From 0f288c318653ebf67d8afdcdce524bcb1c136648 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 16 Oct 2023 20:58:35 +0200 Subject: [PATCH 001/456] Fix delete_fragments handlers according to new model. (#4414) This updates the handlers for delete_fragments and delete_fragments_list to the new handler model that takes in an opened array. SC-34826 --- TYPE: IMPROVEMENT DESC: Fix delete_fragments handlers according to new model. --- test/src/unit-capi-array.cc | 22 +- tiledb/sm/array/array.cc | 14 +- tiledb/sm/array/array.h | 4 +- tiledb/sm/c_api/tiledb.cc | 125 +- tiledb/sm/c_api/tiledb_serialization.h | 18 +- tiledb/sm/rest/rest_client.cc | 26 +- tiledb/sm/rest/rest_client.h | 13 +- tiledb/sm/serialization/fragments.cc | 75 +- tiledb/sm/serialization/fragments.h | 63 +- tiledb/sm/serialization/tiledb-rest.capnp | 12 +- tiledb/sm/serialization/tiledb-rest.capnp.c++ | 15444 +++++++++------- tiledb/sm/serialization/tiledb-rest.capnp.h | 162 +- 12 files changed, 9474 insertions(+), 6504 deletions(-) diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 94cee2f7a3c0..83d23382691b 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -2686,6 +2686,14 @@ TEST_CASE_METHOD( CHECK(tiledb::test::num_commits(array_name) == 2); CHECK(tiledb::test::num_fragments(array_name) == 2); + // Reopen for modify exclusive. + tiledb_array_free(&array); + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_open(ctx_, array, TILEDB_MODIFY_EXCLUSIVE); + REQUIRE(rc == TILEDB_OK); + // ALlocate buffer tiledb_buffer_t* buff; rc = tiledb_buffer_alloc(ctx_, &buff); @@ -2693,14 +2701,15 @@ TEST_CASE_METHOD( SECTION("delete_fragments") { // Serialize fragment timestamps and deserialize delete request - tiledb::sm::serialization::fragments_timestamps_serialize( - array_name, + tiledb::sm::serialization::serialize_delete_fragments_timestamps_request( + array->array_->config(), start_timestamp, end_timestamp, tiledb::sm::SerializationType::CAPNP, &buff->buffer()); - rc = tiledb_deserialize_array_delete_fragments_timestamps_request( + rc = tiledb_handle_array_delete_fragments_timestamps_request( ctx_, + array, (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, buff); REQUIRE(rc == TILEDB_OK); @@ -2729,13 +2738,14 @@ TEST_CASE_METHOD( fragments.emplace_back(URI(uri2)); // Serialize fragments list and deserialize delete request - tiledb::sm::serialization::fragments_list_serialize( - array_name, + tiledb::sm::serialization::serialize_delete_fragments_list_request( + array->array_->config(), fragments, tiledb::sm::SerializationType::CAPNP, &buff->buffer()); - rc = tiledb_deserialize_array_delete_fragments_list_request( + rc = tiledb_handle_array_delete_fragments_list_request( ctx_, + array, (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, buff); REQUIRE(rc == TILEDB_OK); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index b9ea8f516a5b..730063ae24fb 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -558,18 +558,17 @@ void Array::delete_fragments( throw ArrayException( "[delete_fragments] Remote array with no REST client."); } - rest_client->delete_fragments_from_rest( - uri, timestamp_start, timestamp_end); + rest_client->post_delete_fragments_to_rest( + uri, this, timestamp_start, timestamp_end); } else { storage_manager_->delete_fragments( uri.c_str(), timestamp_start, timestamp_end); } } -void Array::delete_fragments_list( - const URI& uri, const std::vector& fragment_uris) { +void Array::delete_fragments_list(const std::vector& fragment_uris) { // Check that data deletion is allowed - ensure_array_is_valid_for_delete(uri); + ensure_array_is_valid_for_delete(array_uri_); // Delete fragments_list if (remote_) { @@ -578,10 +577,11 @@ void Array::delete_fragments_list( throw ArrayException( "[delete_fragments_list] Remote array with no REST client."); } - rest_client->delete_fragments_list_from_rest(uri, fragment_uris); + rest_client->post_delete_fragments_list_to_rest( + array_uri_, this, fragment_uris); } else { auto array_dir = ArrayDirectory( - resources_, uri, 0, std::numeric_limits::max()); + resources_, array_uri_, 0, std::numeric_limits::max()); array_dir.delete_fragments_list(fragment_uris); } } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 7c03262d42cf..24965a75a830 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -233,13 +233,11 @@ class Array { /** * Deletes the fragments with the given URIs from the Array with given URI. * - * @param uri The uri of the Array whose fragments are to be deleted. * @param fragment_uris The uris of the fragments to be deleted. * * @pre The Array must be open for exclusive writes */ - void delete_fragments_list( - const URI& uri, const std::vector& fragment_uris); + void delete_fragments_list(const std::vector& fragment_uris); /** Returns a constant pointer to the encryption key. */ const EncryptionKey* encryption_key() const; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index f6fd7038f491..111474dd9d41 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2470,7 +2470,7 @@ capi_return_t tiledb_array_delete_fragments_list( // Delete fragments list try { - array->array_->delete_fragments_list(uri, uris); + array->array_->delete_fragments_list(uris); } catch (...) { throw_if_not_ok(array->array_->close()); delete array; @@ -4022,112 +4022,58 @@ int32_t tiledb_serialize_array_max_buffer_sizes( return TILEDB_OK; } -capi_return_t tiledb_deserialize_array_delete_fragments_timestamps_request( +capi_return_t tiledb_handle_array_delete_fragments_timestamps_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* buffer) { - api::ensure_buffer_is_valid(buffer); - - // Deserialize buffer - auto [uri_str, timestamp_start, timestamp_end] = - tiledb::sm::serialization::fragments_timestamps_deserialize( - (tiledb::sm::SerializationType)serialize_type, buffer->buffer()); - - auto uri = tiledb::sm::URI(uri_str); - if (uri.is_invalid()) { - throw api::CAPIStatusException( - "Failed to delete fragments; Invalid input uri"); - } - - // Allocate an array object - tiledb_array_t* array = new (std::nothrow) tiledb_array_t; - try { - array->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); - } catch (...) { - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to create array"); + const tiledb_buffer_t* request) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + throw std::invalid_argument("Array paramter must be valid."); } - // Set array open timestamps - array->array_->set_timestamp_start(timestamp_start); - array->array_->set_timestamp_end(timestamp_end); + api::ensure_buffer_is_valid(request); - // Open the array for exclusive modification - throw_if_not_ok(array->array_->open( - static_cast(TILEDB_MODIFY_EXCLUSIVE), - static_cast(TILEDB_NO_ENCRYPTION), - nullptr, - 0)); + // Deserialize buffer + auto [timestamp_start, timestamp_end] = tiledb::sm::serialization:: + deserialize_delete_fragments_timestamps_request( + (tiledb::sm::SerializationType)serialize_type, request->buffer()); // Delete fragments try { - array->array_->delete_fragments(uri, timestamp_start, timestamp_end); + array->array_->delete_fragments( + array->array_->array_uri(), timestamp_start, timestamp_end); } catch (...) { - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; throw api::CAPIStatusException("Failed to delete fragments"); } - // Close and delete the array - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; - return TILEDB_OK; } -capi_return_t tiledb_deserialize_array_delete_fragments_list_request( +capi_return_t tiledb_handle_array_delete_fragments_list_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* buffer) { - api::ensure_buffer_is_valid(buffer); - - // Deserialize buffer - auto [uri_str, uris] = tiledb::sm::serialization::fragments_list_deserialize( - (tiledb::sm::SerializationType)serialize_type, buffer->buffer()); - - auto uri = tiledb::sm::URI(uri_str); - if (uri.is_invalid()) { - throw api::CAPIStatusException( - "Failed to delete_fragments_list; Invalid input uri"); + const tiledb_buffer_t* request) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + throw std::invalid_argument("Array paramter must be valid."); } - // Allocate an array object - tiledb_array_t* array = new (std::nothrow) tiledb_array_t; - try { - array->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); - } catch (...) { - delete array; - array = nullptr; - throw api::CAPIStatusException("Failed to create array"); - } + api::ensure_buffer_is_valid(request); - // Open the array for exclusive modification - throw_if_not_ok(array->array_->open( - static_cast(TILEDB_MODIFY_EXCLUSIVE), - static_cast(TILEDB_NO_ENCRYPTION), - nullptr, - 0)); + // Deserialize buffer + auto uris = + tiledb::sm::serialization::deserialize_delete_fragments_list_request( + array->array_->array_uri(), + (tiledb::sm::SerializationType)serialize_type, + request->buffer()); // Delete fragments list try { - array->array_->delete_fragments_list(uri, uris); + array->array_->delete_fragments_list(uris); } catch (...) { - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; throw api::CAPIStatusException("Failed to delete fragments_list"); } - // Close and delete the array - throw_if_not_ok(array->array_->close()); - delete array; - array = nullptr; - return TILEDB_OK; } @@ -7033,23 +6979,24 @@ int32_t tiledb_serialize_array_max_buffer_sizes( ctx, array, subarray, serialize_type, buffer); } -capi_return_t tiledb_deserialize_array_delete_fragments_timestamps_request( +capi_return_t tiledb_handle_array_delete_fragments_timestamps_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* buffer) noexcept { + const tiledb_buffer_t* request) noexcept { return api_entry< - tiledb::api:: - tiledb_deserialize_array_delete_fragments_timestamps_request>( - ctx, serialize_type, buffer); + tiledb::api::tiledb_handle_array_delete_fragments_timestamps_request>( + ctx, array, serialize_type, request); } -capi_return_t tiledb_deserialize_array_delete_fragments_list_request( +capi_return_t tiledb_handle_array_delete_fragments_list_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* buffer) noexcept { + const tiledb_buffer_t* request) noexcept { return api_entry< - tiledb::api::tiledb_deserialize_array_delete_fragments_list_request>( - ctx, serialize_type, buffer); + tiledb::api::tiledb_handle_array_delete_fragments_list_request>( + ctx, array, serialize_type, request); } int32_t tiledb_serialize_array_metadata( diff --git a/tiledb/sm/c_api/tiledb_serialization.h b/tiledb/sm/c_api/tiledb_serialization.h index ad7d1bfb862a..5edcb27971e9 100644 --- a/tiledb/sm/c_api/tiledb_serialization.h +++ b/tiledb/sm/c_api/tiledb_serialization.h @@ -432,32 +432,36 @@ TILEDB_EXPORT int32_t tiledb_serialize_array_max_buffer_sizes( tiledb_buffer_t** buffer) TILEDB_NOEXCEPT; /** - * Deserializes a delete fragments request from the given buffer. + * Process a delete fragments request. * * @param ctx The TileDB context. + * @param array The TileDB Array. * @param serialization_type Type of serialization to use - * @param buffer Buffer containing serialized fragment timestamps. + * @param request Buffer containing serialized fragment timestamps. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT -capi_return_t tiledb_deserialize_array_delete_fragments_timestamps_request( +capi_return_t tiledb_handle_array_delete_fragments_timestamps_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialization_type, - const tiledb_buffer_t* buffer) TILEDB_NOEXCEPT; + const tiledb_buffer_t* request) TILEDB_NOEXCEPT; /** - * Deserializes a delete fragments list request from the given buffer. + * Process a delete fragments list request. * * @param ctx The TileDB context. + * @param array The TileDB Array. * @param serialization_type Type of serialization to use * @param buffer Buffer containing serialized fragments list. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT -capi_return_t tiledb_deserialize_array_delete_fragments_list_request( +capi_return_t tiledb_handle_array_delete_fragments_list_request( tiledb_ctx_t* ctx, + tiledb_array_t* array, tiledb_serialization_type_t serialization_type, - const tiledb_buffer_t* buffer) TILEDB_NOEXCEPT; + const tiledb_buffer_t* request) TILEDB_NOEXCEPT; /** * Serializes the array metadata into the given buffer. diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 819025eba42d..26184ed66ed1 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -395,11 +395,14 @@ void RestClient::delete_array_from_rest(const URI& uri) { stats_, url, serialization_type_, &returned_data, cache_key)); } -void RestClient::delete_fragments_from_rest( - const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end) { +void RestClient::post_delete_fragments_to_rest( + const URI& uri, + Array* array, + uint64_t timestamp_start, + uint64_t timestamp_end) { Buffer buff; - serialization::fragments_timestamps_serialize( - uri.to_string(), + serialization::serialize_delete_fragments_timestamps_request( + array->config(), timestamp_start, timestamp_end, serialization_type_, @@ -429,11 +432,11 @@ void RestClient::delete_fragments_from_rest( cache_key)); } -void RestClient::delete_fragments_list_from_rest( - const URI& uri, const std::vector& fragment_uris) { +void RestClient::post_delete_fragments_list_to_rest( + const URI& uri, Array* array, const std::vector& fragment_uris) { Buffer buff; - serialization::fragments_list_serialize( - uri.to_string(), fragment_uris, serialization_type_, &buff); + serialization::serialize_delete_fragments_list_request( + array->config(), fragment_uris, serialization_type_, &buff); // Wrap in a list BufferList serialized; throw_if_not_ok(serialized.add_buffer(std::move(buff))); @@ -1555,13 +1558,14 @@ void RestClient::delete_array_from_rest(const URI&) { Status_RestError("Cannot use rest client; serialization not enabled.")); } -void RestClient::delete_fragments_from_rest(const URI&, uint64_t, uint64_t) { +void RestClient::post_delete_fragments_to_rest( + const URI&, Array*, uint64_t, uint64_t) { throw StatusException( Status_RestError("Cannot use rest client; serialization not enabled.")); } -void RestClient::delete_fragments_list_from_rest( - const URI&, const std::vector&) { +void RestClient::post_delete_fragments_list_to_rest( + const URI&, Array*, const std::vector&) { throw StatusException( Status_RestError("Cannot use rest client; serialization not enabled.")); } diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index e1b05e908f4d..63d9044db9e2 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -147,25 +147,30 @@ class RestClient { * at the given URI from the REST server. * * @param uri Array URI to delete fragments from + * @param array Array to delete fragments from * @param timestamp_start The start timestamp at which to delete fragments * @param timestamp_end The end timestamp at which to delete fragments * * #TODO Implement API endpoint on TileDBCloud. */ - void delete_fragments_from_rest( - const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end); + void post_delete_fragments_to_rest( + const URI& uri, + Array* array, + uint64_t timestamp_start, + uint64_t timestamp_end); /** * Deletes the fragments with the given URIs from the array at the given URI * from the REST server. * * @param uri Array URI to delete fragments from + * @param array Array to delete fragments from * @param fragment_uris The uris of the fragments to be deleted * * #TODO Implement API endpoint on TileDBCloud. */ - void delete_fragments_list_from_rest( - const URI& uri, const std::vector& fragment_uris); + void post_delete_fragments_list_to_rest( + const URI& uri, Array* array, const std::vector& fragment_uris); /** * Deregisters an array at the given URI from the REST server. diff --git a/tiledb/sm/serialization/fragments.cc b/tiledb/sm/serialization/fragments.cc index d3b4f2160c61..5a2624c38260 100644 --- a/tiledb/sm/serialization/fragments.cc +++ b/tiledb/sm/serialization/fragments.cc @@ -58,25 +58,24 @@ class FragmentsSerializationException : public StatusException { #ifdef TILEDB_SERIALIZATION void fragments_timestamps_to_capnp( - std::string uri, + const Config& config, uint64_t start_timestamp, uint64_t end_timestamp, - capnp::ArrayDeleteFragmentsTimestampsRequest::Builder* builder) { - builder->setUri(uri); - builder->setStartTimestamp(start_timestamp); - builder->setEndTimestamp(end_timestamp); + capnp::ArrayDeleteFragmentsTimestampsRequest::Builder& builder) { + auto config_builder = builder.initConfig(); + throw_if_not_ok(config_to_capnp(config, &config_builder)); + + builder.setStartTimestamp(start_timestamp); + builder.setEndTimestamp(end_timestamp); } -std::tuple fragments_timestamps_from_capnp( +std::tuple fragments_timestamps_from_capnp( const capnp::ArrayDeleteFragmentsTimestampsRequest::Reader& reader) { - return { - reader.getUri().cStr(), - reader.getStartTimestamp(), - reader.getEndTimestamp()}; + return {reader.getStartTimestamp(), reader.getEndTimestamp()}; } -void fragments_timestamps_serialize( - std::string uri, +void serialize_delete_fragments_timestamps_request( + const Config& config, uint64_t start_timestamp, uint64_t end_timestamp, SerializationType serialize_type, @@ -87,7 +86,7 @@ void fragments_timestamps_serialize( auto builder = message.initRoot(); fragments_timestamps_to_capnp( - uri, start_timestamp, end_timestamp, &builder); + config, start_timestamp, end_timestamp, builder); // Copy to buffer serialized_buffer->reset_size(); @@ -130,7 +129,7 @@ void fragments_timestamps_serialize( } } -std::tuple fragments_timestamps_deserialize( +std::tuple deserialize_delete_fragments_timestamps_request( SerializationType serialize_type, const Buffer& serialized_buffer) { try { switch (serialize_type) { @@ -176,37 +175,39 @@ std::tuple fragments_timestamps_deserialize( } void fragments_list_to_capnp( - std::string uri, + const Config& config, const std::vector& fragments, - capnp::ArrayDeleteFragmentsListRequest::Builder* builder) { - builder->setUri(uri); - auto entries_builder = builder->initEntries(fragments.size()); + capnp::ArrayDeleteFragmentsListRequest::Builder& builder) { + auto config_builder = builder.initConfig(); + throw_if_not_ok(config_to_capnp(config, &config_builder)); + + auto entries_builder = builder.initEntries(fragments.size()); for (size_t i = 0; i < fragments.size(); i++) { const auto& relative_uri = serialize_array_uri_to_relative(fragments[i]); entries_builder.set(i, relative_uri); } } -std::tuple> fragments_list_from_capnp( +std::vector fragments_list_from_capnp( + const URI& array_uri, const capnp::ArrayDeleteFragmentsListRequest::Reader& reader) { - auto uri = reader.getUri().cStr(); if (reader.hasEntries()) { std::vector fragments; fragments.reserve(reader.getEntries().size()); auto get_entries_reader = reader.getEntries(); for (auto entry : get_entries_reader) { fragments.emplace_back( - deserialize_array_uri_to_absolute(entry.cStr(), URI(uri))); + deserialize_array_uri_to_absolute(entry.cStr(), URI(array_uri))); } - return {uri, fragments}; + return fragments; } else { throw FragmentsSerializationException( "[fragments_list_from_capnp] There are no fragments to deserialize"); } } -void fragments_list_serialize( - std::string uri, +void serialize_delete_fragments_list_request( + const Config& config, const std::vector& fragments, SerializationType serialize_type, Buffer* serialized_buffer) { @@ -219,7 +220,7 @@ void fragments_list_serialize( // Serialize ::capnp::MallocMessageBuilder message; auto builder = message.initRoot(); - fragments_list_to_capnp(uri, fragments, &builder); + fragments_list_to_capnp(config, fragments, builder); // Copy to buffer serialized_buffer->reset_size(); @@ -261,8 +262,10 @@ void fragments_list_serialize( } } -std::tuple> fragments_list_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer) { +std::vector deserialize_delete_fragments_list_request( + const URI& array_uri, + SerializationType serialize_type, + const Buffer& serialized_buffer) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -275,7 +278,7 @@ std::tuple> fragments_list_deserialize( builder); auto reader = builder.asReader(); // Deserialize - return fragments_list_from_capnp(reader); + return fragments_list_from_capnp(array_uri, reader); } case SerializationType::CAPNP: { const auto mBytes = @@ -286,7 +289,7 @@ std::tuple> fragments_list_deserialize( auto reader = msg_reader.getRoot(); // Deserialize - return fragments_list_from_capnp(reader); + return fragments_list_from_capnp(array_uri, reader); } default: { throw FragmentsSerializationException( @@ -304,26 +307,26 @@ std::tuple> fragments_list_deserialize( } #else -void fragments_timestamps_serialize( - std::string, uint64_t, uint64_t, SerializationType, Buffer*) { +void serialize_delete_fragments_timestamps_request( + uint64_t, uint64_t, SerializationType, Buffer*) { throw FragmentsSerializationException( "Cannot serialize; serialization not enabled."); } -std::tuple fragments_timestamps_deserialize( +std::tuple deserialize_delete_fragments_timestamps_request( SerializationType, const Buffer&) { throw FragmentsSerializationException( "Cannot deserialize; serialization not enabled."); } -void fragments_list_serialize( - std::string, const std::vector&, SerializationType, Buffer*) { +void serialize_delete_fragments_list_request( + const std::vector&, SerializationType, Buffer*) { throw FragmentsSerializationException( "Cannot serialize; serialization not enabled."); } -std::tuple> fragments_list_deserialize( - SerializationType, const Buffer&) { +std::vector deserialize_delete_fragments_list_request( + const URI&, SerializationType, const Buffer&) { throw FragmentsSerializationException( "Cannot deserialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/fragments.h b/tiledb/sm/serialization/fragments.h index e4cd66534a73..636205f9073d 100644 --- a/tiledb/sm/serialization/fragments.h +++ b/tiledb/sm/serialization/fragments.h @@ -44,74 +44,31 @@ using namespace tiledb::common; namespace tiledb::sm { class Buffer; +class Config; enum class SerializationType : uint8_t; namespace serialization { -#ifdef TILEDB_SERIALIZATION -/** - * Convert ArrayDeleteFragmentsTimestampsRequest to Cap'n Proto message - * - * @param uri the URI of the fragments' parent array - * @param start_timestamp the start timestamp to serialize - * @param end_timestamp the end timestamp to serialize - * @param builder cap'n proto class - */ -void fragments_timestamps_to_capnp( - std::string uri, - uint64_t start_timestamp, - uint64_t end_timestamp, - capnp::ArrayDeleteFragmentsTimestampsRequest::Builder* builder); - -/** - * Convert Cap'n Proto message to ArrayDeleteFragmentsTimestampsRequest - * - * @param reader cap'n proto class - * @return a tuple of uri, start_timestamp, end_timestamp - */ -std::tuple fragments_timestamps_from_capnp( - const capnp::ArrayDeleteFragmentsTimestampsRequest::Reader& reader); - -/** - * Convert ArrayDeleteFragmentsListRequest to Cap'n Proto message - * - * @param uri the URI of the fragments' parent array - * @param fragments fragments to serialize - * @param builder cap'n proto class - */ -void fragments_list_to_capnp( - std::string uri, - const std::vector& fragments, - capnp::ArrayDeleteFragmentsListRequest::Builder* builder); - -/** - * Convert Cap'n Proto message to ArrayDeleteFragmentsListRequest - * - * @param reader cap'n proto class - * @return a tuple of uri, vector of deserialized fragments - */ -std::tuple> fragments_list_from_capnp( - const capnp::ArrayDeleteFragmentsListRequest::Reader& reader); -#endif - -void fragments_timestamps_serialize( - std::string uri, +void serialize_delete_fragments_timestamps_request( + const Config& config, uint64_t start_timestamp, uint64_t end_timestamp, SerializationType serialize_type, Buffer* serialized_buffer); -std::tuple fragments_timestamps_deserialize( +std::tuple deserialize_delete_fragments_timestamps_request( SerializationType serialize_type, const Buffer& serialized_buffer); -void fragments_list_serialize( - std::string uri, +void serialize_delete_fragments_list_request( + const Config& config, const std::vector& fragments, SerializationType serialize_type, Buffer* serialized_buffer); -std::tuple> fragments_list_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer); +std::vector deserialize_delete_fragments_list_request( + const URI& array_uri, + SerializationType serialize_type, + const Buffer& serialized_buffer); } // namespace serialization } // namespace tiledb::sm diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 7531bdb7f884..f76b4107e57b 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1200,14 +1200,22 @@ struct BufferedChunk { } struct ArrayDeleteFragmentsListRequest { - uri @0 :Text; + config @0 :Config; + # Config + entries @1 :List(Text); + # Fragment list to delete } struct ArrayDeleteFragmentsTimestampsRequest { - uri @0 :Text; + config @0 :Config; + # Config + startTimestamp @1 :UInt64; + # Start timestamp for the delete + endTimestamp @2 :UInt64; + # End timestamp for the delete } struct ArrayConsolidationRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index e42459c84556..2a5a77c3f800 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -5,6688 +5,9733 @@ namespace capnp { namespace schemas { -static const ::capnp::_::AlignedData<208> b_ce5904e6f9410cec = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 236, 12, 65, 249, 230, 4, 89, - 206, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, - 65, 114, 114, 97, 121, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 40, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 1, 0, 0, 3, 0, 1, 0, 32, 1, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, - 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 1, 0, 0, 3, 0, 1, 0, 52, 1, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 0, - 3, 0, 1, 0, 72, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 58, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 3, 0, 1, 0, - 92, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 89, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 84, 1, 0, 0, 3, 0, 1, 0, 112, 1, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, - 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, - 1, 0, 0, 3, 0, 1, 0, 132, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 1, 0, 0, 50, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 1, 0, 0, 3, - 0, 1, 0, 152, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 149, 1, 0, 0, 58, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 144, 1, 0, 0, 3, 0, 1, 0, 172, - 1, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 169, 1, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 164, 1, 0, 0, 3, 0, 1, 0, 192, 1, 0, 0, 2, - 0, 1, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 1, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 184, 1, - 0, 0, 3, 0, 1, 0, 212, 1, 0, 0, 2, 0, 1, 0, 105, - 110, 116, 56, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 105, 110, 116, 56, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 110, 116, 49, - 54, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 117, 105, 110, 116, 49, 54, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 7, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 105, 110, 116, 51, 50, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, - 105, 110, 116, 51, 50, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 8, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 105, 110, 116, 54, 52, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 105, 110, 116, - 54, 52, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 102, 108, 111, 97, 116, 51, 50, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 10, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 102, 108, 111, 97, 116, 54, 52, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; +static const ::capnp::_::AlignedData<208> b_ce5904e6f9410cec = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 236, 12, 65, 249, 230, 4, 89, 206, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 10, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 55, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 111, 109, 97, 105, 110, + 65, 114, 114, 97, 121, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 40, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 52, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 1, 0, 0, 3, 0, 1, 0, + 72, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 92, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 1, 0, 0, 3, 0, 1, 0, + 112, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 1, 0, 0, 3, 0, 1, 0, + 132, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 1, 0, 0, 3, 0, 1, 0, + 152, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 1, 0, 0, 3, 0, 1, 0, + 172, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 169, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 1, 0, 0, 3, 0, 1, 0, + 192, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 1, 0, 0, 3, 0, 1, 0, + 212, 1, 0, 0, 2, 0, 1, 0, + 105, 110, 116, 56, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 56, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 49, 54, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 49, 54, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 51, 50, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 51, 50, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 54, 52, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 54, 52, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 51, 50, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 54, 52, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_ce5904e6f9410cec = b_ce5904e6f9410cec.words; #if !CAPNP_LITE static const uint16_t m_ce5904e6f9410cec[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_ce5904e6f9410cec[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ce5904e6f9410cec = { - 0xce5904e6f9410cec, - b_ce5904e6f9410cec.words, - 208, - nullptr, - m_ce5904e6f9410cec, - 0, - 10, - i_ce5904e6f9410cec, - nullptr, - nullptr, - {&s_ce5904e6f9410cec, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<47> b_e3dadf2bf211bc97 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 151, 188, 17, 242, 43, 223, 218, - 227, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 170, 0, 0, 0, 29, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 75, 86, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 34, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, - 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, 0, - 52, 0, 0, 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, 108, - 117, 101, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -}}; + 0xce5904e6f9410cec, b_ce5904e6f9410cec.words, 208, nullptr, m_ce5904e6f9410cec, + 0, 10, i_ce5904e6f9410cec, nullptr, nullptr, { &s_ce5904e6f9410cec, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<47> b_e3dadf2bf211bc97 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 170, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 75, 86, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_e3dadf2bf211bc97 = b_e3dadf2bf211bc97.words; #if !CAPNP_LITE static const uint16_t m_e3dadf2bf211bc97[] = {0, 1}; static const uint16_t i_e3dadf2bf211bc97[] = {0, 1}; const ::capnp::_::RawSchema s_e3dadf2bf211bc97 = { - 0xe3dadf2bf211bc97, - b_e3dadf2bf211bc97.words, - 47, - nullptr, - m_e3dadf2bf211bc97, - 0, - 2, - i_e3dadf2bf211bc97, - nullptr, - nullptr, - {&s_e3dadf2bf211bc97, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<37> b_b6c95b4b8111ad36 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 54, 173, 17, 129, 75, 91, 201, - 182, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 102, 105, 103, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 13, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 36, 0, 0, - 0, 2, 0, 1, 0, 101, 110, 116, 114, 105, 101, 115, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 151, 188, 17, 242, 43, 223, 218, - 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xe3dadf2bf211bc97, b_e3dadf2bf211bc97.words, 47, nullptr, m_e3dadf2bf211bc97, + 0, 2, i_e3dadf2bf211bc97, nullptr, nullptr, { &s_e3dadf2bf211bc97, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<37> b_b6c95b4b8111ad36 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 110, 102, 105, 103, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_b6c95b4b8111ad36 = b_b6c95b4b8111ad36.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_b6c95b4b8111ad36[] = { - &s_e3dadf2bf211bc97, + &s_e3dadf2bf211bc97, }; static const uint16_t m_b6c95b4b8111ad36[] = {0}; static const uint16_t i_b6c95b4b8111ad36[] = {0}; const ::capnp::_::RawSchema s_b6c95b4b8111ad36 = { - 0xb6c95b4b8111ad36, - b_b6c95b4b8111ad36.words, - 37, - d_b6c95b4b8111ad36, - m_b6c95b4b8111ad36, - 1, - 1, - i_b6c95b4b8111ad36, - nullptr, - nullptr, - {&s_b6c95b4b8111ad36, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<217> b_a45730f57e0460b4 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 180, 96, 4, 126, 245, 48, 87, - 164, 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 44, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 106, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 1, 0, 0, - 3, 0, 1, 0, 48, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 45, 1, 0, 0, 82, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 44, 1, 0, 0, 3, 0, 1, 0, - 56, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 53, 1, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 48, 1, 0, 0, 3, 0, 1, 0, 60, 1, 0, 0, - 2, 0, 1, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, - 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, - 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, - 1, 0, 0, 3, 0, 1, 0, 68, 1, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 0, 0, 146, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 1, 0, 0, 3, - 0, 1, 0, 80, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, - 3, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 77, 1, 0, 0, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 76, 1, 0, 0, 3, 0, 1, 0, 160, - 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 157, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 156, 1, 0, 0, 3, 0, 1, 0, 168, 1, 0, 0, 2, - 0, 1, 0, 7, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 1, 0, - 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 1, - 0, 0, 3, 0, 1, 0, 176, 1, 0, 0, 2, 0, 1, 0, 8, - 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 173, 1, 0, 0, 122, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 1, 0, 0, 3, 0, - 1, 0, 184, 1, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 7, - 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 181, 1, 0, 0, 162, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 184, 1, 0, 0, 3, 0, 1, 0, 212, 1, - 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 2, 0, 0, 0, 0, - 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 212, 1, 0, 0, 3, 0, 1, 0, 224, 1, 0, 0, 2, 0, - 1, 0, 101, 110, 100, 84, 105, 109, 101, 115, 116, 97, 109, 112, 0, - 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, - 117, 101, 114, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 114, 105, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 115, 116, 97, 114, 116, 84, 105, 109, 101, 115, 116, 97, 109, 112, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 114, 114, - 97, 121, 83, 99, 104, 101, 109, 97, 76, 97, 116, 101, 115, 116, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 254, - 150, 226, 152, 47, 227, 29, 215, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 97, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 115, 65, - 108, 108, 0, 16, 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, - 148, 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, - 0, 0, 2, 0, 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, - 8, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 8, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, 227, 29, 215, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 110, 111, 110, 69, 109, 112, 116, - 121, 68, 111, 109, 97, 105, 110, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 227, 236, 72, 148, 84, 100, 130, 161, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 97, 114, 114, 97, 121, 77, 101, 116, 97, 100, - 97, 116, 97, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 187, - 49, 206, 223, 175, 220, 55, 151, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 97, 114, 114, 97, 121, 68, 105, 114, 101, 99, 116, 111, 114, - 121, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 24, 217, 9, 229, - 12, 242, 23, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, - 114, 97, 103, 109, 101, 110, 116, 77, 101, 116, 97, 100, 97, 116, 97, - 65, 108, 108, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 97, 202, 231, 39, 252, 82, 227, 205, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 111, 112, 101, 110, 101, 100, 65, 116, 69, - 110, 100, 84, 105, 109, 101, 115, 116, 97, 109, 112, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xb6c95b4b8111ad36, b_b6c95b4b8111ad36.words, 37, d_b6c95b4b8111ad36, m_b6c95b4b8111ad36, + 1, 1, i_b6c95b4b8111ad36, nullptr, nullptr, { &s_b6c95b4b8111ad36, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<217> b_a45730f57e0460b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 96, 4, 126, 245, 48, 87, 164, + 18, 0, 0, 0, 1, 0, 3, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 8, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 111, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 44, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 1, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 1, 0, 0, 3, 0, 1, 0, + 56, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 1, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 1, 0, 0, 3, 0, 1, 0, + 60, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 1, 0, 0, 3, 0, 1, 0, + 68, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 1, 0, 0, 3, 0, 1, 0, + 80, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 1, 0, 0, 3, 0, 1, 0, + 160, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 1, 0, 0, 3, 0, 1, 0, + 168, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 1, 0, 0, 3, 0, 1, 0, + 176, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 1, 0, 0, 3, 0, 1, 0, + 184, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 1, 0, 0, 3, 0, 1, 0, + 212, 1, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 224, 1, 0, 0, 2, 0, 1, 0, + 101, 110, 100, 84, 105, 109, 101, 115, + 116, 97, 109, 112, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 117, 101, 114, 121, 84, 121, 112, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 114, 116, 84, 105, 109, + 101, 115, 116, 97, 109, 112, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 76, 97, 116, 101, 115, + 116, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 115, 65, 108, 108, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 111, 110, 69, 109, 112, 116, 121, + 68, 111, 109, 97, 105, 110, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 227, 236, 72, 148, 84, 100, 130, 161, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 77, 101, 116, + 97, 100, 97, 116, 97, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 187, 49, 206, 223, 175, 220, 55, 151, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 68, 105, 114, + 101, 99, 116, 111, 114, 121, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 24, 217, 9, 229, 12, 242, 23, 147, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 77, 101, 116, 97, 100, 97, 116, 97, + 65, 108, 108, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 112, 101, 110, 101, 100, 65, 116, + 69, 110, 100, 84, 105, 109, 101, 115, + 116, 97, 109, 112, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a45730f57e0460b4 = b_a45730f57e0460b4.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a45730f57e0460b4[] = { - &s_9317f20ce509d918, - &s_9737dcafdfce31bb, - &s_a18264549448ece3, - &s_cde352fc27e7ca61, - &s_d71de32f98e296fe, - &s_f179c194ae71718c, + &s_9317f20ce509d918, + &s_9737dcafdfce31bb, + &s_a18264549448ece3, + &s_cde352fc27e7ca61, + &s_d71de32f98e296fe, + &s_f179c194ae71718c, }; static const uint16_t m_a45730f57e0460b4[] = {8, 7, 4, 5, 0, 9, 6, 10, 1, 3, 2}; static const uint16_t i_a45730f57e0460b4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; -KJ_CONSTEXPR( - const)::capnp::_::RawBrandedSchema::Dependency bd_a45730f57e0460b4[] = { - {16777221, - ::tiledb::sm::serialization::capnp:: - Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>:: - _capnpPrivate::brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a45730f57e0460b4[] = { + { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a45730f57e0460b4 = { - 0xa45730f57e0460b4, - b_a45730f57e0460b4.words, - 217, - d_a45730f57e0460b4, - m_a45730f57e0460b4, - 6, - 11, - i_a45730f57e0460b4, - nullptr, - nullptr, - {&s_a45730f57e0460b4, - nullptr, - bd_a45730f57e0460b4, - 0, - sizeof(bd_a45730f57e0460b4) / sizeof(bd_a45730f57e0460b4[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_facceeafd4472c68 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 104, 44, 71, 212, 175, 238, 204, - 250, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 79, - 112, 101, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 0, 0, 0, 3, 0, 1, 0, 56, 0, 0, 0, 2, 0, 1, - 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 113, 117, 101, 114, 121, 84, 121, 112, 101, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xa45730f57e0460b4, b_a45730f57e0460b4.words, 217, d_a45730f57e0460b4, m_a45730f57e0460b4, + 6, 11, i_a45730f57e0460b4, nullptr, nullptr, { &s_a45730f57e0460b4, nullptr, bd_a45730f57e0460b4, 0, sizeof(bd_a45730f57e0460b4) / sizeof(bd_a45730f57e0460b4[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_facceeafd4472c68 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 104, 44, 71, 212, 175, 238, 204, 250, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 79, + 112, 101, 110, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 117, 101, 114, 121, 84, 121, 112, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_facceeafd4472c68 = b_facceeafd4472c68.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_facceeafd4472c68[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_facceeafd4472c68[] = {0, 1}; static const uint16_t i_facceeafd4472c68[] = {0, 1}; const ::capnp::_::RawSchema s_facceeafd4472c68 = { - 0xfacceeafd4472c68, - b_facceeafd4472c68.words, - 49, - d_facceeafd4472c68, - m_facceeafd4472c68, - 1, - 2, - i_facceeafd4472c68, - nullptr, - nullptr, - {&s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, - 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, - 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 204, 1, 0, 0, 3, 0, 1, 0, 216, 1, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, - 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 1, 0, 0, 3, 0, 1, 0, 240, 1, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 1, 0, 0, 74, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 1, 0, 0, - 3, 0, 1, 0, 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 1, 0, 0, 82, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 244, 1, 0, 0, 3, 0, 1, 0, - 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2, 0, 0, 3, 0, 1, 0, 12, 2, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, - 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 2, 0, 0, 3, 0, 1, 0, 16, 2, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2, 0, 0, 170, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 0, 0, 3, - 0, 1, 0, 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, - 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 24, 2, 0, 0, 3, 0, 1, 0, 36, - 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 28, 2, 0, 0, 3, 0, 1, 0, 40, 2, 0, 0, 2, - 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 2, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 2, - 0, 0, 3, 0, 1, 0, 60, 2, 0, 0, 2, 0, 1, 0, 10, - 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 57, 2, 0, 0, 138, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 2, 0, 0, 3, 0, - 1, 0, 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 72, 2, 0, 0, 3, 0, 1, 0, 84, 2, - 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, - 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 76, 2, 0, 0, 3, 0, 1, 0, 88, 2, 0, 0, 2, 0, - 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 2, 0, 0, - 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 2, 0, - 0, 3, 0, 1, 0, 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 109, 2, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 108, 2, 0, 0, 3, 0, 1, - 0, 136, 2, 0, 0, 2, 0, 1, 0, 15, 0, 0, 0, 13, 0, - 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 133, 2, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 2, 0, 0, 3, 0, 1, 0, 160, 2, 0, - 0, 2, 0, 1, 0, 16, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, - 2, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 2, 0, 0, 3, 0, 1, 0, 188, 2, 0, 0, 2, 0, 1, - 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 116, - 116, 114, 105, 98, 117, 116, 101, 115, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 106, 215, 227, 109, 245, 120, - 173, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 97, 112, - 97, 99, 105, 116, 121, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 101, 108, 108, 79, 114, - 100, 101, 114, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 99, 111, 111, 114, 100, 115, 70, 105, 108, - 116, 101, 114, 80, 105, 112, 101, 108, 105, 110, 101, 0, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 245, 196, 234, 51, 247, 131, 69, - 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 111, 109, 97, - 105, 110, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 76, 117, 100, - 118, 68, 15, 3, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 111, 102, 102, 115, 101, 116, 70, 105, 108, 116, 101, 114, 80, 105, 112, - 101, 108, 105, 110, 101, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 245, 196, 234, 51, 247, 131, 69, 188, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 116, 105, 108, 101, 79, 114, 100, 101, 114, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 117, 114, 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 118, 101, 114, 115, 105, 111, 110, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 108, 108, - 111, 119, 115, 68, 117, 112, 108, 105, 99, 97, 116, 101, 115, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 118, 97, 108, 105, 100, 105, 116, 121, 70, 105, 108, 116, 101, - 114, 80, 105, 112, 101, 108, 105, 110, 101, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 245, 196, 234, 51, 247, 131, 69, 188, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 109, 101, - 115, 116, 97, 109, 112, 82, 97, 110, 103, 101, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 105, 109, 101, 110, - 115, 105, 111, 110, 76, 97, 98, 101, 108, 115, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 222, 209, 12, 209, 98, 141, 255, 206, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 110, 117, 109, 101, 114, - 97, 116, 105, 111, 110, 115, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 180, 185, 33, 204, 25, 47, 11, 208, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 101, 110, 117, 109, 101, 114, 97, - 116, 105, 111, 110, 80, 97, 116, 104, 77, 97, 112, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 151, 188, 17, - 242, 43, 223, 218, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xfacceeafd4472c68, b_facceeafd4472c68.words, 49, d_facceeafd4472c68, m_facceeafd4472c68, + 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 15, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 83, + 99, 104, 101, 109, 97, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 68, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 64, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, + 97, 114, 114, 97, 121, 84, 121, 112, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 115, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 106, 215, 227, 109, 245, 120, 173, 146, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 97, 112, 97, 99, 105, 116, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 79, 114, 100, 101, + 114, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 111, 114, 100, 115, 70, 105, + 108, 116, 101, 114, 80, 105, 112, 101, + 108, 105, 110, 101, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 111, 109, 97, 105, 110, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 76, 117, 100, 118, 68, 15, 3, 222, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 70, 105, + 108, 116, 101, 114, 80, 105, 112, 101, + 108, 105, 110, 101, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 79, 114, 100, 101, + 114, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 108, 108, 111, 119, 115, 68, 117, + 112, 108, 105, 99, 97, 116, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 105, 116, 121, + 70, 105, 108, 116, 101, 114, 80, 105, + 112, 101, 108, 105, 110, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 82, 97, 110, 103, 101, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 105, 109, 101, 110, 115, 105, 111, + 110, 76, 97, 98, 101, 108, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 222, 209, 12, 209, 98, 141, 255, 206, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d71de32f98e296fe = b_d71de32f98e296fe.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { - &s_92ad78f56de3d76a, - &s_bc4583f733eac4f5, - &s_ceff8d62d10cd1de, - &s_d00b2f19cc21b9b4, - &s_de030f447664754c, - &s_e3dadf2bf211bc97, -}; -static const uint16_t m_d71de32f98e296fe[] = { - 10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + &s_92ad78f56de3d76a, + &s_bc4583f733eac4f5, + &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, + &s_de030f447664754c, + &s_e3dadf2bf211bc97, +}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, - b_d71de32f98e296fe.words, - 315, - d_d71de32f98e296fe, - m_d71de32f98e296fe, - 6, - 17, - i_d71de32f98e296fe, - nullptr, - nullptr, - {&s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 222, 209, 12, 209, 98, 141, 255, - 206, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, - 105, 111, 110, 76, 97, 98, 101, 108, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 40, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, - 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, - 0, 3, 0, 1, 0, 20, 1, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 42, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 3, 0, 1, - 0, 24, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 1, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 1, 0, 0, 3, 0, 1, 0, 28, 1, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, - 1, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 24, 1, 0, 0, 3, 0, 1, 0, 36, 1, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, - 3, 0, 1, 0, 40, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, - 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 37, 1, 0, 0, 42, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 3, 0, 1, 0, - 44, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 41, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 40, 1, 0, 0, 3, 0, 1, 0, 52, 1, 0, 0, - 2, 0, 1, 0, 7, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 1, - 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, - 1, 0, 0, 3, 0, 1, 0, 60, 1, 0, 0, 2, 0, 1, 0, - 8, 0, 0, 0, 65, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 57, 1, 0, 0, 74, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 1, 0, 0, 3, - 0, 1, 0, 68, 1, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, - 5, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 65, 1, 0, 0, 58, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 60, 1, 0, 0, 3, 0, 1, 0, 72, - 1, 0, 0, 2, 0, 1, 0, 100, 105, 109, 101, 110, 115, 105, 111, - 110, 73, 100, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 117, 114, 105, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 116, 116, - 114, 105, 98, 117, 116, 101, 78, 97, 109, 101, 0, 0, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 121, - 112, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, - 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, - 120, 116, 101, 114, 110, 97, 108, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 101, 108, 97, - 116, 105, 118, 101, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 115, 99, 104, 101, 109, 97, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, 227, - 29, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 222, 209, 12, 209, 98, 141, 255, 206, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 6, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 55, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 105, 109, 101, 110, 115, + 105, 111, 110, 76, 97, 98, 101, 108, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 40, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 1, 0, 0, 3, 0, 1, 0, + 20, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 1, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 1, 0, 0, 3, 0, 1, 0, + 28, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 1, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 1, 0, 0, 3, 0, 1, 0, + 52, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 64, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 1, 0, 0, 3, 0, 1, 0, + 60, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 65, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 1, 0, 0, 3, 0, 1, 0, + 68, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 1, 0, 0, 3, 0, 1, 0, + 72, 1, 0, 0, 2, 0, 1, 0, + 100, 105, 109, 101, 110, 115, 105, 111, + 110, 73, 100, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 78, 97, 109, 101, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 120, 116, 101, 114, 110, 97, 108, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 108, 97, 116, 105, 118, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 99, 104, 101, 109, 97, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_ceff8d62d10cd1de = b_ceff8d62d10cd1de.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_ceff8d62d10cd1de[] = { - &s_d71de32f98e296fe, + &s_d71de32f98e296fe, }; static const uint16_t m_ceff8d62d10cd1de[] = {3, 6, 0, 7, 1, 4, 8, 9, 5, 2}; static const uint16_t i_ceff8d62d10cd1de[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { - 0xceff8d62d10cd1de, - b_ceff8d62d10cd1de.words, - 174, - d_ceff8d62d10cd1de, - m_ceff8d62d10cd1de, - 1, - 10, - i_ceff8d62d10cd1de, - nullptr, - nullptr, - {&s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, - 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, - 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 20, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, - 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, - 0, 3, 0, 1, 0, 156, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 3, 0, 1, - 0, 180, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 177, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 176, 0, 0, 0, 3, 0, 1, 0, 204, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, - 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 0, 0, 0, 3, 0, 1, 0, 232, 0, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 154, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, - 3, 0, 1, 0, 4, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, - 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, - 111, 65, 100, 100, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 106, 215, 227, 109, 245, 120, 173, 146, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 116, 105, 109, 101, 115, 116, 97, 109, 112, 82, 97, 110, - 103, 101, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 101, 110, 117, 109, 101, 114, 97, 116, 105, 111, 110, 115, 84, - 111, 65, 100, 100, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 180, 185, 33, 204, 25, 47, 11, 208, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 110, 117, 109, 101, 114, - 97, 116, 105, 111, 110, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -}}; + 0xceff8d62d10cd1de, b_ceff8d62d10cd1de.words, 174, d_ceff8d62d10cd1de, m_ceff8d62d10cd1de, + 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 212, 48, 130, 84, 103, 29, 184, 161, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 83, + 99, 104, 101, 109, 97, 69, 118, 111, + 108, 117, 116, 105, 111, 110, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 180, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 115, 84, 111, 68, 114, 111, 112, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 115, 84, 111, 65, 100, 100, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 106, 215, 227, 109, 245, 120, 173, 146, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 82, 97, 110, 103, 101, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 68, 114, + 111, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a1b81d67548230d4 = b_a1b81d67548230d4.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { - &s_92ad78f56de3d76a, - &s_d00b2f19cc21b9b4, + &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, - b_a1b81d67548230d4.words, - 122, - d_a1b81d67548230d4, - m_a1b81d67548230d4, - 2, - 5, - i_a1b81d67548230d4, - nullptr, - nullptr, - {&s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, - 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, - 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 236, 0, 0, 0, 3, 0, 1, 0, 248, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, - 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, 252, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, - 3, 0, 1, 0, 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 122, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 4, 1, 0, 0, 3, 0, 1, 0, 16, 1, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1, - 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 1, 0, 0, 3, 0, 1, 0, 24, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 146, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 1, 0, 0, 3, - 0, 1, 0, 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, - 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 3, 0, 1, 0, 40, - 1, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 5, 0, 0, 0, - 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 37, 1, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 36, 1, 0, 0, 3, 0, 1, 0, 48, 1, 0, 0, 2, - 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, - 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 110, 97, 109, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 102, 105, 108, 116, 101, 114, 80, - 105, 112, 101, 108, 105, 110, 101, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 245, 196, 234, 51, 247, 131, 69, 188, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 102, 105, 108, 108, 86, 97, 108, 117, 101, 0, - 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 110, 117, 108, 108, 97, 98, 108, 101, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, - 105, 108, 108, 86, 97, 108, 117, 101, 86, 97, 108, 105, 100, 105, 116, - 121, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 101, 110, 117, 109, 101, 114, 97, - 116, 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 106, 215, 227, 109, 245, 120, 173, 146, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 6, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 116, 116, 114, 105, 98, + 117, 116, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 36, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 33, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 116, 101, 114, 80, 105, + 112, 101, 108, 105, 110, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 108, 86, 97, 108, 117, + 101, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 117, 108, 108, 97, 98, 108, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 108, 86, 97, 108, 117, + 101, 86, 97, 108, 105, 100, 105, 116, + 121, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_92ad78f56de3d76a = b_92ad78f56de3d76a.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { - &s_bc4583f733eac4f5, + &s_bc4583f733eac4f5, }; static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, - b_92ad78f56de3d76a.words, - 160, - d_92ad78f56de3d76a, - m_92ad78f56de3d76a, - 1, - 9, - i_92ad78f56de3d76a, - nullptr, - nullptr, - {&s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 180, 185, 33, 204, 25, 47, 11, - 208, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 110, 117, 109, 101, 114, - 97, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 181, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 176, 0, 0, 0, 3, 0, 1, 0, 188, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, - 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 184, 0, 0, 0, 3, 0, 1, 0, 196, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 193, 0, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, - 3, 0, 1, 0, 200, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 90, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 3, 0, 1, 0, - 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 32, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 205, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 200, 0, 0, 0, 3, 0, 1, 0, 212, 0, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, - 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, - 0, 0, 0, 3, 0, 1, 0, 216, 0, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 66, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 3, - 0, 1, 0, 220, 0, 0, 0, 2, 0, 1, 0, 110, 97, 109, 101, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 112, 97, 116, 104, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, - 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, - 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, - 100, 101, 114, 101, 100, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 100, 97, 116, 97, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 111, 102, 102, 115, 101, 116, 115, 0, 13, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; #if !CAPNP_LITE static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { - 0xd00b2f19cc21b9b4, - b_d00b2f19cc21b9b4.words, - 125, - nullptr, - m_d00b2f19cc21b9b4, - 0, - 7, - i_d00b2f19cc21b9b4, - nullptr, - nullptr, - {&s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 162, 146, 250, 18, 129, 87, 10, - 210, 18, 0, 0, 0, 1, 0, 6, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 66, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, - 117, 116, 101, 66, 117, 102, 102, 101, 114, 72, 101, 97, 100, 101, 114, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 28, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, - 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 0, 0, - 0, 3, 0, 1, 0, 188, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 210, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 192, 0, 0, 0, 3, 0, 1, - 0, 204, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 201, 0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 204, 0, 0, 0, 3, 0, 1, 0, 216, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, - 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 220, 0, 0, 0, 3, 0, 1, 0, 232, 0, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 18, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 240, 0, 0, 0, - 3, 0, 1, 0, 252, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, - 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 2, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, - 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 1, 0, 0, 42, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 20, 1, 0, 0, 3, 0, 1, 0, 32, 1, 0, 0, - 2, 0, 1, 0, 110, 97, 109, 101, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 102, 105, 120, 101, 100, 76, 101, - 110, 66, 117, 102, 102, 101, 114, 83, 105, 122, 101, 73, 110, 66, 121, - 116, 101, 115, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 118, 97, 114, 76, 101, 110, 66, 117, 102, - 102, 101, 114, 83, 105, 122, 101, 73, 110, 66, 121, 116, 101, 115, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, 108, 105, - 100, 105, 116, 121, 76, 101, 110, 66, 117, 102, 102, 101, 114, 83, 105, - 122, 101, 73, 110, 66, 121, 116, 101, 115, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 105, 103, 105, 110, - 97, 108, 70, 105, 120, 101, 100, 76, 101, 110, 66, 117, 102, 102, 101, - 114, 83, 105, 122, 101, 73, 110, 66, 121, 116, 101, 115, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 111, 114, 105, 103, 105, 110, 97, 108, 86, 97, 114, 76, 101, 110, 66, - 117, 102, 102, 101, 114, 83, 105, 122, 101, 73, 110, 66, 121, 116, 101, - 115, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, - 105, 103, 105, 110, 97, 108, 86, 97, 108, 105, 100, 105, 116, 121, 76, - 101, 110, 66, 117, 102, 102, 101, 114, 83, 105, 122, 101, 73, 110, 66, - 121, 116, 101, 115, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 162, 146, 250, 18, 129, 87, 10, 210, + 18, 0, 0, 0, 1, 0, 6, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 66, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 116, 116, 114, 105, 98, + 117, 116, 101, 66, 117, 102, 102, 101, + 114, 72, 101, 97, 100, 101, 114, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 210, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 0, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 234, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 18, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 0, 0, 0, 2, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 3, 0, 1, 0, + 12, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 42, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 120, 101, 100, 76, 101, 110, + 66, 117, 102, 102, 101, 114, 83, 105, + 122, 101, 73, 110, 66, 121, 116, 101, + 115, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 114, 76, 101, 110, 66, 117, + 102, 102, 101, 114, 83, 105, 122, 101, + 73, 110, 66, 121, 116, 101, 115, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 105, 116, 121, + 76, 101, 110, 66, 117, 102, 102, 101, + 114, 83, 105, 122, 101, 73, 110, 66, + 121, 116, 101, 115, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 105, 103, 105, 110, 97, 108, + 70, 105, 120, 101, 100, 76, 101, 110, + 66, 117, 102, 102, 101, 114, 83, 105, + 122, 101, 73, 110, 66, 121, 116, 101, + 115, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 105, 103, 105, 110, 97, 108, + 86, 97, 114, 76, 101, 110, 66, 117, + 102, 102, 101, 114, 83, 105, 122, 101, + 73, 110, 66, 121, 116, 101, 115, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 105, 103, 105, 110, 97, 108, + 86, 97, 108, 105, 100, 105, 116, 121, + 76, 101, 110, 66, 117, 102, 102, 101, + 114, 83, 105, 122, 101, 73, 110, 66, + 121, 116, 101, 115, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d20a578112fa92a2 = b_d20a578112fa92a2.words; #if !CAPNP_LITE static const uint16_t m_d20a578112fa92a2[] = {1, 0, 4, 6, 5, 3, 2}; static const uint16_t i_d20a578112fa92a2[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d20a578112fa92a2 = { - 0xd20a578112fa92a2, - b_d20a578112fa92a2.words, - 143, - nullptr, - m_d20a578112fa92a2, - 0, - 7, - i_d20a578112fa92a2, - nullptr, - nullptr, - {&s_d20a578112fa92a2, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<104> b_95e26a84d32d8223 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 35, 130, 45, 211, 132, 106, 226, - 149, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, - 105, 111, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 24, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 153, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 148, 0, 0, 0, 3, 0, 1, 0, 160, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, - 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 0, 0, 0, 3, 0, 1, 0, 168, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, - 3, 0, 1, 0, 172, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 160, 159, - 176, 109, 83, 82, 166, 162, 169, 0, 0, 0, 90, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2, 0, 0, - 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 149, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 144, 0, 0, 0, 3, 0, 1, 0, 156, 0, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, - 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, - 0, 0, 0, 3, 0, 1, 0, 164, 0, 0, 0, 2, 0, 1, 0, - 110, 97, 109, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 110, 117, 108, 108, 84, 105, 108, 101, 69, 120, 116, - 101, 110, 116, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 116, 121, 112, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 116, 105, 108, 101, 69, 120, 116, 101, 110, 116, - 0, 0, 0, 0, 0, 0, 100, 111, 109, 97, 105, 110, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 236, 12, 65, 249, 230, 4, 89, 206, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 105, 108, 116, 101, - 114, 80, 105, 112, 101, 108, 105, 110, 101, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 245, 196, 234, 51, 247, 131, 69, 188, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -}}; + 0xd20a578112fa92a2, b_d20a578112fa92a2.words, 143, nullptr, m_d20a578112fa92a2, + 0, 7, i_d20a578112fa92a2, nullptr, nullptr, { &s_d20a578112fa92a2, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<104> b_95e26a84d32d8223 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 35, 130, 45, 211, 132, 106, 226, 149, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 87, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 105, 109, 101, 110, 115, + 105, 111, 110, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 24, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 160, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 0, 0, 0, 3, 0, 1, 0, + 168, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 0, 0, 0, 3, 0, 1, 0, + 172, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 160, 159, 176, 109, 83, 82, 166, 162, + 169, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 164, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 117, 108, 108, 84, 105, 108, 101, + 69, 120, 116, 101, 110, 116, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 69, 120, 116, 101, + 110, 116, 0, 0, 0, 0, 0, 0, + 100, 111, 109, 97, 105, 110, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 236, 12, 65, 249, 230, 4, 89, 206, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 116, 101, 114, 80, 105, + 112, 101, 108, 105, 110, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_95e26a84d32d8223 = b_95e26a84d32d8223.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_95e26a84d32d8223[] = { - &s_a2a652536db09fa0, - &s_bc4583f733eac4f5, - &s_ce5904e6f9410cec, + &s_a2a652536db09fa0, + &s_bc4583f733eac4f5, + &s_ce5904e6f9410cec, }; static const uint16_t m_95e26a84d32d8223[] = {4, 5, 0, 1, 3, 2}; static const uint16_t i_95e26a84d32d8223[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_95e26a84d32d8223 = { - 0x95e26a84d32d8223, - b_95e26a84d32d8223.words, - 104, - d_95e26a84d32d8223, - m_95e26a84d32d8223, - 3, - 6, - i_95e26a84d32d8223, - nullptr, - nullptr, - {&s_95e26a84d32d8223, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<168> b_a2a652536db09fa0 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 160, 159, 176, 109, 83, 82, 166, - 162, 28, 0, 0, 0, 1, 0, 2, 0, 35, 130, 45, 211, 132, 106, - 226, 149, 4, 0, 7, 0, 1, 0, 10, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, - 105, 111, 110, 46, 116, 105, 108, 101, 69, 120, 116, 101, 110, 116, 0, - 0, 40, 0, 0, 0, 3, 0, 4, 0, 0, 0, 255, 255, 1, 0, - 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 9, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 4, 1, 0, 0, 3, 0, 1, 0, 16, 1, 0, - 0, 2, 0, 1, 0, 1, 0, 254, 255, 1, 0, 0, 0, 0, 0, - 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, - 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 1, 0, 0, 3, 0, 1, 0, 20, 1, 0, 0, 2, 0, 1, - 0, 2, 0, 253, 255, 2, 0, 0, 0, 0, 0, 1, 0, 5, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, - 3, 0, 1, 0, 24, 1, 0, 0, 2, 0, 1, 0, 3, 0, 252, - 255, 2, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 21, 1, 0, 0, 58, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 16, 1, 0, 0, 3, 0, 1, 0, - 28, 1, 0, 0, 2, 0, 1, 0, 4, 0, 251, 255, 1, 0, 0, - 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 25, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 20, 1, 0, 0, 3, 0, 1, 0, 32, 1, 0, 0, - 2, 0, 1, 0, 5, 0, 250, 255, 1, 0, 0, 0, 0, 0, 1, - 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 1, - 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, - 1, 0, 0, 3, 0, 1, 0, 36, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 249, 255, 1, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 1, 0, 0, 50, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 1, 0, 0, 3, - 0, 1, 0, 40, 1, 0, 0, 2, 0, 1, 0, 7, 0, 248, 255, - 1, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 37, 1, 0, 0, 58, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 32, 1, 0, 0, 3, 0, 1, 0, 44, - 1, 0, 0, 2, 0, 1, 0, 8, 0, 247, 255, 1, 0, 0, 0, - 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 41, 1, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 36, 1, 0, 0, 3, 0, 1, 0, 48, 1, 0, 0, 2, - 0, 1, 0, 9, 0, 246, 255, 1, 0, 0, 0, 0, 0, 1, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1, - 0, 0, 3, 0, 1, 0, 52, 1, 0, 0, 2, 0, 1, 0, 105, - 110, 116, 56, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 117, 105, 110, 116, 56, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 105, 110, 116, 49, 54, 0, 0, 0, - 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 105, 110, 116, - 49, 54, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 110, 116, 51, 50, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 117, 105, 110, 116, 51, 50, 0, 0, 8, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 105, 110, 116, 54, 52, 0, 0, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 105, 110, - 116, 54, 52, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 102, 108, 111, 97, 116, 51, 50, 0, 10, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 102, 108, 111, 97, 116, 54, 52, 0, 11, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x95e26a84d32d8223, b_95e26a84d32d8223.words, 104, d_95e26a84d32d8223, m_95e26a84d32d8223, + 3, 6, i_95e26a84d32d8223, nullptr, nullptr, { &s_95e26a84d32d8223, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<168> b_a2a652536db09fa0 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 160, 159, 176, 109, 83, 82, 166, 162, + 28, 0, 0, 0, 1, 0, 2, 0, + 35, 130, 45, 211, 132, 106, 226, 149, + 4, 0, 7, 0, 1, 0, 10, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 55, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 105, 109, 101, 110, 115, + 105, 111, 110, 46, 116, 105, 108, 101, + 69, 120, 116, 101, 110, 116, 0, 0, + 40, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 255, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 254, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 1, 0, 0, 3, 0, 1, 0, + 20, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 253, 255, 2, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 252, 255, 2, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 1, 0, 0, 3, 0, 1, 0, + 28, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 251, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 250, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 249, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 248, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 247, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 246, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 1, 0, 0, 3, 0, 1, 0, + 52, 1, 0, 0, 2, 0, 1, 0, + 105, 110, 116, 56, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 56, 0, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 49, 54, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 49, 54, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 51, 50, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 51, 50, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 54, 52, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 54, 52, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 51, 50, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 54, 52, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a2a652536db09fa0 = b_a2a652536db09fa0.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a2a652536db09fa0[] = { - &s_95e26a84d32d8223, + &s_95e26a84d32d8223, }; static const uint16_t m_a2a652536db09fa0[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_a2a652536db09fa0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_a2a652536db09fa0 = { - 0xa2a652536db09fa0, - b_a2a652536db09fa0.words, - 168, - d_a2a652536db09fa0, - m_a2a652536db09fa0, - 1, - 10, - i_a2a652536db09fa0, - nullptr, - nullptr, - {&s_a2a652536db09fa0, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<85> b_de030f447664754c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 76, 117, 100, 118, 68, 15, 3, - 222, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 96, 0, 0, 0, 3, 0, 1, 0, 108, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, - 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 0, 0, 0, 3, 0, 1, 0, 132, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 82, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, - 3, 0, 1, 0, 140, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 42, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 3, 0, 1, 0, - 144, 0, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 79, 114, 100, - 101, 114, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 100, 105, 109, 101, 110, 115, 105, 111, 110, 115, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 35, 130, 45, 211, 132, 106, 226, 149, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 116, 105, 108, 101, 79, 114, 100, 101, 114, 0, 0, - 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 116, 121, 112, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xa2a652536db09fa0, b_a2a652536db09fa0.words, 168, d_a2a652536db09fa0, m_a2a652536db09fa0, + 1, 10, i_a2a652536db09fa0, nullptr, nullptr, { &s_a2a652536db09fa0, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<85> b_de030f447664754c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 76, 117, 100, 118, 68, 15, 3, 222, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 111, 109, 97, 105, 110, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 140, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 99, 101, 108, 108, 79, 114, 100, 101, + 114, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 105, 109, 101, 110, 115, 105, 111, + 110, 115, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 35, 130, 45, 211, 132, 106, 226, 149, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 79, 114, 100, 101, + 114, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_de030f447664754c = b_de030f447664754c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_de030f447664754c[] = { - &s_95e26a84d32d8223, + &s_95e26a84d32d8223, }; static const uint16_t m_de030f447664754c[] = {0, 1, 2, 3}; static const uint16_t i_de030f447664754c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_de030f447664754c = { - 0xde030f447664754c, - b_de030f447664754c.words, - 85, - d_de030f447664754c, - m_de030f447664754c, - 1, - 4, - i_de030f447664754c, - nullptr, - nullptr, - {&s_de030f447664754c, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<47> b_fa787661cd3563a4 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 164, 99, 53, 205, 97, 118, 120, - 250, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 114, 114, 111, 114, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, - 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 66, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, 0, - 52, 0, 0, 0, 2, 0, 1, 0, 99, 111, 100, 101, 0, 0, 0, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 101, 115, - 115, 97, 103, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -}}; + 0xde030f447664754c, b_de030f447664754c.words, 85, d_de030f447664754c, m_de030f447664754c, + 1, 4, i_de030f447664754c, nullptr, nullptr, { &s_de030f447664754c, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<47> b_fa787661cd3563a4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 164, 99, 53, 205, 97, 118, 120, 250, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 114, 114, 111, 114, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 100, 101, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 115, 115, 97, 103, 101, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_fa787661cd3563a4 = b_fa787661cd3563a4.words; #if !CAPNP_LITE static const uint16_t m_fa787661cd3563a4[] = {0, 1}; static const uint16_t i_fa787661cd3563a4[] = {0, 1}; const ::capnp::_::RawSchema s_fa787661cd3563a4 = { - 0xfa787661cd3563a4, - b_fa787661cd3563a4.words, - 47, - nullptr, - m_fa787661cd3563a4, - 0, - 2, - i_fa787661cd3563a4, - nullptr, - nullptr, - {&s_fa787661cd3563a4, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_fda1cb9663a55b70 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 112, 91, 165, 99, 150, 203, 161, - 253, 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 108, 111, 97, 116, 83, - 99, 97, 108, 101, 67, 111, 110, 102, 105, 103, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, - 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, - 0, 3, 0, 1, 0, 76, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 3, 0, 1, - 0, 80, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 77, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 76, 0, 0, 0, 3, 0, 1, 0, 88, 0, 0, - 0, 2, 0, 1, 0, 115, 99, 97, 108, 101, 0, 0, 0, 11, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 102, 102, 115, 101, 116, - 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 121, - 116, 101, 87, 105, 100, 116, 104, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xfa787661cd3563a4, b_fa787661cd3563a4.words, 47, nullptr, m_fa787661cd3563a4, + 0, 2, i_fa787661cd3563a4, nullptr, nullptr, { &s_fa787661cd3563a4, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<65> b_fda1cb9663a55b70 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 112, 91, 165, 99, 150, 203, 161, 253, + 18, 0, 0, 0, 1, 0, 3, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 108, 111, 97, 116, 83, + 99, 97, 108, 101, 67, 111, 110, 102, + 105, 103, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 76, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 88, 0, 0, 0, 2, 0, 1, 0, + 115, 99, 97, 108, 101, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 121, 116, 101, 87, 105, 100, 116, + 104, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_fda1cb9663a55b70 = b_fda1cb9663a55b70.words; #if !CAPNP_LITE static const uint16_t m_fda1cb9663a55b70[] = {2, 1, 0}; static const uint16_t i_fda1cb9663a55b70[] = {0, 1, 2}; const ::capnp::_::RawSchema s_fda1cb9663a55b70 = { - 0xfda1cb9663a55b70, - b_fda1cb9663a55b70.words, - 65, - nullptr, - m_fda1cb9663a55b70, - 0, - 3, - i_fda1cb9663a55b70, - nullptr, - nullptr, - {&s_fda1cb9663a55b70, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<94> b_fd5825d8c6e74d78 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 120, 77, 231, 198, 216, 37, 88, - 253, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 234, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 101, 98, 112, 67, 111, - 110, 102, 105, 103, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 125, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 0, 0, 0, 3, 0, 1, 0, 132, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, - 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 124, 0, 0, 0, 3, 0, 1, 0, 136, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 40, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 74, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, - 3, 0, 1, 0, 144, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 66, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 145, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 140, 0, 0, 0, 3, 0, 1, 0, 152, 0, 0, 0, - 2, 0, 1, 0, 113, 117, 97, 108, 105, 116, 121, 0, 10, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 102, 111, 114, 109, 97, 116, 0, - 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 111, 115, - 115, 108, 101, 115, 115, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 120, 116, 101, 110, 116, - 88, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 120, - 116, 101, 110, 116, 89, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xfda1cb9663a55b70, b_fda1cb9663a55b70.words, 65, nullptr, m_fda1cb9663a55b70, + 0, 3, i_fda1cb9663a55b70, nullptr, nullptr, { &s_fda1cb9663a55b70, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<94> b_fd5825d8c6e74d78 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 120, 77, 231, 198, 216, 37, 88, 253, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 234, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 87, 101, 98, 112, 67, 111, + 110, 102, 105, 103, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 40, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 0, 0, 0, 3, 0, 1, 0, + 148, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 113, 117, 97, 108, 105, 116, 121, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 111, 114, 109, 97, 116, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 111, 115, 115, 108, 101, 115, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 120, 116, 101, 110, 116, 88, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 120, 116, 101, 110, 116, 89, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_fd5825d8c6e74d78 = b_fd5825d8c6e74d78.words; #if !CAPNP_LITE static const uint16_t m_fd5825d8c6e74d78[] = {3, 4, 1, 2, 0}; static const uint16_t i_fd5825d8c6e74d78[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_fd5825d8c6e74d78 = { - 0xfd5825d8c6e74d78, - b_fd5825d8c6e74d78.words, - 94, - nullptr, - m_fd5825d8c6e74d78, - 0, - 5, - i_fd5825d8c6e74d78, - nullptr, - nullptr, - {&s_fd5825d8c6e74d78, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<74> b_e7175047415b3f97 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 151, 63, 91, 65, 71, 80, 23, - 231, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 97, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 92, 0, 0, 0, 3, 0, 1, 0, 104, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 127, 137, 171, 179, 50, 248, 234, 156, 101, - 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 13, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 138, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, - 3, 0, 1, 0, 92, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 90, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, 116, 121, 112, 101, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 97, 116, - 97, 0, 0, 0, 0, 102, 108, 111, 97, 116, 83, 99, 97, 108, 101, - 67, 111, 110, 102, 105, 103, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 112, 91, 165, 99, 150, 203, 161, 253, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 101, 98, 112, 67, - 111, 110, 102, 105, 103, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 120, 77, 231, 198, 216, 37, 88, 253, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -}}; + 0xfd5825d8c6e74d78, b_fd5825d8c6e74d78.words, 94, nullptr, m_fd5825d8c6e74d78, + 0, 5, i_fd5825d8c6e74d78, nullptr, nullptr, { &s_fd5825d8c6e74d78, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<74> b_e7175047415b3f97 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 151, 63, 91, 65, 71, 80, 23, 231, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 105, 108, 116, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 127, 137, 171, 179, 50, 248, 234, 156, + 101, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 0, 0, 0, 3, 0, 1, 0, + 100, 0, 0, 0, 2, 0, 1, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 83, 99, 97, + 108, 101, 67, 111, 110, 102, 105, 103, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 112, 91, 165, 99, 150, 203, 161, 253, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 119, 101, 98, 112, 67, 111, 110, 102, + 105, 103, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 120, 77, 231, 198, 216, 37, 88, 253, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_e7175047415b3f97 = b_e7175047415b3f97.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_e7175047415b3f97[] = { - &s_9ceaf832b3ab897f, - &s_fd5825d8c6e74d78, - &s_fda1cb9663a55b70, + &s_9ceaf832b3ab897f, + &s_fd5825d8c6e74d78, + &s_fda1cb9663a55b70, }; static const uint16_t m_e7175047415b3f97[] = {1, 2, 0, 3}; static const uint16_t i_e7175047415b3f97[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_e7175047415b3f97 = { - 0xe7175047415b3f97, - b_e7175047415b3f97.words, - 74, - d_e7175047415b3f97, - m_e7175047415b3f97, - 3, - 4, - i_e7175047415b3f97, - nullptr, - nullptr, - {&s_e7175047415b3f97, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<197> b_9ceaf832b3ab897f = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 127, 137, 171, 179, 50, 248, 234, - 156, 25, 0, 0, 0, 1, 0, 2, 0, 151, 63, 91, 65, 71, 80, - 23, 231, 4, 0, 7, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, - 46, 100, 97, 116, 97, 0, 0, 0, 48, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 255, 255, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 1, 0, 0, - 3, 0, 1, 0, 72, 1, 0, 0, 2, 0, 1, 0, 1, 0, 254, - 255, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 50, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 64, 1, 0, 0, 3, 0, 1, 0, - 76, 1, 0, 0, 2, 0, 1, 0, 2, 0, 253, 255, 2, 0, 0, - 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 73, 1, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 68, 1, 0, 0, 3, 0, 1, 0, 80, 1, 0, 0, - 2, 0, 1, 0, 3, 0, 252, 255, 2, 0, 0, 0, 0, 0, 1, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 1, - 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, - 1, 0, 0, 3, 0, 1, 0, 84, 1, 0, 0, 2, 0, 1, 0, - 4, 0, 251, 255, 1, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 1, 0, 0, 50, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 1, 0, 0, 3, - 0, 1, 0, 88, 1, 0, 0, 2, 0, 1, 0, 5, 0, 250, 255, - 1, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 85, 1, 0, 0, 58, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 80, 1, 0, 0, 3, 0, 1, 0, 92, - 1, 0, 0, 2, 0, 1, 0, 6, 0, 249, 255, 1, 0, 0, 0, - 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 89, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 84, 1, 0, 0, 3, 0, 1, 0, 96, 1, 0, 0, 2, - 0, 1, 0, 7, 0, 248, 255, 1, 0, 0, 0, 0, 0, 1, 0, - 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 1, 0, - 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 1, - 0, 0, 3, 0, 1, 0, 100, 1, 0, 0, 2, 0, 1, 0, 8, - 0, 247, 255, 1, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 97, 1, 0, 0, 50, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 1, 0, 0, 3, 0, - 1, 0, 104, 1, 0, 0, 2, 0, 1, 0, 9, 0, 246, 255, 1, - 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 101, 1, 0, 0, 58, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 96, 1, 0, 0, 3, 0, 1, 0, 108, 1, - 0, 0, 2, 0, 1, 0, 10, 0, 245, 255, 1, 0, 0, 0, 0, - 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 1, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 100, 1, 0, 0, 3, 0, 1, 0, 112, 1, 0, 0, 2, 0, - 1, 0, 11, 0, 244, 255, 1, 0, 0, 0, 0, 0, 1, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1, 0, 0, - 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 1, 0, - 0, 3, 0, 1, 0, 116, 1, 0, 0, 2, 0, 1, 0, 116, 101, - 120, 116, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 98, 121, 116, 101, 115, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 105, 110, 116, 56, 0, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 105, 110, 116, 56, - 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, - 110, 116, 49, 54, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 117, 105, 110, 116, 49, 54, 0, 0, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 105, 110, 116, 51, 50, 0, 0, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 105, 110, 116, - 51, 50, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 110, 116, 54, 52, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 117, 105, 110, 116, 54, 52, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 102, 108, 111, 97, 116, 51, 50, - 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 108, 111, - 97, 116, 54, 52, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -}}; + 0xe7175047415b3f97, b_e7175047415b3f97.words, 74, d_e7175047415b3f97, m_e7175047415b3f97, + 3, 4, i_e7175047415b3f97, nullptr, nullptr, { &s_e7175047415b3f97, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<197> b_9ceaf832b3ab897f = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 127, 137, 171, 179, 50, 248, 234, 156, + 25, 0, 0, 0, 1, 0, 2, 0, + 151, 63, 91, 65, 71, 80, 23, 231, + 4, 0, 7, 0, 1, 0, 12, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 167, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 105, 108, 116, 101, 114, + 46, 100, 97, 116, 97, 0, 0, 0, + 48, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 255, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 1, 0, 0, 3, 0, 1, 0, + 72, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 254, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 76, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 253, 255, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 1, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 1, 0, 0, 3, 0, 1, 0, + 80, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 252, 255, 2, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 1, 0, 0, 3, 0, 1, 0, + 84, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 251, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 1, 0, 0, 3, 0, 1, 0, + 88, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 250, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 85, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 1, 0, 0, 3, 0, 1, 0, + 92, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 249, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 1, 0, 0, 3, 0, 1, 0, + 96, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 248, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 1, 0, 0, 3, 0, 1, 0, + 100, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 247, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 1, 0, 0, 3, 0, 1, 0, + 104, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 246, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 1, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 1, 0, 0, 3, 0, 1, 0, + 108, 1, 0, 0, 2, 0, 1, 0, + 10, 0, 245, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 1, 0, 0, 3, 0, 1, 0, + 112, 1, 0, 0, 2, 0, 1, 0, + 11, 0, 244, 255, 1, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 1, 0, 0, 3, 0, 1, 0, + 116, 1, 0, 0, 2, 0, 1, 0, + 116, 101, 120, 116, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 121, 116, 101, 115, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 56, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 56, 0, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 49, 54, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 49, 54, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 51, 50, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 51, 50, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 116, 54, 52, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 105, 110, 116, 54, 52, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 51, 50, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 10, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 108, 111, 97, 116, 54, 52, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9ceaf832b3ab897f = b_9ceaf832b3ab897f.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9ceaf832b3ab897f[] = { - &s_e7175047415b3f97, + &s_e7175047415b3f97, }; -static const uint16_t m_9ceaf832b3ab897f[] = { - 1, 10, 11, 4, 6, 8, 2, 0, 5, 7, 9, 3}; -static const uint16_t i_9ceaf832b3ab897f[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; +static const uint16_t m_9ceaf832b3ab897f[] = {1, 10, 11, 4, 6, 8, 2, 0, 5, 7, 9, 3}; +static const uint16_t i_9ceaf832b3ab897f[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; const ::capnp::_::RawSchema s_9ceaf832b3ab897f = { - 0x9ceaf832b3ab897f, - b_9ceaf832b3ab897f.words, - 197, - d_9ceaf832b3ab897f, - m_9ceaf832b3ab897f, - 1, - 12, - i_9ceaf832b3ab897f, - nullptr, - nullptr, - {&s_9ceaf832b3ab897f, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<38> b_bc4583f733eac4f5 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 245, 196, 234, 51, 247, 131, 69, - 188, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, - 80, 105, 112, 101, 108, 105, 110, 101, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 3, 0, 1, 0, 36, 0, 0, 0, 2, 0, 1, 0, 102, 105, - 108, 116, 101, 114, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 151, 63, 91, 65, 71, 80, 23, 231, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0x9ceaf832b3ab897f, b_9ceaf832b3ab897f.words, 197, d_9ceaf832b3ab897f, m_9ceaf832b3ab897f, + 1, 12, i_9ceaf832b3ab897f, nullptr, nullptr, { &s_9ceaf832b3ab897f, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<38> b_bc4583f733eac4f5 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 245, 196, 234, 51, 247, 131, 69, 188, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 105, 108, 116, 101, 114, + 80, 105, 112, 101, 108, 105, 110, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 102, 105, 108, 116, 101, 114, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 63, 91, 65, 71, 80, 23, 231, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_bc4583f733eac4f5 = b_bc4583f733eac4f5.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_bc4583f733eac4f5[] = { - &s_e7175047415b3f97, + &s_e7175047415b3f97, }; static const uint16_t m_bc4583f733eac4f5[] = {0}; static const uint16_t i_bc4583f733eac4f5[] = {0}; const ::capnp::_::RawSchema s_bc4583f733eac4f5 = { - 0xbc4583f733eac4f5, - b_bc4583f733eac4f5.words, - 38, - d_bc4583f733eac4f5, - m_bc4583f733eac4f5, - 1, - 1, - i_bc4583f733eac4f5, - nullptr, - nullptr, - {&s_bc4583f733eac4f5, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_f179c194ae71718c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 140, 113, 113, 174, 148, 193, 121, - 241, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 21, 0, 0, 0, 178, 0, 0, 0, 29, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 0, - 0, 0, 23, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 0, 0, 0, - 4, 0, 0, 0, 1, 0, 1, 0, 234, 250, 246, 170, 200, 20, 85, - 219, 1, 0, 0, 0, 50, 0, 0, 0, 69, 110, 116, 114, 121, 0, - 0, 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 13, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 56, 0, - 0, 0, 2, 0, 1, 0, 101, 110, 116, 114, 105, 101, 115, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 234, 250, 246, 170, 200, 20, - 85, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, 0, 0, - 2, 0, 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, - 5, 0, 0, 0, 34, 0, 0, 0, 5, 0, 0, 0, 50, 0, 0, - 0, 75, 101, 121, 0, 0, 0, 0, 0, 86, 97, 108, 117, 101, 0, - 0, 0, -}}; + 0xbc4583f733eac4f5, b_bc4583f733eac4f5.words, 38, d_bc4583f733eac4f5, m_bc4583f733eac4f5, + 1, 1, i_bc4583f733eac4f5, nullptr, nullptr, { &s_bc4583f733eac4f5, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_f179c194ae71718c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, + 21, 0, 0, 0, 178, 0, 0, 0, + 29, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 23, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 234, 250, 246, 170, 200, 20, 85, 219, + 1, 0, 0, 0, 50, 0, 0, 0, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 234, 250, 246, 170, 200, 20, 85, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 1, 0, + 5, 0, 0, 0, 34, 0, 0, 0, + 5, 0, 0, 0, 50, 0, 0, 0, + 75, 101, 121, 0, 0, 0, 0, 0, + 86, 97, 108, 117, 101, 0, 0, 0, } +}; ::capnp::word const* const bp_f179c194ae71718c = b_f179c194ae71718c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_f179c194ae71718c[] = { - &s_db5514c8aaf6faea, + &s_db5514c8aaf6faea, }; static const uint16_t m_f179c194ae71718c[] = {0}; static const uint16_t i_f179c194ae71718c[] = {0}; -KJ_CONSTEXPR(const)::capnp::_::RawBrandedSchema::Dependency - bd_f179c194ae71718c[] = { - {16777216, - ::tiledb::sm::serialization::capnp::Map< - ::capnp::AnyPointer, - ::capnp::AnyPointer>::Entry::_capnpPrivate::brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_f179c194ae71718c[] = { + { 16777216, ::tiledb::sm::serialization::capnp::Map< ::capnp::AnyPointer, ::capnp::AnyPointer>::Entry::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_f179c194ae71718c = { - 0xf179c194ae71718c, - b_f179c194ae71718c.words, - 49, - d_f179c194ae71718c, - m_f179c194ae71718c, - 1, - 1, - i_f179c194ae71718c, - nullptr, - nullptr, - {&s_f179c194ae71718c, - nullptr, - bd_f179c194ae71718c, - 0, - sizeof(bd_f179c194ae71718c) / sizeof(bd_f179c194ae71718c[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_db5514c8aaf6faea = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 234, 250, 246, 170, 200, 20, 85, - 219, 22, 0, 0, 0, 1, 0, 0, 0, 140, 113, 113, 174, 148, 193, - 121, 241, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 46, 69, 110, - 116, 114, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 0, 0, 0, 3, 0, 1, 0, 52, 0, 0, 0, 2, 0, 1, - 0, 107, 101, 121, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, 148, - 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 118, 97, 108, 117, 101, 0, 0, 0, 18, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 140, - 113, 113, 174, 148, 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, - 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xf179c194ae71718c, b_f179c194ae71718c.words, 49, d_f179c194ae71718c, m_f179c194ae71718c, + 1, 1, i_f179c194ae71718c, nullptr, nullptr, { &s_f179c194ae71718c, nullptr, bd_f179c194ae71718c, 0, sizeof(bd_f179c194ae71718c) / sizeof(bd_f179c194ae71718c[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<48> b_db5514c8aaf6faea = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 234, 250, 246, 170, 200, 20, 85, 219, + 22, 0, 0, 0, 1, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 46, 69, 110, + 116, 114, 121, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 18, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 18, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 18, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 18, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_db5514c8aaf6faea = b_db5514c8aaf6faea.words; #if !CAPNP_LITE static const uint16_t m_db5514c8aaf6faea[] = {0, 1}; static const uint16_t i_db5514c8aaf6faea[] = {0, 1}; const ::capnp::_::RawSchema s_db5514c8aaf6faea = { - 0xdb5514c8aaf6faea, - b_db5514c8aaf6faea.words, - 48, - nullptr, - m_db5514c8aaf6faea, - 0, - 2, - i_db5514c8aaf6faea, - nullptr, - nullptr, - {&s_db5514c8aaf6faea, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_c6b5bb09d4611252 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 82, 18, 97, 212, 9, 187, 181, - 198, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, - 116, 51, 50, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 198, 165, 33, 37, 95, 10, 78, 136, 1, 0, 0, 0, 50, 0, - 0, 0, 69, 110, 116, 114, 121, 0, 0, 0, 4, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 1, 0, 36, 0, 0, 0, 2, 0, 1, 0, 101, - 110, 116, 114, 105, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 198, 165, 33, 37, 95, 10, 78, 136, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xdb5514c8aaf6faea, b_db5514c8aaf6faea.words, 48, nullptr, m_db5514c8aaf6faea, + 0, 2, i_db5514c8aaf6faea, nullptr, nullptr, { &s_db5514c8aaf6faea, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<40> b_c6b5bb09d4611252 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 82, 18, 97, 212, 9, 187, 181, 198, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 85, 73, 110, + 116, 51, 50, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 198, 165, 33, 37, 95, 10, 78, 136, + 1, 0, 0, 0, 50, 0, 0, 0, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 198, 165, 33, 37, 95, 10, 78, 136, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c6b5bb09d4611252 = b_c6b5bb09d4611252.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c6b5bb09d4611252[] = { - &s_884e0a5f2521a5c6, + &s_884e0a5f2521a5c6, }; static const uint16_t m_c6b5bb09d4611252[] = {0}; static const uint16_t i_c6b5bb09d4611252[] = {0}; const ::capnp::_::RawSchema s_c6b5bb09d4611252 = { - 0xc6b5bb09d4611252, - b_c6b5bb09d4611252.words, - 40, - d_c6b5bb09d4611252, - m_c6b5bb09d4611252, - 1, - 1, - i_c6b5bb09d4611252, - nullptr, - nullptr, - {&s_c6b5bb09d4611252, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_884e0a5f2521a5c6 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 198, 165, 33, 37, 95, 10, 78, - 136, 28, 0, 0, 0, 1, 0, 1, 0, 82, 18, 97, 212, 9, 187, - 181, 198, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, - 116, 51, 50, 46, 69, 110, 116, 114, 121, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, - 0, 52, 0, 0, 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xc6b5bb09d4611252, b_c6b5bb09d4611252.words, 40, d_c6b5bb09d4611252, m_c6b5bb09d4611252, + 1, 1, i_c6b5bb09d4611252, nullptr, nullptr, { &s_c6b5bb09d4611252, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_884e0a5f2521a5c6 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 198, 165, 33, 37, 95, 10, 78, 136, + 28, 0, 0, 0, 1, 0, 1, 0, + 82, 18, 97, 212, 9, 187, 181, 198, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 85, 73, 110, + 116, 51, 50, 46, 69, 110, 116, 114, + 121, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_884e0a5f2521a5c6 = b_884e0a5f2521a5c6.words; #if !CAPNP_LITE static const uint16_t m_884e0a5f2521a5c6[] = {0, 1}; static const uint16_t i_884e0a5f2521a5c6[] = {0, 1}; const ::capnp::_::RawSchema s_884e0a5f2521a5c6 = { - 0x884e0a5f2521a5c6, - b_884e0a5f2521a5c6.words, - 49, - nullptr, - m_884e0a5f2521a5c6, - 0, - 2, - i_884e0a5f2521a5c6, - nullptr, - nullptr, - {&s_884e0a5f2521a5c6, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_a83707d3ba24dd32 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 50, 221, 36, 186, 211, 7, 55, - 168, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 218, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, - 54, 52, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 175, 43, 58, 51, 180, 204, 202, 169, 1, 0, 0, 0, 50, 0, - 0, 0, 69, 110, 116, 114, 121, 0, 0, 0, 4, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 1, 0, 36, 0, 0, 0, 2, 0, 1, 0, 101, - 110, 116, 114, 105, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 175, 43, 58, 51, 180, 204, 202, 169, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0x884e0a5f2521a5c6, b_884e0a5f2521a5c6.words, 49, nullptr, m_884e0a5f2521a5c6, + 0, 2, i_884e0a5f2521a5c6, nullptr, nullptr, { &s_884e0a5f2521a5c6, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<40> b_a83707d3ba24dd32 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 50, 221, 36, 186, 211, 7, 55, 168, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 218, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 73, 110, 116, + 54, 52, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 175, 43, 58, 51, 180, 204, 202, 169, + 1, 0, 0, 0, 50, 0, 0, 0, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 175, 43, 58, 51, 180, 204, 202, 169, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a83707d3ba24dd32 = b_a83707d3ba24dd32.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a83707d3ba24dd32[] = { - &s_a9caccb4333a2baf, + &s_a9caccb4333a2baf, }; static const uint16_t m_a83707d3ba24dd32[] = {0}; static const uint16_t i_a83707d3ba24dd32[] = {0}; const ::capnp::_::RawSchema s_a83707d3ba24dd32 = { - 0xa83707d3ba24dd32, - b_a83707d3ba24dd32.words, - 40, - d_a83707d3ba24dd32, - m_a83707d3ba24dd32, - 1, - 1, - i_a83707d3ba24dd32, - nullptr, - nullptr, - {&s_a83707d3ba24dd32, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_a9caccb4333a2baf = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 175, 43, 58, 51, 180, 204, 202, - 169, 27, 0, 0, 0, 1, 0, 1, 0, 50, 221, 36, 186, 211, 7, - 55, 168, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, - 54, 52, 46, 69, 110, 116, 114, 121, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, - 0, 52, 0, 0, 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xa83707d3ba24dd32, b_a83707d3ba24dd32.words, 40, d_a83707d3ba24dd32, m_a83707d3ba24dd32, + 1, 1, i_a83707d3ba24dd32, nullptr, nullptr, { &s_a83707d3ba24dd32, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_a9caccb4333a2baf = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 175, 43, 58, 51, 180, 204, 202, 169, + 27, 0, 0, 0, 1, 0, 1, 0, + 50, 221, 36, 186, 211, 7, 55, 168, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 73, 110, 116, + 54, 52, 46, 69, 110, 116, 114, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a9caccb4333a2baf = b_a9caccb4333a2baf.words; #if !CAPNP_LITE static const uint16_t m_a9caccb4333a2baf[] = {0, 1}; static const uint16_t i_a9caccb4333a2baf[] = {0, 1}; const ::capnp::_::RawSchema s_a9caccb4333a2baf = { - 0xa9caccb4333a2baf, - b_a9caccb4333a2baf.words, - 49, - nullptr, - m_a9caccb4333a2baf, - 0, - 2, - i_a9caccb4333a2baf, - nullptr, - nullptr, - {&s_a9caccb4333a2baf, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_d3c5983c670e0f42 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 66, 15, 14, 103, 60, 152, 197, - 211, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, - 116, 54, 52, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 154, 36, 205, 167, 45, 69, 215, 130, 1, 0, 0, 0, 50, 0, - 0, 0, 69, 110, 116, 114, 121, 0, 0, 0, 4, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 1, 0, 36, 0, 0, 0, 2, 0, 1, 0, 101, - 110, 116, 114, 105, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 154, 36, 205, 167, 45, 69, 215, 130, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xa9caccb4333a2baf, b_a9caccb4333a2baf.words, 49, nullptr, m_a9caccb4333a2baf, + 0, 2, i_a9caccb4333a2baf, nullptr, nullptr, { &s_a9caccb4333a2baf, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<40> b_d3c5983c670e0f42 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 66, 15, 14, 103, 60, 152, 197, 211, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 85, 73, 110, + 116, 54, 52, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 154, 36, 205, 167, 45, 69, 215, 130, + 1, 0, 0, 0, 50, 0, 0, 0, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 154, 36, 205, 167, 45, 69, 215, 130, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d3c5983c670e0f42 = b_d3c5983c670e0f42.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_d3c5983c670e0f42[] = { - &s_82d7452da7cd249a, + &s_82d7452da7cd249a, }; static const uint16_t m_d3c5983c670e0f42[] = {0}; static const uint16_t i_d3c5983c670e0f42[] = {0}; const ::capnp::_::RawSchema s_d3c5983c670e0f42 = { - 0xd3c5983c670e0f42, - b_d3c5983c670e0f42.words, - 40, - d_d3c5983c670e0f42, - m_d3c5983c670e0f42, - 1, - 1, - i_d3c5983c670e0f42, - nullptr, - nullptr, - {&s_d3c5983c670e0f42, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_82d7452da7cd249a = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 154, 36, 205, 167, 45, 69, 215, - 130, 28, 0, 0, 0, 1, 0, 1, 0, 66, 15, 14, 103, 60, 152, - 197, 211, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, - 116, 54, 52, 46, 69, 110, 116, 114, 121, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, - 0, 52, 0, 0, 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xd3c5983c670e0f42, b_d3c5983c670e0f42.words, 40, d_d3c5983c670e0f42, m_d3c5983c670e0f42, + 1, 1, i_d3c5983c670e0f42, nullptr, nullptr, { &s_d3c5983c670e0f42, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_82d7452da7cd249a = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 154, 36, 205, 167, 45, 69, 215, 130, + 28, 0, 0, 0, 1, 0, 1, 0, + 66, 15, 14, 103, 60, 152, 197, 211, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 85, 73, 110, + 116, 54, 52, 46, 69, 110, 116, 114, + 121, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_82d7452da7cd249a = b_82d7452da7cd249a.words; #if !CAPNP_LITE static const uint16_t m_82d7452da7cd249a[] = {0, 1}; static const uint16_t i_82d7452da7cd249a[] = {0, 1}; const ::capnp::_::RawSchema s_82d7452da7cd249a = { - 0x82d7452da7cd249a, - b_82d7452da7cd249a.words, - 49, - nullptr, - m_82d7452da7cd249a, - 0, - 2, - i_82d7452da7cd249a, - nullptr, - nullptr, - {&s_82d7452da7cd249a, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_9354174d952207d2 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 210, 7, 34, 149, 77, 23, 84, - 147, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 234, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, - 97, 116, 54, 52, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 52, 41, 49, 63, 188, 205, 128, 187, 1, 0, 0, 0, 50, 0, - 0, 0, 69, 110, 116, 114, 121, 0, 0, 0, 4, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 1, 0, 36, 0, 0, 0, 2, 0, 1, 0, 101, - 110, 116, 114, 105, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 52, 41, 49, 63, 188, 205, 128, 187, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0x82d7452da7cd249a, b_82d7452da7cd249a.words, 49, nullptr, m_82d7452da7cd249a, + 0, 2, i_82d7452da7cd249a, nullptr, nullptr, { &s_82d7452da7cd249a, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<40> b_9354174d952207d2 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 210, 7, 34, 149, 77, 23, 84, 147, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 234, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 70, 108, 111, + 97, 116, 54, 52, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 52, 41, 49, 63, 188, 205, 128, 187, + 1, 0, 0, 0, 50, 0, 0, 0, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 52, 41, 49, 63, 188, 205, 128, 187, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9354174d952207d2 = b_9354174d952207d2.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9354174d952207d2[] = { - &s_bb80cdbc3f312934, + &s_bb80cdbc3f312934, }; static const uint16_t m_9354174d952207d2[] = {0}; static const uint16_t i_9354174d952207d2[] = {0}; const ::capnp::_::RawSchema s_9354174d952207d2 = { - 0x9354174d952207d2, - b_9354174d952207d2.words, - 40, - d_9354174d952207d2, - m_9354174d952207d2, - 1, - 1, - i_9354174d952207d2, - nullptr, - nullptr, - {&s_9354174d952207d2, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_bb80cdbc3f312934 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 52, 41, 49, 63, 188, 205, 128, - 187, 29, 0, 0, 0, 1, 0, 1, 0, 210, 7, 34, 149, 77, 23, - 84, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, - 97, 116, 54, 52, 46, 69, 110, 116, 114, 121, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, - 0, 52, 0, 0, 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0x9354174d952207d2, b_9354174d952207d2.words, 40, d_9354174d952207d2, m_9354174d952207d2, + 1, 1, i_9354174d952207d2, nullptr, nullptr, { &s_9354174d952207d2, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_bb80cdbc3f312934 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 52, 41, 49, 63, 188, 205, 128, 187, + 29, 0, 0, 0, 1, 0, 1, 0, + 210, 7, 34, 149, 77, 23, 84, 147, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 112, 70, 108, 111, + 97, 116, 54, 52, 46, 69, 110, 116, + 114, 121, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_bb80cdbc3f312934 = b_bb80cdbc3f312934.words; #if !CAPNP_LITE static const uint16_t m_bb80cdbc3f312934[] = {0, 1}; static const uint16_t i_bb80cdbc3f312934[] = {0, 1}; const ::capnp::_::RawSchema s_bb80cdbc3f312934 = { - 0xbb80cdbc3f312934, - b_bb80cdbc3f312934.words, - 49, - nullptr, - m_bb80cdbc3f312934, - 0, - 2, - i_bb80cdbc3f312934, - nullptr, - nullptr, - {&s_bb80cdbc3f312934, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_c7e036a11506a1a4 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 164, 161, 6, 21, 161, 54, 224, - 199, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 116, 97, 116, 115, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 58, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, - 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 74, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 3, 0, 1, 0, - 56, 0, 0, 0, 2, 0, 1, 0, 116, 105, 109, 101, 114, 115, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 210, 7, 34, 149, 77, 23, - 84, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, 117, - 110, 116, 101, 114, 115, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 66, 15, 14, 103, 60, 152, 197, 211, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xbb80cdbc3f312934, b_bb80cdbc3f312934.words, 49, nullptr, m_bb80cdbc3f312934, + 0, 2, i_bb80cdbc3f312934, nullptr, nullptr, { &s_bb80cdbc3f312934, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<48> b_c7e036a11506a1a4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 116, 97, 116, 115, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 116, 105, 109, 101, 114, 115, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 210, 7, 34, 149, 77, 23, 84, 147, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 117, 110, 116, 101, 114, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 66, 15, 14, 103, 60, 152, 197, 211, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c7e036a11506a1a4 = b_c7e036a11506a1a4.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c7e036a11506a1a4[] = { - &s_9354174d952207d2, - &s_d3c5983c670e0f42, + &s_9354174d952207d2, + &s_d3c5983c670e0f42, }; static const uint16_t m_c7e036a11506a1a4[] = {1, 0}; static const uint16_t i_c7e036a11506a1a4[] = {0, 1}; const ::capnp::_::RawSchema s_c7e036a11506a1a4 = { - 0xc7e036a11506a1a4, - b_c7e036a11506a1a4.words, - 48, - d_c7e036a11506a1a4, - m_c7e036a11506a1a4, - 2, - 2, - i_c7e036a11506a1a4, - nullptr, - nullptr, - {&s_c7e036a11506a1a4, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<90> b_8bf6c1d37e748294 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 148, 130, 116, 126, 211, 193, 246, - 139, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 85, 110, 111, 114, 100, 101, - 114, 101, 100, 87, 114, 105, 116, 101, 114, 83, 116, 97, 116, 101, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 16, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, - 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, - 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 3, 0, 1, - 0, 128, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 125, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 124, 0, 0, 0, 3, 0, 1, 0, 152, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, - 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 0, 0, 0, 3, 0, 1, 0, 160, 0, 0, 0, 2, 0, 1, - 0, 105, 115, 67, 111, 111, 114, 100, 115, 80, 97, 115, 115, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 101, - 108, 108, 80, 111, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 99, 111, 111, 114, 100, 68, 117, 112, 115, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 102, 114, 97, 103, 77, 101, 116, 97, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 97, 202, 231, - 39, 252, 82, 227, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xc7e036a11506a1a4, b_c7e036a11506a1a4.words, 48, d_c7e036a11506a1a4, m_c7e036a11506a1a4, + 2, 2, i_c7e036a11506a1a4, nullptr, nullptr, { &s_c7e036a11506a1a4, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<90> b_8bf6c1d37e748294 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 148, 130, 116, 126, 211, 193, 246, 139, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 85, 110, 111, 114, 100, 101, + 114, 101, 100, 87, 114, 105, 116, 101, + 114, 83, 116, 97, 116, 101, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 128, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 160, 0, 0, 0, 2, 0, 1, 0, + 105, 115, 67, 111, 111, 114, 100, 115, + 80, 97, 115, 115, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 80, 111, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 111, 114, 100, 68, 117, 112, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 77, 101, 116, 97, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_8bf6c1d37e748294 = b_8bf6c1d37e748294.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_8bf6c1d37e748294[] = { - &s_cde352fc27e7ca61, + &s_cde352fc27e7ca61, }; static const uint16_t m_8bf6c1d37e748294[] = {1, 2, 3, 0}; static const uint16_t i_8bf6c1d37e748294[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_8bf6c1d37e748294 = { - 0x8bf6c1d37e748294, - b_8bf6c1d37e748294.words, - 90, - d_8bf6c1d37e748294, - m_8bf6c1d37e748294, - 1, - 4, - i_8bf6c1d37e748294, - nullptr, - nullptr, - {&s_8bf6c1d37e748294, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<147> b_8ba60147a0e6735e = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 94, 115, 230, 160, 71, 1, 166, - 139, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 101, 114, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 32, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 209, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 208, 0, 0, 0, 3, 0, 1, 0, 220, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, - 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, 228, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 98, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, - 3, 0, 1, 0, 236, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 74, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 241, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 240, 0, 0, 0, 3, 0, 1, 0, 252, 0, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, - 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 244, - 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 253, 0, 0, 0, 154, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, - 0, 1, 0, 12, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, - 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 1, 0, 0, 170, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 3, 0, 1, 0, 24, - 1, 0, 0, 2, 0, 1, 0, 99, 104, 101, 99, 107, 67, 111, 111, - 114, 100, 68, 117, 112, 115, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 99, 104, 101, 99, 107, 67, 111, 111, 114, 100, 79, - 79, 66, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 100, 101, 100, 117, 112, 67, 111, 111, 114, 100, 115, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 117, - 98, 97, 114, 114, 97, 121, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 236, 12, 65, 249, 230, 4, 89, 206, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 117, 98, 97, 114, - 114, 97, 121, 82, 97, 110, 103, 101, 115, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 201, 218, 138, 19, 236, 13, 162, 219, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 115, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 164, 161, 6, 21, 161, 54, 224, - 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 108, 111, 98, - 97, 108, 87, 114, 105, 116, 101, 83, 116, 97, 116, 101, 86, 49, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 39, 68, - 15, 57, 181, 48, 199, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 117, 110, 111, 114, 100, 101, 114, 101, 100, 87, 114, 105, 116, 101, - 114, 83, 116, 97, 116, 101, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 148, 130, 116, 126, 211, 193, 246, 139, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -}}; + 0x8bf6c1d37e748294, b_8bf6c1d37e748294.words, 90, d_8bf6c1d37e748294, m_8bf6c1d37e748294, + 1, 4, i_8bf6c1d37e748294, nullptr, nullptr, { &s_8bf6c1d37e748294, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<147> b_8ba60147a0e6735e = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 94, 115, 230, 160, 71, 1, 166, 139, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 199, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 87, 114, 105, 116, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 32, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 0, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 216, 0, 0, 0, 3, 0, 1, 0, + 228, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 225, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 224, 0, 0, 0, 3, 0, 1, 0, + 236, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 233, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 244, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 3, 0, 1, 0, + 12, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 99, 104, 101, 99, 107, 67, 111, 111, + 114, 100, 68, 117, 112, 115, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 104, 101, 99, 107, 67, 111, 111, + 114, 100, 79, 79, 66, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 100, 117, 112, 67, 111, 111, + 114, 100, 115, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 236, 12, 65, 249, 230, 4, 89, 206, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 82, 97, 110, 103, 101, 115, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 108, 111, 98, 97, 108, 87, 114, + 105, 116, 101, 83, 116, 97, 116, 101, + 86, 49, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 39, 68, 15, 57, 181, 48, 199, 192, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 110, 111, 114, 100, 101, 114, 101, + 100, 87, 114, 105, 116, 101, 114, 83, + 116, 97, 116, 101, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 148, 130, 116, 126, 211, 193, 246, 139, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_8ba60147a0e6735e = b_8ba60147a0e6735e.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_8ba60147a0e6735e[] = { - &s_8bf6c1d37e748294, - &s_c0c730b5390f4427, - &s_c7e036a11506a1a4, - &s_ce5904e6f9410cec, - &s_dba20dec138adac9, + &s_8bf6c1d37e748294, + &s_c0c730b5390f4427, + &s_c7e036a11506a1a4, + &s_ce5904e6f9410cec, + &s_dba20dec138adac9, }; static const uint16_t m_8ba60147a0e6735e[] = {0, 1, 2, 6, 5, 3, 4, 7}; static const uint16_t i_8ba60147a0e6735e[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_8ba60147a0e6735e = { - 0x8ba60147a0e6735e, - b_8ba60147a0e6735e.words, - 147, - d_8ba60147a0e6735e, - m_8ba60147a0e6735e, - 5, - 8, - i_8ba60147a0e6735e, - nullptr, - nullptr, - {&s_8ba60147a0e6735e, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<106> b_86cfc12d74ed4aa0 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 160, 74, 237, 116, 45, 193, 207, - 134, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, - 97, 121, 82, 97, 110, 103, 101, 115, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 20, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, - 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, - 0, 3, 0, 1, 0, 132, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 130, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 3, 0, 1, - 0, 140, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 137, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 0, 0, 0, 3, 0, 1, 0, 144, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, - 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 140, 0, 0, 0, 3, 0, 1, 0, 168, 0, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 138, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, - 3, 0, 1, 0, 196, 0, 0, 0, 2, 0, 1, 0, 116, 121, 112, - 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 104, 97, 115, 68, 101, 102, 97, 117, 108, 116, 82, 97, 110, 103, - 101, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 117, - 102, 102, 101, 114, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 98, 117, 102, 102, 101, 114, 83, 105, 122, 101, 115, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 98, 117, 102, 102, 101, 114, 83, 116, 97, 114, 116, 83, 105, 122, - 101, 115, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x8ba60147a0e6735e, b_8ba60147a0e6735e.words, 147, d_8ba60147a0e6735e, m_8ba60147a0e6735e, + 5, 8, i_8ba60147a0e6735e, nullptr, nullptr, { &s_8ba60147a0e6735e, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<106> b_86cfc12d74ed4aa0 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 160, 74, 237, 116, 45, 193, 207, 134, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 117, 98, 97, 114, 114, + 97, 121, 82, 97, 110, 103, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 140, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 0, 0, 0, 3, 0, 1, 0, + 168, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 168, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 97, 115, 68, 101, 102, 97, 117, + 108, 116, 82, 97, 110, 103, 101, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 117, 102, 102, 101, 114, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 117, 102, 102, 101, 114, 83, 105, + 122, 101, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 117, 102, 102, 101, 114, 83, 116, + 97, 114, 116, 83, 105, 122, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_86cfc12d74ed4aa0 = b_86cfc12d74ed4aa0.words; #if !CAPNP_LITE static const uint16_t m_86cfc12d74ed4aa0[] = {2, 3, 4, 1, 0}; static const uint16_t i_86cfc12d74ed4aa0[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_86cfc12d74ed4aa0 = { - 0x86cfc12d74ed4aa0, - b_86cfc12d74ed4aa0.words, - 106, - nullptr, - m_86cfc12d74ed4aa0, - 0, - 5, - i_86cfc12d74ed4aa0, - nullptr, - nullptr, - {&s_86cfc12d74ed4aa0, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_f7aa276767b422e7 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 231, 34, 180, 103, 103, 39, 170, - 247, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 97, 98, 101, 108, 83, - 117, 98, 97, 114, 114, 97, 121, 82, 97, 110, 103, 101, 115, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, - 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, - 0, 3, 0, 1, 0, 80, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 42, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, 1, - 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 81, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 76, 0, 0, 0, 3, 0, 1, 0, 88, 0, 0, - 0, 2, 0, 1, 0, 100, 105, 109, 101, 110, 115, 105, 111, 110, 73, - 100, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 110, 97, 109, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 114, 97, 110, 103, 101, 115, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 160, 74, 237, 116, 45, 193, 207, 134, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x86cfc12d74ed4aa0, b_86cfc12d74ed4aa0.words, 106, nullptr, m_86cfc12d74ed4aa0, + 0, 5, i_86cfc12d74ed4aa0, nullptr, nullptr, { &s_86cfc12d74ed4aa0, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<65> b_f7aa276767b422e7 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 231, 34, 180, 103, 103, 39, 170, 247, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 76, 97, 98, 101, 108, 83, + 117, 98, 97, 114, 114, 97, 121, 82, + 97, 110, 103, 101, 115, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 88, 0, 0, 0, 2, 0, 1, 0, + 100, 105, 109, 101, 110, 115, 105, 111, + 110, 73, 100, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 97, 110, 103, 101, 115, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 160, 74, 237, 116, 45, 193, 207, 134, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_f7aa276767b422e7 = b_f7aa276767b422e7.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_f7aa276767b422e7[] = { - &s_86cfc12d74ed4aa0, + &s_86cfc12d74ed4aa0, }; static const uint16_t m_f7aa276767b422e7[] = {0, 1, 2}; static const uint16_t i_f7aa276767b422e7[] = {0, 1, 2}; const ::capnp::_::RawSchema s_f7aa276767b422e7 = { - 0xf7aa276767b422e7, - b_f7aa276767b422e7.words, - 65, - d_f7aa276767b422e7, - m_f7aa276767b422e7, - 1, - 3, - i_f7aa276767b422e7, - nullptr, - nullptr, - {&s_f7aa276767b422e7, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<158> b_dba20dec138adac9 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 201, 218, 138, 19, 236, 13, 162, - 219, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 218, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, - 97, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 181, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 176, 0, 0, 0, 3, 0, 1, 0, 188, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, - 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 0, 0, 0, 3, 0, 1, 0, 208, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 0, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, - 3, 0, 1, 0, 212, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 146, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 3, 0, 1, 0, - 240, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 237, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 236, 0, 0, 0, 3, 0, 1, 0, 8, 1, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, - 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 1, 0, 0, 3, 0, 1, 0, 88, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, 85, 1, 0, 0, 122, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 1, 0, 0, 3, - 0, 1, 0, 96, 1, 0, 0, 2, 0, 1, 0, 108, 97, 121, 111, - 117, 116, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 114, 97, 110, 103, 101, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 160, 74, 237, 116, 45, 193, 207, 134, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 115, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 164, 161, 6, 21, 161, 54, 224, 199, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 114, 101, 108, 101, 118, - 97, 110, 116, 70, 114, 97, 103, 109, 101, 110, 116, 115, 0, 0, 0, - 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 108, 97, 98, 101, 108, 82, 97, 110, 103, 101, 115, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 231, 34, - 180, 103, 103, 39, 170, 247, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 82, 97, 110, 103, 101, - 115, 0, 16, 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, 148, - 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, 0, - 0, 2, 0, 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 8, - 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 160, 74, 237, 116, 45, 193, 207, 134, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 99, 111, 97, 108, 101, 115, 99, 101, - 82, 97, 110, 103, 101, 115, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xf7aa276767b422e7, b_f7aa276767b422e7.words, 65, d_f7aa276767b422e7, m_f7aa276767b422e7, + 1, 3, i_f7aa276767b422e7, nullptr, nullptr, { &s_f7aa276767b422e7, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<158> b_dba20dec138adac9 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 6, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 218, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 117, 98, 97, 114, 114, + 97, 121, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 88, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 85, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 1, 0, 0, 3, 0, 1, 0, + 96, 1, 0, 0, 2, 0, 1, 0, + 108, 97, 121, 111, 117, 116, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 97, 110, 103, 101, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 160, 74, 237, 116, 45, 193, 207, 134, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 108, 101, 118, 97, 110, 116, + 70, 114, 97, 103, 109, 101, 110, 116, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 98, 101, 108, 82, 97, 110, + 103, 101, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 231, 34, 180, 103, 103, 39, 170, 247, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 82, 97, 110, 103, 101, 115, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 160, 74, 237, 116, 45, 193, 207, 134, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 97, 108, 101, 115, 99, 101, + 82, 97, 110, 103, 101, 115, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_dba20dec138adac9 = b_dba20dec138adac9.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_dba20dec138adac9[] = { - &s_86cfc12d74ed4aa0, - &s_c7e036a11506a1a4, - &s_f179c194ae71718c, - &s_f7aa276767b422e7, + &s_86cfc12d74ed4aa0, + &s_c7e036a11506a1a4, + &s_f179c194ae71718c, + &s_f7aa276767b422e7, }; static const uint16_t m_dba20dec138adac9[] = {5, 6, 4, 0, 1, 3, 2}; static const uint16_t i_dba20dec138adac9[] = {0, 1, 2, 3, 4, 5, 6}; -KJ_CONSTEXPR( - const)::capnp::_::RawBrandedSchema::Dependency bd_dba20dec138adac9[] = { - {16777221, - ::tiledb::sm::serialization::capnp::Map< - ::capnp::Text, - ::tiledb::sm::serialization::capnp::SubarrayRanges>::_capnpPrivate:: - brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_dba20dec138adac9[] = { + { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::SubarrayRanges>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_dba20dec138adac9 = { - 0xdba20dec138adac9, - b_dba20dec138adac9.words, - 158, - d_dba20dec138adac9, - m_dba20dec138adac9, - 4, - 7, - i_dba20dec138adac9, - nullptr, - nullptr, - {&s_dba20dec138adac9, - nullptr, - bd_dba20dec138adac9, - 0, - sizeof(bd_dba20dec138adac9) / sizeof(bd_dba20dec138adac9[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<155> b_ff14003c70494585 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 133, 69, 73, 112, 60, 0, 20, - 255, 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, - 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, - 97, 121, 80, 97, 114, 116, 105, 116, 105, 111, 110, 101, 114, 0, 0, - 0, 8, 0, 0, 0, 1, 0, 1, 0, 15, 37, 35, 120, 249, 123, - 107, 248, 9, 0, 0, 0, 114, 0, 0, 0, 33, 66, 114, 136, 114, - 228, 217, 253, 9, 0, 0, 0, 50, 0, 0, 0, 80, 97, 114, 116, - 105, 116, 105, 111, 110, 73, 110, 102, 111, 0, 0, 0, 83, 116, 97, - 116, 101, 0, 0, 0, 32, 0, 0, 0, 3, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 3, 0, 1, - 0, 220, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 217, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 212, 0, 0, 0, 3, 0, 1, 0, 240, 0, 0, - 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, - 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, 244, 0, 0, 0, 2, 0, 1, - 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, - 3, 0, 1, 0, 248, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 106, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 244, 0, 0, 0, 3, 0, 1, 0, - 0, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 253, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 252, 0, 0, 0, 3, 0, 1, 0, 8, 1, 0, 0, - 2, 0, 1, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, - 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, - 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, - 1, 0, 0, 3, 0, 1, 0, 20, 1, 0, 0, 2, 0, 1, 0, - 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 1, 0, 0, 50, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 3, - 0, 1, 0, 24, 1, 0, 0, 2, 0, 1, 0, 115, 117, 98, 97, - 114, 114, 97, 121, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 201, 218, 138, 19, 236, 13, 162, 219, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 98, 117, 100, 103, 101, 116, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 45, 205, 230, 7, - 27, 146, 225, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, - 117, 114, 114, 101, 110, 116, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 15, 37, 35, 120, 249, 123, 107, 248, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 115, 116, 97, 116, 101, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 33, 66, 114, 136, 114, 228, 217, 253, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 109, 101, 109, 111, 114, 121, 66, 117, - 100, 103, 101, 116, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 109, 101, 109, 111, 114, 121, 66, 117, 100, 103, 101, - 116, 86, 97, 114, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 109, 101, 109, 111, 114, 121, 66, 117, 100, 103, 101, 116, 86, 97, - 108, 105, 100, 105, 116, 121, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 115, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 164, 161, 6, 21, 161, 54, 224, 199, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xdba20dec138adac9, b_dba20dec138adac9.words, 158, d_dba20dec138adac9, m_dba20dec138adac9, + 4, 7, i_dba20dec138adac9, nullptr, nullptr, { &s_dba20dec138adac9, nullptr, bd_dba20dec138adac9, 0, sizeof(bd_dba20dec138adac9) / sizeof(bd_dba20dec138adac9[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<155> b_ff14003c70494585 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 133, 69, 73, 112, 60, 0, 20, 255, + 18, 0, 0, 0, 1, 0, 3, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 0, 0, 0, 199, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 117, 98, 97, 114, 114, + 97, 121, 80, 97, 114, 116, 105, 116, + 105, 111, 110, 101, 114, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 15, 37, 35, 120, 249, 123, 107, 248, + 9, 0, 0, 0, 114, 0, 0, 0, + 33, 66, 114, 136, 114, 228, 217, 253, + 9, 0, 0, 0, 50, 0, 0, 0, + 80, 97, 114, 116, 105, 116, 105, 111, + 110, 73, 110, 102, 111, 0, 0, 0, + 83, 116, 97, 116, 101, 0, 0, 0, + 32, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 244, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 245, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 253, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 1, 0, 0, 3, 0, 1, 0, + 20, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 117, 100, 103, 101, 116, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 45, 205, 230, 7, 27, 146, 225, 155, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 117, 114, 114, 101, 110, 116, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 15, 37, 35, 120, 249, 123, 107, 248, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 101, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 33, 66, 114, 136, 114, 228, 217, 253, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 109, 111, 114, 121, 66, 117, + 100, 103, 101, 116, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 109, 111, 114, 121, 66, 117, + 100, 103, 101, 116, 86, 97, 114, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 109, 111, 114, 121, 66, 117, + 100, 103, 101, 116, 86, 97, 108, 105, + 100, 105, 116, 121, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_ff14003c70494585 = b_ff14003c70494585.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_ff14003c70494585[] = { - &s_9be1921b07e6cd2d, - &s_c7e036a11506a1a4, - &s_dba20dec138adac9, - &s_f86b7bf97823250f, - &s_fdd9e47288724221, + &s_9be1921b07e6cd2d, + &s_c7e036a11506a1a4, + &s_dba20dec138adac9, + &s_f86b7bf97823250f, + &s_fdd9e47288724221, }; static const uint16_t m_ff14003c70494585[] = {1, 2, 4, 6, 5, 3, 7, 0}; static const uint16_t i_ff14003c70494585[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_ff14003c70494585 = { - 0xff14003c70494585, - b_ff14003c70494585.words, - 155, - d_ff14003c70494585, - m_ff14003c70494585, - 5, - 8, - i_ff14003c70494585, - nullptr, - nullptr, - {&s_ff14003c70494585, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_f86b7bf97823250f = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 15, 37, 35, 120, 249, 123, 107, - 248, 38, 0, 0, 0, 1, 0, 3, 0, 133, 69, 73, 112, 60, 0, - 20, 255, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 162, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, - 97, 121, 80, 97, 114, 116, 105, 116, 105, 111, 110, 101, 114, 46, 80, - 97, 114, 116, 105, 116, 105, 111, 110, 73, 110, 102, 111, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 16, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, - 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, - 0, 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 50, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 3, 0, - 1, 0, 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 109, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 104, 0, 0, 0, 3, 0, 1, 0, 116, 0, - 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 128, 0, 0, 0, 0, - 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 112, 0, 0, 0, 3, 0, 1, 0, 124, 0, 0, 0, 2, 0, - 1, 0, 115, 117, 98, 97, 114, 114, 97, 121, 0, 0, 0, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 201, 218, 138, 19, - 236, 13, 162, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, - 116, 97, 114, 116, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 101, 110, 100, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 115, 112, 108, 105, 116, 77, 117, 108, - 116, 105, 82, 97, 110, 103, 101, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xff14003c70494585, b_ff14003c70494585.words, 155, d_ff14003c70494585, m_ff14003c70494585, + 5, 8, i_ff14003c70494585, nullptr, nullptr, { &s_ff14003c70494585, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<83> b_f86b7bf97823250f = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 15, 37, 35, 120, 249, 123, 107, 248, + 38, 0, 0, 0, 1, 0, 3, 0, + 133, 69, 73, 112, 60, 0, 20, 255, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 117, 98, 97, 114, 114, + 97, 121, 80, 97, 114, 116, 105, 116, + 105, 111, 110, 101, 114, 46, 80, 97, + 114, 116, 105, 116, 105, 111, 110, 73, + 110, 102, 111, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 128, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 114, 116, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 100, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 112, 108, 105, 116, 77, 117, 108, + 116, 105, 82, 97, 110, 103, 101, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_f86b7bf97823250f = b_f86b7bf97823250f.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_f86b7bf97823250f[] = { - &s_dba20dec138adac9, + &s_dba20dec138adac9, }; static const uint16_t m_f86b7bf97823250f[] = {2, 3, 1, 0}; static const uint16_t i_f86b7bf97823250f[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_f86b7bf97823250f = { - 0xf86b7bf97823250f, - b_f86b7bf97823250f.words, - 83, - d_f86b7bf97823250f, - m_f86b7bf97823250f, - 1, - 4, - i_f86b7bf97823250f, - nullptr, - nullptr, - {&s_f86b7bf97823250f, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<90> b_fdd9e47288724221 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 33, 66, 114, 136, 114, 228, 217, - 253, 38, 0, 0, 0, 1, 0, 2, 0, 133, 69, 73, 112, 60, 0, - 20, 255, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, - 97, 121, 80, 97, 114, 116, 105, 116, 105, 111, 110, 101, 114, 46, 83, - 116, 97, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 97, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 92, 0, 0, 0, 3, 0, 1, 0, 104, 0, - 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 101, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 96, 0, 0, 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, - 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, - 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, - 0, 3, 0, 1, 0, 132, 0, 0, 0, 2, 0, 1, 0, 3, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 129, 0, 0, 0, 90, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 3, 0, 1, - 0, 156, 0, 0, 0, 2, 0, 1, 0, 115, 116, 97, 114, 116, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 110, - 100, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 115, 105, 110, 103, 108, 101, 82, 97, 110, 103, 101, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 201, 218, - 138, 19, 236, 13, 162, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 109, 117, 108, 116, 105, 82, 97, 110, 103, 101, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 201, 218, 138, - 19, 236, 13, 162, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xf86b7bf97823250f, b_f86b7bf97823250f.words, 83, d_f86b7bf97823250f, m_f86b7bf97823250f, + 1, 4, i_f86b7bf97823250f, nullptr, nullptr, { &s_f86b7bf97823250f, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<90> b_fdd9e47288724221 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 33, 66, 114, 136, 114, 228, 217, 253, + 38, 0, 0, 0, 1, 0, 2, 0, + 133, 69, 73, 112, 60, 0, 20, 255, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 98, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 117, 98, 97, 114, 114, + 97, 121, 80, 97, 114, 116, 105, 116, + 105, 111, 110, 101, 114, 46, 83, 116, + 97, 116, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, + 115, 116, 97, 114, 116, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 100, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 110, 103, 108, 101, 82, 97, + 110, 103, 101, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 117, 108, 116, 105, 82, 97, 110, + 103, 101, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_fdd9e47288724221 = b_fdd9e47288724221.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_fdd9e47288724221[] = { - &s_dba20dec138adac9, + &s_dba20dec138adac9, }; static const uint16_t m_fdd9e47288724221[] = {1, 3, 2, 0}; static const uint16_t i_fdd9e47288724221[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_fdd9e47288724221 = { - 0xfdd9e47288724221, - b_fdd9e47288724221.words, - 90, - d_fdd9e47288724221, - m_fdd9e47288724221, - 1, - 4, - i_fdd9e47288724221, - nullptr, - nullptr, - {&s_fdd9e47288724221, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_cbe1e7c13508aa2c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 44, 170, 8, 53, 193, 231, 225, - 203, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, - 97, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 97, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 96, 0, 0, 0, 3, 0, 1, 0, 108, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, - 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 0, 0, 0, 3, 0, 1, 0, 116, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 98, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, - 3, 0, 1, 0, 124, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, 162, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 3, 0, 1, 0, - 136, 0, 0, 0, 2, 0, 1, 0, 111, 118, 101, 114, 102, 108, 111, - 119, 101, 100, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 110, 115, 112, 108, 105, 116, 116, 97, 98, - 108, 101, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 105, 110, 105, 116, 105, 97, 108, 105, 122, 101, 100, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, - 117, 98, 97, 114, 114, 97, 121, 80, 97, 114, 116, 105, 116, 105, 111, - 110, 101, 114, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 133, 69, 73, 112, 60, 0, 20, 255, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xfdd9e47288724221, b_fdd9e47288724221.words, 90, d_fdd9e47288724221, m_fdd9e47288724221, + 1, 4, i_fdd9e47288724221, nullptr, nullptr, { &s_fdd9e47288724221, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<83> b_cbe1e7c13508aa2c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 44, 170, 8, 53, 193, 231, 225, 203, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 82, 101, 97, 100, 83, 116, + 97, 116, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 111, 118, 101, 114, 102, 108, 111, 119, + 101, 100, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 110, 115, 112, 108, 105, 116, 116, + 97, 98, 108, 101, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 105, 116, 105, 97, 108, 105, + 122, 101, 100, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 80, 97, 114, 116, 105, 116, 105, 111, + 110, 101, 114, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 133, 69, 73, 112, 60, 0, 20, 255, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_cbe1e7c13508aa2c = b_cbe1e7c13508aa2c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_cbe1e7c13508aa2c[] = { - &s_ff14003c70494585, + &s_ff14003c70494585, }; static const uint16_t m_cbe1e7c13508aa2c[] = {2, 0, 3, 1}; static const uint16_t i_cbe1e7c13508aa2c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { - 0xcbe1e7c13508aa2c, - b_cbe1e7c13508aa2c.words, - 83, - d_cbe1e7c13508aa2c, - m_cbe1e7c13508aa2c, - 1, - 4, - i_cbe1e7c13508aa2c, - nullptr, - nullptr, - {&s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, - 218, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, - 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 16, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, - 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, - 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 3, 0, 1, - 0, 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 104, 0, 0, 0, 3, 0, 1, 0, 116, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, - 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 112, 0, 0, 0, 3, 0, 1, 0, 124, 0, 0, 0, 2, 0, 1, - 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 111, 112, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 117, 115, 101, 69, 110, 117, 109, 101, 114, - 97, 116, 105, 111, 110, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -}}; + 0xcbe1e7c13508aa2c, b_cbe1e7c13508aa2c.words, 83, d_cbe1e7c13508aa2c, m_cbe1e7c13508aa2c, + 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 9, 116, 197, 117, 246, 167, 198, 218, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 110, 100, 105, 116, + 105, 111, 110, 67, 108, 97, 117, 115, + 101, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 102, 105, 101, 108, 100, 78, 97, 109, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 112, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, - b_dac6a7f675c57409.words, - 81, - nullptr, - m_dac6a7f675c57409, - 0, - 4, - i_dac6a7f675c57409, - nullptr, - nullptr, - {&s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<147> b_afc739d5c01e6496 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, - 175, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 83, 84, 78, 111, 100, - 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 32, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 209, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 208, 0, 0, 0, 3, 0, 1, 0, 220, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 217, - 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, 228, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 0, 0, 0, 50, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 220, 0, 0, 0, - 3, 0, 1, 0, 232, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 229, 0, 0, 0, 26, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 233, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 232, 0, 0, 0, 3, 0, 1, 0, 4, 1, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, - 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 3, 0, 1, 0, 12, 1, 0, 0, 2, 0, 1, 0, - 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 122, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 3, - 0, 1, 0, 20, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, - 5, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 17, 1, 0, 0, 66, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 1, 0, 0, 3, 0, 1, 0, 24, - 1, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, - 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, - 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 118, 97, 108, 117, 101, 0, 0, 0, 13, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 111, 112, 0, 0, 0, 0, 0, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 104, 105, 108, 100, 114, - 101, 110, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 150, 100, 30, 192, 213, 57, 199, 175, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, 109, 98, 105, 110, 97, - 116, 105, 111, 110, 79, 112, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 115, 101, 69, 110, 117, 109, 101, 114, 97, - 116, 105, 111, 110, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 111, 102, 102, 115, 101, 116, 115, 0, 13, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -}}; + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<147> b_afc739d5c01e6496 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 150, 100, 30, 192, 213, 57, 199, 175, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 6, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 210, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 199, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 83, 84, 78, 111, 100, + 101, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 32, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 216, 0, 0, 0, 3, 0, 1, 0, + 228, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 225, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 229, 0, 0, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 224, 0, 0, 0, 3, 0, 1, 0, + 236, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 233, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 3, 0, 1, 0, + 12, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 1, 0, 0, 3, 0, 1, 0, + 20, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 1, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, + 105, 115, 69, 120, 112, 114, 101, 115, + 115, 105, 111, 110, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 101, 108, 100, 78, 97, 109, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 112, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 104, 105, 108, 100, 114, 101, 110, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 150, 100, 30, 192, 213, 57, 199, 175, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 98, 105, 110, 97, 116, + 105, 111, 110, 79, 112, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_afc739d5c01e6496 = b_afc739d5c01e6496.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { - &s_afc739d5c01e6496, + &s_afc739d5c01e6496, }; static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 7, 3, 6, 2}; static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, - b_afc739d5c01e6496.words, - 147, - d_afc739d5c01e6496, - m_afc739d5c01e6496, - 1, - 8, - i_afc739d5c01e6496, - nullptr, - nullptr, - {&s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 111, 192, 31, 135, 185, 124, 245, - 234, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, - 105, 111, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 12, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 69, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 64, 0, 0, 0, 3, 0, 1, 0, 92, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, - 0, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 92, 0, 0, 0, 3, 0, 1, 0, 120, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 42, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, - 3, 0, 1, 0, 124, 0, 0, 0, 2, 0, 1, 0, 99, 108, 97, - 117, 115, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 9, 116, 197, 117, 246, 167, 198, 218, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 99, 108, 97, 117, 115, 101, 67, 111, 109, 98, 105, 110, - 97, 116, 105, 111, 110, 79, 112, 115, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 114, 101, 101, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 150, 100, 30, 192, - 213, 57, 199, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 147, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 8, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 110, 100, 105, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 0, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 99, 108, 97, 117, 115, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 9, 116, 197, 117, 246, 167, 198, 218, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 108, 97, 117, 115, 101, 67, 111, + 109, 98, 105, 110, 97, 116, 105, 111, + 110, 79, 112, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 114, 101, 101, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 150, 100, 30, 192, 213, 57, 199, 175, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_eaf57cb9871fc06f = b_eaf57cb9871fc06f.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_eaf57cb9871fc06f[] = { - &s_afc739d5c01e6496, - &s_dac6a7f675c57409, + &s_afc739d5c01e6496, + &s_dac6a7f675c57409, }; static const uint16_t m_eaf57cb9871fc06f[] = {1, 0, 2}; static const uint16_t i_eaf57cb9871fc06f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_eaf57cb9871fc06f = { - 0xeaf57cb9871fc06f, - b_eaf57cb9871fc06f.words, - 73, - d_eaf57cb9871fc06f, - m_eaf57cb9871fc06f, - 2, - 3, - i_eaf57cb9871fc06f, - nullptr, - nullptr, - {&s_eaf57cb9871fc06f, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<113> b_e19754f813ccf79c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 156, 247, 204, 19, 248, 84, 151, - 225, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 82, - 101, 97, 100, 101, 114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 24, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 153, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 148, 0, 0, 0, 3, 0, 1, 0, 160, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, - 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 0, 0, 0, 3, 0, 1, 0, 168, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 82, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, - 3, 0, 1, 0, 176, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 82, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 3, 0, 1, 0, - 184, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 181, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 176, 0, 0, 0, 3, 0, 1, 0, 188, 0, 0, 0, - 2, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, - 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, - 0, 0, 0, 3, 0, 1, 0, 200, 0, 0, 0, 2, 0, 1, 0, - 108, 97, 121, 111, 117, 116, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 115, 117, 98, 97, 114, 114, 97, 121, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 201, 218, - 138, 19, 236, 13, 162, 219, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 114, 101, 97, 100, 83, 116, 97, 116, 101, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 44, 170, 8, 53, 193, - 231, 225, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, - 110, 100, 105, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 111, 192, 31, 135, 185, 124, 245, 234, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 115, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 164, 161, 6, 21, - 161, 54, 224, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, - 105, 109, 76, 97, 98, 101, 108, 73, 110, 99, 114, 101, 97, 115, 105, - 110, 103, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xeaf57cb9871fc06f, b_eaf57cb9871fc06f.words, 73, d_eaf57cb9871fc06f, m_eaf57cb9871fc06f, + 2, 3, i_eaf57cb9871fc06f, nullptr, nullptr, { &s_eaf57cb9871fc06f, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<113> b_e19754f813ccf79c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 156, 247, 204, 19, 248, 84, 151, 225, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 87, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 81, 117, 101, 114, 121, 82, + 101, 97, 100, 101, 114, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 24, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 160, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 0, 0, 0, 3, 0, 1, 0, + 168, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 0, 0, 0, 3, 0, 1, 0, + 176, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 0, 0, 0, 3, 0, 1, 0, + 184, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 108, 97, 121, 111, 117, 116, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 97, 100, 83, 116, 97, 116, + 101, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 44, 170, 8, 53, 193, 231, 225, 203, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 105, 109, 76, 97, 98, 101, 108, + 73, 110, 99, 114, 101, 97, 115, 105, + 110, 103, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_e19754f813ccf79c = b_e19754f813ccf79c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_e19754f813ccf79c[] = { - &s_c7e036a11506a1a4, - &s_cbe1e7c13508aa2c, - &s_dba20dec138adac9, - &s_eaf57cb9871fc06f, + &s_c7e036a11506a1a4, + &s_cbe1e7c13508aa2c, + &s_dba20dec138adac9, + &s_eaf57cb9871fc06f, }; static const uint16_t m_e19754f813ccf79c[] = {3, 5, 0, 2, 4, 1}; static const uint16_t i_e19754f813ccf79c[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_e19754f813ccf79c = { - 0xe19754f813ccf79c, - b_e19754f813ccf79c.words, - 113, - d_e19754f813ccf79c, - m_e19754f813ccf79c, - 4, - 6, - i_e19754f813ccf79c, - nullptr, - nullptr, - {&s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 108, 49, 93, 21, 237, 95, 79, - 215, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 202, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 101, 108, 101, 116, 101, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, 1, 0, 52, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, - 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 0, 0, 0, 3, 0, 1, 0, 56, 0, 0, 0, 2, 0, 1, - 0, 99, 111, 110, 100, 105, 116, 105, 111, 110, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 111, 192, 31, 135, 185, - 124, 245, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, - 97, 116, 115, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 164, - 161, 6, 21, 161, 54, 224, 199, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xe19754f813ccf79c, b_e19754f813ccf79c.words, 113, d_e19754f813ccf79c, m_e19754f813ccf79c, + 4, 6, i_e19754f813ccf79c, nullptr, nullptr, { &s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 101, 108, 101, 116, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d74f5fed155d316c = b_d74f5fed155d316c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_d74f5fed155d316c[] = { - &s_c7e036a11506a1a4, - &s_eaf57cb9871fc06f, + &s_c7e036a11506a1a4, + &s_eaf57cb9871fc06f, }; static const uint16_t m_d74f5fed155d316c[] = {0, 1}; static const uint16_t i_d74f5fed155d316c[] = {0, 1}; const ::capnp::_::RawSchema s_d74f5fed155d316c = { - 0xd74f5fed155d316c, - b_d74f5fed155d316c.words, - 49, - d_d74f5fed155d316c, - m_d74f5fed155d316c, - 2, - 2, - i_d74f5fed155d316c, - nullptr, - nullptr, - {&s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 231, 136, 33, 216, 234, 124, 248, - 222, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 115, 117, 108, 116, - 67, 101, 108, 108, 83, 108, 97, 98, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 16, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, - 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, - 0, 3, 0, 1, 0, 104, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 3, 0, 1, - 0, 108, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 100, 0, 0, 0, 3, 0, 1, 0, 112, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, - 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 0, 0, 0, 3, 0, 1, 0, 116, 0, 0, 0, 2, 0, 1, - 0, 102, 114, 97, 103, 73, 100, 120, 0, 8, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 116, 105, 108, 101, 73, 100, 120, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 114, 116, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 101, - 110, 103, 116, 104, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 49, d_d74f5fed155d316c, m_d74f5fed155d316c, + 2, 2, i_d74f5fed155d316c, nullptr, nullptr, { &s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 231, 136, 33, 216, 234, 124, 248, 222, + 18, 0, 0, 0, 1, 0, 4, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 82, 101, 115, 117, 108, 116, + 67, 101, 108, 108, 83, 108, 97, 98, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 102, 114, 97, 103, 73, 100, 120, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 73, 100, 120, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 114, 116, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 101, 110, 103, 116, 104, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_def87cead82188e7 = b_def87cead82188e7.words; #if !CAPNP_LITE static const uint16_t m_def87cead82188e7[] = {0, 3, 2, 1}; static const uint16_t i_def87cead82188e7[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_def87cead82188e7 = { - 0xdef87cead82188e7, - b_def87cead82188e7.words, - 79, - nullptr, - m_def87cead82188e7, - 0, - 4, - i_def87cead82188e7, - nullptr, - nullptr, - {&s_def87cead82188e7, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_c1a2d010de779de5 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 229, 157, 119, 222, 16, 208, 162, - 193, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, - 110, 116, 73, 110, 100, 101, 120, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 0, 0, 0, 3, 0, 1, 0, 52, 0, 0, 0, 2, 0, 1, - 0, 116, 105, 108, 101, 73, 100, 120, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 99, 101, 108, 108, 73, 100, 120, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xdef87cead82188e7, b_def87cead82188e7.words, 79, nullptr, m_def87cead82188e7, + 0, 4, i_def87cead82188e7, nullptr, nullptr, { &s_def87cead82188e7, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<48> b_c1a2d010de779de5 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 229, 157, 119, 222, 16, 208, 162, 193, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 114, 97, 103, 109, 101, + 110, 116, 73, 110, 100, 101, 120, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 116, 105, 108, 101, 73, 100, 120, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 73, 100, 120, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c1a2d010de779de5 = b_c1a2d010de779de5.words; #if !CAPNP_LITE static const uint16_t m_c1a2d010de779de5[] = {1, 0}; static const uint16_t i_c1a2d010de779de5[] = {0, 1}; const ::capnp::_::RawSchema s_c1a2d010de779de5 = { - 0xc1a2d010de779de5, - b_c1a2d010de779de5.words, - 48, - nullptr, - m_c1a2d010de779de5, - 0, - 2, - i_c1a2d010de779de5, - nullptr, - nullptr, - {&s_c1a2d010de779de5, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<76> b_c86c77b5f6a2bf0f = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 15, 191, 162, 246, 181, 119, 108, - 200, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, - 97, 116, 101, 73, 110, 100, 101, 120, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, - 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, - 0, 3, 0, 1, 0, 96, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 3, 0, 1, - 0, 120, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 117, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 0, 0, 0, 3, 0, 1, 0, 132, 0, 0, - 0, 2, 0, 1, 0, 114, 101, 115, 117, 108, 116, 67, 101, 108, 108, - 83, 108, 97, 98, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 231, 136, 33, 216, 234, 124, 248, 222, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 102, 114, 97, 103, 84, 105, 108, 101, 73, 100, 120, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 229, 157, 119, 222, 16, 208, 162, 193, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 100, 111, 110, 101, 65, 100, 100, 105, 110, 103, 82, 101, - 115, 117, 108, 116, 84, 105, 108, 101, 115, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xc1a2d010de779de5, b_c1a2d010de779de5.words, 48, nullptr, m_c1a2d010de779de5, + 0, 2, i_c1a2d010de779de5, nullptr, nullptr, { &s_c1a2d010de779de5, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<76> b_c86c77b5f6a2bf0f = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 15, 191, 162, 246, 181, 119, 108, 200, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 82, 101, 97, 100, 83, 116, + 97, 116, 101, 73, 110, 100, 101, 120, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 96, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 0, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 114, 101, 115, 117, 108, 116, 67, 101, + 108, 108, 83, 108, 97, 98, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 231, 136, 33, 216, 234, 124, 248, 222, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 84, 105, 108, 101, + 73, 100, 120, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 229, 157, 119, 222, 16, 208, 162, 193, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 111, 110, 101, 65, 100, 100, 105, + 110, 103, 82, 101, 115, 117, 108, 116, + 84, 105, 108, 101, 115, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c86c77b5f6a2bf0f = b_c86c77b5f6a2bf0f.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c86c77b5f6a2bf0f[] = { - &s_c1a2d010de779de5, - &s_def87cead82188e7, + &s_c1a2d010de779de5, + &s_def87cead82188e7, }; static const uint16_t m_c86c77b5f6a2bf0f[] = {2, 1, 0}; static const uint16_t i_c86c77b5f6a2bf0f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_c86c77b5f6a2bf0f = { - 0xc86c77b5f6a2bf0f, - b_c86c77b5f6a2bf0f.words, - 76, - d_c86c77b5f6a2bf0f, - m_c86c77b5f6a2bf0f, - 2, - 3, - i_c86c77b5f6a2bf0f, - nullptr, - nullptr, - {&s_c86c77b5f6a2bf0f, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<96> b_9b9a5fc7713a8692 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 146, 134, 58, 113, 199, 95, 154, - 155, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 101, 114, - 73, 110, 100, 101, 120, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 125, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 120, 0, 0, 0, 3, 0, 1, 0, 132, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, - 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 0, 0, 0, 3, 0, 1, 0, 140, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 0, 82, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, - 3, 0, 1, 0, 148, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 82, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 3, 0, 1, 0, - 156, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, - 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 153, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 148, 0, 0, 0, 3, 0, 1, 0, 160, 0, 0, 0, - 2, 0, 1, 0, 108, 97, 121, 111, 117, 116, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 115, 117, 98, 97, 114, 114, 97, - 121, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 201, 218, 138, 19, 236, 13, 162, 219, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 114, 101, 97, 100, 83, 116, 97, 116, 101, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 15, - 191, 162, 246, 181, 119, 108, 200, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 99, 111, 110, 100, 105, 116, 105, 111, 110, 0, 0, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 111, 192, 31, 135, - 185, 124, 245, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, - 116, 97, 116, 115, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 164, 161, 6, 21, 161, 54, 224, 199, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -}}; + 0xc86c77b5f6a2bf0f, b_c86c77b5f6a2bf0f.words, 76, d_c86c77b5f6a2bf0f, m_c86c77b5f6a2bf0f, + 2, 3, i_c86c77b5f6a2bf0f, nullptr, nullptr, { &s_c86c77b5f6a2bf0f, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<96> b_9b9a5fc7713a8692 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 146, 134, 58, 113, 199, 95, 154, 155, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 82, 101, 97, 100, 101, 114, + 73, 110, 100, 101, 120, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 0, 0, 0, 3, 0, 1, 0, + 140, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 0, 0, 0, 3, 0, 1, 0, + 148, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 0, 0, 0, 3, 0, 1, 0, + 156, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 153, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 160, 0, 0, 0, 2, 0, 1, 0, + 108, 97, 121, 111, 117, 116, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 117, 98, 97, 114, 114, 97, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 201, 218, 138, 19, 236, 13, 162, 219, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 97, 100, 83, 116, 97, 116, + 101, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 15, 191, 162, 246, 181, 119, 108, 200, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9b9a5fc7713a8692 = b_9b9a5fc7713a8692.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9b9a5fc7713a8692[] = { - &s_c7e036a11506a1a4, - &s_c86c77b5f6a2bf0f, - &s_dba20dec138adac9, - &s_eaf57cb9871fc06f, + &s_c7e036a11506a1a4, + &s_c86c77b5f6a2bf0f, + &s_dba20dec138adac9, + &s_eaf57cb9871fc06f, }; static const uint16_t m_9b9a5fc7713a8692[] = {3, 0, 2, 4, 1}; static const uint16_t i_9b9a5fc7713a8692[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { - 0x9b9a5fc7713a8692, - b_9b9a5fc7713a8692.words, - 96, - d_9b9a5fc7713a8692, - m_9b9a5fc7713a8692, - 4, - 5, - i_9b9a5fc7713a8692, - nullptr, - nullptr, - {&s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<367> b_96ba49d0f8b23ccc = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, - 150, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 16, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, - 0, 159, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, - 0, 0, 0, 0, 1, 0, 1, 0, 84, 0, 0, 0, 3, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 2, 0, 0, 186, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 2, 0, 0, - 3, 0, 1, 0, 92, 2, 0, 0, 2, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 89, 2, 0, 0, 58, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 84, 2, 0, 0, 3, 0, 1, 0, - 96, 2, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, - 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 93, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 88, 2, 0, 0, 3, 0, 1, 0, 100, 2, 0, 0, - 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, - 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 2, - 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, - 2, 0, 0, 3, 0, 1, 0, 104, 2, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 2, 0, 0, 58, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 2, 0, 0, 3, - 0, 1, 0, 108, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, - 5, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 105, 2, 0, 0, 58, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 100, 2, 0, 0, 3, 0, 1, 0, 112, - 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 6, 0, 0, 0, - 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 109, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 104, 2, 0, 0, 3, 0, 1, 0, 116, 2, 0, 0, 2, - 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 2, 0, - 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 2, - 0, 0, 3, 0, 1, 0, 132, 2, 0, 0, 2, 0, 1, 0, 8, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 129, 2, 0, 0, 186, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 2, 0, 0, 3, 0, - 1, 0, 144, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 141, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 148, 2, 0, 0, 3, 0, 1, 0, 160, 2, - 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, - 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 156, 2, 0, 0, 3, 0, 1, 0, 168, 2, 0, 0, 2, 0, - 1, 0, 11, 0, 0, 0, 192, 0, 0, 0, 0, 0, 1, 0, 11, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 2, 0, 0, - 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 2, 0, - 0, 3, 0, 1, 0, 184, 2, 0, 0, 2, 0, 1, 0, 12, 0, - 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 181, 2, 0, 0, 146, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 184, 2, 0, 0, 3, 0, 1, - 0, 196, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 8, 0, - 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 193, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 188, 2, 0, 0, 3, 0, 1, 0, 200, 2, 0, - 0, 2, 0, 1, 0, 14, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, - 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 192, 2, 0, 0, 3, 0, 1, 0, 204, 2, 0, 0, 2, 0, 1, - 0, 15, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 15, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 2, 0, 0, 98, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 2, 0, 0, - 3, 0, 1, 0, 212, 2, 0, 0, 2, 0, 1, 0, 16, 0, 0, - 0, 11, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 209, 2, 0, 0, 98, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 208, 2, 0, 0, 3, 0, 1, 0, - 220, 2, 0, 0, 2, 0, 1, 0, 17, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 217, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 212, 2, 0, 0, 3, 0, 1, 0, 224, 2, 0, 0, - 2, 0, 1, 0, 18, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, - 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 221, 2, - 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, - 2, 0, 0, 3, 0, 1, 0, 252, 2, 0, 0, 2, 0, 1, 0, - 19, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 2, 0, 0, 122, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 2, 0, 0, 3, - 0, 1, 0, 20, 3, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, - 15, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 17, 3, 0, 0, 178, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 20, 3, 0, 0, 3, 0, 1, 0, 32, - 3, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, - 101, 66, 117, 102, 102, 101, 114, 72, 101, 97, 100, 101, 114, 115, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 162, 146, 250, 18, - 129, 87, 10, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, - 97, 121, 111, 117, 116, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 115, 116, 97, 116, 117, 115, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 114, 105, 116, - 101, 114, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 94, 115, 230, - 160, 71, 1, 166, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 114, 101, 97, 100, 101, 114, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 156, 247, 204, 19, 248, 84, 151, 225, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 97, 114, 114, 97, 121, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 180, 96, 4, 126, 245, 48, 87, 164, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 116, 111, 116, 97, 108, 70, 105, - 120, 101, 100, 76, 101, 110, 103, 116, 104, 66, 117, 102, 102, 101, 114, - 66, 121, 116, 101, 115, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 111, 116, 97, 108, 86, 97, 114, 76, - 101, 110, 66, 117, 102, 102, 101, 114, 66, 121, 116, 101, 115, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 111, 116, 97, - 108, 86, 97, 108, 105, 100, 105, 116, 121, 66, 117, 102, 102, 101, 114, - 66, 121, 116, 101, 115, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, 114, 79, 102, 102, - 115, 101, 116, 115, 77, 111, 100, 101, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 118, 97, 114, 79, 102, 102, 115, 101, 116, - 115, 65, 100, 100, 69, 120, 116, 114, 97, 69, 108, 101, 109, 101, 110, - 116, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 118, 97, 114, 79, 102, 102, 115, 101, 116, 115, 66, - 105, 116, 115, 105, 122, 101, 0, 0, 0, 0, 0, 0, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, 110, 102, 105, 103, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, 129, 75, - 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, - 97, 116, 115, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 164, - 161, 6, 21, 161, 54, 224, 199, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 114, 101, 97, 100, 101, 114, 73, 110, 100, 101, 120, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 146, 134, 58, 113, - 199, 95, 154, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, - 101, 110, 115, 101, 82, 101, 97, 100, 101, 114, 0, 0, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 156, 247, 204, 19, 248, 84, 151, - 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 101, 108, 101, - 116, 101, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 108, 49, 93, - 21, 237, 95, 79, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 119, 114, 105, 116, 116, 101, 110, 70, 114, 97, 103, 109, 101, 110, 116, - 73, 110, 102, 111, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 82, 167, 44, 41, 29, 197, 54, 167, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 119, 114, 105, 116, 116, 101, 110, 66, - 117, 102, 102, 101, 114, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 101, 100, 68, 105, - 109, 76, 97, 98, 101, 108, 82, 101, 97, 100, 101, 114, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 156, 247, 204, 19, 248, 84, 151, - 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x9b9a5fc7713a8692, b_9b9a5fc7713a8692.words, 96, d_9b9a5fc7713a8692, m_9b9a5fc7713a8692, + 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<367> b_96ba49d0f8b23ccc = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 204, 60, 178, 248, 208, 73, 186, 150, + 18, 0, 0, 0, 1, 0, 4, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 16, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 25, 0, 0, 0, 159, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 81, 117, 101, 114, 121, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 84, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 2, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 2, 0, 0, 3, 0, 1, 0, + 92, 2, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 96, 2, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 2, 0, 0, 3, 0, 1, 0, + 100, 2, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 2, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 2, 0, 0, 3, 0, 1, 0, + 104, 2, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 2, 0, 0, 3, 0, 1, 0, + 108, 2, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 2, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 2, 0, 0, 3, 0, 1, 0, + 116, 2, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 2, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 2, 0, 0, 3, 0, 1, 0, + 132, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 2, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 144, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 2, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 156, 2, 0, 0, 3, 0, 1, 0, + 168, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 192, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 2, 0, 0, 210, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 2, 0, 0, 3, 0, 1, 0, + 184, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 2, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 2, 0, 0, 3, 0, 1, 0, + 196, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 2, 0, 0, 3, 0, 1, 0, + 200, 2, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 2, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 192, 2, 0, 0, 3, 0, 1, 0, + 204, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 2, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 2, 0, 0, 3, 0, 1, 0, + 212, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 2, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 2, 0, 0, 3, 0, 1, 0, + 220, 2, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 217, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 2, 0, 0, 3, 0, 1, 0, + 224, 2, 0, 0, 2, 0, 1, 0, + 18, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 18, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 221, 2, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 224, 2, 0, 0, 3, 0, 1, 0, + 252, 2, 0, 0, 2, 0, 1, 0, + 19, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 249, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 248, 2, 0, 0, 3, 0, 1, 0, + 20, 3, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 1, 0, 20, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 3, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 3, 0, 0, 3, 0, 1, 0, + 32, 3, 0, 0, 2, 0, 1, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 66, 117, 102, 102, 101, 114, 72, + 101, 97, 100, 101, 114, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 162, 146, 250, 18, 129, 87, 10, 210, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 121, 111, 117, 116, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 117, 115, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 119, 114, 105, 116, 101, 114, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 94, 115, 230, 160, 71, 1, 166, 139, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 97, 100, 101, 114, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 156, 247, 204, 19, 248, 84, 151, 225, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 96, 4, 126, 245, 48, 87, 164, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 111, 116, 97, 108, 70, 105, 120, + 101, 100, 76, 101, 110, 103, 116, 104, + 66, 117, 102, 102, 101, 114, 66, 121, + 116, 101, 115, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 111, 116, 97, 108, 86, 97, 114, + 76, 101, 110, 66, 117, 102, 102, 101, + 114, 66, 121, 116, 101, 115, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 111, 116, 97, 108, 86, 97, 108, + 105, 100, 105, 116, 121, 66, 117, 102, + 102, 101, 114, 66, 121, 116, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 114, 79, 102, 102, 115, 101, + 116, 115, 77, 111, 100, 101, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 114, 79, 102, 102, 115, 101, + 116, 115, 65, 100, 100, 69, 120, 116, + 114, 97, 69, 108, 101, 109, 101, 110, + 116, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 114, 79, 102, 102, 115, 101, + 116, 115, 66, 105, 116, 115, 105, 122, + 101, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 97, 100, 101, 114, 73, 110, + 100, 101, 120, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 146, 134, 58, 113, 199, 95, 154, 155, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 110, 115, 101, 82, 101, 97, + 100, 101, 114, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 156, 247, 204, 19, 248, 84, 151, 225, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 108, 101, 116, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 119, 114, 105, 116, 116, 101, 110, 70, + 114, 97, 103, 109, 101, 110, 116, 73, + 110, 102, 111, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 82, 167, 44, 41, 29, 197, 54, 167, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 119, 114, 105, 116, 116, 101, 110, 66, + 117, 102, 102, 101, 114, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 68, + 105, 109, 76, 97, 98, 101, 108, 82, + 101, 97, 100, 101, 114, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 156, 247, 204, 19, 248, 84, 151, 225, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_96ba49d0f8b23ccc = b_96ba49d0f8b23ccc.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { - &s_8ba60147a0e6735e, - &s_9b9a5fc7713a8692, - &s_a45730f57e0460b4, - &s_a736c51d292ca752, - &s_b6c95b4b8111ad36, - &s_c7e036a11506a1a4, - &s_d20a578112fa92a2, - &s_d74f5fed155d316c, - &s_e19754f813ccf79c, -}; -static const uint16_t m_96ba49d0f8b23ccc[] = { - 6, 0, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; -static const uint16_t i_96ba49d0f8b23ccc[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}; + &s_8ba60147a0e6735e, + &s_9b9a5fc7713a8692, + &s_a45730f57e0460b4, + &s_a736c51d292ca752, + &s_b6c95b4b8111ad36, + &s_c7e036a11506a1a4, + &s_d20a578112fa92a2, + &s_d74f5fed155d316c, + &s_e19754f813ccf79c, +}; +static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; +static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, - b_96ba49d0f8b23ccc.words, - 367, - d_96ba49d0f8b23ccc, - m_96ba49d0f8b23ccc, - 9, - 21, - i_96ba49d0f8b23ccc, - nullptr, - nullptr, - {&s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 11, 95, 78, 44, 164, 242, 246, - 157, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, - 116, 121, 68, 111, 109, 97, 105, 110, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, - 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, - 0, 3, 0, 1, 0, 80, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 66, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, 1, - 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 81, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 76, 0, 0, 0, 3, 0, 1, 0, 104, 0, 0, - 0, 2, 0, 1, 0, 110, 111, 110, 69, 109, 112, 116, 121, 68, 111, - 109, 97, 105, 110, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 236, - 12, 65, 249, 230, 4, 89, 206, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 105, 115, 69, 109, 112, 116, 121, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 115, 105, 122, 101, 115, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 367, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 9, 21, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 11, 95, 78, 44, 164, 242, 246, 157, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 78, 111, 110, 69, 109, 112, + 116, 121, 68, 111, 109, 97, 105, 110, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 110, 111, 110, 69, 109, 112, 116, 121, + 68, 111, 109, 97, 105, 110, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 236, 12, 65, 249, 230, 4, 89, 206, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 115, 69, 109, 112, 116, 121, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 115, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9df6f2a42c4e5f0b = b_9df6f2a42c4e5f0b.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9df6f2a42c4e5f0b[] = { - &s_ce5904e6f9410cec, + &s_ce5904e6f9410cec, }; static const uint16_t m_9df6f2a42c4e5f0b[] = {1, 0, 2}; static const uint16_t i_9df6f2a42c4e5f0b[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9df6f2a42c4e5f0b = { - 0x9df6f2a42c4e5f0b, - b_9df6f2a42c4e5f0b.words, - 69, - d_9df6f2a42c4e5f0b, - m_9df6f2a42c4e5f0b, - 1, - 3, - i_9df6f2a42c4e5f0b, - nullptr, - nullptr, - {&s_9df6f2a42c4e5f0b, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_a18264549448ece3 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 227, 236, 72, 148, 84, 100, 130, - 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, - 116, 121, 68, 111, 109, 97, 105, 110, 76, 105, 115, 116, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 3, 0, 1, 0, 40, 0, 0, 0, 2, 0, 1, 0, 110, 111, - 110, 69, 109, 112, 116, 121, 68, 111, 109, 97, 105, 110, 115, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 11, 95, 78, 44, 164, 242, - 246, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x9df6f2a42c4e5f0b, b_9df6f2a42c4e5f0b.words, 69, d_9df6f2a42c4e5f0b, m_9df6f2a42c4e5f0b, + 1, 3, i_9df6f2a42c4e5f0b, nullptr, nullptr, { &s_9df6f2a42c4e5f0b, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<39> b_a18264549448ece3 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 227, 236, 72, 148, 84, 100, 130, 161, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 78, 111, 110, 69, 109, 112, + 116, 121, 68, 111, 109, 97, 105, 110, + 76, 105, 115, 116, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 3, 0, 1, 0, + 40, 0, 0, 0, 2, 0, 1, 0, + 110, 111, 110, 69, 109, 112, 116, 121, + 68, 111, 109, 97, 105, 110, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 11, 95, 78, 44, 164, 242, 246, 157, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a18264549448ece3 = b_a18264549448ece3.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a18264549448ece3[] = { - &s_9df6f2a42c4e5f0b, + &s_9df6f2a42c4e5f0b, }; static const uint16_t m_a18264549448ece3[] = {0}; static const uint16_t i_a18264549448ece3[] = {0}; const ::capnp::_::RawSchema s_a18264549448ece3 = { - 0xa18264549448ece3, - b_a18264549448ece3.words, - 39, - d_a18264549448ece3, - m_a18264549448ece3, - 1, - 1, - i_a18264549448ece3, - nullptr, - nullptr, - {&s_a18264549448ece3, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_9be1921b07e6cd2d = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 45, 205, 230, 7, 27, 146, 225, - 155, 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, - 117, 116, 101, 66, 117, 102, 102, 101, 114, 83, 105, 122, 101, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 16, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, - 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, - 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 98, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 3, 0, 1, - 0, 116, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 113, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 112, 0, 0, 0, 3, 0, 1, 0, 124, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, - 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, 132, 0, 0, 0, 2, 0, 1, - 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 102, - 102, 115, 101, 116, 66, 121, 116, 101, 115, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 97, 116, 97, 66, - 121, 116, 101, 115, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 118, 97, 108, 105, 100, 105, 116, 121, - 66, 121, 116, 101, 115, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xa18264549448ece3, b_a18264549448ece3.words, 39, d_a18264549448ece3, m_a18264549448ece3, + 1, 1, i_a18264549448ece3, nullptr, nullptr, { &s_a18264549448ece3, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<83> b_9be1921b07e6cd2d = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 45, 205, 230, 7, 27, 146, 225, 155, + 18, 0, 0, 0, 1, 0, 3, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 116, 116, 114, 105, 98, + 117, 116, 101, 66, 117, 102, 102, 101, + 114, 83, 105, 122, 101, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 0, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 66, 121, + 116, 101, 115, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 66, 121, 116, 101, + 115, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 105, 100, 105, 116, 121, + 66, 121, 116, 101, 115, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9be1921b07e6cd2d = b_9be1921b07e6cd2d.words; #if !CAPNP_LITE static const uint16_t m_9be1921b07e6cd2d[] = {0, 2, 1, 3}; static const uint16_t i_9be1921b07e6cd2d[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_9be1921b07e6cd2d = { - 0x9be1921b07e6cd2d, - b_9be1921b07e6cd2d.words, - 83, - nullptr, - m_9be1921b07e6cd2d, - 0, - 4, - i_9be1921b07e6cd2d, - nullptr, - nullptr, - {&s_9be1921b07e6cd2d, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_f01116579e9ea98e = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 142, 169, 158, 158, 87, 22, 17, - 240, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 120, 66, 117, 102, - 102, 101, 114, 83, 105, 122, 101, 115, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 3, 0, 1, 0, 40, 0, 0, 0, 2, 0, 1, 0, 109, 97, - 120, 66, 117, 102, 102, 101, 114, 83, 105, 122, 101, 115, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 45, 205, 230, 7, 27, 146, - 225, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x9be1921b07e6cd2d, b_9be1921b07e6cd2d.words, 83, nullptr, m_9be1921b07e6cd2d, + 0, 4, i_9be1921b07e6cd2d, nullptr, nullptr, { &s_9be1921b07e6cd2d, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<39> b_f01116579e9ea98e = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 142, 169, 158, 158, 87, 22, 17, 240, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 97, 120, 66, 117, 102, + 102, 101, 114, 83, 105, 122, 101, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 3, 0, 1, 0, + 40, 0, 0, 0, 2, 0, 1, 0, + 109, 97, 120, 66, 117, 102, 102, 101, + 114, 83, 105, 122, 101, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 45, 205, 230, 7, 27, 146, 225, 155, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_f01116579e9ea98e = b_f01116579e9ea98e.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_f01116579e9ea98e[] = { - &s_9be1921b07e6cd2d, + &s_9be1921b07e6cd2d, }; static const uint16_t m_f01116579e9ea98e[] = {0}; static const uint16_t i_f01116579e9ea98e[] = {0}; const ::capnp::_::RawSchema s_f01116579e9ea98e = { - 0xf01116579e9ea98e, - b_f01116579e9ea98e.words, - 39, - d_f01116579e9ea98e, - m_f01116579e9ea98e, - 1, - 1, - i_f01116579e9ea98e, - nullptr, - nullptr, - {&s_f01116579e9ea98e, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<41> b_9737dcafdfce31bb = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 187, 49, 206, 223, 175, 220, 55, - 151, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, - 101, 116, 97, 100, 97, 116, 97, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 81, 214, 46, 177, 195, 225, 111, 146, 1, 0, 0, 0, 114, 0, - 0, 0, 77, 101, 116, 97, 100, 97, 116, 97, 69, 110, 116, 114, 121, - 0, 0, 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 13, 0, 0, 0, 66, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 36, - 0, 0, 0, 2, 0, 1, 0, 101, 110, 116, 114, 105, 101, 115, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, - 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 81, 214, 46, 177, 195, - 225, 111, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 39, d_f01116579e9ea98e, m_f01116579e9ea98e, + 1, 1, i_f01116579e9ea98e, nullptr, nullptr, { &s_f01116579e9ea98e, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<41> b_9737dcafdfce31bb = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 187, 49, 206, 223, 175, 220, 55, 151, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 77, + 101, 116, 97, 100, 97, 116, 97, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 81, 214, 46, 177, 195, 225, 111, 146, + 1, 0, 0, 0, 114, 0, 0, 0, + 77, 101, 116, 97, 100, 97, 116, 97, + 69, 110, 116, 114, 121, 0, 0, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 36, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 81, 214, 46, 177, 195, 225, 111, 146, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9737dcafdfce31bb = b_9737dcafdfce31bb.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9737dcafdfce31bb[] = { - &s_926fe1c3b12ed651, + &s_926fe1c3b12ed651, }; static const uint16_t m_9737dcafdfce31bb[] = {0}; static const uint16_t i_9737dcafdfce31bb[] = {0}; const ::capnp::_::RawSchema s_9737dcafdfce31bb = { - 0x9737dcafdfce31bb, - b_9737dcafdfce31bb.words, - 41, - d_9737dcafdfce31bb, - m_9737dcafdfce31bb, - 1, - 1, - i_9737dcafdfce31bb, - nullptr, - nullptr, - {&s_9737dcafdfce31bb, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<96> b_926fe1c3b12ed651 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 81, 214, 46, 177, 195, 225, 111, - 146, 32, 0, 0, 0, 1, 0, 1, 0, 187, 49, 206, 223, 175, 220, - 55, 151, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 114, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, - 101, 116, 97, 100, 97, 116, 97, 46, 77, 101, 116, 97, 100, 97, 116, - 97, 69, 110, 116, 114, 121, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 125, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 120, 0, 0, 0, 3, 0, 1, 0, 132, 0, - 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 124, 0, 0, 0, 3, 0, 1, 0, 136, 0, 0, 0, 2, 0, - 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, - 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, - 0, 3, 0, 1, 0, 144, 0, 0, 0, 2, 0, 1, 0, 3, 0, - 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 3, 0, 1, - 0, 148, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 32, 0, - 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 145, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 140, 0, 0, 0, 3, 0, 1, 0, 152, 0, 0, - 0, 2, 0, 1, 0, 107, 101, 121, 0, 0, 0, 0, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, 101, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, - 108, 117, 101, 78, 117, 109, 0, 0, 0, 0, 0, 0, 0, 0, 8, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 97, 108, 117, 101, - 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, - 101, 108, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -}}; + 0x9737dcafdfce31bb, b_9737dcafdfce31bb.words, 41, d_9737dcafdfce31bb, m_9737dcafdfce31bb, + 1, 1, i_9737dcafdfce31bb, nullptr, nullptr, { &s_9737dcafdfce31bb, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<96> b_926fe1c3b12ed651 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 81, 214, 46, 177, 195, 225, 111, 146, + 32, 0, 0, 0, 1, 0, 1, 0, + 187, 49, 206, 223, 175, 220, 55, 151, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 114, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 77, + 101, 116, 97, 100, 97, 116, 97, 46, + 77, 101, 116, 97, 100, 97, 116, 97, + 69, 110, 116, 114, 121, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 132, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 0, 0, 0, 3, 0, 1, 0, + 148, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 107, 101, 121, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 78, 117, 109, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 97, 108, 117, 101, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 108, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_926fe1c3b12ed651 = b_926fe1c3b12ed651.words; #if !CAPNP_LITE static const uint16_t m_926fe1c3b12ed651[] = {4, 0, 1, 3, 2}; static const uint16_t i_926fe1c3b12ed651[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_926fe1c3b12ed651 = { - 0x926fe1c3b12ed651, - b_926fe1c3b12ed651.words, - 96, - nullptr, - m_926fe1c3b12ed651, - 0, - 5, - i_926fe1c3b12ed651, - nullptr, - nullptr, - {&s_926fe1c3b12ed651, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<310> b_9317f20ce509d918 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 24, 217, 9, 229, 12, 242, 23, - 147, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 12, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 10, 1, 0, 0, 37, 0, 0, 0, - 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, - 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, - 105, 114, 101, 99, 116, 111, 114, 121, 0, 0, 0, 0, 0, 0, 0, - 0, 8, 0, 0, 0, 1, 0, 1, 0, 38, 158, 236, 45, 8, 124, - 210, 217, 9, 0, 0, 0, 122, 0, 0, 0, 124, 8, 45, 82, 175, - 69, 183, 158, 9, 0, 0, 0, 226, 0, 0, 0, 84, 105, 109, 101, - 115, 116, 97, 109, 112, 101, 100, 85, 82, 73, 0, 0, 68, 101, 108, - 101, 116, 101, 65, 110, 100, 85, 112, 100, 97, 116, 101, 84, 105, 108, - 101, 76, 111, 99, 97, 116, 105, 111, 110, 0, 0, 0, 0, 0, 56, - 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 121, 1, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 124, 1, 0, 0, 3, 0, 1, 0, 152, 1, 0, 0, 2, - 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 1, 0, - 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 1, - 0, 0, 3, 0, 1, 0, 180, 1, 0, 0, 2, 0, 1, 0, 2, - 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 177, 1, 0, 0, 130, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 176, 1, 0, 0, 3, 0, - 1, 0, 204, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, - 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 201, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 204, 1, 0, 0, 3, 0, 1, 0, 216, 1, - 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, - 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 216, 1, 0, 0, 3, 0, 1, 0, 244, 1, 0, 0, 2, 0, - 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 5, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 241, 1, 0, 0, - 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 1, 0, - 0, 3, 0, 1, 0, 20, 2, 0, 0, 2, 0, 1, 0, 6, 0, - 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 17, 2, 0, 0, 194, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 20, 2, 0, 0, 3, 0, 1, - 0, 48, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 7, 0, - 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 45, 2, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 48, 2, 0, 0, 3, 0, 1, 0, 76, 2, 0, - 0, 2, 0, 1, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, - 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, - 2, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 2, 0, 0, 3, 0, 1, 0, 108, 2, 0, 0, 2, 0, 1, - 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 2, 0, 0, 114, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 2, 0, 0, - 3, 0, 1, 0, 132, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, - 0, 10, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 129, 2, 0, 0, 138, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 132, 2, 0, 0, 3, 0, 1, 0, - 160, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 11, 0, 0, - 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 157, 2, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 164, 2, 0, 0, 3, 0, 1, 0, 192, 2, 0, 0, - 2, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 2, - 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, - 2, 0, 0, 3, 0, 1, 0, 200, 2, 0, 0, 2, 0, 1, 0, - 13, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 197, 2, 0, 0, 106, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 196, 2, 0, 0, 3, - 0, 1, 0, 208, 2, 0, 0, 2, 0, 1, 0, 117, 110, 102, 105, - 108, 116, 101, 114, 101, 100, 70, 114, 97, 103, 109, 101, 110, 116, 85, - 114, 105, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 99, 111, 110, 115, 111, 108, 105, 100, 97, 116, 101, 100, - 67, 111, 109, 109, 105, 116, 85, 114, 105, 115, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 114, 114, 97, 121, - 83, 99, 104, 101, 109, 97, 85, 114, 105, 115, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 97, 116, 101, 115, 116, - 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 85, 114, 105, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, - 114, 114, 97, 121, 77, 101, 116, 97, 85, 114, 105, 115, 84, 111, 86, - 97, 99, 117, 117, 109, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 97, 114, 114, 97, 121, 77, 101, 116, 97, - 86, 97, 99, 85, 114, 105, 115, 84, 111, 86, 97, 99, 117, 117, 109, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 99, 111, 109, 109, 105, 116, 85, 114, 105, - 115, 84, 111, 67, 111, 110, 115, 111, 108, 105, 100, 97, 116, 101, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, - 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, - 109, 109, 105, 116, 85, 114, 105, 115, 84, 111, 86, 97, 99, 117, 117, - 109, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 99, 111, 110, 115, 111, 108, 105, 100, 97, 116, - 101, 100, 67, 111, 109, 109, 105, 116, 85, 114, 105, 115, 84, 111, 86, - 97, 99, 117, 117, 109, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 97, 114, 114, 97, 121, 77, 101, 116, 97, 85, - 114, 105, 115, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 38, 158, 236, 45, 8, 124, 210, 217, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 102, 114, 97, 103, 109, 101, 110, 116, 77, 101, 116, - 97, 85, 114, 105, 115, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 101, 108, 101, - 116, 101, 65, 110, 100, 85, 112, 100, 97, 116, 101, 84, 105, 108, 101, - 76, 111, 99, 97, 116, 105, 111, 110, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 124, 8, 45, 82, 175, 69, 183, - 158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 109, 101, - 115, 116, 97, 109, 112, 83, 116, 97, 114, 116, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 109, 101, 115, 116, 97, - 109, 112, 69, 110, 100, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0x926fe1c3b12ed651, b_926fe1c3b12ed651.words, 96, nullptr, m_926fe1c3b12ed651, + 0, 5, i_926fe1c3b12ed651, nullptr, nullptr, { &s_926fe1c3b12ed651, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<310> b_9317f20ce509d918 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 24, 217, 9, 229, 12, 242, 23, 147, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 12, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 0, 0, 0, 23, 3, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 68, + 105, 114, 101, 99, 116, 111, 114, 121, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 38, 158, 236, 45, 8, 124, 210, 217, + 9, 0, 0, 0, 122, 0, 0, 0, + 124, 8, 45, 82, 175, 69, 183, 158, + 9, 0, 0, 0, 226, 0, 0, 0, + 84, 105, 109, 101, 115, 116, 97, 109, + 112, 101, 100, 85, 82, 73, 0, 0, + 68, 101, 108, 101, 116, 101, 65, 110, + 100, 85, 112, 100, 97, 116, 101, 84, + 105, 108, 101, 76, 111, 99, 97, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 56, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 1, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 1, 0, 0, 3, 0, 1, 0, + 152, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 1, 0, 0, 186, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 1, 0, 0, 3, 0, 1, 0, + 180, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 1, 0, 0, 3, 0, 1, 0, + 204, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 201, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 1, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 216, 1, 0, 0, 3, 0, 1, 0, + 244, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 1, 0, 0, 202, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 248, 1, 0, 0, 3, 0, 1, 0, + 20, 2, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 17, 2, 0, 0, 194, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 2, 0, 0, 3, 0, 1, 0, + 48, 2, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 2, 0, 0, 3, 0, 1, 0, + 76, 2, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 2, 0, 0, 250, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 2, 0, 0, 3, 0, 1, 0, + 108, 2, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 2, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 2, 0, 0, 3, 0, 1, 0, + 132, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 2, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 226, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 2, 0, 0, 3, 0, 1, 0, + 192, 2, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 2, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 2, 0, 0, 3, 0, 1, 0, + 200, 2, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 2, 0, 0, 3, 0, 1, 0, + 208, 2, 0, 0, 2, 0, 1, 0, + 117, 110, 102, 105, 108, 116, 101, 114, + 101, 100, 70, 114, 97, 103, 109, 101, + 110, 116, 85, 114, 105, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 111, 108, 105, 100, + 97, 116, 101, 100, 67, 111, 109, 109, + 105, 116, 85, 114, 105, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 85, 114, 105, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 116, 101, 115, 116, 65, 114, + 114, 97, 121, 83, 99, 104, 101, 109, + 97, 85, 114, 105, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 77, 101, 116, + 97, 85, 114, 105, 115, 84, 111, 86, + 97, 99, 117, 117, 109, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 77, 101, 116, + 97, 86, 97, 99, 85, 114, 105, 115, + 84, 111, 86, 97, 99, 117, 117, 109, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 109, 105, 116, 85, 114, + 105, 115, 84, 111, 67, 111, 110, 115, + 111, 108, 105, 100, 97, 116, 101, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 109, 105, 116, 85, 114, + 105, 115, 84, 111, 86, 97, 99, 117, + 117, 109, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 115, 111, 108, 105, 100, + 97, 116, 101, 100, 67, 111, 109, 109, + 105, 116, 85, 114, 105, 115, 84, 111, + 86, 97, 99, 117, 117, 109, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 77, 101, 116, + 97, 85, 114, 105, 115, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 38, 158, 236, 45, 8, 124, 210, 217, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 77, 101, 116, 97, 85, 114, 105, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 108, 101, 116, 101, 65, 110, + 100, 85, 112, 100, 97, 116, 101, 84, + 105, 108, 101, 76, 111, 99, 97, 116, + 105, 111, 110, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 124, 8, 45, 82, 175, 69, 183, 158, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 83, 116, 97, 114, 116, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 69, 110, 100, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9317f20ce509d918 = b_9317f20ce509d918.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_9317f20ce509d918[] = { - &s_9eb745af522d087c, - &s_d9d27c082dec9e26, + &s_9eb745af522d087c, + &s_d9d27c082dec9e26, }; -static const uint16_t m_9317f20ce509d918[] = { - 9, 4, 5, 2, 6, 7, 1, 8, 11, 10, 3, 13, 12, 0}; -static const uint16_t i_9317f20ce509d918[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; +static const uint16_t m_9317f20ce509d918[] = {9, 4, 5, 2, 6, 7, 1, 8, 11, 10, 3, 13, 12, 0}; +static const uint16_t i_9317f20ce509d918[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; const ::capnp::_::RawSchema s_9317f20ce509d918 = { - 0x9317f20ce509d918, - b_9317f20ce509d918.words, - 310, - d_9317f20ce509d918, - m_9317f20ce509d918, - 2, - 14, - i_9317f20ce509d918, - nullptr, - nullptr, - {&s_9317f20ce509d918, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<67> b_d9d27c082dec9e26 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 38, 158, 236, 45, 8, 124, 210, - 217, 33, 0, 0, 0, 1, 0, 2, 0, 24, 217, 9, 229, 12, 242, - 23, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 130, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, - 105, 114, 101, 99, 116, 111, 114, 121, 46, 84, 105, 109, 101, 115, 116, - 97, 109, 112, 101, 100, 85, 82, 73, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 12, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 69, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 64, 0, 0, 0, 3, 0, 1, 0, 76, 0, - 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 72, 0, 0, 0, 3, 0, 1, 0, 84, 0, 0, 0, 2, 0, - 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, - 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, 0, - 0, 3, 0, 1, 0, 92, 0, 0, 0, 2, 0, 1, 0, 117, 114, - 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 116, 105, 109, 101, 115, 116, 97, 109, 112, 83, 116, 97, 114, - 116, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, - 105, 109, 101, 115, 116, 97, 109, 112, 69, 110, 100, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x9317f20ce509d918, b_9317f20ce509d918.words, 310, d_9317f20ce509d918, m_9317f20ce509d918, + 2, 14, i_9317f20ce509d918, nullptr, nullptr, { &s_9317f20ce509d918, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<67> b_d9d27c082dec9e26 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 38, 158, 236, 45, 8, 124, 210, 217, + 33, 0, 0, 0, 1, 0, 2, 0, + 24, 217, 9, 229, 12, 242, 23, 147, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 68, + 105, 114, 101, 99, 116, 111, 114, 121, + 46, 84, 105, 109, 101, 115, 116, 97, + 109, 112, 101, 100, 85, 82, 73, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 76, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 83, 116, 97, 114, 116, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 69, 110, 100, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d9d27c082dec9e26 = b_d9d27c082dec9e26.words; #if !CAPNP_LITE static const uint16_t m_d9d27c082dec9e26[] = {2, 1, 0}; static const uint16_t i_d9d27c082dec9e26[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d9d27c082dec9e26 = { - 0xd9d27c082dec9e26, - b_d9d27c082dec9e26.words, - 67, - nullptr, - m_d9d27c082dec9e26, - 0, - 3, - i_d9d27c082dec9e26, - nullptr, - nullptr, - {&s_d9d27c082dec9e26, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_9eb745af522d087c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 124, 8, 45, 82, 175, 69, 183, - 158, 33, 0, 0, 0, 1, 0, 1, 0, 24, 217, 9, 229, 12, 242, - 23, 147, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 234, 1, 0, 0, 49, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, - 105, 114, 101, 99, 116, 111, 114, 121, 46, 68, 101, 108, 101, 116, 101, - 65, 110, 100, 85, 112, 100, 97, 116, 101, 84, 105, 108, 101, 76, 111, - 99, 97, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 12, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 69, 0, 0, 0, 34, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 3, 0, 1, 0, 76, - 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 73, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 72, 0, 0, 0, 3, 0, 1, 0, 84, 0, 0, 0, 2, - 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, - 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, - 0, 0, 3, 0, 1, 0, 88, 0, 0, 0, 2, 0, 1, 0, 117, - 114, 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 99, 111, 110, 100, 105, 116, 105, 111, 110, 77, 97, 114, - 107, 101, 114, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 111, 102, 102, 115, 101, 116, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xd9d27c082dec9e26, b_d9d27c082dec9e26.words, 67, nullptr, m_d9d27c082dec9e26, + 0, 3, i_d9d27c082dec9e26, nullptr, nullptr, { &s_d9d27c082dec9e26, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<68> b_9eb745af522d087c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 124, 8, 45, 82, 175, 69, 183, 158, + 33, 0, 0, 0, 1, 0, 1, 0, + 24, 217, 9, 229, 12, 242, 23, 147, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 68, + 105, 114, 101, 99, 116, 111, 114, 121, + 46, 68, 101, 108, 101, 116, 101, 65, + 110, 100, 85, 112, 100, 97, 116, 101, + 84, 105, 108, 101, 76, 111, 99, 97, + 116, 105, 111, 110, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 76, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 88, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 77, 97, 114, 107, 101, 114, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_9eb745af522d087c = b_9eb745af522d087c.words; #if !CAPNP_LITE static const uint16_t m_9eb745af522d087c[] = {1, 2, 0}; static const uint16_t i_9eb745af522d087c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9eb745af522d087c = { - 0x9eb745af522d087c, - b_9eb745af522d087c.words, - 68, - nullptr, - m_9eb745af522d087c, - 0, - 3, - i_9eb745af522d087c, - nullptr, - nullptr, - {&s_9eb745af522d087c, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<95> b_8cd4e323f1feea3b = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 59, 234, 254, 241, 35, 227, 212, - 140, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, - 116, 101, 100, 82, 101, 115, 117, 108, 116, 83, 105, 122, 101, 0, 0, - 0, 8, 0, 0, 0, 1, 0, 1, 0, 105, 82, 86, 133, 118, 70, - 200, 146, 9, 0, 0, 0, 90, 0, 0, 0, 34, 28, 89, 38, 105, - 145, 167, 189, 9, 0, 0, 0, 90, 0, 0, 0, 82, 101, 115, 117, - 108, 116, 83, 105, 122, 101, 0, 0, 0, 0, 0, 0, 77, 101, 109, - 111, 114, 121, 83, 105, 122, 101, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 40, 0, 0, 0, 3, 0, 1, 0, 124, 0, 0, 0, 2, 0, - 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 0, - 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 0, 0, - 0, 3, 0, 1, 0, 204, 0, 0, 0, 2, 0, 1, 0, 114, 101, - 115, 117, 108, 116, 83, 105, 122, 101, 115, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, 148, 193, 121, 241, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, 0, 0, 2, 0, - 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, - 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 105, 82, 86, 133, 118, 70, 200, 146, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 109, 101, 109, 111, 114, 121, 83, 105, 122, 101, 115, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 140, 113, - 113, 174, 148, 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, - 4, 0, 0, 0, 2, 0, 1, 0, 140, 113, 113, 174, 148, 193, 121, - 241, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 0, - 0, 0, 8, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 34, 28, 89, 38, 105, 145, 167, 189, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x9eb745af522d087c, b_9eb745af522d087c.words, 68, nullptr, m_9eb745af522d087c, + 0, 3, i_9eb745af522d087c, nullptr, nullptr, { &s_9eb745af522d087c, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<95> b_8cd4e323f1feea3b = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 59, 234, 254, 241, 35, 227, 212, 140, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 115, 116, 105, 109, 97, + 116, 101, 100, 82, 101, 115, 117, 108, + 116, 83, 105, 122, 101, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 105, 82, 86, 133, 118, 70, 200, 146, + 9, 0, 0, 0, 90, 0, 0, 0, + 34, 28, 89, 38, 105, 145, 167, 189, + 9, 0, 0, 0, 90, 0, 0, 0, + 82, 101, 115, 117, 108, 116, 83, 105, + 122, 101, 0, 0, 0, 0, 0, 0, + 77, 101, 109, 111, 114, 121, 83, 105, + 122, 101, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, + 114, 101, 115, 117, 108, 116, 83, 105, + 122, 101, 115, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 105, 82, 86, 133, 118, 70, 200, 146, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 109, 111, 114, 121, 83, 105, + 122, 101, 115, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 34, 28, 89, 38, 105, 145, 167, 189, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_8cd4e323f1feea3b = b_8cd4e323f1feea3b.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_8cd4e323f1feea3b[] = { - &s_f179c194ae71718c, + &s_f179c194ae71718c, }; static const uint16_t m_8cd4e323f1feea3b[] = {1, 0}; static const uint16_t i_8cd4e323f1feea3b[] = {0, 1}; -KJ_CONSTEXPR( - const)::capnp::_::RawBrandedSchema::Dependency bd_8cd4e323f1feea3b[] = { - {16777216, - ::tiledb::sm::serialization::capnp::Map< - ::capnp::Text, - ::tiledb::sm::serialization::capnp::EstimatedResultSize::ResultSize>:: - _capnpPrivate::brand()}, - {16777217, - ::tiledb::sm::serialization::capnp::Map< - ::capnp::Text, - ::tiledb::sm::serialization::capnp::EstimatedResultSize::MemorySize>:: - _capnpPrivate::brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_8cd4e323f1feea3b[] = { + { 16777216, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::EstimatedResultSize::ResultSize>::_capnpPrivate::brand() }, + { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::EstimatedResultSize::MemorySize>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_8cd4e323f1feea3b = { - 0x8cd4e323f1feea3b, - b_8cd4e323f1feea3b.words, - 95, - d_8cd4e323f1feea3b, - m_8cd4e323f1feea3b, - 1, - 2, - i_8cd4e323f1feea3b, - nullptr, - nullptr, - {&s_8cd4e323f1feea3b, - nullptr, - bd_8cd4e323f1feea3b, - 0, - sizeof(bd_8cd4e323f1feea3b) / sizeof(bd_8cd4e323f1feea3b[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_92c8467685565269 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 105, 82, 86, 133, 118, 70, 200, - 146, 38, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, - 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, - 116, 101, 100, 82, 101, 115, 117, 108, 116, 83, 105, 122, 101, 46, 82, - 101, 115, 117, 108, 116, 83, 105, 122, 101, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, - 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, - 0, 0, 3, 0, 1, 0, 80, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 66, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, - 1, 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 81, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 80, 0, 0, 0, 3, 0, 1, 0, 92, 0, - 0, 0, 2, 0, 1, 0, 115, 105, 122, 101, 70, 105, 120, 101, 100, - 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 115, 105, 122, 101, 86, 97, 114, 0, 11, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 115, 105, 122, 101, 86, 97, 108, 105, - 100, 105, 116, 121, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0x8cd4e323f1feea3b, b_8cd4e323f1feea3b.words, 95, d_8cd4e323f1feea3b, m_8cd4e323f1feea3b, + 1, 2, i_8cd4e323f1feea3b, nullptr, nullptr, { &s_8cd4e323f1feea3b, nullptr, bd_8cd4e323f1feea3b, 0, sizeof(bd_8cd4e323f1feea3b) / sizeof(bd_8cd4e323f1feea3b[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<68> b_92c8467685565269 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 105, 82, 86, 133, 118, 70, 200, 146, + 38, 0, 0, 0, 1, 0, 3, 0, + 59, 234, 254, 241, 35, 227, 212, 140, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 115, 116, 105, 109, 97, + 116, 101, 100, 82, 101, 115, 117, 108, + 116, 83, 105, 122, 101, 46, 82, 101, + 115, 117, 108, 116, 83, 105, 122, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 115, 105, 122, 101, 70, 105, 120, 101, + 100, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 86, 97, 114, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 86, 97, 108, 105, + 100, 105, 116, 121, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 11, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_92c8467685565269 = b_92c8467685565269.words; #if !CAPNP_LITE static const uint16_t m_92c8467685565269[] = {0, 2, 1}; static const uint16_t i_92c8467685565269[] = {0, 1, 2}; const ::capnp::_::RawSchema s_92c8467685565269 = { - 0x92c8467685565269, - b_92c8467685565269.words, - 68, - nullptr, - m_92c8467685565269, - 0, - 3, - i_92c8467685565269, - nullptr, - nullptr, - {&s_92c8467685565269, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_bda7916926591c22 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 34, 28, 89, 38, 105, 145, 167, - 189, 38, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, - 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, - 116, 101, 100, 82, 101, 115, 117, 108, 116, 83, 105, 122, 101, 46, 77, - 101, 109, 111, 114, 121, 83, 105, 122, 101, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, - 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, - 0, 0, 3, 0, 1, 0, 80, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 66, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, - 1, 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 81, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 80, 0, 0, 0, 3, 0, 1, 0, 92, 0, - 0, 0, 2, 0, 1, 0, 115, 105, 122, 101, 70, 105, 120, 101, 100, - 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 115, 105, 122, 101, 86, 97, 114, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 115, 105, 122, 101, 86, 97, 108, 105, - 100, 105, 116, 121, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0x92c8467685565269, b_92c8467685565269.words, 68, nullptr, m_92c8467685565269, + 0, 3, i_92c8467685565269, nullptr, nullptr, { &s_92c8467685565269, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<68> b_bda7916926591c22 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 34, 28, 89, 38, 105, 145, 167, 189, + 38, 0, 0, 0, 1, 0, 3, 0, + 59, 234, 254, 241, 35, 227, 212, 140, + 0, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 115, 116, 105, 109, 97, + 116, 101, 100, 82, 101, 115, 117, 108, + 116, 83, 105, 122, 101, 46, 77, 101, + 109, 111, 114, 121, 83, 105, 122, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 115, 105, 122, 101, 70, 105, 120, 101, + 100, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 86, 97, 114, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 86, 97, 108, 105, + 100, 105, 116, 121, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_bda7916926591c22 = b_bda7916926591c22.words; #if !CAPNP_LITE static const uint16_t m_bda7916926591c22[] = {0, 2, 1}; static const uint16_t i_bda7916926591c22[] = {0, 1, 2}; const ::capnp::_::RawSchema s_bda7916926591c22 = { - 0xbda7916926591c22, - b_bda7916926591c22.words, - 68, - nullptr, - m_bda7916926591c22, - 0, - 3, - i_bda7916926591c22, - nullptr, - nullptr, - {&s_bda7916926591c22, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<34> b_c95970eb9310dc9c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 156, 220, 16, 147, 235, 112, 89, - 201, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, - 110, 116, 73, 110, 102, 111, 82, 101, 113, 117, 101, 115, 116, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 3, 0, 1, 0, 20, 0, 0, 0, 2, 0, 1, 0, 99, 111, - 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, - 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xbda7916926591c22, b_bda7916926591c22.words, 68, nullptr, m_bda7916926591c22, + 0, 3, i_bda7916926591c22, nullptr, nullptr, { &s_bda7916926591c22, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<34> b_c95970eb9310dc9c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 156, 220, 16, 147, 235, 112, 89, 201, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 114, 97, 103, 109, 101, + 110, 116, 73, 110, 102, 111, 82, 101, + 113, 117, 101, 115, 116, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 20, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c95970eb9310dc9c = b_c95970eb9310dc9c.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c95970eb9310dc9c[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_c95970eb9310dc9c[] = {0}; static const uint16_t i_c95970eb9310dc9c[] = {0}; const ::capnp::_::RawSchema s_c95970eb9310dc9c = { - 0xc95970eb9310dc9c, - b_c95970eb9310dc9c.words, - 34, - d_c95970eb9310dc9c, - m_c95970eb9310dc9c, - 1, - 1, - i_c95970eb9310dc9c, - nullptr, - nullptr, - {&s_c95970eb9310dc9c, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<66> b_d42e7b38b33e3d29 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 41, 61, 62, 179, 56, 123, 46, - 212, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, - 70, 114, 97, 103, 109, 101, 110, 116, 73, 110, 102, 111, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, - 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, - 0, 3, 0, 1, 0, 80, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 42, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, 1, - 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 81, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 80, 0, 0, 0, 3, 0, 1, 0, 92, 0, 0, - 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 83, 99, 104, 101, 109, - 97, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 109, 101, 116, 97, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 97, 202, 231, 39, 252, 82, 227, 205, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, 109, 101, 110, 116, 83, - 105, 122, 101, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -}}; + 0xc95970eb9310dc9c, b_c95970eb9310dc9c.words, 34, d_c95970eb9310dc9c, m_c95970eb9310dc9c, + 1, 1, i_c95970eb9310dc9c, nullptr, nullptr, { &s_c95970eb9310dc9c, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<66> b_d42e7b38b33e3d29 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 41, 61, 62, 179, 56, 123, 46, 212, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 105, 110, 103, 108, 101, + 70, 114, 97, 103, 109, 101, 110, 116, + 73, 110, 102, 111, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 78, 97, 109, 101, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 116, 97, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 83, 105, 122, 101, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d42e7b38b33e3d29 = b_d42e7b38b33e3d29.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_d42e7b38b33e3d29[] = { - &s_cde352fc27e7ca61, + &s_cde352fc27e7ca61, }; static const uint16_t m_d42e7b38b33e3d29[] = {0, 2, 1}; static const uint16_t i_d42e7b38b33e3d29[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d42e7b38b33e3d29 = { - 0xd42e7b38b33e3d29, - b_d42e7b38b33e3d29.words, - 66, - d_d42e7b38b33e3d29, - m_d42e7b38b33e3d29, - 1, - 3, - i_d42e7b38b33e3d29, - nullptr, - nullptr, - {&s_d42e7b38b33e3d29, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<109> b_a000530ab1d17816 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 22, 120, 209, 177, 10, 83, 0, - 160, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 250, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, - 110, 116, 73, 110, 102, 111, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 97, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 100, 0, 0, 0, 3, 0, 1, 0, 112, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 109, - 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 108, 0, 0, 0, 3, 0, 1, 0, 192, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 106, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 0, - 3, 0, 1, 0, 216, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 74, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 212, 0, 0, 0, 3, 0, 1, 0, - 240, 0, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 83, 99, - 104, 101, 109, 97, 76, 97, 116, 101, 115, 116, 0, 0, 0, 0, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, - 227, 29, 215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 114, - 114, 97, 121, 83, 99, 104, 101, 109, 97, 115, 65, 108, 108, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, 148, 193, 121, 241, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, 0, 0, 2, 0, - 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, - 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 254, 150, 226, 152, 47, 227, 29, 215, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 102, 114, 97, 103, 109, 101, 110, 116, 73, 110, 102, - 111, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 41, 61, 62, 179, 56, 123, 46, 212, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 116, 111, 86, 97, 99, 117, 117, 109, 0, 0, 0, 0, - 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xd42e7b38b33e3d29, b_d42e7b38b33e3d29.words, 66, d_d42e7b38b33e3d29, m_d42e7b38b33e3d29, + 1, 3, i_d42e7b38b33e3d29, nullptr, nullptr, { &s_d42e7b38b33e3d29, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<109> b_a000530ab1d17816 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 22, 120, 209, 177, 10, 83, 0, 160, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 250, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 114, 97, 103, 109, 101, + 110, 116, 73, 110, 102, 111, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 76, 97, 116, 101, 115, + 116, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 83, 99, 104, + 101, 109, 97, 115, 65, 108, 108, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 73, 110, 102, 111, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 41, 61, 62, 179, 56, 123, 46, 212, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 111, 86, 97, 99, 117, 117, 109, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a000530ab1d17816 = b_a000530ab1d17816.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a000530ab1d17816[] = { - &s_d42e7b38b33e3d29, - &s_d71de32f98e296fe, - &s_f179c194ae71718c, + &s_d42e7b38b33e3d29, + &s_d71de32f98e296fe, + &s_f179c194ae71718c, }; static const uint16_t m_a000530ab1d17816[] = {0, 1, 2, 3}; static const uint16_t i_a000530ab1d17816[] = {0, 1, 2, 3}; -KJ_CONSTEXPR( - const)::capnp::_::RawBrandedSchema::Dependency bd_a000530ab1d17816[] = { - {16777217, - ::tiledb::sm::serialization::capnp:: - Map<::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>:: - _capnpPrivate::brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a000530ab1d17816[] = { + { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a000530ab1d17816 = { - 0xa000530ab1d17816, - b_a000530ab1d17816.words, - 109, - d_a000530ab1d17816, - m_a000530ab1d17816, - 3, - 4, - i_a000530ab1d17816, - nullptr, - nullptr, - {&s_a000530ab1d17816, - nullptr, - bd_a000530ab1d17816, - 0, - sizeof(bd_a000530ab1d17816) / sizeof(bd_a000530ab1d17816[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_838188de0fd57580 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 128, 117, 213, 15, 222, 136, 129, - 131, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, - 101, 116, 97, 100, 97, 116, 97, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 0, 0, 0, 3, 0, 1, 0, 56, 0, 0, 0, 2, 0, 1, - 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 109, 101, 116, 97, 100, 97, 116, 97, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 187, - 49, 206, 223, 175, 220, 55, 151, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xa000530ab1d17816, b_a000530ab1d17816.words, 109, d_a000530ab1d17816, m_a000530ab1d17816, + 3, 4, i_a000530ab1d17816, nullptr, nullptr, { &s_a000530ab1d17816, nullptr, bd_a000530ab1d17816, 0, sizeof(bd_a000530ab1d17816) / sizeof(bd_a000530ab1d17816[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_838188de0fd57580 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 128, 117, 213, 15, 222, 136, 129, 131, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 77, + 101, 116, 97, 100, 97, 116, 97, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 116, 97, 100, 97, 116, 97, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 187, 49, 206, 223, 175, 220, 55, 151, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_838188de0fd57580 = b_838188de0fd57580.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_838188de0fd57580[] = { - &s_9737dcafdfce31bb, - &s_b6c95b4b8111ad36, + &s_9737dcafdfce31bb, + &s_b6c95b4b8111ad36, }; static const uint16_t m_838188de0fd57580[] = {0, 1}; static const uint16_t i_838188de0fd57580[] = {0, 1}; const ::capnp::_::RawSchema s_838188de0fd57580 = { - 0x838188de0fd57580, - b_838188de0fd57580.words, - 49, - d_838188de0fd57580, - m_838188de0fd57580, - 2, - 2, - i_838188de0fd57580, - nullptr, - nullptr, - {&s_838188de0fd57580, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<79> b_c41bcc7e8cc58f18 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 24, 143, 197, 140, 126, 204, 27, - 196, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, - 101, 109, 98, 101, 114, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 97, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 92, 0, 0, 0, 3, 0, 1, 0, 104, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, - 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, 108, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 74, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, - 3, 0, 1, 0, 116, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 42, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 3, 0, 1, 0, - 120, 0, 0, 0, 2, 0, 1, 0, 117, 114, 105, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 121, 112, - 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 114, 101, 108, 97, 116, 105, 118, 101, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 97, - 109, 101, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0x838188de0fd57580, b_838188de0fd57580.words, 49, d_838188de0fd57580, m_838188de0fd57580, + 2, 2, i_838188de0fd57580, nullptr, nullptr, { &s_838188de0fd57580, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<79> b_c41bcc7e8cc58f18 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 24, 143, 197, 140, 126, 204, 27, 196, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 77, + 101, 109, 98, 101, 114, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 16, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 92, 0, 0, 0, 3, 0, 1, 0, + 104, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 101, 108, 97, 116, 105, 118, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c41bcc7e8cc58f18 = b_c41bcc7e8cc58f18.words; #if !CAPNP_LITE static const uint16_t m_c41bcc7e8cc58f18[] = {3, 2, 1, 0}; static const uint16_t i_c41bcc7e8cc58f18[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_c41bcc7e8cc58f18 = { - 0xc41bcc7e8cc58f18, - b_c41bcc7e8cc58f18.words, - 79, - nullptr, - m_c41bcc7e8cc58f18, - 0, - 4, - i_c41bcc7e8cc58f18, - nullptr, - nullptr, - {&s_c41bcc7e8cc58f18, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<60> b_dcdd20e1b79e915a = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 90, 145, 158, 183, 225, 32, 221, - 220, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 0, - 4, 0, 0, 0, 1, 0, 1, 0, 193, 117, 180, 21, 199, 16, 234, - 162, 1, 0, 0, 0, 106, 0, 0, 0, 71, 114, 111, 117, 112, 68, - 101, 116, 97, 105, 108, 115, 0, 0, 0, 0, 8, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, - 0, 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 50, 0, 0, - 0, 45, 0, 0, 0, 31, 0, 0, 0, 76, 0, 0, 0, 3, 0, - 1, 0, 88, 0, 0, 0, 2, 0, 1, 0, 99, 111, 110, 102, 105, - 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, 129, - 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, - 114, 111, 117, 112, 0, 0, 0, 4, 0, 0, 0, 1, 0, 2, 0, - 61, 124, 46, 28, 214, 31, 91, 250, 4, 0, 0, 0, 2, 0, 1, - 0, 16, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 50, - 0, 0, 0, 103, 114, 111, 117, 112, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 193, 117, 180, - 21, 199, 16, 234, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xc41bcc7e8cc58f18, b_c41bcc7e8cc58f18.words, 79, nullptr, m_c41bcc7e8cc58f18, + 0, 4, i_c41bcc7e8cc58f18, nullptr, nullptr, { &s_c41bcc7e8cc58f18, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<60> b_dcdd20e1b79e915a = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 90, 145, 158, 183, 225, 32, 221, 220, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 193, 117, 180, 21, 199, 16, 234, 162, + 1, 0, 0, 0, 106, 0, 0, 0, + 71, 114, 111, 117, 112, 68, 101, 116, + 97, 105, 108, 115, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 45, 0, 0, 0, 31, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 88, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 114, 111, 117, 112, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 2, 0, + 61, 124, 46, 28, 214, 31, 91, 250, + 4, 0, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 50, 0, 0, 0, + 103, 114, 111, 117, 112, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 193, 117, 180, 21, 199, 16, 234, 162, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_dcdd20e1b79e915a = b_dcdd20e1b79e915a.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_dcdd20e1b79e915a[] = { - &s_a2ea10c715b475c1, - &s_b6c95b4b8111ad36, + &s_a2ea10c715b475c1, + &s_b6c95b4b8111ad36, }; static const uint16_t m_dcdd20e1b79e915a[] = {0, 1}; static const uint16_t i_dcdd20e1b79e915a[] = {0, 1}; const ::capnp::_::RawSchema s_dcdd20e1b79e915a = { - 0xdcdd20e1b79e915a, - b_dcdd20e1b79e915a.words, - 60, - d_dcdd20e1b79e915a, - m_dcdd20e1b79e915a, - 2, - 2, - i_dcdd20e1b79e915a, - nullptr, - nullptr, - {&s_dcdd20e1b79e915a, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<54> b_a2ea10c715b475c1 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 193, 117, 180, 21, 199, 16, 234, - 162, 24, 0, 0, 0, 1, 0, 0, 0, 90, 145, 158, 183, 225, 32, - 221, 220, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 46, - 71, 114, 111, 117, 112, 68, 101, 116, 97, 105, 108, 115, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 64, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 3, 0, 1, - 0, 72, 0, 0, 0, 2, 0, 1, 0, 109, 101, 109, 98, 101, 114, - 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 24, 143, 197, - 140, 126, 204, 27, 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 109, 101, 116, 97, 100, 97, 116, 97, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 187, 49, 206, 223, 175, 220, - 55, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xdcdd20e1b79e915a, b_dcdd20e1b79e915a.words, 60, d_dcdd20e1b79e915a, m_dcdd20e1b79e915a, + 2, 2, i_dcdd20e1b79e915a, nullptr, nullptr, { &s_dcdd20e1b79e915a, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<54> b_a2ea10c715b475c1 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 193, 117, 180, 21, 199, 16, 234, 162, + 24, 0, 0, 0, 1, 0, 0, 0, + 90, 145, 158, 183, 225, 32, 221, 220, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 46, + 71, 114, 111, 117, 112, 68, 101, 116, + 97, 105, 108, 115, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 64, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 61, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 60, 0, 0, 0, 3, 0, 1, 0, + 72, 0, 0, 0, 2, 0, 1, 0, + 109, 101, 109, 98, 101, 114, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 24, 143, 197, 140, 126, 204, 27, 196, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 116, 97, 100, 97, 116, 97, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 187, 49, 206, 223, 175, 220, 55, 151, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a2ea10c715b475c1 = b_a2ea10c715b475c1.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a2ea10c715b475c1[] = { - &s_9737dcafdfce31bb, - &s_c41bcc7e8cc58f18, + &s_9737dcafdfce31bb, + &s_c41bcc7e8cc58f18, }; static const uint16_t m_a2ea10c715b475c1[] = {0, 1}; static const uint16_t i_a2ea10c715b475c1[] = {0, 1}; const ::capnp::_::RawSchema s_a2ea10c715b475c1 = { - 0xa2ea10c715b475c1, - b_a2ea10c715b475c1.words, - 54, - d_a2ea10c715b475c1, - m_a2ea10c715b475c1, - 2, - 2, - i_a2ea10c715b475c1, - nullptr, - nullptr, - {&s_a2ea10c715b475c1, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<64> b_c4e54a63294eddb7 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 183, 221, 78, 41, 99, 74, 229, - 196, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, - 112, 100, 97, 116, 101, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 64, 222, 155, 117, 70, 30, 176, 131, 1, 0, 0, 0, 154, 0, - 0, 0, 71, 114, 111, 117, 112, 85, 112, 100, 97, 116, 101, 68, 101, - 116, 97, 105, 108, 115, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, - 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, - 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, - 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, - 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 98, 0, - 0, 0, 49, 0, 0, 0, 31, 0, 0, 0, 84, 0, 0, 0, 3, - 0, 1, 0, 96, 0, 0, 0, 2, 0, 1, 0, 99, 111, 110, 102, - 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, - 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 103, 114, 111, 117, 112, 85, 112, 100, 97, 116, 101, 0, 0, 0, 0, - 0, 4, 0, 0, 0, 1, 0, 2, 0, 61, 124, 46, 28, 214, 31, - 91, 250, 4, 0, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 0, - 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 114, 0, 0, 0, 103, 114, 111, - 117, 112, 95, 99, 104, 97, 110, 103, 101, 115, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 64, - 222, 155, 117, 70, 30, 176, 131, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xa2ea10c715b475c1, b_a2ea10c715b475c1.words, 54, d_a2ea10c715b475c1, m_a2ea10c715b475c1, + 2, 2, i_a2ea10c715b475c1, nullptr, nullptr, { &s_a2ea10c715b475c1, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<64> b_c4e54a63294eddb7 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 183, 221, 78, 41, 99, 74, 229, 196, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 85, + 112, 100, 97, 116, 101, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 64, 222, 155, 117, 70, 30, 176, 131, + 1, 0, 0, 0, 154, 0, 0, 0, + 71, 114, 111, 117, 112, 85, 112, 100, + 97, 116, 101, 68, 101, 116, 97, 105, + 108, 115, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 98, 0, 0, 0, + 49, 0, 0, 0, 31, 0, 0, 0, + 84, 0, 0, 0, 3, 0, 1, 0, + 96, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 114, 111, 117, 112, 85, 112, 100, + 97, 116, 101, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 2, 0, + 61, 124, 46, 28, 214, 31, 91, 250, + 4, 0, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 0, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 114, 0, 0, 0, + 103, 114, 111, 117, 112, 95, 99, 104, + 97, 110, 103, 101, 115, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 64, 222, 155, 117, 70, 30, 176, 131, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c4e54a63294eddb7 = b_c4e54a63294eddb7.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c4e54a63294eddb7[] = { - &s_83b01e46759bde40, - &s_b6c95b4b8111ad36, + &s_83b01e46759bde40, + &s_b6c95b4b8111ad36, }; static const uint16_t m_c4e54a63294eddb7[] = {0, 1}; static const uint16_t i_c4e54a63294eddb7[] = {0, 1}; const ::capnp::_::RawSchema s_c4e54a63294eddb7 = { - 0xc4e54a63294eddb7, - b_c4e54a63294eddb7.words, - 64, - d_c4e54a63294eddb7, - m_c4e54a63294eddb7, - 2, - 2, - i_c4e54a63294eddb7, - nullptr, - nullptr, - {&s_c4e54a63294eddb7, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<82> b_83b01e46759bde40 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 64, 222, 155, 117, 70, 30, 176, - 131, 30, 0, 0, 0, 1, 0, 0, 0, 183, 221, 78, 41, 99, 74, - 229, 196, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, - 112, 100, 97, 116, 101, 46, 71, 114, 111, 117, 112, 85, 112, 100, 97, - 116, 101, 68, 101, 116, 97, 105, 108, 115, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 130, 0, 0, 0, 45, 0, 0, 0, 31, 0, 0, 0, 84, 0, - 0, 0, 3, 0, 1, 0, 112, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 106, 0, 0, - 0, 113, 0, 0, 0, 31, 0, 0, 0, 148, 0, 0, 0, 3, 0, - 1, 0, 176, 0, 0, 0, 2, 0, 1, 0, 109, 101, 109, 98, 101, - 114, 115, 84, 111, 82, 101, 109, 111, 118, 101, 0, 4, 0, 0, 0, - 1, 0, 2, 0, 61, 124, 46, 28, 214, 31, 91, 250, 4, 0, 0, - 0, 2, 0, 1, 0, 24, 0, 0, 0, 0, 0, 1, 0, 12, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 146, 0, 0, 0, 109, 101, 109, 98, 101, 114, 115, 95, - 116, 111, 95, 114, 101, 109, 111, 118, 101, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 109, 101, 109, 98, 101, 114, 115, 84, - 111, 65, 100, 100, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 2, - 0, 61, 124, 46, 28, 214, 31, 91, 250, 4, 0, 0, 0, 2, 0, - 1, 0, 20, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 122, 0, 0, 0, 109, 101, 109, 98, 101, 114, 115, 95, 116, 111, 95, - 97, 100, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 24, 143, 197, 140, 126, 204, 27, - 196, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xc4e54a63294eddb7, b_c4e54a63294eddb7.words, 64, d_c4e54a63294eddb7, m_c4e54a63294eddb7, + 2, 2, i_c4e54a63294eddb7, nullptr, nullptr, { &s_c4e54a63294eddb7, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<82> b_83b01e46759bde40 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 64, 222, 155, 117, 70, 30, 176, 131, + 30, 0, 0, 0, 1, 0, 0, 0, + 183, 221, 78, 41, 99, 74, 229, 196, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 85, + 112, 100, 97, 116, 101, 46, 71, 114, + 111, 117, 112, 85, 112, 100, 97, 116, + 101, 68, 101, 116, 97, 105, 108, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 130, 0, 0, 0, + 45, 0, 0, 0, 31, 0, 0, 0, + 84, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 0, 0, 0, 106, 0, 0, 0, + 113, 0, 0, 0, 31, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 176, 0, 0, 0, 2, 0, 1, 0, + 109, 101, 109, 98, 101, 114, 115, 84, + 111, 82, 101, 109, 111, 118, 101, 0, + 4, 0, 0, 0, 1, 0, 2, 0, + 61, 124, 46, 28, 214, 31, 91, 250, + 4, 0, 0, 0, 2, 0, 1, 0, + 24, 0, 0, 0, 0, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 146, 0, 0, 0, + 109, 101, 109, 98, 101, 114, 115, 95, + 116, 111, 95, 114, 101, 109, 111, 118, + 101, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 101, 109, 98, 101, 114, 115, 84, + 111, 65, 100, 100, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 2, 0, + 61, 124, 46, 28, 214, 31, 91, 250, + 4, 0, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 0, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 122, 0, 0, 0, + 109, 101, 109, 98, 101, 114, 115, 95, + 116, 111, 95, 97, 100, 100, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 24, 143, 197, 140, 126, 204, 27, 196, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_83b01e46759bde40 = b_83b01e46759bde40.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_83b01e46759bde40[] = { - &s_c41bcc7e8cc58f18, + &s_c41bcc7e8cc58f18, }; static const uint16_t m_83b01e46759bde40[] = {1, 0}; static const uint16_t i_83b01e46759bde40[] = {0, 1}; const ::capnp::_::RawSchema s_83b01e46759bde40 = { - 0x83b01e46759bde40, - b_83b01e46759bde40.words, - 82, - d_83b01e46759bde40, - m_83b01e46759bde40, - 1, - 2, - i_83b01e46759bde40, - nullptr, - nullptr, - {&s_83b01e46759bde40, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<64> b_fb7f36ad4d8ffe84 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 132, 254, 143, 77, 173, 54, 127, - 251, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, - 114, 101, 97, 116, 101, 0, 0, 0, 4, 0, 0, 0, 1, 0, 1, - 0, 169, 134, 94, 215, 154, 69, 253, 213, 1, 0, 0, 0, 154, 0, - 0, 0, 71, 114, 111, 117, 112, 67, 114, 101, 97, 116, 101, 68, 101, - 116, 97, 105, 108, 115, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, - 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, - 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, - 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, - 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 106, 0, - 0, 0, 49, 0, 0, 0, 31, 0, 0, 0, 84, 0, 0, 0, 3, - 0, 1, 0, 96, 0, 0, 0, 2, 0, 1, 0, 99, 111, 110, 102, - 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, - 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 103, 114, 111, 117, 112, 68, 101, 116, 97, 105, 108, 115, 0, 0, 0, - 0, 4, 0, 0, 0, 1, 0, 2, 0, 61, 124, 46, 28, 214, 31, - 91, 250, 4, 0, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 0, - 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 114, 0, 0, 0, 103, 114, 111, - 117, 112, 95, 100, 101, 116, 97, 105, 108, 115, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 169, - 134, 94, 215, 154, 69, 253, 213, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0x83b01e46759bde40, b_83b01e46759bde40.words, 82, d_83b01e46759bde40, m_83b01e46759bde40, + 1, 2, i_83b01e46759bde40, nullptr, nullptr, { &s_83b01e46759bde40, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<64> b_fb7f36ad4d8ffe84 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 132, 254, 143, 77, 173, 54, 127, 251, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 67, + 114, 101, 97, 116, 101, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 169, 134, 94, 215, 154, 69, 253, 213, + 1, 0, 0, 0, 154, 0, 0, 0, + 71, 114, 111, 117, 112, 67, 114, 101, + 97, 116, 101, 68, 101, 116, 97, 105, + 108, 115, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 106, 0, 0, 0, + 49, 0, 0, 0, 31, 0, 0, 0, + 84, 0, 0, 0, 3, 0, 1, 0, + 96, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 114, 111, 117, 112, 68, 101, 116, + 97, 105, 108, 115, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 2, 0, + 61, 124, 46, 28, 214, 31, 91, 250, + 4, 0, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 0, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 114, 0, 0, 0, + 103, 114, 111, 117, 112, 95, 100, 101, + 116, 97, 105, 108, 115, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 169, 134, 94, 215, 154, 69, 253, 213, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_fb7f36ad4d8ffe84 = b_fb7f36ad4d8ffe84.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_fb7f36ad4d8ffe84[] = { - &s_b6c95b4b8111ad36, - &s_d5fd459ad75e86a9, + &s_b6c95b4b8111ad36, + &s_d5fd459ad75e86a9, }; static const uint16_t m_fb7f36ad4d8ffe84[] = {0, 1}; static const uint16_t i_fb7f36ad4d8ffe84[] = {0, 1}; const ::capnp::_::RawSchema s_fb7f36ad4d8ffe84 = { - 0xfb7f36ad4d8ffe84, - b_fb7f36ad4d8ffe84.words, - 64, - d_fb7f36ad4d8ffe84, - m_fb7f36ad4d8ffe84, - 2, - 2, - i_fb7f36ad4d8ffe84, - nullptr, - nullptr, - {&s_fb7f36ad4d8ffe84, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<36> b_d5fd459ad75e86a9 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 169, 134, 94, 215, 154, 69, 253, - 213, 30, 0, 0, 0, 1, 0, 0, 0, 132, 254, 143, 77, 173, 54, - 127, 251, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 138, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, - 114, 101, 97, 116, 101, 46, 71, 114, 111, 117, 112, 67, 114, 101, 97, - 116, 101, 68, 101, 116, 97, 105, 108, 115, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, - 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 3, 0, 1, 0, 20, 0, 0, 0, 2, 0, 1, 0, 117, - 114, 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, -}}; + 0xfb7f36ad4d8ffe84, b_fb7f36ad4d8ffe84.words, 64, d_fb7f36ad4d8ffe84, m_fb7f36ad4d8ffe84, + 2, 2, i_fb7f36ad4d8ffe84, nullptr, nullptr, { &s_fb7f36ad4d8ffe84, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<36> b_d5fd459ad75e86a9 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 169, 134, 94, 215, 154, 69, 253, 213, + 30, 0, 0, 0, 1, 0, 0, 0, + 132, 254, 143, 77, 173, 54, 127, 251, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 114, 111, 117, 112, 67, + 114, 101, 97, 116, 101, 46, 71, 114, + 111, 117, 112, 67, 114, 101, 97, 116, + 101, 68, 101, 116, 97, 105, 108, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 20, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d5fd459ad75e86a9 = b_d5fd459ad75e86a9.words; #if !CAPNP_LITE static const uint16_t m_d5fd459ad75e86a9[] = {0}; static const uint16_t i_d5fd459ad75e86a9[] = {0}; const ::capnp::_::RawSchema s_d5fd459ad75e86a9 = { - 0xd5fd459ad75e86a9, - b_d5fd459ad75e86a9.words, - 36, - nullptr, - m_d5fd459ad75e86a9, - 0, - 1, - i_d5fd459ad75e86a9, - nullptr, - nullptr, - {&s_d5fd459ad75e86a9, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<119> b_c0c730b5390f4427 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 39, 68, 15, 57, 181, 48, 199, - 192, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 108, 111, 98, 97, 108, - 87, 114, 105, 116, 101, 83, 116, 97, 116, 101, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 20, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, - 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, - 0, 3, 0, 1, 0, 136, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 3, 0, 1, - 0, 144, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 141, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 140, 0, 0, 0, 3, 0, 1, 0, 152, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, - 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, 164, 0, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 178, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 0, 0, 0, - 3, 0, 1, 0, 248, 0, 0, 0, 2, 0, 1, 0, 99, 101, 108, - 108, 115, 87, 114, 105, 116, 116, 101, 110, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 66, 15, 14, 103, 60, 152, 197, 211, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, 77, 101, - 116, 97, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 97, 202, 231, 39, 252, 82, 227, 205, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 108, 97, 115, 116, 67, 101, 108, 108, 67, - 111, 111, 114, 100, 115, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, - 115, 183, 55, 236, 13, 60, 189, 216, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 108, 97, 115, 116, 72, 105, 108, 98, 101, 114, 116, 86, - 97, 108, 117, 101, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 109, 117, 108, 116, 105, 80, 97, - 114, 116, 85, 112, 108, 111, 97, 100, 83, 116, 97, 116, 101, 115, 0, - 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 140, 113, 113, 174, 148, - 193, 121, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 1, 0, 0, 0, 31, 0, 0, 0, 4, 0, 0, - 0, 2, 0, 1, 0, 140, 113, 113, 174, 148, 193, 121, 241, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 39, 0, 0, 0, 8, - 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 0, 0, 245, 59, 94, 77, 115, 182, 146, 212, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, -}}; + 0xd5fd459ad75e86a9, b_d5fd459ad75e86a9.words, 36, nullptr, m_d5fd459ad75e86a9, + 0, 1, i_d5fd459ad75e86a9, nullptr, nullptr, { &s_d5fd459ad75e86a9, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<119> b_c0c730b5390f4427 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 39, 68, 15, 57, 181, 48, 199, 192, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 71, 108, 111, 98, 97, 108, + 87, 114, 105, 116, 101, 83, 116, 97, + 116, 101, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 0, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 0, 0, 0, 3, 0, 1, 0, + 164, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 161, 0, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, + 99, 101, 108, 108, 115, 87, 114, 105, + 116, 116, 101, 110, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 66, 15, 14, 103, 60, 152, 197, 211, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 77, 101, 116, 97, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 67, 101, 108, 108, + 67, 111, 111, 114, 100, 115, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 115, 183, 55, 236, 13, 60, 189, 216, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 72, 105, 108, 98, + 101, 114, 116, 86, 97, 108, 117, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 109, 117, 108, 116, 105, 80, 97, 114, + 116, 85, 112, 108, 111, 97, 100, 83, + 116, 97, 116, 101, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 0, 0, 31, 0, 0, 0, + 4, 0, 0, 0, 2, 0, 1, 0, + 140, 113, 113, 174, 148, 193, 121, 241, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 39, 0, 0, 0, + 8, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 245, 59, 94, 77, 115, 182, 146, 212, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_c0c730b5390f4427 = b_c0c730b5390f4427.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_c0c730b5390f4427[] = { - &s_cde352fc27e7ca61, - &s_d3c5983c670e0f42, - &s_d8bd3c0dec37b773, - &s_f179c194ae71718c, + &s_cde352fc27e7ca61, + &s_d3c5983c670e0f42, + &s_d8bd3c0dec37b773, + &s_f179c194ae71718c, }; static const uint16_t m_c0c730b5390f4427[] = {0, 1, 2, 3, 4}; static const uint16_t i_c0c730b5390f4427[] = {0, 1, 2, 3, 4}; -KJ_CONSTEXPR(const)::capnp::_::RawBrandedSchema::Dependency - bd_c0c730b5390f4427[] = { - {16777220, - ::tiledb::sm::serialization::capnp::Map< - ::capnp::Text, - ::tiledb::sm::serialization::capnp::MultiPartUploadState>:: - _capnpPrivate::brand()}, +KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_c0c730b5390f4427[] = { + { 16777220, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::MultiPartUploadState>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_c0c730b5390f4427 = { - 0xc0c730b5390f4427, - b_c0c730b5390f4427.words, - 119, - d_c0c730b5390f4427, - m_c0c730b5390f4427, - 4, - 5, - i_c0c730b5390f4427, - nullptr, - nullptr, - {&s_c0c730b5390f4427, - nullptr, - bd_c0c730b5390f4427, - 0, - sizeof(bd_c0c730b5390f4427) / sizeof(bd_c0c730b5390f4427[0]), - nullptr}, - true}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_d8bd3c0dec37b773 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 115, 183, 55, 236, 13, 60, 189, - 216, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, - 67, 111, 111, 114, 100, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 12, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 69, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 64, 0, 0, 0, 3, 0, 1, 0, 108, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, - 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 0, 0, 0, 3, 0, 1, 0, 128, 0, 0, 0, 2, 0, 1, - 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, 106, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, - 3, 0, 1, 0, 152, 0, 0, 0, 2, 0, 1, 0, 99, 111, 111, - 114, 100, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 6, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 115, 105, 122, 101, 115, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 105, 110, 103, - 108, 101, 79, 102, 102, 115, 101, 116, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xc0c730b5390f4427, b_c0c730b5390f4427.words, 119, d_c0c730b5390f4427, m_c0c730b5390f4427, + 4, 5, i_c0c730b5390f4427, nullptr, nullptr, { &s_c0c730b5390f4427, nullptr, bd_c0c730b5390f4427, 0, sizeof(bd_c0c730b5390f4427) / sizeof(bd_c0c730b5390f4427[0]), nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<80> b_d8bd3c0dec37b773 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 115, 183, 55, 236, 13, 60, 189, 216, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 83, 105, 110, 103, 108, 101, + 67, 111, 111, 114, 100, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 128, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 111, 114, 100, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 115, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 110, 103, 108, 101, 79, 102, + 102, 115, 101, 116, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d8bd3c0dec37b773 = b_d8bd3c0dec37b773.words; #if !CAPNP_LITE static const uint16_t m_d8bd3c0dec37b773[] = {0, 2, 1}; static const uint16_t i_d8bd3c0dec37b773[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d8bd3c0dec37b773 = { - 0xd8bd3c0dec37b773, - b_d8bd3c0dec37b773.words, - 80, - nullptr, - m_d8bd3c0dec37b773, - 0, - 3, - i_d8bd3c0dec37b773, - nullptr, - nullptr, - {&s_d8bd3c0dec37b773, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<612> b_cde352fc27e7ca61 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 97, 202, 231, 39, 252, 82, 227, - 205, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 22, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 26, 1, 0, 0, 37, 0, 0, 0, - 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, - 0, 95, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, - 110, 116, 77, 101, 116, 97, 100, 97, 116, 97, 0, 0, 0, 0, 0, - 0, 4, 0, 0, 0, 1, 0, 1, 0, 158, 107, 3, 136, 78, 143, - 170, 137, 1, 0, 0, 0, 154, 0, 0, 0, 71, 101, 110, 101, 114, - 105, 99, 84, 105, 108, 101, 79, 102, 102, 115, 101, 116, 115, 0, 0, - 0, 0, 0, 0, 116, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 29, 3, 0, 0, 82, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 28, 3, 0, 0, 3, 0, 1, 0, - 56, 3, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 53, 3, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 52, 3, 0, 0, 3, 0, 1, 0, 80, 3, 0, 0, - 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 3, - 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 80, - 3, 0, 0, 3, 0, 1, 0, 108, 3, 0, 0, 2, 0, 1, 0, - 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 3, 0, 0, 98, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 3, 0, 0, 3, - 0, 1, 0, 116, 3, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 113, 3, 0, 0, 114, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 112, 3, 0, 0, 3, 0, 1, 0, 124, - 3, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 121, 3, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 120, 3, 0, 0, 3, 0, 1, 0, 132, 3, 0, 0, 2, - 0, 1, 0, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, - 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 3, 0, - 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 3, - 0, 0, 3, 0, 1, 0, 140, 3, 0, 0, 2, 0, 1, 0, 7, - 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 137, 3, 0, 0, 114, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 3, 0, 0, 3, 0, - 1, 0, 148, 3, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 4, - 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 145, 3, 0, 0, 98, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 144, 3, 0, 0, 3, 0, 1, 0, 188, 3, - 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 5, 0, 0, 0, 0, - 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 185, 3, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 184, 3, 0, 0, 3, 0, 1, 0, 228, 3, 0, 0, 2, 0, - 1, 0, 10, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 10, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 225, 3, 0, 0, - 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 3, 0, - 0, 3, 0, 1, 0, 12, 4, 0, 0, 2, 0, 1, 0, 11, 0, - 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 9, 4, 0, 0, 162, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 12, 4, 0, 0, 3, 0, 1, - 0, 56, 4, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 8, 0, - 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 53, 4, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 52, 4, 0, 0, 3, 0, 1, 0, 96, 4, 0, - 0, 2, 0, 1, 0, 13, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, - 4, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 4, 0, 0, 3, 0, 1, 0, 140, 4, 0, 0, 2, 0, 1, - 0, 14, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 4, 0, 0, 114, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 4, 0, 0, - 3, 0, 1, 0, 180, 4, 0, 0, 2, 0, 1, 0, 15, 0, 0, - 0, 11, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 177, 4, 0, 0, 138, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 180, 4, 0, 0, 3, 0, 1, 0, - 224, 4, 0, 0, 2, 0, 1, 0, 16, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 221, 4, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 220, 4, 0, 0, 3, 0, 1, 0, 8, 5, 0, 0, - 2, 0, 1, 0, 17, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, - 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, - 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, - 5, 0, 0, 3, 0, 1, 0, 48, 5, 0, 0, 2, 0, 1, 0, - 18, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 5, 0, 0, 106, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 5, 0, 0, 3, - 0, 1, 0, 88, 5, 0, 0, 2, 0, 1, 0, 19, 0, 0, 0, - 15, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 85, 5, 0, 0, 106, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 84, 5, 0, 0, 3, 0, 1, 0, 128, - 5, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 16, 0, 0, 0, - 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 125, 5, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 124, 5, 0, 0, 3, 0, 1, 0, 152, 5, 0, 0, 2, - 0, 1, 0, 21, 0, 0, 0, 17, 0, 0, 0, 0, 0, 1, 0, - 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 5, 0, - 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 152, 5, - 0, 0, 3, 0, 1, 0, 180, 5, 0, 0, 2, 0, 1, 0, 22, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 22, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 177, 5, 0, 0, 66, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 5, 0, 0, 3, 0, - 1, 0, 184, 5, 0, 0, 2, 0, 1, 0, 23, 0, 0, 0, 18, - 0, 0, 0, 0, 0, 1, 0, 23, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 181, 5, 0, 0, 122, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 180, 5, 0, 0, 3, 0, 1, 0, 208, 5, - 0, 0, 2, 0, 1, 0, 24, 0, 0, 0, 3, 0, 0, 0, 0, - 0, 1, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 5, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 204, 5, 0, 0, 3, 0, 1, 0, 216, 5, 0, 0, 2, 0, - 1, 0, 25, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1, 0, 25, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 5, 0, 0, - 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 5, 0, - 0, 3, 0, 1, 0, 224, 5, 0, 0, 2, 0, 1, 0, 26, 0, - 0, 0, 20, 0, 0, 0, 0, 0, 1, 0, 26, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 221, 5, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 216, 5, 0, 0, 3, 0, 1, - 0, 228, 5, 0, 0, 2, 0, 1, 0, 27, 0, 0, 0, 2, 0, - 0, 0, 0, 0, 1, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 225, 5, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 228, 5, 0, 0, 3, 0, 1, 0, 240, 5, 0, - 0, 2, 0, 1, 0, 28, 0, 0, 0, 21, 0, 0, 0, 0, 0, - 1, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, - 5, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 236, 5, 0, 0, 3, 0, 1, 0, 248, 5, 0, 0, 2, 0, 1, - 0, 102, 105, 108, 101, 83, 105, 122, 101, 115, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 102, 105, 108, 101, 86, 97, 114, 83, 105, 122, 101, 115, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, - 105, 108, 101, 86, 97, 108, 105, 100, 105, 116, 121, 83, 105, 122, 101, - 115, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, 109, 101, 110, 116, 85, - 114, 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 104, 97, 115, 84, 105, 109, 101, 115, 116, 97, 109, 112, - 115, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 97, 115, 68, 101, 108, 101, 116, 101, 77, 101, 116, 97, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 112, 97, - 114, 115, 101, 84, 105, 108, 101, 78, 117, 109, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 73, 110, - 100, 101, 120, 66, 97, 115, 101, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 79, 102, 102, 115, 101, - 116, 115, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, 114, 79, - 102, 102, 115, 101, 116, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, 114, - 83, 105, 122, 101, 115, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, - 108, 105, 100, 105, 116, 121, 79, 102, 102, 115, 101, 116, 115, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 116, 105, 108, 101, 77, 105, 110, 66, 117, 102, 102, 101, - 114, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 116, 105, 108, 101, 77, 105, 110, 86, 97, 114, 66, - 117, 102, 102, 101, 114, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, - 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, - 108, 101, 77, 97, 120, 66, 117, 102, 102, 101, 114, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, - 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, - 105, 108, 101, 77, 97, 120, 86, 97, 114, 66, 117, 102, 102, 101, 114, - 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 6, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 83, 117, 109, - 115, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 6, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 78, 117, - 108, 108, 67, 111, 117, 110, 116, 115, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, 109, - 101, 110, 116, 77, 105, 110, 115, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, - 109, 101, 110, 116, 77, 97, 120, 115, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 114, 97, - 103, 109, 101, 110, 116, 83, 117, 109, 115, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 114, 97, 103, - 109, 101, 110, 116, 78, 117, 108, 108, 67, 111, 117, 110, 116, 115, 0, - 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 118, 101, 114, 115, 105, 111, 110, 0, 8, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 116, 105, 109, 101, 115, 116, 97, 109, - 112, 82, 97, 110, 103, 101, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 108, 97, 115, 116, 84, 105, 108, 101, 67, - 101, 108, 108, 78, 117, 109, 0, 9, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 110, 111, 110, 69, 109, 112, 116, 121, 68, 111, 109, 97, - 105, 110, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 227, 236, 72, - 148, 84, 100, 130, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 114, 116, 114, 101, 101, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 104, 97, 115, 67, 111, 110, 115, 111, 108, 105, 100, - 97, 116, 101, 100, 70, 111, 111, 116, 101, 114, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 116, 79, 102, 102, 115, - 101, 116, 115, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 158, 107, 3, 136, 78, 143, 170, 137, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, -}}; + 0xd8bd3c0dec37b773, b_d8bd3c0dec37b773.words, 80, nullptr, m_d8bd3c0dec37b773, + 0, 3, i_d8bd3c0dec37b773, nullptr, nullptr, { &s_d8bd3c0dec37b773, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<612> b_cde352fc27e7ca61 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 18, 0, 0, 0, 1, 0, 4, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 22, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 0, 0, 0, 95, 6, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 114, 97, 103, 109, 101, + 110, 116, 77, 101, 116, 97, 100, 97, + 116, 97, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 1, 0, 1, 0, + 158, 107, 3, 136, 78, 143, 170, 137, + 1, 0, 0, 0, 154, 0, 0, 0, + 71, 101, 110, 101, 114, 105, 99, 84, + 105, 108, 101, 79, 102, 102, 115, 101, + 116, 115, 0, 0, 0, 0, 0, 0, + 116, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 3, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 28, 3, 0, 0, 3, 0, 1, 0, + 56, 3, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 3, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 3, 0, 0, 3, 0, 1, 0, + 80, 3, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 3, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 3, 0, 0, 3, 0, 1, 0, + 108, 3, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 3, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 3, 0, 0, 3, 0, 1, 0, + 116, 3, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 3, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 3, 0, 0, 3, 0, 1, 0, + 124, 3, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 121, 3, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 120, 3, 0, 0, 3, 0, 1, 0, + 132, 3, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 129, 3, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 3, 0, 0, 3, 0, 1, 0, + 140, 3, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 3, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 3, 0, 0, 3, 0, 1, 0, + 148, 3, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 3, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 3, 0, 0, 3, 0, 1, 0, + 188, 3, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 3, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 3, 0, 0, 3, 0, 1, 0, + 228, 3, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 225, 3, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 224, 3, 0, 0, 3, 0, 1, 0, + 12, 4, 0, 0, 2, 0, 1, 0, + 11, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 11, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 4, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 4, 0, 0, 3, 0, 1, 0, + 56, 4, 0, 0, 2, 0, 1, 0, + 12, 0, 0, 0, 8, 0, 0, 0, + 0, 0, 1, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 53, 4, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 52, 4, 0, 0, 3, 0, 1, 0, + 96, 4, 0, 0, 2, 0, 1, 0, + 13, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 1, 0, 13, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 93, 4, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 96, 4, 0, 0, 3, 0, 1, 0, + 140, 4, 0, 0, 2, 0, 1, 0, + 14, 0, 0, 0, 10, 0, 0, 0, + 0, 0, 1, 0, 14, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 137, 4, 0, 0, 114, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 4, 0, 0, 3, 0, 1, 0, + 180, 4, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 11, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 4, 0, 0, 138, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 4, 0, 0, 3, 0, 1, 0, + 224, 4, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 221, 4, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 220, 4, 0, 0, 3, 0, 1, 0, + 8, 5, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 5, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 4, 5, 0, 0, 3, 0, 1, 0, + 48, 5, 0, 0, 2, 0, 1, 0, + 18, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 18, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 5, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 5, 0, 0, 3, 0, 1, 0, + 88, 5, 0, 0, 2, 0, 1, 0, + 19, 0, 0, 0, 15, 0, 0, 0, + 0, 0, 1, 0, 19, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 85, 5, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 84, 5, 0, 0, 3, 0, 1, 0, + 128, 5, 0, 0, 2, 0, 1, 0, + 20, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 20, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 5, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 5, 0, 0, 3, 0, 1, 0, + 152, 5, 0, 0, 2, 0, 1, 0, + 21, 0, 0, 0, 17, 0, 0, 0, + 0, 0, 1, 0, 21, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 5, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 152, 5, 0, 0, 3, 0, 1, 0, + 180, 5, 0, 0, 2, 0, 1, 0, + 22, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 22, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 177, 5, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 5, 0, 0, 3, 0, 1, 0, + 184, 5, 0, 0, 2, 0, 1, 0, + 23, 0, 0, 0, 18, 0, 0, 0, + 0, 0, 1, 0, 23, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 5, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 180, 5, 0, 0, 3, 0, 1, 0, + 208, 5, 0, 0, 2, 0, 1, 0, + 24, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 24, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 5, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 5, 0, 0, 3, 0, 1, 0, + 216, 5, 0, 0, 2, 0, 1, 0, + 25, 0, 0, 0, 19, 0, 0, 0, + 0, 0, 1, 0, 25, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 5, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 212, 5, 0, 0, 3, 0, 1, 0, + 224, 5, 0, 0, 2, 0, 1, 0, + 26, 0, 0, 0, 20, 0, 0, 0, + 0, 0, 1, 0, 26, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 221, 5, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 216, 5, 0, 0, 3, 0, 1, 0, + 228, 5, 0, 0, 2, 0, 1, 0, + 27, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 27, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 225, 5, 0, 0, 178, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 228, 5, 0, 0, 3, 0, 1, 0, + 240, 5, 0, 0, 2, 0, 1, 0, + 28, 0, 0, 0, 21, 0, 0, 0, + 0, 0, 1, 0, 28, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 5, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 5, 0, 0, 3, 0, 1, 0, + 248, 5, 0, 0, 2, 0, 1, 0, + 102, 105, 108, 101, 83, 105, 122, 101, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 101, 86, 97, 114, 83, + 105, 122, 101, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 105, 108, 101, 86, 97, 108, 105, + 100, 105, 116, 121, 83, 105, 122, 101, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 85, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 97, 115, 84, 105, 109, 101, 115, + 116, 97, 109, 112, 115, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 97, 115, 68, 101, 108, 101, 116, + 101, 77, 101, 116, 97, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 112, 97, 114, 115, 101, 84, 105, + 108, 101, 78, 117, 109, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 73, 110, 100, 101, + 120, 66, 97, 115, 101, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 79, 102, 102, 115, + 101, 116, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 114, 79, + 102, 102, 115, 101, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 114, 83, + 105, 122, 101, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 108, 105, + 100, 105, 116, 121, 79, 102, 102, 115, + 101, 116, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 105, 110, 66, + 117, 102, 102, 101, 114, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 105, 110, 86, + 97, 114, 66, 117, 102, 102, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 97, 120, 66, + 117, 102, 102, 101, 114, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 97, 120, 86, + 97, 114, 66, 117, 102, 102, 101, 114, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 83, 117, 109, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 78, 117, 108, 108, + 67, 111, 117, 110, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 77, 105, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 77, 97, 120, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 6, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 83, 117, 109, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 78, 117, 108, 108, 67, 111, 117, 110, + 116, 115, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 118, 101, 114, 115, 105, 111, 110, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 82, 97, 110, 103, 101, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 108, 97, 115, 116, 84, 105, 108, 101, + 67, 101, 108, 108, 78, 117, 109, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 111, 110, 69, 109, 112, 116, 121, + 68, 111, 109, 97, 105, 110, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 227, 236, 72, 148, 84, 100, 130, 161, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 114, 116, 114, 101, 101, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 104, 97, 115, 67, 111, 110, 115, 111, + 108, 105, 100, 97, 116, 101, 100, 70, + 111, 111, 116, 101, 114, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 103, 116, 79, 102, 102, 115, 101, 116, + 115, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 158, 107, 3, 136, 78, 143, 170, 137, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_cde352fc27e7ca61 = b_cde352fc27e7ca61.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_cde352fc27e7ca61[] = { - &s_89aa8f4e88036b9e, - &s_a18264549448ece3, -}; -static const uint16_t m_cde352fc27e7ca61[] = { - 0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, - 6, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; -static const uint16_t i_cde352fc27e7ca61[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}; + &s_89aa8f4e88036b9e, + &s_a18264549448ece3, +}; +static const uint16_t m_cde352fc27e7ca61[] = {0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, 6, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; +static const uint16_t i_cde352fc27e7ca61[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}; const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { - 0xcde352fc27e7ca61, - b_cde352fc27e7ca61.words, - 612, - d_cde352fc27e7ca61, - m_cde352fc27e7ca61, - 2, - 29, - i_cde352fc27e7ca61, - nullptr, - nullptr, - {&s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<235> b_89aa8f4e88036b9e = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 158, 107, 3, 136, 78, 143, 170, - 137, 35, 0, 0, 0, 1, 0, 3, 0, 97, 202, 231, 39, 252, 82, - 227, 205, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 178, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, - 110, 116, 77, 101, 116, 97, 100, 97, 116, 97, 46, 71, 101, 110, 101, - 114, 105, 99, 84, 105, 108, 101, 79, 102, 102, 115, 101, 116, 115, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 44, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 1, 0, - 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 1, - 0, 0, 3, 0, 1, 0, 44, 1, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 41, 1, 0, 0, 98, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 1, 0, 0, 3, 0, - 1, 0, 68, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 65, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 64, 1, 0, 0, 3, 0, 1, 0, 92, 1, - 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, - 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 1, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 88, 1, 0, 0, 3, 0, 1, 0, 116, 1, 0, 0, 2, 0, - 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 1, 0, 0, - 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 1, 0, - 0, 3, 0, 1, 0, 144, 1, 0, 0, 2, 0, 1, 0, 5, 0, - 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 141, 1, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 140, 1, 0, 0, 3, 0, 1, - 0, 168, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, - 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 165, 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 164, 1, 0, 0, 3, 0, 1, 0, 192, 1, 0, - 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, - 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, - 1, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, 216, 1, 0, 0, 2, 0, 1, - 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 1, 0, 0, 170, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 1, 0, 0, - 3, 0, 1, 0, 244, 1, 0, 0, 2, 0, 1, 0, 9, 0, 0, - 0, 1, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 241, 1, 0, 0, 10, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 252, 1, 0, 0, 3, 0, 1, 0, - 8, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 2, 0, 0, - 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 5, 2, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 12, 2, 0, 0, 3, 0, 1, 0, 24, 2, 0, 0, - 2, 0, 1, 0, 114, 116, 114, 101, 101, 0, 0, 0, 9, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 79, 102, 102, - 115, 101, 116, 115, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, 114, 79, - 102, 102, 115, 101, 116, 115, 0, 0, 14, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, 114, 83, 105, - 122, 101, 115, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 116, 105, 108, 101, 86, 97, 108, 105, 100, 105, - 116, 121, 79, 102, 102, 115, 101, 116, 115, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, - 101, 77, 105, 110, 79, 102, 102, 115, 101, 116, 115, 0, 0, 14, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, - 77, 97, 120, 79, 102, 102, 115, 101, 116, 115, 0, 0, 14, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 83, - 117, 109, 79, 102, 102, 115, 101, 116, 115, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 78, 117, - 108, 108, 67, 111, 117, 110, 116, 79, 102, 102, 115, 101, 116, 115, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 102, 114, 97, 103, 109, 101, 110, 116, 77, 105, 110, 77, 97, 120, - 83, 117, 109, 78, 117, 108, 108, 67, 111, 117, 110, 116, 79, 102, 102, - 115, 101, 116, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 112, 114, 111, 99, 101, 115, 115, 101, - 100, 67, 111, 110, 100, 105, 116, 105, 111, 110, 115, 79, 102, 102, 115, - 101, 116, 115, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 612, d_cde352fc27e7ca61, m_cde352fc27e7ca61, + 2, 29, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<235> b_89aa8f4e88036b9e = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 158, 107, 3, 136, 78, 143, 170, 137, + 35, 0, 0, 0, 1, 0, 3, 0, + 97, 202, 231, 39, 252, 82, 227, 205, + 8, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 111, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 70, 114, 97, 103, 109, 101, + 110, 116, 77, 101, 116, 97, 100, 97, + 116, 97, 46, 71, 101, 110, 101, 114, + 105, 99, 84, 105, 108, 101, 79, 102, + 102, 115, 101, 116, 115, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 44, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 44, 1, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 1, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 1, 0, 0, 3, 0, 1, 0, + 68, 1, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 65, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 1, 0, 0, 3, 0, 1, 0, + 92, 1, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 89, 1, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 88, 1, 0, 0, 3, 0, 1, 0, + 116, 1, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 1, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 1, 0, 0, 3, 0, 1, 0, + 144, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 1, 0, 0, 3, 0, 1, 0, + 168, 1, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 165, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 164, 1, 0, 0, 3, 0, 1, 0, + 192, 1, 0, 0, 2, 0, 1, 0, + 7, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 1, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 189, 1, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 216, 1, 0, 0, 3, 0, 1, 0, + 244, 1, 0, 0, 2, 0, 1, 0, + 9, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 241, 1, 0, 0, 10, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 252, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, + 10, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 5, 2, 0, 0, 218, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 2, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, + 114, 116, 114, 101, 101, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 79, 102, 102, 115, + 101, 116, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 114, 79, + 102, 102, 115, 101, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 114, 83, + 105, 122, 101, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 86, 97, 108, 105, + 100, 105, 116, 121, 79, 102, 102, 115, + 101, 116, 115, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 105, 110, 79, + 102, 102, 115, 101, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 77, 97, 120, 79, + 102, 102, 115, 101, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 83, 117, 109, 79, + 102, 102, 115, 101, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 78, 117, 108, 108, + 67, 111, 117, 110, 116, 79, 102, 102, + 115, 101, 116, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 77, 105, 110, 77, 97, 120, 83, 117, + 109, 78, 117, 108, 108, 67, 111, 117, + 110, 116, 79, 102, 102, 115, 101, 116, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 114, 111, 99, 101, 115, 115, 101, + 100, 67, 111, 110, 100, 105, 116, 105, + 111, 110, 115, 79, 102, 102, 115, 101, + 116, 115, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_89aa8f4e88036b9e = b_89aa8f4e88036b9e.words; #if !CAPNP_LITE static const uint16_t m_89aa8f4e88036b9e[] = {9, 10, 0, 6, 5, 8, 1, 7, 4, 2, 3}; static const uint16_t i_89aa8f4e88036b9e[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; const ::capnp::_::RawSchema s_89aa8f4e88036b9e = { - 0x89aa8f4e88036b9e, - b_89aa8f4e88036b9e.words, - 235, - nullptr, - m_89aa8f4e88036b9e, - 0, - 11, - i_89aa8f4e88036b9e, - nullptr, - nullptr, - {&s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<106> b_d492b6734d5e3bf5 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 245, 59, 94, 77, 115, 182, 146, - 212, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 117, 108, 116, 105, 80, - 97, 114, 116, 85, 112, 108, 111, 97, 100, 83, 116, 97, 116, 101, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 20, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 125, 0, 0, 0, - 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, - 0, 3, 0, 1, 0, 136, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 74, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 3, 0, 1, - 0, 144, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 141, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 136, 0, 0, 0, 3, 0, 1, 0, 148, 0, 0, - 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, - 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, - 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 144, 0, 0, 0, 3, 0, 1, 0, 172, 0, 0, 0, 2, 0, 1, - 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 0, 0, 0, 122, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 0, 0, - 3, 0, 1, 0, 196, 0, 0, 0, 2, 0, 1, 0, 112, 97, 114, - 116, 78, 117, 109, 98, 101, 114, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 117, 112, 108, 111, 97, 100, - 73, 100, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 117, 115, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 111, 109, 112, 108, - 101, 116, 101, 100, 80, 97, 114, 116, 115, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, - 0, 0, 0, 0, 0, 0, 37, 134, 61, 177, 215, 235, 232, 189, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 117, 102, 102, 101, 114, - 101, 100, 67, 104, 117, 110, 107, 115, 0, 0, 14, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 63, 176, 196, 171, 157, 188, 138, 205, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 235, nullptr, m_89aa8f4e88036b9e, + 0, 11, i_89aa8f4e88036b9e, nullptr, nullptr, { &s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<106> b_d492b6734d5e3bf5 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 245, 59, 94, 77, 115, 182, 146, 212, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 4, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 77, 117, 108, 116, 105, 80, + 97, 114, 116, 85, 112, 108, 111, 97, + 100, 83, 116, 97, 116, 101, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 20, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 125, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 136, 0, 0, 0, 3, 0, 1, 0, + 148, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 145, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 144, 0, 0, 0, 3, 0, 1, 0, + 172, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 169, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 168, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 112, 97, 114, 116, 78, 117, 109, 98, + 101, 114, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 112, 108, 111, 97, 100, 73, 100, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 117, 115, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 111, 109, 112, 108, 101, 116, 101, + 100, 80, 97, 114, 116, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 37, 134, 61, 177, 215, 235, 232, 189, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 98, 117, 102, 102, 101, 114, 101, 100, + 67, 104, 117, 110, 107, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 63, 176, 196, 171, 157, 188, 138, 205, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_d492b6734d5e3bf5 = b_d492b6734d5e3bf5.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_d492b6734d5e3bf5[] = { - &s_bde8ebd7b13d8625, - &s_cd8abc9dabc4b03f, + &s_bde8ebd7b13d8625, + &s_cd8abc9dabc4b03f, }; static const uint16_t m_d492b6734d5e3bf5[] = {4, 3, 0, 2, 1}; static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_d492b6734d5e3bf5 = { - 0xd492b6734d5e3bf5, - b_d492b6734d5e3bf5.words, - 106, - d_d492b6734d5e3bf5, - m_d492b6734d5e3bf5, - 2, - 5, - i_d492b6734d5e3bf5, - nullptr, - nullptr, - {&s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 37, 134, 61, 177, 215, 235, 232, - 189, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 109, 112, 108, 101, - 116, 101, 100, 80, 97, 114, 116, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 0, 0, 0, 3, 0, 1, 0, 56, 0, 0, 0, 2, 0, 1, - 0, 101, 84, 97, 103, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 112, 97, 114, 116, 78, 117, 109, 98, 101, 114, - 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 106, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, + 2, 5, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 37, 134, 61, 177, 215, 235, 232, 189, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 109, 112, 108, 101, + 116, 101, 100, 80, 97, 114, 116, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 101, 84, 97, 103, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 114, 116, 78, 117, 109, 98, + 101, 114, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_bde8ebd7b13d8625 = b_bde8ebd7b13d8625.words; #if !CAPNP_LITE static const uint16_t m_bde8ebd7b13d8625[] = {0, 1}; static const uint16_t i_bde8ebd7b13d8625[] = {0, 1}; const ::capnp::_::RawSchema s_bde8ebd7b13d8625 = { - 0xbde8ebd7b13d8625, - b_bde8ebd7b13d8625.words, - 49, - nullptr, - m_bde8ebd7b13d8625, - 0, - 2, - i_bde8ebd7b13d8625, - nullptr, - nullptr, - {&s_bde8ebd7b13d8625, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<54> b_a736c51d292ca752 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 82, 167, 44, 41, 29, 197, 54, - 167, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 50, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 116, 101, - 110, 70, 114, 97, 103, 109, 101, 110, 116, 73, 110, 102, 111, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, - 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, - 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 122, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 3, 0, 1, - 0, 72, 0, 0, 0, 2, 0, 1, 0, 117, 114, 105, 0, 0, 0, - 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, - 109, 101, 115, 116, 97, 109, 112, 82, 97, 110, 103, 101, 0, 0, 14, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, - 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xbde8ebd7b13d8625, b_bde8ebd7b13d8625.words, 49, nullptr, m_bde8ebd7b13d8625, + 0, 2, i_bde8ebd7b13d8625, nullptr, nullptr, { &s_bde8ebd7b13d8625, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<54> b_a736c51d292ca752 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 82, 167, 44, 41, 29, 197, 54, 167, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 87, 114, 105, 116, 116, 101, + 110, 70, 114, 97, 103, 109, 101, 110, + 116, 73, 110, 102, 111, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 72, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 109, 101, 115, 116, 97, 109, + 112, 82, 97, 110, 103, 101, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_a736c51d292ca752 = b_a736c51d292ca752.words; #if !CAPNP_LITE static const uint16_t m_a736c51d292ca752[] = {1, 0}; static const uint16_t i_a736c51d292ca752[] = {0, 1}; const ::capnp::_::RawSchema s_a736c51d292ca752 = { - 0xa736c51d292ca752, - b_a736c51d292ca752.words, - 54, - nullptr, - m_a736c51d292ca752, - 0, - 2, - i_a736c51d292ca752, - nullptr, - nullptr, - {&s_a736c51d292ca752, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_cd8abc9dabc4b03f = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 63, 176, 196, 171, 157, 188, 138, - 205, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 2, 1, 0, 0, 33, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 66, 117, 102, 102, 101, 114, - 101, 100, 67, 104, 117, 110, 107, 0, 0, 0, 0, 0, 1, 0, 1, - 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 41, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, 0, - 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, - 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 40, 0, 0, 0, 3, 0, 1, 0, 52, 0, 0, 0, 2, 0, 1, - 0, 117, 114, 105, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 115, 105, 122, 101, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xa736c51d292ca752, b_a736c51d292ca752.words, 54, nullptr, m_a736c51d292ca752, + 0, 2, i_a736c51d292ca752, nullptr, nullptr, { &s_a736c51d292ca752, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<48> b_cd8abc9dabc4b03f = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 63, 176, 196, 171, 157, 188, 138, 205, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 66, 117, 102, 102, 101, 114, + 101, 100, 67, 104, 117, 110, 107, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 34, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 117, 114, 105, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 105, 122, 101, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_cd8abc9dabc4b03f = b_cd8abc9dabc4b03f.words; #if !CAPNP_LITE static const uint16_t m_cd8abc9dabc4b03f[] = {1, 0}; static const uint16_t i_cd8abc9dabc4b03f[] = {0, 1}; const ::capnp::_::RawSchema s_cd8abc9dabc4b03f = { - 0xcd8abc9dabc4b03f, - b_cd8abc9dabc4b03f.words, - 48, - nullptr, - m_cd8abc9dabc4b03f, - 0, - 2, - i_cd8abc9dabc4b03f, - nullptr, - nullptr, - {&s_cd8abc9dabc4b03f, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<55> b_cfea684b4bcd0721 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 33, 7, 205, 75, 75, 104, 234, - 207, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 146, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, - 101, 108, 101, 116, 101, 70, 114, 97, 103, 109, 101, 110, 116, 115, 76, - 105, 115, 116, 82, 101, 113, 117, 101, 115, 116, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, - 0, 0, 3, 0, 1, 0, 48, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 66, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 3, 0, - 1, 0, 68, 0, 0, 0, 2, 0, 1, 0, 117, 114, 105, 0, 0, - 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, - 110, 116, 114, 105, 101, 115, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xcd8abc9dabc4b03f, b_cd8abc9dabc4b03f.words, 48, nullptr, m_cd8abc9dabc4b03f, + 0, 2, i_cd8abc9dabc4b03f, nullptr, nullptr, { &s_cd8abc9dabc4b03f, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<55> b_cfea684b4bcd0721 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 33, 7, 205, 75, 75, 104, 234, 207, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 146, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 68, + 101, 108, 101, 116, 101, 70, 114, 97, + 103, 109, 101, 110, 116, 115, 76, 105, + 115, 116, 82, 101, 113, 117, 101, 115, + 116, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 68, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 116, 114, 105, 101, 115, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_cfea684b4bcd0721 = b_cfea684b4bcd0721.words; #if !CAPNP_LITE -static const uint16_t m_cfea684b4bcd0721[] = {1, 0}; +static const ::capnp::_::RawSchema* const d_cfea684b4bcd0721[] = { + &s_b6c95b4b8111ad36, +}; +static const uint16_t m_cfea684b4bcd0721[] = {0, 1}; static const uint16_t i_cfea684b4bcd0721[] = {0, 1}; const ::capnp::_::RawSchema s_cfea684b4bcd0721 = { - 0xcfea684b4bcd0721, - b_cfea684b4bcd0721.words, - 55, - nullptr, - m_cfea684b4bcd0721, - 0, - 2, - i_cfea684b4bcd0721, - nullptr, - nullptr, - {&s_cfea684b4bcd0721, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_aaeeafe1e9f3ea1c = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 28, 234, 243, 233, 225, 175, 238, - 170, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 194, 1, 0, 0, 45, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, - 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, - 101, 108, 101, 116, 101, 70, 114, 97, 103, 109, 101, 110, 116, 115, 84, - 105, 109, 101, 115, 116, 97, 109, 112, 115, 82, 101, 113, 117, 101, 115, - 116, 0, 0, 0, 0, 0, 1, 0, 1, 0, 12, 0, 0, 0, 3, - 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, - 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, - 0, 0, 3, 0, 1, 0, 76, 0, 0, 0, 2, 0, 1, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 122, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 0, 0, 0, 3, 0, - 1, 0, 84, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 81, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 80, 0, 0, 0, 3, 0, 1, 0, 92, 0, - 0, 0, 2, 0, 1, 0, 117, 114, 105, 0, 0, 0, 0, 0, 12, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 114, 116, - 84, 105, 109, 101, 115, 116, 97, 109, 112, 0, 0, 9, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 101, 110, 100, 84, 105, 109, 101, 115, - 116, 97, 109, 112, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, -}}; + 0xcfea684b4bcd0721, b_cfea684b4bcd0721.words, 55, d_cfea684b4bcd0721, m_cfea684b4bcd0721, + 1, 2, i_cfea684b4bcd0721, nullptr, nullptr, { &s_cfea684b4bcd0721, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<68> b_aaeeafe1e9f3ea1c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 28, 234, 243, 233, 225, 175, 238, 170, + 18, 0, 0, 0, 1, 0, 2, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 68, + 101, 108, 101, 116, 101, 70, 114, 97, + 103, 109, 101, 110, 116, 115, 84, 105, + 109, 101, 115, 116, 97, 109, 112, 115, + 82, 101, 113, 117, 101, 115, 116, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 0, 0, 3, 0, 1, 0, + 76, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 73, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 72, 0, 0, 0, 3, 0, 1, 0, + 84, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 81, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 114, 116, 84, 105, 109, + 101, 115, 116, 97, 109, 112, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 100, 84, 105, 109, 101, 115, + 116, 97, 109, 112, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_aaeeafe1e9f3ea1c = b_aaeeafe1e9f3ea1c.words; #if !CAPNP_LITE -static const uint16_t m_aaeeafe1e9f3ea1c[] = {2, 1, 0}; +static const ::capnp::_::RawSchema* const d_aaeeafe1e9f3ea1c[] = { + &s_b6c95b4b8111ad36, +}; +static const uint16_t m_aaeeafe1e9f3ea1c[] = {0, 2, 1}; static const uint16_t i_aaeeafe1e9f3ea1c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_aaeeafe1e9f3ea1c = { - 0xaaeeafe1e9f3ea1c, - b_aaeeafe1e9f3ea1c.words, - 68, - nullptr, - m_aaeeafe1e9f3ea1c, - 0, - 3, - i_aaeeafe1e9f3ea1c, - nullptr, - nullptr, - {&s_aaeeafe1e9f3ea1c, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 210, 148, 17, 3, 97, 86, 163, - 245, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 67, - 111, 110, 115, 111, 108, 105, 100, 97, 116, 105, 111, 110, 82, 101, 113, - 117, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 13, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 20, 0, - 0, 0, 2, 0, 1, 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0xaaeeafe1e9f3ea1c, b_aaeeafe1e9f3ea1c.words, 68, d_aaeeafe1e9f3ea1c, m_aaeeafe1e9f3ea1c, + 1, 3, i_aaeeafe1e9f3ea1c, nullptr, nullptr, { &s_aaeeafe1e9f3ea1c, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 210, 148, 17, 3, 97, 86, 163, 245, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 98, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 67, + 111, 110, 115, 111, 108, 105, 100, 97, + 116, 105, 111, 110, 82, 101, 113, 117, + 101, 115, 116, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 20, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_f5a35661031194d2 = b_f5a35661031194d2.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_f5a35661031194d2[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_f5a35661031194d2[] = {0}; static const uint16_t i_f5a35661031194d2[] = {0}; const ::capnp::_::RawSchema s_f5a35661031194d2 = { - 0xf5a35661031194d2, - b_f5a35661031194d2.words, - 35, - d_f5a35661031194d2, - m_f5a35661031194d2, - 1, - 1, - i_f5a35661031194d2, - nullptr, - nullptr, - {&s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<34> b_e68edfc0939e63df = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 223, 99, 158, 147, 192, 223, 142, - 230, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 42, 1, 0, 0, 37, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 86, - 97, 99, 117, 117, 109, 82, 101, 113, 117, 101, 115, 116, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, 0, 4, 0, 0, 0, 3, 0, - 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, - 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, - 0, 3, 0, 1, 0, 20, 0, 0, 0, 2, 0, 1, 0, 99, 111, - 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, - 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, -}}; + 0xf5a35661031194d2, b_f5a35661031194d2.words, 35, d_f5a35661031194d2, m_f5a35661031194d2, + 1, 1, i_f5a35661031194d2, nullptr, nullptr, { &s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<34> b_e68edfc0939e63df = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 223, 99, 158, 147, 192, 223, 142, 230, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 114, 114, 97, 121, 86, + 97, 99, 117, 117, 109, 82, 101, 113, + 117, 101, 115, 116, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 20, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_e68edfc0939e63df = b_e68edfc0939e63df.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_e68edfc0939e63df[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_e68edfc0939e63df[] = {0}; static const uint16_t i_e68edfc0939e63df[] = {0}; const ::capnp::_::RawSchema s_e68edfc0939e63df = { - 0xe68edfc0939e63df, - b_e68edfc0939e63df.words, - 34, - d_e68edfc0939e63df, - m_e68edfc0939e63df, - 1, - 1, - i_e68edfc0939e63df, - nullptr, - nullptr, - {&s_e68edfc0939e63df, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<55> b_891a70a671f15cf6 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 246, 92, 241, 113, 166, 112, 26, - 137, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, - 117, 109, 101, 114, 97, 116, 105, 111, 110, 115, 82, 101, 113, 117, 101, - 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, - 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 44, 0, 0, 0, 3, 0, 1, 0, 72, 0, 0, 0, 2, 0, - 1, 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 101, 110, 117, 109, 101, 114, 97, 116, 105, - 111, 110, 115, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0xe68edfc0939e63df, b_e68edfc0939e63df.words, 34, d_e68edfc0939e63df, m_e68edfc0939e63df, + 1, 1, i_e68edfc0939e63df, nullptr, nullptr, { &s_e68edfc0939e63df, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<55> b_891a70a671f15cf6 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 246, 92, 241, 113, 166, 112, 26, 137, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 82, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 76, 111, 97, 100, 69, 110, + 117, 109, 101, 114, 97, 116, 105, 111, + 110, 115, 82, 101, 113, 117, 101, 115, + 116, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 72, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_891a70a671f15cf6 = b_891a70a671f15cf6.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_891a70a671f15cf6[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_891a70a671f15cf6[] = {0, 1}; static const uint16_t i_891a70a671f15cf6[] = {0, 1}; const ::capnp::_::RawSchema s_891a70a671f15cf6 = { - 0x891a70a671f15cf6, - b_891a70a671f15cf6.words, - 55, - d_891a70a671f15cf6, - m_891a70a671f15cf6, - 1, - 2, - i_891a70a671f15cf6, - nullptr, - nullptr, - {&s_891a70a671f15cf6, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_805c080c10c1e959 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 89, 233, 193, 16, 12, 8, 92, - 128, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 90, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, - 117, 109, 101, 114, 97, 116, 105, 111, 110, 115, 82, 101, 115, 112, 111, - 110, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 13, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 1, 0, 40, 0, - 0, 0, 2, 0, 1, 0, 101, 110, 117, 109, 101, 114, 97, 116, 105, - 111, 110, 115, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 3, 0, 1, 0, 16, 0, 0, 0, 0, 0, - 0, 0, 180, 185, 33, 204, 25, 47, 11, 208, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, -}}; + 0x891a70a671f15cf6, b_891a70a671f15cf6.words, 55, d_891a70a671f15cf6, m_891a70a671f15cf6, + 1, 2, i_891a70a671f15cf6, nullptr, nullptr, { &s_891a70a671f15cf6, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<40> b_805c080c10c1e959 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 89, 233, 193, 16, 12, 8, 92, 128, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 90, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 76, 111, 97, 100, 69, 110, + 117, 109, 101, 114, 97, 116, 105, 111, + 110, 115, 82, 101, 115, 112, 111, 110, + 115, 101, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 3, 0, 1, 0, + 40, 0, 0, 0, 2, 0, 1, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_805c080c10c1e959 = b_805c080c10c1e959.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_805c080c10c1e959[] = { - &s_d00b2f19cc21b9b4, + &s_d00b2f19cc21b9b4, }; static const uint16_t m_805c080c10c1e959[] = {0}; static const uint16_t i_805c080c10c1e959[] = {0}; const ::capnp::_::RawSchema s_805c080c10c1e959 = { - 0x805c080c10c1e959, - b_805c080c10c1e959.words, - 40, - d_805c080c10c1e959, - m_805c080c10c1e959, - 1, - 1, - i_805c080c10c1e959, - nullptr, - nullptr, - {&s_805c080c10c1e959, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_83f094010132ff21 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 33, 255, 50, 1, 1, 148, 240, - 131, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 74, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, - 114, 97, 121, 83, 99, 104, 101, 109, 97, 82, 101, 113, 117, 101, 115, - 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 36, 0, 0, 0, 3, 0, 1, 0, 48, 0, - 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 48, 0, 0, 0, 3, 0, 1, 0, 60, 0, 0, 0, 2, 0, - 1, 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, - 0, 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 105, 110, 99, 108, 117, 100, 101, 69, 110, - 117, 109, 101, 114, 97, 116, 105, 111, 110, 115, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x805c080c10c1e959, b_805c080c10c1e959.words, 40, d_805c080c10c1e959, m_805c080c10c1e959, + 1, 1, i_805c080c10c1e959, nullptr, nullptr, { &s_805c080c10c1e959, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<52> b_83f094010132ff21 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 33, 255, 50, 1, 1, 148, 240, 131, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 74, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 76, 111, 97, 100, 65, 114, + 114, 97, 121, 83, 99, 104, 101, 109, + 97, 82, 101, 113, 117, 101, 115, 116, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 48, 0, 0, 0, 3, 0, 1, 0, + 60, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 99, 108, 117, 100, 101, 69, + 110, 117, 109, 101, 114, 97, 116, 105, + 111, 110, 115, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_83f094010132ff21 = b_83f094010132ff21.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_83f094010132ff21[] = { - &s_b6c95b4b8111ad36, + &s_b6c95b4b8111ad36, }; static const uint16_t m_83f094010132ff21[] = {0, 1}; static const uint16_t i_83f094010132ff21[] = {0, 1}; const ::capnp::_::RawSchema s_83f094010132ff21 = { - 0x83f094010132ff21, - b_83f094010132ff21.words, - 52, - d_83f094010132ff21, - m_83f094010132ff21, - 1, - 2, - i_83f094010132ff21, - nullptr, - nullptr, - {&s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr}, - false}; -#endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = {{ - 0, 0, 0, 0, 5, 0, 6, 0, 241, 29, 154, 172, 89, 127, 225, - 235, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, - 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21, 0, 0, 0, 82, 1, 0, 0, 41, 0, 0, 0, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, - 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, - 114, 97, 121, 83, 99, 104, 101, 109, 97, 82, 101, 115, 112, 111, 110, - 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 4, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 13, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 1, 0, 20, 0, - 0, 0, 2, 0, 1, 0, 115, 99, 104, 101, 109, 97, 0, 0, 16, - 0, 0, 0, 0, 0, 0, 0, 254, 150, 226, 152, 47, 227, 29, 215, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}}; + 0x83f094010132ff21, b_83f094010132ff21.words, 52, d_83f094010132ff21, m_83f094010132ff21, + 1, 2, i_83f094010132ff21, nullptr, nullptr, { &s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 241, 29, 154, 172, 89, 127, 225, 235, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 82, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 76, 111, 97, 100, 65, 114, + 114, 97, 121, 83, 99, 104, 101, 109, + 97, 82, 101, 115, 112, 111, 110, 115, + 101, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 3, 0, 1, 0, + 20, 0, 0, 0, 2, 0, 1, 0, + 115, 99, 104, 101, 109, 97, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 254, 150, 226, 152, 47, 227, 29, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; ::capnp::word const* const bp_ebe17f59ac9a1df1 = b_ebe17f59ac9a1df1.words; #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_ebe17f59ac9a1df1[] = { - &s_d71de32f98e296fe, + &s_d71de32f98e296fe, }; static const uint16_t m_ebe17f59ac9a1df1[] = {0}; static const uint16_t i_ebe17f59ac9a1df1[] = {0}; const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { - 0xebe17f59ac9a1df1, - b_ebe17f59ac9a1df1.words, - 35, - d_ebe17f59ac9a1df1, - m_ebe17f59ac9a1df1, - 1, - 1, - i_ebe17f59ac9a1df1, - nullptr, - nullptr, - {&s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr}, - false}; + 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 35, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, + 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false +}; #endif // !CAPNP_LITE } // namespace schemas } // namespace capnp @@ -6790,8 +9835,7 @@ constexpr uint16_t ArraySchemaEvolution::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArraySchemaEvolution::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArraySchemaEvolution::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArraySchemaEvolution::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -6827,8 +9871,7 @@ constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind AttributeBufferHeader::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - AttributeBufferHeader::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* AttributeBufferHeader::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -6852,8 +9895,7 @@ constexpr uint16_t Dimension::TileExtent::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind Dimension::TileExtent::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - Dimension::TileExtent::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* Dimension::TileExtent::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7057,8 +10099,7 @@ constexpr uint16_t UnorderedWriterState::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind UnorderedWriterState::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - UnorderedWriterState::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* UnorderedWriterState::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7094,8 +10135,7 @@ constexpr uint16_t LabelSubarrayRanges::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind LabelSubarrayRanges::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - LabelSubarrayRanges::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* LabelSubarrayRanges::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7119,23 +10159,19 @@ constexpr uint16_t SubarrayPartitioner::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind SubarrayPartitioner::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - SubarrayPartitioner::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* SubarrayPartitioner::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE // SubarrayPartitioner::PartitionInfo #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t - SubarrayPartitioner::PartitionInfo::_capnpPrivate::dataWordSize; -constexpr uint16_t - SubarrayPartitioner::PartitionInfo::_capnpPrivate::pointerCount; +constexpr uint16_t SubarrayPartitioner::PartitionInfo::_capnpPrivate::dataWordSize; +constexpr uint16_t SubarrayPartitioner::PartitionInfo::_capnpPrivate::pointerCount; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind SubarrayPartitioner::PartitionInfo::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - SubarrayPartitioner::PartitionInfo::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* SubarrayPartitioner::PartitionInfo::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7147,8 +10183,7 @@ constexpr uint16_t SubarrayPartitioner::State::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind SubarrayPartitioner::State::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - SubarrayPartitioner::State::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* SubarrayPartitioner::State::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7304,8 +10339,7 @@ constexpr uint16_t NonEmptyDomainList::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind NonEmptyDomainList::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - NonEmptyDomainList::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* NonEmptyDomainList::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7317,8 +10351,7 @@ constexpr uint16_t AttributeBufferSize::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind AttributeBufferSize::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - AttributeBufferSize::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* AttributeBufferSize::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7354,8 +10387,7 @@ constexpr uint16_t ArrayMetadata::MetadataEntry::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArrayMetadata::MetadataEntry::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayMetadata::MetadataEntry::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArrayMetadata::MetadataEntry::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7379,24 +10411,19 @@ constexpr uint16_t ArrayDirectory::TimestampedURI::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArrayDirectory::TimestampedURI::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayDirectory::TimestampedURI::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArrayDirectory::TimestampedURI::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE // ArrayDirectory::DeleteAndUpdateTileLocation #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t - ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::dataWordSize; -constexpr uint16_t - ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::pointerCount; +constexpr uint16_t ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::dataWordSize; +constexpr uint16_t ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::pointerCount; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr ::capnp::Kind - ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::schema; +constexpr ::capnp::Kind ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* ArrayDirectory::DeleteAndUpdateTileLocation::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7408,8 +10435,7 @@ constexpr uint16_t EstimatedResultSize::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind EstimatedResultSize::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - EstimatedResultSize::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* EstimatedResultSize::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7421,8 +10447,7 @@ constexpr uint16_t EstimatedResultSize::ResultSize::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind EstimatedResultSize::ResultSize::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - EstimatedResultSize::ResultSize::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* EstimatedResultSize::ResultSize::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7434,8 +10459,7 @@ constexpr uint16_t EstimatedResultSize::MemorySize::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind EstimatedResultSize::MemorySize::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - EstimatedResultSize::MemorySize::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* EstimatedResultSize::MemorySize::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7447,8 +10471,7 @@ constexpr uint16_t FragmentInfoRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind FragmentInfoRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - FragmentInfoRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* FragmentInfoRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7460,8 +10483,7 @@ constexpr uint16_t SingleFragmentInfo::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind SingleFragmentInfo::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - SingleFragmentInfo::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* SingleFragmentInfo::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7521,8 +10543,7 @@ constexpr uint16_t Group::GroupDetails::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind Group::GroupDetails::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - Group::GroupDetails::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* Group::GroupDetails::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7546,8 +10567,7 @@ constexpr uint16_t GroupUpdate::GroupUpdateDetails::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind GroupUpdate::GroupUpdateDetails::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - GroupUpdate::GroupUpdateDetails::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* GroupUpdate::GroupUpdateDetails::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7571,8 +10591,7 @@ constexpr uint16_t GroupCreate::GroupCreateDetails::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind GroupCreate::GroupCreateDetails::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - GroupCreate::GroupCreateDetails::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* GroupCreate::GroupCreateDetails::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7614,17 +10633,13 @@ constexpr ::capnp::_::RawSchema const* FragmentMetadata::_capnpPrivate::schema; // FragmentMetadata::GenericTileOffsets #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t - FragmentMetadata::GenericTileOffsets::_capnpPrivate::dataWordSize; -constexpr uint16_t - FragmentMetadata::GenericTileOffsets::_capnpPrivate::pointerCount; +constexpr uint16_t FragmentMetadata::GenericTileOffsets::_capnpPrivate::dataWordSize; +constexpr uint16_t FragmentMetadata::GenericTileOffsets::_capnpPrivate::pointerCount; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr ::capnp::Kind - FragmentMetadata::GenericTileOffsets::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - FragmentMetadata::GenericTileOffsets::_capnpPrivate::schema; +constexpr ::capnp::Kind FragmentMetadata::GenericTileOffsets::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* FragmentMetadata::GenericTileOffsets::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7636,8 +10651,7 @@ constexpr uint16_t MultiPartUploadState::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind MultiPartUploadState::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - MultiPartUploadState::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* MultiPartUploadState::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7661,8 +10675,7 @@ constexpr uint16_t WrittenFragmentInfo::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind WrittenFragmentInfo::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - WrittenFragmentInfo::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* WrittenFragmentInfo::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7686,24 +10699,19 @@ constexpr uint16_t ArrayDeleteFragmentsListRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArrayDeleteFragmentsListRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayDeleteFragmentsListRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArrayDeleteFragmentsListRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE // ArrayDeleteFragmentsTimestampsRequest #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr uint16_t - ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::dataWordSize; -constexpr uint16_t - ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::pointerCount; +constexpr uint16_t ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::dataWordSize; +constexpr uint16_t ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::pointerCount; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL -constexpr ::capnp::Kind - ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::schema; +constexpr ::capnp::Kind ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* ArrayDeleteFragmentsTimestampsRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7715,8 +10723,7 @@ constexpr uint16_t ArrayConsolidationRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArrayConsolidationRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayConsolidationRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArrayConsolidationRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7728,8 +10735,7 @@ constexpr uint16_t ArrayVacuumRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind ArrayVacuumRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - ArrayVacuumRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* ArrayVacuumRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7741,8 +10747,7 @@ constexpr uint16_t LoadEnumerationsRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind LoadEnumerationsRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - LoadEnumerationsRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* LoadEnumerationsRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7754,8 +10759,7 @@ constexpr uint16_t LoadEnumerationsResponse::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind LoadEnumerationsResponse::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - LoadEnumerationsResponse::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* LoadEnumerationsResponse::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7767,8 +10771,7 @@ constexpr uint16_t LoadArraySchemaRequest::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind LoadArraySchemaRequest::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - LoadArraySchemaRequest::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* LoadArraySchemaRequest::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE @@ -7780,12 +10783,13 @@ constexpr uint16_t LoadArraySchemaResponse::_capnpPrivate::pointerCount; #if !CAPNP_LITE #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr ::capnp::Kind LoadArraySchemaResponse::_capnpPrivate::kind; -constexpr ::capnp::_::RawSchema const* - LoadArraySchemaResponse::_capnpPrivate::schema; +constexpr ::capnp::_::RawSchema const* LoadArraySchemaResponse::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE -} // namespace capnp -} // namespace serialization -} // namespace sm -} // namespace tiledb + +} // namespace +} // namespace +} // namespace +} // namespace + diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index d1c0db8ff671..0e68ba5f8446 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -13981,8 +13981,8 @@ class ArrayDeleteFragmentsListRequest::Reader { } #endif // !CAPNP_LITE - inline bool hasUri() const; - inline ::capnp::Text::Reader getUri() const; + inline bool hasConfig() const; + inline ::tiledb::sm::serialization::capnp::Config::Reader getConfig() const; inline bool hasEntries() const; inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader getEntries() @@ -14027,12 +14027,15 @@ class ArrayDeleteFragmentsListRequest::Builder { } #endif // !CAPNP_LITE - inline bool hasUri(); - inline ::capnp::Text::Builder getUri(); - inline void setUri(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initUri(unsigned int size); - inline void adoptUri(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownUri(); + inline bool hasConfig(); + inline ::tiledb::sm::serialization::capnp::Config::Builder getConfig(); + inline void setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value); + inline ::tiledb::sm::serialization::capnp::Config::Builder initConfig(); + inline void adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> + disownConfig(); inline bool hasEntries(); inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder @@ -14069,6 +14072,8 @@ class ArrayDeleteFragmentsListRequest::Pipeline { : _typeless(kj::mv(typeless)) { } + inline ::tiledb::sm::serialization::capnp::Config::Pipeline getConfig(); + private: ::capnp::AnyPointer::Pipeline _typeless; friend class ::capnp::PipelineHook; @@ -14096,8 +14101,8 @@ class ArrayDeleteFragmentsTimestampsRequest::Reader { } #endif // !CAPNP_LITE - inline bool hasUri() const; - inline ::capnp::Text::Reader getUri() const; + inline bool hasConfig() const; + inline ::tiledb::sm::serialization::capnp::Config::Reader getConfig() const; inline ::uint64_t getStartTimestamp() const; @@ -14142,12 +14147,15 @@ class ArrayDeleteFragmentsTimestampsRequest::Builder { } #endif // !CAPNP_LITE - inline bool hasUri(); - inline ::capnp::Text::Builder getUri(); - inline void setUri(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initUri(unsigned int size); - inline void adoptUri(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownUri(); + inline bool hasConfig(); + inline ::tiledb::sm::serialization::capnp::Config::Builder getConfig(); + inline void setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value); + inline ::tiledb::sm::serialization::capnp::Config::Builder initConfig(); + inline void adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> + disownConfig(); inline ::uint64_t getStartTimestamp(); inline void setStartTimestamp(::uint64_t value); @@ -14176,6 +14184,8 @@ class ArrayDeleteFragmentsTimestampsRequest::Pipeline { : _typeless(kj::mv(typeless)) { } + inline ::tiledb::sm::serialization::capnp::Config::Pipeline getConfig(); + private: ::capnp::AnyPointer::Pipeline _typeless; friend class ::capnp::PipelineHook; @@ -30980,46 +30990,56 @@ inline void BufferedChunk::Builder::setSize(::uint64_t value) { ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); } -inline bool ArrayDeleteFragmentsListRequest::Reader::hasUri() const { +inline bool ArrayDeleteFragmentsListRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline bool ArrayDeleteFragmentsListRequest::Builder::hasUri() { +inline bool ArrayDeleteFragmentsListRequest::Builder::hasConfig() { return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader ArrayDeleteFragmentsListRequest::Reader::getUri() - const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::tiledb::sm::serialization::capnp::Config::Reader +ArrayDeleteFragmentsListRequest::Reader::getConfig() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline ::capnp::Text::Builder -ArrayDeleteFragmentsListRequest::Builder::getUri() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::tiledb::sm::serialization::capnp::Config::Builder +ArrayDeleteFragmentsListRequest::Builder::getConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void ArrayDeleteFragmentsListRequest::Builder::setUri( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Config::Pipeline +ArrayDeleteFragmentsListRequest::Pipeline::getConfig() { + return ::tiledb::sm::serialization::capnp::Config::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void ArrayDeleteFragmentsListRequest::Builder::setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::set( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), value); } -inline ::capnp::Text::Builder ArrayDeleteFragmentsListRequest::Builder::initUri( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline ::tiledb::sm::serialization::capnp::Config::Builder +ArrayDeleteFragmentsListRequest::Builder::initConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void ArrayDeleteFragmentsListRequest::Builder::adoptUri( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( +inline void ArrayDeleteFragmentsListRequest::Builder::adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::adopt( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); } -inline ::capnp::Orphan<::capnp::Text> -ArrayDeleteFragmentsListRequest::Builder::disownUri() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> +ArrayDeleteFragmentsListRequest::Builder::disownConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } inline bool ArrayDeleteFragmentsListRequest::Reader::hasEntries() const { @@ -31078,46 +31098,56 @@ ArrayDeleteFragmentsListRequest::Builder::disownEntries() { _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } -inline bool ArrayDeleteFragmentsTimestampsRequest::Reader::hasUri() const { +inline bool ArrayDeleteFragmentsTimestampsRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline bool ArrayDeleteFragmentsTimestampsRequest::Builder::hasUri() { +inline bool ArrayDeleteFragmentsTimestampsRequest::Builder::hasConfig() { return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader -ArrayDeleteFragmentsTimestampsRequest::Reader::getUri() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::tiledb::sm::serialization::capnp::Config::Reader +ArrayDeleteFragmentsTimestampsRequest::Reader::getConfig() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline ::capnp::Text::Builder -ArrayDeleteFragmentsTimestampsRequest::Builder::getUri() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::tiledb::sm::serialization::capnp::Config::Builder +ArrayDeleteFragmentsTimestampsRequest::Builder::getConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void ArrayDeleteFragmentsTimestampsRequest::Builder::setUri( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Config::Pipeline +ArrayDeleteFragmentsTimestampsRequest::Pipeline::getConfig() { + return ::tiledb::sm::serialization::capnp::Config::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void ArrayDeleteFragmentsTimestampsRequest::Builder::setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::set( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), value); } -inline ::capnp::Text::Builder -ArrayDeleteFragmentsTimestampsRequest::Builder::initUri(unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline ::tiledb::sm::serialization::capnp::Config::Builder +ArrayDeleteFragmentsTimestampsRequest::Builder::initConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void ArrayDeleteFragmentsTimestampsRequest::Builder::adoptUri( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( +inline void ArrayDeleteFragmentsTimestampsRequest::Builder::adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::adopt( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); } -inline ::capnp::Orphan<::capnp::Text> -ArrayDeleteFragmentsTimestampsRequest::Builder::disownUri() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> +ArrayDeleteFragmentsTimestampsRequest::Builder::disownConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } inline ::uint64_t From e7b99ff92414a0e931aa5677ff9ec199a4be0610 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 16 Oct 2023 22:42:01 +0300 Subject: [PATCH 002/456] Query field C API (#4385) This PR implements: ```C capi_return_t tiledb_query_get_field( tiledb_ctx_t* ctx, tiledb_query_t* query, const char* field, tiledb_query_field_t** field); capi_return_t tiledb_query_field_free( tiledb_ctx_t* ctx, tiledb_query_field_t** field); capi_return_t tiledb_field_datatype( tiledb_ctx_t* ctx, tiledb_query_field_t* field, tiledb_datatype_t* type); capi_return_t tiledb_field_cell_val_num( tiledb_ctx_t* ctx, tiledb_query_field_t* field, uint32_t* cell_val_num); capi_return_t tiledb_field_origin( tiledb_ctx_t* ctx,, tiledb_query_field_t* field, tiledb_field_origin_t* origin); capi_return_t tiledb_field_channel( tiledb_ctx_t* ctx,, tiledb_query_field_t* field, tiledb_query_channel_t** channel); ``` Patch starts at 940a76088cd1b7f7d790efc0012d046b1c655bf3, to be rebased against dev once #4176 gets merged. --- TYPE: NO_HISTORY | FEATURE | C_API DESC: Query field C API --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- CMakeLists.txt | 1 + tiledb/api/c_api/CMakeLists.txt | 3 + .../query_aggregate/query_aggregate_api.cc | 20 + .../query_aggregate_api_internal.h | 25 +- tiledb/api/c_api/query_field/CMakeLists.txt | 50 +++ .../api/c_api/query_field/query_field_api.cc | 197 ++++++++++ .../query_field_api_external_experimental.h | 196 ++++++++++ .../query_field/query_field_api_internal.h | 108 ++++++ .../api/c_api/query_field/test/CMakeLists.txt | 43 +++ .../test/compile_capi_query_field_main.cc | 41 ++ .../query_field/test/unit_capi_query_field.cc | 357 ++++++++++++++++++ tiledb/sm/query/query.cc | 9 + tiledb/sm/query/query.h | 4 + .../readers/aggregators/count_aggregator.h | 7 +- .../query/readers/aggregators/iaggregator.h | 3 + .../readers/aggregators/mean_aggregator.h | 5 + .../readers/aggregators/min_max_aggregator.h | 5 + .../readers/aggregators/sum_aggregator.h | 5 + .../sm/query/readers/aggregators/sum_type.h | 31 +- .../aggregators/test/unit_aggregators.cc | 134 +++++++ 20 files changed, 1204 insertions(+), 40 deletions(-) create mode 100644 tiledb/api/c_api/query_field/CMakeLists.txt create mode 100644 tiledb/api/c_api/query_field/query_field_api.cc create mode 100644 tiledb/api/c_api/query_field/query_field_api_external_experimental.h create mode 100644 tiledb/api/c_api/query_field/query_field_api_internal.h create mode 100644 tiledb/api/c_api/query_field/test/CMakeLists.txt create mode 100644 tiledb/api/c_api/query_field/test/compile_capi_query_field_main.cc create mode 100644 tiledb/api/c_api/query_field/test/unit_capi_query_field.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index b5c59ebb6106..5309ffa73865 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -364,6 +364,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query/query_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query/query_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query_field/query_field_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/string/string_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_enum.h" diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index dcfda0192896..8c4cc35e7940 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -121,3 +121,6 @@ add_subdirectory(query_plan) # `query_aggregate`: no dependencies add_subdirectory(query_aggregate) +# `query_field`: no dependencies +add_subdirectory(query_field) + diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index 92e3f8115a23..f22cf81f9af4 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -51,6 +51,26 @@ const tiledb_channel_operation_handle_t* tiledb_aggregate_count = tiledb_channel_operation_handle_t::make_handle( std::make_shared()); +shared_ptr tiledb_channel_operator_handle_t::make_operation( + const tiledb::sm::FieldInfo& fi) const { + switch (this->value()) { + case TILEDB_QUERY_CHANNEL_OPERATOR_SUM: { + return std::make_shared(fi, this); + } + case TILEDB_QUERY_CHANNEL_OPERATOR_MIN: { + return std::make_shared(fi, this); + } + case TILEDB_QUERY_CHANNEL_OPERATOR_MAX: { + return std::make_shared(fi, this); + } + default: + throw tiledb::api::CAPIStatusException( + "operator has unsupported value: " + + std::to_string(static_cast(this->value()))); + break; + } +} + namespace tiledb::api { /** diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h index 6415c74e4149..5bfef847f215 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h @@ -49,9 +49,6 @@ enum QueryChannelOperator { TILEDB_QUERY_CHANNEL_OPERATOR_MAX }; -void ensure_aggregate_numeric_field( - const tiledb_channel_operator_t* op, const tiledb::sm::FieldInfo& fi); - class Operation { protected: shared_ptr aggregator_; @@ -205,27 +202,7 @@ struct tiledb_channel_operator_handle_t const tiledb::sm::FieldInfo& fi) const; }; -shared_ptr tiledb_channel_operator_handle_t::make_operation( - const tiledb::sm::FieldInfo& fi) const { - switch (this->value()) { - case TILEDB_QUERY_CHANNEL_OPERATOR_SUM: { - return std::make_shared(fi, this); - } - case TILEDB_QUERY_CHANNEL_OPERATOR_MIN: { - return std::make_shared(fi, this); - } - case TILEDB_QUERY_CHANNEL_OPERATOR_MAX: { - return std::make_shared(fi, this); - } - default: - throw tiledb::api::CAPIStatusException( - "operator has unsupported value: " + - std::to_string(static_cast(this->value()))); - break; - } -} - -void ensure_aggregate_numeric_field( +inline void ensure_aggregate_numeric_field( const tiledb_channel_operator_t* op, const tiledb::sm::FieldInfo& fi) { if (fi.var_sized_) { throw tiledb::api::CAPIStatusException( diff --git a/tiledb/api/c_api/query_field/CMakeLists.txt b/tiledb/api/c_api/query_field/CMakeLists.txt new file mode 100644 index 000000000000..9b627780676d --- /dev/null +++ b/tiledb/api/c_api/query_field/CMakeLists.txt @@ -0,0 +1,50 @@ +# +# tiledb/api/c_api/query_field/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +# The object library `capi_query_field` cannot be linked standalone at the +# present time. It depends upon member functions of `class Query`, which does +# not yet have a standalone object library of its own. Without such a +# dependency, `capi_query_field` would have to link (essentially) the whole +# library to get a complete executable. +# +# It is against policy to use the `object_library` environment to define object +# libraries that cannot stand alone. Therefore this definition is directly in +# CMake and does not include a link-completeness target. +include(common NO_POLICY_SCOPE) + +list(APPEND SOURCES + query_field_api.cc + ) +gather_sources(${SOURCES}) + +# +# Object library for other units to depend upon +# +add_library(capi_query_field OBJECT ${SOURCES}) +target_link_libraries(capi_query_field PUBLIC export) +target_link_libraries(capi_query_field PUBLIC baseline $) + +add_test_subdirectory() diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc new file mode 100644 index 000000000000..56674df31bdf --- /dev/null +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -0,0 +1,197 @@ +/** + * @file tiledb/api/c_api/query_field/query_field_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the query_field C API of TileDB. + **/ + +#include "query_field_api_external_experimental.h" +#include "query_field_api_internal.h" +#include "tiledb/api/c_api/datatype/datatype_api_external.h" +#include "tiledb/api/c_api/query/query_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" + +tiledb_field_origin_t FieldFromDimension::origin() { + return TILEDB_DIMENSION_FIELD; +} + +tiledb_field_origin_t FieldFromAttribute::origin() { + return TILEDB_ATTRIBUTE_FIELD; +} + +tiledb_field_origin_t FieldFromAggregate::origin() { + return TILEDB_AGGREGATE_FIELD; +} + +tiledb_query_field_handle_t::tiledb_query_field_handle_t( + tiledb_query_t* query, const char* field_name) + : query_(query->query_) + , field_name_(field_name) + , channel_(tiledb_query_channel_handle_t::make_handle(query)) { + if (query_->array_schema().is_attr(field_name_)) { + field_origin_ = std::make_shared(); + type_ = query_->array_schema().attribute(field_name_)->type(); + cell_val_num_ = + query_->array_schema().attribute(field_name_)->cell_val_num(); + } else if (query_->array_schema().is_dim(field_name_)) { + field_origin_ = std::make_shared(); + type_ = query_->array_schema().dimension_ptr(field_name_)->type(); + cell_val_num_ = + query_->array_schema().dimension_ptr(field_name_)->cell_val_num(); + } else if (query_->is_aggregate(field_name_)) { + field_origin_ = std::make_shared(); + type_ = query_->get_aggregate(field_name_).value()->output_datatype(); + cell_val_num_ = 1; + } else { + throw tiledb::api::CAPIStatusException("There is no field " + field_name_); + } +} + +namespace tiledb::api { + +/** + * Ensure the argument is a valid char pointer + * + * @param field_name A char pointer + */ +inline void ensure_field_name_is_valid(const char* field_name) { + if (!field_name) { + throw CAPIStatusException("argument `field_name` may not be nullptr"); + } +} + +/** + * Ensure the argument is a valid query field handle + * + * @param field A query field handle + */ +inline void ensure_query_field_is_valid( + const tiledb_query_field_handle_t* field) { + ensure_handle_is_valid(field); +} + +capi_return_t tiledb_query_get_field( + tiledb_query_t* query, + const char* field_name, + tiledb_query_field_t** field) { + ensure_query_is_valid(query); + ensure_field_name_is_valid(field_name); + ensure_output_pointer_is_valid(field); + + *field = tiledb_query_field_handle_t::make_handle(query, field_name); + return TILEDB_OK; +} + +capi_return_t tiledb_query_field_free(tiledb_query_field_t** field) { + ensure_output_pointer_is_valid(field); + ensure_query_field_is_valid(*field); + tiledb_query_field_handle_t::break_handle(*field); + return TILEDB_OK; +} + +capi_return_t tiledb_field_datatype( + tiledb_query_field_t* field, tiledb_datatype_t* type) { + ensure_query_field_is_valid(field); + ensure_output_pointer_is_valid(type); + *type = static_cast(field->type()); + return TILEDB_OK; +} + +capi_return_t tiledb_field_cell_val_num( + tiledb_query_field_t* field, uint32_t* cell_val_num) { + ensure_query_field_is_valid(field); + ensure_output_pointer_is_valid(cell_val_num); + *cell_val_num = field->cell_val_num(); + return TILEDB_OK; +} + +capi_return_t tiledb_field_origin( + tiledb_query_field_t* field, tiledb_field_origin_t* origin) { + ensure_query_field_is_valid(field); + ensure_output_pointer_is_valid(origin); + *origin = field->origin(); + return TILEDB_OK; +} + +capi_return_t tiledb_field_channel( + tiledb_query_field_t* field, tiledb_query_channel_handle_t** channel) { + ensure_query_field_is_valid(field); + ensure_output_pointer_is_valid(channel); + *channel = field->channel(); + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_context; + +capi_return_t tiledb_query_get_field( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + const char* field_name, + tiledb_query_field_t** field) noexcept { + return api_entry_context( + ctx, query, field_name, field); +} + +capi_return_t tiledb_query_field_free( + tiledb_ctx_t* ctx, tiledb_query_field_t** field) noexcept { + return api_entry_context(ctx, field); +} + +capi_return_t tiledb_field_datatype( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_datatype_t* type) noexcept { + return api_entry_context( + ctx, field, type); +} + +capi_return_t tiledb_field_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + uint32_t* cell_val_num) noexcept { + return api_entry_context( + ctx, field, cell_val_num); +} + +capi_return_t tiledb_field_origin( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_field_origin_t* origin) noexcept { + return api_entry_context( + ctx, field, origin); +} + +capi_return_t tiledb_field_channel( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_query_channel_handle_t** channel) noexcept { + return api_entry_context( + ctx, field, channel); +} diff --git a/tiledb/api/c_api/query_field/query_field_api_external_experimental.h b/tiledb/api/c_api/query_field/query_field_api_external_experimental.h new file mode 100644 index 000000000000..b1f19e6645f5 --- /dev/null +++ b/tiledb/api/c_api/query_field/query_field_api_external_experimental.h @@ -0,0 +1,196 @@ +/** + * @file + * tiledb/api/c_api/query_field/query_field_api_external_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the query field C API for TileDB. + */ + +#ifndef TILEDB_CAPI_QUERY_FIELD_API_EXTERNAL_EXPERIMENTAL_H +#define TILEDB_CAPI_QUERY_FIELD_API_EXTERNAL_EXPERIMENTAL_H + +#include "../api_external_common.h" +#include "tiledb/api/c_api/datatype/datatype_api_external.h" +#include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct tiledb_query_field_handle_t tiledb_query_field_t; + +/** TileDB query field origin. */ +typedef enum { + TILEDB_ATTRIBUTE_FIELD = 0, + TILEDB_DIMENSION_FIELD, + TILEDB_AGGREGATE_FIELD +} tiledb_field_origin_t; + +/** + * Get a query field handle for the field name passed as argument. + * It is the responsability of the caller to manage the lifetime of the + * field handle and `tiledb_query_field_free` it when appropriate. + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "dimX", &field); + * tiledb_query_field_free(ctx, &field); + * + * @endcode + * + * @param ctx The TileDB context + * @param query A TileDB query + * @param field_name The name of the field for which a handle should be created + * @param field The query field handle + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_query_get_field( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + const char* field_name, + tiledb_query_field_t** field) TILEDB_NOEXCEPT; + +/** + * Frees the resources associated with a TileDB query field handle + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "dimX", &field); + * tiledb_query_field_free(ctx, &field); + * @endcode + * + * @param ctx The TileDB context + * @param field The address of the query field handle pointer + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_query_field_free( + tiledb_ctx_t* ctx, tiledb_query_field_t** field) TILEDB_NOEXCEPT; + +/** + * Get the TileDB datatype for a query field + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "dimX", &field); + * tiledb_datatype_t t; + * tiledb_field_datatype(ctx, field, &t); + * tiledb_query_field_free(ctx, &field); + * @endcode + * + * @param ctx The TileDB context + * @param field The query field handle + * @param type The TileDB datatype to be returned for the field + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_field_datatype( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Get the number of values per cell for a field + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "dimX", &field); + * uint32_t cell_val_num = 0; + * tiledb_field_cell_val_num(ctx, field, &cell_val_num); + * tiledb_query_field_free(ctx, &field); + * @endcode + * + * @param ctx The TileDB context + * @param field The query field handle + * @param cell_val_num The number of values per cell to be returned + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_field_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + uint32_t* cell_val_num) TILEDB_NOEXCEPT; + +/** + * Get the origin type of the passed field + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "dimX", &field); + * tiledb_field_origin_t origin; + * tiledb_field_origin(ctx, field, &origin); + * check_true(origin == TILEDB_DIMENSION_FIELD); + * tiledb_query_field_free(ctx, &field); + * @endcode + * + * @param ctx The TileDB context + * @param field The query field handle + * @param origin The origin type of the field to be returned + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_field_origin( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_field_origin_t* origin) TILEDB_NOEXCEPT; + +/** + * Get the query channel a field it's on + * At the moment, all fields are on the query default channel. + * Aggregates segmentation will add the ability for multiple channels + * to be created and this API will enable quering which channel based + * on the field. + * This API allocates a new query channel handle when called, it's the + * responsability of the caller to free the new query channel handle. + * **Example:** + * + * @code{.c} + * tiledb_query_field_t *field; + * tiledb_query_get_field(ctx, query, "SumX", &field); + * tiledb_query_channel_t *channel; + * tiledb_field_channel(ctx, field, &channel); + * tiledb_query_channel_free(ctx, &channel); + * tiledb_query_field_free(ctx, &field); + * @endcode + * + * @param ctx The TileDB context + * @param field The query field handle + * @param channel The allocated handle for the query channel the field it's on + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_field_channel( + tiledb_ctx_t* ctx, + tiledb_query_field_t* field, + tiledb_query_channel_t** channel) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_QUERY_FIELD_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/query_field/query_field_api_internal.h b/tiledb/api/c_api/query_field/query_field_api_internal.h new file mode 100644 index 000000000000..3e50160537e1 --- /dev/null +++ b/tiledb/api/c_api/query_field/query_field_api_internal.h @@ -0,0 +1,108 @@ +/** + * @file tiledb/api/c_api/query_field/query_field_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internal mechanics of the query field C API + */ + +#ifndef TILEDB_CAPI_QUERY_FIELD_INTERNAL_H +#define TILEDB_CAPI_QUERY_FIELD_INTERNAL_H + +#include "tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h" +#include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/query/query.h" + +class FieldOrigin { + public: + virtual tiledb_field_origin_t origin() = 0; + virtual ~FieldOrigin() = default; +}; + +class FieldFromDimension : public FieldOrigin { + public: + virtual tiledb_field_origin_t origin() override; +}; + +class FieldFromAttribute : public FieldOrigin { + public: + virtual tiledb_field_origin_t origin() override; +}; + +class FieldFromAggregate : public FieldOrigin { + public: + virtual tiledb_field_origin_t origin() override; +}; + +struct tiledb_query_field_handle_t + : public tiledb::api::CAPIHandle { + /** + * Type name + */ + static constexpr std::string_view object_type_name{"tiledb_query_field_t"}; + + private: + tiledb::sm::Query* query_; + std::string field_name_; + std::shared_ptr field_origin_; + tiledb::sm::Datatype type_; + uint32_t cell_val_num_; + tiledb_query_channel_handle_t* channel_; + + public: + /** + * Default constructor doesn't make sense + */ + tiledb_query_field_handle_t() = delete; + + /** + * Ordinary constructor. + * @param query The query object that owns the channel + */ + tiledb_query_field_handle_t(tiledb_query_t* query, const char* field_name); + + ~tiledb_query_field_handle_t() { + tiledb_query_channel_handle_t::break_handle(channel_); + } + + tiledb_field_origin_t origin() { + return field_origin_->origin(); + } + tiledb::sm::Datatype type() { + return type_; + } + uint32_t cell_val_num() { + return cell_val_num_; + } + tiledb_query_channel_handle_t* channel() { + return channel_; + } +}; + +#endif // TILEDB_CAPI_QUERY_FIELD_INTERNAL_H diff --git a/tiledb/api/c_api/query_field/test/CMakeLists.txt b/tiledb/api/c_api/query_field/test/CMakeLists.txt new file mode 100644 index 000000000000..a35093b4295f --- /dev/null +++ b/tiledb/api/c_api/query_field/test/CMakeLists.txt @@ -0,0 +1,43 @@ +# +# tiledb/api/c_api/query_field/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_query_field) + this_target_sources(unit_capi_query_field.cc) + +# The object library `capi_query_field` cannot be linked standalone at the +# present time. It depends upon member functions of `class Query`, which does +# not yet have a standalone object library of its own. Without such a +# dependency, `capi_query_field` would have to link (essentially) the whole +# library to get a complete executable. + this_target_object_libraries(capi_query_field) + this_target_link_libraries(tiledb_test_support_lib) + + if (NOT MSVC) + target_compile_options(unit_capi_query_field PRIVATE -Wno-deprecated-declarations) + endif() +conclude(unit_test) diff --git a/tiledb/api/c_api/query_field/test/compile_capi_query_field_main.cc b/tiledb/api/c_api/query_field/test/compile_capi_query_field_main.cc new file mode 100644 index 000000000000..821a8814f8fd --- /dev/null +++ b/tiledb/api/c_api/query_field/test/compile_capi_query_field_main.cc @@ -0,0 +1,41 @@ +/** + * @file + * tiledb/api/c_api/query_field/test/compile_capi_query_field_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../query_field_api_external_experimental.h" + +int main() { + tiledb_query_get_field(nullptr, nullptr, nullptr, nullptr); + tiledb_query_field_free(nullptr, nullptr); + tiledb_field_datatype(nullptr, nullptr, nullptr); + tiledb_field_cell_val_num(nullptr, nullptr, nullptr); + tiledb_field_origin(nullptr, nullptr, nullptr); + tiledb_field_channel(nullptr, nullptr, nullptr); + + return 1; +} diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc new file mode 100644 index 000000000000..7c540fa1b4b8 --- /dev/null +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -0,0 +1,357 @@ +/** + * @file tiledeb/api/c_api/query_field/test/unit_capi_query_field.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the query field C API. + */ + +#include +#include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" + +using namespace tiledb::test; + +struct QueryFieldFx : TemporaryDirectoryFixture { + QueryFieldFx() { + REQUIRE( + ctx->resources() + .config() + .set("sm.allow_aggregates_experimental", "true") + .ok() == true); + + create_sparse_array(array_name()); + write_sparse_array(array_name()); + } + + std::string array_name() { + return temp_dir_ + "queryfield_array"; + } + + inline void throw_if_setup_failed(const int32_t rc) { + if (rc != TILEDB_OK) { + throw std::runtime_error("test setup failed"); + } + } + + void write_sparse_array(const std::string& path); + void create_sparse_array(const std::string& path); +}; + +void QueryFieldFx::write_sparse_array(const std::string& array_name) { + tiledb_array_t* array; + throw_if_setup_failed(tiledb_array_alloc(ctx, array_name.c_str(), &array)); + throw_if_setup_failed(tiledb_array_open(ctx, array, TILEDB_WRITE)); + + tiledb_query_t* query; + throw_if_setup_failed(tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query)); + + throw_if_setup_failed(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED)); + + int32_t a[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + int32_t b[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + uint64_t a_size = 10 * sizeof(int32_t); + uint64_t b_size = 10 * sizeof(int32_t); + + int64_t d1[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + int64_t d2[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + uint64_t d1_size = 10 * sizeof(int64_t); + uint64_t d2_size = 10 * sizeof(int64_t); + char c_data[] = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + uint64_t c_size = strlen(c_data); + uint64_t c_data_offsets[] = {0, 5, 8, 13, 17, 21, 26, 31, 36, 40}; + uint64_t c_offsets_size = sizeof(c_data_offsets); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "a", a, &a_size)); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "b", b, &b_size)); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size)); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "d2", d2, &d2_size)); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "c", c_data, &c_size)); + throw_if_setup_failed(tiledb_query_set_offsets_buffer( + ctx, query, "c", c_data_offsets, &c_offsets_size)); + + throw_if_setup_failed( + tiledb_query_set_data_buffer(ctx, query, "d", c_data, &c_size)); + throw_if_setup_failed(tiledb_query_set_offsets_buffer( + ctx, query, "d", c_data_offsets, &c_offsets_size)); + + throw_if_setup_failed(tiledb_query_submit(ctx, query)); + + // Clean up + throw_if_setup_failed(tiledb_array_close(ctx, array)); + tiledb_array_free(&array); + tiledb_query_free(&query); +} + +void QueryFieldFx::create_sparse_array(const std::string& array_name) { + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + tiledb_dimension_t* d1; + throw_if_setup_failed(tiledb_dimension_alloc( + ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1)); + tiledb_dimension_t* d2; + throw_if_setup_failed(tiledb_dimension_alloc( + ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2)); + + // Create domain + tiledb_domain_t* domain; + throw_if_setup_failed(tiledb_domain_alloc(ctx, &domain)); + throw_if_setup_failed(tiledb_domain_add_dimension(ctx, domain, d1)); + throw_if_setup_failed(tiledb_domain_add_dimension(ctx, domain, d2)); + + // Create attributes + tiledb_attribute_t* a; + throw_if_setup_failed(tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a)); + tiledb_attribute_t* b; + throw_if_setup_failed(tiledb_attribute_alloc(ctx, "b", TILEDB_INT32, &b)); + tiledb_attribute_t* c; + throw_if_setup_failed( + tiledb_attribute_alloc(ctx, "c", TILEDB_STRING_ASCII, &c)); + throw_if_setup_failed( + tiledb_attribute_set_cell_val_num(ctx, c, TILEDB_VAR_NUM)); + tiledb_attribute_t* d; + throw_if_setup_failed( + tiledb_attribute_alloc(ctx, "d", TILEDB_STRING_UTF8, &d)); + throw_if_setup_failed( + tiledb_attribute_set_cell_val_num(ctx, d, TILEDB_VAR_NUM)); + + // Create array schema + tiledb_array_schema_t* array_schema; + throw_if_setup_failed( + tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema)); + throw_if_setup_failed( + tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR)); + throw_if_setup_failed( + tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR)); + throw_if_setup_failed(tiledb_array_schema_set_capacity(ctx, array_schema, 4)); + throw_if_setup_failed( + tiledb_array_schema_set_domain(ctx, array_schema, domain)); + throw_if_setup_failed( + tiledb_array_schema_add_attribute(ctx, array_schema, a)); + throw_if_setup_failed( + tiledb_array_schema_add_attribute(ctx, array_schema, b)); + throw_if_setup_failed( + tiledb_array_schema_add_attribute(ctx, array_schema, c)); + throw_if_setup_failed( + tiledb_array_schema_add_attribute(ctx, array_schema, d)); + + // check array schema + throw_if_setup_failed(tiledb_array_schema_check(ctx, array_schema)); + + // Create array + throw_if_setup_failed( + tiledb_array_create(ctx, array_name.c_str(), array_schema)); + + // Clean up + tiledb_attribute_free(&a); + tiledb_attribute_free(&b); + tiledb_dimension_free(&d1); + tiledb_dimension_free(&d2); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&array_schema); +} + +TEST_CASE_METHOD( + QueryFieldFx, + "C API: argument validation", + "[capi][query_field][get][args]") { + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + tiledb_query_field_t* field = nullptr; + + SECTION("nullptr context") { + CHECK( + tiledb_query_get_field(nullptr, query, "", &field) == + TILEDB_INVALID_CONTEXT); + } + + SECTION("nullptr query") { + CHECK(tiledb_query_get_field(ctx, nullptr, "", &field) == TILEDB_ERR); + } + + SECTION("nullptr field name") { + CHECK(tiledb_query_get_field(ctx, query, nullptr, &field) == TILEDB_ERR); + } + + SECTION("nullptr output field") { + CHECK(tiledb_query_get_field(ctx, query, "", nullptr) == TILEDB_ERR); + CHECK(tiledb_query_field_free(ctx, nullptr) == TILEDB_ERR); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_ERR); + } + + // Clean up + tiledb_query_free(&query); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); +} + +TEST_CASE_METHOD( + QueryFieldFx, + "C API: argument validation", + "[capi][query_field][access][args]") { + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + tiledb_query_field_t* field = nullptr; + tiledb_datatype_t type; + tiledb_field_origin_t origin; + uint32_t cell_val_num; + tiledb_query_channel_t* channel = nullptr; + + REQUIRE(tiledb_query_get_field(ctx, query, "d1", &field) == TILEDB_OK); + + SECTION("nullptr context") { + CHECK( + tiledb_field_datatype(nullptr, field, &type) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_field_origin(nullptr, field, &origin) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_field_cell_val_num(nullptr, field, &cell_val_num) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_field_channel(nullptr, field, &channel) == + TILEDB_INVALID_CONTEXT); + } + + SECTION("nullptr field") { + CHECK(tiledb_field_datatype(ctx, nullptr, &type) == TILEDB_ERR); + CHECK(tiledb_field_origin(ctx, nullptr, &origin) == TILEDB_ERR); + CHECK(tiledb_field_cell_val_num(ctx, nullptr, &cell_val_num) == TILEDB_ERR); + CHECK(tiledb_field_channel(ctx, nullptr, &channel) == TILEDB_ERR); + } + + SECTION("nullptr output ptr") { + CHECK(tiledb_field_datatype(ctx, field, nullptr) == TILEDB_ERR); + CHECK(tiledb_field_origin(ctx, field, nullptr) == TILEDB_ERR); + CHECK(tiledb_field_cell_val_num(ctx, field, nullptr) == TILEDB_ERR); + CHECK(tiledb_field_channel(ctx, field, nullptr) == TILEDB_ERR); + } + + // Clean up + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + tiledb_query_free(&query); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); +} + +TEST_CASE_METHOD( + QueryFieldFx, "C API: argument validation", "[capi][query_field]") { + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); + int64_t dom[] = {1, 9, 1, 2}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + tiledb_query_field_t* field = nullptr; + tiledb_datatype_t type; + tiledb_field_origin_t origin; + uint32_t cell_val_num = 0; + tiledb_query_channel_t* channel = nullptr; + + // Errors out when the field doesn't exist + CHECK( + tiledb_query_get_field(ctx, query, "non_existent", &field) == TILEDB_ERR); + + // Check field api works on dimension field + REQUIRE(tiledb_query_get_field(ctx, query, "d1", &field) == TILEDB_OK); + + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_DIMENSION_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + + // Let's at least test that this API actually gets the default channel. + // Default channel means all rows, let's count them. We will add more field + // specific tests when more functionality gets implemented for channels. + REQUIRE(tiledb_field_channel(ctx, field, &channel) == TILEDB_OK); + REQUIRE( + tiledb_channel_apply_aggregate( + ctx, channel, "Count", tiledb_aggregate_count) == TILEDB_OK); + uint64_t count = 0; + uint64_t size = 8; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "Count", &count, &size) == + TILEDB_OK); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + CHECK(count == 9); + CHECK(tiledb_query_channel_free(ctx, &channel) == TILEDB_OK); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + + // Check field api works on attribute field + REQUIRE(tiledb_query_get_field(ctx, query, "c", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_STRING_ASCII); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_ATTRIBUTE_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == TILEDB_VAR_NUM); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + + // Check field api works on aggregate field + REQUIRE(tiledb_query_get_field(ctx, query, "Count", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_AGGREGATE_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + + // Clean up + tiledb_query_free(&query); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); +} diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 268c3e6d4fa2..92115ed20486 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -1138,6 +1138,15 @@ Status Query::set_data_buffer( return Status::Ok(); } +std::optional> Query::get_aggregate( + std::string output_field_name) const { + auto it = default_channel_aggregates_.find(output_field_name); + if (it == default_channel_aggregates_.end()) { + return nullopt; + } + return it->second; +} + Status Query::set_offsets_buffer( const std::string& name, uint64_t* const buffer_offsets, diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 7e3100bfb540..5a43e6c0eab9 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -739,6 +739,10 @@ class Query { default_channel_aggregates_.emplace(output_field_name, aggregator); } + /** Returns an aggregate based on the output field. */ + std::optional> get_aggregate( + std::string output_field_name) const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.h b/tiledb/sm/query/readers/aggregators/count_aggregator.h index 82100bece462..7df272d34030 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.h @@ -33,6 +33,7 @@ #ifndef TILEDB_COUNT_AGGREGATOR_H #define TILEDB_COUNT_AGGREGATOR_H +#include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/query/readers/aggregators/aggregate_with_count.h" #include "tiledb/sm/query/readers/aggregators/iaggregator.h" #include "tiledb/sm/query/readers/aggregators/no_op.h" @@ -45,7 +46,6 @@ class QueryBuffer; template class CountAggregatorBase : public OutputBufferValidator, public IAggregator { public: - /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -96,6 +96,11 @@ class CountAggregatorBase : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns the TileDB datatype of the output field for the aggregate. */ + Datatype output_datatype() override { + return Datatype::UINT64; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/iaggregator.h b/tiledb/sm/query/readers/aggregators/iaggregator.h index 40c692b3f8ab..535bb3c560b7 100644 --- a/tiledb/sm/query/readers/aggregators/iaggregator.h +++ b/tiledb/sm/query/readers/aggregators/iaggregator.h @@ -93,6 +93,9 @@ class IAggregator { virtual void copy_to_user_buffer( std::string output_field_name, std::unordered_map& buffers) = 0; + + /** Returns the TileDB datatype of the output field for the aggregate. */ + virtual Datatype output_datatype() = 0; }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.h b/tiledb/sm/query/readers/aggregators/mean_aggregator.h index b2ef6527b2aa..e61fbe62c3d6 100644 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/mean_aggregator.h @@ -110,6 +110,11 @@ class MeanAggregator : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns the TileDB datatype of the output field for the aggregate. */ + Datatype output_datatype() override { + return sum_type_data::tiledb_datatype; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h index b1fe1874e115..e202445b04ac 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h @@ -173,6 +173,11 @@ class ComparatorAggregator : public ComparatorAggregatorBase, std::string output_field_name, std::unordered_map& buffers) override; + /** Returns the TileDB datatype of the output field for the aggregate. */ + Datatype output_datatype() override { + return ComparatorAggregatorBase::field_info_.type_; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.h b/tiledb/sm/query/readers/aggregators/sum_aggregator.h index 24a9643146c4..9676fe81ef89 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.h @@ -109,6 +109,11 @@ class SumAggregator : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns the TileDB datatype of the output field for the aggregate. */ + Datatype output_datatype() override { + return sum_type_data::tiledb_datatype; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/sum_type.h b/tiledb/sm/query/readers/aggregators/sum_type.h index 95f22dc47a48..d0c1f359a7b0 100644 --- a/tiledb/sm/query/readers/aggregators/sum_type.h +++ b/tiledb/sm/query/readers/aggregators/sum_type.h @@ -37,27 +37,28 @@ namespace tiledb::sm { -#define SUM_TYPE_DATA(T, SUM_T) \ - template <> \ - struct sum_type_data { \ - using type = T; \ - typedef SUM_T sum_type; \ +#define SUM_TYPE_DATA(T, SUM_T, T_DT) \ + template <> \ + struct sum_type_data { \ + using type = T; \ + typedef SUM_T sum_type; \ + static constexpr Datatype tiledb_datatype = T_DT; \ }; /** Convert basic type to a sum type. **/ template struct sum_type_data; -SUM_TYPE_DATA(int8_t, int64_t); -SUM_TYPE_DATA(uint8_t, uint64_t); -SUM_TYPE_DATA(int16_t, int64_t); -SUM_TYPE_DATA(uint16_t, uint64_t); -SUM_TYPE_DATA(int32_t, int64_t); -SUM_TYPE_DATA(uint32_t, uint64_t); -SUM_TYPE_DATA(int64_t, int64_t); -SUM_TYPE_DATA(uint64_t, uint64_t); -SUM_TYPE_DATA(float, double); -SUM_TYPE_DATA(double, double); +SUM_TYPE_DATA(int8_t, int64_t, Datatype::INT64); +SUM_TYPE_DATA(uint8_t, uint64_t, Datatype::UINT64); +SUM_TYPE_DATA(int16_t, int64_t, Datatype::INT64); +SUM_TYPE_DATA(uint16_t, uint64_t, Datatype::UINT64); +SUM_TYPE_DATA(int32_t, int64_t, Datatype::INT64); +SUM_TYPE_DATA(uint32_t, uint64_t, Datatype::UINT64); +SUM_TYPE_DATA(int64_t, int64_t, Datatype::INT64); +SUM_TYPE_DATA(uint64_t, uint64_t, Datatype::UINT64); +SUM_TYPE_DATA(float, double, Datatype::FLOAT64); +SUM_TYPE_DATA(double, double, Datatype::FLOAT64); } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 5062388c9434..b99838917176 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -1109,3 +1109,137 @@ TEST_CASE( std::vector res = {0, 4, 0, 0, 2, 3, 0, 0, 3, 6}; basic_string_aggregation_test(res); } + +TEST_CASE( + "NullCount aggregator: output datatype", + "[null-count-aggregator][output-datatype]") { + NullCountAggregator aggregator{FieldInfo{"a1", false, true, 1}}; + CHECK(aggregator.output_datatype() == Datatype::UINT64); +} + +TEST_CASE( + "Count aggregator: output datatype", + "[count-aggregator][output-datatype]") { + CountAggregator aggregator; + CHECK(aggregator.output_datatype() == Datatype::UINT64); +} + +TEST_CASE( + "Sum aggregator: Expected output type", + "[sum-aggregator][output_datatype]") { + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + SumAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + SumAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + SumAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} + .output_datatype() == Datatype::FLOAT64); + CHECK( + SumAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} + .output_datatype() == Datatype::FLOAT64); +} + +TEST_CASE( + "Mean aggregator: Expected output type", + "[mean-aggregator][output_datatype]") { + CHECK( + MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + MeanAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + MeanAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + MeanAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + MeanAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} + .output_datatype() == Datatype::FLOAT64); + CHECK( + MeanAggregator{ + FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} + .output_datatype() == Datatype::FLOAT64); +} + +TEST_CASE( + "Min max aggregator: Expected output type", + "[min-max-aggregator][output_datatype]") { + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::INT8)} + .output_datatype() == Datatype::INT8); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::UINT8)} + .output_datatype() == Datatype::UINT8); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::INT16)} + .output_datatype() == Datatype::INT16); + CHECK( + MinAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT16)} + .output_datatype() == Datatype::UINT16); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::INT32)} + .output_datatype() == Datatype::INT32); + CHECK( + MinAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT32)} + .output_datatype() == Datatype::UINT32); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} + .output_datatype() == Datatype::INT64); + CHECK( + MinAggregator{ + FieldInfo("a1", false, false, 1, Datatype::UINT64)} + .output_datatype() == Datatype::UINT64); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT32)} + .output_datatype() == Datatype::FLOAT32); + CHECK( + MinAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} + .output_datatype() == Datatype::FLOAT64); + CHECK( + MinAggregator{ + FieldInfo("a1", false, false, 1, Datatype::STRING_ASCII)} + .output_datatype() == Datatype::STRING_ASCII); +} From 1df0d65fb6057b9ebbb8d783c2f9c6d260bb6027 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 16 Oct 2023 23:30:57 +0200 Subject: [PATCH 003/456] Aggregates: Adding tests for computation policies. (#4421) This adds tests for all currently existing computation policies for aggregates. SC-33764 --- TYPE: IMPROVEMENT DESC: Aggregates: Adding tests for computation policies. --- .../sm/query/readers/aggregators/sum_type.h | 55 ++++-- .../readers/aggregators/test/CMakeLists.txt | 2 +- .../test/unit_aggregation_policies.cc | 184 ++++++++++++++++++ 3 files changed, 221 insertions(+), 20 deletions(-) create mode 100644 tiledb/sm/query/readers/aggregators/test/unit_aggregation_policies.cc diff --git a/tiledb/sm/query/readers/aggregators/sum_type.h b/tiledb/sm/query/readers/aggregators/sum_type.h index d0c1f359a7b0..ed2b77a0a682 100644 --- a/tiledb/sm/query/readers/aggregators/sum_type.h +++ b/tiledb/sm/query/readers/aggregators/sum_type.h @@ -34,31 +34,48 @@ #ifndef TILEDB_SUM_TYPE_H #define TILEDB_SUM_TYPE_H - namespace tiledb::sm { -#define SUM_TYPE_DATA(T, SUM_T, T_DT) \ - template <> \ - struct sum_type_data { \ - using type = T; \ - typedef SUM_T sum_type; \ - static constexpr Datatype tiledb_datatype = T_DT; \ - }; +template +concept SignedInt = std::signed_integral; + +template +concept UnsignedInt = std::unsigned_integral; + +template +concept FloatingPoint = std::floating_point; + +template +concept SummableType = FloatingPoint || SignedInt || UnsignedInt; /** Convert basic type to a sum type. **/ template -struct sum_type_data; + requires SummableType +struct sum_type_data {}; + +// Signed integer specializations. +template +struct sum_type_data { + using type = T; + typedef int64_t sum_type; + static constexpr Datatype tiledb_datatype = Datatype::INT64; +}; + +// Unsigned integer specializations. +template +struct sum_type_data { + using type = T; + typedef uint64_t sum_type; + static constexpr Datatype tiledb_datatype = Datatype::UINT64; +}; -SUM_TYPE_DATA(int8_t, int64_t, Datatype::INT64); -SUM_TYPE_DATA(uint8_t, uint64_t, Datatype::UINT64); -SUM_TYPE_DATA(int16_t, int64_t, Datatype::INT64); -SUM_TYPE_DATA(uint16_t, uint64_t, Datatype::UINT64); -SUM_TYPE_DATA(int32_t, int64_t, Datatype::INT64); -SUM_TYPE_DATA(uint32_t, uint64_t, Datatype::UINT64); -SUM_TYPE_DATA(int64_t, int64_t, Datatype::INT64); -SUM_TYPE_DATA(uint64_t, uint64_t, Datatype::UINT64); -SUM_TYPE_DATA(float, double, Datatype::FLOAT64); -SUM_TYPE_DATA(double, double, Datatype::FLOAT64); +// Floating point specializations. +template +struct sum_type_data { + using type = T; + typedef double sum_type; + static constexpr Datatype tiledb_datatype = Datatype::FLOAT64; +}; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt b/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt index f16a39e0c62b..9e2904596c12 100644 --- a/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt +++ b/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt @@ -27,6 +27,6 @@ include(unit_test) commence(unit_test aggregators) - this_target_sources(main.cc unit_aggregate_with_count.cc unit_aggregators.cc) + this_target_sources(main.cc unit_aggregate_with_count.cc unit_aggregation_policies.cc unit_aggregators.cc) this_target_object_libraries(aggregators) conclude(unit_test) diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregation_policies.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregation_policies.cc new file mode 100644 index 000000000000..cfcbaec1ca34 --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregation_policies.cc @@ -0,0 +1,184 @@ +/** + * @file unit_aggregation_policies.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the aggregation policy classes. + */ + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/query/readers/aggregators/min_max.h" +#include "tiledb/sm/query/readers/aggregators/no_op.h" +#include "tiledb/sm/query/readers/aggregators/safe_sum.h" +#include "tiledb/sm/query/readers/aggregators/sum_type.h" +#include "tiledb/sm/query/readers/aggregators/validity_policies.h" + +#include + +using namespace tiledb::sm; + +typedef tuple< + uint8_t, + uint16_t, + uint32_t, + uint64_t, + int8_t, + int16_t, + int32_t, + int64_t, + float, + double, + std::string_view> + TypesUnderTest; +TEMPLATE_LIST_TEST_CASE( + "Aggregate policies: no op", + "[aggregate-policies][no-op]", + TypesUnderTest) { + typedef TestType T; + auto op = NoOp(); + op.op(T(), 0, 0); +} + +typedef tuple< + uint8_t, + uint16_t, + uint32_t, + uint64_t, + int8_t, + int16_t, + int32_t, + int64_t> + FixedTypesUnderTest; +TEMPLATE_LIST_TEST_CASE( + "Aggregate policies: safe sum", + "[aggregate-policies][safe-sum]", + FixedTypesUnderTest) { + typedef TestType T; + typedef typename sum_type_data::sum_type SUM_T; + + SUM_T sum; + std::atomic sum_atomic; + T val; + auto op = SafeSum(); + + // Regular sum. + sum = 10; + sum_atomic = sum; + val = 10; + op.op(val, sum, 0); + op.safe_sum(val, sum_atomic); + CHECK(sum == 20); + CHECK(sum_atomic == 20); + + // Overflow. + sum = std::numeric_limits::max(); + sum_atomic = sum; + val = 1; + CHECK_THROWS_WITH(op.op(val, sum, 0), "overflow on sum"); + CHECK_THROWS_WITH(op.safe_sum(val, sum_atomic), "overflow on sum"); + CHECK(sum == std::numeric_limits::max()); + CHECK(sum_atomic == sum); + + // Underflow + if constexpr (std::is_signed::value) { + sum = std::numeric_limits::lowest(); + sum_atomic = sum; + val = -1; + CHECK_THROWS_WITH(op.op(val, sum, 0), "overflow on sum"); + CHECK_THROWS_WITH(op.safe_sum(val, sum_atomic), "overflow on sum"); + CHECK(sum == std::numeric_limits::lowest()); + CHECK(sum_atomic == sum); + } +} + +template +T value(uint64_t v) { + return v; +} + +const std::string values = "0123456789"; +template <> +std::string_view value(uint64_t v) { + return std::string_view(values.data() + v, 1); +} + +template +void min_max_test() { + auto op = MinMax(); + + T val; + T minmax; + Op operation; + bool res; + for (uint64_t left = 0; left < 10; left++) { + for (uint64_t right = 0; right < 10; right++) { + // First value always replaces the min/max. + val = value(left); + minmax = value(right); + op.template op(val, minmax, 0); + CHECK(minmax == val); + + val = value(right); + minmax = value(left); + op.template op(val, minmax, 0); + CHECK(minmax == val); + + // Other value replaces the min/max depending on Op. + for (uint64_t c = 1; c < 10; c++) { + val = value(left); + minmax = value(right); + res = operation(val, minmax); + op.template op(val, minmax, 1); + CHECK(minmax == (res ? val : minmax)); + + val = value(right); + minmax = value(left); + op.template op(val, minmax, 1); + CHECK(minmax == (res ? minmax : val)); + } + } + } +} + +TEMPLATE_LIST_TEST_CASE( + "Aggregate policies: min max", + "[aggregate-policies][min-max]", + TypesUnderTest) { + typedef TestType T; + min_max_test>(); + min_max_test>(); +} + +TEST_CASE("Aggregate policies: no op", "[aggregate-policies][validities]") { + auto n = Null(); + auto nn = NonNull(); + CHECK(n.op(0)); + CHECK(!n.op(1)); + CHECK(!nn.op(0)); + CHECK(nn.op(1)); +} From e487372b8392d7e4e2f0cf8dd77153cd7d3d944e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:56:26 +0200 Subject: [PATCH 004/456] Fix unintialized variables in nightly build. (#4428) --- TYPE: NO_HISTORY DESC: Fix unintialized variables in nightly build. --- .../query_field/test/unit_capi_query_field.cc | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 7c540fa1b4b8..52321606ca79 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -123,10 +123,10 @@ void QueryFieldFx::create_sparse_array(const std::string& array_name) { uint64_t tile_extents[] = {2, 2}; uint64_t dim_domain[] = {1, 10, 1, 10}; - tiledb_dimension_t* d1; + tiledb_dimension_t* d1 = nullptr; throw_if_setup_failed(tiledb_dimension_alloc( ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1)); - tiledb_dimension_t* d2; + tiledb_dimension_t* d2 = nullptr; throw_if_setup_failed(tiledb_dimension_alloc( ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2)); @@ -137,23 +137,23 @@ void QueryFieldFx::create_sparse_array(const std::string& array_name) { throw_if_setup_failed(tiledb_domain_add_dimension(ctx, domain, d2)); // Create attributes - tiledb_attribute_t* a; + tiledb_attribute_t* a = nullptr; throw_if_setup_failed(tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a)); - tiledb_attribute_t* b; + tiledb_attribute_t* b = nullptr; throw_if_setup_failed(tiledb_attribute_alloc(ctx, "b", TILEDB_INT32, &b)); - tiledb_attribute_t* c; + tiledb_attribute_t* c = nullptr; throw_if_setup_failed( tiledb_attribute_alloc(ctx, "c", TILEDB_STRING_ASCII, &c)); throw_if_setup_failed( tiledb_attribute_set_cell_val_num(ctx, c, TILEDB_VAR_NUM)); - tiledb_attribute_t* d; + tiledb_attribute_t* d = nullptr; throw_if_setup_failed( tiledb_attribute_alloc(ctx, "d", TILEDB_STRING_UTF8, &d)); throw_if_setup_failed( tiledb_attribute_set_cell_val_num(ctx, d, TILEDB_VAR_NUM)); // Create array schema - tiledb_array_schema_t* array_schema; + tiledb_array_schema_t* array_schema = nullptr; throw_if_setup_failed( tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema)); throw_if_setup_failed( @@ -192,11 +192,13 @@ TEST_CASE_METHOD( QueryFieldFx, "C API: argument validation", "[capi][query_field][get][args]") { - tiledb_array_t* array; + tiledb_array_t* array = nullptr; + ; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); - tiledb_query_t* query; + tiledb_query_t* query = nullptr; + ; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); tiledb_query_field_t* field = nullptr; @@ -231,11 +233,13 @@ TEST_CASE_METHOD( QueryFieldFx, "C API: argument validation", "[capi][query_field][access][args]") { - tiledb_array_t* array; + tiledb_array_t* array = nullptr; + ; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); - tiledb_query_t* query; + tiledb_query_t* query = nullptr; + ; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); tiledb_query_field_t* field = nullptr; @@ -282,11 +286,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( QueryFieldFx, "C API: argument validation", "[capi][query_field]") { - tiledb_array_t* array; + tiledb_array_t* array = nullptr; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); - tiledb_query_t* query; + tiledb_query_t* query = nullptr; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); From 5200bbb4bf4461c4ac5c9f7f23cdf02c5d2593a6 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:59:45 +0200 Subject: [PATCH 005/456] Aggregators: adding benchmark for aggregate_with_count class. (#4420) --- TYPE: IMPROVEMENT DESC: Aggregators: adding benchmark for aggregate_with_count class. --- tiledb/sm/query/query.cc | 1 - .../readers/aggregators/test/CMakeLists.txt | 2 +- .../aggregators/test/bench_aggregators.cc | 190 ++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 92115ed20486..55aff280373a 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -479,7 +479,6 @@ Status Query::finalize() { RETURN_NOT_OK(strategy_->finalize()); - copy_aggregates_data_to_user_buffer(); status_ = QueryStatus::COMPLETED; return Status::Ok(); } diff --git a/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt b/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt index 9e2904596c12..386fd236bc12 100644 --- a/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt +++ b/tiledb/sm/query/readers/aggregators/test/CMakeLists.txt @@ -27,6 +27,6 @@ include(unit_test) commence(unit_test aggregators) - this_target_sources(main.cc unit_aggregate_with_count.cc unit_aggregation_policies.cc unit_aggregators.cc) + this_target_sources(main.cc bench_aggregators.cc unit_aggregate_with_count.cc unit_aggregation_policies.cc unit_aggregators.cc) this_target_object_libraries(aggregators) conclude(unit_test) diff --git a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc new file mode 100644 index 000000000000..8e7aa018c621 --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc @@ -0,0 +1,190 @@ +/** + * @file bench_aggregators.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Benchmarks the `AggregateWithCount` class. + */ + +#include "tiledb/common/common.h" +#include "tiledb/sm/query/readers/aggregators/aggregate_buffer.h" +#include "tiledb/sm/query/readers/aggregators/aggregate_with_count.h" +#include "tiledb/sm/query/readers/aggregators/min_max.h" +#include "tiledb/sm/query/readers/aggregators/safe_sum.h" +#include "tiledb/sm/query/readers/aggregators/sum_type.h" +#include "tiledb/sm/query/readers/aggregators/validity_policies.h" + +#include + +using namespace tiledb::sm; + +const uint64_t num_cells = 10 * 1024 * 1024; + +// Fixed seed for determinism. +static std::vector generator_seed_arr = { + 0xBE08D299, 0x4E996D11, 0x402A1E10, 0x95379958, 0x22101AA9}; + +static std::atomic generator_seed = 0; +std::once_flag once_flag; +static std::mt19937_64 generator; + +void set_generator_seed() { + // Set the global seed only once + std::call_once(once_flag, []() { + static std::random_device rd; + static std::mt19937 gen(rd()); + static std::uniform_int_distribution dis( + 0, generator_seed_arr.size() - 1); + generator_seed = generator_seed_arr[dis(gen)]; + std::string gen_seed_str = + "Generator seed: " + std::to_string(generator_seed); + puts(gen_seed_str.c_str()); + }); +} + +template +T random(size_t max) { + // Pick generator seed at random. + set_generator_seed(); + + thread_local static std::uniform_int_distribution distribution( + 0, max); + return distribution(generator); +} + +template +struct fixed_type_data { + using type = T; + typedef T fixed_type; +}; + +template <> +struct fixed_type_data { + using type = std::string; + typedef uint64_t fixed_type; +}; + +template +tuple< + std::vector::fixed_type>*, + std::string*, + std::vector*, + std::vector*> +get_data() { + static std::vector::fixed_type> fixed_data; + static std::vector validity_data; + static std::string var_data; + static std::vector bitmap; + static bool data_generated = false; + if (!data_generated) { + fixed_data.resize(num_cells); + validity_data.resize(num_cells); + bitmap.resize(num_cells); + + uint64_t offset = 0; + for (uint64_t c = 0; c < num_cells; c++) { + if constexpr (std::is_same::value) { + uint64_t size = random(20); + std::string value; + for (uint64_t s = 0; s < size; s++) { + var_data += '0' + random(36); + } + + fixed_data[c] = offset; + offset += size; + } else { + fixed_data[c] = random(200); + } + + validity_data[c] = random(1); + bitmap[c] = random(1); + } + + data_generated = true; + } + + return {&fixed_data, &var_data, &validity_data, &bitmap}; +} + +template < + typename T, + typename AggregateT, + typename PolicyT, + typename ValidityPolicyT> +void run_bench() { + const bool var_sized = std::is_same::value; + bool nullable = GENERATE(true, false); + bool use_bitmap = GENERATE(true, false); + bool segmented = GENERATE(true, false); + const uint64_t increment = segmented ? 4 : num_cells; + + // Get data. + auto&& [fd, vd, vald, b] = get_data(); + auto fixed_data = fd; + auto var_data = vd; + auto validity_data = vald; + auto bitmap = b; + + DYNAMIC_SECTION( + "Var sized: " << (var_sized ? "true" : "false") + << ", Nullable: " << (nullable ? "true" : "false") + << ", Use bitmap: " << (use_bitmap ? "true" : "false") + << ", Segmented: " << (segmented ? "true" : "false")) { + BENCHMARK("Bench") { + AggregateWithCount aggregator( + FieldInfo("a1", var_sized, nullable, 1)); + for (uint64_t s = 0; s < num_cells; s += increment) { + AggregateBuffer input_data{ + s, + std::min(s + increment, num_cells), + fixed_data->data(), + var_sized ? std::make_optional(var_data->data()) : nullopt, + nullable ? std::make_optional(validity_data->data()) : nullopt, + false, + use_bitmap ? std::make_optional(bitmap->data()) : nullopt, + 0}; + aggregator.template aggregate(input_data); + } + }; + } +} + +TEST_CASE("Aggregate with count: sum", "[benchmark][sum][.hide]") { + typedef uint64_t T; + run_bench::sum_type, SafeSum, NonNull>(); +} + +typedef tuple MaxTypesUnderTest; +TEMPLATE_LIST_TEST_CASE( + "Aggregate with count: max", "[benchmark][max][.hide]", MaxTypesUnderTest) { + typedef TestType T; + run_bench< + T, + typename type_data::value_type, + MinMax::value_type>>, + NonNull>(); +} From dd0006a13791016e898071ca93734ef2d4524032 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 17 Oct 2023 16:45:54 +0300 Subject: [PATCH 006/456] Run the backport workflow on the `pull_request_target` event. (#4425) It matches what the repository with the backport action does. Hopefully fixes [SC-35571]. https://github.com/zephyrproject-rtos/action-backport/blob/b7bc23473310e9982167a8a295fe91e59fd4bf0a/.github/workflows/backport.yml#L3 --- TYPE: NO_HISTORY --- .github/workflows/backport.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 8aa53de34727..323473671282 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -1,6 +1,6 @@ name: Backport on: - pull_request: + pull_request_target: types: - closed - labeled @@ -24,7 +24,7 @@ jobs: ) steps: - name: Backport - uses: zephyrproject-rtos/action-backport@v2.0.3-3 + uses: zephyrproject-rtos/action-backport@7e74f601d11eaca577742445e87775b5651a965f #tag=v2.0.3-3 with: issue_labels: Backport github_token: ${{ secrets.GITHUB_TOKEN }} From 657f48206e2ca9a91ef72d11ecf33eb26dad019e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 18 Oct 2023 11:45:01 +0200 Subject: [PATCH 007/456] Fix cast issue in nightly build. (#4432) --- TYPE: NO_HISTORY DESC: Fix cast issue in nightly build. --- tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc index 8e7aa018c621..16e0a7d133bc 100644 --- a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc @@ -73,7 +73,7 @@ T random(size_t max) { thread_local static std::uniform_int_distribution distribution( 0, max); - return distribution(generator); + return static_cast(distribution(generator)); } template From dde86ab5b7a5cd6bf5fee337572aa35073fcb699 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 18 Oct 2023 11:45:41 +0200 Subject: [PATCH 008/456] Adding CPP example for consolidation plan. (#4433) This adds a CPP example for the consolidation APIs. It also fixes a doxygen issue in the CPP header. --- TYPE: IMPROVEMENT DESC: Adding CPP example for consolidation plan. --- examples/cpp_api/consolidation_plan.cc | 133 ++++++++++++++++++ .../cpp_api/consolidation_plan_experimental.h | 17 +-- 2 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 examples/cpp_api/consolidation_plan.cc diff --git a/examples/cpp_api/consolidation_plan.cc b/examples/cpp_api/consolidation_plan.cc new file mode 100644 index 000000000000..1754528b2112 --- /dev/null +++ b/examples/cpp_api/consolidation_plan.cc @@ -0,0 +1,133 @@ +/** + * @file consolidation_plan.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 1D sparse array with three + * fragments and generate the consolidation plan. + */ + +#include +#include +#include +#include + +using namespace tiledb; + +// Name of array. +std::string array_name("consolidation_plan_example_array"); + +void create_array() { + // Create a TileDB context. + Context ctx; + + // If the array already exists on disk, return immediately. + if (Object::object(ctx, array_name).type() == Object::Type::Array) + return; + + // The array will be a vector with one dimension "rows", with domain [1,100]. + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "rows", {{1, 100}}, 4)); + + // The array will be sparse. + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + + // Add a single attribute "a" so each (i) cell can store an integer. + schema.add_attribute(Attribute::create(ctx, "a")); + + // Create the (empty) array on disk. + Array::create(array_name, schema); +} + +void write_fragment(int min, int max) { + Context ctx; + + // Open the array for writing and create the query. + Array array(ctx, array_name, TILEDB_WRITE); + Query query(ctx, array); + + // Set layout to global + query.set_layout(TILEDB_GLOBAL_ORDER); + + // Write a fragment with cells [min,max] for "rows" and "a". + std::vector coords_rows_1(max - min); + std::iota(coords_rows_1.begin(), coords_rows_1.end(), min); + std::vector data_1(max - min); + std::iota(data_1.begin(), data_1.end(), min); + query.set_data_buffer("a", data_1).set_data_buffer("rows", coords_rows_1); + query.submit(); + + // Finalize - IMPORTANT! + query.finalize(); + + // Close the array + array.close(); +} + +void write_array() { + // Write a fragment with cells [1,30] for "rows" and "a". + write_fragment(1, 30); + + // Write another fragment with cells [15,44] for "rows" and "a". + write_fragment(15, 44); + + // Write one last fragment with cells [80,89] for "rows" and "a". + write_fragment(80, 89); +} + +void print_consolidation_plan() { + Context ctx; + + // Prepare the array for reading + Array array(ctx, array_name, TILEDB_READ); + + // Create the plan + ConsolidationPlan plan(ctx, array, 1000); + + auto num_nodes = plan.num_nodes(); + std::cout << "Consolidation plan for " << num_nodes << " nodes:" << std::endl; + for (uint64_t n = 0; n < num_nodes; n++) { + auto num_fragments = plan.num_fragments(n); + std::cout << " Node " << n << " with " << num_fragments + << " fragments:" << std::endl; + for (uint64_t f = 0; f < num_fragments; f++) { + std::cout << " " << plan.fragment_uri(n, f) << std::endl; + } + } +} + +int main() { + Context ctx; + if (Object::object(ctx, array_name).type() != Object::Type::Array) { + create_array(); + write_array(); + } + + print_consolidation_plan(); + return 0; +} diff --git a/tiledb/sm/cpp_api/consolidation_plan_experimental.h b/tiledb/sm/cpp_api/consolidation_plan_experimental.h index 32263b8001ff..e5f577210bfa 100644 --- a/tiledb/sm/cpp_api/consolidation_plan_experimental.h +++ b/tiledb/sm/cpp_api/consolidation_plan_experimental.h @@ -40,16 +40,8 @@ namespace tiledb { class ConsolidationPlan { public: /** - * @brief Constructor. This opens the group for the given query type. The - * destructor calls the `close()` method. - * - * **Example:** - * - * @code{.cpp} - * // Open the group for reading - * tiledb::Context ctx; - * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ); - * @endcode + * Constructor. This creates the consolidation plan for an array with the + * given desired maximum fragment size. * * @param ctx TileDB context. * @param array The array. @@ -90,6 +82,8 @@ class ConsolidationPlan { /** * Returns the number of fragments for a node in the consolidation plan. + * + * @param node_idx Node index to retreive the data for. */ uint64_t num_fragments(uint64_t node_idx) const { uint64_t num; @@ -102,6 +96,9 @@ class ConsolidationPlan { /** * Returns the fragment uri for a node/fragment in the consolidation plan. + * + * @param node_idx Node index to retreive the data for. + * @param fragment_idx Fragment index to retreive the data for. */ std::string fragment_uri(uint64_t node_idx, uint64_t fragment_idx) const { const char* uri_c; From ae7ca5797b6d4beb12508872c9670867694fe531 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 18 Oct 2023 18:33:14 +0300 Subject: [PATCH 009/456] Support GCS on Windows. (#4138) [SC-3139](https://app.shortcut.com/tiledb-inc/story/3139/enable-gcs-support-on-windows) This PR adds support for Google Cloud Storage on Windows. Most things to enable it have already been done, this PR finishes it up by adding a switch on the Windows bootstrap script and a GCS CI leg on Windows CI. --- TYPE: FEATURE DESC: Add support for Google Cloud Storage on Windows --- .github/workflows/build-windows.yml | 18 +++++++-- bootstrap.ps1 | 14 ++++++- scripts/install-run-gcs-emu.ps1 | 59 +++++++++++++++++++++++++++++ test/src/unit-gcs.cc | 22 +++++------ test/src/unit-gs.cc | 22 +++++------ tiledb/CMakeLists.txt | 36 ++++++++++++++++-- tiledb/sm/filesystem/gcs.cc | 15 ++++---- tiledb/sm/filesystem/gcs.h | 9 +++++ 8 files changed, 156 insertions(+), 39 deletions(-) create mode 100644 scripts/install-run-gcs-emu.ps1 diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index d830a0d5f057..cca993b2f65f 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -21,11 +21,10 @@ jobs: matrix: os: [windows-2019, windows-2022] #considering https://stackoverflow.com/questions/65035256/how-to-access-matrix-variables-in-github-actions - environ: [azure, s3, serialization] + environ: [azure, s3, gcs, serialization] include: - environ: 'azure' TILEDB_AZURE: ON - TILEDB_USE_CUSTOM_NODEJS: OFF TILEDB_ARROW_TESTS: ON TILEDB_SERIALIZATION: OFF TILEDB_S3: OFF @@ -36,6 +35,13 @@ jobs: TILEDB_SERIALIZATION: OFF TILEDB_AZURE: OFF TILEDB_WEBP: OFF + - environ: 'gcs' + TILEDB_GCS: ON + TILEDB_ARROW_TESTS: ON + TILEDB_SERIALIZATION: OFF + TILEDB_AZURE: OFF + TILEDB_S3: OFF + TILEDB_WEBP: OFF - environ: 'serialization' TILEDB_S3: OFF TILEDB_AZURE: OFF @@ -50,6 +56,7 @@ jobs: TILEDB_GA_IMAGE_NAME: ${{ matrix.os }} TILEDB_S3: ${{ matrix.TILEDB_S3 }} TILEDB_AZURE: ${{ matrix.TILEDB_AZURE }} #azure }} + TILEDB_GCS: ${{ matrix.TILEDB_GCS }} #gcs }} TILEDB_SERIALIZATION: ${{ matrix.TILEDB_SERIALIZATION }} #serialization }} TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }} TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} @@ -166,8 +173,7 @@ jobs: $bootstrapOptions = $bootstrapOptions + " -_EnableCrc32" } if ($env:TILEDB_GCS -eq "ON") { - $bootstrapOptions = "-EnableGCS " + $bootstrapOptions - #NOTE: GCS simulator not yet actually in place. + $bootstrapOptions = "-EnableGcs " + $bootstrapOptions } if ($env:TILEDB_SERIALIZATION -eq "ON") { $bootstrapOptions = "-EnableSerialization " + $bootstrapOptions @@ -287,6 +293,10 @@ jobs: tasklist | findstr /i node.exe } + if ($env:TILEDB_GCS -eq "ON") { + & "$env:BUILD_SOURCESDIRECTORY\scripts\install-run-gcs-emu.ps1" + } + # Actually run tests $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$CMakeBuildType\tiledb_unit.exe -d=yes" diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 0576bb7f457b..6c3c0358406b 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -50,6 +50,9 @@ Enables building with the Azure storage backend. .PARAMETER EnableS3 Enables building with the S3 storage backend. +.PARAMETER EnableGcs +Enables building with the GCS storage backend. + .PARAMETER EnableSerialization Enables building with serialization support. @@ -126,6 +129,7 @@ Param( [switch]$DisableVcpkg, [switch]$EnableAzure, [switch]$EnableS3, + [switch]$EnableGcs, [switch]$EnableSerialization, [switch]$EnableStaticTileDB, [switch]$EnableTools, @@ -208,6 +212,12 @@ if ($EnableS3.IsPresent) { $UseS3 = "ON" } +# Set TileDB GCS flag +$UseGcs = "OFF" +if ($EnableGcs.IsPresent) { + $UseGcs = "ON" +} + $UseSerialization = "OFF" if ($EnableSerialization.IsPresent) { $UseSerialization = "ON" @@ -307,7 +317,7 @@ if ($SourceDirectory -eq $BinaryDirectory) { } # Check cmake binary -if ((Get-Command "cmake.exe" -ErrorAction SilentlyContinue) -eq $null) { +if ($null -eq (Get-Command "cmake.exe" -ErrorAction SilentlyContinue)) { Throw "Unable to find cmake.exe in your PATH." } if ($CMakeGenerator -eq $null) { @@ -316,7 +326,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/scripts/install-run-gcs-emu.ps1 b/scripts/install-run-gcs-emu.ps1 new file mode 100644 index 000000000000..95ce71674ef2 --- /dev/null +++ b/scripts/install-run-gcs-emu.ps1 @@ -0,0 +1,59 @@ +<# +.SYNOPSIS +This is a Powershell script to install/run the Google Cloud Storage Testbench on Windows. + +.DESCRIPTION +This is a Powershell script to install/run the Google Cloud Storage Testbench on Windows. + +.PARAMETER RunTestbench +Attempt to execute the testbench without installing (assumes already installed.) + +.PARAMETER Clean +Remove the existing testbench installation. + +.LINK +https://github.com/TileDB-Inc/TileDB +#> + +[CmdletBinding()] +Param( + [switch]$RunTestbench, + [switch]$Clean +) + +$ErrorActionPreference = "Stop" + +$version = "v0.36.0" +$testbenchPath = "$env:TEMP\storage-testbench-$version" +$venvPath = "$env:TEMP\storage-testbench-venv" + +# Clean the existing testbench installation +if ($Clean) { + Write-Host "Cleaning existing testbench installation..." + Remove-Item -Recurse -Force $testbenchPath + Remove-Item -Recurse -Force $venvPath +} + +if (!$RunTestbench) { + #if testbench is not installed + if (!(Test-Path $testbenchPath)) { + #install testbench + Write-Host "Installing Google Cloud Storage Testbench..." + git clone --branch $version --depth 1 https://github.com/googleapis/storage-testbench.git $testbenchPath + Write-Host "Setting up Python virtual environment..." + py -m venv $venvPath + & $venvPath\Scripts\activate + py -m pip install -e $testbenchPath + } +} + +# Export the CLOUD_STORAGE_EMULATOR_ENDPOINT environment variable +$env:CLOUD_STORAGE_EMULATOR_ENDPOINT = "http://localhost:9000" + +# Run the testbench +& $venvPath\Scripts\activate +# $testbenchCmd = "start `"Google Cloud Storage Testbench`" py testbench_run.py localhost 9000 10" +$testbenchCmd = "start `"Google Cloud Storage Testbench`" /D `"$testbenchPath`" py testbench_run.py localhost 9000 10" +cmd /c $testbenchCmd + +Write-Host "Finished." diff --git a/test/src/unit-gcs.cc b/test/src/unit-gcs.cc index e876e73781e6..f70d9d955ef6 100644 --- a/test/src/unit-gcs.cc +++ b/test/src/unit-gcs.cc @@ -149,7 +149,7 @@ TEST_CASE_METHOD(GCSFx, "Test GCS filesystem, file management", "[gcs]") { REQUIRE(is_empty); // Continue building the hierarchy - bool is_object; + bool is_object = false; REQUIRE(gcs_.touch(URI(file1)).ok()); REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); REQUIRE(is_object); @@ -289,7 +289,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the files do not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); @@ -314,7 +314,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -377,7 +377,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); REQUIRE(!is_object); @@ -395,7 +395,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -458,7 +458,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the files do not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); @@ -483,7 +483,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -546,7 +546,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); REQUIRE(!is_object); @@ -564,7 +564,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -619,7 +619,7 @@ TEST_CASE_METHOD( gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); @@ -637,7 +637,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; diff --git a/test/src/unit-gs.cc b/test/src/unit-gs.cc index 247e1509cbcd..53b1a6605b6a 100644 --- a/test/src/unit-gs.cc +++ b/test/src/unit-gs.cc @@ -134,7 +134,7 @@ TEST_CASE_METHOD(GSFx, "Test GS filesystem, file management", "[gs]") { REQUIRE(is_empty); // Continue building the hierarchy - bool is_object; + bool is_object = false; REQUIRE(gcs_.touch(URI(file1)).ok()); REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); REQUIRE(is_object); @@ -257,7 +257,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the files do not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); @@ -282,7 +282,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -345,7 +345,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); REQUIRE(!is_object); @@ -363,7 +363,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -426,7 +426,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the files do not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); @@ -451,7 +451,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -514,7 +514,7 @@ TEST_CASE_METHOD( gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); REQUIRE(!is_object); @@ -532,7 +532,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; @@ -587,7 +587,7 @@ TEST_CASE_METHOD( gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); // Before flushing, the file does not exist - bool is_object; + bool is_object = false; REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); REQUIRE(!is_object); @@ -605,7 +605,7 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; - uint64_t bytes_read; + uint64_t bytes_read = 0; REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); CHECK(26 == bytes_read); bool allok = true; diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 0a2cc082aea3..9b0e32ec5404 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -592,8 +592,24 @@ endif() if (TILEDB_GCS) find_package(google_cloud_cpp_storage CONFIG REQUIRED) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE CURL::libcurl google-cloud-cpp::storage google-cloud-cpp::experimental-storage-grpc) - install_all_target_libs(google-cloud-cpp::storage) + target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE + google-cloud-cpp::storage + google-cloud-cpp::rest_internal + absl::str_format_internal + absl::strings + absl::strings_internal + absl::crc32c + absl::time + absl::time_zone + absl::crc_internal + absl::int128 + absl::raw_logging_internal + google-cloud-cpp::common + CURL::libcurl + OpenSSL::Crypto + Crc32c::crc32c + ) + install_all_target_libs("google-cloud-cpp::storage;google-cloud-cpp::rest_internal;absl::str_format_internal;absl::strings;absl::strings_internal;absl::crc32c;absl::time;absl::time_zone;absl::crc_internal;absl::int128;absl::raw_logging_internal;google-cloud-cpp::common;CURL::libcurl;OpenSSL::Crypto;Crc32c::crc32c") endif() # Libcurl @@ -719,6 +735,10 @@ if (WIN32) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE userenv ws2_32 wininet winhttp bcrypt ncrypt crypt32 version secur32) endif() + if (TILEDB_GCS) + target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE ws2_32 crypt32) + endif() + if (TILEDB_AZURE) # WebServices, winhttp and crypt32 needed by Azure storage on windows target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE WebServices winhttp crypt32) @@ -967,7 +987,17 @@ if (TILEDB_STATIC) append_dep_lib(LibLZMA::LibLZMA) append_dep_lib(google-cloud-cpp::storage) - append_dep_lib(google-cloud-cpp::experimental-storage-grpc) + append_dep_lib(google-cloud-cpp::common) + append_dep_lib(google-cloud-cpp::rest_internal) + append_dep_lib(absl::str_format_internal) + append_dep_lib(absl::strings) + append_dep_lib(absl::strings_internal) + append_dep_lib(absl::crc32c) + append_dep_lib(absl::time) + append_dep_lib(absl::time_zone) + append_dep_lib(absl::crc_internal) + append_dep_lib(absl::int128) + append_dep_lib(absl::raw_logging_internal) append_dep_lib(libmagic) append_dep_lib(unofficial::libmagic::libmagic) diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 4e4e6b155f47..880ed077268b 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -32,11 +32,6 @@ #ifdef HAVE_GCS -#include -#include -#include "google/cloud/storage/oauth2/credentials.h" -#include "google/cloud/storage/oauth2/google_credentials.h" - #include #include @@ -1102,15 +1097,19 @@ Status GCS::flush_object_direct(const URI& uri) { std::string object_path; RETURN_NOT_OK(parse_gcs_uri(uri, &bucket_name, &object_path)); - absl::string_view write_buffer( - static_cast(write_cache_buffer->data()), - write_cache_buffer->size()); + Buffer buffer_moved; // Protect 'write_cache_map_' from multiple writers. std::unique_lock cache_lock(write_cache_map_lock_); + // Erasing the buffer from the map will free its memory. + // We have to move it to a local variable first. + buffer_moved = std::move(*write_cache_buffer); write_cache_map_.erase(uri.to_string()); cache_lock.unlock(); + absl::string_view write_buffer( + static_cast(buffer_moved.data()), buffer_moved.size()); + google::cloud::StatusOr object_metadata = client_->InsertObject( bucket_name, object_path, std::move(write_buffer)); diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index a84d3adcd9f8..9d57abecb25a 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -35,7 +35,16 @@ #ifdef HAVE_GCS +#if defined(_MSC_VER) +#pragma warning(push) +// One abseil file has a warning that fails on Windows when compiling with +// warnings as errors. +#pragma warning(disable : 4127) // conditional expression is constant +#endif #include +#if defined(_MSC_VER) +#pragma warning(pop) +#endif #include "tiledb/common/rwlock.h" #include "tiledb/common/status.h" From 888c6db3a366594a28df9d96c02c4874122eddcf Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 19 Oct 2023 04:52:13 +0200 Subject: [PATCH 010/456] Aggregates: change to use size instead of min cell/max cell. (#4436) This changes the AggregateWithCount class to compute using a size only instead of min cell/max cell. --- TYPE: IMPROVEMENT DESC: Aggregates: change to use size instead of min cell/max cell. --- .../readers/aggregators/aggregate_buffer.h | 18 +++++++----------- .../readers/aggregators/aggregate_with_count.h | 12 ++++-------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/tiledb/sm/query/readers/aggregators/aggregate_buffer.h b/tiledb/sm/query/readers/aggregators/aggregate_buffer.h index baad19923a70..74cd4806cbbb 100644 --- a/tiledb/sm/query/readers/aggregators/aggregate_buffer.h +++ b/tiledb/sm/query/readers/aggregators/aggregate_buffer.h @@ -91,14 +91,9 @@ class AggregateBuffer { return bitmap_data_.has_value(); } - /** Returns the min cell position to aggregate. */ - uint64_t min_cell() const { - return min_cell_; - } - - /** Returns the max cell position to aggregate. */ - uint64_t max_cell() const { - return max_cell_; + /** Returns the number of cells to aggregate. */ + uint64_t size() const { + return max_cell_ - min_cell_; } /** @@ -111,7 +106,8 @@ class AggregateBuffer { * @return Value. */ template - inline T value_at(const uint64_t cell_idx) const { + inline T value_at(uint64_t cell_idx) const { + cell_idx += min_cell_; if constexpr (std::is_same_v) { if (var_data_.has_value()) { auto offsets = static_cast(fixed_data_); @@ -138,7 +134,7 @@ class AggregateBuffer { * @return Validity value. */ inline uint8_t validity_at(const uint64_t cell_idx) const { - return validity_data_.value()[cell_idx]; + return validity_data_.value()[cell_idx + min_cell_]; } /** @@ -150,7 +146,7 @@ class AggregateBuffer { */ template inline BitmapType bitmap_at(const uint64_t cell_idx) const { - return static_cast(bitmap_data_.value())[cell_idx]; + return static_cast(bitmap_data_.value())[cell_idx + min_cell_]; } private: diff --git a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h index e22186f9cd8e..082d969767df 100644 --- a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h +++ b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h @@ -97,8 +97,7 @@ class AggregateWithCount { if (input_data.has_bitmap()) { if (field_info_.is_nullable_) { // Process for nullable values with bitmap. - for (uint64_t c = input_data.min_cell(); c < input_data.max_cell(); - c++) { + for (uint64_t c = 0; c < input_data.size(); c++) { auto bitmap_val = input_data.bitmap_at(c); if (val_policy.op(input_data.validity_at(c)) && bitmap_val != 0) { auto value = value_at(input_data, c); @@ -110,8 +109,7 @@ class AggregateWithCount { } } else { // Process for non nullable values with bitmap. - for (uint64_t c = input_data.min_cell(); c < input_data.max_cell(); - c++) { + for (uint64_t c = 0; c < input_data.size(); c++) { auto bitmap_val = input_data.bitmap_at(c); auto value = value_at(input_data, c); for (BITMAP_T i = 0; i < bitmap_val; i++) { @@ -123,8 +121,7 @@ class AggregateWithCount { } else { if (field_info_.is_nullable_) { // Process for nullable values with no bitmap. - for (uint64_t c = input_data.min_cell(); c < input_data.max_cell(); - c++) { + for (uint64_t c = 0; c < input_data.size(); c++) { if (val_policy.op(input_data.validity_at(c))) { auto value = value_at(input_data, c); agg_policy.op(value, res, count); @@ -133,8 +130,7 @@ class AggregateWithCount { } } else { // Process for non nullable values with no bitmap. - for (uint64_t c = input_data.min_cell(); c < input_data.max_cell(); - c++) { + for (uint64_t c = 0; c < input_data.size(); c++) { auto value = value_at(input_data, c); agg_policy.op(value, res, count); count++; From 3d4193a8bffaf54c823f4591a4a1652aeb8ba3b4 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 19 Oct 2023 11:16:00 +0200 Subject: [PATCH 011/456] Fix CPP set buffer APIs to work with aggregates. (#4434) This fixes the CPP set buffer APIs to function properly with aggregates using the new query field API. --- TYPE: IMPROVEMENT DESC: Fix CPP set buffer APIs to work with aggregates. --- test/src/test-cppapi-aggregates.cc | 261 ++++-------------- .../api/c_api/query_field/query_field_api.cc | 16 +- .../query_field/test/unit_capi_query_field.cc | 25 +- tiledb/sm/c_api/tiledb_experimental.h | 1 + tiledb/sm/cpp_api/query.h | 113 ++++---- .../readers/aggregators/mean_aggregator.h | 2 +- .../aggregators/test/unit_aggregators.cc | 16 +- 7 files changed, 151 insertions(+), 283 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 91a1aa557b4f..a45915fae5dd 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -1186,12 +1186,7 @@ TEST_CASE_METHOD( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - uint64_t count_data_size = sizeof(uint64_t); - // TODO: Change to real CPPAPI. - CHECK(query.ptr() - ->query_ - ->set_data_buffer("Count", count.data(), &count_data_size) - .ok()); + query.set_data_buffer("Count", count); if (request_data) { query.set_data_buffer("d1", dim1); @@ -1219,8 +1214,8 @@ TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["Count"]) == 1' once we use the - // set_data_buffer api. + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Count"]) == 1); CHECK(count[0] == expected_count); if (request_data) { @@ -1288,22 +1283,9 @@ TEMPLATE_LIST_TEST_CASE_METHOD( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - uint64_t sum_data_size = - sizeof(typename tiledb::sm::sum_type_data::sum_type); - // TODO: Change to real CPPAPI. - CHECK(query.ptr() - ->query_->set_data_buffer("Sum", sum.data(), &sum_data_size) - .ok()); - uint64_t returned_validity_size = 1; + query.set_data_buffer("Sum", sum); if (CppAggregatesFx::nullable_) { - // TODO: Change to real CPPAPI. Use set_validity_buffer from the - // internal query directly because the CPPAPI doesn't know what is - // an aggregate and what the size of an aggregate should be. - CHECK(query.ptr() - ->query_ - ->set_validity_buffer( - "Sum", sum_validity.data(), &returned_validity_size) - .ok()); + query.set_validity_buffer("Sum", sum_validity); } if (request_data) { @@ -1352,8 +1334,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["Sum"]) == 1' once we use the - // set_data_buffer api. + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Sum"]) == 1); CHECK(sum[0] == expected_sum); if (request_data) { @@ -1422,25 +1404,9 @@ TEMPLATE_LIST_TEST_CASE_METHOD( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer from the internal - // query directly because the CPPAPI doesn't know what is an aggregate - // and what the size of an aggregate should be. - uint64_t returned_mean_size = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer("Mean", &mean[0], &returned_mean_size) - .ok()); - uint64_t returned_validity_size = 1; + query.set_data_buffer("Mean", mean); if (CppAggregatesFx::nullable_) { - // TODO: Change to real CPPAPI. Use set_validity_buffer from the - // internal query directly because the CPPAPI doesn't know what is - // an aggregate and what the size of an aggregate should be. - CHECK(query.ptr() - ->query_ - ->set_validity_buffer( - "Mean", mean_validity.data(), &returned_validity_size) - .ok()); + query.set_validity_buffer("Mean", mean_validity); } if (request_data) { @@ -1492,9 +1458,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["Mean"]) == 1' once we use - // the set_data_buffer api. - CHECK(returned_mean_size == 8); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Mean"]) == 1); CHECK(mean[0] == expected_mean); if (request_data) { @@ -1578,26 +1543,9 @@ TEMPLATE_LIST_TEST_CASE( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer and - // set_validity_buffer from the internal query directly because the - // CPPAPI doesn't know what is an aggregate and what the size of an - // aggregate should be. - uint64_t returned_min_max_size = cell_size; - uint64_t returned_validity_size = 1; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "MinMax", min_max.data(), &returned_min_max_size) - .ok()); + query.set_data_buffer("MinMax", min_max.data(), min_max.size()); if (fx.nullable_) { - CHECK(query.ptr() - ->query_ - ->set_validity_buffer( - "MinMax", - min_max_validity.data(), - &returned_validity_size) - .ok()); + query.set_validity_buffer("MinMax", min_max_validity); } if (request_data) { @@ -1648,9 +1596,8 @@ TEMPLATE_LIST_TEST_CASE( } } - // TODO: use 'std::get<1>(result_el["MinMax"]) == 1' once we use the - // set_data_buffer api. - CHECK(returned_min_max_size == cell_size); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["MinMax"]) == min_max.size()); CHECK(min_max == expected_min_max); if (request_data) { @@ -1695,7 +1642,11 @@ TEMPLATE_LIST_TEST_CASE( query.ptr()->query_->add_aggregator_to_default_channel( "MinMax", std::make_shared(tiledb::sm::FieldInfo( - "a1", true, fx.nullable_, TILEDB_VAR_NUM))); + "a1", + true, + fx.nullable_, + TILEDB_VAR_NUM, + tiledb::sm::Datatype::STRING_ASCII))); fx.set_ranges_and_condition_if_needed(array, query, true); @@ -1711,36 +1662,10 @@ TEMPLATE_LIST_TEST_CASE( a1_data.resize(100); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer and - // set_validity_buffer from the internal query directly because the - // CPPAPI doesn't know what is an aggregate and what the size of an - // aggregate should be. - uint64_t returned_min_max_data_size = 10; - uint64_t returned_min_max_offsets_size = 8; - uint64_t returned_validity_size = 1; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "MinMax", - min_max_data.data(), - &returned_min_max_data_size) - .ok()); - CHECK(query.ptr() - ->query_ - ->set_offsets_buffer( - "MinMax", - min_max_offset.data(), - &returned_min_max_offsets_size) - .ok()); + query.set_data_buffer("MinMax", min_max_data); + query.set_offsets_buffer("MinMax", min_max_offset); if (fx.nullable_) { - CHECK(query.ptr() - ->query_ - ->set_validity_buffer( - "MinMax", - min_max_validity.data(), - &returned_validity_size) - .ok()); + query.set_validity_buffer("MinMax", min_max_validity); } if (request_data) { @@ -1789,17 +1714,15 @@ TEMPLATE_LIST_TEST_CASE( } } - // TODO: use 'std::get<1>(result_el["MinMax"]) == 1' once we use the - // set_data_buffer api. - CHECK(returned_min_max_offsets_size == 8); - CHECK(returned_min_max_data_size == expected_min_max.size()); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["MinMax"]) == expected_min_max.size()); + CHECK(std::get<0>(result_el["MinMax"]) == 1); min_max_data.resize(expected_min_max.size()); CHECK(min_max_data == expected_min_max); CHECK(min_max_offset[0] == 0); if (request_data) { - auto result_el = query.result_buffer_elements_nullable(); a1_offsets[std::get<0>(result_el["a1"])] = std::get<1>(result_el["a1"]); fx.validate_data_var( @@ -1873,22 +1796,13 @@ TEMPLATE_LIST_TEST_CASE_METHOD( uint64_t cell_size = std::is_same::value ? CppAggregatesFx::STRING_CELL_VAL_NUM : sizeof(T); - uint64_t null_count = 0; + std::vector null_count(1); std::vector dim1(100); std::vector dim2(100); std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer from the internal - // query directly because the CPPAPI doesn't know what is an aggregate - // and what the size of an aggregate should be. - uint64_t returned_null_count_size = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount", &null_count, &returned_null_count_size) - .ok()); + query.set_data_buffer("NullCount", null_count); if (request_data) { query.set_data_buffer("d1", dim1); @@ -1921,10 +1835,9 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["NullCount"]) == 1' once we use - // the set_data_buffer api. - CHECK(returned_null_count_size == 8); - CHECK(null_count == expected_null_count); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["NullCount"]) == 1); + CHECK(null_count[0] == expected_null_count); if (request_data) { CppAggregatesFx::validate_data( @@ -1972,7 +1885,7 @@ TEST_CASE_METHOD( set_ranges_and_condition_if_needed(array, query, true); // Set the data buffer for the aggregator. - uint64_t null_count = 0; + std::vector null_count(1); std::vector dim1(100); std::vector dim2(100); std::vector a1_offsets(100); @@ -1980,17 +1893,7 @@ TEST_CASE_METHOD( a1_data.resize(100); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer and - // set_validity_buffer from the internal query directly because the - // CPPAPI doesn't know what is an aggregate and what the size of an - // aggregate should be. - uint64_t returned_null_count_size = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount", &null_count, &returned_null_count_size) - .ok()); + query.set_data_buffer("NullCount", null_count); if (request_data) { query.set_data_buffer("d1", dim1); @@ -2023,13 +1926,11 @@ TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["NullCount"]) == 1' once we use - // the set_data_buffer api. - CHECK(returned_null_count_size == 8); - CHECK(null_count == expected_null_count); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["NullCount"]) == 1); + CHECK(null_count[0] == expected_null_count); if (request_data) { - auto result_el = query.result_buffer_elements_nullable(); a1_offsets[std::get<0>(result_el["a1"])] = std::get<1>(result_el["a1"]); validate_data_var( @@ -2087,8 +1988,8 @@ TEST_CASE_METHOD( set_ranges_and_condition_if_needed(array, query, true); // Set the data buffer for the aggregator. - uint64_t null_count = 0; - uint64_t null_count2 = 0; + std::vector null_count(1); + std::vector null_count2(1); std::vector dim1(100); std::vector dim2(100); std::vector a1_offsets(100); @@ -2096,24 +1997,8 @@ TEST_CASE_METHOD( a1_data.resize(100); std::vector a1_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer and - // set_validity_buffer from the internal query directly because the - // CPPAPI doesn't know what is an aggregate and what the size of an - // aggregate should be. - uint64_t returned_null_count_size = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount", &null_count, &returned_null_count_size) - .ok()); - - uint64_t returned_null_count_size2 = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount2", &null_count2, &returned_null_count_size2) - .ok()); + query.set_data_buffer("NullCount", null_count); + query.set_data_buffer("NullCount2", null_count2); // Here we run a few iterations until the query completes and update the // buffers as we go. @@ -2180,13 +2065,12 @@ TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["NullCount"]) == 1' once we use - // the set_data_buffer api. - CHECK(returned_null_count_size == 8); - CHECK(null_count == expected_null_count); + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["NullCount"]) == 1); + CHECK(null_count[0] == expected_null_count); - CHECK(returned_null_count_size2 == 8); - CHECK(null_count2 == expected_null_count); + CHECK(std::get<1>(result_el["NullCount2"]) == 1); + CHECK(null_count2[0] == expected_null_count); validate_data_var( query, dim1, dim2, a1_data, a1_offsets, a1_validity, false); @@ -2240,8 +2124,8 @@ TEST_CASE_METHOD( set_ranges_and_condition_if_needed(array, query, true); // Set the data buffer for the aggregator. - uint64_t null_count = 0; - uint64_t null_count2 = 0; + std::vector null_count(1); + std::vector null_count2(1); std::vector dim1(100); std::vector dim2(100); std::vector a1_offsets(100); @@ -2253,25 +2137,8 @@ TEST_CASE_METHOD( a2_data.resize(100); std::vector a2_validity(100); query.set_layout(layout); - - // TODO: Change to real CPPAPI. Use set_data_buffer and - // set_validity_buffer from the internal query directly because the - // CPPAPI doesn't know what is an aggregate and what the size of an - // aggregate should be. - uint64_t returned_null_count_size = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount", &null_count, &returned_null_count_size) - .ok()); - - uint64_t returned_null_count_size2 = 8; - CHECK(query.ptr() - ->query_ - ->set_data_buffer( - "NullCount2", &null_count2, &returned_null_count_size2) - .ok()); - + query.set_data_buffer("NullCount", null_count); + query.set_data_buffer("NullCount2", null_count2); query.set_data_buffer("d1", dim1.data(), 100); query.set_data_buffer("d2", dim2.data(), 100); query.set_data_buffer("a1", a1_data.data(), 100); @@ -2343,27 +2210,10 @@ TEMPLATE_LIST_TEST_CASE_METHOD( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - uint64_t count_data_size = sizeof(uint64_t); - uint64_t sum_data_size = - sizeof(typename tiledb::sm::sum_type_data::sum_type); - // TODO: Change to real CPPAPI. - CHECK(query.ptr() - ->query_ - ->set_data_buffer("Count", count.data(), &count_data_size) - .ok()); - CHECK(query.ptr() - ->query_->set_data_buffer("Sum", sum.data(), &sum_data_size) - .ok()); - uint64_t returned_validity_size = 1; + query.set_data_buffer("Count", count); + query.set_data_buffer("Sum", sum); if (CppAggregatesFx::nullable_) { - // TODO: Change to real CPPAPI. Use set_validity_buffer from the - // internal query directly because the CPPAPI doesn't know what is - // an aggregate and what the size of an aggregate should be. - CHECK(query.ptr() - ->query_ - ->set_validity_buffer( - "Sum", sum_validity.data(), &returned_validity_size) - .ok()); + query.set_validity_buffer("Sum", sum_validity); } // Here we run a few iterations until the query completes and update the @@ -2452,12 +2302,11 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } } - // TODO: use 'std::get<1>(result_el["Count"]) == 1' once we use the - // set_data_buffer api. + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Count"]) == 1); CHECK(count[0] == expected_count); - // TODO: use 'std::get<1>(result_el["Sum"]) == 1' once we use the - // set_data_buffer api. + CHECK(std::get<1>(result_el["Sum"]) == 1); CHECK(sum[0] == expected_sum); CppAggregatesFx::validate_data( diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc index 56674df31bdf..25bfc88774ce 100644 --- a/tiledb/api/c_api/query_field/query_field_api.cc +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -35,6 +35,7 @@ #include "tiledb/api/c_api/datatype/datatype_api_external.h" #include "tiledb/api/c_api/query/query_api_internal.h" #include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/sm/misc/constants.h" tiledb_field_origin_t FieldFromDimension::origin() { return TILEDB_DIMENSION_FIELD; @@ -53,7 +54,15 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( : query_(query->query_) , field_name_(field_name) , channel_(tiledb_query_channel_handle_t::make_handle(query)) { - if (query_->array_schema().is_attr(field_name_)) { + if (field_name_ == tiledb::sm::constants::coords) { + field_origin_ = std::make_shared(); + type_ = query_->array_schema().domain().dimension_ptr(0)->type(); + cell_val_num_ = 1; + } else if (field_name_ == tiledb::sm::constants::timestamps) { + field_origin_ = std::make_shared(); + type_ = tiledb::sm::constants::timestamp_type; + cell_val_num_ = 1; + } else if (query_->array_schema().is_attr(field_name_)) { field_origin_ = std::make_shared(); type_ = query_->array_schema().attribute(field_name_)->type(); cell_val_num_ = @@ -65,8 +74,9 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( query_->array_schema().dimension_ptr(field_name_)->cell_val_num(); } else if (query_->is_aggregate(field_name_)) { field_origin_ = std::make_shared(); - type_ = query_->get_aggregate(field_name_).value()->output_datatype(); - cell_val_num_ = 1; + auto aggregate = query_->get_aggregate(field_name_).value(); + type_ = aggregate->output_datatype(); + cell_val_num_ = aggregate->var_sized() ? tiledb::sm::constants::var_num : 1; } else { throw tiledb::api::CAPIStatusException("There is no field " + field_name_); } diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 52321606ca79..52550aaf9fec 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -317,9 +317,7 @@ TEST_CASE_METHOD( REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); CHECK(cell_val_num == 1); - // Let's at least test that this API actually gets the default channel. - // Default channel means all rows, let's count them. We will add more field - // specific tests when more functionality gets implemented for channels. + // Check field api works on aggregate field REQUIRE(tiledb_field_channel(ctx, field, &channel) == TILEDB_OK); REQUIRE( tiledb_channel_apply_aggregate( @@ -334,6 +332,27 @@ TEST_CASE_METHOD( CHECK(tiledb_query_channel_free(ctx, &channel) == TILEDB_OK); CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + // Check field api works on timestamp field + REQUIRE( + tiledb_query_get_field(ctx, query, "__timestamps", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_ATTRIBUTE_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + + // Check field api works on coords field + REQUIRE(tiledb_query_get_field(ctx, query, "__coords", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_DIMENSION_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + // Check field api works on attribute field REQUIRE(tiledb_query_get_field(ctx, query, "c", &field) == TILEDB_OK); REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index c962387197d7..69128ce599d8 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -45,6 +45,7 @@ #include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/group/group_api_external_experimental.h" #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" +#include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" #include "tiledb_dimension_label_experimental.h" diff --git a/tiledb/sm/cpp_api/query.h b/tiledb/sm/cpp_api/query.h index b065e99eb896..3cc9f075dd31 100644 --- a/tiledb/sm/cpp_api/query.h +++ b/tiledb/sm/cpp_api/query.h @@ -44,6 +44,7 @@ #include "query_condition.h" #include "subarray.h" #include "tiledb.h" +#include "tiledb_experimental.h" #include "type.h" #include "utils.h" @@ -414,15 +415,8 @@ class Query { for (const auto& b_it : buff_sizes_) { auto attr_name = b_it.first; auto size_tuple = b_it.second; - auto var = - ((attr_name != "__coords") && - ((schema_.has_attribute(attr_name) && - schema_.attribute(attr_name).cell_val_num() == TILEDB_VAR_NUM) || - (schema_.domain().has_dimension(attr_name) && - schema_.domain().dimension(attr_name).cell_val_num() == - TILEDB_VAR_NUM))); auto element_size = element_sizes_.find(attr_name)->second; - elements[attr_name] = var ? + elements[attr_name] = field_var_sized(attr_name) ? std::pair( std::get<0>(size_tuple) / sizeof(uint64_t), std::get<1>(size_tuple) / element_size) : @@ -492,14 +486,8 @@ class Query { for (const auto& b_it : buff_sizes_) { auto attr_name = b_it.first; auto size_tuple = b_it.second; - auto var = - (schema_.has_attribute(attr_name) && - schema_.attribute(attr_name).cell_val_num() == TILEDB_VAR_NUM) || - (schema_.domain().has_dimension(attr_name) && - schema_.domain().dimension(attr_name).cell_val_num() == - TILEDB_VAR_NUM); auto element_size = element_sizes_.find(attr_name)->second; - elements[attr_name] = var ? + elements[attr_name] = field_var_sized(attr_name) ? std::tuple( std::get<0>(size_tuple) / sizeof(uint64_t), std::get<1>(size_tuple) / element_size, @@ -1552,20 +1540,8 @@ class Query { **/ template Query& set_data_buffer(const std::string& name, T* buff, uint64_t nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (name != "__coords" && name != "__timestamps" && !is_attr && !is_dim) { - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - } else if (is_attr) { - impl::type_check(schema_.attribute(name).type()); - } else if (is_dim) { - impl::type_check(schema_.domain().dimension(name).type()); - } else if (name == "__coords") { - impl::type_check(schema_.domain().type()); - } + // Check we have the correct type. + impl::type_check(field_type(name)); return set_data_buffer(name, buff, nelements, sizeof(T)); } @@ -1611,23 +1587,8 @@ class Query { **/ Query& set_data_buffer( const std::string& name, void* buff, uint64_t nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (name != "__coords" && !is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - - // Compute element size (in bytes). - size_t element_size = 0; - if (name == "__coords") - element_size = tiledb_datatype_size(schema_.domain().type()); - else if (is_attr) - element_size = tiledb_datatype_size(schema_.attribute(name).type()); - else if (is_dim) - element_size = - tiledb_datatype_size(schema_.domain().dimension(name).type()); + // Get element size (in bytes). + size_t element_size = tiledb_datatype_size(field_type(name)); return set_data_buffer(name, buff, nelements, element_size); } @@ -1640,16 +1601,7 @@ class Query { **/ Query& set_data_buffer(const std::string& name, std::string& data) { // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); + impl::type_check(field_type(name)); return set_data_buffer(name, &data[0], data.size(), sizeof(char)); } @@ -1784,12 +1736,6 @@ class Query { **/ Query& set_validity_buffer( const std::string& name, std::vector& validity_bytemap) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); return set_validity_buffer( name, validity_bytemap.data(), validity_bytemap.size()); } @@ -2424,6 +2370,49 @@ class Query { /* PRIVATE METHODS */ /* ********************************* */ + /** + * Gets the field type from a field name. + * + * @param name Name of the field. + * @return Field type. + */ + tiledb_datatype_t field_type(const std::string& name) const { + // Get the field from the query. + auto ctx = ctx_.get(); + tiledb_query_field_t* field = nullptr; + ctx.handle_error(tiledb_query_get_field( + ctx.ptr().get(), query_.get(), name.c_str(), &field)); + + // Get the type from the field. + tiledb_datatype_t type; + ctx.handle_error(tiledb_field_datatype(ctx.ptr().get(), field, &type)); + return type; + } + + /** + * Returns if the field is var sized or not. + * + * @param name Name of the field. + * @return If the field is var sized. + */ + bool field_var_sized(const std::string& name) const { + auto ctx = ctx_.get(); + tiledb_query_field_t* field = nullptr; + try { + // Get the field from the query. + ctx.handle_error(tiledb_query_get_field( + ctx.ptr().get(), query_.get(), name.c_str(), &field)); + } catch (std::exception&) { + return false; + } + + // Get the cell_val_num from the field. + uint32_t cell_val_num; + ctx.handle_error( + tiledb_field_cell_val_num(ctx.ptr().get(), field, &cell_val_num)); + return cell_val_num == TILEDB_VAR_NUM; + } + /** * Sets a buffer for a fixed-sized attribute. * diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.h b/tiledb/sm/query/readers/aggregators/mean_aggregator.h index e61fbe62c3d6..9fbd6fb58f9e 100644 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/mean_aggregator.h @@ -112,7 +112,7 @@ class MeanAggregator : public OutputBufferValidator, public IAggregator { /** Returns the TileDB datatype of the output field for the aggregate. */ Datatype output_datatype() override { - return sum_type_data::tiledb_datatype; + return Datatype::FLOAT64; } private: diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index b99838917176..0b8f3b94b0f7 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -1167,32 +1167,32 @@ TEST_CASE( "[mean-aggregator][output_datatype]") { CHECK( MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} - .output_datatype() == Datatype::INT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{ FieldInfo("a1", false, false, 1, Datatype::UINT64)} - .output_datatype() == Datatype::UINT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} - .output_datatype() == Datatype::INT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{ FieldInfo("a1", false, false, 1, Datatype::UINT64)} - .output_datatype() == Datatype::UINT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} - .output_datatype() == Datatype::INT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{ FieldInfo("a1", false, false, 1, Datatype::UINT64)} - .output_datatype() == Datatype::UINT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::INT64)} - .output_datatype() == Datatype::INT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{ FieldInfo("a1", false, false, 1, Datatype::UINT64)} - .output_datatype() == Datatype::UINT64); + .output_datatype() == Datatype::FLOAT64); CHECK( MeanAggregator{FieldInfo("a1", false, false, 1, Datatype::FLOAT64)} .output_datatype() == Datatype::FLOAT64); From 2fb5a952cf38289f81d1922e4f37c7fa81b518bf Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 19 Oct 2023 14:48:31 +0300 Subject: [PATCH 012/456] Stop building gRPC when GCS is enabled. (#4435) Backport of microsoft/vcpkg#34569 to version 2.15.1 of the `google-cloud-cpp` port. The first commit copies the port as-is to the `ports` directory, and the second applies the fix. Dramatically drops build times and binary size. Before this PR, the size of the `build/vcpkg_installed/x64-windows` directory with features `gcs;tests;webp` enabled was __2.5GB__. With this PR it is __708MB__. --- TYPE: BUILD DESC: Stop building gRPC when GCS is enabled. --- ports/google-cloud-cpp/portfile.cmake | 85 + .../google-cloud-cpp/support_absl_cxx17.patch | 19 + ports/google-cloud-cpp/vcpkg.json | 1447 +++++++++++++++++ scripts/install-run-gcs-emu.ps1 | 2 +- 4 files changed, 1552 insertions(+), 1 deletion(-) create mode 100644 ports/google-cloud-cpp/portfile.cmake create mode 100644 ports/google-cloud-cpp/support_absl_cxx17.patch create mode 100644 ports/google-cloud-cpp/vcpkg.json diff --git a/ports/google-cloud-cpp/portfile.cmake b/ports/google-cloud-cpp/portfile.cmake new file mode 100644 index 000000000000..2cae4df0a1d3 --- /dev/null +++ b/ports/google-cloud-cpp/portfile.cmake @@ -0,0 +1,85 @@ +vcpkg_check_linkage(ONLY_STATIC_LIBRARY) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO googleapis/google-cloud-cpp + REF "v${VERSION}" + SHA512 225202a8e799f630f0b07c392bf305c28e21b99ef8dc5a670238a6d08e0e2816cd8ca1c43d7b252bcf5d289f875e64c16413085f63663265169807fd59977e43 + HEAD_REF main + PATCHES + support_absl_cxx17.patch +) + +if ("grpc-common" IN_LIST FEATURES) + vcpkg_add_to_path(PREPEND "${CURRENT_HOST_INSTALLED_DIR}/tools/grpc") +endif () + +set(GOOGLE_CLOUD_CPP_ENABLE "${FEATURES}") +list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "core") +# This feature does not exist, but allows us to simplify the vcpkg.json +# file. +list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "grpc-common") +list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "rest-common") +list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "googleapis") +# google-cloud-cpp uses dialogflow_cx and dialogflow_es. Underscores +# are invalid in `vcpkg` features, we use dashes (`-`) as a separator +# for the `vcpkg` feature name, and convert it here to something that +# `google-cloud-cpp` would like. +if ("dialogflow-cx" IN_LIST FEATURES) + list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-cx") + list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_cx") +endif () +if ("dialogflow-es" IN_LIST FEATURES) + list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-es") + list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_es") +endif () + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + "-DGOOGLE_CLOUD_CPP_ENABLE=${GOOGLE_CLOUD_CPP_ENABLE}" + -DGOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_WERROR=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_CCACHE=OFF + -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF + -DBUILD_TESTING=OFF + # This is needed by the `experimental-storage-grpc` feature until vcpkg + # gets Protobuf >= 4.23.0. It has no effect for other features, so + # it is simpler to just always turn it on. + -DGOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND=ON +) + +vcpkg_cmake_install() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +foreach(feature IN LISTS FEATURES) + set(config_path "lib/cmake/google_cloud_cpp_${feature}") + # Most features get their own package in `google-cloud-cpp`. + # The exceptions are captured by this `if()` command, basically + # things like `core` and `experimental-storage-grpc` are skipped. + if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") + continue() + endif() + vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${feature}" + CONFIG_PATH "${config_path}" + DO_NOT_DELETE_PARENT_CONFIG_PATH) +endforeach() +# These packages are automatically installed depending on what features are +# enabled. +foreach(suffix common googleapis grpc_utils rest_internal opentelemetry dialogflow_cx dialogflow_es) + set(config_path "lib/cmake/google_cloud_cpp_${suffix}") + if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") + continue() + endif() + vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${suffix}" + CONFIG_PATH "${config_path}" + DO_NOT_DELETE_PARENT_CONFIG_PATH) +endforeach() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib/cmake" + "${CURRENT_PACKAGES_DIR}/debug/lib/cmake" + "${CURRENT_PACKAGES_DIR}/debug/share") +file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) + +vcpkg_copy_pdbs() diff --git a/ports/google-cloud-cpp/support_absl_cxx17.patch b/ports/google-cloud-cpp/support_absl_cxx17.patch new file mode 100644 index 000000000000..582b88a52d40 --- /dev/null +++ b/ports/google-cloud-cpp/support_absl_cxx17.patch @@ -0,0 +1,19 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3fb0564..b4a251b 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -29,6 +29,14 @@ if (NOT "${PROJECT_VERSION_PRE_RELEASE}" STREQUAL "") + set(PROJECT_VERSION "${PROJECT_VERSION}-${PROJECT_VERSION_PRE_RELEASE}") + endif () + ++find_package(absl CONFIG REQUIRED) ++ ++# Use CMAKE_CXX_STANDARD=17 if ABSL_USE_CXX17 is set ++if (ABSL_USE_CXX17) ++ message(STATUS "Found absl uses CXX17, enable CXX17 feature.") ++ set(CMAKE_CXX_STANDARD 17) ++endif () ++ + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3) + message( diff --git a/ports/google-cloud-cpp/vcpkg.json b/ports/google-cloud-cpp/vcpkg.json new file mode 100644 index 000000000000..09da2aa504ad --- /dev/null +++ b/ports/google-cloud-cpp/vcpkg.json @@ -0,0 +1,1447 @@ +{ + "name": "google-cloud-cpp", + "version": "2.15.1", + "description": "C++ Client Libraries for Google Cloud Platform APIs.", + "homepage": "https://github.com/googleapis/google-cloud-cpp", + "license": "Apache-2.0", + "supports": "!uwp", + "dependencies": [ + "abseil", + "openssl", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "bigquery", + "bigtable", + "iam", + "pubsub", + "spanner", + "storage" + ], + "features": { + "accessapproval": { + "description": "Access Approval API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "accesscontextmanager": { + "description": "Access Context Manager API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "advisorynotifications": { + "description": "Advisory Notifications API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "alloydb": { + "description": "Alloy DB API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "apigateway": { + "description": "API Gateway API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "apigeeconnect": { + "description": "Apigee Connect API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "apikeys": { + "description": "API Keys API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "appengine": { + "description": "App Engine Admin API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "artifactregistry": { + "description": "Artifact Registry API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "asset": { + "description": "Cloud Asset API C++ Client Library", + "supports": "!windows", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "accesscontextmanager", + "grpc-common", + "osconfig" + ] + } + ] + }, + "assuredworkloads": { + "description": "Assured Workloads API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "automl": { + "description": "Cloud AutoML API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "baremetalsolution": { + "description": "Bare Metal Solution API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "batch": { + "description": "Batch API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "beyondcorp": { + "description": "BeyondCorp API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "bigquery": { + "description": "The Google Cloud BigQuery C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "bigtable": { + "description": "The Google Cloud Bigtable C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "billing": { + "description": "Cloud Billing Budget API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "binaryauthorization": { + "description": "Binary Authorization API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grafeas", + "grpc-common" + ] + } + ] + }, + "certificatemanager": { + "description": "Certificate Manager API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "channel": { + "description": "Cloud Channel API C++ Client Library", + "supports": "!windows", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "cloudbuild": { + "description": "Cloud Build API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "commerce": { + "description": "Cloud Commerce C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "composer": { + "description": "Cloud Composer C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "confidentialcomputing": { + "description": "Confidential Computing API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "connectors": { + "description": "Connectors API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "contactcenterinsights": { + "description": "Contact Center AI Insights API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "container": { + "description": "Kubernetes Engine API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "containeranalysis": { + "description": "Container Analysis API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grafeas", + "grpc-common" + ] + } + ] + }, + "datacatalog": { + "description": "Google Cloud Data Catalog API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "datafusion": { + "description": "Cloud Data Fusion API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "datamigration": { + "description": "Database Migration API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "dataplex": { + "description": "Cloud Dataplex API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "dataproc": { + "description": "Cloud Dataproc API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "datastore": { + "description": "Cloud Datastore API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "datastream": { + "description": "Datastream API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "debugger": { + "description": "Stackdriver Debugger API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "deploy": { + "description": "Google Cloud Deploy API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "dialogflow-cx": { + "description": "Cloud Dialogflow CX API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "dialogflow-es": { + "description": "Cloud Dialogflow ES API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "discoveryengine": { + "description": "Discovery Engine API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "dlp": { + "description": "Cloud Data Loss Prevention (DLP) API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "documentai": { + "description": "Cloud Document AI API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "edgecontainer": { + "description": "Distributed Cloud Edge Container API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "eventarc": { + "description": "Eventarc API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "experimental-opentelemetry": { + "description": "OpenTelemetry C++ GCP Exporter Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "rest-common", + "trace" + ] + }, + "opentelemetry-cpp" + ] + }, + "experimental-storage-grpc": { + "description": "The GCS+gRPC plugin", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common", + "storage" + ] + } + ] + }, + "filestore": { + "description": "Cloud Filestore API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "functions": { + "description": "Cloud Functions API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "gameservices": { + "description": "Game Services API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "gkehub": { + "description": "GKE Hub C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "gkemulticloud": { + "description": "Anthos Multi-Cloud C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "grafeas": { + "description": "Protocol buffers implementing the 'Grafeas API' (metadata about software artifacts)", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "grpc-common": { + "description": "Dependencies used by all gRPC-based libraries", + "dependencies": [ + "grpc", + { + "name": "grpc", + "host": true + }, + "protobuf", + { + "name": "protobuf", + "host": true + } + ] + }, + "iam": { + "description": "The Google Cloud IAM C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "iap": { + "description": "Cloud Identity-Aware Proxy API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "ids": { + "description": "Cloud IDS API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "iot": { + "description": "Cloud IoT API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "kms": { + "description": "Cloud Key Management Service (KMS) API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "language": { + "description": "Cloud Natural Language API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "logging": { + "description": "Google Cloud Logging C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "managedidentities": { + "description": "Managed Service for Microsoft Active Directory API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "memcache": { + "description": "Cloud Memorystore for Memcached API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "metastore": { + "description": "Dataproc Metastore API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "migrationcenter": { + "description": "Migration Center API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "monitoring": { + "description": "Cloud Monitoring API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "networkconnectivity": { + "description": "Network Connectivity API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "networkmanagement": { + "description": "Network Management API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "networksecurity": { + "description": "Secure Web Proxy API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "notebooks": { + "description": "Notebooks API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "optimization": { + "description": "Cloud Optimization API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "orgpolicy": { + "description": "Organization Policy API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "osconfig": { + "description": "Cloud OS Config API C++ Client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "oslogin": { + "description": "Cloud OS Login API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "policysimulator": { + "description": "Policy Simulator API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "policytroubleshooter": { + "description": "Policy Troubleshooter API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "privateca": { + "description": "Certificate Authority API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "profiler": { + "description": "Cloud Profiler API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "pubsub": { + "description": "The Google Cloud Bigtable C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "rapidmigrationassessment": { + "description": "Rapid Migration Assessment C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "recommender": { + "description": "Recommender C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "redis": { + "description": "Google Cloud Memorystore for Redis API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "resourcemanager": { + "description": "Cloud Resource Manager API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "resourcesettings": { + "description": "Resource Settings API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "rest-common": { + "description": "Dependencies used by all REST-based libraries", + "dependencies": [ + { + "name": "curl", + "features": [ + "ssl" + ] + }, + "nlohmann-json" + ] + }, + "retail": { + "description": "Retail API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "run": { + "description": "Cloud Run Admin API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "scheduler": { + "description": "Cloud Scheduler API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "secretmanager": { + "description": "The Google Cloud Secret Manager C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "securitycenter": { + "description": "Security Command Center API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "servicecontrol": { + "description": "Service Control API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "servicedirectory": { + "description": "Service Directory API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "servicemanagement": { + "description": "Service Management API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "serviceusage": { + "description": "Service Usage API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "shell": { + "description": "Cloud Shell API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "spanner": { + "description": "The Google Cloud Spanner C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "speech": { + "description": "The Google Cloud Speech-to-Text C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "storage": { + "description": "The Google Cloud Storage C++ client library", + "dependencies": [ + "crc32c", + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "rest-common" + ] + } + ] + }, + "storageinsights": { + "description": "Storage Insights API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "storagetransfer": { + "description": "Storage Transfer API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "support": { + "description": "Cloud Support API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "talent": { + "description": "Cloud Talent Solution API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "tasks": { + "description": "The Google Cloud Tasks C++ client library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "texttospeech": { + "description": "Cloud Text-to-Speech API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "tpu": { + "description": "Cloud TPU API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "trace": { + "description": "Stackdriver Trace API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "translate": { + "description": "Cloud Translation API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "video": { + "description": "Video Services C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "videointelligence": { + "description": "Cloud Video Intelligence API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "vision": { + "description": "Cloud Vision API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "vmmigration": { + "description": "VM Migration API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "vmwareengine": { + "description": "VMware Engine API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "vpcaccess": { + "description": "Serverless VPC Access API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "webrisk": { + "description": "Web Risk API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "websecurityscanner": { + "description": "Web Security Scanner API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "workflows": { + "description": "Workflow Executions API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + }, + "workstations": { + "description": "Workstations API C++ Client Library", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": [ + "grpc-common" + ] + } + ] + } + } +} diff --git a/scripts/install-run-gcs-emu.ps1 b/scripts/install-run-gcs-emu.ps1 index 95ce71674ef2..59e9a85e7dde 100644 --- a/scripts/install-run-gcs-emu.ps1 +++ b/scripts/install-run-gcs-emu.ps1 @@ -23,7 +23,7 @@ Param( $ErrorActionPreference = "Stop" -$version = "v0.36.0" +$version = "v0.39.0" $testbenchPath = "$env:TEMP\storage-testbench-$version" $venvPath = "$env:TEMP\storage-testbench-venv" From c57974e031cee5d6c0783550fe6335cf360085c2 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 19 Oct 2023 16:04:44 +0300 Subject: [PATCH 013/456] Update `tiledb_group_remove_member` to remove members by name first. (#4391) [SC-34787](https://app.shortcut.com/tiledb-inc/story/34787/convert-apis-to-use-name-instead-of-uri-for-group-item-reference) This PR changes the behavior of the `tiledb_group_remove_member` function to make its semantics around named members more predictable. If you want to remove a named member, call the function and pass its name. If the member does not have a name, pass its URI. There is a behavioral breaking change, where named members cannot be removed by their URI anymore. This is the behavior that was originally specified. The tests were updated to account for that. I found the existing tests pretty comprehensive and did not add another test case. For example removing unnamed group members [is already tested](https://github.com/TileDB-Inc/TileDB/blob/553979ce4b4f1a6d90ec89f992607363f2387d3a/test/src/unit-capi-group.cc#L645). --- TYPE: C_API DESC: Behavior breaking change: `tiledb_group_remove_member` cannot remove named members by URI. --------- Co-authored-by: Julia Dark <24235303+jp-dark@users.noreply.github.com> Co-authored-by: Luc Rancourt --- test/src/unit-capi-group.cc | 84 ++++++++++++ test/src/unit-cppapi-group.cc | 6 +- tiledb/api/c_api/group/group_api.cc | 10 +- .../group/group_api_external_experimental.h | 13 +- tiledb/sm/cpp_api/group_experimental.h | 9 +- tiledb/sm/group/group.cc | 8 +- tiledb/sm/group/group.h | 14 +- tiledb/sm/group/group_details.cc | 126 +++++++++--------- tiledb/sm/group/group_details.h | 35 ++--- tiledb/sm/group/group_details_v1.cc | 21 +-- tiledb/sm/group/group_details_v2.cc | 9 +- tiledb/sm/group/group_member.h | 10 ++ 12 files changed, 208 insertions(+), 137 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index ff615540bc35..008cb3cb4a9b 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -893,12 +893,18 @@ TEST_CASE_METHOD( rc = tiledb_group_open(ctx_, group2, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + // The member has a name; removing it by URI should fail. rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); + REQUIRE(rc != TILEDB_OK); + rc = tiledb_group_remove_member(ctx_, group1, "four"); REQUIRE(rc == TILEDB_OK); // Group is the latest element group1_expected.resize(group1_expected.size() - 1); + // The member has a name; removing it by URI should fail. rc = tiledb_group_remove_member(ctx_, group2, array3_uri.c_str()); + REQUIRE(rc != TILEDB_OK); + rc = tiledb_group_remove_member(ctx_, group2, "three"); REQUIRE(rc == TILEDB_OK); // There should be nothing left in group2 group2_expected.clear(); @@ -936,6 +942,78 @@ TEST_CASE_METHOD( remove_temp_dir(temp_dir); } +TEST_CASE_METHOD( + GroupFx, + "C API: Group, write duplicate members", + "[capi][group][write][duplicate]") { + // Create and open group in write mode + // TODO: refactor for each supported FS. + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + + const tiledb::sm::URI group1_uri(temp_dir + "group1"); + const tiledb::sm::URI group2_uri(temp_dir + "group1/group2"); + const tiledb::sm::URI group3_uri(temp_dir + "group1/group3"); + + REQUIRE(tiledb_group_create(ctx_, group1_uri.c_str()) == TILEDB_OK); + REQUIRE(tiledb_group_create(ctx_, group2_uri.c_str()) == TILEDB_OK); + REQUIRE(tiledb_group_create(ctx_, group3_uri.c_str()) == TILEDB_OK); + + tiledb_group_t* group1; + int rc = tiledb_group_alloc(ctx_, group1_uri.c_str(), &group1); + REQUIRE(rc == TILEDB_OK); + set_group_timestamp(group1, 1); + rc = tiledb_group_open(ctx_, group1, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); + + SECTION("Name-name collision") { + rc = + tiledb_group_add_member(ctx_, group1, group2_uri.c_str(), false, "one"); + REQUIRE(rc == TILEDB_OK); + rc = + tiledb_group_add_member(ctx_, group1, group3_uri.c_str(), false, "one"); + REQUIRE(rc == TILEDB_ERR); + rc = tiledb_group_remove_member(ctx_, group1, "one"); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("Name-URI collision") { + rc = tiledb_group_add_member( + ctx_, group1, group2_uri.c_str(), false, "group2"); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_group_add_member(ctx_, group1, "group2", true, nullptr); + REQUIRE(rc == TILEDB_ERR); + rc = tiledb_group_remove_member(ctx_, group1, "group2"); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("URI-name collision") { + rc = tiledb_group_add_member(ctx_, group1, "group2", true, nullptr); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_group_add_member( + ctx_, group1, group3_uri.c_str(), false, "group2"); + REQUIRE(rc == TILEDB_ERR); + rc = tiledb_group_remove_member(ctx_, group1, "group2"); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("URI-URI collision") { + rc = tiledb_group_add_member( + ctx_, group1, group2_uri.c_str(), false, nullptr); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_group_add_member( + ctx_, group1, group2_uri.c_str(), false, nullptr); + REQUIRE(rc == TILEDB_ERR); + rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); + REQUIRE(rc == TILEDB_ERR); + } + + // Close group + rc = tiledb_group_close(ctx_, group1); + tiledb_group_free(&group1); + remove_temp_dir(temp_dir); +} + TEST_CASE_METHOD( GroupFx, "C API: Group, consolidation", @@ -1321,12 +1399,18 @@ TEST_CASE_METHOD( rc = tiledb_group_open(ctx_, group2, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + // The member has a name; removing it by URI should fail. rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); + REQUIRE(rc != TILEDB_OK); + rc = tiledb_group_remove_member(ctx_, group1, "eight"); REQUIRE(rc == TILEDB_OK); // Group is the latest element group1_expected.resize(group1_expected.size() - 1); + // The member has a name; removing it by URI should fail. rc = tiledb_group_remove_member(ctx_, group2, array3_relative_uri.c_str()); + REQUIRE(rc != TILEDB_OK); + rc = tiledb_group_remove_member(ctx_, group2, "seven"); REQUIRE(rc == TILEDB_OK); // There should be nothing left in group2 group2_expected.clear(); diff --git a/test/src/unit-cppapi-group.cc b/test/src/unit-cppapi-group.cc index cc92cc64cea5..d0169de0b856 100644 --- a/test/src/unit-cppapi-group.cc +++ b/test/src/unit-cppapi-group.cc @@ -788,11 +788,13 @@ TEST_CASE_METHOD( set_group_timestamp(&group2, 2); group2.open(TILEDB_WRITE); - group1.remove_member(group2_uri.to_string()); + REQUIRE_THROWS(group1.remove_member(group2_uri.to_string())); + group1.remove_member("three"); // Group is the latest element group1_expected.resize(group1_expected.size() - 1); - group2.remove_member(array3_relative_uri); + REQUIRE_THROWS(group2.remove_member(array3_relative_uri)); + group2.remove_member("four"); // There should be nothing left in group2 group2_expected.clear(); diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index a8333252b2c4..e5a195ad3eb9 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -276,11 +276,11 @@ capi_return_t tiledb_group_add_member( } capi_return_t tiledb_group_remove_member( - tiledb_group_handle_t* group, const char* group_uri) { + tiledb_group_handle_t* group, const char* name) { ensure_group_is_valid(group); - ensure_group_uri_argument_is_valid(group_uri); + ensure_name_argument_is_valid(name); - throw_if_not_ok(group->group().mark_member_for_removal(group_uri)); + throw_if_not_ok(group->group().mark_member_for_removal(name)); return TILEDB_OK; } @@ -671,9 +671,9 @@ capi_return_t tiledb_group_add_member( } capi_return_t tiledb_group_remove_member( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* uri) noexcept { + tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name) noexcept { return api_entry_context( - ctx, group, uri); + ctx, group, name); } capi_return_t tiledb_group_get_member_count( diff --git a/tiledb/api/c_api/group/group_api_external_experimental.h b/tiledb/api/c_api/group/group_api_external_experimental.h index d75effd73bc6..a661bd15a722 100644 --- a/tiledb/api/c_api/group/group_api_external_experimental.h +++ b/tiledb/api/c_api/group/group_api_external_experimental.h @@ -343,8 +343,8 @@ TILEDB_EXPORT capi_return_t tiledb_group_has_metadata_key( * @param group An group opened in WRITE mode. * @param uri URI of member to add * @param relative is the URI relative to the group - * @param name optional name group member can be given to be looked up by. Set - * to NULL if wishing to remain unset. + * @param name optional name group member can be given to be looked up by. + * Can be set to NULL. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_group_add_member( @@ -366,12 +366,13 @@ TILEDB_EXPORT capi_return_t tiledb_group_add_member( * * @param ctx The TileDB context. * @param group An group opened in WRITE mode. - * @param uri URI of member to remove. Passing a name is also supported if the - * group member was assigned a name. + * @param name Name of member to remove. If the member has no name, this + * parameter should be set to the URI of the member. In that case, only the + * unnamed member with the given URI will be removed. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_group_remove_member( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* uri) TILEDB_NOEXCEPT; + tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name) TILEDB_NOEXCEPT; /** * Get the count of members in a group @@ -476,8 +477,6 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_name( char** uri, tiledb_object_t* type) TILEDB_NOEXCEPT; -/* (clang format was butchering the tiledb_group_add_member() calls) */ -/* clang-format off */ /** * Get a member of a group by name and relative characteristic of that name * diff --git a/tiledb/sm/cpp_api/group_experimental.h b/tiledb/sm/cpp_api/group_experimental.h index 29bbbbdce338..bdd8a07ad6ad 100644 --- a/tiledb/sm/cpp_api/group_experimental.h +++ b/tiledb/sm/cpp_api/group_experimental.h @@ -401,14 +401,15 @@ class Group { /** * Remove a member from a group * - * @param uri of member to remove. Passing a name is also supported if the - * group member was assigned a name. + * @param name Name of member to remove. If the member has no name, + * this parameter should be set to the URI of the member. In that case, only + * the unnamed member with the given URI will be removed. */ - void remove_member(const std::string& uri) { + void remove_member(const std::string& name) { auto& ctx = ctx_.get(); tiledb_ctx_t* c_ctx = ctx.ptr().get(); ctx.handle_error( - tiledb_group_remove_member(c_ctx, group_.get(), uri.c_str())); + tiledb_group_remove_member(c_ctx, group_.get(), name.c_str())); } uint64_t member_count() const { diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index fddea6ca412a..425d3f66187c 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -561,11 +561,7 @@ Status Group::mark_member_for_addition( group_member_uri, relative, name, storage_manager_); } -Status Group::mark_member_for_removal(const URI& uri) { - return mark_member_for_removal(uri.to_string()); -} - -Status Group::mark_member_for_removal(const std::string& uri) { +Status Group::mark_member_for_removal(const std::string& name) { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { @@ -581,7 +577,7 @@ Status Group::mark_member_for_removal(const std::string& uri) { "mode"); } - return group_details_->mark_member_for_removal(uri); + return group_details_->mark_member_for_removal(name); } const std::vector>& Group::members_to_modify() const { diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index bfd9e2dea977..3058d4934e95 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -257,18 +257,12 @@ class Group { /** * Remove a member from a group, this will be flushed to disk on close * - * @param uri of member to remove + * @param name Name of member to remove. If the member has no name, + * this parameter should be set to the URI of the member. In that case, only + * the unnamed member with the given URI will be removed. * @return Status */ - Status mark_member_for_removal(const URI& uri); - - /** - * Remove a member from a group, this will be flushed to disk on close - * - * @param uri of member to remove - * @return Status - */ - Status mark_member_for_removal(const std::string& uri); + Status mark_member_for_removal(const std::string& name); /** * Get the vector of members to modify, used in serialization only diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index d6497de1bfe3..65f9e091d5ec 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -62,39 +62,27 @@ Status GroupDetails::clear() { members_by_name_.clear(); members_vec_.clear(); members_to_modify_.clear(); + member_keys_to_add_.clear(); + member_keys_to_delete_.clear(); return Status::Ok(); } void GroupDetails::add_member(const shared_ptr group_member) { std::lock_guard lck(mtx_); - const std::string& uri = group_member->uri().to_string(); - members_.emplace(uri, group_member); - members_vec_.emplace_back(group_member); - if (group_member->name().has_value()) { - members_by_name_.emplace(group_member->name().value(), group_member); - } + auto key = group_member->key(); + members_[key] = group_member; + // Invalidate the lookup tables. + members_by_name_.clear(); + members_vec_.clear(); } void GroupDetails::delete_member(const shared_ptr group_member) { std::lock_guard lck(mtx_); - const std::string& uri = group_member->uri().to_string(); - auto it = members_.find(uri); - if (it != members_.end()) { - for (size_t i = 0; i < members_vec_.size(); i++) { - if (members_vec_[i] == it->second) { - members_vec_.erase(members_vec_.begin() + i); - break; - } - } - auto name = it->second->name(); - members_.erase(it); - if (group_member->name().has_value()) { - members_by_name_.erase(group_member->name().value()); - } else if (name.has_value()) { - members_by_name_.erase(name.value()); - } - } + members_.erase(group_member->key()); + // Invalidate the lookup tables. + members_by_name_.clear(); + members_vec_.clear(); } Status GroupDetails::mark_member_for_addition( @@ -103,9 +91,6 @@ Status GroupDetails::mark_member_for_addition( std::optional& name, StorageManager* storage_manager) { std::lock_guard lck(mtx_); - // TODO: Safety checks for not double adding, making sure its remove + add, - // etc - URI absolute_group_member_uri = group_member_uri; if (relative) { absolute_group_member_uri = @@ -122,20 +107,25 @@ Status GroupDetails::mark_member_for_addition( auto group_member = tdb::make_shared( HERE(), group_member_uri, type, relative, name, false); + if (!member_keys_to_add_.insert(group_member->key()).second) { + return Status_GroupError( + "Cannot add group member " + group_member->key() + + ", a member with the same name or URI has already been added."); + } + members_to_modify_.emplace_back(group_member); return Status::Ok(); } -Status GroupDetails::mark_member_for_removal(const URI& uri) { - return mark_member_for_removal(uri.to_string()); -} - -Status GroupDetails::mark_member_for_removal(const std::string& uri) { +Status GroupDetails::mark_member_for_removal(const std::string& name) { std::lock_guard lck(mtx_); - auto it = members_.find(uri); - auto it_name = members_by_name_.find(uri); + auto it = members_.find(name); + if (it == members_.end()) { + // try URI to see if we need to convert the local file to file:// + it = members_.find(URI(name).to_string()); + } if (it != members_.end()) { auto member_to_delete = make_shared( it->second->uri(), @@ -143,39 +133,25 @@ Status GroupDetails::mark_member_for_removal(const std::string& uri) { it->second->relative(), it->second->name(), true); - members_to_modify_.emplace_back(member_to_delete); - return Status::Ok(); - } else if (it_name != members_by_name_.end()) { - // If the user passed the name, convert it to the URI for removal - auto member_to_delete = make_shared( - it_name->second->uri(), - it_name->second->type(), - it_name->second->relative(), - it_name->second->name(), - true); - members_to_modify_.emplace_back(member_to_delete); - return Status::Ok(); - } else { - // try URI to see if we need to convert the local file to file:// - URI uri_uri(uri); - it = members_.find(uri_uri.to_string()); - if (it != members_.end()) { - auto member_to_delete = make_shared( - it->second->uri(), - it->second->type(), - it->second->relative(), - it->second->name(), - true); - members_to_modify_.emplace_back(member_to_delete); - return Status::Ok(); - } else { + + if (member_keys_to_add_.count(member_to_delete->key()) != 0) { return Status_GroupError( - "Cannot remove group member " + uri + - ", member does not exist in group."); + "Cannot remove group member " + member_to_delete->key() + + ", a member with the same name or URI has already been added."); } - } - return Status::Ok(); + if (!member_keys_to_delete_.insert(member_to_delete->key()).second) { + return Status_GroupError( + "Cannot remove group member " + member_to_delete->key() + + ", a member with the same name or URI has already been removed."); + } + + members_to_modify_.emplace_back(member_to_delete); + return Status::Ok(); + } + return Status_GroupError( + "Cannot remove group member " + name + + ", member does not exist in group."); } const std::vector>& GroupDetails::members_to_modify() @@ -233,6 +209,14 @@ tuple> GroupDetails::member_by_index(uint64_t index) { std::lock_guard lck(mtx_); + if (members_vec_.size() != members_.size()) { + members_vec_.clear(); + members_vec_.reserve(members_.size()); + for (auto& [key, member] : members_) { + members_vec_.emplace_back(member); + } + } + if (index >= members_vec_.size()) { throw Status_GroupError( "index " + std::to_string(index) + " is larger than member count " + @@ -253,6 +237,22 @@ tuple, bool> GroupDetails::member_by_name(const std::string& name) { std::lock_guard lck(mtx_); + if (members_by_name_.size() != members_.size()) { + members_by_name_.clear(); + members_by_name_.reserve(members_.size()); + for (auto& [key, member] : members_) { + if (member->name().has_value()) { + bool added = + members_by_name_.insert_or_assign(member->name().value(), member) + .second; + // add_member makes sure that the name is unique, so the call to + // insert_or_assign should always add a member. + assert(added); + std::ignore = added; + } + } + } + auto it = members_by_name_.find(name); if (it == members_by_name_.end()) { throw Status_GroupError(name + " does not exist in group"); diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index 33ee4c7720a3..ff5c3edaaf1c 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -80,18 +80,12 @@ class GroupDetails { /** * Remove a member from a group, this will be flushed to disk on close * - * @param uri of member to remove + * @param name Name of member to remove. If the member has no name, + * this parameter should be set to the URI of the member. In that case, only + * the unnamed member with the given URI will be removed. * @return Status */ - Status mark_member_for_removal(const URI& uri); - - /** - * Remove a member from a group, this will be flushed to disk on close - * - * @param uri of member to remove - * @return Status - */ - Status mark_member_for_removal(const std::string& uri); + Status mark_member_for_removal(const std::string& name); /** * Get the vector of members to modify, used in serialization only @@ -169,7 +163,7 @@ class GroupDetails { uint64_t member_count() const; /** - * Get a member by index + * Get a member by index. * * @param index of member * @return Tuple of URI string, ObjectType, optional GroupMember name @@ -178,7 +172,7 @@ class GroupDetails { uint64_t index); /** - * Get a member by name + * Get a member by name. * * @param name of member * @return Tuple of URI string, ObjectType, optional GroupMember name, @@ -196,7 +190,7 @@ class GroupDetails { /** * Apply any pending member additions or removals * - * mutates members_ and clears members_to_add_ and members_to_remove_ + * mutates members_ and clears members_to_modify_ * * @return Status */ @@ -210,26 +204,33 @@ class GroupDetails { /** The group URI. */ URI group_uri_; - /** The mapping of all members of this group. */ + /** The mapping of all members of this group. This is the canonical store of + * the group's members. The key is the member's key(). */ std::unordered_map> members_; /** Vector for index based lookup. */ std::vector> members_vec_; - /** Unordered map of members by their name, if the member doesn't have a name, + /** Unordered map for name based lookup. If the member doesn't have a name, * it will not be in the map. */ std::unordered_map> members_by_name_; /** Mapping of members slated for adding. */ std::vector> members_to_modify_; + /** Set of member keys that have been marked for addition. */ + std::unordered_set member_keys_to_add_; + + /** Set of member keys that have been marked for removal. */ + std::unordered_set member_keys_to_delete_; + /** Mutex for thread safety. */ mutable std::mutex mtx_; - /* Format version. */ + /** Format version. */ const uint32_t version_; - /* Were changes applied and is a write is required */ + /** Were changes applied and is a write is required */ bool changes_applied_; }; } // namespace sm diff --git a/tiledb/sm/group/group_details_v1.cc b/tiledb/sm/group/group_details_v1.cc index 81379ce73c2f..78a578bab3b8 100644 --- a/tiledb/sm/group/group_details_v1.cc +++ b/tiledb/sm/group/group_details_v1.cc @@ -75,21 +75,20 @@ Status GroupDetailsV1::apply_pending_changes() { // Remove members first for (const auto& member : members_to_modify_) { - auto& uri = member->uri(); + auto key = member->key(); if (member->deleted()) { - members_.erase(uri.to_string()); + members_.erase(key); // Check to remove relative URIs - auto uri_str = uri.to_string(); - if (uri_str.find(group_uri_.add_trailing_slash().to_string()) != + if (key.find(group_uri_.add_trailing_slash().to_string()) != std::string::npos) { // Get the substring relative path - auto relative_uri = uri_str.substr( - group_uri_.add_trailing_slash().to_string().size(), uri_str.size()); + auto relative_uri = key.substr( + group_uri_.add_trailing_slash().to_string().size(), key.size()); members_.erase(relative_uri); } } else { - members_.emplace(member->uri().to_string(), member); + members_.emplace(member->key(), member); } } changes_applied_ = !members_to_modify_.empty(); @@ -97,14 +96,6 @@ Status GroupDetailsV1::apply_pending_changes() { members_vec_.clear(); members_by_name_.clear(); - members_vec_.reserve(members_.size()); - for (auto& it : members_) { - members_vec_.emplace_back(it.second); - if (it.second->name().has_value()) { - members_by_name_.emplace(it.second->name().value(), it.second); - } - } - return Status::Ok(); } diff --git a/tiledb/sm/group/group_details_v2.cc b/tiledb/sm/group/group_details_v2.cc index 221d9d7e1db9..7ea933225191 100644 --- a/tiledb/sm/group/group_details_v2.cc +++ b/tiledb/sm/group/group_details_v2.cc @@ -111,20 +111,13 @@ Status GroupDetailsV2::apply_pending_changes() { members_.clear(); members_vec_.clear(); members_by_name_.clear(); - members_vec_.reserve(members_to_modify_.size()); // First add each member to unordered map, overriding if the user adds/removes // it multiple times for (auto& it : members_to_modify_) { - members_[it->uri().to_string()] = it; + members_[it->key()] = it; } - for (auto& it : members_) { - members_vec_.emplace_back(it.second); - if (it.second->name().has_value()) { - members_by_name_.emplace(it.second->name().value(), it.second); - } - } changes_applied_ = !members_to_modify_.empty(); members_to_modify_.clear(); diff --git a/tiledb/sm/group/group_member.h b/tiledb/sm/group/group_member.h index 9bc76ce2dede..19bdb7b2077e 100644 --- a/tiledb/sm/group/group_member.h +++ b/tiledb/sm/group/group_member.h @@ -82,6 +82,16 @@ class GroupMember { /** Return the Name. */ const std::optional name() const; + /** + * Return the discriminating key of the member within a group. No + * multiple members with the same key may exist in a group. + * + * This method returns the member's name, or its URI if it does not exist. + */ + const std::string key() const { + return name_ ? *name_ : uri_.to_string(); + } + /** Return if object is relative. */ bool relative() const; From fa5ef512f956f2ac1a165517cbba0d95357892c7 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Fri, 20 Oct 2023 14:27:00 +0300 Subject: [PATCH 014/456] Aggregates CPP api support (#4438) All C APIs have a cpp api counterpart. --- TYPE: NO_HISTORY | FEATURE | CPP_API DESC: Aggregates CPP api support --------- Co-authored-by: Luc Rancourt --- test/src/test-cppapi-aggregates.cc | 192 +++++++++++------- test/support/src/vfs_helpers.h | 2 +- .../data_order/test/unit_capi_data_order.cc | 2 +- .../c_api/datatype/test/unit_capi_datatype.cc | 2 +- .../filesystem/test/unit_capi_filesystem.cc | 2 +- .../object/test/unit_capi_object_type.cc | 2 +- .../test/unit_capi_object_walk_order.cc | 2 +- tiledb/api/c_api/query/query_api_internal.h | 25 +++ .../c_api/query/test/unit_capi_query_type.cc | 2 +- .../query_aggregate/query_aggregate_api.cc | 13 +- ...uery_aggregate_api_external_experimental.h | 8 +- .../query_aggregate_api_internal.h | 20 +- .../test/unit_capi_query_aggregate.cc | 123 +++++++---- .../query_field/test/unit_capi_query_field.cc | 2 +- .../query_plan/test/unit_capi_query_plan.cc | 2 +- tiledb/sm/cpp_api/channel_operation.h | 129 ++++++++++++ tiledb/sm/cpp_api/channel_operator.h | 77 +++++++ tiledb/sm/cpp_api/deleter.h | 17 +- tiledb/sm/cpp_api/query_channel.h | 142 +++++++++++++ tiledb/sm/cpp_api/query_experimental.h | 60 ++++++ 20 files changed, 691 insertions(+), 133 deletions(-) create mode 100644 tiledb/sm/cpp_api/channel_operation.h create mode 100644 tiledb/sm/cpp_api/channel_operator.h create mode 100644 tiledb/sm/cpp_api/query_channel.h diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index a45915fae5dd..9be386a0d547 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -33,6 +33,7 @@ #include "test/support/src/helpers.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/query/readers/aggregators/count_aggregator.h" #include "tiledb/sm/query/readers/aggregators/mean_aggregator.h" #include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" @@ -137,7 +138,9 @@ struct CppAggregatesFx { template CppAggregatesFx::CppAggregatesFx() : vfs_(ctx_) { - ctx_ = Context(); + Config cfg; + cfg["sm.allow_aggregates_experimental"] = "true"; + ctx_ = Context(cfg); vfs_ = VFS(ctx_); remove_array(); @@ -1172,9 +1175,10 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. - query.ptr()->query_->add_aggregator_to_default_channel( - "Count", std::make_shared()); + // Add a count aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + default_channel.apply_aggregate("Count", CountOperation()); set_ranges_and_condition_if_needed(array, query, false); @@ -1264,12 +1268,13 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. - query.ptr()->query_->add_aggregator_to_default_channel( - "Sum", - std::make_shared>( - tiledb::sm::FieldInfo( - "a1", false, CppAggregatesFx::nullable_, 1))); + // Add a sum aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("Sum", operation); CppAggregatesFx::set_ranges_and_condition_if_needed( array, query, false); @@ -1385,7 +1390,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. + // TODO: Change to real CPPAPI. Add a mean aggregator to the query. query.ptr()->query_->add_aggregator_to_default_channel( "Mean", std::make_shared>( @@ -1476,28 +1481,28 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } typedef tuple< - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>, - std::pair>> + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair, + std::pair> MinMaxFixedTypesUnderTest; TEMPLATE_LIST_TEST_CASE( "C++ API: Aggregates basic min/max", @@ -1506,7 +1511,7 @@ TEMPLATE_LIST_TEST_CASE( typedef decltype(TestType::first) T; typedef decltype(TestType::second) AGG; CppAggregatesFx fx; - bool min = std::is_same>::value; + bool min = std::is_same::value; fx.generate_test_params(); fx.create_array_and_write_fragments(); @@ -1522,13 +1527,12 @@ TEMPLATE_LIST_TEST_CASE( fx.layout_ = layout; Query query(fx.ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. - uint64_t cell_val_num = - std::is_same::value ? fx.STRING_CELL_VAL_NUM : 1; - query.ptr()->query_->add_aggregator_to_default_channel( - "MinMax", - std::make_shared(tiledb::sm::FieldInfo( - "a1", false, fx.nullable_, cell_val_num))); + // Add a min/max aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a1"); + default_channel.apply_aggregate("MinMax", operation); fx.set_ranges_and_condition_if_needed(array, query, false); @@ -1543,7 +1547,17 @@ TEMPLATE_LIST_TEST_CASE( std::vector a1(100 * cell_size); std::vector a1_validity(100); query.set_layout(layout); - query.set_data_buffer("MinMax", min_max.data(), min_max.size()); + if constexpr (std::is_same::value) { + query.set_data_buffer( + "MinMax", + static_cast(static_cast(min_max.data())), + min_max.size()); + } else { + query.set_data_buffer( + "MinMax", + static_cast(static_cast(min_max.data())), + min_max.size() / cell_size); + } if (fx.nullable_) { query.set_validity_buffer("MinMax", min_max_validity); } @@ -1597,7 +1611,9 @@ TEMPLATE_LIST_TEST_CASE( } auto result_el = query.result_buffer_elements_nullable(); - CHECK(std::get<1>(result_el["MinMax"]) == min_max.size()); + CHECK( + std::get<1>(result_el["MinMax"]) == + (std::is_same::value ? 2 : 1)); CHECK(min_max == expected_min_max); if (request_data) { @@ -1612,17 +1628,14 @@ TEMPLATE_LIST_TEST_CASE( array.close(); } -typedef tuple< - tiledb::sm::MinAggregator, - tiledb::sm::MaxAggregator> - AggUnderTest; +typedef tuple AggUnderTest; TEMPLATE_LIST_TEST_CASE( "C++ API: Aggregates basic min/max var", "[cppapi][aggregates][basic][min-max][var]", AggUnderTest) { CppAggregatesFx fx; typedef TestType AGG; - bool min = std::is_same>::value; + bool min = std::is_same::value; fx.generate_test_params(); fx.create_var_array_and_write_fragments(); @@ -1638,15 +1651,12 @@ TEMPLATE_LIST_TEST_CASE( fx.layout_ = layout; Query query(fx.ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. - query.ptr()->query_->add_aggregator_to_default_channel( - "MinMax", - std::make_shared(tiledb::sm::FieldInfo( - "a1", - true, - fx.nullable_, - TILEDB_VAR_NUM, - tiledb::sm::Datatype::STRING_ASCII))); + // Add a min/max aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a1"); + default_channel.apply_aggregate("MinMax", operation); fx.set_ranges_and_condition_if_needed(array, query, true); @@ -1776,7 +1786,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. + // TODO: Change to real CPPAPI. Add a null count aggregator to the + // query. uint64_t cell_val_num = std::is_same::value ? CppAggregatesFx::STRING_CELL_VAL_NUM : 1; @@ -1875,7 +1886,8 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. + // TODO: Change to real CPPAPI. Add a null count aggregator to the + // query. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount", std::make_shared( @@ -1970,7 +1982,8 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. + // TODO: Change to real CPPAPI. Add a null count aggregator to the + // query. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount", std::make_shared( @@ -2107,7 +2120,8 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a count aggregator to the query. + // TODO: Change to real CPPAPI. Add a null count aggregator to the + // query. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount", std::make_shared( @@ -2184,18 +2198,14 @@ TEMPLATE_LIST_TEST_CASE_METHOD( for (tiledb_layout_t layout : CppAggregatesFx::layout_values_) { CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - - // TODO: Change to real CPPAPI. Add a count aggregator to the query. - // We add both sum and count as they are processed separately in the - // dense case. - query.ptr()->query_->add_aggregator_to_default_channel( - "Count", std::make_shared()); - - query.ptr()->query_->add_aggregator_to_default_channel( - "Sum", - std::make_shared>( - tiledb::sm::FieldInfo( - "a1", false, CppAggregatesFx::nullable_, 1))); + // Add a count aggregator to the query. We add both sum and count as + // they are processed separately in the dense case. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + default_channel.apply_aggregate("Count", CountOperation()); + ChannelOperation operation2 = + QueryExperimental::create_unary_aggregate(query, "a1"); + default_channel.apply_aggregate("Sum", operation2); CppAggregatesFx::set_ranges_and_condition_if_needed( array, query, false); @@ -2317,4 +2327,40 @@ TEMPLATE_LIST_TEST_CASE_METHOD( // Close array. array.close(); +} + +TEST_CASE_METHOD( + CppAggregatesFx, + "CPP: Aggregates - Basic", + "[aggregates][cpp_api][args]") { + dense_ = false; + nullable_ = false; + allow_dups_ = false; + create_array_and_write_fragments(); + + Array array{ctx_, ARRAY_NAME, TILEDB_READ}; + Query query(ctx_, array); + query.set_layout(TILEDB_UNORDERED); + + // This throws for and attribute that doesn't exist + CHECK_THROWS(QueryExperimental::create_unary_aggregate( + query, "nonexistent")); + + QueryChannel default_channel = QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a1"); + default_channel.apply_aggregate("Sum", operation); + + // Duplicated output fields are not allowed + CHECK_THROWS(default_channel.apply_aggregate("Sum", operation)); + + // Transition the query state + int64_t sum = 0; + query.set_data_buffer("Sum", &sum, 1); + REQUIRE(query.submit() == Query::Status::COMPLETE); + + // Check api throws if the query state is already >= initialized + CHECK_THROWS( + QueryExperimental::create_unary_aggregate(query, "a1")); + CHECK_THROWS(default_channel.apply_aggregate("Something", operation)); } \ No newline at end of file diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index 2ad0e51b6da9..f19878a895c9 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -675,10 +675,10 @@ struct TemporaryDirectoryFixture { /** Name of the temporary directory to use for this test */ std::string temp_dir_; - private: /** Virtual file system */ tiledb_vfs_t* vfs_; + private: /** Vector of supported filesystems. Used to initialize ``vfs_``. */ const std::vector> supported_filesystems_; }; diff --git a/tiledb/api/c_api/data_order/test/unit_capi_data_order.cc b/tiledb/api/c_api/data_order/test/unit_capi_data_order.cc index 4167aa36a994..76a5b017dd3b 100644 --- a/tiledb/api/c_api/data_order/test/unit_capi_data_order.cc +++ b/tiledb/api/c_api/data_order/test/unit_capi_data_order.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/data_order/test/unit_capi_data_order.cc + * @file tiledb/api/c_api/data_order/test/unit_capi_data_order.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc b/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc index 137fdf93f02d..e77c0810f24a 100644 --- a/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc +++ b/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/datatype/test/unit_capi_datatype.cc + * @file tiledb/api/c_api/datatype/test/unit_capi_datatype.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc b/tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc index 370ddce7e035..55f0069ab142 100644 --- a/tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc +++ b/tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/filesystem/test/unit_capi_filesystem.cc + * @file tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/object/test/unit_capi_object_type.cc b/tiledb/api/c_api/object/test/unit_capi_object_type.cc index 7df39d24bc77..07dc71427491 100644 --- a/tiledb/api/c_api/object/test/unit_capi_object_type.cc +++ b/tiledb/api/c_api/object/test/unit_capi_object_type.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/object/test/unit_capi_object_type.cc + * @file tiledb/api/c_api/object/test/unit_capi_object_type.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/object/test/unit_capi_object_walk_order.cc b/tiledb/api/c_api/object/test/unit_capi_object_walk_order.cc index 0e417608a20b..1397195e2c4c 100644 --- a/tiledb/api/c_api/object/test/unit_capi_object_walk_order.cc +++ b/tiledb/api/c_api/object/test/unit_capi_object_walk_order.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/object/test/unit_capi_object.cc + * @file tiledb/api/c_api/object/test/unit_capi_object.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/query/query_api_internal.h b/tiledb/api/c_api/query/query_api_internal.h index 919ef2a79e5d..dcc24f111549 100644 --- a/tiledb/api/c_api/query/query_api_internal.h +++ b/tiledb/api/c_api/query/query_api_internal.h @@ -35,6 +35,7 @@ #include "tiledb/api/c_api_support/handle/handle.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/enums/query_status.h" namespace tiledb::api { @@ -50,6 +51,30 @@ inline void ensure_query_is_valid(tiledb_query_t* query) { } } +/** + * Validation function for a TileDB query + * Throws in case the query is >= INITIALIZED with regards to its lifetime + * + * @param query A sm::Query pointer + */ +inline void ensure_query_is_not_initialized(const tiledb::sm::Query* query) { + if (query->status() != sm::QueryStatus::UNINITIALIZED) { + throw CAPIStatusException( + "argument `query` is at a too late state of its lifetime"); + } +} + +/** + * Validation function for a TileDB query + * Throws in case the query is >= INITIALIZED with regards to its lifetime + * + * @param query A C api query pointer + */ +inline void ensure_query_is_not_initialized(tiledb_query_t* query) { + ensure_query_is_valid(query); + ensure_query_is_not_initialized(query->query_); +} + } // namespace tiledb::api #endif diff --git a/tiledb/api/c_api/query/test/unit_capi_query_type.cc b/tiledb/api/c_api/query/test/unit_capi_query_type.cc index e1bfcb588005..7283c236cd05 100644 --- a/tiledb/api/c_api/query/test/unit_capi_query_type.cc +++ b/tiledb/api/c_api/query/test/unit_capi_query_type.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/filesystem/test/unit_capi_filesystem.cc + * @file tiledb/api/c_api/filesystem/test/unit_capi_filesystem.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index f22cf81f9af4..a06597b6e17a 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -34,6 +34,7 @@ #include "query_aggregate_api_external_experimental.h" #include "query_aggregate_api_internal.h" #include "tiledb/api/c_api/query/query_api_internal.h" +#include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api_support/c_api_support.h" #include "tiledb/type/apply_with_type.h" @@ -194,12 +195,21 @@ capi_return_t tiledb_create_unary_aggregate( tiledb_channel_operation_t** operation) { ensure_aggregates_enabled_via_config(ctx); ensure_query_is_valid(query); + ensure_query_is_not_initialized(query); ensure_channel_operator_is_valid(op); ensure_output_pointer_is_valid(operation); ensure_input_field_is_valid(input_field_name, op->name()); std::string field_name(input_field_name); auto& schema = query->query_->array_schema(); + // Getting the field errors if there is no input_field_name associated with + // the query + tiledb_query_field_t* field; + if (auto rv = tiledb_query_get_field(ctx, query, input_field_name, &field)) { + return rv; + } + tiledb_query_field_free(ctx, &field); + auto fi = tiledb::sm::FieldInfo( field_name, schema.var_size(field_name), @@ -220,6 +230,7 @@ capi_return_t tiledb_channel_apply_aggregate( const tiledb_channel_operation_t* operation) { ensure_aggregates_enabled_via_config(ctx); ensure_query_channel_is_valid(channel); + ensure_query_is_not_initialized(channel->query_); ensure_output_field_is_valid(output_field_name); ensure_operation_is_valid(operation); channel->add_aggregate(output_field_name, operation); @@ -378,7 +389,7 @@ SumOperation::SumOperation( std::make_shared>(fi); } else { throw std::logic_error( - "Sum aggregates can only be requested on numeric types"); + "SUM aggregates can only be requested on numeric types"); } }; apply_with_type(g, fi.type_); diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h index 53372d02a451..6325ad2aa065 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h @@ -74,7 +74,7 @@ TILEDB_EXPORT extern const tiledb_channel_operation_t* tiledb_aggregate_count; * @endcode * * @param ctx The TileDB context - * @param operator The operator handle to be retrieved + * @param op The operator handle to be retrieved * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT int32_t tiledb_channel_operator_sum_get( @@ -92,14 +92,14 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_sum_get( * @endcode * * @param ctx The TileDB context - * @param operator The operator handle to be retrieved + * @param op The operator handle to be retrieved * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT int32_t tiledb_channel_operator_min_get( tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) TILEDB_NOEXCEPT; /** - * Helper function to access the constant MAx channel operator handle + * Helper function to access the constant MAX channel operator handle * **Example:** * * @code{.c} @@ -110,7 +110,7 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_min_get( * @endcode * * @param ctx The TileDB context - * @param operator The operator handle to be retrieved + * @param op The operator handle to be retrieved * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT int32_t tiledb_channel_operator_max_get( diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h index 5bfef847f215..93e45b4da69b 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h @@ -51,12 +51,14 @@ enum QueryChannelOperator { class Operation { protected: - shared_ptr aggregator_; + shared_ptr aggregator_ = nullptr; public: - [[nodiscard]] shared_ptr aggregator() const { + [[nodiscard]] virtual shared_ptr aggregator() const { return aggregator_; } + + virtual ~Operation(){}; }; class MinOperation : public Operation { @@ -85,9 +87,13 @@ class SumOperation : public Operation { class CountOperation : public Operation { public: - // For nullary operations, default constructor makes sense - CountOperation() { - aggregator_ = std::make_shared(); + CountOperation() = default; + + // For count operations we have a constant handle, create the aggregator when + // requested so that we get a different object for each query. + [[nodiscard]] shared_ptr aggregator() + const override { + return std::make_shared(); } }; @@ -129,10 +135,9 @@ struct tiledb_query_channel_handle_t */ static constexpr std::string_view object_type_name{"tiledb_query_channel_t"}; - private: + public: tiledb::sm::Query* query_; - public: /** * Default constructor doesn't make sense */ @@ -154,6 +159,7 @@ struct tiledb_query_channel_handle_t "An aggregate operation for output field: " + std::string(output_field) + " already exists."); } + // Add the aggregator the the default channel as this is the only channel // type we currently support query_->add_aggregator_to_default_channel( diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index 70568dd00001..284f8a428cb1 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc + * @file tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc * * @section LICENSE * @@ -35,26 +35,48 @@ #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" using namespace tiledb::test; struct QueryAggregateFx : TemporaryDirectoryFixture { - QueryAggregateFx() { + QueryAggregateFx() + : array_name_(temp_dir_ + "queryaggregate_array") { CHECK( ctx->resources() .config() .set("sm.allow_aggregates_experimental", "true") .ok() == true); + + rm_array(); + create_sparse_array(); + write_sparse_array(); + } + ~QueryAggregateFx() { + rm_array(); } - void create_sparse_array(const std::string& path); - void write_sparse_array(const std::string& path); + void create_sparse_array(); + void write_sparse_array(); + void rm_array(); + + std::string array_name_; }; +void QueryAggregateFx::rm_array() { + int32_t is_dir = 0; + tiledb_vfs_is_dir(ctx, vfs_, array_name_.c_str(), &is_dir); + if (is_dir) { + if (tiledb_vfs_remove_dir(ctx, vfs_, array_name_.c_str()) != TILEDB_OK) { + throw std::runtime_error("couldn't delete existing array " + array_name_); + } + } +} + // Writes simple 2d sparse array to test that query aggregate API // basic functionality such as summing or counting works. -void QueryAggregateFx::write_sparse_array(const std::string& array_name) { +void QueryAggregateFx::write_sparse_array() { tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_WRITE) == TILEDB_OK); tiledb_query_t* query; @@ -112,7 +134,7 @@ void QueryAggregateFx::write_sparse_array(const std::string& array_name) { tiledb_query_free(&query); } -void QueryAggregateFx::create_sparse_array(const std::string& array_name) { +void QueryAggregateFx::create_sparse_array() { // Create dimensions uint64_t tile_extents[] = {2, 2}; uint64_t dim_domain[] = {1, 10, 1, 10}; @@ -179,7 +201,7 @@ void QueryAggregateFx::create_sparse_array(const std::string& array_name) { CHECK(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx, array_name.c_str(), array_schema); + rc = tiledb_array_create(ctx, array_name_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -195,12 +217,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: argument validation", "[capi][query_aggregate][args]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -292,6 +310,12 @@ TEST_CASE_METHOD( ctx, default_channel, "duplicate", tiledb_aggregate_count) == TILEDB_ERR); + // Non-existent input field + CHECK( + tiledb_create_unary_aggregate( + ctx, query, tiledb_channel_operator_sum, "nonexistent", &operation) == + TILEDB_ERR); + // Clean up CHECK(tiledb_query_channel_free(ctx, &default_channel) == TILEDB_OK); CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); @@ -303,12 +327,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates COUNT test", "[capi][query_aggregate][count]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -348,12 +368,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates SUM test", "[capi][query_aggregate][sum]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -398,12 +414,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MIN test", "[capi][query_aggregate][min]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -449,12 +461,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MAX test", "[capi][query_aggregate][max]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -499,12 +507,8 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: datatype checks", "[capi][query_aggregate][attr_type]") { - std::string array_name = temp_dir_ + "queryaggregate_array"; - create_sparse_array(array_name); - write_sparse_array(array_name); - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query; @@ -551,3 +555,46 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); } + +TEST_CASE_METHOD( + QueryAggregateFx, + "C API: Query aggregates lifetime test", + "[capi][query_aggregate][lifetime]") { + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); + int64_t dom[] = {1, 2, 1, 1}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + std::vector d(4); + uint64_t size = 1; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "a", d.data(), &size) == + TILEDB_OK); + + // Transition the query state + tiledb_string_handle_t* string_handle; + REQUIRE(tiledb_query_get_plan(ctx, query, &string_handle) == TILEDB_OK); + + tiledb_query_channel_t* default_channel; + REQUIRE( + tiledb_query_get_default_channel(ctx, query, &default_channel) == + TILEDB_OK); + tiledb_channel_operation_t* op; + CHECK( + tiledb_channel_apply_aggregate( + ctx, default_channel, "Count", tiledb_aggregate_count) == TILEDB_ERR); + CHECK( + tiledb_create_unary_aggregate( + ctx, query, tiledb_channel_operator_min, "_", &op) == TILEDB_ERR); + + // Clean up + REQUIRE(tiledb_query_channel_free(ctx, &default_channel) == TILEDB_OK); + REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); + tiledb_query_free(&query); +} diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 52550aaf9fec..98a5ee60433e 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/query_field/test/unit_capi_query_field.cc + * @file tiledb/api/c_api/query_field/test/unit_capi_query_field.cc * * @section LICENSE * diff --git a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc index 7fba402ad0bd..d3a0c5337301 100644 --- a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc +++ b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc @@ -1,5 +1,5 @@ /** - * @file tiledeb/api/c_api/query_plan/test/unit_capi_query_plan.cc + * @file tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc * * @section LICENSE * diff --git a/tiledb/sm/cpp_api/channel_operation.h b/tiledb/sm/cpp_api/channel_operation.h new file mode 100644 index 000000000000..51153faed8b9 --- /dev/null +++ b/tiledb/sm/cpp_api/channel_operation.h @@ -0,0 +1,129 @@ +/** + * @file channel_operation.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB ChannelOperation class. + */ + +#ifndef TILEDB_CPP_API_CHANNEL_OPERATION_H +#define TILEDB_CPP_API_CHANNEL_OPERATION_H + +#include "channel_operator.h" +#include "context.h" +#include "query.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class ChannelOperation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Default constructor required for COUNT */ + ChannelOperation() = default; + + /** + * Create a ChannelOperation by wrapping a pointer allocated by the C API + * + * @param ctx The Context to use. + * @param operation The tiledb_channel_operation_t allocated by the C API. + */ + ChannelOperation(const Context& ctx, tiledb_channel_operation_t* operation) + : deleter_(&ctx) { + operation_ = + std::shared_ptr(operation, deleter_); + } + + ChannelOperation(const ChannelOperation&) = default; + ChannelOperation(ChannelOperation&&) = default; + ChannelOperation& operator=(const ChannelOperation&) = default; + ChannelOperation& operator=(ChannelOperation&&) = default; + + /** Destructor. */ + ~ChannelOperation() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + private: + virtual const tiledb_channel_operation_t* ptr() const { + return operation_.get(); + }; + + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + /** + * Create a ChannelOperation + * + * @param query The TileDB query + * @param input_field The attribute name the aggregate operation will run on + */ + template Op> + static ChannelOperation create( + const Query& query, const std::string& input_field) { + const Context& ctx = query.ctx(); + tiledb_channel_operation_t* operation = nullptr; + ctx.handle_error(tiledb_create_unary_aggregate( + ctx.ptr().get(), + query.ptr().get(), + Op::ptr(), + input_field.c_str(), + &operation)); + return ChannelOperation(ctx, operation); + } + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Pointer to the TileDB C channel operation object. */ + std::shared_ptr operation_; + + friend class QueryExperimental; + friend class QueryChannel; +}; + +class CountOperation : public ChannelOperation { + private: + virtual const tiledb_channel_operation_t* ptr() const { + return tiledb_aggregate_count; + } +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_CHANNEL_OPERATION_H diff --git a/tiledb/sm/cpp_api/channel_operator.h b/tiledb/sm/cpp_api/channel_operator.h new file mode 100644 index 000000000000..323affa3522b --- /dev/null +++ b/tiledb/sm/cpp_api/channel_operator.h @@ -0,0 +1,77 @@ +/** + * @file channel_operator.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB ChannelOperator class. + */ + +#ifndef TILEDB_CPP_API_CHANNEL_OPERATOR_H +#define TILEDB_CPP_API_CHANNEL_OPERATOR_H + +#include "context.h" +#include "query.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class ChannelOperator {}; + +class SumOperator : public ChannelOperator { + public: + /* ********************************* */ + /* API */ + /* ********************************* */ + static const tiledb_channel_operator_t* ptr() { + return tiledb_channel_operator_sum; + } +}; + +class MinOperator : public ChannelOperator { + public: + /* ********************************* */ + /* API */ + /* ********************************* */ + static const tiledb_channel_operator_t* ptr() { + return tiledb_channel_operator_min; + } +}; + +class MaxOperator : public ChannelOperator { + public: + /* ********************************* */ + /* API */ + /* ********************************* */ + static const tiledb_channel_operator_t* ptr() { + return tiledb_channel_operator_max; + } +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_CHANNEL_OPERATOR_H diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index 61f9a5d2c92f..abd5eefefce0 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -61,7 +61,13 @@ class Deleter { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - Deleter() = default; + Deleter() + : ctx_(nullptr) { + } + + Deleter(const Context* ctx) + : ctx_(ctx) { + } /* ********************************* */ /* DELETERS */ @@ -150,10 +156,19 @@ class Deleter { } } + void operator()(tiledb_query_channel_t* p) const { + tiledb_query_channel_free(ctx_->ptr().get(), &p); + } + + void operator()(tiledb_channel_operation_t* p) const { + tiledb_aggregate_free(ctx_->ptr().get(), &p); + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ + const Context* ctx_; }; } // namespace impl diff --git a/tiledb/sm/cpp_api/query_channel.h b/tiledb/sm/cpp_api/query_channel.h new file mode 100644 index 000000000000..612dc6c4ab24 --- /dev/null +++ b/tiledb/sm/cpp_api/query_channel.h @@ -0,0 +1,142 @@ +/** + * @file query_channel.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB QueryChannel class. + */ + +#ifndef TILEDB_CPP_API_QUERY_CHANNEL_H +#define TILEDB_CPP_API_QUERY_CHANNEL_H + +#include "channel_operation.h" +#include "context.h" +#include "query.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class QueryChannel { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Create an QueryChannel by wrapping a pointer allocated by the C API + * + * @param ctx The Context to use. + * @param ch The tiledb_query_channel_t allocated by the C API. + */ + QueryChannel(const Context& ctx, tiledb_query_channel_t* ch) + : ctx_(ctx) + , deleter_(&ctx) { + channel_ = std::shared_ptr(ch, deleter_); + } + + QueryChannel() = delete; + QueryChannel(const QueryChannel&) = default; + QueryChannel(QueryChannel&&) = default; + QueryChannel& operator=(const QueryChannel&) = default; + QueryChannel& operator=(QueryChannel&&) = default; + + /** Destructor. */ + ~QueryChannel() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Apply an aggregate operation on this channel which will + * produce the results on the output field passed as argument + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * auto array = Array(ctx, uri, TILEDB_READ); + * Query query(ctx, array); + * QueryChannel default_channel = + * QueryExperimental::get_default_channel(query); + * default_channel.apply_aggregate("Count", CountOperation{}); + * + * uint64_t count = 0; + * uint64_t size = sizeof(uint64_t); + * query.set_data_buffer("Count", &count, &size) + * query.submit(); + * @endcode + * + * @param output_field the output field where results will be available. + * @param operation the aggregate operation to be applied on the channel + */ + void apply_aggregate( + const std::string& output_field, const ChannelOperation& operation) { + ctx_.get().handle_error(tiledb_channel_apply_aggregate( + ctx_.get().ptr().get(), + channel_.get(), + output_field.c_str(), + operation.ptr())); + } + + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + private: + /** + * Create a QueryChannel + * + * @param query The TileDB query to use. + */ + static QueryChannel create_default_channel(const Query& query) { + const Context& ctx = query.ctx(); + tiledb_query_channel_t* default_channel = nullptr; + ctx.handle_error(tiledb_query_get_default_channel( + ctx.ptr().get(), query.ptr().get(), &default_channel)); + return QueryChannel(ctx, default_channel); + } + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB query. */ + std::reference_wrapper ctx_; + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Pointer to the TileDB C QueryChannel object. */ + std::shared_ptr channel_; + + friend class QueryExperimental; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_QUERY_CHANNEL_H diff --git a/tiledb/sm/cpp_api/query_experimental.h b/tiledb/sm/cpp_api/query_experimental.h index ce0931153616..92906ab21793 100644 --- a/tiledb/sm/cpp_api/query_experimental.h +++ b/tiledb/sm/cpp_api/query_experimental.h @@ -36,6 +36,7 @@ #include "array_schema_experimental.h" #include "context.h" #include "dimension_label_experimental.h" +#include "query_channel.h" #include "tiledb.h" namespace tiledb { @@ -350,6 +351,65 @@ class QueryExperimental { } return elements; } + + /** + * Get a `QueryChannel` instance that represents the default channel + * of the query passed as argument + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * auto array = Array(ctx, uri, TILEDB_READ); + * Query query(ctx, array); + * QueryChannel default_channel = + * QueryExperimental::get_default_channel(query); + * default_channel.apply_aggregate("Count", CountOperation{}); + * + * uint64_t count = 0; + * uint64_t size = sizeof(uint64_t); + * query.set_data_buffer("Count", &count, &size) + * query.submit(); + * @endcode + * + * @param query Query object. + * @return The default query channel. + */ + static QueryChannel get_default_channel(const Query& query) { + return QueryChannel::create_default_channel(query); + } + + /** + * Create an aggregate operation that operates on a single input field + * and produces a single output + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * auto array = Array(ctx, uri, TILEDB_READ); + * Query query(ctx, array); + * query.add_range("dim", 1, 5) + * QueryChannel default_channel = + * QueryExperimental::get_default_channel(query); + * ChannelOperation operation = + * QueryExperimental::create_unary_aggregate(query, "a"); + * default_channel.apply_aggregate("Sum", operation); + * + * double sum = 0; + * uint64_t size = sizeof(double); + * query.set_data_buffer("Sum", &sum, &size) + * query.submit(); + * @endcode + * + * @tparam Op The channel operator type + * @param query Query object. + * @param input_field The attribute name as input for the aggregate + * @return The aggregate operation + */ + template Op> + static ChannelOperation create_unary_aggregate( + const Query& query, const std::string& input_field) { + return ChannelOperation::create(query, input_field); + } }; } // namespace tiledb From 831b5b03ba4e57a62b96e0307f0457e6fab0f9eb Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Fri, 20 Oct 2023 19:05:05 +0300 Subject: [PATCH 015/456] Query aggregates REST support (#4415) Add rest support for query aggregates. Notes to reviewers: - At this stage we should pay special attention to the capnp specification, make sure it's as generic as possible according to the aggregates design document. - I tested this manually with a local rest server and serialization paths seem to work as expected, but overall the feature doesn't currently work because we need Go support on the REST side which I'll file as followup (When the REST server deserializes the query, the query buffers are set to `nullptr` and their sizes to the correct sizes passed by the user. The Go code is supposed to allocate memory and set the buffers again before the query is submitted "locally". The tests work because in the serialization wrappers we emulate this allocation with some stack allocated buffers, see changes in `helpers.cc`) --- TYPE: NO_HISTORY | FEATURE DESC: Query aggregates REST support --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/support/src/helpers.cc | 28 +- tiledb/CMakeLists.txt | 4 + .../query_aggregate/query_aggregate_api.cc | 106 +--- .../query_aggregate_api_internal.h | 90 +-- .../c_api/query_aggregate/test/CMakeLists.txt | 7 +- .../test/unit_capi_query_aggregate.cc | 160 ++++- .../api/c_api/query_field/query_field_api.cc | 3 +- tiledb/sm/misc/constants.h | 21 + tiledb/sm/query/query.cc | 24 +- tiledb/sm/query/query.h | 42 +- .../query/readers/aggregators/CMakeLists.txt | 2 +- .../readers/aggregators/count_aggregator.cc | 6 +- .../readers/aggregators/count_aggregator.h | 15 +- .../query/readers/aggregators/iaggregator.h | 9 +- .../aggregators/input_field_validator.h | 93 +++ .../readers/aggregators/mean_aggregator.cc | 13 +- .../readers/aggregators/mean_aggregator.h | 16 +- .../readers/aggregators/min_max_aggregator.cc | 12 +- .../readers/aggregators/min_max_aggregator.h | 26 +- .../sm/query/readers/aggregators/operation.cc | 53 ++ .../sm/query/readers/aggregators/operation.h | 209 +++++++ .../query/readers/aggregators/query_channel.h | 90 +++ .../readers/aggregators/sum_aggregator.cc | 13 +- .../readers/aggregators/sum_aggregator.h | 16 +- .../aggregators/test/unit_aggregators.cc | 18 +- tiledb/sm/serialization/query.cc | 42 +- tiledb/sm/serialization/query_aggregates.cc | 132 ++++ tiledb/sm/serialization/query_aggregates.h | 66 ++ tiledb/sm/serialization/tiledb-rest.capnp | 31 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 330 +++++++--- tiledb/sm/serialization/tiledb-rest.capnp.h | 575 +++++++++++++++++- 31 files changed, 1924 insertions(+), 328 deletions(-) create mode 100644 tiledb/sm/query/readers/aggregators/input_field_validator.h create mode 100644 tiledb/sm/query/readers/aggregators/operation.cc create mode 100644 tiledb/sm/query/readers/aggregators/operation.h create mode 100644 tiledb/sm/query/readers/aggregators/query_channel.h create mode 100644 tiledb/sm/serialization/query_aggregates.cc create mode 100644 tiledb/sm/serialization/query_aggregates.h diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 45982c3df10d..44611033b3b9 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1850,7 +1850,9 @@ int submit_query_wrapper( // 4. Server -> Client : Send query response std::vector serialized2; rc = serialize_query(server_ctx, server_deser_query, &serialized2, 0); - REQUIRE(rc == TILEDB_OK); + if (rc != TILEDB_OK) { + return rc; + } if (!refactored_query_v2) { // Close array and clean up @@ -1950,13 +1952,31 @@ void allocate_query_buffers_server_side( tiledb_query_t* query, ServerQueryBuffers& query_buffers) { int rc = 0; - const auto buffer_names = query->query_->buffer_names(); + auto buffer_names = query->query_->buffer_names(); + const auto aggregate_names = query->query_->aggregate_buffer_names(); + buffer_names.insert( + buffer_names.end(), aggregate_names.begin(), aggregate_names.end()); + for (uint64_t i = 0; i < buffer_names.size(); i++) { const auto& name = buffer_names[i]; const auto& buff = query->query_->buffer(name); const auto& schema = query->query_->array_schema(); - auto var_size = schema.var_size(name); - auto nullable = schema.is_nullable(name); + + // TODO: This is yet another instance where there needs to be a common + // mechanism for reporting the common properties of a field. + // Refactor to use query_field_t. + bool var_size = false; + bool nullable = false; + if (query->query_->is_aggregate(name)) { + var_size = + query->query_->get_aggregate(name).value()->aggregation_var_sized(); + nullable = + query->query_->get_aggregate(name).value()->aggregation_nullable(); + } else { + var_size = schema.var_size(name); + nullable = schema.is_nullable(name); + } + if (var_size && buff.buffer_var_ == nullptr) { // Variable-sized buffer query_buffers.attr_or_dim_data.emplace_back(*buff.buffer_var_size_); diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 9b0e32ec5404..07acc6d27cfc 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -255,6 +255,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/count_aggregator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/mean_aggregator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/operation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/output_buffer_validator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/safe_sum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/sum_aggregator.cc @@ -286,6 +287,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/fragments.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/global_stats.cc @@ -338,6 +340,7 @@ if (TILEDB_SERIALIZATION) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/fragments.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -359,6 +362,7 @@ if (TILEDB_SERIALIZATION) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/fragments.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/tiledb-rest.capnp.c++ diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index a06597b6e17a..c26cf14ed792 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -36,41 +36,21 @@ #include "tiledb/api/c_api/query/query_api_internal.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api_support/c_api_support.h" -#include "tiledb/type/apply_with_type.h" +// const tiledb_channel_operator_handle_t* tiledb_channel_operator_sum = tiledb_channel_operator_handle_t::make_handle( - TILEDB_QUERY_CHANNEL_OPERATOR_SUM, "SUM"); + tiledb::sm::constants::aggregate_sum_str); const tiledb_channel_operator_handle_t* tiledb_channel_operator_min = tiledb_channel_operator_handle_t::make_handle( - TILEDB_QUERY_CHANNEL_OPERATOR_MIN, "MIN"); + tiledb::sm::constants::aggregate_min_str); const tiledb_channel_operator_handle_t* tiledb_channel_operator_max = tiledb_channel_operator_handle_t::make_handle( - TILEDB_QUERY_CHANNEL_OPERATOR_MAX, "MAX"); + tiledb::sm::constants::aggregate_max_str); const tiledb_channel_operation_handle_t* tiledb_aggregate_count = tiledb_channel_operation_handle_t::make_handle( - std::make_shared()); - -shared_ptr tiledb_channel_operator_handle_t::make_operation( - const tiledb::sm::FieldInfo& fi) const { - switch (this->value()) { - case TILEDB_QUERY_CHANNEL_OPERATOR_SUM: { - return std::make_shared(fi, this); - } - case TILEDB_QUERY_CHANNEL_OPERATOR_MIN: { - return std::make_shared(fi, this); - } - case TILEDB_QUERY_CHANNEL_OPERATOR_MAX: { - return std::make_shared(fi, this); - } - default: - throw tiledb::api::CAPIStatusException( - "operator has unsupported value: " + - std::to_string(static_cast(this->value()))); - break; - } -} + std::make_shared()); namespace tiledb::api { @@ -260,6 +240,12 @@ capi_return_t tiledb_query_channel_free( } // namespace tiledb::api +shared_ptr +tiledb_channel_operator_handle_t::make_operation( + const tiledb::sm::FieldInfo& fi) const { + return tiledb::sm::Operation::make_operation(this->name(), fi); +} + using tiledb::api::api_entry_with_context; capi_return_t tiledb_channel_operator_sum_get( @@ -324,73 +310,3 @@ capi_return_t tiledb_query_channel_free( return tiledb::api::api_entry_with_context< tiledb::api::tiledb_query_channel_free>(ctx, channel); } - -MaxOperation::MaxOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op) { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - if constexpr (tiledb::type::TileDBNumeric) { - ensure_aggregate_numeric_field(op, fi); - } - - // This is a min/max on strings, should be refactored out once we - // change (STRING_ASCII,CHAR) mapping in apply_with_type - if constexpr (std::is_same_v) { - aggregator_ = - std::make_shared>(fi); - } else { - aggregator_ = - std::make_shared>(fi); - } - } else { - throw std::logic_error( - "MAX aggregates can only be requested on numeric and string " - "types"); - } - }; - apply_with_type(g, fi.type_); -} - -MinOperation::MinOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op) { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - if constexpr (tiledb::type::TileDBNumeric) { - ensure_aggregate_numeric_field(op, fi); - } - - // This is a min/max on strings, should be refactored out once we - // change (STRING_ASCII,CHAR) mapping in apply_with_type - if constexpr (std::is_same_v) { - aggregator_ = - std::make_shared>(fi); - } else { - aggregator_ = - std::make_shared>(fi); - } - } else { - throw std::logic_error( - "MIN aggregates can only be requested on numeric and string " - "types"); - } - }; - apply_with_type(g, fi.type_); -} - -SumOperation::SumOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op) { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBNumeric) { - ensure_aggregate_numeric_field(op, fi); - aggregator_ = - std::make_shared>(fi); - } else { - throw std::logic_error( - "SUM aggregates can only be requested on numeric types"); - } - }; - apply_with_type(g, fi.type_); -} diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h index 93e45b4da69b..1c736f077b03 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h @@ -38,64 +38,7 @@ #include "tiledb/api/c_api_support/handle/handle.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/query/query.h" -#include "tiledb/sm/query/readers/aggregators/count_aggregator.h" -#include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" -#include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" - -enum QueryChannelOperator { - TILEDB_QUERY_CHANNEL_OPERATOR_COUNT = 0, - TILEDB_QUERY_CHANNEL_OPERATOR_SUM, - TILEDB_QUERY_CHANNEL_OPERATOR_MIN, - TILEDB_QUERY_CHANNEL_OPERATOR_MAX -}; - -class Operation { - protected: - shared_ptr aggregator_ = nullptr; - - public: - [[nodiscard]] virtual shared_ptr aggregator() const { - return aggregator_; - } - - virtual ~Operation(){}; -}; - -class MinOperation : public Operation { - public: - MinOperation() = delete; - explicit MinOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op); -}; - -class MaxOperation : public Operation { - public: - MaxOperation() = delete; - explicit MaxOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op); -}; - -class SumOperation : public Operation { - public: - SumOperation() = delete; - explicit SumOperation( - const tiledb::sm::FieldInfo& fi, - const tiledb_channel_operator_handle_t* op); -}; - -class CountOperation : public Operation { - public: - CountOperation() = default; - - // For count operations we have a constant handle, create the aggregator when - // requested so that we get a different object for each query. - [[nodiscard]] shared_ptr aggregator() - const override { - return std::make_shared(); - } -}; +#include "tiledb/sm/query/readers/aggregators/operation.h" struct tiledb_channel_operation_handle_t : public tiledb::api::CAPIHandle { @@ -106,7 +49,7 @@ struct tiledb_channel_operation_handle_t "tiledb_channel_operation_t"}; private: - std::shared_ptr operation_; + std::shared_ptr operation_; public: /** @@ -119,7 +62,7 @@ struct tiledb_channel_operation_handle_t * @param operation An internal operation object */ explicit tiledb_channel_operation_handle_t( - const shared_ptr& operation) + const shared_ptr& operation) : operation_{operation} { } @@ -176,7 +119,6 @@ struct tiledb_channel_operator_handle_t "tiledb_channel_operator_handle_t"}; private: - QueryChannelOperator value_; std::string name_; public: @@ -190,36 +132,16 @@ struct tiledb_channel_operator_handle_t * @param op An enum specifying the type of operator * @param name A string representation of the operator */ - explicit tiledb_channel_operator_handle_t( - QueryChannelOperator op, const std::string& name) - : value_{op} - , name_{name} { - } - - [[nodiscard]] inline QueryChannelOperator value() const { - return value_; + explicit tiledb_channel_operator_handle_t(const std::string& name) + : name_{name} { } [[nodiscard]] inline std::string name() const { return name_; } - [[nodiscard]] std::shared_ptr make_operation( + [[nodiscard]] std::shared_ptr make_operation( const tiledb::sm::FieldInfo& fi) const; }; -inline void ensure_aggregate_numeric_field( - const tiledb_channel_operator_t* op, const tiledb::sm::FieldInfo& fi) { - if (fi.var_sized_) { - throw tiledb::api::CAPIStatusException( - op->name() + " aggregates are not supported for var sized attributes."); - } - if (fi.cell_val_num_ != 1) { - throw tiledb::api::CAPIStatusException( - op->name() + - " aggregates are not supported for attributes with cell_val_num " - "greater than one."); - } -} - #endif // TILEDB_CAPI_QUERY_AGGREGATE_INTERNAL_H diff --git a/tiledb/api/c_api/query_aggregate/test/CMakeLists.txt b/tiledb/api/c_api/query_aggregate/test/CMakeLists.txt index ffd39b7b68be..f4e738447846 100644 --- a/tiledb/api/c_api/query_aggregate/test/CMakeLists.txt +++ b/tiledb/api/c_api/query_aggregate/test/CMakeLists.txt @@ -30,7 +30,12 @@ commence(unit_test capi_query_aggregate) this_target_sources(unit_capi_query_aggregate.cc) this_target_object_libraries(capi_query_aggregate) if (NOT MSVC) - target_compile_options(unit_capi_query_aggregate PRIVATE -Wno-deprecated-declarations) + target_compile_options(unit_capi_query_aggregate PRIVATE -Wno-deprecated-declarations) endif() this_target_link_libraries(tiledb_test_support_lib) + + if (TILEDB_SERIALIZATION) + add_definitions(-DTILEDB_SERIALIZATION) + endif() + conclude(unit_test) diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index 284f8a428cb1..fa7f5533311b 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -31,6 +31,7 @@ */ #include +#include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" @@ -40,6 +41,11 @@ using namespace tiledb::test; struct QueryAggregateFx : TemporaryDirectoryFixture { + // Serialization parameters + bool serialize_ = false; + bool refactored_query_v2_ = false; + ServerQueryBuffers server_buffers_; + QueryAggregateFx() : array_name_(temp_dir_ + "queryaggregate_array") { CHECK( @@ -327,6 +333,15 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates COUNT test", "[capi][query_aggregate][count]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -354,7 +369,16 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Count", &count, &size) == TILEDB_OK); - REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); + CHECK(count == 9); // Clean up @@ -368,6 +392,15 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates SUM test", "[capi][query_aggregate][sum]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -399,7 +432,16 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Sum", &sum, &size) == TILEDB_OK); - REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); + CHECK(sum == 55); // Clean up @@ -414,6 +456,15 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MIN test", "[capi][query_aggregate][min]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -445,7 +496,15 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Min", &min, &size) == TILEDB_OK); - REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); CHECK(min == 1); @@ -461,6 +520,15 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MAX test", "[capi][query_aggregate][max]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -492,7 +560,16 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Max", &max, &size) == TILEDB_OK); - REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); + CHECK(max == 10); // Clean up @@ -598,3 +675,78 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); } + +TEST_CASE_METHOD( + QueryAggregateFx, + "C API: Query aggregates serialization test", + "[capi][query_aggregate][serialization][incompletes]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); + + int64_t dom[] = {1, 10, 1, 1}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + tiledb_query_channel_t* default_channel; + REQUIRE( + tiledb_query_get_default_channel(ctx, query, &default_channel) == + TILEDB_OK); + + tiledb_channel_operation_t* sum_op; + REQUIRE( + tiledb_create_unary_aggregate( + ctx, query, tiledb_channel_operator_sum, "a", &sum_op) == TILEDB_OK); + REQUIRE( + tiledb_channel_apply_aggregate(ctx, default_channel, "Sum", sum_op) == + TILEDB_OK); + + uint64_t sum = 0; + uint64_t size = 8; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "Sum", &sum, &size) == + TILEDB_OK); + + uint64_t a_size = 1; + int32_t a; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "a", &a, &a_size) == TILEDB_OK); + + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + + if (serialize_) { + CHECK(rc != 0); + } else { + tiledb_query_status_t status; + REQUIRE(tiledb_query_get_status(ctx, query, &status) == TILEDB_OK); + REQUIRE(status == TILEDB_INCOMPLETE); + CHECK(rc == 0); + } + + // Clean up + CHECK(tiledb_aggregate_free(ctx, &sum_op) == TILEDB_OK); + CHECK(tiledb_query_channel_free(ctx, &default_channel) == TILEDB_OK); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); + tiledb_query_free(&query); +} diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc index 25bfc88774ce..78355dfee2c8 100644 --- a/tiledb/api/c_api/query_field/query_field_api.cc +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -76,7 +76,8 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( field_origin_ = std::make_shared(); auto aggregate = query_->get_aggregate(field_name_).value(); type_ = aggregate->output_datatype(); - cell_val_num_ = aggregate->var_sized() ? tiledb::sm::constants::var_num : 1; + cell_val_num_ = + aggregate->aggregation_var_sized() ? tiledb::sm::constants::var_num : 1; } else { throw tiledb::api::CAPIStatusException("There is no field " + field_name_); } diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 194e2918b297..7afdea1e3a04 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -49,6 +49,27 @@ enum class SerializationType : uint8_t; namespace constants { +// The following aggregate constants are declared static to avoid a SIOF +// issue with constant aggregate operator handles with extern linkage. + +/** The name of COUNT aggregator. */ +static const std::string aggregate_count_str = "COUNT"; + +/** The name of SUM aggregator. */ +static const std::string aggregate_sum_str = "SUM"; + +/** The name of MIN aggregator. */ +static const std::string aggregate_min_str = "MIN"; + +/** The name of MAX aggregator. */ +static const std::string aggregate_max_str = "MAX"; + +/** The name of NULL_COUNT aggregator. */ +static const std::string aggregate_null_count_str = "NULL_COUNT"; + +/** The name of MEAN aggregator. */ +static const std::string aggregate_mean_str = "MEAN"; + /** * Reduction factor (must be in [0.0, 1.0]) for the multi_range subarray * split by the partitioner. If the number is equal to 0.3, then this diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 55aff280373a..a969311f79b5 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -417,6 +417,16 @@ std::vector Query::dimension_label_buffer_names() const { return ret; } +std::vector Query::aggregate_buffer_names() const { + std::vector buffer_names; + buffer_names.reserve(aggregate_buffers_.size()); + + for (const auto& buffer : aggregate_buffers_) { + buffer_names.push_back(buffer.first); + } + return buffer_names; +} + std::vector Query::unwritten_buffer_names() const { std::vector ret; for (auto& name : buffer_names()) { @@ -445,6 +455,12 @@ QueryBuffer Query::buffer(const std::string& name) const { if (buf != label_buffers_.end()) { return buf->second; } + } else if (is_aggregate(name)) { + // Aggregate buffer + auto buf = aggregate_buffers_.find(name); + if (buf != aggregate_buffers_.end()) { + return buf->second; + } } else { // Attribute or dimension auto buf = buffers_.find(name); @@ -1043,7 +1059,8 @@ Status Query::set_data_buffer( // If this is an aggregate buffer, set it and return. if (is_aggregate(name)) { - const bool is_var = default_channel_aggregates_[name]->var_sized(); + const bool is_var = + default_channel_aggregates_[name]->aggregation_var_sized(); if (!is_var) { // Fixed size data buffer aggregate_buffers_[name].set_data_buffer(buffer, buffer_size); @@ -2190,11 +2207,6 @@ void Query::reset_coords_markers() { } void Query::copy_aggregates_data_to_user_buffer() { - if (array_->is_remote() && !default_channel_aggregates_.empty()) { - throw QueryStatusException( - "Cannot submit query; Query aggregates are not supported in REST yet"); - } - for (auto& default_channel_aggregate : default_channel_aggregates_) { default_channel_aggregate.second->copy_to_user_buffer( default_channel_aggregate.first, aggregate_buffers_); diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 5a43e6c0eab9..49fb0d9f7e37 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -51,6 +51,7 @@ #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/query/query_remote_buffer_storage.h" #include "tiledb/sm/query/readers/aggregators/iaggregator.h" +#include "tiledb/sm/query/readers/aggregators/query_channel.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/query/validity_vector.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" @@ -227,6 +228,9 @@ class Query { /** Returns the names of dimension label buffers for the query. */ std::vector dimension_label_buffer_names() const; + /** Returns the names of aggregate buffers for the query. */ + std::vector aggregate_buffer_names() const; + /** * Returns the names of the buffers set by the user for the query not already * written by a previous partial attribute write. @@ -234,7 +238,7 @@ class Query { std::vector unwritten_buffer_names() const; /** - * Gets the query buffer for the input attribute/dimension. + * Gets the query buffer for the input field. * An empty string means the special default attribute. */ QueryBuffer buffer(const std::string& name) const; @@ -739,10 +743,46 @@ class Query { default_channel_aggregates_.emplace(output_field_name, aggregator); } + /** + * Get all aggregates from the default channel. + */ + std::unordered_map> + get_default_channel_aggregates() { + return default_channel_aggregates_; + } + /** Returns an aggregate based on the output field. */ std::optional> get_aggregate( std::string output_field_name) const; + /** + * Get a list of all channels and their aggregates + */ + std::vector get_channels() { + // Currently only the default channel is supported + return {QueryChannel(true, default_channel_aggregates_)}; + } + + /** + * Add a channel to the query + */ + void add_channel(const QueryChannel& channel) { + if (channel.is_default()) { + default_channel_aggregates_ = channel.aggregates(); + return; + } + throw std::logic_error( + "We currently only support a default channel for queries"); + } + + /** + * Returns true if the query has any aggregates on any channels + */ + bool has_aggregates() { + // We only need to check the default channel for now + return default_channel_aggregates_.empty(); + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/CMakeLists.txt b/tiledb/sm/query/readers/aggregators/CMakeLists.txt index 0ed74a6a7366..d4df7081b37b 100644 --- a/tiledb/sm/query/readers/aggregators/CMakeLists.txt +++ b/tiledb/sm/query/readers/aggregators/CMakeLists.txt @@ -31,7 +31,7 @@ include(object_library) # `aggregators` object library # commence(object_library aggregators) - this_target_sources(count_aggregator.cc mean_aggregator.cc min_max_aggregator.cc output_buffer_validator.cc safe_sum.cc sum_aggregator.cc) + this_target_sources(count_aggregator.cc mean_aggregator.cc min_max_aggregator.cc operation.cc output_buffer_validator.cc safe_sum.cc sum_aggregator.cc) this_target_object_libraries(baseline array_schema) conclude(object_library) diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.cc b/tiledb/sm/query/readers/aggregators/count_aggregator.cc index 1dcf824218fe..7193976b0eda 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.cc @@ -91,11 +91,7 @@ void CountAggregatorBase::copy_to_user_buffer( NullCountAggregator::NullCountAggregator(FieldInfo field_info) : CountAggregatorBase(field_info) , field_info_(field_info) { - if (!field_info_.is_nullable_) { - throw CountAggregatorStatusException( - "NullCount aggregates must only be requested for nullable " - "attributes."); - } + ensure_field_nullable(field_info); } // Explicit template instantiations diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.h b/tiledb/sm/query/readers/aggregators/count_aggregator.h index 7df272d34030..3299ccaff4c2 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.h @@ -60,10 +60,15 @@ class CountAggregatorBase : public OutputBufferValidator, public IAggregator { /* ********************************* */ /** Returns if the aggregation is var sized or not. */ - bool var_sized() override { + bool aggregation_var_sized() override { return false; }; + /** Returns if the aggregation is nullable or not. */ + bool aggregation_nullable() override { + return false; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return true; @@ -96,6 +101,11 @@ class CountAggregatorBase : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns name of the aggregate. */ + std::string aggregate_name() override { + return constants::aggregate_count_str; + } + /** Returns the TileDB datatype of the output field for the aggregate. */ Datatype output_datatype() override { return Datatype::UINT64; @@ -134,7 +144,8 @@ class CountAggregator : public CountAggregatorBase { } }; -class NullCountAggregator : public CountAggregatorBase { +class NullCountAggregator : public CountAggregatorBase, + public InputFieldValidator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ diff --git a/tiledb/sm/query/readers/aggregators/iaggregator.h b/tiledb/sm/query/readers/aggregators/iaggregator.h index 535bb3c560b7..61ebb71d3816 100644 --- a/tiledb/sm/query/readers/aggregators/iaggregator.h +++ b/tiledb/sm/query/readers/aggregators/iaggregator.h @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/sm/misc/constants.h" +#include "tiledb/sm/query/readers/aggregators/input_field_validator.h" #include "tiledb/sm/query/readers/aggregators/output_buffer_validator.h" namespace tiledb::sm { @@ -65,7 +66,10 @@ class IAggregator { virtual bool need_recompute_on_overflow() = 0; /** Returns if the aggregation is var sized or not. */ - virtual bool var_sized() = 0; + virtual bool aggregation_var_sized() = 0; + + /** Returns if the aggregation is nullable or not. */ + virtual bool aggregation_nullable() = 0; /** * Validate the result buffer. @@ -94,6 +98,9 @@ class IAggregator { std::string output_field_name, std::unordered_map& buffers) = 0; + /** Returns name of the aggregate, e.g. COUNT, MIN, SUM. */ + virtual std::string aggregate_name() = 0; + /** Returns the TileDB datatype of the output field for the aggregate. */ virtual Datatype output_datatype() = 0; }; diff --git a/tiledb/sm/query/readers/aggregators/input_field_validator.h b/tiledb/sm/query/readers/aggregators/input_field_validator.h new file mode 100644 index 000000000000..86e73cb12446 --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/input_field_validator.h @@ -0,0 +1,93 @@ +/** + * @file input_field_validator.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class InputFieldValidator. + */ + +#ifndef TILEDB_INPUT_FIELD_VALIDATOR_H +#define TILEDB_INPUT_FIELD_VALIDATOR_H + +#include "tiledb/sm/query/readers/aggregators/aggregate_buffer.h" +#include "tiledb/sm/query/readers/aggregators/field_info.h" + +namespace tiledb::sm { + +class InputFieldValidatorStatusException : public StatusException { + public: + explicit InputFieldValidatorStatusException(const std::string& message) + : StatusException("InputFieldValidator", message) { + } +}; + +class InputFieldValidator { + protected: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + InputFieldValidator() { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Ensure the input field is numeric. + * + * @param field_info Field info. + */ + inline static void ensure_field_numeric(const FieldInfo& field_info) { + if (field_info.var_sized_) { + throw InputFieldValidatorStatusException( + "Aggregate is not supported for var sized non-string fields."); + } + if (field_info.cell_val_num_ != 1) { + throw InputFieldValidatorStatusException( + "Aggregate is not supported for non-string fields with cell_val_num " + "greater than one."); + } + } + + /** + * Ensure the input field is nullable. + * + * @param field_info Field info. + */ + inline static void ensure_field_nullable(const FieldInfo& field_info) { + if (!field_info.is_nullable_) { + throw InputFieldValidatorStatusException( + "Aggregate must only be requested for nullable fields."); + } + } +}; + +} // namespace tiledb::sm + +#endif // TILEDB_INPUT_FIELD_VALIDATOR_H diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.cc b/tiledb/sm/query/readers/aggregators/mean_aggregator.cc index 44ae0b329b01..4f4d845630b3 100644 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/mean_aggregator.cc @@ -54,18 +54,7 @@ MeanAggregator::MeanAggregator(const FieldInfo field_info) , validity_value_( field_info_.is_nullable_ ? std::make_optional(0) : nullopt) , sum_overflowed_(false) { - // TODO: These argument validation can be merged for mean/sum and possibly - // other aggregates. (sc-33763). - if (field_info_.var_sized_) { - throw MeanAggregatorStatusException( - "Mean aggregates are not supported for var sized attributes."); - } - - if (field_info_.cell_val_num_ != 1) { - throw MeanAggregatorStatusException( - "Mean aggregates are not supported for attributes with cell_val_num " - "greater than one."); - } + ensure_field_numeric(field_info_); } template diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.h b/tiledb/sm/query/readers/aggregators/mean_aggregator.h index 9fbd6fb58f9e..96e38504188e 100644 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/mean_aggregator.h @@ -46,7 +46,9 @@ namespace tiledb::sm { class QueryBuffer; template -class MeanAggregator : public OutputBufferValidator, public IAggregator { +class MeanAggregator : public InputFieldValidator, + public OutputBufferValidator, + public IAggregator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -74,10 +76,15 @@ class MeanAggregator : public OutputBufferValidator, public IAggregator { } /** Returns if the aggregation is var sized or not. */ - bool var_sized() override { + bool aggregation_var_sized() override { return false; }; + /** Returns if the aggregation is nullable or not. */ + bool aggregation_nullable() override { + return field_info_.is_nullable_; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return true; @@ -110,6 +117,11 @@ class MeanAggregator : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns name of the aggregate. */ + std::string aggregate_name() override { + return constants::aggregate_mean_str; + } + /** Returns the TileDB datatype of the output field for the aggregate. */ Datatype output_datatype() override { return Datatype::FLOAT64; diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc b/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc index 972e740e68b2..f732c40c685a 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc @@ -120,16 +120,8 @@ ComparatorAggregator::ComparatorAggregator(const FieldInfo& field_info) : ComparatorAggregatorBase(field_info) , OutputBufferValidator(field_info) , aggregate_with_count_(field_info) { - if (field_info.var_sized_ && !std::is_same::value) { - throw MinMaxAggregatorStatusException( - "Min/max aggregates are not supported for var sized non-string " - "attributes."); - } - - if (field_info.cell_val_num_ != 1 && !std::is_same::value) { - throw MinMaxAggregatorStatusException( - "Min/max aggregates are not supported for attributes with cell_val_num " - "greater than one."); + if constexpr (!std::is_same::value) { + ensure_field_numeric(field_info); } } diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h index e202445b04ac..57ac4292fee2 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h @@ -104,6 +104,7 @@ class ComparatorAggregatorBase { template class ComparatorAggregator : public ComparatorAggregatorBase, + public InputFieldValidator, public OutputBufferValidator, public IAggregator { protected: @@ -137,10 +138,15 @@ class ComparatorAggregator : public ComparatorAggregatorBase, } /** Returns if the aggregation is var sized or not. */ - bool var_sized() override { + bool aggregation_var_sized() override { return ComparatorAggregatorBase::field_info_.var_sized_; }; + /** Returns if the aggregation is nullable or not. */ + bool aggregation_nullable() override { + return ComparatorAggregatorBase::field_info_.is_nullable_; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return false; @@ -215,6 +221,15 @@ class MinAggregator : public ComparatorAggregator< DISABLE_COPY_AND_COPY_ASSIGN(MinAggregator); DISABLE_MOVE_AND_MOVE_ASSIGN(MinAggregator); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Returns name of the aggregate. */ + std::string aggregate_name() override { + return constants::aggregate_min_str; + } }; template @@ -240,6 +255,15 @@ class MaxAggregator : public ComparatorAggregator< DISABLE_COPY_AND_COPY_ASSIGN(MaxAggregator); DISABLE_MOVE_AND_MOVE_ASSIGN(MaxAggregator); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Returns name of the aggregate, e.g. COUNT, MIN, SUM. */ + std::string aggregate_name() override { + return constants::aggregate_max_str; + } }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/operation.cc b/tiledb/sm/query/readers/aggregators/operation.cc new file mode 100644 index 000000000000..14634b293674 --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/operation.cc @@ -0,0 +1,53 @@ +/** + * @file operation.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class Operation. + */ + +#include "tiledb/sm/query/readers/aggregators/operation.h" + +namespace tiledb::sm { + +shared_ptr Operation::make_operation( + const std::string& name, const std::optional& fi) { + if (name == constants::aggregate_sum_str) { + return common::make_shared(HERE(), fi.value()); + } else if (name == constants::aggregate_min_str) { + return common::make_shared(HERE(), fi.value()); + } else if (name == constants::aggregate_max_str) { + return common::make_shared(HERE(), fi.value()); + } else if (name == constants::aggregate_count_str) { + return common::make_shared(HERE()); + } + + throw std::logic_error( + "Unable to create and aggregate operation using name: " + name); +} + +} // namespace tiledb::sm \ No newline at end of file diff --git a/tiledb/sm/query/readers/aggregators/operation.h b/tiledb/sm/query/readers/aggregators/operation.h new file mode 100644 index 000000000000..cf1867962fc3 --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/operation.h @@ -0,0 +1,209 @@ +/** + * @file operation.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class Operation and its derivates. + */ + +#ifndef TILEDB_OPERATION_H +#define TILEDB_OPERATION_H + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/query/readers/aggregators/count_aggregator.h" +#include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" +#include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" +#include "tiledb/sm/query/readers/aggregators/sum_type.h" +#include "tiledb/type/apply_with_type.h" + +namespace tiledb::sm { + +class Operation : public InputFieldValidator { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + virtual ~Operation(){}; + + /* ********************************* */ + /* API */ + /* ********************************* */ + /** + * @return a shared pointer to the internal aggregator object + */ + [[nodiscard]] virtual shared_ptr aggregator() const { + return aggregator_; + } + + /** + * Create the right operation based on the aggregator name + * + * @param name The name of the aggregator e.g. SUM, MIN + * @param fi A FieldInfo object describing the input field or nullopt for + * nullary operations + * @return The appropriate operation for the name + */ + static shared_ptr make_operation( + const std::string& name, const std::optional& fi); + + /* ********************************* */ + /* PROTECTED ATTRIBUTES */ + /* ********************************* */ + protected: + shared_ptr aggregator_ = nullptr; +}; + +class MinOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + MinOperation() = delete; + + /** + * Construct the operation where the internal aggregator object + * is instantiated to the correct type given the input field type. + * @param fi The FieldInfo for the input field + */ + explicit MinOperation(const tiledb::sm::FieldInfo& fi) { + auto g = [&](auto T) { + if constexpr (tiledb::type::TileDBFundamental) { + if constexpr (tiledb::type::TileDBNumeric) { + ensure_field_numeric(fi); + } + + // This is a min/max on strings, should be refactored out once we + // change (STRING_ASCII,CHAR) mapping in apply_with_type + if constexpr (std::is_same_v) { + aggregator_ = + tdb::make_shared>( + HERE(), fi); + } else { + aggregator_ = + tdb::make_shared>( + HERE(), fi); + } + } else { + throw std::logic_error( + "MIN aggregates can only be requested on numeric and string " + "types"); + } + }; + type::apply_with_type(g, fi.type_); + } +}; + +class MaxOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + MaxOperation() = delete; + + /** + * Construct the operation where the internal aggregator object + * is instantiated to the correct type given the input field type. + * @param fi The FieldInfo for the input field + */ + explicit MaxOperation(const tiledb::sm::FieldInfo& fi) { + auto g = [&](auto T) { + if constexpr (tiledb::type::TileDBFundamental) { + if constexpr (tiledb::type::TileDBNumeric) { + ensure_field_numeric(fi); + } + + // This is a min/max on strings, should be refactored out once we + // change (STRING_ASCII,CHAR) mapping in apply_with_type + if constexpr (std::is_same_v) { + aggregator_ = + tdb::make_shared>( + HERE(), fi); + } else { + aggregator_ = + tdb::make_shared>( + HERE(), fi); + } + } else { + throw std::logic_error( + "MAX aggregates can only be requested on numeric and string " + "types"); + } + }; + type::apply_with_type(g, fi.type_); + } +}; + +class SumOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + SumOperation() = delete; + + /** + * Construct the operation where the internal aggregator object + * is instantiated to the correct type given the input field type. + * @param fi The FieldInfo for the input field + */ + explicit SumOperation(const tiledb::sm::FieldInfo& fi) { + auto g = [&](auto T) { + if constexpr (tiledb::type::TileDBNumeric) { + ensure_field_numeric(fi); + aggregator_ = tdb::make_shared>( + HERE(), fi); + } else { + throw std::logic_error( + "SUM aggregates can only be requested on numeric types"); + } + }; + type::apply_with_type(g, fi.type_); + } +}; + +class CountOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + /** Default constructor */ + CountOperation() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + // For count operations we have a constant handle, create the aggregator when + // requested so that we get a different object for each query. + [[nodiscard]] shared_ptr aggregator() + const override { + return common::make_shared(HERE()); + } +}; + +} // namespace tiledb::sm + +#endif // TILEDB_OPERATION_H diff --git a/tiledb/sm/query/readers/aggregators/query_channel.h b/tiledb/sm/query/readers/aggregators/query_channel.h new file mode 100644 index 000000000000..62d87c15395f --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/query_channel.h @@ -0,0 +1,90 @@ +/** + * @file query_channel.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class QueryChannel. + */ + +#ifndef TILEDB_QUERY_CHANNEL_H +#define TILEDB_QUERY_CHANNEL_H + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/query/readers/aggregators/iaggregator.h" + +namespace tiledb::sm { + +class QueryChannel { + public: + using ChannelAggregates = + std::unordered_map>; + + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Construct a QueryChannel + * + * @param is_default If true, this is the default query channel + * @param aggregates A map of aggregators by output field name + */ + QueryChannel(bool is_default, const ChannelAggregates& aggregates) + : default_(is_default) + , aggregates_{aggregates} { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * @return true if this is the default query channel + */ + bool is_default() const { + return default_; + } + + /** + * @return the map of aggregator pointers + */ + const ChannelAggregates& aggregates() const { + return aggregates_; + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + bool default_; + ChannelAggregates aggregates_; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_QUERY_CHANNEL_H diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc index 8d349ab57713..53a0384fc8ac 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc @@ -53,18 +53,7 @@ SumAggregator::SumAggregator(const FieldInfo field_info) , validity_value_( field_info_.is_nullable_ ? std::make_optional(0) : nullopt) , sum_overflowed_(false) { - // TODO: These argument validation can be merged for mean/sum and possibly - // other aggregates. (sc-33763). - if (field_info_.var_sized_) { - throw SumAggregatorStatusException( - "Sum aggregates are not supported for var sized attributes."); - } - - if (field_info_.cell_val_num_ != 1) { - throw SumAggregatorStatusException( - "Sum aggregates are not supported for attributes with cell_val_num " - "greater than one."); - } + ensure_field_numeric(field_info_); } template diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.h b/tiledb/sm/query/readers/aggregators/sum_aggregator.h index 9676fe81ef89..0088f50fc389 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.h @@ -45,7 +45,9 @@ namespace tiledb::sm { class QueryBuffer; template -class SumAggregator : public OutputBufferValidator, public IAggregator { +class SumAggregator : public InputFieldValidator, + public OutputBufferValidator, + public IAggregator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -73,10 +75,15 @@ class SumAggregator : public OutputBufferValidator, public IAggregator { } /** Returns if the aggregation is var sized or not. */ - bool var_sized() override { + bool aggregation_var_sized() override { return false; }; + /** Returns if the aggregation is nullable or not. */ + bool aggregation_nullable() override { + return field_info_.is_nullable_; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return true; @@ -109,6 +116,11 @@ class SumAggregator : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; + /** Returns name of the aggregate, e.g. COUNT, MIN, SUM. */ + std::string aggregate_name() override { + return constants::aggregate_sum_str; + } + /** Returns the TileDB datatype of the output field for the aggregate. */ Datatype output_datatype() override { return sum_type_data::tiledb_datatype; diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 0b8f3b94b0f7..354f89056525 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -54,17 +54,15 @@ TEMPLATE_LIST_TEST_CASE( SECTION("Var size") { CHECK_THROWS_WITH( TestType(FieldInfo("a1", true, false, 1)), - Catch::Matchers::EndsWith( - "aggregates are not supported for var sized attributes.") || - Catch::Matchers::EndsWith("aggregates are not supported for var " - "sized non-string attributes.")); + "InputFieldValidator: Aggregate is not supported for var sized " + "non-string fields."); } SECTION("Invalid cell val num") { CHECK_THROWS_WITH( TestType(FieldInfo("a1", false, false, 2)), - Catch::Matchers::EndsWith("aggregates are not supported for attributes " - "with cell_val_num greater than one.")); + "InputFieldValidator: Aggregate is not supported for non-string fields " + "with cell_val_num greater than one."); } } @@ -74,8 +72,8 @@ TEST_CASE( SECTION("Non nullable") { CHECK_THROWS_WITH( NullCountAggregator(FieldInfo("a1", false, false, 1)), - "CountAggregator: NullCount aggregates must only be requested for " - "nullable attributes."); + "InputFieldValidator: Aggregate must only be requested for nullable " + "fields."); } } @@ -99,11 +97,11 @@ typedef tuple< TEMPLATE_LIST_TEST_CASE( "Aggregator: var sized", "[aggregator][var-sized]", AggUnderTest) { auto aggregator{make_aggregator(FieldInfo("a1", false, true, 1))}; - CHECK(aggregator.var_sized() == false); + CHECK(aggregator.aggregation_var_sized() == false); if constexpr (std::is_same>::value) { MinAggregator aggregator2(FieldInfo("a1", true, false, 1)); - CHECK(aggregator2.var_sized() == true); + CHECK(aggregator2.aggregation_var_sized() == true); } } diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 05c43207dca9..44e8dabc6931 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -41,6 +41,7 @@ #include "tiledb/sm/serialization/array.h" #include "tiledb/sm/serialization/array_schema.h" #include "tiledb/sm/serialization/capnp_utils.h" +#include "tiledb/sm/serialization/query_aggregates.h" #endif // clang-format on @@ -1414,6 +1415,17 @@ Status query_to_capnp( buffer_names.insert( buffer_names.end(), dim_label_names.begin(), dim_label_names.end()); + // Add aggregate buffers + const auto aggregate_names = query.aggregate_buffer_names(); + buffer_names.insert( + buffer_names.end(), aggregate_names.begin(), aggregate_names.end()); + + // TODO: add API in query to get all buffers in one go + // Deserialization gets the list of buffers from wire then call + // set_data_buffer which knows on which structure to set. We should have a + // function here as well that gets all buffer names or even better, all + // buffers + uint64_t total_fixed_len_bytes = 0; uint64_t total_var_len_bytes = 0; uint64_t total_validity_len_bytes = 0; @@ -1582,6 +1594,16 @@ Status query_to_capnp( } } + // The server should throw if it's about to serialize an incomplete query + // that has aggregates on it, this behavior is currently not supported. + if (!client_side && query.status() == QueryStatus::INCOMPLETE && + !query.has_aggregates()) { + throw Status_SerializationError( + "Aggregates are not currently supported in incomplete remote " + "queries"); + } + query_channels_to_capnp(query, query_builder); + return Status::Ok(); } @@ -1654,6 +1676,11 @@ Status query_from_capnp( } } + // It's important that deserialization of query channels/aggregates happens + // before deserializing buffers. set_data_buffer won't know whether a buffer + // is aggregate or not if the list of aggregates per channel is not populated. + query_channels_from_capnp(query_reader, query); + const auto& schema = query->array_schema(); // Deserialize and set attribute buffers. if (!query_reader.hasAttributeBufferHeaders()) { @@ -1686,8 +1713,19 @@ Status query_from_capnp( uint8_t* existing_validity_buffer = nullptr; uint64_t existing_validity_buffer_size = 0; - auto var_size = schema.var_size(name); - auto nullable = schema.is_nullable(name); + // TODO: This is yet another instance where there needs to be a common + // mechanism for reporting the common properties of a field. + // Refactor to use query_field_t. + bool var_size = false; + bool nullable = false; + auto aggregate = query->get_aggregate(name); + if (aggregate.has_value()) { + var_size = aggregate.value()->aggregation_var_sized(); + nullable = aggregate.value()->aggregation_nullable(); + } else { + var_size = schema.var_size(name); + nullable = schema.is_nullable(name); + } const QueryBuffer& query_buffer = query->buffer(name); if (type == QueryType::READ) { // We use the query_buffer directly in order to get the original buffer diff --git a/tiledb/sm/serialization/query_aggregates.cc b/tiledb/sm/serialization/query_aggregates.cc new file mode 100644 index 000000000000..036139baadba --- /dev/null +++ b/tiledb/sm/serialization/query_aggregates.cc @@ -0,0 +1,132 @@ +/** + * @file query_aggregates.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares serialization functions for Query aggregates. + */ + +#include "tiledb/sm/serialization/query_aggregates.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/query/readers/aggregators/operation.h" + +namespace tiledb::sm::serialization { + +#ifdef TILEDB_SERIALIZATION + +void query_channels_to_capnp( + Query& query, capnp::Query::Builder* query_builder) { + // Here, don't serialize if we don't have channels or if the default channel + // has no aggregates on it. + auto channels = query.get_channels(); + if (channels.empty() || (channels.size() == 1 && channels[0].is_default() && + channels[0].aggregates().empty())) { + return; + } + auto channels_builder = query_builder->initChannels(channels.size()); + for (const auto& channel : channels) { + auto channel_builder = channels_builder[0]; + channel_builder.setDefault(channel.is_default()); + + auto& aggregates = channel.aggregates(); + if (aggregates.empty()) { + continue; + } + + auto aggregates_builder = channel_builder.initAggregates(aggregates.size()); + size_t i = 0; + for (const auto& agg : aggregates) { + auto aggregate_builder = aggregates_builder[i]; + aggregate_builder.setOutputFieldName(agg.first); + aggregate_builder.setName(agg.second->aggregate_name()); + // TODO: Currently COUNT reports that its field name is + // constants::count_of_rows. This causes deserialization code to crash + // because an input field is always assumed to exist, so deserialization + // calls some array schema functions on it. This should be fixed in a + // followup work item by making the aggregators interface return an + // optional field_name + if (agg.second->aggregate_name() != constants::aggregate_count_str) { + aggregate_builder.setInputFieldName(agg.second->field_name()); + } + ++i; + } + } +} + +void query_channels_from_capnp( + const capnp::Query::Reader& query_reader, Query* const query) { + // We might not have channels if the default channel had no aggregates. + if (query_reader.hasChannels()) { + auto channels_reader = query_reader.getChannels(); + // Only the query default channel is on the wire for now + for (size_t i = 0; i < channels_reader.size(); ++i) { + auto channel_reader = channels_reader[i]; + + if (channel_reader.hasAggregates()) { + QueryChannel::ChannelAggregates aggregates; + + auto aggregates_reader = channel_reader.getAggregates(); + for (const auto& aggregate : aggregates_reader) { + if (aggregate.hasOutputFieldName() && aggregate.hasName()) { + auto output_field = aggregate.getOutputFieldName(); + auto name = aggregate.getName(); + + // TODO: Currently COUNT reports that its field name is + // constants::count_of_rows. This causes deserialization code to + // crash because an input field is always assumed to exist, so + // deserialization calls some array schema functions on it. This + // should be fixed in a followup work item by making the aggregators + // interface return an optional field_name + std::optional fi; + if (aggregate.hasInputFieldName()) { + auto input_field = aggregate.getInputFieldName(); + auto& schema = query->array_schema(); + + fi.emplace(tiledb::sm::FieldInfo( + input_field, + schema.var_size(input_field), + schema.is_nullable(input_field), + schema.cell_val_num(input_field), + schema.type(input_field))); + } + + auto operation = Operation::make_operation(name, fi); + aggregates.emplace(output_field, operation->aggregator()); + } + } + if (!aggregates.empty()) { + query->add_channel( + QueryChannel{channel_reader.getDefault(), aggregates}); + } + } + } + } +} + +#endif // TILEDB_SERIALIZATION + +} // namespace tiledb::sm::serialization \ No newline at end of file diff --git a/tiledb/sm/serialization/query_aggregates.h b/tiledb/sm/serialization/query_aggregates.h new file mode 100644 index 000000000000..774c9439c2d9 --- /dev/null +++ b/tiledb/sm/serialization/query_aggregates.h @@ -0,0 +1,66 @@ +/** + * @file query_aggregates.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares serialization functions for Query aggregates. + */ + +#ifndef TILEDB_SERIALIZATION_QUERY_AGGREGATES_H +#define TILEDB_SERIALIZATION_QUERY_AGGREGATES_H + +#ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/serialization/capnp_utils.h" +#endif + +namespace tiledb::sm::serialization { + +#ifdef TILEDB_SERIALIZATION + +/** + * Convert query channels to Cap'n Proto message + * + * @param query A TileDB query + * @param query_builder cap'n proto class + */ +void query_channels_to_capnp( + Query& query, capnp::Query::Builder* query_builder); + +/** + * Deserialize query channels from Cap'n Proto message + * + * @param query_reader capnp reader + * @param query deserialized TileDB query + */ +void query_channels_from_capnp( + const capnp::Query::Reader& query_reader, Query* const query); + +#endif + +} // namespace tiledb::sm::serialization + +#endif // TILEDB_SERIALIZATION_QUERY_AGGREGATES_H \ No newline at end of file diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index f76b4107e57b..77fb19c6886b 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -772,6 +772,10 @@ struct Query { orderedDimLabelReader @20 :QueryReader; # orderedDimLabelReader contains data needed for dense dimension label reads. + + channels @21 :List(QueryChannel); + # channels contains the list of channels (streams of data) within a read + # query. It always contains at least one element, the default channel. } struct NonEmptyDomain { @@ -1253,3 +1257,30 @@ struct LoadArraySchemaResponse { schema @0 :ArraySchema; # The loaded ArraySchema } + +struct QueryChannel { + # structure representing a query channel, that is a stream of data within + # a TileDB query. Such channels can be generated for the purpose of avoiding + # processing result items multiple times in more complex queries such as e.g. + # grouping queries. + + default @0 :Bool; + # True if a channel is the default query channel + + aggregates @1 :List(Aggregate); + # a list of the aggregate operations applied on this channel +} + +struct Aggregate { + # structure representing a query aggregate operation + + outputFieldName @0 :Text; + # name of the result query buffers + + inputFieldName @1 :Text; + # name of the input field the aggregate is applied on + + name @2 :Text; + # the name of aggregate, e.g. COUNT, MEAN, SUM used for constructing the + # correct object during deserialization +} diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 2a5a77c3f800..5e6afe2644a1 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -5397,171 +5397,178 @@ const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<367> b_96ba49d0f8b23ccc = { +static const ::capnp::_::AlignedData<387> b_96ba49d0f8b23ccc = { { 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, 150, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 16, 0, 7, 0, 0, 0, 0, 0, + 17, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 159, 4, 0, 0, + 25, 0, 0, 0, 215, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 84, 0, 0, 0, 3, 0, 4, 0, + 88, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 2, 0, 0, 186, 0, 0, 0, + 89, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 64, 2, 0, 0, 3, 0, 1, 0, - 92, 2, 0, 0, 2, 0, 1, 0, + 92, 2, 0, 0, 3, 0, 1, 0, + 120, 2, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 2, 0, 0, 58, 0, 0, 0, + 117, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 84, 2, 0, 0, 3, 0, 1, 0, - 96, 2, 0, 0, 2, 0, 1, 0, + 112, 2, 0, 0, 3, 0, 1, 0, + 124, 2, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 93, 2, 0, 0, 58, 0, 0, 0, + 121, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 88, 2, 0, 0, 3, 0, 1, 0, - 100, 2, 0, 0, 2, 0, 1, 0, + 116, 2, 0, 0, 3, 0, 1, 0, + 128, 2, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 2, 0, 0, 42, 0, 0, 0, + 125, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 92, 2, 0, 0, 3, 0, 1, 0, - 104, 2, 0, 0, 2, 0, 1, 0, + 120, 2, 0, 0, 3, 0, 1, 0, + 132, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 101, 2, 0, 0, 58, 0, 0, 0, + 129, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 2, 0, 0, 3, 0, 1, 0, - 108, 2, 0, 0, 2, 0, 1, 0, + 124, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 105, 2, 0, 0, 58, 0, 0, 0, + 133, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 100, 2, 0, 0, 3, 0, 1, 0, - 112, 2, 0, 0, 2, 0, 1, 0, + 128, 2, 0, 0, 3, 0, 1, 0, + 140, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 109, 2, 0, 0, 50, 0, 0, 0, + 137, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 104, 2, 0, 0, 3, 0, 1, 0, - 116, 2, 0, 0, 2, 0, 1, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 144, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 113, 2, 0, 0, 226, 0, 0, 0, + 141, 2, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 2, 0, 0, 3, 0, 1, 0, - 132, 2, 0, 0, 2, 0, 1, 0, + 148, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 2, 0, 0, 186, 0, 0, 0, + 157, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 132, 2, 0, 0, 3, 0, 1, 0, - 144, 2, 0, 0, 2, 0, 1, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 172, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 2, 0, 0, 202, 0, 0, 0, + 169, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 2, 0, 0, 3, 0, 1, 0, - 160, 2, 0, 0, 2, 0, 1, 0, + 176, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 2, 0, 0, 122, 0, 0, 0, + 185, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 2, 0, 0, 3, 0, 1, 0, - 168, 2, 0, 0, 2, 0, 1, 0, + 184, 2, 0, 0, 3, 0, 1, 0, + 196, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 192, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 165, 2, 0, 0, 210, 0, 0, 0, + 193, 2, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 172, 2, 0, 0, 3, 0, 1, 0, - 184, 2, 0, 0, 2, 0, 1, 0, + 200, 2, 0, 0, 3, 0, 1, 0, + 212, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 2, 0, 0, 146, 0, 0, 0, + 209, 2, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 184, 2, 0, 0, 3, 0, 1, 0, - 196, 2, 0, 0, 2, 0, 1, 0, + 212, 2, 0, 0, 3, 0, 1, 0, + 224, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 193, 2, 0, 0, 58, 0, 0, 0, + 221, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 2, 0, 0, 3, 0, 1, 0, - 200, 2, 0, 0, 2, 0, 1, 0, + 216, 2, 0, 0, 3, 0, 1, 0, + 228, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 2, 0, 0, 50, 0, 0, 0, + 225, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 192, 2, 0, 0, 3, 0, 1, 0, - 204, 2, 0, 0, 2, 0, 1, 0, + 220, 2, 0, 0, 3, 0, 1, 0, + 232, 2, 0, 0, 2, 0, 1, 0, 15, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 2, 0, 0, 98, 0, 0, 0, + 229, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 2, 0, 0, 3, 0, 1, 0, - 212, 2, 0, 0, 2, 0, 1, 0, + 228, 2, 0, 0, 3, 0, 1, 0, + 240, 2, 0, 0, 2, 0, 1, 0, 16, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 2, 0, 0, 98, 0, 0, 0, + 237, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 2, 0, 0, 3, 0, 1, 0, - 220, 2, 0, 0, 2, 0, 1, 0, + 236, 2, 0, 0, 3, 0, 1, 0, + 248, 2, 0, 0, 2, 0, 1, 0, 17, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 2, 0, 0, 58, 0, 0, 0, + 245, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 2, 0, 0, 3, 0, 1, 0, - 224, 2, 0, 0, 2, 0, 1, 0, + 240, 2, 0, 0, 3, 0, 1, 0, + 252, 2, 0, 0, 2, 0, 1, 0, 18, 0, 0, 0, 13, 0, 0, 0, 0, 0, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 2, 0, 0, 162, 0, 0, 0, + 249, 2, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 2, 0, 0, 3, 0, 1, 0, - 252, 2, 0, 0, 2, 0, 1, 0, + 252, 2, 0, 0, 3, 0, 1, 0, + 24, 3, 0, 0, 2, 0, 1, 0, 19, 0, 0, 0, 14, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 2, 0, 0, 122, 0, 0, 0, + 21, 3, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 248, 2, 0, 0, 3, 0, 1, 0, - 20, 3, 0, 0, 2, 0, 1, 0, + 20, 3, 0, 0, 3, 0, 1, 0, + 48, 3, 0, 0, 2, 0, 1, 0, 20, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 3, 0, 0, 178, 0, 0, 0, + 45, 3, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 3, 0, 0, 3, 0, 1, 0, - 32, 3, 0, 0, 2, 0, 1, 0, + 48, 3, 0, 0, 3, 0, 1, 0, + 60, 3, 0, 0, 2, 0, 1, 0, + 21, 0, 0, 0, 16, 0, 0, 0, + 0, 0, 1, 0, 21, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 57, 3, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 56, 3, 0, 0, 3, 0, 1, 0, + 84, 3, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 66, 117, 102, 102, 101, 114, 72, 101, 97, 100, 101, 114, 115, 0, 0, @@ -5763,6 +5770,19 @@ static const ::capnp::_::AlignedData<367> b_96ba49d0f8b23ccc = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 104, 97, 110, 110, 101, 108, 115, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 217, 229, 74, 254, 11, 77, 45, 202, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -5775,15 +5795,16 @@ static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { &s_a736c51d292ca752, &s_b6c95b4b8111ad36, &s_c7e036a11506a1a4, + &s_ca2d4d0bfe4ae5d9, &s_d20a578112fa92a2, &s_d74f5fed155d316c, &s_e19754f813ccf79c, }; -static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; -static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}; +static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 21, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; +static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 367, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, - 9, 21, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr }, true + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 387, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 10, 22, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { @@ -9733,6 +9754,149 @@ const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<53> b_ca2d4d0bfe4ae5d9 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 217, 229, 74, 254, 11, 77, 45, 202, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 250, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 81, 117, 101, 114, 121, 67, + 104, 97, 110, 110, 101, 108, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 72, 0, 0, 0, 2, 0, 1, 0, + 100, 101, 102, 97, 117, 108, 116, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 103, 103, 114, 101, 103, 97, 116, + 101, 115, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 46, 16, 159, 90, 244, 241, 147, 225, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_ca2d4d0bfe4ae5d9 = b_ca2d4d0bfe4ae5d9.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_ca2d4d0bfe4ae5d9[] = { + &s_e193f1f45a9f102e, +}; +static const uint16_t m_ca2d4d0bfe4ae5d9[] = {1, 0}; +static const uint16_t i_ca2d4d0bfe4ae5d9[] = {0, 1}; +const ::capnp::_::RawSchema s_ca2d4d0bfe4ae5d9 = { + 0xca2d4d0bfe4ae5d9, b_ca2d4d0bfe4ae5d9.words, 53, d_ca2d4d0bfe4ae5d9, m_ca2d4d0bfe4ae5d9, + 1, 2, i_ca2d4d0bfe4ae5d9, nullptr, nullptr, { &s_ca2d4d0bfe4ae5d9, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<65> b_e193f1f45a9f102e = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 46, 16, 159, 90, 244, 241, 147, 225, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 3, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 65, 103, 103, 114, 101, 103, + 97, 116, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 12, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 69, 0, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 68, 0, 0, 0, 3, 0, 1, 0, + 80, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 77, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 76, 0, 0, 0, 3, 0, 1, 0, + 88, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 85, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 80, 0, 0, 0, 3, 0, 1, 0, + 92, 0, 0, 0, 2, 0, 1, 0, + 111, 117, 116, 112, 117, 116, 70, 105, + 101, 108, 100, 78, 97, 109, 101, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 105, 110, 112, 117, 116, 70, 105, 101, + 108, 100, 78, 97, 109, 101, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_e193f1f45a9f102e = b_e193f1f45a9f102e.words; +#if !CAPNP_LITE +static const uint16_t m_e193f1f45a9f102e[] = {1, 2, 0}; +static const uint16_t i_e193f1f45a9f102e[] = {0, 1, 2}; +const ::capnp::_::RawSchema s_e193f1f45a9f102e = { + 0xe193f1f45a9f102e, b_e193f1f45a9f102e.words, 65, nullptr, m_e193f1f45a9f102e, + 0, 3, i_e193f1f45a9f102e, nullptr, nullptr, { &s_e193f1f45a9f102e, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE } // namespace schemas } // namespace capnp @@ -10787,6 +10951,30 @@ constexpr ::capnp::_::RawSchema const* LoadArraySchemaResponse::_capnpPrivate::s #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// QueryChannel +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t QueryChannel::_capnpPrivate::dataWordSize; +constexpr uint16_t QueryChannel::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind QueryChannel::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* QueryChannel::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// Aggregate +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t Aggregate::_capnpPrivate::dataWordSize; +constexpr uint16_t Aggregate::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind Aggregate::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Aggregate::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + } // namespace } // namespace diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 0e68ba5f8446..f4f8b9694c14 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -108,6 +108,8 @@ CAPNP_DECLARE_SCHEMA(891a70a671f15cf6); CAPNP_DECLARE_SCHEMA(805c080c10c1e959); CAPNP_DECLARE_SCHEMA(83f094010132ff21); CAPNP_DECLARE_SCHEMA(ebe17f59ac9a1df1); +CAPNP_DECLARE_SCHEMA(ca2d4d0bfe4ae5d9); +CAPNP_DECLARE_SCHEMA(e193f1f45a9f102e); } // namespace schemas } // namespace capnp @@ -1004,7 +1006,7 @@ struct Query { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 16) + CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 17) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -1685,6 +1687,40 @@ struct LoadArraySchemaResponse { }; }; +struct QueryChannel { + QueryChannel() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(ca2d4d0bfe4ae5d9, 1, 1) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Aggregate { + Aggregate() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(e193f1f45a9f102e, 0, 3) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + // ======================================================================================= class DomainArray::Reader { @@ -8909,6 +8945,12 @@ class Query::Reader { inline ::tiledb::sm::serialization::capnp::QueryReader::Reader getOrderedDimLabelReader() const; + inline bool hasChannels() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Reader + getChannels() const; + private: ::capnp::_::StructReader _reader; template @@ -9144,6 +9186,27 @@ class Query::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::QueryReader> disownOrderedDimLabelReader(); + inline bool hasChannels(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Builder + getChannels(); + inline void setChannels(::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Builder + initChannels(unsigned int size); + inline void adoptChannels( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>> + disownChannels(); + private: ::capnp::_::StructBuilder _builder; template @@ -14843,6 +14906,241 @@ class LoadArraySchemaResponse::Pipeline { }; #endif // !CAPNP_LITE +class QueryChannel::Reader { + public: + typedef QueryChannel Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool getDefault() const; + + inline bool hasAggregates() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Reader + getAggregates() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class QueryChannel::Builder { + public: + typedef QueryChannel Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool getDefault(); + inline void setDefault(bool value); + + inline bool hasAggregates(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Builder + getAggregates(); + inline void setAggregates(::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Builder + initAggregates(unsigned int size); + inline void adoptAggregates(::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>> + disownAggregates(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class QueryChannel::Pipeline { + public: + typedef QueryChannel Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class Aggregate::Reader { + public: + typedef Aggregate Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasOutputFieldName() const; + inline ::capnp::Text::Reader getOutputFieldName() const; + + inline bool hasInputFieldName() const; + inline ::capnp::Text::Reader getInputFieldName() const; + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Aggregate::Builder { + public: + typedef Aggregate Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasOutputFieldName(); + inline ::capnp::Text::Builder getOutputFieldName(); + inline void setOutputFieldName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initOutputFieldName(unsigned int size); + inline void adoptOutputFieldName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownOutputFieldName(); + + inline bool hasInputFieldName(); + inline ::capnp::Text::Builder getInputFieldName(); + inline void setInputFieldName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initInputFieldName(unsigned int size); + inline void adoptInputFieldName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownInputFieldName(); + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Aggregate::Pipeline { + public: + typedef Aggregate Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + // ======================================================================================= inline bool DomainArray::Reader::hasInt8() const { @@ -24856,6 +25154,80 @@ Query::Builder::disownOrderedDimLabelReader() { _builder.getPointerField(::capnp::bounded<15>() * ::capnp::POINTERS)); } +inline bool Query::Reader::hasChannels() const { + return !_reader.getPointerField(::capnp::bounded<16>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Query::Builder::hasChannels() { + return !_builder.getPointerField(::capnp::bounded<16>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Reader +Query::Reader::getChannels() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<16>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Builder +Query::Builder::getChannels() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<16>() * + ::capnp::POINTERS)); +} +inline void Query::Builder::setChannels( + ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<16>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>::Builder +Query::Builder::initChannels(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<16>() * ::capnp::POINTERS), + size); +} +inline void Query::Builder::adoptChannels( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<16>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>> +Query::Builder::disownChannels() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::QueryChannel, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<16>() * + ::capnp::POINTERS)); +} + inline bool NonEmptyDomain::Reader::hasNonEmptyDomain() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -31587,6 +31959,207 @@ LoadArraySchemaResponse::Builder::disownSchema() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool QueryChannel::Reader::getDefault() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool QueryChannel::Builder::getDefault() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void QueryChannel::Builder::setDefault(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool QueryChannel::Reader::hasAggregates() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryChannel::Builder::hasAggregates() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Reader +QueryChannel::Reader::getAggregates() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Builder +QueryChannel::Builder::getAggregates() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} +inline void QueryChannel::Builder::setAggregates( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>::Builder +QueryChannel::Builder::initAggregates(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void QueryChannel::Builder::adoptAggregates( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>> +QueryChannel::Builder::disownAggregates() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Aggregate, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} + +inline bool Aggregate::Reader::hasOutputFieldName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Aggregate::Builder::hasOutputFieldName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Aggregate::Reader::getOutputFieldName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Aggregate::Builder::getOutputFieldName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Aggregate::Builder::setOutputFieldName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Aggregate::Builder::initOutputFieldName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Aggregate::Builder::adoptOutputFieldName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Aggregate::Builder::disownOutputFieldName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Aggregate::Reader::hasInputFieldName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Aggregate::Builder::hasInputFieldName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Aggregate::Reader::getInputFieldName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Aggregate::Builder::getInputFieldName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Aggregate::Builder::setInputFieldName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Aggregate::Builder::initInputFieldName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Aggregate::Builder::adoptInputFieldName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Aggregate::Builder::disownInputFieldName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Aggregate::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Aggregate::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Aggregate::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Aggregate::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Aggregate::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Aggregate::Builder::initName(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Aggregate::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Aggregate::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + } // namespace capnp } // namespace serialization } // namespace sm From 595e40921c942ee6f9e10f384a837d7edf59c06d Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Sat, 21 Oct 2023 05:47:51 -0500 Subject: [PATCH 016/456] Allow for empty enumerations (#4423) This change allows users to create an empty enumeration when generating a schema which is then extended with schema evolution during the write phase. --- TYPE: IMPROVEMENT DESC: Allow for empty enumerations --- test/src/unit-cppapi-enumerations.cc | 50 ++++- test/src/unit-enumerations.cc | 212 +++++++++++++++++++ tiledb/sm/array_schema/enumeration.cc | 115 +++++++--- tiledb/sm/cpp_api/enumeration_experimental.h | 43 +++- 4 files changed, 383 insertions(+), 37 deletions(-) diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc index c9c7ff684888..06cb2f234b1a 100644 --- a/test/src/unit-cppapi-enumerations.cc +++ b/test/src/unit-cppapi-enumerations.cc @@ -47,7 +47,7 @@ struct CPPEnumerationFx { template void check_dump(const T& val); - void create_array(); + void create_array(bool with_empty_enumeration = false); void rm_array(); std::string uri_; @@ -394,7 +394,7 @@ TEST_CASE_METHOD( // Attempt to query with an enumeration value that isn't in the Enumeration QueryCondition qc(ctx_); - qc.init("attr1", "alf", 4, TILEDB_EQ); + qc.init("attr1", "alf", 3, TILEDB_EQ); // Execute the query condition against the array std::vector dim(5); @@ -435,6 +435,34 @@ TEST_CASE_METHOD( REQUIRE(rc != TILEDB_OK); } +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Attempt to query on empty enumeration", + "[enumeration][query][error]") { + create_array(true); + + // Attempt to query with an enumeration value that isn't in the Enumeration + QueryCondition qc(ctx_); + qc.init("attr3", "alf", 3, TILEDB_EQ); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr3(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr3", attr3) + .set_condition(qc); + + // Check that the error message is helpful to users. + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration value not found for field 'attr3'"); + REQUIRE_THROWS_WITH(query.submit(), matcher); +} + CPPEnumerationFx::CPPEnumerationFx() : uri_("enumeration_test_array") , vfs_(ctx_) { @@ -470,7 +498,7 @@ void CPPEnumerationFx::check_dump(const T& val) { vfs_.remove_file(dump_name); } -void CPPEnumerationFx::create_array() { +void CPPEnumerationFx::create_array(bool with_empty_enumeration) { // Create a simple array for testing. This ends up with just five elements in // the array. dim is an int32_t dimension, attr1 is an enumeration with string // values and int32_t attribute values. attr2 is a float attribute. @@ -500,11 +528,22 @@ void CPPEnumerationFx::create_array() { auto attr2 = Attribute::create(ctx_, "attr2"); schema.add_attribute(attr2); + if (with_empty_enumeration) { + auto empty_enmr = Enumeration::create_empty( + ctx_, "empty_enmr", TILEDB_STRING_ASCII, TILEDB_VAR_NUM); + ArraySchemaExperimental::add_enumeration(ctx_, schema, empty_enmr); + + auto attr3 = Attribute::create(ctx_, "attr3"); + AttributeExperimental::set_enumeration_name(ctx_, attr3, "empty_enmr"); + schema.add_attribute(attr3); + } + Array::create(uri_, schema); // Attribute data std::vector attr1_values = {0, 1, 2, 1, 0}; std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + std::vector attr3_values = {0, 0, 0, 0, 0}; Array array(ctx_, uri_, TILEDB_WRITE); Subarray subarray(ctx_, array); @@ -514,6 +553,11 @@ void CPPEnumerationFx::create_array() { .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("attr1", attr1_values) .set_data_buffer("attr2", attr2_values); + + if (with_empty_enumeration) { + query.set_data_buffer("attr3", attr3_values); + } + CHECK_NOTHROW(query.submit()); query.finalize(); array.close(); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 8b86c1c7abda..8d4fd2d3cb02 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -143,6 +143,27 @@ QueryCondition create_qc( /* Testing Enumeration */ /* ********************************* */ +TEST_CASE_METHOD( + EnumerationFx, "Create Empty Enumeration", "[enumeration][empty]") { + Enumeration::create( + default_enmr_name, Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Create Empty Var Sized Enumeration", + "[enumeration][empty]") { + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + nullptr, + 0, + nullptr, + 0); +} + TEST_CASE_METHOD( EnumerationFx, "Basic Boolean Enumeration Creation", @@ -192,6 +213,60 @@ TEST_CASE_METHOD( false); } +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Single Empty Value Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {""}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Single Empty Value Using nullptr", + "[enumeration][error][invalid-offsets-args]") { + uint64_t offsets = 0; + auto enmr = Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + nullptr, + 0, + &offsets, + sizeof(uint64_t)); + std::vector values = {""}; + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Last Value Empty Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"last", "value", "is", ""}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + TEST_CASE_METHOD( EnumerationFx, "Enumeration Creation with Ordered", @@ -239,6 +314,143 @@ TEST_CASE_METHOD( check_enumeration(enmr, default_enmr_name, values, Datatype::INT32, 2, false); } +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Non-zero size for data nullptr", + "[enumeration][error][invalid-data-args]") { + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data buffer must not be nullptr for fixed sized data."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + 10, + nullptr, + 0), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Zero size for data non-nullptr", + "[enumeration][error][invalid-data-args]") { + int val = 5; + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data size; must be non-zero for fixed size data."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, Datatype::INT32, 1, false, &val, 0, nullptr, 0), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Non-zero size for offsets nullptr", + "[enumeration][error][invalid-offsets-args]") { + const char* val = "foo"; + auto matcher = Catch::Matchers::ContainsSubstring( + "Var sized enumeration values require a non-null offsets pointer."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + val, + strlen(val), + nullptr, + 8), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Zero size for offsets non-nullptr", + "[enumeration][error][invalid-offsets-args]") { + const char* val = "foo"; + uint64_t offset = 0; + auto matcher = Catch::Matchers::ContainsSubstring( + "Var sized enumeration values require a non-zero offsets size."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + val, + strlen(val), + &offset, + 0), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Single Empty String Invalid Data Size", + "[enumeration][error][invalid-offsets-args]") { + uint64_t offsets = 0; + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data buffer; must not be nullptr when data_size " + "is non-zero."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + nullptr, + 5, + &offsets, + sizeof(uint64_t)), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Missing Var Data", + "[enumeration][error][invalid-data-args]") { + uint64_t offsets = 5; + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data input, nullptr provided when the provided offsets " + "require data."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + nullptr, + 5, + &offsets, + sizeof(uint64_t)), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid Data Size", + "[enumeration][error][invalid-data-args]") { + uint64_t offsets = 5; + const char* data = "meow"; + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data input, data_size is smaller than the last provided " + "offset."); + REQUIRE_THROWS_WITH( + Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + 2, + &offsets, + sizeof(uint64_t)), + matcher); +} + TEST_CASE_METHOD( EnumerationFx, "Enumeration Creation Error - Invalid empty name - std::string", diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index 4f92de4a6825..dbd527b34fcc 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -86,30 +86,86 @@ Enumeration::Enumeration( throw EnumerationException("Invalid cell_val_num in Enumeration"); } - if (data == nullptr || data_size == 0) { - throw EnumerationException("No attribute value data supplied."); - } - - if (var_size() && (offsets == nullptr || offsets_size == 0)) { - throw EnumerationException( - "Variable length datatype defined but offsets are not present"); - } else if (!var_size() && (offsets != nullptr || offsets_size > 0)) { - throw EnumerationException( - "Fixed length datatype defined but offsets are present"); + // Check if we're creating an empty enumeration and bail. + auto data_empty = (data == nullptr && data_size == 0); + auto offsets_empty = (offsets == nullptr && offsets_size == 0); + if (data_empty && offsets_empty) { + // This is an empty enumeration so we're done checking for argument + // validity. + return; } if (var_size()) { - if (offsets_size % sizeof(uint64_t) != 0) { + if (offsets == nullptr) { + throw EnumerationException( + "Var sized enumeration values require a non-null offsets pointer."); + } + + if (offsets_size == 0) { + throw EnumerationException( + "Var sized enumeration values require a non-zero offsets size."); + } + + if (offsets_size % constants::cell_var_offset_size != 0) { throw EnumerationException( "Invalid offsets size is not a multiple of sizeof(uint64_t)"); } + + // Setup some temporary aliases for quick reference auto offset_values = static_cast(offsets); - uint64_t last_offset = (offsets_size / sizeof(uint64_t)) - 1; - if (offset_values[last_offset] > data_size) { + auto num_offsets = offsets_size / constants::cell_var_offset_size; + + // Check for the edge case of a single value so we can handle the case of + // having a single empty value. + if (num_offsets == 1 && offset_values[0] == 0) { + // If data is nullptr and data_size > 0, then the user appears to have + // intended to provided us with a non-empty value. + if (data_size > 0 && data == nullptr) { + throw EnumerationException( + "Invalid data buffer; must not be nullptr when data_size " + "is non-zero."); + } + // Else, data_size is zero and we don't care what data is. We ignore the + // check for data_size == 0 && data != nullptr here because a common + // use case with our APIs is to use a std::string to contain all of the + // var data which AFAIK never returns nullptr. + } else { + // We have more than one string which requires a non-nullptr data and + // non-zero data size that is greater than or equal to the last + // offset provided. + if (data == nullptr) { + throw EnumerationException( + "Invalid data input, nullptr provided when the provided offsets " + "require data."); + } + + if (data_size < offset_values[num_offsets - 1]) { + throw EnumerationException( + "Invalid data input, data_size is smaller than the last provided " + "offset."); + } + } + } else { // !var_sized() + if (offsets != nullptr) { throw EnumerationException( - "Provided data buffer size is too small for the provided offsets."); + "Fixed length value type defined but offsets is not nullptr."); } - } else { + + if (offsets_size != 0) { + throw EnumerationException( + "Fixed length value type defined but offsets size is non-zero."); + } + + if (data == nullptr) { + throw EnumerationException( + "Invalid data buffer must not be nullptr for fixed sized data."); + } + + if (data_size == 0) { + throw EnumerationException( + "Invalid data size; must be non-zero for fixed size data."); + } + if (data_size % cell_size() != 0) { throw EnumerationException( "Invalid data size is not a multiple of the cell size."); @@ -117,11 +173,7 @@ Enumeration::Enumeration( } throw_if_not_ok(data_.write(data, 0, data_size)); - - if (offsets_size > 0) { - throw_if_not_ok(offsets_.write(offsets, 0, offsets_size)); - } - + throw_if_not_ok(offsets_.write(offsets, 0, offsets_size)); generate_value_map(); } @@ -147,14 +199,20 @@ shared_ptr Enumeration::deserialize( auto ordered = deserializer.read(); auto data_size = deserializer.read(); - const void* data = deserializer.get_ptr(data_size); + const void* data = nullptr; + + if (data_size > 0) { + data = deserializer.get_ptr(data_size); + } uint64_t offsets_size = 0; const void* offsets = nullptr; if (cell_val_num == constants::var_num) { offsets_size = deserializer.read(); - offsets = deserializer.get_ptr(offsets_size); + if (offsets_size > 0) { + offsets = deserializer.get_ptr(offsets_size); + } } return create( @@ -184,11 +242,15 @@ void Enumeration::serialize(Serializer& serializer) const { serializer.write(cell_val_num_); serializer.write(ordered_); serializer.write(data_.size()); - serializer.write(data_.data(), data_.size()); + if (data_.size() > 0) { + serializer.write(data_.data(), data_.size()); + } if (var_size()) { serializer.write(offsets_.size()); - serializer.write(offsets_.data(), offsets_.size()); + if (offsets_.size() > 0) { + serializer.write(offsets_.data(), offsets_.size()); + } } else { assert(cell_val_num_ < constants::var_num); assert(offsets_.size() == 0); @@ -221,6 +283,11 @@ void Enumeration::dump(FILE* out) const { } void Enumeration::generate_value_map() { + // If we've got no data, there are no values to generate. + if (data_.size() == 0) { + return; + } + auto char_data = data_.data_as(); if (var_size()) { auto offsets = offsets_.data_as(); diff --git a/tiledb/sm/cpp_api/enumeration_experimental.h b/tiledb/sm/cpp_api/enumeration_experimental.h index ca9c37820a0a..3bbd76b44180 100644 --- a/tiledb/sm/cpp_api/enumeration_experimental.h +++ b/tiledb/sm/cpp_api/enumeration_experimental.h @@ -223,15 +223,37 @@ class Enumeration { /* STATIC FUNCTIONS */ /* ********************************* */ + /** + * Create an empty enumeration. + * + * @param ctx The context to use. + * @param name The name of the enumeration. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. + * @param cell_val_num The number of values per cell. + * @param ordered Whether or not to consider this enumeration ordered. + * @return Enumeration The newly constructed enumeration. + */ + static Enumeration create_empty( + const Context& ctx, + const std::string& name, + tiledb_datatype_t type, + uint32_t cell_val_num, + bool ordered = false) { + return create( + ctx, name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0); + } + /** * Create an enumeration from a vector of trivial values (i.e., int's or other * integral or floating point values) * * @param ctx The context to use. - * @param values The list of values to use for this enumeration. + * @param name The name of the enumeration. + * @param values A vector of enumeration values * @param ordered Whether or not to consider this enumeration ordered. - * @param type The datatype of the enumeration values. This is automatically - * deduced if not provided. + * @param type A specific type if you want to override the default. + * @return Enumeration The newly constructed enumeration. */ template * = nullptr> static Enumeration create( @@ -279,13 +301,13 @@ class Enumeration { * Create an enumeration from a vector of strings * * @param ctx The context to use. - * @param values The vector of values for the enumeration. - * @param ordered Whether to consider the enumerationv alues as ordered. - * @param type The datatype of the enumeration values. This is automatically - * deduced if not provided. However, this can be used to override the - * deduced type if need be. For instance, TILEDB_STRING_ASCII is the - * default type for strings but TILEDB_STRING_UTF8 can be specified. + * @param name The name of the enumeration. + * @param values A vector of enumeration values + * @param ordered Whether or not to consider this enumeration ordered. + * @param type A specific type if you want to override the default. + * @return Enumeration The newly constructed enumeration. */ + template * = nullptr> static Enumeration create( const Context& ctx, @@ -330,8 +352,9 @@ class Enumeration { * Create an enumeration * * @param ctx The context to use. + * @param name The name of the enumeration. * @param type The datatype of the enumeration values. - * @param cell_val_num The cell_val_num of the enumeration. + * @param cell_val_num The number of values per cell of the values. * @param ordered Whether this enumeration should be considered ordered. * @param data A pointer to a buffer of values for this enumeration. * @param data_size The size of the buffer pointed to by data. From d3c3e5f6c3b53b9c9153861e56db7363804da5cc Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:40:34 +0200 Subject: [PATCH 017/456] Aggregates: combine sum/mean aggregators. (#4440) This combines the sum and mean aggregators into base class. --- TYPE: IMPROVEMENT DESC: Aggregates: combine sum/mean aggregators. --- test/src/test-cppapi-aggregates.cc | 1 - tiledb/CMakeLists.txt | 1 - tiledb/sm/query/query.h | 8 - .../query/readers/aggregators/CMakeLists.txt | 2 +- .../aggregators/aggregate_with_count.h | 2 - .../readers/aggregators/mean_aggregator.cc | 153 ----------------- .../readers/aggregators/mean_aggregator.h | 157 ------------------ .../sm/query/readers/aggregators/safe_sum.cc | 2 - .../readers/aggregators/sum_aggregator.cc | 92 ++++++++-- .../readers/aggregators/sum_aggregator.h | 115 ++++++++++--- .../sm/query/readers/aggregators/sum_type.h | 2 - .../test/compile_aggregators_main.cc | 1 - .../aggregators/test/unit_aggregators.cc | 1 - 13 files changed, 170 insertions(+), 367 deletions(-) delete mode 100644 tiledb/sm/query/readers/aggregators/mean_aggregator.cc delete mode 100644 tiledb/sm/query/readers/aggregators/mean_aggregator.h diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 9be386a0d547..909320ffd265 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -35,7 +35,6 @@ #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/query/readers/aggregators/count_aggregator.h" -#include "tiledb/sm/query/readers/aggregators/mean_aggregator.h" #include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" #include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 07acc6d27cfc..641d23810d9b 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -253,7 +253,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query_condition.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/query_remote_buffer_storage.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/count_aggregator.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/mean_aggregator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/operation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/readers/aggregators/output_buffer_validator.cc diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 49fb0d9f7e37..54759f9d8c70 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -743,14 +743,6 @@ class Query { default_channel_aggregates_.emplace(output_field_name, aggregator); } - /** - * Get all aggregates from the default channel. - */ - std::unordered_map> - get_default_channel_aggregates() { - return default_channel_aggregates_; - } - /** Returns an aggregate based on the output field. */ std::optional> get_aggregate( std::string output_field_name) const; diff --git a/tiledb/sm/query/readers/aggregators/CMakeLists.txt b/tiledb/sm/query/readers/aggregators/CMakeLists.txt index d4df7081b37b..291f96696252 100644 --- a/tiledb/sm/query/readers/aggregators/CMakeLists.txt +++ b/tiledb/sm/query/readers/aggregators/CMakeLists.txt @@ -31,7 +31,7 @@ include(object_library) # `aggregators` object library # commence(object_library aggregators) - this_target_sources(count_aggregator.cc mean_aggregator.cc min_max_aggregator.cc operation.cc output_buffer_validator.cc safe_sum.cc sum_aggregator.cc) + this_target_sources(count_aggregator.cc min_max_aggregator.cc operation.cc output_buffer_validator.cc safe_sum.cc sum_aggregator.cc) this_target_object_libraries(baseline array_schema) conclude(object_library) diff --git a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h index 082d969767df..70ed41adccd5 100644 --- a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h +++ b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h @@ -28,8 +28,6 @@ * @section DESCRIPTION * * This file defines class AggregateWithCount. - * - * TODO: Add more benchmark coverage for this class (sc-33758). */ #ifndef TILEDB_AGGREGATE_WITH_COUNT_H diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.cc b/tiledb/sm/query/readers/aggregators/mean_aggregator.cc deleted file mode 100644 index 4f4d845630b3..000000000000 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.cc +++ /dev/null @@ -1,153 +0,0 @@ -/** - * @file mean_aggregator.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file implements class MeanAggregator. - */ - -#include "tiledb/sm/query/readers/aggregators/mean_aggregator.h" - -#include "tiledb/sm/query/query_buffer.h" -#include "tiledb/sm/query/readers/aggregators/aggregate_buffer.h" - -namespace tiledb::sm { - -class MeanAggregatorStatusException : public StatusException { - public: - explicit MeanAggregatorStatusException(const std::string& message) - : StatusException("MeanAggregator", message) { - } -}; - -template -MeanAggregator::MeanAggregator(const FieldInfo field_info) - : OutputBufferValidator(field_info) - , field_info_(field_info) - , aggregate_with_count_(field_info) - , sum_(0) - , count_(0) - , validity_value_( - field_info_.is_nullable_ ? std::make_optional(0) : nullopt) - , sum_overflowed_(false) { - ensure_field_numeric(field_info_); -} - -template -void MeanAggregator::validate_output_buffer( - std::string output_field_name, - std::unordered_map& buffers) { - if (buffers.count(output_field_name) == 0) { - throw MeanAggregatorStatusException("Result buffer doesn't exist."); - } - - ensure_output_buffer_arithmetic(buffers[output_field_name]); -} - -template -void MeanAggregator::aggregate_data(AggregateBuffer& input_data) { - // TODO: While this could be shared with the sum implementation, it isn't - // because it will be improved soon... Means should always be able to be - // computed with no overflows. The simple formula is 2*(a/2 + b/2) + a%2 + - // b%2, but we need benchmarks to see what the performance hit would be to run - // this more complex formula (sc-32789). - - // Return if a previous aggregation has overflowed. - if (sum_overflowed_) { - return; - } - - try { - tuple::sum_type, uint64_t> res; - - // TODO: This is duplicated across aggregates but will go away with - // sc-33104. - if (input_data.is_count_bitmap()) { - res = aggregate_with_count_.template aggregate(input_data); - } else { - res = aggregate_with_count_.template aggregate(input_data); - } - - const auto value = std::get<0>(res); - const auto count = std::get<1>(res); - SafeSum().safe_sum(value, sum_); - count_ += count; - - // Here we know that if the count is greater than 0, it means at least one - // valid item was found, which means the result is valid. - if (field_info_.is_nullable_ && count) { - validity_value_ = 1; - } - } catch (std::overflow_error& e) { - sum_overflowed_ = true; - } -} - -template -void MeanAggregator::copy_to_user_buffer( - std::string output_field_name, - std::unordered_map& buffers) { - auto& result_buffer = buffers[output_field_name]; - auto s = static_cast(result_buffer.buffer_); - - if (sum_overflowed_) { - *s = std::numeric_limits::max(); - } else { - *s = static_cast(sum_) / count_; - } - - if (result_buffer.buffer_size_) { - *result_buffer.buffer_size_ = sizeof(double); - } - - if (field_info_.is_nullable_) { - auto v = static_cast(result_buffer.validity_vector_.buffer()); - if (sum_overflowed_) { - *v = 0; - } else { - *v = validity_value_.value(); - } - - if (result_buffer.validity_vector_.buffer_size()) { - *result_buffer.validity_vector_.buffer_size() = 1; - } - } -} - -// Explicit template instantiations -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); -template MeanAggregator::MeanAggregator(const FieldInfo); - -} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/mean_aggregator.h b/tiledb/sm/query/readers/aggregators/mean_aggregator.h deleted file mode 100644 index 96e38504188e..000000000000 --- a/tiledb/sm/query/readers/aggregators/mean_aggregator.h +++ /dev/null @@ -1,157 +0,0 @@ -/** - * @file mean_aggregator.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines class MeanAggregator. - */ - -#ifndef TILEDB_MEAN_AGGREGATOR_H -#define TILEDB_MEAN_AGGREGATOR_H - -#include "tiledb/common/common.h" -#include "tiledb/sm/query/readers/aggregators/aggregate_with_count.h" -#include "tiledb/sm/query/readers/aggregators/field_info.h" -#include "tiledb/sm/query/readers/aggregators/iaggregator.h" -#include "tiledb/sm/query/readers/aggregators/safe_sum.h" -#include "tiledb/sm/query/readers/aggregators/sum_type.h" -#include "tiledb/sm/query/readers/aggregators/validity_policies.h" - -namespace tiledb::sm { - -class QueryBuffer; - -template -class MeanAggregator : public InputFieldValidator, - public OutputBufferValidator, - public IAggregator { - public: - /* ********************************* */ - /* CONSTRUCTORS & DESTRUCTORS */ - /* ********************************* */ - - MeanAggregator() = delete; - - /** - * Constructor. - * - * @param field_info Field info. - */ - MeanAggregator(FieldInfo field_info); - - DISABLE_COPY_AND_COPY_ASSIGN(MeanAggregator); - DISABLE_MOVE_AND_MOVE_ASSIGN(MeanAggregator); - - /* ********************************* */ - /* API */ - /* ********************************* */ - - /** Returns the field name for the aggregator. */ - std::string field_name() override { - return field_info_.name_; - } - - /** Returns if the aggregation is var sized or not. */ - bool aggregation_var_sized() override { - return false; - }; - - /** Returns if the aggregation is nullable or not. */ - bool aggregation_nullable() override { - return field_info_.is_nullable_; - } - - /** Returns if the aggregate needs to be recomputed on overflow. */ - bool need_recompute_on_overflow() override { - return true; - } - - /** - * Validate the result buffer. - * - * @param output_field_name Name for the output buffer. - * @param buffers Query buffers. - */ - void validate_output_buffer( - std::string output_field_name, - std::unordered_map& buffers) override; - - /** - * Aggregate data using the aggregator. - * - * @param input_data Input data for aggregation. - */ - void aggregate_data(AggregateBuffer& input_data) override; - - /** - * Copy final data to the user buffer. - * - * @param output_field_name Name for the output buffer. - * @param buffers Query buffers. - */ - void copy_to_user_buffer( - std::string output_field_name, - std::unordered_map& buffers) override; - - /** Returns name of the aggregate. */ - std::string aggregate_name() override { - return constants::aggregate_mean_str; - } - - /** Returns the TileDB datatype of the output field for the aggregate. */ - Datatype output_datatype() override { - return Datatype::FLOAT64; - } - - private: - /* ********************************* */ - /* PRIVATE ATTRIBUTES */ - /* ********************************* */ - - /** Field information. */ - const FieldInfo field_info_; - - /** AggregateWithCount to do summation of AggregateBuffer data. */ - AggregateWithCount::sum_type, SafeSum, NonNull> - aggregate_with_count_; - - /** Computed sum. */ - std::atomic::sum_type> sum_; - - /** Count of values. */ - std::atomic count_; - - /** Computed validity value. */ - optional validity_value_; - - /** Has the sum overflowed. */ - std::atomic sum_overflowed_; -}; - -} // namespace tiledb::sm - -#endif // TILEDB_MEAN_AGGREGATOR_H diff --git a/tiledb/sm/query/readers/aggregators/safe_sum.cc b/tiledb/sm/query/readers/aggregators/safe_sum.cc index ea74f2bfd515..e1f09924dfca 100644 --- a/tiledb/sm/query/readers/aggregators/safe_sum.cc +++ b/tiledb/sm/query/readers/aggregators/safe_sum.cc @@ -28,8 +28,6 @@ * @section DESCRIPTION * * This file implements class SafeSum. - * - * TODO: Testing for this class is not done (sc-33762). */ #include "tiledb/sm/query/readers/aggregators/safe_sum.h" diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc index 53a0384fc8ac..21cb3647eb73 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc @@ -45,7 +45,7 @@ class SumAggregatorStatusException : public StatusException { }; template -SumAggregator::SumAggregator(const FieldInfo field_info) +SumWithCountAggregator::SumWithCountAggregator(const FieldInfo field_info) : OutputBufferValidator(field_info) , field_info_(field_info) , aggregate_with_count_(field_info) @@ -57,7 +57,7 @@ SumAggregator::SumAggregator(const FieldInfo field_info) } template -void SumAggregator::validate_output_buffer( +void SumWithCountAggregator::validate_output_buffer( std::string output_field_name, std::unordered_map& buffers) { if (buffers.count(output_field_name) == 0) { @@ -68,7 +68,7 @@ void SumAggregator::validate_output_buffer( } template -void SumAggregator::aggregate_data(AggregateBuffer& input_data) { +void SumWithCountAggregator::aggregate_data(AggregateBuffer& input_data) { // Return if a previous aggregation has overflowed. if (sum_overflowed_) { return; @@ -76,9 +76,6 @@ void SumAggregator::aggregate_data(AggregateBuffer& input_data) { try { tuple::sum_type, uint64_t> res; - - // TODO: This is duplicated across aggregates but will go away with - // sc-33104. if (input_data.is_count_bitmap()) { res = aggregate_with_count_.template aggregate(input_data); } else { @@ -88,6 +85,7 @@ void SumAggregator::aggregate_data(AggregateBuffer& input_data) { const auto value = std::get<0>(res); const auto count = std::get<1>(res); SafeSum().safe_sum(value, sum_); + count_ += count; // Here we know that if the count is greater than 0, it means at least one // valid item was found, which means the result is valid. @@ -99,6 +97,23 @@ void SumAggregator::aggregate_data(AggregateBuffer& input_data) { } } +template +void SumWithCountAggregator::copy_validity_value_to_user_buffers( + QueryBuffer& result_buffer) { + if (field_info_.is_nullable_) { + auto v = static_cast(result_buffer.validity_vector_.buffer()); + if (sum_overflowed_) { + *v = 0; + } else { + *v = validity_value_.value(); + } + + if (result_buffer.validity_vector_.buffer_size()) { + *result_buffer.validity_vector_.buffer_size() = 1; + } + } +} + template void SumAggregator::copy_to_user_buffer( std::string output_field_name, @@ -106,31 +121,61 @@ void SumAggregator::copy_to_user_buffer( auto& result_buffer = buffers[output_field_name]; auto s = static_cast::sum_type*>(result_buffer.buffer_); - if (sum_overflowed_) { + if (SumWithCountAggregator::sum_overflowed_) { *s = std::numeric_limits::sum_type>::max(); } else { - *s = sum_; + *s = SumWithCountAggregator::sum_; } if (result_buffer.buffer_size_) { *result_buffer.buffer_size_ = sizeof(typename sum_type_data::sum_type); } - if (field_info_.is_nullable_) { - auto v = static_cast(result_buffer.validity_vector_.buffer()); - if (sum_overflowed_) { - *v = 0; - } else { - *v = validity_value_.value(); - } + SumWithCountAggregator::copy_validity_value_to_user_buffers(result_buffer); +} - if (result_buffer.validity_vector_.buffer_size()) { - *result_buffer.validity_vector_.buffer_size() = 1; - } +template +void MeanAggregator::copy_to_user_buffer( + std::string output_field_name, + std::unordered_map& buffers) { + auto& result_buffer = buffers[output_field_name]; + auto s = static_cast(result_buffer.buffer_); + + if (SumWithCountAggregator::sum_overflowed_) { + *s = std::numeric_limits::max(); + } else { + *s = static_cast(SumWithCountAggregator::sum_) / + SumWithCountAggregator::count_; + } + + if (result_buffer.buffer_size_) { + *result_buffer.buffer_size_ = sizeof(double); } + + SumWithCountAggregator::copy_validity_value_to_user_buffers(result_buffer); } // Explicit template instantiations +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator(const FieldInfo); +template SumWithCountAggregator::SumWithCountAggregator( + const FieldInfo); + template SumAggregator::SumAggregator(const FieldInfo); template SumAggregator::SumAggregator(const FieldInfo); template SumAggregator::SumAggregator(const FieldInfo); @@ -142,4 +187,15 @@ template SumAggregator::SumAggregator(const FieldInfo); template SumAggregator::SumAggregator(const FieldInfo); template SumAggregator::SumAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); +template MeanAggregator::MeanAggregator(const FieldInfo); + } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.h b/tiledb/sm/query/readers/aggregators/sum_aggregator.h index 0088f50fc389..14b5316e1c3d 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.h @@ -45,25 +45,25 @@ namespace tiledb::sm { class QueryBuffer; template -class SumAggregator : public InputFieldValidator, - public OutputBufferValidator, - public IAggregator { +class SumWithCountAggregator : public InputFieldValidator, + public OutputBufferValidator, + public IAggregator { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - SumAggregator() = delete; + SumWithCountAggregator() = delete; /** * Constructor. * * @param field_info Field info. */ - SumAggregator(FieldInfo field_info); + SumWithCountAggregator(FieldInfo field_info); - DISABLE_COPY_AND_COPY_ASSIGN(SumAggregator); - DISABLE_MOVE_AND_MOVE_ASSIGN(SumAggregator); + DISABLE_COPY_AND_COPY_ASSIGN(SumWithCountAggregator); + DISABLE_MOVE_AND_MOVE_ASSIGN(SumWithCountAggregator); /* ********************************* */ /* API */ @@ -106,6 +106,60 @@ class SumAggregator : public InputFieldValidator, */ void aggregate_data(AggregateBuffer& input_data) override; + /** + * Copy final validity value to the user buffer. + * + * @param result_buffer Query buffer to copy to. + */ + void copy_validity_value_to_user_buffers(QueryBuffer& result_buffer); + + protected: + /* ********************************* */ + /* PROTECTED ATTRIBUTES */ + /* ********************************* */ + + /** Field information. */ + const FieldInfo field_info_; + + /** AggregateWithCount to do summation of AggregateBuffer data. */ + AggregateWithCount::sum_type, SafeSum, NonNull> + aggregate_with_count_; + + /** Computed sum. */ + std::atomic::sum_type> sum_; + + /** Count of values. */ + std::atomic count_; + + /** Computed validity value. */ + optional validity_value_; + + /** Has the sum overflowed. */ + std::atomic sum_overflowed_; +}; + +template +class SumAggregator : public SumWithCountAggregator { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + SumAggregator() = delete; + + /** + * Constructor. + * + * @param field_info Field info. + */ + SumAggregator(FieldInfo field_info) + : SumWithCountAggregator(field_info) { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + /** * Copy final data to the user buffer. * @@ -125,27 +179,48 @@ class SumAggregator : public InputFieldValidator, Datatype output_datatype() override { return sum_type_data::tiledb_datatype; } +}; - private: +template +class MeanAggregator : public SumWithCountAggregator { + public: /* ********************************* */ - /* PRIVATE ATTRIBUTES */ + /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Field information. */ - const FieldInfo field_info_; + MeanAggregator() = delete; - /** AggregateWithCount to do summation of AggregateBuffer data. */ - AggregateWithCount::sum_type, SafeSum, NonNull> - aggregate_with_count_; + /** + * Constructor. + * + * @param field_info Field info. + */ + MeanAggregator(FieldInfo field_info) + : SumWithCountAggregator(field_info) { + } - /** Computed sum. */ - std::atomic::sum_type> sum_; + /* ********************************* */ + /* API */ + /* ********************************* */ + /** + * Copy final data to the user buffer. + * + * @param output_field_name Name for the output buffer. + * @param buffers Query buffers. + */ + void copy_to_user_buffer( + std::string output_field_name, + std::unordered_map& buffers) override; - /** Computed validity value. */ - optional validity_value_; + /** Returns name of the aggregate, e.g. COUNT, MIN, SUM. */ + std::string aggregate_name() override { + return constants::aggregate_mean_str; + } - /** Has the sum overflowed. */ - std::atomic sum_overflowed_; + /** Returns the TileDB datatype of the output field for the aggregate. */ + Datatype output_datatype() override { + return Datatype::FLOAT64; + } }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/sum_type.h b/tiledb/sm/query/readers/aggregators/sum_type.h index ed2b77a0a682..11cf1403c8bb 100644 --- a/tiledb/sm/query/readers/aggregators/sum_type.h +++ b/tiledb/sm/query/readers/aggregators/sum_type.h @@ -28,8 +28,6 @@ * @section DESCRIPTION * * This file defines sum types in relation to basic types. - * - * TODO: This needs to be improved to remove macros (sc-33764). */ #ifndef TILEDB_SUM_TYPE_H diff --git a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc index ab526e51139a..40cd8b7f251e 100644 --- a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc +++ b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc @@ -28,7 +28,6 @@ #include "../count_aggregator.h" #include "../field_info.h" -#include "../mean_aggregator.h" #include "../min_max_aggregator.h" #include "../sum_aggregator.h" diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 354f89056525..f4fdd0538562 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -34,7 +34,6 @@ #include "tiledb/sm/query/query_buffer.h" #include "tiledb/sm/query/readers/aggregators/aggregate_buffer.h" #include "tiledb/sm/query/readers/aggregators/count_aggregator.h" -#include "tiledb/sm/query/readers/aggregators/mean_aggregator.h" #include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" #include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" From 54bd64b587299d6ebd4f169108cfb6e1c8ca8c72 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 23 Oct 2023 17:19:47 +0300 Subject: [PATCH 018/456] MEAN and NULL_COUNT capi/cppapi wraps (#4443) Add CAPIs and CPPAPIs for mean and null count aggregators. --- TYPE: FEATURE | C_API | CPP_API DESC: MEAN and NULL_COUNT capi/cppapi wraps --- test/src/test-cppapi-aggregates.cc | 68 ++++----- .../query_aggregate/query_aggregate_api.cc | 35 ++++- ...uery_aggregate_api_external_experimental.h | 43 ++++++ .../test/unit_capi_query_aggregate.cc | 138 ++++++++++++++++++ tiledb/sm/cpp_api/channel_operator.h | 20 +++ .../readers/aggregators/count_aggregator.h | 15 +- .../sm/query/readers/aggregators/operation.cc | 4 + .../sm/query/readers/aggregators/operation.h | 43 ++++++ 8 files changed, 322 insertions(+), 44 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 909320ffd265..012135bc29f7 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -1389,12 +1389,12 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a mean aggregator to the query. - query.ptr()->query_->add_aggregator_to_default_channel( - "Mean", - std::make_shared>( - tiledb::sm::FieldInfo( - "a1", false, CppAggregatesFx::nullable_, 1))); + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("Mean", operation); CppAggregatesFx::set_ranges_and_condition_if_needed( array, query, false); @@ -1785,19 +1785,12 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a null count aggregator to the - // query. - uint64_t cell_val_num = std::is_same::value ? - CppAggregatesFx::STRING_CELL_VAL_NUM : - 1; - query.ptr()->query_->add_aggregator_to_default_channel( - "NullCount", - std::make_shared( - tiledb::sm::FieldInfo( - "a1", - false, - CppAggregatesFx::nullable_, - cell_val_num))); + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("NullCount", operation); CppAggregatesFx::set_ranges_and_condition_if_needed( array, query, false); @@ -1885,13 +1878,12 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a null count aggregator to the - // query. - query.ptr()->query_->add_aggregator_to_default_channel( - "NullCount", - std::make_shared( - tiledb::sm::FieldInfo( - "a1", true, nullable_, TILEDB_VAR_NUM))); + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("NullCount", operation); set_ranges_and_condition_if_needed(array, query, true); @@ -1981,12 +1973,12 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a null count aggregator to the - // query. - query.ptr()->query_->add_aggregator_to_default_channel( - "NullCount", - std::make_shared( - tiledb::sm::FieldInfo("a1", true, nullable_, TILEDB_VAR_NUM))); + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("NullCount", operation); // Add another aggregator on the second attribute. We will make the // first attribute get a var size overflow, which should not impact the @@ -2119,12 +2111,12 @@ TEST_CASE_METHOD( layout_ = layout; Query query(ctx_, array, TILEDB_READ); - // TODO: Change to real CPPAPI. Add a null count aggregator to the - // query. - query.ptr()->query_->add_aggregator_to_default_channel( - "NullCount", - std::make_shared( - tiledb::sm::FieldInfo("a1", true, nullable_, TILEDB_VAR_NUM))); + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("NullCount", operation); // Add another aggregator on the second attribute. We will make this // attribute get a var size overflow, which should impact the result of diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index c26cf14ed792..c51f76e31d73 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -37,7 +37,6 @@ #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api_support/c_api_support.h" -// const tiledb_channel_operator_handle_t* tiledb_channel_operator_sum = tiledb_channel_operator_handle_t::make_handle( tiledb::sm::constants::aggregate_sum_str); @@ -47,6 +46,12 @@ const tiledb_channel_operator_handle_t* tiledb_channel_operator_min = const tiledb_channel_operator_handle_t* tiledb_channel_operator_max = tiledb_channel_operator_handle_t::make_handle( tiledb::sm::constants::aggregate_max_str); +const tiledb_channel_operator_handle_t* tiledb_channel_operator_mean = + tiledb_channel_operator_handle_t::make_handle( + tiledb::sm::constants::aggregate_mean_str); +const tiledb_channel_operator_handle_t* tiledb_channel_operator_null_count = + tiledb_channel_operator_handle_t::make_handle( + tiledb::sm::constants::aggregate_null_count_str); const tiledb_channel_operation_handle_t* tiledb_aggregate_count = tiledb_channel_operation_handle_t::make_handle( @@ -127,6 +132,22 @@ capi_return_t tiledb_channel_operator_sum_get( return TILEDB_OK; } +capi_return_t tiledb_channel_operator_mean_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { + ensure_aggregates_enabled_via_config(ctx); + ensure_output_pointer_is_valid(op); + *op = tiledb_channel_operator_mean; + return TILEDB_OK; +} + +capi_return_t tiledb_channel_operator_null_count_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { + ensure_aggregates_enabled_via_config(ctx); + ensure_output_pointer_is_valid(op); + *op = tiledb_channel_operator_null_count; + return TILEDB_OK; +} + capi_return_t tiledb_channel_operator_min_get( tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { ensure_aggregates_enabled_via_config(ctx); @@ -254,6 +275,12 @@ capi_return_t tiledb_channel_operator_sum_get( ctx, op); } +capi_return_t tiledb_channel_operator_mean_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { + return api_entry_with_context( + ctx, op); +} + capi_return_t tiledb_channel_operator_min_get( tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { return api_entry_with_context( @@ -266,6 +293,12 @@ capi_return_t tiledb_channel_operator_max_get( ctx, op); } +capi_return_t tiledb_channel_operator_null_count_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { + return api_entry_with_context< + tiledb::api::tiledb_channel_operator_null_count_get>(ctx, op); +} + capi_return_t tiledb_aggregate_count_get( tiledb_ctx_t* ctx, const tiledb_channel_operation_t** operation) noexcept { return api_entry_with_context( diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h index 6325ad2aa065..c860552d67ef 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h @@ -59,6 +59,12 @@ TILEDB_EXPORT extern const tiledb_channel_operator_t* TILEDB_EXPORT extern const tiledb_channel_operator_t* tiledb_channel_operator_max; +TILEDB_EXPORT extern const tiledb_channel_operator_t* + tiledb_channel_operator_mean; + +TILEDB_EXPORT extern const tiledb_channel_operator_t* + tiledb_channel_operator_null_count; + // Constant aggregate operation handles TILEDB_EXPORT extern const tiledb_channel_operation_t* tiledb_aggregate_count; @@ -133,6 +139,43 @@ TILEDB_EXPORT int32_t tiledb_aggregate_count_get( tiledb_ctx_t* ctx, const tiledb_channel_operation_t** operation) TILEDB_NOEXCEPT; +/** + * Helper function to access the constant MEAN channel operator handle + * **Example:** + * + * @code{.c} + * tiledb_channel_operator_t *operator_mean; + * tiledb_channel_operator_mean_get(ctx, &operator_mean); + * tiledb_channel_operation_t* mean_A; + * tiledb_create_unary_aggregate(ctx, query, operator_mean, "A", mean_A); + * @endcode + * + * @param ctx The TileDB context + * @param op The operator handle to be retrieved + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_channel_operator_mean_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) TILEDB_NOEXCEPT; + +/** + * Helper function to access the constant NULL_COUNT channel operator handle + * **Example:** + * + * @code{.c} + * tiledb_channel_operator_t *operator_nullcount; + * tiledb_channel_operator_null_count_get(ctx, &operator_nullcount); + * tiledb_channel_operation_t* nullcount_A; + * tiledb_create_unary_aggregate(ctx, query, operator_nullcount, "A", + * nullcount_A); + * @endcode + * + * @param ctx The TileDB context + * @param op The operator handle to be retrieved + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_channel_operator_null_count_get( + tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) TILEDB_NOEXCEPT; + /** * Gets the default channel of the query. The default channel consists of all * the rows the query would return as if executed standalone. diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index fa7f5533311b..23b28b506825 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -94,6 +94,8 @@ void QueryAggregateFx::write_sparse_array() { int32_t b[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; uint64_t a_size = 10 * sizeof(int32_t); uint64_t b_size = 10 * sizeof(int32_t); + uint8_t b_validity[] = {1, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + uint64_t b_validity_size = sizeof(b_validity); int64_t d1[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; int64_t d2[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; @@ -110,6 +112,10 @@ void QueryAggregateFx::write_sparse_array() { REQUIRE( tiledb_query_set_data_buffer(ctx, query, "b", b, &b_size) == TILEDB_OK); + REQUIRE( + tiledb_query_set_validity_buffer( + ctx, query, "b", b_validity, &b_validity_size) == TILEDB_OK); + REQUIRE( tiledb_query_set_data_buffer(ctx, query, "d1", d1, &d1_size) == TILEDB_OK); @@ -170,6 +176,8 @@ void QueryAggregateFx::create_sparse_array() { tiledb_attribute_t* b; rc = tiledb_attribute_alloc(ctx, "b", TILEDB_INT32, &b); CHECK(rc == TILEDB_OK); + rc = tiledb_attribute_set_nullable(ctx, b, 1); + CHECK(rc == TILEDB_OK); tiledb_attribute_t* c; rc = tiledb_attribute_alloc(ctx, "c", TILEDB_STRING_ASCII, &c); CHECK(rc == TILEDB_OK); @@ -452,6 +460,71 @@ TEST_CASE_METHOD( tiledb_query_free(&query); } +TEST_CASE_METHOD( + QueryAggregateFx, + "C API: Query aggregates MEAN test", + "[capi][query_aggregate][mean]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); + + int64_t dom[] = {1, 10, 1, 1}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + tiledb_query_channel_t* default_channel; + REQUIRE( + tiledb_query_get_default_channel(ctx, query, &default_channel) == + TILEDB_OK); + + tiledb_channel_operation_t* mean_op; + REQUIRE( + tiledb_create_unary_aggregate( + ctx, query, tiledb_channel_operator_mean, "a", &mean_op) == + TILEDB_OK); + REQUIRE( + tiledb_channel_apply_aggregate(ctx, default_channel, "Mean", mean_op) == + TILEDB_OK); + + double mean = 0; + uint64_t size = 8; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "Mean", &mean, &size) == + TILEDB_OK); + + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); + + CHECK(mean == 5.5); + + // Clean up + CHECK(tiledb_aggregate_free(ctx, &mean_op) == TILEDB_OK); + CHECK(tiledb_query_channel_free(ctx, &default_channel) == TILEDB_OK); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); + tiledb_query_free(&query); +} + TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MIN test", @@ -580,6 +653,71 @@ TEST_CASE_METHOD( tiledb_query_free(&query); } +TEST_CASE_METHOD( + QueryAggregateFx, + "C API: Query aggregates NULL_COUNT test", + "[capi][query_aggregate][null_count]") { + SECTION("- No serialization") { + serialize_ = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("- Serialization") { + serialize_ = true; + } +#endif + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); + + int64_t dom[] = {1, 10, 1, 1}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + tiledb_query_channel_t* default_channel; + REQUIRE( + tiledb_query_get_default_channel(ctx, query, &default_channel) == + TILEDB_OK); + + tiledb_channel_operation_t* nullcount_op; + REQUIRE( + tiledb_create_unary_aggregate( + ctx, query, tiledb_channel_operator_null_count, "b", &nullcount_op) == + TILEDB_OK); + REQUIRE( + tiledb_channel_apply_aggregate( + ctx, default_channel, "Null", nullcount_op) == TILEDB_OK); + + uint64_t nullcount = 0; + uint64_t size = 8; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "Null", &nullcount, &size) == + TILEDB_OK); + + int rc = submit_query_wrapper( + ctx, + array_name_.c_str(), + &query, + server_buffers_, + serialize_, + false, + false); + REQUIRE(rc == 0); + + CHECK(nullcount == 8); + + // Clean up + CHECK(tiledb_aggregate_free(ctx, &nullcount_op) == TILEDB_OK); + CHECK(tiledb_query_channel_free(ctx, &default_channel) == TILEDB_OK); + CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_array_free(&array); + tiledb_query_free(&query); +} + TEST_CASE_METHOD( QueryAggregateFx, "C API: datatype checks", diff --git a/tiledb/sm/cpp_api/channel_operator.h b/tiledb/sm/cpp_api/channel_operator.h index 323affa3522b..53303db905f0 100644 --- a/tiledb/sm/cpp_api/channel_operator.h +++ b/tiledb/sm/cpp_api/channel_operator.h @@ -72,6 +72,26 @@ class MaxOperator : public ChannelOperator { } }; +class MeanOperator : public ChannelOperator { + public: + /* ********************************* */ + /* API */ + /* ********************************* */ + static const tiledb_channel_operator_t* ptr() { + return tiledb_channel_operator_mean; + } +}; + +class NullCountOperator : public ChannelOperator { + public: + /* ********************************* */ + /* API */ + /* ********************************* */ + static const tiledb_channel_operator_t* ptr() { + return tiledb_channel_operator_null_count; + } +}; + } // namespace tiledb #endif // TILEDB_CPP_API_CHANNEL_OPERATOR_H diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.h b/tiledb/sm/query/readers/aggregators/count_aggregator.h index 3299ccaff4c2..19cf38dd5d32 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.h @@ -101,11 +101,6 @@ class CountAggregatorBase : public OutputBufferValidator, public IAggregator { std::string output_field_name, std::unordered_map& buffers) override; - /** Returns name of the aggregate. */ - std::string aggregate_name() override { - return constants::aggregate_count_str; - } - /** Returns the TileDB datatype of the output field for the aggregate. */ Datatype output_datatype() override { return Datatype::UINT64; @@ -142,6 +137,11 @@ class CountAggregator : public CountAggregatorBase { std::string field_name() override { return constants::count_of_rows; } + + /** Returns name of the aggregate. */ + std::string aggregate_name() override { + return constants::aggregate_count_str; + } }; class NullCountAggregator : public CountAggregatorBase, @@ -162,6 +162,11 @@ class NullCountAggregator : public CountAggregatorBase, return field_info_.name_; } + /** Returns name of the aggregate. */ + std::string aggregate_name() override { + return constants::aggregate_null_count_str; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/operation.cc b/tiledb/sm/query/readers/aggregators/operation.cc index 14634b293674..e3afdf52e5e2 100644 --- a/tiledb/sm/query/readers/aggregators/operation.cc +++ b/tiledb/sm/query/readers/aggregators/operation.cc @@ -44,6 +44,10 @@ shared_ptr Operation::make_operation( return common::make_shared(HERE(), fi.value()); } else if (name == constants::aggregate_count_str) { return common::make_shared(HERE()); + } else if (name == constants::aggregate_mean_str) { + return common::make_shared(HERE(), fi.value()); + } else if (name == constants::aggregate_null_count_str) { + return common::make_shared(HERE(), fi.value()); } throw std::logic_error( diff --git a/tiledb/sm/query/readers/aggregators/operation.h b/tiledb/sm/query/readers/aggregators/operation.h index cf1867962fc3..d5c509b90122 100644 --- a/tiledb/sm/query/readers/aggregators/operation.h +++ b/tiledb/sm/query/readers/aggregators/operation.h @@ -204,6 +204,49 @@ class CountOperation : public Operation { } }; +class MeanOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + MeanOperation() = delete; + + /** + * Construct the operation where the internal aggregator object + * is instantiated to the correct type given the input field type. + * @param fi The FieldInfo for the input field + */ + explicit MeanOperation(const tiledb::sm::FieldInfo& fi) { + auto g = [&](auto T) { + if constexpr (tiledb::type::TileDBNumeric) { + ensure_field_numeric(fi); + aggregator_ = tdb::make_shared>( + HERE(), fi); + } else { + throw std::logic_error( + "MEAN aggregates can only be requested on numeric types"); + } + }; + type::apply_with_type(g, fi.type_); + } +}; + +class NullCountOperation : public Operation { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + NullCountOperation() = delete; + + /** + * Construct the NULL_COUNT operation + * @param fi The FieldInfo for the input field + */ + explicit NullCountOperation(const tiledb::sm::FieldInfo& fi) { + aggregator_ = tdb::make_shared(HERE(), fi); + } +}; + } // namespace tiledb::sm #endif // TILEDB_OPERATION_H From 10a45d7ef23827e06055e690e949acacbc76aea3 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:51:58 +0200 Subject: [PATCH 019/456] Fix the aggregates CPP API install files. (#4441) This also changes the use of concepts in std::derived_from to std::is_base_of_v to not require users of the CPP API to use c++ 20. SC-35774 --- TYPE: IMPROVEMENT DESC: Fix the aggregates CPP API install files. --------- Co-authored-by: Dirk Eddelbuettel --- tiledb/CMakeLists.txt | 3 +++ tiledb/sm/cpp_api/channel_operation.h | 4 +++- tiledb/sm/cpp_api/query_experimental.h | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 641d23810d9b..19aed81ca5e4 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -92,6 +92,8 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/as_built_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute_experimental.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/channel_operation.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/channel_operator.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/config.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/consolidation_plan_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/context.h @@ -112,6 +114,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object_iter.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_channel.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_experimental.h diff --git a/tiledb/sm/cpp_api/channel_operation.h b/tiledb/sm/cpp_api/channel_operation.h index 51153faed8b9..31333d92a185 100644 --- a/tiledb/sm/cpp_api/channel_operation.h +++ b/tiledb/sm/cpp_api/channel_operation.h @@ -89,7 +89,9 @@ class ChannelOperation { * @param query The TileDB query * @param input_field The attribute name the aggregate operation will run on */ - template Op> + template < + class Op, + std::enable_if_t, bool> = true> static ChannelOperation create( const Query& query, const std::string& input_field) { const Context& ctx = query.ctx(); diff --git a/tiledb/sm/cpp_api/query_experimental.h b/tiledb/sm/cpp_api/query_experimental.h index 92906ab21793..d70adda77381 100644 --- a/tiledb/sm/cpp_api/query_experimental.h +++ b/tiledb/sm/cpp_api/query_experimental.h @@ -405,7 +405,9 @@ class QueryExperimental { * @param input_field The attribute name as input for the aggregate * @return The aggregate operation */ - template Op> + template < + class Op, + std::enable_if_t, bool> = true> static ChannelOperation create_unary_aggregate( const Query& query, const std::string& input_field) { return ChannelOperation::create(query, input_field); From 2e95dcc48b44e8e74db65198ffbdc05f596fb460 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:06:15 -0400 Subject: [PATCH 020/456] Add random number generator global singleton (#4437) Precursor to `make_unique_directory`, add a global singleton random number generator which sets the seed from `Catch` or from user input. Note that the seed may only be _set_ and _used_ once. --- TYPE: FEATURE DESC: Add random number generator global singleton --- test/support/CMakeLists.txt | 14 +- test/support/tdb_catch.h | 4 +- test/support/tdb_catch_prng.cc | 35 +++++ test/support/tdb_catch_prng.h | 56 ++++++++ test/support/test/CMakeLists.txt | 32 +++++ test/support/test/compile_tdb_catch_main.cc | 34 +++++ test/support/test/unit_tdb_catch.cc | 40 ++++++ tiledb/common/CMakeLists.txt | 1 + tiledb/common/random/CMakeLists.txt | 41 ++++++ tiledb/common/random/prng.cc | 76 ++++++++++ tiledb/common/random/prng.h | 102 +++++++++++++ tiledb/common/random/seeder.cc | 69 +++++++++ tiledb/common/random/seeder.h | 134 ++++++++++++++++++ tiledb/common/random/test/CMakeLists.txt | 32 +++++ .../test/compile_seedable_global_PRNG_main.cc | 38 +++++ .../random/test/unit_seedable_global_PRNG.cc | 130 +++++++++++++++++ 16 files changed, 836 insertions(+), 2 deletions(-) create mode 100644 test/support/tdb_catch_prng.cc create mode 100644 test/support/tdb_catch_prng.h create mode 100644 test/support/test/CMakeLists.txt create mode 100644 test/support/test/compile_tdb_catch_main.cc create mode 100644 test/support/test/unit_tdb_catch.cc create mode 100644 tiledb/common/random/CMakeLists.txt create mode 100644 tiledb/common/random/prng.cc create mode 100644 tiledb/common/random/prng.h create mode 100644 tiledb/common/random/seeder.cc create mode 100644 tiledb/common/random/seeder.h create mode 100644 tiledb/common/random/test/CMakeLists.txt create mode 100644 tiledb/common/random/test/compile_seedable_global_PRNG_main.cc create mode 100644 tiledb/common/random/test/unit_seedable_global_PRNG.cc diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index e33b2595fdc7..0d3176203060 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -4,7 +4,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -97,3 +97,15 @@ target_link_libraries(tiledb_test_support_lib TILEDB_CORE_OBJECTS_ILIB Catch2::Catch2 ) + +# tdb_catch object library +include(common NO_POLICY_SCOPE) +include(object_library) + +commence(object_library tdb_catch) + this_target_sources(tdb_catch_prng.cc) + this_target_link_libraries(Catch2::Catch2) + this_target_object_libraries(seedable_global_PRNG) +conclude(object_library) + +add_test_subdirectory() diff --git a/test/support/tdb_catch.h b/test/support/tdb_catch.h index 9080cdab8081..bdb0f1956f75 100644 --- a/test/support/tdb_catch.h +++ b/test/support/tdb_catch.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -54,6 +54,8 @@ * The actual payload of this file */ #include +#include +#include /* * Clean up preprocessor definitions diff --git a/test/support/tdb_catch_prng.cc b/test/support/tdb_catch_prng.cc new file mode 100644 index 000000000000..3d4614e703fe --- /dev/null +++ b/test/support/tdb_catch_prng.cc @@ -0,0 +1,35 @@ +/** + * @file tdb_catch_prng.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines a Catch2 hook to seed a global random number generator. + */ + +#include "tdb_catch_prng.h" + +CATCH_REGISTER_LISTENER(PRNGSeederFromCatch) diff --git a/test/support/tdb_catch_prng.h b/test/support/tdb_catch_prng.h new file mode 100644 index 000000000000..552185dbac52 --- /dev/null +++ b/test/support/tdb_catch_prng.h @@ -0,0 +1,56 @@ +/** + * @file tdb_catch_prng.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares a Catch2 hook to seed a global random number generator. + */ + +#ifndef TILEDB_TDB_CATCH_PRNG_H +#define TILEDB_TDB_CATCH_PRNG_H + +#include +#include "tiledb/common/random/prng.h" +#include "tiledb/common/random/seeder.h" + +using namespace tiledb::common; + +class PRNGSeederFromCatch : public Catch::EventListenerBase { + public: + /** + * Make visible the base class constructor to default construct class + * testPRNG using base class initialization. + */ + using Catch::EventListenerBase::EventListenerBase; + + void testRunStarting(Catch::TestRunInfo const&) override { + Seeder& seeder_ = Seeder::get(); + seeder_.set_seed(Catch::rngSeed()); + } +}; + +#endif // TILEDB_TEST_SUPPORT_PRNG_H diff --git a/test/support/test/CMakeLists.txt b/test/support/test/CMakeLists.txt new file mode 100644 index 000000000000..812c3ab0330b --- /dev/null +++ b/test/support/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# test/support/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test test_tdb_catch) + this_target_sources(unit_tdb_catch.cc) + this_target_object_libraries(tdb_catch) +conclude(unit_test) diff --git a/test/support/test/compile_tdb_catch_main.cc b/test/support/test/compile_tdb_catch_main.cc new file mode 100644 index 000000000000..a3a228a912fb --- /dev/null +++ b/test/support/test/compile_tdb_catch_main.cc @@ -0,0 +1,34 @@ +/** + * @file test/support/test/compile_tdb_catch_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../tdb_catch_prng.h" + +int main() { + PRNGSeederFromCatch(nullptr); + return 0; +} diff --git a/test/support/test/unit_tdb_catch.cc b/test/support/test/unit_tdb_catch.cc new file mode 100644 index 000000000000..adfc47e39166 --- /dev/null +++ b/test/support/test/unit_tdb_catch.cc @@ -0,0 +1,40 @@ +/** + * @file test/support/test/unit_tdb_catch.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * Tests for the Catch2 hook of the global seedable PRNG facility. + */ + +#include "../tdb_catch_prng.h" + +/** + * Note: This is a singleton validation test. + * Only one test of this kind may run per test suite. + */ +TEST_CASE("tdbCatch: Validate set_seed hook", "[tdbCatch]") { + tiledb::common::Seeder& seeder_ = tiledb::common::Seeder::get(); + CHECK(seeder_.seed().value() == Catch::rngSeed()); +} diff --git a/tiledb/common/CMakeLists.txt b/tiledb/common/CMakeLists.txt index 02177802035f..a7736fd55ec0 100644 --- a/tiledb/common/CMakeLists.txt +++ b/tiledb/common/CMakeLists.txt @@ -43,6 +43,7 @@ add_subdirectory(dynamic_memory) add_subdirectory(exception) add_subdirectory(governor) add_subdirectory(interval) +add_subdirectory(random) add_subdirectory(thread_pool) add_subdirectory(types) diff --git a/tiledb/common/random/CMakeLists.txt b/tiledb/common/random/CMakeLists.txt new file mode 100644 index 000000000000..e6e934ccd4a3 --- /dev/null +++ b/tiledb/common/random/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/common/random/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + prng.cc + seeder.cc +) +gather_sources(${SOURCES}) + +commence(object_library seedable_global_PRNG) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) +conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/common/random/prng.cc b/tiledb/common/random/prng.cc new file mode 100644 index 000000000000..842439b49b4b --- /dev/null +++ b/tiledb/common/random/prng.cc @@ -0,0 +1,76 @@ +/** + * @file prng.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the library-wide 64-bit PRNG facility. + */ + +#include "tiledb/common/random/prng.h" + +namespace tiledb::common { + +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ + +PRNG::PRNG() + : prng_(prng_initial()) + , mtx_{} { +} + +/* ********************************* */ +/* API */ +/* ********************************* */ + +PRNG& PRNG::get() { + static PRNG singleton; + return singleton; +} + +uint64_t PRNG::operator()() { + std::lock_guard lock(mtx_); + return prng_(); +} + +/* ********************************* */ +/* PRIVATE METHODS */ +/* ********************************* */ + +std::mt19937_64 PRNG::prng_initial() { + // Retrieve optional, potentially default-constructed seed. + auto seed{Seeder::get().seed()}; + + // If the seed has been set, set it on the RNG engine. + if (seed.has_value()) { + return std::mt19937_64{seed.value()}; // RVO + } else { + return {}; // RVO + } +} + +} // namespace tiledb::common diff --git a/tiledb/common/random/prng.h b/tiledb/common/random/prng.h new file mode 100644 index 000000000000..d03f5ceacf11 --- /dev/null +++ b/tiledb/common/random/prng.h @@ -0,0 +1,102 @@ +/** + * @file prng.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the library-wide 64-bit PRNG facility. + */ + +#ifndef TILEDB_PRNG_H +#define TILEDB_PRNG_H + +#include +#include + +#include "tiledb/common/random/seeder.h" + +namespace tiledb::common { +class PRNG { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor. + * + * Constructs an mt19937 engine for random number generation. + * If Seeder has been seeded, the seed will be set on the engine. + * Otherwise, it is default-constructed. + */ + PRNG(); + + /** Copy constructor is deleted. */ + PRNG(const PRNG&) = delete; + + /** Move constructor is deleted. */ + PRNG(PRNG&&) = delete; + + /** Copy assignment is deleted. */ + PRNG& operator=(const PRNG&) = delete; + + /** Move assignment is deleted. */ + PRNG& operator=(PRNG&&) = delete; + + /** Destructor. */ + ~PRNG() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Singleton accessor. */ + static PRNG& get(); + + /** Get next in PRNG sequence. */ + uint64_t operator()(); + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** 64-bit mersenne twister engine for random number generation. */ + std::mt19937_64 prng_; + + /** Mutex which protects against simultaneous access to operator() body. */ + std::mutex mtx_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** Default-constructs an mt19937 engine and optionally sets the seed. */ + std::mt19937_64 prng_initial(); +}; +} // namespace tiledb::common + +#endif // TILEDB_PRNG_H diff --git a/tiledb/common/random/seeder.cc b/tiledb/common/random/seeder.cc new file mode 100644 index 000000000000..e52dc7b02036 --- /dev/null +++ b/tiledb/common/random/seeder.cc @@ -0,0 +1,69 @@ +/** + * @file seeder.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the seeder for the library-wide 64-bit RNG facility. + */ + +#include "tiledb/common/random/seeder.h" + +namespace tiledb::common { + +/* ********************************* */ +/* API */ +/* ********************************* */ + +Seeder& Seeder::get() { + static Seeder singleton; + return singleton; +} + +void Seeder::set_seed(uint64_t seed) { + std::lock_guard lock(mtx_); + + if (lifespan_state_ != 0) { + throw std::logic_error("[Seeder::set_seed] Seed has already been set."); + } else { + seed_ = seed; + lifespan_state_ = 1; + } +} + +std::optional Seeder::seed() { + std::lock_guard lock(mtx_); + + if (lifespan_state_ == 2) { + throw std::logic_error( + "[Seeder::seed] Seed can only be used once and has already been used."); + } + + lifespan_state_ = 2; + return seed_; +} + +} // namespace tiledb::common diff --git a/tiledb/common/random/seeder.h b/tiledb/common/random/seeder.h new file mode 100644 index 000000000000..e2807a146fb1 --- /dev/null +++ b/tiledb/common/random/seeder.h @@ -0,0 +1,134 @@ +/** + * @file seeder.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the seeder for the library-wide 64-bit RNG facility. + */ + +#ifndef TILEDB_SEEDER_H +#define TILEDB_SEEDER_H + +#include +#include +#include + +namespace tiledb::common { + +/** + * Class which manages the lifetime state and transitions of a seed + * which may be used to seed a random number generator in class PRNG. + * + * Lifetime state and transitions can be depicted as follows: + * 0 ---> 1 ---> 2 + * default (set_seed) seed is set (seed) seed is used + * but unused + * + * Note that each transition may occur only once. + * i.e. A seed may only be set one time and may only be used one time. + */ +class Seeder { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor. + * + * Default constructed as the initial state of the singleton Seeder with + * no set seed. + */ + Seeder() = default; + + /** Copy constructor is deleted. */ + Seeder(const Seeder&) = delete; + + /** Move constructor is deleted. */ + Seeder(Seeder&&) = delete; + + /** Copy assignment is deleted. */ + Seeder& operator=(const Seeder&) = delete; + + /** Move assignment is deleted. */ + Seeder& operator=(Seeder&&) = delete; + + /** Destructor. */ + ~Seeder() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Singleton accessor. */ + static Seeder& get(); + + /** + * Sets the seed. + * Transitions seed lifetime_state_: 0 (default) --> 1 (set but unused). + * Note that this may be called once and only once. + * + * @param seed Seed to set. + */ + void set_seed(uint64_t seed); + + /** + * Returns the seed, if set and not yet used. + * Transitions seed lifetime_state_: 1 (set but unused) --> 2 (set and used). + * Note: this may be called once and only once. + * + * Note: if the seed is not yet set, return std::nullopt. + * This behavior will allow users to call the default constructor. + * Transitions seed lifetime_state_: 0 (default) --> 2 (used). + * + * @return Seed, if set and not yet used. Else, throw or return nullopt. + */ + std::optional seed(); + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Optional seed object, set by set_seed. */ + std::optional seed_{std::nullopt}; + + /** + * Numerical representation of seeder state. + * + * 0 = default (unseeded) + * 1 = seeded but seed not yet used + * 2 = seeded and seed used + */ + int lifespan_state_{0}; + + /** Mutex which protects transitions of lifespan_state_. */ + std::mutex mtx_; +}; +} // namespace tiledb::common + +#endif // TILEDB_SEEDER_H diff --git a/tiledb/common/random/test/CMakeLists.txt b/tiledb/common/random/test/CMakeLists.txt new file mode 100644 index 000000000000..2a8754ac7dcd --- /dev/null +++ b/tiledb/common/random/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/common/random/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test seedable_global_PRNG) + this_target_sources(unit_seedable_global_PRNG.cc) + this_target_object_libraries(seedable_global_PRNG) +conclude(unit_test) diff --git a/tiledb/common/random/test/compile_seedable_global_PRNG_main.cc b/tiledb/common/random/test/compile_seedable_global_PRNG_main.cc new file mode 100644 index 000000000000..fc063c80e1e5 --- /dev/null +++ b/tiledb/common/random/test/compile_seedable_global_PRNG_main.cc @@ -0,0 +1,38 @@ +/** + * @file tiledb/common/random/test/compile_seedable_global_PRNG_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../prng.h" +#include "../seeder.h" + +using namespace tiledb::common; + +int main() { + (void)PRNG::get(); + (void)Seeder::get(); + return 0; +} diff --git a/tiledb/common/random/test/unit_seedable_global_PRNG.cc b/tiledb/common/random/test/unit_seedable_global_PRNG.cc new file mode 100644 index 000000000000..c0b78e21ce7d --- /dev/null +++ b/tiledb/common/random/test/unit_seedable_global_PRNG.cc @@ -0,0 +1,130 @@ +/** + * @file tiledb/common/random/test/unit_seedable_global_PRNG.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * Tests for the global seedable PRNG facility. + */ + +#include +#include "../prng.h" +#include "../seeder.h" + +#include + +using namespace tiledb::common; + +TEST_CASE( + "SeedableGlobalPRNG: Seeder, default seed", + "[SeedableGlobalPRNG][Seeder][default]") { + // Default seed is std::nullopt + Seeder seeder; + std::optional seed; + + // Use default seed (state 0 -> 2) + CHECK_NOTHROW(seed = seeder.seed()); + CHECK(!seed.has_value()); + + // Try setting seed after it's been used (state 2) + SECTION("try to set seed again") { + CHECK_THROWS_WITH( + seeder.set_seed(123), + Catch::Matchers::ContainsSubstring("Seed has already been set")); + } + + // Try using seed after it's been used (state 2) + SECTION("try to use seed again") { + CHECK_THROWS_WITH( + seeder.seed(), + Catch::Matchers::ContainsSubstring("Seed can only be used once")); + } +} + +TEST_CASE( + "SeedableGlobalPRNG: Seeder, set seed", + "[SeedableGlobalPRNG][Seeder][set_seed]") { + // Set seed (state 0 -> 1) + Seeder seeder; + CHECK_NOTHROW(seeder.set_seed(123)); + + SECTION("try to set seed again") { + CHECK_THROWS_WITH( + seeder.set_seed(456), + Catch::Matchers::ContainsSubstring("Seed has already been set")); + } + + // Use seed, after it's been set but not used (state 1 -> 2) + CHECK(seeder.seed() == 123); + + // Try setting seed after it's been set & used (state 2) + SECTION("try to set seed after it's been set and used") { + CHECK_THROWS_WITH( + seeder.set_seed(456), + Catch::Matchers::ContainsSubstring("Seed has already been set")); + } + + // Try using seed after it's been set & used (state 2) + SECTION("try to use seed after it's been set and used") { + CHECK_THROWS_WITH( + seeder.seed(), + Catch::Matchers::ContainsSubstring("Seed can only be used once")); + } +} + +TEST_CASE( + "SeedableGlobalPRNG: operator", + "[SeedableGlobalPRNG][operator][multiple]") { + PRNG prng; + auto rand_num1 = prng(); + CHECK(rand_num1 != 0); + + auto rand_num2 = prng(); + CHECK(rand_num2 != 0); + CHECK(rand_num1 != rand_num2); + + auto rand_num3 = prng(); + CHECK(rand_num3 != 0); + CHECK(rand_num1 != rand_num3); + CHECK(rand_num2 != rand_num3); +} + +TEST_CASE( + "SeedableGlobalPRNG: Seeder singleton, errors", + "[SeedableGlobalPRNG][Seeder][singleton][errors]") { + // Note: these errors will occur because PRNG sets and uses the singleton. + Seeder& seeder_ = Seeder::get(); + + SECTION("try to set new seed after it's been set") { + CHECK_THROWS_WITH( + seeder_.set_seed(1), + Catch::Matchers::ContainsSubstring("Seed has already been set")); + } + + SECTION("try to use seed after it's been used") { + CHECK_THROWS_WITH( + seeder_.seed(), + Catch::Matchers::ContainsSubstring("Seed can only be used once")); + } +} From ab87e06f4b140a2fdafbf2d6781a93af841de69c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 23 Oct 2023 18:49:34 +0300 Subject: [PATCH 021/456] Fix the release workflow and enable GCS in the Windows binaries. (#4439) This PR fixes the release workflow for macOS by updating the OS image (it used macOS 11 while regular CI uses 12), enables GCS in the released Windows binaries, and does two other changes that should speed-up the release workflow on Windows. --- TYPE: NO_HISTORY --- .github/workflows/build-windows.yml | 23 +------------- .github/workflows/ci-linux_mac.yml | 2 +- azure-pipelines.yml | 7 ++--- scripts/azure-linux_mac-release.yml | 5 ++- scripts/azure-windows-release.yml | 9 ++++-- scripts/ci/patch_vcpkg_triplets.py | 20 ++++++++++++ scripts/ci/posix/patch_vcpkg_triplets.sh | 40 ------------------------ 7 files changed, 35 insertions(+), 71 deletions(-) create mode 100644 scripts/ci/patch_vcpkg_triplets.py delete mode 100755 scripts/ci/posix/patch_vcpkg_triplets.sh diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index cca993b2f65f..2a9dc05c561f 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -129,28 +129,7 @@ jobs: fetch-depth: 0 - name: Prevent vcpkg from building debug variants - shell: python - run: | - import os - - workspace = os.environ["GITHUB_WORKSPACE"] - triplet_paths = [os.path.join(workspace, "external", "vcpkg", "triplets"), - os.path.join(workspace, "ports", "triplets")] - - for triplet_path in triplet_paths: - for path, dnames, fnames in os.walk(triplet_path): - for fname in fnames: - fname = os.path.join(path, fname) - print(fname) - with open(fname, "rb") as handle: - contents = handle.read() - # If a file is already patched we skip patching it again because - # this affects the hashes calculated by vcpkg for caching - if b"VCPKG_BUILD_TYPE release" in contents: - continue - contents += b"\nset(VCPKG_BUILD_TYPE release)\n" - with open(fname, "wb") as handle: - handle.write(contents) + run: python $env:GITHUB_WORKSPACE/scripts/ci/patch_vcpkg_triplets.py - name: core tiledb windows build run: | diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 3b356963507e..b791f002ce3f 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -87,7 +87,7 @@ jobs: echo "${{ toJSON(steps.runvcpkg.outputs) }}" - name: Prevent vpckg from building debug variants - run: ./scripts/ci/posix/patch_vcpkg_triplets.sh + run: python ./scripts/ci/patch_vcpkg_triplets.py - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.3 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5e80c2e8c352..b0d6841e369b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,13 +60,13 @@ stages: strategy: matrix: macOS: - imageName: 'macos-11' + imageName: 'macos-12' CXX: clang++ ARTIFACT_OS: 'macos' ARTIFACT_ARCH: 'x86_64' CMAKE_OSX_ARCHITECTURES: "x86_64" macOS_arm64: - imageName: 'macos-11' + imageName: 'macos-12' CXX: clang++ ARTIFACT_OS: 'macos' ARTIFACT_ARCH: 'arm64' @@ -104,13 +104,12 @@ stages: steps: - template: scripts/azure-linux_mac-release.yml - job: Windows - timeoutInMinutes: 90 + timeoutInMinutes: 120 strategy: matrix: VS2019: imageName: 'windows-2019' # Only S3 variable is currently supported in bootstrap powershell - TILEDB_GCS: OFF TILEDB_HDFS: OFF ARTIFACT_OS: 'windows' ARTIFACT_ARCH: 'x86_64' diff --git a/scripts/azure-linux_mac-release.yml b/scripts/azure-linux_mac-release.yml index ac2d978c94ba..5e8e27f3da30 100755 --- a/scripts/azure-linux_mac-release.yml +++ b/scripts/azure-linux_mac-release.yml @@ -23,6 +23,9 @@ steps: condition: eq(variables['Agent.OS'], 'Darwin') displayName: 'Install brew packages for macOS build' +- bash: python ./scripts/ci/patch_vcpkg_triplets.py + displayName: Prevent vpckg from building debug variants + # Tried using mamba for manylinux missing packages - switched to custom image instead. #- bash: | # curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba @@ -133,7 +136,7 @@ steps: - task: ArchiveFiles@2 inputs: - rootFolderOrFile: '$(Build.Repository.LocalPath)' + rootFolderOrFile: '$(Build.Repository.LocalPath)/build' includeRootFolder: false archiveType: 'tar' # Options: zip, 7z, tar, wim tarCompression: 'gz' # Optional. Options: gz, bz2, xz, none diff --git a/scripts/azure-windows-release.yml b/scripts/azure-windows-release.yml index 98bae098257a..d0722c5d36a5 100644 --- a/scripts/azure-windows-release.yml +++ b/scripts/azure-windows-release.yml @@ -8,6 +8,9 @@ steps: env: { sourceVersion: $(Build.SourceVersion) } displayName: Git Hash 7-digit +- powershell: python ./scripts/ci/patch_vcpkg_triplets.py + displayName: Prevent vpckg from building debug variants + - powershell: | mkdir $env:AGENT_BUILDDIRECTORY\build cd $env:AGENT_BUILDDIRECTORY\build @@ -24,7 +27,7 @@ steps: $host.SetShouldExit(3) } - $bootstrapOptions = "-EnableVerbose -EnableStaticTileDB" + $bootstrapOptions = "-EnableVerbose -EnableStaticTileDB -DisableTests" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } @@ -32,7 +35,7 @@ steps: $bootstrapOptions = "-EnableAzure " + $bootstrapOptions } if ($env:TILEDB_GCS -eq "ON") { - $bootstrapOptions = "-EnableGCS " + $bootstrapOptions + $bootstrapOptions = "-EnableGcs " + $bootstrapOptions #NOTE: GCS simulator not yet actually in place. } if ($env:TILEDB_SERIALIZATION -eq "ON") { @@ -116,7 +119,7 @@ steps: - task: ArchiveFiles@2 inputs: - rootFolderOrFile: '$(Agent.BuildDirectory)\s\' + rootFolderOrFile: '$(Agent.BuildDirectory)\build\' includeRootFolder: false archiveType: 'zip' # Options: zip, 7z, tar, wim archiveFile: $(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-build-dir.zip diff --git a/scripts/ci/patch_vcpkg_triplets.py b/scripts/ci/patch_vcpkg_triplets.py new file mode 100644 index 000000000000..f8007955d237 --- /dev/null +++ b/scripts/ci/patch_vcpkg_triplets.py @@ -0,0 +1,20 @@ +import os + +workspace = os.curdir +triplet_paths = [os.path.join(workspace, "external", "vcpkg", "triplets"), + os.path.join(workspace, "ports", "triplets")] + +for triplet_path in triplet_paths: + for path, dnames, fnames in os.walk(triplet_path): + for fname in fnames: + fname = os.path.join(path, fname) + print(fname) + with open(fname, "rb") as handle: + contents = handle.read() + # If a file is already patched we skip patching it again because + # this affects the hashes calculated by vcpkg for caching + if b"VCPKG_BUILD_TYPE release" in contents: + continue + contents += b"\nset(VCPKG_BUILD_TYPE release)\n" + with open(fname, "wb") as handle: + handle.write(contents) diff --git a/scripts/ci/posix/patch_vcpkg_triplets.sh b/scripts/ci/posix/patch_vcpkg_triplets.sh deleted file mode 100755 index e27942a83697..000000000000 --- a/scripts/ci/posix/patch_vcpkg_triplets.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# The MIT License (MIT) -# -# Copyright (c) 2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Based on the comment here: -# https://github.com/microsoft/vcpkg/issues/143#issuecomment-705778244 ¯\_(ツ)_/¯ -# -# The conditional logic here is to avoid re-patching the triplet files after -# they're restored by run-vcpkg. While re-patching is harmless, it breaks -# run-vcpkg's caching because it changes the generated package hashes. - -TRIPLETS="${GITHUB_WORKSPACE}/external/vcpkg/triplets ${GITHUB_WORKSPACE}/ports/triplets" - -find $TRIPLETS -type f -maxdepth 1 | xargs grep -q "VCPKG_BUILD_TYPE release" -if [ $? -ne 0 ] -then - find $TRIPLETS -type f \ - -exec sh -c "echo \"\nset(VCPKG_BUILD_TYPE release)\n\" >> {}" \; - echo "Triplets patched" -fi From d7349a646cb5bf3f0ed9dc0ecc3c77543cdf3485 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:45:53 -0600 Subject: [PATCH 022/456] Remove `RETURN_NOT_OK_ELSE_TUPLE` (#4444) Remove `RETURN_NOT_OK_ELSE_TUPLE`. It's no longer used. --- TYPE: NO_HISTORY DESC: Remove `RETURN_NOT_OK_ELSE_TUPLE` --- tiledb/common/exception/status.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tiledb/common/exception/status.h b/tiledb/common/exception/status.h index e513705e5572..f718906360b9 100644 --- a/tiledb/common/exception/status.h +++ b/tiledb/common/exception/status.h @@ -88,15 +88,6 @@ namespace common { } \ } while (false) -#define RETURN_NOT_OK_ELSE_TUPLE(s, else_, ...) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - else_; \ - return {_s, __VA_ARGS__}; \ - } \ - } while (false) - /** * The original `Status` class, used as a ubiquitous return value to avoid * throwing exceptions. From b4f43f9b566a3de528563049db72ab2f09cd5b8d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 23 Oct 2023 19:43:07 +0200 Subject: [PATCH 023/456] Remove default constructor argument for FieldInfo. (#4442) This has caused misuse when refactoring tests and should be avoided. --- TYPE: NO_HISTORY DESC: Remove default constructor argument for FieldInfo. --- test/src/test-cppapi-aggregates.cc | 12 +- test/support/src/helper_type.h | 130 +++++++++++++ .../sm/query/readers/aggregators/field_info.h | 2 +- .../aggregators/test/bench_aggregators.cc | 4 +- .../test/compile_aggregators_main.cc | 172 +++++++++--------- .../test/unit_aggregate_with_count.cc | 6 +- .../aggregators/test/unit_aggregators.cc | 56 +++--- 7 files changed, 266 insertions(+), 116 deletions(-) create mode 100644 test/support/src/helper_type.h diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 012135bc29f7..14e91e6b0a62 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -38,6 +38,7 @@ #include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" #include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" +#include #include using namespace tiledb; @@ -1986,8 +1987,9 @@ TEST_CASE_METHOD( // second attribute. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount2", - std::make_shared( - tiledb::sm::FieldInfo("a2", true, nullable_, TILEDB_VAR_NUM))); + std::make_shared< + tiledb::sm::NullCountAggregator>(tiledb::sm::FieldInfo( + "a2", true, nullable_, TILEDB_VAR_NUM, tdb_type))); set_ranges_and_condition_if_needed(array, query, true); @@ -2123,8 +2125,9 @@ TEST_CASE_METHOD( // the first one hence throw an exception. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount2", - std::make_shared( - tiledb::sm::FieldInfo("a2", true, nullable_, TILEDB_VAR_NUM))); + std::make_shared< + tiledb::sm::NullCountAggregator>(tiledb::sm::FieldInfo( + "a2", true, nullable_, TILEDB_VAR_NUM, tdb_type))); set_ranges_and_condition_if_needed(array, query, true); @@ -2189,6 +2192,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( for (tiledb_layout_t layout : CppAggregatesFx::layout_values_) { CppAggregatesFx::layout_ = layout; Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); + // Add a count aggregator to the query. We add both sum and count as // they are processed separately in the dense case. QueryChannel default_channel = diff --git a/test/support/src/helper_type.h b/test/support/src/helper_type.h new file mode 100644 index 000000000000..28fa7a7bd461 --- /dev/null +++ b/test/support/src/helper_type.h @@ -0,0 +1,130 @@ +/** + * @file helper_type.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This defines TileDB datatypes. + */ + +#ifndef TILEDB_TEST_TYPE_H +#define TILEDB_TEST_TYPE_H + +namespace tiledb::test { + +using Datatype = tiledb::sm::Datatype; + +/** + * Convert a type into a tiledb::sm::Datatype. The default for all copyable + * types is char. + */ +template +struct type_to_tiledb {}; + +template <> +struct type_to_tiledb { + using type = std::byte; + static constexpr Datatype tiledb_type = Datatype::BLOB; +}; + +template <> +struct type_to_tiledb { + using type = bool; + static constexpr Datatype tiledb_type = Datatype::BOOL; +}; + +template <> +struct type_to_tiledb { + using type = int8_t; + static constexpr Datatype tiledb_type = Datatype::INT8; +}; + +template <> +struct type_to_tiledb { + using type = uint8_t; + static constexpr Datatype tiledb_type = Datatype::UINT8; +}; + +template <> +struct type_to_tiledb { + using type = int16_t; + static constexpr Datatype tiledb_type = Datatype::INT16; +}; + +template <> +struct type_to_tiledb { + using type = uint16_t; + static constexpr Datatype tiledb_type = Datatype::UINT16; +}; + +template <> +struct type_to_tiledb { + using type = int32_t; + static constexpr Datatype tiledb_type = Datatype::INT32; +}; + +template <> +struct type_to_tiledb { + using type = uint32_t; + static constexpr Datatype tiledb_type = Datatype::UINT32; +}; + +template <> +struct type_to_tiledb { + using type = int64_t; + static constexpr Datatype tiledb_type = Datatype::INT64; +}; + +template <> +struct type_to_tiledb { + using type = uint64_t; + static constexpr Datatype tiledb_type = Datatype::UINT64; +}; + +template <> +struct type_to_tiledb { + using type = float; + static constexpr Datatype tiledb_type = Datatype::FLOAT32; +}; + +template <> +struct type_to_tiledb { + using type = double; + static constexpr Datatype tiledb_type = Datatype::FLOAT64; +}; + +template <> +struct type_to_tiledb { + using type = char; + static const Datatype tiledb_type = Datatype::STRING_ASCII; +}; + +template +constexpr Datatype tdb_type = type_to_tiledb::tiledb_type; + +} // namespace tiledb::test + +#endif // TILEDB_TEST_TYPE_H diff --git a/tiledb/sm/query/readers/aggregators/field_info.h b/tiledb/sm/query/readers/aggregators/field_info.h index 1ee11ba9dd7f..0a6d6071736c 100644 --- a/tiledb/sm/query/readers/aggregators/field_info.h +++ b/tiledb/sm/query/readers/aggregators/field_info.h @@ -67,7 +67,7 @@ class FieldInfo { const bool var_sized, const bool is_nullable, const unsigned cell_val_num, - const Datatype type = Datatype::UINT8) + const Datatype type) : name_(name) , var_sized_(var_sized) , is_nullable_(is_nullable) diff --git a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc index 16e0a7d133bc..d88e135f4d7f 100644 --- a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc @@ -38,9 +38,11 @@ #include "tiledb/sm/query/readers/aggregators/sum_type.h" #include "tiledb/sm/query/readers/aggregators/validity_policies.h" +#include #include using namespace tiledb::sm; +using namespace tiledb::test; const uint64_t num_cells = 10 * 1024 * 1024; @@ -156,7 +158,7 @@ void run_bench() { << ", Segmented: " << (segmented ? "true" : "false")) { BENCHMARK("Bench") { AggregateWithCount aggregator( - FieldInfo("a1", var_sized, nullable, 1)); + FieldInfo("a1", var_sized, nullable, 1, tdb_type)); for (uint64_t s = 0; s < num_cells; s += increment) { AggregateBuffer input_data{ s, diff --git a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc index 40cd8b7f251e..af180a27054f 100644 --- a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc +++ b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc @@ -34,96 +34,96 @@ int main() { tiledb::sm::CountAggregator(); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); - tiledb::sm::MeanAggregator( - tiledb::sm::FieldInfo("Mean", false, false, 1)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MeanAggregator(tiledb::sm::FieldInfo( + "Mean", false, false, 1, tiledb::sm::Datatype::UINT8)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MinAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MinAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); - tiledb::sm::MaxAggregator( - tiledb::sm::FieldInfo("MinMax", false, false, 1)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::MaxAggregator(tiledb::sm::FieldInfo( + "MinMax", false, false, 1, tiledb::sm::Datatype::UINT8)); - tiledb::sm::NullCountAggregator( - tiledb::sm::FieldInfo("NullCount", false, false, 1)); + tiledb::sm::NullCountAggregator(tiledb::sm::FieldInfo( + "NullCount", false, false, 1, tiledb::sm::Datatype::UINT8)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); - tiledb::sm::SumAggregator( - tiledb::sm::FieldInfo("Sum", false, false, 1)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); + tiledb::sm::SumAggregator(tiledb::sm::FieldInfo( + "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); return 0; } \ No newline at end of file diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregate_with_count.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregate_with_count.cc index c4474928ebed..7cb80a64e81c 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregate_with_count.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregate_with_count.cc @@ -37,9 +37,11 @@ #include "tiledb/sm/query/readers/aggregators/sum_type.h" #include "tiledb/sm/query/readers/aggregators/validity_policies.h" +#include #include using namespace tiledb::sm; +using namespace tiledb::test; typedef tuple< uint8_t, @@ -59,9 +61,9 @@ TEMPLATE_LIST_TEST_CASE( FixedTypesUnderTest) { typedef TestType T; AggregateWithCount::sum_type, SafeSum, NonNull> - aggregator(FieldInfo("a1", false, false, 1)); + aggregator(FieldInfo("a1", false, false, 1, tdb_type)); AggregateWithCount::sum_type, SafeSum, NonNull> - aggregator_nullable(FieldInfo("a2", false, true, 1)); + aggregator_nullable(FieldInfo("a2", false, true, 1, tdb_type)); std::vector fixed_data = {1, 2, 3, 4, 5, 5, 4, 3, 2, 1}; std::vector validity_data = {0, 0, 1, 0, 1, 0, 1, 0, 1, 0}; diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index f4fdd0538562..17b475027736 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -37,9 +37,11 @@ #include "tiledb/sm/query/readers/aggregators/min_max_aggregator.h" #include "tiledb/sm/query/readers/aggregators/sum_aggregator.h" +#include #include using namespace tiledb::sm; +using namespace tiledb::test; typedef tuple< SumAggregator, @@ -52,14 +54,14 @@ TEMPLATE_LIST_TEST_CASE( AggUnderTestConstructor) { SECTION("Var size") { CHECK_THROWS_WITH( - TestType(FieldInfo("a1", true, false, 1)), + TestType(FieldInfo("a1", true, false, 1, Datatype::UINT8)), "InputFieldValidator: Aggregate is not supported for var sized " "non-string fields."); } SECTION("Invalid cell val num") { CHECK_THROWS_WITH( - TestType(FieldInfo("a1", false, false, 2)), + TestType(FieldInfo("a1", false, false, 2, Datatype::UINT8)), "InputFieldValidator: Aggregate is not supported for non-string fields " "with cell_val_num greater than one."); } @@ -70,7 +72,7 @@ TEST_CASE( "[null-count-aggregator][constructor]") { SECTION("Non nullable") { CHECK_THROWS_WITH( - NullCountAggregator(FieldInfo("a1", false, false, 1)), + NullCountAggregator(FieldInfo("a1", false, false, 1, Datatype::UINT8)), "InputFieldValidator: Aggregate must only be requested for nullable " "fields."); } @@ -95,11 +97,13 @@ typedef tuple< AggUnderTest; TEMPLATE_LIST_TEST_CASE( "Aggregator: var sized", "[aggregator][var-sized]", AggUnderTest) { - auto aggregator{make_aggregator(FieldInfo("a1", false, true, 1))}; + auto aggregator{make_aggregator( + FieldInfo("a1", false, true, 1, Datatype::UINT8))}; CHECK(aggregator.aggregation_var_sized() == false); if constexpr (std::is_same>::value) { - MinAggregator aggregator2(FieldInfo("a1", true, false, 1)); + MinAggregator aggregator2( + FieldInfo("a1", true, false, 1, Datatype::UINT8)); CHECK(aggregator2.aggregation_var_sized() == true); } } @@ -108,7 +112,8 @@ TEMPLATE_LIST_TEST_CASE( "Aggregators: need recompute", "[aggregators][need-recompute]", AggUnderTest) { - auto aggregator{make_aggregator(FieldInfo("a1", false, true, 1))}; + auto aggregator{make_aggregator( + FieldInfo("a1", false, true, 1, Datatype::UINT8))}; bool need_recompute = true; if constexpr (std::is_same>::value) { need_recompute = false; @@ -118,7 +123,8 @@ TEMPLATE_LIST_TEST_CASE( TEMPLATE_LIST_TEST_CASE( "Aggregators: field name", "[aggregator][field-name]", AggUnderTest) { - auto aggregator{make_aggregator(FieldInfo("a1", false, true, 1))}; + auto aggregator{make_aggregator( + FieldInfo("a1", false, true, 1, Datatype::UINT8))}; if (std::is_same::value) { CHECK(aggregator.field_name() == constants::count_of_rows); } else { @@ -135,14 +141,15 @@ TEMPLATE_LIST_TEST_CASE( "Aggregators: Validate buffer", "[aggregators][validate-buffer]", AggUnderTestValidateBuffer) { - TestType aggregator(FieldInfo("a1", false, false, 1)); - TestType aggregator_nullable(FieldInfo("a2", false, true, 1)); + TestType aggregator(FieldInfo("a1", false, false, 1, Datatype::UINT8)); + TestType aggregator_nullable( + FieldInfo("a2", false, true, 1, Datatype::UINT8)); MinAggregator aggregator_var( - FieldInfo("a1", true, false, constants::var_num)); + FieldInfo("a1", true, false, constants::var_num, Datatype::UINT8)); MinAggregator aggregator_var_wrong_cvn( - FieldInfo("a1", true, false, 11)); + FieldInfo("a1", true, false, 11, Datatype::UINT8)); MinAggregator aggregator_fixed_string( - FieldInfo("a1", false, false, 5)); + FieldInfo("a1", false, false, 5, Datatype::UINT8)); std::unordered_map buffers; @@ -317,7 +324,8 @@ TEMPLATE_LIST_TEST_CASE( "Aggregators: Validate buffer count", "[aggregators][validate-buffer]", AggUnderTestValidateBufferCount) { - auto aggregator{make_aggregator(FieldInfo("a1", false, true, 1))}; + auto aggregator{make_aggregator( + FieldInfo("a1", false, true, 1, Datatype::UINT8))}; std::unordered_map buffers; SECTION("Doesn't exist") { @@ -436,12 +444,12 @@ void basic_aggregation_test(std::vector expected_results) { if constexpr (std::is_same::value) { aggregator.emplace(); } else { - aggregator.emplace(FieldInfo("a1", false, false, 1)); + aggregator.emplace(FieldInfo("a1", false, false, 1, tdb_type)); } } - auto aggregator_nullable{ - make_aggregator(FieldInfo("a1", false, true, 1))}; + auto aggregator_nullable{make_aggregator( + FieldInfo("a1", false, true, 1, tdb_type))}; std::unordered_map buffers; @@ -701,7 +709,8 @@ TEMPLATE_LIST_TEST_CASE( TEST_CASE( "Sum aggregator: signed overflow", "[sum-aggregator][signed-overflow]") { - SumAggregator aggregator(FieldInfo("a1", false, false, 1)); + SumAggregator aggregator( + FieldInfo("a1", false, false, 1, tdb_type)); std::unordered_map buffers; @@ -785,7 +794,8 @@ TEST_CASE( TEST_CASE( "Sum aggregator: unsigned overflow", "[sum-aggregator][unsigned-overflow]") { - SumAggregator aggregator(FieldInfo("a1", false, false, 1)); + SumAggregator aggregator( + FieldInfo("a1", false, false, 1, tdb_type)); std::unordered_map buffers; @@ -823,7 +833,7 @@ TEMPLATE_LIST_TEST_CASE( "Sum aggregator: double overflow", "[sum-aggregator][double-overflow]", DoubleAggUnderTest) { - TestType aggregator(FieldInfo("a1", false, false, 1)); + TestType aggregator(FieldInfo("a1", false, false, 1, tdb_type)); std::unordered_map buffers; @@ -899,11 +909,12 @@ void basic_string_aggregation_test(std::vector expected_results) { // Optionally make the aggregator for non nullable values. optional aggregator; if constexpr (!std::is_same::value) { - aggregator.emplace(FieldInfo("a1", true, false, constants::var_num)); + aggregator.emplace(FieldInfo( + "a1", true, false, constants::var_num, tdb_type)); } AGGREGATOR aggregator_nullable( - FieldInfo("a2", true, true, constants::var_num)); + FieldInfo("a2", true, true, constants::var_num, tdb_type)); std::unordered_map buffers; @@ -1110,7 +1121,8 @@ TEST_CASE( TEST_CASE( "NullCount aggregator: output datatype", "[null-count-aggregator][output-datatype]") { - NullCountAggregator aggregator{FieldInfo{"a1", false, true, 1}}; + NullCountAggregator aggregator{ + FieldInfo{"a1", false, true, 1, Datatype::UINT8}}; CHECK(aggregator.output_datatype() == Datatype::UINT64); } From 03ed30659386383cfd1df4df3e6727b2deb1cf9c Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Tue, 24 Oct 2023 01:51:12 -0500 Subject: [PATCH 024/456] Small documentation correction in header example for aggregates (#4447) As discusssed with @KiterLuc the example in the doxygen header is not quite correct so this PR aims to amend it. --- TYPE: NO_HISTORY DESC: Small correction for example use in doxygen header for two aggregates examples --- tiledb/sm/cpp_api/query_experimental.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tiledb/sm/cpp_api/query_experimental.h b/tiledb/sm/cpp_api/query_experimental.h index d70adda77381..3f6eb9cffa42 100644 --- a/tiledb/sm/cpp_api/query_experimental.h +++ b/tiledb/sm/cpp_api/query_experimental.h @@ -366,8 +366,8 @@ class QueryExperimental { * default_channel.apply_aggregate("Count", CountOperation{}); * * uint64_t count = 0; - * uint64_t size = sizeof(uint64_t); - * query.set_data_buffer("Count", &count, &size) + * uint64_t size = 1; + * query.set_data_buffer("Count", &count, size); * query.submit(); * @endcode * @@ -395,8 +395,8 @@ class QueryExperimental { * default_channel.apply_aggregate("Sum", operation); * * double sum = 0; - * uint64_t size = sizeof(double); - * query.set_data_buffer("Sum", &sum, &size) + * uint64_t size = 1; + * query.set_data_buffer("Sum", &sum, size); * query.submit(); * @endcode * From 83e6589a07bd8c6249035b4fb0c5d0b20bc98106 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 24 Oct 2023 05:48:42 -0500 Subject: [PATCH 025/456] Add ArraySchemaEvolution::extend_enumeration (#4445) This change adds a new API for allowing extension of enumerations without requiring that users drop the attribute and enumeration before recreating them from scratch. A general outline of the new API looks like such: ```cpp tiledb::Array array(ctx, array_uri, TILEDB_READ); auto old_enmr = array->get_enumeration("enumeraton_name"); std::vector values = {"some", "values", "to", "extend"}; auto new_enmr = old_enmr->extend(values); tiledb::ArraySchemaEvolution ase(ctx); ase.extend_enumeration(new_enmr); ase.array_evolve(array_uri); ``` --- TYPE: FEATURE | API DESC: Add ArraySchemaEvolution::extend_enumeration --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- test/src/unit-cppapi-enumerations.cc | 134 +++++ test/src/unit-enumerations.cc | 467 +++++++++++++++++- .../api/c_api/enumeration/enumeration_api.cc | 41 ++ .../enumeration_api_experimental.h | 41 ++ .../enumeration/enumeration_api_internal.h | 19 + tiledb/sm/array_schema/array_schema.cc | 45 ++ tiledb/sm/array_schema/array_schema.h | 10 + .../sm/array_schema/array_schema_evolution.cc | 61 ++- .../sm/array_schema/array_schema_evolution.h | 26 + tiledb/sm/array_schema/enumeration.cc | 121 +++++ tiledb/sm/array_schema/enumeration.h | 28 ++ tiledb/sm/c_api/tiledb.cc | 25 + tiledb/sm/c_api/tiledb_experimental.h | 35 ++ tiledb/sm/cpp_api/array_schema_evolution.h | 25 +- tiledb/sm/cpp_api/enumeration_experimental.h | 97 ++++ .../serialization/array_schema_evolution.cc | 53 +- tiledb/sm/serialization/tiledb-rest.capnp | 7 +- tiledb/sm/serialization/tiledb-rest.capnp.c++ | 67 ++- tiledb/sm/serialization/tiledb-rest.capnp.h | 104 +++- tiledb/sm/storage_manager/storage_manager.cc | 21 + 20 files changed, 1374 insertions(+), 53 deletions(-) diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc index 06cb2f234b1a..15990fc00590 100644 --- a/test/src/unit-cppapi-enumerations.cc +++ b/test/src/unit-cppapi-enumerations.cc @@ -33,6 +33,7 @@ #include #include +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" @@ -122,6 +123,112 @@ TEST_CASE_METHOD( REQUIRE(data == values); } +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Extend Fixed Size", + "[enumeration][extend][fixed-size]") { + std::vector init_values = {1, 2, 3, 4, 5}; + std::vector add_values = {6, 7, 8, 9, 10}; + std::vector final_values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr1 = Enumeration::create(ctx_, enmr_name, init_values, true); + auto enmr2 = enmr1.extend(add_values); + + REQUIRE(enmr2.ptr()->is_extension_of(enmr1.ptr().get())); + + auto data = enmr2.as_vector(); + REQUIRE(data == final_values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Extend Var Size", + "[enumeration][extend][var-size]") { + std::vector init_values = {"fred", "wilma"}; + std::vector add_values = {"barney", "betty"}; + std::vector final_values = {"fred", "wilma", "barney", "betty"}; + auto enmr1 = Enumeration::create(ctx_, enmr_name, init_values, true); + auto enmr2 = enmr1.extend(add_values); + + REQUIRE(enmr2.ptr()->is_extension_of(enmr1.ptr().get())); + + auto data = enmr2.as_vector(); + REQUIRE(data == final_values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Fixed Size Empty Vector extension", + "[enumeration][extend][error]") { + std::vector init_values = {1, 2, 3, 4, 5}; + std::vector add_values = {}; + auto enmr = Enumeration::create(ctx_, enmr_name, init_values, true); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Unable to extend an enumeration with an empty vector."); + REQUIRE_THROWS_WITH(enmr.extend(add_values), matcher); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Var Size Empty Vector extension", + "[enumeration][extend][error]") { + std::vector init_values = {"fred", "wilma"}; + std::vector add_values = {}; + auto enmr = Enumeration::create(ctx_, enmr_name, init_values, true); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Unable to extend an enumeration with an empty vector."); + REQUIRE_THROWS_WITH(enmr.extend(add_values), matcher); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Invalid cell val num extension", + "[enumeration][extend][error]") { + std::vector init_values = {1, 2, 3, 4}; + std::vector add_values = {5}; + auto enmr = Enumeration::create( + ctx_, + enmr_name, + TILEDB_INT32, + 2, + true, + init_values.data(), + init_values.size() * sizeof(int), + nullptr, + 0); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid data size is not a multiple of the cell size."); + REQUIRE_THROWS_WITH(enmr.extend(add_values), matcher); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Fixed Size Wrong Type extension", + "[enumeration][extend][error]") { + std::vector init_values = {1, 2, 3, 4, 5}; + std::vector add_values = {"barney", "betty"}; + auto enmr = Enumeration::create(ctx_, enmr_name, init_values, true); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Error extending fixed sized enumeration with var size data."); + REQUIRE_THROWS_WITH(enmr.extend(add_values), matcher); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Var Size Wrong Type extension", + "[enumeration][extend][error]") { + std::vector init_values = {"fred", "wilma"}; + std::vector add_values = {6, 7, 8, 9, 10}; + auto enmr = Enumeration::create(ctx_, enmr_name, init_values, true); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Error extending var sized enumeration with fixed size data."); + REQUIRE_THROWS_WITH(enmr.extend(add_values), matcher); +} + TEST_CASE_METHOD( CPPEnumerationFx, "CPP: Enumeration API - Dump Basic", @@ -239,6 +346,33 @@ TEST_CASE_METHOD( REQUIRE(rc != TILEDB_OK); } +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: ArraySchemaEvolution - Extend Enumeration", + "[enumeration][array-schema-evolution][extend-enumeration]") { + ArraySchemaEvolution ase(ctx_); + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + CHECK_NOTHROW(ase.extend_enumeration(enmr)); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "C API: ArraySchemaEvolution - Extend Enumeration - Check nullptr", + "[enumeration][array-schema-evolution][drop-enumeration]") { + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + + auto rc = tiledb_array_schema_evolution_extend_enumeration( + ctx_.ptr().get(), nullptr, enmr.ptr().get()); + REQUIRE(rc != TILEDB_OK); + + ArraySchemaEvolution ase(ctx_); + rc = tiledb_array_schema_evolution_extend_enumeration( + ctx_.ptr().get(), ase.ptr().get(), nullptr); + REQUIRE(rc != TILEDB_OK); +} + TEST_CASE_METHOD( CPPEnumerationFx, "CPP: ArraySchemaEvolution - Drop Enumeration", diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 8d4fd2d3cb02..1716f7423484 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -75,6 +75,16 @@ struct EnumerationFx { Datatype type = static_cast(255), std::string enmr_name = default_enmr_name); + shared_ptr create_empty_enumeration( + Datatype type, + uint32_t cell_val_num, + bool ordered = false, + std::string enmr_name = default_enmr_name); + + template + shared_ptr extend_enumeration( + shared_ptr enmr, const std::vector& values); + template void check_enumeration( shared_ptr enmr, @@ -693,6 +703,201 @@ TEST_CASE_METHOD( REQUIRE_THROWS(create_enumeration(values)); } +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Fixed Size", + "[enumeration][extension][fixed]") { + std::vector init_values = {1, 2, 3, 4, 5}; + std::vector extend_values = {6, 7, 8, 9, 10}; + std::vector final_values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr1 = create_enumeration(init_values); + auto enmr2 = extend_enumeration(enmr1, extend_values); + check_enumeration( + enmr2, default_enmr_name, final_values, Datatype::INT32, 1, false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Empty Fixed Size", + "[enumeration][extension][fixed]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr1 = create_empty_enumeration(Datatype::INT32, 1); + auto enmr2 = extend_enumeration(enmr1, values); + check_enumeration( + enmr2, default_enmr_name, values, Datatype::INT32, 1, false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Fixed Size Multi-Cell Value", + "[enumeration][extension][fixed]") { + std::vector init_values = {1, 2, 3, 4}; + std::vector extend_values = {5, 6, 7, 8, 9, 10}; + std::vector final_values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr1 = Enumeration::create( + default_enmr_name, + Datatype::INT32, + 2, + false, + init_values.data(), + init_values.size() * sizeof(int), + nullptr, + 0); + auto enmr2 = extend_enumeration(enmr1, extend_values); + check_enumeration( + enmr2, default_enmr_name, final_values, Datatype::INT32, 2, false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Var Size", + "[enumeration][extension][var-sized]") { + std::vector init_values = {"fred", "wilma"}; + std::vector extend_values = {"barney", "betty"}; + std::vector final_values = {"fred", "wilma", "barney", "betty"}; + auto enmr1 = create_enumeration(init_values); + auto enmr2 = extend_enumeration(enmr1, extend_values); + check_enumeration( + enmr2, + default_enmr_name, + final_values, + Datatype::STRING_ASCII, + constants::var_num, + false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Empty Var Size", + "[enumeration][extension][var-sized]") { + std::vector values = {"fred", "wilma", "barney", "betty"}; + auto enmr1 = + create_empty_enumeration(Datatype::STRING_ASCII, constants::var_num); + auto enmr2 = extend_enumeration(enmr1, values); + check_enumeration( + enmr2, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); + REQUIRE(!enmr1->is_extension_of(enmr2)); + REQUIRE(enmr2->is_extension_of(enmr1)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Data", + "[enumeration][extension][error]") { + std::vector init_values = {"fred", "wilma"}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Unable to extend an enumeration without a data buffer."); + REQUIRE_THROWS_WITH(enmr->extend(nullptr, 10, nullptr, 0), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Data Size", + "[enumeration][extension][error]") { + std::vector init_values = {"fred", "wilma"}; + const char* data = "barneybetty"; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Unable to extend an enumeration with a zero sized data buffer."); + REQUIRE_THROWS_WITH(enmr->extend(data, 0, nullptr, 0), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Offsets", + "[enumeration][extension][error]") { + std::vector init_values = {"fred", "wilma"}; + const char* data = "barneybetty"; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "The offsets buffer is required for this enumeration extension."); + REQUIRE_THROWS_WITH(enmr->extend(data, 11, nullptr, 0), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Offsets Size", + "[enumeration][extension][error]") { + std::vector init_values = {"fred", "wilma"}; + const char* data = "barneybetty"; + uint64_t offsets[2] = {0, 6}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "The offsets buffer for " + "this enumeration extension must have a non-zero size."); + REQUIRE_THROWS_WITH(enmr->extend(data, 11, offsets, 0), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Offsets Size not Multiple of 8", + "[enumeration][extension][error]") { + std::vector init_values = {"fred", "wilma"}; + const char* data = "barneybetty"; + uint64_t offsets[2] = {0, 6}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid offsets size is not a multiple of sizeof(uint64_t)"); + REQUIRE_THROWS_WITH(enmr->extend(data, 11, offsets, 17), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Offsets for Fixed Size Data", + "[enumeration][extension][error]") { + std::vector init_values = {0, 1, 2, 3}; + const char* data = "barneybetty"; + uint64_t offsets[2] = {0, 6}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Offsets buffer provided when extending a fixed sized enumeration."); + REQUIRE_THROWS_WITH(enmr->extend(data, 11, offsets, 16), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Offsets Size for Fixed Size Data", + "[enumeration][extension][error]") { + std::vector init_values = {0, 1, 2, 3}; + std::vector add_values = {4, 5, 6, 7}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Offsets size is non-zero when extending a fixed sized enumeration."); + REQUIRE_THROWS_WITH( + enmr->extend( + add_values.data(), add_values.size() * sizeof(int), nullptr, 16), + matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Extension Invalid Duplicate Value", + "[enumeration][extension][error]") { + std::vector init_values = {0, 1, 2, 3}; + std::vector add_values = {2, 3, 4, 5}; + auto enmr = create_enumeration(init_values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid duplicated value in enumeration"); + REQUIRE_THROWS_WITH( + enmr->extend( + add_values.data(), add_values.size() * sizeof(int), nullptr, 0), + matcher); +} + TEST_CASE_METHOD( EnumerationFx, "Enumeration Serialization - Fixed Size", @@ -1259,6 +1464,90 @@ TEST_CASE_METHOD( REQUIRE_THROWS(schema->drop_enumeration("not_an_enumeration")); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration is nullptr", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto matcher = Catch::Matchers::ContainsSubstring( + "Error adding enumeration. Enumeration must not be nullptr."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(nullptr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Does Not Exist", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "foo"); + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration with name 'foo' does not exist in this ArraySchema."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Not Loaded", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr"); + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration with name 'test_enmr' is not loaded."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Enumeration Not An Extension", + "[enumeration][array-schema][error]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = make_shared(HERE(), array->array_schema_latest()); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr"); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Provided enumeration is not an extension of the current state of " + "'test_enmr'"); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Extend Enumeration - Duplicate Enumeration Path Name", + "[enumeration][array-schema][error]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = make_shared(HERE(), array->array_schema_latest()); + auto enmr1 = schema->get_enumeration("test_enmr"); + + std::vector extra_values = {"manatee", "narwhal", "oppossum"}; + auto enmr2 = extend_enumeration(enmr1, extra_values); + + // We have to force this condition by hand + auto enmr3 = tiledb::sm::Enumeration::create( + enmr2->name(), + // Notice we're reusing the existing path name from enmr1 + enmr1->path_name(), + enmr2->type(), + enmr2->cell_val_num(), + enmr2->ordered(), + enmr2->data().data(), + enmr2->data().size(), + enmr2->offsets().data(), + enmr2->offsets().size()); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration path name for 'test_enmr' already exists in this schema."); + REQUIRE_THROWS_WITH(schema->extend_enumeration(enmr3), matcher); +} + /* ********************************* */ /* Testing ArraySchemaEvolution */ /* ********************************* */ @@ -1357,10 +1646,37 @@ TEST_CASE_METHOD( CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Extend", + "[enumeration][array-schema-evolution][enmr-to-extend]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + auto orig_schema = array->array_schema_latest_ptr(); + + std::vector values_to_add = {"firefly", "gerbil", "hamster"}; + auto old_enmr = orig_schema->get_enumeration("test_enmr"); + REQUIRE(old_enmr != nullptr); + auto new_enmr = extend_enumeration(old_enmr, values_to_add); + + auto ase = make_shared(HERE()); + ase->extend_enumeration(new_enmr); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", - "[enumeration][array-schema-evolution][enmr-to-add]") { + "[enumeration][array-schema-evolution][enmr-to-drop]") { + auto ase = make_shared(HERE()); + CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Add Then Drop Enumeration", + "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); auto ase1 = make_shared(HERE()); @@ -1377,14 +1693,6 @@ TEST_CASE_METHOD( CHECK_NOTHROW(ase2->evolve_schema(new_schema)); } -TEST_CASE_METHOD( - EnumerationFx, - "ArraySchemaEvolution - Drop Enumeration", - "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared(HERE()); - CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); -} - TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", @@ -1512,6 +1820,30 @@ TEST_CASE_METHOD( REQUIRE_THROWS(ase->evolve_schema(orig_schema)); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Extend Enumeration nullptr", + "[enumeration][array-schema-evolution][extend][error]") { + auto ase = make_shared(HERE()); + auto matcher = Catch::Matchers::ContainsSubstring( + "Cannot extend enumeration; Input enumeration is null"); + REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Extend Enumeration Already Extended", + "[enumeration][array-schema-evolution][extend][error]") { + auto ase = make_shared(HERE()); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + auto matcher = Catch::Matchers::ContainsSubstring( + "Cannot extend enumeration; Input enumeration name has already " + "been extended in this evolution."); + REQUIRE_NOTHROW(ase->extend_enumeration(enmr)); + REQUIRE_THROWS_WITH(ase->extend_enumeration(enmr), matcher); +} + /* ********************************* */ /* Testing QueryCondition */ /* ********************************* */ @@ -1556,6 +1888,45 @@ TEST_CASE_METHOD( REQUIRE(data2.rvalue_as() == 2); } +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Enumeration Value After Extension", + "[enumeration][query-condition][extend][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + array->load_all_enumerations(); + + auto schema = array->array_schema_latest_ptr(); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", std::string("gerbil"), QueryConditionOp::EQ); + auto qc2 = qc1; + + // Check that we fail the rewrite before extension. + auto matcher = Catch::Matchers::ContainsSubstring( + "Enumeration value not found for field 'attr1'"); + REQUIRE_THROWS_WITH( + qc1.rewrite_enumeration_conditions(*(schema.get())), matcher); + + // Extend enumeration via schema evolution. + std::vector values_to_add = {"firefly", "gerbil", "hamster"}; + auto old_enmr = schema->get_enumeration("test_enmr"); + auto new_enmr = extend_enumeration(old_enmr, values_to_add); + + auto ase = make_shared(HERE()); + ase->extend_enumeration(new_enmr); + auto st = ctx_.storage_manager()->array_evolve_schema( + array->array_uri(), ase.get(), array->get_encryption_key()); + throw_if_not_ok(st); + + // Check that we can not rewrite the query condition. + array = get_array(QueryType::READ); + array->load_all_enumerations(); + schema = array->array_schema_latest_ptr(); + + REQUIRE_NOTHROW(qc2.rewrite_enumeration_conditions(*(schema.get()))); +} + TEST_CASE_METHOD( EnumerationFx, "QueryCondition - Skip enumeration rewrite", @@ -1774,6 +2145,7 @@ TEST_CASE_METHOD( auto enmrs_to_add1 = ase1.enumeration_names_to_add(); auto enmrs_to_add2 = ase2->enumeration_names_to_add(); + REQUIRE(enmrs_to_add1.size() == 2); REQUIRE(vec_cmp(enmrs_to_add1, enmrs_to_add2)); for (auto& name : enmrs_to_add1) { @@ -1783,6 +2155,38 @@ TEST_CASE_METHOD( } } +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - ArraySchemaEvolution Serialization With Extensions", + "[enumeration][capnp][basic][array-schema-evolution]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + std::vector values1 = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values1, false, Datatype::INT32, "enmr1"); + + std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; + auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); + + ArraySchemaEvolution ase1; + ase1.extend_enumeration(enmr1); + ase1.extend_enumeration(enmr2); + + auto ase2 = ser_des_array_schema_evolution(&ase1, client_side, ser_type); + + auto enmrs_to_extend1 = ase1.enumeration_names_to_extend(); + auto enmrs_to_extend2 = ase2->enumeration_names_to_extend(); + REQUIRE(enmrs_to_extend2.size() == 2); + REQUIRE(vec_cmp(enmrs_to_extend1, enmrs_to_extend2)); + + for (auto& name : enmrs_to_extend1) { + REQUIRE(ase1.enumeration_to_extend(name) != nullptr); + REQUIRE(ase2->enumeration_to_extend(name) != nullptr); + REQUIRE( + ase1.enumeration_to_extend(name) != ase2->enumeration_to_extend(name)); + } +} + TEST_CASE_METHOD( EnumerationFx, "Cap'N Proto - Basic Backwards Compatible Query Serialization", @@ -2036,6 +2440,51 @@ shared_ptr EnumerationFx::create_enumeration( } } +shared_ptr EnumerationFx::create_empty_enumeration( + Datatype type, uint32_t cell_val_num, bool ordered, std::string name) { + return Enumeration::create( + name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0); +} + +template +shared_ptr EnumerationFx::extend_enumeration( + shared_ptr enmr, const std::vector& values) { + if constexpr (std::is_same_v) { + // We have to call out bool specifically because of the stdlib + // specialization for std::vector + std::vector raw_values(values.size()); + for (size_t i = 0; i < values.size(); i++) { + raw_values[i] = values[i] ? 1 : 0; + } + return enmr->extend( + raw_values.data(), raw_values.size() * sizeof(uint8_t), nullptr, 0); + } else if constexpr (std::is_pod_v) { + return enmr->extend(values.data(), values.size() * sizeof(T), nullptr, 0); + } else { + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size(); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size()); + offsets.push_back(curr_offset); + curr_offset += v.size(); + } + + return enmr->extend( + data.data(), + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } +} + template void EnumerationFx::check_enumeration( shared_ptr enmr, diff --git a/tiledb/api/c_api/enumeration/enumeration_api.cc b/tiledb/api/c_api/enumeration/enumeration_api.cc index 797161f2c081..73283621af2d 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api.cc +++ b/tiledb/api/c_api/enumeration/enumeration_api.cc @@ -82,6 +82,29 @@ capi_return_t tiledb_enumeration_alloc( return TILEDB_OK; } +capi_return_t tiledb_enumeration_extend( + tiledb_enumeration_t* old_enumeration, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** new_enumeration) { + ensure_enumeration_is_valid(old_enumeration); + ensure_output_pointer_is_valid(new_enumeration); + + auto new_enmr = + old_enumeration->extend(data, data_size, offsets, offsets_size); + + try { + *new_enumeration = tiledb_enumeration_handle_t::make_handle(new_enmr); + } catch (...) { + *new_enumeration = nullptr; + throw; + } + + return TILEDB_OK; +} + void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) { ensure_output_pointer_is_valid(enumeration); ensure_enumeration_is_valid(*enumeration); @@ -180,6 +203,24 @@ capi_return_t tiledb_enumeration_alloc( enumeration); } +capi_return_t tiledb_enumeration_extend( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* old_enumeration, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** new_enumeration) noexcept { + return api_entry_context( + ctx, + old_enumeration, + data, + data_size, + offsets, + offsets_size, + new_enumeration); +} + void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) noexcept { return api_entry_void(enumeration); } diff --git a/tiledb/api/c_api/enumeration/enumeration_api_experimental.h b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h index 46326047661b..b214ba86fbd8 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api_experimental.h +++ b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h @@ -93,6 +93,47 @@ TILEDB_EXPORT capi_return_t tiledb_enumeration_alloc( uint64_t offsets_size, tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; +/** + * Extend an Enumeration. + * + * This API exists for users that wish to just add more enumeration values to + * an existing enumeration via array schema evolution. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* old_enumeration = load_existing_enumeration(); + * tiledb_enumeration_t* new_enumeration; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_extend( + * ctx, + * old_enumeration + * data, + * data_size, + * nullptr, + * 0, + * &new_enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param old_enumeration The enumeration to extend. + * @param data A pointer to the enumeration value data to add. + * @param data_size The length of the data buffer provided. + * @param offsets A pointer to the offsets buffer if enumeration is var sized. + * @param offsets_size The length of the offsets buffer, zero if no offsets. + * @param new_enumeration The newly created extended enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_extend( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* old_enumeration, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** new_enumeration) TILEDB_NOEXCEPT; + /** * Destroys a TileDB enumeration, freeing associated memory. * diff --git a/tiledb/api/c_api/enumeration/enumeration_api_internal.h b/tiledb/api/c_api/enumeration/enumeration_api_internal.h index 2a984d27e03a..ea1d1398ba4a 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api_internal.h +++ b/tiledb/api/c_api/enumeration/enumeration_api_internal.h @@ -71,6 +71,25 @@ struct tiledb_enumeration_handle_t return enumeration_; } + /** + * Extend a given enumeration. + */ + [[nodiscard]] shared_ptr extend( + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) const { + return enumeration_->extend(data, data_size, offsets, offsets_size); + } + + /** + * Returns wether or not an enumeration is an extension of this one. + */ + [[nodiscard]] inline bool is_extension_of( + tiledb_enumeration_handle_t* rhs) const { + return enumeration_->is_extension_of(rhs->enumeration_); + } + /** * Return the name of the enumeration. */ diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 68374d690dee..88c263582ff4 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1070,6 +1070,51 @@ void ArraySchema::add_enumeration(shared_ptr enmr) { enumeration_path_map_[enmr->name()] = enmr->path_name(); } +void ArraySchema::extend_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaException( + "Error adding enumeration. Enumeration must not be nullptr."); + } + + auto it = enumeration_map_.find(enmr->name()); + if (it == enumeration_map_.end()) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration with name '" + enmr->name() + + "' does not exist in this ArraySchema."); + } + + if (it->second == nullptr) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration with name '" + enmr->name() + + "' is not loaded."); + } + + if (!enmr->is_extension_of(it->second)) { + throw ArraySchemaException( + "Error extending enumeration. Provided enumeration is not an extension " + "of the current state of '" + + enmr->name() + "'"); + } + + if (enumeration_path_map_.find(enmr->name()) == enumeration_path_map_.end()) { + throw ArraySchemaException( + "Error extending enumeration. Invalid enumeration path map state for " + "enumeration '" + + enmr->name() + "'"); + } + + for (auto& enmr_path : enumeration_path_map_) { + if (enmr->path_name() == enmr_path.second) { + throw ArraySchemaException( + "Error extending enumeration. Enumeration path name for '" + + enmr->name() + "' already exists in this schema."); + } + } + + enumeration_map_[enmr->name()] = enmr; + enumeration_path_map_[enmr->name()] = enmr->path_name(); +} + void ArraySchema::store_enumeration(shared_ptr enmr) { if (enmr == nullptr) { throw ArraySchemaException( diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index 934389284b52..8a352df7550a 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -387,6 +387,16 @@ class ArraySchema { */ void add_enumeration(shared_ptr enmr); + /** + * Extend an Enumeration on this ArraySchema. + * + * N.B., this method is intended to be called via ArraySchemaEvolution. + * Calling it from anywhere else is likely incorrect. + * + * @param enmr The extended enumeration. + */ + void extend_enumeration(shared_ptr enmr); + /** * Check if an enumeration exists with the given name. * diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 6c6fb66809a1..7ef355d05a5f 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -79,11 +79,14 @@ ArraySchemaEvolution::ArraySchemaEvolution( std::unordered_map> attrs_to_add, std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, + std::unordered_map> + enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range) : attributes_to_add_map_(attrs_to_add) , attributes_to_drop_(attrs_to_drop) , enumerations_to_add_map_(enmrs_to_add) + , enumerations_to_extend_map_(enmrs_to_extend) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { } @@ -112,6 +115,10 @@ shared_ptr ArraySchemaEvolution::evolve_schema( schema->add_enumeration(enmr.second); } + for (auto& enmr : enumerations_to_extend_map_) { + schema->extend_enumeration(enmr.second); + } + // Add attributes. for (auto& attr : attributes_to_add_map_) { throw_if_not_ok(schema->add_attribute(attr.second, false)); @@ -250,15 +257,63 @@ shared_ptr ArraySchemaEvolution::enumeration_to_add( return it->second; } +void ArraySchemaEvolution::extend_enumeration( + shared_ptr enmr) { + std::lock_guard lock(mtx_); + + if (enmr == nullptr) { + throw ArraySchemaEvolutionException( + "Cannot extend enumeration; Input enumeration is null"); + } + + auto it = enumerations_to_extend_map_.find(enmr->name()); + if (it != enumerations_to_extend_map_.end()) { + throw ArraySchemaEvolutionException( + "Cannot extend enumeration; Input enumeration name has already " + "been extended in this evolution."); + } + + enumerations_to_extend_map_[enmr->name()] = enmr; +} + +std::vector ArraySchemaEvolution::enumeration_names_to_extend() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_extend_map_.size()); + for (auto elem : enumerations_to_extend_map_) { + names.push_back(elem.first); + } + + return names; +} + +shared_ptr ArraySchemaEvolution::enumeration_to_extend( + const std::string& name) const { + std::lock_guard lock(mtx_); + auto it = enumerations_to_extend_map_.find(name); + + if (it == enumerations_to_extend_map_.end()) { + return nullptr; + } + + return it->second; +} + void ArraySchemaEvolution::drop_enumeration( const std::string& enumeration_name) { std::lock_guard lock(mtx_); enumerations_to_drop_.insert(enumeration_name); - auto it = enumerations_to_add_map_.find(enumeration_name); - if (it != enumerations_to_add_map_.end()) { + auto add_it = enumerations_to_add_map_.find(enumeration_name); + if (add_it != enumerations_to_add_map_.end()) { // Reset the pointer and erase it - enumerations_to_add_map_.erase(it); + enumerations_to_add_map_.erase(add_it); + } + + auto extend_it = enumerations_to_extend_map_.find(enumeration_name); + if (extend_it != enumerations_to_extend_map_.end()) { + enumerations_to_extend_map_.erase(extend_it); } } diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 14d98b98d0d7..be2f2bead0b0 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -83,6 +83,8 @@ class ArraySchemaEvolution { std::unordered_set attrs_to_drop, std::unordered_map> enmrs_to_add, + std::unordered_map> + enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range); @@ -142,6 +144,26 @@ class ArraySchemaEvolution { shared_ptr enumeration_to_add( const std::string& name) const; + /** + * Extend an enumeration. + * + * @param enmr The enumeration with its extension. + */ + void extend_enumeration(shared_ptr enmr); + + /** Returns the names of the enumerations to extend. */ + std::vector enumeration_names_to_extend() const; + + /** + * Returns a constant pointer to the selected enumeration or nullptr if it + * does not exist. + * + * @param name The name of the enumeration to extend. + * @return shared_ptr The enumeration to extend. + */ + shared_ptr enumeration_to_extend( + const std::string& name) const; + /** * Drops an enumeration * @@ -175,6 +197,10 @@ class ArraySchemaEvolution { std::unordered_map> enumerations_to_add_map_; + /** Enumerations to extend. */ + std::unordered_map> + enumerations_to_extend_map_; + /** The names of array enumerations to be dropped. */ std::unordered_set enumerations_to_drop_; diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index dbd527b34fcc..de6d3a270491 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -227,6 +227,127 @@ shared_ptr Enumeration::deserialize( offsets_size); } +shared_ptr Enumeration::extend( + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) const { + if (data == nullptr) { + throw EnumerationException( + "Unable to extend an enumeration without a data buffer."); + } + + if (data_size == 0) { + throw EnumerationException( + "Unable to extend an enumeration with a zero sized data buffer."); + } + + if (var_size()) { + if (offsets == nullptr) { + throw EnumerationException( + "The offsets buffer is required for this enumeration extension."); + } + + if (offsets_size == 0) { + throw EnumerationException( + "The offsets buffer for this enumeration extension must " + "have a non-zero size."); + } + + if (offsets_size % sizeof(uint64_t) != 0) { + throw EnumerationException( + "Invalid offsets size is not a multiple of sizeof(uint64_t)"); + } + } else { + if (offsets != nullptr) { + throw EnumerationException( + "Offsets buffer provided when extending a fixed sized enumeration."); + } + if (offsets_size != 0) { + throw EnumerationException( + "Offsets size is non-zero when extending a fixed sized enumeration."); + } + } + + Buffer new_data(data_.size() + data_size); + throw_if_not_ok(new_data.write(data_.data(), data_.size())); + throw_if_not_ok(new_data.write(data, data_size)); + + const void* new_offsets_ptr = nullptr; + uint64_t new_offsets_size = 0; + + Buffer new_offsets(offsets_.size() + offsets_size); + + if (var_size()) { + // First we write our existing offsets + throw_if_not_ok(new_offsets.write(offsets_.data(), offsets_.size())); + + // All new offsets have to be rewritten to be relative to the length + // of the current data array. + const uint64_t* offsets_arr = static_cast(offsets); + uint64_t num_offsets = offsets_size / sizeof(uint64_t); + for (uint64_t i = 0; i < num_offsets; i++) { + uint64_t new_offset = offsets_arr[i] + data_.size(); + throw_if_not_ok(new_offsets.write(&new_offset, sizeof(uint64_t))); + } + + new_offsets_ptr = new_offsets.data(); + new_offsets_size = new_offsets.size(); + } + + return create( + name_, + "", + type_, + cell_val_num_, + ordered_, + new_data.data(), + new_data.size(), + new_offsets_ptr, + new_offsets_size); +} + +bool Enumeration::is_extension_of(shared_ptr other) const { + if (name_ != other->name()) { + return false; + } + + if (type_ != other->type()) { + return false; + } + + if (cell_val_num_ != other->cell_val_num()) { + return false; + } + + if (ordered_ != other->ordered()) { + return false; + } + + auto other_data = other->data(); + if (data_.size() <= other_data.size()) { + return false; + } + + if (std::memcmp(data_.data(), other_data.data(), other_data.size()) != 0) { + return false; + } + + if (var_size()) { + auto other_offsets = other->offsets(); + if (offsets_.size() <= other_offsets.size()) { + return false; + } + + if (std::memcmp( + offsets_.data(), other_offsets.data(), other_offsets.size()) != 0) { + return false; + } + } + + return true; +} + void Enumeration::serialize(Serializer& serializer) const { serializer.write(constants::enumerations_version); diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h index 2c62a1008306..82e37ce78159 100644 --- a/tiledb/sm/array_schema/enumeration.h +++ b/tiledb/sm/array_schema/enumeration.h @@ -178,6 +178,34 @@ class Enumeration { */ static shared_ptr deserialize(Deserializer& deserializer); + /** + * Create a new enumeration by extending an existing enumeration's + * list of values. + * + * The returned Enumeration can then be used by the + * ArraySchemaEvolution::extend_enumeration to update the schema. + * + * @param data A pointer to the enumerations values to add. + * @param data_size The length of the buffer pointed to by data. + * @param offsets A pointer to a buffer of offsets data. Must be provided + * if and only if the enumeration being extended is var sized. + * @param offsets_size The size of the offsets buffer or zero if no buffer + * is supplied. + * @return shared_ptr The extended enumeration. + */ + shared_ptr extend( + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) const; + + /** + * Check if this enumeration is an extension of the provided Enumeration. + * + * @return bool Whether this enumeration is an extension or not. + */ + bool is_extension_of(shared_ptr other) const; + /** * Serializes the enumeration into a buffer. * diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 111474dd9d41..10d55edc3c3c 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -917,6 +917,22 @@ capi_return_t tiledb_array_schema_evolution_add_enumeration( return TILEDB_OK; } +capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + auto enmr = enumeration->copy(); + array_schema_evolution->array_schema_evolution_->extend_enumeration(enmr); + + return TILEDB_OK; +} + capi_return_t tiledb_array_schema_evolution_drop_enumeration( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -5650,6 +5666,15 @@ int32_t tiledb_array_schema_evolution_add_enumeration( ctx, array_schema_evolution, enmr); } +capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enmr) noexcept { + return api_entry< + tiledb::api::tiledb_array_schema_evolution_extend_enumeration>( + ctx, array_schema_evolution, enmr); +} + capi_return_t tiledb_array_schema_evolution_drop_enumeration( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 69128ce599d8..a814a34043e3 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -212,6 +212,41 @@ TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_add_enumeration( tiledb_array_schema_evolution_t* array_schema_evolution, tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; +/** + * Extends an enumeration during array schema evolution. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* original_enmr = get_existing_enumeration(); + * const void* data = get_new_data(); + * uint64_t data_size = get_new_data_size(); + * tiledb_enumeration_t* new_enmr; + * tiledb_enumeration_extend( + * ctx, + * original_enmr, + * data, + * data_size, + * nullptr, + * 0, + * &new_enmr); + * tiledb_array_schema_evolution_extend_enumeration( + * ctx, + * array_schema_evolution, + * enmr); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param enumeration The enumeration to be extended. This should be the result + * of a call to tiledb_enumeration_extend. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_extend_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + /** * Drops an enumeration from an array schema evolution. * diff --git a/tiledb/sm/cpp_api/array_schema_evolution.h b/tiledb/sm/cpp_api/array_schema_evolution.h index a8397974e0a4..31cfa5523194 100644 --- a/tiledb/sm/cpp_api/array_schema_evolution.h +++ b/tiledb/sm/cpp_api/array_schema_evolution.h @@ -163,7 +163,7 @@ class ArraySchemaEvolution { * values)); * @endcode * - * @param attr The Attribute to add + * @param enmr The Enumeration to add. * @return Reference to this `ArraySchemaEvolution` instance. */ ArraySchemaEvolution& add_enumeration(const Enumeration& enmr) { @@ -173,6 +173,29 @@ class ArraySchemaEvolution { return *this; } + /** + * Extends an Enumeration during array schema evolution. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Enumeration old_enmr = array->get_enumeration("some_enumeration"); + * std::vector new_values = {"cyan", "magenta", "mauve"}; + * tiledb::Enumeration new_enmr = old_enmr->extend(new_values); + * tiledb::ArraySchemaEvolution schema_evolution(ctx); + * schema_evolution.extend_enumeration(new_enmr); + * @endcode + * + * @param enmr The Enumeration to extend. + * @return Reference to this `ArraySchemaEvolution` instance. + */ + ArraySchemaEvolution& extend_enumeration(const Enumeration& enmr) { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_schema_evolution_extend_enumeration( + ctx.ptr().get(), evolution_.get(), enmr.ptr().get())); + return *this; + } + /** * Drops an enumeration. * diff --git a/tiledb/sm/cpp_api/enumeration_experimental.h b/tiledb/sm/cpp_api/enumeration_experimental.h index 3bbd76b44180..950f94537976 100644 --- a/tiledb/sm/cpp_api/enumeration_experimental.h +++ b/tiledb/sm/cpp_api/enumeration_experimental.h @@ -76,6 +76,103 @@ class Enumeration { return enumeration_; } + /** + * Extend this enumeration using the provided values. + * + * @param values The values to extend the enumeration with. + * @return The newly created enumeration. + */ + template * = nullptr> + Enumeration extend(std::vector values) { + if (values.size() == 0) { + throw TileDBError( + "Unable to extend an enumeration with an empty vector."); + } + + if (cell_val_num() == TILEDB_VAR_NUM) { + throw TileDBError( + "Error extending var sized enumeration with fixed size data."); + } + + if constexpr (std::is_same_v) { + // This awkward dance for std::vector is due to the fact that + // C++ provides a template specialization that uses a bitmap which does + // not provide `data()` member method. + std::vector converted(values.size()); + for (size_t i = 0; i < values.size(); i++) { + converted[i] = values[i] ? 1 : 0; + } + + return extend(converted.data(), converted.size() * sizeof(T), nullptr, 0); + } else { + return extend(values.data(), values.size() * sizeof(T), nullptr, 0); + } + } + + /** + * Extend this enumeration using the provided string values. + * + * @param values The string values to extend the enumeration with. + * @return The newly created enumeration. + */ + template * = nullptr> + Enumeration extend(std::vector>& values) { + if (values.size() == 0) { + throw TileDBError( + "Unable to extend an enumeration with an empty vector."); + } + + if (cell_val_num() != TILEDB_VAR_NUM) { + throw TileDBError( + "Error extending fixed sized enumeration with var size data."); + } + + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size() * sizeof(T); + } + + std::vector data(total_size, 0); + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data.data() + curr_offset, v.data(), v.size() * sizeof(T)); + offsets.push_back(curr_offset); + curr_offset += v.size() * sizeof(T); + } + + return extend( + data.data(), + data.size(), + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } + + /** + * Extend this enumeration using the provided pointers. + * + * @return The newly created enumeration. + */ + Enumeration extend( + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + tiledb_enumeration_t* old_enmr = enumeration_.get(); + tiledb_enumeration_t* new_enmr = nullptr; + ctx_.get().handle_error(tiledb_enumeration_extend( + ctx_.get().ptr().get(), + old_enmr, + data, + data_size, + offsets, + offsets_size, + &new_enmr)); + return Enumeration(ctx_, new_enmr); + } + /** * Get the name of the enumeration * diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index ae993b384a4c..c60b9882a674 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -105,13 +105,13 @@ Status array_schema_evolution_to_capnp( attribute_to_capnp(attr_to_add, &attribute_builder); } + // Enumerations to add auto enmr_names_to_add = array_schema_evolution->enumeration_names_to_add(); - auto num_enmrs = enmr_names_to_add.size(); - - if (num_enmrs > 0) { + if (enmr_names_to_add.size() > 0) { auto enmrs_to_add_builder = - array_schema_evolution_builder->initEnumerationsToAdd(num_enmrs); - for (size_t i = 0; i < num_enmrs; i++) { + array_schema_evolution_builder->initEnumerationsToAdd( + enmr_names_to_add.size()); + for (size_t i = 0; i < enmr_names_to_add.size(); i++) { auto enmr = array_schema_evolution->enumeration_to_add(enmr_names_to_add[i]); auto builder = enmrs_to_add_builder[i]; @@ -119,16 +119,31 @@ Status array_schema_evolution_to_capnp( } } - // Enumerations to drop - std::vector enmr_names_to_drop = - array_schema_evolution->enumeration_names_to_drop(); - - auto enumerations_to_drop_builder = - array_schema_evolution_builder->initEnumerationsToDrop( - enmr_names_to_drop.size()); + // Enumerations to extend + auto enmr_names_to_extend = + array_schema_evolution->enumeration_names_to_extend(); + if (enmr_names_to_extend.size() > 0) { + auto enmrs_to_extend_builder = + array_schema_evolution_builder->initEnumerationsToExtend( + enmr_names_to_extend.size()); + for (size_t i = 0; i < enmr_names_to_extend.size(); i++) { + auto enmr = array_schema_evolution->enumeration_to_extend( + enmr_names_to_extend[i]); + auto builder = enmrs_to_extend_builder[i]; + enumeration_to_capnp(enmr, builder); + } + } - for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { - enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + // Enumerations to drop + auto enmr_names_to_drop = array_schema_evolution->enumeration_names_to_drop(); + if (enmr_names_to_drop.size() > 0) { + auto enumerations_to_drop_builder = + array_schema_evolution_builder->initEnumerationsToDrop( + enmr_names_to_drop.size()); + + for (size_t i = 0; i < enmr_names_to_drop.size(); i++) { + enumerations_to_drop_builder.set(i, enmr_names_to_drop[i]); + } } auto timestamp_builder = @@ -166,6 +181,15 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_add[enmr->name()] = enmr; } + // Create enumerations to extend + std::unordered_map> + enmrs_to_extend; + auto enmrs_to_extend_reader = evolution_reader.getEnumerationsToExtend(); + for (auto enmr_reader : enmrs_to_extend_reader) { + auto enmr = enumeration_from_capnp(enmr_reader); + enmrs_to_extend[enmr->name()] = enmr; + } + // Create enumerations to drop std::unordered_set enmrs_to_drop; auto enmrs_to_drop_reader = evolution_reader.getEnumerationsToDrop(); @@ -187,6 +211,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( attrs_to_add, attrs_to_drop, enmrs_to_add, + enmrs_to_extend, enmrs_to_drop, ts_range)); } diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 77fb19c6886b..6535efeb0c62 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -179,6 +179,9 @@ struct ArraySchemaEvolution { enumerationsToDrop @4 :List(Text); # Enumeration names to be dropped + + enumerationsToExtend @5 :List(Enumeration); + # Enumerations to be extended. } struct Attribute { @@ -1259,7 +1262,7 @@ struct LoadArraySchemaResponse { } struct QueryChannel { - # structure representing a query channel, that is a stream of data within + # structure representing a query channel, that is a stream of data within # a TileDB query. Such channels can be generated for the purpose of avoiding # processing result items multiple times in more complex queries such as e.g. # grouping queries. @@ -1281,6 +1284,6 @@ struct Aggregate { # name of the input field the aggregate is applied on name @2 :Text; - # the name of aggregate, e.g. COUNT, MEAN, SUM used for constructing the + # the name of aggregate, e.g. COUNT, MEAN, SUM used for constructing the # correct object during deserialization } diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 5e6afe2644a1..d2d8f5821fe3 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -1157,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<143> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 31, 1, 0, 0, + 33, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1176,42 +1176,49 @@ static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 20, 0, 0, 0, 3, 0, 4, 0, + 24, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 125, 0, 0, 0, 138, 0, 0, 0, + 153, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 128, 0, 0, 0, 3, 0, 1, 0, - 156, 0, 0, 0, 2, 0, 1, 0, + 156, 0, 0, 0, 3, 0, 1, 0, + 184, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 130, 0, 0, 0, + 181, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 122, 0, 0, 0, + 205, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 146, 0, 0, 0, + 229, 0, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 0, 0, 0, 3, 0, 1, 0, - 232, 0, 0, 0, 2, 0, 1, 0, + 232, 0, 0, 0, 3, 0, 1, 0, + 4, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 229, 0, 0, 0, 154, 0, 0, 0, + 1, 1, 0, 0, 154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 4, 1, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 32, 1, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 1, 0, 0, 170, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 32, 1, 0, 0, 3, 0, 1, 0, + 60, 1, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1277,6 +1284,20 @@ static const ::capnp::_::AlignedData<122> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 69, 120, + 116, 101, 110, 100, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1287,11 +1308,11 @@ static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 5, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 122, d_a1b81d67548230d4, m_a1b81d67548230d4, - 2, 5, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 143, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 6, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index f4f8b9694c14..dcd1f3aeb43f 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -246,7 +246,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 5) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 6) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -3150,6 +3150,12 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader getEnumerationsToDrop() const; + inline bool hasEnumerationsToExtend() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToExtend() const; + private: ::capnp::_::StructReader _reader; template @@ -3276,6 +3282,28 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> disownEnumerationsToDrop(); + inline bool hasEnumerationsToExtend(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToExtend(); + inline void setEnumerationsToExtend( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToExtend(unsigned int size); + inline void adoptEnumerationsToExtend( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToExtend(); + private: ::capnp::_::StructBuilder _builder; template @@ -17884,6 +17912,80 @@ ArraySchemaEvolution::Builder::disownEnumerationsToDrop() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToExtend() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToExtend() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToExtend() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToExtend() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToExtend( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToExtend(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToExtend( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToExtend() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<5>() * + ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index e7377c6658b6..6b5cfb32a34b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -755,6 +755,27 @@ Status StorageManager::array_evolve_schema( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); + // Load required enumerations before evolution. + auto enmr_names = schema_evolution->enumeration_names_to_extend(); + if (enmr_names.size() > 0) { + std::unordered_set enmr_path_set; + for (auto name : enmr_names) { + enmr_path_set.insert(array_schema->get_enumeration_path_name(name)); + } + std::vector enmr_paths; + for (auto path : enmr_path_set) { + enmr_paths.emplace_back(path); + } + + MemoryTracker tracker; + auto loaded_enmrs = array_dir.load_enumerations_from_paths( + enmr_paths, encryption_key, tracker); + + for (auto enmr : loaded_enmrs) { + array_schema->store_enumeration(enmr); + } + } + // Evolve schema auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); From 9092156f44981f6b3c548d0a2ddba369791d84e6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 25 Oct 2023 01:37:35 -0500 Subject: [PATCH 026/456] Allow forward slashes in enumeration names (#4454) Originally I was going to use the enumeration name in the URI and so disallowed names from containing `/` characters. Given that we don't put names in URIs this is no longer a valid requirement. --- TYPE: IMPROVEMENT DESC: Allow forward slashes in enumeration names --- test/src/unit-enumerations.cc | 5 +++-- tiledb/sm/array_schema/enumeration.cc | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 1716f7423484..7840f81c3475 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -495,11 +495,12 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( EnumerationFx, - "Enumeration Creation Error - Invalid name with slash", + "Enumeration Creation Error - Invalid path_name with slash", "[enumeration][error][invalid-name]") { std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; REQUIRE_THROWS(Enumeration::create( - "an/enumeration", + default_enmr_name, + "an/bad/path", Datatype::INT32, 2, false, diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index de6d3a270491..86ad30d784a2 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -77,9 +77,9 @@ Enumeration::Enumeration( "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); } - if (name.find("/") != std::string::npos) { + if (path_name.find("/") != std::string::npos) { throw EnumerationException( - "Enumeration name must not contain path separators"); + "Enumeration path name must not contain path separators"); } if (cell_val_num == 0) { From bf65f07d2034ff1fa8f3dc4a03e90e52794a6226 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:52:13 +0200 Subject: [PATCH 027/456] Channel operation needs virtual destructor. (#4456) This came out of the maria DB nightlies returning the following error: /usr/local/include/tiledb/channel_operation.h:71:3: error: 'tiledb::ChannelOperation' has virtual functions but non-virtual destructor [-Werror,-Wnon-virtual-dtor] The fix is trivial. --- TYPE: NO_HISTORY DESC: Channel operation needs virtual destructor. --- tiledb/sm/cpp_api/channel_operation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/cpp_api/channel_operation.h b/tiledb/sm/cpp_api/channel_operation.h index 31333d92a185..8223e9733531 100644 --- a/tiledb/sm/cpp_api/channel_operation.h +++ b/tiledb/sm/cpp_api/channel_operation.h @@ -68,7 +68,7 @@ class ChannelOperation { ChannelOperation& operator=(ChannelOperation&&) = default; /** Destructor. */ - ~ChannelOperation() = default; + virtual ~ChannelOperation() = default; /* ********************************* */ /* API */ From 02adaf2d6c7fdb59cd2366ed936d67eb31db9067 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Wed, 25 Oct 2023 15:42:29 +0300 Subject: [PATCH 028/456] Commit test program for S3 write caching (#3863) I've placed this one in cpp_api examples, it compiles ok, but it requires a local REST server to run. --- TYPE: NO_HISTORY | FEATURE DESC: Commit test program for S3 write caching --- .../global_order_write_s3_buffering.cc | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 experimental/experimental_examples/cpp_api/global_order_write_s3_buffering.cc diff --git a/experimental/experimental_examples/cpp_api/global_order_write_s3_buffering.cc b/experimental/experimental_examples/cpp_api/global_order_write_s3_buffering.cc new file mode 100644 index 000000000000..ec1f0ddc78f5 --- /dev/null +++ b/experimental/experimental_examples/cpp_api/global_order_write_s3_buffering.cc @@ -0,0 +1,189 @@ +#include +#include +#include +#include + +using namespace tiledb; + +// This example assumes you have a local deployment of TileDB REST server +// serving at localhost:8181 where test_gow_rest2 is a registered array. +// The example creates/deletes this array directly from S3, but as long as the +// array was registered on the REST server, the test should work fine +std::string array_name("tiledb://demo/test_gow_rest2"); +std::string s3_array("s3://tiledb-robert2/test_gow_rest2"); + +uint64_t tile_extent = 32; +uint64_t capacity = tile_extent; +uint64_t dim_limit = 1572864; // 12mb +uint64_t ncells = 393216; // 3MB + +// Needs to be tile aligned +uint64_t chunk_size = 131072; // 1MB + +std::vector a1; +std::vector a2; +std::vector a2_nullable; + +std::string a3 = "abcd"; +std::vector a3_data; +std::vector a3_offsets; + +// Replace with TILEDB_SPARSE to test a sparse array +tiledb_array_type_t array_type = TILEDB_DENSE; +std::vector coords; + +void create_array(Context& ctx) { + ArraySchema schema(ctx, array_type); + Domain domain(ctx); + domain.add_dimension( + Dimension::create(ctx, "d1", {0, dim_limit}, tile_extent)); + schema.set_domain(domain); + schema.add_attribute(Attribute::create(ctx, "a1")); + schema.add_attribute( + Attribute::create(ctx, "a2").set_nullable(true)); + schema.add_attribute(Attribute::create>(ctx, "a3")); + if (array_type == TILEDB_SPARSE) { + schema.set_capacity(capacity); + } + Array::create(s3_array, schema); +} + +// Each global order write of size chunk_size will create an intermediate S3 +// chunk which can be observed (when pausing execution before +// submit_and_finalize) in the fragment path under e.g. +// frag_uuid/__global_order_write_chunks/a1.tdb_0 +void global_write(Context& ctx) { + Array array(ctx, array_name, TILEDB_WRITE); + + for (uint64_t i = 0; i < ncells; i++) { + a1.push_back(i); + a2.push_back(2 * i); + a2_nullable.push_back(a2.back() % 5 == 0 ? 0 : 1); + a3_offsets.push_back(i % chunk_size * a3.size()); + a3_data.insert(a3_data.end(), a3.begin(), a3.end()); + if (array_type == TILEDB_SPARSE) { + coords.push_back(i); + } + } + + uint64_t last_space_tile = + (ncells / tile_extent + + static_cast(ncells % tile_extent != 0)) * + tile_extent - + 1; + + Query query(ctx, array); + query.set_layout(TILEDB_GLOBAL_ORDER); + + if (array_type == TILEDB_DENSE) { + Subarray subarray(ctx, array); + subarray.add_range(0, (uint64_t)0, last_space_tile); + query.set_subarray(subarray); + } + + uint64_t begin = 0; + while (begin < ncells - chunk_size) { + query.set_data_buffer("a1", a1.data() + begin, chunk_size); + if (array_type == TILEDB_SPARSE) { + query.set_data_buffer("d1", coords.data() + begin, chunk_size); + } + query.set_data_buffer("a2", a2.data() + begin, chunk_size); + query.set_validity_buffer("a2", a2_nullable.data() + begin, chunk_size); + query.set_data_buffer( + "a3", a3_data.data() + begin * a3.size(), chunk_size * a3.size()); + query.set_offsets_buffer("a3", a3_offsets.data() + begin, chunk_size); + query.submit(); + + begin += chunk_size; + } + + query.set_data_buffer("a1", a1.data() + begin, last_space_tile - begin + 1); + if (array_type == TILEDB_SPARSE) { + query.set_data_buffer( + "d1", coords.data() + begin, last_space_tile - begin + 1); + } + query.set_data_buffer("a2", a2.data() + begin, last_space_tile - begin + 1); + query.set_validity_buffer( + "a2", a2_nullable.data() + begin, last_space_tile - begin + 1); + query.set_data_buffer( + "a3", + a3_data.data() + begin * a3.size(), + (last_space_tile - begin + 1) * a3.size()); + query.set_offsets_buffer( + "a3", a3_offsets.data() + begin, last_space_tile - begin + 1); + query.submit_and_finalize(); + if (query.query_status() != Query::Status::COMPLETE) { + throw std::runtime_error("Query incomplete"); + } +} + +void read_and_validate(Context& ctx) { + Array array(ctx, array_name, TILEDB_READ); + + Query query(ctx, array); + query.set_layout(TILEDB_ROW_MAJOR); + if (array_type == TILEDB_DENSE) { + Subarray subarray(ctx, array); + subarray.add_range(0, (uint64_t)0, ncells - 1); + query.set_subarray(subarray); + } else { + query.set_data_buffer("d1", coords); + } + std::vector a1_result(ncells); + std::vector a2_result(ncells); + std::vector a2_result_nullable(ncells); + std::vector a3_result_data(a3.size() * ncells); + std::vector a3_result_offsets(ncells); + query.set_data_buffer("a1", a1_result.data(), a1_result.size()); + query.set_data_buffer("a2", a2_result.data(), a2_result.size()); + query.set_validity_buffer( + "a2", a2_result_nullable.data(), a2_result_nullable.size()); + query.set_data_buffer( + "a3", a3_result_data.data(), a3_result_data.size() * a3.size()); + query.set_offsets_buffer( + "a3", a3_result_offsets.data(), a3_result_offsets.size()); + query.submit(); + + if (query.query_status() != Query::Status::COMPLETE) { + throw std::runtime_error("Query incomplete during read"); + } + + for (uint64_t i = 0; i < ncells; ++i) { + if (a1[i] != a1_result[i]) { + throw std::runtime_error("Incorrect data read on a1"); + } + if (a2[i] != a2_result[i]) { + throw std::runtime_error("Incorrect data read on a2"); + } + if (a2_nullable[i] != a2_result_nullable[i]) { + throw std::runtime_error("Incorrect data read on nullable vector a2"); + } + for (uint64_t j = 0; j < a3.size(); ++j) { + if (a3_data[i * a3.size() + j] != a3_result_data[i * a3.size() + j]) { + throw std::runtime_error("Incorrect data read on a3"); + } + } + } +} + +int main() { + Config cfg; + cfg["rest.username"] = "demo"; + cfg["rest.password"] = "demodemo"; + cfg["rest.server_address"] = "http://localhost:8181"; + + Context ctx(cfg); + + try { + create_array(ctx); + } catch (...) { + tiledb::VFS vfs(ctx); + vfs.remove_dir(s3_array); + std::cout << "Removed existing array" << std::endl; + create_array(ctx); + } + global_write(ctx); + read_and_validate(ctx); + + return 0; +} From b1ff9ae98cce469825a960b10fd173bf3e2d765e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:21:25 +0200 Subject: [PATCH 029/456] Adding missing history for some patch releases. (#4457) Adding missing patch releases from 2.10, 2.12, 2.14 and 2.16. --- TYPE: NO_HISTORY DESC: Adding missing history for some patch releases. --- HISTORY.md | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 9bd9cabdedf1..aedec8539b58 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -105,6 +105,48 @@ * Conditionally disable WhiteboxAllocator tests to fix GCC-13 build. [#4291](https://github.com/TileDB-Inc/TileDB/pull/4291) * Sorted reads tests: faster tests. [#4278](https://github.com/TileDB-Inc/TileDB/pull/4278) +# TileDB v2.16.3 Release Notes + +## Improvements + +* Make generic tile decompression single threaded. [#4256](https://github.com/TileDB-Inc/TileDB/pull/4256) + +## Defects removed + +* Legacy reader: reading non written region causes segfault. [#4253](https://github.com/TileDB-Inc/TileDB/pull/4253) +* Unordered writes: fix deserialization for older clients. [#4246](https://github.com/TileDB-Inc/TileDB/pull/4246) +* Clear pending changes during group delete. [#4267](https://github.com/TileDB-Inc/TileDB/pull/4267) +* Fix serialization of var-size property for nonempty domains. [#4264](https://github.com/TileDB-Inc/TileDB/pull/4264) + +## Build system changes + +* Fix build failures when vcpkg is not enabled. [#4259](https://github.com/TileDB-Inc/TileDB/pull/4259) +* Fix static linking from the GitHub Releases package on Windows. [#4263](https://github.com/TileDB-Inc/TileDB/pull/4263) + +# TileDB v2.16.2 Release Notes + +## Improvements + +* Add support for serialization of vacuum and consolidation requests. (#3902) [#4229](https://github.com/TileDB-Inc/TileDB/pull/4229) + +## Build changes + +* Export Azure::azure-storage-blobs on install when vcpkg is enabled. [#4232](https://github.com/TileDB-Inc/TileDB/pull/4232) + +# TileDB v2.16.1 Release Notes + +## Defects removed + +* Dense reader: fix nullable string values when reading with qc. [#4174](https://github.com/TileDB-Inc/TileDB/pull/4174) +* Allow Subarray::get_est_result_size_nullable on remote arrays. [#4202](https://github.com/TileDB-Inc/TileDB/pull/4202) +* Sparse unordered w/ dups reader: adding ignored tiles. [#4200](https://github.com/TileDB-Inc/TileDB/pull/4200) + +## Build system changes + +* Fix azure static library linking order for superbuild. [#4171](https://github.com/TileDB-Inc/TileDB/pull/4171) +* Build curl with ZStandard support on vcpkg when serialization is enabled. [#4166](https://github.com/TileDB-Inc/TileDB/pull/4166) +* Fix static linking to the release binaries on Windows. [#4208](https://github.com/TileDB-Inc/TileDB/pull/4208) + # TileDB v2.16.0 Release Notes ## Disk Format @@ -396,6 +438,27 @@ Bump to version 19 (.vac vacuum files now use relative filenames). [#4024](https * https://github.com/TileDB-Inc/TileDB/compare/2.14.0...2.15.0 +# TileDB v2.14.2 Release Notes + +## Improvements + +* RLE and dictionary filter only enabled for UTF8 since format version 17. [#3868](https://github.com/TileDB-Inc/TileDB/pull/3868) +* Fragment consolidation: using correct buffer weights. [#3877](https://github.com/TileDB-Inc/TileDB/pull/3877) +* Sparse global order reader: fix read progress update for duplicates. [#3937](https://github.com/TileDB-Inc/TileDB/pull/3937) +* Sparse unordered w/ dups: fix error on double var size overflow. [#3963](https://github.com/TileDB-Inc/TileDB/pull/3963) + +## Bug fixes + +* Deregister a remote array from the Consistency multimap before reopening. [#3859](https://github.com/TileDB-Inc/TileDB/pull/3859) +* Fix buffer size error in deserializing query. [#3851](https://github.com/TileDB-Inc/TileDB/pull/3851) +* Fix support for empty strings for Dictionary and RLE encodings. [#3938](https://github.com/TileDB-Inc/TileDB/pull/3938) + +# TileDB v2.14.1 Release Notes + +## Defects removed + +* Tile metadata generator: fix buffer overflow on string comparison. [#3821](https://github.com/TileDB-Inc/TileDB/pull/3821) + # TileDB v2.14.0 Release Notes ## Announcements @@ -497,7 +560,6 @@ Bump to version 19 (.vac vacuum files now use relative filenames). [#4024](https * Query condition: fix when attribute condition is not in user buffers. [#3713](https://github.com/TileDB-Inc/TileDB/pull/3713) - # TileDB v2.13.0 Release Notes # Announcements @@ -578,6 +640,32 @@ Bump to version 19 (.vac vacuum files now use relative filenames). [#4024](https * https://github.com/TileDB-Inc/TileDB/compare/2.12.0...2.13.0 +# TileDB v2.12.3 Release Notes + +## Improvements + +* Dense consolidation: set correct non-empty domain. [#3635](https://github.com/TileDB-Inc/TileDB/pull/3635) + +## Bug fixes + +* Dense array: Tile var size metadata not loaded on read. [#3645](https://github.com/TileDB-Inc/TileDB/pull/3645) +* Sparse global order reader: fix tile cleanup when ending an iteration. [#3674](https://github.com/TileDB-Inc/TileDB/pull/3674) + +# TileDB v2.12.2 Release Notes + +## Bug fixes + +* Fix use-after-free on a capnp::FlatArrayMessageReader. [#3631](https://github.com/TileDB-Inc/TileDB/pull/3631) + +# TileDB v2.12.1 Release Notes + +## Improvements + +### Defects Removed + +* Sparse global order reader: fixing incomplete reason for rest queries. [#3620](https://github.com/TileDB-Inc/TileDB/pull/3620) +* Add missing query_type in array_open capnp. [#3616](https://github.com/TileDB-Inc/TileDB/pull/3616) + # TileDB v2.12.0 Release Notes ## Disk Format @@ -817,6 +905,44 @@ Bump to version 19 (.vac vacuum files now use relative filenames). [#4024](https * https://github.com/TileDB-Inc/TileDB/compare/2.10.0...2.11.0 +# TileDB v2.10.4 Release Notes + +## Improvements + +* Adding experimental API for getting relevant fragments. [#3413](https://github.com/TileDB-Inc/TileDB/pull/3413) + +## API additions + +### C API + +* Add `tiledb_query_get_relevant_fragment_num` for experimental API to get relevant fragments. [#3413](https://github.com/TileDB-Inc/TileDB/pull/3413) + +# TileDB v2.10.3 Release Notes + +## Improvements + +* Sparse refactored readers, mark empty fragments as fully loaded early. [#3394](https://github.com/TileDB-Inc/TileDB/pull/3394) + +## Bug fixes + +* Fix SC-19287: segfault due to deref nonexistent filestore key [#3359](https://github.com/TileDB-Inc/TileDB/pull/3359) +* Bug Fix: Wrong results when using OR condition with nullable attributes [#3308](https://github.com/TileDB-Inc/TileDB/pull/3308) + +# TileDB v2.10.2 Release Notes + +## Bug fixes + +* Close magic resources [#3319](https://github.com/TileDB-Inc/TileDB/pull/3319) +* Correct simple typo in assignment [#3335](https://github.com/TileDB-Inc/TileDB/pull/3335) +* Datatype for domain must be serialized for backwards client compatibility [#3343](https://github.com/TileDB-Inc/TileDB/pull/3343) +* Sparse global order reader: disable reader for reads with qc. [#3342](https://github.com/TileDB-Inc/TileDB/pull/3342) +* Fix issue with sparse unordered without duplicates query deserialization to use Indexed Reader [#3347](https://github.com/TileDB-Inc/TileDB/pull/3347) + +# TileDB v2.10.1 Release Notes + +## Bug fixes +* compute_results_count_sparse_string: using cached ranges properly. [#3314](https://github.com/TileDB-Inc/TileDB/pull/3314) + # TileDB v2.10.0 Release Notes **Full Changelog**: https://github.com/TileDB-Inc/TileDB/compare/2.9.0...2.10.0 From ea7e8ef1bb558d93b2a8866fe803fd3b751a3f37 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Wed, 25 Oct 2023 10:36:27 -0600 Subject: [PATCH 030/456] Add an aspect template argument to `class CAPIFunction`. (#4430) This PR makes two changes to the template signature of `class CAPIFunction`. The central one is to add an aspect argument that can provide hooks into `CAPIFunction::function`. In this initial version there's a single aspect point (the aspect member function `call`) that is only an observer. The `call` of the default aspect does nothing. The second change in template signature is to infer the return value of the wrapper function and to suppress return statements if that type is `void`. This allows the elimination of subsidiary wrapper for `void` functions to give them a (constant) return value. Tests for the aspect implement `class LoggingAspect`, a primitive call-logger sufficient for testing, and provide an override header to activate it. The override header also generates a traits class for each C API implementation function that provides data for `LoggingAspect`. These elements are exercised by two new test runners that share a common set of tests; only the results differ depending on whether the hook is active or not. In order to automatically generate metadata for API functions, the PR introduces a set of macros to define the C API interface functions. These functions are already essentially boilerplate, but they cannot be defined with function templates because they must have "C" linkage. The macros provide a pair of macro hooks for generating code either before or after the function definition. --- TYPE: IMPROVEMENT DESC: Add an aspect template argument to `class CAPIFunction` --- tiledb/api/c_api/config/config_api.cc | 69 ++++---- tiledb/api/c_api/context/context_api.cc | 32 ++-- tiledb/api/c_api/error/error_api.cc | 5 +- tiledb/api/c_api_support/c_api_support.h | 4 + .../exception_wrapper/CMakeLists.txt | 1 - .../exception_wrapper/capi_definition.h | 61 +++++++ .../exception_wrapper/exception_wrapper.h | 111 +++++++------ .../exception_wrapper/test/CMakeLists.txt | 50 ++++++ .../exception_wrapper/test/hook/DIRECTORY.md | 5 + .../test/hook/capi_function_override.h | 63 ++++++++ .../exception_wrapper/test/hook_common.h | 45 ++++++ .../exception_wrapper/test/logging_aspect.h | 113 +++++++++++++ .../exception_wrapper/test/unit_capi_hook.cc | 153 ++++++++++++++++++ .../test/unit_capi_hook_with.cc | 37 +++++ .../test/unit_capi_hook_without.cc | 38 +++++ 15 files changed, 687 insertions(+), 100 deletions(-) create mode 100644 tiledb/api/c_api_support/exception_wrapper/capi_definition.h create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/hook/DIRECTORY.md create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/hook/capi_function_override.h create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/hook_common.h create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/logging_aspect.h create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook.cc create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_with.cc create mode 100644 tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_without.cc diff --git a/tiledb/api/c_api/config/config_api.cc b/tiledb/api/c_api/config/config_api.cc index fdcc8b1fa35b..4bd2819dfff0 100644 --- a/tiledb/api/c_api/config/config_api.cc +++ b/tiledb/api/c_api/config/config_api.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -174,8 +174,7 @@ capi_return_t tiledb_config_iter_done( using tiledb::api::api_entry_error; -capi_return_t tiledb_config_alloc( - tiledb_config_t** config, tiledb_error_t** error) noexcept { +CAPI_INTERFACE(config_alloc, tiledb_config_t** config, tiledb_error_t** error) { return api_entry_error(error, config); } @@ -184,48 +183,53 @@ capi_return_t tiledb_config_alloc( * can return an error. * Possible errors: `config` may be null or an invalid handle. */ -void tiledb_config_free(tiledb_config_t** config) noexcept { - return tiledb::api::api_entry_void(config); +CAPI_INTERFACE_VOID(config_free, tiledb_config_t** config) { + tiledb::api::api_entry_void(config); } -capi_return_t tiledb_config_set( +CAPI_INTERFACE( + config_set, tiledb_config_t* config, const char* param, const char* value, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, param, value); } -capi_return_t tiledb_config_get( +CAPI_INTERFACE( + config_get, tiledb_config_t* config, const char* param, const char** value, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, param, value); } -capi_return_t tiledb_config_unset( +CAPI_INTERFACE( + config_unset, tiledb_config_t* config, const char* param, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, param); } -capi_return_t tiledb_config_load_from_file( +CAPI_INTERFACE( + config_load_from_file, tiledb_config_t* config, const char* filename, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, filename); } -capi_return_t tiledb_config_save_to_file( +CAPI_INTERFACE( + config_save_to_file, tiledb_config_t* config, const char* filename, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, filename); } @@ -235,26 +239,31 @@ capi_return_t tiledb_config_save_to_file( * Possible errors: Both `lhs` and `rhs` may be null or an invalid handle. * `equal` may be a null pointer */ -capi_return_t tiledb_config_compare( - tiledb_config_t* lhs, tiledb_config_t* rhs, uint8_t* equal) noexcept { +CAPI_INTERFACE( + config_compare, + tiledb_config_t* lhs, + tiledb_config_t* rhs, + uint8_t* equal) { return tiledb::api::api_entry_plain( lhs, rhs, equal); } -capi_return_t tiledb_config_iter_alloc( +CAPI_INTERFACE( + config_iter_alloc, tiledb_config_t* config, const char* prefix, tiledb_config_iter_t** config_iter, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, prefix, config_iter); } -capi_return_t tiledb_config_iter_reset( +CAPI_INTERFACE( + config_iter_reset, tiledb_config_t* config, tiledb_config_iter_t* config_iter, const char* prefix, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config, config_iter, prefix); } @@ -264,30 +273,34 @@ capi_return_t tiledb_config_iter_reset( * can return an error. * Possible errors: `config` may be null or an invalid handle. */ -void tiledb_config_iter_free(tiledb_config_iter_t** config_iter) noexcept { +CAPI_INTERFACE_VOID(config_iter_free, tiledb_config_iter_t** config_iter) { tiledb::api::api_entry_void( config_iter); } -capi_return_t tiledb_config_iter_here( +CAPI_INTERFACE( + config_iter_here, tiledb_config_iter_t* config_iter, const char** param, const char** value, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config_iter, param, value); } -capi_return_t tiledb_config_iter_next( - tiledb_config_iter_t* config_iter, tiledb_error_t** error) noexcept { +CAPI_INTERFACE( + config_iter_next, + tiledb_config_iter_t* config_iter, + tiledb_error_t** error) { return api_entry_error( error, config_iter); } -capi_return_t tiledb_config_iter_done( +CAPI_INTERFACE( + config_iter_done, tiledb_config_iter_t* config_iter, int32_t* done, - tiledb_error_t** error) noexcept { + tiledb_error_t** error) { return api_entry_error( error, config_iter, done); } \ No newline at end of file diff --git a/tiledb/api/c_api/context/context_api.cc b/tiledb/api/c_api/context/context_api.cc index 3b0eb566fbe4..df5fd2c2f05c 100644 --- a/tiledb/api/c_api/context/context_api.cc +++ b/tiledb/api/c_api/context/context_api.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -138,8 +138,8 @@ using tiledb::api::api_entry_with_context; /* * API Audit: No channel to return error message (failure code only) */ -capi_return_t tiledb_ctx_alloc( - tiledb_config_handle_t* config, tiledb_ctx_handle_t** ctx) noexcept { +CAPI_INTERFACE( + ctx_alloc, tiledb_config_handle_t* config, tiledb_ctx_handle_t** ctx) { return tiledb::api::api_entry_plain( config, ctx); } @@ -173,40 +173,42 @@ capi_return_t tiledb_ctx_alloc_with_error( /* * API Audit: void return */ -void tiledb_ctx_free(tiledb_ctx_handle_t** ctx) noexcept { +CAPI_INTERFACE_VOID(ctx_free, tiledb_ctx_handle_t** ctx) { return tiledb::api::api_entry_void(ctx); } -capi_return_t tiledb_ctx_get_stats( - tiledb_ctx_t* ctx, char** stats_json) noexcept { +CAPI_INTERFACE(ctx_get_stats, tiledb_ctx_t* ctx, char** stats_json) { return api_entry_with_context( ctx, stats_json); } -capi_return_t tiledb_ctx_get_config( - tiledb_ctx_t* ctx, tiledb_config_handle_t** config) noexcept { +CAPI_INTERFACE( + ctx_get_config, tiledb_ctx_t* ctx, tiledb_config_handle_t** config) { return api_entry_with_context( ctx, config); } -capi_return_t tiledb_ctx_get_last_error( - tiledb_ctx_t* ctx, tiledb_error_handle_t** err) noexcept { +CAPI_INTERFACE( + ctx_get_last_error, tiledb_ctx_t* ctx, tiledb_error_handle_t** err) { return api_entry_with_context( ctx, err); } -capi_return_t tiledb_ctx_is_supported_fs( - tiledb_ctx_t* ctx, tiledb_filesystem_t fs, int32_t* is_supported) noexcept { +CAPI_INTERFACE( + ctx_is_supported_fs, + tiledb_ctx_t* ctx, + tiledb_filesystem_t fs, + int32_t* is_supported) { return api_entry_with_context( ctx, fs, is_supported); } -capi_return_t tiledb_ctx_cancel_tasks(tiledb_ctx_t* ctx) noexcept { +CAPI_INTERFACE(ctx_cancel_tasks, tiledb_ctx_t* ctx) { return api_entry_with_context(ctx); } -capi_return_t tiledb_ctx_set_tag( - tiledb_ctx_t* ctx, const char* key, const char* value) noexcept { +CAPI_INTERFACE( + ctx_set_tag, tiledb_ctx_t* ctx, const char* key, const char* value) { return api_entry_with_context( ctx, key, value); } diff --git a/tiledb/api/c_api/error/error_api.cc b/tiledb/api/c_api/error/error_api.cc index 88f6e3e5f951..e265cd859bc2 100644 --- a/tiledb/api/c_api/error/error_api.cc +++ b/tiledb/api/c_api/error/error_api.cc @@ -62,12 +62,11 @@ void tiledb_error_free(tiledb_error_handle_t** err) { } // namespace tiledb::api -capi_return_t tiledb_error_message( - tiledb_error_handle_t* err, const char** errmsg) noexcept { +CAPI_INTERFACE(error_message, tiledb_error_handle_t* err, const char** errmsg) { return tiledb::api::api_entry_plain( err, errmsg); } -void tiledb_error_free(tiledb_error_handle_t** err) noexcept { +CAPI_INTERFACE_VOID(error_free, tiledb_error_handle_t** err) { return tiledb::api::api_entry_void(err); } diff --git a/tiledb/api/c_api_support/c_api_support.h b/tiledb/api/c_api_support/c_api_support.h index 90e736a745b7..4ab9dbeb6e05 100644 --- a/tiledb/api/c_api_support/c_api_support.h +++ b/tiledb/api/c_api_support/c_api_support.h @@ -37,6 +37,10 @@ #define TILEDB_CAPI_SUPPORT_H #include "argument_validation.h" +#include "tiledb/api/c_api_support/exception_wrapper/capi_definition.h" #include "tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h" +#if __has_include("capi_function_override.h") +#include "capi_function_override.h" +#endif #endif // TILEDB_CAPI_SUPPORT_H \ No newline at end of file diff --git a/tiledb/api/c_api_support/exception_wrapper/CMakeLists.txt b/tiledb/api/c_api_support/exception_wrapper/CMakeLists.txt index 2c658392b104..7d6669f31a57 100644 --- a/tiledb/api/c_api_support/exception_wrapper/CMakeLists.txt +++ b/tiledb/api/c_api_support/exception_wrapper/CMakeLists.txt @@ -33,7 +33,6 @@ include(object_library) # only. `exception_wrapper.cc` is an empty source file needed to allow the # OBJECT syntax. -# No actual source files at present. list(APPEND SOURCES exception_wrapper.cc ) diff --git a/tiledb/api/c_api_support/exception_wrapper/capi_definition.h b/tiledb/api/c_api_support/exception_wrapper/capi_definition.h new file mode 100644 index 000000000000..cd45eb219cfd --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/capi_definition.h @@ -0,0 +1,61 @@ +/** + * @file tiledb/api/c_api_support/exception_wrapper/hook.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines macros to define C API functions. + */ + +#ifndef TILEDB_CAPI_DEFINITION_H +#define TILEDB_CAPI_DEFINITION_H + +/* clang-format off */ + +/* + * Hook for additional code to be generated before each C API interface + * function. + */ +#define CAPI_PREFIX(name) + +/* + * Declaration clause for a C API interface function. Follow with a `{ ... }` + * block defining the body. + */ +#define CAPI_INTERFACE(root, ...) \ +CAPI_PREFIX(root) \ +capi_return_t tiledb_##root(__VA_ARGS__) noexcept + +/* + * A variant of CAPI_INTERFACE for the handful of `void` returns. + */ +#define CAPI_INTERFACE_VOID(root, ...) \ +CAPI_PREFIX(root) \ +void tiledb_##root(__VA_ARGS__) noexcept + +/* clang-format on */ + +#endif // TILEDB_CAPI_DEFINITION_H diff --git a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h index 3a29e6401b47..e79227e05281 100644 --- a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h +++ b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h @@ -514,18 +514,43 @@ using ExceptionActionCtxErr = detail::ExceptionActionDetailCtxErr; //------------------------------------------------------- // Exception wrapper //------------------------------------------------------- +/** + * Null aspect for `class CAPIFunction` has null operations for all aspects. + * @tparam f + */ +template +class CAPIFunctionNullAspect { + public: + template + static void call(Args...) { + } +}; + +/** + * Selection struct defines the default aspect type for CAPIFunction. This class + * is always used with second template argument as `void`. This definition is + * for the general case; a specialization can override it. + */ +template +struct CAPIFunctionSelector { + using aspect_type = CAPIFunctionNullAspect; +}; + /** * Non-specialized wrapper for implementations functions for the C API. May * only be used as a specialization. */ -template +template < + auto f, + class H, + class A = typename CAPIFunctionSelector::aspect_type> class CAPIFunction; /** * Wrapper for implementations functions for the C API */ -template -class CAPIFunction { +template +class CAPIFunction { public: /** * Forwarded alias to template parameter H. @@ -539,7 +564,7 @@ class CAPIFunction { * @param args Arguments to an API implementation function * @return */ - static capi_return_t function(H& h, Args... args) { + static R function(H& h, Args... args) { /* * The order of the catch blocks is not arbitrary: * - `std::bad_alloc` comes first because it overrides other problems @@ -561,27 +586,45 @@ class CAPIFunction { * Note that we don't need std::forward here because all the arguments * must have "C" linkage. */ - auto x{f(args...)}; - h.action_on_success(); - return x; + A::call(args...); + if constexpr (std::same_as) { + f(args...); + h.action_on_success(); + } else { + auto x{f(args...)}; + h.action_on_success(); + return x; + } } catch (const std::bad_alloc& e) { h.action(e); - return TILEDB_OOM; + if constexpr (!std::same_as) { + return TILEDB_OOM; + } } catch (const detail::InvalidContextException& e) { h.action(e); - return TILEDB_INVALID_CONTEXT; + if constexpr (!std::same_as) { + return TILEDB_INVALID_CONTEXT; + } } catch (const detail::InvalidErrorException& e) { h.action(e); - return TILEDB_INVALID_ERROR; + if constexpr (!std::same_as) { + return TILEDB_INVALID_ERROR; + } } catch (const StatusException& e) { h.action(e); - return TILEDB_ERR; + if constexpr (!std::same_as) { + return TILEDB_ERR; + } } catch (const std::exception& e) { h.action(e); - return TILEDB_ERR; + if constexpr (!std::same_as) { + return TILEDB_ERR; + } } catch (...) { h.action(CAPIException("unknown exception type; no further information")); - return TILEDB_ERR; + if constexpr (!std::same_as) { + return TILEDB_ERR; + } } }; @@ -668,43 +711,6 @@ template constexpr auto api_entry_plain = CAPIFunction::function_plain; -/** - * Declaration only defined through a specialization. - * - * @tparam f An API implementation function - */ -template -struct CAPIFunctionVoid; - -/** - * Wrapper class for API implementation functions with `void` return. - * - * We require a separate wrapper class here so we can match the template - * argument `f` to a function with void return, since `CAPIFunction` only - * matches those that return `capi_return_t`. - * - * @tparam Args Argument types for the function - * @tparam f An API implementation function - */ -template -struct CAPIFunctionVoid { - /** - * Function transformer changes an API implementation function with `void` - * return to one returning a (trivially constant) `capi_return_t` value. - * - * This function is used to match the function signature in `CAPIFunction`, - * which requires a return value. This allows us to reuse its wrapper function - * without duplicating code. - * - * @param args Arguments passed to the function - * @return TILEDB_OK - */ - inline static capi_return_t function_from_void(Args... args) { - f(args...); - return TILEDB_OK; - } -}; - /** * Function transformer changes an API implementation function with `void` * return to an API interface function, also with `void` return. @@ -712,9 +718,8 @@ struct CAPIFunctionVoid { * @tparam f An implementation function. */ template -constexpr auto api_entry_void = CAPIFunction< - CAPIFunctionVoid::function_from_void, - tiledb::api::ExceptionAction>::void_function; +constexpr auto api_entry_void = + CAPIFunction::void_function; /** * Declaration only defined through a specialization. diff --git a/tiledb/api/c_api_support/exception_wrapper/test/CMakeLists.txt b/tiledb/api/c_api_support/exception_wrapper/test/CMakeLists.txt index 43841b0aa8fe..f0b8845e4b06 100644 --- a/tiledb/api/c_api_support/exception_wrapper/test/CMakeLists.txt +++ b/tiledb/api/c_api_support/exception_wrapper/test/CMakeLists.txt @@ -34,3 +34,53 @@ commence(unit_test capi_exception_wrapper) # top-level dependency required for the test. capi_context_stub) conclude(unit_test) + +# +# Hook Tests +# +# These following two unit test runners are a pair, one compiled without an API +# hook compiled in, one without. They run the same test, although with different +# expectations about the results. The hook is specified in two different ways: +# * In the ordinary way, with an additional include directory +# * Specifically for testing, with a compile definition. +# The two different ways are used to verify that the hook is activated or not +# as appropriate. +# +# Changing the API hook changes the API object libraries. The API hook is +# perforce a low-level mechanism. It changes the definition of C API calls. The +# exception handler is intertwined with a few of the API classes, notably the +# handles for context and error (and config gets pulled in as a dependency). As +# a result, whereas the without-hook test runner can use an existing object +# library, the with-hook ones needs to recompile an equivalent. +# + +# Hook tests without the hook +commence(unit_test capi_ew_without_hook) + this_target_sources( + unit_capi_hook.cc + unit_capi_hook_without.cc) + this_target_object_libraries( + capi_context_stub) +conclude(unit_test) + +# Hook tests with the hook included. +commence(unit_test capi_ew_with_hook) + this_target_sources( + unit_capi_hook.cc + unit_capi_hook_with.cc + ../../../c_api/config/config_api.cc + ../../../c_api/context/context_api.cc + ../../../c_api/error/error_api.cc + ../../../../sm/storage_manager/context.cc + ../../../../sm/storage_manager/context_resources.cc + ) + this_target_link_libraries(export) + this_target_object_libraries( + exception_wrapper + storage_manager_stub + vfs + ) + target_compile_definitions(unit_capi_ew_with_hook PUBLIC WITH_HOOK) + target_include_directories( + unit_capi_ew_with_hook PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/hook) +conclude(unit_test) diff --git a/tiledb/api/c_api_support/exception_wrapper/test/hook/DIRECTORY.md b/tiledb/api/c_api_support/exception_wrapper/test/hook/DIRECTORY.md new file mode 100644 index 000000000000..b1bcef7cb37f --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/hook/DIRECTORY.md @@ -0,0 +1,5 @@ +# Directory for C API hook testing + +A user activates the C API hook by writing an override header and building the library with its directory as part of the include path. This is a compile-time mechanism, so in order to test it well, the same code should execute both with and without a test hook compiled in. + +This directory proivdes a place to put an override header that's not ordinarily included in the build. \ No newline at end of file diff --git a/tiledb/api/c_api_support/exception_wrapper/test/hook/capi_function_override.h b/tiledb/api/c_api_support/exception_wrapper/test/hook/capi_function_override.h new file mode 100644 index 000000000000..b1e5fbd5ac87 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/hook/capi_function_override.h @@ -0,0 +1,63 @@ +/** + * @file + * tiledb/api/c_api_support/exception_wrapper/test/hook/capi_function_override.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines macros to define C API functions. + */ + +#ifndef TILEDB_EXCEPTION_WRAPPER_TEST_CAPI_FUNCTION_OVERRIDE_H +#define TILEDB_EXCEPTION_WRAPPER_TEST_CAPI_FUNCTION_OVERRIDE_H + +#include "../logging_aspect.h" + +/* + * CAPI_PREFIX is an extension point within the macros for defining C API + * interface functions. The override macro defines a trait class for each C API + * implementation function. The trait class is a specialization of an otherwise + * undefined class template. Each trait class contains the name of its + * implementation function; the name is subsequently picked up by `class + * LoggingAspect`. + */ +#undef CAPI_PREFIX +#define CAPI_PREFIX(root) \ + template <> \ + struct CAPIFunctionNameTrait { \ + static constexpr std::string_view name{"tiledb_" #root}; \ + }; + +/** + * Specialization of the aspect selector to `void` overrides the default (the + * null aspect) to compile with the logging aspect instead. + */ +template +struct tiledb::api::CAPIFunctionSelector { + using aspect_type = LoggingAspect; +}; + +#endif // TILEDB_EXCEPTION_WRAPPER_TEST_CAPI_FUNCTION_OVERRIDE_H diff --git a/tiledb/api/c_api_support/exception_wrapper/test/hook_common.h b/tiledb/api/c_api_support/exception_wrapper/test/hook_common.h new file mode 100644 index 000000000000..d767405f94b0 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/hook_common.h @@ -0,0 +1,45 @@ +/** + * @file tiledb/api/c_api_support/exception_wrapper/test/hook_common.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines macros to define C API functions. + */ + +#ifndef TILEDB_EXCEPTION_WRAPPER_TEST_HOOK_COMMON_H +#define TILEDB_EXCEPTION_WRAPPER_TEST_HOOK_COMMON_H + +/* + * Convert the possible command line argument WITH_HOOK into a C++ constant. + */ +#if defined(WITH_HOOK) +constexpr bool compiled_with_hook{true}; +#else +constexpr bool compiled_with_hook{false}; +#endif + +#endif // TILEDB_EXCEPTION_WRAPPER_TEST_HOOK_COMMON_H diff --git a/tiledb/api/c_api_support/exception_wrapper/test/logging_aspect.h b/tiledb/api/c_api_support/exception_wrapper/test/logging_aspect.h new file mode 100644 index 000000000000..5776d3ac6920 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/logging_aspect.h @@ -0,0 +1,113 @@ +/** + * @file tiledb/api/c_api_support/exception_wrapper/test/logging_aspect.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines a logging aspect sufficient to verify that the C API + * aspect system in the exception wrapper is working. + */ + +#ifndef TILEDB_EXCEPTION_WRAPPER_TEST_LOGGING_ASPECT_H +#define TILEDB_EXCEPTION_WRAPPER_TEST_LOGGING_ASPECT_H + +#include + +/** + * Default trait class declaration has no members; it's only defined a + * specialization. + */ +template +struct CAPIFunctionNameTrait; + +/** + * Base class for the logging aspect class template. + * + * This class mimics a global logger. It's rudimentary but suffices for testing. + * Instead of a growing log, it has a single "log entry". It's not C.41, but it + * doesn't need to be, since it has an extremely limited purpose. + */ +class LABase { + protected: + /** + * The "log entry". There's only one. + */ + static std::string msg_; + /** + * Whether `call` has been called since the last reset. + */ + static bool touched_; + + public: + /** + * This isn't a C.41 class, as it's a wrapper around some static variables. In + * lieu of a real construct, we have a reset function. + */ + static void reset() { + msg_ = ""; + touched_ = false; + } + /** Accessor for the "log entry" */ + static inline std::string message() { + return msg_; + } + /** Accessor for the call history flag */ + static inline bool touched() { + return touched_; + } +}; + +/** + * Logging aspect for the exception wrapper from a C API function + * + * @tparam f C API implementation function + */ +template +class LoggingAspect; + +/** + * Specialization of the logging aspect that infers the return type and argument + * types of the template parameter. + * + * @tparam R The return type of `f` + * @tparam Args The argument types of `f` + * @tparam f C API implementation function + */ +template +class LoggingAspect : public LABase { + public: + /** + * Record the name of the function is the "log entry" + * + * @param ... Arguments are ignored + */ + static void call(Args...) { + msg_ = CAPIFunctionNameTrait::name; + touched_ = true; + } +}; + +#endif // TILEDB_EXCEPTION_WRAPPER_TEST_LOGGING_ASPECT_H diff --git a/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook.cc b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook.cc new file mode 100644 index 000000000000..37d8a47a4414 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook.cc @@ -0,0 +1,153 @@ +/** + * @file tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + */ + +#include + +#include "../../c_api_support.h" +#include "hook_common.h" +#include "logging_aspect.h" + +using namespace tiledb::api; +using namespace tiledb::api::detail; + +// from `logging_aspect.h` +std::string LABase::msg_{}; +bool LABase::touched_{false}; + +template +using selector_type = CAPIFunctionSelector; + +/** + * API function that does nothing. + */ +capi_return_t tf_null() { + return TILEDB_OK; +} +/** + * Metadata for null function. + */ +template <> +struct CAPIFunctionNameTrait { + static constexpr std::string_view name{"tf_null"}; +}; + +namespace tiledb::api { +/** + * API implementation function that does nothing. + */ +capi_return_t tiledb_capi_nil(int) { + return 0; +} +} // namespace tiledb::api +/* + * API definitions with macros. + */ +CAPI_INTERFACE(capi_nil, int x) { + return tiledb::api::api_entry_plain(x); +} + +/** + * The wrapped function with an unconditional invocation of the logging + * aspect as an explicit template argument. + */ +using null_always_wrapped_for_logging = tiledb::api:: + CAPIFunction>; +/** + * The wrapped function conditional upon an override as to whether the + * logging aspect is compiled in or not. The aspect argument is omitted, the + * default applies, and overriding is possible. + */ +using null_maybe_wrapped_for_logging = + tiledb::api::CAPIFunction; + +TEST_CASE("Compile consistency") { + /* + * In all cases verify that the default aspect is being used if and only if + * the hook is not enabled. + */ + CHECK( + compiled_with_hook != std::same_as< + selector_type::aspect_type, + CAPIFunctionNullAspect>); + if constexpr (compiled_with_hook) { + /* + * In the case "with hook", check that the aspect is what the test defines. + */ + CHECK(std::same_as< + selector_type::aspect_type, + LoggingAspect>); + } +} + +TEST_CASE("Hook unconditional") { + LABase::reset(); + CHECK(LABase::touched() == false); + CHECK(LABase::message() == ""); + tiledb::api::ExceptionAction h; + null_always_wrapped_for_logging().function(h); + CHECK(LABase::touched() == true); + CHECK(LABase::message() == "tf_null"); +} + +/* + * Test that the hook is invoked if and only if it's compiled in. This is the + * same as the previous test but for the last line. + */ +TEST_CASE("Hook conditional for touch") { + LABase::reset(); + CHECK(LABase::touched() == false); + tiledb::api::ExceptionAction h; + null_maybe_wrapped_for_logging().function(h); + CHECK(LABase::touched() == compiled_with_hook); +} + +TEST_CASE("Hook conditional with text 1") { + LABase::reset(); + CHECK(LABase::message() == ""); + tiledb::api::ExceptionAction h; + null_maybe_wrapped_for_logging().function(h); + if constexpr (compiled_with_hook) { + CHECK(LABase::message() == "tf_null"); + } else { + CHECK(LABase::message() == ""); + } +} + +TEST_CASE("Hook conditional with text 2") { + LABase::reset(); + CHECK(LABase::message() == ""); + tiledb::api::ExceptionAction h; + ::tiledb_capi_nil(0); + if constexpr (compiled_with_hook) { + CHECK(LABase::message() == "tiledb_capi_nil"); + } else { + CHECK(LABase::message() == ""); + } +} diff --git a/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_with.cc b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_with.cc new file mode 100644 index 000000000000..c8b7f4be76a5 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_with.cc @@ -0,0 +1,37 @@ +/** + * @file tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_with.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + */ + +#include + +#include "hook_common.h" + +TEST_CASE("Compile definition - with hook") { + CHECK(compiled_with_hook == true); +} diff --git a/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_without.cc b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_without.cc new file mode 100644 index 000000000000..97e2463a57f0 --- /dev/null +++ b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_without.cc @@ -0,0 +1,38 @@ +/** + * @file + * tiledb/api/c_api_support/exception_wrapper/test/unit_capi_hook_without.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + */ + +#include + +#include "hook_common.h" + +TEST_CASE("Compile definition - without hook") { + CHECK(compiled_with_hook == false); +} From 677c495b89132c154535b6231ea6ab9c3b0a0fe0 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:04:08 +0200 Subject: [PATCH 031/456] Remove Status from fragment_metadata.cc. (#4461) This removes usages of Status in the fragment metadata code. --- TYPE: NO_HISTORY DESC: Remove Status from fragment_metadata.cc. --- test/src/unit-tile-metadata.cc | 140 +--- tiledb/sm/array/array.cc | 6 +- .../fragment_meta_consolidator.cc | 4 +- tiledb/sm/fragment/fragment_info.cc | 13 +- tiledb/sm/fragment/fragment_metadata.cc | 711 ++++++++---------- tiledb/sm/fragment/fragment_metadata.h | 144 ++-- tiledb/sm/misc/constants.cc | 3 + tiledb/sm/misc/constants.h | 3 + tiledb/sm/query/legacy/reader.cc | 27 +- tiledb/sm/query/readers/dense_reader.cc | 8 +- tiledb/sm/query/readers/filtered_data.h | 12 +- .../query/readers/ordered_dim_label_reader.cc | 13 +- tiledb/sm/query/readers/reader_base.cc | 37 +- tiledb/sm/query/readers/reader_base.h | 7 +- .../query/readers/sparse_index_reader_base.cc | 11 +- .../sm/query/writers/global_order_writer.cc | 27 +- tiledb/sm/query/writers/ordered_writer.cc | 2 +- tiledb/sm/query/writers/unordered_writer.cc | 2 +- tiledb/sm/query/writers/writer_base.cc | 58 +- tiledb/sm/serialization/fragment_metadata.cc | 2 +- tiledb/sm/storage_manager/storage_manager.cc | 4 +- tiledb/sm/subarray/subarray.cc | 13 +- tools/src/commands/info_command.cc | 6 +- 23 files changed, 494 insertions(+), 759 deletions(-) diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index 884ed3394488..5464cffb1896 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -290,29 +290,16 @@ struct CPPFixedTileMetadataFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - auto st = frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); - CHECK(st.ok()); + frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); // Load the metadata and validate coords netadata. bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { - std::vector names_min{"d"}; - auto st = - frag_meta[f]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"d"}; - st = frag_meta[f]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"d"}; - st = frag_meta[f]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"d"}; - st = frag_meta[f]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"d"}; + frag_meta[f]->load_tile_min_values(enc_key, names); + frag_meta[f]->load_tile_max_values(enc_key, names); + frag_meta[f]->load_tile_sum_values(enc_key, names); + frag_meta[f]->load_tile_null_count_values(enc_key, names); // Validation. // Min/max/sum for all null tile are invalid. @@ -459,22 +446,11 @@ struct CPPFixedTileMetadataFx { } // Load attribute metadata. - std::vector names_min{"a"}; - st = frag_meta[f]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"a"}; - st = frag_meta[f]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"a"}; - st = frag_meta[f]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"a"}; - st = frag_meta[f]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"a"}; + frag_meta[f]->load_tile_min_values(enc_key, names); + frag_meta[f]->load_tile_max_values(enc_key, names); + frag_meta[f]->load_tile_sum_values(enc_key, names); + frag_meta[f]->load_tile_null_count_values(enc_key, names); // Validate attribute metadta. // Min/max/sum for all null tile are invalid. @@ -837,29 +813,16 @@ struct CPPVarTileMetadataFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - auto st = frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); - CHECK(st.ok()); + frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); // Load the metadata and validate coords netadata. bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { - std::vector names_min{"d"}; - auto st = - frag_meta[f]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"d"}; - st = frag_meta[f]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"d"}; - st = frag_meta[f]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"d"}; - st = frag_meta[f]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"d"}; + frag_meta[f]->load_tile_min_values(enc_key, names); + frag_meta[f]->load_tile_max_values(enc_key, names); + frag_meta[f]->load_tile_sum_values(enc_key, names); + frag_meta[f]->load_tile_null_count_values(enc_key, names); // Validation. // Min/max/sum for all null tile are invalid. @@ -953,22 +916,11 @@ struct CPPVarTileMetadataFx { } // Load attribute metadata. - std::vector names_min{"a"}; - st = frag_meta[f]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"a"}; - st = frag_meta[f]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"a"}; - st = frag_meta[f]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"a"}; - st = frag_meta[f]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"a"}; + frag_meta[f]->load_tile_min_values(enc_key, names); + frag_meta[f]->load_tile_max_values(enc_key, names); + frag_meta[f]->load_tile_sum_values(enc_key, names); + frag_meta[f]->load_tile_null_count_values(enc_key, names); // Validate attribute metadata. // Min/max/sum for all null tile are invalid. @@ -1185,8 +1137,7 @@ struct CPPFixedTileMetadataPartialFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - auto st = frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); - CHECK(st.ok()); + frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); // Do fragment metadata first for attribute. { @@ -1211,22 +1162,11 @@ struct CPPFixedTileMetadataPartialFx { } // Load attribute metadata. - std::vector names_min{"a"}; - st = frag_meta[0]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"a"}; - st = frag_meta[0]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"a"}; - st = frag_meta[0]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"a"}; - st = frag_meta[0]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"a"}; + frag_meta[0]->load_tile_min_values(enc_key, names); + frag_meta[0]->load_tile_max_values(enc_key, names); + frag_meta[0]->load_tile_sum_values(enc_key, names); + frag_meta[0]->load_tile_null_count_values(enc_key, names); std::vector correct_tile_mins{1.1, 2.1, 3.2, 4.1}; std::vector correct_tile_maxs{1.7, 2.6, 3.8, 4.9}; @@ -1366,8 +1306,7 @@ struct CPPVarTileMetadataPartialFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - auto st = frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); - CHECK(st.ok()); + frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); // Do fragment metadata first for attribute. { @@ -1385,22 +1324,11 @@ struct CPPVarTileMetadataPartialFx { } // Load attribute metadata. - std::vector names_min{"a"}; - st = frag_meta[0]->load_tile_min_values(enc_key, std::move(names_min)); - CHECK(st.ok()); - - std::vector names_max{"a"}; - st = frag_meta[0]->load_tile_max_values(enc_key, std::move(names_max)); - CHECK(st.ok()); - - std::vector names_sum{"a"}; - st = frag_meta[0]->load_tile_sum_values(enc_key, std::move(names_sum)); - CHECK(st.ok()); - - std::vector names_null_count{"a"}; - st = frag_meta[0]->load_tile_null_count_values( - enc_key, std::move(names_null_count)); - CHECK(st.ok()); + std::vector names{"a"}; + frag_meta[0]->load_tile_min_values(enc_key, names); + frag_meta[0]->load_tile_max_values(enc_key, names); + frag_meta[0]->load_tile_sum_values(enc_key, names); + frag_meta[0]->load_tile_null_count_values(enc_key, names); std::vector correct_tile_mins{"1.1", "2.1", "3.2", "4.1"}; std::vector correct_tile_maxs{"1.7", "2.6", "3.8", "4.9"}; diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 730063ae24fb..6179e760954e 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1211,8 +1211,9 @@ std::unordered_map Array::get_average_var_cell_sizes() return Status::Ok(); } - return fragment_metadata_[f]->load_tile_var_sizes( + fragment_metadata_[f]->load_tile_var_sizes( *encryption_key_, var_name); + return Status::Ok(); })); } @@ -1414,8 +1415,7 @@ Status Array::compute_max_buffer_sizes( // arrays, this will not be accurate, as it accounts only for the // non-empty regions of the subarray. for (auto& meta : fragment_metadata_) { - RETURN_NOT_OK( - meta->add_max_buffer_sizes(*encryption_key_, subarray, buffer_sizes)); + meta->add_max_buffer_sizes(*encryption_key_, subarray, buffer_sizes); } // Prepare an NDRange for the subarray diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 0360846cfeff..8fe893842e4e 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -121,12 +121,12 @@ Status FragmentMetaConsolidator::consolidate( auto status = parallel_for( storage_manager_->compute_tp(), 0, tiles.size(), [&](size_t i) { SizeComputationSerializer size_computation_serializer; - throw_if_not_ok(meta[i]->write_footer(size_computation_serializer)); + meta[i]->write_footer(size_computation_serializer); tiles[i].reset(tdb_new( WriterTile, WriterTile::from_generic(size_computation_serializer.size()))); Serializer serializer(tiles[i]->data(), tiles[i]->size()); - throw_if_not_ok(meta[i]->write_footer(serializer)); + meta[i]->write_footer(serializer); return Status::Ok(); }); diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 72b140abdb04..a252fbad2238 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -500,7 +500,7 @@ Status FragmentInfo::get_mbr_num(uint32_t fid, uint64_t* mbr_num) { } auto meta = single_fragment_info_vec_[fid].meta(); - RETURN_NOT_OK(meta->load_rtree(enc_key_)); + meta->load_rtree(enc_key_); *mbr_num = meta->mbrs().size(); return Status::Ok(); @@ -522,7 +522,7 @@ Status FragmentInfo::get_mbr( Status_FragmentInfoError("Cannot get MBR; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - RETURN_NOT_OK(meta->load_rtree(enc_key_)); + meta->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -604,7 +604,7 @@ Status FragmentInfo::get_mbr_var_size( Status_FragmentInfoError("Cannot get MBR; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - RETURN_NOT_OK(meta->load_rtree(enc_key_)); + meta->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -684,7 +684,7 @@ Status FragmentInfo::get_mbr_var( Status_FragmentInfoError("Cannot get MBR var; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - RETURN_NOT_OK(meta->load_rtree(enc_key_)); + meta->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -914,7 +914,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { } if (preload_rtrees & !meta->dense()) { - RETURN_NOT_OK(meta->load_rtree(enc_key_)); + meta->load_rtree(enc_key_); } return Status::Ok(); @@ -1067,8 +1067,7 @@ tuple> FragmentInfo::load( new_fragment_uri, timestamp_range, !sparse); - RETURN_NOT_OK_TUPLE( - meta->load(enc_key_, nullptr, 0, array_schemas_all_), nullopt); + meta->load(enc_key_, nullptr, 0, array_schemas_all_); // This is important for format version > 2 sparse = !meta->dense(); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index f4afa12b5acc..97a805e5afee 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -162,10 +162,10 @@ FragmentMetadata& FragmentMetadata::operator=(const FragmentMetadata& other) { /* API */ /* ****************************** */ -Status FragmentMetadata::set_mbr(uint64_t tile, const NDRange& mbr) { +void FragmentMetadata::set_mbr(uint64_t tile, const NDRange& mbr) { // For easy reference tile += tile_index_base_; - RETURN_NOT_OK(rtree_.set_leaf(tile, mbr)); + throw_if_not_ok(rtree_.set_leaf(tile, mbr)); return expand_non_empty_domain(mbr); } @@ -502,31 +502,32 @@ std::vector FragmentMetadata::dim_types() const { return ret; } -Status FragmentMetadata::add_max_buffer_sizes( +void FragmentMetadata::add_max_buffer_sizes( const EncryptionKey& encryption_key, const void* subarray, std::unordered_map>* buffer_sizes) { // Dense case - if (dense_) - return add_max_buffer_sizes_dense(subarray, buffer_sizes); + if (dense_) { + add_max_buffer_sizes_dense(subarray, buffer_sizes); + } else { + // Convert subarray to NDRange + auto dim_num = array_schema_->dim_num(); + auto sub_ptr = (const unsigned char*)subarray; + NDRange sub_nd(dim_num); + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + auto r_size{2 * array_schema_->dimension_ptr(d)->coord_size()}; + sub_nd[d] = Range(&sub_ptr[offset], r_size); + offset += r_size; + } - // Convert subarray to NDRange - auto dim_num = array_schema_->dim_num(); - auto sub_ptr = (const unsigned char*)subarray; - NDRange sub_nd(dim_num); - uint64_t offset = 0; - for (unsigned d = 0; d < dim_num; ++d) { - auto r_size{2 * array_schema_->dimension_ptr(d)->coord_size()}; - sub_nd[d] = Range(&sub_ptr[offset], r_size); - offset += r_size; + // Sparse case + add_max_buffer_sizes_sparse(encryption_key, sub_nd, buffer_sizes); } - - // Sparse case - return add_max_buffer_sizes_sparse(encryption_key, sub_nd, buffer_sizes); } -Status FragmentMetadata::add_max_buffer_sizes_dense( +void FragmentMetadata::add_max_buffer_sizes_dense( const void* subarray, std::unordered_map>* buffer_sizes) { @@ -589,16 +590,14 @@ Status FragmentMetadata::add_max_buffer_sizes_dense( return add_max_buffer_sizes_dense( static_cast(subarray), buffer_sizes); default: - return LOG_STATUS(Status_FragmentMetadataError( + throw FragmentMetadataStatusException( "Cannot compute add read buffer sizes for dense array; Unsupported " - "domain type")); + "domain type"); } - - return Status::Ok(); } template -Status FragmentMetadata::add_max_buffer_sizes_dense( +void FragmentMetadata::add_max_buffer_sizes_dense( const T* subarray, std::unordered_map>* buffer_sizes) { @@ -617,16 +616,14 @@ Status FragmentMetadata::add_max_buffer_sizes_dense( } } } - - return Status::Ok(); } -Status FragmentMetadata::add_max_buffer_sizes_sparse( +void FragmentMetadata::add_max_buffer_sizes_sparse( const EncryptionKey& encryption_key, const NDRange& subarray, std::unordered_map>* buffer_sizes) { - RETURN_NOT_OK(load_rtree(encryption_key)); + load_rtree(encryption_key); // Get tile overlap std::vector is_default(subarray.size(), false); @@ -660,8 +657,6 @@ Status FragmentMetadata::add_max_buffer_sizes_sparse( } } } - - return Status::Ok(); } uint64_t FragmentMetadata::fragment_size() const { @@ -693,13 +688,12 @@ uint64_t FragmentMetadata::fragment_size() const { return size; } -Status FragmentMetadata::get_tile_overlap( +void FragmentMetadata::get_tile_overlap( const NDRange& range, std::vector& is_default, TileOverlap* tile_overlap) { assert(version_ <= 2 || loaded_metadata_.rtree_); *tile_overlap = rtree_.get_tile_overlap(range, is_default); - return Status::Ok(); } void FragmentMetadata::compute_tile_bitmap( @@ -708,7 +702,7 @@ void FragmentMetadata::compute_tile_bitmap( rtree_.compute_tile_bitmap(range, d, tile_bitmap); } -Status FragmentMetadata::init_domain(const NDRange& non_empty_domain) { +void FragmentMetadata::init_domain(const NDRange& non_empty_domain) { auto& domain{array_schema_->domain()}; // Sanity check @@ -731,15 +725,13 @@ Status FragmentMetadata::init_domain(const NDRange& non_empty_domain) { domain_ = non_empty_domain_; domain.expand_to_tiles(&domain_); } - - return Status::Ok(); } -Status FragmentMetadata::init(const NDRange& non_empty_domain) { +void FragmentMetadata::init(const NDRange& non_empty_domain) { // For easy reference auto num = num_dims_and_attrs(); - RETURN_NOT_OK(init_domain(non_empty_domain)); + init_domain(non_empty_domain); // Set last tile cell number last_tile_cell_num_ = 0; @@ -780,11 +772,9 @@ Status FragmentMetadata::init(const NDRange& non_empty_domain) { fragment_maxs_.resize(num); fragment_sums_.resize(num); fragment_null_counts_.resize(num); - - return Status::Ok(); } -Status FragmentMetadata::load( +void FragmentMetadata::load( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, uint64_t offset, @@ -794,7 +784,7 @@ Status FragmentMetadata::load( // Load the metadata file size when we are not reading from consolidated // buffer if (fragment_metadata_tile == nullptr) { - RETURN_NOT_OK( + throw_if_not_ok( storage_manager_->vfs()->file_size(meta_uri, &meta_file_size_)); } @@ -804,10 +794,10 @@ Status FragmentMetadata::load( if (format_version <= 2) { return load_v1_v2(encryption_key, array_schemas); + } else { + return load_v3_or_higher( + encryption_key, fragment_metadata_tile, offset, array_schemas); } - - return load_v3_or_higher( - encryption_key, fragment_metadata_tile, offset, array_schemas); } void FragmentMetadata::store(const EncryptionKey& encryption_key) { @@ -824,13 +814,13 @@ void FragmentMetadata::store(const EncryptionKey& encryption_key) { } try { if (version_ <= 10) { - throw_if_not_ok(store_v7_v10(encryption_key)); + store_v7_v10(encryption_key); } else if (version_ == 11) { - throw_if_not_ok(store_v11(encryption_key)); + store_v11(encryption_key); } else if (version_ <= 14) { - throw_if_not_ok(store_v12_v14(encryption_key)); + store_v12_v14(encryption_key); } else { - throw_if_not_ok(store_v15_or_higher(encryption_key)); + store_v15_or_higher(encryption_key); } return; } catch (...) { @@ -843,7 +833,7 @@ void FragmentMetadata::store(const EncryptionKey& encryption_key) { } } -Status FragmentMetadata::store_v7_v10(const EncryptionKey& encryption_key) { +void FragmentMetadata::store_v7_v10(const EncryptionKey& encryption_key) { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); auto num = num_dims_and_attrs(); @@ -851,7 +841,7 @@ Status FragmentMetadata::store_v7_v10(const EncryptionKey& encryption_key) { // Store R-Tree gt_offsets_.rtree_ = offset; - throw_if_not_ok(store_rtree(encryption_key, &nbytes)); + store_rtree(encryption_key, &nbytes); offset += nbytes; // Store tile offsets @@ -887,13 +877,13 @@ Status FragmentMetadata::store_v7_v10(const EncryptionKey& encryption_key) { } // Store footer - throw_if_not_ok(store_footer(encryption_key)); + store_footer(encryption_key); // Close file - return storage_manager_->vfs()->close_file(fragment_metadata_uri); + throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); } -Status FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { +void FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); auto num = num_dims_and_attrs(); @@ -901,7 +891,7 @@ Status FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { // Store R-Tree gt_offsets_.rtree_ = offset; - RETURN_NOT_OK_ELSE(store_rtree(encryption_key, &nbytes), clean_up()); + store_rtree(encryption_key, &nbytes); offset += nbytes; // Store tile offsets @@ -969,13 +959,13 @@ Status FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { } // Store footer - RETURN_NOT_OK_ELSE(store_footer(encryption_key), clean_up()); + store_footer(encryption_key); // Close file - return storage_manager_->vfs()->close_file(fragment_metadata_uri); + throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); } -Status FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { +void FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); auto num = num_dims_and_attrs(); @@ -983,7 +973,7 @@ Status FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { // Store R-Tree gt_offsets_.rtree_ = offset; - throw_if_not_ok(store_rtree(encryption_key, &nbytes)); + store_rtree(encryption_key, &nbytes); offset += nbytes; // Store tile offsets @@ -1056,13 +1046,13 @@ Status FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { offset += nbytes; // Store footer - throw_if_not_ok(store_footer(encryption_key)); + store_footer(encryption_key); // Close file - return storage_manager_->vfs()->close_file(fragment_metadata_uri); + throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); } -Status FragmentMetadata::store_v15_or_higher( +void FragmentMetadata::store_v15_or_higher( const EncryptionKey& encryption_key) { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); @@ -1071,7 +1061,7 @@ Status FragmentMetadata::store_v15_or_higher( // Store R-Tree gt_offsets_.rtree_ = offset; - throw_if_not_ok(store_rtree(encryption_key, &nbytes)); + store_rtree(encryption_key, &nbytes); offset += nbytes; // Store tile offsets @@ -1149,13 +1139,13 @@ Status FragmentMetadata::store_v15_or_higher( offset += nbytes; // Store footer - throw_if_not_ok(store_footer(encryption_key)); + store_footer(encryption_key); // Close file - return storage_manager_->vfs()->close_file(fragment_metadata_uri); + throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); } -Status FragmentMetadata::set_num_tiles(uint64_t num_tiles) { +void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { for (auto& it : idx_map_) { auto i = it.second; assert(num_tiles >= tile_offsets_[i].size()); @@ -1197,8 +1187,6 @@ Status FragmentMetadata::set_num_tiles(uint64_t num_tiles) { throw_if_not_ok(rtree_.set_leaf_num(num_tiles)); sparse_tile_num_ = num_tiles; } - - return Status::Ok(); } void FragmentMetadata::set_last_tile_cell_num(uint64_t cell_num) { @@ -1213,10 +1201,10 @@ uint64_t FragmentMetadata::tile_num() const { return sparse_tile_num_; } -tuple> FragmentMetadata::encode_name( - const std::string& name) const { - if (version_ <= 7) - return {Status::Ok(), name}; +std::string FragmentMetadata::encode_name(const std::string& name) const { + if (version_ <= 7) { + return name; + } if (version_ == 8) { static const std::unordered_map percent_encoding{ @@ -1255,15 +1243,14 @@ tuple> FragmentMetadata::encode_name( percent_encoded_name << percent_encoding.at(c); } - return {Status::Ok(), percent_encoded_name.str()}; + return percent_encoded_name.str(); } assert(version_ > 8); const auto iter = idx_map_.find(name); - if (iter == idx_map_.end()) - return { - Status_FragmentMetadataError("Name " + name + " not in idx_map_"), - std::nullopt}; + if (iter == idx_map_.end()) { + throw FragmentMetadataStatusException("Name " + name + " not in idx_map_"); + } const unsigned idx = iter->second; @@ -1271,7 +1258,7 @@ tuple> FragmentMetadata::encode_name( for (unsigned i = 0; i < attributes.size(); ++i) { const std::string attr_name = attributes[i]->name(); if (attr_name == name) { - return {Status::Ok(), "a" + std::to_string(idx)}; + return "a" + std::to_string(idx); } } @@ -1279,68 +1266,53 @@ tuple> FragmentMetadata::encode_name( const auto& dim_name{array_schema_->dimension_ptr(i)->name()}; if (dim_name == name) { const unsigned dim_idx = idx - array_schema_->attribute_num() - 1; - return {Status::Ok(), "d" + std::to_string(dim_idx)}; + return "d" + std::to_string(dim_idx); } } if (name == constants::coords) { - return {Status::Ok(), name}; + return name; } if (name == constants::timestamps) { - return {Status::Ok(), "t"}; + return "t"; } if (name == constants::delete_timestamps) { - return {Status::Ok(), "dt"}; + return "dt"; } if (name == constants::delete_condition_index) { - return {Status::Ok(), "dci"}; + return "dci"; } - auto err = "Unable to locate dimension/attribute " + name; - return {Status_FragmentMetadataError(err), std::nullopt}; + throw FragmentMetadataStatusException( + "Unable to locate dimension/attribute " + name); } -tuple> FragmentMetadata::uri( - const std::string& name) const { - auto&& [st, encoded_name] = encode_name(name); - if (!st.ok()) - return {st, std::nullopt}; - - return {st, fragment_uri_.join_path(*encoded_name + constants::file_suffix)}; +URI FragmentMetadata::uri(const std::string& name) const { + auto encoded_name = encode_name(name); + return fragment_uri_.join_path(encoded_name + constants::file_suffix); } -tuple> FragmentMetadata::var_uri( - const std::string& name) const { - auto&& [st, encoded_name] = encode_name(name); - if (!st.ok()) - return {st, std::nullopt}; - - return { - st, - fragment_uri_.join_path(*encoded_name + "_var" + constants::file_suffix)}; +URI FragmentMetadata::var_uri(const std::string& name) const { + auto encoded_name = encode_name(name); + return fragment_uri_.join_path( + encoded_name + "_var" + constants::file_suffix); } -tuple> FragmentMetadata::validity_uri( - const std::string& name) const { - auto&& [st, encoded_name] = encode_name(name); - if (!st.ok()) - return {st, std::nullopt}; - - return { - st, - fragment_uri_.join_path( - *encoded_name + "_validity" + constants::file_suffix)}; +URI FragmentMetadata::validity_uri(const std::string& name) const { + auto encoded_name = encode_name(name); + return fragment_uri_.join_path( + encoded_name + "_validity" + constants::file_suffix); } const std::string& FragmentMetadata::array_schema_name() { return array_schema_name_; } -Status FragmentMetadata::load_tile_offsets( - const EncryptionKey& encryption_key, std::vector&& names) { +void FragmentMetadata::load_tile_offsets( + const EncryptionKey& encryption_key, std::vector& names) { // Sort 'names' in ascending order of their index. The // motivation is to load the offsets in order of their // layout for sequential reads to the file. @@ -1357,26 +1329,26 @@ Status FragmentMetadata::load_tile_offsets( // var offsets. Load all of the fixed offsets // first. for (const auto& name : names) { - RETURN_NOT_OK(load_tile_offsets(encryption_key, idx_map_[name])); + load_tile_offsets(encryption_key, idx_map_[name]); } // Load all of the var offsets. for (const auto& name : names) { - if (array_schema_->var_size(name)) - RETURN_NOT_OK(load_tile_var_offsets(encryption_key, idx_map_[name])); + if (array_schema_->var_size(name)) { + load_tile_var_offsets(encryption_key, idx_map_[name]); + } } // Load all of the var offsets. for (const auto& name : names) { - if (array_schema_->is_nullable(name)) - RETURN_NOT_OK(load_tile_validity_offsets(encryption_key, idx_map_[name])); + if (array_schema_->is_nullable(name)) { + load_tile_validity_offsets(encryption_key, idx_map_[name]); + } } - - return Status::Ok(); } -Status FragmentMetadata::load_tile_min_values( - const EncryptionKey& encryption_key, std::vector&& names) { +void FragmentMetadata::load_tile_min_values( + const EncryptionKey& encryption_key, std::vector& names) { // Sort 'names' in ascending order of their index. The // motivation is to load the offsets in order of their // layout for sequential reads to the file. @@ -1391,14 +1363,12 @@ Status FragmentMetadata::load_tile_min_values( // Load all the min values. for (const auto& name : names) { - RETURN_NOT_OK(load_tile_min_values(encryption_key, idx_map_[name])); + load_tile_min_values(encryption_key, idx_map_[name]); } - - return Status::Ok(); } -Status FragmentMetadata::load_tile_max_values( - const EncryptionKey& encryption_key, std::vector&& names) { +void FragmentMetadata::load_tile_max_values( + const EncryptionKey& encryption_key, std::vector& names) { // Sort 'names' in ascending order of their index. The // motivation is to load the offsets in order of their // layout for sequential reads to the file. @@ -1413,14 +1383,12 @@ Status FragmentMetadata::load_tile_max_values( // Load all the max values. for (const auto& name : names) { - RETURN_NOT_OK(load_tile_max_values(encryption_key, idx_map_[name])); + load_tile_max_values(encryption_key, idx_map_[name]); } - - return Status::Ok(); } -Status FragmentMetadata::load_tile_sum_values( - const EncryptionKey& encryption_key, std::vector&& names) { +void FragmentMetadata::load_tile_sum_values( + const EncryptionKey& encryption_key, std::vector& names) { // Sort 'names' in ascending order of their index. The // motivation is to load the offsets in order of their // layout for sequential reads to the file. @@ -1435,14 +1403,12 @@ Status FragmentMetadata::load_tile_sum_values( // Load all the sum values. for (const auto& name : names) { - RETURN_NOT_OK(load_tile_sum_values(encryption_key, idx_map_[name])); + load_tile_sum_values(encryption_key, idx_map_[name]); } - - return Status::Ok(); } -Status FragmentMetadata::load_tile_null_count_values( - const EncryptionKey& encryption_key, std::vector&& names) { +void FragmentMetadata::load_tile_null_count_values( + const EncryptionKey& encryption_key, std::vector& names) { // Sort 'names' in ascending order of their index. The // motivation is to load the offsets in order of their // layout for sequential reads to the file. @@ -1457,27 +1423,24 @@ Status FragmentMetadata::load_tile_null_count_values( // Load all the null count values. for (const auto& name : names) { - RETURN_NOT_OK(load_tile_null_count_values(encryption_key, idx_map_[name])); + load_tile_null_count_values(encryption_key, idx_map_[name]); } - - return Status::Ok(); } -Status FragmentMetadata::load_fragment_min_max_sum_null_count( +void FragmentMetadata::load_fragment_min_max_sum_null_count( const EncryptionKey& encryption_key) { - if (loaded_metadata_.fragment_min_max_sum_null_count_) - return Status::Ok(); + if (loaded_metadata_.fragment_min_max_sum_null_count_) { + return; + } - if (version_ <= 11) - return Status::Ok(); + if (version_ <= 11) { + return; + } std::lock_guard lock(mtx_); - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.fragment_min_max_sum_null_count_offset_); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_fragment_min_max_sum_null_count_size", tile.size()); @@ -1485,27 +1448,22 @@ Status FragmentMetadata::load_fragment_min_max_sum_null_count( load_fragment_min_max_sum_null_count(deserializer); loaded_metadata_.fragment_min_max_sum_null_count_ = true; - - return Status::Ok(); } -Status FragmentMetadata::load_processed_conditions( +void FragmentMetadata::load_processed_conditions( const EncryptionKey& encryption_key) { if (loaded_metadata_.processed_conditions_) { - return Status::Ok(); + return; } if (version_ <= 15) { - return Status::Ok(); + return; } std::lock_guard lock(mtx_); - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.processed_conditions_offsets_); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_processed_conditions_size", tile.size()); @@ -1513,8 +1471,6 @@ Status FragmentMetadata::load_processed_conditions( load_processed_conditions(deserializer); loaded_metadata_.processed_conditions_ = true; - - return Status::Ok(); } uint64_t FragmentMetadata::file_offset( @@ -1648,7 +1604,7 @@ uint64_t FragmentMetadata::tile_var_size( template T FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx) { + const std::string& name, uint64_t tile_idx) const { const auto var_size = array_schema_->var_size(name); if (var_size) { throw FragmentMetadataStatusException( @@ -1673,13 +1629,13 @@ T FragmentMetadata::get_tile_min_as( } auto size = array_schema_->cell_size(name); - void* min = &tile_min_buffer_[idx][tile_idx * size]; - return *static_cast(min); + const void* min = &tile_min_buffer_[idx][tile_idx * size]; + return *static_cast(min); } template <> std::string_view FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx) { + const std::string& name, uint64_t tile_idx) const { const auto type = array_schema_->type(name); const auto var_size = array_schema_->var_size(name); if (!var_size && type != Datatype::STRING_ASCII && type != Datatype::CHAR) { @@ -1717,18 +1673,18 @@ std::string_view FragmentMetadata::get_tile_min_as( return {}; } - char* min = &tile_min_var_buffer_[idx][min_offset]; + const char* min = &tile_min_var_buffer_[idx][min_offset]; return {min, size}; } else { auto size = static_cast(array_schema_->cell_size(name)); - void* min = &tile_min_buffer_[idx][tile_idx * size]; - return {static_cast(min), size}; + const void* min = &tile_min_buffer_[idx][tile_idx * size]; + return {static_cast(min), size}; } } template T FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx) { + const std::string& name, uint64_t tile_idx) const { const auto var_size = array_schema_->var_size(name); if (var_size) { throw FragmentMetadataStatusException( @@ -1753,13 +1709,13 @@ T FragmentMetadata::get_tile_max_as( } auto size = array_schema_->cell_size(name); - void* max = &tile_max_buffer_[idx][tile_idx * size]; - return *static_cast(max); + const void* max = &tile_max_buffer_[idx][tile_idx * size]; + return *static_cast(max); } template <> std::string_view FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx) { + const std::string& name, uint64_t tile_idx) const { const auto type = array_schema_->type(name); const auto var_size = array_schema_->var_size(name); if (!var_size && type != Datatype::STRING_ASCII && type != Datatype::CHAR) { @@ -1797,17 +1753,17 @@ std::string_view FragmentMetadata::get_tile_max_as( return {}; } - char* max = &tile_max_var_buffer_[idx][max_offset]; + const char* max = &tile_max_var_buffer_[idx][max_offset]; return {max, size}; } else { auto size = static_cast(array_schema_->cell_size(name)); - void* max = &tile_max_buffer_[idx][tile_idx * size]; - return {static_cast(max), size}; + const void* max = &tile_max_buffer_[idx][tile_idx * size]; + return {static_cast(max), size}; } } -void* FragmentMetadata::get_tile_sum( - const std::string& name, uint64_t tile_idx) { +const void* FragmentMetadata::get_tile_sum( + const std::string& name, uint64_t tile_idx) const { auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; @@ -1824,12 +1780,12 @@ void* FragmentMetadata::get_tile_sum( "Trying to access tile sum metadata that's not present"); } - void* sum = &tile_sums_[idx][tile_idx * sizeof(uint64_t)]; + const void* sum = &tile_sums_[idx][tile_idx * sizeof(uint64_t)]; return sum; } uint64_t FragmentMetadata::get_tile_null_count( - const std::string& name, uint64_t tile_idx) { + const std::string& name, uint64_t tile_idx) const { auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; @@ -1963,7 +1919,7 @@ bool FragmentMetadata::operator<(const FragmentMetadata& metadata) const { fragment_uri_ < metadata.fragment_uri_); } -Status FragmentMetadata::write_footer(Serializer& serializer) const { +void FragmentMetadata::write_footer(Serializer& serializer) const { write_version(serializer); if (version_ >= 10) { write_array_schema_name(serializer); @@ -1985,43 +1941,37 @@ Status FragmentMetadata::write_footer(Serializer& serializer) const { write_file_var_sizes(serializer); write_file_validity_sizes(serializer); write_generic_tile_offsets(serializer); - return Status::Ok(); } -Status FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { - if (version_ <= 2) - return Status::Ok(); +void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { + if (version_ <= 2) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.rtree_) - return Status::Ok(); - - auto&& [st, tile_opt] = - read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; + if (loaded_metadata_.rtree_) { + return; + } + auto tile = read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); storage_manager_->stats()->add_counter("read_rtree_size", tile.size()); // Use the serialized buffer size to approximate memory usage of the rtree. if (memory_tracker_ != nullptr && !memory_tracker_->take_memory( tile.size(), MemoryTracker::MemoryType::RTREE)) { - return LOG_STATUS(Status_FragmentMetadataError( + throw FragmentMetadataStatusException( "Cannot load R-tree; Insufficient memory budget; Needed " + std::to_string(tile.size()) + " but only had " + std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget()))); + " from budget " + std::to_string(memory_tracker_->get_memory_budget())); } Deserializer deserializer(tile.data(), tile.size()); rtree_.deserialize(deserializer, &array_schema_->domain(), version_); loaded_metadata_.rtree_ = true; - - return Status::Ok(); } void FragmentMetadata::free_rtree() { @@ -2089,15 +2039,16 @@ void FragmentMetadata::free_tile_offsets() { } } -Status FragmentMetadata::load_tile_var_sizes( +void FragmentMetadata::load_tile_var_sizes( const EncryptionKey& encryption_key, const std::string& name) { - if (version_ <= 2) - return Status::Ok(); + if (version_ <= 2) { + return; + } auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; - return (load_tile_var_sizes(encryption_key, idx)); + load_tile_var_sizes(encryption_key, idx); } /* ****************************** */ @@ -2108,7 +2059,7 @@ uint64_t FragmentMetadata::footer_size() const { return footer_size_; } -Status FragmentMetadata::get_footer_offset_and_size( +void FragmentMetadata::get_footer_offset_and_size( uint64_t* offset, uint64_t* size) const { auto name = fragment_uri_.remove_trailing_slash().last_path_part(); auto fragment_format_version = utils::parse::get_fragment_version(name); @@ -2126,14 +2077,12 @@ Status FragmentMetadata::get_footer_offset_and_size( URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); uint64_t size_offset = meta_file_size_ - sizeof(uint64_t); - RETURN_NOT_OK(storage_manager_->vfs()->read( + throw_if_not_ok(storage_manager_->vfs()->read( fragment_metadata_uri, size_offset, size, sizeof(uint64_t))); *offset = meta_file_size_ - *size - sizeof(uint64_t); storage_manager_->stats()->add_counter( "read_frag_meta_size", sizeof(uint64_t)); } - - return Status::Ok(); } uint64_t FragmentMetadata::footer_size_v3_v4() const { @@ -2372,70 +2321,65 @@ void FragmentMetadata::get_subarray_tile_domain( } } -Status FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { +void FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { std::lock_guard lock(mtx_); // Case the non-empty domain is not initialized yet if (non_empty_domain_.empty()) { non_empty_domain_ = mbr; - return Status::Ok(); + return; } // Expand existing non-empty domain array_schema_->domain().expand_ndrange(mbr, &non_empty_domain_); - - return Status::Ok(); } -Status FragmentMetadata::load_tile_offsets( +void FragmentMetadata::load_tile_offsets( const EncryptionKey& encryption_key, unsigned idx) { if (version_ <= 2) { - return Status::Ok(); + return; } // If the tile offset is already loaded, exit early to avoid the lock - if (loaded_metadata_.tile_offsets_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_offsets_[idx]) { + return; + } std::lock_guard lock(tile_offsets_mtx_[idx]); - if (loaded_metadata_.tile_offsets_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_offsets_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter("read_tile_offsets_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_offsets(idx, deserializer); loaded_metadata_.tile_offsets_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_var_offsets( +void FragmentMetadata::load_tile_var_offsets( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 2) - return Status::Ok(); + if (version_ <= 2) { + return; + } // If the tile var offset is already loaded, exit early to avoid the lock - if (loaded_metadata_.tile_var_offsets_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_var_offsets_[idx]) { + return; + } std::lock_guard lock(tile_var_offsets_mtx_[idx]); - if (loaded_metadata_.tile_var_offsets_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_var_offsets_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_tile_var_offsets_size", tile.size()); @@ -2443,25 +2387,22 @@ Status FragmentMetadata::load_tile_var_offsets( load_tile_var_offsets(idx, deserializer); loaded_metadata_.tile_var_offsets_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_var_sizes( +void FragmentMetadata::load_tile_var_sizes( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 2) - return Status::Ok(); + if (version_ <= 2) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_var_sizes_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_var_sizes_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_sizes_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_tile_var_sizes_size", tile.size()); @@ -2469,126 +2410,111 @@ Status FragmentMetadata::load_tile_var_sizes( load_tile_var_sizes(idx, deserializer); loaded_metadata_.tile_var_sizes_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_validity_offsets( +void FragmentMetadata::load_tile_validity_offsets( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 6) - return Status::Ok(); + if (version_ <= 6) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_validity_offsets_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_validity_offsets_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_validity_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_tile_validity_offsets_size", tile.size()); ConstBuffer cbuff(tile.data(), tile.size()); - RETURN_NOT_OK(load_tile_validity_offsets(idx, &cbuff)); + load_tile_validity_offsets(idx, &cbuff); loaded_metadata_.tile_validity_offsets_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_min_values( +void FragmentMetadata::load_tile_min_values( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 10) - return Status::Ok(); + if (version_ < constants::tile_metadata_min_version) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_min_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_min_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_min_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter("read_tile_min_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_min_values(idx, deserializer); loaded_metadata_.tile_min_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_max_values( +void FragmentMetadata::load_tile_max_values( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 10) - return Status::Ok(); + if (version_ < constants::tile_metadata_min_version) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_max_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_max_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_max_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter("read_tile_max_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_max_values(idx, deserializer); loaded_metadata_.tile_max_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_sum_values( +void FragmentMetadata::load_tile_sum_values( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 10) - return Status::Ok(); + if (version_ < constants::tile_metadata_min_version) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_sum_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_sum_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_sum_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter("read_tile_sum_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_sum_values(idx, deserializer); loaded_metadata_.tile_sum_[idx] = true; - - return Status::Ok(); } -Status FragmentMetadata::load_tile_null_count_values( +void FragmentMetadata::load_tile_null_count_values( const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 10) - return Status::Ok(); + if (version_ < constants::tile_metadata_min_version) { + return; + } std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_null_count_[idx]) - return Status::Ok(); + if (loaded_metadata_.tile_null_count_[idx]) { + return; + } - auto&& [st, tile_opt] = read_generic_tile_from_file( + auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_null_count_offsets_[idx]); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - storage_manager_->stats()->add_counter( "read_tile_null_count_size", tile.size()); @@ -2596,8 +2522,6 @@ Status FragmentMetadata::load_tile_null_count_values( load_tile_null_count_values(idx, deserializer); loaded_metadata_.tile_null_count_[idx] = true; - - return Status::Ok(); } // ===== FORMAT ===== @@ -2861,7 +2785,6 @@ void FragmentMetadata::load_non_empty_domain_v5_or_higher( // Applicable only to versions 1 and 2 void FragmentMetadata::load_tile_offsets(Deserializer& deserializer) { - Status st; uint64_t tile_offsets_num = 0; unsigned int attribute_num = array_schema_->attribute_num(); @@ -2934,7 +2857,6 @@ void FragmentMetadata::load_tile_offsets( // tile_var_offsets_attr#_#1 (uint64_t) // tile_ver_offsets_attr#_#2 (uint64_t) ... void FragmentMetadata::load_tile_var_offsets(Deserializer& deserializer) { - Status st; unsigned int attribute_num = array_schema_->attribute_num(); uint64_t tile_var_offsets_num = 0; @@ -3007,7 +2929,6 @@ void FragmentMetadata::load_tile_var_offsets( // tile_var_sizes__attr#_#1 (uint64_t) // tile_var_sizes_attr#_#2 (uint64_t) ... void FragmentMetadata::load_tile_var_sizes(Deserializer& deserializer) { - Status st; unsigned int attribute_num = array_schema_->attribute_num(); uint64_t tile_var_sizes_num = 0; @@ -3070,18 +2991,15 @@ void FragmentMetadata::load_tile_var_sizes( } } -Status FragmentMetadata::load_tile_validity_offsets( +void FragmentMetadata::load_tile_validity_offsets( unsigned idx, ConstBuffer* buff) { - Status st; uint64_t tile_validity_offsets_num = 0; // Get number of tile offsets - st = buff->read(&tile_validity_offsets_num, sizeof(uint64_t)); - if (!st.ok()) { - return LOG_STATUS( - Status_FragmentMetadataError("Cannot load fragment metadata; Reading " - "number of validity tile offsets " - "failed")); + if (!buff->read(&tile_validity_offsets_num, sizeof(uint64_t)).ok()) { + throw FragmentMetadataStatusException( + "Cannot load fragment metadata; Reading number of validity tile " + "offsets failed"); } // Get tile offsets @@ -3090,26 +3008,22 @@ Status FragmentMetadata::load_tile_validity_offsets( if (memory_tracker_ != nullptr && !memory_tracker_->take_memory( size, MemoryTracker::MemoryType::TILE_OFFSETS)) { - return LOG_STATUS(Status_FragmentMetadataError( + throw FragmentMetadataStatusException( "Cannot load tile validity offsets; Insufficient memory budget; " "Needed " + std::to_string(size) + " but only had " + std::to_string(memory_tracker_->get_memory_available()) + " from budget " + - std::to_string(memory_tracker_->get_memory_budget()))); + std::to_string(memory_tracker_->get_memory_budget())); } tile_validity_offsets_[idx].resize(tile_validity_offsets_num); - st = buff->read(&tile_validity_offsets_[idx][0], size); - - if (!st.ok()) { - return LOG_STATUS(Status_FragmentMetadataError( + if (!buff->read(&tile_validity_offsets_[idx][0], size).ok()) { + throw FragmentMetadataStatusException( "Cannot load fragment metadata; Reading validity tile offsets " - "failed")); + "failed"); } } - - return Status::Ok(); } // ===== FORMAT ===== @@ -3124,7 +3038,6 @@ Status FragmentMetadata::load_tile_validity_offsets( // tile_min_values#_buffer_var void FragmentMetadata::load_tile_min_values( unsigned idx, Deserializer& deserializer) { - Status st; uint64_t buffer_size = 0; uint64_t var_buffer_size = 0; @@ -3170,7 +3083,6 @@ void FragmentMetadata::load_tile_min_values( // tile_max_values#_buffer_var void FragmentMetadata::load_tile_max_values( unsigned idx, Deserializer& deserializer) { - Status st; uint64_t buffer_size = 0; uint64_t var_buffer_size = 0; @@ -3577,7 +3489,7 @@ void FragmentMetadata::load_array_schema_name(Deserializer& deserializer) { deserializer.read(&array_schema_name_[0], size); } -Status FragmentMetadata::load_v1_v2( +void FragmentMetadata::load_v1_v2( const EncryptionKey& encryption_key, const std::unordered_map>& array_schemas) { @@ -3587,7 +3499,7 @@ Status FragmentMetadata::load_v1_v2( GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); auto&& [st, tile_opt] = tile_io.read_generic(0, encryption_key, storage_manager_->config()); - RETURN_NOT_OK(st); + throw_if_not_ok(st); auto& tile = *tile_opt; storage_manager_->stats()->add_counter("read_frag_meta_size", tile.size()); @@ -3599,7 +3511,7 @@ Status FragmentMetadata::load_v1_v2( if (schema != array_schemas.end()) { set_array_schema(schema->second); } else { - return Status_FragmentMetadataError( + throw FragmentMetadataStatusException( "Could not find schema" + array_schema_name_ + " in map of schemas loaded.\n" + "Consider reloading the array to check for new array schemas."); @@ -3618,34 +3530,31 @@ Status FragmentMetadata::load_v1_v2( load_file_sizes(deserializer); load_file_var_sizes(deserializer); load_file_validity_sizes(deserializer); - - return Status::Ok(); } -Status FragmentMetadata::load_v3_or_higher( +void FragmentMetadata::load_v3_or_higher( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, uint64_t offset, std::unordered_map> array_schemas) { - RETURN_NOT_OK(load_footer( - encryption_key, fragment_metadata_tile, offset, array_schemas)); - return Status::Ok(); + load_footer(encryption_key, fragment_metadata_tile, offset, array_schemas); } -Status FragmentMetadata::load_footer( +void FragmentMetadata::load_footer( const EncryptionKey&, Tile* fragment_metadata_tile, uint64_t offset, std::unordered_map> array_schemas) { std::lock_guard lock(mtx_); - if (loaded_metadata_.footer_) - return Status::Ok(); + if (loaded_metadata_.footer_) { + return; + } std::shared_ptr tile; if (fragment_metadata_tile == nullptr) { has_consolidated_footer_ = false; - RETURN_NOT_OK(read_file_footer(tile, &footer_offset_, &footer_size_)); + read_file_footer(tile, &footer_offset_, &footer_size_); fragment_metadata_tile = tile.get(); offset = 0; @@ -3743,8 +3652,6 @@ Status FragmentMetadata::load_footer( if (footer_size_ == 0) { footer_size_ = starting_deserializer_size - deserializer.size(); } - - return Status::Ok(); } // ===== FORMAT ===== @@ -3865,13 +3772,11 @@ void FragmentMetadata::write_last_tile_cell_num(Serializer& serializer) const { serializer.write(last_tile_cell_num); } -Status FragmentMetadata::store_rtree( +void FragmentMetadata::store_rtree( const EncryptionKey& encryption_key, uint64_t* nbytes) { auto rtree_tile = write_rtree(); - RETURN_NOT_OK(write_generic_tile_to_file(encryption_key, rtree_tile, nbytes)); + write_generic_tile_to_file(encryption_key, rtree_tile, nbytes); storage_manager_->stats()->add_counter("write_rtree_size", *nbytes); - - return Status::Ok(); } WriterTile FragmentMetadata::write_rtree() { @@ -3928,7 +3833,7 @@ void FragmentMetadata::write_non_empty_domain(Serializer& serializer) const { } } -tuple> FragmentMetadata::read_generic_tile_from_file( +Tile FragmentMetadata::read_generic_tile_from_file( const EncryptionKey& encryption_key, uint64_t offset) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); @@ -3937,12 +3842,12 @@ tuple> FragmentMetadata::read_generic_tile_from_file( GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); auto&& [st, tile_opt] = tile_io.read_generic(offset, encryption_key, storage_manager_->config()); - RETURN_NOT_OK_TUPLE(st, nullopt); + throw_if_not_ok(st); - return {Status::Ok(), std::move(*tile_opt)}; + return std::move(*tile_opt); } -Status FragmentMetadata::read_file_footer( +void FragmentMetadata::read_file_footer( std::shared_ptr& tile, uint64_t* footer_offset, uint64_t* footer_size) const { @@ -3950,7 +3855,7 @@ Status FragmentMetadata::read_file_footer( std::string(constants::fragment_metadata_filename)); // Get footer offset - RETURN_NOT_OK(get_footer_offset_and_size(footer_offset, footer_size)); + get_footer_offset_and_size(footer_offset, footer_size); tile = make_shared(HERE(), Tile::from_generic(*footer_size)); @@ -3959,23 +3864,22 @@ Status FragmentMetadata::read_file_footer( if (memory_tracker_ != nullptr && !memory_tracker_->take_memory( *footer_size, MemoryTracker::MemoryType::FOOTER)) { - return LOG_STATUS(Status_FragmentMetadataError( + throw FragmentMetadataStatusException( "Cannot load file footer; Insufficient memory budget; Needed " + std::to_string(*footer_size) + " but only had " + std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget()))); + " from budget " + std::to_string(memory_tracker_->get_memory_budget())); } // Read footer - return storage_manager_->vfs()->read( + throw_if_not_ok(storage_manager_->vfs()->read( fragment_metadata_uri, *footer_offset, tile->data_as(), - *footer_size); + *footer_size)); } -Status FragmentMetadata::write_generic_tile_to_file( +void FragmentMetadata::write_generic_tile_to_file( const EncryptionKey& encryption_key, WriterTile& tile, uint64_t* nbytes) const { @@ -3983,24 +3887,22 @@ Status FragmentMetadata::write_generic_tile_to_file( std::string(constants::fragment_metadata_filename)); GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); - RETURN_NOT_OK(tile_io.write_generic(&tile, encryption_key, nbytes)); - - return Status::Ok(); + throw_if_not_ok(tile_io.write_generic(&tile, encryption_key, nbytes)); } -Status FragmentMetadata::write_footer_to_file(WriterTile& tile) const { +void FragmentMetadata::write_footer_to_file(WriterTile& tile) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); uint64_t size = tile.size(); - RETURN_NOT_OK(storage_manager_->vfs()->write( + throw_if_not_ok(storage_manager_->vfs()->write( fragment_metadata_uri, tile.data(), tile.size())); // Write the size in the end if there is at least one var-sized dimension - if (!array_schema_->domain().all_dims_fixed() || version_ >= 10) - return storage_manager_->vfs()->write( - fragment_metadata_uri, &size, sizeof(uint64_t)); - return Status::Ok(); + if (!array_schema_->domain().all_dims_fixed() || version_ >= 10) { + throw_if_not_ok(storage_manager_->vfs()->write( + fragment_metadata_uri, &size, sizeof(uint64_t))); + } } void FragmentMetadata::store_tile_offsets( @@ -4012,8 +3914,7 @@ void FragmentMetadata::store_tile_offsets( Serializer serializer(tile.data(), tile.size()); write_tile_offsets(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_tile_offsets_size", *nbytes); } @@ -4040,8 +3941,7 @@ void FragmentMetadata::store_tile_var_offsets( Serializer serializer(tile.data(), tile.size()); write_tile_var_offsets(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter( "write_tile_var_offsets_size", *nbytes); @@ -4070,8 +3970,7 @@ void FragmentMetadata::store_tile_var_sizes( Serializer serializer(tile.data(), tile.size()); write_tile_var_sizes(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_tile_var_sizes_size", *nbytes); } @@ -4098,8 +3997,7 @@ void FragmentMetadata::store_tile_validity_offsets( Serializer serializer(tile.data(), tile.size()); write_tile_validity_offsets(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter( "write_tile_validity_offsets_size", *nbytes); @@ -4128,15 +4026,12 @@ void FragmentMetadata::store_tile_mins( Serializer serializer(tile.data(), tile.size()); write_tile_mins(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_mins_size", *nbytes); } void FragmentMetadata::write_tile_mins(unsigned idx, Serializer& serializer) { - Status st; - // Write size of buffer uint64_t tile_mins_buffer_size = tile_min_buffer_[idx].size(); serializer.write(tile_mins_buffer_size); @@ -4165,15 +4060,12 @@ void FragmentMetadata::store_tile_maxs( Serializer serializer(tile.data(), tile.size()); write_tile_maxs(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_maxs_size", *nbytes); } void FragmentMetadata::write_tile_maxs(unsigned idx, Serializer& serializer) { - Status st; - // Write size of buffer uint64_t tile_maxs_buffer_size = tile_max_buffer_[idx].size(); serializer.write(tile_maxs_buffer_size); @@ -4202,8 +4094,7 @@ void FragmentMetadata::store_tile_sums( Serializer serializer(tile.data(), tile.size()); write_tile_sums(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_sums_size", *nbytes); } @@ -4228,8 +4119,7 @@ void FragmentMetadata::store_tile_null_counts( Serializer serializer(tile.data(), tile.size()); write_tile_null_counts(idx, serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_null_counts_size", *nbytes); } @@ -4249,7 +4139,6 @@ void FragmentMetadata::write_tile_null_counts( void FragmentMetadata::store_fragment_min_max_sum_null_count( uint64_t num, const EncryptionKey& encryption_key, uint64_t* nbytes) { - Status st; Buffer buff; auto serialize_data = [&](Serializer& serializer) { @@ -4282,8 +4171,7 @@ void FragmentMetadata::store_fragment_min_max_sum_null_count( Serializer serializer(tile.data(), tile.size()); serialize_data(serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter("write_null_counts_size", *nbytes); } @@ -4309,8 +4197,7 @@ void FragmentMetadata::store_processed_conditions( Serializer serializer(tile.data(), tile.size()); serialize_processed_conditions(serializer); - - throw_if_not_ok(write_generic_tile_to_file(encryption_key, tile, nbytes)); + write_generic_tile_to_file(encryption_key, tile, nbytes); storage_manager_->stats()->add_counter( "write_processed_conditions_size", *nbytes); @@ -4633,19 +4520,17 @@ void FragmentMetadata::write_has_delete_meta(Serializer& serializer) const { serializer.write(has_delete_meta_); } -Status FragmentMetadata::store_footer(const EncryptionKey&) { +void FragmentMetadata::store_footer(const EncryptionKey&) { SizeComputationSerializer size_computation_serializer; - RETURN_NOT_OK(write_footer(size_computation_serializer)); + write_footer(size_computation_serializer); WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; Serializer serializer(tile.data(), tile.size()); - RETURN_NOT_OK(write_footer(serializer)); - RETURN_NOT_OK(write_footer_to_file(tile)); + write_footer(serializer); + write_footer_to_file(tile); storage_manager_->stats()->add_counter( "write_frag_meta_footer_size", tile.size()); - - return Status::Ok(); } void FragmentMetadata::resize_tile_offsets_vectors(uint64_t size) { @@ -4741,53 +4626,53 @@ template std::vector> FragmentMetadata::compute_overlapping_tile_ids_cov( const double* subarray) const; template int8_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint8_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int16_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint16_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int32_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint32_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int64_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template char FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint64_t FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template float FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template double FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template std::byte FragmentMetadata::get_tile_min_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int8_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint8_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int16_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint16_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int32_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint32_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template int64_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template uint64_t FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template float FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template double FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template std::byte FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; template char FragmentMetadata::get_tile_max_as( - const std::string& name, uint64_t tile_idx); + const std::string& name, uint64_t tile_idx) const; } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 7e89f6f83a9c..14c4a10caed4 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -40,7 +40,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/types.h" @@ -179,9 +178,8 @@ class FragmentMetadata { * maps an attribute to a buffer size pair. For fix-sized attributes, only * the first size is useful. For var-sized attributes, the first is the * offsets size, whereas the second is the data size. - * @return Status */ - Status add_max_buffer_sizes( + void add_max_buffer_sizes( const EncryptionKey& encryption_key, const void* subarray, std::unordered_map>* @@ -198,9 +196,8 @@ class FragmentMetadata { * maps an attribute to a buffer size pair. For fix-sized attributes, only * the first size is useful. For var-sized attributes, the first is the * offsets size, whereas the second is the data size. - * @return Status */ - Status add_max_buffer_sizes_dense( + void add_max_buffer_sizes_dense( const void* subarray, std::unordered_map>* buffer_sizes); @@ -217,10 +214,9 @@ class FragmentMetadata { * maps an attribute to a buffer size pair. For fix-sized attributes, only * the first size is useful. For var-sized attributes, the first is the * offsets size, whereas the second is the data size. - * @return Status */ template - Status add_max_buffer_sizes_dense( + void add_max_buffer_sizes_dense( const T* subarray, std::unordered_map>* buffer_sizes); @@ -237,9 +233,8 @@ class FragmentMetadata { * maps an attribute to a buffer size pair. For fix-sized attributes, only * the first size is useful. For var-sized attributes, the first is the * offsets size, whereas the second is the data size. - * @return Status */ - Status add_max_buffer_sizes_sparse( + void add_max_buffer_sizes_sparse( const EncryptionKey& encryption_key, const NDRange& subarray, std::unordered_map>* @@ -418,7 +413,7 @@ class FragmentMetadata { /** * Retrieves the overlap of all MBRs with the input ND range. */ - Status get_tile_overlap( + void get_tile_overlap( const NDRange& range, std::vector& is_default, TileOverlap* tile_overlap); @@ -434,20 +429,19 @@ class FragmentMetadata { * * @param non_empty_domain The non-empty domain in which the array read/write * will be constrained. - * @return Status */ - Status init(const NDRange& non_empty_domain); + void init(const NDRange& non_empty_domain); /** * Initializes the fragment's internal domain and non-empty domain members */ - Status init_domain(const NDRange& non_empty_domain); + void init_domain(const NDRange& non_empty_domain); /** * Loads the basic metadata from storage or `f_buff` for later * versions if it is not `nullptr`. */ - Status load( + void load( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, uint64_t offset, @@ -461,28 +455,28 @@ class FragmentMetadata { * * Applicable to format versions 7 to 10. */ - Status store_v7_v10(const EncryptionKey& encryption_key); + void store_v7_v10(const EncryptionKey& encryption_key); /** * Stores all the metadata to storage. * * Applicable to format versions 11. */ - Status store_v11(const EncryptionKey& encryption_key); + void store_v11(const EncryptionKey& encryption_key); /** * Stores all the metadata to storage. * * Applicable to format versions 12 or higher. */ - Status store_v12_v14(const EncryptionKey& encryption_key); + void store_v12_v14(const EncryptionKey& encryption_key); /** * Stores all the metadata to storage. * * Applicable to format versions 15 or higher. */ - Status store_v15_or_higher(const EncryptionKey& encryption_key); + void store_v15_or_higher(const EncryptionKey& encryption_key); /** * Simply sets the number of cells for the last tile. @@ -498,18 +492,16 @@ class FragmentMetadata { * * @param tile The tile index whose MBR will be set. * @param mbr The MBR to be set. - * @return Status */ - Status set_mbr(uint64_t tile, const NDRange& mbr); + void set_mbr(uint64_t tile, const NDRange& mbr); /** * Resizes the per-tile metadata vectors for the given number of tiles. This * is not serialized, and is only used during writes. * * @param num_tiles Number of tiles - * @return Status */ - Status set_num_tiles(uint64_t num_tiles); + void set_num_tiles(uint64_t num_tiles); /** * Sets the tile "index base" which is added to the tile index in the set_*() @@ -684,13 +676,13 @@ class FragmentMetadata { uint64_t tile_num() const; /** Returns the URI of the input attribute/dimension. */ - tuple> uri(const std::string& name) const; + URI uri(const std::string& name) const; /** Returns the URI of the input variable-sized attribute/dimension. */ - tuple> var_uri(const std::string& name) const; + URI var_uri(const std::string& name) const; /** Returns the validity URI of the input nullable attribute. */ - tuple> validity_uri(const std::string& name) const; + URI validity_uri(const std::string& name) const; /** Return the array schema name. */ const std::string& array_schema_name(); @@ -801,7 +793,7 @@ class FragmentMetadata { * @return Value. */ template - T get_tile_min_as(const std::string& name, uint64_t tile_idx); + T get_tile_min_as(const std::string& name, uint64_t tile_idx) const; /** * Retrieves the tile max value for a given attribute or dimension and tile @@ -813,7 +805,7 @@ class FragmentMetadata { * @return Value. */ template - T get_tile_max_as(const std::string& name, uint64_t tile_idx); + T get_tile_max_as(const std::string& name, uint64_t tile_idx) const; /** * Retrieves the tile sum value for a given attribute or dimension and tile @@ -823,7 +815,7 @@ class FragmentMetadata { * @param tile_idx The index of the tile in the metadata. * @return Sum. */ - void* get_tile_sum(const std::string& name, uint64_t tile_idx); + const void* get_tile_sum(const std::string& name, uint64_t tile_idx) const; /** * Retrieves the tile null count value for a given attribute or dimension @@ -833,7 +825,8 @@ class FragmentMetadata { * @param tile_idx The index of the tile in the metadata. * @return Count. */ - uint64_t get_tile_null_count(const std::string& name, uint64_t tile_idx); + uint64_t get_tile_null_count( + const std::string& name, uint64_t tile_idx) const; /** * Retrieves the min value for a given attribute or dimension. @@ -904,10 +897,10 @@ class FragmentMetadata { bool operator<(const FragmentMetadata& metadata) const; /** Serializes the fragment metadata footer into the input buffer. */ - Status write_footer(Serializer& serializer) const; + void write_footer(Serializer& serializer) const; /** Loads the R-tree from storage. */ - Status load_rtree(const EncryptionKey& encryption_key); + void load_rtree(const EncryptionKey& encryption_key); /** Frees the memory associated with the rtree. */ void free_rtree(); @@ -919,7 +912,7 @@ class FragmentMetadata { * Loads the variable tile sizes for the input attribute or dimension idx * from storage. * */ - Status load_tile_var_sizes( + void load_tile_var_sizes( const EncryptionKey& encryption_key, const std::string& name); /** @@ -927,58 +920,52 @@ class FragmentMetadata { * * @param encryption_key The key the array got opened with. * @param names The attribute/dimension names. - * @return Status */ - Status load_tile_offsets( - const EncryptionKey& encryption_key, std::vector&& names); + void load_tile_offsets( + const EncryptionKey& encryption_key, std::vector& names); /** * Loads min values for the attribute names. * * @param encryption_key The key the array got opened with. * @param names The attribute names. - * @return Status */ - Status load_tile_min_values( - const EncryptionKey& encryption_key, std::vector&& names); + void load_tile_min_values( + const EncryptionKey& encryption_key, std::vector& names); /** * Loads max values for the attribute names. * * @param encryption_key The key the array got opened with. * @param names The attribute names. - * @return Status */ - Status load_tile_max_values( - const EncryptionKey& encryption_key, std::vector&& names); + void load_tile_max_values( + const EncryptionKey& encryption_key, std::vector& names); /** * Loads sum values for the attribute names. * * @param encryption_key The key the array got opened with. * @param names The attribute names. - * @return Status */ - Status load_tile_sum_values( - const EncryptionKey& encryption_key, std::vector&& names); + void load_tile_sum_values( + const EncryptionKey& encryption_key, std::vector& names); /** * Loads null count values for the attribute names. * * @param encryption_key The key the array got opened with. * @param names The attribute names. - * @return Status */ - Status load_tile_null_count_values( - const EncryptionKey& encryption_key, std::vector&& names); + void load_tile_null_count_values( + const EncryptionKey& encryption_key, std::vector& names); /** * Loads the min max sum null count values for the fragment. * * @param encryption_key The key the array got opened with. - * @return Status */ - Status load_fragment_min_max_sum_null_count( + void load_fragment_min_max_sum_null_count( const EncryptionKey& encryption_key); /** @@ -987,9 +974,8 @@ class FragmentMetadata { * this fragment and don't need to be applied again. * * @param encryption_key The key the array got opened with. - * @return Status */ - Status load_processed_conditions(const EncryptionKey& encryption_key); + void load_processed_conditions(const EncryptionKey& encryption_key); /** * Checks if the fragment overlaps partially (not fully) with a given @@ -1421,7 +1407,7 @@ class FragmentMetadata { * Retrieves the offset in the fragment metadata file of the footer * (which contains the generic tile offsets) along with its size. */ - Status get_footer_offset_and_size(uint64_t* offset, uint64_t* size) const; + void get_footer_offset_and_size(uint64_t* offset, uint64_t* size) const; /** * Returns the size of the fragment metadata footer @@ -1469,55 +1455,51 @@ class FragmentMetadata { /** * Expands the non-empty domain using the input MBR. */ - Status expand_non_empty_domain(const NDRange& mbr); + void expand_non_empty_domain(const NDRange& mbr); /** * Loads the tile offsets for the input attribute or dimension idx * from storage. */ - Status load_tile_offsets(const EncryptionKey& encryption_key, unsigned idx); + void load_tile_offsets(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the variable tile offsets for the input attribute or dimension idx * from storage. */ - Status load_tile_var_offsets( - const EncryptionKey& encryption_key, unsigned idx); + void load_tile_var_offsets(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the variable tile sizes for the input attribute or dimension idx * from storage. * */ - Status load_tile_var_sizes(const EncryptionKey& encryption_key, unsigned idx); + void load_tile_var_sizes(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the validity tile offsets for the input attribute idx from storage. */ - Status load_tile_validity_offsets( + void load_tile_validity_offsets( const EncryptionKey& encryption_key, unsigned idx); /** * Loads the min values for the input attribute idx from storage. */ - Status load_tile_min_values( - const EncryptionKey& encryption_key, unsigned idx); + void load_tile_min_values(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the max values for the input attribute idx from storage. */ - Status load_tile_max_values( - const EncryptionKey& encryption_key, unsigned idx); + void load_tile_max_values(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the sum values for the input attribute idx from storage. */ - Status load_tile_sum_values( - const EncryptionKey& encryption_key, unsigned idx); + void load_tile_sum_values(const EncryptionKey& encryption_key, unsigned idx); /** * Loads the null count values for the input attribute idx from storage. */ - Status load_tile_null_count_values( + void load_tile_null_count_values( const EncryptionKey& encryption_key, unsigned idx); /** Loads the generic tile offsets from the buffer. */ @@ -1693,7 +1675,7 @@ class FragmentMetadata { * Loads the validity tile offsets for the input attribute from the * input buffer. */ - Status load_tile_validity_offsets(unsigned idx, ConstBuffer* buff); + void load_tile_validity_offsets(unsigned idx, ConstBuffer* buff); /** * Loads the min values for the input attribute from the input buffer. @@ -1703,7 +1685,6 @@ class FragmentMetadata { /** * Loads the max values for the input attribute from the input buffer. */ - // Status load_tile_max_values(unsigned idx, ConstBuffer* buff); void load_tile_max_values(unsigned idx, Deserializer& deserializer); /** @@ -1737,7 +1718,7 @@ class FragmentMetadata { void load_sparse_tile_num(Deserializer& deserializer); /** Loads the basic metadata from storage (version 2 or before). */ - Status load_v1_v2( + void load_v1_v2( const EncryptionKey& encryption_key, const std::unordered_map>& array_schemas); @@ -1746,7 +1727,7 @@ class FragmentMetadata { * Loads the basic metadata from storage or the input `f_buff` if * it is not `nullptr` (version 3 or after). */ - Status load_v3_or_higher( + void load_v3_or_higher( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, uint64_t offset, @@ -1758,7 +1739,7 @@ class FragmentMetadata { * the footer will be loaded from the file, otherwise it * will be loaded from `f_buff`. */ - Status load_footer( + void load_footer( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, uint64_t offset, @@ -1799,12 +1780,11 @@ class FragmentMetadata { * * @param encryption_key The encryption key. * @param nbytes The total number of bytes written for the R-tree. - * @return Status */ - Status store_rtree(const EncryptionKey& encryption_key, uint64_t* nbytes); + void store_rtree(const EncryptionKey& encryption_key, uint64_t* nbytes); /** Stores a footer with the basic information. */ - Status store_footer(const EncryptionKey& encryption_key); + void store_footer(const EncryptionKey& encryption_key); /** Writes the R-tree to a tile. */ WriterTile write_rtree(); @@ -1818,7 +1798,6 @@ class FragmentMetadata { * @param idx The index of the attribute or dimension. * @param encryption_key The encryption key. * @param nbytes The total number of bytes written for the tile offsets. - * @return Status */ void store_tile_offsets( unsigned idx, const EncryptionKey& encryption_key, uint64_t* nbytes); @@ -1836,7 +1815,6 @@ class FragmentMetadata { * @param idx The index of the attribute or dimension. * @param encryption_key The encryption key. * @param nbytes The total number of bytes written for the tile var offsets. - * @return Status */ void store_tile_var_offsets( unsigned idx, const EncryptionKey& encryption_key, uint64_t* nbytes); @@ -1854,7 +1832,6 @@ class FragmentMetadata { * @param idx The index of the attribute or dimension. * @param encryption_key The encryption key. * @param nbytes The total number of bytes written for the tile var sizes. - * @return Status */ void store_tile_var_sizes( unsigned idx, const EncryptionKey& encryption_key, uint64_t* nbytes); @@ -1872,7 +1849,6 @@ class FragmentMetadata { * @param encryption_key The encryption key. * @param nbytes The total number of bytes written for the validity tile * offsets. - * @return Status */ void store_tile_validity_offsets( unsigned idx, const EncryptionKey& encryption_key, uint64_t* nbytes); @@ -1999,14 +1975,14 @@ class FragmentMetadata { * Reads the contents of a generic tile starting at the input offset, * and returns a tile. */ - tuple> read_generic_tile_from_file( + Tile read_generic_tile_from_file( const EncryptionKey& encryption_key, uint64_t offset) const; /** * Reads the fragment metadata file footer (which contains the generic tile * offsets) into the input buffer. */ - Status read_file_footer( + void read_file_footer( std::shared_ptr& tile, uint64_t* footer_offset, uint64_t* footer_size) const; @@ -2018,9 +1994,8 @@ class FragmentMetadata { * @param encryption_key The encryption key. * @param tile The tile whose contents the function will write. * @param nbytes The total number of bytes written to the file. - * @return Status */ - Status write_generic_tile_to_file( + void write_generic_tile_to_file( const EncryptionKey& encryption_key, WriterTile& tile, uint64_t* nbytes) const; @@ -2031,7 +2006,7 @@ class FragmentMetadata { * retrieval upon reading (as its size is predictable based on the * number of attributes). */ - Status write_footer_to_file(WriterTile&) const; + void write_footer_to_file(WriterTile&) const; /** * Simple clean up function called in the case of error. It removes the @@ -2044,10 +2019,9 @@ class FragmentMetadata { * motiviation is to encode illegal/reserved file name characters. * * @param name The dimension/attribute name. - * return Status, the encoded dimension/attribute name. + * @return the encoded dimension/attribute name. */ - tuple> encode_name( - const std::string& name) const; + std::string encode_name(const std::string& name) const; /** * This builds the index mapping for attribute/dimension name to id. diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 952d83acb65a..3b75f7bb1d26 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -701,6 +701,9 @@ const format_version_t deletes_min_version = 16; /** The lowest version supported for updates. */ const format_version_t updates_min_version = 16; +/** The lowest version supported for tile min/max/sum/null count data. */ +const format_version_t tile_metadata_min_version = 11; + /** The lowest version supported format version for enumerations. */ const format_version_t enumerations_min_format_version = 20; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 7afdea1e3a04..9555af422dbd 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -690,6 +690,9 @@ extern const format_version_t deletes_min_version; /** The lowest version supported for updates. */ extern const format_version_t updates_min_version; +/** The lowest version supported for tile min/max/sum/null count data. */ +extern const format_version_t tile_metadata_min_version; + /** The lowest version supported for enumerations. */ extern const format_version_t enumerations_min_format_version; diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index b612068d87a4..9cbafc695951 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -336,7 +336,7 @@ Status Reader::load_initial_data() { // Load processed conditions from fragment metadata. if (delete_and_update_conditions_.size() > 0) { - throw_if_not_ok(load_processed_conditions()); + load_processed_conditions(); } initial_data_loaded_ = true; @@ -1403,12 +1403,11 @@ Status Reader::process_tiles( // Pre-load all attribute offsets into memory for attributes // to be read. - RETURN_NOT_OK(load_tile_offsets(subarray.relevant_fragments(), read_names)); + load_tile_offsets(subarray.relevant_fragments(), read_names); // Pre-load all var attribute var tile sizes into memory for attributes // to be read. - RETURN_NOT_OK( - load_tile_var_sizes(subarray.relevant_fragments(), var_size_read_names)); + load_tile_var_sizes(subarray.relevant_fragments(), var_size_read_names); // Get the maximum number of attributes to read and unfilter in parallel. // Each attribute requires additional memory to buffer reads into @@ -1628,9 +1627,9 @@ Status Reader::compute_result_coords( // ignore fragments with a version >= 5. auto& subarray = read_state_.partitioner_.current(); std::vector zipped_coords_names = {constants::coords}; - RETURN_CANCEL_OR_ERROR(load_tile_offsets( + load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), - zipped_coords_names)); + zipped_coords_names); // Preload unzipped coordinate tile offsets. Note that this will // ignore fragments with a version < 5. @@ -1644,11 +1643,11 @@ Status Reader::compute_result_coords( if (array_schema_.var_size(name)) var_size_dim_names.emplace_back(name); } - RETURN_CANCEL_OR_ERROR(load_tile_offsets( - read_state_.partitioner_.subarray().relevant_fragments(), dim_names)); - RETURN_CANCEL_OR_ERROR(load_tile_var_sizes( + load_tile_offsets( + read_state_.partitioner_.subarray().relevant_fragments(), dim_names); + load_tile_var_sizes( read_state_.partitioner_.subarray().relevant_fragments(), - var_size_dim_names)); + var_size_dim_names); // Read and unfilter zipped coordinate tiles. Note that // this will ignore fragments with a version >= 5. @@ -1663,8 +1662,8 @@ Status Reader::compute_result_coords( // Read and unfilter timestamps, if required. if (use_timestamps_) { std::vector timestamps = {constants::timestamps}; - RETURN_CANCEL_OR_ERROR(load_tile_offsets( - read_state_.partitioner_.subarray().relevant_fragments(), timestamps)); + load_tile_offsets( + read_state_.partitioner_.subarray().relevant_fragments(), timestamps); RETURN_CANCEL_OR_ERROR( read_and_unfilter_attribute_tiles(timestamps, tmp_result_tiles)); } @@ -1672,9 +1671,9 @@ Status Reader::compute_result_coords( // Read and unfilter delete timestamps. { std::vector delete_timestamps = {constants::delete_timestamps}; - RETURN_CANCEL_OR_ERROR(load_tile_offsets( + load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), - delete_timestamps)); + delete_timestamps); RETURN_CANCEL_OR_ERROR( read_and_unfilter_attribute_tiles(delete_timestamps, tmp_result_tiles)); } diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index bffc573e7682..065ee571ea61 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -346,10 +346,10 @@ Status DenseReader::dense_read() { // Pre-load all attribute offsets into memory for attributes // in query condition to be read. - RETURN_CANCEL_OR_ERROR(load_tile_var_sizes( - read_state_.partitioner_.subarray().relevant_fragments(), var_names)); - RETURN_CANCEL_OR_ERROR(load_tile_offsets( - read_state_.partitioner_.subarray().relevant_fragments(), names)); + load_tile_var_sizes( + read_state_.partitioner_.subarray().relevant_fragments(), var_names); + load_tile_offsets( + read_state_.partitioner_.subarray().relevant_fragments(), names); uint64_t t_start = 0; uint64_t t_end = 0; diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index 2a899f9e0d79..89dbb39cf4a2 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -439,21 +439,15 @@ class FilteredData { inline URI file_uri(const FragmentMetadata* fragment, const TileType type) { switch (type) { case TileType::FIXED: { - auto&& [status, uri]{fragment->uri(name_)}; - throw_if_not_ok(status); - return std::move(*uri); + return fragment->uri(name_); } case TileType::VAR: { - auto&& [status, uri]{fragment->var_uri(name_)}; - throw_if_not_ok(status); - return std::move(*uri); + return fragment->var_uri(name_); } case TileType::NULLABLE: { - auto&& [status, uri]{fragment->validity_uri(name_)}; - throw_if_not_ok(status); - return std::move(*uri); + return fragment->validity_uri(name_); } default: diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index cf288aa486f8..fc45cf0c474d 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -185,8 +185,8 @@ Status OrderedDimLabelReader::dowork() { // `tile_var_offsets_`, `tile_validity_offsets_` and `tile_var_sizes_` in // `fragment_metadata_`. std::vector names = {label_name_}; - RETURN_NOT_OK(load_tile_offsets(subarray_.relevant_fragments(), names)); - RETURN_NOT_OK(load_tile_var_sizes(subarray_.relevant_fragments(), names)); + load_tile_offsets(subarray_.relevant_fragments(), names); + load_tile_var_sizes(subarray_.relevant_fragments(), names); // Load the dimension labels min/max values. load_label_min_max_values(); @@ -347,12 +347,9 @@ void OrderedDimLabelReader::load_label_min_max_values() { fragment_metadata_.size(), [&](const uint64_t i) { auto& fragment = fragment_metadata_[i]; - std::vector names_min = {label_name_}; - RETURN_NOT_OK(fragment->load_tile_min_values( - *encryption_key, std::move(names_min))); - std::vector names_max = {label_name_}; - RETURN_NOT_OK(fragment->load_tile_max_values( - *encryption_key, std::move(names_max))); + std::vector names = {label_name_}; + fragment->load_tile_min_values(*encryption_key, names); + fragment->load_tile_max_values(*encryption_key, names); return Status::Ok(); })); } diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 22a0f4791f48..c4d8ddf573db 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -441,13 +441,13 @@ bool ReaderBase::include_timestamps(const unsigned f) const { !dups || timestamps_needed); } -Status ReaderBase::load_tile_offsets( +void ReaderBase::load_tile_offsets( const RelevantFragments& relevant_fragments, const std::vector& names) { auto timer_se = stats_->start_timer("load_tile_offsets"); const auto encryption_key = array_->encryption_key(); - const auto status = parallel_for( + throw_if_not_ok(parallel_for( storage_manager_->compute_tp(), 0, relevant_fragments.size(), @@ -466,23 +466,18 @@ Status ReaderBase::load_tile_offsets( filtered_names.emplace_back(name); } - RETURN_NOT_OK(fragment->load_tile_offsets( - *encryption_key, std::move(filtered_names))); + fragment->load_tile_offsets(*encryption_key, filtered_names); return Status::Ok(); - }); - - RETURN_NOT_OK(status); - - return Status::Ok(); + })); } -Status ReaderBase::load_tile_var_sizes( +void ReaderBase::load_tile_var_sizes( const RelevantFragments& relevant_fragments, const std::vector& names) { auto timer_se = stats_->start_timer("load_tile_var_sizes"); const auto encryption_key = array_->encryption_key(); - const auto status = parallel_for( + throw_if_not_ok(parallel_for( storage_manager_->compute_tp(), 0, relevant_fragments.size(), @@ -503,23 +498,19 @@ Status ReaderBase::load_tile_var_sizes( continue; } - throw_if_not_ok(fragment->load_tile_var_sizes(*encryption_key, name)); + fragment->load_tile_var_sizes(*encryption_key, name); } return Status::Ok(); - }); - - RETURN_NOT_OK(status); - - return Status::Ok(); + })); } -Status ReaderBase::load_processed_conditions() { +void ReaderBase::load_processed_conditions() { auto timer_se = stats_->start_timer("load_processed_conditions"); const auto encryption_key = array_->encryption_key(); // Load all fragments in parallel. - const auto status = parallel_for( + throw_if_not_ok(parallel_for( storage_manager_->compute_tp(), 0, fragment_metadata_.size(), @@ -527,15 +518,11 @@ Status ReaderBase::load_processed_conditions() { auto& fragment = fragment_metadata_[i]; if (fragment->has_delete_meta()) { - RETURN_NOT_OK(fragment->load_processed_conditions(*encryption_key)); + fragment->load_processed_conditions(*encryption_key); } return Status::Ok(); - }); - - RETURN_NOT_OK(status); - - return Status::Ok(); + })); } Status ReaderBase::read_and_unfilter_attribute_tiles( diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index bdb7d5f9be4a..ee5a32b60ffc 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -395,7 +395,7 @@ class ReaderBase : public StrategyBase { * @param names The attribute/dimension names. * @return Status */ - Status load_tile_offsets( + void load_tile_offsets( const RelevantFragments& relevant_fragments, const std::vector& names); @@ -426,9 +426,8 @@ class ReaderBase : public StrategyBase { * * @param relevant_fragments List of relevant fragments. * @param names The attribute/dimension names. - * @return Status */ - Status load_tile_var_sizes( + void load_tile_var_sizes( const RelevantFragments& relevant_fragments, const std::vector& names); @@ -438,7 +437,7 @@ class ReaderBase : public StrategyBase { * @param subarray The subarray to load processed conditions for. * @return Status */ - Status load_processed_conditions(); + void load_processed_conditions(); /** * Read and unfilter attribute tiles. diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 1fb6a489ab25..73ee17740f17 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -358,7 +358,7 @@ Status SparseIndexReaderBase::load_initial_data() { // Load processed conditions from fragment metadata. if (delete_and_update_conditions_.size() > 0) { - throw_if_not_ok(load_processed_conditions()); + load_processed_conditions(); } // Make a list of dim/attr that will be loaded for query condition. @@ -499,16 +499,15 @@ void SparseIndexReaderBase::load_tile_offsets_for_fragments( // Preload zipped coordinate tile offsets. Note that this will // ignore fragments with a version >= 5. std::vector zipped_coords_names = {constants::coords}; - throw_if_not_ok(load_tile_offsets(relevant_fragments, zipped_coords_names)); + load_tile_offsets(relevant_fragments, zipped_coords_names); // Preload unzipped coordinate tile offsets. Note that this will // ignore fragments with a version < 5. - throw_if_not_ok(load_tile_offsets(relevant_fragments, dim_names_)); + load_tile_offsets(relevant_fragments, dim_names_); // Load tile offsets and var sizes for attributes. - throw_if_not_ok(load_tile_var_sizes(relevant_fragments, var_size_to_load_)); - throw_if_not_ok( - load_tile_offsets(relevant_fragments, attr_tile_offsets_to_load_)); + load_tile_var_sizes(relevant_fragments, var_size_to_load_); + load_tile_offsets(relevant_fragments, attr_tile_offsets_to_load_); } Status SparseIndexReaderBase::read_and_unfilter_coords( diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index d389075cf6ad..0a9428e1e816 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -251,36 +251,31 @@ GlobalOrderWriter::multipart_upload_state(bool client) { // TODO: to be refactored, there are multiple places in writers where // we iterate over the internal fragment files manually for (const auto& name : buffer_names()) { - auto&& [st1, uri] = meta->uri(name); - RETURN_NOT_OK_TUPLE(st1, {}); + auto uri = meta->uri(name); - auto&& [st2, state] = storage_manager_->vfs()->multipart_upload_state(*uri); + auto&& [st2, state] = storage_manager_->vfs()->multipart_upload_state(uri); RETURN_NOT_OK_TUPLE(st2, {}); // If there is no entry for this uri, probably multipart upload is disabled // or no write was issued so far if (!state.has_value()) { return {Status::Ok(), {}}; } - result[uri->remove_trailing_slash().last_path_part()] = std::move(*state); + result[uri.remove_trailing_slash().last_path_part()] = std::move(*state); if (array_schema_.var_size(name)) { - auto&& [status, var_uri] = meta->var_uri(name); - RETURN_NOT_OK_TUPLE(status, {}); - + auto var_uri = meta->var_uri(name); auto&& [st, var_state] = - storage_manager_->vfs()->multipart_upload_state(*var_uri); + storage_manager_->vfs()->multipart_upload_state(var_uri); RETURN_NOT_OK_TUPLE(st, {}); - result[var_uri->remove_trailing_slash().last_path_part()] = + result[var_uri.remove_trailing_slash().last_path_part()] = std::move(*var_state); } if (array_schema_.is_nullable(name)) { - auto&& [status, validity_uri] = meta->validity_uri(name); - RETURN_NOT_OK_TUPLE(status, {}); - + auto validity_uri = meta->validity_uri(name); auto&& [st, val_state] = - storage_manager_->vfs()->multipart_upload_state(*validity_uri); + storage_manager_->vfs()->multipart_upload_state(validity_uri); RETURN_NOT_OK_TUPLE(st, {}); - result[validity_uri->remove_trailing_slash().last_path_part()] = + result[validity_uri.remove_trailing_slash().last_path_part()] = std::move(*val_state); } } @@ -813,7 +808,7 @@ Status GlobalOrderWriter::global_write() { // Set new number of tiles in the fragment metadata auto new_num_tiles = frag_meta->tile_index_base() + num; - throw_if_not_ok(frag_meta->set_num_tiles(new_num_tiles)); + frag_meta->set_num_tiles(new_num_tiles); if (new_num_tiles == 0) { throw GlobalOrderWriterStatusException( @@ -851,7 +846,7 @@ Status GlobalOrderWriter::global_write_handle_last_tile() { // Reserve space for the last tile in the fragment metadata auto meta = global_write_state_->frag_meta_; - throw_if_not_ok(meta->set_num_tiles(meta->tile_index_base() + 1)); + meta->set_num_tiles(meta->tile_index_base() + 1); // Filter last tiles RETURN_CANCEL_OR_ERROR(filter_last_tiles(cell_num_last_tiles)); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index b4fd70e85916..b87a3fd3e5b4 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -205,7 +205,7 @@ Status OrderedWriter::ordered_write() { auto tile_num = dense_tiler.tile_num(); // Set number of tiles in the fragment metadata - throw_if_not_ok(frag_meta->set_num_tiles(tile_num)); + frag_meta->set_num_tiles(tile_num); // Prepare, filter and write tiles for all attributes auto attr_num = buffers_.size(); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 2875df2b37bb..a02b30223e43 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -695,7 +695,7 @@ Status UnorderedWriter::unordered_write() { auto tile_num = it->second.size(); if (is_coords_pass_) { // Set the number of tiles in the metadata - throw_if_not_ok(frag_meta_->set_num_tiles(tile_num)); + frag_meta_->set_num_tiles(tile_num); stats_->add_counter("tile_num", tile_num); stats_->add_counter("cell_num", cell_pos_.size()); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 7aa0640c4d6a..3bd52a1282f3 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -607,21 +607,12 @@ Status WriterBase::close_files(shared_ptr meta) const { file_uris.reserve(buffer_name.size() * 3); for (const auto& name : buffer_name) { - auto&& [status, uri] = meta->uri(name); - RETURN_NOT_OK(status); - - file_uris.emplace_back(*uri); + file_uris.emplace_back(meta->uri(name)); if (array_schema_.var_size(name)) { - auto&& [status, var_uri] = meta->var_uri(name); - RETURN_NOT_OK(status); - - file_uris.emplace_back(*var_uri); + file_uris.emplace_back(meta->var_uri(name)); } if (array_schema_.is_nullable(name)) { - auto&& [status, validity_uri] = meta->validity_uri(name); - RETURN_NOT_OK(status); - - file_uris.emplace_back(*validity_uri); + file_uris.emplace_back(meta->validity_uri(name)); } } @@ -706,7 +697,7 @@ void WriterBase::set_coords_metadata( start_tile_idx, end_tile_idx, [&](uint64_t i) { - throw_if_not_ok(meta->set_mbr(i - start_tile_idx, mbrs[i])); + meta->set_mbr(i - start_tile_idx, mbrs[i]); return Status::Ok(); }); throw_if_not_ok(status); @@ -819,7 +810,7 @@ Status WriterBase::create_fragment( has_timestamps, has_delete_metadata); - RETURN_NOT_OK((frag_meta)->init(subarray_.ndrange(0))); + frag_meta->init(subarray_.ndrange(0)); return Status::Ok(); } @@ -1117,25 +1108,10 @@ Status WriterBase::write_tiles( // For easy reference const bool var_size = array_schema_.var_size(name); const bool nullable = array_schema_.is_nullable(name); - auto&& [status, uri] = frag_meta->uri(name); - RETURN_NOT_OK(status); + auto uri = frag_meta->uri(name); - Status st; - optional var_uri; - if (!var_size) - var_uri = URI(""); - else { - tie(st, var_uri) = frag_meta->var_uri(name); - RETURN_NOT_OK(st); - } - - optional validity_uri; - if (!nullable) - validity_uri = URI(""); - else { - tie(st, validity_uri) = frag_meta->validity_uri(name); - RETURN_NOT_OK(st); - } + URI var_uri = var_size ? frag_meta->var_uri(name) : URI(""); + URI validity_uri = nullable ? frag_meta->validity_uri(name) : URI(""); // Compute and set var buffer sizes for the min/max metadata const auto has_min_max_md = has_min_max_metadata(name, var_size); @@ -1150,7 +1126,7 @@ Status WriterBase::write_tiles( auto& tile = (*tiles)[i]; auto& t = var_size ? tile.offset_tile() : tile.fixed_tile(); RETURN_NOT_OK(storage_manager_->vfs()->write( - *uri, + uri, t.filtered_buffer().data(), t.filtered_buffer().size(), remote_global_order_write)); @@ -1160,7 +1136,7 @@ Status WriterBase::write_tiles( if (var_size) { auto& t_var = tile.var_tile(); RETURN_NOT_OK(storage_manager_->vfs()->write( - *var_uri, + var_uri, t_var.filtered_buffer().data(), t_var.filtered_buffer().size(), remote_global_order_write)); @@ -1185,7 +1161,7 @@ Status WriterBase::write_tiles( if (nullable) { auto& t_val = tile.validity_tile(); RETURN_NOT_OK(storage_manager_->vfs()->write( - *validity_uri, + validity_uri, t_val.filtered_buffer().data(), t_val.filtered_buffer().size(), remote_global_order_write)); @@ -1199,19 +1175,13 @@ Status WriterBase::write_tiles( // writes if (close_files) { std::vector closing_uris; - auto&& [st1, uri] = frag_meta->uri(name); - RETURN_NOT_OK(st1); - closing_uris.push_back(*uri); + closing_uris.push_back(frag_meta->uri(name)); if (var_size) { - auto&& [st2, var_uri] = frag_meta->var_uri(name); - RETURN_NOT_OK(st2); - closing_uris.push_back(*var_uri); + closing_uris.push_back(frag_meta->var_uri(name)); } if (nullable) { - auto&& [st2, validity_uri] = frag_meta->validity_uri(name); - RETURN_NOT_OK(st2); - closing_uris.push_back(*validity_uri); + closing_uris.push_back(frag_meta->validity_uri(name)); } for (auto& u : closing_uris) { if (layout_ == Layout::GLOBAL_ORDER) { diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index 7c1763ea9e81..0c66efd20dae 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -390,7 +390,7 @@ Status fragment_metadata_from_capnp( // Whilst sparse gets its domain calculated, dense needs to have it // set here from the deserialized data if (array_schema->dense()) { - throw_if_not_ok(frag_meta->init_domain(*ndrange)); + frag_meta->init_domain(*ndrange); } else { const auto& frag0_dom = *ndrange; frag_meta->non_empty_domain().assign(frag0_dom.begin(), frag0_dom.end()); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 6b5cfb32a34b..ca1fa41e8b27 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -2011,8 +2011,8 @@ StorageManagerCanonical::load_fragment_metadata( } // Load fragment metadata - RETURN_NOT_OK(metadata->load( - encryption_key, fragment_metadata_tile, offset, array_schemas_all)); + metadata->load( + encryption_key, fragment_metadata_tile, offset, array_schemas_all); fragment_metadata[f] = metadata; return Status::Ok(); diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 275615ddf905..46ca4abe07dd 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -2952,7 +2952,8 @@ Status Subarray::load_relevant_fragment_rtrees( auto status = parallel_for(compute_tp, 0, relevant_fragments_.size(), [&](uint64_t f) { - return meta[relevant_fragments_[f]]->load_rtree(*encryption_key); + meta[relevant_fragments_[f]]->load_rtree(*encryption_key); + return Status::Ok(); }); RETURN_NOT_OK(status); @@ -3005,8 +3006,8 @@ Status Subarray::compute_relevant_fragment_tile_overlap( compute_tile_overlap(r + tile_overlap->range_idx_start(), frag_idx); } else { // Sparse fragment const auto& range = this->ndrange(r + tile_overlap->range_idx_start()); - RETURN_NOT_OK(meta->get_tile_overlap( - range, is_default_, tile_overlap->at(frag_idx, r))); + meta->get_tile_overlap( + range, is_default_, tile_overlap->at(frag_idx, r)); } } @@ -3043,10 +3044,12 @@ Status Subarray::load_relevant_fragment_tile_var_sizes( // Gracefully skip loading tile sizes for attributes added in schema // evolution that do not exists in this fragment const auto& schema = meta[f]->array_schema(); - if (!schema->is_field(var_name)) + if (!schema->is_field(var_name)) { return Status::Ok(); + } - return meta[f]->load_tile_var_sizes(*encryption_key, var_name); + meta[f]->load_tile_var_sizes(*encryption_key, var_name); + return Status::Ok(); }); RETURN_NOT_OK(status); diff --git a/tools/src/commands/info_command.cc b/tools/src/commands/info_command.cc index ba4e040a2b3b..eeb028e904eb 100644 --- a/tools/src/commands/info_command.cc +++ b/tools/src/commands/info_command.cc @@ -142,8 +142,8 @@ void InfoCommand::print_tile_sizes() const { uint64_t tile_num = f->tile_num(); std::vector names; names.push_back(name); - THROW_NOT_OK(f->load_tile_offsets(enc_key, std::move(names))); - THROW_NOT_OK(f->load_tile_var_sizes(enc_key, name)); + f->load_tile_offsets(enc_key, names); + f->load_tile_var_sizes(enc_key, name); for (uint64_t tile_idx = 0; tile_idx < tile_num; tile_idx++) { persisted_tile_size += f->persisted_tile_size(name, tile_idx); in_memory_tile_size += f->tile_size(name, tile_idx); @@ -298,7 +298,7 @@ void InfoCommand::write_text_mbrs() const { auto fragment_metadata = array.fragment_metadata(); std::stringstream text; for (const auto& f : fragment_metadata) { - THROW_NOT_OK(f->load_rtree(*encryption_key)); + f->load_rtree(*encryption_key); const auto& mbrs = f->mbrs(); for (const auto& mbr : mbrs) { auto str_mbr = mbr_to_string(mbr, schema.domain()); From 768c5ac3fadfe6d298433a452c8aaee34705c50f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 26 Oct 2023 15:15:11 +0300 Subject: [PATCH 032/456] Fix docs of `tiledb_array_schema_evolution_extend_enumeration`. (#4464) There is not an `enmr` variable in the example. --- TYPE: NO_HISTORY --- tiledb/sm/c_api/tiledb_experimental.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index a814a34043e3..645c65dd13ab 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -233,7 +233,7 @@ TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_add_enumeration( * tiledb_array_schema_evolution_extend_enumeration( * ctx, * array_schema_evolution, - * enmr); + * new_enmr); * @endcode * * @param ctx The TileDB context. From c44b321a7e095d295a75ead676e6b31b2d1e0886 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 26 Oct 2023 18:25:22 +0300 Subject: [PATCH 033/456] Cache searching for the toolchain file and checking for AVX2. (#4449) Saves a couple seconds when repeatedly configuring the CMake project --- TYPE: BUILD DESC: Cache more information when configuring the CMake project. --- cmake/Modules/CheckAVX2Support.cmake | 46 +++++++++------------------- cmake/Options/TileDBToolchain.cmake | 20 ++++++------ 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/cmake/Modules/CheckAVX2Support.cmake b/cmake/Modules/CheckAVX2Support.cmake index 5fc7c9401bbf..b91fcf7acf44 100644 --- a/cmake/Modules/CheckAVX2Support.cmake +++ b/cmake/Modules/CheckAVX2Support.cmake @@ -30,26 +30,6 @@ include(CheckCXXSourceRuns) include(CMakePushCheckState) -# -# Tries to build and run an AVX2 program with the given compiler flag. -# If successful, sets cache variable HAVE_AVX2 to 1. -# -function (CheckAVX2Flag FLAG) - cmake_push_check_state() - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") - unset(HAVE_AVX2 CACHE) - check_cxx_source_runs(" - #include - int main() { - __m256i packed = _mm256_set_epi32(-1, -2, -3, -4, -5, -6, -7, -8); - __m256i absolute_values = _mm256_abs_epi32(packed); - return 0; - }" - HAVE_AVX2 - ) - cmake_pop_check_state() -endfunction() - # # Determines if AVX2 is available. # @@ -67,17 +47,21 @@ function (CheckAVX2Support) endif() if (MSVC) - CheckAVX2Flag(/arch:AVX2) - if (HAVE_AVX2) - set(COMPILER_AVX2_FLAG "/arch:AVX2" CACHE STRING "Compiler flag for AVX2 support.") - endif() + set(COMPILER_AVX2_FLAG "/arch:AVX2" CACHE STRING "Compiler flag for AVX2 support.") else() - CheckAVX2Flag(-mavx2) - if (HAVE_AVX2) - set(COMPILER_AVX2_FLAG "-mavx2" CACHE STRING "Compiler flag for AVX2 support.") - endif() + set(COMPILER_AVX2_FLAG "-mavx2" CACHE STRING "Compiler flag for AVX2 support.") endif() - set(COMPILER_SUPPORTS_AVX2 "${HAVE_AVX2}" CACHE BOOL "True if the compiler supports AVX2." FORCE) - unset(HAVE_AVX2 CACHE) -endfunction() \ No newline at end of file + cmake_push_check_state() + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_source_runs(" + #include + int main() { + __m256i packed = _mm256_set_epi32(-1, -2, -3, -4, -5, -6, -7, -8); + __m256i absolute_values = _mm256_abs_epi32(packed); + return 0; + }" + COMPILER_SUPPORTS_AVX2 + ) + cmake_pop_check_state() +endfunction() diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index c2a3bff5225e..5f09ce3b9557 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -11,15 +11,17 @@ if (NOT TILEDB_SUPERBUILD) return() endif() -if(DEFINED ENV{VCPKG_ROOT}) - set(CMAKE_TOOLCHAIN_FILE - "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" - CACHE STRING "Vcpkg toolchain file") -else() - include(init-submodule) - set(CMAKE_TOOLCHAIN_FILE - "${CMAKE_CURRENT_SOURCE_DIR}/external/vcpkg/scripts/buildsystems/vcpkg.cmake" - CACHE STRING "Vcpkg toolchain file") +if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) + if(DEFINED ENV{VCPKG_ROOT}) + set(CMAKE_TOOLCHAIN_FILE + "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" + CACHE STRING "Vcpkg toolchain file") + else() + include(init-submodule) + set(CMAKE_TOOLCHAIN_FILE + "${CMAKE_CURRENT_SOURCE_DIR}/external/vcpkg/scripts/buildsystems/vcpkg.cmake" + CACHE STRING "Vcpkg toolchain file") + endif() endif() if(APPLE AND NOT DEFINED VCPKG_TARGET_TRIPLET) From e96ea29aee65c3a3e7fd83ba61e1ff6625f177bd Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 26 Oct 2023 20:06:16 +0300 Subject: [PATCH 034/456] Decouple the code in the `fragment` directory from `StorageManager`. (#4453) [SC-23374](https://app.shortcut.com/tiledb-inc/story/23374/move-storagemanager-load-fragments-to-array) The `StorageManager` methods that load fragment info and metadata were moved to the `FragmentInfo` and `FragmentMetadata` classes. These classes also use `ContextResources` instead of `StorageManager`. I was going to make a `fragment` object library but it also depends on `rest` which is not yet componentized. --- TYPE: IMPROVEMENT DESC: Refactor the fragment info loading code to reduce coupling. --- test/src/unit-average-cell-size.cc | 2 +- tiledb/sm/array/array.cc | 66 +++--- tiledb/sm/array/array.h | 19 +- tiledb/sm/array/test/unit_consistency.h | 1 + tiledb/sm/c_api/tiledb.cc | 2 +- .../sm/consolidator/fragment_consolidator.cc | 6 +- tiledb/sm/fragment/fragment_info.cc | 155 ++++++++++--- tiledb/sm/fragment/fragment_info.h | 36 +++- tiledb/sm/fragment/fragment_metadata.cc | 191 ++++++++++------ tiledb/sm/fragment/fragment_metadata.h | 55 ++++- tiledb/sm/query/readers/filtered_data.h | 1 + .../sm/query/writers/global_order_writer.cc | 3 +- tiledb/sm/query/writers/unordered_writer.cc | 2 +- tiledb/sm/query/writers/writer_base.cc | 2 +- tiledb/sm/query_plan/test/unit_query_plan.cc | 1 + tiledb/sm/serialization/array.cc | 3 +- tiledb/sm/serialization/fragment_metadata.cc | 6 +- tiledb/sm/serialization/fragment_metadata.h | 4 +- tiledb/sm/serialization/query.cc | 2 +- tiledb/sm/serialization/query_aggregates.h | 2 + tiledb/sm/storage_manager/storage_manager.cc | 203 ------------------ .../storage_manager_canonical.h | 105 --------- tiledb/sm/subarray/subarray.cc | 1 + 23 files changed, 391 insertions(+), 477 deletions(-) diff --git a/test/src/unit-average-cell-size.cc b/test/src/unit-average-cell-size.cc index f05cd410eb7e..a6663fb1e342 100644 --- a/test/src/unit-average-cell-size.cc +++ b/test/src/unit-average-cell-size.cc @@ -212,7 +212,7 @@ struct CPPAverageCellSizeFx { ->open_without_fragments( sm::EncryptionType::NO_ENCRYPTION, nullptr, 0) .ok()); - REQUIRE(array_for_reads->load_fragments(uris).ok()); + array_for_reads->load_fragments(uris); auto avg_cell_sizes = array_for_reads->get_average_var_cell_sizes(); CHECK(avg_cell_sizes["d2"] == d2_size); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 6179e760954e..1b6f22eaa005 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -209,10 +209,8 @@ Status Array::open_without_fragments( array_dir_ = ArrayDirectory( resources_, array_uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); } - auto&& [array_schema, array_schemas] = open_for_reads_without_fragments(); - - array_schema_latest_ = array_schema.value(); - array_schemas_all_ = array_schemas.value(); + std::tie(array_schema_latest_, array_schemas_all_) = + open_for_reads_without_fragments(); } } catch (std::exception& e) { set_array_closed(); @@ -223,15 +221,20 @@ Status Array::open_without_fragments( return Status::Ok(); } -Status Array::load_fragments( +void Array::load_fragments( const std::vector& fragments_to_load) { - auto&& [st, fragment_metadata] = - storage_manager_->array_load_fragments(this, fragments_to_load); - RETURN_NOT_OK(st); + auto timer_se = resources_.stats().start_timer("sm_array_load_fragments"); - fragment_metadata_ = std::move(fragment_metadata.value()); - - return Status::Ok(); + // Load the fragment metadata + std::unordered_map> offsets; + fragment_metadata_ = FragmentMetadata::load( + resources_, + memory_tracker(), + array_schema_latest_ptr(), + array_schemas_all(), + *encryption_key(), + fragments_to_load, + offsets); } Status Array::open( @@ -380,13 +383,8 @@ Status Array::open( array_dir_ = ArrayDirectory( resources_, array_uri_, timestamp_start_, timestamp_end_opened_at_); } - auto&& [array_schema_latest, array_schemas, fragment_metadata] = + std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = open_for_reads(); - - // Set schemas - array_schema_latest_ = array_schema_latest.value(); - array_schemas_all_ = array_schemas.value(); - fragment_metadata_ = fragment_metadata.value(); } else if ( query_type == QueryType::WRITE || query_type == QueryType::MODIFY_EXCLUSIVE) { @@ -895,13 +893,9 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { return LOG_STATUS(Status_ArrayDirectoryError(le.what())); } - auto&& [array_schema_latest, array_schemas, fragment_metadata] = + std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = open_for_reads(); - array_schema_latest_ = array_schema_latest.value(); - array_schemas_all_ = array_schemas.value(); - fragment_metadata_ = fragment_metadata.value(); - return Status::Ok(); } @@ -1254,39 +1248,35 @@ std::unordered_map Array::get_average_var_cell_sizes() /* ********************************* */ tuple< - optional>, - optional>>, - optional>>> + shared_ptr, + std::unordered_map>, + std::vector>> Array::open_for_reads() { auto timer_se = resources_.stats().start_timer( "array_open_read_load_schemas_and_fragment_meta"); - auto&& [st, array_schema_latest, array_schemas_all, fragment_metadata] = - storage_manager_->load_array_schemas_and_fragment_metadata( - array_directory(), memory_tracker(), *encryption_key()); + auto result = FragmentInfo::load_array_schemas_and_fragment_metadata( + resources_, array_directory(), memory_tracker(), *encryption_key()); - throw_if_not_ok(st); - - auto version = array_schema_latest.value()->version(); + auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); - return {array_schema_latest, array_schemas_all, fragment_metadata}; + return result; } tuple< - optional>, - optional>>> + shared_ptr, + std::unordered_map>> Array::open_for_reads_without_fragments() { auto timer_se = resources_.stats().start_timer( "array_open_read_without_fragments_load_schemas"); // Load array schemas - auto&& [array_schema_latest, array_schemas_all] = - array_dir_.load_array_schemas(*encryption_key()); + auto result = array_dir_.load_array_schemas(*encryption_key()); - auto version = array_schema_latest->version(); + auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); - return {array_schema_latest, array_schemas_all}; + return result; } tuple< diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 24965a75a830..8023c5efc9e6 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -179,9 +179,8 @@ class Array { * Reload the array with the specified fragments. * * @param fragments_to_load The list of fragments to load. - * @return Status */ - Status load_fragments(const std::vector& fragments_to_load); + void load_fragments(const std::vector& fragments_to_load); /** * Opens the array for reading. @@ -716,9 +715,8 @@ class Array { * `timestamp_start` and `timestamp_end`. * * @param array The array to be opened. - * @return tuple of Status, latest ArraySchema, map of all array schemas and + * @return tuple latest ArraySchema, map of all array schemas and * vector of FragmentMetadata - * Status Ok on success, else error * ArraySchema The array schema to be retrieved after the * array is opened. * ArraySchemaMap Map of all array schemas found keyed by name @@ -726,24 +724,23 @@ class Array { * after the array is opened. */ tuple< - optional>, - optional>>, - optional>>> + shared_ptr, + std::unordered_map>, + std::vector>> open_for_reads(); /** * Opens an array for reads without fragments. * * @param array The array to be opened. - * @return tuple of Status, latest ArraySchema and map of all array schemas - * Status Ok on success, else error + * @return tuple of latest ArraySchema and map of all array schemas * ArraySchema The array schema to be retrieved after the * array is opened. * ArraySchemaMap Map of all array schemas found keyed by name */ tuple< - optional>, - optional>>> + shared_ptr, + std::unordered_map>> open_for_reads_without_fragments(); /** Opens an array for writes. diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 2b37c7292ac5..8e2aea539fe2 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -43,6 +43,7 @@ #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 10d55edc3c3c..4c0d787d58a8 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -4504,7 +4504,7 @@ int32_t tiledb_fragment_info_alloc( // Allocate a fragment info object (*fragment_info)->fragment_info_ = - new (std::nothrow) tiledb::sm::FragmentInfo(uri, ctx->storage_manager()); + new (std::nothrow) tiledb::sm::FragmentInfo(uri, ctx->resources()); if ((*fragment_info)->fragment_info_ == nullptr) { delete *fragment_info; *fragment_info = nullptr; diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 58c818c604d4..a4045621a802 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -107,7 +107,7 @@ Status FragmentConsolidator::consolidate( // must be fetched (even before `config_.timestamp_start_`), // to compute the anterior ND range that can help determine // which dense fragments are consolidatable. - FragmentInfo fragment_info(URI(array_name), storage_manager_); + FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); auto st = fragment_info.load( array_for_reads->array_directory(), config_.timestamp_start_, @@ -216,7 +216,7 @@ Status FragmentConsolidator::consolidate_fragments( } // Get all fragment info - FragmentInfo fragment_info(URI(array_name), storage_manager_); + FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); auto st = fragment_info.load( array_for_reads->array_directory(), 0, @@ -371,7 +371,7 @@ Status FragmentConsolidator::consolidate_internal( URI* new_fragment_uri) { auto timer_se = stats_->start_timer("consolidate_internal"); - RETURN_NOT_OK(array_for_reads->load_fragments(to_consolidate)); + array_for_reads->load_fragments(to_consolidate); if (array_for_reads->is_empty()) { return Status::Ok(); diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index a252fbad2238..41354d2166be 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/parse_uri.h" namespace tiledb::sm { @@ -51,15 +52,14 @@ namespace tiledb::sm { /* ****************************** */ FragmentInfo::FragmentInfo() - : storage_manager_(nullptr) + : resources_(nullptr) , unconsolidated_metadata_num_(0) { } -FragmentInfo::FragmentInfo( - const URI& array_uri, StorageManager* storage_manager) +FragmentInfo::FragmentInfo(const URI& array_uri, ContextResources& resources) : array_uri_(array_uri) - , config_(storage_manager->config()) - , storage_manager_(storage_manager) + , config_(resources.config()) + , resources_(&resources) , unconsolidated_metadata_num_(0) { } @@ -778,7 +778,7 @@ shared_ptr FragmentInfo::get_array_schema(uint32_t fid) { EncryptionKey encryption_key; return ArrayDirectory::load_array_schema_from_uri( - storage_manager_->resources(), schema_uri, encryption_key); + *resources_, schema_uri, encryption_key); } Status FragmentInfo::get_array_schema_name( @@ -825,7 +825,7 @@ Status FragmentInfo::load() { RETURN_NOT_OK(set_default_timestamp_range()); if (array_uri_.is_tiledb()) { - auto rest_client = storage_manager_->rest_client(); + auto rest_client = resources_->rest_client(); if (rest_client == nullptr) { return LOG_STATUS(Status_ArrayError( "Cannot load fragment info; remote array with no REST client.")); @@ -841,10 +841,7 @@ Status FragmentInfo::load() { // Create an ArrayDirectory object and load ArrayDirectory array_dir( - storage_manager_->resources(), - array_uri_, - timestamp_start_, - timestamp_end_); + *resources_, array_uri_, timestamp_start_, timestamp_end_); return load(array_dir); } @@ -858,10 +855,7 @@ Status FragmentInfo::load( // Create an ArrayDirectory object and load ArrayDirectory array_dir( - storage_manager_->resources(), - array_uri_, - timestamp_start_, - timestamp_end_); + *resources_, array_uri_, timestamp_start_, timestamp_end_); return load(array_dir); } @@ -889,25 +883,22 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { } // Get the array schemas and fragment metadata. - auto&& [st_schemas, array_schema_latest, array_schemas_all, fragment_metadata] = - storage_manager_->load_array_schemas_and_fragment_metadata( - array_dir, nullptr, enc_key_); - RETURN_NOT_OK(st_schemas); - const auto& fragment_metadata_value = fragment_metadata.value(); - array_schema_latest_ = array_schema_latest.value(); - array_schemas_all_ = std::move(array_schemas_all.value()); - auto fragment_num = (uint32_t)fragment_metadata_value.size(); + std::vector> fragment_metadata; + std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata) = + load_array_schemas_and_fragment_metadata( + *resources_, array_dir, nullptr, enc_key_); + auto fragment_num = (uint32_t)fragment_metadata.size(); // Get fragment sizes std::vector sizes(fragment_num, 0); RETURN_NOT_OK(parallel_for( - storage_manager_->compute_tp(), + &resources_->compute_tp(), 0, fragment_num, - [this, &fragment_metadata_value, &sizes, preload_rtrees](uint64_t i) { + [this, &fragment_metadata, &sizes, preload_rtrees](uint64_t i) { // Get fragment size. Applicable only to relevant fragments, including // fragments that are in the range [timestamp_start_, timestamp_end_]. - auto meta = fragment_metadata_value[i]; + auto meta = fragment_metadata[i]; if (meta->timestamp_range().first >= timestamp_start_ && meta->timestamp_range().second <= timestamp_end_) { sizes[i] = meta->fragment_size(); @@ -926,7 +917,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { // Create the vector that will store the SingleFragmentInfo objects for (uint64_t fid = 0; fid < fragment_num; fid++) { - const auto meta = fragment_metadata_value[fid]; + const auto meta = fragment_metadata[fid]; const auto& array_schema = meta->array_schema(); const auto& non_empty_domain = meta->non_empty_domain(); @@ -987,6 +978,106 @@ Status FragmentInfo::load_and_replace( return Status::Ok(); } +tuple>> +load_consolidated_fragment_meta( + ContextResources& resources, const URI& uri, const EncryptionKey& enc_key) { + auto timer_se = + resources.stats().start_timer("sm_read_load_consolidated_frag_meta"); + + // No consolidated fragment metadata file + if (uri.to_string().empty()) + throw StatusException(Status_FragmentInfoError( + "Cannot load consolidated fragment metadata; URI is empty.")); + + auto&& tile = GenericTileIO::load(resources, uri, 0, enc_key); + + resources.stats().add_counter("consolidated_frag_meta_size", tile.size()); + + uint32_t fragment_num; + Deserializer deserializer(tile.data(), tile.size()); + fragment_num = deserializer.read(); + + uint64_t name_size, offset; + std::string name; + std::vector> ret; + ret.reserve(fragment_num); + for (uint32_t f = 0; f < fragment_num; ++f) { + name_size = deserializer.read(); + name.resize(name_size); + deserializer.read(&name[0], name_size); + offset = deserializer.read(); + ret.emplace_back(name, offset); + } + + return {std::move(tile), std::move(ret)}; +} + +std::tuple< + shared_ptr, + std::unordered_map>, + std::vector>> +FragmentInfo::load_array_schemas_and_fragment_metadata( + ContextResources& resources, + const ArrayDirectory& array_dir, + MemoryTracker* memory_tracker, + const EncryptionKey& enc_key) { + auto timer_se = resources.stats().start_timer( + "sm_load_array_schemas_and_fragment_metadata"); + + // Load array schemas + std::shared_ptr array_schema_latest; + std::unordered_map> + array_schemas_all; + std::tie(array_schema_latest, array_schemas_all) = + array_dir.load_array_schemas(enc_key); + + const auto filtered_fragment_uris = [&]() { + auto timer_se = + resources.stats().start_timer("sm_load_filtered_fragment_uris"); + return array_dir.filtered_fragment_uris(array_schema_latest->dense()); + }(); + const auto& meta_uris = array_dir.fragment_meta_uris(); + const auto& fragments_to_load = filtered_fragment_uris.fragment_uris(); + + // Get the consolidated fragment metadatas + std::vector> fragment_metadata_tiles(meta_uris.size()); + std::vector>> offsets_vectors( + meta_uris.size()); + throw_if_not_ok( + parallel_for(&resources.compute_tp(), 0, meta_uris.size(), [&](size_t i) { + auto&& [tile_opt, offsets] = + load_consolidated_fragment_meta(resources, meta_uris[i], enc_key); + fragment_metadata_tiles[i] = + make_shared(HERE(), std::move(tile_opt)); + offsets_vectors[i] = std::move(offsets); + return Status::Ok(); + })); + + // Get the unique fragment metadatas into a map. + std::unordered_map> offsets; + for (uint64_t i = 0; i < offsets_vectors.size(); i++) { + for (auto& offset : offsets_vectors[i]) { + if (offsets.count(offset.first) == 0) { + offsets.emplace( + offset.first, + std::make_pair(fragment_metadata_tiles[i].get(), offset.second)); + } + } + } + + // Load the fragment metadata + auto&& fragment_metadata = FragmentMetadata::load( + resources, + memory_tracker, + array_schema_latest, + array_schemas_all, + enc_key, + fragments_to_load, + offsets); + + return {array_schema_latest, array_schemas_all, fragment_metadata}; +} + const std::vector& FragmentInfo::single_fragment_info_vec() const { return single_fragment_info_vec_; @@ -1032,7 +1123,7 @@ Status FragmentInfo::set_default_timestamp_range() { tuple> FragmentInfo::load( const URI& new_fragment_uri) const { SingleFragmentInfo ret; - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_->vfs(); const auto& array_schema_latest = single_fragment_info_vec_.back().meta()->array_schema(); @@ -1049,7 +1140,7 @@ tuple> FragmentInfo::load( if (fragment_version <= 2) { URI coords_uri = new_fragment_uri.join_path(constants::coords + constants::file_suffix); - RETURN_NOT_OK_TUPLE(vfs->is_file(coords_uri, &sparse), nullopt); + RETURN_NOT_OK_TUPLE(vfs.is_file(coords_uri, &sparse), nullopt); } else { // Do nothing. It does not matter what the `sparse` value // is, since the FragmentMetadata object will load the correct @@ -1061,7 +1152,7 @@ tuple> FragmentInfo::load( // Get fragment non-empty domain auto meta = make_shared( HERE(), - storage_manager_, + resources_, nullptr, array_schema_latest, new_fragment_uri, @@ -1137,7 +1228,7 @@ FragmentInfo FragmentInfo::clone() const { clone.array_schemas_all_ = array_schemas_all_; clone.config_ = config_; clone.single_fragment_info_vec_ = single_fragment_info_vec_; - clone.storage_manager_ = storage_manager_; + clone.resources_ = resources_; clone.to_vacuum_ = to_vacuum_; clone.unconsolidated_metadata_num_ = unconsolidated_metadata_num_; clone.anterior_ndrange_ = anterior_ndrange_; @@ -1153,7 +1244,7 @@ void FragmentInfo::swap(FragmentInfo& fragment_info) { std::swap(array_schemas_all_, fragment_info.array_schemas_all_); std::swap(config_, fragment_info.config_); std::swap(single_fragment_info_vec_, fragment_info.single_fragment_info_vec_); - std::swap(storage_manager_, fragment_info.storage_manager_); + std::swap(resources_, fragment_info.resources_); std::swap(to_vacuum_, fragment_info.to_vacuum_); std::swap( unconsolidated_metadata_num_, fragment_info.unconsolidated_metadata_num_); diff --git a/tiledb/sm/fragment/fragment_info.h b/tiledb/sm/fragment/fragment_info.h index 1539af3834f8..e488af94c8a2 100644 --- a/tiledb/sm/fragment/fragment_info.h +++ b/tiledb/sm/fragment/fragment_info.h @@ -40,7 +40,7 @@ #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/fragment/single_fragment_info.h" -#include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -58,7 +58,7 @@ class FragmentInfo { FragmentInfo(); /** Constructor. */ - FragmentInfo(const URI& array_uri, StorageManager* storage_manager); + FragmentInfo(const URI& array_uri, ContextResources& resources); /** Destructor. */ ~FragmentInfo(); @@ -298,6 +298,34 @@ class FragmentInfo { const URI& new_fragment_uri, const std::vector& to_replace); + /** + * Returns the array schemas and fragment metadata for the given array. + * The function will focus only on relevant schemas and metadata as + * dictated by the input URI manager. + * + * @param array_dir The ArrayDirectory object used to retrieve the + * various URIs in the array directory. + * @param memory_tracker The memory tracker of the array + * for which the fragment metadata is loaded. + * @param enc_key The encryption key to use. + * @return tuple latest ArraySchema, map of all array schemas and + * vector of FragmentMetadata + * ArraySchema The array schema to be retrieved after the + * array is opened. + * ArraySchemaMap Map of all array schemas found keyed by name + * fragment_metadata The fragment metadata to be retrieved + * after the array is opened. + */ + static tuple< + shared_ptr, + std::unordered_map>, + std::vector>> + load_array_schemas_and_fragment_metadata( + ContextResources& resources, + const ArrayDirectory& array_dir, + MemoryTracker* memory_tracker, + const EncryptionKey& enc_key); + /** Returns the vector with the info about individual fragments. */ const std::vector& single_fragment_info_vec() const; @@ -395,8 +423,8 @@ class FragmentInfo { /** Information about fragments in the array. */ std::vector single_fragment_info_vec_; - /** The storage manager. */ - StorageManager* storage_manager_; + /** The context resources. */ + ContextResources* resources_; /** The URIs of the fragments to vacuum. */ std::vector to_vacuum_; diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 97a805e5afee..85980985d45c 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -81,7 +81,7 @@ FragmentMetadata::FragmentMetadata() { } FragmentMetadata::FragmentMetadata( - StorageManager* storage_manager, + ContextResources* resources, MemoryTracker* memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, @@ -89,7 +89,7 @@ FragmentMetadata::FragmentMetadata( bool dense, bool has_timestamps, bool has_deletes_meta) - : storage_manager_(storage_manager) + : resources_(resources) , memory_tracker_(memory_tracker) , array_schema_(array_schema) , dense_(dense) @@ -115,7 +115,7 @@ FragmentMetadata::~FragmentMetadata() = default; // Copy initialization FragmentMetadata::FragmentMetadata(const FragmentMetadata& other) { - storage_manager_ = other.storage_manager_; + resources_ = other.resources_; array_schema_ = other.array_schema_; dense_ = other.dense_; fragment_uri_ = other.fragment_uri_; @@ -136,7 +136,7 @@ FragmentMetadata::FragmentMetadata(const FragmentMetadata& other) { } FragmentMetadata& FragmentMetadata::operator=(const FragmentMetadata& other) { - storage_manager_ = other.storage_manager_; + resources_ = other.resources_; array_schema_ = other.array_schema_; dense_ = other.dense_; fragment_uri_ = other.fragment_uri_; @@ -377,7 +377,7 @@ void FragmentMetadata::compute_fragment_min_max_sum_null_count() { // Process all attributes in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, idx_map_.size(), [&](uint64_t n) { + &resources_->compute_tp(), 0, idx_map_.size(), [&](uint64_t n) { // For easy reference. const auto& name = names[n]; const auto& idx = idx_map_[name]; @@ -675,8 +675,7 @@ uint64_t FragmentMetadata::fragment_size() const { if (meta_file_size == 0) { auto meta_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); - throw_if_not_ok( - storage_manager_->vfs()->file_size(meta_uri, &meta_file_size)); + throw_if_not_ok(resources_->vfs().file_size(meta_uri, &meta_file_size)); } // Validate that the meta_file_size is not zero, either preloaded or fetched // above @@ -774,6 +773,86 @@ void FragmentMetadata::init(const NDRange& non_empty_domain) { fragment_null_counts_.resize(num); } +std::vector> FragmentMetadata::load( + ContextResources& resources, + MemoryTracker* memory_tracker, + const shared_ptr array_schema_latest, + const std::unordered_map>& + array_schemas_all, + const EncryptionKey& encryption_key, + const std::vector& fragments_to_load, + const std::unordered_map>& + offsets) { + auto timer_se = resources.stats().start_timer("sm_load_fragment_metadata"); + + // Load the metadata for each fragment + auto fragment_num = fragments_to_load.size(); + std::vector> fragment_metadata; + fragment_metadata.resize(fragment_num); + auto status = + parallel_for(&resources.compute_tp(), 0, fragment_num, [&](size_t f) { + const auto& sf = fragments_to_load[f]; + + URI coords_uri = + sf.uri_.join_path(constants::coords + constants::file_suffix); + + auto name = sf.uri_.remove_trailing_slash().last_path_part(); + auto format_version = utils::parse::get_fragment_version(name); + + // Note that the fragment metadata version is >= the array schema + // version. Therefore, the check below is defensive and will always + // ensure backwards compatibility. + shared_ptr metadata; + if (format_version <= 2) { + bool sparse; + RETURN_NOT_OK(resources.vfs().is_file(coords_uri, &sparse)); + metadata = make_shared( + HERE(), + &resources, + memory_tracker, + array_schema_latest, + sf.uri_, + sf.timestamp_range_, + !sparse); + } else { + // Fragment format version > 2 + metadata = make_shared( + HERE(), + &resources, + memory_tracker, + array_schema_latest, + sf.uri_, + sf.timestamp_range_); + } + + // Potentially find the basic fragment metadata in the consolidated + // metadata buffer + Tile* fragment_metadata_tile = nullptr; + uint64_t offset = 0; + + auto it = offsets.end(); + if (metadata->format_version() >= 9) { + it = offsets.find(name); + } else { + it = offsets.find(sf.uri_.to_string()); + } + if (it != offsets.end()) { + fragment_metadata_tile = it->second.first; + offset = it->second.second; + } + + // Load fragment metadata + metadata->load( + encryption_key, fragment_metadata_tile, offset, array_schemas_all); + + fragment_metadata[f] = metadata; + return Status::Ok(); + }); + throw_if_not_ok(status); + + return fragment_metadata; +} + void FragmentMetadata::load( const EncryptionKey& encryption_key, Tile* fragment_metadata_tile, @@ -784,8 +863,7 @@ void FragmentMetadata::load( // Load the metadata file size when we are not reading from consolidated // buffer if (fragment_metadata_tile == nullptr) { - throw_if_not_ok( - storage_manager_->vfs()->file_size(meta_uri, &meta_file_size_)); + throw_if_not_ok(resources_->vfs().file_size(meta_uri, &meta_file_size_)); } // Get fragment name version @@ -801,8 +879,7 @@ void FragmentMetadata::load( } void FragmentMetadata::store(const EncryptionKey& encryption_key) { - auto timer_se = - storage_manager_->stats()->start_timer("write_store_frag_meta"); + auto timer_se = resources_->stats().start_timer("write_store_frag_meta"); if (version_ < 7) { auto fragment_metadata_uri = @@ -880,7 +957,7 @@ void FragmentMetadata::store_v7_v10(const EncryptionKey& encryption_key) { store_footer(encryption_key); // Close file - throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().close_file(fragment_metadata_uri)); } void FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { @@ -962,7 +1039,7 @@ void FragmentMetadata::store_v11(const EncryptionKey& encryption_key) { store_footer(encryption_key); // Close file - throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().close_file(fragment_metadata_uri)); } void FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { @@ -1049,7 +1126,7 @@ void FragmentMetadata::store_v12_v14(const EncryptionKey& encryption_key) { store_footer(encryption_key); // Close file - throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().close_file(fragment_metadata_uri)); } void FragmentMetadata::store_v15_or_higher( @@ -1142,7 +1219,7 @@ void FragmentMetadata::store_v15_or_higher( store_footer(encryption_key); // Close file - throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().close_file(fragment_metadata_uri)); } void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { @@ -1441,7 +1518,7 @@ void FragmentMetadata::load_fragment_min_max_sum_null_count( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.fragment_min_max_sum_null_count_offset_); - storage_manager_->stats()->add_counter( + resources_->stats().add_counter( "read_fragment_min_max_sum_null_count_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); @@ -1464,7 +1541,7 @@ void FragmentMetadata::load_processed_conditions( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.processed_conditions_offsets_); - storage_manager_->stats()->add_counter( + resources_->stats().add_counter( "read_processed_conditions_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); @@ -1955,7 +2032,7 @@ void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { } auto tile = read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); - storage_manager_->stats()->add_counter("read_rtree_size", tile.size()); + resources_->stats().add_counter("read_rtree_size", tile.size()); // Use the serialized buffer size to approximate memory usage of the rtree. if (memory_tracker_ != nullptr && @@ -2077,11 +2154,10 @@ void FragmentMetadata::get_footer_offset_and_size( URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); uint64_t size_offset = meta_file_size_ - sizeof(uint64_t); - throw_if_not_ok(storage_manager_->vfs()->read( + throw_if_not_ok(resources_->vfs().read( fragment_metadata_uri, size_offset, size, sizeof(uint64_t))); *offset = meta_file_size_ - *size - sizeof(uint64_t); - storage_manager_->stats()->add_counter( - "read_frag_meta_size", sizeof(uint64_t)); + resources_->stats().add_counter("read_frag_meta_size", sizeof(uint64_t)); } } @@ -2353,7 +2429,7 @@ void FragmentMetadata::load_tile_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_offsets_[idx]); - storage_manager_->stats()->add_counter("read_tile_offsets_size", tile.size()); + resources_->stats().add_counter("read_tile_offsets_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_offsets(idx, deserializer); @@ -2380,8 +2456,7 @@ void FragmentMetadata::load_tile_var_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_offsets_[idx]); - storage_manager_->stats()->add_counter( - "read_tile_var_offsets_size", tile.size()); + resources_->stats().add_counter("read_tile_var_offsets_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_var_offsets(idx, deserializer); @@ -2403,8 +2478,7 @@ void FragmentMetadata::load_tile_var_sizes( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_sizes_[idx]); - storage_manager_->stats()->add_counter( - "read_tile_var_sizes_size", tile.size()); + resources_->stats().add_counter("read_tile_var_sizes_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_var_sizes(idx, deserializer); @@ -2426,7 +2500,7 @@ void FragmentMetadata::load_tile_validity_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_validity_offsets_[idx]); - storage_manager_->stats()->add_counter( + resources_->stats().add_counter( "read_tile_validity_offsets_size", tile.size()); ConstBuffer cbuff(tile.data(), tile.size()); @@ -2449,7 +2523,7 @@ void FragmentMetadata::load_tile_min_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_min_offsets_[idx]); - storage_manager_->stats()->add_counter("read_tile_min_size", tile.size()); + resources_->stats().add_counter("read_tile_min_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_min_values(idx, deserializer); @@ -2471,7 +2545,7 @@ void FragmentMetadata::load_tile_max_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_max_offsets_[idx]); - storage_manager_->stats()->add_counter("read_tile_max_size", tile.size()); + resources_->stats().add_counter("read_tile_max_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_max_values(idx, deserializer); @@ -2493,7 +2567,7 @@ void FragmentMetadata::load_tile_sum_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_sum_offsets_[idx]); - storage_manager_->stats()->add_counter("read_tile_sum_size", tile.size()); + resources_->stats().add_counter("read_tile_sum_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_sum_values(idx, deserializer); @@ -2515,8 +2589,7 @@ void FragmentMetadata::load_tile_null_count_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_null_count_offsets_[idx]); - storage_manager_->stats()->add_counter( - "read_tile_null_count_size", tile.size()); + resources_->stats().add_counter("read_tile_null_count_size", tile.size()); Deserializer deserializer(tile.data(), tile.size()); load_tile_null_count_values(idx, deserializer); @@ -3496,13 +3569,13 @@ void FragmentMetadata::load_v1_v2( URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); // Read metadata - GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); + GenericTileIO tile_io(*resources_, fragment_metadata_uri); auto&& [st, tile_opt] = - tile_io.read_generic(0, encryption_key, storage_manager_->config()); + tile_io.read_generic(0, encryption_key, resources_->config()); throw_if_not_ok(st); auto& tile = *tile_opt; - storage_manager_->stats()->add_counter("read_frag_meta_size", tile.size()); + resources_->stats().add_counter("read_frag_meta_size", tile.size()); // Pre-v10 format fragments we need to set the schema and schema name to // the "old" schema. This way "old" fragments are still loaded fine @@ -3776,7 +3849,7 @@ void FragmentMetadata::store_rtree( const EncryptionKey& encryption_key, uint64_t* nbytes) { auto rtree_tile = write_rtree(); write_generic_tile_to_file(encryption_key, rtree_tile, nbytes); - storage_manager_->stats()->add_counter("write_rtree_size", *nbytes); + resources_->stats().add_counter("write_rtree_size", *nbytes); } WriterTile FragmentMetadata::write_rtree() { @@ -3839,9 +3912,9 @@ Tile FragmentMetadata::read_generic_tile_from_file( std::string(constants::fragment_metadata_filename)); // Read metadata - GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); + GenericTileIO tile_io(*resources_, fragment_metadata_uri); auto&& [st, tile_opt] = - tile_io.read_generic(offset, encryption_key, storage_manager_->config()); + tile_io.read_generic(offset, encryption_key, resources_->config()); throw_if_not_ok(st); return std::move(*tile_opt); @@ -3859,7 +3932,7 @@ void FragmentMetadata::read_file_footer( tile = make_shared(HERE(), Tile::from_generic(*footer_size)); - storage_manager_->stats()->add_counter("read_frag_meta_size", *footer_size); + resources_->stats().add_counter("read_frag_meta_size", *footer_size); if (memory_tracker_ != nullptr && !memory_tracker_->take_memory( @@ -3872,7 +3945,7 @@ void FragmentMetadata::read_file_footer( } // Read footer - throw_if_not_ok(storage_manager_->vfs()->read( + throw_if_not_ok(resources_->vfs().read( fragment_metadata_uri, *footer_offset, tile->data_as(), @@ -3886,7 +3959,7 @@ void FragmentMetadata::write_generic_tile_to_file( URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); - GenericTileIO tile_io(storage_manager_->resources(), fragment_metadata_uri); + GenericTileIO tile_io(*resources_, fragment_metadata_uri); throw_if_not_ok(tile_io.write_generic(&tile, encryption_key, nbytes)); } @@ -3895,12 +3968,12 @@ void FragmentMetadata::write_footer_to_file(WriterTile& tile) const { std::string(constants::fragment_metadata_filename)); uint64_t size = tile.size(); - throw_if_not_ok(storage_manager_->vfs()->write( - fragment_metadata_uri, tile.data(), tile.size())); + throw_if_not_ok( + resources_->vfs().write(fragment_metadata_uri, tile.data(), tile.size())); // Write the size in the end if there is at least one var-sized dimension if (!array_schema_->domain().all_dims_fixed() || version_ >= 10) { - throw_if_not_ok(storage_manager_->vfs()->write( + throw_if_not_ok(resources_->vfs().write( fragment_metadata_uri, &size, sizeof(uint64_t))); } } @@ -3916,7 +3989,7 @@ void FragmentMetadata::store_tile_offsets( write_tile_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_tile_offsets_size", *nbytes); + resources_->stats().add_counter("write_tile_offsets_size", *nbytes); } void FragmentMetadata::write_tile_offsets( @@ -3943,8 +4016,7 @@ void FragmentMetadata::store_tile_var_offsets( write_tile_var_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter( - "write_tile_var_offsets_size", *nbytes); + resources_->stats().add_counter("write_tile_var_offsets_size", *nbytes); } void FragmentMetadata::write_tile_var_offsets( @@ -3972,7 +4044,7 @@ void FragmentMetadata::store_tile_var_sizes( write_tile_var_sizes(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_tile_var_sizes_size", *nbytes); + resources_->stats().add_counter("write_tile_var_sizes_size", *nbytes); } void FragmentMetadata::write_tile_var_sizes( @@ -3999,8 +4071,7 @@ void FragmentMetadata::store_tile_validity_offsets( write_tile_validity_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter( - "write_tile_validity_offsets_size", *nbytes); + resources_->stats().add_counter("write_tile_validity_offsets_size", *nbytes); } void FragmentMetadata::write_tile_validity_offsets( @@ -4028,7 +4099,7 @@ void FragmentMetadata::store_tile_mins( write_tile_mins(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_mins_size", *nbytes); + resources_->stats().add_counter("write_mins_size", *nbytes); } void FragmentMetadata::write_tile_mins(unsigned idx, Serializer& serializer) { @@ -4062,7 +4133,7 @@ void FragmentMetadata::store_tile_maxs( write_tile_maxs(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_maxs_size", *nbytes); + resources_->stats().add_counter("write_maxs_size", *nbytes); } void FragmentMetadata::write_tile_maxs(unsigned idx, Serializer& serializer) { @@ -4096,7 +4167,7 @@ void FragmentMetadata::store_tile_sums( write_tile_sums(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_sums_size", *nbytes); + resources_->stats().add_counter("write_sums_size", *nbytes); } void FragmentMetadata::write_tile_sums(unsigned idx, Serializer& serializer) { @@ -4121,7 +4192,7 @@ void FragmentMetadata::store_tile_null_counts( write_tile_null_counts(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_null_counts_size", *nbytes); + resources_->stats().add_counter("write_null_counts_size", *nbytes); } void FragmentMetadata::write_tile_null_counts( @@ -4173,7 +4244,7 @@ void FragmentMetadata::store_fragment_min_max_sum_null_count( serialize_data(serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter("write_null_counts_size", *nbytes); + resources_->stats().add_counter("write_null_counts_size", *nbytes); } void FragmentMetadata::store_processed_conditions( @@ -4199,8 +4270,7 @@ void FragmentMetadata::store_processed_conditions( serialize_processed_conditions(serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); - storage_manager_->stats()->add_counter( - "write_processed_conditions_size", *nbytes); + resources_->stats().add_counter("write_processed_conditions_size", *nbytes); } template @@ -4529,8 +4599,7 @@ void FragmentMetadata::store_footer(const EncryptionKey&) { write_footer(serializer); write_footer_to_file(tile); - storage_manager_->stats()->add_counter( - "write_frag_meta_footer_size", tile.size()); + resources_->stats().add_counter("write_frag_meta_footer_size", tile.size()); } void FragmentMetadata::resize_tile_offsets_vectors(uint64_t size) { @@ -4554,8 +4623,8 @@ void FragmentMetadata::clean_up() { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); - throw_if_not_ok(storage_manager_->vfs()->close_file(fragment_metadata_uri)); - throw_if_not_ok(storage_manager_->vfs()->remove_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().close_file(fragment_metadata_uri)); + throw_if_not_ok(resources_->vfs().remove_file(fragment_metadata_uri)); } const shared_ptr& FragmentMetadata::array_schema() const { diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 14c4a10caed4..504914c16065 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -44,7 +44,7 @@ #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/types.h" #include "tiledb/sm/rtree/rtree.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; using namespace tiledb::type; @@ -74,7 +74,7 @@ class FragmentMetadata { /** * Constructor. * - * @param storage_manager A storage manager instance. + * @param resources A context resources instance. * @param memory_tracker The memory tracker of the array this fragment * metadata corresponds to. * @param array_schema The schema of the array the fragment belongs to. @@ -87,7 +87,7 @@ class FragmentMetadata { * @param has_delete_meta Does the fragment contains delete metadata. */ FragmentMetadata( - StorageManager* storage_manager, + ContextResources* resources, MemoryTracker* memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, @@ -447,6 +447,45 @@ class FragmentMetadata { uint64_t offset, std::unordered_map> array_schemas); + /** + * Loads the fragment metadata of an open array given a vector of + * fragment URIs `fragments_to_load`. + * The function stores the fragment metadata of each fragment + * in `fragments_to_load` into the returned vector, such + * that there is a one-to-one correspondence between the two vectors. + * + * If `meta_buf` has data, then some fragment metadata may be contained + * in there and does not need to be loaded from storage. In that + * case, `offsets` helps identifying each fragment metadata in the + * buffer. + * + * @param resources A context resources instance. + * @param memory_tracker The memory tracker of the array + * for which the metadata is loaded. This will be passed to + * the constructor of each of the metadata loaded. + * @param array_schema_latest The latest array schema. + * @param array_schemas_all All the array schemas in a map keyed by the + * schema filename. + * @param encryption_key The encryption key to use. + * @param fragments_to_load The fragments whose metadata to load. + * @param offsets A map from a fragment name to an offset in `meta_buff` + * where the basic fragment metadata can be found. If the offset + * cannot be found, then the metadata of that fragment will be loaded from + * storage instead. + * @return Vector of FragmentMetadata is the fragment metadata to be + * retrieved. + */ + static std::vector> load( + ContextResources& resources, + MemoryTracker* memory_tracker, + const shared_ptr array_schema, + const std::unordered_map>& + array_schemas_all, + const EncryptionKey& encryption_key, + const std::vector& fragments_to_load, + const std::unordered_map>& + offsets); + /** Stores all the metadata to storage. */ void store(const EncryptionKey& encryption_key); @@ -1162,12 +1201,12 @@ class FragmentMetadata { return gt_offsets_; } - /** set the SM pointer during deserialization*/ - void set_storage_manager(StorageManager* sm) { - storage_manager_ = sm; + /** set the CR pointer during deserialization*/ + void set_context_resources(ContextResources* cr) { + resources_ = cr; } - /** set the SM pointer during deserialization*/ + /** set the memory tracker pointer during deserialization*/ void set_memory_tracker(MemoryTracker* memory_tracker) { memory_tracker_ = memory_tracker; } @@ -1208,7 +1247,7 @@ class FragmentMetadata { /* ********************************* */ /** The storage manager. */ - StorageManager* storage_manager_; + ContextResources* resources_; /** * The memory tracker of the array this fragment metadata corresponds to. diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index 89dbb39cf4a2..b1fca72a2cd6 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/status.h" +#include "tiledb/sm/storage_manager/storage_manager.h" using namespace tiledb::common; diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 0a9428e1e816..b2e86ca9c139 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -192,7 +192,8 @@ Status GlobalOrderWriter::alloc_global_write_state() { // Alloc FragmentMetadata object global_write_state_->frag_meta_ = make_shared(HERE()); // Used in serialization when FragmentMetadata is built from ground up - global_write_state_->frag_meta_->set_storage_manager(storage_manager_); + global_write_state_->frag_meta_->set_context_resources( + &storage_manager_->resources()); return Status::Ok(); } diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index a02b30223e43..8d736a140357 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -183,7 +183,7 @@ Status UnorderedWriter::alloc_frag_meta() { // Alloc FragmentMetadata object. frag_meta_ = make_shared(HERE()); // Used in serialization when FragmentMetadata is built from ground up. - frag_meta_->set_storage_manager(storage_manager_); + frag_meta_->set_context_resources(&storage_manager_->resources()); return Status::Ok(); } diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 3bd52a1282f3..42e073e3d39b 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -801,7 +801,7 @@ Status WriterBase::create_fragment( buffers_.count(constants::delete_timestamps) != 0; frag_meta = make_shared( HERE(), - storage_manager_, + &storage_manager_->resources(), nullptr, array_->array_schema_latest_ptr(), fragment_uri_, diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index de34d6b11e59..45709aa8cf95 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -40,6 +40,7 @@ #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/stats/stats.h" +#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb; diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 861ef7059176..812430e97ac3 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -50,6 +50,7 @@ #include "tiledb/sm/serialization/array_directory.h" #include "tiledb/sm/serialization/array_schema.h" #include "tiledb/sm/serialization/fragment_metadata.h" +#include "tiledb/sm/storage_manager/storage_manager.h" using namespace tiledb::common; using namespace tiledb::sm::stats; @@ -314,7 +315,7 @@ Status array_from_capnp( array->array_schema_latest_ptr(), frag_meta_reader, meta, - storage_manager, + &storage_manager->resources(), array->memory_tracker())); if (client_side) { meta->set_rtree_loaded(); diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index 0c66efd20dae..74482c0b4c15 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -120,12 +120,12 @@ Status fragment_metadata_from_capnp( const shared_ptr& array_schema, const capnp::FragmentMetadata::Reader& frag_meta_reader, shared_ptr frag_meta, - StorageManager* storage_manager, + ContextResources* resources, MemoryTracker* memory_tracker) { // TODO: consider a new constructor for fragment meta or using the // existing one - if (storage_manager) { - frag_meta->set_storage_manager(storage_manager); + if (resources) { + frag_meta->set_context_resources(resources); } if (memory_tracker) { frag_meta->set_memory_tracker(memory_tracker); diff --git a/tiledb/sm/serialization/fragment_metadata.h b/tiledb/sm/serialization/fragment_metadata.h index be740a10f492..59868fd4794c 100644 --- a/tiledb/sm/serialization/fragment_metadata.h +++ b/tiledb/sm/serialization/fragment_metadata.h @@ -57,7 +57,7 @@ namespace serialization { * @param array_schema the schema of the array the metadata belongs * @param frag_meta_reader cap'n proto class * @param frag_meta fragment metadata object to deserialize into - * @param storage_manager storage manager associated + * @param resources ContextResources associated * @param memory_tracker memory tracker associated * @return Status */ @@ -65,7 +65,7 @@ Status fragment_metadata_from_capnp( const shared_ptr& array_schema, const capnp::FragmentMetadata::Reader& frag_meta_reader, shared_ptr frag_meta, - StorageManager* storage_manager = nullptr, + ContextResources* resources = nullptr, MemoryTracker* memory_tracker = nullptr); /** diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 44e8dabc6931..65070c0f40e5 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -74,7 +74,7 @@ #include "tiledb/sm/serialization/config.h" #include "tiledb/sm/serialization/fragment_metadata.h" #include "tiledb/sm/serialization/query.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" +#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/subarray_partitioner.h" using namespace tiledb::common; diff --git a/tiledb/sm/serialization/query_aggregates.h b/tiledb/sm/serialization/query_aggregates.h index 774c9439c2d9..c54cfa503cc5 100644 --- a/tiledb/sm/serialization/query_aggregates.h +++ b/tiledb/sm/serialization/query_aggregates.h @@ -34,7 +34,9 @@ #define TILEDB_SERIALIZATION_QUERY_AGGREGATES_H #ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/query/query.h" #include "tiledb/sm/serialization/capnp_utils.h" + #endif namespace tiledb::sm::serialization { diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ca1fa41e8b27..772b2ce30e59 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -169,87 +169,6 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { return Status::Ok(); } -std::tuple< - Status, - optional>, - optional>>, - optional>>> -StorageManagerCanonical::load_array_schemas_and_fragment_metadata( - const ArrayDirectory& array_dir, - MemoryTracker* memory_tracker, - const EncryptionKey& enc_key) { - auto timer_se = - stats()->start_timer("sm_load_array_schemas_and_fragment_metadata"); - - // Load array schemas - auto&& [array_schema_latest, array_schemas_all] = - array_dir.load_array_schemas(enc_key); - - const auto filtered_fragment_uris = - load_filtered_fragment_uris(array_schema_latest->dense(), array_dir); - const auto& meta_uris = array_dir.fragment_meta_uris(); - const auto& fragments_to_load = filtered_fragment_uris.fragment_uris(); - - // Get the consolidated fragment metadatas - std::vector> fragment_metadata_tiles(meta_uris.size()); - std::vector>> offsets_vectors( - meta_uris.size()); - auto status = parallel_for(compute_tp(), 0, meta_uris.size(), [&](size_t i) { - auto&& [st, tile_opt, offsets] = - load_consolidated_fragment_meta(meta_uris[i], enc_key); - RETURN_NOT_OK(st); - fragment_metadata_tiles[i] = - make_shared(HERE(), std::move(*tile_opt)); - offsets_vectors[i] = std::move(offsets.value()); - return st; - }); - RETURN_NOT_OK_TUPLE(status, nullopt, nullopt, nullopt); - - // Get the unique fragment metadatas into a map. - std::unordered_map> offsets; - for (uint64_t i = 0; i < offsets_vectors.size(); i++) { - for (auto& offset : offsets_vectors[i]) { - if (offsets.count(offset.first) == 0) { - offsets.emplace( - offset.first, - std::make_pair(fragment_metadata_tiles[i].get(), offset.second)); - } - } - } - - // Load the fragment metadata - auto&& [st_fragment_meta, fragment_metadata] = load_fragment_metadata( - memory_tracker, - array_schema_latest, - array_schemas_all, - enc_key, - fragments_to_load, - offsets); - RETURN_NOT_OK_TUPLE(st_fragment_meta, nullopt, nullopt, nullopt); - - return { - Status::Ok(), array_schema_latest, array_schemas_all, fragment_metadata}; -} - -tuple>>> -StorageManagerCanonical::array_load_fragments( - Array* array, const std::vector& fragments_to_load) { - auto timer_se = stats()->start_timer("sm_array_load_fragments"); - - // Load the fragment metadata - std::unordered_map> offsets; - auto&& [st_fragment_meta, fragment_metadata] = load_fragment_metadata( - array->memory_tracker(), - array->array_schema_latest_ptr(), - array->array_schemas_all(), - *array->encryption_key(), - fragments_to_load, - offsets); - RETURN_NOT_OK_TUPLE(st_fragment_meta, nullopt); - - return {Status::Ok(), fragment_metadata}; -} - Status StorageManagerCanonical::array_consolidate( const char* array_name, EncryptionType encryption_type, @@ -1943,128 +1862,6 @@ void StorageManagerCanonical::load_group_metadata( /* PRIVATE METHODS */ /* ****************************** */ -tuple>>> -StorageManagerCanonical::load_fragment_metadata( - MemoryTracker* memory_tracker, - const shared_ptr& array_schema_latest, - const std::unordered_map>& - array_schemas_all, - const EncryptionKey& encryption_key, - const std::vector& fragments_to_load, - const std::unordered_map>& - offsets) { - auto timer_se = stats()->start_timer("sm_load_fragment_metadata"); - - // Load the metadata for each fragment - auto fragment_num = fragments_to_load.size(); - std::vector> fragment_metadata; - fragment_metadata.resize(fragment_num); - auto status = parallel_for(compute_tp(), 0, fragment_num, [&](size_t f) { - const auto& sf = fragments_to_load[f]; - - URI coords_uri = - sf.uri_.join_path(constants::coords + constants::file_suffix); - - auto name = sf.uri_.remove_trailing_slash().last_path_part(); - auto format_version = utils::parse::get_fragment_version(name); - - // Note that the fragment metadata version is >= the array schema - // version. Therefore, the check below is defensive and will always - // ensure backwards compatibility. - shared_ptr metadata; - if (format_version <= 2) { - bool sparse; - RETURN_NOT_OK(vfs()->is_file(coords_uri, &sparse)); - metadata = make_shared( - HERE(), - this, - memory_tracker, - array_schema_latest, - sf.uri_, - sf.timestamp_range_, - !sparse); - } else { - // Fragment format version > 2 - metadata = make_shared( - HERE(), - this, - memory_tracker, - array_schema_latest, - sf.uri_, - sf.timestamp_range_); - } - - // Potentially find the basic fragment metadata in the consolidated - // metadata buffer - Tile* fragment_metadata_tile = nullptr; - uint64_t offset = 0; - - auto it = offsets.end(); - if (metadata->format_version() >= 9) { - it = offsets.find(name); - } else { - it = offsets.find(sf.uri_.to_string()); - } - if (it != offsets.end()) { - fragment_metadata_tile = it->second.first; - offset = it->second.second; - } - - // Load fragment metadata - metadata->load( - encryption_key, fragment_metadata_tile, offset, array_schemas_all); - - fragment_metadata[f] = metadata; - return Status::Ok(); - }); - RETURN_NOT_OK_TUPLE(status, nullopt); - - return {Status::Ok(), fragment_metadata}; -} - -tuple< - Status, - optional, - optional>>> -StorageManagerCanonical::load_consolidated_fragment_meta( - const URI& uri, const EncryptionKey& enc_key) { - auto timer_se = stats()->start_timer("sm_read_load_consolidated_frag_meta"); - - // No consolidated fragment metadata file - if (uri.to_string().empty()) - return {Status::Ok(), nullopt, nullopt}; - - auto&& tile = GenericTileIO::load(resources_, uri, 0, enc_key); - - stats()->add_counter("consolidated_frag_meta_size", tile.size()); - - uint32_t fragment_num; - Deserializer deserializer(tile.data(), tile.size()); - fragment_num = deserializer.read(); - - uint64_t name_size, offset; - std::string name; - std::vector> ret; - ret.reserve(fragment_num); - for (uint32_t f = 0; f < fragment_num; ++f) { - name_size = deserializer.read(); - name.resize(name_size); - deserializer.read(&name[0], name_size); - offset = deserializer.read(); - ret.emplace_back(name, offset); - } - - return {Status::Ok(), std::move(tile), ret}; -} - -const ArrayDirectory::FilteredFragmentUris -StorageManagerCanonical::load_filtered_fragment_uris( - const bool dense, const ArrayDirectory& array_dir) { - auto timer_se = stats()->start_timer("sm_load_filtered_fragment_uris"); - - return array_dir.filtered_fragment_uris(dense); -} - Status StorageManagerCanonical::set_default_tags() { const auto version = std::to_string(constants::library_version[0]) + "." + std::to_string(constants::library_version[1]) + "." + diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index d2a502978f9c..903157f833de 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -227,35 +227,6 @@ class StorageManagerCanonical { tdb_shared_ptr group, const EncryptionKey& encryption_key); - /** - * Returns the array schemas and fragment metadata for the given array. - * The function will focus only on relevant schemas and metadata as - * dictated by the input URI manager. - * - * @param array_dir The ArrayDirectory object used to retrieve the - * various URIs in the array directory. - * @param memory_tracker The memory tracker of the array - * for which the fragment metadata is loaded. - * @param enc_key The encryption key to use. - * @return tuple of Status, latest ArraySchema, map of all array schemas and - * vector of FragmentMetadata - * Status Ok on success, else error - * ArraySchema The array schema to be retrieved after the - * array is opened. - * ArraySchemaMap Map of all array schemas found keyed by name - * fragment_metadata The fragment metadata to be retrieved - * after the array is opened. - */ - tuple< - Status, - optional>, - optional>>, - optional>>> - load_array_schemas_and_fragment_metadata( - const ArrayDirectory& array_dir, - MemoryTracker* memory_tracker, - const EncryptionKey& enc_key); - /** * Opens an group for reads. * @@ -275,17 +246,6 @@ class StorageManagerCanonical { std::tuple>> group_open_for_writes(Group* group); - /** - * Load fragments for an already open array. - * - * @param array The open array. - * @param fragment_info The list of fragment info. - * @return Status, the fragment metadata to be loaded. - */ - tuple>>> - array_load_fragments( - Array* array, const std::vector& fragment_info); - /** * Consolidates the fragments of an array into a single one. * @@ -929,71 +889,6 @@ class StorageManagerCanonical { /** Increment the count of in-progress queries. */ void increment_in_progress(); - /** - * Loads the fragment metadata of an open array given a vector of - * fragment URIs `fragments_to_load`. - * The function stores the fragment metadata of each fragment - * in `fragments_to_load` into the returned vector, such - * that there is a one-to-one correspondence between the two vectors. - * - * If `meta_buf` has data, then some fragment metadata may be contained - * in there and does not need to be loaded from storage. In that - * case, `offsets` helps identifying each fragment metadata in the - * buffer. - * - * @param memory_tracker The memory tracker of the array - * for which the metadata is loaded. This will be passed to - * the constructor of each of the metadata loaded. - * @param array_schema_latest The latest array schema. - * @param array_schemas_all All the array schemas in a map keyed by the - * schema filename. - * @param encryption_key The encryption key to use. - * @param fragments_to_load The fragments whose metadata to load. - * @param offsets A map from a fragment name to an offset in `meta_buff` - * where the basic fragment metadata can be found. If the offset - * cannot be found, then the metadata of that fragment will be loaded from - * storage instead. - * @return tuple of Status and vector of FragmentMetadata - * Status Ok on success, else error - * Vector of FragmentMetadata is the fragment metadata to be retrieved. - */ - tuple>>> - load_fragment_metadata( - MemoryTracker* memory_tracker, - const shared_ptr& array_schema, - const std::unordered_map>& - array_schemas_all, - const EncryptionKey& encryption_key, - const std::vector& fragments_to_load, - const std::unordered_map>& - offsets); - - /** - * Loads the latest consolidated fragment metadata from storage. - * - * @param uri The URI of the consolidated fragment metadata. - * @param enc_key The encryption key that may be needed to access the file. - * @param f_buff The buffer to hold the consolidated fragment metadata. - * @return Status, vector from the fragment name to the offset in `f_buff` - * where the basic fragment metadata starts. - */ - tuple< - Status, - optional, - optional>>> - load_consolidated_fragment_meta(const URI& uri, const EncryptionKey& enc_key); - - /** - * Loads the filtered fragment URIs from the array directory. - * - * @param dense Is this a dense array. - * @param array_dir The ArrayDirectory object used to retrieve the - * various URIs in the array directory. - * @return Filtered fragment URIs. - */ - const ArrayDirectory::FilteredFragmentUris load_filtered_fragment_uris( - const bool dense, const ArrayDirectory& array_dir); - /** Block until there are zero in-progress queries. */ void wait_for_zero_in_progress(); diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 46ca4abe07dd..6638786cd763 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -57,6 +57,7 @@ #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/rtree/rtree.h" #include "tiledb/sm/stats/global_stats.h" +#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/subarray/relevant_fragment_generator.h" #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/apply_with_type.h" From 41c96a6bd9b0cd24d4702409ed9f90f2a30a5886 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:37:04 +0200 Subject: [PATCH 035/456] Aggregates: allow aggregates pipeline the processing of full tiles. (#4465) This adds a new aggregate_full_tile interface to iaggregator so that we can later use the data in the fragment metadata to improve aggregation performance. The data is packaged in the new FullTileData class, which will later be produced by the fragment metadata. Note that this only implements the aggregators side of the work... Reader/FragmentMetadata changes will follow. --- TYPE: FEATURE DESC: Aggregates: allow aggregates pipeline the processing of full tiles. --- .../readers/aggregators/count_aggregator.cc | 13 ++ .../readers/aggregators/count_aggregator.h | 7 + .../query/readers/aggregators/iaggregator.h | 8 + .../readers/aggregators/min_max_aggregator.cc | 31 +++- .../readers/aggregators/min_max_aggregator.h | 49 ++++++ .../readers/aggregators/sum_aggregator.cc | 41 +++-- .../readers/aggregators/sum_aggregator.h | 26 ++- .../aggregators/test/unit_aggregators.cc | 163 +++++++++++++++--- .../query/readers/aggregators/tile_metadata.h | 161 +++++++++++++++++ 9 files changed, 447 insertions(+), 52 deletions(-) create mode 100644 tiledb/sm/query/readers/aggregators/tile_metadata.h diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.cc b/tiledb/sm/query/readers/aggregators/count_aggregator.cc index 7193976b0eda..b3c1adca2a2a 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.cc @@ -76,6 +76,19 @@ void CountAggregatorBase::aggregate_data( count_ += std::get<1>(res); } +template +void CountAggregatorBase::aggregate_tile_with_frag_md( + TileMetadata& tile_metadata) { + uint64_t count; + if constexpr (std::is_same::value) { + count = tile_metadata.count(); + } else { + count = tile_metadata.null_count(); + } + + count_ += count; +} + template void CountAggregatorBase::copy_to_user_buffer( std::string output_field_name, diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.h b/tiledb/sm/query/readers/aggregators/count_aggregator.h index 19cf38dd5d32..a2cd3c8eded7 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.h @@ -91,6 +91,13 @@ class CountAggregatorBase : public OutputBufferValidator, public IAggregator { */ void aggregate_data(AggregateBuffer& input_data) override; + /** + * Aggregate a tile with fragment metadata. + * + * @param tile_metadata Tile metadata for aggregation. + */ + void aggregate_tile_with_frag_md(TileMetadata& tile_metadata) override; + /** * Copy final data to the user buffer. * diff --git a/tiledb/sm/query/readers/aggregators/iaggregator.h b/tiledb/sm/query/readers/aggregators/iaggregator.h index 61ebb71d3816..9130b535a60f 100644 --- a/tiledb/sm/query/readers/aggregators/iaggregator.h +++ b/tiledb/sm/query/readers/aggregators/iaggregator.h @@ -37,6 +37,7 @@ #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/query/readers/aggregators/input_field_validator.h" #include "tiledb/sm/query/readers/aggregators/output_buffer_validator.h" +#include "tiledb/sm/query/readers/aggregators/tile_metadata.h" namespace tiledb::sm { @@ -88,6 +89,13 @@ class IAggregator { */ virtual void aggregate_data(AggregateBuffer& input_data) = 0; + /** + * Aggregate a tile with fragment metadata. + * + * @param tile_metadata Tile metadata for aggregation. + */ + virtual void aggregate_tile_with_frag_md(TileMetadata& tile_metadata) = 0; + /** * Copy final data to the user buffer. * diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc b/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc index f732c40c685a..c1b92b0f266d 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.cc @@ -145,12 +145,32 @@ void ComparatorAggregator::aggregate_data(AggregateBuffer& input_data) { res = aggregate_with_count_.template aggregate(input_data); } + const auto value = std::get<0>(res); + const auto count = std::get<1>(res); + update_value(value, count); +} + +template +void ComparatorAggregator::aggregate_tile_with_frag_md( + TileMetadata& tile_metadata) { + const auto value = tile_metadata_value(tile_metadata); + const auto count = tile_metadata.count() - tile_metadata.null_count(); + update_value(value, count); +} + +template +void ComparatorAggregator::copy_to_user_buffer( + std::string output_field_name, + std::unordered_map& buffers) { + ComparatorAggregatorBase::copy_to_user_buffer(output_field_name, buffers); +} + +template +void ComparatorAggregator::update_value(VALUE_T value, uint64_t count) { { // This might be called on multiple threads, the final result stored in // value_ should be computed in a thread safe manner. std::unique_lock lock(value_mtx_); - const auto value = std::get<0>(res); - const auto count = std::get<1>(res); if (count > 0 && (ComparatorAggregatorBase::value_ == std::nullopt || op_(value, ComparatorAggregatorBase::value_.value()))) { @@ -165,13 +185,6 @@ void ComparatorAggregator::aggregate_data(AggregateBuffer& input_data) { } } -template -void ComparatorAggregator::copy_to_user_buffer( - std::string output_field_name, - std::unordered_map& buffers) { - ComparatorAggregatorBase::copy_to_user_buffer(output_field_name, buffers); -} - // Explicit template instantiations template ComparatorAggregator>::ComparatorAggregator( const FieldInfo&); diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h index 57ac4292fee2..e5350b2a678d 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h @@ -124,6 +124,9 @@ class ComparatorAggregator : public ComparatorAggregatorBase, */ ComparatorAggregator(const FieldInfo& field_info); + /** Virtual destructor. */ + virtual ~ComparatorAggregator() = default; + DISABLE_COPY_AND_COPY_ASSIGN(ComparatorAggregator); DISABLE_MOVE_AND_MOVE_ASSIGN(ComparatorAggregator); @@ -169,6 +172,13 @@ class ComparatorAggregator : public ComparatorAggregatorBase, */ void aggregate_data(AggregateBuffer& input_data) override; + /** + * Aggregate a tile with fragment metadata. + * + * @param tile_metadata Tile metadata for aggregation. + */ + void aggregate_tile_with_frag_md(TileMetadata& tile_metadata) override; + /** * Copy final data to the user buffer. * @@ -185,6 +195,21 @@ class ComparatorAggregator : public ComparatorAggregatorBase, } private: + /* ********************************* */ + /* PRIVATE FUNCTIONS */ + /* ********************************* */ + + /** Returns the tile metadata value for the full tile data. */ + virtual VALUE_T tile_metadata_value(TileMetadata& tile_metadata) = 0; + + /** + * Update the value of the aggregation, if required. + * + * @param value Candidate value. + * @param count Count of values considered. + */ + void update_value(VALUE_T value, uint64_t count); + /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ @@ -204,6 +229,8 @@ class MinAggregator : public ComparatorAggregator< T, std::less::value_type>> { public: + using VALUE_T = typename type_data::value_type; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -230,6 +257,16 @@ class MinAggregator : public ComparatorAggregator< std::string aggregate_name() override { return constants::aggregate_min_str; } + + private: + /* ********************************* */ + /* PRIVATE FUNCTIONS */ + /* ********************************* */ + + /** Returns the tile metadata value for the full tile data. */ + VALUE_T tile_metadata_value(TileMetadata& tile_metadata) override { + return tile_metadata.min_as(); + } }; template @@ -237,6 +274,8 @@ class MaxAggregator : public ComparatorAggregator< T, std::greater::value_type>> { public: + using VALUE_T = typename type_data::value_type; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -264,6 +303,16 @@ class MaxAggregator : public ComparatorAggregator< std::string aggregate_name() override { return constants::aggregate_max_str; } + + private: + /* ********************************* */ + /* PRIVATE FUNCTIONS */ + /* ********************************* */ + + /** Returns the tile metadata value for the full tile data. */ + VALUE_T tile_metadata_value(TileMetadata& tile_metadata) override { + return tile_metadata.max_as(); + } }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc index 21cb3647eb73..fd58ebc61ceb 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.cc +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.cc @@ -74,27 +74,28 @@ void SumWithCountAggregator::aggregate_data(AggregateBuffer& input_data) { return; } + tuple::sum_type, uint64_t> res; try { - tuple::sum_type, uint64_t> res; if (input_data.is_count_bitmap()) { res = aggregate_with_count_.template aggregate(input_data); } else { res = aggregate_with_count_.template aggregate(input_data); } - - const auto value = std::get<0>(res); - const auto count = std::get<1>(res); - SafeSum().safe_sum(value, sum_); - count_ += count; - - // Here we know that if the count is greater than 0, it means at least one - // valid item was found, which means the result is valid. - if (field_info_.is_nullable_ && count) { - validity_value_ = 1; - } } catch (std::overflow_error& e) { sum_overflowed_ = true; } + + const auto sum = std::get<0>(res); + const auto count = std::get<1>(res); + update_sum(sum, count); +} + +template +void SumWithCountAggregator::aggregate_tile_with_frag_md( + TileMetadata& tile_metadata) { + const auto sum = tile_metadata.sum_as(); + const auto count = tile_metadata.count() - tile_metadata.null_count(); + update_sum(sum, count); } template @@ -114,6 +115,22 @@ void SumWithCountAggregator::copy_validity_value_to_user_buffers( } } +template +void SumWithCountAggregator::update_sum(SUM_T sum, uint64_t count) { + try { + SafeSum().safe_sum(sum, sum_); + count_ += count; + + // Here we know that if the count is greater than 0, it means at least one + // valid item was found, which means the result is valid. + if (field_info_.is_nullable_ && count) { + validity_value_ = 1; + } + } catch (std::overflow_error& e) { + sum_overflowed_ = true; + } +} + template void SumAggregator::copy_to_user_buffer( std::string output_field_name, diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.h b/tiledb/sm/query/readers/aggregators/sum_aggregator.h index 14b5316e1c3d..7b15180061ac 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.h @@ -49,6 +49,8 @@ class SumWithCountAggregator : public InputFieldValidator, public OutputBufferValidator, public IAggregator { public: + using SUM_T = typename sum_type_data::sum_type; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -106,6 +108,13 @@ class SumWithCountAggregator : public InputFieldValidator, */ void aggregate_data(AggregateBuffer& input_data) override; + /** + * Aggregate a tile with fragment metadata. + * + * @param tile_metadata Tile metadata for aggregation. + */ + void aggregate_tile_with_frag_md(TileMetadata& tile_metadata) override; + /** * Copy final validity value to the user buffer. * @@ -114,6 +123,18 @@ class SumWithCountAggregator : public InputFieldValidator, void copy_validity_value_to_user_buffers(QueryBuffer& result_buffer); protected: + /* ********************************* */ + /* PROTECTED FUNCTIONS */ + /* ********************************* */ + + /** + * Update the sum. + * + * @param sum Sum. + * @param count Count of values summed. + */ + void update_sum(SUM_T sum, uint64_t count); + /* ********************************* */ /* PROTECTED ATTRIBUTES */ /* ********************************* */ @@ -122,11 +143,10 @@ class SumWithCountAggregator : public InputFieldValidator, const FieldInfo field_info_; /** AggregateWithCount to do summation of AggregateBuffer data. */ - AggregateWithCount::sum_type, SafeSum, NonNull> - aggregate_with_count_; + AggregateWithCount aggregate_with_count_; /** Computed sum. */ - std::atomic::sum_type> sum_; + std::atomic sum_; /** Count of values. */ std::atomic count_; diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 17b475027736..6ec03eca483b 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -469,6 +469,30 @@ void basic_aggregation_test(std::vector expected_results) { std::vector validity_data = {0, 0, 1, 0, 1, 0, 1, 0, 1, 0}; SECTION("No bitmap") { + ByteVecValue zero(8); + ByteVecValue full_tile_sum(8); + if constexpr (!std::is_same::value) { + zero.assign_as::sum_type>(0); + full_tile_sum.assign_as::sum_type>( + 10); + } + + auto tile_metadata_all_null = TileMetadata( + 10, + 10, + &fixed_data[0], + sizeof(T), + &fixed_data[0], + sizeof(T), + zero.data()); + auto tile_metadata = TileMetadata( + 10, + 5, + &fixed_data[0], + sizeof(T), + &fixed_data[0], + sizeof(T), + full_tile_sum.data()); if (aggregator.has_value()) { // Regular attribute. AggregateBuffer input_data{ @@ -476,6 +500,14 @@ void basic_aggregation_test(std::vector expected_results) { aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); check_value(RES(), res, expected_results[0]); + + aggregator->aggregate_tile_with_frag_md(tile_metadata_all_null); + aggregator->copy_to_user_buffer("Agg", buffers); + check_value(RES(), res, expected_results[1]); + + aggregator->aggregate_tile_with_frag_md(tile_metadata); + aggregator->copy_to_user_buffer("Agg", buffers); + check_value(RES(), res, expected_results[2]); } // Nullable attribute. @@ -490,7 +522,17 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data2); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value(RES(), res2, expected_results[1]); + check_value(RES(), res2, expected_results[3]); + check_validity(validity, 1); + + aggregator_nullable.aggregate_tile_with_frag_md(tile_metadata_all_null); + aggregator_nullable.copy_to_user_buffer("Agg2", buffers); + check_value(RES(), res2, expected_results[4]); + check_validity(validity, 1); + + aggregator_nullable.aggregate_tile_with_frag_md(tile_metadata); + aggregator_nullable.copy_to_user_buffer("Agg2", buffers); + check_value(RES(), res2, expected_results[5]); check_validity(validity, 1); } @@ -502,13 +544,13 @@ void basic_aggregation_test(std::vector expected_results) { 2, 10, fixed_data.data(), nullopt, nullopt, false, bitmap.data(), 1}; aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); - check_value(RES(), res, expected_results[2]); + check_value(RES(), res, expected_results[6]); AggregateBuffer input_data2{ 0, 2, fixed_data.data(), nullopt, nullopt, false, bitmap.data(), 1}; aggregator->aggregate_data(input_data2); aggregator->copy_to_user_buffer("Agg", buffers); - check_value(RES(), res, expected_results[3]); + check_value(RES(), res, expected_results[7]); } // Nullable attribute. @@ -529,10 +571,10 @@ void basic_aggregation_test(std::vector expected_results) { res2.data()[0] = '0'; } - if (is_nan(expected_results[4])) { + if (is_nan(expected_results[8])) { CHECK(is_nan(*static_cast(static_cast(res2.data())))); } else { - check_value(RES(), res2, expected_results[4]); + check_value(RES(), res2, expected_results[8]); } check_validity(validity, 0); @@ -547,7 +589,7 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data4); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value(RES(), res2, expected_results[5]); + check_value(RES(), res2, expected_results[9]); check_validity(validity, 1); } @@ -566,7 +608,7 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); - check_value(RES(), res, expected_results[6]); + check_value(RES(), res, expected_results[10]); AggregateBuffer input_data2{ 0, @@ -579,7 +621,7 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data2); aggregator->copy_to_user_buffer("Agg", buffers); - check_value(RES(), res, expected_results[7]); + check_value(RES(), res, expected_results[11]); } // Nullable attribute. @@ -594,7 +636,7 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data3); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value(RES(), res2, expected_results[8]); + check_value(RES(), res2, expected_results[12]); check_validity(validity, 1); AggregateBuffer input_data4{ @@ -608,7 +650,7 @@ void basic_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data4); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value(RES(), res2, expected_results[9]); + check_value(RES(), res2, expected_results[13]); check_validity(validity, 1); } } @@ -633,7 +675,7 @@ TEMPLATE_LIST_TEST_CASE( basic_aggregation_test< T, typename sum_type_data::sum_type, - SumAggregator>({27, 14, 11, 14, 0, 6, 29, 34, 22, 22}); + SumAggregator>({27, 27, 37, 14, 14, 24, 11, 14, 0, 6, 29, 34, 22, 22}); } TEMPLATE_LIST_TEST_CASE( @@ -643,7 +685,11 @@ TEMPLATE_LIST_TEST_CASE( typedef TestType T; basic_aggregation_test>( {(27.0 / 8.0), + (27.0 / 8.0), + (37.0 / 13.0), + (14.0 / 4.0), (14.0 / 4.0), + (24.0 / 9.0), (11.0 / 3.0), (14.0 / 5.0), std::numeric_limits::signaling_NaN(), @@ -684,9 +730,9 @@ TEMPLATE_LIST_TEST_CASE( AggUnderTestMinMax) { typedef decltype(TestType::first) T; typedef decltype(TestType::second) AGG; - std::vector res = {1, 2, 2, 1, 0, 2, 1, 1, 2, 2}; + std::vector res = {1, 1, 1, 2, 2, 1, 2, 1, 0, 2, 1, 1, 2, 2}; if (std::is_same>::value) { - res = {5, 5, 5, 5, 0, 4, 5, 5, 4, 4}; + res = {5, 5, 5, 5, 5, 5, 5, 5, 0, 4, 5, 5, 4, 4}; } basic_aggregation_test(res); } @@ -694,7 +740,7 @@ TEMPLATE_LIST_TEST_CASE( TEST_CASE( "Count aggregator: Basic aggregation", "[count-aggregator][basic-aggregation]") { - std::vector res = {8, 8, 3, 5, 2, 5, 10, 13, 10, 13}; + std::vector res = {8, 18, 28, 8, 18, 28, 3, 5, 2, 5, 10, 13, 10, 13}; basic_aggregation_test(res); } @@ -703,7 +749,7 @@ TEMPLATE_LIST_TEST_CASE( "[null-count-aggregator][basic-aggregation]", FixedTypesUnderTest) { typedef TestType T; - std::vector res = {0, 4, 0, 0, 2, 3, 0, 0, 3, 6}; + std::vector res = {0, 0, 0, 4, 14, 19, 0, 0, 2, 3, 0, 0, 3, 6}; basic_aggregation_test(res); } @@ -947,6 +993,24 @@ void basic_string_aggregation_test(std::vector expected_results) { std::vector validity_data = {0, 0, 1, 0, 1, 0, 1, 0, 1, 0}; SECTION("No bitmap") { + ByteVecValue unused(8); + auto tile_metadata_all_null = TileMetadata( + 10, + 10, + &var_data[offsets[0]], + offsets[1] - offsets[0], + &var_data[offsets[5]], + offsets[6] - offsets[5], + unused.data()); + auto tile_metadata = TileMetadata( + 10, + 5, + &var_data[offsets[0]], + offsets[1] - offsets[0], + &var_data[offsets[5]], + offsets[6] - offsets[5], + unused.data()); + if (aggregator.has_value()) { // Regular attribute. AggregateBuffer input_data{ @@ -954,6 +1018,14 @@ void basic_string_aggregation_test(std::vector expected_results) { aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); check_value_string(fixed_data, value_size, value, expected_results[0]); + + aggregator->aggregate_tile_with_frag_md(tile_metadata_all_null); + aggregator->copy_to_user_buffer("Agg", buffers); + check_value_string(fixed_data, value_size, value, expected_results[1]); + + aggregator->aggregate_tile_with_frag_md(tile_metadata); + aggregator->copy_to_user_buffer("Agg", buffers); + check_value_string(fixed_data, value_size, value, expected_results[2]); } // Nullable attribute. @@ -968,8 +1040,16 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data2); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value_string(fixed_data2, value_size2, value2, expected_results[1]); + check_value_string(fixed_data2, value_size2, value2, expected_results[3]); check_validity(validity, 1); + + aggregator_nullable.aggregate_tile_with_frag_md(tile_metadata_all_null); + aggregator_nullable.copy_to_user_buffer("Agg2", buffers); + check_value_string(fixed_data2, value_size2, value2, expected_results[4]); + + aggregator_nullable.aggregate_tile_with_frag_md(tile_metadata); + aggregator_nullable.copy_to_user_buffer("Agg2", buffers); + check_value_string(fixed_data2, value_size2, value2, expected_results[5]); } SECTION("Regular bitmap") { @@ -987,7 +1067,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); - check_value_string(fixed_data, value_size, value, expected_results[2]); + check_value_string(fixed_data, value_size, value, expected_results[6]); AggregateBuffer input_data2{ 0, @@ -1000,7 +1080,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data2); aggregator->copy_to_user_buffer("Agg", buffers); - check_value_string(fixed_data, value_size, value, expected_results[3]); + check_value_string(fixed_data, value_size, value, expected_results[7]); } // Nullable attribute. @@ -1015,7 +1095,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data3); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value_string(fixed_data2, value_size2, value2, expected_results[4]); + check_value_string(fixed_data2, value_size2, value2, expected_results[8]); check_validity(validity, 0); AggregateBuffer input_data4{ @@ -1029,7 +1109,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data4); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value_string(fixed_data2, value_size2, value2, expected_results[5]); + check_value_string(fixed_data2, value_size2, value2, expected_results[9]); check_validity(validity, 1); } @@ -1048,7 +1128,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data); aggregator->copy_to_user_buffer("Agg", buffers); - check_value_string(fixed_data, value_size, value, expected_results[6]); + check_value_string(fixed_data, value_size, value, expected_results[10]); AggregateBuffer input_data2{ 0, @@ -1061,7 +1141,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator->aggregate_data(input_data2); aggregator->copy_to_user_buffer("Agg", buffers); - check_value_string(fixed_data, value_size, value, expected_results[7]); + check_value_string(fixed_data, value_size, value, expected_results[11]); } // Nullable attribute. @@ -1076,7 +1156,7 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data3); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value_string(fixed_data2, value_size2, value2, expected_results[8]); + check_value_string(fixed_data2, value_size2, value2, expected_results[12]); check_validity(validity, 1); AggregateBuffer input_data4{ @@ -1090,12 +1170,12 @@ void basic_string_aggregation_test(std::vector expected_results) { 1}; aggregator_nullable.aggregate_data(input_data4); aggregator_nullable.copy_to_user_buffer("Agg2", buffers); - check_value_string(fixed_data2, value_size2, value2, expected_results[9]); + check_value_string(fixed_data2, value_size2, value2, expected_results[13]); check_validity(validity, 1); } } -typedef tuple, MinAggregator> +typedef tuple, MaxAggregator> MinMaxAggUnderTest; TEMPLATE_LIST_TEST_CASE( "Min max aggregator: Basic string aggregation", @@ -1103,9 +1183,36 @@ TEMPLATE_LIST_TEST_CASE( MinMaxAggUnderTest) { typedef TestType AGGREGATOR; std::vector res = { - "1", "2222", "2222", "11", "", "2222", "1", "1", "2222", "2222"}; + "1", + "1", + "1", + "2222", + "2222", + "11", + "2222", + "11", + "", + "2222", + "1", + "1", + "2222", + "2222"}; if (std::is_same>::value) { - res = {"5555", "555", "5555", "5555", "", "4", "5555", "5555", "4", "4"}; + res = { + "5555", + "5555", + "5555", + "555", + "555", + "5555", + "5555", + "5555", + "", + "4", + "5555", + "5555", + "4", + "4"}; } basic_string_aggregation_test(res); @@ -1114,7 +1221,7 @@ TEMPLATE_LIST_TEST_CASE( TEST_CASE( "NullCount aggregator: Basic string aggregation", "[null-count-aggregator][basic-string-aggregation]") { - std::vector res = {0, 4, 0, 0, 2, 3, 0, 0, 3, 6}; + std::vector res = {0, 0, 0, 4, 14, 19, 0, 0, 2, 3, 0, 0, 3, 6}; basic_string_aggregation_test(res); } diff --git a/tiledb/sm/query/readers/aggregators/tile_metadata.h b/tiledb/sm/query/readers/aggregators/tile_metadata.h new file mode 100644 index 000000000000..b239fd3b865d --- /dev/null +++ b/tiledb/sm/query/readers/aggregators/tile_metadata.h @@ -0,0 +1,161 @@ +/** + * @file tile_metadata.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class TileMetadata. + */ + +#ifndef TILEDB_TILE_METADATA_H +#define TILEDB_TILE_METADATA_H + +#include "tiledb/common/common.h" + +namespace tiledb::sm { + +class TileMetadataStatusException : public StatusException { + public: + explicit TileMetadataStatusException(const std::string& message) + : StatusException("TileMetadata", message) { + } +}; + +class TileMetadata { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor. + * + * @param count Number of cells for this tile. + * @param null_count Number of null values for this tile. + * @param min Pointer to the min data. + * @param min_size Size of the min data. + * @param max Pointer to the max data. + * @param max_size Size of the max data. + * @param sum Pointer to the sum data. + */ + TileMetadata( + const uint64_t count, + const uint64_t null_count, + const void* min, + const storage_size_t min_size, + const void* max, + const storage_size_t max_size, + const void* sum) + : count_(count) + , null_count_(null_count) + , min_(min) + , min_size_(min_size) + , max_(max) + , max_size_(max_size) + , sum_(sum) { + } + + DISABLE_COPY_AND_COPY_ASSIGN(TileMetadata); + DISABLE_MOVE_AND_MOVE_ASSIGN(TileMetadata); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Returns the count. */ + inline uint64_t count() const { + return count_; + } + + /** Returns the null count. */ + inline uint64_t null_count() const { + return null_count_; + } + + /** Returns the min as a specific type. */ + template + T min_as() const { + if constexpr (std::is_same_v) { + return {static_cast(min_), min_size_}; + } else { + if (min_size_ != sizeof(T)) { + throw TileMetadataStatusException("Unexpected min size."); + } + + return *static_cast(min_); + } + } + + /** Returns the max as a specific type. */ + template + T max_as() const { + if constexpr (std::is_same_v) { + return {static_cast(max_), max_size_}; + } else { + if (max_size_ != sizeof(T)) { + throw TileMetadataStatusException("Unexpected max size."); + } + + return *static_cast(max_); + } + } + + /** Returns the sum as a specific type. */ + template + T sum_as() const { + return *static_cast(sum_); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Count of cells. */ + const uint64_t count_; + + /** Null count. */ + const uint64_t null_count_; + + /** Min value. */ + const void* min_; + + /** Min size. */ + const storage_size_t min_size_; + + /** Max value. */ + const void* max_; + + /** Max size. */ + const storage_size_t max_size_; + + /** Sum value. */ + const void* sum_; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_TILE_METADATA_H From f04b453a032d0703e8ab3d07bdaa899e85bf9478 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 27 Oct 2023 21:02:40 +0200 Subject: [PATCH 036/456] Fragment metadata: adding get_tile_metadata. (#4466) This adds the ability to get the tile metadata to later be used by the aggregates pipeline. --- TYPE: IMPROVEMENT DESC: Fragment metadata: adding get_tile_metadata. --- test/src/unit-tile-metadata.cc | 307 ++++++++++++++++++++++-- tiledb/sm/fragment/fragment_metadata.cc | 69 +++++- tiledb/sm/fragment/fragment_metadata.h | 14 ++ 3 files changed, 373 insertions(+), 17 deletions(-) diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index 5464cffb1896..f5853379ce0d 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -35,6 +35,7 @@ #include "test/support/src/helpers.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/query/readers/aggregators/tile_metadata.h" #include "tiledb/sm/tile/tile_metadata_generator.h" using namespace tiledb; @@ -296,6 +297,7 @@ struct CPPFixedTileMetadataFx { bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { std::vector names{"d"}; + frag_meta[f]->load_rtree(enc_key); frag_meta[f]->load_tile_min_values(enc_key, names); frag_meta[f]->load_tile_max_values(enc_key, names); frag_meta[f]->load_tile_sum_values(enc_key, names); @@ -347,6 +349,12 @@ struct CPPFixedTileMetadataFx { // Validate sum. auto sum = frag_meta[f]->get_tile_sum("d", tile_idx); CHECK(*(int64_t*)sum == correct_sum); + + // Validate the tile metadata structure. + auto full_tile_data = frag_meta[f]->get_tile_metadata("d", tile_idx); + CHECK(correct_min == full_tile_data.min_as()); + CHECK(correct_max == full_tile_data.max_as()); + CHECK(correct_sum == full_tile_data.sum_as()); } } } @@ -483,11 +491,12 @@ struct CPPFixedTileMetadataFx { // For strings, the index is stored in a signed value, switch to // the index to unsigned. - int64_t idx = (int64_t)correct_tile_mins_[f][tile_idx] - - (int64_t)std::numeric_limits::min(); + int64_t min_idx = (int64_t)correct_tile_mins_[f][tile_idx] - + (int64_t)std::numeric_limits::min(); CHECK( 0 == - strncmp(min.data(), string_ascii_[idx].c_str(), cell_val_num)); + strncmp( + min.data(), string_ascii_[min_idx].c_str(), cell_val_num)); // Validate max. const auto max = @@ -496,11 +505,12 @@ struct CPPFixedTileMetadataFx { // For strings, the index is stored in a signed value, switch to // the index to unsigned. - idx = (int64_t)correct_tile_maxs_[f][tile_idx] - - (int64_t)std::numeric_limits::min(); + int64_t max_idx = (int64_t)correct_tile_maxs_[f][tile_idx] - + (int64_t)std::numeric_limits::min(); CHECK( 0 == - strncmp(max.data(), string_ascii_[idx].c_str(), cell_val_num)); + strncmp( + max.data(), string_ascii_[max_idx].c_str(), cell_val_num)); // Validate no sum. CHECK_THROWS_WITH( @@ -508,6 +518,16 @@ struct CPPFixedTileMetadataFx { "FragmentMetadata: Trying to access tile sum metadata that's " "not " "present"); + + // Validate the tile metadata structure. + auto full_tile_data = + frag_meta[f]->get_tile_metadata("a", tile_idx); + CHECK( + string_ascii_[min_idx] == + full_tile_data.min_as()); + CHECK( + string_ascii_[max_idx] == + full_tile_data.max_as()); } } else { (void)cell_val_num; @@ -528,13 +548,29 @@ struct CPPFixedTileMetadataFx { memcmp( &max, &correct_tile_maxs_[f][tile_idx], sizeof(TestType))); + // Validate the tile metadata structure. + auto full_tile_data = + frag_meta[f]->get_tile_metadata("a", tile_idx); + CHECK( + correct_tile_mins_[f][tile_idx] == + full_tile_data.min_as()); + CHECK( + correct_tile_maxs_[f][tile_idx] == + full_tile_data.max_as()); + if constexpr (!std::is_same::value) { // Validate sum. auto sum = frag_meta[f]->get_tile_sum("a", tile_idx); if constexpr (std::is_integral_v) { CHECK(*(int64_t*)sum == correct_tile_sums_int_[f][tile_idx]); + CHECK( + correct_tile_sums_int_[f][tile_idx] == + full_tile_data.sum_as()); } else { CHECK(*(double*)sum == correct_tile_sums_double_[f][tile_idx]); + CHECK( + correct_tile_sums_double_[f][tile_idx] == + full_tile_data.sum_as()); } } } @@ -555,6 +591,20 @@ struct CPPFixedTileMetadataFx { } } + if constexpr (!std::is_same::value) { + // Validate the full tile data structure for null count + for (uint64_t tile_idx = 0; tile_idx < num_tiles_; tile_idx++) { + auto full_tile_data = frag_meta[f]->get_tile_metadata("a", tile_idx); + if (nullable) { + CHECK( + full_tile_data.null_count() == + correct_tile_null_counts_[f][tile_idx]); + } else { + CHECK(full_tile_data.null_count() == 0); + } + } + } + // Close array. rc = tiledb_array_close(ctx, array); CHECK(rc == TILEDB_OK); @@ -819,6 +869,7 @@ struct CPPVarTileMetadataFx { bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { std::vector names{"d"}; + frag_meta[f]->load_rtree(enc_key); frag_meta[f]->load_tile_min_values(enc_key, names); frag_meta[f]->load_tile_max_values(enc_key, names); frag_meta[f]->load_tile_sum_values(enc_key, names); @@ -870,6 +921,12 @@ struct CPPVarTileMetadataFx { // Validate sum. auto sum = frag_meta[f]->get_tile_sum("d", tile_idx); CHECK(*(int64_t*)sum == correct_sum); + + // Validate the tile metadata structure. + auto full_tile_data = frag_meta[f]->get_tile_metadata("d", tile_idx); + CHECK(correct_min == full_tile_data.min_as()); + CHECK(correct_max == full_tile_data.max_as()); + CHECK(correct_sum == full_tile_data.sum_as()); } } } @@ -929,26 +986,35 @@ struct CPPVarTileMetadataFx { // Validate min. const auto min = frag_meta[f]->get_tile_min_as("a", tile_idx); - int idx = correct_tile_mins_[f][tile_idx]; - CHECK(min.size() == strings_[idx].size()); + int min_idx = correct_tile_mins_[f][tile_idx]; + CHECK(min.size() == strings_[min_idx].size()); CHECK( - 0 == - strncmp(min.data(), strings_[idx].c_str(), strings_[idx].size())); + 0 == strncmp( + min.data(), + strings_[min_idx].c_str(), + strings_[min_idx].size())); // Validate max. const auto max = frag_meta[f]->get_tile_max_as("a", tile_idx); - idx = correct_tile_maxs_[f][tile_idx]; - CHECK(max.size() == strings_[idx].size()); + int max_idx = correct_tile_maxs_[f][tile_idx]; + CHECK(max.size() == strings_[max_idx].size()); CHECK( - 0 == - strncmp(max.data(), strings_[idx].c_str(), strings_[idx].size())); + 0 == strncmp( + max.data(), + strings_[max_idx].c_str(), + strings_[max_idx].size())); // Validate no sum. CHECK_THROWS_WITH( frag_meta[f]->get_tile_sum("a", tile_idx), "FragmentMetadata: Trying to access tile sum metadata that's not " "present"); + + // Validate the tile metadata structure. + auto full_tile_data = frag_meta[f]->get_tile_metadata("a", tile_idx); + CHECK(strings_[min_idx] == full_tile_data.min_as()); + CHECK(strings_[max_idx] == full_tile_data.max_as()); } } @@ -1185,6 +1251,15 @@ struct CPPFixedTileMetadataPartialFx { // Validate sum. auto sum = frag_meta[0]->get_tile_sum("a", tile_idx); CHECK(*(double*)sum - correct_tile_sums[tile_idx] < 0.0001); + + // Validate the tile metadata structure. + auto full_tile_data = frag_meta[0]->get_tile_metadata("a", tile_idx); + CHECK(correct_tile_mins[tile_idx] == full_tile_data.min_as()); + CHECK(correct_tile_maxs[tile_idx] == full_tile_data.max_as()); + CHECK( + std::abs( + correct_tile_sums[tile_idx] - full_tile_data.sum_as()) < + 0.0001); } // Close array. @@ -1244,7 +1319,7 @@ struct CPPVarTileMetadataPartialFx { tiledb_domain_add_dimension(ctx, domain, d1); tiledb_domain_add_dimension(ctx, domain, d2); - // Create a single attribute "a" so each (i,j) cell can store an integer + // Create a single attribute "a" so each (i,j) cell can store a string tiledb_attribute_t* a; tiledb_attribute_alloc(ctx, "a", TILEDB_STRING_ASCII, &a); tiledb_attribute_set_cell_val_num(ctx, a, TILEDB_VAR_NUM); @@ -1350,6 +1425,15 @@ struct CPPVarTileMetadataPartialFx { CHECK( 0 == memcmp(max.data(), correct_tile_maxs[tile_idx].data(), max.size())); + + // Validate the tile metadata structure. + auto full_tile_data = frag_meta[0]->get_tile_metadata("a", tile_idx); + CHECK( + correct_tile_mins[tile_idx] == + full_tile_data.min_as()); + CHECK( + correct_tile_maxs[tile_idx] == + full_tile_data.max_as()); } // Close array. @@ -1375,4 +1459,197 @@ TEST_CASE_METHOD( create_array(); write_fragment(); check_metadata(); +} + +struct CPPTileMetadataStringDimFx { + CPPTileMetadataStringDimFx() + : vfs_(ctx_) { + if (vfs_.is_dir(ARRAY_NAME)) + vfs_.remove_dir(ARRAY_NAME); + } + + ~CPPTileMetadataStringDimFx() { + if (vfs_.is_dir(ARRAY_NAME)) + vfs_.remove_dir(ARRAY_NAME); + } + + void create_array() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // The array will be two string dimension "d1" and "d2". + tiledb_dimension_t* d1; + tiledb_dimension_alloc(ctx, "d1", TILEDB_STRING_ASCII, 0, 0, &d1); + tiledb_dimension_t* d2; + tiledb_dimension_alloc(ctx, "d2", TILEDB_STRING_ASCII, 0, 0, &d2); + + // Create domain + tiledb_domain_t* domain; + tiledb_domain_alloc(ctx, &domain); + tiledb_domain_add_dimension(ctx, domain, d1); + tiledb_domain_add_dimension(ctx, domain, d2); + + // Create a single attribute "a" so each (i,j) cell can store a double + tiledb_attribute_t* a; + tiledb_attribute_alloc(ctx, "a", TILEDB_FLOAT64, &a); + + // Create array schema + tiledb_array_schema_t* array_schema; + tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema); + tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_domain(ctx, array_schema, domain); + tiledb_array_schema_add_attribute(ctx, array_schema, a); + + // Create array + tiledb_array_create(ctx, ARRAY_NAME, array_schema); + + // Clean up + tiledb_attribute_free(&a); + tiledb_dimension_free(&d1); + tiledb_dimension_free(&d2); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&array_schema); + tiledb_ctx_free(&ctx); + } + + void write_fragment() { + // Write to the array. + auto array = tiledb::Array(ctx_, ARRAY_NAME, TILEDB_WRITE); + auto query = tiledb::Query(ctx_, array, TILEDB_WRITE); + + std::string d1 = "abbcccdddd"; + std::vector d1_offsets{0, 1, 3, 6}; + std::string d2 = "abcd"; + std::vector d2_offsets{0, 1, 2, 3}; + std::vector a{4, 5, 6, 7}; + query.set_layout(TILEDB_UNORDERED); + query.set_data_buffer("d1", d1).set_offsets_buffer("d1", d1_offsets); + query.set_data_buffer("d2", d2).set_offsets_buffer("d2", d2_offsets); + query.set_data_buffer("a", a); + + query.submit(); + query.finalize(); + array.close(); + } + + void check_metadata() { + // Open array. + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx, ARRAY_NAME, &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx, array, TILEDB_READ); + CHECK(rc == TILEDB_OK); + + // Load fragment metadata. + auto frag_meta = array->array_->fragment_metadata(); + auto& enc_key = array->array_->get_encryption_key(); + frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); + + // Do fragment metadata first. + { + // Validate mins. + auto& min = frag_meta[0]->get_min("a"); + CHECK(min.size() == sizeof(double)); + CHECK(*static_cast(static_cast(min.data())) == 4); + + CHECK_THROWS_WITH( + frag_meta[0]->get_min("d1"), + "FragmentMetadata: Trying to access fragment min metadata that's " + "not present"); + + CHECK_THROWS_WITH( + frag_meta[0]->get_min("d2"), + "FragmentMetadata: Trying to access fragment min metadata that's " + "not present"); + + // Validate maxs. + auto& max = frag_meta[0]->get_max("a"); + CHECK(max.size() == sizeof(double)); + CHECK(*static_cast(static_cast(max.data())) == 7); + + CHECK_THROWS_WITH( + frag_meta[0]->get_max("d1"), + "FragmentMetadata: Trying to access fragment max metadata that's " + "not present"); + + CHECK_THROWS_WITH( + frag_meta[0]->get_max("d2"), + "FragmentMetadata: Trying to access fragment max metadata that's " + "not present"); + } + + // Load metadata. + std::vector names{"a", "d1", "d2"}; + frag_meta[0]->load_rtree(enc_key); + frag_meta[0]->load_tile_min_values(enc_key, names); + frag_meta[0]->load_tile_max_values(enc_key, names); + frag_meta[0]->load_tile_sum_values(enc_key, names); + frag_meta[0]->load_tile_null_count_values(enc_key, names); + + // Validate min. + CHECK(frag_meta[0]->get_tile_min_as("a", 0) == 4); + CHECK_THROWS_WITH( + frag_meta[0]->get_tile_min_as("d1", 0), + "FragmentMetadata: Trying to access tile min metadata that's not " + "present"); + CHECK_THROWS_WITH( + frag_meta[0]->get_tile_min_as("d2", 0), + "FragmentMetadata: Trying to access tile min metadata that's not " + "present"); + + // Validate max. + CHECK(frag_meta[0]->get_tile_max_as("a", 0) == 7); + CHECK_THROWS_WITH( + frag_meta[0]->get_tile_max_as("d1", 0), + "FragmentMetadata: Trying to access tile max metadata that's not " + "present"); + CHECK_THROWS_WITH( + frag_meta[0]->get_tile_max_as("d2", 0), + "FragmentMetadata: Trying to access tile max metadata that's not " + "present"); + + // Validate sum. + CHECK(*(double*)frag_meta[0]->get_tile_sum("a", 0) == 22); + + // Validate the tile metadata structure. + auto full_tile_data_a = frag_meta[0]->get_tile_metadata("a", 0); + CHECK(4 == full_tile_data_a.min_as()); + CHECK(7 == full_tile_data_a.max_as()); + CHECK(22 == full_tile_data_a.sum_as()); + + auto full_tile_data_d1 = frag_meta[0]->get_tile_metadata("d1", 0); + CHECK("a" == full_tile_data_d1.min_as()); + CHECK("dddd" == full_tile_data_d1.max_as()); + + auto full_tile_data_d2 = frag_meta[0]->get_tile_metadata("d2", 0); + CHECK("a" == full_tile_data_d2.min_as()); + CHECK("d" == full_tile_data_d2.max_as()); + + // Close array. + rc = tiledb_array_close(ctx, array); + CHECK(rc == TILEDB_OK); + + // Clean up. + tiledb_array_free(&array); + tiledb_ctx_free(&ctx); + } + + const char* ARRAY_NAME = "tile_metadata_unit_array"; + const uint64_t tile_extent_ = 4; + tiledb::Context ctx_; + tiledb::VFS vfs_; +}; + +TEST_CASE_METHOD( + CPPTileMetadataStringDimFx, + "TileMetadata: string dims", + "[tile-metadata][string-dims]") { + // Create the array. + create_array(); + write_fragment(); + check_metadata(); } \ No newline at end of file diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 85980985d45c..bb81f84c7595 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -46,6 +46,7 @@ #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/query/readers/aggregators/tile_metadata.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" @@ -1707,7 +1708,11 @@ T FragmentMetadata::get_tile_min_as( auto size = array_schema_->cell_size(name); const void* min = &tile_min_buffer_[idx][tile_idx * size]; - return *static_cast(min); + if constexpr (std::is_same_v) { + return min; + } else { + return *static_cast(min); + } } template <> @@ -1787,7 +1792,11 @@ T FragmentMetadata::get_tile_max_as( auto size = array_schema_->cell_size(name); const void* max = &tile_max_buffer_[idx][tile_idx * size]; - return *static_cast(max); + if constexpr (std::is_same_v) { + return max; + } else { + return *static_cast(max); + } } template <> @@ -1960,6 +1969,62 @@ uint64_t FragmentMetadata::get_null_count(const std::string& name) { return fragment_null_counts_[idx]; } +TileMetadata FragmentMetadata::get_tile_metadata( + const std::string& name, const uint64_t tile_idx) const { + auto var_size = array_schema_->var_size(name); + auto is_dim = array_schema_->is_dim(name); + auto count = cell_num(tile_idx); + + if (name == constants::count_of_rows) { + return {count, 0, nullptr, 0, nullptr, 0, nullptr}; + } + + uint64_t null_count = 0; + if (array_schema_->is_nullable(name)) { + null_count = get_tile_null_count(name, tile_idx); + } + + unsigned dim_idx = 0; + const NDRange* mbr = nullptr; + if (is_dim) { + throw_if_not_ok( + array_schema_->domain().get_dimension_index(name, &dim_idx)); + mbr = &rtree_.leaf(tile_idx); + } + + if (var_size) { + std::string_view min = + is_dim ? mbr->at(dim_idx).start_str() : + get_tile_min_as(name, tile_idx); + std::string_view max = + is_dim ? mbr->at(dim_idx).end_str() : + get_tile_max_as(name, tile_idx); + return { + count, + null_count, + min.data(), + min.size(), + max.data(), + max.size(), + nullptr}; + } else { + auto cell_size = array_schema_->cell_size(name); + const void* min = is_dim ? mbr->at(dim_idx).start_fixed() : + get_tile_min_as(name, tile_idx); + const void* max = is_dim ? mbr->at(dim_idx).end_fixed() : + get_tile_max_as(name, tile_idx); + + const auto type = array_schema_->type(name); + const auto cell_val_num = array_schema_->cell_val_num(name); + const void* sum = nullptr; + if (TileMetadataGenerator::has_sum_metadata(type, false, cell_val_num)) { + sum = get_tile_sum(name, tile_idx); + } + + return {count, null_count, min, cell_size, max, cell_size, sum}; + } +} + void FragmentMetadata::set_processed_conditions( std::vector& processed_conditions) { processed_conditions_ = processed_conditions; diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 504914c16065..3b4e822b888f 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -59,6 +59,7 @@ namespace sm { class ArraySchema; class Buffer; class EncryptionKey; +class TileMetadata; class MemoryTracker; /** Stores the metadata structures of a fragment. */ @@ -289,6 +290,10 @@ class FragmentMetadata { return has_delete_meta_; } + inline bool has_tile_metadata() { + return version_ >= constants::tile_metadata_min_version; + } + /** Returns the sizes of each attribute file. */ inline const std::vector& file_sizes() const { return file_sizes_; @@ -899,6 +904,15 @@ class FragmentMetadata { */ uint64_t get_null_count(const std::string& name); + /** + * Returns the tile metadata for a tile. + * + * @param name Name of the attribute to get the data for. + * @param tile_idx Tile index. + */ + TileMetadata get_tile_metadata( + const std::string& name, const uint64_t tile_idx) const; + /** * Set the processed conditions. The processed conditions is the list * of delete/update conditions that have already been applied for this From 9921fc66808fc2c89a2a05dd4f43a8cd457b0cbf Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 30 Oct 2023 15:32:33 +0200 Subject: [PATCH 037/456] Improve the MinGW builds and simplify bundling the static library dependencies. (#4448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [SC-35901](https://app.shortcut.com/tiledb-inc/story/35901/copy-the-vcpkg-installed-directory-to-the-mingw-package) As a preparation towards stopping bundling our static library dependencies (which would lead to significant simplifications in the build system), this PR updates the MinGW builds (which supply the binaries the R API uses on Windows) to stop relying on that automatic bundling. Our approach becomes more blunt, just copying the contents of the build directory's `vcpkg_installed` directory, where vcpkg places all libraries we might need. Nothing (fingers crossed) should change on the side of the R API. These are the libraries that are bundled before and after this PR:
Before ``` -rw-r--r-- 0 root root 267222 Οκτ 19 16:08 ucrt64/lib/libaws-c-common.a -rw-r--r-- 0 root root 28638 Οκτ 19 16:08 ucrt64/lib/libaws-c-event-stream.a -rw-r--r-- 0 root root 47078 Οκτ 19 16:08 ucrt64/lib/libaws-checksums.a -rw-r--r-- 0 root root 1540524 Οκτ 19 16:08 ucrt64/lib/libaws-cpp-sdk-cognito-identity.a -rw-r--r-- 0 root root 2315096 Οκτ 19 16:08 ucrt64/lib/libaws-cpp-sdk-core.a -rw-r--r-- 0 root root 319782 Οκτ 19 16:08 ucrt64/lib/libaws-cpp-sdk-identity-management.a -rw-r--r-- 0 root root 5501212 Οκτ 19 16:08 ucrt64/lib/libaws-cpp-sdk-s3.a -rw-r--r-- 0 root root 676178 Οκτ 19 16:08 ucrt64/lib/libaws-cpp-sdk-sts.a -rw-r--r-- 0 root root 69652 Οκτ 19 16:08 ucrt64/lib/libbz2.a -rw-r--r-- 0 root root 452820 Οκτ 19 16:08 ucrt64/lib/libfmt.a -rw-r--r-- 0 root root 241756 Οκτ 19 16:08 ucrt64/lib/liblibmagic.a -rw-r--r-- 0 root root 121288 Οκτ 19 16:08 ucrt64/lib/liblz4.a -rw-r--r-- 0 root root 785406 Οκτ 19 16:08 ucrt64/lib/libpcre2-8.a -rw-r--r-- 0 root root 3934 Οκτ 19 16:08 ucrt64/lib/libpcre2-posix.a -rw-r--r-- 0 root root 33576 Οκτ 19 16:08 ucrt64/lib/libsharpyuv.a -rw-r--r-- 0 root root 1404130 Οκτ 19 16:08 ucrt64/lib/libspdlog.a -rw-r--r-- 0 root root 340224 Οκτ 19 16:08 ucrt64/lib/libtiledb.dll.a -rw-r--r-- 0 root root 41120496 Οκτ 19 16:08 ucrt64/lib/libtiledbstatic.a -rw-r--r-- 0 root root 881632 Οκτ 19 16:08 ucrt64/lib/libwebp.a -rw-r--r-- 0 root root 485960 Οκτ 19 16:08 ucrt64/lib/libwebpdecoder.a -rw-r--r-- 0 root root 14684 Οκτ 19 16:08 ucrt64/lib/libwebpdemux.a -rw-r--r-- 0 root root 46538 Οκτ 19 16:08 ucrt64/lib/libwebpmux.a -rw-r--r-- 0 root root 102320 Οκτ 19 16:08 ucrt64/lib/libz.a -rw-r--r-- 0 root root 814174 Οκτ 19 16:08 ucrt64/lib/libzstd.a ```
After ``` -rw-r--r-- 0 root root 321018 Οκτ 20 14:33 ucrt64/lib/libaws-c-auth.a -rw-r--r-- 0 root root 78040 Οκτ 20 14:33 ucrt64/lib/libaws-c-cal.a -rw-r--r-- 0 root root 356172 Οκτ 20 14:33 ucrt64/lib/libaws-c-common.a -rw-r--r-- 0 root root 8570 Οκτ 20 14:33 ucrt64/lib/libaws-c-compression.a -rw-r--r-- 0 root root 121990 Οκτ 20 14:33 ucrt64/lib/libaws-c-event-stream.a -rw-r--r-- 0 root root 539052 Οκτ 20 14:33 ucrt64/lib/libaws-c-http.a -rw-r--r-- 0 root root 417204 Οκτ 20 14:33 ucrt64/lib/libaws-c-io.a -rw-r--r-- 0 root root 415280 Οκτ 20 14:33 ucrt64/lib/libaws-c-mqtt.a -rw-r--r-- 0 root root 242642 Οκτ 20 14:33 ucrt64/lib/libaws-c-s3.a -rw-r--r-- 0 root root 130632 Οκτ 20 14:33 ucrt64/lib/libaws-c-sdkutils.a -rw-r--r-- 0 root root 39962 Οκτ 20 14:33 ucrt64/lib/libaws-checksums.a -rw-r--r-- 0 root root 448814 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-access-management.a -rw-r--r-- 0 root root 849090 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-cognito-identity.a -rw-r--r-- 0 root root 3630884 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-core.a -rw-r--r-- 0 root root 4055126 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-iam.a -rw-r--r-- 0 root root 346146 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-identity-management.a -rw-r--r-- 0 root root 3799166 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-s3.a -rw-r--r-- 0 root root 495284 Οκτ 20 14:33 ucrt64/lib/libaws-cpp-sdk-sts.a -rw-r--r-- 0 root root 1237918 Οκτ 20 14:33 ucrt64/lib/libaws-crt-cpp.a -rw-r--r-- 0 root root 98636 Οκτ 20 14:33 ucrt64/lib/libbz2.a -rw-r--r-- 0 root root 452820 Οκτ 20 14:33 ucrt64/lib/libfmt.a -rw-r--r-- 0 root root 241756 Οκτ 20 14:33 ucrt64/lib/liblibmagic.a -rw-r--r-- 0 root root 212328 Οκτ 20 14:33 ucrt64/lib/liblz4.a -rw-r--r-- 0 root root 304626 Οκτ 20 14:33 ucrt64/lib/liblzma.a -rw-r--r-- 0 root root 719352 Οκτ 20 14:33 ucrt64/lib/libpcre2-16.a -rw-r--r-- 0 root root 689066 Οκτ 20 14:33 ucrt64/lib/libpcre2-32.a -rw-r--r-- 0 root root 785406 Οκτ 20 14:33 ucrt64/lib/libpcre2-8.a -rw-r--r-- 0 root root 3934 Οκτ 20 14:33 ucrt64/lib/libpcre2-posix.a -rw-r--r-- 0 root root 33576 Οκτ 20 14:33 ucrt64/lib/libsharpyuv.a -rw-r--r-- 0 root root 1404130 Οκτ 20 14:33 ucrt64/lib/libspdlog.a -rw-r--r-- 0 root root 340224 Οκτ 20 14:33 ucrt64/lib/libtiledb.dll.a -rw-r--r-- 0 root root 41075680 Οκτ 20 14:33 ucrt64/lib/libtiledbstatic.a -rw-r--r-- 0 root root 881632 Οκτ 20 14:33 ucrt64/lib/libwebp.a -rw-r--r-- 0 root root 485960 Οκτ 20 14:33 ucrt64/lib/libwebpdecoder.a -rw-r--r-- 0 root root 14684 Οκτ 20 14:33 ucrt64/lib/libwebpdemux.a -rw-r--r-- 0 root root 46538 Οκτ 20 14:33 ucrt64/lib/libwebpmux.a -rw-r--r-- 0 root root 123968 Οκτ 20 14:33 ucrt64/lib/libzlib.a -rw-r--r-- 0 root root 1227472 Οκτ 20 14:33 ucrt64/lib/libzstd.a ```
Notice that no libraries have stoppped from being bundled. Because we now bundle libraries we don't actually need, the package's size increased: | |Uncompressed|Compressed| |-|------------|----------| |Before|73.1MB|9.5MB| |After|112.5MB|13.1MB| Also the package now includes some useless files like around one thousand headers for the AWS SDK, but [they seem to be excluded from being uploaded](https://github.com/TileDB-Inc/rwinlib-tiledb/blob/aab71de1dd9d1c3b25ed2b1c6c6274c28e5e4f7f/update.sh#L33). I also made some improvement that should speed-up the MinGW builds (build dependencies in Release mode only, cache them and disable tests). --- TYPE: IMPROVEMENT DESC: Improve building the Windows binaries for the R API. --- .github/workflows/build-rtools40.yml | 8 + .github/workflows/mingw-w64-tiledb/PKGBUILD | 16 +- .../system-ports/aws-sdk-cpp/portfile.cmake | 1 - .../system-ports/aws-sdk-cpp/vcpkg.json | 837 ------------------ .../system-ports/bzip2/portfile.cmake | 1 - .../system-ports/bzip2/vcpkg.json | 9 - .../system-ports/curl/portfile.cmake | 1 - .../system-ports/curl/vcpkg.json | 73 -- .../system-ports/lz4/portfile.cmake | 1 - .../system-ports/lz4/vcpkg.json | 4 - .../system-ports/zlib/portfile.cmake | 1 - .../system-ports/zlib/vcpkg.json | 4 - .../system-ports/zstd/portfile.cmake | 1 - .../system-ports/zstd/vcpkg.json | 4 - CMakeLists.txt | 4 - cmake/Options/BuildOptions.cmake | 1 + cmake/TileDB-Superbuild.cmake | 1 + ports/aws-sdk-cpp/portfile.cmake | 1 + .../remove-checked-array-iterator.patch | 51 ++ ...c.cmake => x64-mingw-static-release.cmake} | 2 + ...c.cmake => x86-mingw-static-release.cmake} | 2 + 21 files changed, 72 insertions(+), 951 deletions(-) delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/vcpkg.json delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/bzip2/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/bzip2/vcpkg.json delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/curl/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/curl/vcpkg.json delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/lz4/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/lz4/vcpkg.json delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/zlib/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/zlib/vcpkg.json delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/zstd/portfile.cmake delete mode 100644 .github/workflows/mingw-w64-tiledb/system-ports/zstd/vcpkg.json create mode 100644 ports/aws-sdk-cpp/remove-checked-array-iterator.patch rename ports/triplets/{x64-mingw-static.cmake => x64-mingw-static-release.cmake} (84%) rename ports/triplets/{x86-mingw-static.cmake => x86-mingw-static-release.cmake} (84%) diff --git a/.github/workflows/build-rtools40.yml b/.github/workflows/build-rtools40.yml index 73c122e0ecba..12226ef73728 100644 --- a/.github/workflows/build-rtools40.yml +++ b/.github/workflows/build-rtools40.yml @@ -17,6 +17,13 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 1 + # Configure required environment variables for vcpkg to use + # GitHub's Action Cache + - uses: actions/github-script@v6 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install rtools40 if needed uses: r-windows/install-rtools@master - name: Building TileDB with Rtools40 @@ -27,6 +34,7 @@ jobs: env: TILEDB_HOME: ${{ github.workspace }} MINGW_ARCH: ${{ matrix.msystem }} + VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' shell: c:\rtools40\usr\bin\bash.exe --login {0} - name: "Upload binaries" uses: actions/upload-artifact@v3 diff --git a/.github/workflows/mingw-w64-tiledb/PKGBUILD b/.github/workflows/mingw-w64-tiledb/PKGBUILD index bc594b1209ec..8881b2bb86fc 100644 --- a/.github/workflows/mingw-w64-tiledb/PKGBUILD +++ b/.github/workflows/mingw-w64-tiledb/PKGBUILD @@ -7,13 +7,6 @@ pkgdesc="Storage management library for sparse and dense array data (mingw-w64)" arch=("any") url="https://tiledb.com/" license=("MIT") -depends=("${MINGW_PACKAGE_PREFIX}-lz4" - "${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" - "${MINGW_PACKAGE_PREFIX}-bzip2" - "${MINGW_PACKAGE_PREFIX}-curl" - "${MINGW_PACKAGE_PREFIX}-zlib" - "${MINGW_PACKAGE_PREFIX}-zstd" - ) makedepends=("${MINGW_PACKAGE_PREFIX}-cmake" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip") @@ -26,9 +19,9 @@ build() { if [ "$CARCH" == "i686" ]; then export CFLAGS="-mfpmath=sse -msse2" export CXXFLAGS="-mfpmath=sse -msse2" - vcpkg_triplet="x86-mingw-static" + vcpkg_triplet="x86-mingw-static-release" else - vcpkg_triplet="x64-mingw-static" + vcpkg_triplet="x64-mingw-static-release" fi @@ -39,11 +32,12 @@ build() { -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DTILEDB_STATIC=ON \ + -DTILEDB_INSTALL_STATIC_DEPS=OFF \ + -DTILEDB_TESTS=OFF \ -DTILEDB_S3=ON \ -DCOMPILER_SUPPORTS_AVX2=OFF \ -DTILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK=ON \ -DTILEDB_WERROR=OFF \ - -DVCPKG_OVERLAY_PORTS=../.github/workflows/mingw-w64-tiledb/system-ports \ -DVCPKG_TARGET_TRIPLET=${vcpkg_triplet} \ .. make @@ -60,6 +54,8 @@ build() { package() { cd ${source_dir}/build-${MINGW_CHOST} make DESTDIR="${pkgdir}" -C tiledb install + # Copy vcpkg_installed to the mingw package. + cp -r vcpkg_installed/${vcpkg_triplet}/. ${pkgdir}/${MINGW_ARCH} } check (){ diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/vcpkg.json deleted file mode 100644 index d7f820f0dd43..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/aws-sdk-cpp/vcpkg.json +++ /dev/null @@ -1,837 +0,0 @@ -{ - "name": "aws-sdk-cpp", - "version-string": "system", - "features": { - "access-management": { - "description": "C++ SDK for the AWS access-management service" - }, - "accessanalyzer": { - "description": "C++ SDK for the AWS accessanalyzer service" - }, - "acm": { - "description": "C++ SDK for the AWS acm service" - }, - "acm-pca": { - "description": "C++ SDK for the AWS acm-pca service" - }, - "alexaforbusiness": { - "description": "C++ SDK for the AWS alexaforbusiness service" - }, - "amp": { - "description": "C++ SDK for the AWS amp service" - }, - "amplify": { - "description": "C++ SDK for the AWS amplify service" - }, - "amplifybackend": { - "description": "C++ SDK for the AWS amplifybackend service" - }, - "apigateway": { - "description": "C++ SDK for the AWS apigateway service" - }, - "apigatewaymanagementapi": { - "description": "C++ SDK for the AWS apigatewaymanagementapi service" - }, - "apigatewayv2": { - "description": "C++ SDK for the AWS apigatewayv2 service" - }, - "appconfig": { - "description": "C++ SDK for the AWS appconfig service" - }, - "appflow": { - "description": "C++ SDK for the AWS appflow service" - }, - "appintegrations": { - "description": "C++ SDK for the AWS appintegrations service" - }, - "application-autoscaling": { - "description": "C++ SDK for the AWS application-autoscaling service" - }, - "application-insights": { - "description": "C++ SDK for the AWS application-insights service" - }, - "applicationcostprofiler": { - "description": "C++ SDK for the AWS applicationcostprofiler service" - }, - "appmesh": { - "description": "C++ SDK for the AWS appmesh service" - }, - "apprunner": { - "description": "C++ SDK for the AWS apprunner service" - }, - "appstream": { - "description": "C++ SDK for the AWS appstream service" - }, - "appsync": { - "description": "C++ SDK for the AWS appsync service" - }, - "athena": { - "description": "C++ SDK for the AWS athena service" - }, - "auditmanager": { - "description": "C++ SDK for the AWS auditmanager service" - }, - "autoscaling": { - "description": "C++ SDK for the AWS autoscaling service" - }, - "autoscaling-plans": { - "description": "C++ SDK for the AWS autoscaling-plans service" - }, - "awsmigrationhub": { - "description": "C++ SDK for the AWS AWSMigrationHub service" - }, - "awstransfer": { - "description": "C++ SDK for the AWS awstransfer service" - }, - "backup": { - "description": "C++ SDK for the AWS backup service" - }, - "batch": { - "description": "C++ SDK for the AWS batch service" - }, - "braket": { - "description": "C++ SDK for the AWS braket service" - }, - "budgets": { - "description": "C++ SDK for the AWS budgets service" - }, - "ce": { - "description": "C++ SDK for the AWS ce service" - }, - "chime": { - "description": "C++ SDK for the AWS chime service" - }, - "cloud9": { - "description": "C++ SDK for the AWS cloud9 service" - }, - "clouddirectory": { - "description": "C++ SDK for the AWS clouddirectory service" - }, - "cloudformation": { - "description": "C++ SDK for the AWS cloudformation service" - }, - "cloudfront": { - "description": "C++ SDK for the AWS cloudfront service" - }, - "cloudhsm": { - "description": "C++ SDK for the AWS cloudhsm service" - }, - "cloudhsmv2": { - "description": "C++ SDK for the AWS cloudhsmv2 service" - }, - "cloudsearch": { - "description": "C++ SDK for the AWS cloudsearch service" - }, - "cloudsearchdomain": { - "description": "C++ SDK for the AWS cloudsearchdomain service" - }, - "cloudtrail": { - "description": "C++ SDK for the AWS cloudtrail service" - }, - "codeartifact": { - "description": "C++ SDK for the AWS codeartifact service" - }, - "codebuild": { - "description": "C++ SDK for the AWS codebuild service" - }, - "codecommit": { - "description": "C++ SDK for the AWS codecommit service" - }, - "codedeploy": { - "description": "C++ SDK for the AWS codedeploy service" - }, - "codeguru-reviewer": { - "description": "C++ SDK for the AWS codeguru-reviewer service" - }, - "codeguruprofiler": { - "description": "C++ SDK for the AWS codeguruprofiler service" - }, - "codepipeline": { - "description": "C++ SDK for the AWS codepipeline service" - }, - "codestar": { - "description": "C++ SDK for the AWS codestar service" - }, - "codestar-connections": { - "description": "C++ SDK for the AWS codestar-connections service" - }, - "codestar-notifications": { - "description": "C++ SDK for the AWS codestar-notifications service" - }, - "cognito-identity": { - "description": "C++ SDK for the AWS cognito-identity service" - }, - "cognito-idp": { - "description": "C++ SDK for the AWS cognito-idp service" - }, - "cognito-sync": { - "description": "C++ SDK for the AWS cognito-sync service" - }, - "comprehend": { - "description": "C++ SDK for the AWS comprehend service" - }, - "comprehendmedical": { - "description": "C++ SDK for the AWS comprehendmedical service" - }, - "compute-optimizer": { - "description": "C++ SDK for the AWS compute-optimizer service" - }, - "config": { - "description": "C++ SDK for the AWS config service" - }, - "connect": { - "description": "C++ SDK for the AWS connect service" - }, - "connect-contact-lens": { - "description": "C++ SDK for the AWS connect-contact-lens service" - }, - "connectparticipant": { - "description": "C++ SDK for the AWS connectparticipant service" - }, - "cur": { - "description": "C++ SDK for the AWS cur service" - }, - "customer-profiles": { - "description": "C++ SDK for the AWS customer-profiles service" - }, - "databrew": { - "description": "C++ SDK for the AWS databrew service" - }, - "dataexchange": { - "description": "C++ SDK for the AWS dataexchange service" - }, - "datapipeline": { - "description": "C++ SDK for the AWS datapipeline service" - }, - "datasync": { - "description": "C++ SDK for the AWS datasync service" - }, - "dax": { - "description": "C++ SDK for the AWS dax service" - }, - "detective": { - "description": "C++ SDK for the AWS detective service" - }, - "devicefarm": { - "description": "C++ SDK for the AWS devicefarm service" - }, - "devops-guru": { - "description": "C++ SDK for the AWS devops-guru service" - }, - "directconnect": { - "description": "C++ SDK for the AWS directconnect service" - }, - "discovery": { - "description": "C++ SDK for the AWS discovery service" - }, - "dlm": { - "description": "C++ SDK for the AWS dlm service" - }, - "dms": { - "description": "C++ SDK for the AWS dms service" - }, - "docdb": { - "description": "C++ SDK for the AWS docdb service" - }, - "ds": { - "description": "C++ SDK for the AWS ds service" - }, - "dynamodb": { - "description": "C++ SDK for the AWS dynamodb service" - }, - "dynamodbstreams": { - "description": "C++ SDK for the AWS dynamodbstreams service" - }, - "ebs": { - "description": "C++ SDK for the AWS ebs service" - }, - "ec2": { - "description": "C++ SDK for the AWS ec2 service" - }, - "ec2-instance-connect": { - "description": "C++ SDK for the AWS ec2-instance-connect service" - }, - "ecr": { - "description": "C++ SDK for the AWS ecr service" - }, - "ecr-public": { - "description": "C++ SDK for the AWS ecr-public service" - }, - "ecs": { - "description": "C++ SDK for the AWS ecs service" - }, - "eks": { - "description": "C++ SDK for the AWS eks service" - }, - "elastic-inference": { - "description": "C++ SDK for the AWS elastic-inference service" - }, - "elasticache": { - "description": "C++ SDK for the AWS elasticache service" - }, - "elasticbeanstalk": { - "description": "C++ SDK for the AWS elasticbeanstalk service" - }, - "elasticfilesystem": { - "description": "C++ SDK for the AWS elasticfilesystem service" - }, - "elasticloadbalancing": { - "description": "C++ SDK for the AWS elasticloadbalancing service" - }, - "elasticloadbalancingv2": { - "description": "C++ SDK for the AWS elasticloadbalancingv2 service" - }, - "elasticmapreduce": { - "description": "C++ SDK for the AWS elasticmapreduce service" - }, - "elastictranscoder": { - "description": "C++ SDK for the AWS elastictranscoder service" - }, - "email": { - "description": "C++ SDK for the AWS email service" - }, - "emr-containers": { - "description": "C++ SDK for the AWS emr-containers service" - }, - "es": { - "description": "C++ SDK for the AWS es service" - }, - "eventbridge": { - "description": "C++ SDK for the AWS eventbridge service" - }, - "events": { - "description": "C++ SDK for the AWS events service" - }, - "finspace": { - "description": "C++ SDK for the AWS finspace service" - }, - "finspace-data": { - "description": "C++ SDK for the AWS finspace-data service" - }, - "firehose": { - "description": "C++ SDK for the AWS firehose service" - }, - "fis": { - "description": "C++ SDK for the AWS fis service" - }, - "fms": { - "description": "C++ SDK for the AWS fms service" - }, - "forecast": { - "description": "C++ SDK for the AWS forecast service" - }, - "forecastquery": { - "description": "C++ SDK for the AWS forecastquery service" - }, - "frauddetector": { - "description": "C++ SDK for the AWS frauddetector service" - }, - "fsx": { - "description": "C++ SDK for the AWS fsx service" - }, - "gamelift": { - "description": "C++ SDK for the AWS gamelift service" - }, - "glacier": { - "description": "C++ SDK for the AWS glacier service" - }, - "globalaccelerator": { - "description": "C++ SDK for the AWS globalaccelerator service" - }, - "glue": { - "description": "C++ SDK for the AWS glue service" - }, - "greengrass": { - "description": "C++ SDK for the AWS greengrass service" - }, - "greengrassv2": { - "description": "C++ SDK for the AWS greengrassv2 service" - }, - "groundstation": { - "description": "C++ SDK for the AWS groundstation service" - }, - "guardduty": { - "description": "C++ SDK for the AWS guardduty service" - }, - "health": { - "description": "C++ SDK for the AWS health service" - }, - "healthlake": { - "description": "C++ SDK for the AWS healthlake service" - }, - "honeycode": { - "description": "C++ SDK for the AWS honeycode service" - }, - "iam": { - "description": "C++ SDK for the AWS iam service" - }, - "identity-management": { - "description": "C++ SDK for the AWS identity-management service" - }, - "identitystore": { - "description": "C++ SDK for the AWS identitystore service" - }, - "imagebuilder": { - "description": "C++ SDK for the AWS imagebuilder service" - }, - "importexport": { - "description": "C++ SDK for the AWS importexport service" - }, - "inspector": { - "description": "C++ SDK for the AWS inspector service" - }, - "iot": { - "description": "C++ SDK for the AWS iot service" - }, - "iot-data": { - "description": "C++ SDK for the AWS iot-data service" - }, - "iot-jobs-data": { - "description": "C++ SDK for the AWS iot-jobs-data service" - }, - "iot1click-devices": { - "description": "C++ SDK for the AWS iot1click-devices service" - }, - "iot1click-projects": { - "description": "C++ SDK for the AWS iot1click-projects service" - }, - "iotanalytics": { - "description": "C++ SDK for the AWS iotanalytics service" - }, - "iotdeviceadvisor": { - "description": "C++ SDK for the AWS iotdeviceadvisor service" - }, - "iotevents": { - "description": "C++ SDK for the AWS iotevents service" - }, - "iotevents-data": { - "description": "C++ SDK for the AWS iotevents-data service" - }, - "iotfleethub": { - "description": "C++ SDK for the AWS iotfleethub service" - }, - "iotsecuretunneling": { - "description": "C++ SDK for the AWS iotsecuretunneling service" - }, - "iotsitewise": { - "description": "C++ SDK for the AWS iotsitewise service" - }, - "iotthingsgraph": { - "description": "C++ SDK for the AWS iotthingsgraph service" - }, - "iotwireless": { - "description": "C++ SDK for the AWS iotwireless service" - }, - "ivs": { - "description": "C++ SDK for the AWS ivs service" - }, - "kafka": { - "description": "C++ SDK for the AWS kafka service" - }, - "kendra": { - "description": "C++ SDK for the AWS kendra service" - }, - "kinesis": { - "description": "C++ SDK for the AWS kinesis service" - }, - "kinesis-video-archived-media": { - "description": "C++ SDK for the AWS kinesis-video-archived-media service" - }, - "kinesis-video-media": { - "description": "C++ SDK for the AWS kinesis-video-media service" - }, - "kinesis-video-signaling": { - "description": "C++ SDK for the AWS kinesis-video-signaling service" - }, - "kinesisanalytics": { - "description": "C++ SDK for the AWS kinesisanalytics service" - }, - "kinesisanalyticsv2": { - "description": "C++ SDK for the AWS kinesisanalyticsv2 service" - }, - "kinesisvideo": { - "description": "C++ SDK for the AWS kinesisvideo service" - }, - "kms": { - "description": "C++ SDK for the AWS kms service" - }, - "lakeformation": { - "description": "C++ SDK for the AWS lakeformation service" - }, - "lambda": { - "description": "C++ SDK for the AWS lambda service" - }, - "lex": { - "description": "C++ SDK for the AWS lex service" - }, - "lex-models": { - "description": "C++ SDK for the AWS lex-models service" - }, - "lexv2-models": { - "description": "C++ SDK for the AWS lexv2-models service" - }, - "lexv2-runtime": { - "description": "C++ SDK for the AWS lexv2-runtime service" - }, - "license-manager": { - "description": "C++ SDK for the AWS license-manager service" - }, - "lightsail": { - "description": "C++ SDK for the AWS lightsail service" - }, - "location": { - "description": "C++ SDK for the AWS location service" - }, - "logs": { - "description": "C++ SDK for the AWS logs service" - }, - "lookoutequipment": { - "description": "C++ SDK for the AWS lookoutequipment service" - }, - "lookoutmetrics": { - "description": "C++ SDK for the AWS lookoutmetrics service" - }, - "lookoutvision": { - "description": "C++ SDK for the AWS lookoutvision service" - }, - "machinelearning": { - "description": "C++ SDK for the AWS machinelearning service" - }, - "macie": { - "description": "C++ SDK for the AWS macie service" - }, - "macie2": { - "description": "C++ SDK for the AWS macie2 service" - }, - "managedblockchain": { - "description": "C++ SDK for the AWS managedblockchain service" - }, - "marketplace-catalog": { - "description": "C++ SDK for the AWS marketplace-catalog service" - }, - "marketplace-entitlement": { - "description": "C++ SDK for the AWS marketplace-entitlement service" - }, - "marketplacecommerceanalytics": { - "description": "C++ SDK for the AWS marketplacecommerceanalytics service" - }, - "mediaconnect": { - "description": "C++ SDK for the AWS mediaconnect service" - }, - "mediaconvert": { - "description": "C++ SDK for the AWS mediaconvert service" - }, - "medialive": { - "description": "C++ SDK for the AWS medialive service" - }, - "mediapackage": { - "description": "C++ SDK for the AWS mediapackage service" - }, - "mediapackage-vod": { - "description": "C++ SDK for the AWS mediapackage-vod service" - }, - "mediastore": { - "description": "C++ SDK for the AWS mediastore service" - }, - "mediastore-data": { - "description": "C++ SDK for the AWS mediastore-data service" - }, - "mediatailor": { - "description": "C++ SDK for the AWS mediatailor service" - }, - "meteringmarketplace": { - "description": "C++ SDK for the AWS meteringmarketplace service" - }, - "mgn": { - "description": "C++ SDK for the AWS mgn service" - }, - "migrationhub-config": { - "description": "C++ SDK for the AWS migrationhub-config service" - }, - "mobile": { - "description": "C++ SDK for the AWS mobile service" - }, - "monitoring": { - "description": "C++ SDK for the AWS monitoring service" - }, - "mq": { - "description": "C++ SDK for the AWS mq service" - }, - "mturk-requester": { - "description": "C++ SDK for the AWS mturk-requester service" - }, - "mwaa": { - "description": "C++ SDK for the AWS mwaa service" - }, - "neptune": { - "description": "C++ SDK for the AWS neptune service" - }, - "network-firewall": { - "description": "C++ SDK for the AWS network-firewall service" - }, - "networkmanager": { - "description": "C++ SDK for the AWS networkmanager service" - }, - "nimble": { - "description": "C++ SDK for the AWS nimble service" - }, - "opsworks": { - "description": "C++ SDK for the AWS opsworks service" - }, - "opsworkscm": { - "description": "C++ SDK for the AWS opsworkscm service" - }, - "organizations": { - "description": "C++ SDK for the AWS organizations service" - }, - "outposts": { - "description": "C++ SDK for the AWS outposts service" - }, - "personalize": { - "description": "C++ SDK for the AWS personalize service" - }, - "personalize-events": { - "description": "C++ SDK for the AWS personalize-events service" - }, - "personalize-runtime": { - "description": "C++ SDK for the AWS personalize-runtime service" - }, - "pi": { - "description": "C++ SDK for the AWS pi service" - }, - "pinpoint": { - "description": "C++ SDK for the AWS pinpoint service" - }, - "pinpoint-email": { - "description": "C++ SDK for the AWS pinpoint-email service" - }, - "polly": { - "description": "C++ SDK for the AWS polly service" - }, - "pricing": { - "description": "C++ SDK for the AWS pricing service" - }, - "qldb": { - "description": "C++ SDK for the AWS qldb service" - }, - "qldb-session": { - "description": "C++ SDK for the AWS qldb-session service" - }, - "queues": { - "description": "C++ SDK for the AWS queues service" - }, - "quicksight": { - "description": "C++ SDK for the AWS quicksight service" - }, - "ram": { - "description": "C++ SDK for the AWS ram service" - }, - "rds": { - "description": "C++ SDK for the AWS rds service" - }, - "rds-data": { - "description": "C++ SDK for the AWS rds-data service" - }, - "redshift": { - "description": "C++ SDK for the AWS redshift service" - }, - "redshift-data": { - "description": "C++ SDK for the AWS redshift-data service" - }, - "rekognition": { - "description": "C++ SDK for the AWS rekognition service" - }, - "resource-groups": { - "description": "C++ SDK for the AWS resource-groups service" - }, - "resourcegroupstaggingapi": { - "description": "C++ SDK for the AWS resourcegroupstaggingapi service" - }, - "robomaker": { - "description": "C++ SDK for the AWS robomaker service" - }, - "route53": { - "description": "C++ SDK for the AWS route53 service" - }, - "route53domains": { - "description": "C++ SDK for the AWS route53domains service" - }, - "route53resolver": { - "description": "C++ SDK for the AWS route53resolver service" - }, - "s3": { - "description": "C++ SDK for the AWS s3 service" - }, - "s3-crt": { - "description": "C++ SDK for the AWS s3-crt service" - }, - "s3-encryption": { - "description": "C++ SDK for the AWS s3-encryption service" - }, - "s3control": { - "description": "C++ SDK for the AWS s3control service" - }, - "s3outposts": { - "description": "C++ SDK for the AWS s3outposts service" - }, - "sagemaker": { - "description": "C++ SDK for the AWS sagemaker service" - }, - "sagemaker-a2i-runtime": { - "description": "C++ SDK for the AWS sagemaker-a2i-runtime service" - }, - "sagemaker-edge": { - "description": "C++ SDK for the AWS sagemaker-edge service" - }, - "sagemaker-featurestore-runtime": { - "description": "C++ SDK for the AWS sagemaker-featurestore-runtime service" - }, - "sagemaker-runtime": { - "description": "C++ SDK for the AWS sagemaker-runtime service" - }, - "savingsplans": { - "description": "C++ SDK for the AWS savingsplans service" - }, - "schemas": { - "description": "C++ SDK for the AWS schemas service" - }, - "sdb": { - "description": "C++ SDK for the AWS sdb service" - }, - "secretsmanager": { - "description": "C++ SDK for the AWS secretsmanager service" - }, - "securityhub": { - "description": "C++ SDK for the AWS securityhub service" - }, - "serverlessrepo": { - "description": "C++ SDK for the AWS serverlessrepo service" - }, - "service-quotas": { - "description": "C++ SDK for the AWS service-quotas service" - }, - "servicecatalog": { - "description": "C++ SDK for the AWS servicecatalog service" - }, - "servicecatalog-appregistry": { - "description": "C++ SDK for the AWS servicecatalog-appregistry service" - }, - "servicediscovery": { - "description": "C++ SDK for the AWS servicediscovery service" - }, - "sesv2": { - "description": "C++ SDK for the AWS sesv2 service" - }, - "shield": { - "description": "C++ SDK for the AWS shield service" - }, - "signer": { - "description": "C++ SDK for the AWS signer service" - }, - "sms": { - "description": "C++ SDK for the AWS sms service" - }, - "sms-voice": { - "description": "C++ SDK for the AWS sms-voice service" - }, - "snowball": { - "description": "C++ SDK for the AWS snowball service" - }, - "sns": { - "description": "C++ SDK for the AWS sns service" - }, - "sqs": { - "description": "C++ SDK for the AWS sqs service" - }, - "ssm": { - "description": "C++ SDK for the AWS ssm service" - }, - "ssm-contacts": { - "description": "C++ SDK for the AWS ssm-contacts service" - }, - "ssm-incidents": { - "description": "C++ SDK for the AWS ssm-incidents service" - }, - "sso": { - "description": "C++ SDK for the AWS sso service" - }, - "sso-admin": { - "description": "C++ SDK for the AWS sso-admin service" - }, - "sso-oidc": { - "description": "C++ SDK for the AWS sso-oidc service" - }, - "states": { - "description": "C++ SDK for the AWS states service" - }, - "storagegateway": { - "description": "C++ SDK for the AWS storagegateway service" - }, - "sts": { - "description": "C++ SDK for the AWS sts service" - }, - "support": { - "description": "C++ SDK for the AWS support service" - }, - "swf": { - "description": "C++ SDK for the AWS swf service" - }, - "synthetics": { - "description": "C++ SDK for the AWS synthetics service" - }, - "text-to-speech": { - "description": "C++ SDK for the AWS text-to-speech service" - }, - "textract": { - "description": "C++ SDK for the AWS textract service" - }, - "timestream-query": { - "description": "C++ SDK for the AWS timestream-query service" - }, - "timestream-write": { - "description": "C++ SDK for the AWS timestream-write service" - }, - "transcribe": { - "description": "C++ SDK for the AWS transcribe service" - }, - "transcribestreaming": { - "description": "C++ SDK for the AWS transcribestreaming service" - }, - "transfer": { - "description": "C++ SDK for the AWS transfer service" - }, - "translate": { - "description": "C++ SDK for the AWS translate service" - }, - "waf": { - "description": "C++ SDK for the AWS waf service" - }, - "waf-regional": { - "description": "C++ SDK for the AWS waf-regional service" - }, - "wafv2": { - "description": "C++ SDK for the AWS wafv2 service" - }, - "wellarchitected": { - "description": "C++ SDK for the AWS wellarchitected service" - }, - "workdocs": { - "description": "C++ SDK for the AWS workdocs service" - }, - "worklink": { - "description": "C++ SDK for the AWS worklink service" - }, - "workmail": { - "description": "C++ SDK for the AWS workmail service" - }, - "workmailmessageflow": { - "description": "C++ SDK for the AWS workmailmessageflow service" - }, - "workspaces": { - "description": "C++ SDK for the AWS workspaces service" - }, - "xray": { - "description": "C++ SDK for the AWS xray service" - } - } - } \ No newline at end of file diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/vcpkg.json deleted file mode 100644 index 2693dc898d1b..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/bzip2/vcpkg.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "bzip2", - "version-string": "system", - "features": { - "tool": { - "description": "Builds bzip2 executable" - } - } -} diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/curl/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/curl/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/curl/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/curl/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/curl/vcpkg.json deleted file mode 100644 index eccb5dc561fe..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/curl/vcpkg.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "name": "curl", - "version-string": "system", - "features": { - "brotli": { - "description": "brotli support (brotli)" - }, - "c-ares": { - "description": "c-ares support" - }, - "http2": { - "description": "HTTP2 support" - }, - "idn": { - "description": "Default IDN support" - }, - "idn2": { - "description": "idn2 support (libidn2)" - }, - "mbedtls": { - "description": "SSL support (mbedTLS)" - }, - "non-http": { - "description": "Enables protocols beyond HTTP/HTTPS/HTTP2" - }, - "openssl": { - "description": "SSL support (OpenSSL)" - }, - "schannel": { - "description": "SSL support (Secure Channel)", - "supports": "windows & !uwp" - }, - "sectransp": { - "description": "SSL support (sectransp)", - "supports": "osx | ios" - }, - "ssh": { - "description": "SSH support via libssh2" - }, - "ssl": { - "description": "Default SSL backend" - }, - "sspi": { - "description": "SSPI support", - "supports": "windows & !uwp" - }, - "tool": { - "description": "Builds curl executable", - "supports": "!uwp" - }, - "websockets": { - "description": "WebSocket support (experimental)" - }, - "winidn": { - "description": "WinIDN support", - "supports": "windows" - }, - "winldap": { - "description": "LDAP support (WinLDAP). This feature does not include LDAPS support.", - "supports": "windows & !uwp" - }, - "winssl": { - "description": "Legacy name for schannel", - "supports": "windows & !uwp" - }, - "wolfssl": { - "description": "SSL support (wolfSSL)" - }, - "zstd": { - "description": "ZStandard support (zstd)" - } - } -} diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/lz4/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/lz4/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/lz4/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/lz4/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/lz4/vcpkg.json deleted file mode 100644 index 8a6bb87805f0..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/lz4/vcpkg.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "lz4", - "version-string": "system" -} diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/zlib/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/zlib/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/zlib/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/zlib/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/zlib/vcpkg.json deleted file mode 100644 index 204cc4028f34..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/zlib/vcpkg.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "zlib", - "version-string": "system" -} diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/zstd/portfile.cmake b/.github/workflows/mingw-w64-tiledb/system-ports/zstd/portfile.cmake deleted file mode 100644 index 065116c276ad..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/zstd/portfile.cmake +++ /dev/null @@ -1 +0,0 @@ -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/.github/workflows/mingw-w64-tiledb/system-ports/zstd/vcpkg.json b/.github/workflows/mingw-w64-tiledb/system-ports/zstd/vcpkg.json deleted file mode 100644 index f5f557bc8f59..000000000000 --- a/.github/workflows/mingw-w64-tiledb/system-ports/zstd/vcpkg.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "zstd", - "version-string": "system" -} diff --git a/CMakeLists.txt b/CMakeLists.txt index 5309ffa73865..df428fc15897 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,10 +126,6 @@ if (TILEDB_CMAKE_IDE) set(TILEDB_EP_BASE "${CMAKE_CURRENT_BINARY_DIR}/externals") endif() -# When building static TileDB, we also need to install any static dependencies -# built as external projects. -set(TILEDB_INSTALL_STATIC_DEPS ${TILEDB_STATIC}) - # Set main TileDB Source set(TILEDB_BASE_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/tiledb") # Set experimental TileDB Source diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 6a0538d1792c..d85b52d0f008 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -22,6 +22,7 @@ option(TILEDB_CPP_API "Enables building of the TileDB C++ API." ON) option(TILEDB_CMAKE_IDE "(Used for CLion builds). Disables superbuild and sets the EP install dir." OFF) option(TILEDB_STATS "Enables internal TileDB statistics gathering." ON) option(TILEDB_STATIC "Enables building TileDB as a static library." OFF) +cmake_dependent_option(TILEDB_INSTALL_STATIC_DEPS "Installs static dependencies" ON "TILEDB_STATIC" OFF) option(TILEDB_TESTS "If true, enables building the TileDB unit test suite" ON) option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 14378b7f1c54..f9c128bb0107 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -48,6 +48,7 @@ set(INHERITED_CMAKE_ARGS -DTILEDB_EP_BASE=${TILEDB_EP_BASE} -DTILEDB_STATS=${TILEDB_STATS} -DTILEDB_STATIC=${TILEDB_STATIC} + -DTILEDB_INSTALL_STATIC_DEPS=${TILEDB_INSTALL_STATIC_DEPS} -DTILEDB_TESTS=${TILEDB_TESTS} -DTILEDB_TOOLS=${TILEDB_TOOLS} -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake index 99031ac1274a..f8a5ef097143 100644 --- a/ports/aws-sdk-cpp/portfile.cmake +++ b/ports/aws-sdk-cpp/portfile.cmake @@ -9,6 +9,7 @@ vcpkg_from_github( patch-relocatable-rpath.patch fix-aws-root.patch lock-curl-http-and-tls-settings.patch + remove-checked-array-iterator.patch ) string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) diff --git a/ports/aws-sdk-cpp/remove-checked-array-iterator.patch b/ports/aws-sdk-cpp/remove-checked-array-iterator.patch new file mode 100644 index 000000000000..2e69672335fd --- /dev/null +++ b/ports/aws-sdk-cpp/remove-checked-array-iterator.patch @@ -0,0 +1,51 @@ +diff --git "a/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" "b/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" +--- "a/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" ++++ "b/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" +@@ -54,11 +54,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 +- std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); +-#else + std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); +-#endif // MSVC + } + } + +@@ -82,11 +78,7 @@ namespace Aws + if(arr->m_size > 0 && arr->m_data) + { + size_t arraySize = arr->m_size; +-#ifdef _WIN32 +- std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); +-#else + std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); +-#endif // MSVC + location += arraySize; + } + } +@@ -101,11 +93,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 +- std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); +-#else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +-#endif // MSVC + } + } + +@@ -134,11 +122,7 @@ namespace Aws + { + m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); + +-#ifdef _WIN32 +- std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); +-#else + std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); +-#endif // MSVC + } + + return *this; diff --git a/ports/triplets/x64-mingw-static.cmake b/ports/triplets/x64-mingw-static-release.cmake similarity index 84% rename from ports/triplets/x64-mingw-static.cmake rename to ports/triplets/x64-mingw-static-release.cmake index 28600f0ccd70..603a8c69754b 100644 --- a/ports/triplets/x64-mingw-static.cmake +++ b/ports/triplets/x64-mingw-static-release.cmake @@ -4,3 +4,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_ENV_PASSTHROUGH PATH) set(VCPKG_CMAKE_SYSTEM_NAME MinGW) + +set(VCPKG_BUILD_TYPE release) diff --git a/ports/triplets/x86-mingw-static.cmake b/ports/triplets/x86-mingw-static-release.cmake similarity index 84% rename from ports/triplets/x86-mingw-static.cmake rename to ports/triplets/x86-mingw-static-release.cmake index 3769b85f0c7c..a32921d9f6ea 100644 --- a/ports/triplets/x86-mingw-static.cmake +++ b/ports/triplets/x86-mingw-static-release.cmake @@ -4,3 +4,5 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_ENV_PASSTHROUGH PATH) set(VCPKG_CMAKE_SYSTEM_NAME MinGW) + +set(VCPKG_BUILD_TYPE release) From 589f0b6209dee556a74facdfcf05a119f3a2ec30 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 31 Oct 2023 10:07:23 +0200 Subject: [PATCH 038/456] Improve experience when performing too large GCS direct uploads. (#4047) * When `vfs.gcs.use_multi_part_upload` is disabled, there is a maximum possible size of objects we can write. This size used to be the same with multi-part uploads (`vfs.gcs.max_parallel_ops` * `vfs.gcs.multi_part_size`), and this PR introduces a dedicated config option for that (`vfs.gcs_max_direct_upload_size`, defaulting to 10 GiB). * If this limit is exceeded, we now print a more explanatory message that directs the user which config option to change. [SC-27746](https://app.shortcut.com/tiledb-inc/story/27746/gcs-write-failure-when-multipart-is-disabled) --- TYPE: IMPROVEMENT DESC: Improve experience when performing too large GCS direct uploads. TYPE: CONFIG DESC: Add vfs.gcs.max_direct_upload_size config option. --- test/src/unit-capi-config.cc | 4 + test/src/unit-cppapi-config.cc | 2 +- test/src/unit-gcs.cc | 94 ++----------------- test/src/unit-gs.cc | 94 ++----------------- tiledb/api/c_api/config/config_api_external.h | 4 + tiledb/sm/config/config.cc | 4 + tiledb/sm/config/config.h | 3 + tiledb/sm/cpp_api/config.h | 4 + tiledb/sm/filesystem/gcs.cc | 13 ++- 9 files changed, 42 insertions(+), 180 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 04fa1fde00ce..68a430f222cf 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -307,6 +307,7 @@ void check_save_to_file() { ss << "vfs.azure.use_block_list_upload true\n"; ss << "vfs.file.posix_directory_permissions 755\n"; ss << "vfs.file.posix_file_permissions 644\n"; + ss << "vfs.gcs.max_direct_upload_size 10737418240\n"; ss << "vfs.gcs.max_parallel_ops " << std::thread::hardware_concurrency() << "\n"; ss << "vfs.gcs.multi_part_size 5242880\n"; @@ -693,6 +694,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["vfs.gcs.multi_part_size"] = "5242880"; all_param_values["vfs.gcs.use_multi_part_upload"] = "true"; all_param_values["vfs.gcs.request_timeout_ms"] = "3000"; + all_param_values["vfs.gcs.max_direct_upload_size"] = "10737418240"; all_param_values["vfs.azure.storage_account_name"] = ""; all_param_values["vfs.azure.storage_account_key"] = ""; all_param_values["vfs.azure.storage_sas_token"] = ""; @@ -761,6 +763,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { vfs_param_values["gcs.multi_part_size"] = "5242880"; vfs_param_values["gcs.use_multi_part_upload"] = "true"; vfs_param_values["gcs.request_timeout_ms"] = "3000"; + vfs_param_values["gcs.max_direct_upload_size"] = "10737418240"; vfs_param_values["azure.storage_account_name"] = ""; vfs_param_values["azure.storage_account_key"] = ""; vfs_param_values["azure.storage_sas_token"] = ""; @@ -822,6 +825,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { gcs_param_values["multi_part_size"] = "5242880"; gcs_param_values["use_multi_part_upload"] = "true"; gcs_param_values["request_timeout_ms"] = "3000"; + gcs_param_values["max_direct_upload_size"] = "10737418240"; std::map azure_param_values; azure_param_values["storage_account_name"] = ""; diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 10cf2e453b9d..3b08cde07057 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -67,7 +67,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") { names.push_back(it->first); } // Check number of VFS params in default config object. - CHECK(names.size() == 63); + CHECK(names.size() == 64); } TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") { diff --git a/test/src/unit-gcs.cc b/test/src/unit-gcs.cc index f70d9d955ef6..a897f965d34e 100644 --- a/test/src/unit-gcs.cc +++ b/test/src/unit-gcs.cc @@ -340,22 +340,21 @@ TEST_CASE_METHOD( } TEST_CASE_METHOD( - GCSFx, - "Test GCS filesystem I/O, non-multipart, serial", - "[gcs][non-multipart][serial]") { + GCSFx, "Test GCS filesystem I/O, non-multipart", "[gcs][non-multipart]") { Config config; const uint64_t max_parallel_ops = 1; - const uint64_t multi_part_size = 4 * 1024 * 1024; + const uint64_t write_cache_max_size = 4 * 1024 * 1024; REQUIRE( config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) .ok()); REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) + REQUIRE(config + .set( + "vfs.gcs.max_direct_upload_size", + std::to_string(write_cache_max_size)) .ok()); init_gcs(std::move(config)); - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - // Prepare buffers uint64_t buffer_size_large = write_cache_max_size; auto write_buffer_large = new char[buffer_size_large]; @@ -508,87 +507,6 @@ TEST_CASE_METHOD( REQUIRE(allok); } -TEST_CASE_METHOD( - GCSFx, - "Test GCS filesystem I/O, non-multipart, concurrent", - "[gcs][non-multipart][concurrent]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - !gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(smallfile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - TEST_CASE_METHOD( GCSFx, "Test GCS filesystem I/O, multipart, composition", diff --git a/test/src/unit-gs.cc b/test/src/unit-gs.cc index 53b1a6605b6a..c1cc1b367daf 100644 --- a/test/src/unit-gs.cc +++ b/test/src/unit-gs.cc @@ -308,22 +308,21 @@ TEST_CASE_METHOD( } TEST_CASE_METHOD( - GSFx, - "Test GS filesystem I/O, non-multipart, serial", - "[gs][non-multipart][serial]") { + GSFx, "Test GS filesystem I/O, non-multipart", "[gs][non-multipart]") { Config config; const uint64_t max_parallel_ops = 1; - const uint64_t multi_part_size = 4 * 1024 * 1024; + const uint64_t write_cache_max_size = 4 * 1024 * 1024; REQUIRE( config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) .ok()); REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) + REQUIRE(config + .set( + "vfs.gcs.max_direct_upload_size", + std::to_string(write_cache_max_size)) .ok()); init_gcs(std::move(config)); - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - // Prepare buffers uint64_t buffer_size_large = write_cache_max_size; auto write_buffer_large = new char[buffer_size_large]; @@ -476,87 +475,6 @@ TEST_CASE_METHOD( REQUIRE(allok); } -TEST_CASE_METHOD( - GSFx, - "Test GS filesystem I/O, non-multipart, concurrent", - "[gs][non-multipart][concurrent]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - !gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(smallfile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - TEST_CASE_METHOD( GSFx, "Test GS filesystem I/O, multipart, composition", diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index ef0308a79e11..78bea4376f4b 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -405,6 +405,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `vfs.gcs.request_timeout_ms`
* The maximum amount of time to retry network requests to GCS.
* **Default**: "3000" + * - `vfs.gcs.max_direct_upload_size`
+ * The maximum size in bytes of a direct upload to GCS. Ignored + * if `vfs.gcs.use_multi_part_upload` is set to true.
+ * **Default**: "10737418240" * - `vfs.s3.region`
* The S3 region, if S3 is enabled.
* **Default**: us-east-1 diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index ea4d25740e14..ea16c611f19a 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -190,6 +190,7 @@ const std::string Config::VFS_GCS_MAX_PARALLEL_OPS = const std::string Config::VFS_GCS_MULTI_PART_SIZE = "5242880"; const std::string Config::VFS_GCS_USE_MULTI_PART_UPLOAD = "true"; const std::string Config::VFS_GCS_REQUEST_TIMEOUT_MS = "3000"; +const std::string Config::VFS_GCS_MAX_DIRECT_UPLOAD_SIZE = "10737418240"; const std::string Config::VFS_S3_REGION = "us-east-1"; const std::string Config::VFS_S3_AWS_ACCESS_KEY_ID = ""; const std::string Config::VFS_S3_AWS_SECRET_ACCESS_KEY = ""; @@ -425,6 +426,9 @@ const std::map default_config_values = { "vfs.gcs.use_multi_part_upload", Config::VFS_GCS_USE_MULTI_PART_UPLOAD), std::make_pair( "vfs.gcs.request_timeout_ms", Config::VFS_GCS_REQUEST_TIMEOUT_MS), + std::make_pair( + "vfs.gcs.max_direct_upload_size", + Config::VFS_GCS_MAX_DIRECT_UPLOAD_SIZE), std::make_pair("vfs.s3.region", Config::VFS_S3_REGION), std::make_pair( "vfs.s3.aws_access_key_id", Config::VFS_S3_AWS_ACCESS_KEY_ID), diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 3453ef139169..0826d406ce1e 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -469,6 +469,9 @@ class Config { /** GCS request timeout in milliseconds. */ static const std::string VFS_GCS_REQUEST_TIMEOUT_MS; + /** GCS maximum buffer size for non-multipart uploads. */ + static const std::string VFS_GCS_MAX_DIRECT_UPLOAD_SIZE; + /** S3 region. */ static const std::string VFS_S3_REGION; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 02c66b180d19..3ad09185e789 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -583,6 +583,10 @@ class Config { * - `vfs.gcs.request_timeout_ms`
* The maximum amount of time to retry network requests to GCS.
* **Default**: "3000" + * - `vfs.gcs.max_direct_upload_size`
+ * The maximum size in bytes of a direct upload to GCS. Ignored + * if `vfs.gcs.use_multi_part_upload` is set to true.
+ * **Default**: "10737418240" * - `vfs.s3.region`
* The S3 region, if S3 is enabled.
* **Default**: us-east-1 diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 880ed077268b..52a076acdc3a 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -103,8 +103,14 @@ Status GCS::init(const Config& config, ThreadPool* const thread_pool) { RETURN_NOT_OK(config.get( "vfs.gcs.request_timeout_ms", &request_timeout_ms_, &found)); assert(found); + uint64_t max_direct_upload_size; + RETURN_NOT_OK(config.get( + "vfs.gcs.max_direct_upload_size", &max_direct_upload_size, &found)); + assert(found); - write_cache_max_size_ = max_parallel_ops_ * multi_part_part_size_; + write_cache_max_size_ = use_multi_part_upload_ ? + max_parallel_ops_ * multi_part_part_size_ : + max_direct_upload_size; state_ = State::INITIALIZED; return Status::Ok(); @@ -692,8 +698,9 @@ Status GCS::write( if (!use_multi_part_upload_) { if (nbytes_filled != length) { std::stringstream errmsg; - errmsg << "Direct write failed! " << nbytes_filled - << " bytes written to buffer, " << length << " bytes requested."; + errmsg << "Cannot write more than " << write_cache_max_size_ + << " bytes without multi-part uploads. This limit can be " + "configured with the 'vfs.gcs.max_direct_upload_size' option."; return LOG_STATUS(Status_GCSError(errmsg.str())); } else { return Status::Ok(); From 29a9244797a4a6c480b65d8fad14a224a07fe9aa Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Tue, 31 Oct 2023 05:49:33 -0400 Subject: [PATCH 039/456] Refactor filestore API so that it uses internal TileDB library calls. (#3763) Refactor `tiledb/c_api/tiledb_filestore.cc` so that it uses only internal API calls, as opposed to the C++ external API calls. --- TYPE: IMPROVEMENT DESC: Refactor filestore API so that it uses internal TileDB library calls. --------- Co-authored-by: Luc Rancourt --- tiledb/sm/c_api/tiledb_filestore.cc | 816 +++++++++++++++------------- 1 file changed, 446 insertions(+), 370 deletions(-) diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 368eb2b839b4..d659eafe5092 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -31,60 +31,66 @@ * This file defines the C API of TileDB for filestore. **/ -// Avoid deprecation warnings for the cpp api -#define TILEDB_DEPRECATED - #include "tiledb/api/c_api/attribute/attribute_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" #include "tiledb/api/c_api_support/c_api_support.h" -#include "tiledb/common/common-std.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/array/array.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/c_api/api_argument_validator.h" -#include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_experimental.h" -#include "tiledb/sm/cpp_api/array.h" -#include "tiledb/sm/cpp_api/array_schema.h" -#include "tiledb/sm/cpp_api/attribute.h" -#include "tiledb/sm/cpp_api/context.h" -#include "tiledb/sm/cpp_api/dimension.h" -#include "tiledb/sm/cpp_api/filter_list.h" -#include "tiledb/sm/cpp_api/query.h" -#include "tiledb/sm/cpp_api/subarray.h" -#include "tiledb/sm/cpp_api/vfs.h" +#include "tiledb/sm/compressors/zstd_compressor.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/compressor.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/enums/layout.h" #include "tiledb/sm/enums/mime_type.h" +#include "tiledb/sm/enums/query_type.h" +#include "tiledb/sm/enums/vfs_mode.h" +#include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/filter/compression_filter.h" +#include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/mgc_dict.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/storage_manager/context.h" +#include "tiledb/sm/subarray/subarray.h" +#include "tiledb/type/range/range.h" namespace tiledb::common::detail { // Forward declarations uint64_t compute_tile_extent_based_on_file_size(uint64_t file_size); -std::pair> libmagic_get_mime( - void* data, uint64_t size); -std::pair> libmagic_get_mime_encoding( - void* data, uint64_t size); +std::string libmagic_get_mime(void* data, uint64_t size); +std::string libmagic_get_mime_encoding(void* data, uint64_t size); bool libmagic_file_is_compressed(void* data, uint64_t size); -Status read_file_header( - const VFS& vfs, const char* uri, std::vector& header); +void read_file_header( + tiledb::sm::VFS& vfs, const char* uri, std::vector& header); std::pair strip_file_extension(const char* file_uri); -std::pair> get_buffer_size_from_config( - const Context& context, uint64_t tile_extent); +uint64_t get_buffer_size_from_config( + const tiledb::sm::Config& config, uint64_t tile_extent); int32_t tiledb_filestore_schema_create( tiledb_ctx_t* ctx, const char* uri, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR) { - *array_schema = nullptr; - return TILEDB_ERR; - } - - Context context(ctx, false); + tiledb::sm::Context& context = ctx->context(); uint64_t tile_extent = tiledb::sm::constants::filestore_default_tile_extent; bool is_compressed_libmagic = true; if (uri) { // The user provided a uri, let's examine the file and get some insights // Get the file size, calculate a reasonable tile extent - VFS vfs(context); - uint64_t file_size = vfs.file_size(std::string(uri)); + tiledb::sm::VFS vfs( + &context.resources().stats(), + context.compute_tp(), + context.io_tp(), + context.storage_manager()->config()); + uint64_t file_size; + throw_if_not_ok(vfs.file_size(tiledb::sm::URI(uri), &file_size)); if (file_size) { tile_extent = compute_tile_extent_based_on_file_size(file_size); } @@ -92,177 +98,188 @@ int32_t tiledb_filestore_schema_create( // Detect if the file is compressed or not uint64_t size = std::min(file_size, static_cast(1024)); std::vector header(size); - auto st = read_file_header(vfs, uri, header); + // Don't fail if compression cannot be detected, log a message // and default to uncompressed array - if (st.ok()) { + try { + read_file_header(vfs, uri, header); is_compressed_libmagic = libmagic_file_is_compressed(header.data(), size); - } else { - LOG_STATUS_NO_RETURN_VALUE( - Status_Error("Compression couldn't be detected - " + st.message())); + } catch (const std::exception& e) { + LOG_STATUS_NO_RETURN_VALUE(Status_Error( + std::string("Compression couldn't be detected - ") + e.what())); } } - *array_schema = new (std::nothrow) tiledb_array_schema_t; - if (*array_schema == nullptr) { - auto st = Status_Error( - "Failed to create TileDB Array Schema object; Memory allocation error"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_OOM; - } - + *array_schema = new tiledb_array_schema_t; try { - ArraySchema schema(context, TILEDB_DENSE); - // Share ownership of the internal ArraySchema ptr // All other calls for adding domains, attributes, etc // create copies of the underlying core objects from within // the cpp objects constructed here - (*array_schema)->array_schema_ = schema.ptr()->array_schema_; + (*array_schema)->array_schema_ = make_shared( + HERE(), tiledb::sm::ArrayType::DENSE); + auto& schema = (*array_schema)->array_schema_; + + // Define the range of the dimension. + uint64_t range_lo = 0; + uint64_t range_hi = std::numeric_limits::max() - tile_extent - 1; + tiledb::type::Range range_obj(&range_lo, &range_hi, sizeof(uint64_t)); - auto dim = Dimension::create( - context, + // Define the tile extent as a tiledb::sm::ByteVecValue. + std::vector tile_extent_vec(sizeof(uint64_t), 0); + memcpy(tile_extent_vec.data(), &tile_extent, sizeof(uint64_t)); + + auto dim = make_shared( + HERE(), tiledb::sm::constants::filestore_dimension_name, - {0, std::numeric_limits::max() - tile_extent - 1}, - tile_extent); + tiledb::sm::Datatype::UINT64, + 1, + range_obj, + tiledb::sm::FilterPipeline{}, + tiledb::sm::ByteVecValue(std::move(tile_extent_vec))); - Domain domain(context); - domain.add_dimension(dim); + auto domain = make_shared(HERE()); + throw_if_not_ok(domain->add_dimension(dim)); - auto attr = Attribute::create( - context, tiledb::sm::constants::filestore_attribute_name, TILEDB_BLOB); + auto attr = make_shared( + HERE(), + tiledb::sm::constants::filestore_attribute_name, + tiledb::sm::Datatype::BLOB); // If the input file is not compressed, add our own compression if (!is_compressed_libmagic) { - FilterList filter(context); - filter.add_filter({context, TILEDB_FILTER_ZSTD}); - attr.set_filter_list(filter); + tiledb::sm::FilterPipeline filter; + filter.add_filter(tiledb::sm::CompressionFilter( + tiledb::sm::Compressor::ZSTD, + tiledb::sm::ZStd::default_level(), + tiledb::sm::Datatype::ANY)); + attr->set_filter_pipeline(filter); } - schema.set_domain(domain); - schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); - schema.add_attribute(attr); + throw_if_not_ok(schema->set_domain(domain)); + throw_if_not_ok(schema->set_tile_order(tiledb::sm::Layout::ROW_MAJOR)); + throw_if_not_ok(schema->set_cell_order(tiledb::sm::Layout::ROW_MAJOR)); + throw_if_not_ok(schema->add_attribute(attr)); } catch (const std::exception& e) { - auto st = Status_Error( - std::string("Internal TileDB uncaught exception; ") + e.what()); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); + (*array_schema)->array_schema_ = nullptr; delete *array_schema; - return TILEDB_ERR; + throw api::CAPIStatusException( + std::string("Internal TileDB uncaught exception; ") + e.what()); } return TILEDB_OK; } +inline void ensure_uri_is_valid(const char* uri) { + if (uri == nullptr) { + throw api::CAPIException("Invalid uri pointer"); + } +} + +inline void ensure_buffer_is_valid(void* p) { + if (p == nullptr) { + throw api::CAPIException("Invalid pointer"); + } +} + int32_t tiledb_filestore_uri_import( tiledb_ctx_t* ctx, const char* filestore_array_uri, const char* file_uri, tiledb_mime_type_t) { - if (sanity_check(ctx) == TILEDB_ERR || !filestore_array_uri || !file_uri) { - return TILEDB_ERR; - } + api::ensure_context_is_valid(ctx); + ensure_uri_is_valid(filestore_array_uri); + ensure_uri_is_valid(file_uri); - Context context(ctx, false); + tiledb::sm::Context& context = ctx->context(); // Get the file size - VFS vfs(context); - uint64_t file_size = vfs.file_size(std::string(file_uri)); + tiledb::sm::VFS vfs( + &context.resources().stats(), + context.compute_tp(), + context.io_tp(), + context.storage_manager()->config()); + uint64_t file_size; + throw_if_not_ok(vfs.file_size(tiledb::sm::URI(file_uri), &file_size)); if (!file_size) { return TILEDB_OK; // NOOP } // Sync up the fragment timestamp and metadata timestamp - Array array( - context, - std::string(filestore_array_uri), - TILEDB_WRITE, - TemporalPolicy(TimeTravel, tiledb_timestamp_now_ms())); + uint64_t time_now = tiledb_timestamp_now_ms(); + auto array = make_shared( + HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + throw_if_not_ok(array->open( + tiledb::sm::QueryType::WRITE, + 0, + time_now, + tiledb::sm::EncryptionType::NO_ENCRYPTION, + nullptr, + 0)); // Detect mimetype and encoding with libmagic uint64_t size = std::min(file_size, static_cast(1024)); std::vector header(size); - auto st = read_file_header(vfs, file_uri, header); - if (!st.ok()) { - save_error(ctx, st); - LOG_STATUS_NO_RETURN_VALUE(Status_Error( - "MIME type and encoding couldn't be detected - " + st.message())); - return TILEDB_ERR; - } - auto&& [st1, mime] = libmagic_get_mime(header.data(), header.size()); - if (!st1.ok()) { - save_error(ctx, st1); - LOG_STATUS_NO_RETURN_VALUE(st1); - return TILEDB_ERR; - } - auto&& [st2, mime_encoding] = - libmagic_get_mime_encoding(header.data(), header.size()); - if (!st2.ok()) { - save_error(ctx, st2); - LOG_STATUS_NO_RETURN_VALUE(st2); - return TILEDB_ERR; - } + read_file_header(vfs, file_uri, header); + auto mime = libmagic_get_mime(header.data(), header.size()); + auto mime_encoding = libmagic_get_mime_encoding(header.data(), header.size()); // We need to dump all the relevant metadata at this point so that // clients have all the necessary info when consuming the array - array.put_metadata( - tiledb::sm::constants::filestore_metadata_size_key, - TILEDB_UINT64, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_size_key.c_str(), + tiledb::sm::Datatype::UINT64, 1, &file_size); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_mime_encoding_key, - TILEDB_STRING_UTF8, - static_cast(mime_encoding.value().size()), - mime_encoding.value().c_str()); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_mime_type_key, - TILEDB_STRING_UTF8, - static_cast(mime.value().size()), - mime.value().c_str()); + array->put_metadata( + tiledb::sm::constants::filestore_metadata_mime_encoding_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, + static_cast(mime_encoding.size()), + mime_encoding.c_str()); + array->put_metadata( + tiledb::sm::constants::filestore_metadata_mime_type_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, + static_cast(mime.size()), + mime.c_str()); auto&& [fname, fext] = strip_file_extension(file_uri); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_original_filename_key, - TILEDB_STRING_UTF8, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_original_filename_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, static_cast(fname.size()), fname.c_str()); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_file_extension_key, - TILEDB_STRING_UTF8, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_file_extension_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, static_cast(fext.size()), fext.c_str()); // Write the data in batches using the global order writer - VFS::filebuf fb(vfs); - if (!fb.open(std::string(file_uri), std::ios::in)) { - auto st = Status_Error( + if (!vfs.open_file(tiledb::sm::URI(file_uri), tiledb::sm::VFSMode::VFS_READ) + .ok()) { + throw api::CAPIException( "Failed to open the file; Invalid file URI or incorrect file " "permissions"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; } - std::istream input(&fb); // tiledb:// uri hack // We need to special case on tiledb uris until we implement // serialization for global order writes. Until then, we write // timestamped fragments in row-major order. - bool is_tiledb_uri = array.ptr()->array_->is_remote(); + bool is_tiledb_uri = array->is_remote(); uint64_t tile_extent = compute_tile_extent_based_on_file_size(file_size); - auto&& [st3, buffer_size] = get_buffer_size_from_config(context, tile_extent); - if (!st3.ok()) { - LOG_STATUS_NO_RETURN_VALUE(st3); - save_error(ctx, st3); - return TILEDB_ERR; - } - - Query query(context, array); - query.set_layout(TILEDB_GLOBAL_ORDER); - std::vector buffer(*buffer_size); - - Subarray subarray(context, array); + auto buffer_size = get_buffer_size_from_config( + context.storage_manager()->config(), tile_extent); + + tiledb::sm::Query query(context.storage_manager(), array); + throw_if_not_ok(query.set_layout(tiledb::sm::Layout::GLOBAL_ORDER)); + std::vector buffer(buffer_size); + + tiledb::sm::Subarray subarray( + array.get(), + nullptr, + context.storage_manager()->logger(), + true, + context.storage_manager()); // We need to get the right end boundary of the last space tile. // The last chunk either falls exactly on the end of the file // or it goes beyond the end of the file so that it's equal in size @@ -272,158 +289,193 @@ int32_t tiledb_filestore_uri_import( static_cast(file_size % tile_extent != 0)) * tile_extent - 1; - subarray.add_range(0, static_cast(0), last_space_tile_boundary); + uint64_t range_arr[] = {static_cast(0), last_space_tile_boundary}; + tiledb::type::Range subarray_range( + static_cast(range_arr), sizeof(uint64_t) * 2); + throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); query.set_subarray(subarray); - auto tiledb_cloud_fix = [&](uint64_t start, uint64_t end, uint64_t nbytes) { - Query query(context, array); - query.set_layout(TILEDB_ROW_MAJOR); - Subarray subarray(context, array); - subarray.add_range(0, start, end); - query.set_subarray(subarray); - query.set_data_buffer( - tiledb::sm::constants::filestore_attribute_name, buffer.data(), nbytes); - query.submit(); + auto tiledb_cloud_fix = [&](uint64_t start, uint64_t end) { + tiledb::sm::Query query(context.storage_manager(), array); + throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); + tiledb::sm::Subarray subarray_cloud_fix( + array.get(), + nullptr, + context.storage_manager()->logger(), + true, + context.storage_manager()); + + uint64_t cloud_fix_range_arr[] = {start, end}; + tiledb::type::Range subarray_range_cloud_fix( + static_cast(cloud_fix_range_arr), sizeof(uint64_t) * 2); + throw_if_not_ok( + subarray_cloud_fix.add_range(0, std::move(subarray_range_cloud_fix))); + query.set_subarray(subarray_cloud_fix); + uint64_t data_buff_len = end - start + 1; + throw_if_not_ok(query.set_data_buffer( + tiledb::sm::constants::filestore_attribute_name, + buffer.data(), + &data_buff_len)); + throw_if_not_ok(query.submit()); + }; + + auto read_wrapper = + [&file_size, &vfs, &file_uri, &buffer](uint64_t start) -> uint64_t { + if (start >= file_size) + return 0; + + uint64_t readlen = buffer.size(); + if (start + buffer.size() > file_size) { + readlen = file_size - start; + } + throw_if_not_ok( + vfs.read(tiledb::sm::URI(file_uri), start, buffer.data(), readlen)); + return readlen; }; uint64_t start_range = 0; - uint64_t end_range = *buffer_size - 1; - while (input.read(reinterpret_cast(buffer.data()), *buffer_size)) { - if (is_tiledb_uri) { - tiledb_cloud_fix(start_range, end_range, buffer.size()); - start_range += *buffer_size; - end_range += *buffer_size; - } else { - query.set_data_buffer( - tiledb::sm::constants::filestore_attribute_name, - buffer.data(), - *buffer_size); - query.submit(); + uint64_t end_range = buffer_size - 1; + uint64_t readlen = 0; + while ((readlen = read_wrapper(start_range)) > 0) { + uint64_t end_cloud_fix = end_range; + uint64_t query_buffer_len = buffer_size; + if (readlen < buffer_size) { + end_cloud_fix = start_range + readlen; + query_buffer_len = last_space_tile_boundary - + file_size / (buffer_size) * (buffer_size) + 1; + std::memset(buffer.data() + readlen, 0, buffer_size - readlen); } - } - // If the end of the file was reached, but less than buffer_size bytes - // were read, write the read bytes into the array. - // Check input.gcount() to guard against the case when file_size is - // a multiple of buffer_size, thus eof gets set but there are no more - // bytes to read - if (input.eof() && input.gcount() > 0) { - size_t read_bytes = input.gcount(); - // Initialize the remaining empty cells to 0 - std::memset(buffer.data() + read_bytes, 0, *buffer_size - read_bytes); - uint64_t last_tile_size = last_space_tile_boundary - - file_size / (*buffer_size) * (*buffer_size) + 1; if (is_tiledb_uri) { - tiledb_cloud_fix(start_range, start_range + read_bytes - 1, read_bytes); + tiledb_cloud_fix(start_range, end_cloud_fix); } else { - query.set_data_buffer( + throw_if_not_ok(query.set_data_buffer( tiledb::sm::constants::filestore_attribute_name, buffer.data(), - last_tile_size); - query.submit(); + &query_buffer_len)); + throw_if_not_ok(query.submit()); } - } else if (!input.eof()) { + + start_range += readlen; + end_range += readlen; + } + + if (start_range < file_size) { // Something must have gone wrong whilst reading the file - auto st = Status_Error("Error whilst reading the file"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - fb.close(); - return TILEDB_ERR; + throw_if_not_ok(vfs.close_file(tiledb::sm::URI(file_uri))); + throw api::CAPIStatusException("Error whilst reading the file"); } if (!is_tiledb_uri) { // Dump the fragment on disk - query.finalize(); + throw_if_not_ok(query.finalize()); } - fb.close(); + throw_if_not_ok(vfs.close_file(tiledb::sm::URI(file_uri))); + + throw_if_not_ok(array->close()); return TILEDB_OK; } int32_t tiledb_filestore_uri_export( tiledb_ctx_t* ctx, const char* file_uri, const char* filestore_array_uri) { - if (sanity_check(ctx) == TILEDB_ERR || !filestore_array_uri || !file_uri) { - return TILEDB_ERR; - } - - Context context(ctx, false); - - VFS vfs(context); - VFS::filebuf fb(vfs); - if (!fb.open(std::string(file_uri), std::ios::out)) { - auto st = Status_Error( + ensure_uri_is_valid(filestore_array_uri); + ensure_uri_is_valid(file_uri); + + tiledb::sm::Context& context = ctx->context(); + tiledb::sm::VFS vfs( + &context.resources().stats(), + context.compute_tp(), + context.io_tp(), + context.storage_manager()->config()); + if (!vfs.open_file(tiledb::sm::URI(file_uri), tiledb::sm::VFSMode::VFS_WRITE) + .ok()) { + throw api::CAPIException( "Failed to open the file; Invalid file URI or incorrect file " "permissions"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; } - std::ostream output(&fb); + auto array = make_shared( + HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + throw_if_not_ok(array->open( + tiledb::sm::QueryType::READ, + tiledb::sm::EncryptionType::NO_ENCRYPTION, + nullptr, + 0)); - Array array(context, std::string(filestore_array_uri), TILEDB_READ); - tiledb_datatype_t dtype; + tiledb::sm::Datatype dtype; uint32_t num; - const void* size; - array.get_metadata( - tiledb::sm::constants::filestore_metadata_size_key, &dtype, &num, &size); - if (!size) { - auto st = Status_Error( + const void* file_size_ptr = nullptr; + array->get_metadata( + tiledb::sm::constants::filestore_metadata_size_key.c_str(), + &dtype, + &num, + &file_size_ptr); + if (!file_size_ptr) { + throw api::CAPIException( "The array metadata doesn't contain the " + tiledb::sm::constants::filestore_metadata_size_key + "key"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; } - uint64_t file_size = *static_cast(size); + uint64_t file_size = *static_cast(file_size_ptr); uint64_t tile_extent = compute_tile_extent_based_on_file_size(file_size); - auto&& [st3, buffer_size] = get_buffer_size_from_config(context, tile_extent); - if (!st3.ok()) { - LOG_STATUS_NO_RETURN_VALUE(st3); - save_error(ctx, st3); - return TILEDB_ERR; - } + auto buffer_size = get_buffer_size_from_config( + context.storage_manager()->config(), tile_extent); - std::vector data(*buffer_size); + std::vector data(buffer_size); uint64_t start_range = 0; - uint64_t end_range = std::min(file_size, *buffer_size) - 1; + uint64_t end_range = std::min(file_size, buffer_size) - 1; do { uint64_t write_size = end_range - start_range + 1; - Subarray subarray(context, array); - subarray.add_range(0, start_range, end_range); - Query query(context, array); - query.set_layout(TILEDB_ROW_MAJOR); + tiledb::sm::Subarray subarray( + array.get(), + nullptr, + context.storage_manager()->logger(), + true, + context.storage_manager()); + uint64_t subarray_range_arr[] = {start_range, end_range}; + tiledb::type::Range subarray_range( + static_cast(subarray_range_arr), sizeof(uint64_t) * 2); + throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + + tiledb::sm::Query query(context.storage_manager(), array); + throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); // Cloud compatibility hack. Currently stored tiledb file arrays have a // TILEDB_UINT8 attribute. We should pass the right datatype here to // support reads from existing tiledb file arrays. + auto&& [st, schema] = array->get_array_schema(); + throw_if_not_ok(st); auto attr_type = - array.schema() - .attribute(tiledb::sm::constants::filestore_attribute_name) - .type(); - if (attr_type == TILEDB_UINT8) { - query.set_data_buffer( + schema.value() + ->attribute(tiledb::sm::constants::filestore_attribute_name) + ->type(); + if (attr_type == tiledb::sm::Datatype::UINT8) { + throw_if_not_ok(query.set_data_buffer( tiledb::sm::constants::filestore_attribute_name, reinterpret_cast(data.data()), - write_size); + &write_size)); } else { - query.set_data_buffer( + throw_if_not_ok(query.set_data_buffer( tiledb::sm::constants::filestore_attribute_name, data.data(), - write_size); + &write_size)); } - query.submit(); + throw_if_not_ok(query.submit()); - output.write(reinterpret_cast(data.data()), write_size); + throw_if_not_ok(vfs.write( + tiledb::sm::URI(file_uri), + reinterpret_cast(data.data()), + write_size)); start_range = end_range + 1; - end_range = std::min(file_size - 1, end_range + *buffer_size); + end_range = std::min(file_size - 1, end_range + buffer_size); } while (start_range <= end_range); - output.flush(); - fb.close(); + throw_if_not_ok(vfs.close_file(tiledb::sm::URI(file_uri))); + + throw_if_not_ok(array->close()); return TILEDB_OK; } @@ -434,76 +486,83 @@ int32_t tiledb_filestore_buffer_import( void* buf, size_t size, tiledb_mime_type_t) { - if (sanity_check(ctx) == TILEDB_ERR || !filestore_array_uri || !buf) { - return TILEDB_ERR; - } + api::ensure_context_is_valid(ctx); + ensure_uri_is_valid(filestore_array_uri); + ensure_buffer_is_valid(buf); if (!size) { return TILEDB_OK; // NOOP } - Context context(ctx, false); + tiledb::sm::Context& context = ctx->context(); // Sync up the fragment timestamp and metadata timestamp - Array array( - context, - std::string(filestore_array_uri), - TILEDB_WRITE, - TemporalPolicy(TimeTravel, tiledb_timestamp_now_ms())); + uint64_t time_now = tiledb_timestamp_now_ms(); + auto array = make_shared( + HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + throw_if_not_ok(array->open( + tiledb::sm::QueryType::WRITE, + 0, + time_now, + tiledb::sm::EncryptionType::NO_ENCRYPTION, + nullptr, + 0)); // Detect mimetype and encoding with libmagic uint64_t s = std::min(size, static_cast(1024)); - auto&& [st1, mime]{libmagic_get_mime(buf, s)}; - if (!st1.ok()) { - save_error(ctx, st1); - LOG_STATUS_NO_RETURN_VALUE(st1); - return TILEDB_ERR; - } - auto&& [st2, mime_encoding]{libmagic_get_mime_encoding(buf, s)}; - if (!st2.ok()) { - save_error(ctx, st2); - LOG_STATUS_NO_RETURN_VALUE(st2); - return TILEDB_ERR; - } + auto mime = libmagic_get_mime(buf, s); + auto mime_encoding = libmagic_get_mime_encoding(buf, s); // We need to dump all the relevant metadata at this point so that // clients have all the necessary info when consuming the array - array.put_metadata( - tiledb::sm::constants::filestore_metadata_size_key, - TILEDB_UINT64, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_size_key.c_str(), + tiledb::sm::Datatype::UINT64, 1, &size); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_mime_encoding_key, - TILEDB_STRING_UTF8, - static_cast(mime_encoding.value().size()), - mime_encoding.value().c_str()); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_mime_type_key, - TILEDB_STRING_UTF8, - static_cast(mime.value().size()), - mime.value().c_str()); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_original_filename_key, - TILEDB_STRING_UTF8, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_mime_encoding_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, + static_cast(mime_encoding.size()), + mime_encoding.c_str()); + array->put_metadata( + tiledb::sm::constants::filestore_metadata_mime_type_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, + static_cast(mime.size()), + mime.c_str()); + array->put_metadata( + tiledb::sm::constants::filestore_metadata_original_filename_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, static_cast(0), ""); - array.put_metadata( - tiledb::sm::constants::filestore_metadata_file_extension_key, - TILEDB_STRING_UTF8, + array->put_metadata( + tiledb::sm::constants::filestore_metadata_file_extension_key.c_str(), + tiledb::sm::Datatype::STRING_UTF8, static_cast(0), ""); - Query query(context, array); - query.set_layout(TILEDB_ROW_MAJOR); + tiledb::sm::Query query(context.storage_manager(), array); + throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); + + tiledb::sm::Subarray subarray( + array.get(), + nullptr, + context.storage_manager()->logger(), + true, + context.storage_manager()); + uint64_t subarray_range_arr[] = { + static_cast(0), static_cast(size - 1)}; + tiledb::type::Range subarray_range( + static_cast(subarray_range_arr), sizeof(uint64_t) * 2); + throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); - Subarray subarray(context, array); - subarray.add_range( - 0, static_cast(0), static_cast(size - 1)); query.set_subarray(subarray); - query.set_data_buffer( - tiledb::sm::constants::filestore_attribute_name, buf, size); - query.submit(); + uint64_t size_tmp = size; + throw_if_not_ok(query.set_data_buffer( + tiledb::sm::constants::filestore_attribute_name, buf, &size_tmp)); + throw_if_not_ok(query.submit()); + + throw_if_not_ok(array->close()); return TILEDB_OK; } @@ -514,60 +573,84 @@ int32_t tiledb_filestore_buffer_export( size_t offset, void* buf, size_t size) { - if (sanity_check(ctx) == TILEDB_ERR || !filestore_array_uri || !buf) { - return TILEDB_ERR; - } - - Context context(ctx, false); - - Array array(context, std::string(filestore_array_uri), TILEDB_READ); + api::ensure_context_is_valid(ctx); + ensure_uri_is_valid(filestore_array_uri); + ensure_buffer_is_valid(buf); + + tiledb::sm::Context& context = ctx->context(); + auto array = make_shared( + HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + throw_if_not_ok(array->open( + tiledb::sm::QueryType::READ, + tiledb::sm::EncryptionType::NO_ENCRYPTION, + nullptr, + 0)); // Check whether the user requested more data than the array contains. // Return an error if that's the case. // This is valid only when the array metadata contains the file_size key - tiledb_datatype_t dtype; + tiledb::sm::Datatype dtype; uint32_t num; - const void* file_size; - array.get_metadata( - tiledb::sm::constants::filestore_metadata_size_key, + const void* file_size = nullptr; + array->get_metadata( + tiledb::sm::constants::filestore_metadata_size_key.c_str(), &dtype, &num, &file_size); - if (file_size && *static_cast(file_size) < offset + size) { - auto st = - Status_Error("The number of bytes requested is bigger than the array"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - - Subarray subarray(context, array); - subarray.add_range( - 0, - static_cast(offset), - static_cast(offset + size - 1)); - Query query(context, array); - query.set_layout(TILEDB_ROW_MAJOR); + if (!file_size) { + throw(Status_Error( + "The array metadata doesn't contain the " + + tiledb::sm::constants::filestore_metadata_size_key + "key")); + } else if (*static_cast(file_size) < offset + size) { + throw( + Status_Error("The number of bytes requested is bigger than the array")); + } + + tiledb::sm::Subarray subarray( + array.get(), + nullptr, + context.storage_manager()->logger(), + true, + context.storage_manager()); + uint64_t subarray_range_arr[] = { + static_cast(offset), static_cast(offset + size - 1)}; + tiledb::type::Range subarray_range( + static_cast(subarray_range_arr), sizeof(uint64_t) * 2); + throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + + tiledb::sm::Query query(context.storage_manager(), array); + throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); - query.set_data_buffer( - tiledb::sm::constants::filestore_attribute_name, buf, size); - query.submit(); + uint64_t size_tmp = size; + throw_if_not_ok(query.set_data_buffer( + tiledb::sm::constants::filestore_attribute_name, buf, &size_tmp)); + throw_if_not_ok(query.submit()); + + throw_if_not_ok(array->close()); return TILEDB_OK; } int32_t tiledb_filestore_size( tiledb_ctx_t* ctx, const char* filestore_array_uri, size_t* size) { - if (sanity_check(ctx) == TILEDB_ERR || !filestore_array_uri || !size) { - return TILEDB_ERR; - } - - Context context(ctx, false); - Array array(context, std::string(filestore_array_uri), TILEDB_READ); - - tiledb_datatype_t dtype; - if (!array.has_metadata( - tiledb::sm::constants::filestore_metadata_size_key, &dtype)) { + api::ensure_context_is_valid(ctx); + ensure_uri_is_valid(filestore_array_uri); + ensure_buffer_is_valid(size); + + tiledb::sm::Context& context = ctx->context(); + tiledb::sm::Array array( + tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + + throw_if_not_ok(array.open( + tiledb::sm::QueryType::READ, + tiledb::sm::EncryptionType::NO_ENCRYPTION, + nullptr, + 0)); + + std::optional type = array.metadata_type( + tiledb::sm::constants::filestore_metadata_size_key.c_str()); + bool has_key = type.has_value(); + if (!has_key) { LOG_STATUS_NO_RETURN_VALUE(Status_Error( std::string("Filestore size key not found in array metadata; this " "filestore may not have been imported: ") + @@ -575,14 +658,22 @@ int32_t tiledb_filestore_size( return TILEDB_ERR; } uint32_t num; - const void* file_size; + const void* file_size = nullptr; array.get_metadata( - tiledb::sm::constants::filestore_metadata_size_key, - &dtype, + tiledb::sm::constants::filestore_metadata_size_key.c_str(), + &type.value(), &num, &file_size); + + if (!file_size) { + throw(std::logic_error( + "The array metadata should contain the " + + tiledb::sm::constants::filestore_metadata_size_key + "key")); + } *size = *static_cast(file_size); + throw_if_not_ok(array.close()); + return TILEDB_OK; } @@ -615,50 +706,44 @@ uint64_t compute_tile_extent_based_on_file_size(uint64_t file_size) { } } -std::pair> libmagic_get_mime( - void* data, uint64_t size) { +std::string libmagic_get_mime(void* data, uint64_t size) { magic_t magic = magic_open(MAGIC_MIME_TYPE); if (tiledb::sm::magic_dict::magic_mgc_embedded_load(magic)) { std::string errmsg(magic_error(magic)); magic_close(magic); - return { - Status_Error(std::string("Cannot load magic database - ") + errmsg), - nullopt}; + throw api::CAPIStatusException( + std::string("Cannot load magic database - ") + errmsg); } auto rv = magic_buffer(magic, data, size); if (!rv) { std::string errmsg(magic_error(magic)); magic_close(magic); - return { - Status_Error(std::string("Cannot get the mime type - ") + errmsg), - nullopt}; + throw api::CAPIStatusException( + std::string("Cannot get the mime type - ") + errmsg); } std::string mime(rv); magic_close(magic); - return {Status::Ok(), mime}; + return mime; } -std::pair> libmagic_get_mime_encoding( - void* data, uint64_t size) { +std::string libmagic_get_mime_encoding(void* data, uint64_t size) { magic_t magic = magic_open(MAGIC_MIME_ENCODING); if (tiledb::sm::magic_dict::magic_mgc_embedded_load(magic)) { std::string errmsg(magic_error(magic)); magic_close(magic); - return { - Status_Error(std::string("Cannot load magic database - ") + errmsg), - nullopt}; + throw api::CAPIStatusException( + std::string("Cannot load magic database - ") + errmsg); } auto rv = magic_buffer(magic, data, size); if (!rv) { std::string errmsg(magic_error(magic)); magic_close(magic); - return { - Status_Error(std::string("Cannot get the mime encoding - ") + errmsg), - nullopt}; + throw api::CAPIStatusException( + std::string("Cannot get the mime encoding - ") + errmsg); } std::string mime(rv); magic_close(magic); - return {Status::Ok(), mime}; + return mime; } bool libmagic_file_is_compressed(void* data, uint64_t size) { @@ -687,18 +772,12 @@ bool libmagic_file_is_compressed(void* data, uint64_t size) { return compressed_mime_types.find(mime) != compressed_mime_types.end(); } -Status read_file_header( - const VFS& vfs, const char* uri, std::vector& header) { - VFS::filebuf fb(vfs); - if (!fb.open(uri, std::ios::in)) { - return Status_Error("the file couldn't be opened"); - } - std::istream input(&fb); - if (!input.read(header.data(), header.size())) { - return Status_Error("the file couldn't be read"); - } - fb.close(); - return Status::Ok(); +void read_file_header( + tiledb::sm::VFS& vfs, const char* uri, std::vector& header) { + const tiledb::sm::URI uri_obj = tiledb::sm::URI(uri); + throw_if_not_ok(vfs.open_file(uri_obj, tiledb::sm::VFSMode::VFS_READ)); + throw_if_not_ok(vfs.read(uri_obj, 0, header.data(), header.size())); + throw_if_not_ok(vfs.close_file(uri_obj)); } std::pair strip_file_extension(const char* file_uri) { @@ -710,27 +789,24 @@ std::pair strip_file_extension(const char* file_uri) { return {uri.substr(fname_pos, ext_pos - fname_pos), ext}; } -std::pair> get_buffer_size_from_config( - const Context& context, uint64_t tile_extent) { +uint64_t get_buffer_size_from_config( + const tiledb::sm::Config& config, uint64_t tile_extent) { bool found = false; uint64_t buffer_size; - auto st = context.config().ptr()->config().get( - "filestore.buffer_size", &buffer_size, &found); + auto st = config.get("filestore.buffer_size", &buffer_size, &found); - RETURN_NOT_OK_TUPLE(st, nullopt); + throw_if_not_ok(st); assert(found); if (buffer_size < tile_extent) { - auto st = Status_Error( + throw api::CAPIStatusException( "The buffer size configured via filestore.buffer_size" "is smaller than current " + std::to_string(tile_extent) + " tile extent"); - return {st, nullopt}; } // Round the buffer size down to the nearest tile buffer_size = buffer_size / tile_extent * tile_extent; - - return {Status::Ok(), buffer_size}; + return buffer_size; } } // namespace tiledb::common::detail @@ -739,7 +815,7 @@ using tiledb::api::api_entry_plain; template constexpr auto api_entry = tiledb::api::api_entry_with_context; -TILEDB_EXPORT int32_t tiledb_filestore_schema_create( +int32_t tiledb_filestore_schema_create( tiledb_ctx_t* ctx, const char* uri, tiledb_array_schema_t** array_schema) noexcept { @@ -747,7 +823,7 @@ TILEDB_EXPORT int32_t tiledb_filestore_schema_create( ctx, uri, array_schema); } -TILEDB_EXPORT int32_t tiledb_filestore_uri_import( +int32_t tiledb_filestore_uri_import( tiledb_ctx_t* ctx, const char* filestore_array_uri, const char* file_uri, @@ -756,7 +832,7 @@ TILEDB_EXPORT int32_t tiledb_filestore_uri_import( ctx, filestore_array_uri, file_uri, mime_type); } -TILEDB_EXPORT int32_t tiledb_filestore_uri_export( +int32_t tiledb_filestore_uri_export( tiledb_ctx_t* ctx, const char* file_uri, const char* filestore_array_uri) noexcept { @@ -764,7 +840,7 @@ TILEDB_EXPORT int32_t tiledb_filestore_uri_export( ctx, file_uri, filestore_array_uri); } -TILEDB_EXPORT int32_t tiledb_filestore_buffer_import( +int32_t tiledb_filestore_buffer_import( tiledb_ctx_t* ctx, const char* filestore_array_uri, void* buf, @@ -774,7 +850,7 @@ TILEDB_EXPORT int32_t tiledb_filestore_buffer_import( ctx, filestore_array_uri, buf, size, mime_type); } -TILEDB_EXPORT int32_t tiledb_filestore_buffer_export( +int32_t tiledb_filestore_buffer_export( tiledb_ctx_t* ctx, const char* filestore_array_uri, size_t offset, @@ -784,18 +860,18 @@ TILEDB_EXPORT int32_t tiledb_filestore_buffer_export( ctx, filestore_array_uri, offset, buf, size); } -TILEDB_EXPORT int32_t tiledb_filestore_size( +int32_t tiledb_filestore_size( tiledb_ctx_t* ctx, const char* filestore_array_uri, size_t* size) noexcept { return api_entry( ctx, filestore_array_uri, size); } -TILEDB_EXPORT int32_t tiledb_mime_type_to_str( +int32_t tiledb_mime_type_to_str( tiledb_mime_type_t mime_type, const char** str) noexcept { return api_entry_plain(mime_type, str); } -TILEDB_EXPORT int32_t tiledb_mime_type_from_str( +int32_t tiledb_mime_type_from_str( const char* str, tiledb_mime_type_t* mime_type) noexcept { return api_entry_plain(str, mime_type); -} +} \ No newline at end of file From 71a2cc40d561dfa8b98aec1cba51904e46f76fcb Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Tue, 31 Oct 2023 12:39:36 +0200 Subject: [PATCH 040/456] Group metadata doesn't get serialized (#3147) As @snagles reported, the group metadata seems to be written correctly, but it's empty when you try to read it. I managed to reproduce it locally by opening a TileDB Cloud group, then created a test for it by serializing-deserializing a group with metadata locally. The serialization code is calling `Group::metadata()` to get the md object, but the `Metadata` content is not brought up from disk, so the group looks like it has no metadata. `Group::load_metadata` needs to be called to make sure we serialize a synced version of `Metadata`. --- TYPE: BUG DESC: Group metadata doesn't get serialized --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/unit-capi-group.cc | 68 ++++++++++++++++++++++++++++++++ tiledb/sm/rest/rest_client.cc | 4 +- tiledb/sm/serialization/group.cc | 45 +++++++++++---------- tiledb/sm/serialization/group.h | 17 ++++---- 4 files changed, 103 insertions(+), 31 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 008cb3cb4a9b..fa539870102b 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -1762,4 +1762,72 @@ TEST_CASE_METHOD( tiledb_group_free(&group4_read); remove_temp_dir(temp_dir); } + +TEST_CASE_METHOD( + GroupFx, + "C API: Group metadata serialization", + "[capi][group][metadata][serialization]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + + tiledb::sm::URI group_uri(temp_dir + "group"); + REQUIRE(tiledb_group_create(ctx_, group_uri.c_str()) == TILEDB_OK); + tiledb::sm::URI group_deserialized_uri(temp_dir + "group_deserialized"); + REQUIRE( + tiledb_group_create(ctx_, group_deserialized_uri.c_str()) == TILEDB_OK); + + tiledb_group_t* group = nullptr; + REQUIRE(tiledb_group_alloc(ctx_, group_uri.c_str(), &group) == TILEDB_OK); + REQUIRE(tiledb_group_open(ctx_, group, TILEDB_WRITE) == TILEDB_OK); + + int value = 123; + REQUIRE( + tiledb_group_put_metadata( + ctx_, group, "testmetadata", TILEDB_INT32, 1, &value) == TILEDB_OK); + + REQUIRE(tiledb_group_close(ctx_, group) == TILEDB_OK); + tiledb_group_free(&group); + + // Reopen in read mode + REQUIRE(tiledb_group_alloc(ctx_, group_uri.c_str(), &group) == TILEDB_OK); + REQUIRE(tiledb_group_open(ctx_, group, TILEDB_READ) == TILEDB_OK); + + tiledb_group_t* group_deserialized = nullptr; + REQUIRE( + tiledb_group_alloc( + ctx_, group_deserialized_uri.c_str(), &group_deserialized) == + TILEDB_OK); + REQUIRE( + tiledb_group_open(ctx_, group_deserialized, TILEDB_WRITE) == TILEDB_OK); + + REQUIRE( + tiledb_group_serialize(ctx_, group, group_deserialized, TILEDB_CAPNP) == + TILEDB_OK); + + REQUIRE(tiledb_group_close(ctx_, group_deserialized) == TILEDB_OK); + tiledb_group_free(&group_deserialized); + + // Reopen in read mode + REQUIRE( + tiledb_group_alloc( + ctx_, group_deserialized_uri.c_str(), &group_deserialized) == + TILEDB_OK); + REQUIRE( + tiledb_group_open(ctx_, group_deserialized, TILEDB_READ) == TILEDB_OK); + + tiledb_datatype_t dtype; + uint32_t num; + const void* val; + tiledb_group_get_metadata( + ctx_, group_deserialized, "testmetadata", &dtype, &num, &val); + + REQUIRE(val != nullptr); + CHECK(*static_cast(val) == 123); + + REQUIRE(tiledb_group_close(ctx_, group) == TILEDB_OK); + tiledb_group_free(&group); + REQUIRE(tiledb_group_close(ctx_, group_deserialized) == TILEDB_OK); + tiledb_group_free(&group_deserialized); + remove_temp_dir(temp_dir); +} #endif diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 26184ed66ed1..f69b55c4d234 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -1276,7 +1276,7 @@ Status RestClient::post_group_metadata_from_rest(const URI& uri, Group* group) { Buffer buff; RETURN_NOT_OK(serialization::group_metadata_serialize( - group, serialization_type_, &buff)); + group, serialization_type_, &buff, false)); // Wrap in a list BufferList serialized; RETURN_NOT_OK(serialized.add_buffer(std::move(buff))); @@ -1319,7 +1319,7 @@ Status RestClient::put_group_metadata_to_rest(const URI& uri, Group* group) { Buffer buff; RETURN_NOT_OK(serialization::group_metadata_serialize( - group, serialization_type_, &buff)); + group, serialization_type_, &buff, true)); // Wrap in a list BufferList serialized; RETURN_NOT_OK(serialized.add_buffer(std::move(buff))); diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index 1a87d22bcaa0..5e73744c87ba 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -65,12 +65,20 @@ namespace serialization { #ifdef TILEDB_SERIALIZATION Status group_metadata_to_capnp( - const Group* group, capnp::GroupMetadata::Builder* group_metadata_builder) { + Group* group, + capnp::GroupMetadata::Builder* group_metadata_builder, + bool load) { // Set config auto config_builder = group_metadata_builder->initConfig(); RETURN_NOT_OK(config_to_capnp(group->config(), &config_builder)); - const Metadata* metadata = group->metadata(); + Metadata* metadata; + if (load) { + RETURN_NOT_OK(group->metadata(&metadata)); + } else { + metadata = const_cast(group->metadata()); + } + if (metadata->num()) { auto metadata_builder = group_metadata_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &metadata_builder)); @@ -135,13 +143,12 @@ group_member_from_capnp(capnp::GroupMember::Reader* group_member_reader) { } Status group_details_to_capnp( - const Group* group, - capnp::Group::GroupDetails::Builder* group_details_builder) { + Group* group, capnp::Group::GroupDetails::Builder* group_details_builder) { if (group == nullptr) return LOG_STATUS( Status_SerializationError("Error serializing group; group is null.")); - auto& group_details = group->group_details(); + auto group_details = group->group_details(); if (group_details != nullptr) { const auto& group_members = group->members(); @@ -158,7 +165,8 @@ Status group_details_to_capnp( } } - const Metadata* metadata = group->metadata(); + Metadata* metadata; + RETURN_NOT_OK(group->metadata(&metadata)); if (metadata->num()) { auto group_metadata_builder = group_details_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &group_metadata_builder)); @@ -189,8 +197,7 @@ Status group_details_from_capnp( return Status::Ok(); } -Status group_to_capnp( - const Group* group, capnp::Group::Builder* group_builder) { +Status group_to_capnp(Group* group, capnp::Group::Builder* group_builder) { if (group == nullptr) return LOG_STATUS( Status_SerializationError("Error serializing group; group is null.")); @@ -364,9 +371,7 @@ Status group_create_to_capnp( } Status group_serialize( - const Group* group, - SerializationType serialize_type, - Buffer* serialized_buffer) { + Group* group, SerializationType serialize_type, Buffer* serialized_buffer) { try { ::capnp::MallocMessageBuilder message; capnp::Group::Builder groupBuilder = message.initRoot(); @@ -464,9 +469,7 @@ Status group_deserialize( } Status group_details_serialize( - const Group* group, - SerializationType serialize_type, - Buffer* serialized_buffer) { + Group* group, SerializationType serialize_type, Buffer* serialized_buffer) { try { ::capnp::MallocMessageBuilder message; capnp::Group::GroupDetails::Builder groupDetailsBuilder = @@ -724,14 +727,16 @@ Status group_create_serialize( } Status group_metadata_serialize( - const Group* group, + Group* group, SerializationType serialize_type, - Buffer* serialized_buffer) { + Buffer* serialized_buffer, + bool load) { try { ::capnp::MallocMessageBuilder message; capnp::GroupMetadata::Builder group_metadata_builder = message.initRoot(); - RETURN_NOT_OK(group_metadata_to_capnp(group, &group_metadata_builder)); + RETURN_NOT_OK( + group_metadata_to_capnp(group, &group_metadata_builder, load)); serialized_buffer->reset_size(); serialized_buffer->reset_offset(); @@ -779,7 +784,7 @@ Status group_metadata_serialize( #else -Status group_serialize(const Group*, SerializationType, Buffer*) { +Status group_serialize(Group*, SerializationType, Buffer*) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } @@ -789,7 +794,7 @@ Status group_deserialize(Group*, SerializationType, const Buffer&) { "Cannot deserialize; serialization not enabled.")); } -Status group_details_serialize(const Group*, SerializationType, Buffer*) { +Status group_details_serialize(Group*, SerializationType, Buffer*) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } @@ -814,7 +819,7 @@ Status group_create_serialize(const Group*, SerializationType, Buffer*) { "Cannot serialize; serialization not enabled.")); } -Status group_metadata_serialize(const Group*, SerializationType, Buffer*) { +Status group_metadata_serialize(Group*, SerializationType, Buffer*, bool) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/group.h b/tiledb/sm/serialization/group.h index daace986f0fa..5e239e6b16fa 100644 --- a/tiledb/sm/serialization/group.h +++ b/tiledb/sm/serialization/group.h @@ -57,9 +57,7 @@ namespace serialization { * @return Status */ Status group_serialize( - const Group* group, - SerializationType serialize_type, - Buffer* serialized_buffer); + Group* group, SerializationType serialize_type, Buffer* serialized_buffer); /** * Deserialize a group via Cap'n proto @@ -84,9 +82,7 @@ Status group_deserialize( * @return Status */ Status group_details_serialize( - const Group* group, - SerializationType serialize_type, - Buffer* serialized_buffer); + Group* group, SerializationType serialize_type, Buffer* serialized_buffer); /** * Deserialize a group details via Cap'n proto @@ -149,15 +145,18 @@ Status group_create_serialize( * @param serialize_type format to serialize into Cap'n Proto or JSON * @param serialized_buffer buffer to store serialized bytes in * serialize the array URI + * @param load flag signaling whether or not metadata should be fetched from + * storage * @return Status */ Status group_metadata_serialize( - const Group* group, + Group* group, SerializationType serialize_type, - Buffer* serialized_buffer); + Buffer* serialized_buffer, + bool load); } // namespace serialization } // namespace sm } // namespace tiledb -#endif // TILEDB_SERIALIZATION_GROUP_H \ No newline at end of file +#endif // TILEDB_SERIALIZATION_GROUP_H From b8ec44fd9a5b66d7fa632db430b9ec250ccd891e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 31 Oct 2023 13:03:20 +0200 Subject: [PATCH 041/456] Encode Win32 error messages in UTF-8. (#3987) Currently, Win32 error messages are retrieved using the `FormatMessageA` API, which returns them in ANSI encoding. This results in corrupted error messages when the computer's language is other than English. With this PR, we use `FormatMessageW` which returns the message in UTF-16 encoding, which then gets converted to UTF-8 and stored in the encoding-agnostic `std::string`. __Before__ ![image](https://user-images.githubusercontent.com/12659251/226977534-68f56838-6e9c-4e04-92ce-ae0824ef7c56.png) __After__ ![image](https://user-images.githubusercontent.com/12659251/226977191-6c024bc4-99c3-4d91-b28b-5a3407038da4.png) __After, if the console's output mode is set to UTF-8__ ![image](https://user-images.githubusercontent.com/12659251/226976565-6f0670f7-9f4b-4cd0-ab53-b542f79522cf.png) The C# API (at least, don't know about the others) needs to be updated to support incoming UTF-8 strings from the Core. IIRC, POSIX works in UTF-8 by default so it doesn't need any changes, right? [SC-26131](https://app.shortcut.com/tiledb-inc/story/26131/win32-vfs-error-messages-in-greek-are-corrupted) --- TYPE: IMPROVEMENT DESC: Encode Win32 error messages in UTF-8. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/CMakeLists.txt | 2 ++ test/src/unit-win-filesystem.cc | 34 ++++++++++++++++++++++ tiledb/sm/filesystem/win.cc | 50 ++++++++++++++++++++------------- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 42a839d50e6b..8dd294ed847a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -287,6 +287,8 @@ add_executable( add_dependencies(tiledb_unit tiledb_test_support_lib) +target_compile_options(tiledb_unit PRIVATE "$<$:/utf-8>") + # We want tests to continue as normal even as the API is changing, # so don't warn for deprecations, since they'll be escalated to errors. if (NOT MSVC) diff --git a/test/src/unit-win-filesystem.cc b/test/src/unit-win-filesystem.cc index 7168524cba9f..6e888a44bb9b 100644 --- a/test/src/unit-win-filesystem.cc +++ b/test/src/unit-win-filesystem.cc @@ -42,6 +42,8 @@ #include "tiledb/sm/filesystem/path_win.h" #include "tiledb/sm/filesystem/win.h" +#include + using namespace tiledb::common; using namespace tiledb::sm; @@ -285,4 +287,36 @@ TEST_CASE_METHOD( Crypto::MD5_DIGEST_BYTES) == 0); } +// Uses RAII to temporarily change the Win32 thread UI language. +class ChangeThreadUILanguage { + public: + ChangeThreadUILanguage(LANGID langid) { + old_langid_ = ::GetThreadUILanguage(); + ::SetThreadUILanguage(langid); + } + ~ChangeThreadUILanguage() { + ::SetThreadUILanguage(old_langid_); + } + + private: + LANGID old_langid_; +}; + +// This test requires the Greek language pack to be installed. +TEST_CASE("Test UTF-8 error messages", "[.hide][windows][utf8-msgs]") { + // Change the thread UI language to Greek, to test that an error message with + // Unicode characters is received correctly. + ChangeThreadUILanguage change_langid( + MAKELANGID(LANG_GREEK, SUBLANG_GREEK_GREECE)); + + Win win; + REQUIRE(win.init(Config()).ok()); + // NUL is a special file on Windows; deleting it should always fail. + Status st = win.remove_file("NUL"); + REQUIRE(!st.ok()); + auto message = st.message(); + auto expected = u8"Δεν επιτρέπεται η πρόσβαση."; // Access denied. + REQUIRE(message.find((char*)expected) != std::string::npos); +} + #endif // _WIN32 diff --git a/tiledb/sm/filesystem/win.cc b/tiledb/sm/filesystem/win.cc index 57cf37432530..b642a804baec 100644 --- a/tiledb/sm/filesystem/win.cc +++ b/tiledb/sm/filesystem/win.cc @@ -31,33 +31,33 @@ */ #ifdef _WIN32 -#include "win.h" -#include "path_win.h" -#include "tiledb/common/common.h" -#include "tiledb/common/filesystem/directory_entry.h" -#include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger.h" -#include "tiledb/common/scoped_executor.h" -#include "tiledb/common/stdx_string.h" -#include "tiledb/sm/misc/constants.h" -#include "tiledb/sm/misc/tdb_math.h" -#include "tiledb/sm/misc/utils.h" -#include "uri.h" - #if !defined(NOMINMAX) #define NOMINMAX // suppress definition of min/max macros in Windows headers #endif #include #include -#include -#include // For INTERNET_MAX_URL_LENGTH #include #include +#include #include #include +#include #include #include +#include "path_win.h" +#include "tiledb/common/common.h" +#include "tiledb/common/filesystem/directory_entry.h" +#include "tiledb/common/heap_memory.h" +#include "tiledb/common/logger.h" +#include "tiledb/common/scoped_executor.h" +#include "tiledb/common/stdx_string.h" +#include "tiledb/sm/misc/constants.h" +#include "tiledb/sm/misc/tdb_math.h" +#include "tiledb/sm/misc/utils.h" +#include "uri.h" +#include "win.h" + using namespace tiledb::common; using tiledb::common::filesystem::directory_entry; @@ -67,14 +67,22 @@ namespace sm { namespace { /** Returns the last Windows error message string. */ std::string get_last_error_msg_desc(decltype(GetLastError()) gle) { - LPVOID lpMsgBuf = nullptr; - if (FormatMessage( + LPWSTR lpMsgBuf = nullptr; + // FormatMessageW allocates a buffer that must be freed with LocalFree. + if (FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, gle, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR)&lpMsgBuf, + // By passing zero as the language ID, Windows will try the following + // languages in order: + // * Language neutral + // * Thread LANGID, based on the thread's locale value + // * User default LANGID, based on the user's default locale value + // * System default LANGID, based on the system default locale value + // * US English + 0, + (LPWSTR)&lpMsgBuf, 0, NULL) == 0) { if (lpMsgBuf) { @@ -82,7 +90,9 @@ std::string get_last_error_msg_desc(decltype(GetLastError()) gle) { } return "unknown error"; } - std::string msg(reinterpret_cast(lpMsgBuf)); + // Convert to UTF-8. + std::string msg = + std::wstring_convert>{}.to_bytes(lpMsgBuf); LocalFree(lpMsgBuf); return msg; } From aede0f1fea8f9921e47520a08985cf75d572bb67 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Tue, 31 Oct 2023 05:47:24 -0600 Subject: [PATCH 042/456] Convert C API interface functions to CAPI_INTERFACE (#4460) Convert C API interface functions to use `CAPI_INTERFACE`. This completes the work in #4430. There are a very small number of functions not converted. One is not in the C API (i.e. it doesn't have `extern "C"` linkage), but is used by the C++ API, and on top of that it's for a deprecated operation. The others are small, informational functions that do return values directly and do not use `capi_return_t`. --- TYPE: NO_HISTORY DESC: Convert C API interface functions to CAPI_INTERFACE --- tiledb/api/c_api/attribute/attribute_api.cc | 87 +- tiledb/api/c_api/buffer/buffer_api.cc | 25 +- .../api/c_api/buffer_list/buffer_list_api.cc | 26 +- tiledb/api/c_api/data_order/data_order_api.cc | 8 +- tiledb/api/c_api/datatype/datatype_api.cc | 7 +- tiledb/api/c_api/dimension/dimension_api.cc | 54 +- .../dimension_label/dimension_label_api.cc | 39 +- tiledb/api/c_api/domain/domain_api.cc | 41 +- .../api/c_api/enumeration/enumeration_api.cc | 49 +- tiledb/api/c_api/filesystem/filesystem_api.cc | 8 +- tiledb/api/c_api/filter/filter_api.cc | 42 +- .../api/c_api/filter_list/filter_list_api.cc | 31 +- tiledb/api/c_api/group/group_api.cc | 154 ++- tiledb/api/c_api/object/object_api.cc | 16 +- tiledb/api/c_api/query/query_api.cc | 8 +- .../query_aggregate/query_aggregate_api.cc | 59 +- .../api/c_api/query_field/query_field_api.cc | 29 +- tiledb/api/c_api/query_plan/query_plan_api.cc | 5 +- tiledb/api/c_api/string/string_api.cc | 6 +- tiledb/api/c_api/vfs/vfs_api.cc | 132 +- .../exception_wrapper/capi_definition.h | 7 + tiledb/sm/c_api/tiledb.cc | 1231 ++++++++++------- tiledb/sm/c_api/tiledb_dimension_label.cc | 116 +- tiledb/sm/c_api/tiledb_experimental.h | 2 +- tiledb/sm/c_api/tiledb_filestore.cc | 63 +- 25 files changed, 1340 insertions(+), 905 deletions(-) diff --git a/tiledb/api/c_api/attribute/attribute_api.cc b/tiledb/api/c_api/attribute/attribute_api.cc index 9d9efdde6133..62d434c32977 100644 --- a/tiledb/api/c_api/attribute/attribute_api.cc +++ b/tiledb/api/c_api/attribute/attribute_api.cc @@ -205,150 +205,167 @@ capi_return_t tiledb_attribute_get_enumeration_name( using tiledb::api::api_entry_context; -int32_t tiledb_attribute_alloc( +CAPI_INTERFACE( + attribute_alloc, tiledb_ctx_t* ctx, const char* name, tiledb_datatype_t type, - tiledb_attribute_handle_t** attr) noexcept { + tiledb_attribute_handle_t** attr) { return api_entry_context( ctx, name, type, attr); } -void tiledb_attribute_free(tiledb_attribute_handle_t** attr) noexcept { +CAPI_INTERFACE_VOID(attribute_free, tiledb_attribute_handle_t** attr) { return tiledb::api::api_entry_void(attr); } -int32_t tiledb_attribute_set_nullable( +CAPI_INTERFACE( + attribute_set_nullable, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, - uint8_t nullable) noexcept { + uint8_t nullable) { return api_entry_context( ctx, attr, nullable); } -int32_t tiledb_attribute_set_filter_list( +CAPI_INTERFACE( + attribute_set_filter_list, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, - tiledb_filter_list_t* filter_list) noexcept { + tiledb_filter_list_t* filter_list) { return api_entry_context( ctx, attr, filter_list); } -int32_t tiledb_attribute_set_cell_val_num( +CAPI_INTERFACE( + attribute_set_cell_val_num, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, - uint32_t cell_val_num) noexcept { + uint32_t cell_val_num) { return api_entry_context( ctx, attr, cell_val_num); } -int32_t tiledb_attribute_get_name( +CAPI_INTERFACE( + attribute_get_name, tiledb_ctx_t* ctx, const tiledb_attribute_handle_t* attr, - const char** name) noexcept { + const char** name) { return api_entry_context( ctx, attr, name); } -int32_t tiledb_attribute_get_type( +CAPI_INTERFACE( + attribute_get_type, tiledb_ctx_t* ctx, const tiledb_attribute_handle_t* attr, - tiledb_datatype_t* type) noexcept { + tiledb_datatype_t* type) { return api_entry_context( ctx, attr, type); } -int32_t tiledb_attribute_get_nullable( +CAPI_INTERFACE( + attribute_get_nullable, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, - uint8_t* nullable) noexcept { + uint8_t* nullable) { return api_entry_context( ctx, attr, nullable); } -int32_t tiledb_attribute_get_filter_list( +CAPI_INTERFACE( + attribute_get_filter_list, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, - tiledb_filter_list_t** filter_list) noexcept { + tiledb_filter_list_t** filter_list) { return api_entry_context( ctx, attr, filter_list); } -int32_t tiledb_attribute_get_cell_val_num( +CAPI_INTERFACE( + attribute_get_cell_val_num, tiledb_ctx_t* ctx, const tiledb_attribute_handle_t* attr, - uint32_t* cell_val_num) noexcept { + uint32_t* cell_val_num) { return api_entry_context( ctx, attr, cell_val_num); } -int32_t tiledb_attribute_get_cell_size( +CAPI_INTERFACE( + attribute_get_cell_size, tiledb_ctx_t* ctx, const tiledb_attribute_handle_t* attr, - uint64_t* cell_size) noexcept { + uint64_t* cell_size) { return api_entry_context( ctx, attr, cell_size); } -int32_t tiledb_attribute_dump( +CAPI_INTERFACE( + attribute_dump, tiledb_ctx_t* ctx, const tiledb_attribute_handle_t* attr, - FILE* out) noexcept { + FILE* out) { return api_entry_context(ctx, attr, out); } -int32_t tiledb_attribute_set_fill_value( +CAPI_INTERFACE( + attribute_set_fill_value, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, const void* value, - uint64_t size) noexcept { + uint64_t size) { return api_entry_context( ctx, attr, value, size); } -int32_t tiledb_attribute_get_fill_value( +CAPI_INTERFACE( + attribute_get_fill_value, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, const void** value, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry_context( ctx, attr, value, size); } -int32_t tiledb_attribute_set_fill_value_nullable( +CAPI_INTERFACE( + attribute_set_fill_value_nullable, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, const void* value, uint64_t size, - uint8_t valid) noexcept { + uint8_t valid) { return api_entry_context< tiledb::api::tiledb_attribute_set_fill_value_nullable>( ctx, attr, value, size, valid); } -int32_t tiledb_attribute_get_fill_value_nullable( +CAPI_INTERFACE( + attribute_get_fill_value_nullable, tiledb_ctx_t* ctx, tiledb_attribute_handle_t* attr, const void** value, uint64_t* size, - uint8_t* valid) noexcept { + uint8_t* valid) { return api_entry_context< tiledb::api::tiledb_attribute_get_fill_value_nullable>( ctx, attr, value, size, valid); } -capi_return_t tiledb_attribute_set_enumeration_name( +CAPI_INTERFACE( + attribute_set_enumeration_name, tiledb_ctx_t* ctx, tiledb_attribute_t* attr, - const char* enumeration_name) noexcept { + const char* enumeration_name) { return api_entry_context( ctx, attr, enumeration_name); } -capi_return_t tiledb_attribute_get_enumeration_name( +CAPI_INTERFACE( + attribute_get_enumeration_name, tiledb_ctx_t* ctx, tiledb_attribute_t* attr, - tiledb_string_t** name) noexcept { + tiledb_string_t** name) { return api_entry_context( ctx, attr, name); } diff --git a/tiledb/api/c_api/buffer/buffer_api.cc b/tiledb/api/c_api/buffer/buffer_api.cc index 335441c8f6dc..6b6a28de7dec 100644 --- a/tiledb/api/c_api/buffer/buffer_api.cc +++ b/tiledb/api/c_api/buffer/buffer_api.cc @@ -98,45 +98,48 @@ capi_return_t tiledb_buffer_set_data( using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; -capi_return_t tiledb_buffer_alloc( - tiledb_ctx_t* ctx, tiledb_buffer_t** buffer) noexcept { +CAPI_INTERFACE(buffer_alloc, tiledb_ctx_t* ctx, tiledb_buffer_t** buffer) { return api_entry_context(ctx, buffer); } -void tiledb_buffer_free(tiledb_buffer_t** buffer) noexcept { +CAPI_INTERFACE_VOID(buffer_free, tiledb_buffer_t** buffer) { return api_entry_void(buffer); } -capi_return_t tiledb_buffer_set_type( +CAPI_INTERFACE( + buffer_set_type, tiledb_ctx_t* ctx, tiledb_buffer_t* buffer, - tiledb_datatype_t datatype) noexcept { + tiledb_datatype_t datatype) { return api_entry_context( ctx, buffer, datatype); } -capi_return_t tiledb_buffer_get_type( +CAPI_INTERFACE( + buffer_get_type, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, - tiledb_datatype_t* datatype) noexcept { + tiledb_datatype_t* datatype) { return api_entry_context( ctx, buffer, datatype); } -capi_return_t tiledb_buffer_get_data( +CAPI_INTERFACE( + buffer_get_data, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, void** data, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry_context( ctx, buffer, data, size); } -capi_return_t tiledb_buffer_set_data( +CAPI_INTERFACE( + buffer_set_data, tiledb_ctx_t* ctx, tiledb_buffer_t* buffer, void* data, - uint64_t size) noexcept { + uint64_t size) { return api_entry_context( ctx, buffer, data, size); } diff --git a/tiledb/api/c_api/buffer_list/buffer_list_api.cc b/tiledb/api/c_api/buffer_list/buffer_list_api.cc index 7b19a77db3e8..6e7fdfa93a73 100644 --- a/tiledb/api/c_api/buffer_list/buffer_list_api.cc +++ b/tiledb/api/c_api/buffer_list/buffer_list_api.cc @@ -121,45 +121,49 @@ capi_return_t tiledb_buffer_list_flatten( using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; -capi_return_t tiledb_buffer_list_alloc( - tiledb_ctx_t* ctx, tiledb_buffer_list_t** buffer_list) noexcept { +CAPI_INTERFACE( + buffer_list_alloc, tiledb_ctx_t* ctx, tiledb_buffer_list_t** buffer_list) { return api_entry_context( ctx, buffer_list); } -void tiledb_buffer_list_free(tiledb_buffer_list_t** buffer_list) noexcept { +CAPI_INTERFACE_VOID(buffer_list_free, tiledb_buffer_list_t** buffer_list) { return api_entry_void(buffer_list); } -capi_return_t tiledb_buffer_list_get_num_buffers( +CAPI_INTERFACE( + buffer_list_get_num_buffers, tiledb_ctx_t* ctx, const tiledb_buffer_list_t* buffer_list, - uint64_t* num_buffers) noexcept { + uint64_t* num_buffers) { return api_entry_context( ctx, buffer_list, num_buffers); } -capi_return_t tiledb_buffer_list_get_buffer( +CAPI_INTERFACE( + buffer_list_get_buffer, tiledb_ctx_t* ctx, const tiledb_buffer_list_t* buffer_list, uint64_t buffer_idx, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry_context( ctx, buffer_list, buffer_idx, buffer); } -capi_return_t tiledb_buffer_list_get_total_size( +CAPI_INTERFACE( + buffer_list_get_total_size, tiledb_ctx_t* ctx, const tiledb_buffer_list_t* buffer_list, - uint64_t* total_size) noexcept { + uint64_t* total_size) { return api_entry_context( ctx, buffer_list, total_size); } -capi_return_t tiledb_buffer_list_flatten( +CAPI_INTERFACE( + buffer_list_flatten, tiledb_ctx_t* ctx, tiledb_buffer_list_t* buffer_list, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry_context( ctx, buffer_list, buffer); } diff --git a/tiledb/api/c_api/data_order/data_order_api.cc b/tiledb/api/c_api/data_order/data_order_api.cc index 9b8888c2edc4..815e931b22e7 100644 --- a/tiledb/api/c_api/data_order/data_order_api.cc +++ b/tiledb/api/c_api/data_order/data_order_api.cc @@ -55,14 +55,14 @@ capi_return_t tiledb_data_order_from_str( using tiledb::api::api_entry_plain; -capi_return_t tiledb_data_order_to_str( - tiledb_data_order_t data_order, const char** str) noexcept { +CAPI_INTERFACE( + data_order_to_str, tiledb_data_order_t data_order, const char** str) { return api_entry_plain( data_order, str); } -capi_return_t tiledb_data_order_from_str( - const char* str, tiledb_data_order_t* data_order) noexcept { +CAPI_INTERFACE( + data_order_from_str, const char* str, tiledb_data_order_t* data_order) { return api_entry_plain( str, data_order); } diff --git a/tiledb/api/c_api/datatype/datatype_api.cc b/tiledb/api/c_api/datatype/datatype_api.cc index 21a213fc67e2..854b49fe2a56 100644 --- a/tiledb/api/c_api/datatype/datatype_api.cc +++ b/tiledb/api/c_api/datatype/datatype_api.cc @@ -61,13 +61,12 @@ uint64_t tiledb_datatype_size(tiledb_datatype_t type) { using tiledb::api::api_entry_plain; -capi_return_t tiledb_datatype_to_str( - tiledb_datatype_t datatype, const char** str) noexcept { +CAPI_INTERFACE(datatype_to_str, tiledb_datatype_t datatype, const char** str) { return api_entry_plain(datatype, str); } -capi_return_t tiledb_datatype_from_str( - const char* str, tiledb_datatype_t* datatype) noexcept { +CAPI_INTERFACE( + datatype_from_str, const char* str, tiledb_datatype_t* datatype) { return api_entry_plain(str, datatype); } diff --git a/tiledb/api/c_api/dimension/dimension_api.cc b/tiledb/api/c_api/dimension/dimension_api.cc index 594d14e3bdb5..c8a831d7a040 100644 --- a/tiledb/api/c_api/dimension/dimension_api.cc +++ b/tiledb/api/c_api/dimension/dimension_api.cc @@ -141,86 +141,98 @@ int32_t tiledb_dimension_dump(const tiledb_dimension_t* dim, FILE* out) { using tiledb::api::api_entry_context; -int32_t tiledb_dimension_alloc( +CAPI_INTERFACE( + dimension_alloc, tiledb_ctx_t* ctx, const char* name, tiledb_datatype_t type, const void* dim_domain, const void* tile_extent, - tiledb_dimension_t** dim) noexcept { + tiledb_dimension_t** dim) { return api_entry_context( ctx, name, type, dim_domain, tile_extent, dim); } -void tiledb_dimension_free(tiledb_dimension_t** dim) noexcept { +CAPI_INTERFACE_VOID(dimension_free, tiledb_dimension_t** dim) { return tiledb::api::api_entry_void(dim); } -int32_t tiledb_dimension_set_filter_list( +CAPI_INTERFACE( + dimension_set_filter_list, tiledb_ctx_t* ctx, tiledb_dimension_t* dim, - tiledb_filter_list_t* filter_list) noexcept { + tiledb_filter_list_t* filter_list) { return api_entry_context( ctx, dim, filter_list); } -int32_t tiledb_dimension_set_cell_val_num( +CAPI_INTERFACE( + dimension_set_cell_val_num, tiledb_ctx_t* ctx, tiledb_dimension_t* dim, - uint32_t cell_val_num) noexcept { + uint32_t cell_val_num) { return api_entry_context( ctx, dim, cell_val_num); } -int32_t tiledb_dimension_get_filter_list( +CAPI_INTERFACE( + dimension_get_filter_list, tiledb_ctx_t* ctx, tiledb_dimension_t* dim, - tiledb_filter_list_t** filter_list) noexcept { + tiledb_filter_list_t** filter_list) { return api_entry_context( ctx, dim, filter_list); } -int32_t tiledb_dimension_get_cell_val_num( +CAPI_INTERFACE( + dimension_get_cell_val_num, tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, - uint32_t* cell_val_num) noexcept { + uint32_t* cell_val_num) { return api_entry_context( ctx, dim, cell_val_num); } -int32_t tiledb_dimension_get_name( +CAPI_INTERFACE( + dimension_get_name, tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, - const char** name) noexcept { + const char** name) { return api_entry_context( ctx, dim, name); } -int32_t tiledb_dimension_get_type( +CAPI_INTERFACE( + dimension_get_type, tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, - tiledb_datatype_t* type) noexcept { + tiledb_datatype_t* type) { return api_entry_context( ctx, dim, type); } -int32_t tiledb_dimension_get_domain( +CAPI_INTERFACE( + dimension_get_domain, tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, - const void** domain) noexcept { + const void** domain) { return api_entry_context( ctx, dim, domain); } -int32_t tiledb_dimension_get_tile_extent( +CAPI_INTERFACE( + dimension_get_tile_extent, tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, - const void** tile_extent) noexcept { + const void** tile_extent) { return api_entry_context( ctx, dim, tile_extent); } -int32_t tiledb_dimension_dump( - tiledb_ctx_t* ctx, const tiledb_dimension_t* dim, FILE* out) noexcept { +CAPI_INTERFACE( + dimension_dump, + tiledb_ctx_t* ctx, + const tiledb_dimension_t* dim, + FILE* out) { return api_entry_context(ctx, dim, out); } diff --git a/tiledb/api/c_api/dimension_label/dimension_label_api.cc b/tiledb/api/c_api/dimension_label/dimension_label_api.cc index 1476596066b4..41e348ef63b6 100644 --- a/tiledb/api/c_api/dimension_label/dimension_label_api.cc +++ b/tiledb/api/c_api/dimension_label/dimension_label_api.cc @@ -105,66 +105,73 @@ capi_return_t tiledb_dimension_label_get_uri( using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; -void tiledb_dimension_label_free( - tiledb_dimension_label_t** dim_label) noexcept { +CAPI_INTERFACE_VOID( + dimension_label_free, tiledb_dimension_label_t** dim_label) { return api_entry_void(dim_label); } -capi_return_t tiledb_dimension_label_get_dimension_index( +CAPI_INTERFACE( + dimension_label_get_dimension_index, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - uint32_t* dim_index) noexcept { + uint32_t* dim_index) { return api_entry_context< tiledb::api::tiledb_dimension_label_get_dimension_index>( ctx, dim_label, dim_index); } -capi_return_t tiledb_dimension_label_get_label_attr_name( +CAPI_INTERFACE( + dimension_label_get_label_attr_name, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - const char** label_attr_name) noexcept { + const char** label_attr_name) { return api_entry_context< tiledb::api::tiledb_dimension_label_get_label_attr_name>( ctx, dim_label, label_attr_name); } -capi_return_t tiledb_dimension_label_get_label_cell_val_num( +CAPI_INTERFACE( + dimension_label_get_label_cell_val_num, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - uint32_t* label_cell_val_num) noexcept { + uint32_t* label_cell_val_num) { return api_entry_context< tiledb::api::tiledb_dimension_label_get_label_cell_val_num>( ctx, dim_label, label_cell_val_num); } -capi_return_t tiledb_dimension_label_get_label_order( +CAPI_INTERFACE( + dimension_label_get_label_order, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - tiledb_data_order_t* label_order) noexcept { + tiledb_data_order_t* label_order) { return api_entry_context( ctx, dim_label, label_order); } -capi_return_t tiledb_dimension_label_get_label_type( +CAPI_INTERFACE( + dimension_label_get_label_type, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - tiledb_datatype_t* label_type) noexcept { + tiledb_datatype_t* label_type) { return api_entry_context( ctx, dim_label, label_type); } -capi_return_t tiledb_dimension_label_get_name( +CAPI_INTERFACE( + dimension_label_get_name, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - const char** name) noexcept { + const char** name) { return api_entry_context( ctx, dim_label, name); } -capi_return_t tiledb_dimension_label_get_uri( +CAPI_INTERFACE( + dimension_label_get_uri, tiledb_ctx_t* ctx, tiledb_dimension_label_t* dim_label, - const char** uri) noexcept { + const char** uri) { return api_entry_context( ctx, dim_label, uri); } diff --git a/tiledb/api/c_api/domain/domain_api.cc b/tiledb/api/c_api/domain/domain_api.cc index f942e7e25f9a..3a694a97c117 100644 --- a/tiledb/api/c_api/domain/domain_api.cc +++ b/tiledb/api/c_api/domain/domain_api.cc @@ -146,65 +146,72 @@ int32_t tiledb_domain_dump(const tiledb_domain_t* domain, FILE* out) { using tiledb::api::api_entry_context; -int32_t tiledb_domain_alloc( - tiledb_ctx_t* ctx, tiledb_domain_t** domain) noexcept { +CAPI_INTERFACE(domain_alloc, tiledb_ctx_t* ctx, tiledb_domain_t** domain) { return api_entry_context(ctx, domain); } -void tiledb_domain_free(tiledb_domain_t** domain) noexcept { +CAPI_INTERFACE_VOID(domain_free, tiledb_domain_t** domain) { return tiledb::api::api_entry_void(domain); } -int32_t tiledb_domain_get_type( +CAPI_INTERFACE( + domain_get_type, tiledb_ctx_t* ctx, const tiledb_domain_t* domain, - tiledb_datatype_t* type) noexcept { + tiledb_datatype_t* type) { return api_entry_context( ctx, domain, type); } -int32_t tiledb_domain_get_ndim( - tiledb_ctx_t* ctx, const tiledb_domain_t* domain, uint32_t* ndim) noexcept { +CAPI_INTERFACE( + domain_get_ndim, + tiledb_ctx_t* ctx, + const tiledb_domain_t* domain, + uint32_t* ndim) { return api_entry_context( ctx, domain, ndim); } -int32_t tiledb_domain_add_dimension( +CAPI_INTERFACE( + domain_add_dimension, tiledb_ctx_t* ctx, tiledb_domain_t* domain, - tiledb_dimension_t* dim) noexcept { + tiledb_dimension_t* dim) { return api_entry_context( ctx, domain, dim); } -int32_t tiledb_domain_get_dimension_from_index( +CAPI_INTERFACE( + domain_get_dimension_from_index, tiledb_ctx_t* ctx, const tiledb_domain_t* domain, uint32_t index, - tiledb_dimension_t** dim) noexcept { + tiledb_dimension_t** dim) { return api_entry_context( ctx, domain, index, dim); } -int32_t tiledb_domain_get_dimension_from_name( +CAPI_INTERFACE( + domain_get_dimension_from_name, tiledb_ctx_t* ctx, const tiledb_domain_t* domain, const char* name, - tiledb_dimension_t** dim) noexcept { + tiledb_dimension_t** dim) { return api_entry_context( ctx, domain, name, dim); } -int32_t tiledb_domain_has_dimension( +CAPI_INTERFACE( + domain_has_dimension, tiledb_ctx_t* ctx, const tiledb_domain_t* domain, const char* name, - int32_t* has_dim) noexcept { + int32_t* has_dim) { return api_entry_context( ctx, domain, name, has_dim); } -int32_t tiledb_domain_dump( - tiledb_ctx_t* ctx, const tiledb_domain_t* domain, FILE* out) noexcept { +CAPI_INTERFACE( + domain_dump, tiledb_ctx_t* ctx, const tiledb_domain_t* domain, FILE* out) { return api_entry_context(ctx, domain, out); } diff --git a/tiledb/api/c_api/enumeration/enumeration_api.cc b/tiledb/api/c_api/enumeration/enumeration_api.cc index 73283621af2d..d73759bef31c 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api.cc +++ b/tiledb/api/c_api/enumeration/enumeration_api.cc @@ -179,7 +179,8 @@ capi_return_t tiledb_enumeration_dump( using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; -capi_return_t tiledb_enumeration_alloc( +CAPI_INTERFACE( + enumeration_alloc, tiledb_ctx_t* ctx, const char* name, tiledb_datatype_t type, @@ -189,7 +190,7 @@ capi_return_t tiledb_enumeration_alloc( uint64_t data_size, const void* offsets, uint64_t offsets_size, - tiledb_enumeration_t** enumeration) noexcept { + tiledb_enumeration_t** enumeration) { return api_entry_context( ctx, name, @@ -203,14 +204,15 @@ capi_return_t tiledb_enumeration_alloc( enumeration); } -capi_return_t tiledb_enumeration_extend( +CAPI_INTERFACE( + enumeration_extend, tiledb_ctx_t* ctx, tiledb_enumeration_t* old_enumeration, const void* data, uint64_t data_size, const void* offsets, uint64_t offsets_size, - tiledb_enumeration_t** new_enumeration) noexcept { + tiledb_enumeration_t** new_enumeration) { return api_entry_context( ctx, old_enumeration, @@ -221,62 +223,71 @@ capi_return_t tiledb_enumeration_extend( new_enumeration); } -void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) noexcept { +CAPI_INTERFACE_VOID(enumeration_free, tiledb_enumeration_t** enumeration) { return api_entry_void(enumeration); } -capi_return_t tiledb_enumeration_get_name( +CAPI_INTERFACE( + enumeration_get_name, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, - tiledb_string_t** name) noexcept { + tiledb_string_t** name) { return api_entry_context( ctx, enumeration, name); } -capi_return_t tiledb_enumeration_get_type( +CAPI_INTERFACE( + enumeration_get_type, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, - tiledb_datatype_t* type) noexcept { + tiledb_datatype_t* type) { return api_entry_context( ctx, enumeration, type); } -capi_return_t tiledb_enumeration_get_cell_val_num( +CAPI_INTERFACE( + enumeration_get_cell_val_num, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, - uint32_t* cell_val_num) noexcept { + uint32_t* cell_val_num) { return api_entry_context( ctx, enumeration, cell_val_num); } -capi_return_t tiledb_enumeration_get_ordered( +CAPI_INTERFACE( + enumeration_get_ordered, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, - int* ordered) noexcept { + int* ordered) { return api_entry_context( ctx, enumeration, ordered); } -capi_return_t tiledb_enumeration_get_data( +CAPI_INTERFACE( + enumeration_get_data, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, const void** data, - uint64_t* data_size) noexcept { + uint64_t* data_size) { return api_entry_context( ctx, enumeration, data, data_size); } -capi_return_t tiledb_enumeration_get_offsets( +CAPI_INTERFACE( + enumeration_get_offsets, tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, const void** offsets, - uint64_t* offsets_size) noexcept { + uint64_t* offsets_size) { return api_entry_context( ctx, enumeration, offsets, offsets_size); } -capi_return_t tiledb_enumeration_dump( - tiledb_ctx_t* ctx, tiledb_enumeration_t* enumeration, FILE* out) noexcept { +CAPI_INTERFACE( + enumeration_dump, + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + FILE* out) { return api_entry_context( ctx, enumeration, out); } diff --git a/tiledb/api/c_api/filesystem/filesystem_api.cc b/tiledb/api/c_api/filesystem/filesystem_api.cc index 932f3316037b..2fab52108d10 100644 --- a/tiledb/api/c_api/filesystem/filesystem_api.cc +++ b/tiledb/api/c_api/filesystem/filesystem_api.cc @@ -58,14 +58,14 @@ capi_return_t tiledb_filesystem_from_str( using tiledb::api::api_entry_plain; -capi_return_t tiledb_filesystem_to_str( - tiledb_filesystem_t filesystem, const char** str) noexcept { +CAPI_INTERFACE( + filesystem_to_str, tiledb_filesystem_t filesystem, const char** str) { return api_entry_plain( filesystem, str); } -capi_return_t tiledb_filesystem_from_str( - const char* str, tiledb_filesystem_t* filesystem) noexcept { +CAPI_INTERFACE( + filesystem_from_str, const char* str, tiledb_filesystem_t* filesystem) { return api_entry_plain( str, filesystem); } diff --git a/tiledb/api/c_api/filter/filter_api.cc b/tiledb/api/c_api/filter/filter_api.cc index 33c4179acf25..392e1ffb2bb5 100644 --- a/tiledb/api/c_api/filter/filter_api.cc +++ b/tiledb/api/c_api/filter/filter_api.cc @@ -122,64 +122,72 @@ capi_return_t tiledb_filter_option_from_str( using tiledb::api::api_entry_context; using tiledb::api::api_entry_plain; -capi_return_t tiledb_filter_alloc( +CAPI_INTERFACE( + filter_alloc, tiledb_ctx_t* ctx, tiledb_filter_type_t type, - tiledb_filter_t** filter) noexcept { + tiledb_filter_t** filter) { return tiledb::api::api_entry_with_context( ctx, type, filter); } -void tiledb_filter_free(tiledb_filter_t** filter) noexcept { +CAPI_INTERFACE_VOID(filter_free, tiledb_filter_t** filter) { return tiledb::api::api_entry_void(filter); } -capi_return_t tiledb_filter_get_type( +CAPI_INTERFACE( + filter_get_type, tiledb_ctx_t* ctx, tiledb_filter_t* filter, - tiledb_filter_type_t* type) noexcept { + tiledb_filter_type_t* type) { return api_entry_context( ctx, filter, type); } -capi_return_t tiledb_filter_set_option( +CAPI_INTERFACE( + filter_set_option, tiledb_ctx_t* ctx, tiledb_filter_t* filter, tiledb_filter_option_t option, - const void* value) noexcept { + const void* value) { return api_entry_context( ctx, filter, option, value); } -capi_return_t tiledb_filter_get_option( +CAPI_INTERFACE( + filter_get_option, tiledb_ctx_t* ctx, tiledb_filter_t* filter, tiledb_filter_option_t option, - void* value) noexcept { + void* value) { return api_entry_context( ctx, filter, option, value); } -capi_return_t tiledb_filter_type_to_str( - tiledb_filter_type_t filter_type, const char** str) noexcept { +CAPI_INTERFACE( + filter_type_to_str, tiledb_filter_type_t filter_type, const char** str) { return api_entry_plain( filter_type, str); } -capi_return_t tiledb_filter_type_from_str( - const char* str, tiledb_filter_type_t* filter_type) noexcept { +CAPI_INTERFACE( + filter_type_from_str, const char* str, tiledb_filter_type_t* filter_type) { return api_entry_plain( str, filter_type); } -capi_return_t tiledb_filter_option_to_str( - tiledb_filter_option_t filter_option, const char** str) noexcept { +CAPI_INTERFACE( + filter_option_to_str, + tiledb_filter_option_t filter_option, + const char** str) { return api_entry_plain( filter_option, str); } -capi_return_t tiledb_filter_option_from_str( - const char* str, tiledb_filter_option_t* filter_option) noexcept { +CAPI_INTERFACE( + filter_option_from_str, + const char* str, + tiledb_filter_option_t* filter_option) { return api_entry_plain( str, filter_option); } diff --git a/tiledb/api/c_api/filter_list/filter_list_api.cc b/tiledb/api/c_api/filter_list/filter_list_api.cc index 3eb65a006235..3340059a295f 100644 --- a/tiledb/api/c_api/filter_list/filter_list_api.cc +++ b/tiledb/api/c_api/filter_list/filter_list_api.cc @@ -108,55 +108,60 @@ capi_return_t tiledb_filter_list_get_max_chunk_size( using tiledb::api::api_entry_context; -capi_return_t tiledb_filter_list_alloc( - tiledb_ctx_t* ctx, tiledb_filter_list_t** filter_list) noexcept { +CAPI_INTERFACE( + filter_list_alloc, tiledb_ctx_t* ctx, tiledb_filter_list_t** filter_list) { return tiledb::api::api_entry_with_context< tiledb::api::tiledb_filter_list_alloc>(ctx, filter_list); } -void tiledb_filter_list_free(tiledb_filter_list_t** filter_list) noexcept { +CAPI_INTERFACE_VOID(filter_list_free, tiledb_filter_list_t** filter_list) { return tiledb::api::api_entry_void( filter_list); } -capi_return_t tiledb_filter_list_add_filter( +CAPI_INTERFACE( + filter_list_add_filter, tiledb_ctx_t* ctx, tiledb_filter_list_t* filter_list, - tiledb_filter_t* filter) noexcept { + tiledb_filter_t* filter) { return api_entry_context( ctx, filter_list, filter); } -capi_return_t tiledb_filter_list_set_max_chunk_size( +CAPI_INTERFACE( + filter_list_set_max_chunk_size, tiledb_ctx_t* ctx, tiledb_filter_list_t* filter_list, - uint32_t max_chunk_size) noexcept { + uint32_t max_chunk_size) { return api_entry_context( ctx, filter_list, max_chunk_size); } -capi_return_t tiledb_filter_list_get_nfilters( +CAPI_INTERFACE( + filter_list_get_nfilters, tiledb_ctx_t* ctx, const tiledb_filter_list_t* filter_list, - uint32_t* nfilters) noexcept { + uint32_t* nfilters) { return api_entry_context( ctx, filter_list, nfilters); } -capi_return_t tiledb_filter_list_get_filter_from_index( +CAPI_INTERFACE( + filter_list_get_filter_from_index, tiledb_ctx_t* ctx, const tiledb_filter_list_t* filter_list, uint32_t index, - tiledb_filter_t** filter) noexcept { + tiledb_filter_t** filter) { return api_entry_context< tiledb::api::tiledb_filter_list_get_filter_from_index>( ctx, filter_list, index, filter); } -capi_return_t tiledb_filter_list_get_max_chunk_size( +CAPI_INTERFACE( + filter_list_get_max_chunk_size, tiledb_ctx_t* ctx, const tiledb_filter_list_t* filter_list, - uint32_t* max_chunk_size) noexcept { + uint32_t* max_chunk_size) { return api_entry_context( ctx, filter_list, max_chunk_size); } diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index e5a195ad3eb9..bf7b17ff2198 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -549,95 +549,109 @@ using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; using tiledb::api::api_entry_with_context; -capi_return_t tiledb_group_create( - tiledb_ctx_t* ctx, const char* group_uri) noexcept { +CAPI_INTERFACE(group_create, tiledb_ctx_t* ctx, const char* group_uri) { return api_entry_with_context( ctx, group_uri); } -capi_return_t tiledb_group_alloc( - tiledb_ctx_t* ctx, const char* group_uri, tiledb_group_t** group) noexcept { +CAPI_INTERFACE( + group_alloc, + tiledb_ctx_t* ctx, + const char* group_uri, + tiledb_group_t** group) { return api_entry_with_context( ctx, group_uri, group); } -capi_return_t tiledb_group_open( +CAPI_INTERFACE( + group_open, tiledb_ctx_t* ctx, tiledb_group_t* group, - tiledb_query_type_t query_type) noexcept { + tiledb_query_type_t query_type) { return api_entry_context( ctx, group, query_type); } -capi_return_t tiledb_group_close( - tiledb_ctx_t* ctx, tiledb_group_t* group) noexcept { +CAPI_INTERFACE(group_close, tiledb_ctx_t* ctx, tiledb_group_t* group) { return api_entry_context(ctx, group); } -void tiledb_group_free(tiledb_group_t** group) noexcept { +CAPI_INTERFACE_VOID(group_free, tiledb_group_t** group) { return api_entry_void(group); } -capi_return_t tiledb_group_set_config( +CAPI_INTERFACE( + group_set_config, tiledb_ctx_t* ctx, tiledb_group_t* group, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry_context( ctx, group, config); } -capi_return_t tiledb_group_get_config( +CAPI_INTERFACE( + group_get_config, tiledb_ctx_t* ctx, tiledb_group_t* group, - tiledb_config_t** config) noexcept { + tiledb_config_t** config) { return api_entry_context( ctx, group, config); } -capi_return_t tiledb_group_put_metadata( +CAPI_INTERFACE( + group_put_metadata, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key, tiledb_datatype_t value_type, uint32_t value_num, - const void* value) noexcept { + const void* value) { return api_entry_context( ctx, group, key, value_type, value_num, value); } -capi_return_t tiledb_group_delete_group( +CAPI_INTERFACE( + group_delete_group, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* uri, - const uint8_t recursive) noexcept { + const uint8_t recursive) { return api_entry_context( ctx, group, uri, recursive); } -capi_return_t tiledb_group_delete_metadata( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key) noexcept { +CAPI_INTERFACE( + group_delete_metadata, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* key) { return api_entry_context( ctx, group, key); } -capi_return_t tiledb_group_get_metadata( +CAPI_INTERFACE( + group_get_metadata, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key, tiledb_datatype_t* value_type, uint32_t* value_num, - const void** value) noexcept { + const void** value) { return api_entry_context( ctx, group, key, value_type, value_num, value); } -capi_return_t tiledb_group_get_metadata_num( - tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* num) noexcept { +CAPI_INTERFACE( + group_get_metadata_num, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t* num) { return api_entry_context( ctx, group, num); } -capi_return_t tiledb_group_get_metadata_from_index( +CAPI_INTERFACE( + group_get_metadata_from_index, tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t index, @@ -645,153 +659,175 @@ capi_return_t tiledb_group_get_metadata_from_index( uint32_t* key_len, tiledb_datatype_t* value_type, uint32_t* value_num, - const void** value) noexcept { + const void** value) { return api_entry_context( ctx, group, index, key, key_len, value_type, value_num, value); } -capi_return_t tiledb_group_has_metadata_key( +CAPI_INTERFACE( + group_has_metadata_key, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key, tiledb_datatype_t* value_type, - int32_t* has_key) noexcept { + int32_t* has_key) { return api_entry_context( ctx, group, key, value_type, has_key); } -capi_return_t tiledb_group_add_member( +CAPI_INTERFACE( + group_add_member, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* uri, const uint8_t relative, - const char* name) noexcept { + const char* name) { return api_entry_context( ctx, group, uri, relative, name); } -capi_return_t tiledb_group_remove_member( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name) noexcept { +CAPI_INTERFACE( + group_remove_member, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name) { return api_entry_context( ctx, group, name); } -capi_return_t tiledb_group_get_member_count( - tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* count) noexcept { +CAPI_INTERFACE( + group_get_member_count, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t* count) { return api_entry_context( ctx, group, count); } -capi_return_t tiledb_group_get_member_by_index( +CAPI_INTERFACE( + group_get_member_by_index, tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t index, char** uri, tiledb_object_t* type, - char** name) noexcept { + char** name) { return api_entry_context( ctx, group, index, uri, type, name); } -capi_return_t tiledb_group_get_member_by_name( +CAPI_INTERFACE( + group_get_member_by_name, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name, char** uri, - tiledb_object_t* type) noexcept { + tiledb_object_t* type) { return api_entry_context( ctx, group, name, uri, type); } -capi_return_t tiledb_group_get_is_relative_uri_by_name( +CAPI_INTERFACE( + group_get_is_relative_uri_by_name, tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name, - uint8_t* relative) noexcept { + uint8_t* relative) { return api_entry_context< tiledb::api::tiledb_group_get_is_relative_uri_by_name>( ctx, group, name, relative); } -capi_return_t tiledb_group_is_open( - tiledb_ctx_t* ctx, tiledb_group_t* group, int32_t* is_open) noexcept { +CAPI_INTERFACE( + group_is_open, tiledb_ctx_t* ctx, tiledb_group_t* group, int32_t* is_open) { return api_entry_context( ctx, group, is_open); } -capi_return_t tiledb_group_get_uri( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char** group_uri) noexcept { +CAPI_INTERFACE( + group_get_uri, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char** group_uri) { return api_entry_context( ctx, group, group_uri); } -capi_return_t tiledb_group_get_query_type( +CAPI_INTERFACE( + group_get_query_type, tiledb_ctx_t* ctx, tiledb_group_t* group, - tiledb_query_type_t* query_type) noexcept { + tiledb_query_type_t* query_type) { return api_entry_context( ctx, group, query_type); } -capi_return_t tiledb_group_dump_str( +CAPI_INTERFACE( + group_dump_str, tiledb_ctx_t* ctx, tiledb_group_t* group, char** dump_ascii, - const uint8_t recursive) noexcept { + const uint8_t recursive) { return api_entry_context( ctx, group, dump_ascii, recursive); } -capi_return_t tiledb_serialize_group( +CAPI_INTERFACE( + serialize_group, tiledb_ctx_t* ctx, const tiledb_group_t* group, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer_list) noexcept { + tiledb_buffer_t** buffer_list) { return api_entry_context( ctx, group, serialize_type, client_side, buffer_list); } -capi_return_t tiledb_deserialize_group( +CAPI_INTERFACE( + deserialize_group, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_group_t* group) noexcept { + tiledb_group_t* group) { return api_entry_context( ctx, buffer, serialize_type, client_side, group); } -capi_return_t tiledb_serialize_group_metadata( +CAPI_INTERFACE( + serialize_group_metadata, tiledb_ctx_t* ctx, const tiledb_group_t* group, tiledb_serialization_type_t serialization_type, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry_context( ctx, group, serialization_type, buffer); } -capi_return_t tiledb_deserialize_group_metadata( +CAPI_INTERFACE( + deserialize_group_metadata, tiledb_ctx_t* ctx, tiledb_group_t* group, tiledb_serialization_type_t serialization_type, - const tiledb_buffer_t* buffer) noexcept { + const tiledb_buffer_t* buffer) { return api_entry_context( ctx, group, serialization_type, buffer); } -capi_return_t tiledb_group_consolidate_metadata( +CAPI_INTERFACE( + group_consolidate_metadata, tiledb_ctx_t* ctx, const char* group_uri, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry_with_context( ctx, group_uri, config); } -capi_return_t tiledb_group_vacuum_metadata( +CAPI_INTERFACE( + group_vacuum_metadata, tiledb_ctx_t* ctx, const char* group_uri, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry_with_context( ctx, group_uri, config); } diff --git a/tiledb/api/c_api/object/object_api.cc b/tiledb/api/c_api/object/object_api.cc index 1fb3d9de9b1f..1fdae8ab6f23 100644 --- a/tiledb/api/c_api/object/object_api.cc +++ b/tiledb/api/c_api/object/object_api.cc @@ -88,26 +88,26 @@ capi_return_t tiledb_walk_order_from_str( using tiledb::api::api_entry_plain; using tiledb::api::api_entry_with_context; -capi_return_t tiledb_object_type_to_str( - tiledb_object_t object_type, const char** str) noexcept { +CAPI_INTERFACE( + object_type_to_str, tiledb_object_t object_type, const char** str) { return api_entry_plain( object_type, str); } -capi_return_t tiledb_object_type_from_str( - const char* str, tiledb_object_t* object_type) noexcept { +CAPI_INTERFACE( + object_type_from_str, const char* str, tiledb_object_t* object_type) { return api_entry_plain( str, object_type); } -capi_return_t tiledb_walk_order_to_str( - tiledb_walk_order_t walk_order, const char** str) noexcept { +CAPI_INTERFACE( + walk_order_to_str, tiledb_walk_order_t walk_order, const char** str) { return api_entry_plain( walk_order, str); } -capi_return_t tiledb_walk_order_from_str( - const char* str, tiledb_walk_order_t* walk_order) noexcept { +CAPI_INTERFACE( + walk_order_from_str, const char* str, tiledb_walk_order_t* walk_order) { return api_entry_plain( str, walk_order); } diff --git a/tiledb/api/c_api/query/query_api.cc b/tiledb/api/c_api/query/query_api.cc index 58f30c062756..496bec15014a 100644 --- a/tiledb/api/c_api/query/query_api.cc +++ b/tiledb/api/c_api/query/query_api.cc @@ -57,14 +57,14 @@ int32_t tiledb_query_type_from_str( using tiledb::api::api_entry_plain; -int32_t tiledb_query_type_to_str( - tiledb_query_type_t query_type, const char** str) noexcept { +CAPI_INTERFACE( + query_type_to_str, tiledb_query_type_t query_type, const char** str) { return api_entry_plain( query_type, str); } -int32_t tiledb_query_type_from_str( - const char* str, tiledb_query_type_t* query_type) noexcept { +CAPI_INTERFACE( + query_type_from_str, const char* str, tiledb_query_type_t* query_type) { return api_entry_plain( str, query_type); } diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index c51f76e31d73..e61dede8383b 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -269,77 +269,92 @@ tiledb_channel_operator_handle_t::make_operation( using tiledb::api::api_entry_with_context; -capi_return_t tiledb_channel_operator_sum_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { +CAPI_INTERFACE( + channel_operator_sum_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operator_t** op) { return api_entry_with_context( ctx, op); } -capi_return_t tiledb_channel_operator_mean_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { +CAPI_INTERFACE( + channel_operator_mean_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operator_t** op) { return api_entry_with_context( ctx, op); } -capi_return_t tiledb_channel_operator_min_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { +CAPI_INTERFACE( + channel_operator_min_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operator_t** op) { return api_entry_with_context( ctx, op); } -capi_return_t tiledb_channel_operator_max_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { +CAPI_INTERFACE( + channel_operator_max_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operator_t** op) { return api_entry_with_context( ctx, op); } -capi_return_t tiledb_channel_operator_null_count_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) noexcept { +CAPI_INTERFACE( + channel_operator_null_count_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operator_t** op) { return api_entry_with_context< tiledb::api::tiledb_channel_operator_null_count_get>(ctx, op); } -capi_return_t tiledb_aggregate_count_get( - tiledb_ctx_t* ctx, const tiledb_channel_operation_t** operation) noexcept { +CAPI_INTERFACE( + aggregate_count_get, + tiledb_ctx_t* ctx, + const tiledb_channel_operation_t** operation) { return api_entry_with_context( ctx, operation); } -capi_return_t tiledb_query_get_default_channel( +CAPI_INTERFACE( + query_get_default_channel, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_query_channel_t** channel) noexcept { + tiledb_query_channel_t** channel) { return api_entry_with_context( ctx, query, channel); } -capi_return_t tiledb_create_unary_aggregate( +CAPI_INTERFACE( + create_unary_aggregate, tiledb_ctx_t* ctx, tiledb_query_t* query, const tiledb_channel_operator_t* op, const char* input_field_name, - tiledb_channel_operation_t** operation) noexcept { + tiledb_channel_operation_t** operation) { return api_entry_with_context( ctx, query, op, input_field_name, operation); } -capi_return_t tiledb_channel_apply_aggregate( +CAPI_INTERFACE( + channel_apply_aggregate, tiledb_ctx_t* ctx, tiledb_query_channel_t* channel, const char* output_field_name, - const tiledb_channel_operation_t* operation) noexcept { + const tiledb_channel_operation_t* operation) { return api_entry_with_context( ctx, channel, output_field_name, operation); } -capi_return_t tiledb_aggregate_free( - tiledb_ctx_t* ctx, tiledb_channel_operation_t** operation) noexcept { +CAPI_INTERFACE( + aggregate_free, tiledb_ctx_t* ctx, tiledb_channel_operation_t** operation) { return tiledb::api::api_entry_with_context< tiledb::api::tiledb_aggregate_free>(ctx, operation); } -capi_return_t tiledb_query_channel_free( - tiledb_ctx_t* ctx, tiledb_query_channel_t** channel) noexcept { +CAPI_INTERFACE( + query_channel_free, tiledb_ctx_t* ctx, tiledb_query_channel_t** channel) { return tiledb::api::api_entry_with_context< tiledb::api::tiledb_query_channel_free>(ctx, channel); } diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc index 78355dfee2c8..c6b88154c041 100644 --- a/tiledb/api/c_api/query_field/query_field_api.cc +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -161,48 +161,53 @@ capi_return_t tiledb_field_channel( using tiledb::api::api_entry_context; -capi_return_t tiledb_query_get_field( +CAPI_INTERFACE( + query_get_field, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* field_name, - tiledb_query_field_t** field) noexcept { + tiledb_query_field_t** field) { return api_entry_context( ctx, query, field_name, field); } -capi_return_t tiledb_query_field_free( - tiledb_ctx_t* ctx, tiledb_query_field_t** field) noexcept { +CAPI_INTERFACE( + query_field_free, tiledb_ctx_t* ctx, tiledb_query_field_t** field) { return api_entry_context(ctx, field); } -capi_return_t tiledb_field_datatype( +CAPI_INTERFACE( + field_datatype, tiledb_ctx_t* ctx, tiledb_query_field_t* field, - tiledb_datatype_t* type) noexcept { + tiledb_datatype_t* type) { return api_entry_context( ctx, field, type); } -capi_return_t tiledb_field_cell_val_num( +CAPI_INTERFACE( + field_cell_val_num, tiledb_ctx_t* ctx, tiledb_query_field_t* field, - uint32_t* cell_val_num) noexcept { + uint32_t* cell_val_num) { return api_entry_context( ctx, field, cell_val_num); } -capi_return_t tiledb_field_origin( +CAPI_INTERFACE( + field_origin, tiledb_ctx_t* ctx, tiledb_query_field_t* field, - tiledb_field_origin_t* origin) noexcept { + tiledb_field_origin_t* origin) { return api_entry_context( ctx, field, origin); } -capi_return_t tiledb_field_channel( +CAPI_INTERFACE( + field_channel, tiledb_ctx_t* ctx, tiledb_query_field_t* field, - tiledb_query_channel_handle_t** channel) noexcept { + tiledb_query_channel_handle_t** channel) { return api_entry_context( ctx, field, channel); } diff --git a/tiledb/api/c_api/query_plan/query_plan_api.cc b/tiledb/api/c_api/query_plan/query_plan_api.cc index 9d622c583f05..e9d7bbfd3267 100644 --- a/tiledb/api/c_api/query_plan/query_plan_api.cc +++ b/tiledb/api/c_api/query_plan/query_plan_api.cc @@ -59,10 +59,11 @@ capi_return_t tiledb_query_get_plan( using tiledb::api::api_entry_with_context; -capi_return_t tiledb_query_get_plan( +CAPI_INTERFACE( + query_get_plan, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_string_handle_t** rv) noexcept { + tiledb_string_handle_t** rv) { return api_entry_with_context( ctx, query, rv); } diff --git a/tiledb/api/c_api/string/string_api.cc b/tiledb/api/c_api/string/string_api.cc index 21ba1c00e5ab..060124559810 100644 --- a/tiledb/api/c_api/string/string_api.cc +++ b/tiledb/api/c_api/string/string_api.cc @@ -56,12 +56,12 @@ capi_return_t tiledb_string_free(tiledb_string_handle_t** s) { } // namespace tiledb::api -capi_return_t tiledb_string_view( - tiledb_string_t* s, const char** data, size_t* length) noexcept { +CAPI_INTERFACE( + string_view, tiledb_string_t* s, const char** data, size_t* length) { return tiledb::api::api_entry_plain( s, data, length); } -capi_return_t tiledb_string_free(tiledb_string_handle_t** s) noexcept { +CAPI_INTERFACE(string_free, tiledb_string_handle_t** s) { return tiledb::api::api_entry_plain(s); } diff --git a/tiledb/api/c_api/vfs/vfs_api.cc b/tiledb/api/c_api/vfs/vfs_api.cc index 521782e69e9f..04144755f0b9 100644 --- a/tiledb/api/c_api/vfs/vfs_api.cc +++ b/tiledb/api/c_api/vfs/vfs_api.cc @@ -316,213 +316,231 @@ capi_return_t tiledb_vfs_touch(tiledb_vfs_t* vfs, const char* uri) { using tiledb::api::api_entry_context; using tiledb::api::api_entry_plain; -capi_return_t tiledb_vfs_mode_to_str( - tiledb_vfs_mode_t vfs_mode, const char** str) noexcept { +CAPI_INTERFACE(vfs_mode_to_str, tiledb_vfs_mode_t vfs_mode, const char** str) { return api_entry_plain(vfs_mode, str); } -capi_return_t tiledb_vfs_mode_from_str( - const char* str, tiledb_vfs_mode_t* vfs_mode) noexcept { +CAPI_INTERFACE( + vfs_mode_from_str, const char* str, tiledb_vfs_mode_t* vfs_mode) { return api_entry_plain(str, vfs_mode); } -capi_return_t tiledb_vfs_alloc( - tiledb_ctx_t* ctx, tiledb_config_t* config, tiledb_vfs_t** vfs) noexcept { +CAPI_INTERFACE( + vfs_alloc, tiledb_ctx_t* ctx, tiledb_config_t* config, tiledb_vfs_t** vfs) { return tiledb::api::api_entry_with_context( ctx, config, vfs); } -void tiledb_vfs_free(tiledb_vfs_t** vfs) noexcept { +CAPI_INTERFACE_VOID(vfs_free, tiledb_vfs_t** vfs) { return tiledb::api::api_entry_void(vfs); } -capi_return_t tiledb_vfs_get_config( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, tiledb_config_t** config) noexcept { +CAPI_INTERFACE( + vfs_get_config, + tiledb_ctx_t* ctx, + tiledb_vfs_t* vfs, + tiledb_config_t** config) { return api_entry_context( ctx, vfs, config); } -capi_return_t tiledb_vfs_create_bucket( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_create_bucket, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context( ctx, vfs, uri); } -capi_return_t tiledb_vfs_remove_bucket( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_remove_bucket, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context( ctx, vfs, uri); } -capi_return_t tiledb_vfs_empty_bucket( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_empty_bucket, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } -capi_return_t tiledb_vfs_is_empty_bucket( +CAPI_INTERFACE( + vfs_is_empty_bucket, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry_context( ctx, vfs, uri, is_empty); } -capi_return_t tiledb_vfs_is_bucket( +CAPI_INTERFACE( + vfs_is_bucket, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - int32_t* is_bucket) noexcept { + int32_t* is_bucket) { return api_entry_context( ctx, vfs, uri, is_bucket); } -capi_return_t tiledb_vfs_create_dir( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_create_dir, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } -capi_return_t tiledb_vfs_is_dir( +CAPI_INTERFACE( + vfs_is_dir, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - int32_t* is_dir) noexcept { + int32_t* is_dir) { return api_entry_context( ctx, vfs, uri, is_dir); } -capi_return_t tiledb_vfs_remove_dir( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_remove_dir, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } -capi_return_t tiledb_vfs_is_file( +CAPI_INTERFACE( + vfs_is_file, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - int32_t* is_file) noexcept { + int32_t* is_file) { return api_entry_context( ctx, vfs, uri, is_file); } -capi_return_t tiledb_vfs_remove_file( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_remove_file, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } -capi_return_t tiledb_vfs_dir_size( +CAPI_INTERFACE( + vfs_dir_size, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry_context( ctx, vfs, uri, size); } -capi_return_t tiledb_vfs_file_size( +CAPI_INTERFACE( + vfs_file_size, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry_context( ctx, vfs, uri, size); } -capi_return_t tiledb_vfs_move_file( +CAPI_INTERFACE( + vfs_move_file, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* old_uri, - const char* new_uri) noexcept { + const char* new_uri) { return api_entry_context( ctx, vfs, old_uri, new_uri); } -capi_return_t tiledb_vfs_move_dir( +CAPI_INTERFACE( + vfs_move_dir, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* old_uri, - const char* new_uri) noexcept { + const char* new_uri) { return api_entry_context( ctx, vfs, old_uri, new_uri); } -capi_return_t tiledb_vfs_copy_file( +CAPI_INTERFACE( + vfs_copy_file, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* old_uri, - const char* new_uri) noexcept { + const char* new_uri) { return api_entry_context( ctx, vfs, old_uri, new_uri); } -capi_return_t tiledb_vfs_copy_dir( +CAPI_INTERFACE( + vfs_copy_dir, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* old_uri, - const char* new_uri) noexcept { + const char* new_uri) { return api_entry_context( ctx, vfs, old_uri, new_uri); } -capi_return_t tiledb_vfs_open( +CAPI_INTERFACE( + vfs_open, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri, tiledb_vfs_mode_t mode, - tiledb_vfs_fh_t** fh) noexcept { + tiledb_vfs_fh_t** fh) { return api_entry_context( ctx, vfs, uri, mode, fh); } -capi_return_t tiledb_vfs_close( - tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh) noexcept { +CAPI_INTERFACE(vfs_close, tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh) { return api_entry_context(ctx, fh); } -capi_return_t tiledb_vfs_read( +CAPI_INTERFACE( + vfs_read, tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh, uint64_t offset, void* buffer, - uint64_t nbytes) noexcept { + uint64_t nbytes) { return api_entry_context( ctx, fh, offset, buffer, nbytes); } -capi_return_t tiledb_vfs_write( +CAPI_INTERFACE( + vfs_write, tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh, const void* buffer, - uint64_t nbytes) noexcept { + uint64_t nbytes) { return api_entry_context( ctx, fh, buffer, nbytes); } -capi_return_t tiledb_vfs_sync(tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh) noexcept { +CAPI_INTERFACE(vfs_sync, tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh) { return api_entry_context(ctx, fh); } -capi_return_t tiledb_vfs_ls( +CAPI_INTERFACE( + vfs_ls, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* path, int32_t (*callback)(const char*, void*), - void* data) noexcept { + void* data) { return api_entry_context( ctx, vfs, path, callback, data); } -void tiledb_vfs_fh_free(tiledb_vfs_fh_t** fh) noexcept { +CAPI_INTERFACE_VOID(vfs_fh_free, tiledb_vfs_fh_t** fh) { return tiledb::api::api_entry_void(fh); } -capi_return_t tiledb_vfs_fh_is_closed( - tiledb_ctx_t* ctx, tiledb_vfs_fh_t* fh, int32_t* is_closed) noexcept { +CAPI_INTERFACE( + vfs_fh_is_closed, + tiledb_ctx_t* ctx, + tiledb_vfs_fh_t* fh, + int32_t* is_closed) { return api_entry_context( ctx, fh, is_closed); } -capi_return_t tiledb_vfs_touch( - tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) noexcept { +CAPI_INTERFACE( + vfs_touch, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } diff --git a/tiledb/api/c_api_support/exception_wrapper/capi_definition.h b/tiledb/api/c_api_support/exception_wrapper/capi_definition.h index cd45eb219cfd..619844622cde 100644 --- a/tiledb/api/c_api_support/exception_wrapper/capi_definition.h +++ b/tiledb/api/c_api_support/exception_wrapper/capi_definition.h @@ -56,6 +56,13 @@ capi_return_t tiledb_##root(__VA_ARGS__) noexcept CAPI_PREFIX(root) \ void tiledb_##root(__VA_ARGS__) noexcept +/* + * A variant of CAPI_INTERFACE for the handful of functions without arguments. + */ +#define CAPI_INTERFACE_NULL(root) \ +CAPI_PREFIX(root) \ +capi_return_t tiledb_##root(void) noexcept + /* clang-format on */ #endif // TILEDB_CAPI_DEFINITION_H diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 4c0d787d58a8..b4f5394211b2 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -5251,60 +5251,68 @@ constexpr auto api_entry = tiledb::api::api_entry_with_context; /* ****************************** */ /* ENUMS TO/FROM STR */ /* ****************************** */ -int32_t tiledb_array_type_to_str( - tiledb_array_type_t array_type, const char** str) noexcept { +CAPI_INTERFACE( + array_type_to_str, tiledb_array_type_t array_type, const char** str) { return api_entry_plain( array_type, str); } -int32_t tiledb_array_type_from_str( - const char* str, tiledb_array_type_t* array_type) noexcept { +CAPI_INTERFACE( + array_type_from_str, const char* str, tiledb_array_type_t* array_type) { return api_entry_plain( str, array_type); } -int32_t tiledb_layout_to_str( - tiledb_layout_t layout, const char** str) noexcept { +CAPI_INTERFACE(layout_to_str, tiledb_layout_t layout, const char** str) { return api_entry_plain(layout, str); } -int32_t tiledb_layout_from_str( - const char* str, tiledb_layout_t* layout) noexcept { +CAPI_INTERFACE(layout_from_str, const char* str, tiledb_layout_t* layout) { return api_entry_plain(str, layout); } -int32_t tiledb_encryption_type_to_str( - tiledb_encryption_type_t encryption_type, const char** str) noexcept { +CAPI_INTERFACE( + encryption_type_to_str, + tiledb_encryption_type_t encryption_type, + const char** str) { return api_entry_plain( encryption_type, str); } -int32_t tiledb_encryption_type_from_str( - const char* str, tiledb_encryption_type_t* encryption_type) noexcept { +CAPI_INTERFACE( + encryption_type_from_str, + const char* str, + tiledb_encryption_type_t* encryption_type) { return api_entry_plain( str, encryption_type); } -int32_t tiledb_query_status_to_str( - tiledb_query_status_t query_status, const char** str) noexcept { +CAPI_INTERFACE( + query_status_to_str, tiledb_query_status_t query_status, const char** str) { return api_entry_plain( query_status, str); } -int32_t tiledb_query_status_from_str( - const char* str, tiledb_query_status_t* query_status) noexcept { +CAPI_INTERFACE( + query_status_from_str, + const char* str, + tiledb_query_status_t* query_status) { return api_entry_plain( str, query_status); } -int32_t tiledb_serialization_type_to_str( - tiledb_serialization_type_t serialization_type, const char** str) noexcept { +CAPI_INTERFACE( + serialization_type_to_str, + tiledb_serialization_type_t serialization_type, + const char** str) { return api_entry_plain( serialization_type, str); } -int32_t tiledb_serialization_type_from_str( - const char* str, tiledb_serialization_type_t* serialization_type) noexcept { +CAPI_INTERFACE( + serialization_type_from_str, + const char* str, + tiledb_serialization_type_t* serialization_type) { return api_entry_plain( str, serialization_type); } @@ -5357,14 +5365,14 @@ void tiledb_version(int32_t* major, int32_t* minor, int32_t* rev) noexcept { /* LOGGING */ /* ********************************* */ -capi_return_t tiledb_log_warn(tiledb_ctx_t* ctx, const char* message) { +CAPI_INTERFACE(log_warn, tiledb_ctx_t* ctx, const char* message) { return api_entry(ctx, message); } /* ********************************* */ /* AS BUILT */ /* ********************************* */ -capi_return_t tiledb_as_built_dump(tiledb_string_t** out) noexcept { +CAPI_INTERFACE(as_built_dump, tiledb_string_t** out) { return api_entry_plain(out); } @@ -5372,143 +5380,161 @@ capi_return_t tiledb_as_built_dump(tiledb_string_t** out) noexcept { /* ARRAY SCHEMA */ /* ****************************** */ -int32_t tiledb_array_schema_alloc( +CAPI_INTERFACE( + array_schema_alloc, tiledb_ctx_t* ctx, tiledb_array_type_t array_type, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, array_type, array_schema); } -void tiledb_array_schema_free(tiledb_array_schema_t** array_schema) noexcept { +CAPI_INTERFACE_VOID(array_schema_free, tiledb_array_schema_t** array_schema) { return api_entry_void(array_schema); } -int32_t tiledb_array_schema_add_attribute( +CAPI_INTERFACE( + array_schema_add_attribute, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_attribute_t* attr) noexcept { + tiledb_attribute_t* attr) { return api_entry( ctx, array_schema, attr); } -int32_t tiledb_array_schema_set_allows_dups( +CAPI_INTERFACE( + array_schema_set_allows_dups, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - int allows_dups) noexcept { + int allows_dups) { return api_entry( ctx, array_schema, allows_dups); } -int32_t tiledb_array_schema_get_allows_dups( +CAPI_INTERFACE( + array_schema_get_allows_dups, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - int* allows_dups) noexcept { + int* allows_dups) { return api_entry( ctx, array_schema, allows_dups); } -int32_t tiledb_array_schema_get_version( +CAPI_INTERFACE( + array_schema_get_version, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - uint32_t* version) noexcept { + uint32_t* version) { return api_entry( ctx, array_schema, version); } -int32_t tiledb_array_schema_set_domain( +CAPI_INTERFACE( + array_schema_set_domain, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_domain_t* domain) noexcept { + tiledb_domain_t* domain) { return api_entry( ctx, array_schema, domain); } -int32_t tiledb_array_schema_set_capacity( +CAPI_INTERFACE( + array_schema_set_capacity, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - uint64_t capacity) noexcept { + uint64_t capacity) { return api_entry( ctx, array_schema, capacity); } -int32_t tiledb_array_schema_set_cell_order( +CAPI_INTERFACE( + array_schema_set_cell_order, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_layout_t cell_order) noexcept { + tiledb_layout_t cell_order) { return api_entry( ctx, array_schema, cell_order); } -int32_t tiledb_array_schema_set_tile_order( +CAPI_INTERFACE( + array_schema_set_tile_order, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_layout_t tile_order) noexcept { + tiledb_layout_t tile_order) { return api_entry( ctx, array_schema, tile_order); } -int32_t tiledb_array_schema_timestamp_range( +CAPI_INTERFACE( + array_schema_timestamp_range, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint64_t* lo, - uint64_t* hi) noexcept { + uint64_t* hi) { return api_entry( ctx, array_schema, lo, hi); } -int32_t tiledb_array_schema_add_enumeration( +CAPI_INTERFACE( + array_schema_add_enumeration, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_enumeration_t* enumeration) noexcept { + tiledb_enumeration_t* enumeration) { return api_entry( ctx, array_schema, enumeration); } -int32_t tiledb_array_schema_set_coords_filter_list( +CAPI_INTERFACE( + array_schema_set_coords_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) noexcept { + tiledb_filter_list_t* filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_set_offsets_filter_list( +CAPI_INTERFACE( + array_schema_set_offsets_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) noexcept { + tiledb_filter_list_t* filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_set_validity_filter_list( +CAPI_INTERFACE( + array_schema_set_validity_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t* filter_list) noexcept { + tiledb_filter_list_t* filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_check( - tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema) noexcept { +CAPI_INTERFACE( + array_schema_check, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema) { return api_entry(ctx, array_schema); } -int32_t tiledb_array_schema_load( +CAPI_INTERFACE( + array_schema_load, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, array_uri, array_schema); } -int32_t tiledb_array_schema_load_with_key( +CAPI_INTERFACE( + array_schema_load_with_key, tiledb_ctx_t* ctx, const char* array_uri, tiledb_encryption_type_t encryption_type, const void* encryption_key, uint32_t key_length, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, array_uri, @@ -5518,109 +5544,122 @@ int32_t tiledb_array_schema_load_with_key( array_schema); } -int32_t tiledb_array_schema_get_array_type( +CAPI_INTERFACE( + array_schema_get_array_type, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - tiledb_array_type_t* array_type) noexcept { + tiledb_array_type_t* array_type) { return api_entry( ctx, array_schema, array_type); } -int32_t tiledb_array_schema_get_capacity( +CAPI_INTERFACE( + array_schema_get_capacity, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - uint64_t* capacity) noexcept { + uint64_t* capacity) { return api_entry( ctx, array_schema, capacity); } -int32_t tiledb_array_schema_get_cell_order( +CAPI_INTERFACE( + array_schema_get_cell_order, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - tiledb_layout_t* cell_order) noexcept { + tiledb_layout_t* cell_order) { return api_entry( ctx, array_schema, cell_order); } -int32_t tiledb_array_schema_get_coords_filter_list( +CAPI_INTERFACE( + array_schema_get_coords_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) noexcept { + tiledb_filter_list_t** filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_get_offsets_filter_list( +CAPI_INTERFACE( + array_schema_get_offsets_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) noexcept { + tiledb_filter_list_t** filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_get_validity_filter_list( +CAPI_INTERFACE( + array_schema_get_validity_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - tiledb_filter_list_t** filter_list) noexcept { + tiledb_filter_list_t** filter_list) { return api_entry( ctx, array_schema, filter_list); } -int32_t tiledb_array_schema_get_domain( +CAPI_INTERFACE( + array_schema_get_domain, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - tiledb_domain_t** domain) noexcept { + tiledb_domain_t** domain) { return api_entry( ctx, array_schema, domain); } -int32_t tiledb_array_schema_get_tile_order( +CAPI_INTERFACE( + array_schema_get_tile_order, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - tiledb_layout_t* tile_order) noexcept { + tiledb_layout_t* tile_order) { return api_entry( ctx, array_schema, tile_order); } -int32_t tiledb_array_schema_get_attribute_num( +CAPI_INTERFACE( + array_schema_get_attribute_num, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - uint32_t* attribute_num) noexcept { + uint32_t* attribute_num) { return api_entry( ctx, array_schema, attribute_num); } -int32_t tiledb_array_schema_dump( +CAPI_INTERFACE( + array_schema_dump, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, - FILE* out) noexcept { + FILE* out) { return api_entry( ctx, array_schema, out); } -int32_t tiledb_array_schema_get_attribute_from_index( +CAPI_INTERFACE( + array_schema_get_attribute_from_index, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, uint32_t index, - tiledb_attribute_t** attr) noexcept { + tiledb_attribute_t** attr) { return api_entry( ctx, array_schema, index, attr); } -int32_t tiledb_array_schema_get_attribute_from_name( +CAPI_INTERFACE( + array_schema_get_attribute_from_name, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, const char* name, - tiledb_attribute_t** attr) noexcept { + tiledb_attribute_t** attr) { return api_entry( ctx, array_schema, name, attr); } -int32_t tiledb_array_schema_has_attribute( +CAPI_INTERFACE( + array_schema_has_attribute, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, const char* name, - int32_t* has_attr) noexcept { + int32_t* has_attr) { return api_entry( ctx, array_schema, name, has_attr); } @@ -5629,56 +5668,63 @@ int32_t tiledb_array_schema_has_attribute( /* SCHEMA EVOLUTION */ /* ********************************* */ -int32_t tiledb_array_schema_evolution_alloc( +CAPI_INTERFACE( + array_schema_evolution_alloc, tiledb_ctx_t* ctx, - tiledb_array_schema_evolution_t** array_schema_evolution) noexcept { + tiledb_array_schema_evolution_t** array_schema_evolution) { return api_entry( ctx, array_schema_evolution); } -void tiledb_array_schema_evolution_free( - tiledb_array_schema_evolution_t** array_schema_evolution) noexcept { +CAPI_INTERFACE_VOID( + array_schema_evolution_free, + tiledb_array_schema_evolution_t** array_schema_evolution) { return api_entry_void( array_schema_evolution); } -int32_t tiledb_array_schema_evolution_add_attribute( +CAPI_INTERFACE( + array_schema_evolution_add_attribute, tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, - tiledb_attribute_t* attr) noexcept { + tiledb_attribute_t* attr) { return api_entry( ctx, array_schema_evolution, attr); } -int32_t tiledb_array_schema_evolution_drop_attribute( +CAPI_INTERFACE( + array_schema_evolution_drop_attribute, tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, - const char* attribute_name) noexcept { + const char* attribute_name) { return api_entry( ctx, array_schema_evolution, attribute_name); } -int32_t tiledb_array_schema_evolution_add_enumeration( +CAPI_INTERFACE( + array_schema_evolution_add_enumeration, tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, - tiledb_enumeration_t* enmr) noexcept { + tiledb_enumeration_t* enmr) { return api_entry( ctx, array_schema_evolution, enmr); } -capi_return_t tiledb_array_schema_evolution_extend_enumeration( +CAPI_INTERFACE( + array_schema_evolution_extend_enumeration, tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, - tiledb_enumeration_t* enmr) noexcept { + tiledb_enumeration_t* enmr) { return api_entry< tiledb::api::tiledb_array_schema_evolution_extend_enumeration>( ctx, array_schema_evolution, enmr); } -capi_return_t tiledb_array_schema_evolution_drop_enumeration( +CAPI_INTERFACE( + array_schema_evolution_drop_enumeration, tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, - const char* enumeration_name) noexcept { + const char* enumeration_name) { return api_entry( ctx, array_schema_evolution, enumeration_name); } @@ -5697,400 +5743,451 @@ TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( /* QUERY */ /* ****************************** */ -int32_t tiledb_query_alloc( +CAPI_INTERFACE( + query_alloc, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_query_type_t query_type, - tiledb_query_t** query) noexcept { + tiledb_query_t** query) { return api_entry( ctx, array, query_type, query); } -int32_t tiledb_query_get_stats( - tiledb_ctx_t* ctx, tiledb_query_t* query, char** stats_json) noexcept { +CAPI_INTERFACE( + query_get_stats, + tiledb_ctx_t* ctx, + tiledb_query_t* query, + char** stats_json) { return api_entry(ctx, query, stats_json); } -int32_t tiledb_query_set_config( +CAPI_INTERFACE( + query_set_config, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry(ctx, query, config); } -int32_t tiledb_query_get_config( +CAPI_INTERFACE( + query_get_config, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_config_t** config) noexcept { + tiledb_config_t** config) { return api_entry(ctx, query, config); } -int32_t tiledb_query_set_subarray( +CAPI_INTERFACE( + query_set_subarray, tiledb_ctx_t* ctx, tiledb_query_t* query, - const void* subarray_vals) noexcept { + const void* subarray_vals) { return api_entry( ctx, query, subarray_vals); } -int32_t tiledb_query_set_subarray_t( +CAPI_INTERFACE( + query_set_subarray_t, tiledb_ctx_t* ctx, tiledb_query_t* query, - const tiledb_subarray_t* subarray) noexcept { + const tiledb_subarray_t* subarray) { return api_entry( ctx, query, subarray); } -int32_t tiledb_query_set_data_buffer( +CAPI_INTERFACE( + query_set_data_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, void* buffer, - uint64_t* buffer_size) noexcept { + uint64_t* buffer_size) { return api_entry( ctx, query, name, buffer, buffer_size); } -int32_t tiledb_query_set_offsets_buffer( +CAPI_INTERFACE( + query_set_offsets_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, uint64_t* buffer_offsets, - uint64_t* buffer_offsets_size) noexcept { + uint64_t* buffer_offsets_size) { return api_entry( ctx, query, name, buffer_offsets, buffer_offsets_size); } -int32_t tiledb_query_set_validity_buffer( +CAPI_INTERFACE( + query_set_validity_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, uint8_t* buffer_validity, - uint64_t* buffer_validity_size) noexcept { + uint64_t* buffer_validity_size) { return api_entry( ctx, query, name, buffer_validity, buffer_validity_size); } -int32_t tiledb_query_get_data_buffer( +CAPI_INTERFACE( + query_get_data_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, void** buffer, - uint64_t** buffer_size) noexcept { + uint64_t** buffer_size) { return api_entry( ctx, query, name, buffer, buffer_size); } -int32_t tiledb_query_get_offsets_buffer( +CAPI_INTERFACE( + query_get_offsets_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, uint64_t** buffer, - uint64_t** buffer_size) noexcept { + uint64_t** buffer_size) { return api_entry( ctx, query, name, buffer, buffer_size); } -int32_t tiledb_query_get_validity_buffer( +CAPI_INTERFACE( + query_get_validity_buffer, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* name, uint8_t** buffer, - uint64_t** buffer_size) noexcept { + uint64_t** buffer_size) { return api_entry( ctx, query, name, buffer, buffer_size); } -int32_t tiledb_query_set_layout( - tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_layout_t layout) noexcept { +CAPI_INTERFACE( + query_set_layout, + tiledb_ctx_t* ctx, + tiledb_query_t* query, + tiledb_layout_t layout) { return api_entry(ctx, query, layout); } -int32_t tiledb_query_set_condition( +CAPI_INTERFACE( + query_set_condition, tiledb_ctx_t* const ctx, tiledb_query_t* const query, - const tiledb_query_condition_t* const cond) noexcept { + const tiledb_query_condition_t* const cond) { return api_entry(ctx, query, cond); } -int32_t tiledb_query_finalize( - tiledb_ctx_t* ctx, tiledb_query_t* query) noexcept { +CAPI_INTERFACE(query_finalize, tiledb_ctx_t* ctx, tiledb_query_t* query) { return api_entry(ctx, query); } -int32_t tiledb_query_submit_and_finalize( - tiledb_ctx_t* ctx, tiledb_query_t* query) noexcept { +CAPI_INTERFACE( + query_submit_and_finalize, tiledb_ctx_t* ctx, tiledb_query_t* query) { return api_entry(ctx, query); } -void tiledb_query_free(tiledb_query_t** query) noexcept { +CAPI_INTERFACE_VOID(query_free, tiledb_query_t** query) { return api_entry_void(query); } -int32_t tiledb_query_submit(tiledb_ctx_t* ctx, tiledb_query_t* query) noexcept { +CAPI_INTERFACE(query_submit, tiledb_ctx_t* ctx, tiledb_query_t* query) { return api_entry(ctx, query); } -int32_t tiledb_query_submit_async( +CAPI_INTERFACE( + query_submit_async, tiledb_ctx_t* ctx, tiledb_query_t* query, void (*callback)(void*), - void* callback_data) noexcept { + void* callback_data) { return api_entry( ctx, query, callback, callback_data); } -int32_t tiledb_query_has_results( - tiledb_ctx_t* ctx, tiledb_query_t* query, int32_t* has_results) noexcept { +CAPI_INTERFACE( + query_has_results, + tiledb_ctx_t* ctx, + tiledb_query_t* query, + int32_t* has_results) { return api_entry( ctx, query, has_results); } -int32_t tiledb_query_get_status( +CAPI_INTERFACE( + query_get_status, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_query_status_t* status) noexcept { + tiledb_query_status_t* status) { return api_entry(ctx, query, status); } -int32_t tiledb_query_get_type( +CAPI_INTERFACE( + query_get_type, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_query_type_t* query_type) noexcept { + tiledb_query_type_t* query_type) { return api_entry(ctx, query, query_type); } -int32_t tiledb_query_get_layout( +CAPI_INTERFACE( + query_get_layout, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_layout_t* query_layout) noexcept { + tiledb_layout_t* query_layout) { return api_entry( ctx, query, query_layout); } -int32_t tiledb_query_get_array( - tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_array_t** array) noexcept { +CAPI_INTERFACE( + query_get_array, + tiledb_ctx_t* ctx, + tiledb_query_t* query, + tiledb_array_t** array) { return api_entry(ctx, query, array); } -int32_t tiledb_query_add_range( +CAPI_INTERFACE( + query_add_range, tiledb_ctx_t* ctx, tiledb_query_t* query, uint32_t dim_idx, const void* start, const void* end, - const void* stride) noexcept { + const void* stride) { return api_entry( ctx, query, dim_idx, start, end, stride); } -int32_t tiledb_query_add_point_ranges( +CAPI_INTERFACE( + query_add_point_ranges, tiledb_ctx_t* ctx, tiledb_query_t* query, uint32_t dim_idx, const void* start, - uint64_t count) noexcept { + uint64_t count) { return api_entry( ctx, query, dim_idx, start, count); } -int32_t tiledb_query_add_range_by_name( +CAPI_INTERFACE( + query_add_range_by_name, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* dim_name, const void* start, const void* end, - const void* stride) noexcept { + const void* stride) { return api_entry( ctx, query, dim_name, start, end, stride); } -int32_t tiledb_query_add_range_var( +CAPI_INTERFACE( + query_add_range_var, tiledb_ctx_t* ctx, tiledb_query_t* query, uint32_t dim_idx, const void* start, uint64_t start_size, const void* end, - uint64_t end_size) noexcept { + uint64_t end_size) { return api_entry( ctx, query, dim_idx, start, start_size, end, end_size); } -int32_t tiledb_query_add_range_var_by_name( +CAPI_INTERFACE( + query_add_range_var_by_name, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* dim_name, const void* start, uint64_t start_size, const void* end, - uint64_t end_size) noexcept { + uint64_t end_size) { return api_entry( ctx, query, dim_name, start, start_size, end, end_size); } -int32_t tiledb_query_get_range_num( +CAPI_INTERFACE( + query_get_range_num, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t dim_idx, - uint64_t* range_num) noexcept { + uint64_t* range_num) { return api_entry( ctx, query, dim_idx, range_num); } -int32_t tiledb_query_get_range_num_from_name( +CAPI_INTERFACE( + query_get_range_num_from_name, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* dim_name, - uint64_t* range_num) noexcept { + uint64_t* range_num) { return api_entry( ctx, query, dim_name, range_num); } -int32_t tiledb_query_get_range( +CAPI_INTERFACE( + query_get_range, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t dim_idx, uint64_t range_idx, const void** start, const void** end, - const void** stride) noexcept { + const void** stride) { return api_entry( ctx, query, dim_idx, range_idx, start, end, stride); } -int32_t tiledb_query_get_range_from_name( +CAPI_INTERFACE( + query_get_range_from_name, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* dim_name, uint64_t range_idx, const void** start, const void** end, - const void** stride) noexcept { + const void** stride) { return api_entry( ctx, query, dim_name, range_idx, start, end, stride); } -int32_t tiledb_query_get_range_var_size( +CAPI_INTERFACE( + query_get_range_var_size, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t dim_idx, uint64_t range_idx, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry( ctx, query, dim_idx, range_idx, start_size, end_size); } -int32_t tiledb_query_get_range_var_size_from_name( +CAPI_INTERFACE( + query_get_range_var_size_from_name, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* dim_name, uint64_t range_idx, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry( ctx, query, dim_name, range_idx, start_size, end_size); } -int32_t tiledb_query_get_range_var( +CAPI_INTERFACE( + query_get_range_var, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t dim_idx, uint64_t range_idx, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, query, dim_idx, range_idx, start, end); } -int32_t tiledb_query_get_range_var_from_name( +CAPI_INTERFACE( + query_get_range_var_from_name, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* dim_name, uint64_t range_idx, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, query, dim_name, range_idx, start, end); } -int32_t tiledb_query_get_est_result_size( +CAPI_INTERFACE( + query_get_est_result_size, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* name, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry( ctx, query, name, size); } -int32_t tiledb_query_get_est_result_size_var( +CAPI_INTERFACE( + query_get_est_result_size_var, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* name, uint64_t* size_off, - uint64_t* size_val) noexcept { + uint64_t* size_val) { return api_entry( ctx, query, name, size_off, size_val); } -int32_t tiledb_query_get_est_result_size_nullable( +CAPI_INTERFACE( + query_get_est_result_size_nullable, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* name, uint64_t* size_val, - uint64_t* size_validity) noexcept { + uint64_t* size_validity) { return api_entry( ctx, query, name, size_val, size_validity); } -int32_t tiledb_query_get_est_result_size_var_nullable( +CAPI_INTERFACE( + query_get_est_result_size_var_nullable, tiledb_ctx_t* ctx, const tiledb_query_t* query, const char* name, uint64_t* size_off, uint64_t* size_val, - uint64_t* size_validity) noexcept { + uint64_t* size_validity) { return api_entry( ctx, query, name, size_off, size_val, size_validity); } -int32_t tiledb_query_get_fragment_num( - tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t* num) noexcept { +CAPI_INTERFACE( + query_get_fragment_num, + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + uint32_t* num) { return api_entry(ctx, query, num); } -int32_t tiledb_query_get_fragment_uri( +CAPI_INTERFACE( + query_get_fragment_uri, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint64_t idx, - const char** uri) noexcept { + const char** uri) { return api_entry( ctx, query, idx, uri); } -int32_t tiledb_query_get_fragment_timestamp_range( +CAPI_INTERFACE( + query_get_fragment_timestamp_range, tiledb_ctx_t* ctx, const tiledb_query_t* query, uint64_t idx, uint64_t* t1, - uint64_t* t2) noexcept { + uint64_t* t2) { return api_entry( ctx, query, idx, t1, t2); } -int32_t tiledb_query_get_subarray_t( +CAPI_INTERFACE( + query_get_subarray_t, tiledb_ctx_t* ctx, const tiledb_query_t* query, - tiledb_subarray_t** subarray) noexcept { + tiledb_subarray_t** subarray) { return api_entry( ctx, query, subarray); } -int32_t tiledb_query_get_relevant_fragment_num( +CAPI_INTERFACE( + query_get_relevant_fragment_num, tiledb_ctx_t* ctx, const tiledb_query_t* query, - uint64_t* relevant_fragment_num) noexcept { + uint64_t* relevant_fragment_num) { return api_entry( ctx, query, relevant_fragment_num); } @@ -6099,179 +6196,196 @@ int32_t tiledb_query_get_relevant_fragment_num( /* SUBARRAY */ /* ****************************** */ -int32_t tiledb_subarray_alloc( +CAPI_INTERFACE( + subarray_alloc, tiledb_ctx_t* ctx, const tiledb_array_t* array, - tiledb_subarray_t** subarray) noexcept { + tiledb_subarray_t** subarray) { return api_entry(ctx, array, subarray); } -int32_t tiledb_subarray_set_config( +CAPI_INTERFACE( + subarray_set_config, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, subarray, config); } -void tiledb_subarray_free(tiledb_subarray_t** subarray) noexcept { +CAPI_INTERFACE_VOID(subarray_free, tiledb_subarray_t** subarray) { return api_entry_void(subarray); } -int32_t tiledb_subarray_set_coalesce_ranges( +CAPI_INTERFACE( + subarray_set_coalesce_ranges, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, - int coalesce_ranges) noexcept { + int coalesce_ranges) { return api_entry( ctx, subarray, coalesce_ranges); } -int32_t tiledb_subarray_set_subarray( +CAPI_INTERFACE( + subarray_set_subarray, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray_obj, - const void* subarray_vals) noexcept { + const void* subarray_vals) { return api_entry( ctx, subarray_obj, subarray_vals); } -int32_t tiledb_subarray_add_range( +CAPI_INTERFACE( + subarray_add_range, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, const void* end, - const void* stride) noexcept { + const void* stride) { return api_entry( ctx, subarray, dim_idx, start, end, stride); } -int32_t tiledb_subarray_add_point_ranges( +CAPI_INTERFACE( + subarray_add_point_ranges, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, - uint64_t count) noexcept { + uint64_t count) { return api_entry( ctx, subarray, dim_idx, start, count); } -int32_t tiledb_subarray_add_range_by_name( +CAPI_INTERFACE( + subarray_add_range_by_name, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, const char* dim_name, const void* start, const void* end, - const void* stride) noexcept { + const void* stride) { return api_entry( ctx, subarray, dim_name, start, end, stride); } -int32_t tiledb_subarray_add_range_var( +CAPI_INTERFACE( + subarray_add_range_var, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, uint64_t start_size, const void* end, - uint64_t end_size) noexcept { + uint64_t end_size) { return api_entry( ctx, subarray, dim_idx, start, start_size, end, end_size); } -int32_t tiledb_subarray_add_range_var_by_name( +CAPI_INTERFACE( + subarray_add_range_var_by_name, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, const char* dim_name, const void* start, uint64_t start_size, const void* end, - uint64_t end_size) noexcept { + uint64_t end_size) { return api_entry( ctx, subarray, dim_name, start, start_size, end, end_size); } -int32_t tiledb_subarray_get_range_num( +CAPI_INTERFACE( + subarray_get_range_num, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, uint32_t dim_idx, - uint64_t* range_num) noexcept { + uint64_t* range_num) { return api_entry( ctx, subarray, dim_idx, range_num); } -int32_t tiledb_subarray_get_range_num_from_name( +CAPI_INTERFACE( + subarray_get_range_num_from_name, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, - uint64_t* range_num) noexcept { + uint64_t* range_num) { return api_entry( ctx, subarray, dim_name, range_num); } -int32_t tiledb_subarray_get_range( +CAPI_INTERFACE( + subarray_get_range, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, const void** start, const void** end, - const void** stride) noexcept { + const void** stride) { return api_entry( ctx, subarray, dim_idx, range_idx, start, end, stride); } -int32_t tiledb_subarray_get_range_var_size( +CAPI_INTERFACE( + subarray_get_range_var_size, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry( ctx, subarray, dim_idx, range_idx, start_size, end_size); } -int32_t tiledb_subarray_get_range_from_name( +CAPI_INTERFACE( + subarray_get_range_from_name, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, const void** start, const void** end, - const void** stride) noexcept { + const void** stride) { return api_entry( ctx, subarray, dim_name, range_idx, start, end, stride); } -int32_t tiledb_subarray_get_range_var_size_from_name( +CAPI_INTERFACE( + subarray_get_range_var_size_from_name, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry( ctx, subarray, dim_name, range_idx, start_size, end_size); } -int32_t tiledb_subarray_get_range_var( +CAPI_INTERFACE( + subarray_get_range_var, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, subarray, dim_idx, range_idx, start, end); } -int32_t tiledb_subarray_get_range_var_from_name( +CAPI_INTERFACE( + subarray_get_range_var_from_name, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, subarray, dim_name, range_idx, start, end); } @@ -6280,12 +6394,15 @@ int32_t tiledb_subarray_get_range_var_from_name( /* QUERY CONDITION */ /* ****************************** */ -int32_t tiledb_query_condition_alloc( - tiledb_ctx_t* const ctx, tiledb_query_condition_t** const cond) noexcept { +CAPI_INTERFACE( + query_condition_alloc, + tiledb_ctx_t* const ctx, + tiledb_query_condition_t** const cond) { return api_entry(ctx, cond); } -capi_return_t tiledb_query_condition_alloc_set_membership( +CAPI_INTERFACE( + query_condition_alloc_set_membership, tiledb_ctx_t* ctx, const char* field_name, const void* data, @@ -6293,49 +6410,53 @@ capi_return_t tiledb_query_condition_alloc_set_membership( const void* offsets, uint64_t offsets_size, tiledb_query_condition_op_t op, - tiledb_query_condition_t** cond) noexcept { + tiledb_query_condition_t** cond) { return api_entry_context< tiledb::api::tiledb_query_condition_alloc_set_membership>( ctx, field_name, data, data_size, offsets, offsets_size, op, cond); } -void tiledb_query_condition_free(tiledb_query_condition_t** cond) noexcept { +CAPI_INTERFACE_VOID(query_condition_free, tiledb_query_condition_t** cond) { return api_entry_void(cond); } -int32_t tiledb_query_condition_init( +CAPI_INTERFACE( + query_condition_init, tiledb_ctx_t* const ctx, tiledb_query_condition_t* const cond, const char* const attribute_name, const void* const condition_value, const uint64_t condition_value_size, - const tiledb_query_condition_op_t op) noexcept { + const tiledb_query_condition_op_t op) { return api_entry( ctx, cond, attribute_name, condition_value, condition_value_size, op); } -int32_t tiledb_query_condition_combine( +CAPI_INTERFACE( + query_condition_combine, tiledb_ctx_t* const ctx, const tiledb_query_condition_t* const left_cond, const tiledb_query_condition_t* const right_cond, const tiledb_query_condition_combination_op_t combination_op, - tiledb_query_condition_t** const combined_cond) noexcept { + tiledb_query_condition_t** const combined_cond) { return api_entry( ctx, left_cond, right_cond, combination_op, combined_cond); } -int32_t tiledb_query_condition_negate( +CAPI_INTERFACE( + query_condition_negate, tiledb_ctx_t* const ctx, const tiledb_query_condition_t* const cond, - tiledb_query_condition_t** const negated_cond) noexcept { + tiledb_query_condition_t** const negated_cond) { return api_entry( ctx, cond, negated_cond); } -capi_return_t tiledb_query_condition_set_use_enumeration( +CAPI_INTERFACE( + query_condition_set_use_enumeration, tiledb_ctx_t* const ctx, const tiledb_query_condition_t* const cond, - int use_enumeration) noexcept { + int use_enumeration) { return api_entry( ctx, cond, use_enumeration); } @@ -6344,12 +6465,13 @@ capi_return_t tiledb_query_condition_set_use_enumeration( /* UPDATE CONDITION */ /* ****************************** */ -int32_t tiledb_query_add_update_value( +CAPI_INTERFACE( + query_add_update_value, tiledb_ctx_t* ctx, tiledb_query_t* query, const char* field_name, const void* update_value, - uint64_t update_value_size) noexcept { + uint64_t update_value_size) { return api_entry( ctx, query, field_name, update_value, update_value_size); } @@ -6358,147 +6480,169 @@ int32_t tiledb_query_add_update_value( /* ARRAY */ /* ****************************** */ -int32_t tiledb_array_alloc( - tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_t** array) noexcept { +CAPI_INTERFACE( + array_alloc, + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_array_t** array) { return api_entry(ctx, array_uri, array); } -int32_t tiledb_array_set_open_timestamp_start( +CAPI_INTERFACE( + array_set_open_timestamp_start, tiledb_ctx_t* ctx, tiledb_array_t* array, - uint64_t timestamp_start) noexcept { + uint64_t timestamp_start) { return api_entry( ctx, array, timestamp_start); } -int32_t tiledb_array_set_open_timestamp_end( - tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t timestamp_end) noexcept { +CAPI_INTERFACE( + array_set_open_timestamp_end, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + uint64_t timestamp_end) { return api_entry( ctx, array, timestamp_end); } -int32_t tiledb_array_get_open_timestamp_start( +CAPI_INTERFACE( + array_get_open_timestamp_start, tiledb_ctx_t* ctx, tiledb_array_t* array, - uint64_t* timestamp_start) noexcept { + uint64_t* timestamp_start) { return api_entry( ctx, array, timestamp_start); } -int32_t tiledb_array_get_open_timestamp_end( +CAPI_INTERFACE( + array_get_open_timestamp_end, tiledb_ctx_t* ctx, tiledb_array_t* array, - uint64_t* timestamp_end) noexcept { + uint64_t* timestamp_end) { return api_entry( ctx, array, timestamp_end); } -int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) noexcept { +CAPI_INTERFACE(array_delete, tiledb_ctx_t* ctx, const char* uri) { return api_entry(ctx, uri); } -int32_t tiledb_array_delete_array( - tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri) noexcept { +CAPI_INTERFACE( + array_delete_array, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + const char* uri) { return api_entry(ctx, array, uri); } -int32_t tiledb_array_delete_fragments( +CAPI_INTERFACE( + array_delete_fragments, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri, uint64_t timestamp_start, - uint64_t timestamp_end) noexcept { + uint64_t timestamp_end) { return api_entry( ctx, array, uri, timestamp_start, timestamp_end); } -capi_return_t tiledb_array_delete_fragments_v2( +CAPI_INTERFACE( + array_delete_fragments_v2, tiledb_ctx_t* ctx, const char* uri_str, uint64_t timestamp_start, - uint64_t timestamp_end) noexcept { + uint64_t timestamp_end) { return api_entry( ctx, uri_str, timestamp_start, timestamp_end); } -capi_return_t tiledb_array_delete_fragments_list( +CAPI_INTERFACE( + array_delete_fragments_list, tiledb_ctx_t* ctx, const char* uri_str, const char* fragment_uris[], - const size_t num_fragments) noexcept { + const size_t num_fragments) { return api_entry( ctx, uri_str, fragment_uris, num_fragments); } -int32_t tiledb_array_open( +CAPI_INTERFACE( + array_open, tiledb_ctx_t* ctx, tiledb_array_t* array, - tiledb_query_type_t query_type) noexcept { + tiledb_query_type_t query_type) { return api_entry(ctx, array, query_type); } -int32_t tiledb_array_is_open( - tiledb_ctx_t* ctx, tiledb_array_t* array, int32_t* is_open) noexcept { +CAPI_INTERFACE( + array_is_open, tiledb_ctx_t* ctx, tiledb_array_t* array, int32_t* is_open) { return api_entry(ctx, array, is_open); } -int32_t tiledb_array_reopen(tiledb_ctx_t* ctx, tiledb_array_t* array) noexcept { +CAPI_INTERFACE(array_reopen, tiledb_ctx_t* ctx, tiledb_array_t* array) { return api_entry(ctx, array); } -int32_t tiledb_array_set_config( +CAPI_INTERFACE( + array_set_config, tiledb_ctx_t* ctx, tiledb_array_t* array, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry(ctx, array, config); } -int32_t tiledb_array_get_config( +CAPI_INTERFACE( + array_get_config, tiledb_ctx_t* ctx, tiledb_array_t* array, - tiledb_config_t** config) noexcept { + tiledb_config_t** config) { return api_entry(ctx, array, config); } -int32_t tiledb_array_close(tiledb_ctx_t* ctx, tiledb_array_t* array) noexcept { +CAPI_INTERFACE(array_close, tiledb_ctx_t* ctx, tiledb_array_t* array) { return api_entry(ctx, array); } -void tiledb_array_free(tiledb_array_t** array) noexcept { +CAPI_INTERFACE_VOID(array_free, tiledb_array_t** array) { return api_entry_void(array); } -int32_t tiledb_array_get_schema( +CAPI_INTERFACE( + array_get_schema, tiledb_ctx_t* ctx, tiledb_array_t* array, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, array, array_schema); } -int32_t tiledb_array_get_query_type( +CAPI_INTERFACE( + array_get_query_type, tiledb_ctx_t* ctx, tiledb_array_t* array, - tiledb_query_type_t* query_type) noexcept { + tiledb_query_type_t* query_type) { return api_entry( ctx, array, query_type); } -int32_t tiledb_array_create( +CAPI_INTERFACE( + array_create, tiledb_ctx_t* ctx, const char* array_uri, - const tiledb_array_schema_t* array_schema) noexcept { + const tiledb_array_schema_t* array_schema) { return api_entry( ctx, array_uri, array_schema); } -int32_t tiledb_array_create_with_key( +CAPI_INTERFACE( + array_create_with_key, tiledb_ctx_t* ctx, const char* array_uri, const tiledb_array_schema_t* array_schema, tiledb_encryption_type_t encryption_type, const void* encryption_key, - uint32_t key_length) noexcept { + uint32_t key_length) { return api_entry( ctx, array_uri, @@ -6508,165 +6652,189 @@ int32_t tiledb_array_create_with_key( key_length); } -int32_t tiledb_array_consolidate( +CAPI_INTERFACE( + array_consolidate, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, array_uri, config); } -int32_t tiledb_array_consolidate_with_key( +CAPI_INTERFACE( + array_consolidate_with_key, tiledb_ctx_t* ctx, const char* array_uri, tiledb_encryption_type_t encryption_type, const void* encryption_key, uint32_t key_length, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, array_uri, encryption_type, encryption_key, key_length, config); } -int32_t tiledb_array_consolidate_fragments( +CAPI_INTERFACE( + array_consolidate_fragments, tiledb_ctx_t* ctx, const char* array_uri, const char** fragment_uris, const uint64_t num_fragments, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, array_uri, fragment_uris, num_fragments, config); } -int32_t tiledb_array_vacuum( +CAPI_INTERFACE( + array_vacuum, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry(ctx, array_uri, config); } -int32_t tiledb_array_get_non_empty_domain( +CAPI_INTERFACE( + array_get_non_empty_domain, tiledb_ctx_t* ctx, tiledb_array_t* array, void* domain, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry( ctx, array, domain, is_empty); } -int32_t tiledb_array_get_non_empty_domain_from_index( +CAPI_INTERFACE( + array_get_non_empty_domain_from_index, tiledb_ctx_t* ctx, tiledb_array_t* array, uint32_t idx, void* domain, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry( ctx, array, idx, domain, is_empty); } -int32_t tiledb_array_get_non_empty_domain_from_name( +CAPI_INTERFACE( + array_get_non_empty_domain_from_name, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* name, void* domain, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry( ctx, array, name, domain, is_empty); } -int32_t tiledb_array_get_non_empty_domain_var_size_from_index( +CAPI_INTERFACE( + array_get_non_empty_domain_var_size_from_index, tiledb_ctx_t* ctx, tiledb_array_t* array, uint32_t idx, uint64_t* start_size, uint64_t* end_size, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry< tiledb::api::tiledb_array_get_non_empty_domain_var_size_from_index>( ctx, array, idx, start_size, end_size, is_empty); } -int32_t tiledb_array_get_non_empty_domain_var_size_from_name( +CAPI_INTERFACE( + array_get_non_empty_domain_var_size_from_name, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* name, uint64_t* start_size, uint64_t* end_size, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry< tiledb::api::tiledb_array_get_non_empty_domain_var_size_from_name>( ctx, array, name, start_size, end_size, is_empty); } -int32_t tiledb_array_get_non_empty_domain_var_from_index( +CAPI_INTERFACE( + array_get_non_empty_domain_var_from_index, tiledb_ctx_t* ctx, tiledb_array_t* array, uint32_t idx, void* start, void* end, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry< tiledb::api::tiledb_array_get_non_empty_domain_var_from_index>( ctx, array, idx, start, end, is_empty); } -int32_t tiledb_array_get_non_empty_domain_var_from_name( +CAPI_INTERFACE( + array_get_non_empty_domain_var_from_name, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* name, void* start, void* end, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry< tiledb::api::tiledb_array_get_non_empty_domain_var_from_name>( ctx, array, name, start, end, is_empty); } -int32_t tiledb_array_get_uri( - tiledb_ctx_t* ctx, tiledb_array_t* array, const char** array_uri) noexcept { +CAPI_INTERFACE( + array_get_uri, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + const char** array_uri) { return api_entry(ctx, array, array_uri); } -int32_t tiledb_array_encryption_type( +CAPI_INTERFACE( + array_encryption_type, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_encryption_type_t* encryption_type) noexcept { + tiledb_encryption_type_t* encryption_type) { return api_entry( ctx, array_uri, encryption_type); } -int32_t tiledb_array_put_metadata( +CAPI_INTERFACE( + array_put_metadata, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* key, tiledb_datatype_t value_type, uint32_t value_num, - const void* value) noexcept { + const void* value) { return api_entry( ctx, array, key, value_type, value_num, value); } -int32_t tiledb_array_delete_metadata( - tiledb_ctx_t* ctx, tiledb_array_t* array, const char* key) noexcept { +CAPI_INTERFACE( + array_delete_metadata, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + const char* key) { return api_entry(ctx, array, key); } -int32_t tiledb_array_get_metadata( +CAPI_INTERFACE( + array_get_metadata, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* key, tiledb_datatype_t* value_type, uint32_t* value_num, - const void** value) noexcept { + const void** value) { return api_entry( ctx, array, key, value_type, value_num, value); } -int32_t tiledb_array_get_metadata_num( - tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t* num) noexcept { +CAPI_INTERFACE( + array_get_metadata_num, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + uint64_t* num) { return api_entry(ctx, array, num); } -int32_t tiledb_array_get_metadata_from_index( +CAPI_INTERFACE( + array_get_metadata_from_index, tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t index, @@ -6674,47 +6842,53 @@ int32_t tiledb_array_get_metadata_from_index( uint32_t* key_len, tiledb_datatype_t* value_type, uint32_t* value_num, - const void** value) noexcept { + const void** value) { return api_entry( ctx, array, index, key, key_len, value_type, value_num, value); } -int32_t tiledb_array_has_metadata_key( +CAPI_INTERFACE( + array_has_metadata_key, tiledb_ctx_t* ctx, tiledb_array_t* array, const char* key, tiledb_datatype_t* value_type, - int32_t* has_key) noexcept { + int32_t* has_key) { return api_entry( ctx, array, key, value_type, has_key); } -int32_t tiledb_array_evolve( +CAPI_INTERFACE( + array_evolve, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_array_schema_evolution_t* array_schema_evolution) noexcept { + tiledb_array_schema_evolution_t* array_schema_evolution) { return api_entry( ctx, array_uri, array_schema_evolution); } -capi_return_t tiledb_array_get_enumeration( +CAPI_INTERFACE( + array_get_enumeration, tiledb_ctx_t* ctx, const tiledb_array_t* array, const char* attr_name, - tiledb_enumeration_t** enumeration) noexcept { + tiledb_enumeration_t** enumeration) { return api_entry( ctx, array, attr_name, enumeration); } -capi_return_t tiledb_array_load_all_enumerations( - tiledb_ctx_t* ctx, const tiledb_array_t* array) noexcept { +CAPI_INTERFACE( + array_load_all_enumerations, + tiledb_ctx_t* ctx, + const tiledb_array_t* array) { return api_entry(ctx, array); } -int32_t tiledb_array_upgrade_version( +CAPI_INTERFACE( + array_upgrade_version, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, array_uri, config); } @@ -6723,35 +6897,40 @@ int32_t tiledb_array_upgrade_version( /* OBJECT MANAGEMENT */ /* ****************************** */ -int32_t tiledb_object_type( - tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) noexcept { +CAPI_INTERFACE( + object_type, tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) { return api_entry(ctx, path, type); } -int32_t tiledb_object_remove(tiledb_ctx_t* ctx, const char* path) noexcept { +CAPI_INTERFACE(object_remove, tiledb_ctx_t* ctx, const char* path) { return api_entry(ctx, path); } -int32_t tiledb_object_move( - tiledb_ctx_t* ctx, const char* old_path, const char* new_path) noexcept { +CAPI_INTERFACE( + object_move, + tiledb_ctx_t* ctx, + const char* old_path, + const char* new_path) { return api_entry(ctx, old_path, new_path); } -int32_t tiledb_object_walk( +CAPI_INTERFACE( + object_walk, tiledb_ctx_t* ctx, const char* path, tiledb_walk_order_t order, int32_t (*callback)(const char*, tiledb_object_t, void*), - void* data) noexcept { + void* data) { return api_entry( ctx, path, order, callback, data); } -int32_t tiledb_object_ls( +CAPI_INTERFACE( + object_ls, tiledb_ctx_t* ctx, const char* path, int32_t (*callback)(const char*, tiledb_object_t, void*), - void* data) noexcept { + void* data) { return api_entry(ctx, path, callback, data); } @@ -6759,11 +6938,12 @@ int32_t tiledb_object_ls( /* URI */ /* ****************************** */ -int32_t tiledb_uri_to_path( +CAPI_INTERFACE( + uri_to_path, tiledb_ctx_t* ctx, const char* uri, char* path_out, - uint32_t* path_length) noexcept { + uint32_t* path_length) { return api_entry( ctx, uri, path_out, path_length); } @@ -6772,35 +6952,35 @@ int32_t tiledb_uri_to_path( /* Stats */ /* ****************************** */ -int32_t tiledb_stats_enable() noexcept { +CAPI_INTERFACE_NULL(stats_enable) { return api_entry_plain(); } -int32_t tiledb_stats_disable() noexcept { +CAPI_INTERFACE_NULL(stats_disable) { return api_entry_plain(); } -int32_t tiledb_stats_reset() noexcept { +CAPI_INTERFACE_NULL(stats_reset) { return api_entry_plain(); } -int32_t tiledb_stats_dump(FILE* out) noexcept { +CAPI_INTERFACE(stats_dump, FILE* out) { return api_entry_plain(out); } -int32_t tiledb_stats_dump_str(char** out) noexcept { +CAPI_INTERFACE(stats_dump_str, char** out) { return api_entry_plain(out); } -int32_t tiledb_stats_raw_dump(FILE* out) noexcept { +CAPI_INTERFACE(stats_raw_dump, FILE* out) { return api_entry_plain(out); } -int32_t tiledb_stats_raw_dump_str(char** out) noexcept { +CAPI_INTERFACE(stats_raw_dump_str, char** out) { return api_entry_plain(out); } -int32_t tiledb_stats_free_str(char** out) noexcept { +CAPI_INTERFACE(stats_free_str, char** out) { return api_entry_plain(out); } @@ -6808,11 +6988,12 @@ int32_t tiledb_stats_free_str(char** out) noexcept { /* Heap Profiler */ /* ****************************** */ -int32_t tiledb_heap_profiler_enable( +CAPI_INTERFACE( + heap_profiler_enable, const char* const file_name_prefix, const uint64_t dump_interval_ms, const uint64_t dump_interval_bytes, - const uint64_t dump_threshold_bytes) noexcept { + const uint64_t dump_threshold_bytes) { return api_entry_plain( file_name_prefix, dump_interval_ms, @@ -6824,126 +7005,138 @@ int32_t tiledb_heap_profiler_enable( /* Serialization */ /* ****************************** */ -int32_t tiledb_serialize_array( +CAPI_INTERFACE( + serialize_array, tiledb_ctx_t* ctx, const tiledb_array_t* array, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_array( +CAPI_INTERFACE( + deserialize_array, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_array_t** array) noexcept { + tiledb_array_t** array) { return api_entry( ctx, buffer, serialize_type, client_side, array); } -int32_t tiledb_serialize_array_schema( +CAPI_INTERFACE( + serialize_array_schema, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array_schema, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_array_schema( +CAPI_INTERFACE( + deserialize_array_schema, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, buffer, serialize_type, client_side, array_schema); } -int32_t tiledb_serialize_array_open( +CAPI_INTERFACE( + serialize_array_open, tiledb_ctx_t* ctx, const tiledb_array_t* array, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_array_open( +CAPI_INTERFACE( + deserialize_array_open, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_array_t** array) noexcept { + tiledb_array_t** array) { return api_entry( ctx, buffer, serialize_type, client_side, array); } -int32_t tiledb_serialize_array_schema_evolution( +CAPI_INTERFACE( + serialize_array_schema_evolution, tiledb_ctx_t* ctx, const tiledb_array_schema_evolution_t* array_schema_evolution, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array_schema_evolution, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_array_schema_evolution( +CAPI_INTERFACE( + deserialize_array_schema_evolution, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_array_schema_evolution_t** array_schema_evolution) noexcept { + tiledb_array_schema_evolution_t** array_schema_evolution) { return api_entry( ctx, buffer, serialize_type, client_side, array_schema_evolution); } -int32_t tiledb_serialize_query( +CAPI_INTERFACE( + serialize_query, tiledb_ctx_t* ctx, const tiledb_query_t* query, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_list_t** buffer_list) noexcept { + tiledb_buffer_list_t** buffer_list) { return api_entry( ctx, query, serialize_type, client_side, buffer_list); } -int32_t tiledb_deserialize_query( +CAPI_INTERFACE( + deserialize_query, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_query_t* query) noexcept { + tiledb_query_t* query) { return api_entry( ctx, buffer, serialize_type, client_side, query); } -int32_t tiledb_deserialize_query_and_array( +CAPI_INTERFACE( + deserialize_query_and_array, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, const char* array_uri, tiledb_query_t** query, - tiledb_array_t** array) noexcept { + tiledb_array_t** array) { return api_entry( ctx, buffer, serialize_type, client_side, array_uri, query, array); } -int32_t tiledb_serialize_array_nonempty_domain( +CAPI_INTERFACE( + serialize_array_nonempty_domain, tiledb_ctx_t* ctx, const tiledb_array_t* array, const void* nonempty_domain, int32_t is_empty, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array, @@ -6954,14 +7147,15 @@ int32_t tiledb_serialize_array_nonempty_domain( buffer); } -int32_t tiledb_deserialize_array_nonempty_domain( +CAPI_INTERFACE( + deserialize_array_nonempty_domain, tiledb_ctx_t* ctx, const tiledb_array_t* array, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, void* nonempty_domain, - int32_t* is_empty) noexcept { + int32_t* is_empty) { return api_entry( ctx, array, @@ -6972,173 +7166,190 @@ int32_t tiledb_deserialize_array_nonempty_domain( is_empty); } -int32_t tiledb_serialize_array_non_empty_domain_all_dimensions( +CAPI_INTERFACE( + serialize_array_non_empty_domain_all_dimensions, tiledb_ctx_t* ctx, const tiledb_array_t* array, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry< tiledb::api::tiledb_serialize_array_non_empty_domain_all_dimensions>( ctx, array, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_array_non_empty_domain_all_dimensions( +CAPI_INTERFACE( + deserialize_array_non_empty_domain_all_dimensions, tiledb_ctx_t* ctx, tiledb_array_t* array, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, - int32_t client_side) noexcept { + int32_t client_side) { return api_entry< tiledb::api::tiledb_deserialize_array_non_empty_domain_all_dimensions>( ctx, array, buffer, serialize_type, client_side); } -int32_t tiledb_serialize_array_max_buffer_sizes( +CAPI_INTERFACE( + serialize_array_max_buffer_sizes, tiledb_ctx_t* ctx, const tiledb_array_t* array, const void* subarray, tiledb_serialization_type_t serialize_type, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array, subarray, serialize_type, buffer); } -capi_return_t tiledb_handle_array_delete_fragments_timestamps_request( +CAPI_INTERFACE( + handle_array_delete_fragments_timestamps_request, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* request) noexcept { + const tiledb_buffer_t* request) { return api_entry< tiledb::api::tiledb_handle_array_delete_fragments_timestamps_request>( ctx, array, serialize_type, request); } -capi_return_t tiledb_handle_array_delete_fragments_list_request( +CAPI_INTERFACE( + handle_array_delete_fragments_list_request, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* request) noexcept { + const tiledb_buffer_t* request) { return api_entry< tiledb::api::tiledb_handle_array_delete_fragments_list_request>( ctx, array, serialize_type, request); } -int32_t tiledb_serialize_array_metadata( +CAPI_INTERFACE( + serialize_array_metadata, tiledb_ctx_t* ctx, const tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, array, serialize_type, buffer); } -int32_t tiledb_deserialize_array_metadata( +CAPI_INTERFACE( + deserialize_array_metadata, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_serialization_type_t serialize_type, - const tiledb_buffer_t* buffer) noexcept { + const tiledb_buffer_t* buffer) { return api_entry( ctx, array, serialize_type, buffer); } -int32_t tiledb_serialize_query_est_result_sizes( +CAPI_INTERFACE( + serialize_query_est_result_sizes, tiledb_ctx_t* ctx, const tiledb_query_t* query, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, query, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_query_est_result_sizes( +CAPI_INTERFACE( + deserialize_query_est_result_sizes, tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_serialization_type_t serialize_type, int32_t client_side, - const tiledb_buffer_t* buffer) noexcept { + const tiledb_buffer_t* buffer) { return api_entry( ctx, query, serialize_type, client_side, buffer); } -int32_t tiledb_serialize_config( +CAPI_INTERFACE( + serialize_config, tiledb_ctx_t* ctx, const tiledb_config_t* config, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, config, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_config( +CAPI_INTERFACE( + deserialize_config, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_config_t** config) noexcept { + tiledb_config_t** config) { return api_entry_context( ctx, buffer, serialize_type, client_side, config); } -int32_t tiledb_serialize_fragment_info_request( +CAPI_INTERFACE( + serialize_fragment_info_request, tiledb_ctx_t* ctx, const tiledb_fragment_info_t* fragment_info, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, fragment_info, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_fragment_info_request( +CAPI_INTERFACE( + deserialize_fragment_info_request, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_fragment_info_t* fragment_info) noexcept { + tiledb_fragment_info_t* fragment_info) { return api_entry( ctx, buffer, serialize_type, client_side, fragment_info); } -int32_t tiledb_serialize_fragment_info( +CAPI_INTERFACE( + serialize_fragment_info, tiledb_ctx_t* ctx, const tiledb_fragment_info_t* fragment_info, tiledb_serialization_type_t serialize_type, int32_t client_side, - tiledb_buffer_t** buffer) noexcept { + tiledb_buffer_t** buffer) { return api_entry( ctx, fragment_info, serialize_type, client_side, buffer); } -int32_t tiledb_deserialize_fragment_info( +CAPI_INTERFACE( + deserialize_fragment_info, tiledb_ctx_t* ctx, const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, const char* array_uri, int32_t client_side, - tiledb_fragment_info_t* fragment_info) noexcept { + tiledb_fragment_info_t* fragment_info) { return api_entry( ctx, buffer, serialize_type, array_uri, client_side, fragment_info); } -capi_return_t tiledb_handle_load_array_schema_request( +CAPI_INTERFACE( + handle_load_array_schema_request, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_serialization_type_t serialization_type, const tiledb_buffer_t* request, - tiledb_buffer_t* response) noexcept { + tiledb_buffer_t* response) { return api_entry( ctx, array, serialization_type, request, response); } -capi_return_t tiledb_handle_load_enumerations_request( +CAPI_INTERFACE( + handle_load_enumerations_request, tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_serialization_type_t serialization_type, const tiledb_buffer_t* request, - tiledb_buffer_t* response) noexcept { + tiledb_buffer_t* response) { return api_entry( ctx, array, serialization_type, request, response); } @@ -7159,348 +7370,384 @@ int32_t tiledb::impl::tiledb_query_submit_async_func( /* FRAGMENT INFO */ /* ****************************** */ -int32_t tiledb_fragment_info_alloc( +CAPI_INTERFACE( + fragment_info_alloc, tiledb_ctx_t* ctx, const char* array_uri, - tiledb_fragment_info_t** fragment_info) noexcept { + tiledb_fragment_info_t** fragment_info) { return api_entry( ctx, array_uri, fragment_info); } -void tiledb_fragment_info_free( - tiledb_fragment_info_t** fragment_info) noexcept { +CAPI_INTERFACE_VOID( + fragment_info_free, tiledb_fragment_info_t** fragment_info) { return api_entry_void(fragment_info); } -int32_t tiledb_fragment_info_set_config( +CAPI_INTERFACE( + fragment_info_set_config, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - tiledb_config_t* config) noexcept { + tiledb_config_t* config) { return api_entry( ctx, fragment_info, config); } -int32_t tiledb_fragment_info_get_config( +CAPI_INTERFACE( + fragment_info_get_config, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - tiledb_config_t** config) noexcept { + tiledb_config_t** config) { return api_entry( ctx, fragment_info, config); } -int32_t tiledb_fragment_info_load( - tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info) noexcept { +CAPI_INTERFACE( + fragment_info_load, + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info) { return api_entry(ctx, fragment_info); } -int32_t tiledb_fragment_info_get_fragment_name( +CAPI_INTERFACE( + fragment_info_get_fragment_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - const char** name) noexcept { + const char** name) { return api_entry( ctx, fragment_info, fid, name); } -int32_t tiledb_fragment_info_get_fragment_name_v2( +CAPI_INTERFACE( + fragment_info_get_fragment_name_v2, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - tiledb_string_t** name) noexcept { + tiledb_string_t** name) { return api_entry( ctx, fragment_info, fid, name); } -int32_t tiledb_fragment_info_get_fragment_num( +CAPI_INTERFACE( + fragment_info_get_fragment_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - uint32_t* fragment_num) noexcept { + uint32_t* fragment_num) { return api_entry( ctx, fragment_info, fragment_num); } -int32_t tiledb_fragment_info_get_fragment_uri( +CAPI_INTERFACE( + fragment_info_get_fragment_uri, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - const char** uri) noexcept { + const char** uri) { return api_entry( ctx, fragment_info, fid, uri); } -int32_t tiledb_fragment_info_get_fragment_size( +CAPI_INTERFACE( + fragment_info_get_fragment_size, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - uint64_t* size) noexcept { + uint64_t* size) { return api_entry( ctx, fragment_info, fid, size); } -int32_t tiledb_fragment_info_get_dense( +CAPI_INTERFACE( + fragment_info_get_dense, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - int32_t* dense) noexcept { + int32_t* dense) { return api_entry( ctx, fragment_info, fid, dense); } -int32_t tiledb_fragment_info_get_sparse( +CAPI_INTERFACE( + fragment_info_get_sparse, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - int32_t* sparse) noexcept { + int32_t* sparse) { return api_entry( ctx, fragment_info, fid, sparse); } -int32_t tiledb_fragment_info_get_timestamp_range( +CAPI_INTERFACE( + fragment_info_get_timestamp_range, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint64_t* start, - uint64_t* end) noexcept { + uint64_t* end) { return api_entry( ctx, fragment_info, fid, start, end); } -int32_t tiledb_fragment_info_get_non_empty_domain_from_index( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t did, - void* domain) noexcept { + void* domain) { return api_entry< tiledb::api::tiledb_fragment_info_get_non_empty_domain_from_index>( ctx, fragment_info, fid, did, domain); } -int32_t tiledb_fragment_info_get_non_empty_domain_from_name( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, const char* dim_name, - void* domain) noexcept { + void* domain) { return api_entry< tiledb::api::tiledb_fragment_info_get_non_empty_domain_from_name>( ctx, fragment_info, fid, dim_name, domain); } -int32_t tiledb_fragment_info_get_non_empty_domain_var_size_from_index( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_var_size_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t did, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry< tiledb::api:: tiledb_fragment_info_get_non_empty_domain_var_size_from_index>( ctx, fragment_info, fid, did, start_size, end_size); } -int32_t tiledb_fragment_info_get_non_empty_domain_var_size_from_name( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_var_size_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, const char* dim_name, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry< tiledb::api:: tiledb_fragment_info_get_non_empty_domain_var_size_from_name>( ctx, fragment_info, fid, dim_name, start_size, end_size); } -int32_t tiledb_fragment_info_get_non_empty_domain_var_from_index( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_var_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t did, void* start, - void* end) noexcept { + void* end) { return api_entry< tiledb::api::tiledb_fragment_info_get_non_empty_domain_var_from_index>( ctx, fragment_info, fid, did, start, end); } -int32_t tiledb_fragment_info_get_non_empty_domain_var_from_name( +CAPI_INTERFACE( + fragment_info_get_non_empty_domain_var_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, const char* dim_name, void* start, - void* end) noexcept { + void* end) { return api_entry< tiledb::api::tiledb_fragment_info_get_non_empty_domain_var_from_name>( ctx, fragment_info, fid, dim_name, start, end); } -int32_t tiledb_fragment_info_get_mbr_num( +CAPI_INTERFACE( + fragment_info_get_mbr_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - uint64_t* mbr_num) noexcept { + uint64_t* mbr_num) { return api_entry( ctx, fragment_info, fid, mbr_num); } -int32_t tiledb_fragment_info_get_mbr_from_index( +CAPI_INTERFACE( + fragment_info_get_mbr_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, uint32_t did, - void* mbr) noexcept { + void* mbr) { return api_entry( ctx, fragment_info, fid, mid, did, mbr); } -int32_t tiledb_fragment_info_get_mbr_from_name( +CAPI_INTERFACE( + fragment_info_get_mbr_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, const char* dim_name, - void* mbr) noexcept { + void* mbr) { return api_entry( ctx, fragment_info, fid, mid, dim_name, mbr); } -int32_t tiledb_fragment_info_get_mbr_var_size_from_index( +CAPI_INTERFACE( + fragment_info_get_mbr_var_size_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, uint32_t did, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry< tiledb::api::tiledb_fragment_info_get_mbr_var_size_from_index>( ctx, fragment_info, fid, mid, did, start_size, end_size); } -int32_t tiledb_fragment_info_get_mbr_var_size_from_name( +CAPI_INTERFACE( + fragment_info_get_mbr_var_size_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, const char* dim_name, uint64_t* start_size, - uint64_t* end_size) noexcept { + uint64_t* end_size) { return api_entry< tiledb::api::tiledb_fragment_info_get_mbr_var_size_from_name>( ctx, fragment_info, fid, mid, dim_name, start_size, end_size); } -int32_t tiledb_fragment_info_get_mbr_var_from_index( +CAPI_INTERFACE( + fragment_info_get_mbr_var_from_index, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, uint32_t did, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, fragment_info, fid, mid, did, start, end); } -int32_t tiledb_fragment_info_get_mbr_var_from_name( +CAPI_INTERFACE( + fragment_info_get_mbr_var_from_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t mid, const char* dim_name, void* start, - void* end) noexcept { + void* end) { return api_entry( ctx, fragment_info, fid, mid, dim_name, start, end); } -int32_t tiledb_fragment_info_get_cell_num( +CAPI_INTERFACE( + fragment_info_get_cell_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - uint64_t* cell_num) noexcept { + uint64_t* cell_num) { return api_entry( ctx, fragment_info, fid, cell_num); } -int32_t tiledb_fragment_info_get_total_cell_num( +CAPI_INTERFACE( + fragment_info_get_total_cell_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - uint64_t* cell_num) noexcept { + uint64_t* cell_num) { return api_entry( ctx, fragment_info, cell_num); } -int32_t tiledb_fragment_info_get_version( +CAPI_INTERFACE( + fragment_info_get_version, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - uint32_t* version) noexcept { + uint32_t* version) { return api_entry( ctx, fragment_info, fid, version); } -int32_t tiledb_fragment_info_has_consolidated_metadata( +CAPI_INTERFACE( + fragment_info_has_consolidated_metadata, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - int32_t* has) noexcept { + int32_t* has) { return api_entry( ctx, fragment_info, fid, has); } -int32_t tiledb_fragment_info_get_unconsolidated_metadata_num( +CAPI_INTERFACE( + fragment_info_get_unconsolidated_metadata_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - uint32_t* unconsolidated) noexcept { + uint32_t* unconsolidated) { return api_entry< tiledb::api::tiledb_fragment_info_get_unconsolidated_metadata_num>( ctx, fragment_info, unconsolidated); } -int32_t tiledb_fragment_info_get_to_vacuum_num( +CAPI_INTERFACE( + fragment_info_get_to_vacuum_num, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, - uint32_t* to_vacuum_num) noexcept { + uint32_t* to_vacuum_num) { return api_entry( ctx, fragment_info, to_vacuum_num); } -int32_t tiledb_fragment_info_get_to_vacuum_uri( +CAPI_INTERFACE( + fragment_info_get_to_vacuum_uri, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - const char** uri) noexcept { + const char** uri) { return api_entry( ctx, fragment_info, fid, uri); } -int32_t tiledb_fragment_info_get_array_schema( +CAPI_INTERFACE( + fragment_info_get_array_schema, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - tiledb_array_schema_t** array_schema) noexcept { + tiledb_array_schema_t** array_schema) { return api_entry( ctx, fragment_info, fid, array_schema); } -int32_t tiledb_fragment_info_get_array_schema_name( +CAPI_INTERFACE( + fragment_info_get_array_schema_name, tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t fid, - const char** schema_name) noexcept { + const char** schema_name) { return api_entry( ctx, fragment_info, fid, schema_name); } -int32_t tiledb_fragment_info_dump( +CAPI_INTERFACE( + fragment_info_dump, tiledb_ctx_t* ctx, const tiledb_fragment_info_t* fragment_info, - FILE* out) noexcept { + FILE* out) { return api_entry( ctx, fragment_info, out); } @@ -7509,65 +7756,71 @@ int32_t tiledb_fragment_info_dump( /* EXPERIMENTAL APIs */ /* ********************************* */ -TILEDB_EXPORT int32_t tiledb_query_get_status_details( +CAPI_INTERFACE( + query_get_status_details, tiledb_ctx_t* ctx, tiledb_query_t* query, - tiledb_query_status_details_t* status) noexcept { + tiledb_query_status_details_t* status) { return api_entry( ctx, query, status); } -int32_t tiledb_consolidation_plan_create_with_mbr( +CAPI_INTERFACE( + consolidation_plan_create_with_mbr, tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t fragment_size, - tiledb_consolidation_plan_t** consolidation_plan) noexcept { + tiledb_consolidation_plan_t** consolidation_plan) { return api_entry( ctx, array, fragment_size, consolidation_plan); } -void tiledb_consolidation_plan_free( - tiledb_consolidation_plan_t** consolidation_plan) noexcept { +CAPI_INTERFACE_VOID( + consolidation_plan_free, tiledb_consolidation_plan_t** consolidation_plan) { return api_entry_void( consolidation_plan); } -int32_t tiledb_consolidation_plan_get_num_nodes( +CAPI_INTERFACE( + consolidation_plan_get_num_nodes, tiledb_ctx_t* ctx, tiledb_consolidation_plan_t* consolidation_plan, - uint64_t* num_nodes) noexcept { + uint64_t* num_nodes) { return api_entry( ctx, consolidation_plan, num_nodes); } -int32_t tiledb_consolidation_plan_get_num_fragments( +CAPI_INTERFACE( + consolidation_plan_get_num_fragments, tiledb_ctx_t* ctx, tiledb_consolidation_plan_t* consolidation_plan, uint64_t node_index, - uint64_t* num_fragments) noexcept { + uint64_t* num_fragments) { return api_entry( ctx, consolidation_plan, node_index, num_fragments); } -int32_t tiledb_consolidation_plan_get_fragment_uri( +CAPI_INTERFACE( + consolidation_plan_get_fragment_uri, tiledb_ctx_t* ctx, tiledb_consolidation_plan_t* consolidation_plan, uint64_t node_index, uint64_t fragment_index, - const char** uri) noexcept { + const char** uri) { return api_entry( ctx, consolidation_plan, node_index, fragment_index, uri); } -int32_t tiledb_consolidation_plan_dump_json_str( +CAPI_INTERFACE( + consolidation_plan_dump_json_str, tiledb_ctx_t* ctx, const tiledb_consolidation_plan_t* consolidation_plan, - char** out) noexcept { + char** out) { return api_entry( ctx, consolidation_plan, out); } -int32_t tiledb_consolidation_plan_free_json_str(char** out) noexcept { +CAPI_INTERFACE(consolidation_plan_free_json_str, char** out) { return api_entry_plain( out); } diff --git a/tiledb/sm/c_api/tiledb_dimension_label.cc b/tiledb/sm/c_api/tiledb_dimension_label.cc index 5db966e77e25..dfab3f8c32cc 100644 --- a/tiledb/sm/c_api/tiledb_dimension_label.cc +++ b/tiledb/sm/c_api/tiledb_dimension_label.cc @@ -35,9 +35,7 @@ #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_dimension_label_experimental.h" -using namespace tiledb::common; - -namespace tiledb::common::detail { +namespace tiledb::api { capi_return_t tiledb_array_schema_add_dimension_label( tiledb_ctx_t* ctx, @@ -204,7 +202,7 @@ capi_return_t tiledb_subarray_has_label_ranges( return TILEDB_OK; } -} // namespace tiledb::common::detail +} // namespace tiledb::api using tiledb::api::api_entry_context; using tiledb::api::api_entry_with_context; @@ -212,153 +210,173 @@ using tiledb::api::api_entry_with_context; template constexpr auto api_entry = tiledb::api::api_entry_with_context; -capi_return_t tiledb_array_schema_add_dimension_label( +CAPI_INTERFACE( + array_schema_add_dimension_label, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const uint32_t dim_index, const char* name, tiledb_data_order_t label_order, - tiledb_datatype_t label_type) noexcept { - return api_entry( + tiledb_datatype_t label_type) { + return api_entry( ctx, array_schema, dim_index, name, label_order, label_type); } -capi_return_t tiledb_array_schema_get_dimension_label_from_name( +CAPI_INTERFACE( + array_schema_get_dimension_label_from_name, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, - tiledb_dimension_label_t** dim_label) noexcept { + tiledb_dimension_label_t** dim_label) { return api_entry_with_context< - detail::tiledb_array_schema_get_dimension_label_from_name>( + tiledb::api::tiledb_array_schema_get_dimension_label_from_name>( ctx, array_schema, label_name, dim_label); } -capi_return_t tiledb_array_schema_has_dimension_label( +CAPI_INTERFACE( + array_schema_has_dimension_label, tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, const char* name, - int32_t* has_dim_label) noexcept { - return api_entry_context( + int32_t* has_dim_label) { + return api_entry_context< + tiledb::api::tiledb_array_schema_has_dimension_label>( ctx, array_schema, name, has_dim_label); } -capi_return_t tiledb_array_schema_set_dimension_label_filter_list( +CAPI_INTERFACE( + array_schema_set_dimension_label_filter_list, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, - tiledb_filter_list_t* filter_list) noexcept { - return api_entry( + tiledb_filter_list_t* filter_list) { + return api_entry< + tiledb::api::tiledb_array_schema_set_dimension_label_filter_list>( ctx, array_schema, label_name, filter_list); } -capi_return_t tiledb_array_schema_set_dimension_label_tile_extent( +CAPI_INTERFACE( + array_schema_set_dimension_label_tile_extent, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, const char* label_name, tiledb_datatype_t type, - const void* tile_extent) noexcept { - return api_entry( + const void* tile_extent) { + return api_entry< + tiledb::api::tiledb_array_schema_set_dimension_label_tile_extent>( ctx, array_schema, label_name, type, tile_extent); } -capi_return_t tiledb_array_schema_get_dimension_label_num( +CAPI_INTERFACE( + array_schema_get_dimension_label_num, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, - uint64_t* dim_label_num) noexcept { - return api_entry_context( + uint64_t* dim_label_num) { + return api_entry_context< + tiledb::api::tiledb_array_schema_get_dimension_label_num>( ctx, array_schema, dim_label_num); } -capi_return_t tiledb_array_schema_get_dimension_label_from_index( +CAPI_INTERFACE( + array_schema_get_dimension_label_from_index, tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint64_t dim_label_index, - tiledb_dimension_label_t** dim_label) noexcept { + tiledb_dimension_label_t** dim_label) { return api_entry_context< - detail::tiledb_array_schema_get_dimension_label_from_index>( + tiledb::api::tiledb_array_schema_get_dimension_label_from_index>( ctx, array_schema, dim_label_index, dim_label); } -capi_return_t tiledb_subarray_add_label_range( +CAPI_INTERFACE( + subarray_add_label_range, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, const char* label_name, const void* start, const void* end, - const void* stride) noexcept { - return api_entry_context( + const void* stride) { + return api_entry_context( ctx, subarray, label_name, start, end, stride); } -capi_return_t tiledb_subarray_add_label_range_var( +CAPI_INTERFACE( + subarray_add_label_range_var, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, const char* label_name, const void* start, uint64_t start_size, const void* end, - uint64_t end_size) noexcept { - return api_entry_context( + uint64_t end_size) { + return api_entry_context( ctx, subarray, label_name, start, start_size, end, end_size); } -capi_return_t tiledb_subarray_get_label_name( +CAPI_INTERFACE( + subarray_get_label_name, tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, uint32_t dim_idx, - const char** label_name) noexcept { - return api_entry_context( + const char** label_name) { + return api_entry_context( ctx, subarray, dim_idx, label_name); } -capi_return_t tiledb_subarray_get_label_range( +CAPI_INTERFACE( + subarray_get_label_range, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, const void** start, const void** end, - const void** stride) noexcept { - return api_entry_context( + const void** stride) { + return api_entry_context( ctx, subarray, dim_name, range_idx, start, end, stride); } -capi_return_t tiledb_subarray_get_label_range_num( +CAPI_INTERFACE( + subarray_get_label_range_num, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, - uint64_t* range_num) noexcept { - return api_entry_context( + uint64_t* range_num) { + return api_entry_context( ctx, subarray, dim_name, range_num); } -capi_return_t tiledb_subarray_get_label_range_var( +CAPI_INTERFACE( + subarray_get_label_range_var, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, void* start, - void* end) noexcept { - return api_entry_context( + void* end) { + return api_entry_context( ctx, subarray, dim_name, range_idx, start, end); } -capi_return_t tiledb_subarray_get_label_range_var_size( +CAPI_INTERFACE( + subarray_get_label_range_var_size, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, uint64_t* start_size, - uint64_t* end_size) noexcept { - return api_entry_context( + uint64_t* end_size) { + return api_entry_context< + tiledb::api::tiledb_subarray_get_label_range_var_size>( ctx, subarray, dim_name, range_idx, start_size, end_size); } -capi_return_t tiledb_subarray_has_label_ranges( +CAPI_INTERFACE( + subarray_has_label_ranges, tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray, const uint32_t dim_idx, - int32_t* has_label_ranges) noexcept { - return api_entry_context( + int32_t* has_label_ranges) { + return api_entry_context( ctx, subarray, dim_idx, has_label_ranges); } diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 645c65dd13ab..7c8c5c8bed72 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -75,7 +75,7 @@ extern "C" { * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t -tiledb_log_warn(tiledb_ctx_t* ctx, const char* message); +tiledb_log_warn(tiledb_ctx_t* ctx, const char* message) TILEDB_NOEXCEPT; /* ********************************* */ /* AS BUILT */ diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index d659eafe5092..9c0a38342a51 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -62,7 +62,7 @@ #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/range/range.h" -namespace tiledb::common::detail { +namespace tiledb::api { // Forward declarations uint64_t compute_tile_extent_based_on_file_size(uint64_t file_size); @@ -809,69 +809,78 @@ uint64_t get_buffer_size_from_config( return buffer_size; } -} // namespace tiledb::common::detail +} // namespace tiledb::api using tiledb::api::api_entry_plain; template constexpr auto api_entry = tiledb::api::api_entry_with_context; -int32_t tiledb_filestore_schema_create( +CAPI_INTERFACE( + filestore_schema_create, tiledb_ctx_t* ctx, const char* uri, - tiledb_array_schema_t** array_schema) noexcept { - return api_entry( + tiledb_array_schema_t** array_schema) { + return api_entry( ctx, uri, array_schema); } -int32_t tiledb_filestore_uri_import( +CAPI_INTERFACE( + filestore_uri_import, tiledb_ctx_t* ctx, const char* filestore_array_uri, const char* file_uri, - tiledb_mime_type_t mime_type) noexcept { - return api_entry( + tiledb_mime_type_t mime_type) { + return api_entry( ctx, filestore_array_uri, file_uri, mime_type); } -int32_t tiledb_filestore_uri_export( +CAPI_INTERFACE( + filestore_uri_export, tiledb_ctx_t* ctx, const char* file_uri, - const char* filestore_array_uri) noexcept { - return api_entry( + const char* filestore_array_uri) { + return api_entry( ctx, file_uri, filestore_array_uri); } -int32_t tiledb_filestore_buffer_import( +CAPI_INTERFACE( + filestore_buffer_import, tiledb_ctx_t* ctx, const char* filestore_array_uri, void* buf, size_t size, - tiledb_mime_type_t mime_type) noexcept { - return api_entry( + tiledb_mime_type_t mime_type) { + return api_entry( ctx, filestore_array_uri, buf, size, mime_type); } -int32_t tiledb_filestore_buffer_export( +CAPI_INTERFACE( + filestore_buffer_export, tiledb_ctx_t* ctx, const char* filestore_array_uri, size_t offset, void* buf, - size_t size) noexcept { - return api_entry( + size_t size) { + return api_entry( ctx, filestore_array_uri, offset, buf, size); } -int32_t tiledb_filestore_size( - tiledb_ctx_t* ctx, const char* filestore_array_uri, size_t* size) noexcept { - return api_entry( +CAPI_INTERFACE( + filestore_size, + tiledb_ctx_t* ctx, + const char* filestore_array_uri, + size_t* size) { + return api_entry( ctx, filestore_array_uri, size); } -int32_t tiledb_mime_type_to_str( - tiledb_mime_type_t mime_type, const char** str) noexcept { - return api_entry_plain(mime_type, str); +CAPI_INTERFACE( + mime_type_to_str, tiledb_mime_type_t mime_type, const char** str) { + return api_entry_plain(mime_type, str); } -int32_t tiledb_mime_type_from_str( - const char* str, tiledb_mime_type_t* mime_type) noexcept { - return api_entry_plain(str, mime_type); -} \ No newline at end of file +CAPI_INTERFACE( + mime_type_from_str, const char* str, tiledb_mime_type_t* mime_type) { + return api_entry_plain( + str, mime_type); +} From af3cbf82cf66a5158d00100cc4f0b2cb6ad87c37 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 31 Oct 2023 17:27:32 +0200 Subject: [PATCH 043/456] Stop including the static library when releasing on Windows. (#4469) This PR stops passing `-EnableStaticTileDB` on the Windows release workflow, which means that the release artifacts will include include only the dynamic library, like on other platforms. The size of these artifacts on Windows drops from 175MB [to just 7MB](https://dev.azure.com/TileDB-Inc/CI/_build/results?buildId=35970&view=artifacts&pathAsName=false&type=publishedArtifacts). Users that want to statically link to TileDB will have to build it themselves. TileDB-Java used to do that until TileDB-Inc/TileDB-Java#314, and according to [a search](https://github.com/search?q=TileDB%3A%3Atiledb_static++NOT+is%3Aarchived+NOT+is%3Afork&type=code) of the `TileDB::tiledb_static` string on public non-archived and non-forked repositories, none of these uses would be broken. > [!NOTE] > TileDB-R links statically on Windows as well but it does not use these artifacts so it will not be affected. --- TYPE: BREAKING_BEHAVIOR DESC: The Windows release artifacts no longer include static libraries. --- scripts/azure-windows-release.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/scripts/azure-windows-release.yml b/scripts/azure-windows-release.yml index d0722c5d36a5..6b36ca90d339 100644 --- a/scripts/azure-windows-release.yml +++ b/scripts/azure-windows-release.yml @@ -27,7 +27,7 @@ steps: $host.SetShouldExit(3) } - $bootstrapOptions = "-EnableVerbose -EnableStaticTileDB -DisableTests" + $bootstrapOptions = "-EnableVerbose -DisableTests" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } @@ -41,10 +41,9 @@ steps: if ($env:TILEDB_SERIALIZATION -eq "ON") { $bootstrapOptions = "-EnableSerialization " + $bootstrapOptions } - # static already added above as initial default - # if ($env:TILEDB_STATIC -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableStaticTileDB" - # } + if ($env:TILEDB_STATIC -eq "ON") { + $bootstrapOptions = $bootstrapOptions + " -EnableStaticTileDB" + } # if ($env:TILEDB_HDFS -eq "ON") { # $bootstrapOptions = $bootstrapOptions + " -EnableHDFS" #} @@ -60,10 +59,6 @@ steps: # if ($env:TILEDB_CI_TSAN -eq "ON") { # $bootstrapOptions = $bootstrapOptions + " -EnableSanitizer thread -EnableDebug" # } - # static already added above as initial default - # if ($env:TILEDB_FORCE_BUILD_DEPS" -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableBuildDeps" - # } if ($env:TILEDB_WEBP -eq "OFF") { $bootstrapOptions = $bootstrapOptions + " -DisableWebP" } From 9163edf046e5a6c25c6fab36c0dab84f62890cce Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 1 Nov 2023 10:04:20 -0500 Subject: [PATCH 044/456] Fix serialization of empty enumerations (#4472) This was an obvious oversight in the original PR. I managed to cover enumeration extension serialization but forgot to include serialization of empty arrays. --- TYPE: BUG DESC: Fix serialization of empty enumerations --- test/src/unit-enumerations.cc | 33 ++++++++++++++++++++++++++ tiledb/sm/c_api/tiledb.cc | 2 +- tiledb/sm/serialization/enumeration.cc | 23 +++++++++--------- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 7840f81c3475..ba2b686aa452 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2119,6 +2119,39 @@ TEST_CASE_METHOD( REQUIRE(loaded_names2.empty()); } +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - ArraySchema Serialization with Empty Enumerations", + "[enumeration][capnp][array-schema][empty-enumerations]") { + create_array(); + + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = create_schema(); + + auto enmr1 = Enumeration::create( + "empty_fixed", Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + auto enmr2 = Enumeration::create( + "empty_var", + Datatype::STRING_ASCII, + constants::var_num, + false, + nullptr, + 0, + nullptr, + 0); + + schema1->add_enumeration(enmr1); + schema1->add_enumeration(enmr2); + + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); +} + TEST_CASE_METHOD( EnumerationFx, "Cap'N Proto - Basic ArraySchemaEvolution Serialization", diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index b4f5394211b2..cad0bd29c9b3 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3595,7 +3595,7 @@ int32_t tiledb_deserialize_array_schema( } catch (...) { delete *array_schema; *array_schema = nullptr; - return TILEDB_ERR; + throw; } return TILEDB_OK; diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 6cdcaee2f3c0..34314cd64eb2 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -58,10 +58,12 @@ void enumeration_to_capnp( enmr_builder.setOrdered(enumeration->ordered()); auto dspan = enumeration->data(); - enmr_builder.setData(::kj::arrayPtr(dspan.data(), dspan.size())); + if (dspan.size() > 0) { + enmr_builder.setData(::kj::arrayPtr(dspan.data(), dspan.size())); + } - if (enumeration->var_size()) { - auto ospan = enumeration->offsets(); + auto ospan = enumeration->offsets(); + if (ospan.size() > 0) { enmr_builder.setOffsets(::kj::arrayPtr(ospan.data(), ospan.size())); } } @@ -73,15 +75,14 @@ shared_ptr enumeration_from_capnp( Datatype datatype = Datatype::ANY; throw_if_not_ok(datatype_enum(reader.getType(), &datatype)); - if (!reader.hasData()) { - throw SerializationStatusException( - "[Deserialization::enumeration_from_capnp] Deserialization of " - "Enumeration is missing its data buffer."); - } + const void* data = nullptr; + uint64_t data_size = 0; - auto data_reader = reader.getData().asBytes(); - auto data = data_reader.begin(); - auto data_size = data_reader.size(); + if (reader.hasData()) { + auto data_reader = reader.getData().asBytes(); + data = data_reader.begin(); + data_size = data_reader.size(); + } const void* offsets = nullptr; uint64_t offsets_size = 0; From 89f23eb5447ab8018d492d11056eeb316106f859 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 2 Nov 2023 10:13:30 +0200 Subject: [PATCH 045/456] Add `tiledb_group_get_member_by(index|name)_v2`. (#4019) [SC-23227](https://app.shortcut.com/tiledb-inc/story/23227/fix-tiledb-group-get-member-by-index-api) The APIs' implementation on the Core side is very similar to #3842. Just like that PR, the old functions were left untested. ~~For the C++ API I created a `StringHandleHolder` class that manages the handles' lifetime, similar to [what I have done in C#](https://github.com/TileDB-Inc/TileDB-CSharp/blob/main/sources/TileDB.CSharp/Marshalling/StringHandleHolder.cs). It is easier and safer to use than the existing way of using unique pointers.~~ --- TYPE: C_API DESC: Add new C API functions to get group members and deprecate the existing ones. --- doc/policy/api_changes.md | 5 + test/src/unit-capi-group.cc | 62 ++++++++-- tiledb/CMakeLists.txt | 1 + tiledb/api/CMakeLists.txt | 1 + tiledb/api/c_api/group/group_api.cc | 80 +++++++++++++ .../group/group_api_external_experimental.h | 95 +++++++++++++-- tiledb/api/cpp_api_support/CMakeLists.txt | 29 +++++ .../api/cpp_api_support/test/CMakeLists.txt | 32 +++++ .../test/unit_cppapi_string.cc | 90 ++++++++++++++ tiledb/sm/cpp_api/capi_string.h | 111 ++++++++++++++++++ tiledb/sm/cpp_api/deleter.h | 7 -- tiledb/sm/cpp_api/fragment_info.h | 8 +- tiledb/sm/cpp_api/group_experimental.h | 27 ++--- 13 files changed, 497 insertions(+), 51 deletions(-) create mode 100644 tiledb/api/cpp_api_support/CMakeLists.txt create mode 100644 tiledb/api/cpp_api_support/test/CMakeLists.txt create mode 100644 tiledb/api/cpp_api_support/test/unit_cppapi_string.cc create mode 100644 tiledb/sm/cpp_api/capi_string.h diff --git a/doc/policy/api_changes.md b/doc/policy/api_changes.md index ec935b877de4..25c689d52216 100644 --- a/doc/policy/api_changes.md +++ b/doc/policy/api_changes.md @@ -74,6 +74,11 @@ Function removal shall be announced one release cycle before removal, following - `tiledb_query_submit_async` +### 2.18.0..2.19.0 + +- `tiledb_group_get_member_by_index` +- `tiledb_group_get_member_by_name` + ## Deprecation History Generation
diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index fa539870102b..2f32c87a00fd 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -86,7 +86,7 @@ struct GroupFx { std::string get_golden_ls(const std::string& path); static std::string random_name(const std::string& prefix); std::vector> read_group( - tiledb_group_t* group) const; + tiledb_group_t* group, bool use_get_member_by_index_v2 = true) const; void set_group_timestamp( tiledb_group_t* group, const uint64_t& timestamp) const; void write_group_metadata(const char* group_uri) const; @@ -230,20 +230,49 @@ void GroupFx::vacuum(const char* group_uri) const { } std::vector> GroupFx::read_group( - tiledb_group_t* group) const { + tiledb_group_t* group, bool use_get_member_by_index_v2) const { std::vector> ret; uint64_t count = 0; - char* uri; - tiledb_object_t type; - char* name; int rc = tiledb_group_get_member_count(ctx_, group, &count); REQUIRE(rc == TILEDB_OK); - for (uint64_t i = 0; i < count; i++) { - rc = tiledb_group_get_member_by_index(ctx_, group, i, &uri, &type, &name); - REQUIRE(rc == TILEDB_OK); - ret.emplace_back(uri, type); - std::free(uri); - std::free(name); + if (use_get_member_by_index_v2) { + for (uint64_t i = 0; i < count; i++) { + std::string uri; + tiledb_object_t type; + tiledb_string_t *uri_handle, *name_handle; + rc = tiledb_group_get_member_by_index_v2( + ctx_, group, i, &uri_handle, &type, &name_handle); + REQUIRE(rc == TILEDB_OK); + const char* uri_str; + size_t uri_size; + rc = tiledb_string_view(uri_handle, &uri_str, &uri_size); + CHECK(rc == TILEDB_OK); + uri = std::string(uri_str, uri_size); + rc = tiledb_string_free(&uri_handle); + CHECK(rc == TILEDB_OK); + if (name_handle) { + rc = tiledb_string_free(&name_handle); + CHECK(rc == TILEDB_OK); + } + ret.emplace_back(uri, type); + } + } else { + // When tiledb_group_get_member_by_index gets removed, the else part can + // simply be removed. + for (uint64_t i = 0; i < count; i++) { + std::string uri; + tiledb_object_t type; + char *uri_ptr, *name_ptr; + rc = tiledb_group_get_member_by_index( + ctx_, group, i, &uri_ptr, &type, &name_ptr); + REQUIRE(rc == TILEDB_OK); + uri = uri_ptr == nullptr ? "" : std::string(uri_ptr); + std::free(uri_ptr); + if (name_ptr) { + std::free(name_ptr); + } + ret.emplace_back(uri, type); + } } return ret; } @@ -549,6 +578,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( GroupFx, "C API: Group, write/read", "[capi][group][metadata][read]") { + bool use_get_member_by_index_v2 = true; + SECTION("Using tiledb_group_get_member_by_index_v2") { + use_get_member_by_index_v2 = true; + } + SECTION("Using tiledb_group_get_member_by_index") { + use_get_member_by_index_v2 = false; + } // Create and open group in write mode // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); @@ -618,12 +654,12 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); std::vector> group1_received = - read_group(group1); + read_group(group1, use_get_member_by_index_v2); REQUIRE_THAT( group1_received, Catch::Matchers::UnorderedEquals(group1_expected)); std::vector> group2_received = - read_group(group2); + read_group(group2, use_get_member_by_index_v2); REQUIRE_THAT( group2_received, Catch::Matchers::UnorderedEquals(group2_expected)); diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 19aed81ca5e4..1ce86d43f4fe 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -120,6 +120,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/schema_base.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/stats.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/capi_string.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/subarray.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/subarray_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/type.h diff --git a/tiledb/api/CMakeLists.txt b/tiledb/api/CMakeLists.txt index a0d6c8911c71..54c307cb9117 100644 --- a/tiledb/api/CMakeLists.txt +++ b/tiledb/api/CMakeLists.txt @@ -29,6 +29,7 @@ include(common-root) add_subdirectory(c_api) add_subdirectory(c_api_support) add_subdirectory(c_api_test_support) +add_subdirectory(cpp_api_support) get_gathered(API_SOURCES) set(TILEDB_API_SOURCES ${API_SOURCES} PARENT_SCOPE) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index bf7b17ff2198..4d6e374bcb17 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -41,6 +41,7 @@ #include "../buffer/buffer_api_internal.h" #include "../config/config_api_internal.h" #include "../context/context_api_internal.h" +#include "../string/string_api_internal.h" #include "group_api_internal.h" @@ -306,6 +307,10 @@ capi_return_t tiledb_group_get_member_by_index( ensure_output_pointer_is_valid(type); ensure_output_pointer_is_valid(name); + LOG_WARN( + "tiledb_group_get_member_by_index is deprecated. Please use " + "tiledb_group_get_member_by_index_v2 instead."); + char* tmp_uri = nullptr; char* tmp_name = nullptr; @@ -343,6 +348,34 @@ capi_return_t tiledb_group_get_member_by_index( return TILEDB_OK; } +capi_return_t tiledb_group_get_member_by_index_v2( + tiledb_group_handle_t* group, + uint64_t index, + tiledb_string_t** uri, + tiledb_object_t* type, + tiledb_string_t** name) { + ensure_group_is_valid(group); + ensure_output_pointer_is_valid(uri); + ensure_output_pointer_is_valid(type); + ensure_output_pointer_is_valid(name); + + auto&& [uri_str, object_type, name_str] = + group->group().member_by_index(index); + + *uri = tiledb_string_handle_t::make_handle(uri_str); + *type = static_cast(object_type); + try { + *name = name_str.has_value() ? + tiledb_string_handle_t::make_handle(*name_str) : + nullptr; + } catch (...) { + tiledb_string_handle_t::break_handle(*uri); + throw; + } + + return TILEDB_OK; +} + capi_return_t tiledb_group_get_member_by_name( tiledb_group_handle_t* group, const char* name, @@ -353,6 +386,10 @@ capi_return_t tiledb_group_get_member_by_name( ensure_output_pointer_is_valid(uri); ensure_output_pointer_is_valid(type); + LOG_WARN( + "tiledb_group_get_member_by_name is deprecated. Please use " + "tiledb_group_get_member_by_name_v2 instead."); + auto&& [uri_str, object_type, name_str, ignored_relative] = group->group().member_by_name(name); @@ -366,6 +403,26 @@ capi_return_t tiledb_group_get_member_by_name( return TILEDB_OK; } +capi_return_t tiledb_group_get_member_by_name_v2( + tiledb_group_handle_t* group, + const char* name, + tiledb_string_t** uri, + tiledb_object_t* type) { + ensure_group_is_valid(group); + ensure_output_pointer_is_valid(uri); + ensure_output_pointer_is_valid(type); + + std::string uri_str; + sm::ObjectType object_type; + std::tie(uri_str, object_type, std::ignore, std::ignore) = + group->group().member_by_name(name); + + *uri = tiledb_string_handle_t::make_handle(std::move(uri_str)); + *type = static_cast(object_type); + + return TILEDB_OK; +} + capi_return_t tiledb_group_get_is_relative_uri_by_name( tiledb_group_handle_t* group, const char* name, uint8_t* is_relative) { ensure_group_is_valid(group); @@ -716,6 +773,18 @@ CAPI_INTERFACE( ctx, group, index, uri, type, name); } +CAPI_INTERFACE( + group_get_member_by_index_v2, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t index, + tiledb_string_t** uri, + tiledb_object_t* type, + tiledb_string_t** name) { + return api_entry_context( + ctx, group, index, uri, type, name); +} + CAPI_INTERFACE( group_get_member_by_name, tiledb_ctx_t* ctx, @@ -727,6 +796,17 @@ CAPI_INTERFACE( ctx, group, name, uri, type); } +CAPI_INTERFACE( + group_get_member_by_name_v2, + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name, + tiledb_string_t** uri, + tiledb_object_t* type) { + return api_entry_context( + ctx, group, name, uri, type); +} + CAPI_INTERFACE( group_get_is_relative_uri_by_name, tiledb_ctx_t* ctx, diff --git a/tiledb/api/c_api/group/group_api_external_experimental.h b/tiledb/api/c_api/group/group_api_external_experimental.h index a661bd15a722..1d5fdc81341c 100644 --- a/tiledb/api/c_api/group/group_api_external_experimental.h +++ b/tiledb/api/c_api/group/group_api_external_experimental.h @@ -400,7 +400,8 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_count( tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* count) TILEDB_NOEXCEPT; /** - * Get a member of a group by index and details of group + * Get a member of a group by index and details of group. + * Deprecated, use \p tiledb_group_get_member_by_index_v2 instead. * * **Example:** * @@ -424,14 +425,12 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_count( * @param ctx The TileDB context. * @param group An group opened in READ mode. * @param index index of member to fetch - * @param uri URI of member, The caller takes ownership - * of the c-string. + * @param uri URI of member. * @param type type of member - * @param name name of member, The caller takes ownership - * of the c-string. NULL if name was not set + * @param name name of member. NULL if name was not set * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ -TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index( +TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_index( tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t index, @@ -448,6 +447,46 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index( * tiledb_group_t* group; * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tiledb_string_t *uri, *name; + * tiledb_object_t type; + * tiledb_group_get_member_by_index_v2(ctx, group, 0, &uri, &type, &name); + * + * tiledb_string_free(uri); + * tiledb_string_free(name); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param index index of member to fetch + * @param uri Handle to the URI of the member. + * @param type type of member + * @param name Handle to the name of the member. NULL if name was not set + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index_v2( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t index, + tiledb_string_t** uri, + tiledb_object_t* type, + tiledb_string_t** name) TILEDB_NOEXCEPT; + +/** + * Get a member of a group by name and details of group. + * Deprecated, use \p tiledb_group_get_member_by_name_v2. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", * "group2"); @@ -465,18 +504,56 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index( * @param ctx The TileDB context. * @param group An group opened in READ mode. * @param name name of member to fetch - * @param uri URI of member, The caller takes ownership - * of the c-string. + * @param uri URI of member * @param type type of member * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ -TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_name( +TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_name( tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name, char** uri, tiledb_object_t* type) TILEDB_NOEXCEPT; +/** + * Get a member of a group by name and details of group. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", + * "group2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tilledb_string_t *uri; + * tiledb_object_t type; + * tiledb_group_get_member_by_name(ctx, group, "array1", &uri, &type); + * + * tiledb_string_free(uri); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param name name of member to fetch + * @param uri Handle to the URI of the member. + * @param type type of member + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_name_v2( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name, + tiledb_string_t** uri, + tiledb_object_t* type) TILEDB_NOEXCEPT; + +/* (clang format was butchering the tiledb_group_add_member() calls) */ +/* clang-format off */ /** * Get a member of a group by name and relative characteristic of that name * diff --git a/tiledb/api/cpp_api_support/CMakeLists.txt b/tiledb/api/cpp_api_support/CMakeLists.txt new file mode 100644 index 000000000000..b8a0972697c0 --- /dev/null +++ b/tiledb/api/cpp_api_support/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# tiledb/api/cpp_api_support/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) + +add_test_subdirectory() diff --git a/tiledb/api/cpp_api_support/test/CMakeLists.txt b/tiledb/api/cpp_api_support/test/CMakeLists.txt new file mode 100644 index 000000000000..30acaadf619e --- /dev/null +++ b/tiledb/api/cpp_api_support/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/api/cpp_api_support/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test cppapi_string) + this_target_sources(unit_cppapi_string.cc) + this_target_object_libraries(capi_string) +conclude(unit_test) diff --git a/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc new file mode 100644 index 000000000000..ffddb3f5012d --- /dev/null +++ b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc @@ -0,0 +1,90 @@ +/** + * @file tiledb/api/cpp_api_support/test/unit_cppapi_string.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines tests for the CAPIString class of the TileDB C++ API. + */ + +#include + +#include "tiledb/api/c_api/string/string_api_internal.h" +#include "tiledb/sm/cpp_api/capi_string.h" + +using namespace tiledb::impl; + +TEST_CASE( + "CAPIString: Test constructor with null parameter throws", + "[capi_string][null-param]") { + REQUIRE_THROWS_AS(CAPIString(nullptr), std::invalid_argument); +} + +TEST_CASE( + "CAPIString: Test constructor with non-null parameter pointing to null " + "handle throws", + "[capi_string][null-param-ptr]") { + tiledb_string_t* string = nullptr; + REQUIRE_THROWS_AS(CAPIString(&string), std::invalid_argument); +} + +TEST_CASE( + "CAPIString: Test creating string handle and getting its value", + "[capi_string][get]") { + const std::string test_string = "hello"; + tiledb_string_t* handle = tiledb_string_t::make_handle(test_string); + std::string result; + + SECTION("convert_to_string") { + auto result_maybe = convert_to_string(&handle); + REQUIRE(result_maybe); + result = *result_maybe; + } + SECTION("CAPIString") { + result = CAPIString(&handle).str(); + } + + REQUIRE(handle == nullptr); + REQUIRE(result == test_string); +} + +TEST_CASE( + "CAPIString: Test that accessing freed handle fails", + "[capi_string][freed_handle]") { + const std::string test_string = "hello"; + tiledb_string_t* handle = tiledb_string_t::make_handle(test_string); + tiledb_string_t* handle_copy = handle; + std::ignore = convert_to_string(&handle); + const char* chars = nullptr; + size_t length = 0; + REQUIRE(tiledb_string_view(handle_copy, &chars, &length) == TILEDB_ERR); +} + +TEST_CASE( + "CAPIString: Test convert_to_string with null handle", "[capi_string]") { + tiledb_string_t* handle = nullptr; + REQUIRE(!convert_to_string(&handle).has_value()); +} diff --git a/tiledb/sm/cpp_api/capi_string.h b/tiledb/sm/cpp_api/capi_string.h new file mode 100644 index 000000000000..bd8ffa467687 --- /dev/null +++ b/tiledb/sm/cpp_api/capi_string.h @@ -0,0 +1,111 @@ +/** + * @file capi_string.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB CAPIString object. + */ + +#ifndef TILEDB_CPP_API_CAPI_STRING_H +#define TILEDB_CPP_API_CAPI_STRING_H + +#include +#include +#include +#include "exception.h" +#include "log.h" +#include "tiledb.h" + +namespace tiledb::impl { + +/** + * Manages the lifetime of a tiledb_string_t* handle and provides operations on + * it. + */ +class CAPIString { + public: + /** + * Constructor. Takes ownership of the handle. + * + * @param handle A pointer to the string handle. Must not be null and must not + * point to a null handle. + */ + CAPIString(tiledb_string_t** handle) { + if (handle == nullptr || *handle == nullptr) { + throw std::invalid_argument( + "Pointer to string handle cannot be null or point to null handle."); + } + string_ = *handle; + *handle = nullptr; + } + + ~CAPIString() { + auto result = tiledb_status(tiledb_string_free(&string_)); + if (result != TILEDB_OK) { + log_warn("Could not free string; Error code: " + std::to_string(result)); + } + } + + // Disable copy and move. Because this class owns a resource, + // copying it must not be supported, but moving it could be. + CAPIString(const CAPIString&) = delete; + CAPIString operator=(const CAPIString&) = delete; + CAPIString(const CAPIString&&) = delete; + CAPIString operator=(const CAPIString&&) = delete; + + std::string str() const { + const char* c; + size_t size; + auto status = tiledb_status(tiledb_string_view(string_, &c, &size)); + if (status != TILEDB_OK) { + throw TileDBError( + "Could not view string; Error code: " + std::to_string(status)); + } + return {c, size}; + } + + private: + /** The C API string handle. Invariant: must not be null. */ + tiledb_string_t* string_; +}; + +/** + * Returns a C++ string with the handle's data. The handle is subsequently + * freed. + * + * If the handle is null returns nullopt. + */ +inline std::optional convert_to_string(tiledb_string_t** handle) { + if (*handle == nullptr) { + return std::nullopt; + } + return CAPIString(handle).str(); +} + +} // namespace tiledb::impl + +#endif // TILEDB_CPP_API_CAPI_STRING_H diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index abd5eefefce0..90d466c44039 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -149,13 +149,6 @@ class Deleter { tiledb_consolidation_plan_free(&p); } - void operator()(tiledb_string_t* p) const { - capi_status_t result = tiledb_status(tiledb_string_free(&p)); - if (result != TILEDB_OK) { - log_warn("Could not free string; Error code: " + std::to_string(result)); - } - } - void operator()(tiledb_query_channel_t* p) const { tiledb_query_channel_free(ctx_->ptr().get(), &p); } diff --git a/tiledb/sm/cpp_api/fragment_info.h b/tiledb/sm/cpp_api/fragment_info.h index 72f9cf91ec4d..1e254657fdb4 100644 --- a/tiledb/sm/cpp_api/fragment_info.h +++ b/tiledb/sm/cpp_api/fragment_info.h @@ -34,6 +34,7 @@ #define TILEDB_CPP_API_FRAGMENT_INFO_H #include "array_schema.h" +#include "capi_string.h" #include "context.h" #include "deleter.h" #include "exception.h" @@ -97,12 +98,7 @@ class FragmentInfo { tiledb_string_t* name; ctx.handle_error(tiledb_fragment_info_get_fragment_name_v2( ctx.ptr().get(), fragment_info_.get(), fid, &name)); - auto name_ptr = - std::unique_ptr(name); - const char* name_c; - size_t length; - ctx.handle_error(tiledb_string_view(name_ptr.get(), &name_c, &length)); - return std::string(name_c, length); + return impl::convert_to_string(&name).value(); } /** diff --git a/tiledb/sm/cpp_api/group_experimental.h b/tiledb/sm/cpp_api/group_experimental.h index bdd8a07ad6ad..daae9387a2a3 100644 --- a/tiledb/sm/cpp_api/group_experimental.h +++ b/tiledb/sm/cpp_api/group_experimental.h @@ -35,6 +35,7 @@ #ifndef TILEDB_CPP_API_GROUP_EXPERIMENTAL_H #define TILEDB_CPP_API_GROUP_EXPERIMENTAL_H +#include "capi_string.h" #include "context.h" #include "object.h" #include "tiledb.h" @@ -424,31 +425,25 @@ class Group { tiledb::Object member(uint64_t index) const { auto& ctx = ctx_.get(); tiledb_ctx_t* c_ctx = ctx.ptr().get(); - char* uri; + tiledb_string_t* uri; tiledb_object_t type; - char* name; - ctx.handle_error(tiledb_group_get_member_by_index( + tiledb_string_t* name; + ctx.handle_error(tiledb_group_get_member_by_index_v2( c_ctx, group_.get(), index, &uri, &type, &name)); - std::string uri_str(uri); - std::free(uri); - std::optional name_opt = std::nullopt; - if (name != nullptr) { - name_opt = name; - std::free(name); - } - return tiledb::Object(type, uri_str, name_opt); + return tiledb::Object( + type, + impl::convert_to_string(&uri).value(), + impl::convert_to_string(&name)); } tiledb::Object member(std::string name) const { auto& ctx = ctx_.get(); tiledb_ctx_t* c_ctx = ctx.ptr().get(); - char* uri; + tiledb_string_t* uri; tiledb_object_t type; - ctx.handle_error(tiledb_group_get_member_by_name( + ctx.handle_error(tiledb_group_get_member_by_name_v2( c_ctx, group_.get(), name.c_str(), &uri, &type)); - std::string uri_str(uri); - std::free(uri); - return tiledb::Object(type, uri_str, name); + return tiledb::Object(type, impl::convert_to_string(&uri).value(), name); } /** From 9c206beaf6d938b1b77d1abf131e199a57a109c6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 2 Nov 2023 04:26:52 -0500 Subject: [PATCH 046/456] Add Enumeration REST CI Tests (#4474) The Enumeration routes have been merge to TileDB-Cloud-REST so we can finally merge the REST CI tests. --- TYPE: NO_HISTORY DESC: Add Enumeration REST CI Tests. --- test/CMakeLists.txt | 1 + test/src/unit-rest-enumerations.cc | 235 +++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 test/src/unit-rest-enumerations.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8dd294ed847a..f9ecbc6f7bd1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -262,6 +262,7 @@ endif() if (TILEDB_TESTS_ENABLE_REST) list(APPEND TILEDB_UNIT_TEST_SOURCES src/unit-capi-rest-dense_array.cc + src/unit-rest-enumerations.cc ) endif() diff --git a/test/src/unit-rest-enumerations.cc b/test/src/unit-rest-enumerations.cc new file mode 100644 index 000000000000..c7f13c01b194 --- /dev/null +++ b/test/src/unit-rest-enumerations.cc @@ -0,0 +1,235 @@ +/** + * @file unit-rest-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests serialization of Enumerations via a REST server. + */ + +#include "test/support/src/vfs_helpers.h" +#include "test/support/tdb_catch.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +using namespace tiledb; + +struct RESTEnumerationFx { + RESTEnumerationFx(); + ~RESTEnumerationFx(); + + void create_array(const std::string& array_name); + void rm_array(); + + std::string bucket_; + std::string uri_; + Config cfg_; + Context ctx_; + VFS vfs_; +}; + +TEST_CASE_METHOD( + RESTEnumerationFx, + "Create array test", + "[rest][enumeration][create-array]") { + create_array("simple-array-create"); +} + +TEST_CASE_METHOD( + RESTEnumerationFx, + "Simple enumeration query", + "[rest][enumeration][simple-query]") { + create_array("simple-query"); + + Array array(ctx_, uri_, TILEDB_READ); + Subarray subarray(ctx_, array); + subarray.set_subarray({1, 5}); + + QueryCondition qc(ctx_); + qc.init("attr1", "wilma", 5, TILEDB_EQ); + + std::vector attr1_read(5); + std::vector attr2_read(5); + + Query query(ctx_, array); + query.set_subarray(subarray) + .set_condition(qc) + .set_data_buffer("attr1", attr1_read) + .set_data_buffer("attr2", attr2_read); + + REQUIRE(query.submit() == Query::Status::COMPLETE); + REQUIRE(attr1_read[1] == 1); + REQUIRE(attr1_read[3] == 1); +} + +TEST_CASE_METHOD( + RESTEnumerationFx, + "Get enumeration", + "[rest][enumeration][get-enumeration]") { + create_array("get-enumeration"); + + Array array(ctx_, uri_, TILEDB_READ); + auto enmr = ArrayExperimental::get_enumeration(ctx_, array, "my_enum"); + + std::vector expected = {"fred", "wilma", "barney", "pebbles"}; + REQUIRE(enmr.as_vector() == expected); +} + +TEST_CASE_METHOD( + RESTEnumerationFx, + "Get previously loaded enumeration", + "[rest][enumeration][get-enumeration]") { + create_array("get-enumeration"); + + Array array(ctx_, uri_, TILEDB_READ); + auto enmr1 = ArrayExperimental::get_enumeration(ctx_, array, "my_enum"); + auto enmr2 = ArrayExperimental::get_enumeration(ctx_, array, "my_enum"); + + REQUIRE(enmr1.ptr()->copy() == enmr2.ptr()->copy()); + + std::vector expected = {"fred", "wilma", "barney", "pebbles"}; + REQUIRE(enmr2.as_vector() == expected); +} + +TEST_CASE_METHOD( + RESTEnumerationFx, + "Enumeration Extension", + "[rest][enumeration][extension]") { + create_array("get-enumeration"); + + Array old_array(ctx_, uri_, TILEDB_READ); + auto old_enmr = ArrayExperimental::get_enumeration(ctx_, old_array, "fruit"); + + std::vector fruit = { + "apple", "blueberry", "cherry", "durian", "elderberry"}; + auto new_enmr = old_enmr.extend(fruit); + + ArraySchemaEvolution ase(ctx_); + ase.extend_enumeration(new_enmr); + ase.array_evolve(uri_); + + Array new_array(ctx_, uri_, TILEDB_READ); + auto enmr = ArrayExperimental::get_enumeration(ctx_, new_array, "fruit"); + REQUIRE(enmr.as_vector() == fruit); +} + +Config& setup_config(Config& cfg) { + cfg["vfs.s3.endpoint_override"] = "localhost:9999"; + cfg["vfs.s3.scheme"] = "https"; + cfg["vfs.s3.use_virtual_addressing"] = "false"; + cfg["ssl.verify"] = "false"; + return cfg; +} + +RESTEnumerationFx::RESTEnumerationFx() + : bucket_("s3://enumeration-tests") + , ctx_(setup_config(cfg_)) + , vfs_(ctx_) { + if (!vfs_.is_bucket(bucket_)) { + vfs_.create_bucket(bucket_); + } +} + +RESTEnumerationFx::~RESTEnumerationFx() { + rm_array(); + if (vfs_.is_bucket(bucket_)) { + vfs_.remove_bucket(bucket_); + } +} + +void RESTEnumerationFx::create_array(const std::string& array_name) { + uri_ = "tiledb://unit/" + bucket_ + "/" + array_name; + + // Ensure that no array exists at uri_ + rm_array(); + + // Create a simple array for testing. This ends up with just five elements in + // the array. dim is an int32_t dimension, attr1 is an enumeration with string + // values and int32_t attribute values. attr2 is a float attribute. + // + // The array data is summarized as below, however, pay attention to the fact + // that attr1 is storing integral index values instead of the raw string data. + // + // dim = {1, 2, 3, 4, 5} + // attr1 = {"fred", "wilma", "barney", "wilma", "fred"} + // attr2 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f} + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + // The list of string values in the attr1 enumeration + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, "my_enum", values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr1 = Attribute::create(ctx_, "attr1"); + AttributeExperimental::set_enumeration_name(ctx_, attr1, "my_enum"); + schema.add_attribute(attr1); + + auto attr2 = Attribute::create(ctx_, "attr2"); + schema.add_attribute(attr2); + + auto fruit = Enumeration::create_empty( + ctx_, "fruit", TILEDB_STRING_ASCII, TILEDB_VAR_NUM); + ArraySchemaExperimental::add_enumeration(ctx_, schema, fruit); + + auto attr3 = Attribute::create(ctx_, "attr3"); + AttributeExperimental::set_enumeration_name(ctx_, attr3, "fruit"); + schema.add_attribute(attr3); + + Array::create(uri_, schema); + + // Attribute data + std::vector attr1_values = {0, 1, 2, 1, 0}; + std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + std::vector attr3_values = {0, 1, 2, 3, 4}; + + Array array(ctx_, uri_, TILEDB_WRITE); + Subarray subarray(ctx_, array); + subarray.set_subarray({1, 5}); + Query query(ctx_, array); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("attr1", attr1_values) + .set_data_buffer("attr2", attr2_values) + .set_data_buffer("attr3", attr3_values); + CHECK_NOTHROW(query.submit()); + query.finalize(); + array.close(); +} + +void RESTEnumerationFx::rm_array() { + auto obj = Object::object(ctx_, uri_); + if (obj.type() == Object::Type::Array) { + Array::delete_array(ctx_, uri_); + } +} From 73d15c66ae56a307902fed4c4fbb20200a72a5fe Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:22:56 +0100 Subject: [PATCH 047/456] Remove unused compression defines. (#4480) --- TYPE: IMPROVEMENT DESC: Remove unused compression defines. --- tiledb/api/c_api/api_external_common.h | 3 --- tiledb/sm/filter/compression_filter.h | 3 --- 2 files changed, 6 deletions(-) diff --git a/tiledb/api/c_api/api_external_common.h b/tiledb/api/c_api/api_external_common.h index b42a63c6cead..c1143fb31bd3 100644 --- a/tiledb/api/c_api/api_external_common.h +++ b/tiledb/api/c_api/api_external_common.h @@ -156,9 +156,6 @@ TILEDB_EXPORT capi_status_t tiledb_status_code(capi_return_t x); * Invalid error argument would prevent errors from being reported correctly. */ #define TILEDB_INVALID_ERROR (-4) - -/** Default compression level */ -#define TILEDB_COMPRESSION_FILTER_DEFAULT_LEVEL (-30000) /**@}*/ #ifdef __cplusplus diff --git a/tiledb/sm/filter/compression_filter.h b/tiledb/sm/filter/compression_filter.h index dcfe1943da9e..4925e79610e2 100644 --- a/tiledb/sm/filter/compression_filter.h +++ b/tiledb/sm/filter/compression_filter.h @@ -162,9 +162,6 @@ class CompressionFilter : public Filter { /** The format version. */ uint32_t version_; - /** The default filter compression level. */ - static constexpr int default_level_ = -30000; - /** Mutex guarding zstd_compress_ctx_pool */ std::mutex zstd_compress_ctx_pool_mtx_; From 081bcc5f7ce4bee576f08b97de348236ac88d429 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 2 Nov 2023 12:41:50 +0100 Subject: [PATCH 048/456] Aggregates: use tile metadata. (#4468) This allows the readers to use the tile metadata to do aggregation. The tiles still get loaded into memory even when not required but that will be addressed in a follow up change. --- TYPE: IMPROVEMENT DESC: Aggregates: use tile metadata. --- format_spec/FORMAT_SPEC.md | 2 +- test/src/test-cppapi-aggregates.cc | 6 +- tiledb/sm/misc/constants.cc | 2 +- tiledb/sm/query/readers/dense_reader.cc | 96 ++++++++++--------- tiledb/sm/query/readers/dense_reader.h | 48 ++++++++++ tiledb/sm/query/readers/reader_base.cc | 38 ++++++++ tiledb/sm/query/readers/reader_base.h | 12 +++ tiledb/sm/query/readers/result_space_tile.h | 29 ++++++ .../readers/sparse_global_order_reader.cc | 52 ++++++---- .../readers/sparse_global_order_reader.h | 26 +++++ .../query/readers/sparse_index_reader_base.cc | 3 + .../sparse_unordered_with_dups_reader.cc | 61 +++++++----- .../sparse_unordered_with_dups_reader.h | 17 ++++ tiledb/sm/tile/tile_metadata_generator.cc | 2 +- 14 files changed, 298 insertions(+), 96 deletions(-) diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index ee0db16d584e..a6212a805285 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -4,7 +4,7 @@ title: Format Specification **Notes:** -* The current TileDB format version number is **20** (`uint32_t`). +* The current TileDB format version number is **21** (`uint32_t`). * Data written by TileDB and referenced in this document is **little-endian** with the following exceptions: diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 14e91e6b0a62..60fc27adc9f4 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -620,9 +620,9 @@ void CppAggregatesFx::create_array_and_write_fragments() { write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1, validity_values); write_sparse({4, 5, 6, 7}, {2, 2, 3, 3}, {2, 4, 2, 3}, 3, validity_values); write_sparse( - {8, 9, 10, 11}, {2, 1, 3, 4}, {1, 3, 1, 1}, 4, validity_values); - write_sparse( - {12, 13, 14, 15}, {4, 3, 3, 4}, {2, 3, 4, 4}, 6, validity_values); + {8, 9, 10, 11}, {2, 1, 3, 4}, {1, 3, 1, 1}, 5, validity_values); + write_sparse({12, 13}, {4, 3}, {2, 3}, 7, validity_values); + write_sparse({14, 15}, {3, 4}, {4, 4}, 9, validity_values); } } diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 3b75f7bb1d26..5c9897940c5c 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -676,7 +676,7 @@ const int32_t library_version[3] = { TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH}; /** The TileDB serialization base format version number. */ -const format_version_t base_format_version = 20; +const format_version_t base_format_version = 21; /** * The TileDB serialization format version number. diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 065ee571ea61..828bcac5bf8b 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -350,6 +350,8 @@ Status DenseReader::dense_read() { read_state_.partitioner_.subarray().relevant_fragments(), var_names); load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), names); + load_tile_metadata( + read_state_.partitioner_.subarray().relevant_fragments(), names); uint64_t t_start = 0; uint64_t t_end = 0; @@ -756,15 +758,12 @@ tuple> DenseReader::compute_result_tiles( bool done = false; while (!done && t_end < tile_coords.size()) { const DimType* tc = (DimType*)&tile_coords[t_end][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException("Tile coordinates not found"); - } + auto& result_space_tile = result_space_tiles.at(tc); // Compute the required memory to load the query condition tiles for the // current result space tile. uint64_t condition_memory = 0; - for (const auto& result_tile : it->second.result_tiles()) { + for (const auto& result_tile : result_space_tile.result_tiles()) { auto& rt = result_tile.second; for (uint64_t n = 0; n < condition_names.size(); n++) { condition_memory += @@ -785,7 +784,7 @@ tuple> DenseReader::compute_result_tiles( // current space tile. for (uint64_t n = condition_names.size(); n < names.size(); n++) { uint64_t tile_memory = 0; - for (const auto& result_tile : it->second.result_tiles()) { + for (const auto& result_tile : result_space_tile.result_tiles()) { auto& rt = result_tile.second; tile_memory += get_attribute_tile_size(names[n], rt.frag_idx(), rt.tile_idx()); @@ -808,7 +807,7 @@ tuple> DenseReader::compute_result_tiles( // Add the result tiles for this space tile to the returned list to // process. - for (const auto& result_tile : it->second.result_tiles()) { + for (const auto& result_tile : result_space_tile.result_tiles()) { result_tiles.push_back(const_cast(&result_tile.second)); } @@ -911,21 +910,17 @@ Status DenseReader::apply_query_condition( [&](uint64_t t, uint64_t range_thread_idx) { // Find out result space tile and tile subarray. const DimType* tc = (DimType*)&tile_coords[t][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException( - "Tile coordinates not found"); - } + auto& result_space_tile = result_space_tiles.at(tc); // Iterate over all coordinates, retrieved in cell slab. - const auto& frag_domains = it->second.frag_domains(); + const auto& frag_domains = result_space_tile.frag_domains(); TileCellSlabIter iter( range_thread_idx, num_range_threads, subarray, tile_subarrays[t], tile_extents, - it->second.start_coords(), + result_space_tile.start_coords(), range_info, cell_order); @@ -965,13 +960,22 @@ Status DenseReader::apply_query_condition( *(fragment_metadata_[frag_domains[i].fid()] ->array_schema() .get()), - it->second.result_tile(frag_domains[i].fid()), + result_space_tile.result_tile(frag_domains[i].fid()), start, end - start + 1, iter.pos_in_tile(), stride, iter.cell_slab_coords().data(), dest_ptr)); + + // If any cell doesn't match the query condition, signal + // it in the space tile. + for (uint64_t c = start; c <= end; c++) { + if (dest_ptr[c] == 0) { + result_space_tile.set_qc_filtered_results(); + break; + } + } } } @@ -1085,16 +1089,13 @@ Status DenseReader::copy_attribute( [&](uint64_t t, uint64_t range_thread_idx) { // Find out result space tile and tile subarray. const DimType* tc = (DimType*)&tile_coords[t][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException("Tile coordinates not found"); - } + auto& result_space_tile = result_space_tiles.at(tc); // Copy the tile offsets. return copy_offset_tiles( name, tile_extents, - it->second, + result_space_tile, subarray, tile_subarrays[t], subarray_start_cell, @@ -1148,15 +1149,12 @@ Status DenseReader::copy_attribute( [&](uint64_t t, uint64_t range_thread_idx) { // Find out result space tile and tile subarray. const DimType* tc = (DimType*)&tile_coords[t][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException("Tile coordinates not found"); - } + auto& result_space_tile = result_space_tiles.at(tc); return copy_var_tiles( name, tile_extents, - it->second, + result_space_tile, subarray, tile_subarrays[t], subarray_start_cell, @@ -1193,16 +1191,13 @@ Status DenseReader::copy_attribute( [&](uint64_t t, uint64_t range_thread_idx) { // Find out result space tile and tile subarray. const DimType* tc = (DimType*)&tile_coords[t][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException("Tile coordinates not found"); - } + auto& result_space_tile = result_space_tiles.at(tc); // Copy the tile fixed values. RETURN_NOT_OK(copy_fixed_tiles( name, tile_extents, - it->second, + result_space_tile, subarray, tile_subarrays[t], global_order ? tile_offsets[t] : 0, @@ -1298,24 +1293,33 @@ Status DenseReader::process_aggregates( [&](uint64_t t, uint64_t range_thread_idx) { // Find out result space tile and tile subarray. const DimType* tc = (DimType*)&tile_coords[t][0]; - auto it = result_space_tiles.find(tc); - if (it == result_space_tiles.end()) { - throw DenseReaderStatusException("Tile coordinates not found"); + auto& result_space_tile = result_space_tiles.at(tc); + if (can_aggregate_tile_with_frag_md( + name, result_space_tile, tile_subarrays[t])) { + if (range_thread_idx == 0) { + auto& rt = result_space_tile.single_result_tile(); + auto tile_idx = rt.tile_idx(); + auto& frag_md = fragment_metadata_[rt.frag_idx()]; + auto md = frag_md->get_tile_metadata(name, tile_idx); + auto& aggregates = aggregates_[name]; + for (auto& aggregate : aggregates) { + aggregate->aggregate_tile_with_frag_md(md); + } + } + } else { + RETURN_NOT_OK(aggregate_tiles( + name, + tile_extents, + result_space_tile, + subarray, + tile_subarrays[t], + global_order ? tile_offsets[t] : 0, + range_info, + aggregate_bitmap, + range_thread_idx, + num_range_threads)); } - // Copy the tile fixed values. - RETURN_NOT_OK(aggregate_tiles( - name, - tile_extents, - it->second, - subarray, - tile_subarrays[t], - global_order ? tile_offsets[t] : 0, - range_info, - aggregate_bitmap, - range_thread_idx, - num_range_threads)); - return Status::Ok(); }); RETURN_NOT_OK(status); diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 8cdba9f7c737..74f7624b6beb 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -267,6 +267,54 @@ class DenseReader : public ReaderBase, public IQueryStrategy { ResultTile::TileTuple* tile_tuple, optional bitmap_data); + /** + * Returns wether or not we can aggregate the tile with only the fragment + * metadata. + * + * @param name Name of the field to process. + * @param rst Result space tile. + * @param tile_subarray Tile subarray. + * @return If we can do the aggregation with the frag md or not. + */ + template + inline bool can_aggregate_tile_with_frag_md( + const std::string& name, + ResultSpaceTile& rst, + const Subarray& tile_subarray) { + // Make sure there are no filtered results by the query condition and that + // there are only one fragment domain for this tile. Having more fragment + // domains for a tile means we'll have to merge data for many sources so we + // cannot aggregate the full tile. + if (rst.qc_filtered_results() || rst.frag_domains().size() != 1) { + return false; + } + + // Now we can get the fragment metadata of the only result tile in this + // space tile. + const auto& rt = rst.single_result_tile(); + const auto frag_md = fragment_metadata_[rt.frag_idx()]; + + // Make sure this tile isn't cropped by ranges and the fragment metadata has + // tile metadata. + if (tile_subarray.cell_num() != rt.cell_num() || + frag_md->has_tile_metadata()) { + return false; + } + + // Fixed size nullable strings had incorrect min/max metadata for dense + // until version 20. + const auto type = array_schema_.type(name); + if ((type == Datatype::STRING_ASCII || type == Datatype::CHAR) && + array_schema_.cell_val_num(name) != constants::var_num && + array_schema_.is_nullable(name)) { + if (frag_md->version() <= 20) { + return false; + } + } + + return true; + } + /** Process aggregates for a given field. */ template Status process_aggregates( diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index c4d8ddf573db..a75d2ca4f2e2 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -505,6 +505,44 @@ void ReaderBase::load_tile_var_sizes( })); } +void ReaderBase::load_tile_metadata( + const RelevantFragments& relevant_fragments, + const std::vector& names) { + auto timer_se = stats_->start_timer("load_tile_metadata"); + const auto encryption_key = array_->encryption_key(); + + throw_if_not_ok(parallel_for( + storage_manager_->compute_tp(), + 0, + relevant_fragments.size(), + [&](const uint64_t i) { + auto frag_idx = relevant_fragments[i]; + auto& fragment = fragment_metadata_[frag_idx]; + + // Generate the list of name with aggregates. + const auto& schema = fragment->array_schema(); + std::vector to_load; + for (auto& n : names) { + // Not a member of array schema, this field was added in array + // schema evolution, ignore for this fragment's tile metadata. + if (!schema->is_field(n)) { + continue; + } + + if (aggregates_.count(n) != 0) { + to_load.emplace_back(n); + } + } + + fragment->load_tile_max_values(*encryption_key, to_load); + fragment->load_tile_min_values(*encryption_key, to_load); + fragment->load_tile_sum_values(*encryption_key, to_load); + fragment->load_tile_null_count_values(*encryption_key, to_load); + + return Status::Ok(); + })); +} + void ReaderBase::load_processed_conditions() { auto timer_se = stats_->start_timer("load_processed_conditions"); const auto encryption_key = array_->encryption_key(); diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index ee5a32b60ffc..109e29ac2668 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -431,6 +431,18 @@ class ReaderBase : public StrategyBase { const RelevantFragments& relevant_fragments, const std::vector& names); + /* + * Loads tile metadata for each attribute/dimension name into + * their associated element in `fragment_metadata_`. This is done for + * attributes with aggregates. + * + * @param relevant_fragments List of relevant fragments. + * @param names The attribute/dimension names. + */ + void load_tile_metadata( + const RelevantFragments& relevant_fragments, + const std::vector& names); + /** * Loads processed conditions from fragment metadata. * diff --git a/tiledb/sm/query/readers/result_space_tile.h b/tiledb/sm/query/readers/result_space_tile.h index 9d7a6125eba2..8f15945d4ecc 100644 --- a/tiledb/sm/query/readers/result_space_tile.h +++ b/tiledb/sm/query/readers/result_space_tile.h @@ -152,6 +152,32 @@ class ResultSpaceTile { result_tiles_ == rst.result_tiles_; } + /** The query condition filtered a result for this tile. */ + inline void set_qc_filtered_results() { + qc_filtered_results_ = true; + } + + /** + * Returns if the query condition filtered any results for this tile or not. + */ + inline bool qc_filtered_results() const { + return qc_filtered_results_; + } + + /** + * Returns the only result tile in this space tile or throws if there are more + * than one. + */ + inline ResultTile& single_result_tile() { + if (result_tiles_.size() != 1) { + throw std::runtime_error( + "Shouldn't call single_result_tile on tiles with more than one " + "fragment domain."); + } + + return result_tiles_[frag_domains_[0].fid()]; + } + private: /** The (global) coordinates of the first cell in the space tile. */ std::vector start_coords_; @@ -168,6 +194,9 @@ class ResultSpaceTile { * `(fragment id) -> (result tile)`. */ std::map result_tiles_; + + /** Did the query condition filter any result for this space tile. */ + bool qc_filtered_results_ = false; }; } // namespace sm diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 298ad72117fc..6c037e1a1ccc 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -2098,26 +2098,40 @@ void SparseGlobalOrderReader::process_aggregates( [&](uint64_t i, uint64_t range_thread_idx) { // For easy reference. auto& rcs = result_cell_slabs[i]; - auto rt = static_cast*>( - result_cell_slabs[i].tile_); - - // Compute parallelization parameters. - auto&& [min_pos, max_pos, dest_cell_offset, skip_aggregate] = - compute_parallelization_parameters( - range_thread_idx, - num_range_threads, - rcs.start_, - rcs.length_, - cell_offsets[i]); - if (skip_aggregate) { - return Status::Ok(); - } + if (can_aggregate_tile_with_frag_md(result_cell_slabs[i])) { + if (range_thread_idx == 0) { + auto rt = result_cell_slabs[i].tile_; + auto md = fragment_metadata_[rt->frag_idx()]->get_tile_metadata( + name, rt->tile_idx()); + for (auto& aggregate : aggregates) { + aggregate->aggregate_tile_with_frag_md(md); + } + } + } else { + // Compute parallelization parameters. + auto&& [min_pos, max_pos, dest_cell_offset, skip_aggregate] = + compute_parallelization_parameters( + range_thread_idx, + num_range_threads, + rcs.start_, + rcs.length_, + cell_offsets[i]); + if (skip_aggregate) { + return Status::Ok(); + } - // Compute aggregate. - AggregateBuffer aggregate_buffer{make_aggregate_buffer( - name, var_sized, nullable, cell_val_num, min_pos, max_pos, *rt)}; - for (auto& aggregate : aggregates) { - aggregate->aggregate_data(aggregate_buffer); + // Compute aggregate. + AggregateBuffer aggregate_buffer{make_aggregate_buffer( + name, + var_sized, + nullable, + cell_val_num, + min_pos, + max_pos, + *result_cell_slabs[i].tile_)}; + for (auto& aggregate : aggregates) { + aggregate->aggregate_data(aggregate_buffer); + } } return Status::Ok(); diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.h b/tiledb/sm/query/readers/sparse_global_order_reader.h index 0999694f6a77..1f8a7cddfdb9 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.h +++ b/tiledb/sm/query/readers/sparse_global_order_reader.h @@ -567,6 +567,32 @@ class SparseGlobalOrderReader : public SparseIndexReaderBase, const uint64_t max_cell, ResultTile& rt); + /** + * Returns wether or not we can aggregate the tile with only the fragment + * metadata. + * + * @param rcs Result cell slab. + * @return If we can do the aggregation with the frag md or not. + */ + inline bool can_aggregate_tile_with_frag_md(ResultCellSlab& rcs) { + auto rt = static_cast*>(rcs.tile_); + auto& frag_md = fragment_metadata_[rt->frag_idx()]; + + // Here we only aggregate a full tile if first of all there are no missing + // cells in the bitmap. This can be validated with 'copy_full_tile'. Second, + // we only do it when a full tile is used in the result cell slab structure + // by making sure that the cell slab starts at 0 and ends at the end of the + // tile. When we perform the merge to order everything in global order for + // this reader, we might end up not using a cell in a tile at all because it + // has a duplicate entry (with the same coordinates) written at a later + // timestamp. There is no way to know that this happened in a tile at the + // moment so the best we can do for now is to use fragment metadata only + // when a full tile was merged in the cell slab structure. Finally, we check + // the fragment metadata has indeed tile metadata. + return rt->copy_full_tile() && rcs.start_ == 0 && + rcs.length_ == rt->cell_num() && frag_md->has_tile_metadata(); + } + /** * Process aggregates. * diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 73ee17740f17..0243dcb696f7 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -508,6 +508,9 @@ void SparseIndexReaderBase::load_tile_offsets_for_fragments( // Load tile offsets and var sizes for attributes. load_tile_var_sizes(relevant_fragments, var_size_to_load_); load_tile_offsets(relevant_fragments, attr_tile_offsets_to_load_); + + // Load tile metadata. + load_tile_metadata(relevant_fragments, attr_tile_offsets_to_load_); } Status SparseIndexReaderBase::read_and_unfilter_coords( diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 75e35126bdc1..a829fe9d38fd 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -1936,32 +1936,43 @@ void SparseUnorderedWithDupsReader::process_aggregates( return Status::Ok(); } - uint64_t cell_num = - fragment_metadata_[rt->frag_idx()]->cell_num(rt->tile_idx()); - auto&& [skip_aggregate, src_min_pos, src_max_pos, dest_cell_offset] = - compute_parallelization_parameters( - range_thread_idx, - num_range_threads, - 0, - cell_num, - cell_offsets[i], - nullptr); - if (skip_aggregate) { - return Status::Ok(); - } + if (can_aggregate_tile_with_frag_md(rt)) { + if (range_thread_idx == 0) { + auto t = rt->tile_idx(); + auto md = + fragment_metadata_[rt->frag_idx()]->get_tile_metadata(name, t); + for (auto& aggregate : aggregates) { + aggregate->aggregate_tile_with_frag_md(md); + } + } + } else { + uint64_t cell_num = + fragment_metadata_[rt->frag_idx()]->cell_num(rt->tile_idx()); + auto&& [skip_aggregate, src_min_pos, src_max_pos, dest_cell_offset] = + compute_parallelization_parameters( + range_thread_idx, + num_range_threads, + 0, + cell_num, + cell_offsets[i], + nullptr); + if (skip_aggregate) { + return Status::Ok(); + } - // Compute aggregate. - AggregateBuffer aggregate_buffer{make_aggregate_buffer( - name, - var_sized, - nullable, - cell_val_num, - count_bitmap, - src_min_pos, - src_max_pos, - *rt)}; - for (auto& aggregate : aggregates) { - aggregate->aggregate_data(aggregate_buffer); + // Compute aggregate. + AggregateBuffer aggregate_buffer{make_aggregate_buffer( + name, + var_sized, + nullable, + cell_val_num, + count_bitmap, + src_min_pos, + src_max_pos, + *rt)}; + for (auto& aggregate : aggregates) { + aggregate->aggregate_data(aggregate_buffer); + } } return Status::Ok(); diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h index 27d0dee32b7f..386dbe5aebe9 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h @@ -512,6 +512,23 @@ class SparseUnorderedWithDupsReader : public SparseIndexReaderBase, const uint64_t max_cell, UnorderedWithDupsResultTile& rt); + /** + * Returns wether or not we can aggregate the tile with only the fragment + * metadata. + * + * @param rt Result tile. + * @return If we can do the aggregation with the frag md or not. + */ + inline bool can_aggregate_tile_with_frag_md( + UnorderedWithDupsResultTile* rt) { + auto& frag_md = fragment_metadata_[rt->frag_idx()]; + + // Here we only aggregate a full tile if first of all there are no missing + // cells in the bitmap. This can be validated with 'copy_full_tile'. + // Finally, we check the fragment metadata has indeed tile metadata. + return rt->copy_full_tile() && frag_md->has_tile_metadata(); + } + /** * Process aggregates. * diff --git a/tiledb/sm/tile/tile_metadata_generator.cc b/tiledb/sm/tile/tile_metadata_generator.cc index 1b0ae7aad969..9f3ca7d2b4e5 100644 --- a/tiledb/sm/tile/tile_metadata_generator.cc +++ b/tiledb/sm/tile/tile_metadata_generator.cc @@ -458,7 +458,7 @@ void TileMetadataGenerator::min_max_nullable( uint64_t start, uint64_t end) { // Get pointers to the data and cell num. - auto value = tile.data_as(); + auto value = tile.data_as() + cell_size_ * start; auto validity_values = validity_tile.data_as(); // Process cell by cell. From 7bd026278ddd7c6cc48fc518ced57146f923db15 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 2 Nov 2023 21:11:53 +0200 Subject: [PATCH 049/456] Integrate baseline object library into the main build. (#4446) [SC-15115](https://app.shortcut.com/tiledb-inc/story/15115/integrate-baseline-object-library-into-the-main-build) Supersedes #2916. --- TYPE: IMPROVEMENT DESC: Integrate baseline object library into the main build --- test/CMakeLists.txt | 1 + tiledb/CMakeLists.txt | 43 +++++++++++++++------ tiledb/api/c_api/array/test/CMakeLists.txt | 2 +- tiledb/common/dynamic_memory/CMakeLists.txt | 5 --- tiledb/common/exception/CMakeLists.txt | 6 --- tiledb/common/governor/CMakeLists.txt | 5 --- tiledb/sm/array/test/CMakeLists.txt | 2 +- tiledb/sm/subarray/test/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 2 +- 9 files changed, 36 insertions(+), 32 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f9ecbc6f7bd1..6b844e322e19 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -305,6 +305,7 @@ target_include_directories( target_link_libraries(tiledb_unit PUBLIC TILEDB_CORE_OBJECTS_ILIB + TILEDB_CORE_OBJECTS Catch2::Catch2 tiledb_test_support_lib ) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 1ce86d43f4fe..c5f6928e5296 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -47,6 +47,9 @@ cmake_policy(SET CMP0063 NEW) # * CMAKE_HP_PTHREADS_INIT - If the found thread library is the HP thread library. find_package(Threads REQUIRED) +# Compile all core sources with PIC +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + ############################################################ # Subdirectories ############################################################ @@ -140,9 +143,6 @@ endif() # List of all core source files set(TILEDB_CORE_SOURCES - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/heap_memory.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/heap_profiler.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/logger.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/memory.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/stdx_string.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/common/interval/interval.cc @@ -411,9 +411,17 @@ add_library(TILEDB_CORE_OBJECTS OBJECT ${TILEDB_EXTERNALS_SOURCES} ) -# Use BUILD_INTERFACE generator to work around -# https://gitlab.kitware.com/cmake/cmake/-/issues/20736 -target_link_libraries(TILEDB_CORE_OBJECTS PRIVATE $) +target_sources(TILEDB_CORE_OBJECTS + PUBLIC + $ +) + +target_link_libraries(TILEDB_CORE_OBJECTS + PRIVATE + common + PUBLIC + baseline +) target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) @@ -447,9 +455,6 @@ add_custom_target( COMMENT "Re-generate ${MGC_GZIPPED_BIN_INPUT_FILE} for embedded magic.mgc support" ) -# Compile all core sources with PIC -set_property(TARGET TILEDB_CORE_OBJECTS PROPERTY POSITION_INDEPENDENT_CODE ON) - target_include_directories(TILEDB_CORE_OBJECTS PRIVATE "${TILEDB_CORE_INCLUDE_DIR}" @@ -468,8 +473,14 @@ if (WIN32 AND TILEDB_STATIC) ) add_dependencies(TILEDB_CORE_OBJECTS_STATIC gen_mgc_unarch) - # Compile all core sources with PIC - set_property(TARGET TILEDB_CORE_OBJECTS_STATIC PROPERTY POSITION_INDEPENDENT_CODE ON) + target_sources(TILEDB_CORE_OBJECTS_STATIC + PUBLIC + $) + + target_link_libraries(TILEDB_CORE_OBJECTS_STATIC + PUBLIC + baseline + ) target_compile_definitions(TILEDB_CORE_OBJECTS_STATIC PRIVATE @@ -1069,6 +1080,7 @@ endif() target_link_libraries(tiledb_shared PRIVATE $ + TILEDB_CORE_OBJECTS ) if (TILEDB_STATIC) @@ -1089,7 +1101,10 @@ if (TILEDB_STATIC) ) # Create the target add_library(tiledb_static STATIC - $ + $) + target_link_libraries(tiledb_static + PRIVATE + $ ) target_compile_definitions(tiledb_static INTERFACE @@ -1103,6 +1118,10 @@ if (TILEDB_STATIC) ) else() add_library(tiledb_static STATIC $) + target_link_libraries(tiledb_static + PRIVATE + $ + ) set_target_properties(tiledb_static PROPERTIES OUTPUT_NAME "tiledb" diff --git a/tiledb/api/c_api/array/test/CMakeLists.txt b/tiledb/api/c_api/array/test/CMakeLists.txt index 9e7805c294a6..ebd617387ae8 100644 --- a/tiledb/api/c_api/array/test/CMakeLists.txt +++ b/tiledb/api/c_api/array/test/CMakeLists.txt @@ -30,7 +30,6 @@ find_package(Catch_EP REQUIRED) # #TODO: link only against a new CAPI array handle, not all core objects add_executable( unit_capi_array EXCLUDE_FROM_ALL - $ ) target_link_libraries(unit_capi_array @@ -38,6 +37,7 @@ target_link_libraries(unit_capi_array Catch2::Catch2WithMain export TILEDB_CORE_OBJECTS_ILIB + TILEDB_CORE_OBJECTS ) # Sources for tests diff --git a/tiledb/common/dynamic_memory/CMakeLists.txt b/tiledb/common/dynamic_memory/CMakeLists.txt index 8dcfb713e7eb..72dcee8f4e4e 100644 --- a/tiledb/common/dynamic_memory/CMakeLists.txt +++ b/tiledb/common/dynamic_memory/CMakeLists.txt @@ -26,11 +26,6 @@ include(common NO_POLICY_SCOPE) -list(APPEND SOURCES - dynamic_memory.cc -) -gather_sources(${SOURCES}) - list(APPEND DEPENDENT_SOURCES ../governor/governor.cc ../heap_profiler.cc diff --git a/tiledb/common/exception/CMakeLists.txt b/tiledb/common/exception/CMakeLists.txt index d0f26aa62cf1..d8f011d425e0 100644 --- a/tiledb/common/exception/CMakeLists.txt +++ b/tiledb/common/exception/CMakeLists.txt @@ -27,10 +27,4 @@ include(common NO_POLICY_SCOPE) -list(APPEND SOURCES - # These sources are both in the `baseline` object library - exception.cc status.cc -) -gather_sources(${SOURCES}) - add_test_subdirectory() diff --git a/tiledb/common/governor/CMakeLists.txt b/tiledb/common/governor/CMakeLists.txt index f769afa0aad8..5d1ceae34569 100644 --- a/tiledb/common/governor/CMakeLists.txt +++ b/tiledb/common/governor/CMakeLists.txt @@ -27,11 +27,6 @@ include(common NO_POLICY_SCOPE) -list(APPEND SOURCES - governor.cc -) -gather_sources(${SOURCES}) - if (FALSE AND TILEDB_TESTS) # reserved for later find_package(Catch_EP REQUIRED) diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 62b222de11ea..6ef9982394d6 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -33,6 +33,6 @@ conclude(unit_test) commence(unit_test consistency) this_target_sources(main.cc unit_consistency.cc) - this_target_sources($) + this_target_link_libraries(TILEDB_CORE_OBJECTS) this_target_link_libraries(TILEDB_CORE_OBJECTS_ILIB) conclude(unit_test) diff --git a/tiledb/sm/subarray/test/CMakeLists.txt b/tiledb/sm/subarray/test/CMakeLists.txt index c3d243a50d95..843a552ffb79 100644 --- a/tiledb/sm/subarray/test/CMakeLists.txt +++ b/tiledb/sm/subarray/test/CMakeLists.txt @@ -33,7 +33,7 @@ conclude(unit_test) commence(unit_test add_ranges_list) this_target_sources(main.cc unit_add_ranges_list.cc) - this_target_link_libraries($) + this_target_link_libraries(TILEDB_CORE_OBJECTS) this_target_link_libraries(tiledb_test_support_lib) target_include_directories(unit_add_ranges_list PRIVATE "$") diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3a55d9147108..218e238029e6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -35,7 +35,6 @@ add_executable(tiledb EXCLUDE_FROM_ALL src/commands/help_command.cc src/commands/info_command.cc src/main/tiledb.cc - $ ) # List of all tool targets @@ -45,6 +44,7 @@ set(TILEDB_TOOLS tiledb) foreach(TOOL ${TILEDB_TOOLS}) target_link_libraries(${TOOL} PRIVATE TILEDB_CORE_OBJECTS_ILIB + TILEDB_CORE_OBJECTS clipp::clipp ) target_include_directories(${TOOL} PRIVATE From 836f981637a04674f17a02a6418121aa4a8ed76d Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 2 Nov 2023 14:45:38 -0500 Subject: [PATCH 050/456] Remove UnitTestConfig::array_encryption_key_length (#4482) This field existed to force invalid key lengths. This is unnecessary since we can just set the invalid key lengths in the test. The reason I stopped to remove this is because the test state is improperly persisted between unit tests. If you run `tiledb_unit -d y [capi][encryption]` it will fail because the UnitTestConfig is not reset. It took me so long to debug this issue that I decided to light it on fire and just remove the whole global state and write the tests properly. --- TYPE: IMPROVEMENT DESC: Remove UnitTestConfig::array_encryption_key_length --- test/src/unit-capi-array.cc | 47 ++++++++-------- test/src/unit-capi-fragment_info.cc | 11 ---- test/src/unit-capi-metadata.cc | 9 ---- test/src/unit-cppapi-metadata.cc | 3 -- test/support/src/helpers.cc | 15 +----- test/support/src/helpers.h | 10 ---- tiledb/sm/array/array.cc | 15 ++---- tiledb/sm/crypto/encryption_key.cc | 5 +- tiledb/sm/global_state/unit_test_config.h | 3 -- tiledb/sm/group/group.cc | 10 +--- tiledb/sm/storage_manager/storage_manager.cc | 56 +++----------------- 11 files changed, 44 insertions(+), 140 deletions(-) diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 83d23382691b..6d351eeb2cb4 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -315,7 +315,6 @@ void ArrayFx::create_dense_vector(const std::string& path) { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.reset(); REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_, cfg).ok()); tiledb_config_free(&cfg); } @@ -531,7 +530,8 @@ TEST_CASE_METHOD( SECTION("- API calls with encrypted schema") { const char key[] = "0123456789abcdeF0123456789abcdeF"; - uint32_t key_len = (uint32_t)strlen(key); + const char bad_key_len[] = "bad_key_len"; + const char wrong_key[] = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"; // Check error with invalid key length tiledb_config_t* cfg; @@ -541,10 +541,9 @@ TEST_CASE_METHOD( REQUIRE(err == nullptr); rc = tiledb_config_set(cfg, "sm.encryption_type", "AES_256_GCM", &err); REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", bad_key_len, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set(31); tiledb_ctx_t* ctx_invalid_key_len_1; tiledb_vfs_t* vfs_invalid_key_len_1; REQUIRE(vfs_test_init( @@ -559,8 +558,6 @@ TEST_CASE_METHOD( rc = tiledb_config_set( cfg, "sm.encryption_type", "TILEDB_NO_ENCRYPTION", &err); REQUIRE(err == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len); tiledb_ctx_t* ctx_invalid_key_len_2; tiledb_vfs_t* vfs_invalid_key_len_2; REQUIRE(vfs_test_init( @@ -577,6 +574,8 @@ TEST_CASE_METHOD( // Create array with proper key rc = tiledb_config_set(cfg, "sm.encryption_type", "AES_256_GCM", &err); REQUIRE(err == nullptr); + rc = tiledb_config_set(cfg, "sm.encryption_key", key, &err); + REQUIRE(err == nullptr); tiledb_ctx_t* ctx_proper_key; tiledb_vfs_t* vfs_proper_key; REQUIRE(vfs_test_init(fs_vec_, &ctx_proper_key, &vfs_proper_key, cfg).ok()); @@ -626,12 +625,11 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); REQUIRE(is_open == 0); - // Check error with bad key - char bad_key[32]; + // Check error with wrong key rc = tiledb_config_set(cfg, "sm.encryption_type", "AES_256_GCM", &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", bad_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", wrong_key, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); @@ -644,12 +642,11 @@ TEST_CASE_METHOD( // Check error with bad key length REQUIRE( - tiledb_config_set(cfg, "sm.encryption_key", key, &err) == TILEDB_OK); + tiledb_config_set(cfg, "sm.encryption_key", bad_key_len, &err) == + TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); REQUIRE(rc == TILEDB_OK); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len - 1); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_ERR); rc = tiledb_array_is_open(ctx_, array, &is_open); @@ -657,8 +654,10 @@ TEST_CASE_METHOD( REQUIRE(is_open == 0); // Use correct key - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len); + REQUIRE( + tiledb_config_set(cfg, "sm.encryption_key", key, &err) == TILEDB_OK); + REQUIRE(err == nullptr); + rc = tiledb_array_set_config(ctx_, array, cfg); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_is_open(ctx_, array, &is_open); @@ -669,7 +668,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); REQUIRE(rc == TILEDB_OK); - rc = tiledb_config_set(cfg, "sm.encryption_key", bad_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", key, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); @@ -683,6 +682,10 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_ERR); // Opening an array with a bad key should fail + REQUIRE( + tiledb_config_set(cfg, "sm.encryption_key", wrong_key, &err) == + TILEDB_OK); + REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array2, cfg); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array2, TILEDB_READ); @@ -702,7 +705,7 @@ TEST_CASE_METHOD( // Check with bad key rc = tiledb_config_set(cfg, "sm.encryption_type", "AES_256_GCM", &err); REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", bad_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", wrong_key, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); tiledb_ctx_t* ctx_bad_key; @@ -730,12 +733,17 @@ TEST_CASE_METHOD( tiledb_vfs_free(&vfs_correct_key); // Check opening after closing still requires a key. + rc = tiledb_config_set(cfg, "sm.encryption_key", "", &err); + REQUIRE(rc == TILEDB_OK); + REQUIRE(err == nullptr); + rc = tiledb_array_set_config(ctx_, array, cfg); + REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_ERR); rc = tiledb_array_is_open(ctx_, array, &is_open); REQUIRE(rc == TILEDB_OK); REQUIRE(is_open == 0); - rc = tiledb_config_set(cfg, "sm.encryption_key", bad_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", wrong_key, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); @@ -814,7 +822,6 @@ TEST_CASE_METHOD( rc = tiledb_config_set(cfg, "sm.encryption_key", key, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.reset(); rc = tiledb_array_set_config(ctx_, array, cfg); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); @@ -833,7 +840,6 @@ TEST_CASE_METHOD( REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); REQUIRE(rc == TILEDB_OK); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set(0); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_is_open(ctx_, array, &is_open); @@ -939,9 +945,6 @@ TEST_CASE_METHOD( rc = tiledb_array_set_config(ctx_, array, cfg); CHECK(rc == TILEDB_OK); tiledb_config_free(&cfg); - uint32_t key_len = (uint32_t)strlen(encryption_key_); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len); } rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index 79821d576840..f817f2cce1f7 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -196,17 +196,14 @@ TEST_CASE( encrypt = GENERATE(false, true); tiledb_encryption_type_t encryption_type; const char* key; - int key_length; uint64_t expected_fragment_size; if (encrypt) { encryption_type = tiledb_encryption_type_t::TILEDB_AES_256_GCM; key = "12345678901234567890123456789012"; - key_length = 32; expected_fragment_size = 5585; } else { encryption_type = tiledb_encryption_type_t::TILEDB_NO_ENCRYPTION; key = ""; - key_length = 0; expected_fragment_size = 3202; } @@ -218,7 +215,6 @@ TEST_CASE( array_name, encryption_type, key, - key_length, TILEDB_DENSE, {"d"}, {TILEDB_UINT64}, @@ -323,7 +319,6 @@ TEST_CASE( array_name, encryption_type, key, - key_length, 1, subarray, TILEDB_ROW_MAJOR, @@ -380,7 +375,6 @@ TEST_CASE( array_name, encryption_type, key, - key_length, 2, subarray, TILEDB_ROW_MAJOR, @@ -398,7 +392,6 @@ TEST_CASE( array_name, encryption_type, key, - key_length, 3, subarray, TILEDB_ROW_MAJOR, @@ -593,7 +586,6 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { array_name, TILEDB_AES_256_GCM, key, - 32, TILEDB_SPARSE, {"d1", "d2"}, {TILEDB_UINT64, TILEDB_UINT64}, @@ -635,7 +627,6 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { array_name, TILEDB_AES_256_GCM, key, - 32, 1, TILEDB_UNORDERED, buffers, @@ -656,7 +647,6 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { array_name, TILEDB_AES_256_GCM, key, - 32, 2, TILEDB_UNORDERED, buffers, @@ -677,7 +667,6 @@ TEST_CASE("C API: Test MBR fragment info", "[capi][fragment_info][mbr]") { array_name, TILEDB_AES_256_GCM, key, - 32, 3, TILEDB_UNORDERED, buffers, diff --git a/test/src/unit-capi-metadata.cc b/test/src/unit-capi-metadata.cc index 5a5ec56771eb..a69e4656616f 100644 --- a/test/src/unit-capi-metadata.cc +++ b/test/src/unit-capi-metadata.cc @@ -65,8 +65,6 @@ struct CMetadataFx { const char* ARRAY_NAME = "test_metadata"; tiledb_array_t* array_ = nullptr; const char* key_ = "0123456789abcdeF0123456789abcdeF"; - const uint32_t key_len_ = - (uint32_t)strlen("0123456789abcdeF0123456789abcdeF"); const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; void create_default_array_1d(); @@ -149,7 +147,6 @@ void CMetadataFx::create_default_array_1d_with_key() { array_name_, enc_type_, key_, - key_len_, TILEDB_DENSE, {"d"}, {TILEDB_UINT64}, @@ -230,8 +227,6 @@ TEST_CASE_METHOD( rc = tiledb_config_set(config, "sm.encryption_key", key_, &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len_); rc = tiledb_array_set_config(ctx_, array, config); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); @@ -1030,8 +1025,6 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); rc = tiledb_array_set_config(ctx_, array, config); REQUIRE(rc == TILEDB_OK); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len_); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); @@ -1163,8 +1156,6 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); rc = tiledb_array_set_config(ctx_, array, config); REQUIRE(rc == TILEDB_OK); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len_); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); diff --git a/test/src/unit-cppapi-metadata.cc b/test/src/unit-cppapi-metadata.cc index f7be8a520728..23e095b63db4 100644 --- a/test/src/unit-cppapi-metadata.cc +++ b/test/src/unit-cppapi-metadata.cc @@ -66,8 +66,6 @@ struct CPPMetadataFx { const char* ARRAY_NAME = "test_metadata"; tiledb_array_t* array_ = nullptr; const char* key_ = "0123456789abcdeF0123456789abcdeF"; - const uint32_t key_len_ = - (uint32_t)strlen("0123456789abcdeF0123456789abcdeF"); const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; void create_default_array_1d(); @@ -135,7 +133,6 @@ void CPPMetadataFx::create_default_array_1d_with_key() { array_name_, enc_type_, key_, - key_len_, TILEDB_DENSE, {"d"}, {TILEDB_UINT64}, diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 44611033b3b9..5d18264dbc6d 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -558,7 +558,6 @@ void create_array( const std::string& array_name, tiledb_encryption_type_t enc_type, const char* key, - uint32_t key_len, tiledb_array_type_t array_type, const std::vector& dim_names, const std::vector& dim_types, @@ -651,8 +650,6 @@ void create_array( rc = tiledb_config_set(config, "sm.encryption_key", key, &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len); tiledb_ctx_t* ctx_array; REQUIRE(tiledb_ctx_alloc(config, &ctx_array) == TILEDB_OK); rc = tiledb_array_create(ctx_array, array_name.c_str(), array_schema); @@ -1079,7 +1076,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, tiledb_layout_t layout, const QueryBuffers& buffers) { @@ -1088,7 +1084,6 @@ void write_array( array_name, encryption_type, key, - key_len, timestamp, nullptr, layout, @@ -1122,7 +1117,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, const void* subarray, tiledb_layout_t layout, @@ -1133,7 +1127,6 @@ void write_array( array_name, encryption_type, key, - key_len, timestamp, subarray, layout, @@ -1157,7 +1150,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, tiledb_layout_t layout, const QueryBuffers& buffers, @@ -1167,7 +1159,6 @@ void write_array( array_name, encryption_type, key, - key_len, timestamp, nullptr, layout, @@ -1187,8 +1178,7 @@ void write_array( ctx, array_name, TILEDB_NO_ENCRYPTION, - nullptr, - 0, + "", timestamp, subarray, layout, @@ -1201,7 +1191,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, const void* sub, tiledb_layout_t layout, @@ -1232,8 +1221,6 @@ void write_array( REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx, array, cfg); REQUIRE(rc == TILEDB_OK); - tiledb::sm::UnitTestConfig::instance().array_encryption_key_length.set( - key_len); } rc = tiledb_array_open(ctx, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 9dc90cdcd251..6f4b40af17f7 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -307,7 +307,6 @@ void create_array( * @param array_name The array name. * @param enc_type The encryption type. * @param key The key to encrypt the array with. - * @param key_len The key length. * @param array_type The array type (dense or sparse). * @param dim_names The names of dimensions. * @param dim_types The types of dimensions. @@ -327,7 +326,6 @@ void create_array( const std::string& array_name, tiledb_encryption_type_t enc_type, const char* key, - uint32_t key_len, tiledb_array_type_t array_type, const std::vector& dim_names, const std::vector& dim_types, @@ -577,7 +575,6 @@ void write_array( * @param array_name The array name. * @param encyrption_type The type of encryption. * @param key The encryption key. - * @param key_len The encryption key length. * @param timestamp The timestamp to write at. * @param layout The layout to write into. * @param buffers The attribute/dimension buffers to be written. @@ -587,7 +584,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, tiledb_layout_t layout, const QueryBuffers& buffers); @@ -649,7 +645,6 @@ void write_array( * @param array_name The array name. * @param encyrption_type The type of encryption. * @param key The encryption key. - * @param key_len The encryption key length. * @param timestamp The timestamp to write at. * @param subarray The subarray to write into. * @param layout The layout to write into. @@ -660,7 +655,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, const void* subarray, tiledb_layout_t layout, @@ -691,7 +685,6 @@ void write_array( * @param array_name The array name. * @param encyrption_type The type of encryption. * @param key The encryption key. - * @param key_len The encryption key length. * @param timestamp The timestamp to write at. * @param layout The layout to write into. * @param buffers The attribute/dimension buffers to be written. @@ -702,7 +695,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, tiledb_layout_t layout, const QueryBuffers& buffers, @@ -737,7 +729,6 @@ void write_array( * @param array_name The array name. * @param encyrption_type The type of encryption. * @param key The encryption key. - * @param key_len The encryption key length. * @param timestamp The timestamp to write at. * @param subarray The subarray to write into. * @param layout The layout to write into. @@ -749,7 +740,6 @@ void write_array( const std::string& array_name, tiledb_encryption_type_t encryption_type, const char* key, - uint64_t key_len, uint64_t timestamp, const void* subarray, tiledb_layout_t layout, diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 1b6f22eaa005..1a140954676c 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -212,9 +212,10 @@ Status Array::open_without_fragments( std::tie(array_schema_latest_, array_schemas_all_) = open_for_reads_without_fragments(); } - } catch (std::exception& e) { + } catch (...) { set_array_closed(); - throw Status_ArrayError(e.what()); + std::throw_with_nested( + ArrayException("Error opening array without fragments.")); } is_opening_or_closing_ = false; @@ -307,6 +308,7 @@ Status Array::open( if (!encryption_key_from_cfg.empty()) { encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); std::string encryption_type_from_cfg; bool found = false; encryption_type_from_cfg = config_.get("sm.encryption_type", &found); @@ -317,16 +319,9 @@ Status Array::open( } encryption_type = et.value(); - if (EncryptionKey::is_valid_key_length( + if (!EncryptionKey::is_valid_key_length( encryption_type, static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } else { encryption_key = nullptr; key_length = 0; } diff --git a/tiledb/sm/crypto/encryption_key.cc b/tiledb/sm/crypto/encryption_key.cc index d41bce646451..888a1cac6fde 100644 --- a/tiledb/sm/crypto/encryption_key.cc +++ b/tiledb/sm/crypto/encryption_key.cc @@ -80,9 +80,10 @@ Status EncryptionKey::set_key( EncryptionType encryption_type, const void* key_bytes, uint32_t key_length) { - if (!is_valid_key_length(encryption_type, key_length)) - return LOG_STATUS(Status_EncryptionError( + if (!is_valid_key_length(encryption_type, key_length)) { + throw StatusException(Status_EncryptionError( "Cannot create key; invalid key length for encryption type.")); + } encryption_type_ = encryption_type; key_length_ = key_length; diff --git a/tiledb/sm/global_state/unit_test_config.h b/tiledb/sm/global_state/unit_test_config.h index f1623bd1145a..b28ce2eb19d3 100644 --- a/tiledb/sm/global_state/unit_test_config.h +++ b/tiledb/sm/global_state/unit_test_config.h @@ -126,9 +126,6 @@ class UnitTestConfig { /** For every nth multipart upload request, return a non-OK status. */ Attribute s3_fail_every_nth_upload_request; - /** Used when setting a non-standard key_length. */ - Attribute array_encryption_key_length; - private: /** Constructor. */ UnitTestConfig() = default; diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 425d3f66187c..5f16cb1b9c29 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -110,6 +110,7 @@ Status Group::open( if (!encryption_key_from_cfg.empty()) { encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); std::string encryption_type_from_cfg; encryption_type_from_cfg = config_.get("sm.encryption_type", &found); assert(found); @@ -117,16 +118,9 @@ Status Group::open( RETURN_NOT_OK(st); encryption_type = et.value(); - if (EncryptionKey::is_valid_key_length( + if (!EncryptionKey::is_valid_key_length( encryption_type, static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } else { encryption_key = nullptr; key_length = 0; } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 772b2ce30e59..775f066d518d 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -205,6 +205,7 @@ Status StorageManagerCanonical::array_consolidate( if (!encryption_key_from_cfg.empty()) { encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); std::string encryption_type_from_cfg; bool found = false; encryption_type_from_cfg = config.get("sm.encryption_type", &found); @@ -213,16 +214,9 @@ Status StorageManagerCanonical::array_consolidate( RETURN_NOT_OK(st); encryption_type = et.value(); - if (EncryptionKey::is_valid_key_length( + if (!EncryptionKey::is_valid_key_length( encryption_type, static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } else { encryption_key = nullptr; key_length = 0; } @@ -268,6 +262,7 @@ Status StorageManagerCanonical::fragments_consolidate( if (!encryption_key_from_cfg.empty()) { encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); std::string encryption_type_from_cfg; bool found = false; encryption_type_from_cfg = config.get("sm.encryption_type", &found); @@ -276,16 +271,9 @@ Status StorageManagerCanonical::fragments_consolidate( RETURN_NOT_OK(st); encryption_type = et.value(); - if (EncryptionKey::is_valid_key_length( + if (!EncryptionKey::is_valid_key_length( encryption_type, static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } else { encryption_key = nullptr; key_length = 0; } @@ -504,6 +492,7 @@ Status StorageManagerCanonical::array_metadata_consolidate( if (!encryption_key_from_cfg.empty()) { encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); std::string encryption_type_from_cfg; bool found = false; encryption_type_from_cfg = config.get("sm.encryption_type", &found); @@ -512,16 +501,9 @@ Status StorageManagerCanonical::array_metadata_consolidate( RETURN_NOT_OK(st); encryption_type = et.value(); - if (EncryptionKey::is_valid_key_length( + if (!EncryptionKey::is_valid_key_length( encryption_type, static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } else { encryption_key = nullptr; key_length = 0; } @@ -612,21 +594,10 @@ Status StorageManagerCanonical::array_create( RETURN_NOT_OK( encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); } else { - uint32_t key_length = 0; - if (EncryptionKey::is_valid_key_length( - encryption_type_cfg, - static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } RETURN_NOT_OK(encryption_key_cfg.set_key( encryption_type_cfg, (const void*)encryption_key_from_cfg.c_str(), - key_length)); + static_cast(encryption_key_from_cfg.size()))); } st = store_array_schema(array_schema, encryption_key_cfg); } else { @@ -740,21 +711,10 @@ Status StorageManagerCanonical::array_upgrade_version( if (encryption_key_from_cfg.empty()) { RETURN_NOT_OK(encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); } else { - uint32_t key_length = 0; - if (EncryptionKey::is_valid_key_length( - encryption_type_cfg, - static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } RETURN_NOT_OK(encryption_key_cfg.set_key( encryption_type_cfg, (const void*)encryption_key_from_cfg.c_str(), - key_length)); + static_cast(encryption_key_from_cfg.size()))); } auto&& array_schema = array_dir.load_array_schema_latest(encryption_key_cfg); From 6134a69e60a66ae706311594d3c83a96a799d991 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 2 Nov 2023 15:26:34 -0500 Subject: [PATCH 051/456] Remove use of Status by Tile classes (#4483) This is a standard mechanical removal of the Status class from the headers in `tiledb/sm/tile/*.h` and then fixing every compiler error that comes up. --- TYPE: NO_HISTORY DESC: Remove use of Status by Tile classes --- .../test-capi-dimension-label-encrypted.cc | 4 +- test/src/unit-DenseTiler.cc | 2 +- test/src/unit-filter-pipeline.cc | 270 +++++++++--------- test/src/unit-tile-metadata-generator.cc | 6 +- .../fragment_meta_consolidator.cc | 2 +- tiledb/sm/fragment/fragment_metadata.cc | 13 +- tiledb/sm/query/legacy/reader.cc | 6 +- tiledb/sm/query/readers/reader_base.cc | 2 +- tiledb/sm/query/readers/result_tile.cc | 21 +- tiledb/sm/query/test/unit_query_condition.cc | 95 +++--- tiledb/sm/query/writers/dense_tiler.cc | 12 +- .../sm/query/writers/global_order_writer.cc | 120 ++++---- tiledb/sm/query/writers/unordered_writer.cc | 38 ++- tiledb/sm/storage_manager/storage_manager.cc | 9 +- tiledb/sm/tile/generic_tile_io.cc | 81 +++--- tiledb/sm/tile/generic_tile_io.h | 10 +- tiledb/sm/tile/test/unit_tile.cc | 33 ++- tiledb/sm/tile/tile.cc | 36 +-- tiledb/sm/tile/tile.h | 9 +- 19 files changed, 362 insertions(+), 407 deletions(-) diff --git a/test/src/test-capi-dimension-label-encrypted.cc b/test/src/test-capi-dimension-label-encrypted.cc index 13057bf23b1c..22233a9ace9d 100644 --- a/test/src/test-capi-dimension-label-encrypted.cc +++ b/test/src/test-capi-dimension-label-encrypted.cc @@ -101,7 +101,7 @@ TEST_CASE_METHOD( tiledb_array_schema_t* loaded_array_schema{nullptr}; check_tiledb_error_with( tiledb_array_schema_load(ctx, array_name.c_str(), &loaded_array_schema), - "[TileDB::TileIO] Error: Error reading generic tile; tile is encrypted " + "GenericTileIO: Error reading generic tile; tile is encrypted " "with AES_256_GCM but given key is for NO_ENCRYPTION"); // Check the array schema can be loaded with the encryption key. @@ -123,7 +123,7 @@ TEST_CASE_METHOD( tiledb_array_schema_t* loaded_label_array_schema{nullptr}; require_tiledb_error_with( tiledb_array_schema_load(ctx, dim_label_uri, &loaded_label_array_schema), - "[TileDB::TileIO] Error: Error reading generic tile; tile is encrypted " + "GenericTileIO: Error reading generic tile; tile is encrypted " "with AES_256_GCM but given key is for NO_ENCRYPTION"); // Check the dimension label can be opened with the encryption key. diff --git a/test/src/unit-DenseTiler.cc b/test/src/unit-DenseTiler.cc index 011f0ccd8deb..785e07e8909f 100644 --- a/test/src/unit-DenseTiler.cc +++ b/test/src/unit-DenseTiler.cc @@ -174,7 +174,7 @@ template bool DenseTilerFx::check_tile(WriterTile& tile, const std::vector& data) { std::vector tile_data(data.size()); CHECK(tile.size_as() == data.size()); - CHECK(tile.read(&tile_data[0], 0, data.size() * sizeof(T)).ok()); + CHECK_NOTHROW(tile.read(&tile_data[0], 0, data.size() * sizeof(T))); CHECK(tile_data == data); return tile_data == data; } diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index 04ba7d7dc412..ed43e62fe13f 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -75,7 +75,7 @@ WriterTile make_increasing_tile(const uint64_t nelts) { WriterTile tile( constants::format_version, Datatype::UINT64, cell_size, tile_size); for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); } return tile; @@ -93,12 +93,10 @@ WriterTile make_offsets_tile(std::vector& offsets) { // Set up test data for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK(offsets_tile - .write( - &offsets[i], - i * constants::cell_var_offset_size, - constants::cell_var_offset_size) - .ok()); + CHECK_NOTHROW(offsets_tile.write( + &offsets[i], + i * constants::cell_var_offset_size, + constants::cell_var_offset_size)); } return offsets_tile; @@ -652,8 +650,8 @@ TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -741,8 +739,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } @@ -798,8 +796,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -845,8 +843,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -938,8 +936,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -990,8 +988,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1050,8 +1048,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1097,8 +1095,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1190,8 +1188,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1242,8 +1240,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1301,8 +1299,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1394,8 +1392,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } @@ -1441,8 +1439,8 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1464,8 +1462,8 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1489,8 +1487,8 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1569,8 +1567,8 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1595,8 +1593,8 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1623,8 +1621,8 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1689,8 +1687,8 @@ TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1754,8 +1752,8 @@ TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1856,8 +1854,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1922,8 +1920,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -1987,8 +1985,8 @@ TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2088,8 +2086,8 @@ TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } @@ -2166,8 +2164,8 @@ TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2253,8 +2251,8 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { for (uint64_t n = 0; n < nelts; n++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == n); } } @@ -2284,8 +2282,8 @@ TEST_CASE( for (uint64_t n = 0; n < nelts; n++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == n); } @@ -2305,8 +2303,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile2, sha_256_pipeline); for (uint64_t n = 0; n < nelts; n++) { uint64_t elt = 0; - CHECK(unfiltered_tile2.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == n); } } @@ -2359,8 +2357,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2383,8 +2381,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2406,7 +2404,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = (uint64_t)rng(gen); - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } CHECK( @@ -2418,8 +2416,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK((int64_t)elt == rng(gen_copy)); } } @@ -2442,7 +2440,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint32_t val = (uint32_t)rng(gen); - CHECK(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } CHECK( @@ -2454,8 +2452,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { int32_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); CHECK(elt == rng(gen_copy)); } } @@ -2470,7 +2468,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = i % 257; - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } CHECK( @@ -2482,8 +2480,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i % 257); } } @@ -2594,8 +2592,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2620,8 +2618,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2645,7 +2643,7 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = (uint64_t)rng(gen); - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) @@ -2657,8 +2655,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK((int64_t)elt == rng(gen_copy)); } } @@ -2682,7 +2680,7 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint32_t val = (uint32_t)rng(gen); - CHECK(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } std::vector offsets32(offsets); @@ -2698,12 +2696,10 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK(offsets_tile32 - .write( - &offsets32[i], - i * constants::cell_var_offset_size, - constants::cell_var_offset_size) - .ok()); + CHECK_NOTHROW(offsets_tile32.write( + &offsets32[i], + i * constants::cell_var_offset_size, + constants::cell_var_offset_size)); } CHECK( @@ -2716,8 +2712,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { int32_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); CHECK(elt == rng(gen_copy)); } } @@ -2733,7 +2729,7 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = i % 257; - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } auto offsets_tile = make_offsets_tile(offsets); @@ -2747,8 +2743,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i % 257); } } @@ -2803,8 +2799,8 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2826,8 +2822,8 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2837,7 +2833,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { auto tile = make_increasing_tile(nelts); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } CHECK( @@ -2948,8 +2944,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2975,8 +2971,8 @@ TEST_CASE( run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -2989,7 +2985,7 @@ TEST_CASE( WriterTile::set_max_tile_chunk_size(80); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; - CHECK(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } CHECK( @@ -3021,8 +3017,8 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -3039,7 +3035,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } CHECK( @@ -3051,8 +3047,8 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; - CHECK(unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); CHECK(elt == i); } } @@ -3110,8 +3106,8 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -3129,7 +3125,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } CHECK( @@ -3142,8 +3138,8 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; - CHECK(unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); CHECK(elt == i); } } @@ -3172,8 +3168,8 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -3190,7 +3186,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } CHECK( @@ -3202,8 +3198,8 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; - CHECK(unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); CHECK(elt == i); } } @@ -3261,8 +3257,8 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -3280,7 +3276,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } CHECK( @@ -3293,8 +3289,8 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; - CHECK(unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); CHECK(elt == i); } } @@ -3335,8 +3331,8 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } @@ -3358,8 +3354,8 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t)) - .ok()); + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } @@ -3403,7 +3399,8 @@ void testing_float_scaling_filter() { for (uint64_t i = 0; i < nelts; i++) { FloatingType f = dis(gen); - CHECK(tile.write(&f, i * sizeof(FloatingType), sizeof(FloatingType)).ok()); + CHECK_NOTHROW( + tile.write(&f, i * sizeof(FloatingType), sizeof(FloatingType))); IntType val = static_cast(round( (f - static_cast(foffset)) / @@ -3437,9 +3434,8 @@ void testing_float_scaling_filter() { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { FloatingType elt = 0.0f; - CHECK(unfiltered_tile - .read(&elt, i * sizeof(FloatingType), sizeof(FloatingType)) - .ok()); + CHECK_NOTHROW(unfiltered_tile.read( + &elt, i * sizeof(FloatingType), sizeof(FloatingType))); CHECK(elt == float_result_vec[i]); } } @@ -3481,7 +3477,7 @@ void testing_xor_filter(Datatype t) { for (uint64_t i = 0; i < nelts; i++) { T val = static_cast(dis(gen)); - CHECK(tile.write(&val, i * sizeof(T), sizeof(T)).ok()); + CHECK_NOTHROW(tile.write(&val, i * sizeof(T), sizeof(T))); results.push_back(val); } @@ -3499,7 +3495,7 @@ void testing_xor_filter(Datatype t) { run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { T elt = 0; - CHECK(unfiltered_tile.read(&elt, i * sizeof(T), sizeof(T)).ok()); + CHECK_NOTHROW(unfiltered_tile.read(&elt, i * sizeof(T), sizeof(T))); CHECK(elt == results[i]); } } @@ -3598,7 +3594,7 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { sizeof(float), sizeof(float) * data.size()); for (size_t i = 0; i < data.size(); i++) { - CHECK(tile.write(&data[i], i * sizeof(float), sizeof(float)).ok()); + CHECK_NOTHROW(tile.write(&data[i], i * sizeof(float), sizeof(float))); } ThreadPool tp(4); @@ -3625,7 +3621,7 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; for (size_t i = 0; i < data.size(); i++) { float val = 0; - CHECK(unfiltered_tile.read(&val, i * sizeof(float), sizeof(float)).ok()); + CHECK_NOTHROW(unfiltered_tile.read(&val, i * sizeof(float), sizeof(float))); if (pipeline.has_filter(tiledb::sm::FilterType::FILTER_SCALE_FLOAT)) { // Loss of precision from rounding in FloatScale filter. CHECK(val == results[i]); diff --git a/test/src/unit-tile-metadata-generator.cc b/test/src/unit-tile-metadata-generator.cc index f03eb692ff90..cd3a1b07c2e3 100644 --- a/test/src/unit-tile-metadata-generator.cc +++ b/test/src/unit-tile-metadata-generator.cc @@ -387,8 +387,8 @@ TEST_CASE( *offsets_tile_buff = offset; auto& val = strings[values[i]]; - CHECK( - writer_tile.var_tile().write_var(val.c_str(), offset, val.size()).ok()); + CHECK_NOTHROW( + writer_tile.var_tile().write_var(val.c_str(), offset, val.size())); offset += val.size(); offsets_tile_buff++; @@ -446,7 +446,7 @@ TEST_CASE( // Initialize var tile. std::string data = "12312"; - CHECK(writer_tile.var_tile().write_var(data.c_str(), 0, 5).ok()); + CHECK_NOTHROW(writer_tile.var_tile().write_var(data.c_str(), 0, 5)); writer_tile.var_tile().set_size(5); // Call the tile metadata generator. diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 8fe893842e4e..1ce41e41f342 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -173,7 +173,7 @@ Status FragmentMetaConsolidator::consolidate( GenericTileIO tile_io(storage_manager_->resources(), uri); [[maybe_unused]] uint64_t nbytes = 0; - RETURN_NOT_OK(tile_io.write_generic(&tile, enc_key, &nbytes)); + tile_io.write_generic(&tile, enc_key, &nbytes); RETURN_NOT_OK(storage_manager_->vfs()->close_file(uri)); return Status::Ok(); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index bb81f84c7595..18b62a8691c2 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -3635,10 +3635,7 @@ void FragmentMetadata::load_v1_v2( std::string(constants::fragment_metadata_filename)); // Read metadata GenericTileIO tile_io(*resources_, fragment_metadata_uri); - auto&& [st, tile_opt] = - tile_io.read_generic(0, encryption_key, resources_->config()); - throw_if_not_ok(st); - auto& tile = *tile_opt; + auto tile = tile_io.read_generic(0, encryption_key, resources_->config()); resources_->stats().add_counter("read_frag_meta_size", tile.size()); @@ -3978,11 +3975,7 @@ Tile FragmentMetadata::read_generic_tile_from_file( // Read metadata GenericTileIO tile_io(*resources_, fragment_metadata_uri); - auto&& [st, tile_opt] = - tile_io.read_generic(offset, encryption_key, resources_->config()); - throw_if_not_ok(st); - - return std::move(*tile_opt); + return tile_io.read_generic(offset, encryption_key, resources_->config()); } void FragmentMetadata::read_file_footer( @@ -4025,7 +4018,7 @@ void FragmentMetadata::write_generic_tile_to_file( std::string(constants::fragment_metadata_filename)); GenericTileIO tile_io(*resources_, fragment_metadata_uri); - throw_if_not_ok(tile_io.write_generic(&tile, encryption_key, nbytes)); + tile_io.write_generic(&tile, encryption_key, nbytes); } void FragmentMetadata::write_footer_to_file(WriterTile& tile) const { diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 9cbafc695951..2022fa448341 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -1360,11 +1360,11 @@ Status Reader::copy_partitioned_var_cells( const uint64_t tile_var_offset = tile_offsets[cell_idx] - tile_offsets[0]; - RETURN_NOT_OK(tile_var->read(var_dest, tile_var_offset, cell_var_size)); + tile_var->read(var_dest, tile_var_offset, cell_var_size); if (nullable) - RETURN_NOT_OK(tile_validity->read( - validity_dest, cell_idx, constants::cell_validity_size)); + tile_validity->read( + validity_dest, cell_idx, constants::cell_validity_size); } } diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index a75d2ca4f2e2..a187e80063e6 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -772,7 +772,7 @@ Status ReaderBase::zip_tile_coordinates( array_schema_.filters(name).get_filter() != nullptr; auto version = tile->format_version(); if (version > 1 || using_compression) { - RETURN_NOT_OK(tile->zip_coordinates()); + tile->zip_coordinates(); } } return Status::Ok(); diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index b941389b9488..7848d8a9da85 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -282,14 +282,11 @@ std::string_view ResultTile::coord_string( const auto& coord_tile_val = coord_tiles_[dim_idx].second->var_tile(); offsets_t offset = 0; - Status st = - coord_tile_off.read(&offset, pos * sizeof(uint64_t), sizeof(uint64_t)); - assert(st.ok()); + coord_tile_off.read(&offset, pos * sizeof(uint64_t), sizeof(uint64_t)); offsets_t next_offset = 0; - st = coord_tile_off.read( + coord_tile_off.read( &next_offset, (pos + 1) * sizeof(uint64_t), sizeof(uint64_t)); - assert(st.ok()); auto size = next_offset - offset; @@ -389,7 +386,8 @@ Status ResultTile::read( auto cell_size = tile.cell_size(); auto nbytes = len * cell_size; auto offset = pos * cell_size; - return tile.read(buffer, offset, nbytes); + tile.read(buffer, offset, nbytes); + return Status::Ok(); } else if ( name == constants::coords && !coord_tiles_[0].first.empty() && !coords_tile_.has_value()) { @@ -404,8 +402,7 @@ Status ResultTile::read( auto& coord_tile = coord_tiles_[d].second->fixed_tile(); auto cell_size = coord_tile.cell_size(); auto tile_offset = (pos + c) * cell_size; - RETURN_NOT_OK( - coord_tile.read(buff + buff_offset, tile_offset, cell_size)); + coord_tile.read(buff + buff_offset, tile_offset, cell_size); buff_offset += cell_size; } } @@ -433,8 +430,7 @@ Status ResultTile::read( auto dim_size = cell_size / domain_->dim_num(); uint64_t offset = pos * cell_size + dim_size * dim_offset; for (uint64_t c = 0; c < len; ++c) { - RETURN_NOT_OK(coords_tile_->fixed_tile().read( - buff + (c * dim_size), offset, dim_size)); + coords_tile_->fixed_tile().read(buff + (c * dim_size), offset, dim_size); offset += cell_size; } }; @@ -464,9 +460,8 @@ Status ResultTile::read_nullable( auto validity_nbytes = len * validity_cell_size; auto validity_offset = pos * validity_cell_size; - RETURN_NOT_OK(tile.read(buffer, offset, nbytes)); - RETURN_NOT_OK( - tile_validity.read(buffer_validity, validity_offset, validity_nbytes)); + tile.read(buffer, offset, nbytes); + tile_validity.read(buffer_validity, validity_offset, validity_nbytes); return Status::Ok(); } diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 6b5f82f891a8..07a471dd4649 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1458,7 +1458,7 @@ void test_apply_tile( values[i * 2] = 'a'; values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * cells * sizeof(char)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, 2 * cells * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -1468,9 +1468,8 @@ void test_apply_tile( offsets[i] = offset; offset += 2; } - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -1479,8 +1478,8 @@ void test_apply_tile( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } test_apply_operators( @@ -1506,7 +1505,7 @@ void test_apply_tile( for (uint64_t i = 0; i < cells; ++i) { values[i] = static_cast(i); } - REQUIRE(tile->write(values.data(), 0, cells * sizeof(T)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, cells * sizeof(T))); test_apply_operators( field_name, @@ -1770,7 +1769,8 @@ TEST_CASE( values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char)).ok()); + REQUIRE_NOTHROW( + tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -1783,9 +1783,8 @@ TEST_CASE( offsets[cells - 2] = offset; offsets[cells - 1] = offset; offsets[cells] = offset; - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -1794,8 +1793,8 @@ TEST_CASE( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } // Empty string or null string as condition value @@ -2170,7 +2169,7 @@ void test_apply_tile_dense( values[i * 2] = 'a'; values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * cells * sizeof(char)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, 2 * cells * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -2180,9 +2179,8 @@ void test_apply_tile_dense( offsets[i] = offset; offset += 2; } - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -2191,8 +2189,8 @@ void test_apply_tile_dense( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } test_apply_operators_dense( @@ -2218,7 +2216,7 @@ void test_apply_tile_dense( for (uint64_t i = 0; i < cells; ++i) { values[i] = static_cast(i); } - REQUIRE(tile->write(values.data(), 0, cells * sizeof(T)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, cells * sizeof(T))); test_apply_operators_dense( field_name, @@ -2487,7 +2485,8 @@ TEST_CASE( values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char)).ok()); + REQUIRE_NOTHROW( + tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -2500,9 +2499,8 @@ TEST_CASE( offsets[cells - 2] = offset; offsets[cells - 1] = offset; offsets[cells] = offset; - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -2511,8 +2509,8 @@ TEST_CASE( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } // Empty string or null string as condition value @@ -2870,7 +2868,7 @@ void test_apply_tile_sparse( values[i * 2] = 'a'; values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * cells * sizeof(char)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, 2 * cells * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -2880,9 +2878,8 @@ void test_apply_tile_sparse( offsets[i] = offset; offset += 2; } - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -2891,8 +2888,8 @@ void test_apply_tile_sparse( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } test_apply_operators_sparse( @@ -2918,7 +2915,7 @@ void test_apply_tile_sparse( for (uint64_t i = 0; i < cells; ++i) { values[i] = static_cast(i); } - REQUIRE(tile->write(values.data(), 0, cells * sizeof(T)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, cells * sizeof(T))); test_apply_operators_sparse( field_name, @@ -3838,7 +3835,7 @@ TEST_CASE( for (uint64_t i = 0; i < cells; ++i) { values[i] = i; } - REQUIRE(tile->write(values.data(), 0, cells * sizeof(uint64_t)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, cells * sizeof(uint64_t))); std::vector tp_vec; populate_test_params_vector(field_name, &result_tile, tp_vec); @@ -4122,12 +4119,12 @@ TEST_CASE( Tile* const tile = &tile_tuple->var_tile(); std::vector offsets = {0, 5, 8, 13, 17, 21, 26, 31, 36, 40, 44}; - REQUIRE(tile->write(data.c_str(), 0, data.size()).ok()); + REQUIRE_NOTHROW(tile->write(data.c_str(), 0, data.size())); // Write the tile offsets. Tile* const tile_offsets = &tile_tuple->fixed_tile(); - REQUIRE(tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); std::vector tp_vec; populate_string_test_params_vector(field_name, &result_tile, tp_vec); @@ -4533,12 +4530,12 @@ TEST_CASE( ResultTile::TileTuple* const tile_tuple = result_tile.tile_tuple(field_name); Tile* const tile = &tile_tuple->var_tile(); - REQUIRE(tile->write(data.c_str(), 0, data.size()).ok()); + REQUIRE_NOTHROW(tile->write(data.c_str(), 0, data.size())); // Write the tile offsets. Tile* const tile_offsets = &tile_tuple->fixed_tile(); - REQUIRE(tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); std::vector tp_vec; populate_utf8_string_test_params_vector(field_name, &result_tile, tp_vec); @@ -4798,15 +4795,15 @@ TEST_CASE( // Populate the data tile. std::vector values = { 3.4f, 1.3f, 2.2f, 4.5f, 2.8f, 2.1f, 1.7f, 3.3f, 1.9f, 4.2f}; - REQUIRE(tile->write(values.data(), 0, cells * sizeof(float)).ok()); + REQUIRE_NOTHROW(tile->write(values.data(), 0, cells * sizeof(float))); Tile* const tile_validity = &tile_tuple->validity_tile(); std::vector validity(cells); for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); std::vector tp_vec; populate_nullable_test_params_vector(field_name, &result_tile, tp_vec); @@ -4907,7 +4904,8 @@ TEST_CASE( values[(i * 2) + 1] = 'a' + static_cast(i); } - REQUIRE(tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char)).ok()); + REQUIRE_NOTHROW( + tile->write(values.data(), 0, 2 * (cells - 2) * sizeof(char))); if (var_size) { Tile* const tile_offsets = &tile_tuple->fixed_tile(); @@ -4920,9 +4918,8 @@ TEST_CASE( offsets[cells - 2] = offset; offsets[cells - 1] = offset; offsets[cells] = offset; - REQUIRE( - tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t)) - .ok()); + REQUIRE_NOTHROW( + tile_offsets->write(offsets.data(), 0, (cells + 1) * sizeof(uint64_t))); } if (nullable) { @@ -4931,8 +4928,8 @@ TEST_CASE( for (uint64_t i = 0; i < cells; ++i) { validity[i] = i % 2; } - REQUIRE( - tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t)).ok()); + REQUIRE_NOTHROW( + tile_validity->write(validity.data(), 0, cells * sizeof(uint8_t))); } // Empty string or null string as condition value diff --git a/tiledb/sm/query/writers/dense_tiler.cc b/tiledb/sm/query/writers/dense_tiler.cc index 941ab247166f..4b6fe9c5abb0 100644 --- a/tiledb/sm/query/writers/dense_tiler.cc +++ b/tiledb/sm/query/writers/dense_tiler.cc @@ -228,7 +228,7 @@ Status DenseTiler::get_tile( // Fill entire tile with MAX_UINT64 std::vector to_write( cell_num_in_tile, std::numeric_limits::max()); - RETURN_NOT_OK(tile_pos.write(to_write.data(), 0, tile_off_size)); + tile_pos.write(to_write.data(), 0, tile_off_size); to_write.clear(); to_write.shrink_to_fit(); @@ -250,10 +250,10 @@ Status DenseTiler::get_tile( auto& tile_val = tile.var_tile(); for (uint64_t i = 0; i < cell_num_in_tile; ++i) { pos = tile_pos_buff[i]; - RETURN_NOT_OK(tile_off.write(&offset, tile_off_offset, sizeof(offset))); + tile_off.write(&offset, tile_off_offset, sizeof(offset)); tile_off_offset += sizeof(offset); if (pos == std::numeric_limits::max()) { // Empty - RETURN_NOT_OK(tile_val.write_var(&fill_var[0], offset, cell_size)); + tile_val.write_var(&fill_var[0], offset, cell_size); offset += cell_size; } else { // Non-empty val_offset = ((offsets_bytesize_ == 8) ? @@ -267,8 +267,7 @@ Status DenseTiler::get_tile( mul - val_offset) : buff_var_size - val_offset; - RETURN_NOT_OK( - tile_val.write_var(&buff_var[val_offset], offset, val_size)); + tile_val.write_var(&buff_var[val_offset], offset, val_size); offset += val_size; } } @@ -540,8 +539,7 @@ Status DenseTiler::copy_tile( auto d = dim_num - 1; while (true) { // Copy a slab - RETURN_NOT_OK( - tile.write(&buff[sub_offsets[d]], tile_offsets[d], copy_nbytes)); + tile.write(&buff[sub_offsets[d]], tile_offsets[d], copy_nbytes); // Advance cell coordinates, tile and buffer offsets auto last_dim_changed = d; diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index b2e86ca9c139..4f045fc8ef6b 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -935,15 +935,15 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( if (last_tile_cell_idx != 0) { if (coord_dups.empty()) { do { - RETURN_NOT_OK(last_tile.fixed_tile().write( + last_tile.fixed_tile().write( buffer + cell_idx * cell_size, last_tile_cell_idx * cell_size, - cell_size)); + cell_size); if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++cell_idx; ++last_tile_cell_idx; @@ -951,15 +951,15 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( } else { do { if (coord_dups.find(cell_idx) == coord_dups.end()) { - RETURN_NOT_OK(last_tile.fixed_tile().write( + last_tile.fixed_tile().write( buffer + cell_idx * cell_size, last_tile_cell_idx * cell_size, - cell_size)); + cell_size); if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++last_tile_cell_idx; } @@ -998,14 +998,14 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( // Write all remaining cells one by one if (coord_dups.empty()) { for (uint64_t i = 0; i < cell_num_to_write;) { - RETURN_NOT_OK(tile_it->fixed_tile().write( - buffer + cell_idx * cell_size, 0, cell_size * cell_num_per_tile)); + tile_it->fixed_tile().write( + buffer + cell_idx * cell_size, 0, cell_size * cell_num_per_tile); if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, 0, - constants::cell_validity_size * cell_num_per_tile)); + constants::cell_validity_size * cell_num_per_tile); } cell_idx += cell_num_per_tile; @@ -1021,14 +1021,14 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( } if (coord_dups.find(cell_idx) == coord_dups.end()) { - RETURN_NOT_OK(tile_it->fixed_tile().write( - buffer + cell_idx * cell_size, cell_idx * cell_size, cell_size)); + tile_it->fixed_tile().write( + buffer + cell_idx * cell_size, cell_idx * cell_size, cell_size); if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } } @@ -1039,29 +1039,29 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( last_tile_cell_idx = 0; if (coord_dups.empty()) { for (; cell_idx < cell_num; ++cell_idx, ++last_tile_cell_idx) { - RETURN_NOT_OK(last_tile.fixed_tile().write( + last_tile.fixed_tile().write( buffer + cell_idx * cell_size, last_tile_cell_idx * cell_size, - cell_size)); + cell_size); if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } } else { for (; cell_idx < cell_num; ++cell_idx) { if (coord_dups.find(cell_idx) == coord_dups.end()) { - RETURN_NOT_OK(last_tile.fixed_tile().write( + last_tile.fixed_tile().write( buffer + cell_idx * cell_size, last_tile_cell_idx * cell_size, - cell_size)); + cell_size); if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx * constants::cell_validity_size, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++last_tile_cell_idx; } @@ -1108,10 +1108,10 @@ Status GlobalOrderWriter::prepare_full_tiles_var( if (coord_dups.empty()) { do { // Write offset. - RETURN_NOT_OK(last_tile.offset_tile().write( + last_tile.offset_tile().write( &last_var_offset, last_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1121,16 +1121,16 @@ Status GlobalOrderWriter::prepare_full_tiles_var( prepare_buffer_offset( buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(last_tile.var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + last_tile.var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++cell_idx; @@ -1140,10 +1140,10 @@ Status GlobalOrderWriter::prepare_full_tiles_var( do { if (coord_dups.find(cell_idx) == coord_dups.end()) { // Write offset. - RETURN_NOT_OK(last_tile.offset_tile().write( + last_tile.offset_tile().write( &last_var_offset, last_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1154,16 +1154,16 @@ Status GlobalOrderWriter::prepare_full_tiles_var( prepare_buffer_offset( buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(last_tile.var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + last_tile.var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++last_tile_cell_idx; } @@ -1218,10 +1218,10 @@ Status GlobalOrderWriter::prepare_full_tiles_var( } // Write offset. - RETURN_NOT_OK(tile_it->offset_tile().write( + tile_it->offset_tile().write( &last_var_offset, current_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1232,16 +1232,16 @@ Status GlobalOrderWriter::prepare_full_tiles_var( prepare_buffer_offset( buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(tile_it->var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + tile_it->var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_idx, current_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } } else { @@ -1255,10 +1255,10 @@ Status GlobalOrderWriter::prepare_full_tiles_var( } // Write offset. - RETURN_NOT_OK(tile_it->offset_tile().write( + tile_it->offset_tile().write( &last_var_offset, current_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1269,16 +1269,16 @@ Status GlobalOrderWriter::prepare_full_tiles_var( prepare_buffer_offset( buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(tile_it->var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + tile_it->var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_idx, current_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++current_tile_cell_idx; @@ -1296,10 +1296,10 @@ Status GlobalOrderWriter::prepare_full_tiles_var( if (coord_dups.empty()) { for (; cell_idx < cell_num; ++cell_idx, ++last_tile_cell_idx) { // Write offset. - RETURN_NOT_OK(last_tile.offset_tile().write( + last_tile.offset_tile().write( &last_var_offset, last_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1309,26 +1309,26 @@ Status GlobalOrderWriter::prepare_full_tiles_var( *buffer_var_size - buff_offset : prepare_buffer_offset(buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(last_tile.var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + last_tile.var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } } else { for (; cell_idx < cell_num; ++cell_idx) { if (coord_dups.find(cell_idx) == coord_dups.end()) { // Write offset. - RETURN_NOT_OK(last_tile.offset_tile().write( + last_tile.offset_tile().write( &last_var_offset, last_tile_cell_idx * sizeof(last_var_offset), - sizeof(last_var_offset))); + sizeof(last_var_offset)); // Write var-sized value(s). auto buff_offset = @@ -1338,16 +1338,16 @@ Status GlobalOrderWriter::prepare_full_tiles_var( prepare_buffer_offset( buffer, cell_idx + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(last_tile.var_tile().write_var( - buffer_var + buff_offset, last_var_offset, var_size)); + last_tile.var_tile().write_var( + buffer_var + buff_offset, last_var_offset, var_size); last_var_offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(last_tile.validity_tile().write( + last_tile.validity_tile().write( buffer_validity + cell_idx, last_tile_cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++last_tile_cell_idx; diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 8d736a140357..05448057c2cd 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -455,13 +455,13 @@ Status UnorderedWriter::prepare_tiles_fixed( cell_idx = 0; } - RETURN_NOT_OK(tile_it->fixed_tile().write( - buffer + cell_pos_[i] * cell_size, cell_idx * cell_size, cell_size)); + tile_it->fixed_tile().write( + buffer + cell_pos_[i] * cell_size, cell_idx * cell_size, cell_size); if (nullable) - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_pos_[i] * constants::cell_validity_size, cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } else { for (uint64_t i = 0; i < cell_num; ++i) { @@ -473,13 +473,13 @@ Status UnorderedWriter::prepare_tiles_fixed( cell_idx = 0; } - RETURN_NOT_OK(tile_it->fixed_tile().write( - buffer + cell_pos_[i] * cell_size, cell_idx * cell_size, cell_size)); + tile_it->fixed_tile().write( + buffer + cell_pos_[i] * cell_size, cell_idx * cell_size, cell_size); if (nullable) - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_pos_[i] * constants::cell_validity_size, cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); ++cell_idx; } } @@ -530,8 +530,8 @@ Status UnorderedWriter::prepare_tiles_var( } // Write offset. - RETURN_NOT_OK(tile_it->offset_tile().write( - &offset, cell_idx * sizeof(offset), sizeof(offset))); + tile_it->offset_tile().write( + &offset, cell_idx * sizeof(offset), sizeof(offset)); // Write var-sized value(s). auto buff_offset = @@ -542,16 +542,15 @@ Status UnorderedWriter::prepare_tiles_var( prepare_buffer_offset( buffer, cell_pos_[i] + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(tile_it->var_tile().write_var( - buffer_var + buff_offset, offset, var_size)); + tile_it->var_tile().write_var(buffer_var + buff_offset, offset, var_size); offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_pos_[i], cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } } } else { @@ -567,8 +566,8 @@ Status UnorderedWriter::prepare_tiles_var( } // Write offset. - RETURN_NOT_OK(tile_it->offset_tile().write( - &offset, cell_idx * sizeof(offset), sizeof(offset))); + tile_it->offset_tile().write( + &offset, cell_idx * sizeof(offset), sizeof(offset)); // Write var-sized value(s). auto buff_offset = @@ -579,16 +578,15 @@ Status UnorderedWriter::prepare_tiles_var( prepare_buffer_offset( buffer, cell_pos_[i] + 1, attr_datatype_size) - buff_offset; - RETURN_NOT_OK(tile_it->var_tile().write_var( - buffer_var + buff_offset, offset, var_size)); + tile_it->var_tile().write_var(buffer_var + buff_offset, offset, var_size); offset += var_size; // Write validity value(s). if (nullable) { - RETURN_NOT_OK(tile_it->validity_tile().write( + tile_it->validity_tile().write( buffer_validity + cell_pos_[i], cell_idx * constants::cell_validity_size, - constants::cell_validity_size)); + constants::cell_validity_size); } ++cell_idx; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 775f066d518d..1812eda358d2 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1651,13 +1651,8 @@ Status StorageManagerCanonical::store_data_to_generic_tile( WriterTile& tile, const URI& uri, const EncryptionKey& encryption_key) { GenericTileIO tile_io(resources_, uri); uint64_t nbytes = 0; - Status st = tile_io.write_generic(&tile, encryption_key, &nbytes); - - if (st.ok()) { - st = vfs()->close_file(uri); - } - - return st; + tile_io.write_generic(&tile, encryption_key, &nbytes); + return vfs()->close_file(uri); } void StorageManagerCanonical::wait_for_zero_in_progress() { diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index ece221c65cb9..5968114d7866 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -46,6 +46,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for locally generated status exceptions. */ +class GenericTileIOException : public StatusException { + public: + explicit GenericTileIOException(const std::string& msg) + : StatusException("GenericTileIO", msg) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -69,21 +77,15 @@ Tile GenericTileIO::load( // Get encryption key from config if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { EncryptionKey cfg_enc_key(resources.config()); - auto&& [st1, tile_opt] = - tile_io.read_generic(offset, cfg_enc_key, resources.config()); - throw_if_not_ok(st1); - return std::move(*tile_opt); + return tile_io.read_generic(offset, cfg_enc_key, resources.config()); } else { - auto&& [st1, tile_opt] = - tile_io.read_generic(offset, encryption_key, resources.config()); - throw_if_not_ok(st1); - return std::move(*tile_opt); + return tile_io.read_generic(offset, encryption_key, resources.config()); } stdx::unreachable(); } -tuple> GenericTileIO::read_generic( +Tile GenericTileIO::read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config) { @@ -91,17 +93,14 @@ tuple> GenericTileIO::read_generic( if (encryption_key.encryption_type() != (EncryptionType)header.encryption_type) { - return { - LOG_STATUS(Status_TileIOError( - "Error reading generic tile; tile is encrypted with " + - encryption_type_str((EncryptionType)header.encryption_type) + - " but given key is for " + - encryption_type_str(encryption_key.encryption_type()))), - nullopt}; + throw GenericTileIOException( + "Error reading generic tile; tile is encrypted with " + + encryption_type_str((EncryptionType)header.encryption_type) + + " but given key is for " + + encryption_type_str(encryption_key.encryption_type())); } - RETURN_NOT_OK_TUPLE( - configure_encryption_filter(&header, encryption_key), nullopt); + configure_encryption_filter(&header, encryption_key); const auto tile_data_offset = GenericTileHeader::BASE_SIZE + header.filter_pipeline_size; @@ -117,20 +116,18 @@ tuple> GenericTileIO::read_generic( header.persisted_size); // Read the tile. - RETURN_NOT_OK_TUPLE( - resources_.vfs().read( - uri_, - file_offset + tile_data_offset, - tile.filtered_data(), - header.persisted_size), - nullopt); + throw_if_not_ok(resources_.vfs().read( + uri_, + file_offset + tile_data_offset, + tile.filtered_data(), + header.persisted_size)); // Unfilter assert(tile.filtered()); header.filters.run_reverse_generic_tile(&resources_.stats(), tile, config); assert(!tile.filtered()); - return {Status::Ok(), std::move(tile)}; + return tile; } GenericTileIO::GenericTileHeader GenericTileIO::read_generic_tile_header( @@ -171,28 +168,26 @@ GenericTileIO::GenericTileHeader GenericTileIO::read_generic_tile_header( return header; } -Status GenericTileIO::write_generic( +void GenericTileIO::write_generic( WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes) { // Create a header GenericTileHeader header; - RETURN_NOT_OK(init_generic_tile_header(tile, &header, encryption_key)); + init_generic_tile_header(tile, &header, encryption_key); // Filter tile assert(!tile->filtered()); - RETURN_NOT_OK(header.filters.run_forward( + throw_if_not_ok(header.filters.run_forward( &resources_.stats(), tile, nullptr, &resources_.compute_tp())); header.persisted_size = tile->filtered_buffer().size(); assert(tile->filtered()); - RETURN_NOT_OK(write_generic_tile_header(&header)); + write_generic_tile_header(&header); - RETURN_NOT_OK(resources_.vfs().write( + throw_if_not_ok(resources_.vfs().write( uri_, tile->filtered_buffer().data(), tile->filtered_buffer().size())); *nbytes = GenericTileIO::GenericTileHeader::BASE_SIZE + header.filter_pipeline_size + header.persisted_size; - - return Status::Ok(); } template @@ -208,7 +203,7 @@ void GenericTileIO::serialize_generic_tile_header( header.filters.serialize(serializer); } -Status GenericTileIO::write_generic_tile_header(GenericTileHeader* header) { +void GenericTileIO::write_generic_tile_header(GenericTileHeader* header) { SizeComputationSerializer fp_size_computation_serializer; header->filters.serialize(fp_size_computation_serializer); header->filter_pipeline_size = fp_size_computation_serializer.size(); @@ -221,12 +216,10 @@ Status GenericTileIO::write_generic_tile_header(GenericTileHeader* header) { serialize_generic_tile_header(serializer, *header); // Write buffer to file - RETURN_NOT_OK(resources_.vfs().write(uri_, data.data(), data.size())); - - return Status::Ok(); + throw_if_not_ok(resources_.vfs().write(uri_, data.data(), data.size())); } -Status GenericTileIO::configure_encryption_filter( +void GenericTileIO::configure_encryption_filter( GenericTileHeader* header, const EncryptionKey& encryption_key) const { switch ((EncryptionType)header->encryption_type) { case EncryptionType::NO_ENCRYPTION: @@ -235,20 +228,18 @@ Status GenericTileIO::configure_encryption_filter( case EncryptionType::AES_256_GCM: { auto* f = header->filters.get_filter(); if (f == nullptr) - return Status_TileIOError( + throw GenericTileIOException( "Error getting generic tile; no encryption filter."); f->set_key(encryption_key); break; } default: - return Status_TileIOError( + throw GenericTileIOException( "Error getting generic tile; invalid encryption type."); } - - return Status::Ok(); } -Status GenericTileIO::init_generic_tile_header( +void GenericTileIO::init_generic_tile_header( WriterTile* tile, GenericTileHeader* header, const EncryptionKey& encryption_key) const { @@ -262,10 +253,8 @@ Status GenericTileIO::init_generic_tile_header( constants::generic_tile_compression_level, tile->type())); - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + throw_if_not_ok(FilterPipeline::append_encryption_filter( &header->filters, encryption_key)); - - return Status::Ok(); } } // namespace sm diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index 3eb489544d91..7410784df30f 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -143,7 +143,7 @@ class GenericTileIO { * @param config The storage manager's config. * @return Status, Tile */ - tuple> read_generic( + Tile read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config); @@ -172,7 +172,7 @@ class GenericTileIO { * @param nbytes The total number of bytes written to the file. * @return Status */ - Status write_generic( + void write_generic( WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes); /** @@ -190,7 +190,7 @@ class GenericTileIO { * @param header The header to write * @return Status */ - Status write_generic_tile_header(GenericTileHeader* header); + void write_generic_tile_header(GenericTileHeader* header); private: /* ********************************* */ @@ -210,7 +210,7 @@ class GenericTileIO { * @param encryption_key The encryption key to use. * @return Status */ - Status configure_encryption_filter( + void configure_encryption_filter( GenericTileHeader* header, const EncryptionKey& encryption_key) const; /** @@ -223,7 +223,7 @@ class GenericTileIO { * @param encryption_key The encryption key to use. * @return Status */ - Status init_generic_tile_header( + void init_generic_tile_header( WriterTile* tile, GenericTileHeader* header, const EncryptionKey& encryption_key) const; diff --git a/tiledb/sm/tile/test/unit_tile.cc b/tiledb/sm/tile/test/unit_tile.cc index e56a802199e0..1c6b549d89b5 100644 --- a/tiledb/sm/tile/test/unit_tile.cc +++ b/tiledb/sm/tile/test/unit_tile.cc @@ -57,7 +57,7 @@ TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { } // Write the buffer to the test Tile. - CHECK(tile.write(write_buffer.data(), 0, tile_size).ok()); + CHECK_NOTHROW(tile.write(write_buffer.data(), 0, tile_size)); CHECK(tile.size() == tile_size); // Ensure the internal data was deep-copied: @@ -67,40 +67,43 @@ TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { // Test a partial read at offset 8, which should be a uint32_t with // a value of two. uint32_t two = 0; - CHECK(tile.read(&two, 8, sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.read(&two, 8, sizeof(uint32_t))); CHECK(two == 2); // Test a full read. std::vector read_buffer(buffer_len); uint64_t read_offset = 0; - CHECK(tile.read(read_buffer.data(), read_offset, tile_size).ok()); + CHECK_NOTHROW(tile.read(read_buffer.data(), read_offset, tile_size)); CHECK(memcmp(read_buffer.data(), write_buffer.data(), tile_size) == 0); // Test a write at a non-zero offset. Overwrite the two at offset 8. uint32_t magic = 5234549; - CHECK(tile.write(&magic, 8, sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.write(&magic, 8, sizeof(uint32_t))); // Read the magic number to ensure the '2' value was overwritten. two = 0; - CHECK(tile.read(&two, 8, sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.read(&two, 8, sizeof(uint32_t))); CHECK(two == magic); // Restore the state without the magic number. two = 2; - CHECK(tile.write(&two, 8, sizeof(uint32_t)).ok()); + CHECK_NOTHROW(tile.write(&two, 8, sizeof(uint32_t))); // Test a read at an out-of-bounds offset. memset(read_buffer.data(), 0, tile_size); read_offset = tile_size; - CHECK(!tile.read(read_buffer.data(), read_offset, tile_size).ok()); + auto matcher = Catch::Matchers::ContainsSubstring("Read tile overflow"); + CHECK_THROWS_WITH( + tile.read(read_buffer.data(), read_offset, tile_size), matcher); // Test a read at a valid offset but with a size that // exceeds the written buffer size. const uint32_t large_buffer_size = tile_size * 2; std::vector large_read_buffer(buffer_len * 2); read_offset = 0; - CHECK(!tile.read(large_read_buffer.data(), read_offset, large_buffer_size) - .ok()); + CHECK_THROWS_WITH( + tile.read(large_read_buffer.data(), read_offset, large_buffer_size), + matcher); // Free the write buffer to ensure that it was deep-copied // within the initial write. @@ -108,7 +111,7 @@ TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { write_buffer.clear(); memset(read_buffer.data(), 0, tile_size); read_offset = 0; - CHECK(tile.read(read_buffer.data(), read_offset, tile_size).ok()); + CHECK_NOTHROW(tile.read(read_buffer.data(), read_offset, tile_size)); CHECK(memcmp(read_buffer.data(), write_buffer_copy.data(), tile_size) == 0); } @@ -130,7 +133,7 @@ TEST_CASE("Tile: Test move constructor", "[Tile][move_constructor]") { } // Write the buffer to the first test Tile. - CHECK(tile1.write(buffer.data(), 0, tile_size).ok()); + CHECK_NOTHROW(tile1.write(buffer.data(), 0, tile_size)); // Instantiate a second test tile with the move constructor. Tile tile2(std::move(tile1)); @@ -148,7 +151,7 @@ TEST_CASE("Tile: Test move constructor", "[Tile][move_constructor]") { // written to the first test tile. std::vector read_buffer(buffer_len); uint64_t read_offset = 0; - CHECK(tile2.read(read_buffer.data(), read_offset, tile_size).ok()); + CHECK_NOTHROW(tile2.read(read_buffer.data(), read_offset, tile_size)); CHECK(memcmp(read_buffer.data(), buffer.data(), tile_size) == 0); } @@ -170,7 +173,7 @@ TEST_CASE("Tile: Test move-assignment", "[Tile][move_assignment]") { } // Write the buffer to the first test Tile. - CHECK(tile1.write(buffer.data(), 0, tile_size).ok()); + CHECK_NOTHROW(tile1.write(buffer.data(), 0, tile_size)); // Instantiate a third test tile with the move constructor. Tile tile2 = std::move(tile1); @@ -188,6 +191,6 @@ TEST_CASE("Tile: Test move-assignment", "[Tile][move_assignment]") { // written to the first test tile. std::vector read_buffer(buffer_len); uint64_t read_offset = 0; - CHECK(tile2.read(read_buffer.data(), read_offset, tile_size).ok()); + CHECK_NOTHROW(tile2.read(read_buffer.data(), read_offset, tile_size)); CHECK(memcmp(read_buffer.data(), buffer.data(), tile_size) == 0); -} \ No newline at end of file +} diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index b873858e300c..76bd8af3f06a 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -31,6 +31,7 @@ */ #include "tiledb/sm/tile/tile.h" +#include "tiledb/common/exception/exception.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/sm/enums/datatype.h" @@ -44,9 +45,9 @@ namespace tiledb { namespace sm { /** Class for locally generated status exceptions. */ -class TileStatusException : public StatusException { +class TileException : public StatusException { public: - explicit TileStatusException(const std::string& msg) + explicit TileException(const std::string& msg) : StatusException("Tile", msg) { } }; @@ -87,7 +88,7 @@ uint32_t WriterTile::compute_chunk_size( chunk_size64 = chunk_size64 / dim_cell_size * dim_cell_size; chunk_size64 = std::max(chunk_size64, dim_cell_size); if (chunk_size64 > std::numeric_limits::max()) { - throw TileStatusException("Chunk size exceeds uint32_t"); + throw TileException("Chunk size exceeds uint32_t"); } return static_cast(chunk_size64); @@ -197,29 +198,24 @@ void TileBase::swap(TileBase& tile) { std::swap(type_, tile.type_); } -Status TileBase::read( +void TileBase::read( void* const buffer, const uint64_t offset, const uint64_t nbytes) const { if (nbytes > size_ - offset) { - return LOG_STATUS(Status_TileError( - "Read tile overflow; may not read beyond buffer size")); + throw TileException("Read tile overflow; may not read beyond buffer size"); } std::memcpy(buffer, data_.get() + offset, nbytes); - return Status::Ok(); } -Status TileBase::write(const void* data, uint64_t offset, uint64_t nbytes) { +void TileBase::write(const void* data, uint64_t offset, uint64_t nbytes) { if (nbytes > size_ - offset) { - return LOG_STATUS( - Status_TileError("Write tile overflow; would write out of bounds")); + throw TileException("Write tile overflow; would write out of bounds"); } std::memcpy(data_.get() + offset, data, nbytes); size_ = std::max(offset + nbytes, size_); - - return Status::Ok(); } -Status Tile::zip_coordinates() { +void Tile::zip_coordinates() { assert(zipped_coords_dim_num_ > 0); // For easy reference @@ -245,8 +241,6 @@ Status Tile::zip_coordinates() { // Clean up tdb_free((void*)tile_tmp); - - return Status::Ok(); } uint64_t Tile::load_chunk_data(ChunkData& chunk_data) { @@ -256,7 +250,7 @@ uint64_t Tile::load_chunk_data(ChunkData& chunk_data) { uint64_t Tile::load_offsets_chunk_data(ChunkData& chunk_data) { auto s = size(); if (s < 8) { - throw std::runtime_error("Offsets tile should at least be 8 bytes."); + throw TileException("Offsets tile should at least be 8 bytes."); } return load_chunk_data(chunk_data, s - 8); @@ -274,8 +268,7 @@ void WriterTile::clear_data() { size_ = 0; } -Status WriterTile::write_var( - const void* data, uint64_t offset, uint64_t nbytes) { +void WriterTile::write_var(const void* data, uint64_t offset, uint64_t nbytes) { if (size_ - offset < nbytes) { auto new_alloc_size = size_ == 0 ? offset + nbytes : size_; while (new_alloc_size < offset + nbytes) @@ -284,14 +277,13 @@ Status WriterTile::write_var( auto new_data = static_cast(tdb_realloc(data_.release(), new_alloc_size)); if (new_data == nullptr) { - return LOG_STATUS(Status_TileError( - "Cannot reallocate buffer; Memory allocation failed")); + throw TileException("Cannot reallocate buffer; Memory allocation failed"); } data_.reset(new_data); size_ = new_alloc_size; } - return write(data, offset, nbytes); + write(data, offset, nbytes); } void WriterTile::swap(WriterTile& tile) { @@ -332,7 +324,7 @@ uint64_t Tile::load_chunk_data( } if (total_orig_size != expected_original_size) { - throw TileStatusException("Incorrect unfiltered tile size allocated."); + throw TileException("Incorrect unfiltered tile size allocated."); } return total_orig_size; diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index 0360be55d0c2..0bfcd906d4e3 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -109,7 +109,7 @@ class TileBase { * buffer of size nbytes. Does not mutate the internal offset. * Thread-safe among readers. */ - Status read(void* buffer, uint64_t offset, uint64_t nbytes) const; + void read(void* buffer, uint64_t offset, uint64_t nbytes) const; /** Returns the tile size. */ inline uint64_t size() const { @@ -127,7 +127,7 @@ class TileBase { * @note This function assumes that the tile buffer has already been * properly allocated. It does not alter the tile offset and size. */ - Status write(const void* data, uint64_t offset, uint64_t nbytes); + void write(const void* data, uint64_t offset, uint64_t nbytes); /** * Adds an extra offset at the end of this tile representing the size of the @@ -265,7 +265,7 @@ class Tile : public TileBase { * Zips the coordinate values such that a cell's coordinates across * all dimensions appear contiguously in the buffer. */ - Status zip_coordinates(); + void zip_coordinates(); /** * Reads the chunk data of a tile buffer and populates a chunk data structure. @@ -433,9 +433,8 @@ class WriterTile : public TileBase { * @param data Pointer to the data to write. * @param offset Offset to write into the tile buffer. * @param nbytes Number of bytes to write. - * @return Status. */ - Status write_var(const void* data, uint64_t offset, uint64_t nbytes); + void write_var(const void* data, uint64_t offset, uint64_t nbytes); /** * Sets the size of the tile. From a68824829aa3a618ed56d5997d1db5f6cf12eb2d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 3 Nov 2023 00:12:22 +0200 Subject: [PATCH 052/456] Deduplicate definition of the list of object libraries to link. (#4485) See https://github.com/TileDB-Inc/TileDB/pull/4446#issuecomment-1791032419. --- TYPE: NO_HISTORY --- tiledb/CMakeLists.txt | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index c5f6928e5296..15314a67dff9 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -411,16 +411,22 @@ add_library(TILEDB_CORE_OBJECTS OBJECT ${TILEDB_EXTERNALS_SOURCES} ) +# List of libraries to be linked to TILEDB_CORE_OBJECTS and TILEDB_CORE_OBJECTS_STATIC. +set(TILEDB_CORE_OBJECTS_LIBS + baseline +) + +list(TRANSFORM TILEDB_CORE_OBJECTS_LIBS PREPEND "$" OUTPUT_VARIABLE TILEDB_CORE_OBJECTS_LIBS_SOURCES) + target_sources(TILEDB_CORE_OBJECTS PUBLIC - $ + ${TILEDB_CORE_OBJECTS_LIBS_SOURCES} ) target_link_libraries(TILEDB_CORE_OBJECTS - PRIVATE - common PUBLIC - baseline + ${TILEDB_CORE_OBJECTS_LIBS} ) target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) @@ -475,11 +481,12 @@ if (WIN32 AND TILEDB_STATIC) target_sources(TILEDB_CORE_OBJECTS_STATIC PUBLIC - $) + ${TILEDB_CORE_OBJECTS_LIBS_SOURCES} + ) target_link_libraries(TILEDB_CORE_OBJECTS_STATIC PUBLIC - baseline + ${TILEDB_CORE_OBJECTS_LIBS} ) target_compile_definitions(TILEDB_CORE_OBJECTS_STATIC From 1f4ed0a69eca76a01183cca45a9d48fa0be5ecc6 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 3 Nov 2023 10:33:13 +0200 Subject: [PATCH 053/456] Use `FetchContent` to get vcpkg. (#4484) [SC-36205](https://app.shortcut.com/tiledb-inc/story/36205/use-fetchcontent-to-acquire-vcpkg) Currently, building TileDB from a GitHub archive fails by default[^1], because we use vcpkg from a submodule, and if we are not on a cloned repository that submodules does not exist. With this PR we change our approach to using CMake's [`FetchContent`](https://cmake.org/cmake/help/latest/module/FetchContent.html) module, and remove the `externals/vcpkg` submodule. [This is what the Azure SDK for C++ does.](https://github.com/Azure/azure-sdk-for-cpp/blob/12407e8bfcb9bc1aa43b253c1d0ec93bf795ae3b/cmake-modules/AzureVcpkg.cmake#L14-L40) This has the disadvantage that by default[^1] the vcpkg repository will be downloaded for every clean configure. It can by mitigated by specifying a `VCPKG_ROOT` environment variable to a local path of the vcpkg repository. [^1]: Unless the vcpkg toolchain file is manually specified. --- TYPE: BUILD DESC: Remove the vcpkg submodule and download it automatically when configuring --- .github/workflows/ci-linux_mac.yml | 12 ------------ .gitmodules | 3 --- cmake/Options/TileDBToolchain.cmake | 19 ++++++++++++++----- cmake/init-submodule.cmake | 27 --------------------------- external/vcpkg | 1 - scripts/ci/patch_vcpkg_triplets.py | 3 +-- 6 files changed, 15 insertions(+), 50 deletions(-) delete mode 100644 .gitmodules delete mode 100644 cmake/init-submodule.cmake delete mode 160000 external/vcpkg diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index b791f002ce3f..a02ac7ed0aff 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -74,18 +74,6 @@ jobs: core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11 - id: runvcpkg - with: - vcpkgJsonGlob: 'vcpkg.json' - vcpkgDirectory: '${{ github.workspace }}/external/vcpkg' - - - name: Prints output of run-vcpkg's action. - run: | - echo "run-vcpkg outputs:" - echo "${{ toJSON(steps.runvcpkg.outputs) }}" - - name: Prevent vpckg from building debug variants run: python ./scripts/ci/patch_vcpkg_triplets.py diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 78b70609888e..000000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "external/vcpkg"] - path = external/vcpkg - url = https://github.com/microsoft/vcpkg diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 5f09ce3b9557..73dab473f508 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -16,11 +16,20 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "Vcpkg toolchain file") - else() - include(init-submodule) - set(CMAKE_TOOLCHAIN_FILE - "${CMAKE_CURRENT_SOURCE_DIR}/external/vcpkg/scripts/buildsystems/vcpkg.cmake" - CACHE STRING "Vcpkg toolchain file") + elseif(NOT DEFINED ENV{TILEDB_DISABLE_AUTO_VCPKG}) + # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake + message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") + # To help with resolving conflicts, when you update the commit, also update its date. + set(VCPKG_COMMIT_STRING 1b4d69f3028d74401a001aa316986a670ca6289a) # 2023-09-27 + message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") + include(FetchContent) + FetchContent_Declare( + vcpkg + GIT_REPOSITORY https://github.com/microsoft/vcpkg.git + GIT_TAG ${VCPKG_COMMIT_STRING} + ) + FetchContent_MakeAvailable(vcpkg) + set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "Vcpkg toolchain file") endif() endif() diff --git a/cmake/init-submodule.cmake b/cmake/init-submodule.cmake deleted file mode 100644 index 75416e038275..000000000000 --- a/cmake/init-submodule.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# credit: https://cliutils.gitlab.io/modern-cmake/chapters/projects/submodule.html - -find_package(Git QUIET) - -# NOTE: cannot use PROJECT_SOURCE_DIR here because we need to run this *before* -# `project()` is called in order to ensure vcpkg is available. - -if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") - # Update submodules as needed - if(_TILEDB_CMAKE_INIT_GIT_SUBMODULES) - message(STATUS "Update submodules") - # TODO check pinned version here? - execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_RESULT - ECHO_OUTPUT_VARIABLE - COMMAND_ECHO STDOUT) - if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(FATAL_ERROR "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") - endif() - endif() -endif() - -if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/external/vcpkg/LICENSE.txt") - message(FATAL_ERROR "The submodules were not downloaded! _TILEDB_CMAKE_INIT_GIT_SUBMODULES" - " was turned off or failed. Please update submodules and try again.") -endif() diff --git a/external/vcpkg b/external/vcpkg deleted file mode 160000 index 1b4d69f3028d..000000000000 --- a/external/vcpkg +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1b4d69f3028d74401a001aa316986a670ca6289a diff --git a/scripts/ci/patch_vcpkg_triplets.py b/scripts/ci/patch_vcpkg_triplets.py index f8007955d237..6436e8f8db8f 100644 --- a/scripts/ci/patch_vcpkg_triplets.py +++ b/scripts/ci/patch_vcpkg_triplets.py @@ -1,8 +1,7 @@ import os workspace = os.curdir -triplet_paths = [os.path.join(workspace, "external", "vcpkg", "triplets"), - os.path.join(workspace, "ports", "triplets")] +triplet_paths = [os.path.join(workspace, "ports", "triplets")] for triplet_path in triplet_paths: for path, dnames, fnames in os.walk(triplet_path): From 8474db3517e098c41128bfdd7128d06c1a0fde0d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:11:35 +0100 Subject: [PATCH 054/456] Aggregates: don't read tiles when using fragment metadata. (#4481) This change modifies the readers to not read tiles when using the fragment metadata as it is not necessary. --- TYPE: IMPROVEMENT DESC: Aggregates: don't read tiles when using fragment metadata. --- test/src/test-cppapi-aggregates.cc | 150 ++++++++++++++++-- tiledb/sm/query/readers/dense_reader.cc | 127 +++++++++++---- tiledb/sm/query/readers/dense_reader.h | 36 ++++- .../readers/sparse_global_order_reader.cc | 72 +++++++-- .../readers/sparse_global_order_reader.h | 10 ++ .../query/readers/sparse_index_reader_base.cc | 10 +- .../query/readers/sparse_index_reader_base.h | 17 +- .../sparse_unordered_with_dups_reader.cc | 29 +++- 8 files changed, 383 insertions(+), 68 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 60fc27adc9f4..8cec791f7ebe 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -130,6 +130,8 @@ struct CppAggregatesFx { std::vector& a1_offsets, std::vector& a1_validity, const bool validate_count = true); + void validate_tiles_read(Query& query, bool is_count = false); + void validate_tiles_read_var(Query& query); void remove_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); @@ -1090,7 +1092,8 @@ void CppAggregatesFx::validate_data_var( a1_data_vec.emplace_back(v); } - // Generate an expected vector taking into consideration the query condition. + // Generate an expected vector taking into consideration the query + // condition. std::vector expected_a1_with_qc; expected_a1_with_qc.reserve(expected_a1.size()); for (uint64_t c = 0; c < expected_a1.size(); c++) { @@ -1138,6 +1141,119 @@ void CppAggregatesFx::validate_data_var( } } +template +void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { + // Validate the number of tiles read. + auto stats = query.stats(); + + // Parse num_tiles_read from the stats. + std::string to_find = + "\"Context.StorageManager.Query.Reader.num_tiles_read\": "; + auto start_pos = stats.find(to_find); + + uint64_t num_tiles_read = 0; + if (start_pos != std::string::npos) { + start_pos += to_find.length(); + auto end_pos = stats.find("\n", start_pos); + auto str = stats.substr(start_pos, end_pos - start_pos); + num_tiles_read = std::stoull(str); + } + + uint64_t expected_num_tiles_read; + if (dense_) { + // Dense has 5 tiles. If we request data or have a query condition, we'll + // read all of them. + if (request_data_ || set_qc_) { + expected_num_tiles_read = 5; + } else if (set_ranges_) { + // If we request range, we split all tiles, we'll have to read all. + expected_num_tiles_read = is_count ? 0 : 5; + } else { + // One space tile has two result tiles, we'll have to read them. + expected_num_tiles_read = is_count ? 0 : 2; + } + } else { + if (request_data_) { + if (set_ranges_) { + expected_num_tiles_read = 12; + } else { + expected_num_tiles_read = 15; + } + } else { + if (set_ranges_) { + if (allow_dups_) { + expected_num_tiles_read = is_count ? 8 : 10; + } else { + expected_num_tiles_read = is_count ? 8 : 11; + } + } else { + if (allow_dups_) { + expected_num_tiles_read = 0; + } else { + expected_num_tiles_read = is_count ? 10 : 14; + } + } + } + } + + CHECK(num_tiles_read == expected_num_tiles_read); +} + +template +void CppAggregatesFx::validate_tiles_read_var(Query& query) { + // Validate the number of tiles read. + auto stats = query.stats(); + + // Parse num_tiles_read from the stats. + std::string to_find = + "\"Context.StorageManager.Query.Reader.num_tiles_read\": "; + auto start_pos = stats.find(to_find); + + uint64_t num_tiles_read = 0; + if (start_pos != std::string::npos) { + start_pos += to_find.length(); + auto end_pos = stats.find("\n", start_pos); + auto str = stats.substr(start_pos, end_pos - start_pos); + num_tiles_read = std::stoull(str); + } + + uint64_t expected_num_tiles_read; + if (dense_) { + // Dense has 5 tiles. If we request data or have a query condition or set + // ranges, we'll read all of them. + if (request_data_ || set_qc_ || set_ranges_) { + expected_num_tiles_read = 5; + } else { + // One space tile has two result tiles, we'll have to read them. + expected_num_tiles_read = 2; + } + } else { + if (request_data_) { + if (set_ranges_) { + expected_num_tiles_read = 9; + } else { + expected_num_tiles_read = 12; + } + } else { + if (set_ranges_) { + if (allow_dups_) { + expected_num_tiles_read = 8; + } else { + expected_num_tiles_read = 9; + } + } else { + if (allow_dups_) { + expected_num_tiles_read = 0; + } else { + expected_num_tiles_read = 12; + } + } + } + } + + CHECK(num_tiles_read == expected_num_tiles_read); +} + template void CppAggregatesFx::remove_array(const std::string& array_name) { if (!is_array(array_name)) @@ -1225,6 +1341,8 @@ TEST_CASE_METHOD( if (request_data) { validate_data(query, dim1, dim2, a1, a1_validity); } + + validate_tiles_read(query, true); } } } @@ -1347,6 +1465,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::validate_data( query, dim1, dim2, a1, a1_validity); } + + CppAggregatesFx::validate_tiles_read(query); } } } @@ -1471,6 +1591,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::validate_data( query, dim1, dim2, a1, a1_validity); } + + CppAggregatesFx::validate_tiles_read(query); } } } @@ -1619,6 +1741,8 @@ TEMPLATE_LIST_TEST_CASE( if (request_data) { fx.validate_data(query, dim1, dim2, a1, a1_validity); } + + fx.validate_tiles_read(query); } } } @@ -1738,6 +1862,8 @@ TEMPLATE_LIST_TEST_CASE( fx.validate_data_var( query, dim1, dim2, a1_data, a1_offsets, a1_validity); } + + fx.validate_tiles_read_var(query); } } } @@ -1847,6 +1973,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CppAggregatesFx::validate_data( query, dim1, dim2, a1, a1_validity); } + + CppAggregatesFx::validate_tiles_read(query); } } } @@ -1940,6 +2068,8 @@ TEST_CASE_METHOD( validate_data_var( query, dim1, dim2, a1_data, a1_offsets, a1_validity); } + + validate_tiles_read_var(query); } } } @@ -1982,9 +2112,9 @@ TEST_CASE_METHOD( default_channel.apply_aggregate("NullCount", operation); // Add another aggregator on the second attribute. We will make the - // first attribute get a var size overflow, which should not impact the - // results of the second attribute as we don't request data for the - // second attribute. + // first attribute get a var size overflow, which should not impact + // the results of the second attribute as we don't request data for + // the second attribute. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount2", std::make_shared< @@ -2006,8 +2136,8 @@ TEST_CASE_METHOD( query.set_data_buffer("NullCount", null_count); query.set_data_buffer("NullCount2", null_count2); - // Here we run a few iterations until the query completes and update the - // buffers as we go. + // Here we run a few iterations until the query completes and update + // the buffers as we go. uint64_t var_buffer_size = dense_ ? 20 : 3; uint64_t curr_var_buffer_size = 0; uint64_t curr_elem = 0; @@ -2121,8 +2251,8 @@ TEST_CASE_METHOD( default_channel.apply_aggregate("NullCount", operation); // Add another aggregator on the second attribute. We will make this - // attribute get a var size overflow, which should impact the result of - // the first one hence throw an exception. + // attribute get a var size overflow, which should impact the result + // of the first one hence throw an exception. query.ptr()->query_->add_aggregator_to_default_channel( "NullCount2", std::make_shared< @@ -2221,8 +2351,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( query.set_validity_buffer("Sum", sum_validity); } - // Here we run a few iterations until the query completes and update the - // buffers as we go. + // Here we run a few iterations until the query completes and update + // the buffers as we go. uint64_t curr_elem = 0; uint64_t num_elems = CppAggregatesFx::dense_ ? 20 : 4; uint64_t num_iters = 0; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 828bcac5bf8b..b32237a89ebd 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -350,8 +350,10 @@ Status DenseReader::dense_read() { read_state_.partitioner_.subarray().relevant_fragments(), var_names); load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), names); - load_tile_metadata( - read_state_.partitioner_.subarray().relevant_fragments(), names); + if (!aggregates_.empty()) { + load_tile_metadata( + read_state_.partitioner_.subarray().relevant_fragments(), names); + } uint64_t t_start = 0; uint64_t t_end = 0; @@ -389,15 +391,14 @@ Status DenseReader::dense_read() { stats_->add_counter("internal_loop_num", 1); // Get result tiles to process on this iteration. - auto&& [ret_t_end, result_tiles_ret] = compute_result_tiles( + t_end = compute_space_tiles_end( names, condition_names, subarray, t_start, result_space_tiles, + tile_subarrays, compute_task); - t_end = ret_t_end; - auto result_tiles = std::move(result_tiles_ret); // Add the number of cells to process to subarray_end_cell. for (uint64_t t = t_start; t < t_end; t++) { @@ -421,7 +422,6 @@ Status DenseReader::dense_read() { t_end, condition_names, tile_extents, - result_tiles, tile_subarrays, tile_offsets, range_info, @@ -446,12 +446,21 @@ Status DenseReader::dense_read() { continue; } + // Get the tiles to load for this attribute. + auto result_tiles = result_tiles_to_load( + name, + condition_names, + subarray, + t_start, + t_end, + result_space_tiles, + tile_subarrays); + std::vector filtered_data; - if (condition_names.count(name) == 0) { - // Read and unfilter tiles. - to_read[0] = name; - filtered_data = std::move(read_attribute_tiles(to_read, result_tiles)); - } + + // Read and unfilter tiles. + to_read[0] = name; + filtered_data = std::move(read_attribute_tiles(to_read, result_tiles)); if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); @@ -471,10 +480,8 @@ Status DenseReader::dense_read() { num_range_threads, result_tiles, condition_names]() { - // Unfilter tiles if required. - if (condition_names.count(name) == 0) { - RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); - } + // Unfilter tiles. + RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); // Only copy names that are present in the user buffers. if (buffers_.count(name) != 0) { @@ -722,12 +729,13 @@ DenseReader::field_names_to_process( * the new iteration. */ template -tuple> DenseReader::compute_result_tiles( +uint64_t DenseReader::compute_space_tiles_end( const std::vector& names, const std::unordered_set& condition_names, Subarray& subarray, uint64_t t_start, std::map>& result_space_tiles, + const DynamicArray& tile_subarrays, ThreadPool::Task& compute_task) { // For easy reference. const auto& tile_coords = subarray.tile_coords(); @@ -744,16 +752,20 @@ tuple> DenseReader::compute_result_tiles( const uint64_t upper_memory_limit = std::min(tile_upper_memory_limit_ / 2, available_memory); - // Keep track of the required memory to load the result space tiles. The first - // element of this vector is an aggregate of all query condition fields, - // followed by the rest of the buffers in names. + // Keep track of the required memory to load the result space tiles. uint64_t required_memory_query_condition = 0; std::vector required_memory(names.size() - condition_names.size()); + // Cache which fields are for aggregation only. + std::vector aggregate_only_field(names.size() - condition_names.size()); + for (uint64_t n = condition_names.size(); n < names.size(); n++) { + auto& name = names[n]; + aggregate_only_field[n - condition_names.size()] = aggregate_only(name); + } + // Create the vector of result tiles to operate on. We stop once we reach the // end or the memory budget. The memory budget is combined for all query // condition attributes. - std::vector result_tiles; uint64_t t_end{t_start}; bool done = false; while (!done && t_end < tile_coords.size()) { @@ -784,6 +796,17 @@ tuple> DenseReader::compute_result_tiles( // current space tile. for (uint64_t n = condition_names.size(); n < names.size(); n++) { uint64_t tile_memory = 0; + uint64_t r_idx = n - condition_names.size(); + + // We might not need to load this tile into memory at all for aggregation + // only. + if (aggregate_only_field[r_idx] && + can_aggregate_tile_with_frag_md( + names[n], result_space_tile, tile_subarrays[t_end])) { + continue; + } + + // Compute memory usage for all result tiles in the space tile. for (const auto& result_tile : result_space_tile.result_tiles()) { auto& rt = result_tile.second; tile_memory += @@ -791,7 +814,6 @@ tuple> DenseReader::compute_result_tiles( } // If we reached the memory budget, stop. Always include the first tile. - uint64_t r_idx = n - condition_names.size(); if (t_end != t_start && required_memory[r_idx] + tile_memory > upper_memory_limit) { done = true; @@ -805,15 +827,8 @@ tuple> DenseReader::compute_result_tiles( break; } - // Add the result tiles for this space tile to the returned list to - // process. - for (const auto& result_tile : result_space_tile.result_tiles()) { - result_tiles.push_back(const_cast(&result_tile.second)); - } - t_end++; } - std::sort(result_tiles.begin(), result_tiles.end(), result_tile_cmp); // If we only include one tile, make sure we don't exceed the memory budget. if (t_end == t_start + 1) { @@ -833,7 +848,46 @@ tuple> DenseReader::compute_result_tiles( } } - return {t_end, result_tiles}; + return t_end; +} + +template +std::vector DenseReader::result_tiles_to_load( + const optional name, + const std::unordered_set& condition_names, + const Subarray& subarray, + const uint64_t t_start, + const uint64_t t_end, + std::map>& result_space_tiles, + const DynamicArray& tile_subarrays) const { + // For easy reference. + const auto& tile_coords = subarray.tile_coords(); + const bool agg_only = name.has_value() && aggregate_only(name.value()); + + // If the result is already loaded in query condition, return the empty list; + std::vector ret; + if (name.has_value() && condition_names.count(name.value()) != 0) { + return ret; + } + + for (uint64_t t = t_start; t < t_end; t++) { + // Add the result tiles for this space tile to the returned list to + // process unless it's for an aggregate only field and that tile can be + // processed with fragment metadata only. + const DimType* tc = (DimType*)&tile_coords[t][0]; + auto& result_space_tile = result_space_tiles.at(tc); + if (agg_only && can_aggregate_tile_with_frag_md( + name.value(), result_space_tile, tile_subarrays[t])) { + continue; + } + + for (const auto& result_tile : result_space_tile.result_tiles()) { + ret.push_back(const_cast(&result_tile.second)); + } + } + std::sort(ret.begin(), ret.end(), result_tile_cmp); + + return ret; } /** @@ -849,7 +903,6 @@ Status DenseReader::apply_query_condition( const uint64_t t_end, const std::unordered_set& condition_names, const std::vector& tile_extents, - std::vector& result_tiles, DynamicArray& tile_subarrays, std::vector& tile_offsets, const std::vector>& range_info, @@ -868,6 +921,16 @@ Status DenseReader::apply_query_condition( } } + // Get the result tiles to process. + auto result_tiles = result_tiles_to_load( + nullopt, + condition_names, + subarray, + t_start, + t_end, + result_space_tiles, + tile_subarrays); + // Read and unfilter query condition attributes. std::vector filtered_data = read_attribute_tiles(qc_names, result_tiles); @@ -1528,7 +1591,7 @@ Status DenseReader::copy_fixed_tiles( } // Apply query condition results to this slab. - if (condition_.has_value()) { + if (condition_.has_value() && result_space_tile.qc_filtered_results()) { for (uint64_t c = 0; c < iter.cell_slab_length(); c++) { if (!(qc_result[c + cell_offset] & 0x1)) { memcpy( @@ -1706,7 +1769,7 @@ Status DenseReader::copy_offset_tiles( } } - if (condition_.has_value()) { + if (condition_.has_value() && result_space_tile.qc_filtered_results()) { // Apply query condition results to this slab. for (uint64_t c = 0; c < iter.cell_slab_length(); c++) { if (!(qc_result[c + cell_offset] & 0x1)) { diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 74f7624b6beb..81b61901c1ec 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -200,18 +200,30 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::unordered_set& condition_names); /** - * Compute the result tiles to process on an iteration to respect the memory - * budget. + * Compute the result space tiles to process on an iteration to respect the + * memory budget. */ template - tuple> compute_result_tiles( + uint64_t compute_space_tiles_end( const std::vector& names, const std::unordered_set& condition_names, Subarray& subarray, uint64_t t_start, std::map>& result_space_tiles, + const DynamicArray& tile_subarrays, ThreadPool::Task& compute_task); + /** Compute the result tiles to load for a name. */ + template + std::vector result_tiles_to_load( + const optional name, + const std::unordered_set& condition_names, + const Subarray& subarray, + const uint64_t t_start, + const uint64_t t_end, + std::map>& result_space_tiles, + const DynamicArray& tile_subarrays) const; + /** Apply the query condition. */ template Status apply_query_condition( @@ -221,7 +233,6 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const uint64_t t_end, const std::unordered_set& condition_names, const std::vector& tile_extents, - std::vector& result_tiles, DynamicArray& tile_subarrays, std::vector& tile_offsets, const std::vector>& range_info, @@ -267,6 +278,19 @@ class DenseReader : public ReaderBase, public IQueryStrategy { ResultTile::TileTuple* tile_tuple, optional bitmap_data); + /** Returns wether the field is for aggregation only or not. */ + bool aggregate_only(const std::string& name) const { + if (condition_.has_value() && condition_->field_names().count(name) != 0) { + return false; + } + + if (buffers_.count(name) != 0) { + return false; + } + + return true; + } + /** * Returns wether or not we can aggregate the tile with only the fragment * metadata. @@ -280,7 +304,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { inline bool can_aggregate_tile_with_frag_md( const std::string& name, ResultSpaceTile& rst, - const Subarray& tile_subarray) { + const Subarray& tile_subarray) const { // Make sure there are no filtered results by the query condition and that // there are only one fragment domain for this tile. Having more fragment // domains for a tile means we'll have to merge data for many sources so we @@ -297,7 +321,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { // Make sure this tile isn't cropped by ranges and the fragment metadata has // tile metadata. if (tile_subarray.cell_num() != rt.cell_num() || - frag_md->has_tile_metadata()) { + !frag_md->has_tile_metadata()) { return false; } diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 6c037e1a1ccc..a78452f9caa6 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -1646,6 +1646,7 @@ SparseGlobalOrderReader::respect_copy_memory_budget( storage_manager_->compute_tp(), 0, names.size(), [&](uint64_t i) { // For easy reference. const auto& name = names[i]; + const bool agg_only = aggregate_only(name); const auto var_sized = array_schema_.var_size(name); uint64_t* mem_usage = &total_mem_usage_per_attr[i]; const bool is_timestamps = name == constants::timestamps || @@ -1667,6 +1668,13 @@ SparseGlobalOrderReader::respect_copy_memory_budget( // Get the size for all tiles. uint64_t idx = 0; for (; idx < max_cs_idx; idx++) { + // Skip this tile if it's aggregate only and we can aggregate it with + // the fragment metadata only. + if (agg_only && + can_aggregate_tile_with_frag_md(result_cell_slabs[idx])) { + continue; + } + auto rt = static_cast*>( result_cell_slabs[idx].tile_); const auto f = rt->frag_idx(); @@ -1809,6 +1817,28 @@ SparseGlobalOrderReader::compute_var_size_offsets( return {user_buffers_full, new_var_buffer_size}; } +template +std::vector +SparseGlobalOrderReader::result_tiles_to_load( + std::vector& result_cell_slabs, bool aggregate_only) { + std::vector result_tiles; + { + std::unordered_set found_tiles; + for (auto& rcs : result_cell_slabs) { + if (rcs.length_ != 0) { + if (found_tiles.count(rcs.tile_) == 0) { + found_tiles.emplace(rcs.tile_); + + if (!aggregate_only || !can_aggregate_tile_with_frag_md(rcs)) + result_tiles.emplace_back(rcs.tile_); + } + } + } + } + std::sort(result_tiles.begin(), result_tiles.end(), result_tile_cmp); + return result_tiles; +} + template template void SparseGlobalOrderReader::process_slabs( @@ -1847,28 +1877,38 @@ void SparseGlobalOrderReader::process_slabs( return; } - // Make a list of unique result tiles. - std::vector result_tiles; - { - std::unordered_set found_tiles; - for (auto& rcs : result_cell_slabs) { - if (rcs.length_ != 0) { - if (found_tiles.count(rcs.tile_) == 0) { - found_tiles.emplace(rcs.tile_); - result_tiles.emplace_back(rcs.tile_); - } - } - } - } - std::sort(result_tiles.begin(), result_tiles.end(), result_tile_cmp); - // Read a few attributes a a time. + std::vector result_tiles = + result_tiles_to_load(result_cell_slabs, false); std::optional last_field_to_overflow{std::nullopt}; uint64_t buffer_idx{0}; + optional> result_tiles_agg_only; while (buffer_idx < names.size()) { + // Generate a list of filtered result tiles for aggregates only fields. + bool agg_only = aggregate_only(names[buffer_idx]); + if (agg_only && result_tiles_agg_only == nullopt) { + result_tiles_agg_only = result_tiles_to_load(result_cell_slabs, true); + + // If we hit an overflow, it might have changed the tiles we need to load. + // Recompute the memory usage. This is because a tile where we might have + // included the full tile in a cell slab (0 to 'cell_num()') and not + // loaded might now be truncated to fit the user buffers. Since we can't + // use the fragment metadata to compute the aggregates for this tile on + // this iteration, the memory usage for this attribute needs to be + // recomputed. + if (last_field_to_overflow != nullopt) { + mem_usage_per_attr = respect_copy_memory_budget( + names, result_cell_slabs, user_buffers_full); + } + } + // Read and unfilter as many attributes as can fit in the budget. auto names_to_copy = read_and_unfilter_attributes( - names, mem_usage_per_attr, &buffer_idx, result_tiles); + names, + mem_usage_per_attr, + &buffer_idx, + agg_only ? *result_tiles_agg_only : result_tiles, + agg_only); for (const auto& name : names_to_copy) { // For easy reference. diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.h b/tiledb/sm/query/readers/sparse_global_order_reader.h index 1f8a7cddfdb9..3df66b98d5df 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.h +++ b/tiledb/sm/query/readers/sparse_global_order_reader.h @@ -512,6 +512,16 @@ class SparseGlobalOrderReader : public SparseIndexReaderBase, std::vector& cell_offsets, QueryBuffer& query_buffer); + /** + * Get the sorted unique result tile list from the result cell slabs. + * + * @param result_cell_slabs Result cell slabs. + * @param aggregate_only Are we generating the list for aggregate only fields? + * @return vector of result tiles. + */ + std::vector result_tiles_to_load( + std::vector& result_cell_slabs, bool aggregate_only); + /** * Copy cell slabs. * diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 0243dcb696f7..5299be7fcce6 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -834,7 +834,8 @@ std::vector SparseIndexReaderBase::read_and_unfilter_attributes( const std::vector& names, const std::vector& mem_usage_per_attr, uint64_t* buffer_idx, - std::vector& result_tiles) { + std::vector& result_tiles, + bool agg_only) { auto timer_se = stats_->start_timer("read_and_unfilter_attributes"); const uint64_t memory_budget = available_memory(); @@ -844,6 +845,13 @@ std::vector SparseIndexReaderBase::read_and_unfilter_attributes( while (*buffer_idx < names.size()) { auto& name = names[*buffer_idx]; + // Stop processing if we are doing non aggregates only fields and we hit an + // aggregates only field. Aggregates only field will pass in a filteted list + // of tiles to load. + if (!agg_only && aggregate_only(name)) { + break; + } + auto attr_mem_usage = mem_usage_per_attr[*buffer_idx]; if (memory_used + attr_mem_usage < memory_budget) { memory_used += attr_mem_usage; diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index 074179a1eda0..4b7ac9118825 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -671,6 +671,19 @@ class SparseIndexReaderBase : public ReaderBase { template void apply_query_condition(std::vector& result_tiles); + /** Returns wether the field is for aggregation only or not. */ + bool aggregate_only(const std::string& name) const { + if (qc_loaded_attr_names_set_.count(name) != 0) { + return false; + } + + if (buffers_.count(name) != 0) { + return false; + } + + return true; + } + /** * Read and unfilter as many attributes as can fit in the memory budget and * return the names loaded in 'names_to_copy'. Also keep the 'buffer_idx' @@ -680,6 +693,7 @@ class SparseIndexReaderBase : public ReaderBase { * @param mem_usage_per_attr Computed per attribute memory usage. * @param buffer_idx Stores/return the current buffer index in process. * @param result_tiles Result tiles to process. + * @param agg_only Are we loading for aggregates only field. * * @return names_to_copy. */ @@ -687,7 +701,8 @@ class SparseIndexReaderBase : public ReaderBase { const std::vector& names, const std::vector& mem_usage_per_attr, uint64_t* buffer_idx, - std::vector& result_tiles); + std::vector& result_tiles, + bool agg_only); /** * Get the field names to process. diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index a829fe9d38fd..5308d354b556 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -1473,6 +1473,7 @@ SparseUnorderedWithDupsReader::respect_copy_memory_budget( storage_manager_->compute_tp(), 0, names.size(), [&](uint64_t i) { // For easy reference. const auto& name = names[i]; + const bool agg_only = aggregate_only(name); const auto var_sized = array_schema_.var_size(name); auto mem_usage = &total_mem_usage_per_attr[i]; const bool is_timestamps = name == constants::timestamps || @@ -1490,7 +1491,14 @@ SparseUnorderedWithDupsReader::respect_copy_memory_budget( uint64_t idx = 0; for (; idx < max_rt_idx; idx++) { // Size of the tile in memory. - auto rt = (UnorderedWithDupsResultTile*)result_tiles[idx]; + auto rt = static_cast*>( + result_tiles[idx]); + + // Skip this tile if it's aggregate only and we can aggregate it with + // the fragment metadata only. + if (agg_only && can_aggregate_tile_with_frag_md(rt)) { + continue; + } // Skip for fields added in schema evolution. if (!fragment_metadata_[rt->frag_idx()]->array_schema()->is_field( @@ -1649,10 +1657,27 @@ bool SparseUnorderedWithDupsReader::process_tiles( // Read a few attributes at a time. std::optional last_field_to_overflow{std::nullopt}; uint64_t buffer_idx{0}; + optional> result_tiles_agg_only; while (buffer_idx < names.size()) { + // Generate a list of filtered result tiles for aggregates only fields. + bool agg_only = aggregate_only(names[buffer_idx]); + if (agg_only && result_tiles_agg_only == nullopt) { + result_tiles_agg_only = std::vector(); + for (auto& rt : result_tiles) { + if (!can_aggregate_tile_with_frag_md( + static_cast*>(rt))) { + result_tiles_agg_only->emplace_back(rt); + } + } + } + // Read and unfilter as many attributes as can fit in the budget. auto names_to_copy = read_and_unfilter_attributes( - names, mem_usage_per_attr, &buffer_idx, result_tiles); + names, + mem_usage_per_attr, + &buffer_idx, + agg_only ? *result_tiles_agg_only : result_tiles, + agg_only); // Process one field at a time for buffers in memory. for (const auto& name : names_to_copy) { From 4c75bd60669a6ee97743c8d155ccc84d66f3f881 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 3 Nov 2023 19:43:24 +0200 Subject: [PATCH 055/456] Add a table with the history of the storage format versions. (#4487) Extracted from #4213 into its own PR that can be merged more quickly. Also applied suggestions in https://github.com/TileDB-Inc/TileDB/pull/4213#discussion_r1276790191. --- TYPE: FORMAT DESC: Add a table with the history of the storage format versions. --------- Co-authored-by: Paul J. Davis --- format_spec/FORMAT_SPEC.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index a6212a805285..1039fb183272 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -11,6 +11,32 @@ title: Format Specification - [Dictionary filter](filters/dictionary_encoding.md) - RLE filter +## History + +|Format version|TileDB version|Description| +|-|-|-| +|1|1.4|[Decouple format and library version](https://github.com/TileDB-Inc/TileDB/commit/610f087515b6de5c3290b09dab30c6943ec77feb)| +|2|1.5|[Always split coordinate tiles](https://github.com/TileDB-Inc/TileDB/commit/9394b38bdfbacd606d673896b4ae87e7968b7c2f)| +|3|1.6|[Parallelize fragment metadata loading](https://github.com/TileDB-Inc/TileDB/commit/a2eb6237e622c3a17691dbe04c9223ba099f7466)| +|4|1.7|[Remove KV storage](https://github.com/TileDB-Inc/TileDB/commit/e733f7baa85a41e25e5834a220234397d6038401)| +|5|2.0|[Split coordinates into individual files](https://github.com/TileDB-Inc/TileDB/commit/d3543bdbc4ee7c2ed1f2de8cee42b04c6ec8eafc)| +|6|2.1|[Implement attribute fill values](https://github.com/TileDB-Inc/TileDB/commit/eaafa47c97af0ee654a0ca2e97da7b8d941e672b)| +|7|2.2|[Nullable attribute support](https://github.com/TileDB-Inc/TileDB/commit/a7fd8d6dd74bb4fa1ae25a6f995da93812f92c20)| +|8|2.3|[Percent encode attribute/dimension file names](https://github.com/TileDB-Inc/TileDB/commit/97c5c4b0aa35cfd96197558ffc1189860b4adc6f)| +|9|2.3|[Name attribute/dimension files by index](https://github.com/TileDB-Inc/TileDB/commit/9a2ed1c22242f097300c2909baf6cb671a7ee33e)| +|10|2.4|[Added array schema evolution](https://github.com/TileDB-Inc/TileDB/commit/41e5e8f4b185f49777560d637b1d61de498364ce)| +|11|2.7|[Store integral cells, aka, don't split cells across chunks](https://github.com/TileDB-Inc/TileDB/commit/beab5113526b7156c8c6492542f1681555c8ae87)| +|12|2.8|[New array directory structure](https://github.com/TileDB-Inc/TileDB/commit/ce204ad1ea5b40f006f4a6ddf240d89c08b3235b)| +|13|2.9|[Add dictionary filter](https://github.com/TileDB-Inc/TileDB/commit/5637e8c678451c9d2356ccada118b504c8ca85f0)| +|14|2.10|[Consolidation with timestamps, add has_timestamps to footer](https://github.com/TileDB-Inc/TileDB/commit/31a3dce8db254efc36f6d28249febed41bba3bcd)| +|15|2.11|[Remove consolidate with timestamps config](https://github.com/TileDB-Inc/TileDB/commit/6b49739e79d804dc56eb0a7e422823ae6f002276)| +|16|2.12|[Implement delete strategy](https://github.com/TileDB-Inc/TileDB/commit/8d64b1f38177113379fa741016136dbd2b06fcfd)| +|17|2.14|[Add dimension labels and data order](https://github.com/TileDB-Inc/TileDB/commit/bb433fcf12dc74a38c7e843808ec1e593b16ce71)| +|18|2.15|[Dimension Labels no longer experimental](https://github.com/TileDB-Inc/TileDB/commit/c3a1bb47e7237f50e8ed9e33abfaa3161e23ff64)| +|19|2.16|[Vac files now use relative URIs](https://github.com/TileDB-Inc/TileDB/commit/ef3236a526b67c50138436a16f67ad274c2ca037)| +|20|2.17|[Enumerations](https://github.com/TileDB-Inc/TileDB/commit/c0d7c6a50fdeffbcc7d8c9ba4a29230fe22baed6)| +|21|2.19|[Tile metadata are now correctly calculated for nullable fixed size strings on dense arrays](https://github.com/TileDB-Inc/TileDB/commit/081bcc5f7ce4bee576f08b97de348236ac88d429)| + ## Table of Contents * **Array** From f055b0f1d4b83ae3c53e44ecfd60171b92ba0ee1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 3 Nov 2023 21:15:12 +0200 Subject: [PATCH 056/456] Fix MinGW build failures. (#4488) --- TYPE: NO_HISTORY --- .github/workflows/build-rtools40.yml | 16 ++++++++++++++++ .gitignore | 3 +++ scripts/ci/print_logs.sh | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-rtools40.yml b/.github/workflows/build-rtools40.yml index 12226ef73728..13ed0ed62132 100644 --- a/.github/workflows/build-rtools40.yml +++ b/.github/workflows/build-rtools40.yml @@ -17,6 +17,21 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Get vcpkg commit to checkout + id: vcpkg_commit + shell: bash + run: | + echo "ref=$(grep -oP '(?<=set\(VCPKG_COMMIT_STRING )\w*' cmake/Options/TileDBToolchain.cmake | head -n 1)" >> "$GITHUB_OUTPUT" + # We clone vcpkg ourselves because having FetchContent do it inside + # the build directory causes compilation errors due to long paths. + - name: Checkout vcpkg repository + uses: actions/checkout@v3 + with: + repository: microsoft/vcpkg + path: vcpkg + ref: ${{ steps.vcpkg_commit.outputs.ref }} + # Vcpkg requires fetching all commits. + fetch-depth: 0 # Configure required environment variables for vcpkg to use # GitHub's Action Cache - uses: actions/github-script@v6 @@ -35,6 +50,7 @@ jobs: TILEDB_HOME: ${{ github.workspace }} MINGW_ARCH: ${{ matrix.msystem }} VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' + VCPKG_ROOT: ${{ github.workspace }}/vcpkg shell: c:\rtools40\usr\bin\bash.exe --login {0} - name: "Upload binaries" uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index af5cc15c559b..45d3caf24475 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ CMakeUserPresets.json docs/.quarto # Let CI build this /docs + +vcpkg +external/vcpkg diff --git a/scripts/ci/print_logs.sh b/scripts/ci/print_logs.sh index 35de2a196cc1..211eb711ef01 100644 --- a/scripts/ci/print_logs.sh +++ b/scripts/ci/print_logs.sh @@ -28,7 +28,7 @@ set -e pipefail # Display log files if the build failed echo "Dumping log files for failed build" -for f in $(find $GITHUB_WORKSPACE/{build,external} -name *.log); +for f in $(find $GITHUB_WORKSPACE/build -name *.log); do echo "::group::$f" cat $f From 78cd95102cfc787ab6d6ef5662795575534f8410 Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Sat, 4 Nov 2023 05:55:25 -0400 Subject: [PATCH 057/456] Split `timestamp_end_opened_at_` into 2 separate timestamps (#3633) The private `Array` variable `timestamp_end_opened_at_` is being used for two different use cases: 1. Specify end of timestamp range for fragments to load from file. 2. Specify timestamp to use for new fragments, metadata, etc. Split the private variable into two variables to make this explicit. --- TYPE: IMPROVEMENT DESC: Internal clean-up of array timestamps --- tiledb/sm/array/array.cc | 155 +++++++++++++++++++------------ tiledb/sm/array/array.h | 107 +++++++++++++++------ tiledb/sm/serialization/array.cc | 28 ++---- 3 files changed, 184 insertions(+), 106 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 1a140954676c..5f9350ad0f2a 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -91,9 +91,10 @@ Array::Array( , encryption_key_(make_shared(HERE())) , is_open_(false) , is_opening_or_closing_(false) - , timestamp_start_(0) - , timestamp_end_(UINT64_MAX) - , timestamp_end_opened_at_(UINT64_MAX) + , array_dir_timestamp_start_(0) + , user_set_timestamp_end_(nullopt) + , array_dir_timestamp_end_(UINT64_MAX) + , new_component_timestamp_(nullopt) , storage_manager_(storage_manager) , resources_(storage_manager_->resources()) , config_(resources_.config()) @@ -245,8 +246,8 @@ Status Array::open( uint32_t key_length) { return Array::open( query_type, - timestamp_start_, - timestamp_end_, + array_dir_timestamp_start_, + user_set_timestamp_end_.value_or(UINT64_MAX), encryption_type, encryption_key, key_length); @@ -265,16 +266,17 @@ Status Array::open( // Checks if (is_open()) { return LOG_STATUS( - Status_ArrayError("Cannot open array; Array already open")); + Status_ArrayError("Cannot open array; Array already open.")); } metadata_.clear(); metadata_loaded_ = false; non_empty_domain_computed_ = false; - timestamp_start_ = timestamp_start; - timestamp_end_opened_at_ = timestamp_end; query_type_ = query_type; + set_timestamps( + timestamp_start, timestamp_end, query_type_ == QueryType::READ); + /* Note: the open status MUST be exception safe. If anything interrupts the * opening process, it will throw and the array will be set as closed. */ try { @@ -338,19 +340,6 @@ Status Array::open( throw StatusException(st); } - if (timestamp_end_opened_at_ == UINT64_MAX) { - if (query_type == QueryType::READ) { - timestamp_end_opened_at_ = utils::time::timestamp_now_ms(); - } else if ( - query_type == QueryType::WRITE || - query_type == QueryType::MODIFY_EXCLUSIVE || - query_type == QueryType::DELETE || query_type == QueryType::UPDATE) { - timestamp_end_opened_at_ = 0; - } else { - throw ArrayException("Cannot open array; Unsupported query type."); - } - } - if (remote_) { auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { @@ -360,9 +349,7 @@ Status Array::open( if (!use_refactored_array_open()) { auto&& [st, array_schema_latest] = rest_client->get_array_schema_from_rest(array_uri_); - if (!st.ok()) { - throw StatusException(st); - } + throw_if_not_ok(st); array_schema_latest_ = array_schema_latest.value(); } else { auto st = rest_client->post_array_from_rest( @@ -376,7 +363,10 @@ Status Array::open( auto timer_se = resources_.stats().start_timer("array_open_read_load_directory"); array_dir_ = ArrayDirectory( - resources_, array_uri_, timestamp_start_, timestamp_end_opened_at_); + resources_, + array_uri_, + array_dir_timestamp_start_, + array_dir_timestamp_end_); } std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = open_for_reads(); @@ -389,8 +379,8 @@ Status Array::open( array_dir_ = ArrayDirectory( resources_, array_uri_, - timestamp_start_, - timestamp_end_opened_at_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, ArrayDirectoryMode::SCHEMA_ONLY); } auto&& [st, array_schema_latest, array_schemas] = open_for_writes(); @@ -400,7 +390,9 @@ Status Array::open( array_schema_latest_ = array_schema_latest.value(); array_schemas_all_ = array_schemas.value(); - metadata_.reset(timestamp_end_opened_at_); + // Set the timestamp + metadata_.reset(timestamp_for_new_component()); + } else if ( query_type == QueryType::DELETE || query_type == QueryType::UPDATE) { { @@ -409,8 +401,8 @@ Status Array::open( array_dir_ = ArrayDirectory( resources_, array_uri_, - timestamp_start_, - timestamp_end_opened_at_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, ArrayDirectoryMode::READ); } auto&& [st, array_schema_latest, array_schemas] = open_for_writes(); @@ -440,7 +432,8 @@ Status Array::open( return LOG_STATUS(Status_ArrayError(err.str())); } - metadata_.reset(timestamp_end_opened_at_); + // Updates the timestamp to use for metadata. + metadata_.reset(timestamp_for_new_component()); } else { throw ArrayException("Cannot open array; Unsupported query type."); } @@ -480,13 +473,15 @@ Status Array::close() { // metadata it won't trigger a deadlock metadata_loaded_ = true; auto rest_client = resources_.rest_client(); - if (rest_client == nullptr) + if (rest_client == nullptr) { throw Status_ArrayError( "Error closing array; remote array with no REST client."); - st = rest_client->post_array_metadata_to_rest( - array_uri_, timestamp_start_, timestamp_end_opened_at_, this); - if (!st.ok()) - throw StatusException(st); + } + throw_if_not_ok(rest_client->post_array_metadata_to_rest( + array_uri_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, + this)); } // Storage manager does not own the array schema for remote arrays. @@ -510,7 +505,6 @@ Status Array::close() { array_schemas_all_.clear(); metadata_.clear(); metadata_loaded_ = false; - } catch (std::exception& e) { is_opening_or_closing_ = false; throw Status_ArrayError(e.what()); @@ -618,8 +612,8 @@ std::vector> Array::get_enumerations( loaded = rest_client->post_enumerations_from_rest( array_uri_, - timestamp_start_, - timestamp_end_opened_at_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, this, names_to_load); } else { @@ -820,43 +814,58 @@ const EncryptionKey& Array::get_encryption_key() const { } Status Array::reopen() { - if (timestamp_end_ == timestamp_end_opened_at_) { - // The user has not set `timestamp_end_` since it was last opened. - // In this scenario, re-open at the current timestamp. - return reopen(timestamp_start_, utils::time::timestamp_now_ms()); + // Note: Array will only reopen for reads. This is why we are checking the + // timestamp for the array directory and not new components. This needs to be + // updated if non-read reopens are allowed. + if (!user_set_timestamp_end_.has_value() || + user_set_timestamp_end_.value() == array_dir_timestamp_end_) { + // The user has not set `timestamp_end_` since it was last opened or set it + // to use the current timestamp. In this scenario, re-open at the current + // timestamp. + return reopen(array_dir_timestamp_start_, utils::time::timestamp_now_ms()); } else { - // The user has set `timestamp_end_`. Reopen at that time stamp. - return reopen(timestamp_start_, timestamp_end_); + // The user has changed the end timestamp. Reopen at the user set timestamp. + return reopen(array_dir_timestamp_start_, user_set_timestamp_end_.value()); } } Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { auto timer = resources_.stats().start_timer("array_reopen"); + // Check the array was opened already in READ mode. if (!is_open_) { return LOG_STATUS( Status_ArrayError("Cannot reopen array; Array is not open")); } - if (query_type_ != QueryType::READ) { - return LOG_STATUS( - Status_ArrayError("Cannot reopen array; Array was " - "not opened in read mode")); + return LOG_STATUS(Status_ArrayError( + "Cannot reopen array; Array was not opened in read mode")); + } + + // Update the user set timestamp and the timestamp range to pass to the array + // directory. + if (timestamp_end == UINT64_MAX) { + user_set_timestamp_end_ = nullopt; + array_dir_timestamp_end_ = utils::time::timestamp_now_ms(); + + } else { + user_set_timestamp_end_ = timestamp_end; + array_dir_timestamp_end_ = timestamp_end; } + array_dir_timestamp_start_ = timestamp_start; + // Reset the last max buffer sizes. clear_last_max_buffer_sizes(); - timestamp_start_ = timestamp_start; - timestamp_end_opened_at_ = timestamp_end; + // Reopen metadata. fragment_metadata_.clear(); metadata_.clear(); metadata_loaded_ = false; + + // Reset the non-empty domain - may be different. non_empty_domain_.clear(); non_empty_domain_computed_ = false; - if (timestamp_end_opened_at_ == UINT64_MAX) { - timestamp_end_opened_at_ = utils::time::timestamp_now_ms(); - } - + // Use open to reopen a remote array. if (remote_) { try { set_array_closed(); @@ -873,14 +882,15 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { encryption_key_->key().size()); } + // Reload the array directory in READ mode (reopen only supports reads). try { { auto timer_se = resources_.stats().start_timer("array_reopen_directory"); array_dir_ = ArrayDirectory( resources_, array_uri_, - timestamp_start_, - timestamp_end_opened_at_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, query_type_ == QueryType::READ ? ArrayDirectoryMode::READ : ArrayDirectoryMode::SCHEMA_ONLY); } @@ -888,6 +898,7 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { return LOG_STATUS(Status_ArrayDirectoryError(le.what())); } + // Reopen the array and update private variables. std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = open_for_reads(); @@ -1132,6 +1143,19 @@ bool Array::serialize_metadata() const { return serialize_metadata_array_open; } +void Array::set_timestamps( + uint64_t timestamp_start, uint64_t timestamp_end, bool set_current_time) { + array_dir_timestamp_start_ = timestamp_start; + array_dir_timestamp_end_ = (set_current_time && timestamp_end == UINT64_MAX) ? + utils::time::timestamp_now_ms() : + timestamp_end; + if (timestamp_end == 0 || timestamp_end == UINT64_MAX) { + new_component_timestamp_ = nullopt; + } else { + new_component_timestamp_ = timestamp_end; + } +} + bool Array::use_refactored_array_open() const { auto found = false; auto refactored_array_open = false; @@ -1146,6 +1170,10 @@ bool Array::use_refactored_array_open() const { return refactored_array_open || use_refactored_query_submit(); } +uint64_t Array::timestamp_for_new_component() const { + return new_component_timestamp_.value_or(utils::time::timestamp_now_ms()); +} + bool Array::use_refactored_query_submit() const { auto found = false; auto refactored_query_submit = false; @@ -1503,7 +1531,10 @@ Status Array::load_metadata() { "Cannot load metadata; remote array with no REST client.")); } RETURN_NOT_OK(rest_client->get_array_metadata_from_rest( - array_uri_, timestamp_start_, timestamp_end_opened_at_, this)); + array_uri_, + array_dir_timestamp_start_, + timestamp_end_opened_at(), + this)); } else { do_load_metadata(); } @@ -1519,7 +1550,7 @@ Status Array::load_remote_non_empty_domain() { "Cannot load metadata; remote array with no REST client.")); } RETURN_NOT_OK(rest_client->get_array_non_empty_domain( - this, timestamp_start_, timestamp_end_opened_at_)); + this, array_dir_timestamp_start_, timestamp_end_opened_at())); non_empty_domain_computed_ = true; } return Status::Ok(); @@ -1541,7 +1572,11 @@ ArrayDirectory& Array::load_array_directory() { ArrayDirectoryMode::READ; array_dir_ = ArrayDirectory( - resources_, array_uri_, timestamp_start_, timestamp_end_opened_at_, mode); + resources_, + array_uri_, + array_dir_timestamp_start_, + array_dir_timestamp_end_, + mode); return array_dir_; } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 8023c5efc9e6..37a0f5353543 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -341,36 +341,75 @@ class Array { */ Status reopen(uint64_t timestamp_start, uint64_t timestamp_end); - /** Returns the start timestamp. */ + /** Returns the start timestamp used to load the array directory. */ inline uint64_t timestamp_start() const { - return timestamp_start_; + return array_dir_timestamp_start_; } - /** Returns the end timestamp. */ + /** + * Returns the end timestamp as set by the user. + * + * This may differ from the actual timestamp in use if the array has not yet + * been opened, the user has changed this value, or if using the sentinel + * value of `UINT64_MAX`. + */ inline uint64_t timestamp_end() const { - return timestamp_end_; + return user_set_timestamp_end_.value_or(UINT64_MAX); } - /** Returns the timestamp at which the array was opened. */ + /** + * Returns the timestamp at which the array was opened. + * + * WARNING: This is a legacy function that is needed to support the current + * API and REST calls. Do not use in new code. + */ inline uint64_t timestamp_end_opened_at() const { - return timestamp_end_opened_at_; + return query_type_ == QueryType::READ ? + array_dir_timestamp_end_ : + new_component_timestamp_.value_or(0); } + /** + * Returns the timestamp to use when writing components (fragment, + * metadata, etc.) + * + * If set to use the lastest time, this will get the time when called. + */ + uint64_t timestamp_for_new_component() const; + /** Directly set the timestamp start value. */ inline void set_timestamp_start(uint64_t timestamp_start) { - timestamp_start_ = timestamp_start; + array_dir_timestamp_start_ = timestamp_start; } /** Directly set the timestamp end value. */ inline void set_timestamp_end(uint64_t timestamp_end) { - timestamp_end_ = timestamp_end; + if (timestamp_end == UINT64_MAX) { + user_set_timestamp_end_ = nullopt; + } else { + user_set_timestamp_end_ = timestamp_end; + } } - /** Directly set the timestamp end opened at value. */ - inline void set_timestamp_end_opened_at( - const uint64_t timestamp_end_opened_at) { - timestamp_end_opened_at_ = timestamp_end_opened_at; - } + /** + * Set the internal timestamps. + * + * Note for sentinel values for `timestamp_end`: + * * `timestamp_end == UINT64_MAX`: + * The array directory end timestamp will be set to the current time if + * ``set_current_time=True``. New components will use the time at query + * submission. + * * `timestamp_end` == 0: + * The new component timestamp will use the time at query submission. + * + * @param timestamp_start The starting timestamp for opening the array + * directory. + * @param timstamp_end The ending timestamp for opening the array directory + * and setting new components. See above comments for sentinel values `0` and + * `UINT64_MAX`. + */ + void set_timestamps( + uint64_t timetamp_start, uint64_t timestamp_end, bool set_current_time); /** Directly set the array config. * @@ -631,28 +670,40 @@ class Array { QueryType query_type_ = QueryType::READ; /** - * The starting timestamp between to open `open_array_` at. - * In TileDB, timestamps are in ms elapsed since - * 1970-01-01 00:00:00 +0000 (UTC). + * Starting timestamp to open fragments between. + * + * Timestamps are ms elapsed since 1970-01-01 00:00:00 +0000 (UTC). */ - uint64_t timestamp_start_; + uint64_t array_dir_timestamp_start_; /** - * The ending timestamp between to open `open_array_` at. - * In TileDB, timestamps are in ms elapsed since - * 1970-01-01 00:00:00 +0000 (UTC). A value of UINT64_T - * will be interpretted as the current timestamp. + * Timestamp set by the user. + * + * This is used when setting the end timestamp for loading the array directory + * and the timestamp to use when creating fragments, metadata, etc. This may + * be changed by the user at any time. + * + * Timestamps are ms elapsed since 1970-01-01 00:00:00 +0000 (UTC). If set to + * `nullopt`, use the current time. */ - uint64_t timestamp_end_; + optional user_set_timestamp_end_; /** - * The ending timestamp that the array was last opened - * at. This is useful when `timestamp_end_` has been - * set to UINT64_T. In this scenario, this variable will - * store the timestamp for the time that the array was - * opened. + * Ending timestamp to open fragments between. + * + * Timestamps are ms elapsed since 1970-01-01 00:00:00 +0000 (UTC). Set to a + * sentinel value of UINT64_MAX before the array is opened. + */ + uint64_t array_dir_timestamp_end_; + + /** + * The timestamp to use when creating fragments, delete/update commits, + * metadata, etc. + * + * Timestamps are ms elapsed since 1970-01-01 00:00:00 +0000 (UTC). If set to + * `nullopt`, use the current time. */ - uint64_t timestamp_end_opened_at_; + optional new_component_timestamp_; /** TileDB storage manager. */ StorageManager* storage_manager_; diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 812430e97ac3..b0d3f3c93b78 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -240,8 +240,6 @@ Status array_from_capnp( if (array_reader.hasUri()) { array->set_uri_serialized(array_reader.getUri().cStr()); } - array->set_timestamp_start(array_reader.getStartTimestamp()); - array->set_timestamp_end(array_reader.getEndTimestamp()); if (array_reader.hasQueryType()) { auto query_type_str = array_reader.getQueryType(); @@ -252,22 +250,16 @@ Status array_from_capnp( array->set_serialized_array_open(); } - array->set_timestamp_end_opened_at(array_reader.getOpenedAtEndTimestamp()); - if (array->timestamp_end_opened_at() == UINT64_MAX) { - if (query_type == QueryType::READ) { - array->set_timestamp_end_opened_at( - tiledb::sm::utils::time::timestamp_now_ms()); - } else if ( - query_type == QueryType::WRITE || - query_type == QueryType::MODIFY_EXCLUSIVE || - query_type == QueryType::DELETE || query_type == QueryType::UPDATE) { - array->set_timestamp_end_opened_at(0); - } else { - throw StatusException(Status_SerializationError( - "Cannot open array; Unsupported query type.")); - } - } - } + array->set_timestamps( + array_reader.getStartTimestamp(), + array_reader.getEndTimestamp(), + query_type == QueryType::READ); + } else { + array->set_timestamps( + array_reader.getStartTimestamp(), + array_reader.getEndTimestamp(), + false); + }; if (array_reader.hasArraySchemasAll()) { std::unordered_map> all_schemas; From b5cff65ba0553da316c72e73fc3e152542e066dd Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:42:53 +0100 Subject: [PATCH 058/456] Aggregates: enable full support. (#4489) This removes the need for a configuration setting to use aggregates. It will be merged into 2.18. Also fixed is the serialization code paths so that we can run queries with aggregates on 2.18. I tested that the aggregates work with a REST server. --- TYPE: NO_HISTORY DESC: Aggregates: enable full support. --- test/src/test-cppapi-aggregates.cc | 4 +- test/src/unit-capi-config.cc | 2 - tiledb/api/c_api/config/config_api_external.h | 5 --- .../query_aggregate/query_aggregate_api.cc | 43 +++++-------------- .../test/unit_capi_query_aggregate.cc | 6 --- .../query_field/test/unit_capi_query_field.cc | 6 --- tiledb/sm/config/config.cc | 6 --- tiledb/sm/config/config.h | 3 -- tiledb/sm/cpp_api/config.h | 5 --- tiledb/sm/serialization/query.cc | 39 ++++++++++++++--- 10 files changed, 45 insertions(+), 74 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 8cec791f7ebe..0d82018241cd 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -140,9 +140,7 @@ struct CppAggregatesFx { template CppAggregatesFx::CppAggregatesFx() : vfs_(ctx_) { - Config cfg; - cfg["sm.allow_aggregates_experimental"] = "true"; - ctx_ = Context(cfg); + ctx_ = Context(); vfs_ = VFS(ctx_); remove_array(); diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 68a430f222cf..02ddccfe7d55 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -241,7 +241,6 @@ void check_save_to_file() { ss << "rest.server_serialization_format CAPNP\n"; ss << "rest.use_refactored_array_open false\n"; ss << "rest.use_refactored_array_open_and_query_submit false\n"; - ss << "sm.allow_aggregates_experimental false\n"; ss << "sm.allow_separate_attribute_writes false\n"; ss << "sm.allow_updates_experimental false\n"; ss << "sm.check_coord_dups true\n"; @@ -611,7 +610,6 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["rest.load_enumerations_on_array_open"] = "false"; all_param_values["rest.use_refactored_array_open"] = "true"; all_param_values["rest.use_refactored_array_open_and_query_submit"] = "true"; - all_param_values["sm.allow_aggregates_experimental"] = "false"; all_param_values["sm.allow_separate_attribute_writes"] = "false"; all_param_values["sm.allow_updates_experimental"] = "false"; all_param_values["sm.encryption_key"] = ""; diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 78bea4376f4b..2d34b3109b1a 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -91,11 +91,6 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * * **Parameters** * - * - `sm.allow_aggregates_experimental`
- * **Experimental**
- * Allow query aggregates APIs. Experimental for testing purposes, - * do not use.
- * **Default**: false * - `sm.allow_separate_attribute_writes`
* **Experimental**
* Allow separate attribute write queries.
diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index e61dede8383b..7e63c9242dfd 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -114,69 +114,50 @@ inline void ensure_query_channel_is_valid( ensure_handle_is_valid(channel); } -inline void ensure_aggregates_enabled_via_config(tiledb_ctx_t* ctx) { - bool enabled = ctx->context().resources().config().get( - "sm.allow_aggregates_experimental", sm::Config::MustFindMarker()); - if (!enabled) { - throw CAPIStatusException( - "The aggregates API is disabled. Please set the " - "sm.allow_aggregates_experimental config option to enable it"); - } -} - capi_return_t tiledb_channel_operator_sum_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operator_t** op) { ensure_output_pointer_is_valid(op); *op = tiledb_channel_operator_sum; return TILEDB_OK; } capi_return_t tiledb_channel_operator_mean_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operator_t** op) { ensure_output_pointer_is_valid(op); *op = tiledb_channel_operator_mean; return TILEDB_OK; } capi_return_t tiledb_channel_operator_null_count_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operator_t** op) { ensure_output_pointer_is_valid(op); *op = tiledb_channel_operator_null_count; return TILEDB_OK; } capi_return_t tiledb_channel_operator_min_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operator_t** op) { ensure_output_pointer_is_valid(op); *op = tiledb_channel_operator_min; return TILEDB_OK; } capi_return_t tiledb_channel_operator_max_get( - tiledb_ctx_t* ctx, const tiledb_channel_operator_t** op) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operator_t** op) { ensure_output_pointer_is_valid(op); *op = tiledb_channel_operator_max; return TILEDB_OK; } capi_return_t tiledb_aggregate_count_get( - tiledb_ctx_t* ctx, const tiledb_channel_operation_t** operation) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, const tiledb_channel_operation_t** operation) { ensure_output_pointer_is_valid(operation); *operation = tiledb_aggregate_count; return TILEDB_OK; } capi_return_t tiledb_query_get_default_channel( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - tiledb_query_channel_t** channel) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, tiledb_query_t* query, tiledb_query_channel_t** channel) { ensure_query_is_valid(query); ensure_output_pointer_is_valid(channel); @@ -194,7 +175,6 @@ capi_return_t tiledb_create_unary_aggregate( const tiledb_channel_operator_t* op, const char* input_field_name, tiledb_channel_operation_t** operation) { - ensure_aggregates_enabled_via_config(ctx); ensure_query_is_valid(query); ensure_query_is_not_initialized(query); ensure_channel_operator_is_valid(op); @@ -225,11 +205,10 @@ capi_return_t tiledb_create_unary_aggregate( } capi_return_t tiledb_channel_apply_aggregate( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_query_channel_t* channel, const char* output_field_name, const tiledb_channel_operation_t* operation) { - ensure_aggregates_enabled_via_config(ctx); ensure_query_channel_is_valid(channel); ensure_query_is_not_initialized(channel->query_); ensure_output_field_is_valid(output_field_name); @@ -240,8 +219,7 @@ capi_return_t tiledb_channel_apply_aggregate( } capi_return_t tiledb_aggregate_free( - tiledb_ctx_t* ctx, tiledb_channel_operation_t** operation) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, tiledb_channel_operation_t** operation) { ensure_output_pointer_is_valid(operation); ensure_operation_is_valid(*operation); tiledb_channel_operation_handle_t::break_handle(*operation); @@ -250,8 +228,7 @@ capi_return_t tiledb_aggregate_free( } capi_return_t tiledb_query_channel_free( - tiledb_ctx_t* ctx, tiledb_query_channel_t** channel) { - ensure_aggregates_enabled_via_config(ctx); + tiledb_ctx_t*, tiledb_query_channel_t** channel) { ensure_output_pointer_is_valid(channel); ensure_query_channel_is_valid(*channel); tiledb_query_channel_handle_t::break_handle(*channel); diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index 23b28b506825..3945c4637576 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -48,12 +48,6 @@ struct QueryAggregateFx : TemporaryDirectoryFixture { QueryAggregateFx() : array_name_(temp_dir_ + "queryaggregate_array") { - CHECK( - ctx->resources() - .config() - .set("sm.allow_aggregates_experimental", "true") - .ok() == true); - rm_array(); create_sparse_array(); write_sparse_array(); diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 98a5ee60433e..6fa5a79f1c05 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -40,12 +40,6 @@ using namespace tiledb::test; struct QueryFieldFx : TemporaryDirectoryFixture { QueryFieldFx() { - REQUIRE( - ctx->resources() - .config() - .set("sm.allow_aggregates_experimental", "true") - .ok() == true); - create_sparse_array(array_name()); write_sparse_array(array_name()); } diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index ea16c611f19a..c640de474f9a 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -96,7 +96,6 @@ const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false"; const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "false"; -const std::string Config::SM_ALLOW_AGGREGATES_EXPERIMENTAL = "false"; const std::string Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES = "false"; const std::string Config::SM_ALLOW_UPDATES_EXPERIMENTAL = "false"; const std::string Config::SM_ENCRYPTION_KEY = ""; @@ -268,9 +267,6 @@ const std::map default_config_values = { std::make_pair("config.logging_level", Config::CONFIG_LOGGING_LEVEL), std::make_pair( "config.logging_format", Config::CONFIG_LOGGING_DEFAULT_FORMAT), - std::make_pair( - "sm.allow_aggregates_experimental", - Config::SM_ALLOW_AGGREGATES_EXPERIMENTAL), std::make_pair( "sm.allow_separate_attribute_writes", Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES), @@ -717,8 +713,6 @@ Status Config::sanity_check( if (value != "DEFAULT" && value != "JSON") return LOG_STATUS( Status_ConfigError("Invalid logging format parameter value")); - } else if (param == "sm.allow_aggregates_experimental") { - RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "sm.allow_separate_attribute_writes") { RETURN_NOT_OK(utils::parse::convert(value, &v)); } else if (param == "sm.allow_updates_experimental") { diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 0826d406ce1e..ec7fcdb113ca 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -130,9 +130,6 @@ class Config { /** The default format for logging. */ static const std::string CONFIG_LOGGING_DEFAULT_FORMAT; - /** Allow aggregates API or not. */ - static const std::string SM_ALLOW_AGGREGATES_EXPERIMENTAL; - /** Allow separate attribute writes or not. */ static const std::string SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 3ad09185e789..3c3e09f0053c 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -265,11 +265,6 @@ class Config { * * **Parameters** * - * - `sm.allow_aggregates_experimental`
- * **Experimental**
- * Allow aggregates APIs. Experimental for testing purposes, - * do not use.
- * **Default**: false * - `sm.allow_separate_attribute_writes`
* **Experimental**
* Allow separate attribute write queries.
diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 65070c0f40e5..e67212f97696 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -2009,6 +2009,15 @@ Status query_from_capnp( Buffer offsets_buff(nullptr, fixedlen_size); Buffer varlen_buff(nullptr, varlen_size); Buffer validitylen_buff(nullptr, validitylen_size); + + // Aggregates don't have incomplete queries, and only set the results on + // completion, so we don't need to have go allocate memory. + if (query->is_aggregate(name)) { + offsets_buff = Buffer(fixedlen_size); + varlen_buff = Buffer(fixedlen_size); + validitylen_buff = Buffer(validitylen_size); + } + // For the server on reads we want to set the original user requested // buffer sizes This handles the case of incomplete queries where on // the second `submit()` call the client's buffer size will be the @@ -2044,19 +2053,39 @@ Status query_from_capnp( attr_state->validity_len_data.swap(validitylen_buff); if (var_size) { throw_if_not_ok(query->set_data_buffer( - name, nullptr, &attr_state->var_len_size, false, true)); + name, + attr_state->var_len_data.data(), + &attr_state->var_len_size, + false, + true)); throw_if_not_ok(query->set_offsets_buffer( - name, nullptr, &attr_state->fixed_len_size, false, true)); + name, + attr_state->fixed_len_data.data_as(), + &attr_state->fixed_len_size, + false, + true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( - name, nullptr, &attr_state->validity_len_size, false, true)); + name, + attr_state->validity_len_data.data_as(), + &attr_state->validity_len_size, + false, + true)); } } else { throw_if_not_ok(query->set_data_buffer( - name, nullptr, &attr_state->fixed_len_size, false, true)); + name, + attr_state->fixed_len_data.data(), + &attr_state->fixed_len_size, + false, + true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( - name, nullptr, &attr_state->validity_len_size, false, true)); + name, + attr_state->validity_len_data.data_as(), + &attr_state->validity_len_size, + false, + true)); } } } else if (query_type == QueryType::WRITE) { From c377557d2a51f3d8aab96e673e7f251b1320cd1d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:42:21 +0100 Subject: [PATCH 059/456] Aggregates: adding examples. (#4490) This adds simple examples for aggregates. --- TYPE: NO_HISTORY DESC: Aggregates: adding examples. --- examples/c_api/aggregates.c | 232 ++++++++++++++++++ examples/cpp_api/aggregates.cc | 143 +++++++++++ ...uery_aggregate_api_external_experimental.h | 12 +- 3 files changed, 381 insertions(+), 6 deletions(-) create mode 100644 examples/c_api/aggregates.c create mode 100644 examples/cpp_api/aggregates.cc diff --git a/examples/c_api/aggregates.c b/examples/c_api/aggregates.c new file mode 100644 index 000000000000..5e1e166f168d --- /dev/null +++ b/examples/c_api/aggregates.c @@ -0,0 +1,232 @@ +/** + * @file aggregates.c + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2018-2022 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 2D sparse array, write some data + * to it in global order, and read the data back with aggregates. + */ + +#include +#include +#include +#include + +// Name of array. +const char* array_name = "aggregates_array"; + +void create_array() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. + int dim_domain[] = {1, 4, 1, 4}; + int tile_extents[] = {4, 4}; + tiledb_dimension_t* d1; + tiledb_dimension_alloc( + ctx, "rows", TILEDB_INT32, &dim_domain[0], &tile_extents[0], &d1); + tiledb_dimension_t* d2; + tiledb_dimension_alloc( + ctx, "cols", TILEDB_INT32, &dim_domain[2], &tile_extents[1], &d2); + + // Create domain + tiledb_domain_t* domain; + tiledb_domain_alloc(ctx, &domain); + tiledb_domain_add_dimension(ctx, domain, d1); + tiledb_domain_add_dimension(ctx, domain, d2); + + // Create a single attribute "a" so each (i,j) cell can store an integer + tiledb_attribute_t* a; + tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a); + + // Create array schema + tiledb_array_schema_t* array_schema; + tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema); + tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); + tiledb_array_schema_set_domain(ctx, array_schema, domain); + tiledb_array_schema_add_attribute(ctx, array_schema, a); + + // Create array + tiledb_array_create(ctx, array_name, array_schema); + + // Clean up + tiledb_attribute_free(&a); + tiledb_dimension_free(&d1); + tiledb_dimension_free(&d2); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&array_schema); + tiledb_ctx_free(&ctx); +} + +void write_array() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // Open array for writing + tiledb_array_t* array; + tiledb_array_alloc(ctx, array_name, &array); + tiledb_array_open(ctx, array, TILEDB_WRITE); + + // Prepare data for first write + int coords_rows_1[] = {1, 2}; + int coords_cols_1[] = {1, 4}; + uint64_t coords_size_1 = sizeof(coords_rows_1); + int data_1[] = {1, 2}; + uint64_t data_size_1 = sizeof(data_1); + + // Create the query + tiledb_query_t* query; + tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + tiledb_query_set_data_buffer(ctx, query, "a", data_1, &data_size_1); + tiledb_query_set_data_buffer( + ctx, query, "rows", coords_rows_1, &coords_size_1); + tiledb_query_set_data_buffer( + ctx, query, "cols", coords_cols_1, &coords_size_1); + + // Submit first query + tiledb_query_submit(ctx, query); + + // Prepare data for second write + int coords_rows_2[] = {3}; + int coords_cols_2[] = {3}; + uint64_t coords_size_2 = sizeof(coords_rows_2); + int data_2[] = {3}; + uint64_t data_size_2 = sizeof(data_2); + + // Reset buffers + tiledb_query_set_data_buffer(ctx, query, "a", data_2, &data_size_2); + tiledb_query_set_data_buffer( + ctx, query, "rows", coords_rows_2, &coords_size_2); + tiledb_query_set_data_buffer( + ctx, query, "cols", coords_cols_2, &coords_size_2); + + // Submit second query + tiledb_query_submit(ctx, query); + + // Finalize query (IMPORTANT) + tiledb_query_finalize(ctx, query); + + // Close array + tiledb_array_close(ctx, array); + + // Clean up + tiledb_array_free(&array); + tiledb_query_free(&query); + tiledb_ctx_free(&ctx); +} + +void read_array() { + // Create TileDB context + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + + // Open array for reading + tiledb_array_t* array; + tiledb_array_alloc(ctx, array_name, &array); + tiledb_array_open(ctx, array, TILEDB_READ); + + // Read entire array + tiledb_subarray_t* subarray; + tiledb_subarray_alloc(ctx, array, &subarray); + int subarray_v[] = {1, 4, 1, 4}; + tiledb_subarray_set_subarray(ctx, subarray, subarray_v); + + // Calculate maximum buffer sizes + uint64_t count_size = 8; + uint64_t sum_size = 8; + + // Prepare the vector that will hold the result (1 cells) + uint64_t* count = (uint64_t*)malloc(count_size); + int64_t* sum = (int64_t*)malloc(sum_size); + + // Create query + tiledb_query_t* query; + tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + + // Get the default channel from the query + tiledb_query_channel_t* default_channel; + tiledb_query_get_default_channel(ctx, query, &default_channel); + + // Apply count aggregate + const tiledb_channel_operation_t* count_aggregate; + tiledb_aggregate_count_get(ctx, &count_aggregate); + tiledb_channel_apply_aggregate( + ctx, default_channel, "Count", count_aggregate); + + // Apply sum aggregate + const tiledb_channel_operator_t* operator_sum; + tiledb_channel_operator_sum_get(ctx, &operator_sum); + tiledb_channel_operation_t* sum_a; + tiledb_create_unary_aggregate(ctx, query, operator_sum, "a", &sum_a); + tiledb_channel_apply_aggregate(ctx, default_channel, "Sum", sum_a); + + tiledb_query_set_subarray_t(ctx, query, subarray); + tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + tiledb_query_set_data_buffer(ctx, query, "Count", count, &count_size); + tiledb_query_set_data_buffer(ctx, query, "Sum", sum, &sum_size); + + // Submit query + tiledb_query_submit(ctx, query); + + // Close array + tiledb_array_close(ctx, array); + + // Print out the results. + printf("Count has data %i\n", (int)count[0]); + printf("Sum has data %i\n", (int)sum[0]); + + // Clean up + free((void*)count); + free((void*)sum); + tiledb_subarray_free(&subarray); + tiledb_aggregate_free(ctx, &sum_a); + tiledb_query_channel_free(ctx, &default_channel); + tiledb_array_free(&array); + tiledb_query_free(&query); + tiledb_ctx_free(&ctx); +} + +int main() { + // Get object type + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(NULL, &ctx); + tiledb_object_t type; + tiledb_object_type(ctx, array_name, &type); + tiledb_ctx_free(&ctx); + + if (type != TILEDB_ARRAY) { + create_array(); + write_array(); + } + + read_array(); + return 0; +} diff --git a/examples/cpp_api/aggregates.cc b/examples/cpp_api/aggregates.cc new file mode 100644 index 000000000000..294ebfbf7359 --- /dev/null +++ b/examples/cpp_api/aggregates.cc @@ -0,0 +1,143 @@ +/** + * @file aggregates.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * When run, this program will create a simple 2D sparse array, write some data + * to it in global order, and read the data back with aggregates. + */ + +#include +#include +#include + +using namespace tiledb; + +// Name of array +std::string array_name("aggregates_array"); + +void create_array() { + // Create a TileDB context. + Context ctx; + + // If the array already exists on disk, return immediately. + if (Object::object(ctx, array_name).type() == Object::Type::Array) + return; + + // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "rows", {{1, 4}}, 4)) + .add_dimension(Dimension::create(ctx, "cols", {{1, 4}}, 4)); + + // The array will be sparse. + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + + // Add a single attribute "a" so each (i,j) cell can store an integer. + schema.add_attribute(Attribute::create(ctx, "a")); + + // Create the (empty) array on disk. + Array::create(array_name, schema); +} + +void write_array() { + // Open the array for writing and create the query. + Context ctx; + Array array(ctx, array_name, TILEDB_WRITE); + Query query(ctx, array); + + // Set layout to global + query.set_layout(TILEDB_GLOBAL_ORDER); + + // Submit first query + std::vector coords_rows_1 = {1, 2}; + std::vector coords_cols_1 = {1, 4}; + std::vector data_1 = {1, 2}; + query.set_data_buffer("a", data_1) + .set_data_buffer("rows", coords_rows_1) + .set_data_buffer("cols", coords_cols_1); + query.submit(); + + // Submit second query + std::vector coords_rows_2 = {3}; + std::vector coords_cols_2 = {3}; + std::vector data_2 = {3}; + query.set_data_buffer("a", data_2) + .set_data_buffer("rows", coords_rows_2) + .set_data_buffer("cols", coords_cols_2); + query.submit(); + + // Finalize - IMPORTANT! + query.finalize(); + + // Close the array + array.close(); +} + +void read_array() { + Context ctx; + + // Prepare the array for reading + Array array(ctx, array_name, TILEDB_READ); + + // Read the whole array + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(1, 1, 4); + + std::vector count(1); + std::vector sum(1); + + // Create a query + Query query(ctx, array); + + // Add aggregates for sum and count on the default channel. + QueryChannel default_channel = QueryExperimental::get_default_channel(query); + default_channel.apply_aggregate("Count", CountOperation()); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate(query, "a"); + default_channel.apply_aggregate("Sum", operation); + + // Set layout and buffers. + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("Count", count) + .set_data_buffer("Sum", sum); + + // Submit the query and close the array. + query.submit(); + array.close(); + + // Print out the results. + std::cout << "Count: " << count[0] << std::endl; + std::cout << "Sum: " << sum[0] << std::endl; +} + +int main() { + create_array(); + write_array(); + read_array(); + return 0; +} diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h index c860552d67ef..aef7d2de5e55 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h @@ -73,7 +73,7 @@ TILEDB_EXPORT extern const tiledb_channel_operation_t* tiledb_aggregate_count; * **Example:** * * @code{.c} - * tiledb_channel_operator_t *operator_sum; + * const tiledb_channel_operator_t *operator_sum; * tiledb_channel_operator_sum_get(ctx, &operator_sum); * tiledb_channel_operation_t* sum_A; * tiledb_create_unary_aggregate(ctx, query, operator_sum, "A", sum_A); @@ -91,7 +91,7 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_sum_get( * **Example:** * * @code{.c} - * tiledb_channel_operator_t *operator_min; + * const tiledb_channel_operator_t *operator_min; * tiledb_channel_operator_min_get(ctx, &operator_min); * tiledb_channel_operation_t* min_A; * tiledb_create_unary_aggregate(ctx, query, operator_min, "A", min_A); @@ -109,7 +109,7 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_min_get( * **Example:** * * @code{.c} - * tiledb_channel_operator_t *operator_max; + * const tiledb_channel_operator_t *operator_max; * tiledb_channel_operator_max_get(ctx, &operator_max); * tiledb_channel_operation_t* max_A; * tiledb_create_unary_aggregate(ctx, query, operator_max, "A", max_A); @@ -127,7 +127,7 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_max_get( * **Example:** * * @code{.c} - * tiledb_channel_operation_t* count_aggregate; + * const tiledb_channel_operation_t* count_aggregate; * tiledb_aggregate_count_get(ctx, &count_aggregate); * @endcode * @@ -144,7 +144,7 @@ TILEDB_EXPORT int32_t tiledb_aggregate_count_get( * **Example:** * * @code{.c} - * tiledb_channel_operator_t *operator_mean; + * const tiledb_channel_operator_t *operator_mean; * tiledb_channel_operator_mean_get(ctx, &operator_mean); * tiledb_channel_operation_t* mean_A; * tiledb_create_unary_aggregate(ctx, query, operator_mean, "A", mean_A); @@ -162,7 +162,7 @@ TILEDB_EXPORT int32_t tiledb_channel_operator_mean_get( * **Example:** * * @code{.c} - * tiledb_channel_operator_t *operator_nullcount; + * const tiledb_channel_operator_t *operator_nullcount; * tiledb_channel_operator_null_count_get(ctx, &operator_nullcount); * tiledb_channel_operation_t* nullcount_A; * tiledb_create_unary_aggregate(ctx, query, operator_nullcount, "A", From 6533869ab3e598fe8caa72d4df4027895520cac2 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Mon, 6 Nov 2023 13:55:35 -0500 Subject: [PATCH 060/456] Bump REST CI timeout to 90m (#4494) Bumps REST CI timeout to 90m from the default of 1h to prevent failures we've seen where the dispatched workflow takes just over 1h to complete normally. --- TYPE: NO_HISTORY DESC: Bump REST CI timeout to 90m --- .github/workflows/ci-rest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 2d13db08c0e9..5f9353cc9229 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -27,3 +27,4 @@ jobs: # Pass TileDB core ref to test against REST. # github.head_ref will only be set for PRs, so fallback to github.ref_name if triggered via push event. inputs: '{ "tiledb_ref": "${{ github.head_ref || github.ref_name }}"}' + wait-for-completion-timeout: 90m From 5efd7f53df6646c3b5894b0f7b06e1314f68c214 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 7 Nov 2023 11:01:27 +0200 Subject: [PATCH 061/456] Add a test for write access failures. (#4496) [SC-35061](https://app.shortcut.com/tiledb-inc/story/35061/add-tests-for-write-access-failures) See also https://github.com/TileDB-Inc/TileDB/pull/4052#pullrequestreview-1395205256. The test runs on non-Windows only. --- TYPE: NO_HISTORY --- test/src/unit-capi-query.cc | 100 +++++++++++++++++++++++++++++++++ test/support/src/vfs_helpers.h | 36 ++++++++++++ 2 files changed, 136 insertions(+) diff --git a/test/src/unit-capi-query.cc b/test/src/unit-capi-query.cc index 4d5de516471c..816ffce71902 100644 --- a/test/src/unit-capi-query.cc +++ b/test/src/unit-capi-query.cc @@ -41,6 +41,7 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" +#include "tiledb/platform/platform.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/win.h" #else @@ -704,3 +705,102 @@ TEST_CASE_METHOD( tiledb_array_free(&rarray); remove_temp_dir(temp_dir); } + +TEST_CASE_METHOD( + QueryFx, + "C API: Test query write failure", + "[capi][query][write-failure]") { + // DenyWriteAccess is not supported on Windows. + if constexpr (tiledb::platform::is_os_windows) + return; + SupportedFsLocal local_fs; + std::string temp_dir = local_fs.temp_dir(); + std::string array_name = temp_dir + "write_failure"; + create_temp_dir(temp_dir); + + tiledb_layout_t layout = + GENERATE(TILEDB_ROW_MAJOR, TILEDB_UNORDERED, TILEDB_GLOBAL_ORDER); + tiledb_array_type_t array_type = + layout == TILEDB_UNORDERED ? TILEDB_SPARSE : TILEDB_DENSE; + + tiledb_ctx_t* ctx; + int rc; + { + tiledb_array_schema_t* schema; + tiledb_domain_t* domain; + tiledb_dimension_t* dim; + tiledb_attribute_t* attr; + + rc = tiledb_ctx_alloc(nullptr, &ctx); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_schema_alloc(ctx, array_type, &schema); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_domain_alloc(ctx, &domain); + REQUIRE(rc == TILEDB_OK); + + int bounds[] = {0, 4}; + int extent = 1; + rc = tiledb_dimension_alloc(ctx, "d1", TILEDB_INT32, bounds, &extent, &dim); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain, dim); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_set_domain(ctx, schema, domain); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_attribute_alloc(ctx, "attr1", TILEDB_INT32, &attr); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_add_attribute(ctx, schema, attr); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_create(ctx, array_name.c_str(), schema); + REQUIRE(rc == TILEDB_OK); + + tiledb_attribute_free(&attr); + tiledb_dimension_free(&dim); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&schema); + } + + { + DenyWriteAccess dwa{array_name + "/__fragments"}; + + tiledb_array_t* array; + tiledb_query_t* query; + rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_layout(ctx, query, layout); + REQUIRE(rc == TILEDB_OK); + int data[] = {0, 1, 2, 3, 4}; + uint64_t data_size = sizeof(data); + rc = tiledb_query_set_data_buffer(ctx, query, "attr1", data, &data_size); + REQUIRE(rc == TILEDB_OK); + if (array_type == TILEDB_SPARSE) { + rc = tiledb_query_set_data_buffer(ctx, query, "d1", data, &data_size); + REQUIRE(rc == TILEDB_OK); + } + + rc = tiledb_query_submit(ctx, query); + REQUIRE(rc != TILEDB_OK); + + tiledb_error_t* err; + rc = tiledb_ctx_get_last_error(ctx, &err); + REQUIRE(rc == TILEDB_OK); + const char* msg; + rc = tiledb_error_message(err, &msg); + REQUIRE(rc == TILEDB_OK); + REQUIRE( + std::string(msg).find("Cannot create directory") != std::string::npos); + + tiledb_error_free(&err); + tiledb_query_free(&query); + tiledb_array_free(&array); + } + tiledb_ctx_free(&ctx); +} diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index f19878a895c9..f0b2017aaa4f 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -33,6 +33,7 @@ #ifndef TILEDB_VFS_HELPERS_H #define TILEDB_VFS_HELPERS_H +#include #include "test/support/src/helpers.h" #include "test/support/tdb_catch.h" @@ -683,6 +684,41 @@ struct TemporaryDirectoryFixture { const std::vector> supported_filesystems_; }; +/** + * Denies write access to a local filesystem path. + * + * Not supported on Windows. The permissions function there sets the + * readonly bit on the path, which is not supported on directories. + * + * To support it on Windows we would have to add and remove Access Control + * Lists, which is a nontrivial thing to do. + */ +class DenyWriteAccess { + public: + DenyWriteAccess() = delete; + + DenyWriteAccess(const std::string& path) + : path_(path) + , previous_perms_(std::filesystem::status(path).permissions()) { + std::filesystem::permissions( + path_, + std::filesystem::perms::owner_write, + std::filesystem::perm_options::remove); + } + + ~DenyWriteAccess() { + std::filesystem::permissions( + path_, previous_perms_, std::filesystem::perm_options::replace); + } + + private: + /** The path. */ + const std::string path_; + + /** The previous permissions of the path. */ + const std::filesystem::perms previous_perms_; +}; + } // End of namespace test } // End of namespace tiledb From d355b09f76d1b52ff76f49c4753570288fc57a7b Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 8 Nov 2023 20:24:50 +0100 Subject: [PATCH 062/456] Free query fields in set buffer cpp api after use. (#4501) --- TYPE: NO_HISTORY DESC: Free query fields in set buffer cpp api after use. --- tiledb/sm/cpp_api/query.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tiledb/sm/cpp_api/query.h b/tiledb/sm/cpp_api/query.h index 3cc9f075dd31..a7603be2d90e 100644 --- a/tiledb/sm/cpp_api/query.h +++ b/tiledb/sm/cpp_api/query.h @@ -2386,6 +2386,9 @@ class Query { // Get the type from the field. tiledb_datatype_t type; ctx.handle_error(tiledb_field_datatype(ctx.ptr().get(), field, &type)); + + // Free the field. + ctx.handle_error(tiledb_query_field_free(ctx.ptr().get(), &field)); return type; } @@ -2410,6 +2413,9 @@ class Query { uint32_t cell_val_num; ctx.handle_error( tiledb_field_cell_val_num(ctx.ptr().get(), field, &cell_val_num)); + + // Free the field. + ctx.handle_error(tiledb_query_field_free(ctx.ptr().get(), &field)); return cell_val_num == TILEDB_VAR_NUM; } From 147fa7ae24d7e1a5d31f68177b85a1e66ab65f58 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Thu, 9 Nov 2023 04:10:21 -0500 Subject: [PATCH 063/456] Add base class for VFS filesystems (#4497) Adds `FilesystemBase` class to use as a common base class between filesystems. This also updates `tiledb::sm::Posix` to inherit from `FilesystemBase`, remaining filesystems are to be converted in separate PRs. --- TYPE: IMPROVEMENT DESC: Add base class for VFS filesystems --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/CMakeLists.txt | 1 + tiledb/sm/filesystem/CMakeLists.txt | 1 + tiledb/sm/filesystem/filesystem_base.cc | 33 ++++ tiledb/sm/filesystem/filesystem_base.h | 196 ++++++++++++++++++++++++ tiledb/sm/filesystem/posix.cc | 152 ++++++++---------- tiledb/sm/filesystem/posix.h | 80 ++++------ tiledb/sm/filesystem/vfs.cc | 34 ++-- tiledb/sm/filesystem/vfs.h | 1 + 8 files changed, 346 insertions(+), 152 deletions(-) create mode 100644 tiledb/sm/filesystem/filesystem_base.cc create mode 100644 tiledb/sm/filesystem/filesystem_base.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 15314a67dff9..c5be9feeddc9 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -202,6 +202,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs_file_handle.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/win.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/filesystem_base.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/bit_width_reduction_filter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/bitshuffle_filter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/byteshuffle_filter.cc diff --git a/tiledb/sm/filesystem/CMakeLists.txt b/tiledb/sm/filesystem/CMakeLists.txt index 09011b18f9e9..7cfae52b99ce 100644 --- a/tiledb/sm/filesystem/CMakeLists.txt +++ b/tiledb/sm/filesystem/CMakeLists.txt @@ -39,6 +39,7 @@ commence(object_library vfs) vfs.cc vfs_file_handle.cc win.cc + filesystem_base.cc ../curl/curl_init.cc ) if (TILEDB_S3) diff --git a/tiledb/sm/filesystem/filesystem_base.cc b/tiledb/sm/filesystem/filesystem_base.cc new file mode 100644 index 000000000000..88ff55b45d3e --- /dev/null +++ b/tiledb/sm/filesystem/filesystem_base.cc @@ -0,0 +1,33 @@ +/** + * @file filesystem_base.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements the FilesystemBase class. + */ + +#include "filesystem_base.h" diff --git a/tiledb/sm/filesystem/filesystem_base.h b/tiledb/sm/filesystem/filesystem_base.h new file mode 100644 index 000000000000..b1e9409bc3ef --- /dev/null +++ b/tiledb/sm/filesystem/filesystem_base.h @@ -0,0 +1,196 @@ +/** + * @file filesystem.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the Filesystem base class. + */ + +#ifndef TILEDB_FILESYSTEMBASE_H +#define TILEDB_FILESYSTEMBASE_H + +#include "tiledb/common/filesystem/directory_entry.h" +#include "uri.h" + +#include + +namespace tiledb::sm { + +class VFS; + +class FilesystemBase { + public: + FilesystemBase() = default; + + virtual ~FilesystemBase() = default; + + /** + * Creates a directory. + * + * - On S3, this is a noop. + * - On all other backends, if the directory exists, the function + * just succeeds without doing anything. + * + * @param uri The URI of the directory. + * @return Status + */ + virtual Status create_dir(const URI& uri) const = 0; + + /** + * Creates an empty file. + * + * @param uri The URI of the file. + * @return Status + */ + virtual Status touch(const URI& uri) const = 0; + + /** + * Checks if a directory exists. + * + * @param uri The URI to check for existence. + * @return True if the directory exists, else False. + */ + virtual bool is_dir(const URI& uri) const = 0; + + /** + * Checks if a file exists. + * + * @param uri The URI to check for existence. + * @return True if the file exists, else False. + */ + virtual bool is_file(const URI& uri) const = 0; + + /** + * Removes a given directory (recursive) + * + * @param uri The uri of the directory to be removed + * @return Status + */ + virtual Status remove_dir(const URI& uri) const = 0; + + /** + * Deletes a file. + * + * @param uri The URI of the file. + * @return Status + */ + virtual Status remove_file(const URI& uri) const = 0; + + /** + * Retrieves the size of a file. + * + * @param uri The URI of the file. + * @param size The file size to be retrieved. + * @return Status + */ + virtual Status file_size(const URI& uri, uint64_t* size) const = 0; + + /** + * Retrieves all the entries contained in the parent. + * + * @param parent The target directory to list. + * @return All entries that are contained in the parent + */ + virtual tuple< + Status, + optional>> + ls_with_sizes(const URI& parent) const = 0; + + /** + * Renames a file. + * Both URI must be of the same backend type. (e.g. both s3://, file://, etc) + * + * @param old_uri The old URI. + * @param new_uri The new URI. + * @return Status + */ + virtual Status move_file(const URI& old_uri, const URI& new_uri) const = 0; + + /** + * Copies a file. + * Both URI must be of the same backend type. (e.g. both s3://, file://, etc) + * + * @param old_uri The old URI. + * @param new_uri The new URI. + * @return Status + */ + virtual Status copy_file(const URI& old_uri, const URI& new_uri) const = 0; + + /** + * Copies directory. + * Both URI must be of the same backend type. (e.g. both s3://, file://, etc) + * + * @param old_uri The old URI. + * @param new_uri The new URI. + * @return Status + */ + virtual Status copy_dir(const URI& old_uri, const URI& new_uri) const = 0; + + /** + * Reads from a file. + * + * @param uri The URI of the file. + * @param offset The offset where the read begins. + * @param buffer The buffer to read into. + * @param nbytes Number of bytes to read. + * @param use_read_ahead Whether to use the read-ahead cache. + * @return Status + */ + virtual Status read( + const URI& uri, + uint64_t offset, + void* buffer, + uint64_t nbytes, + bool use_read_ahead = true) = 0; + + /** + * Syncs (flushes) a file. Note that for S3 this is a noop. + * + * @param uri The URI of the file. + * @return Status + */ + virtual Status sync(const URI& uri) = 0; + + /** + * Writes the contents of a buffer into a file. + * + * @param uri The URI of the file. + * @param buffer The buffer to write from. + * @param buffer_size The buffer size. + * @param remote_global_order_write Remote global order write + * @return Status + */ + virtual Status write( + const URI& uri, + const void* buffer, + uint64_t buffer_size, + bool remote_global_order_write = false) = 0; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_FILESYSTEMBASE_H diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index 58df81e50239..a209601f2521 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -59,9 +59,16 @@ using tiledb::common::filesystem::directory_entry; namespace tiledb { namespace sm { -Posix::Posix() - : default_config_(Config()) - , config_(default_config_) { +Posix::Posix(const Config& config) { + // Initialize member variables with posix config parameters. + + // File and directory permissions are set by the user in octal. + std::string permissions = config.get( + "vfs.file.posix_file_permissions", Config::must_find); + file_permissions_ = std::strtol(permissions.c_str(), nullptr, 8); + permissions = config.get( + "vfs.file.posix_directory_permissions", Config::must_find); + directory_permissions_ = std::strtol(permissions.c_str(), nullptr, 8); } bool Posix::both_slashes(char a, char b) { @@ -151,18 +158,16 @@ std::string Posix::abs_path_internal(const std::string& path) { return ret_dir; } -Status Posix::create_dir(const std::string& path) const { +Status Posix::create_dir(const URI& uri) const { // If the directory does not exist, create it - if (is_dir(path)) { + auto path = uri.to_path(); + if (is_dir(uri)) { return LOG_STATUS(Status_IOError( std::string("Cannot create directory '") + path + "'; Directory already exists")); } - uint32_t permissions = 0; - RETURN_NOT_OK(get_posix_directory_permissions(&permissions)); - - if (mkdir(path.c_str(), permissions) != 0) { + if (mkdir(path.c_str(), directory_permissions_) != 0) { return LOG_STATUS(Status_IOError( std::string("Cannot create directory '") + path + "'; " + strerror(errno))); @@ -170,11 +175,11 @@ Status Posix::create_dir(const std::string& path) const { return Status::Ok(); } -Status Posix::touch(const std::string& filename) const { - uint32_t permissions = 0; - RETURN_NOT_OK(get_posix_file_permissions(&permissions)); +Status Posix::touch(const URI& uri) const { + auto filename = uri.to_path(); - int fd = ::open(filename.c_str(), O_WRONLY | O_CREAT | O_SYNC, permissions); + int fd = + ::open(filename.c_str(), O_WRONLY | O_CREAT | O_SYNC, file_permissions_); if (fd == -1 || ::close(fd) != 0) { return LOG_STATUS(Status_IOError( std::string("Failed to create file '") + filename + "'; " + @@ -199,7 +204,8 @@ int Posix::unlink_cb(const char* fpath, const struct stat*, int, struct FTW*) { return rc; } -Status Posix::remove_dir(const std::string& path) const { +Status Posix::remove_dir(const URI& uri) const { + auto path = uri.to_path(); int rc = nftw(path.c_str(), unlink_cb, 64, FTW_DEPTH | FTW_PHYS); if (rc) return LOG_STATUS(Status_IOError( @@ -208,7 +214,8 @@ Status Posix::remove_dir(const std::string& path) const { return Status::Ok(); } -Status Posix::remove_file(const std::string& path) const { +Status Posix::remove_file(const URI& uri) const { + auto path = uri.to_path(); if (remove(path.c_str()) != 0) { return LOG_STATUS(Status_IOError( std::string("Cannot delete file '") + path + "'; " + strerror(errno))); @@ -216,7 +223,8 @@ Status Posix::remove_file(const std::string& path) const { return Status::Ok(); } -Status Posix::file_size(const std::string& path, uint64_t* size) const { +Status Posix::file_size(const URI& uri, uint64_t* size) const { + auto path = uri.to_path(); int fd = open(path.c_str(), O_RDONLY); if (fd == -1) { return LOG_STATUS(Status_IOError( @@ -231,22 +239,16 @@ Status Posix::file_size(const std::string& path, uint64_t* size) const { return Status::Ok(); } -Status Posix::init(const Config& config) { - config_ = config; - - return Status::Ok(); -} - -bool Posix::is_dir(const std::string& path) const { +bool Posix::is_dir(const URI& uri) const { struct stat st; memset(&st, 0, sizeof(struct stat)); - return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode); + return stat(uri.to_path().c_str(), &st) == 0 && S_ISDIR(st.st_mode); } -bool Posix::is_file(const std::string& path) const { +bool Posix::is_file(const URI& uri) const { struct stat st; memset(&st, 0, sizeof(struct stat)); - return (stat(path.c_str(), &st) == 0) && !S_ISDIR(st.st_mode); + return (stat(uri.to_path().c_str(), &st) == 0) && !S_ISDIR(st.st_mode); } Status Posix::ls( @@ -286,7 +288,7 @@ tuple>> Posix::ls_with_sizes( entries.emplace_back(abspath, 0, true); } else { uint64_t size; - RETURN_NOT_OK_TUPLE(file_size(abspath, &size), nullopt); + RETURN_NOT_OK_TUPLE(file_size(URI(abspath), &size), nullopt); entries.emplace_back(abspath, size, false); } } @@ -299,26 +301,25 @@ tuple>> Posix::ls_with_sizes( return {Status::Ok(), entries}; } -Status Posix::move_path( - const std::string& old_path, const std::string& new_path) { - if (rename(old_path.c_str(), new_path.c_str()) != 0) { +Status Posix::move_file(const URI& old_path, const URI& new_path) const { + if (rename(old_path.to_path().c_str(), new_path.to_path().c_str()) != 0) { return LOG_STATUS( Status_IOError(std::string("Cannot move path: ") + strerror(errno))); } return Status::Ok(); } -Status Posix::copy_file( - const std::string& old_path, const std::string& new_path) { - std::ifstream src(old_path, std::ios::binary); - std::ofstream dst(new_path, std::ios::binary); +Status Posix::copy_file(const URI& old_uri, const URI& new_uri) const { + std::ifstream src(old_uri.to_path(), std::ios::binary); + std::ofstream dst(new_uri.to_path(), std::ios::binary); dst << src.rdbuf(); return Status::Ok(); } -Status Posix::copy_dir( - const std::string& old_path, const std::string& new_path) { - RETURN_NOT_OK(create_dir(new_path)); +Status Posix::copy_dir(const URI& old_uri, const URI& new_uri) const { + auto old_path = old_uri.to_path(); + auto new_path = new_uri.to_path(); + RETURN_NOT_OK(create_dir(new_uri)); std::vector paths; RETURN_NOT_OK(ls(old_path, &paths)); @@ -327,20 +328,20 @@ Status Posix::copy_dir( path_queue.emplace(std::move(path)); while (!path_queue.empty()) { - std::string file_name_abs = path_queue.front(); - std::string file_name = file_name_abs.substr(old_path.length()); + const std::string file_name_abs = path_queue.front(); + const std::string file_name = file_name_abs.substr(old_path.length()); path_queue.pop(); - if (is_dir(file_name_abs)) { - RETURN_NOT_OK(create_dir(new_path + "/" + file_name)); + if (is_dir(URI(file_name_abs))) { + RETURN_NOT_OK(create_dir(URI(new_path + "/" + file_name))); std::vector child_paths; RETURN_NOT_OK(ls(file_name_abs, &child_paths)); for (auto& path : child_paths) path_queue.emplace(std::move(path)); } else { - assert(is_file(file_name_abs)); - RETURN_NOT_OK( - copy_file(old_path + "/" + file_name, new_path + "/" + file_name)); + assert(is_file(URI(file_name_abs))); + RETURN_NOT_OK(copy_file( + URI(old_path + "/" + file_name), URI(new_path + "/" + file_name))); } } @@ -402,13 +403,15 @@ void Posix::purge_dots_from_path(std::string* path) { } Status Posix::read( - const std::string& path, + const URI& uri, uint64_t offset, void* buffer, - uint64_t nbytes) const { + uint64_t nbytes, + [[maybe_unused]] bool use_read_ahead) { // Checks + auto path = uri.to_path(); uint64_t file_size; - RETURN_NOT_OK(this->file_size(path, &file_size)); + RETURN_NOT_OK(this->file_size(URI(path), &file_size)); if (offset + nbytes > file_size) return LOG_STATUS( Status_IOError("Cannot read from file; Read exceeds file size")); @@ -438,17 +441,15 @@ Status Posix::read( return st; } -Status Posix::sync(const std::string& path) { - uint32_t permissions = 0; +Status Posix::sync(const URI& uri) { + auto path = uri.to_path(); // Open file int fd = -1; - if (is_dir(path)) { // DIRECTORY - RETURN_NOT_OK(get_posix_directory_permissions(&permissions)); - fd = open(path.c_str(), O_RDONLY, permissions); - } else if (is_file(path)) { // FILE - RETURN_NOT_OK(get_posix_file_permissions(&permissions)); - fd = open(path.c_str(), O_WRONLY | O_APPEND | O_CREAT, permissions); + if (is_dir(URI(path))) { // DIRECTORY + fd = open(path.c_str(), O_RDONLY, directory_permissions_); + } else if (is_file(URI(path))) { // FILE + fd = open(path.c_str(), O_WRONLY | O_APPEND | O_CREAT, file_permissions_); } else return Status_Ok(); // If file does not exist, exit @@ -477,7 +478,11 @@ Status Posix::sync(const std::string& path) { } Status Posix::write( - const std::string& path, const void* buffer, uint64_t buffer_size) { + const URI& uri, + const void* buffer, + uint64_t buffer_size, + [[maybe_unused]] bool remote_global_order_write) { + auto path = uri.to_path(); // Check for valid inputs before attempting the actual // write system call. This is to avoid a bug on macOS // Ventura 13.0 on Apple's M1 processors. @@ -492,14 +497,11 @@ Status Posix::write( } } - uint32_t permissions = 0; - RETURN_NOT_OK(get_posix_file_permissions(&permissions)); - // Get file offset (equal to file size) Status st; uint64_t file_offset = 0; - if (is_file(path)) { - st = file_size(path, &file_offset); + if (is_file(URI(path))) { + st = file_size(URI(path), &file_offset); if (!st.ok()) { std::stringstream errmsg; errmsg << "Cannot write to file '" << path << "'; " << st.message(); @@ -508,7 +510,7 @@ Status Posix::write( } // Open or create file. - int fd = open(path.c_str(), O_WRONLY | O_CREAT, permissions); + int fd = open(path.c_str(), O_WRONLY | O_CREAT, file_permissions_); if (fd == -1) { return LOG_STATUS(Status_IOError( std::string("Cannot open file '") + path + "'; " + strerror(errno))); @@ -545,32 +547,6 @@ Status Posix::write_at( return Status::Ok(); } -Status Posix::get_posix_file_permissions(uint32_t* permissions) const { - // Get config params - bool found = false; - std::string posix_permissions = - config_.get().get("vfs.file.posix_file_permissions", &found); - assert(found); - - // Permissions are passed in octal notation by the user - *permissions = std::strtol(posix_permissions.c_str(), NULL, 8); - - return Status::Ok(); -} - -Status Posix::get_posix_directory_permissions(uint32_t* permissions) const { - // Get config params - bool found = false; - std::string posix_permissions = - config_.get().get("vfs.file.posix_directory_permissions", &found); - assert(found); - - // Permissions are passed in octal notation by the user - *permissions = std::strtol(posix_permissions.c_str(), NULL, 8); - - return Status::Ok(); -} - } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/filesystem/posix.h b/tiledb/sm/filesystem/posix.h index 04c41bd80274..c8399980f3d7 100644 --- a/tiledb/sm/filesystem/posix.h +++ b/tiledb/sm/filesystem/posix.h @@ -44,6 +44,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" +#include "tiledb/sm/filesystem/filesystem_base.h" using namespace tiledb::common; @@ -60,13 +61,18 @@ class URI; /** * This class implements the POSIX filesystem functions. */ -class Posix { +class Posix : public FilesystemBase { public: + /** Default constructor. */ + Posix() + : Posix(Config()) { + } + /** Constructor. */ - Posix(); + explicit Posix(const Config& config); /** Destructor. */ - ~Posix() = default; + ~Posix() override = default; /** * Returns the absolute posix (string) path of the input in the @@ -80,7 +86,7 @@ class Posix { * @param dir The name of the directory to be created. * @return Status */ - Status create_dir(const std::string& path) const; + Status create_dir(const URI& uri) const override; /** * Creates an empty file. @@ -88,7 +94,7 @@ class Posix { * @param filename The name of the file to be created. * @return Status */ - Status touch(const std::string& filename) const; + Status touch(const URI& uri) const override; /** * Returns the directory where the program is executed. @@ -105,9 +111,7 @@ class Posix { * @param path The path of the directory to be deleted. * @return Status */ - Status remove_dir(const std::string& path) const; - - /** Deletes the file in the input path. */ + Status remove_dir(const URI& path) const override; /** * Removes a given path. @@ -115,7 +119,7 @@ class Posix { * @param path The path of the file / empty directory to be deleted. * @return Status */ - Status remove_file(const std::string& path) const; + Status remove_file(const URI& path) const override; /** * Returns the size of the input file. @@ -124,15 +128,7 @@ class Posix { * @param nbytes Pointer to a value * @return Status */ - Status file_size(const std::string& path, uint64_t* size) const; - - /** - * Initialize this instance with the given config. - * - * @param config Config parameters. - * @return Status - */ - Status init(const Config& config); + Status file_size(const URI& path, uint64_t* size) const override; /** * Checks if the input is an existing directory. @@ -140,7 +136,7 @@ class Posix { * @param dir The directory to be checked. * @return *True* if *dir* is an existing directory, and *False* otherwise. */ - bool is_dir(const std::string& path) const; + bool is_dir(const URI& uri) const override; /** * Checks if the input is an existing file. @@ -148,7 +144,7 @@ class Posix { * @param file The file to be checked. * @return *True* if *file* is an existing file, and *false* otherwise. */ - bool is_file(const std::string& path) const; + bool is_file(const URI& uri) const override; /** * @@ -168,34 +164,37 @@ class Posix { * @return A list of directory_entry objects */ tuple>> - ls_with_sizes(const URI& uri) const; + ls_with_sizes(const URI& uri) const override; /** * Move a given filesystem path. + * Both URI must be of the same file:// backend type. * * @param old_path The old path. * @param new_path The new path. * @return Status */ - Status move_path(const std::string& old_path, const std::string& new_path); + Status move_file(const URI& old_path, const URI& new_path) const override; /** * Copy a given filesystem file. + * Both URI must be of the same file:// backend type. * * @param old_path The old path. * @param new_path The new path. * @return Status */ - Status copy_file(const std::string& old_path, const std::string& new_path); + Status copy_file(const URI& old_uri, const URI& new_uri) const override; /** * Copy a given filesystem directory. + * Both URI must be of the same file:// backend type. * * @param old_path The old path. * @param new_path The new path. * @return Status */ - Status copy_dir(const std::string& old_path, const std::string& new_path); + Status copy_dir(const URI& old_path, const URI& new_path) const override; /** * Reads data from a file into a buffer. @@ -207,10 +206,11 @@ class Posix { * @return Status. */ Status read( - const std::string& path, + const URI& uri, uint64_t offset, void* buffer, - uint64_t nbytes) const; + uint64_t nbytes, + bool use_read_ahead = true) override; /** * Syncs a file or directory. @@ -218,7 +218,7 @@ class Posix { * @param path The name of the file. * @return Status */ - Status sync(const std::string& path); + Status sync(const URI& uri) override; /** * Writes the input buffer to a file. @@ -232,15 +232,12 @@ class Posix { * @return Status */ Status write( - const std::string& path, const void* buffer, uint64_t buffer_size); + const URI& uri, + const void* buffer, + uint64_t buffer_size, + bool remote_global_order_write = false) override; private: - /** Default config. */ - Config default_config_; - - /** Config parameters inherited from parent VFS. */ - std::reference_wrapper config_; - static void adjacent_slashes_dedup(std::string* path); static bool both_slashes(char a, char b); @@ -292,19 +289,8 @@ class Posix { static Status write_at( int fd, uint64_t file_offset, const void* buffer, uint64_t buffer_size); - /** - * Parse config to get posix permissions for creating new files - * @param permissions parsed permissions are set to this parameter - * @return Status - */ - Status get_posix_file_permissions(uint32_t* permissions) const; - - /** - * Parse config to get posix permissions for creating new directories - * @param permissions parsed permissions are set to this parameter - * @return Status - */ - Status get_posix_directory_permissions(uint32_t* permissions) const; + private: + uint32_t file_permissions_, directory_permissions_; }; } // namespace sm diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 9dffdb328e40..70a9d030141c 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -108,7 +108,7 @@ VFS::VFS( #ifdef _WIN32 throw_if_not_ok(win_.init(config_)); #else - throw_if_not_ok(posix_.init(config_)); + posix_ = Posix(config_); #endif supported_fs_.insert(Filesystem::MEMFS); @@ -164,7 +164,7 @@ Status VFS::create_dir(const URI& uri) const { #ifdef _WIN32 return win_.create_dir(uri.to_path()); #else - return posix_.create_dir(uri.to_path()); + return posix_.create_dir(uri); #endif } if (uri.is_hdfs()) { @@ -242,7 +242,7 @@ Status VFS::touch(const URI& uri) const { #ifdef _WIN32 return win_.touch(uri.to_path()); #else - return posix_.touch(uri.to_path()); + return posix_.touch(uri); #endif } if (uri.is_hdfs()) { @@ -400,7 +400,7 @@ Status VFS::remove_dir(const URI& uri) const { #ifdef _WIN32 return win_.remove_dir(uri.to_path()); #else - return posix_.remove_dir(uri.to_path()); + return posix_.remove_dir(uri); #endif } else if (uri.is_hdfs()) { #ifdef HAVE_HDFS @@ -452,7 +452,7 @@ Status VFS::remove_file(const URI& uri) const { #ifdef _WIN32 return win_.remove_file(uri.to_path()); #else - return posix_.remove_file(uri.to_path()); + return posix_.remove_file(uri); #endif } if (uri.is_hdfs()) { @@ -539,7 +539,7 @@ Status VFS::file_size(const URI& uri, uint64_t* size) const { #ifdef _WIN32 return win_.file_size(uri.to_path(), size); #else - return posix_.file_size(uri.to_path(), size); + return posix_.file_size(uri, size); #endif } if (uri.is_hdfs()) { @@ -583,7 +583,7 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { #ifdef _WIN32 *is_dir = win_.is_dir(uri.to_path()); #else - *is_dir = posix_.is_dir(uri.to_path()); + *is_dir = posix_.is_dir(uri); #endif return Status::Ok(); } @@ -634,7 +634,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { #ifdef _WIN32 *is_file = win_.is_file(uri.to_path()); #else - *is_file = posix_.is_file(uri.to_path()); + *is_file = posix_.is_file(uri); #endif return Status::Ok(); } @@ -733,7 +733,7 @@ tuple>> VFS::ls_with_sizes( // return an empty `uris` vector. if (!(parent.is_s3() || parent.is_gcs() || parent.is_azure())) { bool flag = false; - RETURN_NOT_OK_TUPLE(is_dir(parent, &flag), nullopt); + RETURN_NOT_OK_TUPLE(this->is_dir(parent, &flag), nullopt); if (!flag) { return {Status::Ok(), std::vector()}; @@ -820,7 +820,7 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef _WIN32 return win_.move_path(old_uri.to_path(), new_uri.to_path()); #else - return posix_.move_path(old_uri.to_path(), new_uri.to_path()); + return posix_.move_file(old_uri, new_uri); #endif } return LOG_STATUS(Status_VFSError( @@ -900,7 +900,7 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef _WIN32 return win_.move_path(old_uri.to_path(), new_uri.to_path()); #else - return posix_.move_path(old_uri.to_path(), new_uri.to_path()); + return posix_.move_file(old_uri, new_uri); #endif } return LOG_STATUS(Status_VFSError( @@ -987,7 +987,7 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { return LOG_STATUS(Status_IOError( std::string("Copying files on Windows is not yet supported."))); #else - return posix_.copy_file(old_uri.to_path(), new_uri.to_path()); + return posix_.copy_file(old_uri, new_uri); #endif } return LOG_STATUS(Status_VFSError( @@ -1062,7 +1062,7 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { return LOG_STATUS(Status_IOError( std::string("Copying directories on Windows is not yet supported."))); #else - return posix_.copy_dir(old_uri.to_path(), new_uri.to_path()); + return posix_.copy_dir(old_uri, new_uri); #endif } return LOG_STATUS(Status_VFSError( @@ -1206,7 +1206,7 @@ Status VFS::read_impl( #ifdef _WIN32 return win_.read(uri.to_path(), offset, buffer, nbytes); #else - return posix_.read(uri.to_path(), offset, buffer, nbytes); + return posix_.read(uri, offset, buffer, nbytes, false); #endif } if (uri.is_hdfs()) { @@ -1361,7 +1361,7 @@ Status VFS::sync(const URI& uri) { #ifdef _WIN32 return win_.sync(uri.to_path()); #else - return posix_.sync(uri.to_path()); + return posix_.sync(uri); #endif } if (uri.is_hdfs()) { @@ -1457,7 +1457,7 @@ Status VFS::close_file(const URI& uri) { #ifdef _WIN32 return win_.sync(uri.to_path()); #else - return posix_.sync(uri.to_path()); + return posix_.sync(uri); #endif } if (uri.is_hdfs()) { @@ -1521,7 +1521,7 @@ Status VFS::write( #ifdef _WIN32 return win_.write(uri.to_path(), buffer, buffer_size); #else - return posix_.write(uri.to_path(), buffer, buffer_size); + return posix_.write(uri, buffer, buffer_size); #endif } if (uri.is_hdfs()) { diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 2b59d7617b2c..910a020bfb65 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -39,6 +39,7 @@ #include #include +#include "filesystem_base.h" #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/macros.h" From 9acc6024895b2dc6e0a62977e4b6cfa128be9739 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 9 Nov 2023 14:23:17 -0800 Subject: [PATCH 064/456] Add include array to cpp_api type.h (#4504) The file `type.h` in `tiledb/sm/cpp_api` used `std::array` but does not `#include `. This causes failure when compiling with `clang++-14` on ubuntu. Added `#include ` to `tiledb/sm/cpp_api/type.h`. --- TYPE: BUG DESC: Added `#include ` to `tiledb/sm/cpp_api/type.h`. --- tiledb/sm/cpp_api/type.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tiledb/sm/cpp_api/type.h b/tiledb/sm/cpp_api/type.h index 2642837fe42d..8f5edaee9eca 100644 --- a/tiledb/sm/cpp_api/type.h +++ b/tiledb/sm/cpp_api/type.h @@ -37,6 +37,7 @@ #include "tiledb.h" +#include #include #include #include From cb8c5c22ce0789906a05587a4684e750f3f6fa56 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Fri, 10 Nov 2023 03:06:41 -0600 Subject: [PATCH 065/456] Remove extraneous TILEDB_EXPORT tags (#4507) --- TYPE: IMPROVEMENT DESC: Remove extraneous TILEDB_EXPORT tags --- tiledb/sm/c_api/tiledb.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index cad0bd29c9b3..940cad49d96e 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -382,7 +382,7 @@ int32_t tiledb_array_schema_timestamp_range( return TILEDB_OK; } -TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( +int32_t tiledb_array_schema_add_enumeration( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_enumeration_t* enumeration) { @@ -2050,7 +2050,7 @@ int32_t tiledb_query_condition_alloc( return TILEDB_OK; } -TILEDB_EXPORT capi_return_t tiledb_query_condition_alloc_set_membership( +capi_return_t tiledb_query_condition_alloc_set_membership( const char* field_name, const void* data, uint64_t data_size, From a648a7e7eaff7e9380c5b58f9675e96049344ae5 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 10 Nov 2023 14:10:23 +0100 Subject: [PATCH 066/456] Groups: don't try to load metadata on client open requests. (#4508) This change makes sure that the client doesn't try to load the group metadata client side when doing a REST request that will end up returning the group metadata. This would actually end up causing two REST requests, one to the load metadata route, then one to the group open route. --- TYPE: NO_HISTORY DESC: Groups: don't try to load metadata on client open requests. --- tiledb/sm/serialization/group.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index 5e73744c87ba..1aac053fa940 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -166,7 +166,12 @@ Status group_details_to_capnp( } Metadata* metadata; - RETURN_NOT_OK(group->metadata(&metadata)); + if (group->group_uri().is_tiledb()) { + metadata = const_cast(group->metadata()); + } else { + RETURN_NOT_OK(group->metadata(&metadata)); + } + if (metadata->num()) { auto group_metadata_builder = group_details_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &group_metadata_builder)); From 3a4ed9c4d250265452f1c475ef1f77868e820f99 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 13 Nov 2023 15:14:28 +0200 Subject: [PATCH 067/456] Use `find_package` for the compression libraries when vcpkg is enabled. (#4500) All of our compression libraries provide native CMake integration with `find_package`. This PR makes use of it when vcpkg is enabled. --- TYPE: BUILD DESC: Use the provided CMake packages to import the compression libraries when vcpkg is enabled. --- cmake/Modules/FindBzip2_EP.cmake | 16 +++++++++----- cmake/Modules/FindLZ4_EP.cmake | 16 +++++++++----- cmake/Modules/FindZstd_EP.cmake | 22 ++++++++++++++----- .../cpp_api/CMakeLists.txt | 2 +- tiledb/CMakeLists.txt | 12 +++++----- tiledb/sm/compressors/CMakeLists.txt | 2 +- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/cmake/Modules/FindBzip2_EP.cmake b/cmake/Modules/FindBzip2_EP.cmake index ad42d31a13c3..8c5f828491b7 100644 --- a/cmake/Modules/FindBzip2_EP.cmake +++ b/cmake/Modules/FindBzip2_EP.cmake @@ -29,11 +29,17 @@ # - BZIP2_INCLUDE_DIR, directory containing headers # - BZIP2_LIBRARIES, the Bzip2 library path # - BZIP2_FOUND, whether Bzip2 has been found -# - The Bzip2::Bzip2 imported target +# - The BZip2::BZip2 imported target # Include some common helper functions. include(TileDBCommon) +if(TILEDB_VCPKG) + find_package(BZip2 REQUIRED) + install_target_libs(BZip2::BZip2) + return() +endif() + # First check for a static version in the EP prefix. find_library(BZIP2_LIBRARIES NAMES @@ -128,9 +134,9 @@ if (NOT BZIP2_FOUND) endif() endif() -if (BZIP2_FOUND AND NOT TARGET Bzip2::Bzip2) - add_library(Bzip2::Bzip2 UNKNOWN IMPORTED) - set_target_properties(Bzip2::Bzip2 PROPERTIES +if (BZIP2_FOUND AND NOT TARGET BZip2::BZip2) + add_library(BZip2::BZip2 UNKNOWN IMPORTED) + set_target_properties(BZip2::BZip2 PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}" ) @@ -138,5 +144,5 @@ endif() # If we built a static EP, install it if required. if (BZIP2_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Bzip2::Bzip2) + install_target_libs(BZip2::BZip2) endif() diff --git a/cmake/Modules/FindLZ4_EP.cmake b/cmake/Modules/FindLZ4_EP.cmake index a8ca7aa63ff7..efdabea8a1a8 100644 --- a/cmake/Modules/FindLZ4_EP.cmake +++ b/cmake/Modules/FindLZ4_EP.cmake @@ -29,11 +29,17 @@ # - LZ4_INCLUDE_DIR, directory containing headers # - LZ4_LIBRARIES, the LZ4 library path # - LZ4_FOUND, whether LZ4 has been found -# - The LZ4::LZ4 imported target +# - The lz4::lz4 imported target # Include some common helper functions. include(TileDBCommon) +if(TILEDB_VCPKG) + find_package(lz4 REQUIRED) + install_target_libs(lz4::lz4) + return() +endif() + # First check for a static version in the EP prefix. find_library(LZ4_LIBRARIES NAMES @@ -116,9 +122,9 @@ endif() set(ignoreUnusedWarning "${TILEDB_LZ4_EP_BUILT}") -if (LZ4_FOUND AND NOT TARGET LZ4::LZ4) - add_library(LZ4::LZ4 UNKNOWN IMPORTED) - set_target_properties(LZ4::LZ4 PROPERTIES +if (LZ4_FOUND AND NOT TARGET lz4::lz4) + add_library(lz4::lz4 UNKNOWN IMPORTED) + set_target_properties(lz4::lz4 PROPERTIES IMPORTED_LOCATION "${LZ4_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" ) @@ -126,5 +132,5 @@ endif() # If we built a static EP, install it if required. if (LZ4_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(LZ4::LZ4) + install_target_libs(lz4::lz4) endif() diff --git a/cmake/Modules/FindZstd_EP.cmake b/cmake/Modules/FindZstd_EP.cmake index 64ca6fbcc823..a4736afe7d0b 100644 --- a/cmake/Modules/FindZstd_EP.cmake +++ b/cmake/Modules/FindZstd_EP.cmake @@ -30,11 +30,22 @@ # - ZSTD_INCLUDE_DIR, directory containing headers # - ZSTD_LIBRARIES, the Zstd library path # - ZSTD_FOUND, whether Zstd has been found -# - The Zstd::Zstd imported target +# - The ${ZSTD_TARGET} imported target # Include some common helper functions. include(TileDBCommon) +if (TILEDB_VCPKG) + find_package(zstd CONFIG REQUIRED) + if (TARGET zstd::libzstd_static) + set(ZSTD_TARGET zstd::libzstd_static) + else() + set(ZSTD_TARGET zstd::libzstd_shared) + endif() + install_target_libs(${ZSTD_TARGET}) + return() +endif() + # First check for a static version in the EP prefix. find_library(ZSTD_LIBRARIES NAMES @@ -118,15 +129,16 @@ if (NOT ZSTD_FOUND) endif() endif() -if (ZSTD_FOUND AND NOT TARGET Zstd::Zstd) - add_library(Zstd::Zstd UNKNOWN IMPORTED) - set_target_properties(Zstd::Zstd PROPERTIES +if (ZSTD_FOUND AND NOT ZSTD_TARGET) + add_library(zstd::libzstd UNKNOWN IMPORTED) + set_target_properties(zstd::libzstd PROPERTIES IMPORTED_LOCATION "${ZSTD_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" ) + set(ZSTD_TARGET zstd::libzstd) endif() # If we built a static EP, install it if required. if (ZSTD_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Zstd::Zstd) + install_target_libs(${ZSTD_TARGET}) endif() diff --git a/experimental/experimental_examples/cpp_api/CMakeLists.txt b/experimental/experimental_examples/cpp_api/CMakeLists.txt index 5dc968536489..8da6d16d2d2c 100644 --- a/experimental/experimental_examples/cpp_api/CMakeLists.txt +++ b/experimental/experimental_examples/cpp_api/CMakeLists.txt @@ -33,7 +33,7 @@ find_package(Bzip2_EP REQUIRED) function(build_TileDB_experimental_example_cppapi TARGET) add_executable(${TARGET}_exp_cpp EXCLUDE_FROM_ALL ${TARGET}.cc) target_link_libraries(${TARGET}_exp_cpp PUBLIC local_install tiledb_shared) - target_link_libraries(${TARGET}_exp_cpp PUBLIC Bzip2::Bzip2) + target_link_libraries(${TARGET}_exp_cpp PUBLIC BZip2::BZip2) if (NOT WIN32) # On Linux, must explicitly link -lpthread -ldl in order for static linking # to libzstd or libcurl to work. diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index c5be9feeddc9..24c583618199 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -695,10 +695,10 @@ find_package(Zstd_EP REQUIRED) find_package(Magic_EP REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE - Bzip2::Bzip2 - LZ4::LZ4 + BZip2::BZip2 + lz4::lz4 ZLIB::ZLIB - Zstd::Zstd + ${ZSTD_TARGET} libmagic ) if (TILEDB_VCPKG) @@ -988,10 +988,10 @@ if (TILEDB_STATIC) endif() endmacro() - append_dep_lib(Bzip2::Bzip2) - append_dep_lib(LZ4::LZ4) + append_dep_lib(BZip2::BZip2) + append_dep_lib(lz4::lz4) append_dep_lib(ZLIB::ZLIB) - append_dep_lib(Zstd::Zstd) + append_dep_lib(${ZSTD_TARGET}) append_dep_lib(CapnProto::capnp) append_dep_lib(CapnProto::capnp-json) append_dep_lib(CapnProto::kj) diff --git a/tiledb/sm/compressors/CMakeLists.txt b/tiledb/sm/compressors/CMakeLists.txt index c690d7049279..8ebe27f5102d 100644 --- a/tiledb/sm/compressors/CMakeLists.txt +++ b/tiledb/sm/compressors/CMakeLists.txt @@ -40,7 +40,7 @@ commence(object_library compressors) find_package(LZ4_EP REQUIRED) find_package(Zlib_EP REQUIRED) find_package(Zstd_EP REQUIRED) - this_target_link_libraries(Bzip2::Bzip2 LZ4::LZ4 ZLIB::ZLIB Zstd::Zstd) + this_target_link_libraries(BZip2::BZip2 lz4::lz4 ZLIB::ZLIB ${ZSTD_TARGET}) conclude(object_library) # From 540db914c2f3f913d3b58016d33829b17b5108c7 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:44:38 +0100 Subject: [PATCH 068/456] Groups: add the ability to delete by URI back. (#4512) This breaking change needs to be reverted because it breaks cloud. --- TYPE: NO_HISTORY DESC: Groups: add the ability to delete by URI back. --- test/src/unit-capi-group.cc | 46 +++-- test/src/unit-cppapi-group.cc | 126 +++++++++++- tiledb/api/c_api/group/group_api.cc | 6 +- .../group/group_api_external_experimental.h | 12 +- tiledb/sm/cpp_api/group_experimental.h | 12 +- tiledb/sm/group/group.cc | 15 +- tiledb/sm/group/group_details.cc | 181 ++++++++++++------ tiledb/sm/group/group_details.h | 63 +++--- tiledb/sm/group/group_details_v1.cc | 6 +- tiledb/sm/group/group_details_v1.h | 8 +- tiledb/sm/group/group_details_v2.cc | 9 +- tiledb/sm/group/group_details_v2.h | 10 +- 12 files changed, 351 insertions(+), 143 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 2f32c87a00fd..25632aa9720f 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -818,6 +818,8 @@ TEST_CASE_METHOD( GroupFx, "C API: Group, write/read with named members", "[capi][group][metadata][read]") { + bool remove_by_name = GENERATE(true, false); + // Create and open group in write mode // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); @@ -929,19 +931,25 @@ TEST_CASE_METHOD( rc = tiledb_group_open(ctx_, group2, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); - // The member has a name; removing it by URI should fail. - rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); - REQUIRE(rc != TILEDB_OK); - rc = tiledb_group_remove_member(ctx_, group1, "four"); + // Remove by bame or URI. + if (remove_by_name) { + rc = tiledb_group_remove_member(ctx_, group1, "four"); + } else { + rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); + } REQUIRE(rc == TILEDB_OK); + // Group is the latest element group1_expected.resize(group1_expected.size() - 1); - // The member has a name; removing it by URI should fail. - rc = tiledb_group_remove_member(ctx_, group2, array3_uri.c_str()); - REQUIRE(rc != TILEDB_OK); - rc = tiledb_group_remove_member(ctx_, group2, "three"); + // Remove by bame or URI. + if (remove_by_name) { + rc = tiledb_group_remove_member(ctx_, group2, "three"); + } else { + rc = tiledb_group_remove_member(ctx_, group2, array3_uri.c_str()); + } REQUIRE(rc == TILEDB_OK); + // There should be nothing left in group2 group2_expected.clear(); @@ -1311,6 +1319,8 @@ TEST_CASE_METHOD( GroupFx, "C API: Group, write/read, relative with named members", "[capi][group][metadata][read]") { + bool remove_by_name = GENERATE(true, false); + // Create and open group in write mode // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); @@ -1435,18 +1445,22 @@ TEST_CASE_METHOD( rc = tiledb_group_open(ctx_, group2, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); - // The member has a name; removing it by URI should fail. - rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); - REQUIRE(rc != TILEDB_OK); - rc = tiledb_group_remove_member(ctx_, group1, "eight"); + // Remove by bame or URI. + if (remove_by_name) { + rc = tiledb_group_remove_member(ctx_, group1, "eight"); + } else { + rc = tiledb_group_remove_member(ctx_, group1, group2_uri.c_str()); + } REQUIRE(rc == TILEDB_OK); // Group is the latest element group1_expected.resize(group1_expected.size() - 1); - // The member has a name; removing it by URI should fail. - rc = tiledb_group_remove_member(ctx_, group2, array3_relative_uri.c_str()); - REQUIRE(rc != TILEDB_OK); - rc = tiledb_group_remove_member(ctx_, group2, "seven"); + // Remove by bame or URI. + if (remove_by_name) { + rc = tiledb_group_remove_member(ctx_, group2, "seven"); + } else { + rc = tiledb_group_remove_member(ctx_, group2, array3_relative_uri.c_str()); + } REQUIRE(rc == TILEDB_OK); // There should be nothing left in group2 group2_expected.clear(); diff --git a/test/src/unit-cppapi-group.cc b/test/src/unit-cppapi-group.cc index d0169de0b856..acb63ce10d26 100644 --- a/test/src/unit-cppapi-group.cc +++ b/test/src/unit-cppapi-group.cc @@ -690,6 +690,8 @@ TEST_CASE_METHOD( GroupCPPFx, "C++ API: Group, write/read, relative named", "[cppapi][group][read]") { + bool remove_by_name = GENERATE(true, false); + // Create and open group in write mode // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); @@ -788,13 +790,21 @@ TEST_CASE_METHOD( set_group_timestamp(&group2, 2); group2.open(TILEDB_WRITE); - REQUIRE_THROWS(group1.remove_member(group2_uri.to_string())); - group1.remove_member("three"); + if (remove_by_name) { + group1.remove_member("three"); + } else { + group1.remove_member(group2_uri.to_string()); + } + // Group is the latest element group1_expected.resize(group1_expected.size() - 1); - REQUIRE_THROWS(group2.remove_member(array3_relative_uri)); - group2.remove_member("four"); + if (remove_by_name) { + group2.remove_member("four"); + } else { + group2.remove_member(array3_relative_uri); + } + // There should be nothing left in group2 group2_expected.clear(); @@ -822,6 +832,114 @@ TEST_CASE_METHOD( remove_temp_dir(temp_dir); } +TEST_CASE_METHOD( + GroupCPPFx, + "C++ API: Group, delete by URI, duplicates", + "[cppapi][group][delete]") { + bool nameless_uri = GENERATE(true, false); + + // Create and open group in write mode + // TODO: refactor for each supported FS. + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + + tiledb::sm::URI group1_uri(temp_dir + "group1"); + tiledb::Group::create(ctx_, group1_uri.to_string()); + + REQUIRE( + tiledb_vfs_create_dir( + ctx_.ptr().get(), vfs_, (temp_dir + "group1/arrays").c_str()) == + TILEDB_OK); + + const std::string array1_relative_uri("arrays/array1"); + const tiledb::sm::URI array1_uri(temp_dir + "group1/arrays/array1"); + const std::string array2_relative_uri("arrays/array2"); + const tiledb::sm::URI array2_uri(temp_dir + "group1/arrays/array2"); + create_array(array1_uri.to_string()); + create_array(array2_uri.to_string()); + + // Set expected + std::vector group1_expected = { + tiledb::Object( + tiledb::Object::Type::Array, array1_uri.to_string(), "one"), + tiledb::Object( + tiledb::Object::Type::Array, array2_uri.to_string(), "two"), + nameless_uri ? + tiledb::Object( + tiledb::Object::Type::Array, array2_uri.to_string(), nullopt) : + tiledb::Object( + tiledb::Object::Type::Array, array2_uri.to_string(), "three"), + }; + + tiledb::Group group1(ctx_, group1_uri.to_string(), TILEDB_WRITE); + group1.close(); + set_group_timestamp(&group1, 1); + group1.open(TILEDB_WRITE); + + group1.add_member(array1_relative_uri, true, "one"); + group1.add_member(array2_relative_uri, true, "two"); + group1.add_member( + array2_relative_uri, + true, + nameless_uri ? nullopt : std::make_optional("three")); + + // Close group from write mode + group1.close(); + + // Reopen in read mode + group1.open(TILEDB_READ); + + std::vector group1_received = read_group(group1); + REQUIRE_THAT( + group1_received, Catch::Matchers::UnorderedEquals(group1_expected)); + + bool is_relative; + is_relative = group1.is_relative("one"); + REQUIRE(is_relative == true); + is_relative = group1.is_relative("two"); + REQUIRE(is_relative == true); + + if (!nameless_uri) { + is_relative = group1.is_relative("three"); + REQUIRE(is_relative == true); + } + + // Close group + group1.close(); + + // Remove assets from group + set_group_timestamp(&group1, 2); + group1.open(TILEDB_WRITE); + if (nameless_uri) { + group1.remove_member(array2_relative_uri); + } else { + CHECK_THROWS_WITH( + group1.remove_member(array2_relative_uri), + Catch::Matchers::ContainsSubstring( + "there are multiple members with the " + "same URI, please remove by name.")); + group1.remove_member("three"); + } + + // Group is the latest element + group1_expected.resize(group1_expected.size() - 1); + + // Close group + group1.close(); + + // Check read again + set_group_timestamp(&group1, 2); + group1.open(TILEDB_READ); + + group1_received = read_group(group1); + REQUIRE_THAT( + group1_received, Catch::Matchers::UnorderedEquals(group1_expected)); + + // Close group + group1.close(); + remove_temp_dir(temp_dir); +} + /** Test Exception For Assertability */ class GroupDtorDoesntThrowException : public std::exception { public: diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 4d6e374bcb17..0191f6acad7e 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -277,11 +277,11 @@ capi_return_t tiledb_group_add_member( } capi_return_t tiledb_group_remove_member( - tiledb_group_handle_t* group, const char* name) { + tiledb_group_handle_t* group, const char* name_or_uri) { ensure_group_is_valid(group); - ensure_name_argument_is_valid(name); + ensure_name_argument_is_valid(name_or_uri); - throw_if_not_ok(group->group().mark_member_for_removal(name)); + throw_if_not_ok(group->group().mark_member_for_removal(name_or_uri)); return TILEDB_OK; } diff --git a/tiledb/api/c_api/group/group_api_external_experimental.h b/tiledb/api/c_api/group/group_api_external_experimental.h index 1d5fdc81341c..8ddcbf89ac6d 100644 --- a/tiledb/api/c_api/group/group_api_external_experimental.h +++ b/tiledb/api/c_api/group/group_api_external_experimental.h @@ -366,13 +366,17 @@ TILEDB_EXPORT capi_return_t tiledb_group_add_member( * * @param ctx The TileDB context. * @param group An group opened in WRITE mode. - * @param name Name of member to remove. If the member has no name, this - * parameter should be set to the URI of the member. In that case, only the - * unnamed member with the given URI will be removed. + * @param name_or_uri Name or URI of member to remove. If the URI is + * registered multiple times in the group, the name needs to be specified so + * that the correct one can be removed. Note that if a URI is registered as + * both a named and unnamed member, the unnamed member will be removed + * successfully using the URI. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ TILEDB_EXPORT capi_return_t tiledb_group_remove_member( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* name) TILEDB_NOEXCEPT; + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name_or_uri) TILEDB_NOEXCEPT; /** * Get the count of members in a group diff --git a/tiledb/sm/cpp_api/group_experimental.h b/tiledb/sm/cpp_api/group_experimental.h index daae9387a2a3..f7bb3aa5601e 100644 --- a/tiledb/sm/cpp_api/group_experimental.h +++ b/tiledb/sm/cpp_api/group_experimental.h @@ -402,15 +402,17 @@ class Group { /** * Remove a member from a group * - * @param name Name of member to remove. If the member has no name, - * this parameter should be set to the URI of the member. In that case, only - * the unnamed member with the given URI will be removed. + * @param name_or_uri Name or URI of member to remove. If the URI is + * registered multiple times in the group, the name needs to be specified so + * that the correct one can be removed. Note that if a URI is registered as + * both a named and unnamed member, the unnamed member will be removed + * successfully using the URI. */ - void remove_member(const std::string& name) { + void remove_member(const std::string& name_or_uri) { auto& ctx = ctx_.get(); tiledb_ctx_t* c_ctx = ctx.ptr().get(); ctx.handle_error( - tiledb_group_remove_member(c_ctx, group_.get(), name.c_str())); + tiledb_group_remove_member(c_ctx, group_.get(), name_or_uri.c_str())); } uint64_t member_count() const { diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 5f16cb1b9c29..f519460fd422 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -253,10 +253,10 @@ Status Group::close() { try { // If changes haven't been applied, apply them if (!changes_applied_) { - RETURN_NOT_OK(group_details_->apply_pending_changes()); + group_details_->apply_pending_changes(); changes_applied_ = group_details_->changes_applied(); } - RETURN_NOT_OK(storage_manager_->group_close_for_writes(this)); + throw_if_not_ok(storage_manager_->group_close_for_writes(this)); } catch (StatusException& exc) { std::string msg = exc.what(); msg += " : Was storage for the group moved or deleted before closing?"; @@ -340,7 +340,7 @@ void Group::delete_group(const URI& uri, bool recursive) { } // Clear metadata and other pending changes to avoid patching a deleted group. metadata_.clear(); - throw_if_not_ok(group_details_->clear()); + group_details_->clear(); // Close the deleted group throw_if_not_ok(this->close()); @@ -521,7 +521,8 @@ void Group::set_config(Config config) { } Status Group::clear() { - return group_details_->clear(); + group_details_->clear(); + return Status::Ok(); } void Group::add_member(const shared_ptr group_member) { @@ -551,8 +552,9 @@ Status Group::mark_member_for_addition( "Cannot get member; Group was not opened in write or modify_exclusive " "mode"); } - return group_details_->mark_member_for_addition( + group_details_->mark_member_for_addition( group_member_uri, relative, name, storage_manager_); + return Status::Ok(); } Status Group::mark_member_for_removal(const std::string& name) { @@ -571,7 +573,8 @@ Status Group::mark_member_for_removal(const std::string& name) { "mode"); } - return group_details_->mark_member_for_removal(name); + group_details_->mark_member_for_removal(name); + return Status::Ok(); } const std::vector>& Group::members_to_modify() const { diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index 65f9e091d5ec..1338521e9be0 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -57,15 +57,12 @@ GroupDetails::GroupDetails(const URI& group_uri, uint32_t version) , changes_applied_(false) { } -Status GroupDetails::clear() { +void GroupDetails::clear() { members_.clear(); - members_by_name_.clear(); - members_vec_.clear(); + invalidate_lookups(); members_to_modify_.clear(); member_keys_to_add_.clear(); member_keys_to_delete_.clear(); - - return Status::Ok(); } void GroupDetails::add_member(const shared_ptr group_member) { @@ -73,19 +70,17 @@ void GroupDetails::add_member(const shared_ptr group_member) { auto key = group_member->key(); members_[key] = group_member; // Invalidate the lookup tables. - members_by_name_.clear(); - members_vec_.clear(); + invalidate_lookups(); } void GroupDetails::delete_member(const shared_ptr group_member) { std::lock_guard lck(mtx_); members_.erase(group_member->key()); // Invalidate the lookup tables. - members_by_name_.clear(); - members_vec_.clear(); + invalidate_lookups(); } -Status GroupDetails::mark_member_for_addition( +void GroupDetails::mark_member_for_addition( const URI& group_member_uri, const bool& relative, std::optional& name, @@ -97,9 +92,10 @@ Status GroupDetails::mark_member_for_addition( group_uri_.join_path(group_member_uri.to_string()); } ObjectType type = ObjectType::INVALID; - RETURN_NOT_OK(storage_manager->object_type(absolute_group_member_uri, &type)); + throw_if_not_ok( + storage_manager->object_type(absolute_group_member_uri, &type)); if (type == ObjectType::INVALID) { - return Status_GroupError( + throw GroupDetailsException( "Cannot add group member " + absolute_group_member_uri.to_string() + ", type is INVALID. The member likely does not exist."); } @@ -108,50 +104,89 @@ Status GroupDetails::mark_member_for_addition( HERE(), group_member_uri, type, relative, name, false); if (!member_keys_to_add_.insert(group_member->key()).second) { - return Status_GroupError( + throw GroupDetailsException( "Cannot add group member " + group_member->key() + ", a member with the same name or URI has already been added."); } members_to_modify_.emplace_back(group_member); - - return Status::Ok(); } -Status GroupDetails::mark_member_for_removal(const std::string& name) { +void GroupDetails::mark_member_for_removal(const std::string& name_or_uri) { std::lock_guard lck(mtx_); - auto it = members_.find(name); + // Try to find the member by key. + shared_ptr member_to_delete = nullptr; + auto it = members_.find(name_or_uri); if (it == members_.end()) { // try URI to see if we need to convert the local file to file:// - it = members_.find(URI(name).to_string()); + it = members_.find(URI(name_or_uri).to_string()); } + + // We found the member by key, set the member to delete pointer. if (it != members_.end()) { - auto member_to_delete = make_shared( + member_to_delete = make_shared( it->second->uri(), it->second->type(), it->second->relative(), it->second->name(), true); + } else { + // Try to lookup by URI. + ensure_lookup_by_uri(); + + // Make sure the user cannot delete members by URI when there are more than + // one group member with the same URI. + auto it_dup = duplicated_uris_->find(name_or_uri); + if (it_dup == duplicated_uris_->end()) { + // try URI to see if we need to convert the local file to file:// + it_dup = duplicated_uris_->find(URI(name_or_uri).to_string()); + } + if (it_dup != duplicated_uris_->end()) { + throw GroupDetailsException( + "Cannot remove group member " + name_or_uri + + ", there are multiple members with the same URI, please remove by " + "name."); + } + + // Try to find the member by URI. + auto it_by_url = members_by_uri_->find(name_or_uri); + if (it_by_url == members_by_uri_->end()) { + // try URI to see if we need to convert the local file to file:// + it_by_url = members_by_uri_->find(URI(name_or_uri).to_string()); + } + + // The member was found, set the member to delete pointer. + if (it_by_url != members_by_uri_->end()) { + member_to_delete = make_shared( + it_by_url->second->uri(), + it_by_url->second->type(), + it_by_url->second->relative(), + it_by_url->second->name(), + true); + } + } + // Delete the member, if set. + if (member_to_delete != nullptr) { if (member_keys_to_add_.count(member_to_delete->key()) != 0) { - return Status_GroupError( + throw GroupDetailsException( "Cannot remove group member " + member_to_delete->key() + ", a member with the same name or URI has already been added."); } if (!member_keys_to_delete_.insert(member_to_delete->key()).second) { - return Status_GroupError( + throw GroupDetailsException( "Cannot remove group member " + member_to_delete->key() + ", a member with the same name or URI has already been removed."); } members_to_modify_.emplace_back(member_to_delete); - return Status::Ok(); + } else { + throw GroupDetailsException( + "Cannot remove group member " + name_or_uri + + ", member does not exist in group."); } - return Status_GroupError( - "Cannot remove group member " + name + - ", member does not exist in group."); } const std::vector>& GroupDetails::members_to_modify() @@ -167,7 +202,7 @@ GroupDetails::members() const { } void GroupDetails::serialize(Serializer&) { - throw StatusException(Status_GroupError("Invalid call to Group::serialize")); + throw GroupDetailsException("Invalid call to Group::serialize"); } std::optional> GroupDetails::deserialize( @@ -180,8 +215,8 @@ std::optional> GroupDetails::deserialize( return GroupDetailsV2::deserialize(deserializer, group_uri); } - throw StatusException(Status_GroupError( - "Unsupported group version " + std::to_string(version))); + throw GroupDetailsException( + "Unsupported group version " + std::to_string(version)); } std::optional> GroupDetails::deserialize( @@ -209,22 +244,14 @@ tuple> GroupDetails::member_by_index(uint64_t index) { std::lock_guard lck(mtx_); - if (members_vec_.size() != members_.size()) { - members_vec_.clear(); - members_vec_.reserve(members_.size()); - for (auto& [key, member] : members_) { - members_vec_.emplace_back(member); - } - } - - if (index >= members_vec_.size()) { - throw Status_GroupError( + if (index >= members_.size()) { + throw GroupDetailsException( "index " + std::to_string(index) + " is larger than member count " + - std::to_string(members_vec_.size())); + std::to_string(members_.size())); } - auto member = members_vec_[index]; - + ensure_lookup_by_index(); + auto member = members_vec_->at(index); std::string uri = member->uri().to_string(); if (member->relative()) { uri = group_uri_.join_path(member->uri().to_string()).to_string(); @@ -237,25 +264,12 @@ tuple, bool> GroupDetails::member_by_name(const std::string& name) { std::lock_guard lck(mtx_); - if (members_by_name_.size() != members_.size()) { - members_by_name_.clear(); - members_by_name_.reserve(members_.size()); - for (auto& [key, member] : members_) { - if (member->name().has_value()) { - bool added = - members_by_name_.insert_or_assign(member->name().value(), member) - .second; - // add_member makes sure that the name is unique, so the call to - // insert_or_assign should always add a member. - assert(added); - std::ignore = added; - } - } - } + auto it = members_.find(name); - auto it = members_by_name_.find(name); - if (it == members_by_name_.end()) { - throw Status_GroupError(name + " does not exist in group"); + // If we didn't find the key in the members list or if the found member is a + // nameless member, return as not found. + if (it == members_.end() || !it->second->name().has_value()) { + throw GroupDetailsException(name + " does not exist in group"); } auto member = it->second; @@ -271,5 +285,54 @@ format_version_t GroupDetails::version() const { return version_; } +void GroupDetails::ensure_lookup_by_index() { + // Populate the the member by index lookup if it hasn't been generated. + if (members_vec_ == nullopt) { + members_vec_.emplace(); + members_vec_->reserve(members_.size()); + for (auto& [key, member] : members_) { + members_vec_->emplace_back(member); + } + } +} + +void GroupDetails::ensure_lookup_by_uri() { + // Populate the the member by uri lookup if it hasn't been generated. + if (members_by_uri_ == nullopt) { + if (duplicated_uris_ != nullopt) { + GroupDetailsException("`duplicated_uris_` should not be generated."); + } + + members_by_uri_.emplace(); + duplicated_uris_.emplace(); + for (auto& [key, member] : members_) { + const std::string& uri = member->uri().to_string(); + + // See if the URI is already duplicated. + auto dup_it = duplicated_uris_->find(uri); + if (dup_it != duplicated_uris_->end()) { + dup_it->second++; + continue; + } + + // See if the URI already exists. + auto it = members_by_uri_->find(uri); + if (it != members_by_uri_->end()) { + members_by_uri_->erase(it); + duplicated_uris_->emplace(uri, 2); + continue; + } + + members_by_uri_->emplace(uri, member); + } + } +} + +void GroupDetails::invalidate_lookups() { + members_vec_ = nullopt; + members_by_uri_ = nullopt; + duplicated_uris_ = nullopt; +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index ff5c3edaaf1c..05071d7fcdd6 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -35,7 +35,6 @@ #include -#include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/query_type.h" @@ -58,10 +57,8 @@ class GroupDetails { /** * Clear a group - * - * @return */ - Status clear(); + void clear(); /** * Add a member to a group, this will be flushed to disk on close @@ -69,9 +66,8 @@ class GroupDetails { * @param group_member_uri group member uri * @param relative is this URI relative * @param name optional name for member - * @return Status */ - Status mark_member_for_addition( + void mark_member_for_addition( const URI& group_member_uri, const bool& relative, std::optional& name, @@ -80,12 +76,13 @@ class GroupDetails { /** * Remove a member from a group, this will be flushed to disk on close * - * @param name Name of member to remove. If the member has no name, - * this parameter should be set to the URI of the member. In that case, only - * the unnamed member with the given URI will be removed. - * @return Status + * @param name_or_uri Name or URI of member to remove. If the URI is + * registered multiple times in the group, the name needs to be specified so + * that the correct one can be removed. Note that if a URI is registered as + * both a named and unnamed member, the unnamed member will be removed + * successfully using the URI. */ - Status mark_member_for_removal(const std::string& name); + void mark_member_for_removal(const std::string& name_or_uri); /** * Get the vector of members to modify, used in serialization only @@ -120,7 +117,6 @@ class GroupDetails { * * @param buff The buffer to serialize the data into. * @param version The format spec version. - * @return Status */ virtual void serialize(Serializer& serializer); @@ -129,7 +125,7 @@ class GroupDetails { * * @param deserializer The buffer to deserialize from. * @param version The format spec version. - * @return Status and Attribute + * @return Group detail. */ static std::optional> deserialize( Deserializer& deserializer, const URI& group_uri); @@ -140,7 +136,7 @@ class GroupDetails { * @param deserializers List of buffers for each details file to deserialize * from. * @param version The format spec version. - * @return Status and Attribute + * @return Group detail. */ static std::optional> deserialize( const std::vector>& deserializer, @@ -191,10 +187,8 @@ class GroupDetails { * Apply any pending member additions or removals * * mutates members_ and clears members_to_modify_ - * - * @return Status */ - virtual Status apply_pending_changes() = 0; + virtual void apply_pending_changes() = 0; protected: /* ********************************* */ @@ -204,18 +198,26 @@ class GroupDetails { /** The group URI. */ URI group_uri_; - /** The mapping of all members of this group. This is the canonical store of - * the group's members. The key is the member's key(). */ + /** + * The mapping of all members of this group. This is the canonical store of + * the group's members. The key is the member's key(). + */ std::unordered_map> members_; /** Vector for index based lookup. */ - std::vector> members_vec_; + optional>> members_vec_; + + /** + * Map of members for uri based lookup. Note that if a URI is found more than + * once, it will not be found here, but in `duplicated_uris_`. + */ + optional>> + members_by_uri_; - /** Unordered map for name based lookup. If the member doesn't have a name, - * it will not be in the map. */ - std::unordered_map> members_by_name_; + /** Set of duplicated URIs, with count. */ + optional> duplicated_uris_; - /** Mapping of members slated for adding. */ + /** Mapping of members slated for addition/removal. */ std::vector> members_to_modify_; /** Set of member keys that have been marked for addition. */ @@ -232,6 +234,19 @@ class GroupDetails { /** Were changes applied and is a write is required */ bool changes_applied_; + + /* ********************************* */ + /* PROTECTED METHODS */ + /* ********************************* */ + + /** Ensure we have built lookup table for members by index. */ + void ensure_lookup_by_index(); + + /** Ensure we have built lookup table for members by uri. */ + void ensure_lookup_by_uri(); + + /** Invalidate the built lookup tables. */ + void invalidate_lookups(); }; } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/group/group_details_v1.cc b/tiledb/sm/group/group_details_v1.cc index 78a578bab3b8..06de3313a770 100644 --- a/tiledb/sm/group/group_details_v1.cc +++ b/tiledb/sm/group/group_details_v1.cc @@ -70,7 +70,7 @@ shared_ptr GroupDetailsV1::deserialize( return group; } -Status GroupDetailsV1::apply_pending_changes() { +void GroupDetailsV1::apply_pending_changes() { std::lock_guard lck(mtx_); // Remove members first @@ -94,9 +94,7 @@ Status GroupDetailsV1::apply_pending_changes() { changes_applied_ = !members_to_modify_.empty(); members_to_modify_.clear(); - members_vec_.clear(); - members_by_name_.clear(); - return Status::Ok(); + invalidate_lookups(); } } // namespace sm diff --git a/tiledb/sm/group/group_details_v1.h b/tiledb/sm/group/group_details_v1.h index 8696553790dc..c5881d1c3aa9 100644 --- a/tiledb/sm/group/group_details_v1.h +++ b/tiledb/sm/group/group_details_v1.h @@ -35,7 +35,6 @@ #include -#include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/query_type.h" @@ -63,7 +62,6 @@ class GroupDetailsV1 : public GroupDetails { * Serializes the object members into a binary buffer. * * @param buff The buffer to serialize the data into. - * @return Status */ void serialize(Serializer& serializer) override; @@ -72,7 +70,7 @@ class GroupDetailsV1 : public GroupDetails { * * @param buff The buffer to deserialize from. * @param version The format spec version. - * @return Status and Attribute + * @return Group detail */ static shared_ptr deserialize( Deserializer& deserializer, const URI& group_uri); @@ -82,10 +80,8 @@ class GroupDetailsV1 : public GroupDetails { * Apply any pending member additions or removals * * mutates members_ and clears members_to_modify_; - * - * @return Status */ - Status apply_pending_changes() override; + void apply_pending_changes() override; private: /* Format version for class. */ diff --git a/tiledb/sm/group/group_details_v2.cc b/tiledb/sm/group/group_details_v2.cc index 7ea933225191..cdc60e51d8ce 100644 --- a/tiledb/sm/group/group_details_v2.cc +++ b/tiledb/sm/group/group_details_v2.cc @@ -105,12 +105,13 @@ shared_ptr GroupDetailsV2::deserialize( return group; } -Status GroupDetailsV2::apply_pending_changes() { +void GroupDetailsV2::apply_pending_changes() { std::lock_guard lck(mtx_); + // In groups V2, we use `members_` to write the list of additions/removals to + // disk. Start with a fresh list. members_.clear(); - members_vec_.clear(); - members_by_name_.clear(); + invalidate_lookups(); // First add each member to unordered map, overriding if the user adds/removes // it multiple times @@ -120,8 +121,6 @@ Status GroupDetailsV2::apply_pending_changes() { changes_applied_ = !members_to_modify_.empty(); members_to_modify_.clear(); - - return Status::Ok(); } } // namespace sm diff --git a/tiledb/sm/group/group_details_v2.h b/tiledb/sm/group/group_details_v2.h index 70abb7e3084c..71e70ae9aa43 100644 --- a/tiledb/sm/group/group_details_v2.h +++ b/tiledb/sm/group/group_details_v2.h @@ -35,7 +35,6 @@ #include -#include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/query_type.h" @@ -64,7 +63,6 @@ class GroupDetailsV2 : public GroupDetails { * Serializes the object members into a binary buffer. * * @param buff The buffer to serialize the data into. - * @return Status */ void serialize(Serializer& serializer) override; @@ -73,7 +71,7 @@ class GroupDetailsV2 : public GroupDetails { * * @param buff The buffer to deserialize from. * @param version The format spec version. - * @return Status and Attribute + * @return Group detail */ static shared_ptr deserialize( Deserializer& deserializer, const URI& group_uri); @@ -83,7 +81,7 @@ class GroupDetailsV2 : public GroupDetails { * * @param buff The buffer to deserialize from. * @param version The format spec version. - * @return Status and Attribute + * @return Group detail */ static shared_ptr deserialize( const std::vector>& deserializer, @@ -94,10 +92,8 @@ class GroupDetailsV2 : public GroupDetails { * Apply any pending member additions or removals * * mutates members_ and clears members_to_modify_ - * - * @return Status */ - Status apply_pending_changes() override; + void apply_pending_changes() override; private: /* Format version for class. */ From 52c4525ccb520f41b13d21dd9e5dea8e993fc80e Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Mon, 13 Nov 2023 09:55:39 -0700 Subject: [PATCH 069/456] Remove function `sanity_check()` for contexts (#4506) The function `sanity_check()` for contexts has returned a constant `TILEDB_OK` for a while now; that check was moved into the exception wrappers. At the time, the function was stubbed out to avoid combining the large number of lines that would change with other, more significant ones. This PR completes the removal of this function. --- TYPE: NO_HISTORY DESC: Remove function `sanity_check()` for contexts --- tiledb/sm/c_api/api_argument_validator.h | 8 - tiledb/sm/c_api/tiledb.cc | 506 ++++++++-------------- tiledb/sm/c_api/tiledb_dimension_label.cc | 6 +- 3 files changed, 181 insertions(+), 339 deletions(-) diff --git a/tiledb/sm/c_api/api_argument_validator.h b/tiledb/sm/c_api/api_argument_validator.h index a2d02548caf2..cda354585d60 100644 --- a/tiledb/sm/c_api/api_argument_validator.h +++ b/tiledb/sm/c_api/api_argument_validator.h @@ -81,14 +81,6 @@ inline int32_t sanity_check( return TILEDB_OK; } -/** - * This function is dead code. Validity of the context is now checked in the - * exception wrapper. - */ -inline constexpr int32_t sanity_check(tiledb_ctx_t*) { - return TILEDB_OK; -} - inline int32_t sanity_check( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema) { if (array_schema == nullptr || array_schema->array_schema_ == nullptr) { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 940cad49d96e..3d3b5ce33449 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -244,9 +244,6 @@ int32_t tiledb_array_schema_alloc( tiledb_ctx_t* ctx, tiledb_array_type_t array_type, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; - // Create array schema struct *array_schema = new (std::nothrow) tiledb_array_schema_t; if (*array_schema == nullptr) { @@ -296,8 +293,7 @@ int32_t tiledb_array_schema_add_attribute( int32_t tiledb_array_schema_set_allows_dups( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, int allows_dups) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(array_schema->array_schema_->set_allows_dups(allows_dups)); return TILEDB_OK; @@ -305,8 +301,7 @@ int32_t tiledb_array_schema_set_allows_dups( int32_t tiledb_array_schema_get_allows_dups( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, int* allows_dups) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *allows_dups = (int)array_schema->array_schema_->allows_dups(); return TILEDB_OK; @@ -314,8 +309,7 @@ int32_t tiledb_array_schema_get_allows_dups( int32_t tiledb_array_schema_get_version( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint32_t* version) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *version = (uint32_t)array_schema->array_schema_->version(); return TILEDB_OK; @@ -325,8 +319,7 @@ int32_t tiledb_array_schema_set_domain( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_domain_t* domain) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( array_schema->array_schema_->set_domain(domain->copy_domain())); @@ -335,8 +328,7 @@ int32_t tiledb_array_schema_set_domain( int32_t tiledb_array_schema_set_capacity( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, uint64_t capacity) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; array_schema->array_schema_->set_capacity(capacity); return TILEDB_OK; @@ -346,8 +338,7 @@ int32_t tiledb_array_schema_set_cell_order( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_layout_t cell_order) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(array_schema->array_schema_->set_cell_order( static_cast(cell_order))); @@ -358,8 +349,7 @@ int32_t tiledb_array_schema_set_tile_order( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_layout_t tile_order) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(array_schema->array_schema_->set_tile_order( static_cast(tile_order))); @@ -371,8 +361,7 @@ int32_t tiledb_array_schema_timestamp_range( tiledb_array_schema_t* array_schema, uint64_t* lo, uint64_t* hi) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; auto timestamp_range = array_schema->array_schema_->timestamp_range(); @@ -401,8 +390,7 @@ int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { return TILEDB_ERR; } api::ensure_filter_list_is_valid(filter_list); @@ -417,8 +405,7 @@ int32_t tiledb_array_schema_set_offsets_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { return TILEDB_ERR; } api::ensure_filter_list_is_valid(filter_list); @@ -434,8 +421,7 @@ int32_t tiledb_array_schema_set_validity_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { return TILEDB_ERR; } api::ensure_filter_list_is_valid(filter_list); @@ -459,9 +445,6 @@ int32_t tiledb_array_schema_load( tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; - // Create array schema *array_schema = new (std::nothrow) tiledb_array_schema_t; if (*array_schema == nullptr) { @@ -540,9 +523,6 @@ int32_t tiledb_array_schema_load_with_key( const void* encryption_key, uint32_t key_length, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; - // Create array schema *array_schema = new (std::nothrow) tiledb_array_schema_t; if (*array_schema == nullptr) { @@ -631,8 +611,7 @@ int32_t tiledb_array_schema_get_array_type( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, tiledb_array_type_t* array_type) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *array_type = static_cast( array_schema->array_schema_->array_type()); @@ -643,8 +622,7 @@ int32_t tiledb_array_schema_get_capacity( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, uint64_t* capacity) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *capacity = array_schema->array_schema_->capacity(); return TILEDB_OK; @@ -654,8 +632,7 @@ int32_t tiledb_array_schema_get_cell_order( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, tiledb_layout_t* cell_order) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *cell_order = static_cast(array_schema->array_schema_->cell_order()); @@ -666,8 +643,7 @@ int32_t tiledb_array_schema_get_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; api::ensure_output_pointer_is_valid(filter_list); // Copy-construct a separate FilterPipeline object @@ -680,8 +656,7 @@ int32_t tiledb_array_schema_get_offsets_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; api::ensure_output_pointer_is_valid(filter_list); // Copy-construct a separate FilterPipeline object @@ -694,8 +669,7 @@ int32_t tiledb_array_schema_get_validity_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, tiledb_filter_list_t** filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; api::ensure_output_pointer_is_valid(filter_list); // Copy-construct a separate FilterPipeline object @@ -721,8 +695,7 @@ int32_t tiledb_array_schema_get_tile_order( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, tiledb_layout_t* tile_order) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *tile_order = static_cast(array_schema->array_schema_->tile_order()); @@ -733,8 +706,7 @@ int32_t tiledb_array_schema_get_attribute_num( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, uint32_t* attribute_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; *attribute_num = array_schema->array_schema_->attribute_num(); return TILEDB_OK; @@ -742,8 +714,7 @@ int32_t tiledb_array_schema_get_attribute_num( int32_t tiledb_array_schema_dump( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema, FILE* out) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; array_schema->array_schema_->dump(out); return TILEDB_OK; @@ -814,8 +785,7 @@ int32_t tiledb_array_schema_has_attribute( const tiledb_array_schema_t* array_schema, const char* name, int32_t* has_attr) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { return TILEDB_ERR; } @@ -835,8 +805,6 @@ int32_t tiledb_array_schema_evolution_alloc( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t** array_schema_evolution) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; // Create schema evolution struct *array_schema_evolution = new (std::nothrow) tiledb_array_schema_evolution_t; @@ -892,8 +860,7 @@ int32_t tiledb_array_schema_evolution_drop_attribute( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, const char* attribute_name) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; array_schema_evolution->array_schema_evolution_->drop_attribute( @@ -955,8 +922,7 @@ int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_array_schema_evolution_t* array_schema_evolution, uint64_t lo, uint64_t hi) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; array_schema_evolution->array_schema_evolution_->set_timestamp_range( @@ -974,7 +940,7 @@ int32_t tiledb_query_alloc( tiledb_query_type_t query_type, tiledb_query_t** query) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Error if array is not open @@ -1038,7 +1004,7 @@ int32_t tiledb_query_alloc( int32_t tiledb_query_get_stats( tiledb_ctx_t* ctx, tiledb_query_t* query, char** stats_json) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; if (stats_json == nullptr) @@ -1078,7 +1044,7 @@ int32_t tiledb_query_get_config( int32_t tiledb_query_set_subarray( tiledb_ctx_t* ctx, tiledb_query_t* query, const void* subarray_vals) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Set subarray @@ -1092,8 +1058,7 @@ int32_t tiledb_query_set_subarray_t( tiledb_query_t* query, const tiledb_subarray_t* subarray) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, query) == TILEDB_ERR || + if (sanity_check(ctx, query) == TILEDB_ERR || sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; @@ -1108,7 +1073,7 @@ int32_t tiledb_query_set_data_buffer( const char* name, void* buffer, uint64_t* buffer_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Set attribute buffer @@ -1123,7 +1088,7 @@ int32_t tiledb_query_set_offsets_buffer( const char* name, uint64_t* buffer_offsets, uint64_t* buffer_offsets_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Set attribute buffer @@ -1139,7 +1104,7 @@ int32_t tiledb_query_set_validity_buffer( const char* name, uint8_t* buffer_validity, uint64_t* buffer_validity_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Set attribute buffer @@ -1156,7 +1121,7 @@ int32_t tiledb_query_get_data_buffer( void** buffer, uint64_t** buffer_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Get attribute buffer @@ -1172,7 +1137,7 @@ int32_t tiledb_query_get_offsets_buffer( uint64_t** buffer, uint64_t** buffer_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Get attribute buffer @@ -1188,7 +1153,7 @@ int32_t tiledb_query_get_validity_buffer( uint8_t** buffer, uint64_t** buffer_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Get attribute buffer @@ -1201,7 +1166,7 @@ int32_t tiledb_query_get_validity_buffer( int32_t tiledb_query_set_layout( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_layout_t layout) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Set layout @@ -1216,8 +1181,7 @@ int32_t tiledb_query_set_condition( tiledb_query_t* const query, const tiledb_query_condition_t* const cond) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, query) == TILEDB_ERR || + if (sanity_check(ctx, query) == TILEDB_ERR || sanity_check(ctx, cond) == TILEDB_ERR) return TILEDB_ERR; @@ -1233,7 +1197,7 @@ int32_t tiledb_query_finalize(tiledb_ctx_t* ctx, tiledb_query_t* query) { return TILEDB_OK; // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Flush query @@ -1249,7 +1213,7 @@ int32_t tiledb_query_submit_and_finalize( return TILEDB_OK; // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->submit_and_finalize()); @@ -1267,7 +1231,7 @@ void tiledb_query_free(tiledb_query_t** query) { int32_t tiledb_query_submit(tiledb_ctx_t* ctx, tiledb_query_t* query) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->submit()); @@ -1281,7 +1245,7 @@ int32_t tiledb_query_submit_async( void (*callback)(void*), void* callback_data) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->submit_async(callback, callback_data)); @@ -1291,7 +1255,7 @@ int32_t tiledb_query_submit_async( int32_t tiledb_query_has_results( tiledb_ctx_t* ctx, tiledb_query_t* query, int32_t* has_results) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *has_results = query->query_->has_results(); @@ -1302,7 +1266,7 @@ int32_t tiledb_query_has_results( int32_t tiledb_query_get_status( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_query_status_t* status) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *status = (tiledb_query_status_t)query->query_->status(); @@ -1313,7 +1277,7 @@ int32_t tiledb_query_get_status( int32_t tiledb_query_get_type( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_query_type_t* query_type) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *query_type = static_cast(query->query_->type()); @@ -1324,7 +1288,7 @@ int32_t tiledb_query_get_type( int32_t tiledb_query_get_layout( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_layout_t* query_layout) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *query_layout = static_cast(query->query_->layout()); @@ -1335,7 +1299,7 @@ int32_t tiledb_query_get_layout( int32_t tiledb_query_get_array( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_array_t** array) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Create array datatype @@ -1361,7 +1325,7 @@ int32_t tiledb_query_add_range( const void* start, const void* end, const void* stride) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1375,7 +1339,7 @@ int32_t tiledb_query_add_point_ranges( uint32_t dim_idx, const void* start, uint64_t count) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; /* @@ -1400,7 +1364,7 @@ int32_t tiledb_query_add_range_by_name( const void* start, const void* end, const void* stride) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1416,7 +1380,7 @@ int32_t tiledb_query_add_range_var( uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1432,7 +1396,7 @@ int32_t tiledb_query_add_range_var_by_name( uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1445,7 +1409,7 @@ int32_t tiledb_query_get_range_num( const tiledb_query_t* query, uint32_t dim_idx, uint64_t* range_num) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1458,7 +1422,7 @@ int32_t tiledb_query_get_range_num_from_name( const tiledb_query_t* query, const char* dim_name, uint64_t* range_num) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1474,7 +1438,7 @@ int32_t tiledb_query_get_range( const void** start, const void** end, const void** stride) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1490,7 +1454,7 @@ int32_t tiledb_query_get_range_from_name( const void** start, const void** end, const void** stride) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1505,7 +1469,7 @@ int32_t tiledb_query_get_range_var_size( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t which_subarray(query); @@ -1520,7 +1484,7 @@ int32_t tiledb_query_get_range_var_size_from_name( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1535,7 +1499,7 @@ int32_t tiledb_query_get_range_var( uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1550,7 +1514,7 @@ int32_t tiledb_query_get_range_var_from_name( uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; tiledb_subarray_transient_local_t query_subarray(query); @@ -1563,7 +1527,7 @@ int32_t tiledb_query_get_est_result_size( const tiledb_query_t* query, const char* name, uint64_t* size) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_est_result_size(name, size)); @@ -1577,7 +1541,7 @@ int32_t tiledb_query_get_est_result_size_var( const char* name, uint64_t* size_off, uint64_t* size_val) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_est_result_size(name, size_off, size_val)); @@ -1591,7 +1555,7 @@ int32_t tiledb_query_get_est_result_size_nullable( const char* name, uint64_t* size_val, uint64_t* size_validity) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_est_result_size_nullable( @@ -1607,7 +1571,7 @@ int32_t tiledb_query_get_est_result_size_var_nullable( uint64_t* size_off, uint64_t* size_val, uint64_t* size_validity) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_est_result_size_nullable( @@ -1618,7 +1582,7 @@ int32_t tiledb_query_get_est_result_size_var_nullable( int32_t tiledb_query_get_fragment_num( tiledb_ctx_t* ctx, const tiledb_query_t* query, uint32_t* num) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_written_fragment_num(num)); @@ -1631,7 +1595,7 @@ int32_t tiledb_query_get_fragment_uri( const tiledb_query_t* query, uint64_t idx, const char** uri) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(query->query_->get_written_fragment_uri(idx, uri)); @@ -1645,7 +1609,7 @@ int32_t tiledb_query_get_fragment_timestamp_range( uint64_t idx, uint64_t* t1, uint64_t* t2) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -1660,7 +1624,7 @@ int32_t tiledb_query_get_subarray_t( tiledb_subarray_t** subarray) { *subarray = nullptr; // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *subarray = new (std::nothrow) tiledb_subarray_t; @@ -1681,7 +1645,7 @@ int32_t tiledb_query_get_relevant_fragment_num( tiledb_ctx_t* ctx, const tiledb_query_t* query, uint64_t* relevant_fragment_num) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; *relevant_fragment_num = @@ -1697,8 +1661,7 @@ int32_t tiledb_query_add_update_value( const void* update_value, uint64_t update_value_size) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, query) == TILEDB_ERR) { + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; } @@ -1773,8 +1736,7 @@ void tiledb_subarray_free(tiledb_subarray_t** subarray) { int32_t tiledb_subarray_set_coalesce_ranges( tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, int coalesce_ranges) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -1787,8 +1749,7 @@ int32_t tiledb_subarray_set_subarray( tiledb_ctx_t* ctx, tiledb_subarray_t* subarray_obj, const void* subarray_vals) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray_obj) == TILEDB_ERR) + if (sanity_check(ctx, subarray_obj) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray_obj->subarray_->set_subarray(subarray_vals)); @@ -1803,8 +1764,7 @@ int32_t tiledb_subarray_add_range( const void* start, const void* end, const void* stride) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->add_range(dim_idx, start, end, stride)); @@ -1818,8 +1778,7 @@ int32_t tiledb_subarray_add_point_ranges( uint32_t dim_idx, const void* start, uint64_t count) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->add_point_ranges(dim_idx, start, count)); @@ -1834,8 +1793,7 @@ int32_t tiledb_subarray_add_range_by_name( const void* start, const void* end, const void* stride) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -1852,8 +1810,7 @@ int32_t tiledb_subarray_add_range_var( uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->add_range_var( @@ -1870,8 +1827,7 @@ int32_t tiledb_subarray_add_range_var_by_name( uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->add_range_var_by_name( @@ -1885,8 +1841,7 @@ int32_t tiledb_subarray_get_range_num( const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t* range_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->get_range_num(dim_idx, range_num)); @@ -1899,8 +1854,7 @@ int32_t tiledb_subarray_get_range_num_from_name( const tiledb_subarray_t* subarray, const char* dim_name, uint64_t* range_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -1917,8 +1871,7 @@ int32_t tiledb_subarray_get_range( const void** start, const void** end, const void** stride) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -1934,8 +1887,7 @@ int32_t tiledb_subarray_get_range_var_size( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->get_range_var_size( @@ -1952,8 +1904,7 @@ int32_t tiledb_subarray_get_range_from_name( const void** start, const void** end, const void** stride) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->get_range_from_name( @@ -1969,8 +1920,7 @@ int32_t tiledb_subarray_get_range_var_size_from_name( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->get_range_var_size_from_name( @@ -1986,8 +1936,7 @@ int32_t tiledb_subarray_get_range_var( uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -2003,8 +1952,7 @@ int32_t tiledb_subarray_get_range_var_from_name( uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(subarray->subarray_->get_range_var_from_name( @@ -2019,11 +1967,6 @@ int32_t tiledb_subarray_get_range_var_from_name( int32_t tiledb_query_condition_alloc( tiledb_ctx_t* const ctx, tiledb_query_condition_t** const cond) { - if (sanity_check(ctx) == TILEDB_ERR) { - *cond = nullptr; - return TILEDB_ERR; - } - // Create query condition struct *cond = new (std::nothrow) tiledb_query_condition_t; if (*cond == nullptr) { @@ -2107,8 +2050,7 @@ int32_t tiledb_query_condition_init( const void* const condition_value, const uint64_t condition_value_size, const tiledb_query_condition_op_t op) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, cond) == TILEDB_ERR) { + if (sanity_check(ctx, cond) == TILEDB_ERR) { return TILEDB_ERR; } @@ -2135,8 +2077,7 @@ int32_t tiledb_query_condition_combine( const tiledb_query_condition_combination_op_t combination_op, tiledb_query_condition_t** const combined_cond) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, left_cond) == TILEDB_ERR || + if (sanity_check(ctx, left_cond) == TILEDB_ERR || (combination_op != TILEDB_NOT && sanity_check(ctx, right_cond) == TILEDB_ERR) || (combination_op == TILEDB_NOT && right_cond != nullptr)) @@ -2220,11 +2161,6 @@ capi_return_t tiledb_query_condition_set_use_enumeration( int32_t tiledb_array_alloc( tiledb_ctx_t* ctx, const char* array_uri, tiledb_array_t** array) { - if (sanity_check(ctx) == TILEDB_ERR) { - *array = nullptr; - return TILEDB_ERR; - } - // Create array struct *array = new (std::nothrow) tiledb_array_t; if (*array == nullptr) { @@ -2266,7 +2202,7 @@ int32_t tiledb_array_alloc( int32_t tiledb_array_set_open_timestamp_start( tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t timestamp_start) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; array->array_->set_timestamp_start(timestamp_start); @@ -2276,7 +2212,7 @@ int32_t tiledb_array_set_open_timestamp_start( int32_t tiledb_array_set_open_timestamp_end( tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t timestamp_end) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; array->array_->set_timestamp_end(timestamp_end); @@ -2286,7 +2222,7 @@ int32_t tiledb_array_set_open_timestamp_end( int32_t tiledb_array_get_open_timestamp_start( tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t* timestamp_start) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; *timestamp_start = array->array_->timestamp_start(); @@ -2296,7 +2232,7 @@ int32_t tiledb_array_get_open_timestamp_start( int32_t tiledb_array_get_open_timestamp_end( tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t* timestamp_end) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; *timestamp_end = array->array_->timestamp_end_opened_at(); @@ -2305,9 +2241,6 @@ int32_t tiledb_array_get_open_timestamp_end( } int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; - // Allocate an array object tiledb_array_t* array = new (std::nothrow) tiledb_array_t; try { @@ -2344,7 +2277,7 @@ int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) { int32_t tiledb_array_delete_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; try { @@ -2365,7 +2298,7 @@ int32_t tiledb_array_delete_fragments( const char* uri, uint64_t timestamp_start, uint64_t timestamp_end) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; LOG_WARN( @@ -2504,7 +2437,7 @@ capi_return_t tiledb_array_delete_fragments_list( int32_t tiledb_array_open( tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_query_type_t query_type) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Open array @@ -2519,7 +2452,7 @@ int32_t tiledb_array_open( int32_t tiledb_array_is_open( tiledb_ctx_t* ctx, tiledb_array_t* array, int32_t* is_open) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; *is_open = (int32_t)array->array_->is_open(); @@ -2528,7 +2461,7 @@ int32_t tiledb_array_is_open( } int32_t tiledb_array_reopen(tiledb_ctx_t* ctx, tiledb_array_t* array) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Reopen array @@ -2557,7 +2490,7 @@ int32_t tiledb_array_get_config( } int32_t tiledb_array_close(tiledb_ctx_t* ctx, tiledb_array_t* array) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Close array @@ -2577,8 +2510,7 @@ int32_t tiledb_array_get_schema( tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array) == TILEDB_ERR) { + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; } @@ -2607,8 +2539,7 @@ int32_t tiledb_array_get_schema( int32_t tiledb_array_get_query_type( tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_query_type_t* query_type) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array) == TILEDB_ERR) { + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; } @@ -2630,8 +2561,7 @@ int32_t tiledb_array_create( const char* array_uri, const tiledb_array_schema_t* array_schema) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; // Check array name @@ -2696,8 +2626,7 @@ int32_t tiledb_array_create_with_key( const void* encryption_key, uint32_t key_length) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; // Check array name @@ -2788,8 +2717,6 @@ int32_t tiledb_array_consolidate_with_key( uint32_t key_length, tiledb_config_t* config) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; throw_if_not_ok(ctx->storage_manager()->array_consolidate( array_uri, @@ -2809,8 +2736,6 @@ int32_t tiledb_array_consolidate_fragments( const uint64_t num_fragments, tiledb_config_t* config) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; // Convert the list of fragments to a vector std::vector uris; @@ -2843,7 +2768,7 @@ int32_t tiledb_array_vacuum( int32_t tiledb_array_get_non_empty_domain( tiledb_ctx_t* ctx, tiledb_array_t* array, void* domain, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b; @@ -2862,7 +2787,7 @@ int32_t tiledb_array_get_non_empty_domain_from_index( uint32_t idx, void* domain, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b; @@ -2881,7 +2806,7 @@ int32_t tiledb_array_get_non_empty_domain_from_name( const char* name, void* domain, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b; @@ -2901,7 +2826,7 @@ int32_t tiledb_array_get_non_empty_domain_var_size_from_index( uint64_t* start_size, uint64_t* end_size, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b = true; @@ -2922,7 +2847,7 @@ int32_t tiledb_array_get_non_empty_domain_var_size_from_name( uint64_t* start_size, uint64_t* end_size, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b = true; @@ -2943,7 +2868,7 @@ int32_t tiledb_array_get_non_empty_domain_var_from_index( void* start, void* end, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b = true; @@ -2964,7 +2889,7 @@ int32_t tiledb_array_get_non_empty_domain_var_from_name( void* start, void* end, int32_t* is_empty) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; bool is_empty_b = true; @@ -2981,7 +2906,7 @@ int32_t tiledb_array_get_non_empty_domain_var_from_name( int32_t tiledb_array_get_uri( tiledb_ctx_t* ctx, tiledb_array_t* array, const char** array_uri) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; *array_uri = array->array_.get()->array_uri().c_str(); @@ -2994,8 +2919,7 @@ int32_t tiledb_array_encryption_type( const char* array_uri, tiledb_encryption_type_t* encryption_type) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR || array_uri == nullptr || - encryption_type == nullptr) + if (array_uri == nullptr || encryption_type == nullptr) return TILEDB_ERR; // For easy reference @@ -3035,7 +2959,7 @@ int32_t tiledb_array_put_metadata( tiledb_datatype_t value_type, uint32_t value_num, const void* value) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Put metadata @@ -3047,7 +2971,7 @@ int32_t tiledb_array_put_metadata( int32_t tiledb_array_delete_metadata( tiledb_ctx_t* ctx, tiledb_array_t* array, const char* key) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Put metadata @@ -3063,7 +2987,7 @@ int32_t tiledb_array_get_metadata( tiledb_datatype_t* value_type, uint32_t* value_num, const void** value) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Get metadata @@ -3077,7 +3001,7 @@ int32_t tiledb_array_get_metadata( int32_t tiledb_array_get_metadata_num( tiledb_ctx_t* ctx, tiledb_array_t* array, uint64_t* num) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Get metadata num @@ -3095,7 +3019,7 @@ int32_t tiledb_array_get_metadata_from_index( tiledb_datatype_t* value_type, uint32_t* value_num, const void** value) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Get metadata @@ -3113,7 +3037,7 @@ int32_t tiledb_array_has_metadata_key( const char* key, tiledb_datatype_t* value_type, int32_t* has_key) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; // Check whether metadata has_key @@ -3131,8 +3055,7 @@ int32_t tiledb_array_evolve( const char* array_uri, tiledb_array_schema_evolution_t* array_schema_evolution) { // Sanity Checks - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; // Check array name @@ -3190,8 +3113,6 @@ capi_return_t tiledb_array_load_all_enumerations( int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { // Sanity Checks - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; // Check array name tiledb::sm::URI uri(array_uri); @@ -3217,9 +3138,6 @@ int32_t tiledb_array_upgrade_version( int32_t tiledb_object_type( tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; - auto uri = tiledb::sm::URI(path); tiledb::sm::ObjectType object_type; throw_if_not_ok(ctx->storage_manager()->object_type(uri, &object_type)); @@ -3229,16 +3147,12 @@ int32_t tiledb_object_type( } int32_t tiledb_object_remove(tiledb_ctx_t* ctx, const char* path) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; throw_if_not_ok(ctx->storage_manager()->object_remove(path)); return TILEDB_OK; } int32_t tiledb_object_move( tiledb_ctx_t* ctx, const char* old_path, const char* new_path) { - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; throw_if_not_ok(ctx->storage_manager()->object_move(old_path, new_path)); return TILEDB_OK; } @@ -3250,8 +3164,7 @@ int32_t tiledb_object_walk( int32_t (*callback)(const char*, tiledb_object_t, void*), void* data) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; + if (callback == nullptr) { auto st = Status_Error("Cannot initiate walk; Invalid callback function"); LOG_STATUS_NO_RETURN_VALUE(st); @@ -3297,8 +3210,7 @@ int32_t tiledb_object_ls( int32_t (*callback)(const char*, tiledb_object_t, void*), void* data) { // Sanity checks - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; + if (callback == nullptr) { auto st = Status_Error("Cannot initiate object ls; Invalid callback function"); @@ -3343,9 +3255,8 @@ int32_t tiledb_object_ls( /* ****************************** */ int32_t tiledb_uri_to_path( - tiledb_ctx_t* ctx, const char* uri, char* path_out, uint32_t* path_length) { - if (sanity_check(ctx) == TILEDB_ERR || uri == nullptr || - path_out == nullptr || path_length == nullptr) + tiledb_ctx_t*, const char* uri, char* path_out, uint32_t* path_length) { + if (uri == nullptr || path_out == nullptr || path_length == nullptr) return TILEDB_ERR; std::string path = tiledb::sm::URI::to_path(uri); @@ -3459,7 +3370,7 @@ int32_t tiledb_serialize_array( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -3487,8 +3398,6 @@ int32_t tiledb_deserialize_array( int32_t, tiledb_array_t** array) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -3547,8 +3456,7 @@ int32_t tiledb_serialize_array_schema( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema) == TILEDB_ERR) + if (sanity_check(ctx, array_schema) == TILEDB_ERR) return TILEDB_ERR; // Create buffer @@ -3610,8 +3518,7 @@ int32_t tiledb_serialize_array_open( // Currently no different behaviour is required if array open is serialized by // the client or the Cloud server, so the variable is unused // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array) == TILEDB_ERR) { + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; } @@ -3640,12 +3547,6 @@ int32_t tiledb_deserialize_array_open( tiledb_array_t** array) { // Currently no different behaviour is required if array open is deserialized // by the client or the Cloud server - - // Sanity check - if (sanity_check(ctx) == TILEDB_ERR) { - return TILEDB_ERR; - } - api::ensure_buffer_is_valid(buffer); // Create array struct @@ -3704,8 +3605,7 @@ int32_t tiledb_serialize_array_schema_evolution( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -3733,8 +3633,6 @@ int32_t tiledb_deserialize_array_schema_evolution( int32_t, tiledb_array_schema_evolution_t** array_schema_evolution) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -3769,7 +3667,7 @@ int32_t tiledb_serialize_query( int32_t client_side, tiledb_buffer_list_t** buffer_list) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Allocate a buffer list @@ -3797,7 +3695,7 @@ int32_t tiledb_deserialize_query( int32_t client_side, tiledb_query_t* query) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -3822,7 +3720,7 @@ int32_t tiledb_deserialize_query_and_array( tiledb_query_t** query, tiledb_array_t** array) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || query == nullptr) { + if (query == nullptr) { return TILEDB_ERR; } @@ -3913,7 +3811,7 @@ int32_t tiledb_serialize_array_nonempty_domain( int32_t, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -3944,7 +3842,7 @@ int32_t tiledb_deserialize_array_nonempty_domain( void* nonempty_domain, int32_t* is_empty) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -3969,7 +3867,7 @@ int32_t tiledb_serialize_array_non_empty_domain_all_dimensions( int32_t, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -3996,7 +3894,7 @@ int32_t tiledb_deserialize_array_non_empty_domain_all_dimensions( tiledb_serialization_type_t serialize_type, int32_t) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -4016,7 +3914,7 @@ int32_t tiledb_serialize_array_max_buffer_sizes( tiledb_serialization_type_t serialize_type, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -4099,7 +3997,7 @@ int32_t tiledb_serialize_array_metadata( tiledb_serialization_type_t serialize_type, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -4133,7 +4031,7 @@ int32_t tiledb_deserialize_array_metadata( tiledb_serialization_type_t serialize_type, const tiledb_buffer_t* buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -4154,7 +4052,7 @@ int32_t tiledb_serialize_query_est_result_sizes( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; auto buf = tiledb_buffer_handle_t::make_handle(); @@ -4182,7 +4080,7 @@ int32_t tiledb_deserialize_query_est_result_sizes( int32_t client_side, const tiledb_buffer_t* buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; api::ensure_buffer_is_valid(buffer); @@ -4203,8 +4101,7 @@ int32_t tiledb_serialize_config( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR) - return TILEDB_ERR; + api::ensure_config_is_valid(config); auto buf = tiledb_buffer_handle_t::make_handle(); @@ -4264,8 +4161,7 @@ int32_t tiledb_serialize_fragment_info_request( // serialized by the client or the Cloud server // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) { + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) { return TILEDB_ERR; } @@ -4296,8 +4192,7 @@ int32_t tiledb_deserialize_fragment_info_request( // serialized by the client or the Cloud server // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) { + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) { return TILEDB_ERR; } @@ -4322,8 +4217,7 @@ int32_t tiledb_serialize_fragment_info( int32_t client_side, tiledb_buffer_t** buffer) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) { + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) { return TILEDB_ERR; } @@ -4357,8 +4251,7 @@ int32_t tiledb_deserialize_fragment_info( // deserialized by the client or the Cloud server // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) { + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) { return TILEDB_ERR; } @@ -4453,8 +4346,7 @@ int32_t tiledb_query_submit_async_func( tiledb_query_t* query, void* callback_func, void* callback_data) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, query) == TILEDB_ERR || callback_func == nullptr) + if (sanity_check(ctx, query) == TILEDB_ERR || callback_func == nullptr) return TILEDB_ERR; std::function callback = @@ -4474,11 +4366,6 @@ int32_t tiledb_fragment_info_alloc( tiledb_ctx_t* ctx, const char* array_uri, tiledb_fragment_info_t** fragment_info) { - if (sanity_check(ctx) == TILEDB_ERR) { - *fragment_info = nullptr; - return TILEDB_ERR; - } - // Create fragment info struct *fragment_info = new (std::nothrow) tiledb_fragment_info_t; if (*fragment_info == nullptr) { @@ -4547,8 +4434,7 @@ int32_t tiledb_fragment_info_get_config( tiledb_fragment_info_t* fragment_info, tiledb_config_t** config) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; api::ensure_output_pointer_is_valid(config); @@ -4559,8 +4445,7 @@ int32_t tiledb_fragment_info_get_config( int32_t tiledb_fragment_info_load( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; // Load fragment info @@ -4574,8 +4459,7 @@ int32_t tiledb_fragment_info_get_fragment_name( tiledb_fragment_info_t* fragment_info, uint32_t fid, const char** name) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; LOG_WARN( @@ -4612,8 +4496,7 @@ int32_t tiledb_fragment_info_get_fragment_num( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t* fragment_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; *fragment_num = fragment_info->fragment_info_->fragment_num(); @@ -4626,8 +4509,7 @@ int32_t tiledb_fragment_info_get_fragment_uri( tiledb_fragment_info_t* fragment_info, uint32_t fid, const char** uri) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_fragment_uri(fid, uri)); @@ -4640,8 +4522,7 @@ int32_t tiledb_fragment_info_get_fragment_size( tiledb_fragment_info_t* fragment_info, uint32_t fid, uint64_t* size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_fragment_size(fid, size)); @@ -4654,8 +4535,7 @@ int32_t tiledb_fragment_info_get_dense( tiledb_fragment_info_t* fragment_info, uint32_t fid, int32_t* dense) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_dense(fid, dense)); @@ -4668,8 +4548,7 @@ int32_t tiledb_fragment_info_get_sparse( tiledb_fragment_info_t* fragment_info, uint32_t fid, int32_t* sparse) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_sparse(fid, sparse)); @@ -4683,8 +4562,7 @@ int32_t tiledb_fragment_info_get_timestamp_range( uint32_t fid, uint64_t* start, uint64_t* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -4699,8 +4577,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_from_index( uint32_t fid, uint32_t did, void* domain) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -4715,8 +4592,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_from_name( uint32_t fid, const char* dim_name, void* domain) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_non_empty_domain( @@ -4732,8 +4608,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_var_size_from_index( uint32_t did, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_non_empty_domain_var_size( @@ -4749,8 +4624,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_var_size_from_name( const char* dim_name, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_non_empty_domain_var_size( @@ -4766,8 +4640,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_var_from_index( uint32_t did, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_non_empty_domain_var( @@ -4783,8 +4656,7 @@ int32_t tiledb_fragment_info_get_non_empty_domain_var_from_name( const char* dim_name, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_non_empty_domain_var( @@ -4798,8 +4670,7 @@ int32_t tiledb_fragment_info_get_mbr_num( tiledb_fragment_info_t* fragment_info, uint32_t fid, uint64_t* mbr_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_mbr_num(fid, mbr_num)); @@ -4814,8 +4685,7 @@ int32_t tiledb_fragment_info_get_mbr_from_index( uint32_t mid, uint32_t did, void* mbr) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_mbr(fid, mid, did, mbr)); @@ -4830,8 +4700,7 @@ int32_t tiledb_fragment_info_get_mbr_from_name( uint32_t mid, const char* dim_name, void* mbr) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -4848,8 +4717,7 @@ int32_t tiledb_fragment_info_get_mbr_var_size_from_index( uint32_t did, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_mbr_var_size( @@ -4866,8 +4734,7 @@ int32_t tiledb_fragment_info_get_mbr_var_size_from_name( const char* dim_name, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_mbr_var_size( @@ -4884,8 +4751,7 @@ int32_t tiledb_fragment_info_get_mbr_var_from_index( uint32_t did, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -4902,8 +4768,7 @@ int32_t tiledb_fragment_info_get_mbr_var_from_name( const char* dim_name, void* start, void* end) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_mbr_var( @@ -4917,8 +4782,7 @@ int32_t tiledb_fragment_info_get_cell_num( tiledb_fragment_info_t* fragment_info, uint32_t fid, uint64_t* cell_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_cell_num(fid, cell_num)); @@ -4930,8 +4794,7 @@ int32_t tiledb_fragment_info_get_total_cell_num( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint64_t* cell_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_total_cell_num(cell_num)); @@ -4944,8 +4807,7 @@ int32_t tiledb_fragment_info_get_version( tiledb_fragment_info_t* fragment_info, uint32_t fid, uint32_t* version) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_version(fid, version)); @@ -4958,8 +4820,7 @@ int32_t tiledb_fragment_info_has_consolidated_metadata( tiledb_fragment_info_t* fragment_info, uint32_t fid, int32_t* has) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -4972,8 +4833,7 @@ int32_t tiledb_fragment_info_get_unconsolidated_metadata_num( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t* unconsolidated) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; *unconsolidated = @@ -4986,8 +4846,7 @@ int32_t tiledb_fragment_info_get_to_vacuum_num( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info, uint32_t* to_vacuum_num) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; *to_vacuum_num = fragment_info->fragment_info_->to_vacuum_num(); @@ -5000,8 +4859,7 @@ int32_t tiledb_fragment_info_get_to_vacuum_uri( tiledb_fragment_info_t* fragment_info, uint32_t fid, const char** uri) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok(fragment_info->fragment_info_->get_to_vacuum_uri(fid, uri)); @@ -5014,8 +4872,7 @@ int32_t tiledb_fragment_info_get_array_schema( tiledb_fragment_info_t* fragment_info, uint32_t fid, tiledb_array_schema_t** array_schema) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; // Create array schema @@ -5039,8 +4896,7 @@ int32_t tiledb_fragment_info_get_array_schema_name( tiledb_fragment_info_t* fragment_info, uint32_t fid, const char** schema_name) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; throw_if_not_ok( @@ -5053,8 +4909,7 @@ int32_t tiledb_fragment_info_get_array_schema_name( int32_t tiledb_fragment_info_dump( tiledb_ctx_t* ctx, const tiledb_fragment_info_t* fragment_info, FILE* out) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, fragment_info) == TILEDB_ERR) + if (sanity_check(ctx, fragment_info) == TILEDB_ERR) return TILEDB_ERR; fragment_info->fragment_info_->dump(out); return TILEDB_OK; @@ -5069,7 +4924,7 @@ int32_t tiledb_query_get_status_details( tiledb_query_t* query, tiledb_query_status_details_t* status) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) return TILEDB_ERR; // Currently only one detailed reason. Retrieve it and set to user struct. @@ -5087,8 +4942,7 @@ int32_t tiledb_consolidation_plan_create_with_mbr( uint64_t fragment_size, tiledb_consolidation_plan_t** consolidation_plan) { // Sanity check - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, array) == TILEDB_ERR) { + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; } @@ -5134,8 +4988,7 @@ int32_t tiledb_consolidation_plan_get_num_nodes( tiledb_ctx_t* ctx, tiledb_consolidation_plan_t* consolidation_plan, uint64_t* num_nodes) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { + if (sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { return TILEDB_ERR; } @@ -5148,8 +5001,7 @@ int32_t tiledb_consolidation_plan_get_num_fragments( tiledb_consolidation_plan_t* consolidation_plan, uint64_t node_index, uint64_t* num_fragments) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { + if (sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { return TILEDB_ERR; } @@ -5172,8 +5024,7 @@ int32_t tiledb_consolidation_plan_get_fragment_uri( uint64_t node_index, uint64_t fragment_index, const char** uri) { - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { + if (sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { return TILEDB_ERR; } @@ -5198,8 +5049,7 @@ int32_t tiledb_consolidation_plan_dump_json_str( return TILEDB_ERR; } - if (sanity_check(ctx) == TILEDB_ERR || - sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { + if (sanity_check(ctx, consolidation_plan) == TILEDB_ERR) { return TILEDB_ERR; } diff --git a/tiledb/sm/c_api/tiledb_dimension_label.cc b/tiledb/sm/c_api/tiledb_dimension_label.cc index dfab3f8c32cc..866365ea61ef 100644 --- a/tiledb/sm/c_api/tiledb_dimension_label.cc +++ b/tiledb/sm/c_api/tiledb_dimension_label.cc @@ -44,7 +44,7 @@ capi_return_t tiledb_array_schema_add_dimension_label( const char* name, tiledb_data_order_t label_order, tiledb_datatype_t label_type) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array_schema)) { + if (sanity_check(ctx, array_schema)) { return TILEDB_ERR; } array_schema->array_schema_->add_dimension_label( @@ -84,7 +84,7 @@ capi_return_t tiledb_array_schema_set_dimension_label_filter_list( tiledb_array_schema_t* array_schema, const char* label_name, tiledb_filter_list_t* filter_list) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array_schema)) { + if (sanity_check(ctx, array_schema)) { return TILEDB_ERR; } api::ensure_filter_list_is_valid(filter_list); @@ -99,7 +99,7 @@ capi_return_t tiledb_array_schema_set_dimension_label_tile_extent( const char* label_name, tiledb_datatype_t type, const void* tile_extent) { - if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, array_schema)) { + if (sanity_check(ctx, array_schema)) { return TILEDB_ERR; } array_schema->array_schema_->set_dimension_label_tile_extent( From 53aa8a3b3a37ec7e6cafa896c1875e5b4a4a5cc3 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:31:32 -0500 Subject: [PATCH 070/456] Add class UniqueDirectory to create a unique, temporary directory (#4498) Add class `TemporaryDirectory` to create a unique, temporary directory using the PRNG --- TYPE: FEATURE DESC: Add class TemporaryDirectory to generate a PRNG-seeded unique directory. --- .gitignore | 1 - test/src/unit-capi-nullable.cc | 117 +++++++-------- test/src/unit-capi-smoke-test.cc | 133 ++++++++---------- test/src/unit-capi-vfs.cc | 131 +++++------------ test/src/unit-win-filesystem.cc | 32 ++--- test/support/src/helpers.cc | 87 +++++------- test/support/src/helpers.h | 20 ++- tiledb/CMakeLists.txt | 1 + .../query_field/test/unit_capi_query_field.cc | 7 +- tiledb/sm/filesystem/CMakeLists.txt | 4 +- .../filesystem/temporary_local_directory.cc | 78 ++++++++++ .../sm/filesystem/temporary_local_directory.h | 77 ++++++++++ 12 files changed, 372 insertions(+), 316 deletions(-) create mode 100644 tiledb/sm/filesystem/temporary_local_directory.cc create mode 100644 tiledb/sm/filesystem/temporary_local_directory.h diff --git a/.gitignore b/.gitignore index 45d3caf24475..8a62bc5943b7 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ examples/bin/* examples/obj/* scripts/deps-staging/* .idea -*temp* *.DS_Store doc/venv doc/source/__pycache__ diff --git a/test/src/unit-capi-nullable.cc b/test/src/unit-capi-nullable.cc index eb980553bf4e..793dc2cbbf11 100644 --- a/test/src/unit-capi-nullable.cc +++ b/test/src/unit-capi-nullable.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2020-2021 TileDB, Inc. + * @copyright Copyright (c) 2020-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,10 +32,14 @@ #include #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #include +#include #include using namespace std; @@ -44,8 +48,6 @@ using namespace tiledb::test; class NullableArrayFx { public: - const string& FILE_TEMP_DIR = tiledb::test::get_temp_path(); - // Serialization parameters bool serialize_ = false; bool refactored_query_v2_ = false; @@ -144,19 +146,16 @@ class NullableArrayFx { /** The C-API VFS object. */ tiledb_vfs_t* vfs_; - /** - * Creates a directory using `vfs_`. - * - * @param path The directory path. - */ - void create_dir(const string& path); + /** The unique local directory object. */ + TemporaryLocalDirectory temp_dir_; /** - * Removes a directory using `vfs_`. + * Compute the full array path given an array name. * - * @param path The directory path. + * @param array_name The array name. + * @return The full array path. */ - void remove_dir(const string& path); + const string array_path(const string& array_name); /** * Creates a TileDB array. @@ -201,40 +200,31 @@ class NullableArrayFx { const void* subarray); }; -NullableArrayFx::NullableArrayFx() { +NullableArrayFx::NullableArrayFx() + : serialize_(false) { // Create a config. tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; - REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_config_alloc(&config, &error)); + throw_if_setup_failed(error == nullptr); // Create the context. - REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_ctx_alloc(config, &ctx_)); + throw_if_setup_failed(ctx_ != nullptr); // Create the VFS. - REQUIRE(tiledb_vfs_alloc(ctx_, config, &vfs_) == TILEDB_OK); - + throw_if_setup_failed(tiledb_vfs_alloc(ctx_, config, &vfs_)); + throw_if_setup_failed(vfs_ != nullptr); tiledb_config_free(&config); - - serialize_ = false; } NullableArrayFx::~NullableArrayFx() { - remove_dir(FILE_TEMP_DIR); tiledb_ctx_free(&ctx_); tiledb_vfs_free(&vfs_); } -void NullableArrayFx::create_dir(const string& path) { - REQUIRE(tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); -} - -void NullableArrayFx::remove_dir(const string& path) { - int is_dir = 0; - REQUIRE(tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir) == TILEDB_OK); - if (is_dir) - REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); +const string NullableArrayFx::array_path(const string& array_name) { + return temp_dir_.path() + array_name; } void NullableArrayFx::create_array( @@ -244,9 +234,6 @@ void NullableArrayFx::create_array( const vector& test_attrs, const tiledb_layout_t cell_order, const tiledb_layout_t tile_order) { - remove_dir(FILE_TEMP_DIR); - create_dir(FILE_TEMP_DIR); - // Create the dimensions. vector dims; dims.reserve(test_dims.size()); @@ -313,8 +300,7 @@ void NullableArrayFx::create_array( REQUIRE(rc == TILEDB_OK); // Create array - rc = tiledb_array_create( - ctx_, (FILE_TEMP_DIR + array_name).c_str(), array_schema); + rc = tiledb_array_create(ctx_, array_path(array_name).c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Free attributes. @@ -340,8 +326,7 @@ void NullableArrayFx::write( const tiledb_layout_t layout) { // Open the array for writing. tiledb_array_t* array; - int rc = - tiledb_array_alloc(ctx_, (FILE_TEMP_DIR + array_name).c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_path(array_name).c_str(), &array); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); @@ -427,7 +412,7 @@ void NullableArrayFx::write( // Submit query rc = submit_query_wrapper( ctx_, - FILE_TEMP_DIR + array_name, + array_path(array_name), &query, server_buffers_, serialize_, @@ -447,8 +432,7 @@ void NullableArrayFx::read( const void* const subarray) { // Open the array for reading. tiledb_array_t* array; - int rc = - tiledb_array_alloc(ctx_, (FILE_TEMP_DIR + array_name).c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_path(array_name).c_str(), &array); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); @@ -535,7 +519,7 @@ void NullableArrayFx::read( // Submit query rc = submit_query_wrapper( ctx_, - FILE_TEMP_DIR + array_name, + array_path(array_name), &query, server_buffers_, serialize_, @@ -868,11 +852,6 @@ TEST_CASE_METHOD( NullableArrayFx, "C API: Test 2D array with nullable attributes", "[capi][2d][nullable]") { - vector attrs; - attrs.emplace_back("a1", TILEDB_INT32, 1, true); - attrs.emplace_back("a2", TILEDB_INT32, 1, true); - attrs.emplace_back("a3", TILEDB_INT32, TILEDB_VAR_NUM, true); - SECTION("no serialization") { serialize_ = false; } @@ -883,23 +862,31 @@ TEST_CASE_METHOD( } #endif - for (auto attr_iter = attrs.begin(); attr_iter != attrs.end(); ++attr_iter) { - vector test_attrs(attrs.begin(), attr_iter + 1); - for (const tiledb_array_type_t array_type : {TILEDB_DENSE, TILEDB_SPARSE}) { - for (const tiledb_layout_t cell_order : - {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}) { - for (const tiledb_layout_t tile_order : - {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}) { - for (const tiledb_layout_t write_order : - {TILEDB_ROW_MAJOR, - TILEDB_COL_MAJOR, - TILEDB_UNORDERED, - TILEDB_GLOBAL_ORDER}) { - do_2d_nullable_test( - test_attrs, array_type, cell_order, tile_order, write_order); - } - } - } - } + // Define the attributes. + vector attrs; + attrs.emplace_back("a1", TILEDB_INT32, 1, true); + attrs.emplace_back("a2", TILEDB_INT32, 1, true); + attrs.emplace_back("a3", TILEDB_INT32, TILEDB_VAR_NUM, true); + + // Generate test conditions + auto num_attrs{GENERATE(1, 2, 3)}; + auto array_type{GENERATE(TILEDB_DENSE, TILEDB_SPARSE)}; + auto cell_order{GENERATE(TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR)}; + auto tile_order{GENERATE(TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR)}; + auto write_order{GENERATE( + TILEDB_ROW_MAJOR, + TILEDB_COL_MAJOR, + TILEDB_UNORDERED, + TILEDB_GLOBAL_ORDER)}; + + DYNAMIC_SECTION( + array_type_str((ArrayType)array_type) + << " array with " << num_attrs << " attribute(s). " + << layout_str((Layout)cell_order) << " cells, " + << layout_str((Layout)tile_order) << " tiles, " + << layout_str((Layout)write_order) << " writes") { + vector test_attrs(attrs.begin(), attrs.begin() + num_attrs); + do_2d_nullable_test( + test_attrs, array_type, cell_order, tile_order, write_order); } } diff --git a/test/src/unit-capi-smoke-test.cc b/test/src/unit-capi-smoke-test.cc index d6a4edbe1313..95f536078054 100644 --- a/test/src/unit-capi-smoke-test.cc +++ b/test/src/unit-capi-smoke-test.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021-2022 TileDB, Inc. + * @copyright Copyright (c) 2021-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,24 +33,23 @@ #include #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #include +#include #include using namespace std; using namespace tiledb::sm; using namespace tiledb::test; -static const char encryption_key[] = "unittestunittestunittestunittest"; -static const string array_name = "smoke_test_array"; - class SmokeTestFx { public: - const string& FILE_TEMP_DIR = tiledb::test::get_temp_path(); - /** * Wraps data to build a dimension with the C-API. */ @@ -215,19 +214,22 @@ class SmokeTestFx { /** The C-API VFS object. */ tiledb_vfs_t* vfs_; - /** - * Creates a directory using `vfs_`. - * - * @param path The directory path. - */ - void create_dir(const string& path); + /** The unique local directory object. */ + TemporaryLocalDirectory temp_dir_; + + /** The encryption key. */ + const char encryption_key_[33] = "unittestunittestunittestunittest"; + + /** The name of the array. */ + const string array_name_ = "smoke_test_array"; /** - * Removes a directory using `vfs_`. + * Compute the full array path given an array name. * - * @param path The directory path. + * @param array_name The array name. + * @return The full array path. */ - void remove_dir(const string& path); + const string array_path(const string& array_name); /** * Creates a TileDB array. @@ -378,34 +380,26 @@ SmokeTestFx::SmokeTestFx() { // Create a config. tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; - REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_config_alloc(&config, &error)); + throw_if_setup_failed(error == nullptr); // Create the context. - REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_ctx_alloc(config, &ctx_)); + throw_if_setup_failed(ctx_ != nullptr); // Create the VFS. - REQUIRE(tiledb_vfs_alloc(ctx_, config, &vfs_) == TILEDB_OK); - + throw_if_setup_failed(tiledb_vfs_alloc(ctx_, config, &vfs_)); + throw_if_setup_failed(vfs_ != nullptr); tiledb_config_free(&config); } SmokeTestFx::~SmokeTestFx() { - remove_dir(FILE_TEMP_DIR); tiledb_ctx_free(&ctx_); tiledb_vfs_free(&vfs_); } -void SmokeTestFx::create_dir(const string& path) { - REQUIRE(tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); -} - -void SmokeTestFx::remove_dir(const string& path) { - int is_dir = 0; - REQUIRE(tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir) == TILEDB_OK); - if (is_dir) - REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); +const string SmokeTestFx::array_path(const string& array_name) { + return temp_dir_.path() + array_name; } void SmokeTestFx::create_array( @@ -415,9 +409,6 @@ void SmokeTestFx::create_array( tiledb_layout_t cell_order, tiledb_layout_t tile_order, tiledb_encryption_type_t encryption_type) { - remove_dir(FILE_TEMP_DIR); - create_dir(FILE_TEMP_DIR); - // Create the dimensions. vector dims; dims.reserve(test_dims.size()); @@ -498,14 +489,14 @@ void SmokeTestFx::create_array( rc = tiledb_config_set( config, "sm.encryption_type", encryption_type_string.c_str(), &error); REQUIRE(error == nullptr); - rc = tiledb_config_set(config, "sm.encryption_key", encryption_key, &error); + rc = + tiledb_config_set(config, "sm.encryption_key", encryption_key_, &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK); tiledb_config_free(&config); } - rc = tiledb_array_create( - ctx_, (FILE_TEMP_DIR + array_name).c_str(), array_schema); + rc = tiledb_array_create(ctx_, array_path(array_name_).c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Free attributes. @@ -531,8 +522,7 @@ void SmokeTestFx::write( tiledb_encryption_type_t encryption_type) { // Open the array for writing (with or without encryption). tiledb_array_t* array; - int rc = - tiledb_array_alloc(ctx_, (FILE_TEMP_DIR + array_name).c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_path(array_name_).c_str(), &array); REQUIRE(rc == TILEDB_OK); if (encryption_type != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -546,7 +536,7 @@ void SmokeTestFx::write( cfg, "sm.encryption_type", encryption_type_string.c_str(), &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); @@ -661,8 +651,7 @@ void SmokeTestFx::read( tiledb_query_condition_combination_op_t combination_op) { // Open the array for reading (with or without encryption). tiledb_array_t* array; - int rc = - tiledb_array_alloc(ctx_, (FILE_TEMP_DIR + array_name).c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_path(array_name_).c_str(), &array); REQUIRE(rc == TILEDB_OK); if (encryption_type != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -676,7 +665,7 @@ void SmokeTestFx::read( cfg, "sm.encryption_type", encryption_type_string.c_str(), &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key, &err); + rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_set_config(ctx_, array, cfg); @@ -1397,35 +1386,33 @@ TEST_CASE_METHOD( const uint64_t d3_tile_extent = 5; dims.emplace_back("d3", TILEDB_UINT64, d3_domain, d3_tile_extent); - for (auto attr_iter = attrs.begin(); attr_iter != attrs.end(); ++attr_iter) { - vector test_attrs(attrs.begin(), attr_iter + 1); - for (const tiledb_array_type_t array_type : {TILEDB_DENSE, TILEDB_SPARSE}) { - for (const tiledb_layout_t cell_order : - {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}) { - for (const tiledb_layout_t tile_order : - {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}) { - for (const tiledb_encryption_type_t encryption_type : - {TILEDB_NO_ENCRYPTION, TILEDB_AES_256_GCM}) { - for (const tiledb_layout_t write_order : - {TILEDB_ROW_MAJOR, TILEDB_UNORDERED}) { - vector test_dims; - for (const test_dim_t& dim : dims) { - test_dims.emplace_back(dim); - - smoke_test( - test_attrs, - query_conditions_vec, - test_dims, - array_type, - cell_order, - tile_order, - write_order, - encryption_type); - } - } - } - } - } - } + // Generate test conditions + auto num_attrs{GENERATE(1, 2, 3)}; + auto num_dims{GENERATE(1, 2, 3)}; + auto array_type{GENERATE(TILEDB_DENSE, TILEDB_SPARSE)}; + auto cell_order{GENERATE(TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR)}; + auto tile_order{GENERATE(TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR)}; + auto encryption_type{GENERATE(TILEDB_NO_ENCRYPTION, TILEDB_AES_256_GCM)}; + auto write_order{GENERATE(TILEDB_ROW_MAJOR, TILEDB_UNORDERED)}; + + DYNAMIC_SECTION( + array_type_str((ArrayType)array_type) + << " array with " << num_attrs << " attribute(s) and " << num_dims + << " dimension(s)." << layout_str((Layout)cell_order) << " cells, " + << layout_str((Layout)tile_order) << " tiles, " + << layout_str((Layout)write_order) << " writes, " + << encryption_type_str((EncryptionType)encryption_type) + << " encryption") { + vector test_attrs(attrs.begin(), attrs.begin() + num_attrs); + vector test_dims(dims.begin(), dims.begin() + num_dims); + smoke_test( + test_attrs, + query_conditions_vec, + test_dims, + array_type, + cell_order, + tile_order, + write_order, + encryption_type); } } diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index 22d44888b543..32dcc313926a 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,7 +32,9 @@ #include #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" @@ -42,12 +44,16 @@ #endif #include +#include #include #include using namespace tiledb::test; struct VFSFx { + // The unique directory object + tiledb::sm::TemporaryLocalDirectory temp_dir_{"tiledb_test_"}; + const std::string HDFS_TEMP_DIR = "hdfs://localhost:9000/tiledb_test/"; const std::string S3_PREFIX = "s3://"; const std::string S3_BUCKET = S3_PREFIX + random_name("tiledb") + "/"; @@ -60,7 +66,7 @@ struct VFSFx { #ifndef _WIN32 "file://" + #endif - tiledb::test::get_temp_path(); + temp_dir_.path(); const std::string MEMFS_TEMP_DIR = std::string("mem://tiledb_test/"); // TileDB context and vfs @@ -83,109 +89,20 @@ struct VFSFx { void check_append(const std::string& path); void check_ls(const std::string& path); static std::string random_name(const std::string& prefix); - void set_supported_fs(); - void set_num_vfs_threads(unsigned num_threads); }; VFSFx::VFSFx() { - ctx_ = nullptr; - vfs_ = nullptr; - - // Supported filesystems - set_supported_fs(); - - // Create context and VFS with 1 thread - set_num_vfs_threads(1); -} - -VFSFx::~VFSFx() { - tiledb_vfs_free(&vfs_); - tiledb_ctx_free(&ctx_); -} - -void VFSFx::set_supported_fs() { - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - + // Supported filesystem vector bool supports_gcs; // unused get_supported_fs( &supports_s3_, &supports_hdfs_, &supports_azure_, &supports_gcs); - tiledb_ctx_free(&ctx); + create_ctx_and_vfs(supports_s3_, supports_azure_, &ctx_, &vfs_); } -void VFSFx::set_num_vfs_threads(unsigned num_threads) { - if (vfs_ != nullptr) - tiledb_vfs_free(&vfs_); - if (ctx_ != nullptr) - tiledb_ctx_free(&ctx_); - - // Create TileDB context - tiledb_config_t* config = nullptr; - tiledb_error_t* error = nullptr; - REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); - REQUIRE(error == nullptr); - if (supports_s3_) { -#ifndef TILEDB_TESTS_AWS_S3_CONFIG - REQUIRE( - tiledb_config_set( - config, "vfs.s3.endpoint_override", "localhost:9999", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set(config, "vfs.s3.scheme", "https", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, "vfs.s3.use_virtual_addressing", "false", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set(config, "vfs.s3.verify_ssl", "false", &error) == - TILEDB_OK); - REQUIRE(error == nullptr); -#endif - } - if (supports_azure_) { - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.storage_account_name", - "devstoreaccount1", - &error) == TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw==", - &error) == TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.blob_endpoint", - "http://127.0.0.1:10000/devstoreaccount1", - &error) == TILEDB_OK); - } - - // Set number of threads across all backends. - REQUIRE( - tiledb_config_set( - config, - "vfs.s3.max_parallel_ops", - std::to_string(num_threads).c_str(), - &error) == TILEDB_OK); - // Set very small parallelization threshold (ignored when there is only 1 - // thread). - REQUIRE( - tiledb_config_set( - config, "vfs.min_parallel_size", std::to_string(1).c_str(), &error) == - TILEDB_OK); - REQUIRE(error == nullptr); - - REQUIRE(tiledb_ctx_alloc(config, &ctx_) == TILEDB_OK); - REQUIRE(error == nullptr); - int rc = tiledb_vfs_alloc(ctx_, config, &vfs_); - REQUIRE(rc == TILEDB_OK); - tiledb_config_free(&config); +VFSFx::~VFSFx() { + tiledb_vfs_free(&vfs_); + tiledb_ctx_free(&ctx_); } void VFSFx::check_vfs(const std::string& path) { @@ -329,7 +246,7 @@ void VFSFx::check_vfs(const std::string& path) { REQUIRE(!(bool)is_empty); } - if (!supports_s3_) { + if (!supports_s3_ && path != FILE_TEMP_DIR) { rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); REQUIRE(rc == TILEDB_OK); } @@ -953,7 +870,25 @@ TEST_CASE_METHOD( TEST_CASE_METHOD(VFSFx, "C API: Test VFS parallel I/O", "[capi][vfs]") { tiledb_stats_enable(); tiledb_stats_reset(); - set_num_vfs_threads(4); + + tiledb_config_t* config; + REQUIRE(tiledb_vfs_get_config(ctx_, vfs_, &config) == TILEDB_OK); + + // Set number of threads to 4. + tiledb_error_t* error = nullptr; + REQUIRE( + tiledb_config_set( + config, + "vfs.s3.max_parallel_ops", + std::to_string(4).c_str(), + &error) == TILEDB_OK); + // Set very small parallelization threshold (ignored when there is only 1 + // thread). + REQUIRE( + tiledb_config_set( + config, "vfs.min_parallel_size", std::to_string(1).c_str(), &error) == + TILEDB_OK); + REQUIRE(error == nullptr); if (supports_s3_) { check_vfs(S3_TEMP_DIR); diff --git a/test/src/unit-win-filesystem.cc b/test/src/unit-win-filesystem.cc index 6e888a44bb9b..e128e967cff9 100644 --- a/test/src/unit-win-filesystem.cc +++ b/test/src/unit-win-filesystem.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,10 +36,13 @@ #include "test/support/src/helpers.h" #include +#include +#include "tiledb/common/random/prng.h" #include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/crypto.h" #include "tiledb/sm/filesystem/path_win.h" +#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/filesystem/win.h" #include @@ -60,23 +63,15 @@ static bool ends_with(const std::string& value, const std::string& suffix) { } struct WinFx { - const std::string& TEMP_DIR = tiledb::test::get_temp_path(); Win win_; Config vfs_config_; + TemporaryLocalDirectory temp_dir_; WinFx() { REQUIRE(win_.init(vfs_config_).ok()); - - if (path_exists(TEMP_DIR)) { - REQUIRE(win_.remove_dir(TEMP_DIR).ok()); - } } - ~WinFx() { - if (path_exists(TEMP_DIR)) { - REQUIRE(win_.remove_dir(TEMP_DIR).ok()); - } - } + ~WinFx() = default; bool path_exists(std::string path) { return win_.is_file(path) || win_.is_dir(path); @@ -85,8 +80,9 @@ struct WinFx { TEST_CASE_METHOD(WinFx, "Test Windows filesystem", "[windows][filesystem]") { using tiledb::sm::path_win::is_win_path; - const std::string test_dir_path = TEMP_DIR + "/win_tests"; - const std::string test_file_path = TEMP_DIR + "/win_tests/tiledb_test_file"; + const std::string test_dir_path = temp_dir_.path() + "/win_tests"; + const std::string test_file_path = + temp_dir_.path() + "/win_tests/tiledb_test_file"; URI test_dir(test_dir_path); URI test_file(test_file_path); Status st; @@ -143,9 +139,7 @@ TEST_CASE_METHOD(WinFx, "Test Windows filesystem", "[windows][filesystem]") { Win::abs_path("path1\\path2\\..\\path3") == Win::current_dir() + "\\path1\\path3"); - CHECK(!win_.is_dir(TEMP_DIR)); - st = win_.create_dir(TEMP_DIR); - CHECK(st.ok()); + CHECK(win_.is_dir(temp_dir_.path())); CHECK(!win_.is_dir(test_dir.to_path())); st = win_.create_dir(test_dir.to_path()); CHECK(st.ok()); @@ -246,11 +240,7 @@ TEST_CASE_METHOD(WinFx, "Test Windows filesystem", "[windows][filesystem]") { TEST_CASE_METHOD( WinFx, "Test writing large files", "[.nightly_only][windows][large-file]") { const uint64_t five_gigabytes = static_cast(5) << 30; - - REQUIRE(win_.create_dir(TEMP_DIR).ok()); - - std::string file = TEMP_DIR + "\\large-file"; - + std::string file = temp_dir_.path() + "\\large-file"; std::vector buffer(five_gigabytes); // We use a prime period to catch errors where the 4GB buffer chunks are diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 5d18264dbc6d..fa9f61abecfa 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -61,18 +61,20 @@ std::mutex catch2_macro_mutex; namespace tiledb { namespace test { -const std::string& get_temp_path() { - // Ensure the path has a trailing delimiter. - static std::string temp_path = - (std::filesystem::temp_directory_path() / - ("tiledb_test_" + std::to_string(getpid())) / "") - .string(); +// Command line arguments. +std::string g_vfs; - return temp_path; +void throw_if_setup_failed(capi_return_t rc) { + if (rc != TILEDB_OK) { + throw std::runtime_error("Test setup failed."); + } } -// Command line arguments. -std::string g_vfs; +void throw_if_setup_failed(bool condition) { + if (!condition) { + throw std::runtime_error("Test setup failed."); + } +} void check_tiledb_error_with( tiledb_ctx_t* ctx, int rc, const std::string& expected_msg, bool contains) { @@ -790,54 +792,43 @@ void create_ctx_and_vfs( // Create TileDB context tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; - REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_config_alloc(&config, &error)); + throw_if_setup_failed(error == nullptr); if (s3_supported) { #ifndef TILEDB_TESTS_AWS_S3_CONFIG - REQUIRE( - tiledb_config_set( - config, "vfs.s3.endpoint_override", "localhost:9999", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set(config, "vfs.s3.scheme", "https", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, "vfs.s3.use_virtual_addressing", "false", &error) == - TILEDB_OK); - REQUIRE( - tiledb_config_set(config, "vfs.s3.verify_ssl", "false", &error) == - TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_config_set( + config, "vfs.s3.endpoint_override", "localhost:9999", &error)); + throw_if_setup_failed( + tiledb_config_set(config, "vfs.s3.scheme", "https", &error)); + throw_if_setup_failed(tiledb_config_set( + config, "vfs.s3.use_virtual_addressing", "false", &error)); + throw_if_setup_failed( + tiledb_config_set(config, "vfs.s3.verify_ssl", "false", &error)); + throw_if_setup_failed(error == nullptr); #endif } if (azure_supported) { - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.storage_account_name", - "devstoreaccount1", - &error) == TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw==", - &error) == TILEDB_OK); - REQUIRE( - tiledb_config_set( - config, - "vfs.azure.blob_endpoint", - "http://127.0.0.1:10000/devstoreaccount1", - &error) == TILEDB_OK); + throw_if_setup_failed(tiledb_config_set( + config, "vfs.azure.storage_account_name", "devstoreaccount1", &error)); + throw_if_setup_failed(tiledb_config_set( + config, + "vfs.azure.storage_account_key", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==", + &error)); + throw_if_setup_failed(tiledb_config_set( + config, + "vfs.azure.blob_endpoint", + "http://127.0.0.1:10000/devstoreaccount1", + &error)); } - REQUIRE(tiledb_ctx_alloc(config, ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + throw_if_setup_failed(tiledb_ctx_alloc(config, ctx)); + throw_if_setup_failed(ctx != nullptr); // Create VFS *vfs = nullptr; - REQUIRE(tiledb_vfs_alloc(*ctx, config, vfs) == TILEDB_OK); + throw_if_setup_failed(tiledb_vfs_alloc(*ctx, config, vfs)); + throw_if_setup_failed(vfs != nullptr); tiledb_config_free(&config); } diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 6f4b40af17f7..cbda05fcff8b 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -84,13 +84,27 @@ static tiledb::sm::stats::Stats g_helper_stats("test"); // objects that require a parent `Logger` object. shared_ptr g_helper_logger(void); -const std::string& get_temp_path(); - // For easy reference typedef std::pair Compressor; template using SubarrayRanges = std::vector>; +/** + * Throws if the return code is not OK. + * For use in test setup for object allocation. + * + * @param rc Return code from a TileDB C-API setup function. + */ +void throw_if_setup_failed(int rc); + +/** + * Throws if the condition is not met. + * For use in test setup for object allocation. + * + * @param condition Condition to check from a TileDB C-API setup function. + */ +void throw_if_setup_failed(bool condition); + /** * Check the return code for a TileDB C-API function is TILEDB_ERR and * compare the last error message from the local TileDB context to an expected diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 24c583618199..bd1a3076b4de 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -198,6 +198,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/s3_thread_pool_executor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/ssl_config.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/temporary_local_directory.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/uri.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs_file_handle.cc diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 6fa5a79f1c05..40e1042065e0 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -31,6 +31,7 @@ */ #include +#include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" @@ -48,12 +49,6 @@ struct QueryFieldFx : TemporaryDirectoryFixture { return temp_dir_ + "queryfield_array"; } - inline void throw_if_setup_failed(const int32_t rc) { - if (rc != TILEDB_OK) { - throw std::runtime_error("test setup failed"); - } - } - void write_sparse_array(const std::string& path); void create_sparse_array(const std::string& path); }; diff --git a/tiledb/sm/filesystem/CMakeLists.txt b/tiledb/sm/filesystem/CMakeLists.txt index 7cfae52b99ce..de8770cb2df3 100644 --- a/tiledb/sm/filesystem/CMakeLists.txt +++ b/tiledb/sm/filesystem/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2021-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,6 +35,7 @@ commence(object_library vfs) mem_filesystem.cc path_win.cc posix.cc + temporary_local_directory.cc uri.cc vfs.cc vfs_file_handle.cc @@ -60,6 +61,7 @@ commence(object_library vfs) cert_file config math + seedable_global_PRNG stats stringx thread_pool diff --git a/tiledb/sm/filesystem/temporary_local_directory.cc b/tiledb/sm/filesystem/temporary_local_directory.cc new file mode 100644 index 000000000000..ef58438dd6df --- /dev/null +++ b/tiledb/sm/filesystem/temporary_local_directory.cc @@ -0,0 +1,78 @@ +/** + * @file temporary_local_directory.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class TemporaryLocalDirectory. + */ + +#include "tiledb/sm/filesystem/temporary_local_directory.h" +#include "tiledb/common/random/prng.h" + +#include +#include + +using namespace tiledb::common; + +namespace tiledb::sm { + +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ + +TemporaryLocalDirectory::TemporaryLocalDirectory(std::string prefix) { + // Generate random number using global PRNG + PRNG& prng = PRNG::get(); + auto rand = prng(); + + // Generate random local path + path_ = (std::filesystem::temp_directory_path() / + (prefix + std::to_string(rand)) / "") + .string(); + + // Create a unique directory + std::filesystem::create_directory(path_); +} + +TemporaryLocalDirectory::~TemporaryLocalDirectory() { + // Remove the unique directory + std::error_code ec; + auto removed = std::filesystem::remove_all(path_, ec); + if (removed == static_cast(-1)) { + LOG_ERROR(ec.message()); + } +} + +/* ********************************* */ +/* API */ +/* ********************************* */ + +const std::string& TemporaryLocalDirectory::path() { + return path_; +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filesystem/temporary_local_directory.h b/tiledb/sm/filesystem/temporary_local_directory.h new file mode 100644 index 000000000000..9a3024486efa --- /dev/null +++ b/tiledb/sm/filesystem/temporary_local_directory.h @@ -0,0 +1,77 @@ +/** + * @file temporary_local_directory.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares class TemporaryLocalDirectory. + */ + +#ifndef TILEDB_TEMPORARY_LOCAL_DIRECTORY_H +#define TILEDB_TEMPORARY_LOCAL_DIRECTORY_H + +#include "tiledb/sm/filesystem/vfs.h" + +namespace tiledb::sm { + +/** + * Maintenance of a local, unique directory. + * + * This class will create a randomly-seeded directory on the local filesystem + * with an optional prefix. Upon destruction of the object, the local directory + * is removed. The path to the unique directory may be accessed for additional + * reads / writes. + */ +class TemporaryLocalDirectory { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Constructor which will create a unique local directory. */ + TemporaryLocalDirectory(std::string prefix = {}); + + /** Destructor which will remove the directory. */ + ~TemporaryLocalDirectory(); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Return the path of the unique directory. */ + const std::string& path(); + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The path of the unique directory. */ + std::string path_; +}; +} // namespace tiledb::sm + +#endif // TILEDB_TEMPORARY_LOCAL_DIRECTORY_H From b76be49980e98b99ff6cb1006e73f4964ec8c379 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 Nov 2023 11:09:44 +0200 Subject: [PATCH 071/456] Remove `TILEDB_INSTALL_STATIC_DEPS`. (#4505) > [!NOTE] > ~~Depends on #4500.~~ [SC-36531](https://app.shortcut.com/tiledb-inc/story/36531/remove-tiledb-install-static-deps) This PR removes the now-unused `TILEDB_INSTALL_STATIC_DEPS` option and all the infrastructure supporting it. This frees us from having to reason about the dependencies of our dependencies (at least when vcpkg is enabled), and allows us to more easily update libraries in the future (for example updating Curl after that will be a piece of cake). Instead of installing our static dependencies, we let the consuming project find them at configure time through the [`find_dependency`](https://cmake.org/cmake/help/latest/module/CMakeFindDependencyMacro.html) command[^1]. This matches what [other](https://github.com/Azure/azure-sdk-for-cpp/blob/0ca8a1d9d5726be74ba0985e69c711af6b4a89b2/sdk/core/azure-core/vcpkg/Config.cmake.in#L6-L19) [CMake](https://github.com/curl/curl/blob/9fb6cc54c59760bee23c730f668653895e713a25/CMake/curl-config.cmake.in#L26-L32) [libraries](https://github.com/googleapis/google-cloud-cpp/blob/43c6142d5b802deb07f8e3094e13367a6219a844/google/cloud/config.cmake.in#L15-L18) do. I tested this by successfully running the static linkage tests. I also ran a full build with every feature enabled and vcpkg disabled, to make sure the legacy build does not get broken, and will also trigger a nightly build run. [^1]: One possible place to find them is vcpkg; in fact this is what we do in the static linkage tests: we pass `vcpkg_installed` to the CMake prefix path, and all these calls to `find_dependency` will be resolved from there. --- TYPE: BUILD DESC: Remove bundling our static library dependencies; they are now found from the CMake config file. --- .github/workflows/build-windows.yml | 43 +++-- .github/workflows/ci-linux_mac.yml | 1 + .github/workflows/mingw-w64-tiledb/PKGBUILD | 1 - bootstrap | 7 + bootstrap.ps1 | 14 +- cmake/Modules/FindAWSSDK_EP.cmake | 34 +--- cmake/Modules/FindAzureCore_EP.cmake | 5 - cmake/Modules/FindAzureStorageBlobs_EP.cmake | 5 - cmake/Modules/FindAzureStorageCommon_EP.cmake | 8 +- cmake/Modules/FindBzip2_EP.cmake | 6 - cmake/Modules/FindCapnp_EP.cmake | 14 +- cmake/Modules/FindCrc32c_EP.cmake | 5 - cmake/Modules/FindCurl_EP.cmake | 6 - cmake/Modules/FindLZ4_EP.cmake | 6 - cmake/Modules/FindMagic_EP.cmake | 11 -- cmake/Modules/FindOpenSSL_EP.cmake | 8 - cmake/Modules/FindSpdlog_EP.cmake | 8 - cmake/Modules/FindWIL_EP.cmake | 8 - cmake/Modules/FindWebp_EP.cmake | 16 -- cmake/Modules/FindZlib_EP.cmake | 6 - cmake/Modules/FindZstd_EP.cmake | 6 - cmake/Modules/TileDBCommon.cmake | 84 --------- cmake/Options/BuildOptions.cmake | 2 +- cmake/Options/TileDBToolchain.cmake | 4 +- cmake/TileDB-Superbuild.cmake | 1 - cmake/inputs/Config.cmake.in | 36 +++- .../{arm64-macos.cmake => arm64-osx.cmake} | 0 .../{x64-macos.cmake => x64-osx.cmake} | 0 scripts/azure-linux_mac-release.yml | 2 +- scripts/azure-windows-release.yml | 2 +- scripts/ci/build_benchmarks.sh | 2 +- test/packaging/CMakeLists.txt | 13 +- tiledb/CMakeLists.txt | 175 +----------------- 33 files changed, 98 insertions(+), 441 deletions(-) rename ports/triplets/{arm64-macos.cmake => arm64-osx.cmake} (100%) rename ports/triplets/{x64-macos.cmake => x64-osx.cmake} (100%) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2a9dc05c561f..e1d6762ea082 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -131,7 +131,8 @@ jobs: - name: Prevent vcpkg from building debug variants run: python $env:GITHUB_WORKSPACE/scripts/ci/patch_vcpkg_triplets.py - - name: core tiledb windows build + - name: Configure TileDB + shell: pwsh run: | $ErrorView = "NormalView" @@ -147,9 +148,6 @@ jobs: } if ($env:TILEDB_AZURE -eq "ON") { $bootstrapOptions = "-EnableAzure " + $bootstrapOptions - - # to be removed when build of crc32 is to be used by gcs - $bootstrapOptions = $bootstrapOptions + " -_EnableCrc32" } if ($env:TILEDB_GCS -eq "ON") { $bootstrapOptions = "-EnableGcs " + $bootstrapOptions @@ -202,8 +200,12 @@ jobs: Write-Host "Bootstrap failed." exit $LastExitCode } + - name: Build TileDB + shell: pwsh + run: | + $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE - cmake --build $env:BUILD_BUILDDIRECTORY --config $CMakeBuildType -j $env:NUMBER_OF_PROCESSORS 2>&1 + cmake --build $env:BUILD_BUILDDIRECTORY --config $CMakeBuildType -j 2>&1 if ($LastExitCode -ne 0) { Write-Host "Build failed. CMake exit status: " $LastExitCocde @@ -216,14 +218,11 @@ jobs: Write-Host "Installation failed." $host.SetShouldExit($LastExitCode) } - shell: powershell - name: 'Test' id: test - shell: powershell + shell: pwsh run: | - write-host "begin run: 'Test'" - $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE # Clone backwards compatibility test arrays @@ -245,9 +244,9 @@ jobs: } # CMake exits with non-0 status if there are any warnings during the build, so - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j $env:NUMBER_OF_PROCESSORS --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal if ($env:TILEDB_AZURE -eq "ON") { if($env.TILEDB_USE_CUSTOM_NODE_JS) { @@ -304,7 +303,11 @@ jobs: $host.SetShouldExit($LastExitCode) } } - # Build the examples + - name: Build examples + shell: pwsh + run: | + $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE + cmake --build $env:BUILD_BUILDDIRECTORY --target examples --config $CMakeBuildType -v if ($LastExitCode -ne 0) { @@ -380,15 +383,19 @@ jobs: Set-Location -path $TestAppDir Remove-Item -Path $TestAppDataDir -recurse -force -ErrorAction SilentlyContinue - # Build examples + - name: Build CMake project example + shell: pwsh + run: | cd $env:BUILD_SOURCESDIRECTORY\examples\cmake_project + $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE + mkdir build cd build # Build zip artifact - cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILD_DIRECTORY\vcpkg_installed\x64-windows-mod" .. + cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. cmake --build . --config $CMakeBuildType -v @@ -402,11 +409,13 @@ jobs: Write-Host "cmd: '$cmd'" Invoke-Expression $cmd - # Packaging test + - name: Packaging test + shell: pwsh + run: | cd $env:BUILD_SOURCESDIRECTORY\test\packaging mkdir build cd build - cmake -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist" .. + cmake -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. # The testing is in the test/packaging/CMakeLists.txt file, just invoked, # there is no other code to be built in the produced projects. diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index a02ac7ed0aff..ba204063f7a0 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -136,6 +136,7 @@ jobs: env: ASAN_OPTIONS: ${{ inputs.asan && 'detect_leaks=0' || '' }} LD_PRELOAD: ${{ inputs.asan && '/usr/lib/x86_64-linux-gnu/libasan.so.6' || '' }} + VCPKG_TARGET_TRIPLET: ${{ runner.os == 'Linux' && 'x64-linux' || 'x64-osx' }} run: | set -e pipefail diff --git a/.github/workflows/mingw-w64-tiledb/PKGBUILD b/.github/workflows/mingw-w64-tiledb/PKGBUILD index 8881b2bb86fc..1ec3de5f780f 100644 --- a/.github/workflows/mingw-w64-tiledb/PKGBUILD +++ b/.github/workflows/mingw-w64-tiledb/PKGBUILD @@ -32,7 +32,6 @@ build() { -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DTILEDB_STATIC=ON \ - -DTILEDB_INSTALL_STATIC_DEPS=OFF \ -DTILEDB_TESTS=OFF \ -DTILEDB_S3=ON \ -DCOMPILER_SUPPORTS_AVX2=OFF \ diff --git a/bootstrap b/bootstrap index 8ade2249dabf..0269a9faedbf 100755 --- a/bootstrap +++ b/bootstrap @@ -47,6 +47,10 @@ Configuration: --disable-avx2 disables use of AVX2 instructions --disable-webp disables building of webp library and linkage test --disable-vcpkg disables use of vcpkg for downloading and building dependencies + --disable-install-find-dependencies + Disables calls to find_dependency in the exported config file. + Should be set to ON when building a shared library with all + dependencies statically linked. --enable-static-tiledb enables building TileDB as a static library --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) @@ -116,6 +120,7 @@ tiledb_build_webp="ON" tiledb_build_abseil="OFF" tiledb_tests_enable_rest="OFF" tiledb_tests_aws_s3_config="OFF" +tiledb_install_find_dependencies="ON" enable_multiple="" while test $# != 0; do case "$1" in @@ -134,6 +139,7 @@ while test $# != 0; do --disable-avx2) tiledb_disable_avx2="-DCOMPILER_SUPPORTS_AVX2=FALSE";; --disable-webp) tiledb_build_webp="OFF";; --disable-vcpkg) tiledb_vcpkg="OFF";; + --disable-install-find-dependencies) tiledb_install_find_dependencies="OFF";; --enable-assertions) tiledb_assertions="ON";; --enable-debug) build_type="Debug";; --enable-release-symbols) build_type="RelWithDebInfo";; @@ -255,6 +261,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ + -DTILEDB_INSTALL_FIND_DEPENDENCIES=${tiledb_install_find_dependencies} \ ${tiledb_disable_avx2} \ "${source_dir}" || die "failed to configure the project" diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 6c3c0358406b..306f3a81b9f4 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -101,6 +101,9 @@ Disables internal TileDB statistics. .PARAMETER DisableVcpkg Disables building dependencies with vcpkg. +.PARAMETER DisableInstallFindDependencies +Disables calling find_dependency in the exported config file. Should be set when when building a shared library with all dependencies statically linked. + .PARAMETER BuildProcesses Number of parallel compile jobs. @@ -145,6 +148,7 @@ Param( [switch]$DisableCppApi, [switch]$DisableTests, [switch]$DisableStats, + [switch]$DisableInstallFindDependencies, [Alias('J')] [int] $BuildProcesses = $env:NUMBER_OF_PROCESSORS @@ -243,6 +247,14 @@ if ($DisableStats.IsPresent) { $Stats = "OFF" } +$InstallFindDependencies = "ON" +if ($DisableInstallFindDependencies.IsPresent) { + $InstallFindDependencies = "OFF" + if ($EnableStaticTileDB.IsPresent) { + Write-Warning "DisableInstallFindDependencies is ignored when building a static library." + } +} + $BuildWebP="ON" if ($DisableWebP.IsPresent) { $BuildWebP="OFF" @@ -326,7 +338,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 -DTILEDB_INSTALL_FIND_DEPENDENCIES=$InstallFindDependencies $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Modules/FindAWSSDK_EP.cmake b/cmake/Modules/FindAWSSDK_EP.cmake index b4a8cdfe1f70..0682e959c6d0 100644 --- a/cmake/Modules/FindAWSSDK_EP.cmake +++ b/cmake/Modules/FindAWSSDK_EP.cmake @@ -34,25 +34,8 @@ include(TileDBCommon) set(AWS_SERVICES identity-management sts s3) if(TILEDB_VCPKG) - # Provides: ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} - # TODO: We may need to conditionally use ${AWSSDK_PLATFORM_DEPS} here for dynamically linked deps, but - # it lists bare "pthread;curl" which leads to linkage of system versions. For static linkage, we - # handle those elsewhere at the moment. + # Provides: ${AWSSDK_LINK_LIBRARIES} find_package(AWSSDK REQUIRED QUIET COMPONENTS ${AWS_SERVICES}) - - if (TILEDB_STATIC) - - set(AWSSDK_EXTRA_LIBS) - # Since AWS SDK 1.11, AWSSDK_THIRD_PARTY_LIBS was replaced by AWSSDK_COMMON_RUNTIME_LIBS. - foreach(TARGET IN LISTS AWSSDK_THIRD_PARTY_LIBS AWSSDK_COMMON_RUNTIME_LIBS) - if (TARGET AWS::${TARGET}) - list(APPEND AWSSDK_EXTRA_LIBS "AWS::${TARGET}") - endif() - endforeach() - install_all_target_libs("${AWSSDK_EXTRA_LIBS}") - endif() - - install_all_target_libs("${AWSSDK_LINK_LIBRARIES}") return() endif() @@ -188,18 +171,3 @@ if (NOT AWSSDK_FOUND) message(FATAL_ERROR "Could not find AWSSDK (required).") endif () endif () - -if (AWSSDK_FOUND) - if (TILEDB_STATIC) - set(AWSSDK_EXTRA_LIBS) - # Since AWS SDK 1.11, AWSSDK_THIRD_PARTY_LIBS was replaced by AWSSDK_COMMON_RUNTIME_LIBS. - foreach(TARGET IN LISTS AWSSDK_THIRD_PARTY_LIBS AWSSDK_COMMON_RUNTIME_LIBS) - if (TARGET AWS::${TARGET}) - list(APPEND AWSSDK_EXTRA_LIBS "AWS::${TARGET}") - endif() - endforeach() - install_all_target_libs("${AWSSDK_EXTRA_LIBS}") - endif() - - install_all_target_libs("${AWSSDK_LINK_LIBRARIES}") -endif () diff --git a/cmake/Modules/FindAzureCore_EP.cmake b/cmake/Modules/FindAzureCore_EP.cmake index 3f01216b1060..f58739fdac1e 100644 --- a/cmake/Modules/FindAzureCore_EP.cmake +++ b/cmake/Modules/FindAzureCore_EP.cmake @@ -146,8 +146,3 @@ if (AZURECORE_FOUND AND NOT TARGET Azure::azure-core) INTERFACE_INCLUDE_DIRECTORIES "${AZURECORE_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (AZURECORE_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Azure::azure-core) -endif() diff --git a/cmake/Modules/FindAzureStorageBlobs_EP.cmake b/cmake/Modules/FindAzureStorageBlobs_EP.cmake index 9cc6be904bf0..fa1fb8677c8d 100644 --- a/cmake/Modules/FindAzureStorageBlobs_EP.cmake +++ b/cmake/Modules/FindAzureStorageBlobs_EP.cmake @@ -134,8 +134,3 @@ if (AZURE_STORAGE_BLOBS_FOUND AND NOT TARGET Azure::azure-storage-blobs) INTERFACE_INCLUDE_DIRECTORIES "${AZURE_STORAGE_BLOBS_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (AZURE_STORAGE_BLOBS_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Azure::azure-storage-blobs) -endif() diff --git a/cmake/Modules/FindAzureStorageCommon_EP.cmake b/cmake/Modules/FindAzureStorageCommon_EP.cmake index 8e3fd8f90b68..d750c623e8a3 100644 --- a/cmake/Modules/FindAzureStorageCommon_EP.cmake +++ b/cmake/Modules/FindAzureStorageCommon_EP.cmake @@ -136,9 +136,7 @@ if (AZURE_STORAGE_COMMON_FOUND AND NOT TARGET Azure::azure-storage-common) IMPORTED_LOCATION "${AZURE_STORAGE_COMMON_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${AZURE_STORAGE_COMMON_INCLUDE_DIR}" ) -endif() - -# If we built a static EP, install it if required. -if (AZURE_STORAGE_COMMON_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Azure::azure-storage-common) + if (WIN32) + target_link_libraries(Azure::azure-storage-common INTERFACE WebServices) + endif() endif() diff --git a/cmake/Modules/FindBzip2_EP.cmake b/cmake/Modules/FindBzip2_EP.cmake index 8c5f828491b7..6a77d8b410d7 100644 --- a/cmake/Modules/FindBzip2_EP.cmake +++ b/cmake/Modules/FindBzip2_EP.cmake @@ -36,7 +36,6 @@ include(TileDBCommon) if(TILEDB_VCPKG) find_package(BZip2 REQUIRED) - install_target_libs(BZip2::BZip2) return() endif() @@ -141,8 +140,3 @@ if (BZIP2_FOUND AND NOT TARGET BZip2::BZip2) INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (BZIP2_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(BZip2::BZip2) -endif() diff --git a/cmake/Modules/FindCapnp_EP.cmake b/cmake/Modules/FindCapnp_EP.cmake index 9a1ddbb3d9e1..d177cb0f33f3 100644 --- a/cmake/Modules/FindCapnp_EP.cmake +++ b/cmake/Modules/FindCapnp_EP.cmake @@ -34,7 +34,6 @@ include(TileDBCommon) if (TILEDB_VCPKG AND TILEDB_SERIALIZATION) find_package(CapnProto REQUIRED) - install_all_target_libs("CapnProto::capnp;CapnProto::capnp-json;CapnProto::kj") return() endif() @@ -114,13 +113,7 @@ else() endif() -if(WIN32) - if(TILEDB_CAPNP_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(CapnProto::capnp) - install_target_libs(CapnProto::kj) - install_target_libs(CapnProto::capnp-json) - endif() -elseif(CAPNP_FOUND AND NOT WIN32) +if(CAPNP_FOUND AND NOT WIN32) # We handle the found case first because ubuntu's install of capnproto is missing libcapnp-json # so we must first make sure the found case has all the appropriate libs, else we will build from source # List of all required Capnp libraries. @@ -130,11 +123,6 @@ elseif(CAPNP_FOUND AND NOT WIN32) message(FATAL_ERROR "Required target CapnProto::${LIB} not defined") elseif (TARGET CapnProto::${LIB}) message(STATUS "Found CapnProto lib: ${LIB}") - # If we built a static EP, install it if required. - if (TILEDB_CAPNP_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - message(STATUS "Adding CapnProto lib: ${LIB} to install target") - install_target_libs(CapnProto::${LIB}) - endif() elseif (NOT TARGET CapnProto::${LIB}) message(STATUS "NOT Found Target for CapnProto lib: ${LIB}") set(SHOULD_CAPNP_LIBRARIES 1) diff --git a/cmake/Modules/FindCrc32c_EP.cmake b/cmake/Modules/FindCrc32c_EP.cmake index ec3f8d0b864e..296c6049cf6e 100644 --- a/cmake/Modules/FindCrc32c_EP.cmake +++ b/cmake/Modules/FindCrc32c_EP.cmake @@ -90,8 +90,3 @@ if (Crc32c_FOUND AND NOT TARGET Crc32c::crc32c) INTERFACE_INCLUDE_DIRECTORIES "${Crc32c_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (TILEDB_CRC32C_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(Crc32c::crc32c) -endif() diff --git a/cmake/Modules/FindCurl_EP.cmake b/cmake/Modules/FindCurl_EP.cmake index 289340ffe843..068790906911 100644 --- a/cmake/Modules/FindCurl_EP.cmake +++ b/cmake/Modules/FindCurl_EP.cmake @@ -31,7 +31,6 @@ endif() if (TILEDB_VCPKG) find_package(CURL REQUIRED) - install_target_libs(CURL::libcurl) return() endif() @@ -245,8 +244,3 @@ if (CURL_FOUND) endif() endif() - -# If we built a static EP, install it if required. -if (TILEDB_CURL_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(CURL::libcurl) -endif() diff --git a/cmake/Modules/FindLZ4_EP.cmake b/cmake/Modules/FindLZ4_EP.cmake index efdabea8a1a8..b044992998e7 100644 --- a/cmake/Modules/FindLZ4_EP.cmake +++ b/cmake/Modules/FindLZ4_EP.cmake @@ -36,7 +36,6 @@ include(TileDBCommon) if(TILEDB_VCPKG) find_package(lz4 REQUIRED) - install_target_libs(lz4::lz4) return() endif() @@ -129,8 +128,3 @@ if (LZ4_FOUND AND NOT TARGET lz4::lz4) INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (LZ4_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(lz4::lz4) -endif() diff --git a/cmake/Modules/FindMagic_EP.cmake b/cmake/Modules/FindMagic_EP.cmake index 418dbcf4ddab..efc039d39062 100644 --- a/cmake/Modules/FindMagic_EP.cmake +++ b/cmake/Modules/FindMagic_EP.cmake @@ -38,9 +38,6 @@ if(TILEDB_VCPKG) find_package(unofficial-libmagic CONFIG) if (unofficial-libmagic_FOUND) set(libmagic_DICTIONARY ${unofficial-libmagic_DICTIONARY}) - install_target_libs(unofficial::libmagic::libmagic) - install_target_libs(PCRE2::8BIT) - install_target_libs(PCRE2::POSIX) else() find_path(libmagic_INCLUDE_DIR NAMES magic.h) find_library(libmagic_LIBRARIES magic) @@ -65,7 +62,6 @@ if(TILEDB_VCPKG) IMPORTED_LOCATION "${libmagic_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${libmagic_INCLUDE_DIR}" ) - install_target_libs(unofficial::libmagic::libmagic) endif() add_library(libmagic ALIAS unofficial::libmagic::libmagic) return() @@ -170,10 +166,3 @@ if (libmagic_FOUND AND NOT TARGET libmagic) INTERFACE_INCLUDE_DIRECTORIES "${libmagic_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (TILEDB_LIBMAGIC_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(libmagic) - install_target_libs(pcre2-posix) - install_target_libs(pcre2-8) -endif() diff --git a/cmake/Modules/FindOpenSSL_EP.cmake b/cmake/Modules/FindOpenSSL_EP.cmake index e711e91ad87c..56a6e944df6a 100644 --- a/cmake/Modules/FindOpenSSL_EP.cmake +++ b/cmake/Modules/FindOpenSSL_EP.cmake @@ -35,8 +35,6 @@ include(TileDBCommon) if (TILEDB_VCPKG) find_package(OpenSSL REQUIRED) - target_link_libraries(OpenSSL::SSL) - install_all_target_libs(OpenSSL::SSL) return() endif() @@ -155,9 +153,3 @@ if (OPENSSL_FOUND) ) endif() endif() - -# If we built a static EP, install it if required. -if (TILEDB_OPENSSL_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(OpenSSL::SSL) - install_target_libs(OpenSSL::Crypto) -endif() diff --git a/cmake/Modules/FindSpdlog_EP.cmake b/cmake/Modules/FindSpdlog_EP.cmake index e07c646ffc25..a3d3eb83edc8 100644 --- a/cmake/Modules/FindSpdlog_EP.cmake +++ b/cmake/Modules/FindSpdlog_EP.cmake @@ -34,9 +34,7 @@ include(TileDBCommon) if (TILEDB_VCPKG) - find_package(fmt REQUIRED) find_package(spdlog CONFIG REQUIRED) - install_all_target_libs("fmt::fmt;spdlog::spdlog") endif() @@ -126,9 +124,3 @@ elseif(TARGET spdlog::spdlog) endif() endif() endif() - - -# If we built a static EP, install it if required. -if (TILEDB_SPDLOG_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(spdlog::spdlog) -endif() diff --git a/cmake/Modules/FindWIL_EP.cmake b/cmake/Modules/FindWIL_EP.cmake index ad037f19f703..7f2f301a0ee2 100644 --- a/cmake/Modules/FindWIL_EP.cmake +++ b/cmake/Modules/FindWIL_EP.cmake @@ -93,11 +93,3 @@ if (wil_FOUND AND NOT TARGET wil::wil) INTERFACE_INCLUDE_DIRECTORIES "${wil_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if(TARGET wil::wil) - get_target_property(target_type wil::wil TYPE) - if (target_type STREQUAL STATIC_LIBRARY AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(wil::wil) - endif() -endif() diff --git a/cmake/Modules/FindWebp_EP.cmake b/cmake/Modules/FindWebp_EP.cmake index ecbffdd7f388..64766f5b6f42 100644 --- a/cmake/Modules/FindWebp_EP.cmake +++ b/cmake/Modules/FindWebp_EP.cmake @@ -28,13 +28,7 @@ include(TileDBCommon) if (TILEDB_VCPKG) - find_package(Threads REQUIRED) find_package(WebP REQUIRED) - install_target_libs(WebP::webp) - install_target_libs(WebP::webpdecoder) - install_target_libs(WebP::webpdemux) - install_target_libs(WebP::libwebpmux) - install_target_libs(WebP::sharpyuv) return() endif() @@ -88,13 +82,3 @@ if(NOT TILEDB_WEBP_EP_BUILT) message(FATAL_ERROR "Unable to find Webp") endif() endif() - -# Always building static EP, install it.. -if (TILEDB_WEBP_EP_BUILT) - # One install_target_libs() with all of these was only installing the first item. - install_target_libs(WebP::webp) - install_target_libs(WebP::webpdecoder) - install_target_libs(WebP::webpdemux) - install_target_libs(WebP::libwebpmux) - install_target_libs(WebP::sharpyuv) -endif() diff --git a/cmake/Modules/FindZlib_EP.cmake b/cmake/Modules/FindZlib_EP.cmake index 4cde4294257f..78a232648409 100644 --- a/cmake/Modules/FindZlib_EP.cmake +++ b/cmake/Modules/FindZlib_EP.cmake @@ -35,7 +35,6 @@ include(TileDBCommon) if (TILEDB_VCPKG) find_package(ZLIB REQUIRED) - install_target_libs(ZLIB::ZLIB) return() endif() @@ -127,8 +126,3 @@ if (ZLIB_FOUND AND NOT TARGET ZLIB::ZLIB ) INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_INCLUDE_DIR}" ) endif() - -# If we built a static EP, install it if required. -if (TILEDB_ZLIB_EP_BUILT AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(ZLIB::ZLIB) -endif() diff --git a/cmake/Modules/FindZstd_EP.cmake b/cmake/Modules/FindZstd_EP.cmake index a4736afe7d0b..bd43cab030ff 100644 --- a/cmake/Modules/FindZstd_EP.cmake +++ b/cmake/Modules/FindZstd_EP.cmake @@ -42,7 +42,6 @@ if (TILEDB_VCPKG) else() set(ZSTD_TARGET zstd::libzstd_shared) endif() - install_target_libs(${ZSTD_TARGET}) return() endif() @@ -137,8 +136,3 @@ if (ZSTD_FOUND AND NOT ZSTD_TARGET) ) set(ZSTD_TARGET zstd::libzstd) endif() - -# If we built a static EP, install it if required. -if (ZSTD_STATIC_EP_FOUND AND TILEDB_INSTALL_STATIC_DEPS) - install_target_libs(${ZSTD_TARGET}) -endif() diff --git a/cmake/Modules/TileDBCommon.cmake b/cmake/Modules/TileDBCommon.cmake index c73fde1ed11a..d8099d02707a 100644 --- a/cmake/Modules/TileDBCommon.cmake +++ b/cmake/Modules/TileDBCommon.cmake @@ -30,89 +30,5 @@ # Get library directory for multiarch linux distros include(GNUInstallDirs) -# -# Stores the IMPORTED_LOCATION* target property of LIB_TARGET in RESULT_VAR. -# On Windows, preferentially tries the IMPORTED_IMPLIB* target property instead. -# -function(get_imported_location RESULT_VAR LIB_TARGET) - if (WIN32) - # Try several methods to find the imported location. - get_target_property(TMP ${LIB_TARGET} IMPORTED_IMPLIB) - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_IMPLIB_RELWITHDEBINFO) - endif() - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_IMPLIB_RELEASE) - endif() - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_IMPLIB_DEBUG) - endif() - endif() - # Try several methods to find the imported location. - if (TMP MATCHES "NOTFOUND" OR NOT WIN32) - get_target_property(TMP ${LIB_TARGET} IMPORTED_LOCATION) - endif() - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_LOCATION_RELWITHDEBINFO) - endif() - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_LOCATION_RELEASE) - endif() - if (TMP MATCHES "NOTFOUND") - get_target_property(TMP ${LIB_TARGET} IMPORTED_LOCATION_DEBUG) - endif() - set(${RESULT_VAR} "${TMP}" PARENT_SCOPE) -endfunction() - -# -# Concatenates the library name of the given imported target to the installation -# prefix and stores the result in RESULT_VAR. If the given library does not reside -# in the external projects build directory (i.e. it was not built by an EP), then -# just return the imported location. -# -function(get_installed_location RESULT_VAR LIB_TARGET) - get_imported_location(IMP_LOC ${LIB_TARGET}) - if ("${IMP_LOC}" MATCHES "^${TILEDB_EP_BASE}") - get_filename_component(LIB_NAME "${IMP_LOC}" NAME) - set(INSTALLED_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${LIB_NAME}") - else() - set(INSTALLED_PATH "${IMP_LOC}") - endif() - set(${RESULT_VAR} "${INSTALLED_PATH}" PARENT_SCOPE) -endfunction() - -# -# Adds imported libraries from the given target to the TileDB installation -# manifest. -# -function(install_target_libs LIB_TARGET) - if (NOT TILEDB_INSTALL_STATIC_DEPS) - return() - endif() - get_imported_location(TARGET_LIBRARIES ${LIB_TARGET}) - if (TARGET_LIBRARIES MATCHES "NOTFOUND") - message(FATAL_ERROR "Could not determine library location for ${LIB_TARGET}") - endif() - if (WIN32 AND "${TARGET_LIBRARIES}" MATCHES "${CMAKE_SHARED_LIBRARY_SUFFIX}$") - install(FILES ${TARGET_LIBRARIES} DESTINATION ${CMAKE_INSTALL_BINDIR}) - else() - install(FILES ${TARGET_LIBRARIES} DESTINATION ${CMAKE_INSTALL_LIBDIR}) - endif() -endfunction() - -# -# Adds imported libraries from the given list of targets to the TileDB installation -# manifest. -# -function (install_all_target_libs TARGET_LIST) - foreach(TGT ${TARGET_LIST}) - if (TARGET ${TGT}) - install_target_libs(${TGT}) - else() - message(WARNING "Requested install for non-existant target '${TGT}'") - endif() - endforeach() -endfunction() - # Suppress warnings from find_package_handle_standard_args usage w/ *_EP targets set(FPHSA_NAME_MISMATCHED TRUE) diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index d85b52d0f008..5ac53c289836 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -22,7 +22,7 @@ option(TILEDB_CPP_API "Enables building of the TileDB C++ API." ON) option(TILEDB_CMAKE_IDE "(Used for CLion builds). Disables superbuild and sets the EP install dir." OFF) option(TILEDB_STATS "Enables internal TileDB statistics gathering." ON) option(TILEDB_STATIC "Enables building TileDB as a static library." OFF) -cmake_dependent_option(TILEDB_INSTALL_STATIC_DEPS "Installs static dependencies" ON "TILEDB_STATIC" OFF) +cmake_dependent_option(TILEDB_INSTALL_FIND_DEPENDENCIES "Enables calls to find_dependency in the exported config file. Should be disabled only when building a shared library with all dependencies statically linked." ON "NOT TILEDB_STATIC" ON) option(TILEDB_TESTS "If true, enables building the TileDB unit test suite" ON) option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 73dab473f508..c68053f5d423 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -35,9 +35,9 @@ endif() if(APPLE AND NOT DEFINED VCPKG_TARGET_TRIPLET) if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - set(VCPKG_TARGET_TRIPLET "x64-macos") + set(VCPKG_TARGET_TRIPLET "x64-osx") elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") - set(VCPKG_TARGET_TRIPLET "arm64-macos") + set(VCPKG_TARGET_TRIPLET "arm64-osx") endif() endif() diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index f9c128bb0107..14378b7f1c54 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -48,7 +48,6 @@ set(INHERITED_CMAKE_ARGS -DTILEDB_EP_BASE=${TILEDB_EP_BASE} -DTILEDB_STATS=${TILEDB_STATS} -DTILEDB_STATIC=${TILEDB_STATIC} - -DTILEDB_INSTALL_STATIC_DEPS=${TILEDB_INSTALL_STATIC_DEPS} -DTILEDB_TESTS=${TILEDB_TESTS} -DTILEDB_TOOLS=${TILEDB_TOOLS} -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in index d6e48b3481ab..0967ecc05973 100644 --- a/cmake/inputs/Config.cmake.in +++ b/cmake/inputs/Config.cmake.in @@ -10,13 +10,39 @@ @PACKAGE_INIT@ +if(@TILEDB_INSTALL_FIND_DEPENDENCIES@) # TILEDB_INSTALL_FIND_DEPENDENCIES + include(CMakeFindDependencyMacro) + find_dependency(Threads) + find_dependency(BZip2) + find_dependency(lz4) + find_dependency(spdlog) + find_dependency(ZLIB) + find_dependency(zstd) + find_dependency(unofficial-libmagic) + if(NOT WIN32) + find_dependency(OpenSSL) + endif() + if(@TILEDB_S3@) # TILEDB_S3 + find_dependency(AWSSDK COMPONENTS @AWS_SERVICES@) + endif() + if(@TILEDB_AZURE@) # TILEDB_AZURE + find_dependency(azure-storage-blobs-cpp) + endif() + if(@TILEDB_GCS@) # TILEDB_GCS + find_dependency(google_cloud_cpp_storage) + endif() + if(@TILEDB_SERIALIZATION@) # TILEDB_SERIALIZATION + find_dependency(CapnProto) + find_dependency(CURL) + endif() + if(@TILEDB_WEBP@) # TILEDB_WEBP + find_dependency(WebP) + endif() +endif() + include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") check_required_components("@PROJECT_NAME@") -# Targets required for linking against tiledb_static: -@TILEDB_STATIC_DEP_STRING@ -# -- End required targets for static - # Define a convenience all-caps variable if (NOT DEFINED TILEDB_FOUND) if (TARGET TileDB::tiledb_shared) @@ -24,4 +50,4 @@ if (NOT DEFINED TILEDB_FOUND) else() set(TILEDB_FOUND FALSE) endif() -endif() \ No newline at end of file +endif() diff --git a/ports/triplets/arm64-macos.cmake b/ports/triplets/arm64-osx.cmake similarity index 100% rename from ports/triplets/arm64-macos.cmake rename to ports/triplets/arm64-osx.cmake diff --git a/ports/triplets/x64-macos.cmake b/ports/triplets/x64-osx.cmake similarity index 100% rename from ports/triplets/x64-macos.cmake rename to ports/triplets/x64-osx.cmake diff --git a/scripts/azure-linux_mac-release.yml b/scripts/azure-linux_mac-release.yml index 5e8e27f3da30..09051edbe3f5 100755 --- a/scripts/azure-linux_mac-release.yml +++ b/scripts/azure-linux_mac-release.yml @@ -60,7 +60,7 @@ steps: # Set up arguments for bootstrap.sh BUILD_BINARIESDIRECTORY=${BUILD_BINARIESDIRECTORY:-$BUILD_REPOSITORY_LOCALPATH/build/dist} - cmake_args="-DCMAKE_INSTALL_PREFIX=${BUILD_BINARIESDIRECTORY} -DTILEDB_TESTS=OFF -DTILEDB_INSTALL_LIBDIR=lib"; + cmake_args="-DCMAKE_INSTALL_PREFIX=${BUILD_BINARIESDIRECTORY} -DTILEDB_TESTS=OFF -DTILEDB_INSTALL_LIBDIR=lib -DTILEDB_INSTALL_FIND_DEPENDENCIES=OFF"; mkdir -p ${BUILD_BINARIESDIRECTORY} # Enable TILEDB_STATIC by default diff --git a/scripts/azure-windows-release.yml b/scripts/azure-windows-release.yml index 6b36ca90d339..66351f473e21 100644 --- a/scripts/azure-windows-release.yml +++ b/scripts/azure-windows-release.yml @@ -27,7 +27,7 @@ steps: $host.SetShouldExit(3) } - $bootstrapOptions = "-EnableVerbose -DisableTests" + $bootstrapOptions = "-EnableVerbose -DisableTests -DisableInstallFindDependencies" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } diff --git a/scripts/ci/build_benchmarks.sh b/scripts/ci/build_benchmarks.sh index 7e800a5213da..18b26bcf3613 100644 --- a/scripts/ci/build_benchmarks.sh +++ b/scripts/ci/build_benchmarks.sh @@ -29,7 +29,7 @@ set -xeuo pipefail pushd $GITHUB_WORKSPACE/test/benchmarking mkdir -p build cd build -cmake -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/build/dist ../src +cmake -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/build/dist;$GITHUB_WORKSPACE/build/vcpkg_installed/$VCPKG_TARGET_TRIPLET" ../src make popd diff --git a/test/packaging/CMakeLists.txt b/test/packaging/CMakeLists.txt index 21f1ece5dab4..a1ba1bc31d9f 100644 --- a/test/packaging/CMakeLists.txt +++ b/test/packaging/CMakeLists.txt @@ -29,19 +29,12 @@ cmake_minimum_required(VERSION 3.21) -project(Test_TileDBConfigVersion_h NONE) - -if (NOT CMAKE_PREFIX_PATH) - message(FATAL_ERROR "Need parameter -DCMAKE_PREFIX_PATH=") -else() - if (NOT EXISTS "${CMAKE_PREFIX_PATH}/include/TileDB/tiledb_version.h") - message(FATAL_ERROR "Missing tiledb_version.h at ${CMAKE_PREFIX_PATH}/include/TileDB/ !") - endif() -endif() +project(Test_TileDBConfigVersion_h) # Parse version numbers out of tiledb_version.h file -file(READ "${CMAKE_PREFIX_PATH}/include/TileDB/tiledb_version.h" ver) +find_file(tiledb_version_h tiledb_version.h PATH_SUFFIXES "include/TileDB" REQUIRED) +file(READ ${tiledb_version_h} ver) string(REGEX MATCH "TILEDB_VERSION_MAJOR ([0-9]*)" _ ${ver}) set(ver_major ${CMAKE_MATCH_1}) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index bd1a3076b4de..9d39b8458854 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -551,10 +551,6 @@ target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE ${TileDB_blosc_LINK_OPT # Find OpenSSL first in case it's needed for S3 or Azure if (NOT WIN32) find_package(OpenSSL_EP REQUIRED) - if (TILEDB_VCPKG) - install_target_libs(OpenSSL::SSL) - install_target_libs(OpenSSL::Crypto) - endif() endif() # S3 dependencies @@ -565,7 +561,6 @@ if (TILEDB_S3) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE ${AWSSDK_LINK_LIBRARIES} - ${AWSSDK_EXTRA_LIBS} ) endif() @@ -575,26 +570,10 @@ if (TILEDB_AZURE) if (TILEDB_VCPKG) find_package(azure-storage-blobs-cpp CONFIG REQUIRED) - if (NOT WIN32) - if (NOT TARGET LibLZMA::LibLZMA) - find_package(LibLZMA REQUIRED) - endif() - if (NOT TARGET LibXml2::LibXml2) - find_package(LibXml2 REQUIRED) - install_target_libs(LibXml2::LibXml2) - endif() - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - LibXml2::LibXml2 - LibLZMA::LibLZMA) - endif() target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE - Azure::azure-storage-common Azure::azure-storage-blobs - Azure::azure-core ) - install_all_target_libs("Azure::azure-storage-blobs;Azure::azure-storage-common;Azure::azure-core") else() find_package(AzureCore_EP REQUIRED) find_package(AzureStorageCommon_EP REQUIRED) @@ -602,7 +581,6 @@ if (TILEDB_AZURE) if (NOT WIN32) if (NOT TARGET LibXml2::LibXml2) find_package(LibXml2 REQUIRED) - install_target_libs(LibXml2::LibXml2) endif() endif() target_link_libraries(TILEDB_CORE_OBJECTS_ILIB @@ -621,39 +599,16 @@ if (TILEDB_GCS) find_package(google_cloud_cpp_storage CONFIG REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE google-cloud-cpp::storage - google-cloud-cpp::rest_internal - absl::str_format_internal - absl::strings - absl::strings_internal - absl::crc32c - absl::time - absl::time_zone - absl::crc_internal - absl::int128 - absl::raw_logging_internal - google-cloud-cpp::common - CURL::libcurl - OpenSSL::Crypto - Crc32c::crc32c ) - install_all_target_libs("google-cloud-cpp::storage;google-cloud-cpp::rest_internal;absl::str_format_internal;absl::strings;absl::strings_internal;absl::crc32c;absl::time;absl::time_zone;absl::crc_internal;absl::int128;absl::raw_logging_internal;google-cloud-cpp::common;CURL::libcurl;OpenSSL::Crypto;Crc32c::crc32c") endif() # Libcurl -if (TILEDB_S3 OR TILEDB_GCS OR TILEDB_AZURE OR TILEDB_SERIALIZATION) - if (NOT WIN32) - find_package(Curl_EP REQUIRED) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - CURL::libcurl - ) - elseif (WIN32 AND TILEDB_SERIALIZATION) - find_package(Curl_EP REQUIRED) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - CURL::libcurl +if (TILEDB_SERIALIZATION) + find_package(Curl_EP REQUIRED) + target_link_libraries(TILEDB_CORE_OBJECTS_ILIB + INTERFACE + CURL::libcurl ) - endif() endif() # HDFS dependencies @@ -698,24 +653,11 @@ target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE BZip2::BZip2 lz4::lz4 + spdlog::spdlog ZLIB::ZLIB ${ZSTD_TARGET} libmagic ) -if (TILEDB_VCPKG) - # vcpkg uses SPDLOG_FMT_EXTERNAL=1 - - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - fmt::fmt - spdlog::spdlog - ) -else() - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - spdlog::spdlog - ) -endif() if(TILEDB_CRC32) find_package(crc32c_EP REQUIRED) @@ -735,7 +677,6 @@ if(TILEDB_WEBP) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE WebP::webp - WebP::sharpyuv ) add_definitions(-DTILEDB_WEBP) endif() @@ -751,25 +692,6 @@ endif() # Win32 specific libraries if (WIN32) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE shlwapi rpcrt4 bcrypt) - if(TILEDB_SERIALIZATION) - #ws2_32.lib, crypt32.lib needed to satisfy curl dependencies - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE ws2_32 crypt32) - endif() - - if (TILEDB_S3) - # the AWSSDK does not include links to some transitive dependencies - # ref: githubcom/awsaws-sdk-cpp/issues/1074#issuecomment-466252911 - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE userenv ws2_32 wininet winhttp bcrypt ncrypt crypt32 version secur32) - endif() - - if (TILEDB_GCS) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE ws2_32 crypt32) - endif() - - if (TILEDB_AZURE) - # WebServices, winhttp and crypt32 needed by Azure storage on windows - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE WebServices winhttp crypt32) - endif() endif() # macOS specific libraries @@ -953,89 +875,6 @@ if(TILEDB_SERIALIZATION) endif() -############################################################ -# Generated dependency information (for installation) -############################################################ - -# This section defines variables that will be used in the generation of the -# TileDBConfig.cmake file, used for find_package(TileDB). This is only required -# for static TileDB, as its dependencies are not embedded in the static object -# (as they are for the shared object). -if (TILEDB_STATIC) - # TILEDB_STATIC_DEP_STRING will be inserted literally into TileDBConfig.cmake. - set(TILEDB_STATIC_DEP_STRING) - # For get_installed_location(). - include(TileDBCommon) - - # Helper to generate CMake code - macro(append_dep_lib APPEND_LIB) - if (TARGET ${APPEND_LIB}) - get_installed_location(TARGET_LOC ${APPEND_LIB}) - cmake_path(GET TARGET_LOC FILENAME TARGET_LOC_FILENAME) - get_target_property(TARGET_LIBS ${APPEND_LIB} INTERFACE_LINK_LIBRARIES) - - string(CONCAT TILEDB_STATIC_DEP_STRING - "${TILEDB_STATIC_DEP_STRING}" - "if (NOT TARGET ${APPEND_LIB})\n" - " add_library(${APPEND_LIB} UNKNOWN IMPORTED)\n" - " set_target_properties(${APPEND_LIB} PROPERTIES IMPORTED_LOCATION \${PACKAGE_PREFIX_DIR}/lib/${TARGET_LOC_FILENAME})\n" - ) - - string(CONCAT TILEDB_STATIC_DEP_STRING - "${TILEDB_STATIC_DEP_STRING}" - "endif()\n" - ) - - endif() - endmacro() - - append_dep_lib(BZip2::BZip2) - append_dep_lib(lz4::lz4) - append_dep_lib(ZLIB::ZLIB) - append_dep_lib(${ZSTD_TARGET}) - append_dep_lib(CapnProto::capnp) - append_dep_lib(CapnProto::capnp-json) - append_dep_lib(CapnProto::kj) - append_dep_lib(OpenSSL::SSL) - append_dep_lib(OpenSSL::Crypto) - append_dep_lib(CURL::libcurl) - - foreach(tgt ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_EXTRA_LIBS}) - append_dep_lib(${tgt}) - endforeach() - - append_dep_lib(fmt::fmt) - append_dep_lib(spdlog::spdlog) - - append_dep_lib(Azure::azure-storage-blobs) - append_dep_lib(Azure::azure-storage-common) - append_dep_lib(Azure::azure-core) - append_dep_lib(LibXml2::LibXml2) - append_dep_lib(LibLZMA::LibLZMA) - - append_dep_lib(google-cloud-cpp::storage) - append_dep_lib(google-cloud-cpp::common) - append_dep_lib(google-cloud-cpp::rest_internal) - append_dep_lib(absl::str_format_internal) - append_dep_lib(absl::strings) - append_dep_lib(absl::strings_internal) - append_dep_lib(absl::crc32c) - append_dep_lib(absl::time) - append_dep_lib(absl::time_zone) - append_dep_lib(absl::crc_internal) - append_dep_lib(absl::int128) - append_dep_lib(absl::raw_logging_internal) - - append_dep_lib(libmagic) - append_dep_lib(unofficial::libmagic::libmagic) - append_dep_lib(PCRE2::8BIT) - append_dep_lib(PCRE2::POSIX) - append_dep_lib(Crc32c::crc32c) - append_dep_lib(WebP::webp) - append_dep_lib(WebP::sharpyuv) - -endif() - ############################################################ # TileDB static and shared library targets ############################################################ @@ -1263,7 +1102,6 @@ set(PROJECT_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/TileDBConfig.cmake") # This process requires these variables to be defined at this point: # * TARGETS_EXPORT_NAME # * PROJECT_NAME -# * TILEDB_STATIC_DEP_STRING configure_package_config_file( "${TILEDB_CMAKE_INPUTS_DIR}/Config.cmake.in" "${PROJECT_CONFIG}" @@ -1308,4 +1146,3 @@ configure_file( ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tiledb.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - From a15e9c6c771b3107bd8ce2dc207b2bab90fde5cf Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:10:10 +0100 Subject: [PATCH 072/456] Null count aggregate: only load validity tiles when possible. (#4513) This will only read/unfilter the validity tile when a field is required for null count aggregation only. --- TYPE: IMPROVEMENT DESC: Null count aggregate: only load validity tiles when possible. --- test/src/test-cppapi-aggregates.cc | 196 ++++++++++++++++-- tiledb/sm/query/legacy/reader.cc | 8 +- .../readers/aggregators/count_aggregator.h | 10 + .../query/readers/aggregators/iaggregator.h | 3 + .../readers/aggregators/min_max_aggregator.h | 5 + .../readers/aggregators/sum_aggregator.h | 5 + tiledb/sm/query/readers/dense_reader.cc | 35 ++-- tiledb/sm/query/readers/dense_reader.h | 13 -- tiledb/sm/query/readers/filtered_data.h | 41 ++-- tiledb/sm/query/readers/reader_base.cc | 162 +++++++++------ tiledb/sm/query/readers/reader_base.h | 140 ++++++++++--- tiledb/sm/query/readers/result_tile.h | 5 +- .../readers/sparse_global_order_reader.cc | 3 +- .../query/readers/sparse_index_reader_base.cc | 12 +- .../query/readers/sparse_index_reader_base.h | 13 -- .../sparse_unordered_with_dups_reader.cc | 3 +- 16 files changed, 471 insertions(+), 183 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 0d82018241cd..f10f57b67a3b 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -131,7 +131,9 @@ struct CppAggregatesFx { std::vector& a1_validity, const bool validate_count = true); void validate_tiles_read(Query& query, bool is_count = false); + void validate_tiles_read_null_count(Query& query); void validate_tiles_read_var(Query& query); + void validate_tiles_read_null_count_var(Query& query); void remove_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); @@ -1139,24 +1141,28 @@ void CppAggregatesFx::validate_data_var( } } -template -void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { - // Validate the number of tiles read. - auto stats = query.stats(); - +uint64_t get_stat(std::string name, std::string& stats) { // Parse num_tiles_read from the stats. std::string to_find = - "\"Context.StorageManager.Query.Reader.num_tiles_read\": "; + "\"Context.StorageManager.Query.Reader." + name + "\": "; auto start_pos = stats.find(to_find); - uint64_t num_tiles_read = 0; if (start_pos != std::string::npos) { start_pos += to_find.length(); auto end_pos = stats.find("\n", start_pos); auto str = stats.substr(start_pos, end_pos - start_pos); - num_tiles_read = std::stoull(str); + return std::stoull(str); } + return 0; +} + +template +void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { + // Validate the number of tiles read. + auto stats = query.stats(); + + uint64_t num_tiles_read = get_stat("num_tiles_read", stats); uint64_t expected_num_tiles_read; if (dense_) { // Dense has 5 tiles. If we request data or have a query condition, we'll @@ -1164,21 +1170,30 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { if (request_data_ || set_qc_) { expected_num_tiles_read = 5; } else if (set_ranges_) { - // If we request range, we split all tiles, we'll have to read all. + // If we request range, we split all tiles, we'll have to read all instead + // of using fragment metadata. expected_num_tiles_read = is_count ? 0 : 5; } else { - // One space tile has two result tiles, we'll have to read them. + // One space tile has two result tiles, we'll have to read them instead of + // using fragment metadata. expected_num_tiles_read = is_count ? 0 : 2; } } else { if (request_data_) { + // Sparse has 5 tiles * 3 (2 dims/1 attr). One of them will be filtered + // out when we set ranges. if (set_ranges_) { + // If we set ranges, we filter out one of the 5 tiles. expected_num_tiles_read = 12; } else { + // Everything is read. expected_num_tiles_read = 15; } } else { if (set_ranges_) { + // For count, we only read dimension tiles, one of which is filtered so + // we read 2 dims * 4 tiles. For the attribute, we can process 2 tiles + // with duplicates and 1 without using the fragment metadata. if (allow_dups_) { expected_num_tiles_read = is_count ? 8 : 10; } else { @@ -1186,8 +1201,13 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { } } else { if (allow_dups_) { + // No ranges, array with duplicates can do it all with fragment + // metadata only. expected_num_tiles_read = 0; } else { + // Arrays without duplicates need to run deduplication, so we read the + // dimension tiles (2 dims * 5 tiles). Only one tile for the attribute + // can be processed with fragment metadata only. expected_num_tiles_read = is_count ? 10 : 14; } } @@ -1198,23 +1218,76 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { } template -void CppAggregatesFx::validate_tiles_read_var(Query& query) { +void CppAggregatesFx::validate_tiles_read_null_count(Query& query) { // Validate the number of tiles read. auto stats = query.stats(); - // Parse num_tiles_read from the stats. - std::string to_find = - "\"Context.StorageManager.Query.Reader.num_tiles_read\": "; - auto start_pos = stats.find(to_find); - - uint64_t num_tiles_read = 0; - if (start_pos != std::string::npos) { - start_pos += to_find.length(); - auto end_pos = stats.find("\n", start_pos); - auto str = stats.substr(start_pos, end_pos - start_pos); - num_tiles_read = std::stoull(str); + uint64_t num_tiles_alloced = get_stat("tiles_allocated", stats); + uint64_t num_tiles_unfiltered = get_stat("tiles_unfiltered", stats); + uint64_t expected_num_tiles; + if (dense_) { + // Dense has 5 tiles. If we request data or have a query condition, we'll + // read all of them. + if (request_data_ || set_qc_) { + expected_num_tiles = 10; + } else if (set_ranges_) { + // If we request range, we split all tiles, we'll have to read all, but + // only nullable tiles. + expected_num_tiles = 5; + } else { + // One space tile has two result tiles, we'll have to read them. + expected_num_tiles = 2; + } + } else { + // Sparse has 5 tiles * 4 (2 dims/1 attr (fixed tile + nullable tile)). One + // of them will be filtered out when we set ranges. + if (request_data_) { + if (set_ranges_) { + // If we set ranges, we filter out one of the 5 tiles. + expected_num_tiles = 16; + } else { + // Everything is read. + expected_num_tiles = 20; + } + } else { + if (set_ranges_) { + // We read dimension tiles to filter ranges, one of which is filtered so + // we read 2 dims * 4 tiles. For the attribute, we can process 2 tiles + // with duplicates and 1 without using the fragment metadata. Note that + // we only add one per space tile for the attribute because we only read + // the validity tile. + if (allow_dups_) { + expected_num_tiles = 10; + } else { + expected_num_tiles = 11; + } + } else { + if (allow_dups_) { + // No ranges, array with duplicates can do it all with fragment + // metadata only. + expected_num_tiles = 0; + } else { + // Arrays without duplicates need to run deduplication, so we read the + // dimension tiles (2 dims * 5 tiles). Only one tile for the attribute + // can be processed with fragment metadata only. Note that we only add + // one per space tile for the attribute because we only read the + // validity tile. + expected_num_tiles = 14; + } + } + } } + CHECK(num_tiles_alloced == expected_num_tiles); + CHECK(num_tiles_unfiltered == expected_num_tiles); +} + +template +void CppAggregatesFx::validate_tiles_read_var(Query& query) { + // Validate the number of tiles read. + auto stats = query.stats(); + + uint64_t num_tiles_read = get_stat("num_tiles_read", stats); uint64_t expected_num_tiles_read; if (dense_) { // Dense has 5 tiles. If we request data or have a query condition or set @@ -1226,7 +1299,11 @@ void CppAggregatesFx::validate_tiles_read_var(Query& query) { expected_num_tiles_read = 2; } } else { + // Sparse has 4 tiles * 3 (2 dims/1 attr). One of them will be filtered + // out when we set ranges. if (request_data_) { + // We request data so everything is read. With ranges we read 3 tiles, + // without 4. 2 dims, 1 attr, means we have to multiply by 3. if (set_ranges_) { expected_num_tiles_read = 9; } else { @@ -1234,12 +1311,20 @@ void CppAggregatesFx::validate_tiles_read_var(Query& query) { } } else { if (set_ranges_) { + // To process ranges, we read 3 tiles * 2 dims at a minimum. For the + // attribute, the array with duplicates can process one tile with the + // fragment metadata, which is not the case for the array with no + // duplicates. if (allow_dups_) { expected_num_tiles_read = 8; } else { expected_num_tiles_read = 9; } } else { + // No ranges, the array with no duplicates can do it all with only + // fragment metadata. The array with no duplicates cannot because the + // cell slab structure never includes a full tile. It needs to read all + // coordinate tiles and all attribute tiles. if (allow_dups_) { expected_num_tiles_read = 0; } else { @@ -1252,6 +1337,71 @@ void CppAggregatesFx::validate_tiles_read_var(Query& query) { CHECK(num_tiles_read == expected_num_tiles_read); } +template +void CppAggregatesFx::validate_tiles_read_null_count_var(Query& query) { + // Validate the number of tiles read. + auto stats = query.stats(); + + uint64_t num_tiles_alloced = get_stat("tiles_allocated", stats); + uint64_t num_tiles_unfiltered = get_stat("tiles_unfiltered", stats); + uint64_t expected_num_tiles; + if (dense_) { + // Dense has 5 tiles. If we request data or have a query condition we'll + // read all of them. + if (request_data_ || set_qc_) { + expected_num_tiles = 15; + } else if (set_ranges_) { + // If we set ranges, we only read the validity tiles. + expected_num_tiles = 5; + } else { + // One space tile has two result tiles, we'll have to read them. + expected_num_tiles = 2; + } + } else { + // Sparse has 4 tiles * 5 (2 dims/1 attr (offset tile + var tile + nullable + // tile)). One of them will be filtered out when we set ranges. + if (request_data_) { + // We request data so everything is read. With ranges we read 3 tiles, + // without 4. 2 dims, 1 nullable var attr, means we have to multiply by 5. + if (set_ranges_) { + expected_num_tiles = 15; + } else { + expected_num_tiles = 20; + } + } else { + if (set_ranges_) { + // To process ranges, we read 3 tiles * 2 dims at a minimum. For the + // attribute, the array with duplicates can process one tile with the + // fragment metadata, which is not the case for the array with no + // duplicates. Note that we only add one per space tile for the + // attribute because we only read the validity tile. + if (allow_dups_) { + expected_num_tiles = 8; + } else { + // 3 * 2 for dims, 3 * 1 for attr. + expected_num_tiles = 9; + } + } else { + // No ranges, the array with no duplicates can do it all with only + // fragment metadata. The array with no duplicates cannot because the + // cell slab structure never includes a full tile. It needs to read all + // coordinate tiles and all attribute tiles. Note that we only add one + // per space tile for the attribute because we only read the validity + // tile. + if (allow_dups_) { + expected_num_tiles = 0; + } else { + // 4 * 2 for dims, 4 * 1 for attr. + expected_num_tiles = 12; + } + } + } + } + + CHECK(num_tiles_alloced == expected_num_tiles); + CHECK(num_tiles_unfiltered == expected_num_tiles); +} + template void CppAggregatesFx::remove_array(const std::string& array_name) { if (!is_array(array_name)) @@ -1973,6 +2123,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } CppAggregatesFx::validate_tiles_read(query); + CppAggregatesFx::validate_tiles_read_null_count(query); } } } @@ -2068,6 +2219,7 @@ TEST_CASE_METHOD( } validate_tiles_read_var(query); + validate_tiles_read_null_count_var(query); } } } diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 2022fa448341..ebba83157217 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -1664,8 +1664,8 @@ Status Reader::compute_result_coords( std::vector timestamps = {constants::timestamps}; load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), timestamps); - RETURN_CANCEL_OR_ERROR( - read_and_unfilter_attribute_tiles(timestamps, tmp_result_tiles)); + RETURN_CANCEL_OR_ERROR(read_and_unfilter_attribute_tiles( + NameToLoad::from_string_vec(timestamps), tmp_result_tiles)); } // Read and unfilter delete timestamps. @@ -1674,8 +1674,8 @@ Status Reader::compute_result_coords( load_tile_offsets( read_state_.partitioner_.subarray().relevant_fragments(), delete_timestamps); - RETURN_CANCEL_OR_ERROR( - read_and_unfilter_attribute_tiles(delete_timestamps, tmp_result_tiles)); + RETURN_CANCEL_OR_ERROR(read_and_unfilter_attribute_tiles( + NameToLoad::from_string_vec(delete_timestamps), tmp_result_tiles)); } // Compute the read coordinates for all fragments for each subarray range. diff --git a/tiledb/sm/query/readers/aggregators/count_aggregator.h b/tiledb/sm/query/readers/aggregators/count_aggregator.h index a2cd3c8eded7..514602c844a2 100644 --- a/tiledb/sm/query/readers/aggregators/count_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/count_aggregator.h @@ -149,6 +149,11 @@ class CountAggregator : public CountAggregatorBase { std::string aggregate_name() override { return constants::aggregate_count_str; } + + /** Returns if the aggregation is for validity only data. */ + bool aggregation_validity_only() override { + return false; + } }; class NullCountAggregator : public CountAggregatorBase, @@ -174,6 +179,11 @@ class NullCountAggregator : public CountAggregatorBase, return constants::aggregate_null_count_str; } + /** Returns if the aggregation is for validity only data. */ + bool aggregation_validity_only() override { + return true; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query/readers/aggregators/iaggregator.h b/tiledb/sm/query/readers/aggregators/iaggregator.h index 9130b535a60f..a8fb11d9dda9 100644 --- a/tiledb/sm/query/readers/aggregators/iaggregator.h +++ b/tiledb/sm/query/readers/aggregators/iaggregator.h @@ -72,6 +72,9 @@ class IAggregator { /** Returns if the aggregation is nullable or not. */ virtual bool aggregation_nullable() = 0; + /** Returns if the aggregation is for validity only data. */ + virtual bool aggregation_validity_only() = 0; + /** * Validate the result buffer. * diff --git a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h index e5350b2a678d..48d8f67c87c9 100644 --- a/tiledb/sm/query/readers/aggregators/min_max_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/min_max_aggregator.h @@ -150,6 +150,11 @@ class ComparatorAggregator : public ComparatorAggregatorBase, return ComparatorAggregatorBase::field_info_.is_nullable_; } + /** Returns if the aggregation is for validity only data. */ + bool aggregation_validity_only() override { + return false; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return false; diff --git a/tiledb/sm/query/readers/aggregators/sum_aggregator.h b/tiledb/sm/query/readers/aggregators/sum_aggregator.h index 7b15180061ac..f1a6eff93fdb 100644 --- a/tiledb/sm/query/readers/aggregators/sum_aggregator.h +++ b/tiledb/sm/query/readers/aggregators/sum_aggregator.h @@ -86,6 +86,11 @@ class SumWithCountAggregator : public InputFieldValidator, return field_info_.is_nullable_; } + /** Returns if the aggregation is for validity only data. */ + bool aggregation_validity_only() override { + return false; + } + /** Returns if the aggregate needs to be recomputed on overflow. */ bool need_recompute_on_overflow() override { return true; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index b32237a89ebd..5149a7121638 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -337,12 +337,11 @@ Status DenseReader::dense_read() { } // Compute attribute names to load and copy. - std::unordered_set condition_names; if (condition_.has_value()) { - condition_names = condition_->field_names(); + qc_loaded_attr_names_set_ = condition_->field_names(); } - auto&& [names, var_names] = field_names_to_process(condition_names); + auto&& [names, var_names] = field_names_to_process(qc_loaded_attr_names_set_); // Pre-load all attribute offsets into memory for attributes // in query condition to be read. @@ -393,7 +392,7 @@ Status DenseReader::dense_read() { // Get result tiles to process on this iteration. t_end = compute_space_tiles_end( names, - condition_names, + qc_loaded_attr_names_set_, subarray, t_start, result_space_tiles, @@ -420,7 +419,7 @@ Status DenseReader::dense_read() { subarray, t_start, t_end, - condition_names, + qc_loaded_attr_names_set_, tile_extents, tile_subarrays, tile_offsets, @@ -440,7 +439,6 @@ Status DenseReader::dense_read() { // Process all attributes, names starts with the query condition names to // clear the memory. Also, a name in names might not be in the user buffers // so we might skip the copy but still clear the memory. - std::vector to_read(1); for (auto& name : names) { if (name == constants::coords || array_schema_.is_dim(name)) { continue; @@ -449,7 +447,7 @@ Status DenseReader::dense_read() { // Get the tiles to load for this attribute. auto result_tiles = result_tiles_to_load( name, - condition_names, + qc_loaded_attr_names_set_, subarray, t_start, t_end, @@ -459,8 +457,10 @@ Status DenseReader::dense_read() { std::vector filtered_data; // Read and unfilter tiles. - to_read[0] = name; - filtered_data = std::move(read_attribute_tiles(to_read, result_tiles)); + bool validity_only = null_count_aggregate_only(name); + std::vector to_load; + to_load.emplace_back(name, validity_only); + filtered_data = std::move(read_attribute_tiles(to_load, result_tiles)); if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); @@ -473,15 +473,15 @@ Status DenseReader::dense_read() { [&, filtered_data = std::move(filtered_data), name, + validity_only, t_start, t_end, subarray_start_cell, subarray_end_cell, num_range_threads, - result_tiles, - condition_names]() { + result_tiles]() { // Unfilter tiles. - RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); + RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); // Only copy names that are present in the user buffers. if (buffers_.count(name) != 0) { @@ -932,8 +932,8 @@ Status DenseReader::apply_query_condition( tile_subarrays); // Read and unfilter query condition attributes. - std::vector filtered_data = - read_attribute_tiles(qc_names, result_tiles); + std::vector filtered_data = read_attribute_tiles( + NameToLoad::from_string_vec(qc_names), result_tiles); if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); @@ -956,7 +956,7 @@ Status DenseReader::apply_query_condition( // Unfilter tiles. for (auto& name : qc_names) { - RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); + RETURN_NOT_OK(unfilter_tiles(name, false, result_tiles)); } if (stride == UINT64_MAX) { @@ -1897,6 +1897,7 @@ Status DenseReader::aggregate_tiles( const auto cell_size = var_size ? constants::cell_var_offset_size : array_schema_.cell_size(name); auto& aggregates = aggregates_[name]; + const bool validity_only = null_count_aggregate_only(name); // Cache tile tuples. std::vector tile_tuples(frag_domains.size()); @@ -1949,7 +1950,7 @@ Status DenseReader::aggregate_tiles( if (stride == 1) { // Compute aggregate. AggregateBuffer aggregate_buffer{make_aggregate_buffer( - var_size, + var_size & !validity_only, nullable, cell_size, iter.pos_in_tile() + start, @@ -1965,7 +1966,7 @@ Status DenseReader::aggregate_tiles( // Compute aggregate. auto start_cell = iter.pos_in_tile() + (start + i) * stride; AggregateBuffer aggregate_buffer{make_aggregate_buffer( - var_size, + var_size & !validity_only, nullable, cell_size, start_cell, diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 81b61901c1ec..5e6e90d0f657 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -278,19 +278,6 @@ class DenseReader : public ReaderBase, public IQueryStrategy { ResultTile::TileTuple* tile_tuple, optional bitmap_data); - /** Returns wether the field is for aggregation only or not. */ - bool aggregate_only(const std::string& name) const { - if (condition_.has_value() && condition_->field_names().count(name) != 0) { - return false; - } - - if (buffers_.count(name) != 0) { - return false; - } - - return true; - } - /** * Returns wether or not we can aggregate the tile with only the fragment * metadata. diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index b1fca72a2cd6..abb2f8884ac5 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -153,9 +153,10 @@ class FilteredData { * @param result_tiles Sorted list (per fragment/tile index) of result tiles. * Only the fragment index and tile index of each result tiles is used here. * Nothing is mutated inside of the vector. - * @param name Name of the attribute. - * @param var_sized Is the attribute var sized? - * @param nullable Is the attribute nullable? + * @param name Name of the field. + * @param var_sized Is the field var sized? + * @param nullable Is the field nullable? + * @param validity_only Is the field read for validity only? * @param storage_manager Storage manager. * @param read_tasks Read tasks to queue new tasks on for new data blocks. */ @@ -169,6 +170,7 @@ class FilteredData { const std::string& name, const bool var_sized, const bool nullable, + const bool validity_only, StorageManager* storage_manager, std::vector& read_tasks) : name_(name) @@ -181,6 +183,8 @@ class FilteredData { return; } + uint64_t tiles_allocated = 0; + // Store data on the datablock in progress for fixed, var and nullable data. std::optional current_frag_idx{nullopt}; storage_size_t current_fixed_offset{0}; @@ -199,18 +203,22 @@ class FilteredData { // Make new blocks, if required as we go for fixed, var and nullable data. auto fragment{fragment_metadata[rt->frag_idx()].get()}; - make_new_block_if_required( - fragment, - min_batch_size, - max_batch_size, - min_batch_gap, - current_frag_idx, - current_fixed_offset, - current_fixed_size, - rt, - TileType::FIXED); - - if (var_sized) { + if (!validity_only) { + tiles_allocated++; + make_new_block_if_required( + fragment, + min_batch_size, + max_batch_size, + min_batch_gap, + current_frag_idx, + current_fixed_offset, + current_fixed_size, + rt, + TileType::FIXED); + } + + if (var_sized && !validity_only) { + tiles_allocated++; make_new_block_if_required( fragment, min_batch_size, @@ -224,6 +232,7 @@ class FilteredData { } if (nullable) { + tiles_allocated++; make_new_block_if_required( fragment, min_batch_size, @@ -258,6 +267,8 @@ class FilteredData { queue_last_block_for_read(TileType::NULLABLE); } + reader.stats()->add_counter("tiles_allocated", tiles_allocated); + current_fixed_data_block_ = fixed_data_blocks_.begin(); current_var_data_block_ = var_data_blocks_.begin(); current_nullable_data_block_ = nullable_data_blocks_.begin(); diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index a187e80063e6..9f73eec3368f 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -564,7 +564,7 @@ void ReaderBase::load_processed_conditions() { } Status ReaderBase::read_and_unfilter_attribute_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const { // The filtered data here contains the memory allocations for all of the // filtered data that is read by `read_attribute_tiles`. To prevent @@ -583,7 +583,8 @@ Status ReaderBase::read_and_unfilter_attribute_tiles( // been used and the tiles are unfiltered so the data can be deleted. auto filtered_data{read_attribute_tiles(names, result_tiles)}; for (auto& name : names) { - RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); + RETURN_NOT_OK( + unfilter_tiles(name.name(), name.validity_only(), result_tiles)); } return Status::Ok(); @@ -596,14 +597,14 @@ Status ReaderBase::read_and_unfilter_coordinate_tiles( // information about the lifetime of this object. auto filtered_data{read_coordinate_tiles(names, result_tiles)}; for (auto& name : names) { - RETURN_NOT_OK(unfilter_tiles(name, result_tiles)); + RETURN_NOT_OK(unfilter_tiles(name, false, result_tiles)); } return Status::Ok(); } std::vector ReaderBase::read_attribute_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_attribute_tiles"); return read_tiles(names, result_tiles); @@ -613,11 +614,11 @@ std::vector ReaderBase::read_coordinate_tiles( const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_coordinate_tiles"); - return read_tiles(names, result_tiles); + return read_tiles(NameToLoad::from_string_vec(names), result_tiles); } std::vector ReaderBase::read_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_tiles"); std::vector filtered_data; @@ -632,7 +633,10 @@ std::vector ReaderBase::read_tiles( filtered_data.reserve(names.size()); // Run all attributes independently. - for (auto name : names) { + for (auto n : names) { + auto& name = n.name(); + auto val_only = n.validity_only(); + // Create the filtered data blocks. This will also kick off the read for the // data blocks right after the memory is allocated so that we can optimize // read and memory allocations. @@ -648,6 +652,7 @@ std::vector ReaderBase::read_tiles( name, var_sized, nullable, + val_only, storage_manager_, read_tasks); @@ -665,7 +670,7 @@ std::vector ReaderBase::read_tiles( // Construct a TileSizes class. ResultTile::TileSizes tile_sizes{ - fragment, name, var_sized, nullable, tile_idx}; + fragment, name, var_sized, nullable, val_only, tile_idx}; // Construct a tile data class. // See the explanation in 'read_and_unfilter_attribute_tiles' for more @@ -677,8 +682,12 @@ std::vector ReaderBase::read_tiles( // 'unfilter_tiles' so that the filter pipeline can stop using the // 'ResultTile' object to get access to the filtered data. ResultTile::TileData tile_data{ - filtered_data.back().fixed_filtered_data(fragment.get(), tile), - filtered_data.back().var_filtered_data(fragment.get(), tile), + val_only ? + nullptr : + filtered_data.back().fixed_filtered_data(fragment.get(), tile), + val_only ? + nullptr : + filtered_data.back().var_filtered_data(fragment.get(), tile), filtered_data.back().nullable_filtered_data(fragment.get(), tile)}; // Initialize the tile(s) @@ -714,6 +723,7 @@ std::vector ReaderBase::read_tiles( tuple, optional, optional> ReaderBase::load_tile_chunk_data( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable, @@ -735,25 +745,29 @@ ReaderBase::load_tile_chunk_data( } const auto t = &tile_tuple->fixed_tile(); - const auto t_var = var_size ? &tile_tuple->var_tile() : nullptr; + const auto t_var = + var_size && !validity_only ? &tile_tuple->var_tile() : nullptr; const auto t_validity = nullable ? &tile_tuple->validity_tile() : nullptr; uint64_t unfiltered_tile_size = 0, unfiltered_tile_var_size = 0, unfiltered_tile_validity_size = 0; const FilterPipeline& filters = array_schema_.filters(name); - if (!var_size || - !filters.skip_offsets_filtering(t_var->type(), array_schema_.version())) { + if (!validity_only) { + if (!var_size || !filters.skip_offsets_filtering( + t_var->type(), array_schema_.version())) { + if (var_size) { + unfiltered_tile_size = t->load_offsets_chunk_data(tile_chunk_data); + } else { + unfiltered_tile_size = t->load_chunk_data(tile_chunk_data); + } + } + if (var_size) { - unfiltered_tile_size = t->load_offsets_chunk_data(tile_chunk_data); - } else { - unfiltered_tile_size = t->load_chunk_data(tile_chunk_data); + unfiltered_tile_var_size = t_var->load_chunk_data(tile_chunk_var_data); } } - if (var_size) { - unfiltered_tile_var_size = t_var->load_chunk_data(tile_chunk_var_data); - } if (nullable) { unfiltered_tile_validity_size = t_validity->load_chunk_data(tile_chunk_validity_data); @@ -780,6 +794,7 @@ Status ReaderBase::zip_tile_coordinates( Status ReaderBase::post_process_unfiltered_tile( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable) const { @@ -802,7 +817,7 @@ Status ReaderBase::post_process_unfiltered_tile( throw_if_not_ok(zip_tile_coordinates(name, &t)); - if (var_size) { + if (var_size && !validity_only) { auto& t_var = tile_tuple->var_tile(); t_var.clear_filtered_buffer(); throw_if_not_ok(zip_tile_coordinates(name, &t_var)); @@ -820,6 +835,7 @@ Status ReaderBase::post_process_unfiltered_tile( Status ReaderBase::unfilter_tiles( const std::string& name, + const bool validity_only, const std::vector& result_tiles) const { const auto stat_type = (array_schema_.is_attr(name)) ? "unfilter_attr_tiles" : "unfilter_coord_tiles"; @@ -863,6 +879,7 @@ Status ReaderBase::unfilter_tiles( auto&& [st, tile_size, tile_var_size, tile_validity_size] = load_tile_chunk_data( name, + validity_only, result_tiles[i], var_size, nullable, @@ -890,6 +907,7 @@ Status ReaderBase::unfilter_tiles( [&](uint64_t i, uint64_t range_thread_idx) { return unfilter_tile( name, + validity_only, result_tiles[i], var_size, nullable, @@ -904,7 +922,7 @@ Status ReaderBase::unfilter_tiles( // Perform required post-processing of unfiltered tiles for (size_t i = 0; i < num_tiles; i++) { RETURN_NOT_OK(post_process_unfiltered_tile( - name, result_tiles[i], var_size, nullable)); + name, validity_only, result_tiles[i], var_size, nullable)); } return Status::Ok(); @@ -912,6 +930,7 @@ Status ReaderBase::unfilter_tiles( Status ReaderBase::unfilter_tile( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable, @@ -935,7 +954,7 @@ Status ReaderBase::unfilter_tile( } auto t = &tile_tuple->fixed_tile(); - auto t_var = var_size ? &tile_tuple->var_tile() : nullptr; + auto t_var = var_size && !validity_only ? &tile_tuple->var_tile() : nullptr; auto t_validity = nullable ? &tile_tuple->validity_tile() : nullptr; FilterPipeline fixed_filters; @@ -943,18 +962,20 @@ Status ReaderBase::unfilter_tile( FilterPipeline validity_filters; // Create our filter pipelines - if (!var_size) { - fixed_filters = array_schema_.filters(name); - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( - &fixed_filters, array_->get_encryption_key())); - } else { - fixed_filters = array_schema_.cell_var_offsets_filters(); - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( - &fixed_filters, array_->get_encryption_key())); + if (!validity_only) { + if (!var_size) { + fixed_filters = array_schema_.filters(name); + RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + &fixed_filters, array_->get_encryption_key())); + } else { + fixed_filters = array_schema_.cell_var_offsets_filters(); + RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + &fixed_filters, array_->get_encryption_key())); - var_filters = array_schema_.filters(name); - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( - &var_filters, array_->get_encryption_key())); + var_filters = array_schema_.filters(name); + RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + &var_filters, array_->get_encryption_key())); + } } if (nullable) { @@ -964,48 +985,52 @@ Status ReaderBase::unfilter_tile( } bool skip_offsets_filtering = false; - if (var_size) { + if (var_size && !validity_only) { skip_offsets_filtering = var_filters.skip_offsets_filtering( t_var->type(), array_schema_.version()); } auto concurrency_level = storage_manager_->compute_tp()->concurrency_level(); - // Unfiltered fixed data - if (!skip_offsets_filtering && - thread_idx <= tile_chunk_fixed_data.filtered_chunks_.size() - 1) { - // Compute chunk boundaries - auto&& [t_min, t_max] = compute_chunk_min_max( - tile_chunk_fixed_data.chunk_offsets_.size(), num_threads, thread_idx); - - // Reverse the tile filters. - RETURN_NOT_OK(fixed_filters.run_reverse( - stats_, - t, - nullptr, - tile_chunk_fixed_data, - t_min, - t_max, - concurrency_level, - storage_manager_->config())); - } + if (!validity_only) { + // Unfiltered fixed data + if (!skip_offsets_filtering && + thread_idx <= tile_chunk_fixed_data.filtered_chunks_.size() - 1) { + // Compute chunk boundaries + auto&& [t_min, t_max] = compute_chunk_min_max( + tile_chunk_fixed_data.chunk_offsets_.size(), num_threads, thread_idx); + + // Reverse the tile filters. + stats_->add_counter("tiles_unfiltered", 1); + RETURN_NOT_OK(fixed_filters.run_reverse( + stats_, + t, + nullptr, + tile_chunk_fixed_data, + t_min, + t_max, + concurrency_level, + storage_manager_->config())); + } - // Prevent processing past the end of chunks in case there are more - // threads than chunks. - if (var_size && - thread_idx <= tile_chunk_var_data.filtered_chunks_.size() - 1) { - auto&& [tvar_min, tvar_max] = compute_chunk_min_max( - tile_chunk_var_data.chunk_offsets_.size(), num_threads, thread_idx); - // Reverse the filters of tile var data - RETURN_NOT_OK(var_filters.run_reverse( - stats_, - t_var, - skip_offsets_filtering ? t : nullptr, - tile_chunk_var_data, - tvar_min, - tvar_max, - concurrency_level, - storage_manager_->config())); + // Prevent processing past the end of chunks in case there are more + // threads than chunks. + if (var_size && + thread_idx <= tile_chunk_var_data.filtered_chunks_.size() - 1) { + auto&& [tvar_min, tvar_max] = compute_chunk_min_max( + tile_chunk_var_data.chunk_offsets_.size(), num_threads, thread_idx); + // Reverse the filters of tile var data + stats_->add_counter("tiles_unfiltered", 1); + RETURN_NOT_OK(var_filters.run_reverse( + stats_, + t_var, + skip_offsets_filtering ? t : nullptr, + tile_chunk_var_data, + tvar_min, + tvar_max, + concurrency_level, + storage_manager_->config())); + } } // Prevent processing past the end of chunks in case there are more @@ -1019,6 +1044,7 @@ Status ReaderBase::unfilter_tile( thread_idx); // Reverse the tile validity filters. + stats_->add_counter("tiles_unfiltered", 1); RETURN_NOT_OK(validity_filters.run_reverse( stats_, t_validity, diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index 109e29ac2668..651a19855940 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -64,6 +64,60 @@ class ReaderBase : public StrategyBase { /* TYPE DEFINITIONS */ /* ********************************* */ + class NameToLoad { + public: + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + /** Return a NameToLoad vector from a string vector. */ + static std::vector from_string_vec( + const std::vector& names_to_load) { + std::vector ret; + ret.reserve(names_to_load.size()); + for (auto& name : names_to_load) { + ret.emplace_back(name); + } + + return ret; + } + + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Constructor. */ + NameToLoad(std::string name, bool validity_only = false) + : name_(name) + , validity_only_(validity_only) { + } + + /* ********************************* */ + /* PUBLIC METHODS */ + /* ********************************* */ + + /** @returns the name to load. */ + const std::string& name() const { + return name_; + } + + /** @returns if the field needs to be loaded for validity only. */ + bool validity_only() const { + return validity_only_; + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Field name to load. */ + const std::string name_; + + /** Load validity only for the field. */ + const bool validity_only_; + }; + /** The state for a read query. */ struct ReadState { /** @@ -291,6 +345,35 @@ class ReaderBase : public StrategyBase { /* PROTECTED METHODS */ /* ********************************* */ + /** Returns wether the field is for aggregation only or not. */ + bool aggregate_only(const std::string& name) const { + if (qc_loaded_attr_names_set_.count(name) != 0) { + return false; + } + + if (buffers_.count(name) != 0) { + return false; + } + + return true; + } + + /** Returns wether the field is for null count aggregation only or not. */ + bool null_count_aggregate_only(const std::string& name) const { + if (!aggregate_only(name)) { + return false; + } + + auto& aggregates = aggregates_.at(name); + for (auto& aggregate : aggregates) { + if (!aggregate->aggregation_validity_only()) { + return false; + } + } + + return true; + } + /** * Returns if we need to process partial timestamp condition for this * fragment. @@ -301,9 +384,9 @@ class ReaderBase : public StrategyBase { bool process_partial_timestamps(FragmentMetadata& frag_meta) const; /** - * Deletes the tiles on the input attribute/dimension from the result tiles. + * Deletes the tiles on the input field from the result tiles. * - * @param name The attribute/dimension name. + * @param name The field name. * @param result_tiles The result tiles to delete from. * @param min_result_tile The minimum index to start clearing tiles at. * @return void @@ -388,11 +471,11 @@ class ReaderBase : public StrategyBase { } /** - * Loads tile offsets for each attribute/dimension name into + * Loads tile offsets for each field name into * their associated element in `fragment_metadata_`. * * @param relevant_fragments List of relevant fragments. - * @param names The attribute/dimension names. + * @param names The field names. * @return Status */ void load_tile_offsets( @@ -421,23 +504,23 @@ class ReaderBase : public StrategyBase { Status add_delete_timestamps_condition(); /** - * Loads tile var sizes for each attribute/dimension name into + * Loads tile var sizes for each field name into * their associated element in `fragment_metadata_`. * * @param relevant_fragments List of relevant fragments. - * @param names The attribute/dimension names. + * @param names The field names. */ void load_tile_var_sizes( const RelevantFragments& relevant_fragments, const std::vector& names); /* - * Loads tile metadata for each attribute/dimension name into + * Loads tile metadata for each field name into * their associated element in `fragment_metadata_`. This is done for * attributes with aggregates. * * @param relevant_fragments List of relevant fragments. - * @param names The attribute/dimension names. + * @param names The field names. */ void load_tile_metadata( const RelevantFragments& relevant_fragments, @@ -460,7 +543,7 @@ class ReaderBase : public StrategyBase { * @return Status. */ Status read_and_unfilter_attribute_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const; /** @@ -488,7 +571,7 @@ class ReaderBase : public StrategyBase { * @return Filtered data blocks. */ std::vector read_attribute_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const; /** @@ -514,35 +597,39 @@ class ReaderBase : public StrategyBase { * Concurrently executes across each name in `names` and each result tile * in 'result_tiles'. * - * @param names The attribute/dimension names. + * @param names The field names. * @param result_tiles The retrieved tiles will be stored inside the * `ResultTile` instances in this vector. + * @param validity_only Is the field read for validity only. * @return Filtered data blocks. */ std::vector read_tiles( - const std::vector& names, + const std::vector& names, const std::vector& result_tiles) const; /** - * Filters the tiles on a particular attribute/dimension from all input + * Filters the tiles on a particular field from all input * fragments based on the tile info in `result_tiles`. * - * @param name Attribute/dimension whose tiles will be unfiltered. + * @param name Field whose tiles will be unfiltered. + * @param validity_only Unfilter for the validity tile only? * @param result_tiles Vector containing the tiles to be unfiltered. * @return Status */ Status unfilter_tiles( const std::string& name, + const bool validity_only, const std::vector& result_tiles) const; /** * Unfilter a specific range of chunks in tile * - * @param name Attribute/dimension the tile belong to. + * @param name field the tile belong to. + * @param validity_only Unfilter the validity tile only. * @param tile Offsets tile to be unfiltered. - * @param var_size True if the attribute/dimension is var-sized, false + * @param var_size True if the field is var-sized, false * otherwise - * @param nullable True if the attribute/dimension is nullable, false + * @param nullable True if the field is nullable, false * otherwise * @param range_thread_idx Current range thread index. * @param num_range_threads Total number of range threads. @@ -554,6 +641,7 @@ class ReaderBase : public StrategyBase { */ Status unfilter_tile( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable, @@ -602,7 +690,7 @@ class ReaderBase : public StrategyBase { * https://github.com/TileDB-Inc/TileDB/issues/1053. For format version > 4, * a tile never stores coordinates * - * @param name Attribute/dimension the tile belongs to. + * @param name field the tile belongs to. * @param tile Tile to zip the coordinates if needed. * @return Status */ @@ -612,11 +700,12 @@ class ReaderBase : public StrategyBase { * Reads the chunk data of all tile buffers and stores them in a data * structure together with the offsets between them * - * @param name Attribute/dimension the tile belong to. + * @param name field the tile belong to. + * @param validity_only Is the field read for validity only. * @param tile Offsets tile to be unfiltered. - * @param var_size True if the attribute/dimension is var-sized, false + * @param var_size True if the field is var-sized, false * otherwise - * @param nullable True if the attribute/dimension is nullable, false + * @param nullable True if the field is nullable, false * otherwise * @param tile_chunk_data Tile/offsets tile chunk info, buffers and * offsets @@ -629,6 +718,7 @@ class ReaderBase : public StrategyBase { tuple, optional, optional> load_tile_chunk_data( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable, @@ -639,11 +729,12 @@ class ReaderBase : public StrategyBase { /** * Perform some necessary post-processing on a tile that was just unfiiltered * - * @param name Attribute/dimension the tile belong to. + * @param name field the tile belong to. + * @param validity_only Is the field read for validity only. * @param tile Offsets tile that was just unfiltered. - * @param var_size True if the attribute/dimension is var-sized, false + * @param var_size True if the field is var-sized, false * otherwise - * @param nullable True if the attribute/dimension is nullable, false + * @param nullable True if the field is nullable, false * otherwise * @param unfiltered_tile_size Size of the unfiltered tile buffer * @param unfiltered_tile_var_size Size of the unfiltered tile_var buffer @@ -653,6 +744,7 @@ class ReaderBase : public StrategyBase { */ Status post_process_unfiltered_tile( const std::string& name, + const bool validity_only, ResultTile* const tile, const bool var_size, const bool nullable) const; diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index f67bb41fb4d7..192fcc3332e2 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -91,11 +91,12 @@ class ResultTile { std::string name, const bool var_size, const bool nullable, + const bool validity_only, const uint64_t tile_idx) - : tile_size_(fragment->tile_size(name, tile_idx)) + : tile_size_(validity_only ? 0 : fragment->tile_size(name, tile_idx)) , tile_persisted_size_(fragment->persisted_tile_size(name, tile_idx)) , tile_var_size_( - var_size ? + var_size && !validity_only ? std::optional(fragment->tile_var_size(name, tile_idx)) : std::nullopt) , tile_var_persisted_size_( diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index a78452f9caa6..b53666197cde 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -2117,6 +2117,7 @@ void SparseGlobalOrderReader::process_aggregates( std::vector& cell_offsets, std::vector& result_cell_slabs) { auto& aggregates = aggregates_[name]; + const bool validity_only = null_count_aggregate_only(name); bool var_sized = false; bool nullable = false; @@ -2163,7 +2164,7 @@ void SparseGlobalOrderReader::process_aggregates( // Compute aggregate. AggregateBuffer aggregate_buffer{make_aggregate_buffer( name, - var_sized, + var_sized && !validity_only, nullable, cell_val_num, min_pos, diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 5299be7fcce6..f3c1e326e08e 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -547,8 +547,14 @@ Status SparseIndexReaderBase::read_and_unfilter_coords( std::back_inserter(attr_to_load)); // Read and unfilter attribute tiles. + std::vector to_load; + to_load.reserve(attr_to_load.size()); + for (auto& name : attr_to_load) { + to_load.emplace_back(name); + } + RETURN_CANCEL_OR_ERROR( - read_and_unfilter_attribute_tiles(attr_to_load, result_tiles)); + read_and_unfilter_attribute_tiles(to_load, result_tiles)); logger_->debug("Done reading and unfiltering coords tiles"); return Status::Ok(); @@ -839,7 +845,7 @@ std::vector SparseIndexReaderBase::read_and_unfilter_attributes( auto timer_se = stats_->start_timer("read_and_unfilter_attributes"); const uint64_t memory_budget = available_memory(); - std::vector names_to_read; + std::vector names_to_read; std::vector names_to_copy; uint64_t memory_used = 0; while (*buffer_idx < names.size()) { @@ -858,7 +864,7 @@ std::vector SparseIndexReaderBase::read_and_unfilter_attributes( // We only read attributes, so dimensions have 0 cost. if (attr_mem_usage != 0) { - names_to_read.emplace_back(name); + names_to_read.emplace_back(name, null_count_aggregate_only(name)); } names_to_copy.emplace_back(name); diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index 4b7ac9118825..c87cc445b557 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -671,19 +671,6 @@ class SparseIndexReaderBase : public ReaderBase { template void apply_query_condition(std::vector& result_tiles); - /** Returns wether the field is for aggregation only or not. */ - bool aggregate_only(const std::string& name) const { - if (qc_loaded_attr_names_set_.count(name) != 0) { - return false; - } - - if (buffers_.count(name) != 0) { - return false; - } - - return true; - } - /** * Read and unfilter as many attributes as can fit in the memory budget and * return the names loaded in 'names_to_copy'. Also keep the 'buffer_idx' diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 5308d354b556..bcb055d9cf0f 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -1931,6 +1931,7 @@ void SparseUnorderedWithDupsReader::process_aggregates( std::vector& cell_offsets, std::vector& result_tiles) { auto& aggregates = aggregates_[name]; + const bool validity_only = null_count_aggregate_only(name); bool var_sized = false; bool nullable = false; @@ -1988,7 +1989,7 @@ void SparseUnorderedWithDupsReader::process_aggregates( // Compute aggregate. AggregateBuffer aggregate_buffer{make_aggregate_buffer( name, - var_sized, + var_sized && !validity_only, nullable, cell_val_num, count_bitmap, From a4b6571553c7d57e77d77c1203332d972a0c28e0 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 Nov 2023 13:05:24 +0200 Subject: [PATCH 073/456] Move the manylinux CI job to GitHub Actions. (#4491) With this PR, the last CI job moves off Azure DevOps. --- TYPE: NO_HISTORY --- .github/workflows/ci-linux_mac.yml | 30 ++- .github/workflows/full-ci.yml | 12 ++ azure-pipelines.yml | 30 --- scripts/azure-linux_mac.yml | 283 ----------------------------- test/src/unit-capi-query.cc | 5 + 5 files changed, 46 insertions(+), 314 deletions(-) delete mode 100755 scripts/azure-linux_mac.yml diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index ba204063f7a0..f16fabbf4534 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -14,7 +14,7 @@ on: type: string bootstrap_args: default: '' - required: true + required: false type: string matrix_image: default: '' @@ -28,6 +28,14 @@ on: default: 'g++' required: false type: string + matrix_compiler_cflags: + default: '' + required: false + type: string + matrix_compiler_cxxflags: + default: '' + required: false + type: string timeout: default: 90 description: 'Job timeout (minutes)' @@ -38,13 +46,23 @@ on: description: 'Enable Address Sanitizer' required: false type: boolean + manylinux: + default: false + description: 'Enable manylinux builds' + required: false + type: boolean env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF TILEDB_CI_BACKEND: ${{ inputs.ci_backend }} TILEDB_CI_OS: runner.os + # Installing Python does not work on manylinux. + TILEDB_ARROW_TESTS: ${{ !inputs.manylinux && 'ON' || 'OFF' }} + TILEDB_MANYLINUX: ${{ !inputs.manylinux && 'ON' || 'OFF' }} CXX: ${{ inputs.matrix_compiler_cxx }} CC: ${{ inputs.matrix_compiler_cc }} + CFLAGS: ${{ inputs.matrix_compiler_cflags }} + CXXFLAGS: ${{ inputs.matrix_compiler_cxxflags }} bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' SCCACHE_GHA_ENABLED: "true" @@ -56,6 +74,7 @@ jobs: os: - ${{ inputs.matrix_image }} runs-on: ${{matrix.os}} + container: ${{inputs.manylinux && 'quay.io/pypa/manylinux2014_x86_64' || ''}} if: ${{ startsWith(github.ref , 'refs/tags') != true && startsWith(github.ref , 'build-') != true }} timeout-minutes: ${{ inputs.timeout || 90 }} @@ -85,14 +104,23 @@ jobs: run: ./scripts/ci/posix/prelim.sh shell: bash + - name: Install manylinux prerequisites + if: inputs.manylinux + run: | + set -e pipefail + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 + echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc + # Need this for virtualenv and arrow tests if enabled - name: 'Install Python' uses: actions/setup-python@v4 + if: ${{ !inputs.manylinux }} with: python-version: '3.8' cache: 'pip' - name: 'Set up Python dependencies' + if: ${{ !inputs.manylinux }} run: | set -e pipefail python -m pip install --upgrade pip virtualenv diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 13ebef77d481..a710d9761fbe 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -112,6 +112,17 @@ jobs: timeout: 120 bootstrap_args: '--enable-azure' + ci_manylinux: + uses: ./.github/workflows/ci-linux_mac.yml + with: + ci_backend: MANYLINUX + matrix_image: ubuntu-20.04 + matrix_compiler_cflags: "-lrt" + matrix_compiler_cxxflags: "-lrt" + timeout: 120 + bootstrap_args: '--enable-serialization' + manylinux: true + ci_msvc: uses: ./.github/workflows/build-windows.yml @@ -145,6 +156,7 @@ jobs: ci8, ci9, ci10, + ci_manylinux, ci_msvc, backward_compatibility, standalone diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b0d6841e369b..236ae26cc838 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,17 +5,6 @@ trigger: - release-* - refs/tags/* - build- -pr: - branches: - include: - - '*' # must quote since "*" is a YAML reserved character; we want a string - paths: - exclude: - - '.github/workflows/quarto-render.yml' - - '_quarto.yml' - - 'quarto-materials/*' - - '**/.md' - - 'doc/source/conf.py' variables: - name: MANYLINUX_IMAGE @@ -24,25 +13,6 @@ variables: value: ghcr.io/ihnorton/tiledb-manylinux2014_x86_64:2023-04-02 stages: - - stage: CI - condition: and(not(startsWith(variables['Build.SourceBranch'], 'refs/tags')), not(startsWith(variables['Build.SourceBranchName'], 'build-'))) - variables: - BACKWARDS_COMPATIBILITY_ARRAYS: OFF - jobs: - - job: linux_manylinux - pool: { vmImage: $(imageName) } - container: ${{ variables.MANYLINUX_IMAGE }} - variables: - imageName: 'ubuntu-20.04' - TILEDB_SERIALIZATION: ON - CXX: g++ - CC: gcc - CXXFLAGS: "-lrt" - CFLAGS: "-lrt" - USE_MANYLINUX: ON - steps: - - template: scripts/azure-linux_mac.yml - - stage: Build_Release condition: or(or(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), eq(variables['Build.SourceBranchName'], 'dev')), startsWith(variables['Build.SourceBranchName'], 'release-')), startsWith(variables['Build.SourceBranchName'], 'build-')) variables: diff --git a/scripts/azure-linux_mac.yml b/scripts/azure-linux_mac.yml deleted file mode 100755 index 57e06eff32e4..000000000000 --- a/scripts/azure-linux_mac.yml +++ /dev/null @@ -1,283 +0,0 @@ -steps: -- task: NodeTool@0 # Needed for compatibility with manylinux image. - inputs: - versionSpec: '16.x' - displayName: 'Install Node.js 16' - -- bash: | - echo "'uname -s' is:" - echo "uname: " $(uname) - echo "uname -m: " $(uname -m) - echo "uname -r:" $(uname -r) - echo "uname -s: " $(uname -s) - echo "uname -v: " $(uname -v) - printenv - displayName: 'Print env' - -# Need this for virtualenv and arrow tests if enabled -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.8' - condition: eq(variables['TILEDB_ARROW_TESTS'], 'ON') - -- bash: | - set -e pipefail - python -m pip install --upgrade pip virtualenv - if [[ "$TILEDB_ARROW_TESTS" == "ON" ]]; then - pip install pyarrow pybind11 numpy - fi - condition: eq(variables['TILEDB_ARROW_TESTS'], 'ON') - -- bash: | - #https://stackoverflow.com/questions/65278351/no-core-dump-file-generated-after-segmentation-fault-in-macos-big-sur - sysctl -a | grep core - - #https://developer.apple.com/forums/thread/53023?answerId=158378022#158378022 - #sudo chmod 1775 /cores - #https://developer.apple.com/forums/thread/127503?answerId=401103022#401103022 - #so seems after change to 1777, the 'runner' attempt did core dump, while the sudo version did not... - sudo chmod 1777 /cores - sudo chown $(whoami) /cores - - sudo ls -l / - - ps -A - ls -l /cores - condition: and(eq(variables['Agent.OS'], 'Darwin'), startsWith(variables['imageName'], 'macOS-')) - displayName: 'prep macOS for "runner" core dumps' - - -- bash: | - brew install autoconf automake - condition: eq(variables['Agent.OS'], 'Darwin') - -- bash: | - # enable core dumps - ulimit -c # should output 0 if disabled - ulimit -c unlimited # Enable core dumps to be captured (must be in same run block) - ulimit -c # should output 'unlimited' now - - # Azure sets "SYSTEM=build" for unknown reasons, which breaks the OpenSSL configure script - # - openssl configure uses ENV{SYSTEM} if available: - # https://github.com/openssl/openssl/blob/6d745d740d37d680ff696486218b650512bbbbc6/config#L56 - # - error description: - # https://developercommunity.visualstudio.com/content/problem/602584/openssl-build-error-when-using-pipelines.htm - unset SYSTEM - - # azure bash does not treat intermediate failure as error - # https://github.com/Microsoft/azure-pipelines-yaml/issues/135 - set -xe pipefail - - git config --global user.name 'Azure Pipeline' - git config --global user.email 'no-reply@tiledb.io' - - if [[ "$BACKWARDS_COMPATIBILITY_ARRAYS" == "ON" ]]; then - git clone https://github.com/TileDB-Inc/TileDB-Unit-Test-Arrays.git --branch 2.5.0 test/inputs/arrays/read_compatibility_test - fi - # displayName: 'Clone Unit-Test-Arrays' - - # - bash: | - # Start HDFS server if enabled - if [[ "$TILEDB_HDFS" == "ON" ]]; then - # - ssh to localhost is required for HDFS launch... - # - /home/vsts has permissions g+w and is owned by user 'docker' - # for VSTS purposes, so disable ssh strictness - sudo sed -i "s/StrictModes\ yes/StrictModes\ no/g" /etc/ssh/sshd_config - - source scripts/install-hadoop.sh - source scripts/run-hadoop.sh - fi - - # Start minio server if S3 is enabled - if [[ "$TILEDB_S3" == "ON" ]]; then - source scripts/install-minio.sh; - source scripts/run-minio.sh; - fi - - # Start Azurite if Azure is enabled - if [[ "$TILEDB_AZURE" == "ON" ]]; then - source scripts/install-azurite.sh; - source scripts/run-azurite.sh; - fi - - # Start GCS Emulator if GCS is enabled - if [[ "$TILEDB_GCS" == "ON" ]]; then - source scripts/install-gcs-emu.sh; - source scripts/run-gcs-emu.sh; - fi - - # Set up arguments for bootstrap.sh - bootstrap_args="--enable=verbose"; - - # Enable TILEDB_STATIC by default - [ "$TILEDB_STATIC" ] || TILEDB_STATIC=ON - if [[ "$TILEDB_STATIC" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-static-tiledb"; - fi - if [[ "$TILEDB_HDFS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-hdfs"; - fi; - if [[ "$TILEDB_S3" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-s3"; - fi; - if [[ "$TILEDB_AZURE" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-azure"; - fi; - if [[ "$TILEDB_GCS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-gcs"; - fi; - if [[ "$TILEDB_TOOLS" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-tools"; - fi - if [[ "$TILEDB_DEBUG" == "ON" ]]; then - bootstrap_args="${bootstrap_args} --enable-debug"; - fi - if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then - # Add address sanitizer flag if necessary - bootstrap_args="${bootstrap_args} --enable-sanitizer=address --enable-debug"; - fi - if [[ "$TILEDB_CI_TSAN" == "ON" ]]; then - # Add thread sanitizer flag if necessary - bootstrap_args="${bootstrap_args} --enable-sanitizer=thread --enable-debug"; - fi - if [[ "$TILEDB_SERIALIZATION" == "ON" ]]; then - # Add serialization flag if necessary - bootstrap_args="${bootstrap_args} --enable-serialization"; - fi - if [[ "$TILEDB_FORCE_BUILD_DEPS" == "ON" ]]; then - # Add superbuild flag - bootstrap_args="${bootstrap_args} --force-build-all-deps"; - fi - if [[ "$TILEDB_WEBP" == "OFF" ]]; then - bootstrap_args="${bootstrap_args} --disable-webp"; - fi; - if [[ "$AGENT_OS" == "Darwin" ]]; then - # We want to be able to print a stack trace when a core dump occurs - sudo chmod 1777 /cores - # release symbols closer to 'release' than debug, and will hopefully have 'enough' - # context for reasonable stack. 'debug' (lack of) optimization less likely to - # to produce problems that might be encountered by actual 'release' build. - bootstrap_args="${bootstrap_args} --enable-release-symbols"; - fi - - # displayName: 'Install dependencies' - - mkdir -p $BUILD_REPOSITORY_LOCALPATH/build - cd $BUILD_REPOSITORY_LOCALPATH/build - - # Configure and build TileDB - echo "Bootstrapping with '$bootstrap_args'" - $BUILD_REPOSITORY_LOCALPATH/bootstrap $bootstrap_args - - make -j4 - make examples -j4 - make -C tiledb install - - #- bash: | - cd $BUILD_REPOSITORY_LOCALPATH/build - ls -la - - if [[ ( "$AGENT_OS" == "Linux" && "$TILEDB_S3" == "ON" ) ]]; then - # make sure docker is still running... - printenv - docker ps -a - fi - - make -j4 -C tiledb tiledb_unit - make -j4 -C tiledb tiledb_regression - - if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then - export ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.5 - fi - - if [[ "$AGENT_OS" == "Darwin" && "$TILEDB_GCS" == "ON" ]]; then - # GCS unit tests are temporarily unsupported on CI for MACOS. Fake success with - # this echo. - echo "##vso[task.setvariable variable=TILEDB_CI_SUCCESS]1" - else - # run directly the executable, cmake catches the segfault and blocks - # the core dump - ./tiledb/test/tiledb_unit -d yes - ./tiledb/test/regression/tiledb_regression -d yes - fi - - # Kill the running Minio server, OSX only because Linux runs it within - # docker. - if [[ ( "$AGENT_OS" == "Darwin" && "$TILEDB_S3" == "ON" ) ]]; then - kill -9 $MINIO_PID - fi - - # Kill the running Azurite server - if [[ "$TILEDB_AZURE" == "ON" ]]; then - kill -9 $AZURITE_PID - fi - - # Kill the running GCS emulator server Linux only because OSX does not - # run the emulator - if [[ "$AGENT_OS" != "Darwin" && "$TILEDB_GCS" == "ON" ]]; then - kill -9 $GCS_PID - fi - - displayName: 'Build and test libtiledb' - -- bash: | - set -x - ls -la /cores - cntfiles=0 - for f in $(find /cores -name 'core.*'); - do - echo "stack trace for $f" - lldb -c $f --batch -o 'bt all' -o 'image list' -o 're r -a' -o 'di -F intel -f -m' -o 'quit' - (( cntfiles = cntfiles + 1 )) - done; - #mv whatever we can, possible we won't get all due to permissions - #and archiving may have chance of failing due to size constraints in runner VM - mkdir -p $BUILD_REPOSITORY_LOCALPATH/build/core - mv -v /cores/core.* $BUILD_REPOSITORY_LOCALPATH/build/core/ - ls -la $BUILD_REPOSITORY_LOCALPATH/build/core - if [[ cntfiles -ne 0 ]] ; then - export TDB_COREFILE_CNT=$cntfiles - fi - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - displayName: 'Print stack trace' - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Build.Repository.LocalPath)' - includeRootFolder: false - archiveType: 'tar' - tarCompression: 'gz' - archiveFile: '$(Build.ArtifactStagingDirectory)/$(ARTIFACT_OS)-build-dir.tar.gz' - replaceExistingArchive: true - verbose: true - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)/$(ARTIFACT_OS)-build-dir.tar.gz' - artifactName: 'build-dirs' - condition: and(failed(), eq(variables['Agent.OS'], 'Darwin')) # only run this job if the build step failed - -- bash: | - # tiledb_unit is configured to set a job-level variable TILEDB_CI_SUCCESS=1 - # following the test run. If this variable is not set, the build should fail. - # see https://github.com/TileDB-Inc/TileDB/pull/1400 (5f0623f4d3) - if [[ "$TILEDB_CI_SUCCESS" -ne 1 ]]; then - exit 1; - fi - displayName: 'Test status check' - - -- bash: | - set -e pipefail - # Display log files if the build failed - echo "Dumping log files for failed build" - echo "----------------------------------" - for f in $(find $BUILD_REPOSITORY_LOCALPATH/build -name *.log); - do echo "------" - echo $f - echo "======" - cat $f - done; - condition: failed() # only run this job if the build step failed - displayName: "Print log files (failed build only)" diff --git a/test/src/unit-capi-query.cc b/test/src/unit-capi-query.cc index 816ffce71902..8a0b3c1c9cca 100644 --- a/test/src/unit-capi-query.cc +++ b/test/src/unit-capi-query.cc @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -713,6 +714,10 @@ TEST_CASE_METHOD( // DenyWriteAccess is not supported on Windows. if constexpr (tiledb::platform::is_os_windows) return; + // The test fails on Manylinux. Skip it. + char* manylinux_var = getenv("TILEDB_MANYLINUX"); + if (manylinux_var && strlen(manylinux_var) > 0) + return; SupportedFsLocal local_fs; std::string temp_dir = local_fs.temp_dir(); std::string array_name = temp_dir + "write_failure"; From 1c00b3e72e2e3c0fbcc3b95732993e6cc73d65ed Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Thu, 16 Nov 2023 15:57:28 +0200 Subject: [PATCH 074/456] Query Plan REST support - part 1: handler (#4516) We need to add support for serializing the `Query Plan` via REST, in order to make it usable with TileDB Cloud. This first part is adding the C-API handler Cloud needs to call from Go to implement this feature. Notes: 1. This is as per this minimal design doc that I just wrote and is not yet reviewed : https://www.notion.so/Query-Plan-Request-Handler-a383a260ff9e40aebb013f36306c0753?pvs=4 2. This includes a change in the `QueryPlan` constructor to produce a _sorted_ list of attributes and dimensions. I needed this sort of predicability for testing, please let me know if this is not acceptable as a change for some reason. 3. I am currently serializing the `json` dump of the `QueryPlan` but it's worth thinking a bit about the future and considering serializing the `QueryPlan` object instead, in case there will be info in the object that would be non recoverable from `json` such as averages or so. I'd like the reviewers' inputs :) --- TYPE: IMPROVEMENT DESC: Query Plan REST support - part 1: handler --- test/src/unit-request-handlers.cc | 174 +- tiledb/CMakeLists.txt | 3 + .../query_plan/test/unit_capi_query_plan.cc | 4 +- tiledb/sm/c_api/tiledb.cc | 41 + tiledb/sm/c_api/tiledb_serialization.h | 19 + tiledb/sm/query_plan/query_plan.cc | 3 + tiledb/sm/serialization/enumeration.cc | 15 +- tiledb/sm/serialization/enumeration.h | 2 +- tiledb/sm/serialization/query.h | 11 + tiledb/sm/serialization/query_plan.cc | 314 ++++ tiledb/sm/serialization/query_plan.h | 104 ++ tiledb/sm/serialization/tiledb-rest.capnp | 13 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 1503 +++++++++++------ tiledb/sm/serialization/tiledb-rest.capnp.h | 389 +++++ 14 files changed, 2038 insertions(+), 557 deletions(-) create mode 100644 tiledb/sm/serialization/query_plan.cc create mode 100644 tiledb/sm/serialization/query_plan.h diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 3e3e0f8ee80b..d7555ec7dc4b 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -34,14 +34,17 @@ #include "test/support/tdb_catch.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" +#include "tiledb/api/c_api/string/string_api_internal.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/c_api/tiledb_serialization.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/serialization/array_schema.h" +#include "tiledb/sm/serialization/query_plan.h" #include "tiledb/sm/storage_manager/context.h" using namespace tiledb::sm; @@ -57,9 +60,6 @@ struct RequestHandlerFx { shared_ptr get_array(QueryType type); - shared_ptr create_string_enumeration( - std::string name, std::vector& values); - URI uri_; Config cfg_; Context ctx_; @@ -74,6 +74,18 @@ struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { virtual shared_ptr create_schema() override; ArraySchema call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype); + + shared_ptr create_string_enumeration( + std::string name, std::vector& values); +}; + +struct HandleQueryPlanRequestFx : RequestHandlerFx { + HandleQueryPlanRequestFx() + : RequestHandlerFx("query_plan_handler") { + } + + virtual shared_ptr create_schema() override; + std::string call_handler(SerializationType stype, Query& query); }; /* ********************************* */ @@ -153,6 +165,107 @@ TEST_CASE_METHOD( REQUIRE(rval != TILEDB_OK); } +/* ******************************************** */ +/* Testing Query Plan serialization */ +/* ******************************************** */ + +TEST_CASE_METHOD( + HandleQueryPlanRequestFx, + "tiledb_handle_query_plan_request - check json", + "[request_handler][query_plan][full]") { + auto stype = GENERATE(SerializationType::JSON, SerializationType::CAPNP); + + // Create and open array + create_array(); + tiledb_ctx_t* ctx = nullptr; + REQUIRE(tiledb_ctx_alloc(NULL, &ctx) == TILEDB_OK); + tiledb_array_t* array = nullptr; + REQUIRE(tiledb_array_alloc(ctx, uri_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + + // Create query + tiledb_query_t* query = nullptr; + REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); + REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_ROW_MAJOR) == TILEDB_OK); + int32_t dom[] = {1, 2, 1, 2}; + REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + + uint64_t size = 1; + std::vector a1(2); + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "attr1", a1.data(), &size) == + TILEDB_OK); + std::vector a2(2); + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "attr2", a2.data(), &size) == + TILEDB_OK); + std::vector a3(2); + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "attr3", a3.data(), &size) == + TILEDB_OK); + + // Use C API to get the query plan + tiledb_string_handle_t* query_plan = nullptr; + REQUIRE(tiledb_query_get_plan(ctx, query, &query_plan) == TILEDB_OK); + + // Call handler to get query plan via serialized req/deserialized response + auto query_plan_ser_deser = call_handler(stype, *query->query_); + + // Compare the two query plans + REQUIRE(query_plan->view() == query_plan_ser_deser); + + // Clean up + REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); + tiledb_query_free(&query); + tiledb_array_free(&array); + tiledb_ctx_free(&ctx); +} + +TEST_CASE_METHOD( + HandleQueryPlanRequestFx, + "tiledb_handle_query_plan_request - error checks", + "[request_handler][query_plan][errors]") { + create_array(); + + auto ctx = tiledb::Context(); + auto array = tiledb::Array(ctx, uri_.to_string(), TILEDB_READ); + auto stype = TILEDB_CAPNP; + auto req_buf = tiledb_buffer_handle_t::make_handle(); + auto resp_buf = tiledb_buffer_handle_t::make_handle(); + + auto rval = tiledb_handle_query_plan_request( + nullptr, + array.ptr().get(), + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_query_plan_request( + ctx.ptr().get(), + nullptr, + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_query_plan_request( + ctx.ptr().get(), + array.ptr().get(), + static_cast(stype), + nullptr, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_query_plan_request( + ctx.ptr().get(), + array.ptr().get(), + static_cast(stype), + req_buf, + nullptr); + REQUIRE(rval != TILEDB_OK); +} + /* ********************************* */ /* Testing Support Code */ /* ********************************* */ @@ -187,7 +300,8 @@ shared_ptr RequestHandlerFx::get_array(QueryType type) { return array; } -shared_ptr RequestHandlerFx::create_string_enumeration( +shared_ptr +HandleLoadArraySchemaRequestFx::create_string_enumeration( std::string name, std::vector& values) { uint64_t total_size = 0; for (auto v : values) { @@ -263,4 +377,56 @@ ArraySchema HandleLoadArraySchemaRequestFx::call_handler( stype, resp_buf->buffer()); } +shared_ptr HandleQueryPlanRequestFx::create_schema() { + auto schema = make_shared(HERE(), ArrayType::DENSE); + schema->set_capacity(10000); + throw_if_not_ok(schema->set_cell_order(Layout::ROW_MAJOR)); + throw_if_not_ok(schema->set_tile_order(Layout::ROW_MAJOR)); + uint32_t dim_domain[] = {1, 10, 1, 10}; + + auto dim1 = make_shared(HERE(), "dim1", Datatype::INT32); + throw_if_not_ok(dim1->set_domain(&dim_domain[0])); + auto dim2 = make_shared(HERE(), "dim2", Datatype::INT32); + throw_if_not_ok(dim2->set_domain(&dim_domain[2])); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim1)); + throw_if_not_ok(dom->add_dimension(dim2)); + throw_if_not_ok(schema->set_domain(dom)); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + throw_if_not_ok(schema->add_attribute(attr1)); + auto attr2 = make_shared(HERE(), "attr2", Datatype::INT32); + throw_if_not_ok(schema->add_attribute(attr2)); + auto attr3 = make_shared(HERE(), "attr3", Datatype::INT64); + throw_if_not_ok(schema->add_attribute(attr3)); + + return schema; +} + +std::string HandleQueryPlanRequestFx::call_handler( + SerializationType stype, Query& query) { + auto ctx = tiledb::Context(); + auto array = tiledb::Array(ctx, uri_.to_string(), TILEDB_READ); + auto req_buf = tiledb_buffer_handle_t::make_handle(); + auto resp_buf = tiledb_buffer_handle_t::make_handle(); + + serialization::serialize_query_plan_request( + cfg_, query, stype, req_buf->buffer()); + auto rval = tiledb_handle_query_plan_request( + ctx.ptr().get(), + array.ptr().get(), + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval == TILEDB_OK); + + auto query_plan = + serialization::deserialize_query_plan_response(stype, resp_buf->buffer()); + + tiledb_buffer_handle_t::break_handle(req_buf); + tiledb_buffer_handle_t::break_handle(resp_buf); + + return query_plan; +} #endif // TILEDB_SERIALIZATION diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 9d39b8458854..72665ac4a70c 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -293,6 +293,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_plan.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/stats/global_stats.cc @@ -346,6 +347,7 @@ if (TILEDB_SERIALIZATION) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_plan.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -368,6 +370,7 @@ if (TILEDB_SERIALIZATION) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/group.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_aggregates.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/query_plan.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/consolidation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/vacuum.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/serialization/tiledb-rest.capnp.c++ diff --git a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc index d3a0c5337301..8a794b1ddba2 100644 --- a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc +++ b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc @@ -304,7 +304,7 @@ TEST_CASE_METHOD( CHECK(json_plan["TileDB Query Plan"]["Query.Strategy.Name"] == "DenseReader"); CHECK( json_plan["TileDB Query Plan"]["Query.Attributes"] == - std::vector({"a2", "a1"})); + std::vector({"a1", "a2"})); CHECK( json_plan["TileDB Query Plan"]["Query.Dimensions"] == std::vector({"dim_1", "dim_2"})); @@ -370,7 +370,7 @@ TEST_CASE_METHOD( "GlobalOrderWriter"); CHECK( json_plan["TileDB Query Plan"]["Query.Attributes"] == - std::vector({"b", "a", "__coords"})); + std::vector({"__coords", "a", "b"})); CHECK( json_plan["TileDB Query Plan"]["Query.Dimensions"] == std::vector()); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 3d3b5ce33449..2a6d65a98031 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -81,6 +81,7 @@ #include "tiledb/sm/serialization/fragment_info.h" #include "tiledb/sm/serialization/fragments.h" #include "tiledb/sm/serialization/query.h" +#include "tiledb/sm/serialization/query_plan.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/context.h" #include "tiledb/sm/subarray/subarray.h" @@ -4337,6 +4338,35 @@ capi_return_t tiledb_handle_load_enumerations_request( return TILEDB_OK; } +capi_return_t tiledb_handle_query_plan_request( + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + throw std::invalid_argument("Array paramter must be valid."); + } + + api::ensure_buffer_is_valid(request); + api::ensure_buffer_is_valid(response); + + tiledb::sm::Query query(ctx->storage_manager(), array->array_); + tiledb::sm::serialization::deserialize_query_plan_request( + static_cast(serialization_type), + request->buffer(), + ctx->resources().compute_tp(), + query); + sm::QueryPlan plan(query); + + tiledb::sm::serialization::serialize_query_plan_response( + plan.dump_json(), + static_cast(serialization_type), + response->buffer()); + + return TILEDB_OK; +} + /* ****************************** */ /* C++ API */ /* ****************************** */ @@ -7204,6 +7234,17 @@ CAPI_INTERFACE( ctx, array, serialization_type, request, response); } +CAPI_INTERFACE( + handle_query_plan_request, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) { + return api_entry( + ctx, array, serialization_type, request, response); +} + /* ****************************** */ /* C++ API */ /* ****************************** */ diff --git a/tiledb/sm/c_api/tiledb_serialization.h b/tiledb/sm/c_api/tiledb_serialization.h index 5edcb27971e9..435befdd79e3 100644 --- a/tiledb/sm/c_api/tiledb_serialization.h +++ b/tiledb/sm/c_api/tiledb_serialization.h @@ -773,6 +773,7 @@ TILEDB_EXPORT capi_return_t tiledb_handle_load_array_schema_request( * * @param ctx The TileDB context. * @param array The TileDB Array. + * @param serialization_type The type of Cap'n Proto serialization used. * @param request A buffer containing the LoadEnumerationsRequest Capnp message. * @param response An allocated buffer that will contain the * LoadEnumerationsResponse Capnp message. @@ -785,6 +786,24 @@ TILEDB_EXPORT capi_return_t tiledb_handle_load_enumerations_request( const tiledb_buffer_t* request, tiledb_buffer_t* response) TILEDB_NOEXCEPT; +/** + * Process a query plan request. + * + * @param ctx The TileDB context. + * @param array The TileDB Array. + * @param serialization_type The type of Cap'n Proto serialization used. + * @param request A buffer containing the QueryPlanRequest Capnp message. + * @param response An allocated buffer that will contain the QueryPlanResponse + * Capnp message. + * @return capi_return_t TILEDB_OK on success, TILEDB_ERR on error. + */ +TILEDB_EXPORT capi_return_t tiledb_handle_query_plan_request( + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) TILEDB_NOEXCEPT; + #ifdef __cplusplus } #endif diff --git a/tiledb/sm/query_plan/query_plan.cc b/tiledb/sm/query_plan/query_plan.cc index 308a85425f6c..9c5d6cbcd947 100644 --- a/tiledb/sm/query_plan/query_plan.cc +++ b/tiledb/sm/query_plan/query_plan.cc @@ -75,6 +75,9 @@ QueryPlan::QueryPlan(Query& query) { if (query.is_dense()) { dimensions_ = query.array()->array_schema_latest().dim_names(); } + + std::sort(attributes_.begin(), attributes_.end()); + std::sort(dimensions_.begin(), dimensions_.end()); } /* ********************************* */ diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 34314cd64eb2..16eb87916636 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -40,7 +40,9 @@ // clang-format on #include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/config/config.h" #include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/enumeration.h" using namespace tiledb::common; @@ -122,7 +124,7 @@ void load_enumerations_request_to_capnp( } std::vector load_enumerations_request_from_capnp( - capnp::LoadEnumerationsRequest::Reader& reader) { + const capnp::LoadEnumerationsRequest::Reader& reader) { std::vector ret; if (reader.hasEnumerations()) { for (auto name_reader : reader.getEnumerations()) { @@ -148,7 +150,7 @@ void load_enumerations_response_to_capnp( std::vector> load_enumerations_response_from_capnp( - capnp::LoadEnumerationsResponse::Reader& reader) { + const capnp::LoadEnumerationsResponse::Reader& reader) { std::vector> ret; if (reader.hasEnumerations()) { auto enmr_readers = reader.getEnumerations(); @@ -252,7 +254,7 @@ std::vector deserialize_load_enumerations_request( } void serialize_load_enumerations_response( - const std::vector> enumerations, + const std::vector>& enumerations, SerializationType serialize_type, Buffer& response) { try { @@ -346,7 +348,10 @@ deserialize_load_enumerations_response( #else void serialize_load_enumerations_request( - const std::vector&, SerializationType, Buffer&) { + const Config&, + const std::vector&, + SerializationType, + Buffer&) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } @@ -358,7 +363,7 @@ std::vector deserialize_load_enumerations_request( } void serialize_load_enumerations_response( - const std::vector>, + const std::vector>&, SerializationType, Buffer&) { throw Status_SerializationError( diff --git a/tiledb/sm/serialization/enumeration.h b/tiledb/sm/serialization/enumeration.h index 0c74702276ae..85e5faaaf7a3 100644 --- a/tiledb/sm/serialization/enumeration.h +++ b/tiledb/sm/serialization/enumeration.h @@ -81,7 +81,7 @@ std::vector deserialize_load_enumerations_request( SerializationType serialization_type, const Buffer& request); void serialize_load_enumerations_response( - const std::vector> enumerations, + const std::vector>& enumerations, SerializationType serialization_type, Buffer& response); diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 85eda96ad09b..c2d5e077236f 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -266,6 +266,17 @@ void ordered_dim_label_reader_from_capnp( OrderedDimLabelReader* reader, ThreadPool* compute_tp); +Status query_to_capnp( + Query& query, capnp::Query::Builder* query_builder, const bool client_side); + +Status query_from_capnp( + const capnp::Query::Reader& query_reader, + const SerializationContext context, + void* buffer_start, + CopyState* const copy_state, + Query* const query, + ThreadPool* compute_tp); + #endif } // namespace serialization diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc new file mode 100644 index 000000000000..74e36f4de934 --- /dev/null +++ b/tiledb/sm/serialization/query_plan.cc @@ -0,0 +1,314 @@ +/** + * @file query_plan.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements serialization for a query plan + */ + +// clang-format off +#ifdef TILEDB_SERIALIZATION +#include +#include +#include +#include "tiledb/sm/serialization/capnp_utils.h" +#endif +// clang-format on + +#include "tiledb/sm/query_plan/query_plan.h" +#include "tiledb/common/thread_pool.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/query.h" +#include "tiledb/sm/serialization/query_plan.h" + +using namespace tiledb::common; + +namespace tiledb::sm::serialization { + +#ifdef TILEDB_SERIALIZATION + +void query_plan_request_to_capnp( + capnp::QueryPlanRequest::Builder& builder, + const Config& config, + Query& query) { + auto config_builder = builder.initConfig(); + throw_if_not_ok(config_to_capnp(config, &config_builder)); + + auto query_builder = builder.initQuery(); + throw_if_not_ok(query_to_capnp(query, &query_builder, true)); +} + +void query_plan_request_from_capnp( + const capnp::QueryPlanRequest::Reader& reader, + ThreadPool& compute_tp, + Query& query) { + if (reader.hasQuery()) { + auto query_reader = reader.getQuery(); + throw_if_not_ok(query_from_capnp( + query_reader, + SerializationContext::SERVER, + nullptr, + nullptr, + &query, + &compute_tp)); + } +} + +void query_plan_response_to_capnp( + capnp::QueryPlanResponse::Builder& builder, const std::string& query_plan) { + builder.setQueryPlan(query_plan); +} + +std::string query_plan_response_from_capnp( + const capnp::QueryPlanResponse::Reader& reader) { + std::string query_plan; + if (reader.hasQueryPlan()) { + query_plan = reader.getQueryPlan().cStr(); + } + + return query_plan; +} + +void serialize_query_plan_request( + const Config& config, + Query& query, + SerializationType serialization_type, + Buffer& request) { + try { + ::capnp::MallocMessageBuilder message; + capnp::QueryPlanRequest::Builder builder = + message.initRoot(); + query_plan_request_to_capnp(builder, config, query); + + request.reset_size(); + request.reset_offset(); + + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + kj::String capnp_json = json.encode(builder); + const auto json_len = capnp_json.size(); + const char nul = '\0'; + // size does not include needed null terminator, so add +1 + throw_if_not_ok(request.realloc(json_len + 1)); + throw_if_not_ok(request.write(capnp_json.cStr(), json_len)); + throw_if_not_ok(request.write(&nul, 1)); + break; + } + case SerializationType::CAPNP: { + kj::Array<::capnp::word> protomessage = messageToFlatArray(message); + kj::ArrayPtr message_chars = protomessage.asChars(); + const auto nbytes = message_chars.size(); + throw_if_not_ok(request.realloc(nbytes)); + throw_if_not_ok(request.write(message_chars.begin(), nbytes)); + break; + } + default: { + throw Status_SerializationError( + "Error serializing query plan request; " + "Unknown serialization type passed"); + } + } + + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error serializing query plan request; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error serializing query plan request; exception " + + std::string(e.what())); + } +} + +void deserialize_query_plan_request( + const SerializationType serialization_type, + const Buffer& request, + ThreadPool& compute_tp, + Query& query) { + try { + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + json.handleByAnnotation(); + ::capnp::MallocMessageBuilder message_builder; + capnp::QueryPlanRequest::Builder builder = + message_builder.initRoot(); + json.decode( + kj::StringPtr(static_cast(request.data())), builder); + capnp::QueryPlanRequest::Reader reader = builder.asReader(); + query_plan_request_from_capnp(reader, compute_tp, query); + break; + } + case SerializationType::CAPNP: { + const auto mBytes = reinterpret_cast(request.data()); + ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( + reinterpret_cast(mBytes), + request.size() / sizeof(::capnp::word))); + capnp::QueryPlanRequest::Reader reader = + array_reader.getRoot(); + query_plan_request_from_capnp(reader, compute_tp, query); + break; + } + default: { + throw Status_SerializationError( + "Error deserializing query plan request; " + "Unknown serialization type passed"); + } + } + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error deserializing query plan request; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error deserializing query plan request; exception " + + std::string(e.what())); + } +} + +void serialize_query_plan_response( + const std::string& query_plan, + SerializationType serialization_type, + Buffer& response) { + try { + ::capnp::MallocMessageBuilder message; + capnp::QueryPlanResponse::Builder builder = + message.initRoot(); + query_plan_response_to_capnp(builder, query_plan); + + response.reset_size(); + response.reset_offset(); + + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + kj::String capnp_json = json.encode(builder); + const auto json_len = capnp_json.size(); + const char nul = '\0'; + // size does not include needed null terminator, so add +1 + throw_if_not_ok(response.realloc(json_len + 1)); + throw_if_not_ok(response.write(capnp_json.cStr(), json_len)); + throw_if_not_ok(response.write(&nul, 1)); + break; + } + case SerializationType::CAPNP: { + kj::Array<::capnp::word> protomessage = messageToFlatArray(message); + kj::ArrayPtr message_chars = protomessage.asChars(); + const auto nbytes = message_chars.size(); + throw_if_not_ok(response.realloc(nbytes)); + throw_if_not_ok(response.write(message_chars.begin(), nbytes)); + break; + } + default: { + throw Status_SerializationError( + "Error serializing query plan response; " + "Unknown serialization type passed"); + } + } + + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error serializing query plan response; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error serializing query plan response; exception " + + std::string(e.what())); + } +} + +std::string deserialize_query_plan_response( + SerializationType serialization_type, const Buffer& response) { + try { + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + ::capnp::MallocMessageBuilder message_builder; + capnp::QueryPlanResponse::Builder builder = + message_builder.initRoot(); + json.decode( + kj::StringPtr(static_cast(response.data())), builder); + capnp::QueryPlanResponse::Reader reader = builder.asReader(); + return query_plan_response_from_capnp(reader); + } + case SerializationType::CAPNP: { + const auto mBytes = reinterpret_cast(response.data()); + ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( + reinterpret_cast(mBytes), + response.size() / sizeof(::capnp::word))); + capnp::QueryPlanResponse::Reader reader = + array_reader.getRoot(); + return query_plan_response_from_capnp(reader); + } + default: { + throw Status_SerializationError( + "Error deserializing query plan response; " + "Unknown serialization type passed"); + } + } + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error deserializing query plan response; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error deserializing query plan response; exception " + + std::string(e.what())); + } +} + +#else + +void serialize_query_plan_request( + const Config&, Query&, const SerializationType, Buffer&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +void deserialize_query_plan_request( + const SerializationType, const Buffer&, ThreadPool&, Query&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +void serialize_query_plan_response( + const std::string&, const SerializationType, Buffer&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +std::string deserialize_query_plan_response( + const SerializationType, const Buffer&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +#endif // TILEDB_SERIALIZATION + +} // namespace tiledb::sm::serialization diff --git a/tiledb/sm/serialization/query_plan.h b/tiledb/sm/serialization/query_plan.h new file mode 100644 index 000000000000..7eb46506889b --- /dev/null +++ b/tiledb/sm/serialization/query_plan.h @@ -0,0 +1,104 @@ +/** + * @file query_plan.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares serialization functions for Query Plan. + */ + +#ifndef TILEDB_SERIALIZATION_QUERY_PLAN_H +#define TILEDB_SERIALIZATION_QUERY_PLAN_H + +#ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/serialization/capnp_utils.h" +#endif + +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/query_plan/query_plan.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +enum class SerializationType : uint8_t; + +namespace serialization { + +/** + * Serialize a Query Plan request to cap'n proto object + * + * @param config Config to serialize. + * @param query The query for which the plan is requested. + * @param serialization_type Format to serialize into: Cap'n Proto or JSON. + * @param request Buffer to store serialized bytes in. + */ +void serialize_query_plan_request( + const Config& config, + Query& query, + const SerializationType serialization_type, + Buffer& request); + +/** + * Deserialize a Query Plan request to cap'n proto object + * + * @param serialization_type Format to serialize from: Cap'n Proto or JSON. + * @param request Buffer to read serialized bytes from. + * @param compute_tp The thread pool for compute-bound tasks. + * @param query The query for which the plan is requested. + */ +void deserialize_query_plan_request( + const SerializationType serialization_type, + const Buffer& request, + ThreadPool& compute_tp, + Query& query); + +/** + * Serialize a Query Plan response to cap'n proto object + * + * @param query_plan The query plan to serialize. + * @param serialization_type Format to serialize into: Cap'n Proto or JSON. + * @param response Buffer to store serialized bytes in. + */ +void serialize_query_plan_response( + const std::string& query_plan, + const SerializationType serialization_type, + Buffer& response); + +/** + * Deserialize a Query Plan request to cap'n proto object + * + * @param serialization_type Format to serialize from: Cap'n Proto or JSON. + * @param response Buffer to read serialized bytes from. + * @return The requested query plan as a string. + */ +std::string deserialize_query_plan_response( + const SerializationType serialization_type, const Buffer& response); + +} // namespace serialization +} // namespace tiledb::sm + +#endif // TILEDB_SERIALIZATION_QUERY_PLAN_H diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 6535efeb0c62..bfb439e99bd9 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1261,6 +1261,19 @@ struct LoadArraySchemaResponse { # The loaded ArraySchema } +struct QueryPlanRequest { + config @0 :Config; + # Config + + query @1 : Query; + # the query for which we request the plan +} + +struct QueryPlanResponse { + queryPlan @0 :Text; + # The returned query plan +} + struct QueryChannel { # structure representing a query channel, that is a stream of data within # a TileDB query. Such channels can be generated for the purpose of avoiding diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index d2d8f5821fe3..89e59d603af9 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -5,19 +5,22 @@ namespace capnp { namespace schemas { -static const ::capnp::_::AlignedData<208> b_ce5904e6f9410cec = { +static const ::capnp::_::AlignedData<211> b_ce5904e6f9410cec = { { 0, 0, 0, 0, 5, 0, 6, 0, 236, 12, 65, 249, 230, 4, 89, 206, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 55, 2, 0, 0, + 41, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, @@ -220,23 +223,26 @@ static const ::capnp::_::AlignedData<208> b_ce5904e6f9410cec = { static const uint16_t m_ce5904e6f9410cec[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_ce5904e6f9410cec[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ce5904e6f9410cec = { - 0xce5904e6f9410cec, b_ce5904e6f9410cec.words, 208, nullptr, m_ce5904e6f9410cec, + 0xce5904e6f9410cec, b_ce5904e6f9410cec.words, 211, nullptr, m_ce5904e6f9410cec, 0, 10, i_ce5904e6f9410cec, nullptr, nullptr, { &s_ce5904e6f9410cec, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<47> b_e3dadf2bf211bc97 = { +static const ::capnp::_::AlignedData<50> b_e3dadf2bf211bc97 = { { 0, 0, 0, 0, 5, 0, 6, 0, 151, 188, 17, 242, 43, 223, 218, 227, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 170, 0, 0, 0, - 29, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 106, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 75, 86, 0, 0, 0, 0, @@ -278,23 +284,26 @@ static const ::capnp::_::AlignedData<47> b_e3dadf2bf211bc97 = { static const uint16_t m_e3dadf2bf211bc97[] = {0, 1}; static const uint16_t i_e3dadf2bf211bc97[] = {0, 1}; const ::capnp::_::RawSchema s_e3dadf2bf211bc97 = { - 0xe3dadf2bf211bc97, b_e3dadf2bf211bc97.words, 47, nullptr, m_e3dadf2bf211bc97, + 0xe3dadf2bf211bc97, b_e3dadf2bf211bc97.words, 50, nullptr, m_e3dadf2bf211bc97, 0, 2, i_e3dadf2bf211bc97, nullptr, nullptr, { &s_e3dadf2bf211bc97, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<37> b_b6c95b4b8111ad36 = { +static const ::capnp::_::AlignedData<40> b_b6c95b4b8111ad36 = { { 0, 0, 0, 0, 5, 0, 6, 0, 54, 173, 17, 129, 75, 91, 201, 182, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 102, 105, 103, @@ -329,23 +338,26 @@ static const ::capnp::_::RawSchema* const d_b6c95b4b8111ad36[] = { static const uint16_t m_b6c95b4b8111ad36[] = {0}; static const uint16_t i_b6c95b4b8111ad36[] = {0}; const ::capnp::_::RawSchema s_b6c95b4b8111ad36 = { - 0xb6c95b4b8111ad36, b_b6c95b4b8111ad36.words, 37, d_b6c95b4b8111ad36, m_b6c95b4b8111ad36, + 0xb6c95b4b8111ad36, b_b6c95b4b8111ad36.words, 40, d_b6c95b4b8111ad36, m_b6c95b4b8111ad36, 1, 1, i_b6c95b4b8111ad36, nullptr, nullptr, { &s_b6c95b4b8111ad36, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<217> b_a45730f57e0460b4 = { +static const ::capnp::_::AlignedData<220> b_a45730f57e0460b4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 180, 96, 4, 126, 245, 48, 87, 164, - 18, 0, 0, 0, 1, 0, 3, 0, + 42, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 0, 0, 0, - 29, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 111, 2, 0, 0, + 37, 0, 0, 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 0, @@ -568,23 +580,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a45730f57e0460b4 { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a45730f57e0460b4 = { - 0xa45730f57e0460b4, b_a45730f57e0460b4.words, 217, d_a45730f57e0460b4, m_a45730f57e0460b4, + 0xa45730f57e0460b4, b_a45730f57e0460b4.words, 220, d_a45730f57e0460b4, m_a45730f57e0460b4, 6, 11, i_a45730f57e0460b4, nullptr, nullptr, { &s_a45730f57e0460b4, nullptr, bd_a45730f57e0460b4, 0, sizeof(bd_a45730f57e0460b4) / sizeof(bd_a45730f57e0460b4[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_facceeafd4472c68 = { +static const ::capnp::_::AlignedData<52> b_facceeafd4472c68 = { { 0, 0, 0, 0, 5, 0, 6, 0, 104, 44, 71, 212, 175, 238, 204, 250, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 79, @@ -631,23 +646,26 @@ static const ::capnp::_::RawSchema* const d_facceeafd4472c68[] = { static const uint16_t m_facceeafd4472c68[] = {0, 1}; static const uint16_t i_facceeafd4472c68[] = {0, 1}; const ::capnp::_::RawSchema s_facceeafd4472c68 = { - 0xfacceeafd4472c68, b_facceeafd4472c68.words, 49, d_facceeafd4472c68, m_facceeafd4472c68, + 0xfacceeafd4472c68, b_facceeafd4472c68.words, 52, d_facceeafd4472c68, m_facceeafd4472c68, 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<318> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 191, 3, 0, 0, + 41, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, @@ -965,23 +983,26 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 318, d_d71de32f98e296fe, m_d71de32f98e296fe, 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { +static const ::capnp::_::AlignedData<177> b_ceff8d62d10cd1de = { { 0, 0, 0, 0, 5, 0, 6, 0, 222, 209, 12, 209, 98, 141, 255, 206, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 55, 2, 0, 0, + 45, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -1153,23 +1174,26 @@ static const ::capnp::_::RawSchema* const d_ceff8d62d10cd1de[] = { static const uint16_t m_ceff8d62d10cd1de[] = {3, 6, 0, 7, 1, 4, 8, 9, 5, 2}; static const uint16_t i_ceff8d62d10cd1de[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { - 0xceff8d62d10cd1de, b_ceff8d62d10cd1de.words, 174, d_ceff8d62d10cd1de, m_ceff8d62d10cd1de, + 0xceff8d62d10cd1de, b_ceff8d62d10cd1de.words, 177, d_ceff8d62d10cd1de, m_ceff8d62d10cd1de, 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<143> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<146> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 58, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 250, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 87, 1, 0, 0, + 45, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, @@ -1311,23 +1335,26 @@ static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 5, 2}; static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 143, d_a1b81d67548230d4, m_a1b81d67548230d4, + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 146, d_a1b81d67548230d4, m_a1b81d67548230d4, 2, 6, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<163> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 255, 1, 0, 0, + 41, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -1485,23 +1512,26 @@ static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 163, d_92ad78f56de3d76a, m_92ad78f56de3d76a, 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { +static const ::capnp::_::AlignedData<128> b_d00b2f19cc21b9b4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 180, 185, 33, 204, 25, 47, 11, 208, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 143, 1, 0, 0, + 41, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 110, 117, 109, 101, 114, @@ -1621,23 +1651,26 @@ static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { - 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 128, nullptr, m_d00b2f19cc21b9b4, 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { +static const ::capnp::_::AlignedData<146> b_d20a578112fa92a2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 162, 146, 250, 18, 129, 87, 10, 210, - 18, 0, 0, 0, 1, 0, 6, 0, + 42, 0, 0, 0, 1, 0, 6, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 66, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 2, 2, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 143, 1, 0, 0, + 45, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -1775,23 +1808,26 @@ static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { static const uint16_t m_d20a578112fa92a2[] = {1, 0, 4, 6, 5, 3, 2}; static const uint16_t i_d20a578112fa92a2[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d20a578112fa92a2 = { - 0xd20a578112fa92a2, b_d20a578112fa92a2.words, 143, nullptr, m_d20a578112fa92a2, + 0xd20a578112fa92a2, b_d20a578112fa92a2.words, 146, nullptr, m_d20a578112fa92a2, 0, 7, i_d20a578112fa92a2, nullptr, nullptr, { &s_d20a578112fa92a2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<104> b_95e26a84d32d8223 = { +static const ::capnp::_::AlignedData<107> b_95e26a84d32d8223 = { { 0, 0, 0, 0, 5, 0, 6, 0, 35, 130, 45, 211, 132, 106, 226, 149, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 41, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -1895,23 +1931,26 @@ static const ::capnp::_::RawSchema* const d_95e26a84d32d8223[] = { static const uint16_t m_95e26a84d32d8223[] = {4, 5, 0, 1, 3, 2}; static const uint16_t i_95e26a84d32d8223[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_95e26a84d32d8223 = { - 0x95e26a84d32d8223, b_95e26a84d32d8223.words, 104, d_95e26a84d32d8223, m_95e26a84d32d8223, + 0x95e26a84d32d8223, b_95e26a84d32d8223.words, 107, d_95e26a84d32d8223, m_95e26a84d32d8223, 3, 6, i_95e26a84d32d8223, nullptr, nullptr, { &s_95e26a84d32d8223, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<168> b_a2a652536db09fa0 = { +static const ::capnp::_::AlignedData<171> b_a2a652536db09fa0 = { { 0, 0, 0, 0, 5, 0, 6, 0, 160, 159, 176, 109, 83, 82, 166, 162, - 28, 0, 0, 0, 1, 0, 2, 0, + 52, 0, 0, 0, 1, 0, 2, 0, 35, 130, 45, 211, 132, 106, 226, 149, 4, 0, 7, 0, 1, 0, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 58, 1, 0, 0, + 21, 0, 0, 0, 250, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 55, 2, 0, 0, + 41, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -2077,23 +2116,26 @@ static const ::capnp::_::RawSchema* const d_a2a652536db09fa0[] = { static const uint16_t m_a2a652536db09fa0[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_a2a652536db09fa0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_a2a652536db09fa0 = { - 0xa2a652536db09fa0, b_a2a652536db09fa0.words, 168, d_a2a652536db09fa0, m_a2a652536db09fa0, + 0xa2a652536db09fa0, b_a2a652536db09fa0.words, 171, d_a2a652536db09fa0, m_a2a652536db09fa0, 1, 10, i_a2a652536db09fa0, nullptr, nullptr, { &s_a2a652536db09fa0, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<85> b_de030f447664754c = { +static const ::capnp::_::AlignedData<88> b_de030f447664754c = { { 0, 0, 0, 0, 5, 0, 6, 0, 76, 117, 100, 118, 68, 15, 3, 222, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, @@ -2176,23 +2218,26 @@ static const ::capnp::_::RawSchema* const d_de030f447664754c[] = { static const uint16_t m_de030f447664754c[] = {0, 1, 2, 3}; static const uint16_t i_de030f447664754c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_de030f447664754c = { - 0xde030f447664754c, b_de030f447664754c.words, 85, d_de030f447664754c, m_de030f447664754c, + 0xde030f447664754c, b_de030f447664754c.words, 88, d_de030f447664754c, m_de030f447664754c, 1, 4, i_de030f447664754c, nullptr, nullptr, { &s_de030f447664754c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<47> b_fa787661cd3563a4 = { +static const ::capnp::_::AlignedData<50> b_fa787661cd3563a4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 164, 99, 53, 205, 97, 118, 120, 250, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 0, 0, 0, - 29, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 114, 114, 111, 114, 0, @@ -2234,23 +2279,26 @@ static const ::capnp::_::AlignedData<47> b_fa787661cd3563a4 = { static const uint16_t m_fa787661cd3563a4[] = {0, 1}; static const uint16_t i_fa787661cd3563a4[] = {0, 1}; const ::capnp::_::RawSchema s_fa787661cd3563a4 = { - 0xfa787661cd3563a4, b_fa787661cd3563a4.words, 47, nullptr, m_fa787661cd3563a4, + 0xfa787661cd3563a4, b_fa787661cd3563a4.words, 50, nullptr, m_fa787661cd3563a4, 0, 2, i_fa787661cd3563a4, nullptr, nullptr, { &s_fa787661cd3563a4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_fda1cb9663a55b70 = { +static const ::capnp::_::AlignedData<68> b_fda1cb9663a55b70 = { { 0, 0, 0, 0, 5, 0, 6, 0, 112, 91, 165, 99, 150, 203, 161, 253, - 18, 0, 0, 0, 1, 0, 3, 0, + 42, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 218, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 108, 111, 97, 116, 83, @@ -2310,23 +2358,26 @@ static const ::capnp::_::AlignedData<65> b_fda1cb9663a55b70 = { static const uint16_t m_fda1cb9663a55b70[] = {2, 1, 0}; static const uint16_t i_fda1cb9663a55b70[] = {0, 1, 2}; const ::capnp::_::RawSchema s_fda1cb9663a55b70 = { - 0xfda1cb9663a55b70, b_fda1cb9663a55b70.words, 65, nullptr, m_fda1cb9663a55b70, + 0xfda1cb9663a55b70, b_fda1cb9663a55b70.words, 68, nullptr, m_fda1cb9663a55b70, 0, 3, i_fda1cb9663a55b70, nullptr, nullptr, { &s_fda1cb9663a55b70, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<94> b_fd5825d8c6e74d78 = { +static const ::capnp::_::AlignedData<97> b_fd5825d8c6e74d78 = { { 0, 0, 0, 0, 5, 0, 6, 0, 120, 77, 231, 198, 216, 37, 88, 253, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 170, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 31, 1, 0, 0, + 41, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 101, 98, 112, 67, 111, @@ -2415,23 +2466,26 @@ static const ::capnp::_::AlignedData<94> b_fd5825d8c6e74d78 = { static const uint16_t m_fd5825d8c6e74d78[] = {3, 4, 1, 2, 0}; static const uint16_t i_fd5825d8c6e74d78[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_fd5825d8c6e74d78 = { - 0xfd5825d8c6e74d78, b_fd5825d8c6e74d78.words, 94, nullptr, m_fd5825d8c6e74d78, + 0xfd5825d8c6e74d78, b_fd5825d8c6e74d78.words, 97, nullptr, m_fd5825d8c6e74d78, 0, 5, i_fd5825d8c6e74d78, nullptr, nullptr, { &s_fd5825d8c6e74d78, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<74> b_e7175047415b3f97 = { +static const ::capnp::_::AlignedData<77> b_e7175047415b3f97 = { { 0, 0, 0, 0, 5, 0, 6, 0, 151, 63, 91, 65, 71, 80, 23, 231, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2505,23 +2559,26 @@ static const ::capnp::_::RawSchema* const d_e7175047415b3f97[] = { static const uint16_t m_e7175047415b3f97[] = {1, 2, 0, 3}; static const uint16_t i_e7175047415b3f97[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_e7175047415b3f97 = { - 0xe7175047415b3f97, b_e7175047415b3f97.words, 74, d_e7175047415b3f97, m_e7175047415b3f97, + 0xe7175047415b3f97, b_e7175047415b3f97.words, 77, d_e7175047415b3f97, m_e7175047415b3f97, 3, 4, i_e7175047415b3f97, nullptr, nullptr, { &s_e7175047415b3f97, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<197> b_9ceaf832b3ab897f = { +static const ::capnp::_::AlignedData<200> b_9ceaf832b3ab897f = { { 0, 0, 0, 0, 5, 0, 6, 0, 127, 137, 171, 179, 50, 248, 234, 156, - 25, 0, 0, 0, 1, 0, 2, 0, + 49, 0, 0, 0, 1, 0, 2, 0, 151, 63, 91, 65, 71, 80, 23, 231, 4, 0, 7, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 167, 2, 0, 0, + 37, 0, 0, 0, 167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2716,23 +2773,26 @@ static const ::capnp::_::RawSchema* const d_9ceaf832b3ab897f[] = { static const uint16_t m_9ceaf832b3ab897f[] = {1, 10, 11, 4, 6, 8, 2, 0, 5, 7, 9, 3}; static const uint16_t i_9ceaf832b3ab897f[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; const ::capnp::_::RawSchema s_9ceaf832b3ab897f = { - 0x9ceaf832b3ab897f, b_9ceaf832b3ab897f.words, 197, d_9ceaf832b3ab897f, m_9ceaf832b3ab897f, + 0x9ceaf832b3ab897f, b_9ceaf832b3ab897f.words, 200, d_9ceaf832b3ab897f, m_9ceaf832b3ab897f, 1, 12, i_9ceaf832b3ab897f, nullptr, nullptr, { &s_9ceaf832b3ab897f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<38> b_bc4583f733eac4f5 = { +static const ::capnp::_::AlignedData<41> b_bc4583f733eac4f5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 196, 234, 51, 247, 131, 69, 188, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2768,23 +2828,26 @@ static const ::capnp::_::RawSchema* const d_bc4583f733eac4f5[] = { static const uint16_t m_bc4583f733eac4f5[] = {0}; static const uint16_t i_bc4583f733eac4f5[] = {0}; const ::capnp::_::RawSchema s_bc4583f733eac4f5 = { - 0xbc4583f733eac4f5, b_bc4583f733eac4f5.words, 38, d_bc4583f733eac4f5, m_bc4583f733eac4f5, + 0xbc4583f733eac4f5, b_bc4583f733eac4f5.words, 41, d_bc4583f733eac4f5, m_bc4583f733eac4f5, 1, 1, i_bc4583f733eac4f5, nullptr, nullptr, { &s_bc4583f733eac4f5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_f179c194ae71718c = { +static const ::capnp::_::AlignedData<52> b_f179c194ae71718c = { { 0, 0, 0, 0, 5, 0, 6, 0, 140, 113, 113, 174, 148, 193, 121, 241, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 21, 0, 0, 0, 178, 0, 0, 0, - 29, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 114, 1, 0, 0, + 41, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 49, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 129, 0, 0, 0, 23, 0, 0, 0, + 141, 0, 0, 0, 23, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 0, 0, 0, @@ -2834,23 +2897,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_f179c194ae71718c { 16777216, ::tiledb::sm::serialization::capnp::Map< ::capnp::AnyPointer, ::capnp::AnyPointer>::Entry::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_f179c194ae71718c = { - 0xf179c194ae71718c, b_f179c194ae71718c.words, 49, d_f179c194ae71718c, m_f179c194ae71718c, + 0xf179c194ae71718c, b_f179c194ae71718c.words, 52, d_f179c194ae71718c, m_f179c194ae71718c, 1, 1, i_f179c194ae71718c, nullptr, nullptr, { &s_f179c194ae71718c, nullptr, bd_f179c194ae71718c, 0, sizeof(bd_f179c194ae71718c) / sizeof(bd_f179c194ae71718c[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_db5514c8aaf6faea = { +static const ::capnp::_::AlignedData<51> b_db5514c8aaf6faea = { { 0, 0, 0, 0, 5, 0, 6, 0, 234, 250, 246, 170, 200, 20, 85, 219, - 22, 0, 0, 0, 1, 0, 0, 0, + 46, 0, 0, 0, 1, 0, 0, 0, 140, 113, 113, 174, 148, 193, 121, 241, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 46, 69, 110, @@ -2893,23 +2959,26 @@ static const ::capnp::_::AlignedData<48> b_db5514c8aaf6faea = { static const uint16_t m_db5514c8aaf6faea[] = {0, 1}; static const uint16_t i_db5514c8aaf6faea[] = {0, 1}; const ::capnp::_::RawSchema s_db5514c8aaf6faea = { - 0xdb5514c8aaf6faea, b_db5514c8aaf6faea.words, 48, nullptr, m_db5514c8aaf6faea, + 0xdb5514c8aaf6faea, b_db5514c8aaf6faea.words, 51, nullptr, m_db5514c8aaf6faea, 0, 2, i_db5514c8aaf6faea, nullptr, nullptr, { &s_db5514c8aaf6faea, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_c6b5bb09d4611252 = { +static const ::capnp::_::AlignedData<43> b_c6b5bb09d4611252 = { { 0, 0, 0, 0, 5, 0, 6, 0, 82, 18, 97, 212, 9, 187, 181, 198, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 53, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -2947,23 +3016,26 @@ static const ::capnp::_::RawSchema* const d_c6b5bb09d4611252[] = { static const uint16_t m_c6b5bb09d4611252[] = {0}; static const uint16_t i_c6b5bb09d4611252[] = {0}; const ::capnp::_::RawSchema s_c6b5bb09d4611252 = { - 0xc6b5bb09d4611252, b_c6b5bb09d4611252.words, 40, d_c6b5bb09d4611252, m_c6b5bb09d4611252, + 0xc6b5bb09d4611252, b_c6b5bb09d4611252.words, 43, d_c6b5bb09d4611252, m_c6b5bb09d4611252, 1, 1, i_c6b5bb09d4611252, nullptr, nullptr, { &s_c6b5bb09d4611252, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_884e0a5f2521a5c6 = { +static const ::capnp::_::AlignedData<52> b_884e0a5f2521a5c6 = { { 0, 0, 0, 0, 5, 0, 6, 0, 198, 165, 33, 37, 95, 10, 78, 136, - 28, 0, 0, 0, 1, 0, 1, 0, + 52, 0, 0, 0, 1, 0, 1, 0, 82, 18, 97, 212, 9, 187, 181, 198, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 18, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 210, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3007,23 +3079,26 @@ static const ::capnp::_::AlignedData<49> b_884e0a5f2521a5c6 = { static const uint16_t m_884e0a5f2521a5c6[] = {0, 1}; static const uint16_t i_884e0a5f2521a5c6[] = {0, 1}; const ::capnp::_::RawSchema s_884e0a5f2521a5c6 = { - 0x884e0a5f2521a5c6, b_884e0a5f2521a5c6.words, 49, nullptr, m_884e0a5f2521a5c6, + 0x884e0a5f2521a5c6, b_884e0a5f2521a5c6.words, 52, nullptr, m_884e0a5f2521a5c6, 0, 2, i_884e0a5f2521a5c6, nullptr, nullptr, { &s_884e0a5f2521a5c6, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_a83707d3ba24dd32 = { +static const ::capnp::_::AlignedData<43> b_a83707d3ba24dd32 = { { 0, 0, 0, 0, 5, 0, 6, 0, 50, 221, 36, 186, 211, 7, 55, 168, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 154, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 53, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, @@ -3061,23 +3136,26 @@ static const ::capnp::_::RawSchema* const d_a83707d3ba24dd32[] = { static const uint16_t m_a83707d3ba24dd32[] = {0}; static const uint16_t i_a83707d3ba24dd32[] = {0}; const ::capnp::_::RawSchema s_a83707d3ba24dd32 = { - 0xa83707d3ba24dd32, b_a83707d3ba24dd32.words, 40, d_a83707d3ba24dd32, m_a83707d3ba24dd32, + 0xa83707d3ba24dd32, b_a83707d3ba24dd32.words, 43, d_a83707d3ba24dd32, m_a83707d3ba24dd32, 1, 1, i_a83707d3ba24dd32, nullptr, nullptr, { &s_a83707d3ba24dd32, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_a9caccb4333a2baf = { +static const ::capnp::_::AlignedData<52> b_a9caccb4333a2baf = { { 0, 0, 0, 0, 5, 0, 6, 0, 175, 43, 58, 51, 180, 204, 202, 169, - 27, 0, 0, 0, 1, 0, 1, 0, + 51, 0, 0, 0, 1, 0, 1, 0, 50, 221, 36, 186, 211, 7, 55, 168, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, @@ -3121,23 +3199,26 @@ static const ::capnp::_::AlignedData<49> b_a9caccb4333a2baf = { static const uint16_t m_a9caccb4333a2baf[] = {0, 1}; static const uint16_t i_a9caccb4333a2baf[] = {0, 1}; const ::capnp::_::RawSchema s_a9caccb4333a2baf = { - 0xa9caccb4333a2baf, b_a9caccb4333a2baf.words, 49, nullptr, m_a9caccb4333a2baf, + 0xa9caccb4333a2baf, b_a9caccb4333a2baf.words, 52, nullptr, m_a9caccb4333a2baf, 0, 2, i_a9caccb4333a2baf, nullptr, nullptr, { &s_a9caccb4333a2baf, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_d3c5983c670e0f42 = { +static const ::capnp::_::AlignedData<43> b_d3c5983c670e0f42 = { { 0, 0, 0, 0, 5, 0, 6, 0, 66, 15, 14, 103, 60, 152, 197, 211, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 53, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3175,23 +3256,26 @@ static const ::capnp::_::RawSchema* const d_d3c5983c670e0f42[] = { static const uint16_t m_d3c5983c670e0f42[] = {0}; static const uint16_t i_d3c5983c670e0f42[] = {0}; const ::capnp::_::RawSchema s_d3c5983c670e0f42 = { - 0xd3c5983c670e0f42, b_d3c5983c670e0f42.words, 40, d_d3c5983c670e0f42, m_d3c5983c670e0f42, + 0xd3c5983c670e0f42, b_d3c5983c670e0f42.words, 43, d_d3c5983c670e0f42, m_d3c5983c670e0f42, 1, 1, i_d3c5983c670e0f42, nullptr, nullptr, { &s_d3c5983c670e0f42, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_82d7452da7cd249a = { +static const ::capnp::_::AlignedData<52> b_82d7452da7cd249a = { { 0, 0, 0, 0, 5, 0, 6, 0, 154, 36, 205, 167, 45, 69, 215, 130, - 28, 0, 0, 0, 1, 0, 1, 0, + 52, 0, 0, 0, 1, 0, 1, 0, 66, 15, 14, 103, 60, 152, 197, 211, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 18, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 210, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3235,23 +3319,26 @@ static const ::capnp::_::AlignedData<49> b_82d7452da7cd249a = { static const uint16_t m_82d7452da7cd249a[] = {0, 1}; static const uint16_t i_82d7452da7cd249a[] = {0, 1}; const ::capnp::_::RawSchema s_82d7452da7cd249a = { - 0x82d7452da7cd249a, b_82d7452da7cd249a.words, 49, nullptr, m_82d7452da7cd249a, + 0x82d7452da7cd249a, b_82d7452da7cd249a.words, 52, nullptr, m_82d7452da7cd249a, 0, 2, i_82d7452da7cd249a, nullptr, nullptr, { &s_82d7452da7cd249a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_9354174d952207d2 = { +static const ::capnp::_::AlignedData<43> b_9354174d952207d2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 210, 7, 34, 149, 77, 23, 84, 147, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 170, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 53, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, @@ -3289,23 +3376,26 @@ static const ::capnp::_::RawSchema* const d_9354174d952207d2[] = { static const uint16_t m_9354174d952207d2[] = {0}; static const uint16_t i_9354174d952207d2[] = {0}; const ::capnp::_::RawSchema s_9354174d952207d2 = { - 0x9354174d952207d2, b_9354174d952207d2.words, 40, d_9354174d952207d2, m_9354174d952207d2, + 0x9354174d952207d2, b_9354174d952207d2.words, 43, d_9354174d952207d2, m_9354174d952207d2, 1, 1, i_9354174d952207d2, nullptr, nullptr, { &s_9354174d952207d2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_bb80cdbc3f312934 = { +static const ::capnp::_::AlignedData<52> b_bb80cdbc3f312934 = { { 0, 0, 0, 0, 5, 0, 6, 0, 52, 41, 49, 63, 188, 205, 128, 187, - 29, 0, 0, 0, 1, 0, 1, 0, + 53, 0, 0, 0, 1, 0, 1, 0, 210, 7, 34, 149, 77, 23, 84, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 218, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, @@ -3349,23 +3439,26 @@ static const ::capnp::_::AlignedData<49> b_bb80cdbc3f312934 = { static const uint16_t m_bb80cdbc3f312934[] = {0, 1}; static const uint16_t i_bb80cdbc3f312934[] = {0, 1}; const ::capnp::_::RawSchema s_bb80cdbc3f312934 = { - 0xbb80cdbc3f312934, b_bb80cdbc3f312934.words, 49, nullptr, m_bb80cdbc3f312934, + 0xbb80cdbc3f312934, b_bb80cdbc3f312934.words, 52, nullptr, m_bb80cdbc3f312934, 0, 2, i_bb80cdbc3f312934, nullptr, nullptr, { &s_bb80cdbc3f312934, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_c7e036a11506a1a4 = { +static const ::capnp::_::AlignedData<51> b_c7e036a11506a1a4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 164, 161, 6, 21, 161, 54, 224, 199, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 0, 0, 0, - 29, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 116, 97, 116, 115, 0, @@ -3412,23 +3505,26 @@ static const ::capnp::_::RawSchema* const d_c7e036a11506a1a4[] = { static const uint16_t m_c7e036a11506a1a4[] = {1, 0}; static const uint16_t i_c7e036a11506a1a4[] = {0, 1}; const ::capnp::_::RawSchema s_c7e036a11506a1a4 = { - 0xc7e036a11506a1a4, b_c7e036a11506a1a4.words, 48, d_c7e036a11506a1a4, m_c7e036a11506a1a4, + 0xc7e036a11506a1a4, b_c7e036a11506a1a4.words, 51, d_c7e036a11506a1a4, m_c7e036a11506a1a4, 2, 2, i_c7e036a11506a1a4, nullptr, nullptr, { &s_c7e036a11506a1a4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<90> b_8bf6c1d37e748294 = { +static const ::capnp::_::AlignedData<93> b_8bf6c1d37e748294 = { { 0, 0, 0, 0, 5, 0, 6, 0, 148, 130, 116, 126, 211, 193, 246, 139, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 58, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 250, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 231, 0, 0, 0, + 45, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 85, 110, 111, 114, 100, 101, @@ -3516,23 +3612,26 @@ static const ::capnp::_::RawSchema* const d_8bf6c1d37e748294[] = { static const uint16_t m_8bf6c1d37e748294[] = {1, 2, 3, 0}; static const uint16_t i_8bf6c1d37e748294[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_8bf6c1d37e748294 = { - 0x8bf6c1d37e748294, b_8bf6c1d37e748294.words, 90, d_8bf6c1d37e748294, m_8bf6c1d37e748294, + 0x8bf6c1d37e748294, b_8bf6c1d37e748294.words, 93, d_8bf6c1d37e748294, m_8bf6c1d37e748294, 1, 4, i_8bf6c1d37e748294, nullptr, nullptr, { &s_8bf6c1d37e748294, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<147> b_8ba60147a0e6735e = { +static const ::capnp::_::AlignedData<150> b_8ba60147a0e6735e = { { 0, 0, 0, 0, 5, 0, 6, 0, 94, 115, 230, 160, 71, 1, 166, 139, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 41, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 101, 114, @@ -3681,23 +3780,26 @@ static const ::capnp::_::RawSchema* const d_8ba60147a0e6735e[] = { static const uint16_t m_8ba60147a0e6735e[] = {0, 1, 2, 6, 5, 3, 4, 7}; static const uint16_t i_8ba60147a0e6735e[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_8ba60147a0e6735e = { - 0x8ba60147a0e6735e, b_8ba60147a0e6735e.words, 147, d_8ba60147a0e6735e, m_8ba60147a0e6735e, + 0x8ba60147a0e6735e, b_8ba60147a0e6735e.words, 150, d_8ba60147a0e6735e, m_8ba60147a0e6735e, 5, 8, i_8ba60147a0e6735e, nullptr, nullptr, { &s_8ba60147a0e6735e, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<106> b_86cfc12d74ed4aa0 = { +static const ::capnp::_::AlignedData<109> b_86cfc12d74ed4aa0 = { { 0, 0, 0, 0, 5, 0, 6, 0, 160, 74, 237, 116, 45, 193, 207, 134, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 31, 1, 0, 0, + 45, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -3798,23 +3900,26 @@ static const ::capnp::_::AlignedData<106> b_86cfc12d74ed4aa0 = { static const uint16_t m_86cfc12d74ed4aa0[] = {2, 3, 4, 1, 0}; static const uint16_t i_86cfc12d74ed4aa0[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_86cfc12d74ed4aa0 = { - 0x86cfc12d74ed4aa0, b_86cfc12d74ed4aa0.words, 106, nullptr, m_86cfc12d74ed4aa0, + 0x86cfc12d74ed4aa0, b_86cfc12d74ed4aa0.words, 109, nullptr, m_86cfc12d74ed4aa0, 0, 5, i_86cfc12d74ed4aa0, nullptr, nullptr, { &s_86cfc12d74ed4aa0, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_f7aa276767b422e7 = { +static const ::capnp::_::AlignedData<68> b_f7aa276767b422e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 34, 180, 103, 103, 39, 170, 247, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 97, 98, 101, 108, 83, @@ -3877,23 +3982,26 @@ static const ::capnp::_::RawSchema* const d_f7aa276767b422e7[] = { static const uint16_t m_f7aa276767b422e7[] = {0, 1, 2}; static const uint16_t i_f7aa276767b422e7[] = {0, 1, 2}; const ::capnp::_::RawSchema s_f7aa276767b422e7 = { - 0xf7aa276767b422e7, b_f7aa276767b422e7.words, 65, d_f7aa276767b422e7, m_f7aa276767b422e7, + 0xf7aa276767b422e7, b_f7aa276767b422e7.words, 68, d_f7aa276767b422e7, m_f7aa276767b422e7, 1, 3, i_f7aa276767b422e7, nullptr, nullptr, { &s_f7aa276767b422e7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<158> b_dba20dec138adac9 = { +static const ::capnp::_::AlignedData<161> b_dba20dec138adac9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 201, 218, 138, 19, 236, 13, 162, 219, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 154, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 143, 1, 0, 0, + 41, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4055,23 +4163,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_dba20dec138adac9 { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::SubarrayRanges>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_dba20dec138adac9 = { - 0xdba20dec138adac9, b_dba20dec138adac9.words, 158, d_dba20dec138adac9, m_dba20dec138adac9, + 0xdba20dec138adac9, b_dba20dec138adac9.words, 161, d_dba20dec138adac9, m_dba20dec138adac9, 4, 7, i_dba20dec138adac9, nullptr, nullptr, { &s_dba20dec138adac9, nullptr, bd_dba20dec138adac9, 0, sizeof(bd_dba20dec138adac9) / sizeof(bd_dba20dec138adac9[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<155> b_ff14003c70494585 = { +static const ::capnp::_::AlignedData<158> b_ff14003c70494585 = { { 0, 0, 0, 0, 5, 0, 6, 0, 133, 69, 73, 112, 60, 0, 20, 255, - 18, 0, 0, 0, 1, 0, 3, 0, + 42, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 0, 0, 0, 199, 1, 0, 0, + 73, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4228,23 +4339,26 @@ static const ::capnp::_::RawSchema* const d_ff14003c70494585[] = { static const uint16_t m_ff14003c70494585[] = {1, 2, 4, 6, 5, 3, 7, 0}; static const uint16_t i_ff14003c70494585[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_ff14003c70494585 = { - 0xff14003c70494585, b_ff14003c70494585.words, 155, d_ff14003c70494585, m_ff14003c70494585, + 0xff14003c70494585, b_ff14003c70494585.words, 158, d_ff14003c70494585, m_ff14003c70494585, 5, 8, i_ff14003c70494585, nullptr, nullptr, { &s_ff14003c70494585, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_f86b7bf97823250f = { +static const ::capnp::_::AlignedData<86> b_f86b7bf97823250f = { { 0, 0, 0, 0, 5, 0, 6, 0, 15, 37, 35, 120, 249, 123, 107, 248, - 38, 0, 0, 0, 1, 0, 3, 0, + 62, 0, 0, 0, 1, 0, 3, 0, 133, 69, 73, 112, 60, 0, 20, 255, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 98, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 53, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4325,23 +4439,26 @@ static const ::capnp::_::RawSchema* const d_f86b7bf97823250f[] = { static const uint16_t m_f86b7bf97823250f[] = {2, 3, 1, 0}; static const uint16_t i_f86b7bf97823250f[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_f86b7bf97823250f = { - 0xf86b7bf97823250f, b_f86b7bf97823250f.words, 83, d_f86b7bf97823250f, m_f86b7bf97823250f, + 0xf86b7bf97823250f, b_f86b7bf97823250f.words, 86, d_f86b7bf97823250f, m_f86b7bf97823250f, 1, 4, i_f86b7bf97823250f, nullptr, nullptr, { &s_f86b7bf97823250f, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<90> b_fdd9e47288724221 = { +static const ::capnp::_::AlignedData<93> b_fdd9e47288724221 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 66, 114, 136, 114, 228, 217, 253, - 38, 0, 0, 0, 1, 0, 2, 0, + 62, 0, 0, 0, 1, 0, 2, 0, 133, 69, 73, 112, 60, 0, 20, 255, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 98, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 34, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 231, 0, 0, 0, + 49, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4429,23 +4546,26 @@ static const ::capnp::_::RawSchema* const d_fdd9e47288724221[] = { static const uint16_t m_fdd9e47288724221[] = {1, 3, 2, 0}; static const uint16_t i_fdd9e47288724221[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_fdd9e47288724221 = { - 0xfdd9e47288724221, b_fdd9e47288724221.words, 90, d_fdd9e47288724221, m_fdd9e47288724221, + 0xfdd9e47288724221, b_fdd9e47288724221.words, 93, d_fdd9e47288724221, m_fdd9e47288724221, 1, 4, i_fdd9e47288724221, nullptr, nullptr, { &s_fdd9e47288724221, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_cbe1e7c13508aa2c = { +static const ::capnp::_::AlignedData<86> b_cbe1e7c13508aa2c = { { 0, 0, 0, 0, 5, 0, 6, 0, 44, 170, 8, 53, 193, 231, 225, 203, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, @@ -4526,23 +4646,26 @@ static const ::capnp::_::RawSchema* const d_cbe1e7c13508aa2c[] = { static const uint16_t m_cbe1e7c13508aa2c[] = {2, 0, 3, 1}; static const uint16_t i_cbe1e7c13508aa2c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { - 0xcbe1e7c13508aa2c, b_cbe1e7c13508aa2c.words, 83, d_cbe1e7c13508aa2c, m_cbe1e7c13508aa2c, + 0xcbe1e7c13508aa2c, b_cbe1e7c13508aa2c.words, 86, d_cbe1e7c13508aa2c, m_cbe1e7c13508aa2c, 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<84> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 18, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 210, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 231, 0, 0, 0, + 45, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, @@ -4618,23 +4741,26 @@ static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 84, nullptr, m_dac6a7f675c57409, 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<147> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<150> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 210, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 146, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 41, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 83, 84, 78, 111, 100, @@ -4779,23 +4905,26 @@ static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 7, 3, 6, 2}; static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 147, d_afc739d5c01e6496, m_afc739d5c01e6496, + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 150, d_afc739d5c01e6496, m_afc739d5c01e6496, 1, 8, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { +static const ::capnp::_::AlignedData<76> b_eaf57cb9871fc06f = { { 0, 0, 0, 0, 5, 0, 6, 0, 111, 192, 31, 135, 185, 124, 245, 234, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, @@ -4867,23 +4996,26 @@ static const ::capnp::_::RawSchema* const d_eaf57cb9871fc06f[] = { static const uint16_t m_eaf57cb9871fc06f[] = {1, 0, 2}; static const uint16_t i_eaf57cb9871fc06f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_eaf57cb9871fc06f = { - 0xeaf57cb9871fc06f, b_eaf57cb9871fc06f.words, 73, d_eaf57cb9871fc06f, m_eaf57cb9871fc06f, + 0xeaf57cb9871fc06f, b_eaf57cb9871fc06f.words, 76, d_eaf57cb9871fc06f, m_eaf57cb9871fc06f, 2, 3, i_eaf57cb9871fc06f, nullptr, nullptr, { &s_eaf57cb9871fc06f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<113> b_e19754f813ccf79c = { +static const ::capnp::_::AlignedData<116> b_e19754f813ccf79c = { { 0, 0, 0, 0, 5, 0, 6, 0, 156, 247, 204, 19, 248, 84, 151, 225, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 41, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 82, @@ -4997,23 +5129,26 @@ static const ::capnp::_::RawSchema* const d_e19754f813ccf79c[] = { static const uint16_t m_e19754f813ccf79c[] = {3, 5, 0, 2, 4, 1}; static const uint16_t i_e19754f813ccf79c[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_e19754f813ccf79c = { - 0xe19754f813ccf79c, b_e19754f813ccf79c.words, 113, d_e19754f813ccf79c, m_e19754f813ccf79c, + 0xe19754f813ccf79c, b_e19754f813ccf79c.words, 116, d_e19754f813ccf79c, m_e19754f813ccf79c, 4, 6, i_e19754f813ccf79c, nullptr, nullptr, { &s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = { +static const ::capnp::_::AlignedData<52> b_d74f5fed155d316c = { { 0, 0, 0, 0, 5, 0, 6, 0, 108, 49, 93, 21, 237, 95, 79, 215, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 101, 108, 101, 116, 101, @@ -5061,23 +5196,26 @@ static const ::capnp::_::RawSchema* const d_d74f5fed155d316c[] = { static const uint16_t m_d74f5fed155d316c[] = {0, 1}; static const uint16_t i_d74f5fed155d316c[] = {0, 1}; const ::capnp::_::RawSchema s_d74f5fed155d316c = { - 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 49, d_d74f5fed155d316c, m_d74f5fed155d316c, + 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 52, d_d74f5fed155d316c, m_d74f5fed155d316c, 2, 2, i_d74f5fed155d316c, nullptr, nullptr, { &s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { +static const ::capnp::_::AlignedData<82> b_def87cead82188e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 136, 33, 216, 234, 124, 248, 222, - 18, 0, 0, 0, 1, 0, 4, 0, + 42, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 231, 0, 0, 0, + 45, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 115, 117, 108, 116, @@ -5151,23 +5289,26 @@ static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { static const uint16_t m_def87cead82188e7[] = {0, 3, 2, 1}; static const uint16_t i_def87cead82188e7[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_def87cead82188e7 = { - 0xdef87cead82188e7, b_def87cead82188e7.words, 79, nullptr, m_def87cead82188e7, + 0xdef87cead82188e7, b_def87cead82188e7.words, 82, nullptr, m_def87cead82188e7, 0, 4, i_def87cead82188e7, nullptr, nullptr, { &s_def87cead82188e7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_c1a2d010de779de5 = { +static const ::capnp::_::AlignedData<51> b_c1a2d010de779de5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 229, 157, 119, 222, 16, 208, 162, 193, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 1, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -5210,23 +5351,26 @@ static const ::capnp::_::AlignedData<48> b_c1a2d010de779de5 = { static const uint16_t m_c1a2d010de779de5[] = {1, 0}; static const uint16_t i_c1a2d010de779de5[] = {0, 1}; const ::capnp::_::RawSchema s_c1a2d010de779de5 = { - 0xc1a2d010de779de5, b_c1a2d010de779de5.words, 48, nullptr, m_c1a2d010de779de5, + 0xc1a2d010de779de5, b_c1a2d010de779de5.words, 51, nullptr, m_c1a2d010de779de5, 0, 2, i_c1a2d010de779de5, nullptr, nullptr, { &s_c1a2d010de779de5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<76> b_c86c77b5f6a2bf0f = { +static const ::capnp::_::AlignedData<79> b_c86c77b5f6a2bf0f = { { 0, 0, 0, 0, 5, 0, 6, 0, 15, 191, 162, 246, 181, 119, 108, 200, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, @@ -5301,23 +5445,26 @@ static const ::capnp::_::RawSchema* const d_c86c77b5f6a2bf0f[] = { static const uint16_t m_c86c77b5f6a2bf0f[] = {2, 1, 0}; static const uint16_t i_c86c77b5f6a2bf0f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_c86c77b5f6a2bf0f = { - 0xc86c77b5f6a2bf0f, b_c86c77b5f6a2bf0f.words, 76, d_c86c77b5f6a2bf0f, m_c86c77b5f6a2bf0f, + 0xc86c77b5f6a2bf0f, b_c86c77b5f6a2bf0f.words, 79, d_c86c77b5f6a2bf0f, m_c86c77b5f6a2bf0f, 2, 3, i_c86c77b5f6a2bf0f, nullptr, nullptr, { &s_c86c77b5f6a2bf0f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<96> b_9b9a5fc7713a8692 = { +static const ::capnp::_::AlignedData<99> b_9b9a5fc7713a8692 = { { 0, 0, 0, 0, 5, 0, 6, 0, 146, 134, 58, 113, 199, 95, 154, 155, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 31, 1, 0, 0, + 41, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 101, 114, @@ -5414,23 +5561,26 @@ static const ::capnp::_::RawSchema* const d_9b9a5fc7713a8692[] = { static const uint16_t m_9b9a5fc7713a8692[] = {3, 0, 2, 4, 1}; static const uint16_t i_9b9a5fc7713a8692[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { - 0x9b9a5fc7713a8692, b_9b9a5fc7713a8692.words, 96, d_9b9a5fc7713a8692, m_9b9a5fc7713a8692, + 0x9b9a5fc7713a8692, b_9b9a5fc7713a8692.words, 99, d_9b9a5fc7713a8692, m_9b9a5fc7713a8692, 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<387> b_96ba49d0f8b23ccc = { +static const ::capnp::_::AlignedData<390> b_96ba49d0f8b23ccc = { { 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, 150, - 18, 0, 0, 0, 1, 0, 4, 0, + 42, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 17, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 0, 0, 0, - 29, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 215, 4, 0, 0, + 37, 0, 0, 0, 215, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, @@ -5824,23 +5974,26 @@ static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 21, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 387, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 390, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, 10, 22, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { +static const ::capnp::_::AlignedData<72> b_9df6f2a42c4e5f0b = { { 0, 0, 0, 0, 5, 0, 6, 0, 11, 95, 78, 44, 164, 242, 246, 157, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, @@ -5907,23 +6060,26 @@ static const ::capnp::_::RawSchema* const d_9df6f2a42c4e5f0b[] = { static const uint16_t m_9df6f2a42c4e5f0b[] = {1, 0, 2}; static const uint16_t i_9df6f2a42c4e5f0b[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9df6f2a42c4e5f0b = { - 0x9df6f2a42c4e5f0b, b_9df6f2a42c4e5f0b.words, 69, d_9df6f2a42c4e5f0b, m_9df6f2a42c4e5f0b, + 0x9df6f2a42c4e5f0b, b_9df6f2a42c4e5f0b.words, 72, d_9df6f2a42c4e5f0b, m_9df6f2a42c4e5f0b, 1, 3, i_9df6f2a42c4e5f0b, nullptr, nullptr, { &s_9df6f2a42c4e5f0b, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_a18264549448ece3 = { +static const ::capnp::_::AlignedData<42> b_a18264549448ece3 = { { 0, 0, 0, 0, 5, 0, 6, 0, 227, 236, 72, 148, 84, 100, 130, 161, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 42, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, @@ -5960,23 +6116,26 @@ static const ::capnp::_::RawSchema* const d_a18264549448ece3[] = { static const uint16_t m_a18264549448ece3[] = {0}; static const uint16_t i_a18264549448ece3[] = {0}; const ::capnp::_::RawSchema s_a18264549448ece3 = { - 0xa18264549448ece3, b_a18264549448ece3.words, 39, d_a18264549448ece3, m_a18264549448ece3, + 0xa18264549448ece3, b_a18264549448ece3.words, 42, d_a18264549448ece3, m_a18264549448ece3, 1, 1, i_a18264549448ece3, nullptr, nullptr, { &s_a18264549448ece3, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_9be1921b07e6cd2d = { +static const ::capnp::_::AlignedData<86> b_9be1921b07e6cd2d = { { 0, 0, 0, 0, 5, 0, 6, 0, 45, 205, 230, 7, 27, 146, 225, 155, - 18, 0, 0, 0, 1, 0, 3, 0, + 42, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 231, 0, 0, 0, + 45, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -6054,23 +6213,26 @@ static const ::capnp::_::AlignedData<83> b_9be1921b07e6cd2d = { static const uint16_t m_9be1921b07e6cd2d[] = {0, 2, 1, 3}; static const uint16_t i_9be1921b07e6cd2d[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_9be1921b07e6cd2d = { - 0x9be1921b07e6cd2d, b_9be1921b07e6cd2d.words, 83, nullptr, m_9be1921b07e6cd2d, + 0x9be1921b07e6cd2d, b_9be1921b07e6cd2d.words, 86, nullptr, m_9be1921b07e6cd2d, 0, 4, i_9be1921b07e6cd2d, nullptr, nullptr, { &s_9be1921b07e6cd2d, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_f01116579e9ea98e = { +static const ::capnp::_::AlignedData<42> b_f01116579e9ea98e = { { 0, 0, 0, 0, 5, 0, 6, 0, 142, 169, 158, 158, 87, 22, 17, 240, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 120, 66, 117, 102, @@ -6107,23 +6269,26 @@ static const ::capnp::_::RawSchema* const d_f01116579e9ea98e[] = { static const uint16_t m_f01116579e9ea98e[] = {0}; static const uint16_t i_f01116579e9ea98e[] = {0}; const ::capnp::_::RawSchema s_f01116579e9ea98e = { - 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 39, d_f01116579e9ea98e, m_f01116579e9ea98e, + 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 42, d_f01116579e9ea98e, m_f01116579e9ea98e, 1, 1, i_f01116579e9ea98e, nullptr, nullptr, { &s_f01116579e9ea98e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<41> b_9737dcafdfce31bb = { +static const ::capnp::_::AlignedData<44> b_9737dcafdfce31bb = { { 0, 0, 0, 0, 5, 0, 6, 0, 187, 49, 206, 223, 175, 220, 55, 151, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 1, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 57, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, @@ -6162,23 +6327,26 @@ static const ::capnp::_::RawSchema* const d_9737dcafdfce31bb[] = { static const uint16_t m_9737dcafdfce31bb[] = {0}; static const uint16_t i_9737dcafdfce31bb[] = {0}; const ::capnp::_::RawSchema s_9737dcafdfce31bb = { - 0x9737dcafdfce31bb, b_9737dcafdfce31bb.words, 41, d_9737dcafdfce31bb, m_9737dcafdfce31bb, + 0x9737dcafdfce31bb, b_9737dcafdfce31bb.words, 44, d_9737dcafdfce31bb, m_9737dcafdfce31bb, 1, 1, i_9737dcafdfce31bb, nullptr, nullptr, { &s_9737dcafdfce31bb, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<96> b_926fe1c3b12ed651 = { +static const ::capnp::_::AlignedData<99> b_926fe1c3b12ed651 = { { 0, 0, 0, 0, 5, 0, 6, 0, 81, 214, 46, 177, 195, 225, 111, 146, - 32, 0, 0, 0, 1, 0, 1, 0, + 56, 0, 0, 0, 1, 0, 1, 0, 187, 49, 206, 223, 175, 220, 55, 151, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 114, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 50, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 31, 1, 0, 0, + 49, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, @@ -6269,23 +6437,26 @@ static const ::capnp::_::AlignedData<96> b_926fe1c3b12ed651 = { static const uint16_t m_926fe1c3b12ed651[] = {4, 0, 1, 3, 2}; static const uint16_t i_926fe1c3b12ed651[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_926fe1c3b12ed651 = { - 0x926fe1c3b12ed651, b_926fe1c3b12ed651.words, 96, nullptr, m_926fe1c3b12ed651, + 0x926fe1c3b12ed651, b_926fe1c3b12ed651.words, 99, nullptr, m_926fe1c3b12ed651, 0, 5, i_926fe1c3b12ed651, nullptr, nullptr, { &s_926fe1c3b12ed651, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<310> b_9317f20ce509d918 = { +static const ::capnp::_::AlignedData<313> b_9317f20ce509d918 = { { 0, 0, 0, 0, 5, 0, 6, 0, 24, 217, 9, 229, 12, 242, 23, 147, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 12, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 1, 0, 0, - 37, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 202, 1, 0, 0, + 49, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 0, 0, 0, 23, 3, 0, 0, + 85, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6594,23 +6765,26 @@ static const ::capnp::_::RawSchema* const d_9317f20ce509d918[] = { static const uint16_t m_9317f20ce509d918[] = {9, 4, 5, 2, 6, 7, 1, 8, 11, 10, 3, 13, 12, 0}; static const uint16_t i_9317f20ce509d918[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; const ::capnp::_::RawSchema s_9317f20ce509d918 = { - 0x9317f20ce509d918, b_9317f20ce509d918.words, 310, d_9317f20ce509d918, m_9317f20ce509d918, + 0x9317f20ce509d918, b_9317f20ce509d918.words, 313, d_9317f20ce509d918, m_9317f20ce509d918, 2, 14, i_9317f20ce509d918, nullptr, nullptr, { &s_9317f20ce509d918, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<67> b_d9d27c082dec9e26 = { +static const ::capnp::_::AlignedData<70> b_d9d27c082dec9e26 = { { 0, 0, 0, 0, 5, 0, 6, 0, 38, 158, 236, 45, 8, 124, 210, 217, - 33, 0, 0, 0, 1, 0, 2, 0, + 57, 0, 0, 0, 1, 0, 2, 0, 24, 217, 9, 229, 12, 242, 23, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 66, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 175, 0, 0, 0, + 49, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6672,23 +6846,26 @@ static const ::capnp::_::AlignedData<67> b_d9d27c082dec9e26 = { static const uint16_t m_d9d27c082dec9e26[] = {2, 1, 0}; static const uint16_t i_d9d27c082dec9e26[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d9d27c082dec9e26 = { - 0xd9d27c082dec9e26, b_d9d27c082dec9e26.words, 67, nullptr, m_d9d27c082dec9e26, + 0xd9d27c082dec9e26, b_d9d27c082dec9e26.words, 70, nullptr, m_d9d27c082dec9e26, 0, 3, i_d9d27c082dec9e26, nullptr, nullptr, { &s_d9d27c082dec9e26, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_9eb745af522d087c = { +static const ::capnp::_::AlignedData<71> b_9eb745af522d087c = { { 0, 0, 0, 0, 5, 0, 6, 0, 124, 8, 45, 82, 175, 69, 183, 158, - 33, 0, 0, 0, 1, 0, 1, 0, + 57, 0, 0, 0, 1, 0, 1, 0, 24, 217, 9, 229, 12, 242, 23, 147, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 170, 2, 0, 0, + 61, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 57, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6751,23 +6928,26 @@ static const ::capnp::_::AlignedData<68> b_9eb745af522d087c = { static const uint16_t m_9eb745af522d087c[] = {1, 2, 0}; static const uint16_t i_9eb745af522d087c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9eb745af522d087c = { - 0x9eb745af522d087c, b_9eb745af522d087c.words, 68, nullptr, m_9eb745af522d087c, + 0x9eb745af522d087c, b_9eb745af522d087c.words, 71, nullptr, m_9eb745af522d087c, 0, 3, i_9eb745af522d087c, nullptr, nullptr, { &s_9eb745af522d087c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<95> b_8cd4e323f1feea3b = { +static const ::capnp::_::AlignedData<98> b_8cd4e323f1feea3b = { { 0, 0, 0, 0, 5, 0, 6, 0, 59, 234, 254, 241, 35, 227, 212, 140, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 65, 0, 0, 0, 119, 0, 0, 0, + 77, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -6864,23 +7044,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_8cd4e323f1feea3b { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::EstimatedResultSize::MemorySize>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_8cd4e323f1feea3b = { - 0x8cd4e323f1feea3b, b_8cd4e323f1feea3b.words, 95, d_8cd4e323f1feea3b, m_8cd4e323f1feea3b, + 0x8cd4e323f1feea3b, b_8cd4e323f1feea3b.words, 98, d_8cd4e323f1feea3b, m_8cd4e323f1feea3b, 1, 2, i_8cd4e323f1feea3b, nullptr, nullptr, { &s_8cd4e323f1feea3b, nullptr, bd_8cd4e323f1feea3b, 0, sizeof(bd_8cd4e323f1feea3b) / sizeof(bd_8cd4e323f1feea3b[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_92c8467685565269 = { +static const ::capnp::_::AlignedData<71> b_92c8467685565269 = { { 0, 0, 0, 0, 5, 0, 6, 0, 105, 82, 86, 133, 118, 70, 200, 146, - 38, 0, 0, 0, 1, 0, 3, 0, + 62, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 74, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 53, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -6943,23 +7126,26 @@ static const ::capnp::_::AlignedData<68> b_92c8467685565269 = { static const uint16_t m_92c8467685565269[] = {0, 2, 1}; static const uint16_t i_92c8467685565269[] = {0, 1, 2}; const ::capnp::_::RawSchema s_92c8467685565269 = { - 0x92c8467685565269, b_92c8467685565269.words, 68, nullptr, m_92c8467685565269, + 0x92c8467685565269, b_92c8467685565269.words, 71, nullptr, m_92c8467685565269, 0, 3, i_92c8467685565269, nullptr, nullptr, { &s_92c8467685565269, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_bda7916926591c22 = { +static const ::capnp::_::AlignedData<71> b_bda7916926591c22 = { { 0, 0, 0, 0, 5, 0, 6, 0, 34, 28, 89, 38, 105, 145, 167, 189, - 38, 0, 0, 0, 1, 0, 3, 0, + 62, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 74, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 53, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -7022,23 +7208,26 @@ static const ::capnp::_::AlignedData<68> b_bda7916926591c22 = { static const uint16_t m_bda7916926591c22[] = {0, 2, 1}; static const uint16_t i_bda7916926591c22[] = {0, 1, 2}; const ::capnp::_::RawSchema s_bda7916926591c22 = { - 0xbda7916926591c22, b_bda7916926591c22.words, 68, nullptr, m_bda7916926591c22, + 0xbda7916926591c22, b_bda7916926591c22.words, 71, nullptr, m_bda7916926591c22, 0, 3, i_bda7916926591c22, nullptr, nullptr, { &s_bda7916926591c22, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<34> b_c95970eb9310dc9c = { +static const ::capnp::_::AlignedData<37> b_c95970eb9310dc9c = { { 0, 0, 0, 0, 5, 0, 6, 0, 156, 220, 16, 147, 235, 112, 89, 201, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -7070,23 +7259,26 @@ static const ::capnp::_::RawSchema* const d_c95970eb9310dc9c[] = { static const uint16_t m_c95970eb9310dc9c[] = {0}; static const uint16_t i_c95970eb9310dc9c[] = {0}; const ::capnp::_::RawSchema s_c95970eb9310dc9c = { - 0xc95970eb9310dc9c, b_c95970eb9310dc9c.words, 34, d_c95970eb9310dc9c, m_c95970eb9310dc9c, + 0xc95970eb9310dc9c, b_c95970eb9310dc9c.words, 37, d_c95970eb9310dc9c, m_c95970eb9310dc9c, 1, 1, i_c95970eb9310dc9c, nullptr, nullptr, { &s_c95970eb9310dc9c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<66> b_d42e7b38b33e3d29 = { +static const ::capnp::_::AlignedData<69> b_d42e7b38b33e3d29 = { { 0, 0, 0, 0, 5, 0, 6, 0, 41, 61, 62, 179, 56, 123, 46, 212, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 42, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, @@ -7150,23 +7342,26 @@ static const ::capnp::_::RawSchema* const d_d42e7b38b33e3d29[] = { static const uint16_t m_d42e7b38b33e3d29[] = {0, 2, 1}; static const uint16_t i_d42e7b38b33e3d29[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d42e7b38b33e3d29 = { - 0xd42e7b38b33e3d29, b_d42e7b38b33e3d29.words, 66, d_d42e7b38b33e3d29, m_d42e7b38b33e3d29, + 0xd42e7b38b33e3d29, b_d42e7b38b33e3d29.words, 69, d_d42e7b38b33e3d29, m_d42e7b38b33e3d29, 1, 3, i_d42e7b38b33e3d29, nullptr, nullptr, { &s_d42e7b38b33e3d29, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<109> b_a000530ab1d17816 = { +static const ::capnp::_::AlignedData<112> b_a000530ab1d17816 = { { 0, 0, 0, 0, 5, 0, 6, 0, 22, 120, 209, 177, 10, 83, 0, 160, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 186, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -7278,23 +7473,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a000530ab1d17816 { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a000530ab1d17816 = { - 0xa000530ab1d17816, b_a000530ab1d17816.words, 109, d_a000530ab1d17816, m_a000530ab1d17816, + 0xa000530ab1d17816, b_a000530ab1d17816.words, 112, d_a000530ab1d17816, m_a000530ab1d17816, 3, 4, i_a000530ab1d17816, nullptr, nullptr, { &s_a000530ab1d17816, nullptr, bd_a000530ab1d17816, 0, sizeof(bd_a000530ab1d17816) / sizeof(bd_a000530ab1d17816[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_838188de0fd57580 = { +static const ::capnp::_::AlignedData<52> b_838188de0fd57580 = { { 0, 0, 0, 0, 5, 0, 6, 0, 128, 117, 213, 15, 222, 136, 129, 131, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 1, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, @@ -7342,23 +7540,26 @@ static const ::capnp::_::RawSchema* const d_838188de0fd57580[] = { static const uint16_t m_838188de0fd57580[] = {0, 1}; static const uint16_t i_838188de0fd57580[] = {0, 1}; const ::capnp::_::RawSchema s_838188de0fd57580 = { - 0x838188de0fd57580, b_838188de0fd57580.words, 49, d_838188de0fd57580, m_838188de0fd57580, + 0x838188de0fd57580, b_838188de0fd57580.words, 52, d_838188de0fd57580, m_838188de0fd57580, 2, 2, i_838188de0fd57580, nullptr, nullptr, { &s_838188de0fd57580, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<79> b_c41bcc7e8cc58f18 = { +static const ::capnp::_::AlignedData<82> b_c41bcc7e8cc58f18 = { { 0, 0, 0, 0, 5, 0, 6, 0, 24, 143, 197, 140, 126, 204, 27, 196, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, @@ -7432,23 +7633,26 @@ static const ::capnp::_::AlignedData<79> b_c41bcc7e8cc58f18 = { static const uint16_t m_c41bcc7e8cc58f18[] = {3, 2, 1, 0}; static const uint16_t i_c41bcc7e8cc58f18[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_c41bcc7e8cc58f18 = { - 0xc41bcc7e8cc58f18, b_c41bcc7e8cc58f18.words, 79, nullptr, m_c41bcc7e8cc58f18, + 0xc41bcc7e8cc58f18, b_c41bcc7e8cc58f18.words, 82, nullptr, m_c41bcc7e8cc58f18, 0, 4, i_c41bcc7e8cc58f18, nullptr, nullptr, { &s_c41bcc7e8cc58f18, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<60> b_dcdd20e1b79e915a = { +static const ::capnp::_::AlignedData<63> b_dcdd20e1b79e915a = { { 0, 0, 0, 0, 5, 0, 6, 0, 90, 145, 158, 183, 225, 32, 221, 220, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 0, 0, 0, - 29, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 53, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 0, @@ -7507,23 +7711,26 @@ static const ::capnp::_::RawSchema* const d_dcdd20e1b79e915a[] = { static const uint16_t m_dcdd20e1b79e915a[] = {0, 1}; static const uint16_t i_dcdd20e1b79e915a[] = {0, 1}; const ::capnp::_::RawSchema s_dcdd20e1b79e915a = { - 0xdcdd20e1b79e915a, b_dcdd20e1b79e915a.words, 60, d_dcdd20e1b79e915a, m_dcdd20e1b79e915a, + 0xdcdd20e1b79e915a, b_dcdd20e1b79e915a.words, 63, d_dcdd20e1b79e915a, m_dcdd20e1b79e915a, 2, 2, i_dcdd20e1b79e915a, nullptr, nullptr, { &s_dcdd20e1b79e915a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<54> b_a2ea10c715b475c1 = { +static const ::capnp::_::AlignedData<57> b_a2ea10c715b475c1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 193, 117, 180, 21, 199, 16, 234, 162, - 24, 0, 0, 0, 1, 0, 0, 0, + 48, 0, 0, 0, 1, 0, 0, 0, 90, 145, 158, 183, 225, 32, 221, 220, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 42, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 46, @@ -7576,23 +7783,26 @@ static const ::capnp::_::RawSchema* const d_a2ea10c715b475c1[] = { static const uint16_t m_a2ea10c715b475c1[] = {0, 1}; static const uint16_t i_a2ea10c715b475c1[] = {0, 1}; const ::capnp::_::RawSchema s_a2ea10c715b475c1 = { - 0xa2ea10c715b475c1, b_a2ea10c715b475c1.words, 54, d_a2ea10c715b475c1, m_a2ea10c715b475c1, + 0xa2ea10c715b475c1, b_a2ea10c715b475c1.words, 57, d_a2ea10c715b475c1, m_a2ea10c715b475c1, 2, 2, i_a2ea10c715b475c1, nullptr, nullptr, { &s_a2ea10c715b475c1, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<64> b_c4e54a63294eddb7 = { +static const ::capnp::_::AlignedData<67> b_c4e54a63294eddb7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 183, 221, 78, 41, 99, 74, 229, 196, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 119, 0, 0, 0, + 61, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, @@ -7655,23 +7865,26 @@ static const ::capnp::_::RawSchema* const d_c4e54a63294eddb7[] = { static const uint16_t m_c4e54a63294eddb7[] = {0, 1}; static const uint16_t i_c4e54a63294eddb7[] = {0, 1}; const ::capnp::_::RawSchema s_c4e54a63294eddb7 = { - 0xc4e54a63294eddb7, b_c4e54a63294eddb7.words, 64, d_c4e54a63294eddb7, m_c4e54a63294eddb7, + 0xc4e54a63294eddb7, b_c4e54a63294eddb7.words, 67, d_c4e54a63294eddb7, m_c4e54a63294eddb7, 2, 2, i_c4e54a63294eddb7, nullptr, nullptr, { &s_c4e54a63294eddb7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<82> b_83b01e46759bde40 = { +static const ::capnp::_::AlignedData<85> b_83b01e46759bde40 = { { 0, 0, 0, 0, 5, 0, 6, 0, 64, 222, 155, 117, 70, 30, 176, 131, - 30, 0, 0, 0, 1, 0, 0, 0, + 54, 0, 0, 0, 1, 0, 0, 0, 183, 221, 78, 41, 99, 74, 229, 196, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 74, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 53, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, @@ -7751,23 +7964,26 @@ static const ::capnp::_::RawSchema* const d_83b01e46759bde40[] = { static const uint16_t m_83b01e46759bde40[] = {1, 0}; static const uint16_t i_83b01e46759bde40[] = {0, 1}; const ::capnp::_::RawSchema s_83b01e46759bde40 = { - 0x83b01e46759bde40, b_83b01e46759bde40.words, 82, d_83b01e46759bde40, m_83b01e46759bde40, + 0x83b01e46759bde40, b_83b01e46759bde40.words, 85, d_83b01e46759bde40, m_83b01e46759bde40, 1, 2, i_83b01e46759bde40, nullptr, nullptr, { &s_83b01e46759bde40, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<64> b_fb7f36ad4d8ffe84 = { +static const ::capnp::_::AlignedData<67> b_fb7f36ad4d8ffe84 = { { 0, 0, 0, 0, 5, 0, 6, 0, 132, 254, 143, 77, 173, 54, 127, 251, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 119, 0, 0, 0, + 61, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, @@ -7830,23 +8046,26 @@ static const ::capnp::_::RawSchema* const d_fb7f36ad4d8ffe84[] = { static const uint16_t m_fb7f36ad4d8ffe84[] = {0, 1}; static const uint16_t i_fb7f36ad4d8ffe84[] = {0, 1}; const ::capnp::_::RawSchema s_fb7f36ad4d8ffe84 = { - 0xfb7f36ad4d8ffe84, b_fb7f36ad4d8ffe84.words, 64, d_fb7f36ad4d8ffe84, m_fb7f36ad4d8ffe84, + 0xfb7f36ad4d8ffe84, b_fb7f36ad4d8ffe84.words, 67, d_fb7f36ad4d8ffe84, m_fb7f36ad4d8ffe84, 2, 2, i_fb7f36ad4d8ffe84, nullptr, nullptr, { &s_fb7f36ad4d8ffe84, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<36> b_d5fd459ad75e86a9 = { +static const ::capnp::_::AlignedData<39> b_d5fd459ad75e86a9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 169, 134, 94, 215, 154, 69, 253, 213, - 30, 0, 0, 0, 1, 0, 0, 0, + 54, 0, 0, 0, 1, 0, 0, 0, 132, 254, 143, 77, 173, 54, 127, 251, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 74, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 53, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, @@ -7877,23 +8096,26 @@ static const ::capnp::_::AlignedData<36> b_d5fd459ad75e86a9 = { static const uint16_t m_d5fd459ad75e86a9[] = {0}; static const uint16_t i_d5fd459ad75e86a9[] = {0}; const ::capnp::_::RawSchema s_d5fd459ad75e86a9 = { - 0xd5fd459ad75e86a9, b_d5fd459ad75e86a9.words, 36, nullptr, m_d5fd459ad75e86a9, + 0xd5fd459ad75e86a9, b_d5fd459ad75e86a9.words, 39, nullptr, m_d5fd459ad75e86a9, 0, 1, i_d5fd459ad75e86a9, nullptr, nullptr, { &s_d5fd459ad75e86a9, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<119> b_c0c730b5390f4427 = { +static const ::capnp::_::AlignedData<122> b_c0c730b5390f4427 = { { 0, 0, 0, 0, 5, 0, 6, 0, 39, 68, 15, 57, 181, 48, 199, 192, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 218, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 31, 1, 0, 0, + 45, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 108, 111, 98, 97, 108, @@ -8016,23 +8238,26 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_c0c730b5390f4427 { 16777220, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::MultiPartUploadState>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_c0c730b5390f4427 = { - 0xc0c730b5390f4427, b_c0c730b5390f4427.words, 119, d_c0c730b5390f4427, m_c0c730b5390f4427, + 0xc0c730b5390f4427, b_c0c730b5390f4427.words, 122, d_c0c730b5390f4427, m_c0c730b5390f4427, 4, 5, i_c0c730b5390f4427, nullptr, nullptr, { &s_c0c730b5390f4427, nullptr, bd_c0c730b5390f4427, 0, sizeof(bd_c0c730b5390f4427) / sizeof(bd_c0c730b5390f4427[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_d8bd3c0dec37b773 = { +static const ::capnp::_::AlignedData<83> b_d8bd3c0dec37b773 = { { 0, 0, 0, 0, 5, 0, 6, 0, 115, 183, 55, 236, 13, 60, 189, 216, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, @@ -8107,23 +8332,26 @@ static const ::capnp::_::AlignedData<80> b_d8bd3c0dec37b773 = { static const uint16_t m_d8bd3c0dec37b773[] = {0, 2, 1}; static const uint16_t i_d8bd3c0dec37b773[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d8bd3c0dec37b773 = { - 0xd8bd3c0dec37b773, b_d8bd3c0dec37b773.words, 80, nullptr, m_d8bd3c0dec37b773, + 0xd8bd3c0dec37b773, b_d8bd3c0dec37b773.words, 83, nullptr, m_d8bd3c0dec37b773, 0, 3, i_d8bd3c0dec37b773, nullptr, nullptr, { &s_d8bd3c0dec37b773, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<612> b_cde352fc27e7ca61 = { +static const ::capnp::_::AlignedData<615> b_cde352fc27e7ca61 = { { 0, 0, 0, 0, 5, 0, 6, 0, 97, 202, 231, 39, 252, 82, 227, 205, - 18, 0, 0, 0, 1, 0, 4, 0, + 42, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 22, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 1, 0, 0, - 37, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 218, 1, 0, 0, + 49, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 95, 6, 0, 0, + 65, 0, 0, 0, 95, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -8734,23 +8962,26 @@ static const ::capnp::_::RawSchema* const d_cde352fc27e7ca61[] = { static const uint16_t m_cde352fc27e7ca61[] = {0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, 6, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; static const uint16_t i_cde352fc27e7ca61[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}; const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { - 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 612, d_cde352fc27e7ca61, m_cde352fc27e7ca61, + 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 615, d_cde352fc27e7ca61, m_cde352fc27e7ca61, 2, 29, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<235> b_89aa8f4e88036b9e = { +static const ::capnp::_::AlignedData<238> b_89aa8f4e88036b9e = { { 0, 0, 0, 0, 5, 0, 6, 0, 158, 107, 3, 136, 78, 143, 170, 137, - 35, 0, 0, 0, 1, 0, 3, 0, + 59, 0, 0, 0, 1, 0, 3, 0, 97, 202, 231, 39, 252, 82, 227, 205, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 114, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 111, 2, 0, 0, + 53, 0, 0, 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -8980,23 +9211,26 @@ static const ::capnp::_::AlignedData<235> b_89aa8f4e88036b9e = { static const uint16_t m_89aa8f4e88036b9e[] = {9, 10, 0, 6, 5, 8, 1, 7, 4, 2, 3}; static const uint16_t i_89aa8f4e88036b9e[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; const ::capnp::_::RawSchema s_89aa8f4e88036b9e = { - 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 235, nullptr, m_89aa8f4e88036b9e, + 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 238, nullptr, m_89aa8f4e88036b9e, 0, 11, i_89aa8f4e88036b9e, nullptr, nullptr, { &s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<106> b_d492b6734d5e3bf5 = { +static const ::capnp::_::AlignedData<109> b_d492b6734d5e3bf5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 59, 94, 77, 115, 182, 146, 212, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 58, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 250, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 31, 1, 0, 0, + 45, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 117, 108, 116, 105, 80, @@ -9101,23 +9335,26 @@ static const ::capnp::_::RawSchema* const d_d492b6734d5e3bf5[] = { static const uint16_t m_d492b6734d5e3bf5[] = {4, 3, 0, 2, 1}; static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_d492b6734d5e3bf5 = { - 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 106, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, + 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 109, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, 2, 5, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { +static const ::capnp::_::AlignedData<52> b_bde8ebd7b13d8625 = { { 0, 0, 0, 0, 5, 0, 6, 0, 37, 134, 61, 177, 215, 235, 232, 189, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 1, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 109, 112, 108, 101, @@ -9161,23 +9398,26 @@ static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { static const uint16_t m_bde8ebd7b13d8625[] = {0, 1}; static const uint16_t i_bde8ebd7b13d8625[] = {0, 1}; const ::capnp::_::RawSchema s_bde8ebd7b13d8625 = { - 0xbde8ebd7b13d8625, b_bde8ebd7b13d8625.words, 49, nullptr, m_bde8ebd7b13d8625, + 0xbde8ebd7b13d8625, b_bde8ebd7b13d8625.words, 52, nullptr, m_bde8ebd7b13d8625, 0, 2, i_bde8ebd7b13d8625, nullptr, nullptr, { &s_bde8ebd7b13d8625, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<54> b_a736c51d292ca752 = { +static const ::capnp::_::AlignedData<57> b_a736c51d292ca752 = { { 0, 0, 0, 0, 5, 0, 6, 0, 82, 167, 44, 41, 29, 197, 54, 167, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 119, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 116, 101, @@ -9226,23 +9466,26 @@ static const ::capnp::_::AlignedData<54> b_a736c51d292ca752 = { static const uint16_t m_a736c51d292ca752[] = {1, 0}; static const uint16_t i_a736c51d292ca752[] = {0, 1}; const ::capnp::_::RawSchema s_a736c51d292ca752 = { - 0xa736c51d292ca752, b_a736c51d292ca752.words, 54, nullptr, m_a736c51d292ca752, + 0xa736c51d292ca752, b_a736c51d292ca752.words, 57, nullptr, m_a736c51d292ca752, 0, 2, i_a736c51d292ca752, nullptr, nullptr, { &s_a736c51d292ca752, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_cd8abc9dabc4b03f = { +static const ::capnp::_::AlignedData<51> b_cd8abc9dabc4b03f = { { 0, 0, 0, 0, 5, 0, 6, 0, 63, 176, 196, 171, 157, 188, 138, 205, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 1, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 66, 117, 102, 102, 101, 114, @@ -9285,23 +9528,26 @@ static const ::capnp::_::AlignedData<48> b_cd8abc9dabc4b03f = { static const uint16_t m_cd8abc9dabc4b03f[] = {1, 0}; static const uint16_t i_cd8abc9dabc4b03f[] = {0, 1}; const ::capnp::_::RawSchema s_cd8abc9dabc4b03f = { - 0xcd8abc9dabc4b03f, b_cd8abc9dabc4b03f.words, 48, nullptr, m_cd8abc9dabc4b03f, + 0xcd8abc9dabc4b03f, b_cd8abc9dabc4b03f.words, 51, nullptr, m_cd8abc9dabc4b03f, 0, 2, i_cd8abc9dabc4b03f, nullptr, nullptr, { &s_cd8abc9dabc4b03f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<55> b_cfea684b4bcd0721 = { +static const ::capnp::_::AlignedData<58> b_cfea684b4bcd0721 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 7, 205, 75, 75, 104, 234, 207, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 146, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 82, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 53, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -9354,23 +9600,26 @@ static const ::capnp::_::RawSchema* const d_cfea684b4bcd0721[] = { static const uint16_t m_cfea684b4bcd0721[] = {0, 1}; static const uint16_t i_cfea684b4bcd0721[] = {0, 1}; const ::capnp::_::RawSchema s_cfea684b4bcd0721 = { - 0xcfea684b4bcd0721, b_cfea684b4bcd0721.words, 55, d_cfea684b4bcd0721, m_cfea684b4bcd0721, + 0xcfea684b4bcd0721, b_cfea684b4bcd0721.words, 58, d_cfea684b4bcd0721, m_cfea684b4bcd0721, 1, 2, i_cfea684b4bcd0721, nullptr, nullptr, { &s_cfea684b4bcd0721, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_aaeeafe1e9f3ea1c = { +static const ::capnp::_::AlignedData<71> b_aaeeafe1e9f3ea1c = { { 0, 0, 0, 0, 5, 0, 6, 0, 28, 234, 243, 233, 225, 175, 238, 170, - 18, 0, 0, 0, 1, 0, 2, 0, + 42, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 2, 0, 0, + 57, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 53, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -9436,23 +9685,26 @@ static const ::capnp::_::RawSchema* const d_aaeeafe1e9f3ea1c[] = { static const uint16_t m_aaeeafe1e9f3ea1c[] = {0, 2, 1}; static const uint16_t i_aaeeafe1e9f3ea1c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_aaeeafe1e9f3ea1c = { - 0xaaeeafe1e9f3ea1c, b_aaeeafe1e9f3ea1c.words, 68, d_aaeeafe1e9f3ea1c, m_aaeeafe1e9f3ea1c, + 0xaaeeafe1e9f3ea1c, b_aaeeafe1e9f3ea1c.words, 71, d_aaeeafe1e9f3ea1c, m_aaeeafe1e9f3ea1c, 1, 3, i_aaeeafe1e9f3ea1c, nullptr, nullptr, { &s_aaeeafe1e9f3ea1c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { +static const ::capnp::_::AlignedData<38> b_f5a35661031194d2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 210, 148, 17, 3, 97, 86, 163, 245, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 98, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 34, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 49, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 67, @@ -9485,23 +9737,26 @@ static const ::capnp::_::RawSchema* const d_f5a35661031194d2[] = { static const uint16_t m_f5a35661031194d2[] = {0}; static const uint16_t i_f5a35661031194d2[] = {0}; const ::capnp::_::RawSchema s_f5a35661031194d2 = { - 0xf5a35661031194d2, b_f5a35661031194d2.words, 35, d_f5a35661031194d2, m_f5a35661031194d2, + 0xf5a35661031194d2, b_f5a35661031194d2.words, 38, d_f5a35661031194d2, m_f5a35661031194d2, 1, 1, i_f5a35661031194d2, nullptr, nullptr, { &s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<34> b_e68edfc0939e63df = { +static const ::capnp::_::AlignedData<37> b_e68edfc0939e63df = { { 0, 0, 0, 0, 5, 0, 6, 0, 223, 99, 158, 147, 192, 223, 142, 230, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 42, 1, 0, 0, - 37, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 86, @@ -9533,23 +9788,26 @@ static const ::capnp::_::RawSchema* const d_e68edfc0939e63df[] = { static const uint16_t m_e68edfc0939e63df[] = {0}; static const uint16_t i_e68edfc0939e63df[] = {0}; const ::capnp::_::RawSchema s_e68edfc0939e63df = { - 0xe68edfc0939e63df, b_e68edfc0939e63df.words, 34, d_e68edfc0939e63df, m_e68edfc0939e63df, + 0xe68edfc0939e63df, b_e68edfc0939e63df.words, 37, d_e68edfc0939e63df, m_e68edfc0939e63df, 1, 1, i_e68edfc0939e63df, nullptr, nullptr, { &s_e68edfc0939e63df, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<55> b_891a70a671f15cf6 = { +static const ::capnp::_::AlignedData<58> b_891a70a671f15cf6 = { { 0, 0, 0, 0, 5, 0, 6, 0, 246, 92, 241, 113, 166, 112, 26, 137, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 82, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 18, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 119, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, @@ -9602,23 +9860,26 @@ static const ::capnp::_::RawSchema* const d_891a70a671f15cf6[] = { static const uint16_t m_891a70a671f15cf6[] = {0, 1}; static const uint16_t i_891a70a671f15cf6[] = {0, 1}; const ::capnp::_::RawSchema s_891a70a671f15cf6 = { - 0x891a70a671f15cf6, b_891a70a671f15cf6.words, 55, d_891a70a671f15cf6, m_891a70a671f15cf6, + 0x891a70a671f15cf6, b_891a70a671f15cf6.words, 58, d_891a70a671f15cf6, m_891a70a671f15cf6, 1, 2, i_891a70a671f15cf6, nullptr, nullptr, { &s_891a70a671f15cf6, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_805c080c10c1e959 = { +static const ::capnp::_::AlignedData<43> b_805c080c10c1e959 = { { 0, 0, 0, 0, 5, 0, 6, 0, 89, 233, 193, 16, 12, 8, 92, 128, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 90, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 26, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 49, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, @@ -9656,23 +9917,26 @@ static const ::capnp::_::RawSchema* const d_805c080c10c1e959[] = { static const uint16_t m_805c080c10c1e959[] = {0}; static const uint16_t i_805c080c10c1e959[] = {0}; const ::capnp::_::RawSchema s_805c080c10c1e959 = { - 0x805c080c10c1e959, b_805c080c10c1e959.words, 40, d_805c080c10c1e959, m_805c080c10c1e959, + 0x805c080c10c1e959, b_805c080c10c1e959.words, 43, d_805c080c10c1e959, m_805c080c10c1e959, 1, 1, i_805c080c10c1e959, nullptr, nullptr, { &s_805c080c10c1e959, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_83f094010132ff21 = { +static const ::capnp::_::AlignedData<55> b_83f094010132ff21 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 255, 50, 1, 1, 148, 240, 131, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 74, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 119, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, @@ -9722,23 +9986,26 @@ static const ::capnp::_::RawSchema* const d_83f094010132ff21[] = { static const uint16_t m_83f094010132ff21[] = {0, 1}; static const uint16_t i_83f094010132ff21[] = {0, 1}; const ::capnp::_::RawSchema s_83f094010132ff21 = { - 0x83f094010132ff21, b_83f094010132ff21.words, 52, d_83f094010132ff21, m_83f094010132ff21, + 0x83f094010132ff21, b_83f094010132ff21.words, 55, d_83f094010132ff21, m_83f094010132ff21, 1, 2, i_83f094010132ff21, nullptr, nullptr, { &s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { +static const ::capnp::_::AlignedData<38> b_ebe17f59ac9a1df1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 241, 29, 154, 172, 89, 127, 225, 235, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 82, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 18, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 49, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, @@ -9771,23 +10038,142 @@ static const ::capnp::_::RawSchema* const d_ebe17f59ac9a1df1[] = { static const uint16_t m_ebe17f59ac9a1df1[] = {0}; static const uint16_t i_ebe17f59ac9a1df1[] = {0}; const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { - 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 35, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, + 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 38, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<53> b_ca2d4d0bfe4ae5d9 = { +static const ::capnp::_::AlignedData<52> b_e06f571aa93eb314 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 20, 179, 62, 169, 26, 87, 111, 224, + 42, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 218, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 81, 117, 101, 114, 121, 80, + 108, 97, 110, 82, 101, 113, 117, 101, + 115, 116, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 117, 101, 114, 121, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 204, 60, 178, 248, 208, 73, 186, 150, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_e06f571aa93eb314 = b_e06f571aa93eb314.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_e06f571aa93eb314[] = { + &s_96ba49d0f8b23ccc, + &s_b6c95b4b8111ad36, +}; +static const uint16_t m_e06f571aa93eb314[] = {0, 1}; +static const uint16_t i_e06f571aa93eb314[] = {0, 1}; +const ::capnp::_::RawSchema s_e06f571aa93eb314 = { + 0xe06f571aa93eb314, b_e06f571aa93eb314.words, 52, d_e06f571aa93eb314, m_e06f571aa93eb314, + 2, 2, i_e06f571aa93eb314, nullptr, nullptr, { &s_e06f571aa93eb314, nullptr, nullptr, 0, 0, nullptr }, true +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<38> b_9fd8fc2f462b2d06 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 6, 45, 43, 70, 47, 252, 216, 159, + 42, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 226, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 81, 117, 101, 114, 121, 80, + 108, 97, 110, 82, 101, 115, 112, 111, + 110, 115, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 3, 0, 1, 0, + 24, 0, 0, 0, 2, 0, 1, 0, + 113, 117, 101, 114, 121, 80, 108, 97, + 110, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_9fd8fc2f462b2d06 = b_9fd8fc2f462b2d06.words; +#if !CAPNP_LITE +static const uint16_t m_9fd8fc2f462b2d06[] = {0}; +static const uint16_t i_9fd8fc2f462b2d06[] = {0}; +const ::capnp::_::RawSchema s_9fd8fc2f462b2d06 = { + 0x9fd8fc2f462b2d06, b_9fd8fc2f462b2d06.words, 38, nullptr, m_9fd8fc2f462b2d06, + 0, 1, i_9fd8fc2f462b2d06, nullptr, nullptr, { &s_9fd8fc2f462b2d06, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<56> b_ca2d4d0bfe4ae5d9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 217, 229, 74, 254, 11, 77, 45, 202, - 18, 0, 0, 0, 1, 0, 1, 0, + 42, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 186, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 67, @@ -9838,23 +10224,26 @@ static const ::capnp::_::RawSchema* const d_ca2d4d0bfe4ae5d9[] = { static const uint16_t m_ca2d4d0bfe4ae5d9[] = {1, 0}; static const uint16_t i_ca2d4d0bfe4ae5d9[] = {0, 1}; const ::capnp::_::RawSchema s_ca2d4d0bfe4ae5d9 = { - 0xca2d4d0bfe4ae5d9, b_ca2d4d0bfe4ae5d9.words, 53, d_ca2d4d0bfe4ae5d9, m_ca2d4d0bfe4ae5d9, + 0xca2d4d0bfe4ae5d9, b_ca2d4d0bfe4ae5d9.words, 56, d_ca2d4d0bfe4ae5d9, m_ca2d4d0bfe4ae5d9, 1, 2, i_ca2d4d0bfe4ae5d9, nullptr, nullptr, { &s_ca2d4d0bfe4ae5d9, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_e193f1f45a9f102e = { +static const ::capnp::_::AlignedData<68> b_e193f1f45a9f102e = { { 0, 0, 0, 0, 5, 0, 6, 0, 46, 16, 159, 90, 244, 241, 147, 225, - 18, 0, 0, 0, 1, 0, 0, 0, + 42, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 0, 0, 0, - 33, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 103, 103, 114, 101, 103, @@ -9914,7 +10303,7 @@ static const ::capnp::_::AlignedData<65> b_e193f1f45a9f102e = { static const uint16_t m_e193f1f45a9f102e[] = {1, 2, 0}; static const uint16_t i_e193f1f45a9f102e[] = {0, 1, 2}; const ::capnp::_::RawSchema s_e193f1f45a9f102e = { - 0xe193f1f45a9f102e, b_e193f1f45a9f102e.words, 65, nullptr, m_e193f1f45a9f102e, + 0xe193f1f45a9f102e, b_e193f1f45a9f102e.words, 68, nullptr, m_e193f1f45a9f102e, 0, 3, i_e193f1f45a9f102e, nullptr, nullptr, { &s_e193f1f45a9f102e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE @@ -10972,6 +11361,30 @@ constexpr ::capnp::_::RawSchema const* LoadArraySchemaResponse::_capnpPrivate::s #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// QueryPlanRequest +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t QueryPlanRequest::_capnpPrivate::dataWordSize; +constexpr uint16_t QueryPlanRequest::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind QueryPlanRequest::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* QueryPlanRequest::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// QueryPlanResponse +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t QueryPlanResponse::_capnpPrivate::dataWordSize; +constexpr uint16_t QueryPlanResponse::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind QueryPlanResponse::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* QueryPlanResponse::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + // QueryChannel #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr uint16_t QueryChannel::_capnpPrivate::dataWordSize; diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index dcd1f3aeb43f..8b1304b4ac7d 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -108,6 +108,8 @@ CAPNP_DECLARE_SCHEMA(891a70a671f15cf6); CAPNP_DECLARE_SCHEMA(805c080c10c1e959); CAPNP_DECLARE_SCHEMA(83f094010132ff21); CAPNP_DECLARE_SCHEMA(ebe17f59ac9a1df1); +CAPNP_DECLARE_SCHEMA(e06f571aa93eb314); +CAPNP_DECLARE_SCHEMA(9fd8fc2f462b2d06); CAPNP_DECLARE_SCHEMA(ca2d4d0bfe4ae5d9); CAPNP_DECLARE_SCHEMA(e193f1f45a9f102e); @@ -1687,6 +1689,40 @@ struct LoadArraySchemaResponse { }; }; +struct QueryPlanRequest { + QueryPlanRequest() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(e06f571aa93eb314, 0, 2) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct QueryPlanResponse { + QueryPlanResponse() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(9fd8fc2f462b2d06, 0, 1) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + struct QueryChannel { QueryChannel() = delete; @@ -14934,6 +14970,218 @@ class LoadArraySchemaResponse::Pipeline { }; #endif // !CAPNP_LITE +class QueryPlanRequest::Reader { + public: + typedef QueryPlanRequest Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasConfig() const; + inline ::tiledb::sm::serialization::capnp::Config::Reader getConfig() const; + + inline bool hasQuery() const; + inline ::tiledb::sm::serialization::capnp::Query::Reader getQuery() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class QueryPlanRequest::Builder { + public: + typedef QueryPlanRequest Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasConfig(); + inline ::tiledb::sm::serialization::capnp::Config::Builder getConfig(); + inline void setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value); + inline ::tiledb::sm::serialization::capnp::Config::Builder initConfig(); + inline void adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> + disownConfig(); + + inline bool hasQuery(); + inline ::tiledb::sm::serialization::capnp::Query::Builder getQuery(); + inline void setQuery(::tiledb::sm::serialization::capnp::Query::Reader value); + inline ::tiledb::sm::serialization::capnp::Query::Builder initQuery(); + inline void adoptQuery( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Query>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Query> + disownQuery(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class QueryPlanRequest::Pipeline { + public: + typedef QueryPlanRequest Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + inline ::tiledb::sm::serialization::capnp::Config::Pipeline getConfig(); + inline ::tiledb::sm::serialization::capnp::Query::Pipeline getQuery(); + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class QueryPlanResponse::Reader { + public: + typedef QueryPlanResponse Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasQueryPlan() const; + inline ::capnp::Text::Reader getQueryPlan() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class QueryPlanResponse::Builder { + public: + typedef QueryPlanResponse Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasQueryPlan(); + inline ::capnp::Text::Builder getQueryPlan(); + inline void setQueryPlan(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initQueryPlan(unsigned int size); + inline void adoptQueryPlan(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownQueryPlan(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class QueryPlanResponse::Pipeline { + public: + typedef QueryPlanResponse Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class QueryChannel::Reader { public: typedef QueryChannel Reads; @@ -32061,6 +32309,147 @@ LoadArraySchemaResponse::Builder::disownSchema() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool QueryPlanRequest::Reader::hasConfig() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanRequest::Builder::hasConfig() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Config::Reader +QueryPlanRequest::Reader::getConfig() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Config::Builder +QueryPlanRequest::Builder::getConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Config::Pipeline +QueryPlanRequest::Pipeline::getConfig() { + return ::tiledb::sm::serialization::capnp::Config::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void QueryPlanRequest::Builder::setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Config::Builder +QueryPlanRequest::Builder::initConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void QueryPlanRequest::Builder::adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> +QueryPlanRequest::Builder::disownConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool QueryPlanRequest::Reader::hasQuery() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanRequest::Builder::hasQuery() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Query::Reader +QueryPlanRequest::Reader::getQuery() const { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>:: + get(_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Query::Builder +QueryPlanRequest::Builder::getQuery() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>:: + get(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Query::Pipeline +QueryPlanRequest::Pipeline::getQuery() { + return ::tiledb::sm::serialization::capnp::Query::Pipeline( + _typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void QueryPlanRequest::Builder::setQuery( + ::tiledb::sm::serialization::capnp::Query::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Query::Builder +QueryPlanRequest::Builder::initQuery() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>:: + init(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void QueryPlanRequest::Builder::adoptQuery( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Query>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Query> +QueryPlanRequest::Builder::disownQuery() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Query>:: + disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool QueryPlanResponse::Reader::hasQueryPlan() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanResponse::Builder::hasQueryPlan() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader QueryPlanResponse::Reader::getQueryPlan() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::getQueryPlan() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void QueryPlanResponse::Builder::setQueryPlan( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::initQueryPlan( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void QueryPlanResponse::Builder::adoptQueryPlan( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +QueryPlanResponse::Builder::disownQueryPlan() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + inline bool QueryChannel::Reader::getDefault() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } From c7268a7b8d76407310499ada5dbfba607c8ab42d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 Nov 2023 18:40:16 +0200 Subject: [PATCH 075/456] Undeprecate `tiledb_group_create`. (#4522) This API is the only way to create groups. I have discussed this with other people in the past and we should undeprecate it. --- TYPE: C_API DESC: `tiledb_group_create` is no longer marked as deprecated. --- tiledb/api/c_api/group/group_api_external.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/api/c_api/group/group_api_external.h b/tiledb/api/c_api/group/group_api_external.h index b9f5b9df3d2d..a780f395c143 100644 --- a/tiledb/api/c_api/group/group_api_external.h +++ b/tiledb/api/c_api/group/group_api_external.h @@ -56,7 +56,7 @@ typedef struct tiledb_group_handle_t tiledb_group_t; * @param group_uri The group URI. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ -TILEDB_DEPRECATED_EXPORT capi_return_t +TILEDB_EXPORT capi_return_t tiledb_group_create(tiledb_ctx_t* ctx, const char* group_uri) TILEDB_NOEXCEPT; #ifdef __cplusplus From a4f7823e2a1ccc6ff7db7b6d66a15cf43ec5694a Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 Nov 2023 18:47:24 +0200 Subject: [PATCH 076/456] Update curl to version 8.4.0. (#4523) After #4505 it became surprisingly simple. --- TYPE: IMPROVEMENT DESC: Update curl to version 8.4.0. --- cmake/Options/TileDBToolchain.cmake | 2 +- ports/curl/0002_fix_uwp.patch | 15 -- ports/curl/0005_remove_imp_suffix.patch | 12 -- ports/curl/0012-fix-dependency-idn2.patch | 14 -- ports/curl/0020-fix-pc-file.patch | 15 -- ports/curl/0022-deduplicate-libs.patch | 12 -- ports/curl/0023-fix-find-cares.patch | 29 --- ports/curl/CVE-2023-38545_7.87.0.patch | 134 -------------- ports/curl/cmake-project-include.cmake | 82 --------- ports/curl/export-components.patch | 23 --- ports/curl/mbedtls-ws2_32.patch | 14 -- ports/curl/portfile.cmake | 129 ------------- ports/curl/vcpkg-cmake-wrapper.cmake | 68 ------- ports/curl/vcpkg.json | 211 ---------------------- vcpkg.json | 4 + 15 files changed, 5 insertions(+), 759 deletions(-) delete mode 100644 ports/curl/0002_fix_uwp.patch delete mode 100644 ports/curl/0005_remove_imp_suffix.patch delete mode 100644 ports/curl/0012-fix-dependency-idn2.patch delete mode 100644 ports/curl/0020-fix-pc-file.patch delete mode 100644 ports/curl/0022-deduplicate-libs.patch delete mode 100644 ports/curl/0023-fix-find-cares.patch delete mode 100644 ports/curl/CVE-2023-38545_7.87.0.patch delete mode 100644 ports/curl/cmake-project-include.cmake delete mode 100644 ports/curl/export-components.patch delete mode 100644 ports/curl/mbedtls-ws2_32.patch delete mode 100644 ports/curl/portfile.cmake delete mode 100644 ports/curl/vcpkg-cmake-wrapper.cmake delete mode 100644 ports/curl/vcpkg.json diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index c68053f5d423..f6ca3e16c70b 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -20,7 +20,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # To help with resolving conflicts, when you update the commit, also update its date. - set(VCPKG_COMMIT_STRING 1b4d69f3028d74401a001aa316986a670ca6289a) # 2023-09-27 + set(VCPKG_COMMIT_STRING ac2a14f35fcd57d7a38f09af75dd5258e96dd6ac) # 2023-11-16 message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") include(FetchContent) FetchContent_Declare( diff --git a/ports/curl/0002_fix_uwp.patch b/ports/curl/0002_fix_uwp.patch deleted file mode 100644 index 71f1d83c9488..000000000000 --- a/ports/curl/0002_fix_uwp.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt -index 8cea346c3..967312905 100644 ---- a/lib/CMakeLists.txt -+++ b/lib/CMakeLists.txt -@@ -118,6 +118,10 @@ if(CURL_HAS_LTO) - INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) - endif() - -+if(CMAKE_SYSTEM_NAME STREQUAL "WindowsStore") -+ set_property(TARGET ${LIB_NAME} APPEND PROPERTY COMPILE_DEFINITIONS "_WINSOCK_DEPRECATED_NO_WARNINGS") -+endif() -+ - if(WIN32) - if(BUILD_SHARED_LIBS) - if(MSVC) diff --git a/ports/curl/0005_remove_imp_suffix.patch b/ports/curl/0005_remove_imp_suffix.patch deleted file mode 100644 index e52da2585b61..000000000000 --- a/ports/curl/0005_remove_imp_suffix.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt -index 1d71e14..62b7b33 100644 ---- a/lib/CMakeLists.txt -+++ b/lib/CMakeLists.txt -@@ -125,7 +125,6 @@ if(WIN32) - if(MSVC) - # Add "_imp" as a suffix before the extension to avoid conflicting with - # the statically linked "libcurl.lib" -- set_target_properties(${LIB_NAME} PROPERTIES IMPORT_SUFFIX "_imp.lib") - endif() - endif() - endif() diff --git a/ports/curl/0012-fix-dependency-idn2.patch b/ports/curl/0012-fix-dependency-idn2.patch deleted file mode 100644 index 9db27979e1aa..000000000000 --- a/ports/curl/0012-fix-dependency-idn2.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 9f6de81..6702845 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -632,6 +632,7 @@ endif() - # Check for idn2 - option(USE_LIBIDN2 "Use libidn2 for IDN support" ON) - if(USE_LIBIDN2) -- check_library_exists_concat("idn2" idn2_lookup_ul HAVE_LIBIDN2) -+ set(HAVE_LIBIDN2 TRUE) -+ list(INSERT CURL_LIBS 0 ${LIBIDN2_LINK_LIBRARIES}) - else() - set(HAVE_LIBIDN2 OFF) - endif() diff --git a/ports/curl/0020-fix-pc-file.patch b/ports/curl/0020-fix-pc-file.patch deleted file mode 100644 index 21eab0933baf..000000000000 --- a/ports/curl/0020-fix-pc-file.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 8b2e428..ea430f4 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -1505,7 +1505,9 @@ set(includedir "\${prefix}/include") - set(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS}") - set(LIBCURL_LIBS "") - set(libdir "${CMAKE_INSTALL_PREFIX}/lib") --foreach(_lib ${CMAKE_C_IMPLICIT_LINK_LIBRARIES} ${CURL_LIBS}) -+set(CURL_LIBS_FLAT "") -+vcpkg_curl_flatten(CURL_LIBS CURL_LIBS_FLAT) -+foreach(_lib ${CURL_LIBS_FLAT}) - if(TARGET "${_lib}") - set(_libname "${_lib}") - get_target_property(_imported "${_libname}" IMPORTED) diff --git a/ports/curl/0022-deduplicate-libs.patch b/ports/curl/0022-deduplicate-libs.patch deleted file mode 100644 index 16cc0498a3b1..000000000000 --- a/ports/curl/0022-deduplicate-libs.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 09d82f1..f0d99e6 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -1507,6 +1507,7 @@ else() - set(ENABLE_SHARED "no") - set(ENABLE_STATIC "yes") - set(LIBCURL_NO_SHARED "${LIBCURL_LIBS}") -+ set(LIBCURL_LIBS "") - endif() - # "a" (Linux) or "lib" (Windows) - string(REPLACE "." "" libext "${CMAKE_STATIC_LIBRARY_SUFFIX}") diff --git a/ports/curl/0023-fix-find-cares.patch b/ports/curl/0023-fix-find-cares.patch deleted file mode 100644 index 4aa11acfa1c6..000000000000 --- a/ports/curl/0023-fix-find-cares.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/CMake/curl-config.cmake.in b/CMake/curl-config.cmake.in -index 496a92d0e..564415ef6 100644 ---- a/CMake/curl-config.cmake.in -+++ b/CMake/curl-config.cmake.in -@@ -30,6 +30,9 @@ endif() - if(@USE_ZLIB@) - find_dependency(ZLIB @ZLIB_VERSION_MAJOR@) - endif() -+if(@USE_ARES@) -+ find_dependency(c-ares) -+endif() - - include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") - check_required_components("@PROJECT_NAME@") -diff --git a/CMakeLists.txt b/CMakeLists.txt -index b43520751..dbf62751f 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -169,8 +169,8 @@ set(CURL_LIBS "") - - if(ENABLE_ARES) - set(USE_ARES 1) -- find_package(CARES REQUIRED) -- list(APPEND CURL_LIBS ${CARES_LIBRARY}) -+ find_package(c-ares CONFIG REQUIRED) -+ list(APPEND CURL_LIBS c-ares::cares) - endif() - - include(CurlSymbolHiding) diff --git a/ports/curl/CVE-2023-38545_7.87.0.patch b/ports/curl/CVE-2023-38545_7.87.0.patch deleted file mode 100644 index c15c273ea41e..000000000000 --- a/ports/curl/CVE-2023-38545_7.87.0.patch +++ /dev/null @@ -1,134 +0,0 @@ -From 92fd36dd54de9ac845549944692eb33c5aee7343 Mon Sep 17 00:00:00 2001 -From: Jay Satiro -Date: Mon, 9 Oct 2023 17:15:44 -0400 -Subject: [PATCH] socks: return error if hostname too long for remote resolve - -Prior to this change the state machine attempted to change the remote -resolve to a local resolve if the hostname was longer than 255 -characters. Unfortunately that did not work as intended and caused a -security issue. - -This patch applies to curl versions 7.87.0 - 8.1.2. Other versions -that are affected take a different patch. Refer to the CVE advisory -for more information. - -Bug: https://curl.se/docs/CVE-2023-38545.html ---- - lib/socks.c | 8 +++---- - tests/data/Makefile.inc | 2 +- - tests/data/test728 | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ - 3 files changed, 69 insertions(+), 5 deletions(-) - create mode 100644 tests/data/test728 - -diff --git a/lib/socks.c b/lib/socks.c -index d491e08..e7da5b4 100644 ---- a/lib/socks.c -+++ b/lib/socks.c -@@ -539,9 +539,9 @@ static CURLproxycode do_SOCKS5(struct Curl_cfilter *cf, - - /* RFC1928 chapter 5 specifies max 255 chars for domain name in packet */ - if(!socks5_resolve_local && hostname_len > 255) { -- infof(data, "SOCKS5: server resolving disabled for hostnames of " -- "length > 255 [actual len=%zu]", hostname_len); -- socks5_resolve_local = TRUE; -+ failf(data, "SOCKS5: the destination hostname is too long to be " -+ "resolved remotely by the proxy."); -+ return CURLPX_LONG_HOSTNAME; - } - - if(auth & ~(CURLAUTH_BASIC | CURLAUTH_GSSAPI)) -@@ -882,7 +882,7 @@ static CURLproxycode do_SOCKS5(struct Curl_cfilter *cf, - } - else { - socksreq[len++] = 3; -- socksreq[len++] = (char) hostname_len; /* one byte address length */ -+ socksreq[len++] = (unsigned char) hostname_len; /* one byte length */ - memcpy(&socksreq[len], sx->hostname, hostname_len); /* w/o NULL */ - len += hostname_len; - } -diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc -index 3e0221a..64b11de 100644 ---- a/tests/data/Makefile.inc -+++ b/tests/data/Makefile.inc -@@ -99,7 +99,7 @@ test679 test680 test681 test682 test683 test684 test685 \ - \ - test700 test701 test702 test703 test704 test705 test706 test707 test708 \ - test709 test710 test711 test712 test713 test714 test715 test716 test717 \ --test718 test719 test720 test721 \ -+test718 test719 test720 test721 test728 \ - \ - test800 test801 test802 test803 test804 test805 test806 test807 test808 \ - test809 test810 test811 test812 test813 test814 test815 test816 test817 \ -diff --git a/tests/data/test728 b/tests/data/test728 -new file mode 100644 -index 0000000..05bcf28 ---- /dev/null -+++ b/tests/data/test728 -@@ -0,0 +1,64 @@ -+ -+ -+ -+HTTP -+HTTP GET -+SOCKS5 -+SOCKS5h -+followlocation -+ -+ -+ -+# -+# Server-side -+ -+# The hostname in this redirect is 256 characters and too long (> 255) for -+# SOCKS5 remote resolve. curl must return error CURLE_PROXY in this case. -+ -+HTTP/1.1 301 Moved Permanently -+Location: http://AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/ -+Content-Length: 0 -+Connection: close -+ -+ -+ -+ -+# -+# Client-side -+ -+ -+proxy -+ -+ -+http -+socks5 -+ -+ -+SOCKS5h with HTTP redirect to hostname too long -+ -+ -+--no-progress-meter --location --proxy socks5h://%HOSTIP:%SOCKSPORT http://%HOSTIP:%HTTPPORT/%TESTNUMBER -+ -+ -+ -+# -+# Verify data after the test has been "shot" -+ -+ -+GET /%TESTNUMBER HTTP/1.1 -+Host: %HOSTIP:%HTTPPORT -+User-Agent: curl/%VERSION -+Accept: */* -+ -+ -+ -+97 -+ -+# the error message is verified because error code CURLE_PROXY (97) may be -+# returned for any number of reasons and we need to make sure it is -+# specifically for the reason below so that we know the check is working. -+ -+curl: (97) SOCKS5: the destination hostname is too long to be resolved remotely by the proxy. -+ -+ -+ --- -2.7.4 - diff --git a/ports/curl/cmake-project-include.cmake b/ports/curl/cmake-project-include.cmake deleted file mode 100644 index 27254864c20c..000000000000 --- a/ports/curl/cmake-project-include.cmake +++ /dev/null @@ -1,82 +0,0 @@ -# Process the libs and targets in the variable named by `input` -# into a flat list of libs in the variable named by `output`. -# Simplify -framework elements. -# Use -l where possible. -# Avoid duplicates. -function(vcpkg_curl_flatten input output) - set(output_libs "${${output}}") - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - string(REGEX REPLACE ";optimized;[^;]*|;debug" "" input_libs "VCPKG;${${input}}") - else() - string(REGEX REPLACE ";debug;[^;]*|;optimized" "" input_libs "VCPKG;${${input}}") - endif() - list(REMOVE_AT input_libs 0) - while(input_libs) - list(POP_BACK input_libs lib) - if(TARGET "${lib}") - set(import_lib "") - set(import_location "") - get_target_property(type "${lib}" TYPE) - if(NOT type STREQUAL "INTERFACE_LIBRARY") - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - get_target_property(link_libs "${lib}" IMPORTED_LINK_INTERFACE_LIBRARIES_DEBUG) - get_target_property(import_lib "${lib}" IMPORTED_IMPLIB_DEBUG) - get_target_property(import_location "${lib}" IMPORTED_LOCATION_DEBUG) - else() - get_target_property(link_libs "${lib}" IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE) - get_target_property(import_lib "${lib}" IMPORTED_IMPLIB_RELEASE) - get_target_property(import_location "${lib}" IMPORTED_LOCATION_RELEASE) - endif() - if(link_libs) - vcpkg_curl_flatten(link_libs output_libs) - endif() - get_target_property(link_libs "${lib}" IMPORTED_LINK_INTERFACE_LIBRARIES) - if(link_libs) - vcpkg_curl_flatten(link_libs output_libs) - endif() - if(NOT import_lib) - get_target_property(import_lib "${lib}" IMPORTED_IMPLIB) - endif() - if(NOT import_location) - get_target_property(import_location "${lib}" IMPORTED_LOCATION) - endif() - endif() - get_target_property(link_libs "${lib}" INTERFACE_LINK_LIBRARIES) - if(link_libs) - vcpkg_curl_flatten(link_libs output_libs) - endif() - if(import_lib) - set(lib "${import_lib}") - elseif(import_location) - set(lib "${import_location}") - endif() - endif() - if(lib MATCHES "^(.*)/([^/]*)[.]framework$") - if(CMAKE_MATCH_1 IN_LIST CMAKE_C_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES) - set(lib "-framework ${CMAKE_MATCH_2}") - else() - set(lib "-framework ${lib}") - endif() - elseif(WIN32 AND lib MATCHES ".*/${CMAKE_IMPORT_LIBRARY_PREFIX}([^/]*)${CMAKE_IMPORT_LIBRARY_SUFFIX}") - set(lib -l${CMAKE_MATCH_1}) - elseif(lib MATCHES ".*/${CMAKE_STATIC_LIBRARY_PREFIX}([^/]*)${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(lib -l${CMAKE_MATCH_1}) - endif() - if(NOT "${lib}" IN_LIST output_libs) - list(PREPEND output_libs "${lib}") - endif() - endwhile() - set("${output}" "${output_libs}" PARENT_SCOPE) -endfunction() - -if(CURL_USE_LIBSSH2) - find_package(Libssh2 CONFIG REQUIRED) - set(LIBSSH2_FOUND TRUE) - get_target_property(LIBSSH2_INCLUDE_DIR Libssh2::libssh2 INTERFACE_INCLUDE_DIRECTORIES) - set(LIBSSH2_LIBRARY Libssh2::libssh2) -endif() - -if(USE_LIBIDN2) - find_package(PkgConfig REQUIRED) - pkg_check_modules(LIBIDN2 REQUIRED libidn2) -endif() diff --git a/ports/curl/export-components.patch b/ports/curl/export-components.patch deleted file mode 100644 index 1a727e5deacc..000000000000 --- a/ports/curl/export-components.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 874a237..3974956 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -1617,8 +1617,17 @@ if(CURL_ENABLE_EXPORT_TARGET) - ) - endif() - -+set(components_file "${CMAKE_CURRENT_BINARY_DIR}/CURLConfigComponents.cmake") -+file(CONFIGURE OUTPUT "${components_file}" CONTENT [[ -+foreach(z_vcpkg_curl_component IN ITEMS @SUPPORT_FEATURES@ @SUPPORT_PROTOCOLS@) -+ if(z_vcpkg_curl_component MATCHES "^[-_a-zA-Z0-9]*$") -+ set(CURL_${z_vcpkg_curl_component}_FOUND TRUE) -+ endif() -+endforeach() -+]] @ONLY) -+ - install( -- FILES ${version_config} ${project_config} -+ FILES ${version_config} ${project_config} ${components_file} - DESTINATION ${CURL_INSTALL_CMAKE_DIR} - ) - diff --git a/ports/curl/mbedtls-ws2_32.patch b/ports/curl/mbedtls-ws2_32.patch deleted file mode 100644 index 37e4f18c94e6..000000000000 --- a/ports/curl/mbedtls-ws2_32.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/CMake/FindMbedTLS.cmake b/CMake/FindMbedTLS.cmake -index 7bdb197..c8c5a73 100644 ---- a/CMake/FindMbedTLS.cmake -+++ b/CMake/FindMbedTLS.cmake -@@ -26,6 +26,9 @@ find_library(MBEDX509_LIBRARY mbedx509) - find_library(MBEDCRYPTO_LIBRARY mbedcrypto) - - set(MBEDTLS_LIBRARIES "${MBEDTLS_LIBRARY}" "${MBEDX509_LIBRARY}" "${MBEDCRYPTO_LIBRARY}") -+if(WIN32) -+ list(APPEND MBEDTLS_LIBRARIES ws2_32) -+endif() - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(MbedTLS DEFAULT_MSG diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake deleted file mode 100644 index 2492fbb10e80..000000000000 --- a/ports/curl/portfile.cmake +++ /dev/null @@ -1,129 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO curl/curl - REF curl-7_88_1 - SHA512 c5caa1f95580ddbf2041c9c4b885f84d4f5c5fcb905a5ea59f9dbb58a98fc292260f95cb935e963bf83d7dcecf98561deef5ce3ff91cdcb29a080559cff0ed64 - HEAD_REF master - PATCHES - 0002_fix_uwp.patch - 0005_remove_imp_suffix.patch - 0012-fix-dependency-idn2.patch - 0020-fix-pc-file.patch - 0022-deduplicate-libs.patch - mbedtls-ws2_32.patch - export-components.patch - 0023-fix-find-cares.patch - CVE-2023-38545_7.87.0.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - # Support HTTP2 TLS Download https://curl.haxx.se/ca/cacert.pem rename to curl-ca-bundle.crt, copy it to libcurl.dll location. - http2 USE_NGHTTP2 - wolfssl CURL_USE_WOLFSSL - openssl CURL_USE_OPENSSL - mbedtls CURL_USE_MBEDTLS - ssh CURL_USE_LIBSSH2 - tool BUILD_CURL_EXE - c-ares ENABLE_ARES - sspi CURL_WINDOWS_SSPI - brotli CURL_BROTLI - schannel CURL_USE_SCHANNEL - sectransp CURL_USE_SECTRANSP - idn2 USE_LIBIDN2 - winidn USE_WIN32_IDN - winldap USE_WIN32_LDAP - websockets ENABLE_WEBSOCKETS - zstd CURL_ZSTD - INVERTED_FEATURES - non-http HTTP_ONLY - winldap CURL_DISABLE_LDAP # Only WinLDAP support ATM -) - -set(OPTIONS "") -if("idn2" IN_LIST FEATURES) - vcpkg_find_acquire_program(PKGCONFIG) - list(APPEND OPTIONS "-DPKG_CONFIG_EXECUTABLE=${PKGCONFIG}") -endif() - -if("sectransp" IN_LIST FEATURES) - list(APPEND OPTIONS -DCURL_CA_PATH=none -DCURL_CA_BUNDLE=none) -endif() - -# UWP targets -if(VCPKG_TARGET_IS_UWP) - list(APPEND OPTIONS - -DCURL_DISABLE_TELNET=ON - -DENABLE_IPV6=OFF - -DENABLE_UNIX_SOCKETS=OFF - ) -endif() - -if(VCPKG_TARGET_IS_WINDOWS) - list(APPEND OPTIONS -DENABLE_UNICODE=ON) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_PROJECT_INCLUDE=${CMAKE_CURRENT_LIST_DIR}/cmake-project-include.cmake" - ${FEATURE_OPTIONS} - ${OPTIONS} - -DBUILD_TESTING=OFF - -DENABLE_MANUAL=OFF - -DCURL_CA_FALLBACK=ON - -DCURL_USE_LIBPSL=OFF - OPTIONS_DEBUG - -DENABLE_DEBUG=ON -) -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -if ("tool" IN_LIST FEATURES) - vcpkg_copy_tools(TOOL_NAMES curl AUTO_CLEAN) -endif() - -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/CURL) - -vcpkg_fixup_pkgconfig() -set(namespec "curl") -if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - set(namespec "libcurl") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/libcurl.pc" " -lcurl" " -l${namespec}") -endif() -if(NOT DEFINED VCPKG_BUILD_TYPE) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/libcurl.pc" " -lcurl" " -l${namespec}-d") -endif() - -#Fix install path -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/curl-config" "${CURRENT_PACKAGES_DIR}" "\${prefix}") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/curl-config" "${CURRENT_INSTALLED_DIR}" "\${prefix}") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/curl-config" "\nprefix=\${prefix}" [=[prefix=$(CDPATH= cd -- "$(dirname -- "$0")"/../../.. && pwd -P)]=]) -file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/bin") -file(RENAME "${CURRENT_PACKAGES_DIR}/bin/curl-config" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/bin/curl-config") -if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/bin/curl-config") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "${CURRENT_PACKAGES_DIR}" "\${prefix}") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "${CURRENT_INSTALLED_DIR}" "\${prefix}") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "\nprefix=\${prefix}/debug" [=[prefix=$(CDPATH= cd -- "$(dirname -- "$0")"/../../../.. && pwd -P)]=]) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "-lcurl" "-l${namespec}-d") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "curl." "curl-d.") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/bin") - file(RENAME "${CURRENT_PACKAGES_DIR}/debug/bin/curl-config" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/bin/curl-config") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static" OR NOT VCPKG_TARGET_IS_WINDOWS) - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/curl/curl.h" - "#ifdef CURL_STATICLIB" - "#if 1" - ) -endif() - -file(INSTALL "${CURRENT_PORT_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") diff --git a/ports/curl/vcpkg-cmake-wrapper.cmake b/ports/curl/vcpkg-cmake-wrapper.cmake deleted file mode 100644 index c74dc2fb03fe..000000000000 --- a/ports/curl/vcpkg-cmake-wrapper.cmake +++ /dev/null @@ -1,68 +0,0 @@ -cmake_policy(PUSH) -cmake_policy(SET CMP0012 NEW) -cmake_policy(SET CMP0054 NEW) -cmake_policy(SET CMP0057 NEW) - -if(NOT CMAKE_VERSION VERSION_LESS 3.14 AND COMPONENTS IN_LIST ARGS) - include("${CMAKE_CURRENT_LIST_DIR}/CURLConfigComponents.cmake") -endif() - -list(REMOVE_ITEM ARGS "NO_MODULE" "CONFIG" "MODULE") -_find_package(${ARGS} CONFIG) - -if(CURL_FOUND) - include("${CMAKE_ROOT}/Modules/SelectLibraryConfigurations.cmake") - - get_target_property(_curl_include_dirs CURL::libcurl INTERFACE_INCLUDE_DIRECTORIES) - get_target_property(_curl_link_libraries CURL::libcurl INTERFACE_LINK_LIBRARIES) - if(NOT _curl_link_libraries) - set(_curl_link_libraries "") - endif() - if(_curl_link_libraries MATCHES "ZLIB::ZLIB") - string(REGEX REPLACE "([\$]<[^;]*)?ZLIB::ZLIB([^;]*>)?" "${ZLIB_LIBRARIES}" _curl_link_libraries "${_curl_link_libraries}") - endif() - if(_curl_link_libraries MATCHES "OpenSSL::") - string(REGEX REPLACE "([\$]<[^;]*)?OpenSSL::(SSL|Crypto)([^;]*>)?" "${OPENSSL_LIBRARIES}" _curl_link_libraries "${_curl_link_libraries}") - endif() - if(_curl_link_libraries MATCHES "Libssh2::libssh2") - # TODO: move find_dependency(Libssh2 CONFIG) into CURL config - find_package(Libssh2 CONFIG QUIET) - get_target_property(_libssh2_LIBRARY_DEBUG Libssh2::libssh2 IMPORTED_IMPLIB_DEBUG) - get_target_property(_libssh2_LIBRARY_RELEASE Libssh2::libssh2 IMPORTED_IMPLIB_RELEASE) - if(NOT IMPORTED_IMPLIB_DEBUG AND NOT IMPORTED_IMPLIB_RELEASE) - get_target_property(_libssh2_LIBRARY_DEBUG Libssh2::libssh2 IMPORTED_LOCATION_DEBUG) - get_target_property(_libssh2_LIBRARY_RELEASE Libssh2::libssh2 IMPORTED_LOCATION_RELEASE) - endif() - select_library_configurations(_libssh2) - string(REGEX REPLACE "([\$]<[^;]*)?Libssh2::libssh2([^;]*>)?" "${_libssh2_LIBRARIES}" _curl_link_libraries "${_curl_link_libraries}") - unset(_libssh2_LIBRARIES) - unset(_libssh2_LIBRARY_DEBUG) - unset(_libssh2_LIBRARY_RELEASE) - endif() - if(_curl_link_libraries MATCHES "::") - message(WARNING "CURL_LIBRARIES list at least one target. This will not work for use cases where targets are not resolved.") - endif() - - if (CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "WindowsStore") - get_target_property(_curl_location_debug CURL::libcurl IMPORTED_IMPLIB_DEBUG) - get_target_property(_curl_location_release CURL::libcurl IMPORTED_IMPLIB_RELEASE) - endif() - - if(NOT _curl_location_debug AND NOT _curl_location_release) - get_target_property(_curl_location_debug CURL::libcurl IMPORTED_LOCATION_DEBUG) - get_target_property(_curl_location_release CURL::libcurl IMPORTED_LOCATION_RELEASE) - endif() - - set(CURL_INCLUDE_DIRS "${_curl_include_dirs}") - set(CURL_LIBRARY_DEBUG "${_curl_location_debug}" CACHE INTERNAL "vcpkg") - set(CURL_LIBRARY_RELEASE "${_curl_location_release}" CACHE INTERNAL "vcpkg") - select_library_configurations(CURL) - set(CURL_LIBRARIES ${CURL_LIBRARY} ${_curl_link_libraries}) - set(CURL_VERSION_STRING "${CURL_VERSION}") - - unset(_curl_include_dirs) - unset(_curl_link_libraries) - unset(_curl_location_debug) - unset(_curl_location_release) -endif() -cmake_policy(POP) diff --git a/ports/curl/vcpkg.json b/ports/curl/vcpkg.json deleted file mode 100644 index a07a8d013390..000000000000 --- a/ports/curl/vcpkg.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "name": "curl", - "version": "7.88.1", - "port-version": 1, - "description": "A library for transferring data with URLs", - "homepage": "https://curl.se/", - "license": null, - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - }, - "zlib" - ], - "default-features": [ - "non-http", - "ssl" - ], - "features": { - "brotli": { - "description": "brotli support (brotli)", - "dependencies": [ - "brotli" - ] - }, - "c-ares": { - "description": "c-ares support", - "dependencies": [ - "c-ares" - ] - }, - "http2": { - "description": "HTTP2 support", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "ssl" - ] - }, - "nghttp2" - ] - }, - "idn": { - "description": "Default IDN support", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "winidn" - ], - "platform": "windows" - }, - { - "name": "curl", - "default-features": false, - "features": [ - "idn2" - ], - "platform": "!windows" - } - ] - }, - "idn2": { - "description": "idn2 support (libidn2)", - "dependencies": [ - "libidn2" - ] - }, - "mbedtls": { - "description": "SSL support (mbedTLS)", - "dependencies": [ - "mbedtls" - ] - }, - "non-http": { - "description": "Enables protocols beyond HTTP/HTTPS/HTTP2" - }, - "openssl": { - "description": "SSL support (OpenSSL)", - "dependencies": [ - "openssl" - ] - }, - "schannel": { - "description": "SSL support (Secure Channel)", - "supports": "windows & !uwp", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "sspi" - ] - } - ] - }, - "sectransp": { - "description": "SSL support (sectransp)", - "supports": "osx | ios" - }, - "ssh": { - "description": "SSH support via libssh2", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "non-http" - ] - }, - { - "name": "curl", - "default-features": false, - "features": [ - "openssl" - ] - }, - "libssh2" - ] - }, - "ssl": { - "description": "Default SSL backend", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "sectransp" - ], - "platform": "osx | ios" - }, - { - "name": "curl", - "default-features": false, - "features": [ - "schannel" - ], - "platform": "(windows & !uwp) | mingw" - }, - { - "name": "curl", - "default-features": false, - "features": [ - "openssl" - ], - "platform": "(uwp | !windows) & !(osx | ios) & !mingw" - } - ] - }, - "sspi": { - "description": "SSPI support", - "supports": "windows & !uwp" - }, - "tool": { - "description": "Builds curl executable", - "supports": "!uwp" - }, - "websockets": { - "description": "WebSocket support (experimental)" - }, - "winidn": { - "description": "WinIDN support", - "supports": "windows" - }, - "winldap": { - "description": "LDAP support (WinLDAP). This feature does not include LDAPS support.", - "supports": "windows & !uwp", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "non-http" - ] - } - ] - }, - "winssl": { - "description": "Legacy name for schannel", - "supports": "windows & !uwp", - "dependencies": [ - { - "name": "curl", - "default-features": false, - "features": [ - "schannel" - ] - } - ] - }, - "wolfssl": { - "description": "SSL support (wolfSSL)", - "dependencies": [ - "wolfssl" - ] - }, - "zstd": { - "description": "ZStandard support (zstd)", - "dependencies": [ - "zstd" - ] - } - } -} diff --git a/vcpkg.json b/vcpkg.json index 9c2fa1e3f84e..ce2bd65276df 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -112,6 +112,10 @@ { "name": "abseil", "version": "20230802.1" + }, + { + "name": "curl", + "version": "8.4.0" } ] } From f774124c9492baab402e7650a2e2cf9562552546 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:09:15 -0500 Subject: [PATCH 077/456] Add v2_17_3 arrays to backward compatability matrix (#4524) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 8ea6ed7f35b8..2936492f723c 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0", "v2_17_3"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 2c04a1090ef5038574a2a978cea7a5e6241941f2 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:28:19 -0500 Subject: [PATCH 078/456] Move compute_new_fragment_name into StorageFormat (#4520) Move `compute_new_fragment_name` from `tiledb/sm` into the `StorageFormat`. A new file is added to maintain storage format of `fragment_name`, and contains `compute_new_fragment_name` and `generate_fragment_name`. Tests are added for these APIs, as well as `get_timestamp_range`. --- TYPE: FORMAT DESC: Migrate fragment_name parsing into StorageFormat --- tiledb/sm/array/array_directory.cc | 32 +-- tiledb/sm/array/array_directory.h | 13 +- .../sm/consolidator/fragment_consolidator.cc | 8 +- .../fragment_meta_consolidator.cc | 6 +- tiledb/sm/group/group_directory.cc | 31 +-- tiledb/sm/group/group_directory.h | 15 +- .../deletes_and_updates.cc | 10 +- .../array_dimension_label_queries.cc | 4 +- .../sm/query/writers/global_order_writer.cc | 10 +- tiledb/sm/query/writers/writer_base.cc | 10 +- tiledb/sm/storage_manager/storage_manager.cc | 9 +- tiledb/storage_format/uri/CMakeLists.txt | 4 +- tiledb/storage_format/uri/generate_uri.cc | 27 ++- tiledb/storage_format/uri/generate_uri.h | 23 +- tiledb/storage_format/uri/test/CMakeLists.txt | 31 +++ .../uri/test/compile_uri_format_main.cc | 18 +- .../uri/test/unit_uri_format.cc | 211 ++++++++++++++++++ 17 files changed, 340 insertions(+), 122 deletions(-) create mode 100644 tiledb/storage_format/uri/test/CMakeLists.txt create mode 100644 tiledb/storage_format/uri/test/unit_uri_format.cc diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 6e431167146c..2a937fe434c1 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -41,14 +41,13 @@ #include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" -#include "tiledb/storage_format/uri/parse_uri.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Class for ArrayDirectory status exceptions. */ class ArrayDirectoryException : public StatusException { @@ -266,7 +265,7 @@ const uint64_t& ArrayDirectory::timestamp_end() const { void ArrayDirectory::write_commit_ignore_file( const std::vector& commit_uris_to_ignore) { - auto name = compute_new_fragment_name( + auto name = storage_format::generate_consolidated_fragment_name( commit_uris_to_ignore.front(), commit_uris_to_ignore.back(), constants::format_version); @@ -563,27 +562,6 @@ URI ArrayDirectory::get_vacuum_uri(const URI& fragment_uri) const { return URI(temp_uri.to_string() + constants::vacuum_file_suffix); } -std::string ArrayDirectory::compute_new_fragment_name( - const URI& first, const URI& last, format_version_t format_version) const { - // Get uuid - std::string uuid; - throw_if_not_ok(uuid::generate_uuid(&uuid, false)); - - // For creating the new fragment URI - - // Get timestamp ranges - std::pair t_first, t_last; - throw_if_not_ok(utils::parse::get_timestamp_range(first, &t_first)); - throw_if_not_ok(utils::parse::get_timestamp_range(last, &t_last)); - - // Create new URI - std::stringstream ss; - ss << "/__" << t_first.first << "_" << t_last.second << "_" << uuid << "_" - << format_version; - - return ss.str(); -} - bool ArrayDirectory::loaded() const { return loaded_; } @@ -1334,5 +1312,5 @@ shared_ptr ArrayDirectory::load_enumeration( Deserializer deserializer(tile.data(), tile.size()); return Enumeration::deserialize(deserializer); } -} // namespace sm -} // namespace tiledb + +} // namespace tiledb::sm diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 19c69cfe2ce0..ca8faf79ffcb 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -48,8 +48,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Mode for the ArrayDirectory class. */ enum class ArrayDirectoryMode { @@ -456,13 +455,6 @@ class ArrayDirectory { /** Returns the URI for a vacuum file. */ URI get_vacuum_uri(const URI& fragment_uri) const; - /** - * The new fragment name is computed - * as `____`. - */ - std::string compute_new_fragment_name( - const URI& first, const URI& last, format_version_t format_version) const; - /** Returns `true` if `load` has been run. */ bool loaded() const; @@ -832,7 +824,6 @@ class ArrayDirectory { MemoryTracker& memory_tracker) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_ARRAY_DIRECTORY_H diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index a4045621a802..47386fe08522 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ #include "tiledb/sm/query/query.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -649,9 +650,8 @@ Status FragmentConsolidator::create_queries( auto last = (*query_r)->last_fragment_uri(); auto write_version = array_for_reads->array_schema_latest().write_version(); - auto fragment_name = - array_for_reads->array_directory().compute_new_fragment_name( - first, last, write_version); + auto fragment_name = storage_format::generate_consolidated_fragment_name( + first, last, write_version); // Create write query *query_w = tdb_new(Query, storage_manager_, array_for_writes, fragment_name); diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 1ce41e41f342..270ad4d8209f 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb::common; @@ -92,7 +93,8 @@ Status FragmentMetaConsolidator::consolidate( auto first = meta.front()->fragment_uri(); auto last = meta.back()->fragment_uri(); auto write_version = array.array_schema_latest().write_version(); - auto name = array_dir.compute_new_fragment_name(first, last, write_version); + auto name = storage_format::generate_consolidated_fragment_name( + first, last, write_version); auto frag_md_uri = array_dir.get_fragment_metadata_dir(write_version); RETURN_NOT_OK(storage_manager_->vfs()->create_dir(frag_md_uri)); diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc index 87736c7f460e..868c4b745a21 100644 --- a/tiledb/sm/group/group_directory.cc +++ b/tiledb/sm/group/group_directory.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,8 +42,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -168,29 +167,6 @@ Status GroupDirectory::load() { return Status::Ok(); } -tuple> GroupDirectory::compute_new_fragment_name( - const URI& first, const URI& last, format_version_t format_version) const { - // Get uuid - std::string uuid; - RETURN_NOT_OK_TUPLE(uuid::generate_uuid(&uuid, false), nullopt); - - // For creating the new fragment URI - - // Get timestamp ranges - std::pair t_first, t_last; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range(first, &t_first), nullopt); - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range(last, &t_last), nullopt); - - // Create new URI - std::stringstream ss; - ss << "/__" << t_first.first << "_" << t_last.second << "_" << uuid << "_" - << format_version; - - return {Status::Ok(), ss.str()}; -} - bool GroupDirectory::loaded() const { return loaded_; } @@ -384,5 +360,4 @@ bool GroupDirectory::is_vacuum_file(const URI& uri) const { return false; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/group/group_directory.h b/tiledb/sm/group/group_directory.h index d6e6b3e2dd0d..a475e83177dd 100644 --- a/tiledb/sm/group/group_directory.h +++ b/tiledb/sm/group/group_directory.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,8 +42,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Mode for the GroupDirectory class. */ enum class GroupDirectoryMode { @@ -123,13 +122,6 @@ class GroupDirectory { /** Return the latest group details URI. */ const URI& latest_group_details_uri() const; - /** - * The new fragment name is computed - * as `____`. - */ - tuple> compute_new_fragment_name( - const URI& first, const URI& last, format_version_t format_version) const; - /** Returns `true` if `load` has been run. */ bool loaded() const; @@ -243,7 +235,6 @@ class GroupDirectory { bool is_vacuum_file(const URI& uri) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_GROUP_DIRECTORY_H diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index 5630797be21e..6413fdaf10fd 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,8 +41,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class DeleteAndUpdateStatusException : public StatusException { public: @@ -139,7 +138,7 @@ Status DeletesAndUpdates::dowork() { uint64_t timestamp = array_->timestamp_end_opened_at(); auto write_version = array_->array_schema_latest().write_version(); auto new_fragment_str = - storage_format::generate_fragment_name(timestamp, write_version); + storage_format::generate_timestamped_name(timestamp, write_version); // Check that the delete or update isn't in the middle of a fragment // consolidated without timestamps. @@ -191,5 +190,4 @@ std::string DeletesAndUpdates::name() { return "DeletesAndUpdates"; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc index 048d439ca53e..6812c3ee9691 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -90,7 +90,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( // or to get the timestamp_end from the parent array. This fix is // blocked by current discussion on a timestamp refactor design. if (!fragment_name_.has_value()) { - fragment_name_ = storage_format::generate_fragment_name( + fragment_name_ = storage_format::generate_timestamped_name( array->timestamp_end_opened_at(), array->array_schema_latest().write_version()); } diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 4f045fc8ef6b..303e686afc65 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -58,8 +58,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class GlobalOrderWriterStatusException : public StatusException { public: @@ -1438,7 +1437,7 @@ Status GlobalOrderWriter::start_new_fragment() { const auto write_version = array_->array_schema_latest().write_version(); auto frag_dir_uri = array_->array_directory().get_fragments_dir(write_version); - auto new_fragment_str = storage_format::generate_uri( + auto new_fragment_str = storage_format::generate_timestamped_name( fragment_timestamp_range_.first, fragment_timestamp_range_.second, write_version); @@ -1452,5 +1451,4 @@ Status GlobalOrderWriter::start_new_fragment() { return Status::Ok(); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 42e073e3d39b..de83892da003 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -60,8 +60,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class WriterBaseStatusException : public StatusException { public: @@ -225,7 +224,7 @@ WriterBase::WriterBase( auto new_fragment_str = fragment_name.has_value() ? fragment_name.value() : - storage_format::generate_fragment_name(timestamp, write_version); + storage_format::generate_timestamped_name(timestamp, write_version); auto frag_dir_uri = array_->array_directory().get_fragments_dir(write_version); fragment_uri_ = frag_dir_uri.join_path(new_fragment_str); @@ -1205,5 +1204,4 @@ bool WriterBase::remote_query() const { return remote_query_; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 1812eda358d2..4fbb04385940 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -72,6 +72,7 @@ #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -80,8 +81,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -293,7 +293,7 @@ void StorageManagerCanonical::write_consolidated_commits_file( ArrayDirectory array_dir, const std::vector& commit_uris) { // Compute the file name. - auto name = array_dir.compute_new_fragment_name( + auto name = storage_format::generate_consolidated_fragment_name( commit_uris.front(), commit_uris.back(), write_version); // Compute size of consolidated file. Save the sizes of the files to re-use @@ -1876,5 +1876,4 @@ void StorageManagerCanonical::group_metadata_vacuum( consolidator->vacuum(group_name); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 4dbd4dd3f319..3abf73b8ca6f 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -33,3 +33,5 @@ commence(object_library uri_format) this_target_sources(parse_uri.cc generate_uri.cc) this_target_object_libraries(baseline time uuid vfs) conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index 9c6ad41d01f8..1366c35ef0f6 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,7 @@ #include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/uuid.h" +#include "tiledb/storage_format/uri/parse_uri.h" #include @@ -36,10 +37,17 @@ using namespace tiledb::common; namespace tiledb::storage_format { -std::string generate_uri( +std::string generate_timestamped_name( uint64_t timestamp_start, uint64_t timestamp_end, uint32_t version) { std::string uuid; throw_if_not_ok(sm::uuid::generate_uuid(&uuid, false)); + + if (timestamp_start > timestamp_end) { + throw std::logic_error( + "Error generating timestamped name; " + "start timestamp cannot be after end timestamp."); + } + std::stringstream ss; ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid << "_" << version; @@ -47,11 +55,22 @@ std::string generate_uri( return ss.str(); } -std::string generate_fragment_name( +std::string generate_timestamped_name( uint64_t timestamp, format_version_t format_version) { timestamp = (timestamp != 0) ? timestamp : sm::utils::time::timestamp_now_ms(); - return generate_uri(timestamp, timestamp, format_version); + return generate_timestamped_name(timestamp, timestamp, format_version); +} + +std::string generate_consolidated_fragment_name( + const URI& first, const URI& last, format_version_t format_version) { + // Get timestamp ranges + std::pair t_first, t_last; + throw_if_not_ok(utils::parse::get_timestamp_range(first, &t_first)); + throw_if_not_ok(utils::parse::get_timestamp_range(last, &t_last)); + + return generate_timestamped_name( + t_first.first, t_last.second, (uint64_t)format_version); } } // namespace tiledb::storage_format diff --git a/tiledb/storage_format/uri/generate_uri.h b/tiledb/storage_format/uri/generate_uri.h index 9b4d8c2f0c09..727b3c9b8ddd 100644 --- a/tiledb/storage_format/uri/generate_uri.h +++ b/tiledb/storage_format/uri/generate_uri.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,14 +34,14 @@ #define TILEDB_GENERATE_URI_H #include "tiledb/common/common.h" +#include "tiledb/sm/filesystem/uri.h" using namespace tiledb::common; +using namespace tiledb::sm; namespace tiledb::storage_format { /** - * Generates a new URI. - * * Generate a new URI in the form `__t1_t2_uuid_v`, where `t1` is the starting * timestamp, `t2` is the ending timestamp, and `v` is the current format * version. For instance, @@ -55,7 +55,7 @@ namespace tiledb::storage_format { * * @return The new URI. */ -std::string generate_uri( +std::string generate_timestamped_name( uint64_t timestamp_start, uint64_t timestamp_end, uint32_t version); /** @@ -73,9 +73,22 @@ std::string generate_uri( * * @return new fragment name. */ -std::string generate_fragment_name( +std::string generate_timestamped_name( uint64_t timestamp, format_version_t format_version); +/** + * Generates a consolidated fragment name in the form + * `____`. + * + * @param first The first URI. + * @param last The last URI. + * @param format_version The write version. + * + * @return consolidated fragment name. + */ +std::string generate_consolidated_fragment_name( + const URI& first, const URI& last, format_version_t format_version); + } // namespace tiledb::storage_format #endif diff --git a/tiledb/storage_format/uri/test/CMakeLists.txt b/tiledb/storage_format/uri/test/CMakeLists.txt new file mode 100644 index 000000000000..aeaf8a95fa40 --- /dev/null +++ b/tiledb/storage_format/uri/test/CMakeLists.txt @@ -0,0 +1,31 @@ +# +# tiledb/storage_format/uri/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +include(unit_test) + +commence(unit_test uri_format) + this_target_sources(unit_uri_format.cc) + this_target_object_libraries(uri_format) +conclude(unit_test) diff --git a/tiledb/storage_format/uri/test/compile_uri_format_main.cc b/tiledb/storage_format/uri/test/compile_uri_format_main.cc index f93b590e1469..5485f87e622e 100644 --- a/tiledb/storage_format/uri/test/compile_uri_format_main.cc +++ b/tiledb/storage_format/uri/test/compile_uri_format_main.cc @@ -1,11 +1,11 @@ /** - * @file compile_array_schema_main.cc + * @file compile_uri_format_main.cc * * @section LICENSE * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,10 +26,22 @@ * THE SOFTWARE. */ +#include "../generate_uri.h" #include "../parse_uri.h" +using namespace tiledb::storage_format; +using namespace tiledb::sm::utils::parse; + int main() { + (void)generate_timestamped_name(1, 2, 1); + (void)generate_timestamped_name(1, 1); + (void)generate_consolidated_fragment_name( + tiledb::sm::URI{}, tiledb::sm::URI{}, 1); + std::pair x; - (void)tiledb::sm::utils::parse::get_timestamp_range(tiledb::sm::URI{}, &x); + (void)get_timestamp_range(tiledb::sm::URI{}, &x); + (void)get_fragment_version(""); + (void)is_element_of(tiledb::sm::URI{}, tiledb::sm::URI{}); + return 0; } diff --git a/tiledb/storage_format/uri/test/unit_uri_format.cc b/tiledb/storage_format/uri/test/unit_uri_format.cc new file mode 100644 index 000000000000..2c8970db95ee --- /dev/null +++ b/tiledb/storage_format/uri/test/unit_uri_format.cc @@ -0,0 +1,211 @@ +/** + * @file tiledb/storage_format/uri/test/unit_uri_format.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for the storage format's uri manipulation APIs. + */ + +#include + +#include "../generate_uri.h" +#include "../parse_uri.h" +#include "tiledb/sm/misc/constants.h" + +using namespace tiledb::storage_format; +using namespace tiledb::sm::utils::parse; + +TEST_CASE( + "StorageFormat: Generate: generate_timestamped_name", + "[storage_format][generate][generate_timestamped_name]") { + // Note: all format versions will generate the same format of fragment name + uint64_t start = 1; + uint64_t end = 2; + format_version_t format_version = 5; + std::string name; + + SECTION("one timestamp") { + // Generate new fragment name + name = generate_timestamped_name(start, format_version); + + // Check new fragment name's timestamp range + auto name_timestamps{name.substr(0, 6)}; + CHECK(name_timestamps == "/__1_1"); + } + + SECTION("two timestamps") { + // Generate new fragment name + name = generate_timestamped_name(start, end, (uint32_t)format_version); + + // Check new fragment name's timestamp range + auto name_timestamps{name.substr(0, 6)}; + CHECK(name_timestamps == "/__1_2"); + } + + // Check new fragment name's format version + auto name_version{name.substr(name.find_last_of("_"))}; + CHECK(name_version == "_5"); +} + +TEST_CASE( + "StorageFormat: Generate: generate_consolidated_fragment_name", + "[storage_format][generate][generate_consolidated_fragment_name]") { + // Create fragments at timestamps 1 and 2 + auto start = tiledb::sm::URI( + "file:///array_name/__fragments/__1_1_44318efd44f546b18db13edc8d10805b"); + auto end = tiledb::sm::URI( + "file:///array_name/__fragments/__2_2_44318efd44f546b18db13edc8d10806c"); + + SECTION("valid, unique timestamps") { + // Generate fragment name for fragments at timestamps 1, 2 + auto name{generate_consolidated_fragment_name( + start, end, constants::format_version)}; + + // Check new fragment name timestamp range + auto name_substr{name.substr(name.find_last_of("/"), 6)}; + CHECK(name_substr == "/__1_2"); + } + + SECTION("valid, same timestamps") { + // Generate fragment name for fragments at timestamps 1, 1 + auto name{generate_consolidated_fragment_name( + start, start, constants::format_version)}; + + // Check new fragment name timestamp range + auto name_substr{name.substr(name.find_last_of("/"), 6)}; + CHECK(name_substr == "/__1_1"); + } + + SECTION("invalid timestamps") { + // Try to generate fragment name for fragments at timestamps 2, 1 + REQUIRE_THROWS_WITH( + generate_consolidated_fragment_name( + end, start, constants::format_version), + Catch::Matchers::ContainsSubstring( + "start timestamp cannot be after end timestamp")); + } +} + +TEST_CASE( + "StorageFormat: Parse: get_timestamp_range", + "[storage_format][parse][get_timestamp_range]") { + std::pair range; + + SECTION("name version 1") { + // Note: the end timestamp may or may not be present. + // If present, this version sets both range values to the end timestamp. + // As such, if the end_timestamp < start_timestamp, no error is thrown. + SECTION("valid timestamps") { + tiledb::sm::URI frag; + + SECTION("with end timestamp") { + // Create fragment at timestamp 1-2 + frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__44318efd44f546b18db13edc8d10805b_1_2"); + } + + SECTION("without end timestamp") { + // Create fragment at timestamp 2 + frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__44318efd44f546b18db13edc8d10805b_2"); + } + + // Check timestamp range + REQUIRE(get_timestamp_range(frag, &range).ok()); + CHECK(range.first == 2); + CHECK(range.second == 2); + } + + SECTION("invalid timestamps") { + // Create fragment at timestamp 2-1 + auto frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__44318efd44f546b18db13edc8d10805b_2_1"); + + // Check timestamp range + REQUIRE(get_timestamp_range(frag, &range).ok()); + CHECK(range.first == 1); + CHECK(range.second == 1); + } + } + + SECTION("name version 2") { + SECTION("valid timestamps") { + // Create fragment at timestamp 1-2 + auto frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__1_2_44318efd44f546b18db13edc8d10805b"); + + // Check timestamp range + REQUIRE(get_timestamp_range(frag, &range).ok()); + CHECK(range.first == 1); + CHECK(range.second == 2); + } + + SECTION("invalid timestamps") { + // Create fragment at timestamp 2-1 + auto frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__2_1_44318efd44f546b18db13edc8d10805b"); + + // Try to get timestamp range + REQUIRE_THROWS_WITH( + get_timestamp_range(frag, &range), + Catch::Matchers::ContainsSubstring( + "start timestamp cannot be after end timestamp")); + } + } + + SECTION("name version 3") { + SECTION("valid timestamps") { + // Create fragment at timestamp 1-2 + auto frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__1_2_44318efd44f546b18db13edc8d10805b_5"); + + // Check timestamp range + REQUIRE(get_timestamp_range(frag, &range).ok()); + CHECK(range.first == 1); + CHECK(range.second == 2); + } + + SECTION("invalid timestamps") { + // Create fragment at timestamp 2-1 + auto frag = tiledb::sm::URI( + "file:///array_name/__fragments/" + "__2_1_44318efd44f546b18db13edc8d10805b_5"); + + // Try to get timestamp range + REQUIRE_THROWS_WITH( + get_timestamp_range(frag, &range), + Catch::Matchers::ContainsSubstring( + "start timestamp cannot be after end timestamp")); + } + } +} From 2f7361cf03e9ae24e3b0474e656abadffe504440 Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Mon, 20 Nov 2023 20:49:28 +0200 Subject: [PATCH 079/456] Remove tests against hidden libwebp symbols in link_webp_test1.cc (#4525) `link_webp_test1.cc` currently tests linking against a few internal-only symbols of the webp library. This sort of works with a static webp, but fails for a shared version. Here's what objdump shows for these symbols in the latest version of libwebp. Static: ``` objdump -t ~/.conan2/p/libwe0ab986b34fa07/p/lib/*.a | grep -P 'WebPGetDemuxVersion|WebPGetMuxVersion|WebPAllocateDecBuffer|WebPFreeDecBuffer|VP8LNew|VP8LClear' | grep -v '*UND*' | sort -u 0000000000000000 g F .text 0000000000000006 WebPGetMuxVersion 00000000000000f0 g F .text 0000000000000428 .hidden WebPAllocateDecBuffer 0000000000000570 g F .text 000000000000003b WebPFreeDecBuffer 0000000000000a40 g F .text 0000000000000031 .hidden VP8LClearBackwardRefs 0000000000000bb0 g F .text 0000000000000006 WebPGetDemuxVersion 0000000000002860 g F .text 000000000000002f .hidden VP8LNew 0000000000002890 g F .text 0000000000000101 .hidden VP8LClear ``` Shared: ``` objdump -T ~/.conan2/p/libwed5d42768c6b93/p/lib/*.so | grep -P 'WebPGetDemuxVersion|WebPGetMuxVersion|WebPAllocateDecBuffer|WebPFreeDecBuffer|VP8LNew|VP8LClear' | LC_COLLATE=C sort -u 0000000000002780 g DF .text 0000000000000006 Base WebPGetDemuxVersion 0000000000002da0 g DF .text 000000000000003b Base WebPFreeDecBuffer 0000000000003f10 g DF .text 000000000000003b Base WebPFreeDecBuffer 00000000000064a0 g DF .text 0000000000000006 Base WebPGetMuxVersion ``` For context, this cropped up in a PR for adding a TileDB recipe to the Conan Center Index: https://github.com/conan-io/conan-center-index/pull/19381 --- TYPE: BUG DESC: Fix broken linking tests against a shared WebP library --------- Co-authored-by: Theodore Tsirpanis --- tiledb/sm/compressors/test/link_webp_test1.cc | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/tiledb/sm/compressors/test/link_webp_test1.cc b/tiledb/sm/compressors/test/link_webp_test1.cc index c59b166311b6..cfd976a2e6e7 100644 --- a/tiledb/sm/compressors/test/link_webp_test1.cc +++ b/tiledb/sm/compressors/test/link_webp_test1.cc @@ -96,18 +96,6 @@ uint64_t reference_a_value(uint64_t v) { #pragma warning(disable : 4701) // 'config', 'picture_no_alpha' #endif -extern "C" int WebPGetDemuxVersion(void); -extern "C" int WebPGetMuxVersion(void); -extern "C" VP8StatusCode WebPAllocateDecBuffer( - int width, - int height, - const WebPDecoderOptions* const options, - WebPDecBuffer* const buffer); -extern "C" void WebPFreeDecBuffer(WebPDecBuffer* const buffer); -typedef void VP8LDecoder; -extern "C" VP8LDecoder* VP8LNew(void); -extern "C" void VP8LClear(VP8LDecoder*); - int main(int argc, const char* argv[]) { int return_value = -1; const char *in_file = NULL, *out_file = NULL; @@ -138,28 +126,13 @@ int main(int argc, const char* argv[]) { // *** turns out, there is -NO- symbol in webpdecoder that is not also in // webp! left here anyway, in case libraries are ever de-dupped. const int dec_version = WebPGetDecoderVersion(); - auto vp8l_dec = VP8LNew(); - VP8LClear(vp8l_dec); - - VP8StatusCode status; - WebPDecoderOptions webpdecoptions; - WebPDecBuffer webpdec_buffer; - status = WebPAllocateDecBuffer(512, 512, &webpdecoptions, &webpdec_buffer); - if (status == VP8_STATUS_OK) { - WebPFreeDecBuffer(&webpdec_buffer); - } - - // webpdemux - const int demux_version = WebPGetDemuxVersion(); - // webpmux - const int mux_version = WebPGetMuxVersion(); // webp // in testing, even when WebP::webp was -not- added as link item, this // library was still being placed in link_webp_test project. auto enc_version = WebPGetEncoderVersion(); - reference_a_value(dec_version + demux_version + mux_version + enc_version); + reference_a_value(dec_version + enc_version); } if (argc == 1) { From f7390c4a86bb82ddfffbaedc2a52228db96784ad Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Tue, 21 Nov 2023 04:28:01 -0500 Subject: [PATCH 080/456] Update error message for C++ bad filter option type (#4503) --- TYPE: CPP_API DESC: Improve error message in C++ API for bad filter option type --- test/src/unit-cppapi-filter.cc | 13 ++--- tiledb/sm/cpp_api/filter.h | 94 +++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/test/src/unit-cppapi-filter.cc b/test/src/unit-cppapi-filter.cc index ee8bda48d93f..0866ca14c049 100644 --- a/test/src/unit-cppapi-filter.cc +++ b/test/src/unit-cppapi-filter.cc @@ -76,13 +76,11 @@ TEST_CASE("C++ API: Filter options", "[cppapi][filter]") { // Check templated version with wrong type throws exception uint32_t wrong_type_u = 1; REQUIRE_THROWS_AS( - f.set_option(TILEDB_COMPRESSION_LEVEL, wrong_type_u), - std::invalid_argument); + f.set_option(TILEDB_COMPRESSION_LEVEL, wrong_type_u), TileDBError); REQUIRE_THROWS_AS( - f.get_option(TILEDB_COMPRESSION_LEVEL, &wrong_type_u), - std::invalid_argument); + f.get_option(TILEDB_COMPRESSION_LEVEL, &wrong_type_u), TileDBError); REQUIRE_THROWS_AS( - f.get_option(TILEDB_COMPRESSION_LEVEL), std::invalid_argument); + f.get_option(TILEDB_COMPRESSION_LEVEL), TileDBError); // Check that you can bypass type safety (don't do this). f.get_option(TILEDB_COMPRESSION_LEVEL, (void*)&wrong_type_u); @@ -101,10 +99,9 @@ TEST_CASE("C++ API: Filter options", "[cppapi][filter]") { int32_t wrong_type_i = 1; REQUIRE_THROWS_AS(f2.set_option(TILEDB_COMPRESSION_LEVEL, 1), TileDBError); REQUIRE_THROWS_AS( - f2.set_option(TILEDB_BIT_WIDTH_MAX_WINDOW, -1), std::invalid_argument); + f2.set_option(TILEDB_BIT_WIDTH_MAX_WINDOW, -1), TileDBError); REQUIRE_THROWS_AS( - f2.set_option(TILEDB_BIT_WIDTH_MAX_WINDOW, wrong_type_i), - std::invalid_argument); + f2.set_option(TILEDB_BIT_WIDTH_MAX_WINDOW, wrong_type_i), TileDBError); } TEST_CASE("C++ API: Filter lists", "[cppapi][filter]") { diff --git a/tiledb/sm/cpp_api/filter.h b/tiledb/sm/cpp_api/filter.h index 387f6818c267..786780f0702c 100644 --- a/tiledb/sm/cpp_api/filter.h +++ b/tiledb/sm/cpp_api/filter.h @@ -33,13 +33,43 @@ #ifndef TILEDB_CPP_API_FILTER_H #define TILEDB_CPP_API_FILTER_H +#include "exception.h" #include "tiledb.h" +#include "type.h" #include #include namespace tiledb { +template +class FilterOptionTypeError : public TypeError { + public: + FilterOptionTypeError(tiledb_filter_option_t option) + : TypeError( + "Cannot set filter option '" + option_name(option) + + "' with type '" + tiledb::impl::type_to_tiledb().name + + "'; Option value must be '" + + tiledb::impl::type_to_tiledb().name + "'.") { + } + + FilterOptionTypeError( + tiledb_filter_option_t option, const std::string& alternate_type) + : TypeError( + "Cannot set filter option '" + option_name(option) + + "' with type '" + tiledb::impl::type_to_tiledb().name + + "'; Option value must be '" + alternate_type + "' or '" + + tiledb::impl::type_to_tiledb().name + "'.") { + } + + private: + static std::string option_name(tiledb_filter_option_t option) { + const char* option_name; + tiledb_filter_option_to_str(option, &option_name); + return {option_name}; + } +}; + /** * Represents a filter. A filter is used to transform attribute data e.g. * with compression, delta encoding, etc. @@ -335,67 +365,59 @@ class Filter { template void option_value_typecheck(tiledb_filter_option_t option) { std::string type_name = tiledb::impl::type_to_tiledb().name; + const char* option_name; + tiledb_filter_option_to_str(option, &option_name); switch (option) { case TILEDB_COMPRESSION_LEVEL: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be int32_t."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_BIT_WIDTH_MAX_WINDOW: case TILEDB_POSITIVE_DELTA_MAX_WINDOW: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be uint32_t."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_SCALE_FLOAT_BYTEWIDTH: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be uint64_t."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_SCALE_FLOAT_FACTOR: case TILEDB_SCALE_FLOAT_OFFSET: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be double."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_WEBP_QUALITY: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be float."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_WEBP_INPUT_FORMAT: if constexpr ( !std::is_same_v && - !std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be tiledb_filter_webp_format_t or " - "uint8_t."); + !std::is_same_v) { + throw FilterOptionTypeError( + option, "tiledb_filter_webp_format_t"); + } break; case TILEDB_WEBP_LOSSLESS: - if constexpr (!std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be uint8_t."); + if constexpr (!std::is_same_v) { + throw FilterOptionTypeError(option); + } break; case TILEDB_COMPRESSION_REINTERPRET_DATATYPE: if constexpr ( !std::is_same_v && - !std::is_same_v) - throw std::invalid_argument( - "Cannot set option with type '" + type_name + - "'; Option value must be tiledb_datatype_t or uint8_t."); + !std::is_same_v) { + throw FilterOptionTypeError(option, "tiledb_datatype_t"); + } break; default: { - const char* option_str; - tiledb_filter_option_to_str(option, &option_str); throw std::invalid_argument( - "Invalid option '" + std::string(option_str) + "'"); + "Invalid filter option '" + std::string(option_name) + "'"); } } } From 61ef498322bd60efa19ad8063e9989efcd11d8fb Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 23 Nov 2023 04:49:48 -0500 Subject: [PATCH 081/456] Update generate_uri member functions to use the StorageFormat (#4530) Member functions should not generate uuids to create their own timestamped fragment and uri names. Instead, they need only to use the StorageFormat's APIs. --- TYPE: IMPROVEMENT DESC: Use StorageFormat APIs to generate timestamped uris --- tiledb/sm/array/array_directory.cc | 1 - tiledb/sm/array_schema/CMakeLists.txt | 2 +- tiledb/sm/array_schema/array_schema.cc | 67 +++++++------------ tiledb/sm/array_schema/array_schema.h | 12 ++-- .../sm/array_schema/array_schema_evolution.cc | 17 ++--- .../sm/array_schema/array_schema_evolution.h | 9 +-- tiledb/sm/array_schema/attribute.cc | 9 +-- .../consolidator/array_meta_consolidator.cc | 20 +----- .../consolidator/group_meta_consolidator.cc | 20 +----- tiledb/sm/group/group.cc | 36 +++------- tiledb/sm/group/group.h | 17 ++--- tiledb/sm/group/group_details.cc | 9 +-- tiledb/sm/group/group_directory.cc | 1 - tiledb/sm/metadata/CMakeLists.txt | 4 +- tiledb/sm/metadata/metadata.cc | 32 +++------ tiledb/sm/metadata/metadata.h | 12 ++-- tiledb/sm/metadata/test/unit_metadata.cc | 3 +- tiledb/sm/misc/uuid.cc | 10 +-- .../sm/query/writers/global_order_writer.cc | 1 - tiledb/sm/query/writers/ordered_writer.cc | 9 +-- tiledb/sm/query/writers/unordered_writer.cc | 26 +++---- tiledb/sm/query/writers/writer_base.cc | 1 - tiledb/sm/storage_manager/storage_manager.cc | 16 ++--- tiledb/storage_format/uri/generate_uri.cc | 13 ++-- tiledb/storage_format/uri/generate_uri.h | 10 ++- .../uri/test/unit_uri_format.cc | 29 ++++++-- 26 files changed, 145 insertions(+), 241 deletions(-) diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 2a937fe434c1..5a223fd25c75 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -37,7 +37,6 @@ #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index bff3c61bb334..efc7626d2b34 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -72,7 +72,7 @@ conclude(object_library) commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain enumeration time uri_format uuid vfs) + attribute domain enumeration time uri_format vfs) conclude(object_library) # This is linked outside the object_library scope because ContextResources diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 88c263582ff4..1bb33ec150d9 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -84,19 +84,18 @@ ArraySchema::ArraySchema() } ArraySchema::ArraySchema(ArrayType array_type) - : array_type_(array_type) { - allows_dups_ = false; - array_uri_ = URI(); - uri_ = URI(); - name_ = ""; - capacity_ = constants::capacity; - cell_order_ = Layout::ROW_MAJOR; - domain_ = nullptr; - tile_order_ = Layout::ROW_MAJOR; - version_ = constants::format_version; - auto timestamp = utils::time::timestamp_now_ms(); - timestamp_range_ = std::make_pair(timestamp, timestamp); - + : uri_(URI()) + , array_uri_(URI()) + , version_(constants::format_version) + , timestamp_range_(std::make_pair( + utils::time::timestamp_now_ms(), utils::time::timestamp_now_ms())) + , name_("") + , array_type_(array_type) + , allows_dups_(false) + , domain_(nullptr) + , cell_order_(Layout::ROW_MAJOR) + , tile_order_(Layout::ROW_MAJOR) + , capacity_(constants::capacity) { // Set up default filter pipelines for coords, offsets, and validity values. coords_filters_.add_filter(CompressionFilter( constants::coords_compression, @@ -112,7 +111,7 @@ ArraySchema::ArraySchema(ArrayType array_type) Datatype::UINT8)); // Generate URI and name for ArraySchema - throw_if_not_ok(generate_uri()); + generate_uri(); } ArraySchema::ArraySchema( @@ -1564,10 +1563,9 @@ format_version_t ArraySchema::version() const { return version_; } -Status ArraySchema::set_timestamp_range( +void ArraySchema::set_timestamp_range( const std::pair& timestamp_range) { timestamp_range_ = timestamp_range; - return Status::Ok(); } std::pair ArraySchema::timestamp_range() const { @@ -1696,36 +1694,19 @@ void ArraySchema::clear() { timestamp_range_ = std::make_pair(0, 0); } -Status ArraySchema::generate_uri() { - std::string uuid; - RETURN_NOT_OK(uuid::generate_uuid(&uuid, false)); - - auto timestamp = utils::time::timestamp_now_ms(); - timestamp_range_ = std::make_pair(timestamp, timestamp); - std::stringstream ss; - ss << "__" << timestamp_range_.first << "_" << timestamp_range_.second << "_" - << uuid; - name_ = ss.str(); - uri_ = - array_uri_.join_path(constants::array_schema_dir_name).join_path(name_); - - return Status::Ok(); -} - -Status ArraySchema::generate_uri( - const std::pair& timestamp_range) { - std::string uuid; - RETURN_NOT_OK(uuid::generate_uuid(&uuid, false)); +void ArraySchema::generate_uri( + std::optional> timestamp_range) { + if (timestamp_range == std::nullopt) { + auto timestamp = utils::time::timestamp_now_ms(); + timestamp_range_ = std::make_pair(timestamp, timestamp); + } else { + timestamp_range_ = timestamp_range.value(); + } - timestamp_range_ = timestamp_range; - std::stringstream ss; - ss << "__" << timestamp_range_.first << "_" << timestamp_range_.second << "_" - << uuid; - name_ = ss.str(); + name_ = tiledb::storage_format::generate_timestamped_name( + timestamp_range_.first, timestamp_range_.second, std::nullopt); uri_ = array_uri_.join_path(constants::array_schema_dir_name).join_path(name_); - - return Status::Ok(); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index 8a352df7550a..f41308eec537 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -42,7 +42,6 @@ #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -547,7 +546,7 @@ class ArraySchema { format_version_t version() const; /** Set a timestamp range for the array schema */ - Status set_timestamp_range( + void set_timestamp_range( const std::pair& timestamp_range); /** Returns the timestamp range. */ @@ -565,11 +564,10 @@ class ArraySchema { /** Set the schema name. */ void set_name(const std::string& name); - /** Generates a new array schema URI. */ - Status generate_uri(); - - /** Generates a new array schema URI with specified timestamp range. */ - Status generate_uri(const std::pair& timestamp_range); + /** Generates a new array schema URI with optional timestamp range. */ + void generate_uri( + std::optional> timestamp_range = + std::nullopt); private: /* ********************************* */ diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 7ef355d05a5f..7b0c3d1e90bf 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -57,8 +57,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Class for locally generated exceptions. */ class ArraySchemaEvolutionException : public StatusException { @@ -142,11 +141,10 @@ shared_ptr ArraySchemaEvolution::evolve_schema( // Set timestamp, if specified if (std::get<0>(timestamp_range_) != 0) { - throw_if_not_ok(schema.get()->set_timestamp_range(timestamp_range_)); - throw_if_not_ok(schema->generate_uri(timestamp_range_)); + schema->generate_uri(timestamp_range_); } else { // Generate new schema URI - throw_if_not_ok(schema->generate_uri()); + schema->generate_uri(); } return schema; @@ -331,10 +329,10 @@ std::vector ArraySchemaEvolution::enumeration_names_to_drop() void ArraySchemaEvolution::set_timestamp_range( const std::pair& timestamp_range) { if (timestamp_range.first != timestamp_range.second) { - throw std::runtime_error(std::string( + throw ArraySchemaEvolutionException( "Cannot set timestamp range; first element " + std::to_string(timestamp_range.first) + " and second element " + - std::to_string(timestamp_range.second) + " are not equal!")); + std::to_string(timestamp_range.second) + " are not equal!"); } timestamp_range_ = timestamp_range; } @@ -356,5 +354,4 @@ void ArraySchemaEvolution::clear() { timestamp_range_ = {0, 0}; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index be2f2bead0b0..24818c598ca4 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -42,12 +42,10 @@ #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" -#include "tiledb/sm/misc/uuid.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Attribute; class Buffer; @@ -223,7 +221,6 @@ class ArraySchemaEvolution { void clear(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index 17bcea68696b..8588d53e7cfc 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,7 +38,6 @@ #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/misc/parse_argument.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/type/range/range.h" #include @@ -48,8 +47,7 @@ using namespace tiledb::common; using namespace tiledb::type; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Class for locally generated status exceptions. */ class AttributeStatusException : public StatusException { @@ -496,5 +494,4 @@ std::string Attribute::fill_value_str() const { return ret; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 3708bb592f21..9c9c39accc8f 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -108,22 +108,8 @@ Status ArrayMetaConsolidator::consolidate( // Metadata uris to delete const auto to_vacuum = metadata_w->loaded_metadata_uris(); - // Generate new name for consolidated metadata - st = metadata_w->generate_uri(array_uri); - if (!st.ok()) { - throw_if_not_ok(array_for_reads.close()); - throw_if_not_ok(array_for_writes.close()); - return st; - } - - // Get the new URI name - URI new_uri; - st = metadata_w->get_uri(array_uri, &new_uri); - if (!st.ok()) { - throw_if_not_ok(array_for_reads.close()); - throw_if_not_ok(array_for_writes.close()); - return st; - } + // Get the new URI name for consolidated metadata + URI new_uri = metadata_w->get_uri(array_uri); // Close arrays RETURN_NOT_OK_ELSE( diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index ae0db0fc97ae..459f7798c9fa 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -101,22 +101,8 @@ Status GroupMetaConsolidator::consolidate( // Metadata uris to delete const auto to_vacuum = metadata_w->loaded_metadata_uris(); - // Generate new name for consolidated metadata - st = metadata_w->generate_uri(group_uri); - if (!st.ok()) { - throw_if_not_ok(group_for_reads.close()); - throw_if_not_ok(group_for_writes.close()); - return st; - } - - // Get the new URI name - URI new_uri; - st = metadata_w->get_uri(group_uri, &new_uri); - if (!st.ok()) { - throw_if_not_ok(group_for_reads.close()); - throw_if_not_ok(group_for_writes.close()); - return st; - } + // Get the new URI name for consolidated metadata + URI new_uri = metadata_w->get_uri(group_uri); // Close groups RETURN_NOT_OK_ELSE( diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index f519460fd422..adc87909d36b 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -43,13 +43,12 @@ #include "tiledb/sm/group/group_member_v2.h" #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/rest/rest_client.h" +#include "tiledb/storage_format/uri/generate_uri.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class GroupStatusException : public StatusException { public: @@ -600,14 +599,12 @@ const shared_ptr Group::group_directory() const { return group_dir_; } -tuple> Group::generate_detail_uri() const { - auto&& [st, name] = generate_name(); - RETURN_NOT_OK_TUPLE(st, std::nullopt); +URI Group::generate_detail_uri() const { + auto ts_name = tiledb::storage_format::generate_timestamped_name( + timestamp_end_, group_details_->version()); - URI uri = group_uri_.join_path(constants::group_detail_dir_name) - .join_path(name.value()); - - return {Status::Ok(), uri}; + return group_uri_.join_path(constants::group_detail_dir_name) + .join_path(ts_name); } bool Group::changes_applied() const { @@ -708,22 +705,6 @@ std::string Group::dump( /* PROTECTED METHODS */ /* ********************************* */ -tuple> Group::generate_name() const { - std::string uuid; - RETURN_NOT_OK_TUPLE(uuid::generate_uuid(&uuid, false), std::nullopt); - - const auto& version = group_details_->version(); - auto timestamp = - (timestamp_end_ != 0) ? timestamp_end_ : utils::time::timestamp_now_ms(); - std::stringstream ss; - ss << "__" << timestamp << "_" << timestamp << "_" << uuid; - if (version > 1) { - ss << "_" << version; - } - - return {Status::Ok(), ss.str()}; -} - void Group::load_metadata() { if (remote_) { auto rest_client = storage_manager_->rest_client(); @@ -740,5 +721,4 @@ void Group::load_metadata() { metadata_loaded_ = true; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 3058d4934e95..b68f740c9df2 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -47,8 +47,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class GroupDetailsException : public StatusException { public: @@ -301,9 +300,9 @@ class Group { /** * Function to generate a URL of a detail file * - * @return tuple of status and uri + * @return uri */ - tuple> generate_detail_uri() const; + URI generate_detail_uri() const; /** * Have changes been applied to a group in write mode @@ -453,15 +452,7 @@ class Group { * Load group metadata, handles remote groups vs non-remote groups */ void load_metadata(); - - /** - * Generate new name in the form of timestmap_timestamp_uuid - * - * @return tuple of status and optional string - */ - tuple> generate_name() const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_GROUP_H diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index 1338521e9be0..d65df5d2de73 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,13 +43,11 @@ #include "tiledb/sm/group/group_member_v2.h" #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/rest/rest_client.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { GroupDetails::GroupDetails(const URI& group_uri, uint32_t version) : group_uri_(group_uri) @@ -334,5 +332,4 @@ void GroupDetails::invalidate_lookups() { duplicated_uris_ = nullopt; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc index 868c4b745a21..d531c744a41e 100644 --- a/tiledb/sm/group/group_directory.cc +++ b/tiledb/sm/group/group_directory.cc @@ -37,7 +37,6 @@ #include "tiledb/sm/group/group_member.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb::common; diff --git a/tiledb/sm/metadata/CMakeLists.txt b/tiledb/sm/metadata/CMakeLists.txt index d84f5069bd4c..2e6d29177726 100644 --- a/tiledb/sm/metadata/CMakeLists.txt +++ b/tiledb/sm/metadata/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2022 TileDB, Inc. +# Copyright (c) 2021-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -31,7 +31,7 @@ include(object_library) # commence(object_library metadata) this_target_sources(metadata.cc) - this_target_object_libraries(baseline buffer constants time uuid vfs) + this_target_object_libraries(baseline buffer constants time uri_format vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/metadata/metadata.cc b/tiledb/sm/metadata/metadata.cc index ff726690c12b..fbce14b864de 100644 --- a/tiledb/sm/metadata/metadata.cc +++ b/tiledb/sm/metadata/metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,16 +36,14 @@ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" -#include "tiledb/storage_format/serialization/serializers.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include #include using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class MetadataException : public StatusException { public: @@ -103,25 +101,18 @@ void Metadata::clear() { uri_ = URI(); } -Status Metadata::get_uri(const URI& array_uri, URI* meta_uri) { +URI Metadata::get_uri(const URI& array_uri) { if (uri_.to_string().empty()) - RETURN_NOT_OK(generate_uri(array_uri)); + generate_uri(array_uri); - *meta_uri = uri_; - return Status::Ok(); + return uri_; } -Status Metadata::generate_uri(const URI& array_uri) { - std::string uuid; - RETURN_NOT_OK(uuid::generate_uuid(&uuid, false)); - - std::stringstream ss; - ss << "__" << timestamp_range_.first << "_" << timestamp_range_.second << "_" - << uuid; +void Metadata::generate_uri(const URI& array_uri) { + auto ts_name = tiledb::storage_format::generate_timestamped_name( + timestamp_range_.first, timestamp_range_.second, std::nullopt); uri_ = array_uri.join_path(constants::array_metadata_dir_name) - .join_path(ss.str()); - - return Status::Ok(); + .join_path(ts_name); } Metadata Metadata::deserialize( @@ -365,5 +356,4 @@ void Metadata::build_metadata_index() { metadata_index_[i++] = std::make_pair(&(m.first), &(m.second)); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/metadata/metadata.h b/tiledb/sm/metadata/metadata.h index 4da002b92db4..7041223234a5 100644 --- a/tiledb/sm/metadata/metadata.h +++ b/tiledb/sm/metadata/metadata.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,8 +46,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Buffer; class ConstBuffer; @@ -111,10 +110,10 @@ class Metadata { void clear(); /** Retrieves the array metadata URI. */ - Status get_uri(const URI& array_uri, URI* meta_uri); + URI get_uri(const URI& array_uri); /** Generates a new array metadata URI. */ - Status generate_uri(const URI& array_uri); + void generate_uri(const URI& array_uri); /** * Deserializes the input metadata buffers. Note that the buffers are @@ -272,7 +271,6 @@ class Metadata { void build_metadata_index(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_METADATA_H diff --git a/tiledb/sm/metadata/test/unit_metadata.cc b/tiledb/sm/metadata/test/unit_metadata.cc index cf606da4de38..982902e779f6 100644 --- a/tiledb/sm/metadata/test/unit_metadata.cc +++ b/tiledb/sm/metadata/test/unit_metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,7 +38,6 @@ #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" using namespace tiledb; using namespace tiledb::common; diff --git a/tiledb/sm/misc/uuid.cc b/tiledb/sm/misc/uuid.cc index 7183ff3aed40..052cce47bdcd 100644 --- a/tiledb/sm/misc/uuid.cc +++ b/tiledb/sm/misc/uuid.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,9 +45,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { -namespace uuid { +namespace tiledb::sm::uuid { /** Mutex to guard UUID generation. */ static std::mutex uuid_mtx; @@ -173,6 +171,4 @@ Status generate_uuid(std::string* uuid, bool hyphenate) { return Status::Ok(); } -} // namespace uuid -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::uuid diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 303e686afc65..16f6287e9e99 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -44,7 +44,6 @@ #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index b87a3fd3e5b4..13cd1dcf2cce 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,7 +45,6 @@ #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" @@ -59,8 +58,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -393,5 +391,4 @@ Status OrderedWriter::prepare_filter_and_write_tiles( return Status::Ok(); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 05448057c2cd..6c0e42622405 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -45,7 +45,6 @@ #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" @@ -58,12 +57,11 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { -class UnorderWriterStatusException : public StatusException { +class UnorderWriterException : public StatusException { public: - explicit UnorderWriterStatusException(const std::string& message) + explicit UnorderWriterException(const std::string& message) : StatusException("UnorderWriter", message) { } }; @@ -107,7 +105,7 @@ UnorderedWriter::UnorderedWriter( , is_coords_pass_(true) { // Check the layout is unordered. if (layout != Layout::UNORDERED) { - throw UnorderWriterStatusException( + throw UnorderWriterException( "Failed to initialize UnorderedWriter; The unordered writer does not " "support layout " + layout_str(layout)); @@ -115,14 +113,14 @@ UnorderedWriter::UnorderedWriter( // Check the array is sparse. if (array_schema_.dense()) { - throw UnorderWriterStatusException( + throw UnorderWriterException( "Failed to initialize UnorderedWriter; The unordered " "writer does not support dense arrays."); } // Check no ordered attributes. if (array_schema_.has_ordered_attributes()) { - throw UnorderWriterStatusException( + throw UnorderWriterException( "Failed to initialize UnorderedWriter; The unordered writer does not " "support ordered attributes."); } @@ -166,7 +164,7 @@ Status UnorderedWriter::finalize() { if (written_buffers_.size() < array_schema_.dim_num() + array_schema_.attribute_num()) { - throw UnorderWriterStatusException("Not all buffers already written"); + throw UnorderWriterException("Not all buffers already written"); } return Status::Ok(); @@ -643,15 +641,14 @@ Status UnorderedWriter::unordered_write() { if (written_buffers_.size() >= array_schema_.dim_num() + array_schema_.attribute_num()) { - throw UnorderWriterStatusException("All buffers already written"); + throw UnorderWriterException("All buffers already written"); } if (is_coords_pass_) { for (ArraySchema::dimension_size_type d = 0; d < array_schema_.dim_num(); d++) { if (buffers_.count(array_schema_.dimension_ptr(d)->name()) == 0) { - throw UnorderWriterStatusException( - "All dimension buffers should be set"); + throw UnorderWriterException("All dimension buffers should be set"); } } @@ -743,5 +740,4 @@ Status UnorderedWriter::unordered_write() { return Status::Ok(); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index de83892da003..1579c6beac61 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -46,7 +46,6 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/query/hilbert_order.h" #include "tiledb/sm/query/query_macros.h" #include "tiledb/sm/stats/global_stats.h" diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 4fbb04385940..ce066fc632ab 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -63,7 +63,6 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/query/update_value.h" @@ -153,11 +152,10 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { // Store any changes required if (group->changes_applied()) { const URI& group_detail_folder_uri = group->group_detail_uri(); - auto&& [st, group_detail_uri] = group->generate_detail_uri(); - RETURN_NOT_OK(st); + auto group_detail_uri = group->generate_detail_uri(); RETURN_NOT_OK(store_group_detail( group_detail_folder_uri, - group_detail_uri.value(), + group_detail_uri, group->group_details(), *group->encryption_key())); } @@ -535,7 +533,7 @@ Status StorageManagerCanonical::array_create( std::lock_guard lock{object_create_mtx_}; array_schema->set_array_uri(array_uri); - RETURN_NOT_OK(array_schema->generate_uri()); + array_schema->generate_uri(); array_schema->check(config_); // Create array directory @@ -720,14 +718,13 @@ Status StorageManagerCanonical::array_upgrade_version( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key_cfg); if (array_schema->version() < constants::format_version) { - auto st = array_schema->generate_uri(); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); + array_schema->generate_uri(); array_schema->set_version(constants::format_version); // Create array schema directory if necessary URI array_schema_dir_uri = array_uri.join_path(constants::array_schema_dir_name); - st = vfs()->create_dir(array_schema_dir_uri); + auto st = vfs()->create_dir(array_schema_dir_uri); RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); // Store array schema @@ -1639,8 +1636,7 @@ Status StorageManagerCanonical::store_metadata( stats()->add_counter("write_meta_size", serializer.size()); // Create a metadata file name - URI metadata_uri; - RETURN_NOT_OK(metadata->get_uri(uri, &metadata_uri)); + URI metadata_uri = metadata->get_uri(uri); RETURN_NOT_OK(store_data_to_generic_tile(tile, metadata_uri, encryption_key)); diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index 1366c35ef0f6..fa32a2906110 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -38,7 +38,9 @@ using namespace tiledb::common; namespace tiledb::storage_format { std::string generate_timestamped_name( - uint64_t timestamp_start, uint64_t timestamp_end, uint32_t version) { + uint64_t timestamp_start, + uint64_t timestamp_end, + std::optional version) { std::string uuid; throw_if_not_ok(sm::uuid::generate_uuid(&uuid, false)); @@ -49,8 +51,11 @@ std::string generate_timestamped_name( } std::stringstream ss; - ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid << "_" - << version; + ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid; + + if (version.has_value()) { + ss << "_" << version.value(); + } return ss.str(); } @@ -70,7 +75,7 @@ std::string generate_consolidated_fragment_name( throw_if_not_ok(utils::parse::get_timestamp_range(last, &t_last)); return generate_timestamped_name( - t_first.first, t_last.second, (uint64_t)format_version); + t_first.first, t_last.second, format_version); } } // namespace tiledb::storage_format diff --git a/tiledb/storage_format/uri/generate_uri.h b/tiledb/storage_format/uri/generate_uri.h index 727b3c9b8ddd..9802c6df33b2 100644 --- a/tiledb/storage_format/uri/generate_uri.h +++ b/tiledb/storage_format/uri/generate_uri.h @@ -36,6 +36,8 @@ #include "tiledb/common/common.h" #include "tiledb/sm/filesystem/uri.h" +#include + using namespace tiledb::common; using namespace tiledb::sm; @@ -44,19 +46,21 @@ namespace tiledb::storage_format { /** * Generate a new URI in the form `__t1_t2_uuid_v`, where `t1` is the starting * timestamp, `t2` is the ending timestamp, and `v` is the current format - * version. For instance, + * version, if provided. For instance, * `__1458759561320_1458759561480_6ba7b8129dad11d180b400c04fd430c8_3`. * * @param timestamp_start The staring timestamp. It is in ms since * 1970-01-01 00:00:00 +0000 (UTC). * @param timestamp_end The ending timestamp. It is in ms since * 1970-01-01 00:00:00 +0000 (UTC). - * @param version Version number to append to the URI. + * @param version Optional version number to append to the URI. * * @return The new URI. */ std::string generate_timestamped_name( - uint64_t timestamp_start, uint64_t timestamp_end, uint32_t version); + uint64_t timestamp_start, + uint64_t timestamp_end, + std::optional version); /** * Generates a new fragment name. diff --git a/tiledb/storage_format/uri/test/unit_uri_format.cc b/tiledb/storage_format/uri/test/unit_uri_format.cc index 2c8970db95ee..3fc3246f566e 100644 --- a/tiledb/storage_format/uri/test/unit_uri_format.cc +++ b/tiledb/storage_format/uri/test/unit_uri_format.cc @@ -55,20 +55,37 @@ TEST_CASE( // Check new fragment name's timestamp range auto name_timestamps{name.substr(0, 6)}; CHECK(name_timestamps == "/__1_1"); + + // Check new fragment name's format version + auto name_version{name.substr(name.find_last_of("_"))}; + CHECK(name_version == "_5"); } SECTION("two timestamps") { - // Generate new fragment name - name = generate_timestamped_name(start, end, (uint32_t)format_version); + SECTION("with format version") { + // Generate new fragment name + name = generate_timestamped_name(start, end, format_version); + + // Check new fragment name's format version + auto name_version{name.substr(name.find_last_of("_"))}; + CHECK(name_version == "_5"); + } + + SECTION("without format version") { + // Generate new fragment name + name = generate_timestamped_name(start, end, std::nullopt); + + // Ensure new fragment name has no format version (ends in uuid) + // Precautionary measure: Check that the length is greater than 10 + // Note that format versions are currently no longer than 2 chars + auto name_end{name.substr(name.find_last_of("_"))}; + CHECK(name_end.length() > 5); + } // Check new fragment name's timestamp range auto name_timestamps{name.substr(0, 6)}; CHECK(name_timestamps == "/__1_2"); } - - // Check new fragment name's format version - auto name_version{name.substr(name.find_last_of("_"))}; - CHECK(name_version == "_5"); } TEST_CASE( From dd6960cd199eb1ce7946460a08e704cd8e2c8d3a Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 23 Nov 2023 12:26:54 +0200 Subject: [PATCH 082/456] Move `TemporaryLocalDirectory` to the test support library. (#4532) Introduced in #4498, `TemporaryLocalDirectory` is the first class in the Core sources to use the `std::filesystem` library. This started causing failures in the Release workflow for macOS on x64, because `std::filesystem` is supported since macOS 10.15, while we target 10.14 at minimum (because of CRAN policies). Since we don't actually use this class in the Core, this PR moves it to `tiledb_test_support_lib`, which is not built as part of releasing. --- TYPE: NO_HISTORY --- test/src/unit-capi-nullable.cc | 2 +- test/src/unit-capi-smoke-test.cc | 2 +- test/src/unit-capi-vfs.cc | 2 +- test/src/unit-win-filesystem.cc | 2 +- test/support/CMakeLists.txt | 1 + .../support/src}/temporary_local_directory.cc | 2 +- .../filesystem => test/support/src}/temporary_local_directory.h | 0 tiledb/CMakeLists.txt | 1 - tiledb/sm/filesystem/CMakeLists.txt | 1 - 9 files changed, 6 insertions(+), 7 deletions(-) rename {tiledb/sm/filesystem => test/support/src}/temporary_local_directory.cc (97%) rename {tiledb/sm/filesystem => test/support/src}/temporary_local_directory.h (100%) diff --git a/test/src/unit-capi-nullable.cc b/test/src/unit-capi-nullable.cc index 793dc2cbbf11..bd93abc576ac 100644 --- a/test/src/unit-capi-nullable.cc +++ b/test/src/unit-capi-nullable.cc @@ -32,10 +32,10 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" #include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/array_type.h" -#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #include diff --git a/test/src/unit-capi-smoke-test.cc b/test/src/unit-capi-smoke-test.cc index 95f536078054..2255b9a69550 100644 --- a/test/src/unit-capi-smoke-test.cc +++ b/test/src/unit-capi-smoke-test.cc @@ -33,11 +33,11 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" #include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" -#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #include diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index 32dcc313926a..8e0a48453ac2 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -32,9 +32,9 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" #include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" -#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/misc/utils.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" diff --git a/test/src/unit-win-filesystem.cc b/test/src/unit-win-filesystem.cc index e128e967cff9..c28906351e04 100644 --- a/test/src/unit-win-filesystem.cc +++ b/test/src/unit-win-filesystem.cc @@ -34,6 +34,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" #include #include @@ -42,7 +43,6 @@ #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/crypto.h" #include "tiledb/sm/filesystem/path_win.h" -#include "tiledb/sm/filesystem/temporary_local_directory.h" #include "tiledb/sm/filesystem/win.h" #include diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index 0d3176203060..0b97ae844051 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -42,6 +42,7 @@ set(TILEDB_TEST_SUPPORT_SOURCES src/helpers.cc src/helpers-dimension.h src/serialization_wrappers.cc + src/temporary_local_directory.cc src/vfs_helpers.cc ) diff --git a/tiledb/sm/filesystem/temporary_local_directory.cc b/test/support/src/temporary_local_directory.cc similarity index 97% rename from tiledb/sm/filesystem/temporary_local_directory.cc rename to test/support/src/temporary_local_directory.cc index ef58438dd6df..538b3b25253b 100644 --- a/tiledb/sm/filesystem/temporary_local_directory.cc +++ b/test/support/src/temporary_local_directory.cc @@ -30,7 +30,7 @@ * This file defines class TemporaryLocalDirectory. */ -#include "tiledb/sm/filesystem/temporary_local_directory.h" +#include "temporary_local_directory.h" #include "tiledb/common/random/prng.h" #include diff --git a/tiledb/sm/filesystem/temporary_local_directory.h b/test/support/src/temporary_local_directory.h similarity index 100% rename from tiledb/sm/filesystem/temporary_local_directory.h rename to test/support/src/temporary_local_directory.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 72665ac4a70c..123c0a726d6c 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -198,7 +198,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/s3_thread_pool_executor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/ssl_config.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/temporary_local_directory.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/uri.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs_file_handle.cc diff --git a/tiledb/sm/filesystem/CMakeLists.txt b/tiledb/sm/filesystem/CMakeLists.txt index de8770cb2df3..121829379347 100644 --- a/tiledb/sm/filesystem/CMakeLists.txt +++ b/tiledb/sm/filesystem/CMakeLists.txt @@ -35,7 +35,6 @@ commence(object_library vfs) mem_filesystem.cc path_win.cc posix.cc - temporary_local_directory.cc uri.cc vfs.cc vfs_file_handle.cc From f1c8d301351a7eed4801496a27c805dd045e4204 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 23 Nov 2023 13:53:01 +0200 Subject: [PATCH 083/456] Enable calls to `find_dependency` in the exported config only if we are building a static library. (#4526) #4505 replaced bundling of our dependencies' static libraries with calls to CMake's `find_dependency` command. We definitely need these calls when we have installed a static library, and we definitely don't need them in the release artifacts where everything is contained in a shared library. Out of abundance of caution for potential cases where we might need these options in future scenarios, I controlled emitting these calls with `TILEDB_INSTALL_FIND_DEPENDENCIES`, an on-by-default option that was always enabled when building a static library. This caused disruptions in downstream projects such as SOMA because of this change in defaults. My initial thought was to make this option off-by-default if we are not building a static library, but as it turns out we never need it enabled in such cases, because a shared library does not require linking to its dependencies when we link to it, and these dependencies are implementation details and not exposed through our public C API. Contrast this with cases like `azure-storage-blobs-cpp` depending on `azure-storage-common-cpp` at install-time, because the former's public headers depend on the latter's. So this PR removes the `TILEDB_INSTALL_FIND_DEPENDENCIES` option and replaces it with `TILEDB_STATIC`. --- TYPE: NO_HISTORY --- bootstrap | 7 ------- bootstrap.ps1 | 14 +------------- cmake/Options/BuildOptions.cmake | 1 - cmake/inputs/Config.cmake.in | 2 +- scripts/azure-linux_mac-release.yml | 2 +- scripts/azure-windows-release.yml | 2 +- 6 files changed, 4 insertions(+), 24 deletions(-) diff --git a/bootstrap b/bootstrap index 0269a9faedbf..8ade2249dabf 100755 --- a/bootstrap +++ b/bootstrap @@ -47,10 +47,6 @@ Configuration: --disable-avx2 disables use of AVX2 instructions --disable-webp disables building of webp library and linkage test --disable-vcpkg disables use of vcpkg for downloading and building dependencies - --disable-install-find-dependencies - Disables calls to find_dependency in the exported config file. - Should be set to ON when building a shared library with all - dependencies statically linked. --enable-static-tiledb enables building TileDB as a static library --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) @@ -120,7 +116,6 @@ tiledb_build_webp="ON" tiledb_build_abseil="OFF" tiledb_tests_enable_rest="OFF" tiledb_tests_aws_s3_config="OFF" -tiledb_install_find_dependencies="ON" enable_multiple="" while test $# != 0; do case "$1" in @@ -139,7 +134,6 @@ while test $# != 0; do --disable-avx2) tiledb_disable_avx2="-DCOMPILER_SUPPORTS_AVX2=FALSE";; --disable-webp) tiledb_build_webp="OFF";; --disable-vcpkg) tiledb_vcpkg="OFF";; - --disable-install-find-dependencies) tiledb_install_find_dependencies="OFF";; --enable-assertions) tiledb_assertions="ON";; --enable-debug) build_type="Debug";; --enable-release-symbols) build_type="RelWithDebInfo";; @@ -261,7 +255,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ - -DTILEDB_INSTALL_FIND_DEPENDENCIES=${tiledb_install_find_dependencies} \ ${tiledb_disable_avx2} \ "${source_dir}" || die "failed to configure the project" diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 306f3a81b9f4..6c3c0358406b 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -101,9 +101,6 @@ Disables internal TileDB statistics. .PARAMETER DisableVcpkg Disables building dependencies with vcpkg. -.PARAMETER DisableInstallFindDependencies -Disables calling find_dependency in the exported config file. Should be set when when building a shared library with all dependencies statically linked. - .PARAMETER BuildProcesses Number of parallel compile jobs. @@ -148,7 +145,6 @@ Param( [switch]$DisableCppApi, [switch]$DisableTests, [switch]$DisableStats, - [switch]$DisableInstallFindDependencies, [Alias('J')] [int] $BuildProcesses = $env:NUMBER_OF_PROCESSORS @@ -247,14 +243,6 @@ if ($DisableStats.IsPresent) { $Stats = "OFF" } -$InstallFindDependencies = "ON" -if ($DisableInstallFindDependencies.IsPresent) { - $InstallFindDependencies = "OFF" - if ($EnableStaticTileDB.IsPresent) { - Write-Warning "DisableInstallFindDependencies is ignored when building a static library." - } -} - $BuildWebP="ON" if ($DisableWebP.IsPresent) { $BuildWebP="OFF" @@ -338,7 +326,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 -DTILEDB_INSTALL_FIND_DEPENDENCIES=$InstallFindDependencies $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 5ac53c289836..6a0538d1792c 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -22,7 +22,6 @@ option(TILEDB_CPP_API "Enables building of the TileDB C++ API." ON) option(TILEDB_CMAKE_IDE "(Used for CLion builds). Disables superbuild and sets the EP install dir." OFF) option(TILEDB_STATS "Enables internal TileDB statistics gathering." ON) option(TILEDB_STATIC "Enables building TileDB as a static library." OFF) -cmake_dependent_option(TILEDB_INSTALL_FIND_DEPENDENCIES "Enables calls to find_dependency in the exported config file. Should be disabled only when building a shared library with all dependencies statically linked." ON "NOT TILEDB_STATIC" ON) option(TILEDB_TESTS "If true, enables building the TileDB unit test suite" ON) option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in index 0967ecc05973..49c85082268a 100644 --- a/cmake/inputs/Config.cmake.in +++ b/cmake/inputs/Config.cmake.in @@ -10,7 +10,7 @@ @PACKAGE_INIT@ -if(@TILEDB_INSTALL_FIND_DEPENDENCIES@) # TILEDB_INSTALL_FIND_DEPENDENCIES +if(@TILEDB_STATIC@) # TILEDB_STATIC include(CMakeFindDependencyMacro) find_dependency(Threads) find_dependency(BZip2) diff --git a/scripts/azure-linux_mac-release.yml b/scripts/azure-linux_mac-release.yml index 09051edbe3f5..5e8e27f3da30 100755 --- a/scripts/azure-linux_mac-release.yml +++ b/scripts/azure-linux_mac-release.yml @@ -60,7 +60,7 @@ steps: # Set up arguments for bootstrap.sh BUILD_BINARIESDIRECTORY=${BUILD_BINARIESDIRECTORY:-$BUILD_REPOSITORY_LOCALPATH/build/dist} - cmake_args="-DCMAKE_INSTALL_PREFIX=${BUILD_BINARIESDIRECTORY} -DTILEDB_TESTS=OFF -DTILEDB_INSTALL_LIBDIR=lib -DTILEDB_INSTALL_FIND_DEPENDENCIES=OFF"; + cmake_args="-DCMAKE_INSTALL_PREFIX=${BUILD_BINARIESDIRECTORY} -DTILEDB_TESTS=OFF -DTILEDB_INSTALL_LIBDIR=lib"; mkdir -p ${BUILD_BINARIESDIRECTORY} # Enable TILEDB_STATIC by default diff --git a/scripts/azure-windows-release.yml b/scripts/azure-windows-release.yml index 66351f473e21..6b36ca90d339 100644 --- a/scripts/azure-windows-release.yml +++ b/scripts/azure-windows-release.yml @@ -27,7 +27,7 @@ steps: $host.SetShouldExit(3) } - $bootstrapOptions = "-EnableVerbose -DisableTests -DisableInstallFindDependencies" + $bootstrapOptions = "-EnableVerbose -DisableTests" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } From bafca5bad7b10debb776746eddbcad9e44e5b75e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 23 Nov 2023 17:43:43 +0200 Subject: [PATCH 084/456] Move the Release workflow to GitHub Actions (#4518) [SC-36530](https://app.shortcut.com/tiledb-inc/story/36530/release-with-github-actions) This PR updates the Release workflow to run on top of GitHub Actions. The release builds now also run on every PR. To test the new workflow I am making [dummy releases to my fork](https://github.com/teo-tsirpanis/TileDB/releases). This PR will be ready for review once that succeeds. --- TYPE: NO_HISTORY DESC: Move the Release workflow to GitHub Actions. --- .github/workflows/nightly-test.yml | 1 - .github/workflows/release.yml | 163 +++++++++++++++++++++ azure-pipelines.yml | 120 --------------- cmake/Options/BuildOptions.cmake | 1 - ports/triplets/arm64-osx-release.cmake | 9 ++ ports/triplets/arm64-osx.cmake | 1 + ports/triplets/x64-linux-release.cmake | 7 + ports/triplets/x64-osx-release.cmake | 9 ++ ports/triplets/x64-osx.cmake | 1 + ports/triplets/x64-windows-release.cmake | 7 + scripts/azure-linux_mac-release.yml | 179 ----------------------- scripts/azure-windows-release.yml | 130 ---------------- 12 files changed, 197 insertions(+), 431 deletions(-) create mode 100644 .github/workflows/release.yml delete mode 100644 azure-pipelines.yml create mode 100644 ports/triplets/arm64-osx-release.cmake create mode 100644 ports/triplets/x64-linux-release.cmake create mode 100644 ports/triplets/x64-osx-release.cmake create mode 100644 ports/triplets/x64-windows-release.cmake delete mode 100755 scripts/azure-linux_mac-release.yml delete mode 100644 scripts/azure-windows-release.yml diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 9868167bf0ed..eab5a6fc375c 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -30,7 +30,6 @@ jobs: ${{ matrix.os }} - Sanitizer: ${{ matrix.sanitizer || 'none' }} | Experimental: ${{ matrix.experimental || 'OFF' }} | ${{ matrix.config || 'Release' }} env: - MACOSX_DEPLOYMENT_TARGET: 10.15 TILEDB_NIGHTLY_BUILD: 1 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..2eb12cad3ba6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,163 @@ +name: Release + +on: + pull_request: + branches: + - '*' # must quote since "*" is a YAML reserved character; we want a string + paths-ignore: + - '.github/workflows/quarto-render.yml' + - '_quarto.yml' + - 'quarto-materials/*' + - '**/.md' + - 'tiledb/doxygen/source/*' + - 'tiledb/sm/c_api/tiledb_version.h' + push: + branches: + - dev + - 'release-*' + tags: + - '*' + +jobs: + Build-Release: + strategy: + fail-fast: false + matrix: + platform: [macos-arm64, macos-x86_64, linux-x86_64, linux-x86_64-noavx2, windows-x86_64] + include: + - platform: windows-x86_64 + os: windows-2019 + triplet: x64-windows-release + - platform: linux-x86_64 + os: ubuntu-20.04 + manylinux: true + triplet: x64-linux-release + - platform: linux-x86_64-noavx2 + os: ubuntu-20.04 + cmake_args: -DCOMPILER_SUPPORTS_AVX2=OFF + triplet: x64-linux-release + manylinux: true + - platform: macos-x86_64 + os: macos-12 + # By default we have to target macOS 10.14 at minimum according to CRAN policy. + # See https://stat.ethz.ch/pipermail/r-package-devel/2023q4/010078.html + MACOSX_DEPLOYMENT_TARGET: 10.14 + triplet: x64-osx-release + - platform: macos-arm64 + os: macos-12 + cmake_args: -DCMAKE_OSX_ARCHITECTURES=arm64 + MACOSX_DEPLOYMENT_TARGET: 11 + triplet: arm64-osx-release + runs-on: ${{ matrix.os }} + container: ${{ matrix.manylinux && 'quay.io/pypa/manylinux2014_x86_64:2023-11-13-f6b0c51' || '' }} + env: + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.MACOSX_DEPLOYMENT_TARGET }} + VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' + + steps: + - name: Checkout TileDB + # v4 uses node 20 which is incompatible with the libc version of the manylinux image + uses: actions/checkout@v3 + - name: Export GitHub Actions cache variables + uses: actions/github-script@v6 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - name: Set variables + id: get-values + run: | + release_hash=$( echo ${{ github.sha }} | cut -c-8 - ) + ref=${{ github.head_ref || github.ref_name }} + echo "release_hash=$release_hash" >> $GITHUB_OUTPUT + echo "archive_name=tiledb-${{ matrix.platform }}-${ref##*/}-$release_hash" >> $GITHUB_OUTPUT + shell: bash + - name: Install manylinux prerequisites + if: matrix.manylinux + run: | + set -e pipefail + yum install -y redhat-lsb-core centos-release-scl devtoolset-7 + echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc + - name: Configure TileDB + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=./dist \ + -DTILEDB_S3=ON \ + -DTILEDB_AZURE=ON \ + -DTILEDB_GCS=ON \ + -DTILEDB_HDFS=${{ startsWith(matrix.platform, 'windows') && 'OFF' || 'ON' }} \ + -DTILEDB_SERIALIZATION=ON \ + -DTILEDB_WEBP=ON \ + -DTILEDB_STATIC=OFF \ + -DTILEDB_TESTS=OFF \ + -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \ + ${{ matrix.cmake_args }} + shell: bash + - name: Build TileDB + run: cmake --build build --config Release + - name: Install TileDB + run: cmake --build build --config Release --target install-tiledb + - name: Archive installed artifacts (non-Windows) + if: ${{ !startsWith(matrix.platform, 'windows') }} + run: | + tar -czf ${{ steps.get-values.outputs.archive_name }}.tar.gz -C dist . + - name: Archive installed artifacts (Windows) + if: startsWith(matrix.platform, 'windows') + run: | + Compress-Archive -Path dist\* -DestinationPath ${{ steps.get-values.outputs.archive_name }}.zip + shell: pwsh + - name: Upload release artifacts + uses: actions/upload-artifact@v3 + with: + name: tiledb-dist + path: ${{ steps.get-values.outputs.archive_name }}.* + - name: Archive build directory + run: | + tar -czf build-${{ matrix.platform }}.tar.gz -C build . + - name: Upload build directory + uses: actions/upload-artifact@v3 + with: + name: tiledb-build + path: build-${{ matrix.platform }}.tar.gz + - name: "Print log files (failed build only)" + run: | + source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh + if: failure() # only run this job if the build step failed + + Publish-Release: + needs: Build-Release + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + steps: + - name: Download release artifacts + uses: actions/download-artifact@v2 + with: + name: tiledb-dist + path: dist + - name: Publish release artifacts + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const repo = context.repo; + const release = await github.rest.repos.getReleaseByTag({ + owner: repo.owner, + repo: repo.repo, + tag: "${{ github.ref_name }}" + }); + const globber = await glob.create('dist/*'); + for await (const file of globber.globGenerator()) { + await github.rest.repos.uploadReleaseAsset({ + owner: repo.owner, + repo: repo.repo, + release_id: release.data.id, + headers: { + 'content-type': 'application/octet-stream', + 'content-length': fs.statSync(file).size + }, + name: path.basename(file), + data: fs.readFileSync(file) + }); + } diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 236ae26cc838..000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,120 +0,0 @@ -trigger: - branches: - include: - - dev - - release-* - - refs/tags/* - - build- - -variables: - - name: MANYLINUX_IMAGE - #value: quay.io/pypa/manylinux2014_x86_64:2022-11-06-7be974c - # modified from above tag to add linux-headers - value: ghcr.io/ihnorton/tiledb-manylinux2014_x86_64:2023-04-02 - -stages: - - stage: Build_Release - condition: or(or(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), eq(variables['Build.SourceBranchName'], 'dev')), startsWith(variables['Build.SourceBranchName'], 'release-')), startsWith(variables['Build.SourceBranchName'], 'build-')) - variables: - TILEDB_S3: ON - TILEDB_AZURE: ON - TILEDB_GCS: ON - TILEDB_HDFS: ON - TILEDB_STATIC: OFF - TILEDB_SERIALIZATION: ON - MACOSX_DEPLOYMENT_TARGET: 10.14 - SUDO: 'sudo' - jobs: - - job: - timeoutInMinutes: 90 - strategy: - matrix: - macOS: - imageName: 'macos-12' - CXX: clang++ - ARTIFACT_OS: 'macos' - ARTIFACT_ARCH: 'x86_64' - CMAKE_OSX_ARCHITECTURES: "x86_64" - macOS_arm64: - imageName: 'macos-12' - CXX: clang++ - ARTIFACT_OS: 'macos' - ARTIFACT_ARCH: 'arm64' - TILEDB_GCS: OFF - TILEDB_WERROR: OFF - CMAKE_OSX_ARCHITECTURES: "arm64" - BUILD_MAGIC_MACOS_UNIVERSAL: "ON" - MACOSX_DEPLOYMENT_TARGET: 11 - pool: - vmImage: $(imageName) - steps: - - template: scripts/azure-linux_mac-release.yml - - job: linux_manylinux - timeoutInMinutes: 90 - pool: { vmImage: 'ubuntu-20.04' } - container: ${{ variables.MANYLINUX_IMAGE }} - strategy: - matrix: - standard: - CXX: g++ - CFLAGS: "-lrt" - CXXFLAGS: "-lrt" - ARTIFACT_OS: 'linux' - ARTIFACT_ARCH: "x86_64" - TILEDB_AVX2: ON - SUDO: '' - noavx2: - CXX: g++ - CFLAGS: "-lrt" - CXXFLAGS: "-lrt" - ARTIFACT_OS: 'linux' - ARTIFACT_ARCH: "x86_64-noavx2" - TILEDB_AVX2: OFF - SUDO: '' - steps: - - template: scripts/azure-linux_mac-release.yml - - job: Windows - timeoutInMinutes: 120 - strategy: - matrix: - VS2019: - imageName: 'windows-2019' - # Only S3 variable is currently supported in bootstrap powershell - TILEDB_HDFS: OFF - ARTIFACT_OS: 'windows' - ARTIFACT_ARCH: 'x86_64' - pool: - vmImage: $(imageName) - steps: - - template: scripts/azure-windows-release.yml - - # NOTE: this section cannot be conditional because `Build.Repository.Name` is an agent-scoped variable. - - stage: Github_Release - condition: startsWith(variables['Build.SourceBranch'], 'refs/tags') - pool: - vmImage: 'ubuntu-latest' - jobs: - - job: - steps: - # First download artifacts - - task: DownloadBuildArtifacts@0 - inputs: - downloadType: 'specific' - - script: | - echo $sourceVersion - commitHash=${sourceVersion:0:7} - echo $commitHash - echo "##vso[task.setvariable variable=commitHash]$commitHash" ## Set variable for using in other tasks. - env: { sourceVersion: $(Build.SourceVersion) } - displayName: Git Hash 7-digit - - task: GithubRelease@0 - condition: succeeded() # only run this job if the build step succeeded - displayName: 'Add artifacts to GitHub Release' - inputs: - gitHubConnection: TileDB-Inc-Release - repositoryName: $(Build.Repository.Name) - addChangeLog: false - action: edit - tag: $(Build.SourceBranchName) - assets: | - $(Build.ArtifactStagingDirectory)/built-libs/* diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 6a0538d1792c..6714907c34aa 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -36,7 +36,6 @@ option(TILEDB_TESTS_AWS_S3_CONFIG "Use an S3 config appropriate for AWS in tests option(TILEDB_TESTS_ENABLE_REST "Enables REST tests (requires running REST server)" OFF) option(CMAKE_EXPORT_COMPILE_COMMANDS "cmake compile commands" ON) -option(_TILEDB_CMAKE_INIT_GIT_SUBMODULES "Check submodules during build" ON) set(TILEDB_INSTALL_LIBDIR "" CACHE STRING "If non-empty, install TileDB library to this directory instead of CMAKE_INSTALL_LIBDIR.") diff --git a/ports/triplets/arm64-osx-release.cmake b/ports/triplets/arm64-osx-release.cmake new file mode 100644 index 000000000000..0f0d0b2f20b5 --- /dev/null +++ b/ports/triplets/arm64-osx-release.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) + +set(VCPKG_BUILD_TYPE release) diff --git a/ports/triplets/arm64-osx.cmake b/ports/triplets/arm64-osx.cmake index 62325a696759..348949babcfb 100644 --- a/ports/triplets/arm64-osx.cmake +++ b/ports/triplets/arm64-osx.cmake @@ -4,3 +4,4 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES arm64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) diff --git a/ports/triplets/x64-linux-release.cmake b/ports/triplets/x64-linux-release.cmake new file mode 100644 index 000000000000..8a25c72e1e89 --- /dev/null +++ b/ports/triplets/x64-linux-release.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_BUILD_TYPE release) diff --git a/ports/triplets/x64-osx-release.cmake b/ports/triplets/x64-osx-release.cmake new file mode 100644 index 000000000000..f4fa41a3975f --- /dev/null +++ b/ports/triplets/x64-osx-release.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 10.14) + +set(VCPKG_BUILD_TYPE release) diff --git a/ports/triplets/x64-osx.cmake b/ports/triplets/x64-osx.cmake index 8cd8c90b4a69..e8a4c8d9d69e 100644 --- a/ports/triplets/x64-osx.cmake +++ b/ports/triplets/x64-osx.cmake @@ -4,3 +4,4 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) diff --git a/ports/triplets/x64-windows-release.cmake b/ports/triplets/x64-windows-release.cmake new file mode 100644 index 000000000000..cea1b71be19b --- /dev/null +++ b/ports/triplets/x64-windows-release.cmake @@ -0,0 +1,7 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) + +set(VCPKG_BUILD_TYPE release) diff --git a/scripts/azure-linux_mac-release.yml b/scripts/azure-linux_mac-release.yml deleted file mode 100755 index 5e8e27f3da30..000000000000 --- a/scripts/azure-linux_mac-release.yml +++ /dev/null @@ -1,179 +0,0 @@ -steps: -- task: NodeTool@0 # Needed for compatibility with manylinux image. - inputs: - versionSpec: '16.x' - displayName: 'Install Node.js 16' - -- bash: | - echo "'uname -s' is:" - echo "uname: " $(uname) - echo "uname -m: " $(uname -m) - echo "uname -r:" $(uname -r) - echo "uname -s: " $(uname -s) - echo "uname -v: " $(uname -v) - printenv - echo "vcpkg is: " $(which vcpkg || true) - displayName: 'Print env' - -- bash: | - # This stage is needed until https://github.com/actions/runner-images/pull/7125 or similar is merged - # MacOS image 20230214.1 removed implicit installation of pkg-config - set -e pipefail - brew install pkg-config autoconf automake - condition: eq(variables['Agent.OS'], 'Darwin') - displayName: 'Install brew packages for macOS build' - -- bash: python ./scripts/ci/patch_vcpkg_triplets.py - displayName: Prevent vpckg from building debug variants - -# Tried using mamba for manylinux missing packages - switched to custom image instead. -#- bash: | -# curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba -# bin/micromamba create -p $HOME/m -y -# bin/micromamba -p $HOME/m install -y -c conda-forge zip tar unzip -# echo '##vso[task.prependpath]$(HOME)/m/bin' -# condition: eq(variables['Agent.OS'], 'Linux') -# displayName: 'Install vcpkg dependencies (Linux only)' - -- bash: | - set -e pipefail - $SUDO rm -Rf /Library/Developer/CommandLineTools/SDKs/* # Remove SDKs without ARM support - - condition: eq(variables['Agent.OS'], 'Darwin') - displayName: 'Remove sdks for testing (OSX only)' - -- bash: | - export PATH=$HOME/m/bin:$PATH - # Azure sets "SYSTEM=build" for unknown reasons, which breaks the OpenSSL configure script - # - openssl configure uses ENV{SYSTEM} if available: - # https://github.com/openssl/openssl/blob/6d745d740d37d680ff696486218b650512bbbbc6/config#L56 - # - error description: - # https://developercommunity.visualstudio.com/content/problem/602584/openssl-build-error-when-using-pipelines.htm - unset SYSTEM - - # azure bash does not treat intermediate failure as error - # https://github.com/Microsoft/azure-pipelines-yaml/issues/135 - set -e pipefail - - git config --global user.name 'Azure Pipeline' - git config --global user.email 'no-reply@tiledb.io' - - # Set up arguments for bootstrap.sh - BUILD_BINARIESDIRECTORY=${BUILD_BINARIESDIRECTORY:-$BUILD_REPOSITORY_LOCALPATH/build/dist} - cmake_args="-DCMAKE_INSTALL_PREFIX=${BUILD_BINARIESDIRECTORY} -DTILEDB_TESTS=OFF -DTILEDB_INSTALL_LIBDIR=lib"; - mkdir -p ${BUILD_BINARIESDIRECTORY} - - # Enable TILEDB_STATIC by default - [ "$TILEDB_STATIC" ] || TILEDB_STATIC=ON - if [[ "$TILEDB_STATIC" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_STATIC=ON"; - fi - if [[ "$TILEDB_HDFS" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_HDFS=ON"; - fi; - if [[ "$TILEDB_S3" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_S3=ON"; - fi; - if [[ "$TILEDB_AZURE" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_AZURE=ON"; - fi; - if [[ "$TILEDB_GCS" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_GCS=ON"; - fi; - if [[ "$TILEDB_TOOLS" == "ON" ]]; then - cmake_args="${cmake_args} -DTILEDB_TOOLS=ON"; - fi - if [[ "$TILEDB_DEBUG" == "ON" ]]; then - cmake_args="${cmake_args} -DCMAKE_BUILD_TYPE=Debug"; - fi - if [[ "$TILEDB_CI_ASAN" == "ON" ]]; then - # Add address sanitizer flag if necessary - cmake_args="${cmake_args} -DSANITIZER=address"; - fi - if [[ "$TILEDB_CI_TSAN" == "ON" ]]; then - # Add thread sanitizer flag if necessary - cmake_args="${cmake_args} -DSANITIZER=thread"; - fi - if [[ "$TILEDB_SERIALIZATION" == "ON" ]]; then - # Add serialization flag if necessary - cmake_args="${cmake_args} -DTILEDB_SERIALIZATION=ON"; - fi - if [[ "$TILEDB_FORCE_BUILD_DEPS" == "ON" ]]; then - # Add superbuild flag - cmake_args="${cmake_args} -DTILEDB_FORCE_ALL_DEPS=ON"; - fi - if [[ "$TILEDB_WERROR" == "OFF" ]]; then - # Add superbuild flag - cmake_args="${cmake_args} -DTILEDB_WERROR=OFF"; - fi - if [[ "$TILEDB_AVX2" == "OFF" ]]; then - # Add superbuild flag - cmake_args="${cmake_args} -DCOMPILER_SUPPORTS_AVX2=OFF"; - fi - if [[ "$TILEDB_WEBP" == "OFF" ]]; then - cmake_args="${cmake_args} -DTILEDB_WEBP=OFF"; - fi; - if [[ ! -z "$CMAKE_OSX_ARCHITECTURES" ]]; then - cmake_args="${cmake_args} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}"; - fi; - - mkdir -p $BUILD_REPOSITORY_LOCALPATH/build - cd $BUILD_REPOSITORY_LOCALPATH/build - - # Configure and build TileDB - echo "Running cmake with '${cmake_args}'" - cmake .. ${cmake_args} - - make -j4 - make -C tiledb install - displayName: 'Build libtiledb' - continueOnError: true - -- bash: | - cat $BUILD_REPOSITORY_LOCALPATH/build/vcpkg-bootstrap.log || true - displayName: 'Dump vcpkg logs' - continueOnError: true - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Build.Repository.LocalPath)/build' - includeRootFolder: false - archiveType: 'tar' # Options: zip, 7z, tar, wim - tarCompression: 'gz' # Optional. Options: gz, bz2, xz, none - archiveFile: $(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-build-dir.tar.gz - replaceExistingArchive: true - verbose: true # Optional - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-build-dir.tar.gz' - artifactName: 'build-dirs' - condition: succeeded() - -- script: | - echo $sourceVersion - commitHash=${sourceVersion:0:7} - echo $commitHash - echo "##vso[task.setvariable variable=commitHash]$commitHash" ## Set variable for using in other tasks. - env: { sourceVersion: $(Build.SourceVersion) } - displayName: Git Hash 7-digit - -# Archive files -# Compress files into .7z, .tar.gz, or .zip -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Build.BinariesDirectory)' - includeRootFolder: false - archiveType: 'tar' # Options: zip, 7z, tar, wim - tarCompression: 'gz' # Optional. Options: gz, bz2, xz, none - archiveFile: $(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-$(Build.SourceBranchName)-$(commitHash).tar.gz - replaceExistingArchive: true - verbose: true # Optional - condition: succeeded() - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-$(Build.SourceBranchName)-$(commitHash).tar.gz' - artifactName: 'built-libs' - condition: succeeded() - diff --git a/scripts/azure-windows-release.yml b/scripts/azure-windows-release.yml deleted file mode 100644 index 6b36ca90d339..000000000000 --- a/scripts/azure-windows-release.yml +++ /dev/null @@ -1,130 +0,0 @@ -steps: -- script: | - echo $(Build.SourceVersion) - set TestVar=$(Build.SourceVersion) - set $commitHash=%TestVar:~0,7% - echo %$commitHash% - echo ##vso[task.setvariable variable=commitHash]%$commitHash% - env: { sourceVersion: $(Build.SourceVersion) } - displayName: Git Hash 7-digit - -- powershell: python ./scripts/ci/patch_vcpkg_triplets.py - displayName: Prevent vpckg from building debug variants - -- powershell: | - mkdir $env:AGENT_BUILDDIRECTORY\build - cd $env:AGENT_BUILDDIRECTORY\build - - if ($env:imageName -eq "windows-2019") { - $VSCategory = "Enterprise" # alternate 'Community' - if (!(Test-Path "C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin")) { - Write-Host "ERROR***: Missing C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" - $host.SetShouldExit(2) - } - $env:Path += ";C:\Program Files (x86)\Microsoft Visual Studio\2019\${VSCategory}\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" - } else { - Write-Host "Unknown image name: '$($env:imageName)'" - $host.SetShouldExit(3) - } - - $bootstrapOptions = "-EnableVerbose -DisableTests" - if ($env:TILEDB_S3 -eq "ON") { - $bootstrapOptions = "-EnableS3 " + $bootstrapOptions - } - if ($env:TILEDB_AZURE -eq "ON") { - $bootstrapOptions = "-EnableAzure " + $bootstrapOptions - } - if ($env:TILEDB_GCS -eq "ON") { - $bootstrapOptions = "-EnableGcs " + $bootstrapOptions - #NOTE: GCS simulator not yet actually in place. - } - if ($env:TILEDB_SERIALIZATION -eq "ON") { - $bootstrapOptions = "-EnableSerialization " + $bootstrapOptions - } - if ($env:TILEDB_STATIC -eq "ON") { - $bootstrapOptions = $bootstrapOptions + " -EnableStaticTileDB" - } - # if ($env:TILEDB_HDFS -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableHDFS" - #} - if ($env:TILEDB_TOOLS -eq "ON") { - $bootstrapOptions = $bootstrapOptions + " -EnableTools" - } - if ($env:TILEDB_DEBUG -eq "ON") { - $bootstrapOptions = $bootstrapOptions + " -EnableDebug" - } - # if ($env:TILEDB_CI_ASAN -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableSanitizer address -EnableDebug" - # } - # if ($env:TILEDB_CI_TSAN -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableSanitizer thread -EnableDebug" - # } - if ($env:TILEDB_WEBP -eq "OFF") { - $bootstrapOptions = $bootstrapOptions + " -DisableWebP" - } - if ($env:TILEDB_WERROR -eq "OFF") { - $bootstrapOptions = $bootstrapOptions + " -DisableWerror" - } - - # Set install prefix for packaging purposes - $bootstrapOptions = "-Prefix $env:AGENT_BUILDDIRECTORY/s/dist " + $bootstrapOptions - - $bootstrapExpression = "& $env:BUILD_SOURCESDIRECTORY\bootstrap.ps1 " + $bootstrapOptions - Write-Host "bootstrapExpression: $bootstrapExpression" - Invoke-Expression $bootstrapExpression - - if ($LastExitCode -ne 0) { - Write-Host "Bootstrap failed." - $host.SetShouldExit($LastExitCode) - } - - cmake --build $env:AGENT_BUILDDIRECTORY\build --config Release -- /verbosity:minimal - - if ($LastExitCode -ne 0) { - Write-Host "Build failed. CMake exit status: " $LastExitCocde - $host.SetShouldExit($LastExitCode) - } - - cmake --build $env:AGENT_BUILDDIRECTORY\build --target install-tiledb --config Release - - if ($LastExitCode -ne 0) { - Write-Host "Installation failed." - $host.SetShouldExit($LastExitCode) - } - displayName: "Build" - continueOnError: true - -# Archive files -# Compress files into .7z, .tar.gz, or .zip -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Agent.BuildDirectory)\s\dist\' - includeRootFolder: false - archiveType: 'zip' # Options: zip, 7z, tar, wim - archiveFile: $(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-$(Build.SourceBranchName)-$(commitHash).zip - replaceExistingArchive: true - verbose: true # Optional - condition: succeeded() - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)\tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-$(Build.SourceBranchName)-$(commitHash).zip' - artifactName: 'built-libs' - condition: succeeded() - -- task: ArchiveFiles@2 - inputs: - rootFolderOrFile: '$(Agent.BuildDirectory)\build\' - includeRootFolder: false - archiveType: 'zip' # Options: zip, 7z, tar, wim - archiveFile: $(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-build-dir.zip - replaceExistingArchive: true - verbose: true # Optional - continueOnError: true - -- task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.ArtifactStagingDirectory)/tiledb-$(ARTIFACT_OS)-$(ARTIFACT_ARCH)-build-dir.zip' - artifactName: 'build-dirs' - condition: succeeded() - continueOnError: true From 5d89202d297db5cda0f9819b28ba5cade096e09f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 23 Nov 2023 17:53:32 +0200 Subject: [PATCH 085/456] Refactor the group code. (#4378) [SC-37180](https://app.shortcut.com/tiledb-inc/story/37180/finish-group-refactor-pr) This PR refactors the group code to reduce state mutations. More specifically, before writing a group to storage we were calling the `GroupDetails::apply_pending_changes` method which updated the group details' canonical store of members. This PR removes `apply_pending_changes` and replaces it with a `members_to_serialize` method, that computes the members to write to storage without modifying the group details object. This makes the code easier to understand. --- TYPE: IMPROVEMENT DESC: Refactor the group code. --- tiledb/sm/group/group.cc | 23 ++-------- tiledb/sm/group/group.h | 21 +-------- tiledb/sm/group/group_details.cc | 15 ++---- tiledb/sm/group/group_details.h | 48 +++++++++++++------- tiledb/sm/group/group_details_v1.cc | 30 +++++++----- tiledb/sm/group/group_details_v1.h | 18 ++------ tiledb/sm/group/group_details_v2.cc | 34 +++++++------- tiledb/sm/group/group_details_v2.h | 18 ++------ tiledb/sm/serialization/group.cc | 4 +- tiledb/sm/storage_manager/storage_manager.cc | 7 +-- 10 files changed, 94 insertions(+), 124 deletions(-) diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index adc87909d36b..508a9be2f4c8 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -67,8 +67,7 @@ Group::Group(const URI& group_uri, StorageManager* storage_manager) , query_type_(QueryType::READ) , timestamp_start_(0) , timestamp_end_(UINT64_MAX) - , encryption_key_(tdb::make_shared(HERE())) - , changes_applied_(false) { + , encryption_key_(tdb::make_shared(HERE())) { } Status Group::open( @@ -196,7 +195,6 @@ Status Group::open( query_type_ = query_type; is_open_ = true; - changes_applied_ = false; return Status::Ok(); } @@ -250,11 +248,6 @@ Status Group::close() { query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { try { - // If changes haven't been applied, apply them - if (!changes_applied_) { - group_details_->apply_pending_changes(); - changes_applied_ = group_details_->changes_applied(); - } throw_if_not_ok(storage_manager_->group_close_for_writes(this)); } catch (StatusException& exc) { std::string msg = exc.what(); @@ -267,7 +260,8 @@ Status Group::close() { metadata_.clear(); metadata_loaded_ = false; is_open_ = false; - return clear(); + clear(); + return Status::Ok(); } bool Group::is_open() const { @@ -519,9 +513,8 @@ void Group::set_config(Config config) { config_.inherit(config); } -Status Group::clear() { +void Group::clear() { group_details_->clear(); - return Status::Ok(); } void Group::add_member(const shared_ptr group_member) { @@ -607,14 +600,6 @@ URI Group::generate_detail_uri() const { .join_path(ts_name); } -bool Group::changes_applied() const { - return changes_applied_; -} - -void Group::set_changes_applied(const bool changes_applied) { - changes_applied_ = changes_applied; -} - uint64_t Group::member_count() const { std::lock_guard lck(mtx_); // Check if group is open diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index b68f740c9df2..cadbaf667880 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -95,10 +95,8 @@ class Group { /** * Clear a group - * - * @return */ - Status clear(); + void clear(); /** * Deletes data from and closes a group opened in MODIFY_EXCLUSIVE mode. @@ -304,20 +302,6 @@ class Group { */ URI generate_detail_uri() const; - /** - * Have changes been applied to a group in write mode - * @return changes_applied_ - */ - bool changes_applied() const; - - /** - * Set changes applied, only used in serialization - * @param changes_applied should changes be considered to be applied? If so - * then this will enable writes from a deserialized group - * - */ - void set_changes_applied(bool changes_applied); - /** * Get count of members * @@ -441,9 +425,6 @@ class Group { /** Mutex for thread safety. */ mutable std::mutex mtx_; - /* Were changes applied and is a write is required */ - bool changes_applied_; - /* ********************************* */ /* PROTECTED METHODS */ /* ********************************* */ diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index d65df5d2de73..a777732e729d 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -51,8 +51,8 @@ namespace tiledb::sm { GroupDetails::GroupDetails(const URI& group_uri, uint32_t version) : group_uri_(group_uri) - , version_(version) - , changes_applied_(false) { + , is_modified_(false) + , version_(version) { } void GroupDetails::clear() { @@ -61,6 +61,7 @@ void GroupDetails::clear() { members_to_modify_.clear(); member_keys_to_add_.clear(); member_keys_to_delete_.clear(); + is_modified_ = false; } void GroupDetails::add_member(const shared_ptr group_member) { @@ -108,6 +109,7 @@ void GroupDetails::mark_member_for_addition( } members_to_modify_.emplace_back(group_member); + is_modified_ = true; } void GroupDetails::mark_member_for_removal(const std::string& name_or_uri) { @@ -180,6 +182,7 @@ void GroupDetails::mark_member_for_removal(const std::string& name_or_uri) { } members_to_modify_.emplace_back(member_to_delete); + is_modified_ = true; } else { throw GroupDetailsException( "Cannot remove group member " + name_or_uri + @@ -199,10 +202,6 @@ GroupDetails::members() const { return members_; } -void GroupDetails::serialize(Serializer&) { - throw GroupDetailsException("Invalid call to Group::serialize"); -} - std::optional> GroupDetails::deserialize( Deserializer& deserializer, const URI& group_uri) { uint32_t version = 0; @@ -228,10 +227,6 @@ const URI& GroupDetails::group_uri() const { return group_uri_; } -bool GroupDetails::changes_applied() const { - return changes_applied_; -} - uint64_t GroupDetails::member_count() const { std::lock_guard lck(mtx_); diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index 05071d7fcdd6..9e28bf8822d3 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -90,6 +90,25 @@ class GroupDetails { */ const std::vector>& members_to_modify() const; + /** + * Get whether the group has been modified. + * + * This determines whether to write the group details on close. + */ + bool is_modified() const { + return is_modified_; + } + + /** + * Marks the group as modified. + * + * Used only by serialization, to support writing the group details of a + * deserialized group. + */ + void set_modified() { + is_modified_ = true; + } + /** * Get the unordered map of members * @return members @@ -115,10 +134,13 @@ class GroupDetails { /** * Serializes the object members into a binary buffer. * - * @param buff The buffer to serialize the data into. - * @param version The format spec version. + * @param members The members to serialize. Should be retrieved from + * members_to_serialize(). + * @param serializer The buffer to serialize the data into. */ - virtual void serialize(Serializer& serializer); + virtual void serialize( + const std::vector>& members, + Serializer& serializer) const = 0; /** * Returns a Group object from the data in the input binary buffer. @@ -145,12 +167,6 @@ class GroupDetails { /** Returns the group URI. */ const URI& group_uri() const; - /** - * Have changes been applied to a group in write mode - * @return changes_applied_ - */ - bool changes_applied() const; - /** * Get count of members * @@ -184,11 +200,11 @@ class GroupDetails { format_version_t version() const; /** - * Apply any pending member additions or removals - * - * mutates members_ and clears members_to_modify_ + * Get the members to write to storage, after accounting for duplicate members + * and member removals. */ - virtual void apply_pending_changes() = 0; + virtual std::vector> members_to_serialize() + const = 0; protected: /* ********************************* */ @@ -198,6 +214,9 @@ class GroupDetails { /** The group URI. */ URI group_uri_; + /** Whether the group has been modified. */ + bool is_modified_; + /** * The mapping of all members of this group. This is the canonical store of * the group's members. The key is the member's key(). @@ -232,9 +251,6 @@ class GroupDetails { /** Format version. */ const uint32_t version_; - /** Were changes applied and is a write is required */ - bool changes_applied_; - /* ********************************* */ /* PROTECTED METHODS */ /* ********************************* */ diff --git a/tiledb/sm/group/group_details_v1.cc b/tiledb/sm/group/group_details_v1.cc index 06de3313a770..ae5132e092e6 100644 --- a/tiledb/sm/group/group_details_v1.cc +++ b/tiledb/sm/group/group_details_v1.cc @@ -47,12 +47,13 @@ GroupDetailsV1::GroupDetailsV1(const URI& group_uri) // group_member #1 // group_member #2 // ... -void GroupDetailsV1::serialize(Serializer& serializer) { +void GroupDetailsV1::serialize( + const std::vector>& members, + Serializer& serializer) const { serializer.write(GroupDetailsV1::format_version_); - uint64_t group_member_num = members_.size(); - serializer.write(group_member_num); - for (auto& it : members_) { - it.second->serialize(serializer); + serializer.write(members.size()); + for (auto& it : members) { + it->serialize(serializer); } } @@ -70,14 +71,16 @@ shared_ptr GroupDetailsV1::deserialize( return group; } -void GroupDetailsV1::apply_pending_changes() { +std::vector> GroupDetailsV1::members_to_serialize() + const { std::lock_guard lck(mtx_); + decltype(members_) members = members_; // Remove members first for (const auto& member : members_to_modify_) { auto key = member->key(); if (member->deleted()) { - members_.erase(key); + members.erase(key); // Check to remove relative URIs if (key.find(group_uri_.add_trailing_slash().to_string()) != @@ -85,16 +88,19 @@ void GroupDetailsV1::apply_pending_changes() { // Get the substring relative path auto relative_uri = key.substr( group_uri_.add_trailing_slash().to_string().size(), key.size()); - members_.erase(relative_uri); + members.erase(relative_uri); } } else { - members_.emplace(member->key(), member); + members.emplace(member->uri().to_string(), member); } } - changes_applied_ = !members_to_modify_.empty(); - members_to_modify_.clear(); - invalidate_lookups(); + std::vector> result; + result.reserve(members.size()); + for (auto& it : members) { + result.emplace_back(it.second); + } + return result; } } // namespace sm diff --git a/tiledb/sm/group/group_details_v1.h b/tiledb/sm/group/group_details_v1.h index c5881d1c3aa9..b04ed932dc0a 100644 --- a/tiledb/sm/group/group_details_v1.h +++ b/tiledb/sm/group/group_details_v1.h @@ -58,12 +58,9 @@ class GroupDetailsV1 : public GroupDetails { /** Destructor. */ ~GroupDetailsV1() override = default; - /** - * Serializes the object members into a binary buffer. - * - * @param buff The buffer to serialize the data into. - */ - void serialize(Serializer& serializer) override; + void serialize( + const std::vector>& members, + Serializer& serializer) const override; /** * Returns a Group object from the data in the input binary buffer. @@ -75,13 +72,8 @@ class GroupDetailsV1 : public GroupDetails { static shared_ptr deserialize( Deserializer& deserializer, const URI& group_uri); - protected: - /** - * Apply any pending member additions or removals - * - * mutates members_ and clears members_to_modify_; - */ - void apply_pending_changes() override; + std::vector> members_to_serialize() + const override; private: /* Format version for class. */ diff --git a/tiledb/sm/group/group_details_v2.cc b/tiledb/sm/group/group_details_v2.cc index cdc60e51d8ce..b77d8a1cb9a2 100644 --- a/tiledb/sm/group/group_details_v2.cc +++ b/tiledb/sm/group/group_details_v2.cc @@ -47,12 +47,13 @@ GroupDetailsV2::GroupDetailsV2(const URI& group_uri) // group_member #1 // group_member #2 // ... -void GroupDetailsV2::serialize(Serializer& serializer) { +void GroupDetailsV2::serialize( + const std::vector>& members, + Serializer& serializer) const { serializer.write(GroupDetailsV2::format_version_); - uint64_t group_member_num = members_.size(); - serializer.write(group_member_num); - for (auto& it : members_) { - it.second->serialize(serializer); + serializer.write(members.size()); + for (auto& it : members) { + it->serialize(serializer); } } @@ -105,22 +106,23 @@ shared_ptr GroupDetailsV2::deserialize( return group; } -void GroupDetailsV2::apply_pending_changes() { +std::vector> GroupDetailsV2::members_to_serialize() + const { std::lock_guard lck(mtx_); - // In groups V2, we use `members_` to write the list of additions/removals to - // disk. Start with a fresh list. - members_.clear(); - invalidate_lookups(); + decltype(members_) members = members_; - // First add each member to unordered map, overriding if the user adds/removes - // it multiple times - for (auto& it : members_to_modify_) { - members_[it->key()] = it; + // Add each member, overriding if the user adds/removes it multiple times. + for (auto it : members_to_modify_) { + members[it->key()] = it; } - changes_applied_ = !members_to_modify_.empty(); - members_to_modify_.clear(); + std::vector> result; + result.reserve(members.size()); + for (auto& it : members) { + result.emplace_back(it.second); + } + return result; } } // namespace sm diff --git a/tiledb/sm/group/group_details_v2.h b/tiledb/sm/group/group_details_v2.h index 71e70ae9aa43..36ddfd04e6c1 100644 --- a/tiledb/sm/group/group_details_v2.h +++ b/tiledb/sm/group/group_details_v2.h @@ -59,12 +59,9 @@ class GroupDetailsV2 : public GroupDetails { /** Destructor. */ ~GroupDetailsV2() override = default; - /** - * Serializes the object members into a binary buffer. - * - * @param buff The buffer to serialize the data into. - */ - void serialize(Serializer& serializer) override; + void serialize( + const std::vector>& members, + Serializer& serializer) const override; /** * Returns a Group object from the data in the input binary buffer. @@ -87,13 +84,8 @@ class GroupDetailsV2 : public GroupDetails { const std::vector>& deserializer, const URI& group_uri); - protected: - /** - * Apply any pending member additions or removals - * - * mutates members_ and clears members_to_modify_ - */ - void apply_pending_changes() override; + std::vector> members_to_serialize() + const override; private: /* Format version for class. */ diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index 1aac053fa940..3b5d59a27adb 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -197,7 +197,7 @@ Status group_details_from_capnp( group->set_metadata_loaded(true); } - group->set_changes_applied(true); + group->group_details()->set_modified(); return Status::Ok(); } @@ -226,7 +226,7 @@ Status group_from_capnp( } if (group_reader.hasGroup()) { - throw_if_not_ok(group->clear()); + group->clear(); RETURN_NOT_OK(group_details_from_capnp(group_reader.getGroup(), group)); } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ce066fc632ab..ae86f78475aa 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -150,7 +150,7 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { group->group_uri(), *group->encryption_key(), group->unsafe_metadata())); // Store any changes required - if (group->changes_applied()) { + if (group->group_details()->is_modified()) { const URI& group_detail_folder_uri = group->group_detail_uri(); auto group_detail_uri = group->generate_detail_uri(); RETURN_NOT_OK(store_group_detail( @@ -1518,13 +1518,14 @@ Status StorageManagerCanonical::store_group_detail( tdb_shared_ptr group, const EncryptionKey& encryption_key) { // Serialize + auto members = group->members_to_serialize(); SizeComputationSerializer size_computation_serializer; - group->serialize(size_computation_serializer); + group->serialize(members, size_computation_serializer); WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; Serializer serializer(tile.data(), tile.size()); - group->serialize(serializer); + group->serialize(members, serializer); stats()->add_counter("write_group_size", tile.size()); From 84157195d8d2462781599a7facbd53a470cea81d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 24 Nov 2023 10:48:43 +0200 Subject: [PATCH 086/456] Update `install-azurite.sh` to not install Node.js and get the latest version of Azurite. (#4535) Hopefully it fixes CI failures when installing Node.js with apt. All GitHub Actions images contain a recent version of Node.js either way, and we can expect developers to install it themselves if they need to run Azurite. We print a descriptive error message if `npm` is not found, directing to install Node.js. We also remove the pin to specific Azurite versions, matching the behavior on Windows. The Windows script was not updated since it does not pose a problem at the moment. --- TYPE: NO_HISTORY --- scripts/install-azurite.sh | 59 ++++++-------------------------------- 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/scripts/install-azurite.sh b/scripts/install-azurite.sh index 9c56e6e6d407..2567e543ce40 100755 --- a/scripts/install-azurite.sh +++ b/scripts/install-azurite.sh @@ -24,56 +24,13 @@ # SOFTWARE. # -# Installs dependencies for Azurite. +# Installs Azurite. -die() { - echo "$@" 1>&2 ; popd 2>/dev/null; exit 1 -} +# Fail if npm does not exist. +if ! command -v npm &> /dev/null +then + echo "npm could not be found. Please install Node.js." + exit 1 +fi -install_apt_pkgs() { - sudo apt-get -y install libnode-dev node-gyp libssl1.0 npm || die "could not install nodejs dependency" -} - -install_yum_pkgs() { - sudo yum -y install nodejs || die "could not install nodejs dependency" -} - -install_brew_pkgs() { - # Now installed on AZP images by default - if [[ `which node` == "" ]]; then - brew install node || brew link --overwrite node - fi -} - -install_deps() { - AZURITE_PACKAGE="azurite@3.22.0" - if [[ $OSTYPE == linux* ]]; then - if [ -n "$(command -v apt-get)" ]; then - install_apt_pkgs - elif [ -n "$(command -v yum)" ]; then - install_yum_pkgs - fi - - sudo npm config set strict-ssl false - sudo npm cache clean -f - sudo npm install -g n - sudo n stable - sudo npm install -g $AZURITE_PACKAGE - elif [[ $OSTYPE == darwin* ]]; then - if [ -n "$(command -v brew)" ]; then - install_brew_pkgs - else - die "homebrew is not installed!" - fi - - npm install -g $AZURITE_PACKAGE - else - die "unsupported package management system" - fi -} - -run() { - install_deps -} - -run +npm install -g azurite From a84e031242c44d1b8881e5feeec8fb68df250f40 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 24 Nov 2023 11:11:00 +0100 Subject: [PATCH 087/456] Update dev HISTORY with 2.17 and 2.18 content. (#4536) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.17 and 2.18 content. --- HISTORY.md | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index aedec8539b58..530fa5e52361 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,175 @@ +# TileDB v2.18.0 Release Notes + +## Announcements + +* TileDB 2.18, targeted for release in November 2023, includes a preview set of aggregate pushdown APIs. The APIs will be finalized in 2.19 with performance improvements. + +## Disk Format + +* Fix the format specification for group members. [#4380](https://github.com/TileDB-Inc/TileDB/pull/4380) +* Update fragment format spec for info on tile sizes and tile offsets. [#4416](https://github.com/TileDB-Inc/TileDB/pull/4416) + +## Configuration changes + +* Remove vfs.file.max_parallel_ops config option. [#3964](https://github.com/TileDB-Inc/TileDB/pull/3964) + +## Breaking C API changes + +* Behavior breaking change: `tiledb_group_remove_member` cannot remove named members by URI if the URI is duplicated. [#4391](https://github.com/TileDB-Inc/TileDB/pull/4391) + +## New features + +* Support custom headers on s3 requests. [#4400](https://github.com/TileDB-Inc/TileDB/pull/4400) +* Add support for Google Cloud Storage on Windows. [#4138](https://github.com/TileDB-Inc/TileDB/pull/4138) +* Query aggregates REST support. [#4415](https://github.com/TileDB-Inc/TileDB/pull/4415) +* Add random number generator global singleton. [#4437](https://github.com/TileDB-Inc/TileDB/pull/4437) + +## Improvements + +* Add an aspect template argument to `class CAPIFunction`. [#4430](https://github.com/TileDB-Inc/TileDB/pull/4430) +* Fix enumeration value not found error message. [#4339](https://github.com/TileDB-Inc/TileDB/pull/4339) +* Allow forward slashes in enumeration names. [#4454](https://github.com/TileDB-Inc/TileDB/pull/4454) +* Adding extra in memory offset for var tiles on read path. [#4294](https://github.com/TileDB-Inc/TileDB/pull/4294) +* Aggregators: adding mean aggregate. [#4292](https://github.com/TileDB-Inc/TileDB/pull/4292) +* Remove parallel file I/O. [#3964](https://github.com/TileDB-Inc/TileDB/pull/3964) +* C.41 constructor: S3. [#4368](https://github.com/TileDB-Inc/TileDB/pull/4368) +* Changed the logic of the main loop in `Domain::cell_order_cmp_impl`. [#3847](https://github.com/TileDB-Inc/TileDB/pull/3847) +* Add REST client support for delete_fragments and delete_fragments_list. [#3923](https://github.com/TileDB-Inc/TileDB/pull/3923) +* Remove the internal `ValidityVector::init_bytemap` method. [#4071](https://github.com/TileDB-Inc/TileDB/pull/4071) +* Use exceptions for error handling in the Metadata code. [#4012](https://github.com/TileDB-Inc/TileDB/pull/4012) +* Move min/max computation to aggregate_with_count. [#4394](https://github.com/TileDB-Inc/TileDB/pull/4394) +* Add memory tracking to Enumeration loading. [#4395](https://github.com/TileDB-Inc/TileDB/pull/4395) +* Logs for nested exceptions now incorporate all messages. [#4393](https://github.com/TileDB-Inc/TileDB/pull/4393) +* Implement read logging modes. [#4386](https://github.com/TileDB-Inc/TileDB/pull/4386) +* Move counts computation to aggregate_with_count. [#4401](https://github.com/TileDB-Inc/TileDB/pull/4401) +* Fix delete_fragments handlers according to new model. [#4414](https://github.com/TileDB-Inc/TileDB/pull/4414) +* Aggregates: change to use size instead of min cell/max cell. [#4436](https://github.com/TileDB-Inc/TileDB/pull/4436) +* Allow for empty enumerations. [#4423](https://github.com/TileDB-Inc/TileDB/pull/4423) +* Aggregates: combine sum/mean aggregators. [#4440](https://github.com/TileDB-Inc/TileDB/pull/4440) + +### Documentation + +* Update docs for set data buffer APIs. [#4381](https://github.com/TileDB-Inc/TileDB/pull/4381) + +## Defects removed + +* Fix serialization of empty enumerations. [#4472](https://github.com/TileDB-Inc/TileDB/pull/4472) +* Patch curl to fix CVE-2023-38545. [#4408](https://github.com/TileDB-Inc/TileDB/pull/4408) +* Update libwebp to version 1.3.2 to fix CVE-2023-5129. [#4384](https://github.com/TileDB-Inc/TileDB/pull/4384) +* Fix missing WebP::sharpyuv installation command. [#4325](https://github.com/TileDB-Inc/TileDB/pull/4325) +* Fix re-loading enumerations from the REST server. [#4335](https://github.com/TileDB-Inc/TileDB/pull/4335) +* Save error on the context before returning error code in vfs_open. [#4347](https://github.com/TileDB-Inc/TileDB/pull/4347) +* Filter pipeline: fix coord filter type. [#4351](https://github.com/TileDB-Inc/TileDB/pull/4351) +* Fix creation of Enumerations with `std::vector`. [#4362](https://github.com/TileDB-Inc/TileDB/pull/4362) +* Revert change to S3 is_object, is_bucket checks. [#4370](https://github.com/TileDB-Inc/TileDB/pull/4370) +* Specify whether the HDFS backend is enabled in the string returned by `tiledb_as_built_dump`. [#4364](https://github.com/TileDB-Inc/TileDB/pull/4364) +* Fix fragment corruption in the GlobalOrderWriter. [#4383](https://github.com/TileDB-Inc/TileDB/pull/4383) +* Fix bug in FragmentMetadata::get_footer_size. [#4212](https://github.com/TileDB-Inc/TileDB/pull/4212) +* Global order writer: don't try to delete the current fragment if it had failed to be created. [#4052](https://github.com/TileDB-Inc/TileDB/pull/4052) +* Fix bug in ordered label reader for unsigned integers. [#4404](https://github.com/TileDB-Inc/TileDB/pull/4404) +* Add missing include object.h in group_experimental.h. [#4411](https://github.com/TileDB-Inc/TileDB/pull/4411) +* Support Azure Storage SAS tokens that don't start with a question mark. [#4418](https://github.com/TileDB-Inc/TileDB/pull/4418) +* Avoid empty Enumeration REST requests. [#4369](https://github.com/TileDB-Inc/TileDB/pull/4369) +* Make the `initializer_list` overload of `Subarray::set_subarray` zero-copy. [#4062](https://github.com/TileDB-Inc/TileDB/pull/4062) + +## API changes + +### C API + +* New: Aggregate pushdown C API. [#4176](https://github.com/TileDB-Inc/TileDB/pull/4176) +* Implement tiledb_handle_load_array_schema_request. [#4399](https://github.com/TileDB-Inc/TileDB/pull/4399) +* Add ArraySchemaEvolution::extend_enumeration. [#4445](https://github.com/TileDB-Inc/TileDB/pull/4445) +* Allow null cfg in group consolidate_metadata and vacuum_metadata. [#4412](https://github.com/TileDB-Inc/TileDB/pull/4412) + +### C++ API + +* Fix UBSAN error in the Enumeration CPP API. [#4357](https://github.com/TileDB-Inc/TileDB/pull/4357) +* Remove default operation type from query condition set membership API (introduced in 2.17). [#4390](https://github.com/TileDB-Inc/TileDB/pull/4390) +* Fix CPP set buffer APIs to work with aggregates. [#4434](https://github.com/TileDB-Inc/TileDB/pull/4434) +* Adding CPP example for consolidation plan. [#4433](https://github.com/TileDB-Inc/TileDB/pull/4433) +* New: Aggregate API for MEAN and NULL_COUNT capi/cppapi wraps. [#4443](https://github.com/TileDB-Inc/TileDB/pull/4443) + +## Build System Changes + +* Fix static linking failures and build ExampleExe_static on Windows. [#4336](https://github.com/TileDB-Inc/TileDB/pull/4336) +* Run Windows nightly debug build on C:/ drive to avoid out of space errors. [#4352](https://github.com/TileDB-Inc/TileDB/pull/4352) +* Manage dependencies with vcpkg by default. [#4348](https://github.com/TileDB-Inc/TileDB/pull/4348) +* Find the OpenSSL dependency of the AWS SDK on non-Windows only. [#4359](https://github.com/TileDB-Inc/TileDB/pull/4359) +* Use generator expressions when setting compile options for the benchmarks. [#4328](https://github.com/TileDB-Inc/TileDB/pull/4328) +* Bump default compiler to GCC 10 and macos images from 11 to 12. [#4367](https://github.com/TileDB-Inc/TileDB/pull/4367) +* Fix and optimize the Release CI workflow. [#4373](https://github.com/TileDB-Inc/TileDB/pull/4373) +* Stop building gRPC when GCS is enabled. [#4435](https://github.com/TileDB-Inc/TileDB/pull/4435) +* Require only ucrt build from Rtools. [#4361](https://github.com/TileDB-Inc/TileDB/pull/4361) +* Update the AWS SDK to version 1.11.160. [#4214](https://github.com/TileDB-Inc/TileDB/pull/4214) +* Remove warnings from benchmarks. [#4396](https://github.com/TileDB-Inc/TileDB/pull/4396) +* Update `clang-format` to version 16. [#4397](https://github.com/TileDB-Inc/TileDB/pull/4397) +* Update the Google Cloud Storage SDK to its latest version (2.15.1). [#4031](https://github.com/TileDB-Inc/TileDB/pull/4031) +* Update capnproto to version 1.0.1. [#4405](https://github.com/TileDB-Inc/TileDB/pull/4405) +* Fix compilation with GCC 13. [#4331](https://github.com/TileDB-Inc/TileDB/pull/4331) +* Compilation fix for GCC 9.4 of Ubuntu 20.04. [#4355](https://github.com/TileDB-Inc/TileDB/pull/4355) +* Don't error out if DELETE symbol is defined. [#4365](https://github.com/TileDB-Inc/TileDB/pull/4365) + +## Test only changes + +* Aggregates pushdown: adding tests for var size buffer overflow. [#4315](https://github.com/TileDB-Inc/TileDB/pull/4315) +* Aggregates pushdown: adding incomplete tests. [#4301](https://github.com/TileDB-Inc/TileDB/pull/4301) +* Add group backwards compatibility tests. [#3996](https://github.com/TileDB-Inc/TileDB/pull/3996) +* Aggregates: Adding tests for computation policies. [#4421](https://github.com/TileDB-Inc/TileDB/pull/4421) +* Aggregators: adding benchmark for aggregate_with_count class. [#4420](https://github.com/TileDB-Inc/TileDB/pull/4420) + +## Experimental Changes + +* Add support for basic mimo node to TileDB task graph. [#3977](https://github.com/TileDB-Inc/TileDB/pull/3977) +* Verify that dependency-only edges using `std::monostate` work with nodes and edges. [#3939](https://github.com/TileDB-Inc/TileDB/pull/3939) + +# TileDB v2.17.4 Release Notes + +## Defects removed + +* Fix serialization of empty enumerations. [#4472](https://github.com/TileDB-Inc/TileDB/pull/4472) +* Update libwebp to version 1.3.2 to fix CVE-2023-5129. [#4384](https://github.com/TileDB-Inc/TileDB/pull/4384) + +# TileDB v2.17.3 Release Notes + +## Improvements + +* Allow for empty enumerations [#4423](https://github.com/TileDB-Inc/TileDB/pull/4423) + +## API changes + +* Add `ArraySchemaEvolution::extend_enumeration` [#4445](https://github.com/TileDB-Inc/TileDB/pull/4445) + +# TileDB v2.17.2 Release Notes + +## Defects removed + +* Fix fragment corruption in the GlobalOrderWriter. [#4383](https://github.com/TileDB-Inc/TileDB/pull/4383) +* Prepend a question mark to the Azure Storage SAS token if it does not begin with one. [#4426](https://github.com/TileDB-Inc/TileDB/pull/4426) + +# TileDB v2.17.1 Release Notes + +## Improvements + +* Avoid empty Enumeration REST requests. [#4369](https://github.com/TileDB-Inc/TileDB/pull/4369) + +## Defects removed + +* Revert change to S3 is_object, is_bucket checks. [#4371](https://github.com/TileDB-Inc/TileDB/pull/4371) + +## API changes + +### C++ API + +* Add support for `std::vector` in Enumeration CPP API. [#4362](https://github.com/TileDB-Inc/TileDB/pull/4362) +* Fix UBSAN error in the Enumeration CPP API. [#4357](https://github.com/TileDB-Inc/TileDB/pull/4357) + * This is a change from 2.17.0 in the header-only C++ API. +* Don't error out if DELETE symbol is defined. [#4365](https://github.com/TileDB-Inc/TileDB/pull/4365) + +## Build changes + +* Compilation fix for GCC 9.4 of Ubuntu 20.04. [#4355](https://github.com/TileDB-Inc/TileDB/pull/4355) +* Fix static linking failures and build ExampleExe_static on Windows. [#4336](https://github.com/TileDB-Inc/TileDB/pull/4336) + # TileDB v2.17.0 Release Notes ## Disk Format From 217be37a457b4c7e2b8086ca3315dc0085d6aa40 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 27 Nov 2023 11:20:57 +0100 Subject: [PATCH 088/456] Bump dev version to 2.19.0 (#4539) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.19.0 --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index 1f450ccbe49a..97cdc8bf6fc5 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.18' +version = '2.19' # The full version, including alpha/beta/rc tags. -release = '2.18.0' +release = '2.19.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index 872faf93da79..f845c3a1b09a 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 18 +#define TILEDB_VERSION_MINOR 19 #define TILEDB_VERSION_PATCH 0 From 2b1ada70d047a142cd28ca0182902e77dc75b51e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 27 Nov 2023 22:54:43 +0100 Subject: [PATCH 089/456] Fix delta filter deserialization for format ver 19. (#4541) This fixes the delta filter deserialization for version 19. In that version, we already included the extra reinterpret type. --- TYPE: BUG DESC: Fix delta filter deserialization for format ver 19. --- tiledb/sm/filter/filter_create.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/filter/filter_create.cc b/tiledb/sm/filter/filter_create.cc index 7d424c0c0c1c..588c66de1a73 100644 --- a/tiledb/sm/filter/filter_create.cc +++ b/tiledb/sm/filter/filter_create.cc @@ -126,8 +126,8 @@ shared_ptr tiledb::sm::FilterCreate::deserialize( uint8_t compressor_char = deserializer.read(); int compression_level = deserializer.read(); Datatype reinterpret_type = Datatype::ANY; - if (version >= 20 && (filtertype == FilterType::FILTER_DELTA || - filtertype == FilterType::FILTER_DOUBLE_DELTA)) { + if ((version >= 20 && filtertype == FilterType::FILTER_DOUBLE_DELTA) || + (version >= 19 && filtertype == FilterType::FILTER_DELTA)) { uint8_t reinterpret = deserializer.read(); reinterpret_type = static_cast(reinterpret); } From c72013dd41b916b1e357eed86ed4618dd7a607c4 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 28 Nov 2023 04:41:41 -0500 Subject: [PATCH 090/456] Add new timestamped_name file to the format spec (#4533) Add a `timestamped_name` file to the format spec and link to it elsewhere in the spec. --- TYPE: FORMAT DESC: Add timestamped_name file to the Format Spec --- format_spec/FORMAT_SPEC.md | 1 + format_spec/array_file_hierarchy.md | 20 +++++++------------ format_spec/array_schema.md | 8 ++------ format_spec/consolidated_commits_file.md | 8 +------- .../consolidated_fragment_metadata_file.md | 8 +------- format_spec/delete_commit_file.md | 8 +------- format_spec/fragment.md | 13 ++++-------- format_spec/group.md | 12 ++++++----- format_spec/ignore_file.md | 12 ++--------- format_spec/metadata.md | 13 ++++-------- format_spec/timestamped_name.md | 19 ++++++++++++++++++ format_spec/update_commit_file.md | 8 +------- format_spec/vacuum_file.md | 20 +++++++------------ 13 files changed, 57 insertions(+), 93 deletions(-) create mode 100644 format_spec/timestamped_name.md diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index 1039fb183272..c4def7a9f782 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -51,4 +51,5 @@ title: Format Specification * **Other** * [Consolidated Fragment Metadata File](./consolidated_fragment_metadata_file.md) * [Filter Pipeline](./filter_pipeline.md) + * [Timestamped Name](./timestamped_name.md) * [Vacuum Pipeline](./vacuum_file.md) diff --git a/format_spec/array_file_hierarchy.md b/format_spec/array_file_hierarchy.md index ca32b290a441..4c85cfa77d7b 100644 --- a/format_spec/array_file_hierarchy.md +++ b/format_spec/array_file_hierarchy.md @@ -30,21 +30,15 @@ my_array # array folder ``` -A `` above has format `__t1_t2_uuid_v`, where - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - Inside the array folder, you can find the following: * [Array schema](./array_schema.md) folder `__schema`. -* Inside of a fragments folder, any number of [fragment folders](./fragment.md) ``. -* Inside of a commit folder, an empty file `.wrt` associated with every fragment folder ``, where `` is common for the folder and the WRT file. This is used to indicate that fragment `` has been *committed* (i.e., its write process finished successfully) and it is ready for use by TileDB. If the WRT file does not exist, the corresponding fragment folder is ignored by TileDB during the reads. -* Inside the same commit folder, any number of [delete commit files](./delete_commit_file.md) of the form `.del`. -* Inside the same commit folder, any number of [update commit files](./update_commit_file.md) of the form `.upd`. -* Inside the same commit folder, any number of [consolidated commits files](./consolidated_commits_file.md) of the form `.con`. -* Inside the same commit folder, any number of [ignore files](./ignore_file.md) of the form `.ign`. -* Inside of a fragment metadata folder, any number of [consolidated fragment metadata files](./consolidated_fragment_metadata_file.md) of the form `.meta`. +* Inside of a fragments folder, any number of [fragment folders](./fragment.md) [``](./timestamped_name.md). +* Inside of a commit folder, an empty file [``](./timestamped_name.md)`.wrt` associated with every fragment folder [``](./timestamped_name.md), where [``](./timestamped_name.md) is common for the folder and the WRT file. This is used to indicate that fragment [``](./timestamped_name.md) has been *committed* (i.e., its write process finished successfully) and it is ready for use by TileDB. If the WRT file does not exist, the corresponding fragment folder is ignored by TileDB during the reads. +* Inside the same commit folder, any number of [delete commit files](./delete_commit_file.md) of the form [``](./timestamped_name.md)`.del`. +* Inside the same commit folder, any number of [update commit files](./update_commit_file.md) of the form [``](./timestamped_name.md)`.upd`. +* Inside the same commit folder, any number of [consolidated commits files](./consolidated_commits_file.md) of the form [``](./timestamped_name.md)`.con`. +* Inside the same commit folder, any number of [ignore files](./ignore_file.md) of the form [``](./timestamped_name.md)`.ign`. +* Inside of a fragment metadata folder, any number of [consolidated fragment metadata files](./consolidated_fragment_metadata_file.md) of the form [``](./timestamped_name.md)`.meta`. * [Array metadata](./metadata.md) folder `__meta`. * Inside of a labels folder, additional TileDB arrays storing dimension label data. \ No newline at end of file diff --git a/format_spec/array_schema.md b/format_spec/array_schema.md index 6e7c55481c7b..108edbc857f3 100644 --- a/format_spec/array_schema.md +++ b/format_spec/array_schema.md @@ -14,14 +14,10 @@ my_array # array folder |_ ... ``` -`` has format `__timestamp_timestamp_uuid`, where: -* `timestamp` is timestamp in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier - - The array schema folder can contain: -* Any number of [array schema files](#array-schema-file) +* Any number of [array schema files](#array-schema-file) with name [``](./timestamped_name.md). + * Note: the name does _not_ include the format version. ## Previous Array Schema Version diff --git a/format_spec/consolidated_commits_file.md b/format_spec/consolidated_commits_file.md index ec3bfe42e430..eaaed6e97654 100644 --- a/format_spec/consolidated_commits_file.md +++ b/format_spec/consolidated_commits_file.md @@ -2,7 +2,7 @@ title: Consolidated Commits File --- -A consolidated commits file has name `.con` and is located here: +A consolidated commits file has name [``](./timestamped_name.md)`.con` and is located here: ``` my_array # array folder @@ -12,12 +12,6 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - There may be multiple such files in the array commits folder. Each consolidated commits file combines a list of fragments commits, delete or update commits that will be considered together when opening an array. They are timestamped (with the smaller/bigger timestamp from contained elements) so that they might be filtered if the array open start/end time don't intersect anything inside of the file, but any file where there is any intersection will be loaded. There are two situations when these files are added. The first one is when a user decides to improve the open performance of their array (by reducing the listing size) through commits consolidation. The second one is when a user has specified a maximum fragment size for consolidation and multiple fragments needed to be committed at once, this file format will be used so that an atomic file system operation can be performed to do so. When fragments that are included inside of a consolidated file are removed, either through vacuuming or other fragment deletion, an [ignore file](./ignore_file.md) needs to be written so that those commits file can be ignored when opening an array and not cause unnecessary file system operations. diff --git a/format_spec/consolidated_fragment_metadata_file.md b/format_spec/consolidated_fragment_metadata_file.md index 79ed5f537936..4802e25efa06 100644 --- a/format_spec/consolidated_fragment_metadata_file.md +++ b/format_spec/consolidated_fragment_metadata_file.md @@ -2,7 +2,7 @@ title: Consolidated Fragment Metadata File --- -A consolidated fragment metadata file has name `.meta` and is located here: +A consolidated fragment metadata file has name [``](./timestamped_name.md)`.meta` and is located here: ``` my_array # array folder @@ -12,12 +12,6 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - There may be multiple such files in the array folder. Each consolidated fragment metadata file combines the metadata footers of a set of fragments. It has the following on-disk format: | **Field** | **Type** | **Description** | diff --git a/format_spec/delete_commit_file.md b/format_spec/delete_commit_file.md index 825f5c53086e..d824063c1b9d 100644 --- a/format_spec/delete_commit_file.md +++ b/format_spec/delete_commit_file.md @@ -2,7 +2,7 @@ title: Delete Commit File --- -A delete commit file has name `.del` and is located here: +A delete commit file has name [``](./timestamped_name.md)`.del` and is located here: ``` my_array # array folder @@ -12,12 +12,6 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - There may be multiple such files in the array commits folder. Each delete commit file contains a [tile](./tile.md) with a serialized delete condition, which is a tree of nodes. Each node can be a value node or expression node. Expression nodes have the following on disk format: | **Field** | **Type** | **Description** | diff --git a/format_spec/fragment.md b/format_spec/fragment.md index c64e3bdceed7..eddeb2649991 100644 --- a/format_spec/fragment.md +++ b/format_spec/fragment.md @@ -4,7 +4,7 @@ title: Fragment ## Main Structure -A fragment metadata folder is called `` and located here: +A fragment metadata folder is called [``](./timestamped_name.md)` and located here: ``` my_array # array folder @@ -31,11 +31,6 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - There can be any number of fragments in an array. The fragment folder contains: * A single [fragment metadata file](#fragment-metadata-file) named `__fragment_metadata.tdb`. @@ -75,9 +70,9 @@ The fragment metadata file has the following on-disk format: | Variable sums for attribute/dimension N | [Tile Sums](#tile-sums) | The serialized sums for attribute/dimension N | | Tile null counts for attribute/dimension 1 | [Tile Null Count](#tile-null-count) | The serialized null counts for attribute/dimension 1 | | … | … | … | -| Variable maxes for attribute/dimension N | [[Tile Null Count](#tile-null-count) | The serialized null counts for attribute/dimension N | -| Fragment min, max, sum, null count | [[Tile Fragment Min Max Sum Null Count](#tile-fragment-min-max-sum-null-count) | The serialized fragment min max sum null count | -| Processed conditions | [[Tile Processed Conditions](#tile-processed-conditions) | The serialized processed conditions | +| Variable maxes for attribute/dimension N | [Tile Null Count](#tile-null-count) | The serialized null counts for attribute/dimension N | +| Fragment min, max, sum, null count | [Tile Fragment Min Max Sum Null Count](#tile-fragment-min-max-sum-null-count) | The serialized fragment min max sum null count | +| Processed conditions | [Tile Processed Conditions](#tile-processed-conditions) | The serialized processed conditions | | Metadata footer | [Footer](#footer) | Basic metadata gathered in the footer | ### R-Tree diff --git a/format_spec/group.md b/format_spec/group.md index be2882909135..7b438c0dc26c 100644 --- a/format_spec/group.md +++ b/format_spec/group.md @@ -1,17 +1,19 @@ # Group -A group consists of [metadata](./metadata.md) and a file containing group members. - -The current group format version is `2`. +The current group format version (`2`) is a folder called `__group` located here: ``` my_group # Group folder - |_ __tiledb_group.tdb # Empty group file + | ... |_ __group # Group folder |_ # Timestamped group file detailing members - |_ __meta # group metadata folder + |_ ... ``` +The group folder can contain: + +* Any number of group files with name [``](./timestamped_name.md). + ## Group File diff --git a/format_spec/ignore_file.md b/format_spec/ignore_file.md index 0849962bef77..336121a0a39e 100644 --- a/format_spec/ignore_file.md +++ b/format_spec/ignore_file.md @@ -2,23 +2,15 @@ title: Ignore File --- -A ignore file has name `__t1_t2_uuid_v.ign` and is located in the array commit folder: +A ignore file has name [``](./timestamped_name.md)`.ign` and is located in the array commit or metadata folder: ``` my_array # array folder |_ .... |_ __commits # array commit folder - |___t1_t2_uuid_v.ign # ignore file + |_ .ign # ignore file ``` -or in the array metadata folder: - -In the file name: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - The ignore file is a simple text file where each line contains a URI string. The URI is the relative URI based on the top level array URI. Ignored files are used to specify commit files that should be ignored. The timestamps for the file is used to identify the timestamp range that is included inside of the file (with the smaller/bigger timestamp from contained elements). That way if an array is opened at a time that doesn't intersect the timestamp range of the file, it can be omitted. All ignore files that intersect the array open start/end times will be loaded when the array is opened. There are a few situations where those files are required, all linked to [consolidated commits files](./consolidated_commits_file.md). One of them is when fragments are vacuumed after consolidation. If any fragments were included a consolidated commits file, they now should be added to an ignore file so that the system doesn't try to load that fragments. Another is when a fragment that is included in a consolidated commits file is deleted, it needs to be added to an ignore file so that the array can still be opened. diff --git a/format_spec/metadata.md b/format_spec/metadata.md index 426e2dda73dd..bf2ec7153fd8 100644 --- a/format_spec/metadata.md +++ b/format_spec/metadata.md @@ -16,15 +16,10 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - -The metadata folder can contain: -* Any number of [metadata files](#array-metadata-file) -* Any number of [vacuum files](./vacuum_file.md) +The metadata folder can contain any number of [timestamped](./timestamped_name.md): +* [metadata files](#array-metadata-file) +* [vacuum files](./vacuum_file.md) +* Note: the timestamped names do _not_ include the format version. ## Metadata File diff --git a/format_spec/timestamped_name.md b/format_spec/timestamped_name.md new file mode 100644 index 000000000000..35742c7cc961 --- /dev/null +++ b/format_spec/timestamped_name.md @@ -0,0 +1,19 @@ +--- +title: Timestamped Name +--- + +A TileDB object may contain files with format `.ext`, as seen in [array_file_hierarchy.md](./array_file_hierarchy.md), for example. + +A `` has format `__t1_t2_uuid[_v]`, where: + +* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) +* `uuid` is a unique identifier +* `v` is the _optional_ format version, as indicated by `[]` + +## Format History +_Note_: The presence of `[]` is indicative of an optional parameter. +| Format version | TileDB version | Timestamped name format | +| :-: | :-: | :-: | +| 1 - 2 | 1.4 - 1.5 | `__uuid_t1[_t2]` | +| 3 - 4 | 1.6 - 1.7 | `__t1_t2_uuid` | +| 5+ | 2.0+ | `__t1_t2_uuid_[v]` | diff --git a/format_spec/update_commit_file.md b/format_spec/update_commit_file.md index cde6adae6219..87d0b8382ffb 100644 --- a/format_spec/update_commit_file.md +++ b/format_spec/update_commit_file.md @@ -2,7 +2,7 @@ title: Update Commit File --- -A update commit file has name `.upd` and is located here: +A update commit file has name [``](./timestamped_name.md)`.upd` and is located here: ``` my_array # array folder @@ -12,12 +12,6 @@ my_array # array folder |_ ... ``` -`` has format `__t1_t2_uuid_v`, where: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version - There may be multiple such files in the array commits folder. Each update commit file contains a [tile](./tile.md) with a serialized update condition, which is a tree of nodes followed by update values, which is a list of values. Each node for the condition can be a value node or expression node. Expression nodes have the following on disk format: | **Field** | **Type** | **Description** | diff --git a/format_spec/vacuum_file.md b/format_spec/vacuum_file.md index ec58664bcfb5..280e8544dedb 100644 --- a/format_spec/vacuum_file.md +++ b/format_spec/vacuum_file.md @@ -2,13 +2,13 @@ title: Vacuum File --- -A vacuum file has name `__t1_t2_uuid_v.vac` and can be located either in the array commit folder: +A vacuum file has name `[``](./timestamped_name.md)`.vac` and can be located either in the array commit folder: ``` my_array # array folder |_ .... |_ __commits # array commit folder - |___t1_t2_uuid_v.vac # vacuum file + |_ .vac # vacuum file ``` or in the array metadata folder: @@ -16,19 +16,13 @@ or in the array metadata folder: ``` my_array # array folder | ... - |_ __meta # array metadata folder - |_ ... - |_ __t1_t2_uuid_v.vac # vacuum file - |_ ... + | __meta # array metadata folder + | ... + | .vac # vacuum file + | ... ``` -When located in the commits folder, it will include the URI of fragments (in the __fragments folder) that can be vaccumed. When located in the array metadata folder, it will include the URI or array metadata files that can be vaccumed. - -In the file name: - -* `t1` and `t2` are timestamps in milliseconds elapsed since 1970-01-01 00:00:00 +0000 (UTC) -* `uuid` is a unique identifier -* `v` is the format version +When located in the commits folder, it will include the URI of fragments (in the `__fragments` folder) that can be vaccumed. When located in the array metadata folder, it will include the URI or array metadata files that can be vaccumed. The vacuum file is a simple text file where each line contains a URI string: From aea92ab84cdeffd936a9ec88f4dc228c53a8c34a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 29 Nov 2023 02:38:00 -0500 Subject: [PATCH 091/456] Add v2_18_1 arrays to backward compatability matrix (#4546) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 2936492f723c..114bfd4a084e 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0", "v2_17_3"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0", "v2_17_3", "v2_18_1"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From e1929edfa745ab1c46b8fafcbcb56e1032a4ff19 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Thu, 30 Nov 2023 10:37:02 +0200 Subject: [PATCH 092/456] Consolidation plan REST support - part1: handler (#4534) This is adding support for serializing the Consolidation Plan via REST, in order to make it usable with TileDB Cloud. This first part is adding the C-API handler Cloud needs to call from Go to implement this feature. Part 2 that integrates with the rest client will follow as a separate PR. Design doc: https://www.notion.so/Consolidation-Plan-Request-Handler-d25fb00674444bcdaed74a8e46761b97 --- TYPE: IMPROVEMENT DESC: Consolidation plan REST support - part1: handler --- test/src/test-capi-consolidation-plan.cc | 4 +- test/src/test-cppapi-consolidation-plan.cc | 76 +++- test/src/unit-request-handlers.cc | 62 +++ tiledb/sm/c_api/tiledb.cc | 48 +- tiledb/sm/c_api/tiledb_serialization.h | 19 + .../consolidation_plan/consolidation_plan.cc | 8 + .../consolidation_plan/consolidation_plan.h | 12 +- tiledb/sm/serialization/consolidation.cc | 255 +++++++++++ tiledb/sm/serialization/consolidation.h | 49 +++ tiledb/sm/serialization/tiledb-rest.capnp | 13 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 151 +++++++ tiledb/sm/serialization/tiledb-rest.capnp.h | 413 ++++++++++++++++++ 12 files changed, 1088 insertions(+), 22 deletions(-) diff --git a/test/src/test-capi-consolidation-plan.cc b/test/src/test-capi-consolidation-plan.cc index 8e7337621438..7efd70b471a7 100644 --- a/test/src/test-capi-consolidation-plan.cc +++ b/test/src/test-capi-consolidation-plan.cc @@ -215,7 +215,7 @@ TEST_CASE_METHOD( ctx_.ptr().get(), consolidation_plan, 0, &num_fragments)); CHECK(num_fragments == 11); check_last_error( - "Error: ConsolidationPlan: Trying to access a node that doesn't exists"); + "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); const char* frag_uri = nullptr; CHECK( @@ -223,7 +223,7 @@ TEST_CASE_METHOD( ctx_.ptr().get(), consolidation_plan, 0, 0, &frag_uri)); CHECK(frag_uri == nullptr); check_last_error( - "Error: ConsolidationPlan: Trying to access a node that doesn't exists"); + "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); tiledb_consolidation_plan_free(&consolidation_plan); } diff --git a/test/src/test-cppapi-consolidation-plan.cc b/test/src/test-cppapi-consolidation-plan.cc index 5379581d9dde..4a23ec489392 100644 --- a/test/src/test-cppapi-consolidation-plan.cc +++ b/test/src/test-cppapi-consolidation-plan.cc @@ -31,8 +31,13 @@ */ #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/buffer/buffer_api_internal.h" +#include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/consolidation.h" #include @@ -46,6 +51,7 @@ struct CppConsolidationPlanFx { // TileDB context. Context ctx_; VFS vfs_; + Config cfg_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; @@ -67,15 +73,20 @@ struct CppConsolidationPlanFx { bool is_array(const std::string& array_name); void check_last_error(std::string expected); void validate_plan( + uint64_t fragment_size, + const Array& array, ConsolidationPlan& plan, std::vector> expected_plan); + tiledb::sm::ConsolidationPlan call_handler( + uint64_t fragment_size, + const Array& array, + tiledb::sm::SerializationType stype); }; CppConsolidationPlanFx::CppConsolidationPlanFx() : vfs_(ctx_) { - Config config; - config.set("sm.consolidation.buffer_size", "1000"); - ctx_ = Context(config); + cfg_.set("sm.consolidation.buffer_size", "1000"); + ctx_ = Context(cfg_); vfs_ = VFS(ctx_); remove_sparse_array(); @@ -188,9 +199,44 @@ void CppConsolidationPlanFx::check_last_error(std::string expected) { CHECK(msg == expected); } +tiledb::sm::ConsolidationPlan CppConsolidationPlanFx::call_handler( + uint64_t fragment_size, + const Array& array, + tiledb::sm::SerializationType stype) { + auto req_buf = tiledb_buffer_handle_t::make_handle(); + auto resp_buf = tiledb_buffer_handle_t::make_handle(); + + tiledb::sm::serialization::serialize_consolidation_plan_request( + fragment_size, cfg_.ptr()->config(), stype, req_buf->buffer()); + auto rval = tiledb_handle_consolidation_plan_request( + ctx_.ptr().get(), + array.ptr().get(), + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval == TILEDB_OK); + + auto fragments_per_node = + tiledb::sm::serialization::deserialize_consolidation_plan_response( + stype, resp_buf->buffer()); + // construct consolidation plan from the members we got from serialization + return tiledb::sm::ConsolidationPlan(fragment_size, fragments_per_node); +} + void CppConsolidationPlanFx::validate_plan( + [[maybe_unused]] uint64_t fragment_size, + [[maybe_unused]] const Array& array, ConsolidationPlan& plan, std::vector> expected_plan) { +#ifdef TILEDB_SERIALIZATION + auto stype = GENERATE( + tiledb::sm::SerializationType::JSON, + tiledb::sm::SerializationType::CAPNP); + auto deserialized_plan = call_handler(fragment_size, array, stype); + // Now the two plans should be exactly the same. + REQUIRE(plan.dump() == deserialized_plan.dump()); +#endif + // Take all the nodes in the plan, make a string out of them, the string will // be the sorted fragment URIs. std::vector string_plan(plan.num_nodes()); @@ -247,11 +293,11 @@ TEST_CASE_METHOD( CHECK_THROWS_WITH( consolidation_plan.num_fragments(0), - "Error: ConsolidationPlan: Trying to access a node that doesn't exists"); + "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); CHECK_THROWS_WITH( consolidation_plan.fragment_uri(0, 0), - "Error: ConsolidationPlan: Trying to access a node that doesn't exists"); + "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); } TEST_CASE_METHOD( @@ -286,7 +332,7 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 1); // Validate the plan. - validate_plan(consolidation_plan, {{uri1, uri2}}); + validate_plan(1, array, consolidation_plan, {{uri1, uri2}}); } TEST_CASE_METHOD( @@ -312,7 +358,7 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 1); // Validate the plan. - validate_plan(consolidation_plan, {{uri1, uri2}, {uri3, uri4}}); + validate_plan(1, array, consolidation_plan, {{uri1, uri2}, {uri3, uri4}}); } TEST_CASE_METHOD( @@ -338,7 +384,7 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 1); // Validate the plan. - validate_plan(consolidation_plan, {{uri1, uri2, uri3}}); + validate_plan(1, array, consolidation_plan, {{uri1, uri2, uri3}}); } TEST_CASE_METHOD( @@ -361,7 +407,7 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 10 * 1024); // Validate the plan. - validate_plan(consolidation_plan, {{uri1}}); + validate_plan(10 * 1024, array, consolidation_plan, {{uri1}}); } TEST_CASE_METHOD( @@ -380,7 +426,7 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 100 * 1024); // Validate the plan. - validate_plan(consolidation_plan, {{uri1, uri2}}); + validate_plan(100 * 1024, array, consolidation_plan, {{uri1, uri2}}); } TEST_CASE_METHOD( @@ -409,7 +455,7 @@ TEST_CASE_METHOD( // Validate the plan, we should only have a node for the large fragment to be // split. - validate_plan(consolidation_plan, {{uri2}}); + validate_plan(100 * 1024, array, consolidation_plan, {{uri2}}); } TEST_CASE_METHOD( @@ -438,7 +484,7 @@ TEST_CASE_METHOD( // Validate the plan, we should only have a node for the large fragment to be // split. - validate_plan(consolidation_plan, {{uri1, uri3}, {uri2}}); + validate_plan(100 * 1024, array, consolidation_plan, {{uri1, uri3}, {uri2}}); } TEST_CASE_METHOD( @@ -486,5 +532,9 @@ TEST_CASE_METHOD( ConsolidationPlan consolidation_plan(ctx_, array, 100 * 1024); // Validate the plan. - validate_plan(consolidation_plan, {{uri1, uri2, uri3}, {uri6, uri7}, {uri5}}); + validate_plan( + 100 * 1024, + array, + consolidation_plan, + {{uri1, uri2, uri3}, {uri6, uri7}, {uri5}}); } diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index d7555ec7dc4b..7014b5630957 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -88,6 +88,23 @@ struct HandleQueryPlanRequestFx : RequestHandlerFx { std::string call_handler(SerializationType stype, Query& query); }; +struct HandleConsolidationPlanRequestFx : RequestHandlerFx { + HandleConsolidationPlanRequestFx() + : RequestHandlerFx("consolidation_plan_handler") { + } + + virtual shared_ptr create_schema() override { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + return schema; + }; +}; + /* ********************************* */ /* Testing Array Schema Loading */ /* ********************************* */ @@ -266,6 +283,51 @@ TEST_CASE_METHOD( REQUIRE(rval != TILEDB_OK); } +TEST_CASE_METHOD( + HandleConsolidationPlanRequestFx, + "tiledb_handle_consolidation_plan_request - error checks", + "[request_handler][consolidation-plan][errors]") { + create_array(); + + auto ctx = tiledb::Context(); + auto array = tiledb::Array(ctx, uri_.to_string(), TILEDB_READ); + auto stype = TILEDB_CAPNP; + auto req_buf = tiledb_buffer_handle_t::make_handle(); + auto resp_buf = tiledb_buffer_handle_t::make_handle(); + + auto rval = tiledb_handle_consolidation_plan_request( + nullptr, + array.ptr().get(), + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_consolidation_plan_request( + ctx.ptr().get(), + nullptr, + static_cast(stype), + req_buf, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_consolidation_plan_request( + ctx.ptr().get(), + array.ptr().get(), + static_cast(stype), + nullptr, + resp_buf); + REQUIRE(rval != TILEDB_OK); + + rval = tiledb_handle_consolidation_plan_request( + ctx.ptr().get(), + array.ptr().get(), + static_cast(stype), + req_buf, + nullptr); + REQUIRE(rval != TILEDB_OK); +} + /* ********************************* */ /* Testing Support Code */ /* ********************************* */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 2a6d65a98031..afb88d060627 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -77,6 +77,7 @@ #include "tiledb/sm/serialization/array_schema.h" #include "tiledb/sm/serialization/array_schema_evolution.h" #include "tiledb/sm/serialization/config.h" +#include "tiledb/sm/serialization/consolidation.h" #include "tiledb/sm/serialization/enumeration.h" #include "tiledb/sm/serialization/fragment_info.h" #include "tiledb/sm/serialization/fragments.h" @@ -4367,6 +4368,39 @@ capi_return_t tiledb_handle_query_plan_request( return TILEDB_OK; } +capi_return_t tiledb_handle_consolidation_plan_request( + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + throw std::invalid_argument("Array paramter must be valid."); + } + + api::ensure_buffer_is_valid(request); + api::ensure_buffer_is_valid(response); + + // Error if array is not open + if (!array->array_->is_open()) { + throw std::logic_error( + "Cannot get consolidation plan. Input array is not open"); + } + + auto fragment_size = + tiledb::sm::serialization::deserialize_consolidation_plan_request( + static_cast(serialization_type), + request->buffer()); + sm::ConsolidationPlan plan(array->array_, fragment_size); + + tiledb::sm::serialization::serialize_consolidation_plan_response( + plan, + static_cast(serialization_type), + response->buffer()); + + return TILEDB_OK; +} + /* ****************************** */ /* C++ API */ /* ****************************** */ @@ -5083,10 +5117,7 @@ int32_t tiledb_consolidation_plan_dump_json_str( return TILEDB_ERR; } - consolidation_plan->consolidation_plan_->dump(); - std::string str = consolidation_plan->consolidation_plan_->dump(); - ; *out = static_cast(std::malloc(str.size() + 1)); if (*out == nullptr) { @@ -7245,6 +7276,17 @@ CAPI_INTERFACE( ctx, array, serialization_type, request, response); } +CAPI_INTERFACE( + handle_consolidation_plan_request, + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) { + return api_entry( + ctx, array, serialization_type, request, response); +} + /* ****************************** */ /* C++ API */ /* ****************************** */ diff --git a/tiledb/sm/c_api/tiledb_serialization.h b/tiledb/sm/c_api/tiledb_serialization.h index 435befdd79e3..84bfe305ca0d 100644 --- a/tiledb/sm/c_api/tiledb_serialization.h +++ b/tiledb/sm/c_api/tiledb_serialization.h @@ -804,6 +804,25 @@ TILEDB_EXPORT capi_return_t tiledb_handle_query_plan_request( const tiledb_buffer_t* request, tiledb_buffer_t* response) TILEDB_NOEXCEPT; +/** + * Process a consolidation plan request. + * + * @param ctx The TileDB context. + * @param array The TileDB Array. + * @param serialization_type The type of Cap'n Proto serialization used. + * @param request A buffer containing the ConsolidationPlanRequest Capnp + * message. + * @param response An allocated buffer that will contain the + * ConsolidationPlanResponse Capnp message. + * @return capi_return_t TILEDB_OK on success, TILEDB_ERR on error. + */ +TILEDB_EXPORT capi_return_t tiledb_handle_consolidation_plan_request( + tiledb_ctx_t* ctx, + tiledb_array_t* array, + tiledb_serialization_type_t serialization_type, + const tiledb_buffer_t* request, + tiledb_buffer_t* response) TILEDB_NOEXCEPT; + #ifdef __cplusplus } #endif diff --git a/tiledb/sm/consolidation_plan/consolidation_plan.cc b/tiledb/sm/consolidation_plan/consolidation_plan.cc index b27b2318f25f..051306b5db90 100644 --- a/tiledb/sm/consolidation_plan/consolidation_plan.cc +++ b/tiledb/sm/consolidation_plan/consolidation_plan.cc @@ -47,6 +47,14 @@ ConsolidationPlan::ConsolidationPlan( generate(array); } +ConsolidationPlan::ConsolidationPlan( + uint64_t fragment_size, + std::vector> fragment_uris_per_node) + : num_nodes_{fragment_uris_per_node.size()} + , fragment_uris_per_node_{fragment_uris_per_node} + , desired_fragment_size_{fragment_size} { +} + ConsolidationPlan::~ConsolidationPlan() = default; /* ********************************* */ diff --git a/tiledb/sm/consolidation_plan/consolidation_plan.h b/tiledb/sm/consolidation_plan/consolidation_plan.h index de2b66cfcd42..fa41c22d2fe3 100644 --- a/tiledb/sm/consolidation_plan/consolidation_plan.h +++ b/tiledb/sm/consolidation_plan/consolidation_plan.h @@ -64,6 +64,10 @@ class ConsolidationPlan { /** Constructor. */ ConsolidationPlan(shared_ptr array, uint64_t fragment_size); + /** Constructor. */ + ConsolidationPlan( + uint64_t fragment_size, + std::vector> fragment_uris_per_node); /** Destructor. */ ~ConsolidationPlan(); @@ -87,7 +91,7 @@ class ConsolidationPlan { if (node_idx == std::numeric_limits::max() || node_idx + 1 > num_nodes_) { throw ConsolidationPlanStatusException( - "Trying to access a node that doesn't exists"); + "Trying to access a node that doesn't exist."); } return fragment_uris_per_node_[node_idx].size(); @@ -97,7 +101,7 @@ class ConsolidationPlan { * Get the fragment URI for a node/fragment index. * * @param node_idx Index of the node. - * @param node_idx Index of the fragment. + * @param fragment_idx Index of the fragment. * @return The null terminated string for the fragment URI. */ inline const char* get_fragment_uri( @@ -105,13 +109,13 @@ class ConsolidationPlan { if (node_idx == std::numeric_limits::max() || node_idx + 1 > num_nodes_) { throw ConsolidationPlanStatusException( - "Trying to access a node that doesn't exists"); + "Trying to access a node that doesn't exist."); } if (fragment_idx == std::numeric_limits::max() || fragment_idx + 1 > fragment_uris_per_node_[node_idx].size()) { throw ConsolidationPlanStatusException( - "Trying to access a fragment that doesn't exists"); + "Trying to access a fragment that doesn't exist."); } return fragment_uris_per_node_[node_idx][fragment_idx].c_str(); diff --git a/tiledb/sm/serialization/consolidation.cc b/tiledb/sm/serialization/consolidation.cc index 54bf18a6471d..7efababc271d 100644 --- a/tiledb/sm/serialization/consolidation.cc +++ b/tiledb/sm/serialization/consolidation.cc @@ -45,6 +45,7 @@ #include "tiledb/common/logger_public.h" #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/serialization/config.h" +#include "tiledb/sm/serialization/consolidation.h" using namespace tiledb::common; @@ -186,6 +187,236 @@ Status array_consolidation_request_deserialize( return Status::Ok(); } +void consolidation_plan_request_to_capnp( + capnp::ConsolidationPlanRequest::Builder& builder, + const Config& config, + uint64_t fragment_size) { + auto config_builder = builder.initConfig(); + throw_if_not_ok(config_to_capnp(config, &config_builder)); + builder.setFragmentSize(fragment_size); +} + +uint64_t consolidation_plan_request_from_capnp( + const capnp::ConsolidationPlanRequest::Reader& reader) { + return reader.getFragmentSize(); +} + +void consolidation_plan_response_to_capnp( + capnp::ConsolidationPlanResponse::Builder& builder, + const ConsolidationPlan& plan) { + auto node_size = plan.get_num_nodes(); + if (node_size > 0) { + auto node_builder = builder.initFragmentUrisPerNode(node_size); + for (size_t i = 0; i < node_size; i++) { + auto fragment_builder = node_builder.init(i, plan.get_num_fragments(i)); + for (size_t j = 0; j < plan.get_num_fragments(i); j++) { + fragment_builder.set(j, plan.get_fragment_uri(i, j)); + } + } + } +} + +std::vector> consolidation_plan_response_from_capnp( + const capnp::ConsolidationPlanResponse::Reader& reader) { + std::vector> fragment_uris_per_node; + if (reader.hasFragmentUrisPerNode()) { + auto node_reader = reader.getFragmentUrisPerNode(); + fragment_uris_per_node.reserve(node_reader.size()); + for (const auto& fragment_reader : node_reader) { + auto& node = fragment_uris_per_node.emplace_back(); + node.reserve(fragment_reader.size()); + for (const auto& fragment_uri : fragment_reader) { + node.emplace_back(fragment_uri); + } + } + } + + return fragment_uris_per_node; +} + +void serialize_consolidation_plan_request( + uint64_t fragment_size, + const Config& config, + SerializationType serialization_type, + Buffer& request) { + try { + ::capnp::MallocMessageBuilder message; + capnp::ConsolidationPlanRequest::Builder builder = + message.initRoot(); + consolidation_plan_request_to_capnp(builder, config, fragment_size); + + request.reset_size(); + request.reset_offset(); + + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + kj::String capnp_json = json.encode(builder); + const auto json_len = capnp_json.size(); + const char nul = '\0'; + // size does not include needed null terminator, so add +1 + throw_if_not_ok(request.realloc(json_len + 1)); + throw_if_not_ok(request.write(capnp_json.cStr(), json_len)); + throw_if_not_ok(request.write(&nul, 1)); + break; + } + case SerializationType::CAPNP: { + kj::Array<::capnp::word> protomessage = messageToFlatArray(message); + kj::ArrayPtr message_chars = protomessage.asChars(); + const auto nbytes = message_chars.size(); + throw_if_not_ok(request.realloc(nbytes)); + throw_if_not_ok(request.write(message_chars.begin(), nbytes)); + break; + } + default: { + throw Status_SerializationError( + "Error serializing consolidation plan request; " + "Unknown serialization type passed"); + } + } + + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error serializing consolidation plan request; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error serializing consolidation plan request; exception " + + std::string(e.what())); + } +} + +uint64_t deserialize_consolidation_plan_request( + SerializationType serialization_type, const Buffer& request) { + try { + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + ::capnp::MallocMessageBuilder message_builder; + capnp::ConsolidationPlanRequest::Builder builder = + message_builder.initRoot(); + json.decode( + kj::StringPtr(static_cast(request.data())), builder); + capnp::ConsolidationPlanRequest::Reader reader = builder.asReader(); + return consolidation_plan_request_from_capnp(reader); + } + case SerializationType::CAPNP: { + const auto mBytes = reinterpret_cast(request.data()); + ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( + reinterpret_cast(mBytes), + request.size() / sizeof(::capnp::word))); + capnp::ConsolidationPlanRequest::Reader reader = + array_reader.getRoot(); + return consolidation_plan_request_from_capnp(reader); + } + default: { + throw Status_SerializationError( + "Error deserializing consolidation plan request; " + "Unknown serialization type passed"); + } + } + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error deserializing consolidation plan request; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error deserializing consolidation plan request; exception " + + std::string(e.what())); + } +} + +void serialize_consolidation_plan_response( + const ConsolidationPlan& plan, + SerializationType serialization_type, + Buffer& response) { + try { + ::capnp::MallocMessageBuilder message; + capnp::ConsolidationPlanResponse::Builder builder = + message.initRoot(); + consolidation_plan_response_to_capnp(builder, plan); + + response.reset_size(); + response.reset_offset(); + + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + kj::String capnp_json = json.encode(builder); + const auto json_len = capnp_json.size(); + const char nul = '\0'; + // size does not include needed null terminator, so add +1 + throw_if_not_ok(response.realloc(json_len + 1)); + throw_if_not_ok(response.write(capnp_json.cStr(), json_len)); + throw_if_not_ok(response.write(&nul, 1)); + break; + } + case SerializationType::CAPNP: { + kj::Array<::capnp::word> protomessage = messageToFlatArray(message); + kj::ArrayPtr message_chars = protomessage.asChars(); + const auto nbytes = message_chars.size(); + throw_if_not_ok(response.realloc(nbytes)); + throw_if_not_ok(response.write(message_chars.begin(), nbytes)); + break; + } + default: { + throw Status_SerializationError( + "Error serializing consolidation plan response; " + "Unknown serialization type passed"); + } + } + + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error serializing consolidation plan response; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error serializing consolidation plan response; exception " + + std::string(e.what())); + } +} + +std::vector> deserialize_consolidation_plan_response( + SerializationType serialization_type, const Buffer& response) { + try { + switch (serialization_type) { + case SerializationType::JSON: { + ::capnp::JsonCodec json; + ::capnp::MallocMessageBuilder message_builder; + capnp::ConsolidationPlanResponse::Builder builder = + message_builder.initRoot(); + json.decode( + kj::StringPtr(static_cast(response.data())), builder); + capnp::ConsolidationPlanResponse::Reader reader = builder.asReader(); + return consolidation_plan_response_from_capnp(reader); + } + case SerializationType::CAPNP: { + const auto mBytes = reinterpret_cast(response.data()); + ::capnp::FlatArrayMessageReader array_reader(kj::arrayPtr( + reinterpret_cast(mBytes), + response.size() / sizeof(::capnp::word))); + capnp::ConsolidationPlanResponse::Reader reader = + array_reader.getRoot(); + return consolidation_plan_response_from_capnp(reader); + } + default: { + throw Status_SerializationError( + "Error deserializing consolidation plan response; " + "Unknown serialization type passed"); + } + } + } catch (kj::Exception& e) { + throw Status_SerializationError( + "Error deserializing consolidation plan response; kj::Exception: " + + std::string(e.getDescription().cStr())); + } catch (std::exception& e) { + throw Status_SerializationError( + "Error deserializing consolidation plan response; exception " + + std::string(e.what())); + } +} + #else Status array_consolidation_request_serialize( @@ -200,6 +431,30 @@ Status array_consolidation_request_deserialize( "Cannot deserialize; serialization not enabled.")); } +void serialize_consolidation_plan_request( + uint64_t, const Config&, SerializationType, Buffer&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +uint64_t deserialize_consolidation_plan_request( + SerializationType, const Buffer&) { + throw Status_SerializationError( + "Cannot deserialize; serialization not enabled."); +} + +void serialize_consolidation_plan_response( + const ConsolidationPlan&, SerializationType, Buffer&) { + throw Status_SerializationError( + "Cannot serialize; serialization not enabled."); +} + +std::vector> deserialize_consolidation_plan_response( + SerializationType, const Buffer&) { + throw Status_SerializationError( + "Cannot deserialize; serialization not enabled."); +} + #endif // TILEDB_SERIALIZATION } // namespace serialization diff --git a/tiledb/sm/serialization/consolidation.h b/tiledb/sm/serialization/consolidation.h index f36dcbaf72bf..d27c6c6e3466 100644 --- a/tiledb/sm/serialization/consolidation.h +++ b/tiledb/sm/serialization/consolidation.h @@ -39,6 +39,7 @@ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" +#include "tiledb/sm/consolidation_plan/consolidation_plan.h" using namespace tiledb::common; @@ -102,6 +103,54 @@ Status array_consolidation_request_deserialize( SerializationType serialize_type, const Buffer& serialized_buffer); +/** + * Serialize a consolidation plan request via Cap'n Proto. + * + * @param fragment_size maximum fragment size for the cosnolidateion plan. + * @param config config object to serialize. + * @param serialization_type format to serialize into Cap'n Proto or JSON. + * @param request buffer to store serialized bytes in. + */ +void serialize_consolidation_plan_request( + uint64_t fragment_size, + const Config& config, + SerializationType serialization_type, + Buffer& request); + +/** + * Deserialize a consolidation plan request via Cap'n Proto. + * + * @param serialization_type format the data is serialized in: Cap'n Proto of + * JSON. + * @param response buffer to read serialized bytes from. + * @return the deserialized maximum fragment size + */ +uint64_t deserialize_consolidation_plan_request( + SerializationType serialization_type, const Buffer& request); + +/** + * Serialize a consolidation plan response via Cap'n Proto. + * + * @param consolidation_plan consolidation plan to serialize. + * @param serialization_type format to serialize into Cap'n Proto or JSON. + * @param response buffer to store serialized bytes in. + */ +void serialize_consolidation_plan_response( + const ConsolidationPlan& consolidation_plan, + SerializationType serialization_type, + Buffer& response); + +/** + * Deserialize a consolidation plan response via Cap'n Proto. + * + * @param serialization_type format the data is serialized in: Cap'n Proto of + * JSON. + * @param response buffer to read serialized bytes from. + * @return the deserialized consolidation plan info + */ +std::vector> deserialize_consolidation_plan_response( + SerializationType serialization_type, const Buffer& response); + } // namespace serialization } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index bfb439e99bd9..1f06e2261922 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1274,6 +1274,19 @@ struct QueryPlanResponse { # The returned query plan } +struct ConsolidationPlanRequest { + config @0 :Config; + # Config + + fragmentSize @1 :UInt64; + # Maximum fragment size +} + +struct ConsolidationPlanResponse { + fragmentUrisPerNode @0 :List(List(Text)); + # The uris for each node of the consolidation plan +} + struct QueryChannel { # structure representing a query channel, that is a stream of data within # a TileDB query. Such channels can be generated for the purpose of avoiding diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 89e59d603af9..bfb7eda2dffb 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -10158,6 +10158,133 @@ const ::capnp::_::RawSchema s_9fd8fc2f462b2d06 = { 0, 1, i_9fd8fc2f462b2d06, nullptr, nullptr, { &s_9fd8fc2f462b2d06, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<54> b_8965edf5597ce627 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 39, 230, 124, 89, 245, 237, 101, 137, + 42, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 26, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 110, 115, 111, 108, + 105, 100, 97, 116, 105, 111, 110, 80, + 108, 97, 110, 82, 101, 113, 117, 101, + 115, 116, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 102, 105, 103, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 54, 173, 17, 129, 75, 91, 201, 182, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 83, 105, 122, 101, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 9, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_8965edf5597ce627 = b_8965edf5597ce627.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_8965edf5597ce627[] = { + &s_b6c95b4b8111ad36, +}; +static const uint16_t m_8965edf5597ce627[] = {0, 1}; +static const uint16_t i_8965edf5597ce627[] = {0, 1}; +const ::capnp::_::RawSchema s_8965edf5597ce627 = { + 0x8965edf5597ce627, b_8965edf5597ce627.words, 54, d_8965edf5597ce627, m_8965edf5597ce627, + 1, 2, i_8965edf5597ce627, nullptr, nullptr, { &s_8965edf5597ce627, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<48> b_aac8bf9b5211388b = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 139, 56, 17, 82, 155, 191, 200, 170, + 42, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 1, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 34, 2, 0, 0, + 53, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 63, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 47, 115, + 109, 47, 115, 101, 114, 105, 97, 108, + 105, 122, 97, 116, 105, 111, 110, 47, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 67, 111, 110, 115, 111, 108, + 105, 100, 97, 116, 105, 111, 110, 80, + 108, 97, 110, 82, 101, 115, 112, 111, + 110, 115, 101, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 4, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 162, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 3, 0, 1, 0, + 60, 0, 0, 0, 2, 0, 1, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 85, 114, 105, 115, 80, 101, 114, 78, + 111, 100, 101, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_aac8bf9b5211388b = b_aac8bf9b5211388b.words; +#if !CAPNP_LITE +static const uint16_t m_aac8bf9b5211388b[] = {0}; +static const uint16_t i_aac8bf9b5211388b[] = {0}; +const ::capnp::_::RawSchema s_aac8bf9b5211388b = { + 0xaac8bf9b5211388b, b_aac8bf9b5211388b.words, 48, nullptr, m_aac8bf9b5211388b, + 0, 1, i_aac8bf9b5211388b, nullptr, nullptr, { &s_aac8bf9b5211388b, nullptr, nullptr, 0, 0, nullptr }, false +}; +#endif // !CAPNP_LITE static const ::capnp::_::AlignedData<56> b_ca2d4d0bfe4ae5d9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 217, 229, 74, 254, 11, 77, 45, 202, @@ -11385,6 +11512,30 @@ constexpr ::capnp::_::RawSchema const* QueryPlanResponse::_capnpPrivate::schema; #endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL #endif // !CAPNP_LITE +// ConsolidationPlanRequest +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t ConsolidationPlanRequest::_capnpPrivate::dataWordSize; +constexpr uint16_t ConsolidationPlanRequest::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind ConsolidationPlanRequest::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* ConsolidationPlanRequest::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + +// ConsolidationPlanResponse +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr uint16_t ConsolidationPlanResponse::_capnpPrivate::dataWordSize; +constexpr uint16_t ConsolidationPlanResponse::_capnpPrivate::pointerCount; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#if !CAPNP_LITE +#if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr ::capnp::Kind ConsolidationPlanResponse::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* ConsolidationPlanResponse::_capnpPrivate::schema; +#endif // !CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL +#endif // !CAPNP_LITE + // QueryChannel #if CAPNP_NEED_REDUNDANT_CONSTEXPR_DECL constexpr uint16_t QueryChannel::_capnpPrivate::dataWordSize; diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 8b1304b4ac7d..7463097f0f4c 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -110,6 +110,8 @@ CAPNP_DECLARE_SCHEMA(83f094010132ff21); CAPNP_DECLARE_SCHEMA(ebe17f59ac9a1df1); CAPNP_DECLARE_SCHEMA(e06f571aa93eb314); CAPNP_DECLARE_SCHEMA(9fd8fc2f462b2d06); +CAPNP_DECLARE_SCHEMA(8965edf5597ce627); +CAPNP_DECLARE_SCHEMA(aac8bf9b5211388b); CAPNP_DECLARE_SCHEMA(ca2d4d0bfe4ae5d9); CAPNP_DECLARE_SCHEMA(e193f1f45a9f102e); @@ -1723,6 +1725,40 @@ struct QueryPlanResponse { }; }; +struct ConsolidationPlanRequest { + ConsolidationPlanRequest() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(8965edf5597ce627, 1, 1) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct ConsolidationPlanResponse { + ConsolidationPlanResponse() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(aac8bf9b5211388b, 0, 1) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + struct QueryChannel { QueryChannel() = delete; @@ -15182,6 +15218,232 @@ class QueryPlanResponse::Pipeline { }; #endif // !CAPNP_LITE +class ConsolidationPlanRequest::Reader { + public: + typedef ConsolidationPlanRequest Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasConfig() const; + inline ::tiledb::sm::serialization::capnp::Config::Reader getConfig() const; + + inline ::uint64_t getFragmentSize() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class ConsolidationPlanRequest::Builder { + public: + typedef ConsolidationPlanRequest Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasConfig(); + inline ::tiledb::sm::serialization::capnp::Config::Builder getConfig(); + inline void setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value); + inline ::tiledb::sm::serialization::capnp::Config::Builder initConfig(); + inline void adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> + disownConfig(); + + inline ::uint64_t getFragmentSize(); + inline void setFragmentSize(::uint64_t value); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class ConsolidationPlanRequest::Pipeline { + public: + typedef ConsolidationPlanRequest Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + inline ::tiledb::sm::serialization::capnp::Config::Pipeline getConfig(); + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + +class ConsolidationPlanResponse::Reader { + public: + typedef ConsolidationPlanResponse Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasFragmentUrisPerNode() const; + inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Reader + getFragmentUrisPerNode() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class ConsolidationPlanResponse::Builder { + public: + typedef ConsolidationPlanResponse Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasFragmentUrisPerNode(); + inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Builder + getFragmentUrisPerNode(); + inline void setFragmentUrisPerNode( + ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Reader value); + inline void setFragmentUrisPerNode( + ::kj::ArrayPtr< + const ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader> + value); + inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Builder + initFragmentUrisPerNode(unsigned int size); + inline void adoptFragmentUrisPerNode( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>> + disownFragmentUrisPerNode(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class ConsolidationPlanResponse::Pipeline { + public: + typedef ConsolidationPlanResponse Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class QueryChannel::Reader { public: typedef QueryChannel Reads; @@ -32450,6 +32712,157 @@ QueryPlanResponse::Builder::disownQueryPlan() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool ConsolidationPlanRequest::Reader::hasConfig() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ConsolidationPlanRequest::Builder::hasConfig() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Config::Reader +ConsolidationPlanRequest::Reader::getConfig() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Config::Builder +ConsolidationPlanRequest::Builder::getConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Config::Pipeline +ConsolidationPlanRequest::Pipeline::getConfig() { + return ::tiledb::sm::serialization::capnp::Config::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void ConsolidationPlanRequest::Builder::setConfig( + ::tiledb::sm::serialization::capnp::Config::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Config::Builder +ConsolidationPlanRequest::Builder::initConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void ConsolidationPlanRequest::Builder::adoptConfig( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Config>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> +ConsolidationPlanRequest::Builder::disownConfig() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Config>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline ::uint64_t ConsolidationPlanRequest::Reader::getFragmentSize() const { + return _reader.getDataField<::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint64_t ConsolidationPlanRequest::Builder::getFragmentSize() { + return _builder.getDataField<::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConsolidationPlanRequest::Builder::setFragmentSize( + ::uint64_t value) { + _builder.setDataField<::uint64_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool ConsolidationPlanResponse::Reader::hasFragmentUrisPerNode() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ConsolidationPlanResponse::Builder::hasFragmentUrisPerNode() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Reader +ConsolidationPlanResponse::Reader::getFragmentUrisPerNode() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>::get(_reader + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Builder +ConsolidationPlanResponse::Builder::getFragmentUrisPerNode() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>::get(_builder + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} +inline void ConsolidationPlanResponse::Builder::setFragmentUrisPerNode( + ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline void ConsolidationPlanResponse::Builder::setFragmentUrisPerNode( + ::kj::ArrayPtr< + const ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader> + value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>:: + set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>::Builder +ConsolidationPlanResponse::Builder::initFragmentUrisPerNode(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>:: + init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void ConsolidationPlanResponse::Builder::adoptFragmentUrisPerNode( + ::capnp::Orphan<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>:: + adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>> +ConsolidationPlanResponse::Builder::disownFragmentUrisPerNode() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>, + ::capnp::Kind::LIST>>::disown(_builder + .getPointerField( + ::capnp::bounded<0>() * + ::capnp::POINTERS)); +} + inline bool QueryChannel::Reader::getDefault() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } From f4ad55665d9b10d1fe51df9776853d1b9aac42b7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 30 Nov 2023 15:14:00 +0200 Subject: [PATCH 093/456] Clean-up unused build system code. (#4527) This PR: * Removes the `abseil` vcpkg feature, the abseil linkage tests and the relevant bootstrap script options. These are dead code since #4055. * Abseil is still acquired from vcpkg if GCS is enabled. * Deletes unused patches for the ExternalProject of the GCS SDK, which has been removed in #4031. * Removes the vcpkg direct dependency to liblzma. The Core does not directly use liblzma. * Removes the crc32c linkage tests and the related CMake and bootstrap script options. They are not used by CI and their value has declined with the advent of vcpkg. Moreover using exported static libraries with `TILEDB_CRC32` enabled from CMake will fail because there is no `find_dependency(Crc32c)` in the config file. Nor there is a `crc32` vcpkg feature. And this option offers nothing at runtime. * Removes `unit_link_webp` as this was used to prove WebP was properly included into the build. Now that we have a WebP filter, this code is not necessary anymore. --- TYPE: IMPROVEMENT DESC: Clean-up unused build system code. TYPE: BUILD DESC: Remove the CMake and bootstrap script options related to abseil and crc32c linkage tests. --- .github/workflows/build-windows.yml | 12 - CMakeLists.txt | 8 - bootstrap | 13 - bootstrap.ps1 | 18 - cmake/Modules/FindCrc32c_EP.cmake | 92 ---- cmake/Options/BuildOptions.cmake | 1 - cmake/TileDB-Superbuild.cmake | 5 - cmake/inputs/patches/ep_gcssdk/build.patch | 396 ------------------ .../patches/ep_gcssdk/disable_examples.patch | 35 -- .../patches/ep_gcssdk/disable_tests.patch | 23 - ...txt.openssl3md5deprecationmitigation.patch | 15 - test/CMakeLists.txt | 3 - test/external/CMakeLists.txt | 44 -- test/external/src/absl_library_targets.cmake | 171 -------- test/external/src/absl_link_test.cc | 159 ------- test/external/src/crc32_link_test.cc | 45 -- tiledb/CMakeLists.txt | 13 - tiledb/sm/compressors/CMakeLists.txt | 25 -- tiledb/sm/compressors/test/link_webp_test1.cc | 348 --------------- vcpkg.json | 10 - 20 files changed, 1436 deletions(-) delete mode 100644 cmake/Modules/FindCrc32c_EP.cmake delete mode 100644 cmake/inputs/patches/ep_gcssdk/build.patch delete mode 100644 cmake/inputs/patches/ep_gcssdk/disable_examples.patch delete mode 100644 cmake/inputs/patches/ep_gcssdk/disable_tests.patch delete mode 100644 cmake/inputs/patches/ep_gcssdk/v1.22.0.CMakelists.txt.openssl3md5deprecationmitigation.patch delete mode 100644 test/external/CMakeLists.txt delete mode 100644 test/external/src/absl_library_targets.cmake delete mode 100644 test/external/src/absl_link_test.cc delete mode 100644 test/external/src/crc32_link_test.cc delete mode 100644 tiledb/sm/compressors/test/link_webp_test1.cc diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index e1d6762ea082..0c896c58b059 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -291,18 +291,6 @@ jobs: Write-Host "Tests failed. test_assert exit status: " $LastExitCocde $host.SetShouldExit($LastExitCode) } - - - if ($env:TILEDB_WEBP -eq "ON") { - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\compressors\$CMakeBuildType\unit_link_webp.exe" - Write-Host "cmds: '$cmds'" - Invoke-Expression $cmds - - if ($LastExitCode -ne 0) { - Write-Host "unit_link_webp failed. CMake exit status: " $LastExitCocde - $host.SetShouldExit($LastExitCode) - } - } - name: Build examples shell: pwsh run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index df428fc15897..83d51e767ea4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -578,14 +578,6 @@ if (TILEDB_TESTS) # C API array schema add_dependencies(tests unit_capi_filter unit_capi_filter_list unit_capi_dimension_label) - if (TILEDB_CRC32) - add_dependencies(tests unit_link_crc32) - endif() - - if (TILEDB_WEBP) - add_dependencies(tests unit_link_webp) - endif() - if (ENABLE_MAGIC_TEST) add_dependencies(tests unit_mgc_dict) endif() diff --git a/bootstrap b/bootstrap index 8ade2249dabf..455b424334bb 100755 --- a/bootstrap +++ b/bootstrap @@ -63,8 +63,6 @@ Configuration: --enable-tools enables TileDB CLI tools (experimental) --enable-ccache enables use of ccache (if present) --enable-arrow-tests enables the compilation of the arrow adapter unit tests - --enable-_crc32 enables building of crc32 library and linkage test - --enable-abseil enables building of abseil library and linkage test --enable-experimental-features enables experimental TileDB features --enable-rest-tests enables REST tests --enable-aws-s3-config enables AWS S3 configuration for tests @@ -111,9 +109,7 @@ tiledb_tools="OFF" tiledb_ccache="OFF" tiledb_arrow_tests="OFF" tiledb_experimental_features="OFF" -tiledb_build_crc32="OFF" tiledb_build_webp="ON" -tiledb_build_abseil="OFF" tiledb_tests_enable_rest="OFF" tiledb_tests_aws_s3_config="OFF" enable_multiple="" @@ -150,8 +146,6 @@ while test $# != 0; do --enable-tools) tiledb_tools="ON";; --enable-ccache) tiledb_ccache="ON";; --enable-arrow-tests) tiledb_arrow_tests="ON";; - --enable-_crc32) tiledb_build_crc32="ON";; - --enable-abseil) tiledb_build_abseil="ON";; --enable-experimental-features) tiledb_experimental_features="ON";; --enable-rest-tests) tiledb_tests_enable_rest="ON";; --enable-aws-s3-config) tiledb_tests_aws_s3_config="ON";; @@ -181,8 +175,6 @@ for en in "${enables[@]}"; do tools) tiledb_tools="ON";; ccache) tiledb_ccache="ON";; arrow-tests) tiledb_arrow_tests="ON";; - _crc32) tiledb_build_crc32="ON";; - abseil) tiledb_build_abseil="ON";; hdfs) tiledb_hdfs="ON";; static-tiledb) tiledb_static="ON";; experimental-features) tiledb_experimental_features="ON";; @@ -192,10 +184,6 @@ for en in "${enables[@]}"; do esac done -if [ "${tiledb_build_abseil}" = "ON" ]; then - echo "Argument '--build-abseil' is obsolete and will be removed in TileDB 2.18" -fi - if [ "${source_dir}" = "${binary_dir}" ]; then die "cannot build the project in the source directory! Out-of-source build is enforced!" fi @@ -247,7 +235,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_TESTS=${tiledb_tests} \ -DTILEDB_CCACHE=${tiledb_ccache} \ -DTILEDB_ARROW_TESTS=${tiledb_arrow_tests} \ - -DTILEDB_CRC32=${tiledb_build_crc32} \ -DTILEDB_WEBP=${tiledb_build_webp} \ -DTILEDB_FORCE_ALL_DEPS=${tiledb_force_all_deps} \ -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \ diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 6c3c0358406b..48e1533f6bdc 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -68,12 +68,6 @@ Enables building TileDB CLI tools (experimental) .PARAMETER EnableExperimentalFeatures Enables building TileDB Experimental features -.PARAMETER EnableAbseil -(Obsolete) Enables building of Abseil and simple linkage test - -.PARAMETER _EnableCrc32 -Enables building of Crc32 and simple linkage test - .PARAMETER EnableArrowTests Enables the compilation of the arrow adapter unit tests @@ -134,9 +128,7 @@ Param( [switch]$EnableStaticTileDB, [switch]$EnableTools, [switch]$EnableExperimentalFeatures, - [switch]$EnableAbseil, [switch]$EnableBuildDeps, - [switch]$_EnableCrc32, [switch]$EnableArrowTests, [switch]$EnableRestTests, [switch]$EnableAwsS3Config, @@ -263,21 +255,11 @@ if ($EnableExperimentalFeatures.IsPresent) { $TileDBExperimentalFeatures = "ON" } -if ($EnableAbseil.IsPresent) { - # remove in 2.18 - Write-Host "EnableAbseil is deprecated and will be removed" -} - $TileDBBuildDeps = "OFF"; if ($EnableBuildDeps.IsPresent) { $TileDBBuildDeps = "ON" } -$BuildCrc32="OFF" -if ($_EnableCrc32.IsPresent) { - $BuildCrc32="ON" -} - $ArrowTests="OFF" if ($EnableArrowTests.IsPresent) { $ArrowTests="ON" diff --git a/cmake/Modules/FindCrc32c_EP.cmake b/cmake/Modules/FindCrc32c_EP.cmake deleted file mode 100644 index 296c6049cf6e..000000000000 --- a/cmake/Modules/FindCrc32c_EP.cmake +++ /dev/null @@ -1,92 +0,0 @@ -# -# FindCrc32c_EP.cmake -# -# The MIT License -# -# Copyright (c) 2022 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# Finds the Crc32c library, installing with an ExternalProject as necessary. -# This module defines: -# - Crc32c_INCLUDE_DIR, directory containing headers -# - Crc32c_LIBRARIES, the Crc32c library path -# - Crc32c_FOUND, whether Crc32c has been found -# - The Crc32c::crc32c imported target - -# Include some common helper functions. -include(TileDBCommon) - -if(TILEDB_CRC32C_EP_BUILT) - find_package(Crc32c CONFIG REQUIRED - HINTS - ${TILEDB_EP_INSTALL_PREFIX}/lib/cmake - ${TILEDB_DEPS_NO_DEFAULT_PATH}) -elseif (NOT TILEDB_FORCE_ALL_DEPS) - # seems no standard findCrc32c.cmake, so silence warnings with QUIET - find_package(Crc32c QUIET ${TILEDB_DEPS_NO_DEFAULT_PATH}) -endif() - -# if not yet built add it as an external project -if (NOT TILEDB_CRC32C_EP_BUILT AND NOT Crc32c_FOUND) - if (TILEDB_SUPERBUILD) - message(STATUS "Adding crc32c as an external project") - - set(TILEDB_CRC32C_BUILD_VERSION 1.1.2) - set(TILEDB_CRC32C_URL - "https://github.com/google/crc32c/archive/refs/tags/${TILEDB_CRC32C_BUILD_VERSION}.tar.gz") - set(TILEDB_CRC32C_SHA256 - "ac07840513072b7fcebda6e821068aa04889018f24e10e46181068fb214d7e56") - - ExternalProject_Add( - ep_crc32c - EXCLUDE_FROM_ALL ON - PREFIX "externals" - INSTALL_DIR "${TILEDB_EP_INSTALL_PREFIX}" - URL ${TILEDB_CRC32C_URL} - URL_HASH SHA256=${TILEDB_CRC32C_SHA256} - LIST_SEPARATOR | - CMAKE_ARGS - -DCMAKE_PREFIX_PATH=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_INSTALL_PREFIX=${TILEDB_EP_INSTALL_PREFIX} - -DCRC32C_BUILD_TESTS=OFF - -DCRC32C_BUILD_BENCHMARKS=OFF - -DCRC32C_USE_GLOG=OFF - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON - LOG_INSTALL ON) - - list(APPEND TILEDB_EXTERNAL_PROJECTS ep_crc32c) - list(APPEND FORWARD_EP_CMAKE_ARGS - -DTILEDB_CRC32C_EP_BUILT=TRUE - ) - else() - message(FATAL_ERROR "Unable to find crc32c") - endif() -endif () - -if (Crc32c_FOUND AND NOT TARGET Crc32c::crc32c) - message(STATUS "Found crc32c, adding imported target: ${Crc32c_LIBRARIES}") - add_library(Crc32c::crc32c UNKNOWN IMPORTED) - set_target_properties(Crc32c::crc32c PROPERTIES - IMPORTED_LOCATION "${Crc32c_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${Crc32c_INCLUDE_DIR}" - ) -endif() diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 6714907c34aa..3a21254e69f1 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -27,7 +27,6 @@ option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) option(TILEDB_CCACHE "If true, enables use of 'ccache' (if present)" OFF) option(TILEDB_ARROW_TESTS "If true, enables building the arrow adapter unit tests" OFF) -option(TILEDB_CRC32 "If true, enables building crc32 and a simple linkage test" OFF) option(TILEDB_WEBP "If true, enables building webp and a simple linkage test" ON) option(TILEDB_LOG_OUTPUT_ON_FAILURE "If true, print error logs if dependency sub-project build fails" ON) option(TILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK "If true, skip check needed path length for awssdk (TILEDB_S3) dependent builds" OFF) diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 14378b7f1c54..fc3943059bb9 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -52,7 +52,6 @@ set(INHERITED_CMAKE_ARGS -DTILEDB_TOOLS=${TILEDB_TOOLS} -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} -DTILEDB_ARROW_TESTS=${TILEDB_ARROW_TESTS} - -DTILEDB_CRC32=${TILEDB_CRC32} -DTILEDB_WEBP=${TILEDB_WEBP} -DTILEDB_INSTALL_LIBDIR=${TILEDB_INSTALL_LIBDIR} -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} @@ -106,10 +105,6 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindZlib_EP.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindZstd_EP.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindMagic_EP.cmake) -if(TILEDB_CRC32) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindCrc32c_EP.cmake) -endif() - if(TILEDB_WEBP) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindWebp_EP.cmake) endif() diff --git a/cmake/inputs/patches/ep_gcssdk/build.patch b/cmake/inputs/patches/ep_gcssdk/build.patch deleted file mode 100644 index 19f81a836d52..000000000000 --- a/cmake/inputs/patches/ep_gcssdk/build.patch +++ /dev/null @@ -1,396 +0,0 @@ -From b95493b781f372c0afbbfe0c1f8d1603513bd25a Mon Sep 17 00:00:00 2001 -From: Joe Maley -Date: Fri, 8 Jan 2021 10:51:53 -0500 -Subject: [PATCH] GCS build patch for TileDB - ---- - CMakeLists.txt | 5 +++ - super/CMakeLists.txt | 14 +++++-- - super/external/abseil.cmake | 7 ++-- - super/external/c-ares.cmake | 6 ++- - super/external/crc32c.cmake | 6 ++- - super/external/curl.cmake | 63 --------------------------------- - super/external/googletest.cmake | 6 ++- - super/external/grpc.cmake | 16 ++++++-- - super/external/protobuf.cmake | 8 ++-- - super/external/ssl.cmake | 26 -------------- - super/external/zlib.cmake | 46 ------------------------ - 11 files changed, 47 insertions(+), 149 deletions(-) - delete mode 100644 super/external/curl.cmake - delete mode 100644 super/external/ssl.cmake - delete mode 100644 super/external/zlib.cmake - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index e0ffd4a75..cda631c40 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -65,6 +65,11 @@ endif () - option(GOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK - "If enabled, check that the user has defined OPENSSL_ROOT_DIR on macOS" - ON) -+ -+ if (NOT DEFINED ENV{OPENSSL_ROOT_DIR} AND DEFINED OPENSSL_ROOT_DIR) -+ set(ENV{OPENSSL_ROOT_DIR} ${OPENSSL_ROOT_DIR}) -+ endif() -+ - if (APPLE) - # This is an easy mistake to make, and the error messages are very - # confusing. Help our users by giving them some guidance. -diff --git a/super/CMakeLists.txt b/super/CMakeLists.txt -index f5560a620..12ca348a2 100644 ---- a/super/CMakeLists.txt -+++ b/super/CMakeLists.txt -@@ -39,7 +39,6 @@ endif () - - include(external/abseil) - include(external/benchmark) --include(external/curl) - include(external/crc32c) - include(external/googletest) - include(external/grpc) -@@ -55,7 +54,6 @@ set_external_project_vars() - set(GOOGLE_CLOUD_CPP_DEPENDENCIES_LIST - abseil-cpp-project - grpc-project -- curl-project - crc32c-project - nlohmann-json-project - googletest-project -@@ -77,6 +75,15 @@ ExternalProject_Add( - -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} - -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} - -DCMAKE_INSTALL_PREFIX= -+ -DCMAKE_BUILD_TYPE=$ -+ -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -+ -DBUILD_SAMPLES=${BUILD_SAMPLES} -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -+ -DGOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK=OFF -+ -DBUILD_TESTING=${BUILD_TESTING} -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - TEST_COMMAND - ${CMAKE_CTEST_COMMAND} -@@ -89,7 +95,8 @@ ExternalProject_Add( - LOG_DOWNLOAD OFF - LOG_CONFIGURE OFF - LOG_BUILD OFF -- LOG_INSTALL OFF) -+ LOG_INSTALL OFF -+ LOG_OUTPUT_ON_FAILURE ON) - - # This makes it easy to compile the dependencies before the code. - add_custom_target(project-dependencies) -diff --git a/super/external/abseil.cmake b/super/external/abseil.cmake -index 97d2c1551..0b71ee987 100644 ---- a/super/external/abseil.cmake -+++ b/super/external/abseil.cmake -@@ -46,11 +46,14 @@ if (NOT TARGET abseil-cpp-project) - -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} - -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} - -DCMAKE_INSTALL_PREFIX= -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - PATCH_COMMAND ${CMAKE_COMMAND} -P - "${CMAKE_CURRENT_LIST_DIR}/abseil-patch.cmake" -- LOG_DOWNLOAD OFF - LOG_CONFIGURE OFF - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/c-ares.cmake b/super/external/c-ares.cmake -index 5b898e3f7..63bd708b6 100644 ---- a/super/external/c-ares.cmake -+++ b/super/external/c-ares.cmake -@@ -40,9 +40,13 @@ if (NOT TARGET c-ares-project) - -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} - -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} - -DCMAKE_INSTALL_PREFIX= -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/crc32c.cmake b/super/external/crc32c.cmake -index ed35de175..7d54c5ae8 100644 ---- a/super/external/crc32c.cmake -+++ b/super/external/crc32c.cmake -@@ -43,9 +43,13 @@ if (NOT TARGET crc32c-project) - -DCRC32C_BUILD_TESTS=OFF - -DCRC32C_BUILD_BENCHMARKS=OFF - -DCRC32C_USE_GLOG=OFF -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/curl.cmake b/super/external/curl.cmake -deleted file mode 100644 -index 1a695e098..000000000 ---- a/super/external/curl.cmake -+++ /dev/null -@@ -1,63 +0,0 @@ --# ~~~ --# Copyright 2018 Google LLC --# --# Licensed under the Apache License, Version 2.0 (the "License"); --# you may not use this file except in compliance with the License. --# You may obtain a copy of the License at --# --# http://www.apache.org/licenses/LICENSE-2.0 --# --# Unless required by applicable law or agreed to in writing, software --# distributed under the License is distributed on an "AS IS" BASIS, --# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --# See the License for the specific language governing permissions and --# limitations under the License. --# ~~~ -- --include(external/c-ares) --include(external/ssl) --include(external/zlib) -- --if (NOT TARGET curl-project) -- # Give application developers a hook to configure the version and hash -- # downloaded from GitHub. -- set(GOOGLE_CLOUD_CPP_CURL_URL -- "https://curl.haxx.se/download/curl-7.69.1.tar.gz") -- set(GOOGLE_CLOUD_CPP_CURL_SHA256 -- "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98") -- -- set_external_project_build_parallel_level(PARALLEL) -- set_external_project_vars() -- -- include(ExternalProject) -- ExternalProject_Add( -- curl-project -- DEPENDS c-ares-project ssl-project zlib-project -- EXCLUDE_FROM_ALL ON -- PREFIX "${CMAKE_BINARY_DIR}/external/curl" -- INSTALL_DIR "${GOOGLE_CLOUD_CPP_EXTERNAL_PREFIX}" -- URL ${GOOGLE_CLOUD_CPP_CURL_URL} -- URL_HASH SHA256=${GOOGLE_CLOUD_CPP_CURL_SHA256} -- LIST_SEPARATOR | -- CMAKE_ARGS ${GOOGLE_CLOUD_CPP_EXTERNAL_PROJECT_CMAKE_FLAGS} -- -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} -- -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} -- -DCMAKE_INSTALL_PREFIX= -- # libcurl automatically enables a number of protocols. With -- # static libraries this is a problem. The indirect -- # dependencies, such as libldap, become hard to predict and -- # manage. Setting HTTP_ONLY=ON disables all optional -- # protocols and meets our needs. If the application needs a -- # version of libcurl with other protocols enabled they can -- # compile against it by using find_package() and defining the -- # `curl_project` target. -- -DHTTP_ONLY=ON -- -DCMAKE_ENABLE_OPENSSL=ON -- -DENABLE_ARES=ON -- -DCMAKE_DEBUG_POSTFIX= -- BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} -- LOG_DOWNLOAD ON -- LOG_CONFIGURE ON -- LOG_BUILD ON -- LOG_INSTALL ON) --endif () -diff --git a/super/external/googletest.cmake b/super/external/googletest.cmake -index a4e5220e0..f7cd15b61 100644 ---- a/super/external/googletest.cmake -+++ b/super/external/googletest.cmake -@@ -40,9 +40,13 @@ if (NOT TARGET googletest-project) - -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} - -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} - -DCMAKE_INSTALL_PREFIX= -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/grpc.cmake b/super/external/grpc.cmake -index 91f70eaa9..3cac5cfef 100644 ---- a/super/external/grpc.cmake -+++ b/super/external/grpc.cmake -@@ -17,7 +17,6 @@ - include(ExternalProjectHelper) - include(external/abseil) - include(external/c-ares) --include(external/ssl) - include(external/protobuf) - - if (NOT TARGET grpc-project) -@@ -34,7 +33,7 @@ if (NOT TARGET grpc-project) - include(ExternalProject) - ExternalProject_Add( - grpc-project -- DEPENDS c-ares-project protobuf-project ssl-project abseil-cpp-project -+ DEPENDS c-ares-project protobuf-project abseil-cpp-project - EXCLUDE_FROM_ALL ON - PREFIX "${CMAKE_BINARY_DIR}/external/grpc" - INSTALL_DIR "${GOOGLE_CLOUD_CPP_EXTERNAL_PREFIX}" -@@ -52,9 +51,20 @@ if (NOT TARGET grpc-project) - -DgRPC_PROTOBUF_PROVIDER=package - -DgRPC_SSL_PROVIDER=package - -DgRPC_ZLIB_PROVIDER=package -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF -+ -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF -+ -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF -+ -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF -+ -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF -+ -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF -+ -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR} -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/protobuf.cmake b/super/external/protobuf.cmake -index 7cc0aae1e..9351ef34f 100644 ---- a/super/external/protobuf.cmake -+++ b/super/external/protobuf.cmake -@@ -17,7 +17,6 @@ - find_package(Threads REQUIRED) - - include(ExternalProjectHelper) --include(external/zlib) - - if (NOT TARGET protobuf-project) - # Give application developers a hook to configure the version and hash -@@ -33,7 +32,6 @@ if (NOT TARGET protobuf-project) - include(ExternalProject) - ExternalProject_Add( - protobuf-project -- DEPENDS zlib-project - EXCLUDE_FROM_ALL ON - PREFIX "${CMAKE_BINARY_DIR}/external/protobuf" - INSTALL_DIR "${GOOGLE_CLOUD_CPP_EXTERNAL_PREFIX}" -@@ -51,9 +49,13 @@ if (NOT TARGET protobuf-project) - -Dprotobuf_DEBUG_POSTFIX= - -H/cmake - -B -+ -DCMAKE_CXX_FLAGS=-fPIC -+ -DCMAKE_C_FLAGS=-fPIC -+ -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON -- LOG_INSTALL ON) -+ LOG_INSTALL ON -+ LOG_OUTPUT_ON_FAILURE ON) - endif () -diff --git a/super/external/ssl.cmake b/super/external/ssl.cmake -deleted file mode 100644 -index 251eed2da..000000000 ---- a/super/external/ssl.cmake -+++ /dev/null -@@ -1,26 +0,0 @@ --# ~~~ --# Copyright 2018 Google LLC --# --# Licensed under the Apache License, Version 2.0 (the "License"); --# you may not use this file except in compliance with the License. --# You may obtain a copy of the License at --# --# http://www.apache.org/licenses/LICENSE-2.0 --# --# Unless required by applicable law or agreed to in writing, software --# distributed under the License is distributed on an "AS IS" BASIS, --# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --# See the License for the specific language governing permissions and --# limitations under the License. --# ~~~ -- --if (NOT TARGET ssl-project) -- # For OpenSSL we don't really support external projects. OpenSSL build -- # system is notoriously finicky. Use vcpkg on Windows, or install OpenSSL -- # using your operating system packages instead. -- # -- # This file is here to simplify the definition of external projects, such as -- # curl and gRPC, that depend on a SSL library. -- find_package(OpenSSL REQUIRED) -- add_custom_target(ssl-project) --endif () -diff --git a/super/external/zlib.cmake b/super/external/zlib.cmake -deleted file mode 100644 -index f9df5e4cc..000000000 ---- a/super/external/zlib.cmake -+++ /dev/null -@@ -1,46 +0,0 @@ --# ~~~ --# Copyright 2018 Google LLC --# --# Licensed under the Apache License, Version 2.0 (the "License"); --# you may not use this file except in compliance with the License. --# You may obtain a copy of the License at --# --# http://www.apache.org/licenses/LICENSE-2.0 --# --# Unless required by applicable law or agreed to in writing, software --# distributed under the License is distributed on an "AS IS" BASIS, --# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --# See the License for the specific language governing permissions and --# limitations under the License. --# ~~~ -- --if (NOT TARGET zlib-project) -- # Give application developers a hook to configure the version and hash -- # downloaded from GitHub. -- set(GOOGLE_CLOUD_CPP_ZLIB_URL -- "https://github.com/madler/zlib/archive/v1.2.11.tar.gz") -- set(GOOGLE_CLOUD_CPP_ZLIB_SHA256 -- "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff") -- -- set_external_project_build_parallel_level(PARALLEL) -- set_external_project_vars() -- -- include(ExternalProject) -- ExternalProject_Add( -- zlib-project -- EXCLUDE_FROM_ALL ON -- PREFIX "${CMAKE_BINARY_DIR}/external/zlib" -- INSTALL_DIR "${GOOGLE_CLOUD_CPP_EXTERNAL_PREFIX}" -- URL ${GOOGLE_CLOUD_CPP_ZLIB_URL} -- URL_HASH SHA256=${GOOGLE_CLOUD_CPP_ZLIB_SHA256} -- LIST_SEPARATOR | -- CMAKE_ARGS ${GOOGLE_CLOUD_CPP_EXTERNAL_PROJECT_CMAKE_FLAGS} -- -DCMAKE_PREFIX_PATH=${GOOGLE_CLOUD_CPP_PREFIX_PATH} -- -DCMAKE_INSTALL_RPATH=${GOOGLE_CLOUD_CPP_INSTALL_RPATH} -- -DCMAKE_INSTALL_PREFIX= -- BUILD_COMMAND ${CMAKE_COMMAND} --build ${PARALLEL} -- LOG_DOWNLOAD ON -- LOG_CONFIGURE ON -- LOG_BUILD ON -- LOG_INSTALL ON) --endif () --- -2.30.0 - diff --git a/cmake/inputs/patches/ep_gcssdk/disable_examples.patch b/cmake/inputs/patches/ep_gcssdk/disable_examples.patch deleted file mode 100644 index a6c4afce7f8e..000000000000 --- a/cmake/inputs/patches/ep_gcssdk/disable_examples.patch +++ /dev/null @@ -1,35 +0,0 @@ -From 95e54c03d877f89cab681e8ac3c747ca017a22ea Mon Sep 17 00:00:00 2001 -From: Seth Shelnutt -Date: Fri, 22 May 2020 15:29:11 -0400 -Subject: [PATCH] All cloud examples should be wrapped in BUILD_TESTING - ---- -google/cloud/examples/CMakeLists.txt | 10 ++--- - 2 files changed, 41 insertions(+), 39 deletions(-) - -diff --git a/google/cloud/examples/CMakeLists.txt b/google/cloud/examples/CMakeLists.txt -index 5af4627b4..dc052168b 100644 ---- a/google/cloud/examples/CMakeLists.txt -+++ b/google/cloud/examples/CMakeLists.txt -@@ -15,13 +15,13 @@ - # ~~~ - - # Pick the right MSVC runtime libraries. --include(SelectMSVCRuntime) -+if (BUILD_TESTING) -+ include(SelectMSVCRuntime) - --add_executable(gcs2cbt gcs2cbt.cc) --target_link_libraries(gcs2cbt bigtable_client storage_client -+ add_executable(gcs2cbt gcs2cbt.cc) -+ target_link_libraries(gcs2cbt bigtable_client storage_client - google_cloud_cpp_grpc_utils) --google_cloud_cpp_add_common_options(gcs2cbt) --if (BUILD_TESTING) -+ google_cloud_cpp_add_common_options(gcs2cbt) - add_test(NAME gcs2cbt COMMAND gcs2cbt) - set_tests_properties( - gcs2cbt PROPERTIES LABELS --- -2.26.2 - diff --git a/cmake/inputs/patches/ep_gcssdk/disable_tests.patch b/cmake/inputs/patches/ep_gcssdk/disable_tests.patch deleted file mode 100644 index 898d2bb92497..000000000000 --- a/cmake/inputs/patches/ep_gcssdk/disable_tests.patch +++ /dev/null @@ -1,23 +0,0 @@ -From 4749df2b03e70218870dab48b83d090af2451027 Mon Sep 17 00:00:00 2001 -From: Seth Shelnutt -Date: Fri, 22 May 2020 14:06:41 -0400 -Subject: [PATCH] Add option to disable tests - ---- - CMakeLists.txt | 1 + - super/CMakeLists.txt | 1 + - 2 files changed, 2 insertions(+) - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index c9275541b..0d0058b27 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -143,6 +143,7 @@ mark_as_advanced(GOOGLE_CLOUD_CPP_ENABLE_GRPC_UTILS) - - # Enable testing in this directory so we can do a top-level `make test`. This - # also includes the BUILD_TESTING option, which is on by default. -+option(BUILD_TESTING "" OFF) - include(CTest) - - # Each subproject adds dependencies to this target to have their docs generated. -2.26.2 diff --git a/cmake/inputs/patches/ep_gcssdk/v1.22.0.CMakelists.txt.openssl3md5deprecationmitigation.patch b/cmake/inputs/patches/ep_gcssdk/v1.22.0.CMakelists.txt.openssl3md5deprecationmitigation.patch deleted file mode 100644 index 6fff9593c135..000000000000 --- a/cmake/inputs/patches/ep_gcssdk/v1.22.0.CMakelists.txt.openssl3md5deprecationmitigation.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 11327f7c..bdcef236 100755 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -23,6 +23,10 @@ project( - VERSION 1.22.0 - LANGUAGES CXX C) - -+if(UNIX) -+add_compile_options("-Wno-deprecated-declarations") -+endif() -+ - # Configure the Compiler options, we use C++11 features by default. - set(GOOGLE_CLOUD_CPP_CXX_STANDARD - "" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6b844e322e19..fec08249ff7d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -406,6 +406,3 @@ add_subdirectory(ci) if(WIN32) add_subdirectory(performance) endif() - -#add tests related to external items -add_subdirectory(external) diff --git a/test/external/CMakeLists.txt b/test/external/CMakeLists.txt deleted file mode 100644 index d7a7a0a66cc3..000000000000 --- a/test/external/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ -# -# test/external/CMakeLists.txt -# -# -# The MIT License -# -# Copyright (c) 2022 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -if(NOT TILEDB_CRC32) - message("skipping target unit_link_crc32, found NOT TILEDB_CRC32") -else() - message("adding target unit_link_absl, have TILEDB_ABSEIL") - - add_executable(unit_link_crc32 EXCLUDE_FROM_ALL src/crc32_link_test.cc) - - find_package(Crc32c CONFIG REQUIRED - PATHS - ${TILEDB_EP_INSTALL_PREFIX}/lib/cmake - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) - - target_link_libraries(unit_link_crc32 PRIVATE Crc32c::crc32c) - - target_include_directories(unit_link_crc32 PRIVATE ${TILEDB_EP_INSTALL_PREFIX} ${TILEDB_EP_INSTALL_PREFIX}/include ) -endif() diff --git a/test/external/src/absl_library_targets.cmake b/test/external/src/absl_library_targets.cmake deleted file mode 100644 index 5115dd5127f5..000000000000 --- a/test/external/src/absl_library_targets.cmake +++ /dev/null @@ -1,171 +0,0 @@ -# -# test/external/CMakeLists.txt -# -# -# The MIT License -# -# Copyright (c) 2022 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# basic list obtained from an installed abslTargets.cmake foreach(expectedTarget ...) -# at the time this was created, abseil build process created absl:: namespaced target -# for each individual library built. -# -# update list as needed if changes in abseil require - -set(TILEDB_ABSL_LIBRARY_TARGETS - absl::algorithm - absl::algorithm_container - absl::any - absl::atomic_hook - absl::bad_any_cast - absl::bad_any_cast_impl - absl::bad_optional_access - absl::bad_variant_access - absl::base - absl::base_internal - absl::bind_front - absl::bits - absl::btree - absl::city - absl::civil_time - absl::cleanup - absl::cleanup_internal - absl::compare - absl::compressed_tuple - absl::config - absl::container_common - absl::container_memory - absl::cord - absl::cord_internal - absl::cordz_functions - absl::cordz_handle - absl::cordz_info - absl::cordz_sample_token - absl::cordz_statistics - absl::cordz_update_scope - absl::cordz_update_tracker - absl::core_headers - absl::counting_allocator - absl::debugging - absl::debugging_internal - absl::demangle_internal - absl::dynamic_annotations - absl::endian - absl::errno_saver - absl::examine_stack - absl::exponential_biased - absl::failure_signal_handler - absl::fast_type_id - absl::fixed_array - absl::flags - absl::flags_commandlineflag - absl::flags_commandlineflag_internal - absl::flags_config - absl::flags_internal - absl::flags_marshalling - absl::flags_parse - absl::flags_path_util - absl::flags_private_handle_accessor - absl::flags_program_name - absl::flags_reflection - absl::flags_usage - absl::flags_usage_internal - absl::flat_hash_map - absl::flat_hash_set - absl::function_ref - absl::graphcycles_internal - absl::hash - absl::hash_function_defaults - absl::hash_policy_traits - absl::hashtable_debug - absl::hashtable_debug_hooks - absl::hashtablez_sampler - absl::have_sse - absl::inlined_vector - absl::inlined_vector_internal - absl::int128 - absl::kernel_timeout_internal - absl::layout - absl::leak_check - absl::leak_check_disable - absl::log_severity - absl::low_level_hash - absl::malloc_internal - absl::memory - absl::meta - absl::node_hash_map - absl::node_hash_policy - absl::node_hash_set - absl::numeric - absl::numeric_representation - absl::optional - absl::periodic_sampler - absl::pretty_function - absl::random_bit_gen_ref - absl::random_distributions - absl::random_internal_distribution_caller - absl::random_internal_distribution_test_util - absl::random_internal_fast_uniform_bits - absl::random_internal_fastmath - absl::random_internal_generate_real - absl::random_internal_iostream_state_saver - absl::random_internal_mock_helpers - absl::random_internal_nonsecure_base - absl::random_internal_pcg_engine - absl::random_internal_platform - absl::random_internal_pool_urbg - absl::random_internal_randen - absl::random_internal_randen_engine - absl::random_internal_randen_hwaes - absl::random_internal_randen_hwaes_impl - absl::random_internal_randen_slow - absl::random_internal_salted_seed_seq - absl::random_internal_seed_material - absl::random_internal_traits - absl::random_internal_uniform_helper - absl::random_internal_wide_multiply - absl::random_random - absl::random_seed_gen_exception - absl::random_seed_sequences - absl::raw_hash_map - absl::raw_hash_set - absl::raw_logging_internal - absl::sample_recorder - absl::scoped_set_env - absl::span - absl::spinlock_wait - absl::stacktrace - absl::status - absl::statusor - absl::str_format - absl::str_format_internal - absl::strerror - absl::strings - absl::strings_internal - absl::symbolize - absl::synchronization - absl::throw_delegate - absl::time - absl::time_zone - absl::type_traits - absl::utility - absl::variant -) diff --git a/test/external/src/absl_link_test.cc b/test/external/src/absl_link_test.cc deleted file mode 100644 index 0b7818f82e00..000000000000 --- a/test/external/src/absl_link_test.cc +++ /dev/null @@ -1,159 +0,0 @@ -// clang-format off -/** - * @file absl_lilnk_test.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for linkage to libraries of the targets found to be referenced by - * gcssdk v1.22 at the time of creation of absl_link_test.cc. - * - * Those targets were crudely identified by extracting from the following action: - * ep_gcssdk\google>findstr /s absl:: *cmake* -cloud\bigquery\CMakeLists.txt: PUBLIC absl::memory absl::strings google_cloud_cpp_grpc_utils -cloud\bigtable\CMakeLists.txt: PUBLIC absl::memory -cloud\bigtable\CMakeLists.txt: PUBLIC absl::memory -cloud\bigtable\CMakeLists.txt: PRIVATE absl::memory -cloud\bigtable\examples\CMakeLists.txt: absl::time -cloud\bigtable\examples\CMakeLists.txt: absl::time -cloud\CMakeLists.txt: google_cloud_cpp_common PUBLIC absl::flat_hash_map absl::memory -cloud\CMakeLists.txt: absl::optional absl::time Threads::Threads) -cloud\CMakeLists.txt: absl::variant GTest::gmock_main GTest::gmock GTest::gtest) -cloud\CMakeLists.txt:# runtime deps of the absl::optional target (http://github.com/abseil/abseil-cpp -cloud\CMakeLists.txt: PUBLIC absl::function_ref -cloud\CMakeLists.txt: absl::memory -cloud\CMakeLists.txt: absl::time -cloud\CMakeLists.txt: absl::variant -cloud\pubsub\benchmarks\CMakeLists.txt: absl::str_format -cloud\pubsub\CMakeLists.txt: googleapis-c++::pubsub_protos absl::flat_hash_map) -cloud\pubsub\CMakeLists.txt: absl::str_format -cloud\spanner\CMakeLists.txt: PUBLIC absl::fixed_array -cloud\spanner\CMakeLists.txt: absl::memory -cloud\spanner\CMakeLists.txt: absl::numeric -cloud\spanner\CMakeLists.txt: absl::strings -cloud\spanner\CMakeLists.txt: absl::time -cloud\spanner\CMakeLists.txt: absl::memory -cloud\spanner\CMakeLists.txt: absl::numeric -cloud\storage\benchmarks\CMakeLists.txt: absl::strings -cloud\storage\benchmarks\CMakeLists.txt: absl::strings -cloud\storage\CMakeLists.txt: PUBLIC absl::memory -cloud\storage\CMakeLists.txt: absl::strings -cloud\storage\CMakeLists.txt: absl::str_format -cloud\storage\CMakeLists.txt: absl::time -cloud\storage\CMakeLists.txt: absl::variant -cloud\storage\CMakeLists.txt: absl::strings -cloud\storage\CMakeLists.txt: PUBLIC absl::memory -cloud\storage\CMakeLists.txt: PRIVATE absl::memory -cloud\storage\tests\CMakeLists.txt: absl::strings -cloud\testing_util\CMakeLists.txt: PUBLIC absl::symbolize absl::failure_signal_handler - */ -// clang-format on - -#include -#include -#include -#include - -#include // absl::function_ref -#include // absl::fixed_array -#include // absl::flat_hash_map -#include -#include // absl::function_ref -#include // absl::numeric -#include -#include // absl::str_format -#include -#include -#include // absl::optional -#include // absl::variant -#if 0 -// absl::symbolize, absl::failure_signal_handler both in testing_util, not -// verifying these as we do not build gcs tests. -#include a) { - printf( - "Unexpected b < a... (%s) < a (%s)\n", - absl::FormatTime(b).c_str(), - absl::FormatTime(a).c_str()); - } - } - { // hash.h - bool is_hashable = std::is_default_constructible>(); - printf("is_hashable(), %d\n", is_hashable); - } - { // optional - absl::optional empty; - if (!empty) { - printf("empty !empty?\n"); - } - if (empty.has_value()) { - printf("empty unexpectedly has value!\n"); - } - } - { // variant - absl::variant x; - if (x.index()) { - printf("x.index() unexpectedly != zero\n"); - } - } - { // str_format - auto fmtd = absl::StrFormat("%d", 123); - if (fmtd != std::string("123")) { - printf("must not work like I guessed, \"%s\" != \"123\"\n", fmtd.c_str()); - } - } - { // fixed_array - absl::FixedArray fa(5); - *fa.begin() = atoi("99753"); - printf("fa.begin() %d, fa[0] %d\n", *fa.begin(), fa[0]); - } - { // numeric - absl::uint128 u128{1299.3}; - std::stringstream sstr; - sstr << "u128 " << u128; - printf("%s\n", sstr.str().c_str()); - } - - return 0; -} diff --git a/test/external/src/crc32_link_test.cc b/test/external/src/crc32_link_test.cc deleted file mode 100644 index b8cc4195a678..000000000000 --- a/test/external/src/crc32_link_test.cc +++ /dev/null @@ -1,45 +0,0 @@ -/** - * @file crc32_lilnk_test.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for linkage to Crc32c - */ - -#include - -#include - -int main() { - uint8_t buf[32]; - - printf( - "crc32 of indeterminate buffer is %u\n", - crc32c::Crc32c(buf, sizeof(buf))); - - return 0; -} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 123c0a726d6c..ffee1c1fd6d2 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -661,19 +661,6 @@ target_link_libraries(TILEDB_CORE_OBJECTS_ILIB libmagic ) -if(TILEDB_CRC32) - find_package(crc32c_EP REQUIRED) - find_package(crc32c CONFIG REQUIRED - HINTS - ${TILEDB_EP_INSTALL_PREFIX}/lib/cmake - ) - - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - Crc32c::crc32c - ) -endif() - if(TILEDB_WEBP) find_package(Webp_EP REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB diff --git a/tiledb/sm/compressors/CMakeLists.txt b/tiledb/sm/compressors/CMakeLists.txt index 8ebe27f5102d..b2c694c3e05f 100644 --- a/tiledb/sm/compressors/CMakeLists.txt +++ b/tiledb/sm/compressors/CMakeLists.txt @@ -61,31 +61,6 @@ target_include_directories(tdb_gzip_embedded_data PRIVATE $ ) -if (TILEDB_WEBP) - # - # perform webp build and link test for some webp routines - # - find_package(Webp_EP REQUIRED) - add_executable(unit_link_webp) - if (WIN32) - target_link_libraries(unit_link_webp PRIVATE WebP::webp WebP::webpdecoder WebP::webpdemux WebP::libwebpmux) - else() #if (NOT WIN32) - # On Linux, must explicitly link -lpthread -ldl in order for static linking - # to libzstd or libcurl to work. - target_link_libraries(unit_link_webp WebP::webp WebP::webpdecoder WebP::webpdemux WebP::libwebpmux pthread dl) - endif() - target_sources( - unit_link_webp - PRIVATE - test/link_webp_test1.cc - ) - - add_test(NAME "unit_link_webp" - COMMAND "$" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) -endif() - # Link to Threads::Threads if library or flag needed to enable threading. find_package(Threads REQUIRED) diff --git a/tiledb/sm/compressors/test/link_webp_test1.cc b/tiledb/sm/compressors/test/link_webp_test1.cc deleted file mode 100644 index cfd976a2e6e7..000000000000 --- a/tiledb/sm/compressors/test/link_webp_test1.cc +++ /dev/null @@ -1,348 +0,0 @@ -// stripped to point of non-useability to demonstrate linkage success -// based on cwebp.cc from -// https://chromium.googlesource.com/webm/libwebp -// commit a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Use of this source code is governed by a BSD-style license -// that can be found in the COPYING file in the root of the source -// tree. An additional intellectual property rights grant can be found -// in the file PATENTS. All contributing project authors may -// be found in the AUTHORS file in the root of the source tree. -// ----------------------------------------------------------------------------- -// -// simple command line calling the WebPEncode function. -// Encodes a raw .YUV into WebP bitstream -// -// Author: Skal (pascal.massimino@gmail.com) - -#include -#include -#include -#include - -#include "webp/decode.h" -#include "webp/encode.h" -#include "webp/mux_types.h" -#include "webp/types.h" - -//------------------------------------------------------------------------------ - -static void AllocExtraInfo(WebPPicture* const pic) { - const int mb_w = (pic->width + 15) / 16; - const int mb_h = (pic->height + 15) / 16; - pic->extra_info = - (uint8_t*)WebPMalloc(mb_w * mb_h * sizeof(*pic->extra_info)); -} - -// ----------------------------------------------------------------------------- -// Metadata writing. - -enum { - METADATA_EXIF = (1 << 0), - METADATA_ICC = (1 << 1), - METADATA_XMP = (1 << 2), - METADATA_ALL = METADATA_EXIF | METADATA_ICC | METADATA_XMP -}; - -//------------------------------------------------------------------------------ - -static int ProgressReport(int percent, const WebPPicture* const picture) { - fprintf(stderr, "[%s]: %3d %% \r", (char*)picture->user_data, percent); - return 1; // all ok -} - -//------------------------------------------------------------------------------ - -static void HelpShort(void) { - printf("Usage:\n\n"); - printf(" cwebp [options] -q quality input.png -o output.webp\n\n"); - printf("where quality is between 0 (poor) to 100 (very good).\n"); - printf("Typical value is around 80.\n\n"); - printf("Try -longhelp for an exhaustive list of advanced options.\n"); -} - -//------------------------------------------------------------------------------ -// Error messages - -static const char* const kErrorMessages[VP8_ENC_ERROR_LAST] = { - "OK", - "OUT_OF_MEMORY: Out of memory allocating objects", - "BITSTREAM_OUT_OF_MEMORY: Out of memory re-allocating byte buffer", - "NULL_PARAMETER: NULL parameter passed to function", - "INVALID_CONFIGURATION: configuration is invalid", - "BAD_DIMENSION: Bad picture dimension. Maximum width and height " - "allowed is 16383 pixels.", - "PARTITION0_OVERFLOW: Partition #0 is too big to fit 512k.\n" - "To reduce the size of this partition, try using less segments " - "with the -segments option, and eventually reduce the number of " - "header bits using -partition_limit. More details are available " - "in the manual (`man cwebp`)", - "PARTITION_OVERFLOW: Partition is too big to fit 16M", - "BAD_WRITE: Picture writer returned an I/O error", - "FILE_TOO_BIG: File would be too big to fit in 4G", - "USER_ABORT: encoding abort requested by user"}; - -//------------------------------------------------------------------------------ - -uint64_t reference_a_value(uint64_t v) { - return v + 1; -} - -#ifdef _MSC_VER -// do outside of function, because... -//"Use of the warning pragma in the function to change the state of a warning -// number larger than 4699 only takes effect after the end of the function." -#pragma warning(disable : 4701) // 'config', 'picture_no_alpha' -#endif - -int main(int argc, const char* argv[]) { - int return_value = -1; - const char *in_file = NULL, *out_file = NULL; - FILE* out = NULL; - int short_output = 0; - int quiet = 0; - int blend_alpha = 0; - uint32_t background_color = 0xffffffu; - int crop = 0, crop_x = 0, crop_y = 0, crop_w = 0, crop_h = 0; - int resize_w = 0, resize_h = 0; - int lossless_preset = 6; - int use_lossless_preset = -1; // -1=unset, 0=don't use, 1=use it - int show_progress = 0; - int keep_metadata = 0; - WebPPicture picture; - int print_distortion = -1; // -1=off, 0=PSNR, 1=SSIM, 2=LSIM - WebPPicture original_picture; // when PSNR or SSIM is requested - WebPConfig config; - WebPAuxStats stats; - WebPMemoryWriter memory_writer; - int use_memory_writer; - WebPMemoryWriterInit(&memory_writer); - if (!WebPPictureInit(&picture) || !WebPPictureInit(&original_picture) || - !WebPConfigInit(&config)) { - fprintf(stderr, "Error! Version mismatch!\n"); - } - { // webpdecoder - // *** turns out, there is -NO- symbol in webpdecoder that is not also in - // webp! left here anyway, in case libraries are ever de-dupped. - const int dec_version = WebPGetDecoderVersion(); - - // webp - // in testing, even when WebP::webp was -not- added as link item, this - // library was still being placed in link_webp_test project. - auto enc_version = WebPGetEncoderVersion(); - - reference_a_value(dec_version + enc_version); - } - - if (argc == 1) { - HelpShort(); - return 0; - } - // add some non-determinism re in_file so compiler can't, seeing that - // in_file init'd to NULL with no chance of change, avoid generating - // code (vs2019 was not including any of that code) - // following the if (in_file == NULL) check below, tho if - // anyone calls program with argument program will likely fail on - // that input in its current state... - else { - in_file = argv[1]; - } - - if (in_file == NULL) { - fprintf(stderr, "No input file specified!\n"); - HelpShort(); - // Exercising this stripped executable in build, don't report error - return 0; - // goto Error; - } - - if (use_lossless_preset == 1) { - if (!WebPConfigLosslessPreset(&config, lossless_preset)) { - fprintf(stderr, "Invalid lossless preset (-z %d)\n", lossless_preset); - goto Error; - } - } - - // Check for unsupported command line options for lossless mode and log - // warning for such options. - if (!quiet && config.lossless == 1) { - if (config.target_size > 0 || config.target_PSNR > 0) { - fprintf( - stderr, - "Encoding for specified size or PSNR is not supported" - " for lossless encoding. Ignoring such option(s)!\n"); - } - if (config.partition_limit > 0) { - fprintf( - stderr, - "Partition limit option is not required for lossless" - " encoding. Ignoring this option!\n"); - } - } - // If a target size or PSNR was given, but somehow the -pass option was - // omitted, force a reasonable value. - if (config.target_size > 0 || config.target_PSNR > 0) { - if (config.pass == 1) - config.pass = 6; - } - - if (!WebPValidateConfig(&config)) { - fprintf(stderr, "Error! Invalid configuration.\n"); - goto Error; - } - - // Read the input. We need to decide if we prefer ARGB or YUVA - // samples, depending on the expected compression mode (this saves - // some conversion steps). - picture.use_argb = - (config.lossless || config.use_sharp_yuv || config.preprocessing > 0 || - crop || (resize_w | resize_h) > 0); - picture.progress_hook = (show_progress && !quiet) ? ProgressReport : NULL; - - if (blend_alpha) { - WebPBlendAlpha(&picture, background_color); - } - - // The bitstream should be kept in memory when metadata must be appended - // before writing it to a file/stream, and/or when the near-losslessly encoded - // bitstream must be decoded for distortion computation (lossy will modify the - // 'picture' but not the lossless pipeline). - // Otherwise directly write the bitstream to a file. - use_memory_writer = (out_file != NULL && keep_metadata) || - (!quiet && print_distortion >= 0 && config.lossless && - config.near_lossless < 100); - - { - out = NULL; - if (use_memory_writer) { - picture.writer = WebPMemoryWrite; - picture.custom_ptr = (void*)&memory_writer; - } - if (!quiet && !short_output) { - fprintf(stderr, "No output file specified (no -o flag). Encoding will\n"); - fprintf(stderr, "be performed, but its results discarded.\n\n"); - } - } - if (!quiet) { - picture.stats = &stats; - picture.user_data = (void*)in_file; - } - - // Crop & resize. - if (crop != 0) { - // We use self-cropping using a view. - if (!WebPPictureView(&picture, crop_x, crop_y, crop_w, crop_h, &picture)) { - fprintf(stderr, "Error! Cannot crop picture\n"); - goto Error; - } - } - if ((resize_w | resize_h) > 0) { - WebPPicture picture_no_alpha; - if (config.exact) { - // If -exact, we can't premultiply RGB by A otherwise RGB is lost if A=0. - // We rescale an opaque copy and assemble scaled A and non-premultiplied - // RGB channels. This is slower but it's a very uncommon use case. Color - // leak at sharp alpha edges is possible. - if (!WebPPictureCopy(&picture, &picture_no_alpha)) { - fprintf(stderr, "Error! Cannot copy temporary picture\n"); - goto Error; - } - - // We enforced picture.use_argb = 1 above. Now, remove the alpha values. - { - int x, y; - uint32_t* argb_no_alpha = picture_no_alpha.argb; - for (y = 0; y < picture_no_alpha.height; ++y) { - for (x = 0; x < picture_no_alpha.width; ++x) { - argb_no_alpha[x] |= 0xff000000; // Opaque copy. - } - argb_no_alpha += picture_no_alpha.argb_stride; - } - } - - if (!WebPPictureRescale(&picture_no_alpha, resize_w, resize_h)) { - fprintf(stderr, "Error! Cannot resize temporary picture\n"); - goto Error; - } - } - - if (!WebPPictureRescale(&picture, resize_w, resize_h)) { - fprintf(stderr, "Error! Cannot resize picture\n"); - goto Error; - } - - if (config.exact) { // Put back the alpha information. - int x, y; - uint32_t* argb_no_alpha = picture_no_alpha.argb; - uint32_t* argb = picture.argb; - for (y = 0; y < picture_no_alpha.height; ++y) { - for (x = 0; x < picture_no_alpha.width; ++x) { - argb[x] = (argb[x] & 0xff000000) | (argb_no_alpha[x] & 0x00ffffff); - } - argb_no_alpha += picture_no_alpha.argb_stride; - argb += picture.argb_stride; - } - WebPPictureFree(&picture_no_alpha); - } - } - - if (picture.extra_info_type > 0) { - AllocExtraInfo(&picture); - } - // Save original picture for later comparison. Only for lossy as lossless does - // not modify 'picture' (even near-lossless). - if (print_distortion >= 0 && !config.lossless && - !WebPPictureCopy(&picture, &original_picture)) { - fprintf(stderr, "Error! Cannot copy temporary picture\n"); - goto Error; - } - - // Compress. - if (!WebPEncode(&config, &picture)) { - fprintf(stderr, "Error! Cannot encode picture as WebP\n"); - fprintf( - stderr, - "Error code: %d (%s)\n", - picture.error_code, - kErrorMessages[picture.error_code]); - goto Error; - } - - // Get the decompressed image for the lossless pipeline. - if (!quiet && print_distortion >= 0 && config.lossless) { - if (config.near_lossless == 100) { - // Pure lossless: image was not modified, make 'original_picture' a view - // of 'picture' by copying all members except the freeable pointers. - original_picture = picture; - original_picture.memory_ = original_picture.memory_argb_ = NULL; - } else { - // Decode the bitstream stored in 'memory_writer' to get the altered image - // to 'picture'; save the 'original_picture' beforehand. - assert(use_memory_writer); - original_picture = picture; - if (!WebPPictureInit(&picture)) { // Do not free 'picture'. - fprintf(stderr, "Error! Version mismatch!\n"); - goto Error; - } - - picture.use_argb = 1; - picture.stats = original_picture.stats; - } - original_picture.stats = NULL; - } - - return_value = 0; - -Error: - WebPMemoryWriterClear(&memory_writer); - WebPFree(picture.extra_info); - WebPPictureFree(&picture); - WebPPictureFree(&original_picture); - if (out != NULL && out != stdout) { - fclose(out); - } - return return_value; -} - -//------------------------------------------------------------------------------ diff --git a/vcpkg.json b/vcpkg.json index ce2bd65276df..d93075886980 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,7 +4,6 @@ "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", "dependencies": [ "bzip2", - "liblzma", "libmagic", "lz4", { @@ -16,15 +15,6 @@ "zstd" ], "features": { - "abseil": { - "description": "Enables building of abseil library and linkage test", - "dependencies": [ - { - "name": "abseil", - "version>=": "20211102.0" - } - ] - }, "azure": { "description": "Support Azure Blob Storage", "dependencies": [ From df88a4b9295870172b4b50c8ad467bcd76b074e2 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 30 Nov 2023 18:18:42 +0200 Subject: [PATCH 094/456] Move documentation generation and source formatting into dedicated CMake modules. (#4538) [SC-36914](https://app.shortcut.com/tiledb-inc/story/36914/move-the-doc-target-from-the-superbuild-to-the-main-build) --- TYPE: BUILD DESC: Move documentation generation and source formatting into dedicated CMake modules. --- .github/workflows/build-docs.yml | 11 +++--- CMakeLists.txt | 2 ++ cmake/Modules/Doxygen.cmake | 59 ++++++++++++++++++++++++++++++++ cmake/Modules/Format.cmake | 50 +++++++++++++++++++++++++++ cmake/TileDB-Superbuild.cmake | 59 -------------------------------- tiledb/doxygen/README.md | 2 +- 6 files changed, 118 insertions(+), 65 deletions(-) create mode 100644 cmake/Modules/Doxygen.cmake create mode 100644 cmake/Modules/Format.cmake diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 7da4e6f48e9c..1bcc9623790b 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -1,6 +1,9 @@ name: build-docs on: push: + branches: + - dev + - 'release-*' paths-ignore: - '.github/workflows/quarto-render.yml' - '_quarto.yml' @@ -18,11 +21,11 @@ on: - '**/.md' - 'tiledb/doxygen/source/*' - 'tiledb/sm/c_api/tiledb_version.h' + workflow_dispatch: jobs: build: runs-on: ubuntu-22.04 - if: ${{ startsWith(github.ref , 'refs/tags') != true && startsWith(github.ref , 'build-') != true }} timeout-minutes: 90 name: Build Docs steps: @@ -41,7 +44,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 - - name: Install Doxygen (linux only) + - name: Install Doxygen run: | set -e pipefail sudo apt-get update @@ -49,12 +52,10 @@ jobs: sudo apt-get install -y doxygen pip install virtualenv shell: bash - if: ${{ runner.os == 'Linux' }} - - name: Build Doxygen Docs (linux only)' + - name: Build Doxygen Docs run: | # Build the documentation (this does not deploy to RTD). cd $GITHUB_WORKSPACE/tiledb/doxygen; ./local-build.sh; shell: bash - if: ${{ runner.os == 'Linux' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 83d51e767ea4..23ac97ba3ccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,8 @@ include(CIConfig) include(BuildOptions) include(global-policies NO_POLICY_SCOPE) include(TileDBToolchain) +include(Doxygen) +include(Format) ############################################################ # Parse version file diff --git a/cmake/Modules/Doxygen.cmake b/cmake/Modules/Doxygen.cmake new file mode 100644 index 000000000000..a8a2467dc481 --- /dev/null +++ b/cmake/Modules/Doxygen.cmake @@ -0,0 +1,59 @@ +# +# Doxygen.cmake +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +########################################################### +# Doxygen documentation +########################################################### + +find_package(Doxygen) +if(DOXYGEN_FOUND) + file(GLOB_RECURSE TILEDB_C_API_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/*_api_external.h") + list(APPEND TILEDB_C_API_HEADERS + "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h" + "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/c_api/tiledb.h" + ) + file(GLOB TILEDB_CPP_API_HEADERS + "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/cpp_api/*.h" + ) + set(TILEDB_API_HEADERS ${TILEDB_C_API_HEADERS} ${TILEDB_CPP_API_HEADERS}) + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in + COMMAND mkdir -p doxygen + COMMAND echo INPUT = ${CMAKE_CURRENT_SOURCE_DIR}/tiledb/doxygen/mainpage.dox + ${TILEDB_API_HEADERS} > ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in + COMMENT "Preparing for Doxygen documentation" VERBATIM + ) + add_custom_target(doc + Doxygen::doxygen ${CMAKE_CURRENT_SOURCE_DIR}/tiledb/doxygen/Doxyfile.mk > + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile.log 2>&1 + COMMENT "Generating API documentation with Doxygen" VERBATIM + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in + ) +else(DOXYGEN_FOUND) + add_custom_target(doc + _______doc + COMMENT "!! Docs cannot be built. Please install Doxygen and re-run cmake. !!" VERBATIM + ) +endif(DOXYGEN_FOUND) diff --git a/cmake/Modules/Format.cmake b/cmake/Modules/Format.cmake new file mode 100644 index 000000000000..81a9fbb3e3c7 --- /dev/null +++ b/cmake/Modules/Format.cmake @@ -0,0 +1,50 @@ +# +# Format.cmake +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +############################################################ +# "make format" and "make check-format" targets +############################################################ + +set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts") + +find_package(ClangTools) +if (NOT ${CLANG_FORMAT_FOUND}) + find_program(CLANG_FORMAT_BIN NAMES clang-format-16) + if(CLANG_FORMAT_BIN) + set(CLANG_FORMAT_FOUND TRUE) + endif() +endif() +if (${CLANG_FORMAT_FOUND}) + message(STATUS "clang hunt, found ${CLANG_FORMAT_BIN}") + # runs clang format and updates files in place. + + add_custom_target(format ${SCRIPTS_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1) + + # runs clang format and exits with a non-zero exit code if any files need to be reformatted + add_custom_target(check-format ${SCRIPTS_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0) +else() + message(STATUS "was unable to find clang-format") +endif() diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index fc3943059bb9..fe60c1e0036a 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -190,62 +190,3 @@ if (TILEDB_TESTS) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tiledb ) endif() - -############################################################ -# "make format" and "make check-format" targets -############################################################ - -set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts") - -find_package(ClangTools) -if (NOT ${CLANG_FORMAT_FOUND}) - find_program(CLANG_FORMAT_BIN NAMES clang-format-16) - if(CLANG_FORMAT_BIN) - set(CLANG_FORMAT_FOUND TRUE) - endif() -endif() -if (${CLANG_FORMAT_FOUND}) - message(STATUS "clang hunt, found ${CLANG_FORMAT_BIN}") - # runs clang format and updates files in place. - - add_custom_target(format ${SCRIPTS_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 1) - - # runs clang format and exits with a non-zero exit code if any files need to be reformatted - add_custom_target(check-format ${SCRIPTS_DIR}/run-clang-format.sh ${CMAKE_CURRENT_SOURCE_DIR} ${CLANG_FORMAT_BIN} 0) -else() - message(STATUS "was unable to find clang-format") -endif() - -########################################################### -# Doxygen documentation -########################################################### - -find_package(Doxygen) -if(DOXYGEN_FOUND) - file(GLOB_RECURSE TILEDB_C_API_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/*_api_external.h") - list(APPEND TILEDB_C_API_HEADERS - "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h" - "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/c_api/tiledb.h" - ) - file(GLOB TILEDB_CPP_API_HEADERS - "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/cpp_api/*.h" - ) - set(TILEDB_API_HEADERS ${TILEDB_C_API_HEADERS} ${TILEDB_CPP_API_HEADERS}) - add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in - COMMAND mkdir -p doxygen - COMMAND echo INPUT = ${CMAKE_CURRENT_SOURCE_DIR}/tiledb/doxygen/mainpage.dox - ${TILEDB_API_HEADERS} > ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in - COMMENT "Preparing for Doxygen documentation" VERBATIM - ) - add_custom_target(doc - ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tiledb/doxygen/Doxyfile.mk > - ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile.log 2>&1 - COMMENT "Generating API documentation with Doxygen" VERBATIM - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/doxyfile.in - ) -else(DOXYGEN_FOUND) - add_custom_target(doc - _______doc - COMMENT "!! Docs cannot be built. Please install Doxygen and re-run cmake. !!" VERBATIM - ) -endif(DOXYGEN_FOUND) diff --git a/tiledb/doxygen/README.md b/tiledb/doxygen/README.md index 6afbc1d49e96..62ee029497ce 100644 --- a/tiledb/doxygen/README.md +++ b/tiledb/doxygen/README.md @@ -4,7 +4,7 @@ This directory contains the structure used to generate TileDB C and C++ API docu ## Build system integration -Doxygen is integrated in the build system via a `doc` target defined in [cmake/TileDB-Superbuild.cmake](../../cmake/TileDB-Superbuild.cmake). The `doc` target executes `doxygen` against an input file list generated by CMake into `/doxyfile.in`. `doxyfile.in` is included by `source/Doxyfile.mk`, which is specified as the config file in the CMake target definition for `doc`. This process extracts doc strings into XML files for further processing. +Doxygen is integrated in the build system via a `doc` target defined in [cmake/Modules/Doxygen.cmake](../../cmake/Modules/Doxygen.cmake). The `doc` target executes `doxygen` against an input file list generated by CMake into `/doxyfile.in`. `doxyfile.in` is included by `source/Doxyfile.mk`, which is specified as the config file in the CMake target definition for `doc`. This process extracts doc strings into XML files for further processing. ## Description of contents From 278c427f4a70c53091b57510a718cb188cacf10e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:16:43 -0500 Subject: [PATCH 095/456] Add constexpr for Filesystem support (#4548) One step toward `VFS` and `tiledb_unit` cleanup. In `vfs.h`, add `constexpr` to represent whether or not a filesystem is supported. Additionally, do some miscellaneous, related cleanup. --- TYPE: IMPROVEMENT DESC: Use constexpr to check for filesystem support --- test/src/unit-capi-vfs.cc | 73 ++-- test/src/unit-vfs.cc | 17 +- test/support/src/helpers.cc | 139 ++++---- test/support/src/helpers.h | 19 +- test/support/src/vfs_helpers.cc | 23 +- tiledb/sm/filesystem/vfs.cc | 595 ++++++++++++++------------------ tiledb/sm/filesystem/vfs.h | 78 ++++- 7 files changed, 434 insertions(+), 510 deletions(-) diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index 8e0a48453ac2..e71183e44f82 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -73,11 +73,6 @@ struct VFSFx { tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; - // Supported filesystems - bool supports_s3_; - bool supports_hdfs_; - bool supports_azure_; - // Functions VFSFx(); ~VFSFx(); @@ -92,12 +87,7 @@ struct VFSFx { }; VFSFx::VFSFx() { - // Supported filesystem vector - bool supports_gcs; // unused - get_supported_fs( - &supports_s3_, &supports_hdfs_, &supports_azure_, &supports_gcs); - - create_ctx_and_vfs(supports_s3_, supports_azure_, &ctx_, &vfs_); + create_ctx_and_vfs(&ctx_, &vfs_); } VFSFx::~VFSFx() { @@ -106,7 +96,7 @@ VFSFx::~VFSFx() { } void VFSFx::check_vfs(const std::string& path) { - if (supports_s3_) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { // Check S3 bucket functionality if (path == S3_TEMP_DIR) { int is_bucket = 0; @@ -239,29 +229,35 @@ void VFSFx::check_vfs(const std::string& path) { // Ls check_ls(path); - if (supports_s3_ && path == S3_TEMP_DIR) { - int is_empty; - rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!(bool)is_empty); + if constexpr (tiledb::sm::filesystem::s3_enabled) { + if (path == S3_TEMP_DIR) { + int is_empty; + rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!(bool)is_empty); + } } - if (!supports_s3_ && path != FILE_TEMP_DIR) { - rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); + if constexpr (!tiledb::sm::filesystem::s3_enabled) { + if (path != FILE_TEMP_DIR) { + rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); + REQUIRE(rc == TILEDB_OK); + } } - if (supports_s3_ && path == S3_TEMP_DIR) { - rc = tiledb_vfs_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); + if constexpr (tiledb::sm::filesystem::s3_enabled) { + if (path == S3_TEMP_DIR) { + rc = tiledb_vfs_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str()); + REQUIRE(rc == TILEDB_OK); - int is_empty; - rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); - REQUIRE(rc == TILEDB_OK); - REQUIRE((bool)is_empty); + int is_empty; + rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); + REQUIRE(rc == TILEDB_OK); + REQUIRE((bool)is_empty); - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); + rc = tiledb_vfs_remove_bucket(ctx_, vfs_, S3_BUCKET.c_str()); + REQUIRE(rc == TILEDB_OK); + } } } @@ -355,7 +351,7 @@ void VFSFx::check_move(const std::string& path) { REQUIRE(is_file); // Move from one bucket to another (only for S3) - if (supports_s3_) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { if (path == S3_TEMP_DIR) { std::string bucket2 = S3_PREFIX + random_name("tiledb") + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; @@ -387,8 +383,9 @@ void VFSFx::check_move(const std::string& path) { #ifndef _WIN32 void VFSFx::check_copy(const std::string& path) { // Do not support HDFS - if (supports_hdfs_) + if constexpr (tiledb::sm::filesystem::hdfs_enabled) { return; + } // Copy file auto file = path + "file"; @@ -467,7 +464,7 @@ void VFSFx::check_copy(const std::string& path) { REQUIRE(is_file); // Copy from one bucket to another (only for S3) - if (supports_s3_) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { std::string bucket2 = S3_PREFIX + random_name("tiledb") + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; std::string file3 = subdir3 + "file2"; @@ -811,11 +808,11 @@ TEST_CASE_METHOD(VFSFx, "C API: Test virtual filesystem", "[capi][vfs]") { tiledb_stats_enable(); tiledb_stats_reset(); - if (supports_s3_) + if constexpr (tiledb::sm::filesystem::s3_enabled) { check_vfs(S3_TEMP_DIR); - else if (supports_hdfs_) + } else if constexpr (tiledb::sm::filesystem::hdfs_enabled) { check_vfs(HDFS_TEMP_DIR); - else { + } else { check_vfs(FILE_TEMP_DIR); check_vfs(MEMFS_TEMP_DIR); } @@ -825,7 +822,7 @@ TEST_CASE_METHOD( VFSFx, "C API: Test virtual filesystem when S3 is not supported", "[capi][vfs]") { - if (!supports_s3_) { + if constexpr (!tiledb::sm::filesystem::s3_enabled) { tiledb_vfs_t* vfs; int rc = tiledb_vfs_alloc(ctx_, nullptr, &vfs); REQUIRE(rc == TILEDB_OK); @@ -890,9 +887,9 @@ TEST_CASE_METHOD(VFSFx, "C API: Test VFS parallel I/O", "[capi][vfs]") { TILEDB_OK); REQUIRE(error == nullptr); - if (supports_s3_) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { check_vfs(S3_TEMP_DIR); - } else if (supports_hdfs_) { + } else if constexpr (tiledb::sm::filesystem::hdfs_enabled) { check_vfs(HDFS_TEMP_DIR); } else { check_vfs(FILE_TEMP_DIR); diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 48322383372a..366d5fd2ffe3 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2022 TileDB, Inc. + * @copyright Copyright (c) 2018-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -144,16 +144,9 @@ TEST_CASE("VFS: Test long posix paths", "[vfs]") { TEST_CASE("VFS: URI semantics", "[vfs][uri]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); - - bool s3_supported = false; - bool hdfs_supported = false; - bool azure_supported = false; - bool gcs_supported = false; - tiledb::test::get_supported_fs( - &s3_supported, &hdfs_supported, &azure_supported, &gcs_supported); - std::vector> root_pairs; - if (s3_supported) { + + if constexpr (tiledb::sm::filesystem::s3_enabled) { Config config; REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok()); REQUIRE(config.set("vfs.s3.scheme", "https").ok()); @@ -164,13 +157,13 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { URI("s3://" + tiledb::test::random_name("vfs") + "/"), std::move(config)); } - if (hdfs_supported) { + if constexpr (tiledb::sm::filesystem::hdfs_enabled) { Config config; root_pairs.emplace_back( URI("hdfs:///" + tiledb::test::random_name("vfs") + "/"), std::move(config)); } - if (azure_supported) { + if constexpr (tiledb::sm::filesystem::azure_enabled) { Config config; REQUIRE( config.set("vfs.azure.storage_account_name", "devstoreaccount1").ok()); diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index fa9f61abecfa..fdca6bbd4987 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -58,8 +58,7 @@ std::mutex catch2_macro_mutex; -namespace tiledb { -namespace test { +namespace tiledb::test { // Command line arguments. std::string g_vfs; @@ -784,17 +783,13 @@ void create_azure_container( } } -void create_ctx_and_vfs( - bool s3_supported, - bool azure_supported, - tiledb_ctx_t** ctx, - tiledb_vfs_t** vfs) { +void create_ctx_and_vfs(tiledb_ctx_t** ctx, tiledb_vfs_t** vfs) { // Create TileDB context tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; throw_if_setup_failed(tiledb_config_alloc(&config, &error)); throw_if_setup_failed(error == nullptr); - if (s3_supported) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { #ifndef TILEDB_TESTS_AWS_S3_CONFIG throw_if_setup_failed(tiledb_config_set( config, "vfs.s3.endpoint_override", "localhost:9999", &error)); @@ -807,7 +802,7 @@ void create_ctx_and_vfs( throw_if_setup_failed(error == nullptr); #endif } - if (azure_supported) { + if constexpr (tiledb::sm::filesystem::azure_enabled) { throw_if_setup_failed(tiledb_config_set( config, "vfs.azure.storage_account_name", "devstoreaccount1", &error)); throw_if_setup_failed(tiledb_config_set( @@ -832,6 +827,61 @@ void create_ctx_and_vfs( tiledb_config_free(&config); } +void get_supported_fs( + bool* s3_supported, + bool* hdfs_supported, + bool* azure_supported, + bool* gcs_supported) { + // Override VFS support if the user used the '--vfs' command line argument. + if (g_vfs.empty()) { + *s3_supported = tiledb::sm::filesystem::s3_enabled; + *hdfs_supported = tiledb::sm::filesystem::hdfs_enabled; + *azure_supported = tiledb::sm::filesystem::azure_enabled; + *gcs_supported = tiledb::sm::filesystem::gcs_enabled; + } else { + if (!(g_vfs == "native" || g_vfs == "s3" || g_vfs == "hdfs" || + g_vfs == "azure" || g_vfs == "gcs")) { + throw std::runtime_error( + "Failed to get supported fs. Invalid --vfs command line argument."); + } + + if (g_vfs == "native") { + *s3_supported = false; + *hdfs_supported = false; + *azure_supported = false; + *gcs_supported = false; + } + + if (g_vfs == "s3") { + *s3_supported = true; + *hdfs_supported = false; + *azure_supported = false; + *gcs_supported = false; + } + + if (g_vfs == "hdfs") { + *s3_supported = false; + *hdfs_supported = true; + *azure_supported = false; + *gcs_supported = false; + } + + if (g_vfs == "azure") { + *s3_supported = false; + *hdfs_supported = false; + *azure_supported = true; + *gcs_supported = false; + } + + if (g_vfs == "gcs") { + *s3_supported = false; + *hdfs_supported = false; + *azure_supported = false; + *gcs_supported = true; + } + } +} + void create_dir(const std::string& path, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs) { remove_dir(path, ctx, vfs); REQUIRE(tiledb_vfs_create_dir(ctx, vfs, path.c_str()) == TILEDB_OK); @@ -914,73 +964,6 @@ void create_subarray( *returned_subarray = psubarray; } -void get_supported_fs( - bool* s3_supported, - bool* hdfs_supported, - bool* azure_supported, - bool* gcs_supported) { - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - - int is_supported = 0; - int rc = tiledb_ctx_is_supported_fs(ctx, TILEDB_S3, &is_supported); - REQUIRE(rc == TILEDB_OK); - *s3_supported = (bool)is_supported; - rc = tiledb_ctx_is_supported_fs(ctx, TILEDB_HDFS, &is_supported); - REQUIRE(rc == TILEDB_OK); - *hdfs_supported = (bool)is_supported; - rc = tiledb_ctx_is_supported_fs(ctx, TILEDB_AZURE, &is_supported); - REQUIRE(rc == TILEDB_OK); - *azure_supported = (bool)is_supported; - rc = tiledb_ctx_is_supported_fs(ctx, TILEDB_GCS, &is_supported); - REQUIRE(rc == TILEDB_OK); - *gcs_supported = (bool)is_supported; - - // Override VFS support if the user used the '--vfs' command line argument. - if (!g_vfs.empty()) { - REQUIRE( - (g_vfs == "native" || g_vfs == "s3" || g_vfs == "hdfs" || - g_vfs == "azure" || g_vfs == "gcs")); - - if (g_vfs == "native") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "s3") { - *s3_supported = true; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "hdfs") { - *s3_supported = false; - *hdfs_supported = true; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "azure") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = true; - *gcs_supported = false; - } - - if (g_vfs == "gcs") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = true; - } - } - - tiledb_ctx_free(&ctx); -} - void open_array( tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_query_type_t query_type) { int rc = tiledb_array_open(ctx, array, query_type); @@ -2375,6 +2358,4 @@ template void check_counts( template void check_counts( span vals, std::vector expected); -} // End of namespace test - -} // End of namespace tiledb +} // namespace tiledb::test diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index cbda05fcff8b..d1653c73e0b1 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -67,13 +67,11 @@ extern std::mutex catch2_macro_mutex; REQUIRE(a); \ } -namespace tiledb { - -namespace sm { +namespace tiledb::sm { class SubarrayPartitioner; } -namespace test { +namespace tiledb::test { // A dummy `Stats` instance. This is useful for constructing // objects that require a parent `Stats` object. These stats are @@ -485,19 +483,14 @@ void create_subarray( /** * Helper method that creates a TileDB context and a VFS object. * - * @param s3_supported Indicates whether S3 is supported or not. - * @param azure_supported Indicates whether Azure is supported or not. * @param ctx The TileDB context to be created. * @param vfs The VFS object to be created. */ -void create_ctx_and_vfs( - bool s3_supported, - bool azure_supported, - tiledb_ctx_t** ctx, - tiledb_vfs_t** vfs); +void create_ctx_and_vfs(tiledb_ctx_t** ctx, tiledb_vfs_t** vfs); /** * Helper function to get the supported filesystems. + * Supports VFS override via "--vfs" command line argument. * * @param s3_supported Set to `true` if S3 is supported. * @param hdfs_supported Set to `true` if HDFS is supported. @@ -1015,8 +1008,6 @@ void allocate_query_buffers_server_side( tiledb_query_t* query, ServerQueryBuffers& query_buffers); -} // End of namespace test - -} // End of namespace tiledb +} // namespace tiledb::test #endif diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index ef273dcaac59..fc16baa07892 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2020-2021 TileDB, Inc. + * @copyright Copyright (c) 2020-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,31 +46,23 @@ #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" -namespace tiledb { -namespace test { +namespace tiledb::test { std::vector> vfs_test_get_fs_vec() { std::vector> fs_vec; - - bool supports_s3 = false; - bool supports_hdfs = false; - bool supports_azure = false; - bool supports_gcs = false; - get_supported_fs( - &supports_s3, &supports_hdfs, &supports_azure, &supports_gcs); - if (supports_s3) { + if constexpr (tiledb::sm::filesystem::s3_enabled) { fs_vec.emplace_back(std::make_unique()); } - if (supports_hdfs) { + if constexpr (tiledb::sm::filesystem::hdfs_enabled) { fs_vec.emplace_back(std::make_unique()); } - if (supports_azure) { + if constexpr (tiledb::sm::filesystem::azure_enabled) { fs_vec.emplace_back(std::make_unique()); } - if (supports_gcs) { + if constexpr (tiledb::sm::filesystem::gcs_enabled) { fs_vec.emplace_back(std::make_unique()); fs_vec.emplace_back(std::make_unique("gs://")); } @@ -415,5 +407,4 @@ std::string TemporaryDirectoryFixture::create_temporary_array( return array_uri; } -} // End of namespace test -} // End of namespace tiledb +} // namespace tiledb::test diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 70a9d030141c..c517d13517c1 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -49,7 +49,7 @@ #include using namespace tiledb::common; -using tiledb::common::filesystem::directory_entry; +using namespace tiledb::sm::filesystem; namespace tiledb::sm { @@ -81,19 +81,19 @@ VFS::VFS( hdfs_ = tdb_unique_ptr(tdb_new(hdfs::HDFS)); st = hdfs_->init(config_); if (!st.ok()) { - throw std::runtime_error("[VFS::VFS] Failed to initialize HDFS backend."); + throw VFSException("Failed to initialize HDFS backend."); } #endif -#ifdef HAVE_S3 - supported_fs_.insert(Filesystem::S3); -#endif + if constexpr (s3_enabled) { + supported_fs_.insert(Filesystem::S3); + } #ifdef HAVE_AZURE supported_fs_.insert(Filesystem::AZURE); st = azure_.init(config_, io_tp_); if (!st.ok()) { - throw std::runtime_error("[VFS::VFS] Failed to initialize Azure backend."); + throw VFSException("Failed to initialize Azure backend."); } #endif @@ -101,7 +101,7 @@ VFS::VFS( supported_fs_.insert(Filesystem::GCS); st = gcs_.init(config_, io_tp_); if (!st.ok()) { - throw std::runtime_error("[VFS::VFS] Failed to initialize GCS backend."); + throw VFSException("Failed to initialize GCS backend."); } #endif @@ -171,39 +171,37 @@ Status VFS::create_dir(const URI& uri) const { #ifdef HAVE_HDFS return hdfs_->create_dir(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { -#ifdef HAVE_S3 - // It is a noop for S3 - return Status::Ok(); -#else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); -#endif + if constexpr (s3_enabled) { + // It is a noop for S3 + return Status::Ok(); + } else { + throw BuiltWithout("S3"); + } } if (uri.is_azure()) { -#ifdef HAVE_AZURE - // It is a noop for Azure - return Status::Ok(); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); -#endif + if constexpr (azure_enabled) { + // It is a noop for Azure + return Status::Ok(); + } else { + throw BuiltWithout("Azure"); + } } if (uri.is_gcs()) { -#ifdef HAVE_GCS - // It is a noop for GCS - return Status::Ok(); -#else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); -#endif + if constexpr (gcs_enabled) { + // It is a noop for GCS + return Status::Ok(); + } else { + throw BuiltWithout("GCS"); + } } if (uri.is_memfs()) { return memfs_.create_dir(uri.to_path()); } - return LOG_STATUS(Status_VFSError( - std::string("Unsupported URI scheme: ") + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::dir_size(const URI& dir_name, uint64_t* dir_size) const { @@ -211,9 +209,9 @@ Status VFS::dir_size(const URI& dir_name, uint64_t* dir_size) const { bool is_dir; RETURN_NOT_OK(this->is_dir(dir_name, &is_dir)); if (!is_dir) - return LOG_STATUS(Status_VFSError( - std::string("Cannot get directory size; Input '") + - dir_name.to_string() + "' is not a directory")); + throw VFSException( + "Cannot get directory size; Input '" + dir_name.to_string() + + "' is not a directory"); // Get all files in the tree rooted at `dir_name` and add their sizes *dir_size = 0; @@ -249,36 +247,34 @@ Status VFS::touch(const URI& uri) const { #ifdef HAVE_HDFS return hdfs_->touch(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { #ifdef HAVE_S3 return s3_.touch(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.touch(uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.touch(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return memfs_.touch(uri.to_path()); } - return LOG_STATUS(Status_VFSError( - std::string("Unsupported URI scheme: ") + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::cancel_all_tasks() { @@ -291,26 +287,24 @@ Status VFS::create_bucket(const URI& uri) const { #ifdef HAVE_S3 return s3_.create_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("S3 is not supported"))); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.create_container(uri); #else - return LOG_STATUS(Status_VFSError(std::string("Azure is not supported"))); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.create_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("GCS is not supported"))); + throw BuiltWithout("GCS"); #endif } - return LOG_STATUS(Status_VFSError( - std::string("Cannot create bucket; Unsupported URI scheme: ") + - uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::remove_bucket(const URI& uri) const { @@ -318,26 +312,24 @@ Status VFS::remove_bucket(const URI& uri) const { #ifdef HAVE_S3 return s3_.remove_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("S3 is not supported"))); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.remove_container(uri); #else - return LOG_STATUS(Status_VFSError(std::string("Azure is not supported"))); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.remove_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("GCS is not supported"))); + throw BuiltWithout("GCS"); #endif } - return LOG_STATUS(Status_VFSError( - std::string("Cannot remove bucket; Unsupported URI scheme: ") + - uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::empty_bucket(const URI& uri) const { @@ -345,26 +337,24 @@ Status VFS::empty_bucket(const URI& uri) const { #ifdef HAVE_S3 return s3_.empty_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("S3 is not supported"))); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.empty_container(uri); #else - return LOG_STATUS(Status_VFSError(std::string("Azure is not supported"))); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.empty_bucket(uri); #else - return LOG_STATUS(Status_VFSError(std::string("GCS is not supported"))); + throw BuiltWithout("GCS"); #endif } - return LOG_STATUS(Status_VFSError( - std::string("Cannot empty bucket; Unsupported URI scheme: ") + - uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::is_empty_bucket( @@ -373,26 +363,24 @@ Status VFS::is_empty_bucket( #ifdef HAVE_S3 return s3_.is_empty_bucket(uri, is_empty); #else - return LOG_STATUS(Status_VFSError(std::string("S3 is not supported"))); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.is_empty_container(uri, is_empty); #else - return LOG_STATUS(Status_VFSError(std::string("Azure is not supported"))); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.is_empty_bucket(uri, is_empty); #else - return LOG_STATUS(Status_VFSError(std::string("GCS is not supported"))); + throw BuiltWithout("GCS"); #endif } - return LOG_STATUS(Status_VFSError( - std::string("Cannot remove bucket; Unsupported URI scheme: ") + - uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::remove_dir(const URI& uri) const { @@ -406,32 +394,30 @@ Status VFS::remove_dir(const URI& uri) const { #ifdef HAVE_HDFS return hdfs_->remove_dir(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } else if (uri.is_s3()) { #ifdef HAVE_S3 return s3_.remove_dir(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } else if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.remove_dir(uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } else if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.remove_dir(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } else if (uri.is_memfs()) { return memfs_.remove(uri.to_path(), true); } else { - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } } @@ -459,36 +445,34 @@ Status VFS::remove_file(const URI& uri) const { #ifdef HAVE_HDFS return hdfs_->remove_file(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { #ifdef HAVE_S3 return s3_.remove_object(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.remove_blob(uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.remove_object(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return memfs_.remove(uri.to_path(), false); } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } void VFS::remove_files( @@ -546,36 +530,34 @@ Status VFS::file_size(const URI& uri, uint64_t* size) const { #ifdef HAVE_HDFS return hdfs_->file_size(uri, size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { #ifdef HAVE_S3 return s3_.object_size(uri, size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.blob_size(uri, size); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.object_size(uri, size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return memfs_.file_size(uri.to_path(), size); } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::is_dir(const URI& uri, bool* is_dir) const { @@ -592,7 +574,7 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { return hdfs_->is_dir(uri, is_dir); #else *is_dir = false; - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { @@ -600,7 +582,7 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { return s3_.is_dir(uri, is_dir); #else *is_dir = false; - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { @@ -608,8 +590,7 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { return azure_.is_dir(uri, is_dir); #else *is_dir = false; - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { @@ -617,15 +598,14 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { return gcs_.is_dir(uri, is_dir); #else *is_dir = false; - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { *is_dir = memfs_.is_dir(uri.to_path()); return Status::Ok(); } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::is_file(const URI& uri, bool* is_file) const { @@ -643,7 +623,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { return hdfs_->is_file(uri, is_file); #else *is_file = false; - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { @@ -652,7 +632,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { return Status::Ok(); #else *is_file = false; - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { @@ -660,8 +640,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { return azure_.is_blob(uri, is_file); #else *is_file = false; - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { @@ -669,15 +648,14 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { return gcs_.is_object(uri, is_file); #else *is_file = false; - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { *is_file = memfs_.is_file(uri.to_path()); return Status::Ok(); } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { @@ -687,7 +665,7 @@ Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { return Status::Ok(); #else *is_bucket = false; - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { @@ -696,8 +674,7 @@ Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { return Status::Ok(); #else *is_bucket = false; - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { @@ -706,12 +683,11 @@ Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { return Status::Ok(); #else *is_bucket = false; - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::ls(const URI& parent, std::vector* uris) const { @@ -755,8 +731,7 @@ tuple>> VFS::ls_with_sizes( Status st; std::tie(st, entries) = s3_.ls_with_sizes(parent); #else - auto st = - LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + auto st = Status_VFSError("TileDB was built without S3 support"); #endif RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_azure()) { @@ -764,8 +739,7 @@ tuple>> VFS::ls_with_sizes( Status st; std::tie(st, entries) = azure_.ls_with_sizes(parent); #else - auto st = - LOG_STATUS(Status_VFSError("TileDB was built without Azure support")); + auto st = Status_VFSError("TileDB was built without Azure support"); #endif RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_gcs()) { @@ -773,8 +747,7 @@ tuple>> VFS::ls_with_sizes( Status st; std::tie(st, entries) = gcs_.ls_with_sizes(parent); #else - auto st = - LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + auto st = Status_VFSError("TileDB was built without GCS support"); #endif RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_hdfs()) { @@ -782,8 +755,7 @@ tuple>> VFS::ls_with_sizes( Status st; std::tie(st, entries) = hdfs_->ls_with_sizes(parent); #else - auto st = - LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + auto st = Status_VFSError("TileDB was built without HDFS support"); #endif RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_memfs()) { @@ -792,8 +764,7 @@ tuple>> VFS::ls_with_sizes( memfs_.ls_with_sizes(URI("mem://" + parent.to_path())); RETURN_NOT_OK_TUPLE(st, nullopt); } else { - auto st = LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + parent.to_string())); + auto st = Status_VFSError("Unsupported URI scheme: " + parent.to_string()); return {st, std::nullopt}; } parallel_sort( @@ -823,8 +794,7 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { return posix_.move_file(old_uri, new_uri); #endif } - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // HDFS @@ -833,11 +803,9 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_HDFS return hdfs_->move_path(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // S3 @@ -846,10 +814,9 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_S3 return s3_.move_object(old_uri, new_uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // Azure @@ -858,11 +825,9 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_AZURE return azure_.move_object(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // GCS @@ -871,11 +836,9 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_GCS return gcs_.move_object(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // In-memory filesystem @@ -883,14 +846,11 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { if (new_uri.is_memfs()) { return memfs_.move(old_uri.to_path(), new_uri.to_path()); } - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving files"); } // Unsupported filesystem - return LOG_STATUS(Status_VFSError( - "Unsupported URI schemes: " + old_uri.to_string() + ", " + - new_uri.to_string())); + throw UnsupportedURI(old_uri.to_string() + ", " + new_uri.to_string()); } Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { @@ -903,8 +863,7 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { return posix_.move_file(old_uri, new_uri); #endif } - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // HDFS @@ -913,11 +872,9 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_HDFS return hdfs_->move_path(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // S3 @@ -926,10 +883,9 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_S3 return s3_.move_dir(old_uri, new_uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // Azure @@ -938,11 +894,9 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_AZURE return azure_.move_dir(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // GCS @@ -951,11 +905,9 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_GCS return gcs_.move_dir(old_uri, new_uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // In-memory filesystem @@ -963,14 +915,11 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { if (new_uri.is_memfs()) { return memfs_.move(old_uri.to_path(), new_uri.to_path()); } - return LOG_STATUS(Status_VFSError( - "Moving files across filesystems is not supported yet")); + throw UnsupportedOperation("Moving directories"); } // Unsupported filesystem - return LOG_STATUS(Status_VFSError( - "Unsupported URI schemes: " + old_uri.to_string() + ", " + - new_uri.to_string())); + throw UnsupportedURI(old_uri.to_string() + ", " + new_uri.to_string()); } Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { @@ -984,28 +933,24 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { if (old_uri.is_file()) { if (new_uri.is_file()) { #ifdef _WIN32 - return LOG_STATUS(Status_IOError( - std::string("Copying files on Windows is not yet supported."))); + throw VFSException("Copying files on Windows is not yet supported."); #else return posix_.copy_file(old_uri, new_uri); #endif } - return LOG_STATUS(Status_VFSError( - "Copying files across filesystems is not supported yet")); + throw UnsupportedOperation("Copying files"); } // HDFS if (old_uri.is_hdfs()) { - if (new_uri.is_hdfs()) -#ifdef HAVE_HDFS - return LOG_STATUS(Status_IOError( - std::string("Copying files on HDFS is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without HDFS support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying files across filesystems is not supported yet")); + if (new_uri.is_hdfs()) { + if constexpr (hdfs_enabled) { + throw VFSException("Copying files on HDFS is not yet supported."); + } else { + throw BuiltWithout("HDFS"); + } + } + throw UnsupportedOperation("Copying files"); } // S3 @@ -1014,44 +959,37 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_S3 return s3_.copy_file(old_uri, new_uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif - return LOG_STATUS(Status_VFSError( - "Copying files across filesystems is not supported yet")); + throw UnsupportedOperation("Copying files"); } // Azure if (old_uri.is_azure()) { - if (new_uri.is_azure()) -#ifdef HAVE_AZURE - return LOG_STATUS(Status_IOError( - std::string("Copying files on Azure is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying files across filesystems is not supported yet")); + if (new_uri.is_azure()) { + if constexpr (azure_enabled) { + throw VFSException("Copying files on Azure is not yet supported."); + } else { + throw BuiltWithout("Azure"); + } + } + throw UnsupportedOperation("Copying files"); } // GCS if (old_uri.is_gcs()) { - if (new_uri.is_gcs()) -#ifdef HAVE_GCS - return LOG_STATUS(Status_IOError( - std::string("Copying files on GCS is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without GCS support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying files across filesystems is not supported yet")); + if (new_uri.is_gcs()) { + if constexpr (gcs_enabled) { + throw VFSException("Copying files on GCS is not yet supported."); + } else { + throw BuiltWithout("GCS"); + } + } + throw UnsupportedOperation("Copying files"); } // Unsupported filesystem - return LOG_STATUS(Status_VFSError( - "Unsupported URI schemes: " + old_uri.to_string() + ", " + - new_uri.to_string())); + throw UnsupportedURI(old_uri.to_string() + ", " + new_uri.to_string()); } Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { @@ -1059,28 +997,25 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { if (old_uri.is_file()) { if (new_uri.is_file()) { #ifdef _WIN32 - return LOG_STATUS(Status_IOError( - std::string("Copying directories on Windows is not yet supported."))); + throw VFSException( + "Copying directories on Windows is not yet supported."); #else return posix_.copy_dir(old_uri, new_uri); #endif } - return LOG_STATUS(Status_VFSError( - "Copying directories across filesystems is not supported yet")); + throw UnsupportedOperation("Copying directories"); } // HDFS if (old_uri.is_hdfs()) { - if (new_uri.is_hdfs()) -#ifdef HAVE_HDFS - return LOG_STATUS(Status_IOError( - std::string("Copying directories on HDFS is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without HDFS support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying directories across filesystems is not supported yet")); + if (new_uri.is_hdfs()) { + if constexpr (hdfs_enabled) { + throw VFSException("Copying directories on HDFS is not yet supported."); + } else { + throw BuiltWithout("HDFS"); + } + } + throw UnsupportedOperation("Copying directories"); } // S3 @@ -1089,44 +1024,38 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { #ifdef HAVE_S3 return s3_.copy_dir(old_uri, new_uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif - return LOG_STATUS(Status_VFSError( - "Copying directories across filesystems is not supported yet")); + throw UnsupportedOperation("Copying directories"); } // Azure if (old_uri.is_azure()) { - if (new_uri.is_azure()) -#ifdef HAVE_AZURE - return LOG_STATUS(Status_IOError( - std::string("Copying directories on Azure is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying directories across filesystems is not supported yet")); + if (new_uri.is_azure()) { + if constexpr (azure_enabled) { + throw VFSException( + "Copying directories on Azure is not yet supported."); + } else { + throw BuiltWithout("Azure"); + } + } + throw UnsupportedOperation("Copying directories"); } // GCS if (old_uri.is_gcs()) { - if (new_uri.is_gcs()) -#ifdef HAVE_GCS - return LOG_STATUS(Status_IOError( - std::string("Copying directories on GCS is not yet supported."))); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without GCS support")); -#endif - return LOG_STATUS(Status_VFSError( - "Copying directories across filesystems is not supported yet")); + if (new_uri.is_gcs()) { + if constexpr (gcs_enabled) { + throw VFSException("Copying directories on GCS is not yet supported."); + } else { + throw BuiltWithout("GCS"); + } + } + throw UnsupportedOperation("Copying directories"); } // Unsupported filesystem - return LOG_STATUS(Status_VFSError( - "Unsupported URI schemes: " + old_uri.to_string() + ", " + - new_uri.to_string())); + throw UnsupportedURI(old_uri.to_string() + ", " + new_uri.to_string()); } Status VFS::read( @@ -1184,7 +1113,7 @@ Status VFS::read( std::stringstream errmsg; errmsg << "VFS parallel read error '" << uri.to_string() << "'; " << st.message(); - return LOG_STATUS(Status_VFSError(errmsg.str())); + return Status_VFSError(errmsg.str()); } return st; } @@ -1213,7 +1142,7 @@ Status VFS::read_impl( #ifdef HAVE_HDFS return hdfs_->read(uri, offset, buffer, nbytes); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { @@ -1230,7 +1159,7 @@ Status VFS::read_impl( return read_ahead_impl( read_fn, uri, offset, buffer, nbytes, use_read_ahead); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { @@ -1247,8 +1176,7 @@ Status VFS::read_impl( return read_ahead_impl( read_fn, uri, offset, buffer, nbytes, use_read_ahead); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { @@ -1265,15 +1193,14 @@ Status VFS::read_impl( return read_ahead_impl( read_fn, uri, offset, buffer, nbytes, use_read_ahead); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return memfs_.read(uri.to_path(), offset, buffer, nbytes); } - return LOG_STATUS( - Status_VFSError("Unsupported URI schemes: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::read_ahead_impl( @@ -1368,36 +1295,34 @@ Status VFS::sync(const URI& uri) { #ifdef HAVE_HDFS return hdfs_->sync(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { -#ifdef HAVE_S3 - return Status::Ok(); -#else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); -#endif + if constexpr (s3_enabled) { + return Status::Ok(); + } else { + throw BuiltWithout("S3"); + } } if (uri.is_azure()) { -#ifdef HAVE_AZURE - return Status::Ok(); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); -#endif + if constexpr (azure_enabled) { + return Status::Ok(); + } else { + throw BuiltWithout("Azure"); + } } if (uri.is_gcs()) { -#ifdef HAVE_GCS - return Status::Ok(); -#else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); -#endif + if constexpr (gcs_enabled) { + return Status::Ok(); + } else { + throw BuiltWithout("GCS"); + } } if (uri.is_memfs()) { return Status::Ok(); } - return LOG_STATUS( - Status_VFSError("Unsupported URI scheme: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::open_file(const URI& uri, VFSMode mode) { @@ -1407,9 +1332,8 @@ Status VFS::open_file(const URI& uri, VFSMode mode) { switch (mode) { case VFSMode::VFS_READ: if (!is_file) - return LOG_STATUS(Status_VFSError( - std::string("Cannot open file '") + uri.c_str() + - "'; File does not exist")); + throw VFSException( + "Cannot open file '" + uri.to_string() + "'; File does not exist"); break; case VFSMode::VFS_WRITE: if (is_file) @@ -1417,34 +1341,31 @@ Status VFS::open_file(const URI& uri, VFSMode mode) { break; case VFSMode::VFS_APPEND: if (uri.is_s3()) { -#ifdef HAVE_S3 - return LOG_STATUS(Status_VFSError( - std::string("Cannot open file '") + uri.c_str() + - "'; S3 does not support append mode")); -#else - return LOG_STATUS(Status_VFSError( - "Cannot open file; TileDB was built without S3 support")); -#endif + if constexpr (s3_enabled) { + throw VFSException( + "Cannot open file '" + uri.to_string() + + "'; S3 does not support append mode"); + } else { + throw BuiltWithout("S3"); + } } if (uri.is_azure()) { -#ifdef HAVE_AZURE - return LOG_STATUS(Status_VFSError( - std::string("Cannot open file '") + uri.c_str() + - "'; Azure does not support append mode")); -#else - return LOG_STATUS(Status_VFSError( - "Cannot open file; TileDB was built without Azure support")); -#endif + if constexpr (azure_enabled) { + throw VFSException( + "Cannot open file '" + uri.to_string() + + "'; Azure does not support append mode"); + } else { + throw BuiltWithout("Azure"); + } } if (uri.is_gcs()) { -#ifdef HAVE_GCS - return LOG_STATUS(Status_VFSError( - std::string("Cannot open file '") + uri.c_str() + - "'; GCS does not support append mode")); -#else - return LOG_STATUS(Status_VFSError( - "Cannot open file; TileDB was built without GCS support")); -#endif + if constexpr (gcs_enabled) { + throw VFSException( + "Cannot open file '" + uri.to_string() + + "'; GCS does not support append mode"); + } else { + throw BuiltWithout("GCS"); + } } break; } @@ -1464,36 +1385,34 @@ Status VFS::close_file(const URI& uri) { #ifdef HAVE_HDFS return hdfs_->sync(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { #ifdef HAVE_S3 return s3_.flush_object(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.flush_blob(uri); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.flush_object(uri); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return Status::Ok(); } - return LOG_STATUS( - Status_VFSError("Unsupported URI schemes: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } void VFS::finalize_and_close_file(const URI& uri) { @@ -1502,8 +1421,7 @@ void VFS::finalize_and_close_file(const URI& uri) { s3_.finalize_and_flush_object(uri); return; #else - throw StatusException( - Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } throw_if_not_ok(close_file(uri)); @@ -1528,7 +1446,7 @@ Status VFS::write( #ifdef HAVE_HDFS return hdfs_->write(uri, buffer, buffer_size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without HDFS support")); + throw BuiltWithout("HDFS"); #endif } if (uri.is_s3()) { @@ -1539,29 +1457,27 @@ Status VFS::write( } return s3_.write(uri, buffer, buffer_size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } if (uri.is_azure()) { #ifdef HAVE_AZURE return azure_.write(uri, buffer, buffer_size); #else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); + throw BuiltWithout("Azure"); #endif } if (uri.is_gcs()) { #ifdef HAVE_GCS return gcs_.write(uri, buffer, buffer_size); #else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); + throw BuiltWithout("GCS"); #endif } if (uri.is_memfs()) { return memfs_.write(uri.to_path(), buffer, buffer_size); } - return LOG_STATUS( - Status_VFSError("Unsupported URI schemes: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } std::pair> @@ -1595,33 +1511,25 @@ VFS::multipart_upload_state(const URI& uri) { return {Status::Ok(), state}; #else - return { - LOG_STATUS(Status_VFSError("TileDB was built without S3 support")), - nullopt}; + return {Status_VFSError("TileDB was built without S3 support"), nullopt}; #endif } else if (uri.is_azure()) { -#ifdef HAVE_AZURE - return { - LOG_STATUS(Status_VFSError("Not yet supported for Azure")), nullopt}; -#else - return { - LOG_STATUS(Status_VFSError("TileDB was built without Azure support")), - nullopt}; -#endif + if constexpr (azure_enabled) { + return {Status_VFSError("Not yet supported for Azure"), nullopt}; + } else { + return { + Status_VFSError("TileDB was built without Azure support"), nullopt}; + } } else if (uri.is_gcs()) { -#ifdef HAVE_GCS - return {LOG_STATUS(Status_VFSError("Not yet supported for GCS")), nullopt}; -#else - return { - LOG_STATUS(Status_VFSError("TileDB was built without GCS support")), - nullopt}; -#endif + if constexpr (gcs_enabled) { + return {Status_VFSError("Not yet supported for GCS"), nullopt}; + } else { + return {Status_VFSError("TileDB was built without GCS support"), nullopt}; + } } return { - LOG_STATUS( - Status_VFSError("Unsupported URI schemes: " + uri.to_string())), - nullopt}; + Status_VFSError("Unsupported URI schemes: " + uri.to_string()), nullopt}; } Status VFS::set_multipart_upload_state( @@ -1651,25 +1559,23 @@ Status VFS::set_multipart_upload_state( return s3_.set_multipart_upload_state(uri.to_string(), s3_state); #else - return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } else if (uri.is_azure()) { -#ifdef HAVE_AZURE - return LOG_STATUS(Status_VFSError("Not yet supported for Azure")); -#else - return LOG_STATUS( - Status_VFSError("TileDB was built without Azure support")); -#endif + if constexpr (azure_enabled) { + throw VFSException("Not yet supported for Azure"); + } else { + throw BuiltWithout("Azure"); + } } else if (uri.is_gcs()) { -#ifdef HAVE_GCS - return LOG_STATUS(Status_VFSError("Not yet supported for GCS")); -#else - return LOG_STATUS(Status_VFSError("TileDB was built without GCS support")); -#endif + if constexpr (gcs_enabled) { + throw VFSException("Not yet supported for GCS"); + } else { + throw BuiltWithout("GCS"); + } } - return LOG_STATUS( - Status_VFSError("Unsupported URI schemes: " + uri.to_string())); + throw UnsupportedURI(uri.to_string()); } Status VFS::flush_multipart_file_buffer(const URI& uri) { @@ -1681,8 +1587,7 @@ Status VFS::flush_multipart_file_buffer(const URI& uri) { buff->reset_size(); #else - throw StatusException( - Status_VFSError("TileDB was built without S3 support")); + throw BuiltWithout("S3"); #endif } diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 910a020bfb65..95942370577d 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -60,24 +60,90 @@ #ifdef HAVE_GCS #include "tiledb/sm/filesystem/gcs.h" -#endif +#endif // HAVE_GCS #ifdef HAVE_S3 #include "tiledb/sm/filesystem/s3.h" -#endif +#endif // HAVE_S3 #ifdef HAVE_HDFS #include "tiledb/sm/filesystem/hdfs_filesystem.h" -#endif +#endif // HAVE_HDFS #ifdef HAVE_AZURE #include "tiledb/sm/filesystem/azure.h" -#endif +#endif // HAVE_AZURE using namespace tiledb::common; +using tiledb::common::filesystem::directory_entry; namespace tiledb::sm { +namespace filesystem { +class VFSException : public StatusException { + public: + explicit VFSException(const std::string& message) + : StatusException("VFS", message) { + } +}; + +class BuiltWithout : public VFSException { + public: + explicit BuiltWithout(const std::string& filesystem) + : VFSException("TileDB was built without " + filesystem + "support") { + } +}; + +class UnsupportedOperation : public VFSException { + public: + explicit UnsupportedOperation(const std::string& operation) + : VFSException(operation + " across filesystems is not supported yet") { + } +}; + +class UnsupportedURI : public VFSException { + public: + explicit UnsupportedURI(const std::string& uri) + : VFSException("Unsupported URI scheme: " + uri) { + } +}; + +// Local filesystem is always enabled +static constexpr bool local_enabled = true; + +#ifdef _WIN32 +static constexpr bool windows_enabled = true; +static constexpr bool posix_enabled = false; +#else +static constexpr bool windows_enabled = false; +static constexpr bool posix_enabled = true; +#endif + +#ifdef HAVE_GCS +static constexpr bool gcs_enabled = true; +#else +static constexpr bool gcs_enabled = false; +#endif // HAVE_GCS + +#ifdef HAVE_S3 +static constexpr bool s3_enabled = true; +#else +static constexpr bool s3_enabled = false; +#endif // HAVE_S3 + +#ifdef HAVE_HDFS +static constexpr bool hdfs_enabled = true; +#else +static constexpr bool hdfs_enabled = false; +#endif // HAVE_HDFS + +#ifdef HAVE_AZURE +static constexpr bool azure_enabled = true; +#else +static constexpr bool azure_enabled = false; +#endif // HAVE_AZURE +} // namespace filesystem + class Tile; enum class Filesystem : uint8_t; @@ -425,8 +491,8 @@ class VFS : private VFSBase, S3_within_VFS { * @param parent The target directory to list. * @return All entries that are contained in the parent */ - tuple>> - ls_with_sizes(const URI& parent) const; + tuple>> ls_with_sizes( + const URI& parent) const; /** * Renames a file. From fe03460e7b1f1931b08a65c5bd71cff99f24d6f6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Mon, 4 Dec 2023 18:11:37 -0600 Subject: [PATCH 096/456] Add sm.consolidation.total_buffer_size setting (#4550) This adds a new configuration parameter for directly setting the total buffer size to use during fragment consolidation. This deprecates the old sm.consolidation.buffer_size value. We will continue to use the old value if set by users for backwards compatibility. If not set, the new setting is used. --- TYPE: FEATURE DESC: Add sm.consolidation.total_buffer_size setting --- test/src/unit-capi-config.cc | 2 ++ tiledb/api/c_api/config/config_api_external.h | 5 ++++ tiledb/sm/config/config.cc | 6 +++++ tiledb/sm/config/config.h | 3 +++ .../sm/consolidator/fragment_consolidator.cc | 25 +++++++++++++++++-- .../sm/consolidator/fragment_consolidator.h | 2 ++ tiledb/sm/cpp_api/config.h | 5 ++++ 7 files changed, 46 insertions(+), 2 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 02ddccfe7d55..6b5920eaa406 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -260,6 +260,7 @@ void check_save_to_file() { ss << "sm.consolidation.steps 4294967295\n"; ss << "sm.consolidation.timestamp_end " << std::to_string(UINT64_MAX) << "\n"; ss << "sm.consolidation.timestamp_start 0\n"; + ss << "sm.consolidation.total_buffer_size 2147483648\n"; ss << "sm.dedup_coords false\n"; ss << "sm.enable_signal_handlers true\n"; ss << "sm.encryption_type NO_ENCRYPTION\n"; @@ -661,6 +662,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["sm.consolidation.step_min_frags"] = "4294967295"; all_param_values["sm.consolidation.step_max_frags"] = "4294967295"; all_param_values["sm.consolidation.buffer_size"] = "50000000"; + all_param_values["sm.consolidation.total_buffer_size"] = "2147483648"; all_param_values["sm.consolidation.max_fragment_size"] = std::to_string(UINT64_MAX); all_param_values["sm.consolidation.step_size_ratio"] = "0.0"; diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 2d34b3109b1a..b49135aebb1f 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -164,9 +164,14 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * (since the resulting fragment is dense).
* **Default**: 1.0 * - `sm.consolidation.buffer_size`
+ * Deprecated. Prefer `sm.consolidation.total_buffer_size` instead. * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 + * - `sm.consolidation.total_buffer_size`
+ * The size (in bytes) of all attribute buffers used during + * consolidation.
+ * **Default**: 2,147,483,648 * - `sm.consolidation.max_fragment_size`
* **Experimental**
* The size (in bytes) of the maximum on-disk fragment size that will be diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index c640de474f9a..83b11dbe36f2 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -135,6 +135,7 @@ const std::string Config::SM_IO_CONCURRENCY_LEVEL = const std::string Config::SM_SKIP_CHECKSUM_VALIDATION = "false"; const std::string Config::SM_CONSOLIDATION_AMPLIFICATION = "1.0"; const std::string Config::SM_CONSOLIDATION_BUFFER_SIZE = "50000000"; +const std::string Config::SM_CONSOLIDATION_TOTAL_BUFFER_SIZE = "2147483648"; const std::string Config::SM_CONSOLIDATION_MAX_FRAGMENT_SIZE = std::to_string(UINT64_MAX); const std::string Config::SM_CONSOLIDATION_PURGE_DELETED_CELLS = "false"; @@ -332,6 +333,9 @@ const std::map default_config_values = { Config::SM_CONSOLIDATION_AMPLIFICATION), std::make_pair( "sm.consolidation.buffer_size", Config::SM_CONSOLIDATION_BUFFER_SIZE), + std::make_pair( + "sm.consolidation.total_buffer_size", + Config::SM_CONSOLIDATION_TOTAL_BUFFER_SIZE), std::make_pair( "sm.consolidation.max_fragment_size", Config::SM_CONSOLIDATION_MAX_FRAGMENT_SIZE), @@ -741,6 +745,8 @@ Status Config::sanity_check( RETURN_NOT_OK(utils::parse::convert(value, &vf)); } else if (param == "sm.consolidation.buffer_size") { RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); + } else if (param == "sm.consolidation.total_buffer_size") { + RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); } else if (param == "sm.consolidation.max_fragment_size") { RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); } else if (param == "sm.consolidation.purge_deleted_cells") { diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index ec7fcdb113ca..ffc4f4694d4c 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -269,6 +269,9 @@ class Config { /** The buffer size for each attribute used in consolidation. */ static const std::string SM_CONSOLIDATION_BUFFER_SIZE; + /** The total buffer size for all attributes during consolidation. */ + static const std::string SM_CONSOLIDATION_TOTAL_BUFFER_SIZE; + /** The maximum fragment size used in consolidation. */ static const std::string SM_CONSOLIDATION_MAX_FRAGMENT_SIZE; diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 47386fe08522..649c4eea55d9 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -593,7 +593,14 @@ FragmentConsolidator::create_buffers( // Use the old buffer size setting to see how much memory we would use. auto buffer_num = buffer_weights.size(); - uint64_t total_budget = config.buffer_size_ * buffer_num; + uint64_t total_budget = config.total_buffer_size_; + + // If a user set the per-attribute buffer size configuration, we override + // the use of the total_budget_size config setting for backwards + // compatible behavior. + if (config.buffer_size_ != 0) { + total_budget = config.buffer_size_ * buffer_num; + } // Create buffers. std::vector buffers(buffer_num); @@ -896,8 +903,22 @@ Status FragmentConsolidator::set_config(const Config& config) { "sm.consolidation.steps", &config_.steps_, &found)); assert(found); config_.buffer_size_ = 0; + // Only set the buffer_size_ if the user specified a value. Otherwise, we use + // the new sm.consolidation.total_buffer_size instead. + if (merged_config.set_params().count("sm.consolidation.buffer_size") > 0) { + logger_->warn( + "The `sm.consolidation.buffer_size configuration setting has been " + "deprecated. Set consolidation buffer sizes using the newer " + "`sm.consolidation.total_buffer_size` setting."); + RETURN_NOT_OK(merged_config.get( + "sm.consolidation.buffer_size", &config_.buffer_size_, &found)); + assert(found); + } + config_.total_buffer_size_ = 0; RETURN_NOT_OK(merged_config.get( - "sm.consolidation.buffer_size", &config_.buffer_size_, &found)); + "sm.consolidation.total_buffer_size", + &config_.total_buffer_size_, + &found)); assert(found); config_.max_fragment_size_ = 0; RETURN_NOT_OK(merged_config.get( diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index ca3d8e2243ad..c4fc095734c7 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -154,6 +154,8 @@ class FragmentConsolidator : public Consolidator { float amplification_; /** Attribute buffer size. */ uint64_t buffer_size_; + /** Total buffer size for all attributes. */ + uint64_t total_buffer_size_; /** Max fragment size. */ uint64_t max_fragment_size_; /** diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 3c3e09f0053c..34eb820e8b6a 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -340,9 +340,14 @@ class Config { * (since the resulting fragments is dense).
* **Default**: 1.0 * - `sm.consolidation.buffer_size`
+ * Deprecated. Prefer `sm.consolidation.total_buffer_size` instead. * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 + * - `sm.consolidation.total_buffer_size`
+ * The size (in bytes) of all attribute buffers used during + * consolidation.
+ * **Default**: 2,147,483,648 * - `sm.consolidation.max_fragment_size`
* **Experimental**
* The size (in bytes) of the maximum on-disk fragment size that will be From 6042a36df66f60910a809a9e460c776771ad0301 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 5 Dec 2023 18:40:29 -0600 Subject: [PATCH 097/456] Fix array schema when time traveling (#4549) We were using the latest array schema even while traveling. This changes the latest array schema selection when in ArrayDirectoryMode::READ to use the newest array schema that is less than or equal to the current end timestamp. If no array schemas are newer than the timestamp end value, the earliest schema is used. --- TYPE: BUG DESC: Fix array schema selection when time traveling --- test/regression/CMakeLists.txt | 1 + test/regression/targets/sc-35424.cc | 165 ++++++++++++++++++++++++++++ test/src/unit-capi-array_schema.cc | 6 + tiledb/sm/array/array_directory.cc | 41 ++++++- tiledb/sm/array/array_directory.h | 7 ++ 5 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 test/regression/targets/sc-35424.cc diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 3fa7e3c28865..0f193580ac48 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -47,6 +47,7 @@ if (TILEDB_CPP_API) list(APPEND SOURCES targets/sc-25116.cc) list(APPEND SOURCES targets/sc-29682.cc) list(APPEND SOURCES targets/sc-33480.cc) + list(APPEND SOURCES targets/sc-35424.cc) endif() add_executable(tiledb_regression diff --git a/test/regression/targets/sc-35424.cc b/test/regression/targets/sc-35424.cc new file mode 100644 index 000000000000..66b144363f18 --- /dev/null +++ b/test/regression/targets/sc-35424.cc @@ -0,0 +1,165 @@ +#include +#include +#include + +#include +#include + +#include + +using namespace tiledb; + +static void create_array(const std::string& array_uri); +static void write_first_fragment(const std::string& array_uri); +static uint64_t time_travel_destination(); +static void add_attr_b(const std::string& array_uri); +static void write_second_fragment(const std::string& array_uri); + +static void read_without_time_travel(const std::string& array_uri); +static void read_with_time_travel(const std::string& array_uri, uint64_t when); + +TEST_CASE( + "Use the correct schema when time traveling", + "[time-traveling][array-schema][bug][sc35424]") { + std::string array_uri = "test_time_traveling_schema"; + + // Test setup + create_array(array_uri); + write_first_fragment(array_uri); + auto timepoint = time_travel_destination(); + add_attr_b(array_uri); + write_second_fragment(array_uri); + + // Check reads with and without time travel. + read_without_time_travel(array_uri); + read_with_time_travel(array_uri, timepoint); +} + +void create_array(const std::string& array_uri) { + Context ctx; + + auto obj = Object::object(ctx, array_uri); + if (obj.type() != Object::Type::Invalid) { + Object::remove(ctx, array_uri); + } + + auto dim = Dimension::create(ctx, "d", {{0, 1024}}); + + Domain dom(ctx); + dom.add_dimension(dim); + + auto attr = Attribute::create(ctx, "a"); + + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}) + .set_domain(dom) + .add_attribute(attr); + + Array::create(array_uri, schema); +} + +void write_first_fragment(const std::string& array_uri) { + std::vector d_data = {0, 1, 2, 3, 4}; + std::vector a_data = {5, 6, 7, 8, 9}; + + Context ctx; + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("d", d_data) + .set_data_buffer("a", a_data); + REQUIRE(query.submit() == Query::Status::COMPLETE); + array.close(); +} + +uint64_t time_travel_destination() { + // We sleep for 5ms to ensure that our fragments are separated in time + // and allowing us to grab a time guaranteed to be between them. + auto delay = std::chrono::milliseconds(5); + std::this_thread::sleep_for(delay); + + auto timepoint = tiledb_timestamp_now_ms(); + + std::this_thread::sleep_for(delay); + + return timepoint; +} + +void add_attr_b(const std::string& array_uri) { + Context ctx; + auto attr = Attribute::create(ctx, "b"); + + ArraySchemaEvolution ase(ctx); + ase.add_attribute(attr); + ase.array_evolve(array_uri); +} + +void write_second_fragment(const std::string& array_uri) { + std::vector d_data = {5, 6, 7, 8, 9}; + std::vector a_data = {10, 11, 12, 13, 14}; + std::vector b_data = {15, 16, 17, 18, 19}; + + Context ctx; + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("d", d_data) + .set_data_buffer("a", a_data) + .set_data_buffer("b", b_data); + REQUIRE(query.submit() == Query::Status::COMPLETE); + array.close(); +} + +void read_without_time_travel(const std::string& array_uri) { + std::vector d_data(10); + std::vector a_data(10); + std::vector b_data(10); + + Context ctx; + Array array(ctx, array_uri, TILEDB_READ); + Query query(ctx, array, TILEDB_READ); + query.set_data_buffer("d", d_data) + .set_data_buffer("a", a_data) + .set_data_buffer("b", b_data); + + REQUIRE(query.submit() == Query::Status::COMPLETE); + + for (int32_t i = 0; i < 10; i++) { + REQUIRE(d_data[i] == i); + REQUIRE(a_data[i] == i + 5); + + if (i < 5) { + REQUIRE(b_data[i] == INT32_MIN); + } else { + REQUIRE(b_data[i] == i + 10); + } + } +} + +void read_with_time_travel(const std::string& array_uri, uint64_t when) { + std::vector d_data(10, INT_MAX); + std::vector a_data(10, INT_MAX); + std::vector b_data(10, INT_MAX); + + Context ctx; + Array array(ctx, array_uri, TILEDB_READ, TemporalPolicy(TimeTravel, when)); + Query query(ctx, array, TILEDB_READ); + query.set_data_buffer("d", d_data).set_data_buffer("a", a_data); + + auto matcher = Catch::Matchers::ContainsSubstring("There is no field b"); + REQUIRE_THROWS_WITH(query.set_data_buffer("b", b_data), matcher); + + REQUIRE(query.submit() == Query::Status::COMPLETE); + + for (int32_t i = 0; i < 10; i++) { + if (i < 5) { + REQUIRE(d_data[i] == i); + REQUIRE(a_data[i] == i + 5); + REQUIRE(b_data[i] == INT_MAX); + } else { + REQUIRE(d_data[i] == INT_MAX); + REQUIRE(a_data[i] == INT_MAX); + REQUIRE(b_data[i] == INT_MAX); + } + } +} diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 767115574533..643d5cdfa4e2 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -2149,6 +2149,8 @@ TEST_CASE_METHOD( tiledb_array_t* array; rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_set_open_timestamp_end(ctx_, array, now + 1); + REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); tiledb_array_schema_t* read_schema; @@ -2292,6 +2294,8 @@ TEST_CASE_METHOD( tiledb_array_t* array; rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_set_open_timestamp_end(ctx_, array, now + 1); + REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); tiledb_array_schema_t* read_schema; @@ -2384,6 +2388,8 @@ TEST_CASE_METHOD( tiledb_array_t* array; rc = tiledb_array_alloc(ctx_, array_uri.c_str(), &array); REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_set_open_timestamp_end(ctx_, array, now + 1); + REQUIRE(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); REQUIRE(rc == TILEDB_OK); tiledb_array_schema_t* read_schema; diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 5a223fd25c75..86fa172a307c 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -393,8 +393,7 @@ Status ArrayDirectory::load() { "Cannot open array; Array does not exist.")); } - // Set the latest array schema URI - latest_array_schema_uri_ = array_schema_uris_.back(); + latest_array_schema_uri_ = select_latest_array_schema_uri(); assert(!latest_array_schema_uri_.is_invalid()); } @@ -1214,6 +1213,44 @@ Status ArrayDirectory::compute_array_schema_uris( return Status::Ok(); } +URI ArrayDirectory::select_latest_array_schema_uri() { + // Set the latest array schema URI. When in READ mode, the latest array + // schema URI is the schema with the largest timestamp less than or equal + // to the current timestamp_end_. If no schema meets this definition, we + // use the first schema available. + // + // The reason for choosing the oldest array schema URI even when time + // traveling before it existed is to first, not break any arrays that have + // fragments written before the first schema existed. The second reason is + // to not break old arrays that only have the old `__array_schema.tdb` + // URI which does not have timestamps. + if (mode_ != ArrayDirectoryMode::READ) { + return array_schema_uris_.back(); + } + + optional latest_uri = nullopt; + uint64_t latest_ts = 0; + + for (auto& uri : array_schema_uris_) { + auto name = uri.remove_trailing_slash().last_path_part(); + + // Skip the old schema URI name since it doesn't have timestamps + if (name == constants::array_schema_filename) { + continue; + } + + std::pair ts_range; + throw_if_not_ok(utils::parse::get_timestamp_range(uri, &ts_range)); + + if (ts_range.second > latest_ts && ts_range.second <= timestamp_end_) { + latest_uri = uri; + latest_ts = ts_range.second; + } + } + + return latest_uri.value_or(array_schema_uris_.front()); +} + bool ArrayDirectory::is_vacuum_file(const URI& uri) const { if (utils::parse::ends_with(uri.to_string(), constants::vacuum_file_suffix)) return true; diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index ca8faf79ffcb..0d17b64af023 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -764,6 +764,13 @@ class ArrayDirectory { Status compute_array_schema_uris( const std::vector& array_schema_dir_uris); + /** + * Select the URI to use for the latest array schema. + * + * @return URI The latest array schema URI to use. + */ + URI select_latest_array_schema_uri(); + /** * Checks if a fragment overlaps with the array directory timestamp * range. Overlap is partial or full depending on the consolidation From cebbb8484c275979d491be51532bd7cb0a009e0b Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:25:16 -0500 Subject: [PATCH 098/456] Update dev HISTORY with 2.17 and 2.18 patch content. (#4558) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.17 and 2.18 patch content. --- HISTORY.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 530fa5e52361..2b4e85859f23 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,20 @@ +# TileDB v2.18.2 Release Notes + +## Defects removed + +* Fix delta filter deserialization for format ver 19. [#4541](https://github.com/TileDB-Inc/TileDB/pull/4541) + +## Build System Changes + +* Remove the `externals/vcpkg` submodule. Vcpkg is always downloaded by CMake. [#4543](https://github.com/TileDB-Inc/TileDB/pull/4543) + +# TileDB v2.18.1 Release Notes + +## Build System Changes + +* Fix errors when building from a directory that is not a Git repository. [#4529](https://github.com/TileDB-Inc/TileDB/pull/4529) + + # TileDB v2.18.0 Release Notes ## Announcements @@ -122,6 +139,11 @@ * Add support for basic mimo node to TileDB task graph. [#3977](https://github.com/TileDB-Inc/TileDB/pull/3977) * Verify that dependency-only edges using `std::monostate` work with nodes and edges. [#3939](https://github.com/TileDB-Inc/TileDB/pull/3939) +# TileDB v2.17.5 Release Notes + +## Defects removed +* Fix delta filter deserialization for format ver 19. [#4541](https://github.com/TileDB-Inc/TileDB/pull/4541) + # TileDB v2.17.4 Release Notes ## Defects removed From 79266ed6070f352211ee066c77e70bddf1a9a76f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 8 Dec 2023 15:44:05 +0200 Subject: [PATCH 099/456] Combine the `tiledb_shared` and `tiledb_static` CMake targets. (#4528) [SC-36838](https://app.shortcut.com/tiledb-inc/story/36838/combine-the-tiledb-shared-and-tiledb-static-cmake-targets) This PR combines the `tiledb_shared` and `tiledb_static` CMake targets into a `tiledb` target whose linkage is determined by the `BUILD_SHARED_LIBS` variable. This approach has several advantages: * Simplifies the build system and brings it more in line with CMake's conventions. * Avoids excessive building of code. For example the MinGW workflow needs a static library but had to also build a shared library because there is no way to disable that. * Provides a unified `TileDB::tiledb` CMake target that links to TileDB either statically or dynamically, depending on how TileDB was built. * The existing `TileDB::tiledb_static` and `TileDB::tiledb_shared` targets are retained for compatibility. The `TILEDB_STATIC` option was removed and replaced by `BUILD_SHARED_LIBS`. The bootstrap scripts were updated accordingly. --- TYPE: IMPROVEMENT DESC: Export a `TileDB::tiledb` CMake target regardless of static or dynamic linkage. TYPE: BUILD DESC: Remove the `TILEDB_STATIC` option and replace it with `BUILD_SHARED_LIBS` which is disabled by default. TYPE: BUILD DESC: The build system supports building only a static or shared library from the same buildtree. --- .github/workflows/build-windows.yml | 12 +- .github/workflows/ci-linux_mac.yml | 9 +- .github/workflows/mingw-w64-tiledb/PKGBUILD | 1 - .github/workflows/release.yml | 2 +- .github/workflows/unit-test-runs.yml | 4 +- bootstrap | 30 +++- bootstrap.ps1 | 24 +++- cmake/Options/BuildOptions.cmake | 2 +- cmake/TileDB-Superbuild.cmake | 2 +- cmake/inputs/Config.cmake.in | 15 +- examples/c_api/CMakeLists.txt | 25 +--- examples/c_api/vfs.c | 12 +- examples/cmake_project/CMakeLists.txt | 8 +- examples/cpp_api/CMakeLists.txt | 4 +- examples/cpp_api/quickstart_dense.cc | 39 +++-- examples/png_ingestion/CMakeLists.txt | 2 +- .../cpp_api/CMakeLists.txt | 4 +- scripts/ci/build_benchmarks.sh | 26 ---- scripts/run-nix-examples.sh | 30 ++-- test/CMakeLists.txt | 2 +- test/benchmarking/src/CMakeLists.txt | 2 +- test/performance/CMakeLists.txt | 2 +- test/regression/CMakeLists.txt | 2 +- test/support/CMakeLists.txt | 2 +- tiledb/CMakeLists.txt | 133 ++++-------------- tiledb/api/c_api/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 8 +- 27 files changed, 159 insertions(+), 245 deletions(-) mode change 100644 => 100755 scripts/run-nix-examples.sh diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 0c896c58b059..2f8cfcb860fa 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -9,9 +9,8 @@ env: TILEDB_AZURE: OFF TILEDB_GCS: OFF TILEDB_SERIALIZATION: OFF - TILEDB_STATIC: OFF TILEDB_TOOLS: ON - TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVerbose -EnableStaticTileDB' + TILEDB_BASE_BOOTSTRAP_OPTIONS: '-EnableVerbose' jobs: build: @@ -155,10 +154,6 @@ jobs: if ($env:TILEDB_SERIALIZATION -eq "ON") { $bootstrapOptions = "-EnableSerialization " + $bootstrapOptions } - # static already added above as initial default - # if ($env:TILEDB_STATIC -eq "ON") { - # $bootstrapOptions = $bootstrapOptions + " -EnableStaticTileDB" - # } # if ($env:TILEDB_HDFS -eq "ON") { # $bootstrapOptions = $bootstrapOptions + " -EnableHDFS" #} @@ -392,11 +387,6 @@ jobs: Write-Host "cmd: '$cmd'" Invoke-Expression $cmd - #.\$CMakeBuildType\ExampleExe_static.exe - $cmd = ".\$CMakeBuildType\ExampleExe_static.exe" - Write-Host "cmd: '$cmd'" - Invoke-Expression $cmd - - name: Packaging test shell: pwsh run: | diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index f16fabbf4534..32b0dfd59930 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -204,8 +204,9 @@ jobs: ################################################### # Build and run Examples - make -C $GITHUB_WORKSPACE/build -j3 examples - source $GITHUB_WORKSPACE/scripts/run-nix-examples.sh + # The following script must not be sourced. + # It relies on finding its source directory. + $GITHUB_WORKSPACE/scripts/run-nix-examples.sh ################################################### # Build benchmark code @@ -216,6 +217,10 @@ jobs: if: ${{ failure() && startsWith(matrix.os, 'ubuntu-') == true }} # only run this job if the build step failed run: ./scripts/ci/posix/dump-core-stacks.sh + - name: List files in build directory on failure + run: du --exclude _deps -h $GITHUB_WORKSPACE/build + if: ${{ failure() }} + - name: 'Upload failure artifacts (Linux)' # https://github.com/actions/upload-artifact#where-does-the-upload-go if: ${{ startsWith(matrix.os, 'ubuntu-') == true }} # only run this job if the build step failed uses: actions/upload-artifact@v3 diff --git a/.github/workflows/mingw-w64-tiledb/PKGBUILD b/.github/workflows/mingw-w64-tiledb/PKGBUILD index 1ec3de5f780f..7c54c04e909c 100644 --- a/.github/workflows/mingw-w64-tiledb/PKGBUILD +++ b/.github/workflows/mingw-w64-tiledb/PKGBUILD @@ -31,7 +31,6 @@ build() { -DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \ -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ - -DTILEDB_STATIC=ON \ -DTILEDB_TESTS=OFF \ -DTILEDB_S3=ON \ -DCOMPILER_SUPPORTS_AVX2=OFF \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2eb12cad3ba6..77f894ce7f7f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,6 +82,7 @@ jobs: run: | cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=./dist \ -DTILEDB_S3=ON \ -DTILEDB_AZURE=ON \ @@ -89,7 +90,6 @@ jobs: -DTILEDB_HDFS=${{ startsWith(matrix.platform, 'windows') && 'OFF' || 'ON' }} \ -DTILEDB_SERIALIZATION=ON \ -DTILEDB_WEBP=ON \ - -DTILEDB_STATIC=OFF \ -DTILEDB_TESTS=OFF \ -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \ ${{ matrix.cmake_args }} diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index 660c4539be3c..0b16aede88c5 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -9,7 +9,6 @@ env: TILEDB_AZURE: OFF TILEDB_GCS: OFF TILEDB_SERIALIZATION: ON - TILEDB_STATIC: ON TILEDB_TOOLS: ON VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' @@ -59,8 +58,7 @@ jobs: -DTILEDB_GCS=${TILEDB_GCS} \ -DTILEDB_S3=${TILEDB_S3} \ -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} \ - -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS} \ - -DTILEDB_STATIC=${TILEDB_STATIC} + -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS} make -j4 # Build all unit tests make -C tiledb tests -j4 diff --git a/bootstrap b/bootstrap index 455b424334bb..1878c54df6d5 100755 --- a/bootstrap +++ b/bootstrap @@ -37,6 +37,8 @@ Configuration: --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] + --linkage specify the linkage of tiledb. Defaults to static. + [static|shared] --force-build-all-deps force building of all dependencies, even those already installed at system-level --remove-deprecations build TileDB without any deprecated APIs @@ -45,9 +47,9 @@ Configuration: --disable-tests disables building the TileDB tests --disable-stats disables internal TileDB statistics --disable-avx2 disables use of AVX2 instructions - --disable-webp disables building of webp library and linkage test + --disable-webp disables building of webp library --disable-vcpkg disables use of vcpkg for downloading and building dependencies - --enable-static-tiledb enables building TileDB as a static library + --enable-static-tiledb enables building TileDB as a static library (deprecated, use --linkage=static) --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) --enable-debug enable debug build @@ -101,7 +103,7 @@ tiledb_cpp_api="ON" tiledb_force_all_deps="OFF" tiledb_remove_deprecations="OFF" tiledb_stats="ON" -tiledb_static="OFF" +build_shared_libs="OFF" tiledb_disable_avx2="" tiledb_assertions="OFF" tiledb_serialization="OFF" @@ -121,6 +123,7 @@ while test $# != 0; do tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` dependency_dir="$dir";; + --linkage=*) linkage=`arg "$1"`;; --force-build-all-deps) tiledb_force_all_deps="ON";; --remove-deprecations) tiledb_remove_deprecations="ON";; --disable-werror) tiledb_werror="OFF";; @@ -140,7 +143,8 @@ while test $# != 0; do --enable-azure) tiledb_azure="ON";; --enable-gcs) tiledb_gcs="ON";; --enable-serialization) tiledb_serialization="ON";; - --enable-static-tiledb) tiledb_static="ON";; + --enable-static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. TileDB is now built as a static library by default and linkage is controlled with the --linkage option." + enable_static_tiledb="ON";; --enable-sanitizer=*) san=`arg "$1"` sanitizer="$san";; --enable-tools) tiledb_tools="ON";; @@ -176,7 +180,8 @@ for en in "${enables[@]}"; do ccache) tiledb_ccache="ON";; arrow-tests) tiledb_arrow_tests="ON";; hdfs) tiledb_hdfs="ON";; - static-tiledb) tiledb_static="ON";; + static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. TileDB is now built as a static library by default and linkage is controlled with the --linkage option." + enable_static_tiledb="ON";; experimental-features) tiledb_experimental_features="ON";; rest-tests) tiledb_tests_enable_rest="ON";; aws-s3-config) tiledb_tests_aws_s3_config="ON";; @@ -188,6 +193,19 @@ if [ "${source_dir}" = "${binary_dir}" ]; then die "cannot build the project in the source directory! Out-of-source build is enforced!" fi +if [ "${linkage}" = "static" ] || [ "${linkage}" = "" ]; then + build_shared_libs="OFF" +elif [ "${linkage}" = "shared" ]; then + build_shared_libs="ON" +else + die "unknown linkage: ${linkage}" +fi + +# Fail if both --linkage=shared and --enable-static-tiledb are specified +if [ "${build_shared_libs}" = "ON" ] && [ "${enable_static_tiledb}" = "ON" ]; then + die "cannot specify both --linkage=shared and --enable-static-tiledb" +fi + # Check clang compiler if [[ x"${CC}" = x"" ]]; then CC=gcc @@ -219,6 +237,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DCMAKE_C_COMPILER="${c_compiler}" \ -DCMAKE_CXX_COMPILER="${cxx_compiler}" \ -DCMAKE_PREFIX_PATH="${dependency_dir}" \ + -DBUILD_SHARED_LIBS=${build_shared_libs} \ -DTILEDB_VCPKG=${tiledb_vcpkg} \ -DTILEDB_ASSERTIONS=${tiledb_assertions} \ -DTILEDB_VERBOSE=${tiledb_verbose} \ @@ -231,7 +250,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_WERROR=${tiledb_werror} \ -DTILEDB_CPP_API=${tiledb_cpp_api} \ -DTILEDB_STATS=${tiledb_stats} \ - -DTILEDB_STATIC=${tiledb_static} \ -DTILEDB_TESTS=${tiledb_tests} \ -DTILEDB_CCACHE=${tiledb_ccache} \ -DTILEDB_ARROW_TESTS=${tiledb_arrow_tests} \ diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 48e1533f6bdc..bda0743e4699 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -16,6 +16,10 @@ Installs files in tree rooted at PREFIX (defaults to TileDB\dist). .PARAMETER Dependency Semicolon separated list to binary dependencies. +.PARAMETER Linkage +Specify the linkage type to build TileDB with. Valid values are +"static" and "shared". Default is "static". + .PARAMETER CMakeGenerator Optionally specify the CMake generator string, e.g. "Visual Studio 15 2017". Check 'cmake --help' for a list of supported generators. @@ -58,6 +62,7 @@ Enables building with serialization support. .PARAMETER EnableStaticTileDB Enables building TileDB as a static library. +Deprecated, use -Linkage static instead. .PARAMETER EnableBuildDeps Enables building TileDB dependencies from source (superbuild) @@ -113,6 +118,7 @@ https://github.com/TileDB-Inc/TileDB Param( [string]$Prefix, [string]$Dependency, + [string]$Linkage = "static", [string]$CMakeGenerator, [switch]$EnableAssert, [switch]$EnableDebug, @@ -240,9 +246,21 @@ if ($DisableWebP.IsPresent) { $BuildWebP="OFF" } -$TileDBStatic = "OFF"; +$BuildSharedLibs = "OFF"; +if ($Linkage -eq "shared") { + $BuildSharedLibs = "ON" +} +elseif ($Linkage -ne "static") { + Write-Error "Invalid linkage type: $Linkage. Valid values are 'static' and 'shared'." + exit 1 +} + if ($EnableStaticTileDB.IsPresent) { - $TileDBStatic = "ON" + Write-Warning "-EnableStaticTileDB is deprecated and will be removed in a future version. TileDB is now built as a static library by default. Use -Linkage shared to build a shared library." + if ($Linkage -eq "shared") { + Write-Error "Cannot specify -EnableStaticTileDB alongside -Linkage shared." + exit 1 + } } $TileDBTools = "OFF"; @@ -308,7 +326,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 3a21254e69f1..a5a412e6f7fb 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -21,7 +21,7 @@ option(TILEDB_ASSERTIONS "Build with assertions enabled (default off for release option(TILEDB_CPP_API "Enables building of the TileDB C++ API." ON) option(TILEDB_CMAKE_IDE "(Used for CLion builds). Disables superbuild and sets the EP install dir." OFF) option(TILEDB_STATS "Enables internal TileDB statistics gathering." ON) -option(TILEDB_STATIC "Enables building TileDB as a static library." OFF) +option(BUILD_SHARED_LIBS "Enables building TileDB as a shared library." OFF) option(TILEDB_TESTS "If true, enables building the TileDB unit test suite" ON) option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index fe60c1e0036a..5af56ab2dc76 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -24,6 +24,7 @@ list(JOIN CMAKE_PREFIX_PATH "|" CMAKE_PREFIX_PATH_STR) # Forward any additional CMake args to the non-superbuild. set(INHERITED_CMAKE_ARGS + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH_STR} -DCMAKE_BUILD_TYPE=$ @@ -47,7 +48,6 @@ set(INHERITED_CMAKE_ARGS -DSANITIZER=${SANITIZER} -DTILEDB_EP_BASE=${TILEDB_EP_BASE} -DTILEDB_STATS=${TILEDB_STATS} - -DTILEDB_STATIC=${TILEDB_STATIC} -DTILEDB_TESTS=${TILEDB_TESTS} -DTILEDB_TOOLS=${TILEDB_TOOLS} -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in index 49c85082268a..ccea6a22cb32 100644 --- a/cmake/inputs/Config.cmake.in +++ b/cmake/inputs/Config.cmake.in @@ -1,8 +1,9 @@ # # This file attempts to locate the TileDB library. If found, the following # imported targets are created: -# - TileDB::tiledb_shared -# - TileDB::tiledb_static +# - TileDB::tiledb (points to either a static or shared library) +# - TileDB::tiledb_shared (if TileDB is built as a shared library) +# - TileDB::tiledb_static (if TileDB is built as a static library) # And the following variables are defined: # - TILEDB_FOUND # - TileDB_FOUND @@ -10,7 +11,7 @@ @PACKAGE_INIT@ -if(@TILEDB_STATIC@) # TILEDB_STATIC +if(NOT @BUILD_SHARED_LIBS@) # NOT BUILD_SHARED_LIBS include(CMakeFindDependencyMacro) find_dependency(Threads) find_dependency(BZip2) @@ -43,9 +44,15 @@ endif() include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") check_required_components("@PROJECT_NAME@") +if(@BUILD_SHARED_LIBS@ AND NOT TARGET TileDB::tiledb_shared) # BUILD_SHARED_LIBS AND NOT TARGET TileDB::tiledb_shared + add_library(TileDB::tiledb_shared ALIAS TileDB::tiledb) +elseif(NOT TARGET TileDB::tiledb_static) + add_library(TileDB::tiledb_static ALIAS TileDB::tiledb) +endif() + # Define a convenience all-caps variable if (NOT DEFINED TILEDB_FOUND) - if (TARGET TileDB::tiledb_shared) + if (TARGET TileDB::tiledb) set(TILEDB_FOUND TRUE) else() set(TILEDB_FOUND FALSE) diff --git a/examples/c_api/CMakeLists.txt b/examples/c_api/CMakeLists.txt index e145d35d4acf..e52506ebb09e 100644 --- a/examples/c_api/CMakeLists.txt +++ b/examples/c_api/CMakeLists.txt @@ -31,16 +31,12 @@ set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) # Don't use GNU extensions # Add custom target 'examples_c' -add_custom_target(examples_c DEPENDS tiledb_shared) - -if (TILEDB_STATIC) - add_custom_target(static_examples_c DEPENDS tiledb_static) -endif() +add_custom_target(examples_c) # Function that builds an executable per example function(build_TileDB_example_capi TARGET) add_executable(${TARGET}_c EXCLUDE_FROM_ALL ${TARGET}.c) - target_link_libraries(${TARGET}_c PUBLIC local_install tiledb_shared) + target_link_libraries(${TARGET}_c PUBLIC local_install tiledb) if (NOT WIN32) # On Linux, must explicitly link -lpthread -ldl in order for static linking # to libzstd or libcurl to work. @@ -49,20 +45,6 @@ function(build_TileDB_example_capi TARGET) add_dependencies(examples_c ${TARGET}_c) endfunction() -function(build_TileDB_static_example_capi TARGET) - add_executable(static_${TARGET}_c EXCLUDE_FROM_ALL ${TARGET}.c) - target_include_directories(static_${TARGET}_c PUBLIC - "${CMAKE_CURRENT_BINARY_DIR}/../" - ) - target_link_libraries(static_${TARGET}_c tiledb_static) - if (NOT WIN32) - # On Linux, must explicitly link -lpthread -ldl in order for static linking - # to libzstd or libcurl to work. - target_link_libraries(static_${TARGET}_c pthread dl) - endif() - add_dependencies(static_examples_c static_${TARGET}_c) -endfunction() - # Get the example sources file(GLOB TILEDB_EXAMPLE_SOURCES_CAPI "*.c") @@ -80,7 +62,4 @@ foreach(EXAMPLE_SOURCE ${TILEDB_EXAMPLE_SOURCES_CAPI}) # Build example executable build_TileDB_example_capi(${EXAMPLE_BIN}) - if (TILEDB_STATIC) - build_TileDB_static_example_capi(${EXAMPLE_BIN}) - endif() endforeach() diff --git a/examples/c_api/vfs.c b/examples/c_api/vfs.c index 921f65552145..ffbcb0f0f720 100644 --- a/examples/c_api/vfs.c +++ b/examples/c_api/vfs.c @@ -36,7 +36,7 @@ #include #include -void dirs_files() { +static void do_dirs_files() { // Create TileDB context tiledb_ctx_t* ctx; tiledb_ctx_alloc(NULL, &ctx); @@ -92,7 +92,7 @@ void dirs_files() { tiledb_ctx_free(&ctx); } -void write() { +static void do_write() { // Create TileDB context tiledb_ctx_t* ctx; tiledb_ctx_alloc(NULL, &ctx); @@ -132,7 +132,7 @@ void write() { tiledb_ctx_free(&ctx); } -void read() { +static void do_read() { // Create TileDB context tiledb_ctx_t* ctx; tiledb_ctx_alloc(NULL, &ctx); @@ -158,7 +158,7 @@ void read() { } int main() { - dirs_files(); - write(); - read(); + do_dirs_files(); + do_write(); + do_read(); } diff --git a/examples/cmake_project/CMakeLists.txt b/examples/cmake_project/CMakeLists.txt index f002264d3de4..31dcd48d53c2 100644 --- a/examples/cmake_project/CMakeLists.txt +++ b/examples/cmake_project/CMakeLists.txt @@ -50,10 +50,4 @@ add_executable(ExampleExe "src/main.cc") # Link the example program with the TileDB shared library. # This also configures include paths to find the TileDB headers. -target_link_libraries(ExampleExe TileDB::tiledb_shared) - -# Link a second executable with the TileDB static library, if it is available. -if (TARGET TileDB::tiledb_static) - add_executable(ExampleExe_static "src/main.cc") - target_link_libraries(ExampleExe_static TileDB::tiledb_static) -endif() +target_link_libraries(ExampleExe TileDB::tiledb) diff --git a/examples/cpp_api/CMakeLists.txt b/examples/cpp_api/CMakeLists.txt index 134904eb5ad4..0226f9f2b6e5 100644 --- a/examples/cpp_api/CMakeLists.txt +++ b/examples/cpp_api/CMakeLists.txt @@ -26,12 +26,12 @@ # # Add custom target 'examples_cpp' -add_custom_target(examples_cpp DEPENDS tiledb_shared) +add_custom_target(examples_cpp) # Function that builds an executable per example function(build_TileDB_example_cppapi TARGET) add_executable(${TARGET}_cpp EXCLUDE_FROM_ALL ${TARGET}.cc) - target_link_libraries(${TARGET}_cpp PUBLIC local_install tiledb_shared) + target_link_libraries(${TARGET}_cpp PUBLIC local_install tiledb) if (NOT WIN32) # On Linux, must explicitly link -lpthread -ldl in order for static linking # to libzstd or libcurl to work. diff --git a/examples/cpp_api/quickstart_dense.cc b/examples/cpp_api/quickstart_dense.cc index fe6bc8384d76..2c0a3efe6a13 100644 --- a/examples/cpp_api/quickstart_dense.cc +++ b/examples/cpp_api/quickstart_dense.cc @@ -41,34 +41,31 @@ using namespace tiledb; // Name of array. std::string array_name_("mem://quickstart_dense_array"); -// Example-global Context object. -Context ctx_; - -void create_array() { +void create_array(const Context& ctx) { // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. - Domain domain(ctx_); - domain.add_dimension(Dimension::create(ctx_, "rows", {{1, 4}}, 4)) - .add_dimension(Dimension::create(ctx_, "cols", {{1, 4}}, 4)); + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "rows", {{1, 4}}, 4)) + .add_dimension(Dimension::create(ctx, "cols", {{1, 4}}, 4)); // The array will be dense. - ArraySchema schema(ctx_, TILEDB_DENSE); + ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); // Add a single attribute "a" so each (i,j) cell can store an integer. - schema.add_attribute(Attribute::create(ctx_, "a")); + schema.add_attribute(Attribute::create(ctx, "a")); // Create the (empty) array on disk. Array::create(array_name_, schema); } -void write_array() { +void write_array(const Context& ctx) { // Prepare some data for the array std::vector data = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; // Open the array for writing and create the query. - Array array(ctx_, array_name_, TILEDB_WRITE); - Query query(ctx_, array, TILEDB_WRITE); + Array array(ctx, array_name_, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); query.set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", data); // Perform the write and close the array. @@ -76,19 +73,19 @@ void write_array() { array.close(); } -void read_array() { +void read_array(const Context& ctx) { // Prepare the array for reading - Array array(ctx_, array_name_, TILEDB_READ); + Array array(ctx, array_name_, TILEDB_READ); // Slice only rows 1, 2 and cols 2, 3, 4 - Subarray subarray(ctx_, array); + Subarray subarray(ctx, array); subarray.add_range(0, 1, 2).add_range(1, 2, 4); // Prepare the vector that will hold the result (of size 6 elements) std::vector data(6); // Prepare the query - Query query(ctx_, array, TILEDB_READ); + Query query(ctx, array, TILEDB_READ); query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data); @@ -104,11 +101,13 @@ void read_array() { } int main() { - if (Object::object(ctx_, array_name_).type() != Object::Type::Array) { - create_array(); - write_array(); + // Create TileDB context + Context ctx; + if (Object::object(ctx, array_name_).type() != Object::Type::Array) { + create_array(ctx); + write_array(ctx); } - read_array(); + read_array(ctx); return 0; } diff --git a/examples/png_ingestion/CMakeLists.txt b/examples/png_ingestion/CMakeLists.txt index 2e791c8c18f8..6bdf16546c23 100644 --- a/examples/png_ingestion/CMakeLists.txt +++ b/examples/png_ingestion/CMakeLists.txt @@ -50,7 +50,7 @@ add_executable(tiledb_png "src/main.cc") # Link the example program with the TileDB shared library. # This also configures include paths to find the TileDB headers. -target_link_libraries(tiledb_png TileDB::tiledb_shared) +target_link_libraries(tiledb_png TileDB::tiledb) # Find and link with libpng. find_package(Zlib_EP REQUIRED) diff --git a/experimental/experimental_examples/cpp_api/CMakeLists.txt b/experimental/experimental_examples/cpp_api/CMakeLists.txt index 8da6d16d2d2c..192c31892822 100644 --- a/experimental/experimental_examples/cpp_api/CMakeLists.txt +++ b/experimental/experimental_examples/cpp_api/CMakeLists.txt @@ -25,14 +25,14 @@ # THE SOFTWARE. # -add_custom_target(experimental_examples_cpp DEPENDS tiledb_shared) +add_custom_target(experimental_examples_cpp) find_package(Bzip2_EP REQUIRED) # Function that builds an executable per example function(build_TileDB_experimental_example_cppapi TARGET) add_executable(${TARGET}_exp_cpp EXCLUDE_FROM_ALL ${TARGET}.cc) - target_link_libraries(${TARGET}_exp_cpp PUBLIC local_install tiledb_shared) + target_link_libraries(${TARGET}_exp_cpp PUBLIC local_install tiledb) target_link_libraries(${TARGET}_exp_cpp PUBLIC BZip2::BZip2) if (NOT WIN32) # On Linux, must explicitly link -lpthread -ldl in order for static linking diff --git a/scripts/ci/build_benchmarks.sh b/scripts/ci/build_benchmarks.sh index 18b26bcf3613..a3de85b57ab1 100644 --- a/scripts/ci/build_benchmarks.sh +++ b/scripts/ci/build_benchmarks.sh @@ -32,29 +32,3 @@ cd build cmake -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/build/dist;$GITHUB_WORKSPACE/build/vcpkg_installed/$VCPKG_TARGET_TRIPLET" ../src make popd - -testfile=$(mktemp) -mv $testfile $testfile.cc -testfile=$testfile.cc -cat < $testfile -#include -#include -#include -int main(int argc, char **argv) { - int major = 0; - int minor = 0; - int patch = 0; - tiledb_version(&major,&minor,&patch); - auto version = tiledb::version(); - assert(major == std::get<0>(version)); - return 0; -} -EOF -export TESTFILE_LDFLAGS="-ltiledb" -export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/build/dist/lib:/usr/local/lib:${LD_LIBRARY_PATH:-} -$CXX -std=c++11 -g -O0 -Wall -Werror -I$GITHUB_WORKSPACE/build/dist/include -L$GITHUB_WORKSPACE/build/dist/lib $testfile -o $testfile.exe $TESTFILE_LDFLAGS && \ -$testfile.exe && \ -rm -f $testfile $testfile.exe - -ps -U $(whoami) -o comm= | sort | uniq -# displayName: 'Build examples, PNG test, and benchmarks (build-only)' diff --git a/scripts/run-nix-examples.sh b/scripts/run-nix-examples.sh old mode 100644 new mode 100755 index 5743cf882b3d..b99cabd1b83d --- a/scripts/run-nix-examples.sh +++ b/scripts/run-nix-examples.sh @@ -1,20 +1,28 @@ #!/bin/bash +SourceDir="$(dirname $0)/.." BaseDir="$(pwd)" + TestAppDir="$(pwd)/tiledb/examples/c_api" TestAppDataDir="$(pwd)/tiledb/examples/c_api/test_app_data" -for exampleexe in $(ls ${TestAppDir}/*_c) ; +for example in $(ls ${SourceDir}/examples/c_api/*.c) ; do cd ${TestAppDir} rm -rf ${TestAppDataDir} mkdir ${TestAppDataDir} cd ${TestAppDataDir} - echo $exampleexe - $exampleexe; + exampleexe=${example%.c}_c + exampleexe=${exampleexe##*/} + cmake --build ${BaseDir}/tiledb --target ${exampleexe} + echo $TestAppDir/$exampleexe + $TestAppDir/$exampleexe; status=$? + # Remove the executable after running it to prevent disk + # space exhaustion when statically linking to tiledb. + rm $TestAppDir/$exampleexe if (($status != 0)); then echo "FAILED: $exampleexe exited with $status" - echo "::set-output name=TILEDB_CI_SUCCESS::0" + echo "TILEDB_CI_SUCCESS=0" >> $GITHUB_OUTPUT fi done cd ${TestAppDir} @@ -23,10 +31,10 @@ rm -rf ${TestAppDataDir} cd ${BaseDir} TestAppDir="$(pwd)/tiledb/examples/cpp_api" TestAppDataDir="$(pwd)/tiledb/examples/cpp_api/test_app_data" -for exampleexe in $(ls ${TestAppDir}/*_cpp) ; +for example in $(ls ${SourceDir}/examples/cpp_api/*.cc) ; do # Skip running WebP example with no input - if [ "${exampleexe##*/}" == png_ingestion_webp_cpp ]; then + if [ "${example##*/}" == png_ingestion_webp.cc ]; then continue fi; @@ -34,12 +42,16 @@ do rm -rf ${TestAppDataDir} mkdir ${TestAppDataDir} cd ${TestAppDataDir} - echo $exampleexe - $exampleexe; + exampleexe=${example%.cc}_cpp + exampleexe=${exampleexe##*/} + cmake --build ${BaseDir}/tiledb --target ${exampleexe} + echo $TestAppDir/$exampleexe + $TestAppDir/$exampleexe; status=$? + rm $TestAppDir/$exampleexe if (($status != 0)); then echo "FAILED: $exampleexe exited with $status" - echo "::set-output name=TILEDB_CI_SUCCESS::0" + echo "TILEDB_CI_SUCCESS=0" >> $GITHUB_OUTPUT fi done cd ${TestAppDir} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fec08249ff7d..50316237e807 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -360,7 +360,7 @@ endif() # This is necessary only because we are linking directly to the core objects. # Other users (e.g. the examples) do not need this flag. -target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_CORE_OBJECTS_EXPORTS) +target_compile_definitions(tiledb_unit PRIVATE -Dtiledb_EXPORTS) target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_TEST_INPUTS_DIR="${CMAKE_CURRENT_SOURCE_DIR}/inputs" diff --git a/test/benchmarking/src/CMakeLists.txt b/test/benchmarking/src/CMakeLists.txt index 07d08b5ded2a..2fe1ebe09364 100644 --- a/test/benchmarking/src/CMakeLists.txt +++ b/test/benchmarking/src/CMakeLists.txt @@ -72,5 +72,5 @@ foreach(NAME IN LISTS BENCHMARKS) "${NAME}.cc" $ ) - target_link_libraries(${NAME} TileDB::tiledb_shared pthread) + target_link_libraries(${NAME} TileDB::tiledb pthread) endforeach() diff --git a/test/performance/CMakeLists.txt b/test/performance/CMakeLists.txt index f8db58905d26..6352041104c4 100644 --- a/test/performance/CMakeLists.txt +++ b/test/performance/CMakeLists.txt @@ -86,7 +86,7 @@ target_link_libraries(tiledb_explore_msys_handle_leakage # This is necessary only because we are linking directly to the core objects. # Other users (e.g. the examples) do not need this flag. -target_compile_definitions(tiledb_explore_msys_handle_leakage PRIVATE -DTILEDB_CORE_OBJECTS_EXPORTS) +target_compile_definitions(tiledb_explore_msys_handle_leakage PRIVATE -Dtiledb_EXPORTS) target_compile_definitions(tiledb_explore_msys_handle_leakage PRIVATE -DTILEDB_TEST_INPUTS_DIR="${CMAKE_CURRENT_SOURCE_DIR}/inputs" diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 0f193580ac48..2607aaff1316 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -68,7 +68,7 @@ target_link_libraries(tiledb_regression PUBLIC Catch2::Catch2WithMain local_install - tiledb_shared + tiledb ) target_include_directories(tiledb_regression diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index 0b97ae844051..e727cfb91e13 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -87,7 +87,7 @@ if (TILEDB_SERIALIZATION) target_compile_definitions(tiledb_test_support_lib PRIVATE -DTILEDB_SERIALIZATION) endif() -target_compile_definitions(tiledb_test_support_lib PRIVATE -DTILEDB_CORE_OBJECTS_EXPORTS) +target_compile_definitions(tiledb_test_support_lib PRIVATE -Dtiledb_EXPORTS) target_compile_definitions(tiledb_test_support_lib PRIVATE -DTILEDB_TEST_INPUTS_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../inputs" diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index ffee1c1fd6d2..530d2b8c68ba 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -415,7 +415,7 @@ add_library(TILEDB_CORE_OBJECTS OBJECT ${TILEDB_EXTERNALS_SOURCES} ) -# List of libraries to be linked to TILEDB_CORE_OBJECTS and TILEDB_CORE_OBJECTS_STATIC. +# List of libraries to be linked to TILEDB_CORE_OBJECTS. set(TILEDB_CORE_OBJECTS_LIBS baseline ) @@ -474,31 +474,6 @@ target_include_directories(TILEDB_CORE_OBJECTS "${CMAKE_CURRENT_BINARY_DIR}/.." ) -# Build a separate copy of the object library for use with static TileDB -# on Windows. See https://github.com/TileDB-Inc/TileDB/issues/673 -if (WIN32 AND TILEDB_STATIC) - add_library(TILEDB_CORE_OBJECTS_STATIC OBJECT - ${TILEDB_CORE_SOURCES} - ${TILEDB_EXTERNALS_SOURCES} - ) - add_dependencies(TILEDB_CORE_OBJECTS_STATIC gen_mgc_unarch) - - target_sources(TILEDB_CORE_OBJECTS_STATIC - PUBLIC - ${TILEDB_CORE_OBJECTS_LIBS_SOURCES} - ) - - target_link_libraries(TILEDB_CORE_OBJECTS_STATIC - PUBLIC - ${TILEDB_CORE_OBJECTS_LIBS} - ) - - target_compile_definitions(TILEDB_CORE_OBJECTS_STATIC - PRIVATE - -DTILEDB_STATIC_DEFINE - ) -endif() - if (TILEDB_CPP_API) target_include_directories(TILEDB_CORE_OBJECTS PRIVATE @@ -816,11 +791,6 @@ if(TILEDB_SERIALIZATION) target_sources(TILEDB_CORE_OBJECTS PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp.c++ ) - if (WIN32 AND TILEDB_STATIC) - target_sources(TILEDB_CORE_OBJECTS_STATIC PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp.c++ - ) - endif() ################################### # capnproto file generation target @@ -865,10 +835,12 @@ endif() ############################################################ -# TileDB static and shared library targets +# TileDB library targets ############################################################ -add_library(tiledb_shared SHARED $) +# CMake will determine the library's linkage from +# the value of the BUILD_SHARED_LIBS variable. +add_library(tiledb $) file(READ "${CMAKE_CURRENT_SOURCE_DIR}/sm/c_api/tiledb_version.h" ver) @@ -883,7 +855,7 @@ set(ver_patch ${CMAKE_MATCH_1}) set(VERSION "${ver_major}.${ver_minor}.${ver_patch}") -if (WIN32) +if (WIN32 AND BUILD_SHARED_LIBS) # WIN32 RC file creation (for VERSIONINFO) set(TDB_WIN32_FILEVERSION "${ver_major},${ver_minor},${ver_patch},0") set(TDB_WIN32_FILEVERSION_STR "\"${ver_major}.${ver_minor}.${ver_patch}\"") @@ -894,80 +866,32 @@ if (WIN32) ) # The .rc file will be created elsewhere (currently below) via configure_file() - target_sources(tiledb_shared PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/tiledb_win32_verinfo.rc) - target_include_directories(tiledb_shared + target_sources(tiledb PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/tiledb_win32_verinfo.rc) + target_include_directories(tiledb PRIVATE "${TILEDB_CORE_INCLUDE_DIR}" ) endif() -# Target properties -set_target_properties(tiledb_shared - PROPERTIES - OUTPUT_NAME "tiledb" -) - if (TILEDB_VERSION AND CMAKE_SYSTEM_NAME MATCHES "Linux") - set_target_properties(tiledb_shared + set_target_properties(tiledb PROPERTIES SOVERSION "${TILEDB_VERSION_MAJOR}.${TILEDB_VERSION_MINOR}") endif() # Link the dependencies specified earlier -target_link_libraries(tiledb_shared +target_link_libraries(tiledb PRIVATE $ - TILEDB_CORE_OBJECTS + $ ) -if (TILEDB_STATIC) - if (WIN32) - # Copy over all include directories, compile options, etc, from the regular - # core objects. - target_compile_definitions(TILEDB_CORE_OBJECTS_STATIC - PRIVATE - $ - ) - target_compile_options(TILEDB_CORE_OBJECTS_STATIC - PRIVATE - $ - ) - target_include_directories(TILEDB_CORE_OBJECTS_STATIC - PRIVATE - $ - ) - # Create the target - add_library(tiledb_static STATIC - $) - target_link_libraries(tiledb_static - PRIVATE - $ - ) - target_compile_definitions(tiledb_static - INTERFACE - -DTILEDB_STATIC_DEFINE - ) - # On Windows we must name the static library something else to avoid - # name clash with the DLL's "import library" .lib file. - set_target_properties(tiledb_static - PROPERTIES - OUTPUT_NAME "tiledbstatic" - ) - else() - add_library(tiledb_static STATIC $) - target_link_libraries(tiledb_static - PRIVATE - $ - ) - set_target_properties(tiledb_static - PROPERTIES - OUTPUT_NAME "tiledb" - ) - endif() - - target_link_libraries(tiledb_static - PRIVATE - $ +if (WIN32 AND NOT BUILD_SHARED_LIBS) + # On Windows we must name the static library something else to avoid + # name clash with the DLL's "import library" .lib file. + set_target_properties(tiledb + PROPERTIES + OUTPUT_NAME "tiledbstatic" ) endif() @@ -979,13 +903,13 @@ endif() # Generates the file 'tiledb_export.h' suitable for the current compiler. include(GenerateExportHeader) -generate_export_header(TILEDB_CORE_OBJECTS +generate_export_header(tiledb BASE_NAME tiledb ) # Install locally configure_file(${TILEDB_EXPORT_HEADER} ${TILEDB_LOCALINSTALL_INCLUDE}/${TILEDB_EXPORT_HEADER_LOCALINSTALL_PATH} COPYONLY) -target_compile_definitions(TILEDB_CORE_OBJECTS PRIVATE -DTILEDB_CORE_OBJECTS_EXPORTS) +target_compile_definitions(TILEDB_CORE_OBJECTS PRIVATE -Dtiledb_EXPORTS) target_include_directories(TILEDB_CORE_OBJECTS PRIVATE ${TILEDB_EXPORT_HEADER_DIR}) # Add the generated header to the public headers list @@ -994,23 +918,17 @@ list(APPEND TILEDB_PUBLIC_HEADERS ) # Set the public headers, which are the ones that get installed. -set_target_properties(tiledb_shared +set_target_properties(tiledb PROPERTIES PUBLIC_HEADER "${TILEDB_PUBLIC_HEADERS}" ) -if (TILEDB_STATIC) - set_target_properties(tiledb_static - PROPERTIES - PUBLIC_HEADER "${TILEDB_PUBLIC_HEADERS}" - ) -endif() # Don't re-export symbols from static (archive) libraries # Prevents conflicts with other versions of (e.g.) OpenSSL # loaded in the same process namespace, which can cause # crashes if the versions are not compatible. if (CMAKE_SYSTEM_NAME MATCHES "Linux") - set_target_properties(tiledb_shared + set_target_properties(tiledb PROPERTIES LINK_FLAGS "-Wl,--exclude-libs=ALL") endif() @@ -1025,8 +943,8 @@ include(GNUInstallDirs) include(CMakePackageConfigHelpers) # Set rpath so the TileDB dynamic dependencies can be located. -if (NOT WIN32) - set_target_properties(tiledb_shared +if (BUILD_SHARED_LIBS AND NOT WIN32) + set_target_properties(tiledb PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" ) @@ -1039,11 +957,10 @@ endif() # List of targets to install. set(TILEDB_INSTALL_TARGETS - tiledb_shared + tiledb ) -if (TILEDB_STATIC) +if (NOT BUILD_SHARED_LIBS) list(APPEND TILEDB_INSTALL_TARGETS - tiledb_static TILEDB_CORE_OBJECTS_ILIB object_store_definitions ) diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 8c4cc35e7940..11d62110bcde 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -32,7 +32,7 @@ include(common NO_POLICY_SCOPE) # - Include path for `c_api` headers # add_library(export INTERFACE) -target_compile_definitions(export INTERFACE -DTILEDB_CORE_OBJECTS_EXPORTS) +target_compile_definitions(export INTERFACE -Dtiledb_EXPORTS) target_include_directories(export INTERFACE ${TILEDB_EXPORT_HEADER_DIR}) target_include_directories(export INTERFACE ${TILEDB_SOURCE_ROOT}/tiledb/sm/c_api) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 218e238029e6..a2bf0edd0c99 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -31,14 +31,18 @@ else() find_package(Clipp_EP REQUIRED) endif() -add_executable(tiledb EXCLUDE_FROM_ALL +add_executable(tiledb-cli EXCLUDE_FROM_ALL src/commands/help_command.cc src/commands/info_command.cc src/main/tiledb.cc ) +set_target_properties(tiledb-cli PROPERTIES + OUTPUT_NAME tiledb +) + # List of all tool targets -set(TILEDB_TOOLS tiledb) +set(TILEDB_TOOLS tiledb-cli) # Set up link and include directories for all tools foreach(TOOL ${TILEDB_TOOLS}) From 657f942db4076e501fbf77b595421781acc3a966 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:04:17 -0500 Subject: [PATCH 100/456] Bump dev version to 2.20.0. (#4559) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.20.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index 97cdc8bf6fc5..b1523967ee46 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.19' +version = '2.20' # The full version, including alpha/beta/rc tags. -release = '2.19.0' +release = '2.20.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index f845c3a1b09a..ef785a9bf2df 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 19 +#define TILEDB_VERSION_MINOR 20 #define TILEDB_VERSION_PATCH 0 From 70813c585a210ff063fa435ffde3eaa42ff7cfe7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 12 Dec 2023 05:16:24 +0200 Subject: [PATCH 101/456] Build shared libraries by default. (#4569) Restores default behavior to prior to #4528. Supersedes #4560. Fixes TileDB-Inc/TileDB-MariaDB#294 Fixes TileDB-Inc/TileDB-CSharp#339 Fixes TileDB-Inc/TileDB-CSharp#340 Fixes TileDB-Inc/TileDB-CSharp#341 --- TYPE: BUILD DESC: `BUILD_SHARED_LIBS` is enabled by default. --- .github/workflows/build-windows.yml | 2 ++ bootstrap | 14 +++++++------- bootstrap.ps1 | 15 ++++++++------- cmake/Options/BuildOptions.cmake | 2 +- test/support/CMakeLists.txt | 2 +- tiledb/sm/query/ast/test/CMakeLists.txt | 2 +- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2f8cfcb860fa..5862eb93ac3c 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -371,6 +371,8 @@ jobs: run: | cd $env:BUILD_SOURCESDIRECTORY\examples\cmake_project + $env:Path += ";$env:BUILD_BUILDDIRECTORY\dist\bin;$env:BUILD_BUILDDIRECTORY\externals\install\bin" + $CMakeBuildType = $env:TILEDB_CMAKE_BUILD_TYPE mkdir build diff --git a/bootstrap b/bootstrap index 1878c54df6d5..4ac8f1e6c4d7 100755 --- a/bootstrap +++ b/bootstrap @@ -37,7 +37,7 @@ Configuration: --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] - --linkage specify the linkage of tiledb. Defaults to static. + --linkage specify the linkage of tiledb. Defaults to shared. [static|shared] --force-build-all-deps force building of all dependencies, even those already installed at system-level @@ -143,7 +143,7 @@ while test $# != 0; do --enable-azure) tiledb_azure="ON";; --enable-gcs) tiledb_gcs="ON";; --enable-serialization) tiledb_serialization="ON";; - --enable-static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. TileDB is now built as a static library by default and linkage is controlled with the --linkage option." + --enable-static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. Use --linkage=static instead." enable_static_tiledb="ON";; --enable-sanitizer=*) san=`arg "$1"` sanitizer="$san";; @@ -180,7 +180,7 @@ for en in "${enables[@]}"; do ccache) tiledb_ccache="ON";; arrow-tests) tiledb_arrow_tests="ON";; hdfs) tiledb_hdfs="ON";; - static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. TileDB is now built as a static library by default and linkage is controlled with the --linkage option." + static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. Use --linkage=static instead." enable_static_tiledb="ON";; experimental-features) tiledb_experimental_features="ON";; rest-tests) tiledb_tests_enable_rest="ON";; @@ -193,16 +193,16 @@ if [ "${source_dir}" = "${binary_dir}" ]; then die "cannot build the project in the source directory! Out-of-source build is enforced!" fi -if [ "${linkage}" = "static" ] || [ "${linkage}" = "" ]; then - build_shared_libs="OFF" -elif [ "${linkage}" = "shared" ]; then +if [ "${linkage}" = "shared" ] || [ "${linkage}" = "" ]; then build_shared_libs="ON" +elif [ "${linkage}" = "static" ] || [ "${enable_static_tiledb}" = "ON" ]; then + build_shared_libs="OFF" else die "unknown linkage: ${linkage}" fi # Fail if both --linkage=shared and --enable-static-tiledb are specified -if [ "${build_shared_libs}" = "ON" ] && [ "${enable_static_tiledb}" = "ON" ]; then +if [ "${linkage}" = "shared" ] && [ "${enable_static_tiledb}" = "ON" ]; then die "cannot specify both --linkage=shared and --enable-static-tiledb" fi diff --git a/bootstrap.ps1 b/bootstrap.ps1 index bda0743e4699..bf49fce5d671 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -18,7 +18,7 @@ Semicolon separated list to binary dependencies. .PARAMETER Linkage Specify the linkage type to build TileDB with. Valid values are -"static" and "shared". Default is "static". +"static" and "shared". Default is "shared". .PARAMETER CMakeGenerator Optionally specify the CMake generator string, e.g. "Visual Studio 15 @@ -118,7 +118,7 @@ https://github.com/TileDB-Inc/TileDB Param( [string]$Prefix, [string]$Dependency, - [string]$Linkage = "static", + [string]$Linkage = "shared", [string]$CMakeGenerator, [switch]$EnableAssert, [switch]$EnableDebug, @@ -246,17 +246,18 @@ if ($DisableWebP.IsPresent) { $BuildWebP="OFF" } -$BuildSharedLibs = "OFF"; -if ($Linkage -eq "shared") { - $BuildSharedLibs = "ON" +$BuildSharedLibs = "ON"; +if ($Linkage -eq "static") { + $BuildSharedLibs = "OFF" } -elseif ($Linkage -ne "static") { +elseif ($Linkage -ne "shared") { Write-Error "Invalid linkage type: $Linkage. Valid values are 'static' and 'shared'." exit 1 } if ($EnableStaticTileDB.IsPresent) { - Write-Warning "-EnableStaticTileDB is deprecated and will be removed in a future version. TileDB is now built as a static library by default. Use -Linkage shared to build a shared library." + Write-Warning "-EnableStaticTileDB is deprecated and will be removed in a future version. Use -Linkage static instead." + $BuildSharedLibs = "OFF" if ($Linkage -eq "shared") { Write-Error "Cannot specify -EnableStaticTileDB alongside -Linkage shared." exit 1 diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index a5a412e6f7fb..d7ffcf72db23 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -21,7 +21,7 @@ option(TILEDB_ASSERTIONS "Build with assertions enabled (default off for release option(TILEDB_CPP_API "Enables building of the TileDB C++ API." ON) option(TILEDB_CMAKE_IDE "(Used for CLion builds). Disables superbuild and sets the EP install dir." OFF) option(TILEDB_STATS "Enables internal TileDB statistics gathering." ON) -option(BUILD_SHARED_LIBS "Enables building TileDB as a shared library." OFF) +option(BUILD_SHARED_LIBS "Enables building TileDB as a shared library." ON) option(TILEDB_TESTS "If true, enables building the TileDB unit test suite" ON) option(TILEDB_TOOLS "If true, enables building the TileDB tools" OFF) option(TILEDB_SERIALIZATION "If true, enables building with support for query serialization" OFF) diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index e727cfb91e13..977d65a60480 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -50,7 +50,7 @@ if (TILEDB_VERBOSE) add_definitions(-DTILEDB_VERBOSE) endif() -add_library(tiledb_test_support_lib EXCLUDE_FROM_ALL +add_library(tiledb_test_support_lib STATIC EXCLUDE_FROM_ALL $ ${TILEDB_TEST_SUPPORT_SOURCES} ) diff --git a/tiledb/sm/query/ast/test/CMakeLists.txt b/tiledb/sm/query/ast/test/CMakeLists.txt index 0f687210769e..9597f820076e 100644 --- a/tiledb/sm/query/ast/test/CMakeLists.txt +++ b/tiledb/sm/query/ast/test/CMakeLists.txt @@ -36,7 +36,7 @@ set(AST_TEST_SUPPORT_SOURCES ${CMAKE_SOURCE_DIR}/test/support/src/ast_helpers.h ${CMAKE_SOURCE_DIR}/test/support/src/ast_helpers.cc ) -add_library(ast_test_support_lib EXCLUDE_FROM_ALL ${AST_TEST_SUPPORT_SOURCES}) +add_library(ast_test_support_lib STATIC EXCLUDE_FROM_ALL ${AST_TEST_SUPPORT_SOURCES}) # We want tests to continue as normal even as the API is changing, # so don't warn for deprecations, since they'll be escalated to errors. if (NOT MSVC) From ae5cc5ae20aa5c3d3fd9568adadf4b106afdd7b1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 12 Dec 2023 10:59:33 +0200 Subject: [PATCH 102/456] Always install libraries in the `lib` directory. (#4562) Fixes regression introduced in #4518. --- TYPE: BUG DESC: Fix regression when libraries were sometimes installed in the `lib64` directory. --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77f894ce7f7f..6718e9dd43a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -84,6 +84,7 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=./dist \ + -DTILEDB_INSTALL_LIBDIR=lib \ -DTILEDB_S3=ON \ -DTILEDB_AZURE=ON \ -DTILEDB_GCS=ON \ From 9249276d82d60845c94aef0d4419704e566f94e7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 12 Dec 2023 11:00:54 +0200 Subject: [PATCH 103/456] Support running REST CI on workflow-dispatch. (#4568) REST CI can not run on pull requests from forks (because it needs a secret to access a private repository, and that secret is not available there). This PR makes possible one way to run it on such pull requests, by copying the branch to the `TileDB-Inc/TileDB` repo and running it manually, without having to run all other CI jobs. --- TYPE: NO_HISTORY --- .github/workflows/ci-rest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 5f9353cc9229..4c69656c1de5 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -1,5 +1,8 @@ name: REST CI -on: workflow_call + +on: + - workflow_call + - workflow_dispatch jobs: rest-ci: From b4f920d1faaa240eefe8dc551b35e9b4eaa575ac Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:37:15 +0100 Subject: [PATCH 104/456] Add setting to disable install_sigpipe_handler on S3. (#4573) This adds a setting that allows to disable install_sigpipe_handler on S3. When set to `true`, the S3 SDK uses a handler that ignores SIGPIPE signals. --- TYPE: IMPROVEMENT DESC: Add setting to disable install_sigpipe_handler on S3. --- test/src/unit-capi-config.cc | 4 ++++ test/src/unit-cppapi-config.cc | 2 +- tiledb/api/c_api/config/config_api_external.h | 4 ++++ tiledb/sm/config/config.cc | 4 ++++ tiledb/sm/config/config.h | 6 ++++++ tiledb/sm/cpp_api/config.h | 4 ++++ tiledb/sm/filesystem/s3.cc | 3 ++- 7 files changed, 25 insertions(+), 2 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 6b5920eaa406..6418fdcc7b77 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -324,6 +324,7 @@ void check_save_to_file() { ss << "vfs.s3.connect_max_tries 5\n"; ss << "vfs.s3.connect_scale_factor 25\n"; ss << "vfs.s3.connect_timeout_ms 10800\n"; + ss << "vfs.s3.install_sigpipe_handler true\n"; ss << "vfs.s3.logging_level Off\n"; ss << "vfs.s3.max_parallel_ops " << std::thread::hardware_concurrency() << "\n"; @@ -741,6 +742,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["vfs.s3.proxy_username"] = ""; all_param_values["vfs.s3.verify_ssl"] = "true"; all_param_values["vfs.s3.no_sign_request"] = "false"; + all_param_values["vfs.s3.install_sigpipe_handler"] = "true"; all_param_values["vfs.hdfs.username"] = "stavros"; all_param_values["vfs.hdfs.kerb_ticket_cache_path"] = ""; all_param_values["vfs.hdfs.name_node_uri"] = ""; @@ -813,6 +815,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { vfs_param_values["s3.bucket_canned_acl"] = "NOT_SET"; vfs_param_values["s3.object_canned_acl"] = "NOT_SET"; vfs_param_values["s3.config_source"] = "auto"; + vfs_param_values["s3.install_sigpipe_handler"] = "true"; vfs_param_values["hdfs.username"] = "stavros"; vfs_param_values["hdfs.kerb_ticket_cache_path"] = ""; vfs_param_values["hdfs.name_node_uri"] = ""; @@ -877,6 +880,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { s3_param_values["bucket_canned_acl"] = "NOT_SET"; s3_param_values["object_canned_acl"] = "NOT_SET"; s3_param_values["config_source"] = "auto"; + s3_param_values["install_sigpipe_handler"] = "true"; // Create an iterator and iterate over all parameters tiledb_config_iter_t* config_iter = nullptr; diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 3b08cde07057..609ece4c7c0e 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -67,7 +67,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") { names.push_back(it->first); } // Check number of VFS params in default config object. - CHECK(names.size() == 64); + CHECK(names.size() == 65); } TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") { diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index b49135aebb1f..8d240c40680c 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -561,6 +561,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * from config files with support for web tokens, commonly used by EKS/ECS). * **Default**: auto *
+ * - `vfs.s3.install_sigpipe_handler`
+ * When set to `true`, the S3 SDK uses a handler that ignores SIGPIPE + * signals. + * **Default**: "true" * - `vfs.hdfs.name_node_uri"`
* Name node for HDFS.
* **Default**: "" diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 83b11dbe36f2..6afe3ad5c4ce 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -227,6 +227,7 @@ const std::string Config::VFS_S3_NO_SIGN_REQUEST = "false"; const std::string Config::VFS_S3_BUCKET_CANNED_ACL = "NOT_SET"; const std::string Config::VFS_S3_OBJECT_CANNED_ACL = "NOT_SET"; const std::string Config::VFS_S3_CONFIG_SOURCE = "auto"; +const std::string Config::VFS_S3_INSTALL_SIGPIPE_HANDLER = "true"; const std::string Config::VFS_HDFS_KERB_TICKET_CACHE_PATH = ""; const std::string Config::VFS_HDFS_NAME_NODE_URI = ""; const std::string Config::VFS_HDFS_USERNAME = ""; @@ -478,6 +479,9 @@ const std::map default_config_values = { std::make_pair( "vfs.s3.object_canned_acl", Config::VFS_S3_OBJECT_CANNED_ACL), std::make_pair("vfs.s3.config_source", Config::VFS_S3_CONFIG_SOURCE), + std::make_pair( + "vfs.s3.install_sigpipe_handler", + Config::VFS_S3_INSTALL_SIGPIPE_HANDLER), std::make_pair("vfs.hdfs.name_node_uri", Config::VFS_HDFS_NAME_NODE_URI), std::make_pair("vfs.hdfs.username", Config::VFS_HDFS_USERNAME), std::make_pair( diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index ffc4f4694d4c..5754effc26cf 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -568,6 +568,12 @@ class Config { /** Force making an unsigned request to s3 (false). */ static const std::string VFS_S3_NO_SIGN_REQUEST; + /** + * When set to `true`, the S3 SDK uses a handler that ignores SIGPIPE + * signals. + */ + static const std::string VFS_S3_INSTALL_SIGPIPE_HANDLER; + /** HDFS default kerb ticket cache path. */ static const std::string VFS_HDFS_KERB_TICKET_CACHE_PATH; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 34eb820e8b6a..afa8d8dcd251 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -739,6 +739,10 @@ class Config { * from config files with support for web tokens, commonly used by EKS/ECS). * **Default**: auto *
+ * - `vfs.s3.install_sigpipe_handler`
+ * When set to `true`, the S3 SDK uses a handler that ignores SIGPIPE + * signals. + * **Default**: "true" * - `vfs.hdfs.name_node_uri"`
* Name node for HDFS.
* **Default**: "" diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index a6c854984ecc..f645c6149486 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -329,7 +329,8 @@ S3::S3( // the AWS SDK to set its own signal handler to ignore SIGPIPE signals. A // SIGPIPE may be raised from the socket library when the peer disconnects // unexpectedly. - options_.httpOptions.installSigPipeHandler = true; + options_.httpOptions.installSigPipeHandler = + config.get("vfs.s3.install_sigpipe_handler", Config::must_find); // Initialize the library once per process. if (!s3_params_.skip_init_) From 368bb32ca06dbe5ff9915d851aebeb92da96b781 Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Wed, 13 Dec 2023 10:18:07 -0500 Subject: [PATCH 105/456] Jpd/move pipeline tests part 1 (#4565) Move simple filter pipeline unit tests out of `tiledb_unit` and into `filter` directory unit tests. This PR is a first step in refactoring the filter pipeline unit tests for running the pipeline. This is being submitted as an initial PR to isolate code movement from code changes in a follow-up PR. Non-movement changes include: * Create dummy `Stats` object for methods that require stats. * Change `new` and `delete` to `tdb_new` and `tdb_delete`, respectively. * Split test filter classes between header file and source file. --- TYPE: NO_HISTORY --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/unit-filter-pipeline.cc | 1867 +---------------- tiledb/sm/filter/test/CMakeLists.txt | 13 + .../sm/filter/test/add_1_in_place_filter.cc | 102 + tiledb/sm/filter/test/add_1_in_place_filter.h | 77 + .../test/add_1_including_metadata_filter.cc | 161 ++ .../test/add_1_including_metadata_filter.h | 79 + .../filter/test/add_1_out_of_place_filter.cc | 122 ++ .../filter/test/add_1_out_of_place_filter.h | 77 + .../sm/filter/test/add_n_in_place_filter.cc | 112 + tiledb/sm/filter/test/add_n_in_place_filter.h | 84 + .../sm/filter/test/pseudo_checksum_filter.cc | 120 ++ .../sm/filter/test/pseudo_checksum_filter.h | 77 + .../filter/test/unit_run_filter_pipeline.cc | 1546 ++++++++++++++ 13 files changed, 2684 insertions(+), 1753 deletions(-) create mode 100644 tiledb/sm/filter/test/add_1_in_place_filter.cc create mode 100644 tiledb/sm/filter/test/add_1_in_place_filter.h create mode 100644 tiledb/sm/filter/test/add_1_including_metadata_filter.cc create mode 100644 tiledb/sm/filter/test/add_1_including_metadata_filter.h create mode 100644 tiledb/sm/filter/test/add_1_out_of_place_filter.cc create mode 100644 tiledb/sm/filter/test/add_1_out_of_place_filter.h create mode 100644 tiledb/sm/filter/test/add_n_in_place_filter.cc create mode 100644 tiledb/sm/filter/test/add_n_in_place_filter.h create mode 100644 tiledb/sm/filter/test/pseudo_checksum_filter.cc create mode 100644 tiledb/sm/filter/test/pseudo_checksum_filter.h create mode 100644 tiledb/sm/filter/test/unit_run_filter_pipeline.cc diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index ed43e62fe13f..6e20ce2cb97d 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -299,90 +299,6 @@ class Add1OutOfPlace : public tiledb::sm::Filter { } }; -/** - * Simple filter that modifies the input stream by adding a constant value to - * every input element. - */ -class AddNInPlace : public tiledb::sm::Filter { - public: - // Just use a dummy filter type - AddNInPlace(Datatype filter_data_type) - : Filter(FilterType::FILTER_NONE, filter_data_type) { - increment_ = 1; - } - - void dump(FILE* out) const override { - (void)out; - } - - Status run_forward( - const WriterTile&, - WriterTile* const, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output) const override { - auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); - output->reset_offset(); - - uint64_t nelts = input_size / sizeof(uint64_t); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t* val = output->value_ptr(); - *val += increment_; - output->advance_offset(sizeof(uint64_t)); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); - } - - Status run_reverse( - const Tile&, - Tile*, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output, - const tiledb::sm::Config& config) const override { - (void)config; - - auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); - output->reset_offset(); - - uint64_t nelts = input_size / sizeof(uint64_t); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t* val = output->value_ptr(); - *val -= increment_; - output->advance_offset(sizeof(uint64_t)); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); - } - - uint64_t increment() const { - return increment_; - } - - void set_increment(uint64_t increment) { - increment_ = increment; - } - - AddNInPlace* clone_impl() const override { - auto clone = new AddNInPlace(filter_data_type_); - clone->increment_ = increment_; - return clone; - } - - private: - uint64_t increment_; -}; - /** * Simple filter which computes the sum of its input and prepends the sum * to the output. In reverse execute, checks that the sum is correct. @@ -473,194 +389,103 @@ class PseudoChecksumFilter : public tiledb::sm::Filter { } }; -/** - * Simple filter that increments every element of the input stream, writing the - * output to a new buffer. The input metadata is treated as a part of the input - * data. - */ -class Add1IncludingMetadataFilter : public tiledb::sm::Filter { - public: - // Just use a dummy filter type - Add1IncludingMetadataFilter(Datatype filter_data_type) - : Filter(FilterType::FILTER_NONE, filter_data_type) { - } +TEST_CASE("Filter: Test compression", "[filter][compression]") { + tiledb::sm::Config config; - void dump(FILE* out) const override { - (void)out; - } + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); - Status run_forward( - const WriterTile&, - WriterTile* const, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output) const override { - auto input_size = static_cast(input->size()), - input_md_size = static_cast(input_metadata->size()); - auto nelts = input_size / sizeof(uint64_t), - md_nelts = input_md_size / sizeof(uint64_t); + // Set up dummy array schema (needed by compressor filter for cell size, etc). + uint32_t dim_dom[] = {1, 10}; + auto dim{make_shared(HERE(), "", Datatype::INT32)}; + CHECK(dim->set_domain(dim_dom).ok()); + auto domain{make_shared(HERE())}; + CHECK(domain->add_dimension(dim).ok()); + tiledb::sm::ArraySchema schema; + tiledb::sm::Attribute attr("attr", Datatype::UINT64); + CHECK(schema.add_attribute(make_shared(HERE(), attr)) + .ok()); + CHECK(schema.set_domain(domain).ok()); - // Add a new output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size + input_md_size)); - output->reset_offset(); + FilterPipeline pipeline; + ThreadPool tp(4); - // Filter input data - for (uint64_t i = 0; i < nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); - inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); - } - // Finish any remaining bytes to ensure no data loss. - auto rem = input_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); - } + SECTION("- Simple") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - // Now filter input metadata. - for (uint64_t i = 0; i < md_nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input_metadata->read(&inc, sizeof(uint64_t))); - inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); - } - rem = input_md_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input_metadata->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); - } + CHECK( + pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + // Check compression worked + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - // Because this filter modifies the input metadata, we need output metadata - // that allows the original metadata to be reconstructed on reverse. Also - // note that contrary to most filters, we don't forward the input metadata. - RETURN_NOT_OK(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_md_size, sizeof(uint32_t))); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); - return Status::Ok(); + // Check all elements original values. + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } } - Status run_reverse( - const Tile&, - Tile*, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output, - const tiledb::sm::Config& config) const override { - (void)config; - - if (input_metadata->size() != 2 * sizeof(uint32_t)) - return Status_FilterError("Unexpected input metadata length"); - - uint32_t orig_input_size, orig_md_size; - RETURN_NOT_OK(input_metadata->read(&orig_input_size, sizeof(uint32_t))); - RETURN_NOT_OK(input_metadata->read(&orig_md_size, sizeof(uint32_t))); + SECTION("- With checksum stage") { + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - // Add a new output buffer. - RETURN_NOT_OK(output->prepend_buffer(orig_input_size)); - // Add a new output metadata buffer. - RETURN_NOT_OK(output_metadata->prepend_buffer(orig_md_size)); + CHECK( + pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + // Check compression worked + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - // Restore original data - auto nelts = orig_input_size / sizeof(uint64_t); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); - inc--; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); - } - auto rem = orig_input_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); - } + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); - // Restore original metadata - nelts = orig_md_size / sizeof(uint64_t); + // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); - inc--; - RETURN_NOT_OK(output_metadata->write(&inc, sizeof(uint64_t))); - } - rem = orig_md_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output_metadata->write(&byte, sizeof(char))); + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); } - - return Status::Ok(); } - Add1IncludingMetadataFilter* clone_impl() const override { - return new Add1IncludingMetadataFilter(filter_data_type_); - } -}; - -TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + SECTION("- With multiple stages") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); + CHECK( + pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + // Check compression worked + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - // Check all elements unchanged. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i); - offset += sizeof(uint64_t); - } + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); + // Check all elements original values. + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } } } -TEST_CASE( - "Filter: Test empty pipeline var sized", "[filter][empty-pipeline][var]") { +TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { tiledb::sm::Config config; - // Set up test data const uint64_t nelts = 100; auto tile = make_increasing_tile(nelts); @@ -695,152 +520,40 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets); - FilterPipeline pipeline; - ThreadPool tp(4); - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + // Set up dummy array schema (needed by compressor filter for cell size, etc). + uint32_t dim_dom[] = {1, 10}; + auto dim{make_shared(HERE(), "", Datatype::INT32)}; + CHECK(dim->set_domain(dim_dom).ok()); + auto domain{make_shared(HERE())}; + CHECK(domain->add_dimension(dim).ok()); + tiledb::sm::ArraySchema schema; + tiledb::sm::Attribute attr("attr", Datatype::UINT64); + CHECK(schema.add_attribute(make_shared(HERE(), attr)) .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements unchanged. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == el++); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE( - "Filter: Test simple in-place pipeline", "[filter][simple-in-place]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + CHECK(schema.set_domain(domain).ok()); FilterPipeline pipeline; ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - - SECTION("- Single stage") { - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - SECTION("- Multi-stage") { - // Add a few more +1 filters and re-run. - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + SECTION("- Simple") { + WriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - // Check new size and number of chunks + CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + .ok()); + // Check number of chunks CHECK(tile.size() == 0); CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); - offset += sizeof(uint64_t); - } + tile.filtered_buffer().value_at_as(0) == + 9); // Number of chunks auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); run_reverse(config, tp, unfiltered_tile, pipeline); + + // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; CHECK_NOTHROW( @@ -848,92 +561,25 @@ TEST_CASE( CHECK(elt == i); } } -} - -TEST_CASE( - "Filter: Test simple in-place pipeline var", - "[filter][simple-in-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - - SECTION("- Single stage") { + SECTION("- With checksum stage") { WriterTile::set_max_tile_chunk_size(80); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) .ok()); - - // Check new size and number of chunks + // Check number of chunks CHECK(tile.size() == 0); CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile.filtered_buffer().value_at_as(0) == 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); - offset += sizeof(uint64_t); - } - } auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); run_reverse(config, tp, unfiltered_tile, pipeline); + + // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; CHECK_NOTHROW( @@ -942,50 +588,26 @@ TEST_CASE( } } - SECTION("- Multi-stage") { - // Add a few more +1 filters and re-run. + SECTION("- With multiple stages") { WriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) .ok()); - - // Check new size and number of chunks + // Check number of chunks CHECK(tile.size() == 0); CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile.filtered_buffer().value_at_as(0) == 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); - offset += sizeof(uint64_t); - } - } auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); run_reverse(config, tp, unfiltered_tile, pipeline); + + // Check all elements original values. for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; CHECK_NOTHROW( @@ -997,1267 +619,6 @@ TEST_CASE( WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } -TEST_CASE( - "Filter: Test simple out-of-place pipeline", - "[filter][simple-out-of-place]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - - SECTION("- Single stage") { - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Multi-stage") { - // Add a few more +1 filters and re-run. - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE( - "Filter: Test simple out-of-place pipeline var", - "[filter][simple-out-of-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - - SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Multi-stage") { - // Add a few more +1 filters and re-run. - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE( - "Filter: Test mixed in- and out-of-place pipeline", - "[filter][in-out-place]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } -} - -TEST_CASE( - "Filter: Test mixed in- and out-of-place pipeline var", - "[filter][in-out-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - FilterPipeline pipeline; - ThreadPool tp(4); - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test compression", "[filter][compression]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up dummy array schema (needed by compressor filter for cell size, etc). - uint32_t dim_dom[] = {1, 10}; - auto dim{make_shared(HERE(), "", Datatype::INT32)}; - CHECK(dim->set_domain(dim_dom).ok()); - auto domain{make_shared(HERE())}; - CHECK(domain->add_dimension(dim).ok()); - tiledb::sm::ArraySchema schema; - tiledb::sm::Attribute attr("attr", Datatype::UINT64); - CHECK(schema.add_attribute(make_shared(HERE(), attr)) - .ok()); - CHECK(schema.set_domain(domain).ok()); - - FilterPipeline pipeline; - ThreadPool tp(4); - - SECTION("- Simple") { - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With checksum stage") { - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With multiple stages") { - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - // Set up dummy array schema (needed by compressor filter for cell size, etc). - uint32_t dim_dom[] = {1, 10}; - auto dim{make_shared(HERE(), "", Datatype::INT32)}; - CHECK(dim->set_domain(dim_dom).ok()); - auto domain{make_shared(HERE())}; - CHECK(domain->add_dimension(dim).ok()); - tiledb::sm::ArraySchema schema; - tiledb::sm::Attribute attr("attr", Datatype::UINT64); - CHECK(schema.add_attribute(make_shared(HERE(), attr)) - .ok()); - CHECK(schema.set_domain(domain).ok()); - - FilterPipeline pipeline; - ThreadPool tp(4); - - SECTION("- Simple") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With checksum stage") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With multiple stages") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - const uint64_t expected_checksum = 4950; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + - 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements are the same. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Multi-stage") { - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - // Compute the second (final) checksum value. - uint64_t expected_checksum_2 = 0; - for (uint64_t i = 0; i < nelts; i++) - expected_checksum_2 += i + 2; - - // Check new size and number of chunks. - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + - sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 2 * sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Outer checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum_2); - offset += sizeof(uint64_t); - - // Inner checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements are correct. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i + 2); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE( - "Filter: Test pseudo-checksum var", "[filter][pseudo-checksum][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - std::vector expected_checksums{ - 91, 99, 275, 238, 425, 525, 1350, 825, 1122}; - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + - 9 * sizeof(uint64_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // Chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums[i]); - offset += sizeof(uint64_t); - - // Check all elements are the same. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == el++); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Multi-stage") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - std::vector expected_checksums2{ - 119, 111, 297, 252, 445, 545, 1390, 845, 1146}; - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + - 2 * 9 * sizeof(uint64_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 2 * sizeof(uint64_t)); // Chunk metadata size - offset += sizeof(uint32_t); - - // Checksums - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums2[i]); - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums[i]); - offset += sizeof(uint64_t); - - // Check all elements are incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 1); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(AddNInPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - - // Get non-existent filter instance - auto* cksum = pipeline.get_filter(); - CHECK(cksum == nullptr); - - // Modify +N filter - auto* add_n = pipeline.get_filter(); - CHECK(add_n != nullptr); - add_n->set_increment(2); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } -} - -TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(AddNInPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - - // Get non-existent filter instance - auto* cksum = pipeline.get_filter(); - CHECK(cksum == nullptr); - - // Modify +N filter - auto* add_n = pipeline.get_filter(); - CHECK(add_n != nullptr); - add_n->set_increment(2); - - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements are incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { - tiledb::sm::Config config; - - const uint64_t expected_checksum = 5350; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(AddNInPlace(Datatype::UINT64)); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - - // Modify +N filter - auto* add_n = pipeline.get_filter(); - CHECK(add_n != nullptr); - add_n->set_increment(2); - - // Copy pipeline - FilterPipeline pipeline_copy(pipeline); - - // Check +N filter cloned correctly - auto* add_n_2 = pipeline_copy.get_filter(); - CHECK(add_n_2 != add_n); - CHECK(add_n_2 != nullptr); - CHECK(add_n_2->increment() == 2); - - CHECK(pipeline_copy.run_forward(&test::g_helper_stats, &tile, nullptr, &tp) - .ok()); - - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } -} - -TEST_CASE("Filter: Test random pipeline", "[filter][random]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - - EncryptionKey encryption_key; - REQUIRE(encryption_key - .set_key( - EncryptionType::AES_256_GCM, - "abcdefghijklmnopqrstuvwxyz012345", - 32) - .ok()); - - // List of potential filters to use. All of these filters can occur anywhere - // in the pipeline. - std::vector> constructors = { - []() { return new Add1InPlace(Datatype::UINT64); }, - []() { return new Add1OutOfPlace(Datatype::UINT64); }, - []() { return new Add1IncludingMetadataFilter(Datatype::UINT64); }, - []() { return new BitWidthReductionFilter(Datatype::UINT64); }, - []() { return new BitshuffleFilter(Datatype::UINT64); }, - []() { return new ByteshuffleFilter(Datatype::UINT64); }, - []() { - return new CompressionFilter( - tiledb::sm::Compressor::BZIP2, -1, Datatype::UINT64); - }, - []() { return new PseudoChecksumFilter(Datatype::UINT64); }, - []() { return new ChecksumMD5Filter(Datatype::UINT64); }, - []() { return new ChecksumSHA256Filter(Datatype::UINT64); }, - [&encryption_key]() { - return new EncryptionAES256GCMFilter(encryption_key, Datatype::UINT64); - }, - }; - - // List of potential filters that must occur at the beginning of the pipeline. - std::vector> constructors_first = { - // Pos-delta would (correctly) return error after e.g. compression. - []() { return new PositiveDeltaFilter(Datatype::UINT64); }}; - - ThreadPool tp(4); - for (int i = 0; i < 100; i++) { - auto tile = make_increasing_tile(nelts); - - // Construct a random pipeline - FilterPipeline pipeline; - const unsigned max_num_filters = 6; - std::random_device rd; - auto pipeline_seed = rd(); - std::mt19937 gen(pipeline_seed); - std::uniform_int_distribution<> rng_num_filters(0, max_num_filters), - rng_bool(0, 1), rng_constructors(0, (int)(constructors.size() - 1)), - rng_constructors_first(0, (int)(constructors_first.size() - 1)); - - INFO("Random pipeline seed: " << pipeline_seed); - - auto num_filters = (unsigned)rng_num_filters(gen); - for (unsigned j = 0; j < num_filters; j++) { - if (j == 0 && rng_bool(gen) == 1) { - auto idx = (unsigned)rng_constructors_first(gen); - tiledb::sm::Filter* filter = constructors_first[idx](); - pipeline.add_filter(*filter); - delete filter; - } else { - auto idx = (unsigned)rng_constructors(gen); - tiledb::sm::Filter* filter = constructors[idx](); - pipeline.add_filter(*filter); - delete filter; - } - } - - // End result should always be the same as the input. - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t n = 0; n < nelts; n++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == n); - } - } -} - TEST_CASE( "Filter: Test skip checksum validation", "[filter][skip-checksum-validation]") { diff --git a/tiledb/sm/filter/test/CMakeLists.txt b/tiledb/sm/filter/test/CMakeLists.txt index ded1b9f8dc86..879486245d13 100644 --- a/tiledb/sm/filter/test/CMakeLists.txt +++ b/tiledb/sm/filter/test/CMakeLists.txt @@ -36,6 +36,19 @@ commence(unit_test filter_pipeline) this_target_sources(main.cc unit_filter_pipeline.cc) conclude(unit_test) +commence(unit_test run_filter_pipeline) + this_target_object_libraries(filter_pipeline) + this_target_sources( + main.cc + add_1_in_place_filter.cc + add_1_including_metadata_filter.cc + add_1_out_of_place_filter.cc + add_n_in_place_filter.cc + pseudo_checksum_filter.cc + unit_run_filter_pipeline.cc + ) +conclude(unit_test) + commence(unit_test float_scale_input_val) this_target_object_libraries(float_scaling_filter) this_target_sources(main.cc unit_float_scale_input_validation.cc) diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.cc b/tiledb/sm/filter/test/add_1_in_place_filter.cc new file mode 100644 index 000000000000..f4c9425fade2 --- /dev/null +++ b/tiledb/sm/filter/test/add_1_in_place_filter.cc @@ -0,0 +1,102 @@ +/** + * @file add_1_in_place_filter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "add_1_in_place_filter.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +// Just use a dummy filter type +Add1InPlace::Add1InPlace(Datatype filter_data_type) + : Filter(FilterType::FILTER_NONE, filter_data_type) { +} + +void Add1InPlace::dump(FILE* out) const { + (void)out; +} + +Status Add1InPlace::run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const { + auto input_size = input->size(); + RETURN_NOT_OK(output->append_view(input)); + output->reset_offset(); + + uint64_t nelts = input_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t* val = output->value_ptr(); + *val += 1; + output->advance_offset(sizeof(uint64_t)); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + + return Status::Ok(); +} + +Status Add1InPlace::run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const { + (void)config; + + auto input_size = input->size(); + RETURN_NOT_OK(output->append_view(input)); + output->reset_offset(); + + uint64_t nelts = input_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t* val = output->value_ptr(); + *val -= 1; + output->advance_offset(sizeof(uint64_t)); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + + return Status::Ok(); +} + +Add1InPlace* Add1InPlace::clone_impl() const { + return tdb_new(Add1InPlace, filter_data_type_); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.h b/tiledb/sm/filter/test/add_1_in_place_filter.h new file mode 100644 index 000000000000..67b33f33de67 --- /dev/null +++ b/tiledb/sm/filter/test/add_1_in_place_filter.h @@ -0,0 +1,77 @@ +/** + * @file add_1_in_place_filter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simple filter that modifies the input stream by adding 1 to every input + * element. + * + * This filter is for use in running filter pipeline tests. + */ + +#ifndef TILEDB_ADD_1_IN_PLACE_FILTER_H +#define TILEDB_ADD_1_IN_PLACE_FILTER_H + +#include "tiledb/sm/filter/filter.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** + * Simple filter that modifies the input stream by adding 1 to every input + * element. + */ +class Add1InPlace : public tiledb::sm::Filter { + public: + Add1InPlace(Datatype filter_data_type); + + void dump(FILE* out) const override; + + Status run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const override; + + Status run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const override; + + Add1InPlace* clone_impl() const override; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.cc b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc new file mode 100644 index 000000000000..dfc344933076 --- /dev/null +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc @@ -0,0 +1,161 @@ +/** + * @file add_1_including_metadata_filter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "add_1_including_metadata_filter.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { +// Just use a dummy filter type +Add1IncludingMetadataFilter::Add1IncludingMetadataFilter( + Datatype filter_data_type) + : Filter(FilterType::FILTER_NONE, filter_data_type) { +} + +void Add1IncludingMetadataFilter::dump(FILE* out) const { + (void)out; +} + +Status Add1IncludingMetadataFilter::run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const { + auto input_size = static_cast(input->size()), + input_md_size = static_cast(input_metadata->size()); + auto nelts = input_size / sizeof(uint64_t), + md_nelts = input_md_size / sizeof(uint64_t); + + // Add another output buffer. + RETURN_NOT_OK(output->prepend_buffer(input_size + input_md_size)); + output->reset_offset(); + + // Filter input data + for (uint64_t i = 0; i < nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + inc++; + RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + } + // Finish any remaining bytes to ensure no data loss. + auto rem = input_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input->read(&byte, sizeof(char))); + RETURN_NOT_OK(output->write(&byte, sizeof(char))); + } + + // Now filter input metadata. + for (uint64_t i = 0; i < md_nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input_metadata->read(&inc, sizeof(uint64_t))); + inc++; + RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + } + rem = input_md_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input_metadata->read(&byte, sizeof(char))); + RETURN_NOT_OK(output->write(&byte, sizeof(char))); + } + + // Because this filter modifies the input metadata, we need output metadata + // that allows the original metadata to be reconstructed on reverse. Also + // note that contrary to most filters, we don't forward the input metadata. + RETURN_NOT_OK(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); + RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); + RETURN_NOT_OK(output_metadata->write(&input_md_size, sizeof(uint32_t))); + + return Status::Ok(); +} + +Status Add1IncludingMetadataFilter::run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const { + (void)config; + + if (input_metadata->size() != 2 * sizeof(uint32_t)) + return Status_FilterError("Unexpected input metadata length"); + + uint32_t orig_input_size, orig_md_size; + RETURN_NOT_OK(input_metadata->read(&orig_input_size, sizeof(uint32_t))); + RETURN_NOT_OK(input_metadata->read(&orig_md_size, sizeof(uint32_t))); + + // Add another output buffer. + RETURN_NOT_OK(output->prepend_buffer(orig_input_size)); + // Add another output metadata buffer. + RETURN_NOT_OK(output_metadata->prepend_buffer(orig_md_size)); + + // Restore original data + auto nelts = orig_input_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + inc--; + RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + } + auto rem = orig_input_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input->read(&byte, sizeof(char))); + RETURN_NOT_OK(output->write(&byte, sizeof(char))); + } + + // Restore original metadata + nelts = orig_md_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + inc--; + RETURN_NOT_OK(output_metadata->write(&inc, sizeof(uint64_t))); + } + rem = orig_md_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input->read(&byte, sizeof(char))); + RETURN_NOT_OK(output_metadata->write(&byte, sizeof(char))); + } + + return Status::Ok(); +} + +Add1IncludingMetadataFilter* Add1IncludingMetadataFilter::clone_impl() const { + return tdb_new(Add1IncludingMetadataFilter, filter_data_type_); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.h b/tiledb/sm/filter/test/add_1_including_metadata_filter.h new file mode 100644 index 000000000000..c508caff401d --- /dev/null +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.h @@ -0,0 +1,79 @@ +/** + * @file add_1_including_metadata_filter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simple filter that increments every element of the input stream, writing the + * output to another buffer. The input metadata is treated as a part of the + * input data. + * + * This filter is for use in running filter pipeline tests. + */ + +#ifndef TILEDB_ADD_1_INCLUDING_METADATA_FILTER_H +#define TILEDB_ADD_1_INCLUDING_METADATA_FILTER_H + +#include "tiledb/sm/filter/filter.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** + * Simple filter that increments every element of the input stream, writing the + * output to another buffer. The input metadata is treated as a part of the + * input data. + */ +class Add1IncludingMetadataFilter : public tiledb::sm::Filter { + public: + Add1IncludingMetadataFilter(Datatype filter_data_type); + + void dump(FILE* out) const override; + + Status run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const override; + + Status run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const override; + + Add1IncludingMetadataFilter* clone_impl() const override; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.cc b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc new file mode 100644 index 000000000000..aa7c7f475394 --- /dev/null +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc @@ -0,0 +1,122 @@ +/** + * @file add_1_out_of_place_filter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "add_1_out_of_place_filter.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { +// Just use a dummy filter type +Add1OutOfPlace::Add1OutOfPlace(Datatype filter_data_type) + : Filter(FilterType::FILTER_NONE, filter_data_type) { +} + +void Add1OutOfPlace::dump(FILE* out) const { + (void)out; +} + +Status Add1OutOfPlace::run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const { + auto input_size = input->size(); + auto nelts = input_size / sizeof(uint64_t); + + // Add another output buffer. + RETURN_NOT_OK(output->prepend_buffer(input_size)); + output->reset_offset(); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + inc++; + RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + } + + // Finish any remaining bytes to ensure no data loss. + auto rem = input_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input->read(&byte, sizeof(char))); + RETURN_NOT_OK(output->write(&byte, sizeof(char))); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + + return Status::Ok(); +} + +Status Add1OutOfPlace::run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const { + (void)config; + + auto input_size = input->size(); + auto nelts = input->size() / sizeof(uint64_t); + + // Add another output buffer. + RETURN_NOT_OK(output->prepend_buffer(input->size())); + output->reset_offset(); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t inc; + RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + inc--; + RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + } + + auto rem = input_size % sizeof(uint64_t); + for (unsigned i = 0; i < rem; i++) { + char byte; + RETURN_NOT_OK(input->read(&byte, sizeof(char))); + RETURN_NOT_OK(output->write(&byte, sizeof(char))); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + + return Status::Ok(); +} + +Add1OutOfPlace* Add1OutOfPlace::clone_impl() const { + return tdb_new(Add1OutOfPlace, filter_data_type_); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.h b/tiledb/sm/filter/test/add_1_out_of_place_filter.h new file mode 100644 index 000000000000..21065a002f65 --- /dev/null +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.h @@ -0,0 +1,77 @@ +/** + * @file add_1_out_of_place_filter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simple filter that increments every element of the input stream, writing the + * output to another buffer. Does not modify the input stream. + * + * This filter is for use in running filter pipeline tests. + */ + +#ifndef TILEDB_ADD_1_OUT_OF_PLACE_FILTER_H +#define TILEDB_ADD_1_OUT_OF_PLACE_FILTER_H + +#include "tiledb/sm/filter/filter.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** + * Simple filter that increments every element of the input stream, writing the + * output to another buffer. Does not modify the input stream. + */ +class Add1OutOfPlace : public tiledb::sm::Filter { + public: + Add1OutOfPlace(Datatype filter_data_type); + + void dump(FILE* out) const override; + + Status run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const override; + + Status run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const override; + + Add1OutOfPlace* clone_impl() const override; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.cc b/tiledb/sm/filter/test/add_n_in_place_filter.cc new file mode 100644 index 000000000000..15fa7df2f856 --- /dev/null +++ b/tiledb/sm/filter/test/add_n_in_place_filter.cc @@ -0,0 +1,112 @@ +/** + * @file add_n_in_place_filter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "add_n_in_place_filter.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +// Just use a dummy filter type +AddNInPlace::AddNInPlace(Datatype filter_data_type) + : Filter(FilterType::FILTER_NONE, filter_data_type) { + increment_ = 1; +} + +void AddNInPlace::dump(FILE* out) const { + (void)out; +} + +Status AddNInPlace::run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const { + auto input_size = input->size(); + RETURN_NOT_OK(output->append_view(input)); + output->reset_offset(); + + uint64_t nelts = input_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t* val = output->value_ptr(); + *val += increment_; + output->advance_offset(sizeof(uint64_t)); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + return Status::Ok(); +} + +Status AddNInPlace::run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const { + (void)config; + + auto input_size = input->size(); + RETURN_NOT_OK(output->append_view(input)); + output->reset_offset(); + + uint64_t nelts = input_size / sizeof(uint64_t); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t* val = output->value_ptr(); + *val -= increment_; + output->advance_offset(sizeof(uint64_t)); + } + + // Metadata not modified by this filter. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + + return Status::Ok(); +} + +uint64_t AddNInPlace::increment() const { + return increment_; +} + +void AddNInPlace::set_increment(uint64_t increment) { + increment_ = increment; +} + +AddNInPlace* AddNInPlace::clone_impl() const { + auto clone = tdb_new(AddNInPlace, filter_data_type_); + clone->increment_ = increment_; + return clone; +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.h b/tiledb/sm/filter/test/add_n_in_place_filter.h new file mode 100644 index 000000000000..97cba2c3c18c --- /dev/null +++ b/tiledb/sm/filter/test/add_n_in_place_filter.h @@ -0,0 +1,84 @@ +/** + * @file add_n_in_place_filter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simple filter that modifies the input stream by adding a constant value to + * every input element. + * + * This filter is for use in running filter pipeline tests. + */ + +#ifndef TILEDB_TEST_FILTERS_H +#define TILEDB_TEST_FILTERS_H + +#include "tiledb/sm/filter/filter.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** + * Simple filter that modifies the input stream by adding a constant value to + * every input element. + */ +class AddNInPlace : public tiledb::sm::Filter { + public: + AddNInPlace(Datatype filter_data_type); + + void dump(FILE* out) const override; + + Status run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const override; + + Status run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const override; + + uint64_t increment() const; + + void set_increment(uint64_t increment); + + AddNInPlace* clone_impl() const override; + + private: + uint64_t increment_; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.cc b/tiledb/sm/filter/test/pseudo_checksum_filter.cc new file mode 100644 index 000000000000..c65a7c7308ef --- /dev/null +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.cc @@ -0,0 +1,120 @@ +/** + * @file pseudo_checksum_filter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "pseudo_checksum_filter.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { +// Just use a dummy filter type +PseudoChecksumFilter::PseudoChecksumFilter(Datatype filter_data_type) + : Filter(FilterType::FILTER_NONE, filter_data_type) { +} + +void PseudoChecksumFilter::dump(FILE* out) const { + (void)out; +} + +Status PseudoChecksumFilter::run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const { + auto input_size = input->size(); + auto nelts = input_size / sizeof(uint64_t); + + // The input is unmodified by this filter. + RETURN_NOT_OK(output->append_view(input)); + + // Forward the existing metadata and prepend a metadata buffer for the + // checksum. + RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + RETURN_NOT_OK(output_metadata->prepend_buffer(sizeof(uint64_t))); + output_metadata->reset_offset(); + + uint64_t sum = 0; + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val; + RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); + sum += val; + } + + RETURN_NOT_OK(output_metadata->write(&sum, sizeof(uint64_t))); + + return Status::Ok(); +} + +Status PseudoChecksumFilter::run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const { + (void)config; + + auto input_size = input->size(); + auto nelts = input_size / sizeof(uint64_t); + + uint64_t input_sum; + RETURN_NOT_OK(input_metadata->read(&input_sum, sizeof(uint64_t))); + + uint64_t sum = 0; + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val; + RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); + sum += val; + } + + if (sum != input_sum) + return Status_FilterError("Filter error; sum does not match."); + + // The output metadata is just a view on the input metadata, skipping the + // checksum bytes. + RETURN_NOT_OK(output_metadata->append_view( + input_metadata, + sizeof(uint64_t), + input_metadata->size() - sizeof(uint64_t))); + + // The output data is just a view on the unmodified input. + RETURN_NOT_OK(output->append_view(input)); + + return Status::Ok(); +} + +PseudoChecksumFilter* PseudoChecksumFilter::clone_impl() const { + return tdb_new(PseudoChecksumFilter, filter_data_type_); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.h b/tiledb/sm/filter/test/pseudo_checksum_filter.h new file mode 100644 index 000000000000..96197b7ad4c7 --- /dev/null +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.h @@ -0,0 +1,77 @@ +/** + * @file pseudo_checksum_filter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simple filter which computes the sum of its input and prepends the sum + * to the output. In reverse execute, checks that the sum is correct. + * + * This filter is for use in running filter pipeline tests. + */ + +#ifndef TILEDB_PSEUDO_CHECKSUM_FILTER_H +#define TILEDB_PSEUDO_CHECKSUM_FILTER_H + +#include "tiledb/sm/filter/filter.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** + * Simple filter which computes the sum of its input and prepends the sum + * to the output. In reverse execute, checks that the sum is correct. + */ +class PseudoChecksumFilter : public tiledb::sm::Filter { + public: + PseudoChecksumFilter(Datatype filter_data_type); + + void dump(FILE* out) const override; + + Status run_forward( + const WriterTile&, + WriterTile* const, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output) const override; + + Status run_reverse( + const Tile&, + Tile*, + FilterBuffer* input_metadata, + FilterBuffer* input, + FilterBuffer* output_metadata, + FilterBuffer* output, + const tiledb::sm::Config& config) const override; + + PseudoChecksumFilter* clone_impl() const override; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc new file mode 100644 index 000000000000..1c2e02ac3748 --- /dev/null +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -0,0 +1,1546 @@ +/** + * @file unit_run_filter_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline forward and + * backward. Most tests here use simplified filters that perform basic + * operations such as adding 1 to each value in the original data so that the + * filtered data itself can be checked after running the filter pipeline + * forward. + */ + +#include + +#include + +#include "../bit_width_reduction_filter.h" +#include "../bitshuffle_filter.h" +#include "../byteshuffle_filter.h" +#include "../checksum_md5_filter.h" +#include "../checksum_sha256_filter.h" +#include "../compression_filter.h" +#include "../encryption_aes256gcm_filter.h" +#include "../filter.h" +#include "../filter_pipeline.h" +#include "../positive_delta_filter.h" +#include "add_1_in_place_filter.h" +#include "add_1_including_metadata_filter.h" +#include "add_1_out_of_place_filter.h" +#include "add_n_in_place_filter.h" +#include "pseudo_checksum_filter.h" +#include "tiledb/sm/crypto/encryption_key.h" +#include "tiledb/sm/enums/compressor.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/filter_option.h" +#include "tiledb/sm/enums/filter_type.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::sm; + +// A dummy `Stats` instance. +static tiledb::sm::stats::Stats dummy_stats("test"); + +WriterTile make_increasing_tile(const uint64_t nelts) { + const uint64_t tile_size = nelts * sizeof(uint64_t); + const uint64_t cell_size = sizeof(uint64_t); + + WriterTile tile( + constants::format_version, Datatype::UINT64, cell_size, tile_size); + for (uint64_t i = 0; i < nelts; i++) { + CHECK_NOTHROW(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + return tile; +} + +WriterTile make_offsets_tile(std::vector& offsets) { + const uint64_t offsets_tile_size = + offsets.size() * constants::cell_var_offset_size; + + WriterTile offsets_tile( + constants::format_version, + Datatype::UINT64, + constants::cell_var_offset_size, + offsets_tile_size); + + // Set up test data + for (uint64_t i = 0; i < offsets.size(); i++) { + CHECK_NOTHROW(offsets_tile.write( + &offsets[i], + i * constants::cell_var_offset_size, + constants::cell_var_offset_size)); + } + + return offsets_tile; +} + +Tile create_tile_for_unfiltering(uint64_t nelts, WriterTile& tile) { + Tile ret( + tile.format_version(), + tile.type(), + tile.cell_size(), + 0, + tile.cell_size() * nelts, + tile.filtered_buffer().data(), + tile.filtered_buffer().size()); + return ret; +} + +void run_reverse( + const tiledb::sm::Config& config, + ThreadPool& tp, + Tile& unfiltered_tile, + FilterPipeline& pipeline, + bool success = true) { + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + CHECK( + success == pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); +} + +TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements unchanged. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == i); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } +} + +TEST_CASE( + "Filter: Test empty pipeline var sized", "[filter][empty-pipeline][var]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + FilterPipeline pipeline; + ThreadPool tp(4); + WriterTile::set_max_tile_chunk_size(80); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements unchanged. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == el++); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE( + "Filter: Test simple in-place pipeline", "[filter][simple-in-place]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + + SECTION("- Single stage") { + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + // Add a few more +1 filters and re-run. + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } +} + +TEST_CASE( + "Filter: Test simple in-place pipeline var", + "[filter][simple-in-place][var]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + + SECTION("- Single stage") { + WriterTile::set_max_tile_chunk_size(80); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + // Add a few more +1 filters and re-run. + WriterTile::set_max_tile_chunk_size(80); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE( + "Filter: Test simple out-of-place pipeline", + "[filter][simple-out-of-place]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + + SECTION("- Single stage") { + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + // Add a few more +1 filters and re-run. + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } +} + +TEST_CASE( + "Filter: Test simple out-of-place pipeline var", + "[filter][simple-out-of-place][var]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + + SECTION("- Single stage") { + WriterTile::set_max_tile_chunk_size(80); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + // Add a few more +1 filters and re-run. + WriterTile::set_max_tile_chunk_size(80); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE( + "Filter: Test mixed in- and out-of-place pipeline", + "[filter][in-out-place]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } +} + +TEST_CASE( + "Filter: Test mixed in- and out-of-place pipeline var", + "[filter][in-out-place][var]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + FilterPipeline pipeline; + ThreadPool tp(4); + WriterTile::set_max_tile_chunk_size(80); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + const uint64_t expected_checksum = 4950; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + sizeof(uint64_t)); // First chunk metadata size + offset += sizeof(uint32_t); + + // Checksum + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksum); + offset += sizeof(uint64_t); + + // Check all elements are the same. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == i); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + // Compute the second (final) checksum value. + uint64_t expected_checksum_2 = 0; + for (uint64_t i = 0; i < nelts; i++) + expected_checksum_2 += i + 2; + + // Check size and number of chunks. + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + + sizeof(uint64_t) + 3 * sizeof(uint32_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 2 * sizeof(uint64_t)); // First chunk metadata size + offset += sizeof(uint32_t); + + // Outer checksum + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksum_2); + offset += sizeof(uint64_t); + + // Inner checksum + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksum); + offset += sizeof(uint64_t); + + // Check all elements are correct. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == i + 2); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } +} + +TEST_CASE( + "Filter: Test pseudo-checksum var", "[filter][pseudo-checksum][var]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + std::vector expected_checksums{ + 91, 99, 275, 238, 425, 525, 1350, 825, 1122}; + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + WriterTile::set_max_tile_chunk_size(80); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + + 9 * sizeof(uint64_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + sizeof(uint64_t)); // Chunk metadata size + offset += sizeof(uint32_t); + + // Checksum + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksums[i]); + offset += sizeof(uint64_t); + + // Check all elements are the same. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == el++); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Multi-stage") { + WriterTile::set_max_tile_chunk_size(80); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + std::vector expected_checksums2{ + 119, 111, 297, 252, 445, 545, 1390, 845, 1146}; + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + + 2 * 9 * sizeof(uint64_t)); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 2 * sizeof(uint64_t)); // Chunk metadata size + offset += sizeof(uint32_t); + + // Checksums + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksums2[i]); + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksums[i]); + offset += sizeof(uint64_t); + + // Check all elements are incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 1); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { + tiledb::sm::Config config; + + // Set up test data + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(AddNInPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + + // Get non-existent filter instance + auto* cksum = pipeline.get_filter(); + CHECK(cksum == nullptr); + + // Modify +N filter + auto* add_n = pipeline.get_filter(); + CHECK(add_n != nullptr); + add_n->set_increment(2); + + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() != 0); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // First chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } +} + +TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(AddNInPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + + // Get non-existent filter instance + auto* cksum = pipeline.get_filter(); + CHECK(cksum == nullptr); + + // Modify +N filter + auto* add_n = pipeline.get_filter(); + CHECK(add_n != nullptr); + add_n->set_increment(2); + + WriterTile::set_max_tile_chunk_size(80); + CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); + + // Check size and number of chunks + CHECK(tile.size() == 0); + CHECK( + tile.filtered_buffer().size() == + nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); + + offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t el = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 0); // Chunk metadata size + offset += sizeof(uint32_t); + + // Check all elements are incremented. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); + offset += sizeof(uint64_t); + } + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { + tiledb::sm::Config config; + + const uint64_t expected_checksum = 5350; + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(AddNInPlace(Datatype::UINT64)); + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + + // Modify +N filter + auto* add_n = pipeline.get_filter(); + CHECK(add_n != nullptr); + add_n->set_increment(2); + + // Copy pipeline + FilterPipeline pipeline_copy(pipeline); + + // Check +N filter cloned correctly + auto* add_n_2 = pipeline_copy.get_filter(); + CHECK(add_n_2 != add_n); + CHECK(add_n_2 != nullptr); + CHECK(add_n_2->increment() == 2); + + CHECK(pipeline_copy.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() != 0); + + uint64_t offset = 0; + CHECK( + tile.filtered_buffer().value_at_as(offset) == + 1); // Number of chunks + offset += sizeof(uint64_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // First chunk filtered size + offset += sizeof(uint32_t); + CHECK( + tile.filtered_buffer().value_at_as(offset) == + sizeof(uint64_t)); // First chunk metadata size + offset += sizeof(uint32_t); + + // Checksum + CHECK( + tile.filtered_buffer().value_at_as(offset) == + expected_checksum); + offset += sizeof(uint64_t); + + // Check all elements incremented. + for (uint64_t i = 0; i < nelts; i++) { + CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); + offset += sizeof(uint64_t); + } + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } +} + +TEST_CASE("Filter: Test random pipeline", "[filter][random]") { + tiledb::sm::Config config; + + const uint64_t nelts = 100; + + EncryptionKey encryption_key; + REQUIRE(encryption_key + .set_key( + EncryptionType::AES_256_GCM, + "abcdefghijklmnopqrstuvwxyz012345", + 32) + .ok()); + + // List of potential filters to use. All of these filters can occur anywhere + // in the pipeline. + std::vector> constructors = { + []() { return tdb_new(Add1InPlace, Datatype::UINT64); }, + []() { return tdb_new(Add1OutOfPlace, Datatype::UINT64); }, + []() { return tdb_new(Add1IncludingMetadataFilter, Datatype::UINT64); }, + []() { return tdb_new(BitWidthReductionFilter, Datatype::UINT64); }, + []() { return tdb_new(BitshuffleFilter, Datatype::UINT64); }, + []() { return tdb_new(ByteshuffleFilter, Datatype::UINT64); }, + []() { + return tdb_new( + CompressionFilter, + tiledb::sm::Compressor::BZIP2, + -1, + Datatype::UINT64); + }, + []() { return tdb_new(PseudoChecksumFilter, Datatype::UINT64); }, + []() { return tdb_new(ChecksumMD5Filter, Datatype::UINT64); }, + []() { return tdb_new(ChecksumSHA256Filter, Datatype::UINT64); }, + [&encryption_key]() { + return tdb_new( + EncryptionAES256GCMFilter, encryption_key, Datatype::UINT64); + }, + }; + + // List of potential filters that must occur at the beginning of the pipeline. + std::vector> constructors_first = { + // Pos-delta would (correctly) return error after e.g. compression. + []() { return tdb_new(PositiveDeltaFilter, Datatype::UINT64); }}; + + ThreadPool tp(4); + for (int i = 0; i < 100; i++) { + auto tile = make_increasing_tile(nelts); + + // Construct a random pipeline + FilterPipeline pipeline; + const unsigned max_num_filters = 6; + std::random_device rd; + auto pipeline_seed = rd(); + std::mt19937 gen(pipeline_seed); + std::uniform_int_distribution<> rng_num_filters(0, max_num_filters), + rng_bool(0, 1), rng_constructors(0, (int)(constructors.size() - 1)), + rng_constructors_first(0, (int)(constructors_first.size() - 1)); + + INFO("Random pipeline seed: " << pipeline_seed); + + auto num_filters = (unsigned)rng_num_filters(gen); + for (unsigned j = 0; j < num_filters; j++) { + if (j == 0 && rng_bool(gen) == 1) { + auto idx = (unsigned)rng_constructors_first(gen); + tiledb::sm::Filter* filter = constructors_first[idx](); + pipeline.add_filter(*filter); + tdb_delete(filter); + } else { + auto idx = (unsigned)rng_constructors(gen); + tiledb::sm::Filter* filter = constructors[idx](); + pipeline.add_filter(*filter); + tdb_delete(filter); + } + } + + // End result should always be the same as the input. + CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + CHECK(tile.size() == 0); + CHECK(tile.filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile, pipeline); + + for (uint64_t n = 0; n < nelts; n++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == n); + } + } +} From b8e2f0cabd74f3dd04a133c68f21bba15870afa0 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:41:11 -0500 Subject: [PATCH 106/456] Maintain support for --vfs command line arg (#4563) Maintain support for --vfs command line arg --- TYPE: NO_HISTORY --- test/support/src/vfs_helpers.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index fc16baa07892..285981a81482 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -50,25 +50,32 @@ namespace tiledb::test { std::vector> vfs_test_get_fs_vec() { std::vector> fs_vec; - if constexpr (tiledb::sm::filesystem::s3_enabled) { + + bool supports_s3 = false; + bool supports_hdfs = false; + bool supports_azure = false; + bool supports_gcs = false; + get_supported_fs( + &supports_s3, &supports_hdfs, &supports_azure, &supports_gcs); + + if (supports_s3) { fs_vec.emplace_back(std::make_unique()); } - if constexpr (tiledb::sm::filesystem::hdfs_enabled) { + if (supports_hdfs) { fs_vec.emplace_back(std::make_unique()); } - if constexpr (tiledb::sm::filesystem::azure_enabled) { + if (supports_azure) { fs_vec.emplace_back(std::make_unique()); } - if constexpr (tiledb::sm::filesystem::gcs_enabled) { + if (supports_gcs) { fs_vec.emplace_back(std::make_unique()); fs_vec.emplace_back(std::make_unique("gs://")); } fs_vec.emplace_back(std::make_unique()); - fs_vec.emplace_back(std::make_unique()); return fs_vec; From 70a5084c4c183b0bf0a19814930362845bdd3bcc Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 14 Dec 2023 10:39:52 +0200 Subject: [PATCH 107/456] Update the minimum macOS version to 11 on x64. (#4561) [SC-38139](https://app.shortcut.com/tiledb-inc/story/38139/switch-macos-builds-for-r-use-to-mmacosx-version-min-11-0) After further discussion we are allowed by CRAN to target macOS 11. This will allow us to use newer C++ APIs like `std::filesystem`. --- TYPE: BREAKING_BEHAVIOR DESC: The minimum supported macOS version on x64 is 11. --- .github/workflows/release.yml | 4 +--- ports/triplets/x64-osx-release.cmake | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6718e9dd43a5..c4eb697682a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,9 +39,7 @@ jobs: manylinux: true - platform: macos-x86_64 os: macos-12 - # By default we have to target macOS 10.14 at minimum according to CRAN policy. - # See https://stat.ethz.ch/pipermail/r-package-devel/2023q4/010078.html - MACOSX_DEPLOYMENT_TARGET: 10.14 + MACOSX_DEPLOYMENT_TARGET: 11 triplet: x64-osx-release - platform: macos-arm64 os: macos-12 diff --git a/ports/triplets/x64-osx-release.cmake b/ports/triplets/x64-osx-release.cmake index f4fa41a3975f..461a5a727ec9 100644 --- a/ports/triplets/x64-osx-release.cmake +++ b/ports/triplets/x64-osx-release.cmake @@ -4,6 +4,6 @@ set(VCPKG_LIBRARY_LINKAGE static) set(VCPKG_CMAKE_SYSTEM_NAME Darwin) set(VCPKG_OSX_ARCHITECTURES x86_64) -set(VCPKG_OSX_DEPLOYMENT_TARGET 10.14) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) set(VCPKG_BUILD_TYPE release) From 8c7b00e0486325cc30cd401187806ff02fa209e2 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 14 Dec 2023 03:40:24 -0500 Subject: [PATCH 108/456] Rewrite `unit-capi-vfs.cc` to remove fixture functions (#4556) Rewrite `unit-capi-vfs.cc` to remove fixture functions --- TYPE: IMPROVEMENT DESC: Rewrite unit-capi-vfs.cc to remove fixture functions --- test/src/unit-capi-vfs.cc | 1226 ++++++----------- test/support/src/helpers.cc | 44 - test/support/src/helpers.h | 8 - test/support/src/vfs_helpers.h | 63 +- .../c_api_test_support/testsupport_capi_vfs.h | 4 +- 5 files changed, 498 insertions(+), 847 deletions(-) diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index e71183e44f82..da24ded03726 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -33,8 +33,12 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/temporary_local_directory.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/vfs/vfs_api_internal.h" +#include "tiledb/api/c_api_test_support/testsupport_capi_vfs.h" +#include "tiledb/platform/platform.h" #include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/utils.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" @@ -49,850 +53,496 @@ #include using namespace tiledb::test; +using tiledb::api::test_support::ordinary_vfs; -struct VFSFx { - // The unique directory object - tiledb::sm::TemporaryLocalDirectory temp_dir_{"tiledb_test_"}; - - const std::string HDFS_TEMP_DIR = "hdfs://localhost:9000/tiledb_test/"; - const std::string S3_PREFIX = "s3://"; - const std::string S3_BUCKET = S3_PREFIX + random_name("tiledb") + "/"; - const std::string S3_TEMP_DIR = S3_BUCKET + "tiledb_test/"; - const std::string AZURE_PREFIX = "azure://"; - const std::string AZURE_CONTAINER = - AZURE_PREFIX + random_name("tiledb") + "/"; - const std::string AZURE_TEMP_DIR = AZURE_CONTAINER + "tiledb_test/"; - const std::string FILE_TEMP_DIR = -#ifndef _WIN32 - "file://" + -#endif - temp_dir_.path(); - const std::string MEMFS_TEMP_DIR = std::string("mem://tiledb_test/"); - - // TileDB context and vfs - tiledb_ctx_t* ctx_; - tiledb_vfs_t* vfs_; - - // Functions - VFSFx(); - ~VFSFx(); - void check_vfs(const std::string& path); - void check_write(const std::string& path); - void check_move(const std::string& path); - void check_copy(const std::string& path); - void check_read(const std::string& path); - void check_append(const std::string& path); - void check_ls(const std::string& path); - static std::string random_name(const std::string& prefix); -}; - -VFSFx::VFSFx() { - create_ctx_and_vfs(&ctx_, &vfs_); -} +// The unique directory object +tiledb::sm::TemporaryLocalDirectory temp_dir_{"tiledb_test_"}; -VFSFx::~VFSFx() { - tiledb_vfs_free(&vfs_); - tiledb_ctx_free(&ctx_); +void require_tiledb_ok(capi_return_t rc) { + REQUIRE(rc == TILEDB_OK); } -void VFSFx::check_vfs(const std::string& path) { - if constexpr (tiledb::sm::filesystem::s3_enabled) { - // Check S3 bucket functionality - if (path == S3_TEMP_DIR) { - int is_bucket = 0; - int rc = tiledb_vfs_is_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_bucket); - REQUIRE(rc == TILEDB_OK); - if (is_bucket) { - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); - } - rc = tiledb_vfs_is_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_bucket); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_bucket); +void require_tiledb_err(capi_return_t rc) { + REQUIRE(rc == TILEDB_ERR); +} - rc = tiledb_vfs_create_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_bucket); - REQUIRE(is_bucket); - } - } +int ls_getter(const char* path, void* data) { + auto vec = static_cast*>(data); + vec->emplace_back(path); + return 1; +} - // Create directory, is directory, remove directory - int is_dir = 0; - int rc = tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (is_dir) { - rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); +TEST_CASE( + "C API: Test virtual filesystem when S3 is not supported", "[capi][vfs]") { + if constexpr (!tiledb::sm::filesystem::s3_enabled) { + ordinary_vfs x; + require_tiledb_err(tiledb_vfs_create_bucket(x.ctx, x.vfs, "s3://foo")); } - rc = tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - rc = tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - // Second time succeeds as well - rc = tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); - - // Remove directory recursively - auto subdir = path + "subdir/"; - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - - // Move - rc = tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - int is_file = 0; - std::string some_file = subdir + "some_file"; - rc = tiledb_vfs_touch(ctx_, vfs_, some_file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, some_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - auto subdir2 = path + "subdir2/"; - rc = tiledb_vfs_move_dir(ctx_, vfs_, subdir.c_str(), subdir2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_dir); - - // Invalid file - std::string foo_file = path + "foo"; - rc = tiledb_vfs_is_file(ctx_, vfs_, foo_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - tiledb_vfs_fh_t* fh = nullptr; - rc = tiledb_vfs_open(ctx_, vfs_, foo_file.c_str(), TILEDB_VFS_READ, &fh); - REQUIRE(rc == TILEDB_ERR); - REQUIRE(fh == nullptr); +} - // Touch file - rc = tiledb_vfs_touch(ctx_, vfs_, foo_file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, foo_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_remove_file(ctx_, vfs_, foo_file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, foo_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); +TEST_CASE("C API: Test virtual filesystem config", "[capi][vfs]") { + // Prepare a config + tiledb_error_t* error = nullptr; + tiledb_config_t* config; + require_tiledb_ok(tiledb_config_alloc(&config, &error)); + REQUIRE(error == nullptr); + require_tiledb_ok( + tiledb_config_set(config, "vfs.s3.scheme", "https", &error)); + REQUIRE(error == nullptr); - // Check write and append - check_write(path); - check_append(path); + // Create VFS + ordinary_vfs x(config); - // Read file - check_read(path); + // Get VFS config and check + tiledb_config_t* config2; + require_tiledb_ok(tiledb_vfs_get_config(x.ctx, x.vfs, &config2)); + const char* value; + require_tiledb_ok( + tiledb_config_get(config2, "vfs.s3.scheme", &value, &error)); + REQUIRE(error == nullptr); + CHECK(!strncmp(value, "https", strlen("https"))); - // Move - check_move(path); + // Clean up + tiledb_config_free(&config); + tiledb_config_free(&config2); +} -#ifndef _WIN32 - // Copy - if (path != MEMFS_TEMP_DIR) { - // copy not yet supported for memfs - check_copy(path); - } -#endif +TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { + tiledb_stats_enable(); + tiledb_stats_reset(); - // Ls - check_ls(path); + vfs_config v; + auto config = v.config; + tiledb_error_t* error = nullptr; - if constexpr (tiledb::sm::filesystem::s3_enabled) { - if (path == S3_TEMP_DIR) { - int is_empty; - rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!(bool)is_empty); - } + SECTION("Parallel I/O with 4 threads") { + // Set number of threads to 4. + require_tiledb_ok(tiledb_config_set( + config, "vfs.s3.max_parallel_ops", std::to_string(4).c_str(), &error)); + // Set small parallelization threshold (ignored when there is only 1 thread) + require_tiledb_ok(tiledb_config_set( + config, "vfs.min_parallel_size", std::to_string(1).c_str(), &error)); + REQUIRE(error == nullptr); } - if constexpr (!tiledb::sm::filesystem::s3_enabled) { - if (path != FILE_TEMP_DIR) { - rc = tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()); - REQUIRE(rc == TILEDB_OK); + // Get enabled filesystems + // Note: get_supported_fs handles VFS support override from `--vfs`. + bool s3_enabled; + bool hdfs_enabled; + bool azure_enabled; + bool gcs_enabled; + get_supported_fs(&s3_enabled, &hdfs_enabled, &azure_enabled, &gcs_enabled); + + // Sections to test each filesystem, if enabled + ordinary_vfs x(config); + std::string path = ""; + if (hdfs_enabled) { + SECTION("Filesystem: HDFS") { + path = "hdfs://localhost:9000/tiledb_test/"; } } - if constexpr (tiledb::sm::filesystem::s3_enabled) { - if (path == S3_TEMP_DIR) { - rc = tiledb_vfs_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); - - int is_empty; - rc = tiledb_vfs_is_empty_bucket(ctx_, vfs_, S3_BUCKET.c_str(), &is_empty); - REQUIRE(rc == TILEDB_OK); - REQUIRE((bool)is_empty); + std::string s3_bucket; + if (s3_enabled) { + SECTION("Filesystem: S3") { + path = "s3://" + random_name("tiledb") + "/tiledb_test/"; + s3_bucket = path.substr(0, path.find("tiledb_test/")); - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, S3_BUCKET.c_str()); - REQUIRE(rc == TILEDB_OK); - } - } -} - -void VFSFx::check_move(const std::string& path) { - // Move and remove file - auto file = path + "file"; - auto file2 = path + "file2"; - int is_file = 0; - int rc = tiledb_vfs_touch(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_move_file(ctx_, vfs_, file.c_str(), file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_remove_file(ctx_, vfs_, file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - - // Move directory with subdirectories and files - auto dir = path + "dir/"; - auto dir2 = path + "dir2/"; - auto subdir = path + "dir/subdir/"; - auto subdir2 = path + "dir2/subdir/"; - file = dir + "file"; - file2 = subdir + "file2"; - auto new_file = dir2 + "file"; - auto new_file2 = subdir2 + "file2"; - int is_dir = 0; - rc = tiledb_vfs_create_dir(ctx_, vfs_, dir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - rc = tiledb_vfs_touch(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_touch(ctx_, vfs_, file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_move_dir(ctx_, vfs_, dir.c_str(), dir2.c_str()); - REQUIRE(rc == TILEDB_OK); - - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_dir); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_dir); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_dir); - rc = tiledb_vfs_is_file(ctx_, vfs_, new_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_is_file(ctx_, vfs_, new_file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - - // Move from one bucket to another (only for S3) - if constexpr (tiledb::sm::filesystem::s3_enabled) { - if (path == S3_TEMP_DIR) { - std::string bucket2 = S3_PREFIX + random_name("tiledb") + "/"; - std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; - std::string file3 = subdir3 + "file2"; + // Check S3 bucket functionality int is_bucket = 0; - - rc = tiledb_vfs_is_bucket(ctx_, vfs_, bucket2.c_str(), &is_bucket); - REQUIRE(rc == TILEDB_OK); + require_tiledb_ok( + tiledb_vfs_is_bucket(x.ctx, x.vfs, s3_bucket.c_str(), &is_bucket)); if (is_bucket) { - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, s3_bucket.c_str())); } + require_tiledb_ok( + tiledb_vfs_is_bucket(x.ctx, x.vfs, s3_bucket.c_str(), &is_bucket)); + REQUIRE(!is_bucket); - rc = tiledb_vfs_create_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); + require_tiledb_ok( + tiledb_vfs_create_bucket(x.ctx, x.vfs, s3_bucket.c_str())); + require_tiledb_ok( + tiledb_vfs_is_bucket(x.ctx, x.vfs, s3_bucket.c_str(), &is_bucket)); + REQUIRE(is_bucket); + } + } - rc = tiledb_vfs_move_dir(ctx_, vfs_, subdir2.c_str(), subdir3.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file3.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); + /** Note: Azure testing not currently enabled. + if (azure_enabled) { + SECTION("Filesystem: Azure") { + path = "azure://" + random_name("tiledb") + "/tiledb_test/"; + } + } */ - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); + std::string local_prefix = ""; + SECTION("Filesystem: Local") { + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; } + path = local_prefix + temp_dir_.path(); } -} -#ifndef _WIN32 -void VFSFx::check_copy(const std::string& path) { - // Do not support HDFS - if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - return; + SECTION("Filesystem: MemFS") { + path = "mem://tiledb_test/"; } - // Copy file - auto file = path + "file"; - auto file2 = path + "file2"; - int is_file = 0; - int rc = tiledb_vfs_touch(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_copy_file(ctx_, vfs_, file.c_str(), file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - - // Move directory with subdirectories and files - auto dir = path + "dir/"; - auto dir2 = path + "dir2/"; - auto subdir = path + "dir/subdir/"; - auto subdir2 = path + "dir2/subdir/"; - file = dir + "file"; - file2 = subdir + "file2"; - auto new_file = dir2 + "file"; - auto new_file2 = subdir2 + "file2"; - int is_dir = 0; - rc = tiledb_vfs_create_dir(ctx_, vfs_, dir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (path == S3_TEMP_DIR) - REQUIRE(!is_dir); // No empty dirs exist in S3 - else - REQUIRE(is_dir); - rc = tiledb_vfs_touch(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_touch(ctx_, vfs_, file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); + // Check VFS operations + if (!path.empty()) { + bool backend_is_s3 = tiledb::sm::URI::is_s3(path); - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - if (is_dir) { - rc = tiledb_vfs_remove_dir(ctx_, vfs_, dir2.c_str()); - REQUIRE(rc == TILEDB_OK); - } - rc = tiledb_vfs_copy_dir(ctx_, vfs_, dir.c_str(), dir2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_dir(ctx_, vfs_, dir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_dir); - rc = tiledb_vfs_is_dir(ctx_, vfs_, subdir2.c_str(), &is_dir); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_dir); - rc = tiledb_vfs_is_file(ctx_, vfs_, new_file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_is_file(ctx_, vfs_, new_file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - - // Copy from one bucket to another (only for S3) - if constexpr (tiledb::sm::filesystem::s3_enabled) { - std::string bucket2 = S3_PREFIX + random_name("tiledb") + "/"; - std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; - std::string file3 = subdir3 + "file2"; - int is_bucket = 0; - - rc = tiledb_vfs_is_bucket(ctx_, vfs_, bucket2.c_str(), &is_bucket); - REQUIRE(rc == TILEDB_OK); - if (is_bucket) { - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); + // Create directory + int is_dir = 0; + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, path.c_str(), &is_dir)); + if (is_dir) { + require_tiledb_ok(tiledb_vfs_remove_dir(x.ctx, x.vfs, path.c_str())); } - - rc = tiledb_vfs_create_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); - - rc = tiledb_vfs_copy_dir(ctx_, vfs_, subdir2.c_str(), subdir3.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file3.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, path.c_str(), &is_dir)); + REQUIRE(!is_dir); + require_tiledb_ok(tiledb_vfs_create_dir(x.ctx, x.vfs, path.c_str())); + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, path.c_str(), &is_dir)); + if (backend_is_s3) + REQUIRE(!is_dir); // No empty dirs exist in S3 + else + REQUIRE(is_dir); + // Second time succeeds as well + require_tiledb_ok(tiledb_vfs_create_dir(x.ctx, x.vfs, path.c_str())); + + // Create a subdirectory + auto subdir = path + "subdir/"; + require_tiledb_ok(tiledb_vfs_create_dir(x.ctx, x.vfs, subdir.c_str())); + + // Touch + auto file = path + "file"; + int is_file = 0; + require_tiledb_ok(tiledb_vfs_touch(x.ctx, x.vfs, file.c_str())); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); REQUIRE(is_file); - rc = tiledb_vfs_remove_bucket(ctx_, vfs_, bucket2.c_str()); - REQUIRE(rc == TILEDB_OK); - } -} -#endif - -void VFSFx::check_write(const std::string& path) { - // File write and file size - int is_file = 0; - auto file = path + "file"; - int rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - if (is_file) { - rc = tiledb_vfs_remove_file(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - } - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - std::string to_write = "This will be written to the file"; - tiledb_vfs_fh_t* fh; - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_WRITE, &fh); - REQUIRE(rc == TILEDB_OK); - int is_closed = 0; - rc = tiledb_vfs_fh_is_closed(ctx_, fh, &is_closed); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_closed == 0); - rc = tiledb_vfs_write(ctx_, fh, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_sync(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - - // Only for S3, sync still does not create the file - uint64_t file_size = 0; - if (path.find("s3://") == 0) { + // Invalid file + std::string foo_file = path + "foo"; + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, foo_file.c_str(), &is_file)); REQUIRE(!is_file); - } else { - REQUIRE(is_file); - rc = tiledb_vfs_file_size(ctx_, vfs_, file.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - REQUIRE(file_size == to_write.size()); - } - - // Close file - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_fh_is_closed(ctx_, fh, &is_closed); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_closed == 1); - tiledb_vfs_fh_free(&fh); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); // It is a file even for S3 - rc = tiledb_vfs_file_size(ctx_, vfs_, file.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - REQUIRE(file_size == to_write.size()); - - // Write a second file - auto file2 = path + "file2"; - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - if (is_file) { - rc = tiledb_vfs_remove_file(ctx_, vfs_, file2.c_str()); - REQUIRE(rc == TILEDB_OK); - } - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(!is_file); - tiledb_vfs_fh_t* fh2; - rc = tiledb_vfs_open(ctx_, vfs_, file2.c_str(), TILEDB_VFS_WRITE, &fh2); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_fh_is_closed(ctx_, fh2, &is_closed); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_closed == 0); - rc = tiledb_vfs_write(ctx_, fh2, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh2); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_fh_is_closed(ctx_, fh2, &is_closed); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_closed == 1); - tiledb_vfs_fh_free(&fh2); - rc = tiledb_vfs_is_file(ctx_, vfs_, file2.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_file_size(ctx_, vfs_, file2.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - REQUIRE(file_size == to_write.size()); - - // Directory size - uint64_t dir_size; - rc = tiledb_vfs_dir_size(ctx_, vfs_, path.c_str(), &dir_size); - CHECK(rc == TILEDB_OK); - CHECK(dir_size == 2 * to_write.size()); - - // Write another file in a subdir - std::string subdir = path + "subdir"; - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - auto file3 = subdir + "file3"; - rc = tiledb_vfs_is_file(ctx_, vfs_, file3.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - if (is_file) { - rc = tiledb_vfs_remove_file(ctx_, vfs_, file3.c_str()); - REQUIRE(rc == TILEDB_OK); - } - tiledb_vfs_fh_t* fh3; - rc = tiledb_vfs_open(ctx_, vfs_, file3.c_str(), TILEDB_VFS_WRITE, &fh3); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_write(ctx_, fh3, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh3); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh3); - - rc = tiledb_vfs_dir_size(ctx_, vfs_, path.c_str(), &dir_size); - CHECK(rc == TILEDB_OK); - CHECK(dir_size == 3 * to_write.size()); - - // Check correctness with read - std::string to_read; - to_read.resize(to_write.size()); - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_READ, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_read(ctx_, fh, 0, &to_read[0], file_size); - REQUIRE(rc == TILEDB_OK); - CHECK_THAT(to_read, Catch::Matchers::Equals(to_write)); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - - // Open in WRITE mode again - previous file will be removed - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_WRITE, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_write(ctx_, fh, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_file_size(ctx_, vfs_, file.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - REQUIRE(file_size == to_write.size()); // Not 2*to_write.size() - - // Opening and closing the file without writing, first deletes previous - // file, and then creates an empty file - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_WRITE, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_is_file(ctx_, vfs_, file.c_str(), &is_file); - REQUIRE(rc == TILEDB_OK); - REQUIRE(is_file); - rc = tiledb_vfs_file_size(ctx_, vfs_, file.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - REQUIRE(file_size == 0); // Not 2*to_write.size() -} + tiledb_vfs_fh_t* fh = nullptr; + int rc = + tiledb_vfs_open(x.ctx, x.vfs, foo_file.c_str(), TILEDB_VFS_READ, &fh); + REQUIRE(rc == TILEDB_ERR); + REQUIRE(fh == nullptr); -void VFSFx::check_append(const std::string& path) { - // File write and file size - auto file = path + "file"; - tiledb_vfs_fh_t* fh; + // Check ls + std::vector children; + require_tiledb_ok( + tiledb_vfs_ls(x.ctx, x.vfs, (path).c_str(), ls_getter, &children)); + std::sort(children.begin(), children.end()); +#ifdef _WIN32 + // Normalization only for Windows + file = tiledb::sm::path_win::uri_from_path(file); + subdir = tiledb::sm::path_win::uri_from_path(subdir); +#endif + CHECK(children[0] == file); + if (!backend_is_s3) { + CHECK(children[1] + "/" == subdir); + } - // First write - std::string to_write = "This will be written to the file"; - int rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_WRITE, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_write(ctx_, fh, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); + // Check write + // File write and file size + require_tiledb_ok(tiledb_vfs_remove_file(x.ctx, x.vfs, file.c_str())); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(!is_file); + std::string to_write = "This will be written to the file"; + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_WRITE, &fh)); + int is_closed = 0; + require_tiledb_ok(tiledb_vfs_fh_is_closed(x.ctx, fh, &is_closed)); + REQUIRE(is_closed == 0); + require_tiledb_ok( + tiledb_vfs_write(x.ctx, fh, to_write.c_str(), to_write.size())); + require_tiledb_ok(tiledb_vfs_sync(x.ctx, fh)); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + + // Only for S3, sync still does not create the file + uint64_t file_size = 0; + if (backend_is_s3) { + REQUIRE(!is_file); + } else { + REQUIRE(is_file); + require_tiledb_ok( + tiledb_vfs_file_size(x.ctx, x.vfs, file.c_str(), &file_size)); + REQUIRE(file_size == to_write.size()); + } - // Second write - append - std::string to_write_2 = "This will be appended to the end of the file"; - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_APPEND, &fh); - if (path.find("s3://") == 0) { // S3 does not support append - REQUIRE(rc == TILEDB_ERR); - REQUIRE(fh == nullptr); - } else { - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_write(ctx_, fh, to_write_2.c_str(), to_write_2.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); + // Close file + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + require_tiledb_ok(tiledb_vfs_fh_is_closed(x.ctx, fh, &is_closed)); + REQUIRE(is_closed == 1); tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - uint64_t file_size = 0; - rc = tiledb_vfs_file_size(ctx_, vfs_, file.c_str(), &file_size); - REQUIRE(rc == TILEDB_OK); - uint64_t total_size = to_write.size() + to_write_2.size(); - CHECK(file_size == total_size); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(is_file); // It is a file even for S3 + require_tiledb_ok( + tiledb_vfs_file_size(x.ctx, x.vfs, file.c_str(), &file_size)); + REQUIRE(file_size == to_write.size()); + + // Write another file in a subdir + auto file2 = subdir + "file2"; + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + if (is_file) { + require_tiledb_ok(tiledb_vfs_remove_file(x.ctx, x.vfs, file2.c_str())); + } + tiledb_vfs_fh_t* fh2; + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file2.c_str(), TILEDB_VFS_WRITE, &fh2)); + require_tiledb_ok( + tiledb_vfs_write(x.ctx, fh2, to_write.c_str(), to_write.size())); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh2)); + tiledb_vfs_fh_free(&fh2); + + // Directory size + uint64_t dir_size; + require_tiledb_ok( + tiledb_vfs_dir_size(x.ctx, x.vfs, path.c_str(), &dir_size)); + CHECK(dir_size == 2 * to_write.size()); // Check correctness with read std::string to_read; - to_read.resize(total_size); - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_READ, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_read(ctx_, fh, 0, &to_read[0], total_size); - REQUIRE(rc == TILEDB_OK); - CHECK_THAT(to_read, Catch::Matchers::Equals(to_write + to_write_2)); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); + to_read.resize(to_write.size()); + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_READ, &fh)); + require_tiledb_ok(tiledb_vfs_read(x.ctx, fh, 0, &to_read[0], file_size)); + CHECK_THAT(to_read, Catch::Matchers::Equals(to_write)); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - } - - // Remove file - rc = tiledb_vfs_remove_file(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); -} - -void VFSFx::check_read(const std::string& path) { - auto file = path + "file"; - std::string to_write = "This will be written to the file"; - tiledb_vfs_fh_t* fh; - int rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_WRITE, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_write(ctx_, fh, to_write.c_str(), to_write.size()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - - // Read only the "will be written" portion of the file - std::string to_check = "will be written"; - std::string to_read; - to_read.resize(to_check.size()); - uint64_t offset = 5; - rc = tiledb_vfs_open(ctx_, vfs_, file.c_str(), TILEDB_VFS_READ, &fh); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_read(ctx_, fh, offset, &to_read[0], to_check.size()); - REQUIRE(rc == TILEDB_OK); - CHECK_THAT(to_read, Catch::Matchers::Equals(to_check)); - rc = tiledb_vfs_close(ctx_, fh); - REQUIRE(rc == TILEDB_OK); - tiledb_vfs_fh_free(&fh); - REQUIRE(rc == TILEDB_OK); - // Remove file - rc = tiledb_vfs_remove_file(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); -} - -int ls_getter(const char* path, void* data) { - auto vec = static_cast*>(data); - vec->emplace_back(path); - return 1; -} - -void VFSFx::check_ls(const std::string& path) { - std::string dir = path + "ls_dir"; - std::string file = dir + "/file"; - std::string file2 = dir + "/file2"; - std::string subdir = dir + "/subdir"; - std::string subdir2 = dir + "/subdir2"; - std::string subdir_file = subdir + "/file"; - std::string subdir_file2 = subdir2 + "/file2"; - - // Create directories and files - int rc = tiledb_vfs_create_dir(ctx_, vfs_, dir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_create_dir(ctx_, vfs_, subdir2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_touch(ctx_, vfs_, file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_touch(ctx_, vfs_, file2.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_touch(ctx_, vfs_, subdir_file.c_str()); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_touch(ctx_, vfs_, subdir_file2.c_str()); - REQUIRE(rc == TILEDB_OK); - - // List - std::vector children; - rc = tiledb_vfs_ls(ctx_, vfs_, (dir + "/").c_str(), ls_getter, &children); - REQUIRE(rc == TILEDB_OK); - - // Normalization - for (auto& child : children) { - if (child.back() == '/') - child.pop_back(); - } - -#ifdef _WIN32 - // Normalization only for Windows - file = tiledb::sm::path_win::uri_from_path(file); - file2 = tiledb::sm::path_win::uri_from_path(file2); - subdir = tiledb::sm::path_win::uri_from_path(subdir); - subdir2 = tiledb::sm::path_win::uri_from_path(subdir2); -#endif - - // Check results - std::sort(children.begin(), children.end()); - REQUIRE(children.size() == 4); - CHECK(children[0] == file); - CHECK(children[1] == file2); - CHECK(children[2] == subdir); - CHECK(children[3] == subdir2); -} + // Read only the "will be written" portion of the file + std::string to_check = "will be written"; + to_read.resize(to_check.size()); + uint64_t offset = 5; + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_READ, &fh)); + require_tiledb_ok( + tiledb_vfs_read(x.ctx, fh, offset, &to_read[0], to_check.size())); + CHECK_THAT(to_read, Catch::Matchers::Equals(to_check)); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + tiledb_vfs_fh_free(&fh); -std::string VFSFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} + // Check append + std::string to_write_2 = "This will be appended to the end of the file"; + rc = tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_APPEND, &fh); + if (backend_is_s3) { // S3 does not support append + REQUIRE(rc == TILEDB_ERR); + REQUIRE(fh == nullptr); + } else { + REQUIRE(rc == TILEDB_OK); + require_tiledb_ok( + tiledb_vfs_write(x.ctx, fh, to_write_2.c_str(), to_write_2.size())); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + tiledb_vfs_fh_free(&fh); + uint64_t file_size = 0; + require_tiledb_ok( + tiledb_vfs_file_size(x.ctx, x.vfs, file.c_str(), &file_size)); + uint64_t total_size = to_write.size() + to_write_2.size(); + CHECK(file_size == total_size); + + // Check correctness with read + std::string to_read; + to_read.resize(total_size); + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_READ, &fh)); + require_tiledb_ok(tiledb_vfs_read(x.ctx, fh, 0, &to_read[0], total_size)); + CHECK_THAT(to_read, Catch::Matchers::Equals(to_write + to_write_2)); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + tiledb_vfs_fh_free(&fh); + } -TEST_CASE_METHOD(VFSFx, "C API: Test virtual filesystem", "[capi][vfs]") { - tiledb_stats_enable(); - tiledb_stats_reset(); + // Open in WRITE mode again - previous file will be removed + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_WRITE, &fh)); + require_tiledb_ok( + tiledb_vfs_write(x.ctx, fh, to_write.c_str(), to_write.size())); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + tiledb_vfs_fh_free(&fh); + require_tiledb_ok( + tiledb_vfs_file_size(x.ctx, x.vfs, file.c_str(), &file_size)); + REQUIRE(file_size == to_write.size()); // Not 2*to_write.size() + + // Opening and closing the file without writing, first deletes previous + // file, and then creates an empty file + require_tiledb_ok( + tiledb_vfs_open(x.ctx, x.vfs, file.c_str(), TILEDB_VFS_WRITE, &fh)); + require_tiledb_ok(tiledb_vfs_close(x.ctx, fh)); + tiledb_vfs_fh_free(&fh); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(is_file); + require_tiledb_ok( + tiledb_vfs_file_size(x.ctx, x.vfs, file.c_str(), &file_size)); + REQUIRE(file_size == 0); // Not 2*to_write.size() + + // Move file + file2 = subdir + "file"; + require_tiledb_ok( + tiledb_vfs_move_file(x.ctx, x.vfs, file.c_str(), file2.c_str())); + require_tiledb_ok(tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(!is_file); + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); - if constexpr (tiledb::sm::filesystem::s3_enabled) { - check_vfs(S3_TEMP_DIR); - } else if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - check_vfs(HDFS_TEMP_DIR); - } else { - check_vfs(FILE_TEMP_DIR); - check_vfs(MEMFS_TEMP_DIR); - } -} + // Move directory + auto subdir2 = path + "subdir2/"; + require_tiledb_ok( + tiledb_vfs_move_dir(x.ctx, x.vfs, subdir.c_str(), subdir2.c_str())); + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, subdir.c_str(), &is_dir)); + REQUIRE(!is_dir); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir2.c_str(), &is_dir)); + file2 = subdir2 + "file"; + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); -TEST_CASE_METHOD( - VFSFx, - "C API: Test virtual filesystem when S3 is not supported", - "[capi][vfs]") { - if constexpr (!tiledb::sm::filesystem::s3_enabled) { - tiledb_vfs_t* vfs; - int rc = tiledb_vfs_alloc(ctx_, nullptr, &vfs); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_vfs_create_bucket(ctx_, vfs, "s3://foo"); - REQUIRE(rc == TILEDB_ERR); - tiledb_vfs_free(&vfs); - } -} + // Move from one bucket to another (only for S3) + if (backend_is_s3) { + std::string bucket2 = "s3://" + random_name("tiledb") + "/"; + std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; -TEST_CASE_METHOD( - VFSFx, "C API: Test virtual filesystem config", "[capi][vfs]") { - // Prepare a config - tiledb_error_t* error = nullptr; - tiledb_config_t* config; - int rc = tiledb_config_alloc(&config, &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); - rc = tiledb_config_set(config, "vfs.s3.scheme", "https", &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); + // Remove and recreate bucket if already exists + int is_bucket = 0; + require_tiledb_ok( + tiledb_vfs_is_bucket(x.ctx, x.vfs, bucket2.c_str(), &is_bucket)); + if (is_bucket) { + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, bucket2.c_str())); + } + require_tiledb_ok( + tiledb_vfs_create_bucket(x.ctx, x.vfs, bucket2.c_str())); + + // Move bucket + require_tiledb_ok( + tiledb_vfs_move_dir(x.ctx, x.vfs, subdir2.c_str(), subdir3.c_str())); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir3.c_str(), &is_dir)); + REQUIRE(is_dir); + file2 = subdir3 + "file"; + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); - // Create VFS - tiledb_vfs_t* vfs; - rc = tiledb_vfs_alloc(ctx_, config, &vfs); - REQUIRE(rc == TILEDB_OK); + // Move back + require_tiledb_ok( + tiledb_vfs_move_dir(x.ctx, x.vfs, subdir3.c_str(), subdir2.c_str())); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir2.c_str(), &is_dir)); + REQUIRE(is_dir); - // Get VFS config and check - tiledb_config_t* config2; - rc = tiledb_vfs_get_config(ctx_, vfs, &config2); - REQUIRE(rc == TILEDB_OK); - const char* value; - rc = tiledb_config_get(config2, "vfs.s3.scheme", &value, &error); - REQUIRE(error == nullptr); - CHECK(!strncmp(value, "https", strlen("https"))); + // Remove bucket + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, bucket2.c_str())); + } - // Clean up - tiledb_config_free(&config); - tiledb_config_free(&config2); - tiledb_vfs_free(&vfs); -} + // Check copy (not yet supported for MemFS or HDFS) + if constexpr (!tiledb::platform::is_os_windows) { + if (!tiledb::sm::URI::is_memfs(path) && !tiledb::sm::URI::is_hdfs(path)) { + auto dir = path + "dir/"; + auto file = dir + "file"; + int is_file = 0; + require_tiledb_ok(tiledb_vfs_create_dir(x.ctx, x.vfs, dir.c_str())); + require_tiledb_ok(tiledb_vfs_touch(x.ctx, x.vfs, file.c_str())); + int is_dir = 0; + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, dir.c_str(), &is_dir)); + REQUIRE(is_dir); + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(is_file); + + // Copy file + auto file2 = dir + "file2"; + require_tiledb_ok( + tiledb_vfs_copy_file(x.ctx, x.vfs, file.c_str(), file2.c_str())); + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file.c_str(), &is_file)); + REQUIRE(is_file); + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); + + // Copy directory + auto dir2 = path + "dir2/"; + file2 = dir2 + "file2"; + require_tiledb_ok( + tiledb_vfs_copy_dir(x.ctx, x.vfs, dir.c_str(), dir2.c_str())); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, dir2.c_str(), &is_dir)); + REQUIRE(is_dir); + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); + + // Copy from one bucket to another (only for S3) + if (backend_is_s3) { + std::string bucket2 = "s3://" + random_name("tiledb") + "/"; + std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; + + // Remove and recreate bucket if already exists + int is_bucket = 0; + require_tiledb_ok( + tiledb_vfs_is_bucket(x.ctx, x.vfs, bucket2.c_str(), &is_bucket)); + if (is_bucket) { + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, bucket2.c_str())); + } + require_tiledb_ok( + tiledb_vfs_create_bucket(x.ctx, x.vfs, bucket2.c_str())); + + // Copy bucket + require_tiledb_ok( + tiledb_vfs_copy_dir(x.ctx, x.vfs, dir2.c_str(), subdir3.c_str())); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir3.c_str(), &is_dir)); + REQUIRE(is_dir); + file2 = subdir3 + "file"; + require_tiledb_ok( + tiledb_vfs_is_file(x.ctx, x.vfs, file2.c_str(), &is_file)); + REQUIRE(is_file); + + // Remove bucket + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, bucket2.c_str())); + } + } + } -TEST_CASE_METHOD(VFSFx, "C API: Test VFS parallel I/O", "[capi][vfs]") { - tiledb_stats_enable(); - tiledb_stats_reset(); + // Clean up + if (backend_is_s3) { + // Empty s3 bucket + int is_empty; + require_tiledb_ok(tiledb_vfs_is_empty_bucket( + x.ctx, x.vfs, s3_bucket.c_str(), &is_empty)); + REQUIRE(!(bool)is_empty); - tiledb_config_t* config; - REQUIRE(tiledb_vfs_get_config(ctx_, vfs_, &config) == TILEDB_OK); + require_tiledb_ok( + tiledb_vfs_empty_bucket(x.ctx, x.vfs, s3_bucket.c_str())); - // Set number of threads to 4. - tiledb_error_t* error = nullptr; - REQUIRE( - tiledb_config_set( - config, - "vfs.s3.max_parallel_ops", - std::to_string(4).c_str(), - &error) == TILEDB_OK); - // Set very small parallelization threshold (ignored when there is only 1 - // thread). - REQUIRE( - tiledb_config_set( - config, "vfs.min_parallel_size", std::to_string(1).c_str(), &error) == - TILEDB_OK); - REQUIRE(error == nullptr); + require_tiledb_ok(tiledb_vfs_is_empty_bucket( + x.ctx, x.vfs, s3_bucket.c_str(), &is_empty)); + REQUIRE((bool)is_empty); - if constexpr (tiledb::sm::filesystem::s3_enabled) { - check_vfs(S3_TEMP_DIR); - } else if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - check_vfs(HDFS_TEMP_DIR); - } else { - check_vfs(FILE_TEMP_DIR); - check_vfs(MEMFS_TEMP_DIR); + require_tiledb_ok( + tiledb_vfs_remove_bucket(x.ctx, x.vfs, s3_bucket.c_str())); + } else { + // Remove directory recursively + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, path.c_str(), &is_dir)); + REQUIRE(is_dir); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir2.c_str(), &is_dir)); + REQUIRE(is_dir); + require_tiledb_ok(tiledb_vfs_remove_dir(x.ctx, x.vfs, path.c_str())); + require_tiledb_ok(tiledb_vfs_is_dir(x.ctx, x.vfs, path.c_str(), &is_dir)); + REQUIRE(!is_dir); + require_tiledb_ok( + tiledb_vfs_is_dir(x.ctx, x.vfs, subdir2.c_str(), &is_dir)); + REQUIRE(!is_dir); + } } } diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index fdca6bbd4987..388363f010b0 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -783,50 +783,6 @@ void create_azure_container( } } -void create_ctx_and_vfs(tiledb_ctx_t** ctx, tiledb_vfs_t** vfs) { - // Create TileDB context - tiledb_config_t* config = nullptr; - tiledb_error_t* error = nullptr; - throw_if_setup_failed(tiledb_config_alloc(&config, &error)); - throw_if_setup_failed(error == nullptr); - if constexpr (tiledb::sm::filesystem::s3_enabled) { -#ifndef TILEDB_TESTS_AWS_S3_CONFIG - throw_if_setup_failed(tiledb_config_set( - config, "vfs.s3.endpoint_override", "localhost:9999", &error)); - throw_if_setup_failed( - tiledb_config_set(config, "vfs.s3.scheme", "https", &error)); - throw_if_setup_failed(tiledb_config_set( - config, "vfs.s3.use_virtual_addressing", "false", &error)); - throw_if_setup_failed( - tiledb_config_set(config, "vfs.s3.verify_ssl", "false", &error)); - throw_if_setup_failed(error == nullptr); -#endif - } - if constexpr (tiledb::sm::filesystem::azure_enabled) { - throw_if_setup_failed(tiledb_config_set( - config, "vfs.azure.storage_account_name", "devstoreaccount1", &error)); - throw_if_setup_failed(tiledb_config_set( - config, - "vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw==", - &error)); - throw_if_setup_failed(tiledb_config_set( - config, - "vfs.azure.blob_endpoint", - "http://127.0.0.1:10000/devstoreaccount1", - &error)); - } - throw_if_setup_failed(tiledb_ctx_alloc(config, ctx)); - throw_if_setup_failed(ctx != nullptr); - - // Create VFS - *vfs = nullptr; - throw_if_setup_failed(tiledb_vfs_alloc(*ctx, config, vfs)); - throw_if_setup_failed(vfs != nullptr); - tiledb_config_free(&config); -} - void get_supported_fs( bool* s3_supported, bool* hdfs_supported, diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index d1653c73e0b1..6c0324a23d6d 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -480,14 +480,6 @@ void create_subarray( tiledb::Subarray** subarray, bool coalesce_ranges = false); -/** - * Helper method that creates a TileDB context and a VFS object. - * - * @param ctx The TileDB context to be created. - * @param vfs The VFS object to be created. - */ -void create_ctx_and_vfs(tiledb_ctx_t** ctx, tiledb_vfs_t** vfs); - /** * Helper function to get the supported filesystems. * Supports VFS override via "--vfs" command line argument. diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index f0b2017aaa4f..d5c5a2ec20ec 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2020-2021 TileDB, Inc. + * @copyright Copyright (c) 2020-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,8 +43,7 @@ #include "tiledb/sm/filesystem/posix.h" #endif -namespace tiledb { -namespace test { +namespace tiledb::test { // Forward declaration class SupportedFs; @@ -549,6 +548,61 @@ class SupportedFsMem : public SupportedFs { std::string temp_dir_; }; +/** + * Struct which allocates a config and conditionally sets filesystem-specific + * parameters on it. + */ +struct vfs_config { + /** Config handle. */ + tiledb_config_handle_t* config{nullptr}; + + /** Constructor. */ + vfs_config() { + tiledb_error_t* error = nullptr; + auto rc = tiledb_config_alloc(&config, &error); + if (rc != TILEDB_OK) { + throw std::runtime_error("error creating config handle"); + } + if (error != nullptr) { + throw std::logic_error( + "tiledb_config_alloc returned OK but with non-null error"); + } + + if constexpr (tiledb::sm::filesystem::s3_enabled) { + SupportedFsS3 fs_s3; + auto st = fs_s3.prepare_config(config, error); + if (!st.ok()) { + throw std::runtime_error("error preparing S3 config"); + } + } + + if constexpr (tiledb::sm::filesystem::azure_enabled) { + SupportedFsAzure fs_azure; + auto st = fs_azure.prepare_config(config, error); + if (!st.ok()) { + throw std::runtime_error("error preparing Azure config"); + } + } + } + + /** Copy constructor is deleted. */ + vfs_config(const vfs_config&) = delete; + + /** Move constructor is deleted. */ + vfs_config(vfs_config&&) = delete; + + /** Copy assignment is deleted. */ + vfs_config& operator=(const vfs_config&) = delete; + + /** Move assignment is deleted. */ + vfs_config& operator=(vfs_config&&) = delete; + + /** Destructor. */ + ~vfs_config() { + tiledb_config_free(&config); + } +}; + /** * Fixture for creating a temporary directory for a test case. This fixture * also manages the context and virtual file system for the test case. @@ -719,7 +773,6 @@ class DenyWriteAccess { const std::filesystem::perms previous_perms_; }; -} // End of namespace test -} // End of namespace tiledb +} // namespace tiledb::test #endif diff --git a/tiledb/api/c_api_test_support/testsupport_capi_vfs.h b/tiledb/api/c_api_test_support/testsupport_capi_vfs.h index 5b2172abc5ff..11e7ed85696b 100644 --- a/tiledb/api/c_api_test_support/testsupport_capi_vfs.h +++ b/tiledb/api/c_api_test_support/testsupport_capi_vfs.h @@ -41,12 +41,12 @@ namespace tiledb::api::test_support { struct ordinary_vfs { tiledb_ctx_handle_t* ctx{nullptr}; tiledb_vfs_handle_t* vfs{nullptr}; - ordinary_vfs() { + ordinary_vfs(tiledb_config_t* config = nullptr) { auto rc = tiledb_ctx_alloc(nullptr, &ctx); if (rc != TILEDB_OK) { throw std::runtime_error("error creating test context"); } - rc = tiledb_vfs_alloc(ctx, nullptr, &vfs); + rc = tiledb_vfs_alloc(ctx, config, &vfs); if (rc != TILEDB_OK) { throw std::runtime_error("error creating test vfs"); } From 896dc75f9c7c2c08ede3451c27c43a7304e9d7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Thu, 14 Dec 2023 15:39:14 +0100 Subject: [PATCH 109/456] Add packaging test into CI/CD (#4567) sc-24994 The purpose of this PR is to turn on packaging tests for linux and mac platforms (it currently runs on windows only). Changes: 1. Modify Include find path from 'include/TileDB' to 'include/tiledb' 2. Wrap packaging test logic into shell script 3. Add packaging test into CI/CD pipeline --- TYPE: BUILD DESC: Add packaging tests into linux and mac CI pipelines --- .github/workflows/build-windows.yml | 7 +------ .github/workflows/ci-linux_mac.yml | 5 +++++ test/CMakeLists.txt | 8 ++++++++ test/packaging/CMakeLists.txt | 7 +++++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 5862eb93ac3c..2b898a1a333a 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -392,12 +392,7 @@ jobs: - name: Packaging test shell: pwsh run: | - cd $env:BUILD_SOURCESDIRECTORY\test\packaging - mkdir build - cd build - cmake -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. - # The testing is in the test/packaging/CMakeLists.txt file, just invoked, - # there is no other code to be built in the produced projects. + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb --target check-package - name: 'process crashdumps' shell: cmd diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 32b0dfd59930..4be5997183ff 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -212,6 +212,11 @@ jobs: # Build benchmark code source $GITHUB_WORKSPACE/scripts/ci/build_benchmarks.sh + + ################################################### + # Perform package test + + cmake --build $GITHUB_WORKSPACE/build/tiledb --target check-package - name: 'Dump core stacks on failure' if: ${{ failure() && startsWith(matrix.os, 'ubuntu-') == true }} # only run this job if the build step failed diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 50316237e807..12987e299340 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -406,3 +406,11 @@ add_subdirectory(ci) if(WIN32) add_subdirectory(performance) endif() + +add_custom_target( + check-package + COMMAND ${CMAKE_COMMAND} + -DCMAKE_PREFIX_PATH="${CMAKE_INSTALL_PREFIX}$${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}" + -B ${CMAKE_CURRENT_BINARY_DIR}/check-package-build + ${CMAKE_CURRENT_SOURCE_DIR}/packaging +) diff --git a/test/packaging/CMakeLists.txt b/test/packaging/CMakeLists.txt index a1ba1bc31d9f..a7921767e3e8 100644 --- a/test/packaging/CMakeLists.txt +++ b/test/packaging/CMakeLists.txt @@ -25,7 +25,7 @@ # THE SOFTWARE. # -# Expects invocation wtih -DCMAKE_PREFIX_PATH= +# Expects invocation with -DCMAKE_PREFIX_PATH= cmake_minimum_required(VERSION 3.21) @@ -33,7 +33,10 @@ project(Test_TileDBConfigVersion_h) # Parse version numbers out of tiledb_version.h file -find_file(tiledb_version_h tiledb_version.h PATH_SUFFIXES "include/TileDB" REQUIRED) +find_file( + tiledb_version_h tiledb_version.h + PATH_SUFFIXES "include/tiledb" REQUIRED +) file(READ ${tiledb_version_h} ver) string(REGEX MATCH "TILEDB_VERSION_MAJOR ([0-9]*)" _ ${ver}) From 7e81c04408abd45330e84af6b8b5ad80b7a5f7ca Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 15 Dec 2023 19:26:09 +0200 Subject: [PATCH 110/456] Update the AWS SDK to version 1.11.215. (#4551) [SC-37841](https://app.shortcut.com/tiledb-inc/story/37841/update-aws-sdk-to-at-least-1-11-179) This PR updates the AWS SDK to its latest version as of now (1.11.215). The first commit copies the ports from the latest version on vcpkg (1.11.201), and the second commit updates them to their latest version. The reason for the second update is that version 1.11.205 upstreamed `remove-checked-array-iterator.patch` that fixes MinGW build failures, and I found the opportunity to remove the patch from our code. The ports on vcpkg will be updated with microsoft/vcpkg#35445. Verified by successfully locally building on my machine. The SHA-512 hash for the `s2n` sources (which is not supported on Windows) was computed by manually downloading and hashing it. --- TYPE: IMPROVEMENT DESC: Update the AWS SDK to version 1.11.215. --- ports/aws-c-auth/portfile.cmake | 2 +- ports/aws-c-auth/vcpkg.json | 2 +- ports/aws-c-cal/portfile.cmake | 2 +- ports/aws-c-cal/vcpkg.json | 6 +- ports/aws-c-common/portfile.cmake | 2 +- ports/aws-c-common/vcpkg.json | 2 +- ports/aws-c-http/portfile.cmake | 2 +- ports/aws-c-http/vcpkg.json | 2 +- ports/aws-c-io/portfile.cmake | 2 +- ports/aws-c-io/vcpkg.json | 2 +- ports/aws-c-mqtt/portfile.cmake | 2 +- ports/aws-c-mqtt/vcpkg.json | 2 +- ports/aws-c-s3/portfile.cmake | 2 +- ports/aws-c-s3/vcpkg.json | 2 +- ports/aws-crt-cpp/portfile.cmake | 2 +- ports/aws-crt-cpp/vcpkg.json | 2 +- ports/aws-sdk-cpp/compute_build_only.cmake | 1090 ----------------- .../fix-awsmigrationhub-build.patch | 10 + ports/aws-sdk-cpp/generateFeatures.ps1 | 158 ++- ports/aws-sdk-cpp/portfile.cmake | 202 ++- .../remove-checked-array-iterator.patch | 51 - ports/aws-sdk-cpp/usage | 24 +- ports/aws-sdk-cpp/vcpkg.in.json | 4 +- ports/aws-sdk-cpp/vcpkg.json | 5 +- ports/s2n/portfile.cmake | 3 +- ports/s2n/remove-trycompile.patch | 59 - ports/s2n/vcpkg.json | 2 +- 27 files changed, 220 insertions(+), 1424 deletions(-) delete mode 100644 ports/aws-sdk-cpp/compute_build_only.cmake create mode 100644 ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch delete mode 100644 ports/aws-sdk-cpp/remove-checked-array-iterator.patch delete mode 100644 ports/s2n/remove-trycompile.patch diff --git a/ports/aws-c-auth/portfile.cmake b/ports/aws-c-auth/portfile.cmake index be658b76ac74..00ae5a4f0179 100644 --- a/ports/aws-c-auth/portfile.cmake +++ b/ports/aws-c-auth/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-auth REF "v${VERSION}" - SHA512 a5508c7bee8a8b4fb12d598e270913d3b6e63190faa866c98d841cc5df814f959cda88e62ea794b7456f23c07a8457f7d4af78ac4758211791051293843ba3b5 + SHA512 e247b145132818bb1adf03689fdc98984e6ac50e6d81d0d7caf60d53ba3d3a95bbb4e76f48ac4582365f9038d0d8b29e57947190ec5c34f68e816dd7efcc117d HEAD_REF master ) diff --git a/ports/aws-c-auth/vcpkg.json b/ports/aws-c-auth/vcpkg.json index 9bf2280e88a3..091f50478960 100644 --- a/ports/aws-c-auth/vcpkg.json +++ b/ports/aws-c-auth/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-auth", - "version": "0.7.3", + "version": "0.7.8", "description": "C99 library implementation of AWS client-side authentication: standard credentials providers and signing.", "homepage": "https://github.com/awslabs/aws-c-auth", "license": "Apache-2.0", diff --git a/ports/aws-c-cal/portfile.cmake b/ports/aws-c-cal/portfile.cmake index 4782cce5420e..3cef1fbb2825 100644 --- a/ports/aws-c-cal/portfile.cmake +++ b/ports/aws-c-cal/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-cal REF "v${VERSION}" - SHA512 b1116977b467c2c7f10f84912a3bc2a8329e3ec22c58f19f7b8a244a2b08fb3420bed62791b7ad6f06b8aeff6c361a33ddc0ac28cf781dfa1aafc83a62aa24ec + SHA512 deee106b366522e6781974c92b1aa06542b7857b91a8d4cb59eb0e17247ce7fc3ffacb044c032ff7f2a0f9baca807d4c2d9a14934d4576966f48bfc0661e5edb HEAD_REF master PATCHES remove-libcrypto-messages.patch ) diff --git a/ports/aws-c-cal/vcpkg.json b/ports/aws-c-cal/vcpkg.json index ff6212c8abd6..b1d22ed8eade 100644 --- a/ports/aws-c-cal/vcpkg.json +++ b/ports/aws-c-cal/vcpkg.json @@ -1,12 +1,16 @@ { "name": "aws-c-cal", - "version": "0.6.2", + "version": "0.6.9", "description": "C99 wrapper for cryptography primitives.", "homepage": "https://github.com/awslabs/aws-c-cal", "license": "Apache-2.0", "supports": "!(windows & arm) & !uwp", "dependencies": [ "aws-c-common", + { + "name": "openssl", + "platform": "!windows & !osx" + }, { "name": "vcpkg-cmake", "host": true diff --git a/ports/aws-c-common/portfile.cmake b/ports/aws-c-common/portfile.cmake index 0d00bf2473f6..e96d1a82844e 100644 --- a/ports/aws-c-common/portfile.cmake +++ b/ports/aws-c-common/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-common REF "v${VERSION}" - SHA512 e68f7096bb5a0f908e28f910d51366284713c477ed3ea852136d4f33ad940bcb66a8fe731ba0b1f6e397812f07be426b982c62ea0e3ead0e581f46d3e5801c13 + SHA512 f04bdc01d52b5ad191d320d352a13dd4cb9675127596c2c2229657211bc5fa3cddf05a3b395a0dc0ac5ce2f09cecf54c04b9cfacb08299e24a16a47684560f11 HEAD_REF master PATCHES disable-internal-crt-option.patch # Disable internal crt option because vcpkg contains crt processing flow diff --git a/ports/aws-c-common/vcpkg.json b/ports/aws-c-common/vcpkg.json index fab95ef4f03e..9d7a14ce9cfd 100644 --- a/ports/aws-c-common/vcpkg.json +++ b/ports/aws-c-common/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-common", - "version": "0.9.3", + "version": "0.9.10", "description": "AWS common library for C", "homepage": "https://github.com/awslabs/aws-c-common", "license": "Apache-2.0", diff --git a/ports/aws-c-http/portfile.cmake b/ports/aws-c-http/portfile.cmake index abda1ef1786a..a0cf2430398e 100644 --- a/ports/aws-c-http/portfile.cmake +++ b/ports/aws-c-http/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-http REF "v${VERSION}" - SHA512 451b542926e5eeb85006053ba8835636d6fbb33ff31ecacb0b3e98d667b9768cdb448c172a33cfe400171fff64c37492ec4467da443d99ae32a70292de6d1713 + SHA512 205cf1cc4b766fd42cfca6ab84974e0797f2a52f8cdf64fed0c4327a2b70c251e5c9561b1cf714cbe4b7ef87092a094830071e6621bd0e0e1684b2e66ea718e2 HEAD_REF master ) diff --git a/ports/aws-c-http/vcpkg.json b/ports/aws-c-http/vcpkg.json index 852afc214c50..dcbb887aa9e0 100644 --- a/ports/aws-c-http/vcpkg.json +++ b/ports/aws-c-http/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-http", - "version": "0.7.12", + "version": "0.7.14", "description": "C99 implementation of the HTTP/1.1 and HTTP/2 specifications", "homepage": "https://github.com/awslabs/aws-c-http", "license": "Apache-2.0", diff --git a/ports/aws-c-io/portfile.cmake b/ports/aws-c-io/portfile.cmake index 96bbd9291723..bf331bfbaa20 100644 --- a/ports/aws-c-io/portfile.cmake +++ b/ports/aws-c-io/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-io REF "v${VERSION}" - SHA512 54f6ef23ab7fb65c9d1425d21d59daf1b13a50595f7f06c7d4f74bfb5941a7ddf0185ae3249b940473422c1880bb034c0610876c6e13ae499ccd2dcc888c1b23 + SHA512 cb76ddf7d43c440443c6e5fb33a022f540d8ca95572f36b263836fe6bebfd0790370c84fec37b45475645167ab71bb14de320b9988803ac01f6bbb0d7436949a HEAD_REF master ) diff --git a/ports/aws-c-io/vcpkg.json b/ports/aws-c-io/vcpkg.json index f7720445def0..e6fc427362c2 100644 --- a/ports/aws-c-io/vcpkg.json +++ b/ports/aws-c-io/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-io", - "version": "0.13.32", + "version": "0.13.36", "description": "Handles all IO and TLS work for application protocols.", "homepage": "https://github.com/awslabs/aws-c-io", "license": "Apache-2.0", diff --git a/ports/aws-c-mqtt/portfile.cmake b/ports/aws-c-mqtt/portfile.cmake index 7fbc122c28f6..fd41c8328a84 100644 --- a/ports/aws-c-mqtt/portfile.cmake +++ b/ports/aws-c-mqtt/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-mqtt REF "v${VERSION}" - SHA512 e5b7adf0e4450d93159f996e1e95bcc09b97985b180fa4f13f1f9a1f8901e9e287248f2b7af552e3d2742d29c257ad6f4975458824833054b06794c4f916af96 + SHA512 9664596de78a8778349cf32d5dd207ed8e5a28e1d13505f942c8486c3fbf516ec49121af312d85182ebdc66b464d430fcd2d28606b1c2652a8dfb9c173334681 HEAD_REF master ) diff --git a/ports/aws-c-mqtt/vcpkg.json b/ports/aws-c-mqtt/vcpkg.json index 1c7eb94bca69..1bec811c3463 100644 --- a/ports/aws-c-mqtt/vcpkg.json +++ b/ports/aws-c-mqtt/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-mqtt", - "version": "0.9.5", + "version": "0.9.10", "description": "C99 implementation of the MQTT 3.1.1 specification.", "homepage": "https://github.com/awslabs/aws-c-mqtt", "license": "Apache-2.0", diff --git a/ports/aws-c-s3/portfile.cmake b/ports/aws-c-s3/portfile.cmake index 5453b8e8d6b4..4b65d3999499 100644 --- a/ports/aws-c-s3/portfile.cmake +++ b/ports/aws-c-s3/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-c-s3 REF "v${VERSION}" - SHA512 f96cab50c0f96d6cb3cb8e142cc95c91e9efd0b02d115e91ec5d2fc7c4be350d58342f5846065216a49c080aaf3954fa9732ff423bad20b250afe4227ee3aaef + SHA512 62dca71a857f7c55cb824cf1a81f3dbefa7cf7d50e5fadec5f2e67d4c2e33c2902a483145b4e9390aa5adf68ab65588c7c71e91717b74733db2a15e7a1cc5794 HEAD_REF master ) diff --git a/ports/aws-c-s3/vcpkg.json b/ports/aws-c-s3/vcpkg.json index 0616b5d83a55..362b597c81bc 100644 --- a/ports/aws-c-s3/vcpkg.json +++ b/ports/aws-c-s3/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-c-s3", - "version": "0.3.17", + "version": "0.4.3", "description": "C99 library implementation for communicating with the S3 service, designed for maximizing throughput on high bandwidth EC2 instances.", "homepage": "https://github.com/awslabs/aws-c-s3", "license": "Apache-2.0", diff --git a/ports/aws-crt-cpp/portfile.cmake b/ports/aws-crt-cpp/portfile.cmake index 2addfc63ae65..553fc30c9021 100644 --- a/ports/aws-crt-cpp/portfile.cmake +++ b/ports/aws-crt-cpp/portfile.cmake @@ -2,7 +2,7 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO awslabs/aws-crt-cpp REF "v${VERSION}" - SHA512 88631845bf78e526b3160d0ba6ff67922b09f8e25864bfeeb4cf16f2d217bf00c9686e53e70382129d8209612453190024a16aa3cff153198960c8895f2e1772 + SHA512 4160aae9aaf98556d9fd26a13feb09f7fce2a20e073740d13ab9890fdab158de63918c8fd2f4a2d07e92798e2df47b6696b38e614c9631ebcdddf7ea1c44a126 PATCHES no-werror.patch ) diff --git a/ports/aws-crt-cpp/vcpkg.json b/ports/aws-crt-cpp/vcpkg.json index 6a5f6e4128c3..a211574cd573 100644 --- a/ports/aws-crt-cpp/vcpkg.json +++ b/ports/aws-crt-cpp/vcpkg.json @@ -1,6 +1,6 @@ { "name": "aws-crt-cpp", - "version": "0.23.1", + "version": "0.24.8", "description": "C++ wrapper around the aws-c-* libraries. Provides Cross-Platform Transport Protocols and SSL/TLS implementations for C++.", "homepage": "https://github.com/awslabs/aws-crt-cpp", "license": "Apache-2.0", diff --git a/ports/aws-sdk-cpp/compute_build_only.cmake b/ports/aws-sdk-cpp/compute_build_only.cmake deleted file mode 100644 index 27b93c24bc75..000000000000 --- a/ports/aws-sdk-cpp/compute_build_only.cmake +++ /dev/null @@ -1,1090 +0,0 @@ -# Automatically generated by generateFeatures.ps1 -if("access-management" IN_LIST FEATURES) - list(APPEND BUILD_ONLY access-management) -endif() -if("accessanalyzer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY accessanalyzer) -endif() -if("account" IN_LIST FEATURES) - list(APPEND BUILD_ONLY account) -endif() -if("acm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY acm) -endif() -if("acm-pca" IN_LIST FEATURES) - list(APPEND BUILD_ONLY acm-pca) -endif() -if("alexaforbusiness" IN_LIST FEATURES) - list(APPEND BUILD_ONLY alexaforbusiness) -endif() -if("amp" IN_LIST FEATURES) - list(APPEND BUILD_ONLY amp) -endif() -if("amplify" IN_LIST FEATURES) - list(APPEND BUILD_ONLY amplify) -endif() -if("amplifybackend" IN_LIST FEATURES) - list(APPEND BUILD_ONLY amplifybackend) -endif() -if("amplifyuibuilder" IN_LIST FEATURES) - list(APPEND BUILD_ONLY amplifyuibuilder) -endif() -if("apigateway" IN_LIST FEATURES) - list(APPEND BUILD_ONLY apigateway) -endif() -if("apigatewaymanagementapi" IN_LIST FEATURES) - list(APPEND BUILD_ONLY apigatewaymanagementapi) -endif() -if("apigatewayv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY apigatewayv2) -endif() -if("appconfig" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appconfig) -endif() -if("appconfigdata" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appconfigdata) -endif() -if("appfabric" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appfabric) -endif() -if("appflow" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appflow) -endif() -if("appintegrations" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appintegrations) -endif() -if("application-autoscaling" IN_LIST FEATURES) - list(APPEND BUILD_ONLY application-autoscaling) -endif() -if("application-insights" IN_LIST FEATURES) - list(APPEND BUILD_ONLY application-insights) -endif() -if("applicationcostprofiler" IN_LIST FEATURES) - list(APPEND BUILD_ONLY applicationcostprofiler) -endif() -if("appmesh" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appmesh) -endif() -if("apprunner" IN_LIST FEATURES) - list(APPEND BUILD_ONLY apprunner) -endif() -if("appstream" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appstream) -endif() -if("appsync" IN_LIST FEATURES) - list(APPEND BUILD_ONLY appsync) -endif() -if("arc-zonal-shift" IN_LIST FEATURES) - list(APPEND BUILD_ONLY arc-zonal-shift) -endif() -if("athena" IN_LIST FEATURES) - list(APPEND BUILD_ONLY athena) -endif() -if("auditmanager" IN_LIST FEATURES) - list(APPEND BUILD_ONLY auditmanager) -endif() -if("autoscaling" IN_LIST FEATURES) - list(APPEND BUILD_ONLY autoscaling) -endif() -if("autoscaling-plans" IN_LIST FEATURES) - list(APPEND BUILD_ONLY autoscaling-plans) -endif() -if("awsmigrationhub" IN_LIST FEATURES) - list(APPEND BUILD_ONLY AWSMigrationHub) -endif() -if("awstransfer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY awstransfer) -endif() -if("backup" IN_LIST FEATURES) - list(APPEND BUILD_ONLY backup) -endif() -if("backup-gateway" IN_LIST FEATURES) - list(APPEND BUILD_ONLY backup-gateway) -endif() -if("backupstorage" IN_LIST FEATURES) - list(APPEND BUILD_ONLY backupstorage) -endif() -if("batch" IN_LIST FEATURES) - list(APPEND BUILD_ONLY batch) -endif() -if("billingconductor" IN_LIST FEATURES) - list(APPEND BUILD_ONLY billingconductor) -endif() -if("braket" IN_LIST FEATURES) - list(APPEND BUILD_ONLY braket) -endif() -if("budgets" IN_LIST FEATURES) - list(APPEND BUILD_ONLY budgets) -endif() -if("ce" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ce) -endif() -if("chime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime) -endif() -if("chime-sdk-identity" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime-sdk-identity) -endif() -if("chime-sdk-media-pipelines" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime-sdk-media-pipelines) -endif() -if("chime-sdk-meetings" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime-sdk-meetings) -endif() -if("chime-sdk-messaging" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime-sdk-messaging) -endif() -if("chime-sdk-voice" IN_LIST FEATURES) - list(APPEND BUILD_ONLY chime-sdk-voice) -endif() -if("cleanrooms" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cleanrooms) -endif() -if("cloud9" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloud9) -endif() -if("cloudcontrol" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudcontrol) -endif() -if("clouddirectory" IN_LIST FEATURES) - list(APPEND BUILD_ONLY clouddirectory) -endif() -if("cloudformation" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudformation) -endif() -if("cloudfront" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudfront) -endif() -if("cloudhsm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudhsm) -endif() -if("cloudhsmv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudhsmv2) -endif() -if("cloudsearch" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudsearch) -endif() -if("cloudsearchdomain" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudsearchdomain) -endif() -if("cloudtrail" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudtrail) -endif() -if("cloudtrail-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cloudtrail-data) -endif() -if("codeartifact" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codeartifact) -endif() -if("codebuild" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codebuild) -endif() -if("codecatalyst" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codecatalyst) -endif() -if("codecommit" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codecommit) -endif() -if("codedeploy" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codedeploy) -endif() -if("codeguru-reviewer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codeguru-reviewer) -endif() -if("codeguru-security" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codeguru-security) -endif() -if("codeguruprofiler" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codeguruprofiler) -endif() -if("codepipeline" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codepipeline) -endif() -if("codestar" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codestar) -endif() -if("codestar-connections" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codestar-connections) -endif() -if("codestar-notifications" IN_LIST FEATURES) - list(APPEND BUILD_ONLY codestar-notifications) -endif() -if("cognito-identity" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cognito-identity) -endif() -if("cognito-idp" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cognito-idp) -endif() -if("cognito-sync" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cognito-sync) -endif() -if("comprehend" IN_LIST FEATURES) - list(APPEND BUILD_ONLY comprehend) -endif() -if("comprehendmedical" IN_LIST FEATURES) - list(APPEND BUILD_ONLY comprehendmedical) -endif() -if("compute-optimizer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY compute-optimizer) -endif() -if("config" IN_LIST FEATURES) - list(APPEND BUILD_ONLY config) -endif() -if("connect" IN_LIST FEATURES) - list(APPEND BUILD_ONLY connect) -endif() -if("connect-contact-lens" IN_LIST FEATURES) - list(APPEND BUILD_ONLY connect-contact-lens) -endif() -if("connectcampaigns" IN_LIST FEATURES) - list(APPEND BUILD_ONLY connectcampaigns) -endif() -if("connectcases" IN_LIST FEATURES) - list(APPEND BUILD_ONLY connectcases) -endif() -if("connectparticipant" IN_LIST FEATURES) - list(APPEND BUILD_ONLY connectparticipant) -endif() -if("controltower" IN_LIST FEATURES) - list(APPEND BUILD_ONLY controltower) -endif() -if("cur" IN_LIST FEATURES) - list(APPEND BUILD_ONLY cur) -endif() -if("customer-profiles" IN_LIST FEATURES) - list(APPEND BUILD_ONLY customer-profiles) -endif() -if("databrew" IN_LIST FEATURES) - list(APPEND BUILD_ONLY databrew) -endif() -if("dataexchange" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dataexchange) -endif() -if("datapipeline" IN_LIST FEATURES) - list(APPEND BUILD_ONLY datapipeline) -endif() -if("datasync" IN_LIST FEATURES) - list(APPEND BUILD_ONLY datasync) -endif() -if("dax" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dax) -endif() -if("detective" IN_LIST FEATURES) - list(APPEND BUILD_ONLY detective) -endif() -if("devicefarm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY devicefarm) -endif() -if("devops-guru" IN_LIST FEATURES) - list(APPEND BUILD_ONLY devops-guru) -endif() -if("directconnect" IN_LIST FEATURES) - list(APPEND BUILD_ONLY directconnect) -endif() -if("discovery" IN_LIST FEATURES) - list(APPEND BUILD_ONLY discovery) -endif() -if("dlm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dlm) -endif() -if("dms" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dms) -endif() -if("docdb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY docdb) -endif() -if("docdb-elastic" IN_LIST FEATURES) - list(APPEND BUILD_ONLY docdb-elastic) -endif() -if("drs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY drs) -endif() -if("ds" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ds) -endif() -if("dynamodb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dynamodb) -endif() -if("dynamodbstreams" IN_LIST FEATURES) - list(APPEND BUILD_ONLY dynamodbstreams) -endif() -if("ebs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ebs) -endif() -if("ec2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ec2) -endif() -if("ec2-instance-connect" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ec2-instance-connect) -endif() -if("ecr" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ecr) -endif() -if("ecr-public" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ecr-public) -endif() -if("ecs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ecs) -endif() -if("eks" IN_LIST FEATURES) - list(APPEND BUILD_ONLY eks) -endif() -if("elastic-inference" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elastic-inference) -endif() -if("elasticache" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticache) -endif() -if("elasticbeanstalk" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticbeanstalk) -endif() -if("elasticfilesystem" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticfilesystem) -endif() -if("elasticloadbalancing" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticloadbalancing) -endif() -if("elasticloadbalancingv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticloadbalancingv2) -endif() -if("elasticmapreduce" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elasticmapreduce) -endif() -if("elastictranscoder" IN_LIST FEATURES) - list(APPEND BUILD_ONLY elastictranscoder) -endif() -if("email" IN_LIST FEATURES) - list(APPEND BUILD_ONLY email) -endif() -if("emr-containers" IN_LIST FEATURES) - list(APPEND BUILD_ONLY emr-containers) -endif() -if("emr-serverless" IN_LIST FEATURES) - list(APPEND BUILD_ONLY emr-serverless) -endif() -if("entityresolution" IN_LIST FEATURES) - list(APPEND BUILD_ONLY entityresolution) -endif() -if("es" IN_LIST FEATURES) - list(APPEND BUILD_ONLY es) -endif() -if("eventbridge" IN_LIST FEATURES) - list(APPEND BUILD_ONLY eventbridge) -endif() -if("events" IN_LIST FEATURES) - list(APPEND BUILD_ONLY events) -endif() -if("evidently" IN_LIST FEATURES) - list(APPEND BUILD_ONLY evidently) -endif() -if("finspace" IN_LIST FEATURES) - list(APPEND BUILD_ONLY finspace) -endif() -if("finspace-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY finspace-data) -endif() -if("firehose" IN_LIST FEATURES) - list(APPEND BUILD_ONLY firehose) -endif() -if("fis" IN_LIST FEATURES) - list(APPEND BUILD_ONLY fis) -endif() -if("fms" IN_LIST FEATURES) - list(APPEND BUILD_ONLY fms) -endif() -if("forecast" IN_LIST FEATURES) - list(APPEND BUILD_ONLY forecast) -endif() -if("forecastquery" IN_LIST FEATURES) - list(APPEND BUILD_ONLY forecastquery) -endif() -if("frauddetector" IN_LIST FEATURES) - list(APPEND BUILD_ONLY frauddetector) -endif() -if("fsx" IN_LIST FEATURES) - list(APPEND BUILD_ONLY fsx) -endif() -if("gamelift" IN_LIST FEATURES) - list(APPEND BUILD_ONLY gamelift) -endif() -if("gamesparks" IN_LIST FEATURES) - list(APPEND BUILD_ONLY gamesparks) -endif() -if("glacier" IN_LIST FEATURES) - list(APPEND BUILD_ONLY glacier) -endif() -if("globalaccelerator" IN_LIST FEATURES) - list(APPEND BUILD_ONLY globalaccelerator) -endif() -if("glue" IN_LIST FEATURES) - list(APPEND BUILD_ONLY glue) -endif() -if("grafana" IN_LIST FEATURES) - list(APPEND BUILD_ONLY grafana) -endif() -if("greengrass" IN_LIST FEATURES) - list(APPEND BUILD_ONLY greengrass) -endif() -if("greengrassv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY greengrassv2) -endif() -if("groundstation" IN_LIST FEATURES) - list(APPEND BUILD_ONLY groundstation) -endif() -if("guardduty" IN_LIST FEATURES) - list(APPEND BUILD_ONLY guardduty) -endif() -if("health" IN_LIST FEATURES) - list(APPEND BUILD_ONLY health) -endif() -if("healthlake" IN_LIST FEATURES) - list(APPEND BUILD_ONLY healthlake) -endif() -if("honeycode" IN_LIST FEATURES) - list(APPEND BUILD_ONLY honeycode) -endif() -if("iam" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iam) -endif() -if("identity-management" IN_LIST FEATURES) - list(APPEND BUILD_ONLY identity-management) -endif() -if("identitystore" IN_LIST FEATURES) - list(APPEND BUILD_ONLY identitystore) -endif() -if("imagebuilder" IN_LIST FEATURES) - list(APPEND BUILD_ONLY imagebuilder) -endif() -if("importexport" IN_LIST FEATURES) - list(APPEND BUILD_ONLY importexport) -endif() -if("inspector" IN_LIST FEATURES) - list(APPEND BUILD_ONLY inspector) -endif() -if("inspector2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY inspector2) -endif() -if("internetmonitor" IN_LIST FEATURES) - list(APPEND BUILD_ONLY internetmonitor) -endif() -if("iot" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot) -endif() -if("iot-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot-data) -endif() -if("iot-jobs-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot-jobs-data) -endif() -if("iot-roborunner" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot-roborunner) -endif() -if("iot1click-devices" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot1click-devices) -endif() -if("iot1click-projects" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iot1click-projects) -endif() -if("iotanalytics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotanalytics) -endif() -if("iotdeviceadvisor" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotdeviceadvisor) -endif() -if("iotevents" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotevents) -endif() -if("iotevents-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotevents-data) -endif() -if("iotfleethub" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotfleethub) -endif() -if("iotfleetwise" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotfleetwise) -endif() -if("iotsecuretunneling" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotsecuretunneling) -endif() -if("iotsitewise" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotsitewise) -endif() -if("iotthingsgraph" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotthingsgraph) -endif() -if("iottwinmaker" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iottwinmaker) -endif() -if("iotwireless" IN_LIST FEATURES) - list(APPEND BUILD_ONLY iotwireless) -endif() -if("ivs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ivs) -endif() -if("ivs-realtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ivs-realtime) -endif() -if("ivschat" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ivschat) -endif() -if("kafka" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kafka) -endif() -if("kafkaconnect" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kafkaconnect) -endif() -if("kendra" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kendra) -endif() -if("kendra-ranking" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kendra-ranking) -endif() -if("keyspaces" IN_LIST FEATURES) - list(APPEND BUILD_ONLY keyspaces) -endif() -if("kinesis" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesis) -endif() -if("kinesis-video-archived-media" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesis-video-archived-media) -endif() -if("kinesis-video-media" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesis-video-media) -endif() -if("kinesis-video-signaling" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesis-video-signaling) -endif() -if("kinesis-video-webrtc-storage" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesis-video-webrtc-storage) -endif() -if("kinesisanalytics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesisanalytics) -endif() -if("kinesisanalyticsv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesisanalyticsv2) -endif() -if("kinesisvideo" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kinesisvideo) -endif() -if("kms" IN_LIST FEATURES) - list(APPEND BUILD_ONLY kms) -endif() -if("lakeformation" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lakeformation) -endif() -if("lambda" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lambda) -endif() -if("lex" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lex) -endif() -if("lex-models" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lex-models) -endif() -if("lexv2-models" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lexv2-models) -endif() -if("lexv2-runtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lexv2-runtime) -endif() -if("license-manager" IN_LIST FEATURES) - list(APPEND BUILD_ONLY license-manager) -endif() -if("license-manager-linux-subscriptions" IN_LIST FEATURES) - list(APPEND BUILD_ONLY license-manager-linux-subscriptions) -endif() -if("license-manager-user-subscriptions" IN_LIST FEATURES) - list(APPEND BUILD_ONLY license-manager-user-subscriptions) -endif() -if("lightsail" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lightsail) -endif() -if("location" IN_LIST FEATURES) - list(APPEND BUILD_ONLY location) -endif() -if("logs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY logs) -endif() -if("lookoutequipment" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lookoutequipment) -endif() -if("lookoutmetrics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lookoutmetrics) -endif() -if("lookoutvision" IN_LIST FEATURES) - list(APPEND BUILD_ONLY lookoutvision) -endif() -if("m2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY m2) -endif() -if("machinelearning" IN_LIST FEATURES) - list(APPEND BUILD_ONLY machinelearning) -endif() -if("macie" IN_LIST FEATURES) - list(APPEND BUILD_ONLY macie) -endif() -if("macie2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY macie2) -endif() -if("managedblockchain" IN_LIST FEATURES) - list(APPEND BUILD_ONLY managedblockchain) -endif() -if("managedblockchain-query" IN_LIST FEATURES) - list(APPEND BUILD_ONLY managedblockchain-query) -endif() -if("marketplace-catalog" IN_LIST FEATURES) - list(APPEND BUILD_ONLY marketplace-catalog) -endif() -if("marketplace-entitlement" IN_LIST FEATURES) - list(APPEND BUILD_ONLY marketplace-entitlement) -endif() -if("marketplacecommerceanalytics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY marketplacecommerceanalytics) -endif() -if("mediaconnect" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediaconnect) -endif() -if("mediaconvert" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediaconvert) -endif() -if("medialive" IN_LIST FEATURES) - list(APPEND BUILD_ONLY medialive) -endif() -if("mediapackage" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediapackage) -endif() -if("mediapackage-vod" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediapackage-vod) -endif() -if("mediapackagev2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediapackagev2) -endif() -if("mediastore" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediastore) -endif() -if("mediastore-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediastore-data) -endif() -if("mediatailor" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mediatailor) -endif() -if("medical-imaging" IN_LIST FEATURES) - list(APPEND BUILD_ONLY medical-imaging) -endif() -if("memorydb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY memorydb) -endif() -if("meteringmarketplace" IN_LIST FEATURES) - list(APPEND BUILD_ONLY meteringmarketplace) -endif() -if("mgn" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mgn) -endif() -if("migration-hub-refactor-spaces" IN_LIST FEATURES) - list(APPEND BUILD_ONLY migration-hub-refactor-spaces) -endif() -if("migrationhub-config" IN_LIST FEATURES) - list(APPEND BUILD_ONLY migrationhub-config) -endif() -if("migrationhuborchestrator" IN_LIST FEATURES) - list(APPEND BUILD_ONLY migrationhuborchestrator) -endif() -if("migrationhubstrategy" IN_LIST FEATURES) - list(APPEND BUILD_ONLY migrationhubstrategy) -endif() -if("mobile" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mobile) -endif() -if("monitoring" IN_LIST FEATURES) - list(APPEND BUILD_ONLY monitoring) -endif() -if("mq" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mq) -endif() -if("mturk-requester" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mturk-requester) -endif() -if("mwaa" IN_LIST FEATURES) - list(APPEND BUILD_ONLY mwaa) -endif() -if("neptune" IN_LIST FEATURES) - list(APPEND BUILD_ONLY neptune) -endif() -if("neptunedata" IN_LIST FEATURES) - list(APPEND BUILD_ONLY neptunedata) -endif() -if("network-firewall" IN_LIST FEATURES) - list(APPEND BUILD_ONLY network-firewall) -endif() -if("networkmanager" IN_LIST FEATURES) - list(APPEND BUILD_ONLY networkmanager) -endif() -if("nimble" IN_LIST FEATURES) - list(APPEND BUILD_ONLY nimble) -endif() -if("oam" IN_LIST FEATURES) - list(APPEND BUILD_ONLY oam) -endif() -if("omics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY omics) -endif() -if("opensearch" IN_LIST FEATURES) - list(APPEND BUILD_ONLY opensearch) -endif() -if("opensearchserverless" IN_LIST FEATURES) - list(APPEND BUILD_ONLY opensearchserverless) -endif() -if("opsworks" IN_LIST FEATURES) - list(APPEND BUILD_ONLY opsworks) -endif() -if("opsworkscm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY opsworkscm) -endif() -if("organizations" IN_LIST FEATURES) - list(APPEND BUILD_ONLY organizations) -endif() -if("osis" IN_LIST FEATURES) - list(APPEND BUILD_ONLY osis) -endif() -if("outposts" IN_LIST FEATURES) - list(APPEND BUILD_ONLY outposts) -endif() -if("panorama" IN_LIST FEATURES) - list(APPEND BUILD_ONLY panorama) -endif() -if("payment-cryptography" IN_LIST FEATURES) - list(APPEND BUILD_ONLY payment-cryptography) -endif() -if("payment-cryptography-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY payment-cryptography-data) -endif() -if("pca-connector-ad" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pca-connector-ad) -endif() -if("personalize" IN_LIST FEATURES) - list(APPEND BUILD_ONLY personalize) -endif() -if("personalize-events" IN_LIST FEATURES) - list(APPEND BUILD_ONLY personalize-events) -endif() -if("personalize-runtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY personalize-runtime) -endif() -if("pi" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pi) -endif() -if("pinpoint" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pinpoint) -endif() -if("pinpoint-email" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pinpoint-email) -endif() -if("pinpoint-sms-voice-v2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pinpoint-sms-voice-v2) -endif() -if("pipes" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pipes) -endif() -if("polly" IN_LIST FEATURES) - list(APPEND BUILD_ONLY polly) -endif() -if("pricing" IN_LIST FEATURES) - list(APPEND BUILD_ONLY pricing) -endif() -if("privatenetworks" IN_LIST FEATURES) - list(APPEND BUILD_ONLY privatenetworks) -endif() -if("proton" IN_LIST FEATURES) - list(APPEND BUILD_ONLY proton) -endif() -if("qldb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY qldb) -endif() -if("qldb-session" IN_LIST FEATURES) - list(APPEND BUILD_ONLY qldb-session) -endif() -if("queues" IN_LIST FEATURES) - list(APPEND BUILD_ONLY queues) -endif() -if("quicksight" IN_LIST FEATURES) - list(APPEND BUILD_ONLY quicksight) -endif() -if("ram" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ram) -endif() -if("rbin" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rbin) -endif() -if("rds" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rds) -endif() -if("rds-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rds-data) -endif() -if("redshift" IN_LIST FEATURES) - list(APPEND BUILD_ONLY redshift) -endif() -if("redshift-data" IN_LIST FEATURES) - list(APPEND BUILD_ONLY redshift-data) -endif() -if("redshift-serverless" IN_LIST FEATURES) - list(APPEND BUILD_ONLY redshift-serverless) -endif() -if("rekognition" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rekognition) -endif() -if("resiliencehub" IN_LIST FEATURES) - list(APPEND BUILD_ONLY resiliencehub) -endif() -if("resource-explorer-2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY resource-explorer-2) -endif() -if("resource-groups" IN_LIST FEATURES) - list(APPEND BUILD_ONLY resource-groups) -endif() -if("resourcegroupstaggingapi" IN_LIST FEATURES) - list(APPEND BUILD_ONLY resourcegroupstaggingapi) -endif() -if("robomaker" IN_LIST FEATURES) - list(APPEND BUILD_ONLY robomaker) -endif() -if("rolesanywhere" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rolesanywhere) -endif() -if("route53" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53) -endif() -if("route53-recovery-cluster" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53-recovery-cluster) -endif() -if("route53-recovery-control-config" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53-recovery-control-config) -endif() -if("route53-recovery-readiness" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53-recovery-readiness) -endif() -if("route53domains" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53domains) -endif() -if("route53resolver" IN_LIST FEATURES) - list(APPEND BUILD_ONLY route53resolver) -endif() -if("rum" IN_LIST FEATURES) - list(APPEND BUILD_ONLY rum) -endif() -if("s3" IN_LIST FEATURES) - list(APPEND BUILD_ONLY s3) -endif() -if("s3-crt" IN_LIST FEATURES) - list(APPEND BUILD_ONLY s3-crt) -endif() -if("s3-encryption" IN_LIST FEATURES) - list(APPEND BUILD_ONLY s3-encryption) -endif() -if("s3control" IN_LIST FEATURES) - list(APPEND BUILD_ONLY s3control) -endif() -if("s3outposts" IN_LIST FEATURES) - list(APPEND BUILD_ONLY s3outposts) -endif() -if("sagemaker" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker) -endif() -if("sagemaker-a2i-runtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-a2i-runtime) -endif() -if("sagemaker-edge" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-edge) -endif() -if("sagemaker-featurestore-runtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-featurestore-runtime) -endif() -if("sagemaker-geospatial" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-geospatial) -endif() -if("sagemaker-metrics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-metrics) -endif() -if("sagemaker-runtime" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sagemaker-runtime) -endif() -if("savingsplans" IN_LIST FEATURES) - list(APPEND BUILD_ONLY savingsplans) -endif() -if("scheduler" IN_LIST FEATURES) - list(APPEND BUILD_ONLY scheduler) -endif() -if("schemas" IN_LIST FEATURES) - list(APPEND BUILD_ONLY schemas) -endif() -if("sdb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sdb) -endif() -if("secretsmanager" IN_LIST FEATURES) - list(APPEND BUILD_ONLY secretsmanager) -endif() -if("securityhub" IN_LIST FEATURES) - list(APPEND BUILD_ONLY securityhub) -endif() -if("securitylake" IN_LIST FEATURES) - list(APPEND BUILD_ONLY securitylake) -endif() -if("serverlessrepo" IN_LIST FEATURES) - list(APPEND BUILD_ONLY serverlessrepo) -endif() -if("service-quotas" IN_LIST FEATURES) - list(APPEND BUILD_ONLY service-quotas) -endif() -if("servicecatalog" IN_LIST FEATURES) - list(APPEND BUILD_ONLY servicecatalog) -endif() -if("servicecatalog-appregistry" IN_LIST FEATURES) - list(APPEND BUILD_ONLY servicecatalog-appregistry) -endif() -if("servicediscovery" IN_LIST FEATURES) - list(APPEND BUILD_ONLY servicediscovery) -endif() -if("sesv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sesv2) -endif() -if("shield" IN_LIST FEATURES) - list(APPEND BUILD_ONLY shield) -endif() -if("signer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY signer) -endif() -if("simspaceweaver" IN_LIST FEATURES) - list(APPEND BUILD_ONLY simspaceweaver) -endif() -if("sms" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sms) -endif() -if("sms-voice" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sms-voice) -endif() -if("snow-device-management" IN_LIST FEATURES) - list(APPEND BUILD_ONLY snow-device-management) -endif() -if("snowball" IN_LIST FEATURES) - list(APPEND BUILD_ONLY snowball) -endif() -if("sns" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sns) -endif() -if("sqs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sqs) -endif() -if("ssm" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ssm) -endif() -if("ssm-contacts" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ssm-contacts) -endif() -if("ssm-incidents" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ssm-incidents) -endif() -if("ssm-sap" IN_LIST FEATURES) - list(APPEND BUILD_ONLY ssm-sap) -endif() -if("sso" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sso) -endif() -if("sso-admin" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sso-admin) -endif() -if("sso-oidc" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sso-oidc) -endif() -if("states" IN_LIST FEATURES) - list(APPEND BUILD_ONLY states) -endif() -if("storagegateway" IN_LIST FEATURES) - list(APPEND BUILD_ONLY storagegateway) -endif() -if("sts" IN_LIST FEATURES) - list(APPEND BUILD_ONLY sts) -endif() -if("support" IN_LIST FEATURES) - list(APPEND BUILD_ONLY support) -endif() -if("support-app" IN_LIST FEATURES) - list(APPEND BUILD_ONLY support-app) -endif() -if("swf" IN_LIST FEATURES) - list(APPEND BUILD_ONLY swf) -endif() -if("synthetics" IN_LIST FEATURES) - list(APPEND BUILD_ONLY synthetics) -endif() -if("text-to-speech" IN_LIST FEATURES) - list(APPEND BUILD_ONLY text-to-speech) -endif() -if("textract" IN_LIST FEATURES) - list(APPEND BUILD_ONLY textract) -endif() -if("timestream-query" IN_LIST FEATURES) - list(APPEND BUILD_ONLY timestream-query) -endif() -if("timestream-write" IN_LIST FEATURES) - list(APPEND BUILD_ONLY timestream-write) -endif() -if("tnb" IN_LIST FEATURES) - list(APPEND BUILD_ONLY tnb) -endif() -if("transcribe" IN_LIST FEATURES) - list(APPEND BUILD_ONLY transcribe) -endif() -if("transcribestreaming" IN_LIST FEATURES) - list(APPEND BUILD_ONLY transcribestreaming) -endif() -if("transfer" IN_LIST FEATURES) - list(APPEND BUILD_ONLY transfer) -endif() -if("translate" IN_LIST FEATURES) - list(APPEND BUILD_ONLY translate) -endif() -if("verifiedpermissions" IN_LIST FEATURES) - list(APPEND BUILD_ONLY verifiedpermissions) -endif() -if("voice-id" IN_LIST FEATURES) - list(APPEND BUILD_ONLY voice-id) -endif() -if("vpc-lattice" IN_LIST FEATURES) - list(APPEND BUILD_ONLY vpc-lattice) -endif() -if("waf" IN_LIST FEATURES) - list(APPEND BUILD_ONLY waf) -endif() -if("waf-regional" IN_LIST FEATURES) - list(APPEND BUILD_ONLY waf-regional) -endif() -if("wafv2" IN_LIST FEATURES) - list(APPEND BUILD_ONLY wafv2) -endif() -if("wellarchitected" IN_LIST FEATURES) - list(APPEND BUILD_ONLY wellarchitected) -endif() -if("wisdom" IN_LIST FEATURES) - list(APPEND BUILD_ONLY wisdom) -endif() -if("workdocs" IN_LIST FEATURES) - list(APPEND BUILD_ONLY workdocs) -endif() -if("worklink" IN_LIST FEATURES) - list(APPEND BUILD_ONLY worklink) -endif() -if("workmail" IN_LIST FEATURES) - list(APPEND BUILD_ONLY workmail) -endif() -if("workmailmessageflow" IN_LIST FEATURES) - list(APPEND BUILD_ONLY workmailmessageflow) -endif() -if("workspaces" IN_LIST FEATURES) - list(APPEND BUILD_ONLY workspaces) -endif() -if("workspaces-web" IN_LIST FEATURES) - list(APPEND BUILD_ONLY workspaces-web) -endif() -if("xray" IN_LIST FEATURES) - list(APPEND BUILD_ONLY xray) -endif() diff --git a/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch b/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch new file mode 100644 index 000000000000..c2d98030136e --- /dev/null +++ b/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch @@ -0,0 +1,10 @@ +diff --git a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt +index a8a888d..574b985 100644 +--- a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt ++++ b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt +@@ -1,4 +1,4 @@ +-add_project(aws-cpp-sdk-AWSMigrationHub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) ++add_project(aws-cpp-sdk-awsmigrationhub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) + + file(GLOB AWS_AWSMIGRATIONHUB_HEADERS + "include/aws/AWSMigrationHub/*.h" diff --git a/ports/aws-sdk-cpp/generateFeatures.ps1 b/ports/aws-sdk-cpp/generateFeatures.ps1 index f10ee40e7a2b..d0f9a5aba722 100644 --- a/ports/aws-sdk-cpp/generateFeatures.ps1 +++ b/ports/aws-sdk-cpp/generateFeatures.ps1 @@ -1,85 +1,73 @@ -[CmdletBinding()] -param( - [Parameter(Mandatory=$true)][string]$SourcesRef, - [Parameter(Mandatory=$false)][string]$PortDirectory = $PSScriptRoot, - [Parameter(Mandatory=$false)][string]$vcpkg = "$PSScriptRoot/../../vcpkg" -) - -$ErrorActionPreference = "Stop" - -$ManifestIn = "$PortDirectory/vcpkg.in.json" -$ManifestOut = "$PortDirectory/vcpkg.json" -$CMakeFragmentFile = "$PortDirectory/compute_build_only.cmake" - -$ExtractedSources = "${env:TEMP}/aws-sdk-cpp-generateFeatures-$SourcesRef" -if (-not (Test-Path $ExtractedSources)) { - if (Test-Path "$ExtractedSources.tmp") { - Remove-Item -Force "$ExtractedSources.tmp" - } - git clone "https://github.com/aws/aws-sdk-cpp" "$ExtractedSources.tmp" | Out-Host - git -c "$ExtractedSources.tmp" checkout $SourcesRef - Move-Item "$ExtractedSources.tmp" "$ExtractedSources" -} -Write-Host "Using sources directory: $ExtractedSources" - - -$subfolders = Get-ChildItem -Path "$ExtractedSources\generated\src\aws-cpp-sdk-*", "$ExtractedSources\src\aws-cpp-sdk*" | Sort-Object -Property Name - -$manifest = Get-Content $ManifestIn | ConvertFrom-Json -$manifest | Add-Member ` - -NotePropertyName '$note' ` - -NotePropertyValue 'Automatically generated by generateFeatures.ps1' -$manifest | Add-Member -NotePropertyName 'features' -NotePropertyValue @{} - -$cmakefragmenttext = @("# Automatically generated by generateFeatures.ps1") - -function GetDescription($dir, $modulename) -{ - if (Test-Path "$dir\CMakeLists.txt") - { - $descs = @(Select-String -Path "$dir\CMakeLists.txt" -Pattern "`"C\+\+ SDK for the AWS [^`"]*`"") - if ($descs.count -eq 1) { - $desc = $descs[0].Matches.Value -replace "`"","" - "$desc" - } - else { "C++ SDK for the AWS $modulename service" } - } - else { "C++ SDK for the AWS $modulename service" } -} - -$featureDependencies = @{} -Select-String -Path "$ExtractedSources\cmake\sdksCommon.cmake" -Pattern "list\(APPEND SDK_DEPENDENCY_LIST `"([\w-]+):([\w-,]+)`"\)" -AllMatches ` -| ForEach-Object { $_.Matches } ` -| ForEach-Object { $featureDependencies[$_.Groups[1].Value] = @($_.Groups[2].Value -split "," ` -| Where-Object { $_ -ne "core" }) } - -foreach ($subfolder in $subfolders) -{ - $modulename = $subfolder.name -replace "^aws-cpp-sdk-","" - if ($modulename -match "-tests`$") { continue } - if ($modulename -match "-sample`$") { continue } - if ($modulename -eq "core") { continue } - - $lowermodulename = $modulename.ToLower() - - $featureObj = @{ description = (GetDescription $subfolder $modulename) } - - if ($featureDependencies.ContainsKey($lowermodulename)) { - $featureObj.dependencies = ,@{ name = "aws-sdk-cpp"; "default-features" = $false; "features" = $featureDependencies[$lowermodulename] } - } - - $manifest.features.Add("$lowermodulename", $featureObj) - - $cmakefragmenttext += @( - "if(`"$lowermodulename`" IN_LIST FEATURES)", - " list(APPEND BUILD_ONLY $modulename)", - "endif()" - ) -} - -[IO.File]::WriteAllText($ManifestOut, (ConvertTo-Json -Depth 10 -InputObject $manifest)) - -Write-Verbose ($cmakefragmenttext -join "`n") -[IO.File]::WriteAllText($CMakeFragmentFile, ($cmakefragmenttext -join "`n") +"`n") - -& $vcpkg format-manifest --feature-flags=-manifests $ManifestOut +[CmdletBinding()] +param( + [Parameter(Mandatory=$true)][string]$SourcesRef, + [Parameter(Mandatory=$false)][string]$PortDirectory = $PSScriptRoot, + [Parameter(Mandatory=$false)][string]$vcpkg = "$PSScriptRoot/../../vcpkg" +) + +$ErrorActionPreference = "Stop" + +$ManifestIn = "$PortDirectory/vcpkg.in.json" +$ManifestOut = "$PortDirectory/vcpkg.json" + +$ExtractedSources = "${env:TEMP}/aws-sdk-cpp-generateFeatures-$SourcesRef" +if (-not (Test-Path $ExtractedSources)) { + if (Test-Path "$ExtractedSources.tmp") { + Remove-Item -Force "$ExtractedSources.tmp" + } + git clone "https://github.com/aws/aws-sdk-cpp" "$ExtractedSources.tmp" | Out-Host + git -c "$ExtractedSources.tmp" checkout $SourcesRef + Move-Item "$ExtractedSources.tmp" "$ExtractedSources" +} +Write-Host "Using sources directory: $ExtractedSources" + + +$subfolders = Get-ChildItem -Path "$ExtractedSources\generated\src\aws-cpp-sdk-*", "$ExtractedSources\src\aws-cpp-sdk*" | Sort-Object -Property Name + +$manifest = Get-Content $ManifestIn | ConvertFrom-Json +$manifest | Add-Member ` + -NotePropertyName '$note' ` + -NotePropertyValue 'Automatically generated by generateFeatures.ps1' +$manifest | Add-Member -NotePropertyName 'features' -NotePropertyValue @{} + +function GetDescription($dir, $modulename) +{ + if (Test-Path "$dir\CMakeLists.txt") + { + $descs = @(Select-String -Path "$dir\CMakeLists.txt" -Pattern "`"C\+\+ SDK for the AWS [^`"]*`"") + if ($descs.count -eq 1) { + $desc = $descs[0].Matches.Value -replace "`"","" + "$desc" + } + else { "C++ SDK for the AWS $modulename service" } + } + else { "C++ SDK for the AWS $modulename service" } +} + +$featureDependencies = @{} +Select-String -Path "$ExtractedSources\cmake\sdksCommon.cmake" -Pattern "list\(APPEND SDK_DEPENDENCY_LIST `"([\w-]+):([\w-,]+)`"\)" -AllMatches ` +| ForEach-Object { $_.Matches } ` +| ForEach-Object { $featureDependencies[$_.Groups[1].Value] = @($_.Groups[2].Value -split "," ` +| Where-Object { $_ -ne "core" }) } + +foreach ($subfolder in $subfolders) +{ + $modulename = $subfolder.name -replace "^aws-cpp-sdk-","" + if ($modulename -match "-tests`$") { continue } + if ($modulename -match "-sample`$") { continue } + if ($modulename -eq "core") { continue } + + $lowermodulename = $modulename.ToLower() + + $featureObj = @{ description = (GetDescription $subfolder $modulename) } + + if ($featureDependencies.ContainsKey($lowermodulename)) { + $featureObj.dependencies = ,@{ name = "aws-sdk-cpp"; "default-features" = $false; "features" = $featureDependencies[$lowermodulename] } + } + + $manifest.features.Add("$lowermodulename", $featureObj) +} + +[IO.File]::WriteAllText($ManifestOut, (ConvertTo-Json -Depth 10 -InputObject $manifest)) + +& $vcpkg format-manifest --feature-flags=-manifests $ManifestOut diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake index f8a5ef097143..1dc66c5f4b3c 100644 --- a/ports/aws-sdk-cpp/portfile.cmake +++ b/ports/aws-sdk-cpp/portfile.cmake @@ -1,102 +1,100 @@ -vcpkg_buildpath_length_warning(37) - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO aws/aws-sdk-cpp - REF "${VERSION}" - SHA512 b1a07939dd40f635fdc5b5947c3d679e6a2482b5017a3b26801639785fa1cb3e88414dd216fe64d3fb984d812ff3e8c4103e9b4355d531e533b78f1fa2a7cb01 - PATCHES - patch-relocatable-rpath.patch - fix-aws-root.patch - lock-curl-http-and-tls-settings.patch - remove-checked-array-iterator.patch -) - -string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) - -set(EXTRA_ARGS) -if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - set(rpath "@loader_path") -elseif (VCPKG_TARGET_IS_ANDROID) - set(EXTRA_ARGS "-DTARGET_ARCH=ANDROID" - "-DGIT_EXECUTABLE=--invalid-git-executable--" - "-DGIT_FOUND=TRUE" - "-DNDK_DIR=$ENV{ANDROID_NDK_HOME}" - "-DANDROID_BUILD_ZLIB=FALSE" - "-DANDROID_BUILD_CURL=FALSE" - "-DANDROID_BUILD_OPENSSL=FALSE" - ) -else() - set(rpath "\$ORIGIN") -endif() - -set(BUILD_ONLY core) -include(${CMAKE_CURRENT_LIST_DIR}/compute_build_only.cmake) -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - ${EXTRA_ARGS} - "-DENABLE_UNITY_BUILD=ON" - "-DENABLE_TESTING=OFF" - "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" - "-DBUILD_ONLY=${BUILD_ONLY}" - "-DBUILD_DEPS=OFF" - "-DBUILD_SHARED_LIBS=OFF" - "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" - "-DCMAKE_INSTALL_RPATH=${rpath}" - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files -) -vcpkg_cmake_install() - -foreach(TARGET IN LISTS BUILD_ONLY) - vcpkg_cmake_config_fixup(PACKAGE_NAME "aws-cpp-sdk-${TARGET}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() -vcpkg_cmake_config_fixup(PACKAGE_NAME "AWSSDK" CONFIG_PATH "lib/cmake/AWSSDK") - -vcpkg_copy_pdbs() - -file(GLOB_RECURSE AWS_TARGETS "${CURRENT_PACKAGES_DIR}/share/*/*-targets-*.cmake") -foreach(AWS_TARGET IN LISTS AWS_TARGETS) - file(READ ${AWS_TARGET} _contents) - string(REGEX REPLACE - "bin\\/([A-Za-z0-9_.-]+\\.lib)" - "lib/\\1" - _contents "${_contents}") - file(WRITE ${AWS_TARGET} "${_contents}") -endforeach() - -file(GLOB AWS_CONFIGS "${CURRENT_PACKAGES_DIR}/share/*/aws-cpp-sdk-*-config.cmake") -list(FILTER AWS_CONFIGS EXCLUDE REGEX "aws-cpp-sdk-core-config\\.cmake\$") -foreach(AWS_CONFIG IN LISTS AWS_CONFIGS) - file(READ "${AWS_CONFIG}" _contents) - file(WRITE "${AWS_CONFIG}" "include(CMakeFindDependencyMacro)\nfind_dependency(aws-cpp-sdk-core)\n${_contents}") -endforeach() - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/pkgconfig" - "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig" - "${CURRENT_PACKAGES_DIR}/nuget" - "${CURRENT_PACKAGES_DIR}/debug/nuget" -) - -if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) - file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) - if(LIB_FILES) - file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(REMOVE ${LIB_FILES}) - endif() - file(GLOB DEBUG_LIB_FILES ${CURRENT_PACKAGES_DIR}/debug/bin/*.lib) - if(DEBUG_LIB_FILES) - file(COPY ${DEBUG_LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) - file(REMOVE ${DEBUG_LIB_FILES}) - endif() - - file(APPEND "${CURRENT_PACKAGES_DIR}/include/aws/core/SDKConfig.h" "#ifndef USE_IMPORT_EXPORT\n#define USE_IMPORT_EXPORT\n#endif") -endif() - -configure_file("${CURRENT_PORT_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" @ONLY) - -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) +vcpkg_buildpath_length_warning(37) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO aws/aws-sdk-cpp + REF "${VERSION}" + SHA512 4410eaff815ce7b6c0bc0d37bd4175754d5103d2e3cfd60755df57dad103ab7e7705b79fc6039d2c8b7d1ccec650912f5ff0aa73baa2d9cf6d6608a493d11088 + PATCHES + patch-relocatable-rpath.patch + fix-aws-root.patch + lock-curl-http-and-tls-settings.patch + fix-awsmigrationhub-build.patch +) + +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) + +set(EXTRA_ARGS) +if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + set(rpath "@loader_path") +elseif (VCPKG_TARGET_IS_ANDROID) + set(EXTRA_ARGS "-DTARGET_ARCH=ANDROID" + "-DGIT_EXECUTABLE=--invalid-git-executable--" + "-DGIT_FOUND=TRUE" + "-DNDK_DIR=$ENV{ANDROID_NDK_HOME}" + "-DANDROID_BUILD_ZLIB=FALSE" + "-DANDROID_BUILD_CURL=FALSE" + "-DANDROID_BUILD_OPENSSL=FALSE" + ) +else() + set(rpath "\$ORIGIN") +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + ${EXTRA_ARGS} + "-DENABLE_UNITY_BUILD=ON" + "-DENABLE_TESTING=OFF" + "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" + "-DBUILD_ONLY=${FEATURES}" + "-DBUILD_DEPS=OFF" + "-DBUILD_SHARED_LIBS=OFF" + "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" + "-DCMAKE_INSTALL_RPATH=${rpath}" + "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files +) +vcpkg_cmake_install() + +foreach(TARGET IN LISTS FEATURES) + vcpkg_cmake_config_fixup(PACKAGE_NAME "aws-cpp-sdk-${TARGET}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) +endforeach() +vcpkg_cmake_config_fixup(PACKAGE_NAME "AWSSDK" CONFIG_PATH "lib/cmake/AWSSDK") + +vcpkg_copy_pdbs() + +file(GLOB_RECURSE AWS_TARGETS "${CURRENT_PACKAGES_DIR}/share/*/*-targets-*.cmake") +foreach(AWS_TARGET IN LISTS AWS_TARGETS) + file(READ ${AWS_TARGET} _contents) + string(REGEX REPLACE + "bin\\/([A-Za-z0-9_.-]+\\.lib)" + "lib/\\1" + _contents "${_contents}") + file(WRITE ${AWS_TARGET} "${_contents}") +endforeach() + +file(GLOB AWS_CONFIGS "${CURRENT_PACKAGES_DIR}/share/*/aws-cpp-sdk-*-config.cmake") +list(FILTER AWS_CONFIGS EXCLUDE REGEX "aws-cpp-sdk-core-config\\.cmake\$") +foreach(AWS_CONFIG IN LISTS AWS_CONFIGS) + file(READ "${AWS_CONFIG}" _contents) + file(WRITE "${AWS_CONFIG}" "include(CMakeFindDependencyMacro)\nfind_dependency(aws-cpp-sdk-core)\n${_contents}") +endforeach() + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig" + "${CURRENT_PACKAGES_DIR}/nuget" + "${CURRENT_PACKAGES_DIR}/debug/nuget" +) + +if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) + file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) + if(LIB_FILES) + file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(REMOVE ${LIB_FILES}) + endif() + file(GLOB DEBUG_LIB_FILES ${CURRENT_PACKAGES_DIR}/debug/bin/*.lib) + if(DEBUG_LIB_FILES) + file(COPY ${DEBUG_LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + file(REMOVE ${DEBUG_LIB_FILES}) + endif() + + file(APPEND "${CURRENT_PACKAGES_DIR}/include/aws/core/SDKConfig.h" "#ifndef USE_IMPORT_EXPORT\n#define USE_IMPORT_EXPORT\n#endif") +endif() + +configure_file("${CURRENT_PORT_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" @ONLY) + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-sdk-cpp/remove-checked-array-iterator.patch b/ports/aws-sdk-cpp/remove-checked-array-iterator.patch deleted file mode 100644 index 2e69672335fd..000000000000 --- a/ports/aws-sdk-cpp/remove-checked-array-iterator.patch +++ /dev/null @@ -1,51 +0,0 @@ -diff --git "a/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" "b/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" ---- "a/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" -+++ "b/src/aws-cpp-sdk-core/include/aws/core/utils/Array.h" -@@ -54,11 +54,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -- std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); --#else - std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); --#endif // MSVC - } - } - -@@ -82,11 +78,7 @@ namespace Aws - if(arr->m_size > 0 && arr->m_data) - { - size_t arraySize = arr->m_size; --#ifdef _WIN32 -- std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); --#else - std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); --#endif // MSVC - location += arraySize; - } - } -@@ -101,11 +93,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -- std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); --#else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); --#endif // MSVC - } - } - -@@ -134,11 +122,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -- std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); --#else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); --#endif // MSVC - } - - return *this; diff --git a/ports/aws-sdk-cpp/usage b/ports/aws-sdk-cpp/usage index c526d2c45612..37e1f617b2cb 100644 --- a/ports/aws-sdk-cpp/usage +++ b/ports/aws-sdk-cpp/usage @@ -1,12 +1,12 @@ -The package @PORT@:@TARGET_TRIPLET@ provides CMake targets: - - When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. - find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) - target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) - target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) - - OR - - find_package(aws-cpp-sdk-core REQUIRED) - target_include_directories(main PRIVATE aws-cpp-sdk-core) - target_link_libraries(main PRIVATE aws-cpp-sdk-core) +The package @PORT@:@TARGET_TRIPLET@ provides CMake targets: + + When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. + find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) + target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) + target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) + + OR + + find_package(aws-cpp-sdk-core REQUIRED) + target_include_directories(main PRIVATE aws-cpp-sdk-core) + target_link_libraries(main PRIVATE aws-cpp-sdk-core) diff --git a/ports/aws-sdk-cpp/vcpkg.in.json b/ports/aws-sdk-cpp/vcpkg.in.json index ef92a782dac7..e79a583786ba 100644 --- a/ports/aws-sdk-cpp/vcpkg.in.json +++ b/ports/aws-sdk-cpp/vcpkg.in.json @@ -1,6 +1,6 @@ { "name": "aws-sdk-cpp", - "version": "1.11.160", + "version": "1.11.215", "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", "license": "Apache-2.0", @@ -34,4 +34,4 @@ "kinesis", "s3" ] -} \ No newline at end of file +} diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json index ddb0e0bf1ebd..745435741aa6 100644 --- a/ports/aws-sdk-cpp/vcpkg.json +++ b/ports/aws-sdk-cpp/vcpkg.json @@ -1,7 +1,7 @@ { "$note": "Automatically generated by generateFeatures.ps1", "name": "aws-sdk-cpp", - "version": "1.11.160", + "version": "1.11.215", "description": "AWS SDK for C++", "homepage": "https://github.com/aws/aws-sdk-cpp", "license": "Apache-2.0", @@ -454,9 +454,6 @@ "gamelift": { "description": "C++ SDK for the AWS gamelift service" }, - "gamesparks": { - "description": "C++ SDK for the AWS gamesparks service" - }, "glacier": { "description": "C++ SDK for the AWS glacier service" }, diff --git a/ports/s2n/portfile.cmake b/ports/s2n/portfile.cmake index 98bb2c08a09e..0909ebee5bc5 100644 --- a/ports/s2n/portfile.cmake +++ b/ports/s2n/portfile.cmake @@ -2,10 +2,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO aws/s2n-tls REF "v${VERSION}" - SHA512 cfdfcdaa9d57427211e896845358da85d7498fa0478b6916dd34f36c755100c0421c07ccd624ed2d4baf9fcfe0dde6629159769619ac22e9ef1535ba20bf8979 + SHA512 deead85f2ab22441e1110d442fc93273d96d8dd6a203940cca7ef166fc1c9e7ab75ffe2d550e013e1e1e3266b208904cff94cc2482d6fd00e0546293b0ba11d4 PATCHES fix-cmake-target-path.patch - remove-trycompile.patch ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/ports/s2n/remove-trycompile.patch b/ports/s2n/remove-trycompile.patch deleted file mode 100644 index d9385fd45778..000000000000 --- a/ports/s2n/remove-trycompile.patch +++ /dev/null @@ -1,59 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt (revision 4aec93c6a74aacf60cec6229b35094f56cb0f87b) -+++ b/CMakeLists.txt (date 1675010024800) -@@ -393,55 +393,6 @@ - message(STATUS "Enabling libcrypto interning") - endif() - --# Determine if EVP_md5_sha1 is available in libcrypto --try_compile( -- LIBCRYPTO_SUPPORTS_EVP_MD5_SHA1_HASH -- ${CMAKE_BINARY_DIR} -- SOURCES "${CMAKE_CURRENT_LIST_DIR}/tests/features/evp_md5_sha1.c" -- LINK_LIBRARIES ${LINK_LIB} ${OS_LIBS} --) --if (LIBCRYPTO_SUPPORTS_EVP_MD5_SHA1_HASH) -- target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_LIBCRYPTO_SUPPORTS_EVP_MD5_SHA1_HASH) --endif() -- --# Determine if RC4 is available in libcrypto --try_compile( -- LIBCRYPTO_SUPPORTS_EVP_RC4 -- ${CMAKE_BINARY_DIR} -- SOURCES "${CMAKE_CURRENT_LIST_DIR}/tests/features/evp_rc4.c" -- LINK_LIBRARIES ${LINK_LIB} ${OS_LIBS} --) -- --if (LIBCRYPTO_SUPPORTS_EVP_RC4) -- target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_LIBCRYPTO_SUPPORTS_EVP_RC4) --endif() -- --# Determine if EVP_MD_CTX_set_pkey_ctx is available in libcrypto --try_compile( -- LIBCRYPTO_SUPPORTS_EVP_MD_CTX_SET_PKEY_CTX -- ${CMAKE_BINARY_DIR} -- SOURCES "${CMAKE_CURRENT_LIST_DIR}/tests/features/evp_md_ctx_set_pkey_ctx.c" -- LINK_LIBRARIES ${LINK_LIB} ${OS_LIBS} -- CMAKE_FLAGS ${ADDITIONAL_FLAGS} --) -- --if (LIBCRYPTO_SUPPORTS_EVP_MD_CTX_SET_PKEY_CTX) -- target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_LIBCRYPTO_SUPPORTS_EVP_MD_CTX_SET_PKEY_CTX) --endif() -- --# Determine if Kyber512 implementation from AWS-LC is available --try_compile( -- LIBCRYPTO_SUPPORTS_EVP_KYBER_512 -- ${CMAKE_BINARY_DIR} -- SOURCES "${CMAKE_CURRENT_LIST_DIR}/tests/features/evp_kyber_512.c" -- LINK_LIBRARIES ${LINK_LIB} ${OS_LIBS} -- COMPILE_DEFINITIONS "-Werror" --) -- --if(S2N_AWSLC_KYBER_UNSTABLE AND LIBCRYPTO_SUPPORTS_EVP_KYBER_512) -- target_compile_options(${PROJECT_NAME} PUBLIC -DS2N_LIBCRYPTO_SUPPORTS_KYBER512) --endif() -- - if (NOT DEFINED CMAKE_AR) - message(STATUS "CMAKE_AR undefined, setting to `ar` by default") - SET(CMAKE_AR ar) diff --git a/ports/s2n/vcpkg.json b/ports/s2n/vcpkg.json index 1f26ea583b9e..76f883756f64 100644 --- a/ports/s2n/vcpkg.json +++ b/ports/s2n/vcpkg.json @@ -1,6 +1,6 @@ { "name": "s2n", - "version": "1.3.34", + "version": "1.3.56", "description": "C99 implementation of the TLS/SSL protocols.", "homepage": "https://github.com/aws/s2n-tls", "license": "Apache-2.0", From 7b6b5cf0ac1058592dabb842f05e9dbb8f57a7fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Mon, 18 Dec 2023 09:42:57 +0100 Subject: [PATCH 111/456] Add target link libraries for unit_readers target (#4580) Fix issue with `unit_readers` not linking in Debug mode. This is due to missing object library dependency on `baseline`. --- TYPE: NO_HISTORY DESC: Fix issue with `unit_readers` not linking in Debug mode. --- tiledb/sm/query/readers/test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tiledb/sm/query/readers/test/CMakeLists.txt b/tiledb/sm/query/readers/test/CMakeLists.txt index f3367cdaf7be..8de0a4fa8c51 100644 --- a/tiledb/sm/query/readers/test/CMakeLists.txt +++ b/tiledb/sm/query/readers/test/CMakeLists.txt @@ -28,4 +28,5 @@ include(unit_test) commence(unit_test readers) this_target_sources(main.cc unit_reader_base.cc) + this_target_object_libraries(baseline) conclude(unit_test) From 2be338e0be2aa76c47c4a0ea84b03635a0bab918 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 19 Dec 2023 03:47:26 -0600 Subject: [PATCH 112/456] Don't segfault in ArraySchema::dump (#4583) There was no check for whether an enumeration was loaded before attempting to dump it during a call to ArraySchema::dump. Obviously, dereferencing nullptr is bad so this avoids doing that. The new output now shows the following when a Enumeration isn't loaded: ``` ### Attribute ### - Name: a - Type: INT32 - Nullable: false - Cell val num: 1 - Filters: 0 - Fill value: -2147483648 - Enumeration name: flintstones ### Enumeration ### - Name: flintstones - Loaded: false ``` And after calling `load_all_enumerations`: ``` ### Attribute ### - Name: a - Type: INT32 - Nullable: false - Cell val num: 1 - Filters: 0 - Fill value: -2147483648 - Enumeration name: flintstones ### Enumeration ### - Name: flintstones - Loaded: true - Type: STRING_ASCII - Cell Val Num: 4294967295 - Ordered: false - Element Count: 4 ``` --- TYPE: BUG DESC: Don't segfault in ArraySchema::dump --- test/regression/CMakeLists.txt | 1 + test/regression/targets/sc-38300.cc | 59 ++++++++++++++++++++++++++ tiledb/sm/array_schema/array_schema.cc | 10 ++++- tiledb/sm/array_schema/attribute.cc | 4 ++ tiledb/sm/array_schema/enumeration.cc | 1 + 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 test/regression/targets/sc-38300.cc diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 2607aaff1316..7545cda9b1b8 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -48,6 +48,7 @@ if (TILEDB_CPP_API) list(APPEND SOURCES targets/sc-29682.cc) list(APPEND SOURCES targets/sc-33480.cc) list(APPEND SOURCES targets/sc-35424.cc) + list(APPEND SOURCES targets/sc-38300.cc) endif() add_executable(tiledb_regression diff --git a/test/regression/targets/sc-38300.cc b/test/regression/targets/sc-38300.cc new file mode 100644 index 000000000000..26f07a31c46f --- /dev/null +++ b/test/regression/targets/sc-38300.cc @@ -0,0 +1,59 @@ +#include +#include +#include + +#include +#include + +#include + +using namespace tiledb; + +static void create_array(const std::string& array_uri); +static void dump_schema(const std::string& array_uri); + +TEST_CASE( + "Don't segfault in ArraySchema::dump with unloaded enumerations", + "[array-schema][enumeration][bug][sc38300]") { + std::string array_uri = "test_array_schema_dump"; + + // Test setup + create_array(array_uri); + dump_schema(array_uri); +} + +void create_array(const std::string& array_uri) { + Context ctx; + + auto obj = Object::object(ctx, array_uri); + if (obj.type() != Object::Type::Invalid) { + Object::remove(ctx, array_uri); + } + + auto dim = Dimension::create(ctx, "d", {{0, 1024}}); + + Domain dom(ctx); + dom.add_dimension(dim); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx, "flintstones", values); + + auto attr = Attribute::create(ctx, "a"); + AttributeExperimental::set_enumeration_name(ctx, attr, "flintstones"); + + ArraySchema schema(ctx, TILEDB_SPARSE); + ArraySchemaExperimental::add_enumeration(ctx, schema, enmr); + schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}) + .set_domain(dom) + .add_attribute(attr); + + Array::create(array_uri, schema); +} + +void dump_schema(const std::string& array_uri) { + Context ctx; + Array array(ctx, array_uri, TILEDB_READ); + array.schema().dump(); + ArrayExperimental::load_all_enumerations(ctx, array); + array.schema().dump(); +} diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 1bb33ec150d9..df101ed0313e 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -679,7 +679,15 @@ void ArraySchema::dump(FILE* out) const { for (auto& enmr_iter : enumeration_map_) { fprintf(out, "\n"); - enmr_iter.second->dump(out); + if (enmr_iter.second != nullptr) { + enmr_iter.second->dump(out); + } else { + std::stringstream ss; + ss << "### Enumeration ###" << std::endl; + ss << "- Name: " << enmr_iter.first << std::endl; + ss << "- Loaded: false" << std::endl; + fprintf(out, "%s", ss.str().c_str()); + } } for (auto& label : dimension_labels_) { diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index 8588d53e7cfc..ad4b55d5c0dc 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -224,6 +224,10 @@ void Attribute::dump(FILE* out) const { fprintf(out, "\n"); fprintf(out, "- Data ordering: %s", data_order_str(order_).c_str()); } + if (enumeration_name_.has_value()) { + fprintf(out, "\n"); + fprintf(out, "- Enumeration name: %s", enumeration_name_.value().c_str()); + } fprintf(out, "\n"); } diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index 86ad30d784a2..c6b2ab9633f7 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -396,6 +396,7 @@ void Enumeration::dump(FILE* out) const { std::stringstream ss; ss << "### Enumeration ###" << std::endl; ss << "- Name: " << name_ << std::endl; + ss << "- Loaded: true" << std::endl; ss << "- Type: " << datatype_str(type_) << std::endl; ss << "- Cell Val Num: " << cell_val_num_ << std::endl; ss << "- Ordered: " << (ordered_ ? "true" : "false") << std::endl; From 4c735e9e0b4da5100615dcaa14655cdd01e6bf70 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 20 Dec 2023 20:09:56 +0100 Subject: [PATCH 113/456] Make consolidation tests faster. (#4587) Update some consolidation tests to run faster after the consolidation setting change. --- TYPE: NO_HISTORY DESC: Make consolidation tests fater. --- test/src/unit-capi-fragment_info.cc | 20 ++++++++++++- test/src/unit-capi-sparse_heter.cc | 30 +++++++++++++++++-- test/src/unit-capi-string_dims.cc | 8 ++++- test/src/unit-cppapi-array.cc | 6 +++- ...it-cppapi-consolidation-with-timestamps.cc | 1 + test/src/unit-cppapi-consolidation.cc | 4 ++- test/src/unit-cppapi-fragment_info.cc | 1 + test/src/unit-cppapi-hilbert.cc | 1 + test/src/unit-cppapi-max-fragment-size.cc | 1 + test/src/unit-duplicates.cc | 14 ++++++++- 10 files changed, 79 insertions(+), 7 deletions(-) diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index f817f2cce1f7..bddbf027da65 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -1270,6 +1270,11 @@ TEST_CASE( REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + // Consolidate rc = tiledb_array_consolidate(ctx, array_name.c_str(), config); CHECK(rc == TILEDB_OK); @@ -1630,10 +1635,23 @@ TEST_CASE( buffers, &written_frag_uri_3); + // Consolidate fragments + tiledb_config_t* config = nullptr; + tiledb_error_t* error = nullptr; + REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); + REQUIRE(error == nullptr); + + rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + // Consolidate - rc = tiledb_array_consolidate(ctx, array_name.c_str(), nullptr); + rc = tiledb_array_consolidate(ctx, array_name.c_str(), config); CHECK(rc == TILEDB_OK); + tiledb_config_free(&config); + // Create fragment info object tiledb_fragment_info_t* fragment_info = nullptr; rc = tiledb_fragment_info_alloc(ctx, array_name.c_str(), &fragment_info); diff --git a/test/src/unit-capi-sparse_heter.cc b/test/src/unit-capi-sparse_heter.cc index f7692eef4111..2d7d3db3f9af 100644 --- a/test/src/unit-capi-sparse_heter.cc +++ b/test/src/unit-capi-sparse_heter.cc @@ -988,9 +988,22 @@ TEST_CASE_METHOD( // ####### CONSOLIDATE ####### - int rc = tiledb_array_consolidate(ctx_, array_name.c_str(), nullptr); + // Consolidate fragments + tiledb_config_t* config = nullptr; + tiledb_error_t* error = nullptr; + REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); + REQUIRE(error == nullptr); + + auto rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + + rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); CHECK(rc == TILEDB_OK); + tiledb_config_free(&config); + // Check non-empty domain c_dom_f = {1.1f, 1.5f}; c_dom_i = {1, 6}; @@ -1342,9 +1355,22 @@ TEST_CASE_METHOD( // ####### CONSOLIDATE ####### - int rc = tiledb_array_consolidate(ctx_, array_name.c_str(), nullptr); + // Consolidate fragments + tiledb_config_t* config = nullptr; + tiledb_error_t* error = nullptr; + REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); + REQUIRE(error == nullptr); + + int rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + + rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); CHECK(rc == TILEDB_OK); + tiledb_config_free(&config); + // Check non-empty domain c_dom_i = {1, 6}; c_dom_f = {1.1f, 1.5f}; diff --git a/test/src/unit-capi-string_dims.cc b/test/src/unit-capi-string_dims.cc index c7c3bbc7e450..145fe99347cf 100644 --- a/test/src/unit-capi-string_dims.cc +++ b/test/src/unit-capi-string_dims.cc @@ -2284,6 +2284,9 @@ TEST_CASE_METHOD( rc = tiledb_config_set( config, "sm.consolidation.mode", "fragment_meta", &error); CHECK(rc == TILEDB_OK); + rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + CHECK(rc == TILEDB_OK); // Consolidate fragment metadata rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); @@ -2328,8 +2331,11 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); + rc = tiledb_config_set(config, "sm.consolidation.mode", "fragments", &error); + CHECK(rc == TILEDB_OK); + // Consolidate - rc = tiledb_array_consolidate(ctx_, array_name.c_str(), nullptr); + rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); CHECK(rc == TILEDB_OK); rc = tiledb_array_vacuum(ctx_, array_name.c_str(), nullptr); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 944f7cd34286..77cb140c9c47 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -693,7 +693,9 @@ TEST_CASE("C++ API: Consolidation of empty arrays", "[cppapi][consolidation]") { TEST_CASE( "C++ API: Consolidation of sequential fragment writes", "[cppapi][consolidation][sequential]") { - Context ctx; + tiledb::Config cfg; + cfg["sm.consolidation.total_buffer_size"] = "1048576"; + Context ctx(cfg); VFS vfs(ctx); const std::string array_name = "cpp_unit_array"; @@ -743,6 +745,7 @@ TEST_CASE( TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption]") { const char key[] = "0123456789abcdeF0123456789abcdeF"; tiledb::Config cfg; + cfg["sm.consolidation.total_buffer_size"] = "1048576"; cfg["sm.encryption_type"] = "AES_256_GCM"; cfg["sm.encryption_key"] = key; Context ctx(cfg); @@ -827,6 +830,7 @@ TEST_CASE("C++ API: Encrypted array, std::string key", "[cppapi][encryption]") { tiledb::Config cfg; cfg["sm.encryption_type"] = "AES_256_GCM"; cfg["sm.encryption_key"] = key.c_str(); + cfg["sm.consolidation.total_buffer_size"] = "1048576"; Context ctx(cfg); VFS vfs(ctx); const std::string array_name = "cpp_unit_array"; diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index b5b2dc990880..77aa4dedd6d5 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -1404,6 +1404,7 @@ TEST_CASE_METHOD( // fully removed for overlapping ranges, this test case can be deleted. tiledb::Config cfg; cfg.set("sm.merge_overlapping_ranges_experimental", "false"); + cfg["sm.consolidation.total_buffer_size"] = "1048576"; ctx_ = Context(cfg); sm_ = ctx_.ptr().get()->storage_manager(); vfs_ = VFS(ctx_); diff --git a/test/src/unit-cppapi-consolidation.cc b/test/src/unit-cppapi-consolidation.cc index 7469e3019fac..845d2b611025 100644 --- a/test/src/unit-cppapi-consolidation.cc +++ b/test/src/unit-cppapi-consolidation.cc @@ -120,7 +120,9 @@ TEST_CASE( remove_array(array_name); // Create array - Context ctx; + tiledb::Config cfg; + cfg["sm.consolidation.total_buffer_size"] = "1048576"; + Context ctx(cfg); Domain domain(ctx); auto d = Dimension::create(ctx, "d1", {{10, 110}}, 50); domain.add_dimensions(d); diff --git a/test/src/unit-cppapi-fragment_info.cc b/test/src/unit-cppapi-fragment_info.cc index ba13ee4541c2..c60693010d2b 100644 --- a/test/src/unit-cppapi-fragment_info.cc +++ b/test/src/unit-cppapi-fragment_info.cc @@ -879,6 +879,7 @@ TEST_CASE( // Consolidate fragments Config config; config["sm.consolidation.mode"] = "fragments"; + config["sm.consolidation.total_buffer_size"] = "1048576"; Array::consolidate(ctx, array_name, &config); // Load fragment info diff --git a/test/src/unit-cppapi-hilbert.cc b/test/src/unit-cppapi-hilbert.cc index 68fa034e9598..4cec2a37de1e 100644 --- a/test/src/unit-cppapi-hilbert.cc +++ b/test/src/unit-cppapi-hilbert.cc @@ -1974,6 +1974,7 @@ TEST_CASE( Config config; config["sm.consolidation.mode"] = "fragments"; config["sm.vacuum.mode"] = "fragments"; + config["sm.consolidation.total_buffer_size"] = "1048576"; CHECK_NOTHROW(Array::consolidate(ctx, array_name, &config)); CHECK_NOTHROW(Array::vacuum(ctx, array_name, &config)); auto contents = vfs.ls(get_fragment_dir(array_name)); diff --git a/test/src/unit-cppapi-max-fragment-size.cc b/test/src/unit-cppapi-max-fragment-size.cc index 111e9ef3fe97..2b2f70f9b8c2 100644 --- a/test/src/unit-cppapi-max-fragment-size.cc +++ b/test/src/unit-cppapi-max-fragment-size.cc @@ -243,6 +243,7 @@ struct CPPMaxFragmentSizeFx { uint64_t max_fragment_size = std::numeric_limits::max()) { auto config = ctx_.config(); config["sm.consolidation.max_fragment_size"] = max_fragment_size; + config["sm.consolidation.total_buffer_size"] = "1048576"; Array::consolidate(ctx_, array_name, &config); } diff --git a/test/src/unit-duplicates.cc b/test/src/unit-duplicates.cc index 9b44706101e6..b28c8a437ad0 100644 --- a/test/src/unit-duplicates.cc +++ b/test/src/unit-duplicates.cc @@ -334,11 +334,23 @@ TEST_CASE_METHOD( !std::memcmp(data_c_2, data_r, data_r_size); CHECK(data_c_matches); + // Consolidate fragments + tiledb_config_t* config = nullptr; + tiledb_error_t* error = nullptr; + REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); + REQUIRE(error == nullptr); + + rc = tiledb_config_set( + config, "sm.consolidation.total_buffer_size", "1048576", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + // Consolidate - rc = tiledb_array_consolidate(ctx_, array_name_.c_str(), nullptr); + rc = tiledb_array_consolidate(ctx_, array_name_.c_str(), config); REQUIRE(rc == TILEDB_OK); rc = tiledb_array_vacuum(ctx_, array_name_.c_str(), nullptr); REQUIRE(rc == TILEDB_OK); + tiledb_config_free(&config); // Open array for reading - #2 rc = tiledb_array_alloc(ctx_, array_name_.c_str(), &array); From 93a464c6b7de3ae69fae67fb02034fa19c51017e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 21 Dec 2023 04:42:44 -0500 Subject: [PATCH 114/456] Add function random_label to utilize PRNG for random string generation (#4564) Add function `random_label` to utilize PRNG for random string generation. This function is a migration of `random_name` from `test/support/src/helpers` into `common/random`, which has been modified to use the PRNG. Excess declarations of in-test `random_name` have been removed from `tiledb_unit` --- TYPE: IMPROVEMENT DESC: Add function random_label to utilize PRNG for random string generation --- test/src/unit-capi-array.cc | 8 --- test/src/unit-capi-array_schema.cc | 10 +-- test/src/unit-capi-attributes.cc | 10 +-- test/src/unit-capi-dense_array.cc | 10 +-- test/src/unit-capi-dense_neg.cc | 10 +-- test/src/unit-capi-dense_vector.cc | 10 +-- test/src/unit-capi-group.cc | 10 +-- test/src/unit-capi-object_mgmt.cc | 10 +-- test/src/unit-capi-query.cc | 10 +-- test/src/unit-capi-rest-dense_array.cc | 3 +- test/src/unit-capi-sparse_array.cc | 8 --- test/src/unit-capi-sparse_heter.cc | 10 +-- test/src/unit-capi-sparse_neg.cc | 10 +-- test/src/unit-capi-sparse_neg_2.cc | 12 +--- test/src/unit-capi-sparse_real.cc | 10 +-- test/src/unit-capi-sparse_real_2.cc | 12 +--- test/src/unit-capi-string_dims.cc | 10 +-- test/src/unit-capi-vfs.cc | 8 +-- test/src/unit-cppapi-array.cc | 2 +- test/src/unit-cppapi-vfs.cc | 5 +- test/src/unit-s3-no-multipart.cc | 10 +-- test/src/unit-s3.cc | 10 +-- test/src/unit-vfs.cc | 14 ++--- test/support/src/helpers.cc | 7 --- test/support/src/helpers.h | 10 +-- test/support/src/vfs_helpers.h | 6 +- tiledb/common/random/CMakeLists.txt | 3 +- tiledb/common/random/random_label.cc | 63 +++++++++++++++++++ tiledb/common/random/random_label.h | 57 +++++++++++++++++ .../random/test/unit_seedable_global_PRNG.cc | 14 ++++- 30 files changed, 168 insertions(+), 204 deletions(-) create mode 100644 tiledb/common/random/random_label.cc create mode 100644 tiledb/common/random/random_label.h diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 6d351eeb2cb4..e9e97df65628 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -106,7 +106,6 @@ struct ArrayFx { void create_dense_vector(const std::string& path); void create_dense_array(const std::string& path); void write_fragment(tiledb_array_t* array, uint64_t timestamp); - static std::string random_name(const std::string& prefix); static int get_fragment_timestamps(const char* path, void* data); }; @@ -141,13 +140,6 @@ void ArrayFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string ArrayFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - int ArrayFx::get_fragment_timestamps(const char* path, void* data) { auto data_vec = (std::vector*)data; std::pair timestamp_range; diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 643d5cdfa4e2..6641e5b2eff1 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -123,7 +123,6 @@ struct ArraySchemaFx { void delete_array(const std::string& path); bool is_array(const std::string& path); void load_and_check_array_schema(const std::string& path); - static std::string random_name(const std::string& prefix); int array_create_wrapper( const std::string& path, tiledb_array_schema_t* array_schema); @@ -945,13 +944,6 @@ void ArraySchemaFx::load_and_check_array_schema(const std::string& path) { tiledb_array_schema_free(&array_schema); } -std::string ArraySchemaFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - int ArraySchemaFx::get_schema_file_struct(const char* path, void* data) { auto data_struct = (ArraySchemaFx::schema_file_struct*)data; auto ctx = data_struct->ctx; diff --git a/test/src/unit-capi-attributes.cc b/test/src/unit-capi-attributes.cc index b483ebc52607..01e9c7d6dd40 100644 --- a/test/src/unit-capi-attributes.cc +++ b/test/src/unit-capi-attributes.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021 TileDB Inc. + * @copyright Copyright (c) 2021-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -77,7 +77,6 @@ struct Attributesfx { const std::string& path, const std::string& attr_name, tiledb_datatype_t attr_type); - static std::string random_name(const std::string& prefix); }; Attributesfx::Attributesfx() @@ -105,13 +104,6 @@ void Attributesfx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string Attributesfx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void Attributesfx::create_dense_vector( const std::string& path, const std::string& attr_name, diff --git a/test/src/unit-capi-dense_array.cc b/test/src/unit-capi-dense_array.cc index ae3820aaef56..ce94c7f7c351 100644 --- a/test/src/unit-capi-dense_array.cc +++ b/test/src/unit-capi-dense_array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -150,7 +150,6 @@ struct DenseArrayFx { const tiledb_layout_t layout, std::vector& ranges, std::vector& a1); - static std::string random_name(const std::string& prefix); /** * Creates a 2D dense array. @@ -3094,13 +3093,6 @@ void DenseArrayFx::check_non_empty_domain(const std::string& path) { tiledb_array_free(&array); }; -std::string DenseArrayFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, sorted reads", diff --git a/test/src/unit-capi-dense_neg.cc b/test/src/unit-capi-dense_neg.cc index 14b499039403..734ab104a2b3 100644 --- a/test/src/unit-capi-dense_neg.cc +++ b/test/src/unit-capi-dense_neg.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -76,7 +76,6 @@ struct DenseNegFx { void read_dense_array_global(const std::string& path); void read_dense_array_row(const std::string& path); void read_dense_array_col(const std::string& path); - static std::string random_name(const std::string& prefix); }; DenseNegFx::DenseNegFx() @@ -104,13 +103,6 @@ void DenseNegFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string DenseNegFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void DenseNegFx::create_dense_vector(const std::string& path) { int rc; int64_t dim_domain[] = {-1, 2}; diff --git a/test/src/unit-capi-dense_vector.cc b/test/src/unit-capi-dense_vector.cc index 0488976008d9..a1a78a006d57 100644 --- a/test/src/unit-capi-dense_vector.cc +++ b/test/src/unit-capi-dense_vector.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -72,7 +72,6 @@ struct DenseVectorFx { void check_update(const std::string& path); void create_temp_dir(const std::string& path); void remove_temp_dir(const std::string& path); - static std::string random_name(const std::string& prefix); }; DenseVectorFx::DenseVectorFx() @@ -100,13 +99,6 @@ void DenseVectorFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string DenseVectorFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void DenseVectorFx::create_dense_vector( const std::string& path, tiledb_layout_t cell_order, diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 25632aa9720f..16023cec0fd5 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB Inc. + * @copyright Copyright (c) 2022-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -84,7 +84,6 @@ struct GroupFx { void remove_temp_dir(const std::string& path) const; std::string get_golden_walk(const std::string& path); std::string get_golden_ls(const std::string& path); - static std::string random_name(const std::string& prefix); std::vector> read_group( tiledb_group_t* group, bool use_get_member_by_index_v2 = true) const; void set_group_timestamp( @@ -359,13 +358,6 @@ std::string GroupFx::get_golden_ls(const std::string& path) { return golden; } -std::string GroupFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - TEST_CASE_METHOD( GroupFx, "C API: Test group metadata", "[capi][group][metadata]") { // TODO: refactor for each supported FS. diff --git a/test/src/unit-capi-object_mgmt.cc b/test/src/unit-capi-object_mgmt.cc index 204bc84b31e4..414fd2bb1860 100644 --- a/test/src/unit-capi-object_mgmt.cc +++ b/test/src/unit-capi-object_mgmt.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -74,7 +74,6 @@ struct ObjectMgmtFx { std::string get_golden_walk(const std::string& path); std::string get_golden_ls(const std::string& path); static int write_path(const char* path, tiledb_object_t type, void* data); - static std::string random_name(const std::string& prefix); }; ObjectMgmtFx::ObjectMgmtFx() @@ -357,13 +356,6 @@ int ObjectMgmtFx::write_path( return 1; } -std::string ObjectMgmtFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - TEST_CASE_METHOD( ObjectMgmtFx, "C API: Test object management methods: object_type, delete, move", diff --git a/test/src/unit-capi-query.cc b/test/src/unit-capi-query.cc index 8a0b3c1c9cca..6add17a5782e 100644 --- a/test/src/unit-capi-query.cc +++ b/test/src/unit-capi-query.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -71,7 +71,6 @@ struct QueryFx { void test_get_buffer_write_decoupled(const std::string& path); void test_get_buffer_read(const std::string& path); void test_get_buffer_read_decoupled(const std::string& path); - static std::string random_name(const std::string& prefix); }; QueryFx::QueryFx() @@ -87,13 +86,6 @@ QueryFx::~QueryFx() { tiledb_ctx_free(&ctx_); } -std::string QueryFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void QueryFx::create_temp_dir(const std::string& path) { remove_temp_dir(path); REQUIRE(tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index ba8f5f65932b..11f336b7734b 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -98,7 +98,6 @@ struct DenseArrayRESTFx { void create_dense_array_1_attribute(const std::string& array_name); void write_dense_array(const std::string& array_name); void write_dense_array_missing_attributes(const std::string& array_name); - static std::string random_name(const std::string& prefix); /** * Creates a 2D dense array. diff --git a/test/src/unit-capi-sparse_array.cc b/test/src/unit-capi-sparse_array.cc index e1fe3c2c6a1e..2766031cb88b 100644 --- a/test/src/unit-capi-sparse_array.cc +++ b/test/src/unit-capi-sparse_array.cc @@ -133,7 +133,6 @@ struct SparseArrayFx { const std::vector& a3); void create_temp_dir(const std::string& path); void remove_temp_dir(const std::string& path); - static std::string random_name(const std::string& prefix); void check_sorted_reads( const std::string& array_name, tiledb_filter_type_t compressor, @@ -261,13 +260,6 @@ void SparseArrayFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseArrayFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseArrayFx::create_sparse_array_2D( const std::string& array_name, const int64_t tile_extent_0, diff --git a/test/src/unit-capi-sparse_heter.cc b/test/src/unit-capi-sparse_heter.cc index 2d7d3db3f9af..efe4de2f1956 100644 --- a/test/src/unit-capi-sparse_heter.cc +++ b/test/src/unit-capi-sparse_heter.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -118,7 +118,6 @@ struct SparseHeterFx { const std::vector& buff_d1, const std::vector& buff_d2, const std::vector& buff_a); - static std::string random_name(const std::string& prefix); int tiledb_array_get_non_empty_domain_from_index_wrapper( tiledb_ctx_t* ctx, tiledb_array_t* array, @@ -322,13 +321,6 @@ void SparseHeterFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseHeterFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseHeterFx::check_non_empty_domain_float_int64( const std::string& path, const float* dom_f, diff --git a/test/src/unit-capi-sparse_neg.cc b/test/src/unit-capi-sparse_neg.cc index c87374806741..c2e1bdff3e1b 100644 --- a/test/src/unit-capi-sparse_neg.cc +++ b/test/src/unit-capi-sparse_neg.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -74,7 +74,6 @@ struct SparseNegFx { void read_sparse_array_global(const std::string& path); void read_sparse_array_row(const std::string& path); void read_sparse_array_col(const std::string& path); - static std::string random_name(const std::string& prefix); }; SparseNegFx::SparseNegFx() @@ -102,13 +101,6 @@ void SparseNegFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseNegFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseNegFx::create_sparse_vector(const std::string& path) { int rc; int64_t dim_domain[] = {-1, 2}; diff --git a/test/src/unit-capi-sparse_neg_2.cc b/test/src/unit-capi-sparse_neg_2.cc index db7521475dd3..f94897dd05e3 100644 --- a/test/src/unit-capi-sparse_neg_2.cc +++ b/test/src/unit-capi-sparse_neg_2.cc @@ -1,11 +1,11 @@ /** - * @file unit-capi-sparse_neg.cc + * @file unit-capi-sparse_neg_2.cc * * @section LICENSE * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -73,7 +73,6 @@ struct SparseNegFx2 { void read_sparse_vector(const std::string& path); void read_sparse_array_row(const std::string& path); void read_sparse_array_col(const std::string& path); - static std::string random_name(const std::string& prefix); }; SparseNegFx2::SparseNegFx2() @@ -101,13 +100,6 @@ void SparseNegFx2::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseNegFx2::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseNegFx2::create_sparse_vector(const std::string& path) { int rc; int64_t dim_domain[] = {-1, 2}; diff --git a/test/src/unit-capi-sparse_real.cc b/test/src/unit-capi-sparse_real.cc index 3b6133e3c19d..dead9ad65cb7 100644 --- a/test/src/unit-capi-sparse_real.cc +++ b/test/src/unit-capi-sparse_real.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -66,7 +66,6 @@ struct SparseRealFx { void write_sparse_array_next_partition_bug(const std::string& path); void read_sparse_array(const std::string& path); void read_sparse_array_next_partition_bug(const std::string& path); - static std::string random_name(const std::string& prefix); }; SparseRealFx::SparseRealFx() @@ -94,13 +93,6 @@ void SparseRealFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseRealFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseRealFx::create_sparse_array(const std::string& path) { // Create dimensions float dim_domain[] = {-180.0f, 180.0f, -90.0f, 90.0f}; diff --git a/test/src/unit-capi-sparse_real_2.cc b/test/src/unit-capi-sparse_real_2.cc index aa2fe066abda..97fe65ccd8c2 100644 --- a/test/src/unit-capi-sparse_real_2.cc +++ b/test/src/unit-capi-sparse_real_2.cc @@ -1,11 +1,11 @@ /** - * @file unit-capi-sparse_real.cc + * @file unit-capi-sparse_real_2.cc * * @section LICENSE * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -65,7 +65,6 @@ struct SparseRealFx2 { void write_sparse_array_next_partition_bug(const std::string& path); void read_sparse_array(const std::string& path); void read_sparse_array_next_partition_bug(const std::string& path); - static std::string random_name(const std::string& prefix); }; SparseRealFx2::SparseRealFx2() @@ -93,13 +92,6 @@ void SparseRealFx2::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string SparseRealFx2::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void SparseRealFx2::create_sparse_array(const std::string& path) { // Create dimensions float dim_domain[] = {-180.0f, 180.0f, -90.0f, 90.0f}; diff --git a/test/src/unit-capi-string_dims.cc b/test/src/unit-capi-string_dims.cc index 145fe99347cf..e2d3135ebd96 100644 --- a/test/src/unit-capi-string_dims.cc +++ b/test/src/unit-capi-string_dims.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -80,7 +80,6 @@ struct StringDimsFx { ~StringDimsFx(); void create_temp_dir(const std::string& path); void remove_temp_dir(const std::string& path); - std::string random_name(const std::string& prefix); int array_create_wrapper( const std::string& path, tiledb_array_schema_t* array_schema); int array_schema_load_wrapper( @@ -201,13 +200,6 @@ void StringDimsFx::remove_temp_dir(const std::string& path) { REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); } -std::string StringDimsFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - int StringDimsFx::get_dir_num(const char* path, void* data) { auto data_struct = (StringDimsFx::get_num_struct*)data; auto ctx = data_struct->ctx; diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index da24ded03726..b0568adb99cc 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -145,7 +145,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { std::string s3_bucket; if (s3_enabled) { SECTION("Filesystem: S3") { - path = "s3://" + random_name("tiledb") + "/tiledb_test/"; + path = "s3://" + random_label("tiledb-") + "/tiledb_test/"; s3_bucket = path.substr(0, path.find("tiledb_test/")); // Check S3 bucket functionality @@ -171,7 +171,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { /** Note: Azure testing not currently enabled. if (azure_enabled) { SECTION("Filesystem: Azure") { - path = "azure://" + random_name("tiledb") + "/tiledb_test/"; + path = "azure://" + random_label("tiledb-") + "/tiledb_test/"; } } */ @@ -403,7 +403,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { // Move from one bucket to another (only for S3) if (backend_is_s3) { - std::string bucket2 = "s3://" + random_name("tiledb") + "/"; + std::string bucket2 = "s3://" + random_label("tiledb-") + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; // Remove and recreate bucket if already exists @@ -481,7 +481,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { // Copy from one bucket to another (only for S3) if (backend_is_s3) { - std::string bucket2 = "s3://" + random_name("tiledb") + "/"; + std::string bucket2 = "s3://" + random_label("tiledb-") + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; // Remove and recreate bucket if already exists diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 77cb140c9c47..10525f6229f4 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -1883,7 +1883,7 @@ TEST_CASE("C++ API: Array write and read from MemFS", "[cppapi][memfs]") { TEST_CASE( "C++ API: Array on s3 with empty subfolders", "[cppapi][s3][empty_subfolders]") { - const std::string array_bucket = "s3://" + random_name("tiledb") + "/"; + const std::string array_bucket = "s3://" + random_label("tiledb-") + "/"; const std::string array_name = array_bucket + "cpp_unit_array/"; tiledb::Config cfg; diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index 4548b6fe4b8c..ccdb8a165e56 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2023 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -479,8 +479,7 @@ TEST_CASE( tiledb_ctx_is_supported_fs(ctx.ptr().get(), TILEDB_S3, &s3) == TILEDB_OK); if (s3) { tiledb::VFS vfs(ctx); - std::string bucket_name = - "s3://" + tiledb::test::random_name("tiledb") + "/"; + std::string bucket_name = "s3://" + random_label("tiledb-") + "/"; if (vfs.is_bucket(bucket_name)) { REQUIRE_NOTHROW(vfs.remove_bucket(bucket_name)); } diff --git a/test/src/unit-s3-no-multipart.cc b/test/src/unit-s3-no-multipart.cc index 140a040d8681..1965882ef24e 100644 --- a/test/src/unit-s3-no-multipart.cc +++ b/test/src/unit-s3-no-multipart.cc @@ -51,11 +51,10 @@ struct S3DirectFx { S3DirectFx(); ~S3DirectFx(); static Config set_config_params(); - static std::string random_name(const std::string& prefix); const std::string S3_PREFIX = "s3://"; const tiledb::sm::URI S3_BUCKET = - tiledb::sm::URI(S3_PREFIX + random_name("tiledb") + "/"); + tiledb::sm::URI(S3_PREFIX + random_label("tiledb-") + "/"); const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/"; ThreadPool thread_pool_{2}; tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()}; @@ -109,13 +108,6 @@ Config S3DirectFx::set_config_params() { return config; } -std::string S3DirectFx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << tiledb::sm::utils::time::timestamp_now_ms(); - return ss.str(); -} - TEST_CASE_METHOD( S3DirectFx, "Test S3 filesystem, file I/O with multipart API disabled", diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index 295bece26650..340ab95d433f 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -51,11 +51,10 @@ struct S3Fx { S3Fx(); ~S3Fx(); static Config set_config_params(); - static std::string random_name(const std::string& prefix); const std::string S3_PREFIX = "s3://"; const tiledb::sm::URI S3_BUCKET = - tiledb::sm::URI(S3_PREFIX + random_name("tiledb") + "/"); + tiledb::sm::URI(S3_PREFIX + random_label("tiledb-") + "/"); const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/"; ThreadPool thread_pool_{2}; tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()}; @@ -105,13 +104,6 @@ Config S3Fx::set_config_params() { return config; } -std::string S3Fx::random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << tiledb::sm::utils::time::timestamp_now_ms(); - return ss.str(); -} - TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file management", "[s3]") { /* Create the following file hierarchy: * diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 366d5fd2ffe3..78ed7203b2b2 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -154,14 +154,12 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok()); root_pairs.emplace_back( - URI("s3://" + tiledb::test::random_name("vfs") + "/"), - std::move(config)); + URI("s3://" + random_label("vfs-") + "/"), std::move(config)); } if constexpr (tiledb::sm::filesystem::hdfs_enabled) { Config config; root_pairs.emplace_back( - URI("hdfs:///" + tiledb::test::random_name("vfs") + "/"), - std::move(config)); + URI("hdfs:///" + random_label("vfs-") + "/"), std::move(config)); } if constexpr (tiledb::sm::filesystem::azure_enabled) { Config config; @@ -181,19 +179,17 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { .ok()); root_pairs.emplace_back( - URI("azure://" + tiledb::test::random_name("vfs") + "/"), - std::move(config)); + URI("azure://" + random_label("vfs-") + "/"), std::move(config)); } Config config; #ifdef _WIN32 root_pairs.emplace_back( - URI(tiledb::sm::Win::current_dir() + "\\" + - tiledb::test::random_name("vfs") + "\\"), + URI(tiledb::sm::Win::current_dir() + "\\" + random_label("vfs-") + "\\"), std::move(config)); #else root_pairs.emplace_back( - URI(Posix::current_dir() + "/" + tiledb::test::random_name("vfs") + "/"), + URI(Posix::current_dir() + "/" + random_label("vfs-") + "/"), std::move(config)); #endif diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 388363f010b0..72c60f954e6b 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -926,13 +926,6 @@ void open_array( CHECK(rc == TILEDB_OK); } -std::string random_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << TILEDB_TIMESTAMP_NOW_MS; - return ss.str(); -} - void remove_dir(const std::string& path, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs) { int is_dir = 0; REQUIRE(tiledb_vfs_is_dir(ctx, vfs, path.c_str(), &is_dir) == TILEDB_OK); diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 6c0324a23d6d..f45054d4e429 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -37,6 +37,7 @@ #include "test/support/src/coords_workaround.h" #include "tiledb.h" #include "tiledb/common/common.h" +#include "tiledb/common/random/random_label.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/enums/layout.h" @@ -504,15 +505,6 @@ void get_supported_fs( */ void open_array(tiledb_ctx_t* ctx, tiledb_array_t* array, tiledb_query_type_t); -/** - * Returns a random bucket name, with `prefix` as prefix and using - * the thread id as a "random" suffix. - * - * @param prefix The prefix of the bucket name. - * @return A random bucket name. - */ -std::string random_name(const std::string& prefix); - /** * Helper method that removes a directory. * diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index d5c5a2ec20ec..fea0bf0efa51 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -148,7 +148,7 @@ class SupportedFsS3 : public SupportedFs { public: SupportedFsS3() : s3_prefix_("s3://") - , s3_bucket_(s3_prefix_ + random_name("tiledb") + "/") + , s3_bucket_(s3_prefix_ + random_label("tiledb-") + "/") , temp_dir_(s3_bucket_ + "tiledb_test/") { } @@ -275,7 +275,7 @@ class SupportedFsAzure : public SupportedFs { public: SupportedFsAzure() : azure_prefix_("azure://") - , container_(azure_prefix_ + random_name("tiledb") + "/") + , container_(azure_prefix_ + random_label("tiledb-") + "/") , temp_dir_(container_ + "tiledb_test/") { } @@ -342,7 +342,7 @@ class SupportedFsGCS : public SupportedFs { public: SupportedFsGCS(std::string prefix = "gcs://") : prefix_(prefix) - , bucket_(prefix_ + random_name("tiledb") + "/") + , bucket_(prefix_ + random_label("tiledb-") + "/") , temp_dir_(bucket_ + "tiledb_test/") { } diff --git a/tiledb/common/random/CMakeLists.txt b/tiledb/common/random/CMakeLists.txt index e6e934ccd4a3..edff498f6394 100644 --- a/tiledb/common/random/CMakeLists.txt +++ b/tiledb/common/random/CMakeLists.txt @@ -28,7 +28,8 @@ include(common NO_POLICY_SCOPE) include(object_library) list(APPEND SOURCES - prng.cc + prng.cc + random_label.cc seeder.cc ) gather_sources(${SOURCES}) diff --git a/tiledb/common/random/random_label.cc b/tiledb/common/random/random_label.cc new file mode 100644 index 000000000000..a16a028dd6bc --- /dev/null +++ b/tiledb/common/random/random_label.cc @@ -0,0 +1,63 @@ +/** + * @file random_label.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines a random label generator. + */ + +#include "tiledb/common/random/random_label.h" +#include "tiledb/common/random/prng.h" + +#include +#include + +namespace tiledb::common { + +/** + * Legacy code provides randomness using UUIDs, which are always 128 bits, + * represented as a 32-digit hexadecimal value. + * + * To ensure backward compatibility, this function formats the PRNG-generated + * values to be precisely a 32-digit hexadecimal value. Each value is padded + * with 0s such that it makes up one 16-digit half of the full 32-digit number. + */ +std::string random_label(std::string prefix) { + PRNG& prng = PRNG::get(); + std::stringstream ss; + + // Generate and format a 128-bit, 32-digit hexadecimal random number + auto rand1 = prng(); + ss << std::hex << std::setw(16) << std::setfill('0') << rand1; + auto rand2 = prng(); + ss << std::hex << std::setw(16) << std::setfill('0') << rand2; + + // Return label string + return prefix + ss.str(); +} + +} // namespace tiledb::common \ No newline at end of file diff --git a/tiledb/common/random/random_label.h b/tiledb/common/random/random_label.h new file mode 100644 index 000000000000..103a613c5e4a --- /dev/null +++ b/tiledb/common/random/random_label.h @@ -0,0 +1,57 @@ +/** + * @file random_label.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares a random label generator. + */ + +#ifndef TILEDB_HELPERS_H +#define TILEDB_HELPERS_H + +#include + +namespace tiledb::common { + +/** + * Returns a PRNG-generated random label with the optionally-provided prefix. + * + * Given prefix "tiledb-", this function will return a label with syntax + * tiledb-<32-digit hexadecimal random number>. + * (Ex. tiledb-f258d22d4db9139204eef2b4b5d860cc). + * + * Note: the random number is actually the combination of two 16-digit numbers. + * The values are 0-padded to ensure exactly a 128-bit, 32-digit length. + * + * @param prefix The optional prefix of the label. + * @return A random label. + */ +std::string random_label(std::string prefix = ""); + +} // namespace tiledb::common + +#endif // TILEDB_HELPERS_H diff --git a/tiledb/common/random/test/unit_seedable_global_PRNG.cc b/tiledb/common/random/test/unit_seedable_global_PRNG.cc index c0b78e21ce7d..b3d906207a84 100644 --- a/tiledb/common/random/test/unit_seedable_global_PRNG.cc +++ b/tiledb/common/random/test/unit_seedable_global_PRNG.cc @@ -30,10 +30,9 @@ #include #include "../prng.h" +#include "../random_label.h" #include "../seeder.h" -#include - using namespace tiledb::common; TEST_CASE( @@ -96,7 +95,7 @@ TEST_CASE( TEST_CASE( "SeedableGlobalPRNG: operator", "[SeedableGlobalPRNG][operator][multiple]") { - PRNG prng; + PRNG& prng = PRNG::get(); auto rand_num1 = prng(); CHECK(rand_num1 != 0); @@ -128,3 +127,12 @@ TEST_CASE( Catch::Matchers::ContainsSubstring("Seed can only be used once")); } } + +TEST_CASE("random_label", "[random_label]") { + auto rand_label1 = random_label(); + CHECK(rand_label1.length() == 32); + + auto rand_label2 = random_label(); + CHECK(rand_label2.length() == 32); + CHECK(rand_label1 != rand_label2); +} From 0d71748f0d05b5cd66a44202ed0b5bd980b3275b Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Fri, 22 Dec 2023 05:01:30 -0500 Subject: [PATCH 115/456] Reduce amount of copy-paste in filter pipeline tests (#4585) * Create classes for checking filtered buffer data. * Create a class and subclass for generating/checking tile data. * Create class that generates copy-pasted var. length test data. * Create functions for running/checking the filter pipeline. * Use new classes/funcitons in run pipeline tests. --- TYPE: NO_HISTORY DESC: Refactor filter pipeline unit tests --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- tiledb/sm/filter/test/CMakeLists.txt | 1 + .../sm/filter/test/filtered_tile_checker.cc | 76 + tiledb/sm/filter/test/filtered_tile_checker.h | 502 +++++ tiledb/sm/filter/test/tile_data_generator.h | 209 ++ .../filter/test/unit_run_filter_pipeline.cc | 1787 ++++++----------- 5 files changed, 1401 insertions(+), 1174 deletions(-) create mode 100644 tiledb/sm/filter/test/filtered_tile_checker.cc create mode 100644 tiledb/sm/filter/test/filtered_tile_checker.h create mode 100644 tiledb/sm/filter/test/tile_data_generator.h diff --git a/tiledb/sm/filter/test/CMakeLists.txt b/tiledb/sm/filter/test/CMakeLists.txt index 879486245d13..b1fc76cc3d65 100644 --- a/tiledb/sm/filter/test/CMakeLists.txt +++ b/tiledb/sm/filter/test/CMakeLists.txt @@ -44,6 +44,7 @@ commence(unit_test run_filter_pipeline) add_1_including_metadata_filter.cc add_1_out_of_place_filter.cc add_n_in_place_filter.cc + filtered_tile_checker.cc pseudo_checksum_filter.cc unit_run_filter_pipeline.cc ) diff --git a/tiledb/sm/filter/test/filtered_tile_checker.cc b/tiledb/sm/filter/test/filtered_tile_checker.cc new file mode 100644 index 000000000000..91142b6f51e2 --- /dev/null +++ b/tiledb/sm/filter/test/filtered_tile_checker.cc @@ -0,0 +1,76 @@ +/** + * @file filtered_tile_checker.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "filtered_tile_checker.h" +#include + +using namespace tiledb::common; + +namespace tiledb::sm { + +void ChunkChecker::check( + const FilteredBuffer& buffer, + const FilteredTileChunkInfo& buffer_info, + uint64_t chunk_index) const { + auto chunk_info = buffer_info.chunk_info(chunk_index); + auto chunk_offset = buffer_info.chunk_offset(chunk_index); + + // Check the value of the original chunk length. + CHECK( + chunk_info.original_chunk_length() == + expected_chunk_info().original_chunk_length()); + + // Check the metadata. + check_metadata(buffer, chunk_info, chunk_offset); + + // Check the filtered chunk data. + check_filtered_data(buffer, chunk_info, chunk_offset); +} + +void FilteredTileChecker::check(const FilteredBuffer& buffer) const { + // Get a summary of all chunk information. + FilteredTileChunkInfo buffer_chunk_info{buffer}; + + // Check the size of the filtered buffer matches the expected total size. + CHECK(buffer.size() == buffer_chunk_info.size()); + + // Check the number of chunks. + auto nchunks_actual = buffer.value_at_as(0); + auto nchunks_expected = chunk_checkers_.size(); + CHECK(nchunks_actual == nchunks_expected); + + // Check each individual chunk. + auto nchunks_check = std::min(nchunks_actual, nchunks_expected); + for (uint64_t chunk_index = 0; chunk_index < nchunks_check; ++chunk_index) { + INFO("Chunk number: " << chunk_index); + chunk_checkers_[chunk_index]->check(buffer, buffer_chunk_info, chunk_index); + } +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/filtered_tile_checker.h b/tiledb/sm/filter/test/filtered_tile_checker.h new file mode 100644 index 000000000000..a2b7354d8302 --- /dev/null +++ b/tiledb/sm/filter/test/filtered_tile_checker.h @@ -0,0 +1,502 @@ +/** + * @file filtered_tile_checker.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains helper classes for checking the data in a filtered tile + * is as expected after running a filter pipeline forward. + * + * The main class here is the FilteredTileChecker. It checks all chunks in the + * filtered tile with the use of additional helper classes. + * + * - Summary of lengths and data offsets. + * + * - ChunkInfo: Info for chunk component lengths and offsets. + * + * - FilteredTileChunkInfo: Summary of info for all chunks in a tile. + * + * - Checking individual chunks: + * + * - ChunkChecker: Abstract base class for testing the data in a chunk is as + * expected. + * + * - GridChunkChecker: ChunkChecker for fixed grid data with checksum + * metadata. + * + */ + +#ifndef TILEDB_FILTERED_BUFFER_CHECKER_H +#define TILEDB_FILTERED_BUFFER_CHECKER_H + +#include +#include +#include "tiledb/sm/tile/filtered_buffer.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::sm; + +namespace tiledb::sm { + +/** + * Summary of chunk information for all chunks in a filtered tile. + */ +class FilteredTileChunkInfo; + +/** + * Stores the lengths and offsets for data in a chunk. + */ +class ChunkInfo { + public: + /** Constructor for an empty chunk. */ + ChunkInfo() = default; + + /** + * Constructor from chunk component sizes. + * + * Uses prescriptive data lengths. Useful for defining expected chunk info + * in a tests. + * + * @param original_chunk_length Length of unfiltered data. + * @param filtered_chunk_length Length of filtered data stored in the chunk. + * @param metadata_length Length of the metadata stored in the chunk. + */ + ChunkInfo( + uint32_t original_chunk_length, + uint32_t filtered_chunk_length, + uint32_t metadata_length) + : original_chunk_length_{original_chunk_length} + , filtered_chunk_length_{filtered_chunk_length} + , metadata_length_{metadata_length} { + } + + /** Constructor from filtered buffer and chunk offset. */ + ChunkInfo(const FilteredBuffer& buffer, uint32_t chunk_offset) + : original_chunk_length_{buffer.value_at_as(chunk_offset)} + , filtered_chunk_length_{buffer.value_at_as( + chunk_offset + sizeof(uint32_t))} + , metadata_length_{ + buffer.value_at_as(chunk_offset + 2 * sizeof(uint32_t))} { + } + + /** Returns the length of the original, unfiltered data. */ + inline uint32_t original_chunk_length() const { + return original_chunk_length_; + } + + /** Returns the length of the filtered data stored in the chunk. */ + inline uint32_t filtered_chunk_length() const { + return filtered_chunk_length_; + } + + /** Returns the offset from the chunk start to the filtered data. */ + inline uint64_t filtered_chunk_offset() const { + return 3 * sizeof(uint32_t) + static_cast(metadata_length_); + } + + /** Returns the length of the metadata stored in the chunk. */ + inline uint32_t metadata_length() const { + return metadata_length_; + } + + /** Returns the offset from the chunk start to the metadata. */ + inline uint64_t metadata_offset() const { + return 3 * sizeof(uint32_t); + } + + /** + * Returns the totol size of the data in the chunk including size + * information. + */ + inline uint64_t size() const { + return 3 * sizeof(uint32_t) + + static_cast(filtered_chunk_length_) + + static_cast(metadata_length_); + } + + private: + uint32_t original_chunk_length_{}; + uint32_t filtered_chunk_length_{}; + uint32_t metadata_length_{}; +}; + +/** + * Class for checking chunk data and metadata values using Catch2 testing + * macros. + */ +class ChunkChecker { + public: + virtual ~ChunkChecker() = default; + + /** + * Uses Catch2 macros to check the data in a chunk is as expected. + * + * @param tile The tile to check chunk data on. + * @param tile_info The summary chunk info for all chunks in the filtered + * buffer. + * @param chunk_index The index of which chunk to check. + */ + void check( + const FilteredBuffer& tile, + const FilteredTileChunkInfo& tile_info, + uint64_t chunk_index) const; + + /** + * Uses Catch2 macros to check the data in a chunk is as expected. + * + * @param tile The tile to check chunk data on. + * @param chunk_info Summary of the chunk parameters for the chunk being + * checked. + * @param chunk_offset The offset from the start of the tile to the chunk + * being checked. + */ + virtual void check_filtered_data( + const FilteredBuffer& tile, + const ChunkInfo& chunk_info, + uint64_t chunk_offset) const = 0; + + /** + * Uses Catch2 macros to check the metadata in a chunk is as expected. + * + * @param tile The tile to check chunk data on. + * @param chunk_info Summary of the chunk parameters for the chunk being + * checked. + * @param chunk_offset The offset from the start of the tile to the chunk + * being checked. + */ + virtual void check_metadata( + const FilteredBuffer& buffer, + const ChunkInfo& chunk_info, + uint64_t chunk_offset) const = 0; + + protected: + /** Returns the chunk info that is expected for the chunk being tested. */ + virtual const ChunkInfo& expected_chunk_info() const = 0; +}; + +/** + * A chunk checker for data that increases by a fixed amount at each + * subsequent element (a grid with fixed-spacing). It supports checksum + * metadata. + * + * This is for testing data on file using the test-specific filters. + * + * @tparam Type of the filtered data. + */ +template +class GridChunkChecker : public ChunkChecker { + public: + /** + * Constructor for checker with no metadata. + * + * @param original_chunk_length The length of the original, unfiltered data. + * @param num_filtered_elements The number of filtered elements stored in the + * chunk. + * @param starting_value The value of the first element in the chunk. + * @param spacing The difference between subsequent elements. + */ + GridChunkChecker( + uint32_t original_chunk_length, + uint32_t num_filtered_elements, + T starting_value, + T spacing) + : expected_chunk_info_{original_chunk_length, static_cast(num_filtered_elements * sizeof(T)), 0} + , num_filtered_elements_{num_filtered_elements} + , starting_value_{starting_value} + , spacing_{spacing} + , checksum_{} { + } + + /** + * Constructor for checker with metadata. + * + * @param original_chunk_length The length of the original, unfiltered data. + * @param num_filtered_elements The number of filtered elements stored in the + * chunk. + * @param starting_value The value of the first element in the chunk. + * @param spacing The difference between subsequent elements. + * @param checksum A vector of expected checksum values. + */ + GridChunkChecker( + uint32_t original_chunk_length, + uint32_t num_filtered_elements, + T starting_value, + T spacing, + std::vector checksum) + : expected_chunk_info_{original_chunk_length, static_cast(num_filtered_elements * sizeof(T)), 0} + , num_filtered_elements_{num_filtered_elements} + , starting_value_{starting_value} + , spacing_{spacing} + , checksum_{checksum} { + } + + /** + * Uses Catch2 macros to check the data in a chunk stored is as expected. + * + * @param tile The tile to check chunk data on. + * @param chunk_info The summary chunk info for all chunks in the filtered + * buffer. + * @param chunk_offset The index of which chunk to check. + */ + void check_filtered_data( + const FilteredBuffer& tile, + const ChunkInfo& chunk_info, + uint64_t chunk_offset) const override { + // Check the size of the filtered data. If it does not match the + // expected size, then end the test as a failure. + REQUIRE( + chunk_info.filtered_chunk_length() == + expected_chunk_info_.filtered_chunk_length()); + + // Check the data. + auto data_offset = chunk_info.filtered_chunk_offset() + chunk_offset; + for (uint32_t index = 0; index < num_filtered_elements_; ++index) { + auto offset = data_offset + index * sizeof(T); + T expected_value = starting_value_ + index * spacing_; + T actual_value = tile.value_at_as(offset); + CHECK(actual_value == expected_value); + } + } + + /** + * Uses Catch2 macros to check the metadata in a chunk is as expected. + * + * @param tile The tile to check chunk data on. + * @param chunk_info Summary of the chunk parameters for the chunk being + * checked. + * @param chunk_offset The offset from the start of the tile to the chunk + * being checked. + */ + void check_metadata( + const FilteredBuffer& tile, + const ChunkInfo& chunk_info, + uint64_t chunk_offset) const override { + // Check the size of the metadata. If it does not match the expected + // size, then end the test as a failure. + REQUIRE( + chunk_info.metadata_length() == checksum_.size() * sizeof(uint64_t)); + + // Check the metadata values. + auto metadata_offset = chunk_info.metadata_offset() + chunk_offset; + for (uint64_t index = 0; index < checksum_.size(); ++index) { + auto offset = metadata_offset + index * sizeof(uint64_t); + auto actual_checksum = tile.value_at_as(offset); + auto expected_checksum = checksum_[index]; + CHECK(actual_checksum == expected_checksum); + } + } + + protected: + /** Returns the chunk info that is expected for the chunk being tested. */ + const ChunkInfo& expected_chunk_info() const override { + return expected_chunk_info_; + } + + private: + ChunkInfo expected_chunk_info_{}; + uint32_t num_filtered_elements_{}; + T starting_value_{}; + T spacing_{}; + std::vector checksum_{}; +}; + +/** + * Summary of chunk information for all chunks in a filtered tile. + */ +class FilteredTileChunkInfo { + public: + /** + * Constructor. + * + * @param buffer The FilteredBuffer to create the summary for. + */ + FilteredTileChunkInfo(const FilteredBuffer& buffer) + : nchunks_{buffer.value_at_as(0)} + , chunk_info_{} + , offsets_{} { + chunk_info_.reserve(nchunks_); + offsets_.reserve(nchunks_); + uint64_t current_offset = sizeof(uint64_t); + for (uint64_t index = 0; index < nchunks_; ++index) { + chunk_info_.emplace_back(buffer, current_offset); + offsets_.push_back(current_offset); + current_offset += chunk_info_[index].size(); + } + size_ = current_offset; + } + + /** Returns the chunk info for the requested chunk. */ + inline const ChunkInfo& chunk_info(uint64_t index) const { + return chunk_info_[index]; + } + + /** Returns the offset for accessing the requested chunk. */ + inline uint64_t chunk_offset(uint64_t index) const { + return offsets_[index]; + } + + /** Returns the total number of chunks. */ + inline uint64_t nchunks() const { + return nchunks_; + } + + /** Returns the total size of the chunk. */ + inline uint64_t size() const { + return size_; + } + + private: + uint64_t nchunks_{}; + std::vector chunk_info_; + std::vector offsets_{}; + uint64_t size_{}; +}; + +/** + * Class for checking all filtered data using Catch2 macros. + */ +class FilteredTileChecker { + public: + /** + * Factory for creating a FilteredTileChecker where the data forms a grid with + * fixed spacing. + * + * @tparam T Datatype for the filtered data. + * @param elements_per_chunk A vector storing the number of elements expected + * on each chunk. + * @param starting_value The value of the first element. + * @param spacing The spacing between subsequent values. + */ + template + static FilteredTileChecker create_uncompressed_with_grid_chunks( + const std::vector& elements_per_chunk, + T starting_value = 0, + T spacing = 1) { + FilteredTileChecker checker{}; + T start{starting_value}; + for (auto nelements_chunk : elements_per_chunk) { + uint32_t data_size{static_cast(nelements_chunk * sizeof(T))}; + checker.add_grid_chunk_checker( + data_size, nelements_chunk, start, spacing); + start += nelements_chunk * spacing; + } + return checker; + } + + /** + * Factory for creating a FilteredTileChecker where the data forms a grid with + * fixed spacing. + * + * @tparam T Datatype for the filtered data. + * @param elements_per_chunk A vector storing the number of elements expected + * on each chunk. Must be the same length as `checksum_per_chunk`. + * @param checksum_per_chunk A vector of vectors storing checksum metadata + * expected on each chunk. Must be the same length as `elements_per_chunk`. + * @param starting_value The value of the first element. + * @param spacing The spacing between subsequent values. + */ + template + static FilteredTileChecker create_uncompressed_with_grid_chunks( + const std::vector& elements_per_chunk, + const std::vector>& checksum_per_chunk, + T starting_value = 0, + T spacing = 1) { + FilteredTileChecker checker{}; + T start{starting_value}; + if (elements_per_chunk.size() != checksum_per_chunk.size()) { + throw std::runtime_error( + "Mismatched test parameters for filtered tile checker."); + } + for (uint64_t chunk_index = 0; chunk_index < elements_per_chunk.size(); + ++chunk_index) { + auto nelements = elements_per_chunk[chunk_index]; + auto checksum = checksum_per_chunk[chunk_index]; + uint32_t data_size{static_cast(nelements * sizeof(T))}; + checker.add_grid_chunk_checker( + data_size, nelements, start, spacing, checksum); + start += nelements * spacing; + } + return checker; + } + + /** Constructor for an empty tile checker. */ + FilteredTileChecker() = default; + + template + void add_grid_chunk_checker( + uint32_t original_chunk_length, + uint32_t num_filtered_elements, + T starting_value, + T spacing) { + chunk_checkers_.emplace_back(tdb_new( + GridChunkChecker, + original_chunk_length, + num_filtered_elements, + starting_value, + spacing)); + } + + /** + * Add an additional grid chunk checker. + * + * @tparam T Datatype of the filtered data. + * @param original_chunk_length The length of the original, unfiltered data. + * @param num_filtered_elements Number of elements in the chunk. + * @param starting_value The value of the first element. + * @param spacing Spacing between subsequent elements. + * @param checksum Vector of checksums stored as metadata. + */ + template + void add_grid_chunk_checker( + uint32_t original_chunk_length, + uint32_t num_filtered_elements, + T starting_value, + T spacing, + std::vector checksum) { + chunk_checkers_.emplace_back(tdb_new( + GridChunkChecker, + original_chunk_length, + num_filtered_elements, + starting_value, + spacing, + checksum)); + } + + /** + * Use Catch2 macros to check all chunks have the expected data stored in + * them. + * + * @param tile The tile to check. + */ + void check(const FilteredBuffer& tile) const; + + private: + std::vector> chunk_checkers_; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/tile_data_generator.h b/tiledb/sm/filter/test/tile_data_generator.h new file mode 100644 index 000000000000..2b88491e16ae --- /dev/null +++ b/tiledb/sm/filter/test/tile_data_generator.h @@ -0,0 +1,209 @@ +/** + * @file tile_data_generator.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * Class for generating data on a writer data and checking for that same data + * on another tile. + */ + +#ifndef TILEDB_INPUT_TILE_TEST_DATA_H +#define TILEDB_INPUT_TILE_TEST_DATA_H + +#include +#include +#include +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +class TileDataGenerator { + public: + virtual ~TileDataGenerator() = default; + + virtual uint64_t cell_size() const = 0; + /** + * Checks if the provide tile has the same data as a writer tile created + * by this class. + */ + virtual void check_tile_data(const Tile& tile) const = 0; + + /** Returns the datatype of the original data stored in this test. */ + virtual Datatype datatype() const = 0; + + /** + * Returns an empty writer tile with enough room for the input data. + */ + WriterTile create_empty_writer_tile() const { + return WriterTile( + constants::format_version, + datatype(), + cell_size(), + original_tile_size()); + } + + /** + * Returns the writer tile and optional writer offsets tile. + * + * If the data is fixed, the offsets tile will be a null opt. + * + * @returns [writer_tile, writer_offsets_tile] The writer tile with the input + * test data and the writer offsets tile with the (optional) input offsets + * data. + */ + virtual std::tuple> + create_writer_tiles() const = 0; + + /** + * Returns a tile with the data from the filtered buffer and enough room + * for the original tile data. + **/ + Tile create_filtered_buffer_tile(FilteredBuffer& filtered_buffer) const { + return Tile( + constants::format_version, + datatype(), + cell_size(), + 0, + original_tile_size(), + filtered_buffer.data(), + filtered_buffer.size()); + } + + /** Returns the size of the original unfiltered data. */ + virtual uint64_t original_tile_size() const = 0; +}; + +/** + * This is a simple tile data generator that simply contains incremental values + * to store int he tile. + * + * @WARNING This test data modifies the max tile chunk size. This is required to + * get the expected data chunks in the filtered buffer. + */ +template +class IncrementTileDataGenerator : public TileDataGenerator { + public: + /** + * Constructor for variable length data with multiple chunks. + * + * @param cells_per_value Number of cells in each value. Used for generating + * offsets. + */ + IncrementTileDataGenerator(const std::vector& cells_per_value) + : num_elements_{std::accumulate( + cells_per_value.cbegin(), + cells_per_value.cend(), + static_cast(0))} + , cells_per_value_{cells_per_value} + , original_tile_size_{num_elements_ * sizeof(T)} { + } + + /** Constructor for fixed data with all data in a single chunk. */ + IncrementTileDataGenerator(uint64_t num_elements) + : num_elements_{num_elements} + , cells_per_value_{} + , cells_per_chunk_{num_elements_} + , original_tile_size_{num_elements_ * sizeof(T)} { + } + + ~IncrementTileDataGenerator() { + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + } + + uint64_t cell_size() const override { + return sizeof(T); + } + + void check_tile_data(const Tile& tile) const override { + T expected{}; + for (uint64_t index = 0; index < num_elements_; ++index) { + T element{}; + CHECK_NOTHROW(tile.read(&element, index * sizeof(T), sizeof(T))); + CHECK(element == expected++); + } + } + + std::tuple> create_writer_tiles() + const override { + // Writer tile. + auto tile = create_empty_writer_tile(); + T value{}; + for (uint64_t index = 0; index < num_elements_; ++index) { + CHECK_NOTHROW(tile.write(&value, index * sizeof(T), sizeof(T))); + ++value; + } + + // If no cells per value data, then this is fixed length data and there is + // no offsets tile. + if (cells_per_value_.empty()) { + return {std::move(tile), std::nullopt}; + } + + // If cells_per_value_ is not empty, construct a vector of offsets values. + std::vector offsets; + offsets.reserve(cells_per_value_.size() + 1); + offsets.push_back(0); + for (auto num_cells : cells_per_value_) { + offsets.push_back(offsets.back() + num_cells * sizeof(uint64_t)); + } + offsets.pop_back(); + + // Write the offsets tile. + WriterTile offsets_tile( + constants::format_version, + Datatype::UINT64, + constants::cell_var_offset_size, + offsets.size() * constants::cell_var_offset_size); + for (uint64_t index = 0; index < offsets.size(); ++index) { + CHECK_NOTHROW(offsets_tile.write( + &offsets[index], + index * constants::cell_var_offset_size, + constants::cell_var_offset_size)); + } + + return {std::move(tile), std::move(offsets_tile)}; + } + + Datatype datatype() const override { + return type; + } + + uint64_t original_tile_size() const override { + return original_tile_size_; + } + + private: + uint64_t num_elements_; + std::vector cells_per_value_; + std::vector cells_per_chunk_; + uint64_t original_tile_size_; +}; + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 1c2e02ac3748..c068d61c8334 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -32,8 +32,34 @@ * operations such as adding 1 to each value in the original data so that the * filtered data itself can be checked after running the filter pipeline * forward. + * + * + * Notes on variable length data: + * + * The filtered pipeline will break-up tile data into chunks for filtering. + * Below we describe the decision process for adding to the existing chunk vs + * creating a new chunk for variable length data. + * + * Define the following when adding a value of variable length data: + * + * * "current size": the size of the current chunk before adding the data + * * "new size": the size of the current chunk if the new data is added to it + * * "target size": the target size for chunks + * * "min size": 50% the target size for chunks + * * "max size": 150% the target size for chunks + * + * A new chunk is created if the total size > target size. + * + * When a new chunk is created, if either of the following are met, then add + * the current component to the existing chunk: + * + * Condition 1. current size < min size + * Condition 2. new size < max size + * */ +#include +#include #include #include @@ -52,7 +78,9 @@ #include "add_1_including_metadata_filter.h" #include "add_1_out_of_place_filter.h" #include "add_n_in_place_filter.h" +#include "filtered_tile_checker.h" #include "pseudo_checksum_filter.h" +#include "tile_data_generator.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/datatype.h" @@ -66,406 +94,341 @@ using namespace tiledb::sm; // A dummy `Stats` instance. static tiledb::sm::stats::Stats dummy_stats("test"); -WriterTile make_increasing_tile(const uint64_t nelts) { - const uint64_t tile_size = nelts * sizeof(uint64_t); - const uint64_t cell_size = sizeof(uint64_t); +class SimpleTestData {}; - WriterTile tile( - constants::format_version, Datatype::UINT64, cell_size, tile_size); - for (uint64_t i = 0; i < nelts; i++) { - CHECK_NOTHROW(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); +/** + * Original variable length test from the pipeline tests. + * + * For this test target size is 10 cells per chunk. Below is a list of value + * cell lengths, the cell they are added to, and the rational. + * + * target = 8 cells + * min = 4 cells + * max = 12 cells + * + * | # Cells | Previous/New # Cell in Chunk | Notes | + * |:-------:|:--------|:------------------------------------------------------| + * | 4 | 0 / 4 | chunk 0: initial chunk | + * | 10 | 4 / 14 | chunk 0: new > max, prev. <= min (next new) | + * | 6 | 0 / 6 | chunk 1: new <= target | + * | 11 | 6 / 11 | chunk 2: target < new <= max, prev. > min (next new) | + * | 7 | 0 / 7 | chunk 3: new <= target | + * | 9 | 7 / 16 | chunk 4: new > max, prev. > min (this new) | + * | 1 | 9 / 10 | chunk 4: new <= target | + * | 10 | 10 / 20 | chunk 5: new > max, prev. > min (this new) | + * | 20 | 0 / 20 | chunk 6: new > max, prev. < min (next new) | + * | 2 | 0 / 2 | chunk 7: new <= target | + * | 2 | 2 / 4 | chunk 7: new <= target | + * | 2 | 4 / 6 | chunk 7: new <= target | + * | 2 | 6 / 8 | chunk 7: new <= target | + * | 2 | 8 / 10 | chunk 7: new <= target | + * | 12 | 10 / 24 | chunk 8: new > max, prev. > min (this new) | + * + */ +class SimpleVariableTestData { + public: + SimpleVariableTestData() + : target_ncells_per_chunk_{10} + , elements_per_chunk_{14, 6, 11, 7, 10, 10, 20, 10, 12} + , tile_data_generator_{ + {4, 10, 6, 11, 7, 9, 1, 10, 20, 2, 2, 2, 2, 2, 12}} { + WriterTile::set_max_tile_chunk_size( + target_ncells_per_chunk_ * sizeof(uint64_t)); + } + ~SimpleVariableTestData() { + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } - return tile; -} + /** Returns the number elements (cells) stored in each chunk. */ + const std::vector& elements_per_chunk() const { + return elements_per_chunk_; + } -WriterTile make_offsets_tile(std::vector& offsets) { - const uint64_t offsets_tile_size = - offsets.size() * constants::cell_var_offset_size; - - WriterTile offsets_tile( - constants::format_version, - Datatype::UINT64, - constants::cell_var_offset_size, - offsets_tile_size); - - // Set up test data - for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK_NOTHROW(offsets_tile.write( - &offsets[i], - i * constants::cell_var_offset_size, - constants::cell_var_offset_size)); + const TileDataGenerator& tile_data_generator() const { + return tile_data_generator_; } - return offsets_tile; -} + private: + uint64_t target_ncells_per_chunk_{}; + std::vector elements_per_chunk_{}; + IncrementTileDataGenerator tile_data_generator_; +}; + +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Filtered buffer data is as expected. + * 3. Pipeline runs backward without error. + * 4. Result from roundtrip matches the original data. + */ +void check_run_pipeline_full( + Config& config, + ThreadPool& tp, + WriterTile& tile, + std::optional& offsets_tile, + FilterPipeline& pipeline, + const TileDataGenerator* test_data, + const FilteredTileChecker& filtered_buffer_checker) { + // Run the pipeline forward. + CHECK(pipeline + .run_forward( + &dummy_stats, + &tile, + offsets_tile.has_value() ? &offsets_tile.value() : nullptr, + &tp) + .ok()); + + // Check the original unfiltered data was removed. + CHECK(tile.size() == 0); + + // Check the filtered buffer has the expected data. + auto filtered_buffer = tile.filtered_buffer(); + filtered_buffer_checker.check(filtered_buffer); -Tile create_tile_for_unfiltering(uint64_t nelts, WriterTile& tile) { - Tile ret( - tile.format_version(), - tile.type(), - tile.cell_size(), - 0, - tile.cell_size() * nelts, - tile.filtered_buffer().data(), - tile.filtered_buffer().size()); - return ret; + // Run the data in reverse. + auto unfiltered_tile = + test_data->create_filtered_buffer_tile(filtered_buffer); + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + CHECK(pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); + + // Check the original data is reverted. + test_data->check_tile_data(unfiltered_tile); } -void run_reverse( - const tiledb::sm::Config& config, +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Pipeline runs backward without error. + * 3. Result from roundtrip matches the original data. + */ +void check_run_pipeline_roundtrip( + Config& config, ThreadPool& tp, - Tile& unfiltered_tile, + WriterTile& tile, + std::optional& offsets_tile, FilterPipeline& pipeline, - bool success = true) { + TileDataGenerator* test_data) { + // Run the pipeline forward. + CHECK(pipeline + .run_forward( + &dummy_stats, + &tile, + offsets_tile.has_value() ? &offsets_tile.value() : nullptr, + &tp) + .ok()); + + // Check the original unfiltered data was removed. + CHECK(tile.size() == 0); + + // Run the data in reverse. + auto unfiltered_tile = + test_data->create_filtered_buffer_tile(tile.filtered_buffer()); ChunkData chunk_data; unfiltered_tile.load_chunk_data(chunk_data); - CHECK( - success == pipeline - .run_reverse( - &dummy_stats, - &unfiltered_tile, - nullptr, - chunk_data, - 0, - chunk_data.filtered_chunks_.size(), - tp.concurrency_level(), - config) - .ok()); + CHECK(pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); + + // Check the original data is reverted. + test_data->check_tile_data(unfiltered_tile); } TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; + // Create pipeline. FilterPipeline pipeline; + + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 0, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); +} + +TEST_CASE( + "Filter: Test empty pipeline on uint16 data", "[filter][empty-pipeline]") { + // Create TileDB needed for running pipeline. + Config config; ThreadPool tp(4); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - // Check size and number of chunks - CHECK(tile.size() == 0); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements unchanged. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i); - offset += sizeof(uint64_t); - } + // Create pipeline. + FilterPipeline pipeline; - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 0, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE( "Filter: Test empty pipeline var sized", "[filter][empty-pipeline][var]") { - tiledb::sm::Config config; - - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - FilterPipeline pipeline; + // Create TileDB needed for running pipeline. + Config config; ThreadPool tp(4); - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements unchanged. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == el++); - offset += sizeof(uint64_t); - } - } + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + // Create pipeline to test and expected filtered data checker. + FilterPipeline pipeline; + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 0, 1); + + // Run the round-trip test. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE( "Filter: Test simple in-place pipeline", "[filter][simple-in-place]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 1, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { // Add a few more +1 filters and re-run. pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); - offset += sizeof(uint64_t); - } - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 3, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } } TEST_CASE( "Filter: Test simple in-place pipeline var", "[filter][simple-in-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - auto offsets_tile = make_offsets_tile(offsets); + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 1, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { @@ -473,757 +436,348 @@ TEST_CASE( WriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); - offset += sizeof(uint64_t); - } - } - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 3, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } TEST_CASE( "Filter: Test simple out-of-place pipeline", "[filter][simple-out-of-place]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; + // Create pipeline to test. FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 1)); - offset += sizeof(uint64_t); - } + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 1, 1); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { // Add a few more +1 filters and re-run. pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 3)); - offset += sizeof(uint64_t); - } - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 3, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } } TEST_CASE( "Filter: Test simple out-of-place pipeline var", "[filter][simple-out-of-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - auto offsets_tile = make_offsets_tile(offsets); + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 1, 1); + + // run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { - // Add a few more +1 filters and re-run. - WriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 2); - offset += sizeof(uint64_t); - } - } - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 3, 1); + + // run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } TEST_CASE( "Filter: Test mixed in- and out-of-place pipeline", "[filter][in-out-place]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; + // Create filter pipeline. FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 4, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE( "Filter: Test mixed in- and out-of-place pipeline var", "[filter][in-out-place][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - auto offsets_tile = make_offsets_tile(offsets); + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; - ThreadPool tp(4); - WriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 4, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - // Set up test data - const uint64_t nelts = 100; - const uint64_t expected_checksum = 4950; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; + // Create filter pipeline. FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + const uint64_t expected_checksum = 4950; SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + - 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements are the same. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create filtered buffer checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, {{expected_checksum}}, 0, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { + // Add filters. pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); + // Create filtered buffer checker. + // const uint64_t expected_checksum = 2 * 4950; // Compute the second (final) checksum value. - uint64_t expected_checksum_2 = 0; - for (uint64_t i = 0; i < nelts; i++) - expected_checksum_2 += i + 2; - - // Check size and number of chunks. - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + sizeof(uint64_t) + - sizeof(uint64_t) + 3 * sizeof(uint32_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 2 * sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Outer checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum_2); - offset += sizeof(uint64_t); - - // Inner checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements are correct. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == i + 2); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + const uint64_t expected_checksum_2 = 5150; + + // Create filtered buffer checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, + {{expected_checksum_2, expected_checksum}}, + 2, + 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } } TEST_CASE( "Filter: Test pseudo-checksum var", "[filter][pseudo-checksum][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - std::vector expected_checksums{ - 91, 99, 275, 238, 425, 525, 1350, 825, 1122}; + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); + // Create filter pipeline. FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + - 9 * sizeof(uint64_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // Chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums[i]); - offset += sizeof(uint64_t); - - // Check all elements are the same. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == el++); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + std::vector> expected_checksums{ + {91}, {99}, {275}, {238}, {425}, {525}, {1350}, {825}, {1122}}; + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, expected_checksums, 0, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } SECTION("- Multi-stage") { - WriterTile::set_max_tile_chunk_size(80); + // Update pipeline. pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - std::vector expected_checksums2{ - 119, 111, 297, 252, 445, 545, 1390, 845, 1146}; - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t) + - 2 * 9 * sizeof(uint64_t)); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 2 * sizeof(uint64_t)); // Chunk metadata size - offset += sizeof(uint32_t); - - // Checksums - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums2[i]); - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksums[i]); - offset += sizeof(uint64_t); - - // Check all elements are incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 1); - offset += sizeof(uint64_t); - } - } - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create expected filtered data checker. + std::vector> expected_checksums{ + {119, 91}, + {111, 99}, + {297, 275}, + {252, 238}, + {445, 425}, + {545, 525}, + {1390, 1350}, + {845, 825}, + {1146, 1122}}; + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, expected_checksums, 2, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - // Set up test data - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; + // Create filter pipeline. FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(AddNInPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); @@ -1237,85 +791,34 @@ TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { CHECK(add_n != nullptr); add_n->set_increment(2); - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // First chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create pipeline to test and expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 4, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - auto offsets_tile = make_offsets_tile(offsets); + // Set-up test data. + SimpleVariableTestData test_data{}; + auto&& [tile, offsets_tile] = + test_data.tile_data_generator().create_writer_tiles(); + const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(AddNInPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); @@ -1329,66 +832,36 @@ TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { CHECK(add_n != nullptr); add_n->set_increment(2); - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&dummy_stats, &tile, &offsets_tile, &tp).ok()); - - // Check size and number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().size() == - nelts * sizeof(uint64_t) + sizeof(uint64_t) + 3 * 9 * sizeof(uint32_t)); - - offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t el = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 0); // Chunk metadata size - offset += sizeof(uint32_t); - - // Check all elements are incremented. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == ++el + 3); - offset += sizeof(uint64_t); - } - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + // Create expected filtered data checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, 4, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &test_data.tile_data_generator(), + filtered_buffer_checker); } TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { - tiledb::sm::Config config; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); - const uint64_t expected_checksum = 5350; + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + std::vector elements_per_chunk{100}; - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + const uint64_t expected_checksum = 5350; FilterPipeline pipeline; - ThreadPool tp(4); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(AddNInPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); @@ -1408,57 +881,28 @@ TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { CHECK(add_n_2 != nullptr); CHECK(add_n_2->increment() == 2); - CHECK(pipeline_copy.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile.filtered_buffer().value_at_as(offset) == - 1); // Number of chunks - offset += sizeof(uint64_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // First chunk filtered size - offset += sizeof(uint32_t); - CHECK( - tile.filtered_buffer().value_at_as(offset) == - sizeof(uint64_t)); // First chunk metadata size - offset += sizeof(uint32_t); - - // Checksum - CHECK( - tile.filtered_buffer().value_at_as(offset) == - expected_checksum); - offset += sizeof(uint64_t); - - // Check all elements incremented. - for (uint64_t i = 0; i < nelts; i++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == (i + 4)); - offset += sizeof(uint64_t); - } - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } + // Create filtered buffer checker. + auto filtered_buffer_checker = + FilteredTileChecker::create_uncompressed_with_grid_chunks( + elements_per_chunk, {{expected_checksum}}, 4, 1); + + // Run the pipeline tests. + check_run_pipeline_full( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + filtered_buffer_checker); } TEST_CASE("Filter: Test random pipeline", "[filter][random]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); + // Create an encryption key. EncryptionKey encryption_key; REQUIRE(encryption_key .set_key( @@ -1492,14 +936,18 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { }, }; - // List of potential filters that must occur at the beginning of the pipeline. + // List of potential filters that must occur at the beginning of the + // pipeline. std::vector> constructors_first = { // Pos-delta would (correctly) return error after e.g. compression. []() { return tdb_new(PositiveDeltaFilter, Datatype::UINT64); }}; - ThreadPool tp(4); + // Create tile data generator. + IncrementTileDataGenerator tile_data_generator( + 100); for (int i = 0; i < 100; i++) { - auto tile = make_increasing_tile(nelts); + // Create fresh input tiles. + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); // Construct a random pipeline FilterPipeline pipeline; @@ -1528,19 +976,10 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { } } - // End result should always be the same as the input. - CHECK(pipeline.run_forward(&dummy_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - for (uint64_t n = 0; n < nelts; n++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == n); - } + // Check the pipelines run forward and backward without error and return the + // input data. + // Run the pipeline tests. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); } } From 7b0a4eeea18a999a7f4d35ded8baf987f20184e2 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 22 Dec 2023 12:08:11 +0200 Subject: [PATCH 116/456] Add vcpkg triplets for ASAN. (#4515) [SC-34454](https://app.shortcut.com/tiledb-inc/story/34454/pass-sanitizer-flags-into-vcpkg-triple) This PR defines `x64-windows-asan`, `x64-linux-asan`, `x64-osx-asan` and `arm64-macos-asan` triplets, that pass compiler options to enable Address Sanitizer. To enable ASAN you have to specify the `TILEDB_VCPKG_BASE_TRIPLET` variable (or the related bootstrap script options) to the triplet without the `-asan` suffix. As an attempt to reduce duplication, I tried passing the flags with a CMake toolchain file, but could not make it work. --- TYPE: BUILD DESC: Add vcpkg triplets for Address Sanitizer. --------- Co-authored-by: Eric Hughes --- .github/workflows/full-ci.yml | 2 +- .github/workflows/nightly-test.yml | 3 +- CMakeLists.txt | 21 +----- CONTRIBUTING.md | 27 ++++++- bootstrap | 8 +- bootstrap.ps1 | 12 ++- cmake/Modules/Sanitizer.cmake | 105 ++++++++++++++++++++++++++ cmake/Options/BuildOptions.cmake | 2 + cmake/Options/TileDBToolchain.cmake | 34 +++++++-- cmake/TileDB-Superbuild.cmake | 2 +- ports/triplets/arm64-osx-asan.cmake | 9 +++ ports/triplets/x64-linux-asan.cmake | 8 ++ ports/triplets/x64-osx-asan.cmake | 9 +++ ports/triplets/x64-windows-asan.cmake | 9 +++ tiledb/CMakeLists.txt | 4 +- 15 files changed, 222 insertions(+), 33 deletions(-) create mode 100644 cmake/Modules/Sanitizer.cmake create mode 100644 ports/triplets/arm64-osx-asan.cmake create mode 100644 ports/triplets/x64-linux-asan.cmake create mode 100644 ports/triplets/x64-osx-asan.cmake create mode 100644 ports/triplets/x64-windows-asan.cmake diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index a710d9761fbe..a525a9cf382a 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -101,7 +101,7 @@ jobs: matrix_compiler_cc: 'gcc-10' matrix_compiler_cxx: 'g++-10' timeout: 120 - bootstrap_args: '--enable-serialization' + bootstrap_args: '--enable-serialization --vcpkg-base-triplet=x64-linux' asan: true ci10: diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index eab5a6fc375c..db03f78205ed 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -16,6 +16,7 @@ jobs: - os: macos-latest - os: macos-latest # ASAN build sanitizer: "address" + base_triplet: "x64-osx" - os: macos-latest experimental: ON - os: windows-latest @@ -46,7 +47,7 @@ jobs: EXPERIMENTAL: ${{ matrix.experimental || 'OFF' }} working-directory: ${{ matrix.working_directory || github.workspace }} run: | - cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DSANITIZER=$SANITIZER_ARG + cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }} - name: Configure TileDB CMake (Windows) if: contains(matrix.os, 'windows') diff --git a/CMakeLists.txt b/CMakeLists.txt index 23ac97ba3ccd..82b7b12fd03d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Options") include(CIConfig) include(BuildOptions) include(global-policies NO_POLICY_SCOPE) +include(Sanitizer) include(TileDBToolchain) include(Doxygen) include(Format) @@ -250,24 +251,8 @@ if (COMPILER_SUPPORTS_AVX2) add_compile_options(${COMPILER_AVX2_FLAG}) endif() -# HACK: Set the sanitizer configuration globally after the -# superbuild has finished. We want to enable sanitization for -# TileDB, but not our dependencies. -# -# The reasoning for setting this globally is due to false positives -# in the `AddressSanitizerContainerOverflow` checks [1]. -# -# [1] https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow#false-positives -if (SANITIZER) - string(TOLOWER ${SANITIZER} SANITIZER) - if (NOT SANITIZER MATCHES "^(address|memory|leak|thread|undefined)$") - message(FATAL_ERROR "Unknown clang sanitizer: ${SANITIZER})") - else() - message(STATUS "The TileDB library is compiled with sanitizer ${SANITIZER} enabled") - endif() - add_compile_options(-DTILEDB_SANITIZER=${SANITIZER}) - add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=${SANITIZER}) - add_link_options(-fsanitize=${SANITIZER}) +if(TILEDB_SANITIZER) + validate_sanitizer_options() endif() ####################################################### diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 606481f12c43..71d27bd3f953 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,27 @@ Formatting conventions: - comments are good, TileDB uses [doxygen](http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html) for class doc strings. - format code using [clang-format](https://clang.llvm.org/docs/ClangFormat.html) +### Building with sanitizers + +TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as the vcpkg base triplet corresponding to your platform. The following platforms support sanitizers: + +* `arm64-osx` +* `x64-linux` +* `x64-osx` +* `x64-windows` + +> [!NOTE] +> Currently only the `address` sanitizer is supported. + +```bash +cd TileDB && mkdir build-asan && cd build-asan +../bootstrap --enable-sanitizer=address --vcpkg-base-triplet=x64-linux +make && make check +``` + +> [!IMPORTANT] +> To avoid errors, building with sanitizers must be done in a separate build directory. + ### Pull Requests: - `dev` is the development branch, all PR’s should be rebased on top of the latest `dev` commit. @@ -78,7 +99,7 @@ Formatting conventions: - Submit a PR, writing a descriptive message. If a PR closes an open issue, reference the issue in the PR message (ex. If an issue closes issue number 10, you would write `closes #10`) -- Make sure CI (continuous integration) is passing for your PR -- click `Show all checks` in the pull request status box at the bottom of each PR page. The continous integration project pages will also list all recently-built PRs: +- Make sure CI (continuous integration) is passing for your PR -- click `Show all checks` in the pull request status box at the bottom of each PR page. The continuous integration project pages will also list all recently-built PRs: - [Azure Pipelines](https://dev.azure.com/TileDB-Inc/CI/_build) ### Documentation Pull Requests: @@ -107,8 +128,8 @@ This will install all the required packages in a Python virtual environment, and - [Organization](https://github.com/TileDB-Inc/) -* Github / Git +* GitHub / Git - [Git cheatsheet](https://services.github.com/on-demand/downloads/github-git-cheat-sheet/) - - [Github Documentation](https://help.github.com/) + - [GitHub Documentation](https://help.github.com/) - [Forking a Repo](https://help.github.com/articles/fork-a-repo/) - [More Learning Resources](https://help.github.com/articles/git-and-github-learning-resources/) diff --git a/bootstrap b/bootstrap index 4ac8f1e6c4d7..627ca30b593e 100755 --- a/bootstrap +++ b/bootstrap @@ -34,6 +34,7 @@ Configuration: --help print this message --prefix=PREFIX install files in tree rooted at PREFIX ['"${default_prefix}"'] + --vcpkg-base-triplet=TRIPLET vcpkg base triplet to use when building with sanitizers --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] @@ -52,6 +53,7 @@ Configuration: --enable-static-tiledb enables building TileDB as a static library (deprecated, use --linkage=static) --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) + Must manually specify a vcpkg base triplet. --enable-debug enable debug build --enable-assertions enable assertions in compiled code --enable-release-symbols enable create symbols for release build @@ -88,6 +90,7 @@ Example: # Parse arguments prefix_dirs="${default_prefix}" +vcpkg_base_triplet="" tiledb_vcpkg="ON" dependency_dir="${default_dependency}" sanitizer="" @@ -119,6 +122,8 @@ while test $# != 0; do case "$1" in --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; + --vcpkg-base-triplet=*) vcpkg_base_triplet=`arg "$1"` + vcpkg_base_triplet="-DTILEDB_VCPKG_BASE_TRIPLET=${vcpkg_base_triplet}";; --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` @@ -256,11 +261,12 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_WEBP=${tiledb_build_webp} \ -DTILEDB_FORCE_ALL_DEPS=${tiledb_force_all_deps} \ -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \ - -DSANITIZER="${sanitizer}" \ + -DTILEDB_SANITIZER="${sanitizer}" \ -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ ${tiledb_disable_avx2} \ + ${vcpkg_base_triplet} \ "${source_dir}" || die "failed to configure the project" echo 'bootstrap success. Run "make" to build, "make check" to test, or "make -C tiledb install" to install.' diff --git a/bootstrap.ps1 b/bootstrap.ps1 index bf49fce5d671..9bb1ee23158a 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -13,6 +13,10 @@ develop TileDB. .PARAMETER Prefix Installs files in tree rooted at PREFIX (defaults to TileDB\dist). +.PARAMETER VcpkgBaseTriplet +Optionally specify the vcpkg target triplet, e.g. "x64-windows-release". +Defaults to automatically detecting it from the environment. + .PARAMETER Dependency Semicolon separated list to binary dependencies. @@ -117,6 +121,7 @@ https://github.com/TileDB-Inc/TileDB [CmdletBinding()] Param( [string]$Prefix, + [string]$VcpkgBaseTriplet, [string]$Dependency, [string]$Linkage = "shared", [string]$CMakeGenerator, @@ -163,6 +168,11 @@ $DefaultPrefix = Join-Path $BinaryDirectory "dist" # Choose the default dependency install prefix. $DefaultDependency = $DefaultPrefix +# Set the vcpkg base triplet. +if ($VcpkgBaseTriplet.IsPresent) { + $VcpkgBaseTriplet = "-DTILEDB_VCPKG_BASE_TRIPLET=$VcpkgBaseTriplet" +} + # Set assertion mode # No-op for a debug build. $AssertionMode = "OFF" @@ -327,7 +337,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake new file mode 100644 index 000000000000..7acf3407021d --- /dev/null +++ b/cmake/Modules/Sanitizer.cmake @@ -0,0 +1,105 @@ +# +# cmake/Modules/Sanitizers.cmake +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Sanitizers +# https://github.com/google/sanitizers +# https://github.com/google/sanitizers/wiki/AddressSanitizer +# +# GCC +# all: https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html +# All the sanitizer options are here, together with all the similar options gcc offers. +# +# Clang +# address: https://clang.llvm.org/docs/AddressSanitizer.html +# Recommends linking with `clang++`. No warning if otherwise at this time. +# undefined behavior: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html +# memory: https://clang.llvm.org/docs/MemorySanitizer.html +# +# MSVC +# Only supports the address sanitizer +# https://docs.microsoft.com/en-us/cpp/sanitizers/asan +# https://docs.microsoft.com/en-us/cpp/sanitizers/asan-building +# https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ +# https://devblogs.microsoft.com/cppblog/address-sanitizer-for-msvc-now-generally-available/ + +if(NOT TILEDB_SANITIZER) + return() +endif() + +# The basic sanitizer option is standard across compilers +add_compile_options(-fsanitize=${TILEDB_SANITIZER}) + +# Check that the sanitizer name is well-formed +if (NOT TILEDB_SANITIZER MATCHES "^[-A-Za-z]*$") + message(FATAL_ERROR "Bad sanitizer specification \"${sanitizer}\";" + " permissible characters are only alphabetic and hyphen") +endif() + +# Verify that the sanitizer is one that some compiler supports +string(TOLOWER ${TILEDB_SANITIZER} TILEDB_SANITIZER) +if (NOT TILEDB_SANITIZER MATCHES "^address$") + message(FATAL_ERROR "Unsupported sanitizer ${sanitizer}") +endif() + +# Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. +add_compile_definitions("$<$:CATCH_CONFIG_NO_WINDOWS_SEH>") +# Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. +add_link_options("$<$:/INCREMENTAL:NO>") + +# Ordinary gcc/clang behavior. +add_compile_options("$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>") +add_link_options("$<$>:-fsanitize=${TILEDB_SANITIZER}>") +if(TILEDB_SANITIZER STREQUAL "address") + # There may be problems if clang tries to link the ASAN library statically + add_link_options("$<$:-shared-libasan>") +endif() + +# Validate sanitizer options. +# This must be called after the project() command, where the compiler is known. +macro(validate_sanitizer_options) + # For known compilers, check that the sanitizer is supported. + # If we know it's not supported, we'll fail so that we avoid false confidence. + # If we don't know, we'll warn that it might not work. + if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + if (TILEDB_SANITIZER STREQUAL "address") + # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 + # and was announced as "fully supported" in version 16.9 + if (MSVC_VERSION LESS 1924) + message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) + endif() + if (MSVC_VERSION LESS 1929) + message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) + endif() + else() + # MSVC support only the address sanitizer + message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") + endif() + + # May also need to explicitly remove /RTC flags + + elseif (NOT (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")) + message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") + endif() +endmacro() diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index d7ffcf72db23..fd439db47006 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -10,6 +10,8 @@ include(CMakeDependentOption) option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing dependencies)." ON) option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." ON) cmake_dependent_option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF "NOT TILEDB_VCPKG" OFF) +option(TILEDB_SANITIZER "Sets the sanitizers to use. Only address is currently supported." "") +option(TILEDB_VCPKG_BASE_TRIPLET "Sets the base vcpkg triplet when building with sanitizers." "") option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF) option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF) option(TILEDB_S3 "Enables S3/minio support using aws-cpp-sdk" OFF) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index f6ca3e16c70b..59ed7574c109 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -1,3 +1,28 @@ +# +# cmake/Options/TileDBToolchain.cmake +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ############################################################ # TileDB Toolchain Setup ############################################################ @@ -33,12 +58,11 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() endif() -if(APPLE AND NOT DEFINED VCPKG_TARGET_TRIPLET) - if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - set(VCPKG_TARGET_TRIPLET "x64-osx") - elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") - set(VCPKG_TARGET_TRIPLET "arm64-osx") +if(TILEDB_SANITIZER STREQUAL "address") + if(NOT TILEDB_VCPKG_BASE_TRIPLET) + message(FATAL_ERROR "TILEDB_VCPKG_BASE_TRIPLET must be defined when building with ASAN.") endif() + set(VCPKG_TARGET_TRIPLET "${TILEDB_VCPKG_BASE_TRIPLET}-asan") endif() set(VCPKG_INSTALL_OPTIONS "--no-print-usage") diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 5af56ab2dc76..f1802fbcb1a6 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -45,7 +45,7 @@ set(INHERITED_CMAKE_ARGS -DTILEDB_CPP_API=${TILEDB_CPP_API} -DTILEDB_FORCE_ALL_DEPS=${TILEDB_FORCE_ALL_DEPS} -DTILEDB_REMOVE_DEPRECATIONS=${TILEDB_REMOVE_DEPRECATIONS} - -DSANITIZER=${SANITIZER} + -DTILEDB_SANITIZER=${TILEDB_SANITIZER} -DTILEDB_EP_BASE=${TILEDB_EP_BASE} -DTILEDB_STATS=${TILEDB_STATS} -DTILEDB_TESTS=${TILEDB_TESTS} diff --git a/ports/triplets/arm64-osx-asan.cmake b/ports/triplets/arm64-osx-asan.cmake new file mode 100644 index 000000000000..4aa2d22af543 --- /dev/null +++ b/ports/triplets/arm64-osx-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-linux-asan.cmake b/ports/triplets/x64-linux-asan.cmake new file mode 100644 index 000000000000..747d7d620207 --- /dev/null +++ b/ports/triplets/x64-linux-asan.cmake @@ -0,0 +1,8 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-osx-asan.cmake b/ports/triplets/x64-osx-asan.cmake new file mode 100644 index 000000000000..aac0688e2c7c --- /dev/null +++ b/ports/triplets/x64-osx-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-windows-asan.cmake b/ports/triplets/x64-windows-asan.cmake new file mode 100644 index 000000000000..d19486d2cfed --- /dev/null +++ b/ports/triplets/x64-windows-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) + +# bigobj is needed for capnp. +set(VCPKG_C_FLAGS "/fsanitize=address /bigobj") +set(VCPKG_CXX_FLAGS "/fsanitize=address /bigobj") diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 530d2b8c68ba..3cc6cea153ce 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -598,10 +598,10 @@ if (TILEDB_HDFS) endif() # Sanitizer linker flags -if (SANITIZER) +if (TILEDB_SANITIZER) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE - -fsanitize=${SANITIZER} + -fsanitize=${TILEDB_SANITIZER} ) endif() From b35ecdc3b1f8202b1f918f3e26b91d2fe85a91e5 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Fri, 22 Dec 2023 06:51:24 -0500 Subject: [PATCH 117/456] Add internal definitions for S3 ls recursive (#4467) This adds internal definitions for `sm::VFS::ls_recursive` and `S3::ls_filtered` for S3 only. Results are collected using a `S3Scanner` class that manages fetching results from S3, and advances `LsScanIterator` until the next accepted result is reached. The `S3Scanner::iterator()` method returns an iterator for the scan which can be passed to STL constructors or algorithms using input iterators. --- TYPE: FEATURE DESC: Add internal definitions for S3 ls recursive --- test/src/unit-s3.cc | 113 +++++ test/src/unit-vfs.cc | 91 ++++ test/support/src/vfs_helpers.cc | 87 +++- test/support/src/vfs_helpers.h | 170 ++++++- tiledb/CMakeLists.txt | 1 + tiledb/sm/filesystem/CMakeLists.txt | 1 + tiledb/sm/filesystem/filesystem_base.h | 4 +- tiledb/sm/filesystem/ls_scanner.cc | 33 ++ tiledb/sm/filesystem/ls_scanner.h | 244 ++++++++++ tiledb/sm/filesystem/s3.cc | 119 +---- tiledb/sm/filesystem/s3.h | 441 +++++++++++++++++- tiledb/sm/filesystem/test/CMakeLists.txt | 2 +- tiledb/sm/filesystem/test/unit_ls_filtered.cc | 124 +++++ tiledb/sm/filesystem/vfs.cc | 52 +-- tiledb/sm/filesystem/vfs.h | 53 ++- 15 files changed, 1354 insertions(+), 181 deletions(-) create mode 100644 tiledb/sm/filesystem/ls_scanner.cc create mode 100644 tiledb/sm/filesystem/ls_scanner.h create mode 100644 tiledb/sm/filesystem/test/unit_ls_filtered.cc diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index 340ab95d433f..219cfa98f750 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -34,6 +34,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/s3.h" @@ -546,4 +547,116 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") { try_with_bucket_object_canned_acl("public_read_write", "public_read_write"); try_with_bucket_object_canned_acl("authenticated_read", "authenticated_read"); } + +TEST_CASE( + "S3: S3Scanner iterator to populate vector", "[s3][ls-scan-iterator]") { + S3Test s3_test({10, 50}); + bool recursive = true; + // 1000 is the default max_keys for S3. This is the same default used by + // S3Scanner. Testing with small max_keys validates the iterator handles batch + // collection and filtering appropriately. + int max_keys = GENERATE(1000, 10, 7); + + DYNAMIC_SECTION("Testing with " << max_keys << " max keys from S3") { + FileFilter file_filter; + auto expected = s3_test.expected_results(); + + SECTION("Accept all objects") { + file_filter = [](const std::string_view&, uint64_t) { return true; }; + std::sort(expected.begin(), expected.end()); + } + + SECTION("Reject all objects") { + file_filter = [](const std::string_view&, uint64_t) { return false; }; + } + + SECTION("Filter objects including 'test_file_1' in key") { + file_filter = [](const std::string_view& path, uint64_t) { + if (path.find("test_file_1") != std::string::npos) { + return true; + } + return false; + }; + } + + SECTION("Scan for a single object") { + file_filter = [](const std::string_view& path, uint64_t) { + if (path.find("test_file_50") != std::string::npos) { + return true; + } + return false; + }; + } + + // Filter expected results to apply file_filter. + std::erase_if(expected, [&file_filter](const auto& a) { + return !file_filter(a.first, a.second); + }); + + auto scan = s3_test.get_s3().scanner( + s3_test.temp_dir_, file_filter, accept_all_dirs, recursive, max_keys); + std::vector results_vector(scan.begin(), scan.end()); + + CHECK(results_vector.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + auto s3_object = results_vector[i]; + CHECK(file_filter(s3_object.GetKey(), s3_object.GetSize())); + auto full_uri = s3_test.temp_dir_.to_string() + "/" + s3_object.GetKey(); + CHECK(full_uri == expected[i].first); + CHECK(static_cast(s3_object.GetSize()) == expected[i].second); + } + } +} + +TEST_CASE("S3: S3Scanner iterator", "[s3][ls-scan-iterator]") { + S3Test s3_test({10, 50, 7}); + bool recursive = true; + int max_keys = GENERATE(1000, 11); + + std::vector results_vector; + DYNAMIC_SECTION("Testing with " << max_keys << " max keys from S3") { + auto scan = s3_test.get_s3().scanner( + s3_test.temp_dir_, + VFSTest::accept_all_files, + accept_all_dirs, + recursive, + max_keys); + + SECTION("for loop") { + SECTION("range based for") { + for (const auto& result : scan) { + results_vector.push_back(result); + } + } + SECTION("prefix operator") { + for (auto it = scan.begin(); it != scan.end(); ++it) { + results_vector.push_back(*it); + } + } + SECTION("postfix operator") { + for (auto it = scan.begin(); it != scan.end(); it++) { + results_vector.push_back(*it); + } + } + } + + SECTION("vector::assign") { + results_vector.assign(scan.begin(), scan.end()); + } + + SECTION("std::move") { + std::move(scan.begin(), scan.end(), std::back_inserter(results_vector)); + } + } + + auto expected = s3_test.expected_results(); + CHECK(results_vector.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + auto s3_object = results_vector[i]; + auto full_uri = s3_test.temp_dir_.to_string() + "/" + s3_object.GetKey(); + CHECK(full_uri == expected[i].first); + CHECK(static_cast(s3_object.GetSize()) == expected[i].second); + } +} + #endif diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 78ed7203b2b2..eee3542d1d6b 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -37,6 +37,7 @@ #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" #endif +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/tile/tile.h" using namespace tiledb::common; @@ -340,3 +341,93 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { // Clean up REQUIRE(vfs_ls.remove_dir(URI(path)).ok()); } + +// Currently only S3 is supported for VFS::ls_recursive. +using TestBackends = std::tuple; +TEMPLATE_LIST_TEST_CASE( + "VFS: Test internal ls_filtered recursion argument", + "[vfs][ls_filtered][recursion]", + TestBackends) { + TestType fs({10, 50}); + if (!fs.is_supported()) { + return; + } + + bool recursive = GENERATE(true, false); + DYNAMIC_SECTION( + fs.temp_dir_.backend_name() + << " ls_filtered with recursion: " << (recursive ? "true" : "false")) { +#ifdef HAVE_S3 + // If testing with recursion use the root directory, otherwise use a subdir. + auto path = recursive ? fs.temp_dir_ : fs.temp_dir_.join_path("subdir_1"); + auto ls_objects = fs.get_s3().ls_filtered( + path, VFSTestBase::accept_all_files, accept_all_dirs, recursive); + + auto expected = fs.expected_results(); + if (!recursive) { + // If non-recursive, all objects in the first directory should be + // returned. + std::erase_if(expected, [](const auto& p) { + return p.first.find("subdir_1") == std::string::npos; + }); + } + + CHECK(ls_objects.size() == expected.size()); + CHECK(ls_objects == expected); +#endif + } +} + +TEST_CASE( + "VFS: ls_recursive throws for unsupported backends", + "[vfs][ls_recursive]") { + // Local and mem fs tests are in tiledb/sm/filesystem/test/unit_ls_filtered.cc + std::string prefix = GENERATE("s3://", "hdfs://", "azure://", "gcs://"); + VFSTest vfs_test({1}, prefix); + if (!vfs_test.is_supported()) { + return; + } + std::string backend = vfs_test.temp_dir_.backend_name(); + + if (vfs_test.temp_dir_.is_s3()) { + DYNAMIC_SECTION(backend << " supported backend should not throw") { + CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, VFSTestBase::accept_all_files)); + } + } else { + DYNAMIC_SECTION(backend << " unsupported backend should throw") { + CHECK_THROWS_WITH( + vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, VFSTestBase::accept_all_files), + Catch::Matchers::ContainsSubstring( + "storage backend is not supported")); + } + } +} + +TEST_CASE( + "VFS: Throwing FileFilter for ls_recursive", + "[vfs][ls_recursive][file-filter]") { + std::string prefix = "s3://"; + VFSTest vfs_test({0}, prefix); + if (!vfs_test.is_supported()) { + return; + } + + auto file_filter = [](const std::string_view&, uint64_t) -> bool { + throw std::logic_error("Throwing FileFilter"); + }; + SECTION("Throwing FileFilter with 0 objects should not throw") { + CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, file_filter, tiledb::sm::accept_all_dirs)); + } + SECTION("Throwing FileFilter with N objects should throw") { + vfs_test.vfs_.touch(vfs_test.temp_dir_.join_path("file")).ok(); + CHECK_THROWS_AS( + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + std::logic_error); + CHECK_THROWS_WITH( + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + Catch::Matchers::ContainsSubstring("Throwing FileFilter")); + } +} diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index 285981a81482..14bde730bd1a 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -42,12 +42,15 @@ // after tdb_catch.h. #include "test/support/src/serialization_wrappers.h" -#include #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" namespace tiledb::test { +tiledb::sm::URI test_dir(const std::string& prefix) { + return tiledb::sm::URI(prefix + "tiledb-" + std::to_string(PRNG::get()())); +} + std::vector> vfs_test_get_fs_vec() { std::vector> fs_vec; @@ -414,4 +417,86 @@ std::string TemporaryDirectoryFixture::create_temporary_array( return array_uri; } +VFSTestBase::VFSTestBase( + const std::vector& test_tree, const std::string& prefix) + : test_tree_(test_tree) + , compute_(4) + , io_(4) + , vfs_(&tiledb::test::g_helper_stats, &io_, &compute_, create_test_config()) + , prefix_(prefix) + , temp_dir_(tiledb::test::test_dir(prefix_)) + , is_supported_(vfs_.supports_uri_scheme(temp_dir_)) { + // TODO: Throw when we can provide a list of supported filesystems to Catch2. +} + +VFSTestBase::~VFSTestBase() { + if (vfs_.supports_uri_scheme(temp_dir_)) { + bool is_dir = false; + vfs_.is_dir(temp_dir_, &is_dir).ok(); + if (is_dir) { + vfs_.remove_dir(temp_dir_).ok(); + } + } +} + +tiledb::sm::Config VFSTestBase::create_test_config() { + tiledb::sm::Config cfg; + if constexpr (!tiledb::test::aws_s3_config) { + // Set up connection to minio backend emulator. + cfg.set("vfs.s3.endpoint_override", "localhost:9999").ok(); + cfg.set("vfs.s3.scheme", "https").ok(); + cfg.set("vfs.s3.use_virtual_addressing", "false").ok(); + cfg.set("vfs.s3.verify_ssl", "false").ok(); + } + cfg.set("vfs.azure.storage_account_name", "devstoreaccount1").ok(); + cfg.set( + "vfs.azure.storage_account_key", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==") + .ok(); + cfg.set("vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1") + .ok(); + return cfg; +} + +VFSTest::VFSTest( + const std::vector& test_tree, const std::string& prefix) + : VFSTestBase(test_tree, prefix) { + if (!is_supported()) { + return; + } + + if (temp_dir_.is_file() || temp_dir_.is_memfs() || temp_dir_.is_hdfs()) { + vfs_.create_dir(temp_dir_).ok(); + } else { + vfs_.create_bucket(temp_dir_).ok(); + } + for (size_t i = 1; i <= test_tree_.size(); i++) { + sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); + // VFS::create_dir is a no-op for S3. + vfs_.create_dir(path).ok(); + for (size_t j = 1; j <= test_tree_[i - 1]; j++) { + auto object_uri = path.join_path("test_file_" + std::to_string(j)); + vfs_.touch(object_uri).ok(); + std::string data(j * 10, 'a'); + vfs_.open_file(object_uri, sm::VFSMode::VFS_WRITE).ok(); + vfs_.write(object_uri, data.data(), data.size()).ok(); + vfs_.close_file(object_uri).ok(); + expected_results().emplace_back(object_uri.to_string(), data.size()); + } + } + std::sort(expected_results().begin(), expected_results().end()); +} + +LocalFsTest::LocalFsTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "file://") { +#ifdef _WIN32 + temp_dir_ = + tiledb::test::test_dir(prefix_ + tiledb::sm::Win::current_dir() + "/"); +#else + temp_dir_ = + tiledb::test::test_dir(prefix_ + tiledb::sm::Posix::current_dir() + "/"); +#endif +} + } // namespace tiledb::test diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index fea0bf0efa51..8416ce889d69 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -33,21 +33,32 @@ #ifndef TILEDB_VFS_HELPERS_H #define TILEDB_VFS_HELPERS_H +#include #include #include "test/support/src/helpers.h" -#include "test/support/tdb_catch.h" - -#ifdef _WIN32 -#include "tiledb/sm/filesystem/win.h" -#else -#include "tiledb/sm/filesystem/posix.h" -#endif +#include "tiledb/sm/enums/vfs_mode.h" +#include "tiledb/sm/filesystem/vfs.h" namespace tiledb::test { // Forward declaration class SupportedFs; +#ifdef TILEDB_TESTS_AWS_CONFIG +constexpr bool aws_s3_config = true; +#else +constexpr bool aws_s3_config = false; +#endif + +/** + * Generates a random temp directory URI for use in VFS tests. + * + * @param prefix A prefix to use for the temp directory name. Should include + * `s3://`, `mem://` or other URI prefix for the backend. + * @return URI which the caller can use to create a temp directory. + */ +tiledb::sm::URI test_dir(const std::string& prefix); + /** * Create the vector of supported filesystems. */ @@ -773,6 +784,151 @@ class DenyWriteAccess { const std::filesystem::perms previous_perms_; }; +/** + * Base class use for VFS and file system test objects. Deriving classes are + * responsible for creating a temporary directory and populating it with test + * objects for the related file system. + */ +class VFSTestBase { + protected: + /** + * Requires derived class to create a temporary directory. + * + * @param test_tree Vector used to build test directory and objects. + * For each element we create a nested directory with N objects. + * @param prefix The URI prefix to use for the test directory. + */ + VFSTestBase(const std::vector& test_tree, const std::string& prefix); + + public: + /** Type definition for objects returned from ls_recursive */ + using LsObjects = std::vector>; + + virtual ~VFSTestBase(); + + /** + * @return True if the URI prefix is supported by the build, else false. + */ + inline bool is_supported() const { + return is_supported_; + } + + inline LsObjects& expected_results() { + return expected_results_; + } + + /** + * Creates a config for testing VFS storage backends over local emulators. + * + * @return Fully initialized configuration for testing VFS storage backends. + */ + static tiledb::sm::Config create_test_config(); + + /** FilePredicate for passing to ls_filtered that accepts all files. */ + static bool accept_all_files(const std::string_view&, uint64_t) { + return true; + } + + std::vector test_tree_; + ThreadPool compute_, io_; + tiledb::sm::VFS vfs_; + std::string prefix_; + tiledb::sm::URI temp_dir_; + + private: + LsObjects expected_results_; + bool is_supported_; +}; + +/** + * Test object for tiledb::sm::VFS functionality. When constructed, this test + * object creates a temporary directory and populates it using the test_tree + * vector passed to the constructor. For each element in the vector, we create a + * nested directory with N objects. The constructor also writes `10 * N` bytes + * of data to each object created for testing returned object sizes are correct. + * + * This test object can be used for any valid VFS URI prefix, and is not + * specific to any one backend. + */ +class VFSTest : public VFSTestBase { + public: + VFSTest(const std::vector& test_tree, const std::string& prefix); +}; + +/** Test object for tiledb::sm::S3 functionality. */ +class S3Test : public VFSTestBase, protected tiledb::sm::S3_within_VFS { + public: + explicit S3Test(const std::vector& test_tree) + : VFSTestBase(test_tree, "s3://") + , S3_within_VFS(&tiledb::test::g_helper_stats, &io_, vfs_.config()) { +#ifdef HAVE_S3 + s3().create_bucket(temp_dir_).ok(); + for (size_t i = 1; i <= test_tree_.size(); i++) { + sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); + // VFS::create_dir is a no-op for S3; Just create objects. + for (size_t j = 1; j <= test_tree_[i - 1]; j++) { + auto object_uri = path.join_path("test_file_" + std::to_string(j)); + s3().touch(object_uri).ok(); + std::string data(j * 10, 'a'); + s3().write(object_uri, data.data(), data.size()).ok(); + s3().flush_object(object_uri).ok(); + expected_results().emplace_back(object_uri.to_string(), data.size()); + } + } + std::sort(expected_results().begin(), expected_results().end()); +#endif + } + +#ifdef HAVE_S3 + /** Expose protected accessor from S3_within_VFS. */ + tiledb::sm::S3& get_s3() { + return s3(); + } + + /** Expose protected const accessor from S3_within_VFS. */ + const tiledb::sm::S3& get_s3() const { + return s3(); + } +#endif +}; + +/** Stub test object for tiledb::sm::Win and Posix functionality. */ +class LocalFsTest : public VFSTestBase { + public: + explicit LocalFsTest(const std::vector& test_tree); +}; + +/** Stub test object for tiledb::sm::Azure functionality. */ +class AzureTest : public VFSTestBase { + public: + explicit AzureTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "azure://") { + } +}; + +/** Stub test object for tiledb::sm::GCS functionality. */ +class GCSTest : public VFSTestBase { + public: + explicit GCSTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "gcs://") { + } +}; + +/** Stub test object for tiledb::sm::HDFS functionality. */ +class HDFSTest : public VFSTestBase { + public: + explicit HDFSTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "hdfs://") { + } +}; + +/** Stub test object for tiledb::sm::MemFilesystem functionality. */ +class MemFsTest : public VFSTestBase { + public: + explicit MemFsTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "mem://") { + } +}; } // namespace tiledb::test #endif diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 3cc6cea153ce..b9e02b88725b 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -201,6 +201,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/uri.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs_file_handle.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/ls_scanner.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/win.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/filesystem_base.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/bit_width_reduction_filter.cc diff --git a/tiledb/sm/filesystem/CMakeLists.txt b/tiledb/sm/filesystem/CMakeLists.txt index 121829379347..fd0e1f34a40e 100644 --- a/tiledb/sm/filesystem/CMakeLists.txt +++ b/tiledb/sm/filesystem/CMakeLists.txt @@ -38,6 +38,7 @@ commence(object_library vfs) uri.cc vfs.cc vfs_file_handle.cc + ls_scanner.cc win.cc filesystem_base.cc ../curl/curl_init.cc diff --git a/tiledb/sm/filesystem/filesystem_base.h b/tiledb/sm/filesystem/filesystem_base.h index b1e9409bc3ef..89be313d1163 100644 --- a/tiledb/sm/filesystem/filesystem_base.h +++ b/tiledb/sm/filesystem/filesystem_base.h @@ -1,5 +1,5 @@ /** - * @file filesystem.h + * @file filesystem_base.h * * @section LICENSE * @@ -40,8 +40,6 @@ namespace tiledb::sm { -class VFS; - class FilesystemBase { public: FilesystemBase() = default; diff --git a/tiledb/sm/filesystem/ls_scanner.cc b/tiledb/sm/filesystem/ls_scanner.cc new file mode 100644 index 000000000000..bcccd82870ec --- /dev/null +++ b/tiledb/sm/filesystem/ls_scanner.cc @@ -0,0 +1,33 @@ +/** + * @file ls_scanner.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This defines the LsScanner class and related types used for VFS. + */ + +#include "ls_scanner.h" diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h new file mode 100644 index 000000000000..724ae667e05e --- /dev/null +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -0,0 +1,244 @@ +/** + * @file ls_scanner.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This defines the LsScanner class and related types used for VFS. + */ + +#ifndef TILEDB_LS_SCANNER_H +#define TILEDB_LS_SCANNER_H + +#include "tiledb/common/exception/exception.h" +#include "tiledb/sm/filesystem/uri.h" + +#include +#include +#include + +/** Inclusion predicate for objects collected by ls */ +template +concept FilePredicate = true; + +/** + * DirectoryPredicate is currently unused, but is kept here for adding directory + * pruning support in the future. + */ +template +concept DirectoryPredicate = true; + +namespace tiledb::sm { +class LsScanException : public StatusException { + public: + explicit LsScanException(const std::string& message) + : StatusException("LsScan", message) { + } +}; + +using FileFilter = std::function; + +using DirectoryFilter = std::function; +/** Static DirectoryFilter used as default argument. */ +[[maybe_unused]] static bool accept_all_dirs(const std::string_view&) { + return true; +} + +/** Type defintion for objects returned from ls_recursive. */ +using LsObjects = std::vector>; + +/** + * LsScanIterator iterates over the results of ls requests wrapped by classes + * deriving from LsScanner. See S3Scanner as an example. + * + * The iterator is implemented as an input iterator, where the end() iterator + * is default constructed, resulting in an LsScanIterator::ptr_ that is a + * non-dereferencable iterator. + * + * + * @tparam scanner_type The LsScanner type that created this iterator. + * @tparam T The data type stored by this iterator. + * TODO: Discuss using T for iterator type instead of underlying data type. + */ +template +class LsScanIterator { + public: + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = typename std::vector::const_iterator; + using reference = const T&; + using iterator_category = std::input_iterator_tag; + + /** + * Default constructor. + * + * This constructor is used to construct the end() iterator, where ptr_ is + * default constructed as a non-de-referencable iterator. + */ + LsScanIterator() noexcept = default; + + /** + * Constructor. + * + * @param scanner The scanner that created this iterator. + */ + explicit LsScanIterator(scanner_type* scanner) noexcept + : scanner_(scanner) + , ptr_(scanner->begin_) { + } + + /** + * Constructor. + * + * @param scanner The scanner that created this iterator. + * @param ptr Pointer to set as the current object. + */ + LsScanIterator(scanner_type* scanner, pointer ptr) noexcept + : scanner_(scanner) + , ptr_(ptr) { + } + + /** Copy constructor. */ + LsScanIterator(const LsScanIterator& rhs) noexcept { + ptr_ = rhs.ptr_; + scanner_ = rhs.scanner_; + } + + /** Copy assignment operator. */ + LsScanIterator& operator=(LsScanIterator rhs) noexcept { + if (&rhs != this) { + std::swap(ptr_, rhs.ptr_); + std::swap(scanner_, rhs.scanner_); + } + return *this; + } + + inline void ensure_dereferenceable() const { + if (ptr_ == pointer()) { + throw LsScanException("Failed to dereference invalid iterator."); + } + } + + /** + * Dereference operator. + * + * @return The current object being visited. + */ + constexpr reference operator*() const { + ensure_dereferenceable(); + return *ptr_; + } + + /** + * Dereference operator. + * + * @return The current object being visited. + */ + constexpr pointer operator->() const { + ensure_dereferenceable(); + return ptr_; + } + + /** + * Prefix increment operator. + * Calls the scanner's next() method to advance to the next object. + * + * @return Reference to this iterator after advancing to the next object. + */ + LsScanIterator& operator++() { + if (++ptr_ != pointer()) { + scanner_->next(ptr_); + } + return *this; + } + + /** + * Postfix increment operator. + * Calls next() method to advance to the next object via prefix operator. + * + * @return Copy of this iterator prior to advancing to the next object. + */ + LsScanIterator operator++(int) { + LsScanIterator tmp(*this); + operator++(); + return tmp; + } + + /** Inequality operator. */ + friend constexpr bool operator!=( + const LsScanIterator& lhs, const LsScanIterator& rhs) noexcept { + return !(lhs.ptr_ == rhs.ptr_); + } + + /** Equality operator. */ + friend constexpr bool operator==( + const LsScanIterator& lhs, const LsScanIterator& rhs) noexcept { + return lhs.ptr_ == rhs.ptr_; + } + + private: + /** Pointer to the scanner that created this iterator. */ + scanner_type* scanner_; + + /** Pointer to the current object. */ + pointer ptr_; +}; + +/** + * LsScanner is a base class for scanning a filesystem for objects that match + * the given file and directory predicates. This should be used as a common + * base class for future filesystem scanner implementations, similar to + * S3Scanner. + * + * @tparam F The FilePredicate type used to filter object results. + * @tparam D The DirectoryPredicate type used to prune prefix results. + */ +template +class LsScanner { + public: + /** Constructor. */ + LsScanner( + const URI& prefix, F file_filter, D dir_filter, bool recursive = false) + : prefix_(prefix) + , file_filter_(file_filter) + , dir_filter_(dir_filter) + , is_recursive_(recursive) { + } + + protected: + /** URI prefix being scanned and filtered for results. */ + const URI prefix_; + /** File predicate used to filter file or object results. */ + const F file_filter_; + /** Directory predicate used to prune directory or prefix results. */ + const D dir_filter_; + /** Whether or not to recursively scan the prefix. */ + const bool is_recursive_; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_LS_SCANNER_H diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index f645c6149486..c85dc30125cf 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -162,67 +162,6 @@ using namespace tiledb::common; namespace tiledb::sm { -namespace { - -/** - * Return the exception name and error message from the given outcome object. - * - * @tparam R AWS result type - * @tparam E AWS error type - * @param outcome Outcome to retrieve error message from - * @return Error message string - */ -template -std::string outcome_error_message(const Aws::Utils::Outcome& outcome) { - if (outcome.IsSuccess()) { - return "Success"; - } - - auto err = outcome.GetError(); - Aws::StringStream ss; - - ss << "[Error Type: " << static_cast(err.GetErrorType()) << "]" - << " [HTTP Response Code: " << static_cast(err.GetResponseCode()) - << "]"; - - if (!err.GetExceptionName().empty()) { - ss << " [Exception: " << err.GetExceptionName() << "]"; - } - - // For some reason, these symbols are not exposed when building with MINGW - // so for now we just disable adding the tags on Windows. - if constexpr (!platform::is_os_windows) { - if (!err.GetRemoteHostIpAddress().empty()) { - ss << " [Remote IP: " << err.GetRemoteHostIpAddress() << "]"; - } - - if (!err.GetRequestId().empty()) { - ss << " [Request ID: " << err.GetRequestId() << "]"; - } - } - - if (err.GetResponseHeaders().size() > 0) { - ss << " [Headers:"; - for (auto&& h : err.GetResponseHeaders()) { - ss << " '" << h.first << "' = '" << h.second << "'"; - } - ss << "]"; - } - - ss << " : " << err.GetMessage(); - - return ss.str(); -} - -} // namespace - -class S3Exception : public StatusException { - public: - explicit S3Exception(const std::string& message) - : StatusException("S3", message) { - } -}; - S3Parameters::Headers S3Parameters::load_headers(const Config& cfg) { Headers ret; auto iter = ConfigIter(cfg, constants::s3_header_prefix); @@ -236,58 +175,6 @@ S3Parameters::Headers S3Parameters::load_headers(const Config& cfg) { return ret; } -/** - * Helper class which overrides Aws::S3::S3Client to set headers from - * vfs.s3.custom_headers.* - * - * @note The AWS SDK does not have a common base class, so there's no - * straightforward way to add a header to a unique request before submitting - * it. This class exists solely to override the S3Client, adding custom headers - * upon building the Http Request. - */ -class TileDBS3Client : public Aws::S3::S3Client { - public: - TileDBS3Client( - const S3Parameters& s3_params, - const Aws::Client::ClientConfiguration& client_config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, - bool use_virtual_addressing) - : Aws::S3::S3Client(client_config, sign_payloads, use_virtual_addressing) - , params_(s3_params) { - } - - TileDBS3Client( - const S3Parameters& s3_params, - const std::shared_ptr& creds, - const Aws::Client::ClientConfiguration& client_config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, - bool use_virtual_addressing) - : Aws::S3::S3Client( - creds, client_config, sign_payloads, use_virtual_addressing) - , params_(s3_params) { - } - - virtual void BuildHttpRequest( - const Aws::AmazonWebServiceRequest& request, - const std::shared_ptr& httpRequest) - const override { - S3Client::BuildHttpRequest(request, httpRequest); - - // Set header from S3Parameters custom headers - for (auto& [key, val] : params_.custom_headers_) { - httpRequest->SetHeaderValue(key, val); - } - } - - protected: - /** - * A reference to the S3 configuration parameters, which stores the header. - * - * @note Until the removal of init_client(), this must be const-qualified. - */ - const S3Parameters& params_; -}; - /* ********************************* */ /* GLOBAL VARIABLES */ /* ********************************* */ @@ -1626,17 +1513,17 @@ Status S3::fill_file_buffer( return Status::Ok(); } -std::string S3::add_front_slash(const std::string& path) const { +std::string S3::add_front_slash(const std::string& path) { return (path.front() != '/') ? (std::string("/") + path) : path; } -std::string S3::remove_front_slash(const std::string& path) const { +std::string S3::remove_front_slash(const std::string& path) { if (path.front() == '/') return path.substr(1, path.length()); return path; } -std::string S3::remove_trailing_slash(const std::string& path) const { +std::string S3::remove_trailing_slash(const std::string& path) { if (path.back() == '/') { return path.substr(0, path.length() - 1); } diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 038472876bb9..1bdf02107794 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -34,10 +34,15 @@ #define TILEDB_S3_H #ifdef HAVE_S3 + +#include "filesystem_base.h" +#include "ls_scanner.h" #include "tiledb/common/common.h" +#include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/rwlock.h" #include "tiledb/common/status.h" #include "tiledb/common/thread_pool.h" +#include "tiledb/platform/platform.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/curl/curl_init.h" @@ -86,7 +91,67 @@ using tiledb::common::filesystem::directory_entry; namespace tiledb::sm { -class TileDBS3Client; +/** Class for S3 status exceptions. */ +class S3Exception : public StatusException { + public: + explicit S3Exception(const std::string& msg) + : StatusException("S3", msg) { + } +}; + +namespace { + +/** + * Return the exception name and error message from the given outcome object. + * + * @tparam R AWS result type + * @tparam E AWS error type + * @param outcome Outcome to retrieve error message from + * @return Error message string + */ +template +std::string outcome_error_message(const Aws::Utils::Outcome& outcome) { + if (outcome.IsSuccess()) { + return "Success"; + } + + auto err = outcome.GetError(); + Aws::StringStream ss; + + ss << "[Error Type: " << static_cast(err.GetErrorType()) << "]" + << " [HTTP Response Code: " << static_cast(err.GetResponseCode()) + << "]"; + + if (!err.GetExceptionName().empty()) { + ss << " [Exception: " << err.GetExceptionName() << "]"; + } + + // For some reason, these symbols are not exposed when building with MINGW + // so for now we just disable adding the tags on Windows. + if constexpr (!platform::is_os_windows) { + if (!err.GetRemoteHostIpAddress().empty()) { + ss << " [Remote IP: " << err.GetRemoteHostIpAddress() << "]"; + } + + if (!err.GetRequestId().empty()) { + ss << " [Request ID: " << err.GetRequestId() << "]"; + } + } + + if (err.GetResponseHeaders().size() > 0) { + ss << " [Headers:"; + for (auto&& h : err.GetResponseHeaders()) { + ss << " '" << h.first << "' = '" << h.second << "'"; + } + ss << "]"; + } + + ss << " : " << err.GetMessage(); + + return ss.str(); +} + +} // namespace /** * The s3-specific configuration parameters. @@ -271,6 +336,219 @@ struct S3Parameters { std::string config_source_; }; +/** + * Helper class which overrides Aws::S3::S3Client to set headers from + * vfs.s3.custom_headers.* + * + * @note The AWS SDK does not have a common base class, so there's no + * straightforward way to add a header to a unique request before submitting + * it. This class exists solely to override the S3Client, adding custom headers + * upon building the Http Request. + */ +class TileDBS3Client : public Aws::S3::S3Client { + public: + TileDBS3Client( + const S3Parameters& s3_params, + const Aws::Client::ClientConfiguration& client_config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, + bool use_virtual_addressing) + : Aws::S3::S3Client(client_config, sign_payloads, use_virtual_addressing) + , params_(s3_params) { + } + + TileDBS3Client( + const S3Parameters& s3_params, + const std::shared_ptr& creds, + const Aws::Client::ClientConfiguration& client_config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy sign_payloads, + bool use_virtual_addressing) + : Aws::S3::S3Client( + creds, client_config, sign_payloads, use_virtual_addressing) + , params_(s3_params) { + } + + void BuildHttpRequest( + const Aws::AmazonWebServiceRequest& request, + const std::shared_ptr& httpRequest) + const override { + S3Client::BuildHttpRequest(request, httpRequest); + + // Set header from S3Parameters custom headers + for (auto& [key, val] : params_.custom_headers_) { + httpRequest->SetHeaderValue(key, val); + } + } + + inline bool requester_pays() const { + return params_.requester_pays_; + } + + protected: + /** + * A reference to the S3 configuration parameters, which stores the header. + * + * @note Until the removal of init_client(), this must be const-qualified. + */ + const S3Parameters& params_; +}; + +/** + * S3Scanner wraps the AWS ListObjectsV2 request and provides an iterator for + * results. If we reach the end of the current batch of results and results are + * truncated, we fetch the next batch of results from S3. + * + * For each batch of results collected by fetch_results(), the begin_ and end_ + * members are initialized to the first and last elements of the batch. The scan + * steps through each result in the range [begin_, end_), using next() to + * advance to the next object accepted by the filters for this scan. + * + * @section Known Defect + * The iterators of S3Scanner are initialized by the AWS ListObjectsV2 + * results and there is no way to determine if they are different from + * iterators returned by a previous request. To be able to detect this, we + * can track the batch number and compare it to the batch number associated + * with the iterator returned by the previous request. Batch number can be + * tracked by the total number of times we submit a ListObjectsV2 request + * within fetch_results(). + * + * @tparam F The FilePredicate type used to filter object results. + * @tparam D The DirectoryPredicate type used to prune prefix results. + */ +template +class S3Scanner : public LsScanner { + public: + /** Declare LsScanIterator as a friend class for access to call next(). */ + template + friend class LsScanIterator; + using Iterator = LsScanIterator, Aws::S3::Model::Object>; + + /** Constructor. */ + S3Scanner( + const std::shared_ptr& client, + const URI& prefix, + F file_filter, + D dir_filter = accept_all_dirs, + bool recursive = false, + int max_keys = 1000); + + /** + * Returns true if there are more results to fetch from S3. + */ + [[nodiscard]] inline bool more_to_fetch() const { + return list_objects_outcome_.GetResult().GetIsTruncated(); + } + + /** + * @return Iterator to the beginning of the results being iterated on. + * Input iterators are single-pass, so we return a copy of this iterator at + * it's current position. + */ + Iterator begin() { + return Iterator(this); + } + + /** + * @return Default constructed iterator, which marks the end of results using + * nullptr. + */ + Iterator end() { + return Iterator(); + } + + private: + /** + * Advance to the next object accepted by the filters for this scan. + * + * @param ptr Reference to the current data pointer. + * @sa LsScanIterator::operator++() + */ + void next(typename Iterator::pointer& ptr); + + /** + * If the iterator is at the end of the current batch, this will fetch the + * next batch of results from S3. This does not check if the results are + * accepted by the filters for this scan. + * + * @param ptr Reference to the current data iterator. + */ + void advance(typename Iterator::pointer& ptr) { + ptr++; + if (ptr == end_) { + if (more_to_fetch()) { + // Fetch results and reset the iterator. + ptr = fetch_results(); + } else { + // Set the pointer to nullptr to indicate the end of results. + end_ = ptr = typename Iterator::pointer(); + } + } + } + + /** + * Fetch the next batch of results from S3. This also handles setting the + * continuation token for the next request, if the results were truncated. + * + * @return A pointer to the first result in the new batch. The return value + * is used to update the pointer managed by the iterator during traversal. + * If the request returned no results, this will return nullptr to mark the + * end of the scan. + * @sa LsScanIterator::operator++() + * @sa S3Scanner::next(typename Iterator::pointer&) + */ + typename Iterator::pointer fetch_results() { + // If this is our first request, GetIsTruncated() will be false. + if (more_to_fetch()) { + // If results are truncated on a subsequent request, we set the next + // continuation token before resubmitting our request. + Aws::String next_marker = + list_objects_outcome_.GetResult().GetNextContinuationToken(); + if (next_marker.empty()) { + throw S3Exception( + "Failed to retrieve next continuation token for ListObjectsV2 " + "request."); + } + list_objects_request_.SetContinuationToken(std::move(next_marker)); + } else if (list_objects_outcome_.IsSuccess()) { + // If we have previously submitted a successful request and there are no + // more results, we've reached the end of the scan. + begin_ = end_ = typename Iterator::pointer(); + return end_; + } + + list_objects_outcome_ = client_->ListObjectsV2(list_objects_request_); + if (!list_objects_outcome_.IsSuccess()) { + throw S3Exception( + std::string("Error while listing with prefix '") + + this->prefix_.add_trailing_slash().to_string() + "' and delimiter '" + + delimiter_ + "'" + outcome_error_message(list_objects_outcome_)); + } + // Update pointers to the newly fetched results. + begin_ = list_objects_outcome_.GetResult().GetContents().begin(); + end_ = list_objects_outcome_.GetResult().GetContents().end(); + + if (list_objects_outcome_.GetResult().GetContents().empty()) { + // If the request returned no results, we've reached the end of the scan. + // We hit this case when the number of objects in the bucket is a multiple + // of the current max_keys. + return end_; + } + + return begin_; + } + + /** Pointer to the S3 client initialized by VFS. */ + shared_ptr client_; + /** Delimiter used for ListObjects request. */ + std::string delimiter_; + /** Iterators for the current objects fetched from S3. */ + typename Iterator::pointer begin_, end_; + + /** The current request being scanned. */ + Aws::S3::Model::ListObjectsV2Request list_objects_request_; + /** The current request outcome being scanned. */ + Aws::S3::Model::ListObjectsV2Outcome list_objects_outcome_; +}; + /** * This class implements the various S3 filesystem functions. It also * maintains buffer caches for writing into the various attribute files. @@ -413,18 +691,72 @@ class S3 { /** * * Lists objects and object information that start with `prefix`. + * For recursive results, an empty string can be passed as the `delimiter`. * * @param prefix The parent path to list sub-paths. - * @param delimiter The uri is truncated to the first delimiter - * @param max_paths The maximum number of paths to be retrieved - * @return A list of directory_entry objects + * @param delimiter The uri is truncated to the first delimiter. + * @param max_paths The maximum number of paths to be retrieved. + * @return Status tuple where second is a list of directory_entry objects. */ - tuple>> - ls_with_sizes( + tuple>> ls_with_sizes( const URI& prefix, const std::string& delimiter = "/", int max_paths = -1) const; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + */ + template + LsObjects ls_filtered( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false) const { + throw_if_not_ok(init_client()); + S3Scanner s3_scanner(client_, parent, f, d, recursive); + // Prepend each object key with the bucket URI. + auto prefix = parent.to_string(); + prefix = prefix.substr(0, prefix.find('/', 5)); + + LsObjects objects; + for (auto object : s3_scanner) { + objects.emplace_back(prefix + "/" + object.GetKey(), object.GetSize()); + } + return objects; + } + + /** + * Constructs a scanner for listing S3 objects. The scanner can be used to + * retrieve an InputIterator for passing to algorithms such as `std::copy_if` + * or STL constructors supporting initialization via input iterators. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + * @param max_keys The maximum number of keys to retrieve per request. + * @return Fully constructed S3Scanner object. + */ + template + S3Scanner scanner( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false, + int max_keys = 1000) const { + throw_if_not_ok(init_client()); + return S3Scanner(client_, parent, f, d, recursive, max_keys); + } + /** * Renames an object. * @@ -577,6 +909,28 @@ class S3 { */ void global_order_write(const URI& uri, const void* buffer, uint64_t length); + /* ********************************* */ + /* PUBLIC STATIC METHODS */ + /* ********************************* */ + + /** + * Returns the input `path` after adding a `/` character + * at the front if it does not exist. + */ + static std::string add_front_slash(const std::string& path); + + /** + * Returns the input `path` after removing a potential `/` character + * from the front if it exists. + */ + static std::string remove_front_slash(const std::string& path); + + /** + * Returns the input `path` after removing a potential `/` character + * from the end if it exists. + */ + static std::string remove_trailing_slash(const std::string& path); + private: /* ********************************* */ /* PRIVATE DATATYPES */ @@ -973,24 +1327,6 @@ class S3 { uint64_t length, uint64_t* nbytes_filled); - /** - * Returns the input `path` after adding a `/` character - * at the front if it does not exist. - */ - std::string add_front_slash(const std::string& path) const; - - /** - * Returns the input `path` after removing a potential `/` character - * from the front if it exists. - */ - std::string remove_front_slash(const std::string& path) const; - - /** - * Returns the input `path` after removing a potential `/` character - * from the end if it exists. - */ - std::string remove_trailing_slash(const std::string& path) const; - /** * Writes the contents of the input buffer to the S3 object given by * the input `uri` as a new series of multipart uploads. It then @@ -1192,6 +1528,63 @@ class S3 { const URI& attribute_uri, const std::string& chunk_name); }; +template +S3Scanner::S3Scanner( + const shared_ptr& client, + const URI& prefix, + F file_filter, + D dir_filter, + bool recursive, + int max_keys) + : LsScanner(prefix, file_filter, dir_filter, recursive) + , client_(client) + , delimiter_(this->is_recursive_ ? "" : "/") { + const auto prefix_dir = prefix.add_trailing_slash(); + auto prefix_str = prefix_dir.to_string(); + Aws::Http::URI aws_uri = prefix_str.c_str(); + if (!prefix_dir.is_s3()) { + throw S3Exception("URI is not an S3 URI: " + prefix_str); + } + + list_objects_request_.SetBucket(aws_uri.GetAuthority()); + const std::string aws_uri_str(S3::remove_front_slash(aws_uri.GetPath())); + list_objects_request_.SetPrefix(aws_uri_str.c_str()); + // Empty delimiter returns recursive results from S3. + list_objects_request_.SetDelimiter(delimiter_.c_str()); + // The default max_keys for ListObjects is 1000. + list_objects_request_.SetMaxKeys(max_keys); + + if (client_->requester_pays()) { + list_objects_request_.SetRequestPayer( + Aws::S3::Model::RequestPayer::requester); + } + fetch_results(); + next(begin_); +} + +template +void S3Scanner::next(typename Iterator::pointer& ptr) { + if (ptr == end_) { + ptr = fetch_results(); + } + + while (ptr != end_) { + auto object = *ptr; + uint64_t size = object.GetSize(); + std::string path = "s3://" + list_objects_request_.GetBucket() + + S3::add_front_slash(object.GetKey()); + + // TODO: Add support for directory pruning. + if (this->file_filter_(path, size)) { + // Iterator is at the next object within results accepted by the filters. + return; + } else { + // Object was rejected by the FilePredicate, do not include it in results. + advance(ptr); + } + } +} + } // namespace tiledb::sm #endif // HAVE_S3 diff --git a/tiledb/sm/filesystem/test/CMakeLists.txt b/tiledb/sm/filesystem/test/CMakeLists.txt index faa4f65f6157..f41457f6231d 100644 --- a/tiledb/sm/filesystem/test/CMakeLists.txt +++ b/tiledb/sm/filesystem/test/CMakeLists.txt @@ -27,7 +27,7 @@ include(unit_test) commence(unit_test vfs) this_target_object_libraries(vfs) - this_target_sources(main.cc unit_uri.cc) + this_target_sources(main.cc unit_uri.cc unit_ls_filtered.cc) conclude(unit_test) commence(unit_test vfs_read_log_modes) diff --git a/tiledb/sm/filesystem/test/unit_ls_filtered.cc b/tiledb/sm/filesystem/test/unit_ls_filtered.cc new file mode 100644 index 000000000000..8af2ec8f225a --- /dev/null +++ b/tiledb/sm/filesystem/test/unit_ls_filtered.cc @@ -0,0 +1,124 @@ +/** + * @file unit_ls_filtered.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests internal ls recursive filter. + */ + +#include +#include +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/filesystem/vfs.h" + +class VFSTest { + public: + /** + * Requires derived class to create a temporary directory. + * + * @param test_tree Vector used to build test directory and objects. + * For each element we create a nested directory with N objects. + * @param prefix The URI prefix to use for the test directory. + */ + explicit VFSTest( + const std::vector& test_tree, const std::string& prefix) + : stats_("unit_ls_filtered") + , io_(4) + , compute_(4) + , vfs_(&stats_, &io_, &compute_, tiledb::sm::Config()) + , test_tree_(test_tree) + , prefix_(prefix) + , temp_dir_(prefix_) { + } + + virtual ~VFSTest() { + bool is_dir = false; + vfs_.is_dir(temp_dir_, &is_dir).ok(); + if (is_dir) { + vfs_.remove_dir(temp_dir_).ok(); + } + } + + /** FilePredicate for passing to ls_filtered that accepts all files. */ + static bool accept_all_files(const std::string_view&, uint64_t) { + return true; + } + + /** Resources needed to construct VFS */ + tiledb::sm::stats::Stats stats_; + ThreadPool io_, compute_; + tiledb::sm::VFS vfs_; + + std::vector test_tree_; + std::string prefix_; + tiledb::sm::URI temp_dir_; + + private: + tiledb::sm::LsObjects expected_results_; +}; + +// TODO: Disable shouldfail when file:// or mem:// support is added. +TEST_CASE( + "VFS: Throwing FileFilter ls_recursive", + "[vfs][ls_recursive][!shouldfail]") { + std::string prefix = GENERATE("file://", "mem://"); + prefix += std::filesystem::current_path().string() + "/ls_filtered_test"; + + VFSTest vfs_test({0}, prefix); + auto file_filter = [](const std::string_view&, uint64_t) -> bool { + throw std::logic_error("Throwing FileFilter"); + }; + SECTION("Throwing FileFilter with 0 objects should not throw") { + CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, file_filter, tiledb::sm::accept_all_dirs)); + } + SECTION("Throwing FileFilter with N objects should throw") { + vfs_test.vfs_.touch(vfs_test.temp_dir_.join_path("file")).ok(); + CHECK_THROWS_AS( + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + std::logic_error); + CHECK_THROWS_WITH( + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + Catch::Matchers::ContainsSubstring("Throwing FileFilter")); + } +} + +TEST_CASE( + "VFS: ls_recursive throws for unsupported filesystems", + "[vfs][ls_recursive]") { + std::string prefix = GENERATE("file://", "mem://"); + prefix += std::filesystem::current_path().string() + "/ls_filtered_test"; + + VFSTest vfs_test({1}, prefix); + std::string backend = vfs_test.temp_dir_.backend_name(); + DYNAMIC_SECTION(backend << " unsupported backend should throw") { + CHECK_THROWS_WITH( + vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, VFSTest::accept_all_files), + Catch::Matchers::ContainsSubstring("storage backend is not supported")); + } +} diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index c517d13517c1..9faa1ac3fa55 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -252,7 +252,7 @@ Status VFS::touch(const URI& uri) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.touch(uri); + return s3().touch(uri); #else throw BuiltWithout("S3"); #endif @@ -285,7 +285,7 @@ Status VFS::cancel_all_tasks() { Status VFS::create_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.create_bucket(uri); + return s3().create_bucket(uri); #else throw BuiltWithout("S3"); #endif @@ -310,7 +310,7 @@ Status VFS::create_bucket(const URI& uri) const { Status VFS::remove_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.remove_bucket(uri); + return s3().remove_bucket(uri); #else throw BuiltWithout("S3"); #endif @@ -335,7 +335,7 @@ Status VFS::remove_bucket(const URI& uri) const { Status VFS::empty_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.empty_bucket(uri); + return s3().empty_bucket(uri); #else throw BuiltWithout("S3"); #endif @@ -361,7 +361,7 @@ Status VFS::is_empty_bucket( const URI& uri, [[maybe_unused]] bool* is_empty) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.is_empty_bucket(uri, is_empty); + return s3().is_empty_bucket(uri, is_empty); #else throw BuiltWithout("S3"); #endif @@ -398,7 +398,7 @@ Status VFS::remove_dir(const URI& uri) const { #endif } else if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.remove_dir(uri); + return s3().remove_dir(uri); #else throw BuiltWithout("S3"); #endif @@ -450,7 +450,7 @@ Status VFS::remove_file(const URI& uri) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.remove_object(uri); + return s3().remove_object(uri); #else throw BuiltWithout("S3"); #endif @@ -535,7 +535,7 @@ Status VFS::file_size(const URI& uri, uint64_t* size) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.object_size(uri, size); + return s3().object_size(uri, size); #else throw BuiltWithout("S3"); #endif @@ -579,7 +579,7 @@ Status VFS::is_dir(const URI& uri, bool* is_dir) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.is_dir(uri, is_dir); + return s3().is_dir(uri, is_dir); #else *is_dir = false; throw BuiltWithout("S3"); @@ -628,7 +628,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - RETURN_NOT_OK(s3_.is_object(uri, is_file)); + RETURN_NOT_OK(s3().is_object(uri, is_file)); return Status::Ok(); #else *is_file = false; @@ -661,7 +661,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { if (uri.is_s3()) { #ifdef HAVE_S3 - RETURN_NOT_OK(s3_.is_bucket(uri, is_bucket)); + RETURN_NOT_OK(s3().is_bucket(uri, is_bucket)); return Status::Ok(); #else *is_bucket = false; @@ -729,7 +729,7 @@ tuple>> VFS::ls_with_sizes( } else if (parent.is_s3()) { #ifdef HAVE_S3 Status st; - std::tie(st, entries) = s3_.ls_with_sizes(parent); + std::tie(st, entries) = s3().ls_with_sizes(parent); #else auto st = Status_VFSError("TileDB was built without S3 support"); #endif @@ -812,7 +812,7 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { if (old_uri.is_s3()) { if (new_uri.is_s3()) #ifdef HAVE_S3 - return s3_.move_object(old_uri, new_uri); + return s3().move_object(old_uri, new_uri); #else throw BuiltWithout("S3"); #endif @@ -881,7 +881,7 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { if (old_uri.is_s3()) { if (new_uri.is_s3()) #ifdef HAVE_S3 - return s3_.move_dir(old_uri, new_uri); + return s3().move_dir(old_uri, new_uri); #else throw BuiltWithout("S3"); #endif @@ -957,7 +957,7 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { if (old_uri.is_s3()) { if (new_uri.is_s3()) #ifdef HAVE_S3 - return s3_.copy_file(old_uri, new_uri); + return s3().copy_file(old_uri, new_uri); #else throw BuiltWithout("S3"); #endif @@ -1022,7 +1022,7 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { if (old_uri.is_s3()) { if (new_uri.is_s3()) #ifdef HAVE_S3 - return s3_.copy_dir(old_uri, new_uri); + return s3().copy_dir(old_uri, new_uri); #else throw BuiltWithout("S3"); #endif @@ -1149,7 +1149,7 @@ Status VFS::read_impl( #ifdef HAVE_S3 const auto read_fn = std::bind( &S3::read, - &s3_, + &s3(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, @@ -1390,7 +1390,7 @@ Status VFS::close_file(const URI& uri) { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3_.flush_object(uri); + return s3().flush_object(uri); #else throw BuiltWithout("S3"); #endif @@ -1418,7 +1418,7 @@ Status VFS::close_file(const URI& uri) { void VFS::finalize_and_close_file(const URI& uri) { if (uri.is_s3()) { #ifdef HAVE_S3 - s3_.finalize_and_flush_object(uri); + s3().finalize_and_flush_object(uri); return; #else throw BuiltWithout("S3"); @@ -1452,10 +1452,10 @@ Status VFS::write( if (uri.is_s3()) { #ifdef HAVE_S3 if (remote_global_order_write) { - s3_.global_order_write_buffered(uri, buffer, buffer_size); + s3().global_order_write_buffered(uri, buffer, buffer_size); return Status::Ok(); } - return s3_.write(uri, buffer, buffer_size); + return s3().write(uri, buffer, buffer_size); #else throw BuiltWithout("S3"); #endif @@ -1487,7 +1487,7 @@ VFS::multipart_upload_state(const URI& uri) { } else if (uri.is_s3()) { #ifdef HAVE_S3 VFS::MultiPartUploadState state; - auto s3_state = s3_.multipart_upload_state(uri); + auto s3_state = s3().multipart_upload_state(uri); if (!s3_state.has_value()) { return {Status::Ok(), nullopt}; } @@ -1553,11 +1553,11 @@ Status VFS::set_multipart_upload_state( // Chunk URI gets reconstructed from the serialized chunk name // and the real attribute uri s3_state.buffered_chunks.emplace_back( - s3_.generate_chunk_uri(uri, chunk.uri).to_string(), chunk.size); + s3().generate_chunk_uri(uri, chunk.uri).to_string(), chunk.size); } } - return s3_.set_multipart_upload_state(uri.to_string(), s3_state); + return s3().set_multipart_upload_state(uri.to_string(), s3_state); #else throw BuiltWithout("S3"); #endif @@ -1582,8 +1582,8 @@ Status VFS::flush_multipart_file_buffer(const URI& uri) { if (uri.is_s3()) { #ifdef HAVE_S3 Buffer* buff = nullptr; - throw_if_not_ok(s3_.get_file_buffer(uri, &buff)); - s3_.global_order_write(uri, buff->data(), buff->size()); + throw_if_not_ok(s3().get_file_buffer(uri, &buff)); + s3().global_order_write(uri, buff->data(), buff->size()); buff->reset_size(); #else diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 95942370577d..99c5ac4e77e0 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -40,6 +40,7 @@ #include #include "filesystem_base.h" +#include "ls_scanner.h" #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/macros.h" @@ -227,15 +228,29 @@ struct VFSBase { /** The S3 filesystem. */ #ifdef HAVE_S3 -struct S3_within_VFS { +class S3_within_VFS { + /** Private member variable */ S3 s3_; + + protected: template S3_within_VFS(Args&&... args) : s3_(std::forward(args)...) { } + + /** Protected accessor for the S3 object. */ + inline tiledb::sm::S3& s3() { + return s3_; + } + + /** Protected accessor for the const S3 object. */ + inline const tiledb::sm::S3& s3() const { + return s3_; + } }; #else -struct S3_within_VFS { +class S3_within_VFS { + protected: template S3_within_VFS(Args&&...) { } // empty constructor @@ -246,7 +261,7 @@ struct S3_within_VFS { * This class implements a virtual filesystem that directs filesystem-related * function execution to the appropriate backend based on the input URI. */ -class VFS : private VFSBase, S3_within_VFS { +class VFS : private VFSBase, protected S3_within_VFS { public: /* ********************************* */ /* TYPE DEFINITIONS */ @@ -494,6 +509,38 @@ class VFS : private VFSBase, S3_within_VFS { tuple>> ls_with_sizes( const URI& parent) const; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * Currently only S3 is supported for ls_recursive. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @return Vector of results with each entry being a pair of the string URI + * and object size. + */ + template + LsObjects ls_recursive( + const URI& parent, + [[maybe_unused]] F f, + [[maybe_unused]] D d = accept_all_dirs) const { + if (parent.is_s3()) { +#ifdef HAVE_S3 + return s3().ls_filtered(parent, f, d, true); +#else + throw filesystem::VFSException("TileDB was built without S3 support"); +#endif + } else { + throw filesystem::VFSException( + "Recursive ls over " + parent.backend_name() + + " storage backend is not supported."); + } + } + /** * Renames a file. * From 18cb02034d19229f5d4f43768840710492b785bc Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Sat, 23 Dec 2023 17:12:55 +0100 Subject: [PATCH 118/456] Fix narrowing conversions in nightlies. (#4593) --- TYPE: NO_HISTORY DESC: Fix narrowing conversions in nightlies. --- tiledb/sm/filter/test/filtered_tile_checker.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/filter/test/filtered_tile_checker.h b/tiledb/sm/filter/test/filtered_tile_checker.h index a2b7354d8302..9252a997483d 100644 --- a/tiledb/sm/filter/test/filtered_tile_checker.h +++ b/tiledb/sm/filter/test/filtered_tile_checker.h @@ -400,8 +400,8 @@ class FilteredTileChecker { for (auto nelements_chunk : elements_per_chunk) { uint32_t data_size{static_cast(nelements_chunk * sizeof(T))}; checker.add_grid_chunk_checker( - data_size, nelements_chunk, start, spacing); - start += nelements_chunk * spacing; + data_size, static_cast(nelements_chunk), start, spacing); + start += static_cast(nelements_chunk) * spacing; } return checker; } From 1e6a38c648efac81d364066fe3656db1b77c5820 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Sun, 24 Dec 2023 14:57:13 +0100 Subject: [PATCH 119/456] Fix narrowing conversions in nightlies 2. (#4595) --- TYPE: NO_HISTORY DESC: Fix narrowing conversions in nightlies 2. --- tiledb/sm/filter/test/filtered_tile_checker.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/filter/test/filtered_tile_checker.h b/tiledb/sm/filter/test/filtered_tile_checker.h index 9252a997483d..5f0784583d21 100644 --- a/tiledb/sm/filter/test/filtered_tile_checker.h +++ b/tiledb/sm/filter/test/filtered_tile_checker.h @@ -436,8 +436,12 @@ class FilteredTileChecker { auto checksum = checksum_per_chunk[chunk_index]; uint32_t data_size{static_cast(nelements * sizeof(T))}; checker.add_grid_chunk_checker( - data_size, nelements, start, spacing, checksum); - start += nelements * spacing; + data_size, + static_cast(nelements), + start, + spacing, + checksum); + start += static_cast(nelements) * spacing; } return checker; } From e442cb65b8d98bd312cd60e7ebf3d25ad848b78e Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 4 Jan 2024 09:58:59 -0700 Subject: [PATCH 120/456] `class BudgetUnavailable` (#4598) Added new exception classes for resource budgeting. `class BudgetUnavailable` translates to a special C API return code. --- TYPE: NO_HISTORY DESC: --- tiledb/api/c_api/api_external_common.h | 4 ++ .../exception_wrapper/exception_wrapper.h | 5 ++ .../test/unit_capi_exception_wrapper.cc | 16 +++++++ tiledb/common/exception/exception.h | 46 +++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/tiledb/api/c_api/api_external_common.h b/tiledb/api/c_api/api_external_common.h index c1143fb31bd3..9f67f672f003 100644 --- a/tiledb/api/c_api/api_external_common.h +++ b/tiledb/api/c_api/api_external_common.h @@ -156,6 +156,10 @@ TILEDB_EXPORT capi_status_t tiledb_status_code(capi_return_t x); * Invalid error argument would prevent errors from being reported correctly. */ #define TILEDB_INVALID_ERROR (-4) +/** + * Invalid error argument would prevent errors from being reported correctly. + */ +#define TILEDB_BUDGET_UNAVAILABLE (-5) /**@}*/ #ifdef __cplusplus diff --git a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h index e79227e05281..43fa2c675023 100644 --- a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h +++ b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h @@ -605,6 +605,11 @@ class CAPIFunction { if constexpr (!std::same_as) { return TILEDB_INVALID_CONTEXT; } + } catch (const BudgetUnavailable& e) { + h.action(e); + if constexpr (!std::same_as) { + return TILEDB_BUDGET_UNAVAILABLE; + } } catch (const detail::InvalidErrorException& e) { h.action(e); if constexpr (!std::same_as) { diff --git a/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_exception_wrapper.cc b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_exception_wrapper.cc index 4216da18b80e..0eaf9dd59b3a 100644 --- a/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_exception_wrapper.cc +++ b/tiledb/api/c_api_support/exception_wrapper/test/unit_capi_exception_wrapper.cc @@ -289,3 +289,19 @@ TEST_CASE("api_entry_error - throw") { CHECK(error->message() == "Test: error"); tiledb_error_free(&error); } + +capi_return_t tf_budget_never_available() { + throw BudgetUnavailable("Test", "budget unavailable"); +} +TEST_CASE("BudgetUnavailable - special return value from CAPIFunction") { + auto rc{tiledb::api::api_entry_plain()}; + CHECK(tiledb_status(rc) == TILEDB_BUDGET_UNAVAILABLE); +} + +capi_return_t tf_budget_exceeded() { + throw BudgetExceeded("Test", "budget exceeded"); +} +TEST_CASE("BudgetExceeded - ordinary return value from CAPIFunction") { + auto rc{tiledb::api::api_entry_plain()}; + CHECK(tiledb_status(rc) == TILEDB_ERR); +} diff --git a/tiledb/common/exception/exception.h b/tiledb/common/exception/exception.h index 58b022e5d505..7093d8a31b1f 100644 --- a/tiledb/common/exception/exception.h +++ b/tiledb/common/exception/exception.h @@ -185,6 +185,52 @@ inline void throw_if_not_ok(const Status& st) { } } +/** + * An exception that refuses to start an operation because the estimate of + * resources to complete the operation exceeds the available budget for those + * resources. + * + * This exception should only be thrown _before_ an operation commences. Once an + * operation starts, if (for whatever reason) the budget estimate was wrong, the + * proper exception to throw is `BudgetExceeded`. The reason for this is that + * the exception wrapper catches this exception in a specific `catch` clause, + * whereupon it returns the special value `TILEDB_BUDGET_UNAVAILABLE` to the C + * API caller. `BudgetExceeded`, by contrast, is an ordinary exception that will + * terminate an in-progress operation like any other fatal error would. + * + * @section Maturity Notes + * + * This exception makes no attempt to state what kind of budget was unavailable. + * In order to do this in the exception itself there would need to be data + * structures available that formalized what budget categories existed, what the + * limits are, what the request would have required, etc. The constructor for + * this exception might eventually take additional constructor arguments for + * this purpose. + */ +class BudgetUnavailable : public StatusException { + public: + /** + * Ordinary constructor has same signature as `BudgetExceeded`. + */ + BudgetUnavailable(const std::string origin, const std::string message) + : StatusException(origin, message) { + } +}; + +/** + * An exception that terminates an already-started operation because there is + * not enough budget of some resource to complete the operation. + */ +class BudgetExceeded : public StatusException { + public: + /** + * Ordinary constructor has same signature as `BudgetUnavailable`. + */ + BudgetExceeded(const std::string origin, const std::string message) + : StatusException(origin, message) { + } +}; + } // namespace tiledb::common #endif // TILEDB_COMMON_EXCEPTION_H From 9360208aadd261694d6596b737cb1eb6ff815c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Thu, 4 Jan 2024 22:18:21 +0100 Subject: [PATCH 121/456] CentOS Stream 9 Dockerfile (#4591) Add centos dockerfile that should run nightly. --- TYPE: IMPROVEMENT DESC: Adds centos stream 9 dockerfile that builds tiledb --- .github/workflows/nightly-dockerfile.yml | 41 ++++++++++++++++++++++++ examples/Dockerfile/centos.dockerfile | 14 ++++++++ examples/Dockerfile/rocky.dockerfile | 14 ++++++++ 3 files changed, 69 insertions(+) create mode 100644 .github/workflows/nightly-dockerfile.yml create mode 100644 examples/Dockerfile/centos.dockerfile create mode 100644 examples/Dockerfile/rocky.dockerfile diff --git a/.github/workflows/nightly-dockerfile.yml b/.github/workflows/nightly-dockerfile.yml new file mode 100644 index 000000000000..78c988b26cdb --- /dev/null +++ b/.github/workflows/nightly-dockerfile.yml @@ -0,0 +1,41 @@ +name: Nightly-Dockerfile + +on: + schedule: + # runs every day at 2:50 UTC + - cron: "50 02 * * *" + workflow_call: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + dockerfile: + - centos.dockerfile + - rocky.dockerfile + name: ${{ matrix.dockerfile }} + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + - name: Build the Dockerfile + run: docker build -f examples/Dockerfile/${{ matrix.dockerfile }} -t tiledb:dev . + + create_issue_on_fail: + permissions: + issues: write + runs-on: ubuntu-latest + needs: build + if: failure() || cancelled() + steps: + - uses: actions/checkout@v4 + - name: Create Issue if Build Fails + uses: TileDB-Inc/github-actions/open-issue@main + with: + name: Nightly Dev Build + label: CI,nightly + assignee: dudoslav diff --git a/examples/Dockerfile/centos.dockerfile b/examples/Dockerfile/centos.dockerfile new file mode 100644 index 000000000000..7d10ba40cb27 --- /dev/null +++ b/examples/Dockerfile/centos.dockerfile @@ -0,0 +1,14 @@ +FROM dokken/centos-stream-9 + +RUN yum -y install git g++ python3-pip curl zip unzip tar perl \ + && pip install cmake + +WORKDIR /opt/tiledb +COPY . . + +RUN cmake . -B build -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_MAKE_PROGRAM=make \ + -DTILEDB_AZURE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_TOOLS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr \ + && cmake --build build \ + && cmake --build build --target install-tiledb \ + && ldconfig \ No newline at end of file diff --git a/examples/Dockerfile/rocky.dockerfile b/examples/Dockerfile/rocky.dockerfile new file mode 100644 index 000000000000..d1b1ce028c47 --- /dev/null +++ b/examples/Dockerfile/rocky.dockerfile @@ -0,0 +1,14 @@ +FROM rockylinux:9 + +RUN yum -y install git g++ python3-pip zip unzip tar perl \ + && pip install cmake + +WORKDIR /opt/tiledb +COPY . . + +RUN cmake . -B build -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_MAKE_PROGRAM=make \ + -DTILEDB_AZURE=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_TOOLS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr \ + && cmake --build build \ + && cmake --build build --target install-tiledb \ + && ldconfig From 65d1ce40a4aebeee18aed19b7f590952692f3042 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 5 Jan 2024 19:17:40 +0100 Subject: [PATCH 122/456] Fix out of order consolidation. (#4597) The following issue https://github.com/TileDB-Inc/TileDB-Py/issues/1885 reported that running consolidation in this order invalidates an array: - consolidate fragments - consolidate commits - vacuum fragments - vacuum commits This fixes two issues: - We sometimes would not delete the commit files when vacuuming fragments. - We would vacuum an ignore file when a consolidated commits file still needed it. --- TYPE: BUG DESC: Fix out of order consolidation. --- test/src/unit-capi-consolidation.cc | 26 +++++++++++++++++++ tiledb/sm/array/array_directory.cc | 10 +++++-- .../sm/consolidator/fragment_consolidator.cc | 11 ++++++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index a75d1650c579..b6ecef3a88c0 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -7217,4 +7217,30 @@ TEST_CASE_METHOD( "progress can be made") == msg); remove_sparse_string_array(); +} + +TEST_CASE_METHOD( + ConsolidationFx, + "C API: Test consolidation, fragments/commits out of order", + "[capi][consolidation][fragments-commits][out-of-order]") { +#ifdef TILEDB_SERIALIZATION + serialize_ = GENERATE(true, false); + if (serialize_) { + refactored_query_v2_ = GENERATE(true, false); + } +#endif + + remove_sparse_array(); + create_sparse_array(); + + write_sparse_full(); + write_sparse_unordered(); + consolidate_sparse("fragments"); + consolidate_sparse("commits"); + vacuum_sparse("fragments"); + vacuum_sparse("commits"); + read_sparse_full_unordered(); + check_commits_dir_sparse(1, 0, 1); + + remove_sparse_array(); } \ No newline at end of file diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 86fa172a307c..4676cf507a45 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -805,11 +805,17 @@ ArrayDirectory::load_consolidated_commit_uris( for (auto& meta_file : meta_files) { std::stringstream ss(meta_file.second); uint64_t count = 0; + bool all_in_set = true; for (std::string uri_str; std::getline(ss, uri_str);) { - count += uris_set.count(uri_.to_string() + uri_str); + if (uris_set.count(uri_.to_string() + uri_str) > 0) { + count++; + } else { + all_in_set = false; + break; + } } - if (count == uris_set.size()) { + if (all_in_set && count == uris_set.size()) { for (auto& uri : commits_dir_uris) { if (stdx::string::ends_with( uri.to_string(), constants::con_commits_file_suffix)) { diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 649c4eea55d9..80cb15a78c7a 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -315,10 +315,9 @@ void FragmentConsolidator::vacuum(const char* array_name) { array_dir.write_commit_ignore_file(commit_uris_to_ignore); } - // Delete the commit and vacuum files + // Delete the vacuum files. auto vfs = storage_manager_->vfs(); auto compute_tp = storage_manager_->compute_tp(); - vfs->remove_files(compute_tp, filtered_fragment_uris.commit_uris_to_vacuum()); vfs->remove_files( compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); @@ -327,6 +326,14 @@ void FragmentConsolidator::vacuum(const char* array_name) { compute_tp, 0, fragment_uris_to_vacuum.size(), [&](size_t i) { RETURN_NOT_OK(vfs->remove_dir(fragment_uris_to_vacuum[i])); + // Remove the commit file, if present. + auto commit_uri = array_dir.get_commit_uri(fragment_uris_to_vacuum[i]); + bool is_file = false; + RETURN_NOT_OK(vfs->is_file(commit_uri, &is_file)); + if (is_file) { + RETURN_NOT_OK(vfs->remove_file(commit_uri)); + } + return Status::Ok(); })); } From e397cdb2f939d4006132e081a40eca60659eed2d Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Mon, 8 Jan 2024 15:53:55 +0200 Subject: [PATCH 123/456] Fix consolidation plan to print relative paths in output (#4604) This issue was found by @anastasop while testing Cloud support for Consolidation plan. So far consolidation plan would print full URIs for fragments. However, for remote arrays we should be only printing out relative URIs. So, this fix replaces full with relative URIs for any type of array consolidation plan. --- TYPE: IMPROVEMENT DESC: Fix consolidation plan to print relative paths in output --- test/src/test-cppapi-consolidation-plan.cc | 3 ++- tiledb/sm/consolidation_plan/consolidation_plan.h | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/src/test-cppapi-consolidation-plan.cc b/test/src/test-cppapi-consolidation-plan.cc index 4a23ec489392..02c9e373a6d8 100644 --- a/test/src/test-cppapi-consolidation-plan.cc +++ b/test/src/test-cppapi-consolidation-plan.cc @@ -37,6 +37,7 @@ #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/serialization/consolidation.h" #include @@ -170,7 +171,7 @@ std::string CppConsolidationPlanFx::write_sparse( // Close array. array->close(); - return query.fragment_uri(0); + return tiledb::sm::URI(query.fragment_uri(0)).last_path_part(); } void CppConsolidationPlanFx::remove_array(const std::string& array_name) { diff --git a/tiledb/sm/consolidation_plan/consolidation_plan.h b/tiledb/sm/consolidation_plan/consolidation_plan.h index fa41c22d2fe3..d33e28ebaf8b 100644 --- a/tiledb/sm/consolidation_plan/consolidation_plan.h +++ b/tiledb/sm/consolidation_plan/consolidation_plan.h @@ -198,8 +198,10 @@ class ConsolidationPlan { std::vector ret; ret.reserve(fragment_indexes_.size()); for (auto& idx : fragment_indexes_) { - ret.emplace_back( - array_->fragment_metadata()[idx]->fragment_uri().c_str()); + ret.emplace_back(array_->fragment_metadata()[idx] + ->fragment_uri() + .last_path_part() + .c_str()); } return ret; From ae2ae932cb158b257a9063ca17262b8f51b42997 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Wed, 10 Jan 2024 13:15:05 +0200 Subject: [PATCH 124/456] Finalize should error out for remote global order writes (#4600) Calling finalize() was allowed as noop in remote global order writes, but it should have been treated as incorrect usage. --- TYPE: BUG DESC: Finalize should error out for remote global order writes --- .../src/unit-cppapi-global-order-writes-remote.cc | 15 ++++++++++++++- tiledb/sm/query/query.cc | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/src/unit-cppapi-global-order-writes-remote.cc b/test/src/unit-cppapi-global-order-writes-remote.cc index 26fbaed120ec..2bdfb4eddfc2 100644 --- a/test/src/unit-cppapi-global-order-writes-remote.cc +++ b/test/src/unit-cppapi-global-order-writes-remote.cc @@ -116,7 +116,7 @@ struct RemoteGlobalOrderWriteFx { array.close(); } - void write_array() { + void write_array(bool check_finalize_fails = false) { Array array(ctx_, array_uri_, TILEDB_WRITE); Query query(ctx_, array); query.set_layout(TILEDB_GLOBAL_ORDER); @@ -182,6 +182,13 @@ struct RemoteGlobalOrderWriteFx { if (i + submit_cell_count_ < total_cell_count_) { query.submit(); } else { + if constexpr (rest_tests) { + if (check_finalize_fails) { + CHECK_THROWS_WITH( + query.finalize(), + Catch::Matchers::ContainsSubstring("submit_and_finalize")); + } + } // IMPORTANT: Submit final write query and close the array. // We must do this within loop; Else our buffers will be out of scope. query.submit_and_finalize(); @@ -480,3 +487,9 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { fx.run_test(); } } + +TEST_CASE("Remote global order writes finalize errors", "[rest][finalize]") { + RemoteGlobalOrderWriteFx fx(20, 10, 3, true, true); + fx.create_array(); + fx.write_array(true); +} diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index a969311f79b5..316376123f9b 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -475,7 +475,7 @@ QueryBuffer Query::buffer(const std::string& name) const { Status Query::finalize() { if (status_ == QueryStatus::UNINITIALIZED || - status_ == QueryStatus::INITIALIZED) { + (status_ == QueryStatus::INITIALIZED && !array_->is_remote())) { return Status::Ok(); } From f6dd42e357fd84e9b3a4af2d3c1595c202ab6bda Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:53:54 +0100 Subject: [PATCH 125/456] Fix aggregate leak when field doesn't exist. (#4610) When a field didn't exist, we would throw on the creation of the query field handle after making a handle for the channel. This handle wouldn't get broken after we throw. This solves the issue by creating the handle for the channel after we validated that the field exists. --- TYPE: BUG DESC: Fix aggregate leak when field doesn't exist. --- tiledb/api/c_api/query_field/query_field_api.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc index c6b88154c041..6e396af2e35c 100644 --- a/tiledb/api/c_api/query_field/query_field_api.cc +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -52,8 +52,7 @@ tiledb_field_origin_t FieldFromAggregate::origin() { tiledb_query_field_handle_t::tiledb_query_field_handle_t( tiledb_query_t* query, const char* field_name) : query_(query->query_) - , field_name_(field_name) - , channel_(tiledb_query_channel_handle_t::make_handle(query)) { + , field_name_(field_name) { if (field_name_ == tiledb::sm::constants::coords) { field_origin_ = std::make_shared(); type_ = query_->array_schema().domain().dimension_ptr(0)->type(); @@ -81,6 +80,8 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( } else { throw tiledb::api::CAPIStatusException("There is no field " + field_name_); } + + channel_ = tiledb_query_channel_handle_t::make_handle(query); } namespace tiledb::api { From 0e6a7164872c7e3c0a5005918620782d3b8e0279 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 10 Jan 2024 18:45:57 +0100 Subject: [PATCH 126/456] Call get field function directly in tiledb_create_unary_aggregate. (#4588) This changes tiledb_create_unary_aggregate to stop calling tiledb_query_get_field and call the internal API instead. This will allow exception to propagate properly. --- TYPE: NO_HISTORY DESC: Call get field function directly in tiledb_create_unary_aggregate. --- .../c_api/query_aggregate/query_aggregate_api.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index 7e63c9242dfd..786d663b8160 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -35,6 +35,7 @@ #include "query_aggregate_api_internal.h" #include "tiledb/api/c_api/query/query_api_internal.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" +#include "tiledb/api/c_api/query_field/query_field_api_internal.h" #include "tiledb/api/c_api_support/c_api_support.h" const tiledb_channel_operator_handle_t* tiledb_channel_operator_sum = @@ -170,7 +171,7 @@ capi_return_t tiledb_query_get_default_channel( } capi_return_t tiledb_create_unary_aggregate( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_query_t* query, const tiledb_channel_operator_t* op, const char* input_field_name, @@ -184,12 +185,10 @@ capi_return_t tiledb_create_unary_aggregate( auto& schema = query->query_->array_schema(); // Getting the field errors if there is no input_field_name associated with - // the query - tiledb_query_field_t* field; - if (auto rv = tiledb_query_get_field(ctx, query, input_field_name, &field)) { - return rv; - } - tiledb_query_field_free(ctx, &field); + // the query. + tiledb_query_field_t* field = + tiledb_query_field_handle_t::make_handle(query, input_field_name); + tiledb_query_field_handle_t::break_handle(field); auto fi = tiledb::sm::FieldInfo( field_name, From acb3ea0cfbbe348d5b6de526ff7470448f6eb33d Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 10 Jan 2024 11:58:44 -0600 Subject: [PATCH 127/456] Add build system hook (#4586) This is a minor change to allow for using the newly added C API macros in projects that build libtiledb from source. --- TYPE: NO_HISTORY DESC: Add build system hook --- CMakeLists.txt | 13 +++++++++++++ cmake/TileDB-Superbuild.cmake | 1 + vcpkg.json | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82b7b12fd03d..cc7c43f05abf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -593,3 +593,16 @@ add_custom_target( COMMAND rmdir "${CMAKE_INSTALL_PREFIX}/include/tiledb" COMMAND echo "TileDB uninstalled" ) + +########################################################### +# Run the optional extra cmake include +########################################################### + +if(TILEDB_EXTRA_CMAKE_INCLUDE) + get_filename_component( + EXTRA_INCLUDE_ABSPATH + "${TILEDB_EXTRA_CMAKE_INCLUDE}" + ABSOLUTE + ) + include("${EXTRA_INCLUDE_ABSPATH}") +endif() diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index f1802fbcb1a6..62e07f9b7902 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -34,6 +34,7 @@ set(INHERITED_CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} -DCOMPILER_SUPPORTS_AVX2=${COMPILER_SUPPORTS_AVX2} + -DTILEDB_EXTRA_CMAKE_INCLUDE=${TILEDB_EXTRA_CMAKE_INCLUDE} -DTILEDB_VCPKG=${TILEDB_VCPKG} -DTILEDB_VERBOSE=${TILEDB_VERBOSE} -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS} diff --git a/vcpkg.json b/vcpkg.json index d93075886980..86a75075b40f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,7 @@ { "name": "tiledb", "version-string": "2.15.0", - "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", + "builtin-baseline": "c0b1007fe7fc6ccc902e6c708fd6a670ddad8f9d", "dependencies": [ "bzip2", "libmagic", From 31500c5a26b224843113589875f36f78f0be8fbf Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 10 Jan 2024 19:59:32 +0200 Subject: [PATCH 128/456] Use 7-digit commit hashes in release artifacts. (#4599) [SC-38683](https://app.shortcut.com/tiledb-inc/story/38683) Fixes a regression introduced in #4518. --- TYPE: BUILD DESC: Fix regression where release artifacts had 8-digit commit hashes. --- .github/workflows/release.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c4eb697682a9..201d5413f5b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: - name: Set variables id: get-values run: | - release_hash=$( echo ${{ github.sha }} | cut -c-8 - ) + release_hash=$( echo ${{ github.sha }} | cut -c-7 - ) ref=${{ github.head_ref || github.ref_name }} echo "release_hash=$release_hash" >> $GITHUB_OUTPUT echo "archive_name=tiledb-${{ matrix.platform }}-${ref##*/}-$release_hash" >> $GITHUB_OUTPUT @@ -124,8 +124,24 @@ jobs: source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed - Publish-Release: + Test-Release-Artifacts: needs: Build-Release + runs-on: ubuntu-latest + steps: + - name: Download release artifacts + uses: actions/download-artifact@v2 + with: + name: tiledb-dist + path: dist + - name: Test names of release artifacts + run: | + if [ ls dist/ | grep -Ev -- '^tiledb-(linux|macos|windows)+-(x86_64|arm64)(-noavx2)?-.+-[0-9a-f]{7}\.(zip|tar\.gz)$' ]; then + echo "Some release artifact names do not match expected pattern" + exit 1 + fi + + Publish-Release: + needs: Test-Release-Artifacts if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: From 6aa599fc94800e9f442eae9c2ed6838f8db44a55 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Wed, 10 Jan 2024 11:52:31 -0700 Subject: [PATCH 129/456] Refactor fragment consolidation with `class FragmentConsolidationWorkspace` (#4605) This PR puts the query buffers used inside the fragment consolidator into their own class. This class deletes its copy constructor but provides a move constructor. This avoids copying the buffers in the `return` statement of `create_buffers`, which not only doubles the instantaneous memory consumption during that statement, but does a bunch of spurious copying of empty buffers. In addition, this change prepares for memory as a managed resource, providing a single location for the allocators of the containers for these buffers to be specified. --- TYPE: NO_HISTORY DESC: `class FragmentConsolidationWorkspace` --- tiledb/sm/consolidator/consolidator.h | 24 +-- .../sm/consolidator/fragment_consolidator.cc | 71 ++++--- .../sm/consolidator/fragment_consolidator.h | 177 +++++++++++------- .../test/unit_fragment_consolidator.cc | 21 ++- 4 files changed, 168 insertions(+), 125 deletions(-) diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index b7811bef99ab..529097e6885e 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -125,6 +125,18 @@ class Consolidator { */ virtual void vacuum(const char* array_name); + /* ********************************* */ + /* TYPE DEFINITIONS */ + /* ********************************* */ + + /** Consolidation configuration parameters. */ + struct ConsolidationConfigBase { + /** Start time for consolidation. */ + uint64_t timestamp_start_; + /** End time for consolidation. */ + uint64_t timestamp_end_; + }; + protected: /* ********************************* */ /* PROTECTED METHODS */ @@ -144,18 +156,6 @@ class Consolidator { */ void check_array_uri(const char* array_name); - /* ********************************* */ - /* TYPE DEFINITIONS */ - /* ********************************* */ - - /** Consolidation configuration parameters. */ - struct ConsolidationConfigBase { - /** Start time for consolidation. */ - uint64_t timestamp_start_; - /** End time for consolidation. */ - uint64_t timestamp_end_; - }; - /* ********************************* */ /* PROTECTED ATTRIBUTES */ /* ********************************* */ diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 80cb15a78c7a..946889e670de 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -51,9 +51,9 @@ using namespace tiledb::common; namespace tiledb::sm { -class FragmentConsolidatorStatusException : public StatusException { +class FragmentConsolidatorException : public StatusException { public: - explicit FragmentConsolidatorStatusException(const std::string& message) + explicit FragmentConsolidatorException(const std::string& message) : StatusException("FragmentConsolidator", message) { } }; @@ -293,7 +293,7 @@ Status FragmentConsolidator::consolidate_fragments( void FragmentConsolidator::vacuum(const char* array_name) { if (array_name == nullptr) { - throw Status_StorageManagerError( + throw FragmentConsolidatorException( "Cannot vacuum fragments; Array name cannot be null"); } @@ -348,7 +348,7 @@ bool FragmentConsolidator::are_consolidatable( size_t start, size_t end, const NDRange& union_non_empty_domains) const { - auto anterior_ndrange = fragment_info.anterior_ndrange(); + const auto& anterior_ndrange = fragment_info.anterior_ndrange(); if (anterior_ndrange.size() != 0 && domain.overlap(union_non_empty_domains, anterior_ndrange)) return false; @@ -419,8 +419,8 @@ Status FragmentConsolidator::consolidate_internal( // Prepare buffers auto average_var_cell_sizes = array_for_reads->get_average_var_cell_sizes(); - auto&& [buffers, buffer_sizes] = - create_buffers(stats_, config_, array_schema, average_var_cell_sizes); + FragmentConsolidationWorkspace cw{ + create_buffers(stats_, config_, array_schema, average_var_cell_sizes)}; // Create queries auto query_r = (Query*)nullptr; @@ -451,12 +451,13 @@ Status FragmentConsolidator::consolidate_internal( } // Read from one array and write to the other - st = copy_array(query_r, query_w, &buffers, &buffer_sizes); - if (!st.ok()) { + try { + copy_array(query_r, query_w, cw); + } catch (...) { tdb_delete(query_r); tdb_delete(query_w); - return st; - } + throw; + }; // Finalize write query st = query_w->finalize(); @@ -495,47 +496,41 @@ Status FragmentConsolidator::consolidate_internal( return st; } -Status FragmentConsolidator::copy_array( - Query* query_r, - Query* query_w, - std::vector* buffers, - std::vector* buffer_sizes) { +void FragmentConsolidator::copy_array( + Query* query_r, Query* query_w, FragmentConsolidationWorkspace& cw) { auto timer_se = stats_->start_timer("consolidate_copy_array"); // Set the read query buffers outside the repeated submissions. // The Reader will reset the query buffer sizes to the original // sizes, not the potentially smaller sizes of the results after // the query submission. - RETURN_NOT_OK(set_query_buffers(query_r, buffers, buffer_sizes)); + set_query_buffers(query_r, cw); do { // READ - RETURN_NOT_OK(query_r->submit()); + throw_if_not_ok(query_r->submit()); // If Consolidation cannot make any progress, throw. The first buffer will - // always contain fixed size data, wether it is tile offsets for var size + // always contain fixed size data, whether it is tile offsets for var size // attribute/dimension or the actual fixed size data so we can use its size // to know if any cells were written or not. - if (buffer_sizes->at(0) == 0) { - throw FragmentConsolidatorStatusException( + if (cw.sizes().at(0) == 0) { + throw FragmentConsolidatorException( "Consolidation read 0 cells, no progress can be made"); } // Set explicitly the write query buffers, as the sizes may have // been altered by the read query. - RETURN_NOT_OK(set_query_buffers(query_w, buffers, buffer_sizes)); + set_query_buffers(query_w, cw); // WRITE - RETURN_NOT_OK(query_w->submit()); + throw_if_not_ok(query_w->submit()); } while (query_r->status() == QueryStatus::INCOMPLETE); - - return Status::Ok(); } -tuple, std::vector> -FragmentConsolidator::create_buffers( +FragmentConsolidationWorkspace FragmentConsolidator::create_buffers( stats::Stats* stats, - const ConsolidationConfig& config, + const FragmentConsolidationConfig& config, const ArraySchema& array_schema, std::unordered_map& avg_cell_sizes) { auto timer_se = stats->start_timer("consolidate_create_buffers"); @@ -610,10 +605,11 @@ FragmentConsolidator::create_buffers( } // Create buffers. - std::vector buffers(buffer_num); - std::vector buffer_sizes(buffer_num); - auto total_weights = - std::accumulate(buffer_weights.begin(), buffer_weights.end(), 0); + FragmentConsolidationWorkspace cw{buffer_num}; + std::vector& buffers{cw.buffers()}; + std::vector& buffer_sizes{cw.sizes()}; + auto total_weights = std::accumulate( + buffer_weights.begin(), buffer_weights.end(), static_cast(0)); // Allocate space for each buffer. uint64_t adjusted_budget = total_budget / total_weights * total_weights; @@ -624,7 +620,7 @@ FragmentConsolidator::create_buffers( } // Success - return {buffers, buffer_sizes}; + return cw; } Status FragmentConsolidator::create_queries( @@ -810,10 +806,11 @@ Status FragmentConsolidator::compute_next_to_consolidate( return Status::Ok(); } -Status FragmentConsolidator::set_query_buffers( - Query* query, - std::vector* buffers, - std::vector* buffer_sizes) const { +void FragmentConsolidator::set_query_buffers( + Query* query, FragmentConsolidationWorkspace& cw) const { + std::vector* buffers{&cw.buffers()}; + std::vector* buffer_sizes{&cw.sizes()}; + const auto& array_schema = query->array_schema(); auto dim_num = array_schema.dim_num(); auto dense = array_schema.dense(); @@ -892,8 +889,6 @@ Status FragmentConsolidator::set_query_buffers( &(*buffer_sizes)[bid])); ++bid; } - - return Status::Ok(); } Status FragmentConsolidator::set_config(const Config& config) { diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index c4fc095734c7..1087e71c9f7e 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -53,6 +53,107 @@ class Config; class Query; class URI; +/* ********************************* */ +/* TYPE DEFINITIONS */ +/* ********************************* */ + +/** Consolidation configuration parameters. */ +struct FragmentConsolidationConfig : Consolidator::ConsolidationConfigBase { + /** + * Include timestamps in the consolidated fragment or not. + */ + bool with_timestamps_; + /** + * Include delete metadata in the consolidated fragment or not. + */ + bool with_delete_meta_; + /** + * The factor by which the size of the dense fragment resulting + * from consolidating a set of fragments (containing at least one + * dense fragment) can be amplified. This is important when + * the union of the non-empty domains of the fragments to be + * consolidated have a lot of empty cells, which the consolidated + * fragment will have to fill with the special fill value + * (since the resulting fragments is dense). + */ + float amplification_; + /** Attribute buffer size. */ + uint64_t buffer_size_; + /** Total buffer size for all attributes. */ + uint64_t total_buffer_size_; + /** Max fragment size. */ + uint64_t max_fragment_size_; + /** + * Number of consolidation steps performed in a single + * consolidation invocation. + */ + uint32_t steps_; + /** Minimum number of fragments to consolidate in a single step. */ + uint32_t min_frags_; + /** Maximum number of fragments to consolidate in a single step. */ + uint32_t max_frags_; + /** + * Minimum size ratio for two fragments to be considered for + * consolidation. + */ + float size_ratio_; + /** Is the refactored reader in use or not */ + bool use_refactored_reader_; + /** Purge deleted cells or not. */ + bool purge_deleted_cells_; +}; + +/** + * Consolidation workspace holds the large buffers used by the operation. + */ +class FragmentConsolidationWorkspace { + std::vector buffers_; + std::vector sizes_; + + public: + FragmentConsolidationWorkspace() = delete; + explicit FragmentConsolidationWorkspace(size_t n) + : buffers_(n) + , sizes_(n) { + } + /** + * Copy constructor is deleted to avoid copying large amounts of memory + */ + FragmentConsolidationWorkspace(const FragmentConsolidationWorkspace&) = + delete; + + /** + * Move constructor + */ + FragmentConsolidationWorkspace(FragmentConsolidationWorkspace&&) = default; + + /** + * Copy assignment is deleted to avoid copying large amounts of memory + */ + const FragmentConsolidationWorkspace& operator=( + const FragmentConsolidationWorkspace&) = delete; + + /** + * Move assignment is deleted for convenience + */ + const FragmentConsolidationWorkspace& operator=( + FragmentConsolidationWorkspace&&) = delete; + + /** + * Accessor for buffers + */ + std::vector& buffers() { + return buffers_; + } + + /** + * Access for sizes + */ + std::vector& sizes() { + return sizes_; + }; +}; + /** Handles fragment consolidation. */ class FragmentConsolidator : public Consolidator { friend class WhiteboxFragmentConsolidator; @@ -128,56 +229,6 @@ class FragmentConsolidator : public Consolidator { void vacuum(const char* array_name); private: - /* ********************************* */ - /* TYPE DEFINITIONS */ - /* ********************************* */ - - /** Consolidation configuration parameters. */ - struct ConsolidationConfig : Consolidator::ConsolidationConfigBase { - /** - * Include timestamps in the consolidated fragment or not. - */ - bool with_timestamps_; - /** - * Include delete metadata in the consolidated fragment or not. - */ - bool with_delete_meta_; - /** - * The factor by which the size of the dense fragment resulting - * from consolidating a set of fragments (containing at least one - * dense fragment) can be amplified. This is important when - * the union of the non-empty domains of the fragments to be - * consolidated have a lot of empty cells, which the consolidated - * fragment will have to fill with the special fill value - * (since the resulting fragments is dense). - */ - float amplification_; - /** Attribute buffer size. */ - uint64_t buffer_size_; - /** Total buffer size for all attributes. */ - uint64_t total_buffer_size_; - /** Max fragment size. */ - uint64_t max_fragment_size_; - /** - * Number of consolidation steps performed in a single - * consolidation invocation. - */ - uint32_t steps_; - /** Minimum number of fragments to consolidate in a single step. */ - uint32_t min_frags_; - /** Maximum number of fragments to consolidate in a single step. */ - uint32_t max_frags_; - /** - * Minimum size ratio for two fragments to be considered for - * consolidation. - */ - float size_ratio_; - /** Is the refactored reader in use or not */ - bool use_refactored_reader_; - /** Purge deleted cells or not. */ - bool purge_deleted_cells_; - }; - /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ @@ -239,13 +290,10 @@ class FragmentConsolidator : public Consolidator { * * @param query_r The read query. * @param query_w The write query. - * @return Status + * @param cw A workspace containing buffers for the queries */ - Status copy_array( - Query* query_r, - Query* query_w, - std::vector* buffers, - std::vector* buffer_sizes); + void copy_array( + Query* query_r, Query* query_w, FragmentConsolidationWorkspace& cw); /** * Creates the buffers that will be used upon reading the input fragments and @@ -256,11 +304,11 @@ class FragmentConsolidator : public Consolidator { * @param config The consolidation config. * @param array_schema The array schema. * @param avg_cell_sizes The average cell sizes. - * @return Buffers, Buffer sizes. + * @return a consolidation workspace containing the buffers */ - static tuple, std::vector> create_buffers( + static FragmentConsolidationWorkspace create_buffers( stats::Stats* stats, - const ConsolidationConfig& config, + const FragmentConsolidationConfig& config, const ArraySchema& array_schema, std::unordered_map& avg_cell_sizes); @@ -315,13 +363,10 @@ class FragmentConsolidator : public Consolidator { * if the array is sparse in the end. * * @param query The query to set the buffers to. - * @param The buffers to set. - * @param buffer_sizes The buffer sizes. + * @param cw Consolidation workspace containing the buffers */ - Status set_query_buffers( - Query* query, - std::vector* buffers, - std::vector* buffer_sizes) const; + void set_query_buffers( + Query* query, FragmentConsolidationWorkspace& cw) const; /** Writes the vacuum file that contains the URIs of the consolidated * fragments. */ @@ -336,7 +381,7 @@ class FragmentConsolidator : public Consolidator { /* ********************************* */ /** Consolidation configuration parameters. */ - ConsolidationConfig config_; + FragmentConsolidationConfig config_; }; } // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index a1b92493168d..14f8f9edf75c 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -43,7 +43,7 @@ using namespace tiledb::sm; namespace tiledb::sm { class WhiteboxFragmentConsolidator { public: - static tuple, std::vector> create_buffers( + static FragmentConsolidationWorkspace create_buffers( stats::Stats* stats, const bool with_timestamps, const bool with_delete_meta, @@ -51,7 +51,7 @@ class WhiteboxFragmentConsolidator { const ArraySchema& schema, std::unordered_map& avg_cell_sizes) { // Create config. - FragmentConsolidator::ConsolidationConfig cfg; + FragmentConsolidationConfig cfg; cfg.with_timestamps_ = with_timestamps; cfg.with_delete_meta_ = with_delete_meta; cfg.buffer_size_ = buffer_size; @@ -232,13 +232,16 @@ TEST_CASE( } // Create buffers. - auto&& [buffers, buffer_sizes] = WhiteboxFragmentConsolidator::create_buffers( - &statistics, - with_timestamps, - with_delete_meta, - 1000, - *schema, - avg_cell_sizes); + FragmentConsolidationWorkspace cw{ + WhiteboxFragmentConsolidator::create_buffers( + &statistics, + with_timestamps, + with_delete_meta, + 1000, + *schema, + avg_cell_sizes)}; + std::vector& buffers{cw.buffers()}; + std::vector& buffer_sizes{cw.sizes()}; // Validate. CHECK(buffers.size() == expected_sizes.size()); From b1432109fd2dfd66f8b9ad7f4615aa3e3603ac41 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 10 Jan 2024 20:31:15 +0100 Subject: [PATCH 130/456] Deprecate sm.consolidation.total_buffer_size. (#4613) We are going to use sm.mem.total_bugdet to allocate consolidation buffers. --- TYPE: NO_HISTORY DESC: Deprecate sm.consolidation.total_buffer_size. --- tiledb/api/c_api/config/config_api_external.h | 3 ++- tiledb/sm/cpp_api/config.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 8d240c40680c..cbbcaec71f85 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -164,11 +164,12 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * (since the resulting fragment is dense).
* **Default**: 1.0 * - `sm.consolidation.buffer_size`
- * Deprecated. Prefer `sm.consolidation.total_buffer_size` instead. + * **Deprecated** * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 * - `sm.consolidation.total_buffer_size`
+ * **Deprecated** * The size (in bytes) of all attribute buffers used during * consolidation.
* **Default**: 2,147,483,648 diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index afa8d8dcd251..dc4046238450 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -340,11 +340,12 @@ class Config { * (since the resulting fragments is dense).
* **Default**: 1.0 * - `sm.consolidation.buffer_size`
- * Deprecated. Prefer `sm.consolidation.total_buffer_size` instead. + * **Deprecated** * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 * - `sm.consolidation.total_buffer_size`
+ * **Deprecated** * The size (in bytes) of all attribute buffers used during * consolidation.
* **Default**: 2,147,483,648 From a89484c17c4cbfcbdfc562ffcff47d534c27814e Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Thu, 11 Jan 2024 20:19:17 +0200 Subject: [PATCH 131/456] Fix traversal limit in array deserialization (#4606) This is a similar issue to the one fixed here: https://github.com/TileDB-Inc/TileDB/pull/4287, we are hitting the traversal limit when loading fragment metadata as part of opening the array when queryv3 is enabled. The fix is also similar, we just need to extend it to array deserialization, as we hit the same error when trying to open an array with large metadata. --- TYPE: IMPROVEMENT DESC: Fix traversal limit in array deserialization --- tiledb/sm/serialization/array.cc | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index b0d3f3c93b78..fad14c9f2c0c 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -299,9 +299,9 @@ Status array_from_capnp( if (array_reader.hasFragmentMetadataAll()) { array->fragment_metadata().clear(); - array->fragment_metadata().reserve( - array_reader.getFragmentMetadataAll().size()); - for (auto frag_meta_reader : array_reader.getFragmentMetadataAll()) { + auto fragment_metadata_all_reader = array_reader.getFragmentMetadataAll(); + array->fragment_metadata().reserve(fragment_metadata_all_reader.size()); + for (auto frag_meta_reader : fragment_metadata_all_reader) { auto meta = make_shared(HERE()); RETURN_NOT_OK(fragment_metadata_from_capnp( array->array_schema_latest_ptr(), @@ -556,11 +556,21 @@ Status array_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = storage_manager->config() + .get("rest.capnp_traversal_limit") + .value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); capnp::Array::Reader array_reader = reader.getRoot(); RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array)); break; From 6b291176e6c63ce33bdd3e00418d7c80ae066acf Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 12 Jan 2024 09:01:22 +0100 Subject: [PATCH 132/456] Pass AWS client config when creating `STSAssumeRoleCredentialsProvider`. (#4616) This fixes an issue with assume role where the client configuration was not getting passed to the `STSAssumeRoleCredentialsProvider`, which resulted in options like the CA path not being honored when the AWS SDK was making HTTP requests for assume role. --- TYPE: BUG DESC: Fix HTTP requests for AWS assume role not honoring config options. Co-authored-by: Theodore Tsirpanis --- tiledb/sm/filesystem/s3.cc | 6 +++--- tiledb/sm/filesystem/s3.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index c85dc30125cf..3af7b981c08e 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -1307,8 +1308,7 @@ Status S3::init_client() const { // check for client configuration on create, which can be slow if aws is not // configured on a users systems due to ec2 metadata check - client_config_ = tdb_unique_ptr( - tdb_new(Aws::Client::ClientConfiguration)); + client_config_ = make_shared(HERE()); s3_tp_executor_ = make_shared(HERE(), vfs_thread_pool_); @@ -1403,7 +1403,7 @@ Status S3::init_client() const { session_name, external_id, load_frequency, - nullptr); + make_shared(HERE(), client_config)); break; } case 7: { diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 1bdf02107794..0f8f0765274b 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -1240,7 +1240,7 @@ class S3 { mutable std::mutex client_init_mtx_; /** Configuration object used to initialize the client. */ - mutable tdb_unique_ptr client_config_; + mutable shared_ptr client_config_; /** The executor used by 'client_'. */ mutable shared_ptr s3_tp_executor_; From d227c22c0901a986b846f44199e86da013dee7ac Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:49:19 +0100 Subject: [PATCH 133/456] Move common strategy parameters to class. (#4620) This moves common strategy parameters to a class so that adding/updating parameters is easier moving forward. Note that this doesn't move the stats and logger as those get modified in many places down the construction chain so that will require a later change that can be done with more care. --- TYPE: NO_HISTORY DESC: Move common strategy parameters to class. --- test/src/unit-Reader.cc | 8 +- .../deletes_and_updates.cc | 25 +--- .../deletes_and_updates/deletes_and_updates.h | 11 +- tiledb/sm/query/legacy/reader.cc | 32 +--- tiledb/sm/query/legacy/reader.h | 11 +- tiledb/sm/query/query.cc | 135 ++++------------- tiledb/sm/query/readers/dense_reader.cc | 30 +--- tiledb/sm/query/readers/dense_reader.h | 11 +- .../query/readers/ordered_dim_label_reader.cc | 33 +---- .../query/readers/ordered_dim_label_reader.h | 13 +- tiledb/sm/query/readers/reader_base.cc | 40 ++--- tiledb/sm/query/readers/reader_base.h | 12 +- .../readers/sparse_global_order_reader.cc | 60 ++------ .../readers/sparse_global_order_reader.h | 13 +- .../query/readers/sparse_index_reader_base.cc | 32 +--- .../query/readers/sparse_index_reader_base.h | 11 +- .../sparse_unordered_with_dups_reader.cc | 58 +------- .../sparse_unordered_with_dups_reader.h | 13 +- tiledb/sm/query/strategy_base.cc | 23 +-- tiledb/sm/query/strategy_base.h | 137 +++++++++++++++++- .../sm/query/writers/global_order_writer.cc | 24 +-- tiledb/sm/query/writers/global_order_writer.h | 10 +- tiledb/sm/query/writers/ordered_writer.cc | 24 +-- tiledb/sm/query/writers/ordered_writer.h | 10 +- tiledb/sm/query/writers/unordered_writer.cc | 24 +-- tiledb/sm/query/writers/unordered_writer.h | 10 +- tiledb/sm/query/writers/writer_base.cc | 24 +-- tiledb/sm/query/writers/writer_base.h | 10 +- 28 files changed, 267 insertions(+), 577 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 07256378aa79..ba32e601ff95 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -163,9 +163,7 @@ TEST_CASE_METHOD( .ok()); Subarray subarray(&array, &g_helper_stats, g_helper_logger()); DefaultChannelAggregates default_channel_aggregates; - Reader reader( - &g_helper_stats, - g_helper_logger(), + auto params = StrategyParams( context.storage_manager(), &array, config, @@ -174,7 +172,9 @@ TEST_CASE_METHOD( subarray, Layout::ROW_MAJOR, condition, - default_channel_aggregates); + default_channel_aggregates, + false); + Reader reader(&g_helper_stats, g_helper_logger(), params); unsigned dim_num = 2; auto size = 2 * sizeof(int32_t); int32_t domain_vec[] = {1, 10, 1, 15}; diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index 6413fdaf10fd..7896d94361ec 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -57,25 +57,10 @@ class DeleteAndUpdateStatusException : public StatusException { DeletesAndUpdates::DeletesAndUpdates( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - std::vector& update_values, - bool skip_checks_serialization) - : StrategyBase( - stats, - logger->clone("Deletes", ++logger_id_), - storage_manager, - array, - config, - buffers, - subarray, - layout) - , condition_(condition) + StrategyParams& params, + std::vector& update_values) + : StrategyBase(stats, logger->clone("Deletes", ++logger_id_), params) + , condition_(params.condition()) , update_values_(update_values) { // Sanity checks if (storage_manager_ == nullptr) { @@ -98,7 +83,7 @@ DeletesAndUpdates::DeletesAndUpdates( "Cannot initialize deletes; Subarrays are not supported"); } - if (!skip_checks_serialization && !condition_.has_value()) { + if (!params.skip_checks_serialization() && !condition_.has_value()) { throw DeleteAndUpdateStatusException( "Cannot initialize deletes; One condition is needed"); } diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.h b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.h index 7cda7e32e2a8..c6ec5a9d34fd 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.h +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.h @@ -54,15 +54,8 @@ class DeletesAndUpdates : public StrategyBase, public IQueryStrategy { DeletesAndUpdates( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - std::vector& update_values, - bool skip_checks_serialization = false); + StrategyParams& params, + std::vector& update_values); /** Destructor. */ ~DeletesAndUpdates(); diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index ebba83157217..075c50259930 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -107,52 +107,32 @@ inline IterT skip_invalid_elements(IterT it, const IterT& end) { Reader::Reader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization, + StrategyParams& params, bool remote_query) - : ReaderBase( - stats, - logger->clone("Reader", ++logger_id_), - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates) { + : ReaderBase(stats, logger->clone("Reader", ++logger_id_), params) { // Sanity checks if (storage_manager_ == nullptr) { throw ReaderStatusException( "Cannot initialize reader; Storage manager not set"); } - if (!default_channel_aggregates.empty()) { + if (!params.default_channel_aggregates().empty()) { throw ReaderStatusException( "Cannot initialize reader; Reader cannot process aggregates"); } - if (!skip_checks_serialization && buffers_.empty()) { + if (!params.skip_checks_serialization() && buffers_.empty()) { throw ReaderStatusException("Cannot initialize reader; Buffers not set"); } - if (!skip_checks_serialization && array_schema_.dense() && + if (!params.skip_checks_serialization() && array_schema_.dense() && !subarray_.is_set()) { throw ReaderStatusException( "Cannot initialize reader; Dense reads must have a subarray set"); } // Check subarray - check_subarray(remote_query && array->array_schema_latest().dense()); + check_subarray(remote_query && params.array()->array_schema_latest().dense()); // Initialize the read state init_read_state(); diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index ee3b6e93b2cd..057841d10a47 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -67,16 +67,7 @@ class Reader : public ReaderBase, public IQueryStrategy { Reader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization = false, + StrategyParams& params, bool remote_query = false); /** Destructor. */ diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 316376123f9b..0b9736930809 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -1789,6 +1789,17 @@ bool Query::is_aggregate(std::string output_field_name) const { /* ****************************** */ Status Query::create_strategy(bool skip_checks_serialization) { + auto params = StrategyParams( + storage_manager_, + array_, + config_, + buffers_, + aggregate_buffers_, + subarray_, + layout_, + condition_, + default_channel_aggregates_, + skip_checks_serialization); if (type_ == QueryType::WRITE || type_ == QueryType::MODIFY_EXCLUSIVE) { if (layout_ == Layout::COL_MAJOR || layout_ == Layout::ROW_MAJOR) { if (!array_schema_->dense()) { @@ -1800,17 +1811,11 @@ Status Query::create_strategy(bool skip_checks_serialization) { OrderedWriter, stats_->create_child("Writer"), logger_, - storage_manager_, - array_, - config_, - buffers_, - subarray_, - layout_, + params, written_fragment_info_, coords_info_, remote_query_, - fragment_name_, - skip_checks_serialization)); + fragment_name_)); } else if (layout_ == Layout::UNORDERED) { if (array_schema_->dense()) { return Status_QueryError( @@ -1821,37 +1826,25 @@ Status Query::create_strategy(bool skip_checks_serialization) { UnorderedWriter, stats_->create_child("Writer"), logger_, - storage_manager_, - array_, - config_, - buffers_, - subarray_, - layout_, + params, written_fragment_info_, coords_info_, written_buffers_, remote_query_, - fragment_name_, - skip_checks_serialization)); + fragment_name_)); } else if (layout_ == Layout::GLOBAL_ORDER) { strategy_ = tdb_unique_ptr(tdb_new( GlobalOrderWriter, stats_->create_child("Writer"), logger_, - storage_manager_, - array_, - config_, - buffers_, - subarray_, - layout_, + params, fragment_size_, written_fragment_info_, disable_checks_consolidation_, processed_conditions_, coords_info_, remote_query_, - fragment_name_, - skip_checks_serialization)); + fragment_name_)); } else { return Status_QueryError( "Cannot create strategy; unsupported layout " + layout_str(layout_)); @@ -1867,17 +1860,8 @@ Status Query::create_strategy(bool skip_checks_serialization) { OrderedDimLabelReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - dimension_label_increasing_, - skip_checks_serialization)); + params, + dimension_label_increasing_)); } else if (use_refactored_sparse_unordered_with_dups_reader( layout_, *array_schema_)) { auto&& [st, non_overlapping_ranges]{Query::non_overlapping_ranges()}; @@ -1889,31 +1873,13 @@ Status Query::create_strategy(bool skip_checks_serialization) { SparseUnorderedWithDupsReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - skip_checks_serialization)); + params)); } else { strategy_ = tdb_unique_ptr(tdb_new( SparseUnorderedWithDupsReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - skip_checks_serialization)); + params)); } } else if ( use_refactored_sparse_global_order_reader(layout_, *array_schema_) && @@ -1929,65 +1895,29 @@ Status Query::create_strategy(bool skip_checks_serialization) { SparseGlobalOrderReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - consolidation_with_timestamps_, - skip_checks_serialization)); + params, + consolidation_with_timestamps_)); } else { strategy_ = tdb_unique_ptr(tdb_new( SparseGlobalOrderReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - consolidation_with_timestamps_, - skip_checks_serialization)); + params, + consolidation_with_timestamps_)); } } else if (use_refactored_dense_reader(*array_schema_, all_dense)) { strategy_ = tdb_unique_ptr(tdb_new( DenseReader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - skip_checks_serialization, + params, remote_query_)); } else { strategy_ = tdb_unique_ptr(tdb_new( Reader, stats_->create_child("Reader"), logger_, - storage_manager_, - array_, - config_, - buffers_, - aggregate_buffers_, - subarray_, - layout_, - condition_, - default_channel_aggregates_, - skip_checks_serialization, + params, remote_query_)); } } else if (type_ == QueryType::DELETE || type_ == QueryType::UPDATE) { @@ -1995,15 +1925,8 @@ Status Query::create_strategy(bool skip_checks_serialization) { DeletesAndUpdates, stats_->create_child("Deletes"), logger_, - storage_manager_, - array_, - config_, - buffers_, - subarray_, - layout_, - condition_, - update_values_, - skip_checks_serialization)); + params, + update_values_)); } else { return logger_->status( Status_QueryError("Cannot create strategy; unsupported query type")); diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 5149a7121638..7cd916fe336d 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -71,30 +71,10 @@ class DenseReaderStatusException : public StatusException { DenseReader::DenseReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization, + StrategyParams& params, bool remote_query) - : ReaderBase( - stats, - logger->clone("DenseReader", ++logger_id_), - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates) - , array_memory_tracker_(array->memory_tracker()) { + : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) + , array_memory_tracker_(params.array()->memory_tracker()) { elements_mode_ = false; // Sanity checks. @@ -103,13 +83,13 @@ DenseReader::DenseReader( "Cannot initialize dense reader; Storage manager not set"); } - if (!skip_checks_serialization && buffers_.empty() && + if (!params.skip_checks_serialization() && buffers_.empty() && aggregate_buffers_.empty()) { throw DenseReaderStatusException( "Cannot initialize dense reader; Buffers not set"); } - if (!skip_checks_serialization && !subarray_.is_set()) { + if (!params.skip_checks_serialization() && !subarray_.is_set()) { throw DenseReaderStatusException( "Cannot initialize reader; Dense reads must have a subarray set"); } diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 5e6e90d0f657..8560171edecb 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -90,16 +90,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { DenseReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization = false, + StrategyParams& params, bool remote_query = false); /** Destructor. */ diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index fc45cf0c474d..0c618d9f17cf 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -70,31 +70,12 @@ class OrderedDimLabelReaderStatusException : public StatusException { OrderedDimLabelReader::OrderedDimLabelReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool increasing_labels, - bool skip_checks_serialization) + StrategyParams& params, + bool increasing_labels) : ReaderBase( - stats, - logger->clone("OrderedDimLabelReader", ++logger_id_), - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates) + stats, logger->clone("OrderedDimLabelReader", ++logger_id_), params) , ranges_( - subarray.get_attribute_ranges(array_schema_.attributes()[0]->name())) + subarray_.get_attribute_ranges(array_schema_.attributes()[0]->name())) , label_name_(array_schema_.attributes()[0]->name()) , label_type_(array_schema_.attributes()[0]->type()) , label_var_size_(array_schema_.attributes()[0]->var_size()) @@ -107,17 +88,17 @@ OrderedDimLabelReader::OrderedDimLabelReader( "Cannot initialize ordered dim label reader; Storage manager not set"); } - if (!default_channel_aggregates.empty()) { + if (!params.default_channel_aggregates().empty()) { throw OrderedDimLabelReaderStatusException( "Cannot initialize reader; Reader cannot process aggregates"); } - if (!skip_checks_serialization && buffers_.empty()) { + if (!params.skip_checks_serialization() && buffers_.empty()) { throw OrderedDimLabelReaderStatusException( "Cannot initialize ordered dim label reader; Buffers not set"); } - if (!skip_checks_serialization && buffers_.size() != 1) { + if (!params.skip_checks_serialization() && buffers_.size() != 1) { throw OrderedDimLabelReaderStatusException( "Cannot initialize ordered dim label reader with " + std::to_string(buffers_.size()) + " buffers; Only one buffer allowed"); diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.h b/tiledb/sm/query/readers/ordered_dim_label_reader.h index b321d6922a94..3c5dd9ff8ef1 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.h +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.h @@ -252,17 +252,8 @@ class OrderedDimLabelReader : public ReaderBase, public IQueryStrategy { OrderedDimLabelReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool increasing_order, - bool skip_checks_serialization); + StrategyParams& params, + bool increasing_order); /** Destructor. */ ~OrderedDimLabelReader() = default; diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 9f73eec3368f..7f6eb29c5e80 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -72,46 +72,28 @@ class ReaderBaseStatusException : public StatusException { /* ****************************** */ ReaderBase::ReaderBase( - stats::Stats* stats, - shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates) - : StrategyBase( - stats, - logger, - storage_manager, - array, - config, - buffers, - subarray, - layout) - , condition_(condition) + stats::Stats* stats, shared_ptr logger, StrategyParams& params) + : StrategyBase(stats, logger, params) + , condition_(params.condition()) , user_requested_timestamps_(false) , use_timestamps_(false) , initial_data_loaded_(false) - , max_batch_size_(config.get("vfs.max_batch_size").value()) - , min_batch_gap_(config.get("vfs.min_batch_gap").value()) - , min_batch_size_(config.get("vfs.min_batch_size").value()) - , aggregate_buffers_(aggregate_buffers) { - if (array != nullptr) - fragment_metadata_ = array->fragment_metadata(); + , max_batch_size_(config_.get("vfs.max_batch_size").value()) + , min_batch_gap_(config_.get("vfs.min_batch_gap").value()) + , min_batch_size_(config_.get("vfs.min_batch_size").value()) + , aggregate_buffers_(params.aggregate_buffers()) { + if (params.array() != nullptr) + fragment_metadata_ = params.array()->fragment_metadata(); timestamps_needed_for_deletes_and_updates_.resize(fragment_metadata_.size()); - if (layout_ == Layout::GLOBAL_ORDER && subarray.range_num() > 1) { + if (layout_ == Layout::GLOBAL_ORDER && subarray_.range_num() > 1) { throw ReaderBaseStatusException( "Cannot initialize reader; Multi-range reads are not supported on a " "global order query."); } // Validate the aggregates and store the requested aggregates by field name. - for (auto& aggregate : default_channel_aggregates) { + for (auto& aggregate : params.default_channel_aggregates()) { aggregate.second->validate_output_buffer( aggregate.first, aggregate_buffers_); aggregates_[aggregate.second->field_name()].emplace_back(aggregate.second); diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index 651a19855940..ca69ada01249 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -165,17 +165,7 @@ class ReaderBase : public StrategyBase { /** Constructor. */ ReaderBase( - stats::Stats* stats, - shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates); + stats::Stats* stats, shared_ptr logger, StrategyParams& params); /** Destructor. */ ~ReaderBase() = default; diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index b53666197cde..e2ddeb8ce511 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -71,36 +71,18 @@ template SparseGlobalOrderReader::SparseGlobalOrderReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool consolidation_with_timestamps, - bool skip_checks_serialization) + StrategyParams& params, + bool consolidation_with_timestamps) : SparseIndexReaderBase( "sparse_global_order", stats, - logger->clone("SparseGlobalOrderReader", ++logger_id_), - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates, - skip_checks_serialization, + logger->clone("SparseUnorderedWithDupsReader", ++logger_id_), + params, true) - , result_tiles_leftover_(array->fragment_metadata().size()) - , memory_used_for_coords_(array->fragment_metadata().size()) + , result_tiles_leftover_(array_->fragment_metadata().size()) + , memory_used_for_coords_(array_->fragment_metadata().size()) , consolidation_with_timestamps_(consolidation_with_timestamps) - , last_cells_(array->fragment_metadata().size()) + , last_cells_(array_->fragment_metadata().size()) , tile_offsets_loaded_(false) { // Initialize memory budget variables. refresh_config(); @@ -2234,33 +2216,9 @@ void SparseGlobalOrderReader::end_iteration( // Explicit template instantiations template SparseGlobalOrderReader::SparseGlobalOrderReader( - stats::Stats*, - shared_ptr, - StorageManager*, - Array*, - Config&, - std::unordered_map&, - std::unordered_map&, - Subarray&, - Layout, - std::optional&, - DefaultChannelAggregates&, - bool, - bool); + stats::Stats*, shared_ptr, StrategyParams&, bool); template SparseGlobalOrderReader::SparseGlobalOrderReader( - stats::Stats*, - shared_ptr, - StorageManager*, - Array*, - Config&, - std::unordered_map&, - std::unordered_map&, - Subarray&, - Layout, - std::optional&, - DefaultChannelAggregates&, - bool, - bool); + stats::Stats*, shared_ptr, StrategyParams&, bool); } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.h b/tiledb/sm/query/readers/sparse_global_order_reader.h index 3df66b98d5df..d213af6a253e 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.h +++ b/tiledb/sm/query/readers/sparse_global_order_reader.h @@ -71,17 +71,8 @@ class SparseGlobalOrderReader : public SparseIndexReaderBase, SparseGlobalOrderReader( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool consolidation_with_timestamps, - bool skip_checks_serialization = false); + StrategyParams& params, + bool consolidation_with_timestamps); /** Destructor. */ ~SparseGlobalOrderReader() = default; diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index f3c1e326e08e..615e8ecbdb1c 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -67,33 +67,13 @@ SparseIndexReaderBase::SparseIndexReaderBase( std::string reader_string, stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization, + StrategyParams& params, bool include_coords) - : ReaderBase( - stats, - logger, - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates) - , tmp_read_state_(array->fragment_metadata().size()) - , memory_budget_(config, reader_string) + : ReaderBase(stats, logger, params) + , tmp_read_state_(array_->fragment_metadata().size()) + , memory_budget_(config_, reader_string) , include_coords_(include_coords) - , array_memory_tracker_(array->memory_tracker()) + , array_memory_tracker_(params.array()->memory_tracker()) , memory_used_for_coords_total_(0) , deletes_consolidation_no_purge_( buffers_.count(constants::delete_timestamps) != 0) @@ -104,7 +84,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( "Cannot initialize reader; Storage manager not set"); } - if (!skip_checks_serialization && buffers_.empty() && + if (!params.skip_checks_serialization() && buffers_.empty() && aggregate_buffers_.empty()) { throw SparseIndexReaderBaseStatusException( "Cannot initialize reader; Buffers not set"); diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index c87cc445b557..33657acdc1b5 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -493,16 +493,7 @@ class SparseIndexReaderBase : public ReaderBase { std::string reader_string, stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization, + StrategyParams& params, bool include_coords); /** Destructor. */ diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index bcb055d9cf0f..e5740481c82b 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -67,40 +67,16 @@ class SparseUnorderedWithDupsReaderStatusException : public StatusException { template SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( - stats::Stats* stats, - shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization) + stats::Stats* stats, shared_ptr logger, StrategyParams& params) : SparseIndexReaderBase( - "sparse_unordered_with_dups", - stats, - logger->clone("SparseUnorderedWithDupsReader", ++logger_id_), - storage_manager, - array, - config, - buffers, - aggregate_buffers, - subarray, - layout, - condition, - default_channel_aggregates, - skip_checks_serialization, - false) + "sparse_unordered_with_dups", stats, logger, params, false) , tile_offsets_min_frag_idx_(std::numeric_limits::max()) , tile_offsets_max_frag_idx_(0) { // Initialize memory budget variables. refresh_config(); - // Get the setting that allows to partially load tile offsets. This is done - // for this reader only for now. + // Get the setting that allows to partially load tile offsets. This is + // done for this reader only for now. bool found = false; if (!config_ .get( @@ -2050,31 +2026,9 @@ void SparseUnorderedWithDupsReader::end_iteration( // Explicit template instantiations template SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( - stats::Stats*, - shared_ptr, - StorageManager*, - Array*, - Config&, - std::unordered_map&, - std::unordered_map&, - Subarray&, - Layout, - std::optional&, - DefaultChannelAggregates&, - bool); + stats::Stats*, shared_ptr, StrategyParams&); template SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( - stats::Stats*, - shared_ptr, - StorageManager*, - Array*, - Config&, - std::unordered_map&, - std::unordered_map&, - Subarray&, - Layout, - std::optional&, - DefaultChannelAggregates&, - bool); + stats::Stats*, shared_ptr, StrategyParams&); } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h index 386dbe5aebe9..7f271682c5f9 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h @@ -68,18 +68,7 @@ class SparseUnorderedWithDupsReader : public SparseIndexReaderBase, /** Constructor. */ SparseUnorderedWithDupsReader( - stats::Stats* stats, - shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - std::unordered_map& aggregate_buffers, - Subarray& subarray, - Layout layout, - std::optional& condition, - DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization = false); + stats::Stats* stats, shared_ptr logger, StrategyParams& params); /** Destructor. */ ~SparseUnorderedWithDupsReader() = default; diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 4f2bf511481f..393a1bff0c67 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -46,23 +46,16 @@ namespace sm { /* ****************************** */ StrategyBase::StrategyBase( - stats::Stats* stats, - shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout) + stats::Stats* stats, shared_ptr logger, StrategyParams& params) : stats_(stats) , logger_(logger) - , array_(array) - , array_schema_(array->array_schema_latest()) - , config_(config) - , buffers_(buffers) - , layout_(layout) - , storage_manager_(storage_manager) - , subarray_(subarray) + , array_(params.array()) + , array_schema_(params.array()->array_schema_latest()) + , config_(params.config()) + , buffers_(params.buffers()) + , layout_(params.layout()) + , storage_manager_(params.storage_manager()) + , subarray_(params.subarray()) , offsets_format_mode_(Config::SM_OFFSETS_FORMAT_MODE) , offsets_extra_element_(false) , offsets_bitsize_(constants::cell_var_offset_size * 8) { diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 6fb7e243aaef..fd2cce5a277e 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -45,27 +45,148 @@ namespace sm { class Array; class ArraySchema; +class IAggregator; enum class Layout : uint8_t; class Subarray; class QueryBuffer; +class QueryCondition; -/** Processes read or write queries. */ -class StrategyBase { +typedef std::unordered_map> + DefaultChannelAggregates; + +/** + * Class used to pass in common parameters to strategies. This will make it + * easier to change parameters moving fowards. + */ +class StrategyParams { public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Constructor. */ - StrategyBase( - stats::Stats* stats, - shared_ptr logger, + StrategyParams( StorageManager* storage_manager, Array* array, Config& config, std::unordered_map& buffers, + std::unordered_map& aggregate_buffers, Subarray& subarray, - Layout layout); + Layout layout, + std::optional& condition, + DefaultChannelAggregates& default_channel_aggregates, + bool skip_checks_serialization) + : storage_manager_(storage_manager) + , array_(array) + , config_(config) + , buffers_(buffers) + , aggregate_buffers_(aggregate_buffers) + , subarray_(subarray) + , layout_(layout) + , condition_(condition) + , default_channel_aggregates_(default_channel_aggregates) + , skip_checks_serialization_(skip_checks_serialization) { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Return the storage manager. */ + inline StorageManager* storage_manager() { + return storage_manager_; + }; + + /** Return the array. */ + inline Array* array() { + return array_; + }; + + /** Return the config. */ + inline Config& config() { + return config_; + }; + + /** Return the buffers. */ + inline std::unordered_map& buffers() { + return buffers_; + }; + + /** Return the aggregate buffers. */ + inline std::unordered_map& aggregate_buffers() { + return aggregate_buffers_; + }; + + /** Return the subarray. */ + inline Subarray& subarray() { + return subarray_; + }; + + /** Return the layout. */ + inline Layout layout() { + return layout_; + }; + + /** Return the condition. */ + inline std::optional& condition() { + return condition_; + } + + /** Return the default channel aggregates. */ + inline DefaultChannelAggregates& default_channel_aggregates() { + return default_channel_aggregates_; + } + + /** Return if we should skip checks for serialization. */ + inline bool skip_checks_serialization() { + return skip_checks_serialization_; + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Storage manager. */ + StorageManager* storage_manager_; + + /** Array. */ + Array* array_; + + /** Config for query-level parameters only. */ + Config& config_; + + /** Buffers. */ + std::unordered_map& buffers_; + + /** Aggregate buffers. */ + std::unordered_map& aggregate_buffers_; + + /** Query subarray (initially the whole domain by default). */ + Subarray& subarray_; + + /** Layout of the cells in the result of the subarray. */ + Layout layout_; + + /** Query condition. */ + std::optional& condition_; + + /** Default channel aggregates. */ + DefaultChannelAggregates& default_channel_aggregates_; + + /** Skip checks for serialization. */ + bool skip_checks_serialization_; +}; + +/** Processes read or write queries. */ +class StrategyBase { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Constructor. */ + StrategyBase( + stats::Stats* stats, shared_ptr logger, StrategyParams& params); /** Destructor. */ ~StrategyBase() = default; @@ -119,7 +240,7 @@ class StrategyBase { * Maps attribute/dimension names to their buffers. * `TILEDB_COORDS` may be used for the special zipped coordinates * buffer. - * */ + */ std::unordered_map& buffers_; /** The layout of the cells in the result of the subarray. */ diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 16f6287e9e99..cba606f6cd0c 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -73,43 +73,31 @@ class GlobalOrderWriterStatusException : public StatusException { GlobalOrderWriter::GlobalOrderWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, uint64_t fragment_size, std::vector& written_fragment_info, bool disable_checks_consolidation, std::vector& processed_conditions, Query::CoordsInfo& coords_info, bool remote_query, - optional fragment_name, - bool skip_checks_serialization) + optional fragment_name) : WriterBase( stats, logger, - storage_manager, - array, - config, - buffers, - subarray, - layout, + params, written_fragment_info, disable_checks_consolidation, coords_info, remote_query, - fragment_name, - skip_checks_serialization) + fragment_name) , processed_conditions_(processed_conditions) , fragment_size_(fragment_size) , current_fragment_size_(0) { // Check the layout is global order. - if (layout != Layout::GLOBAL_ORDER) { + if (layout_ != Layout::GLOBAL_ORDER) { throw GlobalOrderWriterStatusException( "Failed to initialize global order writer. Layout " + - layout_str(layout) + " is not global order."); + layout_str(layout_) + " is not global order."); } // Check no ordered attributes. diff --git a/tiledb/sm/query/writers/global_order_writer.h b/tiledb/sm/query/writers/global_order_writer.h index d65bf31d79d8..bf48eafb2320 100644 --- a/tiledb/sm/query/writers/global_order_writer.h +++ b/tiledb/sm/query/writers/global_order_writer.h @@ -108,20 +108,14 @@ class GlobalOrderWriter : public WriterBase { GlobalOrderWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, uint64_t fragment_size, std::vector& written_fragment_info, bool disable_checks_consolidation, std::vector& processed_conditions, Query::CoordsInfo& coords_info_, bool remote_query, - optional fragment_name = nullopt, - bool skip_checks_serialization = false); + optional fragment_name = nullopt); /** Destructor. */ ~GlobalOrderWriter(); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 13cd1dcf2cce..714b6b7d9b2f 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -67,38 +67,26 @@ namespace tiledb::sm { OrderedWriter::OrderedWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, Query::CoordsInfo& coords_info, bool remote_query, - optional fragment_name, - bool skip_checks_serialization) + optional fragment_name) : WriterBase( stats, logger, - storage_manager, - array, - config, - buffers, - subarray, - layout, + params, written_fragment_info, false, coords_info, remote_query, - fragment_name, - skip_checks_serialization) + fragment_name) , frag_uri_(std::nullopt) { - if (layout != Layout::ROW_MAJOR && layout != Layout::COL_MAJOR) { + if (layout_ != Layout::ROW_MAJOR && layout_ != Layout::COL_MAJOR) { throw StatusException(Status_WriterError( "Failed to initialize OrderedWriter; The ordered writer does not " "support layout " + - layout_str(layout))); + layout_str(layout_))); } if (!array_schema_.dense()) { diff --git a/tiledb/sm/query/writers/ordered_writer.h b/tiledb/sm/query/writers/ordered_writer.h index de6c52a4d133..14887f26a846 100644 --- a/tiledb/sm/query/writers/ordered_writer.h +++ b/tiledb/sm/query/writers/ordered_writer.h @@ -55,17 +55,11 @@ class OrderedWriter : public WriterBase { OrderedWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, Query::CoordsInfo& coords_info_, bool remote_query, - optional fragment_name = nullopt, - bool skip_checks_serialization = false); + optional fragment_name = nullopt); /** Destructor. */ ~OrderedWriter(); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 6c0e42622405..2084f9feb2ac 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -73,42 +73,30 @@ class UnorderWriterException : public StatusException { UnorderedWriter::UnorderedWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, Query::CoordsInfo& coords_info, std::unordered_set& written_buffers, bool remote_query, - optional fragment_name, - bool skip_checks_serialization) + optional fragment_name) : WriterBase( stats, logger, - storage_manager, - array, - config, - buffers, - subarray, - layout, + params, written_fragment_info, false, coords_info, remote_query, - fragment_name, - skip_checks_serialization) + fragment_name) , frag_uri_(std::nullopt) , written_buffers_(written_buffers) , is_coords_pass_(true) { // Check the layout is unordered. - if (layout != Layout::UNORDERED) { + if (layout_ != Layout::UNORDERED) { throw UnorderWriterException( "Failed to initialize UnorderedWriter; The unordered writer does not " "support layout " + - layout_str(layout)); + layout_str(layout_)); } // Check the array is sparse. diff --git a/tiledb/sm/query/writers/unordered_writer.h b/tiledb/sm/query/writers/unordered_writer.h index d5227a0eb9c4..2136ba509e74 100644 --- a/tiledb/sm/query/writers/unordered_writer.h +++ b/tiledb/sm/query/writers/unordered_writer.h @@ -55,18 +55,12 @@ class UnorderedWriter : public WriterBase { UnorderedWriter( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, Query::CoordsInfo& coords_info, std::unordered_set& written_buffers, bool remote_query, - optional fragment_name = nullopt, - bool skip_checks_serialization = false); + optional fragment_name = nullopt); /** Destructor. */ ~UnorderedWriter(); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 1579c6beac61..6e90fb3b7040 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -75,27 +75,13 @@ class WriterBaseStatusException : public StatusException { WriterBase::WriterBase( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, bool disable_checks_consolidation, Query::CoordsInfo& coords_info, bool remote_query, - optional fragment_name, - bool skip_checks_serialization) - : StrategyBase( - stats, - logger->clone("Writer", ++logger_id_), - storage_manager, - array, - config, - buffers, - subarray, - layout) + optional fragment_name) + : StrategyBase(stats, logger->clone("Writer", ++logger_id_), params) , disable_checks_consolidation_(disable_checks_consolidation) , coords_info_(coords_info) , check_coord_dups_(false) @@ -110,7 +96,7 @@ WriterBase::WriterBase( "Cannot initialize query; Storage manager not set"); } - if (!skip_checks_serialization && buffers_.empty()) { + if (!params.skip_checks_serialization() && buffers_.empty()) { throw WriterBaseStatusException( "Cannot initialize writer; Buffers not set"); } @@ -202,7 +188,7 @@ WriterBase::WriterBase( check_extra_element(); } - if (!skip_checks_serialization) { + if (!params.skip_checks_serialization()) { // Consolidation might set a subarray that is not tile aligned. if (!disable_checks_consolidation) { check_subarray(); diff --git a/tiledb/sm/query/writers/writer_base.h b/tiledb/sm/query/writers/writer_base.h index 486ff060a695..e289f5e936dc 100644 --- a/tiledb/sm/query/writers/writer_base.h +++ b/tiledb/sm/query/writers/writer_base.h @@ -70,18 +70,12 @@ class WriterBase : public StrategyBase, public IQueryStrategy { WriterBase( stats::Stats* stats, shared_ptr logger, - StorageManager* storage_manager, - Array* array, - Config& config, - std::unordered_map& buffers, - Subarray& subarray, - Layout layout, + StrategyParams& params, std::vector& written_fragment_info, bool disable_checks_consolidation, Query::CoordsInfo& coords_info_, bool remote_query, - optional fragment_name = nullopt, - bool skip_checks_serialization = false); + optional fragment_name = nullopt); /** Destructor. */ ~WriterBase(); From 4e7b97826f7f634e8d332f4192e33c73aaa8a5fb Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:10:04 -0500 Subject: [PATCH 134/456] Remove uuid in favor of random_label (#4589) Remove uuid in favor of random_label --- TYPE: IMPROVEMENT DESC: Remove uuid in favor of random_label --- test/src/unit-capi-vfs.cc | 8 +- test/src/unit-cppapi-array.cc | 2 +- test/src/unit-cppapi-vfs.cc | 2 +- test/src/unit-s3-no-multipart.cc | 2 +- test/src/unit-s3.cc | 2 +- test/src/unit-vfs.cc | 10 +- test/support/src/vfs_helpers.h | 6 +- tiledb/CMakeLists.txt | 1 - tiledb/common/random/random_label.cc | 4 +- tiledb/common/random/random_label.h | 10 +- tiledb/sm/array/CMakeLists.txt | 3 +- tiledb/sm/array_schema/CMakeLists.txt | 7 +- tiledb/sm/array_schema/enumeration.cc | 8 +- tiledb/sm/misc/CMakeLists.txt | 18 +-- tiledb/sm/misc/test/CMakeLists.txt | 5 +- tiledb/sm/misc/test/compile_uuid_main.cc | 34 ----- tiledb/sm/misc/test/unit_uuid.cc | 87 ----------- tiledb/sm/misc/uuid.cc | 174 ---------------------- tiledb/sm/misc/uuid.h | 59 -------- tiledb/storage_format/uri/CMakeLists.txt | 2 +- tiledb/storage_format/uri/generate_uri.cc | 8 +- 21 files changed, 35 insertions(+), 417 deletions(-) delete mode 100644 tiledb/sm/misc/test/compile_uuid_main.cc delete mode 100644 tiledb/sm/misc/test/unit_uuid.cc delete mode 100644 tiledb/sm/misc/uuid.cc delete mode 100644 tiledb/sm/misc/uuid.h diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index b0568adb99cc..e50339e01ece 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -145,7 +145,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { std::string s3_bucket; if (s3_enabled) { SECTION("Filesystem: S3") { - path = "s3://" + random_label("tiledb-") + "/tiledb_test/"; + path = "s3://tiledb-" + random_label() + "/tiledb_test/"; s3_bucket = path.substr(0, path.find("tiledb_test/")); // Check S3 bucket functionality @@ -171,7 +171,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { /** Note: Azure testing not currently enabled. if (azure_enabled) { SECTION("Filesystem: Azure") { - path = "azure://" + random_label("tiledb-") + "/tiledb_test/"; + path = "azure://tiledb-" + random_label() + "/tiledb_test/"; } } */ @@ -403,7 +403,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { // Move from one bucket to another (only for S3) if (backend_is_s3) { - std::string bucket2 = "s3://" + random_label("tiledb-") + "/"; + std::string bucket2 = "s3://tiledb-" + random_label() + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; // Remove and recreate bucket if already exists @@ -481,7 +481,7 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { // Copy from one bucket to another (only for S3) if (backend_is_s3) { - std::string bucket2 = "s3://" + random_label("tiledb-") + "/"; + std::string bucket2 = "s3://tiledb-" + random_label() + "/"; std::string subdir3 = bucket2 + "tiledb_test/subdir3/"; // Remove and recreate bucket if already exists diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 10525f6229f4..6e5b55ae0143 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -1883,7 +1883,7 @@ TEST_CASE("C++ API: Array write and read from MemFS", "[cppapi][memfs]") { TEST_CASE( "C++ API: Array on s3 with empty subfolders", "[cppapi][s3][empty_subfolders]") { - const std::string array_bucket = "s3://" + random_label("tiledb-") + "/"; + const std::string array_bucket = "s3://tiledb-" + random_label() + "/"; const std::string array_name = array_bucket + "cpp_unit_array/"; tiledb::Config cfg; diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index ccdb8a165e56..e4f4d40437f5 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -479,7 +479,7 @@ TEST_CASE( tiledb_ctx_is_supported_fs(ctx.ptr().get(), TILEDB_S3, &s3) == TILEDB_OK); if (s3) { tiledb::VFS vfs(ctx); - std::string bucket_name = "s3://" + random_label("tiledb-") + "/"; + std::string bucket_name = "s3://tiledb-" + random_label() + "/"; if (vfs.is_bucket(bucket_name)) { REQUIRE_NOTHROW(vfs.remove_bucket(bucket_name)); } diff --git a/test/src/unit-s3-no-multipart.cc b/test/src/unit-s3-no-multipart.cc index 1965882ef24e..272b09d7bd06 100644 --- a/test/src/unit-s3-no-multipart.cc +++ b/test/src/unit-s3-no-multipart.cc @@ -54,7 +54,7 @@ struct S3DirectFx { const std::string S3_PREFIX = "s3://"; const tiledb::sm::URI S3_BUCKET = - tiledb::sm::URI(S3_PREFIX + random_label("tiledb-") + "/"); + tiledb::sm::URI(S3_PREFIX + "tiledb-" + random_label() + "/"); const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/"; ThreadPool thread_pool_{2}; tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()}; diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index 219cfa98f750..bc24ffc37418 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -55,7 +55,7 @@ struct S3Fx { const std::string S3_PREFIX = "s3://"; const tiledb::sm::URI S3_BUCKET = - tiledb::sm::URI(S3_PREFIX + random_label("tiledb-") + "/"); + tiledb::sm::URI(S3_PREFIX + "tiledb-" + random_label() + "/"); const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/"; ThreadPool thread_pool_{2}; tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()}; diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index eee3542d1d6b..189bbe384ce3 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -155,12 +155,12 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok()); root_pairs.emplace_back( - URI("s3://" + random_label("vfs-") + "/"), std::move(config)); + URI("s3://vfs-" + random_label() + "/"), std::move(config)); } if constexpr (tiledb::sm::filesystem::hdfs_enabled) { Config config; root_pairs.emplace_back( - URI("hdfs:///" + random_label("vfs-") + "/"), std::move(config)); + URI("hdfs:///vfs-" + random_label() + "/"), std::move(config)); } if constexpr (tiledb::sm::filesystem::azure_enabled) { Config config; @@ -180,17 +180,17 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { .ok()); root_pairs.emplace_back( - URI("azure://" + random_label("vfs-") + "/"), std::move(config)); + URI("azure://vfs-" + random_label() + "/"), std::move(config)); } Config config; #ifdef _WIN32 root_pairs.emplace_back( - URI(tiledb::sm::Win::current_dir() + "\\" + random_label("vfs-") + "\\"), + URI(tiledb::sm::Win::current_dir() + "\\vfs-" + random_label() + "\\"), std::move(config)); #else root_pairs.emplace_back( - URI(Posix::current_dir() + "/" + random_label("vfs-") + "/"), + URI(Posix::current_dir() + "/vfs-" + random_label() + "/"), std::move(config)); #endif diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index 8416ce889d69..72170aac4cce 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -159,7 +159,7 @@ class SupportedFsS3 : public SupportedFs { public: SupportedFsS3() : s3_prefix_("s3://") - , s3_bucket_(s3_prefix_ + random_label("tiledb-") + "/") + , s3_bucket_(s3_prefix_ + "tiledb-" + random_label() + "/") , temp_dir_(s3_bucket_ + "tiledb_test/") { } @@ -286,7 +286,7 @@ class SupportedFsAzure : public SupportedFs { public: SupportedFsAzure() : azure_prefix_("azure://") - , container_(azure_prefix_ + random_label("tiledb-") + "/") + , container_(azure_prefix_ + "tiledb-" + random_label() + "/") , temp_dir_(container_ + "tiledb_test/") { } @@ -353,7 +353,7 @@ class SupportedFsGCS : public SupportedFs { public: SupportedFsGCS(std::string prefix = "gcs://") : prefix_(prefix) - , bucket_(prefix_ + random_label("tiledb-") + "/") + , bucket_(prefix_ + "tiledb-" + random_label() + "/") , temp_dir_(bucket_ + "tiledb_test/") { } diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index b9e02b88725b..a10c5350b4ae 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -243,7 +243,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/tdb_time.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/types.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/uuid.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc diff --git a/tiledb/common/random/random_label.cc b/tiledb/common/random/random_label.cc index a16a028dd6bc..e636c539110d 100644 --- a/tiledb/common/random/random_label.cc +++ b/tiledb/common/random/random_label.cc @@ -46,7 +46,7 @@ namespace tiledb::common { * values to be precisely a 32-digit hexadecimal value. Each value is padded * with 0s such that it makes up one 16-digit half of the full 32-digit number. */ -std::string random_label(std::string prefix) { +std::string random_label() { PRNG& prng = PRNG::get(); std::stringstream ss; @@ -57,7 +57,7 @@ std::string random_label(std::string prefix) { ss << std::hex << std::setw(16) << std::setfill('0') << rand2; // Return label string - return prefix + ss.str(); + return ss.str(); } } // namespace tiledb::common \ No newline at end of file diff --git a/tiledb/common/random/random_label.h b/tiledb/common/random/random_label.h index 103a613c5e4a..600a5831039b 100644 --- a/tiledb/common/random/random_label.h +++ b/tiledb/common/random/random_label.h @@ -38,19 +38,15 @@ namespace tiledb::common { /** - * Returns a PRNG-generated random label with the optionally-provided prefix. - * - * Given prefix "tiledb-", this function will return a label with syntax - * tiledb-<32-digit hexadecimal random number>. - * (Ex. tiledb-f258d22d4db9139204eef2b4b5d860cc). + * Returns a PRNG-generated label as a 32-digit hexadecimal random number. + * (Ex. f258d22d4db9139204eef2b4b5d860cc). * * Note: the random number is actually the combination of two 16-digit numbers. * The values are 0-padded to ensure exactly a 128-bit, 32-digit length. * - * @param prefix The optional prefix of the label. * @return A random label. */ -std::string random_label(std::string prefix = ""); +std::string random_label(); } // namespace tiledb::common diff --git a/tiledb/sm/array/CMakeLists.txt b/tiledb/sm/array/CMakeLists.txt index 1865ba11bec9..ca712b353983 100644 --- a/tiledb/sm/array/CMakeLists.txt +++ b/tiledb/sm/array/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -37,7 +37,6 @@ commence(object_library array) baseline generic_tile_io uri_format - uuid vfs ) if(TILEDB_STATS) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index efc7626d2b34..b5e58e29af30 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2022 TileDB, Inc. +# Copyright (c) 2021-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -38,8 +38,7 @@ commence(object_library attribute) constants filter_pipeline range - stringx - uuid) + stringx) conclude(object_library) # @@ -63,7 +62,7 @@ conclude(object_library) # commence(object_library enumeration) this_target_sources(enumeration.cc) - this_target_object_libraries(buffer constants uuid) + this_target_object_libraries(buffer constants seedable_global_PRNG) conclude(object_library) # diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index c6b2ab9633f7..3079e8b03d99 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -33,7 +33,7 @@ #include #include -#include "tiledb/sm/misc/uuid.h" +#include "tiledb/common/random/random_label.h" #include "enumeration.h" @@ -71,10 +71,8 @@ Enumeration::Enumeration( } if (path_name_.empty()) { - std::string tmp_uuid; - throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); - path_name_ = - "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); + path_name_ = "__" + tiledb::common::random_label() + "_" + + std::to_string(constants::enumerations_version); } if (path_name.find("/") != std::string::npos) { diff --git a/tiledb/sm/misc/CMakeLists.txt b/tiledb/sm/misc/CMakeLists.txt index 3e86c7cd4a9d..3c8733c69d56 100644 --- a/tiledb/sm/misc/CMakeLists.txt +++ b/tiledb/sm/misc/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2021-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -71,22 +71,6 @@ commence(object_library time) this_target_sources(tdb_time.cc) conclude(object_library) -# -# `uuid` object library -# -commence(object_library uuid) - this_target_sources(uuid.cc) - this_target_object_libraries(baseline) - if(WIN32) - this_target_link_libraries(rpcrt4) - else() - find_package(OpenSSL_EP REQUIRED) - this_target_link_libraries(OpenSSL::Crypto) - endif() -conclude(object_library) - -add_test_subdirectory() - # # `mgc_dict.*` tests are declared in this directory for the moment. # diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index 0c6bd18206d1..be13e19e3d31 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ include(unit_test) commence(unit_test misc) - this_target_object_libraries(math uuid) + this_target_object_libraries(math) this_target_link_libraries(tiledb_test_support_lib) # change to `this_target_include_directories` when available target_include_directories(unit_misc PRIVATE "${CMAKE_SOURCE_DIR}") @@ -36,6 +36,5 @@ commence(unit_test misc) unit_hilbert.cc unit_integral_type_casts.cc unit_math.cc - unit_uuid.cc ) conclude(unit_test) diff --git a/tiledb/sm/misc/test/compile_uuid_main.cc b/tiledb/sm/misc/test/compile_uuid_main.cc deleted file mode 100644 index 2db30c1d6e52..000000000000 --- a/tiledb/sm/misc/test/compile_uuid_main.cc +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @file compile_uuid_main.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "../uuid.h" - -int main() { - (void)tiledb::sm::uuid::generate_uuid(nullptr, false); - return 0; -} diff --git a/tiledb/sm/misc/test/unit_uuid.cc b/tiledb/sm/misc/test/unit_uuid.cc deleted file mode 100644 index 8dbf830b29cd..000000000000 --- a/tiledb/sm/misc/test/unit_uuid.cc +++ /dev/null @@ -1,87 +0,0 @@ -/** - * @file unit_uuid.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests the UUID utility functions. - */ - -#include -#include -#include -#include - -#include "tiledb/sm/global_state/global_state.h" -#include "tiledb/sm/misc/uuid.h" - -using namespace tiledb::sm; - -std::mutex catch2_macro_mutex; - -// A thread-safe variant of the REQUIRE macro. -#define REQUIRE_SAFE(a) \ - { \ - std::lock_guard lock(catch2_macro_mutex); \ - REQUIRE(a); \ - } - -void cancel_all_tasks(StorageManager*) { -} - -TEST_CASE("UUID: Test generate", "[uuid]") { - SECTION("- Serial") { - std::string uuid0, uuid1, uuid2; - REQUIRE(uuid::generate_uuid(&uuid0).ok()); - REQUIRE(uuid0.length() == 36); - REQUIRE(uuid::generate_uuid(&uuid1).ok()); - REQUIRE(uuid1.length() == 36); - REQUIRE(uuid0 != uuid1); - - REQUIRE(uuid::generate_uuid(&uuid2, false).ok()); - REQUIRE(uuid2.length() == 32); - } - - SECTION("- Threaded") { - const unsigned nthreads = 20; - std::vector uuids(nthreads); - std::vector threads; - for (unsigned i = 0; i < nthreads; i++) { - threads.emplace_back([&uuids, i]() { - std::string& uuid = uuids[i]; - REQUIRE_SAFE(uuid::generate_uuid(&uuid).ok()); - REQUIRE_SAFE(uuid.length() == 36); - }); - } - for (auto& t : threads) { - t.join(); - } - // Check uniqueness - std::set uuid_set; - uuid_set.insert(uuids.begin(), uuids.end()); - REQUIRE(uuid_set.size() == uuids.size()); - } -} diff --git a/tiledb/sm/misc/uuid.cc b/tiledb/sm/misc/uuid.cc deleted file mode 100644 index 052cce47bdcd..000000000000 --- a/tiledb/sm/misc/uuid.cc +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @file uuid.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines a platform-independent UUID generator. - */ - -#include -#include - -#include "tiledb/sm/misc/uuid.h" - -#ifdef _WIN32 -#include -#else -#include -#include -#include -#endif - -using namespace tiledb::common; - -namespace tiledb::sm::uuid { - -/** Mutex to guard UUID generation. */ -static std::mutex uuid_mtx; - -#ifdef _WIN32 - -/** - * Generate a UUID using Win32 RPC API. - */ -Status generate_uuid_win32(std::string* uuid_str) { - if (uuid_str == nullptr) - return Status_UtilsError("Null UUID string argument"); - - UUID uuid; - RPC_STATUS rc = UuidCreate(&uuid); - if (rc != RPC_S_OK) - return Status_UtilsError("Unable to generate Win32 UUID: creation error"); - - char* buf = nullptr; - rc = UuidToStringA(&uuid, reinterpret_cast(&buf)); - if (rc != RPC_S_OK) - return Status_UtilsError( - "Unable to generate Win32 UUID: string conversion error"); - - *uuid_str = std::string(buf); - - rc = RpcStringFreeA(reinterpret_cast(&buf)); - if (rc != RPC_S_OK) - return Status_UtilsError("Unable to generate Win32 UUID: free error"); - - return Status::Ok(); -} - -#else - -/** - * Generate a UUID using OpenSSL. - * - * Initially from: https://gist.github.com/kvelakur/9069c9896577c3040030 - * "Generating a Version 4 UUID using OpenSSL" - */ -Status generate_uuid_openssl(std::string* uuid_str) { - if (uuid_str == nullptr) - return Status_UtilsError("Null UUID string argument"); - - union { - struct { - uint32_t time_low; - uint16_t time_mid; - uint16_t time_hi_and_version; - uint8_t clk_seq_hi_res; - uint8_t clk_seq_low; - uint8_t node[6]; - }; - uint8_t __rnd[16]; - } uuid; - - int rc = RAND_bytes(uuid.__rnd, sizeof(uuid)); - if (rc < 1) { - char err_msg[256]; - ERR_error_string_n(ERR_get_error(), err_msg, sizeof(err_msg)); - return Status_UtilsError( - "Cannot generate random bytes with OpenSSL: " + std::string(err_msg)); - } - - // Refer Section 4.2 of RFC-4122 - // https://tools.ietf.org/html/rfc4122#section-4.2 - uuid.clk_seq_hi_res = (uint8_t)((uuid.clk_seq_hi_res & 0x3F) | 0x80); - uuid.time_hi_and_version = - (uint16_t)((uuid.time_hi_and_version & 0x0FFF) | 0x4000); - - // Format the UUID as a string. - char buf[128]; - rc = snprintf( - buf, - sizeof(buf), - "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", - uuid.time_low, - uuid.time_mid, - uuid.time_hi_and_version, - uuid.clk_seq_hi_res, - uuid.clk_seq_low, - uuid.node[0], - uuid.node[1], - uuid.node[2], - uuid.node[3], - uuid.node[4], - uuid.node[5]); - - if (rc < 0) - return Status_UtilsError("Error formatting UUID string"); - - *uuid_str = std::string(buf); - - return Status::Ok(); -} - -#endif - -Status generate_uuid(std::string* uuid, bool hyphenate) { - if (uuid == nullptr) - return Status_UtilsError("Null UUID string argument"); - - std::string uuid_str; - { - // OpenSSL is not threadsafe, so grab a lock here. We are locking in the - // Windows case as well just to be careful. - std::unique_lock lck(uuid_mtx); -#ifdef _WIN32 - RETURN_NOT_OK(generate_uuid_win32(&uuid_str)); -#else - RETURN_NOT_OK(generate_uuid_openssl(&uuid_str)); -#endif - } - - uuid->clear(); - for (unsigned i = 0; i < uuid_str.length(); i++) { - if (uuid_str[i] == '-' && !hyphenate) - continue; - uuid->push_back(uuid_str[i]); - } - - return Status::Ok(); -} - -} // namespace tiledb::sm::uuid diff --git a/tiledb/sm/misc/uuid.h b/tiledb/sm/misc/uuid.h deleted file mode 100644 index 5999007f8b59..000000000000 --- a/tiledb/sm/misc/uuid.h +++ /dev/null @@ -1,59 +0,0 @@ -/** - * @file uuid.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declares a platform-independent UUID generator. - */ - -#ifndef TILEDB_UUID_H -#define TILEDB_UUID_H - -#include "tiledb/common/status.h" - -using namespace tiledb::common; - -namespace tiledb { -namespace sm { -namespace uuid { - -/** - * Generates a 128-bit UUID. The string is formatted with hyphens like: - * 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx' where 'x' is a hexadecimal digit. - * Note: this function internally acquires a lock. - * - * @param uuid Output parameter which will store the UUID in string format. - * @param hyphenate If false, the UUID string will not be hyphenated. - * @return Status - */ -Status generate_uuid(std::string* uuid, bool hyphenate = true); - -} // namespace uuid -} // namespace sm -} // namespace tiledb - -#endif diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 3abf73b8ca6f..2f54390eee2b 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -31,7 +31,7 @@ include(object_library) # commence(object_library uri_format) this_target_sources(parse_uri.cc generate_uri.cc) - this_target_object_libraries(baseline time uuid vfs) + this_target_object_libraries(baseline time vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index fa32a2906110..3506fe7e66d6 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -27,8 +27,8 @@ */ #include "tiledb/storage_format/uri/generate_uri.h" +#include "tiledb/common/random/random_label.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -41,9 +41,6 @@ std::string generate_timestamped_name( uint64_t timestamp_start, uint64_t timestamp_end, std::optional version) { - std::string uuid; - throw_if_not_ok(sm::uuid::generate_uuid(&uuid, false)); - if (timestamp_start > timestamp_end) { throw std::logic_error( "Error generating timestamped name; " @@ -51,7 +48,8 @@ std::string generate_timestamped_name( } std::stringstream ss; - ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid; + ss << "/__" << timestamp_start << "_" << timestamp_end << "_" + << random_label(); if (version.has_value()) { ss << "_" << version.value(); From b8bc74a71e9ec8e61973ef48480430b60a0ae4d8 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Fri, 12 Jan 2024 13:14:26 -0500 Subject: [PATCH 135/456] Add experimental C API for ls_recursive. (#4615) This adds experimental C API for ls_recursive, which is currently only supported over S3. --- TYPE: FEATURE DESC: Add experimental C API for ls_recursive. --------- Co-authored-by: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> --- CMakeLists.txt | 1 + test/CMakeLists.txt | 1 + .../c_api/vfs/test/unit_capi_ls_recursive.cc | 113 ++++++++++++++++++ tiledb/api/c_api/vfs/test/unit_capi_vfs.cc | 106 ++++++++++++++++ tiledb/api/c_api/vfs/vfs_api.cc | 29 +++++ tiledb/api/c_api/vfs/vfs_api_experimental.h | 102 ++++++++++++++++ tiledb/api/c_api/vfs/vfs_api_internal.h | 9 ++ tiledb/sm/c_api/tiledb_experimental.h | 1 + tiledb/sm/filesystem/ls_scanner.h | 72 +++++++++++ tiledb/sm/filesystem/vfs.h | 23 ++-- 10 files changed, 450 insertions(+), 7 deletions(-) create mode 100644 tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc create mode 100644 tiledb/api/c_api/vfs/vfs_api_experimental.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cc7c43f05abf..c84b2d66fe08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -352,6 +352,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/string/string_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/vfs_api_experimental.h" ) set(TILEDB_C_API_RELATIVE_HEADER_BASE "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 12987e299340..ec67498ef670 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -203,6 +203,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-SubarrayPartitioner-sparse.cc src/unit-vfs.cc src/unit-win-filesystem.cc + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc" ) if (TILEDB_CPP_API) diff --git a/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc new file mode 100644 index 000000000000..5792fe82fd67 --- /dev/null +++ b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc @@ -0,0 +1,113 @@ +/** + * @file tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C API for tiledb_vfs_ls_recursive. + * + * TODO: This test is built and ran as part of tiledb_unit. Once we're able to + * execute these tests in CI, we should build this test as a separate unit. + */ + +#include +#include "test/support/src/vfs_helpers.h" + +using namespace tiledb::test; + +TEST_CASE("C API: ls_recursive callback", "[vfs][ls-recursive]") { + using tiledb::sm::LsObjects; + S3Test s3_test({10, 50}); + if (!s3_test.is_supported()) { + return; + } + auto expected = s3_test.expected_results(); + + vfs_config vfs_config; + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(vfs_config.config, &ctx); + tiledb_vfs_t* vfs; + tiledb_vfs_alloc(ctx, vfs_config.config, &vfs); + + LsObjects data; + tiledb_ls_callback_t cb = [](const char* path, + size_t path_len, + uint64_t object_size, + void* data) -> int32_t { + auto* ls_data = static_cast(data); + ls_data->push_back({{path, path_len}, object_size}); + return 1; + }; + + // This callback will return 0 exactly once. Traversal should stop immediately + // and not continue to the next object. + SECTION("callback stops traversal") { + cb = [](const char* path, + size_t path_len, + uint64_t object_size, + void* data) -> int32_t { + // There's no precheck here to push_back, so the vector size will match + // the number of times the callback was executed. + auto* ls_data = static_cast(data); + ls_data->push_back({{path, path_len}, object_size}); + // Stop traversal after we collect 10 results. + return ls_data->size() != 10; + }; + expected.resize(10); + } + + CHECK( + tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) == + TILEDB_OK); + CHECK(data.size() == expected.size()); + CHECK(data == expected); +} + +TEST_CASE("C API: ls_recursive throwing callback", "[vfs][ls-recursive]") { + using tiledb::sm::LsObjects; + S3Test s3_test({10, 50}); + if (!s3_test.is_supported()) { + return; + } + auto expected = s3_test.expected_results(); + + vfs_config vfs_config; + tiledb_ctx_t* ctx; + tiledb_ctx_alloc(vfs_config.config, &ctx); + tiledb_vfs_t* vfs; + tiledb_vfs_alloc(ctx, vfs_config.config, &vfs); + + LsObjects data; + tiledb_ls_callback_t cb = + [](const char*, size_t, uint64_t, void*) -> int32_t { + throw std::runtime_error("Throwing callback"); + }; + + CHECK( + tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) == + TILEDB_ERR); + CHECK(data.empty()); +} diff --git a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc index d7e2ec7c9ce0..985406fb9195 100644 --- a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc +++ b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc @@ -675,3 +675,109 @@ TEST_CASE( CHECK(error != nullptr); } + +TEST_CASE("C API: tiledb_vfs_ls_recursive argument validation", "[capi][vfs]") { + /* + * No "success" sections here; too much overhead to set up. + */ + ordinary_vfs x; + int32_t data; + auto cb = [](const char*, size_t, uint64_t, void*) { return 0; }; + SECTION("null context") { + auto rc{tiledb_vfs_ls_recursive(nullptr, x.vfs, TEST_URI, cb, &data)}; + CHECK(tiledb_status(rc) == TILEDB_INVALID_CONTEXT); + } + SECTION("null vfs") { + auto rc{tiledb_vfs_ls_recursive(x.ctx, nullptr, TEST_URI, cb, &data)}; + CHECK(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null uri") { + auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, nullptr, cb, &data)}; + CHECK(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null callback") { + auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, TEST_URI, nullptr, &data)}; + CHECK(tiledb_status(rc) == TILEDB_ERR); + } + SECTION("null data ptr") { + auto rc{tiledb_vfs_ls_recursive(x.ctx, x.vfs, TEST_URI, cb, nullptr)}; + CHECK(tiledb_status(rc) == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: VFS recursive ls unsupported backends", + "[capi][vfs][ls-recursive]") { + ordinary_vfs vfs; + int ls_data; + auto cb = [](const char*, size_t, uint64_t, void*) { return 0; }; + // Recursive ls is currently only supported for S3. + tiledb::sm::URI uri{GENERATE( + "file:///path/", + "mem:///path/", + "azure://path/", + "gcs://path/", + "hdfs://path/")}; + DYNAMIC_SECTION( + "Test recursive ls usupported backend over " << uri.backend_name()) { + if (!vfs.vfs->vfs()->supports_uri_scheme(uri)) { + return; + } + CHECK( + tiledb_vfs_ls_recursive(vfs.ctx, vfs.vfs, uri.c_str(), cb, &ls_data) == + TILEDB_ERR); + } +} + +TEST_CASE( + "C API: CallbackWrapper operator() validation", + "[ls-recursive][callback][wrapper]") { + tiledb::sm::LsObjects data; + auto cb = [](const char* path, + size_t path_len, + uint64_t object_size, + void* data) -> int32_t { + if (object_size > 100) { + // Throw if object size is greater than 100 bytes. + throw std::runtime_error("Throwing callback"); + } else if (!std::string(path, path_len).ends_with(".txt")) { + // Reject non-txt files. + return 0; + } + auto* ls_data = static_cast(data); + ls_data->push_back({{path, path_len}, object_size}); + return 1; + }; + tiledb::sm::CallbackWrapper wrapper(cb, &data); + + SECTION("Callback return 1 signals to continue traversal") { + CHECK(wrapper("file.txt", 10) == 1); + CHECK(data.size() == 1); + } + SECTION("Callback return 0 signals to stop traversal") { + CHECK_THROWS_AS(wrapper("some/dir/", 0) == 0, tiledb::sm::LsStopTraversal); + } + SECTION("Callback exception is propagated") { + CHECK_THROWS_WITH(wrapper("path", 101) == 0, "Throwing callback"); + } +} + +TEST_CASE( + "C API: CallbackWrapper construction validation", + "[ls-recursive][callback][wrapper]") { + using tiledb::sm::CallbackWrapper; + tiledb::sm::LsObjects data; + auto cb = [](const char*, size_t, uint64_t, void*) -> int32_t { return 1; }; + SECTION("Null callback") { + CHECK_THROWS(CallbackWrapper(nullptr, &data)); + } + SECTION("Null data") { + CHECK_THROWS(CallbackWrapper(cb, nullptr)); + } + SECTION("Null callback and data") { + CHECK_THROWS(CallbackWrapper(nullptr, nullptr)); + } + SECTION("Valid callback and data") { + CHECK_NOTHROW(CallbackWrapper(cb, &data)); + } +} diff --git a/tiledb/api/c_api/vfs/vfs_api.cc b/tiledb/api/c_api/vfs/vfs_api.cc index 04144755f0b9..8491f82ea094 100644 --- a/tiledb/api/c_api/vfs/vfs_api.cc +++ b/tiledb/api/c_api/vfs/vfs_api.cc @@ -31,6 +31,7 @@ */ #include "tiledb/api/c_api_support/c_api_support.h" +#include "vfs_api_experimental.h" #include "vfs_api_internal.h" namespace tiledb::api { @@ -311,6 +312,23 @@ capi_return_t tiledb_vfs_touch(tiledb_vfs_t* vfs, const char* uri) { return TILEDB_OK; } +capi_return_t tiledb_vfs_ls_recursive( + tiledb_vfs_t* vfs, + const char* path, + tiledb_ls_callback_t callback, + void* data) { + ensure_vfs_is_valid(vfs); + if (path == nullptr) { + throw CAPIStatusException("Invalid TileDB object: VFS passed a null path."); + } else if (callback == nullptr) { + throw CAPIStatusException( + "Invalid TileDB object: Callback function is null."); + } + ensure_output_pointer_is_valid(data); + vfs->ls_recursive(tiledb::sm::URI(path), callback, data); + return TILEDB_OK; +} + } // namespace tiledb::api using tiledb::api::api_entry_context; @@ -544,3 +562,14 @@ CAPI_INTERFACE( vfs_touch, tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const char* uri) { return api_entry_context(ctx, vfs, uri); } + +CAPI_INTERFACE( + vfs_ls_recursive, + tiledb_ctx_t* ctx, + tiledb_vfs_t* vfs, + const char* path, + tiledb_ls_callback_t callback, + void* data) { + return api_entry_context( + ctx, vfs, path, callback, data); +} diff --git a/tiledb/api/c_api/vfs/vfs_api_experimental.h b/tiledb/api/c_api/vfs/vfs_api_experimental.h new file mode 100644 index 000000000000..655a4a84138e --- /dev/null +++ b/tiledb/api/c_api/vfs/vfs_api_experimental.h @@ -0,0 +1,102 @@ +/** + * @file tiledb/api/c_api/vfs/vfs_api_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental VFS C API for TileDB. + */ + +#ifndef TILEDB_VFS_API_EXPERIMENTAL_H +#define TILEDB_VFS_API_EXPERIMENTAL_H + +#include "tiledb/api/c_api/api_external_common.h" +#include "tiledb/api/c_api/context/context_api_external.h" +#include "tiledb/api/c_api/vfs/vfs_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Typedef for ls_recursive callback function invoked on each object collected. + * + * @param path The path of a visited object for the relative filesystem. + * @param path_len The length of the path. + * @param object_size The size of the object at the current path. + * @param data Data passed to the callback used to store collected results. + */ +typedef int32_t (*tiledb_ls_callback_t)( + const char* path, size_t path_len, uint64_t object_size, void* data); + +/** + * Visits the children of `path` recursively, invoking the callback for each + * entry. The callback should return 1 to continue traversal, 0 to stop, or -1 + * on error. The callback is responsible for writing gathered entries into the + * `data` buffer, for example using a pointer to a user-defined struct. + * + * Currently only S3 is supported, and the `path` must be a valid S3 URI. + * + * **Example:** + * + * @code{.c} + * int my_callback( + * const char* path, size_t path_length, uint64_t file_size, void* data) { + * MyCbStruct cb_data = static_cast(data); + * // Perform custom callback behavior here. + * return 1; // Continue traversal to next entry. + * } + * MyCbStruct* cb_data = allocate_cb_struct(); + * + * tiledb_vfs_ls_recursive(ctx, vfs, "s3://bucket/foo", my_callback, &cb_data); + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] vfs The virtual filesystem object. + * @param[in] path The path in which the traversal will occur. + * @param[in] callback + * The callback function to be applied on every visited object. + * The callback should return `0` if the iteration must stop, and `1` + * if the iteration must continue. It takes as input the currently visited + * path, the length of the currently visited path, the size of the file, and + * user provided buffer for paths and object sizes in the form of a struct + * pointer. The callback returns `-1` upon error. Note that `path` in the + * callback will be an **absolute** path. + * @param[in] data Data pointer passed into the callback for storing results. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_vfs_ls_recursive( + tiledb_ctx_t* ctx, + tiledb_vfs_t* vfs, + const char* path, + tiledb_ls_callback_t callback, + void* data) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_VFS_API_EXPERIMENTAL_H \ No newline at end of file diff --git a/tiledb/api/c_api/vfs/vfs_api_internal.h b/tiledb/api/c_api/vfs/vfs_api_internal.h index a91d3aef08e8..fb8dd98f6512 100644 --- a/tiledb/api/c_api/vfs/vfs_api_internal.h +++ b/tiledb/api/c_api/vfs/vfs_api_internal.h @@ -38,6 +38,7 @@ #include "tiledb/sm/enums/vfs_mode.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/filesystem/vfs_file_handle.h" +#include "vfs_api_experimental.h" #include "vfs_api_external.h" /** Handle `struct` for API VFS objects. */ @@ -145,6 +146,14 @@ struct tiledb_vfs_handle_t Status touch(const tiledb::sm::URI& uri) const { return vfs_.touch(uri); } + + void ls_recursive( + const tiledb::sm::URI& parent, + tiledb_ls_callback_t cb, + void* data) const { + tiledb::sm::CallbackWrapper wrapper(cb, data); + vfs_.ls_recursive(parent, wrapper); + } }; /** Handle `struct` for API VFS file handle objects. */ diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 7c8c5c8bed72..bebb1c9ae9c9 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -47,6 +47,7 @@ #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" +#include "tiledb/api/c_api/vfs/vfs_api_experimental.h" #include "tiledb_dimension_label_experimental.h" /* ********************************* */ diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h index 724ae667e05e..e357e76d7c65 100644 --- a/tiledb/sm/filesystem/ls_scanner.h +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -59,6 +59,19 @@ class LsScanException : public StatusException { } }; +/** + * Exception thrown when the user callback signals to stop traversal. This will + * not result in an error and is only used to stop traversal early. + * + * @sa VFS::ls_recursive + */ +class LsStopTraversal : public LsScanException { + public: + explicit LsStopTraversal() + : LsScanException("Callback signaled to stop traversal") { + } +}; + using FileFilter = std::function; using DirectoryFilter = std::function; @@ -67,6 +80,18 @@ using DirectoryFilter = std::function; return true; } +/** + * Typedef for ls C API callback as std::function for passing to C++ + * + * @param path The path of a visited object for the relative filesystem. + * @param path_len The length of the path. + * @param object_size The size of the object at the current path. + * @param data Data passed to the callback used to store collected results. + * @return 1 if the callback should continue to the next object, 0 to stop + * traversal, or -1 if an error occurred. + */ +typedef std::function LsCallback; + /** Type defintion for objects returned from ls_recursive. */ using LsObjects = std::vector>; @@ -239,6 +264,53 @@ class LsScanner { const bool is_recursive_; }; +/** + * Wrapper for the C API callback function to be passed to the C++ API. + */ +class CallbackWrapper { + public: + /** Default constructor is deleted */ + CallbackWrapper() = delete; + + /** Constructor */ + CallbackWrapper(LsCallback cb, void* data) + : cb_(cb) + , data_(data) { + if (cb_ == nullptr) { + throw LsScanException("ls_recursive callback function cannot be null"); + } else if (data_ == nullptr) { + throw LsScanException("ls_recursive data cannot be null"); + } + } + + /** + * Operator to wrap the FilePredicate used in the C++ API. + * This will throw a LsStopTraversal exception if the user callback returns 0, + * and will throw a LsScanException if the user callback returns -1. + * + * @param path The path of the object. + * @param size The size of the object in bytes. + * @return True if the object should be included, False otherwise. + */ + bool operator()(std::string_view path, const uint64_t& size) const { + int ret = cb_(path.data(), path.size(), size, data_); + if (ret == 0) { + // Throw an exception to stop traversal, which will be caught by the C++ + // internal ls_recursive implementation to stop traversal. + throw LsStopTraversal(); + } else if (ret == -1) { + throw LsScanException("Error in user callback"); + } + return ret; + } + + private: + /** CAPI callback as function object */ + LsCallback cb_; + /** User data for callback */ + void* data_; +}; + } // namespace tiledb::sm #endif // TILEDB_LS_SCANNER_H diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 99c5ac4e77e0..1549b754fa5d 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -528,17 +528,26 @@ class VFS : private VFSBase, protected S3_within_VFS { const URI& parent, [[maybe_unused]] F f, [[maybe_unused]] D d = accept_all_dirs) const { - if (parent.is_s3()) { + LsObjects results; + try { + if (parent.is_s3()) { #ifdef HAVE_S3 - return s3().ls_filtered(parent, f, d, true); + results = s3().ls_filtered(parent, f, d, true); #else - throw filesystem::VFSException("TileDB was built without S3 support"); + throw filesystem::VFSException("TileDB was built without S3 support"); #endif - } else { - throw filesystem::VFSException( - "Recursive ls over " + parent.backend_name() + - " storage backend is not supported."); + } else { + throw filesystem::VFSException( + "Recursive ls over " + parent.backend_name() + + " storage backend is not supported."); + } + } catch (LsStopTraversal& e) { + // Do nothing, callback signaled to stop traversal. + } catch (...) { + // Rethrow exception thrown by the callback. + throw; } + return results; } /** From 45361effee71d1300d05c1e89822c3a5cffdb51d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 12 Jan 2024 19:15:00 +0100 Subject: [PATCH 136/456] Keep opened array resources alive until query ends. (#4601) This change moves all the opened array resources used in strategies to a class so that they can be acquired as a shared pointer when the strategy begins and kept alive until the strategy is done. [sc-38872] --- TYPE: IMPROVEMENT DESC: Keep opened array resources alive until query ends. --------- Co-authored-by: Paul J. Davis --- test/src/unit-Reader.cc | 5 +- test/src/unit-capi-array.cc | 2 +- test/src/unit-capi-fragment_info.cc | 2 + test/src/unit-cppapi-array.cc | 49 +++ test/src/unit-cppapi-deletes.cc | 2 +- test/src/unit-cppapi-subarray.cc | 26 -- test/src/unit-cppapi-update-queries.cc | 2 +- tiledb/sm/array/array.cc | 405 +++++++++--------- tiledb/sm/array/array.h | 327 +++++++++++--- tiledb/sm/array/array_directory.h | 2 +- tiledb/sm/array/test/CMakeLists.txt | 1 + tiledb/sm/array/test/unit_array_directory.cc | 17 +- tiledb/sm/c_api/tiledb.cc | 21 +- .../consolidator/array_meta_consolidator.cc | 17 +- tiledb/sm/query/query.cc | 10 +- tiledb/sm/query/query.h | 9 +- tiledb/sm/query/readers/dense_reader.cc | 2 +- .../query/readers/sparse_index_reader_base.cc | 2 +- tiledb/sm/query/strategy_base.h | 33 +- tiledb/sm/serialization/array.cc | 22 +- tiledb/sm/serialization/capnp_utils.h | 12 +- tiledb/sm/storage_manager/storage_manager.cc | 21 +- .../storage_manager_canonical.h | 5 +- .../subarray/relevant_fragment_generator.cc | 18 +- .../sm/subarray/relevant_fragment_generator.h | 10 +- tiledb/sm/subarray/subarray.cc | 37 +- tiledb/sm/subarray/subarray.h | 29 +- .../sm/subarray/test/unit_add_ranges_list.cc | 3 - 28 files changed, 678 insertions(+), 413 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index ba32e601ff95..dbc2cd34287e 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -165,7 +165,7 @@ TEST_CASE_METHOD( DefaultChannelAggregates default_channel_aggregates; auto params = StrategyParams( context.storage_manager(), - &array, + array.opened_array(), config, buffers, aggregate_buffers, @@ -173,7 +173,8 @@ TEST_CASE_METHOD( Layout::ROW_MAJOR, condition, default_channel_aggregates, - false); + false, + array.memory_tracker()); Reader reader(&g_helper_stats, g_helper_logger(), params); unsigned dim_num = 2; auto size = 2 * sizeof(int32_t); diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index e9e97df65628..129a219a8f56 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -2337,7 +2337,7 @@ TEST_CASE_METHOD( // Check the retrieved non empty domain auto non_empty_domain = new_array->array_->loaded_non_empty_domain(); - CHECK(non_empty_domain->empty() == false); + CHECK(non_empty_domain.empty() == false); // Check the retrieved metadata Datatype type; diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index bddbf027da65..57bda54927e1 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -1279,6 +1279,8 @@ TEST_CASE( rc = tiledb_array_consolidate(ctx, array_name.c_str(), config); CHECK(rc == TILEDB_OK); + tiledb_config_free(&config); + // Load fragment info rc = tiledb_fragment_info_load(ctx, fragment_info); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 6e5b55ae0143..ff4204fe3464 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -2088,3 +2088,52 @@ TEST_CASE( vfs.remove_dir(get_commit_dir(array_read.uri())); vfs.remove_dir(array_read.uri() + "/__schema"); } + +TEST_CASE( + "C++ API: Close array with running query", "[cppapi][close-before-read]") { + const std::string array_name = "cpp_unit_array"; + Context ctx; + VFS vfs(ctx); + + if (vfs.is_dir(array_name)) + vfs.remove_dir(array_name); + + // Create + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "rows", {{0, 3}}, 4)) + .add_dimension(Dimension::create(ctx, "cols", {{0, 3}}, 4)); + ArraySchema schema(ctx, TILEDB_DENSE); + schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + schema.add_attribute(Attribute::create(ctx, "a")); + Array::create(array_name, schema); + + // Write + std::vector data_w = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + Array array_w(ctx, array_name, TILEDB_WRITE); + Query query_w(ctx, array_w); + query_w.set_subarray(Subarray(ctx, array_w).set_subarray({0, 3, 0, 3})) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("a", data_w); + query_w.submit(); + array_w.close(); + + // Open for read. + Array array(ctx, array_name, TILEDB_READ); + std::vector subarray_read = {0, 3, 0, 3}; + std::vector a_read(16); + + Query query(ctx, array); + query.set_subarray(subarray_read) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("a", a_read); + query.submit_async(); + array.close(); + + uint64_t i = 0; + while (query.query_status() != Query::Status::COMPLETE) { + i++; + } + + CHECK(i > 0); +} \ No newline at end of file diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 8306717301a7..409a349fcd1b 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -458,7 +458,7 @@ void DeletesFx::check_delete_conditions( // Load delete conditions. auto&& [st, delete_conditions, update_values] = - sm_->load_delete_and_update_conditions(*array_ptr); + sm_->load_delete_and_update_conditions(*(array_ptr->opened_array())); REQUIRE(st.ok()); REQUIRE(delete_conditions->size() == qcs.size()); diff --git a/test/src/unit-cppapi-subarray.cc b/test/src/unit-cppapi-subarray.cc index d1197c580e74..96455c04ff7c 100644 --- a/test/src/unit-cppapi-subarray.cc +++ b/test/src/unit-cppapi-subarray.cc @@ -415,32 +415,6 @@ TEST_CASE("C++ API: Test subarray (dense)", "[cppapi][dense][subarray]") { // default expected to err. REQUIRE_THROWS(subarray.set_subarray({1, 4, -1, 3})); tiledb::Config config; - // The config option should be ignored but we set anyway. - config.set("sm.read_range_oob", "error"); - subarray.set_config(config); - REQUIRE_THROWS(subarray.set_subarray({1, 4, -1, 3})); - } - - SECTION("- set_subarray Write ranges oob warn - Subarray-query") { - tiledb::Array array(ctx, array_name, TILEDB_WRITE); - tiledb::Query query(ctx, array); - // default (dense) WRITE should always err/throw on oob - REQUIRE_THROWS(query.set_subarray({1, 4, -1, 3})); - tiledb::Config config; - config.set("sm.read_range_oob", "warn"); - query.set_config(config); - // (dense) WRITE should ignore sm.read_range_oob config option - // and err/throw on oob - REQUIRE_THROWS(query.set_subarray({1, 4, -1, 3})); - } - - SECTION("- set_subarray Write ranges oob warn - Subarray-cppapi") { - tiledb::Array array(ctx, array_name, TILEDB_WRITE); - tiledb::Subarray subarray(ctx, array); - REQUIRE_THROWS(subarray.set_subarray({1, 4, -1, 3})); - tiledb::Config config; - config.set("sm.read_range_oob", "warn"); - subarray.set_config(config); REQUIRE_THROWS(subarray.set_subarray({1, 4, -1, 3})); } diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index a91f5576870b..76d9cbbe68e4 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -203,7 +203,7 @@ void UpdatesFx::check_update_conditions( // Load delete conditions. auto&& [st, conditions, update_values] = - sm_->load_delete_and_update_conditions(*array_ptr); + sm_->load_delete_and_update_conditions(*(array_ptr->opened_array())); REQUIRE(st.ok()); REQUIRE(conditions->size() == qcs.size()); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 5f9350ad0f2a..ef6c451c5845 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -84,11 +84,8 @@ Array::Array( const URI& array_uri, StorageManager* storage_manager, ConsistencyController& cc) - : array_schema_latest_(nullptr) - , array_uri_(array_uri) - , array_dir_(storage_manager->resources(), array_uri) + : array_uri_(array_uri) , array_uri_serialized_(array_uri) - , encryption_key_(make_shared(HERE())) , is_open_(false) , is_opening_or_closing_(false) , array_dir_timestamp_start_(0) @@ -99,9 +96,6 @@ Array::Array( , resources_(storage_manager_->resources()) , config_(resources_.config()) , remote_(array_uri.is_tiledb()) - , metadata_() - , metadata_loaded_(false) - , non_empty_domain_computed_(false) , consistency_controller_(cc) , consistency_sentry_(nullopt) { } @@ -110,38 +104,16 @@ Array::Array( /* API */ /* ********************************* */ -void Array::set_array_schema_latest( - const shared_ptr& array_schema) { - array_schema_latest_ = array_schema; -} - -const ArraySchema& Array::array_schema_latest() const { - return *(array_schema_latest_.get()); -} - -shared_ptr Array::array_schema_latest_ptr() const { - return array_schema_latest_; -} - -void Array::set_array_schemas_all( - std::unordered_map>& all_schemas) { - array_schemas_all_ = all_schemas; -} - const URI& Array::array_uri() const { return array_uri_; } -const ArrayDirectory& Array::array_directory() const { - return array_dir_; -} - const URI& Array::array_uri_serialized() const { return array_uri_serialized_; } const EncryptionKey* Array::encryption_key() const { - return encryption_key_.get(); + return opened_array_->encryption_key(); } // Used in Consolidator @@ -162,9 +134,16 @@ Status Array::open_without_fragments( "remote arrays are not supported.")); } - metadata_.clear(); - metadata_loaded_ = false; - non_empty_domain_computed_ = false; + opened_array_ = make_shared( + HERE(), + resources_, + array_uri_, + encryption_type, + encryption_key, + key_length, + timestamp_start(), + timestamp_end_opened_at(), + is_remote()); /* Note: query_type_ MUST be set before calling set_array_open() because it will be examined by the ConsistencyController. */ @@ -175,12 +154,6 @@ Status Array::open_without_fragments( try { set_array_open(query_type_); - // Copy the key bytes. - st = encryption_key_->set_key(encryption_type, encryption_key, key_length); - if (!st.ok()) { - throw StatusException(st); - } - if (remote_) { auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { @@ -195,7 +168,7 @@ Status Array::open_without_fragments( if (!st.ok()) { throw StatusException(st); } - array_schema_latest_ = array_schema_latest.value(); + set_array_schema_latest(array_schema_latest.value()); } else { auto st = rest_client->post_array_from_rest( array_uri_, storage_manager_, this); @@ -207,11 +180,13 @@ Status Array::open_without_fragments( { auto timer_se = resources_.stats().start_timer( "array_open_without_fragments_load_directory"); - array_dir_ = ArrayDirectory( - resources_, array_uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); + set_array_directory(ArrayDirectory( + resources_, array_uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ)); } - std::tie(array_schema_latest_, array_schemas_all_) = + auto&& [array_schema_latest, array_schemas_all] = open_for_reads_without_fragments(); + set_array_schema_latest(array_schema_latest); + set_array_schemas_all(std::move(array_schemas_all)); } } catch (...) { set_array_closed(); @@ -229,14 +204,14 @@ void Array::load_fragments( // Load the fragment metadata std::unordered_map> offsets; - fragment_metadata_ = FragmentMetadata::load( + set_fragment_metadata(FragmentMetadata::load( resources_, memory_tracker(), - array_schema_latest_ptr(), + opened_array_->array_schema_latest_ptr(), array_schemas_all(), *encryption_key(), fragments_to_load, - offsets); + offsets)); } Status Array::open( @@ -269,9 +244,6 @@ Status Array::open( Status_ArrayError("Cannot open array; Array already open.")); } - metadata_.clear(); - metadata_loaded_ = false; - non_empty_domain_computed_ = false; query_type_ = query_type; set_timestamps( @@ -334,11 +306,16 @@ Status Array::open( "Cannot open array; encrypted remote arrays are not supported."); } - // Copy the key bytes. - st = encryption_key_->set_key(encryption_type, encryption_key, key_length); - if (!st.ok()) { - throw StatusException(st); - } + opened_array_ = make_shared( + HERE(), + resources_, + array_uri_, + encryption_type, + encryption_key, + key_length, + this->timestamp_start(), + timestamp_end_opened_at(), + is_remote()); if (remote_) { auto rest_client = resources_.rest_client(); @@ -350,7 +327,7 @@ Status Array::open( auto&& [st, array_schema_latest] = rest_client->get_array_schema_from_rest(array_uri_); throw_if_not_ok(st); - array_schema_latest_ = array_schema_latest.value(); + set_array_schema_latest(array_schema_latest.value()); } else { auto st = rest_client->post_array_from_rest( array_uri_, storage_manager_, this); @@ -362,57 +339,59 @@ Status Array::open( { auto timer_se = resources_.stats().start_timer("array_open_read_load_directory"); - array_dir_ = ArrayDirectory( + set_array_directory(ArrayDirectory( resources_, array_uri_, array_dir_timestamp_start_, - array_dir_timestamp_end_); + array_dir_timestamp_end_)); } - std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = + auto&& [array_schema_latest, array_schemas_all, fragment_metadata] = open_for_reads(); + set_array_schema_latest(array_schema_latest); + set_array_schemas_all(std::move(array_schemas_all)); + set_fragment_metadata(std::move(fragment_metadata)); } else if ( query_type == QueryType::WRITE || query_type == QueryType::MODIFY_EXCLUSIVE) { { auto timer_se = resources_.stats().start_timer("array_open_write_load_directory"); - array_dir_ = ArrayDirectory( + set_array_directory(ArrayDirectory( resources_, array_uri_, array_dir_timestamp_start_, array_dir_timestamp_end_, - ArrayDirectoryMode::SCHEMA_ONLY); + ArrayDirectoryMode::SCHEMA_ONLY)); } auto&& [st, array_schema_latest, array_schemas] = open_for_writes(); throw_if_not_ok(st); // Set schemas - array_schema_latest_ = array_schema_latest.value(); - array_schemas_all_ = array_schemas.value(); + set_array_schema_latest(array_schema_latest.value()); + set_array_schemas_all(std::move(array_schemas.value())); // Set the timestamp - metadata_.reset(timestamp_for_new_component()); - + opened_array_->metadata().reset(timestamp_for_new_component()); } else if ( query_type == QueryType::DELETE || query_type == QueryType::UPDATE) { { auto timer_se = resources_.stats().start_timer( "array_open_delete_or_update_load_directory"); - array_dir_ = ArrayDirectory( + set_array_directory(ArrayDirectory( resources_, array_uri_, array_dir_timestamp_start_, array_dir_timestamp_end_, - ArrayDirectoryMode::READ); + ArrayDirectoryMode::READ)); } - auto&& [st, array_schema_latest, array_schemas] = open_for_writes(); + auto&& [st, latest, array_schemas] = open_for_writes(); throw_if_not_ok(st); // Set schemas - array_schema_latest_ = array_schema_latest.value(); - array_schemas_all_ = array_schemas.value(); + set_array_schema_latest(latest.value()); + set_array_schemas_all(std::move(array_schemas.value())); - auto version = array_schema_latest_->version(); + auto version = array_schema_latest().version(); if (query_type == QueryType::DELETE && version < constants::deletes_min_version) { std::stringstream err; @@ -433,7 +412,7 @@ Status Array::open( } // Updates the timestamp to use for metadata. - metadata_.reset(timestamp_for_new_component()); + opened_array_->metadata().reset(timestamp_for_new_component()); } else { throw ArrayException("Cannot open array; Unsupported query type."); } @@ -455,10 +434,7 @@ Status Array::close() { return Status::Ok(); } - non_empty_domain_.clear(); - non_empty_domain_computed_ = false; clear_last_max_buffer_sizes(); - fragment_metadata_.clear(); try { set_array_closed(); @@ -468,10 +444,10 @@ Status Array::close() { // user if ((query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) && - metadata_.num() > 0) { + opened_array_->metadata().num() > 0) { // Set metadata loaded to be true so when serialization fetchs the // metadata it won't trigger a deadlock - metadata_loaded_ = true; + set_metadata_loaded(true); auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { throw Status_ArrayError( @@ -483,15 +459,11 @@ Status Array::close() { array_dir_timestamp_end_, this)); } - - // Storage manager does not own the array schema for remote arrays. - array_schema_latest_.reset(); } else { - array_schema_latest_.reset(); if (query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { st = storage_manager_->store_metadata( - array_uri_, *encryption_key_.get(), &metadata_); + array_uri_, *encryption_key(), &opened_array_->metadata()); if (!st.ok()) { throw StatusException(st); } @@ -502,9 +474,7 @@ Status Array::close() { } } - array_schemas_all_.clear(); - metadata_.clear(); - metadata_loaded_ = false; + opened_array_.reset(); } catch (std::exception& e) { is_opening_or_closing_ = false; throw Status_ArrayError(e.what()); @@ -587,7 +557,7 @@ std::vector> Array::get_enumerations( // Dedupe requested names and filter out anything already loaded. std::unordered_set enmrs_to_load; for (auto& enmr_name : enumeration_names) { - if (array_schema_latest_->is_enumeration_loaded(enmr_name)) { + if (array_schema_latest().is_enumeration_loaded(enmr_name)) { continue; } enmrs_to_load.insert(enmr_name); @@ -620,25 +590,25 @@ std::vector> Array::get_enumerations( // Create a vector of paths to be loaded. std::vector paths_to_load; for (auto& enmr_name : enmrs_to_load) { - auto path = array_schema_latest_->get_enumeration_path_name(enmr_name); + auto path = array_schema_latest().get_enumeration_path_name(enmr_name); paths_to_load.push_back(path); } // Load the enumerations from storage - loaded = array_dir_.load_enumerations_from_paths( - paths_to_load, get_encryption_key(), memory_tracker_); + loaded = array_directory().load_enumerations_from_paths( + paths_to_load, *encryption_key(), memory_tracker_); } // Store the loaded enumerations in the schema for (auto& enmr : loaded) { - array_schema_latest_->store_enumeration(enmr); + opened_array_->array_schema_latest_ptr()->store_enumeration(enmr); } } // Return the requested list of enumerations std::vector> ret(enumeration_names.size()); for (size_t i = 0; i < enumeration_names.size(); i++) { - ret[i] = array_schema_latest_->get_enumeration(enumeration_names[i]); + ret[i] = array_schema_latest().get_enumeration(enumeration_names[i]); } return ret; } @@ -648,11 +618,11 @@ void Array::load_all_enumerations() { throw ArrayException("Unable to load all enumerations; Array is not open."); } // Load all enumerations, discarding the returned list of loaded enumerations. - get_enumerations(array_schema_latest_->get_enumeration_names()); + get_enumerations(array_schema_latest().get_enumeration_names()); } bool Array::is_empty() const { - return fragment_metadata_.empty(); + return opened_array_->fragment_metadata().empty(); } bool Array::is_open() { @@ -664,10 +634,6 @@ bool Array::is_remote() const { return remote_; } -std::vector> Array::fragment_metadata() const { - return fragment_metadata_; -} - tuple>> Array::get_array_schema() const { // Error if the array is not open @@ -677,7 +643,7 @@ tuple>> Array::get_array_schema() Status_ArrayError("Cannot get array schema; Array is not open")), nullopt}; - return {Status::Ok(), array_schema_latest_}; + return {Status::Ok(), opened_array_->array_schema_latest_ptr()}; } QueryType Array::get_query_type() const { @@ -706,22 +672,22 @@ Status Array::get_max_buffer_size( } // Not applicable to heterogeneous domains - if (!array_schema_latest_->domain().all_dims_same_type()) { + if (!array_schema_latest().domain().all_dims_same_type()) { return LOG_STATUS( Status_ArrayError("Cannot get max buffer size; Function not " "applicable to heterogeneous domains")); } // Not applicable to variable-sized dimensions - if (!array_schema_latest_->domain().all_dims_fixed()) { + if (!array_schema_latest().domain().all_dims_fixed()) { return LOG_STATUS(Status_ArrayError( "Cannot get max buffer size; Function not " "applicable to domains with variable-sized dimensions")); } // Check if name is attribute or dimension - bool is_dim = array_schema_latest_->is_dim(name); - bool is_attr = array_schema_latest_->is_attr(name); + bool is_dim = array_schema_latest().is_dim(name); + bool is_attr = array_schema_latest().is_attr(name); // Check if attribute/dimension exists if (name != constants::coords && !is_dim && !is_attr) { @@ -731,7 +697,7 @@ Status Array::get_max_buffer_size( } // Check if attribute/dimension is fixed sized - if (array_schema_latest_->var_size(name)) { + if (array_schema_latest().var_size(name)) { return LOG_STATUS(Status_ArrayError( std::string("Cannot get max buffer size; Attribute/Dimension '") + name + "' is var-sized")); @@ -772,14 +738,14 @@ Status Array::get_max_buffer_size( } // Not applicable to heterogeneous domains - if (!array_schema_latest_->domain().all_dims_same_type()) { + if (!array_schema_latest().domain().all_dims_same_type()) { return LOG_STATUS( Status_ArrayError("Cannot get max buffer size; Function not " "applicable to heterogeneous domains")); } // Not applicable to variable-sized dimensions - if (!array_schema_latest_->domain().all_dims_fixed()) { + if (!array_schema_latest().domain().all_dims_fixed()) { return LOG_STATUS(Status_ArrayError( "Cannot get max buffer size; Function not " "applicable to domains with variable-sized dimensions")); @@ -796,7 +762,7 @@ Status Array::get_max_buffer_size( } // Check if attribute/dimension is var-sized - if (!array_schema_latest_->var_size(name)) { + if (!array_schema_latest().var_size(name)) { return LOG_STATUS(Status_ArrayError( std::string("Cannot get max buffer size; Attribute/Dimension '") + name + "' is fixed-sized")); @@ -809,10 +775,6 @@ Status Array::get_max_buffer_size( return Status::Ok(); } -const EncryptionKey& Array::get_encryption_key() const { - return *encryption_key_; -} - Status Array::reopen() { // Note: Array will only reopen for reads. This is why we are checking the // timestamp for the array directory and not new components. This needs to be @@ -857,13 +819,17 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { clear_last_max_buffer_sizes(); // Reopen metadata. - fragment_metadata_.clear(); - metadata_.clear(); - metadata_loaded_ = false; - - // Reset the non-empty domain - may be different. - non_empty_domain_.clear(); - non_empty_domain_computed_ = false; + auto key = opened_array_->encryption_key(); + opened_array_ = make_shared( + HERE(), + resources_, + array_uri_, + key->encryption_type(), + key->key().data(), + key->key().size(), + this->timestamp_start(), + timestamp_end_opened_at(), + is_remote()); // Use open to reopen a remote array. if (remote_) { @@ -877,30 +843,33 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { return open( query_type_, - encryption_key_->encryption_type(), - encryption_key_->key().data(), - encryption_key_->key().size()); + encryption_key()->encryption_type(), + encryption_key()->key().data(), + encryption_key()->key().size()); } // Reload the array directory in READ mode (reopen only supports reads). try { { auto timer_se = resources_.stats().start_timer("array_reopen_directory"); - array_dir_ = ArrayDirectory( + set_array_directory(ArrayDirectory( resources_, array_uri_, array_dir_timestamp_start_, array_dir_timestamp_end_, query_type_ == QueryType::READ ? ArrayDirectoryMode::READ : - ArrayDirectoryMode::SCHEMA_ONLY); + ArrayDirectoryMode::SCHEMA_ONLY)); } } catch (const std::logic_error& le) { return LOG_STATUS(Status_ArrayDirectoryError(le.what())); } // Reopen the array and update private variables. - std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata_) = + auto&& [array_schema_latest, array_schemas_all, fragment_metadata] = open_for_reads(); + set_array_schema_latest(array_schema_latest); + set_array_schemas_all(std::move(array_schemas_all)); + set_fragment_metadata(std::move(fragment_metadata)); return Status::Ok(); } @@ -931,7 +900,7 @@ void Array::delete_metadata(const char* key) { throw ArrayException("Cannot delete metadata. Key cannot be null"); } - metadata_.del(key); + opened_array_->metadata().del(key); } void Array::put_metadata( @@ -962,7 +931,7 @@ void Array::put_metadata( throw ArrayException("Cannot put metadata; Value type cannot be ANY"); } - metadata_.put(key, value_type, value_num, value); + opened_array_->metadata().put(key, value_type, value_num, value); } void Array::get_metadata( @@ -987,11 +956,11 @@ void Array::get_metadata( } // Load array metadata, if not loaded yet - if (!metadata_loaded_) { + if (!metadata_loaded()) { throw_if_not_ok(load_metadata()); } - metadata_.get(key, value_type, value_num, value); + opened_array_->metadata().get(key, value_type, value_num, value); } void Array::get_metadata( @@ -1013,11 +982,12 @@ void Array::get_metadata( } // Load array metadata, if not loaded yet - if (!metadata_loaded_) { + if (!metadata_loaded()) { throw_if_not_ok(load_metadata()); } - metadata_.get(index, key, key_len, value_type, value_num, value); + opened_array_->metadata().get( + index, key, key_len, value_type, value_num, value); } uint64_t Array::metadata_num() { @@ -1033,11 +1003,11 @@ uint64_t Array::metadata_num() { } // Load array metadata, if not loaded yet - if (!metadata_loaded_) { + if (!metadata_loaded()) { throw_if_not_ok(load_metadata()); } - return metadata_.num(); + return opened_array_->metadata().num(); } std::optional Array::metadata_type(const char* key) { @@ -1058,43 +1028,35 @@ std::optional Array::metadata_type(const char* key) { } // Load array metadata, if not loaded yet - if (!metadata_loaded_) { + if (!metadata_loaded()) { throw_if_not_ok(load_metadata()); } - return metadata_.metadata_type(key); + return opened_array_->metadata().metadata_type(key); } Metadata* Array::unsafe_metadata() { - return &metadata_; + return &opened_array_->metadata(); } Status Array::metadata(Metadata** metadata) { // Load array metadata for array opened for reads, if not loaded yet - if (query_type_ == QueryType::READ && !metadata_loaded_) { + if (query_type_ == QueryType::READ && !metadata_loaded()) { RETURN_NOT_OK(load_metadata()); } - *metadata = &metadata_; + *metadata = &opened_array_->metadata(); return Status::Ok(); } -NDRange* Array::loaded_non_empty_domain() { - return &non_empty_domain_; -} - -tuple> Array::non_empty_domain() { - if (!non_empty_domain_computed_) { +const NDRange Array::non_empty_domain() { + if (!non_empty_domain_computed()) { // Compute non-empty domain - RETURN_NOT_OK_TUPLE(compute_non_empty_domain(), nullopt); + throw_if_not_ok(compute_non_empty_domain()); } - return {Status::Ok(), non_empty_domain_}; -} - -void Array::set_non_empty_domain(const NDRange& non_empty_domain) { - non_empty_domain_ = non_empty_domain; + return loaded_non_empty_domain(); } MemoryTracker* Array::memory_tracker() { @@ -1194,12 +1156,17 @@ std::unordered_map Array::get_average_var_cell_sizes() const { std::unordered_map ret; + // Keep the current opened array alive for the duration of this call. + auto opened_array = opened_array_; + auto& fragment_metadata = opened_array->fragment_metadata(); + auto& array_schema_latest = opened_array->array_schema_latest(); + // Find the names of the var-sized dimensions or attributes. std::vector var_names; // Start with dimensions. - for (unsigned d = 0; d < array_schema_latest_->dim_num(); d++) { - auto dim = array_schema_latest_->dimension_ptr(d); + for (unsigned d = 0; d < array_schema_latest.dim_num(); d++) { + auto dim = array_schema_latest.dimension_ptr(d); if (dim->var_size()) { var_names.emplace_back(dim->name()); ret.emplace(dim->name(), 0); @@ -1207,7 +1174,7 @@ std::unordered_map Array::get_average_var_cell_sizes() } // Now attributes. - for (auto& attr : array_schema_latest_->attributes()) { + for (auto& attr : array_schema_latest.attributes()) { if (attr->var_size()) { var_names.emplace_back(attr->name()); ret.emplace(attr->name(), 0); @@ -1219,17 +1186,17 @@ std::unordered_map Array::get_average_var_cell_sizes() throw_if_not_ok(parallel_for( &resources_.compute_tp(), 0, - fragment_metadata_.size(), + fragment_metadata.size(), [&](const unsigned f) { // Gracefully skip loading tile sizes for attributes added in schema // evolution that do not exists in this fragment. - const auto& schema = fragment_metadata_[f]->array_schema(); + const auto& schema = fragment_metadata[f]->array_schema(); if (!schema->is_field(var_name)) { return Status::Ok(); } - fragment_metadata_[f]->load_tile_var_sizes( - *encryption_key_, var_name); + fragment_metadata[f]->load_tile_var_sizes( + *encryption_key(), var_name); return Status::Ok(); })); } @@ -1242,18 +1209,18 @@ std::unordered_map Array::get_average_var_cell_sizes() auto& var_name = var_names[n]; // Sum the total tile size and cell num for each fragments. - for (unsigned f = 0; f < fragment_metadata_.size(); f++) { + for (unsigned f = 0; f < fragment_metadata.size(); f++) { // Skip computation for fields that don't exist for a particular // fragment. - const auto& schema = fragment_metadata_[f]->array_schema(); + const auto& schema = fragment_metadata[f]->array_schema(); if (!schema->is_field(var_name)) { continue; } // Go through all tiles. - for (uint64_t t = 0; t < fragment_metadata_[f]->tile_num(); t++) { - total_size += fragment_metadata_[f]->tile_var_size(var_name, t); - cell_num += fragment_metadata_[f]->cell_num(t); + for (uint64_t t = 0; t < fragment_metadata[f]->tile_num(); t++) { + total_size += fragment_metadata[f]->tile_var_size(var_name, t); + cell_num += fragment_metadata[f]->cell_num(t); } } @@ -1294,7 +1261,7 @@ Array::open_for_reads_without_fragments() { "array_open_read_without_fragments_load_schemas"); // Load array schemas - auto result = array_dir_.load_array_schemas(*encryption_key()); + auto result = array_directory().load_array_schemas(*encryption_key()); auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); @@ -1319,7 +1286,7 @@ Array::open_for_writes() { // Load array schemas auto&& [array_schema_latest, array_schemas_all] = - array_dir_.load_array_schemas(*encryption_key_); + array_directory().load_array_schemas(*encryption_key()); // If building experimentally, this library should not be able to // write to newer-versioned or older-versioned arrays @@ -1363,16 +1330,16 @@ void Array::clear_last_max_buffer_sizes() { Status Array::compute_max_buffer_sizes(const void* subarray) { // Applicable only to domains where all dimensions have the same type - if (!array_schema_latest_->domain().all_dims_same_type()) { + if (!array_schema_latest().domain().all_dims_same_type()) { return LOG_STATUS( Status_ArrayError("Cannot compute max buffer sizes; Inapplicable when " "dimension domains have different types")); } // Allocate space for max buffer sizes subarray - auto dim_num = array_schema_latest_->dim_num(); + auto dim_num = array_schema_latest().dim_num(); auto coord_size{ - array_schema_latest_->domain().dimension_ptr(0)->coord_size()}; + array_schema_latest().domain().dimension_ptr(0)->coord_size()}; auto subarray_size = 2 * dim_num * coord_size; last_max_buffer_sizes_subarray_.resize(subarray_size); @@ -1383,7 +1350,7 @@ Status Array::compute_max_buffer_sizes(const void* subarray) { last_max_buffer_sizes_.clear(); // Get all attributes and coordinates - auto attributes = array_schema_latest_->attributes(); + auto attributes = array_schema_latest().attributes(); last_max_buffer_sizes_.clear(); for (const auto& attr : attributes) last_max_buffer_sizes_[attr->name()] = @@ -1392,7 +1359,7 @@ Status Array::compute_max_buffer_sizes(const void* subarray) { std::pair(0, 0); for (unsigned d = 0; d < dim_num; ++d) last_max_buffer_sizes_ - [array_schema_latest_->domain().dimension_ptr(d)->name()] = + [array_schema_latest().domain().dimension_ptr(d)->name()] = std::pair(0, 0); RETURN_NOT_OK(compute_max_buffer_sizes(subarray, &last_max_buffer_sizes_)); @@ -1416,64 +1383,68 @@ Status Array::compute_max_buffer_sizes( } return rest_client->get_array_max_buffer_sizes( - array_uri_, *(array_schema_latest_.get()), subarray, buffer_sizes); + array_uri_, array_schema_latest(), subarray, buffer_sizes); } + // Keep the current opened array alive for the duration of this call. + auto opened_array = opened_array_; + auto& fragment_metadata = opened_array->fragment_metadata(); + auto& array_schema_latest = opened_array->array_schema_latest(); + // Return if there are no metadata - if (fragment_metadata_.empty()) { + if (fragment_metadata.empty()) { return Status::Ok(); } // First we calculate a rough upper bound. Especially for dense // arrays, this will not be accurate, as it accounts only for the // non-empty regions of the subarray. - for (auto& meta : fragment_metadata_) { - meta->add_max_buffer_sizes(*encryption_key_, subarray, buffer_sizes); + for (auto& meta : fragment_metadata) { + meta->add_max_buffer_sizes(*encryption_key(), subarray, buffer_sizes); } // Prepare an NDRange for the subarray - auto dim_num = array_schema_latest_->dim_num(); + auto dim_num = array_schema_latest.dim_num(); NDRange sub(dim_num); auto sub_ptr = (const unsigned char*)subarray; uint64_t offset = 0; for (unsigned d = 0; d < dim_num; ++d) { - auto r_size{2 * array_schema_latest_->dimension_ptr(d)->coord_size()}; + auto r_size{2 * array_schema_latest.dimension_ptr(d)->coord_size()}; sub[d] = Range(&sub_ptr[offset], r_size); offset += r_size; } // Rectify bound for dense arrays - if (array_schema_latest_->dense()) { - auto cell_num = array_schema_latest_->domain().cell_num(sub); + if (array_schema_latest.dense()) { + auto cell_num = array_schema_latest.domain().cell_num(sub); // `cell_num` becomes 0 when `subarray` is huge, leading to a // `uint64_t` overflow. if (cell_num != 0) { for (auto& it : *buffer_sizes) { - if (array_schema_latest_->var_size(it.first)) { + if (array_schema_latest.var_size(it.first)) { it.second.first = cell_num * constants::cell_var_offset_size; it.second.second += - cell_num * datatype_size(array_schema_latest_->type(it.first)); + cell_num * datatype_size(array_schema_latest.type(it.first)); } else { - it.second.first = - cell_num * array_schema_latest_->cell_size(it.first); + it.second.first = cell_num * array_schema_latest.cell_size(it.first); } } } } // Rectify bound for sparse arrays with integer domain, without duplicates - if (!array_schema_latest_->dense() && !array_schema_latest_->allows_dups() && - array_schema_latest_->domain().all_dims_int()) { - auto cell_num = array_schema_latest_->domain().cell_num(sub); + if (!array_schema_latest.dense() && !array_schema_latest.allows_dups() && + array_schema_latest.domain().all_dims_int()) { + auto cell_num = array_schema_latest.domain().cell_num(sub); // `cell_num` becomes 0 when `subarray` is huge, leading to a // `uint64_t` overflow. if (cell_num != 0) { for (auto& it : *buffer_sizes) { - if (!array_schema_latest_->var_size(it.first)) { + if (!array_schema_latest.var_size(it.first)) { // Check for overflow uint64_t new_size = - cell_num * array_schema_latest_->cell_size(it.first); - if (new_size / array_schema_latest_->cell_size((it.first)) != + cell_num * array_schema_latest.cell_size(it.first); + if (new_size / array_schema_latest.cell_size((it.first)) != cell_num) { continue; } @@ -1489,14 +1460,14 @@ Status Array::compute_max_buffer_sizes( } void Array::do_load_metadata() { - if (!array_dir_.loaded()) { + if (!array_directory().loaded()) { throw ArrayException( "Cannot load metadata; array directory is not loaded."); } auto timer_se = resources_.stats().start_timer("sm_load_array_metadata"); // Determine which array metadata to load - const auto& array_metadata_to_load = array_dir_.array_meta_uris(); + const auto& array_metadata_to_load = array_directory().array_meta_uris(); auto metadata_num = array_metadata_to_load.size(); std::vector> metadata_tiles(metadata_num); @@ -1504,7 +1475,8 @@ void Array::do_load_metadata() { parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { const auto& uri = array_metadata_to_load[m].uri_; - auto&& tile = GenericTileIO::load(resources_, uri, 0, *encryption_key_); + auto&& tile = + GenericTileIO::load(resources_, uri, 0, *encryption_key()); metadata_tiles[m] = tdb::make_shared(HERE(), std::move(tile)); return Status::Ok(); @@ -1517,10 +1489,10 @@ void Array::do_load_metadata() { } resources_.stats().add_counter("read_array_meta_size", meta_size); - metadata_ = Metadata::deserialize(metadata_tiles); + opened_array_->metadata() = Metadata::deserialize(metadata_tiles); // Sets the loaded metadata URIs - metadata_.set_loaded_metadata_uris(array_metadata_to_load); + opened_array_->metadata().set_loaded_metadata_uris(array_metadata_to_load); } Status Array::load_metadata() { @@ -1538,7 +1510,7 @@ Status Array::load_metadata() { } else { do_load_metadata(); } - metadata_loaded_ = true; + set_metadata_loaded(true); return Status::Ok(); } @@ -1551,14 +1523,14 @@ Status Array::load_remote_non_empty_domain() { } RETURN_NOT_OK(rest_client->get_array_non_empty_domain( this, array_dir_timestamp_start_, timestamp_end_opened_at())); - non_empty_domain_computed_ = true; + set_non_empty_domain_computed(true); } return Status::Ok(); } -ArrayDirectory& Array::load_array_directory() { - if (array_dir_.loaded()) { - return array_dir_; +const ArrayDirectory& Array::load_array_directory() { + if (array_directory().loaded()) { + return array_directory(); } if (remote_) { @@ -1571,45 +1543,51 @@ ArrayDirectory& Array::load_array_directory() { ArrayDirectoryMode::SCHEMA_ONLY : ArrayDirectoryMode::READ; - array_dir_ = ArrayDirectory( + set_array_directory(ArrayDirectory( resources_, array_uri_, array_dir_timestamp_start_, array_dir_timestamp_end_, - mode); + mode)); - return array_dir_; + return array_directory(); } Status Array::compute_non_empty_domain() { + // Keep the current opened array alive for the duration of this call. + auto opened_array = opened_array_; + auto& fragment_metadata = opened_array->fragment_metadata(); + auto& array_schema_latest = opened_array->array_schema_latest(); + auto& non_empty_domain = opened_array->non_empty_domain(); + if (remote_) { RETURN_NOT_OK(load_remote_non_empty_domain()); - } else if (!fragment_metadata_.empty()) { - const auto& frag0_dom = fragment_metadata_[0]->non_empty_domain(); - non_empty_domain_.assign(frag0_dom.begin(), frag0_dom.end()); + } else if (!fragment_metadata.empty()) { + const auto& frag0_dom = fragment_metadata[0]->non_empty_domain(); + non_empty_domain.assign(frag0_dom.begin(), frag0_dom.end()); - auto metadata_num = fragment_metadata_.size(); + auto metadata_num = fragment_metadata.size(); for (size_t j = 1; j < metadata_num; ++j) { - const auto& meta_dom = fragment_metadata_[j]->non_empty_domain(); + const auto& meta_dom = fragment_metadata[j]->non_empty_domain(); // Validate that this fragment's non-empty domain is set // It should _always_ be set, however we've seen cases where disk // corruption or other out-of-band activity can cause the fragment to be // corrupt for these cases we want to check to prevent any segfaults // later. if (!meta_dom.empty()) { - array_schema_latest_->domain().expand_ndrange( - meta_dom, &non_empty_domain_); + array_schema_latest.domain().expand_ndrange( + meta_dom, &non_empty_domain); } else { // If the fragment's non-empty domain is indeed empty, lets log it so // the user gets a message warning that this fragment might be corrupt // Note: LOG_STATUS only prints if TileDB is built in verbose mode. LOG_STATUS_NO_RETURN_VALUE(Status_ArrayError( "Non empty domain unexpectedly empty for fragment: " + - fragment_metadata_[j]->fragment_uri().to_string())); + fragment_metadata[j]->fragment_uri().to_string())); } } } - non_empty_domain_computed_ = true; + opened_array->non_empty_domain_computed() = true; return Status::Ok(); } @@ -1631,6 +1609,21 @@ void Array::set_array_open(const QueryType& query_type) { is_open_ = true; } +void Array::set_serialized_array_open() { + is_open_ = true; + + opened_array_ = make_shared( + HERE(), + resources_, + array_uri_, + EncryptionType::NO_ENCRYPTION, + nullptr, + 0, + timestamp_start(), + timestamp_end_opened_at(), + array_uri_.is_tiledb()); +} + void Array::set_array_closed() { std::lock_guard lock(mtx_); diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 37a0f5353543..d9f0c8b6f48d 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -58,6 +58,200 @@ class SchemaEvolution; class FragmentMetadata; enum class QueryType : uint8_t; +/** + * Class to store opened array resources. The class is not C.41 compliant as the + * serialization code doesn't allow it. It needs to be refactored. This object + * will not change after the array as been opened though (see note below for the + * consolidation exception), so eventually we can make this object C.41 + * compliant. That object allows queries or subarrays can take a reference to + * the resources which is going to be stored in a shared pointer and guarantee + * that the resources will be kept alive for the whole duration of the query. + * + * For consolidation, the opened array resources will be changed after the array + * is opened. The consolidator does load fragments separately once it figured + * out what it wants to consolidate. This is not an issue as we control the + * consolidator code and that code is built around that functionality. + */ +class OpenedArray { + public: + /* No default constructor. */ + OpenedArray() = delete; + + /** + * Construct a new Opened Array object. + * + * @param resources The context resources to use. + * @param array_uri The URI of the array. + * @param encryption_type Encryption type. + * @param key_bytes Encryption key data. + * @param key_length Encryption key length. + * @param timestamp_start Start timestamp used to load the array directory. + * @param timestamp_end_opened_at Timestamp at which the array was opened. + * @param is_remote Is the array remote? + */ + OpenedArray( + ContextResources& resources, + const URI& array_uri, + EncryptionType encryption_type, + const void* key_bytes, + uint32_t key_length, + uint64_t timestamp_start, + uint64_t timestamp_end_opened_at, + bool is_remote) + : array_dir_(ArrayDirectory(resources, array_uri)) + , array_schema_latest_(nullptr) + , metadata_() + , metadata_loaded_(false) + , non_empty_domain_computed_(false) + , encryption_key_(make_shared(HERE())) + , timestamp_start_(timestamp_start) + , timestamp_end_opened_at_(timestamp_end_opened_at) + , is_remote_(is_remote) { + throw_if_not_ok( + encryption_key_->set_key(encryption_type, key_bytes, key_length)); + } + + /** Returns the array directory object. */ + inline const ArrayDirectory& array_directory() const { + return array_dir_.value(); + } + + /** Sets the array directory. */ + inline void set_array_directory(const ArrayDirectory&& dir) { + array_dir_ = dir; + } + + /** Returns the latest array schema. */ + inline const ArraySchema& array_schema_latest() const { + return *(array_schema_latest_.get()); + } + + /** Returns the latest array schema as a shared pointer. */ + inline shared_ptr array_schema_latest_ptr() const { + return array_schema_latest_; + } + + /** Sets the latest array schema. */ + inline void set_array_schema_latest( + const shared_ptr& array_schema) { + array_schema_latest_ = array_schema; + } + + /** Returns all array schemas. */ + inline const std::unordered_map>& + array_schemas_all() const { + return array_schemas_all_; + } + + /** Sets all array schema. */ + inline void set_array_schemas_all( + std::unordered_map>&& all_schemas) { + array_schemas_all_ = std::move(all_schemas); + } + + /** Gets a reference to the metadata. */ + inline Metadata& metadata() { + return metadata_; + } + + /** Get a reference to the `metadata_loaded_` value. */ + inline bool& metadata_loaded() { + return metadata_loaded_; + } + + /** Get a reference to the `non_empty_domain_computed_` value. */ + inline bool& non_empty_domain_computed() { + return non_empty_domain_computed_; + } + + /** Gets a reference to the non empty domain. */ + inline NDRange& non_empty_domain() { + return non_empty_domain_; + } + + /** Gets a reference to the fragment metadata. */ + inline std::vector>& fragment_metadata() { + return fragment_metadata_; + } + + /** Returns a constant pointer to the encryption key. */ + inline const EncryptionKey* encryption_key() const { + return encryption_key_.get(); + } + + /** + * Returns a reference to the private encryption key. + */ + inline const EncryptionKey& get_encryption_key() const { + return *encryption_key_; + } + + /** Returns the start timestamp used to load the array directory. */ + inline uint64_t timestamp_start() const { + return timestamp_start_; + } + + /** + * Returns the timestamp at which the array was opened. + * + * WARNING: This is a legacy function that is needed to support the current + * API and REST calls. Do not use in new code. + */ + inline uint64_t timestamp_end_opened_at() const { + return timestamp_end_opened_at_; + } + + /** Returns if the array is remote or not. */ + inline bool is_remote() const { + return is_remote_; + } + + private: + /** The array directory object for listing URIs. */ + optional array_dir_; + + /** The latest array schema. */ + shared_ptr array_schema_latest_; + + /** + * A map of all array_schemas_all + */ + std::unordered_map> array_schemas_all_; + + /** The array metadata. */ + Metadata metadata_; + + /** True if the array metadata is loaded. */ + bool metadata_loaded_; + + /** True if the non_empty_domain has been computed */ + bool non_empty_domain_computed_; + + /** The non-empty domain of the array. */ + NDRange non_empty_domain_; + + /** The metadata of the fragments the array was opened with. */ + std::vector> fragment_metadata_; + + /** + * The private encryption key used to encrypt the array. + * + * Note: This is the only place in TileDB where the user's private key + * bytes should be stored. Whenever a key is needed, a pointer to this + * memory region should be passed instead of a copy of the bytes. + */ + shared_ptr encryption_key_; + + /** The start timestamp used to load the array directory. */ + uint64_t timestamp_start_; + + /** The timestamp at which the array was opened. */ + uint64_t timestamp_end_opened_at_; + + /** Is this a remote array? */ + bool is_remote_; +}; + /** * Free function that returns a reference to the ConsistencyController object. */ @@ -103,37 +297,53 @@ class Array { /* API */ /* ********************************* */ + /** Returns the opened array. */ + inline const shared_ptr opened_array() const { + return opened_array_; + }; + /** Returns the array directory object. */ - const ArrayDirectory& array_directory() const; + inline const ArrayDirectory& array_directory() const { + return opened_array_->array_directory(); + } /** Set the array directory. */ - void set_array_directory(const ArrayDirectory& dir) { - array_dir_ = dir; + inline void set_array_directory(const ArrayDirectory&& dir) { + opened_array_->set_array_directory(std::move(dir)); } /** Sets the latest array schema. * @param array_schema The array schema to set. */ - void set_array_schema_latest(const shared_ptr& array_schema); + inline void set_array_schema_latest( + const shared_ptr& array_schema) { + opened_array_->set_array_schema_latest(array_schema); + } /** Returns the latest array schema. */ - const ArraySchema& array_schema_latest() const; + inline const ArraySchema& array_schema_latest() const { + return opened_array_->array_schema_latest(); + } /** Returns the latest array schema as a shared pointer. */ - shared_ptr array_schema_latest_ptr() const; + inline shared_ptr array_schema_latest_ptr() const { + return opened_array_->array_schema_latest_ptr(); + } /** Returns array schemas map. */ inline const std::unordered_map>& array_schemas_all() const { - return array_schemas_all_; + return opened_array_->array_schemas_all(); } /** * Sets all array schemas. * @param all_schemas The array schemas to set. */ - void set_array_schemas_all( - std::unordered_map>& all_schemas); + inline void set_array_schemas_all( + std::unordered_map>&& all_schemas) { + opened_array_->set_array_schemas_all(std::move(all_schemas)); + } /** Returns the array URI. */ const URI& array_uri() const; @@ -242,16 +452,19 @@ class Array { const EncryptionKey* encryption_key() const; /** - * Returns the fragment metadata of the array. If the array is not open, - * an empty vector is returned. + * Accessor to the fragment metadata of the array. */ - std::vector> fragment_metadata() const; + inline std::vector>& fragment_metadata() { + return opened_array_->fragment_metadata(); + } /** - * Accessor to the fragment metadata of the array. + * Sets fragment metadata. + * @param fragment_metadata The fragment metadata. */ - inline std::vector>& fragment_metadata() { - return fragment_metadata_; + inline void set_fragment_metadata( + std::vector>&& fragment_metadata) { + opened_array_->fragment_metadata() = fragment_metadata; } /** @@ -321,7 +534,9 @@ class Array { /** * Returns a reference to the private encryption key. */ - const EncryptionKey& get_encryption_key() const; + inline const EncryptionKey& get_encryption_key() const { + return opened_array_->get_encryption_key(); + } /** * Re-opens the array. This effectively updates the "view" of the array, @@ -531,37 +746,40 @@ class Array { /** Set if array metadata is loaded already for this array or not */ inline void set_metadata_loaded(const bool is_loaded) { - metadata_loaded_ = is_loaded; + opened_array_->metadata_loaded() = is_loaded; } /** Check if array metadata is loaded already for this array or not */ - inline bool& metadata_loaded() { - return metadata_loaded_; + inline bool metadata_loaded() { + return opened_array_->metadata_loaded(); } /** Check if non emtpy domain is loaded already for this array or not */ - inline bool& non_empty_domain_computed() { - return non_empty_domain_computed_; + inline bool non_empty_domain_computed() { + return opened_array_->non_empty_domain_computed(); } - /** Returns the non-empty domain of the opened array. - * If the non_empty_domain has not been computed or loaded - * it will be loaded first - * */ - tuple> non_empty_domain(); - /** - * Retrieves the array metadata object that is already loadad. - * If it's not yet loaded it will be empty. + * Returns the non-empty domain of the opened array. If the non_empty_domain + * has not been computed or loaded it will be loaded first */ - NDRange* loaded_non_empty_domain(); + const NDRange non_empty_domain(); + /** + * Retrieves the array metadata object that is already loaded. If it's not yet + * loaded it will be empty. + */ + inline NDRange& loaded_non_empty_domain() { + return opened_array_->non_empty_domain(); + } /** Returns the non-empty domain of the opened array. */ - void set_non_empty_domain(const NDRange& non_empty_domain); + inline void set_non_empty_domain(const NDRange& non_empty_domain) { + opened_array_->non_empty_domain() = non_empty_domain; + } /** Set if the non_empty_domain is computed already for this array or not */ inline void set_non_empty_domain_computed(const bool is_computed) { - non_empty_domain_computed_ = is_computed; + opened_array_->non_empty_domain_computed() = is_computed; } /** Returns the memory tracker. */ @@ -569,7 +787,8 @@ class Array { /** * Checks the config to see if non empty domain should be serialized on array - * open. */ + * open. + */ bool serialize_non_empty_domain() const; /** @@ -598,9 +817,7 @@ class Array { /** * Sets the array state as open, used in serialization */ - inline void set_serialized_array_open() { - is_open_ = true; - } + void set_serialized_array_open(); /** Set the query type to open the array for. */ inline void set_query_type(QueryType query_type) { @@ -619,27 +836,19 @@ class Array { std::unordered_map get_average_var_cell_sizes() const; /** Load array directory for non-remote arrays */ - ArrayDirectory& load_array_directory(); + const ArrayDirectory& load_array_directory(); private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ - /** The latest array schema. */ - shared_ptr array_schema_latest_; - - /** - * A map of all array_schemas_all - */ - std::unordered_map> array_schemas_all_; + /** The opened array that can be used by queries. */ + shared_ptr opened_array_; /** The array URI. */ URI array_uri_; - /** The array directory object for listing URIs. */ - ArrayDirectory array_dir_; - /** This is a backwards compatible URI from serialization * In TileDB 2.5 we removed sending the URI but 2.4 and older were * unconditionally setting the URI, so things got set to an empty stirng Now @@ -648,18 +857,6 @@ class Array { */ URI array_uri_serialized_; - /** - * The private encryption key used to encrypt the array. - * - * Note: This is the only place in TileDB where the user's private key - * bytes should be stored. Whenever a key is needed, a pointer to this - * memory region should be passed instead of a copy of the bytes. - */ - shared_ptr encryption_key_; - - /** The metadata of the fragments the array was opened with. */ - std::vector> fragment_metadata_; - /** `True` if the array has been opened. */ std::atomic is_open_; @@ -727,18 +924,6 @@ class Array { /** True if the array is remote (has `tiledb://` URI scheme). */ bool remote_; - /** The array metadata. */ - Metadata metadata_; - - /** True if the array metadata is loaded. */ - bool metadata_loaded_; - - /** True if the non_empty_domain has been computed */ - bool non_empty_domain_computed_; - - /** The non-empty domain of the array. */ - NDRange non_empty_domain_; - /** Memory tracker for the array. */ MemoryTracker memory_tracker_; diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 0d17b64af023..115b1ac8c5f5 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -286,7 +286,7 @@ class ArrayDirectory { /* ********************************* */ /** - * Constructor. + * Constructor. Needed for serialization. * * @param resources A reference to a ContextResources instance * @param uri The URI of the array directory diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 6ef9982394d6..94b34c77bf70 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -27,6 +27,7 @@ include(unit_test) commence(unit_test array) + this_target_compile_definitions(-DTILEDB_TEST_INPUTS_DIR="${CMAKE_SOURCE_DIR}/test/inputs/") this_target_sources(main.cc unit_array_directory.cc) this_target_link_libraries(array context_resources) conclude(unit_test) diff --git a/tiledb/sm/array/test/unit_array_directory.cc b/tiledb/sm/array/test/unit_array_directory.cc index 1f227e28a6f6..940ace2fad56 100644 --- a/tiledb/sm/array/test/unit_array_directory.cc +++ b/tiledb/sm/array/test/unit_array_directory.cc @@ -40,6 +40,9 @@ using namespace tiledb::common; using namespace tiledb::sm; +static const std::string arrays_dir = + std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays"; + namespace tiledb::sm { class WhiteboxArrayDirectory { public: @@ -50,12 +53,6 @@ class WhiteboxArrayDirectory { return array_directory.timestamps_overlap( fragment_timestamp_range, consolidation_with_timestamps); } - - void set_open_timestamps( - ArrayDirectory& array_directory, uint64_t start, uint64_t end) const { - array_directory.timestamp_start_ = start; - array_directory.timestamp_end_ = end; - } }; } // namespace tiledb::sm @@ -66,8 +63,12 @@ TEST_CASE( Config cfg; auto logger = make_shared(HERE(), "foo"); ContextResources resources{cfg, logger, 1, 1, ""}; - ArrayDirectory array_dir(resources, URI()); - wb_array_dir.set_open_timestamps(array_dir, 2, 4); + ArrayDirectory array_dir( + resources, + URI(arrays_dir + "/dense_array_v1_3_0"), + 2, + 4, + ArrayDirectoryMode::READ); // Only full overlap should be included for regular fragments. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index afb88d060627..5eef66c81abb 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -495,9 +495,9 @@ int32_t tiledb_array_schema_load( auto storage_manager{ctx->storage_manager()}; // Load URIs from the array directory - tiledb::sm::ArrayDirectory array_dir(storage_manager->resources(), uri); + optional array_dir; try { - array_dir = tiledb::sm::ArrayDirectory( + array_dir.emplace( storage_manager->resources(), uri, 0, @@ -512,7 +512,7 @@ int32_t tiledb_array_schema_load( } // Load latest array schema - auto&& array_schema_latest = array_dir.load_array_schema_latest(key); + auto&& array_schema_latest = array_dir->load_array_schema_latest(key); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -586,9 +586,9 @@ int32_t tiledb_array_schema_load_with_key( auto storage_manager{ctx->storage_manager()}; // Load URIs from the array directory - tiledb::sm::ArrayDirectory array_dir(storage_manager->resources(), uri); + optional array_dir; try { - array_dir = tiledb::sm::ArrayDirectory( + array_dir.emplace( storage_manager->resources(), uri, 0, @@ -603,7 +603,7 @@ int32_t tiledb_array_schema_load_with_key( } // Load latest array schema - auto&& array_schema_latest = array_dir.load_array_schema_latest(key); + auto&& array_schema_latest = array_dir->load_array_schema_latest(key); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -1719,7 +1719,8 @@ int32_t tiledb_subarray_set_config( if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; api::ensure_config_is_valid(config); - subarray->subarray_->set_config(config->config()); + subarray->subarray_->set_config( + tiledb::sm::QueryType::READ, config->config()); return TILEDB_OK; } @@ -2929,9 +2930,9 @@ int32_t tiledb_array_encryption_type( auto uri = tiledb::sm::URI(array_uri); // Load URIs from the array directory - tiledb::sm::ArrayDirectory array_dir(storage_manager->resources(), uri); + optional array_dir; try { - array_dir = tiledb::sm::ArrayDirectory( + array_dir.emplace( storage_manager->resources(), uri, 0, @@ -2947,7 +2948,7 @@ int32_t tiledb_array_encryption_type( // Get encryption type tiledb::sm::EncryptionType enc; throw_if_not_ok( - ctx->storage_manager()->array_get_encryption(array_dir, &enc)); + ctx->storage_manager()->array_get_encryption(array_dir.value(), &enc)); *encryption_type = static_cast(enc); diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 9c9c39accc8f..1677120d0835 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -111,6 +111,14 @@ Status ArrayMetaConsolidator::consolidate( // Get the new URI name for consolidated metadata URI new_uri = metadata_w->get_uri(array_uri); + // Write vac files relative to the array URI. This was fixed for reads in + // version 19 so only do this for arrays starting with version 19. + size_t base_uri_size = 0; + if (array_for_reads.array_schema_latest_ptr() == nullptr || + array_for_reads.array_schema_latest().write_version() >= 19) { + base_uri_size = array_for_reads.array_uri().to_string().size(); + } + // Close arrays RETURN_NOT_OK_ELSE( array_for_reads.close(), throw_if_not_ok(array_for_writes.close())); @@ -119,15 +127,6 @@ Status ArrayMetaConsolidator::consolidate( // Write vacuum file URI vac_uri = URI(new_uri.to_string() + constants::vacuum_file_suffix); - size_t base_uri_size = 0; - - // Write vac files relative to the array URI. This was fixed for reads in - // version 19 so only do this for arrays starting with version 19. - if (array_for_reads.array_schema_latest_ptr() == nullptr || - array_for_reads.array_schema_latest().write_version() >= 19) { - base_uri_size = array_for_reads.array_uri().to_string().size(); - } - std::stringstream ss; for (const auto& uri : to_vacuum) { ss << uri.to_string().substr(base_uri_size) << "\n"; diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 0b9736930809..32bcd7199b64 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -84,6 +84,7 @@ Query::Query( optional fragment_name) : array_shared_(array) , array_(array_shared_.get()) + , opened_array_(array->opened_array()) , array_schema_(array->array_schema_latest_ptr()) , type_(array_->get_query_type()) , layout_( @@ -129,7 +130,7 @@ Query::Query( config_ = storage_manager->config(); // Set initial subarray configuration - subarray_.set_config(config_); + subarray_.set_config(type_, config_); rest_scratch_ = make_shared(HERE()); } @@ -983,7 +984,7 @@ void Query::set_config(const Config& config) { // Set subarray's config for backwards compatibility // Users expect the query config to effect the subarray based on existing // behavior before subarray was exposed directly - subarray_.set_config(config_); + subarray_.set_config(type_, config_); } Status Query::set_coords_buffer(void* buffer, uint64_t* buffer_size) { @@ -1791,7 +1792,7 @@ bool Query::is_aggregate(std::string output_field_name) const { Status Query::create_strategy(bool skip_checks_serialization) { auto params = StrategyParams( storage_manager_, - array_, + opened_array_, config_, buffers_, aggregate_buffers_, @@ -1799,7 +1800,8 @@ Status Query::create_strategy(bool skip_checks_serialization) { layout_, condition_, default_channel_aggregates_, - skip_checks_serialization); + skip_checks_serialization, + array_->memory_tracker()); if (type_ == QueryType::WRITE || type_ == QueryType::MODIFY_EXCLUSIVE) { if (layout_ == Layout::COL_MAJOR || layout_ == Layout::ROW_MAJOR) { if (!array_schema_->dense()) { diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 54759f9d8c70..c5c8ecba2dba 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -784,10 +784,15 @@ class Query { * Ensures that the Array object exists as long as the Query object exists. */ shared_ptr array_shared_; - /** The array the query is associated with. - * Cached copy of array_shared_.get(). */ + /** + * The array the query is associated with. + * Cached copy of array_shared_.get(). + */ Array* array_; + /** Keeps a copy of the opened array object at construction. */ + shared_ptr opened_array_; + /** The array schema. */ shared_ptr array_schema_; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 7cd916fe336d..df18d001bd92 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -74,7 +74,7 @@ DenseReader::DenseReader( StrategyParams& params, bool remote_query) : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) - , array_memory_tracker_(params.array()->memory_tracker()) { + , array_memory_tracker_(params.memory_tracker()) { elements_mode_ = false; // Sanity checks. diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 615e8ecbdb1c..b47e6450359b 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -73,7 +73,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( , tmp_read_state_(array_->fragment_metadata().size()) , memory_budget_(config_, reader_string) , include_coords_(include_coords) - , array_memory_tracker_(params.array()->memory_tracker()) + , array_memory_tracker_(params.memory_tracker()) , memory_used_for_coords_total_(0) , deletes_consolidation_no_purge_( buffers_.count(constants::delete_timestamps) != 0) diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index fd2cce5a277e..91bc71946403 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -43,16 +43,17 @@ namespace tiledb { namespace sm { -class Array; +class OpenedArray; class ArraySchema; class IAggregator; enum class Layout : uint8_t; +class MemoryTracker; class Subarray; class QueryBuffer; class QueryCondition; -typedef std::unordered_map> - DefaultChannelAggregates; +using DefaultChannelAggregates = + std::unordered_map>; /** * Class used to pass in common parameters to strategies. This will make it @@ -66,7 +67,7 @@ class StrategyParams { StrategyParams( StorageManager* storage_manager, - Array* array, + shared_ptr array, Config& config, std::unordered_map& buffers, std::unordered_map& aggregate_buffers, @@ -74,7 +75,8 @@ class StrategyParams { Layout layout, std::optional& condition, DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization) + bool skip_checks_serialization, + MemoryTracker* memory_tracker) : storage_manager_(storage_manager) , array_(array) , config_(config) @@ -84,7 +86,8 @@ class StrategyParams { , layout_(layout) , condition_(condition) , default_channel_aggregates_(default_channel_aggregates) - , skip_checks_serialization_(skip_checks_serialization) { + , skip_checks_serialization_(skip_checks_serialization) + , memory_tracker_(memory_tracker) { } /* ********************************* */ @@ -97,7 +100,7 @@ class StrategyParams { }; /** Return the array. */ - inline Array* array() { + inline shared_ptr array() { return array_; }; @@ -141,6 +144,10 @@ class StrategyParams { return skip_checks_serialization_; } + inline MemoryTracker* memory_tracker() { + return memory_tracker_; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -150,7 +157,7 @@ class StrategyParams { StorageManager* storage_manager_; /** Array. */ - Array* array_; + shared_ptr array_; /** Config for query-level parameters only. */ Config& config_; @@ -175,6 +182,9 @@ class StrategyParams { /** Skip checks for serialization. */ bool skip_checks_serialization_; + + /** Memory tracker. */ + MemoryTracker* memory_tracker_; }; /** Processes read or write queries. */ @@ -227,8 +237,11 @@ class StrategyBase { /** The class logger. */ shared_ptr logger_; - /** The array. */ - const Array* array_; + /** + * A shared pointer to the opened array which ensures that the query can + * still access it even after the array is closed. + */ + shared_ptr array_; /** The array schema. */ const ArraySchema& array_schema_; diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index fad14c9f2c0c..7792c1ca5b6d 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -246,9 +246,6 @@ Status array_from_capnp( QueryType query_type = QueryType::READ; RETURN_NOT_OK(query_type_enum(query_type_str, &query_type)); array->set_query_type(query_type); - if (!array->is_open()) { - array->set_serialized_array_open(); - } array->set_timestamps( array_reader.getStartTimestamp(), @@ -261,6 +258,10 @@ Status array_from_capnp( false); }; + if (!array->is_open()) { + array->set_serialized_array_open(); + } + if (array_reader.hasArraySchemasAll()) { std::unordered_map> all_schemas; auto all_schemas_reader = array_reader.getArraySchemasAll(); @@ -275,7 +276,7 @@ Status array_from_capnp( make_shared(HERE(), schema); } } - array->set_array_schemas_all(all_schemas); + array->set_array_schemas_all(std::move(all_schemas)); } if (array_reader.hasArraySchemaLatest()) { @@ -294,13 +295,17 @@ Status array_from_capnp( array_directory_reader, storage_manager->resources(), array->array_uri()); - array->set_array_directory(*array_dir); + array->set_array_directory(std::move(*array_dir)); + } else { + array->set_array_directory( + ArrayDirectory(storage_manager->resources(), array->array_uri())); } if (array_reader.hasFragmentMetadataAll()) { - array->fragment_metadata().clear(); + auto fragment_metadata = array->fragment_metadata(); + fragment_metadata.clear(); auto fragment_metadata_all_reader = array_reader.getFragmentMetadataAll(); - array->fragment_metadata().reserve(fragment_metadata_all_reader.size()); + fragment_metadata.reserve(fragment_metadata_all_reader.size()); for (auto frag_meta_reader : fragment_metadata_all_reader) { auto meta = make_shared(HERE()); RETURN_NOT_OK(fragment_metadata_from_capnp( @@ -312,8 +317,9 @@ Status array_from_capnp( if (client_side) { meta->set_rtree_loaded(); } - array->fragment_metadata().emplace_back(meta); + fragment_metadata.emplace_back(meta); } + array->set_fragment_metadata(std::move(fragment_metadata)); } if (array_reader.hasNonEmptyDomain()) { diff --git a/tiledb/sm/serialization/capnp_utils.h b/tiledb/sm/serialization/capnp_utils.h index 4794c81858e1..6162f96483c0 100644 --- a/tiledb/sm/serialization/capnp_utils.h +++ b/tiledb/sm/serialization/capnp_utils.h @@ -484,14 +484,10 @@ Status serialize_non_empty_domain_rv( */ template Status serialize_non_empty_domain(CapnpT& builder, tiledb::sm::Array* array) { - auto&& [st, nonEmptyDomain_opt] = array->non_empty_domain(); - RETURN_NOT_OK(st); - if (nonEmptyDomain_opt.has_value()) { - return serialize_non_empty_domain_rv( - builder, - nonEmptyDomain_opt.value(), - array->array_schema_latest().dim_num()); - } + return serialize_non_empty_domain_rv( + builder, + array->non_empty_domain(), + array->array_schema_latest().dim_num()); return Status::Ok(); } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ae86f78475aa..ab174b12674e 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -773,12 +773,8 @@ Status StorageManagerCanonical::array_get_non_empty_domain( return logger_->status(Status_StorageManagerError( "Cannot get non-empty domain; Array not opened for reads")); - auto&& [st, domain_opt] = array->non_empty_domain(); - RETURN_NOT_OK(st); - if (domain_opt.has_value()) { - *domain = domain_opt.value(); - *is_empty = domain->empty(); - } + *domain = array->non_empty_domain(); + *is_empty = domain->empty(); return Status::Ok(); } @@ -1259,8 +1255,10 @@ tuple< Status, optional>, optional>>> -StorageManagerCanonical::load_delete_and_update_conditions(const Array& array) { - auto& locations = array.array_directory().delete_and_update_tiles_location(); +StorageManagerCanonical::load_delete_and_update_conditions( + const OpenedArray& opened_array) { + auto& locations = + opened_array.array_directory().delete_and_update_tiles_location(); auto conditions = std::vector(locations.size()); auto update_values = std::vector>(locations.size()); @@ -1270,7 +1268,10 @@ StorageManagerCanonical::load_delete_and_update_conditions(const Array& array) { // Read the condition from storage. auto&& tile = GenericTileIO::load( - resources_, uri, locations[i].offset(), *array.encryption_key()); + resources_, + uri, + locations[i].offset(), + *(opened_array.encryption_key())); if (tiledb::sm::utils::parse::ends_with( locations[i].condition_marker(), @@ -1290,7 +1291,7 @@ StorageManagerCanonical::load_delete_and_update_conditions(const Array& array) { throw Status_StorageManagerError("Unknown condition marker extension"); } - throw_if_not_ok(conditions[i].check(array.array_schema_latest())); + throw_if_not_ok(conditions[i].check(opened_array.array_schema_latest())); return Status::Ok(); }); RETURN_NOT_OK_TUPLE(status, nullopt, nullopt); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 903157f833de..78fef0fcb559 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -66,6 +66,7 @@ namespace tiledb { namespace sm { class Array; +class OpenedArray; class ArrayDirectory; class ArraySchema; class ArraySchemaEvolution; @@ -597,14 +598,14 @@ class StorageManagerCanonical { /** * Loads the delete and update conditions from storage. * - * @param array The array. + * @param opened_array The opened array. * @return Status, vector of the conditions, vector of the update values. */ tuple< Status, optional>, optional>>> - load_delete_and_update_conditions(const Array& array); + load_delete_and_update_conditions(const OpenedArray& opened_array); /** Removes a TileDB object (group, array). */ Status object_remove(const char* path) const; diff --git a/tiledb/sm/subarray/relevant_fragment_generator.cc b/tiledb/sm/subarray/relevant_fragment_generator.cc index ad4fe4bb0080..3ff313b67451 100644 --- a/tiledb/sm/subarray/relevant_fragment_generator.cc +++ b/tiledb/sm/subarray/relevant_fragment_generator.cc @@ -46,12 +46,14 @@ namespace tiledb { namespace sm { RelevantFragmentGenerator::RelevantFragmentGenerator( - const Array& array, const Subarray& subarray, stats::Stats* stats) + const shared_ptr opened_array, + const Subarray& subarray, + stats::Stats* stats) : stats_(stats) - , array_(array) + , array_(opened_array) , subarray_(subarray) { - auto dim_num = array_.array_schema_latest().dim_num(); - auto fragment_num = array_.fragment_metadata().size(); + auto dim_num = array_->array_schema_latest().dim_num(); + auto fragment_num = array_->fragment_metadata().size(); // Create a fragment bytemap for each dimension. Each // non-zero byte represents an overlap between a fragment @@ -94,8 +96,8 @@ bool RelevantFragmentGenerator::update_range_coords( RelevantFragments RelevantFragmentGenerator::compute_relevant_fragments( ThreadPool* const compute_tp) { auto timer_se = stats_->start_timer("compute_relevant_frags"); - auto dim_num = array_.array_schema_latest().dim_num(); - auto fragment_num = array_.fragment_metadata().size(); + auto dim_num = array_->array_schema_latest().dim_num(); + auto fragment_num = array_->fragment_metadata().size(); // Populate the fragment bytemap for each dimension in parallel. throw_if_not_ok(parallel_for(compute_tp, 0, dim_num, [&](const uint32_t d) { @@ -122,8 +124,8 @@ Status RelevantFragmentGenerator::compute_relevant_fragments_for_dim( const std::vector& start_coords, const std::vector& end_coords, std::vector* const frag_bytemap) const { - const auto meta = array_.fragment_metadata(); - auto dim{array_.array_schema_latest().dimension_ptr(dim_idx)}; + const auto meta = array_->fragment_metadata(); + auto dim{array_->array_schema_latest().dimension_ptr(dim_idx)}; return parallel_for(compute_tp, 0, fragment_num, [&](const uint64_t f) { // We're done when we have already determined fragment `f` to diff --git a/tiledb/sm/subarray/relevant_fragment_generator.h b/tiledb/sm/subarray/relevant_fragment_generator.h index f8c0651b3f58..362c7038c632 100644 --- a/tiledb/sm/subarray/relevant_fragment_generator.h +++ b/tiledb/sm/subarray/relevant_fragment_generator.h @@ -43,7 +43,7 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -class Array; +class OpenedArray; class RelevantFragments; class Subarray; class SubarrayTileOverlap; @@ -72,7 +72,9 @@ class RelevantFragmentGenerator { /** Constructor a generator. */ RelevantFragmentGenerator( - const Array& array, const Subarray& subarray, stats::Stats* stats); + const shared_ptr opened_array, + const Subarray& subarray, + stats::Stats* stats); /* ********************************* */ /* API */ @@ -144,8 +146,8 @@ class RelevantFragmentGenerator { /** The class stats. */ stats::Stats* stats_; - /** Reference to the opened array. */ - const Array& array_; + /** Reference to the opened opened array. */ + const shared_ptr array_; /** Reference to the subarray. */ const Subarray& subarray_; diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 6638786cd763..4f4d6e62f5e5 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -99,7 +99,7 @@ Subarray::Subarray( const bool coalesce_ranges, StorageManager* storage_manager) : Subarray( - array, + array->opened_array(), Layout::UNORDERED, parent_stats, logger, @@ -114,17 +114,33 @@ Subarray::Subarray( shared_ptr logger, const bool coalesce_ranges, StorageManager* storage_manager) + : Subarray( + array->opened_array(), + layout, + parent_stats, + logger, + coalesce_ranges, + storage_manager) { +} + +Subarray::Subarray( + const shared_ptr opened_array, + const Layout layout, + Stats* const parent_stats, + shared_ptr logger, + const bool coalesce_ranges, + StorageManager* storage_manager) : stats_( parent_stats ? parent_stats->create_child("Subarray") : storage_manager ? storage_manager->stats()->create_child("subSubarray") : nullptr) , logger_(logger->clone("Subarray", ++logger_id_)) - , array_(array) + , array_(opened_array) , layout_(layout) - , cell_order_(array_->array_schema_latest().cell_order()) + , cell_order_(opened_array->array_schema_latest().cell_order()) , est_result_size_computed_(false) - , relevant_fragments_(array->fragment_metadata().size()) + , relevant_fragments_(opened_array->fragment_metadata().size()) , coalesce_ranges_(coalesce_ranges) , ranges_sorted_(false) { if (!parent_stats && !storage_manager) { @@ -734,7 +750,7 @@ Status Subarray::get_range_var_from_name( return get_range_var(dim_idx, range_idx, start, end); } -const Array* Subarray::array() const { +const shared_ptr Subarray::array() const { return array_; } @@ -1097,12 +1113,9 @@ void Subarray::set_layout(Layout layout) { layout_ = layout; } -void Subarray::set_config(const Config& config) { +void Subarray::set_config(const QueryType query_type, const Config& config) { config_.inherit(config); - - QueryType array_query_type{array_->get_query_type()}; - - if (array_query_type == QueryType::READ) { + if (query_type == QueryType::READ) { bool found = false; std::string read_range_oob_str = config.get("sm.read_range_oob", &found); assert(found); @@ -2586,7 +2599,7 @@ Status Subarray::precompute_tile_overlap( // each successive loop. The intent is to minimize the number of loops // at the risk of exceeding our target maximum memory usage for the // tile overlap data. - RelevantFragmentGenerator relevant_fragment_generator(*array_, *this, stats_); + RelevantFragmentGenerator relevant_fragment_generator(array_, *this, stats_); ComputeRelevantTileOverlapCtx tile_overlap_ctx; SubarrayTileOverlap tile_overlap( fragment_num, tile_overlap_start, tmp_tile_overlap_end); @@ -2642,7 +2655,7 @@ Status Subarray::precompute_all_ranges_tile_overlap( // Compute relevant fragments and load rtrees. ComputeRelevantTileOverlapCtx tile_overlap_ctx; - RelevantFragmentGenerator relevant_fragment_generator(*array_, *this, stats_); + RelevantFragmentGenerator relevant_fragment_generator(array_, *this, stats_); relevant_fragment_generator.update_range_coords(nullptr); relevant_fragments_ = relevant_fragment_generator.compute_relevant_fragments(compute_tp); diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index afa2a03c562b..4b2fa5563f09 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -69,6 +69,7 @@ namespace sm { class Array; class ArraySchema; +class OpenedArray; class DimensionLabel; class EncryptionKey; class FragIdx; @@ -247,6 +248,26 @@ class Subarray { bool coalesce_ranges = true, StorageManager* storage_manager = nullptr); + /** + * Constructor. + * + * @param opened_array The opened array the subarray is associated with. + * @param layout The layout of the values of the subarray (of the results + * if the subarray is used for reads, or of the values provided + * by the user for writes). + * @param parent_stats The parent stats to inherit from. + * @param logger The parent logger to clone and use for logging + * @param coalesce_ranges When enabled, ranges will attempt to coalesce + * with existing ranges as they are added. + */ + Subarray( + const shared_ptr opened_array, + Layout layout, + stats::Stats* parent_stats, + shared_ptr logger, + bool coalesce_ranges = true, + StorageManager* storage_manager = nullptr); + /** * Copy constructor. This performs a deep copy (including memcpy of * underlying buffers). @@ -273,7 +294,7 @@ class Subarray { /* ********************************* */ /** Sets config for query-level parameters only. */ - void set_config(const Config& config); + void set_config(const QueryType query_type, const Config& config); /** * Get the config of the writer @@ -593,8 +614,8 @@ class Subarray { void* start, void* end) const; - /** Returns the array the subarray is associated with. */ - const Array* array() const; + /** Returns the opened array the subarray is associated with. */ + const shared_ptr array() const; /** * Returns the number of cells in the subarray. @@ -1388,7 +1409,7 @@ class Subarray { shared_ptr logger_; /** The array the subarray object is associated with. */ - const Array* array_; + shared_ptr array_; /** Stores the estimated result size for each array attribute/dimension. */ std::unordered_map est_result_size_; diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 9253ceb8cc70..c8168cc65ed7 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -71,12 +71,9 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { make_shared(HERE()); CHECK(sp_as->set_domain(sp_dom).ok()); CHECK(sp_as->add_attribute(sp_attrib).ok()); - // sp_as->add_dimension(); tiledb::sm::Config cfg; tiledb::sm::Context ctx(cfg); tiledb::sm::Array a(tiledb::sm::URI{"mem://junk"}, ctx.storage_manager()); - a.set_array_schema_latest(sp_as); - // a.create(); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); CHECK(ctx.storage_manager()->array_create(a.array_uri(), sp_as, ek).ok()); From 1579cfc85246c84ccd1c7495ee8a772806b2cc1d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 12 Jan 2024 22:50:28 +0100 Subject: [PATCH 137/456] Update back compat arrays after delta filter addition. (#4623) This switches the tests to the new versions of the backwards compatibility arrays for 2.16, 2.17 and 2.18 that include the delta filter. Older branches will still use the old arrays... Once we get a few releases done, we should remove the old arrays. --- TYPE: NO_HISTORY DESC: Update back compat arrays after delta filter addition. --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- tiledb/sm/array/array.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 114bfd4a084e..8693873d8436 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_0", "v2_13_0", "v2_14_0", "v2_15_0", "v2_16_0", "v2_17_3", "v2_18_1"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index ef6c451c5845..58e99219032d 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -826,7 +826,7 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { array_uri_, key->encryption_type(), key->key().data(), - key->key().size(), + static_cast(key->key().size()), this->timestamp_start(), timestamp_end_opened_at(), is_remote()); From 7902e8b16c78aaec38f0a67dd79d9baba9fcfe09 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Fri, 12 Jan 2024 22:22:40 -0600 Subject: [PATCH 138/456] Don't reallocate buffers in fragment consolidation (#4614) Avoid needless buffer reallocations when consolidating fragments. --- TYPE: IMPROVEMENT DESC: Avoid needless buffer reallocations when consolidating fragments. --- test/regression/CMakeLists.txt | 1 + test/regression/targets/sc-36372.cc | 191 +++++++++++++++ .../sm/consolidator/fragment_consolidator.cc | 218 ++++++++++-------- .../sm/consolidator/fragment_consolidator.h | 78 +++---- .../test/unit_fragment_consolidator.cc | 39 +--- 5 files changed, 350 insertions(+), 177 deletions(-) create mode 100644 test/regression/targets/sc-36372.cc diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 7545cda9b1b8..a14dd3198b48 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -48,6 +48,7 @@ if (TILEDB_CPP_API) list(APPEND SOURCES targets/sc-29682.cc) list(APPEND SOURCES targets/sc-33480.cc) list(APPEND SOURCES targets/sc-35424.cc) + list(APPEND SOURCES targets/sc-36372.cc) list(APPEND SOURCES targets/sc-38300.cc) endif() diff --git a/test/regression/targets/sc-36372.cc b/test/regression/targets/sc-36372.cc new file mode 100644 index 000000000000..ebbc7589b8ae --- /dev/null +++ b/test/regression/targets/sc-36372.cc @@ -0,0 +1,191 @@ +#include +#include + +#include + +#include + +TEST_CASE( + "C++ API: Consolidation slowness in create_buffer with large number of " + "attributes" + "[cppapi][consolidation][sc36372]") { + std::string array_name = "cpp_unit_array_36372"; + + tiledb::Config cfg; + cfg["sm.consolidation.step_min_frags"] = 2; + cfg["sm.consolidation.step_max_frags"] = 4; + tiledb::Context ctx(cfg); + tiledb::VFS vfs(ctx); + + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + tiledb::Domain domain(ctx); + auto domain_lo = std::numeric_limits::min(); + auto domain_hi = std::numeric_limits::max() - 1; + + // Create and initialize dimension. + auto d0 = tiledb::Dimension::create( + ctx, "d0", {{domain_lo, domain_hi}}, 2); + auto d1 = tiledb::Dimension::create( + ctx, "d1", {{domain_lo, domain_hi}}, 4); + auto d2 = tiledb::Dimension::create( + ctx, "d2", {{domain_lo, domain_hi}}, 50); + auto d3 = tiledb::Dimension::create( + ctx, "d3", {{domain_lo, domain_hi}}, 200); + auto d4 = tiledb::Dimension::create( + ctx, "d4", {{domain_lo, domain_hi}}, 2); + auto d5 = tiledb::Dimension::create( + ctx, "d5", {{domain_lo, domain_hi}}, 2); + + domain.add_dimensions(d0, d1, d2, d3, d4, d5); + + auto a0 = tiledb::Attribute::create(ctx, "a0").set_cell_val_num( + TILEDB_VAR_NUM); + auto a1 = tiledb::Attribute::create(ctx, "a1").set_cell_val_num( + TILEDB_VAR_NUM); + auto a2 = tiledb::Attribute::create(ctx, "a2").set_cell_val_num( + TILEDB_VAR_NUM); + auto a3 = tiledb::Attribute::create(ctx, "a3").set_cell_val_num( + TILEDB_VAR_NUM); + auto a4 = tiledb::Attribute::create(ctx, "a4").set_cell_val_num( + TILEDB_VAR_NUM); + auto a5 = tiledb::Attribute::create(ctx, "a5").set_cell_val_num( + TILEDB_VAR_NUM); + auto a6 = tiledb::Attribute::create(ctx, "a6").set_cell_val_num( + TILEDB_VAR_NUM); + auto a7 = tiledb::Attribute::create(ctx, "a7").set_cell_val_num( + TILEDB_VAR_NUM); + auto a8 = tiledb::Attribute::create(ctx, "a8").set_cell_val_num( + TILEDB_VAR_NUM); + auto a9 = tiledb::Attribute::create(ctx, "a9").set_cell_val_num( + TILEDB_VAR_NUM); + auto a10 = tiledb::Attribute::create(ctx, "a10") + .set_cell_val_num(TILEDB_VAR_NUM); + auto a11 = tiledb::Attribute::create(ctx, "a11") + .set_cell_val_num(TILEDB_VAR_NUM); + auto a12 = tiledb::Attribute::create(ctx, "a12") + .set_cell_val_num(TILEDB_VAR_NUM); + auto a13 = tiledb::Attribute::create(ctx, "a13") + .set_cell_val_num(TILEDB_VAR_NUM); + auto a14 = tiledb::Attribute::create(ctx, "a14") + .set_cell_val_num(TILEDB_VAR_NUM); + + tiledb::ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.add_attributes( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14); + schema.set_capacity(10000000); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_ROW_MAJOR); + CHECK_NOTHROW(tiledb::Array::create(array_name, schema)); + + // Perform Write + std::vector d0_data(1, 0); + std::vector d1_data(1, 0); + std::vector d2_data(1, 0); + std::vector d3_data(1, 0); + std::vector d4_data(1, 0); + std::vector d5_data(1, 0); + std::vector a0_data(1, 0); + std::vector a0_offsets(1, 0); + std::vector a1_data(1, 0); + std::vector a1_offsets(1, 0); + std::vector a2_data(1, 0); + std::vector a2_offsets(1, 0); + std::vector a3_data(1, 0); + std::vector a3_offsets(1, 0); + std::vector a4_data(1, 0); + std::vector a4_offsets(1, 0); + std::vector a5_data(1, 0); + std::vector a5_offsets(1, 0); + std::vector a6_data(1, 0); + std::vector a6_offsets(1, 0); + std::vector a7_data(1, 0); + std::vector a7_offsets(1, 0); + std::vector a8_data(1, 0); + std::vector a8_offsets(1, 0); + std::vector a9_data(1, 0); + std::vector a9_offsets(1, 0); + std::vector a10_data(1, 0); + std::vector a10_offsets(1, 0); + std::vector a11_data(1, 0); + std::vector a11_offsets(1, 0); + std::vector a12_data(1, 0); + std::vector a12_offsets(1, 0); + std::vector a13_data(1, 0); + std::vector a13_offsets(1, 0); + std::vector a14_data(1, 0); + std::vector a14_offsets(1, 0); + + uint8_t fragments_to_create = 196; + tiledb::Array array(ctx, array_name, TILEDB_WRITE); + for (uint8_t i = 0; i < fragments_to_create; i++) { + d0_data[0] = i; + d1_data[0] = i; + d2_data[0] = i; + d3_data[0] = i; + d4_data[0] = i; + d5_data[0] = i; + a0_data[0] = i; + a1_data[0] = i; + a2_data[0] = i; + a3_data[0] = i; + a4_data[0] = i; + a5_data[0] = i; + a6_data[0] = i; + a7_data[0] = i; + a8_data[0] = i; + a9_data[0] = i; + a10_data[0] = i; + a11_data[0] = i; + a12_data[0] = i; + a13_data[0] = i; + a14_data[0] = i; + + tiledb::Query query(ctx, array); + query.set_data_buffer("d0", d0_data); + query.set_data_buffer("d1", d1_data); + query.set_data_buffer("d2", d2_data); + query.set_data_buffer("d3", d3_data); + query.set_data_buffer("d4", d4_data); + query.set_data_buffer("d5", d5_data); + + query.set_data_buffer("a0", a0_data).set_offsets_buffer("a0", a0_offsets); + query.set_data_buffer("a1", a1_data).set_offsets_buffer("a1", a1_offsets); + query.set_data_buffer("a2", a2_data).set_offsets_buffer("a2", a2_offsets); + query.set_data_buffer("a3", a3_data).set_offsets_buffer("a3", a3_offsets); + query.set_data_buffer("a4", a4_data).set_offsets_buffer("a4", a4_offsets); + query.set_data_buffer("a5", a5_data).set_offsets_buffer("a5", a5_offsets); + query.set_data_buffer("a6", a6_data).set_offsets_buffer("a6", a6_offsets); + query.set_data_buffer("a7", a7_data).set_offsets_buffer("a7", a7_offsets); + query.set_data_buffer("a8", a8_data).set_offsets_buffer("a8", a8_offsets); + query.set_data_buffer("a9", a9_data).set_offsets_buffer("a9", a9_offsets); + query.set_data_buffer("a10", a10_data) + .set_offsets_buffer("a10", a10_offsets); + query.set_data_buffer("a11", a11_data) + .set_offsets_buffer("a11", a11_offsets); + query.set_data_buffer("a12", a12_data) + .set_offsets_buffer("a12", a12_offsets); + query.set_data_buffer("a13", a13_data) + .set_offsets_buffer("a13", a13_offsets); + query.set_data_buffer("a14", a14_data) + .set_offsets_buffer("a14", a14_offsets); + + query.submit(); + } + + // Consolidate + tiledb::Stats::enable(); + tiledb::Array::consolidate(ctx, array_name); + tiledb::Stats::dump(); + + // Vacuum + tiledb::Array::vacuum(ctx, array_name); + + // Cleanup. + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } +} diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 946889e670de..eede94d2390b 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -58,6 +58,111 @@ class FragmentConsolidatorException : public StatusException { } }; +void FragmentConsolidationWorkspace::resize_buffers( + stats::Stats* stats, + const FragmentConsolidationConfig& config, + const ArraySchema& array_schema, + std::unordered_map& avg_cell_sizes) { + auto timer_se = stats->start_timer("resize_buffers"); + + // For easy reference + auto attribute_num = array_schema.attribute_num(); + auto& domain{array_schema.domain()}; + auto dim_num = array_schema.dim_num(); + auto sparse = !array_schema.dense(); + + // Calculate buffer weights. We reserve the maximum possible number of buffers + // to make only one allocation. If an attribute is var size and nullable, it + // has 3 buffers, dimensions only have 2 as they cannot be nullable. Then one + // buffer for timestamps, and 2 for delete metadata. + std::vector buffer_weights; + buffer_weights.reserve(attribute_num * 3 + dim_num * 2 + 3); + for (unsigned i = 0; i < attribute_num; ++i) { + const auto attr = array_schema.attributes()[i]; + const auto var_size = attr->var_size(); + + // First buffer is either the var size offsets or the fixed size data. + buffer_weights.emplace_back( + var_size ? constants::cell_var_offset_size : attr->cell_size()); + + // For var size attributes, add the data buffer weight. + if (var_size) { + buffer_weights.emplace_back(avg_cell_sizes[attr->name()]); + } + + // For nullable attributes, add the validity buffer weight. + if (attr->nullable()) { + buffer_weights.emplace_back(constants::cell_validity_size); + } + } + + // Add weights for sparse dimensions. + if (sparse) { + for (unsigned i = 0; i < dim_num; ++i) { + const auto dim = domain.dimension_ptr(i); + const auto var_size = dim->var_size(); + + // First buffer is either the var size offsets or the fixed size data. + buffer_weights.emplace_back( + var_size ? constants::cell_var_offset_size : dim->coord_size()); + + // For var size attributes, add the data buffer weight. + if (var_size) { + buffer_weights.emplace_back(avg_cell_sizes[dim->name()]); + } + } + } + + // Add weights for timestamp attribute. + if (config.with_timestamps_ && sparse) { + buffer_weights.emplace_back(constants::timestamp_size); + } + + // Adding buffers for delete meta, one for timestamp and one for condition + // index. + if (config.with_delete_meta_) { + buffer_weights.emplace_back(constants::timestamp_size); + buffer_weights.emplace_back(sizeof(uint64_t)); + } + + // Use the old buffer size setting to see how much memory we would use. + auto buffer_num = buffer_weights.size(); + uint64_t total_budget = config.total_buffer_size_; + + // If a user set the per-attribute buffer size configuration, we override + // the use of the total_budget_size config setting for backwards + // compatible behavior. + if (config.buffer_size_ != 0) { + total_budget = config.buffer_size_ * buffer_num; + } + + // Make sure the buffers and sizes vectors are large enough. + if (buffer_num > buffers_.size()) { + buffers_.resize(buffer_num); + sizes_.resize(buffer_num); + } + + // Create buffers. + auto total_weights = std::accumulate( + buffer_weights.begin(), buffer_weights.end(), static_cast(0)); + + // Allocate space for each buffer. + uint64_t adjusted_budget = total_budget / total_weights * total_weights; + + if (std::max(1, adjusted_budget) > backing_buffer_.size()) { + backing_buffer_.resize(std::max(1, adjusted_budget)); + } + + // Finally allocate spans for the buffers. + size_t offset = 0; + for (unsigned i = 0; i < buffer_num; ++i) { + sizes_[i] = std::max( + 1, adjusted_budget * buffer_weights[i] / total_weights); + buffers_[i] = span(&(backing_buffer_[offset]), sizes_[i]); + offset += sizes_[i]; + } +} + /* ****************************** */ /* CONSTRUCTOR */ /* ****************************** */ @@ -122,6 +227,8 @@ Status FragmentConsolidator::consolidate( return st; } + FragmentConsolidationWorkspace cw; + uint32_t step = 0; std::vector to_consolidate; do { @@ -154,7 +261,8 @@ Status FragmentConsolidator::consolidate( array_for_writes, to_consolidate, union_non_empty_domains, - &new_fragment_uri); + &new_fragment_uri, + cw); if (!st.ok()) { throw_if_not_ok(array_for_reads->close()); throw_if_not_ok(array_for_writes->close()); @@ -260,6 +368,8 @@ Status FragmentConsolidator::consolidate_fragments( "Cannot consolidate; Not all fragments could be found")); } + FragmentConsolidationWorkspace cw; + // Consolidate the selected fragments URI new_fragment_uri; st = consolidate_internal( @@ -267,7 +377,8 @@ Status FragmentConsolidator::consolidate_fragments( array_for_writes, to_consolidate, union_non_empty_domains, - &new_fragment_uri); + &new_fragment_uri, + cw); if (!st.ok()) { throw_if_not_ok(array_for_reads->close()); throw_if_not_ok(array_for_writes->close()); @@ -376,7 +487,8 @@ Status FragmentConsolidator::consolidate_internal( shared_ptr array_for_writes, const std::vector& to_consolidate, const NDRange& union_non_empty_domains, - URI* new_fragment_uri) { + URI* new_fragment_uri, + FragmentConsolidationWorkspace& cw) { auto timer_se = stats_->start_timer("consolidate_internal"); array_for_reads->load_fragments(to_consolidate); @@ -419,8 +531,7 @@ Status FragmentConsolidator::consolidate_internal( // Prepare buffers auto average_var_cell_sizes = array_for_reads->get_average_var_cell_sizes(); - FragmentConsolidationWorkspace cw{ - create_buffers(stats_, config_, array_schema, average_var_cell_sizes)}; + cw.resize_buffers(stats_, config_, array_schema, average_var_cell_sizes); // Create queries auto query_r = (Query*)nullptr; @@ -528,101 +639,6 @@ void FragmentConsolidator::copy_array( } while (query_r->status() == QueryStatus::INCOMPLETE); } -FragmentConsolidationWorkspace FragmentConsolidator::create_buffers( - stats::Stats* stats, - const FragmentConsolidationConfig& config, - const ArraySchema& array_schema, - std::unordered_map& avg_cell_sizes) { - auto timer_se = stats->start_timer("consolidate_create_buffers"); - - // For easy reference - auto attribute_num = array_schema.attribute_num(); - auto& domain{array_schema.domain()}; - auto dim_num = array_schema.dim_num(); - auto sparse = !array_schema.dense(); - - // Calculate buffer weights. We reserve the maximum possible number of buffers - // to make only one allocation. If an attribute is var size and nullable, it - // has 3 buffers, dimensions only have 2 as they cannot be nullable. Then one - // buffer for timestamps, and 2 for delete metadata. - std::vector buffer_weights; - buffer_weights.reserve(attribute_num * 3 + dim_num * 2 + 3); - for (unsigned i = 0; i < attribute_num; ++i) { - const auto attr = array_schema.attributes()[i]; - const auto var_size = attr->var_size(); - - // First buffer is either the var size offsets or the fixed size data. - buffer_weights.emplace_back( - var_size ? constants::cell_var_offset_size : attr->cell_size()); - - // For var size attributes, add the data buffer weight. - if (var_size) { - buffer_weights.emplace_back(avg_cell_sizes[attr->name()]); - } - - // For nullable attributes, add the validity buffer weight. - if (attr->nullable()) { - buffer_weights.emplace_back(constants::cell_validity_size); - } - } - - if (sparse) { - for (unsigned i = 0; i < dim_num; ++i) { - const auto dim = domain.dimension_ptr(i); - const auto var_size = dim->var_size(); - - // First buffer is either the var size offsets or the fixed size data. - buffer_weights.emplace_back( - var_size ? constants::cell_var_offset_size : dim->coord_size()); - - // For var size attributes, add the data buffer weight. - if (var_size) { - buffer_weights.emplace_back(avg_cell_sizes[dim->name()]); - } - } - } - - if (config.with_timestamps_ && sparse) { - buffer_weights.emplace_back(constants::timestamp_size); - } - - // Adding buffers for delete meta, one for timestamp and one for condition - // index. - if (config.with_delete_meta_) { - buffer_weights.emplace_back(constants::timestamp_size); - buffer_weights.emplace_back(sizeof(uint64_t)); - } - - // Use the old buffer size setting to see how much memory we would use. - auto buffer_num = buffer_weights.size(); - uint64_t total_budget = config.total_buffer_size_; - - // If a user set the per-attribute buffer size configuration, we override - // the use of the total_budget_size config setting for backwards - // compatible behavior. - if (config.buffer_size_ != 0) { - total_budget = config.buffer_size_ * buffer_num; - } - - // Create buffers. - FragmentConsolidationWorkspace cw{buffer_num}; - std::vector& buffers{cw.buffers()}; - std::vector& buffer_sizes{cw.sizes()}; - auto total_weights = std::accumulate( - buffer_weights.begin(), buffer_weights.end(), static_cast(0)); - - // Allocate space for each buffer. - uint64_t adjusted_budget = total_budget / total_weights * total_weights; - for (unsigned i = 0; i < buffer_num; ++i) { - buffer_sizes[i] = std::max( - 1, adjusted_budget * buffer_weights[i] / total_weights); - buffers[i].resize(buffer_sizes[i]); - } - - // Success - return cw; -} - Status FragmentConsolidator::create_queries( shared_ptr array_for_reads, shared_ptr array_for_writes, @@ -808,7 +824,7 @@ Status FragmentConsolidator::compute_next_to_consolidate( void FragmentConsolidator::set_query_buffers( Query* query, FragmentConsolidationWorkspace& cw) const { - std::vector* buffers{&cw.buffers()}; + std::vector>* buffers{&cw.buffers()}; std::vector* buffer_sizes{&cw.sizes()}; const auto& array_schema = query->array_schema(); diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 1087e71c9f7e..0f6c0fc041d6 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -107,42 +107,35 @@ struct FragmentConsolidationConfig : Consolidator::ConsolidationConfigBase { * Consolidation workspace holds the large buffers used by the operation. */ class FragmentConsolidationWorkspace { - std::vector buffers_; - std::vector sizes_; - public: - FragmentConsolidationWorkspace() = delete; - explicit FragmentConsolidationWorkspace(size_t n) - : buffers_(n) - , sizes_(n) { - } - /** - * Copy constructor is deleted to avoid copying large amounts of memory - */ - FragmentConsolidationWorkspace(const FragmentConsolidationWorkspace&) = - delete; + FragmentConsolidationWorkspace() = default; - /** - * Move constructor - */ - FragmentConsolidationWorkspace(FragmentConsolidationWorkspace&&) = default; + // Disable copy and move construction/assignment so we don't have + // to think about it. + DISABLE_COPY_AND_COPY_ASSIGN(FragmentConsolidationWorkspace); + DISABLE_MOVE_AND_MOVE_ASSIGN(FragmentConsolidationWorkspace); /** - * Copy assignment is deleted to avoid copying large amounts of memory - */ - const FragmentConsolidationWorkspace& operator=( - const FragmentConsolidationWorkspace&) = delete; - - /** - * Move assignment is deleted for convenience + * Resize the buffers that will be used upon reading the input fragments and + * writing into the new fragment. It also retrieves the number of buffers + * created. + * + * @param stats The stats. + * @param config The consolidation config. + * @param array_schema The array schema. + * @param avg_cell_sizes The average cell sizes. + * @return a consolidation workspace containing the buffers */ - const FragmentConsolidationWorkspace& operator=( - FragmentConsolidationWorkspace&&) = delete; + void resize_buffers( + stats::Stats* stats, + const FragmentConsolidationConfig& config, + const ArraySchema& array_schema, + std::unordered_map& avg_cell_sizes); /** * Accessor for buffers */ - std::vector& buffers() { + std::vector>& buffers() { return buffers_; } @@ -152,6 +145,16 @@ class FragmentConsolidationWorkspace { std::vector& sizes() { return sizes_; }; + + private: + /*** The backing buffer used for all buffers. */ + std::vector backing_buffer_; + + /*** Spans that point to non-overlapping sections of the buffer. */ + std::vector> buffers_; + + /*** The size of each span. */ + std::vector sizes_; }; /** Handles fragment consolidation. */ @@ -274,6 +277,7 @@ class FragmentConsolidator : public Consolidator { * fragments are *not* all sparse. * @param new_fragment_uri The URI of the fragment created after * consolidating the `to_consolidate` fragments. + * @param cw A workspace containing buffers for the queries * @return Status */ Status consolidate_internal( @@ -281,7 +285,8 @@ class FragmentConsolidator : public Consolidator { shared_ptr array_for_writes, const std::vector& to_consolidate, const NDRange& union_non_empty_domains, - URI* new_fragment_uri); + URI* new_fragment_uri, + FragmentConsolidationWorkspace& cw); /** * Copies the array by reading from the fragments to be consolidated @@ -295,23 +300,6 @@ class FragmentConsolidator : public Consolidator { void copy_array( Query* query_r, Query* query_w, FragmentConsolidationWorkspace& cw); - /** - * Creates the buffers that will be used upon reading the input fragments and - * writing into the new fragment. It also retrieves the number of buffers - * created. - * - * @param stats The stats. - * @param config The consolidation config. - * @param array_schema The array schema. - * @param avg_cell_sizes The average cell sizes. - * @return a consolidation workspace containing the buffers - */ - static FragmentConsolidationWorkspace create_buffers( - stats::Stats* stats, - const FragmentConsolidationConfig& config, - const ArraySchema& array_schema, - std::unordered_map& avg_cell_sizes); - /** * Creates the queries needed for consolidation. It also retrieves * the number of fragments to be consolidated and the URI of the diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index 14f8f9edf75c..3e2f9d9e2fb1 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -40,28 +40,6 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; -namespace tiledb::sm { -class WhiteboxFragmentConsolidator { - public: - static FragmentConsolidationWorkspace create_buffers( - stats::Stats* stats, - const bool with_timestamps, - const bool with_delete_meta, - const uint64_t buffer_size, - const ArraySchema& schema, - std::unordered_map& avg_cell_sizes) { - // Create config. - FragmentConsolidationConfig cfg; - cfg.with_timestamps_ = with_timestamps; - cfg.with_delete_meta_ = with_delete_meta; - cfg.buffer_size_ = buffer_size; - - return FragmentConsolidator::create_buffers( - stats, cfg, schema, avg_cell_sizes); - } -}; -} // namespace tiledb::sm - shared_ptr make_schema( const bool sparse, const std::vector dim_types, @@ -232,15 +210,14 @@ TEST_CASE( } // Create buffers. - FragmentConsolidationWorkspace cw{ - WhiteboxFragmentConsolidator::create_buffers( - &statistics, - with_timestamps, - with_delete_meta, - 1000, - *schema, - avg_cell_sizes)}; - std::vector& buffers{cw.buffers()}; + FragmentConsolidationConfig cfg; + cfg.with_timestamps_ = with_timestamps; + cfg.with_delete_meta_ = with_delete_meta; + cfg.buffer_size_ = 1000; + + FragmentConsolidationWorkspace cw; + cw.resize_buffers(&statistics, cfg, *schema, avg_cell_sizes); + std::vector>& buffers{cw.buffers()}; std::vector& buffer_sizes{cw.sizes()}; // Validate. From 588f143625693c50fbd370b93e028d1e4c63a492 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Sat, 13 Jan 2024 06:23:06 +0200 Subject: [PATCH 139/456] Fix recursively dumping a group with a nonexistent subgroup. (#4582) [SC-38499](https://app.shortcut.com/tiledb-inc/story/38499/recursively-dumping-a-group-fails-when-a-subgroup-does-not-exist) --- TYPE: BUG DESC: Fix recursively dumping a group with a nonexistent subgroup. --- test/src/unit-capi-group.cc | 48 +++++++++++++++++++++++++++++++++++++ tiledb/sm/group/group.cc | 22 +++++++++++++---- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 16023cec0fd5..802faa304970 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -1490,6 +1490,54 @@ TEST_CASE_METHOD( remove_temp_dir(temp_dir); } +TEST_CASE_METHOD(GroupFx, "C API: Group, dump", "[capi][group][dump]") { + // Create and open group in write mode + // TODO: refactor for each supported FS. + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + + tiledb::sm::URI group1_uri(temp_dir + "group1"); + REQUIRE(tiledb_group_create(ctx_, group1_uri.c_str()) == TILEDB_OK); + + tiledb::sm::URI group2_uri(temp_dir + "group2"); + REQUIRE(tiledb_group_create(ctx_, group2_uri.c_str()) == TILEDB_OK); + + tiledb_group_t* group1; + int rc = tiledb_group_alloc(ctx_, group1_uri.c_str(), &group1); + REQUIRE(rc == TILEDB_OK); + set_group_timestamp(group1, 1); + rc = tiledb_group_open(ctx_, group1, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_group_add_member(ctx_, group1, group2_uri.c_str(), 0, "group2"); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_group_close(ctx_, group1); + REQUIRE(rc == TILEDB_OK); + + // Delete the subgroup to test that recursively dumping its parent will not + // fail. + rc = tiledb_object_remove(ctx_, group2_uri.c_str()); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_group_open(ctx_, group1, TILEDB_READ); + REQUIRE(rc == TILEDB_OK); + + char* group_dump; + rc = tiledb_group_dump_str(ctx_, group1, &group_dump, 1); + REQUIRE(rc == TILEDB_OK); + REQUIRE(group_dump != nullptr); + REQUIRE( + std::string(group_dump) == + "group1 GROUP\n|-- group2 GROUP (does not exist)\n"); + free(group_dump); + + rc = tiledb_group_close(ctx_, group1); + REQUIRE(rc == TILEDB_OK); + + tiledb_group_free(&group1); + remove_temp_dir(temp_dir); +} + #ifdef TILEDB_SERIALIZATION TEST_CASE_METHOD( GroupFx, diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 508a9be2f4c8..c6965f8c41a1 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -669,13 +669,27 @@ std::string Group::dump( for (const auto& member_entry : members()) { const auto& it = member_entry.second; - ss << "|" << indent << l_indent << " " << *it << std::endl; + URI member_uri = it->uri(); + if (it->relative()) { + member_uri = group_uri_.join_path(it->uri().to_string()); + } + + ss << "|" << indent << l_indent << " " << *it; + bool do_recurse = false; if (it->type() == ObjectType::GROUP && recursive) { - URI member_uri = it->uri(); - if (it->relative()) { - member_uri = group_uri_.join_path(it->uri().to_string()); + // Before listing the group's members, check if the group exists in + // storage. If it does not, leave a message in the string. + ObjectType member_type = ObjectType::INVALID; + throw_if_not_ok(storage_manager_->object_type(member_uri, &member_type)); + if (member_type == ObjectType::GROUP) { + do_recurse = true; + } else { + ss << " (does not exist)"; } + } + ss << std::endl; + if (do_recurse) { Group group_rec(member_uri, storage_manager_); throw_if_not_ok(group_rec.open(QueryType::READ)); ss << group_rec.dump(indent_size, num_indents + 2, recursive, false); From d017dafa4f845fad3e792290c228f409b5755146 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Sat, 13 Jan 2024 06:25:16 +0200 Subject: [PATCH 140/456] Add a document with build instructions. (#4581) [SC-34997](https://app.shortcut.com/tiledb-inc/story/34997/update-our-public-build-instructions-with-vcpkg-era-instructions) --- TYPE: FEATURE DESC: Add a document with build instructions. --- bootstrap.ps1 | 13 ++++- doc/dev/BUILD.md | 128 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 doc/dev/BUILD.md diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 9bb1ee23158a..e9a7e7c3895e 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -24,6 +24,9 @@ Semicolon separated list to binary dependencies. Specify the linkage type to build TileDB with. Valid values are "static" and "shared". Default is "shared". +.PARAMETER RemoveDeprecations +Build TileDB without any deprecated APIs. + .PARAMETER CMakeGenerator Optionally specify the CMake generator string, e.g. "Visual Studio 15 2017". Check 'cmake --help' for a list of supported generators. @@ -124,6 +127,7 @@ Param( [string]$VcpkgBaseTriplet, [string]$Dependency, [string]$Linkage = "shared", + [switch]$RemoveDeprecations, [string]$CMakeGenerator, [switch]$EnableAssert, [switch]$EnableDebug, @@ -199,6 +203,13 @@ if ($EnableVerbose.IsPresent) { $Verbosity = "ON" } +if ($RemoveDeprecations.IsPresent) { + $_RemoveDeprecations = "ON" +} +else { + $_RemoveDeprecations = "OFF" +} + # Set vcpkg flag $UseVcpkg = "ON" if ($EnableVcpkg.IsPresent) { @@ -337,7 +348,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/doc/dev/BUILD.md b/doc/dev/BUILD.md new file mode 100644 index 000000000000..4f0e0bc898c0 --- /dev/null +++ b/doc/dev/BUILD.md @@ -0,0 +1,128 @@ +--- +title: Building TileDB from source +--- + +## Prerequisites + +* CMake 3.21 or greater +* C++ compiler with C++20 support + * The minimum compiler versions TileDB is being tested in CI are: + * MSVC from Visual Studio 2019 16.11 + * GCC 10.3 on Linux and Windows via MinGW + * Apple Clang 14 +* Git (required by vcpkg) +* curl (required by vcpkg on non-Windows) + +## Downloading the source code + +Begin by downloading a release tarball or by cloning the TileDB GitHub repo and checking out a release tag. In the following script `` is the version you wish to use (e.g., `2.18.0`) + +``` +git clone https://github.com/TileDB-Inc/TileDB.git +cd TileDB +git checkout +``` + +## Configuring the build tree + +### Configuring with the bootstrap scripts + +TileDB provides bootstrap scripts to allow easier configuration of a build. The bootstrap script on macOS and Linux is located in [bootstrap](../../bootstrap), while on Windows it is located in [bootstrap.ps1](../../bootstrap.ps1) and runs on [PowerShell](https://learn.microsoft.com/en-us/powershell/). + +You can use the bootstrap scripts like this: + +**macOS and Linux** + +```bash +mkdir build +cd build +# You can see a list of all available options by running ../bootstrap --help +../bootstrap +``` + +**Windows** + +```powershell +mkdir build +cd build +# You can see a list of all available options by running Get-Help ..\bootstrap.ps1 -Detailed +..\bootstrap.ps1 +``` + +### Configuring with CMake + +Alternatively, you can configure the build by directly running CMake. The following example works in all operating systems: + +```bash +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja -DTILEDB_S3=ON +``` + +### Configuration options + +The following are the most common configuration options: + +|macOS/Linux flag|Windows flag|CMake variable|Description| +|----------------|------------|--------------|-----------| +|`--prefix=PREFIX`|`-Prefix=PREFIX`|`CMAKE_INSTALL_PREFIX=`|Install files in tree rooted at `PREFIX` (defaults to `TileDB/dist`)| +|`--linkage=shared/static`|`-Linkage=shared/static`|`BUILD_SHARED_LIBS=ON/OFF`|Linkage of the compiled TileDB library (defaults to `shared`) | +|`--remove-deprecations`|`-RemoveDeprecations`|`TILEDB_REMOVE_DEPRECATIONS=ON`|Build TileDB without deprecated APIs| +|`--enable-debug`|`-EnableDebug`|`CMAKE_BUILD_TYPE=Debug`|Enables debug build| +|`--enable-release-symbols`|`-EnableReleaseSymbols`|`CMAKE_BUILD_TYPE=RelWithDebInfo`|Enables release build with debug symbols| +|`--enable-coverage`|`-EnableCoverage`|`CMAKE_BUILD_TYPE=Coverage`|Enables build with code coverage support| +|`--enable-verbose`|`-EnableVerbose`|`TILEDB_VERBOSE=ON`|Enables verbose status messages| +|`--enable-hdfs`|_(HDFS is unsupported on Windows)_|`TILEDB_HDFS=ON`|Enables building with HDFS storage backend support| +|`--enable-s3`|`-EnableS3`|`TILEDB_S3=ON`|Enables building with S3 storage backend support| +|`--enable-azure`|`-EnableAzure`|`TILEDB_AZURE=ON`|Enables building with Azure Blob Storage backend support| +|`--enable-gcs`|`-EnableGcs`|`TILEDB_GCS=ON`|Enables building with Google Cloud Storage backend support| +|`--enable-serialization`|`-EnableSerialization`|`TILEDB_SERIALIZATION=ON`|Enables building with Serialization and TileDB Cloud support| +|`--disable-avx2`|_(Unavailable)_|`COMPILER_SUPPORTS_AVX2=FALSE`|Disables use of the AVX2 instruction set| +|`--disable-webp`|`-DisableWebp`|`TILEDB_WEBP=OFF`|Disables building with support for the WebP filter| +|`--disable-werror`|`-DisableWerror`|`TILEDB_WERROR=OFF`|Disables building with the `-Werror` flag| +|`--disable-cpp-api`|`-DisableCppApi`|`TILEDB_CPP_API=OFF`|Disables building the TileDB C++ API| +|`--disable-stats`|`-DisableStats`|`TILEDB_STATS=OFF`|Disables internal TileDB statistics| +|`--disable-tests`|`-DisableTests`|`TILEDB_TESTS=OFF`|Disables building the TileDB test suite| + +> [!TIP] +> You can see all TileDB-specific CMake variables in [BuildOptions.cmake](../../cmake/Options/BuildOptions.cmake). + +## Building and installing + +Once the build tree has been configured, you can build and install TileDB by running the following commands from the build tree: + +```bash +cmake --build . -j +cmake --build . --target install-tiledb +``` + +If you are building with a multi-config generator (e.g., Visual Studio), you will have to specify the configuration to build with the `--config` option. Also you can directly invoke the underlying build tool: + +```bash +make -j4 +make install-tiledb +``` + +The following are the most important targets: + +|Target|Description| +|------|-----------| +|`install-tiledb`|Installs the TileDB library and headers.| +|`check`|Builds and runs all TileDB tests.| +|`examples`|Builds all TileDB examples.| + +## Advanced + +### Dependency management with vcpkg + +TileDB uses [vcpkg](https://vcpkg.io) to download and build its dependencies. By default, the vcpkg repository will be cloned automatically when configuring the build tree. To avoid repeatedly cloning the repository, and take full advantage of vcpkg's [binary caching](https://learn.microsoft.com/en-us/vcpkg/users/binarycaching) feature, you are recommended to clone the vcpkg repository manually and set the `VCPKG_ROOT` environment variable on your machine to the path of the repository. + +> [!NOTE] +> If you have set `VCPKG_ROOT` on your machine and encountered errors like `error: no version database entry for at ` it means that your vcpkg repository is out of date. You can update it by running `git pull` in the repository directory. + +Vcpkg will not be automatically downloaded if: + +* The `TILEDB_DISABLE_AUTO_VCPKG` environment variable has been defined. +* The build tree has been configured by directly calling CMake and the `CMAKE_TOOLCHAIN_FILE` variable has been set by the user. + +In these cases no dependencies CMake will find the dependencies based on the rules of the [`find_package`](https://cmake.org/cmake/help/latest/command/find_package.html#command:find_package) command. The user is responsible for providing them. From cfd6eb22554d27ee8426ad2387b37219e271ef58 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Sat, 13 Jan 2024 05:00:42 -0500 Subject: [PATCH 141/456] Add experimental CPP API for ls_recursive. (#4625) This adds the experimental CPP APIs for ls_recursive, which is currently only supported over S3. --- TYPE: FEATURE DESC: Add experimental CPP API for ls_recursive. --- test/src/unit-cppapi-vfs.cc | 186 +++++++++++++++++++ tiledb/api/c_api/vfs/test/unit_capi_vfs.cc | 16 +- tiledb/api/c_api/vfs/vfs_api_internal.h | 2 +- tiledb/sm/cpp_api/vfs_experimental.h | 205 +++++++++++++++++++++ tiledb/sm/filesystem/ls_scanner.h | 47 ++++- 5 files changed, 443 insertions(+), 13 deletions(-) create mode 100644 tiledb/sm/cpp_api/vfs_experimental.h diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index e4f4d40437f5..52bf25ac41f2 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -32,7 +32,9 @@ #include #include +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/vfs_experimental.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" @@ -500,3 +502,187 @@ TEST_CASE( } } } + +TEST_CASE("CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]") { + using namespace tiledb::test; + S3Test s3_test({10, 100, 0}); + if (!s3_test.is_supported()) { + return; + } + auto expected_results = s3_test.expected_results(); + + vfs_config cfg; + tiledb::Context ctx(tiledb::Config(&cfg.config)); + tiledb::VFS vfs(ctx); + + tiledb::VFSExperimental::LsObjects ls_objects; + // Predicate filter to apply to ls_recursive. + tiledb::VFSExperimental::LsInclude include; + // Callback to populate ls_objects vector using a filter. + tiledb::VFSExperimental::LsCallback cb = [&](std::string_view path, + uint64_t size) { + if (include(path, size)) { + ls_objects.emplace_back(path, size); + } + return true; + }; + + SECTION("Default filter (include all)") { + include = [](std::string_view, uint64_t) { return true; }; + } + SECTION("Custom filter (include none)") { + include = [](std::string_view, uint64_t) { return false; }; + } + + bool include_result = true; + SECTION("Custom filter (include half)") { + include = [&include_result](std::string_view, uint64_t) { + include_result = !include_result; + return include_result; + }; + } + + SECTION("Custom filter (search for test_file_50)") { + include = [](std::string_view object_name, uint64_t) { + return object_name.find("test_file_50") != std::string::npos; + }; + } + SECTION("Custom filter (search for test_file_1*)") { + include = [](std::string_view object_name, uint64_t) { + return object_name.find("test_file_1") != std::string::npos; + }; + } + SECTION("Custom filter (reject files over 50 bytes)") { + include = [](std::string_view, uint64_t size) { return size <= 50; }; + } + + // Test collecting results with LsInclude predicate. + auto results = tiledb::VFSExperimental::ls_recursive_filter( + ctx, vfs, s3_test.temp_dir_.to_string(), include); + std::erase_if(expected_results, [&include](const auto& object) { + return !include(object.first, object.second); + }); + CHECK(results.size() == expected_results.size()); + CHECK(expected_results == results); + + // Test collecting results with LsCallback, writing data into ls_objects. + tiledb::VFSExperimental::ls_recursive( + ctx, vfs, s3_test.temp_dir_.to_string(), cb); + CHECK(ls_objects.size() == expected_results.size()); + CHECK(expected_results == ls_objects); +} + +TEST_CASE("CPP API: Callback stops traversal", "[cppapi][vfs][ls-recursive]") { + using namespace tiledb::test; + S3Test s3_test({10, 50, 15}); + if (!s3_test.is_supported()) { + return; + } + auto expected_results = s3_test.expected_results(); + + vfs_config cfg; + tiledb::Context ctx(tiledb::Config(&cfg.config)); + tiledb::VFS vfs(ctx); + + tiledb::VFSExperimental::LsObjects ls_objects; + size_t cb_count = GENERATE(1, 10, 11, 50); + auto cb = [&](std::string_view path, uint64_t size) { + // Always emplace to check the callback is not invoked more than `cb_count`. + ls_objects.emplace_back(path, size); + // Signal to stop traversal when we have seen `cb_count` objects. + if (ls_objects.size() == cb_count) { + return false; + } + return true; + }; + tiledb::VFSExperimental::ls_recursive( + ctx, vfs, s3_test.temp_dir_.to_string(), cb); + expected_results.resize(cb_count); + CHECK(ls_objects.size() == cb_count); + CHECK(ls_objects == expected_results); +} + +TEST_CASE("CPP API: Throwing filter", "[cppapi][vfs][ls-recursive]") { + using namespace tiledb::test; + S3Test s3_test({0}); + if (!s3_test.is_supported()) { + return; + } + + vfs_config cfg; + tiledb::Context ctx(tiledb::Config(&cfg.config)); + tiledb::VFS vfs(ctx); + + tiledb::VFSExperimental::LsInclude filter = [](std::string_view, + uint64_t) -> bool { + throw std::runtime_error("Throwing filter"); + }; + auto path = s3_test.temp_dir_.to_string(); + + // If the test directory is empty the filter should not throw. + SECTION("Throwing filter with 0 objects should not throw") { + CHECK_NOTHROW( + tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter)); + CHECK_NOTHROW( + tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter)); + } + SECTION("Throwing filter with N objects should throw") { + vfs.touch(s3_test.temp_dir_.join_path("test_file").to_string()); + CHECK_THROWS_AS( + tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter), + std::runtime_error); + CHECK_THROWS_WITH( + tiledb::VFSExperimental::ls_recursive_filter(ctx, vfs, path, filter), + Catch::Matchers::ContainsSubstring("Throwing filter")); + CHECK_THROWS_AS( + tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter), + std::runtime_error); + CHECK_THROWS_WITH( + tiledb::VFSExperimental::ls_recursive(ctx, vfs, path, filter), + Catch::Matchers::ContainsSubstring("Throwing filter")); + } +} + +TEST_CASE( + "CPP API: CallbackWrapperCPP construction validation", + "[ls-recursive][callback][wrapper]") { + using tiledb::sm::CallbackWrapperCPP; + tiledb::VFSExperimental::LsObjects data; + auto cb = [&](std::string_view, uint64_t) -> bool { return true; }; + SECTION("Null callback") { + CHECK_THROWS(CallbackWrapperCPP(nullptr)); + } + SECTION("Valid callback") { + CHECK_NOTHROW(CallbackWrapperCPP(cb)); + } +} + +TEST_CASE( + "CPP API: CallbackWrapperCPP operator() validation", + "[ls-recursive][callback][wrapper]") { + tiledb::VFSExperimental::LsObjects data; + auto cb = [&](std::string_view path, uint64_t object_size) -> bool { + if (object_size > 100) { + // Throw if object size is greater than 100 bytes. + throw std::runtime_error("Throwing callback"); + } else if (!path.ends_with(".txt")) { + // Reject non-txt files. + return false; + } + data.emplace_back(path, object_size); + return true; + }; + tiledb::sm::CallbackWrapperCPP wrapper(cb); + + SECTION("Callback return true accepts object") { + CHECK(wrapper("file.txt", 10) == true); + CHECK(data.size() == 1); + } + SECTION("Callback return false rejects object") { + CHECK(wrapper("some/dir/", 0) == false); + CHECK(data.empty()); + } + SECTION("Callback exception is propagated") { + CHECK_THROWS_WITH(wrapper("path", 101) == 0, "Throwing callback"); + } +} diff --git a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc index 985406fb9195..1a2984ebda5d 100644 --- a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc +++ b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc @@ -730,7 +730,7 @@ TEST_CASE( } TEST_CASE( - "C API: CallbackWrapper operator() validation", + "C API: CallbackWrapperCAPI operator() validation", "[ls-recursive][callback][wrapper]") { tiledb::sm::LsObjects data; auto cb = [](const char* path, @@ -748,7 +748,7 @@ TEST_CASE( ls_data->push_back({{path, path_len}, object_size}); return 1; }; - tiledb::sm::CallbackWrapper wrapper(cb, &data); + tiledb::sm::CallbackWrapperCAPI wrapper(cb, &data); SECTION("Callback return 1 signals to continue traversal") { CHECK(wrapper("file.txt", 10) == 1); @@ -763,21 +763,21 @@ TEST_CASE( } TEST_CASE( - "C API: CallbackWrapper construction validation", + "C API: CallbackWrapperCAPI construction validation", "[ls-recursive][callback][wrapper]") { - using tiledb::sm::CallbackWrapper; + using tiledb::sm::CallbackWrapperCAPI; tiledb::sm::LsObjects data; auto cb = [](const char*, size_t, uint64_t, void*) -> int32_t { return 1; }; SECTION("Null callback") { - CHECK_THROWS(CallbackWrapper(nullptr, &data)); + CHECK_THROWS(CallbackWrapperCAPI(nullptr, &data)); } SECTION("Null data") { - CHECK_THROWS(CallbackWrapper(cb, nullptr)); + CHECK_THROWS(CallbackWrapperCAPI(cb, nullptr)); } SECTION("Null callback and data") { - CHECK_THROWS(CallbackWrapper(nullptr, nullptr)); + CHECK_THROWS(CallbackWrapperCAPI(nullptr, nullptr)); } SECTION("Valid callback and data") { - CHECK_NOTHROW(CallbackWrapper(cb, &data)); + CHECK_NOTHROW(CallbackWrapperCAPI(cb, &data)); } } diff --git a/tiledb/api/c_api/vfs/vfs_api_internal.h b/tiledb/api/c_api/vfs/vfs_api_internal.h index fb8dd98f6512..b590d755ec84 100644 --- a/tiledb/api/c_api/vfs/vfs_api_internal.h +++ b/tiledb/api/c_api/vfs/vfs_api_internal.h @@ -151,7 +151,7 @@ struct tiledb_vfs_handle_t const tiledb::sm::URI& parent, tiledb_ls_callback_t cb, void* data) const { - tiledb::sm::CallbackWrapper wrapper(cb, data); + tiledb::sm::CallbackWrapperCAPI wrapper(cb, data); vfs_.ls_recursive(parent, wrapper); } }; diff --git a/tiledb/sm/cpp_api/vfs_experimental.h b/tiledb/sm/cpp_api/vfs_experimental.h new file mode 100644 index 000000000000..bb51b6e93a48 --- /dev/null +++ b/tiledb/sm/cpp_api/vfs_experimental.h @@ -0,0 +1,205 @@ +/** + * @file vfs_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for VFS. + */ + +#ifndef TILEDB_VFS_EXPERIMENTAL_H +#define TILEDB_VFS_EXPERIMENTAL_H + +#include +#include +#include +#include "context.h" +#include "tiledb_experimental.h" +#include "vfs.h" + +namespace tiledb { +class VFSExperimental { + public: + /* ********************************* */ + /* TYPE DEFINITIONS */ + /* ********************************* */ + + /** + * Typedef for ls callback function used to collect results from ls_recursive. + * + * If the callback returns True, the walk will continue. If False, the walk + * will stop. If an error is thrown, the walk will stop and the error will be + * propagated to the caller using std::throw_with_nested. + * + * @param path The path of a visited object for the relative filesystem. + * @param object_size The size of the object at the current path. + * @return True if the walk should continue, else false. + */ + using LsCallback = std::function; + + /** + * Typedef for ls inclusion predicate function used to check if a single + * result should be included in the final results returned from ls_recursive. + * + * If the predicate returns True, the result will be included. If False, the + * result will not be included. If an error is thrown, the walk will stop and + * the error will be propagated. + * + * @param path The path of a visited object for the relative filesystem. + * @param object_size The size of the object at the current path. + * @return True if the result should be included, else false. + */ + using LsInclude = std::function; + + /** + * Default typedef for objects collected by recursive ls, storing a vector of + * pairs for each object path and size. This can be overridden by the client + * to store results into a custom data structure using a custom callback. + * @sa LsCallback + */ + using LsObjects = std::vector>; + + /* ********************************* */ + /* PUBLIC STATIC METHODS */ + /* ********************************* */ + + /** + * Recursively lists objects at the input URI, invoking the provided callback + * on each entry gathered. The callback is passed the data pointer provided + * on each invocation and is responsible for writing the collected results + * into this structure. If the callback returns True, the walk will continue. + * If False, the walk will stop. If an error is thrown, the walk will stop and + * the error will be propagated to the caller using std::throw_with_nested. + * + * Currently only S3 is supported, and the `path` must be a valid S3 URI. + * + * @code{.c} + * VFSExperimental::LsObjects ls_objects; + * VFSExperimental::LsCallback cb = [&](const std::string_view& path, + * uint64_t size) { + * ls_objects.emplace_back(path, size); + * return true; // Continue traversal to next entry. + * } + * + * VFSExperimental::ls_recursive(ctx, vfs, "s3://bucket/foo", cb); + * @endcode + * + * @param ctx The TileDB context. + * @param vfs The VFS instance to use. + * @param uri The base URI to list results recursively. + * @param cb The callback to invoke on each entry. + */ + static void ls_recursive( + const Context& ctx, + const VFS& vfs, + const std::string& uri, + LsCallback cb) { + tiledb::sm::CallbackWrapperCPP wrapper(cb); + ctx.handle_error(tiledb_vfs_ls_recursive( + ctx.ptr().get(), + vfs.ptr().get(), + uri.c_str(), + ls_callback_wrapper, + &wrapper)); + } + + /** + * Recursively lists objects at the input URI, invoking the provided callback + * on each entry gathered. The callback should return true if the entry should + * be included in the results and false otherwise. If no inclusion predicate + * is provided, all results are returned. + * + * Currently only S3 is supported, and the `path` must be a valid S3 URI. + * + * @code{.c} + * VFSExperimental::LsInclude predicate = [](std::string_view path, + * uint64_t object_size) { + * return path.find(".txt") != std::string::npos; + * } + * // Include only files with '.txt' extension using a custom predicate. + * auto ret = VFSExperimental::ls_recursive_filter( + * ctx, vfs, "s3://bucket/foo", predicate); + * + * // Optionally omit the predicate to include all paths collected. + * auto all_paths = VFSExperimental::ls_recursive_filter( + * ctx, vfs, "s3://bucket/foo"); + * @endcode + * + * @param ctx The TileDB context. + * @param vfs The VFS instance to use. + * @param uri The base URI to list results recursively. + * @param include Predicate function to check if a result should be included. + * @return Vector of pairs for each object path and size. + */ + static LsObjects ls_recursive_filter( + const Context& ctx, + const VFS& vfs, + const std::string& uri, + std::optional include = std::nullopt) { + LsObjects ls_objects; + if (include.has_value()) { + auto include_cb = include.value(); + ls_recursive(ctx, vfs, uri, [&](std::string_view path, uint64_t size) { + if (include_cb(path, size)) { + ls_objects.emplace_back(path, size); + } + return true; + }); + } else { + ls_recursive(ctx, vfs, uri, [&](std::string_view path, uint64_t size) { + ls_objects.emplace_back(path, size); + return true; + }); + } + return ls_objects; + } + + private: + /* ********************************* */ + /* PRIVATE STATIC METHODS */ + /* ********************************* */ + + /** + * Callback function for invoking the C++ ls_recursive callback via C API. + * + * @param path The path of a visited object for the relative filesystem. + * @param path_len The length of the path. + * @param object_size The size of the object at the current path. + * @param data Data passed to the callback used to store collected results. + * @return 1 if the callback should continue to the next object, or 0 to stop + * traversal. + * @sa tiledb_ls_callback_t + */ + static int32_t ls_callback_wrapper( + const char* path, size_t path_len, uint64_t object_size, void* data) { + tiledb::sm::CallbackWrapperCPP* cb = + static_cast(data); + return (*cb)({path, path_len}, object_size); + } +}; +} // namespace tiledb + +#endif // TILEDB_VFS_EXPERIMENTAL_H \ No newline at end of file diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h index e357e76d7c65..4f925c55aa0b 100644 --- a/tiledb/sm/filesystem/ls_scanner.h +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -90,7 +90,7 @@ using DirectoryFilter = std::function; * @return 1 if the callback should continue to the next object, 0 to stop * traversal, or -1 if an error occurred. */ -typedef std::function LsCallback; +using LsCallback = std::function; /** Type defintion for objects returned from ls_recursive. */ using LsObjects = std::vector>; @@ -267,13 +267,13 @@ class LsScanner { /** * Wrapper for the C API callback function to be passed to the C++ API. */ -class CallbackWrapper { +class CallbackWrapperCAPI { public: /** Default constructor is deleted */ - CallbackWrapper() = delete; + CallbackWrapperCAPI() = delete; /** Constructor */ - CallbackWrapper(LsCallback cb, void* data) + CallbackWrapperCAPI(LsCallback cb, void* data) : cb_(cb) , data_(data) { if (cb_ == nullptr) { @@ -311,6 +311,45 @@ class CallbackWrapper { void* data_; }; +/** Class to wrap C++ FilePredicate for passing to the C API. */ +class CallbackWrapperCPP { + public: + /** + * Typedef for ls callback function used to check if a single + * result should be included in the final results returned from ls_recursive. + * + * @param path The path of a visited object for the relative filesystem. + * @param size The size of the current object in bytes. + * @return True if the result should be included, else false. + */ + using LsCallback = std::function; + + /** Default constructor is deleted */ + CallbackWrapperCPP() = delete; + + /** Constructor */ + CallbackWrapperCPP(LsCallback cb) + : cb_(cb) { + if (cb_ == nullptr) { + throw LsScanException("ls_recursive callback function cannot be null"); + } + } + + /** + * Operator to wrap the FilePredicate used in the C++ API. + * + * @param path The path of the object. + * @param size The size of the object in bytes. + * @return True if the object should be included, False otherwise. + */ + bool operator()(std::string_view path, uint64_t size) { + return cb_(path, size); + } + + private: + LsCallback cb_; +}; + } // namespace tiledb::sm #endif // TILEDB_LS_SCANNER_H From e221f323414ecf3a446b6944995eeb6d280f15b9 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Sat, 13 Jan 2024 13:49:30 +0100 Subject: [PATCH 142/456] Update dev HISTORY with 2.19.0 and 2.18.3 content. (#4627) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.19.0 and 2.18.3 content. --- HISTORY.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 2b4e85859f23..37e4704ddb30 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,90 @@ +# TileDB v2.19.0 Release Notes + +## Announcements + +* TileDB 2.19, targeted for release in December 2023, includes performance improvements to the aggregate pushdown APIs. + +## Breaking behavior + +* The Windows release artifacts no longer include static libraries. [#4469](https://github.com/TileDB-Inc/TileDB/pull/4469) + +## New features + +* Allow aggregates to pipeline the processing of full tiles. [#4465](https://github.com/TileDB-Inc/TileDB/pull/4465) +* Add class TemporaryDirectory to generate a PRNG-seeded unique directory. [#4498](https://github.com/TileDB-Inc/TileDB/pull/4498) +* Add sm.consolidation.total_buffer_size setting. [#4550](https://github.com/TileDB-Inc/TileDB/pull/4550) +* Use tile metadata in aggregates. [#4468](https://github.com/TileDB-Inc/TileDB/pull/4468) +* Load validity tiles only for null count aggregates when possible. [#4513](https://github.com/TileDB-Inc/TileDB/pull/4513) +* Prevent reading tiles when using fragment metadata in aggregates. [#4481](https://github.com/TileDB-Inc/TileDB/pull/4481) + +## Improvements + +* Refactor the fragment info loading code to reduce coupling. [#4453](https://github.com/TileDB-Inc/TileDB/pull/4453) +* Fragment metadata: adding get_tile_metadata. [#4466](https://github.com/TileDB-Inc/TileDB/pull/4466) +* Improve error reporting when performing too large GCS direct uploads. [#4047](https://github.com/TileDB-Inc/TileDB/pull/4047) +* Refactor filestore API so that it uses internal TileDB library calls. [#3763](https://github.com/TileDB-Inc/TileDB/pull/3763) +* Encode Win32 error messages in UTF-8. [#3987](https://github.com/TileDB-Inc/TileDB/pull/3987) +* Remove unused compression defines. [#4480](https://github.com/TileDB-Inc/TileDB/pull/4480) +* Integrate baseline object library into the main build. [#4446](https://github.com/TileDB-Inc/TileDB/pull/4446) +* Internal clean-up of array timestamps. [#3633](https://github.com/TileDB-Inc/TileDB/pull/3633) +* Add base class for VFS filesystems. [#4497](https://github.com/TileDB-Inc/TileDB/pull/4497) +* Remove extraneous TILEDB_EXPORT tags. [#4507](https://github.com/TileDB-Inc/TileDB/pull/4507) +* Query Plan REST support - part 1: handler. [#4516](https://github.com/TileDB-Inc/TileDB/pull/4516) +* Use StorageFormat APIs to generate timestamped uris. [#4530](https://github.com/TileDB-Inc/TileDB/pull/4530) +* Refactor the group code. [#4378](https://github.com/TileDB-Inc/TileDB/pull/4378) +* Consolidation plan REST support - part1: handler. [#4534](https://github.com/TileDB-Inc/TileDB/pull/4534) +* Use constexpr to check for filesystem support. [#4548](https://github.com/TileDB-Inc/TileDB/pull/4548) +* Add vfs.gcs.max_direct_upload_size config option. [#4047](https://github.com/TileDB-Inc/TileDB/pull/4047) +* Migrate fragment_name parsing into StorageFormat. [#4520](https://github.com/TileDB-Inc/TileDB/pull/4520) +* Add setting to disable install_sigpipe_handler on S3. [#4573](https://github.com/TileDB-Inc/TileDB/pull/4573) + +## Defects removed + +* Group metadata doesn't get serialized. [#3147](https://github.com/TileDB-Inc/TileDB/pull/3147) +* Added `#include ` to `tiledb/sm/cpp_api/type.h`. [#4504](https://github.com/TileDB-Inc/TileDB/pull/4504) +* Fix broken linking tests against a shared WebP library. [#4525](https://github.com/TileDB-Inc/TileDB/pull/4525) +* Fix delta filter deserialization for format ver 19. [#4541](https://github.com/TileDB-Inc/TileDB/pull/4541) +* Fix array schema selection when time traveling. [#4549](https://github.com/TileDB-Inc/TileDB/pull/4549) + +## Documentation + +* Add a table with the history of the storage format versions. [#4487](https://github.com/TileDB-Inc/TileDB/pull/4487) +* Add timestamped_name file to the Format Spec. [#4533](https://github.com/TileDB-Inc/TileDB/pull/4533) + +## API changes + +### C API + +* Add tiledb_group_get_member_by{index|name}_v2 (and deprecate tiledb_group_get_member_by_{index|name}). [#4019](https://github.com/TileDB-Inc/TileDB/pull/4019) +* `tiledb_group_create` is no longer marked as deprecated. [#4522](https://github.com/TileDB-Inc/TileDB/pull/4522) + +### C++ API + +* Improve error message in C++ API for bad filter option type. [#4503](https://github.com/TileDB-Inc/TileDB/pull/4503) + +## Build System Changes + +* Improve building the Windows binaries for the R API. [#4448](https://github.com/TileDB-Inc/TileDB/pull/4448) +* Update curl to version 8.4.0. [#4523](https://github.com/TileDB-Inc/TileDB/pull/4523) +* Clean-up unused build system code. [#4527](https://github.com/TileDB-Inc/TileDB/pull/4527) +* Cache more information when configuring the CMake project. [#4449](https://github.com/TileDB-Inc/TileDB/pull/4449) +* Remove the vcpkg submodule and download it automatically when configuring. [#4484](https://github.com/TileDB-Inc/TileDB/pull/4484) +* Use the provided CMake packages to import the compression libraries when vcpkg is enabled. [#4500](https://github.com/TileDB-Inc/TileDB/pull/4500) +* Remove bundling our static library dependencies; they are now found from the CMake config file. [#4505](https://github.com/TileDB-Inc/TileDB/pull/4505) +* Remove the CMake and bootstrap script options related to abseil and crc32c linkage tests. [#4527](https://github.com/TileDB-Inc/TileDB/pull/4527) +* Move documentation generation and source formatting into dedicated CMake modules. [#4538](https://github.com/TileDB-Inc/TileDB/pull/4538) +* Fix regression when libraries were sometimes installed in the `lib64` directory. [#4572](https://github.com/TileDB-Inc/TileDB/pull/4572) + +## Test only changes + +* Remove UnitTestConfig::array_encryption_key_length. [#4482](https://github.com/TileDB-Inc/TileDB/pull/4482) + +# TileDB v2.18.3 Release Notes + +## New features + +* Add setting to disable install_sigpipe_handler on S3. [#4573](https://github.com/TileDB-Inc/TileDB/pull/4573) + # TileDB v2.18.2 Release Notes ## Defects removed From 31bb094ed1d7b396b679c305d2df7a02dc09beed Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 13 Jan 2024 23:23:42 -0500 Subject: [PATCH 143/456] Update README (#4628) Move Geospatial to the correct section Alphabetize application packages --- TYPE: NO_HISTORY --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fdc5bd144e26..02b74aad5c52 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,11 @@ The TileDB data format is open-source and can be found [here](format_spec/FORMAT ## Application-specific Packages -* [TileDB-Vector-Search](https://github.com/TileDB-Inc/TileDB-Vector-Search): open source, embeddable, and cloud-native vector similarity search database built on top of TileDB in high-performance C++, with an easy-to-use Python API. -* [TileDB-VCF](https://github.com/TileDB-Inc/TileDB-VCF): TileDB library and query engine for genomic variant data. ([documentation](https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics)). -* [TileDB-SOMA](https://github.com/single-cell-data/TileDB-SOMA): TileDB implementation of the [SOMA](https://github.com/single-cell-data/SOMA) specification for single-cell genomic data. ([documentation](https://github.com/single-cell-data/TileDB-SOMA#quick-start)) * [TileDB-BioImaging](https://github.com/TileDB-Inc/TileDB-BioImaging): TileDB library for biomedical imaging, with support for image-optimized compression using WebP. +* [TileDB Geospatial Tools](https://docs.tiledb.com/main/integrations-and-extensions/geospatial) (GDAL, PDAL, Rasterio) +* [TileDB-SOMA](https://github.com/single-cell-data/TileDB-SOMA): TileDB implementation of the [SOMA](https://github.com/single-cell-data/SOMA) specification for single-cell genomic data. ([documentation](https://github.com/single-cell-data/TileDB-SOMA#quick-start)) +* [TileDB-VCF](https://github.com/TileDB-Inc/TileDB-VCF): TileDB library and query engine for genomic variant data. ([documentation](https://docs.tiledb.com/main/integrations-and-extensions/genomics/population-genomics)). +* [TileDB-Vector-Search](https://github.com/TileDB-Inc/TileDB-Vector-Search): open source, embeddable, and cloud-native vector similarity search database built on top of TileDB in high-performance C++, with an easy-to-use Python API. ## APIs @@ -87,7 +88,6 @@ TileDB is also integrated with several popular databases and data science tools: * [MariaDB](https://docs.tiledb.com/main/integrations-and-extensions/sql/mariadb) * [PrestoDB](https://docs.tiledb.com/main/integrations-and-extensions/sql/prestodb) * [Trino](https://docs.tiledb.com/main/integrations-and-extensions/distributed-computing/prestodb-1) -* [Geospatial](https://docs.tiledb.com/main/integrations-and-extensions/geospatial) (GDAL, PDAL, Rasterio) ## Get involved From c2af64c6a78ab08c3fca744b94ae4fdf2e7452d7 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Mon, 15 Jan 2024 13:01:42 +0200 Subject: [PATCH 144/456] Change query plan capnp to use structured data (#4611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR changes the CAPNP format not to serialize the JSON dump of the Query Plan, but to serialize just the required info from the Cloud server (only `Query strategy` for now). It also fixes a couple of issues found during testing, the most ambiguous being: - Part of the information Query Plan prints to the user is `VFS.Backend` . For local queries that’s trivial to get from parsing the array URI. For remote ones, it’s feasible again to get it from URI if it’s of the form `tiledb:///:///` but not if it’s in form `tiledb:///`. To address this for now I am printing `Unknown` as `VFS.Backend` for such array URIs but I am open to better ideas. example QueryPlan output: ``` "TileDB Query Plan": { "Array.Type": "sparse", "Array.URI": "tiledb://unit/s3://tiledb-ypatia/scratch/aov2w20", "Query.Attributes": [ "value" ], "Query.Dimensions": [ "__tiledb_rows" ], "Query.Layout": "global-order", "Query.Strategy.Name": "SparseGlobalOrderReader", "VFS.Backend": "s3" } ``` --- TYPE: NO_HISTORY DESC: Change query plan capnp to use structured data --- test/src/unit-request-handlers.cc | 10 +- tiledb/api/c_api/query_plan/query_plan_api.cc | 6 + tiledb/sm/c_api/tiledb.cc | 4 +- tiledb/sm/filesystem/uri.cc | 12 +- tiledb/sm/query_plan/query_plan.cc | 18 +- tiledb/sm/query_plan/query_plan.h | 10 +- tiledb/sm/serialization/query.cc | 21 +- tiledb/sm/serialization/query.h | 3 +- tiledb/sm/serialization/query_plan.cc | 33 +- tiledb/sm/serialization/query_plan.h | 13 +- tiledb/sm/serialization/tiledb-rest.capnp | 7 +- tiledb/sm/serialization/tiledb-rest.capnp.c++ | 1429 +++++++---------- tiledb/sm/serialization/tiledb-rest.capnp.h | 32 +- 13 files changed, 676 insertions(+), 922 deletions(-) diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 7014b5630957..f3378026adc3 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -85,7 +85,7 @@ struct HandleQueryPlanRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override; - std::string call_handler(SerializationType stype, Query& query); + QueryPlan call_handler(SerializationType stype, Query& query); }; struct HandleConsolidationPlanRequestFx : RequestHandlerFx { @@ -229,7 +229,7 @@ TEST_CASE_METHOD( auto query_plan_ser_deser = call_handler(stype, *query->query_); // Compare the two query plans - REQUIRE(query_plan->view() == query_plan_ser_deser); + REQUIRE(query_plan->view() == query_plan_ser_deser.dump_json()); // Clean up REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); @@ -466,7 +466,7 @@ shared_ptr HandleQueryPlanRequestFx::create_schema() { return schema; } -std::string HandleQueryPlanRequestFx::call_handler( +QueryPlan HandleQueryPlanRequestFx::call_handler( SerializationType stype, Query& query) { auto ctx = tiledb::Context(); auto array = tiledb::Array(ctx, uri_.to_string(), TILEDB_READ); @@ -483,8 +483,8 @@ std::string HandleQueryPlanRequestFx::call_handler( resp_buf); REQUIRE(rval == TILEDB_OK); - auto query_plan = - serialization::deserialize_query_plan_response(stype, resp_buf->buffer()); + auto query_plan = serialization::deserialize_query_plan_response( + query, stype, resp_buf->buffer()); tiledb_buffer_handle_t::break_handle(req_buf); tiledb_buffer_handle_t::break_handle(resp_buf); diff --git a/tiledb/api/c_api/query_plan/query_plan_api.cc b/tiledb/api/c_api/query_plan/query_plan_api.cc index e9d7bbfd3267..cde57b7f9340 100644 --- a/tiledb/api/c_api/query_plan/query_plan_api.cc +++ b/tiledb/api/c_api/query_plan/query_plan_api.cc @@ -48,6 +48,12 @@ capi_return_t tiledb_query_get_plan( throw CAPIStatusException("argument `query` may not be nullptr"); } + if ((*query->query_).array()->is_remote()) { + throw std::logic_error( + "Failed to create a query plan; Remote arrays" + "are not currently supported."); + } + sm::QueryPlan plan(*query->query_); *rv = tiledb_string_handle_t::make_handle(plan.dump_json()); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 5eef66c81abb..93c240dce9b5 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -4347,7 +4347,7 @@ capi_return_t tiledb_handle_query_plan_request( const tiledb_buffer_t* request, tiledb_buffer_t* response) { if (sanity_check(ctx, array) == TILEDB_ERR) { - throw std::invalid_argument("Array paramter must be valid."); + throw std::invalid_argument("Array parameter must be valid."); } api::ensure_buffer_is_valid(request); @@ -4362,7 +4362,7 @@ capi_return_t tiledb_handle_query_plan_request( sm::QueryPlan plan(query); tiledb::sm::serialization::serialize_query_plan_response( - plan.dump_json(), + plan, static_cast(serialization_type), response->buffer()); diff --git a/tiledb/sm/filesystem/uri.cc b/tiledb/sm/filesystem/uri.cc index 3d2deb4a7899..8206ba36fb2b 100644 --- a/tiledb/sm/filesystem/uri.cc +++ b/tiledb/sm/filesystem/uri.cc @@ -326,7 +326,17 @@ std::string URI::to_path(const std::string& uri) { } std::string URI::backend_name() const { - return uri_.substr(0, uri_.find_first_of(':')); + if (is_tiledb(uri_)) { + std::string array_ns, array_uri; + throw_if_not_ok(URI(uri_).get_rest_components(&array_ns, &array_uri)); + auto prefix = array_uri.substr(0, array_uri.find_first_of(':')); + if (prefix == array_uri) { // no `:` separator found in URI + prefix = "Unknown"; + } + return prefix; + } else { + return uri_.substr(0, uri_.find_first_of(':')); + } } std::string URI::to_path() const { diff --git a/tiledb/sm/query_plan/query_plan.cc b/tiledb/sm/query_plan/query_plan.cc index 9c5d6cbcd947..c9aa66cb103c 100644 --- a/tiledb/sm/query_plan/query_plan.cc +++ b/tiledb/sm/query_plan/query_plan.cc @@ -48,20 +48,18 @@ namespace sm { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -QueryPlan::QueryPlan(Query& query) { - if (query.array()->is_remote()) { - throw std::logic_error( - "Failed to create a query plan; Remote arrays" - "are not currently supported."); - } - +QueryPlan::QueryPlan(Query& query, std::string strategy) { array_uri_ = query.array()->array_uri().to_string(); vfs_backend_ = URI(array_uri_).backend_name(); query_layout_ = query.layout(); - // This most probably ends up creating the strategy on the query - auto strategy_ptr = query.strategy(); - strategy_name_ = strategy_ptr->name(); + if (strategy.empty()) { + // This most probably ends up creating the strategy on the query + auto strategy_ptr = query.strategy(); + strategy_name_ = strategy_ptr->name(); + } else { + strategy_name_ = strategy; + } array_type_ = query.array()->array_schema_latest().array_type(); diff --git a/tiledb/sm/query_plan/query_plan.h b/tiledb/sm/query_plan/query_plan.h index c0e544441ca3..a5f2cfdb73d4 100644 --- a/tiledb/sm/query_plan/query_plan.h +++ b/tiledb/sm/query_plan/query_plan.h @@ -65,8 +65,9 @@ class QueryPlan { * Constructor * * @param query A query object for which we want to calculate the plan + * @param strategy The strategy of the query, required only for remote queries */ - QueryPlan(Query& query); + QueryPlan(Query& query, const std::string strategy = ""); /* ****************************** */ /* API */ @@ -81,6 +82,13 @@ class QueryPlan { */ std::string dump_json(uint32_t indent = 4); + /* + * Get the strategy name stored in the query plan + */ + inline std::string strategy() const { + return strategy_name_; + } + private: /* ****************************** */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index e67212f97696..aabfd95adad2 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -1613,7 +1613,8 @@ Status query_from_capnp( void* buffer_start, CopyState* const copy_state, Query* const query, - ThreadPool* compute_tp) { + ThreadPool* compute_tp, + const bool query_plan) { using namespace tiledb::sm; auto array = query->array(); @@ -2114,12 +2115,16 @@ Status query_from_capnp( attr_state->validity_len_data.swap(validity_buff); throw_if_not_ok(query->set_data_buffer( - name, varlen_data, &attr_state->var_len_size, true, true)); + name, varlen_data, &attr_state->var_len_size, !query_plan, true)); throw_if_not_ok(query->set_offsets_buffer( - name, offsets, &attr_state->fixed_len_size, true, true)); + name, offsets, &attr_state->fixed_len_size, !query_plan, true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( - name, validity, &attr_state->validity_len_size, true, true)); + name, + validity, + &attr_state->validity_len_size, + !query_plan, + true)); } } else { auto* data = attribute_buffer_start; @@ -2142,10 +2147,14 @@ Status query_from_capnp( attr_state->validity_len_data.swap(validity_buff); throw_if_not_ok(query->set_data_buffer( - name, data, &attr_state->fixed_len_size, true, true)); + name, data, &attr_state->fixed_len_size, !query_plan, true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( - name, validity, &attr_state->validity_len_size, true, true)); + name, + validity, + &attr_state->validity_len_size, + !query_plan, + true)); } } } else { diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index c2d5e077236f..28821dd69a84 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -275,7 +275,8 @@ Status query_from_capnp( void* buffer_start, CopyState* const copy_state, Query* const query, - ThreadPool* compute_tp); + ThreadPool* compute_tp, + const bool query_plan = false); #endif diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc index 74e36f4de934..07bb764b2144 100644 --- a/tiledb/sm/serialization/query_plan.cc +++ b/tiledb/sm/serialization/query_plan.cc @@ -75,23 +75,24 @@ void query_plan_request_from_capnp( nullptr, nullptr, &query, - &compute_tp)); + &compute_tp, + true)); } } void query_plan_response_to_capnp( - capnp::QueryPlanResponse::Builder& builder, const std::string& query_plan) { - builder.setQueryPlan(query_plan); + capnp::QueryPlanResponse::Builder& builder, const QueryPlan& query_plan) { + builder.setStrategy(query_plan.strategy()); } std::string query_plan_response_from_capnp( const capnp::QueryPlanResponse::Reader& reader) { - std::string query_plan; - if (reader.hasQueryPlan()) { - query_plan = reader.getQueryPlan().cStr(); + std::string strategy; + if (reader.hasStrategy()) { + strategy = reader.getStrategy().cStr(); } - return query_plan; + return strategy; } void serialize_query_plan_request( @@ -193,7 +194,7 @@ void deserialize_query_plan_request( } void serialize_query_plan_response( - const std::string& query_plan, + const QueryPlan& query_plan, SerializationType serialization_type, Buffer& response) { try { @@ -243,8 +244,10 @@ void serialize_query_plan_response( } } -std::string deserialize_query_plan_response( - SerializationType serialization_type, const Buffer& response) { +QueryPlan deserialize_query_plan_response( + Query& query, + SerializationType serialization_type, + const Buffer& response) { try { switch (serialization_type) { case SerializationType::JSON: { @@ -255,7 +258,7 @@ std::string deserialize_query_plan_response( json.decode( kj::StringPtr(static_cast(response.data())), builder); capnp::QueryPlanResponse::Reader reader = builder.asReader(); - return query_plan_response_from_capnp(reader); + return QueryPlan(query, query_plan_response_from_capnp(reader)); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(response.data()); @@ -264,7 +267,7 @@ std::string deserialize_query_plan_response( response.size() / sizeof(::capnp::word))); capnp::QueryPlanResponse::Reader reader = array_reader.getRoot(); - return query_plan_response_from_capnp(reader); + return QueryPlan(query, query_plan_response_from_capnp(reader)); } default: { throw Status_SerializationError( @@ -298,13 +301,13 @@ void deserialize_query_plan_request( } void serialize_query_plan_response( - const std::string&, const SerializationType, Buffer&) { + const QueryPlan&, const SerializationType, Buffer&) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } -std::string deserialize_query_plan_response( - const SerializationType, const Buffer&) { +QueryPlan deserialize_query_plan_response( + Query&, const SerializationType, const Buffer&) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/query_plan.h b/tiledb/sm/serialization/query_plan.h index 7eb46506889b..943e31d65b1b 100644 --- a/tiledb/sm/serialization/query_plan.h +++ b/tiledb/sm/serialization/query_plan.h @@ -84,19 +84,22 @@ void deserialize_query_plan_request( * @param response Buffer to store serialized bytes in. */ void serialize_query_plan_response( - const std::string& query_plan, + const QueryPlan& query_plan, const SerializationType serialization_type, Buffer& response); /** - * Deserialize a Query Plan request to cap'n proto object + * Deserialize a Query Plan response from cap'n proto object * + * @param query The query the plan is requested for. * @param serialization_type Format to serialize from: Cap'n Proto or JSON. * @param response Buffer to read serialized bytes from. - * @return The requested query plan as a string. + * @return The requested query plan. */ -std::string deserialize_query_plan_response( - const SerializationType serialization_type, const Buffer& response); +QueryPlan deserialize_query_plan_response( + Query& query, + const SerializationType serialization_type, + const Buffer& response); } // namespace serialization } // namespace tiledb::sm diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 1f06e2261922..cdc4537baf28 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -71,6 +71,7 @@ struct Array { struct ArrayOpen { config @0 :Config; # Config + queryType @1 :Text; # Query type to open the array for } @@ -1265,13 +1266,13 @@ struct QueryPlanRequest { config @0 :Config; # Config - query @1 : Query; + query @1 :Query; # the query for which we request the plan } struct QueryPlanResponse { - queryPlan @0 :Text; - # The returned query plan + strategy @0 :Text; + # strategy used by the query } struct ConsolidationPlanRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index bfb7eda2dffb..5ea224a7acef 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -5,22 +5,19 @@ namespace capnp { namespace schemas { -static const ::capnp::_::AlignedData<211> b_ce5904e6f9410cec = { +static const ::capnp::_::AlignedData<208> b_ce5904e6f9410cec = { { 0, 0, 0, 0, 5, 0, 6, 0, 236, 12, 65, 249, 230, 4, 89, 206, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 10, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 55, 2, 0, 0, + 29, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, @@ -223,26 +220,23 @@ static const ::capnp::_::AlignedData<211> b_ce5904e6f9410cec = { static const uint16_t m_ce5904e6f9410cec[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_ce5904e6f9410cec[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ce5904e6f9410cec = { - 0xce5904e6f9410cec, b_ce5904e6f9410cec.words, 211, nullptr, m_ce5904e6f9410cec, + 0xce5904e6f9410cec, b_ce5904e6f9410cec.words, 208, nullptr, m_ce5904e6f9410cec, 0, 10, i_ce5904e6f9410cec, nullptr, nullptr, { &s_ce5904e6f9410cec, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<50> b_e3dadf2bf211bc97 = { +static const ::capnp::_::AlignedData<47> b_e3dadf2bf211bc97 = { { 0, 0, 0, 0, 5, 0, 6, 0, 151, 188, 17, 242, 43, 223, 218, 227, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 106, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 170, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 119, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 75, 86, 0, 0, 0, 0, @@ -284,26 +278,23 @@ static const ::capnp::_::AlignedData<50> b_e3dadf2bf211bc97 = { static const uint16_t m_e3dadf2bf211bc97[] = {0, 1}; static const uint16_t i_e3dadf2bf211bc97[] = {0, 1}; const ::capnp::_::RawSchema s_e3dadf2bf211bc97 = { - 0xe3dadf2bf211bc97, b_e3dadf2bf211bc97.words, 50, nullptr, m_e3dadf2bf211bc97, + 0xe3dadf2bf211bc97, b_e3dadf2bf211bc97.words, 47, nullptr, m_e3dadf2bf211bc97, 0, 2, i_e3dadf2bf211bc97, nullptr, nullptr, { &s_e3dadf2bf211bc97, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<40> b_b6c95b4b8111ad36 = { +static const ::capnp::_::AlignedData<37> b_b6c95b4b8111ad36 = { { 0, 0, 0, 0, 5, 0, 6, 0, 54, 173, 17, 129, 75, 91, 201, 182, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 63, 0, 0, 0, + 29, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 102, 105, 103, @@ -338,26 +329,23 @@ static const ::capnp::_::RawSchema* const d_b6c95b4b8111ad36[] = { static const uint16_t m_b6c95b4b8111ad36[] = {0}; static const uint16_t i_b6c95b4b8111ad36[] = {0}; const ::capnp::_::RawSchema s_b6c95b4b8111ad36 = { - 0xb6c95b4b8111ad36, b_b6c95b4b8111ad36.words, 40, d_b6c95b4b8111ad36, m_b6c95b4b8111ad36, + 0xb6c95b4b8111ad36, b_b6c95b4b8111ad36.words, 37, d_b6c95b4b8111ad36, m_b6c95b4b8111ad36, 1, 1, i_b6c95b4b8111ad36, nullptr, nullptr, { &s_b6c95b4b8111ad36, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<220> b_a45730f57e0460b4 = { +static const ::capnp::_::AlignedData<217> b_a45730f57e0460b4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 180, 96, 4, 126, 245, 48, 87, 164, - 42, 0, 0, 0, 1, 0, 3, 0, + 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 111, 2, 0, 0, + 25, 0, 0, 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 0, @@ -580,26 +568,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a45730f57e0460b4 { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a45730f57e0460b4 = { - 0xa45730f57e0460b4, b_a45730f57e0460b4.words, 220, d_a45730f57e0460b4, m_a45730f57e0460b4, + 0xa45730f57e0460b4, b_a45730f57e0460b4.words, 217, d_a45730f57e0460b4, m_a45730f57e0460b4, 6, 11, i_a45730f57e0460b4, nullptr, nullptr, { &s_a45730f57e0460b4, nullptr, bd_a45730f57e0460b4, 0, sizeof(bd_a45730f57e0460b4) / sizeof(bd_a45730f57e0460b4[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_facceeafd4472c68 = { +static const ::capnp::_::AlignedData<49> b_facceeafd4472c68 = { { 0, 0, 0, 0, 5, 0, 6, 0, 104, 44, 71, 212, 175, 238, 204, 250, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 79, @@ -646,26 +631,23 @@ static const ::capnp::_::RawSchema* const d_facceeafd4472c68[] = { static const uint16_t m_facceeafd4472c68[] = {0, 1}; static const uint16_t i_facceeafd4472c68[] = {0, 1}; const ::capnp::_::RawSchema s_facceeafd4472c68 = { - 0xfacceeafd4472c68, b_facceeafd4472c68.words, 52, d_facceeafd4472c68, m_facceeafd4472c68, + 0xfacceeafd4472c68, b_facceeafd4472c68.words, 49, d_facceeafd4472c68, m_facceeafd4472c68, 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<318> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 191, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, @@ -983,26 +965,23 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 318, d_d71de32f98e296fe, m_d71de32f98e296fe, + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<177> b_ceff8d62d10cd1de = { +static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { { 0, 0, 0, 0, 5, 0, 6, 0, 222, 209, 12, 209, 98, 141, 255, 206, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 55, 2, 0, 0, + 33, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -1174,26 +1153,23 @@ static const ::capnp::_::RawSchema* const d_ceff8d62d10cd1de[] = { static const uint16_t m_ceff8d62d10cd1de[] = {3, 6, 0, 7, 1, 4, 8, 9, 5, 2}; static const uint16_t i_ceff8d62d10cd1de[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { - 0xceff8d62d10cd1de, b_ceff8d62d10cd1de.words, 177, d_ceff8d62d10cd1de, m_ceff8d62d10cd1de, + 0xceff8d62d10cd1de, b_ceff8d62d10cd1de.words, 174, d_ceff8d62d10cd1de, m_ceff8d62d10cd1de, 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<146> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<143> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 87, 1, 0, 0, + 33, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 83, @@ -1335,26 +1311,23 @@ static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 4, 5, 2}; static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 146, d_a1b81d67548230d4, m_a1b81d67548230d4, + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 143, d_a1b81d67548230d4, m_a1b81d67548230d4, 2, 6, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<163> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 255, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -1512,26 +1485,23 @@ static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 163, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<128> b_d00b2f19cc21b9b4 = { +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 180, 185, 33, 204, 25, 47, 11, 208, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 143, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 110, 117, 109, 101, 114, @@ -1651,26 +1621,23 @@ static const ::capnp::_::AlignedData<128> b_d00b2f19cc21b9b4 = { static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { - 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 128, nullptr, m_d00b2f19cc21b9b4, + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<146> b_d20a578112fa92a2 = { +static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 162, 146, 250, 18, 129, 87, 10, 210, - 42, 0, 0, 0, 1, 0, 6, 0, + 18, 0, 0, 0, 1, 0, 6, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 2, 2, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 66, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 143, 1, 0, 0, + 33, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -1808,26 +1775,23 @@ static const ::capnp::_::AlignedData<146> b_d20a578112fa92a2 = { static const uint16_t m_d20a578112fa92a2[] = {1, 0, 4, 6, 5, 3, 2}; static const uint16_t i_d20a578112fa92a2[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_d20a578112fa92a2 = { - 0xd20a578112fa92a2, b_d20a578112fa92a2.words, 146, nullptr, m_d20a578112fa92a2, + 0xd20a578112fa92a2, b_d20a578112fa92a2.words, 143, nullptr, m_d20a578112fa92a2, 0, 7, i_d20a578112fa92a2, nullptr, nullptr, { &s_d20a578112fa92a2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<107> b_95e26a84d32d8223 = { +static const ::capnp::_::AlignedData<104> b_95e26a84d32d8223 = { { 0, 0, 0, 0, 5, 0, 6, 0, 35, 130, 45, 211, 132, 106, 226, 149, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -1931,26 +1895,23 @@ static const ::capnp::_::RawSchema* const d_95e26a84d32d8223[] = { static const uint16_t m_95e26a84d32d8223[] = {4, 5, 0, 1, 3, 2}; static const uint16_t i_95e26a84d32d8223[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_95e26a84d32d8223 = { - 0x95e26a84d32d8223, b_95e26a84d32d8223.words, 107, d_95e26a84d32d8223, m_95e26a84d32d8223, + 0x95e26a84d32d8223, b_95e26a84d32d8223.words, 104, d_95e26a84d32d8223, m_95e26a84d32d8223, 3, 6, i_95e26a84d32d8223, nullptr, nullptr, { &s_95e26a84d32d8223, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<171> b_a2a652536db09fa0 = { +static const ::capnp::_::AlignedData<168> b_a2a652536db09fa0 = { { 0, 0, 0, 0, 5, 0, 6, 0, 160, 159, 176, 109, 83, 82, 166, 162, - 52, 0, 0, 0, 1, 0, 2, 0, + 28, 0, 0, 0, 1, 0, 2, 0, 35, 130, 45, 211, 132, 106, 226, 149, 4, 0, 7, 0, 1, 0, 10, 0, 1, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 1, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 55, 2, 0, 0, + 29, 0, 0, 0, 55, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 105, 109, 101, 110, 115, @@ -2116,26 +2077,23 @@ static const ::capnp::_::RawSchema* const d_a2a652536db09fa0[] = { static const uint16_t m_a2a652536db09fa0[] = {8, 9, 2, 4, 6, 0, 3, 5, 7, 1}; static const uint16_t i_a2a652536db09fa0[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; const ::capnp::_::RawSchema s_a2a652536db09fa0 = { - 0xa2a652536db09fa0, b_a2a652536db09fa0.words, 171, d_a2a652536db09fa0, m_a2a652536db09fa0, + 0xa2a652536db09fa0, b_a2a652536db09fa0.words, 168, d_a2a652536db09fa0, m_a2a652536db09fa0, 1, 10, i_a2a652536db09fa0, nullptr, nullptr, { &s_a2a652536db09fa0, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<88> b_de030f447664754c = { +static const ::capnp::_::AlignedData<85> b_de030f447664754c = { { 0, 0, 0, 0, 5, 0, 6, 0, 76, 117, 100, 118, 68, 15, 3, 222, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 111, 109, 97, 105, 110, @@ -2218,26 +2176,23 @@ static const ::capnp::_::RawSchema* const d_de030f447664754c[] = { static const uint16_t m_de030f447664754c[] = {0, 1, 2, 3}; static const uint16_t i_de030f447664754c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_de030f447664754c = { - 0xde030f447664754c, b_de030f447664754c.words, 88, d_de030f447664754c, m_de030f447664754c, + 0xde030f447664754c, b_de030f447664754c.words, 85, d_de030f447664754c, m_de030f447664754c, 1, 4, i_de030f447664754c, nullptr, nullptr, { &s_de030f447664754c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<50> b_fa787661cd3563a4 = { +static const ::capnp::_::AlignedData<47> b_fa787661cd3563a4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 164, 99, 53, 205, 97, 118, 120, 250, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 119, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 114, 114, 111, 114, 0, @@ -2279,26 +2234,23 @@ static const ::capnp::_::AlignedData<50> b_fa787661cd3563a4 = { static const uint16_t m_fa787661cd3563a4[] = {0, 1}; static const uint16_t i_fa787661cd3563a4[] = {0, 1}; const ::capnp::_::RawSchema s_fa787661cd3563a4 = { - 0xfa787661cd3563a4, b_fa787661cd3563a4.words, 50, nullptr, m_fa787661cd3563a4, + 0xfa787661cd3563a4, b_fa787661cd3563a4.words, 47, nullptr, m_fa787661cd3563a4, 0, 2, i_fa787661cd3563a4, nullptr, nullptr, { &s_fa787661cd3563a4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_fda1cb9663a55b70 = { +static const ::capnp::_::AlignedData<65> b_fda1cb9663a55b70 = { { 0, 0, 0, 0, 5, 0, 6, 0, 112, 91, 165, 99, 150, 203, 161, 253, - 42, 0, 0, 0, 1, 0, 3, 0, + 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 108, 111, 97, 116, 83, @@ -2358,26 +2310,23 @@ static const ::capnp::_::AlignedData<68> b_fda1cb9663a55b70 = { static const uint16_t m_fda1cb9663a55b70[] = {2, 1, 0}; static const uint16_t i_fda1cb9663a55b70[] = {0, 1, 2}; const ::capnp::_::RawSchema s_fda1cb9663a55b70 = { - 0xfda1cb9663a55b70, b_fda1cb9663a55b70.words, 68, nullptr, m_fda1cb9663a55b70, + 0xfda1cb9663a55b70, b_fda1cb9663a55b70.words, 65, nullptr, m_fda1cb9663a55b70, 0, 3, i_fda1cb9663a55b70, nullptr, nullptr, { &s_fda1cb9663a55b70, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<97> b_fd5825d8c6e74d78 = { +static const ::capnp::_::AlignedData<94> b_fd5825d8c6e74d78 = { { 0, 0, 0, 0, 5, 0, 6, 0, 120, 77, 231, 198, 216, 37, 88, 253, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 170, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 31, 1, 0, 0, + 29, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 101, 98, 112, 67, 111, @@ -2466,26 +2415,23 @@ static const ::capnp::_::AlignedData<97> b_fd5825d8c6e74d78 = { static const uint16_t m_fd5825d8c6e74d78[] = {3, 4, 1, 2, 0}; static const uint16_t i_fd5825d8c6e74d78[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_fd5825d8c6e74d78 = { - 0xfd5825d8c6e74d78, b_fd5825d8c6e74d78.words, 97, nullptr, m_fd5825d8c6e74d78, + 0xfd5825d8c6e74d78, b_fd5825d8c6e74d78.words, 94, nullptr, m_fd5825d8c6e74d78, 0, 5, i_fd5825d8c6e74d78, nullptr, nullptr, { &s_fd5825d8c6e74d78, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<77> b_e7175047415b3f97 = { +static const ::capnp::_::AlignedData<74> b_e7175047415b3f97 = { { 0, 0, 0, 0, 5, 0, 6, 0, 151, 63, 91, 65, 71, 80, 23, 231, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2559,26 +2505,23 @@ static const ::capnp::_::RawSchema* const d_e7175047415b3f97[] = { static const uint16_t m_e7175047415b3f97[] = {1, 2, 0, 3}; static const uint16_t i_e7175047415b3f97[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_e7175047415b3f97 = { - 0xe7175047415b3f97, b_e7175047415b3f97.words, 77, d_e7175047415b3f97, m_e7175047415b3f97, + 0xe7175047415b3f97, b_e7175047415b3f97.words, 74, d_e7175047415b3f97, m_e7175047415b3f97, 3, 4, i_e7175047415b3f97, nullptr, nullptr, { &s_e7175047415b3f97, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<200> b_9ceaf832b3ab897f = { +static const ::capnp::_::AlignedData<197> b_9ceaf832b3ab897f = { { 0, 0, 0, 0, 5, 0, 6, 0, 127, 137, 171, 179, 50, 248, 234, 156, - 49, 0, 0, 0, 1, 0, 2, 0, + 25, 0, 0, 0, 1, 0, 2, 0, 151, 63, 91, 65, 71, 80, 23, 231, 4, 0, 7, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 167, 2, 0, 0, + 25, 0, 0, 0, 167, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2773,26 +2716,23 @@ static const ::capnp::_::RawSchema* const d_9ceaf832b3ab897f[] = { static const uint16_t m_9ceaf832b3ab897f[] = {1, 10, 11, 4, 6, 8, 2, 0, 5, 7, 9, 3}; static const uint16_t i_9ceaf832b3ab897f[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; const ::capnp::_::RawSchema s_9ceaf832b3ab897f = { - 0x9ceaf832b3ab897f, b_9ceaf832b3ab897f.words, 200, d_9ceaf832b3ab897f, m_9ceaf832b3ab897f, + 0x9ceaf832b3ab897f, b_9ceaf832b3ab897f.words, 197, d_9ceaf832b3ab897f, m_9ceaf832b3ab897f, 1, 12, i_9ceaf832b3ab897f, nullptr, nullptr, { &s_9ceaf832b3ab897f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<41> b_bc4583f733eac4f5 = { +static const ::capnp::_::AlignedData<38> b_bc4583f733eac4f5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 196, 234, 51, 247, 131, 69, 188, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 105, 108, 116, 101, 114, @@ -2828,26 +2768,23 @@ static const ::capnp::_::RawSchema* const d_bc4583f733eac4f5[] = { static const uint16_t m_bc4583f733eac4f5[] = {0}; static const uint16_t i_bc4583f733eac4f5[] = {0}; const ::capnp::_::RawSchema s_bc4583f733eac4f5 = { - 0xbc4583f733eac4f5, b_bc4583f733eac4f5.words, 41, d_bc4583f733eac4f5, m_bc4583f733eac4f5, + 0xbc4583f733eac4f5, b_bc4583f733eac4f5.words, 38, d_bc4583f733eac4f5, m_bc4583f733eac4f5, 1, 1, i_bc4583f733eac4f5, nullptr, nullptr, { &s_bc4583f733eac4f5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_f179c194ae71718c = { +static const ::capnp::_::AlignedData<49> b_f179c194ae71718c = { { 0, 0, 0, 0, 5, 0, 6, 0, 140, 113, 113, 174, 148, 193, 121, 241, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 21, 0, 0, 0, 114, 1, 0, 0, - 41, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 178, 0, 0, 0, + 29, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 141, 0, 0, 0, 23, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, + 129, 0, 0, 0, 23, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 0, 0, 0, @@ -2897,26 +2834,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_f179c194ae71718c { 16777216, ::tiledb::sm::serialization::capnp::Map< ::capnp::AnyPointer, ::capnp::AnyPointer>::Entry::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_f179c194ae71718c = { - 0xf179c194ae71718c, b_f179c194ae71718c.words, 52, d_f179c194ae71718c, m_f179c194ae71718c, + 0xf179c194ae71718c, b_f179c194ae71718c.words, 49, d_f179c194ae71718c, m_f179c194ae71718c, 1, 1, i_f179c194ae71718c, nullptr, nullptr, { &s_f179c194ae71718c, nullptr, bd_f179c194ae71718c, 0, sizeof(bd_f179c194ae71718c) / sizeof(bd_f179c194ae71718c[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<51> b_db5514c8aaf6faea = { +static const ::capnp::_::AlignedData<48> b_db5514c8aaf6faea = { { 0, 0, 0, 0, 5, 0, 6, 0, 234, 250, 246, 170, 200, 20, 85, 219, - 46, 0, 0, 0, 1, 0, 0, 0, + 22, 0, 0, 0, 1, 0, 0, 0, 140, 113, 113, 174, 148, 193, 121, 241, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 46, 69, 110, @@ -2959,26 +2893,23 @@ static const ::capnp::_::AlignedData<51> b_db5514c8aaf6faea = { static const uint16_t m_db5514c8aaf6faea[] = {0, 1}; static const uint16_t i_db5514c8aaf6faea[] = {0, 1}; const ::capnp::_::RawSchema s_db5514c8aaf6faea = { - 0xdb5514c8aaf6faea, b_db5514c8aaf6faea.words, 51, nullptr, m_db5514c8aaf6faea, + 0xdb5514c8aaf6faea, b_db5514c8aaf6faea.words, 48, nullptr, m_db5514c8aaf6faea, 0, 2, i_db5514c8aaf6faea, nullptr, nullptr, { &s_db5514c8aaf6faea, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<43> b_c6b5bb09d4611252 = { +static const ::capnp::_::AlignedData<40> b_c6b5bb09d4611252 = { { 0, 0, 0, 0, 5, 0, 6, 0, 82, 18, 97, 212, 9, 187, 181, 198, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3016,26 +2947,23 @@ static const ::capnp::_::RawSchema* const d_c6b5bb09d4611252[] = { static const uint16_t m_c6b5bb09d4611252[] = {0}; static const uint16_t i_c6b5bb09d4611252[] = {0}; const ::capnp::_::RawSchema s_c6b5bb09d4611252 = { - 0xc6b5bb09d4611252, b_c6b5bb09d4611252.words, 43, d_c6b5bb09d4611252, m_c6b5bb09d4611252, + 0xc6b5bb09d4611252, b_c6b5bb09d4611252.words, 40, d_c6b5bb09d4611252, m_c6b5bb09d4611252, 1, 1, i_c6b5bb09d4611252, nullptr, nullptr, { &s_c6b5bb09d4611252, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_884e0a5f2521a5c6 = { +static const ::capnp::_::AlignedData<49> b_884e0a5f2521a5c6 = { { 0, 0, 0, 0, 5, 0, 6, 0, 198, 165, 33, 37, 95, 10, 78, 136, - 52, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 1, 0, 1, 0, 82, 18, 97, 212, 9, 187, 181, 198, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 210, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3079,26 +3007,23 @@ static const ::capnp::_::AlignedData<52> b_884e0a5f2521a5c6 = { static const uint16_t m_884e0a5f2521a5c6[] = {0, 1}; static const uint16_t i_884e0a5f2521a5c6[] = {0, 1}; const ::capnp::_::RawSchema s_884e0a5f2521a5c6 = { - 0x884e0a5f2521a5c6, b_884e0a5f2521a5c6.words, 52, nullptr, m_884e0a5f2521a5c6, + 0x884e0a5f2521a5c6, b_884e0a5f2521a5c6.words, 49, nullptr, m_884e0a5f2521a5c6, 0, 2, i_884e0a5f2521a5c6, nullptr, nullptr, { &s_884e0a5f2521a5c6, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<43> b_a83707d3ba24dd32 = { +static const ::capnp::_::AlignedData<40> b_a83707d3ba24dd32 = { { 0, 0, 0, 0, 5, 0, 6, 0, 50, 221, 36, 186, 211, 7, 55, 168, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 154, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 218, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, @@ -3136,26 +3061,23 @@ static const ::capnp::_::RawSchema* const d_a83707d3ba24dd32[] = { static const uint16_t m_a83707d3ba24dd32[] = {0}; static const uint16_t i_a83707d3ba24dd32[] = {0}; const ::capnp::_::RawSchema s_a83707d3ba24dd32 = { - 0xa83707d3ba24dd32, b_a83707d3ba24dd32.words, 43, d_a83707d3ba24dd32, m_a83707d3ba24dd32, + 0xa83707d3ba24dd32, b_a83707d3ba24dd32.words, 40, d_a83707d3ba24dd32, m_a83707d3ba24dd32, 1, 1, i_a83707d3ba24dd32, nullptr, nullptr, { &s_a83707d3ba24dd32, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_a9caccb4333a2baf = { +static const ::capnp::_::AlignedData<49> b_a9caccb4333a2baf = { { 0, 0, 0, 0, 5, 0, 6, 0, 175, 43, 58, 51, 180, 204, 202, 169, - 51, 0, 0, 0, 1, 0, 1, 0, + 27, 0, 0, 0, 1, 0, 1, 0, 50, 221, 36, 186, 211, 7, 55, 168, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 73, 110, 116, @@ -3199,26 +3121,23 @@ static const ::capnp::_::AlignedData<52> b_a9caccb4333a2baf = { static const uint16_t m_a9caccb4333a2baf[] = {0, 1}; static const uint16_t i_a9caccb4333a2baf[] = {0, 1}; const ::capnp::_::RawSchema s_a9caccb4333a2baf = { - 0xa9caccb4333a2baf, b_a9caccb4333a2baf.words, 52, nullptr, m_a9caccb4333a2baf, + 0xa9caccb4333a2baf, b_a9caccb4333a2baf.words, 49, nullptr, m_a9caccb4333a2baf, 0, 2, i_a9caccb4333a2baf, nullptr, nullptr, { &s_a9caccb4333a2baf, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<43> b_d3c5983c670e0f42 = { +static const ::capnp::_::AlignedData<40> b_d3c5983c670e0f42 = { { 0, 0, 0, 0, 5, 0, 6, 0, 66, 15, 14, 103, 60, 152, 197, 211, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3256,26 +3175,23 @@ static const ::capnp::_::RawSchema* const d_d3c5983c670e0f42[] = { static const uint16_t m_d3c5983c670e0f42[] = {0}; static const uint16_t i_d3c5983c670e0f42[] = {0}; const ::capnp::_::RawSchema s_d3c5983c670e0f42 = { - 0xd3c5983c670e0f42, b_d3c5983c670e0f42.words, 43, d_d3c5983c670e0f42, m_d3c5983c670e0f42, + 0xd3c5983c670e0f42, b_d3c5983c670e0f42.words, 40, d_d3c5983c670e0f42, m_d3c5983c670e0f42, 1, 1, i_d3c5983c670e0f42, nullptr, nullptr, { &s_d3c5983c670e0f42, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_82d7452da7cd249a = { +static const ::capnp::_::AlignedData<49> b_82d7452da7cd249a = { { 0, 0, 0, 0, 5, 0, 6, 0, 154, 36, 205, 167, 45, 69, 215, 130, - 52, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 1, 0, 1, 0, 66, 15, 14, 103, 60, 152, 197, 211, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 210, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 85, 73, 110, @@ -3319,26 +3235,23 @@ static const ::capnp::_::AlignedData<52> b_82d7452da7cd249a = { static const uint16_t m_82d7452da7cd249a[] = {0, 1}; static const uint16_t i_82d7452da7cd249a[] = {0, 1}; const ::capnp::_::RawSchema s_82d7452da7cd249a = { - 0x82d7452da7cd249a, b_82d7452da7cd249a.words, 52, nullptr, m_82d7452da7cd249a, + 0x82d7452da7cd249a, b_82d7452da7cd249a.words, 49, nullptr, m_82d7452da7cd249a, 0, 2, i_82d7452da7cd249a, nullptr, nullptr, { &s_82d7452da7cd249a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<43> b_9354174d952207d2 = { +static const ::capnp::_::AlignedData<40> b_9354174d952207d2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 210, 7, 34, 149, 77, 23, 84, 147, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 170, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 234, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, @@ -3376,26 +3289,23 @@ static const ::capnp::_::RawSchema* const d_9354174d952207d2[] = { static const uint16_t m_9354174d952207d2[] = {0}; static const uint16_t i_9354174d952207d2[] = {0}; const ::capnp::_::RawSchema s_9354174d952207d2 = { - 0x9354174d952207d2, b_9354174d952207d2.words, 43, d_9354174d952207d2, m_9354174d952207d2, + 0x9354174d952207d2, b_9354174d952207d2.words, 40, d_9354174d952207d2, m_9354174d952207d2, 1, 1, i_9354174d952207d2, nullptr, nullptr, { &s_9354174d952207d2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_bb80cdbc3f312934 = { +static const ::capnp::_::AlignedData<49> b_bb80cdbc3f312934 = { { 0, 0, 0, 0, 5, 0, 6, 0, 52, 41, 49, 63, 188, 205, 128, 187, - 53, 0, 0, 0, 1, 0, 1, 0, + 29, 0, 0, 0, 1, 0, 1, 0, 210, 7, 34, 149, 77, 23, 84, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 112, 70, 108, 111, @@ -3439,26 +3349,23 @@ static const ::capnp::_::AlignedData<52> b_bb80cdbc3f312934 = { static const uint16_t m_bb80cdbc3f312934[] = {0, 1}; static const uint16_t i_bb80cdbc3f312934[] = {0, 1}; const ::capnp::_::RawSchema s_bb80cdbc3f312934 = { - 0xbb80cdbc3f312934, b_bb80cdbc3f312934.words, 52, nullptr, m_bb80cdbc3f312934, + 0xbb80cdbc3f312934, b_bb80cdbc3f312934.words, 49, nullptr, m_bb80cdbc3f312934, 0, 2, i_bb80cdbc3f312934, nullptr, nullptr, { &s_bb80cdbc3f312934, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<51> b_c7e036a11506a1a4 = { +static const ::capnp::_::AlignedData<48> b_c7e036a11506a1a4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 164, 161, 6, 21, 161, 54, 224, 199, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 119, 0, 0, 0, + 25, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 116, 97, 116, 115, 0, @@ -3505,26 +3412,23 @@ static const ::capnp::_::RawSchema* const d_c7e036a11506a1a4[] = { static const uint16_t m_c7e036a11506a1a4[] = {1, 0}; static const uint16_t i_c7e036a11506a1a4[] = {0, 1}; const ::capnp::_::RawSchema s_c7e036a11506a1a4 = { - 0xc7e036a11506a1a4, b_c7e036a11506a1a4.words, 51, d_c7e036a11506a1a4, m_c7e036a11506a1a4, + 0xc7e036a11506a1a4, b_c7e036a11506a1a4.words, 48, d_c7e036a11506a1a4, m_c7e036a11506a1a4, 2, 2, i_c7e036a11506a1a4, nullptr, nullptr, { &s_c7e036a11506a1a4, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<93> b_8bf6c1d37e748294 = { +static const ::capnp::_::AlignedData<90> b_8bf6c1d37e748294 = { { 0, 0, 0, 0, 5, 0, 6, 0, 148, 130, 116, 126, 211, 193, 246, 139, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 231, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 85, 110, 111, 114, 100, 101, @@ -3612,26 +3516,23 @@ static const ::capnp::_::RawSchema* const d_8bf6c1d37e748294[] = { static const uint16_t m_8bf6c1d37e748294[] = {1, 2, 3, 0}; static const uint16_t i_8bf6c1d37e748294[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_8bf6c1d37e748294 = { - 0x8bf6c1d37e748294, b_8bf6c1d37e748294.words, 93, d_8bf6c1d37e748294, m_8bf6c1d37e748294, + 0x8bf6c1d37e748294, b_8bf6c1d37e748294.words, 90, d_8bf6c1d37e748294, m_8bf6c1d37e748294, 1, 4, i_8bf6c1d37e748294, nullptr, nullptr, { &s_8bf6c1d37e748294, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<150> b_8ba60147a0e6735e = { +static const ::capnp::_::AlignedData<147> b_8ba60147a0e6735e = { { 0, 0, 0, 0, 5, 0, 6, 0, 94, 115, 230, 160, 71, 1, 166, 139, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 101, 114, @@ -3780,26 +3681,23 @@ static const ::capnp::_::RawSchema* const d_8ba60147a0e6735e[] = { static const uint16_t m_8ba60147a0e6735e[] = {0, 1, 2, 6, 5, 3, 4, 7}; static const uint16_t i_8ba60147a0e6735e[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_8ba60147a0e6735e = { - 0x8ba60147a0e6735e, b_8ba60147a0e6735e.words, 150, d_8ba60147a0e6735e, m_8ba60147a0e6735e, + 0x8ba60147a0e6735e, b_8ba60147a0e6735e.words, 147, d_8ba60147a0e6735e, m_8ba60147a0e6735e, 5, 8, i_8ba60147a0e6735e, nullptr, nullptr, { &s_8ba60147a0e6735e, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<109> b_86cfc12d74ed4aa0 = { +static const ::capnp::_::AlignedData<106> b_86cfc12d74ed4aa0 = { { 0, 0, 0, 0, 5, 0, 6, 0, 160, 74, 237, 116, 45, 193, 207, 134, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 31, 1, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -3900,26 +3798,23 @@ static const ::capnp::_::AlignedData<109> b_86cfc12d74ed4aa0 = { static const uint16_t m_86cfc12d74ed4aa0[] = {2, 3, 4, 1, 0}; static const uint16_t i_86cfc12d74ed4aa0[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_86cfc12d74ed4aa0 = { - 0x86cfc12d74ed4aa0, b_86cfc12d74ed4aa0.words, 109, nullptr, m_86cfc12d74ed4aa0, + 0x86cfc12d74ed4aa0, b_86cfc12d74ed4aa0.words, 106, nullptr, m_86cfc12d74ed4aa0, 0, 5, i_86cfc12d74ed4aa0, nullptr, nullptr, { &s_86cfc12d74ed4aa0, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_f7aa276767b422e7 = { +static const ::capnp::_::AlignedData<65> b_f7aa276767b422e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 34, 180, 103, 103, 39, 170, 247, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 97, 98, 101, 108, 83, @@ -3982,26 +3877,23 @@ static const ::capnp::_::RawSchema* const d_f7aa276767b422e7[] = { static const uint16_t m_f7aa276767b422e7[] = {0, 1, 2}; static const uint16_t i_f7aa276767b422e7[] = {0, 1, 2}; const ::capnp::_::RawSchema s_f7aa276767b422e7 = { - 0xf7aa276767b422e7, b_f7aa276767b422e7.words, 68, d_f7aa276767b422e7, m_f7aa276767b422e7, + 0xf7aa276767b422e7, b_f7aa276767b422e7.words, 65, d_f7aa276767b422e7, m_f7aa276767b422e7, 1, 3, i_f7aa276767b422e7, nullptr, nullptr, { &s_f7aa276767b422e7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<161> b_dba20dec138adac9 = { +static const ::capnp::_::AlignedData<158> b_dba20dec138adac9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 201, 218, 138, 19, 236, 13, 162, 219, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 154, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 218, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 143, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4163,26 +4055,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_dba20dec138adac9 { 16777221, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::SubarrayRanges>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_dba20dec138adac9 = { - 0xdba20dec138adac9, b_dba20dec138adac9.words, 161, d_dba20dec138adac9, m_dba20dec138adac9, + 0xdba20dec138adac9, b_dba20dec138adac9.words, 158, d_dba20dec138adac9, m_dba20dec138adac9, 4, 7, i_dba20dec138adac9, nullptr, nullptr, { &s_dba20dec138adac9, nullptr, bd_dba20dec138adac9, 0, sizeof(bd_dba20dec138adac9) / sizeof(bd_dba20dec138adac9[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<158> b_ff14003c70494585 = { +static const ::capnp::_::AlignedData<155> b_ff14003c70494585 = { { 0, 0, 0, 0, 5, 0, 6, 0, 133, 69, 73, 112, 60, 0, 20, 255, - 42, 0, 0, 0, 1, 0, 3, 0, + 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 73, 0, 0, 0, 199, 1, 0, 0, + 61, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4339,26 +4228,23 @@ static const ::capnp::_::RawSchema* const d_ff14003c70494585[] = { static const uint16_t m_ff14003c70494585[] = {1, 2, 4, 6, 5, 3, 7, 0}; static const uint16_t i_ff14003c70494585[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_ff14003c70494585 = { - 0xff14003c70494585, b_ff14003c70494585.words, 158, d_ff14003c70494585, m_ff14003c70494585, + 0xff14003c70494585, b_ff14003c70494585.words, 155, d_ff14003c70494585, m_ff14003c70494585, 5, 8, i_ff14003c70494585, nullptr, nullptr, { &s_ff14003c70494585, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<86> b_f86b7bf97823250f = { +static const ::capnp::_::AlignedData<83> b_f86b7bf97823250f = { { 0, 0, 0, 0, 5, 0, 6, 0, 15, 37, 35, 120, 249, 123, 107, 248, - 62, 0, 0, 0, 1, 0, 3, 0, + 38, 0, 0, 0, 1, 0, 3, 0, 133, 69, 73, 112, 60, 0, 20, 255, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 98, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 162, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 231, 0, 0, 0, + 41, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4439,26 +4325,23 @@ static const ::capnp::_::RawSchema* const d_f86b7bf97823250f[] = { static const uint16_t m_f86b7bf97823250f[] = {2, 3, 1, 0}; static const uint16_t i_f86b7bf97823250f[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_f86b7bf97823250f = { - 0xf86b7bf97823250f, b_f86b7bf97823250f.words, 86, d_f86b7bf97823250f, m_f86b7bf97823250f, + 0xf86b7bf97823250f, b_f86b7bf97823250f.words, 83, d_f86b7bf97823250f, m_f86b7bf97823250f, 1, 4, i_f86b7bf97823250f, nullptr, nullptr, { &s_f86b7bf97823250f, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<93> b_fdd9e47288724221 = { +static const ::capnp::_::AlignedData<90> b_fdd9e47288724221 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 66, 114, 136, 114, 228, 217, 253, - 62, 0, 0, 0, 1, 0, 2, 0, + 38, 0, 0, 0, 1, 0, 2, 0, 133, 69, 73, 112, 60, 0, 20, 255, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 34, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 98, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 231, 0, 0, 0, + 37, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 117, 98, 97, 114, 114, @@ -4546,26 +4429,23 @@ static const ::capnp::_::RawSchema* const d_fdd9e47288724221[] = { static const uint16_t m_fdd9e47288724221[] = {1, 3, 2, 0}; static const uint16_t i_fdd9e47288724221[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_fdd9e47288724221 = { - 0xfdd9e47288724221, b_fdd9e47288724221.words, 93, d_fdd9e47288724221, m_fdd9e47288724221, + 0xfdd9e47288724221, b_fdd9e47288724221.words, 90, d_fdd9e47288724221, m_fdd9e47288724221, 1, 4, i_fdd9e47288724221, nullptr, nullptr, { &s_fdd9e47288724221, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<86> b_cbe1e7c13508aa2c = { +static const ::capnp::_::AlignedData<83> b_cbe1e7c13508aa2c = { { 0, 0, 0, 0, 5, 0, 6, 0, 44, 170, 8, 53, 193, 231, 225, 203, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, @@ -4646,26 +4526,23 @@ static const ::capnp::_::RawSchema* const d_cbe1e7c13508aa2c[] = { static const uint16_t m_cbe1e7c13508aa2c[] = {2, 0, 3, 1}; static const uint16_t i_cbe1e7c13508aa2c[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { - 0xcbe1e7c13508aa2c, b_cbe1e7c13508aa2c.words, 86, d_cbe1e7c13508aa2c, m_cbe1e7c13508aa2c, + 0xcbe1e7c13508aa2c, b_cbe1e7c13508aa2c.words, 83, d_cbe1e7c13508aa2c, m_cbe1e7c13508aa2c, 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<84> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 210, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 18, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 231, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, @@ -4741,26 +4618,23 @@ static const ::capnp::_::AlignedData<84> b_dac6a7f675c57409 = { static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 84, nullptr, m_dac6a7f675c57409, + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<150> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<147> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 146, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 210, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 199, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 83, 84, 78, 111, 100, @@ -4905,26 +4779,23 @@ static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 7, 3, 6, 2}; static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6, 7}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 150, d_afc739d5c01e6496, m_afc739d5c01e6496, + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 147, d_afc739d5c01e6496, m_afc739d5c01e6496, 1, 8, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<76> b_eaf57cb9871fc06f = { +static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { { 0, 0, 0, 0, 5, 0, 6, 0, 111, 192, 31, 135, 185, 124, 245, 234, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 100, 105, 116, @@ -4996,26 +4867,23 @@ static const ::capnp::_::RawSchema* const d_eaf57cb9871fc06f[] = { static const uint16_t m_eaf57cb9871fc06f[] = {1, 0, 2}; static const uint16_t i_eaf57cb9871fc06f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_eaf57cb9871fc06f = { - 0xeaf57cb9871fc06f, b_eaf57cb9871fc06f.words, 76, d_eaf57cb9871fc06f, m_eaf57cb9871fc06f, + 0xeaf57cb9871fc06f, b_eaf57cb9871fc06f.words, 73, d_eaf57cb9871fc06f, m_eaf57cb9871fc06f, 2, 3, i_eaf57cb9871fc06f, nullptr, nullptr, { &s_eaf57cb9871fc06f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_e19754f813ccf79c = { +static const ::capnp::_::AlignedData<113> b_e19754f813ccf79c = { { 0, 0, 0, 0, 5, 0, 6, 0, 156, 247, 204, 19, 248, 84, 151, 225, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 87, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 82, @@ -5129,26 +4997,23 @@ static const ::capnp::_::RawSchema* const d_e19754f813ccf79c[] = { static const uint16_t m_e19754f813ccf79c[] = {3, 5, 0, 2, 4, 1}; static const uint16_t i_e19754f813ccf79c[] = {0, 1, 2, 3, 4, 5}; const ::capnp::_::RawSchema s_e19754f813ccf79c = { - 0xe19754f813ccf79c, b_e19754f813ccf79c.words, 116, d_e19754f813ccf79c, m_e19754f813ccf79c, + 0xe19754f813ccf79c, b_e19754f813ccf79c.words, 113, d_e19754f813ccf79c, m_e19754f813ccf79c, 4, 6, i_e19754f813ccf79c, nullptr, nullptr, { &s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_d74f5fed155d316c = { +static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = { { 0, 0, 0, 0, 5, 0, 6, 0, 108, 49, 93, 21, 237, 95, 79, 215, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 138, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 68, 101, 108, 101, 116, 101, @@ -5196,26 +5061,23 @@ static const ::capnp::_::RawSchema* const d_d74f5fed155d316c[] = { static const uint16_t m_d74f5fed155d316c[] = {0, 1}; static const uint16_t i_d74f5fed155d316c[] = {0, 1}; const ::capnp::_::RawSchema s_d74f5fed155d316c = { - 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 52, d_d74f5fed155d316c, m_d74f5fed155d316c, + 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 49, d_d74f5fed155d316c, m_d74f5fed155d316c, 2, 2, i_d74f5fed155d316c, nullptr, nullptr, { &s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<82> b_def87cead82188e7 = { +static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 136, 33, 216, 234, 124, 248, 222, - 42, 0, 0, 0, 1, 0, 4, 0, + 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 231, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 115, 117, 108, 116, @@ -5289,26 +5151,23 @@ static const ::capnp::_::AlignedData<82> b_def87cead82188e7 = { static const uint16_t m_def87cead82188e7[] = {0, 3, 2, 1}; static const uint16_t i_def87cead82188e7[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_def87cead82188e7 = { - 0xdef87cead82188e7, b_def87cead82188e7.words, 82, nullptr, m_def87cead82188e7, + 0xdef87cead82188e7, b_def87cead82188e7.words, 79, nullptr, m_def87cead82188e7, 0, 4, i_def87cead82188e7, nullptr, nullptr, { &s_def87cead82188e7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<51> b_c1a2d010de779de5 = { +static const ::capnp::_::AlignedData<48> b_c1a2d010de779de5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 229, 157, 119, 222, 16, 208, 162, 193, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -5351,26 +5210,23 @@ static const ::capnp::_::AlignedData<51> b_c1a2d010de779de5 = { static const uint16_t m_c1a2d010de779de5[] = {1, 0}; static const uint16_t i_c1a2d010de779de5[] = {0, 1}; const ::capnp::_::RawSchema s_c1a2d010de779de5 = { - 0xc1a2d010de779de5, b_c1a2d010de779de5.words, 51, nullptr, m_c1a2d010de779de5, + 0xc1a2d010de779de5, b_c1a2d010de779de5.words, 48, nullptr, m_c1a2d010de779de5, 0, 2, i_c1a2d010de779de5, nullptr, nullptr, { &s_c1a2d010de779de5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<79> b_c86c77b5f6a2bf0f = { +static const ::capnp::_::AlignedData<76> b_c86c77b5f6a2bf0f = { { 0, 0, 0, 0, 5, 0, 6, 0, 15, 191, 162, 246, 181, 119, 108, 200, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 83, 116, @@ -5445,26 +5301,23 @@ static const ::capnp::_::RawSchema* const d_c86c77b5f6a2bf0f[] = { static const uint16_t m_c86c77b5f6a2bf0f[] = {2, 1, 0}; static const uint16_t i_c86c77b5f6a2bf0f[] = {0, 1, 2}; const ::capnp::_::RawSchema s_c86c77b5f6a2bf0f = { - 0xc86c77b5f6a2bf0f, b_c86c77b5f6a2bf0f.words, 79, d_c86c77b5f6a2bf0f, m_c86c77b5f6a2bf0f, + 0xc86c77b5f6a2bf0f, b_c86c77b5f6a2bf0f.words, 76, d_c86c77b5f6a2bf0f, m_c86c77b5f6a2bf0f, 2, 3, i_c86c77b5f6a2bf0f, nullptr, nullptr, { &s_c86c77b5f6a2bf0f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<99> b_9b9a5fc7713a8692 = { +static const ::capnp::_::AlignedData<96> b_9b9a5fc7713a8692 = { { 0, 0, 0, 0, 5, 0, 6, 0, 146, 134, 58, 113, 199, 95, 154, 155, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 31, 1, 0, 0, + 29, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 82, 101, 97, 100, 101, 114, @@ -5561,26 +5414,23 @@ static const ::capnp::_::RawSchema* const d_9b9a5fc7713a8692[] = { static const uint16_t m_9b9a5fc7713a8692[] = {3, 0, 2, 4, 1}; static const uint16_t i_9b9a5fc7713a8692[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { - 0x9b9a5fc7713a8692, b_9b9a5fc7713a8692.words, 99, d_9b9a5fc7713a8692, m_9b9a5fc7713a8692, + 0x9b9a5fc7713a8692, b_9b9a5fc7713a8692.words, 96, d_9b9a5fc7713a8692, m_9b9a5fc7713a8692, 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<390> b_96ba49d0f8b23ccc = { +static const ::capnp::_::AlignedData<387> b_96ba49d0f8b23ccc = { { 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, 150, - 42, 0, 0, 0, 1, 0, 4, 0, + 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 17, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 215, 4, 0, 0, + 25, 0, 0, 0, 215, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, @@ -5974,26 +5824,23 @@ static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 21, 13, 17, 16, 1, 20, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4, 19, 18}; static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 390, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 387, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, 10, 22, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<72> b_9df6f2a42c4e5f0b = { +static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { { 0, 0, 0, 0, 5, 0, 6, 0, 11, 95, 78, 44, 164, 242, 246, 157, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, @@ -6060,26 +5907,23 @@ static const ::capnp::_::RawSchema* const d_9df6f2a42c4e5f0b[] = { static const uint16_t m_9df6f2a42c4e5f0b[] = {1, 0, 2}; static const uint16_t i_9df6f2a42c4e5f0b[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9df6f2a42c4e5f0b = { - 0x9df6f2a42c4e5f0b, b_9df6f2a42c4e5f0b.words, 72, d_9df6f2a42c4e5f0b, m_9df6f2a42c4e5f0b, + 0x9df6f2a42c4e5f0b, b_9df6f2a42c4e5f0b.words, 69, d_9df6f2a42c4e5f0b, m_9df6f2a42c4e5f0b, 1, 3, i_9df6f2a42c4e5f0b, nullptr, nullptr, { &s_9df6f2a42c4e5f0b, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<42> b_a18264549448ece3 = { +static const ::capnp::_::AlignedData<39> b_a18264549448ece3 = { { 0, 0, 0, 0, 5, 0, 6, 0, 227, 236, 72, 148, 84, 100, 130, 161, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 78, 111, 110, 69, 109, 112, @@ -6116,26 +5960,23 @@ static const ::capnp::_::RawSchema* const d_a18264549448ece3[] = { static const uint16_t m_a18264549448ece3[] = {0}; static const uint16_t i_a18264549448ece3[] = {0}; const ::capnp::_::RawSchema s_a18264549448ece3 = { - 0xa18264549448ece3, b_a18264549448ece3.words, 42, d_a18264549448ece3, m_a18264549448ece3, + 0xa18264549448ece3, b_a18264549448ece3.words, 39, d_a18264549448ece3, m_a18264549448ece3, 1, 1, i_a18264549448ece3, nullptr, nullptr, { &s_a18264549448ece3, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<86> b_9be1921b07e6cd2d = { +static const ::capnp::_::AlignedData<83> b_9be1921b07e6cd2d = { { 0, 0, 0, 0, 5, 0, 6, 0, 45, 205, 230, 7, 27, 146, 225, 155, - 42, 0, 0, 0, 1, 0, 3, 0, + 18, 0, 0, 0, 1, 0, 3, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 231, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 116, 116, 114, 105, 98, @@ -6213,26 +6054,23 @@ static const ::capnp::_::AlignedData<86> b_9be1921b07e6cd2d = { static const uint16_t m_9be1921b07e6cd2d[] = {0, 2, 1, 3}; static const uint16_t i_9be1921b07e6cd2d[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_9be1921b07e6cd2d = { - 0x9be1921b07e6cd2d, b_9be1921b07e6cd2d.words, 86, nullptr, m_9be1921b07e6cd2d, + 0x9be1921b07e6cd2d, b_9be1921b07e6cd2d.words, 83, nullptr, m_9be1921b07e6cd2d, 0, 4, i_9be1921b07e6cd2d, nullptr, nullptr, { &s_9be1921b07e6cd2d, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<42> b_f01116579e9ea98e = { +static const ::capnp::_::AlignedData<39> b_f01116579e9ea98e = { { 0, 0, 0, 0, 5, 0, 6, 0, 142, 169, 158, 158, 87, 22, 17, 240, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 97, 120, 66, 117, 102, @@ -6269,26 +6107,23 @@ static const ::capnp::_::RawSchema* const d_f01116579e9ea98e[] = { static const uint16_t m_f01116579e9ea98e[] = {0}; static const uint16_t i_f01116579e9ea98e[] = {0}; const ::capnp::_::RawSchema s_f01116579e9ea98e = { - 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 42, d_f01116579e9ea98e, m_f01116579e9ea98e, + 0xf01116579e9ea98e, b_f01116579e9ea98e.words, 39, d_f01116579e9ea98e, m_f01116579e9ea98e, 1, 1, i_f01116579e9ea98e, nullptr, nullptr, { &s_f01116579e9ea98e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<44> b_9737dcafdfce31bb = { +static const ::capnp::_::AlignedData<41> b_9737dcafdfce31bb = { { 0, 0, 0, 0, 5, 0, 6, 0, 187, 49, 206, 223, 175, 220, 55, 151, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 0, 0, 0, 63, 0, 0, 0, + 45, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, @@ -6327,26 +6162,23 @@ static const ::capnp::_::RawSchema* const d_9737dcafdfce31bb[] = { static const uint16_t m_9737dcafdfce31bb[] = {0}; static const uint16_t i_9737dcafdfce31bb[] = {0}; const ::capnp::_::RawSchema s_9737dcafdfce31bb = { - 0x9737dcafdfce31bb, b_9737dcafdfce31bb.words, 44, d_9737dcafdfce31bb, m_9737dcafdfce31bb, + 0x9737dcafdfce31bb, b_9737dcafdfce31bb.words, 41, d_9737dcafdfce31bb, m_9737dcafdfce31bb, 1, 1, i_9737dcafdfce31bb, nullptr, nullptr, { &s_9737dcafdfce31bb, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<99> b_926fe1c3b12ed651 = { +static const ::capnp::_::AlignedData<96> b_926fe1c3b12ed651 = { { 0, 0, 0, 0, 5, 0, 6, 0, 81, 214, 46, 177, 195, 225, 111, 146, - 56, 0, 0, 0, 1, 0, 1, 0, + 32, 0, 0, 0, 1, 0, 1, 0, 187, 49, 206, 223, 175, 220, 55, 151, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 50, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 114, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 31, 1, 0, 0, + 37, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 77, @@ -6437,26 +6269,23 @@ static const ::capnp::_::AlignedData<99> b_926fe1c3b12ed651 = { static const uint16_t m_926fe1c3b12ed651[] = {4, 0, 1, 3, 2}; static const uint16_t i_926fe1c3b12ed651[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_926fe1c3b12ed651 = { - 0x926fe1c3b12ed651, b_926fe1c3b12ed651.words, 99, nullptr, m_926fe1c3b12ed651, + 0x926fe1c3b12ed651, b_926fe1c3b12ed651.words, 96, nullptr, m_926fe1c3b12ed651, 0, 5, i_926fe1c3b12ed651, nullptr, nullptr, { &s_926fe1c3b12ed651, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<313> b_9317f20ce509d918 = { +static const ::capnp::_::AlignedData<310> b_9317f20ce509d918 = { { 0, 0, 0, 0, 5, 0, 6, 0, 24, 217, 9, 229, 12, 242, 23, 147, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 12, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 202, 1, 0, 0, - 49, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 10, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 85, 0, 0, 0, 23, 3, 0, 0, + 73, 0, 0, 0, 23, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6765,26 +6594,23 @@ static const ::capnp::_::RawSchema* const d_9317f20ce509d918[] = { static const uint16_t m_9317f20ce509d918[] = {9, 4, 5, 2, 6, 7, 1, 8, 11, 10, 3, 13, 12, 0}; static const uint16_t i_9317f20ce509d918[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}; const ::capnp::_::RawSchema s_9317f20ce509d918 = { - 0x9317f20ce509d918, b_9317f20ce509d918.words, 313, d_9317f20ce509d918, m_9317f20ce509d918, + 0x9317f20ce509d918, b_9317f20ce509d918.words, 310, d_9317f20ce509d918, m_9317f20ce509d918, 2, 14, i_9317f20ce509d918, nullptr, nullptr, { &s_9317f20ce509d918, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<70> b_d9d27c082dec9e26 = { +static const ::capnp::_::AlignedData<67> b_d9d27c082dec9e26 = { { 0, 0, 0, 0, 5, 0, 6, 0, 38, 158, 236, 45, 8, 124, 210, 217, - 57, 0, 0, 0, 1, 0, 2, 0, + 33, 0, 0, 0, 1, 0, 2, 0, 24, 217, 9, 229, 12, 242, 23, 147, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 66, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 130, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 175, 0, 0, 0, + 37, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6846,26 +6672,23 @@ static const ::capnp::_::AlignedData<70> b_d9d27c082dec9e26 = { static const uint16_t m_d9d27c082dec9e26[] = {2, 1, 0}; static const uint16_t i_d9d27c082dec9e26[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d9d27c082dec9e26 = { - 0xd9d27c082dec9e26, b_d9d27c082dec9e26.words, 70, nullptr, m_d9d27c082dec9e26, + 0xd9d27c082dec9e26, b_d9d27c082dec9e26.words, 67, nullptr, m_d9d27c082dec9e26, 0, 3, i_d9d27c082dec9e26, nullptr, nullptr, { &s_d9d27c082dec9e26, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<71> b_9eb745af522d087c = { +static const ::capnp::_::AlignedData<68> b_9eb745af522d087c = { { 0, 0, 0, 0, 5, 0, 6, 0, 124, 8, 45, 82, 175, 69, 183, 158, - 57, 0, 0, 0, 1, 0, 1, 0, + 33, 0, 0, 0, 1, 0, 1, 0, 24, 217, 9, 229, 12, 242, 23, 147, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 170, 2, 0, 0, - 61, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 234, 1, 0, 0, + 49, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 57, 0, 0, 0, 175, 0, 0, 0, + 45, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -6928,26 +6751,23 @@ static const ::capnp::_::AlignedData<71> b_9eb745af522d087c = { static const uint16_t m_9eb745af522d087c[] = {1, 2, 0}; static const uint16_t i_9eb745af522d087c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_9eb745af522d087c = { - 0x9eb745af522d087c, b_9eb745af522d087c.words, 71, nullptr, m_9eb745af522d087c, + 0x9eb745af522d087c, b_9eb745af522d087c.words, 68, nullptr, m_9eb745af522d087c, 0, 3, i_9eb745af522d087c, nullptr, nullptr, { &s_9eb745af522d087c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<98> b_8cd4e323f1feea3b = { +static const ::capnp::_::AlignedData<95> b_8cd4e323f1feea3b = { { 0, 0, 0, 0, 5, 0, 6, 0, 59, 234, 254, 241, 35, 227, 212, 140, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 39, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 119, 0, 0, 0, + 65, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -7044,26 +6864,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_8cd4e323f1feea3b { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::EstimatedResultSize::MemorySize>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_8cd4e323f1feea3b = { - 0x8cd4e323f1feea3b, b_8cd4e323f1feea3b.words, 98, d_8cd4e323f1feea3b, m_8cd4e323f1feea3b, + 0x8cd4e323f1feea3b, b_8cd4e323f1feea3b.words, 95, d_8cd4e323f1feea3b, m_8cd4e323f1feea3b, 1, 2, i_8cd4e323f1feea3b, nullptr, nullptr, { &s_8cd4e323f1feea3b, nullptr, bd_8cd4e323f1feea3b, 0, sizeof(bd_8cd4e323f1feea3b) / sizeof(bd_8cd4e323f1feea3b[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<71> b_92c8467685565269 = { +static const ::capnp::_::AlignedData<68> b_92c8467685565269 = { { 0, 0, 0, 0, 5, 0, 6, 0, 105, 82, 86, 133, 118, 70, 200, 146, - 62, 0, 0, 0, 1, 0, 3, 0, + 38, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 74, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -7126,26 +6943,23 @@ static const ::capnp::_::AlignedData<71> b_92c8467685565269 = { static const uint16_t m_92c8467685565269[] = {0, 2, 1}; static const uint16_t i_92c8467685565269[] = {0, 1, 2}; const ::capnp::_::RawSchema s_92c8467685565269 = { - 0x92c8467685565269, b_92c8467685565269.words, 71, nullptr, m_92c8467685565269, + 0x92c8467685565269, b_92c8467685565269.words, 68, nullptr, m_92c8467685565269, 0, 3, i_92c8467685565269, nullptr, nullptr, { &s_92c8467685565269, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<71> b_bda7916926591c22 = { +static const ::capnp::_::AlignedData<68> b_bda7916926591c22 = { { 0, 0, 0, 0, 5, 0, 6, 0, 34, 28, 89, 38, 105, 145, 167, 189, - 62, 0, 0, 0, 1, 0, 3, 0, + 38, 0, 0, 0, 1, 0, 3, 0, 59, 234, 254, 241, 35, 227, 212, 140, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 74, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 69, 115, 116, 105, 109, 97, @@ -7208,26 +7022,23 @@ static const ::capnp::_::AlignedData<71> b_bda7916926591c22 = { static const uint16_t m_bda7916926591c22[] = {0, 2, 1}; static const uint16_t i_bda7916926591c22[] = {0, 1, 2}; const ::capnp::_::RawSchema s_bda7916926591c22 = { - 0xbda7916926591c22, b_bda7916926591c22.words, 71, nullptr, m_bda7916926591c22, + 0xbda7916926591c22, b_bda7916926591c22.words, 68, nullptr, m_bda7916926591c22, 0, 3, i_bda7916926591c22, nullptr, nullptr, { &s_bda7916926591c22, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<37> b_c95970eb9310dc9c = { +static const ::capnp::_::AlignedData<34> b_c95970eb9310dc9c = { { 0, 0, 0, 0, 5, 0, 6, 0, 156, 220, 16, 147, 235, 112, 89, 201, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -7259,26 +7070,23 @@ static const ::capnp::_::RawSchema* const d_c95970eb9310dc9c[] = { static const uint16_t m_c95970eb9310dc9c[] = {0}; static const uint16_t i_c95970eb9310dc9c[] = {0}; const ::capnp::_::RawSchema s_c95970eb9310dc9c = { - 0xc95970eb9310dc9c, b_c95970eb9310dc9c.words, 37, d_c95970eb9310dc9c, m_c95970eb9310dc9c, + 0xc95970eb9310dc9c, b_c95970eb9310dc9c.words, 34, d_c95970eb9310dc9c, m_c95970eb9310dc9c, 1, 1, i_c95970eb9310dc9c, nullptr, nullptr, { &s_c95970eb9310dc9c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<69> b_d42e7b38b33e3d29 = { +static const ::capnp::_::AlignedData<66> b_d42e7b38b33e3d29 = { { 0, 0, 0, 0, 5, 0, 6, 0, 41, 61, 62, 179, 56, 123, 46, 212, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, @@ -7342,26 +7150,23 @@ static const ::capnp::_::RawSchema* const d_d42e7b38b33e3d29[] = { static const uint16_t m_d42e7b38b33e3d29[] = {0, 2, 1}; static const uint16_t i_d42e7b38b33e3d29[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d42e7b38b33e3d29 = { - 0xd42e7b38b33e3d29, b_d42e7b38b33e3d29.words, 69, d_d42e7b38b33e3d29, m_d42e7b38b33e3d29, + 0xd42e7b38b33e3d29, b_d42e7b38b33e3d29.words, 66, d_d42e7b38b33e3d29, m_d42e7b38b33e3d29, 1, 3, i_d42e7b38b33e3d29, nullptr, nullptr, { &s_d42e7b38b33e3d29, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<112> b_a000530ab1d17816 = { +static const ::capnp::_::AlignedData<109> b_a000530ab1d17816 = { { 0, 0, 0, 0, 5, 0, 6, 0, 22, 120, 209, 177, 10, 83, 0, 160, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 186, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 250, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -7473,26 +7278,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_a000530ab1d17816 { 16777217, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::ArraySchema>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_a000530ab1d17816 = { - 0xa000530ab1d17816, b_a000530ab1d17816.words, 112, d_a000530ab1d17816, m_a000530ab1d17816, + 0xa000530ab1d17816, b_a000530ab1d17816.words, 109, d_a000530ab1d17816, m_a000530ab1d17816, 3, 4, i_a000530ab1d17816, nullptr, nullptr, { &s_a000530ab1d17816, nullptr, bd_a000530ab1d17816, 0, sizeof(bd_a000530ab1d17816) / sizeof(bd_a000530ab1d17816[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_838188de0fd57580 = { +static const ::capnp::_::AlignedData<49> b_838188de0fd57580 = { { 0, 0, 0, 0, 5, 0, 6, 0, 128, 117, 213, 15, 222, 136, 129, 131, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, @@ -7540,26 +7342,23 @@ static const ::capnp::_::RawSchema* const d_838188de0fd57580[] = { static const uint16_t m_838188de0fd57580[] = {0, 1}; static const uint16_t i_838188de0fd57580[] = {0, 1}; const ::capnp::_::RawSchema s_838188de0fd57580 = { - 0x838188de0fd57580, b_838188de0fd57580.words, 52, d_838188de0fd57580, m_838188de0fd57580, + 0x838188de0fd57580, b_838188de0fd57580.words, 49, d_838188de0fd57580, m_838188de0fd57580, 2, 2, i_838188de0fd57580, nullptr, nullptr, { &s_838188de0fd57580, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<82> b_c41bcc7e8cc58f18 = { +static const ::capnp::_::AlignedData<79> b_c41bcc7e8cc58f18 = { { 0, 0, 0, 0, 5, 0, 6, 0, 24, 143, 197, 140, 126, 204, 27, 196, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 231, 0, 0, 0, + 29, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 77, @@ -7633,26 +7432,23 @@ static const ::capnp::_::AlignedData<82> b_c41bcc7e8cc58f18 = { static const uint16_t m_c41bcc7e8cc58f18[] = {3, 2, 1, 0}; static const uint16_t i_c41bcc7e8cc58f18[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_c41bcc7e8cc58f18 = { - 0xc41bcc7e8cc58f18, b_c41bcc7e8cc58f18.words, 82, nullptr, m_c41bcc7e8cc58f18, + 0xc41bcc7e8cc58f18, b_c41bcc7e8cc58f18.words, 79, nullptr, m_c41bcc7e8cc58f18, 0, 4, i_c41bcc7e8cc58f18, nullptr, nullptr, { &s_c41bcc7e8cc58f18, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<63> b_dcdd20e1b79e915a = { +static const ::capnp::_::AlignedData<60> b_dcdd20e1b79e915a = { { 0, 0, 0, 0, 5, 0, 6, 0, 90, 145, 158, 183, 225, 32, 221, 220, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 1, 0, 0, - 41, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 194, 0, 0, 0, + 29, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 0, @@ -7711,26 +7507,23 @@ static const ::capnp::_::RawSchema* const d_dcdd20e1b79e915a[] = { static const uint16_t m_dcdd20e1b79e915a[] = {0, 1}; static const uint16_t i_dcdd20e1b79e915a[] = {0, 1}; const ::capnp::_::RawSchema s_dcdd20e1b79e915a = { - 0xdcdd20e1b79e915a, b_dcdd20e1b79e915a.words, 63, d_dcdd20e1b79e915a, m_dcdd20e1b79e915a, + 0xdcdd20e1b79e915a, b_dcdd20e1b79e915a.words, 60, d_dcdd20e1b79e915a, m_dcdd20e1b79e915a, 2, 2, i_dcdd20e1b79e915a, nullptr, nullptr, { &s_dcdd20e1b79e915a, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<57> b_a2ea10c715b475c1 = { +static const ::capnp::_::AlignedData<54> b_a2ea10c715b475c1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 193, 117, 180, 21, 199, 16, 234, 162, - 48, 0, 0, 0, 1, 0, 0, 0, + 24, 0, 0, 0, 1, 0, 0, 0, 90, 145, 158, 183, 225, 32, 221, 220, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 46, @@ -7783,26 +7576,23 @@ static const ::capnp::_::RawSchema* const d_a2ea10c715b475c1[] = { static const uint16_t m_a2ea10c715b475c1[] = {0, 1}; static const uint16_t i_a2ea10c715b475c1[] = {0, 1}; const ::capnp::_::RawSchema s_a2ea10c715b475c1 = { - 0xa2ea10c715b475c1, b_a2ea10c715b475c1.words, 57, d_a2ea10c715b475c1, m_a2ea10c715b475c1, + 0xa2ea10c715b475c1, b_a2ea10c715b475c1.words, 54, d_a2ea10c715b475c1, m_a2ea10c715b475c1, 2, 2, i_a2ea10c715b475c1, nullptr, nullptr, { &s_a2ea10c715b475c1, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<67> b_c4e54a63294eddb7 = { +static const ::capnp::_::AlignedData<64> b_c4e54a63294eddb7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 183, 221, 78, 41, 99, 74, 229, 196, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 0, 0, 0, 119, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, @@ -7865,26 +7655,23 @@ static const ::capnp::_::RawSchema* const d_c4e54a63294eddb7[] = { static const uint16_t m_c4e54a63294eddb7[] = {0, 1}; static const uint16_t i_c4e54a63294eddb7[] = {0, 1}; const ::capnp::_::RawSchema s_c4e54a63294eddb7 = { - 0xc4e54a63294eddb7, b_c4e54a63294eddb7.words, 67, d_c4e54a63294eddb7, m_c4e54a63294eddb7, + 0xc4e54a63294eddb7, b_c4e54a63294eddb7.words, 64, d_c4e54a63294eddb7, m_c4e54a63294eddb7, 2, 2, i_c4e54a63294eddb7, nullptr, nullptr, { &s_c4e54a63294eddb7, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<85> b_83b01e46759bde40 = { +static const ::capnp::_::AlignedData<82> b_83b01e46759bde40 = { { 0, 0, 0, 0, 5, 0, 6, 0, 64, 222, 155, 117, 70, 30, 176, 131, - 54, 0, 0, 0, 1, 0, 0, 0, + 30, 0, 0, 0, 1, 0, 0, 0, 183, 221, 78, 41, 99, 74, 229, 196, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 74, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 85, @@ -7964,26 +7751,23 @@ static const ::capnp::_::RawSchema* const d_83b01e46759bde40[] = { static const uint16_t m_83b01e46759bde40[] = {1, 0}; static const uint16_t i_83b01e46759bde40[] = {0, 1}; const ::capnp::_::RawSchema s_83b01e46759bde40 = { - 0x83b01e46759bde40, b_83b01e46759bde40.words, 85, d_83b01e46759bde40, m_83b01e46759bde40, + 0x83b01e46759bde40, b_83b01e46759bde40.words, 82, d_83b01e46759bde40, m_83b01e46759bde40, 1, 2, i_83b01e46759bde40, nullptr, nullptr, { &s_83b01e46759bde40, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<67> b_fb7f36ad4d8ffe84 = { +static const ::capnp::_::AlignedData<64> b_fb7f36ad4d8ffe84 = { { 0, 0, 0, 0, 5, 0, 6, 0, 132, 254, 143, 77, 173, 54, 127, 251, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 61, 0, 0, 0, 119, 0, 0, 0, + 49, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, @@ -8046,26 +7830,23 @@ static const ::capnp::_::RawSchema* const d_fb7f36ad4d8ffe84[] = { static const uint16_t m_fb7f36ad4d8ffe84[] = {0, 1}; static const uint16_t i_fb7f36ad4d8ffe84[] = {0, 1}; const ::capnp::_::RawSchema s_fb7f36ad4d8ffe84 = { - 0xfb7f36ad4d8ffe84, b_fb7f36ad4d8ffe84.words, 67, d_fb7f36ad4d8ffe84, m_fb7f36ad4d8ffe84, + 0xfb7f36ad4d8ffe84, b_fb7f36ad4d8ffe84.words, 64, d_fb7f36ad4d8ffe84, m_fb7f36ad4d8ffe84, 2, 2, i_fb7f36ad4d8ffe84, nullptr, nullptr, { &s_fb7f36ad4d8ffe84, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<39> b_d5fd459ad75e86a9 = { +static const ::capnp::_::AlignedData<36> b_d5fd459ad75e86a9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 169, 134, 94, 215, 154, 69, 253, 213, - 54, 0, 0, 0, 1, 0, 0, 0, + 30, 0, 0, 0, 1, 0, 0, 0, 132, 254, 143, 77, 173, 54, 127, 251, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 74, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 138, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 63, 0, 0, 0, + 41, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 114, 111, 117, 112, 67, @@ -8096,26 +7877,23 @@ static const ::capnp::_::AlignedData<39> b_d5fd459ad75e86a9 = { static const uint16_t m_d5fd459ad75e86a9[] = {0}; static const uint16_t i_d5fd459ad75e86a9[] = {0}; const ::capnp::_::RawSchema s_d5fd459ad75e86a9 = { - 0xd5fd459ad75e86a9, b_d5fd459ad75e86a9.words, 39, nullptr, m_d5fd459ad75e86a9, + 0xd5fd459ad75e86a9, b_d5fd459ad75e86a9.words, 36, nullptr, m_d5fd459ad75e86a9, 0, 1, i_d5fd459ad75e86a9, nullptr, nullptr, { &s_d5fd459ad75e86a9, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<122> b_c0c730b5390f4427 = { +static const ::capnp::_::AlignedData<119> b_c0c730b5390f4427 = { { 0, 0, 0, 0, 5, 0, 6, 0, 39, 68, 15, 57, 181, 48, 199, 192, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 31, 1, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 71, 108, 111, 98, 97, 108, @@ -8238,26 +8016,23 @@ KJ_CONSTEXPR(const) ::capnp::_::RawBrandedSchema::Dependency bd_c0c730b5390f4427 { 16777220, ::tiledb::sm::serialization::capnp::Map< ::capnp::Text, ::tiledb::sm::serialization::capnp::MultiPartUploadState>::_capnpPrivate::brand() }, }; const ::capnp::_::RawSchema s_c0c730b5390f4427 = { - 0xc0c730b5390f4427, b_c0c730b5390f4427.words, 122, d_c0c730b5390f4427, m_c0c730b5390f4427, + 0xc0c730b5390f4427, b_c0c730b5390f4427.words, 119, d_c0c730b5390f4427, m_c0c730b5390f4427, 4, 5, i_c0c730b5390f4427, nullptr, nullptr, { &s_c0c730b5390f4427, nullptr, bd_c0c730b5390f4427, 0, sizeof(bd_c0c730b5390f4427) / sizeof(bd_c0c730b5390f4427[0]), nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<83> b_d8bd3c0dec37b773 = { +static const ::capnp::_::AlignedData<80> b_d8bd3c0dec37b773 = { { 0, 0, 0, 0, 5, 0, 6, 0, 115, 183, 55, 236, 13, 60, 189, 216, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 178, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 83, 105, 110, 103, 108, 101, @@ -8332,26 +8107,23 @@ static const ::capnp::_::AlignedData<83> b_d8bd3c0dec37b773 = { static const uint16_t m_d8bd3c0dec37b773[] = {0, 2, 1}; static const uint16_t i_d8bd3c0dec37b773[] = {0, 1, 2}; const ::capnp::_::RawSchema s_d8bd3c0dec37b773 = { - 0xd8bd3c0dec37b773, b_d8bd3c0dec37b773.words, 83, nullptr, m_d8bd3c0dec37b773, + 0xd8bd3c0dec37b773, b_d8bd3c0dec37b773.words, 80, nullptr, m_d8bd3c0dec37b773, 0, 3, i_d8bd3c0dec37b773, nullptr, nullptr, { &s_d8bd3c0dec37b773, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<615> b_cde352fc27e7ca61 = { +static const ::capnp::_::AlignedData<612> b_cde352fc27e7ca61 = { { 0, 0, 0, 0, 5, 0, 6, 0, 97, 202, 231, 39, 252, 82, 227, 205, - 42, 0, 0, 0, 1, 0, 4, 0, + 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, 22, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 1, 0, 0, - 49, 0, 0, 0, 23, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 65, 0, 0, 0, 95, 6, 0, 0, + 53, 0, 0, 0, 95, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -8962,26 +8734,23 @@ static const ::capnp::_::RawSchema* const d_cde352fc27e7ca61[] = { static const uint16_t m_cde352fc27e7ca61[] = {0, 2, 1, 19, 18, 21, 20, 3, 28, 27, 5, 4, 24, 25, 26, 6, 7, 14, 15, 12, 13, 17, 8, 16, 11, 9, 10, 23, 22}; static const uint16_t i_cde352fc27e7ca61[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28}; const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { - 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 615, d_cde352fc27e7ca61, m_cde352fc27e7ca61, + 0xcde352fc27e7ca61, b_cde352fc27e7ca61.words, 612, d_cde352fc27e7ca61, m_cde352fc27e7ca61, 2, 29, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<238> b_89aa8f4e88036b9e = { +static const ::capnp::_::AlignedData<235> b_89aa8f4e88036b9e = { { 0, 0, 0, 0, 5, 0, 6, 0, 158, 107, 3, 136, 78, 143, 170, 137, - 59, 0, 0, 0, 1, 0, 3, 0, + 35, 0, 0, 0, 1, 0, 3, 0, 97, 202, 231, 39, 252, 82, 227, 205, 8, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 114, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 178, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 111, 2, 0, 0, + 41, 0, 0, 0, 111, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 70, 114, 97, 103, 109, 101, @@ -9211,26 +8980,23 @@ static const ::capnp::_::AlignedData<238> b_89aa8f4e88036b9e = { static const uint16_t m_89aa8f4e88036b9e[] = {9, 10, 0, 6, 5, 8, 1, 7, 4, 2, 3}; static const uint16_t i_89aa8f4e88036b9e[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; const ::capnp::_::RawSchema s_89aa8f4e88036b9e = { - 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 238, nullptr, m_89aa8f4e88036b9e, + 0x89aa8f4e88036b9e, b_89aa8f4e88036b9e.words, 235, nullptr, m_89aa8f4e88036b9e, 0, 11, i_89aa8f4e88036b9e, nullptr, nullptr, { &s_89aa8f4e88036b9e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<109> b_d492b6734d5e3bf5 = { +static const ::capnp::_::AlignedData<106> b_d492b6734d5e3bf5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 59, 94, 77, 115, 182, 146, 212, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 250, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 58, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 31, 1, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 77, 117, 108, 116, 105, 80, @@ -9335,26 +9101,23 @@ static const ::capnp::_::RawSchema* const d_d492b6734d5e3bf5[] = { static const uint16_t m_d492b6734d5e3bf5[] = {4, 3, 0, 2, 1}; static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_d492b6734d5e3bf5 = { - 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 109, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, + 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 106, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, 2, 5, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_bde8ebd7b13d8625 = { +static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { { 0, 0, 0, 0, 5, 0, 6, 0, 37, 134, 61, 177, 215, 235, 232, 189, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 109, 112, 108, 101, @@ -9398,26 +9161,23 @@ static const ::capnp::_::AlignedData<52> b_bde8ebd7b13d8625 = { static const uint16_t m_bde8ebd7b13d8625[] = {0, 1}; static const uint16_t i_bde8ebd7b13d8625[] = {0, 1}; const ::capnp::_::RawSchema s_bde8ebd7b13d8625 = { - 0xbde8ebd7b13d8625, b_bde8ebd7b13d8625.words, 52, nullptr, m_bde8ebd7b13d8625, + 0xbde8ebd7b13d8625, b_bde8ebd7b13d8625.words, 49, nullptr, m_bde8ebd7b13d8625, 0, 2, i_bde8ebd7b13d8625, nullptr, nullptr, { &s_bde8ebd7b13d8625, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<57> b_a736c51d292ca752 = { +static const ::capnp::_::AlignedData<54> b_a736c51d292ca752 = { { 0, 0, 0, 0, 5, 0, 6, 0, 82, 167, 44, 41, 29, 197, 54, 167, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 242, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 50, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 87, 114, 105, 116, 116, 101, @@ -9466,26 +9226,23 @@ static const ::capnp::_::AlignedData<57> b_a736c51d292ca752 = { static const uint16_t m_a736c51d292ca752[] = {1, 0}; static const uint16_t i_a736c51d292ca752[] = {0, 1}; const ::capnp::_::RawSchema s_a736c51d292ca752 = { - 0xa736c51d292ca752, b_a736c51d292ca752.words, 57, nullptr, m_a736c51d292ca752, + 0xa736c51d292ca752, b_a736c51d292ca752.words, 54, nullptr, m_a736c51d292ca752, 0, 2, i_a736c51d292ca752, nullptr, nullptr, { &s_a736c51d292ca752, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<51> b_cd8abc9dabc4b03f = { +static const ::capnp::_::AlignedData<48> b_cd8abc9dabc4b03f = { { 0, 0, 0, 0, 5, 0, 6, 0, 63, 176, 196, 171, 157, 188, 138, 205, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 194, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 2, 1, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 66, 117, 102, 102, 101, 114, @@ -9528,26 +9285,23 @@ static const ::capnp::_::AlignedData<51> b_cd8abc9dabc4b03f = { static const uint16_t m_cd8abc9dabc4b03f[] = {1, 0}; static const uint16_t i_cd8abc9dabc4b03f[] = {0, 1}; const ::capnp::_::RawSchema s_cd8abc9dabc4b03f = { - 0xcd8abc9dabc4b03f, b_cd8abc9dabc4b03f.words, 51, nullptr, m_cd8abc9dabc4b03f, + 0xcd8abc9dabc4b03f, b_cd8abc9dabc4b03f.words, 48, nullptr, m_cd8abc9dabc4b03f, 0, 2, i_cd8abc9dabc4b03f, nullptr, nullptr, { &s_cd8abc9dabc4b03f, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<58> b_cfea684b4bcd0721 = { +static const ::capnp::_::AlignedData<55> b_cfea684b4bcd0721 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 7, 205, 75, 75, 104, 234, 207, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 82, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 146, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 119, 0, 0, 0, + 41, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -9600,26 +9354,23 @@ static const ::capnp::_::RawSchema* const d_cfea684b4bcd0721[] = { static const uint16_t m_cfea684b4bcd0721[] = {0, 1}; static const uint16_t i_cfea684b4bcd0721[] = {0, 1}; const ::capnp::_::RawSchema s_cfea684b4bcd0721 = { - 0xcfea684b4bcd0721, b_cfea684b4bcd0721.words, 58, d_cfea684b4bcd0721, m_cfea684b4bcd0721, + 0xcfea684b4bcd0721, b_cfea684b4bcd0721.words, 55, d_cfea684b4bcd0721, m_cfea684b4bcd0721, 1, 2, i_cfea684b4bcd0721, nullptr, nullptr, { &s_cfea684b4bcd0721, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<71> b_aaeeafe1e9f3ea1c = { +static const ::capnp::_::AlignedData<68> b_aaeeafe1e9f3ea1c = { { 0, 0, 0, 0, 5, 0, 6, 0, 28, 234, 243, 233, 225, 175, 238, 170, - 42, 0, 0, 0, 1, 0, 2, 0, + 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 130, 2, 0, 0, - 57, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 194, 1, 0, 0, + 45, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 0, 0, 0, 175, 0, 0, 0, + 41, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 68, @@ -9685,26 +9436,23 @@ static const ::capnp::_::RawSchema* const d_aaeeafe1e9f3ea1c[] = { static const uint16_t m_aaeeafe1e9f3ea1c[] = {0, 2, 1}; static const uint16_t i_aaeeafe1e9f3ea1c[] = {0, 1, 2}; const ::capnp::_::RawSchema s_aaeeafe1e9f3ea1c = { - 0xaaeeafe1e9f3ea1c, b_aaeeafe1e9f3ea1c.words, 71, d_aaeeafe1e9f3ea1c, m_aaeeafe1e9f3ea1c, + 0xaaeeafe1e9f3ea1c, b_aaeeafe1e9f3ea1c.words, 68, d_aaeeafe1e9f3ea1c, m_aaeeafe1e9f3ea1c, 1, 3, i_aaeeafe1e9f3ea1c, nullptr, nullptr, { &s_aaeeafe1e9f3ea1c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<38> b_f5a35661031194d2 = { +static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 210, 148, 17, 3, 97, 86, 163, 245, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 34, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 98, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 67, @@ -9737,26 +9485,23 @@ static const ::capnp::_::RawSchema* const d_f5a35661031194d2[] = { static const uint16_t m_f5a35661031194d2[] = {0}; static const uint16_t i_f5a35661031194d2[] = {0}; const ::capnp::_::RawSchema s_f5a35661031194d2 = { - 0xf5a35661031194d2, b_f5a35661031194d2.words, 38, d_f5a35661031194d2, m_f5a35661031194d2, + 0xf5a35661031194d2, b_f5a35661031194d2.words, 35, d_f5a35661031194d2, m_f5a35661031194d2, 1, 1, i_f5a35661031194d2, nullptr, nullptr, { &s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<37> b_e68edfc0939e63df = { +static const ::capnp::_::AlignedData<34> b_e68edfc0939e63df = { { 0, 0, 0, 0, 5, 0, 6, 0, 223, 99, 158, 147, 192, 223, 142, 230, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 234, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 42, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 114, 114, 97, 121, 86, @@ -9788,26 +9533,23 @@ static const ::capnp::_::RawSchema* const d_e68edfc0939e63df[] = { static const uint16_t m_e68edfc0939e63df[] = {0}; static const uint16_t i_e68edfc0939e63df[] = {0}; const ::capnp::_::RawSchema s_e68edfc0939e63df = { - 0xe68edfc0939e63df, b_e68edfc0939e63df.words, 37, d_e68edfc0939e63df, m_e68edfc0939e63df, + 0xe68edfc0939e63df, b_e68edfc0939e63df.words, 34, d_e68edfc0939e63df, m_e68edfc0939e63df, 1, 1, i_e68edfc0939e63df, nullptr, nullptr, { &s_e68edfc0939e63df, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<58> b_891a70a671f15cf6 = { +static const ::capnp::_::AlignedData<55> b_891a70a671f15cf6 = { { 0, 0, 0, 0, 5, 0, 6, 0, 246, 92, 241, 113, 166, 112, 26, 137, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 18, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 82, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, @@ -9860,26 +9602,23 @@ static const ::capnp::_::RawSchema* const d_891a70a671f15cf6[] = { static const uint16_t m_891a70a671f15cf6[] = {0, 1}; static const uint16_t i_891a70a671f15cf6[] = {0, 1}; const ::capnp::_::RawSchema s_891a70a671f15cf6 = { - 0x891a70a671f15cf6, b_891a70a671f15cf6.words, 58, d_891a70a671f15cf6, m_891a70a671f15cf6, + 0x891a70a671f15cf6, b_891a70a671f15cf6.words, 55, d_891a70a671f15cf6, m_891a70a671f15cf6, 1, 2, i_891a70a671f15cf6, nullptr, nullptr, { &s_891a70a671f15cf6, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<43> b_805c080c10c1e959 = { +static const ::capnp::_::AlignedData<40> b_805c080c10c1e959 = { { 0, 0, 0, 0, 5, 0, 6, 0, 89, 233, 193, 16, 12, 8, 92, 128, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 90, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 69, 110, @@ -9917,26 +9656,23 @@ static const ::capnp::_::RawSchema* const d_805c080c10c1e959[] = { static const uint16_t m_805c080c10c1e959[] = {0}; static const uint16_t i_805c080c10c1e959[] = {0}; const ::capnp::_::RawSchema s_805c080c10c1e959 = { - 0x805c080c10c1e959, b_805c080c10c1e959.words, 43, d_805c080c10c1e959, m_805c080c10c1e959, + 0x805c080c10c1e959, b_805c080c10c1e959.words, 40, d_805c080c10c1e959, m_805c080c10c1e959, 1, 1, i_805c080c10c1e959, nullptr, nullptr, { &s_805c080c10c1e959, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<55> b_83f094010132ff21 = { +static const ::capnp::_::AlignedData<52> b_83f094010132ff21 = { { 0, 0, 0, 0, 5, 0, 6, 0, 33, 255, 50, 1, 1, 148, 240, 131, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 10, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 74, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, @@ -9986,26 +9722,23 @@ static const ::capnp::_::RawSchema* const d_83f094010132ff21[] = { static const uint16_t m_83f094010132ff21[] = {0, 1}; static const uint16_t i_83f094010132ff21[] = {0, 1}; const ::capnp::_::RawSchema s_83f094010132ff21 = { - 0x83f094010132ff21, b_83f094010132ff21.words, 55, d_83f094010132ff21, m_83f094010132ff21, + 0x83f094010132ff21, b_83f094010132ff21.words, 52, d_83f094010132ff21, m_83f094010132ff21, 1, 2, i_83f094010132ff21, nullptr, nullptr, { &s_83f094010132ff21, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<38> b_ebe17f59ac9a1df1 = { +static const ::capnp::_::AlignedData<35> b_ebe17f59ac9a1df1 = { { 0, 0, 0, 0, 5, 0, 6, 0, 241, 29, 154, 172, 89, 127, 225, 235, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 18, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 82, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 76, 111, 97, 100, 65, 114, @@ -10038,26 +9771,23 @@ static const ::capnp::_::RawSchema* const d_ebe17f59ac9a1df1[] = { static const uint16_t m_ebe17f59ac9a1df1[] = {0}; static const uint16_t i_ebe17f59ac9a1df1[] = {0}; const ::capnp::_::RawSchema s_ebe17f59ac9a1df1 = { - 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 38, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, + 0xebe17f59ac9a1df1, b_ebe17f59ac9a1df1.words, 35, d_ebe17f59ac9a1df1, m_ebe17f59ac9a1df1, 1, 1, i_ebe17f59ac9a1df1, nullptr, nullptr, { &s_ebe17f59ac9a1df1, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<52> b_e06f571aa93eb314 = { +static const ::capnp::_::AlignedData<49> b_e06f571aa93eb314 = { { 0, 0, 0, 0, 5, 0, 6, 0, 20, 179, 62, 169, 26, 87, 111, 224, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 218, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 26, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 119, 0, 0, 0, + 33, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 80, @@ -10105,26 +9835,23 @@ static const ::capnp::_::RawSchema* const d_e06f571aa93eb314[] = { static const uint16_t m_e06f571aa93eb314[] = {0, 1}; static const uint16_t i_e06f571aa93eb314[] = {0, 1}; const ::capnp::_::RawSchema s_e06f571aa93eb314 = { - 0xe06f571aa93eb314, b_e06f571aa93eb314.words, 52, d_e06f571aa93eb314, m_e06f571aa93eb314, + 0xe06f571aa93eb314, b_e06f571aa93eb314.words, 49, d_e06f571aa93eb314, m_e06f571aa93eb314, 2, 2, i_e06f571aa93eb314, nullptr, nullptr, { &s_e06f571aa93eb314, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<38> b_9fd8fc2f462b2d06 = { +static const ::capnp::_::AlignedData<35> b_9fd8fc2f462b2d06 = { { 0, 0, 0, 0, 5, 0, 6, 0, 6, 45, 43, 70, 47, 252, 216, 159, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 226, 1, 0, 0, - 49, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 34, 1, 0, 0, + 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 80, @@ -10135,12 +9862,12 @@ static const ::capnp::_::AlignedData<38> b_9fd8fc2f462b2d06 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 82, 0, 0, 0, + 13, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 3, 0, 1, 0, 24, 0, 0, 0, 2, 0, 1, 0, - 113, 117, 101, 114, 121, 80, 108, 97, - 110, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 114, 97, 116, 101, 103, 121, + 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10154,26 +9881,23 @@ static const ::capnp::_::AlignedData<38> b_9fd8fc2f462b2d06 = { static const uint16_t m_9fd8fc2f462b2d06[] = {0}; static const uint16_t i_9fd8fc2f462b2d06[] = {0}; const ::capnp::_::RawSchema s_9fd8fc2f462b2d06 = { - 0x9fd8fc2f462b2d06, b_9fd8fc2f462b2d06.words, 38, nullptr, m_9fd8fc2f462b2d06, + 0x9fd8fc2f462b2d06, b_9fd8fc2f462b2d06.words, 35, nullptr, m_9fd8fc2f462b2d06, 0, 1, i_9fd8fc2f462b2d06, nullptr, nullptr, { &s_9fd8fc2f462b2d06, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<54> b_8965edf5597ce627 = { +static const ::capnp::_::AlignedData<51> b_8965edf5597ce627 = { { 0, 0, 0, 0, 5, 0, 6, 0, 39, 230, 124, 89, 245, 237, 101, 137, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 26, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 90, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 119, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 115, 111, 108, @@ -10222,26 +9946,23 @@ static const ::capnp::_::RawSchema* const d_8965edf5597ce627[] = { static const uint16_t m_8965edf5597ce627[] = {0, 1}; static const uint16_t i_8965edf5597ce627[] = {0, 1}; const ::capnp::_::RawSchema s_8965edf5597ce627 = { - 0x8965edf5597ce627, b_8965edf5597ce627.words, 54, d_8965edf5597ce627, m_8965edf5597ce627, + 0x8965edf5597ce627, b_8965edf5597ce627.words, 51, d_8965edf5597ce627, m_8965edf5597ce627, 1, 2, i_8965edf5597ce627, nullptr, nullptr, { &s_8965edf5597ce627, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<48> b_aac8bf9b5211388b = { +static const ::capnp::_::AlignedData<45> b_aac8bf9b5211388b = { { 0, 0, 0, 0, 5, 0, 6, 0, 139, 56, 17, 82, 155, 191, 200, 170, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 34, 2, 0, 0, - 53, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 98, 1, 0, 0, + 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 49, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 67, 111, 110, 115, 111, 108, @@ -10281,26 +10002,23 @@ static const ::capnp::_::AlignedData<48> b_aac8bf9b5211388b = { static const uint16_t m_aac8bf9b5211388b[] = {0}; static const uint16_t i_aac8bf9b5211388b[] = {0}; const ::capnp::_::RawSchema s_aac8bf9b5211388b = { - 0xaac8bf9b5211388b, b_aac8bf9b5211388b.words, 48, nullptr, m_aac8bf9b5211388b, + 0xaac8bf9b5211388b, b_aac8bf9b5211388b.words, 45, nullptr, m_aac8bf9b5211388b, 0, 1, i_aac8bf9b5211388b, nullptr, nullptr, { &s_aac8bf9b5211388b, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<56> b_ca2d4d0bfe4ae5d9 = { +static const ::capnp::_::AlignedData<53> b_ca2d4d0bfe4ae5d9 = { { 0, 0, 0, 0, 5, 0, 6, 0, 217, 229, 74, 254, 11, 77, 45, 202, - 42, 0, 0, 0, 1, 0, 1, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 186, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 250, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 119, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 67, @@ -10351,26 +10069,23 @@ static const ::capnp::_::RawSchema* const d_ca2d4d0bfe4ae5d9[] = { static const uint16_t m_ca2d4d0bfe4ae5d9[] = {1, 0}; static const uint16_t i_ca2d4d0bfe4ae5d9[] = {0, 1}; const ::capnp::_::RawSchema s_ca2d4d0bfe4ae5d9 = { - 0xca2d4d0bfe4ae5d9, b_ca2d4d0bfe4ae5d9.words, 56, d_ca2d4d0bfe4ae5d9, m_ca2d4d0bfe4ae5d9, + 0xca2d4d0bfe4ae5d9, b_ca2d4d0bfe4ae5d9.words, 53, d_ca2d4d0bfe4ae5d9, m_ca2d4d0bfe4ae5d9, 1, 2, i_ca2d4d0bfe4ae5d9, nullptr, nullptr, { &s_ca2d4d0bfe4ae5d9, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<68> b_e193f1f45a9f102e = { +static const ::capnp::_::AlignedData<65> b_e193f1f45a9f102e = { { 0, 0, 0, 0, 5, 0, 6, 0, 46, 16, 159, 90, 244, 241, 147, 225, - 42, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 21, 0, 0, 0, 162, 1, 0, 0, - 45, 0, 0, 0, 7, 0, 0, 0, + 21, 0, 0, 0, 226, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 41, 0, 0, 0, 175, 0, 0, 0, + 29, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 116, 105, 108, 101, 100, 98, 47, 115, - 109, 47, 115, 101, 114, 105, 97, 108, - 105, 122, 97, 116, 105, 111, 110, 47, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 65, 103, 103, 114, 101, 103, @@ -10430,7 +10145,7 @@ static const ::capnp::_::AlignedData<68> b_e193f1f45a9f102e = { static const uint16_t m_e193f1f45a9f102e[] = {1, 2, 0}; static const uint16_t i_e193f1f45a9f102e[] = {0, 1, 2}; const ::capnp::_::RawSchema s_e193f1f45a9f102e = { - 0xe193f1f45a9f102e, b_e193f1f45a9f102e.words, 68, nullptr, m_e193f1f45a9f102e, + 0xe193f1f45a9f102e, b_e193f1f45a9f102e.words, 65, nullptr, m_e193f1f45a9f102e, 0, 3, i_e193f1f45a9f102e, nullptr, nullptr, { &s_e193f1f45a9f102e, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 7463097f0f4c..4d493a76910c 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -15140,8 +15140,8 @@ class QueryPlanResponse::Reader { } #endif // !CAPNP_LITE - inline bool hasQueryPlan() const; - inline ::capnp::Text::Reader getQueryPlan() const; + inline bool hasStrategy() const; + inline ::capnp::Text::Reader getStrategy() const; private: ::capnp::_::StructReader _reader; @@ -15182,12 +15182,12 @@ class QueryPlanResponse::Builder { } #endif // !CAPNP_LITE - inline bool hasQueryPlan(); - inline ::capnp::Text::Builder getQueryPlan(); - inline void setQueryPlan(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initQueryPlan(unsigned int size); - inline void adoptQueryPlan(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownQueryPlan(); + inline bool hasStrategy(); + inline ::capnp::Text::Builder getStrategy(); + inline void setStrategy(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initStrategy(unsigned int size); + inline void adoptStrategy(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownStrategy(); private: ::capnp::_::StructBuilder _builder; @@ -32672,42 +32672,42 @@ QueryPlanRequest::Builder::disownQuery() { _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } -inline bool QueryPlanResponse::Reader::hasQueryPlan() const { +inline bool QueryPlanResponse::Reader::hasStrategy() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline bool QueryPlanResponse::Builder::hasQueryPlan() { +inline bool QueryPlanResponse::Builder::hasStrategy() { return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader QueryPlanResponse::Reader::getQueryPlan() const { +inline ::capnp::Text::Reader QueryPlanResponse::Reader::getStrategy() const { return ::capnp::_::PointerHelpers<::capnp::Text>::get( _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline ::capnp::Text::Builder QueryPlanResponse::Builder::getQueryPlan() { +inline ::capnp::Text::Builder QueryPlanResponse::Builder::getStrategy() { return ::capnp::_::PointerHelpers<::capnp::Text>::get( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void QueryPlanResponse::Builder::setQueryPlan( +inline void QueryPlanResponse::Builder::setStrategy( ::capnp::Text::Reader value) { ::capnp::_::PointerHelpers<::capnp::Text>::set( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), value); } -inline ::capnp::Text::Builder QueryPlanResponse::Builder::initQueryPlan( +inline ::capnp::Text::Builder QueryPlanResponse::Builder::initStrategy( unsigned int size) { return ::capnp::_::PointerHelpers<::capnp::Text>::init( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), size); } -inline void QueryPlanResponse::Builder::adoptQueryPlan( +inline void QueryPlanResponse::Builder::adoptStrategy( ::capnp::Orphan<::capnp::Text>&& value) { ::capnp::_::PointerHelpers<::capnp::Text>::adopt( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::Text> -QueryPlanResponse::Builder::disownQueryPlan() { +QueryPlanResponse::Builder::disownStrategy() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } From 8594341a444ca475705babf6c000e183107f52d3 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 16 Jan 2024 15:38:05 -0600 Subject: [PATCH 145/456] Fix doxygen build step. (#4639) Fix doxygen build step. --- TYPE: NO_HISTORY DESC: Fix doxygen build step. --- tiledb/api/c_api/config/config_api_external.h | 2 +- tiledb/doxygen/local-build.sh | 2 +- tiledb/doxygen/source/conf.py | 2 +- tiledb/doxygen/source/requirements.txt | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index cbbcaec71f85..3401f8ebbc6b 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -566,7 +566,7 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * When set to `true`, the S3 SDK uses a handler that ignores SIGPIPE * signals. * **Default**: "true" - * - `vfs.hdfs.name_node_uri"`
+ * - `vfs.hdfs.name_node_uri`
* Name node for HDFS.
* **Default**: "" * - `vfs.hdfs.username`
diff --git a/tiledb/doxygen/local-build.sh b/tiledb/doxygen/local-build.sh index 827904d45f2c..c207d4eab52e 100755 --- a/tiledb/doxygen/local-build.sh +++ b/tiledb/doxygen/local-build.sh @@ -92,7 +92,7 @@ build_site() { # Note: # -E disables the build cache (slower builds). # -W enables warnings as errors. - sphinx-build -E -W -T -b html -d ${build_dir}/doctrees -D language=en ${source_dir} ${build_dir}/html ${BUILD_DIR_ARG:-} || \ + sphinx-build -E -W -T -b html -d ${build_dir}/doctrees ${source_dir} ${build_dir}/html ${BUILD_DIR_ARG:-} || \ die "could not build sphinx site" } diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index b1523967ee46..af509ced1fe9 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -107,7 +107,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/tiledb/doxygen/source/requirements.txt b/tiledb/doxygen/source/requirements.txt index 926e0f8c4177..d759deff6cde 100644 --- a/tiledb/doxygen/source/requirements.txt +++ b/tiledb/doxygen/source/requirements.txt @@ -1,6 +1,6 @@ -breathe==4.26.1 -sphinxcontrib-contentui==0.2.5 -sphinx_rtd_theme==0.5.0 -docutils < 0.18 # pin pending https://github.com/sphinx-doc/sphinx/issues/9777 -cmake>=3.21 -jinja2<3.1 # pinned due to incompatibility with sphinx 3.4 +breathe +cmake +docutils +jinja2 +sphinx_rtd_theme +sphinxcontrib-contentui From 4498063a30fa36f4f02323540e22211e3b2750e4 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 17 Jan 2024 18:11:38 +0200 Subject: [PATCH 146/456] Pass AWS client configuration to `STSProfileWithWebIdentityCredentialsProvider`. (#4641) This PR applies a similar fix to #4616 but for `STSProfileWithWebIdentityCredentialsProvider`. That class was also adjusted to prevent a memory leak, to use the user-provided `STSClient` if available, and to use public APIs. The issue has been validated to be fixed. --- TYPE: BUG DESC: Fix HTTP requests for AWS assume role with web identity not honoring config options. --- tiledb/sm/filesystem/s3.cc | 9 ++- ...ofileWithWebIdentityCredentialsProvider.cc | 74 ++++++++++--------- ...rofileWithWebIdentityCredentialsProvider.h | 10 ++- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index 3af7b981c08e..c4611d796c39 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -1421,7 +1421,14 @@ Status S3::init_client() const { } case 16: { credentials_provider_ = make_shared< - Aws::Auth::STSProfileWithWebIdentityCredentialsProvider>(HERE()); + Aws::Auth::STSProfileWithWebIdentityCredentialsProvider>( + HERE(), + Aws::Auth::GetConfigProfileName(), + std::chrono::minutes(60), + [client_config](const auto& credentials) { + return make_shared( + HERE(), credentials, client_config); + }); break; } default: { diff --git a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc index 07b14ff9fcd2..35c08b93259b 100644 --- a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc +++ b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc @@ -48,6 +48,7 @@ #include #include #include +#include #include @@ -77,8 +78,8 @@ STSProfileWithWebIdentityCredentialsProvider:: STSProfileWithWebIdentityCredentialsProvider( const Aws::String& profileName, std::chrono::minutes duration, - const std::function& - stsClientFactory) + const std::function( + const AWSCredentials&)>& stsClientFactory) : m_profileName(profileName) , m_duration(duration) , m_reloadFrequency( @@ -430,27 +431,22 @@ STSProfileWithWebIdentityCredentialsProvider::GetCredentialsFromSTS( const Aws::String& externalID) { using namespace Aws::STS::Model; if (m_stsClientFactory) { - return GetCredentialsFromSTSInternal( - roleArn, externalID, m_stsClientFactory(credentials)); + auto client = m_stsClientFactory(credentials); + return GetCredentialsFromSTSInternal(roleArn, externalID, client.get()); } Aws::STS::STSClient stsClient{credentials}; return GetCredentialsFromSTSInternal(roleArn, externalID, &stsClient); } -AWSCredentials -STSProfileWithWebIdentityCredentialsProvider::GetCredentialsFromWebIdentity( - const Config::Profile& profile) { +AWSCredentials STSProfileWithWebIdentityCredentialsProvider:: + GetCredentialsFromWebIdentityInternal( + const Config::Profile& profile, Aws::STS::STSClient* client) { + using namespace Aws::STS::Model; const Aws::String& m_roleArn = profile.GetRoleArn(); Aws::String m_tokenFile = profile.GetValue("web_identity_token_file"); Aws::String m_sessionName = profile.GetValue("role_session_name"); - auto tmpRegion = profile.GetRegion(); - if (tmpRegion.empty()) { - // Set same default as STSAssumeRoleWebIdentityCredentialsProvider - tmpRegion = Aws::Region::US_EAST_1; - } - if (m_sessionName.empty()) { m_sessionName = Aws::Utils::UUID::RandomUUID(); } @@ -467,30 +463,40 @@ STSProfileWithWebIdentityCredentialsProvider::GetCredentialsFromWebIdentity( return {}; } - Internal::STSCredentialsClient::STSAssumeRoleWithWebIdentityRequest request{ - m_sessionName, m_roleArn, m_token}; - - Aws::Client::ClientConfiguration config; - config.scheme = Aws::Http::Scheme::HTTPS; - config.region = tmpRegion; + AssumeRoleWithWebIdentityRequest request; + request.SetRoleArn(m_roleArn); + request.SetRoleSessionName(m_sessionName); + request.SetWebIdentityToken(m_token); - Aws::Vector retryableErrors; - retryableErrors.push_back("IDPCommunicationError"); - retryableErrors.push_back("InvalidIdentityToken"); - - config.retryStrategy = - Aws::MakeShared( - CLASS_TAG, retryableErrors, 3 /*maxRetries*/); + auto outcome = client->AssumeRoleWithWebIdentity(request); + if (outcome.IsSuccess()) { + const auto& modelCredentials = outcome.GetResult().GetCredentials(); + AWS_LOGSTREAM_TRACE( + CLASS_TAG, + "Successfully retrieved credentials with AWS_ACCESS_KEY: " + << modelCredentials.GetAccessKeyId()); + return { + modelCredentials.GetAccessKeyId(), + modelCredentials.GetSecretAccessKey(), + modelCredentials.GetSessionToken(), + modelCredentials.GetExpiration()}; + } else { + AWS_LOGSTREAM_ERROR(CLASS_TAG, "failed to assume role" << m_roleArn); + } + return {}; +} - auto m_client = - Aws::MakeUnique(CLASS_TAG, config); - auto result = m_client->GetAssumeRoleWithWebIdentityCredentials(request); - AWS_LOGSTREAM_TRACE( - CLASS_TAG, - "Successfully retrieved credentials with AWS_ACCESS_KEY: " - << result.creds.GetAWSAccessKeyId()); +AWSCredentials +STSProfileWithWebIdentityCredentialsProvider::GetCredentialsFromWebIdentity( + const Config::Profile& profile) { + using namespace Aws::STS::Model; + if (m_stsClientFactory) { + auto client = m_stsClientFactory({}); + return GetCredentialsFromWebIdentityInternal(profile, client.get()); + } - return result.creds; + Aws::STS::STSClient stsClient{AWSCredentials{}}; + return GetCredentialsFromWebIdentityInternal(profile, &stsClient); } #endif // HAVE_S3s \ No newline at end of file diff --git a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h index 4c0678cb94c0..d50185af1293 100644 --- a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h +++ b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h @@ -97,8 +97,8 @@ class /* AWS_IDENTITY_MANAGEMENT_API */ STSProfileWithWebIdentityCredentialsProvider( const Aws::String& profileName, std::chrono::minutes duration, - const std::function& - stsClientFactory); + const std::function( + const AWSCredentials&)>& stsClientFactory); /** * Fetches the credentials set from STS following the rules defined in the @@ -132,11 +132,15 @@ class /* AWS_IDENTITY_MANAGEMENT_API */ const Aws::String& externalID, Aws::STS::STSClient* client); + AWSCredentials GetCredentialsFromWebIdentityInternal( + const Config::Profile& profile, Aws::STS::STSClient* client); + Aws::String m_profileName; AWSCredentials m_credentials; const std::chrono::minutes m_duration; const std::chrono::milliseconds m_reloadFrequency; - std::function m_stsClientFactory; + std::function(const AWSCredentials&)> + m_stsClientFactory; }; } // namespace Auth } // namespace Aws From 08d27747a0a2ada59d2e5c1b2ea78985ed04d7bf Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 18 Jan 2024 04:47:55 -0500 Subject: [PATCH 147/456] Add v2_19_1 arrays to backward compatibility matrix (#4636) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 8693873d8436..d36324cf1ec8 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 0da5c1a84d23c76eaa80c82e87990c1084c6aed2 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 19 Jan 2024 18:21:46 +0100 Subject: [PATCH 148/456] Make sure buffer is large enough for fragment consolidation buffers. (#4632) --- TYPE: NO_HISTORY DESC: Make sure buffer is large enough for fragment consolidation buffers. --- .../sm/consolidator/fragment_consolidator.cc | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index eede94d2390b..8cf8a0a6aa62 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -136,28 +136,40 @@ void FragmentConsolidationWorkspace::resize_buffers( total_budget = config.buffer_size_ * buffer_num; } - // Make sure the buffers and sizes vectors are large enough. - if (buffer_num > buffers_.size()) { - buffers_.resize(buffer_num); - sizes_.resize(buffer_num); - } - - // Create buffers. + // Calculate the size of individual buffers by assigning a weight based + // percentage of the total buffer size. auto total_weights = std::accumulate( buffer_weights.begin(), buffer_weights.end(), static_cast(0)); - // Allocate space for each buffer. uint64_t adjusted_budget = total_budget / total_weights * total_weights; - if (std::max(1, adjusted_budget) > backing_buffer_.size()) { - backing_buffer_.resize(std::max(1, adjusted_budget)); + if (buffer_num > buffers_.size()) { + sizes_.resize(buffer_num); } - // Finally allocate spans for the buffers. size_t offset = 0; for (unsigned i = 0; i < buffer_num; ++i) { sizes_[i] = std::max( 1, adjusted_budget * buffer_weights[i] / total_weights); + offset += sizes_[i]; + } + + // Total size is the final offset value + size_t final_budget = offset; + + // Ensure that our backing buffer is large enough to reference the final + // budget number of bytes. + if (final_budget > backing_buffer_.size()) { + backing_buffer_.resize(final_budget); + } + + // Finally, update our spans referencing the backing buffer. + if (buffer_num > buffers_.size()) { + buffers_.resize(buffer_num); + } + + offset = 0; + for (unsigned i = 0; i < buffer_num; ++i) { buffers_[i] = span(&(backing_buffer_[offset]), sizes_[i]); offset += sizes_[i]; } From e1b22b366c66061582e323146ebc94f2b574b760 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 22 Jan 2024 12:59:10 +0100 Subject: [PATCH 149/456] Bump dev version to 2.21.0. (#4656) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.21.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index af509ced1fe9..b75e2e30713b 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.20' +version = '2.21' # The full version, including alpha/beta/rc tags. -release = '2.20.0' +release = '2.21.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index ef785a9bf2df..91b07fb241d8 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 20 +#define TILEDB_VERSION_MINOR 21 #define TILEDB_VERSION_PATCH 0 From 6f0a21129e8a4f7383a4bc42a9d84ee3e8c4c2b1 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:49:11 +0100 Subject: [PATCH 150/456] Update HISTORY for 2.20. (#4655) --- TYPE: NO_HISTORY DESC: Update HISTORY for 2.20. --------- Co-authored-by: Theodore Tsirpanis --- HISTORY.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 37e4704ddb30..686ccd951c69 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,48 @@ +# TileDB v2.20.0 Release Notes + +## Breaking behavior + +* The minimum supported macOS version on x64 is 11. [#4561](https://github.com/TileDB-Inc/TileDB/pull/4561) + +## New features + +* Add internal definitions for S3 ls recursive [#4467](https://github.com/TileDB-Inc/TileDB/pull/4467) +* Add experimental C API for ls_recursive. [#4615](https://github.com/TileDB-Inc/TileDB/pull/4615) +* Add experimental CPP API for ls_recursive. [#4625](https://github.com/TileDB-Inc/TileDB/pull/4625) + +## Improvements + +* Keep opened array resources alive until query ends. [#4601](https://github.com/TileDB-Inc/TileDB/pull/4601) +* Avoid needless buffer reallocations when consolidating fragments. [#4614](https://github.com/TileDB-Inc/TileDB/pull/4614) +* Update the AWS SDK to version 1.11.215. [#4551](https://github.com/TileDB-Inc/TileDB/pull/4551) +* Fix consolidation plan to print relative paths in output. [#4604](https://github.com/TileDB-Inc/TileDB/pull/4604) +* Fix traversal limit in array deserialization. [#4606](https://github.com/TileDB-Inc/TileDB/pull/4606) +* Add function random_label to utilize PRNG for random string generation. [#4564](https://github.com/TileDB-Inc/TileDB/pull/4564) +* Remove uuid in favor of random_label. [#4589](https://github.com/TileDB-Inc/TileDB/pull/4589) + +## Defects removed + +* Fix regression when libraries were sometimes installed in the `lib64` directory. [#4562](https://github.com/TileDB-Inc/TileDB/pull/4562) +* Don't segfault in ArraySchema::dump when enums are not loaded. [#4583](https://github.com/TileDB-Inc/TileDB/pull/4583) +* Fix out of order consolidation. [#4597](https://github.com/TileDB-Inc/TileDB/pull/4597) +* Finalize should error out for remote global order writes. [#4600](https://github.com/TileDB-Inc/TileDB/pull/4600) +* Fix aggregate leak when field doesn't exist. [#4610](https://github.com/TileDB-Inc/TileDB/pull/4610) +* Fix HTTP requests for AWS assume role not honoring config options. [#4616](https://github.com/TileDB-Inc/TileDB/pull/4616) +* Fix recursively dumping a group with a nonexistent subgroup. [#4582](https://github.com/TileDB-Inc/TileDB/pull/4582) +* Fix HTTP requests for AWS assume role with web identity not honoring config options. [#4641](https://github.com/TileDB-Inc/TileDB/pull/4641) + +## Build System Changes + +* Adds centos stream 9 dockerfile that builds tiledb [#4591](https://github.com/TileDB-Inc/TileDB/pull/4591) +* Export a `TileDB::tiledb` CMake target regardless of static or dynamic linkage. [#4528](https://github.com/TileDB-Inc/TileDB/pull/4528) +* Add a document with build instructions. [#4581](https://github.com/TileDB-Inc/TileDB/pull/4581) +* Remove the `TILEDB_STATIC` option and replace it with `BUILD_SHARED_LIBS`. [#4528](https://github.com/TileDB-Inc/TileDB/pull/4528) +* The build system supports building only a static or shared library from the same buildtree. [#4528](https://github.com/TileDB-Inc/TileDB/pull/4528) +* `BUILD_SHARED_LIBS` is enabled by default. [#4569](https://github.com/TileDB-Inc/TileDB/pull/4569) +* Add packaging tests into linux and mac CI pipelines. [#4567](https://github.com/TileDB-Inc/TileDB/pull/4567) +* Add vcpkg triplets for Address Sanitizer. [#4515](https://github.com/TileDB-Inc/TileDB/pull/4515) +* Fix regression where release artifacts had 8-digit commit hashes. [#4599](https://github.com/TileDB-Inc/TileDB/pull/4599) + # TileDB v2.19.0 Release Notes ## Announcements From 8b6501e865f9f4d9cdc9bf57213089e19bef4b50 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 23 Jan 2024 17:01:37 +0200 Subject: [PATCH 151/456] Add more fields to Query Plan capnp and other improvements. (#4633) This PR includes some fixes to the cap'n proto design of Query Plan : - Add as many fields as possible to the cap'n proto model, so that we don't rely on their availability on the client - Do not display `VFS.Backend` field in JSON for remote arrays - Remove `query_plan` toggle from `query_from_capnp`, and replace with generic `allocate_buffers` flag that transfers the responsibility of deciding if query buffers should be allocated to the caller and removes magic `true`/`false` values from the code. (+ small fixes to array directory serialization that I stumbled upon) --- TYPE: NO_HISTORY DESC: Add more fields to Query Plan capnp and other improvements --- tiledb/sm/filesystem/uri.cc | 8 +- tiledb/sm/query_plan/query_plan.cc | 58 ++-- tiledb/sm/query_plan/query_plan.h | 51 +++- tiledb/sm/serialization/array_directory.cc | 83 ++--- tiledb/sm/serialization/query.cc | 56 +++- tiledb/sm/serialization/query.h | 2 +- tiledb/sm/serialization/query_plan.cc | 68 ++++- tiledb/sm/serialization/tiledb-rest.capnp | 16 +- tiledb/sm/serialization/tiledb-rest.capnp.c++ | 94 +++++- tiledb/sm/serialization/tiledb-rest.capnp.h | 285 ++++++++++++++++-- 10 files changed, 602 insertions(+), 119 deletions(-) diff --git a/tiledb/sm/filesystem/uri.cc b/tiledb/sm/filesystem/uri.cc index 8206ba36fb2b..0c7823387850 100644 --- a/tiledb/sm/filesystem/uri.cc +++ b/tiledb/sm/filesystem/uri.cc @@ -327,13 +327,7 @@ std::string URI::to_path(const std::string& uri) { std::string URI::backend_name() const { if (is_tiledb(uri_)) { - std::string array_ns, array_uri; - throw_if_not_ok(URI(uri_).get_rest_components(&array_ns, &array_uri)); - auto prefix = array_uri.substr(0, array_uri.find_first_of(':')); - if (prefix == array_uri) { // no `:` separator found in URI - prefix = "Unknown"; - } - return prefix; + return ""; } else { return uri_.substr(0, uri_.find_first_of(':')); } diff --git a/tiledb/sm/query_plan/query_plan.cc b/tiledb/sm/query_plan/query_plan.cc index c9aa66cb103c..d52160cb2aa3 100644 --- a/tiledb/sm/query_plan/query_plan.cc +++ b/tiledb/sm/query_plan/query_plan.cc @@ -48,18 +48,14 @@ namespace sm { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -QueryPlan::QueryPlan(Query& query, std::string strategy) { +QueryPlan::QueryPlan(Query& query) { array_uri_ = query.array()->array_uri().to_string(); vfs_backend_ = URI(array_uri_).backend_name(); query_layout_ = query.layout(); - if (strategy.empty()) { - // This most probably ends up creating the strategy on the query - auto strategy_ptr = query.strategy(); - strategy_name_ = strategy_ptr->name(); - } else { - strategy_name_ = strategy; - } + // This most probably ends up creating the strategy on the query + auto strategy_ptr = query.strategy(); + strategy_name_ = strategy_ptr->name(); array_type_ = query.array()->array_schema_latest().array_type(); @@ -78,19 +74,47 @@ QueryPlan::QueryPlan(Query& query, std::string strategy) { std::sort(dimensions_.begin(), dimensions_.end()); } +QueryPlan::QueryPlan( + Query& query, + Layout layout, + const std::string& strategy_name, + ArrayType array_type, + const std::vector& attributes, + const std::vector& dimensions) + : array_uri_{query.array()->array_uri().to_string()} + , vfs_backend_{URI(array_uri_).backend_name()} + , query_layout_{layout} + , strategy_name_{strategy_name} + , array_type_{array_type} + , attributes_{attributes} + , dimensions_{dimensions} { +} + /* ********************************* */ /* API */ /* ********************************* */ std::string QueryPlan::dump_json(uint32_t indent) { - nlohmann::json rv = { - {"TileDB Query Plan", - {{"Array.URI", array_uri_}, - {"Array.Type", array_type_str(array_type_)}, - {"VFS.Backend", vfs_backend_}, - {"Query.Layout", layout_str(query_layout_)}, - {"Query.Strategy.Name", strategy_name_}, - {"Query.Attributes", attributes_}, - {"Query.Dimensions", dimensions_}}}}; + nlohmann::json rv; + if (!URI(array_uri_).is_tiledb()) { + rv = { + {"TileDB Query Plan", + {{"Array.URI", array_uri_}, + {"Array.Type", array_type_str(array_type_)}, + {"VFS.Backend", vfs_backend_}, + {"Query.Layout", layout_str(query_layout_)}, + {"Query.Strategy.Name", strategy_name_}, + {"Query.Attributes", attributes_}, + {"Query.Dimensions", dimensions_}}}}; + } else { + rv = { + {"TileDB Query Plan", + {{"Array.URI", array_uri_}, + {"Array.Type", array_type_str(array_type_)}, + {"Query.Layout", layout_str(query_layout_)}, + {"Query.Strategy.Name", strategy_name_}, + {"Query.Attributes", attributes_}, + {"Query.Dimensions", dimensions_}}}}; + } return rv.dump(indent); } diff --git a/tiledb/sm/query_plan/query_plan.h b/tiledb/sm/query_plan/query_plan.h index a5f2cfdb73d4..c8e676b16e3c 100644 --- a/tiledb/sm/query_plan/query_plan.h +++ b/tiledb/sm/query_plan/query_plan.h @@ -65,9 +65,26 @@ class QueryPlan { * Constructor * * @param query A query object for which we want to calculate the plan - * @param strategy The strategy of the query, required only for remote queries */ - QueryPlan(Query& query, const std::string strategy = ""); + QueryPlan(Query& query); + + /** + * Constructor + * + * @param query A query object for which we want to calculate the plan. + * @param layout The query layout. + * @param strategy_name The name of the strategy the query uses. + * @param array_type The type of the array. + * @param attributes The attribute names set in the query. + * @param dimensions The dimension names set in the query. + */ + QueryPlan( + Query& query, + Layout layout, + const std::string& strategy_name, + ArrayType array_type, + const std::vector& attributes, + const std::vector& dimensions); /* ****************************** */ /* API */ @@ -83,12 +100,40 @@ class QueryPlan { std::string dump_json(uint32_t indent = 4); /* - * Get the strategy name stored in the query plan + * Get the query layout stored in the query plan. + */ + inline Layout query_layout() const { + return query_layout_; + } + + /* + * Get the strategy name stored in the query plan. */ inline std::string strategy() const { return strategy_name_; } + /* + * Get the array type stored in the query plan. + */ + inline ArrayType array_type() const { + return array_type_; + } + + /* + * Get the attribute names stored in the query plan. + */ + inline std::vector attributes() const { + return attributes_; + } + + /* + * Get the dimension names stored in the query plan. + */ + inline std::vector dimensions() const { + return dimensions_; + } + private: /* ****************************** */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/serialization/array_directory.cc b/tiledb/sm/serialization/array_directory.cc index e66e9bfbfdb1..41df530e6908 100644 --- a/tiledb/sm/serialization/array_directory.cc +++ b/tiledb/sm/serialization/array_directory.cc @@ -239,9 +239,11 @@ shared_ptr array_directory_from_capnp( // Get unfiltered fragment uris if (array_directory_reader.hasUnfilteredFragmentUris()) { - for (auto uri : array_directory_reader.getUnfilteredFragmentUris()) { - array_directory->unfiltered_fragment_uris().reserve( - array_directory_reader.getUnfilteredFragmentUris().size()); + auto unfiltered_fragment_uris = + array_directory_reader.getUnfilteredFragmentUris(); + array_directory->unfiltered_fragment_uris().reserve( + unfiltered_fragment_uris.size()); + for (auto uri : unfiltered_fragment_uris) { array_directory->unfiltered_fragment_uris().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -249,9 +251,11 @@ shared_ptr array_directory_from_capnp( // Get consolidated commit uris if (array_directory_reader.hasConsolidatedCommitUris()) { - for (auto uri : array_directory_reader.getConsolidatedCommitUris()) { - array_directory->consolidated_commit_uris_set().reserve( - array_directory_reader.getConsolidatedCommitUris().size()); + auto consolidated_commit_uris = + array_directory_reader.getConsolidatedCommitUris(); + array_directory->consolidated_commit_uris_set().reserve( + consolidated_commit_uris.size()); + for (auto uri : consolidated_commit_uris) { array_directory->consolidated_commit_uris_set().emplace( deserialize_array_uri_to_absolute(uri.cStr(), array_uri).to_string()); } @@ -259,9 +263,9 @@ shared_ptr array_directory_from_capnp( // Get array schema uris if (array_directory_reader.hasArraySchemaUris()) { - for (auto uri : array_directory_reader.getArraySchemaUris()) { - array_directory->array_schema_uris().reserve( - array_directory_reader.getArraySchemaUris().size()); + auto array_schema_uris = array_directory_reader.getArraySchemaUris(); + array_directory->array_schema_uris().reserve(array_schema_uris.size()); + for (auto uri : array_schema_uris) { array_directory->array_schema_uris().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -277,9 +281,11 @@ shared_ptr array_directory_from_capnp( // Get array meta uris to vacuum if (array_directory_reader.hasArrayMetaUrisToVacuum()) { - for (auto uri : array_directory_reader.getArrayMetaUrisToVacuum()) { - array_directory->array_meta_uris_to_vacuum().reserve( - array_directory_reader.getArrayMetaUrisToVacuum().size()); + auto array_meta_uris_to_vacuum = + array_directory_reader.getArrayMetaUrisToVacuum(); + array_directory->array_meta_uris_to_vacuum().reserve( + array_meta_uris_to_vacuum.size()); + for (auto uri : array_meta_uris_to_vacuum) { array_directory->array_meta_uris_to_vacuum().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -287,9 +293,11 @@ shared_ptr array_directory_from_capnp( // Get array meta vac uris to vacuum if (array_directory_reader.hasArrayMetaVacUrisToVacuum()) { - for (auto uri : array_directory_reader.getArrayMetaVacUrisToVacuum()) { - array_directory->array_meta_vac_uris_to_vacuum().reserve( - array_directory_reader.getArrayMetaVacUrisToVacuum().size()); + auto array_meta_vac_uris_to_vacuum = + array_directory_reader.getArrayMetaVacUrisToVacuum(); + array_directory->array_meta_vac_uris_to_vacuum().reserve( + array_meta_vac_uris_to_vacuum.size()); + for (auto uri : array_meta_vac_uris_to_vacuum) { array_directory->array_meta_vac_uris_to_vacuum().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -297,9 +305,11 @@ shared_ptr array_directory_from_capnp( // Get commit uris to consolidate if (array_directory_reader.hasCommitUrisToConsolidate()) { - for (auto uri : array_directory_reader.getCommitUrisToConsolidate()) { - array_directory->commit_uris_to_consolidate().reserve( - array_directory_reader.getCommitUrisToConsolidate().size()); + auto commit_uris_to_consolidate = + array_directory_reader.getCommitUrisToConsolidate(); + array_directory->commit_uris_to_consolidate().reserve( + commit_uris_to_consolidate.size()); + for (auto uri : commit_uris_to_consolidate) { array_directory->commit_uris_to_consolidate().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -307,9 +317,10 @@ shared_ptr array_directory_from_capnp( // Get commit uris to vacuum if (array_directory_reader.hasCommitUrisToVacuum()) { - for (auto uri : array_directory_reader.getCommitUrisToVacuum()) { - array_directory->commit_uris_to_vacuum().reserve( - array_directory_reader.getCommitUrisToVacuum().size()); + auto commit_uris_to_vacuum = array_directory_reader.getCommitUrisToVacuum(); + array_directory->commit_uris_to_vacuum().reserve( + commit_uris_to_vacuum.size()); + for (auto uri : commit_uris_to_vacuum) { array_directory->commit_uris_to_vacuum().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -317,10 +328,11 @@ shared_ptr array_directory_from_capnp( // Get consolidated commit uris to vacuum if (array_directory_reader.hasConsolidatedCommitUrisToVacuum()) { - for (auto uri : - array_directory_reader.getConsolidatedCommitUrisToVacuum()) { - array_directory->consolidated_commits_uris_to_vacuum().reserve( - array_directory_reader.getConsolidatedCommitUrisToVacuum().size()); + auto consolidated_commits_uris_to_vacuum = + array_directory_reader.getConsolidatedCommitUrisToVacuum(); + array_directory->consolidated_commits_uris_to_vacuum().reserve( + consolidated_commits_uris_to_vacuum.size()); + for (auto uri : consolidated_commits_uris_to_vacuum) { array_directory->consolidated_commits_uris_to_vacuum().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -328,9 +340,9 @@ shared_ptr array_directory_from_capnp( // Get array meta uris if (array_directory_reader.hasArrayMetaUris()) { - for (auto timestamp_reader : array_directory_reader.getArrayMetaUris()) { - array_directory->array_meta_uris().reserve( - array_directory_reader.getArrayMetaUris().size()); + auto array_meta_uris = array_directory_reader.getArrayMetaUris(); + array_directory->array_meta_uris().reserve(array_meta_uris.size()); + for (auto timestamp_reader : array_meta_uris) { array_directory->array_meta_uris().emplace_back( URI(deserialize_array_uri_to_absolute( timestamp_reader.getUri().cStr(), array_uri)), @@ -342,9 +354,9 @@ shared_ptr array_directory_from_capnp( // Get fragment meta uris if (array_directory_reader.hasFragmentMetaUris()) { - for (auto uri : array_directory_reader.getFragmentMetaUris()) { - array_directory->fragment_meta_uris().reserve( - array_directory_reader.getFragmentMetaUris().size()); + auto fragment_meta_uris = array_directory_reader.getFragmentMetaUris(); + array_directory->fragment_meta_uris().reserve(fragment_meta_uris.size()); + for (auto uri : fragment_meta_uris) { array_directory->fragment_meta_uris().emplace_back( deserialize_array_uri_to_absolute(uri.cStr(), array_uri)); } @@ -352,10 +364,11 @@ shared_ptr array_directory_from_capnp( // Get delete tiles location if (array_directory_reader.hasDeleteAndUpdateTileLocation()) { - for (auto del_tile_reader : - array_directory_reader.getDeleteAndUpdateTileLocation()) { - array_directory->delete_and_update_tiles_location().reserve( - array_directory_reader.getDeleteAndUpdateTileLocation().size()); + auto delete_and_update_tiles_location = + array_directory_reader.getDeleteAndUpdateTileLocation(); + array_directory->delete_and_update_tiles_location().reserve( + delete_and_update_tiles_location.size()); + for (auto del_tile_reader : delete_and_update_tiles_location) { array_directory->delete_and_update_tiles_location().emplace_back( URI(deserialize_array_uri_to_absolute( del_tile_reader.getUri().cStr(), array_uri)), diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index aabfd95adad2..11351aa98414 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -1614,7 +1614,7 @@ Status query_from_capnp( CopyState* const copy_state, Query* const query, ThreadPool* compute_tp, - const bool query_plan) { + const bool allocate_buffers) { using namespace tiledb::sm; auto array = query->array(); @@ -2057,20 +2057,20 @@ Status query_from_capnp( name, attr_state->var_len_data.data(), &attr_state->var_len_size, - false, + allocate_buffers, true)); throw_if_not_ok(query->set_offsets_buffer( name, attr_state->fixed_len_data.data_as(), &attr_state->fixed_len_size, - false, + allocate_buffers, true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( name, attr_state->validity_len_data.data_as(), &attr_state->validity_len_size, - false, + allocate_buffers, true)); } } else { @@ -2078,14 +2078,14 @@ Status query_from_capnp( name, attr_state->fixed_len_data.data(), &attr_state->fixed_len_size, - false, + allocate_buffers, true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( name, attr_state->validity_len_data.data_as(), &attr_state->validity_len_size, - false, + allocate_buffers, true)); } } @@ -2115,15 +2115,23 @@ Status query_from_capnp( attr_state->validity_len_data.swap(validity_buff); throw_if_not_ok(query->set_data_buffer( - name, varlen_data, &attr_state->var_len_size, !query_plan, true)); + name, + varlen_data, + &attr_state->var_len_size, + allocate_buffers, + true)); throw_if_not_ok(query->set_offsets_buffer( - name, offsets, &attr_state->fixed_len_size, !query_plan, true)); + name, + offsets, + &attr_state->fixed_len_size, + allocate_buffers, + true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( name, validity, &attr_state->validity_len_size, - !query_plan, + allocate_buffers, true)); } } else { @@ -2147,13 +2155,13 @@ Status query_from_capnp( attr_state->validity_len_data.swap(validity_buff); throw_if_not_ok(query->set_data_buffer( - name, data, &attr_state->fixed_len_size, !query_plan, true)); + name, data, &attr_state->fixed_len_size, allocate_buffers, true)); if (nullable) { throw_if_not_ok(query->set_validity_buffer( name, validity, &attr_state->validity_len_size, - !query_plan, + allocate_buffers, true)); } } @@ -2547,7 +2555,8 @@ Status do_query_deserialize( const SerializationContext context, CopyState* const copy_state, Query* query, - ThreadPool* compute_tp) { + ThreadPool* compute_tp, + const bool allocate_buffers) { if (serialize_type == SerializationType::JSON) return LOG_STATUS(Status_SerializationError( "Cannot deserialize query; json format not supported.")); @@ -2565,7 +2574,13 @@ Status do_query_deserialize( query_builder); capnp::Query::Reader query_reader = query_builder.asReader(); return query_from_capnp( - query_reader, context, nullptr, copy_state, query, compute_tp); + query_reader, + context, + nullptr, + copy_state, + query, + compute_tp, + allocate_buffers); } case SerializationType::CAPNP: { // Capnp FlatArrayMessageReader requires 64-bit alignment. @@ -2595,7 +2610,13 @@ Status do_query_deserialize( auto attribute_buffer_start = reader.getEnd(); auto buffer_start = const_cast<::capnp::word*>(attribute_buffer_start); return query_from_capnp( - query_reader, context, buffer_start, copy_state, query, compute_tp); + query_reader, + context, + buffer_start, + copy_state, + query, + compute_tp, + allocate_buffers); } default: return LOG_STATUS(Status_SerializationError( @@ -2637,6 +2658,7 @@ Status query_deserialize( original_copy_state = tdb_unique_ptr(tdb_new(CopyState, *copy_state)); } + auto allocate_buffers = query->type() == QueryType::WRITE; // Deserialize 'serialized_buffer'. const Status st = do_query_deserialize( @@ -2645,7 +2667,8 @@ Status query_deserialize( clientside ? SerializationContext::CLIENT : SerializationContext::SERVER, copy_state, query, - compute_tp); + compute_tp, + allocate_buffers); // If the deserialization failed, deserialize 'original_buffer' // into 'query' to ensure that 'query' is in the state it was before the @@ -2663,7 +2686,8 @@ Status query_deserialize( SerializationContext::BACKUP, copy_state, query, - compute_tp); + compute_tp, + allocate_buffers); if (!st2.ok()) { LOG_ERROR(st2.message()); return st2; diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 28821dd69a84..dced5df32e4b 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -276,7 +276,7 @@ Status query_from_capnp( CopyState* const copy_state, Query* const query, ThreadPool* compute_tp, - const bool query_plan = false); + const bool allocate_buffers); #endif diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc index 07bb764b2144..9c5caea9a5fd 100644 --- a/tiledb/sm/serialization/query_plan.cc +++ b/tiledb/sm/serialization/query_plan.cc @@ -40,8 +40,11 @@ // clang-format on #include "tiledb/sm/query_plan/query_plan.h" + #include "tiledb/common/thread_pool.h" #include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/layout.h" #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/serialization/query.h" #include "tiledb/sm/serialization/query_plan.h" @@ -76,23 +79,68 @@ void query_plan_request_from_capnp( nullptr, &query, &compute_tp, - true)); + false)); } } void query_plan_response_to_capnp( capnp::QueryPlanResponse::Builder& builder, const QueryPlan& query_plan) { - builder.setStrategy(query_plan.strategy()); + builder.setQueryLayout(layout_str(query_plan.query_layout())); + builder.setStrategyName(query_plan.strategy()); + builder.setArrayType(array_type_str(query_plan.array_type())); + const auto& attributes = query_plan.attributes(); + if (!attributes.empty()) { + auto attributes_builder = builder.initAttributeNames(attributes.size()); + for (size_t i = 0; i < attributes.size(); i++) { + attributes_builder.set(i, attributes[i]); + } + } + + const auto& dimensions = query_plan.dimensions(); + if (!dimensions.empty()) { + auto dimensions_builder = builder.initDimensionNames(dimensions.size()); + for (size_t i = 0; i < dimensions.size(); i++) { + dimensions_builder.set(i, dimensions[i]); + } + } } -std::string query_plan_response_from_capnp( - const capnp::QueryPlanResponse::Reader& reader) { - std::string strategy; - if (reader.hasStrategy()) { - strategy = reader.getStrategy().cStr(); +QueryPlan query_plan_response_from_capnp( + const capnp::QueryPlanResponse::Reader& reader, Query& query) { + auto layout{Layout::ROW_MAJOR}; + if (reader.hasQueryLayout()) { + throw_if_not_ok(layout_enum(reader.getQueryLayout().cStr(), &layout)); + } + + std::string strategy{}; + if (reader.hasStrategyName()) { + strategy = reader.getStrategyName().cStr(); + } + + auto array_type{ArrayType::DENSE}; + if (reader.hasQueryLayout()) { + throw_if_not_ok(array_type_enum(reader.getArrayType().cStr(), &array_type)); + } + + std::vector attributes; + if (reader.hasAttributeNames()) { + auto attribute_names = reader.getAttributeNames(); + attributes.reserve(attribute_names.size()); + for (auto attr : attribute_names) { + attributes.emplace_back(attr); + } + } + + std::vector dimensions; + if (reader.hasDimensionNames()) { + auto dimension_names = reader.getDimensionNames(); + dimensions.reserve(dimension_names.size()); + for (auto dim : dimension_names) { + dimensions.emplace_back(dim); + } } - return strategy; + return QueryPlan(query, layout, strategy, array_type, attributes, dimensions); } void serialize_query_plan_request( @@ -258,7 +306,7 @@ QueryPlan deserialize_query_plan_response( json.decode( kj::StringPtr(static_cast(response.data())), builder); capnp::QueryPlanResponse::Reader reader = builder.asReader(); - return QueryPlan(query, query_plan_response_from_capnp(reader)); + return query_plan_response_from_capnp(reader, query); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(response.data()); @@ -267,7 +315,7 @@ QueryPlan deserialize_query_plan_response( response.size() / sizeof(::capnp::word))); capnp::QueryPlanResponse::Reader reader = array_reader.getRoot(); - return QueryPlan(query, query_plan_response_from_capnp(reader)); + return query_plan_response_from_capnp(reader, query); } default: { throw Status_SerializationError( diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index cdc4537baf28..cb48696310c0 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1271,8 +1271,20 @@ struct QueryPlanRequest { } struct QueryPlanResponse { - strategy @0 :Text; - # strategy used by the query + queryLayout @0 :Text; + # query layout + + strategyName @1 :Text; + # name of strategy used by the query + + arrayType @2 :Text; + # type of array + + attributeNames @3 :List(Text); + # names of attributes in the query + + dimensionNames @4 :List(Text); + # names of dimensions in the query } struct ConsolidationPlanRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index 5ea224a7acef..f5770f9a81f5 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -9839,17 +9839,17 @@ const ::capnp::_::RawSchema s_e06f571aa93eb314 = { 2, 2, i_e06f571aa93eb314, nullptr, nullptr, { &s_e06f571aa93eb314, nullptr, nullptr, 0, 0, nullptr }, true }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_9fd8fc2f462b2d06 = { +static const ::capnp::_::AlignedData<107> b_9fd8fc2f462b2d06 = { { 0, 0, 0, 0, 5, 0, 6, 0, 6, 45, 43, 70, 47, 252, 216, 159, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 1, 0, 7, 0, 0, 0, 0, 0, + 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 34, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 63, 0, 0, 0, + 33, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -9858,31 +9858,103 @@ static const ::capnp::_::AlignedData<35> b_9fd8fc2f462b2d06 = { 108, 97, 110, 82, 101, 115, 112, 111, 110, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 4, 0, + 20, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 74, 0, 0, 0, + 125, 0, 0, 0, 98, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 136, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 0, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 141, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 140, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 149, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 176, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 172, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 113, 117, 101, 114, 121, 76, 97, 121, + 111, 117, 116, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 3, 0, 1, 0, - 24, 0, 0, 0, 2, 0, 1, 0, 115, 116, 114, 97, 116, 101, 103, 121, + 78, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 114, 114, 97, 121, 84, 121, 112, + 101, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 97, 116, 116, 114, 105, 98, 117, 116, + 101, 78, 97, 109, 101, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 105, 109, 101, 110, 115, 105, 111, + 110, 78, 97, 109, 101, 115, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_9fd8fc2f462b2d06 = b_9fd8fc2f462b2d06.words; #if !CAPNP_LITE -static const uint16_t m_9fd8fc2f462b2d06[] = {0}; -static const uint16_t i_9fd8fc2f462b2d06[] = {0}; +static const uint16_t m_9fd8fc2f462b2d06[] = {2, 3, 4, 0, 1}; +static const uint16_t i_9fd8fc2f462b2d06[] = {0, 1, 2, 3, 4}; const ::capnp::_::RawSchema s_9fd8fc2f462b2d06 = { - 0x9fd8fc2f462b2d06, b_9fd8fc2f462b2d06.words, 35, nullptr, m_9fd8fc2f462b2d06, - 0, 1, i_9fd8fc2f462b2d06, nullptr, nullptr, { &s_9fd8fc2f462b2d06, nullptr, nullptr, 0, 0, nullptr }, false + 0x9fd8fc2f462b2d06, b_9fd8fc2f462b2d06.words, 107, nullptr, m_9fd8fc2f462b2d06, + 0, 5, i_9fd8fc2f462b2d06, nullptr, nullptr, { &s_9fd8fc2f462b2d06, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<51> b_8965edf5597ce627 = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 4d493a76910c..4951375c0824 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -1716,7 +1716,7 @@ struct QueryPlanResponse { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(9fd8fc2f462b2d06, 0, 1) + CAPNP_DECLARE_STRUCT_HEADER(9fd8fc2f462b2d06, 0, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -15140,8 +15140,22 @@ class QueryPlanResponse::Reader { } #endif // !CAPNP_LITE - inline bool hasStrategy() const; - inline ::capnp::Text::Reader getStrategy() const; + inline bool hasQueryLayout() const; + inline ::capnp::Text::Reader getQueryLayout() const; + + inline bool hasStrategyName() const; + inline ::capnp::Text::Reader getStrategyName() const; + + inline bool hasArrayType() const; + inline ::capnp::Text::Reader getArrayType() const; + + inline bool hasAttributeNames() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getAttributeNames() const; + + inline bool hasDimensionNames() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getDimensionNames() const; private: ::capnp::_::StructReader _reader; @@ -15182,12 +15196,56 @@ class QueryPlanResponse::Builder { } #endif // !CAPNP_LITE - inline bool hasStrategy(); - inline ::capnp::Text::Builder getStrategy(); - inline void setStrategy(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initStrategy(unsigned int size); - inline void adoptStrategy(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownStrategy(); + inline bool hasQueryLayout(); + inline ::capnp::Text::Builder getQueryLayout(); + inline void setQueryLayout(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initQueryLayout(unsigned int size); + inline void adoptQueryLayout(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownQueryLayout(); + + inline bool hasStrategyName(); + inline ::capnp::Text::Builder getStrategyName(); + inline void setStrategyName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initStrategyName(unsigned int size); + inline void adoptStrategyName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownStrategyName(); + + inline bool hasArrayType(); + inline ::capnp::Text::Builder getArrayType(); + inline void setArrayType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initArrayType(unsigned int size); + inline void adoptArrayType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownArrayType(); + + inline bool hasAttributeNames(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getAttributeNames(); + inline void setAttributeNames( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setAttributeNames( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initAttributeNames(unsigned int size); + inline void adoptAttributeNames( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownAttributeNames(); + + inline bool hasDimensionNames(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getDimensionNames(); + inline void setDimensionNames( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setDimensionNames( + ::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initDimensionNames(unsigned int size); + inline void adoptDimensionNames( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownDimensionNames(); private: ::capnp::_::StructBuilder _builder; @@ -32672,46 +32730,239 @@ QueryPlanRequest::Builder::disownQuery() { _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } -inline bool QueryPlanResponse::Reader::hasStrategy() const { +inline bool QueryPlanResponse::Reader::hasQueryLayout() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline bool QueryPlanResponse::Builder::hasStrategy() { +inline bool QueryPlanResponse::Builder::hasQueryLayout() { return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader QueryPlanResponse::Reader::getStrategy() const { +inline ::capnp::Text::Reader QueryPlanResponse::Reader::getQueryLayout() const { return ::capnp::_::PointerHelpers<::capnp::Text>::get( _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline ::capnp::Text::Builder QueryPlanResponse::Builder::getStrategy() { +inline ::capnp::Text::Builder QueryPlanResponse::Builder::getQueryLayout() { return ::capnp::_::PointerHelpers<::capnp::Text>::get( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline void QueryPlanResponse::Builder::setStrategy( +inline void QueryPlanResponse::Builder::setQueryLayout( ::capnp::Text::Reader value) { ::capnp::_::PointerHelpers<::capnp::Text>::set( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), value); } -inline ::capnp::Text::Builder QueryPlanResponse::Builder::initStrategy( +inline ::capnp::Text::Builder QueryPlanResponse::Builder::initQueryLayout( unsigned int size) { return ::capnp::_::PointerHelpers<::capnp::Text>::init( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), size); } -inline void QueryPlanResponse::Builder::adoptStrategy( +inline void QueryPlanResponse::Builder::adoptQueryLayout( ::capnp::Orphan<::capnp::Text>&& value) { ::capnp::_::PointerHelpers<::capnp::Text>::adopt( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::Text> -QueryPlanResponse::Builder::disownStrategy() { +QueryPlanResponse::Builder::disownQueryLayout() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool QueryPlanResponse::Reader::hasStrategyName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanResponse::Builder::hasStrategyName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader QueryPlanResponse::Reader::getStrategyName() + const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::getStrategyName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void QueryPlanResponse::Builder::setStrategyName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::initStrategyName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void QueryPlanResponse::Builder::adoptStrategyName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +QueryPlanResponse::Builder::disownStrategyName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool QueryPlanResponse::Reader::hasArrayType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanResponse::Builder::hasArrayType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader QueryPlanResponse::Reader::getArrayType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::getArrayType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void QueryPlanResponse::Builder::setArrayType( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder QueryPlanResponse::Builder::initArrayType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void QueryPlanResponse::Builder::adoptArrayType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +QueryPlanResponse::Builder::disownArrayType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline bool QueryPlanResponse::Reader::hasAttributeNames() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanResponse::Builder::hasAttributeNames() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +QueryPlanResponse::Reader::getAttributeNames() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +QueryPlanResponse::Builder::getAttributeNames() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void QueryPlanResponse::Builder::setAttributeNames( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline void QueryPlanResponse::Builder::setAttributeNames( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +QueryPlanResponse::Builder::initAttributeNames(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void QueryPlanResponse::Builder::adoptAttributeNames( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +QueryPlanResponse::Builder::disownAttributeNames() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool QueryPlanResponse::Reader::hasDimensionNames() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool QueryPlanResponse::Builder::hasDimensionNames() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +QueryPlanResponse::Reader::getDimensionNames() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +QueryPlanResponse::Builder::getDimensionNames() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void QueryPlanResponse::Builder::setDimensionNames( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline void QueryPlanResponse::Builder::setDimensionNames( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +QueryPlanResponse::Builder::initDimensionNames(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void QueryPlanResponse::Builder::adoptDimensionNames( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +QueryPlanResponse::Builder::disownDimensionNames() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool ConsolidationPlanRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); From 598e84eb3f6f77a7058ef68a26447984b545e3e4 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 23 Jan 2024 17:56:56 +0100 Subject: [PATCH 152/456] Improve large dense aggregate reads with tile metadata only. (#4657) The dense reader was creating a bitmap to compute all aggregate results at the top level of the read. For large aggregate reads where we don't need to load any tiles, this could be quite large. It also would turn out to be completely unnecessary. This fix moves the bitmap to the lower level of the read, where a smaller bitmap can be created only if necessary. --- TYPE: IMPROVEMENT DESC: Improve large dense aggregate reads with tile metadata only. --- tiledb/sm/query/readers/dense_reader.cc | 25 +++++++++++++------------ tiledb/sm/query/readers/dense_reader.h | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index df18d001bd92..501cdd5b4ad3 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -1319,13 +1319,6 @@ Status DenseReader::process_aggregates( const auto& tile_coords = subarray.tile_coords(); const auto global_order = layout_ == Layout::GLOBAL_ORDER; - std::vector aggregate_bitmap; - if (condition_.has_value()) { - aggregate_bitmap = qc_result; - } else { - aggregate_bitmap.resize(subarray.cell_num(), 1); - } - // Process values in parallel. auto status = parallel_for_2d( storage_manager_->compute_tp(), @@ -1358,7 +1351,7 @@ Status DenseReader::process_aggregates( tile_subarrays[t], global_order ? tile_offsets[t] : 0, range_info, - aggregate_bitmap, + qc_result, range_thread_idx, num_range_threads)); } @@ -1863,7 +1856,7 @@ Status DenseReader::aggregate_tiles( const Subarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, - std::vector& aggregate_bitmap, + const std::vector& qc_result, const uint64_t range_thread_idx, const uint64_t num_range_threads) { // For easy reference @@ -1909,6 +1902,14 @@ Status DenseReader::aggregate_tiles( cell_offset = iter.dest_offset_row_col(); } + std::vector aggregate_bitmap(iter.cell_slab_length(), 1); + if (condition_.has_value()) { + memcpy( + aggregate_bitmap.data(), + qc_result.data() + cell_offset, + iter.cell_slab_length()); + } + // Iterate through all fragment domains and copy data. for (uint64_t fd = 0; fd < frag_domains.size(); fd++) { // If the cell slab overlaps this fragment domain range, copy data. @@ -1936,7 +1937,7 @@ Status DenseReader::aggregate_tiles( iter.pos_in_tile() + start, iter.pos_in_tile() + end + 1, tile_tuples[fd], - &aggregate_bitmap[cell_offset + start])}; + aggregate_bitmap.data() + start)}; for (auto& aggregate : aggregates) { aggregate->aggregate_data(aggregate_buffer); } @@ -1952,7 +1953,7 @@ Status DenseReader::aggregate_tiles( start_cell, start_cell + 1, tile_tuples[fd], - &aggregate_bitmap[cell_offset + start + i])}; + aggregate_bitmap.data() + start + i)}; for (auto& aggregate : aggregates) { aggregate->aggregate_data(aggregate_buffer); } @@ -1964,7 +1965,7 @@ Status DenseReader::aggregate_tiles( // fragments. if (fd != frag_domains.size() - 1) { for (uint64_t c = start; c <= end; c++) { - aggregate_bitmap[cell_offset + c] = 0; + aggregate_bitmap[c] = 0; } } diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 8560171edecb..16c997299acf 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -400,7 +400,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const Subarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, - std::vector& aggregate_bitmap, + const std::vector& qc_result, const uint64_t range_thread_idx, const uint64_t num_range_threads); From 310dd6cbdea47327fb96c8d9673e1bc8961d448e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 24 Jan 2024 03:40:11 -0500 Subject: [PATCH 153/456] Add datatypes GEOM_WKB and GEOM_WKT (#4640) Add datatypes GEOM_WKB and GEOM_WKT, which represent `std::byte` --- TYPE: FEATURE DESC: Add datatypes GEOM_WKB and GEOM_WKT --- test/src/test-cppapi-aggregates.cc | 31 ++++++++- test/src/unit-backwards_compat.cc | 18 +++-- test/src/unit-capi-attributes.cc | 13 ++-- test/src/unit-cppapi-array.cc | 67 +++++++++++++++++++ test/src/unit-enum-helpers.cc | 4 +- tiledb/api/c_api/datatype/datatype_api_enum.h | 6 +- .../c_api/datatype/test/unit_capi_datatype.cc | 6 +- .../c_api/dimension/dimension_api_external.h | 8 +-- tiledb/sm/array_schema/dimension.cc | 10 +-- tiledb/sm/array_schema/dimension.h | 14 ++-- tiledb/sm/array_schema/domain.cc | 9 +-- tiledb/sm/array_schema/test/unit_dimension.cc | 7 +- tiledb/sm/compressors/dd_compressor.cc | 28 ++++---- tiledb/sm/compressors/delta_compressor.cc | 37 +++++----- tiledb/sm/cpp_api/arrow_io_impl.h | 13 ++-- tiledb/sm/cpp_api/dimension.h | 12 ++-- tiledb/sm/cpp_api/type.h | 21 ++++-- tiledb/sm/enums/datatype.h | 38 ++++++++--- .../sm/filter/bit_width_reduction_filter.cc | 14 ++-- tiledb/sm/filter/filter_buffer.cc | 10 +-- tiledb/sm/filter/positive_delta_filter.cc | 18 ++--- tiledb/sm/filter/test/unit_filter_pipeline.cc | 6 +- tiledb/sm/fragment/fragment_metadata.cc | 10 +-- tiledb/sm/misc/constants.cc | 27 +++++--- tiledb/sm/misc/constants.h | 17 +++-- tiledb/sm/misc/integral_type_casts.h | 13 ++-- tiledb/sm/misc/parse_argument.cc | 4 +- .../sm/misc/test/unit_integral_type_casts.cc | 11 +-- tiledb/sm/query/ast/query_ast.cc | 17 +++-- tiledb/sm/query/query_condition.cc | 24 ++++--- tiledb/sm/query/test/unit_query_condition.cc | 29 +++++++- tiledb/sm/serialization/capnp_utils.h | 27 ++++---- tiledb/sm/tile/tile_metadata_generator.cc | 18 +++-- tiledb/storage_format/DIRECTORY.md | 2 + 34 files changed, 399 insertions(+), 190 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index f10f57b67a3b..aa5a313a732e 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -516,7 +516,7 @@ std::vector CppAggregatesFx::make_data_buff( } } else { for (auto& v : values) { - data.emplace_back(v); + data.emplace_back(static_cast(v)); } } @@ -1624,6 +1624,31 @@ TEMPLATE_LIST_TEST_CASE_METHOD( array.close(); } +typedef tuple BlobTypeUnderTest; +TEMPLATE_LIST_TEST_CASE_METHOD( + CppAggregatesFx, + "C++ API: Aggregates basic sum, std::byte", + "[cppapi][aggregates][basic][sum][byte]", + BlobTypeUnderTest) { + typedef TestType T; + CppAggregatesFx::generate_test_params(); + CppAggregatesFx::create_array_and_write_fragments(); + Array array{ + CppAggregatesFx::ctx_, CppAggregatesFx::ARRAY_NAME, TILEDB_READ}; + Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); + + // Add a sum aggregator to the query. + QueryChannel default_channel = QueryExperimental::get_default_channel(query); + + REQUIRE_THROWS_WITH( + QueryExperimental::create_unary_aggregate(query, "a1"), + Catch::Matchers::ContainsSubstring( + "Datatype::BLOB is not a valid Datatype")); + + // Close array. + array.close(); +} + typedef tuple< uint8_t, uint16_t, @@ -2607,7 +2632,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( TEST_CASE_METHOD( CppAggregatesFx, "CPP: Aggregates - Basic", - "[aggregates][cpp_api][args]") { + "[cppapi][aggregates][args]") { dense_ = false; nullable_ = false; allow_dups_ = false; diff --git a/test/src/unit-backwards_compat.cc b/test/src/unit-backwards_compat.cc index 22ac6e956cd3..685c942cf35e 100644 --- a/test/src/unit-backwards_compat.cc +++ b/test/src/unit-backwards_compat.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -275,7 +275,9 @@ TEST_CASE( uint8_t* validity = static_cast(malloc(sizeof(uint8_t))); switch (attr.second.type()) { - case TILEDB_BLOB: { + case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: { set_buffer_wrapper( query, attribute_name, @@ -577,7 +579,9 @@ TEST_CASE( Attribute attribute = array->schema().attribute(buff.first); switch (attribute.type()) { - case TILEDB_BLOB: { + case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: { REQUIRE( static_cast(std::get<1>(buffer))[0] == static_cast(1)); @@ -780,7 +784,9 @@ TEST_CASE( uint8_t* validity = static_cast(malloc(sizeof(uint8_t))); switch (attr.second.type()) { - case TILEDB_BLOB: { + case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: { set_buffer_wrapper( query, attribute_name, @@ -1118,7 +1124,9 @@ TEST_CASE( Attribute attribute = array->schema().attribute(buff.first); switch (attribute.type()) { - case TILEDB_BLOB: { + case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: { REQUIRE( static_cast(std::get<1>(buffer))[0] == static_cast(1)); diff --git a/test/src/unit-capi-attributes.cc b/test/src/unit-capi-attributes.cc index 01e9c7d6dd40..ca74361c9ed7 100644 --- a/test/src/unit-capi-attributes.cc +++ b/test/src/unit-capi-attributes.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021-2023 TileDB Inc. + * @copyright Copyright (c) 2021-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -286,8 +286,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( Attributesfx, - "C API: Test attributes with tiledb_blob datatype", - "[capi][attributes][tiledb_blob]") { + "C API: Test attributes with std::byte", + "[capi][attributes][byte]") { + auto datatype = GENERATE(TILEDB_BLOB, TILEDB_GEOM_WKB, TILEDB_GEOM_WKT); + SECTION("no serialization") { serialize_ = false; } @@ -306,7 +308,7 @@ TEST_CASE_METHOD( continue; } - std::string attr_name = "attr"; + std::string attr_name = "a"; // Create new TileDB context with file lock config disabled, rest the // same. @@ -324,7 +326,7 @@ TEST_CASE_METHOD( create_temp_dir(temp_dir); - create_dense_vector(array_name, attr_name, TILEDB_BLOB); + create_dense_vector(array_name, attr_name, datatype); // Prepare cell buffers uint8_t buffer_write[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -383,6 +385,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); + rc = submit_query_wrapper( ctx_, array_name, diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index ff4204fe3464..c25ada51c8a5 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -1198,6 +1198,73 @@ TEST_CASE( vfs.remove_dir(array_name); } +TEST_CASE( + "C++ API: Writing single byte cell with global order", + "[cppapi][std::byte]") { + bool serialize = false, refactored_query_v2 = false; + const std::string array_name = "cpp_unit_array"; + + Context ctx; + VFS vfs(ctx); + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + // Create + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "rows", {{0, 0}}, 1)); + ArraySchema schema(ctx, TILEDB_DENSE); + schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); + schema.add_attribute( + Attribute::create(ctx, "a", tiledb_datatype_t::TILEDB_BLOB)); + Array::create(array_name, schema); + + // Write + std::byte data_w{1}; + Array array_w(ctx, array_name, TILEDB_WRITE); + Query query_w(ctx, array_w); + query_w.set_layout(TILEDB_GLOBAL_ORDER).set_data_buffer("a", &data_w, 1); + + SECTION("no serialization") { + serialize = false; + } +#ifdef TILEDB_SERIALIZATION + SECTION("serialization enabled global order write") { + serialize = true; + refactored_query_v2 = GENERATE(true, false); + } +#endif + + // Submit query + test::ServerQueryBuffers server_buffers_; + auto rc = test::submit_query_wrapper( + ctx, + array_name, + &query_w, + server_buffers_, + serialize, + refactored_query_v2); + REQUIRE(rc == TILEDB_OK); + array_w.close(); + + // Read + Array array(ctx, array_name, TILEDB_READ); + Query query(ctx, array); + Subarray subarray(ctx, array); + subarray.add_range(0, 0, 0); + std::byte data; + query.set_layout(TILEDB_ROW_MAJOR) + .set_subarray(subarray) + .set_data_buffer("a", &data, 1); + query.submit(); + array.close(); + + REQUIRE(data == data_w); + + if (vfs.is_dir(array_name)) + vfs.remove_dir(array_name); +} + TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { const std::string array_name = "cpp_unit_array"; Context ctx; diff --git a/test/src/unit-enum-helpers.cc b/test/src/unit-enum-helpers.cc index 555b8f993707..1ff811d35ab7 100644 --- a/test/src/unit-enum-helpers.cc +++ b/test/src/unit-enum-helpers.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -58,6 +58,8 @@ TEST_CASE( } REQUIRE_THROWS(datatype_max_integral_value(Datatype::BLOB)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::GEOM_WKB)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::GEOM_WKT)); REQUIRE_THROWS(datatype_max_integral_value(Datatype::FLOAT64)); REQUIRE_THROWS(datatype_max_integral_value(Datatype::STRING_ASCII)); } diff --git a/tiledb/api/c_api/datatype/datatype_api_enum.h b/tiledb/api/c_api/datatype/datatype_api_enum.h index 74608d3f2c1e..eb20848e0e39 100644 --- a/tiledb/api/c_api/datatype/datatype_api_enum.h +++ b/tiledb/api/c_api/datatype/datatype_api_enum.h @@ -1,7 +1,7 @@ /* * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -114,4 +114,8 @@ TILEDB_DATATYPE_ENUM(BLOB) = 40, /** Boolean */ TILEDB_DATATYPE_ENUM(BOOL) = 41, + /** Geometry data in well-known binary (WKB) format, stored as std::byte */ + TILEDB_DATATYPE_ENUM(GEOM_WKB) = 42, + /** Geometry data in well-known text (WKT) format, stored as std::byte */ + TILEDB_DATATYPE_ENUM(GEOM_WKT) = 43, #endif diff --git a/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc b/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc index e77c0810f24a..f26c61b284f6 100644 --- a/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc +++ b/tiledb/api/c_api/datatype/test/unit_capi_datatype.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -106,7 +106,9 @@ TEST_CASE( TestCase(TILEDB_TIME_FS, "TIME_FS", 38), TestCase(TILEDB_TIME_AS, "TIME_AS", 39), TestCase(TILEDB_BLOB, "BLOB", 40), - TestCase(TILEDB_BOOL, "BOOL", 41)); + TestCase(TILEDB_BOOL, "BOOL", 41), + TestCase(TILEDB_GEOM_WKB, "GEOM_WKB", 42), + TestCase(TILEDB_GEOM_WKT, "GEOM_WKT", 43)); DYNAMIC_SECTION("[" << test.name_ << "]") { test.run(); diff --git a/tiledb/api/c_api/dimension/dimension_api_external.h b/tiledb/api/c_api/dimension/dimension_api_external.h index 89867bf21df8..8dc2a073a0fe 100644 --- a/tiledb/api/c_api/dimension/dimension_api_external.h +++ b/tiledb/api/c_api/dimension/dimension_api_external.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -62,9 +62,9 @@ typedef struct tiledb_dimension_handle_t tiledb_dimension_t; * * Note: as laid out in the Storage Format, * the following Datatypes are not valid for Dimension: - * TILEDB_CHAR, TILEDB_BLOB, TILEDB_BOOL, TILEDB_STRING_UTF8, - * TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, TILEDB_STRING_UCS2, - * TILEDB_STRING_UCS4, TILEDB_ANY + * TILEDB_CHAR, TILEDB_BLOB, TILEDB_GEOM_WKB, TILEDB_GEOM_WKT, TILEDB_BOOL, + * TILEDB_STRING_UTF8, TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, + * TILEDB_STRING_UCS2, TILEDB_STRING_UCS4, TILEDB_ANY * * @param ctx The TileDB context. * @param name The dimension name. diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 4e9f2d148f9e..947aaaca2476 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,8 +50,7 @@ tiledb::common::blank::blank() : tiledb::sm::Dimension{"", tiledb::sm::Datatype::INT32} { } -namespace tiledb { -namespace sm { +namespace tiledb::sm { class DimensionException : public StatusException { public: @@ -1555,6 +1554,8 @@ void Dimension::ensure_datatype_is_supported(Datatype type) const { switch (type) { case Datatype::CHAR: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::STRING_UTF8: case Datatype::STRING_UTF16: @@ -1795,5 +1796,4 @@ void Dimension::set_smaller_than_func() { apply_with_type(g, type_); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 791a8be99bff..0e2a0014e377 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -56,8 +56,7 @@ namespace tiledb::type { class Range; } -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Buffer; class ConstBuffer; @@ -70,9 +69,9 @@ enum class Datatype : uint8_t; * * Note: as laid out in the Storage Format, * the following Datatypes are not valid for Dimension: - * TILEDB_CHAR, TILEDB_BLOB, TILEDB_BOOL, TILEDB_STRING_UTF8, - * TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, TILEDB_STRING_UCS2, - * TILEDB_STRING_UCS4, TILEDB_ANY + * TILEDB_CHAR, TILEDB_BLOB, TILEDB_GEOM_WKB, TILEDB_GEOM_WKT, TILEDB_BOOL, + * TILEDB_STRING_UTF8, TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, + * TILEDB_STRING_UCS2, TILEDB_STRING_UCS4, TILEDB_ANY */ class Dimension { public: @@ -1085,8 +1084,7 @@ class Dimension { void set_smaller_than_func(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm namespace tiledb::common { template <> diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 6ae9ecc14150..139cca4ecac2 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -918,6 +918,8 @@ void Domain::set_tile_cell_order_cmp_funcs() { cell_order_cmp_func_2_[d] = nullptr; break; case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::CHAR: case Datatype::BOOL: case Datatype::STRING_UTF8: @@ -926,9 +928,8 @@ void Domain::set_tile_cell_order_cmp_funcs() { case Datatype::STRING_UCS2: case Datatype::STRING_UCS4: case Datatype::ANY: - tile_order_cmp_func_[d] = nullptr; - cell_order_cmp_func_[d] = nullptr; - cell_order_cmp_func_2_[d] = nullptr; + throw std::invalid_argument( + "Unsupported dimension datatype " + datatype_str(type)); } } } diff --git a/tiledb/sm/array_schema/test/unit_dimension.cc b/tiledb/sm/array_schema/test/unit_dimension.cc index a9df3c59c2c0..55e246c08b6c 100644 --- a/tiledb/sm/array_schema/test/unit_dimension.cc +++ b/tiledb/sm/array_schema/test/unit_dimension.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021 TileDB, Inc. + * @copyright Copyright (c) 2021-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -221,6 +221,8 @@ TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { std::vector valid_unsupported_datatypes = { Datatype::CHAR, Datatype::BLOB, + Datatype::GEOM_WKB, + Datatype::GEOM_WKT, Datatype::BOOL, Datatype::STRING_UTF8, Datatype::STRING_UTF16, @@ -241,7 +243,8 @@ TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { } SECTION("- invalid Datatypes") { - std::vector> invalid_datatypes = {42, 100}; + // Note: Ensure this test is updated each time a new datatype is added. + std::vector> invalid_datatypes = {44, 100}; for (auto type : invalid_datatypes) { try { diff --git a/tiledb/sm/compressors/dd_compressor.cc b/tiledb/sm/compressors/dd_compressor.cc index 92c31029a169..0010e2b05805 100644 --- a/tiledb/sm/compressors/dd_compressor.cc +++ b/tiledb/sm/compressors/dd_compressor.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,8 +43,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { const uint64_t DoubleDelta::OVERHEAD = 17; @@ -56,6 +55,8 @@ Status DoubleDelta::compress( Datatype type, ConstBuffer* input_buffer, Buffer* output_buffer) { switch (type) { case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: return DoubleDelta::compress(input_buffer, output_buffer); case Datatype::INT8: return DoubleDelta::compress(input_buffer, output_buffer); @@ -109,14 +110,14 @@ Status DoubleDelta::compress( return DoubleDelta::compress(input_buffer, output_buffer); case Datatype::FLOAT32: case Datatype::FLOAT64: - return LOG_STATUS(Status_CompressionError( - "Cannot compress tile with DoubleDelta; Float " - "datatypes are not supported")); + return LOG_STATUS( + Status_CompressionError("DoubleDelta tile compression is not yet " + "supported for float types.")); } assert(false); return LOG_STATUS(Status_CompressionError( - "Cannot compress tile with DoubleDelta; Not supported datatype")); + "Cannot compress tile with DoubleDelta; Unsupported datatype")); } Status DoubleDelta::decompress( @@ -125,6 +126,8 @@ Status DoubleDelta::decompress( PreallocatedBuffer* output_buffer) { switch (type) { case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: return DoubleDelta::decompress(input_buffer, output_buffer); case Datatype::INT8: return DoubleDelta::decompress(input_buffer, output_buffer); @@ -178,14 +181,14 @@ Status DoubleDelta::decompress( return DoubleDelta::decompress(input_buffer, output_buffer); case Datatype::FLOAT32: case Datatype::FLOAT64: - return LOG_STATUS(Status_CompressionError( - "Cannot decompress tile with DoubleDelta; Float " - "datatypes are not supported")); + return LOG_STATUS( + Status_CompressionError("DoubleDelta tile decompression is not yet " + "supported for float types.")); } assert(false); return LOG_STATUS(Status_CompressionError( - "Cannot decompress tile with DoubleDelta; Not supported datatype")); + "Cannot decompress tile with DoubleDelta; Unupported datatype")); } uint64_t DoubleDelta::overhead(uint64_t) { @@ -469,5 +472,4 @@ template Status DoubleDelta::decompress( template Status DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -}; // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/compressors/delta_compressor.cc b/tiledb/sm/compressors/delta_compressor.cc index 70c3794223ef..5f595f9f18c1 100644 --- a/tiledb/sm/compressors/delta_compressor.cc +++ b/tiledb/sm/compressors/delta_compressor.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,8 +43,14 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { + +class DeltaCompressorException : public StatusException { + public: + explicit DeltaCompressorException(const std::string& message) + : StatusException("DeltaCompressor", message) { + } +}; const uint64_t Delta::OVERHEAD = 17; @@ -56,6 +62,8 @@ void Delta::compress( Datatype type, ConstBuffer* input_buffer, Buffer* output_buffer) { switch (type) { case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: Delta::compress(input_buffer, output_buffer); break; case Datatype::INT8: @@ -121,12 +129,11 @@ void Delta::compress( break; case Datatype::FLOAT32: case Datatype::FLOAT64: - throw StatusException( - Status_CompressionError("Cannot compress tile with Delta; Float " - "datatypes are not supported")); + throw DeltaCompressorException( + "Compression is not yet supported for float datatypes."); default: - throw StatusException(Status_CompressionError( - "Cannot compress tile with Delta; Unsupported datatype")); + throw DeltaCompressorException( + "Compression failed; Unsupported datatype"); } } @@ -136,6 +143,8 @@ void Delta::decompress( PreallocatedBuffer* output_buffer) { switch (type) { case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: Delta::decompress(input_buffer, output_buffer); break; case Datatype::INT8: @@ -201,12 +210,11 @@ void Delta::decompress( break; case Datatype::FLOAT32: case Datatype::FLOAT64: - throw StatusException( - Status_CompressionError("Cannot decompress tile with Delta; Float " - "datatypes are not supported")); + throw DeltaCompressorException( + "Decompression is not yet supported for float datatypes."); default: - throw StatusException(Status_CompressionError( - "Cannot compress tile with Delta; Unsupported datatype")); + throw DeltaCompressorException( + "Decompression failed; Unsupported datatype"); } } @@ -303,5 +311,4 @@ template void Delta::decompress( template void Delta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -}; // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/cpp_api/arrow_io_impl.h b/tiledb/sm/cpp_api/arrow_io_impl.h index 0c56435573c2..c315e99ccd40 100644 --- a/tiledb/sm/cpp_api/arrow_io_impl.h +++ b/tiledb/sm/cpp_api/arrow_io_impl.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2020-2021 TileDB, Inc. + * @copyright Copyright (c) 2020-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -99,8 +99,7 @@ using _TileDBError = tiledb::TileDBError; #define TDB_LERROR tiledb::TileDBError #endif -namespace tiledb { -namespace arrow { +namespace tiledb::arrow { /* ****************************** */ /* Helper types */ @@ -250,6 +249,8 @@ ArrowInfo tiledb_buffer_arrow_fmt(BufferInfo bufferinfo, bool use_list = true) { case TILEDB_STRING_UCS2: case TILEDB_STRING_UCS4: case TILEDB_ANY: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: default: break; } @@ -845,10 +846,6 @@ void query_set_buffer_arrow_array( importer.import_(name, arw_array, arw_schema); } -} // end namespace arrow -} // end namespace tiledb - -/* End TileDB Arrow IO public API implementation */ -/* ************************************************************************ */ +} // namespace tiledb::arrow #endif // TILEDB_ARROW_H diff --git a/tiledb/sm/cpp_api/dimension.h b/tiledb/sm/cpp_api/dimension.h index c33561f86e24..bbca6f92031a 100644 --- a/tiledb/sm/cpp_api/dimension.h +++ b/tiledb/sm/cpp_api/dimension.h @@ -7,7 +7,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -63,9 +63,9 @@ namespace tiledb { * * Note: as laid out in the Storage Format, * the following Datatypes are not valid for Dimension: - * TILEDB_CHAR, TILEDB_BLOB, TILEDB_BOOL, TILEDB_STRING_UTF8, - * TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, TILEDB_STRING_UCS2, - * TILEDB_STRING_UCS4, TILEDB_ANY + * TILEDB_CHAR, TILEDB_BLOB, TILEDB_GEOM_WKB, TILEDB_GEOM_WKT, TILEDB_BOOL, + * TILEDB_STRING_UTF8, TILEDB_STRING_UTF16, TILEDB_STRING_UTF32, + * TILEDB_STRING_UCS2, TILEDB_STRING_UCS4, TILEDB_ANY **/ class Dimension { public: @@ -254,6 +254,8 @@ class Dimension { // representation return ""; case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: case TILEDB_CHAR: case TILEDB_BOOL: case TILEDB_STRING_UTF8: @@ -373,6 +375,8 @@ class Dimension { // representation return ""; case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: case TILEDB_CHAR: case TILEDB_BOOL: case TILEDB_STRING_UTF8: diff --git a/tiledb/sm/cpp_api/type.h b/tiledb/sm/cpp_api/type.h index 8f5edaee9eca..db7331bf5c3f 100644 --- a/tiledb/sm/cpp_api/type.h +++ b/tiledb/sm/cpp_api/type.h @@ -48,9 +48,7 @@ #include #include -namespace tiledb { - -namespace impl { +namespace tiledb::impl { /** Used to defer compilation of static_assert until type is known. **/ template @@ -183,6 +181,20 @@ struct tiledb_to_type { static constexpr const char* name = "BLOB"; }; +template <> +struct tiledb_to_type { + using type = std::byte; + static const tiledb_datatype_t tiledb_type = TILEDB_GEOM_WKB; + static constexpr const char* name = "GEOM_WKB"; +}; + +template <> +struct tiledb_to_type { + using type = std::byte; + static const tiledb_datatype_t tiledb_type = TILEDB_GEOM_WKT; + static constexpr const char* name = "GEOM_WKT"; +}; + template <> struct tiledb_to_type { using type = uint8_t; @@ -500,7 +512,6 @@ struct TypeHandler> { } }; -} // namespace impl -} // namespace tiledb +} // namespace tiledb::impl #endif // TILEDB_CPP_API_TYPE_H diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index dfa59ef3f420..8c21280b267b 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,8 +42,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { #if defined(_WIN32) && defined(TIME_MS) #pragma message("WARNING: Windows.h may have already been included before") @@ -78,6 +77,10 @@ inline uint64_t datatype_size(Datatype type) noexcept { return sizeof(char); case Datatype::BLOB: return sizeof(std::byte); + case Datatype::GEOM_WKB: + return sizeof(std::byte); + case Datatype::GEOM_WKT: + return sizeof(std::byte); case Datatype::BOOL: return sizeof(uint8_t); case Datatype::INT8: @@ -150,6 +153,10 @@ inline const std::string& datatype_str(Datatype type) { return constants::char_str; case Datatype::BLOB: return constants::blob_str; + case Datatype::GEOM_WKB: + return constants::geom_wkb_str; + case Datatype::GEOM_WKT: + return constants::geom_wkt_str; case Datatype::BOOL: return constants::bool_str; case Datatype::INT8: @@ -242,6 +249,10 @@ inline Status datatype_enum( *datatype = Datatype::CHAR; else if (datatype_str == constants::blob_str) *datatype = Datatype::BLOB; + else if (datatype_str == constants::geom_wkb_str) + *datatype = Datatype::GEOM_WKB; + else if (datatype_str == constants::geom_wkt_str) + *datatype = Datatype::GEOM_WKT; else if (datatype_str == constants::bool_str) *datatype = Datatype::BOOL; else if (datatype_str == constants::int8_str) @@ -397,28 +408,36 @@ inline bool datatype_is_time(Datatype type) { type == Datatype::TIME_AS); } +/** Returns true if the input datatype is a std::byte type. */ +inline bool datatype_is_byte(Datatype type) { + return ( + type == Datatype::BOOL || type == Datatype::GEOM_WKB || + type == Datatype::GEOM_WKT); +} + /** Returns true if the input datatype is a boolean type. */ inline bool datatype_is_boolean(Datatype type) { return (type == Datatype::BOOL); } -/** Throws error if the input Datatype's enum is not between 0 and 41. */ +/** Throws error if the input Datatype's enum is not between 0 and 43. */ inline void ensure_datatype_is_valid(uint8_t datatype_enum) { - if (datatype_enum > 41) { + if (datatype_enum > 43) { throw std::runtime_error( "Invalid Datatype (" + std::to_string(datatype_enum) + ")"); } } -/** Throws error if the input Datatype's enum is not between 0 and 41. */ +/** Throws error if the input Datatype's enum is not between 0 and 43. */ inline void ensure_datatype_is_valid(Datatype type) { ensure_datatype_is_valid(::stdx::to_underlying(type)); } -/** Throws error if: +/** + * Throws error if: * * the datatype string is not valid as a datatype. - * the datatype string's enum is not between 0 and 41. + * the datatype string's enum is not between 0 and 43. **/ inline void ensure_datatype_is_valid(const std::string& datatype_str) { Datatype datatype_type; @@ -521,7 +540,6 @@ inline void ensure_ordered_attribute_datatype_is_valid(Datatype type) { } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_DATATYPE_H diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index a411f0d33c7b..df47c6f05fd8 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,8 +41,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Compute the number of bits required to represent a signed integral value. */ template @@ -100,7 +99,7 @@ void BitWidthReductionFilter::dump(FILE* out) const { bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const { if (datatype_is_integer(datatype) || datatype_is_datetime(datatype) || - datatype_is_time(datatype) || datatype == Datatype::BLOB) { + datatype_is_time(datatype) || datatype_is_byte(datatype)) { return true; } return false; @@ -164,6 +163,8 @@ Status BitWidthReductionFilter::run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); case Datatype::INT8: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::UINT8: default: @@ -337,6 +338,8 @@ Status BitWidthReductionFilter::run_reverse( tile, offsets_tile, input_metadata, input, output_metadata, output); case Datatype::INT8: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::UINT8: default: @@ -564,5 +567,4 @@ void BitWidthReductionFilter::serialize_impl(Serializer& serializer) const { serializer.write(max_window_size_); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/filter_buffer.cc b/tiledb/sm/filter/filter_buffer.cc index 34d34b76d1b2..68373b5322e6 100644 --- a/tiledb/sm/filter/filter_buffer.cc +++ b/tiledb/sm/filter/filter_buffer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,8 +40,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { FilterBuffer::BufferOrView::BufferOrView(const shared_ptr& buffer) { underlying_buffer_ = buffer; @@ -211,6 +210,8 @@ std::vector FilterBuffer::buffers_as(Datatype datatype) const { case Datatype::FLOAT64: return buffers_as(); case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::UINT8: return buffers_as(); @@ -652,5 +653,4 @@ template std::vector FilterBuffer::buffers_as() const; template std::vector FilterBuffer::buffers_as() const; template std::vector FilterBuffer::buffers_as() const; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/positive_delta_filter.cc b/tiledb/sm/filter/positive_delta_filter.cc index 8287dc4ec861..9b469efe16c2 100644 --- a/tiledb/sm/filter/positive_delta_filter.cc +++ b/tiledb/sm/filter/positive_delta_filter.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,8 +41,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { PositiveDeltaFilter::PositiveDeltaFilter(Datatype filter_data_type) : Filter(FilterType::FILTER_POSITIVE_DELTA, filter_data_type) { @@ -63,7 +62,7 @@ void PositiveDeltaFilter::dump(FILE* out) const { bool PositiveDeltaFilter::accepts_input_datatype(Datatype datatype) const { if (datatype_is_integer(datatype) || datatype_is_datetime(datatype) || - datatype_is_time(datatype) || datatype == Datatype::BLOB) { + datatype_is_time(datatype) || datatype_is_byte(datatype)) { return true; } return false; @@ -77,13 +76,15 @@ Status PositiveDeltaFilter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { /* Note: Arithmetic operations cannot be performed on std::byte. - We will use uint8_t for the Datatype::BLOB case as it is the same size as + We will use uint8_t for the byte-type cases as it is the same size as std::byte and can have arithmetic performed on it. */ switch (filter_data_type_) { case Datatype::INT8: return run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::UINT8: return run_forward( @@ -254,13 +255,15 @@ Status PositiveDeltaFilter::run_reverse( FilterBuffer* output, const Config&) const { /* Note: Arithmetic operations cannot be performed on std::byte. - We will use uint8_t for the Datatype::BLOB case as it is the same size as + We will use uint8_t for the byte-type cases as it is the same size as std::byte and can have arithmetic perfomed on it. */ switch (filter_data_type_) { case Datatype::INT8: return run_reverse( tile, offsets_tile, input_metadata, input, output_metadata, output); case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::BOOL: case Datatype::UINT8: return run_reverse( @@ -419,5 +422,4 @@ void PositiveDeltaFilter::serialize_impl(Serializer& serializer) const { serializer.write(max_window_size_); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/unit_filter_pipeline.cc b/tiledb/sm/filter/test/unit_filter_pipeline.cc index 414074950199..f3c250085720 100644 --- a/tiledb/sm/filter/test/unit_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_filter_pipeline.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -185,6 +185,8 @@ TEST_CASE( CHECK( fp_with.use_tile_chunking(is_var_sized, version, Datatype::DATETIME_AS)); CHECK(fp_with.use_tile_chunking(is_var_sized, version, Datatype::BLOB)); + CHECK(fp_with.use_tile_chunking(is_var_sized, version, Datatype::GEOM_WKB)); + CHECK(fp_with.use_tile_chunking(is_var_sized, version, Datatype::GEOM_WKT)); CHECK(fp_with.use_tile_chunking(is_var_sized, version, Datatype::INT32)); CHECK(fp_with.use_tile_chunking(is_var_sized, version, Datatype::FLOAT64)); } @@ -238,6 +240,8 @@ TEST_CASE( CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::TIME_MS, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::DATETIME_AS, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::BLOB, version)); + CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::GEOM_WKB, version)); + CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::GEOM_WKT, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::INT32, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::FLOAT64, version)); } \ No newline at end of file diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 18b62a8691c2..b56698bbd3ae 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -64,8 +64,7 @@ using namespace tiledb::common; using namespace tiledb::type; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class FragmentMetadataStatusException : public StatusException { public: @@ -454,6 +453,8 @@ void FragmentMetadata::compute_fragment_min_max_sum_null_count() { compute_fragment_min_max_sum(name); break; case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: compute_fragment_min_max_sum(name); break; default: @@ -4801,5 +4802,4 @@ template std::byte FragmentMetadata::get_tile_max_as( template char FragmentMetadata::get_tile_max_as( const std::string& name, uint64_t tile_idx) const; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 5c9897940c5c..cadde0492afe 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,10 +51,7 @@ #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/misc/utils.h" -namespace tiledb { -namespace sm { - -namespace constants { +namespace tiledb::sm::constants { /** * Reduction factor (must be in [0.0, 1.0]) for the multi_range subarray @@ -187,6 +184,12 @@ const char empty_char = std::numeric_limits::min(); /** The special value for an empty blob. */ constexpr std::byte empty_blob{0}; +/** The special value for an empty geom_wkb. */ +constexpr std::byte empty_geom_wkb{0}; + +/** The special value for an empty geom_wkt. */ +constexpr std::byte empty_geom_wkt{0}; + /** The special value for an empty bool. */ const uint8_t empty_bool = 0; @@ -491,6 +494,12 @@ const std::string char_str = "CHAR"; /** The string representation for type blob. */ const std::string blob_str = "BLOB"; +/** The string representation for type geom_wkb. */ +const std::string geom_wkb_str = "GEOM_WKB"; + +/** The string representation for type geom_wkt. */ +const std::string geom_wkt_str = "GEOM_WKT"; + /** The string representation for type bool. */ const std::string bool_str = "BOOL"; @@ -797,6 +806,10 @@ const void* fill_value(Datatype type) { switch (type) { case Datatype::BLOB: return &constants::empty_blob; + case Datatype::GEOM_WKB: + return &constants::empty_geom_wkb; + case Datatype::GEOM_WKT: + return &constants::empty_geom_wkt; case Datatype::BOOL: return &constants::empty_bool; case Datatype::INT8: @@ -864,7 +877,5 @@ const void* fill_value(Datatype type) { } const std::string config_delimiter = ","; -} // namespace constants -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::constants diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 9555af422dbd..0cccd9f49987 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,9 +40,7 @@ #include "tiledb/common/common.h" -namespace tiledb { -namespace sm { - +namespace tiledb::sm { enum class Datatype : uint8_t; enum class Compressor : uint8_t; enum class SerializationType : uint8_t; @@ -489,6 +487,12 @@ extern const std::string char_str; /** The string representation for type blob. */ extern const std::string blob_str; +/** The string representation for type geom_wkb. */ +extern const std::string geom_wkb_str; + +/** The string representation for type geom_wkt. */ +extern const std::string geom_wkt_str; + /** The string representation for type bool. */ extern const std::string bool_str; @@ -785,9 +789,8 @@ extern const uint64_t s3_min_multipart_part_size; * global order writes intermediate chunks */ extern const std::string s3_multipart_buffering_dirname; -} // namespace constants -} // namespace sm -} // namespace tiledb +} // namespace constants +} // namespace tiledb::sm #endif // TILEDB_CONSTANTS_H diff --git a/tiledb/sm/misc/integral_type_casts.h b/tiledb/sm/misc/integral_type_casts.h index a82611765368..50a781e0b789 100644 --- a/tiledb/sm/misc/integral_type_casts.h +++ b/tiledb/sm/misc/integral_type_casts.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -96,12 +96,9 @@ template void safe_integral_cast_to_datatype( Source value, Datatype type, ByteVecValue& dest) { if (!datatype_is_integer(type)) { - throw std::invalid_argument("Datatype must be integral"); - } - - if (type == Datatype::BLOB) { throw std::invalid_argument( - "Datatype::BLOB not supported in integral conversion"); + "Unsupported datatype " + datatype_str(type) + + "; Datatype must be integral"); } switch (type) { @@ -133,7 +130,9 @@ void safe_integral_cast_to_datatype( dest.assign_as(safe_integral_cast(value)); return; default: - throw std::logic_error("Definitions of integral types are mismatched."); + throw std::logic_error( + "Definitions of integral types are mismatched on datatype " + + datatype_str(type)); } ::stdx::unreachable(); diff --git a/tiledb/sm/misc/parse_argument.cc b/tiledb/sm/misc/parse_argument.cc index 9ce6c6804668..a813cad7f50e 100644 --- a/tiledb/sm/misc/parse_argument.cc +++ b/tiledb/sm/misc/parse_argument.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -319,6 +319,8 @@ std::string to_str(const void* value, Datatype type) { ss << *(const int64_t*)value; break; case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: // For printing to string use unsigned int value ss << *(const uint8_t*)value; break; diff --git a/tiledb/sm/misc/test/unit_integral_type_casts.cc b/tiledb/sm/misc/test/unit_integral_type_casts.cc index 26bb9dd190a6..4b266fe0a489 100644 --- a/tiledb/sm/misc/test/unit_integral_type_casts.cc +++ b/tiledb/sm/misc/test/unit_integral_type_casts.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -222,10 +222,13 @@ TEST_CASE("util::safe_integral_cast_to_datatype", "[safe-integral-casts]") { TEST_CASE( "util::safe_integral_cast_to_datatype bad type", "[safe-integral-casts][error]") { + auto datatype = GENERATE( + Datatype::BLOB, + Datatype::GEOM_WKB, + Datatype::GEOM_WKT, + Datatype::STRING_ASCII); ByteVecValue bvv; - REQUIRE_THROWS(utils::safe_integral_cast_to_datatype(5, Datatype::BLOB, bvv)); - REQUIRE_THROWS( - utils::safe_integral_cast_to_datatype(5, Datatype::STRING_ASCII, bvv)); + REQUIRE_THROWS(utils::safe_integral_cast_to_datatype(5, datatype, bvv)); } template diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index bf240b6effbd..94403e57aaa9 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,8 +36,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { static inline bool supported_string_type(Datatype type) { return ( @@ -319,15 +318,16 @@ Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { // Ensure that the attribute type is valid. switch (type) { case Datatype::ANY: - return Status_QueryConditionError( - "Value node attribute type may not be of type 'ANY': " + field_name_); case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: case Datatype::STRING_UCS2: case Datatype::STRING_UCS4: + case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: return Status_QueryConditionError( - "Only ASCII and UTF-8 string types are currently supported." + - field_name_); + "Unsupported value node attribute type " + datatype_str(type) + + " on field " + field_name_); default: break; } @@ -592,5 +592,4 @@ void ASTNodeExpr::set_use_enumeration(bool use_enumeration) { } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 5ae5314b6d1d..80e02a5e9ce7 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021-2022 TileDB, Inc. + * @copyright Copyright (c) 2021-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,8 +51,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { QueryCondition::QueryCondition() { } @@ -1091,6 +1090,8 @@ void QueryCondition::apply_ast_node( } break; case Datatype::ANY: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: case Datatype::STRING_UCS2: @@ -1098,9 +1099,8 @@ void QueryCondition::apply_ast_node( default: throw std::runtime_error( "QueryCondition::apply_ast_node: Cannot perform query comparison; " - "Unsupported query " - "conditional type on " + - node->get_field_name()); + "Unsupported datatype " + + datatype_str(type) + " on " + node->get_field_name()); } return; @@ -1847,6 +1847,8 @@ void QueryCondition::apply_ast_node_dense( } break; case Datatype::ANY: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: case Datatype::STRING_UCS2: @@ -1854,9 +1856,8 @@ void QueryCondition::apply_ast_node_dense( default: throw std::runtime_error( "Cannot perform query comparison; Unsupported query conditional " - "type " - "on " + - node->get_field_name()); + "type " + + datatype_str(type) + " on " + node->get_field_name()); } } @@ -2621,6 +2622,8 @@ void QueryCondition::apply_ast_node_sparse( } break; case Datatype::ANY: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: case Datatype::STRING_UCS2: @@ -2767,5 +2770,4 @@ template Status QueryCondition::apply_sparse( const ArraySchema& array_schema, ResultTile&, std::vector&); template Status QueryCondition::apply_sparse( const ArraySchema& array_schema, ResultTile&, std::vector&); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 07a471dd4649..0d53549ac7e6 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021-2022 TileDB, Inc. + * @copyright Copyright (c) 2021-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -185,6 +185,33 @@ TEST_CASE( check_ast_str(query_condition.negated_condition(), "foo GE 62 61 72"); } +TEST_CASE("QueryCondition: Test blob type", "[QueryCondition][blob]") { + std::string field_name = "blob_attr"; + std::byte value{5}; + + QueryCondition query_condition; + REQUIRE(query_condition + .init( + std::string(field_name), + &value, + sizeof(value), + QueryConditionOp::LT) + .ok()); + + shared_ptr array_schema = make_shared(HERE()); + shared_ptr attr = + make_shared(HERE(), "blob_attr", Datatype::BLOB); + REQUIRE(array_schema->add_attribute(attr).ok()); + std::vector result_cell_slabs; + std::vector> frag_md; + + REQUIRE_THROWS_WITH( + query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1), + Catch::Matchers::ContainsSubstring( + "Cannot perform query comparison; Unsupported datatype " + + datatype_str(Datatype::BLOB))); +} + TEST_CASE( "QueryCondition: Test AST construction, basic", "[QueryCondition][ast][api]") { diff --git a/tiledb/sm/serialization/capnp_utils.h b/tiledb/sm/serialization/capnp_utils.h index 6162f96483c0..8bad689116b8 100644 --- a/tiledb/sm/serialization/capnp_utils.h +++ b/tiledb/sm/serialization/capnp_utils.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2019-2021 TileDB, Inc. + * @copyright Copyright (c) 2019-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -48,11 +48,12 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Attribute; +class Dimension; +} // namespace tiledb::sm -namespace serialization { +namespace tiledb::sm::serialization { /** Class for query status exceptions. */ class SerializationStatusException : public StatusException { @@ -143,14 +144,6 @@ inline URI deserialize_array_uri_to_absolute( return array_uri.join_path(uri); } -}; // namespace serialization -}; // namespace sm -}; // namespace tiledb - -namespace tiledb { -namespace sm { -class Dimension; -namespace serialization { namespace utils { /** @@ -187,6 +180,8 @@ Status set_capnp_array_ptr( case tiledb::sm::Datatype::STRING_ASCII: case tiledb::sm::Datatype::STRING_UTF8: case tiledb::sm::Datatype::BLOB: + case tiledb::sm::Datatype::GEOM_WKB: + case tiledb::sm::Datatype::GEOM_WKT: case tiledb::sm::Datatype::BOOL: case tiledb::sm::Datatype::UINT8: builder.setUint8(kj::arrayPtr(static_cast(ptr), size)); @@ -268,6 +263,8 @@ Status set_capnp_scalar( builder.setInt8(*static_cast(value)); break; case tiledb::sm::Datatype::BLOB: + case tiledb::sm::Datatype::GEOM_WKB: + case tiledb::sm::Datatype::GEOM_WKT: case tiledb::sm::Datatype::BOOL: case tiledb::sm::Datatype::UINT8: builder.setUint8(*static_cast(value)); @@ -377,6 +374,8 @@ Status copy_capnp_list( RETURN_NOT_OK(copy_capnp_list(reader.getInt8(), buffer)); break; case tiledb::sm::Datatype::BLOB: + case tiledb::sm::Datatype::GEOM_WKB: + case tiledb::sm::Datatype::GEOM_WKT: case tiledb::sm::Datatype::BOOL: case tiledb::sm::Datatype::UINT8: if (reader.hasUint8()) @@ -696,9 +695,7 @@ Status deserialize_coords( } } // namespace utils -} // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::serialization #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/tile/tile_metadata_generator.cc b/tiledb/sm/tile/tile_metadata_generator.cc index 9f3ca7d2b4e5..e7134f06d016 100644 --- a/tiledb/sm/tile/tile_metadata_generator.cc +++ b/tiledb/sm/tile/tile_metadata_generator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 WriterTileDB, Inc. + * @copyright Copyright (c) 2017-2024 WriterTileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -35,8 +35,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ****************************** */ /* STRUCTURED BINDINGS APIS */ @@ -222,10 +221,12 @@ bool TileMetadataGenerator::has_min_max_metadata( return false; } - // No min max for any, blob, or non ascii strings. + // No min max for any, byte-type, or non ascii strings. switch (type) { case Datatype::ANY: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::STRING_UTF8: case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: @@ -244,10 +245,12 @@ bool TileMetadataGenerator::has_sum_metadata( if (var_size || cell_val_num != 1) return false; - // No sum for any, blob, or non ascii strings. + // No sum for any, byte-type, or non ascii strings. switch (type) { case Datatype::ANY: case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: case Datatype::STRING_UTF8: case Datatype::STRING_UTF16: case Datatype::STRING_UTF32: @@ -358,6 +361,8 @@ void TileMetadataGenerator::process_cell_slab( process_cell_range(tile, start, end); break; case Datatype::BLOB: + case Datatype::GEOM_WKB: + case Datatype::GEOM_WKT: process_cell_range(tile, start, end); break; default: @@ -616,5 +621,4 @@ inline void TileMetadataGenerator::min_max_var( } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/storage_format/DIRECTORY.md b/tiledb/storage_format/DIRECTORY.md index ebc38e8a1fb9..064ae3629cad 100644 --- a/tiledb/storage_format/DIRECTORY.md +++ b/tiledb/storage_format/DIRECTORY.md @@ -20,6 +20,8 @@ Please note that this information is repeated in the C API, C++ API, and the Dim * TILEDB_CHAR * TILEDB_BLOB +* TILEDB_GEOM_WKB +* TILEDB_GEOM_WKT * TILEDB_BOOL * TILEDB_STRING_UTF8 * TILEDB_STRING_UTF16 From b49e3aa4774261ecc2b91b07d5fbe9dd33043b8c Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 25 Jan 2024 01:44:37 -0600 Subject: [PATCH 154/456] Remove dead code from FragmentInfo and FragmentMetadata (#4667) This just removes some unused constructor and assignment operators that are no longer used now that these classes are almost exclusively used with shared pointers. --- TYPE: NO_HISTORY DESC: Remove dead code from FragmentInfo and FragmentMetadata --- tiledb/sm/fragment/fragment_info.cc | 59 ------------------------- tiledb/sm/fragment/fragment_info.h | 25 ++--------- tiledb/sm/fragment/fragment_metadata.cc | 45 ------------------- tiledb/sm/fragment/fragment_metadata.h | 6 +-- 4 files changed, 5 insertions(+), 130 deletions(-) diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 41354d2166be..20e51afd2648 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -51,11 +51,6 @@ namespace tiledb::sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -FragmentInfo::FragmentInfo() - : resources_(nullptr) - , unconsolidated_metadata_num_(0) { -} - FragmentInfo::FragmentInfo(const URI& array_uri, ContextResources& resources) : array_uri_(array_uri) , config_(resources.config()) @@ -65,28 +60,6 @@ FragmentInfo::FragmentInfo(const URI& array_uri, ContextResources& resources) FragmentInfo::~FragmentInfo() = default; -FragmentInfo::FragmentInfo(const FragmentInfo& fragment_info) - : FragmentInfo() { - auto clone = fragment_info.clone(); - swap(clone); -} - -FragmentInfo::FragmentInfo(FragmentInfo&& fragment_info) - : FragmentInfo() { - swap(fragment_info); -} - -FragmentInfo& FragmentInfo::operator=(const FragmentInfo& fragment_info) { - auto clone = fragment_info.clone(); - swap(clone); - return *this; -} - -FragmentInfo& FragmentInfo::operator=(FragmentInfo&& fragment_info) { - swap(fragment_info); - return *this; -} - /* ********************************* */ /* API */ /* ********************************* */ @@ -1221,36 +1194,4 @@ Status FragmentInfo::replace( return Status::Ok(); } -FragmentInfo FragmentInfo::clone() const { - FragmentInfo clone; - clone.array_uri_ = array_uri_; - clone.array_schema_latest_ = array_schema_latest_; - clone.array_schemas_all_ = array_schemas_all_; - clone.config_ = config_; - clone.single_fragment_info_vec_ = single_fragment_info_vec_; - clone.resources_ = resources_; - clone.to_vacuum_ = to_vacuum_; - clone.unconsolidated_metadata_num_ = unconsolidated_metadata_num_; - clone.anterior_ndrange_ = anterior_ndrange_; - clone.timestamp_start_ = timestamp_start_; - clone.timestamp_end_ = timestamp_end_; - - return clone; -} - -void FragmentInfo::swap(FragmentInfo& fragment_info) { - std::swap(array_uri_, fragment_info.array_uri_); - std::swap(array_schema_latest_, fragment_info.array_schema_latest_); - std::swap(array_schemas_all_, fragment_info.array_schemas_all_); - std::swap(config_, fragment_info.config_); - std::swap(single_fragment_info_vec_, fragment_info.single_fragment_info_vec_); - std::swap(resources_, fragment_info.resources_); - std::swap(to_vacuum_, fragment_info.to_vacuum_); - std::swap( - unconsolidated_metadata_num_, fragment_info.unconsolidated_metadata_num_); - std::swap(anterior_ndrange_, fragment_info.anterior_ndrange_); - std::swap(timestamp_start_, fragment_info.timestamp_start_); - std::swap(timestamp_end_, fragment_info.timestamp_end_); -} - } // namespace tiledb::sm diff --git a/tiledb/sm/fragment/fragment_info.h b/tiledb/sm/fragment/fragment_info.h index e488af94c8a2..eb06049543ec 100644 --- a/tiledb/sm/fragment/fragment_info.h +++ b/tiledb/sm/fragment/fragment_info.h @@ -54,8 +54,7 @@ class FragmentInfo { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Constructor. */ - FragmentInfo(); + FragmentInfo() = delete; /** Constructor. */ FragmentInfo(const URI& array_uri, ContextResources& resources); @@ -63,17 +62,8 @@ class FragmentInfo { /** Destructor. */ ~FragmentInfo(); - /** Copy constructor. */ - FragmentInfo(const FragmentInfo& fragment_info); - - /** Move constructor. */ - FragmentInfo(FragmentInfo&& fragment_info); - - /** Copy-assign operator. */ - FragmentInfo& operator=(const FragmentInfo& fragment_info); - - /** Move-assign operator. */ - FragmentInfo& operator=(FragmentInfo&& fragment_info); + DISABLE_COPY_AND_COPY_ASSIGN(FragmentInfo); + DISABLE_MOVE_AND_MOVE_ASSIGN(FragmentInfo); /* ********************************* */ /* API */ @@ -484,15 +474,6 @@ class FragmentInfo { Status replace( const SingleFragmentInfo& new_single_fragment_info, const std::vector& to_replace); - - /** Returns a copy of this object. */ - FragmentInfo clone() const; - - /** - * Swaps the contents (all field values) of this object with the - * given object. - */ - void swap(FragmentInfo& fragment_info); }; } // namespace sm diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index b56698bbd3ae..aca8c928dcff 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -113,51 +113,6 @@ FragmentMetadata::FragmentMetadata( FragmentMetadata::~FragmentMetadata() = default; -// Copy initialization -FragmentMetadata::FragmentMetadata(const FragmentMetadata& other) { - resources_ = other.resources_; - array_schema_ = other.array_schema_; - dense_ = other.dense_; - fragment_uri_ = other.fragment_uri_; - timestamp_range_ = other.timestamp_range_; - has_consolidated_footer_ = other.has_consolidated_footer_; - rtree_ = other.rtree_; - meta_file_size_ = other.meta_file_size_; - version_ = other.version_; - tile_index_base_ = other.tile_index_base_; - has_timestamps_ = other.has_timestamps_; - has_delete_meta_ = other.has_delete_meta_; - sparse_tile_num_ = other.sparse_tile_num_; - footer_size_ = other.footer_size_; - footer_offset_ = other.footer_offset_; - idx_map_ = other.idx_map_; - array_schema_name_ = other.array_schema_name_; - array_uri_ = other.array_uri_; -} - -FragmentMetadata& FragmentMetadata::operator=(const FragmentMetadata& other) { - resources_ = other.resources_; - array_schema_ = other.array_schema_; - dense_ = other.dense_; - fragment_uri_ = other.fragment_uri_; - timestamp_range_ = other.timestamp_range_; - has_consolidated_footer_ = other.has_consolidated_footer_; - rtree_ = other.rtree_; - meta_file_size_ = other.meta_file_size_; - version_ = other.version_; - tile_index_base_ = other.tile_index_base_; - has_timestamps_ = other.has_timestamps_; - has_delete_meta_ = other.has_delete_meta_; - sparse_tile_num_ = other.sparse_tile_num_; - footer_size_ = other.footer_size_; - footer_offset_ = other.footer_offset_; - idx_map_ = other.idx_map_; - array_schema_name_ = other.array_schema_name_; - array_uri_ = other.array_uri_; - - return *this; -} - /* ****************************** */ /* API */ /* ****************************** */ diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 3b4e822b888f..2ef0ab30d416 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -100,10 +100,8 @@ class FragmentMetadata { /** Destructor. */ ~FragmentMetadata(); - // Copy initialization - FragmentMetadata(const FragmentMetadata& other); - - FragmentMetadata& operator=(const FragmentMetadata& other); + DISABLE_COPY_AND_COPY_ASSIGN(FragmentMetadata); + DISABLE_MOVE_AND_MOVE_ASSIGN(FragmentMetadata); /* ********************************* */ /* TYPE DEFINITIONS */ From 5804d004c6e42dd3a3163817e19bbfade424250f Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 25 Jan 2024 00:45:24 -0700 Subject: [PATCH 155/456] Remove dead code from `class StorageManager` (#4665) * Remove member variables and code to track open groups. The resulting container was not used for any purpose outside of `Debug` builds, and of very limited utility within them. * Remove unreferenced member variable `xlock_cv_`. This PR supports the goal of shrinking the storage manager. --- TYPE: NO_HISTORY DESC: Remove dead code from `class StorageManager` --- tiledb/sm/storage_manager/storage_manager.cc | 24 ++----------------- .../storage_manager_canonical.h | 13 ---------- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ab174b12674e..1f0bdc60f828 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -132,19 +132,12 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::group_close_for_reads(Group* group) { - assert(open_groups_.find(group) != open_groups_.end()); - - // Remove entry from open groups - std::lock_guard lock{open_groups_mtx_}; - open_groups_.erase(group); - +Status StorageManagerCanonical::group_close_for_reads(Group*) { + // Closing a group does nothing at present return Status::Ok(); } Status StorageManagerCanonical::group_close_for_writes(Group* group) { - assert(open_groups_.find(group) != open_groups_.end()); - // Flush the group metadata RETURN_NOT_OK(store_metadata( group->group_uri(), *group->encryption_key(), group->unsafe_metadata())); @@ -159,11 +152,6 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { group->group_details(), *group->encryption_key())); } - - // Remove entry from open groups - std::lock_guard lock{open_groups_mtx_}; - open_groups_.erase(group); - return Status::Ok(); } @@ -1738,10 +1726,6 @@ StorageManagerCanonical::group_open_for_reads(Group* group) { load_group_details(group->group_directory(), *group->encryption_key()); RETURN_NOT_OK_TUPLE(st, std::nullopt); - // Mark the array as now open - std::lock_guard lock{open_groups_mtx_}; - open_groups_.insert(group); - if (group_deserialized.has_value()) { return {Status::Ok(), group_deserialized.value()}; } @@ -1760,10 +1744,6 @@ StorageManagerCanonical::group_open_for_writes(Group* group) { load_group_details(group->group_directory(), *group->encryption_key()); RETURN_NOT_OK_TUPLE(st, std::nullopt); - // Mark the array as now open - std::lock_guard lock{open_groups_mtx_}; - open_groups_.insert(group); - if (group_deserialized.has_value()) { return {Status::Ok(), group_deserialized.value()}; } diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 78fef0fcb559..8ad416ac45c2 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -845,25 +845,12 @@ class StorageManagerCanonical { /** Mutex protecting cancellation_in_progress_. */ std::mutex cancellation_in_progress_mtx_; - /** - * The condition variable for exlcusively locking arrays. This is used - * to wait for an array to be closed, before being exclusively locked - * by `array_xlock`. - */ - std::condition_variable xlock_cv_; - /** Mutex for providing thread-safety upon creating TileDB objects. */ std::mutex object_create_mtx_; /** Stores the TileDB configuration parameters. */ Config config_; - /** Keeps track of which groups are open. */ - std::set open_groups_; - - /** Mutex for managing open groups. */ - std::mutex open_groups_mtx_; - /** Count of the number of queries currently in progress. */ uint64_t queries_in_progress_; From 798a939a6dc7c9dcad3d24eea4d5e50516786ce7 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Thu, 25 Jan 2024 04:47:27 -0500 Subject: [PATCH 156/456] Move consolidation calls out of StorageManager. (#4668) This moves consolidation out of StorageManager using static methods in `tiledb::sm::Consolidator`. The subclasses that derive from Consolidator will still need access to the StorageManager for opening the arrays and groups in the `consolidate` and `vacuum` overrides. --- TYPE: NO_HISTORY DESC: Move consolidation calls out of StorageManager. --- tiledb/sm/c_api/tiledb.cc | 25 +- .../sm/consolidator/array_meta_consolidator.h | 4 +- .../sm/consolidator/commits_consolidator.cc | 4 +- tiledb/sm/consolidator/commits_consolidator.h | 4 +- tiledb/sm/consolidator/consolidator.cc | 203 +++++++++++++- tiledb/sm/consolidator/consolidator.h | 79 ++++++ .../sm/consolidator/fragment_consolidator.cc | 2 +- .../sm/consolidator/fragment_consolidator.h | 4 +- .../consolidator/fragment_meta_consolidator.h | 4 +- .../sm/consolidator/group_meta_consolidator.h | 4 +- .../sm/query/writers/global_order_writer.cc | 8 +- tiledb/sm/storage_manager/storage_manager.cc | 247 ------------------ .../storage_manager_canonical.h | 85 ------ 13 files changed, 312 insertions(+), 361 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 93c240dce9b5..7bafd132752c 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -58,6 +58,7 @@ #include "tiledb/sm/c_api/api_argument_validator.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/config/config_iter.h" +#include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/cpp_api/core_interface.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" @@ -2702,13 +2703,13 @@ int32_t tiledb_array_create_with_key( int32_t tiledb_array_consolidate( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { api::ensure_config_is_valid_if_present(config); - throw_if_not_ok(ctx->storage_manager()->array_consolidate( + tiledb::sm::Consolidator::array_consolidate( array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0, - (config == nullptr) ? ctx->storage_manager()->config() : - config->config())); + (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + ctx->storage_manager()); return TILEDB_OK; } @@ -2721,13 +2722,13 @@ int32_t tiledb_array_consolidate_with_key( tiledb_config_t* config) { // Sanity checks - throw_if_not_ok(ctx->storage_manager()->array_consolidate( + tiledb::sm::Consolidator::array_consolidate( array_uri, static_cast(encryption_type), encryption_key, key_length, - (config == nullptr) ? ctx->storage_manager()->config() : - config->config())); + (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + ctx->storage_manager()); return TILEDB_OK; } @@ -2747,24 +2748,24 @@ int32_t tiledb_array_consolidate_fragments( uris.emplace_back(fragment_uris[i]); } - throw_if_not_ok(ctx->storage_manager()->fragments_consolidate( + tiledb::sm::Consolidator::fragments_consolidate( array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0, uris, - (config == nullptr) ? ctx->storage_manager()->config() : - config->config())); + (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + ctx->storage_manager()); return TILEDB_OK; } int32_t tiledb_array_vacuum( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { - ctx->storage_manager()->array_vacuum( + tiledb::sm::Consolidator::array_vacuum( array_uri, - (config == nullptr) ? ctx->storage_manager()->config() : - config->config()); + (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + ctx->storage_manager()); return TILEDB_OK; } diff --git a/tiledb/sm/consolidator/array_meta_consolidator.h b/tiledb/sm/consolidator/array_meta_consolidator.h index 74de41f9fe3a..6da4cc997485 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.h +++ b/tiledb/sm/consolidator/array_meta_consolidator.h @@ -85,14 +85,14 @@ class ArrayMetaConsolidator : public Consolidator { const char* array_name, EncryptionType encryption_type, const void* encryption_key, - uint32_t key_length); + uint32_t key_length) override; /** * Performs the vacuuming operation. * * @param array_name URI of array to consolidate. */ - void vacuum(const char* array_name); + void vacuum(const char* array_name) override; private: /* ********************************* */ diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index d81fe90cc804..b01167d93487 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -95,8 +95,8 @@ Status CommitsConsolidator::consolidate( // Get the file name. auto& to_consolidate = array_dir.commit_uris_to_consolidate(); - storage_manager_->write_consolidated_commits_file( - write_version, array_dir, to_consolidate); + Consolidator::write_consolidated_commits_file( + write_version, array_dir, to_consolidate, storage_manager_); return Status::Ok(); } diff --git a/tiledb/sm/consolidator/commits_consolidator.h b/tiledb/sm/consolidator/commits_consolidator.h index 0a6433c7c200..27ec1c4375b4 100644 --- a/tiledb/sm/consolidator/commits_consolidator.h +++ b/tiledb/sm/consolidator/commits_consolidator.h @@ -83,14 +83,14 @@ class CommitsConsolidator : public Consolidator { const char* array_name, EncryptionType encryption_type, const void* encryption_key, - uint32_t key_length); + uint32_t key_length) override; /** * Performs the vacuuming operation. * * @param array_name URI of array to consolidate. */ - void vacuum(const char* array_name); + void vacuum(const char* array_name) override; }; } // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 26851c2446c1..db2eba17e2c8 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -32,12 +32,16 @@ #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/common/logger.h" +#include "tiledb/common/stdx_string.h" #include "tiledb/sm/consolidator/array_meta_consolidator.h" #include "tiledb/sm/consolidator/commits_consolidator.h" #include "tiledb/sm/consolidator/fragment_consolidator.h" #include "tiledb/sm/consolidator/fragment_meta_consolidator.h" #include "tiledb/sm/consolidator/group_meta_consolidator.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/storage_format/uri/generate_uri.h" using namespace tiledb::common; @@ -121,12 +125,207 @@ Status Consolidator::consolidate( } void Consolidator::vacuum([[maybe_unused]] const char* array_name) { - throw Status_ConsolidatorError("Cannot vacuum; Invalid object"); + throw ConsolidatorException("Cannot vacuum; Invalid object"); +} + +void Consolidator::array_consolidate( + const char* array_name, + EncryptionType encryption_type, + const void* encryption_key, + uint32_t key_length, + const Config& config, + StorageManager* storage_manager) { + // Check array URI + URI array_uri(array_name); + if (array_uri.is_invalid()) { + throw ConsolidatorException("Cannot consolidate array; Invalid URI"); + } + + // Check if array exists + ObjectType obj_type; + throw_if_not_ok(storage_manager->object_type(array_uri, &obj_type)); + + if (obj_type != ObjectType::ARRAY) { + throw ConsolidatorException( + "Cannot consolidate array; Array does not exist"); + } + + if (array_uri.is_tiledb()) { + throw_if_not_ok(storage_manager->rest_client()->post_consolidation_to_rest( + array_uri, config)); + } else { + // Get encryption key from config + std::string encryption_key_from_cfg; + if (!encryption_key) { + bool found = false; + encryption_key_from_cfg = config.get("sm.encryption_key", &found); + assert(found); + } + + if (!encryption_key_from_cfg.empty()) { + encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); + std::string encryption_type_from_cfg; + bool found = false; + encryption_type_from_cfg = config.get("sm.encryption_type", &found); + assert(found); + auto [st, et] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st); + encryption_type = et.value(); + + if (!EncryptionKey::is_valid_key_length( + encryption_type, + static_cast(encryption_key_from_cfg.size()))) { + encryption_key = nullptr; + key_length = 0; + } + } + + // Consolidate + auto mode = Consolidator::mode_from_config(config); + auto consolidator = Consolidator::create(mode, config, storage_manager); + throw_if_not_ok(consolidator->consolidate( + array_name, encryption_type, encryption_key, key_length)); + } +} + +void Consolidator::fragments_consolidate( + const char* array_name, + EncryptionType encryption_type, + const void* encryption_key, + uint32_t key_length, + const std::vector fragment_uris, + const Config& config, + StorageManager* storage_manager) { + // Check array URI + URI array_uri(array_name); + if (array_uri.is_invalid()) { + throw ConsolidatorException("Cannot consolidate array; Invalid URI"); + } + + // Check if array exists + ObjectType obj_type; + throw_if_not_ok(storage_manager->object_type(array_uri, &obj_type)); + + if (obj_type != ObjectType::ARRAY) { + throw ConsolidatorException( + "Cannot consolidate array; Array does not exist"); + } + + // Get encryption key from config + std::string encryption_key_from_cfg; + if (!encryption_key) { + bool found = false; + encryption_key_from_cfg = config.get("sm.encryption_key", &found); + assert(found); + } + + if (!encryption_key_from_cfg.empty()) { + encryption_key = encryption_key_from_cfg.c_str(); + key_length = static_cast(encryption_key_from_cfg.size()); + std::string encryption_type_from_cfg; + bool found = false; + encryption_type_from_cfg = config.get("sm.encryption_type", &found); + assert(found); + auto [st, et] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st); + encryption_type = et.value(); + + if (!EncryptionKey::is_valid_key_length( + encryption_type, + static_cast(encryption_key_from_cfg.size()))) { + encryption_key = nullptr; + key_length = 0; + } + } + + // Consolidate + auto consolidator = Consolidator::create( + ConsolidationMode::FRAGMENT, config, storage_manager); + auto fragment_consolidator = + dynamic_cast(consolidator.get()); + throw_if_not_ok(fragment_consolidator->consolidate_fragments( + array_name, encryption_type, encryption_key, key_length, fragment_uris)); +} + +void Consolidator::write_consolidated_commits_file( + format_version_t write_version, + ArrayDirectory array_dir, + const std::vector& commit_uris, + StorageManager* storage_manager) { + // Compute the file name. + auto name = storage_format::generate_consolidated_fragment_name( + commit_uris.front(), commit_uris.back(), write_version); + + // Compute size of consolidated file. Save the sizes of the files to re-use + // below. + storage_size_t total_size = 0; + const auto base_uri_size = array_dir.uri().to_string().size(); + std::vector file_sizes(commit_uris.size()); + for (uint64_t i = 0; i < commit_uris.size(); i++) { + const auto& uri = commit_uris[i]; + total_size += uri.to_string().size() - base_uri_size + 1; + + // If the file is a delete, add the file size to the count and the size of + // the size variable. + if (stdx::string::ends_with( + uri.to_string(), constants::delete_file_suffix)) { + throw_if_not_ok(storage_manager->vfs()->file_size(uri, &file_sizes[i])); + total_size += file_sizes[i]; + total_size += sizeof(storage_size_t); + } + } + + // Write consolidated file, URIs are relative to the array URI. + std::vector data(total_size); + storage_size_t file_index = 0; + for (uint64_t i = 0; i < commit_uris.size(); i++) { + // Add the uri. + const auto& uri = commit_uris[i]; + std::string relative_uri = uri.to_string().substr(base_uri_size) + "\n"; + memcpy(&data[file_index], relative_uri.data(), relative_uri.size()); + file_index += relative_uri.size(); + + // For deletes, read the delete condition to the output file. + if (stdx::string::ends_with( + uri.to_string(), constants::delete_file_suffix)) { + memcpy(&data[file_index], &file_sizes[i], sizeof(storage_size_t)); + file_index += sizeof(storage_size_t); + throw_if_not_ok(storage_manager->vfs()->read( + uri, 0, &data[file_index], file_sizes[i])); + file_index += file_sizes[i]; + } + } + + // Write the file to storage. + URI consolidated_commits_uri = + array_dir.get_commits_dir(write_version) + .join_path(name + constants::con_commits_file_suffix); + throw_if_not_ok(storage_manager->vfs()->write( + consolidated_commits_uri, data.data(), data.size())); + throw_if_not_ok(storage_manager->vfs()->close_file(consolidated_commits_uri)); +} + +void Consolidator::array_vacuum( + const char* array_name, + const Config& config, + StorageManager* storage_manager) { + URI array_uri(array_name); + if (array_uri.is_tiledb()) { + throw_if_not_ok( + storage_manager->rest_client()->post_vacuum_to_rest(array_uri, config)); + return; + } + + auto mode = Consolidator::mode_from_config(config, true); + auto consolidator = Consolidator::create(mode, config, storage_manager); + consolidator->vacuum(array_name); } void Consolidator::check_array_uri(const char* array_name) { if (URI(array_name).is_tiledb()) { - throw std::logic_error("Consolidation is not supported for remote arrays."); + throw ConsolidatorException( + "Consolidation is not supported for remote arrays."); } } diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 529097e6885e..b11ca6120b35 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -46,6 +46,13 @@ using namespace tiledb::common; namespace tiledb::sm { +class ConsolidatorException : public StatusException { + public: + explicit ConsolidatorException(const std::string& msg) + : StatusException("Consolidator", msg) { + } +}; + class ArraySchema; class Config; class Query; @@ -125,6 +132,78 @@ class Consolidator { */ virtual void vacuum(const char* array_name); + /** + * Consolidates the fragments of an array into a single one. + * + * @param array_name The name of the array to be consolidated. + * @param encryption_type The encryption type of the array + * @param encryption_key If the array is encrypted, the private encryption + * key. For unencrypted arrays, pass `nullptr`. + * @param key_length The length in bytes of the encryption key. + * @param config Configuration parameters for the consolidation + * (`nullptr` means default, which will use the config associated with + * this instance). + * @param storage_manager The storage manager. + */ + static void array_consolidate( + const char* array_name, + EncryptionType encryption_type, + const void* encryption_key, + uint32_t key_length, + const Config& config, + StorageManager* storage_manager); + + /** + * Consolidates the fragments of an array into a single one. + * + * @param array_name The name of the array to be consolidated. + * @param encryption_type The encryption type of the array + * @param encryption_key If the array is encrypted, the private encryption + * key. For unencrypted arrays, pass `nullptr`. + * @param key_length The length in bytes of the encryption key. + * @param fragment_uris URIs of the fragments to consolidate. + * @param config Configuration parameters for the consolidation + * (`nullptr` means default, which will use the config associated with + * this instance). + * @param storage_manager The storage manager. + */ + static void fragments_consolidate( + const char* array_name, + EncryptionType encryption_type, + const void* encryption_key, + uint32_t key_length, + const std::vector fragment_uris, + const Config& config, + StorageManager* storage_manager); + + /** + * Writes a consolidated commits file. + * + * @param write_version Write version. + * @param array_dir ArrayDirectory where the data is stored. + * @param commit_uris Commit files to include. + * @param storage_manager The storage manager. + */ + static void write_consolidated_commits_file( + format_version_t write_version, + ArrayDirectory array_dir, + const std::vector& commit_uris, + StorageManager* storage_manager); + + /** + * Cleans up the array, such as its consolidated fragments and array + * metadata. Note that this will coarsen the granularity of time traveling + * (see docs for more information). + * + * @param array_name The name of the array to be vacuumed. + * @param config Configuration parameters for vacuuming. + * @param storage_manager The storage manager. + */ + static void array_vacuum( + const char* array_name, + const Config& config, + StorageManager* storage_manager); + /* ********************************* */ /* TYPE DEFINITIONS */ /* ********************************* */ diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 8cf8a0a6aa62..6b78edcf8910 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -184,7 +184,7 @@ FragmentConsolidator::FragmentConsolidator( : Consolidator(storage_manager) { auto st = set_config(config); if (!st.ok()) { - throw std::logic_error(st.message()); + throw FragmentConsolidatorException(st.message()); } } diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 0f6c0fc041d6..7375e644cee6 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -199,7 +199,7 @@ class FragmentConsolidator : public Consolidator { const char* array_name, EncryptionType encryption_type, const void* encryption_key, - uint32_t key_length); + uint32_t key_length) override; /** * Consolidates only the fragments of the input array using a list of @@ -229,7 +229,7 @@ class FragmentConsolidator : public Consolidator { * * @param array_name URI of array to vacuum. */ - void vacuum(const char* array_name); + void vacuum(const char* array_name) override; private: /* ********************************* */ diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.h b/tiledb/sm/consolidator/fragment_meta_consolidator.h index ff667f9e4ab6..230d3490f5b2 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.h +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.h @@ -83,14 +83,14 @@ class FragmentMetaConsolidator : public Consolidator { const char* array_name, EncryptionType encryption_type, const void* encryption_key, - uint32_t key_length); + uint32_t key_length) override; /** * Performs the vacuuming operation. * * @param array_name URI of array to consolidate. */ - void vacuum(const char* array_name); + void vacuum(const char* array_name) override; }; } // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/group_meta_consolidator.h b/tiledb/sm/consolidator/group_meta_consolidator.h index 82eef23e8b4e..e83618e925bb 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.h +++ b/tiledb/sm/consolidator/group_meta_consolidator.h @@ -83,14 +83,14 @@ class GroupMetaConsolidator : public Consolidator { const char* group_name, EncryptionType encryption_type, const void* encryption_key, - uint32_t key_length); + uint32_t key_length) override; /** * Performs the vacuuming operation. * * @param group_name URI of group to consolidate. */ - void vacuum(const char* group_name); + void vacuum(const char* group_name) override; private: /* ********************************* */ diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index cba606f6cd0c..5d943e792010 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -37,6 +37,7 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/fragment/fragment_metadata.h" #include "tiledb/sm/misc/comparators.h" #include "tiledb/sm/misc/hilbert.h" @@ -713,8 +714,11 @@ Status GlobalOrderWriter::finalize_global_write_state() { commit_uris.emplace_back(commit_uri); auto write_version = array_->array_schema_latest().write_version(); - storage_manager_->write_consolidated_commits_file( - write_version, array_->array_directory(), commit_uris); + Consolidator::write_consolidated_commits_file( + write_version, + array_->array_directory(), + commit_uris, + storage_manager_); } // Delete global write state diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 1f0bdc60f828..bae92437b4f5 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -155,181 +155,6 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { return Status::Ok(); } -Status StorageManagerCanonical::array_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const Config& config) { - // Check array URI - URI array_uri(array_name); - if (array_uri.is_invalid()) { - return logger_->status( - Status_StorageManagerError("Cannot consolidate array; Invalid URI")); - } - - // Check if array exists - ObjectType obj_type; - RETURN_NOT_OK(object_type(array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate array; Array does not exist")); - } - - if (array_uri.is_tiledb()) { - return rest_client()->post_consolidation_to_rest(array_uri, config); - } - - // Get encryption key from config - std::string encryption_key_from_cfg; - if (!encryption_key) { - bool found = false; - encryption_key_from_cfg = config.get("sm.encryption_key", &found); - assert(found); - } - - if (!encryption_key_from_cfg.empty()) { - encryption_key = encryption_key_from_cfg.c_str(); - key_length = static_cast(encryption_key_from_cfg.size()); - std::string encryption_type_from_cfg; - bool found = false; - encryption_type_from_cfg = config.get("sm.encryption_type", &found); - assert(found); - auto [st, et] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st); - encryption_type = et.value(); - - if (!EncryptionKey::is_valid_key_length( - encryption_type, - static_cast(encryption_key_from_cfg.size()))) { - encryption_key = nullptr; - key_length = 0; - } - } - - // Consolidate - auto mode = Consolidator::mode_from_config(config); - auto consolidator = Consolidator::create(mode, config, this); - return consolidator->consolidate( - array_name, encryption_type, encryption_key, key_length); -} - -Status StorageManagerCanonical::fragments_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const std::vector fragment_uris, - const Config& config) { - // Check array URI - URI array_uri(array_name); - if (array_uri.is_invalid()) { - return logger_->status( - Status_StorageManagerError("Cannot consolidate array; Invalid URI")); - } - - // Check if array exists - ObjectType obj_type; - RETURN_NOT_OK(object_type(array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate array; Array does not exist")); - } - - // Get encryption key from config - std::string encryption_key_from_cfg; - if (!encryption_key) { - bool found = false; - encryption_key_from_cfg = config.get("sm.encryption_key", &found); - assert(found); - } - - if (!encryption_key_from_cfg.empty()) { - encryption_key = encryption_key_from_cfg.c_str(); - key_length = static_cast(encryption_key_from_cfg.size()); - std::string encryption_type_from_cfg; - bool found = false; - encryption_type_from_cfg = config.get("sm.encryption_type", &found); - assert(found); - auto [st, et] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st); - encryption_type = et.value(); - - if (!EncryptionKey::is_valid_key_length( - encryption_type, - static_cast(encryption_key_from_cfg.size()))) { - encryption_key = nullptr; - key_length = 0; - } - } - - // Consolidate - auto consolidator = - Consolidator::create(ConsolidationMode::FRAGMENT, config, this); - auto fragment_consolidator = - dynamic_cast(consolidator.get()); - return fragment_consolidator->consolidate_fragments( - array_name, encryption_type, encryption_key, key_length, fragment_uris); -} - -void StorageManagerCanonical::write_consolidated_commits_file( - format_version_t write_version, - ArrayDirectory array_dir, - const std::vector& commit_uris) { - // Compute the file name. - auto name = storage_format::generate_consolidated_fragment_name( - commit_uris.front(), commit_uris.back(), write_version); - - // Compute size of consolidated file. Save the sizes of the files to re-use - // below. - storage_size_t total_size = 0; - const auto base_uri_size = array_dir.uri().to_string().size(); - std::vector file_sizes(commit_uris.size()); - for (uint64_t i = 0; i < commit_uris.size(); i++) { - const auto& uri = commit_uris[i]; - total_size += uri.to_string().size() - base_uri_size + 1; - - // If the file is a delete, add the file size to the count and the size of - // the size variable. - if (stdx::string::ends_with( - uri.to_string(), constants::delete_file_suffix)) { - throw_if_not_ok(vfs()->file_size(uri, &file_sizes[i])); - total_size += file_sizes[i]; - total_size += sizeof(storage_size_t); - } - } - - // Write consolidated file, URIs are relative to the array URI. - std::vector data(total_size); - storage_size_t file_index = 0; - for (uint64_t i = 0; i < commit_uris.size(); i++) { - // Add the uri. - const auto& uri = commit_uris[i]; - std::string relative_uri = uri.to_string().substr(base_uri_size) + "\n"; - memcpy(&data[file_index], relative_uri.data(), relative_uri.size()); - file_index += relative_uri.size(); - - // For deletes, read the delete condition to the output file. - if (stdx::string::ends_with( - uri.to_string(), constants::delete_file_suffix)) { - memcpy(&data[file_index], &file_sizes[i], sizeof(storage_size_t)); - file_index += sizeof(storage_size_t); - throw_if_not_ok(vfs()->read(uri, 0, &data[file_index], file_sizes[i])); - file_index += file_sizes[i]; - } - } - - // Write the file to storage. - URI consolidated_commits_uri = - array_dir.get_commits_dir(write_version) - .join_path(name + constants::con_commits_file_suffix); - throw_if_not_ok( - vfs()->write(consolidated_commits_uri, data.data(), data.size())); - throw_if_not_ok(vfs()->close_file(consolidated_commits_uri)); -} - void StorageManagerCanonical::delete_array(const char* array_name) { if (array_name == nullptr) { throw std::invalid_argument("[delete_array] Array name cannot be null"); @@ -430,78 +255,6 @@ void StorageManagerCanonical::delete_group(const char* group_name) { vfs()->remove_dirs(compute_tp(), dirs); } -void StorageManagerCanonical::array_vacuum( - const char* array_name, const Config& config) { - URI array_uri(array_name); - if (array_uri.is_tiledb()) { - throw_if_not_ok(rest_client()->post_vacuum_to_rest(array_uri, config)); - return; - } - - auto mode = Consolidator::mode_from_config(config, true); - auto consolidator = Consolidator::create(mode, config, this); - consolidator->vacuum(array_name); -} - -Status StorageManagerCanonical::array_metadata_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const Config& config) { - // Check array URI - URI array_uri(array_name); - if (array_uri.is_invalid()) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate array metadata; Invalid URI")); - } - // Check if array exists - ObjectType obj_type; - RETURN_NOT_OK(object_type(array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate array metadata; Array does not exist")); - } - - if (array_uri.is_tiledb()) { - return rest_client()->post_consolidation_to_rest(array_uri, config); - } - - // Get encryption key from config - std::string encryption_key_from_cfg; - if (!encryption_key) { - bool found = false; - encryption_key_from_cfg = config.get("sm.encryption_key", &found); - assert(found); - } - - if (!encryption_key_from_cfg.empty()) { - encryption_key = encryption_key_from_cfg.c_str(); - key_length = static_cast(encryption_key_from_cfg.size()); - std::string encryption_type_from_cfg; - bool found = false; - encryption_type_from_cfg = config.get("sm.encryption_type", &found); - assert(found); - auto [st, et] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st); - encryption_type = et.value(); - - if (!EncryptionKey::is_valid_key_length( - encryption_type, - static_cast(encryption_key_from_cfg.size()))) { - encryption_key = nullptr; - key_length = 0; - } - } - - // Consolidate - auto consolidator = - Consolidator::create(ConsolidationMode::ARRAY_META, config, this); - return consolidator->consolidate( - array_name, encryption_type, encryption_key, key_length); -} - Status StorageManagerCanonical::array_create( const URI& array_uri, const shared_ptr& array_schema, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 8ad416ac45c2..689f889fe420 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -247,60 +247,6 @@ class StorageManagerCanonical { std::tuple>> group_open_for_writes(Group* group); - /** - * Consolidates the fragments of an array into a single one. - * - * @param array_name The name of the array to be consolidated. - * @param encryption_type The encryption type of the array - * @param encryption_key If the array is encrypted, the private encryption - * key. For unencrypted arrays, pass `nullptr`. - * @param key_length The length in bytes of the encryption key. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config associated with - * this instance). - * @return Status - */ - Status array_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const Config& config); - - /** - * Consolidates the fragments of an array into a single one. - * - * @param array_name The name of the array to be consolidated. - * @param encryption_type The encryption type of the array - * @param encryption_key If the array is encrypted, the private encryption - * key. For unencrypted arrays, pass `nullptr`. - * @param key_length The length in bytes of the encryption key. - * @param fragment_uris URIs of the fragments to consolidate. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config associated with - * this instance). - * @return Status - */ - Status fragments_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const std::vector fragment_uris, - const Config& config); - - /** - * Writes a consolidated commits file. - * - * @param write_version Write version. - * @param array_dir ArrayDirectory where the data is stored. - * @param commit_uris Commit files to include. - */ - void write_consolidated_commits_file( - format_version_t write_version, - ArrayDirectory array_dir, - const std::vector& commit_uris); - /** * Cleans up the array data. * @@ -325,37 +271,6 @@ class StorageManagerCanonical { */ void delete_group(const char* group_name); - /** - * Cleans up the array, such as its consolidated fragments and array - * metadata. Note that this will coarsen the granularity of time traveling - * (see docs for more information). - * - * @param array_name The name of the array to be vacuumed. - * @param config Configuration parameters for vacuuming. - */ - void array_vacuum(const char* array_name, const Config& config); - - /** - * Consolidates the metadata of an array into a single file. - * - * @param array_name The name of the array whose metadata will be - * consolidated. - * @param encryption_type The encryption type of the array - * @param encryption_key If the array is encrypted, the private encryption - * key. For unencrypted arrays, pass `nullptr`. - * @param key_length The length in bytes of the encryption key. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config associated with - * this instance). - * @return Status - */ - Status array_metadata_consolidate( - const char* array_name, - EncryptionType encryption_type, - const void* encryption_key, - uint32_t key_length, - const Config& config); - /** * Creates a TileDB array storing its schema. * From 0df8863dfc54d2e1e60abc65ef6ec77b0468bd4b Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 25 Jan 2024 03:48:21 -0600 Subject: [PATCH 157/456] Fix unit_query_plan test (#4666) This test was failing when re-run after a failure because it does not attempt to remove the existing array directory that may exist. This just adds the obvious fix of removing the `query_plan_array` directory at the beginning of the test. --- TYPE: NO_HISTORY DESC: Fix unit_query_plan --- tiledb/sm/query_plan/test/unit_query_plan.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 45709aa8cf95..23ce164c092b 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -30,6 +30,7 @@ * This file tests the QueryPlan class */ +#include #include #include "../query_plan.h" #include "external/include/nlohmann/json.hpp" @@ -55,6 +56,10 @@ struct QueryPlanFx { void destroy_array(const std::shared_ptr& array); + URI array_uri(const std::string& uri); + + TemporaryLocalDirectory temp_dir_; + Config cfg_; shared_ptr logger_; ContextResources resources_; @@ -97,7 +102,10 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { void QueryPlanFx::destroy_array(const std::shared_ptr& array) { REQUIRE(array->close().ok()); - REQUIRE(sm_->vfs()->remove_dir(array->array_uri()).ok()); +} + +URI QueryPlanFx::array_uri(const std::string& array_name) { + return URI(temp_dir_.path() + array_name); } QueryPlanFx::QueryPlanFx() @@ -107,7 +115,7 @@ QueryPlanFx::QueryPlanFx() } TEST_CASE_METHOD(QueryPlanFx, "Query plan dump_json", "[query_plan][dump]") { - const URI uri = URI("query_plan_array"); + const URI uri = array_uri("query_plan_array"); auto array = create_array(uri); From ca11ed28ae051e3811a758eec25b88d1db79a9f6 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 25 Jan 2024 05:03:53 -0500 Subject: [PATCH 158/456] Integrate filesystem unit tests into a single test. (#4635) Begin work on integrating _all_ `unit-.cc` tests into a _single_ `unit-vfs.cc` test. Eventually, the individual tests will be removed entirely and the single test will exist solely as a _unit_ test in `tiledb/sm/filesystem/test`. This work begins that transition. --- TYPE: NO_HISTORY DESC: Integrate filesystem unit tests into a single test. --- test/src/unit-azure.cc | 137 +-------- test/src/unit-gcs.cc | 133 +-------- test/src/unit-hdfs-filesystem.cc | 38 +-- test/src/unit-s3.cc | 127 -------- test/src/unit-vfs.cc | 479 +++++++++++++++++++------------ 5 files changed, 298 insertions(+), 616 deletions(-) diff --git a/test/src/unit-azure.cc b/test/src/unit-azure.cc index cb0efff4dcdc..f0ad9fbdc361 100644 --- a/test/src/unit-azure.cc +++ b/test/src/unit-azure.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -131,146 +131,11 @@ std::string AzureFx::random_container_name(const std::string& prefix) { return ss.str(); } -TEST_CASE_METHOD(AzureFx, "Test Azure filesystem, file management", "[azure]") { - Config config; - REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok()); - - auto settings = - GENERATE(from_range(test_settings.begin(), test_settings.end())); - init_azure(std::move(config), settings); - - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that container is empty - bool is_empty; - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(is_empty); - - // Continue building the hierarchy - bool is_blob = false; - REQUIRE(azure_.touch(URI(file1)).ok()); - REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file2)).ok()); - REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file3)).ok()); - REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file4)).ok()); - REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.touch(URI(file5)).ok()); - REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok()); - REQUIRE(is_blob); - - // Check that container is not empty - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(!is_empty); - - // Check invalid file - REQUIRE(azure_.is_blob(URI(TEST_DIR + "foo"), &is_blob).ok()); - REQUIRE(!is_blob); - - // List with prefix - std::vector paths; - REQUIRE(azure_.ls(URI(TEST_DIR), &paths).ok()); - REQUIRE(paths.size() == 3); - paths.clear(); - REQUIRE(azure_.ls(URI(dir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(azure_.ls(URI(subdir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - REQUIRE(azure_.is_dir(URI(file1), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(azure_.is_dir(URI(file4), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - REQUIRE(azure_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(azure_.write(URI(file3), s.data(), s.size()).ok()); - REQUIRE(azure_.flush_blob(URI(file3)).ok()); - - auto&& [status, rv] = azure_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - REQUIRE(azure_.move_object(URI(file5), URI(file6)).ok()); - REQUIRE(azure_.is_blob(URI(file5), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file6), &is_blob).ok()); - REQUIRE(is_blob); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Move directory - REQUIRE(azure_.move_dir(URI(dir), URI(dir2)).ok()); - REQUIRE(azure_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(!is_dir); - REQUIRE(azure_.is_dir(URI(dir2), &is_dir).ok()); - REQUIRE(is_dir); - paths.clear(); - REQUIRE(azure_.ls(AZURE_CONTAINER, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Remove files - REQUIRE(azure_.remove_blob(URI(file4)).ok()); - REQUIRE(azure_.is_blob(URI(file4), &is_blob).ok()); - REQUIRE(!is_blob); - - // Remove directories - REQUIRE(azure_.remove_dir(URI(dir2)).ok()); - REQUIRE(azure_.is_blob(URI(file1), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file2), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(file3), &is_blob).ok()); - REQUIRE(!is_blob); -} - TEST_CASE_METHOD( AzureFx, "Test Azure filesystem, file I/O", "[azure][multipart]") { Config config; const uint64_t max_parallel_ops = 2; const uint64_t block_list_block_size = 4 * 1024 * 1024; - REQUIRE(config.set("vfs.azure.use_block_list_upload", "true").ok()); REQUIRE( config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops)) .ok()); diff --git a/test/src/unit-gcs.cc b/test/src/unit-gcs.cc index a897f965d34e..e8065a9e665a 100644 --- a/test/src/unit-gcs.cc +++ b/test/src/unit-gcs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -120,137 +120,6 @@ TEST_CASE_METHOD(GCSFx, "Test GCS init", "[gcs]") { } } -TEST_CASE_METHOD(GCSFx, "Test GCS filesystem, file management", "[gcs]") { - Config config; - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - init_gcs(std::move(config)); - - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that bucket is empty - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); - - // Continue building the hierarchy - bool is_object = false; - REQUIRE(gcs_.touch(URI(file1)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file2)).ok()); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file3)).ok()); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file5)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(is_object); - - // Check that bucket is not empty - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(!is_empty); - - // Check invalid file - REQUIRE(gcs_.is_object(URI(TEST_DIR + "foo"), &is_object).ok()); - REQUIRE(!is_object); - - // List with prefix - std::vector paths; - REQUIRE(gcs_.ls(URI(TEST_DIR), &paths).ok()); - REQUIRE(paths.size() == 3); - paths.clear(); - REQUIRE(gcs_.ls(URI(dir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(URI(subdir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - REQUIRE(gcs_.is_dir(URI(file1), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(file4), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - REQUIRE(gcs_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(gcs_.write(URI(file3), s.data(), s.size()).ok()); - REQUIRE(gcs_.flush_object(URI(file3)).ok()); - - auto&& [status, rv] = gcs_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - REQUIRE(gcs_.move_object(URI(file5), URI(file6)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file6), &is_object).ok()); - REQUIRE(is_object); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Move directory - REQUIRE(gcs_.move_dir(URI(dir), URI(dir2)).ok()); - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(!is_dir); - REQUIRE(gcs_.is_dir(URI(dir2), &is_dir).ok()); - REQUIRE(is_dir); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Remove files - REQUIRE(gcs_.remove_object(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(!is_object); - - // Remove directories - REQUIRE(gcs_.remove_dir(URI(dir2)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(!is_object); -} - TEST_CASE_METHOD( GCSFx, "Test GCS filesystem I/O, multipart, serial", diff --git a/test/src/unit-hdfs-filesystem.cc b/test/src/unit-hdfs-filesystem.cc index 24ff6c0ae4f6..4715ccb098f3 100644 --- a/test/src/unit-hdfs-filesystem.cc +++ b/test/src/unit-hdfs-filesystem.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2023 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -118,42 +118,6 @@ TEST_CASE("Test HDFS filesystem", "[hdfs]") { } CHECK(allok); - std::vector paths; - st = hdfs.ls(URI("hdfs:///"), &paths); - CHECK(st.ok()); - CHECK(paths.size() > 0); - - // ls_with_sizes - // Dir structure: - // ...../subdir - // ...../subdir/file - // ...../subdir/subsubdir - - std::string subdir = "hdfs://localhost:9000/tiledb_test/subdir"; - std::string file = subdir + "/file"; - std::string subsubdir = subdir + "/subsubdir"; - - CHECK(hdfs.create_dir(URI(subdir)).ok()); - CHECK(hdfs.create_dir(URI(subsubdir)).ok()); - CHECK(hdfs.touch(URI(file)).ok()); - - std::string s = "abcdef"; - CHECK(hdfs.write(URI(file), s.data(), s.size()).ok()); - - auto&& [status, rv] = hdfs.ls_with_sizes(URI(subdir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file); - CHECK(children[1].path().native() == subsubdir.substr(0, subsubdir.size())); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - // Cleanup - CHECK(hdfs.remove_dir(URI(subdir)).ok()); - uint64_t nbytes = 0; st = hdfs.file_size(URI("hdfs:///tiledb_test/tiledb_test_file"), &nbytes); CHECK(st.ok()); diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index bc24ffc37418..a6e4d05eff27 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -105,133 +105,6 @@ Config S3Fx::set_config_params() { return config; } -TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file management", "[s3]") { - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that bucket is empty - bool is_empty; - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); - CHECK(is_empty); - - // Continue building the hierarchy - bool exists = false; - CHECK(s3_.touch(URI(file1)).ok()); - CHECK(s3_.is_object(URI(file1), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file2)).ok()); - CHECK(s3_.is_object(URI(file2), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file3)).ok()); - CHECK(s3_.is_object(URI(file3), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file4)).ok()); - CHECK(s3_.is_object(URI(file4), &exists).ok()); - CHECK(exists); - CHECK(s3_.touch(URI(file5)).ok()); - CHECK(s3_.is_object(URI(file5), &exists).ok()); - CHECK(exists); - - // Check that the bucket is not empty - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); - CHECK(!is_empty); - - // Check invalid file - CHECK(s3_.is_object(URI(TEST_DIR + "foo"), &exists).ok()); - CHECK(!exists); - - // List with prefix - std::vector paths; - CHECK(s3_.ls(URI(TEST_DIR), &paths).ok()); - CHECK(paths.size() == 3); - paths.clear(); - CHECK(s3_.ls(URI(dir), &paths).ok()); - CHECK(paths.size() == 2); - paths.clear(); - CHECK(s3_.ls(URI(subdir), &paths).ok()); - CHECK(paths.size() == 2); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - CHECK(s3_.is_dir(URI(file1), &is_dir).ok()); - CHECK(!is_dir); // Not a dir - CHECK(s3_.is_dir(URI(file4), &is_dir).ok()); - CHECK(!is_dir); // Not a dir - CHECK(s3_.is_dir(URI(dir), &is_dir).ok()); - CHECK(is_dir); // This is viewed as a dir - CHECK(s3_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - CHECK(is_dir); // This is viewed as a dir - - // ls_with_sizes - std::string s = "abcdef"; - CHECK(s3_.write(URI(file3), s.data(), s.size()).ok()); - CHECK(s3_.flush_object(URI(file3)).ok()); - - auto&& [status, rv] = s3_.ls_with_sizes(URI(dir)); - auto children = *rv; - REQUIRE(status.ok()); - - REQUIRE(children.size() == 2); - CHECK(children[0].path().native() == file3); - CHECK(children[1].path().native() == subdir.substr(0, subdir.size() - 1)); - - CHECK(children[0].file_size() == s.size()); - // Directories don't get a size - CHECK(children[1].file_size() == 0); - - // Move file - CHECK(s3_.move_object(URI(file5), URI(file6)).ok()); - CHECK(s3_.is_object(URI(file5), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file6), &exists).ok()); - CHECK(exists); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Move directory - CHECK(s3_.move_dir(URI(dir), URI(dir2)).ok()); - CHECK(s3_.is_dir(URI(dir), &is_dir).ok()); - CHECK(!is_dir); - CHECK(s3_.is_dir(URI(dir2), &is_dir).ok()); - CHECK(is_dir); - paths.clear(); - CHECK(s3_.ls(S3_BUCKET, &paths, "").ok()); // No delimiter - CHECK(paths.size() == 5); - - // Remove files - CHECK(s3_.remove_object(URI(file4)).ok()); - CHECK(s3_.is_object(URI(file4), &exists).ok()); - CHECK(!exists); - - // Remove directories - CHECK(s3_.remove_dir(URI(dir2)).ok()); - CHECK(s3_.is_object(URI(file1), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file2), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(file3), &exists).ok()); - CHECK(!exists); -} - TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { // Prepare buffers uint64_t buffer_size = 5 * 1024 * 1024; diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 189bbe384ce3..fe70a04dd7a5 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -31,215 +31,223 @@ */ #include -#include #include "test/support/src/helpers.h" +#include "test/support/src/temporary_local_directory.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/filesystem/vfs.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" #endif -#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/tile/tile.h" +#include + using namespace tiledb::common; using namespace tiledb::sm; using namespace tiledb::test; -#ifdef _WIN32 +// The unique local directory object +tiledb::sm::TemporaryLocalDirectory unit_vfs_dir_{"tiledb_test"}; + +void require_tiledb_ok(Status st) { + REQUIRE(st.ok()); +} + +void require_tiledb_err(Status st) { + REQUIRE(!st.ok()); +} + +Config set_config_params() { + Config config; + + if constexpr (tiledb::sm::filesystem::gcs_enabled) { + require_tiledb_ok(config.set("vfs.gcs.project_id", "TODO")); + } -TEST_CASE("VFS: Test long paths (Win32)", "[vfs][windows]") { + if constexpr (tiledb::sm::filesystem::s3_enabled) { + require_tiledb_ok(config.set("vfs.s3.endpoint_override", "localhost:9999")); + require_tiledb_ok(config.set("vfs.s3.scheme", "https")); + require_tiledb_ok(config.set("vfs.s3.use_virtual_addressing", "false")); + require_tiledb_ok(config.set("vfs.s3.verify_ssl", "false")); + } + + if constexpr (tiledb::sm::filesystem::azure_enabled) { + require_tiledb_ok( + config.set("vfs.azure.storage_account_name", "devstoreaccount1")); + require_tiledb_ok(config.set( + "vfs.azure.storage_account_key", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==")); + require_tiledb_ok(config.set( + "vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1")); + // Currently disabled because it does not work with the Azurite emulator + // The SAS path was manually tested against the Azure Blob Service. + // require_tiledb_ok(config.set( + // "vfs.azure.storage_account_name", "devstoreaccount2")); + // require_tiledb_ok(config.set("vfs.azure.storage_sas_token", "")); + // require_tiledb_ok(config.set( + // "vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount2")); + } + + return config; +} + +TEST_CASE("VFS: Test long local paths", "[vfs]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); - VFS vfs_long_path_win{&g_helper_stats, &compute_tp, &io_tp, Config{}}; - std::string tmpdir_base = tiledb::sm::Win::current_dir() + "\\tiledb_test\\"; - REQUIRE(vfs_long_path_win.create_dir(URI(tmpdir_base)).ok()); + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, Config{}}; SECTION("- Deep hierarchy") { - // On some Windows platforms, the path length of a directory must be <= 248 - // chars. On others (that have opted in to a configuration that allows - // long paths) the limit is ~32,767. Here we check for either case. - std::string tmpdir = tmpdir_base; + // Create a nested path with a long total length + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string tmpdir = local_prefix + unit_vfs_dir_.path(); bool success = true; while (tmpdir.size() < 512) { - tmpdir += "subdir\\"; - success &= vfs_long_path_win.create_dir(URI(tmpdir)).ok(); + tmpdir += "subdir/"; + success &= vfs.create_dir(URI(tmpdir)).ok(); + if constexpr (tiledb::sm::filesystem::posix_enabled) { + REQUIRE(success); + } } + // On some Windows platforms, the path length of a directory must be <= 248 + // chars. On others (that have opted in to a configuration that allows + // long paths) the limit is ~32,767. Here we check for either case. if (success) { // Check we can create files within the deep hierarchy URI testfile(tmpdir + "file.txt"); REQUIRE(!testfile.is_invalid()); bool exists = false; - REQUIRE(vfs_long_path_win.is_file(testfile, &exists).ok()); - if (exists) - REQUIRE(vfs_long_path_win.remove_file(testfile).ok()); - REQUIRE(vfs_long_path_win.touch(testfile).ok()); - REQUIRE(vfs_long_path_win.remove_file(testfile).ok()); + require_tiledb_ok(vfs.is_file(testfile, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_file(testfile)); + } + require_tiledb_ok(vfs.touch(testfile)); + require_tiledb_ok(vfs.remove_file(testfile)); } else { // Don't check anything; directory creation failed. } } SECTION("- Too long name") { + // This may not be long enough on some filesystems to pass the fail check. std::string name; - for (unsigned i = 0; i < 256; i++) + for (unsigned i = 0; i < 256; i++) { name += "x"; + } + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string tmpdir = local_prefix + unit_vfs_dir_.path(); + URI testfile(tmpdir + name); + + // Creating the URI and checking its existence is fine on posix + if constexpr (tiledb::sm::filesystem::posix_enabled) { + REQUIRE(!testfile.is_invalid()); + bool exists = false; + require_tiledb_ok(vfs.is_file(testfile, &exists)); + + // Creating the file is not + require_tiledb_err(vfs.touch(testfile)); + } // Creating the URI is invalid on Win32 (failure to canonicalize path) - URI testfile(tmpdir_base + name); - REQUIRE(testfile.is_invalid()); + if constexpr (tiledb::sm::filesystem::windows_enabled) { + REQUIRE(testfile.is_invalid()); + } } - - REQUIRE(vfs_long_path_win.remove_dir(URI(tmpdir_base)).ok()); } -#else - -TEST_CASE("VFS: Test long posix paths", "[vfs]") { +TEST_CASE("VFS: URI semantics and file management", "[vfs][uri]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); - VFS vfs_long_path_pos{&g_helper_stats, &compute_tp, &io_tp, Config{}}; - - std::string tmpdir_base = Posix::current_dir() + "/tiledb_test/"; - REQUIRE(vfs_long_path_pos.create_dir(URI(tmpdir_base)).ok()); - - SECTION("- Deep hierarchy") { - // Create a nested path with a long total length - std::string tmpdir = tmpdir_base; - while (tmpdir.size() < 512) { - tmpdir += "subdir/"; - REQUIRE(vfs_long_path_pos.create_dir(URI(tmpdir)).ok()); + Config config = set_config_params(); + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; + + // Sections to test each enabled filesystem + URI path; + std::string local_prefix = ""; + SECTION("Filesystem: Local") { + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; } - - // Check we can create files within the deep hierarchy - URI testfile("file://" + tmpdir + "file.txt"); - REQUIRE(!testfile.is_invalid()); - bool exists = false; - REQUIRE(vfs_long_path_pos.is_file(testfile, &exists).ok()); - if (exists) - REQUIRE(vfs_long_path_pos.remove_file(testfile).ok()); - REQUIRE(vfs_long_path_pos.touch(testfile).ok()); - REQUIRE(vfs_long_path_pos.remove_file(testfile).ok()); + path = URI(local_prefix + unit_vfs_dir_.path()); } - SECTION("- Too long name") { - // This may not be long enough on some filesystems to pass the fail check. - std::string name; - for (unsigned i = 0; i < 256; i++) - name += "x"; - - // Creating the URI and checking its existence is fine - URI testfile("file://" + tmpdir_base + name); - REQUIRE(!testfile.is_invalid()); - bool exists = false; - REQUIRE(vfs_long_path_pos.is_file(testfile, &exists).ok()); - - // Creating the file is not - REQUIRE(!vfs_long_path_pos.touch(testfile).ok()); + if constexpr (tiledb::sm::filesystem::gcs_enabled) { + SECTION("Filesystem: GCS") { + path = URI("gcs://vfs-" + random_label() + "/"); + } } - REQUIRE(vfs_long_path_pos.remove_dir(URI(tmpdir_base)).ok()); -} - -#endif - -TEST_CASE("VFS: URI semantics", "[vfs][uri]") { - ThreadPool compute_tp(4); - ThreadPool io_tp(4); - std::vector> root_pairs; - if constexpr (tiledb::sm::filesystem::s3_enabled) { - Config config; - REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok()); - REQUIRE(config.set("vfs.s3.scheme", "https").ok()); - REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok()); - REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok()); - - root_pairs.emplace_back( - URI("s3://vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: S3") { + path = URI("s3://vfs-" + random_label() + "/"); + } } + if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - Config config; - root_pairs.emplace_back( - URI("hdfs:///vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: HDFS") { + path = URI("hdfs:///vfs-" + random_label() + "/"); + } } + if constexpr (tiledb::sm::filesystem::azure_enabled) { - Config config; - REQUIRE( - config.set("vfs.azure.storage_account_name", "devstoreaccount1").ok()); - REQUIRE(config - .set( - "vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4" - "I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw==") - .ok()); - REQUIRE(config - .set( - "vfs.azure.blob_endpoint", - "http://127.0.0.1:10000/devstoreaccount1") - .ok()); - - root_pairs.emplace_back( - URI("azure://vfs-" + random_label() + "/"), std::move(config)); + SECTION("Filesystem: Azure") { + path = URI("azure://vfs-" + random_label() + "/"); + } } - Config config; -#ifdef _WIN32 - root_pairs.emplace_back( - URI(tiledb::sm::Win::current_dir() + "\\vfs-" + random_label() + "\\"), - std::move(config)); -#else - root_pairs.emplace_back( - URI(Posix::current_dir() + "/vfs-" + random_label() + "/"), - std::move(config)); -#endif - - for (const auto& root_pair : root_pairs) { - const URI& root = root_pair.first; - const Config& config = root_pair.second; - - VFS vfs_uri{&g_helper_stats, &compute_tp, &io_tp, config}; - - bool exists = false; - if (root.is_s3() || root.is_azure()) { - REQUIRE(vfs_uri.is_bucket(root, &exists).ok()); - if (exists) { - REQUIRE(vfs_uri.remove_bucket(root).ok()); - } - REQUIRE(vfs_uri.create_bucket(root).ok()); - } else { - REQUIRE(vfs_uri.is_dir(root, &exists).ok()); - if (exists) { - REQUIRE(vfs_uri.remove_dir(root).ok()); - } - REQUIRE(vfs_uri.create_dir(root).ok()); + // Set up + bool exists = false; + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.is_bucket(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_bucket(path)); } + require_tiledb_ok(vfs.create_bucket(path)); + } else { + require_tiledb_ok(vfs.is_dir(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_dir(path)); + } + require_tiledb_ok(vfs.create_dir(path)); + } - std::string dir1 = root.to_string() + "dir1"; - REQUIRE(vfs_uri.create_dir(URI(dir1)).ok()); - - std::string dir2 = root.to_string() + "dir1/dir2/"; - REQUIRE(vfs_uri.create_dir(URI(dir2)).ok()); - - URI file1(root.to_string() + "file1"); - REQUIRE(vfs_uri.touch(file1).ok()); - - URI file2(root.to_string() + "file2"); - REQUIRE(vfs_uri.touch(file2).ok()); - - URI file3(root.to_string() + "dir1/file3"); - REQUIRE(vfs_uri.touch(file3).ok()); - - URI file4(root.to_string() + "dir1/dir2/file4"); - REQUIRE(vfs_uri.touch(file4).ok()); - - URI file5(root.to_string() + "file5/"); - REQUIRE(!vfs_uri.touch(file5).ok()); - + /* Create the following file hierarchy: + * + * path/dir1/subdir/file1 + * path/dir1/subdir/file2 + * path/dir1/file3 + * path/file4 + * path/file5 + */ + auto dir1 = URI(path.to_string() + "dir1/"); + auto subdir = URI(dir1.to_string() + "subdir/"); + auto file1 = URI(subdir.to_string() + "file1"); + auto file2 = URI(subdir.to_string() + "file2"); + auto file3 = URI(dir1.to_string() + "file3"); + auto file4 = URI(path.to_string() + "file4"); + auto file5 = URI(path.to_string() + "file5"); + require_tiledb_ok(vfs.create_dir(URI(dir1))); + require_tiledb_ok(vfs.create_dir(URI(subdir))); + require_tiledb_ok(vfs.touch(file1)); + require_tiledb_ok(vfs.touch(file2)); + require_tiledb_ok(vfs.touch(file3)); + require_tiledb_ok(vfs.touch(file4)); + require_tiledb_ok(vfs.touch(file5)); + + /** + * URI Semantics + */ + { + std::vector expected_uri_names = {"file4", "file5", "dir1"}; std::vector uris; - REQUIRE(vfs_uri.ls(root, &uris).ok()); - - std::vector expected_uri_names = {"file1", "file2", "dir1"}; + require_tiledb_ok(vfs.ls(path, &uris)); for (const auto& uri : uris) { // Ensure that the URIs do not contain a trailing backslash. @@ -268,12 +276,126 @@ TEST_CASE("VFS: URI semantics", "[vfs][uri]") { // Verify we found all expected file/dir names. REQUIRE(expected_uri_names.empty()); - - if (root.is_s3() || root.is_azure()) { - REQUIRE(vfs_uri.remove_bucket(root).ok()); - } else { - REQUIRE(vfs_uri.remove_dir(root).ok()); + } // URI Semantics + + /** + * File Management + */ + { + // Check invalid file + require_tiledb_ok(vfs.is_file(URI(path.to_string() + "foo"), &exists)); + CHECK(!exists); + + // List with prefix + std::vector paths; + require_tiledb_ok(vfs.ls(path, &paths)); + CHECK(paths.size() == 3); + paths.clear(); + require_tiledb_ok(vfs.ls(dir1, &paths)); + CHECK(paths.size() == 2); + paths.clear(); + require_tiledb_ok(vfs.ls(subdir, &paths)); + CHECK(paths.size() == 2); + paths.clear(); + + // Check if a directory exists + require_tiledb_ok(vfs.is_dir(file1, &exists)); + CHECK(!exists); // Not a dir + require_tiledb_ok(vfs.is_dir(file4, &exists)); + CHECK(!exists); // Not a dir + require_tiledb_ok(vfs.is_dir(dir1, &exists)); + CHECK(exists); // This is viewed as a dir + require_tiledb_ok(vfs.is_dir(URI(path.to_string() + "dir1"), &exists)); + CHECK(exists); // This is viewed as a dir + + // Check ls_with_sizes + URI ls_dir = dir1; + URI ls_subdir = subdir; + URI ls_file = file3; + if (path.is_hdfs()) { + // HDFS requires localhost-resolved paths for ls_with_sizes + auto localdir1 = + URI("hdfs://localhost:9000/vfs-" + random_label() + "/dir1/"); + require_tiledb_ok(vfs.create_dir(localdir1)); + auto localsubdir = URI(localdir1.to_string() + "subdir/"); + require_tiledb_ok(vfs.create_dir(localsubdir)); + auto localfile3 = URI(localdir1.to_string() + "file3"); + require_tiledb_ok(vfs.touch(localfile3)); + ls_dir = localdir1; + ls_subdir = localsubdir; + ls_file = localfile3; + } + std::string s = "abcdef"; + require_tiledb_ok(vfs.write(ls_file, s.data(), s.size())); + require_tiledb_ok(vfs.close_file(ls_file)); + auto&& [status, opt_children] = vfs.ls_with_sizes(ls_dir); + require_tiledb_ok(status); + auto children = opt_children.value(); +#ifdef _WIN32 + // Normalization only for Windows + ls_file = URI(tiledb::sm::path_win::uri_from_path(ls_file.to_string())); +#endif + REQUIRE(children.size() == 2); + CHECK(URI(children[0].path().native()) == ls_file); + CHECK( + URI(children[1].path().native()) == ls_subdir.remove_trailing_slash()); + CHECK(children[0].file_size() == s.size()); + CHECK(children[1].file_size() == 0); // Directories don't get a size + + if (path.is_hdfs()) { + // Clean up + require_tiledb_ok(vfs.remove_dir(ls_dir)); + require_tiledb_ok(vfs.is_dir(ls_dir, &exists)); + CHECK(!exists); } + + // Move file + auto file6 = URI(path.to_string() + "file6"); + require_tiledb_ok(vfs.move_file(file5, file6)); + require_tiledb_ok(vfs.is_file(file5, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file6, &exists)); + CHECK(exists); + paths.clear(); + + // Move directory + auto dir2 = URI(path.to_string() + "dir2/"); + require_tiledb_ok(vfs.move_dir(dir1, URI(dir2))); + require_tiledb_ok(vfs.is_dir(dir1, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_dir(dir2, &exists)); + CHECK(exists); + paths.clear(); + + // Remove files + require_tiledb_ok(vfs.remove_file(file4)); + require_tiledb_ok(vfs.is_file(file4, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.remove_file(file6)); + require_tiledb_ok(vfs.is_file(file6, &exists)); + CHECK(!exists); + + // Remove directories + require_tiledb_ok(vfs.remove_dir(dir2)); + require_tiledb_ok(vfs.is_file(file1, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file2, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(file3, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_dir(dir2, &exists)); + CHECK(!exists); + } // File Management + + // Clean up + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.remove_bucket(path)); + require_tiledb_ok(vfs.is_bucket(path, &exists)); + REQUIRE(!exists); + } else { + require_tiledb_ok(vfs.remove_dir(path)); + require_tiledb_ok(vfs.is_dir(path, &exists)); + REQUIRE(!exists); } } @@ -282,44 +404,35 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { ThreadPool io_tp(4); VFS vfs_ls{&g_helper_stats, &compute_tp, &io_tp, Config{}}; -#ifdef _WIN32 - std::string path = tiledb::sm::Win::current_dir() + "\\vfs_test\\"; -#else - std::string path = - std::string("file://") + tiledb::sm::Posix::current_dir() + "/vfs_test/"; -#endif - - // Clean up - bool is_dir = false; - REQUIRE(vfs_ls.is_dir(URI(path), &is_dir).ok()); - if (is_dir) - REQUIRE(vfs_ls.remove_dir(URI(path)).ok()); - + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + std::string path = local_prefix + unit_vfs_dir_.path(); std::string dir = path + "ls_dir"; std::string file = dir + "/file"; std::string subdir = dir + "/subdir"; std::string subdir_file = subdir + "/file"; // Create directories and files - REQUIRE(vfs_ls.create_dir(URI(path)).ok()); - REQUIRE(vfs_ls.create_dir(URI(dir)).ok()); - REQUIRE(vfs_ls.create_dir(URI(subdir)).ok()); - REQUIRE(vfs_ls.touch(URI(file)).ok()); - REQUIRE(vfs_ls.touch(URI(subdir_file)).ok()); + require_tiledb_ok(vfs_ls.create_dir(URI(path))); + require_tiledb_ok(vfs_ls.create_dir(URI(dir))); + require_tiledb_ok(vfs_ls.create_dir(URI(subdir))); + require_tiledb_ok(vfs_ls.touch(URI(file))); + require_tiledb_ok(vfs_ls.touch(URI(subdir_file))); // Write to file std::string s1 = "abcdef"; - REQUIRE(vfs_ls.write(URI(file), s1.data(), s1.size()).ok()); + require_tiledb_ok(vfs_ls.write(URI(file), s1.data(), s1.size())); // Write to subdir file std::string s2 = "abcdef"; - REQUIRE(vfs_ls.write(URI(subdir_file), s2.data(), s2.size()).ok()); + require_tiledb_ok(vfs_ls.write(URI(subdir_file), s2.data(), s2.size())); // List auto&& [status, rv] = vfs_ls.ls_with_sizes(URI(dir)); auto children = *rv; - - REQUIRE(status.ok()); + require_tiledb_ok(status); #ifdef _WIN32 // Normalization only for Windows @@ -329,17 +442,15 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { // Check results REQUIRE(children.size() == 2); - REQUIRE(children[0].path().native() == URI(file).to_path()); REQUIRE(children[1].path().native() == URI(subdir).to_path()); - REQUIRE(children[0].file_size() == 6); // Directories don't get a size REQUIRE(children[1].file_size() == 0); // Clean up - REQUIRE(vfs_ls.remove_dir(URI(path)).ok()); + require_tiledb_ok(vfs_ls.remove_dir(URI(path))); } // Currently only S3 is supported for VFS::ls_recursive. From 2085d8cb8469093b0b075b99053e5a0dffc93c6c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 25 Jan 2024 15:12:01 +0200 Subject: [PATCH 159/456] Bump the vcpkg `builtin-baseline` and remove most custom ports. (#4553) [SC-37848](https://app.shortcut.com/tiledb-inc/story/37848/bump-the-vcpkg-builtin-baseline-and-remove-most-remaining-custom-ports) This PR bumps the vcpkg `builtin-baseline` which causes all of our dependencies to be updated to the most recent versions available on vcpkg. This removes the need for our custom ports and most were removed. The README of the ports folder was also updated to highlight that custom ports should be temporary. Apart from the remaining port overlays in the `ports` directory, the precise package versions we are using can be found in https://github.com/microsoft/vcpkg/blob/72010900b7cee36cea77aebb97695095c9358eaf/versions/baseline.json. --- TYPE: BUILD DESC: Update vcpkg version baseline to https://github.com/microsoft/vcpkg/commit/72010900b7cee36cea77aebb97695095c9358eaf. --- .github/workflows/release.yml | 3 + .github/workflows/unit-test-runs.yml | 2 +- cmake/Options/TileDBToolchain.cmake | 2 +- ports/README.md | 30 +- ports/aws-c-auth/portfile.cmake | 32 - ports/aws-c-auth/vcpkg.json | 25 - ports/aws-c-cal/portfile.cmake | 33 - .../aws-c-cal/remove-libcrypto-messages.patch | 14 - ports/aws-c-cal/vcpkg.json | 23 - .../disable-internal-crt-option.patch | 20 - ports/aws-c-common/portfile.cmake | 35 - ports/aws-c-common/vcpkg.json | 18 - ports/aws-c-compression/portfile.cmake | 32 - ports/aws-c-compression/vcpkg.json | 19 - ports/aws-c-event-stream/portfile.cmake | 32 - ports/aws-c-event-stream/vcpkg.json | 21 - ports/aws-c-http/portfile.cmake | 32 - ports/aws-c-http/vcpkg.json | 22 - ports/aws-c-io/portfile.cmake | 32 - ports/aws-c-io/vcpkg.json | 24 - ports/aws-c-mqtt/portfile.cmake | 32 - ports/aws-c-mqtt/vcpkg.json | 24 - ports/aws-c-s3/portfile.cmake | 32 - ports/aws-c-s3/vcpkg.json | 26 - ports/aws-c-sdkutils/portfile.cmake | 32 - ports/aws-c-sdkutils/vcpkg.json | 19 - ports/aws-checksums/portfile.cmake | 39 - ports/aws-checksums/vcpkg.json | 19 - ports/aws-crt-cpp/no-werror.patch | 15 - ports/aws-crt-cpp/portfile.cmake | 38 - ports/aws-crt-cpp/vcpkg.json | 32 - ports/aws-sdk-cpp/fix-aws-root.patch | 22 - ports/aws-sdk-cpp/generateFeatures.ps1 | 73 - .../lock-curl-http-and-tls-settings.patch | 20 - .../aws-sdk-cpp/patch-relocatable-rpath.patch | 12 - ports/aws-sdk-cpp/portfile.cmake | 100 -- ports/aws-sdk-cpp/usage | 12 - ports/aws-sdk-cpp/vcpkg.in.json | 37 - ports/aws-sdk-cpp/vcpkg.json | 1183 -------------- ports/azure-storage-common-cpp/portfile.cmake | 32 + ports/azure-storage-common-cpp/vcpkg.json | 36 + ports/bzip2/CMakeLists.txt | 45 - ports/bzip2/bzip2.pc.in | 11 - ports/bzip2/fix-import-export-macros.patch | 40 - ports/bzip2/portfile.cmake | 56 - ports/bzip2/usage | 4 - ports/bzip2/vcpkg.json | 24 - ports/catch2/fix-install-path.patch | 45 - ports/catch2/fix-uwp-build.patch | 34 - ports/catch2/portfile.cmake | 36 - ports/catch2/vcpkg.json | 18 - ports/crc32c/fix_clang-cl.patch | 13 - ports/crc32c/portfile.cmake | 33 - ports/crc32c/vcpkg.json | 17 - ports/fmt/fix-format-conflict.patch | 15 - ports/fmt/fix-write-batch.patch | 13 - ports/fmt/portfile.cmake | 66 - ports/fmt/usage | 8 - ports/fmt/vcpkg.json | 18 - ports/google-cloud-cpp/portfile.cmake | 85 - .../google-cloud-cpp/support_absl_cxx17.patch | 19 - ports/google-cloud-cpp/vcpkg.json | 1447 ----------------- ports/liblzma/add_support_ios.patch | 20 - ports/liblzma/build-tools.patch | 20 - ports/liblzma/fix_config_include.patch | 12 - ports/liblzma/portfile.cmake | 84 - ports/liblzma/usage | 9 - ports/liblzma/vcpkg-cmake-wrapper.cmake | 64 - ports/liblzma/vcpkg.json | 24 - ports/liblzma/win_output_name.patch | 17 - ports/libxml2/disable-docs.patch | 20 - ports/libxml2/fix_cmakelist.patch | 108 -- ports/libxml2/portfile.cmake | 89 - ports/libxml2/usage | 4 - ports/libxml2/vcpkg-cmake-wrapper.cmake | 3 - ports/libxml2/vcpkg.json | 57 - ...mory-corruption-with-negative-memmov.patch | 26 - ports/lz4/CMakeLists.txt | 57 - ports/lz4/portfile.cmake | 42 - ports/lz4/vcpkg.json | 18 - ports/nlohmann-json/portfile.cmake | 48 - ports/nlohmann-json/usage | 12 - ports/nlohmann-json/vcpkg.json | 22 - ports/s2n/fix-cmake-target-path.patch | 40 - ports/s2n/portfile.cmake | 46 - ports/s2n/vcpkg.json | 24 - ports/spdlog/fmt-header.patch | 13 - ports/spdlog/portfile.cmake | 73 - ports/spdlog/vcpkg.json | 30 - ports/vcpkg-cmake-config/copyright | 23 - ports/vcpkg-cmake-config/portfile.cmake | 12 - .../vcpkg-port-config.cmake | 1 - ports/vcpkg-cmake-config/vcpkg.json | 7 - .../vcpkg_cmake_config_fixup.cmake | 258 --- ports/vcpkg-cmake/portfile.cmake | 14 - ports/vcpkg-cmake/vcpkg-port-config.cmake | 3 - ports/vcpkg-cmake/vcpkg.json | 6 - ports/vcpkg-cmake/vcpkg_cmake_build.cmake | 91 -- ports/vcpkg-cmake/vcpkg_cmake_configure.cmake | 320 ---- ports/vcpkg-cmake/vcpkg_cmake_install.cmake | 21 - ...id-inclusions-when-HAVE_-is-set-to-0.patch | 53 - ports/zlib/0002-skip-building-examples.patch | 17 - ...0003-build-static-or-shared-not-both.patch | 53 - ports/zlib/0004-android-and-mingw-fixes.patch | 31 - ports/zlib/portfile.cmake | 53 - ports/zlib/usage | 4 - ports/zlib/vcpkg-cmake-wrapper.cmake | 12 - ports/zlib/vcpkg.json | 13 - ports/zstd/install_pkgpc.patch | 13 - ports/zstd/no-static-suffix.patch | 13 - ports/zstd/portfile.cmake | 55 - ports/zstd/usage | 4 - ports/zstd/vcpkg.json | 18 - .../unit-cppapi-global-order-writes-remote.cc | 1 + test/src/unit-dimension.cc | 2 +- test/src/unit-enumerations.cc | 2 +- vcpkg.json | 34 +- 117 files changed, 105 insertions(+), 6344 deletions(-) delete mode 100644 ports/aws-c-auth/portfile.cmake delete mode 100644 ports/aws-c-auth/vcpkg.json delete mode 100644 ports/aws-c-cal/portfile.cmake delete mode 100644 ports/aws-c-cal/remove-libcrypto-messages.patch delete mode 100644 ports/aws-c-cal/vcpkg.json delete mode 100644 ports/aws-c-common/disable-internal-crt-option.patch delete mode 100644 ports/aws-c-common/portfile.cmake delete mode 100644 ports/aws-c-common/vcpkg.json delete mode 100644 ports/aws-c-compression/portfile.cmake delete mode 100644 ports/aws-c-compression/vcpkg.json delete mode 100644 ports/aws-c-event-stream/portfile.cmake delete mode 100644 ports/aws-c-event-stream/vcpkg.json delete mode 100644 ports/aws-c-http/portfile.cmake delete mode 100644 ports/aws-c-http/vcpkg.json delete mode 100644 ports/aws-c-io/portfile.cmake delete mode 100644 ports/aws-c-io/vcpkg.json delete mode 100644 ports/aws-c-mqtt/portfile.cmake delete mode 100644 ports/aws-c-mqtt/vcpkg.json delete mode 100644 ports/aws-c-s3/portfile.cmake delete mode 100644 ports/aws-c-s3/vcpkg.json delete mode 100644 ports/aws-c-sdkutils/portfile.cmake delete mode 100644 ports/aws-c-sdkutils/vcpkg.json delete mode 100644 ports/aws-checksums/portfile.cmake delete mode 100644 ports/aws-checksums/vcpkg.json delete mode 100644 ports/aws-crt-cpp/no-werror.patch delete mode 100644 ports/aws-crt-cpp/portfile.cmake delete mode 100644 ports/aws-crt-cpp/vcpkg.json delete mode 100644 ports/aws-sdk-cpp/fix-aws-root.patch delete mode 100644 ports/aws-sdk-cpp/generateFeatures.ps1 delete mode 100644 ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch delete mode 100644 ports/aws-sdk-cpp/patch-relocatable-rpath.patch delete mode 100644 ports/aws-sdk-cpp/portfile.cmake delete mode 100644 ports/aws-sdk-cpp/usage delete mode 100644 ports/aws-sdk-cpp/vcpkg.in.json delete mode 100644 ports/aws-sdk-cpp/vcpkg.json create mode 100644 ports/azure-storage-common-cpp/portfile.cmake create mode 100644 ports/azure-storage-common-cpp/vcpkg.json delete mode 100644 ports/bzip2/CMakeLists.txt delete mode 100644 ports/bzip2/bzip2.pc.in delete mode 100644 ports/bzip2/fix-import-export-macros.patch delete mode 100644 ports/bzip2/portfile.cmake delete mode 100644 ports/bzip2/usage delete mode 100644 ports/bzip2/vcpkg.json delete mode 100644 ports/catch2/fix-install-path.patch delete mode 100644 ports/catch2/fix-uwp-build.patch delete mode 100644 ports/catch2/portfile.cmake delete mode 100644 ports/catch2/vcpkg.json delete mode 100644 ports/crc32c/fix_clang-cl.patch delete mode 100644 ports/crc32c/portfile.cmake delete mode 100644 ports/crc32c/vcpkg.json delete mode 100644 ports/fmt/fix-format-conflict.patch delete mode 100644 ports/fmt/fix-write-batch.patch delete mode 100644 ports/fmt/portfile.cmake delete mode 100644 ports/fmt/usage delete mode 100644 ports/fmt/vcpkg.json delete mode 100644 ports/google-cloud-cpp/portfile.cmake delete mode 100644 ports/google-cloud-cpp/support_absl_cxx17.patch delete mode 100644 ports/google-cloud-cpp/vcpkg.json delete mode 100644 ports/liblzma/add_support_ios.patch delete mode 100644 ports/liblzma/build-tools.patch delete mode 100644 ports/liblzma/fix_config_include.patch delete mode 100644 ports/liblzma/portfile.cmake delete mode 100644 ports/liblzma/usage delete mode 100644 ports/liblzma/vcpkg-cmake-wrapper.cmake delete mode 100644 ports/liblzma/vcpkg.json delete mode 100644 ports/liblzma/win_output_name.patch delete mode 100644 ports/libxml2/disable-docs.patch delete mode 100644 ports/libxml2/fix_cmakelist.patch delete mode 100644 ports/libxml2/portfile.cmake delete mode 100644 ports/libxml2/usage delete mode 100644 ports/libxml2/vcpkg-cmake-wrapper.cmake delete mode 100644 ports/libxml2/vcpkg.json delete mode 100755 ports/lz4/0001-Fix-potential-memory-corruption-with-negative-memmov.patch delete mode 100644 ports/lz4/CMakeLists.txt delete mode 100644 ports/lz4/portfile.cmake delete mode 100644 ports/lz4/vcpkg.json delete mode 100644 ports/nlohmann-json/portfile.cmake delete mode 100644 ports/nlohmann-json/usage delete mode 100644 ports/nlohmann-json/vcpkg.json delete mode 100644 ports/s2n/fix-cmake-target-path.patch delete mode 100644 ports/s2n/portfile.cmake delete mode 100644 ports/s2n/vcpkg.json delete mode 100644 ports/spdlog/fmt-header.patch delete mode 100644 ports/spdlog/portfile.cmake delete mode 100644 ports/spdlog/vcpkg.json delete mode 100644 ports/vcpkg-cmake-config/copyright delete mode 100644 ports/vcpkg-cmake-config/portfile.cmake delete mode 100644 ports/vcpkg-cmake-config/vcpkg-port-config.cmake delete mode 100644 ports/vcpkg-cmake-config/vcpkg.json delete mode 100644 ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake delete mode 100644 ports/vcpkg-cmake/portfile.cmake delete mode 100644 ports/vcpkg-cmake/vcpkg-port-config.cmake delete mode 100644 ports/vcpkg-cmake/vcpkg.json delete mode 100644 ports/vcpkg-cmake/vcpkg_cmake_build.cmake delete mode 100644 ports/vcpkg-cmake/vcpkg_cmake_configure.cmake delete mode 100644 ports/vcpkg-cmake/vcpkg_cmake_install.cmake delete mode 100644 ports/zlib/0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch delete mode 100644 ports/zlib/0002-skip-building-examples.patch delete mode 100644 ports/zlib/0003-build-static-or-shared-not-both.patch delete mode 100644 ports/zlib/0004-android-and-mingw-fixes.patch delete mode 100644 ports/zlib/portfile.cmake delete mode 100644 ports/zlib/usage delete mode 100644 ports/zlib/vcpkg-cmake-wrapper.cmake delete mode 100644 ports/zlib/vcpkg.json delete mode 100644 ports/zstd/install_pkgpc.patch delete mode 100644 ports/zstd/no-static-suffix.patch delete mode 100644 ports/zstd/portfile.cmake delete mode 100644 ports/zstd/usage delete mode 100644 ports/zstd/vcpkg.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 201d5413f5b0..9c373fd607a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,9 @@ jobs: - name: Checkout TileDB # v4 uses node 20 which is incompatible with the libc version of the manylinux image uses: actions/checkout@v3 + - name: 'Homebrew setup' + run: brew install automake pkg-config + if: ${{ startsWith(matrix.os, 'macos-') == true }} - name: Export GitHub Actions cache variables uses: actions/github-script@v6 with: diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index 0b16aede88c5..2c17b2dd6670 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v3 - name: 'Homebrew setup' - run: brew install pkg-config + run: brew install automake pkg-config if: ${{ startsWith(matrix.os, 'macos-') == true }} # Configure required environment variables for vcpkg to use diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 59ed7574c109..d80215128a8a 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -45,7 +45,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # To help with resolving conflicts, when you update the commit, also update its date. - set(VCPKG_COMMIT_STRING ac2a14f35fcd57d7a38f09af75dd5258e96dd6ac) # 2023-11-16 + set(VCPKG_COMMIT_STRING 72010900b7cee36cea77aebb97695095c9358eaf) # 2023-12-05 message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") include(FetchContent) FetchContent_Declare( diff --git a/ports/README.md b/ports/README.md index 7313b9cd2bfb..7faf08053d33 100644 --- a/ports/README.md +++ b/ports/README.md @@ -1,22 +1,36 @@ -Port Overlays -=== +# Port Overlays -This directory contains the custom port overlays we're using for building depdencies. We're using overlays because some of our dependencies are quite old and getting working combinations from modern vcpkg was proving untenable. Overtime as our dependency versions are modernized, most if not all of the overlay ports will likely be removed. +This directory contains the custom vcpkg port overlays we're using for building dependencies. -Updating a Port ---- +## Adding a Port -The easiest way to update a port is to find the version of the port in the `microsoft/vcpkg` repository and extract it using a tool. So, for instance if we wanted to update our curl dependency from 7.80.0 to 7.84.0, we would look in the `path/to/microsoft/vcpkg/versions/c-/curl.json` file and find the tree-ish listed for 7.84.0. One thing to pay attention to here is that there can be multiple port versions for a given dependency version. So we want to pick the higest port version for the dependency at the version we are upgrading. In our hypothetical curl case that I may have just done, this gives us a treeish value of: +> [!IMPORTANT] +> Port overlays should be used as a temporary measure to fix issues with upstream ports or to add new ports that are not yet available to vcpkg. Once the upstream ports are fixed or added, the overlay ports should be removed. - `588fa4742c417db9d7c0f89e652b618296388d1e` +If modifying an existing port, you have to first determine the version of the port in the `microsoft/vcpkg` repository and extract it using a tool. If the port does not have a version pin in the [`vcpkg.json`](../vcpkg.json) manifest (in either a `version>=` field or an entry in the `overrides` section), browse the `microsoft/vcpkg` repository in the commit specified in the `builtin-baseline` field and copy the port directory from there. + +If the port does have a version pin, and for instance we wanted to modify the curl port, which is in version 8.4.0, we would look in the [`versions/c-/curl.json` file](https://github.com/microsoft/vcpkg/blob/master/versions/c-/curl.json) and find the treeish listed for 8.4.0. One thing to pay attention to here is that there can be multiple port versions for a given dependency version. So we want to pick the highest port version for the dependency at the version we are upgrading. In our hypothetical curl case that I may have just done, this gives us a treeish value of: + + `6125c796d6e2913a89a2996d7082375ce16b02dd` Once we have the tree-ish, we just need to be able to extract all of the files and store them in our overlay ports directory. The easiest approach for this is to use something like [this script](https://gist.github.com/mhl/498447/b245d48f2a22301415a30ca8a68241f96e0b3861) to do just that. If you put that script on your path (and remember to `chmod +x path/to/extract-tree-from-git.py`) you can follow these simple steps for updating the port: ```bash $ rm ports/curl/* $ cd path/to/microsoft/vcpkg -$ extract-tree-from-git.py 588fa4742c417db9d7c0f89e652b618296388d1e path/to/tiledb/ports/curl/ +$ extract-tree-from-git.py 6125c796d6e2913a89a2996d7082375ce16b02dd path/to/tiledb/ports/curl/ $ cd path/to/tiledb $ git add ports $ git commit ``` + +After copying the port, add an entry to the table below. You should also contribute your changes to vcpkg and/or the upstream package repository. + +## List of port overlays + +|Port|Reason| +|----|------| +|`libmagic`|Updating to the upstream port deferred due to failures.| +|`openssl`|Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024.| +|`pcre2`|To be removed alongside libmagic.| +|`azure-storage-common-cpp`|Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221).| diff --git a/ports/aws-c-auth/portfile.cmake b/ports/aws-c-auth/portfile.cmake deleted file mode 100644 index 00ae5a4f0179..000000000000 --- a/ports/aws-c-auth/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-auth - REF "v${VERSION}" - SHA512 e247b145132818bb1adf03689fdc98984e6ac50e6d81d0d7caf60d53ba3d3a95bbb4e76f48ac4582365f9038d0d8b29e57947190ec5c34f68e816dd7efcc117d - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-auth/vcpkg.json b/ports/aws-c-auth/vcpkg.json deleted file mode 100644 index 091f50478960..000000000000 --- a/ports/aws-c-auth/vcpkg.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "aws-c-auth", - "version": "0.7.8", - "description": "C99 library implementation of AWS client-side authentication: standard credentials providers and signing.", - "homepage": "https://github.com/awslabs/aws-c-auth", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-cal", - "aws-c-http", - "aws-c-sdkutils", - { - "name": "s2n", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-cal/portfile.cmake b/ports/aws-c-cal/portfile.cmake deleted file mode 100644 index 3cef1fbb2825..000000000000 --- a/ports/aws-c-cal/portfile.cmake +++ /dev/null @@ -1,33 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-cal - REF "v${VERSION}" - SHA512 deee106b366522e6781974c92b1aa06542b7857b91a8d4cb59eb0e17247ce7fc3ffacb044c032ff7f2a0f9baca807d4c2d9a14934d4576966f48bfc0661e5edb - HEAD_REF master - PATCHES remove-libcrypto-messages.patch -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-cal/remove-libcrypto-messages.patch b/ports/aws-c-cal/remove-libcrypto-messages.patch deleted file mode 100644 index 5a1d5dc35d01..000000000000 --- a/ports/aws-c-cal/remove-libcrypto-messages.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/cmake/modules/Findcrypto.cmake b/cmake/modules/Findcrypto.cmake -index fed83bb..9c1ae28 100644 ---- a/cmake/modules/Findcrypto.cmake -+++ b/cmake/modules/Findcrypto.cmake -@@ -105,9 +105,6 @@ else() - set(CRYPTO_FOUND true) - set(crypto_FOUND true) - -- message(STATUS "LibCrypto Include Dir: ${crypto_INCLUDE_DIR}") -- message(STATUS "LibCrypto Shared Lib: ${crypto_SHARED_LIBRARY}") -- message(STATUS "LibCrypto Static Lib: ${crypto_STATIC_LIBRARY}") - if (NOT TARGET AWS::crypto AND - (EXISTS "${crypto_LIBRARY}") - ) diff --git a/ports/aws-c-cal/vcpkg.json b/ports/aws-c-cal/vcpkg.json deleted file mode 100644 index b1d22ed8eade..000000000000 --- a/ports/aws-c-cal/vcpkg.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "aws-c-cal", - "version": "0.6.9", - "description": "C99 wrapper for cryptography primitives.", - "homepage": "https://github.com/awslabs/aws-c-cal", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-common", - { - "name": "openssl", - "platform": "!windows & !osx" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-common/disable-internal-crt-option.patch b/ports/aws-c-common/disable-internal-crt-option.patch deleted file mode 100644 index 1a0fabd4f2b8..000000000000 --- a/ports/aws-c-common/disable-internal-crt-option.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/cmake/AwsCFlags.cmake b/cmake/AwsCFlags.cmake -index 470f6db..537536b 100644 ---- a/cmake/AwsCFlags.cmake -+++ b/cmake/AwsCFlags.cmake -@@ -82,15 +82,6 @@ function(aws_set_common_properties target) - list(APPEND AWS_C_FLAGS /DAWS_SUPPORT_WIN7=1) - endif() - -- # Set MSVC runtime libary. -- # Note: there are other ways of doing this if we bump our CMake minimum to 3.14+ -- # See: https://cmake.org/cmake/help/latest/policy/CMP0091.html -- if (AWS_STATIC_MSVC_RUNTIME_LIBRARY OR STATIC_CRT) -- list(APPEND AWS_C_FLAGS "/MT$<$:d>") -- else() -- list(APPEND AWS_C_FLAGS "/MD$<$:d>") -- endif() -- - else() - list(APPEND AWS_C_FLAGS -Wall -Wstrict-prototypes) - diff --git a/ports/aws-c-common/portfile.cmake b/ports/aws-c-common/portfile.cmake deleted file mode 100644 index e96d1a82844e..000000000000 --- a/ports/aws-c-common/portfile.cmake +++ /dev/null @@ -1,35 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-common - REF "v${VERSION}" - SHA512 f04bdc01d52b5ad191d320d352a13dd4cb9675127596c2c2229657211bc5fa3cddf05a3b395a0dc0ac5ce2f09cecf54c04b9cfacb08299e24a16a47684560f11 - HEAD_REF master - PATCHES - disable-internal-crt-option.patch # Disable internal crt option because vcpkg contains crt processing flow -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake) # central macros -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-common/vcpkg.json b/ports/aws-c-common/vcpkg.json deleted file mode 100644 index 9d7a14ce9cfd..000000000000 --- a/ports/aws-c-common/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "aws-c-common", - "version": "0.9.10", - "description": "AWS common library for C", - "homepage": "https://github.com/awslabs/aws-c-common", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-compression/portfile.cmake b/ports/aws-c-compression/portfile.cmake deleted file mode 100644 index 3104c34f4b1f..000000000000 --- a/ports/aws-c-compression/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-compression - REF "v${VERSION}" - SHA512 7813ae3d19336a187dd218d22748a09b397aea68f344e44ce0472490babdd7c7a4bb94d586a38d7f0c7d3b6f29502ec9ccf080b020e15fd24891ec1b3cdb4663 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-compression/vcpkg.json b/ports/aws-c-compression/vcpkg.json deleted file mode 100644 index 351ad87cea14..000000000000 --- a/ports/aws-c-compression/vcpkg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "aws-c-compression", - "version": "0.2.17", - "description": "C99 implementation of huffman encoding/decoding", - "homepage": "https://github.com/awslabs/aws-c-compression", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-common", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-event-stream/portfile.cmake b/ports/aws-c-event-stream/portfile.cmake deleted file mode 100644 index e725a88532b0..000000000000 --- a/ports/aws-c-event-stream/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-event-stream - REF "v${VERSION}" - SHA512 295ee4b1f647f5b15790e000cd5f0102e64ef03912c1c7378739f52229b82090384c66367c4caf8fa7f34f3275b3da71bc031cf6000053ad31d53ce075721da2 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-event-stream/vcpkg.json b/ports/aws-c-event-stream/vcpkg.json deleted file mode 100644 index 1c3135a5ed57..000000000000 --- a/ports/aws-c-event-stream/vcpkg.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "aws-c-event-stream", - "version": "0.3.2", - "description": "C99 implementation of the vnd.amazon.event-stream content-type.", - "homepage": "https://github.com/awslabs/aws-c-event-stream", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-common", - "aws-c-io", - "aws-checksums", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-http/portfile.cmake b/ports/aws-c-http/portfile.cmake deleted file mode 100644 index a0cf2430398e..000000000000 --- a/ports/aws-c-http/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-http - REF "v${VERSION}" - SHA512 205cf1cc4b766fd42cfca6ab84974e0797f2a52f8cdf64fed0c4327a2b70c251e5c9561b1cf714cbe4b7ef87092a094830071e6621bd0e0e1684b2e66ea718e2 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-http/vcpkg.json b/ports/aws-c-http/vcpkg.json deleted file mode 100644 index dcbb887aa9e0..000000000000 --- a/ports/aws-c-http/vcpkg.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "aws-c-http", - "version": "0.7.14", - "description": "C99 implementation of the HTTP/1.1 and HTTP/2 specifications", - "homepage": "https://github.com/awslabs/aws-c-http", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-cal", - "aws-c-common", - "aws-c-compression", - "aws-c-io", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-io/portfile.cmake b/ports/aws-c-io/portfile.cmake deleted file mode 100644 index bf331bfbaa20..000000000000 --- a/ports/aws-c-io/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-io - REF "v${VERSION}" - SHA512 cb76ddf7d43c440443c6e5fb33a022f540d8ca95572f36b263836fe6bebfd0790370c84fec37b45475645167ab71bb14de320b9988803ac01f6bbb0d7436949a - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-io/vcpkg.json b/ports/aws-c-io/vcpkg.json deleted file mode 100644 index e6fc427362c2..000000000000 --- a/ports/aws-c-io/vcpkg.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "aws-c-io", - "version": "0.13.36", - "description": "Handles all IO and TLS work for application protocols.", - "homepage": "https://github.com/awslabs/aws-c-io", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-cal", - "aws-c-common", - { - "name": "s2n", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-mqtt/portfile.cmake b/ports/aws-c-mqtt/portfile.cmake deleted file mode 100644 index fd41c8328a84..000000000000 --- a/ports/aws-c-mqtt/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-mqtt - REF "v${VERSION}" - SHA512 9664596de78a8778349cf32d5dd207ed8e5a28e1d13505f942c8486c3fbf516ec49121af312d85182ebdc66b464d430fcd2d28606b1c2652a8dfb9c173334681 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-mqtt/vcpkg.json b/ports/aws-c-mqtt/vcpkg.json deleted file mode 100644 index 1bec811c3463..000000000000 --- a/ports/aws-c-mqtt/vcpkg.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "aws-c-mqtt", - "version": "0.9.10", - "description": "C99 implementation of the MQTT 3.1.1 specification.", - "homepage": "https://github.com/awslabs/aws-c-mqtt", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-http", - "aws-c-io", - { - "name": "s2n", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-s3/portfile.cmake b/ports/aws-c-s3/portfile.cmake deleted file mode 100644 index 4b65d3999499..000000000000 --- a/ports/aws-c-s3/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-s3 - REF "v${VERSION}" - SHA512 62dca71a857f7c55cb824cf1a81f3dbefa7cf7d50e5fadec5f2e67d4c2e33c2902a483145b4e9390aa5adf68ab65588c7c71e91717b74733db2a15e7a1cc5794 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-s3/vcpkg.json b/ports/aws-c-s3/vcpkg.json deleted file mode 100644 index 362b597c81bc..000000000000 --- a/ports/aws-c-s3/vcpkg.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "aws-c-s3", - "version": "0.4.3", - "description": "C99 library implementation for communicating with the S3 service, designed for maximizing throughput on high bandwidth EC2 instances.", - "homepage": "https://github.com/awslabs/aws-c-s3", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-auth", - "aws-c-http", - "aws-c-sdkutils", - "aws-checksums", - { - "name": "s2n", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-c-sdkutils/portfile.cmake b/ports/aws-c-sdkutils/portfile.cmake deleted file mode 100644 index 53ed5fdc8a0f..000000000000 --- a/ports/aws-c-sdkutils/portfile.cmake +++ /dev/null @@ -1,32 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-c-sdkutils - REF "v${VERSION}" - SHA512 34d5bc5190fc6890533247d304b791625f59b0f60aca972b03e00f684b37dd3b806b7c371f1f8b048a30cb9960326d1534c53874621c9630215e597ba8700685 - HEAD_REF master -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-c-sdkutils/vcpkg.json b/ports/aws-c-sdkutils/vcpkg.json deleted file mode 100644 index bb91dda4e0db..000000000000 --- a/ports/aws-c-sdkutils/vcpkg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "aws-c-sdkutils", - "version": "0.1.12", - "description": "C99 library implementing AWS SDK specific utilities. Includes utilities for ARN parsing, reading AWS profiles, etc...", - "homepage": "https://github.com/awslabs/aws-c-sdkutils", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-common", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-checksums/portfile.cmake b/ports/aws-checksums/portfile.cmake deleted file mode 100644 index 07234e6ca53b..000000000000 --- a/ports/aws-checksums/portfile.cmake +++ /dev/null @@ -1,39 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-checksums - REF "v${VERSION}" - SHA512 b75f5442db9a61f8856756c4a784339fd446effca0cdb02c67e51ce9f14ea76f5ca94d29a69f2a452c63c868598489343ec1d097432a8a0159868731422cfbf4 - HEAD_REF master -) - -if (VCPKG_CRT_LINKAGE STREQUAL static) - set(STATIC_CRT_LNK ON) -else() - set(STATIC_CRT_LNK OFF) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DSTATIC_CRT=${STATIC_CRT_LNK} - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-checksums/vcpkg.json b/ports/aws-checksums/vcpkg.json deleted file mode 100644 index c915c21576cc..000000000000 --- a/ports/aws-checksums/vcpkg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "aws-checksums", - "version": "0.1.17", - "description": "Cross-Platform HW accelerated CRC32c and CRC32 with fallback to efficient SW implementations.", - "homepage": "https://github.com/awslabs/aws-checksums", - "license": "Apache-2.0", - "supports": "!(windows & arm)", - "dependencies": [ - "aws-c-common", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-crt-cpp/no-werror.patch b/ports/aws-crt-cpp/no-werror.patch deleted file mode 100644 index 27b7c8f8ab6e..000000000000 --- a/ports/aws-crt-cpp/no-werror.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt (revision f2adef31d778cfe90b8a5bb377425f825ebf92f0) -+++ b/CMakeLists.txt (date 1675009099302) -@@ -337,9 +337,9 @@ - # set extra warning flags - if(AWS_WARNINGS_ARE_ERRORS) - if(MSVC) -- target_compile_options(${PROJECT_NAME} PRIVATE /W4 /WX /wd4068) -+ target_compile_options(${PROJECT_NAME} PRIVATE /W4 /wd4068) - else() -- target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wno-long-long -pedantic -Werror) -+ target_compile_options(${PROJECT_NAME} PRIVATE -Wall -Wno-long-long -pedantic) - endif() - endif() - diff --git a/ports/aws-crt-cpp/portfile.cmake b/ports/aws-crt-cpp/portfile.cmake deleted file mode 100644 index 553fc30c9021..000000000000 --- a/ports/aws-crt-cpp/portfile.cmake +++ /dev/null @@ -1,38 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO awslabs/aws-crt-cpp - REF "v${VERSION}" - SHA512 4160aae9aaf98556d9fd26a13feb09f7fce2a20e073740d13ab9890fdab158de63918c8fd2f4a2d07e92798e2df47b6696b38e614c9631ebcdddf7ea1c44a126 - PATCHES - no-werror.patch -) - -string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" STATIC_CRT) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - "-DSTATIC_CRT=${STATIC_CRT}" - -DBUILD_DEPS=OFF - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files - -DBUILD_TESTING=FALSE -) - -vcpkg_cmake_install() - -string(REPLACE "dynamic" "shared" subdir "${VCPKG_LIBRARY_LINKAGE}") -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake/${subdir}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/${PORT}/cmake") -vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/${PORT}/${PORT}-config.cmake" [[/${type}/]] "/") - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/${PORT}" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/${PORT}" -) - -vcpkg_copy_pdbs() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-crt-cpp/vcpkg.json b/ports/aws-crt-cpp/vcpkg.json deleted file mode 100644 index a211574cd573..000000000000 --- a/ports/aws-crt-cpp/vcpkg.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "aws-crt-cpp", - "version": "0.24.8", - "description": "C++ wrapper around the aws-c-* libraries. Provides Cross-Platform Transport Protocols and SSL/TLS implementations for C++.", - "homepage": "https://github.com/awslabs/aws-crt-cpp", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-c-auth", - "aws-c-cal", - "aws-c-common", - "aws-c-compression", - "aws-c-event-stream", - "aws-c-http", - "aws-c-io", - "aws-c-mqtt", - "aws-c-s3", - "aws-checksums", - { - "name": "s2n", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/aws-sdk-cpp/fix-aws-root.patch b/ports/aws-sdk-cpp/fix-aws-root.patch deleted file mode 100644 index da4365ad7218..000000000000 --- a/ports/aws-sdk-cpp/fix-aws-root.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/cmake/AWSSDKConfig.cmake b/cmake/AWSSDKConfig.cmake ---- a/cmake/AWSSDKConfig.cmake (revision 2f90f9fd6c56460bd382243aa215fcddcb5883c8) -+++ b/cmake/AWSSDKConfig.cmake (date 1636913220527) -@@ -54,18 +54,14 @@ - string(REPLACE ";" "${AWS_MODULE_DIR};" SYSTEM_MODULE_PATH "${CMAKE_SYSTEM_PREFIX_PATH}${AWS_MODULE_DIR}") - list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH} ${SYSTEM_MODULE_PATH}) - --# On Windows, dlls are treated as runtime target and installed in bindir - if (WIN32 AND AWSSDK_INSTALL_AS_SHARED_LIBS) -- set(AWSSDK_INSTALL_LIBDIR "${AWSSDK_INSTALL_BINDIR}") - # If installed CMake scripts are associated with dll library, define USE_IMPORT_EXPORT for customers - add_definitions(-DUSE_IMPORT_EXPORT) - endif() - - - # Compute the default installation root relative to this file. --# from prefix/lib/cmake/AWSSDK/xx.cmake to prefix - get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) --get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) - get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) - get_filename_component(AWSSDK_DEFAULT_ROOT_DIR "${AWSSDK_DEFAULT_ROOT_DIR}" PATH) - get_filename_component(AWS_NATIVE_SDK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) diff --git a/ports/aws-sdk-cpp/generateFeatures.ps1 b/ports/aws-sdk-cpp/generateFeatures.ps1 deleted file mode 100644 index d0f9a5aba722..000000000000 --- a/ports/aws-sdk-cpp/generateFeatures.ps1 +++ /dev/null @@ -1,73 +0,0 @@ -[CmdletBinding()] -param( - [Parameter(Mandatory=$true)][string]$SourcesRef, - [Parameter(Mandatory=$false)][string]$PortDirectory = $PSScriptRoot, - [Parameter(Mandatory=$false)][string]$vcpkg = "$PSScriptRoot/../../vcpkg" -) - -$ErrorActionPreference = "Stop" - -$ManifestIn = "$PortDirectory/vcpkg.in.json" -$ManifestOut = "$PortDirectory/vcpkg.json" - -$ExtractedSources = "${env:TEMP}/aws-sdk-cpp-generateFeatures-$SourcesRef" -if (-not (Test-Path $ExtractedSources)) { - if (Test-Path "$ExtractedSources.tmp") { - Remove-Item -Force "$ExtractedSources.tmp" - } - git clone "https://github.com/aws/aws-sdk-cpp" "$ExtractedSources.tmp" | Out-Host - git -c "$ExtractedSources.tmp" checkout $SourcesRef - Move-Item "$ExtractedSources.tmp" "$ExtractedSources" -} -Write-Host "Using sources directory: $ExtractedSources" - - -$subfolders = Get-ChildItem -Path "$ExtractedSources\generated\src\aws-cpp-sdk-*", "$ExtractedSources\src\aws-cpp-sdk*" | Sort-Object -Property Name - -$manifest = Get-Content $ManifestIn | ConvertFrom-Json -$manifest | Add-Member ` - -NotePropertyName '$note' ` - -NotePropertyValue 'Automatically generated by generateFeatures.ps1' -$manifest | Add-Member -NotePropertyName 'features' -NotePropertyValue @{} - -function GetDescription($dir, $modulename) -{ - if (Test-Path "$dir\CMakeLists.txt") - { - $descs = @(Select-String -Path "$dir\CMakeLists.txt" -Pattern "`"C\+\+ SDK for the AWS [^`"]*`"") - if ($descs.count -eq 1) { - $desc = $descs[0].Matches.Value -replace "`"","" - "$desc" - } - else { "C++ SDK for the AWS $modulename service" } - } - else { "C++ SDK for the AWS $modulename service" } -} - -$featureDependencies = @{} -Select-String -Path "$ExtractedSources\cmake\sdksCommon.cmake" -Pattern "list\(APPEND SDK_DEPENDENCY_LIST `"([\w-]+):([\w-,]+)`"\)" -AllMatches ` -| ForEach-Object { $_.Matches } ` -| ForEach-Object { $featureDependencies[$_.Groups[1].Value] = @($_.Groups[2].Value -split "," ` -| Where-Object { $_ -ne "core" }) } - -foreach ($subfolder in $subfolders) -{ - $modulename = $subfolder.name -replace "^aws-cpp-sdk-","" - if ($modulename -match "-tests`$") { continue } - if ($modulename -match "-sample`$") { continue } - if ($modulename -eq "core") { continue } - - $lowermodulename = $modulename.ToLower() - - $featureObj = @{ description = (GetDescription $subfolder $modulename) } - - if ($featureDependencies.ContainsKey($lowermodulename)) { - $featureObj.dependencies = ,@{ name = "aws-sdk-cpp"; "default-features" = $false; "features" = $featureDependencies[$lowermodulename] } - } - - $manifest.features.Add("$lowermodulename", $featureObj) -} - -[IO.File]::WriteAllText($ManifestOut, (ConvertTo-Json -Depth 10 -InputObject $manifest)) - -& $vcpkg format-manifest --feature-flags=-manifests $ManifestOut diff --git a/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch b/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch deleted file mode 100644 index 0f7a3b191830..000000000000 --- a/ports/aws-sdk-cpp/lock-curl-http-and-tls-settings.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/src/aws-cpp-sdk-core/CMakeLists.txt b/src/aws-cpp-sdk-core/CMakeLists.txt -index c44546b0e..b66888362 100644 ---- a/src/aws-cpp-sdk-core/CMakeLists.txt -+++ b/src/aws-cpp-sdk-core/CMakeLists.txt -@@ -113,13 +113,8 @@ if(ENABLE_CURL_CLIENT) - int main() { - CURL* handle = curl_easy_init(); - return curl_easy_setopt(handle, CURLOPT_PROXY_SSLCERT, \"client.pem\"); }") -- if (CMAKE_CROSSCOMPILING) -- check_c_source_compiles("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) -- check_c_source_compiles("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) -- else() -- check_c_source_runs("${CHECK_CURL_HAS_H2}" CURL_HAS_H2) -- check_c_source_runs("${CHECK_CURL_HAS_TLS_PROXY}" CURL_HAS_TLS_PROXY) -- endif() -+ set(CURL_HAS_H2 OFF) -+ set(CURL_HAS_TLS_PROXY ON) - elseif(ENABLE_WINDOWS_CLIENT) - # NOTE: HTTP/2 is not supported when using IXML_HTTP_REQUEST_2 - if(USE_IXML_HTTP_REQUEST_2) diff --git a/ports/aws-sdk-cpp/patch-relocatable-rpath.patch b/ports/aws-sdk-cpp/patch-relocatable-rpath.patch deleted file mode 100644 index 7d68126be591..000000000000 --- a/ports/aws-sdk-cpp/patch-relocatable-rpath.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 41d220d5fa..f6ee9a2a74 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -147,7 +147,6 @@ if (LEGACY_BUILD) - endif () - - # Add Linker search paths to RPATH so as to fix the problem where some linkers can't find cross-compiled dependent libraries in customer paths when linking executables. -- set(CMAKE_INSTALL_RPATH_USE_LINK_PATH true) - - # build the sdk targets - project("aws-cpp-sdk-all" VERSION "${PROJECT_VERSION}" LANGUAGES CXX) diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake deleted file mode 100644 index 1dc66c5f4b3c..000000000000 --- a/ports/aws-sdk-cpp/portfile.cmake +++ /dev/null @@ -1,100 +0,0 @@ -vcpkg_buildpath_length_warning(37) - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO aws/aws-sdk-cpp - REF "${VERSION}" - SHA512 4410eaff815ce7b6c0bc0d37bd4175754d5103d2e3cfd60755df57dad103ab7e7705b79fc6039d2c8b7d1ccec650912f5ff0aa73baa2d9cf6d6608a493d11088 - PATCHES - patch-relocatable-rpath.patch - fix-aws-root.patch - lock-curl-http-and-tls-settings.patch - fix-awsmigrationhub-build.patch -) - -string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) - -set(EXTRA_ARGS) -if(VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - set(rpath "@loader_path") -elseif (VCPKG_TARGET_IS_ANDROID) - set(EXTRA_ARGS "-DTARGET_ARCH=ANDROID" - "-DGIT_EXECUTABLE=--invalid-git-executable--" - "-DGIT_FOUND=TRUE" - "-DNDK_DIR=$ENV{ANDROID_NDK_HOME}" - "-DANDROID_BUILD_ZLIB=FALSE" - "-DANDROID_BUILD_CURL=FALSE" - "-DANDROID_BUILD_OPENSSL=FALSE" - ) -else() - set(rpath "\$ORIGIN") -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - ${EXTRA_ARGS} - "-DENABLE_UNITY_BUILD=ON" - "-DENABLE_TESTING=OFF" - "-DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}" - "-DBUILD_ONLY=${FEATURES}" - "-DBUILD_DEPS=OFF" - "-DBUILD_SHARED_LIBS=OFF" - "-DAWS_SDK_WARNINGS_ARE_ERRORS=OFF" - "-DCMAKE_INSTALL_RPATH=${rpath}" - "-DCMAKE_MODULE_PATH=${CURRENT_INSTALLED_DIR}/share/aws-c-common" # use extra cmake files -) -vcpkg_cmake_install() - -foreach(TARGET IN LISTS FEATURES) - vcpkg_cmake_config_fixup(PACKAGE_NAME "aws-cpp-sdk-${TARGET}" CONFIG_PATH "lib/cmake/aws-cpp-sdk-${TARGET}" DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() -vcpkg_cmake_config_fixup(PACKAGE_NAME "AWSSDK" CONFIG_PATH "lib/cmake/AWSSDK") - -vcpkg_copy_pdbs() - -file(GLOB_RECURSE AWS_TARGETS "${CURRENT_PACKAGES_DIR}/share/*/*-targets-*.cmake") -foreach(AWS_TARGET IN LISTS AWS_TARGETS) - file(READ ${AWS_TARGET} _contents) - string(REGEX REPLACE - "bin\\/([A-Za-z0-9_.-]+\\.lib)" - "lib/\\1" - _contents "${_contents}") - file(WRITE ${AWS_TARGET} "${_contents}") -endforeach() - -file(GLOB AWS_CONFIGS "${CURRENT_PACKAGES_DIR}/share/*/aws-cpp-sdk-*-config.cmake") -list(FILTER AWS_CONFIGS EXCLUDE REGEX "aws-cpp-sdk-core-config\\.cmake\$") -foreach(AWS_CONFIG IN LISTS AWS_CONFIGS) - file(READ "${AWS_CONFIG}" _contents) - file(WRITE "${AWS_CONFIG}" "include(CMakeFindDependencyMacro)\nfind_dependency(aws-cpp-sdk-core)\n${_contents}") -endforeach() - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/pkgconfig" - "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig" - "${CURRENT_PACKAGES_DIR}/nuget" - "${CURRENT_PACKAGES_DIR}/debug/nuget" -) - -if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) - file(GLOB LIB_FILES ${CURRENT_PACKAGES_DIR}/bin/*.lib) - if(LIB_FILES) - file(COPY ${LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(REMOVE ${LIB_FILES}) - endif() - file(GLOB DEBUG_LIB_FILES ${CURRENT_PACKAGES_DIR}/debug/bin/*.lib) - if(DEBUG_LIB_FILES) - file(COPY ${DEBUG_LIB_FILES} DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) - file(REMOVE ${DEBUG_LIB_FILES}) - endif() - - file(APPEND "${CURRENT_PACKAGES_DIR}/include/aws/core/SDKConfig.h" "#ifndef USE_IMPORT_EXPORT\n#define USE_IMPORT_EXPORT\n#endif") -endif() - -configure_file("${CURRENT_PORT_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" @ONLY) - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/aws-sdk-cpp/usage b/ports/aws-sdk-cpp/usage deleted file mode 100644 index 37e1f617b2cb..000000000000 --- a/ports/aws-sdk-cpp/usage +++ /dev/null @@ -1,12 +0,0 @@ -The package @PORT@:@TARGET_TRIPLET@ provides CMake targets: - - When using AWSSDK, AWSSDK_ROOT_DIR must be defined by the user. - find_package(AWSSDK CONFIG COMPONENTS core dynamodb kinesis s3 REQUIRED) - target_include_directories(main PRIVATE ${AWSSDK_INCLUDE_DIRS}) - target_link_libraries(main PRIVATE ${AWSSDK_LIBRARIES}) - - OR - - find_package(aws-cpp-sdk-core REQUIRED) - target_include_directories(main PRIVATE aws-cpp-sdk-core) - target_link_libraries(main PRIVATE aws-cpp-sdk-core) diff --git a/ports/aws-sdk-cpp/vcpkg.in.json b/ports/aws-sdk-cpp/vcpkg.in.json deleted file mode 100644 index e79a583786ba..000000000000 --- a/ports/aws-sdk-cpp/vcpkg.in.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "aws-sdk-cpp", - "version": "1.11.215", - "description": "AWS SDK for C++", - "homepage": "https://github.com/aws/aws-sdk-cpp", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-crt-cpp", - { - "name": "curl", - "default-features": false, - "features": [ - "ssl" - ], - "platform": "!uwp & !windows" - }, - { - "name": "openssl", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - }, - "zlib" - ], - "default-features": [ - "dynamodb", - "kinesis", - "s3" - ] -} diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json deleted file mode 100644 index 745435741aa6..000000000000 --- a/ports/aws-sdk-cpp/vcpkg.json +++ /dev/null @@ -1,1183 +0,0 @@ -{ - "$note": "Automatically generated by generateFeatures.ps1", - "name": "aws-sdk-cpp", - "version": "1.11.215", - "description": "AWS SDK for C++", - "homepage": "https://github.com/aws/aws-sdk-cpp", - "license": "Apache-2.0", - "supports": "!(windows & arm) & !uwp", - "dependencies": [ - "aws-crt-cpp", - { - "name": "curl", - "default-features": false, - "features": [ - "ssl" - ], - "platform": "!uwp & !windows" - }, - { - "name": "openssl", - "platform": "!uwp & !windows" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - }, - "zlib" - ], - "default-features": [ - "dynamodb", - "kinesis", - "s3" - ], - "features": { - "access-management": { - "description": "C++ SDK for the AWS access-management service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "cognito-identity", - "iam" - ] - } - ] - }, - "accessanalyzer": { - "description": "C++ SDK for the AWS accessanalyzer service" - }, - "account": { - "description": "C++ SDK for the AWS account service" - }, - "acm": { - "description": "C++ SDK for the AWS acm service" - }, - "acm-pca": { - "description": "C++ SDK for the AWS acm-pca service" - }, - "alexaforbusiness": { - "description": "C++ SDK for the AWS alexaforbusiness service" - }, - "amp": { - "description": "C++ SDK for the AWS amp service" - }, - "amplify": { - "description": "C++ SDK for the AWS amplify service" - }, - "amplifybackend": { - "description": "C++ SDK for the AWS amplifybackend service" - }, - "amplifyuibuilder": { - "description": "C++ SDK for the AWS amplifyuibuilder service" - }, - "apigateway": { - "description": "C++ SDK for the AWS apigateway service" - }, - "apigatewaymanagementapi": { - "description": "C++ SDK for the AWS apigatewaymanagementapi service" - }, - "apigatewayv2": { - "description": "C++ SDK for the AWS apigatewayv2 service" - }, - "appconfig": { - "description": "C++ SDK for the AWS appconfig service" - }, - "appconfigdata": { - "description": "C++ SDK for the AWS appconfigdata service" - }, - "appfabric": { - "description": "C++ SDK for the AWS appfabric service" - }, - "appflow": { - "description": "C++ SDK for the AWS appflow service" - }, - "appintegrations": { - "description": "C++ SDK for the AWS appintegrations service" - }, - "application-autoscaling": { - "description": "C++ SDK for the AWS application-autoscaling service" - }, - "application-insights": { - "description": "C++ SDK for the AWS application-insights service" - }, - "applicationcostprofiler": { - "description": "C++ SDK for the AWS applicationcostprofiler service" - }, - "appmesh": { - "description": "C++ SDK for the AWS appmesh service" - }, - "apprunner": { - "description": "C++ SDK for the AWS apprunner service" - }, - "appstream": { - "description": "C++ SDK for the AWS appstream service" - }, - "appsync": { - "description": "C++ SDK for the AWS appsync service" - }, - "arc-zonal-shift": { - "description": "C++ SDK for the AWS arc-zonal-shift service" - }, - "athena": { - "description": "C++ SDK for the AWS athena service" - }, - "auditmanager": { - "description": "C++ SDK for the AWS auditmanager service" - }, - "autoscaling": { - "description": "C++ SDK for the AWS autoscaling service" - }, - "autoscaling-plans": { - "description": "C++ SDK for the AWS autoscaling-plans service" - }, - "awsmigrationhub": { - "description": "C++ SDK for the AWS AWSMigrationHub service" - }, - "awstransfer": { - "description": "C++ SDK for the AWS awstransfer service" - }, - "backup": { - "description": "C++ SDK for the AWS backup service" - }, - "backup-gateway": { - "description": "C++ SDK for the AWS backup-gateway service" - }, - "backupstorage": { - "description": "C++ SDK for the AWS backupstorage service" - }, - "batch": { - "description": "C++ SDK for the AWS batch service" - }, - "billingconductor": { - "description": "C++ SDK for the AWS billingconductor service" - }, - "braket": { - "description": "C++ SDK for the AWS braket service" - }, - "budgets": { - "description": "C++ SDK for the AWS budgets service" - }, - "ce": { - "description": "C++ SDK for the AWS ce service" - }, - "chime": { - "description": "C++ SDK for the AWS chime service" - }, - "chime-sdk-identity": { - "description": "C++ SDK for the AWS chime-sdk-identity service" - }, - "chime-sdk-media-pipelines": { - "description": "C++ SDK for the AWS chime-sdk-media-pipelines service" - }, - "chime-sdk-meetings": { - "description": "C++ SDK for the AWS chime-sdk-meetings service" - }, - "chime-sdk-messaging": { - "description": "C++ SDK for the AWS chime-sdk-messaging service" - }, - "chime-sdk-voice": { - "description": "C++ SDK for the AWS chime-sdk-voice service" - }, - "cleanrooms": { - "description": "C++ SDK for the AWS cleanrooms service" - }, - "cloud9": { - "description": "C++ SDK for the AWS cloud9 service" - }, - "cloudcontrol": { - "description": "C++ SDK for the AWS cloudcontrol service" - }, - "clouddirectory": { - "description": "C++ SDK for the AWS clouddirectory service" - }, - "cloudformation": { - "description": "C++ SDK for the AWS cloudformation service" - }, - "cloudfront": { - "description": "C++ SDK for the AWS cloudfront service" - }, - "cloudhsm": { - "description": "C++ SDK for the AWS cloudhsm service" - }, - "cloudhsmv2": { - "description": "C++ SDK for the AWS cloudhsmv2 service" - }, - "cloudsearch": { - "description": "C++ SDK for the AWS cloudsearch service" - }, - "cloudsearchdomain": { - "description": "C++ SDK for the AWS cloudsearchdomain service" - }, - "cloudtrail": { - "description": "C++ SDK for the AWS cloudtrail service" - }, - "cloudtrail-data": { - "description": "C++ SDK for the AWS cloudtrail-data service" - }, - "codeartifact": { - "description": "C++ SDK for the AWS codeartifact service" - }, - "codebuild": { - "description": "C++ SDK for the AWS codebuild service" - }, - "codecatalyst": { - "description": "C++ SDK for the AWS codecatalyst service" - }, - "codecommit": { - "description": "C++ SDK for the AWS codecommit service" - }, - "codedeploy": { - "description": "C++ SDK for the AWS codedeploy service" - }, - "codeguru-reviewer": { - "description": "C++ SDK for the AWS codeguru-reviewer service" - }, - "codeguru-security": { - "description": "C++ SDK for the AWS codeguru-security service" - }, - "codeguruprofiler": { - "description": "C++ SDK for the AWS codeguruprofiler service" - }, - "codepipeline": { - "description": "C++ SDK for the AWS codepipeline service" - }, - "codestar": { - "description": "C++ SDK for the AWS codestar service" - }, - "codestar-connections": { - "description": "C++ SDK for the AWS codestar-connections service" - }, - "codestar-notifications": { - "description": "C++ SDK for the AWS codestar-notifications service" - }, - "cognito-identity": { - "description": "C++ SDK for the AWS cognito-identity service" - }, - "cognito-idp": { - "description": "C++ SDK for the AWS cognito-idp service" - }, - "cognito-sync": { - "description": "C++ SDK for the AWS cognito-sync service" - }, - "comprehend": { - "description": "C++ SDK for the AWS comprehend service" - }, - "comprehendmedical": { - "description": "C++ SDK for the AWS comprehendmedical service" - }, - "compute-optimizer": { - "description": "C++ SDK for the AWS compute-optimizer service" - }, - "config": { - "description": "C++ SDK for the AWS config service" - }, - "connect": { - "description": "C++ SDK for the AWS connect service" - }, - "connect-contact-lens": { - "description": "C++ SDK for the AWS connect-contact-lens service" - }, - "connectcampaigns": { - "description": "C++ SDK for the AWS connectcampaigns service" - }, - "connectcases": { - "description": "C++ SDK for the AWS connectcases service" - }, - "connectparticipant": { - "description": "C++ SDK for the AWS connectparticipant service" - }, - "controltower": { - "description": "C++ SDK for the AWS controltower service" - }, - "cur": { - "description": "C++ SDK for the AWS cur service" - }, - "customer-profiles": { - "description": "C++ SDK for the AWS customer-profiles service" - }, - "databrew": { - "description": "C++ SDK for the AWS databrew service" - }, - "dataexchange": { - "description": "C++ SDK for the AWS dataexchange service" - }, - "datapipeline": { - "description": "C++ SDK for the AWS datapipeline service" - }, - "datasync": { - "description": "C++ SDK for the AWS datasync service" - }, - "dax": { - "description": "C++ SDK for the AWS dax service" - }, - "detective": { - "description": "C++ SDK for the AWS detective service" - }, - "devicefarm": { - "description": "C++ SDK for the AWS devicefarm service" - }, - "devops-guru": { - "description": "C++ SDK for the AWS devops-guru service" - }, - "directconnect": { - "description": "C++ SDK for the AWS directconnect service" - }, - "discovery": { - "description": "C++ SDK for the AWS discovery service" - }, - "dlm": { - "description": "C++ SDK for the AWS dlm service" - }, - "dms": { - "description": "C++ SDK for the AWS dms service" - }, - "docdb": { - "description": "C++ SDK for the AWS docdb service" - }, - "docdb-elastic": { - "description": "C++ SDK for the AWS docdb-elastic service" - }, - "drs": { - "description": "C++ SDK for the AWS drs service" - }, - "ds": { - "description": "C++ SDK for the AWS ds service" - }, - "dynamodb": { - "description": "C++ SDK for the AWS dynamodb service" - }, - "dynamodbstreams": { - "description": "C++ SDK for the AWS dynamodbstreams service" - }, - "ebs": { - "description": "C++ SDK for the AWS ebs service" - }, - "ec2": { - "description": "C++ SDK for the AWS ec2 service" - }, - "ec2-instance-connect": { - "description": "C++ SDK for the AWS ec2-instance-connect service" - }, - "ecr": { - "description": "C++ SDK for the AWS ecr service" - }, - "ecr-public": { - "description": "C++ SDK for the AWS ecr-public service" - }, - "ecs": { - "description": "C++ SDK for the AWS ecs service" - }, - "eks": { - "description": "C++ SDK for the AWS eks service" - }, - "elastic-inference": { - "description": "C++ SDK for the AWS elastic-inference service" - }, - "elasticache": { - "description": "C++ SDK for the AWS elasticache service" - }, - "elasticbeanstalk": { - "description": "C++ SDK for the AWS elasticbeanstalk service" - }, - "elasticfilesystem": { - "description": "C++ SDK for the AWS elasticfilesystem service" - }, - "elasticloadbalancing": { - "description": "C++ SDK for the AWS elasticloadbalancing service" - }, - "elasticloadbalancingv2": { - "description": "C++ SDK for the AWS elasticloadbalancingv2 service" - }, - "elasticmapreduce": { - "description": "C++ SDK for the AWS elasticmapreduce service" - }, - "elastictranscoder": { - "description": "C++ SDK for the AWS elastictranscoder service" - }, - "email": { - "description": "C++ SDK for the AWS email service" - }, - "emr-containers": { - "description": "C++ SDK for the AWS emr-containers service" - }, - "emr-serverless": { - "description": "C++ SDK for the AWS emr-serverless service" - }, - "entityresolution": { - "description": "C++ SDK for the AWS entityresolution service" - }, - "es": { - "description": "C++ SDK for the AWS es service" - }, - "eventbridge": { - "description": "C++ SDK for the AWS eventbridge service" - }, - "events": { - "description": "C++ SDK for the AWS events service" - }, - "evidently": { - "description": "C++ SDK for the AWS evidently service" - }, - "finspace": { - "description": "C++ SDK for the AWS finspace service" - }, - "finspace-data": { - "description": "C++ SDK for the AWS finspace-data service" - }, - "firehose": { - "description": "C++ SDK for the AWS firehose service" - }, - "fis": { - "description": "C++ SDK for the AWS fis service" - }, - "fms": { - "description": "C++ SDK for the AWS fms service" - }, - "forecast": { - "description": "C++ SDK for the AWS forecast service" - }, - "forecastquery": { - "description": "C++ SDK for the AWS forecastquery service" - }, - "frauddetector": { - "description": "C++ SDK for the AWS frauddetector service" - }, - "fsx": { - "description": "C++ SDK for the AWS fsx service" - }, - "gamelift": { - "description": "C++ SDK for the AWS gamelift service" - }, - "glacier": { - "description": "C++ SDK for the AWS glacier service" - }, - "globalaccelerator": { - "description": "C++ SDK for the AWS globalaccelerator service" - }, - "glue": { - "description": "C++ SDK for the AWS glue service" - }, - "grafana": { - "description": "C++ SDK for the AWS grafana service" - }, - "greengrass": { - "description": "C++ SDK for the AWS greengrass service" - }, - "greengrassv2": { - "description": "C++ SDK for the AWS greengrassv2 service" - }, - "groundstation": { - "description": "C++ SDK for the AWS groundstation service" - }, - "guardduty": { - "description": "C++ SDK for the AWS guardduty service" - }, - "health": { - "description": "C++ SDK for the AWS health service" - }, - "healthlake": { - "description": "C++ SDK for the AWS healthlake service" - }, - "honeycode": { - "description": "C++ SDK for the AWS honeycode service" - }, - "iam": { - "description": "C++ SDK for the AWS iam service" - }, - "identity-management": { - "description": "C++ SDK for the AWS identity-management service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "cognito-identity", - "sts" - ] - } - ] - }, - "identitystore": { - "description": "C++ SDK for the AWS identitystore service" - }, - "imagebuilder": { - "description": "C++ SDK for the AWS imagebuilder service" - }, - "importexport": { - "description": "C++ SDK for the AWS importexport service" - }, - "inspector": { - "description": "C++ SDK for the AWS inspector service" - }, - "inspector2": { - "description": "C++ SDK for the AWS inspector2 service" - }, - "internetmonitor": { - "description": "C++ SDK for the AWS internetmonitor service" - }, - "iot": { - "description": "C++ SDK for the AWS iot service" - }, - "iot-data": { - "description": "C++ SDK for the AWS iot-data service" - }, - "iot-jobs-data": { - "description": "C++ SDK for the AWS iot-jobs-data service" - }, - "iot-roborunner": { - "description": "C++ SDK for the AWS iot-roborunner service" - }, - "iot1click-devices": { - "description": "C++ SDK for the AWS iot1click-devices service" - }, - "iot1click-projects": { - "description": "C++ SDK for the AWS iot1click-projects service" - }, - "iotanalytics": { - "description": "C++ SDK for the AWS iotanalytics service" - }, - "iotdeviceadvisor": { - "description": "C++ SDK for the AWS iotdeviceadvisor service" - }, - "iotevents": { - "description": "C++ SDK for the AWS iotevents service" - }, - "iotevents-data": { - "description": "C++ SDK for the AWS iotevents-data service" - }, - "iotfleethub": { - "description": "C++ SDK for the AWS iotfleethub service" - }, - "iotfleetwise": { - "description": "C++ SDK for the AWS iotfleetwise service" - }, - "iotsecuretunneling": { - "description": "C++ SDK for the AWS iotsecuretunneling service" - }, - "iotsitewise": { - "description": "C++ SDK for the AWS iotsitewise service" - }, - "iotthingsgraph": { - "description": "C++ SDK for the AWS iotthingsgraph service" - }, - "iottwinmaker": { - "description": "C++ SDK for the AWS iottwinmaker service" - }, - "iotwireless": { - "description": "C++ SDK for the AWS iotwireless service" - }, - "ivs": { - "description": "C++ SDK for the AWS ivs service" - }, - "ivs-realtime": { - "description": "C++ SDK for the AWS ivs-realtime service" - }, - "ivschat": { - "description": "C++ SDK for the AWS ivschat service" - }, - "kafka": { - "description": "C++ SDK for the AWS kafka service" - }, - "kafkaconnect": { - "description": "C++ SDK for the AWS kafkaconnect service" - }, - "kendra": { - "description": "C++ SDK for the AWS kendra service" - }, - "kendra-ranking": { - "description": "C++ SDK for the AWS kendra-ranking service" - }, - "keyspaces": { - "description": "C++ SDK for the AWS keyspaces service" - }, - "kinesis": { - "description": "C++ SDK for the AWS kinesis service" - }, - "kinesis-video-archived-media": { - "description": "C++ SDK for the AWS kinesis-video-archived-media service" - }, - "kinesis-video-media": { - "description": "C++ SDK for the AWS kinesis-video-media service" - }, - "kinesis-video-signaling": { - "description": "C++ SDK for the AWS kinesis-video-signaling service" - }, - "kinesis-video-webrtc-storage": { - "description": "C++ SDK for the AWS kinesis-video-webrtc-storage service" - }, - "kinesisanalytics": { - "description": "C++ SDK for the AWS kinesisanalytics service" - }, - "kinesisanalyticsv2": { - "description": "C++ SDK for the AWS kinesisanalyticsv2 service" - }, - "kinesisvideo": { - "description": "C++ SDK for the AWS kinesisvideo service" - }, - "kms": { - "description": "C++ SDK for the AWS kms service" - }, - "lakeformation": { - "description": "C++ SDK for the AWS lakeformation service" - }, - "lambda": { - "description": "C++ SDK for the AWS lambda service" - }, - "lex": { - "description": "C++ SDK for the AWS lex service" - }, - "lex-models": { - "description": "C++ SDK for the AWS lex-models service" - }, - "lexv2-models": { - "description": "C++ SDK for the AWS lexv2-models service" - }, - "lexv2-runtime": { - "description": "C++ SDK for the AWS lexv2-runtime service" - }, - "license-manager": { - "description": "C++ SDK for the AWS license-manager service" - }, - "license-manager-linux-subscriptions": { - "description": "C++ SDK for the AWS license-manager-linux-subscriptions service" - }, - "license-manager-user-subscriptions": { - "description": "C++ SDK for the AWS license-manager-user-subscriptions service" - }, - "lightsail": { - "description": "C++ SDK for the AWS lightsail service" - }, - "location": { - "description": "C++ SDK for the AWS location service" - }, - "logs": { - "description": "C++ SDK for the AWS logs service" - }, - "lookoutequipment": { - "description": "C++ SDK for the AWS lookoutequipment service" - }, - "lookoutmetrics": { - "description": "C++ SDK for the AWS lookoutmetrics service" - }, - "lookoutvision": { - "description": "C++ SDK for the AWS lookoutvision service" - }, - "m2": { - "description": "C++ SDK for the AWS m2 service" - }, - "machinelearning": { - "description": "C++ SDK for the AWS machinelearning service" - }, - "macie": { - "description": "C++ SDK for the AWS macie service" - }, - "macie2": { - "description": "C++ SDK for the AWS macie2 service" - }, - "managedblockchain": { - "description": "C++ SDK for the AWS managedblockchain service" - }, - "managedblockchain-query": { - "description": "C++ SDK for the AWS managedblockchain-query service" - }, - "marketplace-catalog": { - "description": "C++ SDK for the AWS marketplace-catalog service" - }, - "marketplace-entitlement": { - "description": "C++ SDK for the AWS marketplace-entitlement service" - }, - "marketplacecommerceanalytics": { - "description": "C++ SDK for the AWS marketplacecommerceanalytics service" - }, - "mediaconnect": { - "description": "C++ SDK for the AWS mediaconnect service" - }, - "mediaconvert": { - "description": "C++ SDK for the AWS mediaconvert service" - }, - "medialive": { - "description": "C++ SDK for the AWS medialive service" - }, - "mediapackage": { - "description": "C++ SDK for the AWS mediapackage service" - }, - "mediapackage-vod": { - "description": "C++ SDK for the AWS mediapackage-vod service" - }, - "mediapackagev2": { - "description": "C++ SDK for the AWS mediapackagev2 service" - }, - "mediastore": { - "description": "C++ SDK for the AWS mediastore service" - }, - "mediastore-data": { - "description": "C++ SDK for the AWS mediastore-data service" - }, - "mediatailor": { - "description": "C++ SDK for the AWS mediatailor service" - }, - "medical-imaging": { - "description": "C++ SDK for the AWS medical-imaging service" - }, - "memorydb": { - "description": "C++ SDK for the AWS memorydb service" - }, - "meteringmarketplace": { - "description": "C++ SDK for the AWS meteringmarketplace service" - }, - "mgn": { - "description": "C++ SDK for the AWS mgn service" - }, - "migration-hub-refactor-spaces": { - "description": "C++ SDK for the AWS migration-hub-refactor-spaces service" - }, - "migrationhub-config": { - "description": "C++ SDK for the AWS migrationhub-config service" - }, - "migrationhuborchestrator": { - "description": "C++ SDK for the AWS migrationhuborchestrator service" - }, - "migrationhubstrategy": { - "description": "C++ SDK for the AWS migrationhubstrategy service" - }, - "mobile": { - "description": "C++ SDK for the AWS mobile service" - }, - "monitoring": { - "description": "C++ SDK for the AWS monitoring service" - }, - "mq": { - "description": "C++ SDK for the AWS mq service" - }, - "mturk-requester": { - "description": "C++ SDK for the AWS mturk-requester service" - }, - "mwaa": { - "description": "C++ SDK for the AWS mwaa service" - }, - "neptune": { - "description": "C++ SDK for the AWS neptune service" - }, - "neptunedata": { - "description": "C++ SDK for the AWS neptunedata service" - }, - "network-firewall": { - "description": "C++ SDK for the AWS network-firewall service" - }, - "networkmanager": { - "description": "C++ SDK for the AWS networkmanager service" - }, - "nimble": { - "description": "C++ SDK for the AWS nimble service" - }, - "oam": { - "description": "C++ SDK for the AWS oam service" - }, - "omics": { - "description": "C++ SDK for the AWS omics service" - }, - "opensearch": { - "description": "C++ SDK for the AWS opensearch service" - }, - "opensearchserverless": { - "description": "C++ SDK for the AWS opensearchserverless service" - }, - "opsworks": { - "description": "C++ SDK for the AWS opsworks service" - }, - "opsworkscm": { - "description": "C++ SDK for the AWS opsworkscm service" - }, - "organizations": { - "description": "C++ SDK for the AWS organizations service" - }, - "osis": { - "description": "C++ SDK for the AWS osis service" - }, - "outposts": { - "description": "C++ SDK for the AWS outposts service" - }, - "panorama": { - "description": "C++ SDK for the AWS panorama service" - }, - "payment-cryptography": { - "description": "C++ SDK for the AWS payment-cryptography service" - }, - "payment-cryptography-data": { - "description": "C++ SDK for the AWS payment-cryptography-data service" - }, - "pca-connector-ad": { - "description": "C++ SDK for the AWS pca-connector-ad service" - }, - "personalize": { - "description": "C++ SDK for the AWS personalize service" - }, - "personalize-events": { - "description": "C++ SDK for the AWS personalize-events service" - }, - "personalize-runtime": { - "description": "C++ SDK for the AWS personalize-runtime service" - }, - "pi": { - "description": "C++ SDK for the AWS pi service" - }, - "pinpoint": { - "description": "C++ SDK for the AWS pinpoint service" - }, - "pinpoint-email": { - "description": "C++ SDK for the AWS pinpoint-email service" - }, - "pinpoint-sms-voice-v2": { - "description": "C++ SDK for the AWS pinpoint-sms-voice-v2 service" - }, - "pipes": { - "description": "C++ SDK for the AWS pipes service" - }, - "polly": { - "description": "C++ SDK for the AWS polly service" - }, - "pricing": { - "description": "C++ SDK for the AWS pricing service" - }, - "privatenetworks": { - "description": "C++ SDK for the AWS privatenetworks service" - }, - "proton": { - "description": "C++ SDK for the AWS proton service" - }, - "qldb": { - "description": "C++ SDK for the AWS qldb service" - }, - "qldb-session": { - "description": "C++ SDK for the AWS qldb-session service" - }, - "queues": { - "description": "C++ SDK for the AWS queues service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "sqs" - ] - } - ] - }, - "quicksight": { - "description": "C++ SDK for the AWS quicksight service" - }, - "ram": { - "description": "C++ SDK for the AWS ram service" - }, - "rbin": { - "description": "C++ SDK for the AWS rbin service" - }, - "rds": { - "description": "C++ SDK for the AWS rds service" - }, - "rds-data": { - "description": "C++ SDK for the AWS rds-data service" - }, - "redshift": { - "description": "C++ SDK for the AWS redshift service" - }, - "redshift-data": { - "description": "C++ SDK for the AWS redshift-data service" - }, - "redshift-serverless": { - "description": "C++ SDK for the AWS redshift-serverless service" - }, - "rekognition": { - "description": "C++ SDK for the AWS rekognition service" - }, - "resiliencehub": { - "description": "C++ SDK for the AWS resiliencehub service" - }, - "resource-explorer-2": { - "description": "C++ SDK for the AWS resource-explorer-2 service" - }, - "resource-groups": { - "description": "C++ SDK for the AWS resource-groups service" - }, - "resourcegroupstaggingapi": { - "description": "C++ SDK for the AWS resourcegroupstaggingapi service" - }, - "robomaker": { - "description": "C++ SDK for the AWS robomaker service" - }, - "rolesanywhere": { - "description": "C++ SDK for the AWS rolesanywhere service" - }, - "route53": { - "description": "C++ SDK for the AWS route53 service" - }, - "route53-recovery-cluster": { - "description": "C++ SDK for the AWS route53-recovery-cluster service" - }, - "route53-recovery-control-config": { - "description": "C++ SDK for the AWS route53-recovery-control-config service" - }, - "route53-recovery-readiness": { - "description": "C++ SDK for the AWS route53-recovery-readiness service" - }, - "route53domains": { - "description": "C++ SDK for the AWS route53domains service" - }, - "route53resolver": { - "description": "C++ SDK for the AWS route53resolver service" - }, - "rum": { - "description": "C++ SDK for the AWS rum service" - }, - "s3": { - "description": "C++ SDK for the AWS s3 service" - }, - "s3-crt": { - "description": "C++ SDK for the AWS s3-crt service" - }, - "s3-encryption": { - "description": "C++ SDK for the AWS s3-encryption service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "kms", - "s3" - ] - } - ] - }, - "s3control": { - "description": "C++ SDK for the AWS s3control service" - }, - "s3outposts": { - "description": "C++ SDK for the AWS s3outposts service" - }, - "sagemaker": { - "description": "C++ SDK for the AWS sagemaker service" - }, - "sagemaker-a2i-runtime": { - "description": "C++ SDK for the AWS sagemaker-a2i-runtime service" - }, - "sagemaker-edge": { - "description": "C++ SDK for the AWS sagemaker-edge service" - }, - "sagemaker-featurestore-runtime": { - "description": "C++ SDK for the AWS sagemaker-featurestore-runtime service" - }, - "sagemaker-geospatial": { - "description": "C++ SDK for the AWS sagemaker-geospatial service" - }, - "sagemaker-metrics": { - "description": "C++ SDK for the AWS sagemaker-metrics service" - }, - "sagemaker-runtime": { - "description": "C++ SDK for the AWS sagemaker-runtime service" - }, - "savingsplans": { - "description": "C++ SDK for the AWS savingsplans service" - }, - "scheduler": { - "description": "C++ SDK for the AWS scheduler service" - }, - "schemas": { - "description": "C++ SDK for the AWS schemas service" - }, - "sdb": { - "description": "C++ SDK for the AWS sdb service" - }, - "secretsmanager": { - "description": "C++ SDK for the AWS secretsmanager service" - }, - "securityhub": { - "description": "C++ SDK for the AWS securityhub service" - }, - "securitylake": { - "description": "C++ SDK for the AWS securitylake service" - }, - "serverlessrepo": { - "description": "C++ SDK for the AWS serverlessrepo service" - }, - "service-quotas": { - "description": "C++ SDK for the AWS service-quotas service" - }, - "servicecatalog": { - "description": "C++ SDK for the AWS servicecatalog service" - }, - "servicecatalog-appregistry": { - "description": "C++ SDK for the AWS servicecatalog-appregistry service" - }, - "servicediscovery": { - "description": "C++ SDK for the AWS servicediscovery service" - }, - "sesv2": { - "description": "C++ SDK for the AWS sesv2 service" - }, - "shield": { - "description": "C++ SDK for the AWS shield service" - }, - "signer": { - "description": "C++ SDK for the AWS signer service" - }, - "simspaceweaver": { - "description": "C++ SDK for the AWS simspaceweaver service" - }, - "sms": { - "description": "C++ SDK for the AWS sms service" - }, - "sms-voice": { - "description": "C++ SDK for the AWS sms-voice service" - }, - "snow-device-management": { - "description": "C++ SDK for the AWS snow-device-management service" - }, - "snowball": { - "description": "C++ SDK for the AWS snowball service" - }, - "sns": { - "description": "C++ SDK for the AWS sns service" - }, - "sqs": { - "description": "C++ SDK for the AWS sqs service" - }, - "ssm": { - "description": "C++ SDK for the AWS ssm service" - }, - "ssm-contacts": { - "description": "C++ SDK for the AWS ssm-contacts service" - }, - "ssm-incidents": { - "description": "C++ SDK for the AWS ssm-incidents service" - }, - "ssm-sap": { - "description": "C++ SDK for the AWS ssm-sap service" - }, - "sso": { - "description": "C++ SDK for the AWS sso service" - }, - "sso-admin": { - "description": "C++ SDK for the AWS sso-admin service" - }, - "sso-oidc": { - "description": "C++ SDK for the AWS sso-oidc service" - }, - "states": { - "description": "C++ SDK for the AWS states service" - }, - "storagegateway": { - "description": "C++ SDK for the AWS storagegateway service" - }, - "sts": { - "description": "C++ SDK for the AWS sts service" - }, - "support": { - "description": "C++ SDK for the AWS support service" - }, - "support-app": { - "description": "C++ SDK for the AWS support-app service" - }, - "swf": { - "description": "C++ SDK for the AWS swf service" - }, - "synthetics": { - "description": "C++ SDK for the AWS synthetics service" - }, - "text-to-speech": { - "description": "C++ SDK for the AWS text-to-speech service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "polly" - ] - } - ] - }, - "textract": { - "description": "C++ SDK for the AWS textract service" - }, - "timestream-query": { - "description": "C++ SDK for the AWS timestream-query service" - }, - "timestream-write": { - "description": "C++ SDK for the AWS timestream-write service" - }, - "tnb": { - "description": "C++ SDK for the AWS tnb service" - }, - "transcribe": { - "description": "C++ SDK for the AWS transcribe service" - }, - "transcribestreaming": { - "description": "C++ SDK for the AWS transcribestreaming service" - }, - "transfer": { - "description": "C++ SDK for the AWS transfer service", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "default-features": false, - "features": [ - "s3" - ] - } - ] - }, - "translate": { - "description": "C++ SDK for the AWS translate service" - }, - "verifiedpermissions": { - "description": "C++ SDK for the AWS verifiedpermissions service" - }, - "voice-id": { - "description": "C++ SDK for the AWS voice-id service" - }, - "vpc-lattice": { - "description": "C++ SDK for the AWS vpc-lattice service" - }, - "waf": { - "description": "C++ SDK for the AWS waf service" - }, - "waf-regional": { - "description": "C++ SDK for the AWS waf-regional service" - }, - "wafv2": { - "description": "C++ SDK for the AWS wafv2 service" - }, - "wellarchitected": { - "description": "C++ SDK for the AWS wellarchitected service" - }, - "wisdom": { - "description": "C++ SDK for the AWS wisdom service" - }, - "workdocs": { - "description": "C++ SDK for the AWS workdocs service" - }, - "worklink": { - "description": "C++ SDK for the AWS worklink service" - }, - "workmail": { - "description": "C++ SDK for the AWS workmail service" - }, - "workmailmessageflow": { - "description": "C++ SDK for the AWS workmailmessageflow service" - }, - "workspaces": { - "description": "C++ SDK for the AWS workspaces service" - }, - "workspaces-web": { - "description": "C++ SDK for the AWS workspaces-web service" - }, - "xray": { - "description": "C++ SDK for the AWS xray service" - } - } -} diff --git a/ports/azure-storage-common-cpp/portfile.cmake b/ports/azure-storage-common-cpp/portfile.cmake new file mode 100644 index 000000000000..482ed9687f13 --- /dev/null +++ b/ports/azure-storage-common-cpp/portfile.cmake @@ -0,0 +1,32 @@ +# NOTE: All changes made to this file will get overwritten by the next port release. +# Please contribute your changes to https://github.com/Azure/azure-sdk-for-cpp. + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Azure/azure-sdk-for-cpp + REF azure-storage-common_12.5.0 + SHA512 46c9cc4ebec54f289ba7270356da4e89e5f8e890004c7a232200b87ca33357064c2f46670a1090fe41ca6962cdbc76d2e3520bb600438cbc0f21f15cf7816f04 +) + +if(EXISTS "${SOURCE_PATH}/sdk/storage/azure-storage-common") + file(REMOVE_RECURSE "${SOURCE_PATH}/sdk/storage/_") + file(REMOVE_RECURSE "${SOURCE_PATH}/sdk/_") + file(REMOVE_RECURSE "${SOURCE_PATH}/_") + + file(RENAME "${SOURCE_PATH}/sdk/storage/azure-storage-common" "${SOURCE_PATH}/sdk/storage/_") + file(RENAME "${SOURCE_PATH}/sdk/storage" "${SOURCE_PATH}/sdk/_") + file(RENAME "${SOURCE_PATH}/sdk" "${SOURCE_PATH}/_") +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/_/_/_" + OPTIONS + -DWARNINGS_AS_ERRORS=OFF + -DBUILD_TESTING=OFF +) + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_copy_pdbs() diff --git a/ports/azure-storage-common-cpp/vcpkg.json b/ports/azure-storage-common-cpp/vcpkg.json new file mode 100644 index 000000000000..8dd26d908204 --- /dev/null +++ b/ports/azure-storage-common-cpp/vcpkg.json @@ -0,0 +1,36 @@ +{ + "$comment": "NOTE: All changes made to this file will get overwritten by the next port release. Please contribute your changes to https://github.com/Azure/azure-sdk-for-cpp.", + "name": "azure-storage-common-cpp", + "version-semver": "12.5.0", + "port-version": 1, + "description": [ + "Microsoft Azure Common Storage SDK for C++", + "This library provides common Azure Storage-related abstractions for Azure SDK." + ], + "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/storage/azure-storage-common", + "license": "MIT", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false, + "version>=": "1.10.3" + }, + { + "name": "libxml2", + "default-features": false, + "platform": "!windows" + }, + { + "name": "openssl", + "platform": "!windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/ports/bzip2/CMakeLists.txt b/ports/bzip2/CMakeLists.txt deleted file mode 100644 index e2da221d7deb..000000000000 --- a/ports/bzip2/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -cmake_minimum_required(VERSION 3.0) -project(bzip2 C) - -if(CMAKE_BUILD_TYPE STREQUAL Debug) - add_definitions(-DBZ_DEBUG) # enable extra assertions -endif() - -set(BZ2_SOURCES - blocksort.c - huffman.c - crctable.c - randtable.c - compress.c - decompress.c - bzlib.c) - -add_library(bz2 ${BZ2_SOURCES}) -set_target_properties(bz2 PROPERTIES - DEBUG_POSTFIX d - VERSION 1.0.6 - SOVERSION 1.0) -if(BUILD_SHARED_LIBS) - target_compile_definitions(bz2 PRIVATE -DBZ_BUILD_DLL) -endif() - -if(MSVC) - add_definitions(-D_CRT_SECURE_NO_WARNINGS) - add_definitions(-D_CRT_SECURE_NO_DEPRECATE) - add_definitions(-D_CRT_NONSTDC_NO_DEPRECATE) -endif() - -install(TARGETS bz2 - RUNTIME DESTINATION bin - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib) - -if(NOT BZIP2_SKIP_TOOLS) - add_executable(bzip2 bzip2.c ${BZ2_SOURCES}) - add_executable(bzip2recover bzip2recover.c ${BZ2_SOURCES}) - install(TARGETS bzip2 bzip2recover DESTINATION tools/bzip2) -endif() - -if(NOT BZIP2_SKIP_HEADERS) - install(FILES bzlib.h DESTINATION include) -endif() diff --git a/ports/bzip2/bzip2.pc.in b/ports/bzip2/bzip2.pc.in deleted file mode 100644 index 91153c13b532..000000000000 --- a/ports/bzip2/bzip2.pc.in +++ /dev/null @@ -1,11 +0,0 @@ -prefix=@BZIP2_PREFIX@ -exec_prefix=${prefix} -libdir=${prefix}/lib -includedir=${prefix}/include - -Name: bzip2 -Description: bzip2 -Version: 1.0.6 -Requires: -Libs: -L${libdir} -l@bzname@ -Cflags: -I${includedir} \ No newline at end of file diff --git a/ports/bzip2/fix-import-export-macros.patch b/ports/bzip2/fix-import-export-macros.patch deleted file mode 100644 index fc67887da4e5..000000000000 --- a/ports/bzip2/fix-import-export-macros.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/bzlib.h b/bzlib.h -index 8277123..84fbd0a 100644 ---- a/bzlib.h -+++ b/bzlib.h -@@ -65,29 +65,23 @@ typedef - } - bz_stream; - -- --#ifndef BZ_IMPORT --#define BZ_EXPORT --#endif -- - #ifndef BZ_NO_STDIO - /* Need a definitition for FILE */ - #include - #endif - - #ifdef _WIN32 --# include - # ifdef small - /* windows.h define small to char */ - # undef small - # endif --# ifdef BZ_EXPORT --# define BZ_API(func) WINAPI func --# define BZ_EXTERN extern -+# define BZ_API(func) func -+# if defined(BZ_BUILD_DLL) -+# define BZ_EXTERN __declspec(dllexport) -+# elif defined(BZ_IMPORT) -+# define BZ_EXTERN __declspec(dllimport) - # else -- /* import windows dll dynamically */ --# define BZ_API(func) (WINAPI * func) --# define BZ_EXTERN -+# define BZ_EXTERN - # endif - #else - # define BZ_API(func) func diff --git a/ports/bzip2/portfile.cmake b/ports/bzip2/portfile.cmake deleted file mode 100644 index c03f48c48fb7..000000000000 --- a/ports/bzip2/portfile.cmake +++ /dev/null @@ -1,56 +0,0 @@ -set(BZIP2_VERSION 1.0.8) -vcpkg_download_distfile(ARCHIVE - URLS "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" - FILENAME "bzip2-${BZIP2_VERSION}.tar.gz" - SHA512 083f5e675d73f3233c7930ebe20425a533feedeaaa9d8cc86831312a6581cefbe6ed0d08d2fa89be81082f2a5abdabca8b3c080bf97218a1bd59dc118a30b9f3 -) - -vcpkg_extract_source_archive_ex( - OUT_SOURCE_PATH SOURCE_PATH - ARCHIVE ${ARCHIVE} - PATCHES fix-import-export-macros.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - INVERTED_FEATURES - tool BZIP2_SKIP_TOOLS -) - -file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}") - -vcpkg_cmake_configure( - SOURCE_PATH ${SOURCE_PATH} - OPTIONS - ${FEATURE_OPTIONS} - OPTIONS_DEBUG - -DBZIP2_SKIP_HEADERS=ON - -DBZIP2_SKIP_TOOLS=ON -) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -file(READ "${CURRENT_PACKAGES_DIR}/include/bzlib.h" BZLIB_H) -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - string(REPLACE "defined(BZ_IMPORT)" "0" BZLIB_H "${BZLIB_H}") -else() - string(REPLACE "defined(BZ_IMPORT)" "1" BZLIB_H "${BZLIB_H}") -endif() -file(WRITE "${CURRENT_PACKAGES_DIR}/include/bzlib.h" "${BZLIB_H}") - -if (NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - set(BZIP2_PREFIX "${CURRENT_INSTALLED_DIR}") - set(bzname bz2) - configure_file("${CMAKE_CURRENT_LIST_DIR}/bzip2.pc.in" "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/bzip2.pc" @ONLY) -endif() - -if (NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - set(BZIP2_PREFIX "${CURRENT_INSTALLED_DIR}/debug") - set(bzname bz2d) - configure_file("${CMAKE_CURRENT_LIST_DIR}/bzip2.pc.in" "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/bzip2.pc" @ONLY) -endif() - -vcpkg_fixup_pkgconfig() - -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) -file(COPY "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/ports/bzip2/usage b/ports/bzip2/usage deleted file mode 100644 index 084bf390ad51..000000000000 --- a/ports/bzip2/usage +++ /dev/null @@ -1,4 +0,0 @@ -The package bzip2 is compatible with built-in CMake targets: - - find_package(BZip2 REQUIRED) - target_link_libraries(main PRIVATE BZip2::BZip2) diff --git a/ports/bzip2/vcpkg.json b/ports/bzip2/vcpkg.json deleted file mode 100644 index b899ec154572..000000000000 --- a/ports/bzip2/vcpkg.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "bzip2", - "version-semver": "1.0.8", - "port-version": 0, - "description": "bzip2 is a freely available, patent free, high-quality data compressor. It typically compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical compressors), whilst being around twice as fast at compression and six times faster at decompression.", - "homepage": "https://sourceware.org/bzip2/", - "documentation": "https://sourceware.org/bzip2/docs.html", - "license": "bzip2-1.0.6", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "tool": { - "description": "Builds bzip2 executable" - } - } -} diff --git a/ports/catch2/fix-install-path.patch b/ports/catch2/fix-install-path.patch deleted file mode 100644 index c89dacdb5855..000000000000 --- a/ports/catch2/fix-install-path.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -164,7 +164,7 @@ - - ## Provide some pkg-config integration - set(PKGCONFIG_INSTALL_DIR -- "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig" -+ "${CMAKE_INSTALL_LIBDIR}/pkgconfig" - CACHE PATH "Path where catch2.pc is installed" - ) - configure_file( -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -401,16 +401,28 @@ - install( - TARGETS - Catch2 -- Catch2WithMain - EXPORT - Catch2Targets - LIBRARY DESTINATION - ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION - ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION - ${CMAKE_INSTALL_BINDIR} - ) -+ -+ install( -+ TARGETS -+ Catch2WithMain -+ EXPORT -+ Catch2Targets -+ LIBRARY DESTINATION -+ ${CMAKE_INSTALL_LIBDIR}/manual-link -+ ARCHIVE DESTINATION -+ ${CMAKE_INSTALL_LIBDIR}/manual-link -+ RUNTIME DESTINATION -+ ${CMAKE_INSTALL_BINDIR} -+ ) - - - install( diff --git a/ports/catch2/fix-uwp-build.patch b/ports/catch2/fix-uwp-build.patch deleted file mode 100644 index feee34cfbae0..000000000000 --- a/ports/catch2/fix-uwp-build.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp -index 4465831d..7555a21f 100644 ---- a/src/catch2/catch_config.cpp -+++ b/src/catch2/catch_config.cpp -@@ -26,7 +26,12 @@ namespace { - # pragma warning( disable : 4996 ) - # endif - -+# if defined( WINAPI_FAMILY ) -+ // UWP doesn't support environment variables. -+ return false; -+# else - return std::getenv( "BAZEL_TEST" ) != nullptr; -+# endif - - # if defined( _MSC_VER ) - # pragma warning( pop ) -@@ -81,6 +86,8 @@ namespace Catch { - } ); - } - -+#if !defined( WINAPI_FAMILY ) -+ // UWP doesn't support environment variables. - if(provideBazelReporterOutput()){ - // Register a JUnit reporter for Bazel. Bazel sets an environment - // variable with the path to XML output. If this file is written to -@@ -102,6 +109,7 @@ namespace Catch { - { "junit", std::string( bazelOutputFilePtr ), {}, {} } ); - } - } -+#endif - - - // We now fixup the reporter specs to handle default output spec, diff --git a/ports/catch2/portfile.cmake b/ports/catch2/portfile.cmake deleted file mode 100644 index 3c6090e51185..000000000000 --- a/ports/catch2/portfile.cmake +++ /dev/null @@ -1,36 +0,0 @@ - -if(VCPKG_TARGET_IS_WINDOWS) - vcpkg_check_linkage(ONLY_STATIC_LIBRARY) -endif() - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO catchorg/Catch2 - REF v3.3.2 - SHA512 3d0c5666509a19be54ea0c48a3c8e1c4a951a2d991a7c9f7fe6d326661464538f1ab9dc573b1b2647f49fb6bef45bbd866142a4ce0fba38545ad182b8d55f61f - HEAD_REF devel - PATCHES - fix-install-path.patch -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DCATCH_INSTALL_DOCS=OFF - -DCMAKE_CXX_STANDARD=17 -) - -vcpkg_cmake_install() - -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Catch2) -vcpkg_fixup_pkgconfig() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -# We remove these folders because they are empty and cause warnings on the library installation -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/catch2/benchmark/internal") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/catch2/generators/internal") - -file(WRITE "${CURRENT_PACKAGES_DIR}/include/catch.hpp" "#include ") -file(INSTALL "${SOURCE_PATH}/LICENSE.txt" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/catch2/vcpkg.json b/ports/catch2/vcpkg.json deleted file mode 100644 index f3ab9ccca03a..000000000000 --- a/ports/catch2/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "catch2", - "version-semver": "3.3.2", - "port-version": 1, - "description": "A modern, header-only test framework for unit testing.", - "homepage": "https://github.com/catchorg/Catch2", - "license": "BSL-1.0", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/crc32c/fix_clang-cl.patch b/ports/crc32c/fix_clang-cl.patch deleted file mode 100644 index d120243209c1..000000000000 --- a/ports/crc32c/fix_clang-cl.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 84907285c..8c446b0c8 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -22,7 +22,7 @@ if(NOT CMAKE_CXX_STANDARD) - endif(NOT CMAKE_CXX_STANDARD) - - # https://github.com/izenecloud/cmake/blob/master/SetCompilerWarningAll.cmake --if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") -+if(MSVC) - # Use the highest warning level for Visual Studio. - set(CMAKE_CXX_WARNING_LEVEL 4) - if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") diff --git a/ports/crc32c/portfile.cmake b/ports/crc32c/portfile.cmake deleted file mode 100644 index 0280002fd2a5..000000000000 --- a/ports/crc32c/portfile.cmake +++ /dev/null @@ -1,33 +0,0 @@ -vcpkg_check_linkage(ONLY_STATIC_LIBRARY) - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO google/crc32c - REF 1.1.2 - SHA512 6325c52b5a6850b9f90086e0c0c86798c008af36e7cfd3a0216184a2d37b3bf7323481ddc6bfe4cbd5b31288b3ee6c69772d03085a13094cf95d00a9756a7196 - HEAD_REF master - PATCHES - fix_clang-cl.patch -) - -vcpkg_cmake_configure( - SOURCE_PATH ${SOURCE_PATH} - OPTIONS - -DCRC32C_BUILD_TESTS=OFF - -DCRC32C_BUILD_BENCHMARKS=OFF - -DCRC32C_USE_GLOG=OFF -) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Crc32c) - -file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) -file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/share) - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/bin ${CURRENT_PACKAGES_DIR}/debug/bin) -endif() - -# Handle copyright -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/crc32c/vcpkg.json b/ports/crc32c/vcpkg.json deleted file mode 100644 index 3acb74812e56..000000000000 --- a/ports/crc32c/vcpkg.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "crc32c", - "version": "1.1.2", - "port-version": 1, - "description": "CRC32C implementation with support for CPU-specific acceleration instructions.", - "homepage": "https://github.com/google/crc32c", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/fmt/fix-format-conflict.patch b/ports/fmt/fix-format-conflict.patch deleted file mode 100644 index 0a2ac6a57a24..000000000000 --- a/ports/fmt/fix-format-conflict.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/include/fmt/format-inl.h b/include/fmt/format-inl.h -index f44df01c..59cc4559 100644 ---- a/include/fmt/format-inl.h -+++ b/include/fmt/format-inl.h -@@ -62,8 +62,8 @@ FMT_FUNC void format_error_code(detail::buffer& out, int error_code, - error_code_size += detail::to_unsigned(detail::count_digits(abs_value)); - auto it = buffer_appender(out); - if (message.size() <= inline_buffer_size - error_code_size) -- format_to(it, FMT_STRING("{}{}"), message, SEP); -- format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code); -+ fmt::format_to(it, FMT_STRING("{}{}"), message, SEP); -+ fmt::format_to(it, FMT_STRING("{}{}"), ERROR_STR, error_code); - FMT_ASSERT(out.size() <= inline_buffer_size, ""); - } - diff --git a/ports/fmt/fix-write-batch.patch b/ports/fmt/fix-write-batch.patch deleted file mode 100644 index 7c17d6037728..000000000000 --- a/ports/fmt/fix-write-batch.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f21cf45..691a632 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -157,7 +157,7 @@ if (MASTER_PROJECT AND CMAKE_GENERATOR MATCHES "Visual Studio") - join(netfxpath - "C:\\Program Files\\Reference Assemblies\\Microsoft\\Framework\\" - ".NETFramework\\v4.0") -- file(WRITE run-msbuild.bat " -+ file(WRITE ${CMAKE_BINARY_DIR}/run-msbuild.bat " - ${MSBUILD_SETUP} - ${CMAKE_MAKE_PROGRAM} -p:FrameworkPathOverride=\"${netfxpath}\" %*") - endif () diff --git a/ports/fmt/portfile.cmake b/ports/fmt/portfile.cmake deleted file mode 100644 index f5f819a01a94..000000000000 --- a/ports/fmt/portfile.cmake +++ /dev/null @@ -1,66 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO fmtlib/fmt - REF 9.1.0 - SHA512 a18442042722dd48e20714ec034a12fcc0576c9af7be5188586970e2edf47529825bdc99af366b1d5891630c8dbf6f63bfa9f012e77ab3d3ed80d1a118e3b2be - HEAD_REF master - PATCHES - fix-write-batch.patch - fix-format-conflict.patch -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DFMT_CMAKE_DIR=share/fmt - -DFMT_TEST=OFF - -DFMT_DOC=OFF -) - -vcpkg_cmake_install() -file(INSTALL "${SOURCE_PATH}/LICENSE.rst" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) -if(VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) - if(VCPKG_TARGET_IS_WINDOWS) - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/lib/fmtd.dll") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/debug/bin") - file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/fmtd.dll" "${CURRENT_PACKAGES_DIR}/debug/bin/fmtd.dll") - endif() - endif() - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/fmt.dll") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/bin") - file(RENAME "${CURRENT_PACKAGES_DIR}/lib/fmt.dll" "${CURRENT_PACKAGES_DIR}/bin/fmt.dll") - endif() - endif() - endif() - - vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/include/fmt/core.h - "defined(FMT_SHARED)" - "1" - ) -endif() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") - -vcpkg_cmake_config_fixup() -vcpkg_fixup_pkgconfig() - -if(VCPKG_TARGET_IS_WINDOWS) - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/share/fmt/fmt-targets-debug.cmake - "lib/fmtd.dll" - "bin/fmtd.dll" - ) - endif() - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/share/fmt/fmt-targets-release.cmake - "lib/fmt.dll" - "bin/fmt.dll" - ) - endif() -endif() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -# Handle post-build CMake instructions -vcpkg_copy_pdbs() -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/ports/fmt/usage b/ports/fmt/usage deleted file mode 100644 index e5a9d7048069..000000000000 --- a/ports/fmt/usage +++ /dev/null @@ -1,8 +0,0 @@ -The package fmt provides CMake targets: - - find_package(fmt CONFIG REQUIRED) - target_link_libraries(main PRIVATE fmt::fmt) - - # Or use the header-only version - find_package(fmt CONFIG REQUIRED) - target_link_libraries(main PRIVATE fmt::fmt-header-only) diff --git a/ports/fmt/vcpkg.json b/ports/fmt/vcpkg.json deleted file mode 100644 index a768a4e960d9..000000000000 --- a/ports/fmt/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "fmt", - "version": "9.1.0", - "port-version": 1, - "description": "Formatting library for C++. It can be used as a safe alternative to printf or as a fast alternative to IOStreams.", - "homepage": "https://github.com/fmtlib/fmt", - "license": "MIT", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/google-cloud-cpp/portfile.cmake b/ports/google-cloud-cpp/portfile.cmake deleted file mode 100644 index 2cae4df0a1d3..000000000000 --- a/ports/google-cloud-cpp/portfile.cmake +++ /dev/null @@ -1,85 +0,0 @@ -vcpkg_check_linkage(ONLY_STATIC_LIBRARY) - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO googleapis/google-cloud-cpp - REF "v${VERSION}" - SHA512 225202a8e799f630f0b07c392bf305c28e21b99ef8dc5a670238a6d08e0e2816cd8ca1c43d7b252bcf5d289f875e64c16413085f63663265169807fd59977e43 - HEAD_REF main - PATCHES - support_absl_cxx17.patch -) - -if ("grpc-common" IN_LIST FEATURES) - vcpkg_add_to_path(PREPEND "${CURRENT_HOST_INSTALLED_DIR}/tools/grpc") -endif () - -set(GOOGLE_CLOUD_CPP_ENABLE "${FEATURES}") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "core") -# This feature does not exist, but allows us to simplify the vcpkg.json -# file. -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "grpc-common") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "rest-common") -list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "googleapis") -# google-cloud-cpp uses dialogflow_cx and dialogflow_es. Underscores -# are invalid in `vcpkg` features, we use dashes (`-`) as a separator -# for the `vcpkg` feature name, and convert it here to something that -# `google-cloud-cpp` would like. -if ("dialogflow-cx" IN_LIST FEATURES) - list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-cx") - list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_cx") -endif () -if ("dialogflow-es" IN_LIST FEATURES) - list(REMOVE_ITEM GOOGLE_CLOUD_CPP_ENABLE "dialogflow-es") - list(APPEND GOOGLE_CLOUD_CPP_ENABLE "dialogflow_es") -endif () - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - "-DGOOGLE_CLOUD_CPP_ENABLE=${GOOGLE_CLOUD_CPP_ENABLE}" - -DGOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_WERROR=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_CCACHE=OFF - -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF - -DBUILD_TESTING=OFF - # This is needed by the `experimental-storage-grpc` feature until vcpkg - # gets Protobuf >= 4.23.0. It has no effect for other features, so - # it is simpler to just always turn it on. - -DGOOGLE_CLOUD_CPP_ENABLE_CTYPE_CORD_WORKAROUND=ON -) - -vcpkg_cmake_install() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -foreach(feature IN LISTS FEATURES) - set(config_path "lib/cmake/google_cloud_cpp_${feature}") - # Most features get their own package in `google-cloud-cpp`. - # The exceptions are captured by this `if()` command, basically - # things like `core` and `experimental-storage-grpc` are skipped. - if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") - continue() - endif() - vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${feature}" - CONFIG_PATH "${config_path}" - DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() -# These packages are automatically installed depending on what features are -# enabled. -foreach(suffix common googleapis grpc_utils rest_internal opentelemetry dialogflow_cx dialogflow_es) - set(config_path "lib/cmake/google_cloud_cpp_${suffix}") - if(NOT IS_DIRECTORY "${CURRENT_PACKAGES_DIR}/${config_path}") - continue() - endif() - vcpkg_cmake_config_fixup(PACKAGE_NAME "google_cloud_cpp_${suffix}" - CONFIG_PATH "${config_path}" - DO_NOT_DELETE_PARENT_CONFIG_PATH) -endforeach() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib/cmake" - "${CURRENT_PACKAGES_DIR}/debug/lib/cmake" - "${CURRENT_PACKAGES_DIR}/debug/share") -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) - -vcpkg_copy_pdbs() diff --git a/ports/google-cloud-cpp/support_absl_cxx17.patch b/ports/google-cloud-cpp/support_absl_cxx17.patch deleted file mode 100644 index 582b88a52d40..000000000000 --- a/ports/google-cloud-cpp/support_absl_cxx17.patch +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 3fb0564..b4a251b 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -29,6 +29,14 @@ if (NOT "${PROJECT_VERSION_PRE_RELEASE}" STREQUAL "") - set(PROJECT_VERSION "${PROJECT_VERSION}-${PROJECT_VERSION_PRE_RELEASE}") - endif () - -+find_package(absl CONFIG REQUIRED) -+ -+# Use CMAKE_CXX_STANDARD=17 if ABSL_USE_CXX17 is set -+if (ABSL_USE_CXX17) -+ message(STATUS "Found absl uses CXX17, enable CXX17 feature.") -+ set(CMAKE_CXX_STANDARD 17) -+endif () -+ - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3) - message( diff --git a/ports/google-cloud-cpp/vcpkg.json b/ports/google-cloud-cpp/vcpkg.json deleted file mode 100644 index 09da2aa504ad..000000000000 --- a/ports/google-cloud-cpp/vcpkg.json +++ /dev/null @@ -1,1447 +0,0 @@ -{ - "name": "google-cloud-cpp", - "version": "2.15.1", - "description": "C++ Client Libraries for Google Cloud Platform APIs.", - "homepage": "https://github.com/googleapis/google-cloud-cpp", - "license": "Apache-2.0", - "supports": "!uwp", - "dependencies": [ - "abseil", - "openssl", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "bigquery", - "bigtable", - "iam", - "pubsub", - "spanner", - "storage" - ], - "features": { - "accessapproval": { - "description": "Access Approval API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "accesscontextmanager": { - "description": "Access Context Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "advisorynotifications": { - "description": "Advisory Notifications API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "alloydb": { - "description": "Alloy DB API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apigateway": { - "description": "API Gateway API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apigeeconnect": { - "description": "Apigee Connect API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "apikeys": { - "description": "API Keys API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "appengine": { - "description": "App Engine Admin API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "artifactregistry": { - "description": "Artifact Registry API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "asset": { - "description": "Cloud Asset API C++ Client Library", - "supports": "!windows", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "accesscontextmanager", - "grpc-common", - "osconfig" - ] - } - ] - }, - "assuredworkloads": { - "description": "Assured Workloads API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "automl": { - "description": "Cloud AutoML API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "baremetalsolution": { - "description": "Bare Metal Solution API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "batch": { - "description": "Batch API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "beyondcorp": { - "description": "BeyondCorp API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "bigquery": { - "description": "The Google Cloud BigQuery C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "bigtable": { - "description": "The Google Cloud Bigtable C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "billing": { - "description": "Cloud Billing Budget API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "binaryauthorization": { - "description": "Binary Authorization API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grafeas", - "grpc-common" - ] - } - ] - }, - "certificatemanager": { - "description": "Certificate Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "channel": { - "description": "Cloud Channel API C++ Client Library", - "supports": "!windows", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "cloudbuild": { - "description": "Cloud Build API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "commerce": { - "description": "Cloud Commerce C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "composer": { - "description": "Cloud Composer C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "confidentialcomputing": { - "description": "Confidential Computing API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "connectors": { - "description": "Connectors API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "contactcenterinsights": { - "description": "Contact Center AI Insights API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "container": { - "description": "Kubernetes Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "containeranalysis": { - "description": "Container Analysis API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grafeas", - "grpc-common" - ] - } - ] - }, - "datacatalog": { - "description": "Google Cloud Data Catalog API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datafusion": { - "description": "Cloud Data Fusion API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datamigration": { - "description": "Database Migration API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dataplex": { - "description": "Cloud Dataplex API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dataproc": { - "description": "Cloud Dataproc API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datastore": { - "description": "Cloud Datastore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "datastream": { - "description": "Datastream API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "debugger": { - "description": "Stackdriver Debugger API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "deploy": { - "description": "Google Cloud Deploy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dialogflow-cx": { - "description": "Cloud Dialogflow CX API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dialogflow-es": { - "description": "Cloud Dialogflow ES API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "discoveryengine": { - "description": "Discovery Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "dlp": { - "description": "Cloud Data Loss Prevention (DLP) API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "documentai": { - "description": "Cloud Document AI API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "edgecontainer": { - "description": "Distributed Cloud Edge Container API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "eventarc": { - "description": "Eventarc API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "experimental-opentelemetry": { - "description": "OpenTelemetry C++ GCP Exporter Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common", - "trace" - ] - }, - "opentelemetry-cpp" - ] - }, - "experimental-storage-grpc": { - "description": "The GCS+gRPC plugin", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common", - "storage" - ] - } - ] - }, - "filestore": { - "description": "Cloud Filestore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "functions": { - "description": "Cloud Functions API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "gameservices": { - "description": "Game Services API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "gkehub": { - "description": "GKE Hub C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "gkemulticloud": { - "description": "Anthos Multi-Cloud C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "grafeas": { - "description": "Protocol buffers implementing the 'Grafeas API' (metadata about software artifacts)", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "grpc-common": { - "description": "Dependencies used by all gRPC-based libraries", - "dependencies": [ - "grpc", - { - "name": "grpc", - "host": true - }, - "protobuf", - { - "name": "protobuf", - "host": true - } - ] - }, - "iam": { - "description": "The Google Cloud IAM C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "iap": { - "description": "Cloud Identity-Aware Proxy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "ids": { - "description": "Cloud IDS API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "iot": { - "description": "Cloud IoT API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "kms": { - "description": "Cloud Key Management Service (KMS) API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "language": { - "description": "Cloud Natural Language API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "logging": { - "description": "Google Cloud Logging C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "managedidentities": { - "description": "Managed Service for Microsoft Active Directory API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "memcache": { - "description": "Cloud Memorystore for Memcached API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "metastore": { - "description": "Dataproc Metastore API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "migrationcenter": { - "description": "Migration Center API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "monitoring": { - "description": "Cloud Monitoring API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networkconnectivity": { - "description": "Network Connectivity API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networkmanagement": { - "description": "Network Management API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "networksecurity": { - "description": "Secure Web Proxy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "notebooks": { - "description": "Notebooks API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "optimization": { - "description": "Cloud Optimization API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "orgpolicy": { - "description": "Organization Policy API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "osconfig": { - "description": "Cloud OS Config API C++ Client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "oslogin": { - "description": "Cloud OS Login API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "policysimulator": { - "description": "Policy Simulator API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "policytroubleshooter": { - "description": "Policy Troubleshooter API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "privateca": { - "description": "Certificate Authority API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "profiler": { - "description": "Cloud Profiler API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "pubsub": { - "description": "The Google Cloud Bigtable C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "rapidmigrationassessment": { - "description": "Rapid Migration Assessment C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "recommender": { - "description": "Recommender C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "redis": { - "description": "Google Cloud Memorystore for Redis API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "resourcemanager": { - "description": "Cloud Resource Manager API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "resourcesettings": { - "description": "Resource Settings API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "rest-common": { - "description": "Dependencies used by all REST-based libraries", - "dependencies": [ - { - "name": "curl", - "features": [ - "ssl" - ] - }, - "nlohmann-json" - ] - }, - "retail": { - "description": "Retail API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "run": { - "description": "Cloud Run Admin API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "scheduler": { - "description": "Cloud Scheduler API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "secretmanager": { - "description": "The Google Cloud Secret Manager C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "securitycenter": { - "description": "Security Command Center API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicecontrol": { - "description": "Service Control API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicedirectory": { - "description": "Service Directory API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "servicemanagement": { - "description": "Service Management API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "serviceusage": { - "description": "Service Usage API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "shell": { - "description": "Cloud Shell API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "spanner": { - "description": "The Google Cloud Spanner C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "speech": { - "description": "The Google Cloud Speech-to-Text C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "storage": { - "description": "The Google Cloud Storage C++ client library", - "dependencies": [ - "crc32c", - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "rest-common" - ] - } - ] - }, - "storageinsights": { - "description": "Storage Insights API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "storagetransfer": { - "description": "Storage Transfer API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "support": { - "description": "Cloud Support API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "talent": { - "description": "Cloud Talent Solution API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "tasks": { - "description": "The Google Cloud Tasks C++ client library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "texttospeech": { - "description": "Cloud Text-to-Speech API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "tpu": { - "description": "Cloud TPU API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "trace": { - "description": "Stackdriver Trace API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "translate": { - "description": "Cloud Translation API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "video": { - "description": "Video Services C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "videointelligence": { - "description": "Cloud Video Intelligence API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vision": { - "description": "Cloud Vision API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vmmigration": { - "description": "VM Migration API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vmwareengine": { - "description": "VMware Engine API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "vpcaccess": { - "description": "Serverless VPC Access API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "webrisk": { - "description": "Web Risk API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "websecurityscanner": { - "description": "Web Security Scanner API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "workflows": { - "description": "Workflow Executions API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - }, - "workstations": { - "description": "Workstations API C++ Client Library", - "dependencies": [ - { - "name": "google-cloud-cpp", - "default-features": false, - "features": [ - "grpc-common" - ] - } - ] - } - } -} diff --git a/ports/liblzma/add_support_ios.patch b/ports/liblzma/add_support_ios.patch deleted file mode 100644 index 79741639b60a..000000000000 --- a/ports/liblzma/add_support_ios.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 52439b3..0b5e371 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -574,6 +574,7 @@ if(HAVE_GETOPT_LONG) - - install(TARGETS xzdec - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}" - COMPONENT xzdec) - - if(UNIX) -@@ -701,6 +702,7 @@ if(NOT MSVC AND HAVE_GETOPT_LONG) - - install(TARGETS xz - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ BUNDLE DESTINATION "${CMAKE_INSTALL_BINDIR}" - COMPONENT xz) - - if(UNIX) diff --git a/ports/liblzma/build-tools.patch b/ports/liblzma/build-tools.patch deleted file mode 100644 index 759345ef23af..000000000000 --- a/ports/liblzma/build-tools.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 03b8301..820d08e 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -584,6 +584,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/liblzma-config.cmake" - COMPONENT liblzma_Development) - - -+if(BUILD_TOOLS) - ############################################################################# - # getopt_long - ############################################################################# -@@ -793,6 +794,7 @@ if(NOT MSVC AND HAVE_GETOPT_LONG) - endforeach() - endif() - endif() -+endif() - - - ############################################################################# diff --git a/ports/liblzma/fix_config_include.patch b/ports/liblzma/fix_config_include.patch deleted file mode 100644 index 91dc4c13ba54..000000000000 --- a/ports/liblzma/fix_config_include.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 34c6aca00..7b3708ab2 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -413,6 +413,7 @@ if(WIN32) - if(BUILD_SHARED_LIBS) - # Add the Windows resource file for liblzma.dll. - target_sources(liblzma PRIVATE src/liblzma/liblzma_w32res.rc) -+ target_include_directories(liblzma PRIVATE windows/vs2019) - - set_target_properties(liblzma PROPERTIES - LINK_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/common/common_w32res.rc" diff --git a/ports/liblzma/portfile.cmake b/ports/liblzma/portfile.cmake deleted file mode 100644 index 8631df28b0f0..000000000000 --- a/ports/liblzma/portfile.cmake +++ /dev/null @@ -1,84 +0,0 @@ -vcpkg_minimum_required(VERSION 2022-10-12) # for ${VERSION} -vcpkg_from_sourceforge( - OUT_SOURCE_PATH SOURCE_PATH - REPO lzmautils - FILENAME "xz-${VERSION}.tar.xz" - SHA512 f890ee5207799fbc7bb9ae031f444d39d82275b0e1b8cc7f01fdb9270050e38849bd1269db2a2f12fe87b5e23e03f9e809a5c3456d066c0a56e6f98d728553ea - PATCHES - fix_config_include.patch - win_output_name.patch # Fix output name on Windows. Autotool build does not generate lib prefixed libraries on windows. - add_support_ios.patch # add install bundle info for support ios - build-tools.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - tools BUILD_TOOLS -) - -if(VCPKG_TARGET_ARCHITECTURE STREQUAL "wasm32") - set(WASM_OPTIONS -DCMAKE_C_BYTE_ORDER=LITTLE_ENDIAN -DCMAKE_CXX_BYTE_ORDER=LITTLE_ENDIAN) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${FEATURE_OPTIONS} - ${WASM_OPTIONS} - -DBUILD_TESTING=OFF - -DCREATE_XZ_SYMLINKS=OFF - -DCREATE_LZMA_SYMLINKS=OFF - MAYBE_UNUSED_VARIABLES - CREATE_XZ_SYMLINKS - CREATE_LZMA_SYMLINKS -) -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -set(exec_prefix "\${prefix}") -set(libdir "\${prefix}/lib") -set(includedir "\${prefix}/include") -set(PACKAGE_URL https://tukaani.org/xz/) -set(PACKAGE_VERSION 5.2.5) -if(NOT VCPKG_TARGET_IS_WINDOWS) - set(PTHREAD_CFLAGS -pthread) -endif() -set(prefix "${CURRENT_INSTALLED_DIR}") -configure_file("${SOURCE_PATH}/src/liblzma/liblzma.pc.in" "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/liblzma.pc" @ONLY) -if (NOT VCPKG_BUILD_TYPE) - set(prefix "${CURRENT_INSTALLED_DIR}/debug") - configure_file("${SOURCE_PATH}/src/liblzma/liblzma.pc.in" "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/liblzma.pc" @ONLY) -endif() -vcpkg_fixup_pkgconfig() - -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/liblzma) - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/lzma.h" "defined(LZMA_API_STATIC)" "1") -else() - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/lzma.h" "defined(LZMA_API_STATIC)" "0") -endif() - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/share/man" -) - -set(TOOLS xz xzdec) -foreach(_tool IN LISTS TOOLS) - if(NOT EXISTS "${CURRENT_PACKAGES_DIR}/bin/${_tool}${VCPKG_TARGET_EXECUTABLE_SUFFIX}") - list(REMOVE_ITEM TOOLS ${_tool}) - endif() -endforeach() -if(TOOLS) - vcpkg_copy_tools(TOOL_NAMES ${TOOLS} AUTO_CLEAN) -endif() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -file(COPY "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -file(COPY "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/liblzma/usage b/ports/liblzma/usage deleted file mode 100644 index b1461c93fc0a..000000000000 --- a/ports/liblzma/usage +++ /dev/null @@ -1,9 +0,0 @@ -liblzma is compatible with built-in CMake targets: - - find_package(LibLZMA REQUIRED) - target_link_libraries(main PRIVATE LibLZMA::LibLZMA) - -liblzma provides CMake targets: - - find_package(liblzma CONFIG REQUIRED) - target_link_libraries(main PRIVATE liblzma::liblzma) diff --git a/ports/liblzma/vcpkg-cmake-wrapper.cmake b/ports/liblzma/vcpkg-cmake-wrapper.cmake deleted file mode 100644 index 826cdba0650b..000000000000 --- a/ports/liblzma/vcpkg-cmake-wrapper.cmake +++ /dev/null @@ -1,64 +0,0 @@ -cmake_policy(PUSH) -cmake_policy(SET CMP0012 NEW) -cmake_policy(SET CMP0057 NEW) -set(z_vcpkg_liblzma_fixup_needed 0) -if(NOT "CONFIG" IN_LIST ARGS AND NOT "NO_MODULE" IN_LIST ARGS AND NOT CMAKE_DISABLE_FIND_PACKAGE_LibLZMA) - get_filename_component(z_vcpkg_liblzma_prefix "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) - get_filename_component(z_vcpkg_liblzma_prefix "${z_vcpkg_liblzma_prefix}" DIRECTORY) - find_path(LIBLZMA_INCLUDE_DIR NAMES lzma.h PATHS "${z_vcpkg_liblzma_prefix}/include" NO_DEFAULT_PATH) - # liblzma doesn't use a debug postfix, but FindLibLZMA.cmake expects it - find_library(LIBLZMA_LIBRARY_RELEASE NAMES lzma PATHS "${z_vcpkg_liblzma_prefix}/lib" NO_DEFAULT_PATH) - find_library(LIBLZMA_LIBRARY_DEBUG NAMES lzma PATHS "${z_vcpkg_liblzma_prefix}/debug/lib" NO_DEFAULT_PATH) - unset(z_vcpkg_liblzma_prefix) - if(CMAKE_VERSION VERSION_LESS 3.16) - # Older versions of FindLibLZMA.cmake need a single lib in LIBLZMA_LIBRARY. - set(z_vcpkg_liblzma_fixup_needed 1) - set(LIBLZMA_LIBRARY "${LIBLZMA_LIBRARY_RELEASE}" CACHE INTERNAL "") - elseif(NOT TARGET LibLZMA::LibLZMA) - set(z_vcpkg_liblzma_fixup_needed 1) - endif() - # Known values, and required. Skip expensive tests. - set(LIBLZMA_HAS_AUTO_DECODER 1 CACHE INTERNAL "") - set(LIBLZMA_HAS_EASY_ENCODER 1 CACHE INTERNAL "") - set(LIBLZMA_HAS_LZMA_PRESET 1 CACHE INTERNAL "") -endif() - -_find_package(${ARGS}) - -if(z_vcpkg_liblzma_fixup_needed) - include(SelectLibraryConfigurations) - select_library_configurations(LIBLZMA) - if(NOT TARGET LibLZMA::LibLZMA) - # Backfill LibLZMA::LibLZMA to versions of cmake before 3.14 - add_library(LibLZMA::LibLZMA UNKNOWN IMPORTED) - if(DEFINED LIBLZMA_INCLUDE_DIRS) - set_target_properties(LibLZMA::LibLZMA PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${LIBLZMA_INCLUDE_DIRS}") - endif() - set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(LibLZMA::LibLZMA PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C" - IMPORTED_LOCATION_RELEASE "${LIBLZMA_LIBRARY_RELEASE}") - if(EXISTS "${LIBLZMA_LIBRARY}") - set_target_properties(LibLZMA::LibLZMA PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${LIBLZMA_LIBRARY}") - endif() - endif() - if(LIBLZMA_LIBRARY_DEBUG) - # Backfill debug variant to versions of cmake before 3.16 - set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(LibLZMA::LibLZMA PROPERTIES IMPORTED_LOCATION_DEBUG "${LIBLZMA_LIBRARY_DEBUG}") - endif() -endif() -if(LIBLZMA_LIBRARIES AND NOT "Threads::Threads" IN_LIST LIBLZMA_LIBRARIES) - set(THREADS_PREFER_PTHREAD_FLAG TRUE) - find_package(Threads) - list(APPEND LIBLZMA_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) - if(TARGET LibLZMA::LibLZMA) - set_property(TARGET LibLZMA::LibLZMA APPEND PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads) - endif() -endif() -unset(z_vcpkg_liblzma_fixup_needed) -cmake_policy(POP) diff --git a/ports/liblzma/vcpkg.json b/ports/liblzma/vcpkg.json deleted file mode 100644 index d0ac62bed495..000000000000 --- a/ports/liblzma/vcpkg.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "liblzma", - "version-semver": "5.4.1", - "port-version": 1, - "description": "Compression library with an API similar to that of zlib.", - "homepage": "https://tukaani.org/xz/", - "license": null, - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "tools": { - "description": "Build tools", - "supports": "!windows, mingw" - } - } -} diff --git a/ports/liblzma/win_output_name.patch b/ports/liblzma/win_output_name.patch deleted file mode 100644 index 7011475de585..000000000000 --- a/ports/liblzma/win_output_name.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index e80b524..cddbccb 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -462,8 +462,11 @@ set_target_properties(liblzma PROPERTIES - - # It's liblzma.so or liblzma.dll, not libliblzma.so or lzma.dll. - # Avoid the name lzma.dll because it would conflict with LZMA SDK. -- PREFIX "" -+ OUTPUT_NAME lzma - ) -+if(WIN32 AND NOT MINGW) -+ set_target_properties(liblzma PROPERTIES RUNTIME_OUTPUT_NAME liblzma) -+endif() - - # Create liblzma-config-version.cmake. We use this spelling instead of - # liblzmaConfig.cmake to make find_package work in case insensitive manner diff --git a/ports/libxml2/disable-docs.patch b/ports/libxml2/disable-docs.patch deleted file mode 100644 index 848961429d65..000000000000 --- a/ports/libxml2/disable-docs.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f922d5ab..70466bc7 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -599,15 +599,5 @@ if(LIBXML2_WITH_PYTHON) - endif() - --install(FILES doc/xml2-config.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT documentation) --install(FILES doc/xmlcatalog.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT documentation) --install(FILES doc/xmllint.1 DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 COMPONENT documentation) --install(DIRECTORY doc/ DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT documentation -- PATTERN "Makefile.*" EXCLUDE -- PATTERN "*.1" EXCLUDE -- PATTERN "*.py" EXCLUDE -- PATTERN "*.res" EXCLUDE -- PATTERN "*.xml" EXCLUDE -- PATTERN "*.xsl" EXCLUDE) - - configure_package_config_file( - libxml2-config.cmake.cmake.in libxml2-config.cmake diff --git a/ports/libxml2/fix_cmakelist.patch b/ports/libxml2/fix_cmakelist.patch deleted file mode 100644 index f8c60d0f1560..000000000000 --- a/ports/libxml2/fix_cmakelist.patch +++ /dev/null @@ -1,108 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f922d5ab..685964b3 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -420,15 +420,15 @@ endif() - if(LIBXML2_WITH_ICU) - target_link_libraries(LibXml2 PRIVATE ICU::data ICU::i18n ICU::uc) - if(WIN32) -- set(ICU_LIBS "-licudt -licuin -licuuc") -+ set(ICU_LIBS "icu-i18n") - else() -- set(ICU_LIBS "-licudata -licui18n -licuuc") -+ set(ICU_LIBS "icu-i18n") - endif() - endif() - - if(LIBXML2_WITH_LZMA) - target_link_libraries(LibXml2 PRIVATE LibLZMA::LibLZMA) -- set(LZMA_LIBS "-llzma") -+ set(LZMA_LIBS "liblzma") - endif() - - if(LIBXML2_WITH_THREADS) -@@ -438,7 +438,7 @@ endif() - - if(LIBXML2_WITH_ZLIB) - target_link_libraries(LibXml2 PRIVATE ZLIB::ZLIB) -- set(Z_LIBS "-lz") -+ set(Z_LIBS "zlib") - endif() - - set_target_properties( -@@ -490,23 +490,9 @@ set_target_properties( - SOVERSION ${LIBXML_MAJOR_VERSION} - ) - -+set(XML_LIB_NAME xml2) - if(MSVC) -- if(BUILD_SHARED_LIBS) -- set_target_properties( -- LibXml2 -- PROPERTIES -- DEBUG_POSTFIX d -- ) -- else() -- set_target_properties( -- LibXml2 -- PROPERTIES -- DEBUG_POSTFIX sd -- MINSIZEREL_POSTFIX s -- RELEASE_POSTFIX s -- RELWITHDEBINFO_POSTFIX s -- ) -- endif() -+ set(XML_LIB_NAME libxml2) - endif() - - install(FILES ${LIBXML2_HDRS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/libxml2/libxml COMPONENT development) -@@ -654,30 +640,30 @@ install(DIRECTORY doc/ DESTINATION ${CMAKE_INSTALL_DATADIR}/doc/libxml2 COMPONEN - - configure_package_config_file( - libxml2-config.cmake.cmake.in libxml2-config.cmake -- INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2-${PROJECT_VERSION} -+ INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2 - ) - - install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/libxml2-config.cmake -- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2-${PROJECT_VERSION} -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2 - COMPONENT development - ) - - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/libxml2-config-version.cmake - VERSION ${PROJECT_VERSION} -- COMPATIBILITY ExactVersion -+ COMPATIBILITY SameMinorVersion - ) - - install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/libxml2-config-version.cmake -- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2-${PROJECT_VERSION} -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2 - COMPONENT development - ) - - install( - EXPORT LibXml2 -- DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2-${PROJECT_VERSION} -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libxml2 - NAMESPACE LibXml2:: - FILE libxml2-export.cmake - COMPONENT development -diff --git a/libxml-2.0.pc.in b/libxml-2.0.pc.in -index 2653a7c5..2eb2f362 100644 ---- a/libxml-2.0.pc.in -+++ b/libxml-2.0.pc.in -@@ -8,6 +8,7 @@ Name: libXML - Version: @VERSION@ - Description: libXML library version2. - Requires: --Libs: -L${libdir} @XML_LIBS@ --Libs.private: @XML_PRIVATE_LIBS@ @LIBS@ -+Requires.private: @ICU_LIBS@ @Z_LIBS@ @LZMA_LIBS@ -+Libs: -L${libdir} -l@XML_LIB_NAME@ -+Libs.private: @THREAD_LIBS@ @ICONV_LIBS@ @LIBM@ @WIN32_EXTRA_LIBADD@ @LIBS@ - Cflags: @XML_INCLUDEDIR@ @XML_CFLAGS@ diff --git a/ports/libxml2/portfile.cmake b/ports/libxml2/portfile.cmake deleted file mode 100644 index d2c9619a2cdd..000000000000 --- a/ports/libxml2/portfile.cmake +++ /dev/null @@ -1,89 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO GNOME/libxml2 - REF f507d167f1755b7eaea09fb1a44d29aab828b6d1 - SHA512 2ac3dcab31111f608a3fe33dde492c9653ad2bd49a792373acdd03d2787e1a4ef70eeb7a3d47cf67eefd43aee2ab75ec50b36cdcd124445ca206de924abb6021 - HEAD_REF master - PATCHES - disable-docs.patch - fix_cmakelist.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - "ftp" LIBXML2_WITH_FTP - "http" LIBXML2_WITH_HTTP - "legacy" LIBXML2_WITH_LEGACY - "lzma" LIBXML2_WITH_LZMA - "zlib" LIBXML2_WITH_ZLIB - "tools" LIBXML2_WITH_PROGRAMS -) -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${FEATURE_OPTIONS} - -DLIBXML2_WITH_TESTS=OFF - -DLIBXML2_WITH_HTML=ON - -DLIBXML2_WITH_C14N=ON - -DLIBXML2_WITH_CATALOG=ON - -DLIBXML2_WITH_DEBUG=ON - -DLIBXML2_WITH_ISO8859X=ON - -DLIBXML2_WITH_ICU=OFF # Culprit of linkage issues? Solving this is probably another PR - -DLIBXML2_WITH_MEM_DEBUG=OFF - -DLIBXML2_WITH_MODULES=ON - -DLIBXML2_WITH_OUTPUT=ON - -DLIBXML2_WITH_PATTERN=ON - -DLIBXML2_WITH_PUSH=ON - -DLIBXML2_WITH_PYTHON=OFF - -DLIBXML2_WITH_READER=ON - -DLIBXML2_WITH_REGEXPS=ON - -DLIBXML2_WITH_RUN_DEBUG=OFF - -DLIBXML2_WITH_SAX1=ON - -DLIBXML2_WITH_SCHEMAS=ON - -DLIBXML2_WITH_SCHEMATRON=ON - -DLIBXML2_WITH_THREADS=ON - -DLIBXML2_WITH_THREAD_ALLOC=OFF - -DLIBXML2_WITH_TREE=ON - -DLIBXML2_WITH_VALID=ON - -DLIBXML2_WITH_WRITER=ON - -DLIBXML2_WITH_XINCLUDE=ON - -DLIBXML2_WITH_XPATH=ON - -DLIBXML2_WITH_XPTR=ON - -DLIBXML2_WITH_ICONV=OFF # [TileDB]: Can't negate in manifest, clobber here -) - -vcpkg_cmake_install() - -vcpkg_cmake_config_fixup(CONFIG_PATH "lib/cmake/libxml2") -vcpkg_fixup_pkgconfig() - -vcpkg_copy_pdbs() - -if("tools" IN_LIST FEATURES) - vcpkg_copy_tools(TOOL_NAMES xmllint xmlcatalog AUTO_CLEAN) -endif() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL static) - set(_file "${CURRENT_PACKAGES_DIR}/include/libxml2/libxml/xmlexports.h") - file(READ "${_file}" _contents) - string(REPLACE "#ifdef LIBXML_STATIC" "#undef LIBXML_STATIC\n#define LIBXML_STATIC\n#ifdef LIBXML_STATIC" _contents "${_contents}") - file(WRITE "${_file}" "${_contents}") -endif() - -file(COPY "${CURRENT_PACKAGES_DIR}/include/libxml2/" DESTINATION "${CURRENT_PACKAGES_DIR}/include") # TODO: Fix usage in all dependent ports hardcoding the wrong include path. - -# Cleanup -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib/xml2Conf.sh" "${CURRENT_PACKAGES_DIR}/debug/lib/xml2Conf.sh") - -file(COPY - "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" - "${CMAKE_CURRENT_LIST_DIR}/usage" - DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" -) -file(INSTALL "${SOURCE_PATH}/Copyright" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/libxml2/usage b/ports/libxml2/usage deleted file mode 100644 index c7a4878e9b96..000000000000 --- a/ports/libxml2/usage +++ /dev/null @@ -1,4 +0,0 @@ -The package libxml2 is compatible with built-in CMake targets: - - find_package(LibXml2 REQUIRED) - target_link_libraries(main PRIVATE LibXml2::LibXml2) diff --git a/ports/libxml2/vcpkg-cmake-wrapper.cmake b/ports/libxml2/vcpkg-cmake-wrapper.cmake deleted file mode 100644 index 705f22c6a35d..000000000000 --- a/ports/libxml2/vcpkg-cmake-wrapper.cmake +++ /dev/null @@ -1,3 +0,0 @@ -list(REMOVE_ITEM ARGS "NO_MODULE" "CONFIG" "MODULE") -_find_package(${ARGS} CONFIG) -set(LIBXML2_FOUND "${LibXml2_FOUND}") # fphsa compatibility diff --git a/ports/libxml2/vcpkg.json b/ports/libxml2/vcpkg.json deleted file mode 100644 index 47f4866b135e..000000000000 --- a/ports/libxml2/vcpkg.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "name": "libxml2", - "version": "2.10.3", - "port-version": 1, - "description": "Libxml2 is the XML C parser and toolkit developed for the Gnome project (but usable outside of the Gnome platform).", - "homepage": "https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home", - "license": "MIT", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "iconv", - "lzma", - "zlib" - ], - "features": { - "ftp": { - "description": "Add the FTP support", - "supports": "!uwp" - }, - "http": { - "description": "Add the HTTP support", - "supports": "!uwp" - }, - "iconv": { - "description": "Add ICONV support", - "dependencies": [ - "libiconv" - ] - }, - "legacy": { - "description": "Add deprecated APIs for compatibility" - }, - "lzma": { - "description": "Use LZMA", - "dependencies": [ - "liblzma" - ] - }, - "tools": { - "description": "Build tools" - }, - "zlib": { - "description": "Use ZLib", - "dependencies": [ - "zlib" - ] - } - } -} diff --git a/ports/lz4/0001-Fix-potential-memory-corruption-with-negative-memmov.patch b/ports/lz4/0001-Fix-potential-memory-corruption-with-negative-memmov.patch deleted file mode 100755 index d6f444810c0f..000000000000 --- a/ports/lz4/0001-Fix-potential-memory-corruption-with-negative-memmov.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 8301a21773ef61656225e264f4f06ae14462bca7 Mon Sep 17 00:00:00 2001 -From: Jasper Lievisse Adriaanse -Date: Fri, 26 Feb 2021 15:21:20 +0100 -Subject: [PATCH 001/120] Fix potential memory corruption with negative - memmove() size - ---- - lib/lz4.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lib/lz4.c b/lib/lz4.c -index 5f524d0..c2f504e 100644 ---- a/lib/lz4.c -+++ b/lib/lz4.c -@@ -1749,7 +1749,7 @@ LZ4_decompress_generic( - const size_t dictSize /* note : = 0 if noDict */ - ) - { -- if (src == NULL) { return -1; } -+ if ((src == NULL) || (outputSize < 0)) { return -1; } - - { const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; --- -2.36.1 - diff --git a/ports/lz4/CMakeLists.txt b/ports/lz4/CMakeLists.txt deleted file mode 100644 index 142a229006f7..000000000000 --- a/ports/lz4/CMakeLists.txt +++ /dev/null @@ -1,57 +0,0 @@ -cmake_minimum_required(VERSION 3.0) -project(lz4 C) - -if(MSVC AND BUILD_SHARED_LIBS) - add_definitions(-DLZ4_DLL_EXPORT) -endif() -add_definitions(-DXXH_NAMESPACE=LZ4_) - -set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "Install prefix") -set(INSTALL_BIN_DIR "bin" CACHE PATH "Path where exe and dll will be installed") -set(INSTALL_LIB_DIR "lib" CACHE PATH "Path where lib will be installed") -set(INSTALL_INCLUDE_DIR "include" CACHE PATH "Path where headers will be installed") -set(INSTALL_CMAKE_DIR "share/lz4" CACHE PATH "Path where cmake configs will be installed") - -file(GLOB LZ4_HEADERS lib/*.h) - -add_library(lz4 - ${LZ4_HEADERS} - lib/lz4.c - lib/lz4frame.c - lib/lz4hc.c - lib/xxhash.c -) - -target_include_directories(lz4 PUBLIC $ $) -set_target_properties(lz4 PROPERTIES PUBLIC_HEADER ${LZ4_HEADERS}) - -install(TARGETS lz4 - EXPORT lz4Config - RUNTIME DESTINATION "${INSTALL_BIN_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" - PUBLIC_HEADER DESTINATION "${INSTALL_INCLUDE_DIR}" - PRIVATE_HEADER DESTINATION "${INSTALL_INCLUDE_DIR}" - COMPONENT dev -) - -FILE(GLOB lz4h "${CMAKE_CURRENT_LIST_DIR}/lib/*.h") -list(REMOVE_ITEM lz4h "${CMAKE_CURRENT_LIST_DIR}/lib/xxhash.h") -INSTALL(FILES ${lz4h} DESTINATION "${INSTALL_INCLUDE_DIR}") - -install(EXPORT lz4Config - FILE lz4-config.cmake - NAMESPACE lz4:: - DESTINATION "${INSTALL_CMAKE_DIR}" -) - -# Export the package for use from the build-tree (this registers the build-tree with a global CMake-registry) -export(PACKAGE lz4) - -# Create pkgconfig module file -file(STRINGS "lib/lz4.h" LZ4_H REGEX "^#define LZ4_VERSION_(MAJOR|MINOR|RELEASE) ") -string(REGEX REPLACE "^.*MAJOR +([0-9]+).*MINOR +([0-9]+).*RELEASE +([0-9]+).*$" "\\1.\\2.\\3" VERSION "${LZ4_H}") -set(INCLUDEDIR [[${prefix}/include]]) -set(LIBDIR [[${prefix}/lib]]) -configure_file(lib/liblz4.pc.in lib/liblz4.pc) -install(FILES "${CMAKE_CURRENT_BINARY_DIR}/lib/liblz4.pc" DESTINATION "lib/pkgconfig") diff --git a/ports/lz4/portfile.cmake b/ports/lz4/portfile.cmake deleted file mode 100644 index a2e3b74f9118..000000000000 --- a/ports/lz4/portfile.cmake +++ /dev/null @@ -1,42 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO lz4/lz4 - REF v1.9.3 - SHA512 c246b0bda881ee9399fa1be490fa39f43b291bb1d9db72dba8a85db1a50aad416a97e9b300eee3d2a4203c2bd88bda2762e81bc229c3aa409ad217eb306a454c - HEAD_REF dev - PATCHES - 0001-Fix-potential-memory-corruption-with-negative-memmov.patch -) - -file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}") - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS_DEBUG - -DCMAKE_DEBUG_POSTFIX=d -) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") - set(DLL_IMPORT "1 && defined(_MSC_VER)") -else() - set(DLL_IMPORT "0") -endif() -foreach(FILE lz4.h lz4frame.h) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/${FILE}" - "defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)" - "${DLL_IMPORT}" - ) -endforeach() - -vcpkg_cmake_config_fixup() -vcpkg_fixup_pkgconfig() -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/liblz4.pc" " -llz4" " -llz4d") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") - -file(INSTALL "${SOURCE_PATH}/lib/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/lz4/vcpkg.json b/ports/lz4/vcpkg.json deleted file mode 100644 index 597e902dd15c..000000000000 --- a/ports/lz4/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "lz4", - "version": "1.9.3", - "port-version": 4, - "description": "Lossless compression algorithm, providing compression speed at 400 MB/s per core.", - "homepage": "https://github.com/lz4/lz4", - "license": "BSD-2-Clause AND GPL-2.0-only", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/ports/nlohmann-json/portfile.cmake b/ports/nlohmann-json/portfile.cmake deleted file mode 100644 index 947cba357282..000000000000 --- a/ports/nlohmann-json/portfile.cmake +++ /dev/null @@ -1,48 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO nlohmann/json - REF v3.11.2 - SHA512 70097c9bcd7a91254acbd41b8b68a6aaa371fc2dd7011f472917f69f1e2d2986155a0339dad791699d542e4a3be44dc49ae72ff73d0ee0ea4b34183296ce19a0 - HEAD_REF master -) - -if(NOT DEFINED nlohmann-json_IMPLICIT_CONVERSIONS) - set(nlohmann-json_IMPLICIT_CONVERSIONS ON) -endif() - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS -FEATURES - "diagnostics" JSON_Diagnostics -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS ${FEATURE_OPTIONS} - -DJSON_Install=ON - -DJSON_MultipleHeaders=ON - -DJSON_BuildTests=OFF - -DJSON_ImplicitConversions=${nlohmann-json_IMPLICIT_CONVERSIONS} -) -vcpkg_cmake_install() -vcpkg_cmake_config_fixup(PACKAGE_NAME "nlohmann_json" CONFIG_PATH "share/cmake/nlohmann_json") -vcpkg_fixup_pkgconfig() - -vcpkg_replace_string( - "${CURRENT_PACKAGES_DIR}/share/nlohmann_json/nlohmann_jsonTargets.cmake" - "{_IMPORT_PREFIX}/nlohmann_json.natvis" - "{_IMPORT_PREFIX}/share/nlohmann_json/nlohmann_json.natvis" -) -if(EXISTS "${CURRENT_PACKAGES_DIR}/nlohmann_json.natvis") - file(RENAME - "${CURRENT_PACKAGES_DIR}/nlohmann_json.natvis" - "${CURRENT_PACKAGES_DIR}/share/nlohmann_json/nlohmann_json.natvis" - ) -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug") - -# Handle copyright -file(INSTALL "${SOURCE_PATH}/LICENSE.MIT" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) - -# Handle usage -file(COPY "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/ports/nlohmann-json/usage b/ports/nlohmann-json/usage deleted file mode 100644 index 19c76d508d09..000000000000 --- a/ports/nlohmann-json/usage +++ /dev/null @@ -1,12 +0,0 @@ -The package nlohmann-json provides CMake targets: - - find_package(nlohmann_json CONFIG REQUIRED) - target_link_libraries(main PRIVATE nlohmann_json::nlohmann_json) - -The package nlohmann-json can be configured to not provide implicit conversions via a custom triplet file: - - set(nlohmann-json_IMPLICIT_CONVERSIONS OFF) - -For more information, see the docs here: - - https://json.nlohmann.me/api/macros/json_use_implicit_conversions/ diff --git a/ports/nlohmann-json/vcpkg.json b/ports/nlohmann-json/vcpkg.json deleted file mode 100644 index cc65b9d204d6..000000000000 --- a/ports/nlohmann-json/vcpkg.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "nlohmann-json", - "version-semver": "3.11.2", - "description": "JSON for Modern C++", - "homepage": "https://github.com/nlohmann/json", - "license": "MIT", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "diagnostics": { - "description": "Build json_diagnostics" - } - } -} diff --git a/ports/s2n/fix-cmake-target-path.patch b/ports/s2n/fix-cmake-target-path.patch deleted file mode 100644 index 723b0b236fba..000000000000 --- a/ports/s2n/fix-cmake-target-path.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt ---- a/CMakeLists.txt (revision 4aec93c6a74aacf60cec6229b35094f56cb0f87b) -+++ b/CMakeLists.txt (date 1675011451052) -@@ -670,7 +670,7 @@ - endif() - - install(EXPORT "${PROJECT_NAME}-targets" -- DESTINATION "${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/cmake/${TARGET_DIR}" -+ DESTINATION "${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/cmake" - NAMESPACE AWS:: - COMPONENT Development) - -diff --git a/cmake/s2n-config.cmake b/cmake/s2n-config.cmake ---- a/cmake/s2n-config.cmake (revision 4aec93c6a74aacf60cec6229b35094f56cb0f87b) -+++ b/cmake/s2n-config.cmake (date 1675011032538) -@@ -6,21 +6,6 @@ - endif() - - list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") --find_dependency(crypto) -- --# Allow static or shared lib to be used. --# If both are installed, choose based on BUILD_SHARED_LIBS. --if (BUILD_SHARED_LIBS) -- if (EXISTS "${CMAKE_CURRENT_LIST_DIR}/shared") -- include(${CMAKE_CURRENT_LIST_DIR}/shared/@PROJECT_NAME@-targets.cmake) -- else() -- include(${CMAKE_CURRENT_LIST_DIR}/static/@PROJECT_NAME@-targets.cmake) -- endif() --else() -- if (EXISTS "${CMAKE_CURRENT_LIST_DIR}/static") -- include(${CMAKE_CURRENT_LIST_DIR}/static/@PROJECT_NAME@-targets.cmake) -- else() -- include(${CMAKE_CURRENT_LIST_DIR}/shared/@PROJECT_NAME@-targets.cmake) -- endif() --endif() -+find_dependency(OpenSSL COMPONENTS Crypto) - -+include(${CMAKE_CURRENT_LIST_DIR}/@CMAKE_PROJECT_NAME@-targets.cmake) -\ No newline at end of file \ No newline at end of file diff --git a/ports/s2n/portfile.cmake b/ports/s2n/portfile.cmake deleted file mode 100644 index 0909ebee5bc5..000000000000 --- a/ports/s2n/portfile.cmake +++ /dev/null @@ -1,46 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO aws/s2n-tls - REF "v${VERSION}" - SHA512 deead85f2ab22441e1110d442fc93273d96d8dd6a203940cca7ef166fc1c9e7ab75ffe2d550e013e1e1e3266b208904cff94cc2482d6fd00e0546293b0ba11d4 - PATCHES - fix-cmake-target-path.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - tests BUILD_TESTING -) - -set(EXTRA_ARGS) -if(VCPKG_TARGET_ARCHITECTURE STREQUAL "wasm32") - set(EXTRA_ARGS "-DS2N_NO_PQ=TRUE") -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${EXTRA_ARGS} - ${FEATURE_OPTIONS} - -DUNSAFE_TREAT_WARNINGS_AS_ERRORS=OFF -) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/s2n/cmake) - -if(BUILD_TESTING) - message(STATUS "Testing") - vcpkg_cmake_build(TARGET test LOGFILE_BASE test) -endif() - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/lib/s2n" - "${CURRENT_PACKAGES_DIR}/debug/share" - "${CURRENT_PACKAGES_DIR}/lib/s2n" - "${CURRENT_PACKAGES_DIR}/share/s2n/modules" -) - -# Handle copyright -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/ports/s2n/vcpkg.json b/ports/s2n/vcpkg.json deleted file mode 100644 index 76f883756f64..000000000000 --- a/ports/s2n/vcpkg.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "s2n", - "version": "1.3.56", - "description": "C99 implementation of the TLS/SSL protocols.", - "homepage": "https://github.com/aws/s2n-tls", - "license": "Apache-2.0", - "supports": "!uwp & !windows", - "dependencies": [ - "openssl", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "tests": { - "description": "Build and run the tests" - } - } -} diff --git a/ports/spdlog/fmt-header.patch b/ports/spdlog/fmt-header.patch deleted file mode 100644 index 670d5a2f03ed..000000000000 --- a/ports/spdlog/fmt-header.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/bench/bench.cpp b/bench/bench.cpp -index 8a46837a6..ae47f047c 100644 ---- a/bench/bench.cpp -+++ b/bench/bench.cpp -@@ -15,7 +15,7 @@ - #if defined(SPDLOG_USE_STD_FORMAT) - # include - #elif defined(SPDLOG_FMT_EXTERNAL) --# include -+# include - #else - # include "spdlog/fmt/bundled/format.h" - #endif diff --git a/ports/spdlog/portfile.cmake b/ports/spdlog/portfile.cmake deleted file mode 100644 index 21a61d9b9ffe..000000000000 --- a/ports/spdlog/portfile.cmake +++ /dev/null @@ -1,73 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO gabime/spdlog - REF v1.11.0 - SHA512 210f3135c7af3ec774ef9a5c77254ce172a44e2fa720bf590e1c9214782bf5c8140ff683403a85b585868bc308286fbdeb1c988e4ed1eb3c75975254ffe75412 - HEAD_REF v1.x - PATCHES - fmt-header.patch # https://github.com/gabime/spdlog/pull/2545 -) - -vcpkg_check_linkage(ONLY_STATIC_LIBRARY) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - benchmark SPDLOG_BUILD_BENCH - wchar SPDLOG_WCHAR_SUPPORT -) - -# SPDLOG_WCHAR_FILENAMES can only be configured in triplet file since it is an alternative (not additive) -if(NOT DEFINED SPDLOG_WCHAR_FILENAMES) - set(SPDLOG_WCHAR_FILENAMES OFF) -endif() -if(NOT VCPKG_TARGET_IS_WINDOWS) - if("wchar" IN_LIST FEATURES) - message(WARNING "Feature 'wchar' is only supported for Windows and has no effect on other platforms.") - elseif(SPDLOG_WCHAR_FILENAMES) - message(FATAL_ERROR "Build option 'SPDLOG_WCHAR_FILENAMES' is for Windows.") - endif() -endif() - -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" SPDLOG_BUILD_SHARED) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${FEATURE_OPTIONS} - -DSPDLOG_FMT_EXTERNAL=ON - -DSPDLOG_INSTALL=ON - -DSPDLOG_BUILD_SHARED=${SPDLOG_BUILD_SHARED} - -DSPDLOG_WCHAR_FILENAMES=${SPDLOG_WCHAR_FILENAMES} - -DSPDLOG_BUILD_EXAMPLE=OFF -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/spdlog) -vcpkg_fixup_pkgconfig() -vcpkg_copy_pdbs() - -# use vcpkg-provided fmt library (see also option SPDLOG_FMT_EXTERNAL above) -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/include/spdlog/fmt/bundled") - -# add support for integration other than cmake -vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/include/spdlog/tweakme.h - "// #define SPDLOG_FMT_EXTERNAL" - "#ifndef SPDLOG_FMT_EXTERNAL\n#define SPDLOG_FMT_EXTERNAL\n#endif" -) -if(SPDLOG_WCHAR_SUPPORT AND VCPKG_TARGET_IS_WINDOWS) - vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/include/spdlog/tweakme.h - "// #define SPDLOG_WCHAR_TO_UTF8_SUPPORT" - "#ifndef SPDLOG_WCHAR_TO_UTF8_SUPPORT\n#define SPDLOG_WCHAR_TO_UTF8_SUPPORT\n#endif" - ) -endif() -if(SPDLOG_WCHAR_FILENAMES) - vcpkg_replace_string(${CURRENT_PACKAGES_DIR}/include/spdlog/tweakme.h - "// #define SPDLOG_WCHAR_FILENAMES" - "#ifndef SPDLOG_WCHAR_FILENAMES\n#define SPDLOG_WCHAR_FILENAMES\n#endif" - ) -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/share") - -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/spdlog/vcpkg.json b/ports/spdlog/vcpkg.json deleted file mode 100644 index a88014f9ad06..000000000000 --- a/ports/spdlog/vcpkg.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "spdlog", - "version-semver": "1.11.0", - "description": "Very fast, header only, C++ logging library", - "homepage": "https://github.com/gabime/spdlog", - "license": "MIT", - "dependencies": [ - "fmt", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "benchmark": { - "description": "Use google benchmark", - "dependencies": [ - "benchmark" - ] - }, - "wchar": { - "description": "Build with wchar_t (Windows only)", - "supports": "windows" - } - } -} diff --git a/ports/vcpkg-cmake-config/copyright b/ports/vcpkg-cmake-config/copyright deleted file mode 100644 index 2e4eac8264fa..000000000000 --- a/ports/vcpkg-cmake-config/copyright +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) Microsoft Corporation - -All rights reserved. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/ports/vcpkg-cmake-config/portfile.cmake b/ports/vcpkg-cmake-config/portfile.cmake deleted file mode 100644 index fc3dbafd5184..000000000000 --- a/ports/vcpkg-cmake-config/portfile.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if(NOT TARGET_TRIPLET STREQUAL _HOST_TRIPLET) - # make FATAL_ERROR in CI when issue #16773 fixed - message(WARNING "vcpkg-cmake-config is a host-only port; please mark it as a host port in your dependencies.") -endif() - -file(INSTALL - "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake" - "${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" - "${CMAKE_CURRENT_LIST_DIR}/copyright" - DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - -set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/ports/vcpkg-cmake-config/vcpkg-port-config.cmake b/ports/vcpkg-cmake-config/vcpkg-port-config.cmake deleted file mode 100644 index 980d411315c7..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg-port-config.cmake +++ /dev/null @@ -1 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake") diff --git a/ports/vcpkg-cmake-config/vcpkg.json b/ports/vcpkg-cmake-config/vcpkg.json deleted file mode 100644 index db965341a7e0..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "vcpkg-cmake-config", - "version-date": "2022-02-06", - "port-version": 1, - "documentation": "https://vcpkg.io/en/docs/README.html", - "license": "MIT" -} diff --git a/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake b/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake deleted file mode 100644 index 368e5809ac21..000000000000 --- a/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake +++ /dev/null @@ -1,258 +0,0 @@ -include_guard(GLOBAL) - -function(vcpkg_cmake_config_fixup) - cmake_parse_arguments(PARSE_ARGV 0 "arg" "DO_NOT_DELETE_PARENT_CONFIG_PATH;NO_PREFIX_CORRECTION" "PACKAGE_NAME;CONFIG_PATH;TOOLS_PATH" "") - - if(DEFINED arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "vcpkg_cmake_config_fixup was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - if(NOT arg_PACKAGE_NAME) - set(arg_PACKAGE_NAME "${PORT}") - endif() - if(NOT arg_CONFIG_PATH) - set(arg_CONFIG_PATH "share/${arg_PACKAGE_NAME}") - endif() - if(NOT arg_TOOLS_PATH) - set(arg_TOOLS_PATH "tools/${PORT}") - endif() - set(target_path "share/${arg_PACKAGE_NAME}") - - string(REPLACE "." "\\." EXECUTABLE_SUFFIX "${VCPKG_TARGET_EXECUTABLE_SUFFIX}") - - set(debug_share "${CURRENT_PACKAGES_DIR}/debug/${target_path}") - set(release_share "${CURRENT_PACKAGES_DIR}/${target_path}") - - if(NOT arg_CONFIG_PATH STREQUAL "share/${arg_PACKAGE_NAME}") - if(arg_CONFIG_PATH STREQUAL "share") - set(arg_CONFIG_PATH z_vcpkg_share) - file(RENAME "${CURRENT_PACKAGES_DIR}/debug/share" "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") - file(RENAME "${CURRENT_PACKAGES_DIR}/share" "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") - endif() - - set(debug_config "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") - set(release_config "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(NOT EXISTS "${debug_config}") - message(FATAL_ERROR "'${debug_config}' does not exist.") - endif() - - # This roundabout handling enables CONFIG_PATH = share - file(MAKE_DIRECTORY "${debug_share}") - file(GLOB files "${debug_config}/*") - file(COPY ${files} DESTINATION "${debug_share}") - file(REMOVE_RECURSE "${debug_config}") - endif() - - file(GLOB files "${release_config}/*") - file(COPY ${files} DESTINATION "${release_share}") - file(REMOVE_RECURSE "${release_config}") - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - get_filename_component(debug_config_dir_name "${debug_config}" NAME) - string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) - if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${debug_config}") - else() - get_filename_component(debug_config_parent_dir "${debug_config}" DIRECTORY) - get_filename_component(debug_config_dir_name "${debug_config_parent_dir}" NAME) - string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) - if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${debug_config_parent_dir}") - endif() - endif() - endif() - - get_filename_component(release_config_dir_name "${release_config}" NAME) - string(TOLOWER "${release_config_dir_name}" release_config_dir_name) - if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${release_config}") - else() - get_filename_component(release_config_parent_dir "${release_config}" DIRECTORY) - get_filename_component(release_config_dir_name "${release_config_parent_dir}" NAME) - string(TOLOWER "${release_config_dir_name}" release_config_dir_name) - if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) - file(REMOVE_RECURSE "${release_config_parent_dir}") - endif() - endif() - endif() - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(NOT EXISTS "${debug_share}") - message(FATAL_ERROR "'${debug_share}' does not exist.") - endif() - endif() - - file(GLOB_RECURSE release_targets - "${release_share}/*-release.cmake" - ) - foreach(release_target IN LISTS release_targets) - file(READ "${release_target}" contents) - string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") - string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \"]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") - file(WRITE "${release_target}" "${contents}") - endforeach() - - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(GLOB_RECURSE debug_targets - "${debug_share}/*-debug.cmake" - ) - foreach(debug_target IN LISTS debug_targets) - file(RELATIVE_PATH debug_target_rel "${debug_share}" "${debug_target}") - - file(READ "${debug_target}" contents) - string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") - string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \";]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") - string(REPLACE "\${_IMPORT_PREFIX}/lib" "\${_IMPORT_PREFIX}/debug/lib" contents "${contents}") - string(REPLACE "\${_IMPORT_PREFIX}/bin" "\${_IMPORT_PREFIX}/debug/bin" contents "${contents}") - file(WRITE "${release_share}/${debug_target_rel}" "${contents}") - - file(REMOVE "${debug_target}") - endforeach() - endif() - - #Fix ${_IMPORT_PREFIX} and absolute paths in cmake generated targets and configs; - #Since those can be renamed we have to check in every *.cmake, but only once. - file(GLOB_RECURSE main_cmakes "${release_share}/*.cmake") - if(NOT DEFINED Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - vcpkg_list(SET Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - endif() - foreach(already_fixed_up IN LISTS Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) - vcpkg_list(REMOVE_ITEM main_cmakes "${already_fixed_up}") - endforeach() - vcpkg_list(APPEND Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP ${main_cmakes}) - set(Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP "${Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP}" CACHE INTERNAL "") - - foreach(main_cmake IN LISTS main_cmakes) - file(READ "${main_cmake}" contents) - # Note: I think the following comment is no longer true, since we now require the path to be `share/blah` - # however, I don't know it for sure. - # - nimazzuc - - #This correction is not correct for all cases. To make it correct for all cases it needs to consider - #original folder deepness to CURRENT_PACKAGES_DIR in comparison to the moved to folder deepness which - #is always at least (>=) 2, e.g. share/${PORT}. Currently the code assumes it is always 2 although - #this requirement is only true for the *Config.cmake. The targets are not required to be in the same - #folder as the *Config.cmake! - if(NOT arg_NO_PREFIX_CORRECTION) - string(REGEX REPLACE -[[get_filename_component\(_IMPORT_PREFIX "\${CMAKE_CURRENT_LIST_FILE}" PATH\)( -get_filename_component\(_IMPORT_PREFIX "\${_IMPORT_PREFIX}" PATH\))*]] -[[get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)]] - contents "${contents}") # see #1044 for details why this replacement is necessary. See #4782 why it must be a regex. - string(REGEX REPLACE -[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\./(\.\./)*" ABSOLUTE\)]] -[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] - contents "${contents}") - string(REGEX REPLACE -[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\.((\\|/)\.\.)*" ABSOLUTE\)]] -[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] - contents "${contents}") # This is a meson-related workaround, see https://github.com/mesonbuild/meson/issues/6955 - endif() - - # Merge release and debug configurations of target property INTERFACE_LINK_LIBRARIES. - string(REPLACE "${release_share}/" "${debug_share}/" debug_cmake "${main_cmake}") - if(DEFINED VCPKG_BUILD_TYPE) - # Skip. Warning: A release-only port in a dual-config installation - # may pull release dependencies into the debug configuration. - elseif(NOT contents MATCHES "INTERFACE_LINK_LIBRARIES") - # Skip. No relevant properties. - elseif(NOT contents MATCHES "# Generated CMake target import file\\.") - # Skip. No safe assumptions about a matching debug import file. - elseif(NOT EXISTS "${debug_cmake}") - message(SEND_ERROR "Did not find a debug import file matching '${main_cmake}'") - else() - file(READ "${debug_cmake}" debug_contents) - while(contents MATCHES "set_target_properties\\(([^ \$]*) PROPERTIES[^)]*\\)") - set(matched_command "${CMAKE_MATCH_0}") - string(REPLACE "+" "\\+" target "${CMAKE_MATCH_1}") - if(NOT debug_contents MATCHES "set_target_properties\\(${target} PROPERTIES[^)]*\\)") - message(SEND_ERROR "Did not find a debug configuration for target '${target}'.") - endif() - set(debug_command "${CMAKE_MATCH_0}") - string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" release_line "${matched_command}") - set(release_libs "${CMAKE_MATCH_1}") - string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" debug_line "${debug_command}") - set(debug_libs "${CMAKE_MATCH_1}") - z_vcpkg_cmake_config_fixup_merge(merged_libs release_libs debug_libs) - string(REPLACE "${release_line}" " INTERFACE_LINK_LIBRARIES \"${merged_libs}\"" updated_command "${matched_command}") - string(REPLACE "set_target_properties" "set_target_properties::done" updated_command "${updated_command}") # Prevend 2nd match - string(REPLACE "${matched_command}" "${updated_command}" contents "${contents}") - endwhile() - string(REPLACE "set_target_properties::done" "set_target_properties" contents "${contents}") # Restore original command - endif() - - #Fix absolute paths to installed dir with ones relative to ${CMAKE_CURRENT_LIST_DIR} - #This happens if vcpkg built libraries are directly linked to a target instead of using - #an imported target. - string(REPLACE "${CURRENT_INSTALLED_DIR}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") - file(TO_CMAKE_PATH "${CURRENT_PACKAGES_DIR}" cmake_current_packages_dir) - string(REPLACE "${cmake_current_packages_dir}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") - # If ${VCPKG_IMPORT_PREFIX} was actually used, inject a definition of it: - string(FIND "${contents}" [[${VCPKG_IMPORT_PREFIX}]] index) - if (NOT index STREQUAL "-1") - get_filename_component(main_cmake_dir "${main_cmake}" DIRECTORY) - # Calculate relative to be a sequence of "../" - file(RELATIVE_PATH relative "${main_cmake_dir}" "${cmake_current_packages_dir}") - string(PREPEND contents "get_filename_component(VCPKG_IMPORT_PREFIX \"\${CMAKE_CURRENT_LIST_DIR}\/${relative}\" ABSOLUTE)\n") - endif() - - file(WRITE "${main_cmake}" "${contents}") - endforeach() - - file(GLOB_RECURSE unused_files - "${debug_share}/*[Tt]argets.cmake" - "${debug_share}/*[Cc]onfig.cmake" - "${debug_share}/*[Cc]onfigVersion.cmake" - "${debug_share}/*[Cc]onfig-version.cmake" - ) - foreach(unused_file IN LISTS unused_files) - file(REMOVE "${unused_file}") - endforeach() - - # Remove /debug// if it's empty. - file(GLOB_RECURSE remaining_files "${debug_share}/*") - if(remaining_files STREQUAL "") - file(REMOVE_RECURSE "${debug_share}") - endif() - - # Remove /debug/share/ if it's empty. - file(GLOB_RECURSE remaining_files "${CURRENT_PACKAGES_DIR}/debug/share/*") - if(remaining_files STREQUAL "") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - endif() -endfunction() - -# Merges link interface library lists for release and debug -# into a single expression which use generator expression as necessary. -function(z_vcpkg_cmake_config_fixup_merge out_var release_var debug_var) - set(release_libs "VCPKG;${${release_var}}") - string(REGEX REPLACE ";optimized;([^;]*)" ";\\1" release_libs "${release_libs}") - string(REGEX REPLACE ";debug;([^;]*)" ";" release_libs "${release_libs}") - list(REMOVE_AT release_libs 0) - list(FILTER release_libs EXCLUDE REGEX [[^\\[$]<\\[$]:]]) - list(TRANSFORM release_libs REPLACE [[^\\[$]<\\[$]>:(.*)>$]] "\\1") - - set(debug_libs "VCPKG;${${debug_var}}") - string(REGEX REPLACE ";optimized;([^;]*)" ";" debug_libs "${debug_libs}") - string(REGEX REPLACE ";debug;([^;]*)" ";\\1" debug_libs "${debug_libs}") - list(REMOVE_AT debug_libs 0) - list(FILTER debug_libs EXCLUDE REGEX [[^\\[$]<\\[$]>:]]) - list(TRANSFORM debug_libs REPLACE [[^\\[$]<\\[$]:(.*)>$]] "\\1") - - set(merged_libs "") - foreach(release_lib debug_lib IN ZIP_LISTS release_libs debug_libs) - if(release_lib STREQUAL debug_lib) - list(APPEND merged_libs "${release_lib}") - else() - if(release_lib) - list(APPEND merged_libs "\\\$<\\\$>:${release_lib}>") - endif() - if(debug_lib) - list(APPEND merged_libs "\\\$<\\\$:${debug_lib}>") - endif() - endif() - endforeach() - set("${out_var}" "${merged_libs}" PARENT_SCOPE) -endfunction() diff --git a/ports/vcpkg-cmake/portfile.cmake b/ports/vcpkg-cmake/portfile.cmake deleted file mode 100644 index 0b7dd502017e..000000000000 --- a/ports/vcpkg-cmake/portfile.cmake +++ /dev/null @@ -1,14 +0,0 @@ -if(VCPKG_CROSSCOMPILING) - # make FATAL_ERROR in CI when issue #16773 fixed - message(WARNING "vcpkg-cmake is a host-only port; please mark it as a host port in your dependencies.") -endif() - -file(INSTALL - "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_configure.cmake" - "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_build.cmake" - "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_install.cmake" - "${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" - DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - -file(INSTALL "${VCPKG_ROOT_DIR}/LICENSE.txt" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) -set(VCPKG_POLICY_CMAKE_HELPER_PORT enabled) diff --git a/ports/vcpkg-cmake/vcpkg-port-config.cmake b/ports/vcpkg-cmake/vcpkg-port-config.cmake deleted file mode 100644 index f2a973d4ebcb..000000000000 --- a/ports/vcpkg-cmake/vcpkg-port-config.cmake +++ /dev/null @@ -1,3 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_configure.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_build.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_install.cmake") diff --git a/ports/vcpkg-cmake/vcpkg.json b/ports/vcpkg-cmake/vcpkg.json deleted file mode 100644 index 943638234744..000000000000 --- a/ports/vcpkg-cmake/vcpkg.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "vcpkg-cmake", - "version-date": "2022-10-30", - "documentation": "https://vcpkg.io/en/docs/maintainers/ports/vcpkg-cmake.html", - "license": "MIT" -} diff --git a/ports/vcpkg-cmake/vcpkg_cmake_build.cmake b/ports/vcpkg-cmake/vcpkg_cmake_build.cmake deleted file mode 100644 index 47933b3fe9f8..000000000000 --- a/ports/vcpkg-cmake/vcpkg_cmake_build.cmake +++ /dev/null @@ -1,91 +0,0 @@ -include_guard(GLOBAL) - -function(vcpkg_cmake_build) - cmake_parse_arguments(PARSE_ARGV 0 "arg" "DISABLE_PARALLEL;ADD_BIN_TO_PATH" "TARGET;LOGFILE_BASE" "") - - if(DEFINED arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "vcpkg_cmake_build was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - if(NOT DEFINED arg_LOGFILE_BASE) - set(arg_LOGFILE_BASE "build") - endif() - vcpkg_list(SET build_param) - vcpkg_list(SET parallel_param) - vcpkg_list(SET no_parallel_param) - - if("${Z_VCPKG_CMAKE_GENERATOR}" STREQUAL "Ninja") - vcpkg_list(SET build_param "-v") # verbose output - vcpkg_list(SET parallel_param "-j${VCPKG_CONCURRENCY}") - vcpkg_list(SET no_parallel_param "-j1") - elseif("${Z_VCPKG_CMAKE_GENERATOR}" MATCHES "^Visual Studio") - vcpkg_list(SET build_param - "/p:VCPkgLocalAppDataDisabled=true" - "/p:UseIntelMKL=No" - ) - vcpkg_list(SET parallel_param "/m") - elseif("${Z_VCPKG_CMAKE_GENERATOR}" STREQUAL "NMake Makefiles") - # No options are currently added for nmake builds - elseif(Z_VCPKG_CMAKE_GENERATOR STREQUAL "Unix Makefiles") - vcpkg_list(SET build_param "VERBOSE=1") - vcpkg_list(SET parallel_param "-j${VCPKG_CONCURRENCY}") - vcpkg_list(SET no_parallel_param "") - elseif(Z_VCPKG_CMAKE_GENERATOR STREQUAL "Xcode") - vcpkg_list(SET parallel_param -jobs "${VCPKG_CONCURRENCY}") - vcpkg_list(SET no_parallel_param -jobs 1) - else() - message(WARNING "Unrecognized GENERATOR setting from vcpkg_cmake_configure().") - endif() - - vcpkg_list(SET target_param) - if(arg_TARGET) - vcpkg_list(SET target_param "--target" "${arg_TARGET}") - endif() - - foreach(build_type IN ITEMS debug release) - if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "${build_type}") - if("${build_type}" STREQUAL "debug") - set(short_build_type "dbg") - set(config "Debug") - else() - set(short_build_type "rel") - set(config "Release") - endif() - - message(STATUS "Building ${TARGET_TRIPLET}-${short_build_type}") - - if(arg_ADD_BIN_TO_PATH) - vcpkg_backup_env_variables(VARS PATH) - if("${build_type}" STREQUAL "debug") - vcpkg_add_to_path(PREPEND "${CURRENT_INSTALLED_DIR}/debug/bin") - else() - vcpkg_add_to_path(PREPEND "${CURRENT_INSTALLED_DIR}/bin") - endif() - endif() - - if(arg_DISABLE_PARALLEL) - vcpkg_execute_build_process( - COMMAND - "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} - -- ${build_param} ${no_parallel_param} - WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${short_build_type}" - LOGNAME "${arg_LOGFILE_BASE}-${TARGET_TRIPLET}-${short_build_type}" - ) - else() - vcpkg_execute_build_process( - COMMAND - "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} - -- ${build_param} ${parallel_param} - NO_PARALLEL_COMMAND - "${CMAKE_COMMAND}" --build . --config "${config}" ${target_param} - -- ${build_param} ${no_parallel_param} - WORKING_DIRECTORY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${short_build_type}" - LOGNAME "${arg_LOGFILE_BASE}-${TARGET_TRIPLET}-${short_build_type}" - ) - endif() - - if(arg_ADD_BIN_TO_PATH) - vcpkg_restore_env_variables(VARS PATH) - endif() - endif() - endforeach() -endfunction() diff --git a/ports/vcpkg-cmake/vcpkg_cmake_configure.cmake b/ports/vcpkg-cmake/vcpkg_cmake_configure.cmake deleted file mode 100644 index 832bbf700790..000000000000 --- a/ports/vcpkg-cmake/vcpkg_cmake_configure.cmake +++ /dev/null @@ -1,320 +0,0 @@ -include_guard(GLOBAL) - -macro(z_vcpkg_cmake_configure_both_set_or_unset var1 var2) - if(DEFINED ${var1} AND NOT DEFINED ${var2}) - message(FATAL_ERROR "If ${var1} is set, then ${var2} must be set.") - elseif(NOT DEFINED ${var1} AND DEFINED ${var2}) - message(FATAL_ERROR "If ${var2} is set, then ${var1} must be set.") - endif() -endmacro() - -function(vcpkg_cmake_configure) - cmake_parse_arguments(PARSE_ARGV 0 "arg" - "PREFER_NINJA;DISABLE_PARALLEL_CONFIGURE;WINDOWS_USE_MSBUILD;NO_CHARSET_FLAG;Z_CMAKE_GET_VARS_USAGE" - "SOURCE_PATH;GENERATOR;LOGFILE_BASE" - "OPTIONS;OPTIONS_DEBUG;OPTIONS_RELEASE;MAYBE_UNUSED_VARIABLES" - ) - - if(NOT arg_Z_CMAKE_GET_VARS_USAGE AND DEFINED CACHE{Z_VCPKG_CMAKE_GENERATOR}) - message(WARNING "${CMAKE_CURRENT_FUNCTION} already called; this function should only be called once.") - endif() - if(arg_PREFER_NINJA) - message(WARNING "PREFER_NINJA has been deprecated in ${CMAKE_CURRENT_FUNCTION}. Please remove it from the portfile!") - endif() - - if(DEFINED arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "${CMAKE_CURRENT_FUNCTION} was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - - if(NOT DEFINED arg_SOURCE_PATH) - message(FATAL_ERROR "SOURCE_PATH must be set") - endif() - if(NOT DEFINED arg_LOGFILE_BASE) - set(arg_LOGFILE_BASE "config-${TARGET_TRIPLET}") - endif() - - set(manually_specified_variables "") - - if(arg_Z_CMAKE_GET_VARS_USAGE) - set(configuring_message "Getting CMake variables for ${TARGET_TRIPLET}") - else() - set(configuring_message "Configuring ${TARGET_TRIPLET}") - - foreach(option IN LISTS arg_OPTIONS arg_OPTIONS_RELEASE arg_OPTIONS_DEBUG) - if("${option}" MATCHES "^-D([^:=]*)[:=]") - vcpkg_list(APPEND manually_specified_variables "${CMAKE_MATCH_1}") - endif() - endforeach() - vcpkg_list(REMOVE_DUPLICATES manually_specified_variables) - foreach(maybe_unused_var IN LISTS arg_MAYBE_UNUSED_VARIABLES) - vcpkg_list(REMOVE_ITEM manually_specified_variables "${maybe_unused_var}") - endforeach() - debug_message("manually specified variables: ${manually_specified_variables}") - endif() - - if(CMAKE_HOST_WIN32) - if(DEFINED ENV{PROCESSOR_ARCHITEW6432}) - set(host_architecture "$ENV{PROCESSOR_ARCHITEW6432}") - else() - set(host_architecture "$ENV{PROCESSOR_ARCHITECTURE}") - endif() - endif() - - set(ninja_host ON) # Ninja availability - if(host_architecture STREQUAL "x86" OR DEFINED ENV{VCPKG_FORCE_SYSTEM_BINARIES}) - # Prebuilt ninja binaries are only provided for x64 hosts - find_program(NINJA NAMES ninja ninja-build) - if(NOT NINJA) - set(ninja_host OFF) - set(arg_DISABLE_PARALLEL_CONFIGURE ON) - set(arg_WINDOWS_USE_MSBUILD ON) - endif() - endif() - - set(generator "") - set(architecture_options "") - if(arg_WINDOWS_USE_MSBUILD AND VCPKG_HOST_IS_WINDOWS AND VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - z_vcpkg_get_visual_studio_generator(OUT_GENERATOR generator OUT_ARCH arch) - vcpkg_list(APPEND architecture_options "-A${arch}") - if(DEFINED VCPKG_PLATFORM_TOOLSET) - vcpkg_list(APPEND arg_OPTIONS "-T${VCPKG_PLATFORM_TOOLSET}") - endif() - if(NOT generator) - message(FATAL_ERROR "Unable to determine appropriate Visual Studio generator for triplet ${TARGET_TRIPLET}: - ENV{VisualStudioVersion} : $ENV{VisualStudioVersion} - VCPKG_TARGET_ARCHITECTURE: ${VCPKG_TARGET_ARCHITECTURE}") - endif() - elseif(DEFINED arg_GENERATOR) - set(generator "${arg_GENERATOR}") - elseif(ninja_host) - set(generator "Ninja") - elseif(NOT VCPKG_HOST_IS_WINDOWS) - set(generator "Unix Makefiles") - endif() - - if(NOT generator) - if(NOT VCPKG_CMAKE_SYSTEM_NAME) - set(VCPKG_CMAKE_SYSTEM_NAME "Windows") - endif() - message(FATAL_ERROR "Unable to determine appropriate generator for: " - "${VCPKG_CMAKE_SYSTEM_NAME}-${VCPKG_TARGET_ARCHITECTURE}-${VCPKG_PLATFORM_TOOLSET}") - endif() - - if(generator STREQUAL "Ninja") - vcpkg_find_acquire_program(NINJA) - vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_MAKE_PROGRAM=${NINJA}") - # If we use Ninja, it must be on PATH for CMake's ExternalProject, - # cf. https://gitlab.kitware.com/cmake/cmake/-/issues/23355. - get_filename_component(ninja_path "${NINJA}" DIRECTORY) - vcpkg_add_to_path("${ninja_path}") - endif() - - set(build_dir_release "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel") - set(build_dir_debug "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg") - file(REMOVE_RECURSE - "${build_dir_release}" - "${build_dir_debug}") - file(MAKE_DIRECTORY "${build_dir_release}") - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(MAKE_DIRECTORY "${build_dir_debug}") - endif() - - if(DEFINED VCPKG_CMAKE_SYSTEM_NAME) - vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_SYSTEM_NAME=${VCPKG_CMAKE_SYSTEM_NAME}") - if(VCPKG_TARGET_IS_UWP AND NOT DEFINED VCPKG_CMAKE_SYSTEM_VERSION) - set(VCPKG_CMAKE_SYSTEM_VERSION 10.0) - elseif(VCPKG_TARGET_IS_ANDROID AND NOT DEFINED VCPKG_CMAKE_SYSTEM_VERSION) - set(VCPKG_CMAKE_SYSTEM_VERSION 21) - endif() - endif() - - if(DEFINED VCPKG_CMAKE_SYSTEM_VERSION) - vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_SYSTEM_VERSION=${VCPKG_CMAKE_SYSTEM_VERSION}") - endif() - - if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") - vcpkg_list(APPEND arg_OPTIONS "-DBUILD_SHARED_LIBS=ON") - elseif(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_list(APPEND arg_OPTIONS "-DBUILD_SHARED_LIBS=OFF") - else() - message(FATAL_ERROR - "Invalid setting for VCPKG_LIBRARY_LINKAGE: \"${VCPKG_LIBRARY_LINKAGE}\". " - "It must be \"static\" or \"dynamic\"") - endif() - - z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS_DEBUG VCPKG_C_FLAGS_DEBUG) - z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS_RELEASE VCPKG_C_FLAGS_RELEASE) - z_vcpkg_cmake_configure_both_set_or_unset(VCPKG_CXX_FLAGS VCPKG_C_FLAGS) - - set(VCPKG_SET_CHARSET_FLAG ON) - if(arg_NO_CHARSET_FLAG) - set(VCPKG_SET_CHARSET_FLAG OFF) - endif() - - if(NOT DEFINED VCPKG_CHAINLOAD_TOOLCHAIN_FILE) - z_vcpkg_select_default_vcpkg_chainload_toolchain() - endif() - - list(JOIN VCPKG_TARGET_ARCHITECTURE "\;" target_architecture_string) - vcpkg_list(APPEND arg_OPTIONS - "-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=${VCPKG_CHAINLOAD_TOOLCHAIN_FILE}" - "-DVCPKG_TARGET_TRIPLET=${TARGET_TRIPLET}" - "-DVCPKG_SET_CHARSET_FLAG=${VCPKG_SET_CHARSET_FLAG}" - "-DVCPKG_PLATFORM_TOOLSET=${VCPKG_PLATFORM_TOOLSET}" - "-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON" - "-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON" - "-DCMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=ON" - "-DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=TRUE" - "-DCMAKE_VERBOSE_MAKEFILE=ON" - "-DVCPKG_APPLOCAL_DEPS=OFF" - "-DCMAKE_TOOLCHAIN_FILE=${SCRIPTS}/buildsystems/vcpkg.cmake" - "-DCMAKE_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION=ON" - "-DVCPKG_CXX_FLAGS=${VCPKG_CXX_FLAGS}" - "-DVCPKG_CXX_FLAGS_RELEASE=${VCPKG_CXX_FLAGS_RELEASE}" - "-DVCPKG_CXX_FLAGS_DEBUG=${VCPKG_CXX_FLAGS_DEBUG}" - "-DVCPKG_C_FLAGS=${VCPKG_C_FLAGS}" - "-DVCPKG_C_FLAGS_RELEASE=${VCPKG_C_FLAGS_RELEASE}" - "-DVCPKG_C_FLAGS_DEBUG=${VCPKG_C_FLAGS_DEBUG}" - "-DVCPKG_CRT_LINKAGE=${VCPKG_CRT_LINKAGE}" - "-DVCPKG_LINKER_FLAGS=${VCPKG_LINKER_FLAGS}" - "-DVCPKG_LINKER_FLAGS_RELEASE=${VCPKG_LINKER_FLAGS_RELEASE}" - "-DVCPKG_LINKER_FLAGS_DEBUG=${VCPKG_LINKER_FLAGS_DEBUG}" - "-DVCPKG_TARGET_ARCHITECTURE=${target_architecture_string}" - "-DCMAKE_INSTALL_LIBDIR:STRING=lib" - "-DCMAKE_INSTALL_BINDIR:STRING=bin" - "-D_VCPKG_ROOT_DIR=${VCPKG_ROOT_DIR}" - "-D_VCPKG_INSTALLED_DIR=${_VCPKG_INSTALLED_DIR}" - "-DVCPKG_MANIFEST_INSTALL=OFF" - "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" - ) - - # Sets configuration variables for macOS builds - foreach(config_var IN ITEMS INSTALL_NAME_DIR OSX_DEPLOYMENT_TARGET OSX_SYSROOT OSX_ARCHITECTURES) - if(DEFINED VCPKG_${config_var}) - vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_${config_var}=${VCPKG_${config_var}}") - endif() - endforeach() - - # Allow overrides / additional configuration variables from triplets - if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS) - vcpkg_list(APPEND arg_OPTIONS ${VCPKG_CMAKE_CONFIGURE_OPTIONS}) - endif() - if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS_RELEASE) - vcpkg_list(APPEND arg_OPTIONS_RELEASE ${VCPKG_CMAKE_CONFIGURE_OPTIONS_RELEASE}) - endif() - if(DEFINED VCPKG_CMAKE_CONFIGURE_OPTIONS_DEBUG) - vcpkg_list(APPEND arg_OPTIONS_DEBUG ${VCPKG_CMAKE_CONFIGURE_OPTIONS_DEBUG}) - endif() - - vcpkg_list(SET rel_command - "${CMAKE_COMMAND}" "${arg_SOURCE_PATH}" - -G "${generator}" - ${architecture_options} - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_PREFIX=${CURRENT_PACKAGES_DIR}" - ${arg_OPTIONS} ${arg_OPTIONS_RELEASE}) - vcpkg_list(SET dbg_command - "${CMAKE_COMMAND}" "${arg_SOURCE_PATH}" - -G "${generator}" - ${architecture_options} - "-DCMAKE_BUILD_TYPE=Debug" - "-DCMAKE_INSTALL_PREFIX=${CURRENT_PACKAGES_DIR}/debug" - ${arg_OPTIONS} ${arg_OPTIONS_DEBUG}) - - if(NOT arg_DISABLE_PARALLEL_CONFIGURE) - vcpkg_list(APPEND arg_OPTIONS "-DCMAKE_DISABLE_SOURCE_CHANGES=ON") - - vcpkg_find_acquire_program(NINJA) - - #parallelize the configure step - set(ninja_configure_contents - "rule CreateProcess\n command = \$process\n\n" - ) - - if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "release") - z_vcpkg_configure_cmake_build_cmakecache(ninja_configure_contents ".." "rel") - endif() - if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "debug") - z_vcpkg_configure_cmake_build_cmakecache(ninja_configure_contents "../../${TARGET_TRIPLET}-dbg" "dbg") - endif() - - file(MAKE_DIRECTORY "${build_dir_release}/vcpkg-parallel-configure") - file(WRITE - "${build_dir_release}/vcpkg-parallel-configure/build.ninja" - "${ninja_configure_contents}") - - message(STATUS "${configuring_message}") - vcpkg_execute_required_process( - COMMAND "${NINJA}" -v - WORKING_DIRECTORY "${build_dir_release}/vcpkg-parallel-configure" - LOGNAME "${arg_LOGFILE_BASE}" - SAVE_LOG_FILES ../../${TARGET_TRIPLET}-dbg/CMakeCache.txt ../CMakeCache.txt - ) - - vcpkg_list(APPEND config_logs - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-out.log" - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-err.log") - else() - if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "debug") - message(STATUS "${configuring_message}-dbg") - vcpkg_execute_required_process( - COMMAND ${dbg_command} - WORKING_DIRECTORY "${build_dir_debug}" - LOGNAME "${arg_LOGFILE_BASE}-dbg" - SAVE_LOG_FILES CMakeCache.txt - ) - vcpkg_list(APPEND config_logs - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-dbg-out.log" - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-dbg-err.log") - endif() - - if(NOT DEFINED VCPKG_BUILD_TYPE OR "${VCPKG_BUILD_TYPE}" STREQUAL "release") - message(STATUS "${configuring_message}-rel") - vcpkg_execute_required_process( - COMMAND ${rel_command} - WORKING_DIRECTORY "${build_dir_release}" - LOGNAME "${arg_LOGFILE_BASE}-rel" - SAVE_LOG_FILES CMakeCache.txt - ) - vcpkg_list(APPEND config_logs - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-rel-out.log" - "${CURRENT_BUILDTREES_DIR}/${arg_LOGFILE_BASE}-rel-err.log") - endif() - endif() - - set(all_unused_variables) - foreach(config_log IN LISTS config_logs) - if(NOT EXISTS "${config_log}") - continue() - endif() - file(READ "${config_log}" log_contents) - debug_message("Reading configure log ${config_log}...") - if(NOT log_contents MATCHES "Manually-specified variables were not used by the project:\n\n(( [^\n]*\n)*)") - continue() - endif() - string(STRIP "${CMAKE_MATCH_1}" unused_variables) # remove leading ` ` and trailing `\n` - string(REPLACE "\n " ";" unused_variables "${unused_variables}") - debug_message("unused variables: ${unused_variables}") - foreach(unused_variable IN LISTS unused_variables) - if(unused_variable IN_LIST manually_specified_variables) - debug_message("manually specified unused variable: ${unused_variable}") - vcpkg_list(APPEND all_unused_variables "${unused_variable}") - else() - debug_message("unused variable (not manually specified): ${unused_variable}") - endif() - endforeach() - endforeach() - - if(DEFINED all_unused_variables) - vcpkg_list(REMOVE_DUPLICATES all_unused_variables) - vcpkg_list(JOIN all_unused_variables "\n " all_unused_variables) - message(WARNING "The following variables are not used in CMakeLists.txt: - ${all_unused_variables} -Please recheck them and remove the unnecessary options from the `vcpkg_cmake_configure` call. -If these options should still be passed for whatever reason, please use the `MAYBE_UNUSED_VARIABLES` argument.") - endif() - - if(NOT arg_Z_CMAKE_GET_VARS_USAGE) - set(Z_VCPKG_CMAKE_GENERATOR "${generator}" CACHE INTERNAL "The generator which was used to configure CMake.") - endif() -endfunction() diff --git a/ports/vcpkg-cmake/vcpkg_cmake_install.cmake b/ports/vcpkg-cmake/vcpkg_cmake_install.cmake deleted file mode 100644 index 2bd8b4ea75ff..000000000000 --- a/ports/vcpkg-cmake/vcpkg_cmake_install.cmake +++ /dev/null @@ -1,21 +0,0 @@ -include_guard(GLOBAL) - -function(vcpkg_cmake_install) - cmake_parse_arguments(PARSE_ARGV 0 "arg" "DISABLE_PARALLEL;ADD_BIN_TO_PATH" "" "") - if(DEFINED arg_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "vcpkg_cmake_install was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") - endif() - - set(args) - foreach(arg IN ITEMS DISABLE_PARALLEL ADD_BIN_TO_PATH) - if(arg_${arg}) - list(APPEND args "${arg}") - endif() - endforeach() - - vcpkg_cmake_build( - ${args} - LOGFILE_BASE install - TARGET install - ) -endfunction() diff --git a/ports/zlib/0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch b/ports/zlib/0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch deleted file mode 100644 index 8fe2b2f5a668..000000000000 --- a/ports/zlib/0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/zconf.h.cmakein b/zconf.h.cmakein -index a7f24cc..a1b359b 100644 ---- a/zconf.h.cmakein -+++ b/zconf.h.cmakein -@@ -434,11 +434,19 @@ typedef uLong FAR uLongf; - #endif - - #ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ --# define Z_HAVE_UNISTD_H -+# if ~(~HAVE_UNISTD_H + 0) == 0 && ~(~HAVE_UNISTD_H + 1) == 1 -+# define Z_HAVE_UNISTD_H -+# elif HAVE_UNISTD_H != 0 -+# define Z_HAVE_UNISTD_H -+# endif - #endif - - #ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ --# define Z_HAVE_STDARG_H -+# if ~(~HAVE_STDARG_H + 0) == 0 && ~(~HAVE_STDARG_H + 1) == 1 -+# define Z_HAVE_STDARG_H -+# elif HAVE_STDARG_H != 0 -+# define Z_HAVE_STDARG_H -+# endif - #endif - - #ifdef STDC -diff --git a/zconf.h.in b/zconf.h.in -index 5e1d68a..32f53c8 100644 ---- a/zconf.h.in -+++ b/zconf.h.in -@@ -432,11 +432,19 @@ typedef uLong FAR uLongf; - #endif - - #ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ --# define Z_HAVE_UNISTD_H -+# if ~(~HAVE_UNISTD_H + 0) == 0 && ~(~HAVE_UNISTD_H + 1) == 1 -+# define Z_HAVE_UNISTD_H -+# elif HAVE_UNISTD_H != 0 -+# define Z_HAVE_UNISTD_H -+# endif - #endif - - #ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ --# define Z_HAVE_STDARG_H -+# if ~(~HAVE_STDARG_H + 0) == 0 && ~(~HAVE_STDARG_H + 1) == 1 -+# define Z_HAVE_STDARG_H -+# elif HAVE_STDARG_H != 0 -+# define Z_HAVE_STDARG_H -+# endif - #endif - - #ifdef STDC - diff --git a/ports/zlib/0002-skip-building-examples.patch b/ports/zlib/0002-skip-building-examples.patch deleted file mode 100644 index 8183f2ab20b0..000000000000 --- a/ports/zlib/0002-skip-building-examples.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index b412dc7..f46c8e6 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -194,6 +194,7 @@ endif() - # Example binaries - #============================================================================ - -+if (0) - add_executable(example test/example.c) - target_link_libraries(example zlib) - add_test(example example) -@@ -211,3 +212,4 @@ if(HAVE_OFF64_T) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - endif() -+endif() diff --git a/ports/zlib/0003-build-static-or-shared-not-both.patch b/ports/zlib/0003-build-static-or-shared-not-both.patch deleted file mode 100644 index c9f2ecf19762..000000000000 --- a/ports/zlib/0003-build-static-or-shared-not-both.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f46c8e6..6fa5575 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -121,9 +121,11 @@ set(ZLIB_SRCS - ) - - if(NOT MINGW) -+ if(BUILD_SHARED_LIBS) - set(ZLIB_DLL_SRCS - win32/zlib1.rc # If present will override custom build rule below. - ) -+ endif() - endif() - - # parse the full version number from zlib.h and include in ZLIB_FULL_VERSION -@@ -144,13 +146,16 @@ if(MINGW) - -I ${CMAKE_CURRENT_BINARY_DIR} - -o ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj - -i ${CMAKE_CURRENT_SOURCE_DIR}/win32/zlib1.rc) -+ if(BUILD_SHARED_LIBS) - set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) -+ endif() - endif(MINGW) - --add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) --add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -+add_library(zlib ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -+if (BUILD_SHARED_LIBS) - set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) - set_target_properties(zlib PROPERTIES SOVERSION 1) -+endif() - - if(NOT CYGWIN) - # This property causes shared libraries on Linux to have the full version -@@ -165,7 +170,7 @@ endif() - - if(UNIX) - # On unix-like platforms the library is almost always called libz -- set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) -+ set_target_properties(zlib PROPERTIES OUTPUT_NAME z) - if(NOT APPLE) - set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") - endif() -@@ -175,7 +180,7 @@ elseif(BUILD_SHARED_LIBS AND WIN32) - endif() - - if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL ) -- install(TARGETS zlib zlibstatic -+ install(TARGETS zlib - RUNTIME DESTINATION "${INSTALL_BIN_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ) diff --git a/ports/zlib/0004-android-and-mingw-fixes.patch b/ports/zlib/0004-android-and-mingw-fixes.patch deleted file mode 100644 index e93173f24618..000000000000 --- a/ports/zlib/0004-android-and-mingw-fixes.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 6fa5575..7c345db 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -56,7 +56,7 @@ endif() - # - check_include_file(unistd.h Z_HAVE_UNISTD_H) - --if(MSVC) -+if(WIN32) - set(CMAKE_DEBUG_POSTFIX "d") - add_definitions(-D_CRT_SECURE_NO_DEPRECATE) - add_definitions(-D_CRT_NONSTDC_NO_DEPRECATE) -@@ -133,7 +133,7 @@ file(READ ${CMAKE_CURRENT_SOURCE_DIR}/zlib.h _zlib_h_contents) - string(REGEX REPLACE ".*#define[ \t]+ZLIB_VERSION[ \t]+\"([-0-9A-Za-z.]+)\".*" - "\\1" ZLIB_FULL_VERSION ${_zlib_h_contents}) - --if(MINGW) -+if(MINGW AND NOT ANDROID) - # This gets us DLL resource information when compiling on MinGW. - if(NOT CMAKE_RC_COMPILER) - set(CMAKE_RC_COMPILER windres.exe) -@@ -149,7 +149,7 @@ if(MINGW) - if(BUILD_SHARED_LIBS) - set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) - endif() --endif(MINGW) -+endif(MINGW AND NOT ANDROID) - - add_library(zlib ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) - if (BUILD_SHARED_LIBS) diff --git a/ports/zlib/portfile.cmake b/ports/zlib/portfile.cmake deleted file mode 100644 index 450cb2c1915b..000000000000 --- a/ports/zlib/portfile.cmake +++ /dev/null @@ -1,53 +0,0 @@ -# When this port is updated, the minizip port should be updated at the same time -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO madler/zlib - REF v1.2.13 - SHA512 44b834fbfb50cca229209b8dbe1f96b258f19a49f5df23b80970b716371d856a4adf525edb4c6e0e645b180ea949cb90f5365a1d896160f297f56794dd888659 - HEAD_REF master - PATCHES - 0001-Prevent-invalid-inclusions-when-HAVE_-is-set-to-0.patch - 0002-skip-building-examples.patch - 0003-build-static-or-shared-not-both.patch - 0004-android-and-mingw-fixes.patch -) - -# This is generated during the cmake build -file(REMOVE "${SOURCE_PATH}/zconf.h") - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DSKIP_INSTALL_FILES=ON - OPTIONS_DEBUG - -DSKIP_INSTALL_HEADERS=ON -) - -vcpkg_cmake_install() -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") - -# Install the pkgconfig file -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - if(VCPKG_TARGET_IS_WINDOWS) - vcpkg_replace_string("${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/zlib.pc" "-lz" "-lzlib") - endif() - file(COPY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/zlib.pc" DESTINATION "${CURRENT_PACKAGES_DIR}/lib/pkgconfig") -endif() -if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - if(VCPKG_TARGET_IS_WINDOWS) - vcpkg_replace_string("${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/zlib.pc" "-lz" "-lzlibd") - endif() - file(COPY "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-dbg/zlib.pc" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig") -endif() - -vcpkg_fixup_pkgconfig() -vcpkg_copy_pdbs() - -if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/zconf.h" "ifdef ZLIB_DLL" "if 0") -else() - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/zconf.h" "ifdef ZLIB_DLL" "if 1") -endif() - -file(COPY "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) diff --git a/ports/zlib/usage b/ports/zlib/usage deleted file mode 100644 index 39d8618221a6..000000000000 --- a/ports/zlib/usage +++ /dev/null @@ -1,4 +0,0 @@ -The package zlib is compatible with built-in CMake targets: - - find_package(ZLIB REQUIRED) - target_link_libraries(main PRIVATE ZLIB::ZLIB) diff --git a/ports/zlib/vcpkg-cmake-wrapper.cmake b/ports/zlib/vcpkg-cmake-wrapper.cmake deleted file mode 100644 index 8624b70867e9..000000000000 --- a/ports/zlib/vcpkg-cmake-wrapper.cmake +++ /dev/null @@ -1,12 +0,0 @@ -find_path(ZLIB_INCLUDE_DIR NAMES zlib.h PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include" NO_DEFAULT_PATH) -find_library(ZLIB_LIBRARY_RELEASE NAMES zlib z PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib" NO_DEFAULT_PATH) -find_library(ZLIB_LIBRARY_DEBUG NAMES zlibd z PATHS "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/lib" NO_DEFAULT_PATH) -if(NOT ZLIB_INCLUDE_DIR OR NOT (ZLIB_LIBRARY_RELEASE OR ZLIB_LIBRARY_DEBUG)) - message(FATAL_ERROR "Broken installation of vcpkg port zlib") -endif() -if(CMAKE_VERSION VERSION_LESS 3.4) - include(SelectLibraryConfigurations) - select_library_configurations(ZLIB) - unset(ZLIB_FOUND) -endif() -_find_package(${ARGS}) diff --git a/ports/zlib/vcpkg.json b/ports/zlib/vcpkg.json deleted file mode 100644 index 7ef1f29d8ed9..000000000000 --- a/ports/zlib/vcpkg.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "zlib", - "version": "1.2.13", - "description": "A compression library", - "homepage": "https://www.zlib.net/", - "license": "Zlib", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - } - ] -} diff --git a/ports/zstd/install_pkgpc.patch b/ports/zstd/install_pkgpc.patch deleted file mode 100644 index a0582d98b21c..000000000000 --- a/ports/zstd/install_pkgpc.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt -index 5f75665..dabc9d5 100644 ---- a/build/cmake/lib/CMakeLists.txt -+++ b/build/cmake/lib/CMakeLists.txt -@@ -130,7 +130,7 @@ if (ZSTD_BUILD_STATIC) - OUTPUT_NAME ${STATIC_LIBRARY_BASE_NAME}) - endif () - --if (UNIX OR MINGW) -+if (1) - # pkg-config - set(PREFIX "${CMAKE_INSTALL_PREFIX}") - set(EXEC_PREFIX "\${prefix}") diff --git a/ports/zstd/no-static-suffix.patch b/ports/zstd/no-static-suffix.patch deleted file mode 100644 index 4e61f96b87d5..000000000000 --- a/ports/zstd/no-static-suffix.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt -index 8234060..765f003 100644 ---- a/build/cmake/lib/CMakeLists.txt -+++ b/build/cmake/lib/CMakeLists.txt -@@ -112,7 +112,7 @@ endif () - - # With MSVC static library needs to be renamed to avoid conflict with import library - if (MSVC OR (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND NOT MINGW)) -- set(STATIC_LIBRARY_BASE_NAME zstd_static) -+ set(STATIC_LIBRARY_BASE_NAME zstd) - else () - set(STATIC_LIBRARY_BASE_NAME zstd) - endif () diff --git a/ports/zstd/portfile.cmake b/ports/zstd/portfile.cmake deleted file mode 100644 index 6d7aa4df8838..000000000000 --- a/ports/zstd/portfile.cmake +++ /dev/null @@ -1,55 +0,0 @@ -vcpkg_minimum_required(VERSION 2022-10-12) # for ${VERSION} -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO facebook/zstd - REF "v${VERSION}" - SHA512 e107508a41fca50845cc2494e64adaba93efb95a2fa486fc962510a8ba4b2180d93067cae9870f119e88e5e8b28a046bc2240b0b23cdd8933d1fb1a6a9668c1e - HEAD_REF dev - PATCHES - install_pkgpc.patch - no-static-suffix.patch -) - -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" ZSTD_BUILD_STATIC) -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" ZSTD_BUILD_SHARED) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}/build/cmake" - OPTIONS - -DZSTD_BUILD_SHARED=${ZSTD_BUILD_SHARED} - -DZSTD_BUILD_STATIC=${ZSTD_BUILD_STATIC} - -DZSTD_LEGACY_SUPPORT=1 - -DZSTD_BUILD_PROGRAMS=0 - -DZSTD_BUILD_TESTS=0 - -DZSTD_BUILD_CONTRIB=0 - -DZSTD_MULTITHREAD_SUPPORT=1 -) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/zstd) -vcpkg_fixup_pkgconfig() - -file(READ "${CURRENT_PACKAGES_DIR}/share/zstd/zstdTargets.cmake" targets) -if(targets MATCHES "-pthread") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/libzstd.pc" " -lzstd" " -lzstd -pthread") - if(NOT VCPKG_BUILD_TYPE) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/libzstd.pc" " -lzstd" " -lzstd -pthread") - endif() -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share") - -if(VCPKG_TARGET_IS_WINDOWS AND VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") - foreach(HEADER IN ITEMS zdict.h zstd.h zstd_errors.h) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/${HEADER}" "defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)" "1" ) - endforeach() -endif() - -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -vcpkg_install_copyright( - COMMENT "ZSTD is dual licensed under BSD and GPLv2." - FILE_LIST - "${SOURCE_PATH}/LICENSE" - "${SOURCE_PATH}/COPYING" -) diff --git a/ports/zstd/usage b/ports/zstd/usage deleted file mode 100644 index f01a2f5cfb7d..000000000000 --- a/ports/zstd/usage +++ /dev/null @@ -1,4 +0,0 @@ -The package zstd provides CMake targets: - - find_package(zstd CONFIG REQUIRED) - target_link_libraries(main PRIVATE $,zstd::libzstd_shared,zstd::libzstd_static>) diff --git a/ports/zstd/vcpkg.json b/ports/zstd/vcpkg.json deleted file mode 100644 index d683b33e7de2..000000000000 --- a/ports/zstd/vcpkg.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "zstd", - "version": "1.5.2", - "port-version": 3, - "description": "Zstandard - Fast real-time compression algorithm", - "homepage": "https://facebook.github.io/zstd/", - "license": "BSD-3-Clause OR GPL-2.0-only", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ] -} diff --git a/test/src/unit-cppapi-global-order-writes-remote.cc b/test/src/unit-cppapi-global-order-writes-remote.cc index 2bdfb4eddfc2..5feee0c12109 100644 --- a/test/src/unit-cppapi-global-order-writes-remote.cc +++ b/test/src/unit-cppapi-global-order-writes-remote.cc @@ -36,6 +36,7 @@ #include #include +#include using namespace tiledb; diff --git a/test/src/unit-dimension.cc b/test/src/unit-dimension.cc index 366903ac7bd9..990790d79418 100644 --- a/test/src/unit-dimension.cc +++ b/test/src/unit-dimension.cc @@ -273,7 +273,7 @@ TEST_CASE( TEST_CASE( "Dimension: Test map_to_uint64, float32", - "[dimension][map_to_uint64][float32") { + "[dimension][map_to_uint64][float32]") { // Create dimensions Dimension d1("d1", Datatype::FLOAT32); float dom1[] = {0.0f, 1.0f}; diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index ba2b686aa452..6d9ae77052d3 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2073,7 +2073,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( EnumerationFx, "Cap'N Proto - Basic New ArraySchema Serialization", - "[enumeration][capnp][basic][initialized-in-ram") { + "[enumeration][capnp][basic][initialized-in-ram]") { auto client_side = GENERATE(true, false); auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); diff --git a/vcpkg.json b/vcpkg.json index 86a75075b40f..c8b9831f013e 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,5 @@ { - "name": "tiledb", - "version-string": "2.15.0", - "builtin-baseline": "c0b1007fe7fc6ccc902e6c708fd6a670ddad8f9d", + "builtin-baseline": "72010900b7cee36cea77aebb97695095c9358eaf", "dependencies": [ "bzip2", "libmagic", @@ -18,16 +16,9 @@ "azure": { "description": "Support Azure Blob Storage", "dependencies": [ - { - "name": "azure-storage-blobs-cpp", - "version>=": "12.6.1" - }, + "azure-storage-blobs-cpp", { "name": "libxml2", - "features": [ - "lzma", - "zlib" - ], "default-features": false, "platform": "!windows" } @@ -38,7 +29,6 @@ "dependencies": [ { "name": "google-cloud-cpp", - "version>=": "2.15.1", "features": [ "storage" ], @@ -59,10 +49,7 @@ "name": "curl", "features": [ "zstd" ] }, - { - "name": "capnproto", - "version>=": "1.0.1" - } + "capnproto" ] }, "s3": { @@ -70,7 +57,6 @@ "dependencies": [ { "name": "aws-sdk-cpp", - "version>=": "1.11.160", "features": ["s3", "identity-management", "sts"], "default-features": false } @@ -91,21 +77,15 @@ "webp": { "description": "Support WebP compression", "dependencies": [ - { - "name": "libwebp", - "version>=": "1.3.2" - } + "libwebp" ] } }, "overrides": [ { - "name": "abseil", - "version": "20230802.1" - }, - { - "name": "curl", - "version": "8.4.0" + "$note": "Remove this when the custom port for libmagic gets removed", + "name": "dirent", + "version": "1.23.2#3" } ] } From 8c701dd11da0e27c4abf2baa2156f7fbddae8380 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Fri, 26 Jan 2024 04:11:48 -0500 Subject: [PATCH 160/456] Update S3 to inherit from FilesystemBase. (#4608) This updates S3 to inherit from FilesystemBase. Since other cloud backends will inherit from FilesystemBase in the future, this adds common cloud functions to the base class and stubs them out as no-ops in Posix. I also reorganized definitions to match declaration order for both Posix and S3 in my last two commits. --- TYPE: NO_HISTORY DESC: Update S3 to inherit from FilesystemBase. --- test/src/unit-s3-no-multipart.cc | 42 +- test/src/unit-s3.cc | 80 +- test/src/unit-vfs.cc | 2 +- test/support/src/vfs_helpers.h | 6 +- tiledb/sm/filesystem/filesystem_base.h | 94 ++- tiledb/sm/filesystem/posix.cc | 570 +++++++------- tiledb/sm/filesystem/posix.h | 192 +++-- tiledb/sm/filesystem/s3.cc | 701 +++++++++--------- tiledb/sm/filesystem/s3.h | 342 ++++++--- .../test/unit_vfs_read_log_modes.cc | 3 +- tiledb/sm/filesystem/vfs.cc | 85 ++- tiledb/sm/filesystem/vfs.h | 6 +- 12 files changed, 1154 insertions(+), 969 deletions(-) diff --git a/test/src/unit-s3-no-multipart.cc b/test/src/unit-s3-no-multipart.cc index 272b09d7bd06..2c973cf29b75 100644 --- a/test/src/unit-s3-no-multipart.cc +++ b/test/src/unit-s3-no-multipart.cc @@ -62,33 +62,30 @@ struct S3DirectFx { S3DirectFx::S3DirectFx() { // Create bucket - bool exists; - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + bool exists = s3_.is_bucket(S3_BUCKET); if (exists) - REQUIRE(s3_.remove_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + exists = s3_.is_bucket(S3_BUCKET); REQUIRE(!exists); - REQUIRE(s3_.create_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.create_bucket(S3_BUCKET)); // Check if bucket is empty - bool is_empty; - REQUIRE(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); } S3DirectFx::~S3DirectFx() { // Empty bucket - bool is_empty; - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); if (!is_empty) { - CHECK(s3_.empty_bucket(S3_BUCKET).ok()); - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + CHECK_NOTHROW(s3_.empty_bucket(S3_BUCKET)); + is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); } // Delete bucket - CHECK(s3_.remove_bucket(S3_BUCKET).ok()); + CHECK_NOTHROW(s3_.remove_bucket(S3_BUCKET)); CHECK(s3_.disconnect().ok()); } @@ -124,10 +121,12 @@ TEST_CASE_METHOD( // Write to two files auto largefile = TEST_DIR + "largefile"; - CHECK(s3_.write(URI(largefile), write_buffer, buffer_size).ok()); - CHECK(s3_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); + CHECK_NOTHROW(s3_.write(URI(largefile), write_buffer, buffer_size)); + CHECK_NOTHROW( + s3_.write(URI(largefile), write_buffer_small, buffer_size_small)); auto smallfile = TEST_DIR + "smallfile"; - CHECK(s3_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); + CHECK_NOTHROW( + s3_.write(URI(smallfile), write_buffer_small, buffer_size_small)); // Before flushing, the files do not exist bool exists = false; @@ -156,7 +155,8 @@ TEST_CASE_METHOD( // Read from the beginning auto read_buffer = new char[26]; uint64_t bytes_read = 0; - CHECK(s3_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); + CHECK_NOTHROW( + s3_.read_impl(URI(largefile), 0, read_buffer, 26, 0, &bytes_read)); assert(26 == bytes_read); bool allok = true; for (int i = 0; i < 26; i++) { @@ -168,7 +168,8 @@ TEST_CASE_METHOD( CHECK(allok); // Read from a different offset - CHECK(s3_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); + CHECK_NOTHROW( + s3_.read_impl(URI(largefile), 11, read_buffer, 26, 0, &bytes_read)); assert(26 == bytes_read); allok = true; for (int i = 0; i < 26; i++) { @@ -182,7 +183,7 @@ TEST_CASE_METHOD( // Try to write 11 MB file, should fail with given buffer configuration auto badfile = TEST_DIR + "badfile"; auto badbuffer = (char*)malloc(11000000); - CHECK(!(s3_.write(URI(badfile), badbuffer, 11000000).ok())); + CHECK_THROWS((s3_.write(URI(badfile), badbuffer, 11000000))); } TEST_CASE_METHOD( @@ -199,9 +200,8 @@ TEST_CASE_METHOD( tiledb::sm::S3 s3{&g_helper_stats, &thread_pool_, cfg}; auto uri = URI(TEST_DIR + "writefailure"); - // This is a buffered write, which is why it returns ok. - auto st = s3.write(uri, "Validate s3 custom headers", 26); - REQUIRE(st.ok()); + // This is a buffered write, which is why it should not throw. + CHECK_NOTHROW(s3.write(uri, "Validate s3 custom headers", 26)); auto matcher = Catch::Matchers::ContainsSubstring( "The Content-Md5 you specified is not valid."); diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index a6e4d05eff27..cf05f8a9b3c4 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -63,33 +63,30 @@ struct S3Fx { S3Fx::S3Fx() { // Create bucket - bool exists; - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + bool exists = s3_.is_bucket(S3_BUCKET); if (exists) - REQUIRE(s3_.remove_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + exists = s3_.is_bucket(S3_BUCKET); REQUIRE(!exists); - REQUIRE(s3_.create_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.create_bucket(S3_BUCKET)); // Check if bucket is empty - bool is_empty; - REQUIRE(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); } S3Fx::~S3Fx() { // Empty bucket - bool is_empty; - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); if (!is_empty) { - CHECK(s3_.empty_bucket(S3_BUCKET).ok()); - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + CHECK_NOTHROW(s3_.empty_bucket(S3_BUCKET)); + is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); } // Delete bucket - CHECK(s3_.remove_bucket(S3_BUCKET).ok()); + CHECK_NOTHROW(s3_.remove_bucket(S3_BUCKET)); CHECK(s3_.disconnect().ok()); } @@ -118,10 +115,12 @@ TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { // Write to two files auto largefile = TEST_DIR + "largefile"; - CHECK(s3_.write(URI(largefile), write_buffer, buffer_size).ok()); - CHECK(s3_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); + CHECK_NOTHROW(s3_.write(URI(largefile), write_buffer, buffer_size)); + CHECK_NOTHROW( + s3_.write(URI(largefile), write_buffer_small, buffer_size_small)); auto smallfile = TEST_DIR + "smallfile"; - CHECK(s3_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); + CHECK_NOTHROW( + s3_.write(URI(smallfile), write_buffer_small, buffer_size_small)); // Before flushing, the files do not exist bool exists = false; @@ -150,7 +149,8 @@ TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { // Read from the beginning auto read_buffer = new char[26]; uint64_t bytes_read = 0; - CHECK(s3_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); + CHECK_NOTHROW( + s3_.read_impl(URI(largefile), 0, read_buffer, 26, 0, &bytes_read)); CHECK(26 == bytes_read); bool allok = true; for (int i = 0; i < 26; i++) { @@ -162,7 +162,8 @@ TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { CHECK(allok); // Read from a different offset - CHECK(s3_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); + CHECK_NOTHROW( + s3_.read_impl(URI(largefile), 11, read_buffer, 26, 0, &bytes_read)); CHECK(26 == bytes_read); allok = true; for (int i = 0; i < 26; i++) { @@ -188,7 +189,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 multiupload abort path", "[s3]") { // Write one large file, the write will fail auto largefile = TEST_DIR + "failed_largefile_" + std::to_string(nth_failure); - CHECK(!s3_.write(URI(largefile), write_buffer, buffer_size).ok()); + CHECK_THROWS(s3_.write(URI(largefile), write_buffer, buffer_size)); // Before flushing, the file does not exist bool exists = false; @@ -240,18 +241,16 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use BucketCannedACL", "[s3]") { tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, config}; // Create bucket - bool exists; - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + bool exists = s3_.is_bucket(S3_BUCKET); if (exists) - REQUIRE(s3_.remove_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + exists = s3_.is_bucket(S3_BUCKET); REQUIRE(!exists); - REQUIRE(s3_.create_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.create_bucket(S3_BUCKET)); // Check if bucket is empty - bool is_empty; - REQUIRE(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); CHECK(s3_.disconnect().ok()); @@ -295,30 +294,29 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") { auto file6 = TEST_DIR + "file6"; // Check that bucket is empty - bool is_empty; - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); // Continue building the hierarchy bool exists = false; - CHECK(s3_.touch(URI(file1)).ok()); + CHECK_NOTHROW(s3_.touch(URI(file1))); CHECK(s3_.is_object(URI(file1), &exists).ok()); CHECK(exists); - CHECK(s3_.touch(URI(file2)).ok()); + CHECK_NOTHROW(s3_.touch(URI(file2))); CHECK(s3_.is_object(URI(file2), &exists).ok()); CHECK(exists); - CHECK(s3_.touch(URI(file3)).ok()); + CHECK_NOTHROW(s3_.touch(URI(file3))); CHECK(s3_.is_object(URI(file3), &exists).ok()); CHECK(exists); - CHECK(s3_.touch(URI(file4)).ok()); + CHECK_NOTHROW(s3_.touch(URI(file4))); CHECK(s3_.is_object(URI(file4), &exists).ok()); CHECK(exists); - CHECK(s3_.touch(URI(file5)).ok()); + CHECK_NOTHROW(s3_.touch(URI(file5))); CHECK(s3_.is_object(URI(file5), &exists).ok()); CHECK(exists); // Check that the bucket is not empty - CHECK(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(!is_empty); // Check invalid file @@ -361,7 +359,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") { CHECK(paths.size() == 5); // Move directory - CHECK(s3_.move_dir(URI(dir), URI(dir2)).ok()); + CHECK_NOTHROW(s3_.move_dir(URI(dir), URI(dir2))); CHECK(s3_.is_dir(URI(dir), &is_dir).ok()); CHECK(!is_dir); CHECK(s3_.is_dir(URI(dir2), &is_dir).ok()); @@ -376,7 +374,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") { CHECK(!exists); // Remove directories - CHECK(s3_.remove_dir(URI(dir2)).ok()); + CHECK_NOTHROW(s3_.remove_dir(URI(dir2))); CHECK(s3_.is_object(URI(file1), &exists).ok()); CHECK(!exists); CHECK(s3_.is_object(URI(file2), &exists).ok()); @@ -394,18 +392,16 @@ TEST_CASE_METHOD(S3Fx, "Test S3 use Bucket/Object CannedACL", "[s3]") { tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, config}; // Create bucket - bool exists; - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + bool exists = s3_.is_bucket(S3_BUCKET); if (exists) - REQUIRE(s3_.remove_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - REQUIRE(s3_.is_bucket(S3_BUCKET, &exists).ok()); + exists = s3_.is_bucket(S3_BUCKET); REQUIRE(!exists); - REQUIRE(s3_.create_bucket(S3_BUCKET).ok()); + REQUIRE_NOTHROW(s3_.create_bucket(S3_BUCKET)); // Check if bucket is empty - bool is_empty; - REQUIRE(s3_.is_empty_bucket(S3_BUCKET, &is_empty).ok()); + bool is_empty = s3_.is_empty_bucket(S3_BUCKET); CHECK(is_empty); exercise_object_canned_acl(); diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index fe70a04dd7a5..ae6fa1d250b6 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -152,7 +152,7 @@ TEST_CASE("VFS: Test long local paths", "[vfs]") { require_tiledb_ok(vfs.is_file(testfile, &exists)); // Creating the file is not - require_tiledb_err(vfs.touch(testfile)); + REQUIRE_THROWS(vfs.touch(testfile)); } // Creating the URI is invalid on Win32 (failure to canonicalize path) diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index 72170aac4cce..c343f023d1a0 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -862,15 +862,15 @@ class S3Test : public VFSTestBase, protected tiledb::sm::S3_within_VFS { : VFSTestBase(test_tree, "s3://") , S3_within_VFS(&tiledb::test::g_helper_stats, &io_, vfs_.config()) { #ifdef HAVE_S3 - s3().create_bucket(temp_dir_).ok(); + s3().create_bucket(temp_dir_); for (size_t i = 1; i <= test_tree_.size(); i++) { sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); // VFS::create_dir is a no-op for S3; Just create objects. for (size_t j = 1; j <= test_tree_[i - 1]; j++) { auto object_uri = path.join_path("test_file_" + std::to_string(j)); - s3().touch(object_uri).ok(); + s3().touch(object_uri); std::string data(j * 10, 'a'); - s3().write(object_uri, data.data(), data.size()).ok(); + s3().write(object_uri, data.data(), data.size()); s3().flush_object(object_uri).ok(); expected_results().emplace_back(object_uri.to_string(), data.size()); } diff --git a/tiledb/sm/filesystem/filesystem_base.h b/tiledb/sm/filesystem/filesystem_base.h index 89be313d1163..ff54707376f3 100644 --- a/tiledb/sm/filesystem/filesystem_base.h +++ b/tiledb/sm/filesystem/filesystem_base.h @@ -33,12 +33,19 @@ #ifndef TILEDB_FILESYSTEMBASE_H #define TILEDB_FILESYSTEMBASE_H +#include "tiledb/common/exception/exception.h" #include "tiledb/common/filesystem/directory_entry.h" #include "uri.h" #include namespace tiledb::sm { +class IOError : public StatusException { + public: + explicit IOError(const std::string& message) + : StatusException("IO Error", message) { + } +}; class FilesystemBase { public: @@ -54,17 +61,15 @@ class FilesystemBase { * just succeeds without doing anything. * * @param uri The URI of the directory. - * @return Status */ - virtual Status create_dir(const URI& uri) const = 0; + virtual void create_dir(const URI& uri) const = 0; /** * Creates an empty file. * * @param uri The URI of the file. - * @return Status */ - virtual Status touch(const URI& uri) const = 0; + virtual void touch(const URI& uri) const = 0; /** * Checks if a directory exists. @@ -86,26 +91,23 @@ class FilesystemBase { * Removes a given directory (recursive) * * @param uri The uri of the directory to be removed - * @return Status */ - virtual Status remove_dir(const URI& uri) const = 0; + virtual void remove_dir(const URI& uri) const = 0; /** * Deletes a file. * * @param uri The URI of the file. - * @return Status */ - virtual Status remove_file(const URI& uri) const = 0; + virtual void remove_file(const URI& uri) const = 0; /** * Retrieves the size of a file. * * @param uri The URI of the file. * @param size The file size to be retrieved. - * @return Status */ - virtual Status file_size(const URI& uri, uint64_t* size) const = 0; + virtual void file_size(const URI& uri, uint64_t* size) const = 0; /** * Retrieves all the entries contained in the parent. @@ -113,9 +115,7 @@ class FilesystemBase { * @param parent The target directory to list. * @return All entries that are contained in the parent */ - virtual tuple< - Status, - optional>> + virtual tuple>> ls_with_sizes(const URI& parent) const = 0; /** @@ -124,9 +124,17 @@ class FilesystemBase { * * @param old_uri The old URI. * @param new_uri The new URI. - * @return Status */ - virtual Status move_file(const URI& old_uri, const URI& new_uri) const = 0; + virtual void move_file(const URI& old_uri, const URI& new_uri) const = 0; + + /** + * Renames a directory. + * Both URI must be of the same backend type. (e.g. both s3://, file://, etc) + * + * @param old_uri The old URI. + * @param new_uri The new URI. + */ + virtual void move_dir(const URI& old_uri, const URI& new_uri) const = 0; /** * Copies a file. @@ -134,9 +142,8 @@ class FilesystemBase { * * @param old_uri The old URI. * @param new_uri The new URI. - * @return Status */ - virtual Status copy_file(const URI& old_uri, const URI& new_uri) const = 0; + virtual void copy_file(const URI& old_uri, const URI& new_uri) const = 0; /** * Copies directory. @@ -144,9 +151,8 @@ class FilesystemBase { * * @param old_uri The old URI. * @param new_uri The new URI. - * @return Status */ - virtual Status copy_dir(const URI& old_uri, const URI& new_uri) const = 0; + virtual void copy_dir(const URI& old_uri, const URI& new_uri) const = 0; /** * Reads from a file. @@ -156,22 +162,20 @@ class FilesystemBase { * @param buffer The buffer to read into. * @param nbytes Number of bytes to read. * @param use_read_ahead Whether to use the read-ahead cache. - * @return Status */ - virtual Status read( + virtual void read( const URI& uri, uint64_t offset, void* buffer, uint64_t nbytes, - bool use_read_ahead = true) = 0; + bool use_read_ahead = true) const = 0; /** * Syncs (flushes) a file. Note that for S3 this is a noop. * * @param uri The URI of the file. - * @return Status */ - virtual Status sync(const URI& uri) = 0; + virtual void sync(const URI& uri) const = 0; /** * Writes the contents of a buffer into a file. @@ -180,13 +184,49 @@ class FilesystemBase { * @param buffer The buffer to write from. * @param buffer_size The buffer size. * @param remote_global_order_write Remote global order write - * @return Status */ - virtual Status write( + virtual void write( const URI& uri, const void* buffer, uint64_t buffer_size, - bool remote_global_order_write = false) = 0; + bool remote_global_order_write) = 0; + + /** + * Checks if an object store bucket exists. + * + * @param uri The name of the object store bucket. + * @return True if the bucket exists, false otherwise. + */ + virtual bool is_bucket(const URI& uri) const = 0; + + /** + * Checks if an object-store bucket is empty. + * + * @param uri The name of the object store bucket. + * @return True if the bucket is empty, false otherwise. + */ + virtual bool is_empty_bucket(const URI& uri) const = 0; + + /** + * Creates an object store bucket. + * + * @param uri The name of the bucket to be created. + */ + virtual void create_bucket(const URI& uri) const = 0; + + /** + * Deletes an object store bucket. + * + * @param uri The name of the bucket to be deleted. + */ + virtual void remove_bucket(const URI& uri) const = 0; + + /** + * Deletes the contents of an object store bucket. + * + * @param uri The name of the bucket to be emptied. + */ + virtual void empty_bucket(const URI& uri) const = 0; }; } // namespace tiledb::sm diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index a209601f2521..70d4e8320bcd 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -54,10 +54,9 @@ #include using namespace tiledb::common; -using tiledb::common::filesystem::directory_entry; +using filesystem::directory_entry; -namespace tiledb { -namespace sm { +namespace tiledb::sm { Posix::Posix(const Config& config) { // Initialize member variables with posix config parameters. @@ -71,164 +70,69 @@ Posix::Posix(const Config& config) { directory_permissions_ = std::strtol(permissions.c_str(), nullptr, 8); } -bool Posix::both_slashes(char a, char b) { - return a == '/' && b == '/'; -} - -Status Posix::read_all(int fd, void* buffer, uint64_t nbytes, uint64_t offset) { - auto bytes = reinterpret_cast(buffer); - uint64_t nread = 0; - do { - ssize_t actual_read = - ::pread(fd, bytes + nread, nbytes - nread, offset + nread); - if (actual_read < 0) { - return LOG_STATUS( - Status_IOError(std::string("POSIX read error: ") + strerror(errno))); - } else if (actual_read == 0) { - break; - } - nread += actual_read; - } while (nread < nbytes); - - if (nread != nbytes) { - return LOG_STATUS(Status_IOError("POSIX incomplete read: EOF reached")); - } - return Status::Ok(); -} - -void Posix::adjacent_slashes_dedup(std::string* path) { - assert(utils::parse::starts_with(*path, "file://")); - path->erase( - std::unique( - path->begin() + std::string("file://").size(), - path->end(), - both_slashes), - path->end()); -} - -std::string Posix::abs_path(const std::string& path) { - std::string resolved_path = abs_path_internal(path); - - // Ensure the returned has the same postfix slash as 'path'. - if (utils::parse::ends_with(path, "/")) { - if (!utils::parse::ends_with(resolved_path, "/")) { - resolved_path = resolved_path + "/"; - } - } else { - if (utils::parse::ends_with(resolved_path, "/")) { - resolved_path = resolved_path.substr(0, resolved_path.length() - 1); - } - } - - return resolved_path; -} - -std::string Posix::abs_path_internal(const std::string& path) { - // Initialize current, home and root - std::string current = current_dir(); - auto env_home_ptr = getenv("HOME"); - std::string home = env_home_ptr != nullptr ? env_home_ptr : current; - std::string root = "/"; - std::string posix_prefix = "file://"; - - // Easy cases - if (path.empty() || path == "." || path == "./") - return posix_prefix + current; - if (path == "~") - return posix_prefix + home; - if (path == "/") - return posix_prefix + root; - - // Other cases - std::string ret_dir; - if (utils::parse::starts_with(path, posix_prefix)) - return path; - else if (utils::parse::starts_with(path, "/")) - ret_dir = posix_prefix + path; - else if (utils::parse::starts_with(path, "~/")) - ret_dir = posix_prefix + home + path.substr(1, path.size() - 1); - else if (utils::parse::starts_with(path, "./")) - ret_dir = posix_prefix + current + path.substr(1, path.size() - 1); - else - ret_dir = posix_prefix + current + "/" + path; - - adjacent_slashes_dedup(&ret_dir); - purge_dots_from_path(&ret_dir); - - return ret_dir; -} - -Status Posix::create_dir(const URI& uri) const { +void Posix::create_dir(const URI& uri) const { // If the directory does not exist, create it auto path = uri.to_path(); if (is_dir(uri)) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot create directory '") + path + - "'; Directory already exists")); + "'; Directory already exists"); } if (mkdir(path.c_str(), directory_permissions_) != 0) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot create directory '") + path + "'; " + - strerror(errno))); + strerror(errno)); } - return Status::Ok(); } -Status Posix::touch(const URI& uri) const { +void Posix::touch(const URI& uri) const { auto filename = uri.to_path(); int fd = ::open(filename.c_str(), O_WRONLY | O_CREAT | O_SYNC, file_permissions_); if (fd == -1 || ::close(fd) != 0) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Failed to create file '") + filename + "'; " + - strerror(errno))); + strerror(errno)); } - - return Status::Ok(); } -std::string Posix::current_dir() { - static std::unique_ptr cwd_(getcwd(nullptr, 0), free); - std::string dir = cwd_.get(); - return dir; +bool Posix::is_dir(const URI& uri) const { + struct stat st; + memset(&st, 0, sizeof(struct stat)); + return stat(uri.to_path().c_str(), &st) == 0 && S_ISDIR(st.st_mode); } -// TODO: it maybe better to use unlinkat for deeply nested recursive directories -// but the path name length limit in TileDB may make this unnecessary -int Posix::unlink_cb(const char* fpath, const struct stat*, int, struct FTW*) { - int rc = remove(fpath); - if (rc) - perror(fpath); - return rc; +bool Posix::is_file(const URI& uri) const { + struct stat st; + memset(&st, 0, sizeof(struct stat)); + return (stat(uri.to_path().c_str(), &st) == 0) && !S_ISDIR(st.st_mode); } -Status Posix::remove_dir(const URI& uri) const { +void Posix::remove_dir(const URI& uri) const { auto path = uri.to_path(); int rc = nftw(path.c_str(), unlink_cb, 64, FTW_DEPTH | FTW_PHYS); - if (rc) - return LOG_STATUS(Status_IOError( + if (rc) { + throw IOError( std::string("Failed to delete path '") + path + "'; " + - strerror(errno))); - return Status::Ok(); + strerror(errno)); + } } -Status Posix::remove_file(const URI& uri) const { +void Posix::remove_file(const URI& uri) const { auto path = uri.to_path(); if (remove(path.c_str()) != 0) { - return LOG_STATUS(Status_IOError( - std::string("Cannot delete file '") + path + "'; " + strerror(errno))); + throw IOError( + std::string("Cannot delete file '") + path + "'; " + strerror(errno)); } - return Status::Ok(); } -Status Posix::file_size(const URI& uri, uint64_t* size) const { +void Posix::file_size(const URI& uri, uint64_t* size) const { auto path = uri.to_path(); int fd = open(path.c_str(), O_RDONLY); if (fd == -1) { - return LOG_STATUS(Status_IOError( - "Cannot get file size of '" + path + "'; " + strerror(errno))); + throw IOError("Cannot get file size of '" + path + "'; " + strerror(errno)); } struct stat st; @@ -236,92 +140,30 @@ Status Posix::file_size(const URI& uri, uint64_t* size) const { *size = (uint64_t)st.st_size; close(fd); - return Status::Ok(); -} - -bool Posix::is_dir(const URI& uri) const { - struct stat st; - memset(&st, 0, sizeof(struct stat)); - return stat(uri.to_path().c_str(), &st) == 0 && S_ISDIR(st.st_mode); -} - -bool Posix::is_file(const URI& uri) const { - struct stat st; - memset(&st, 0, sizeof(struct stat)); - return (stat(uri.to_path().c_str(), &st) == 0) && !S_ISDIR(st.st_mode); -} - -Status Posix::ls( - const std::string& path, std::vector* paths) const { - auto&& [st, entries] = ls_with_sizes(URI(path)); - - RETURN_NOT_OK(st); - - for (auto& fs : *entries) { - paths->emplace_back(fs.path().native()); - } - - return Status::Ok(); } -tuple>> Posix::ls_with_sizes( - const URI& uri) const { - std::string path = uri.to_path(); - struct dirent* next_path = nullptr; - DIR* dir = opendir(path.c_str()); - if (dir == nullptr) { - return {Status::Ok(), nullopt}; - } - - std::vector entries; - - while ((next_path = readdir(dir)) != nullptr) { - if (!strcmp(next_path->d_name, ".") || !strcmp(next_path->d_name, "..")) - continue; - std::string abspath = path + "/" + next_path->d_name; - - // Getting the file size here incurs an additional system call - // via file_size() and ls() calls will feel this too. - // If this penalty becomes noticeable, we should just duplicate - // this implementation in ls() and don't get the size - if (next_path->d_type == DT_DIR) { - entries.emplace_back(abspath, 0, true); - } else { - uint64_t size; - RETURN_NOT_OK_TUPLE(file_size(URI(abspath), &size), nullopt); - entries.emplace_back(abspath, size, false); - } - } - // close parent directory - if (closedir(dir) != 0) { - auto st = LOG_STATUS(Status_IOError( - std::string("Cannot close parent directory; ") + strerror(errno))); - return {st, nullopt}; +void Posix::move_file(const URI& old_path, const URI& new_path) const { + if (rename(old_path.to_path().c_str(), new_path.to_path().c_str()) != 0) { + throw IOError(std::string("Cannot move path: ") + strerror(errno)); } - return {Status::Ok(), entries}; } -Status Posix::move_file(const URI& old_path, const URI& new_path) const { - if (rename(old_path.to_path().c_str(), new_path.to_path().c_str()) != 0) { - return LOG_STATUS( - Status_IOError(std::string("Cannot move path: ") + strerror(errno))); - } - return Status::Ok(); +void Posix::move_dir(const URI& old_uri, const URI& new_uri) const { + move_file(old_uri, new_uri); } -Status Posix::copy_file(const URI& old_uri, const URI& new_uri) const { +void Posix::copy_file(const URI& old_uri, const URI& new_uri) const { std::ifstream src(old_uri.to_path(), std::ios::binary); std::ofstream dst(new_uri.to_path(), std::ios::binary); dst << src.rdbuf(); - return Status::Ok(); } -Status Posix::copy_dir(const URI& old_uri, const URI& new_uri) const { +void Posix::copy_dir(const URI& old_uri, const URI& new_uri) const { auto old_path = old_uri.to_path(); auto new_path = new_uri.to_path(); - RETURN_NOT_OK(create_dir(new_uri)); + create_dir(new_uri); std::vector paths; - RETURN_NOT_OK(ls(old_path, &paths)); + throw_if_not_ok(ls(old_path, &paths)); std::queue path_queue; for (auto& path : paths) @@ -333,115 +175,56 @@ Status Posix::copy_dir(const URI& old_uri, const URI& new_uri) const { path_queue.pop(); if (is_dir(URI(file_name_abs))) { - RETURN_NOT_OK(create_dir(URI(new_path + "/" + file_name))); + create_dir(URI(new_path + "/" + file_name)); std::vector child_paths; - RETURN_NOT_OK(ls(file_name_abs, &child_paths)); + throw_if_not_ok(ls(file_name_abs, &child_paths)); for (auto& path : child_paths) path_queue.emplace(std::move(path)); } else { assert(is_file(URI(file_name_abs))); - RETURN_NOT_OK(copy_file( - URI(old_path + "/" + file_name), URI(new_path + "/" + file_name))); + copy_file( + URI(old_path + "/" + file_name), URI(new_path + "/" + file_name)); } } - - return Status::Ok(); } -void Posix::purge_dots_from_path(std::string* path) { - // Trivial case - if (path == nullptr) - return; - - // Trivial case - uint64_t path_size = path->size(); - if (path_size == 0 || *path == "file:///") - return; - - assert(utils::parse::starts_with(*path, "file:///")); - - // Tokenize - const char* token_c_str = path->c_str() + 8; - std::vector tokens, final_tokens; - std::string token; - - for (uint64_t i = 8; i < path_size; ++i) { - if ((*path)[i] == '/') { - (*path)[i] = '\0'; - token = token_c_str; - if (!token.empty()) - tokens.push_back(token); - token_c_str = path->c_str() + i + 1; - } - } - token = token_c_str; - if (!token.empty()) - tokens.push_back(token); - - // Purge dots - for (auto& t : tokens) { - if (t == ".") // Skip single dots - continue; - - if (t == "..") { - if (final_tokens.empty()) { - // Invalid path - *path = ""; - return; - } - - final_tokens.pop_back(); - } else { - final_tokens.push_back(t); - } - } - - // Assemble final path - *path = "file://"; - for (auto& t : final_tokens) - *path += std::string("/") + t; -} - -Status Posix::read( +void Posix::read( const URI& uri, uint64_t offset, void* buffer, uint64_t nbytes, - [[maybe_unused]] bool use_read_ahead) { + [[maybe_unused]] bool use_read_ahead) const { // Checks auto path = uri.to_path(); uint64_t file_size; - RETURN_NOT_OK(this->file_size(URI(path), &file_size)); + this->file_size(URI(path), &file_size); if (offset + nbytes > file_size) - return LOG_STATUS( - Status_IOError("Cannot read from file; Read exceeds file size")); + throw IOError("Cannot read from file; Read exceeds file size"); // Open file int fd = open(path.c_str(), O_RDONLY); if (fd == -1) { - return LOG_STATUS(Status_IOError( - std::string("Cannot read from file; ") + strerror(errno))); + throw IOError(std::string("Cannot read from file; ") + strerror(errno)); } if (offset > static_cast(std::numeric_limits::max())) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot read from file ' ") + path.c_str() + - "'; offset > typemax(off_t)")); + "'; offset > typemax(off_t)"); } if (nbytes > SSIZE_MAX) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot read from file ' ") + path + - "'; nbytes > SSIZE_MAX")); + "'; nbytes > SSIZE_MAX"); } - Status st = read_all(fd, buffer, nbytes, offset); + throw_if_not_ok(read_all(fd, buffer, nbytes, offset)); // Close file if (close(fd)) { LOG_STATUS_NO_RETURN_VALUE( Status_IOError(std::string("Cannot close file; ") + strerror(errno))); } - return st; } -Status Posix::sync(const URI& uri) { +void Posix::sync(const URI& uri) const { auto path = uri.to_path(); // Open file @@ -450,34 +233,32 @@ Status Posix::sync(const URI& uri) { fd = open(path.c_str(), O_RDONLY, directory_permissions_); } else if (is_file(URI(path))) { // FILE fd = open(path.c_str(), O_WRONLY | O_APPEND | O_CREAT, file_permissions_); - } else - return Status_Ok(); // If file does not exist, exit + } else { + return; // If file does not exist, exit + } // Handle error if (fd == -1) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot open file '") + path + "' for syncing; " + - strerror(errno))); + strerror(errno)); } // Sync if (fsync(fd) != 0) { - return LOG_STATUS(Status_IOError( - std::string("Cannot sync file '") + path + "'; " + strerror(errno))); + throw IOError( + std::string("Cannot sync file '") + path + "'; " + strerror(errno)); } // Close file if (close(fd) != 0) { - return LOG_STATUS(Status_IOError( + throw IOError( std::string("Cannot close synced file '") + path + "'; " + - strerror(errno))); + strerror(errno)); } - - // Success - return Status::Ok(); } -Status Posix::write( +void Posix::write( const URI& uri, const void* buffer, uint64_t buffer_size, @@ -501,19 +282,14 @@ Status Posix::write( Status st; uint64_t file_offset = 0; if (is_file(URI(path))) { - st = file_size(URI(path), &file_offset); - if (!st.ok()) { - std::stringstream errmsg; - errmsg << "Cannot write to file '" << path << "'; " << st.message(); - return LOG_STATUS(Status_IOError(errmsg.str())); - } + file_size(URI(path), &file_offset); } // Open or create file. int fd = open(path.c_str(), O_WRONLY | O_CREAT, file_permissions_); if (fd == -1) { - return LOG_STATUS(Status_IOError( - std::string("Cannot open file '") + path + "'; " + strerror(errno))); + throw IOError( + std::string("Cannot open file '") + path + "'; " + strerror(errno)); } st = write_at(fd, file_offset, buffer, buffer_size); @@ -521,13 +297,218 @@ Status Posix::write( close(fd); std::stringstream errmsg; errmsg << "Cannot write to file '" << path << "'; " << st.message(); - return LOG_STATUS(Status_IOError(errmsg.str())); + throw IOError(errmsg.str()); } if (close(fd) != 0) { - return LOG_STATUS(Status_IOError( - std::string("Cannot close file '") + path + "'; " + strerror(errno))); + throw IOError( + std::string("Cannot close file '") + path + "'; " + strerror(errno)); + } +} + +tuple>> Posix::ls_with_sizes( + const URI& uri) const { + std::string path = uri.to_path(); + struct dirent* next_path = nullptr; + DIR* dir = opendir(path.c_str()); + if (dir == nullptr) { + return {Status::Ok(), nullopt}; + } + + std::vector entries; + + while ((next_path = readdir(dir)) != nullptr) { + if (!strcmp(next_path->d_name, ".") || !strcmp(next_path->d_name, "..")) + continue; + std::string abspath = path + "/" + next_path->d_name; + + // Getting the file size here incurs an additional system call + // via file_size() and ls() calls will feel this too. + // If this penalty becomes noticeable, we should just duplicate + // this implementation in ls() and don't get the size + if (next_path->d_type == DT_DIR) { + entries.emplace_back(abspath, 0, true); + } else { + uint64_t size; + file_size(URI(abspath), &size); + entries.emplace_back(abspath, size, false); + } + } + // close parent directory + if (closedir(dir) != 0) { + auto st = LOG_STATUS(Status_IOError( + std::string("Cannot close parent directory; ") + strerror(errno))); + return {st, nullopt}; } - return st; + return {Status::Ok(), entries}; +} + +Status Posix::ls( + const std::string& path, std::vector* paths) const { + auto&& [st, entries] = ls_with_sizes(URI(path)); + + RETURN_NOT_OK(st); + + for (auto& fs : *entries) { + paths->emplace_back(fs.path().native()); + } + + return Status::Ok(); +} + +std::string Posix::abs_path(const std::string& path) { + std::string resolved_path = abs_path_internal(path); + + // Ensure the returned has the same postfix slash as 'path'. + if (utils::parse::ends_with(path, "/")) { + if (!utils::parse::ends_with(resolved_path, "/")) { + resolved_path = resolved_path + "/"; + } + } else { + if (utils::parse::ends_with(resolved_path, "/")) { + resolved_path = resolved_path.substr(0, resolved_path.length() - 1); + } + } + + return resolved_path; +} + +std::string Posix::current_dir() { + static std::unique_ptr cwd_(getcwd(nullptr, 0), free); + std::string dir = cwd_.get(); + return dir; +} + +void Posix::adjacent_slashes_dedup(std::string* path) { + assert(utils::parse::starts_with(*path, "file://")); + path->erase( + std::unique( + path->begin() + std::string("file://").size(), + path->end(), + both_slashes), + path->end()); +} + +bool Posix::both_slashes(char a, char b) { + return a == '/' && b == '/'; +} + +std::string Posix::abs_path_internal(const std::string& path) { + // Initialize current, home and root + std::string current = current_dir(); + auto env_home_ptr = getenv("HOME"); + std::string home = env_home_ptr != nullptr ? env_home_ptr : current; + std::string root = "/"; + std::string posix_prefix = "file://"; + + // Easy cases + if (path.empty() || path == "." || path == "./") + return posix_prefix + current; + if (path == "~") + return posix_prefix + home; + if (path == "/") + return posix_prefix + root; + + // Other cases + std::string ret_dir; + if (utils::parse::starts_with(path, posix_prefix)) + return path; + else if (utils::parse::starts_with(path, "/")) + ret_dir = posix_prefix + path; + else if (utils::parse::starts_with(path, "~/")) + ret_dir = posix_prefix + home + path.substr(1, path.size() - 1); + else if (utils::parse::starts_with(path, "./")) + ret_dir = posix_prefix + current + path.substr(1, path.size() - 1); + else + ret_dir = posix_prefix + current + "/" + path; + + adjacent_slashes_dedup(&ret_dir); + purge_dots_from_path(&ret_dir); + + return ret_dir; +} + +void Posix::purge_dots_from_path(std::string* path) { + // Trivial case + if (path == nullptr) + return; + + // Trivial case + uint64_t path_size = path->size(); + if (path_size == 0 || *path == "file:///") + return; + + assert(utils::parse::starts_with(*path, "file:///")); + + // Tokenize + const char* token_c_str = path->c_str() + 8; + std::vector tokens, final_tokens; + std::string token; + + for (uint64_t i = 8; i < path_size; ++i) { + if ((*path)[i] == '/') { + (*path)[i] = '\0'; + token = token_c_str; + if (!token.empty()) + tokens.push_back(token); + token_c_str = path->c_str() + i + 1; + } + } + token = token_c_str; + if (!token.empty()) + tokens.push_back(token); + + // Purge dots + for (auto& t : tokens) { + if (t == ".") // Skip single dots + continue; + + if (t == "..") { + if (final_tokens.empty()) { + // Invalid path + *path = ""; + return; + } + + final_tokens.pop_back(); + } else { + final_tokens.push_back(t); + } + } + + // Assemble final path + *path = "file://"; + for (auto& t : final_tokens) + *path += std::string("/") + t; +} + +Status Posix::read_all(int fd, void* buffer, uint64_t nbytes, uint64_t offset) { + auto bytes = reinterpret_cast(buffer); + uint64_t nread = 0; + do { + ssize_t actual_read = + ::pread(fd, bytes + nread, nbytes - nread, offset + nread); + if (actual_read < 0) { + return LOG_STATUS( + Status_IOError(std::string("POSIX read error: ") + strerror(errno))); + } else if (actual_read == 0) { + break; + } + nread += actual_read; + } while (nread < nbytes); + + if (nread != nbytes) { + return LOG_STATUS(Status_IOError("POSIX incomplete read: EOF reached")); + } + return Status::Ok(); +} + +// TODO: it maybe better to use unlinkat for deeply nested recursive directories +// but the path name length limit in TileDB may make this unnecessary +int Posix::unlink_cb(const char* fpath, const struct stat*, int, struct FTW*) { + int rc = remove(fpath); + if (rc) + perror(fpath); + return rc; } Status Posix::write_at( @@ -547,7 +528,6 @@ Status Posix::write_at( return Status::Ok(); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // !_WIN32 diff --git a/tiledb/sm/filesystem/posix.h b/tiledb/sm/filesystem/posix.h index c8399980f3d7..958a4df51a1d 100644 --- a/tiledb/sm/filesystem/posix.h +++ b/tiledb/sm/filesystem/posix.h @@ -74,127 +74,93 @@ class Posix : public FilesystemBase { /** Destructor. */ ~Posix() override = default; - /** - * Returns the absolute posix (string) path of the input in the - * form "file://" - */ - static std::string abs_path(const std::string& path); - /** * Creates a new directory. * * @param dir The name of the directory to be created. - * @return Status */ - Status create_dir(const URI& uri) const override; + void create_dir(const URI& uri) const override; /** * Creates an empty file. * * @param filename The name of the file to be created. - * @return Status */ - Status touch(const URI& uri) const override; + void touch(const URI& uri) const override; /** - * Returns the directory where the program is executed. + * Checks if the input is an existing directory. * - * @return The directory path where the program is executed. If the program - * cannot retrieve the current working directory, the empty string is - * returned. + * @param dir The directory to be checked. + * @return *True* if *dir* is an existing directory, and *False* otherwise. */ - static std::string current_dir(); + bool is_dir(const URI& uri) const override; + + /** + * Checks if the input is an existing file. + * + * @param file The file to be checked. + * @return *True* if *file* is an existing file, and *false* otherwise. + */ + bool is_file(const URI& uri) const override; /** * Removes a given directory recursively. * * @param path The path of the directory to be deleted. - * @return Status */ - Status remove_dir(const URI& path) const override; + void remove_dir(const URI& path) const override; /** * Removes a given path. * * @param path The path of the file / empty directory to be deleted. - * @return Status */ - Status remove_file(const URI& path) const override; + void remove_file(const URI& path) const override; /** * Returns the size of the input file. * * @param path The name of the file whose size is to be retrieved. * @param nbytes Pointer to a value - * @return Status - */ - Status file_size(const URI& path, uint64_t* size) const override; - - /** - * Checks if the input is an existing directory. - * - * @param dir The directory to be checked. - * @return *True* if *dir* is an existing directory, and *False* otherwise. - */ - bool is_dir(const URI& uri) const override; - - /** - * Checks if the input is an existing file. - * - * @param file The file to be checked. - * @return *True* if *file* is an existing file, and *false* otherwise. - */ - bool is_file(const URI& uri) const override; - - /** - * - * Lists files one level deep under a given path. - * - * @param path The parent path to list sub-paths. - * @param paths Pointer to a vector of strings to store the retrieved paths. - * @return Status */ - Status ls(const std::string& path, std::vector* paths) const; + void file_size(const URI& path, uint64_t* size) const override; /** + * Move a given filesystem path. + * Both URI must be of the same file:// backend type. * - * Lists files and file information one level deep under a given path. - * - * @param uri The parent path to list sub-paths. - * @return A list of directory_entry objects + * @param old_uri The old URI. + * @param new_uri The new URI. */ - tuple>> - ls_with_sizes(const URI& uri) const override; + void move_file(const URI& old_uri, const URI& new_uri) const override; /** - * Move a given filesystem path. + * Renames a directory. * Both URI must be of the same file:// backend type. * - * @param old_path The old path. - * @param new_path The new path. - * @return Status + * @param old_uri The old URI. + * @param new_uri The new URI. */ - Status move_file(const URI& old_path, const URI& new_path) const override; + void move_dir(const URI& old_uri, const URI& new_uri) const override; /** * Copy a given filesystem file. * Both URI must be of the same file:// backend type. * - * @param old_path The old path. - * @param new_path The new path. - * @return Status + * @param old_uri The old URI. + * @param new_uri The new URI. */ - Status copy_file(const URI& old_uri, const URI& new_uri) const override; + void copy_file(const URI& old_uri, const URI& new_uri) const override; /** * Copy a given filesystem directory. * Both URI must be of the same file:// backend type. * - * @param old_path The old path. - * @param new_path The new path. - * @return Status + * @param old_uri The old URI. + * @param new_uri The new URI. */ - Status copy_dir(const URI& old_path, const URI& new_path) const override; + void copy_dir(const URI& old_uri, const URI& new_uri) const override; /** * Reads data from a file into a buffer. @@ -203,22 +169,20 @@ class Posix : public FilesystemBase { * @param offset The offset in the file from which the read will start. * @param buffer The buffer into which the data will be written. * @param nbytes The size of the data to be read from the file. - * @return Status. */ - Status read( + void read( const URI& uri, uint64_t offset, void* buffer, uint64_t nbytes, - bool use_read_ahead = true) override; + bool use_read_ahead = true) const override; /** * Syncs a file or directory. * * @param path The name of the file. - * @return Status */ - Status sync(const URI& uri) override; + void sync(const URI& uri) const override; /** * Writes the input buffer to a file. @@ -229,14 +193,96 @@ class Posix : public FilesystemBase { * @param path The name of the file. * @param buffer The input buffer. * @param buffer_size The size of the input buffer. - * @return Status */ - Status write( + void write( const URI& uri, const void* buffer, uint64_t buffer_size, bool remote_global_order_write = false) override; + /** + * Checks if an object store bucket exists. + * + * @param uri The name of the object store bucket. + * @return True if the bucket exists, false otherwise. + */ + bool is_bucket(const URI&) const override { + // No concept of buckets for Posix. + return false; + } + + /** + * Checks if an object-store bucket is empty. + * + * @param uri The name of the object store bucket. + * @return True if the bucket is empty, false otherwise. + */ + bool is_empty_bucket(const URI&) const override { + // No concept of buckets for Posix. + return true; + } + + /** + * Creates an object store bucket. + * + * @param uri The name of the bucket to be created. + */ + void create_bucket(const URI&) const override { + // No-op for Posix, stub function for cloud filesystems. + } + + /** + * Deletes an object store bucket. + * + * @param uri The name of the bucket to be deleted. + */ + void remove_bucket(const URI&) const override { + // No-op for Posix, stub function for cloud filesystems. + } + + /** + * Deletes the contents of an object store bucket. + * + * @param uri The name of the bucket to be emptied. + */ + void empty_bucket(const URI&) const override { + // No-op for Posix, stub function for cloud filesystems. + } + + /** + * + * Lists files and file information one level deep under a given path. + * + * @param uri The parent path to list sub-paths. + * @return A list of directory_entry objects + */ + tuple>> + ls_with_sizes(const URI& uri) const override; + + /** + * Lists files one level deep under a given path. + * + * @param path The parent path to list sub-paths. + * @param paths Pointer to a vector of strings to store the retrieved paths. + * @return Status + */ + Status ls(const std::string& path, std::vector* paths) const; + + /** + * Returns the absolute posix (string) path of the input in the + * form "file://" + */ + static std::string abs_path(const std::string& path); + + /** + * Returns the directory where the program is executed. + * + * @return The directory path where the program is executed. If the program + * cannot retrieve the current working directory, the empty string is + * returned. + */ + static std::string current_dir(); + private: static void adjacent_slashes_dedup(std::string* path); diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index c4611d796c39..b0433752cf57 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -280,12 +280,12 @@ S3::~S3() { /* API */ /* ********************************* */ -Status S3::create_bucket(const URI& bucket) const { - RETURN_NOT_OK(init_client()); +void S3::create_bucket(const URI& bucket) const { + throw_if_not_ok(init_client()); if (!bucket.is_s3()) { - return LOG_STATUS(Status_S3Error( - std::string("URI is not an S3 URI: " + bucket.to_string()))); + throw S3Exception( + std::string("URI is not an S3 URI: " + bucket.to_string())); } Aws::Http::URI aws_uri = bucket.c_str(); @@ -309,21 +309,122 @@ Status S3::create_bucket(const URI& bucket) const { auto create_bucket_outcome = client_->CreateBucket(create_bucket_request); if (!create_bucket_outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( + throw S3Exception( std::string("Failed to create S3 bucket ") + bucket.to_string() + - outcome_error_message(create_bucket_outcome))); + outcome_error_message(create_bucket_outcome)); } - RETURN_NOT_OK(wait_for_bucket_to_be_created(bucket)); + throw_if_not_ok(wait_for_bucket_to_be_created(bucket)); +} - return Status::Ok(); +void S3::empty_bucket(const URI& bucket) const { + throw_if_not_ok(init_client()); + + auto uri_dir = bucket.add_trailing_slash(); + remove_dir(uri_dir); } -Status S3::remove_bucket(const URI& bucket) const { - RETURN_NOT_OK(init_client()); +bool S3::is_empty_bucket(const URI& bucket) const { + throw_if_not_ok(init_client()); + + bool exists = is_bucket(bucket); + if (!exists) { + throw S3Exception("Cannot check if bucket is empty; Bucket does not exist"); + } + + Aws::Http::URI aws_uri = bucket.c_str(); + Aws::S3::Model::ListObjectsV2Request list_objects_request; + list_objects_request.SetBucket(aws_uri.GetAuthority()); + list_objects_request.SetPrefix(""); + list_objects_request.SetDelimiter("/"); + if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) { + list_objects_request.SetRequestPayer(request_payer_); + } + auto list_objects_outcome = client_->ListObjectsV2(list_objects_request); + + if (!list_objects_outcome.IsSuccess()) { + throw S3Exception( + std::string("Failed to list s3 objects in bucket ") + bucket.c_str() + + outcome_error_message(list_objects_outcome)); + } + + return list_objects_outcome.GetResult().GetContents().empty() && + list_objects_outcome.GetResult().GetCommonPrefixes().empty(); +} + +bool S3::is_bucket(const URI& uri) const { + throw_if_not_ok(init_client()); + + if (!uri.is_s3()) { + throw S3Exception(std::string("URI is not an S3 URI: " + uri.to_string())); + } + + Aws::Http::URI aws_uri = uri.c_str(); + Aws::S3::Model::HeadBucketRequest head_bucket_request; + head_bucket_request.SetBucket(aws_uri.GetAuthority()); + auto head_bucket_outcome = client_->HeadBucket(head_bucket_request); + return head_bucket_outcome.IsSuccess(); +} + +void S3::move_dir(const URI& old_uri, const URI& new_uri) const { + throw_if_not_ok(init_client()); + + std::vector paths; + throw_if_not_ok(ls(old_uri, &paths, "")); + for (const auto& path : paths) { + auto suffix = path.substr(old_uri.to_string().size()); + auto new_path = new_uri.join_path(suffix); + throw_if_not_ok(move_object(URI(path), URI(new_path))); + } +} + +void S3::copy_file(const URI& old_uri, const URI& new_uri) const { + throw_if_not_ok(init_client()); + + throw_if_not_ok(copy_object(old_uri, new_uri)); +} + +void S3::copy_dir(const URI& old_uri, const URI& new_uri) const { + throw_if_not_ok(init_client()); + + std::string old_uri_string = old_uri.to_string(); + std::vector paths; + throw_if_not_ok(ls(old_uri, &paths)); + while (!paths.empty()) { + std::string file_name_abs = paths.front(); + URI file_name_uri = URI(file_name_abs); + std::string file_name = file_name_abs.substr(old_uri_string.length()); + paths.erase(paths.begin()); + + bool dir_exists; + throw_if_not_ok(is_dir(file_name_uri, &dir_exists)); + if (dir_exists) { + std::vector child_paths; + throw_if_not_ok(ls(file_name_uri, &child_paths)); + paths.insert(paths.end(), child_paths.begin(), child_paths.end()); + } else { + std::string new_path_string = new_uri.to_string() + file_name; + URI new_path_uri = URI(new_path_string); + throw_if_not_ok(copy_object(file_name_uri, new_path_uri)); + } + } +} + +void S3::read( + const URI& uri, + uint64_t offset, + void* buffer, + uint64_t nbytes, + bool) const { + uint64_t nbytes_read = 0; + throw_if_not_ok(read_impl(uri, offset, buffer, nbytes, 0, &nbytes_read)); +} + +void S3::remove_bucket(const URI& bucket) const { + throw_if_not_ok(init_client()); // Empty bucket - RETURN_NOT_OK(empty_bucket(bucket)); + empty_bucket(bucket); // Delete bucket Aws::Http::URI aws_uri = bucket.c_str(); @@ -331,11 +432,138 @@ Status S3::remove_bucket(const URI& bucket) const { delete_bucket_request.SetBucket(aws_uri.GetAuthority()); auto delete_bucket_outcome = client_->DeleteBucket(delete_bucket_request); if (!delete_bucket_outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( + throw S3Exception( std::string("Failed to remove S3 bucket ") + bucket.to_string() + - outcome_error_message(delete_bucket_outcome))); + outcome_error_message(delete_bucket_outcome)); } - return Status::Ok(); +} + +void S3::remove_dir(const URI& uri) const { + throw_if_not_ok(init_client()); + + std::vector paths; + throw_if_not_ok(ls(uri, &paths, "")); + throw_if_not_ok( + parallel_for(vfs_thread_pool_, 0, paths.size(), [&](size_t i) { + throw_if_not_ok(remove_object(URI(paths[i]))); + return Status::Ok(); + })); +} + +void S3::touch(const URI& uri) const { + throw_if_not_ok(init_client()); + + if (!uri.is_s3()) { + throw S3Exception(std::string( + "Cannot create file; URI is not an S3 URI: " + uri.to_string())); + } + + if (uri.to_string().back() == '/') { + throw S3Exception(std::string( + "Cannot create file; URI is a directory: " + uri.to_string())); + } + + bool exists; + throw_if_not_ok(is_object(uri, &exists)); + if (exists) { + return; + } + + Aws::Http::URI aws_uri = uri.c_str(); + Aws::S3::Model::PutObjectRequest put_object_request; + put_object_request.WithKey(aws_uri.GetPath()) + .WithBucket(aws_uri.GetAuthority()); + + auto request_stream = + Aws::MakeShared(constants::s3_allocation_tag.c_str()); + put_object_request.SetBody(request_stream); + if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) + put_object_request.SetRequestPayer(request_payer_); + if (sse_ != Aws::S3::Model::ServerSideEncryption::NOT_SET) + put_object_request.SetServerSideEncryption(sse_); + if (!s3_params_.sse_kms_key_id_.empty()) + put_object_request.SetSSEKMSKeyId( + Aws::String(s3_params_.sse_kms_key_id_.c_str())); + if (object_canned_acl_ != Aws::S3::Model::ObjectCannedACL::NOT_SET) { + put_object_request.SetACL(object_canned_acl_); + } + + auto put_object_outcome = client_->PutObject(put_object_request); + if (!put_object_outcome.IsSuccess()) { + throw S3Exception( + std::string("Cannot touch object '") + uri.c_str() + + outcome_error_message(put_object_outcome)); + } + + throw_if_not_ok(wait_for_object_to_propagate( + put_object_request.GetBucket(), put_object_request.GetKey())); +} + +void S3::write( + const URI& uri, + const void* buffer, + uint64_t length, + bool remote_global_order_write) { + throw_if_not_ok(init_client()); + + if (!uri.is_s3()) { + throw S3Exception(std::string("URI is not an S3 URI: " + uri.to_string())); + } + + if (remote_global_order_write) { + global_order_write_buffered(uri, buffer, length); + return; + } + + // This write is never considered the last part of an object. The last part is + // only uploaded with flush_object(). + const bool is_last_part = false; + + // Get file buffer + auto buff = (Buffer*)nullptr; + throw_if_not_ok(get_file_buffer(uri, &buff)); + + // Fill file buffer + uint64_t nbytes_filled; + throw_if_not_ok(fill_file_buffer(buff, buffer, length, &nbytes_filled)); + + if ((!s3_params_.use_multipart_upload_) && (nbytes_filled != length)) { + std::stringstream errmsg; + errmsg << "Direct write failed! " << nbytes_filled + << " bytes written to buffer, " << length << " bytes requested."; + throw S3Exception(errmsg.str()); + } + + // Flush file buffer + // multipart objects will flush whenever the writes exceed file_buffer_size_ + // write_direct should just append to buffer and upload later + if (s3_params_.use_multipart_upload_) { + if (buff->size() == file_buffer_size_) + throw_if_not_ok(flush_file_buffer(uri, buff, is_last_part)); + + uint64_t new_length = length - nbytes_filled; + uint64_t offset = nbytes_filled; + // Write chunks + while (new_length > 0) { + if (new_length >= file_buffer_size_) { + throw_if_not_ok(write_multipart( + uri, (char*)buffer + offset, file_buffer_size_, is_last_part)); + offset += file_buffer_size_; + new_length -= file_buffer_size_; + } else { + throw_if_not_ok(fill_file_buffer( + buff, (char*)buffer + offset, new_length, &nbytes_filled)); + offset += nbytes_filled; + new_length -= nbytes_filled; + } + } + assert(offset == length); + } +} + +tuple>> S3::ls_with_sizes( + const URI& parent) const { + return ls_with_sizes(parent, "/", -1); } Status S3::disconnect() { @@ -407,13 +635,6 @@ Status S3::disconnect() { return ret_st; } -Status S3::empty_bucket(const URI& bucket) const { - RETURN_NOT_OK(init_client()); - - auto uri_dir = bucket.add_trailing_slash(); - return remove_dir(uri_dir); -} - Status S3::flush_object(const URI& uri) { RETURN_NOT_OK(init_client()); if (!s3_params_.use_multipart_upload_) { @@ -519,7 +740,7 @@ void S3::finalize_and_flush_object(const URI& uri) { throw_if_not_ok(parallel_for( vfs_thread_pool_, 0, intermediate_chunks.size(), [&](size_t i) { uint64_t length_returned; - throw_if_not_ok(read( + throw_if_not_ok(read_impl( URI(intermediate_chunks[i].uri), 0, merged.data() + offsets[i], @@ -571,106 +792,14 @@ void S3::finalize_and_flush_object(const URI& uri) { unique_wl.unlock(); } -Aws::S3::Model::CompleteMultipartUploadRequest -S3::make_multipart_complete_request(const MultiPartUploadState& state) { - // Add all the completed parts (sorted by part number) to the upload object. - Aws::S3::Model::CompletedMultipartUpload completed_upload; - for (auto& tup : state.completed_parts) { - const Aws::S3::Model::CompletedPart& part = std::get<1>(tup); - completed_upload.AddParts(part); - } - - Aws::S3::Model::CompleteMultipartUploadRequest complete_request; - complete_request.SetBucket(state.bucket); - complete_request.SetKey(state.key); - complete_request.SetUploadId(state.upload_id); - if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) - complete_request.SetRequestPayer(request_payer_); - return complete_request.WithMultipartUpload(std::move(completed_upload)); -} - -Aws::S3::Model::AbortMultipartUploadRequest S3::make_multipart_abort_request( - const MultiPartUploadState& state) { - Aws::S3::Model::AbortMultipartUploadRequest abort_request; - abort_request.SetBucket(state.bucket); - abort_request.SetKey(state.key); - abort_request.SetUploadId(state.upload_id); - if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) - abort_request.SetRequestPayer(request_payer_); - return abort_request; -} - -template -Status S3::finish_flush_object( - const Aws::Utils::Outcome& outcome, - const URI& uri, - Buffer* const buff) { - Aws::Http::URI aws_uri = uri.c_str(); - - UniqueWriteLock unique_wl(&multipart_upload_rwlock_); - multipart_upload_states_.erase(aws_uri.GetPath().c_str()); - unique_wl.unlock(); - - std::unique_lock file_buffers_lck(file_buffers_mtx_); - file_buffers_.erase(uri.to_string()); - file_buffers_lck.unlock(); - tdb_delete(buff); - - if (!outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( - std::string("Failed to flush S3 object ") + uri.c_str() + - outcome_error_message(outcome))); - } - - return Status::Ok(); -} - -Status S3::is_empty_bucket(const URI& bucket, bool* is_empty) const { +Status S3::is_dir(const URI& uri, bool* exists) const { RETURN_NOT_OK(init_client()); - bool exists; - RETURN_NOT_OK(is_bucket(bucket, &exists)); - if (!exists) { - return LOG_STATUS(Status_S3Error( - "Cannot check if bucket is empty; Bucket does not exist")); - } - - Aws::Http::URI aws_uri = bucket.c_str(); - Aws::S3::Model::ListObjectsV2Request list_objects_request; - list_objects_request.SetBucket(aws_uri.GetAuthority()); - list_objects_request.SetPrefix(""); - list_objects_request.SetDelimiter("/"); - if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) { - list_objects_request.SetRequestPayer(request_payer_); - } - auto list_objects_outcome = client_->ListObjectsV2(list_objects_request); - - if (!list_objects_outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( - std::string("Failed to list s3 objects in bucket ") + bucket.c_str() + - outcome_error_message(list_objects_outcome))); - } - - *is_empty = list_objects_outcome.GetResult().GetContents().empty() && - list_objects_outcome.GetResult().GetCommonPrefixes().empty(); - - return Status::Ok(); -} - -Status S3::is_bucket(const URI& uri, bool* const exists) const { - throw_if_not_ok(init_client()); - - if (!uri.is_s3()) { - return LOG_STATUS(Status_S3Error( - std::string("URI is not an S3 URI: " + uri.to_string()))); - } - - Aws::Http::URI aws_uri = uri.c_str(); - Aws::S3::Model::HeadBucketRequest head_bucket_request; - head_bucket_request.SetBucket(aws_uri.GetAuthority()); - auto head_bucket_outcome = client_->HeadBucket(head_bucket_request); - *exists = head_bucket_outcome.IsSuccess(); - + // Potentially add `/` to the end of `uri` + auto uri_dir = uri.add_trailing_slash(); + std::vector paths; + RETURN_NOT_OK(ls(uri_dir, &paths, "/", 1)); + *exists = (bool)paths.size(); return Status::Ok(); } @@ -704,17 +833,6 @@ Status S3::is_object( return Status::Ok(); } -Status S3::is_dir(const URI& uri, bool* exists) const { - RETURN_NOT_OK(init_client()); - - // Potentially add `/` to the end of `uri` - auto uri_dir = uri.add_trailing_slash(); - std::vector paths; - RETURN_NOT_OK(ls(uri_dir, &paths, "/", 1)); - *exists = (bool)paths.size(); - return Status::Ok(); -} - Status S3::ls( const URI& prefix, std::vector* paths, @@ -796,74 +914,25 @@ tuple>> S3::ls_with_sizes( (max_paths != -1 && entries.size() >= static_cast(max_paths)); if (!is_done) { Aws::String next_marker = - list_objects_outcome.GetResult().GetNextContinuationToken(); - if (next_marker.empty()) { - auto st = - LOG_STATUS(Status_S3Error("Failed to retrieve next continuation " - "token for ListObjectsV2 request.")); - return {st, nullopt}; - } - list_objects_request.SetContinuationToken(std::move(next_marker)); - } - } - - return {Status::Ok(), entries}; -} - -Status S3::move_object(const URI& old_uri, const URI& new_uri) { - RETURN_NOT_OK(init_client()); - - RETURN_NOT_OK(copy_object(old_uri, new_uri)); - RETURN_NOT_OK(remove_object(old_uri)); - return Status::Ok(); -} - -Status S3::move_dir(const URI& old_uri, const URI& new_uri) { - RETURN_NOT_OK(init_client()); - - std::vector paths; - RETURN_NOT_OK(ls(old_uri, &paths, "")); - for (const auto& path : paths) { - auto suffix = path.substr(old_uri.to_string().size()); - auto new_path = new_uri.join_path(suffix); - RETURN_NOT_OK(move_object(URI(path), URI(new_path))); + list_objects_outcome.GetResult().GetNextContinuationToken(); + if (next_marker.empty()) { + auto st = + LOG_STATUS(Status_S3Error("Failed to retrieve next continuation " + "token for ListObjectsV2 request.")); + return {st, nullopt}; + } + list_objects_request.SetContinuationToken(std::move(next_marker)); + } } - return Status::Ok(); + return {Status::Ok(), entries}; } -Status S3::copy_file(const URI& old_uri, const URI& new_uri) { +Status S3::move_object(const URI& old_uri, const URI& new_uri) const { RETURN_NOT_OK(init_client()); RETURN_NOT_OK(copy_object(old_uri, new_uri)); - return Status::Ok(); -} - -Status S3::copy_dir(const URI& old_uri, const URI& new_uri) { - RETURN_NOT_OK(init_client()); - - std::string old_uri_string = old_uri.to_string(); - std::vector paths; - RETURN_NOT_OK(ls(old_uri, &paths)); - while (!paths.empty()) { - std::string file_name_abs = paths.front(); - URI file_name_uri = URI(file_name_abs); - std::string file_name = file_name_abs.substr(old_uri_string.length()); - paths.erase(paths.begin()); - - bool dir_exists; - RETURN_NOT_OK(is_dir(file_name_uri, &dir_exists)); - if (dir_exists) { - std::vector child_paths; - RETURN_NOT_OK(ls(file_name_uri, &child_paths)); - paths.insert(paths.end(), child_paths.begin(), child_paths.end()); - } else { - std::string new_path_string = new_uri.to_string() + file_name; - URI new_path_uri = URI(new_path_string); - RETURN_NOT_OK(copy_object(file_name_uri, new_path_uri)); - } - } - + RETURN_NOT_OK(remove_object(old_uri)); return Status::Ok(); } @@ -895,7 +964,7 @@ Status S3::object_size(const URI& uri, uint64_t* nbytes) const { return Status::Ok(); } -Status S3::read( +Status S3::read_impl( const URI& uri, const off_t offset, void* const buffer, @@ -973,127 +1042,6 @@ Status S3::remove_object(const URI& uri) const { return Status::Ok(); } -Status S3::remove_dir(const URI& uri) const { - RETURN_NOT_OK(init_client()); - - std::vector paths; - RETURN_NOT_OK(ls(uri, &paths, "")); - auto status = parallel_for(vfs_thread_pool_, 0, paths.size(), [&](size_t i) { - RETURN_NOT_OK(remove_object(URI(paths[i]))); - return Status::Ok(); - }); - RETURN_NOT_OK(status); - - return Status::Ok(); -} - -Status S3::touch(const URI& uri) const { - RETURN_NOT_OK(init_client()); - - if (!uri.is_s3()) { - return LOG_STATUS(Status_S3Error(std::string( - "Cannot create file; URI is not an S3 URI: " + uri.to_string()))); - } - - if (uri.to_string().back() == '/') { - return LOG_STATUS(Status_S3Error(std::string( - "Cannot create file; URI is a directory: " + uri.to_string()))); - } - - bool exists; - RETURN_NOT_OK(is_object(uri, &exists)); - if (exists) { - return Status::Ok(); - } - - Aws::Http::URI aws_uri = uri.c_str(); - Aws::S3::Model::PutObjectRequest put_object_request; - put_object_request.WithKey(aws_uri.GetPath()) - .WithBucket(aws_uri.GetAuthority()); - - auto request_stream = - Aws::MakeShared(constants::s3_allocation_tag.c_str()); - put_object_request.SetBody(request_stream); - if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) - put_object_request.SetRequestPayer(request_payer_); - if (sse_ != Aws::S3::Model::ServerSideEncryption::NOT_SET) - put_object_request.SetServerSideEncryption(sse_); - if (!s3_params_.sse_kms_key_id_.empty()) - put_object_request.SetSSEKMSKeyId( - Aws::String(s3_params_.sse_kms_key_id_.c_str())); - if (object_canned_acl_ != Aws::S3::Model::ObjectCannedACL::NOT_SET) { - put_object_request.SetACL(object_canned_acl_); - } - - auto put_object_outcome = client_->PutObject(put_object_request); - if (!put_object_outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( - std::string("Cannot touch object '") + uri.c_str() + - outcome_error_message(put_object_outcome))); - } - - throw_if_not_ok(wait_for_object_to_propagate( - put_object_request.GetBucket(), put_object_request.GetKey())); - - return Status::Ok(); -} - -Status S3::write(const URI& uri, const void* buffer, uint64_t length) { - RETURN_NOT_OK(init_client()); - - if (!uri.is_s3()) { - return LOG_STATUS(Status_S3Error( - std::string("URI is not an S3 URI: " + uri.to_string()))); - } - - // This write is never considered the last part of an object. The last part is - // only uploaded with flush_object(). - const bool is_last_part = false; - - // Get file buffer - auto buff = (Buffer*)nullptr; - RETURN_NOT_OK(get_file_buffer(uri, &buff)); - - // Fill file buffer - uint64_t nbytes_filled; - RETURN_NOT_OK(fill_file_buffer(buff, buffer, length, &nbytes_filled)); - - if ((!s3_params_.use_multipart_upload_) && (nbytes_filled != length)) { - std::stringstream errmsg; - errmsg << "Direct write failed! " << nbytes_filled - << " bytes written to buffer, " << length << " bytes requested."; - return LOG_STATUS(Status_S3Error(errmsg.str())); - } - - // Flush file buffer - // multipart objects will flush whenever the writes exceed file_buffer_size_ - // write_direct should just append to buffer and upload later - if (s3_params_.use_multipart_upload_) { - if (buff->size() == file_buffer_size_) - RETURN_NOT_OK(flush_file_buffer(uri, buff, is_last_part)); - - uint64_t new_length = length - nbytes_filled; - uint64_t offset = nbytes_filled; - // Write chunks - while (new_length > 0) { - if (new_length >= file_buffer_size_) { - RETURN_NOT_OK(write_multipart( - uri, (char*)buffer + offset, file_buffer_size_, is_last_part)); - offset += file_buffer_size_; - new_length -= file_buffer_size_; - } else { - RETURN_NOT_OK(fill_file_buffer( - buff, (char*)buffer + offset, new_length, &nbytes_filled)); - offset += nbytes_filled; - new_length -= nbytes_filled; - } - } - assert(offset == length); - } - - return Status::Ok(); -} - void S3::global_order_write_buffered( const URI& uri, const void* buffer, uint64_t length) { throw_if_not_ok(init_client()); @@ -1198,7 +1146,7 @@ void S3::global_order_write( throw_if_not_ok(parallel_for( vfs_thread_pool_, 0, intermediate_chunks.size(), [&](size_t i) { uint64_t length_returned; - throw_if_not_ok(read( + throw_if_not_ok(read_impl( URI(intermediate_chunks[i].uri), 0, merged.data() + offsets[i], @@ -1269,6 +1217,24 @@ void S3::global_order_write( intermediate_chunks.clear(); } +std::string S3::add_front_slash(const std::string& path) { + return (path.front() != '/') ? (std::string("/") + path) : path; +} + +std::string S3::remove_front_slash(const std::string& path) { + if (path.front() == '/') + return path.substr(1, path.length()); + return path; +} + +std::string S3::remove_trailing_slash(const std::string& path) { + if (path.back() == '/') { + return path.substr(0, path.length() - 1); + } + + return path; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ @@ -1472,7 +1438,7 @@ Status S3::init_client() const { return Status::Ok(); } -Status S3::copy_object(const URI& old_uri, const URI& new_uri) { +Status S3::copy_object(const URI& old_uri, const URI& new_uri) const { RETURN_NOT_OK(init_client()); Aws::Http::URI src_uri = old_uri.c_str(); @@ -1520,24 +1486,6 @@ Status S3::fill_file_buffer( return Status::Ok(); } -std::string S3::add_front_slash(const std::string& path) { - return (path.front() != '/') ? (std::string("/") + path) : path; -} - -std::string S3::remove_front_slash(const std::string& path) { - if (path.front() == '/') - return path.substr(1, path.length()); - return path; -} - -std::string S3::remove_trailing_slash(const std::string& path) { - if (path.back() == '/') { - return path.substr(0, path.length() - 1); - } - - return path; -} - Status S3::flush_file_buffer(const URI& uri, Buffer* buff, bool last_part) { RETURN_NOT_OK(init_client()); if (buff->size() > 0) { @@ -1667,8 +1615,7 @@ Status S3::wait_for_bucket_to_be_created(const URI& bucket_uri) const { unsigned attempts_cnt = 0; while (attempts_cnt++ < constants::s3_max_attempts) { - bool exists; - RETURN_NOT_OK(is_bucket(bucket_uri, &exists)); + bool exists = is_bucket(bucket_uri); if (exists) { return Status::Ok(); } @@ -1682,6 +1629,60 @@ Status S3::wait_for_bucket_to_be_created(const URI& bucket_uri) const { " to be created.")); } +Aws::S3::Model::CompleteMultipartUploadRequest +S3::make_multipart_complete_request(const MultiPartUploadState& state) { + // Add all the completed parts (sorted by part number) to the upload object. + Aws::S3::Model::CompletedMultipartUpload completed_upload; + for (auto& tup : state.completed_parts) { + const Aws::S3::Model::CompletedPart& part = std::get<1>(tup); + completed_upload.AddParts(part); + } + + Aws::S3::Model::CompleteMultipartUploadRequest complete_request; + complete_request.SetBucket(state.bucket); + complete_request.SetKey(state.key); + complete_request.SetUploadId(state.upload_id); + if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) + complete_request.SetRequestPayer(request_payer_); + return complete_request.WithMultipartUpload(std::move(completed_upload)); +} + +Aws::S3::Model::AbortMultipartUploadRequest S3::make_multipart_abort_request( + const MultiPartUploadState& state) { + Aws::S3::Model::AbortMultipartUploadRequest abort_request; + abort_request.SetBucket(state.bucket); + abort_request.SetKey(state.key); + abort_request.SetUploadId(state.upload_id); + if (request_payer_ != Aws::S3::Model::RequestPayer::NOT_SET) + abort_request.SetRequestPayer(request_payer_); + return abort_request; +} + +template +Status S3::finish_flush_object( + const Aws::Utils::Outcome& outcome, + const URI& uri, + Buffer* const buff) { + Aws::Http::URI aws_uri = uri.c_str(); + + UniqueWriteLock unique_wl(&multipart_upload_rwlock_); + multipart_upload_states_.erase(aws_uri.GetPath().c_str()); + unique_wl.unlock(); + + std::unique_lock file_buffers_lck(file_buffers_mtx_); + file_buffers_.erase(uri.to_string()); + file_buffers_lck.unlock(); + tdb_delete(buff); + + if (!outcome.IsSuccess()) { + return LOG_STATUS(Status_S3Error( + std::string("Failed to flush S3 object ") + uri.c_str() + + outcome_error_message(outcome))); + } + + return Status::Ok(); +} + Status S3::flush_direct(const URI& uri) { RETURN_NOT_OK(init_client()); @@ -1947,6 +1948,20 @@ Status S3::get_make_upload_part_req( return Status::Ok(); } +Status S3::set_multipart_upload_state( + const std::string& uri, MultiPartUploadState& state) { + Aws::Http::URI aws_uri(uri.c_str()); + std::string uri_path(aws_uri.GetPath().c_str()); + + state.bucket = aws_uri.GetAuthority(); + state.key = aws_uri.GetPath(); + + UniqueWriteLock unique_wl(&multipart_upload_rwlock_); + multipart_upload_states_[uri_path] = state; + + return Status::Ok(); +} + std::optional S3::multipart_upload_state( const URI& uri) { const Aws::Http::URI aws_uri(uri.c_str()); @@ -1970,20 +1985,6 @@ std::optional S3::multipart_upload_state( return rv_state; } -Status S3::set_multipart_upload_state( - const std::string& uri, MultiPartUploadState& state) { - Aws::Http::URI aws_uri(uri.c_str()); - std::string uri_path(aws_uri.GetPath().c_str()); - - state.bucket = aws_uri.GetAuthority(); - state.key = aws_uri.GetPath(); - - UniqueWriteLock unique_wl(&multipart_upload_rwlock_); - multipart_upload_states_[uri_path] = state; - - return Status::Ok(); -} - URI S3::generate_chunk_uri(const URI& attribute_uri, uint64_t id) { auto attribute_name = attribute_uri.remove_trailing_slash().last_path_part(); auto fragment_uri = attribute_uri.parent_path(); diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 0f8f0765274b..ebbe827a59a4 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -553,7 +553,7 @@ class S3Scanner : public LsScanner { * This class implements the various S3 filesystem functions. It also * maintains buffer caches for writing into the various attribute files. */ -class S3 { +class S3 : FilesystemBase { private: /** Forward declaration */ struct MultiPartUploadState; @@ -586,9 +586,213 @@ class S3 { * Creates a bucket. * * @param bucket The name of the bucket to be created. - * @return Status */ - Status create_bucket(const URI& bucket) const; + void create_bucket(const URI& bucket) const override; + + /** + * Removes the contents of an S3 bucket. + * + * @param bucket The URI of the bucket to be emptied. + */ + void empty_bucket(const URI& bucket) const override; + + /** + * Checks if a bucket is empty. + * + * @param bucket The URI of the bucket. + * @return True if the bucket is empty, false otherwise. + */ + bool is_empty_bucket(const URI& bucket) const override; + + /** + * Check if a bucket exists. + * + * @param bucket The name of the bucket. + * @return True if the bucket exists, false otherwise. + */ + bool is_bucket(const URI& uri) const override; + + /** + * Renames a directory. Note that this is an expensive operation. + * The function will essentially copy all objects with directory + * prefix `old_uri` to new objects with prefix `new_uri` and then + * delete the old ones. + * + * @param old_uri The URI of the old path. + * @param new_uri The URI of the new path. + */ + void move_dir(const URI& old_uri, const URI& new_uri) const override; + + /** + * Copies a file. + * + * @param old_uri The URI of the old path. + * @param new_uri The URI of the new path. + */ + void copy_file(const URI& old_uri, const URI& new_uri) const override; + + /** + * Copies a directory. All subdirectories and files are copied. + * + * @param old_uri The URI of the old path. + * @param new_uri The URI of the new path. + */ + void copy_dir(const URI& old_uri, const URI& new_uri) const override; + + /** + * Reads from a file. + * + * @param uri The URI of the file. + * @param offset The offset where the read begins. + * @param buffer The buffer to read into. + * @param nbytes Number of bytes to read. + * @param use_read_ahead Whether to use the read-ahead cache. + */ + void read( + const URI& uri, + uint64_t offset, + void* buffer, + uint64_t nbytes, + bool use_read_ahead = true) const override; + + /** + * Deletes a bucket. + * + * @param bucket The name of the bucket to be deleted. + */ + void remove_bucket(const URI& bucket) const override; + + /** + * Deletes all objects with prefix `prefix/` (if the ending `/` does not + * exist in `prefix`, it is added by the function. + * + * For instance, suppose there exist the following objects: + * - `s3://some_bucket/foo/bar1` + * - `s3://some_bucket/foo/bar2/bar3 + * - `s3://some_bucket/foo/bar4 + * - `s3://some_bucket/foo2` + * + * `remove("s3://some_bucket/foo")` and `remove("s3://some_bucket/foo/")` + * will delete objects: + * + * - `s3://some_bucket/foo/bar1` + * - `s3://some_bucket/foo/bar2/bar3 + * - `s3://some_bucket/foo/bar4 + * + * In contrast, `remove("s3://some_bucket/foo2")` will not delete anything, + * the function internally appends `/` to the end of the URI, and therefore + * there is not object with prefix "s3://some_bucket/foo2/" in this example. + * + * @param prefix The prefix of the objects to be deleted. + */ + void remove_dir(const URI& prefix) const override; + + /** + * Creates an empty object. + * + * @param uri The URI of the object to be created. + */ + void touch(const URI& uri) const override; + + /** + * Writes the input buffer to an S3 object. Note that this is essentially + * an append operation implemented via multipart uploads. + * + * @param uri The URI of the object to be written to. + * @param buffer The input buffer. + * @param length The size of the input buffer. + * @param remote_global_order_write + */ + void write( + const URI& uri, + const void* buffer, + uint64_t length, + bool remote_global_order_write = false) override; + + /** + * Creates a directory. + * + * - On S3, this is a noop. + * - On all other backends, if the directory exists, the function + * just succeeds without doing anything. + * + * @param uri The URI of the directory. + */ + void create_dir(const URI&) const override { + // No-op for S3. + } + + /** + * Checks if a file exists. + * + * @param uri The URI to check for existence. + * @return True if the file exists, else False. + */ + bool is_file(const URI& uri) const override { + bool object = false; + throw_if_not_ok(is_object(uri, &object)); + return object; + } + + /** + * Deletes a file. + * + * @param uri The URI of the file. + */ + void remove_file(const URI& uri) const override { + throw_if_not_ok(remove_object(uri)); + } + + /** + * Retrieves the size of a file. + * + * @param uri The URI of the file. + * @param size The file size to be retrieved. + */ + void file_size(const URI& uri, uint64_t* size) const override { + throw_if_not_ok(object_size(uri, size)); + } + + /** + * Renames a file. + * Both URI must be of the same backend type. (e.g. both s3://, file://, etc) + * + * @param old_uri The old URI. + * @param new_uri The new URI. + */ + void move_file(const URI& old_uri, const URI& new_uri) const override { + throw_if_not_ok(move_object(old_uri, new_uri)); + } + + /** + * Syncs (flushes) a file. Note that for S3 this is a noop. + * + * @param uri The URI of the file. + */ + void sync(const URI&) const override { + // No-op for S3. + } + + /** + * Checks if a directory exists. + * + * @param uri The URI to check for existence. + * @return True if the directory exists, else False. + */ + bool is_dir(const URI& uri) const override { + bool dir = false; + throw_if_not_ok(is_dir(uri, &dir)); + return dir; + } + + /** + * Retrieves all the entries contained in the parent. + * + * @param parent The target directory to list. + * @return All entries that are contained in the parent + */ + tuple>> ls_with_sizes( + const URI& parent) const override; /** * Disconnects a S3 client. @@ -596,10 +800,6 @@ class S3 { * @return Status */ Status disconnect(); - - /** Removes the contents of an S3 bucket. */ - Status empty_bucket(const URI& bucket) const; - /** * Flushes an object to S3, finalizing the multipart upload. * @@ -616,19 +816,6 @@ class S3 { */ void finalize_and_flush_object(const URI& uri); - /** Checks if a bucket is empty. */ - Status is_empty_bucket(const URI& bucket, bool* is_empty) const; - - /** - * Check if a bucket exists. - * - * @param bucket The name of the bucket. - * @param exists Mutates to `true` if `uri` is an existing bucket, - * and `false` otherwise. - * @return Status - */ - Status is_bucket(const URI& uri, bool* exists) const; - /** * Checks if there is an object with prefix `uri/`. For instance, suppose * the following objects exist: @@ -659,6 +846,12 @@ class S3 { */ Status is_object(const URI& uri, bool* exists) const; + /** Checks if the given object exists on S3. */ + Status is_object( + const Aws::String& bucket_name, + const Aws::String& object_key, + bool* exists) const; + /** * Lists the objects that start with `prefix`. Full URI paths are * retrieved for the matched objects. If a delimiter is specified, @@ -699,9 +892,7 @@ class S3 { * @return Status tuple where second is a list of directory_entry objects. */ tuple>> ls_with_sizes( - const URI& prefix, - const std::string& delimiter = "/", - int max_paths = -1) const; + const URI& prefix, const std::string& delimiter, int max_paths) const; /** * Lists objects and object information that start with `prefix`, invoking @@ -764,37 +955,7 @@ class S3 { * @param new_uri The URI of the new path. * @return Status */ - Status move_object(const URI& old_uri, const URI& new_uri); - - /** - * Renames a directory. Note that this is an expensive operation. - * The function will essentially copy all objects with directory - * prefix `old_uri` to new objects with prefix `new_uri` and then - * delete the old ones. - * - * @param old_uri The URI of the old path. - * @param new_uri The URI of the new path. - * @return Status - */ - Status move_dir(const URI& old_uri, const URI& new_uri); - - /** - * Copies a file. - * - * @param old_uri The URI of the old path. - * @param new_uri The URI of the new path. - * @return Status - */ - Status copy_file(const URI& old_uri, const URI& new_uri); - - /** - * Copies a directory. All subdirectories and files are copied. - * - * @param old_uri The URI of the old path. - * @param new_uri The URI of the new path. - * @return Status - */ - Status copy_dir(const URI& old_uri, const URI& new_uri); + Status move_object(const URI& old_uri, const URI& new_uri) const; /** * Returns the size of the input object with a given URI in bytes. @@ -816,21 +977,13 @@ class S3 { * @param length_returned Returns the total length read into `buffer`. * @return Status */ - Status read( + Status read_impl( const URI& uri, - off_t offset, - void* buffer, - uint64_t length, - uint64_t read_ahead_length, - uint64_t* length_returned) const; - - /** - * Deletes a bucket. - * - * @param bucket The name of the bucket to be deleted. - * @return Status - */ - Status remove_bucket(const URI& bucket) const; + const off_t offset, + void* const buffer, + const uint64_t length, + const uint64_t read_ahead_length, + uint64_t* const length_returned) const; /** * Deletes an object with a given URI. @@ -840,51 +993,6 @@ class S3 { */ Status remove_object(const URI& uri) const; - /** - * Deletes all objects with prefix `prefix/` (if the ending `/` does not - * exist in `prefix`, it is added by the function. - * - * For instance, suppose there exist the following objects: - * - `s3://some_bucket/foo/bar1` - * - `s3://some_bucket/foo/bar2/bar3 - * - `s3://some_bucket/foo/bar4 - * - `s3://some_bucket/foo2` - * - * `remove("s3://some_bucket/foo")` and `remove("s3://some_bucket/foo/")` - * will delete objects: - * - * - `s3://some_bucket/foo/bar1` - * - `s3://some_bucket/foo/bar2/bar3 - * - `s3://some_bucket/foo/bar4 - * - * In contrast, `remove("s3://some_bucket/foo2")` will not delete anything, - * the function internally appends `/` to the end of the URI, and therefore - * there is not object with prefix "s3://some_bucket/foo2/" in this example. - * - * @param uri The prefix of the objects to be deleted. - * @return Status - */ - Status remove_dir(const URI& prefix) const; - - /** - * Creates an empty object. - * - * @param uri The URI of the object to be created. - * @return Status - */ - Status touch(const URI& uri) const; - - /** - * Writes the input buffer to an S3 object. Note that this is essentially - * an append operation implemented via multipart uploads. - * - * @param uri The URI of the object to be written to. - * @param buffer The input buffer. - * @param length The size of the input buffer. - * @return Status - */ - Status write(const URI& uri, const void* buffer, uint64_t length); - /** * Writes the input buffer to an S3 object. This function buffers in memory * file_buffer_size_ bytes before calling global_order_write() to execute @@ -1308,7 +1416,7 @@ class S3 { * @param new_uri The newly created object. * @return Status */ - Status copy_object(const URI& old_uri, const URI& new_uri); + Status copy_object(const URI& old_uri, const URI& new_uri) const; /** * Fills the file buffer (given as an input `Buffer` object) from the @@ -1363,12 +1471,6 @@ class S3 { std::string join_authority_and_path( const std::string& authority, const std::string& path) const; - /** Checks if the given object exists on S3. */ - Status is_object( - const Aws::String& bucket_name, - const Aws::String& object_key, - bool* exists) const; - /** Waits for the input object to be propagated. */ Status wait_for_object_to_propagate( const Aws::String& bucketName, const Aws::String& objectKey) const; diff --git a/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc b/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc index 931166dba8a4..5b995533aa19 100644 --- a/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc +++ b/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc @@ -73,9 +73,8 @@ TEST_CASE("VFS Read Log Modes", "[vfs][read-logging-modes]") { for (int i = 0; i < 2; i++) { char buffer[123]; for (auto& uri : uris_to_read) { - auto st = res.vfs().read(URI(uri), 123, buffer, 456); // None of these files exist, so we expect every read to fail. - REQUIRE(!st.ok()); + REQUIRE_THROWS(res.vfs().read(URI(uri), 123, buffer, 456)); } } } diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 9faa1ac3fa55..4d24aad32f20 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -164,7 +164,8 @@ Status VFS::create_dir(const URI& uri) const { #ifdef _WIN32 return win_.create_dir(uri.to_path()); #else - return posix_.create_dir(uri); + posix_.create_dir(uri); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -240,7 +241,8 @@ Status VFS::touch(const URI& uri) const { #ifdef _WIN32 return win_.touch(uri.to_path()); #else - return posix_.touch(uri); + posix_.touch(uri); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -252,7 +254,8 @@ Status VFS::touch(const URI& uri) const { } if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().touch(uri); + s3().touch(uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif @@ -285,7 +288,8 @@ Status VFS::cancel_all_tasks() { Status VFS::create_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().create_bucket(uri); + s3().create_bucket(uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif @@ -310,7 +314,8 @@ Status VFS::create_bucket(const URI& uri) const { Status VFS::remove_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().remove_bucket(uri); + s3().remove_bucket(uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif @@ -335,7 +340,8 @@ Status VFS::remove_bucket(const URI& uri) const { Status VFS::empty_bucket(const URI& uri) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().empty_bucket(uri); + s3().empty_bucket(uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif @@ -361,7 +367,8 @@ Status VFS::is_empty_bucket( const URI& uri, [[maybe_unused]] bool* is_empty) const { if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().is_empty_bucket(uri, is_empty); + *is_empty = s3().is_empty_bucket(uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif @@ -388,7 +395,7 @@ Status VFS::remove_dir(const URI& uri) const { #ifdef _WIN32 return win_.remove_dir(uri.to_path()); #else - return posix_.remove_dir(uri); + posix_.remove_dir(uri); #endif } else if (uri.is_hdfs()) { #ifdef HAVE_HDFS @@ -398,7 +405,7 @@ Status VFS::remove_dir(const URI& uri) const { #endif } else if (uri.is_s3()) { #ifdef HAVE_S3 - return s3().remove_dir(uri); + s3().remove_dir(uri); #else throw BuiltWithout("S3"); #endif @@ -419,6 +426,7 @@ Status VFS::remove_dir(const URI& uri) const { } else { throw UnsupportedURI(uri.to_string()); } + return Status::Ok(); } void VFS::remove_dirs( @@ -438,7 +446,8 @@ Status VFS::remove_file(const URI& uri) const { #ifdef _WIN32 return win_.remove_file(uri.to_path()); #else - return posix_.remove_file(uri); + posix_.remove_file(uri); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -523,7 +532,8 @@ Status VFS::file_size(const URI& uri, uint64_t* size) const { #ifdef _WIN32 return win_.file_size(uri.to_path(), size); #else - return posix_.file_size(uri, size); + posix_.file_size(uri, size); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -661,7 +671,7 @@ Status VFS::is_file(const URI& uri, bool* is_file) const { Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { if (uri.is_s3()) { #ifdef HAVE_S3 - RETURN_NOT_OK(s3().is_bucket(uri, is_bucket)); + *is_bucket = s3().is_bucket(uri); return Status::Ok(); #else *is_bucket = false; @@ -791,7 +801,8 @@ Status VFS::move_file(const URI& old_uri, const URI& new_uri) { #ifdef _WIN32 return win_.move_path(old_uri.to_path(), new_uri.to_path()); #else - return posix_.move_file(old_uri, new_uri); + posix_.move_file(old_uri, new_uri); + return Status::Ok(); #endif } throw UnsupportedOperation("Moving files"); @@ -860,7 +871,8 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { #ifdef _WIN32 return win_.move_path(old_uri.to_path(), new_uri.to_path()); #else - return posix_.move_file(old_uri, new_uri); + posix_.move_dir(old_uri, new_uri); + return Status::Ok(); #endif } throw UnsupportedOperation("Moving directories"); @@ -879,12 +891,14 @@ Status VFS::move_dir(const URI& old_uri, const URI& new_uri) { // S3 if (old_uri.is_s3()) { - if (new_uri.is_s3()) + if (new_uri.is_s3()) { #ifdef HAVE_S3 - return s3().move_dir(old_uri, new_uri); + s3().move_dir(old_uri, new_uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif + } throw UnsupportedOperation("Moving directories"); } @@ -935,7 +949,8 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { #ifdef _WIN32 throw VFSException("Copying files on Windows is not yet supported."); #else - return posix_.copy_file(old_uri, new_uri); + posix_.copy_file(old_uri, new_uri); + return Status::Ok(); #endif } throw UnsupportedOperation("Copying files"); @@ -955,12 +970,14 @@ Status VFS::copy_file(const URI& old_uri, const URI& new_uri) { // S3 if (old_uri.is_s3()) { - if (new_uri.is_s3()) + if (new_uri.is_s3()) { #ifdef HAVE_S3 - return s3().copy_file(old_uri, new_uri); + s3().copy_file(old_uri, new_uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif + } throw UnsupportedOperation("Copying files"); } @@ -1000,7 +1017,8 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { throw VFSException( "Copying directories on Windows is not yet supported."); #else - return posix_.copy_dir(old_uri, new_uri); + posix_.copy_dir(old_uri, new_uri); + return Status::Ok(); #endif } throw UnsupportedOperation("Copying directories"); @@ -1020,12 +1038,14 @@ Status VFS::copy_dir(const URI& old_uri, const URI& new_uri) { // S3 if (old_uri.is_s3()) { - if (new_uri.is_s3()) + if (new_uri.is_s3()) { #ifdef HAVE_S3 - return s3().copy_dir(old_uri, new_uri); + s3().copy_dir(old_uri, new_uri); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif + } throw UnsupportedOperation("Copying directories"); } @@ -1135,7 +1155,8 @@ Status VFS::read_impl( #ifdef _WIN32 return win_.read(uri.to_path(), offset, buffer, nbytes); #else - return posix_.read(uri, offset, buffer, nbytes, false); + posix_.read(uri, offset, buffer, nbytes, false); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -1148,7 +1169,7 @@ Status VFS::read_impl( if (uri.is_s3()) { #ifdef HAVE_S3 const auto read_fn = std::bind( - &S3::read, + &S3::read_impl, &s3(), std::placeholders::_1, std::placeholders::_2, @@ -1288,7 +1309,8 @@ Status VFS::sync(const URI& uri) { #ifdef _WIN32 return win_.sync(uri.to_path()); #else - return posix_.sync(uri); + posix_.sync(uri); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -1378,7 +1400,8 @@ Status VFS::close_file(const URI& uri) { #ifdef _WIN32 return win_.sync(uri.to_path()); #else - return posix_.sync(uri); + posix_.sync(uri); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -1439,7 +1462,8 @@ Status VFS::write( #ifdef _WIN32 return win_.write(uri.to_path(), buffer, buffer_size); #else - return posix_.write(uri, buffer, buffer_size); + posix_.write(uri, buffer, buffer_size); + return Status::Ok(); #endif } if (uri.is_hdfs()) { @@ -1451,11 +1475,8 @@ Status VFS::write( } if (uri.is_s3()) { #ifdef HAVE_S3 - if (remote_global_order_write) { - s3().global_order_write_buffered(uri, buffer, buffer_size); - return Status::Ok(); - } - return s3().write(uri, buffer, buffer_size); + s3().write(uri, buffer, buffer_size, remote_global_order_write); + return Status::Ok(); #else throw BuiltWithout("S3"); #endif diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 1549b754fa5d..08798df4c1ce 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -239,12 +239,12 @@ class S3_within_VFS { } /** Protected accessor for the S3 object. */ - inline tiledb::sm::S3& s3() { + inline S3& s3() { return s3_; } /** Protected accessor for the const S3 object. */ - inline const tiledb::sm::S3& s3() const { + inline const S3& s3() const { return s3_; } }; @@ -336,7 +336,7 @@ class VFS : private VFSBase, protected S3_within_VFS { /** * Return a config object containing the VFS parameters. All other non-VFS - * parameters will are set to default values. + * parameters are set to default values. */ Config config() const; From ffded0cc8c3b961e3cf31ca366a98d92f0c1ee0a Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 29 Jan 2024 14:06:20 +0200 Subject: [PATCH 161/456] Fix documentation of `Subarray::set_config` to match `tiledb_subarray_set_config`. (#4672) See https://github.com/TileDB-Inc/TileDB/blob/1ea150f531e0da288594e410e2d1d986741dfcf2/tiledb/sm/c_api/tiledb.h#L2364-L2375 --- TYPE: BUG DESC: Fix documentation of `Subarray::set_config` to match `tiledb_subarray_set_config`. --- tiledb/sm/cpp_api/subarray.h | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tiledb/sm/cpp_api/subarray.h b/tiledb/sm/cpp_api/subarray.h index 860962b19e80..e165fc4c3110 100644 --- a/tiledb/sm/cpp_api/subarray.h +++ b/tiledb/sm/cpp_api/subarray.h @@ -331,16 +331,7 @@ class Subarray { * Setting configuration with this function overrides the following * Subarray-level parameters only: * - * - `sm.memory_budget` - * - `sm.memory_budget_var` - * - `sm.sub_partitioner_memory_budget` - * - `sm.var_offsets.mode` - * - `sm.var_offsets.extra_element` - * - `sm.var_offsets.bitsize` - * - `sm.check_coord_dups` - * - `sm.check_coord_oob` - * - `sm.check_global_order` - * - `sm.dedup_coords` + * - `sm.read_range_oob` */ Subarray& set_config(const Config& config) { auto ctx = ctx_.get(); From f493b9ef9de92befb4f659cf5a0ea6675868acbc Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 29 Jan 2024 16:25:01 +0100 Subject: [PATCH 162/456] Update dev HISTORY with 2.19.1 and 2.18.4 content. (#4677) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.19.1 and 2.18.4 content. --- HISTORY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 686ccd951c69..91ea067476ee 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -19,6 +19,7 @@ * Fix traversal limit in array deserialization. [#4606](https://github.com/TileDB-Inc/TileDB/pull/4606) * Add function random_label to utilize PRNG for random string generation. [#4564](https://github.com/TileDB-Inc/TileDB/pull/4564) * Remove uuid in favor of random_label. [#4589](https://github.com/TileDB-Inc/TileDB/pull/4589) +* Improve large dense aggregate reads with tile metadata only. [#4657](https://github.com/TileDB-Inc/TileDB/pull/4657) ## Defects removed @@ -43,6 +44,17 @@ * Add vcpkg triplets for Address Sanitizer. [#4515](https://github.com/TileDB-Inc/TileDB/pull/4515) * Fix regression where release artifacts had 8-digit commit hashes. [#4599](https://github.com/TileDB-Inc/TileDB/pull/4599) +# TileDB v2.19.1 Release Notes + +## Improvements + +* Improve large dense aggregate reads with tile metadata only. [#4657](https://github.com/TileDB-Inc/TileDB/pull/4657) + +## Defects removed + +* Fix HTTP requests for AWS assume role with web identity not honoring config options. [#4641](https://github.com/TileDB-Inc/TileDB/pull/4641) +* Fix HTTP requests for AWS assume role not honoring config options. [#4616](https://github.com/TileDB-Inc/TileDB/pull/4616) + # TileDB v2.19.0 Release Notes ## Announcements @@ -124,6 +136,13 @@ * Remove UnitTestConfig::array_encryption_key_length. [#4482](https://github.com/TileDB-Inc/TileDB/pull/4482) +# TileDB v2.18.4 Release Notes + +## Defects removed + +* Fix HTTP requests for AWS assume role not honoring config options. [#4616](https://github.com/TileDB-Inc/TileDB/pull/4616) +* Fix HTTP requests for AWS assume role with web identity not honoring config options. [#4641](https://github.com/TileDB-Inc/TileDB/pull/4641) + # TileDB v2.18.3 Release Notes ## New features From 74ebc44d619abc9908b9996bc504cd943fbb80ce Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:31:23 -0500 Subject: [PATCH 163/456] Add class FragmentID. (#4607) Add class FragmentID to validate, parse and handle components of a fragment uri. --- TYPE: NO_HISTORY DESC: Add class FragmentID. --- format_spec/timestamped_name.md | 2 +- test/src/unit-average-cell-size.cc | 8 +- test/src/unit-capi-array.cc | 15 +- tiledb/CMakeLists.txt | 3 +- tiledb/sm/CMakeLists.txt | 3 +- tiledb/sm/array/CMakeLists.txt | 4 +- tiledb/sm/array/array_directory.cc | 76 +++---- tiledb/sm/array/array_directory.h | 12 +- tiledb/sm/array_schema/CMakeLists.txt | 4 +- tiledb/sm/array_schema/array_schema.cc | 8 +- .../sm/consolidator/fragment_consolidator.cc | 9 +- .../fragment_meta_consolidator.cc | 18 +- tiledb/sm/fragment/CMakeLists.txt | 41 ++++ tiledb/sm/fragment/fragment_identifier.cc | 116 +++++++++++ tiledb/sm/fragment/fragment_identifier.h | 142 +++++++++++++ tiledb/sm/fragment/fragment_info.cc | 14 +- tiledb/sm/fragment/fragment_metadata.cc | 21 +- tiledb/sm/fragment/test/CMakeLists.txt | 32 +++ .../sm/fragment/test/compile_fragment_main.cc | 40 ++++ tiledb/sm/fragment/test/main.cc | 34 ++++ .../fragment/test/unit_fragment_identifier.cc | 189 ++++++++++++++++++ tiledb/sm/group/group_directory.cc | 17 +- .../deletes_and_updates.cc | 13 +- tiledb/sm/query/query_condition.cc | 10 +- tiledb/sm/query/writers/writer_base.cc | 7 +- tiledb/storage_format/uri/CMakeLists.txt | 4 +- tiledb/storage_format/uri/generate_uri.cc | 11 +- tiledb/storage_format/uri/parse_uri.cc | 96 +-------- tiledb/storage_format/uri/parse_uri.h | 17 +- .../uri/test/compile_uri_format_main.cc | 6 +- .../uri/test/unit_uri_format.cc | 102 +--------- 31 files changed, 710 insertions(+), 364 deletions(-) create mode 100644 tiledb/sm/fragment/CMakeLists.txt create mode 100644 tiledb/sm/fragment/fragment_identifier.cc create mode 100644 tiledb/sm/fragment/fragment_identifier.h create mode 100644 tiledb/sm/fragment/test/CMakeLists.txt create mode 100644 tiledb/sm/fragment/test/compile_fragment_main.cc create mode 100644 tiledb/sm/fragment/test/main.cc create mode 100644 tiledb/sm/fragment/test/unit_fragment_identifier.cc diff --git a/format_spec/timestamped_name.md b/format_spec/timestamped_name.md index 35742c7cc961..946af468e5f7 100644 --- a/format_spec/timestamped_name.md +++ b/format_spec/timestamped_name.md @@ -16,4 +16,4 @@ _Note_: The presence of `[]` is indicative of an optional parameter. | :-: | :-: | :-: | | 1 - 2 | 1.4 - 1.5 | `__uuid_t1[_t2]` | | 3 - 4 | 1.6 - 1.7 | `__t1_t2_uuid` | -| 5+ | 2.0+ | `__t1_t2_uuid_[v]` | +| 5+ | 2.0+ | `__t1_t2_uuid[_v]` | diff --git a/test/src/unit-average-cell-size.cc b/test/src/unit-average-cell-size.cc index a6663fb1e342..a5a7a5f07c20 100644 --- a/test/src/unit-average-cell-size.cc +++ b/test/src/unit-average-cell-size.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB Inc. + * @copyright Copyright (c) 2022-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/utils.h" @@ -166,9 +167,8 @@ struct CPPAverageCellSizeFx { REQUIRE(query.submit() == Query::Status::COMPLETE); auto uri = sm::URI(query.fragment_uri(0)); - std::pair timestamps; - REQUIRE(sm::utils::parse::get_timestamp_range(uri, ×tamps).ok()); - return {uri, timestamps}; + sm::FragmentID fragment_id{uri}; + return {uri, fragment_id.timestamp_range()}; } /** diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 129a219a8f56..3da9120f9110 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -53,10 +53,10 @@ #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/global_state/unit_test_config.h" #include "tiledb/sm/serialization/array.h" #include "tiledb/sm/serialization/fragments.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include #include @@ -142,13 +142,10 @@ void ArrayFx::remove_temp_dir(const std::string& path) { int ArrayFx::get_fragment_timestamps(const char* path, void* data) { auto data_vec = (std::vector*)data; - std::pair timestamp_range; - if (tiledb::sm::utils::parse::ends_with( - path, tiledb::sm::constants::write_file_suffix)) { - auto uri = tiledb::sm::URI(path); - if (tiledb::sm::utils::parse::get_timestamp_range(uri, ×tamp_range) - .ok()) - data_vec->push_back(timestamp_range.first); + if (utils::parse::ends_with(path, constants::write_file_suffix)) { + FragmentID fragment_id{path}; + auto timestamp_range{fragment_id.timestamp_range()}; + data_vec->push_back(timestamp_range.first); } return 1; diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index a10c5350b4ae..f4a11f7c0521 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -4,7 +4,7 @@ # # The MIT License # -# Copyright (c) 2017-2023 TileDB, Inc. +# Copyright (c) 2017-2024 TileDB, Inc. # Copyright (c) 2016 MIT and Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -221,6 +221,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/webp_filter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/noop_filter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/positive_delta_filter.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_identifier.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_info.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_metadata.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/global_state/global_state.cc diff --git a/tiledb/sm/CMakeLists.txt b/tiledb/sm/CMakeLists.txt index 29c0492080f4..8f875d7c66fe 100644 --- a/tiledb/sm/CMakeLists.txt +++ b/tiledb/sm/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,6 +35,7 @@ add_subdirectory(consolidator) add_subdirectory(crypto) add_subdirectory(filesystem) add_subdirectory(filter) +add_subdirectory(fragment) add_subdirectory(group) add_subdirectory(metadata) add_subdirectory(misc) diff --git a/tiledb/sm/array/CMakeLists.txt b/tiledb/sm/array/CMakeLists.txt index ca712b353983..737f99cde7b7 100644 --- a/tiledb/sm/array/CMakeLists.txt +++ b/tiledb/sm/array/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022-2023 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,8 +35,8 @@ commence(object_library array) this_target_object_libraries( array_schema baseline + fragment generic_tile_io - uri_format vfs ) if(TILEDB_STATS) diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 4676cf507a45..9710c5607f29 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -535,28 +535,24 @@ URI ArrayDirectory::get_commits_dir(uint32_t write_version) const { } URI ArrayDirectory::get_commit_uri(const URI& fragment_uri) const { - auto name = fragment_uri.remove_trailing_slash().last_path_part(); - auto fragment_version = utils::parse::get_fragment_version(name); - - if (fragment_version < 12) { + FragmentID fragment_id{fragment_uri}; + if (fragment_id.array_format_version() < 12) { return URI(fragment_uri.to_string() + constants::ok_file_suffix); } - auto temp_uri = - uri_.join_path(constants::array_commits_dir_name).join_path(name); + auto temp_uri = uri_.join_path(constants::array_commits_dir_name) + .join_path(fragment_id.name()); return URI(temp_uri.to_string() + constants::write_file_suffix); } URI ArrayDirectory::get_vacuum_uri(const URI& fragment_uri) const { - auto name = fragment_uri.remove_trailing_slash().last_path_part(); - auto fragment_version = utils::parse::get_fragment_version(name); - - if (fragment_version < 12) { + FragmentID fragment_id{fragment_uri}; + if (fragment_id.array_format_version() < 12) { return URI(fragment_uri.to_string() + constants::vacuum_file_suffix); } - auto temp_uri = - uri_.join_path(constants::array_commits_dir_name).join_path(name); + auto temp_uri = uri_.join_path(constants::array_commits_dir_name) + .join_path(fragment_id.name()); return URI(temp_uri.to_string() + constants::vacuum_file_suffix); } @@ -678,11 +674,8 @@ ArrayDirectory::load_commits_dir_uris_v12_or_higher( stdx::string::ends_with( commits_dir_uris[i].to_string(), constants::update_file_suffix)) { // Get the start and end timestamp for this delete/update - std::pair timestamp_range; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range( - commits_dir_uris[i], ×tamp_range), - nullopt); + FragmentID fragment_id{commits_dir_uris[i]}; + auto timestamp_range{fragment_id.timestamp_range()}; // Add the delete tile location if it overlaps the open start/end times if (timestamps_overlap(timestamp_range, false)) { @@ -772,12 +765,8 @@ ArrayDirectory::load_consolidated_commit_uris( auto pos = ss.tellg(); // Get the start and end timestamp for this delete - std::pair delete_timestamp_range; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range( - URI(condition_marker), &delete_timestamp_range), - nullopt, - nullopt); + FragmentID fragment_id{URI(condition_marker)}; + auto delete_timestamp_range{fragment_id.timestamp_range()}; // Add the delete tile location if it overlaps the open start/end // times @@ -1028,9 +1017,8 @@ ArrayDirectory::compute_uris_to_vacuum( auto& uri = uris[i]; // Get the start and end timestamp for this fragment - std::pair fragment_timestamp_range; - RETURN_NOT_OK( - utils::parse::get_timestamp_range(uri, &fragment_timestamp_range)); + FragmentID fragment_id{uri}; + auto fragment_timestamp_range{fragment_id.timestamp_range()}; if (is_vacuum_file(uri)) { vac_file_bitmap[i] = 1; if (timestamps_overlap( @@ -1164,8 +1152,8 @@ ArrayDirectory::compute_filtered_uris( } // Get the start and end timestamp for this fragment - RETURN_NOT_OK(utils::parse::get_timestamp_range( - uri, &fragment_timestamp_ranges[i])); + FragmentID fragment_id{uri}; + fragment_timestamp_ranges[i] = fragment_id.timestamp_range(); if (timestamps_overlap( fragment_timestamp_ranges[i], !full_overlap_only && @@ -1238,16 +1226,13 @@ URI ArrayDirectory::select_latest_array_schema_uri() { uint64_t latest_ts = 0; for (auto& uri : array_schema_uris_) { - auto name = uri.remove_trailing_slash().last_path_part(); - + FragmentID fragment_id{uri}; // Skip the old schema URI name since it doesn't have timestamps - if (name == constants::array_schema_filename) { + if (fragment_id.name() == constants::array_schema_filename) { continue; } - std::pair ts_range; - throw_if_not_ok(utils::parse::get_timestamp_range(uri, &ts_range)); - + auto ts_range{fragment_id.timestamp_range()}; if (ts_range.second > latest_ts && ts_range.second <= timestamp_end_) { latest_uri = uri; latest_ts = ts_range.second; @@ -1270,7 +1255,8 @@ Status ArrayDirectory::is_fragment( const std::unordered_set& consolidated_uris_set, int* is_fragment) const { // If the URI name has a suffix, then it is not a fragment - auto name = uri.remove_trailing_slash().last_path_part(); + FragmentID fragment_id{uri}; + auto name = fragment_id.name(); if (name.find_first_of('.') != std::string::npos) { *is_fragment = 0; return Status::Ok(); @@ -1299,10 +1285,9 @@ Status ArrayDirectory::is_fragment( return Status::Ok(); } - // If the format version is >= 5, then the above suffices to check if + // If the array format version is >= 5, then the above suffices to check if // the URI is indeed a fragment - auto fragment_version = utils::parse::get_fragment_version(name); - if (fragment_version >= 5) { + if (fragment_id.array_format_version() >= 5) { *is_fragment = false; return Status::Ok(); } @@ -1317,15 +1302,12 @@ Status ArrayDirectory::is_fragment( bool ArrayDirectory::consolidation_with_timestamps_supported( const URI& uri) const { - // Get the fragment version from the uri - auto name = uri.remove_trailing_slash().last_path_part(); - auto fragment_version = utils::parse::get_fragment_version(name); - - // get_fragment_version returns UINT32_MAX for versions <= 2 so we should - // explicitly exclude this case when checking if consolidation with timestamps - // is supported on a fragment + // FragmentID::array_format_version() returns UINT32_MAX for versions <= 2 + // so we should explicitly exclude this case when checking if consolidation + // with timestamps is supported on a fragment + FragmentID fragment_id{uri}; return mode_ == ArrayDirectoryMode::READ && - fragment_version >= + fragment_id.array_format_version() >= constants::consolidation_with_timestamps_min_version; } diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 115b1ac8c5f5..ffaf399a7017 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,9 +39,9 @@ #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/stats/stats.h" #include "tiledb/sm/storage_manager/context_resources.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include #include @@ -221,12 +221,8 @@ class ArrayDirectory { : uri_(uri) , condition_marker_(condition_marker) , offset_(offset) { - std::pair timestamps; - if (!utils::parse::get_timestamp_range(URI(condition_marker), ×tamps) - .ok()) { - throw std::logic_error("Error parsing uri."); - } - + FragmentID fragment_id{condition_marker}; + auto timestamps{fragment_id.timestamp_range()}; timestamp_ = timestamps.first; } diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index b5e58e29af30..602d9b9af264 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2023 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -71,7 +71,7 @@ conclude(object_library) commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain enumeration time uri_format vfs) + attribute domain enumeration fragment time uri_format vfs) conclude(object_library) # This is linked outside the object_library scope because ContextResources diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index df101ed0313e..95fc59ace895 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -49,12 +49,12 @@ #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/filter/webp_filter.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/integral_type_casts.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include "tiledb/type/apply_with_type.h" #include @@ -1386,8 +1386,8 @@ ArraySchema ArraySchema::deserialize( } // Populate timestamp range - std::pair timestamp_range; - throw_if_not_ok(utils::parse::get_timestamp_range(uri, ×tamp_range)); + FragmentID fragment_id{uri}; + auto timestamp_range{fragment_id.timestamp_range()}; // Set schema name std::string name = uri.last_path_part(); diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 6b78edcf8910..aa42ee99cfe5 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,12 +36,12 @@ #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_status.h" #include "tiledb/sm/enums/query_type.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/generate_uri.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include #include @@ -518,9 +518,8 @@ Status FragmentConsolidator::consolidate_internal( if (!config_.purge_deleted_cells_ && array_schema.write_version() >= constants::deletes_min_version) { // Get the first fragment first timestamp. - std::pair timestamps; - RETURN_NOT_OK( - utils::parse::get_timestamp_range(to_consolidate[0].uri_, ×tamps)); + FragmentID fragment_id{to_consolidate[0].uri_}; + auto timestamps{fragment_id.timestamp_range()}; for (auto& delete_and_update_tile_location : array_for_reads->array_directory() diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 270ad4d8209f..8b22b37c40c5 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,6 +34,7 @@ #include "tiledb/common/logger.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_type.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/stats/global_stats.h" @@ -41,7 +42,6 @@ #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" #include "tiledb/storage_format/uri/generate_uri.h" -#include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb::common; @@ -101,10 +101,8 @@ Status FragmentMetaConsolidator::consolidate( uri = URI(frag_md_uri.to_string() + name + constants::meta_file_suffix); // Get the consolidated fragment metadata version - auto meta_name = uri.remove_trailing_slash().last_path_part(); - auto pos = meta_name.find_last_of('.'); - meta_name = (pos == std::string::npos) ? meta_name : meta_name.substr(0, pos); - auto meta_version = utils::parse::get_fragment_version(meta_name); + FragmentID fragment_id{uri}; + auto meta_version = fragment_id.array_format_version(); // Calculate offset of first fragment footer uint64_t offset = sizeof(uint32_t); // Fragment num @@ -197,8 +195,8 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { // Get the latest timestamp uint64_t t_latest = 0; for (const auto& uri : fragment_meta_uris) { - std::pair timestamp_range; - throw_if_not_ok(utils::parse::get_timestamp_range(uri, ×tamp_range)); + FragmentID fragment_id{uri}; + auto timestamp_range{fragment_id.timestamp_range()}; if (timestamp_range.second > t_latest) { t_latest = timestamp_range.second; } @@ -211,8 +209,8 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { throw_if_not_ok( parallel_for(compute_tp, 0, fragment_meta_uris.size(), [&](size_t i) { auto& uri = fragment_meta_uris[i]; - std::pair timestamp_range; - RETURN_NOT_OK(utils::parse::get_timestamp_range(uri, ×tamp_range)); + FragmentID fragment_id{uri}; + auto timestamp_range{fragment_id.timestamp_range()}; if (timestamp_range.second != t_latest) RETURN_NOT_OK(vfs->remove_file(uri)); return Status::Ok(); diff --git a/tiledb/sm/fragment/CMakeLists.txt b/tiledb/sm/fragment/CMakeLists.txt new file mode 100644 index 000000000000..6b359240b95a --- /dev/null +++ b/tiledb/sm/fragment/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/sm/fragment/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +# +# `fragment` object library +# +commence(object_library fragment) + this_target_sources(fragment_identifier.cc) + this_target_object_libraries(baseline vfs) + if(TILEDB_STATS) + this_target_compile_definitions(-DTILEDB_STATS) + endif() +conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/sm/fragment/fragment_identifier.cc b/tiledb/sm/fragment/fragment_identifier.cc new file mode 100644 index 000000000000..88ff947224f3 --- /dev/null +++ b/tiledb/sm/fragment/fragment_identifier.cc @@ -0,0 +1,116 @@ +/** + * @file tiledb/sm/fragment/fragment_identifier.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class FragmentID. + */ + +#include +#include // For sscanf PRId64 +#include + +#include "fragment_identifier.h" +#include "tiledb/common/exception/exception.h" + +namespace tiledb::sm { + +class FragmentIDException : public StatusException { + public: + explicit FragmentIDException(const std::string& message) + : StatusException("FragmentID", message) { + } +}; + +class InvalidURIException : public FragmentIDException { + public: + explicit InvalidURIException(const std::string& message) + : FragmentIDException("input URI is invalid. " + message) { + } +}; + +FragmentID::FragmentID(const URI& uri) + : URI(uri) + , name_(uri.remove_trailing_slash().last_path_part()) + , timestamp_range_({0, 0}) + , name_version_(FragmentNameVersion::ONE) { + // Ensure input uri is valid (non-empty) + if (uri.empty()) { + throw InvalidURIException("URI may not be empty."); + } + + // Set name + auto pos = name_.find_last_of('.'); + name_ = (pos == std::string::npos) ? name_ : name_.substr(0, pos); + if (name_.find_last_of('_') == std::string::npos) { + throw InvalidURIException("Provided URI does not contain a fragment name."); + } + + // Set array format version + auto array_format_version_str = name_.substr(name_.find_last_of('_') + 1); + std::stringstream ss(array_format_version_str); + ss >> array_format_version_; + + // Set name version + size_t n = std::count(name_.begin(), name_.end(), '_'); + if (n == 5) { + name_version_ = FragmentNameVersion::THREE; + } else if (array_format_version_str.size() == 32) { + name_version_ = FragmentNameVersion::TWO; + } + + // Set timestamp range + if (name_version_ == FragmentNameVersion::ONE) { + array_format_version_ = 2; + sscanf( + array_format_version_str.c_str(), + (std::string("%") + std::string(PRId64)).c_str(), + (long long int*)×tamp_range_.first); + timestamp_range_.second = timestamp_range_.first; + } else { + if (name_version_ == FragmentNameVersion::TWO) { + array_format_version_ = 4; + } + sscanf( + name_.c_str(), + (std::string("__%") + std::string(PRId64) + "_%" + std::string(PRId64)) + .c_str(), + (long long int*)×tamp_range_.first, + (long long int*)×tamp_range_.second); + } + if (timestamp_range_.first > timestamp_range_.second) { + throw FragmentIDException( + "Failed to construct FragmentID; start timestamp cannot " + "be after end timestamp"); + } +} + +FragmentID::FragmentID(const std::string_view& path) + : FragmentID(URI(path.data())) { +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/fragment/fragment_identifier.h b/tiledb/sm/fragment/fragment_identifier.h new file mode 100644 index 000000000000..52879308a879 --- /dev/null +++ b/tiledb/sm/fragment/fragment_identifier.h @@ -0,0 +1,142 @@ +/** + * @file tiledb/sm/fragment/fragment_identifier.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class FragmentID. + */ + +#ifndef TILEDB_FRAGMENT_ID_H +#define TILEDB_FRAGMENT_ID_H + +#include "tiledb/common/status.h" +#include "tiledb/sm/filesystem/uri.h" + +namespace tiledb::sm { + +using timestamp_range_type = std::pair; + +/** + * The possible fragment name versions. + * + * ONE: __uuid_t1{_t2} + * TWO: __t1_t2_uuid + * THREE: __t1_t2_uuid_version + * + * @ref `format_spec/timestamped_name.md` + */ +enum class FragmentNameVersion { ONE, TWO, THREE }; + +/** + * Validate, parse and handle the different components of a fragment identifer. + * + * @section Known Defects: + * Construction should fail (but currently does not) + * with the following _name_ (not uri) inputs: + * + * Empty. + * "X" Non-empty, no underscores. + * "_" 1 underscore only, no fields. + * "__" 2 underscores only, no fields. + * "___" 3 underscores only, all fields empty (version 1). + * "____" 4 underscores only, all fields empty (versions 2, 3). + * "_____"5 underscores, all fields are empty (version 3). + * Missing fields. + * Correct number of fields, incorrect order. + * Trailing "_". + * Otherwise potentially-malformed names, which may not begin with "__". + */ +class FragmentID : private URI { + private: + /** The fragment name. */ + std::string name_; + /** The timestamp range. */ + timestamp_range_type timestamp_range_; + + /** The fragment name version. */ + FragmentNameVersion name_version_; + + /** The array format version. */ + format_version_t array_format_version_; + + public: + /** Constructor. */ + FragmentID(const URI& uri); + + /** Constructor. */ + FragmentID(const std::string_view& path); + + /** Destructor. */ + ~FragmentID() = default; + + /** Accessor to the fragment name. */ + inline const std::string& name() { + return name_; + } + + /** + * Accessor to the timestamp range. + * For array format version <= 2, only the range start is valid + * (the range end is ignored). + */ + inline timestamp_range_type timestamp_range() { + return timestamp_range_; + } + + /** + * Accessor to the fragment name version. + * Returns an integer corresponding to the FragmentNameVersion. + * + * - Name version 1 corresponds to format versions 1 and 2 + * * __uuid_{_t2} + * - Name version 2 corresponds to format version 3 and 4 + * * __t1_t2_uuid + * - Name version 3 corresponds to format version 5 or higher + * * __t1_t2_uuid_version + */ + inline int name_version() { + if (name_version_ == FragmentNameVersion::ONE) { + return 1; + } else if (name_version_ == FragmentNameVersion::TWO) { + return 2; + } else { + return 3; + } + } + + /** + * Accessor to the array format version. + * Returns UINT32_MAX for name versions <= 2. + */ + inline format_version_t array_format_version() { + return array_format_version_; + } +}; + +} // namespace tiledb::sm + +#endif // TILEDB_FRAGMENT_ID_H diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 20e51afd2648..ec6e3988273a 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2020-2023 TileDB, Inc. + * @copyright Copyright (c) 2020-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,12 +38,12 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/tile/generic_tile_io.h" -#include "tiledb/storage_format/uri/parse_uri.h" namespace tiledb::sm { @@ -1101,16 +1101,12 @@ tuple> FragmentInfo::load( single_fragment_info_vec_.back().meta()->array_schema(); // Get timestamp range - std::pair timestamp_range; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range(new_fragment_uri, ×tamp_range), - nullopt); - auto name = new_fragment_uri.remove_trailing_slash().last_path_part(); - auto fragment_version = utils::parse::get_fragment_version(name); + FragmentID fragment_id{new_fragment_uri}; + auto timestamp_range{fragment_id.timestamp_range()}; // Check if fragment is sparse bool sparse = false; - if (fragment_version <= 2) { + if (fragment_id.array_format_version() <= 2) { URI coords_uri = new_fragment_uri.join_path(constants::coords + constants::file_suffix); RETURN_NOT_OK_TUPLE(vfs.is_file(coords_uri, &sparse), nullopt); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index aca8c928dcff..b337e23bc4dc 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/fragment/fragment_metadata.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" @@ -53,7 +54,6 @@ #include "tiledb/sm/tile/tile.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/storage_format/serialization/serializers.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include "tiledb/type/range/range.h" #include @@ -749,18 +749,15 @@ std::vector> FragmentMetadata::load( auto status = parallel_for(&resources.compute_tp(), 0, fragment_num, [&](size_t f) { const auto& sf = fragments_to_load[f]; - URI coords_uri = sf.uri_.join_path(constants::coords + constants::file_suffix); - auto name = sf.uri_.remove_trailing_slash().last_path_part(); - auto format_version = utils::parse::get_fragment_version(name); - // Note that the fragment metadata version is >= the array schema // version. Therefore, the check below is defensive and will always // ensure backwards compatibility. shared_ptr metadata; - if (format_version <= 2) { + FragmentID fragment_id{sf.uri_}; + if (fragment_id.array_format_version() <= 2) { bool sparse; RETURN_NOT_OK(resources.vfs().is_file(coords_uri, &sparse)); metadata = make_shared( @@ -789,7 +786,7 @@ std::vector> FragmentMetadata::load( auto it = offsets.end(); if (metadata->format_version() >= 9) { - it = offsets.find(name); + it = offsets.find(fragment_id.name()); } else { it = offsets.find(sf.uri_.to_string()); } @@ -824,10 +821,8 @@ void FragmentMetadata::load( } // Get fragment name version - auto name = fragment_uri_.remove_trailing_slash().last_path_part(); - auto format_version = utils::parse::get_fragment_version(name); - - if (format_version <= 2) { + FragmentID fragment_id{fragment_uri_}; + if (fragment_id.array_format_version() <= 2) { return load_v1_v2(encryption_key, array_schemas); } else { return load_v3_or_higher( @@ -2159,8 +2154,8 @@ uint64_t FragmentMetadata::footer_size() const { void FragmentMetadata::get_footer_offset_and_size( uint64_t* offset, uint64_t* size) const { - auto name = fragment_uri_.remove_trailing_slash().last_path_part(); - auto fragment_format_version = utils::parse::get_fragment_version(name); + FragmentID fragment_id{fragment_uri_}; + auto fragment_format_version{fragment_id.array_format_version()}; auto all_fixed = array_schema_->domain().all_dims_fixed(); if (all_fixed && fragment_format_version < 5) { *size = footer_size_v3_v4(); diff --git a/tiledb/sm/fragment/test/CMakeLists.txt b/tiledb/sm/fragment/test/CMakeLists.txt new file mode 100644 index 000000000000..059bc089aefd --- /dev/null +++ b/tiledb/sm/fragment/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/sm/fragment/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test fragment) + this_target_sources(main.cc unit_fragment_identifier.cc) + this_target_link_libraries(fragment) +conclude(unit_test) diff --git a/tiledb/sm/fragment/test/compile_fragment_main.cc b/tiledb/sm/fragment/test/compile_fragment_main.cc new file mode 100644 index 000000000000..bc16f1378db2 --- /dev/null +++ b/tiledb/sm/fragment/test/compile_fragment_main.cc @@ -0,0 +1,40 @@ +/** + * @file compile_fragment_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../fragment_identifier.h" + +using namespace tiledb::sm; + +int main() { + FragmentID x(URI{}); + (void)x.name(); + (void)x.timestamp_range(); + (void)x.name_version(); + (void)x.array_format_version(); + return 0; +} \ No newline at end of file diff --git a/tiledb/sm/fragment/test/main.cc b/tiledb/sm/fragment/test/main.cc new file mode 100644 index 000000000000..f31c0efb18d1 --- /dev/null +++ b/tiledb/sm/fragment/test/main.cc @@ -0,0 +1,34 @@ +/** + * @file tiledb/sm/fragment/test/main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines a test `main()` + */ + +#define CATCH_CONFIG_MAIN +#include diff --git a/tiledb/sm/fragment/test/unit_fragment_identifier.cc b/tiledb/sm/fragment/test/unit_fragment_identifier.cc new file mode 100644 index 000000000000..1ea0286e5bd1 --- /dev/null +++ b/tiledb/sm/fragment/test/unit_fragment_identifier.cc @@ -0,0 +1,189 @@ +/** + * @file tiledb/sm/fragment/test/unit_fragment_identifier.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file tests the FragmentID class + */ + +#include +#include + +#include "tiledb/sm/fragment/fragment_identifier.h" + +using namespace tiledb::sm; + +const std::string frag_dir{"file:///"}; +struct failure_test_case { + std::string path; +}; + +struct success_test_case { + std::string path; // input + // outputs + std::string name; + timestamp_range_type timestamp_range; + int name_version; + format_version_t array_format_version; +}; + +TEST_CASE("FragmentID: Constructor: empty uri", "[fragment_id][empty_uri]") { + failure_test_case failure_case{""}; + REQUIRE_THROWS(FragmentID{failure_case.path}); +} + +// Should fail but succeed (known defects) +TEST_CASE( + "FragmentID: Constructor: Invalid uri", + "[!shouldfail][fragment_id][invalid_uri]") { + std::vector invalid_uris{{{""}, {"_"}, {"X"}}}; + + std::vector empty_fields{{ + {"__"}, // 2 underscores only, no fields + {"___"}, // 3 underscores only, all fields empty (version 1) + {"____"}, // 4 underscores only, all fields empty (versions 2, 3) + {"_____"} // 5 underscores, all fields are empty (version 3) + }}; + + // (Version 1) Expects: __uuid_t + std::vector two_fields{{ + {"__0123456789ABCDEF0123456789ABCDEF_1_"}, // __uuid_t_ + {"__1_0123456789ABCDEF0123456789ABCDEF"}, // __t_uuid + {"___0123456789ABCDEF0123456789ABCDEF1"}, // ___uuidt + {"_0123456789ABCDEF0123456789ABCDEF__1"}, // _uuid__t + {"0123456789ABCDEF0123456789ABCDEF___1"} // uuid___t + }}; + + // (Version 2, 3) Expects: __t1_t2_uuid + std::vector three_fields{{ + {"__1_2_0123456789ABCDEF0123456789ABCDEF_"}, // __t1_t2_uuid_ + {"__1_0123456789ABCDEF0123456789ABCDEF_2"}, // __t1_uuid_t2 + {"__2_0123456789ABCDEF0123456789ABCDEF_1"}, // __t2_uuid_t1 + {"____120123456789ABCDEF0123456789ABCDEF"}, // ____t1t2uuid + {"___1_20123456789ABCDEF0123456789ABCDEF"}, // ___t1_t2uuid + {"__1__20123456789ABCDEF0123456789ABCDEF"}, // __t1__t2uuid + {"_1___20123456789ABCDEF0123456789ABCDEF"}, // _t1___t2uuid + {"1____20123456789ABCDEF0123456789ABCDEF"}, // t1____t2uuid + {"1___2_0123456789ABCDEF0123456789ABCDEF"}, // t1___t2_uuid + {"1__2__0123456789ABCDEF0123456789ABCDEF"}, // t1__t2__uuid + {"1_2___0123456789ABCDEF0123456789ABCDEF"}, // t1_t2___uuid + {"12____0123456789ABCDEF0123456789ABCDEF"}, // t1t2____uuid + }}; + + // (Version 3) Expects: __t1_t2_uuid_v + std::vector four_fields{{ + {"__1_2_0123456789ABCDEF0123456789ABCDEF_5_"}, // __t1_t2_uuid_v_ + {"__1_2_5_0123456789ABCDEF0123456789ABCDEF"}, // __t1_t2_v_uuid + {"__1_0123456789ABCDEF0123456789ABCDEF_2_5"}, // __t1_uuid_t2_v + {"__0123456789ABCDEF0123456789ABCDEF_1_2_5"}, // __uuid_t1_t2_v + {"_____120123456789ABCDEF0123456789ABCDEF5"}, // _____t1t2uuidv + {"____1_20123456789ABCDEF0123456789ABCDEF5"}, // ____t1_t2uuidv + {"___1__20123456789ABCDEF0123456789ABCDEF5"}, // ___t1__t2uuidv + {"__1___20123456789ABCDEF0123456789ABCDEF5"}, // __t1___t2uuidv + {"_1____20123456789ABCDEF0123456789ABCDEF5"}, // _t1____t2uuidv + {"1____20123456789ABCDEF0123456789ABCDEF5"}, // t1_____t2uuidv + {"1___2_0123456789ABCDEF0123456789ABCDEF5"}, // t1____t2_uuidv + {"1__2__0123456789ABCDEF0123456789ABCDEF5"}, // t1__t2___uuidv + {"1_2___0123456789ABCDEF0123456789ABCDEF5"}, // t1_t2___uuidv + {"1_2__0123456789ABCDEF0123456789ABCDEF_5"}, // t1_t2__uuid_v + {"1_2_0123456789ABCDEF0123456789ABCDEF__5"}, // t1_t2_uuid__v + {"1_20123456789ABCDEF0123456789ABCDEF___5"}, // t1_t2uuid___v + }}; + + // Timestamps and uuid are identical + std::vector uuid_timestamps{{ + {"__0123456789ABCDEF0123456789ABCDEF_" + "0123456789ABCDEF0123456789ABCDEF"}, // version1: __uuid_t1 + {"__0123456789ABCDEF0123456789ABCDEF_" + "0123456789ABCDEF0123456789ABCDEF_" + "0123456789ABCDEF0123456789ABCDEF"}, // version1: __uuid_t1_t2, + // versions 2,3: + // __t1_t2_uuid + {"__0123456789ABCDEF0123456789ABCDEF_" + "0123456789ABCDEF0123456789ABCDEF_0123456789ABCDEF0123456789ABCDEF_" + "5"}, // version3: __t1_t2_uuid_v + }}; + + std::vector> failure_cases{ + invalid_uris, + empty_fields, + two_fields, + three_fields, + four_fields, + uuid_timestamps}; + + for (auto failure_case : failure_cases) { + for (auto failure : failure_case) { + auto uri = frag_dir + failure.path; + DYNAMIC_SECTION(uri) { + REQUIRE_THROWS(FragmentID{uri}); + } + } + } +} + +// Should succeed and do +TEST_CASE("FragmentID: Valid uris", "[fragment_id][valid_uri]") { + std::array success_cases{{ + {"file:///__0123456789ABCDEF0123456789ABCDEF_1", + "__0123456789ABCDEF0123456789ABCDEF_1", + std::pair{1, 1}, + 1, + 2}, + {"file:///__0123456789ABCDEF0123456789ABCDEF_1_2", + "__0123456789ABCDEF0123456789ABCDEF_1_2", + std::pair{2, 2}, + 1, + 2}, + {"file:///__0123456789ABCDEF0123456789ABCDEF_2_1", + "__0123456789ABCDEF0123456789ABCDEF_2_1", + std::pair{1, 1}, + 1, + 2}, + {"file:///__1_2_0123456789ABCDEF0123456789ABCDEF", + "__1_2_0123456789ABCDEF0123456789ABCDEF", + std::pair{1, 2}, + 2, + 4}, + {"file:///__1_2_0123456789ABCDEF0123456789ABCDEF_5", + "__1_2_0123456789ABCDEF0123456789ABCDEF_5", + std::pair{1, 2}, + 3, + 5}, + }}; + + for (auto success_case : success_cases) { + auto uri = success_case.path; + DYNAMIC_SECTION(uri) { + FragmentID f{uri}; + CHECK(f.name() == success_case.name); + CHECK(f.timestamp_range() == success_case.timestamp_range); + CHECK(f.name_version() == success_case.name_version); + CHECK(f.array_format_version() == success_case.array_format_version); + } + } +} diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc index d531c744a41e..e4107c22d421 100644 --- a/tiledb/sm/group/group_directory.cc +++ b/tiledb/sm/group/group_directory.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,10 +34,10 @@ #include "tiledb/common/logger.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/group/group_member.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb::common; @@ -243,12 +243,8 @@ GroupDirectory::compute_uris_to_vacuum(const std::vector& uris) const { std::unordered_set non_vac_uris_set; std::unordered_map uris_map; for (size_t i = 0; i < uris.size(); ++i) { - std::pair timestamp_range; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range(uris[i], ×tamp_range), - nullopt, - nullopt); - + FragmentID fragment_id{uris[i]}; + auto timestamp_range{fragment_id.timestamp_range()}; if (is_vacuum_file(uris[i])) { if (timestamp_range.first >= timestamp_start_ && timestamp_range.second <= timestamp_end_) @@ -337,9 +333,8 @@ GroupDirectory::compute_filtered_uris( // Add only URIs whose first timestamp is greater than or equal to the // timestamp_start and whose second timestamp is smaller than or equal to // the timestamp_end - std::pair timestamp_range; - RETURN_NOT_OK_TUPLE( - utils::parse::get_timestamp_range(uri, ×tamp_range), nullopt); + FragmentID fragment_id{uri}; + auto timestamp_range{fragment_id.timestamp_range()}; auto t1 = timestamp_range.first; auto t2 = timestamp_range.second; if (t1 >= timestamp_start_ && t2 <= timestamp_end_) diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index 7896d94361ec..e871dd2b2a21 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ #include "tiledb/sm/query/deletes_and_updates/deletes_and_updates.h" #include "tiledb/common/logger.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/generate_uri.h" @@ -129,12 +130,10 @@ Status DeletesAndUpdates::dowork() { // consolidated without timestamps. auto& frag_uris = array_->array_directory().unfiltered_fragment_uris(); for (auto& uri : frag_uris) { - auto name = uri.remove_trailing_slash().last_path_part(); - auto format_version = utils::parse::get_fragment_version(name); - if (format_version < constants::consolidation_with_timestamps_min_version) { - std::pair fragment_timestamp_range; - RETURN_NOT_OK( - utils::parse::get_timestamp_range(uri, &fragment_timestamp_range)); + FragmentID fragment_id{uri}; + if (fragment_id.array_format_version() < + constants::consolidation_with_timestamps_min_version) { + auto fragment_timestamp_range{fragment_id.timestamp_range()}; if (timestamp >= fragment_timestamp_range.first && timestamp <= fragment_timestamp_range.second) { throw DeleteAndUpdateStatusException( diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 80e02a5e9ce7..336668d86b18 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -35,10 +35,10 @@ #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_condition_combination_op.h" #include "tiledb/sm/enums/query_condition_op.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/fragment/fragment_metadata.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/query/readers/result_cell_slab.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include #include @@ -209,12 +209,8 @@ uint64_t QueryCondition::condition_timestamp() const { return 0; } - std::pair timestamps; - if (!utils::parse::get_timestamp_range(URI(condition_marker_), ×tamps) - .ok()) { - throw std::logic_error("Error parsing condition marker."); - } - + FragmentID fragment_id{condition_marker_}; + auto timestamps{fragment_id.timestamp_range()}; return timestamps.first; } diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 6e90fb3b7040..751899c547d1 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/filter/webp_filter.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/fragment/fragment_metadata.h" #include "tiledb/sm/misc/comparators.h" #include "tiledb/sm/misc/hilbert.h" @@ -213,8 +214,8 @@ WriterBase::WriterBase( auto frag_dir_uri = array_->array_directory().get_fragments_dir(write_version); fragment_uri_ = frag_dir_uri.join_path(new_fragment_str); - throw_if_not_ok(utils::parse::get_timestamp_range( - fragment_uri_, &fragment_timestamp_range_)); + FragmentID fragment_id{fragment_uri_}; + fragment_timestamp_range_ = fragment_id.timestamp_range(); } WriterBase::~WriterBase() { diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 2f54390eee2b..b0b7949accf7 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022-2023 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -31,7 +31,7 @@ include(object_library) # commence(object_library uri_format) this_target_sources(parse_uri.cc generate_uri.cc) - this_target_object_libraries(baseline time vfs) + this_target_object_libraries(baseline fragment time vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index 3506fe7e66d6..d823936c13d2 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/common/random/random_label.h" +#include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/storage_format/uri/parse_uri.h" @@ -68,9 +69,11 @@ std::string generate_timestamped_name( std::string generate_consolidated_fragment_name( const URI& first, const URI& last, format_version_t format_version) { // Get timestamp ranges - std::pair t_first, t_last; - throw_if_not_ok(utils::parse::get_timestamp_range(first, &t_first)); - throw_if_not_ok(utils::parse::get_timestamp_range(last, &t_last)); + tiledb::sm::FragmentID fragment_id_first{first}; + auto t_first{fragment_id_first.timestamp_range()}; + + tiledb::sm::FragmentID fragment_id_last{last}; + auto t_last{fragment_id_last.timestamp_range()}; return generate_timestamped_name( t_first.first, t_last.second, format_version); diff --git a/tiledb/storage_format/uri/parse_uri.cc b/tiledb/storage_format/uri/parse_uri.cc index af64319175a2..24d9896927a5 100644 --- a/tiledb/storage_format/uri/parse_uri.cc +++ b/tiledb/storage_format/uri/parse_uri.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,102 +30,14 @@ * This file contains functions for parsing URIs for storage of an array. */ -#include "parse_uri.h" #include #include // For sscanf PRId64 #include -namespace tiledb::sm::utils::parse { - -/** - * The possible fragment name versions. - */ -enum class FragmentNameVersion { ONE, TWO, THREE }; - -/** - * Retrieves the fragment name version. - * - Name version 1 corresponds to format versions 1 and 2 - * * __uuid_{_t2} - * - Name version 2 corresponds to format version 3 and 4 - * * __t1_t2_uuid - * - Name version 3 corresponds to format version 5 or higher - * * __t1_t2_uuid_version - */ -FragmentNameVersion get_fragment_name_version(const std::string& name) { - // Format Version 3: __t1_t2_uuid_version - size_t n = std::count(name.begin(), name.end(), '_'); - if (n == 5) { - return FragmentNameVersion::THREE; - } - - // Format Version 2: __t1_t2_uuid - auto maybe_uuid = name.substr(name.find_last_of('_') + 1); - if (maybe_uuid.size() == 32) { - return FragmentNameVersion::TWO; - } - - // Else, Format Version 1: __uuid_t1{_t2} - return FragmentNameVersion::ONE; -} - -Status get_timestamp_range( - const URI& uri, std::pair* timestamp_range) { - // Initializations - auto name = uri.remove_trailing_slash().last_path_part(); - *timestamp_range = {0, 0}; - - // Cut the suffix - auto pos = name.find_last_of('.'); - name = (pos == std::string::npos) ? name : name.substr(0, pos); - - // Get fragment name version - auto name_version = get_fragment_name_version(name); - - // FragmentNameVersion::ONE is equivalent to fragment format version <= 2 - if (name_version == FragmentNameVersion::ONE) { - assert(name.find_last_of('_') != std::string::npos); - auto t_str = name.substr(name.find_last_of('_') + 1); - sscanf( - t_str.c_str(), - (std::string("%") + std::string(PRId64)).c_str(), - (long long int*)×tamp_range->first); - timestamp_range->second = timestamp_range->first; - } else { - assert(name.find_last_of('_') != std::string::npos); - sscanf( - name.c_str(), - (std::string("__%") + std::string(PRId64) + "_%" + std::string(PRId64)) - .c_str(), - (long long int*)×tamp_range->first, - (long long int*)×tamp_range->second); - } - - if (timestamp_range->first > timestamp_range->second) { - throw std::logic_error( - "Error retrieving timestamp range from URI; start timestamp cannot " - "be after end timestamp"); - } - - return Status::Ok(); -} - -format_version_t get_fragment_version(const std::string& name) { - auto name_version = get_fragment_name_version(name); - - if (name_version == FragmentNameVersion::ONE) { - return 2; - } - - if (name_version == FragmentNameVersion::TWO) { - return 4; - } +#include "parse_uri.h" +#include "tiledb/common/exception/exception.h" - uint32_t ret; - auto version_str = name.substr(name.find_last_of('_') + 1); - std::stringstream ss(version_str); - ss >> ret; - return ret; -} +namespace tiledb::sm::utils::parse { bool is_element_of(const URI uri, const URI intersecting_uri) { std::string prefix = uri.to_string().substr( diff --git a/tiledb/storage_format/uri/parse_uri.h b/tiledb/storage_format/uri/parse_uri.h index 3013ddb0dcbb..07e3aff6d557 100644 --- a/tiledb/storage_format/uri/parse_uri.h +++ b/tiledb/storage_format/uri/parse_uri.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,21 +38,6 @@ namespace tiledb::sm::utils::parse { -/** - * Retrieves the timestamp range from the input - * URI. For format version <= 2, only the range start is valid - * (the range end is ignored). - */ -Status get_timestamp_range( - const URI& uri, std::pair* timestamp_range); - -/** - * Retrieves the fragment version. This will work only for - * name versions > 2, otherwise the function sets `version` - * to UINT32_MAX. - */ -format_version_t get_fragment_version(const std::string& name); - /** * Returns true if the given URIs have the same "prefix" and could * potentially intersect one another. diff --git a/tiledb/storage_format/uri/test/compile_uri_format_main.cc b/tiledb/storage_format/uri/test/compile_uri_format_main.cc index 5485f87e622e..adca02dc03e9 100644 --- a/tiledb/storage_format/uri/test/compile_uri_format_main.cc +++ b/tiledb/storage_format/uri/test/compile_uri_format_main.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,10 +38,6 @@ int main() { (void)generate_consolidated_fragment_name( tiledb::sm::URI{}, tiledb::sm::URI{}, 1); - std::pair x; - (void)get_timestamp_range(tiledb::sm::URI{}, &x); - (void)get_fragment_version(""); (void)is_element_of(tiledb::sm::URI{}, tiledb::sm::URI{}); - return 0; } diff --git a/tiledb/storage_format/uri/test/unit_uri_format.cc b/tiledb/storage_format/uri/test/unit_uri_format.cc index 3fc3246f566e..477a7eee3b91 100644 --- a/tiledb/storage_format/uri/test/unit_uri_format.cc +++ b/tiledb/storage_format/uri/test/unit_uri_format.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -126,103 +126,3 @@ TEST_CASE( "start timestamp cannot be after end timestamp")); } } - -TEST_CASE( - "StorageFormat: Parse: get_timestamp_range", - "[storage_format][parse][get_timestamp_range]") { - std::pair range; - - SECTION("name version 1") { - // Note: the end timestamp may or may not be present. - // If present, this version sets both range values to the end timestamp. - // As such, if the end_timestamp < start_timestamp, no error is thrown. - SECTION("valid timestamps") { - tiledb::sm::URI frag; - - SECTION("with end timestamp") { - // Create fragment at timestamp 1-2 - frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__44318efd44f546b18db13edc8d10805b_1_2"); - } - - SECTION("without end timestamp") { - // Create fragment at timestamp 2 - frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__44318efd44f546b18db13edc8d10805b_2"); - } - - // Check timestamp range - REQUIRE(get_timestamp_range(frag, &range).ok()); - CHECK(range.first == 2); - CHECK(range.second == 2); - } - - SECTION("invalid timestamps") { - // Create fragment at timestamp 2-1 - auto frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__44318efd44f546b18db13edc8d10805b_2_1"); - - // Check timestamp range - REQUIRE(get_timestamp_range(frag, &range).ok()); - CHECK(range.first == 1); - CHECK(range.second == 1); - } - } - - SECTION("name version 2") { - SECTION("valid timestamps") { - // Create fragment at timestamp 1-2 - auto frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__1_2_44318efd44f546b18db13edc8d10805b"); - - // Check timestamp range - REQUIRE(get_timestamp_range(frag, &range).ok()); - CHECK(range.first == 1); - CHECK(range.second == 2); - } - - SECTION("invalid timestamps") { - // Create fragment at timestamp 2-1 - auto frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__2_1_44318efd44f546b18db13edc8d10805b"); - - // Try to get timestamp range - REQUIRE_THROWS_WITH( - get_timestamp_range(frag, &range), - Catch::Matchers::ContainsSubstring( - "start timestamp cannot be after end timestamp")); - } - } - - SECTION("name version 3") { - SECTION("valid timestamps") { - // Create fragment at timestamp 1-2 - auto frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__1_2_44318efd44f546b18db13edc8d10805b_5"); - - // Check timestamp range - REQUIRE(get_timestamp_range(frag, &range).ok()); - CHECK(range.first == 1); - CHECK(range.second == 2); - } - - SECTION("invalid timestamps") { - // Create fragment at timestamp 2-1 - auto frag = tiledb::sm::URI( - "file:///array_name/__fragments/" - "__2_1_44318efd44f546b18db13edc8d10805b_5"); - - // Try to get timestamp range - REQUIRE_THROWS_WITH( - get_timestamp_range(frag, &range), - Catch::Matchers::ContainsSubstring( - "start timestamp cannot be after end timestamp")); - } - } -} From cab7c288ad77cdada1415d5d484eaf8cb6fe6ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Mon, 29 Jan 2024 19:38:45 +0100 Subject: [PATCH 164/456] Add CPack to CMakeLists.txt for binary and source package generation (#4645) Add CPack to CMakeLists.txt. All default package exports are enabled for tiledb project so the superbuild's `package` command generates outputs based on the current platform (TGZ for Linux). Usage: - *Superbuild* - `make package` invokes `cpack` inside of the EP tiledb. - *Build* - `cpack` for default behaviour or `cpack -G ` for specific output - `make package` - `make package_source` Example output from Superbuild's `make package`: ``` TGZ CPack: Create package using TGZ CPack: Install projects CPack: - Run preinstall target for: TileDB CPack: - Install project: TileDB [] CPack: Create package CPack: - package: /mnt/c/Users/dbara/CLionProjects/TileDB/build_superbuild/tiledb/tiledb-linux-x86_64-2.20.0.tar.gz generated. CPack: - checksum file: /mnt/c/Users/dbara/CLionProjects/TileDB/build_superbuild/tiledb/tiledb-linux-x86_64-2.20.0.tar.gz.sha1 generated. ``` --- TYPE: FEATURE DESC: Add CPack to CMakeLists.txt --- cmake/TileDB-Superbuild.cmake | 7 +++++++ cmake/inputs/CustomCPackOptions.cmake.in | 18 ++++++++++++++++++ tiledb/CMakeLists.txt | 18 ++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 cmake/inputs/CustomCPackOptions.cmake.in diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 62e07f9b7902..1a09e997439c 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -191,3 +191,10 @@ if (TILEDB_TESTS) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tiledb ) endif() + +# make package (.tar.gz for Linux and MacOS, .zip for Windows) +add_custom_target(package + DEPENDS tiledb + COMMAND ${CMAKE_CPACK_COMMAND} --config CPackConfig.cmake -G "$,ZIP,TGZ>" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tiledb +) \ No newline at end of file diff --git a/cmake/inputs/CustomCPackOptions.cmake.in b/cmake/inputs/CustomCPackOptions.cmake.in new file mode 100644 index 000000000000..60f201003463 --- /dev/null +++ b/cmake/inputs/CustomCPackOptions.cmake.in @@ -0,0 +1,18 @@ +set(CPACK_SOURCE_IGNORE_FILES ".*\\.git;.*build.*") + +set(CPACK_PACKAGE_VENDOR "TileDB Inc.") +set(CPACK_PACKAGE_VERSION "@TILEDB_VERSION@") + +set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) + +# Append NOAVX2 if needed +if(NOT ${COMPILER_SUPPORTS_AVX2}) + set(NOAVX2 "-noavx2") +endif() + +# Set output name +set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${NOAVX2}-${CPACK_PACKAGE_VERSION}") +string(TOLOWER ${CPACK_PACKAGE_FILE_NAME} CPACK_PACKAGE_FILE_NAME ) + +# Enable HASH +set(CPACK_PACKAGE_CHECKSUM "SHA256") \ No newline at end of file diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index f4a11f7c0521..06cf1186af6e 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -1053,3 +1053,21 @@ configure_file( ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tiledb.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +# Packaging configuration +configure_file ("${PROJECT_SOURCE_DIR}/cmake/inputs/CustomCPackOptions.cmake.in" + "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake" + @ONLY) +set (CPACK_PROJECT_CONFIG_FILE + "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake") + +# Not all options can be set in CustomCPackOptions.cmake +if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + set(CPACK_SOURCE_GENERATOR "ZIP") + set(CPACK_GENERATOR "ZIP") +else() + set(CPACK_SOURCE_GENERATOR "TGZ") + set(CPACK_GENERATOR "TGZ") +endif() + +include(CPack) From ee6c1bcba2bb0ab845719ae8c8ec00f8342f01b6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 30 Jan 2024 02:35:38 -0600 Subject: [PATCH 165/456] Ignore compile_commands.json (#4680) This is just a quality of life so that developers can symlink compile_commands.json to wherever they place their build directory. The reaoning for this is that a number of editor extensions use this file to configure their LSP interface for improved editor support. --- TYPE: NO_HISTORY DESC: Ignore compile_commands.json --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a62bc5943b7..051e2eb8e318 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ examples/obj/* scripts/deps-staging/* .idea *.DS_Store +compile_commands.json doc/venv doc/source/__pycache__ doc/source/_build From 6e17749bb09b3abd0caaa5e6f6b4c31527f52e9b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 30 Jan 2024 10:41:01 +0200 Subject: [PATCH 166/456] Update to clang-format 17. (#4675) [SC-39200](https://app.shortcut.com/tiledb-inc/story/39200/make-sure-that-clang-format-is-the-same-across-os-jobs) There are no formatting changes. --- TYPE: NO_HISTORY --- cmake/Modules/FindClangTools.cmake | 2 +- cmake/Modules/Format.cmake | 2 +- scripts/ci/check_formatting_linux.sh | 2 +- scripts/install-clangformat.sh | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/Modules/FindClangTools.cmake b/cmake/Modules/FindClangTools.cmake index 99245714df41..c0f94047aa6a 100644 --- a/cmake/Modules/FindClangTools.cmake +++ b/cmake/Modules/FindClangTools.cmake @@ -52,7 +52,7 @@ endif() find_program(CLANG_FORMAT_BIN NAMES - clang-format-16 + clang-format-17 clang-format PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin NO_DEFAULT_PATH diff --git a/cmake/Modules/Format.cmake b/cmake/Modules/Format.cmake index 81a9fbb3e3c7..33e8664632a5 100644 --- a/cmake/Modules/Format.cmake +++ b/cmake/Modules/Format.cmake @@ -32,7 +32,7 @@ set(SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts") find_package(ClangTools) if (NOT ${CLANG_FORMAT_FOUND}) - find_program(CLANG_FORMAT_BIN NAMES clang-format-16) + find_program(CLANG_FORMAT_BIN NAMES clang-format-17) if(CLANG_FORMAT_BIN) set(CLANG_FORMAT_FOUND TRUE) endif() diff --git a/scripts/ci/check_formatting_linux.sh b/scripts/ci/check_formatting_linux.sh index 9eb4563a6b57..99b40e1e331a 100644 --- a/scripts/ci/check_formatting_linux.sh +++ b/scripts/ci/check_formatting_linux.sh @@ -35,4 +35,4 @@ sudo ./scripts/install-clangformat.sh src=$GITHUB_WORKSPACE cd $src -$src/scripts/run-clang-format.sh $src clang-format-16 0 +$src/scripts/run-clang-format.sh $src clang-format-17 0 diff --git a/scripts/install-clangformat.sh b/scripts/install-clangformat.sh index 3bb230b8a5ff..64d48d2464f3 100755 --- a/scripts/install-clangformat.sh +++ b/scripts/install-clangformat.sh @@ -29,9 +29,9 @@ die() { } install_apt_pkg() { - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main' && - apt-get update -qq && apt-get install -qq -y clang-format-16 + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc + add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main' && + apt-get update -qq && apt-get install -qq -y clang-format-17 } install_brew_pkg() { From 15787557d00410d97c6d51e1a1fe7352651a3ee2 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 30 Jan 2024 02:41:33 -0600 Subject: [PATCH 167/456] Add InsertNewlineAtEOF: true to .clang-format (#4681) This is a new option to always insert newlines when formatting C++ source files. This is new as of clang-format 16 which we always include. --- TYPE: NO_HISTORY DESC: Add InsertNewlineAtEOF: true to .clang-format --- .clang-format | 1 + examples/c_api/query_condition_sparse.c | 2 +- examples/cpp_api/query_condition_sparse.cc | 2 +- experimental/test/compile_experimental_main.cc | 2 +- experimental/tiledb/common/dag/edge/test/unit_block_edge.cc | 2 +- experimental/tiledb/common/dag/execution/test/unit_duffs.h | 2 +- .../tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc | 2 +- test/regression/targets/sc-18836.cc | 2 +- test/src/cpp-integration-query-condition.cc | 2 +- test/src/test-capi-consolidation-plan.cc | 2 +- test/src/test-cppapi-aggregates.cc | 2 +- test/src/unit-capi-consolidation.cc | 2 +- test/src/unit-capi-fragment_info.cc | 2 +- test/src/unit-capi-partial-attribute-write.cc | 2 +- test/src/unit-cppapi-array.cc | 2 +- test/src/unit-cppapi-datetimes.cc | 2 +- test/src/unit-cppapi-partial-attribute-write.cc | 2 +- test/src/unit-cppapi-schema.cc | 2 +- test/src/unit-cppapi-time.cc | 2 +- test/src/unit-cppapi-update-queries.cc | 2 +- test/src/unit-ctx.cc | 2 +- test/src/unit-dense-reader.cc | 2 +- test/src/unit-duplicates.cc | 2 +- test/src/unit-result-coords.cc | 2 +- test/src/unit-result-tile.cc | 2 +- test/src/unit-sparse-global-order-reader.cc | 2 +- test/src/unit-tile-metadata.cc | 2 +- tiledb/api/c_api/config/config_api.cc | 2 +- tiledb/api/c_api/domain/domain_api_external.h | 2 +- tiledb/api/c_api/domain/domain_api_internal.h | 2 +- tiledb/api/c_api/vfs/vfs_api_experimental.h | 2 +- tiledb/api/c_api/vfs/vfs_api_external.h | 2 +- tiledb/api/c_api_support/argument_validation.h | 2 +- tiledb/api/c_api_support/c_api_support.h | 2 +- tiledb/api/c_api_support/exception_wrapper/exception_wrapper.cc | 2 +- .../exception_wrapper/test/compile_exception_wrapper_main.cc | 2 +- tiledb/api/c_api_support/handle/handle.cc | 2 +- .../storage_manager_stub/storage_manager_stub.cc | 2 +- tiledb/common/interval/interval.cc | 2 +- tiledb/common/random/random_label.cc | 2 +- tiledb/common/stdx_string.cc | 2 +- tiledb/common/thread_pool/test/compile_thread_pool_main.cc | 2 +- tiledb/common/types/test/compile_datum_main.cc | 2 +- tiledb/platform/platform.h | 2 +- tiledb/sm/array/test/compile_array_main.cc | 2 +- tiledb/sm/array_schema/test/compile_attribute_main.cc | 2 +- tiledb/sm/array_schema/test/compile_dimension_main.cc | 2 +- tiledb/sm/array_schema/test/compile_domain_main.cc | 2 +- tiledb/sm/array_schema/test/unit_dimension.cc | 2 +- tiledb/sm/buffer/test/compile_buffer_list_main.cc | 2 +- tiledb/sm/buffer/test/compile_buffer_main.cc | 2 +- tiledb/sm/c_api/api_argument_validator.h | 2 +- tiledb/sm/compressors/test/compile_compressors_main.cc | 2 +- tiledb/sm/compressors/test/unit_delta_compressor.cc | 2 +- tiledb/sm/compressors/test/unit_dict_compressor.cc | 2 +- tiledb/sm/config/test/compile_config_main.cc | 2 +- tiledb/sm/cpp_api/vfs_experimental.h | 2 +- tiledb/sm/crypto/test/compile_crypto_main.cc | 2 +- .../s3/STSProfileWithWebIdentityCredentialsProvider.cc | 2 +- .../s3/STSProfileWithWebIdentityCredentialsProvider.h | 2 +- tiledb/sm/filesystem/test/compile_vfs_main.cc | 2 +- tiledb/sm/filter/filter_create.cc | 2 +- tiledb/sm/filter/float_scaling_filter.cc | 2 +- tiledb/sm/filter/float_scaling_filter.h | 2 +- tiledb/sm/filter/test/compile_all_filters_main.cc | 2 +- tiledb/sm/filter/test/compile_bitshuffle_filter_main.cc | 2 +- tiledb/sm/filter/test/compile_byteshuffle_filter_main.cc | 2 +- tiledb/sm/filter/test/compile_checksum_filters_main.cc | 2 +- tiledb/sm/filter/test/compile_compression_filter_main.cc | 2 +- tiledb/sm/filter/test/compile_encryption_filters_main.cc | 2 +- tiledb/sm/filter/test/compile_filter_main.cc | 2 +- tiledb/sm/filter/test/compile_filter_pipeline_main.cc | 2 +- tiledb/sm/filter/test/compile_float_scaling_filter_main.cc | 2 +- tiledb/sm/filter/test/compile_xor_filter_main.cc | 2 +- tiledb/sm/filter/test/unit_filter_create.cc | 2 +- tiledb/sm/filter/test/unit_filter_pipeline.cc | 2 +- tiledb/sm/filter/test/unit_float_scale_input_validation.cc | 2 +- tiledb/sm/filter/webp_filter.cc | 2 +- tiledb/sm/filter/xor_filter.cc | 2 +- tiledb/sm/filter/xor_filter.h | 2 +- tiledb/sm/fragment/test/compile_fragment_main.cc | 2 +- tiledb/sm/group/test/compile_group_main.cc | 2 +- tiledb/sm/metadata/test/compile_metadata_main.cc | 2 +- tiledb/sm/query/ast/test/compile_query_ast_main.cc | 2 +- .../sm/query/deletes_and_updates/test/unit_delete_condition.cc | 2 +- tiledb/sm/query/readers/aggregators/operation.cc | 2 +- .../query/readers/aggregators/test/compile_aggregators_main.cc | 2 +- tiledb/sm/query/readers/test/unit_reader_base.cc | 2 +- tiledb/sm/rtree/test/compile_rtree_main.cc | 2 +- tiledb/sm/serialization/array_schema_evolution.h | 2 +- tiledb/sm/serialization/fragment_metadata.h | 2 +- tiledb/sm/serialization/query_aggregates.cc | 2 +- tiledb/sm/serialization/query_aggregates.h | 2 +- tiledb/sm/stats/duration_instrument.h | 2 +- tiledb/sm/stats/test/compile_stats_main.cc | 2 +- tiledb/sm/tile/test/compile_tile_main.cc | 2 +- 96 files changed, 96 insertions(+), 95 deletions(-) diff --git a/.clang-format b/.clang-format index 94da8b24d049..3cb129336a41 100644 --- a/.clang-format +++ b/.clang-format @@ -64,6 +64,7 @@ IndentCaseLabels: true IndentRequiresClause: true IndentWidth: 2 IndentWrappedFunctionNames: false +InsertNewlineAtEOF: true JavaScriptQuotes: Leave JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false diff --git a/examples/c_api/query_condition_sparse.c b/examples/c_api/query_condition_sparse.c index 9c0677716195..3478e7bc8f2f 100644 --- a/examples/c_api/query_condition_sparse.c +++ b/examples/c_api/query_condition_sparse.c @@ -361,4 +361,4 @@ int main() { tiledb_ctx_free(&ctx); return 0; -} \ No newline at end of file +} diff --git a/examples/cpp_api/query_condition_sparse.cc b/examples/cpp_api/query_condition_sparse.cc index 6596d29f9813..af4fd8506b6f 100644 --- a/examples/cpp_api/query_condition_sparse.cc +++ b/examples/cpp_api/query_condition_sparse.cc @@ -289,4 +289,4 @@ int main() { vfs.remove_dir(array_name); return 0; -} \ No newline at end of file +} diff --git a/experimental/test/compile_experimental_main.cc b/experimental/test/compile_experimental_main.cc index 9bb219864866..79123668cff6 100644 --- a/experimental/test/compile_experimental_main.cc +++ b/experimental/test/compile_experimental_main.cc @@ -28,4 +28,4 @@ int main() { return 0; -} \ No newline at end of file +} diff --git a/experimental/tiledb/common/dag/edge/test/unit_block_edge.cc b/experimental/tiledb/common/dag/edge/test/unit_block_edge.cc index b6899cf20620..b4ed8c0e2f6a 100644 --- a/experimental/tiledb/common/dag/edge/test/unit_block_edge.cc +++ b/experimental/tiledb/common/dag/edge/test/unit_block_edge.cc @@ -41,4 +41,4 @@ */ TEST_CASE("BlockEdge: Trivial test", "[block_edge]") { CHECK(true); -} \ No newline at end of file +} diff --git a/experimental/tiledb/common/dag/execution/test/unit_duffs.h b/experimental/tiledb/common/dag/execution/test/unit_duffs.h index 46e6dd6410da..ab1f42c45237 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_duffs.h +++ b/experimental/tiledb/common/dag/execution/test/unit_duffs.h @@ -31,4 +31,4 @@ #ifndef TILEDB_UNIT_DUFFS_H #define TILEDB_UNIT_DUFFS_H #include -#endif // TILEDB_UNIT_DUFFS_H \ No newline at end of file +#endif // TILEDB_UNIT_DUFFS_H diff --git a/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc b/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc index ca6a78ccc8c2..8b956cae2306 100644 --- a/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc +++ b/experimental/tiledb/common/dag/nodes/test/unit_segmented_mimo_nodes.cc @@ -1103,4 +1103,4 @@ TEMPLATE_TEST_CASE( // Repeat one of the tests above but with stop token -#endif \ No newline at end of file +#endif diff --git a/test/regression/targets/sc-18836.cc b/test/regression/targets/sc-18836.cc index e3609fb9bd4c..8906f3ae8f6d 100644 --- a/test/regression/targets/sc-18836.cc +++ b/test/regression/targets/sc-18836.cc @@ -116,4 +116,4 @@ TEST_CASE( if (vfs.is_dir(array_name)) vfs.remove_dir(array_name); -} \ No newline at end of file +} diff --git a/test/src/cpp-integration-query-condition.cc b/test/src/cpp-integration-query-condition.cc index a691d665ab2e..d18e2e428727 100644 --- a/test/src/cpp-integration-query-condition.cc +++ b/test/src/cpp-integration-query-condition.cc @@ -2640,4 +2640,4 @@ TEST_CASE( if (vfs.is_dir(array_name)) { vfs.remove_dir(array_name); } -} \ No newline at end of file +} diff --git a/test/src/test-capi-consolidation-plan.cc b/test/src/test-capi-consolidation-plan.cc index 7efd70b471a7..bce48e219d1d 100644 --- a/test/src/test-capi-consolidation-plan.cc +++ b/test/src/test-capi-consolidation-plan.cc @@ -255,4 +255,4 @@ TEST_CASE_METHOD( tiledb_consolidation_plan_free_json_str(&str); tiledb_consolidation_plan_free(&consolidation_plan); -} \ No newline at end of file +} diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index aa5a313a732e..868592e1752a 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -2663,4 +2663,4 @@ TEST_CASE_METHOD( CHECK_THROWS( QueryExperimental::create_unary_aggregate(query, "a1")); CHECK_THROWS(default_channel.apply_aggregate("Something", operation)); -} \ No newline at end of file +} diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index b6ecef3a88c0..19bbe334bc87 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -7243,4 +7243,4 @@ TEST_CASE_METHOD( check_commits_dir_sparse(1, 0, 1); remove_sparse_array(); -} \ No newline at end of file +} diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index 57bda54927e1..f9e3f615f065 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -2180,4 +2180,4 @@ TEST_CASE( tiledb_vfs_free(&vfs); tiledb_error_free(&error); tiledb_config_free(&config); -} \ No newline at end of file +} diff --git a/test/src/unit-capi-partial-attribute-write.cc b/test/src/unit-capi-partial-attribute-write.cc index e75c3872acd6..07264040f5e7 100644 --- a/test/src/unit-capi-partial-attribute-write.cc +++ b/test/src/unit-capi-partial-attribute-write.cc @@ -284,4 +284,4 @@ TEST_CASE_METHOD( CHECK(dim2 == std::vector({1, 2, 4, 3, 1, 2, 3, 4})); remove_array(); -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index c25ada51c8a5..bab2af037340 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -2203,4 +2203,4 @@ TEST_CASE( } CHECK(i > 0); -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-datetimes.cc b/test/src/unit-cppapi-datetimes.cc index d0a960328f50..898ffb7dd06a 100644 --- a/test/src/unit-cppapi-datetimes.cc +++ b/test/src/unit-cppapi-datetimes.cc @@ -142,4 +142,4 @@ TEST_CASE("C++ API: Datetime dimension", "[cppapi][datetime]") { if (vfs.is_dir(array_name)) vfs.remove_dir(array_name); -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-partial-attribute-write.cc b/test/src/unit-cppapi-partial-attribute-write.cc index de2df05db657..c62ebd8c8d0c 100644 --- a/test/src/unit-cppapi-partial-attribute-write.cc +++ b/test/src/unit-cppapi-partial-attribute-write.cc @@ -503,4 +503,4 @@ TEST_CASE_METHOD( CHECK(dim2 == std::vector({0, 0, 0, 0, 0, 0, 0, 0})); remove_array(); -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-schema.cc b/test/src/unit-cppapi-schema.cc index f1cd258296fd..6dc878aa9f4f 100644 --- a/test/src/unit-cppapi-schema.cc +++ b/test/src/unit-cppapi-schema.cc @@ -367,4 +367,4 @@ TEST_CASE( CHECK_NOTHROW(tiledb::Dimension::create( ctx, "d1", TILEDB_UINT64, domain, &tile_extent)); } -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-time.cc b/test/src/unit-cppapi-time.cc index 04e8e56cfa11..592d45c89446 100644 --- a/test/src/unit-cppapi-time.cc +++ b/test/src/unit-cppapi-time.cc @@ -163,4 +163,4 @@ TEST_CASE("C++ API: Time dimension", "[cppapi][time]") { if (vfs.is_dir(array_name)) vfs.remove_dir(array_name); } -} \ No newline at end of file +} diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index 76d9cbbe68e4..60e13b631d07 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -310,4 +310,4 @@ TEST_CASE_METHOD( check_update_conditions({qc, qc2}, {uvs, uvs2}, 4, encrypt); remove_sparse_array(); -} \ No newline at end of file +} diff --git a/test/src/unit-ctx.cc b/test/src/unit-ctx.cc index af876ecfb510..42b28ef8becf 100644 --- a/test/src/unit-ctx.cc +++ b/test/src/unit-ctx.cc @@ -95,4 +95,4 @@ TEST_CASE("C++ API: Test context tags", "[cppapi][ctx-tags]") { REQUIRE_NOTHROW(ctx.set_tag("tag1", "value3")); REQUIRE(sm->tags().size() == 4); REQUIRE(sm->tags().at("tag1") == "value3"); -} \ No newline at end of file +} diff --git a/test/src/unit-dense-reader.cc b/test/src/unit-dense-reader.cc index ae8e4563720e..51d61cf3ee46 100644 --- a/test/src/unit-dense-reader.cc +++ b/test/src/unit-dense-reader.cc @@ -1422,4 +1422,4 @@ TEST_CASE_METHOD( CHECK(!std::memcmp(c_data_b, data_r_b, c_data_b_size)); CHECK(offsets_r_b_size == c_offsets_b_size); CHECK(!std::memcmp(c_offsets_b, offsets_r_b, c_offsets_b_size)); -} \ No newline at end of file +} diff --git a/test/src/unit-duplicates.cc b/test/src/unit-duplicates.cc index b28c8a437ad0..4ba0745cac5d 100644 --- a/test/src/unit-duplicates.cc +++ b/test/src/unit-duplicates.cc @@ -730,4 +730,4 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); -} \ No newline at end of file +} diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index a57957825bf6..8e0044401b74 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -263,4 +263,4 @@ TEST_CASE_METHOD( REQUIRE(rc2.advance_to_next_cell() == true); REQUIRE(rc2.pos_ == 2); REQUIRE(rc2.advance_to_next_cell() == false); -} \ No newline at end of file +} diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index 542f34570ccb..77677e6a1d3c 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -424,4 +424,4 @@ TEST_CASE_METHOD( num_cells); CHECK(memcmp(result_count.data(), exp_result_count.data(), num_cells) == 0); -} \ No newline at end of file +} diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 0cdf23cac4a6..582f59268ad4 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -1588,4 +1588,4 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); -} \ No newline at end of file +} diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index f5853379ce0d..89281332f040 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -1652,4 +1652,4 @@ TEST_CASE_METHOD( create_array(); write_fragment(); check_metadata(); -} \ No newline at end of file +} diff --git a/tiledb/api/c_api/config/config_api.cc b/tiledb/api/c_api/config/config_api.cc index 4bd2819dfff0..37c978f712f2 100644 --- a/tiledb/api/c_api/config/config_api.cc +++ b/tiledb/api/c_api/config/config_api.cc @@ -303,4 +303,4 @@ CAPI_INTERFACE( tiledb_error_t** error) { return api_entry_error( error, config_iter, done); -} \ No newline at end of file +} diff --git a/tiledb/api/c_api/domain/domain_api_external.h b/tiledb/api/c_api/domain/domain_api_external.h index 4ea7eec0cf2b..8be1b37a5373 100644 --- a/tiledb/api/c_api/domain/domain_api_external.h +++ b/tiledb/api/c_api/domain/domain_api_external.h @@ -231,4 +231,4 @@ TILEDB_EXPORT int32_t tiledb_domain_dump( } #endif -#endif \ No newline at end of file +#endif diff --git a/tiledb/api/c_api/domain/domain_api_internal.h b/tiledb/api/c_api/domain/domain_api_internal.h index e54ada5ea41a..76a427ce8ace 100644 --- a/tiledb/api/c_api/domain/domain_api_internal.h +++ b/tiledb/api/c_api/domain/domain_api_internal.h @@ -126,4 +126,4 @@ inline void ensure_domain_is_valid(const tiledb_domain_handle_t* h) { ensure_handle_is_valid(h); } -#endif \ No newline at end of file +#endif diff --git a/tiledb/api/c_api/vfs/vfs_api_experimental.h b/tiledb/api/c_api/vfs/vfs_api_experimental.h index 655a4a84138e..b70f97629c7d 100644 --- a/tiledb/api/c_api/vfs/vfs_api_experimental.h +++ b/tiledb/api/c_api/vfs/vfs_api_experimental.h @@ -99,4 +99,4 @@ TILEDB_EXPORT capi_return_t tiledb_vfs_ls_recursive( } #endif -#endif // TILEDB_VFS_API_EXPERIMENTAL_H \ No newline at end of file +#endif // TILEDB_VFS_API_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/vfs/vfs_api_external.h b/tiledb/api/c_api/vfs/vfs_api_external.h index 32478eba3bb9..435c9d8522a4 100644 --- a/tiledb/api/c_api/vfs/vfs_api_external.h +++ b/tiledb/api/c_api/vfs/vfs_api_external.h @@ -660,4 +660,4 @@ TILEDB_EXPORT capi_return_t tiledb_vfs_touch( } #endif -#endif // TILEDB_CAPI_VFS_EXTERNAL_H \ No newline at end of file +#endif // TILEDB_CAPI_VFS_EXTERNAL_H diff --git a/tiledb/api/c_api_support/argument_validation.h b/tiledb/api/c_api_support/argument_validation.h index 710509f21d2b..94fd6f782470 100644 --- a/tiledb/api/c_api_support/argument_validation.h +++ b/tiledb/api/c_api_support/argument_validation.h @@ -75,4 +75,4 @@ inline void ensure_output_pointer_is_valid(void* p) { } // namespace tiledb::api -#endif // TILEDB_CAPI_SUPPORT_ARGUMENT_VALIDATION_H \ No newline at end of file +#endif // TILEDB_CAPI_SUPPORT_ARGUMENT_VALIDATION_H diff --git a/tiledb/api/c_api_support/c_api_support.h b/tiledb/api/c_api_support/c_api_support.h index 4ab9dbeb6e05..5113e9ec4e50 100644 --- a/tiledb/api/c_api_support/c_api_support.h +++ b/tiledb/api/c_api_support/c_api_support.h @@ -43,4 +43,4 @@ #include "capi_function_override.h" #endif -#endif // TILEDB_CAPI_SUPPORT_H \ No newline at end of file +#endif // TILEDB_CAPI_SUPPORT_H diff --git a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.cc b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.cc index 7d00f4b44a04..c9e475ce9831 100644 --- a/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.cc +++ b/tiledb/api/c_api_support/exception_wrapper/exception_wrapper.cc @@ -91,4 +91,4 @@ void ETVisitorStdException::item(const std::exception& e) noexcept { } } -} // namespace tiledb::api::detail \ No newline at end of file +} // namespace tiledb::api::detail diff --git a/tiledb/api/c_api_support/exception_wrapper/test/compile_exception_wrapper_main.cc b/tiledb/api/c_api_support/exception_wrapper/test/compile_exception_wrapper_main.cc index 15dd6ef6ced7..88d40af1a315 100644 --- a/tiledb/api/c_api_support/exception_wrapper/test/compile_exception_wrapper_main.cc +++ b/tiledb/api/c_api_support/exception_wrapper/test/compile_exception_wrapper_main.cc @@ -33,4 +33,4 @@ int main() { tiledb::api::detail::ETVisitorStdException v{}; x.visit(v); return 1; -} \ No newline at end of file +} diff --git a/tiledb/api/c_api_support/handle/handle.cc b/tiledb/api/c_api_support/handle/handle.cc index 29b222755209..35e7cda527a6 100644 --- a/tiledb/api/c_api_support/handle/handle.cc +++ b/tiledb/api/c_api_support/handle/handle.cc @@ -30,4 +30,4 @@ #include "handle.h" -namespace tiledb::api {} // namespace tiledb::api \ No newline at end of file +namespace tiledb::api {} // namespace tiledb::api diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_stub.cc b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_stub.cc index ed5e579f30b5..f0f67f57db32 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_stub.cc +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_stub.cc @@ -28,4 +28,4 @@ * @section DESCRIPTION * * This file has definitions for `class StorageManagerStub`. - */ \ No newline at end of file + */ diff --git a/tiledb/common/interval/interval.cc b/tiledb/common/interval/interval.cc index 978aac9c6e17..b1a7214420cc 100644 --- a/tiledb/common/interval/interval.cc +++ b/tiledb/common/interval/interval.cc @@ -32,4 +32,4 @@ #include "interval.h" -namespace tiledb::common::detail {} // namespace tiledb::common::detail \ No newline at end of file +namespace tiledb::common::detail {} // namespace tiledb::common::detail diff --git a/tiledb/common/random/random_label.cc b/tiledb/common/random/random_label.cc index e636c539110d..cc962b94d0ab 100644 --- a/tiledb/common/random/random_label.cc +++ b/tiledb/common/random/random_label.cc @@ -60,4 +60,4 @@ std::string random_label() { return ss.str(); } -} // namespace tiledb::common \ No newline at end of file +} // namespace tiledb::common diff --git a/tiledb/common/stdx_string.cc b/tiledb/common/stdx_string.cc index 4bc975de3591..147017c6ae3b 100644 --- a/tiledb/common/stdx_string.cc +++ b/tiledb/common/stdx_string.cc @@ -54,4 +54,4 @@ size_t common_prefix_size( return size; } -} // namespace tiledb::stdx::string \ No newline at end of file +} // namespace tiledb::stdx::string diff --git a/tiledb/common/thread_pool/test/compile_thread_pool_main.cc b/tiledb/common/thread_pool/test/compile_thread_pool_main.cc index 023a720417fc..f0ee3476dca8 100644 --- a/tiledb/common/thread_pool/test/compile_thread_pool_main.cc +++ b/tiledb/common/thread_pool/test/compile_thread_pool_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::common::ThreadPool); return 0; -} \ No newline at end of file +} diff --git a/tiledb/common/types/test/compile_datum_main.cc b/tiledb/common/types/test/compile_datum_main.cc index 734fa20fb23c..0594c169e6d1 100644 --- a/tiledb/common/types/test/compile_datum_main.cc +++ b/tiledb/common/types/test/compile_datum_main.cc @@ -33,4 +33,4 @@ int main() { (void)sizeof(tiledb::common::UntypedDatumView); (void)sizeof(tiledb::common::DynamicTypedDatumView); return 0; -} \ No newline at end of file +} diff --git a/tiledb/platform/platform.h b/tiledb/platform/platform.h index 102064ae393c..2214acafeb6b 100644 --- a/tiledb/platform/platform.h +++ b/tiledb/platform/platform.h @@ -50,4 +50,4 @@ constexpr bool is_os_linux = true; #endif // _WIN32 } // namespace tiledb::platform -#endif // TILEDB_PLATFORM_H \ No newline at end of file +#endif // TILEDB_PLATFORM_H diff --git a/tiledb/sm/array/test/compile_array_main.cc b/tiledb/sm/array/test/compile_array_main.cc index 759247d1ac71..e5bc86118991 100644 --- a/tiledb/sm/array/test/compile_array_main.cc +++ b/tiledb/sm/array/test/compile_array_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::ArrayDirectory); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/array_schema/test/compile_attribute_main.cc b/tiledb/sm/array_schema/test/compile_attribute_main.cc index aa1dfb6c5fa0..2697028066c6 100644 --- a/tiledb/sm/array_schema/test/compile_attribute_main.cc +++ b/tiledb/sm/array_schema/test/compile_attribute_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Attribute); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/array_schema/test/compile_dimension_main.cc b/tiledb/sm/array_schema/test/compile_dimension_main.cc index fe62cc333ad3..84784bc36de4 100644 --- a/tiledb/sm/array_schema/test/compile_dimension_main.cc +++ b/tiledb/sm/array_schema/test/compile_dimension_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Dimension); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/array_schema/test/compile_domain_main.cc b/tiledb/sm/array_schema/test/compile_domain_main.cc index 4b9399730ea5..2d06c8da1d3d 100644 --- a/tiledb/sm/array_schema/test/compile_domain_main.cc +++ b/tiledb/sm/array_schema/test/compile_domain_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Domain); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/array_schema/test/unit_dimension.cc b/tiledb/sm/array_schema/test/unit_dimension.cc index 55e246c08b6c..5ae3e3f12737 100644 --- a/tiledb/sm/array_schema/test/unit_dimension.cc +++ b/tiledb/sm/array_schema/test/unit_dimension.cc @@ -438,4 +438,4 @@ TEST_CASE("Dimension::oob format") { error == "Coordinate -682.75 is out of domain bounds [-682.73999, 929.42999] on " "dimension 'X'"); -} \ No newline at end of file +} diff --git a/tiledb/sm/buffer/test/compile_buffer_list_main.cc b/tiledb/sm/buffer/test/compile_buffer_list_main.cc index 70a44124bfad..fa69bb5307d6 100644 --- a/tiledb/sm/buffer/test/compile_buffer_list_main.cc +++ b/tiledb/sm/buffer/test/compile_buffer_list_main.cc @@ -32,4 +32,4 @@ int main() { tiledb::sm::BufferList(); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/buffer/test/compile_buffer_main.cc b/tiledb/sm/buffer/test/compile_buffer_main.cc index 01532c683bf4..4e2fc6c00c45 100644 --- a/tiledb/sm/buffer/test/compile_buffer_main.cc +++ b/tiledb/sm/buffer/test/compile_buffer_main.cc @@ -31,4 +31,4 @@ int main() { tiledb::sm::Buffer(); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/c_api/api_argument_validator.h b/tiledb/sm/c_api/api_argument_validator.h index cda354585d60..665420aa3724 100644 --- a/tiledb/sm/c_api/api_argument_validator.h +++ b/tiledb/sm/c_api/api_argument_validator.h @@ -171,4 +171,4 @@ inline int32_t sanity_check( return save_error(ctx, _s); \ }() -#endif // TILEDB_CAPI_HELPERS_H \ No newline at end of file +#endif // TILEDB_CAPI_HELPERS_H diff --git a/tiledb/sm/compressors/test/compile_compressors_main.cc b/tiledb/sm/compressors/test/compile_compressors_main.cc index 0b520262ec51..9cd586af2c13 100644 --- a/tiledb/sm/compressors/test/compile_compressors_main.cc +++ b/tiledb/sm/compressors/test/compile_compressors_main.cc @@ -43,4 +43,4 @@ int main() { (void)sizeof(tiledb::sm::RLE); (void)sizeof(tiledb::sm::ZStd); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/compressors/test/unit_delta_compressor.cc b/tiledb/sm/compressors/test/unit_delta_compressor.cc index 66dd6add00d1..1dbfa55f43ed 100644 --- a/tiledb/sm/compressors/test/unit_delta_compressor.cc +++ b/tiledb/sm/compressors/test/unit_delta_compressor.cc @@ -97,4 +97,4 @@ TEST_CASE("Test delta decompression of a vector", "[decompression][delta]") { std::vector expected{0, 1, 1, 15, 3, 0, 2, 7, 1}; CHECK(uncompressed == expected); -} \ No newline at end of file +} diff --git a/tiledb/sm/compressors/test/unit_dict_compressor.cc b/tiledb/sm/compressors/test/unit_dict_compressor.cc index 6e145154f205..08ec199c3161 100644 --- a/tiledb/sm/compressors/test/unit_dict_compressor.cc +++ b/tiledb/sm/compressors/test/unit_dict_compressor.cc @@ -429,4 +429,4 @@ TEST_CASE( for (uint32_t i = 0; i < expected_offsets.size(); i++) { CHECK(expected_offsets[i] == decompressed_offsets[i]); } -} \ No newline at end of file +} diff --git a/tiledb/sm/config/test/compile_config_main.cc b/tiledb/sm/config/test/compile_config_main.cc index bebcd7802157..389fb5a08847 100644 --- a/tiledb/sm/config/test/compile_config_main.cc +++ b/tiledb/sm/config/test/compile_config_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Config); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/cpp_api/vfs_experimental.h b/tiledb/sm/cpp_api/vfs_experimental.h index bb51b6e93a48..149b27afca84 100644 --- a/tiledb/sm/cpp_api/vfs_experimental.h +++ b/tiledb/sm/cpp_api/vfs_experimental.h @@ -202,4 +202,4 @@ class VFSExperimental { }; } // namespace tiledb -#endif // TILEDB_VFS_EXPERIMENTAL_H \ No newline at end of file +#endif // TILEDB_VFS_EXPERIMENTAL_H diff --git a/tiledb/sm/crypto/test/compile_crypto_main.cc b/tiledb/sm/crypto/test/compile_crypto_main.cc index 570db7fa52f3..ea3eef1c5794 100644 --- a/tiledb/sm/crypto/test/compile_crypto_main.cc +++ b/tiledb/sm/crypto/test/compile_crypto_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Crypto); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc index 35c08b93259b..05e37c58c3df 100644 --- a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc +++ b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.cc @@ -499,4 +499,4 @@ STSProfileWithWebIdentityCredentialsProvider::GetCredentialsFromWebIdentity( return GetCredentialsFromWebIdentityInternal(profile, &stsClient); } -#endif // HAVE_S3s \ No newline at end of file +#endif // HAVE_S3s diff --git a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h index d50185af1293..8af825209a26 100644 --- a/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h +++ b/tiledb/sm/filesystem/s3/STSProfileWithWebIdentityCredentialsProvider.h @@ -145,4 +145,4 @@ class /* AWS_IDENTITY_MANAGEMENT_API */ } // namespace Auth } // namespace Aws #endif // HAVE_S3 -#endif // TILEDB_S3_STS_PROFILE_WITH_WEB_IDENTITY_CREDENTIALS_PROVIDER_H \ No newline at end of file +#endif // TILEDB_S3_STS_PROFILE_WITH_WEB_IDENTITY_CREDENTIALS_PROVIDER_H diff --git a/tiledb/sm/filesystem/test/compile_vfs_main.cc b/tiledb/sm/filesystem/test/compile_vfs_main.cc index bfa47c8055e4..acb4522d142f 100644 --- a/tiledb/sm/filesystem/test/compile_vfs_main.cc +++ b/tiledb/sm/filesystem/test/compile_vfs_main.cc @@ -34,4 +34,4 @@ int main() { ThreadPool io_tp(4); tiledb::sm::VFS x{&stats, &compute_tp, &io_tp, tiledb::sm::Config{}}; return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/filter_create.cc b/tiledb/sm/filter/filter_create.cc index 588c66de1a73..cd8b150b33b0 100644 --- a/tiledb/sm/filter/filter_create.cc +++ b/tiledb/sm/filter/filter_create.cc @@ -204,4 +204,4 @@ shared_ptr tiledb::sm::FilterCreate::deserialize( EncryptionKey encryption_key; return tiledb::sm::FilterCreate::deserialize( deserializer, encryption_key, version, datatype); -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/float_scaling_filter.cc b/tiledb/sm/filter/float_scaling_filter.cc index 8948d04879e7..f11d4703f52b 100644 --- a/tiledb/sm/filter/float_scaling_filter.cc +++ b/tiledb/sm/filter/float_scaling_filter.cc @@ -335,4 +335,4 @@ FloatScalingFilter* FloatScalingFilter::clone_impl() const { } } // namespace sm -} // namespace tiledb \ No newline at end of file +} // namespace tiledb diff --git a/tiledb/sm/filter/float_scaling_filter.h b/tiledb/sm/filter/float_scaling_filter.h index 54bfa0f60bc0..518fe0ed38b1 100644 --- a/tiledb/sm/filter/float_scaling_filter.h +++ b/tiledb/sm/filter/float_scaling_filter.h @@ -207,4 +207,4 @@ class FloatScalingFilter : public Filter { } // namespace sm } // namespace tiledb -#endif // TILEDB_FLOAT_SCALING_FILTER_H \ No newline at end of file +#endif // TILEDB_FLOAT_SCALING_FILTER_H diff --git a/tiledb/sm/filter/test/compile_all_filters_main.cc b/tiledb/sm/filter/test/compile_all_filters_main.cc index 4c65189588f0..e4465679dfe3 100644 --- a/tiledb/sm/filter/test/compile_all_filters_main.cc +++ b/tiledb/sm/filter/test/compile_all_filters_main.cc @@ -36,4 +36,4 @@ int main() { Deserializer& deserializer, const uint32_t version, Datatype datatype)>( tiledb::sm::FilterCreate::deserialize); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_bitshuffle_filter_main.cc b/tiledb/sm/filter/test/compile_bitshuffle_filter_main.cc index d7ca5998f5c2..b15df2548115 100644 --- a/tiledb/sm/filter/test/compile_bitshuffle_filter_main.cc +++ b/tiledb/sm/filter/test/compile_bitshuffle_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::BitshuffleFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_byteshuffle_filter_main.cc b/tiledb/sm/filter/test/compile_byteshuffle_filter_main.cc index 1a4aef581d8f..6ec7b06857f0 100644 --- a/tiledb/sm/filter/test/compile_byteshuffle_filter_main.cc +++ b/tiledb/sm/filter/test/compile_byteshuffle_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::ByteshuffleFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_checksum_filters_main.cc b/tiledb/sm/filter/test/compile_checksum_filters_main.cc index dbbbc7189b51..cd2679370ece 100644 --- a/tiledb/sm/filter/test/compile_checksum_filters_main.cc +++ b/tiledb/sm/filter/test/compile_checksum_filters_main.cc @@ -33,4 +33,4 @@ int main() { (void)sizeof(tiledb::sm::ChecksumMD5Filter); (void)sizeof(tiledb::sm::ChecksumSHA256Filter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_compression_filter_main.cc b/tiledb/sm/filter/test/compile_compression_filter_main.cc index 526a30bf1290..f0f8c69ce221 100644 --- a/tiledb/sm/filter/test/compile_compression_filter_main.cc +++ b/tiledb/sm/filter/test/compile_compression_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::CompressionFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_encryption_filters_main.cc b/tiledb/sm/filter/test/compile_encryption_filters_main.cc index bb65e98e20fd..33bb7a5bbef5 100644 --- a/tiledb/sm/filter/test/compile_encryption_filters_main.cc +++ b/tiledb/sm/filter/test/compile_encryption_filters_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::EncryptionAES256GCMFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_filter_main.cc b/tiledb/sm/filter/test/compile_filter_main.cc index b274df19c2a0..a9af572494f2 100644 --- a/tiledb/sm/filter/test/compile_filter_main.cc +++ b/tiledb/sm/filter/test/compile_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Filter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_filter_pipeline_main.cc b/tiledb/sm/filter/test/compile_filter_pipeline_main.cc index 1149175743eb..0675c0099bfc 100644 --- a/tiledb/sm/filter/test/compile_filter_pipeline_main.cc +++ b/tiledb/sm/filter/test/compile_filter_pipeline_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::FilterPipeline); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_float_scaling_filter_main.cc b/tiledb/sm/filter/test/compile_float_scaling_filter_main.cc index 8285efe26edb..e7702bf1fb9c 100644 --- a/tiledb/sm/filter/test/compile_float_scaling_filter_main.cc +++ b/tiledb/sm/filter/test/compile_float_scaling_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::FloatScalingFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/compile_xor_filter_main.cc b/tiledb/sm/filter/test/compile_xor_filter_main.cc index 2f96980d95ff..73239c20a499 100644 --- a/tiledb/sm/filter/test/compile_xor_filter_main.cc +++ b/tiledb/sm/filter/test/compile_xor_filter_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::XORFilter); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/unit_filter_create.cc b/tiledb/sm/filter/test/unit_filter_create.cc index e2f818b530db..f6def14e9f5c 100644 --- a/tiledb/sm/filter/test/unit_filter_create.cc +++ b/tiledb/sm/filter/test/unit_filter_create.cc @@ -487,4 +487,4 @@ TEST_CASE("Filter: Test WEBP filter deserialization", "[filter][webp]") { CHECK(y0 == extents.first); CHECK(x0 == extents.second); } -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/unit_filter_pipeline.cc b/tiledb/sm/filter/test/unit_filter_pipeline.cc index f3c250085720..4b33569ed467 100644 --- a/tiledb/sm/filter/test/unit_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_filter_pipeline.cc @@ -244,4 +244,4 @@ TEST_CASE( CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::GEOM_WKT, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::INT32, version)); CHECK_FALSE(fp_with.skip_offsets_filtering(Datatype::FLOAT64, version)); -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/test/unit_float_scale_input_validation.cc b/tiledb/sm/filter/test/unit_float_scale_input_validation.cc index e0e4ed8e1a12..5e9b9f3ecde7 100644 --- a/tiledb/sm/filter/test/unit_float_scale_input_validation.cc +++ b/tiledb/sm/filter/test/unit_float_scale_input_validation.cc @@ -94,4 +94,4 @@ TEST_CASE( check_ok(FilterOption::SCALE_FLOAT_OFFSET, &zero_example); check_ok(FilterOption::SCALE_FLOAT_OFFSET, &subnormal_example); check_ok(FilterOption::SCALE_FLOAT_OFFSET, &normal_example); -} \ No newline at end of file +} diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 9ac57a935588..961209f7ecb4 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -403,4 +403,4 @@ template void WebpFilter::set_extents( } // namespace tiledb::sm -#endif // TILEDB_WEBP \ No newline at end of file +#endif // TILEDB_WEBP diff --git a/tiledb/sm/filter/xor_filter.cc b/tiledb/sm/filter/xor_filter.cc index 96e67bd70838..488ffc7fe466 100644 --- a/tiledb/sm/filter/xor_filter.cc +++ b/tiledb/sm/filter/xor_filter.cc @@ -285,4 +285,4 @@ XORFilter* XORFilter::clone_impl() const { } } // namespace sm -} // namespace tiledb \ No newline at end of file +} // namespace tiledb diff --git a/tiledb/sm/filter/xor_filter.h b/tiledb/sm/filter/xor_filter.h index 2910cb926b57..e6decfe71539 100644 --- a/tiledb/sm/filter/xor_filter.h +++ b/tiledb/sm/filter/xor_filter.h @@ -163,4 +163,4 @@ class XORFilter : public Filter { } // namespace sm } // namespace tiledb -#endif // TILEDB_XOR_FILTER_H \ No newline at end of file +#endif // TILEDB_XOR_FILTER_H diff --git a/tiledb/sm/fragment/test/compile_fragment_main.cc b/tiledb/sm/fragment/test/compile_fragment_main.cc index bc16f1378db2..045d9f9ab9ca 100644 --- a/tiledb/sm/fragment/test/compile_fragment_main.cc +++ b/tiledb/sm/fragment/test/compile_fragment_main.cc @@ -37,4 +37,4 @@ int main() { (void)x.name_version(); (void)x.array_format_version(); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/group/test/compile_group_main.cc b/tiledb/sm/group/test/compile_group_main.cc index 59f9bbc70859..1549b189c98b 100644 --- a/tiledb/sm/group/test/compile_group_main.cc +++ b/tiledb/sm/group/test/compile_group_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::GroupDetails); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/metadata/test/compile_metadata_main.cc b/tiledb/sm/metadata/test/compile_metadata_main.cc index 0b09e57369ad..7233c396c70c 100644 --- a/tiledb/sm/metadata/test/compile_metadata_main.cc +++ b/tiledb/sm/metadata/test/compile_metadata_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Metadata); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/query/ast/test/compile_query_ast_main.cc b/tiledb/sm/query/ast/test/compile_query_ast_main.cc index 41def32d8932..8e730e5a9050 100644 --- a/tiledb/sm/query/ast/test/compile_query_ast_main.cc +++ b/tiledb/sm/query/ast/test/compile_query_ast_main.cc @@ -33,4 +33,4 @@ int main() { (void)sizeof(tiledb::sm::ASTNodeVal); (void)sizeof(tiledb::sm::ASTNodeExpr); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc index 93c1f75dc47a..60383922098e 100644 --- a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc +++ b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc @@ -791,4 +791,4 @@ TEST_CASE( REQUIRE( subtree_a.combine(subtree_b, QueryConditionCombinationOp::OR, &qc).ok()); serialize_deserialize_check(qc); -} \ No newline at end of file +} diff --git a/tiledb/sm/query/readers/aggregators/operation.cc b/tiledb/sm/query/readers/aggregators/operation.cc index e3afdf52e5e2..6f66011c293d 100644 --- a/tiledb/sm/query/readers/aggregators/operation.cc +++ b/tiledb/sm/query/readers/aggregators/operation.cc @@ -54,4 +54,4 @@ shared_ptr Operation::make_operation( "Unable to create and aggregate operation using name: " + name); } -} // namespace tiledb::sm \ No newline at end of file +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc index af180a27054f..87402f9a6512 100644 --- a/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc +++ b/tiledb/sm/query/readers/aggregators/test/compile_aggregators_main.cc @@ -126,4 +126,4 @@ int main() { "Sum", false, false, 1, tiledb::sm::Datatype::UINT8)); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/query/readers/test/unit_reader_base.cc b/tiledb/sm/query/readers/test/unit_reader_base.cc index b68d569b580f..5d52bbcbde68 100644 --- a/tiledb/sm/query/readers/test/unit_reader_base.cc +++ b/tiledb/sm/query/readers/test/unit_reader_base.cc @@ -82,4 +82,4 @@ TEST_CASE( ReaderBase::compute_chunk_min_max(10, 1, 1), "Range thread index is greater than number of range threads"); } -} \ No newline at end of file +} diff --git a/tiledb/sm/rtree/test/compile_rtree_main.cc b/tiledb/sm/rtree/test/compile_rtree_main.cc index b50cd2dd7325..d720c36ba364 100644 --- a/tiledb/sm/rtree/test/compile_rtree_main.cc +++ b/tiledb/sm/rtree/test/compile_rtree_main.cc @@ -31,4 +31,4 @@ int main() { tiledb::sm::RTree x{}; return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index 4e65d2bd93ff..c1b1d8d25841 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -74,4 +74,4 @@ Status array_schema_evolution_deserialize( } // namespace serialization } // namespace sm } // namespace tiledb -#endif // TILEDB_SERIALIZATION_ARRAY_SCHEMA_Evolution_H \ No newline at end of file +#endif // TILEDB_SERIALIZATION_ARRAY_SCHEMA_Evolution_H diff --git a/tiledb/sm/serialization/fragment_metadata.h b/tiledb/sm/serialization/fragment_metadata.h index 59868fd4794c..270f50a2a9c4 100644 --- a/tiledb/sm/serialization/fragment_metadata.h +++ b/tiledb/sm/serialization/fragment_metadata.h @@ -103,4 +103,4 @@ Status fragment_metadata_to_capnp( } // namespace sm } // namespace tiledb -#endif // TILEDB_SERIALIZATION_FRAGMENT_METADATA_H \ No newline at end of file +#endif // TILEDB_SERIALIZATION_FRAGMENT_METADATA_H diff --git a/tiledb/sm/serialization/query_aggregates.cc b/tiledb/sm/serialization/query_aggregates.cc index 036139baadba..d3785e1ebd03 100644 --- a/tiledb/sm/serialization/query_aggregates.cc +++ b/tiledb/sm/serialization/query_aggregates.cc @@ -129,4 +129,4 @@ void query_channels_from_capnp( #endif // TILEDB_SERIALIZATION -} // namespace tiledb::sm::serialization \ No newline at end of file +} // namespace tiledb::sm::serialization diff --git a/tiledb/sm/serialization/query_aggregates.h b/tiledb/sm/serialization/query_aggregates.h index c54cfa503cc5..5eaeb7f024a6 100644 --- a/tiledb/sm/serialization/query_aggregates.h +++ b/tiledb/sm/serialization/query_aggregates.h @@ -65,4 +65,4 @@ void query_channels_from_capnp( } // namespace tiledb::sm::serialization -#endif // TILEDB_SERIALIZATION_QUERY_AGGREGATES_H \ No newline at end of file +#endif // TILEDB_SERIALIZATION_QUERY_AGGREGATES_H diff --git a/tiledb/sm/stats/duration_instrument.h b/tiledb/sm/stats/duration_instrument.h index c0d2baf3b3b7..82ca74d6ac4f 100644 --- a/tiledb/sm/stats/duration_instrument.h +++ b/tiledb/sm/stats/duration_instrument.h @@ -85,4 +85,4 @@ class DurationInstrument { } // namespace sm } // namespace tiledb -#endif // TILEDB_DURATION_INSTRUMENT_H \ No newline at end of file +#endif // TILEDB_DURATION_INSTRUMENT_H diff --git a/tiledb/sm/stats/test/compile_stats_main.cc b/tiledb/sm/stats/test/compile_stats_main.cc index 8f09bb43a4f8..22ef30ac6bfc 100644 --- a/tiledb/sm/stats/test/compile_stats_main.cc +++ b/tiledb/sm/stats/test/compile_stats_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::stats::Stats); return 0; -} \ No newline at end of file +} diff --git a/tiledb/sm/tile/test/compile_tile_main.cc b/tiledb/sm/tile/test/compile_tile_main.cc index 71b9fe3618c8..1b8ed29b021a 100644 --- a/tiledb/sm/tile/test/compile_tile_main.cc +++ b/tiledb/sm/tile/test/compile_tile_main.cc @@ -31,4 +31,4 @@ int main() { (void)sizeof(tiledb::sm::Tile); return 0; -} \ No newline at end of file +} From 8a9b23df4847019ba7383ce789cdbbffe7a02d46 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 31 Jan 2024 02:55:02 -0800 Subject: [PATCH 168/456] Remove serialization non C.41 constructors from query condition. (#4684) This change removes non C.41 constructors from condition_from_capnp. --- TYPE: NO_HISTORY DESC: Remove serialization non C.41 constructors from query condition. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/unit-QueryCondition-serialization.cc | 6 ++--- test/src/unit-cppapi-query-condition-sets.cc | 2 +- tiledb/sm/query/query_condition.cc | 4 --- tiledb/sm/query/query_condition.h | 10 ++----- tiledb/sm/serialization/query.cc | 26 ++++++++----------- tiledb/sm/serialization/query.h | 5 ++-- 6 files changed, 18 insertions(+), 35 deletions(-) diff --git a/test/src/unit-QueryCondition-serialization.cc b/test/src/unit-QueryCondition-serialization.cc index edabfca8506a..15cb31d09299 100644 --- a/test/src/unit-QueryCondition-serialization.cc +++ b/test/src/unit-QueryCondition-serialization.cc @@ -51,7 +51,6 @@ TEST_CASE( "QueryCondition serialization: Test serialization", "[QueryCondition][serialization]") { QueryCondition query_condition; - QueryCondition query_condition_clone; SECTION("Test serialization, basic") { std::string field_name = "x"; @@ -271,9 +270,8 @@ TEST_CASE( REQUIRE(tiledb::sm::serialization::condition_to_capnp( query_condition, &condition_builder) .ok()); - REQUIRE(tiledb::sm::serialization::condition_from_capnp( - condition_builder, &query_condition_clone) - .ok()); + auto query_condition_clone = + tiledb::sm::serialization::condition_from_capnp(condition_builder); REQUIRE(tiledb::test::ast_equal( query_condition.ast(), query_condition_clone.ast())); } diff --git a/test/src/unit-cppapi-query-condition-sets.cc b/test/src/unit-cppapi-query-condition-sets.cc index b93181d82d54..76ba87d2918b 100644 --- a/test/src/unit-cppapi-query-condition-sets.cc +++ b/test/src/unit-cppapi-query-condition-sets.cc @@ -924,7 +924,7 @@ QueryCondition CPPQueryConditionFx::serialize_deserialize_qc( throw_if_not_ok(condition_to_capnp(*qc_ptr, &builder)); // Deserialize the query condition. - throw_if_not_ok(condition_from_capnp(builder, ret_ptr)); + *ret_ptr = condition_from_capnp(builder); REQUIRE(tiledb::test::ast_equal(ret_ptr->ast(), qc_ptr->ast())); return ret; diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 336668d86b18..242babd69049 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -2757,10 +2757,6 @@ uint64_t QueryCondition::condition_index() const { return condition_index_; } -void QueryCondition::set_ast(tdb_unique_ptr&& ast) { - tree_ = std::move(ast); -} - // Explicit template instantiations. template Status QueryCondition::apply_sparse( const ArraySchema& array_schema, ResultTile&, std::vector&); diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index af7f0a88c454..327e48d551fd 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -59,10 +59,10 @@ class QueryCondition { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Default constructor. */ + /** Default constructor. Should be used only in the C API. */ QueryCondition(); - /** Constructor for a set membership QueryCondition */ + /** Constructor for a set membership QueryCondition. */ QueryCondition( const std::string& field_name, const void* data, @@ -247,12 +247,6 @@ class QueryCondition { */ QueryCondition negated_condition(); - /** - * Sets the AST. This is internal state to only be used in - * the serialization path. - */ - void set_ast(tdb_unique_ptr&& ast); - /** * Returns the AST object. This is internal state to only be used in testing * and the serialization path. diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 11351aa98414..8a716ca82b74 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -1068,9 +1068,8 @@ tdb_unique_ptr condition_ast_from_capnp( tdb_new(ASTNodeExpr, std::move(ast_nodes), combination_op)); } -Status condition_from_capnp( - const capnp::Condition::Reader& condition_reader, - QueryCondition* const condition) { +QueryCondition condition_from_capnp( + const capnp::Condition::Reader& condition_reader) { if (condition_reader.hasClauses()) { // coming from older API // Accumulating the AST value nodes from the clause list. std::vector> ast_nodes; @@ -1104,19 +1103,20 @@ Status condition_from_capnp( // Constructing the tree from the list of AST nodes. assert(ast_nodes.size() > 0); if (ast_nodes.size() == 1) { - condition->set_ast(std::move(ast_nodes[0])); + return QueryCondition(std::move(ast_nodes[0])); } else { auto tree_ptr = tdb_unique_ptr(tdb_new( ASTNodeExpr, std::move(ast_nodes), QueryConditionCombinationOp::AND)); - condition->set_ast(std::move(tree_ptr)); + return QueryCondition(std::move(tree_ptr)); } } else if (condition_reader.hasTree()) { // Constructing the query condition from the AST representation. // We assume that the deserialized values of the AST are validated properly. auto ast_reader = condition_reader.getTree(); - condition->set_ast(condition_ast_from_capnp(ast_reader)); + return QueryCondition(condition_ast_from_capnp(ast_reader)); } - return Status::Ok(); + throw std::runtime_error( + "condition_from_capnp: serialized QC has no tree or clauses."); } Status reader_from_capnp( @@ -1144,8 +1144,7 @@ Status reader_from_capnp( // Query condition if (reader_reader.hasCondition()) { auto condition_reader = reader_reader.getCondition(); - QueryCondition condition; - RETURN_NOT_OK(condition_from_capnp(condition_reader, &condition)); + QueryCondition condition = condition_from_capnp(condition_reader); RETURN_NOT_OK(query->set_condition(condition)); } @@ -1186,8 +1185,7 @@ Status index_reader_from_capnp( // Query condition if (reader_reader.hasCondition()) { auto condition_reader = reader_reader.getCondition(); - QueryCondition condition; - RETURN_NOT_OK(condition_from_capnp(condition_reader, &condition)); + QueryCondition condition = condition_from_capnp(condition_reader); RETURN_NOT_OK(query->set_condition(condition)); } @@ -1229,8 +1227,7 @@ Status dense_reader_from_capnp( // Query condition if (reader_reader.hasCondition()) { auto condition_reader = reader_reader.getCondition(); - QueryCondition condition; - RETURN_NOT_OK(condition_from_capnp(condition_reader, &condition)); + QueryCondition condition = condition_from_capnp(condition_reader); RETURN_NOT_OK(query->set_condition(condition)); } @@ -1253,8 +1250,7 @@ Status delete_from_capnp( // Query condition if (delete_reader.hasCondition()) { auto condition_reader = delete_reader.getCondition(); - QueryCondition condition; - RETURN_NOT_OK(condition_from_capnp(condition_reader, &condition)); + QueryCondition condition = condition_from_capnp(condition_reader); RETURN_NOT_OK(query->set_condition(condition)); } diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index dced5df32e4b..1a711db6b7de 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -239,9 +239,8 @@ Status unordered_write_state_from_capnp( UnorderedWriter* runordered_writer, SerializationContext context); -Status condition_from_capnp( - const capnp::Condition::Reader& condition_reader, - QueryCondition* const condition); +QueryCondition condition_from_capnp( + const capnp::Condition::Reader& condition_reader); Status condition_to_capnp( const QueryCondition& condition, From 82dc8fe6c69455eba65746ca746fb2010fad7d11 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 1 Feb 2024 19:46:49 +0800 Subject: [PATCH 169/456] unit_thread_pool.cc: add forgotten include (#4690) At least when building with `gcc` this source needs to include ``, otherwise it fails to compile. --- TYPE: NO_HISTORY DESC: Add a missing include into `unit_thread_pool.cc` --------- Co-authored-by: Theodore Tsirpanis --- test/src/unit-tile-metadata-generator.cc | 2 ++ test/src/unit-tile-metadata.cc | 2 ++ tiledb/common/thread_pool/test/unit_thread_pool.cc | 1 + tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc | 2 ++ 4 files changed, 7 insertions(+) diff --git a/test/src/unit-tile-metadata-generator.cc b/test/src/unit-tile-metadata-generator.cc index cd3a1b07c2e3..b754f7fdac96 100644 --- a/test/src/unit-tile-metadata-generator.cc +++ b/test/src/unit-tile-metadata-generator.cc @@ -30,6 +30,8 @@ * Tests the TileMetadataGenerator class. */ +#include + #include #include "test/support/src/helpers.h" #include "tiledb/common/common.h" diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index 89281332f040..f478b2d30953 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -31,6 +31,8 @@ * load_tile_*_values and get_tile_* apis of fragment metadata. */ +#include + #include #include "test/support/src/helpers.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" diff --git a/tiledb/common/thread_pool/test/unit_thread_pool.cc b/tiledb/common/thread_pool/test/unit_thread_pool.cc index 99131dd9084d..9af3936d21ed 100644 --- a/tiledb/common/thread_pool/test/unit_thread_pool.cc +++ b/tiledb/common/thread_pool/test/unit_thread_pool.cc @@ -37,6 +37,7 @@ #include #include #include +#include #include #include "tiledb/common/thread_pool.h" diff --git a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc index d88e135f4d7f..f714249d81d0 100644 --- a/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/bench_aggregators.cc @@ -30,6 +30,8 @@ * Benchmarks the `AggregateWithCount` class. */ +#include + #include "tiledb/common/common.h" #include "tiledb/sm/query/readers/aggregators/aggregate_buffer.h" #include "tiledb/sm/query/readers/aggregators/aggregate_with_count.h" From 404f89f102ed784be3a59b3a307f0c156cdea522 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 5 Feb 2024 10:37:50 +0200 Subject: [PATCH 170/456] Move group open logic out of StorageManager. (#4676) This moves group_open_for_reads, group_open_for_writes and all connected functions out of StorageManager. --- TYPE: NO_HISTORY DESC: Move group open logic out of StorageManager. --- tiledb/api/c_api/group/group_api_internal.h | 2 +- .../consolidator/group_meta_consolidator.cc | 6 +- tiledb/sm/group/group.cc | 152 +++++++++++++++--- tiledb/sm/group/group.h | 56 ++++++- tiledb/sm/storage_manager/storage_manager.cc | 142 +--------------- .../storage_manager_canonical.h | 66 -------- 6 files changed, 195 insertions(+), 229 deletions(-) diff --git a/tiledb/api/c_api/group/group_api_internal.h b/tiledb/api/c_api/group/group_api_internal.h index ec06a6f9a894..1df1977b94ae 100644 --- a/tiledb/api/c_api/group/group_api_internal.h +++ b/tiledb/api/c_api/group/group_api_internal.h @@ -52,7 +52,7 @@ struct tiledb_group_handle_t explicit tiledb_group_handle_t( const tiledb::sm::URI& uri, tiledb::sm::StorageManager* sm) - : group_(uri, sm) { + : group_(sm->resources(), uri, sm) { } [[nodiscard]] inline tiledb::sm::Group& group() { diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index 459f7798c9fa..589b067fe4c7 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -69,12 +69,14 @@ Status GroupMetaConsolidator::consolidate( // Open group for reading auto group_uri = URI(group_name); - Group group_for_reads(group_uri, storage_manager_); + Group group_for_reads( + storage_manager_->resources(), group_uri, storage_manager_); RETURN_NOT_OK(group_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_)); // Open group for writing - Group group_for_writes(group_uri, storage_manager_); + Group group_for_writes( + storage_manager_->resources(), group_uri, storage_manager_); RETURN_NOT_OK_ELSE( group_for_writes.open(QueryType::WRITE), throw_if_not_ok(group_for_reads.close())); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index c6965f8c41a1..aa5a10edb51f 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -44,6 +44,9 @@ #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/stats/global_stats.h" +#include "tiledb/sm/storage_manager/context_resources.h" +#include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" using namespace tiledb::common; @@ -57,7 +60,10 @@ class GroupStatusException : public StatusException { } }; -Group::Group(const URI& group_uri, StorageManager* storage_manager) +Group::Group( + ContextResources& resources, + const URI& group_uri, + StorageManager* storage_manager) : group_uri_(group_uri) , storage_manager_(storage_manager) , config_(storage_manager_->config()) @@ -67,7 +73,8 @@ Group::Group(const URI& group_uri, StorageManager* storage_manager) , query_type_(QueryType::READ) , timestamp_start_(0) , timestamp_end_(UINT64_MAX) - , encryption_key_(tdb::make_shared(HERE())) { + , encryption_key_(tdb::make_shared(HERE())) + , resources_(resources) { } Status Group::open( @@ -159,11 +166,7 @@ Status Group::open( return Status_GroupDirectoryError(le.what()); } - auto&& [st, group_details] = storage_manager_->group_open_for_reads(this); - RETURN_NOT_OK(st); - if (group_details.has_value()) { - group_details_ = group_details.value(); - } + group_open_for_reads(); } else { try { group_dir_ = make_shared( @@ -178,12 +181,7 @@ Status Group::open( return Status_GroupDirectoryError(le.what()); } - auto&& [st, group_details] = storage_manager_->group_open_for_writes(this); - RETURN_NOT_OK(st); - - if (group_details.has_value()) { - group_details_ = group_details.value(); - } + group_open_for_writes(); metadata_.reset(timestamp_end); } @@ -323,7 +321,7 @@ void Group::delete_group(const URI& uri, bool recursive) { if (member->type() == ObjectType::ARRAY) { storage_manager_->delete_array(member_uri.to_string().c_str()); } else if (member->type() == ObjectType::GROUP) { - Group group_rec(member_uri, storage_manager_); + Group group_rec(resources_, member_uri, storage_manager_); throw_if_not_ok(group_rec.open(QueryType::MODIFY_EXCLUSIVE)); group_rec.delete_group(member_uri, true); } @@ -690,7 +688,7 @@ std::string Group::dump( ss << std::endl; if (do_recurse) { - Group group_rec(member_uri, storage_manager_); + Group group_rec(resources_, member_uri, storage_manager_); throw_if_not_ok(group_rec.open(QueryType::READ)); ss << group_rec.dump(indent_size, num_indents + 2, recursive, false); throw_if_not_ok(group_rec.close()); @@ -706,7 +704,7 @@ std::string Group::dump( void Group::load_metadata() { if (remote_) { - auto rest_client = storage_manager_->rest_client(); + auto rest_client = resources_.rest_client(); if (rest_client == nullptr) throw GroupStatusException( "Cannot load metadata; remote group with no REST client."); @@ -714,10 +712,128 @@ void Group::load_metadata() { rest_client->post_group_metadata_from_rest(group_uri_, this)); } else { assert(group_dir_->loaded()); - storage_manager_->load_group_metadata( - group_dir_, *encryption_key_, &metadata_); + load_metadata_from_storage(group_dir_, *encryption_key_, &metadata_); } metadata_loaded_ = true; } +void Group::load_metadata_from_storage( + const shared_ptr& group_dir, + const EncryptionKey& encryption_key, + Metadata* metadata) { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("group_load_metadata_from_storage"); + + // Special case + if (metadata == nullptr) { + return; + } + + // Determine which group metadata to load + const auto& group_metadata_to_load = group_dir->group_meta_uris(); + + auto metadata_num = group_metadata_to_load.size(); + // TBD: Might use DynamicArray when it is more capable. + std::vector> metadata_tiles(metadata_num); + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { + const auto& uri = group_metadata_to_load[m].uri_; + + auto&& tile = GenericTileIO::load(resources_, uri, 0, encryption_key); + metadata_tiles[m] = tdb::make_shared(HERE(), std::move(tile)); + + return Status::Ok(); + })); + + // Compute array metadata size for the statistics + uint64_t meta_size = 0; + for (const auto& t : metadata_tiles) { + meta_size += t->size(); + } + resources_.stats().add_counter("group_read_group_meta_size", meta_size); + + // Copy the deserialized metadata into the original Metadata object + *metadata = Metadata::deserialize(metadata_tiles); + metadata->set_loaded_metadata_uris(group_metadata_to_load); +} + +void Group::group_open_for_reads() { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("group_open_for_reads"); + + // Load group data + load_group_details(); +} + +void Group::load_group_details() { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("load_group_details"); + const URI& latest_group_uri = group_directory()->latest_group_details_uri(); + if (latest_group_uri.is_invalid()) { + return; + } + + // V1 groups did not have the version appended so only have 4 "_" + // (____) + auto part = latest_group_uri.last_path_part(); + if (std::count(part.begin(), part.end(), '_') == 4) { + load_group_from_uri(latest_group_uri); + return; + } + + // V2 and newer should loop over all uris all the time to handle deletes at + // read-time + load_group_from_all_uris(group_directory()->group_detail_uris()); +} + +void Group::load_group_from_uri(const URI& uri) { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("load_group_from_uri"); + + auto&& tile = GenericTileIO::load(resources_, uri, 0, *encryption_key()); + + resources_.stats().add_counter("read_group_size", tile.size()); + + // Deserialize + Deserializer deserializer(tile.data(), tile.size()); + auto opt_group = + GroupDetails::deserialize(deserializer, group_directory()->uri()); + + if (opt_group.has_value()) { + group_details_ = opt_group.value(); + } +} + +void Group::load_group_from_all_uris(const std::vector& uris) { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("load_group_from_all_uris"); + + std::vector> deserializers; + for (auto& uri : uris) { + auto&& tile = + GenericTileIO::load(resources_, uri.uri_, 0, *encryption_key()); + + resources_.stats().add_counter("read_group_size", tile.size()); + + // Deserialize + shared_ptr deserializer = + tdb::make_shared(HERE(), std::move(tile)); + deserializers.emplace_back(deserializer); + } + + auto opt_group = + GroupDetails::deserialize(deserializers, group_directory()->uri()); + + if (opt_group.has_value()) { + group_details_ = opt_group.value(); + } +} + +void Group::group_open_for_writes() { + [[maybe_unused]] auto timer_se = + resources_.stats().start_timer("group_open_for_writes"); + + load_group_details(); +} + } // namespace tiledb::sm diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index cadbaf667880..0cb415405372 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -49,6 +49,8 @@ using namespace tiledb::common; namespace tiledb::sm { +class ContextResources; + class GroupDetailsException : public StatusException { public: explicit GroupDetailsException(const std::string& message) @@ -58,7 +60,23 @@ class GroupDetailsException : public StatusException { class Group { public: - Group(const URI& group_uri, StorageManager* storage_manager); + /** + * Constructs a Group object given a uri and a ContextResources reference. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Group class on StorageManager. For + * now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * Group(ContextResources&, const URI&). + * + * @param resources A ContextResources reference + * @param group_uri The location of the group + * @param storage_manager A StorageManager pointer + * (this will go away in the near future) + */ + Group( + ContextResources& resources, + const URI& group_uri, + StorageManager* storage_manager); /** Destructor. */ ~Group() = default; @@ -425,6 +443,9 @@ class Group { /** Mutex for thread safety. */ mutable std::mutex mtx_; + /** The ContextResources class. */ + ContextResources& resources_; + /* ********************************* */ /* PROTECTED METHODS */ /* ********************************* */ @@ -433,6 +454,39 @@ class Group { * Load group metadata, handles remote groups vs non-remote groups */ void load_metadata(); + + /** + * Load group metadata from disk + */ + void load_metadata_from_storage( + const shared_ptr& group_dir, + const EncryptionKey& encryption_key, + Metadata* metadata); + + /** Opens an group for reads. */ + void group_open_for_reads(); + + /** + * Load group details from disk + */ + void load_group_details(); + + /** + * Load a group detail from URI + * + * @param uri location to load + */ + void load_group_from_uri(const URI& uri); + + /** + * Load a group detail from URIs + * + * @param uris locations to load + */ + void load_group_from_all_uris(const std::vector& uris); + + /** Opens an group for writes. */ + void group_open_for_writes(); }; } // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index bae92437b4f5..51649923d6a0 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -915,7 +915,7 @@ Status StorageManagerCanonical::group_create(const std::string& group_uri) { std::lock_guard lock{object_create_mtx_}; if (uri.is_tiledb()) { - Group group(uri, this); + Group group(resources_, uri, this); RETURN_NOT_OK(rest_client()->post_group_create_to_rest(uri, &group)); return Status::Ok(); } @@ -1404,146 +1404,6 @@ shared_ptr StorageManagerCanonical::logger() const { return logger_; } -tuple>> -StorageManagerCanonical::load_group_from_uri( - const URI& group_uri, const URI& uri, const EncryptionKey& encryption_key) { - auto timer_se = stats()->start_timer("sm_load_group_from_uri"); - - auto&& tile = GenericTileIO::load(resources_, uri, 0, encryption_key); - - stats()->add_counter("read_group_size", tile.size()); - - // Deserialize - Deserializer deserializer(tile.data(), tile.size()); - auto opt_group = GroupDetails::deserialize(deserializer, group_uri); - return {Status::Ok(), opt_group}; -} - -tuple>> -StorageManagerCanonical::load_group_from_all_uris( - const URI& group_uri, - const std::vector& uris, - const EncryptionKey& encryption_key) { - auto timer_se = stats()->start_timer("sm_load_group_from_uri"); - - std::vector> deserializers; - for (auto& uri : uris) { - auto&& tile = GenericTileIO::load(resources_, uri.uri_, 0, encryption_key); - - stats()->add_counter("read_group_size", tile.size()); - - // Deserialize - shared_ptr deserializer = - tdb::make_shared(HERE(), std::move(tile)); - deserializers.emplace_back(deserializer); - } - - auto opt_group = GroupDetails::deserialize(deserializers, group_uri); - return {Status::Ok(), opt_group}; -} - -tuple>> -StorageManagerCanonical::load_group_details( - const shared_ptr& group_directory, - const EncryptionKey& encryption_key) { - auto timer_se = stats()->start_timer("sm_load_group_details"); - const URI& latest_group_uri = group_directory->latest_group_details_uri(); - if (latest_group_uri.is_invalid()) { - // Returning ok because not having the latest group details means the group - // has just been created and no members have been added yet. - return {Status::Ok(), std::nullopt}; - } - - // V1 groups did not have the version appended so only have 4 "_" - // (____) - auto part = latest_group_uri.last_path_part(); - if (std::count(part.begin(), part.end(), '_') == 4) { - return load_group_from_uri( - group_directory->uri(), latest_group_uri, encryption_key); - } - - // V2 and newer should loop over all uris all the time to handle deletes at - // read-time - return load_group_from_all_uris( - group_directory->uri(), - group_directory->group_detail_uris(), - encryption_key); -} - -std::tuple>> -StorageManagerCanonical::group_open_for_reads(Group* group) { - auto timer_se = stats()->start_timer("group_open_for_reads"); - - // Load group data - auto&& [st, group_deserialized] = - load_group_details(group->group_directory(), *group->encryption_key()); - RETURN_NOT_OK_TUPLE(st, std::nullopt); - - if (group_deserialized.has_value()) { - return {Status::Ok(), group_deserialized.value()}; - } - - // Return ok because having no members is acceptable if the group has never - // been written to. - return {Status::Ok(), std::nullopt}; -} - -std::tuple>> -StorageManagerCanonical::group_open_for_writes(Group* group) { - auto timer_se = stats()->start_timer("group_open_for_writes"); - - // Load group data - auto&& [st, group_deserialized] = - load_group_details(group->group_directory(), *group->encryption_key()); - RETURN_NOT_OK_TUPLE(st, std::nullopt); - - if (group_deserialized.has_value()) { - return {Status::Ok(), group_deserialized.value()}; - } - - // Return ok because having no members is acceptable if the group has never - // been written to. - return {Status::Ok(), std::nullopt}; -} - -void StorageManagerCanonical::load_group_metadata( - const shared_ptr& group_dir, - const EncryptionKey& encryption_key, - Metadata* metadata) { - auto timer_se = stats()->start_timer("sm_load_group_metadata"); - - // Special case - if (metadata == nullptr) { - return; - } - - // Determine which group metadata to load - const auto& group_metadata_to_load = group_dir->group_meta_uris(); - - auto metadata_num = group_metadata_to_load.size(); - // TBD: Might use DynamicArray when it is more capable. - std::vector> metadata_tiles(metadata_num); - throw_if_not_ok(parallel_for(compute_tp(), 0, metadata_num, [&](size_t m) { - const auto& uri = group_metadata_to_load[m].uri_; - - auto&& tile = GenericTileIO::load(resources_, uri, 0, encryption_key); - metadata_tiles[m] = tdb::make_shared(HERE(), std::move(tile)); - - return Status::Ok(); - })); - - // Compute array metadata size for the statistics - uint64_t meta_size = 0; - for (const auto& t : metadata_tiles) { - meta_size += t->size(); - } - stats()->add_counter("read_array_meta_size", meta_size); - - // Copy the deserialized metadata into the original Metadata object - *metadata = Metadata::deserialize(metadata_tiles); - metadata->set_loaded_metadata_uris(group_metadata_to_load); -} - /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 689f889fe420..50c374516430 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -166,53 +166,6 @@ class StorageManagerCanonical { */ Status group_close_for_writes(tiledb::sm::Group* group); - /** - * Loads the group metadata from persistent storage based on - * the input URI manager. - */ - void load_group_metadata( - const tdb_shared_ptr& group_dir, - const EncryptionKey& encryption_key, - Metadata* metadata); - - /** - * Load a group detail from URI - * - * @param group_uri group uri - * @param uri location to load - * @param encryption_key encryption key - * @return tuple Status and pointer to group deserialized - */ - tuple>> load_group_from_uri( - const URI& group_uri, - const URI& uri, - const EncryptionKey& encryption_key); - - /** - * Load a group detail from URIs - * - * @param group_uri group uri - * @param uri location to load - * @param encryption_key encryption key - * @return tuple Status and pointer to group deserialized - */ - tuple>> load_group_from_all_uris( - const URI& group_uri, - const std::vector& uris, - const EncryptionKey& encryption_key); - - /** - * Load group details based on group directory - * - * @param group_directory - * @param encryption_key encryption key - * - * @return tuple Status and pointer to group deserialized - */ - tuple>> load_group_details( - const shared_ptr& group_directory, - const EncryptionKey& encryption_key); - /** * Store the group details * @@ -228,25 +181,6 @@ class StorageManagerCanonical { tdb_shared_ptr group, const EncryptionKey& encryption_key); - /** - * Opens an group for reads. - * - * @param group The group to be opened. - * @return tuple of Status, latest GroupSchema and map of all group schemas - * Status Ok on success, else error - */ - std::tuple>> - group_open_for_reads(Group* group); - - /** Opens an group for writes. - * - * @param group The group to open. - * @return tuple of Status, latest GroupSchema and map of all group schemas - * Status Ok on success, else error - */ - std::tuple>> - group_open_for_writes(Group* group); - /** * Cleans up the array data. * From 00edc260e95ab578e60fe3c18b1246f50502d6d7 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 5 Feb 2024 03:41:39 -0500 Subject: [PATCH 171/456] Additional testing of TILEDB_GEOM_* types. (#4692) TYPE: NO_HISTORY --- test/src/test-cppapi-aggregates.cc | 70 +++++++++++++++--------- test/src/unit-capi-attributes.cc | 87 +++++++++++++++++++++++++++--- test/src/unit-cppapi-array.cc | 15 ++++-- 3 files changed, 136 insertions(+), 36 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 868592e1752a..07c4ee4a5a25 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -1624,31 +1624,6 @@ TEMPLATE_LIST_TEST_CASE_METHOD( array.close(); } -typedef tuple BlobTypeUnderTest; -TEMPLATE_LIST_TEST_CASE_METHOD( - CppAggregatesFx, - "C++ API: Aggregates basic sum, std::byte", - "[cppapi][aggregates][basic][sum][byte]", - BlobTypeUnderTest) { - typedef TestType T; - CppAggregatesFx::generate_test_params(); - CppAggregatesFx::create_array_and_write_fragments(); - Array array{ - CppAggregatesFx::ctx_, CppAggregatesFx::ARRAY_NAME, TILEDB_READ}; - Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - - // Add a sum aggregator to the query. - QueryChannel default_channel = QueryExperimental::get_default_channel(query); - - REQUIRE_THROWS_WITH( - QueryExperimental::create_unary_aggregate(query, "a1"), - Catch::Matchers::ContainsSubstring( - "Datatype::BLOB is not a valid Datatype")); - - // Close array. - array.close(); -} - typedef tuple< uint8_t, uint16_t, @@ -2664,3 +2639,48 @@ TEST_CASE_METHOD( QueryExperimental::create_unary_aggregate(query, "a1")); CHECK_THROWS(default_channel.apply_aggregate("Something", operation)); } + +typedef tuple BlobTypeUnderTest; +TEMPLATE_LIST_TEST_CASE( + "C++ API: Aggregates basic sum, std::byte", + "[cppapi][aggregates][basic][sum][byte]", + BlobTypeUnderTest) { + const std::string array_name = "test_byte_aggregates"; + auto datatype = GENERATE( + tiledb_datatype_t::TILEDB_BLOB, + tiledb_datatype_t::TILEDB_GEOM_WKB, + tiledb_datatype_t::TILEDB_GEOM_WKT); + + Context ctx; + VFS vfs(ctx); + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + // Create domain. + Domain domain(ctx); + auto d = Dimension::create(ctx, "d", {{1, 999}}, 2); + domain.add_dimension(d); + + // Create array schema. + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.add_attribute(Attribute::create(ctx, "a", datatype)); + + // Create array and query. + Array::create(array_name, schema); + Array array(ctx, array_name, TILEDB_READ); + Query query(ctx, array, TILEDB_READ); + + // Add a sum aggregator to the query. + QueryChannel default_channel = QueryExperimental::get_default_channel(query); + REQUIRE_THROWS_WITH( + QueryExperimental::create_unary_aggregate(query, "a"), + Catch::Matchers::ContainsSubstring("not a valid Datatype")); + + // Clean up. + array.close(); + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } +} diff --git a/test/src/unit-capi-attributes.cc b/test/src/unit-capi-attributes.cc index ca74361c9ed7..17e61fc3ec2a 100644 --- a/test/src/unit-capi-attributes.cc +++ b/test/src/unit-capi-attributes.cc @@ -302,30 +302,25 @@ TEST_CASE_METHOD( for (const auto& fs : fs_vec_) { std::string temp_dir = fs->temp_dir(); std::string array_name = temp_dir; + std::string attr_name = "a"; // serialization is not supported for memfs arrays if (serialize_ && tiledb::sm::utils::parse::starts_with(array_name, "mem://")) { continue; } - std::string attr_name = "a"; - // Create new TileDB context with file lock config disabled, rest the // same. tiledb_ctx_free(&ctx_); tiledb_vfs_free(&vfs_); - tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); REQUIRE(error == nullptr); - REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok()); - tiledb_config_free(&config); create_temp_dir(temp_dir); - create_dense_vector(array_name, attr_name, datatype); // Prepare cell buffers @@ -366,12 +361,92 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); + uint64_t ts_open = 0; + if (datatype == TILEDB_BLOB) { + // For the BLOB datatype, test with and without schema evolution. + auto evolve = GENERATE(true, false); + if (evolve) { + // Ensure the BLOB type can evolve to both GEOM types. + auto new_type = GENERATE(TILEDB_GEOM_WKB, TILEDB_GEOM_WKT); + // Add a second attribute on the schema and drop the original. + tiledb_array_schema_evolution_t* schema_evolution; + rc = tiledb_array_schema_evolution_alloc(ctx_, &schema_evolution); + REQUIRE(rc == TILEDB_OK); + tiledb_attribute_t* b; + rc = tiledb_attribute_alloc(ctx_, "b", TILEDB_BLOB, &b); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_evolution_add_attribute( + ctx_, schema_evolution, b); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_evolution_drop_attribute( + ctx_, schema_evolution, attr_name.c_str()); + REQUIRE(rc == TILEDB_OK); + // Set timestamp to avoid race condition + ts_open = tiledb_timestamp_now_ms(); + ts_open = ts_open + 1; + rc = tiledb_array_schema_evolution_set_timestamp_range( + ctx_, schema_evolution, ts_open, ts_open); + rc = tiledb_array_evolve(ctx_, array_name.c_str(), schema_evolution); + REQUIRE(rc == TILEDB_OK); + + // Add back the original attribute as new_type and drop "b". + tiledb_array_schema_evolution_t* schema_evolution2; + rc = tiledb_array_schema_evolution_alloc(ctx_, &schema_evolution2); + REQUIRE(rc == TILEDB_OK); + tiledb_attribute_t* attr; + rc = tiledb_attribute_alloc(ctx_, attr_name.c_str(), new_type, &attr); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_evolution_add_attribute( + ctx_, schema_evolution2, attr); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_evolution_drop_attribute( + ctx_, schema_evolution2, "b"); + REQUIRE(rc == TILEDB_OK); + // Set timestamp to avoid race condition + ts_open = tiledb_timestamp_now_ms(); + ts_open = ts_open + 2; + rc = tiledb_array_schema_evolution_set_timestamp_range( + ctx_, schema_evolution2, ts_open, ts_open); + if (serialize_) { + // Serialize the array schema evolution + tiledb_buffer_t* buffer; + rc = tiledb_serialize_array_schema_evolution( + ctx_, + schema_evolution2, + (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, + 0, + &buffer); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_deserialize_array_schema_evolution( + ctx_, + buffer, + (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, + 1, + &schema_evolution2); + REQUIRE(rc == TILEDB_OK); + tiledb_buffer_free(&buffer); + } + rc = tiledb_array_evolve(ctx_, array_name.c_str(), schema_evolution2); + REQUIRE(rc == TILEDB_OK); + + // Clean up + tiledb_attribute_free(&b); + tiledb_attribute_free(&attr); + tiledb_array_schema_evolution_free(&schema_evolution); + tiledb_array_schema_evolution_free(&schema_evolution2); + } + } + int buffer_read[10]; uint64_t buffer_read_size = sizeof(buffer_read); // Open array rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); + if (ts_open != 0) { + rc = tiledb_array_set_open_timestamp_end(ctx_, array, ts_open + 2); + REQUIRE(rc == TILEDB_OK); + } rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index bab2af037340..effff473a670 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -1210,20 +1210,25 @@ TEST_CASE( vfs.remove_dir(array_name); } + auto datatype = GENERATE( + tiledb_datatype_t::TILEDB_BLOB, + tiledb_datatype_t::TILEDB_GEOM_WKB, + tiledb_datatype_t::TILEDB_GEOM_WKT); + // Create Domain domain(ctx); domain.add_dimension(Dimension::create(ctx, "rows", {{0, 0}}, 1)); ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); - schema.add_attribute( - Attribute::create(ctx, "a", tiledb_datatype_t::TILEDB_BLOB)); + schema.add_attribute(Attribute::create(ctx, "a", datatype)); Array::create(array_name, schema); // Write std::byte data_w{1}; Array array_w(ctx, array_name, TILEDB_WRITE); Query query_w(ctx, array_w); - query_w.set_layout(TILEDB_GLOBAL_ORDER).set_data_buffer("a", &data_w, 1); + query_w.set_layout(TILEDB_GLOBAL_ORDER) + .set_data_buffer("a", (void*)(&data_w), 1); SECTION("no serialization") { serialize = false; @@ -1255,7 +1260,7 @@ TEST_CASE( std::byte data; query.set_layout(TILEDB_ROW_MAJOR) .set_subarray(subarray) - .set_data_buffer("a", &data, 1); + .set_data_buffer("a", (void*)(&data), 1); query.submit(); array.close(); From 49e87524c5090f8a06f8f9a4fd14bb24ed807408 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 5 Feb 2024 11:02:04 +0200 Subject: [PATCH 172/456] Remove unused code. (#4689) This PR removes some unused code, specifically: * A patch file that should have been removed in #4553. * The `EncryptionKeyValidation` class that is unused since #1197. * Setting CMake policies to NEW that are already set by `cmake_minimum_required(VERSION 3.21)` --- TYPE: NO_HISTORY --- cmake/common.cmake | 10 -- .../fix-awsmigrationhub-build.patch | 10 -- tiledb/CMakeLists.txt | 8 -- tiledb/sm/crypto/encryption_key_validation.cc | 128 ------------------ tiledb/sm/crypto/encryption_key_validation.h | 92 ------------- 5 files changed, 248 deletions(-) delete mode 100644 ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch delete mode 100644 tiledb/sm/crypto/encryption_key_validation.cc delete mode 100644 tiledb/sm/crypto/encryption_key_validation.h diff --git a/cmake/common.cmake b/cmake/common.cmake index f388e0f60014..0630e8ac02c6 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -91,13 +91,3 @@ endif() include_directories(${CMAKE_SOURCE_DIR}) cmake_path(SET TILEDB_SOURCE_ROOT NORMALIZE ${CMAKE_SOURCE_DIR}) cmake_path(APPEND TILEDB_SOURCE_ROOT "external/include" OUTPUT_VARIABLE TILEDB_EXTERNAL_INCLUDE) - -# -# Policies -# -if (POLICY CMP0076) - # CMP0076 NEW instructs `target_sources` to convert relative paths to absolute ones. - # At present this is only used for unit-test executables, not for the main - # library at the top level. - cmake_policy(SET CMP0076 NEW) -endif() \ No newline at end of file diff --git a/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch b/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch deleted file mode 100644 index c2d98030136e..000000000000 --- a/ports/aws-sdk-cpp/fix-awsmigrationhub-build.patch +++ /dev/null @@ -1,10 +0,0 @@ -diff --git a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt -index a8a888d..574b985 100644 ---- a/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt -+++ b/generated/src/aws-cpp-sdk-AWSMigrationHub/CMakeLists.txt -@@ -1,4 +1,4 @@ --add_project(aws-cpp-sdk-AWSMigrationHub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) -+add_project(aws-cpp-sdk-awsmigrationhub "C++ SDK for the AWS AWSMigrationHub service" aws-cpp-sdk-core) - - file(GLOB AWS_AWSMIGRATIONHUB_HEADERS - "include/aws/AWSMigrationHub/*.h" diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 06cf1186af6e..d35312ad8b18 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -26,13 +26,6 @@ # THE SOFTWARE. # -############################################################ -# CMake policies -############################################################ - -# C++ library, allow the VISIBLITY_PRESET for each c++ target to work -cmake_policy(SET CMP0063 NEW) - ############################################################ # Find packages needed for object libraries ############################################################ @@ -184,7 +177,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/consolidator/group_meta_consolidator.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/crypto/crypto.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/crypto/encryption_key.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/crypto/encryption_key_validation.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/crypto/crypto_openssl.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/crypto/crypto_win32.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/curl/curl_init.cc diff --git a/tiledb/sm/crypto/encryption_key_validation.cc b/tiledb/sm/crypto/encryption_key_validation.cc deleted file mode 100644 index 4980ce32029d..000000000000 --- a/tiledb/sm/crypto/encryption_key_validation.cc +++ /dev/null @@ -1,128 +0,0 @@ -/** - * @file encryption_key_validation.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines a class for encryption key validation. - */ - -#include "tiledb/sm/crypto/encryption_key_validation.h" -#include "tiledb/common/logger.h" -#include "tiledb/sm/crypto/crypto.h" -#include "tiledb/sm/crypto/encryption_key.h" -#include "tiledb/sm/enums/encryption_type.h" - -using namespace tiledb::common; - -namespace tiledb { -namespace sm { - -const std::string EncryptionKeyValidation::ENCRYPTION_KEY_CHECK_DATA = - "TILEDB_ENCRYPTION_KEY_CHECK_DATA"; - -Status EncryptionKeyValidation::check_encryption_key( - const EncryptionKey& encryption_key) { - // First time: encrypt the check data. - if (encryption_key_check_data_.size() == 0) - RETURN_NOT_OK(init_encryption_key_check_data(encryption_key)); - - // Decrypt the data and check that it is the same. - Buffer output; - ConstBuffer input( - encryption_key_check_data_.data(), encryption_key_check_data_.size()); - switch (encryption_key.encryption_type()) { - case EncryptionType::NO_ENCRYPTION: - RETURN_NOT_OK(output.write(&input, input.size())); - break; - case EncryptionType::AES_256_GCM: { - ConstBuffer iv( - encryption_key_check_data_iv_.data(), - encryption_key_check_data_iv_.size()); - ConstBuffer tag( - encryption_key_check_data_tag_.data(), - encryption_key_check_data_tag_.size()); - ConstBuffer key = encryption_key.key(); - RETURN_NOT_OK( - Crypto::decrypt_aes256gcm(&key, &iv, &tag, &input, &output)); - break; - } - default: - return LOG_STATUS(Status_EncryptionError( - "Invalid encryption key; invalid encryption type.")); - } - - if (output.size() != ENCRYPTION_KEY_CHECK_DATA.size()) - return LOG_STATUS(Status_EncryptionError("Invalid encryption key.")); - for (uint64_t i = 0; i < output.size(); i++) { - if (output.value(i * sizeof(char)) != ENCRYPTION_KEY_CHECK_DATA[i]) - return LOG_STATUS(Status_EncryptionError("Invalid encryption key.")); - } - - return Status::Ok(); -} - -Status EncryptionKeyValidation::init_encryption_key_check_data( - const EncryptionKey& encryption_key) { - encryption_key_check_data_.clear(); - encryption_key_check_data_tag_.clear(); - encryption_key_check_data_iv_.clear(); - - ConstBuffer input( - ENCRYPTION_KEY_CHECK_DATA.data(), ENCRYPTION_KEY_CHECK_DATA.size()); - - switch (encryption_key.encryption_type()) { - case EncryptionType::NO_ENCRYPTION: - RETURN_NOT_OK(encryption_key_check_data_.write(&input, input.size())); - break; - case EncryptionType::AES_256_GCM: { - RETURN_NOT_OK( - encryption_key_check_data_iv_.realloc(Crypto::AES256GCM_IV_BYTES)); - RETURN_NOT_OK( - encryption_key_check_data_tag_.realloc(Crypto::AES256GCM_TAG_BYTES)); - ConstBuffer key = encryption_key.key(); - PreallocatedBuffer iv( - encryption_key_check_data_iv_.data(), - encryption_key_check_data_iv_.alloced_size()); - PreallocatedBuffer tag( - encryption_key_check_data_tag_.data(), - encryption_key_check_data_tag_.alloced_size()); - RETURN_NOT_OK(Crypto::encrypt_aes256gcm( - &key, nullptr, &input, &encryption_key_check_data_, &iv, &tag)); - encryption_key_check_data_iv_.advance_size(Crypto::AES256GCM_IV_BYTES); - encryption_key_check_data_tag_.advance_size(Crypto::AES256GCM_TAG_BYTES); - break; - } - default: - return LOG_STATUS(Status_EncryptionError( - "Invalid encryption key; invalid encryption type.")); - } - - return Status::Ok(); -} - -} // namespace sm -} // namespace tiledb diff --git a/tiledb/sm/crypto/encryption_key_validation.h b/tiledb/sm/crypto/encryption_key_validation.h deleted file mode 100644 index 60410fabcbb7..000000000000 --- a/tiledb/sm/crypto/encryption_key_validation.h +++ /dev/null @@ -1,92 +0,0 @@ -/** - * @file encryption_key_validation.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declares a class for encryption key validation. - */ - -#ifndef TILEDB_ENCRYPTION_KEY_VALIDATION_H -#define TILEDB_ENCRYPTION_KEY_VALIDATION_H - -#include "tiledb/common/status.h" -#include "tiledb/sm/buffer/buffer.h" - -using namespace tiledb::common; - -namespace tiledb { -namespace sm { - -class EncryptionKey; - -/** - * Class which securely validates that a given encryption key is the same as a - * previously given encryption key, without storing the key itself. - */ -class EncryptionKeyValidation { - public: - /** - * Checks an encryption key. - * - * On the first invocation, the given key is used to encrypt a buffer of known - * data. On subsequent invocations, the given key is used to decrypt the - * internal encrypted buffer, which is checked for correctness against the - * known data. - * - * An error is returned if the key is invalid. - * - * @param encryption_key The encryption key to check. - * @return Status - */ - Status check_encryption_key(const EncryptionKey& encryption_key); - - private: - /** Constant string value used to check encryption keys. */ - static const std::string ENCRYPTION_KEY_CHECK_DATA; - - /** Buffer holding the encrypted bytes for the check data. */ - Buffer encryption_key_check_data_; - - /** Buffer holding the IV bytes for the check data. */ - Buffer encryption_key_check_data_iv_; - - /** Buffer holding the tag bytes for the check data. */ - Buffer encryption_key_check_data_tag_; - - /** - * Encrypt a known value with the given key and store the encrypted data. - * - * @param encryption_key The encryption key to use. - * @return Status - */ - Status init_encryption_key_check_data(const EncryptionKey& encryption_key); -}; - -} // namespace sm -} // namespace tiledb - -#endif // TILEDB_ENCRYPTION_KEY_VALIDATION_H From ad0371fb8cc87abf9aa7c23e053d40b1f87fa80d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 5 Feb 2024 11:52:54 +0200 Subject: [PATCH 173/456] Fix importing TileDB in CMake versions prior to 3.18. (#4671) #4528 introduced a single `TileDB::tiledb`exported CMake target for TileDB with either static or dynamic linkage. For compatibility with previous versions, the targets `TileDB::tiledb_shared` or `TileDB::tiledb_static` were also defined depending on the linkage, as `ALIAS`es to `TileDB::tiledb`. As it turns out however, we cannot use `ALIAS` targets, because they are always declared in the global scope prior to CMake 3.18 and if `find_package(TileDB)` is not called in the top-level `CMakeLists.txt` file, it will fail with `add_library cannot create ALIAS target "TileDB::tiledb_shared" because target "TileDB::tiledb" is imported but not globally visible.`. Nor can we switch to using `IMPORTED INTERFACE` targets and linking them to `TileDB::tiledb`, because it would bring a minor breaking change[^1]. Because `TileDB::tiledb_shared` would become an `INTERFACE` library, it does not have an `IMPORTED_LOCATION` anymore, which would cause [calls to `install_target_libs(TileDB::tiledb_shared)`](https://github.com/TileDB-Inc/TileDB-VCF/blob/5bcc79b07935ac540c56bf6ed9ee0f5d60bf247e/libtiledbvcf/cmake/Modules/FindTileDB_EP.cmake#L121) to fail. Thankfully there is another solution. We set the [`EXPORT_NAME`](https://cmake.org/cmake/help/latest/prop_tgt/EXPORT_NAME.html) of the `tiledb` target to either `tiledb_shared` or `tiledb_static` depending on the linkage, and define `TileDB::tiledb` as an `IMPORTED INTERFACE` target[^2] to the linkage-specific target. This maintains full compatibility. [^1]: Something similar is the "breaking build system change" I talked about in https://github.com/TileDB-Inc/TileDB/pull/4408#issuecomment-1757962838. After removing the `install_target_libs` calls from this repository, the change in Curl did not afffect us and we could update much more easily. [^2]: In this opposite case the unified target _must_ be an `IMPORTED INTERFACE`. We cannot get the `IMPORTED_LOCATION` of `TileDB::tiledb`, but since the target is new this is not a breaking change. --- TYPE: BUILD DESC: Fix importing TileDB in CMake versions prior to 3.18. --- cmake/inputs/Config.cmake.in | 12 ++++++++---- tiledb/CMakeLists.txt | 9 +++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in index ccea6a22cb32..6723ff3b036a 100644 --- a/cmake/inputs/Config.cmake.in +++ b/cmake/inputs/Config.cmake.in @@ -44,10 +44,14 @@ endif() include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") check_required_components("@PROJECT_NAME@") -if(@BUILD_SHARED_LIBS@ AND NOT TARGET TileDB::tiledb_shared) # BUILD_SHARED_LIBS AND NOT TARGET TileDB::tiledb_shared - add_library(TileDB::tiledb_shared ALIAS TileDB::tiledb) -elseif(NOT TARGET TileDB::tiledb_static) - add_library(TileDB::tiledb_static ALIAS TileDB::tiledb) +if(NOT TARGET TileDB::tiledb) + if(TARGET TileDB::tiledb_shared) + add_library(TileDB::tiledb INTERFACE IMPORTED) + set_target_properties(TileDB::tiledb PROPERTIES INTERFACE_LINK_LIBRARIES TileDB::tiledb_shared) + elseif(TARGET TileDB::tiledb_static) + add_library(TileDB::tiledb INTERFACE IMPORTED) + set_target_properties(TileDB::tiledb PROPERTIES INTERFACE_LINK_LIBRARIES TileDB::tiledb_static) + endif() endif() # Define a convenience all-caps variable diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index d35312ad8b18..8f93a3100373 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -835,6 +835,15 @@ endif() # the value of the BUILD_SHARED_LIBS variable. add_library(tiledb $) +# Export the target as either tiledb_shared or tiledb_static for compatibility. +# The exported config will create the unified tiledb target that links to either +# of them. +if(BUILD_SHARED_LIBS) + set_target_properties(tiledb PROPERTIES EXPORT_NAME tiledb_shared) +else() + set_target_properties(tiledb PROPERTIES EXPORT_NAME tiledb_static) +endif() + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/sm/c_api/tiledb_version.h" ver) string(REGEX MATCH "TILEDB_VERSION_MAJOR ([0-9]*)" _ ${ver}) From 4c6d530b7147baabede0e68bdb6ecc831cc2983e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 5 Feb 2024 21:44:56 +0200 Subject: [PATCH 174/456] Free some disk space before downloading the backwards compatibility test arrays. (#4700) --- TYPE: NO_HISTORY --------- Co-authored-by: Luc Rancourt --- .../build-ubuntu20.04-backwards-compatibility.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index d36324cf1ec8..df0dbbab047c 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -74,6 +74,16 @@ jobs: - name: Update tiledb_unit permissions run: chmod +x $GITHUB_WORKSPACE/build/tiledb/test/tiledb_unit + - name: Free disk space + run: | + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: 'Test backward compatibility' id: test env: From 47b3eb843061cd40bed7eb2856e12411bd51c66d Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Tue, 6 Feb 2024 04:14:09 -0800 Subject: [PATCH 175/456] Remove serialization non C.41 constructors from index_read_state_from_capnp. (#4670) Made index_read_state_from_capnp C41 compliant. --- TYPE: NO_HISTORY DESC: Remove serialization non C.41 constructors from index_read_state_from_capnp --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- .../readers/sparse_global_order_reader.cc | 38 +++++---- .../query/readers/sparse_index_reader_base.cc | 17 ++-- .../query/readers/sparse_index_reader_base.h | 82 +++++++++++++++++-- .../sparse_unordered_with_dups_reader.cc | 41 +++++----- tiledb/sm/serialization/query.cc | 34 ++++---- tiledb/sm/subarray/subarray.cc | 2 +- tiledb/sm/subarray/subarray.h | 2 +- 7 files changed, 142 insertions(+), 74 deletions(-) diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index e2ddeb8ce511..8f12ac915714 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -94,7 +94,7 @@ SparseGlobalOrderReader::SparseGlobalOrderReader( template bool SparseGlobalOrderReader::incomplete() const { - return !read_state_.done_adding_result_tiles_ || + return !read_state_.done_adding_result_tiles() || memory_used_for_coords_total_ != 0; } @@ -131,7 +131,7 @@ Status SparseGlobalOrderReader::dowork() { // Handle empty array. if (fragment_metadata_.empty()) { - read_state_.done_adding_result_tiles_ = true; + read_state_.set_done_adding_result_tiles(true); return Status::Ok(); } @@ -416,7 +416,7 @@ SparseGlobalOrderReader::create_result_tiles( auto tile_num = fragment_metadata_[f]->tile_num(); // Figure out the start index. - auto start = read_state_.frag_idx_[f].tile_idx_; + auto start = read_state_.frag_idx()[f].tile_idx_; if (!result_tiles[f].empty()) { start = std::max(start, result_tiles[f].back().tile_idx() + 1); } @@ -470,7 +470,7 @@ SparseGlobalOrderReader::create_result_tiles( logger_->debug("All result tiles loaded"); } - read_state_.done_adding_result_tiles_ = done_adding_result_tiles; + read_state_.set_done_adding_result_tiles(done_adding_result_tiles); // Return the list of tiles added. std::vector created_tiles; @@ -789,8 +789,8 @@ bool SparseGlobalOrderReader::add_next_cell_to_queue( // Increment the tile index, which should clear all tiles in // end_iteration. if (!result_tiles[frag_idx].empty()) { - read_state_.frag_idx_[frag_idx].tile_idx_++; - read_state_.frag_idx_[frag_idx].cell_idx_ = 0; + uint64_t new_tile_idx = read_state_.frag_idx()[frag_idx].tile_idx_ + 1; + read_state_.set_frag_idx(frag_idx, FragIdx(new_tile_idx, 0)); } // This fragment has more tiles potentially. @@ -876,11 +876,11 @@ void SparseGlobalOrderReader::compute_hilbert_values( template void SparseGlobalOrderReader::update_frag_idx( GlobalOrderResultTile* tile, uint64_t c) { - auto& frag_idx = read_state_.frag_idx_[tile->frag_idx()]; + auto& frag_idx = read_state_.frag_idx()[tile->frag_idx()]; auto t = tile->tile_idx(); if ((t == frag_idx.tile_idx_ && c > frag_idx.cell_idx_) || t > frag_idx.tile_idx_) { - frag_idx = FragIdx(t, c); + read_state_.set_frag_idx(tile->frag_idx(), FragIdx(t, c)); } } @@ -932,8 +932,8 @@ SparseGlobalOrderReader::merge_result_cell_slabs( // Add the tile to the queue. uint64_t cell_idx = - read_state_.frag_idx_[f].tile_idx_ == rt_it[f]->tile_idx() ? - read_state_.frag_idx_[f].cell_idx_ : + read_state_.frag_idx()[f].tile_idx_ == rt_it[f]->tile_idx() ? + read_state_.frag_idx()[f].cell_idx_ : 0; GlobalOrderResultCoords rc(&*(rt_it[f]), cell_idx); bool res = add_next_cell_to_queue( @@ -1716,8 +1716,9 @@ SparseGlobalOrderReader::respect_copy_memory_budget( while (result_cell_slabs.size() > max_cs_idx) { // Revert progress for this slab in read state, and pop it. auto& last_rcs = result_cell_slabs.back(); - read_state_.frag_idx_[last_rcs.tile_->frag_idx()] = - FragIdx(last_rcs.tile_->tile_idx(), last_rcs.start_); + read_state_.set_frag_idx( + last_rcs.tile_->frag_idx(), + FragIdx(last_rcs.tile_->tile_idx(), last_rcs.start_)); result_cell_slabs.pop_back(); } @@ -1758,8 +1759,9 @@ SparseGlobalOrderReader::compute_var_size_offsets( while (query_buffer.original_buffer_var_size_ < new_var_buffer_size) { // Revert progress for this slab in read state, and pop it. auto& last_rcs = result_cell_slabs.back(); - read_state_.frag_idx_[last_rcs.tile_->frag_idx()] = - FragIdx(last_rcs.tile_->tile_idx(), last_rcs.start_); + read_state_.set_frag_idx( + last_rcs.tile_->frag_idx(), + FragIdx(last_rcs.tile_->tile_idx(), last_rcs.start_)); result_cell_slabs.pop_back(); // Update the new var buffer size. @@ -1787,8 +1789,10 @@ SparseGlobalOrderReader::compute_var_size_offsets( new_var_buffer_size = ((OffType*)query_buffer.buffer_)[total_cells]; // Update the cell progress. - read_state_.frag_idx_[last_rcs.tile_->frag_idx()] = - FragIdx(last_rcs.tile_->tile_idx(), last_rcs.start_ + last_rcs.length_); + read_state_.set_frag_idx( + last_rcs.tile_->frag_idx(), + FragIdx( + last_rcs.tile_->tile_idx(), last_rcs.start_ + last_rcs.length_)); // Remove empty cell slab. if (last_rcs.length_ == 0) { @@ -2192,7 +2196,7 @@ void SparseGlobalOrderReader::end_iteration( storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { while (!result_tiles[f].empty() && result_tiles[f].front().tile_idx() < - read_state_.frag_idx_[f].tile_idx_) { + read_state_.frag_idx()[f].tile_idx_) { remove_result_tile(f, result_tiles[f].begin(), result_tiles); } diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index b47e6450359b..3d2861973d71 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -70,6 +70,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( StrategyParams& params, bool include_coords) : ReaderBase(stats, logger, params) + , read_state_(array_->fragment_metadata().size()) , tmp_read_state_(array_->fragment_metadata().size()) , memory_budget_(config_, reader_string) , include_coords_(include_coords) @@ -131,13 +132,13 @@ SparseIndexReaderBase::SparseIndexReaderBase( /* PROTECTED METHODS */ /* ****************************** */ -const typename SparseIndexReaderBase::ReadState* +const typename SparseIndexReaderBase::ReadState& SparseIndexReaderBase::read_state() const { - return &read_state_; + return read_state_; } -typename SparseIndexReaderBase::ReadState* SparseIndexReaderBase::read_state() { - return &read_state_; +void SparseIndexReaderBase::set_read_state(ReadState read_state) { + read_state_ = std::move(read_state); } uint64_t SparseIndexReaderBase::available_memory() { @@ -319,11 +320,10 @@ Status SparseIndexReaderBase::load_initial_data() { } auto timer_se = stats_->start_timer("load_initial_data"); - read_state_.done_adding_result_tiles_ = false; + read_state_.set_done_adding_result_tiles(false); // For easy reference. const auto dim_num = array_schema_.dim_num(); - auto fragment_num = fragment_metadata_.size(); // Load delete conditions. auto&& [st, conditions, update_values] = @@ -366,9 +366,6 @@ Status SparseIndexReaderBase::load_initial_data() { } } - // Make sure there is enough space for tiles data. - read_state_.frag_idx_.resize(fragment_num); - // Calculate ranges of tiles in the subarray, if set. if (subarray_.is_set()) { // At this point, full memory budget is available. @@ -385,7 +382,7 @@ Status SparseIndexReaderBase::load_initial_data() { // below. RETURN_NOT_OK(subarray_.precompute_all_ranges_tile_overlap( storage_manager_->compute_tp(), - read_state_.frag_idx_, + read_state_.frag_idx(), &tmp_read_state_)); if (tmp_read_state_.memory_used_tile_ranges() > diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index 33657acdc1b5..5ef454e20cad 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -296,11 +296,79 @@ class SparseIndexReaderBase : public ReaderBase { * it is really required to determine if a query is incomplete from the client * side of a cloud request. */ - struct ReadState { + class ReadState { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Delete default constructor. */ + ReadState() = delete; + + /** Constructor. + * @param frag_idxs_len The length of the fragment index vector. + */ + ReadState(size_t frag_idxs_len) + : frag_idx_(frag_idxs_len) { + } + + /** Constructor used in deserialization. */ + ReadState(std::vector&& frag_idx, bool done_adding_result_tiles) + : frag_idx_(std::move(frag_idx)) + , done_adding_result_tiles_(done_adding_result_tiles) { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Return whether the tiles that will be processed are loaded in memory. + * @return Done adding result tiles. + */ + inline bool done_adding_result_tiles() const { + return done_adding_result_tiles_; + } + + /** + * Sets the flag that determines whether the tiles that will be processed + * are loaded in memory. + * @param done_adding_result_tiles Done adding result tiles. + */ + inline void set_done_adding_result_tiles(bool done_adding_result_tiles) { + done_adding_result_tiles_ = done_adding_result_tiles; + } + + /** + * Sets a value in the fragment index vector. + * @param idx The index of the vector. + * @param val The value to set frag_idx[idx] to. + */ + inline void set_frag_idx(uint64_t idx, FragIdx val) { + if (idx >= frag_idx_.size()) { + throw std::runtime_error( + "ReadState::set_frag_idx: idx greater than frag_idx_'s size."); + } + frag_idx_[idx] = std::move(val); + } + + /** + * Returns a read-only version of the fragment index vector. + * @return The fragment index vector. + */ + const std::vector& frag_idx() const { + return frag_idx_; + } + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + private: /** The tile index inside of each fragments. */ std::vector frag_idx_; - /** Is the reader done with the query. */ + /** Have all tiles to be processed been loaded in memory? */ bool done_adding_result_tiles_; }; @@ -506,16 +574,16 @@ class SparseIndexReaderBase : public ReaderBase { /** * Returns the current read state. * - * @return pointer to the read state. + * @return const reference to the read state. */ - const ReadState* read_state() const; + const ReadState& read_state() const; /** - * Returns the current read state. + * Sets the new read state. Used only for deserialization. * - * @return pointer to the read state. + * @param read_state New read_state value. */ - ReadState* read_state(); + void set_read_state(ReadState read_state); protected: /* ********************************* */ diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index e5740481c82b..9de064abedd9 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -95,7 +95,7 @@ SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( template bool SparseUnorderedWithDupsReader::incomplete() const { - return !read_state_.done_adding_result_tiles_ || + return !read_state_.done_adding_result_tiles() || !result_tiles_leftover_.empty(); } @@ -149,7 +149,7 @@ Status SparseUnorderedWithDupsReader::dowork() { // Handle empty array. if (fragment_metadata_.empty()) { - read_state_.done_adding_result_tiles_ = true; + read_state_.set_done_adding_result_tiles(true); return Status::Ok(); } @@ -176,7 +176,7 @@ Status SparseUnorderedWithDupsReader::dowork() { // No more tiles to process, done. if (result_tiles.empty()) { - assert(read_state_.done_adding_result_tiles_); + assert(read_state_.done_adding_result_tiles()); break; } @@ -468,7 +468,7 @@ SparseUnorderedWithDupsReader::create_result_tiles() { auto tile_num = fragment_metadata_[f]->tile_num(); // Figure out the start index. - auto start = read_state_.frag_idx_[f].tile_idx_; + auto start = read_state_.frag_idx()[f].tile_idx_; // Add all tiles for this fragment. if (start == tile_num) { @@ -502,7 +502,7 @@ SparseUnorderedWithDupsReader::create_result_tiles() { logger_->debug("All result tiles loaded"); } - read_state_.done_adding_result_tiles_ = done_adding_result_tiles; + read_state_.set_done_adding_result_tiles(done_adding_result_tiles); return result_tiles; } @@ -776,7 +776,7 @@ void SparseUnorderedWithDupsReader::copy_offsets_tiles( // We might have a partially processed result tile from last run. auto min_pos_tile = 0; if (i == 0) { - min_pos_tile = read_state_.frag_idx_[rt->frag_idx()].cell_idx_; + min_pos_tile = read_state_.frag_idx()[rt->frag_idx()].cell_idx_; } auto max_pos_tile = @@ -1254,7 +1254,7 @@ void SparseUnorderedWithDupsReader::copy_fixed_data_tiles( // We might have a partially processed result tile from last run. auto min_pos_tile = 0; if (i == 0) { - min_pos_tile = read_state_.frag_idx_[rt->frag_idx()].cell_idx_; + min_pos_tile = read_state_.frag_idx()[rt->frag_idx()].cell_idx_; } auto max_pos_tile = @@ -1424,7 +1424,7 @@ SparseUnorderedWithDupsReader::resize_fixed_results_to_copy( uint64_t initial_cell_offset = cells_copied(names); uint64_t first_tile_min_pos = - read_state_.frag_idx_[result_tiles[0]->frag_idx()].cell_idx_; + read_state_.frag_idx()[result_tiles[0]->frag_idx()].cell_idx_; return resize_fixed_result_tiles_to_copy( max_num_cells, initial_cell_offset, first_tile_min_pos, result_tiles); @@ -1713,7 +1713,7 @@ bool SparseUnorderedWithDupsReader::process_tiles( if (result_tiles.size() > 0) { auto last_tile = (UnorderedWithDupsResultTile*)result_tiles.back(); - auto& frag_tile_idx = read_state_.frag_idx_[last_tile->frag_idx()]; + auto& frag_tile_idx = read_state_.frag_idx()[last_tile->frag_idx()]; last_tile_cells_copied = cell_offsets[result_tiles.size()] - cell_offsets[result_tiles.size() - 1]; if (frag_tile_idx.tile_idx_ == last_tile->tile_idx()) { @@ -1724,18 +1724,20 @@ bool SparseUnorderedWithDupsReader::process_tiles( // Adjust tile index. for (auto rt : result_tiles) { - read_state_.frag_idx_[rt->frag_idx()] = FragIdx(rt->tile_idx() + 1, 0); + read_state_.set_frag_idx(rt->frag_idx(), FragIdx(rt->tile_idx() + 1, 0)); } // If the last tile is not fully copied, save the cell index. if (result_tiles.size() > 0) { auto last_tile = (UnorderedWithDupsResultTile*)result_tiles.back(); - auto& frag_tile_idx = read_state_.frag_idx_[last_tile->frag_idx()]; if (last_tile->result_num() != last_tile_cells_copied) { - frag_tile_idx.tile_idx_ = last_tile->tile_idx(); - frag_tile_idx.cell_idx_ = - last_tile->pos_with_given_result_sum(0, last_tile_cells_copied) + 1; + read_state_.set_frag_idx( + last_tile->frag_idx(), + FragIdx( + last_tile->tile_idx(), + last_tile->pos_with_given_result_sum(0, last_tile_cells_copied) + + 1)); } } @@ -1803,7 +1805,7 @@ bool SparseUnorderedWithDupsReader::copy_tiles( uint64_t var_buffer_size = 0; if (var_sized) { uint64_t first_tile_min_pos = - read_state_.frag_idx_[result_tiles[0]->frag_idx()].cell_idx_; + read_state_.frag_idx()[result_tiles[0]->frag_idx()].cell_idx_; // Adjust the offsets buffer and make sure all data fits. auto&& [caused_overflow, new_var_buffer_size, new_result_tiles_size] = @@ -1934,7 +1936,7 @@ void SparseUnorderedWithDupsReader::process_aggregates( // The first tile might have already been processed by the last // computation. We only process a tile the first time. - if (i == 0 && read_state_.frag_idx_[rt->frag_idx()].cell_idx_ != 0) { + if (i == 0 && read_state_.frag_idx()[rt->frag_idx()].cell_idx_ != 0) { return Status::Ok(); } @@ -2001,9 +2003,10 @@ template void SparseUnorderedWithDupsReader::end_iteration( ResultTilesList& result_tiles) { // Clear result tiles that are not necessary anymore. - while (!result_tiles.empty() && - result_tiles.front().tile_idx() < - read_state_.frag_idx_[result_tiles.front().frag_idx()].tile_idx_) { + while ( + !result_tiles.empty() && + result_tiles.front().tile_idx() < + read_state_.frag_idx()[result_tiles.front().frag_idx()].tile_idx_) { const auto f = result_tiles.front().frag_idx(); remove_result_tile(f, result_tiles, result_tiles.begin()); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 8a716ca82b74..75162c69347c 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -596,18 +596,18 @@ Status read_state_to_capnp( } Status index_read_state_to_capnp( - const SparseIndexReaderBase::ReadState* read_state, + const SparseIndexReaderBase::ReadState& read_state, capnp::ReaderIndex::Builder* builder) { auto read_state_builder = builder->initReadState(); read_state_builder.setDoneAddingResultTiles( - read_state->done_adding_result_tiles_); + read_state.done_adding_result_tiles()); auto frag_tile_idx_builder = - read_state_builder.initFragTileIdx(read_state->frag_idx_.size()); - for (size_t i = 0; i < read_state->frag_idx_.size(); ++i) { - frag_tile_idx_builder[i].setTileIdx(read_state->frag_idx_[i].tile_idx_); - frag_tile_idx_builder[i].setCellIdx(read_state->frag_idx_[i].cell_idx_); + read_state_builder.initFragTileIdx(read_state.frag_idx().size()); + for (size_t i = 0; i < read_state.frag_idx().size(); ++i) { + frag_tile_idx_builder[i].setTileIdx(read_state.frag_idx()[i].tile_idx_); + frag_tile_idx_builder[i].setCellIdx(read_state.frag_idx()[i].cell_idx_); } return Status::Ok(); @@ -663,25 +663,21 @@ Status read_state_from_capnp( return Status::Ok(); } -Status index_read_state_from_capnp( +tiledb::sm::SparseIndexReaderBase::ReadState index_read_state_from_capnp( const ArraySchema& schema, - const capnp::ReadStateIndex::Reader& read_state_reader, - SparseIndexReaderBase* reader) { - auto read_state = reader->read_state(); - - read_state->done_adding_result_tiles_ = - read_state_reader.getDoneAddingResultTiles(); - + const capnp::ReadStateIndex::Reader& read_state_reader) { + bool done_reading = read_state_reader.getDoneAddingResultTiles(); assert(read_state_reader.hasFragTileIdx()); - read_state->frag_idx_.clear(); + std::vector fragment_indexes; for (const auto rcs : read_state_reader.getFragTileIdx()) { auto tile_idx = rcs.getTileIdx(); auto cell_idx = rcs.getCellIdx(); - read_state->frag_idx_.emplace_back(tile_idx, cell_idx); + fragment_indexes.emplace_back(tile_idx, cell_idx); } - return Status::Ok(); + return tiledb::sm::SparseIndexReaderBase::ReadState( + std::move(fragment_indexes), done_reading); } Status dense_read_state_from_capnp( @@ -1179,8 +1175,8 @@ Status index_reader_from_capnp( // Read state if (reader_reader.hasReadState()) - RETURN_NOT_OK(index_read_state_from_capnp( - schema, reader_reader.getReadState(), reader)); + reader->set_read_state( + index_read_state_from_capnp(schema, reader_reader.getReadState())); // Query condition if (reader_reader.hasCondition()) { diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 4f4d6e62f5e5..486c88b9c949 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -2640,7 +2640,7 @@ Status Subarray::precompute_tile_overlap( Status Subarray::precompute_all_ranges_tile_overlap( ThreadPool* const compute_tp, - std::vector& frag_tile_idx, + const std::vector& frag_tile_idx, ITileRange* tile_ranges) { auto timer_se = stats_->start_timer("read_compute_simple_tile_overlap"); diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 4b2fa5563f09..a16e92b376e0 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -707,7 +707,7 @@ class Subarray { */ Status precompute_all_ranges_tile_overlap( ThreadPool* const compute_tp, - std::vector& frag_tile_idx, + const std::vector& frag_tile_idx, ITileRange* tile_ranges); /** From 645f285da959d57ee0312f231c0c5e46a15cd92f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 8 Feb 2024 17:28:24 +0200 Subject: [PATCH 176/456] Add vcpkg triplets for RelWithDebInfo. (#4669) [SC-39876](https://app.shortcut.com/tiledb-inc/story/39876/make-relwithdebinfo-builds-readily-available-to-core-developers) --- TYPE: BUILD DESC: Add vcpkg triplets for RelWithDebInfo. --- .github/workflows/ci-linux_mac.yml | 2 +- .github/workflows/full-ci.yml | 2 +- CONTRIBUTING.md | 21 ----------- cmake/Options/TileDBToolchain.cmake | 15 ++++++++ doc/dev/BUILD.md | 35 ++++++++++++++++++- ports/triplets/arm64-osx-relwithdebinfo.cmake | 10 ++++++ ports/triplets/x64-linux-relwithdebinfo.cmake | 8 +++++ ports/triplets/x64-osx-relwithdebinfo.cmake | 10 ++++++ 8 files changed, 79 insertions(+), 24 deletions(-) create mode 100644 ports/triplets/arm64-osx-relwithdebinfo.cmake create mode 100644 ports/triplets/x64-linux-relwithdebinfo.cmake create mode 100644 ports/triplets/x64-osx-relwithdebinfo.cmake diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 4be5997183ff..c6c68362ed2f 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -63,7 +63,7 @@ env: CC: ${{ inputs.matrix_compiler_cc }} CFLAGS: ${{ inputs.matrix_compiler_cflags }} CXXFLAGS: ${{ inputs.matrix_compiler_cxxflags }} - bootstrap_args: "--enable-ccache ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" + bootstrap_args: "--enable-ccache --vcpkg-base-triplet=x64-${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'linux' || 'osx' }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' SCCACHE_GHA_ENABLED: "true" diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index a525a9cf382a..a710d9761fbe 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -101,7 +101,7 @@ jobs: matrix_compiler_cc: 'gcc-10' matrix_compiler_cxx: 'g++-10' timeout: 120 - bootstrap_args: '--enable-serialization --vcpkg-base-triplet=x64-linux' + bootstrap_args: '--enable-serialization' asan: true ci10: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 71d27bd3f953..627ab9af9e4b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,27 +61,6 @@ Formatting conventions: - comments are good, TileDB uses [doxygen](http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html) for class doc strings. - format code using [clang-format](https://clang.llvm.org/docs/ClangFormat.html) -### Building with sanitizers - -TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as the vcpkg base triplet corresponding to your platform. The following platforms support sanitizers: - -* `arm64-osx` -* `x64-linux` -* `x64-osx` -* `x64-windows` - -> [!NOTE] -> Currently only the `address` sanitizer is supported. - -```bash -cd TileDB && mkdir build-asan && cd build-asan -../bootstrap --enable-sanitizer=address --vcpkg-base-triplet=x64-linux -make && make check -``` - -> [!IMPORTANT] -> To avoid errors, building with sanitizers must be done in a separate build directory. - ### Pull Requests: - `dev` is the development branch, all PR’s should be rebased on top of the latest `dev` commit. diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index d80215128a8a..40befe0460ae 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -65,6 +65,21 @@ if(TILEDB_SANITIZER STREQUAL "address") set(VCPKG_TARGET_TRIPLET "${TILEDB_VCPKG_BASE_TRIPLET}-asan") endif() +get_cmake_property(is_multi_config GENERATOR_IS_MULTI_CONFIG) +# On Windows vcpkg always builds dependencies with symbols. +# https://github.com/microsoft/vcpkg/blob/master/scripts/toolchains/windows.cmake +if(NOT WIN32 AND NOT is_multi_config AND CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" AND NOT VCPKG_TARGET_TRIPLET) + if(TILEDB_SANITIZER STREQUAL "address") + message(FATAL_ERROR "Cannot enable both RelWithDebInfo and ASAN at the same time.") + endif() + if(TILEDB_VCPKG_BASE_TRIPLET) + message(STATUS "Overriding vcpkg triplet to ${TILEDB_VCPKG_BASE_TRIPLET}-relwithdebinfo") + set(VCPKG_TARGET_TRIPLET "${TILEDB_VCPKG_BASE_TRIPLET}-relwithdebinfo") + else() + message(WARNING "Dependencies will be built without symbols. You have to set either VCPKG_TARGET_TRIPLET or TILEDB_VCPKG_BASE_TRIPLET.") + endif() +endif() + set(VCPKG_INSTALL_OPTIONS "--no-print-usage") macro(tiledb_vcpkg_enable_if tiledb_feature vcpkg_feature) diff --git a/doc/dev/BUILD.md b/doc/dev/BUILD.md index 4f0e0bc898c0..1717cb0fa908 100644 --- a/doc/dev/BUILD.md +++ b/doc/dev/BUILD.md @@ -66,6 +66,7 @@ The following are the most common configuration options: |macOS/Linux flag|Windows flag|CMake variable|Description| |----------------|------------|--------------|-----------| |`--prefix=PREFIX`|`-Prefix=PREFIX`|`CMAKE_INSTALL_PREFIX=`|Install files in tree rooted at `PREFIX` (defaults to `TileDB/dist`)| +|`--vcpkg-base-triplet=TRIPLET`|`-VcpkgBaseTriplet=TRIPLET`|`TILEDB_VCPKG_BASE_TRIPLET=TRIPLET`|Vcpkg base triplet, needed for features like ASAN and RelWithDebInfo| |`--linkage=shared/static`|`-Linkage=shared/static`|`BUILD_SHARED_LIBS=ON/OFF`|Linkage of the compiled TileDB library (defaults to `shared`) | |`--remove-deprecations`|`-RemoveDeprecations`|`TILEDB_REMOVE_DEPRECATIONS=ON`|Build TileDB without deprecated APIs| |`--enable-debug`|`-EnableDebug`|`CMAKE_BUILD_TYPE=Debug`|Enables debug build| @@ -84,6 +85,13 @@ The following are the most common configuration options: |`--disable-stats`|`-DisableStats`|`TILEDB_STATS=OFF`|Disables internal TileDB statistics| |`--disable-tests`|`-DisableTests`|`TILEDB_TESTS=OFF`|Disables building the TileDB test suite| +The supported vcpkg base triplet values are: + +* `arm64-osx` +* `x64-linux` +* `x64-osx` +* `x64-windows` + > [!TIP] > You can see all TileDB-specific CMake variables in [BuildOptions.cmake](../../cmake/Options/BuildOptions.cmake). @@ -125,4 +133,29 @@ Vcpkg will not be automatically downloaded if: * The `TILEDB_DISABLE_AUTO_VCPKG` environment variable has been defined. * The build tree has been configured by directly calling CMake and the `CMAKE_TOOLCHAIN_FILE` variable has been set by the user. -In these cases no dependencies CMake will find the dependencies based on the rules of the [`find_package`](https://cmake.org/cmake/help/latest/command/find_package.html#command:find_package) command. The user is responsible for providing them. +In these cases CMake will find the dependencies based on the rules of the [`find_package`](https://cmake.org/cmake/help/latest/command/find_package.html) command. The user is responsible for providing the dependencies. + +### Building with sanitizers + +TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as the [vcpkg base triplet](#configuration-options) corresponding to your platform: + +> [!NOTE] +> Currently only the `address` sanitizer is supported. + +```bash +cd TileDB && mkdir build-asan && cd build-asan +../bootstrap --enable-sanitizer=address --vcpkg-base-triplet=x64-linux +make && make check +``` + +> [!IMPORTANT] +> To avoid errors, building with sanitizers must be done in a separate build directory. + +### Building with optimizations and debug symbols + +TileDB supports configuring in `RelWithDebInfo` mode, which compiles code with optimizations while also emitting debug symbols. However on non-Windows platforms the dependencies built by vcpkg do not build by default with symbols. To enable that you have to do either of the following: + +* [Specify a vcpkg base triplet](#configuration-options). +* Configure by directly calling CMake and setting a vcpkg triplet with the `VCPKG_DEFAULT_TRIPLET` variable. In this case you are responsible to ensure the appropriate options are passed to the triplet file. + +Configuring in `RelWithDebInfo` mode and enabling ASAN at the same time is not supported. diff --git a/ports/triplets/arm64-osx-relwithdebinfo.cmake b/ports/triplets/arm64-osx-relwithdebinfo.cmake new file mode 100644 index 000000000000..592c7f2ae832 --- /dev/null +++ b/ports/triplets/arm64-osx-relwithdebinfo.cmake @@ -0,0 +1,10 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) + +set(VCPKG_CXX_FLAGS "-g") +set(VCPKG_C_FLAGS "-g") diff --git a/ports/triplets/x64-linux-relwithdebinfo.cmake b/ports/triplets/x64-linux-relwithdebinfo.cmake new file mode 100644 index 000000000000..533c1f3928cb --- /dev/null +++ b/ports/triplets/x64-linux-relwithdebinfo.cmake @@ -0,0 +1,8 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_CXX_FLAGS "-g") +set(VCPKG_C_FLAGS "-g") diff --git a/ports/triplets/x64-osx-relwithdebinfo.cmake b/ports/triplets/x64-osx-relwithdebinfo.cmake new file mode 100644 index 000000000000..db1bfabf76d9 --- /dev/null +++ b/ports/triplets/x64-osx-relwithdebinfo.cmake @@ -0,0 +1,10 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) +set(VCPKG_OSX_DEPLOYMENT_TARGET 11) + +set(VCPKG_CXX_FLAGS "-g") +set(VCPKG_C_FLAGS "-g") From 17bcc858a86d46886d829b2d9a5c7d61081bc810 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Fri, 9 Feb 2024 02:58:04 -0600 Subject: [PATCH 177/456] Fix stats bug in StorageManager (#4708) I just stumbled across the fact that Serializer::size() returns semantically different values depending on whether its in size computing mode vs serialization mode. In size computing mode, it returns the number of bytes "written" which is the required buffer size. In actual serializing stuff mode, it returns how much of the buffer is left to be written, so generally should return 0 after a complete serialization. Just to see if anyone was misusing that I moved size() to the size computing serializer and lo and behold, there was a single case where we had buggy behavior. Fix is to just record the size calculated by the size computing serializer. --- TYPE: BUG DESC: Fix stats bug in StorageManager --- tiledb/sm/storage_manager/storage_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 51649923d6a0..b58e7bf2bc0b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1376,7 +1376,7 @@ Status StorageManagerCanonical::store_metadata( Serializer serializer(tile.data(), tile.size()); metadata->serialize(serializer); - stats()->add_counter("write_meta_size", serializer.size()); + stats()->add_counter("write_meta_size", size_computation_serializer.size()); // Create a metadata file name URI metadata_uri = metadata->get_uri(uri); From 2f03f0c079220b88cdf3a94f5befdd2ce557eca2 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:12:02 +0100 Subject: [PATCH 178/456] Remove random_label in favor of uuid. (#4709) The random label work broke the python tests because it can generate the same URI under certain conditions. --- TYPE: NO_HISTORY DESC: Remove random_label in favor of uuid. --- tiledb/CMakeLists.txt | 1 + tiledb/sm/array/CMakeLists.txt | 1 + tiledb/sm/array_schema/CMakeLists.txt | 5 +- tiledb/sm/array_schema/enumeration.cc | 8 +- tiledb/sm/misc/CMakeLists.txt | 16 ++ tiledb/sm/misc/test/CMakeLists.txt | 3 +- tiledb/sm/misc/test/compile_uuid_main.cc | 34 +++++ tiledb/sm/misc/test/unit_uuid.cc | 87 +++++++++++ tiledb/sm/misc/uuid.cc | 174 ++++++++++++++++++++++ tiledb/sm/misc/uuid.h | 59 ++++++++ tiledb/storage_format/uri/CMakeLists.txt | 2 +- tiledb/storage_format/uri/generate_uri.cc | 8 +- 12 files changed, 388 insertions(+), 10 deletions(-) create mode 100644 tiledb/sm/misc/test/compile_uuid_main.cc create mode 100644 tiledb/sm/misc/test/unit_uuid.cc create mode 100644 tiledb/sm/misc/uuid.cc create mode 100644 tiledb/sm/misc/uuid.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 8f93a3100373..a9222730871a 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -236,6 +236,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/tdb_time.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/types.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/uuid.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc diff --git a/tiledb/sm/array/CMakeLists.txt b/tiledb/sm/array/CMakeLists.txt index 737f99cde7b7..c0b55afab26d 100644 --- a/tiledb/sm/array/CMakeLists.txt +++ b/tiledb/sm/array/CMakeLists.txt @@ -37,6 +37,7 @@ commence(object_library array) baseline fragment generic_tile_io + uuid vfs ) if(TILEDB_STATS) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 602d9b9af264..171331ad3166 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -38,7 +38,8 @@ commence(object_library attribute) constants filter_pipeline range - stringx) + stringx + uuid) conclude(object_library) # @@ -62,7 +63,7 @@ conclude(object_library) # commence(object_library enumeration) this_target_sources(enumeration.cc) - this_target_object_libraries(buffer constants seedable_global_PRNG) + this_target_object_libraries(buffer constants uuid) conclude(object_library) # diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index 3079e8b03d99..c6b2ab9633f7 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -33,7 +33,7 @@ #include #include -#include "tiledb/common/random/random_label.h" +#include "tiledb/sm/misc/uuid.h" #include "enumeration.h" @@ -71,8 +71,10 @@ Enumeration::Enumeration( } if (path_name_.empty()) { - path_name_ = "__" + tiledb::common::random_label() + "_" + - std::to_string(constants::enumerations_version); + std::string tmp_uuid; + throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); + path_name_ = + "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); } if (path_name.find("/") != std::string::npos) { diff --git a/tiledb/sm/misc/CMakeLists.txt b/tiledb/sm/misc/CMakeLists.txt index 3c8733c69d56..f6b033b82741 100644 --- a/tiledb/sm/misc/CMakeLists.txt +++ b/tiledb/sm/misc/CMakeLists.txt @@ -71,6 +71,22 @@ commence(object_library time) this_target_sources(tdb_time.cc) conclude(object_library) +# +# `uuid` object library +# +commence(object_library uuid) + this_target_sources(uuid.cc) + this_target_object_libraries(baseline) + if(WIN32) + this_target_link_libraries(rpcrt4) + else() + find_package(OpenSSL_EP REQUIRED) + this_target_link_libraries(OpenSSL::Crypto) + endif() +conclude(object_library) + +add_test_subdirectory() + # # `mgc_dict.*` tests are declared in this directory for the moment. # diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index be13e19e3d31..c61fe26da6fc 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -26,7 +26,7 @@ include(unit_test) commence(unit_test misc) - this_target_object_libraries(math) + this_target_object_libraries(math uuid) this_target_link_libraries(tiledb_test_support_lib) # change to `this_target_include_directories` when available target_include_directories(unit_misc PRIVATE "${CMAKE_SOURCE_DIR}") @@ -36,5 +36,6 @@ commence(unit_test misc) unit_hilbert.cc unit_integral_type_casts.cc unit_math.cc + unit_uuid.cc ) conclude(unit_test) diff --git a/tiledb/sm/misc/test/compile_uuid_main.cc b/tiledb/sm/misc/test/compile_uuid_main.cc new file mode 100644 index 000000000000..2db30c1d6e52 --- /dev/null +++ b/tiledb/sm/misc/test/compile_uuid_main.cc @@ -0,0 +1,34 @@ +/** + * @file compile_uuid_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2021 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../uuid.h" + +int main() { + (void)tiledb::sm::uuid::generate_uuid(nullptr, false); + return 0; +} diff --git a/tiledb/sm/misc/test/unit_uuid.cc b/tiledb/sm/misc/test/unit_uuid.cc new file mode 100644 index 000000000000..8dbf830b29cd --- /dev/null +++ b/tiledb/sm/misc/test/unit_uuid.cc @@ -0,0 +1,87 @@ +/** + * @file unit_uuid.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2018-2022 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the UUID utility functions. + */ + +#include +#include +#include +#include + +#include "tiledb/sm/global_state/global_state.h" +#include "tiledb/sm/misc/uuid.h" + +using namespace tiledb::sm; + +std::mutex catch2_macro_mutex; + +// A thread-safe variant of the REQUIRE macro. +#define REQUIRE_SAFE(a) \ + { \ + std::lock_guard lock(catch2_macro_mutex); \ + REQUIRE(a); \ + } + +void cancel_all_tasks(StorageManager*) { +} + +TEST_CASE("UUID: Test generate", "[uuid]") { + SECTION("- Serial") { + std::string uuid0, uuid1, uuid2; + REQUIRE(uuid::generate_uuid(&uuid0).ok()); + REQUIRE(uuid0.length() == 36); + REQUIRE(uuid::generate_uuid(&uuid1).ok()); + REQUIRE(uuid1.length() == 36); + REQUIRE(uuid0 != uuid1); + + REQUIRE(uuid::generate_uuid(&uuid2, false).ok()); + REQUIRE(uuid2.length() == 32); + } + + SECTION("- Threaded") { + const unsigned nthreads = 20; + std::vector uuids(nthreads); + std::vector threads; + for (unsigned i = 0; i < nthreads; i++) { + threads.emplace_back([&uuids, i]() { + std::string& uuid = uuids[i]; + REQUIRE_SAFE(uuid::generate_uuid(&uuid).ok()); + REQUIRE_SAFE(uuid.length() == 36); + }); + } + for (auto& t : threads) { + t.join(); + } + // Check uniqueness + std::set uuid_set; + uuid_set.insert(uuids.begin(), uuids.end()); + REQUIRE(uuid_set.size() == uuids.size()); + } +} diff --git a/tiledb/sm/misc/uuid.cc b/tiledb/sm/misc/uuid.cc new file mode 100644 index 000000000000..052cce47bdcd --- /dev/null +++ b/tiledb/sm/misc/uuid.cc @@ -0,0 +1,174 @@ +/** + * @file uuid.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2018-2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines a platform-independent UUID generator. + */ + +#include +#include + +#include "tiledb/sm/misc/uuid.h" + +#ifdef _WIN32 +#include +#else +#include +#include +#include +#endif + +using namespace tiledb::common; + +namespace tiledb::sm::uuid { + +/** Mutex to guard UUID generation. */ +static std::mutex uuid_mtx; + +#ifdef _WIN32 + +/** + * Generate a UUID using Win32 RPC API. + */ +Status generate_uuid_win32(std::string* uuid_str) { + if (uuid_str == nullptr) + return Status_UtilsError("Null UUID string argument"); + + UUID uuid; + RPC_STATUS rc = UuidCreate(&uuid); + if (rc != RPC_S_OK) + return Status_UtilsError("Unable to generate Win32 UUID: creation error"); + + char* buf = nullptr; + rc = UuidToStringA(&uuid, reinterpret_cast(&buf)); + if (rc != RPC_S_OK) + return Status_UtilsError( + "Unable to generate Win32 UUID: string conversion error"); + + *uuid_str = std::string(buf); + + rc = RpcStringFreeA(reinterpret_cast(&buf)); + if (rc != RPC_S_OK) + return Status_UtilsError("Unable to generate Win32 UUID: free error"); + + return Status::Ok(); +} + +#else + +/** + * Generate a UUID using OpenSSL. + * + * Initially from: https://gist.github.com/kvelakur/9069c9896577c3040030 + * "Generating a Version 4 UUID using OpenSSL" + */ +Status generate_uuid_openssl(std::string* uuid_str) { + if (uuid_str == nullptr) + return Status_UtilsError("Null UUID string argument"); + + union { + struct { + uint32_t time_low; + uint16_t time_mid; + uint16_t time_hi_and_version; + uint8_t clk_seq_hi_res; + uint8_t clk_seq_low; + uint8_t node[6]; + }; + uint8_t __rnd[16]; + } uuid; + + int rc = RAND_bytes(uuid.__rnd, sizeof(uuid)); + if (rc < 1) { + char err_msg[256]; + ERR_error_string_n(ERR_get_error(), err_msg, sizeof(err_msg)); + return Status_UtilsError( + "Cannot generate random bytes with OpenSSL: " + std::string(err_msg)); + } + + // Refer Section 4.2 of RFC-4122 + // https://tools.ietf.org/html/rfc4122#section-4.2 + uuid.clk_seq_hi_res = (uint8_t)((uuid.clk_seq_hi_res & 0x3F) | 0x80); + uuid.time_hi_and_version = + (uint16_t)((uuid.time_hi_and_version & 0x0FFF) | 0x4000); + + // Format the UUID as a string. + char buf[128]; + rc = snprintf( + buf, + sizeof(buf), + "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", + uuid.time_low, + uuid.time_mid, + uuid.time_hi_and_version, + uuid.clk_seq_hi_res, + uuid.clk_seq_low, + uuid.node[0], + uuid.node[1], + uuid.node[2], + uuid.node[3], + uuid.node[4], + uuid.node[5]); + + if (rc < 0) + return Status_UtilsError("Error formatting UUID string"); + + *uuid_str = std::string(buf); + + return Status::Ok(); +} + +#endif + +Status generate_uuid(std::string* uuid, bool hyphenate) { + if (uuid == nullptr) + return Status_UtilsError("Null UUID string argument"); + + std::string uuid_str; + { + // OpenSSL is not threadsafe, so grab a lock here. We are locking in the + // Windows case as well just to be careful. + std::unique_lock lck(uuid_mtx); +#ifdef _WIN32 + RETURN_NOT_OK(generate_uuid_win32(&uuid_str)); +#else + RETURN_NOT_OK(generate_uuid_openssl(&uuid_str)); +#endif + } + + uuid->clear(); + for (unsigned i = 0; i < uuid_str.length(); i++) { + if (uuid_str[i] == '-' && !hyphenate) + continue; + uuid->push_back(uuid_str[i]); + } + + return Status::Ok(); +} + +} // namespace tiledb::sm::uuid diff --git a/tiledb/sm/misc/uuid.h b/tiledb/sm/misc/uuid.h new file mode 100644 index 000000000000..5999007f8b59 --- /dev/null +++ b/tiledb/sm/misc/uuid.h @@ -0,0 +1,59 @@ +/** + * @file uuid.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares a platform-independent UUID generator. + */ + +#ifndef TILEDB_UUID_H +#define TILEDB_UUID_H + +#include "tiledb/common/status.h" + +using namespace tiledb::common; + +namespace tiledb { +namespace sm { +namespace uuid { + +/** + * Generates a 128-bit UUID. The string is formatted with hyphens like: + * 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx' where 'x' is a hexadecimal digit. + * Note: this function internally acquires a lock. + * + * @param uuid Output parameter which will store the UUID in string format. + * @param hyphenate If false, the UUID string will not be hyphenated. + * @return Status + */ +Status generate_uuid(std::string* uuid, bool hyphenate = true); + +} // namespace uuid +} // namespace sm +} // namespace tiledb + +#endif diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index b0b7949accf7..e92ebae65092 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -31,7 +31,7 @@ include(object_library) # commence(object_library uri_format) this_target_sources(parse_uri.cc generate_uri.cc) - this_target_object_libraries(baseline fragment time vfs) + this_target_object_libraries(baseline fragment time uuid vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index d823936c13d2..fe0e0a9738aa 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -27,9 +27,9 @@ */ #include "tiledb/storage_format/uri/generate_uri.h" -#include "tiledb/common/random/random_label.h" #include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/misc/uuid.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -42,6 +42,9 @@ std::string generate_timestamped_name( uint64_t timestamp_start, uint64_t timestamp_end, std::optional version) { + std::string uuid; + throw_if_not_ok(sm::uuid::generate_uuid(&uuid, false)); + if (timestamp_start > timestamp_end) { throw std::logic_error( "Error generating timestamped name; " @@ -49,8 +52,7 @@ std::string generate_timestamped_name( } std::stringstream ss; - ss << "/__" << timestamp_start << "_" << timestamp_end << "_" - << random_label(); + ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid; if (version.has_value()) { ss << "_" << version.value(); From 437340d5c548084a88ba1a1452ae723cbfbf1cb4 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 9 Feb 2024 16:14:35 +0100 Subject: [PATCH 179/456] Optimize sparse reads when full non empty domain is requested. (#4710) This change allows to optimize sparse reads where the only range set covers the non-empty domain of the fragments to be read. This is the case for python. --- TYPE: IMPROVEMENT DESC: Optimize sparse reads when full non empty domain is requested. --- tiledb/sm/array/array.h | 1 + .../readers/sparse_global_order_reader.cc | 2 ++ .../sparse_unordered_with_dups_reader.cc | 7 +++--- tiledb/sm/subarray/subarray.cc | 23 +++++++++++++++++++ tiledb/sm/subarray/subarray.h | 11 +++++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index d9f0c8b6f48d..fffcee67387b 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -764,6 +764,7 @@ class Array { * has not been computed or loaded it will be loaded first */ const NDRange non_empty_domain(); + /** * Retrieves the array metadata object that is already loaded. If it's not yet * loaded it will be empty. diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 8f12ac915714..c5a78d31788e 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -119,6 +119,8 @@ Status SparseGlobalOrderReader::dowork() { auto timer_se = stats_->start_timer("dowork"); stats_->add_counter("loop_num", 1); + subarray_.reset_default_ranges(); + // Check that the query condition is valid. if (condition_.has_value()) { throw_if_not_ok(condition_->check(array_schema_)); diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 9de064abedd9..09fd3e70cf16 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -122,12 +122,13 @@ void SparseUnorderedWithDupsReader::refresh_config() { template Status SparseUnorderedWithDupsReader::dowork() { - // Subarray is not known to be explicitly set until buffers are deserialized - include_coords_ = subarray_.is_set(); - auto timer_se = stats_->start_timer("dowork"); stats_->add_counter("loop_num", 1); + // Subarray is not known to be explicitly set until buffers are deserialized + subarray_.reset_default_ranges(); + include_coords_ = subarray_.is_set(); + // Make sure user didn't request delete timestamps. if (buffers_.count(constants::delete_timestamps) != 0) { return logger_->status(Status_SparseUnorderedWithDupsReaderError( diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 486c88b9c949..426beba8ea87 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -2131,6 +2131,29 @@ void Subarray::add_default_label_ranges(dimension_size_type dim_num) { label_range_subset_.resize(dim_num, nullopt); } +void Subarray::reset_default_ranges() { + if (array_->non_empty_domain_computed()) { + auto dim_num = array_->array_schema_latest().dim_num(); + auto& domain{array_->array_schema_latest().domain()}; + + // Process all dimensions one by one. + for (unsigned d = 0; d < dim_num; d++) { + // Only enter the check if there are only one range set on the dimension. + if (!is_default_[d] && range_subset_[d].num_ranges() == 1) { + // If the range set is the same as the non empty domain. + auto& ned = array_->non_empty_domain()[d]; + if (ned == range_subset_[d][0]) { + // Reset the default flag and reset the range subset to be default. + is_default_[d] = true; + auto dim{domain.dimension_ptr(d)}; + range_subset_[d] = RangeSetAndSuperset( + dim->type(), dim->domain(), true, coalesce_ranges_); + } + } + } + } +} + void Subarray::compute_range_offsets() { range_offsets_.clear(); diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index a16e92b376e0..98fca20e9af3 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -1306,6 +1306,17 @@ class Subarray { */ void add_default_label_ranges(dimension_size_type dim_num); + /** + * Reset ranges to default if possible before a read operation for sparse + * reads. We have a lot of optimizations in the sparse readers when no ranges + * are specified. Python will set ranges that are equal to the non empty + * domain, which will negate those optimizations. When the non empty domain is + * computed for the array, it is low performance cost to see if the ranges set + * are equal to the non empty domain. If they are, we can reset them to be + * default. + */ + void reset_default_ranges(); + private: /* ********************************* */ /* PRIVATE DATA TYPES */ From 02b372a1bbe8537d5dbbecdfc4f55b994a3c8f5c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 9 Feb 2024 21:27:12 +0200 Subject: [PATCH 180/456] Add explicit casts to `size_t`. (#4712) Fixes #4704. --- TYPE: BUG DESC: Add explicit casts to fix compile errors. --- tiledb/sm/array_schema/enumeration.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h index 82e37ce78159..950ca447dc92 100644 --- a/tiledb/sm/array_schema/enumeration.h +++ b/tiledb/sm/array_schema/enumeration.h @@ -306,7 +306,8 @@ class Enumeration { size of the buffer pointed to. */ span data() const { - return {static_cast(data_.data()), data_.size()}; + return { + static_cast(data_.data()), static_cast(data_.size())}; } /** @@ -316,7 +317,9 @@ class Enumeration { * the size of the buffer pointed to. */ span offsets() const { - return {static_cast(offsets_.data()), offsets_.size()}; + return { + static_cast(offsets_.data()), + static_cast(offsets_.size())}; } /** From fbc1a0faa3199795e8865240ca53aa0fd51b95dd Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Fri, 9 Feb 2024 15:28:10 -0600 Subject: [PATCH 181/456] Ignore Nova.app configuration (#4716) This is the same as we do for VSCode and CLion. --- TYPE: NO_HISTORY DESC: Ignore Nova.app configuration --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 051e2eb8e318..5c529f44c40e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.nova* .vscode* *.sw? build/* From c45956cd2b7e303ac60434c1f857fcd76450f9fe Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Sat, 10 Feb 2024 19:05:54 +0100 Subject: [PATCH 182/456] Don't use non empty domain on empty array. (#4719) This was introduced in https://github.com/TileDB-Inc/TileDB/pull/4710. In that PR, we could try to use the non empty domain of an array with no fragments, which is empty. This should fix python and R nightlies. --- TYPE: NO_HISTORY DESC: Don't use non empty domain on empty array. --- test/src/unit-cppapi-array.cc | 34 +++++++++++++++++++ .../readers/sparse_global_order_reader.cc | 4 +-- .../sparse_unordered_with_dups_reader.cc | 8 ++--- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index effff473a670..12ec1d2f1dcb 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -2209,3 +2209,37 @@ TEST_CASE( CHECK(i > 0); } + +TEST_CASE("C++ API: Read empty array", "[cppapi][read-empty-array]") { + const std::string array_name_1d = "cpp_unit_array_1d"; + Context ctx; + VFS vfs(ctx); + + bool dups = GENERATE(true, false); + + if (vfs.is_dir(array_name_1d)) { + vfs.remove_dir(array_name_1d); + } + + ArraySchema schema(ctx, TILEDB_SPARSE); + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "d", {{0, 1000}}, 1001)); + schema.set_domain(domain); + schema.add_attribute(Attribute::create(ctx, "a")); + schema.set_allows_dups(dups); + Array::create(array_name_1d, schema); + Array array(ctx, array_name_1d, TILEDB_READ); + + std::vector d(1); + std::vector a(1); + Query q(ctx, array, TILEDB_READ); + q.set_layout(TILEDB_UNORDERED); + q.set_data_buffer("d", d); + q.set_data_buffer("a", a); + q.submit(); + array.close(); + + if (vfs.is_dir(array_name_1d)) { + vfs.remove_dir(array_name_1d); + } +} diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index c5a78d31788e..c4703673426c 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -119,8 +119,6 @@ Status SparseGlobalOrderReader::dowork() { auto timer_se = stats_->start_timer("dowork"); stats_->add_counter("loop_num", 1); - subarray_.reset_default_ranges(); - // Check that the query condition is valid. if (condition_.has_value()) { throw_if_not_ok(condition_->check(array_schema_)); @@ -137,6 +135,8 @@ Status SparseGlobalOrderReader::dowork() { return Status::Ok(); } + subarray_.reset_default_ranges(); + // Load initial data, if not loaded already. throw_if_not_ok(load_initial_data()); purge_deletes_consolidation_ = !deletes_consolidation_no_purge_ && diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 09fd3e70cf16..f51cf512c65f 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -125,10 +125,6 @@ Status SparseUnorderedWithDupsReader::dowork() { auto timer_se = stats_->start_timer("dowork"); stats_->add_counter("loop_num", 1); - // Subarray is not known to be explicitly set until buffers are deserialized - subarray_.reset_default_ranges(); - include_coords_ = subarray_.is_set(); - // Make sure user didn't request delete timestamps. if (buffers_.count(constants::delete_timestamps) != 0) { return logger_->status(Status_SparseUnorderedWithDupsReaderError( @@ -154,6 +150,10 @@ Status SparseUnorderedWithDupsReader::dowork() { return Status::Ok(); } + // Subarray is not known to be explicitly set until buffers are deserialized + subarray_.reset_default_ranges(); + include_coords_ = subarray_.is_set(); + // Load initial data, if not loaded already. Coords are only included if the // subarray is set. throw_if_not_ok(load_initial_data()); From 9b28f75e10b27cf3a7b6604ea273d9acae02a18f Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 12 Feb 2024 19:31:40 +0200 Subject: [PATCH 183/456] Remove serialization non C41 constructors from stats (#4706) With this change in, PR #4685 and others to follow should: a) remove `Subarray::set_stats` and respective call from `serialization/query.cc` b) where `Subarray is constructed in `serialization/query.cc`, change constructor call with the following code snippet: ``` auto &stats_data = stats_from_capnp(reader.getStats()); Subarray subarray(array, layout, query_stats, stats_data, dummy_logger, true); ``` c) The constructor calls parent_stats->create_child(prefix, stats_data); d) When all migrations are done, make `Stats::populate_with_data private` --- TYPE: NO_HISTORY DESC: Remove serialization non C41 constructors from stats --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/sm/query/query.cc | 6 + tiledb/sm/query/query.h | 8 ++ tiledb/sm/query/strategy_base.cc | 4 + tiledb/sm/query/strategy_base.h | 8 ++ tiledb/sm/serialization/query.cc | 151 ++++++++------------- tiledb/sm/stats/global_stats.h | 68 +++++++++- tiledb/sm/stats/stats.cc | 26 +++- tiledb/sm/stats/stats.h | 82 ++++++++++- tiledb/sm/subarray/subarray.cc | 8 +- tiledb/sm/subarray/subarray.h | 10 +- tiledb/sm/subarray/subarray_partitioner.cc | 8 +- tiledb/sm/subarray/subarray_partitioner.h | 9 +- 12 files changed, 277 insertions(+), 111 deletions(-) diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 32bcd7199b64..713dd5ec6915 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -827,6 +827,7 @@ Status Query::process() { if (!only_dim_label_query()) { if (strategy_ != nullptr) { + // The strategy destructor should reset its own Stats object here dynamic_cast(strategy_.get())->stats()->reset(); strategy_ = nullptr; } @@ -917,6 +918,7 @@ Status Query::reset_strategy_with_layout( Layout layout, bool force_legacy_reader) { force_legacy_reader_ = force_legacy_reader; if (strategy_ != nullptr) { + // The strategy destructor should reset its own Stats object here dynamic_cast(strategy_.get())->stats()->reset(); strategy_ = nullptr; } @@ -1663,6 +1665,10 @@ stats::Stats* Query::stats() const { return stats_; } +void Query::set_stats(const stats::StatsData& data) { + stats_->populate_with_data(data); +} + shared_ptr Query::rest_scratch() const { return rest_scratch_; } diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index c5c8ecba2dba..e54c0342f907 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -655,6 +655,14 @@ class Query { /** Returns the internal stats object. */ stats::Stats* stats() const; + /** + * Populate the owned stats instance with data. + * To be removed when the class will get a C41 constructor. + * + * @param data Data to populate the stats with. + */ + void set_stats(const stats::StatsData& data); + /** Returns the scratch space used for REST requests. */ shared_ptr rest_scratch() const; diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 393a1bff0c67..09406c734de9 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -65,6 +65,10 @@ stats::Stats* StrategyBase::stats() const { return stats_; } +void StrategyBase::set_stats(const stats::StatsData& data) { + stats_->populate_with_data(data); +} + /* ****************************** */ /* PROTECTED METHODS */ /* ****************************** */ diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 91bc71946403..b5d1abb983a9 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -208,6 +208,14 @@ class StrategyBase { /** Returns `stats_`. */ stats::Stats* stats() const; + /** + * Populate the owned stats instance with data. + * To be removed when the class will get a C41 constructor. + * + * @param data Data to populate the stats with. + */ + void set_stats(const stats::StatsData& data); + /** Returns the configured offsets format mode. */ std::string offsets_mode() const; diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 75162c69347c..92414cd51d54 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -88,7 +88,7 @@ namespace serialization { shared_ptr dummy_logger = make_shared(HERE(), ""); -Status stats_to_capnp(Stats& stats, capnp::Stats::Builder* stats_builder) { +void stats_to_capnp(const Stats& stats, capnp::Stats::Builder* stats_builder) { // Build counters const auto counters = stats.counters(); if (counters != nullptr && !counters->empty()) { @@ -114,31 +114,29 @@ Status stats_to_capnp(Stats& stats, capnp::Stats::Builder* stats_builder) { ++index; } } - - return Status::Ok(); } -Status stats_from_capnp( - const capnp::Stats::Reader& stats_reader, Stats* stats) { +StatsData stats_from_capnp(const capnp::Stats::Reader& stats_reader) { + std::unordered_map counters; + std::unordered_map timers; + if (stats_reader.hasCounters()) { - auto counters = stats->counters(); auto counters_reader = stats_reader.getCounters(); for (const auto entry : counters_reader.getEntries()) { auto key = std::string_view{entry.getKey().cStr(), entry.getKey().size()}; - (*counters)[std::string{key}] = entry.getValue(); + counters[std::string(key)] = entry.getValue(); } } if (stats_reader.hasTimers()) { - auto timers = stats->timers(); auto timers_reader = stats_reader.getTimers(); for (const auto entry : timers_reader.getEntries()) { auto key = std::string_view{entry.getKey().cStr(), entry.getKey().size()}; - (*timers)[std::string{key}] = entry.getValue(); + timers[std::string(key)] = entry.getValue(); } } - return Status::Ok(); + return stats::StatsData(counters, timers); } void range_buffers_to_capnp( @@ -246,11 +244,9 @@ Status subarray_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = subarray->stats(); - if (stats != nullptr) { - auto stats_builder = builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = subarray->stats(); + auto stats_builder = builder->initStats(); + stats_to_capnp(stats, &stats_builder); if (subarray->relevant_fragments().relevant_fragments_size() > 0) { auto relevant_fragments_builder = builder->initRelevantFragments( @@ -328,11 +324,8 @@ Status subarray_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader.hasStats()) { - stats::Stats* stats = subarray->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader.getStats()); + subarray->set_stats(stats_data); } if (reader.hasRelevantFragments()) { @@ -428,11 +421,9 @@ Status subarray_partitioner_to_capnp( builder->setMemoryBudgetValidity(mem_budget_validity); // If stats object exists set its cap'n proto object - stats::Stats* stats = partitioner.stats(); - if (stats != nullptr) { - auto stats_builder = builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = partitioner.stats(); + auto stats_builder = builder->initStats(); + stats_to_capnp(stats, &stats_builder); return Status::Ok(); } @@ -566,11 +557,8 @@ Status subarray_partitioner_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader.hasStats()) { - auto stats = partitioner->stats(); - // We should always have stats - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader.getStats()); + partitioner->set_stats(stats_data); } return Status::Ok(); @@ -913,11 +901,9 @@ Status reader_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = reader.stats(); - if (stats != nullptr) { - auto stats_builder = reader_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *reader.stats(); + auto stats_builder = reader_builder->initStats(); + stats_to_capnp(stats, &stats_builder); return Status::Ok(); } @@ -947,11 +933,9 @@ Status index_reader_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = reader.stats(); - if (stats != nullptr) { - auto stats_builder = reader_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *reader.stats(); + auto stats_builder = reader_builder->initStats(); + stats_to_capnp(stats, &stats_builder); return Status::Ok(); } @@ -982,11 +966,9 @@ Status dense_reader_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = reader.stats(); - if (stats != nullptr) { - auto stats_builder = reader_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *reader.stats(); + auto stats_builder = reader_builder->initStats(); + stats_to_capnp(stats, &stats_builder); return Status::Ok(); } @@ -1146,11 +1128,8 @@ Status reader_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader_reader.hasStats()) { - stats::Stats* stats = reader->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(reader_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader_reader.getStats()); + reader->set_stats(stats_data); } return Status::Ok(); @@ -1187,11 +1166,8 @@ Status index_reader_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader_reader.hasStats()) { - stats::Stats* stats = reader->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(reader_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader_reader.getStats()); + reader->set_stats(stats_data); } return Status::Ok(); @@ -1229,11 +1205,8 @@ Status dense_reader_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader_reader.hasStats()) { - stats::Stats* stats = reader->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(reader_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader_reader.getStats()); + reader->set_stats(stats_data); } return Status::Ok(); @@ -1252,11 +1225,8 @@ Status delete_from_capnp( // If cap'n proto object has stats set it on c++ object if (delete_reader.hasStats()) { - stats::Stats* stats = delete_strategy->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(delete_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(delete_reader.getStats()); + delete_strategy->set_stats(stats_data); } return Status::Ok(); @@ -1273,11 +1243,9 @@ Status delete_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = delete_strategy.stats(); - if (stats != nullptr) { - auto stats_builder = delete_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *delete_strategy.stats(); + auto stats_builder = delete_builder->initStats(); + stats_to_capnp(stats, &stats_builder); return Status::Ok(); } @@ -1301,11 +1269,9 @@ Status writer_to_capnp( } // If stats object exists set its cap'n proto object - stats::Stats* stats = writer.stats(); - if (stats != nullptr) { - auto stats_builder = writer_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *writer.stats(); + auto stats_builder = writer_builder->initStats(); + stats_to_capnp(stats, &stats_builder); if (query.layout() == Layout::GLOBAL_ORDER) { auto& global_writer = dynamic_cast(writer); @@ -1341,11 +1307,8 @@ Status writer_from_capnp( // If cap'n proto object has stats set it on c++ object if (writer_reader.hasStats()) { - stats::Stats* stats = writer->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(writer_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(writer_reader.getStats()); + writer->set_stats(stats_data); } if (query.layout() == Layout::GLOBAL_ORDER && @@ -1557,10 +1520,10 @@ Status query_to_capnp( RETURN_NOT_OK(config_to_capnp(query.config(), &config_builder)); // If stats object exists set its cap'n proto object - stats::Stats* stats = query.stats(); - if (stats != nullptr) { + auto stats = query.stats(); + if (stats) { auto stats_builder = query_builder->initStats(); - RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); + stats_to_capnp(*stats, &stats_builder); } auto& written_fragment_info = query.get_written_fragment_info(); @@ -2270,11 +2233,8 @@ Status query_from_capnp( // If cap'n proto object has stats set it on c++ object if (query_reader.hasStats()) { - stats::Stats* stats = query->stats(); - // We should always have a stats here - if (stats != nullptr) { - RETURN_NOT_OK(stats_from_capnp(query_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(query_reader.getStats()); + query->set_stats(stats_data); } if (query_reader.hasWrittenFragmentInfo()) { @@ -3180,11 +3140,9 @@ void ordered_dim_label_reader_to_capnp( query.dimension_label_increasing_order()); // If stats object exists set its cap'n proto object - stats::Stats* stats = reader.stats(); - if (stats != nullptr) { - auto stats_builder = reader_builder->initStats(); - throw_if_not_ok(stats_to_capnp(*stats, &stats_builder)); - } + const auto& stats = *reader.stats(); + auto stats_builder = reader_builder->initStats(); + stats_to_capnp(stats, &stats_builder); } void ordered_dim_label_reader_from_capnp( @@ -3212,11 +3170,8 @@ void ordered_dim_label_reader_from_capnp( // If cap'n proto object has stats set it on c++ object if (reader_reader.hasStats()) { - stats::Stats* stats = reader->stats(); - // We should always have a stats here - if (stats != nullptr) { - throw_if_not_ok(stats_from_capnp(reader_reader.getStats(), stats)); - } + auto stats_data = stats_from_capnp(reader_reader.getStats()); + reader->set_stats(stats_data); } } diff --git a/tiledb/sm/stats/global_stats.h b/tiledb/sm/stats/global_stats.h index 7d5635ef4830..66d9e3dfab58 100644 --- a/tiledb/sm/stats/global_stats.h +++ b/tiledb/sm/stats/global_stats.h @@ -49,6 +49,71 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/sm/stats/stats.h" +/** + * Documenting the current stats behavior and architecture as close as + * possible to the code so it's helpful next time someone tries to refactor. + * + * Statistics collection is done at the top level via the GlobalStats class + * defined in this file. + * We maintain a global object called `all_stats` which is used to register + * Stats objects, enable/disable collection, reset or dumping the collected + * stats. + * + * The TileDB C API uses the `all_stats` object directly to execute the + * actions iterated above. + * + * The GlobalStats class owns a list called `registered_stats` that has one + * Stats object registered for each Context used. In consequence, + * ContextResources register a Stats object for each Context created, this + * object serves as the root for the tree of all children Stats used in a + * Context. + * + * As mentioned above, the Stats objects used under a Context form a tree. + * Each Stats object mentains a list of children Stats and a pointer to the + * parent Stats object. + * The Stats object created by ContextResources(named "Context.StorageManager") + * is the only Stats constructed in a standalone fashion using the Stats + * constructor, all the other objects under this root Stats are created via + * the Stats::create_child API. + * + * The (current, please update if not accurate anymore) exhaustive list of + * Stats we maintain under a Context is: + * --------------------------- + * ContextResources + * - Query + * - Reader + * - Writer + * - DenseTiler + * - Subarray + * - Deletes + * - Subarray + * - subSubarray + * - SubarrayPartitioner + * - VFS + * - S3 + * - ArrayDirectory + * - RestClient + * - Consolidator + * --------------------------- + * Please visualize this as a tree, it was much easier to write + * like this, the tree is too wide. + * + * + * Observed issues: + * - Stats objects currently are created via Stats::create_child API from a + * parent stats object. Child objects such as e.g. Subarray only hold a + * pointer to the Stats object, this means that the Stats objects outlive + * the objects they represent and are kept alive by the tree like structure + * defined by the Stats class. + * In theory, a Context running for a long time would OOM the machine with + * Stats objects. + * + * - Stats::populate_flattened_stats aggregate the collected statistic via + * summing. But we also collect ".min", ".max" statistics as well, + * sum-aggregating those is incorrect. Currently the dump function just + * doesn't print those statistics. + */ + namespace tiledb { namespace sm { namespace stats { @@ -158,8 +223,7 @@ class GlobalStats { /* ********************************* */ /** - * The singleton instance holding all global stats counters. The report will - * be automatically made when this object is destroyed (at program termination). + * The singleton instance holding all global stats counters and timers. */ extern GlobalStats all_stats; diff --git a/tiledb/sm/stats/stats.cc b/tiledb/sm/stats/stats.cc index 5773c312b583..82de3ed25494 100644 --- a/tiledb/sm/stats/stats.cc +++ b/tiledb/sm/stats/stats.cc @@ -48,9 +48,14 @@ namespace stats { /* ****************************** */ Stats::Stats(const std::string& prefix) + : Stats(prefix, StatsData{}) { +} + +Stats::Stats(const std::string& prefix, const StatsData& data) : enabled_(true) , prefix_(prefix + ".") , parent_(nullptr) { + this->populate_with_data(data); } /* ****************************** */ @@ -246,8 +251,12 @@ Stats* Stats::parent() { } Stats* Stats::create_child(const std::string& prefix) { + return create_child(prefix, StatsData{}); +} + +Stats* Stats::create_child(const std::string& prefix, const StatsData& data) { std::unique_lock lck(mtx_); - children_.emplace_back(prefix_ + prefix); + children_.emplace_back(prefix_ + prefix, data); Stats* const child = &children_.back(); child->parent_ = this; return child; @@ -272,15 +281,26 @@ void Stats::populate_flattened_stats( } } -std::unordered_map* Stats::timers() { +const std::unordered_map* Stats::timers() const { return &timers_; } /** Return pointer to conters map, used for serialization only. */ -std::unordered_map* Stats::counters() { +const std::unordered_map* Stats::counters() const { return &counters_; } +void Stats::populate_with_data(const StatsData& data) { + auto& timers = data.timers(); + for (const auto& timer : timers) { + timers_[timer.first] = timer.second; + } + auto& counters = data.counters(); + for (const auto& counter : counters) { + counters_[counter.first] = counter.second; + } +} + } // namespace stats } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/stats/stats.h b/tiledb/sm/stats/stats.h index 018ed721f637..47691c8d2943 100644 --- a/tiledb/sm/stats/stats.h +++ b/tiledb/sm/stats/stats.h @@ -51,6 +51,58 @@ namespace tiledb { namespace sm { namespace stats { +/** + * Class that holds measurement data that Stats objects can be + * initialized with. + */ +class StatsData { + public: + /* ****************************** */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ****************************** */ + + /* Default constructor */ + StatsData() = default; + + /** + * Value constructor. + * + * @param counters A map of counters + * @param timers A map of timers + */ + StatsData( + std::unordered_map& counters, + std::unordered_map& timers) + : counters_(counters) + , timers_(timers) { + } + + /* ****************************** */ + /* API */ + /* ****************************** */ + + /** Get a reference to internal counters */ + const std::unordered_map& counters() const { + return counters_; + } + + /** Get a reference to internal timers */ + const std::unordered_map& timers() const { + return timers_; + } + + private: + /* ****************************** */ + /* PRIVATE ATTRIBUTES */ + /* ****************************** */ + + /** Map of counters and values */ + std::unordered_map counters_; + + /** Map of timers and values */ + std::unordered_map timers_; +}; + /** * Class that defines stats counters and methods to manipulate them. */ @@ -72,6 +124,14 @@ class Stats { */ Stats(const std::string& prefix); + /** + * Value constructor. + * + * @param prefix The stat name prefix. + * @param data Initial data to populate the Stats object with. + */ + Stats(const std::string& prefix, const StatsData& data); + /** Destructor. */ ~Stats() = default; @@ -116,11 +176,29 @@ class Stats { /** Creates a child instance, managed by this instance. */ Stats* create_child(const std::string& prefix); + /** + * Creates a child instance, managed by this instance, the instance is + * constructed with initial data. + * + * @param prefix The stat name prefix. + * @param data Initial data to populate the Stats object with. + */ + Stats* create_child(const std::string& prefix, const StatsData& data); + /** Return pointer to timers map, used for serialization only. */ - std::unordered_map* timers(); + const std::unordered_map* timers() const; /** Return pointer to conters map, used for serialization only. */ - std::unordered_map* counters(); + const std::unordered_map* counters() const; + + /** + * Populate the counters and timers internal maps from a StatsData object + * Please be aware that the data is not being added up, it will override the + * existing data on the Stats object. + * + * @param data Data to populate the stats with. + */ + void populate_with_data(const StatsData& data); private: /* ****************************** */ diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 426beba8ea87..2cd6e3709f8f 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -3103,8 +3103,12 @@ RelevantFragments& Subarray::relevant_fragments() { return relevant_fragments_; } -stats::Stats* Subarray::stats() const { - return stats_; +const stats::Stats& Subarray::stats() const { + return *stats_; +} + +void Subarray::set_stats(const stats::StatsData& data) { + stats_->populate_with_data(data); } tuple> Subarray::non_overlapping_ranges_for_dim( diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 98fca20e9af3..04d0af954fed 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -1276,7 +1276,15 @@ class Subarray { std::vector* end_coords) const; /** Returns `stats_`. */ - stats::Stats* stats() const; + const stats::Stats& stats() const; + + /** + * Populate the owned stats instance with data. + * To be removed when the class will get a C41 constructor. + * + * @param data Data to populate the stats with. + */ + void set_stats(const stats::StatsData& data); /** Stores a vector of 1D ranges per dimension. */ std::vector> original_range_idx_; diff --git a/tiledb/sm/subarray/subarray_partitioner.cc b/tiledb/sm/subarray/subarray_partitioner.cc index db6dab81cf42..e22fbf28ac51 100644 --- a/tiledb/sm/subarray/subarray_partitioner.cc +++ b/tiledb/sm/subarray/subarray_partitioner.cc @@ -666,8 +666,12 @@ Subarray& SubarrayPartitioner::subarray() { return subarray_; } -stats::Stats* SubarrayPartitioner::stats() const { - return stats_; +const stats::Stats& SubarrayPartitioner::stats() const { + return *stats_; +} + +void SubarrayPartitioner::set_stats(const stats::StatsData& data) { + stats_->populate_with_data(data); } /* ****************************** */ diff --git a/tiledb/sm/subarray/subarray_partitioner.h b/tiledb/sm/subarray/subarray_partitioner.h index 919b9a1b6261..2915fd7aefe7 100644 --- a/tiledb/sm/subarray/subarray_partitioner.h +++ b/tiledb/sm/subarray/subarray_partitioner.h @@ -333,7 +333,14 @@ class SubarrayPartitioner { Subarray& subarray(); /** Returns `stats_`. */ - stats::Stats* stats() const; + const stats::Stats& stats() const; + + /** Populate the owned stats instance with data. + * To be removed when the class will get a C41 constructor. + * + * @param data Data to populate the stats with. + */ + void set_stats(const stats::StatsData& data); private: /* ********************************* */ From 695cc84b765f62a4be6169b980fc1484d8b6fcbe Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Mon, 12 Feb 2024 14:31:56 -0600 Subject: [PATCH 184/456] Implement std::pmr based memory tracking. (#4683) This implements memory allocation tracking by using the `std::pmr` interface introduced in C++17. --- TYPE: FEATURE DESC: Add std::pmr based memory allocation tracking. --- CMakeLists.txt | 3 + test/src/unit-ReadCellSlabIter.cc | 17 +- test/src/unit-Reader.cc | 7 +- test/src/unit-enumerations.cc | 11 +- test/src/unit-result-coords.cc | 11 +- test/src/unit-result-tile.cc | 15 +- .../unit-sparse-unordered-with-dups-reader.cc | 3 +- test/support/CMakeLists.txt | 3 + test/support/src/helpers.cc | 1 + test/support/src/helpers.h | 1 + test/support/src/mem_helpers.cc | 49 +++ test/support/src/mem_helpers.h | 48 +++ tiledb/common/CMakeLists.txt | 18 +- tiledb/common/memory_tracker.cc | 373 ++++++++++++++++++ tiledb/common/memory_tracker.h | 295 +++++++++++++- tiledb/common/pmr.cc | 41 ++ tiledb/common/pmr.h | 127 ++++++ tiledb/sm/array/array.cc | 21 +- tiledb/sm/array/array.h | 12 +- tiledb/sm/array/array_directory.cc | 12 +- tiledb/sm/array/array_directory.h | 6 +- tiledb/sm/fragment/fragment_info.cc | 7 +- tiledb/sm/fragment/fragment_info.h | 7 +- tiledb/sm/fragment/fragment_metadata.cc | 90 +++-- tiledb/sm/fragment/fragment_metadata.h | 93 +++-- tiledb/sm/query/query.cc | 15 +- tiledb/sm/query/query.h | 3 + tiledb/sm/query/readers/dense_reader.cc | 7 +- tiledb/sm/query/readers/dense_reader.h | 3 - .../readers/sparse_global_order_reader.cc | 6 +- .../query/readers/sparse_index_reader_base.cc | 1 - .../query/readers/sparse_index_reader_base.h | 3 - .../sparse_unordered_with_dups_reader.cc | 6 +- tiledb/sm/query/strategy_base.cc | 7 +- tiledb/sm/query/strategy_base.h | 32 +- tiledb/sm/query/test/CMakeLists.txt | 2 +- tiledb/sm/query/test/unit_query_condition.cc | 36 +- .../sm/query/writers/global_order_writer.cc | 2 +- tiledb/sm/query/writers/ordered_writer.cc | 2 +- tiledb/sm/query/writers/unordered_writer.cc | 4 +- tiledb/sm/query/writers/writer_base.cc | 7 +- tiledb/sm/query/writers/writer_base.h | 3 + tiledb/sm/serialization/array.cc | 9 +- tiledb/sm/serialization/fragment_info.cc | 14 +- tiledb/sm/serialization/fragment_metadata.cc | 12 +- tiledb/sm/serialization/fragment_metadata.h | 4 +- .../sm/storage_manager/context_resources.cc | 12 +- tiledb/sm/storage_manager/context_resources.h | 16 + tiledb/sm/storage_manager/storage_manager.cc | 3 +- vcpkg.json | 1 + 50 files changed, 1244 insertions(+), 237 deletions(-) create mode 100644 test/support/src/mem_helpers.cc create mode 100644 test/support/src/mem_helpers.h create mode 100644 tiledb/common/memory_tracker.cc create mode 100644 tiledb/common/pmr.cc create mode 100644 tiledb/common/pmr.h diff --git a/CMakeLists.txt b/CMakeLists.txt index c84b2d66fe08..9b10402b214f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,9 @@ endif() set(CMAKE_C_VISIBILITY_PRESET hidden) set(CMAKE_CXX_VISIBILITY_PRESET hidden) +# Disable warnings from Boost +set(Boost_NO_WARN_NEW_VERSIONS ON) + ############################################################ # Superbuild setup ############################################################ diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index ce047421e2d1..194f55cb9214 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -33,6 +33,7 @@ #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/tile_domain.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/query/legacy/read_cell_slab_iter.h" @@ -256,7 +257,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -330,7 +331,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -408,7 +409,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -491,7 +492,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -711,7 +712,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -897,7 +898,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -1096,7 +1097,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -1342,7 +1343,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index dbc2cd34287e..b28c0f6e024d 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -36,6 +36,7 @@ #include "tiledb/common/dynamic_memory/dynamic_memory.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/misc/types.h" @@ -164,6 +165,7 @@ TEST_CASE_METHOD( Subarray subarray(&array, &g_helper_stats, g_helper_logger()); DefaultChannelAggregates default_channel_aggregates; auto params = StrategyParams( + array.memory_tracker(), context.storage_manager(), array.opened_array(), config, @@ -173,8 +175,7 @@ TEST_CASE_METHOD( Layout::ROW_MAJOR, condition, default_channel_aggregates, - false, - array.memory_tracker()); + false); Reader reader(&g_helper_stats, g_helper_logger(), params); unsigned dim_num = 2; auto size = 2 * sizeof(int32_t); @@ -253,7 +254,7 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), schema, URI(), std::make_pair(0, 0), diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 6d9ae77052d3..107e80053615 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -33,6 +33,7 @@ #include #include "test/support/tdb_catch.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" @@ -1114,7 +1115,7 @@ TEST_CASE_METHOD( auto enmr_path = schema->get_enumeration_path_name(enmr_name.value()); - MemoryTracker tracker; + auto tracker = ctx_.resources().create_memory_tracker(); auto loaded = ad->load_enumerations_from_paths({enmr_path}, enc_key_, tracker); REQUIRE(loaded.size() == 1); @@ -1138,7 +1139,7 @@ TEST_CASE_METHOD( auto schema = get_array_schema_latest(); auto ad = get_array_directory(); - MemoryTracker tracker; + auto tracker = ctx_.resources().create_memory_tracker(); // Check that this function throws an exception when attempting to load // an unknown enumeration @@ -1163,8 +1164,8 @@ TEST_CASE_METHOD( auto enmr_name = schema->attribute("attr1")->get_enumeration_name(); auto enmr_path = schema->get_enumeration_path_name(enmr_name.value()); - MemoryTracker tracker; - tracker.set_budget(1); + auto tracker = ctx_.resources().create_memory_tracker(); + tracker->set_budget(1); // Check that this function throws an exception when attempting to load // an enumeration that exceeds the memory budget. @@ -1175,7 +1176,7 @@ TEST_CASE_METHOD( matcher); // Check that the fix is to increase the memory budget. - tracker.set_budget(std::numeric_limits::max()); + tracker->set_budget(std::numeric_limits::max()); REQUIRE_NOTHROW( ad->load_enumerations_from_paths({enmr_path}, enc_key_, tracker)); } diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index 8e0044401b74..f86037a29df8 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -29,6 +29,8 @@ * * Tests for the ResultCoords classes. */ + +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" @@ -54,7 +56,7 @@ struct CResultCoordsFx { std::string array_name_; const char* ARRAY_NAME = "test_result_coords"; tiledb_array_t* array_; - std::unique_ptr frag_md_; + std::shared_ptr frag_md_; CResultCoordsFx(uint64_t num_cells); ~CResultCoordsFx(); @@ -104,13 +106,14 @@ CResultCoordsFx::CResultCoordsFx(uint64_t num_cells) { rc = tiledb_array_open(ctx_, array_, TILEDB_READ); REQUIRE(rc == TILEDB_OK); - frag_md_.reset(new FragmentMetadata( - nullptr, + frag_md_ = make_shared( + HERE(), nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), - true)); + true); } CResultCoordsFx::~CResultCoordsFx() { diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index 77677e6a1d3c..62910cbb8138 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -29,6 +29,8 @@ * * Tests for the ResultTile classes. */ + +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/misc/types.h" @@ -59,7 +61,7 @@ struct CResultTileFx { std::string array_name_; const char* ARRAY_NAME = "test_result_coords"; tiledb_array_t* array_; - std::unique_ptr frag_md_; + std::shared_ptr frag_md_; CResultTileFx(); ~CResultTileFx(); @@ -107,13 +109,14 @@ CResultTileFx::CResultTileFx() { rc = tiledb_array_open(ctx_, array_, TILEDB_READ); REQUIRE(rc == TILEDB_OK); - frag_md_.reset(new FragmentMetadata( - nullptr, + frag_md_ = make_shared( + HERE(), nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), - false)); + false); } CResultTileFx::~CResultTileFx() { @@ -188,7 +191,7 @@ TEST_CASE_METHOD( auto& array_schema = array_->array_->array_schema_latest(); FragmentMetadata frag_md( nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), @@ -298,7 +301,7 @@ TEST_CASE_METHOD( auto& array_schema = array_->array_->array_schema_latest(); FragmentMetadata frag_md( nullptr, - nullptr, + create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 74daab6ff01e..2ea31339a5e8 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -32,6 +32,7 @@ #include "test/support/src/helpers.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/query/query_buffer.h" @@ -767,7 +768,7 @@ CSparseUnorderedWithDupsVarDataFx::open_default_array_1d_with_fragments( shared_ptr fragment = make_shared( HERE(), nullptr, - nullptr, + create_test_memory_tracker(), array->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index 977d65a60480..935578a63124 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -41,6 +41,8 @@ set(TILEDB_TEST_SUPPORT_SOURCES src/helpers.h src/helpers.cc src/helpers-dimension.h + src/mem_helpers.h + src/mem_helpers.cc src/serialization_wrappers.cc src/temporary_local_directory.cc src/vfs_helpers.cc @@ -65,6 +67,7 @@ if (NOT MSVC) endif() target_link_libraries(tiledb_test_support_lib PRIVATE $) +target_link_libraries(tiledb_test_support_lib PRIVATE baseline) target_include_directories( tiledb_test_support_lib BEFORE PRIVATE diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 72c60f954e6b..251e6cfbd65b 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -45,6 +45,7 @@ #include "tiledb/api/c_api/context/context_api_external.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index f45054d4e429..283f536ea446 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -35,6 +35,7 @@ #include #include "test/support/src/coords_workaround.h" +#include "test/support/src/mem_helpers.h" #include "tiledb.h" #include "tiledb/common/common.h" #include "tiledb/common/random/random_label.h" diff --git a/test/support/src/mem_helpers.cc b/test/support/src/mem_helpers.cc new file mode 100644 index 000000000000..c92f0cab85f6 --- /dev/null +++ b/test/support/src/mem_helpers.cc @@ -0,0 +1,49 @@ +/** + * @file mem_helpers.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines some test suite helper functions, specific to memory + * tracking. + */ + +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::test { + +shared_ptr create_test_memory_tracker() { + class MemoryTrackerCreator : public sm::MemoryTracker { + public: + MemoryTrackerCreator() + : sm::MemoryTracker() { + } + }; + + return make_shared(HERE()); +} + +} // namespace tiledb::test diff --git a/test/support/src/mem_helpers.h b/test/support/src/mem_helpers.h new file mode 100644 index 000000000000..96a359a1975b --- /dev/null +++ b/test/support/src/mem_helpers.h @@ -0,0 +1,48 @@ +/** + * @file mem_helpers.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares some test suite helper functions specific to memory + * tracking. + */ + +#ifndef TILEDB_MEM_HELPERS_H +#define TILEDB_MEM_HELPERS_H + +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::test { + +/** + * Helper function to create test instances of shared_ptr + */ +shared_ptr create_test_memory_tracker(); + +} // namespace tiledb::test + +#endif // TILEDB_MEM_HELPERS_H diff --git a/tiledb/common/CMakeLists.txt b/tiledb/common/CMakeLists.txt index a7736fd55ec0..92ba0d1d2a03 100644 --- a/tiledb/common/CMakeLists.txt +++ b/tiledb/common/CMakeLists.txt @@ -54,11 +54,21 @@ set(TILEDB_COMMON_SOURCES ${COMMON_SOURCES} PARENT_SCOPE) # `baseline` object library # commence(object_library baseline) - this_target_sources(logger.cc governor/governor.cc - dynamic_memory/dynamic_memory.cc heap_profiler.cc heap_memory.cc - exception/exception.cc exception/status.cc) + this_target_sources( + logger.cc + dynamic_memory/dynamic_memory.cc + exception/exception.cc + exception/status.cc + governor/governor.cc + heap_profiler.cc + heap_memory.cc + memory_tracker.cc + pmr.cc + ) find_package(Spdlog_EP REQUIRED) + find_package(Boost REQUIRED COMPONENTS container) target_link_libraries(baseline PUBLIC spdlog::spdlog) + target_link_libraries(baseline PUBLIC Boost::container) target_link_libraries(baseline PUBLIC common) conclude(object_library) @@ -69,4 +79,4 @@ commence(object_library stringx) this_target_sources(stdx_string.cc) conclude(object_library) -add_test_subdirectory() \ No newline at end of file +add_test_subdirectory() diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc new file mode 100644 index 000000000000..5157efacf9dd --- /dev/null +++ b/tiledb/common/memory_tracker.cc @@ -0,0 +1,373 @@ +/** + * @file memory_tracker.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains implementations for the PMR memory tracking classes. See + * the top level description in memory_tracker.h. + */ + +#include + +#include "external/include/nlohmann/json.hpp" + +#include "tiledb/common/exception/exception.h" +#include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::sm { + +class MemoryTrackerException : public common::StatusException { + public: + explicit MemoryTrackerException(const std::string& message) + : StatusException("MemoryTracker", message) { + } +}; + +std::string memory_type_to_str(MemoryType type) { + switch (type) { + case MemoryType::RTREE: + return "RTree"; + case MemoryType::FOOTER: + return "Footer"; + case MemoryType::TILE_OFFSETS: + return "TileOffsets"; + case MemoryType::TILE_MIN_VALS: + return "TileMinVals"; + case MemoryType::TILE_MAX_VALS: + return "TileMaxVals"; + case MemoryType::TILE_SUMS: + return "TileSums"; + case MemoryType::TILE_NULL_COUNTS: + return "TileNullCounts"; + case MemoryType::ENUMERATION: + return "Enumeration"; + default: + auto val = std::to_string(static_cast(type)); + throw std::logic_error("Invalid memory type: " + val); + } +} + +std::string memory_tracker_type_to_str(MemoryTrackerType type) { + switch (type) { + case MemoryTrackerType::ANONYMOUS: + return "Anonymous"; + case MemoryTrackerType::ARRAY_READ: + return "ArrayRead"; + case MemoryTrackerType::ARRAY_WRITE: + return "ArrayWrite"; + case MemoryTrackerType::QUERY_READ: + return "QueryRead"; + case MemoryTrackerType::QUERY_WRITE: + return "QueryWrite"; + case MemoryTrackerType::CONSOLIDATOR: + return "Consolidator"; + default: + auto val = std::to_string(static_cast(type)); + throw std::logic_error("Invalid memory tracker type: " + val); + } +} + +uint64_t MemoryTrackerResource::get_count() { + return type_counter_.fetch_add(0, std::memory_order_relaxed); +} + +void* MemoryTrackerResource::do_allocate(size_t bytes, size_t alignment) { + total_counter_.fetch_add(bytes, std::memory_order_relaxed); + type_counter_.fetch_add(bytes, std::memory_order_relaxed); + return upstream_->allocate(bytes, alignment); +} + +void MemoryTrackerResource::do_deallocate( + void* ptr, size_t bytes, size_t alignment) { + upstream_->deallocate(ptr, bytes, alignment); + type_counter_.fetch_sub(bytes, std::memory_order_relaxed); + total_counter_.fetch_sub(bytes, std::memory_order_relaxed); +} + +bool MemoryTrackerResource::do_is_equal( + const tdb::pmr::memory_resource& other) const noexcept { + return this == &other; +} + +MemoryTracker::~MemoryTracker() { + assert( + total_counter_.fetch_add(0) == 0 && + "MemoryTracker destructed with outstanding allocations."); +} + +tdb::pmr::memory_resource* MemoryTracker::get_resource(MemoryType type) { + std::lock_guard lg(mutex_); + + // If we've already created an instance for this type, return it. + auto iter = resources_.find(type); + if (iter != resources_.end()) { + return iter->second.get(); + } + + // Add a new counter if it doesn't exist. + if (counters_.find(type) == counters_.end()) { + counters_.emplace(type, 0); + } else { + // There's no outstanding memory resource for this type, so it must be zero. + assert(counters_[type] == 0 && "Invalid memory tracking state."); + } + + // Create and track a shared_ptr to the new memory resource. + auto ret = make_shared( + HERE(), upstream_, total_counter_, counters_[type]); + resources_.emplace(type, ret); + + // Return the raw memory resource pointer for use by pmr containers. + return ret.get(); +} + +std::tuple> +MemoryTracker::get_counts() { + std::lock_guard lg(mutex_); + + auto total = total_counter_.fetch_add(0, std::memory_order_relaxed); + std::unordered_map by_type; + std::vector to_del; + for (auto& [mem_type, resource] : resources_) { + by_type[mem_type] = resource->get_count(); + } + + return {total, by_type}; +} + +uint64_t MemoryTracker::generate_id() { + static std::atomic curr_id{0}; + return curr_id.fetch_add(1); +} + +shared_ptr MemoryTrackerManager::create_tracker() { + /* + * The MemoryTracker class has a protected constructor to hopefully help + * self-document that instances should almost never be created directly + * except in test code. There exists a + * `tiledb::test::create_test_memory_tracker()` API that can be used in + * tests to create untracked instances of MemoryTracker. + * + * This just uses the standard private derived class to enable the use of + * `make_shared` to create instances in specific bits of code. + */ + class MemoryTrackerCreator : public MemoryTracker { + public: + /** + * Pass through to the protected MemoryTracker constructor for + * make_shared. + */ + MemoryTrackerCreator() + : MemoryTracker() { + } + }; + + std::lock_guard lg(mutex_); + + // Delete any expired weak_ptr instances + size_t idx = 0; + while (idx < trackers_.size()) { + if (trackers_[idx].expired()) { + trackers_.erase(trackers_.begin() + idx); + } else { + idx++; + } + } + + // Create a new tracker + auto ret = make_shared(HERE()); + trackers_.emplace(trackers_.begin(), ret); + + return ret; +} + +std::string MemoryTrackerManager::to_json() { + std::lock_guard lg(mutex_); + nlohmann::json rv; + + /* + * The reason for this being a while-loop instead of a for-loop is that we're + * modifying the trackers_ vector while iterating over it. The reference docs + * for std::vector::erase make it sound like a standard for-loop with + * iterators would work here, but the subtle key point in the docs is that + * the end() iterator used during iteration is invalidated after a call to + * erase. The end result of which is that it will lead to a subtle bug when + * every weak_ptr in the vector is expired (such as at shutdown) which leads + * to deleting random bits of memory. Thankfully The address sanitizer + * managed to point out the issue rather quickly. + */ + size_t idx = 0; + while (idx < trackers_.size()) { + auto ptr = trackers_[idx].lock(); + // If the weak_ptr is expired, we just remove it from trackers_ and + // carry on. + if (!ptr) { + trackers_.erase(trackers_.begin() + idx); + continue; + } + + nlohmann::json val; + + // Set an distinguishing id + val["tracker_id"] = std::to_string(ptr->get_id()); + + // Mark the stats with the tracker type. + val["tracker_type"] = memory_tracker_type_to_str(ptr->get_type()); + + // Add memory stats + auto [total, by_type] = ptr->get_counts(); + val["total_memory"] = total; + val["by_type"] = nlohmann::json::object(); + for (auto& [type, count] : by_type) { + val["by_type"][memory_type_to_str(type)] = count; + } + rv.push_back(val); + + idx++; + } + + return rv.dump(); +} + +MemoryTrackerReporter::~MemoryTrackerReporter() { + if (!filename_.has_value()) { + return; + } + + // Scoped lock_guard so we don't hold the lock while waiting for threads + // to join. + { + std::lock_guard lg(mutex_); + stop_ = true; + cv_.notify_all(); + } + + // Wait for the background thread to quit so that we don't cause a segfault + // when we destruct our synchronization primitives. + try { + thread_.join(); + } catch (std::exception& exc) { + LOG_ERROR( + "Error stopping MemoryTrackerReporter thread: " + + std::string(exc.what())); + } +} + +void MemoryTrackerReporter::start() { + if (!filename_.has_value()) { + LOG_INFO("No filename set, not starting the MemoryTrackerReporter."); + return; + } + + { + // Scoped so we release this before the thread starts. Probably unnecessary + // but better safe than sorry. + std::lock_guard lg(mutex_); + if (stop_) { + throw std::runtime_error("MemoryTrackerReporters cannot be restarted."); + } + } + + // Thread start logic mirrored from the ThreadPool. + for (size_t i = 0; i < 3; i++) { + try { + thread_ = std::thread(&MemoryTrackerReporter::run, this); + return; + } catch (const std::system_error& e) { + if (e.code() == std::errc::resource_unavailable_try_again) { + continue; + } + + throw MemoryTrackerException( + "Error starting the MemoryTrackerReporter: " + std::string(e.what())); + } + } + + throw MemoryTrackerException( + "No threads avaiable to start the MemoryTrackerReporter."); +} + +void MemoryTrackerReporter::run() { + std::stringstream ss; + std::ofstream out; + + while (true) { + std::unique_lock lk(mutex_); + cv_.wait_for(lk, std::chrono::milliseconds(1000), [&] { return stop_; }); + + if (stop_) { + return; + } + + // Open the log file, possibly re-opening after encountering an error. Log + // any errors and continue trying in case whatever issue resolves itself. + if (!out.is_open()) { + // Clear any error state. + out.clear(); + out.open(filename_.value(), std::ios::app); + } + + // If we failed to open the file, log a message and try again on the next + // iteration of this loop. This logic is in a background thread so the + // only real other options would be to crash the entire program or exit + // the thread. Retrying to see if its an ephemeral error seems better and + // also informs users that something is wrong with their config while not + // causing excessive chaos. + if (!out) { + LOG_ERROR( + "Error opening MemoryTrackerReporter file: " + filename_.value()); + continue; + } + + // Generate a JSON report from our MemoryTrackerManager. + auto json = manager_->to_json(); + if (json == "null") { + // This happens if the manager doesn't have any trackers registered. + // Rather than log noise we just ignore it. + continue; + } + + // Append our report to the log. + ss.str(""); + ss.clear(); + ss << json << std::endl; + out << ss.str(); + + // If writing to the file fails, we make a note, close it and then attempt + // to re-open it on the next iteration. See the note above on open errors + // for more context. + if (!out) { + LOG_ERROR( + "Error writing to MemoryTrackeReporter file: " + filename_.value()); + out.close(); + continue; + } + } +} + +} // namespace tiledb::sm diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 02d374474b9d..81c5a74cc035 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -28,39 +28,179 @@ * * @section DESCRIPTION * - * This file defines class MemoryTracker. + * This file contains the definitions for classes related to tracking memory + * using the polymorphic memory resources feature introduced in C++17. + * + * There are four main classes to be aware of: + * + * - MemoryTrackerResource + * - MemoryTracker + * - MemoryTrackerManager + * - MemoryTrackerReporter + * + * MemoryTrackerResource + * ===================== + * + * The MemoryTrackerResource class is responsible for actually tracking + * individual allocations. Each MemoryTrackerResource represents a single type + * of memory as enumerated in the MemoryType enum. To create instances of this + * class, users should use the MemoryTrackerManager::get_resource API. + * + * MemoryTracker + * ============= + * + * The MemoryTracker class is responsible for managing instances of + * MemoryTrackerResource. A MemoryTracker represents some section or behavior + * inside the TileDB library as enumerated in the MemoryTrackerType enum. + * Instances of MemoryTracker should be created using the + * MemoryTrackerManager::create_tracker() API or via the helper method + * ContextResources::create_memory_tracker(). Generally speaking, there should + * be very few of these instances outside of test code and instead existing + * instances should be referenced. + * + * For instance, there is currently an existing MemoryTracker member variable + * on both Array and Query. Most code in the library should be using one of + * these two trackers. There are a few specialized instances like in the + * Consolidator or for things like deserializing GenericTileIO tiles. + * + * MemoryTrackerManager + * ==================== + * + * The MemoryTrackerManager is a member variable on the ContextResources + * class. Users should not need to interact with this class directly as its + * just a container that holds references to all the MemoryTracker instances + * for a given context. Its used by the MemoryTrackerReport when logging + * memory usage. + * + * MemoryTrackerReporter + * ===================== + * + * The MemoryTrackerReporter class is a member variable on the ContextResources + * class. Users should not need to interact with this class directly as its + * just used to log memory statistics to a special log file when configured. + * + * Users wishing to run memory usage experiments should use the + * 'sm.memory.tracker.reporter.filename' configuration key to set a filename + * that will contain the logged memory statistics in JSONL format (i.e., JSON + * objects and arrays encoded one per line). At runtime the reporter appends + * a JSON blob once a second to this logfile that can then be analyzed using + * whatever scripts or software as appropriate. */ #ifndef TILEDB_MEMORY_TRACKER_H #define TILEDB_MEMORY_TRACKER_H +#include +#include +#include + +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" +#include "tiledb/sm/config/config.h" namespace tiledb { namespace sm { -class MemoryTracker { +//** The type of memory to track. */ +enum class MemoryType { + RTREE, + FOOTER, + TILE_OFFSETS, + TILE_MIN_VALS, + TILE_MAX_VALS, + TILE_SUMS, + TILE_NULL_COUNTS, + ENUMERATION +}; + +/** The type of MemoryTracker. */ +enum class MemoryTrackerType { + ANONYMOUS, + ARRAY_READ, + ARRAY_WRITE, + QUERY_READ, + QUERY_WRITE, + CONSOLIDATOR +}; + +class MemoryTrackerResource : public tdb::pmr::memory_resource { public: - enum class MemoryType { - RTREE, - FOOTER, - TILE_OFFSETS, - MIN_MAX_SUM_NULL_COUNT, - ENUMERATION - }; + // Disable all default generated constructors. + MemoryTrackerResource() = delete; + DISABLE_COPY_AND_COPY_ASSIGN(MemoryTrackerResource); + DISABLE_MOVE_AND_MOVE_ASSIGN(MemoryTrackerResource); /** Constructor. */ - MemoryTracker() { - memory_usage_ = 0; - memory_budget_ = std::numeric_limits::max(); - }; + explicit MemoryTrackerResource( + tdb::pmr::memory_resource* upstream, + std::atomic& total_counter, + std::atomic& type_counter) + : upstream_(upstream) + , total_counter_(total_counter) + , type_counter_(type_counter) { + } + + /** The number of bytes tracked by this resource. */ + uint64_t get_count(); + + protected: + /** Perform an allocation, returning a pointer to the allocated memory. */ + void* do_allocate(size_t bytes, size_t alignment) override; + /** Deallocate a previously allocated chunk of memory. */ + void do_deallocate(void* p, size_t bytes, size_t alignment) override; + + /** Check if two memory trackers are equal. */ + bool do_is_equal( + const tdb::pmr::memory_resource& other) const noexcept override; + + private: + /** The upstream memory resource to use for the actual allocation. */ + tdb::pmr::memory_resource* upstream_; + + /** A reference to a total counter for the MemoryTracker. */ + std::atomic& total_counter_; + + /** A reference to the memory type counter this resource is tracking. */ + std::atomic& type_counter_; +}; + +class MemoryTracker { + public: /** Destructor. */ - ~MemoryTracker() = default; + ~MemoryTracker(); DISABLE_COPY_AND_COPY_ASSIGN(MemoryTracker); DISABLE_MOVE_AND_MOVE_ASSIGN(MemoryTracker); + /** Get the id of this MemoryTracker instance. */ + inline uint64_t get_id() { + return id_; + } + + /** Get the type of this memory tracker. */ + inline MemoryTrackerType get_type() { + std::lock_guard lg(mutex_); + return type_; + } + + /** Set the type of this memory tracker. */ + void set_type(MemoryTrackerType type) { + std::lock_guard lg(mutex_); + type_ = type; + } + + /** + * Create a memory resource instance. + * + * @param type The type of memory that is being tracked. + * @return A memory resource derived from std::pmr::memory_resource. + */ + tdb::pmr::memory_resource* get_resource(MemoryType); + + /** Return the total and counts of this tracker. */ + std::tuple> get_counts(); + /** * Take memory from the budget. * @@ -139,8 +279,31 @@ class MemoryTracker { return memory_budget_; } + protected: + /** + * Constructor. + * + * This constructor is protected on purpose to discourage creating instances + * of this class that aren't connected to a ContextResources. When writing + * library code, you should almost always be using an existing instance of + * a MemoryTracker from the places those exist, i.e., on an Array, Query, + * or in the Consolidator. Occasionally, we'll need to create new instances + * for specific reasons. In those cases you need to have a reference to the + * ContextResources to call ContextResource::create_memory_tracker(). + * + * For tests that need to have a temporary MemoryTracker instance, there is + * a `create_test_memory_tracker()` API available in the test support library. + */ + MemoryTracker() + : memory_usage_(0) + , memory_budget_(std::numeric_limits::max()) + , id_(generate_id()) + , type_(MemoryTrackerType::ANONYMOUS) + , upstream_(tdb::pmr::get_default_resource()) + , total_counter_(0){}; + private: - /** Protects all member variables. */ + /** Protects all non-atomic member variables. */ std::mutex mutex_; /** Memory usage for tracked structures. */ @@ -151,6 +314,108 @@ class MemoryTracker { /** Memory usage by type. */ std::unordered_map memory_usage_by_type_; + + /** The id of this MemoryTracker. */ + uint64_t id_; + + /** The type of this MemoryTracker. */ + MemoryTrackerType type_; + + /** The upstream memory resource. */ + tdb::pmr::memory_resource* upstream_; + + /** MemoryTrackerResource by MemoryType. */ + std::unordered_map> + resources_; + + /** Memory counters by MemoryType. */ + std::unordered_map> counters_; + + /** The total memory usage of this MemoryTracker. */ + std::atomic total_counter_; + + /** Generate a unique id for this MemoryTracker. */ + static uint64_t generate_id(); +}; + +class MemoryTrackerManager { + public: + /** Constructor. */ + MemoryTrackerManager() = default; + + DISABLE_COPY_AND_COPY_ASSIGN(MemoryTrackerManager); + DISABLE_MOVE_AND_MOVE_ASSIGN(MemoryTrackerManager); + + /** + * Create a new memory tracker. + * + * @return The created MemoryTracker. + */ + shared_ptr create_tracker(); + + /** + * Generate a JSON string representing the current state of tracked memory. + * + * @return A string containing the JSON representation of tracked memory. + */ + std::string to_json(); + + private: + /** A mutext to protect our list of trackers. */ + std::mutex mutex_; + + /** A weak_ptr to the instances of MemoryTracker we create. */ + std::vector> trackers_; +}; + +class MemoryTrackerReporter { + public: + /** + * Constructor. + * + * @param cfg The Config instance for the parent context. + * @param manager The MemoryTrackerManager instance on the context resources. + */ + MemoryTrackerReporter( + const Config& cfg, shared_ptr manager) + : manager_(manager) + , filename_(cfg.get("sm.memory.tracker.reporter.filename")) + , stop_(false) { + } + + /** Destructor. */ + ~MemoryTrackerReporter(); + + DISABLE_COPY_AND_COPY_ASSIGN(MemoryTrackerReporter); + DISABLE_MOVE_AND_MOVE_ASSIGN(MemoryTrackerReporter); + + /** Start the background reporter thread if configured. */ + void start(); + + /** Stop the background reporter thread if started. */ + void stop(); + + /** The background reporter thread's main loop. */ + void run(); + + private: + /** The MemoryTrackerManager instance on the parent ContextResources. */ + shared_ptr manager_; + + /** An filename set in the config. */ + std::optional filename_; + + /** The background reporter thread. */ + std::thread thread_; + + /** A mutex for communication with the background thread. */ + std::mutex mutex_; + + /** A condition variable for signaling the background thread. */ + std::condition_variable cv_; + + /** A stop flag to signal shutdown to the background thread. */ + bool stop_; }; } // namespace sm diff --git a/tiledb/common/pmr.cc b/tiledb/common/pmr.cc new file mode 100644 index 000000000000..92da739a6066 --- /dev/null +++ b/tiledb/common/pmr.cc @@ -0,0 +1,41 @@ +/** + * @file pmr.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains implementation of pmr functions + */ + +#include "pmr.h" + +namespace tiledb::common::pmr { + +memory_resource* get_default_resource() { + return boost::container::pmr::get_default_resource(); +} + +} // namespace tiledb::common::pmr diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h new file mode 100644 index 000000000000..e66179e5fd86 --- /dev/null +++ b/tiledb/common/pmr.h @@ -0,0 +1,127 @@ +/** + * @file tiledb/common/pmr.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * A centralized definition of the polymorphic resource types used by + * TileDB. + */ + +#ifndef TILEDB_COMMON_PMR_H +#define TILEDB_COMMON_PMR_H + +#include + +#include +#include +#include + +#include "common.h" + +namespace tiledb::common::pmr { + +using memory_resource = boost::container::pmr::memory_resource; + +template +using pmr_vector = + std::vector>; + +memory_resource* get_default_resource(); + +template +class vector : public pmr_vector { + public: + // This class exists to ensure that all uses of it are provided with a + // valid std::pmr based allocator. This is so that as we switch from + // std::vector to using this class we don't forget to provide the allocator + // which is quite easy to do. + // + // If these constructors look confusing, just know that all we're doing is + // copying the current definitions from cppreference and then adjusting types + // to require the PMR based allocator. + + // I have absolutely no idea if all of these aliases are required. The + // allocator_type is the important one. I've copied the others just in + // case since I do know that PMR aware containers at least require + // allocator_type. + using value_type = typename pmr_vector::value_type; + using allocator_type = typename pmr_vector::allocator_type; + using size_type = typename pmr_vector::size_type; + using difference_type = typename pmr_vector::difference_type; + using reference = typename pmr_vector::reference; + using const_reference = typename pmr_vector::const_reference; + using pointer = typename pmr_vector::pointer; + using const_pointer = typename pmr_vector::const_pointer; + using iterator = typename pmr_vector::iterator; + using const_iterator = typename pmr_vector::const_iterator; + using reverse_iterator = typename pmr_vector::reverse_iterator; + using const_reverse_iterator = + typename pmr_vector::const_reverse_iterator; + + // Delete all default constructors because they don't require an allocator + constexpr vector() noexcept(noexcept(allocator_type())) = delete; + constexpr vector(const vector& other) = delete; + constexpr vector(vector&& other) noexcept = delete; + + // Delete non-allocator aware copy and move assign. + constexpr vector& operator=(const vector& other) = delete; + constexpr vector& operator=(vector&& other) noexcept = delete; + + constexpr explicit vector(const allocator_type& alloc) noexcept + : pmr_vector(alloc) { + } + + constexpr vector( + size_type count, const Tp& value, const allocator_type& alloc) + : pmr_vector(count, value, alloc) { + } + + constexpr explicit vector(size_type count, const allocator_type& alloc) + : pmr_vector(count, alloc) { + } + + template + constexpr vector(InputIt first, InputIt last, const allocator_type& alloc) + : pmr_vector(first, last, alloc) { + } + + constexpr vector(const vector& other, const allocator_type& alloc) + : pmr_vector(other, alloc) { + } + + constexpr vector(vector&& other, const allocator_type& alloc) + : pmr_vector(other, alloc) { + } + + constexpr vector(std::initializer_list init, const allocator_type& alloc) + : pmr_vector(init, alloc) { + } +}; + +} // namespace tiledb::common::pmr + +#endif // TILEDB_COMMON_PMR_H diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 58e99219032d..7094cd3e6f57 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -33,6 +33,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" @@ -96,6 +97,7 @@ Array::Array( , resources_(storage_manager_->resources()) , config_(resources_.config()) , remote_(array_uri.is_tiledb()) + , memory_tracker_(storage_manager->resources().create_memory_tracker()) , consistency_controller_(cc) , consistency_sentry_(nullopt) { } @@ -148,6 +150,7 @@ Status Array::open_without_fragments( /* Note: query_type_ MUST be set before calling set_array_open() because it will be examined by the ConsistencyController. */ query_type_ = QueryType::READ; + memory_tracker_->set_type(MemoryTrackerType::ARRAY_READ); /* Note: the open status MUST be exception safe. If anything interrupts the * opening process, it will throw and the array will be set as closed. */ @@ -245,6 +248,11 @@ Status Array::open( } query_type_ = query_type; + if (query_type_ == QueryType::READ) { + memory_tracker_->set_type(MemoryTrackerType::ARRAY_READ); + } else { + memory_tracker_->set_type(MemoryTrackerType::ARRAY_WRITE); + } set_timestamps( timestamp_start, timestamp_end, query_type_ == QueryType::READ); @@ -1059,10 +1067,6 @@ const NDRange Array::non_empty_domain() { return loaded_non_empty_domain(); } -MemoryTracker* Array::memory_tracker() { - return &memory_tracker_; -} - bool Array::serialize_non_empty_domain() const { auto found = false; auto serialize_ned_array_open = false; @@ -1624,6 +1628,15 @@ void Array::set_serialized_array_open() { array_uri_.is_tiledb()); } +void Array::set_query_type(QueryType query_type) { + query_type_ = query_type; + if (query_type_ == QueryType::READ) { + memory_tracker_->set_type(MemoryTrackerType::ARRAY_READ); + } else { + memory_tracker_->set_type(MemoryTrackerType::ARRAY_WRITE); + } +} + void Array::set_array_closed() { std::lock_guard lock(mtx_); diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index fffcee67387b..57bf6ef09156 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -38,7 +38,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array/consistency.h" @@ -56,6 +55,7 @@ namespace sm { class ArraySchema; class SchemaEvolution; class FragmentMetadata; +class MemoryTracker; enum class QueryType : uint8_t; /** @@ -784,7 +784,9 @@ class Array { } /** Returns the memory tracker. */ - MemoryTracker* memory_tracker(); + inline shared_ptr memory_tracker() { + return memory_tracker_; + } /** * Checks the config to see if non empty domain should be serialized on array @@ -821,9 +823,7 @@ class Array { void set_serialized_array_open(); /** Set the query type to open the array for. */ - inline void set_query_type(QueryType query_type) { - query_type_ = query_type; - } + void set_query_type(QueryType query_type); /** * Checks the array is open, in MODIFY_EXCLUSIVE mode, before deleting data. @@ -926,7 +926,7 @@ class Array { bool remote_; /** Memory tracker for the array. */ - MemoryTracker memory_tracker_; + shared_ptr memory_tracker_; /** A reference to the object which controls the present Array instance. */ ConsistencyController& consistency_controller_; diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 9710c5607f29..72615f674687 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -32,6 +32,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/filesystem/vfs.h" @@ -190,7 +191,7 @@ std::vector> ArrayDirectory::load_enumerations_from_paths( const std::vector& enumeration_paths, const EncryptionKey& encryption_key, - MemoryTracker& memory_tracker) const { + shared_ptr memory_tracker) const { // This should never be called with an empty list of enumeration paths, but // there's no reason to not check an early return case here given that code // changes. @@ -1314,7 +1315,7 @@ bool ArrayDirectory::consolidation_with_timestamps_supported( shared_ptr ArrayDirectory::load_enumeration( const std::string& enumeration_path, const EncryptionKey& encryption_key, - MemoryTracker& memory_tracker) const { + shared_ptr memory_tracker) const { auto timer_se = resources_.get().stats().start_timer("sm_load_enumeration"); auto enmr_uri = uri_.join_path(constants::array_schema_dir_name) @@ -1324,13 +1325,12 @@ shared_ptr ArrayDirectory::load_enumeration( auto&& tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); resources_.get().stats().add_counter("read_enumeration_size", tile.size()); - if (!memory_tracker.take_memory( - tile.size(), MemoryTracker::MemoryType::ENUMERATION)) { + if (!memory_tracker->take_memory(tile.size(), MemoryType::ENUMERATION)) { throw ArrayDirectoryException( "Error loading enumeration; Insufficient memory budget; Needed " + std::to_string(tile.size()) + " but only had " + - std::to_string(memory_tracker.get_memory_available()) + - " from budget " + std::to_string(memory_tracker.get_memory_budget())); + std::to_string(memory_tracker->get_memory_available()) + + " from budget " + std::to_string(memory_tracker->get_memory_budget())); } Deserializer deserializer(tile.data(), tile.size()); diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index ffaf399a7017..9f29563b981b 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -33,7 +33,6 @@ #ifndef TILEDB_ARRAY_DIRECTORY_H #define TILEDB_ARRAY_DIRECTORY_H -#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/array_schema/array_schema.h" @@ -60,6 +59,7 @@ enum class ArrayDirectoryMode { }; // Forward declaration +class MemoryTracker; class WhiteboxArrayDirectory; /** @@ -391,7 +391,7 @@ class ArrayDirectory { std::vector> load_enumerations_from_paths( const std::vector& enumeration_paths, const EncryptionKey& encryption_key, - MemoryTracker& memory_tracker) const; + shared_ptr memory_tracker) const; /** Returns the array URI. */ const URI& uri() const; @@ -824,7 +824,7 @@ class ArrayDirectory { shared_ptr load_enumeration( const std::string& enumeration_path, const EncryptionKey& encryption_key, - MemoryTracker& memory_tracker) const; + shared_ptr memory_tracker) const; }; } // namespace tiledb::sm diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index ec6e3988273a..941b9313487a 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -856,10 +856,11 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { } // Get the array schemas and fragment metadata. + auto memory_tracker = resources_->create_memory_tracker(); std::vector> fragment_metadata; std::tie(array_schema_latest_, array_schemas_all_, fragment_metadata) = load_array_schemas_and_fragment_metadata( - *resources_, array_dir, nullptr, enc_key_); + *resources_, array_dir, memory_tracker, enc_key_); auto fragment_num = (uint32_t)fragment_metadata.size(); // Get fragment sizes @@ -992,7 +993,7 @@ std::tuple< FragmentInfo::load_array_schemas_and_fragment_metadata( ContextResources& resources, const ArrayDirectory& array_dir, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const EncryptionKey& enc_key) { auto timer_se = resources.stats().start_timer( "sm_load_array_schemas_and_fragment_metadata"); @@ -1122,7 +1123,7 @@ tuple> FragmentInfo::load( auto meta = make_shared( HERE(), resources_, - nullptr, + resources_->create_memory_tracker(), array_schema_latest, new_fragment_uri, timestamp_range, diff --git a/tiledb/sm/fragment/fragment_info.h b/tiledb/sm/fragment/fragment_info.h index eb06049543ec..5f5b3bc5dbd1 100644 --- a/tiledb/sm/fragment/fragment_info.h +++ b/tiledb/sm/fragment/fragment_info.h @@ -313,7 +313,7 @@ class FragmentInfo { load_array_schemas_and_fragment_metadata( ContextResources& resources, const ArrayDirectory& array_dir, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const EncryptionKey& enc_key); /** Returns the vector with the info about individual fragments. */ @@ -349,6 +349,11 @@ class FragmentInfo { return config_; } + /** Returns the context resources. */ + inline ContextResources* resources() const { + return resources_; + } + // Accessors /** Returns array schemas latest. */ diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index b337e23bc4dc..f30fd261232e 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -77,12 +77,29 @@ class FragmentMetadataStatusException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -FragmentMetadata::FragmentMetadata() { +FragmentMetadata::FragmentMetadata( + ContextResources* resources, shared_ptr memory_tracker) + : resources_(resources) + , memory_tracker_(memory_tracker) + , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_validity_offsets_( + memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_min_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_min_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_max_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) + , tile_null_counts_( + memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) { } FragmentMetadata::FragmentMetadata( ContextResources* resources, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, const std::pair& timestamp_range, @@ -104,6 +121,20 @@ FragmentMetadata::FragmentMetadata( , meta_file_size_(0) , rtree_(RTree(&array_schema_->domain(), constants::rtree_fanout)) , tile_index_base_(0) + , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_validity_offsets_( + memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_min_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_min_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_max_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) + , tile_null_counts_( + memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) , version_(array_schema_->write_version()) , timestamp_range_(timestamp_range) , array_uri_(array_schema_->array_uri()) { @@ -732,7 +763,7 @@ void FragmentMetadata::init(const NDRange& non_empty_domain) { std::vector> FragmentMetadata::load( ContextResources& resources, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const shared_ptr array_schema_latest, const std::unordered_map>& array_schemas_all, @@ -2052,8 +2083,7 @@ void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { // Use the serialized buffer size to approximate memory usage of the rtree. if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - tile.size(), MemoryTracker::MemoryType::RTREE)) { + !memory_tracker_->take_memory(tile.size(), MemoryType::RTREE)) { throw FragmentMetadataStatusException( "Cannot load R-tree; Insufficient memory budget; Needed " + std::to_string(tile.size()) + " but only had " + @@ -2070,7 +2100,7 @@ void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { void FragmentMetadata::free_rtree() { auto freed = rtree_.free_memory(); if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory(freed, MemoryTracker::MemoryType::RTREE); + memory_tracker_->release_memory(freed, MemoryType::RTREE); } loaded_metadata_.rtree_ = false; } @@ -2080,8 +2110,7 @@ void FragmentMetadata::free_tile_offsets() { std::lock_guard lock(tile_offsets_mtx_[i]); if (memory_tracker_ != nullptr) { memory_tracker_->release_memory( - tile_offsets_[i].size() * sizeof(uint64_t), - MemoryTracker::MemoryType::TILE_OFFSETS); + tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); } tile_offsets_[i].clear(); loaded_metadata_.tile_offsets_[i] = false; @@ -2092,7 +2121,7 @@ void FragmentMetadata::free_tile_offsets() { if (memory_tracker_ != nullptr) { memory_tracker_->release_memory( tile_var_offsets_[i].size() * sizeof(uint64_t), - MemoryTracker::MemoryType::TILE_OFFSETS); + MemoryType::TILE_OFFSETS); } tile_var_offsets_[i].clear(); loaded_metadata_.tile_var_offsets_[i] = false; @@ -2102,8 +2131,7 @@ void FragmentMetadata::free_tile_offsets() { std::lock_guard lock(tile_offsets_mtx_[i]); if (memory_tracker_ != nullptr) { memory_tracker_->release_memory( - tile_offsets_[i].size() * sizeof(uint64_t), - MemoryTracker::MemoryType::TILE_OFFSETS); + tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); } tile_offsets_[i].clear(); loaded_metadata_.tile_offsets_[i] = false; @@ -2114,7 +2142,7 @@ void FragmentMetadata::free_tile_offsets() { if (memory_tracker_ != nullptr) { memory_tracker_->release_memory( tile_validity_offsets_[i].size() * sizeof(uint64_t), - MemoryTracker::MemoryType::TILE_OFFSETS); + MemoryType::TILE_OFFSETS); } tile_validity_offsets_[i].clear(); loaded_metadata_.tile_validity_offsets_[i] = false; @@ -2125,7 +2153,7 @@ void FragmentMetadata::free_tile_offsets() { if (memory_tracker_ != nullptr) { memory_tracker_->release_memory( tile_var_sizes_[i].size() * sizeof(uint64_t), - MemoryTracker::MemoryType::TILE_OFFSETS); + MemoryType::TILE_OFFSETS); } tile_var_sizes_[i].clear(); loaded_metadata_.tile_var_sizes_[i] = false; @@ -2891,8 +2919,7 @@ void FragmentMetadata::load_tile_offsets(Deserializer& deserializer) { auto size = tile_offsets_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile offsets; Insufficient memory budget; Needed " + std::to_string(size) + " but only had " + @@ -2921,8 +2948,7 @@ void FragmentMetadata::load_tile_offsets( if (tile_offsets_num != 0) { auto size = tile_offsets_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile offsets; Insufficient memory budget; Needed " + std::to_string(size) + " but only had " + @@ -2963,8 +2989,7 @@ void FragmentMetadata::load_tile_var_offsets(Deserializer& deserializer) { auto size = tile_var_offsets_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile var offsets; Insufficient memory budget; " "Needed " + @@ -2994,8 +3019,7 @@ void FragmentMetadata::load_tile_var_offsets( if (tile_var_offsets_num != 0) { auto size = tile_var_offsets_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile var offsets; Insufficient memory budget; " "Needed " + @@ -3034,8 +3058,7 @@ void FragmentMetadata::load_tile_var_sizes(Deserializer& deserializer) { auto size = tile_var_sizes_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile var sizes; Insufficient memory budget; " "Needed " + @@ -3064,8 +3087,7 @@ void FragmentMetadata::load_tile_var_sizes( if (tile_var_sizes_num != 0) { auto size = tile_var_sizes_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile var sizes; Insufficient memory budget; " "Needed " + @@ -3095,8 +3117,7 @@ void FragmentMetadata::load_tile_validity_offsets( if (tile_validity_offsets_num != 0) { auto size = tile_validity_offsets_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::TILE_OFFSETS)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { throw FragmentMetadataStatusException( "Cannot load tile validity offsets; Insufficient memory budget; " "Needed " + @@ -3140,8 +3161,7 @@ void FragmentMetadata::load_tile_min_values( if (buffer_size != 0) { auto size = buffer_size + var_buffer_size; if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::MIN_MAX_SUM_NULL_COUNT)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_MIN_VALS)) { throw FragmentMetadataStatusException( "Cannot load min values; Insufficient memory budget; Needed " + std::to_string(size) + " but only had " + @@ -3185,8 +3205,7 @@ void FragmentMetadata::load_tile_max_values( if (buffer_size != 0) { auto size = buffer_size + var_buffer_size; if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::MIN_MAX_SUM_NULL_COUNT)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_MAX_VALS)) { throw FragmentMetadataStatusException( "Cannot load max values; Insufficient memory budget; Needed " + std::to_string(size) + " but only had " + @@ -3224,8 +3243,7 @@ void FragmentMetadata::load_tile_sum_values( if (tile_sum_num != 0) { auto size = tile_sum_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::MIN_MAX_SUM_NULL_COUNT)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_SUMS)) { throw FragmentMetadataStatusException( "Cannot load sum values; Insufficient memory budget; Needed " + std::to_string(size) + " but only had " + @@ -3257,8 +3275,7 @@ void FragmentMetadata::load_tile_null_count_values( if (tile_null_count_num != 0) { auto size = tile_null_count_num * sizeof(uint64_t); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - size, MemoryTracker::MemoryType::MIN_MAX_SUM_NULL_COUNT)) { + !memory_tracker_->take_memory(size, MemoryType::TILE_NULL_COUNTS)) { throw FragmentMetadataStatusException( "Cannot load null count values; Insufficient memory budget; " "Needed " + @@ -3944,8 +3961,7 @@ void FragmentMetadata::read_file_footer( resources_->stats().add_counter("read_frag_meta_size", *footer_size); if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory( - *footer_size, MemoryTracker::MemoryType::FOOTER)) { + !memory_tracker_->take_memory(*footer_size, MemoryType::FOOTER)) { throw FragmentMetadataStatusException( "Cannot load file footer; Insufficient memory budget; Needed " + std::to_string(*footer_size) + " but only had " + diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 2ef0ab30d416..13cb1676a780 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -40,6 +40,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/types.h" @@ -69,8 +70,15 @@ class FragmentMetadata { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Constructor. */ - FragmentMetadata(); + /** + * Constructor. + * + * @param resources A context resources instance. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + */ + FragmentMetadata( + ContextResources* resources, shared_ptr memory_tracker); /** * Constructor. @@ -89,7 +97,7 @@ class FragmentMetadata { */ FragmentMetadata( ContextResources* resources, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, const std::pair& timestamp_range, @@ -318,53 +326,61 @@ class FragmentMetadata { } /** Returns the tile offsets. */ - inline const std::vector>& tile_offsets() const { + inline const tdb::pmr::vector>& tile_offsets() + const { return tile_offsets_; } /** Returns the variable tile offsets. */ - inline const std::vector>& tile_var_offsets() const { + inline const tdb::pmr::vector>& tile_var_offsets() + const { return tile_var_offsets_; } /** Returns the sizes of the uncompressed variable tiles. */ - inline const std::vector>& tile_var_sizes() const { + inline const tdb::pmr::vector>& tile_var_sizes() + const { return tile_var_sizes_; } /** Returns the validity tile offsets. */ - inline const std::vector>& tile_validity_offsets() - const { + inline const tdb::pmr::vector>& + tile_validity_offsets() const { return tile_validity_offsets_; } /** Returns the tile min buffers. */ - inline const std::vector>& tile_min_buffer() const { + inline const tdb::pmr::vector>& tile_min_buffer() + const { return tile_min_buffer_; } /** Returns the tile min buffers variable length data. */ - inline const std::vector>& tile_min_var_buffer() const { + inline const tdb::pmr::vector>& tile_min_var_buffer() + const { return tile_min_var_buffer_; } /** Returns the tile max buffers. */ - inline const std::vector>& tile_max_buffer() const { + inline const tdb::pmr::vector>& tile_max_buffer() + const { return tile_max_buffer_; } /** Returns the tile max buffers variable length data. */ - inline const std::vector>& tile_max_var_buffer() const { + inline const tdb::pmr::vector>& tile_max_var_buffer() + const { return tile_max_var_buffer_; } /** Returns the tile sum values for fixed sized data. */ - inline const std::vector>& tile_sums() const { + inline const tdb::pmr::vector>& tile_sums() const { return tile_sums_; } /** Returns the tile null count values for attributes/dimensions. */ - inline const std::vector>& tile_null_counts() const { + inline const tdb::pmr::vector>& tile_null_counts() + const { return tile_null_counts_; } @@ -480,7 +496,7 @@ class FragmentMetadata { */ static std::vector> load( ContextResources& resources, - MemoryTracker* memory_tracker, + shared_ptr memory_tracker, const shared_ptr array_schema, const std::unordered_map>& array_schemas_all, @@ -1104,7 +1120,7 @@ class FragmentMetadata { } /** tile_offsets accessor */ - std::vector>& tile_offsets() { + tdb::pmr::vector>& tile_offsets() { return tile_offsets_; } @@ -1114,7 +1130,7 @@ class FragmentMetadata { } /** tile_var_offsets accessor */ - std::vector>& tile_var_offsets() { + tdb::pmr::vector>& tile_var_offsets() { return tile_var_offsets_; } @@ -1124,42 +1140,42 @@ class FragmentMetadata { } /** tile_var_sizes accessor */ - std::vector>& tile_var_sizes() { + tdb::pmr::vector>& tile_var_sizes() { return tile_var_sizes_; } /** tile_validity_offsets accessor */ - std::vector>& tile_validity_offsets() { + tdb::pmr::vector>& tile_validity_offsets() { return tile_validity_offsets_; } /** tile_min_buffer accessor */ - std::vector>& tile_min_buffer() { + tdb::pmr::vector>& tile_min_buffer() { return tile_min_buffer_; } /** tile_min_var_buffer accessor */ - std::vector>& tile_min_var_buffer() { + tdb::pmr::vector>& tile_min_var_buffer() { return tile_min_var_buffer_; } /** tile_max_buffer accessor */ - std::vector>& tile_max_buffer() { + tdb::pmr::vector>& tile_max_buffer() { return tile_max_buffer_; } /** tile_max_var_buffer accessor */ - std::vector>& tile_max_var_buffer() { + tdb::pmr::vector>& tile_max_var_buffer() { return tile_max_var_buffer_; } /** tile_sums accessor */ - std::vector>& tile_sums() { + tdb::pmr::vector>& tile_sums() { return tile_sums_; } /** tile_null_counts accessor */ - std::vector>& tile_null_counts() { + tdb::pmr::vector>& tile_null_counts() { return tile_null_counts_; } @@ -1218,11 +1234,6 @@ class FragmentMetadata { resources_ = cr; } - /** set the memory tracker pointer during deserialization*/ - void set_memory_tracker(MemoryTracker* memory_tracker) { - memory_tracker_ = memory_tracker; - } - /** loaded_metadata_.rtree_ accessor */ void set_rtree_loaded() { loaded_metadata_.rtree_ = true; @@ -1264,7 +1275,7 @@ class FragmentMetadata { /** * The memory tracker of the array this fragment metadata corresponds to. */ - MemoryTracker* memory_tracker_; + shared_ptr memory_tracker_; /** The array schema */ shared_ptr array_schema_; @@ -1357,57 +1368,57 @@ class FragmentMetadata { * The tile offsets in their corresponding attribute files. Meaningful only * when there is compression. */ - std::vector> tile_offsets_; + tdb::pmr::vector> tile_offsets_; /** * The variable tile offsets in their corresponding attribute files. * Meaningful only for variable-sized tiles. */ - std::vector> tile_var_offsets_; + tdb::pmr::vector> tile_var_offsets_; /** * The sizes of the uncompressed variable tiles. * Meaningful only when there is compression for variable tiles. */ - std::vector> tile_var_sizes_; + tdb::pmr::vector> tile_var_sizes_; /** * The validity tile offsets in their corresponding attribute files. * Meaningful only when there is compression. */ - std::vector> tile_validity_offsets_; + tdb::pmr::vector> tile_validity_offsets_; /** * The tile min buffers, for variable attributes/dimensions, this will store * offsets. */ - std::vector> tile_min_buffer_; + tdb::pmr::vector> tile_min_buffer_; /** * The tile min buffers variable length data. */ - std::vector> tile_min_var_buffer_; + tdb::pmr::vector> tile_min_var_buffer_; /** * The tile max buffers, for variable attributes/dimensions, this will store * offsets. */ - std::vector> tile_max_buffer_; + tdb::pmr::vector> tile_max_buffer_; /** * The tile max buffers variable length data. */ - std::vector> tile_max_var_buffer_; + tdb::pmr::vector> tile_max_var_buffer_; /** * The tile sum values, ignored for var sized attributes/dimensions. */ - std::vector> tile_sums_; + tdb::pmr::vector> tile_sums_; /** * The tile null count values for attributes/dimensions. */ - std::vector> tile_null_counts_; + tdb::pmr::vector> tile_null_counts_; /** * Fragment min values. diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 713dd5ec6915..56dc8b86c159 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -35,6 +35,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/common/memory.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/enums/query_status.h" @@ -82,7 +83,9 @@ Query::Query( StorageManager* storage_manager, shared_ptr array, optional fragment_name) - : array_shared_(array) + : query_memory_tracker_( + storage_manager->resources().create_memory_tracker()) + , array_shared_(array) , array_(array_shared_.get()) , opened_array_(array->opened_array()) , array_schema_(array->array_schema_latest_ptr()) @@ -113,6 +116,12 @@ Query::Query( , query_remote_buffer_storage_(std::nullopt) { assert(array->is_open()); + if (array->get_query_type() == QueryType::READ) { + query_memory_tracker_->set_type(MemoryTrackerType::QUERY_READ); + } else { + query_memory_tracker_->set_type(MemoryTrackerType::QUERY_WRITE); + } + subarray_ = Subarray(array_, layout_, stats_, logger_); fragment_metadata_ = array->fragment_metadata(); @@ -1797,6 +1806,7 @@ bool Query::is_aggregate(std::string output_field_name) const { Status Query::create_strategy(bool skip_checks_serialization) { auto params = StrategyParams( + array_->memory_tracker(), storage_manager_, opened_array_, config_, @@ -1806,8 +1816,7 @@ Status Query::create_strategy(bool skip_checks_serialization) { layout_, condition_, default_channel_aggregates_, - skip_checks_serialization, - array_->memory_tracker()); + skip_checks_serialization); if (type_ == QueryType::WRITE || type_ == QueryType::MODIFY_EXCLUSIVE) { if (layout_ == Layout::COL_MAJOR || layout_ == Layout::ROW_MAJOR) { if (!array_schema_->dense()) { diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index e54c0342f907..520934c61df0 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -788,6 +788,9 @@ class Query { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The query memory tracker. */ + shared_ptr query_memory_tracker_; + /** A smart pointer to the array the query is associated with. * Ensures that the Array object exists as long as the Query object exists. */ shared_ptr array_shared_; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 501cdd5b4ad3..688f981c80fd 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -30,8 +30,9 @@ * This file implements class DenseReader. */ +#include "tiledb/sm/query/readers/dense_reader.h" #include "tiledb/common/logger.h" - +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension.h" @@ -40,7 +41,6 @@ #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/query/legacy/cell_slab_iter.h" #include "tiledb/sm/query/query_macros.h" -#include "tiledb/sm/query/readers/dense_reader.h" #include "tiledb/sm/query/readers/filtered_data.h" #include "tiledb/sm/query/readers/result_tile.h" #include "tiledb/sm/stats/global_stats.h" @@ -73,8 +73,7 @@ DenseReader::DenseReader( shared_ptr logger, StrategyParams& params, bool remote_query) - : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) - , array_memory_tracker_(params.memory_tracker()) { + : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) { elements_mode_ = false; // Sanity checks. diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 16c997299acf..37eb8a08d5ea 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -162,9 +162,6 @@ class DenseReader : public ReaderBase, public IQueryStrategy { /** Target upper memory limit for tiles. */ uint64_t tile_upper_memory_limit_; - /** Memory tracker object for the array. */ - MemoryTracker* array_memory_tracker_; - /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index c4703673426c..c83d866ab158 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -248,9 +248,9 @@ void SparseGlobalOrderReader::load_all_tile_offsets() { // Make sure we have enough space for tile offsets data. uint64_t total_tile_offset_usage = tile_offsets_size(subarray_.relevant_fragments()); - uint64_t available_memory = array_memory_tracker_->get_memory_available() - - array_memory_tracker_->get_memory_usage( - MemoryTracker::MemoryType::TILE_OFFSETS); + uint64_t available_memory = + array_memory_tracker_->get_memory_available() - + array_memory_tracker_->get_memory_usage(MemoryType::TILE_OFFSETS); if (total_tile_offset_usage > available_memory) { throw SparseGlobalOrderReaderStatusException( "Cannot load tile offsets, computed size (" + diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 3d2861973d71..88f18e43beb3 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -74,7 +74,6 @@ SparseIndexReaderBase::SparseIndexReaderBase( , tmp_read_state_(array_->fragment_metadata().size()) , memory_budget_(config_, reader_string) , include_coords_(include_coords) - , array_memory_tracker_(params.memory_tracker()) , memory_used_for_coords_total_(0) , deletes_consolidation_no_purge_( buffers_.count(constants::delete_timestamps) != 0) diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index 5ef454e20cad..c9303391001f 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -608,9 +608,6 @@ class SparseIndexReaderBase : public ReaderBase { /** Are dimensions var sized. */ std::vector is_dim_var_size_; - /** Memory tracker object for the array. */ - MemoryTracker* array_memory_tracker_; - /** Memory used for coordinates tiles. */ std::atomic memory_used_for_coords_total_; diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index f51cf512c65f..dcc13a574e44 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -252,9 +252,9 @@ void SparseUnorderedWithDupsReader::load_tile_offsets_data() { bool initial_load = tile_offsets_min_frag_idx_ == std::numeric_limits::max() && tile_offsets_max_frag_idx_ == 0; - uint64_t available_memory = array_memory_tracker_->get_memory_available() - - array_memory_tracker_->get_memory_usage( - MemoryTracker::MemoryType::TILE_OFFSETS); + uint64_t available_memory = + array_memory_tracker_->get_memory_available() - + array_memory_tracker_->get_memory_usage(MemoryType::TILE_OFFSETS); auto& relevant_fragments = subarray_.relevant_fragments(); if (!partial_tile_offsets_loading_) { diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 09406c734de9..a0e3b504f645 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -47,7 +47,8 @@ namespace sm { StrategyBase::StrategyBase( stats::Stats* stats, shared_ptr logger, StrategyParams& params) - : stats_(stats) + : array_memory_tracker_(params.array_memory_tracker()) + , stats_(stats) , logger_(logger) , array_(params.array()) , array_schema_(params.array()->array_schema_latest()) @@ -61,10 +62,6 @@ StrategyBase::StrategyBase( , offsets_bitsize_(constants::cell_var_offset_size * 8) { } -stats::Stats* StrategyBase::stats() const { - return stats_; -} - void StrategyBase::set_stats(const stats::StatsData& data) { stats_->populate_with_data(data); } diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index b5d1abb983a9..db1119597ab5 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -66,6 +66,7 @@ class StrategyParams { /* ********************************* */ StrategyParams( + shared_ptr array_memory_tracker, StorageManager* storage_manager, shared_ptr array, Config& config, @@ -75,9 +76,9 @@ class StrategyParams { Layout layout, std::optional& condition, DefaultChannelAggregates& default_channel_aggregates, - bool skip_checks_serialization, - MemoryTracker* memory_tracker) - : storage_manager_(storage_manager) + bool skip_checks_serialization) + : array_memory_tracker_(array_memory_tracker) + , storage_manager_(storage_manager) , array_(array) , config_(config) , buffers_(buffers) @@ -86,14 +87,18 @@ class StrategyParams { , layout_(layout) , condition_(condition) , default_channel_aggregates_(default_channel_aggregates) - , skip_checks_serialization_(skip_checks_serialization) - , memory_tracker_(memory_tracker) { + , skip_checks_serialization_(skip_checks_serialization) { } /* ********************************* */ /* API */ /* ********************************* */ + /** Return the array memory tracker. */ + inline shared_ptr array_memory_tracker() { + return array_memory_tracker_; + } + /** Return the storage manager. */ inline StorageManager* storage_manager() { return storage_manager_; @@ -144,15 +149,14 @@ class StrategyParams { return skip_checks_serialization_; } - inline MemoryTracker* memory_tracker() { - return memory_tracker_; - } - private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** Array Memory tracker. */ + shared_ptr array_memory_tracker_; + /** Storage manager. */ StorageManager* storage_manager_; @@ -182,9 +186,6 @@ class StrategyParams { /** Skip checks for serialization. */ bool skip_checks_serialization_; - - /** Memory tracker. */ - MemoryTracker* memory_tracker_; }; /** Processes read or write queries. */ @@ -206,7 +207,9 @@ class StrategyBase { /* ********************************* */ /** Returns `stats_`. */ - stats::Stats* stats() const; + inline stats::Stats* stats() const { + return stats_; + } /** * Populate the owned stats instance with data. @@ -239,6 +242,9 @@ class StrategyBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr array_memory_tracker_; + /** The class stats. */ stats::Stats* stats_; diff --git a/tiledb/sm/query/test/CMakeLists.txt b/tiledb/sm/query/test/CMakeLists.txt index b6df998a9f9f..32827f7eae3b 100644 --- a/tiledb/sm/query/test/CMakeLists.txt +++ b/tiledb/sm/query/test/CMakeLists.txt @@ -29,7 +29,7 @@ include(unit_test) commence(unit_test query) this_target_sources(main.cc unit_validity_vector.cc unit_query_condition.cc) # Not actually testing a unit yet, but some things that ought to be units - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(tiledb_test_support_lib) this_target_link_libraries(ast_test_support_lib) # We want tests to continue as normal even as the API is changing, diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 0d53549ac7e6..b4a8c6b0fb96 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -31,7 +31,9 @@ */ #include "test/support/src/ast_helpers.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" @@ -1121,7 +1123,7 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -1158,7 +1160,7 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -1307,7 +1309,7 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -1589,7 +1591,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -1645,7 +1647,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -1757,7 +1759,7 @@ TEST_CASE( frag_md[0] = make_shared( HERE(), nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -2304,7 +2306,7 @@ void test_apply_dense( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -2363,7 +2365,7 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -2474,7 +2476,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -3003,7 +3005,7 @@ void test_apply_sparse( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -3062,7 +3064,7 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -3213,7 +3215,7 @@ void validate_qc_apply( frag_md[0] = make_shared( HERE(), nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -3832,7 +3834,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -4118,7 +4120,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -4472,7 +4474,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -4794,7 +4796,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), @@ -4893,7 +4895,7 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - nullptr, + tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 5d943e792010..a9110d240600 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -177,7 +177,7 @@ Status GlobalOrderWriter::alloc_global_write_state() { global_write_state_.reset(new GlobalWriteState); // Alloc FragmentMetadata object - global_write_state_->frag_meta_ = make_shared(HERE()); + global_write_state_->frag_meta_ = this->create_fragment_metadata(); // Used in serialization when FragmentMetadata is built from ground up global_write_state_->frag_meta_->set_context_resources( &storage_manager_->resources()); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 714b6b7d9b2f..53e77377970c 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -176,7 +176,7 @@ Status OrderedWriter::ordered_write() { auto timer_se = stats_->start_timer("ordered_write"); // Create new fragment - auto frag_meta = make_shared(HERE()); + auto frag_meta = this->create_fragment_metadata(); RETURN_CANCEL_OR_ERROR(create_fragment(true, frag_meta)); frag_uri_ = frag_meta->fragment_uri(); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 2084f9feb2ac..7dc5b57930f7 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -167,7 +167,7 @@ std::string UnorderedWriter::name() { Status UnorderedWriter::alloc_frag_meta() { // Alloc FragmentMetadata object. - frag_meta_ = make_shared(HERE()); + frag_meta_ = this->create_fragment_metadata(); // Used in serialization when FragmentMetadata is built from ground up. frag_meta_->set_context_resources(&storage_manager_->resources()); @@ -653,7 +653,7 @@ Status UnorderedWriter::unordered_write() { } // Create new fragment - frag_meta_ = make_shared(HERE()); + frag_meta_ = this->create_fragment_metadata(); RETURN_CANCEL_OR_ERROR(create_fragment(false, frag_meta_)); } diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 751899c547d1..4c9dfb9c7254 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -309,6 +309,11 @@ void WriterBase::refresh_config() { /* PRIVATE METHODS */ /* ****************************** */ +shared_ptr WriterBase::create_fragment_metadata() { + return make_shared( + HERE(), &storage_manager_->resources(), array_memory_tracker_); +} + Status WriterBase::add_written_fragment_info(const URI& uri) { written_fragment_info_.emplace_back(uri, fragment_timestamp_range_); return Status::Ok(); @@ -787,7 +792,7 @@ Status WriterBase::create_fragment( frag_meta = make_shared( HERE(), &storage_manager_->resources(), - nullptr, + array_memory_tracker_, array_->array_schema_latest_ptr(), fragment_uri_, timestamp_range, diff --git a/tiledb/sm/query/writers/writer_base.h b/tiledb/sm/query/writers/writer_base.h index e289f5e936dc..8438ca75fd75 100644 --- a/tiledb/sm/query/writers/writer_base.h +++ b/tiledb/sm/query/writers/writer_base.h @@ -193,6 +193,9 @@ class WriterBase : public StrategyBase, public IQueryStrategy { /* PROTECTED METHODS */ /* ********************************* */ + /** Utility function for constructing new FragmentMetadata instances. */ + shared_ptr create_fragment_metadata(); + /** Adss a fragment to `written_fragment_info_`. */ Status add_written_fragment_info(const URI& uri); diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 7792c1ca5b6d..6e1e4f04aa78 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -307,13 +307,10 @@ Status array_from_capnp( auto fragment_metadata_all_reader = array_reader.getFragmentMetadataAll(); fragment_metadata.reserve(fragment_metadata_all_reader.size()); for (auto frag_meta_reader : fragment_metadata_all_reader) { - auto meta = make_shared(HERE()); + auto meta = make_shared( + HERE(), &storage_manager->resources(), array->memory_tracker()); RETURN_NOT_OK(fragment_metadata_from_capnp( - array->array_schema_latest_ptr(), - frag_meta_reader, - meta, - &storage_manager->resources(), - array->memory_tracker())); + array->array_schema_latest_ptr(), frag_meta_reader, meta)); if (client_side) { meta->set_rtree_loaded(); } diff --git a/tiledb/sm/serialization/fragment_info.cc b/tiledb/sm/serialization/fragment_info.cc index 4d2badb9e8c9..5be515c7e91d 100644 --- a/tiledb/sm/serialization/fragment_info.cc +++ b/tiledb/sm/serialization/fragment_info.cc @@ -199,8 +199,7 @@ Status fragment_info_request_deserialize( std::tuple> single_fragment_info_from_capnp( const capnp::SingleFragmentInfo::Reader& single_frag_info_reader, - const std::unordered_map>& - array_schemas) { + FragmentInfo* fragment_info) { // Get array schema name std::string schema_name; if (single_frag_info_reader.hasArraySchemaName()) { @@ -213,8 +212,8 @@ single_fragment_info_from_capnp( } // Use the array schema name to find the corresponding array schema - auto schema = array_schemas.find(schema_name); - if (schema == array_schemas.end()) { + auto schema = fragment_info->array_schemas_all().find(schema_name); + if (schema == fragment_info->array_schemas_all().end()) { return { Status_SerializationError( "Could not find schema" + schema_name + @@ -226,7 +225,10 @@ single_fragment_info_from_capnp( shared_ptr meta; if (single_frag_info_reader.hasMeta()) { auto frag_meta_reader = single_frag_info_reader.getMeta(); - meta = make_shared(HERE()); + + auto memory_tracker = fragment_info->resources()->create_memory_tracker(); + meta = make_shared( + HERE(), fragment_info->resources(), memory_tracker); auto st = fragment_metadata_from_capnp(schema->second, frag_meta_reader, meta); } else { @@ -312,7 +314,7 @@ Status fragment_info_from_capnp( fragment_info_list_reader.size()); for (auto single_frag_info_reader : fragment_info_list_reader) { auto&& [st, single_frag_info] = single_fragment_info_from_capnp( - single_frag_info_reader, fragment_info->array_schemas_all()); + single_frag_info_reader, fragment_info); RETURN_NOT_OK(st); fragment_info->single_fragment_info_vec().emplace_back( single_frag_info.value()); diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index 74482c0b4c15..c5e8f581acd0 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -119,17 +119,7 @@ void generic_tile_offsets_from_capnp( Status fragment_metadata_from_capnp( const shared_ptr& array_schema, const capnp::FragmentMetadata::Reader& frag_meta_reader, - shared_ptr frag_meta, - ContextResources* resources, - MemoryTracker* memory_tracker) { - // TODO: consider a new constructor for fragment meta or using the - // existing one - if (resources) { - frag_meta->set_context_resources(resources); - } - if (memory_tracker) { - frag_meta->set_memory_tracker(memory_tracker); - } + shared_ptr frag_meta) { if (frag_meta_reader.hasFileSizes()) { auto filesizes_reader = frag_meta_reader.getFileSizes(); frag_meta->file_sizes().reserve(filesizes_reader.size()); diff --git a/tiledb/sm/serialization/fragment_metadata.h b/tiledb/sm/serialization/fragment_metadata.h index 270f50a2a9c4..281fc875f04e 100644 --- a/tiledb/sm/serialization/fragment_metadata.h +++ b/tiledb/sm/serialization/fragment_metadata.h @@ -64,9 +64,7 @@ namespace serialization { Status fragment_metadata_from_capnp( const shared_ptr& array_schema, const capnp::FragmentMetadata::Reader& frag_meta_reader, - shared_ptr frag_meta, - ContextResources* resources = nullptr, - MemoryTracker* memory_tracker = nullptr); + shared_ptr frag_meta); /** * Serialize Fragment Metadata sizes and offsets diff --git a/tiledb/sm/storage_manager/context_resources.cc b/tiledb/sm/storage_manager/context_resources.cc index 1dd423a163ac..999f0f7566b9 100644 --- a/tiledb/sm/storage_manager/context_resources.cc +++ b/tiledb/sm/storage_manager/context_resources.cc @@ -31,6 +31,7 @@ */ #include "tiledb/sm/storage_manager/context_resources.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/rest/rest_client.h" using namespace tiledb::common; @@ -53,7 +54,10 @@ ContextResources::ContextResources( size_t compute_thread_count, size_t io_thread_count, std::string stats_name) - : config_(config) + : memory_tracker_manager_(make_shared(HERE())) + , memory_tracker_reporter_(make_shared( + HERE(), config, memory_tracker_manager_)) + , config_(config) , logger_(logger) , compute_tp_(compute_thread_count) , io_tp_(io_thread_count) @@ -77,6 +81,12 @@ ContextResources::ContextResources( rest_client_ = client; } } + + memory_tracker_reporter_->start(); +} + +shared_ptr ContextResources::create_memory_tracker() const { + return memory_tracker_manager_->create_tracker(); } } // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/context_resources.h b/tiledb/sm/storage_manager/context_resources.h index 3bf63e6fbcc8..b19cb7f310c9 100644 --- a/tiledb/sm/storage_manager/context_resources.h +++ b/tiledb/sm/storage_manager/context_resources.h @@ -44,6 +44,9 @@ using namespace tiledb::common; namespace tiledb::sm { +class MemoryTracker; +class MemoryTrackerManager; +class MemoryTrackerReporter; class RestClient; /** @@ -109,11 +112,24 @@ class ContextResources { return rest_client_; } + /** + * Create a new MemoryTracker + * + * @return The created MemoryTracker. + */ + shared_ptr create_memory_tracker() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The MemoryTrackerManager for this context. */ + mutable shared_ptr memory_tracker_manager_; + + /** The MemoryTrackerReporter for this context. */ + mutable shared_ptr memory_tracker_reporter_; + /** The configuration for this ContextResources */ mutable Config config_; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index b58e7bf2bc0b..570c0e0f537a 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -396,9 +396,8 @@ Status StorageManager::array_evolve_schema( enmr_paths.emplace_back(path); } - MemoryTracker tracker; auto loaded_enmrs = array_dir.load_enumerations_from_paths( - enmr_paths, encryption_key, tracker); + enmr_paths, encryption_key, resources_.create_memory_tracker()); for (auto enmr : loaded_enmrs) { array_schema->store_enumeration(enmr); diff --git a/vcpkg.json b/vcpkg.json index c8b9831f013e..2220a145d6d9 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,7 @@ { "builtin-baseline": "72010900b7cee36cea77aebb97695095c9358eaf", "dependencies": [ + "boost-container", "bzip2", "libmagic", "lz4", From ec0ff6ea4119a75a90e24abd5122bfe224a35401 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Mon, 12 Feb 2024 23:05:26 -0800 Subject: [PATCH 185/456] Create pmr::unordered_map class for memory tracking. (#4698) Implement a pmr::unordered_map class that uses polymorphic memory resource tracking. --- TYPE: NO_HISTORY DESC: Create pmr::unordered_map class for memory tracking. --- tiledb/common/pmr.h | 158 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index e66179e5fd86..aa36488121c2 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -34,6 +34,7 @@ #ifndef TILEDB_COMMON_PMR_H #define TILEDB_COMMON_PMR_H +#include #include #include @@ -46,6 +47,10 @@ namespace tiledb::common::pmr { using memory_resource = boost::container::pmr::memory_resource; +/* ********************************* */ +/* PMR VECTOR DECLARATION */ +/* ********************************* */ + template using pmr_vector = std::vector>; @@ -122,6 +127,159 @@ class vector : public pmr_vector { } }; +/* ********************************* */ +/* PMR UNORDERED MAP DECLARATION */ +/* ********************************* */ + +template < + class Key, + class T, + class Hash = std::hash, + class KeyEqual = std::equal_to> +using pmr_unordered_map = std::unordered_map< + Key, + T, + Hash, + KeyEqual, + boost::container::pmr::polymorphic_allocator>>; + +template < + class Key, + class T, + class Hash = std::hash, + class KeyEqual = std::equal_to> +class unordered_map : public pmr_unordered_map { + public: + // Type declarations. + using key_type = typename pmr_unordered_map::key_type; + using mapped_type = + typename pmr_unordered_map::mapped_type; + using value_type = + typename pmr_unordered_map::value_type; + using size_type = + typename pmr_unordered_map::size_type; + using difference_type = + typename pmr_unordered_map::difference_type; + using hasher = typename pmr_unordered_map::hasher; + using key_equal = + typename pmr_unordered_map::key_equal; + using allocator_type = + typename pmr_unordered_map::allocator_type; + using reference = + typename pmr_unordered_map::reference; + using const_reference = + typename pmr_unordered_map::const_reference; + using pointer = typename pmr_unordered_map::pointer; + using const_pointer = + typename pmr_unordered_map::const_pointer; + using iterator = typename pmr_unordered_map::iterator; + using const_iterator = + typename pmr_unordered_map::const_iterator; + using local_iterator = + typename pmr_unordered_map::local_iterator; + using node_type = + typename pmr_unordered_map::node_type; + using insert_return_type = + typename pmr_unordered_map::insert_return_type; + + constexpr unordered_map() = delete; + constexpr unordered_map(const unordered_map& other) = delete; + constexpr unordered_map(unordered_map&& other) = delete; + + constexpr explicit unordered_map( + size_type bucket_count, + const Hash& hash, + const key_equal& equal, + const allocator_type& alloc) + : pmr_unordered_map( + bucket_count, hash, equal, alloc) { + } + + constexpr unordered_map(size_type bucket_count, const allocator_type& alloc) + : pmr_unordered_map( + bucket_count, Hash(), KeyEqual(), alloc) { + } + + constexpr unordered_map( + size_type bucket_count, const Hash& hash, const allocator_type& alloc) + : pmr_unordered_map( + bucket_count, hash, KeyEqual(), alloc) { + } + + constexpr explicit unordered_map(const allocator_type& alloc) + : pmr_unordered_map(alloc) { + } + + template + constexpr unordered_map( + InputIt first, + InputIt last, + size_type bucket_count, + const Hash& hash, + const key_equal& equal, + const allocator_type& alloc) + : pmr_unordered_map( + first, last, bucket_count, hash, equal, alloc) { + } + + template + constexpr unordered_map( + InputIt first, + InputIt last, + size_type bucket_count, + const allocator_type& alloc) + : pmr_unordered_map( + first, last, bucket_count, Hash(), KeyEqual(), alloc) { + } + + template + constexpr unordered_map( + InputIt first, + InputIt last, + size_type bucket_count, + const Hash& hash, + const allocator_type& alloc) + : pmr_unordered_map( + first, last, bucket_count, hash, KeyEqual(), alloc) { + } + + constexpr unordered_map( + const unordered_map& other, const allocator_type& alloc) + : pmr_unordered_map(other, alloc) { + } + + constexpr unordered_map(unordered_map&& other, const allocator_type& alloc) + : pmr_unordered_map(other, alloc) { + } + + constexpr unordered_map( + std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const key_equal& equal, + const allocator_type& alloc) + : pmr_unordered_map( + init, bucket_count, hash, equal, alloc) { + } + + constexpr unordered_map( + std::initializer_list init, + size_type bucket_count, + const allocator_type& alloc) + : pmr_unordered_map( + init, bucket_count, Hash(), KeyEqual(), alloc) { + } + + constexpr unordered_map( + std::initializer_list init, + size_type bucket_count, + const Hash& hash, + const allocator_type& alloc) + : pmr_unordered_map( + init, bucket_count, hash, KeyEqual(), alloc) { + } +}; + } // namespace tiledb::common::pmr #endif // TILEDB_COMMON_PMR_H From 3351f2a35f2c39e3014bb590d47d198bc45220f3 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 13 Feb 2024 18:27:47 +0200 Subject: [PATCH 186/456] Lazily initialize the Azure SDK client. (#4703) [SC-40555](https://app.shortcut.com/tiledb-inc/story/40555/lazily-initialize-the-azure-sdk) The other two cloud VFSes already lazily initialize their SDK client, this PR does it for Azure as well. Validated by locally running the Azure tests in Azurite. --- TYPE: IMPROVEMENT DESC: Lazily initialize the Azure SDK client. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/sm/filesystem/azure.cc | 297 +++++++++++++++++----------------- tiledb/sm/filesystem/azure.h | 100 +++++++++--- 2 files changed, 226 insertions(+), 171 deletions(-) diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 504547486d60..4da5bfc33d66 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -43,13 +43,12 @@ #include "tiledb/common/stdx_string.h" #include "tiledb/platform/cert_file.h" #include "tiledb/sm/filesystem/azure.h" -#include "tiledb/sm/filesystem/ssl_config.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/utils.h" static std::shared_ptr<::Azure::Core::Http::HttpTransport> create_transport( - tiledb::sm::SSLConfig& ssl_cfg); + const tiledb::sm::SSLConfig& ssl_cfg); using namespace tiledb::common; using tiledb::common::filesystem::directory_entry; @@ -61,102 +60,96 @@ namespace sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -Azure::Azure() - : write_cache_max_size_(0) - , max_parallel_ops_(1) - , block_list_block_size_(0) - , use_block_list_upload_(false) { +Azure::Azure() { } Azure::~Azure() { } -/* ********************************* */ -/* API */ -/* ********************************* */ - -Status Azure::init(const Config& config, ThreadPool* const thread_pool) { - if (thread_pool == nullptr) { - return LOG_STATUS( - Status_AzureError("Can't initialize with null thread pool.")); - } - - thread_pool_ = thread_pool; - - bool found; - char* tmp = nullptr; - - std::string account_name = - config.get("vfs.azure.storage_account_name", &found); - assert(found); - if (account_name.empty() && - ((tmp = getenv("AZURE_STORAGE_ACCOUNT")) != nullptr)) { - account_name = std::string(tmp); - } - - std::string account_key = config.get("vfs.azure.storage_account_key", &found); - assert(found); - if (account_key.empty() && ((tmp = getenv("AZURE_STORAGE_KEY")) != nullptr)) { - account_key = std::string(tmp); +std::string get_config_with_env_fallback( + const Config& config, const std::string& key, const char* env_name) { + std::string result = config.get(key, Config::must_find); + if (result.empty()) { + char* env = getenv(env_name); + if (env) { + result = getenv(env_name); + } } + return result; +} - std::string sas_token = - config.get("vfs.azure.storage_sas_token", Config::must_find); - if (sas_token.empty() && - ((tmp = getenv("AZURE_STORAGE_SAS_TOKEN")) != nullptr)) { - sas_token = std::string(tmp); - } +std::string get_blob_endpoint( + const Config& config, const std::string& account_name) { + std::string sas_token = get_config_with_env_fallback( + config, "vfs.azure.storage_sas_token", "AZURE_STORAGE_SAS_TOKEN"); - std::string blob_endpoint = config.get("vfs.azure.blob_endpoint", &found); - assert(found); - if (blob_endpoint.empty() && - ((tmp = getenv("AZURE_BLOB_ENDPOINT")) != nullptr)) { - blob_endpoint = std::string(tmp); - } - if (blob_endpoint.empty()) { + std::string result = get_config_with_env_fallback( + config, "vfs.azure.blob_endpoint", "AZURE_BLOB_ENDPOINT"); + if (result.empty()) { if (!account_name.empty()) { - blob_endpoint = "https://" + account_name + ".blob.core.windows.net"; + result = "https://" + account_name + ".blob.core.windows.net"; } else { LOG_WARN( "Neither the 'vfs.azure.storage_account_name' nor the " "'vfs.azure.blob_endpoint' options are specified."); } - } else if (!(utils::parse::starts_with(blob_endpoint, "http://") || - utils::parse::starts_with(blob_endpoint, "https://"))) { + } else if (!(utils::parse::starts_with(result, "http://") || + utils::parse::starts_with(result, "https://"))) { LOG_WARN( "The 'vfs.azure.blob_endpoint' option should include the scheme (HTTP " "or HTTPS)."); } - if (!blob_endpoint.empty() && !sas_token.empty()) { + if (!result.empty() && !sas_token.empty()) { // The question mark is not strictly part of the SAS token // (https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview#sas-token), // but in the Azure Portal the SAS token starts with one. If it does not, we // add the question mark ourselves. if (!utils::parse::starts_with(sas_token, "?")) { - blob_endpoint += '?'; + result += '?'; } - blob_endpoint += sas_token; + result += sas_token; } + return result; +} - RETURN_NOT_OK(config.get( - "vfs.azure.max_parallel_ops", &max_parallel_ops_, &found)); - assert(found); - RETURN_NOT_OK(config.get( - "vfs.azure.block_list_block_size", &block_list_block_size_, &found)); - assert(found); - RETURN_NOT_OK(config.get( - "vfs.azure.use_block_list_upload", &use_block_list_upload_, &found)); - assert(found); +AzureParameters::AzureParameters(const Config& config) + : max_parallel_ops_( + config.get("vfs.azure.max_parallel_ops", Config::must_find)) + , block_list_block_size_(config.get( + "vfs.azure.block_list_block_size", Config::must_find)) + , write_cache_max_size_(max_parallel_ops_ * block_list_block_size_) + , max_retries_( + config.get("vfs.azure.max_retries", Config::must_find)) + , retry_delay_(std::chrono::milliseconds( + config.get("vfs.azure.retry_delay_ms", Config::must_find))) + , max_retry_delay_(std::chrono::milliseconds(config.get( + "vfs.azure.max_retry_delay_ms", Config::must_find))) + , use_block_list_upload_(config.get( + "vfs.azure.use_block_list_upload", Config::must_find)) + , account_name_(get_config_with_env_fallback( + config, "vfs.azure.storage_account_name", "AZURE_STORAGE_ACCOUNT")) + , account_key_(get_config_with_env_fallback( + config, "vfs.azure.storage_account_key", "AZURE_STORAGE_KEY")) + , blob_endpoint_(get_blob_endpoint(config, account_name_)) + , ssl_cfg_(config) { +} - int max_retries = - config.get("vfs.azure.max_retries", Config::must_find); - retry_delay_ = std::chrono::milliseconds( - config.get("vfs.azure.retry_delay_ms", Config::must_find)); - std::chrono::milliseconds max_retry_delay{ - config.get("vfs.azure.retry_delay_ms", Config::must_find)}; +Status Azure::init(const Config& config, ThreadPool* const thread_pool) { + if (thread_pool == nullptr) { + return LOG_STATUS( + Status_AzureError("Can't initialize with null thread pool.")); + } + thread_pool_ = thread_pool; + azure_params_ = config; + return Status::Ok(); +} - write_cache_max_size_ = max_parallel_ops_ * block_list_block_size_; +/* ********************************* */ +/* API */ +/* ********************************* */ +const ::Azure::Storage::Blobs::BlobServiceClient& +Azure::AzureClientSingleton::get(const AzureParameters& params) { // Initialize logging from the Azure SDK. static std::once_flag azure_log_sentinel; std::call_once(azure_log_sentinel, []() { @@ -179,38 +172,40 @@ Status Azure::init(const Config& config, ThreadPool* const thread_pool) { }); }); - ::Azure::Storage::Blobs::BlobClientOptions options; - options.Retry.MaxRetries = max_retries; - options.Retry.RetryDelay = retry_delay_; - options.Retry.MaxRetryDelay = max_retry_delay; - - SSLConfig ssl_cfg = SSLConfig(config); - options.Transport.Transport = create_transport(ssl_cfg); - - // Construct the Azure SDK blob service client. - // We pass a shared key if it was specified. - if (!account_key.empty()) { - client_ = - tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( - ::Azure::Storage::Blobs::BlobServiceClient, - blob_endpoint, - make_shared<::Azure::Storage::StorageSharedKeyCredential>( - HERE(), account_name, account_key), - options)); - } else { - client_ = - tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( - ::Azure::Storage::Blobs::BlobServiceClient, - blob_endpoint, - options)); + std::lock_guard lck(client_init_mtx_); + + if (!client_) { + ::Azure::Storage::Blobs::BlobClientOptions options; + options.Retry.MaxRetries = params.max_retries_; + options.Retry.RetryDelay = params.retry_delay_; + options.Retry.MaxRetryDelay = params.max_retry_delay_; + + options.Transport.Transport = create_transport(params.ssl_cfg_); + + // Construct the Azure SDK blob service client. + // We pass a shared key if it was specified. + if (!params.account_key_.empty()) { + client_ = + tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( + ::Azure::Storage::Blobs::BlobServiceClient, + params.blob_endpoint_, + make_shared<::Azure::Storage::StorageSharedKeyCredential>( + HERE(), params.account_name_, params.account_key_), + options)); + } else { + client_ = + tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( + ::Azure::Storage::Blobs::BlobServiceClient, + params.blob_endpoint_, + options)); + } } - return Status::Ok(); + return *client_; } Status Azure::create_container(const URI& uri) const { - assert(client_); - + const auto& c = client(); if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( std::string("URI is not an Azure URI: " + uri.to_string()))); @@ -222,8 +217,7 @@ Status Azure::create_container(const URI& uri) const { bool created; std::string error_message = ""; try { - created = - client_->GetBlobContainerClient(container_name).Create().Value.Created; + created = c.GetBlobContainerClient(container_name).Create().Value.Created; } catch (const ::Azure::Storage::StorageException& e) { created = false; error_message = "; " + e.Message; @@ -238,15 +232,14 @@ Status Azure::create_container(const URI& uri) const { } Status Azure::empty_container(const URI& container) const { - assert(client_); - return remove_dir(container); } Status Azure::flush_blob(const URI& uri) { - assert(client_); + assert(azure_params_); + const auto& c = client(); - if (!use_block_list_upload_) { + if (!azure_params_->use_block_list_upload_) { return flush_blob_direct(uri); } @@ -308,7 +301,7 @@ Status Azure::flush_blob(const URI& uri) { finish_block_list_upload(uri); try { - client_->GetBlobContainerClient(container_name) + c.GetBlobContainerClient(container_name) .GetBlockBlobClient(blob_path) .CommitBlockList(std::vector(block_ids.begin(), block_ids.end())); } catch (const ::Azure::Storage::StorageException& e) { @@ -334,6 +327,7 @@ void Azure::finish_block_list_upload(const URI& uri) { } Status Azure::flush_blob_direct(const URI& uri) { + auto& c = client(); if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( std::string("URI is not an Azure URI: " + uri.to_string()))); @@ -350,7 +344,7 @@ Status Azure::flush_blob_direct(const URI& uri) { RETURN_NOT_OK(parse_azure_uri(uri, &container_name, &blob_path)); try { - client_->GetBlobContainerClient(container_name) + c.GetBlobContainerClient(container_name) .GetBlockBlobClient(blob_path) .UploadFrom( static_cast(write_cache_buffer->data()), @@ -370,7 +364,7 @@ Status Azure::flush_blob_direct(const URI& uri) { } Status Azure::is_empty_container(const URI& uri, bool* is_empty) const { - assert(client_); + const auto& c = client(); assert(is_empty); if (!uri.is_azure()) { @@ -384,7 +378,7 @@ Status Azure::is_empty_container(const URI& uri, bool* is_empty) const { ::Azure::Storage::Blobs::ListBlobsOptions options; options.PageSizeHint = 1; try { - *is_empty = client_->GetBlobContainerClient(container_name) + *is_empty = c.GetBlobContainerClient(container_name) .ListBlobs(options) .Blobs.empty(); } catch (const ::Azure::Storage::StorageException& e) { @@ -411,11 +405,11 @@ Status Azure::is_container(const URI& uri, bool* const is_container) const { Status Azure::is_container( const std::string& container_name, bool* const is_container) const { - assert(client_); + const auto& c = client(); assert(is_container); try { - client_->GetBlobContainerClient(container_name).GetProperties(); + c.GetBlobContainerClient(container_name).GetProperties(); } catch (const ::Azure::Storage::StorageException& e) { if (e.StatusCode == ::Azure::Core::Http::HttpStatusCode::NotFound) { *is_container = false; @@ -431,7 +425,6 @@ Status Azure::is_container( } Status Azure::is_dir(const URI& uri, bool* const exists) const { - assert(client_); assert(exists); std::vector paths; @@ -454,11 +447,11 @@ Status Azure::is_blob( const std::string& container_name, const std::string& blob_path, bool* const is_blob) const { - assert(client_); + const auto& c = client(); assert(is_blob); try { - client_->GetBlobContainerClient(container_name) + c.GetBlobContainerClient(container_name) .GetBlobClient(blob_path) .GetProperties(); } catch (const ::Azure::Storage::StorageException& e) { @@ -503,7 +496,6 @@ Status Azure::ls( std::vector* paths, const std::string& delimiter, const int max_paths) const { - assert(client_); assert(paths); auto&& [st, entries] = ls_with_sizes(uri, delimiter, max_paths); @@ -518,7 +510,7 @@ Status Azure::ls( tuple>> Azure::ls_with_sizes( const URI& uri, const std::string& delimiter, int max_paths) const { - assert(client_); + const auto& c = client(); const URI uri_dir = uri.add_trailing_slash(); @@ -533,7 +525,7 @@ tuple>> Azure::ls_with_sizes( RETURN_NOT_OK_TUPLE( parse_azure_uri(uri_dir, &container_name, &blob_path), nullopt); - auto container_client = client_->GetBlobContainerClient(container_name); + auto container_client = c.GetBlobContainerClient(container_name); std::vector entries; ::Azure::Storage::Blobs::ListBlobsOptions options; @@ -573,14 +565,14 @@ tuple>> Azure::ls_with_sizes( } Status Azure::move_object(const URI& old_uri, const URI& new_uri) { - assert(client_); RETURN_NOT_OK(copy_blob(old_uri, new_uri)); RETURN_NOT_OK(remove_blob(old_uri)); return Status::Ok(); } Status Azure::copy_blob(const URI& old_uri, const URI& new_uri) { - assert(client_); + assert(azure_params_); + auto& c = client(); if (!old_uri.is_azure()) { return LOG_STATUS(Status_AzureError( @@ -595,7 +587,7 @@ Status Azure::copy_blob(const URI& old_uri, const URI& new_uri) { std::string old_container_name; std::string old_blob_path; RETURN_NOT_OK(parse_azure_uri(old_uri, &old_container_name, &old_blob_path)); - std::string source_uri = client_->GetBlobContainerClient(old_container_name) + std::string source_uri = c.GetBlobContainerClient(old_container_name) .GetBlobClient(old_blob_path) .GetUrl(); @@ -604,10 +596,10 @@ Status Azure::copy_blob(const URI& old_uri, const URI& new_uri) { RETURN_NOT_OK(parse_azure_uri(new_uri, &new_container_name, &new_blob_path)); try { - client_->GetBlobContainerClient(new_container_name) + c.GetBlobContainerClient(new_container_name) .GetBlobClient(new_blob_path) .StartCopyFromUri(source_uri) - .PollUntilDone(retry_delay_); + .PollUntilDone(azure_params_->retry_delay_); } catch (const ::Azure::Storage::StorageException& e) { return LOG_STATUS(Status_AzureError( "Copy blob failed on: " + old_uri.to_string() + "; " + e.Message)); @@ -617,8 +609,6 @@ Status Azure::copy_blob(const URI& old_uri, const URI& new_uri) { } Status Azure::move_dir(const URI& old_uri, const URI& new_uri) { - assert(client_); - std::vector paths; RETURN_NOT_OK(ls(old_uri, &paths, "")); for (const auto& path : paths) { @@ -630,7 +620,7 @@ Status Azure::move_dir(const URI& old_uri, const URI& new_uri) { } Status Azure::blob_size(const URI& uri, uint64_t* const nbytes) const { - assert(client_); + auto& c = client(); assert(nbytes); if (!uri.is_azure()) { @@ -649,8 +639,7 @@ Status Azure::blob_size(const URI& uri, uint64_t* const nbytes) const { options.Prefix = blob_path; options.PageSizeHint = 1; - auto response = - client_->GetBlobContainerClient(container_name).ListBlobs(options); + auto response = c.GetBlobContainerClient(container_name).ListBlobs(options); if (response.Blobs.empty()) { error_message = "Blob does not exist."; @@ -676,7 +665,7 @@ Status Azure::read( const uint64_t length, const uint64_t read_ahead_length, uint64_t* const length_returned) const { - assert(client_); + const auto& c = client(); if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( @@ -696,7 +685,7 @@ Status Azure::read( ::Azure::Storage::Blobs::Models::DownloadBlobResult result; try { - result = client_->GetBlobContainerClient(container_name) + result = c.GetBlobContainerClient(container_name) .GetBlobClient(blob_path) .Download(options) .Value; @@ -717,7 +706,7 @@ Status Azure::read( } Status Azure::remove_container(const URI& uri) const { - assert(client_); + auto& c = client(); // Empty container RETURN_NOT_OK(empty_container(uri)); @@ -728,7 +717,7 @@ Status Azure::remove_container(const URI& uri) const { bool deleted; std::string error_message = ""; try { - deleted = client_->DeleteBlobContainer(container_name).Value.Deleted; + deleted = c.DeleteBlobContainer(container_name).Value.Deleted; } catch (const ::Azure::Storage::StorageException& e) { deleted = false; error_message = "; " + e.Message; @@ -743,7 +732,7 @@ Status Azure::remove_container(const URI& uri) const { } Status Azure::remove_blob(const URI& uri) const { - assert(client_); + auto& c = client(); std::string container_name; std::string blob_path; @@ -752,7 +741,7 @@ Status Azure::remove_blob(const URI& uri) const { bool deleted; std::string error_message = ""; try { - deleted = client_->GetBlobContainerClient(container_name) + deleted = c.GetBlobContainerClient(container_name) .DeleteBlob(blob_path) .Value.Deleted; } catch (const ::Azure::Storage::StorageException& e) { @@ -769,8 +758,6 @@ Status Azure::remove_blob(const URI& uri) const { } Status Azure::remove_dir(const URI& uri) const { - assert(client_); - std::vector paths; RETURN_NOT_OK(ls(uri, &paths, "")); auto status = parallel_for(thread_pool_, 0, paths.size(), [&](size_t i) { @@ -783,7 +770,7 @@ Status Azure::remove_dir(const URI& uri) const { } Status Azure::touch(const URI& uri) const { - assert(client_); + auto& c = client(); if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( @@ -806,7 +793,7 @@ Status Azure::touch(const URI& uri) const { RETURN_NOT_OK(parse_azure_uri(uri, &container_name, &blob_path)); try { - client_->GetBlobContainerClient(container_name) + c.GetBlobContainerClient(container_name) .GetBlockBlobClient(blob_path) .UploadFrom(nullptr, 0); } catch (const ::Azure::Storage::StorageException& e) { @@ -819,6 +806,8 @@ Status Azure::touch(const URI& uri) const { Status Azure::write( const URI& uri, const void* const buffer, const uint64_t length) { + assert(azure_params_); + auto write_cache_max_size = azure_params_->write_cache_max_size_; if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( std::string("URI is not an Azure URI: " + uri.to_string()))); @@ -830,7 +819,7 @@ Status Azure::write( RETURN_NOT_OK( fill_write_cache(write_cache_buffer, buffer, length, &nbytes_filled)); - if (!use_block_list_upload_) { + if (!azure_params_->use_block_list_upload_) { if (nbytes_filled != length) { std::stringstream errmsg; errmsg << "Direct write failed! " << nbytes_filled @@ -841,21 +830,21 @@ Status Azure::write( } } - if (write_cache_buffer->size() == write_cache_max_size_) { + if (write_cache_buffer->size() == write_cache_max_size) { RETURN_NOT_OK(flush_write_cache(uri, write_cache_buffer, false)); } uint64_t new_length = length - nbytes_filled; uint64_t offset = nbytes_filled; while (new_length > 0) { - if (new_length >= write_cache_max_size_) { + if (new_length >= write_cache_max_size) { RETURN_NOT_OK(write_blocks( uri, static_cast(buffer) + offset, - write_cache_max_size_, + write_cache_max_size, false)); - offset += write_cache_max_size_; - new_length -= write_cache_max_size_; + offset += write_cache_max_size; + new_length -= write_cache_max_size; } else { RETURN_NOT_OK(fill_write_cache( write_cache_buffer, @@ -886,12 +875,14 @@ Status Azure::fill_write_cache( const void* const buffer, const uint64_t length, uint64_t* const nbytes_filled) { + assert(azure_params_); assert(write_cache_buffer); assert(buffer); assert(nbytes_filled); - *nbytes_filled = - std::min(write_cache_max_size_ - write_cache_buffer->size(), length); + *nbytes_filled = std::min( + azure_params_->write_cache_max_size_ - write_cache_buffer->size(), + length); if (*nbytes_filled > 0) { RETURN_NOT_OK(write_cache_buffer->write(buffer, *nbytes_filled)); @@ -922,6 +913,8 @@ Status Azure::write_blocks( const void* const buffer, const uint64_t length, const bool last_block) { + assert(azure_params_); + auto block_list_block_size = azure_params_->block_list_block_size_; if (!uri.is_azure()) { return LOG_STATUS(Status_AzureError( std::string("URI is not an Azure URI: " + uri.to_string()))); @@ -933,11 +926,12 @@ Status Azure::write_blocks( // configured max number. Length must be evenly divisible by // block_list_block_size_ unless this is the last block. uint64_t num_ops = last_block ? - utils::math::ceil(length, block_list_block_size_) : - (length / block_list_block_size_); - num_ops = std::min(std::max(num_ops, uint64_t(1)), max_parallel_ops_); + utils::math::ceil(length, block_list_block_size) : + (length / block_list_block_size); + num_ops = std::min( + std::max(num_ops, uint64_t(1)), azure_params_->max_parallel_ops_); - if (!last_block && length % block_list_block_size_ != 0) { + if (!last_block && length % block_list_block_size != 0) { return LOG_STATUS( Status_AzureError("Length not evenly divisible by block size")); } @@ -990,9 +984,9 @@ Status Azure::write_blocks( std::vector tasks; tasks.reserve(num_ops); for (uint64_t i = 0; i < num_ops; i++) { - const uint64_t begin = i * block_list_block_size_; + const uint64_t begin = i * block_list_block_size; const uint64_t end = - std::min((i + 1) * block_list_block_size_ - 1, length - 1); + std::min((i + 1) * block_list_block_size - 1, length - 1); const char* const thread_buffer = reinterpret_cast(buffer) + begin; const uint64_t thread_buffer_len = end - begin + 1; @@ -1024,10 +1018,11 @@ Status Azure::upload_block( const void* const buffer, const uint64_t length, const std::string& block_id) { + const auto& c = client(); ::Azure::Core::IO::MemoryBodyStream stream( static_cast(buffer), static_cast(length)); try { - client_->GetBlobContainerClient(container_name) + c.GetBlobContainerClient(container_name) .GetBlockBlobClient(blob_path) .StageBlock(block_id, stream); } catch (const ::Azure::Storage::StorageException& e) { @@ -1125,7 +1120,7 @@ std::string Azure::BlockListUploadState::next_block_id() { #if defined(_WIN32) #include std::shared_ptr<::Azure::Core::Http::HttpTransport> create_transport( - tiledb::sm::SSLConfig& ssl_cfg) { + const tiledb::sm::SSLConfig& ssl_cfg) { ::Azure::Core::Http::WinHttpTransportOptions transport_opts; if (!ssl_cfg.ca_file().empty()) { @@ -1146,7 +1141,7 @@ std::shared_ptr<::Azure::Core::Http::HttpTransport> create_transport( #else #include std::shared_ptr<::Azure::Core::Http::HttpTransport> create_transport( - tiledb::sm::SSLConfig& ssl_cfg) { + const tiledb::sm::SSLConfig& ssl_cfg) { ::Azure::Core::Http::CurlTransportOptions transport_opts; if (!ssl_cfg.ca_file().empty()) { diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index 70792379434d..3d5ca37e6a7d 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -39,6 +39,7 @@ #include "tiledb/common/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" +#include "tiledb/sm/filesystem/ssl_config.h" #include "tiledb/sm/misc/constants.h" #include "uri.h" @@ -63,6 +64,48 @@ class directory_entry; namespace sm { +/** + * The Azure-specific configuration parameters. + */ +struct AzureParameters { + AzureParameters() = delete; + + AzureParameters(const Config& config); + + /** The maximum number of parallel requests. */ + uint64_t max_parallel_ops_; + + /** The target block size in a block list upload */ + uint64_t block_list_block_size_; + + /** The maximum size of each value-element in 'write_cache_map_'. */ + uint64_t write_cache_max_size_; + + /** The maximum number of retries. */ + int max_retries_; + + /** The minimum time to wait between retries. */ + std::chrono::milliseconds retry_delay_; + + /** The maximum time to wait between retries. */ + std::chrono::milliseconds max_retry_delay_; + + /** Whether or not to use block list upload. */ + bool use_block_list_upload_; + + /** The Blob Storage account name. */ + std::string account_name_; + + /** The Blob Storage account key. */ + std::string account_key_; + + /** The Blob Storage endpoint to connect to. */ + std::string blob_endpoint_; + + /** SSL configuration. */ + SSLConfig ssl_cfg_; +}; + class Azure { public: /* ********************************* */ @@ -310,13 +353,14 @@ class Azure { Status write(const URI& uri, const void* buffer, uint64_t length); /** - * Returns a reference to the Azure blob service client. + * Initializes the Azure blob service client and returns a reference to it. * - * Used for testing. Calling code should include the Azure SDK headers to make + * Calling code should include the Azure SDK headers to make * use of the BlobServiceClient. */ const ::Azure::Storage::Blobs::BlobServiceClient& client() const { - return *client_; + assert(azure_params_); + return client_singleton_.get(*azure_params_); } private: @@ -364,6 +408,32 @@ class Azure { Status st_; }; + /** + * Encapsulates access to an Azure BlobServiceClient. + * + * This class ensures that: + * * Callers access the client in an initialized state. + * * The client gets initialized only once even for concurrent accesses. + */ + class AzureClientSingleton { + public: + /** + * Gets a reference to the Azure BlobServiceClient, and initializes it if it + * is not initialized. + * + * @param params The parameters to initialize the client with. + */ + const ::Azure::Storage::Blobs::BlobServiceClient& get( + const AzureParameters& params); + + private: + /** The Azure blob service client. */ + tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient> client_; + + /** Protects from creating the client many times. */ + std::mutex client_init_mtx_; + }; + /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ @@ -371,8 +441,8 @@ class Azure { /** The VFS thread pool. */ ThreadPool* thread_pool_; - /** The Azure blob service client. */ - tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient> client_; + /** A holder for the Azure blob service client. */ + mutable AzureClientSingleton client_singleton_; /** Maps a blob URI to a write cache buffer. */ std::unordered_map write_cache_map_; @@ -380,20 +450,11 @@ class Azure { /** Protects 'write_cache_map_'. */ std::mutex write_cache_map_lock_; - /** The maximum size of each value-element in 'write_cache_map_'. */ - uint64_t write_cache_max_size_; - - /** The maximum number of parallel requests. */ - uint64_t max_parallel_ops_; - - /** The target block size in a block list upload */ - uint64_t block_list_block_size_; - - /** The minimum time to wait between retries. */ - std::chrono::milliseconds retry_delay_; - - /** Whether or not to use block list upload. */ - bool use_block_list_upload_; + /** + * Contains options to configure connection to Azure. + * After the class becomes C.41 compliant, remove the std::optional. + */ + std::optional azure_params_; /** Maps a blob URI to its block list upload state. */ std::unordered_map @@ -467,7 +528,6 @@ class Azure { * @param length The length of `buffer`. * @param block_id A base64-encoded string that is unique to this block * within the blob. - * @param result The returned future to fetch the async upload result from. * @return Status */ Status upload_block( From 5509b278f9f5b87770bcebf14def84447af8daa9 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 13 Feb 2024 11:55:02 -0600 Subject: [PATCH 187/456] Add the query_memory_tracker (#4721) This allows easy access for readers and writers to the query memory tracker. Originally written as part of the Tile instrumentation PR, this is just an extraction so that it can unblock other work in adding other measurements. --- TYPE: NO_HISTORY DESC: Add the query_memory_tracker --- test/src/unit-Reader.cc | 6 +++++- tiledb/sm/query/query.cc | 1 + tiledb/sm/query/strategy_base.cc | 1 + tiledb/sm/query/strategy_base.h | 14 +++++++++++++- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index b28c0f6e024d..26d13cc93f6f 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -69,12 +69,15 @@ struct ReaderFx { const char* ARRAY_NAME = "reader"; tiledb_array_t* array_ = nullptr; + shared_ptr tracker_; + ReaderFx(); ~ReaderFx(); }; ReaderFx::ReaderFx() - : fs_vec_(vfs_test_get_fs_vec()) { + : fs_vec_(vfs_test_get_fs_vec()) + , tracker_(tiledb::test::create_test_memory_tracker()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); @@ -166,6 +169,7 @@ TEST_CASE_METHOD( DefaultChannelAggregates default_channel_aggregates; auto params = StrategyParams( array.memory_tracker(), + tracker_, context.storage_manager(), array.opened_array(), config, diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 56dc8b86c159..9e3880646dcb 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -1807,6 +1807,7 @@ bool Query::is_aggregate(std::string output_field_name) const { Status Query::create_strategy(bool skip_checks_serialization) { auto params = StrategyParams( array_->memory_tracker(), + query_memory_tracker_, storage_manager_, opened_array_, config_, diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index a0e3b504f645..91efeee59487 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -48,6 +48,7 @@ namespace sm { StrategyBase::StrategyBase( stats::Stats* stats, shared_ptr logger, StrategyParams& params) : array_memory_tracker_(params.array_memory_tracker()) + , query_memory_tracker_(params.query_memory_tracker()) , stats_(stats) , logger_(logger) , array_(params.array()) diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index db1119597ab5..d1d157c10c5d 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -67,6 +67,7 @@ class StrategyParams { StrategyParams( shared_ptr array_memory_tracker, + shared_ptr query_memory_tracker, StorageManager* storage_manager, shared_ptr array, Config& config, @@ -78,6 +79,7 @@ class StrategyParams { DefaultChannelAggregates& default_channel_aggregates, bool skip_checks_serialization) : array_memory_tracker_(array_memory_tracker) + , query_memory_tracker_(query_memory_tracker) , storage_manager_(storage_manager) , array_(array) , config_(config) @@ -99,6 +101,10 @@ class StrategyParams { return array_memory_tracker_; } + inline shared_ptr query_memory_tracker() { + return query_memory_tracker_; + } + /** Return the storage manager. */ inline StorageManager* storage_manager() { return storage_manager_; @@ -157,6 +163,9 @@ class StrategyParams { /** Array Memory tracker. */ shared_ptr array_memory_tracker_; + /** Query Memory tracker. */ + shared_ptr query_memory_tracker_; + /** Storage manager. */ StorageManager* storage_manager_; @@ -242,9 +251,12 @@ class StrategyBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ - /** The memory tracker. */ + /** The array memory tracker. */ shared_ptr array_memory_tracker_; + /** The query memory tracker. */ + shared_ptr query_memory_tracker_; + /** The class stats. */ stats::Stats* stats_; From b4de1c3c8d5b3f0c63bd896847af8c9a09a15da0 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 13 Feb 2024 23:10:22 -0500 Subject: [PATCH 188/456] Don't run Build-Release workflow on every PR, reduce CI overhead (#4726) --- DESC: Don't run Build-Release workflow on every PR, reduce CI overhead TYPE: NO_HISTORY --- .github/workflows/release.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c373fd607a8..956600930c58 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,22 +1,13 @@ name: Release on: - pull_request: - branches: - - '*' # must quote since "*" is a YAML reserved character; we want a string - paths-ignore: - - '.github/workflows/quarto-render.yml' - - '_quarto.yml' - - 'quarto-materials/*' - - '**/.md' - - 'tiledb/doxygen/source/*' - - 'tiledb/sm/c_api/tiledb_version.h' push: branches: - dev - 'release-*' tags: - '*' + workflow_dispatch: jobs: Build-Release: @@ -58,7 +49,7 @@ jobs: uses: actions/checkout@v3 - name: 'Homebrew setup' run: brew install automake pkg-config - if: ${{ startsWith(matrix.os, 'macos-') == true }} + if: ${{ startsWith(matrix.os, 'macos-') == true }} - name: Export GitHub Actions cache variables uses: actions/github-script@v6 with: From e11fe82e3ff17da4454522b782b0450c64c6e7b5 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 14 Feb 2024 00:25:44 -0600 Subject: [PATCH 189/456] Fix bug with new minio behavior (#4725) New behavior doesn't delete masked objects. So check after the first deletion if we need to remove anything that was uncovered. --- TYPE: BUG DESC: Fix bug with new minio behavior --- tiledb/sm/filesystem/s3.cc | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index b0433752cf57..d1d13b5e72d8 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -443,6 +443,33 @@ void S3::remove_dir(const URI& uri) const { std::vector paths; throw_if_not_ok(ls(uri, &paths, "")); + + // Bail early if we don't have anything to delete. + if (paths.empty()) { + return; + } + + throw_if_not_ok( + parallel_for(vfs_thread_pool_, 0, paths.size(), [&](size_t i) { + throw_if_not_ok(remove_object(URI(paths[i]))); + return Status::Ok(); + })); + + // Minio changed their delete behavior when an object masks another object + // with the same prefix. Previously, minio would delete any object with + // a matching prefix. The new behavior is to only delete the object masking + // the "directory" of objects below. To handle this we just run a second + // ls to see if we still have paths to remove, and remove them if so. + + paths.clear(); + throw_if_not_ok(ls(uri, &paths, "")); + + // We got everything on the first pass. + if (paths.empty()) { + return; + } + + // Delete the uncovered object prefixes. throw_if_not_ok( parallel_for(vfs_thread_pool_, 0, paths.size(), [&](size_t i) { throw_if_not_ok(remove_object(URI(paths[i]))); From c49c13b6bb52c5fc676c02fbcb49d0d7155c33c1 Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Thu, 15 Feb 2024 02:23:06 -0500 Subject: [PATCH 190/456] Move tests from tiledb_unit to unit_run_filter_pipeline (#4727) * Update unit tests to use new testing infrastructure. * Remove unneeded test infrastructure from tiledb_unit. --- TYPE: NO_HISTORY Co-authored-by: Julia Dark <> --- test/src/unit-filter-pipeline.cc | 483 ------------------ .../filter/test/unit_run_filter_pipeline.cc | 99 +++- 2 files changed, 98 insertions(+), 484 deletions(-) diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index 6e20ce2cb97d..107385645c5e 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -136,489 +136,6 @@ void run_reverse( .ok()); } -/** - * Simple filter that modifies the input stream by adding 1 to every input - * element. - */ -class Add1InPlace : public tiledb::sm::Filter { - public: - // Just use a dummy filter type - Add1InPlace(Datatype filter_data_type) - : Filter(FilterType::FILTER_NONE, filter_data_type) { - } - - void dump(FILE* out) const override { - (void)out; - } - - Status run_forward( - const WriterTile&, - WriterTile* const, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output) const override { - auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); - output->reset_offset(); - - uint64_t nelts = input_size / sizeof(uint64_t); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t* val = output->value_ptr(); - *val += 1; - output->advance_offset(sizeof(uint64_t)); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); - } - - Status run_reverse( - const Tile&, - Tile*, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output, - const tiledb::sm::Config& config) const override { - (void)config; - - auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); - output->reset_offset(); - - uint64_t nelts = input_size / sizeof(uint64_t); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t* val = output->value_ptr(); - *val -= 1; - output->advance_offset(sizeof(uint64_t)); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); - } - - Add1InPlace* clone_impl() const override { - return new Add1InPlace(filter_data_type_); - } -}; - -/** - * Simple filter that increments every element of the input stream, writing the - * output to a new buffer. Does not modify the input stream. - */ -class Add1OutOfPlace : public tiledb::sm::Filter { - public: - // Just use a dummy filter type - Add1OutOfPlace(Datatype filter_data_type) - : Filter(FilterType::FILTER_NONE, filter_data_type) { - } - - void dump(FILE* out) const override { - (void)out; - } - - Status run_forward( - const WriterTile&, - WriterTile* const, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output) const override { - auto input_size = input->size(); - auto nelts = input_size / sizeof(uint64_t); - - // Add a new output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size)); - output->reset_offset(); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); - inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); - } - - // Finish any remaining bytes to ensure no data loss. - auto rem = input_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); - } - - Status run_reverse( - const Tile&, - Tile*, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output, - const tiledb::sm::Config& config) const override { - (void)config; - - auto input_size = input->size(); - auto nelts = input->size() / sizeof(uint64_t); - - // Add a new output buffer. - RETURN_NOT_OK(output->prepend_buffer(input->size())); - output->reset_offset(); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); - inc--; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); - } - - auto rem = input_size % sizeof(uint64_t); - for (unsigned i = 0; i < rem; i++) { - char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); - } - - // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); - } - - Add1OutOfPlace* clone_impl() const override { - return new Add1OutOfPlace(filter_data_type_); - } -}; - -/** - * Simple filter which computes the sum of its input and prepends the sum - * to the output. In reverse execute, checks that the sum is correct. - */ -class PseudoChecksumFilter : public tiledb::sm::Filter { - public: - // Just use a dummy filter type - PseudoChecksumFilter(Datatype filter_data_type) - : Filter(FilterType::FILTER_NONE, filter_data_type) { - } - - void dump(FILE* out) const override { - (void)out; - } - - Status run_forward( - const WriterTile&, - WriterTile* const, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output) const override { - auto input_size = input->size(); - auto nelts = input_size / sizeof(uint64_t); - - // The input is unmodified by this filter. - RETURN_NOT_OK(output->append_view(input)); - - // Forward the existing metadata and prepend a metadata buffer for the - // checksum. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(sizeof(uint64_t))); - output_metadata->reset_offset(); - - uint64_t sum = 0; - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val; - RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); - sum += val; - } - - RETURN_NOT_OK(output_metadata->write(&sum, sizeof(uint64_t))); - - return Status::Ok(); - } - - Status run_reverse( - const Tile&, - Tile*, - FilterBuffer* input_metadata, - FilterBuffer* input, - FilterBuffer* output_metadata, - FilterBuffer* output, - const tiledb::sm::Config& config) const override { - (void)config; - - auto input_size = input->size(); - auto nelts = input_size / sizeof(uint64_t); - - uint64_t input_sum; - RETURN_NOT_OK(input_metadata->read(&input_sum, sizeof(uint64_t))); - - uint64_t sum = 0; - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val; - RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); - sum += val; - } - - if (sum != input_sum) - return Status_FilterError("Filter error; sum does not match."); - - // The output metadata is just a view on the input metadata, skipping the - // checksum bytes. - RETURN_NOT_OK(output_metadata->append_view( - input_metadata, - sizeof(uint64_t), - input_metadata->size() - sizeof(uint64_t))); - - // The output data is just a view on the unmodified input. - RETURN_NOT_OK(output->append_view(input)); - - return Status::Ok(); - } - - PseudoChecksumFilter* clone_impl() const override { - return new PseudoChecksumFilter(filter_data_type_); - } -}; - -TEST_CASE("Filter: Test compression", "[filter][compression]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up dummy array schema (needed by compressor filter for cell size, etc). - uint32_t dim_dom[] = {1, 10}; - auto dim{make_shared(HERE(), "", Datatype::INT32)}; - CHECK(dim->set_domain(dim_dom).ok()); - auto domain{make_shared(HERE())}; - CHECK(domain->add_dimension(dim).ok()); - tiledb::sm::ArraySchema schema; - tiledb::sm::Attribute attr("attr", Datatype::UINT64); - CHECK(schema.add_attribute(make_shared(HERE(), attr)) - .ok()); - CHECK(schema.set_domain(domain).ok()); - - FilterPipeline pipeline; - ThreadPool tp(4); - - SECTION("- Simple") { - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With checksum stage") { - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With multiple stages") { - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - // Check compression worked - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { - tiledb::sm::Config config; - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets); - - // Set up dummy array schema (needed by compressor filter for cell size, etc). - uint32_t dim_dom[] = {1, 10}; - auto dim{make_shared(HERE(), "", Datatype::INT32)}; - CHECK(dim->set_domain(dim_dom).ok()); - auto domain{make_shared(HERE())}; - CHECK(domain->add_dimension(dim).ok()); - tiledb::sm::ArraySchema schema; - tiledb::sm::Attribute attr("attr", Datatype::UINT64); - CHECK(schema.add_attribute(make_shared(HERE(), attr)) - .ok()); - CHECK(schema.set_domain(domain).ok()); - - FilterPipeline pipeline; - ThreadPool tp(4); - - SECTION("- Simple") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With checksum stage") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- With multiple stages") { - WriterTile::set_max_tile_chunk_size(80); - pipeline.add_filter(Add1InPlace(Datatype::UINT64)); - pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); - pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - // Check number of chunks - CHECK(tile.size() == 0); - CHECK( - tile.filtered_buffer().value_at_as(0) == - 9); // Number of chunks - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline); - - // Check all elements original values. - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - TEST_CASE( "Filter: Test skip checksum validation", "[filter][skip-checksum-validation]") { diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index c068d61c8334..39b3ac171850 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -220,7 +220,7 @@ void check_run_pipeline_roundtrip( WriterTile& tile, std::optional& offsets_tile, FilterPipeline& pipeline, - TileDataGenerator* test_data) { + const TileDataGenerator* test_data) { // Run the pipeline forward. CHECK(pipeline .run_forward( @@ -983,3 +983,100 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { config, tp, tile, offsets_tile, pipeline, &tile_data_generator); } } + +TEST_CASE("Filter: Test compression", "[filter][compression]") { + // Create resources for running pipeline tests. + Config config; + ThreadPool tp(4); + FilterPipeline pipeline; + + // Set-up test data. + IncrementTileDataGenerator tile_data_generator( + 100); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + + SECTION("- Simple") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } + + SECTION("- With checksum stage") { + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } + + SECTION("- With multiple stages") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } +} + +TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { + // Create TileDB resources for running the filter pipeline. + Config config; + ThreadPool tp(4); + + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + + FilterPipeline pipeline; + + SECTION("- Simple") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } + + SECTION("- With checksum stage") { + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } + + SECTION("- With multiple stages") { + pipeline.add_filter(Add1InPlace(Datatype::UINT64)); + pipeline.add_filter(PseudoChecksumFilter(Datatype::UINT64)); + pipeline.add_filter(Add1OutOfPlace(Datatype::UINT64)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, Datatype::UINT64)); + + // Check the pipelines run forward and backward without error and returns + // the input data. + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + } +} From 2f96db64b35d8c889056a7894e4ffc286b303073 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 15 Feb 2024 12:48:27 +0100 Subject: [PATCH 191/456] Update dev HISTORY with 2.20.0 content. (#4728) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.20.0 content. --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 91ea067476ee..1eb09d8accd7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -18,7 +18,6 @@ * Fix consolidation plan to print relative paths in output. [#4604](https://github.com/TileDB-Inc/TileDB/pull/4604) * Fix traversal limit in array deserialization. [#4606](https://github.com/TileDB-Inc/TileDB/pull/4606) * Add function random_label to utilize PRNG for random string generation. [#4564](https://github.com/TileDB-Inc/TileDB/pull/4564) -* Remove uuid in favor of random_label. [#4589](https://github.com/TileDB-Inc/TileDB/pull/4589) * Improve large dense aggregate reads with tile metadata only. [#4657](https://github.com/TileDB-Inc/TileDB/pull/4657) ## Defects removed @@ -43,6 +42,7 @@ * Add packaging tests into linux and mac CI pipelines. [#4567](https://github.com/TileDB-Inc/TileDB/pull/4567) * Add vcpkg triplets for Address Sanitizer. [#4515](https://github.com/TileDB-Inc/TileDB/pull/4515) * Fix regression where release artifacts had 8-digit commit hashes. [#4599](https://github.com/TileDB-Inc/TileDB/pull/4599) +* Fix importing TileDB in CMake versions prior to 3.18. [#4671](https://github.com/TileDB-Inc/TileDB/pull/4671) # TileDB v2.19.1 Release Notes From fbf244ac792bcba007cb9f3cfe91a65aeaaa1acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Fri, 16 Feb 2024 00:36:51 +0100 Subject: [PATCH 192/456] Create issue if release workflow fails (#4729) This PR adds functionality to create issue when release workflow fails. This is how the issue will look: https://github.com/dudoslav/TileDB/issues/1 --- TYPE: NO_HISTORY DESC: Create issue if release workflow fails --------- Co-authored-by: Theodore Tsirpanis --- .github/workflows/release.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 956600930c58..dbabae23bc53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -170,3 +170,17 @@ jobs: data: fs.readFileSync(file) }); } + + Create-Issue-On-Fail: + permissions: + issues: write + runs-on: ubuntu-latest + needs: Publish-Release + if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch' + steps: + - name: Create Issue if Build Fails + uses: TileDB-Inc/github-actions/open-issue@main + with: + name: Release failed + label: bug + assignee: KiterLuc,teo-tsirpanis,davisp \ No newline at end of file From 32401e344a35e29dde6dc23f591d843c85ab8c10 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 15 Feb 2024 18:45:51 -0600 Subject: [PATCH 193/456] Replace boost::container with cpp17::pmr (#4731) This removes the boost dependency to avoid having to make that work in our builds that are still using external packages. This change uses std::pmr when it is detected as available and usable. On platforms that don't have a working std::pmr (notably, macOS < 14), it uses the implementation from Pablo Halpern that is implemented with purely C++11 features. The major downfall to this approach is that if/when we start changing allocator strategies it'll require us to backport/reimplement those allocator strategies for macOS < 14. However, if we end up writing our own allocation strategies they'll work on either implementation seamlessly. --- TYPE: NO_HISTORY DESC: Remove boost::container --- CMakeLists.txt | 9 + cmake/Modules/DetectStdPmr.cmake | 69 +++ cmake/TileDB-Superbuild.cmake | 2 +- cmake/inputs/detect_std_pmr.cc | 7 + tiledb/common/CMakeLists.txt | 5 +- tiledb/common/pmr.cc | 6 +- tiledb/common/pmr.h | 32 +- tiledb/common/polymorphic_allocator/README.md | 15 + .../polymorphic_allocator.cc | 24 + .../polymorphic_allocator.h | 494 ++++++++++++++++++ tiledb/sm/filter/test/tile_data_generator.h | 1 + vcpkg.json | 1 - 12 files changed, 651 insertions(+), 14 deletions(-) create mode 100644 cmake/Modules/DetectStdPmr.cmake create mode 100644 cmake/inputs/detect_std_pmr.cc create mode 100644 tiledb/common/polymorphic_allocator/README.md create mode 100644 tiledb/common/polymorphic_allocator/polymorphic_allocator.cc create mode 100644 tiledb/common/polymorphic_allocator/polymorphic_allocator.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b10402b214f..2d643c3ac7b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,15 @@ if(TILEDB_SANITIZER) validate_sanitizer_options() endif() +include(DetectStdPmr) + +if(TILEDB_USE_CPP17_PMR) + message(STATUS "Building with cpp17::pmr") + add_definitions(-DUSE_CPP17_PMR) +else() + message(STATUS "Building with std::pmr") +endif() + ####################################################### # Header Files ####################################################### diff --git a/cmake/Modules/DetectStdPmr.cmake b/cmake/Modules/DetectStdPmr.cmake new file mode 100644 index 000000000000..3f96e9435490 --- /dev/null +++ b/cmake/Modules/DetectStdPmr.cmake @@ -0,0 +1,69 @@ +# +# DetectStdPmr.cmake +# +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Detect whether polymorphic allocators are available on the system. + +# Special case for macOS when the MACOSX_DEPLOYMENT_TARGET is set to anything +# less than 14. For some reason, std::pmr is still detectable, but the resulting +# binary dies with a dyld missing symbol error. + +if (ENV{MACOSX_DEPLOYMENT_TARGET}) + string(COMPARE LESS "$ENV{MACOSX_DEPLOYMENT_TARGET}" "14" MACOS_BAD_PMR_SUPPORT) + if (MACOS_BAD_PMR_SUPPORT) + set(TILEDB_USE_CPP17_PMR ON) + message(STATUS "Using vendored cpp17::pmr for polymorphic allocators") + return() + endif() +endif() + +# Otherwise, if we're not building a targeted macOS version, we just detect +# whether std::pmr is available. +# +# However CMake makes this extra awesome because try_run appears to have +# changed in a backwards compatible manner. We'll just version check for +# selecting which to run. + +if (CMAKE_VERSION VERSION_LESS "3.25") + try_run( + TILEDB_CAN_RUN_STD_PMR + TILEDB_CAN_COMPILE_STD_PMR + "${CMAKE_CURRENT_BINARY_DIR}" + "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" + ) +else() + try_run( + TILEDB_CAN_RUN_STD_PMR + TILEDB_CAN_COMPILE_STD_PMR + SOURCES "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" + ) +endif() + +if ("${TILEDB_CAN_COMPILE_STD_PMR}" AND "${TILEDB_CAN_RUN_STD_PMR}" EQUAL 0) + message(STATUS "Using std::pmr for polymorphic allocators") +else() + set(TILEDB_USE_CPP17_PMR ON) + message(STATUS "Using vendored cpp17::pmr for polymorphic allocators") +endif() diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 1a09e997439c..cd1b81f9307d 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -197,4 +197,4 @@ add_custom_target(package DEPENDS tiledb COMMAND ${CMAKE_CPACK_COMMAND} --config CPackConfig.cmake -G "$,ZIP,TGZ>" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tiledb -) \ No newline at end of file +) diff --git a/cmake/inputs/detect_std_pmr.cc b/cmake/inputs/detect_std_pmr.cc new file mode 100644 index 000000000000..0d6b0eb93dd6 --- /dev/null +++ b/cmake/inputs/detect_std_pmr.cc @@ -0,0 +1,7 @@ + +#include + +int +main() { + auto resource = std::pmr::get_default_resource(); +} diff --git a/tiledb/common/CMakeLists.txt b/tiledb/common/CMakeLists.txt index 92ba0d1d2a03..3c8fbfb987b5 100644 --- a/tiledb/common/CMakeLists.txt +++ b/tiledb/common/CMakeLists.txt @@ -65,10 +65,11 @@ commence(object_library baseline) memory_tracker.cc pmr.cc ) + if (TILEDB_USE_CPP17_PMR) + this_target_sources(polymorphic_allocator/polymorphic_allocator.cc) + endif() find_package(Spdlog_EP REQUIRED) - find_package(Boost REQUIRED COMPONENTS container) target_link_libraries(baseline PUBLIC spdlog::spdlog) - target_link_libraries(baseline PUBLIC Boost::container) target_link_libraries(baseline PUBLIC common) conclude(object_library) diff --git a/tiledb/common/pmr.cc b/tiledb/common/pmr.cc index 92da739a6066..f37db73927a2 100644 --- a/tiledb/common/pmr.cc +++ b/tiledb/common/pmr.cc @@ -35,7 +35,11 @@ namespace tiledb::common::pmr { memory_resource* get_default_resource() { - return boost::container::pmr::get_default_resource(); +#ifdef USE_CPP17_PMR + return cpp17::pmr::get_default_resource(); +#else + return std::pmr::get_default_resource(); +#endif } } // namespace tiledb::common::pmr diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index aa36488121c2..3f12802d31d1 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -37,25 +37,39 @@ #include #include -#include -#include -#include +#ifdef USE_CPP17_PMR +#include "polymorphic_allocator/polymorphic_allocator.h" +#else +#include +#endif #include "common.h" namespace tiledb::common::pmr { -using memory_resource = boost::container::pmr::memory_resource; +#ifdef USE_CPP17_PMR + +using memory_resource = cpp17::pmr::memory_resource; + +template +using polymorphic_allocator = cpp17::pmr::polymorphic_allocator; + +#else + +using memory_resource = std::pmr::memory_resource; + +template +using polymorphic_allocator = std::pmr::polymorphic_allocator; +#endif + +memory_resource* get_default_resource(); /* ********************************* */ /* PMR VECTOR DECLARATION */ /* ********************************* */ template -using pmr_vector = - std::vector>; - -memory_resource* get_default_resource(); +using pmr_vector = std::vector>; template class vector : public pmr_vector { @@ -141,7 +155,7 @@ using pmr_unordered_map = std::unordered_map< T, Hash, KeyEqual, - boost::container::pmr::polymorphic_allocator>>; + polymorphic_allocator>>; template < class Key, diff --git a/tiledb/common/polymorphic_allocator/README.md b/tiledb/common/polymorphic_allocator/README.md new file mode 100644 index 000000000000..510c0f207a31 --- /dev/null +++ b/tiledb/common/polymorphic_allocator/README.md @@ -0,0 +1,15 @@ +Polymorphic Allocator Fallback Implementation +=== + +This implementation of polymorphic_allocator was pulled from Pablo Halpern's +C++11 implementation available here: + +https://github.com/phalpern/CppCon2017Code/tree/d26e7f4f6c593fe135c6b454aee93486790726b7 + +I have a personal forked copy here in case that repository ever dissappears: + +https://github.com/davisp/phalpern-CppCon2017Code/tree/d26e7f4f6c593fe135c6b454aee93486790726b7 + +The only changes from the original files in that repository are to reformat +using TileDB coding style and adding a handful of `std::` namespace qualifiers +to avoid symbol name clashes with internal TileDB symbols. diff --git a/tiledb/common/polymorphic_allocator/polymorphic_allocator.cc b/tiledb/common/polymorphic_allocator/polymorphic_allocator.cc new file mode 100644 index 000000000000..fe031cc482f1 --- /dev/null +++ b/tiledb/common/polymorphic_allocator/polymorphic_allocator.cc @@ -0,0 +1,24 @@ +/* polymorphic_allocator.cpp -*-C++-*- + * + * Copyright 2012 Pablo Halpern. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + */ + +#include "polymorphic_allocator.h" + +namespace cpp17 { + +atomic pmr::memory_resource::s_default_resource(nullptr); + +pmr::new_delete_resource* pmr::new_delete_resource_singleton() { + // TBD: I think the standard makes this exception-safe, otherwise, we need + // to use 'call_once()' in ''. + static new_delete_resource singleton; + return &singleton; +} + +} // namespace cpp17 + +// end polymorphic_allocator.cpp diff --git a/tiledb/common/polymorphic_allocator/polymorphic_allocator.h b/tiledb/common/polymorphic_allocator/polymorphic_allocator.h new file mode 100644 index 000000000000..654effacb1a2 --- /dev/null +++ b/tiledb/common/polymorphic_allocator/polymorphic_allocator.h @@ -0,0 +1,494 @@ +/* polymorphic_allocator.h -*-C++-*- + * + * Copyright 2017 Pablo Halpern. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + */ + +#ifndef INCLUDED_POLYMORPHIC_ALLOCATOR_DOT_H +#define INCLUDED_POLYMORPHIC_ALLOCATOR_DOT_H + +#include +#include // For max_align_t +#include +#include +#include + +namespace cpp17 { + +using namespace std; + +// The `byte` type is defined exactly this way in C++17's `` (section +// [cstddef.syn]). It is defined here to allow use of +// `polymorphic_allocator` as a vocabulary type. +enum class byte : unsigned char {}; + +namespace pmr { + +// Abstract base class for allocator resources. +// Conforms to the C++17 standard, section [mem.res.class]. +class memory_resource { + static constexpr size_t max_align = alignof(max_align_t); + + static atomic s_default_resource; + + friend memory_resource* set_default_resource(memory_resource*); + friend memory_resource* get_default_resource(); + + public: + virtual ~memory_resource(); + + void* allocate(size_t bytes, size_t alignment = max_align) { + return do_allocate(bytes, alignment); + } + void deallocate(void* p, size_t bytes, size_t alignment = max_align) { + return do_deallocate(p, bytes, alignment); + } + + // `is_equal` is needed because polymorphic allocators are sometimes + // produced as a result of type erasure. In that case, two different + // instances of a polymorphic_memory_resource may actually represent + // the same underlying allocator and should compare equal, even though + // their addresses are different. + bool is_equal(const memory_resource& other) const noexcept { + return do_is_equal(other); + } + + protected: + virtual void* do_allocate(size_t bytes, size_t alignment) = 0; + virtual void do_deallocate(void* p, size_t bytes, size_t alignment) = 0; + virtual bool do_is_equal(const memory_resource& other) const noexcept = 0; +}; + +inline bool operator==(const memory_resource& a, const memory_resource& b) { + // Call `is_equal` rather than using address comparisons because some + // polymorphic allocators are produced as a result of type erasure. In + // that case, `a` and `b` may contain `memory_resource`s with different + // addresses which, nevertheless, should compare equal. + return &a == &b || a.is_equal(b); +} + +inline bool operator!=(const memory_resource& a, const memory_resource& b) { + return !(a == b); +} + +namespace __details { + +// STL allocator that holds a pointer to a polymorphic allocator resource. +// Used to implement `polymorphic_allocator`, which is a scoped allocator. +template +class polymorphic_allocator_imp { + memory_resource* m_resource; + + public: + using value_type = Tp; + + // These types are old-fashioned, pre-C++11 requirements, still needed by + // g++'s `basic_string` implementation. + using size_type = size_t; + using difference_type = ptrdiff_t; + using reference = Tp&; + using const_reference = Tp const&; + using pointer = Tp*; + using const_pointer = Tp const*; + + polymorphic_allocator_imp(); + polymorphic_allocator_imp(memory_resource* r); + + template + polymorphic_allocator_imp(const polymorphic_allocator_imp& other); + + Tp* allocate(size_t n); + void deallocate(Tp* p, size_t n); + + // Return a default-constructed allocator + polymorphic_allocator_imp select_on_container_copy_construction() const; + + memory_resource* resource() const; +}; + +template +bool operator==( + const polymorphic_allocator_imp& a, + const polymorphic_allocator_imp& b); + +template +bool operator!=( + const polymorphic_allocator_imp& a, + const polymorphic_allocator_imp& b); + +template +struct aligned_chunk; + +template <> +struct aligned_chunk<1> { + char x; +}; +template <> +struct aligned_chunk<2> { + short x; +}; +template <> +struct aligned_chunk<4> { + int x; +}; +template <> +struct aligned_chunk<8> { + long long x; +}; +template <> +struct aligned_chunk<16> { + __attribute__((aligned(16))) char x; +}; +template <> +struct aligned_chunk<32> { + __attribute__((aligned(32))) char x; +}; +template <> +struct aligned_chunk<64> { + __attribute__((aligned(64))) char x; +}; + +// Adaptor to make a polymorphic allocator resource type from an STL allocator +// type. This is really a C++20 feature, but it's useful for implementing +// this component. +template +class resource_adaptor_imp : public memory_resource { + typename allocator_traits::template rebind_alloc + m_alloc; + + template + void* allocate_imp(size_t bytes); + + template + void deallocate_imp(void* p, size_t bytes); + + public: + typedef Allocator allocator_type; + + resource_adaptor_imp() = default; + + resource_adaptor_imp(const resource_adaptor_imp&) = default; + + template + resource_adaptor_imp( + Allocator2&& a2, + typename enable_if::value, int>:: + type = 0); + + protected: + void* do_allocate(size_t bytes, size_t alignment = 0) override; + void do_deallocate(void* p, size_t bytes, size_t alignment = 0) override; + + bool do_is_equal(const memory_resource& other) const noexcept override; + + allocator_type get_allocator() const { + return m_alloc; + } +}; + +} // end namespace __details + +// A resource_adaptor converts a traditional STL allocator to a polymorphic +// memory resource. Somehow, this didn't make it into C++17, but it should +// have, so here it is. +// This alias ensures that `resource_adaptor` and +// `resource_adaptor` are always the same type, whether or not +// `T` and `U` are the same type. +template +using resource_adaptor = __details::resource_adaptor_imp< + typename allocator_traits::template rebind_alloc>; + +// Memory resource that uses new and delete. +class new_delete_resource : public resource_adaptor> {}; + +// Return a pointer to a global instance of `new_delete_resource`. +new_delete_resource* new_delete_resource_singleton(); + +// Get the current default resource +memory_resource* get_default_resource(); + +// Set the default resource +memory_resource* set_default_resource(memory_resource* r); + +template +class polymorphic_allocator : public scoped_allocator_adaptor< + __details::polymorphic_allocator_imp> { + typedef __details::polymorphic_allocator_imp Imp; + typedef scoped_allocator_adaptor Base; + + public: + // g++-4.6.3 does not use allocator_traits in shared_ptr, so we have to + // provide an explicit rebind. + template + struct rebind { + typedef polymorphic_allocator other; + }; + + polymorphic_allocator() = default; + polymorphic_allocator(memory_resource* r) + : Base(Imp(r)) { + } + + template + polymorphic_allocator(const polymorphic_allocator& other) + : Base(Imp((other.resource()))) { + } + + template + polymorphic_allocator(const __details::polymorphic_allocator_imp& other) + : Base(other) { + } + + // Return a default-constructed allocator + polymorphic_allocator select_on_container_copy_construction() const { + return polymorphic_allocator(); + } + + memory_resource* resource() const { + return this->outer_allocator().resource(); + } +}; + +template +inline bool operator==( + const polymorphic_allocator& a, const polymorphic_allocator& b) { + return a.outer_allocator() == b.outer_allocator(); +} + +template +inline bool operator!=( + const polymorphic_allocator& a, const polymorphic_allocator& b) { + return !(a == b); +} + +} // end namespace pmr + +/////////////////////////////////////////////////////////////////////////////// +// INLINE AND TEMPLATE FUNCTION IMPLEMENTATIONS +/////////////////////////////////////////////////////////////////////////////// + +inline pmr::memory_resource::~memory_resource() { +} + +inline pmr::memory_resource* pmr::get_default_resource() { + memory_resource* ret = pmr::memory_resource::s_default_resource.load(); + if (nullptr == ret) + ret = new_delete_resource_singleton(); + return ret; +} + +inline pmr::memory_resource* pmr::set_default_resource( + pmr::memory_resource* r) { + if (nullptr == r) + r = new_delete_resource_singleton(); + + // TBD, should use an atomic swap + pmr::memory_resource* prev = get_default_resource(); + pmr::memory_resource::s_default_resource.store(r); + return prev; +} + +template +template +inline pmr::__details::resource_adaptor_imp::resource_adaptor_imp( + Allocator2&& a2, + typename enable_if::value, int>::type) + : m_alloc(forward(a2)) { +} + +template +template +void* pmr::__details::resource_adaptor_imp::allocate_imp( + size_t bytes) { + typedef __details::aligned_chunk chunk; + size_t chunks = (bytes + Align - 1) / Align; + + typedef typename allocator_traits::template rebind_traits + chunk_traits; + typename chunk_traits::allocator_type rebound(m_alloc); + return chunk_traits::allocate(rebound, chunks); +} + +template +template +void pmr::__details::resource_adaptor_imp::deallocate_imp( + void* p, size_t bytes) { + typedef __details::aligned_chunk chunk; + size_t chunks = (bytes + Align - 1) / Align; + + typedef typename allocator_traits::template rebind_traits + chunk_traits; + typename chunk_traits::allocator_type rebound(m_alloc); + return chunk_traits::deallocate(rebound, static_cast(p), chunks); +} + +template +void* pmr::__details::resource_adaptor_imp::do_allocate( + size_t bytes, size_t alignment) { + static const size_t max_natural_alignment = sizeof(max_align_t); + + if (0 == alignment) { + // Choose natural alignment for `bytes` + alignment = ((bytes ^ (bytes - 1)) >> 1) + 1; + if (alignment > max_natural_alignment) + alignment = max_natural_alignment; + } + + switch (alignment) { + case 1: + return allocate_imp<1>(bytes); + case 2: + return allocate_imp<2>(bytes); + case 4: + return allocate_imp<4>(bytes); + case 8: + return allocate_imp<8>(bytes); + case 16: + return allocate_imp<16>(bytes); + case 32: + return allocate_imp<32>(bytes); + case 64: + return allocate_imp<64>(bytes); + default: { + size_t chunks = (bytes + sizeof(void*) + alignment - 1) / 64; + size_t chunkbytes = chunks * 64; + void* original = allocate_imp<64>(chunkbytes); + + // Make room for original pointer storage + char* p = static_cast(original) + sizeof(void*); + + // Round up to nearest alignment boundary + p += alignment - 1; + p -= (size_t(p)) & (alignment - 1); + + // Store original pointer in word before allocated pointer + reinterpret_cast(p)[-1] = original; + + return p; + } + } +} + +template +void pmr::__details::resource_adaptor_imp::do_deallocate( + void* p, size_t bytes, size_t alignment) { + static const size_t max_natural_alignment = sizeof(max_align_t); + + if (0 == alignment) { + // Choose natural alignment for `bytes` + alignment = ((bytes ^ (bytes - 1)) >> 1) + 1; + if (alignment > max_natural_alignment) + alignment = max_natural_alignment; + } + + switch (alignment) { + case 1: + deallocate_imp<1>(p, bytes); + break; + case 2: + deallocate_imp<2>(p, bytes); + break; + case 4: + deallocate_imp<4>(p, bytes); + break; + case 8: + deallocate_imp<8>(p, bytes); + break; + case 16: + deallocate_imp<16>(p, bytes); + break; + case 32: + deallocate_imp<32>(p, bytes); + break; + case 64: + deallocate_imp<64>(p, bytes); + break; + default: { + size_t chunks = (bytes + sizeof(void*) + alignment - 1) / 64; + size_t chunkbytes = chunks * 64; + void* original = reinterpret_cast(p)[-1]; + + deallocate_imp<64>(original, chunkbytes); + } + } +} + +template +bool pmr::__details::resource_adaptor_imp::do_is_equal( + const memory_resource& other) const noexcept { + const resource_adaptor_imp* other_p = + dynamic_cast(&other); + + if (other_p) + return this->m_alloc == other_p->m_alloc; + else + return false; +} + +namespace __pmrd = pmr::__details; + +template +inline __pmrd::polymorphic_allocator_imp::polymorphic_allocator_imp() + : m_resource(get_default_resource()) { +} + +template +inline __pmrd::polymorphic_allocator_imp::polymorphic_allocator_imp( + pmr::memory_resource* r) + : m_resource(r ? r : get_default_resource()) { +} + +template +template +inline __pmrd::polymorphic_allocator_imp::polymorphic_allocator_imp( + const __pmrd::polymorphic_allocator_imp& other) + : m_resource(other.resource()) { +} + +template +inline Tp* __pmrd::polymorphic_allocator_imp::allocate(size_t n) { + return static_cast(m_resource->allocate(n * sizeof(Tp), alignof(Tp))); +} + +template +inline void __pmrd::polymorphic_allocator_imp::deallocate(Tp* p, size_t n) { + m_resource->deallocate(p, n * sizeof(Tp), alignof(Tp)); +} + +template +inline __pmrd::polymorphic_allocator_imp __pmrd::polymorphic_allocator_imp< + Tp>::select_on_container_copy_construction() const { + return __pmrd::polymorphic_allocator_imp(); +} + +template +inline pmr::memory_resource* __pmrd::polymorphic_allocator_imp::resource() + const { + return m_resource; +} + +template +inline bool __pmrd::operator==( + const __pmrd::polymorphic_allocator_imp& a, + const __pmrd::polymorphic_allocator_imp& b) { + // `operator==` for `memory_resource` first checks for equality of + // addresses and calls `is_equal` only if the addresses differ. The call + // `is_equal` because some polymorphic allocators are produced as a result + // of type erasure. In that case, `a` and `b` may contain + // `memory_resource`s with different addresses which, nevertheless, + // should compare equal. + return *a.resource() == *b.resource(); +} + +template +inline bool __pmrd::operator!=( + const __pmrd::polymorphic_allocator_imp& a, + const __pmrd::polymorphic_allocator_imp& b) { + return *a.resource() != *b.resource(); +} + +} // namespace cpp17 + +#endif // ! defined(INCLUDED_POLYMORPHIC_ALLOCATOR_DOT_H) diff --git a/tiledb/sm/filter/test/tile_data_generator.h b/tiledb/sm/filter/test/tile_data_generator.h index 2b88491e16ae..2f4bff73fa6d 100644 --- a/tiledb/sm/filter/test/tile_data_generator.h +++ b/tiledb/sm/filter/test/tile_data_generator.h @@ -35,6 +35,7 @@ #include #include +#include #include #include "tiledb/sm/tile/tile.h" diff --git a/vcpkg.json b/vcpkg.json index 2220a145d6d9..c8b9831f013e 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,7 +1,6 @@ { "builtin-baseline": "72010900b7cee36cea77aebb97695095c9358eaf", "dependencies": [ - "boost-container", "bzip2", "libmagic", "lz4", From 14e99dd6def1310404a591c6a3e75385a42d86a5 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 16 Feb 2024 17:55:28 +0200 Subject: [PATCH 194/456] Reintroduce `TILEDB_STATIC` under a deprecation warning. (#4732) --- TYPE: BUILD DESC: Reintroduce the `TILEDB_STATIC` option under a deprecation warning. --- cmake/Options/BuildOptions.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index fd439db47006..c88c3f10ac0a 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -40,6 +40,15 @@ option(CMAKE_EXPORT_COMPILE_COMMANDS "cmake compile commands" ON) set(TILEDB_INSTALL_LIBDIR "" CACHE STRING "If non-empty, install TileDB library to this directory instead of CMAKE_INSTALL_LIBDIR.") +if (DEFINED TILEDB_STATIC) + message(DEPRECATION "TILEDB_STATIC is deprecated and will be removed in version 2.28, to be released in Q3 2024. Use BUILD_SHARED_LIBS INSTEAD. Building both static and shared libraries is no longer available.") + if (TILEDB_STATIC) + set(BUILD_SHARED_LIBS OFF) + else() + set(BUILD_SHARED_LIBS ON) + endif() +endif() + if (NOT TILEDB_VCPKG) message(DEPRECATION "Disabling TILEDB_VCPKG is deprecated and will be removed in a future version.") endif() From c1e34275035a7cc0b524d2d80987ddb46da588a1 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Fri, 16 Feb 2024 13:24:38 -0500 Subject: [PATCH 195/456] Add memory tracking for RTree. (#4697) Add MemoryTracker to RTree using `tdb::pmr` containers. --- TYPE: NO_HISTORY DESC: Add memory tracking for RTree. --- tiledb/sm/fragment/fragment_metadata.cc | 6 +- tiledb/sm/fragment/fragment_metadata.h | 2 +- tiledb/sm/rtree/rtree.cc | 77 +++------ tiledb/sm/rtree/rtree.h | 54 +++---- tiledb/sm/rtree/test/CMakeLists.txt | 1 + tiledb/sm/rtree/test/compile_rtree_main.cc | 2 +- tiledb/sm/rtree/test/unit_rtree.cc | 161 ++++++++++--------- tiledb/sm/serialization/fragment_metadata.cc | 2 +- 8 files changed, 143 insertions(+), 162 deletions(-) diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index f30fd261232e..65064eb4e2d8 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -81,6 +81,7 @@ FragmentMetadata::FragmentMetadata( ContextResources* resources, shared_ptr memory_tracker) : resources_(resources) , memory_tracker_(memory_tracker) + , rtree_(RTree(nullptr, constants::rtree_fanout, memory_tracker_)) , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) @@ -119,7 +120,8 @@ FragmentMetadata::FragmentMetadata( , has_delete_meta_(has_deletes_meta) , sparse_tile_num_(0) , meta_file_size_(0) - , rtree_(RTree(&array_schema_->domain(), constants::rtree_fanout)) + , rtree_(RTree( + &array_schema_->domain(), constants::rtree_fanout, memory_tracker_)) , tile_index_base_(0) , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) @@ -1576,7 +1578,7 @@ const NDRange& FragmentMetadata::mbr(uint64_t tile_idx) const { return rtree_.leaf(tile_idx); } -const std::vector& FragmentMetadata::mbrs() const { +const tdb::pmr::vector& FragmentMetadata::mbrs() const { return rtree_.leaves(); } diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 13cb1676a780..8321b248f856 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -783,7 +783,7 @@ class FragmentMetadata { const NDRange& mbr(uint64_t tile_idx) const; /** Returns all the MBRs of all tiles in the fragment. */ - const std::vector& mbrs() const; + const tdb::pmr::vector& mbrs() const; /** * Retrieves the size of the tile when it is persisted (e.g. the size of the diff --git a/tiledb/sm/rtree/rtree.cc b/tiledb/sm/rtree/rtree.cc index 8578636c3714..e91a1838f792 100644 --- a/tiledb/sm/rtree/rtree.cc +++ b/tiledb/sm/rtree/rtree.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ #include "tiledb/sm/rtree/rtree.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" @@ -53,43 +54,18 @@ namespace sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -RTree::RTree() { - domain_ = nullptr; - fanout_ = 0; - deserialized_buffer_size_ = 0; -} - -RTree::RTree(const Domain* domain, unsigned fanout) - : domain_(domain) - , fanout_(fanout) { +RTree::RTree( + const Domain* domain, + unsigned fanout, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , domain_(domain) + , fanout_(fanout) + , levels_(memory_tracker_->get_resource(MemoryType::RTREE)) { } RTree::~RTree() = default; -RTree::RTree(const RTree& rtree) - : RTree() { - auto clone = rtree.clone(); - swap(clone); -} - -RTree::RTree(RTree&& rtree) noexcept - : RTree() { - swap(rtree); -} - -RTree& RTree::operator=(const RTree& rtree) { - auto clone = rtree.clone(); - swap(clone); - - return *this; -} - -RTree& RTree::operator=(RTree&& rtree) noexcept { - swap(rtree); - - return *this; -} - /* ****************************** */ /* API */ /* ****************************** */ @@ -240,7 +216,7 @@ const NDRange& RTree::leaf(uint64_t leaf_idx) const { return levels_.back()[leaf_idx]; } -const std::vector& RTree::leaves() const { +const tdb::pmr::vector& RTree::leaves() const { assert(!levels_.empty()); return levels_.back(); } @@ -297,10 +273,10 @@ Status RTree::set_leaf(uint64_t leaf_id, const NDRange& mbr) { return Status::Ok(); } -Status RTree::set_leaves(const std::vector& mbrs) { +Status RTree::set_leaves(const tdb::pmr::vector& mbrs) { levels_.clear(); levels_.resize(1); - levels_[0] = mbrs; + levels_[0].assign(mbrs.begin(), mbrs.end()); return Status::Ok(); } @@ -328,13 +304,21 @@ void RTree::deserialize( deserialize_v5(deserializer, domain); } +void RTree::reset(const Domain* domain, unsigned int fanout) { + domain_ = domain; + fanout_ = fanout; + free_memory(); +} + /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ RTree::Level RTree::build_level(const Level& level) { auto cur_mbr_num = (uint64_t)level.size(); - Level new_level((uint64_t)std::ceil((double)cur_mbr_num / fanout_)); + Level new_level( + (uint64_t)std::ceil((double)cur_mbr_num / fanout_), + memory_tracker_->get_resource(MemoryType::RTREE)); auto new_mbr_num = (uint64_t)new_level.size(); uint64_t mbrs_visited = 0; @@ -344,16 +328,7 @@ RTree::Level RTree::build_level(const Level& level) { domain_->expand_ndrange(level[mbrs_visited], &new_level[i]); } - return new_level; -} - -RTree RTree::clone() const { - RTree clone; - clone.domain_ = domain_; - clone.fanout_ = fanout_; - clone.levels_ = levels_; - - return clone; + return {new_level, memory_tracker_->get_resource(MemoryType::RTREE)}; } void RTree::deserialize_v1_v4( @@ -424,11 +399,5 @@ void RTree::deserialize_v5(Deserializer& deserializer, const Domain* domain) { domain_ = domain; } -void RTree::swap(RTree& rtree) { - std::swap(domain_, rtree.domain_); - std::swap(fanout_, rtree.fanout_); - std::swap(levels_, rtree.levels_); -} - } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/rtree/rtree.h b/tiledb/sm/rtree/rtree.h index ad70a0b8066c..35fd505d7bed 100644 --- a/tiledb/sm/rtree/rtree.h +++ b/tiledb/sm/rtree/rtree.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,6 +36,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/misc/tile_overlap.h" @@ -48,6 +49,7 @@ namespace sm { class Buffer; class ConstBuffer; +class MemoryTracker; enum class Datatype : uint8_t; enum class Layout : uint8_t; @@ -62,27 +64,19 @@ class RTree { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ + RTree() = delete; /** Constructor. */ - RTree(); - - /** Constructor. */ - RTree(const Domain* domain, unsigned fanout); + RTree( + const Domain* domain, + unsigned fanout, + shared_ptr memory_tracker); /** Destructor. */ ~RTree(); - /** Copy constructor. This performs a deep copy. */ - RTree(const RTree& rtree); - - /** Move constructor. */ - RTree(RTree&& rtree) noexcept; - - /** Copy-assign operator. This performs a deep copy. */ - RTree& operator=(const RTree& rtree); - - /** Move-assign operator. */ - RTree& operator=(RTree&& rtree) noexcept; + DISABLE_COPY_AND_COPY_ASSIGN(RTree); + DISABLE_MOVE_AND_MOVE_ASSIGN(RTree); /* ********************************* */ /* API */ @@ -125,7 +119,7 @@ class RTree { const NDRange& leaf(uint64_t leaf_idx) const; /** Returns the leaves of the tree. */ - const std::vector& leaves() const; + const tdb::pmr::vector& leaves() const; /** * Returns the number of leaves that are stored in a (full) subtree @@ -158,7 +152,7 @@ class RTree { * Sets the input MBRs as leaves. This will destroy the existing * RTree. */ - Status set_leaves(const std::vector& mbrs); + Status set_leaves(const tdb::pmr::vector& mbrs); /** * Resizes the leaf level. It destroys the upper levels @@ -176,6 +170,14 @@ class RTree { void deserialize( Deserializer& deserializer, const Domain* domain, uint32_t version); + /** + * Resets the RTree with the input domain and fanout. + * + * @param domain The domain to use for the RTree. + * @param fanout The fanout of the RTree. + */ + void reset(const Domain* domain, unsigned fanout); + private: /* ********************************* */ /* PRIVATE TYPE DEFINITIONS */ @@ -203,7 +205,7 @@ class RTree { * `levels_`, where the first level is the root. This is how * we can infer which tree level each `Level` object corresponds to. */ - typedef std::vector Level; + typedef tdb::pmr::vector Level; /** * Defines an R-Tree level entry, which corresponds to a node @@ -222,6 +224,9 @@ class RTree { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** Memory tracker for the RTree. */ + shared_ptr memory_tracker_; + /** * The domain for which this R-tree provides an index. * @@ -237,7 +242,7 @@ class RTree { * The tree levels. The first level is the root. Note that the root * always consists of a single MBR. */ - std::vector levels_; + tdb::pmr::vector levels_; /** * Stores the size of the buffer used to deserialize the data, used for @@ -252,9 +257,6 @@ class RTree { /** Builds a single tree level on top of the input level. */ Level build_level(const Level& level); - /** Returns a deep copy of this RTree. */ - RTree clone() const; - /** * Deserializes the contents of the object from the input buffer based * on the format version. @@ -272,12 +274,6 @@ class RTree { * Applicable to versions >= 5 */ void deserialize_v5(Deserializer& deserializer, const Domain* domain); - - /** - * Swaps the contents (all field values) of this RTree with the - * given ``rtree``. - */ - void swap(RTree& rtree); }; } // namespace sm diff --git a/tiledb/sm/rtree/test/CMakeLists.txt b/tiledb/sm/rtree/test/CMakeLists.txt index ef40ea8a1224..13aade8948ec 100644 --- a/tiledb/sm/rtree/test/CMakeLists.txt +++ b/tiledb/sm/rtree/test/CMakeLists.txt @@ -28,5 +28,6 @@ include(unit_test) commence(unit_test rtree) this_target_sources(main.cc unit_rtree.cc) + this_target_link_libraries(tiledb_test_support_lib) this_target_object_libraries(rtree) conclude(unit_test) diff --git a/tiledb/sm/rtree/test/compile_rtree_main.cc b/tiledb/sm/rtree/test/compile_rtree_main.cc index d720c36ba364..f1f30d98cb9c 100644 --- a/tiledb/sm/rtree/test/compile_rtree_main.cc +++ b/tiledb/sm/rtree/test/compile_rtree_main.cc @@ -29,6 +29,6 @@ #include "../rtree.h" int main() { - tiledb::sm::RTree x{}; + tiledb::sm::RTree x{nullptr, 0, nullptr}; return 0; } diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 029d75988348..6ca42b686d73 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,7 +30,9 @@ * Tests the `RTree` class. */ +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/layout.h" @@ -40,15 +42,18 @@ #include using namespace tiledb::sm; +using tiledb::test::create_test_memory_tracker; // `mbrs` contains a flattened vector of values (low, high) // per dimension per MBR template -std::vector create_mbrs(const std::vector& mbrs) { +tdb::pmr::vector create_mbrs( + const std::vector& mbrs, shared_ptr tracker) { assert(mbrs.size() % 2 * D == 0); uint64_t mbr_num = (uint64_t)(mbrs.size() / (2 * D)); - std::vector ret(mbr_num); + tdb::pmr::vector ret( + mbr_num, tracker->get_resource(MemoryType::RTREE)); uint64_t r_size = 2 * sizeof(T); for (uint64_t m = 0; m < mbr_num; ++m) { ret[m].resize(D); @@ -57,16 +62,19 @@ std::vector create_mbrs(const std::vector& mbrs) { } } - return ret; + return {ret, tracker->get_resource(MemoryType::RTREE)}; } template -std::vector create_mbrs( - const std::vector& r1, const std::vector& r2) { +tdb::pmr::vector create_mbrs( + const std::vector& r1, + const std::vector& r2, + shared_ptr tracker) { assert(r1.size() == r2.size()); uint64_t mbr_num = (uint64_t)(r1.size() / 2); - std::vector ret(mbr_num); + tdb::pmr::vector ret( + mbr_num, tracker->get_resource(MemoryType::RTREE)); uint64_t r1_size = 2 * sizeof(T1); uint64_t r2_size = 2 * sizeof(T2); for (uint64_t m = 0; m < mbr_num; ++m) { @@ -75,7 +83,7 @@ std::vector create_mbrs( ret[m][1] = Range(&r2[2 * m], r2_size); } - return ret; + return {ret, tracker->get_resource(MemoryType::RTREE)}; } Domain create_domain( @@ -118,7 +126,8 @@ Domain create_domain( TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { // Empty tree - RTree rtree0; + auto tracker = create_test_memory_tracker(); + RTree rtree0(nullptr, 0, tracker); CHECK(rtree0.height() == 0); CHECK(rtree0.dim_num() == 0); CHECK(rtree0.domain() == nullptr); @@ -131,9 +140,9 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { int32_t dim_extent = 10; Domain dom1 = create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); - std::vector mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}); + auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); const Domain d1{dom1}; - RTree rtree1(&d1, 3); + RTree rtree1(&d1, 3, tracker); CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); CHECK(rtree1.set_leaf_num(mbrs_1d.size()).ok()); for (size_t m = 0; m < mbrs_1d.size(); ++m) @@ -191,10 +200,10 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { {Datatype::INT64, Datatype::INT64}, {dim_dom_2, dim_dom_2}, {&dim_extent_2, &dim_extent_2}); - std::vector mbrs_2d = - create_mbrs({1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}); + auto mbrs_2d = create_mbrs( + {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); const Domain d2{dom2}; - RTree rtree2(&d2, 5); + RTree rtree2(&d2, 5, tracker); CHECK(rtree2.set_leaves(mbrs_2d).ok()); rtree2.build_tree(); CHECK(rtree2.height() == 2); @@ -228,12 +237,12 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { // Float datatype float dim_dom_f[] = {1.0, 1000.0}; float dim_extent_f = 10.0; - std::vector mbrs_f = - create_mbrs({1.0f, 3.0f, 5.0f, 10.0f, 20.0f, 22.0f}); + auto mbrs_f = + create_mbrs({1.0f, 3.0f, 5.0f, 10.0f, 20.0f, 22.0f}, tracker); Domain dom2f = create_domain({"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}); const Domain d2f{dom2f}; - RTree rtreef(&d2f, 5); + RTree rtreef(&d2f, 5, tracker); CHECK(rtreef.set_leaves(mbrs_f).ok()); rtreef.build_tree(); @@ -269,14 +278,15 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; Domain dom1 = create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); - std::vector mbrs = create_mbrs({1, 3, 5, 10, 20, 22}); + auto mbrs = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); const Domain d1{dom1}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -315,15 +325,16 @@ TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { TEST_CASE("RTree: Test 1D R-tree, height 3", "[rtree][1d][3h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - std::vector mbrs = create_mbrs( - {1, 3, 5, 10, 20, 22, 30, 35, 36, 38, 40, 49, 50, 51, 65, 69}); + auto mbrs = create_mbrs( + {1, 3, 5, 10, 20, 22, 30, 35, 36, 38, 40, 49, 50, 51, 65, 69}, tracker); Domain dom1 = create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); const Domain d1(dom1); - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -381,6 +392,7 @@ TEST_CASE("RTree: Test 1D R-tree, height 3", "[rtree][1d][3h]") { TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; @@ -389,10 +401,10 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, {&dim_extent, &dim_extent}); - std::vector mbrs = - create_mbrs({1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}); + auto mbrs = create_mbrs( + {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); const Domain d2{dom2}; - RTree rtree(&d2, 3); + RTree rtree(&d2, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -434,6 +446,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; @@ -442,11 +455,12 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, {&dim_extent, &dim_extent}); - std::vector mbrs = create_mbrs( + auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15, 11, 15, 20, 22, 16, 16, - 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}); + 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, + tracker); const Domain d2{dom2}; - RTree rtree(&d2, 3); + RTree rtree(&d2, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -511,6 +525,7 @@ TEST_CASE( "RTree: Test R-Tree, heterogeneous (uint8, int32), basic functions", "[rtree][basic][heter]") { // Create RTree with dimensions uint8, int32 + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); uint8_t uint8_dom[] = {0, 10}; int32_t int32_dom[] = {5, 10}; @@ -521,10 +536,10 @@ TEST_CASE( {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, {&uint8_extent, &int32_extent}); - std::vector mbrs = - create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}); + auto mbrs = + create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); const Domain d1{dom}; - RTree rtree(&d1, 5); + RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -567,6 +582,7 @@ TEST_CASE( "RTree: Test R-Tree, heterogeneous (uint64, float32), basic functions", "[rtree][basic][heter]") { // Create RTree with dimensions uint64, float32 + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); uint64_t uint64_dom[] = {0, 10}; float float_dom[] = {0.1f, 0.9f}; @@ -577,10 +593,10 @@ TEST_CASE( {Datatype::UINT64, Datatype::FLOAT32}, {uint64_dom, float_dom}, {&uint64_extent, &float_extent}); - std::vector mbrs = - create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}); + auto mbrs = + create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); const Domain d1{dom}; - RTree rtree(&d1, 5); + RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -623,6 +639,7 @@ TEST_CASE( "RTree: Test 2D R-tree, height 2, heterogeneous (uint8, int32)", "[rtree][2d][2h][heter]") { // Create RTree with dimensions uint8, int32 + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); uint8_t uint8_dom[] = {0, 200}; int32_t int32_dom[] = {5, 100}; @@ -633,10 +650,10 @@ TEST_CASE( {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, {&uint8_extent, &int32_extent}); - std::vector mbrs = - create_mbrs({0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}); + auto mbrs = create_mbrs( + {0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}, tracker); const Domain d1{dom}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -692,6 +709,7 @@ TEST_CASE( "RTree: Test 2D R-tree, height 3, heterogeneous (uint8, int32)", "[rtree][2d][2h][heter]") { // Create RTree with dimensions uint8, int32 + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); uint8_t uint8_dom[] = {0, 200}; int32_t int32_dom[] = {5, 100}; @@ -702,10 +720,10 @@ TEST_CASE( {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, {&uint8_extent, &int32_extent}); - std::vector mbrs = create_mbrs( - {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}); + auto mbrs = create_mbrs( + {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}, tracker); const Domain d1{dom}; - RTree rtree(&d1, 2); + RTree rtree(&d1, 2, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -788,11 +806,13 @@ TEST_CASE( // `mbrs` contains a flattened vector of values (low, high) // per dimension per MBR template -std::vector create_str_mbrs(const std::vector& mbrs) { +tdb::pmr::vector create_str_mbrs( + const std::vector& mbrs, shared_ptr tracker) { assert(mbrs.size() % 2 * D == 0); uint64_t mbr_num = (uint64_t)(mbrs.size() / (2 * D)); - std::vector ret(mbr_num); + tdb::pmr::vector ret( + mbr_num, tracker->get_resource(MemoryType::RTREE)); for (uint64_t m = 0; m < mbr_num; ++m) { ret[m].resize(D); for (unsigned d = 0; d < D; ++d) { @@ -802,19 +822,21 @@ std::vector create_str_mbrs(const std::vector& mbrs) { } } - return ret; + return {ret, tracker->get_resource(MemoryType::RTREE)}; } // `mbrs` contains a flattened vector of values (low, high) // per dimension per MBR -std::vector create_str_int32_mbrs( +tdb::pmr::vector create_str_int32_mbrs( const std::vector& mbrs_str, - const std::vector mbrs_int) { + const std::vector mbrs_int, + shared_ptr tracker) { assert(mbrs_str.size() == mbrs_int.size()); assert(mbrs_str.size() % 2 == 0); uint64_t mbr_num = (uint64_t)(mbrs_str.size() / 2); - std::vector ret(mbr_num); + tdb::pmr::vector ret( + mbr_num, tracker->get_resource(MemoryType::RTREE)); for (uint64_t m = 0; m < mbr_num; ++m) { ret[m].resize(2); const auto& start = mbrs_str[2 * m]; @@ -824,25 +846,22 @@ std::vector create_str_int32_mbrs( ret[m][1] = Range(range, sizeof(range)); } - return ret; -} - -std::pair range_to_str(const Range& r) { - return std::pair(r.start_str(), r.end_str()); + return {ret, tracker->get_resource(MemoryType::RTREE)}; } TEST_CASE( "RTree: Test 1D R-tree, string dims, height 2", "[rtree][1d][string-dims][2h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); Domain dom1 = create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); - std::vector mbrs = - create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}); + auto mbrs = + create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}, tracker); const Domain d1{dom1}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -913,10 +932,11 @@ TEST_CASE( "RTree: Test 1D R-tree, string dims, height 3", "[rtree][1d][string-dims][3h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); Domain dom1 = create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); - std::vector mbrs = create_str_mbrs<1>( + auto mbrs = create_str_mbrs<1>( {"aa", "b", "eee", @@ -928,10 +948,11 @@ TEST_CASE( "mm", "mmn", "oo", - "oop"}); + "oop"}, + tracker); const Domain d1{dom1}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -1005,28 +1026,19 @@ TEST_CASE( "RTree: Test 2D R-tree, string dims, height 2", "[rtree][2d][string-dims][2h]") { // Build tree + auto tracker = create_test_memory_tracker(); std::vector is_default(2, false); Domain dom = create_domain( {"d1", "d2"}, {Datatype::STRING_ASCII, Datatype::STRING_ASCII}, {nullptr, nullptr}, {nullptr, nullptr}); - std::vector mbrs = create_str_mbrs<2>( - {"aa", - "b", - "eee", - "g", - "gggg", - "ii", - "jj", - "lll", - "m", - "n", - "oo", - "qqq"}); + auto mbrs = create_str_mbrs<2>( + {"aa", "b", "eee", "g", "gggg", "ii", "jj", "lll", "m", "n", "oo", "qqq"}, + tracker); const Domain d1{dom}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -1128,11 +1140,12 @@ TEST_CASE( {Datatype::STRING_ASCII, Datatype::INT32}, {nullptr, dom_int32}, {nullptr, &tile_extent}); - std::vector mbrs = create_str_int32_mbrs( - {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}); + auto tracker = create_test_memory_tracker(); + auto mbrs = create_str_int32_mbrs( + {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}, tracker); const Domain d1{dom}; - RTree rtree(&d1, 3); + RTree rtree(&d1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index c5e8f581acd0..999c860db8fb 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -358,7 +358,7 @@ Status fragment_metadata_from_capnp( auto data = frag_meta_reader.getRtree(); auto& domain = array_schema->domain(); // If there are no levels, we still need domain_ properly initialized - frag_meta->rtree() = RTree(&domain, constants::rtree_fanout); + frag_meta->rtree().reset(&domain, constants::rtree_fanout); Deserializer deserializer(data.begin(), data.size()); // What we actually deserialize is not something written on disk in a // possibly historical format, but what has been serialized in From c6c76154d1e18d3048f22ccc4888b3e2fb07fe89 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Sun, 18 Feb 2024 05:39:14 -0600 Subject: [PATCH 196/456] Move to a process global for test MemoryTracker (#4739) I've seen about three or four different test failures due to `create_test_memory_tracker` returning new instances which leads to life time issues with the memory resources. This changes to a single process singleton to avoid the issue altogether. Once we merge the current PRs that are all using `create_test_memory_tracker` I'll follow up with a PR to rename all of those instances to `get_test_memory_tracker` and remove the `create_test_memory_tracker` to prevent further use. --- TYPE: NO_HISTORY DESC: Move to a process global for test MemoryTracker --- test/support/src/mem_helpers.cc | 14 ++++++++++++-- test/support/src/mem_helpers.h | 12 ++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/test/support/src/mem_helpers.cc b/test/support/src/mem_helpers.cc index c92f0cab85f6..50a18ad5b26e 100644 --- a/test/support/src/mem_helpers.cc +++ b/test/support/src/mem_helpers.cc @@ -35,15 +35,25 @@ namespace tiledb::test { -shared_ptr create_test_memory_tracker() { +shared_ptr get_test_memory_tracker() { class MemoryTrackerCreator : public sm::MemoryTracker { public: MemoryTrackerCreator() : sm::MemoryTracker() { } + + static shared_ptr get_instance() { + static shared_ptr tracker{ + new MemoryTrackerCreator()}; + return tracker; + } }; - return make_shared(HERE()); + return MemoryTrackerCreator::get_instance(); +} + +shared_ptr create_test_memory_tracker() { + return get_test_memory_tracker(); } } // namespace tiledb::test diff --git a/test/support/src/mem_helpers.h b/test/support/src/mem_helpers.h index 96a359a1975b..363a1e78a76c 100644 --- a/test/support/src/mem_helpers.h +++ b/test/support/src/mem_helpers.h @@ -38,6 +38,18 @@ namespace tiledb::test { +/** + * Helper function get the test instance of a shared_ptr + * + * This is the preferred function. The create_test_memory_tracker will be + * replaced shortly and only serves as a proxy to this function while we + * transition the first few PRs to use this new function. + * + * The reasoning here is that creating memory trackers has turned out to be a + * bit of a footgun with lifetime issues. + */ +shared_ptr get_test_memory_tracker(); + /** * Helper function to create test instances of shared_ptr */ From a84789eacc5ac2dee2d94031cd0cdd0c53a8d5f8 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 19 Feb 2024 22:38:15 +0200 Subject: [PATCH 197/456] Support reading V1 group details with explicit version in the name. (#4744) [SC-41275](https://app.shortcut.com/tiledb-inc/story/41275/fix-v1-groups-written-with-latest-version) When version 2 of the group details format was introduced in #3928 (2.16 and also backported to 2.15), the names of group details V2 files were of the format `__t1_t2_id_version`, and V1 kept using `__t1_t2_id` for compatibility. In #4530 (2.19), V1 files started being written with the new format as well. This confused the group reader, which considered group details files with names of the form `__t1_t2_id_1` as V2, merely because they had a version field in their name. This PR fixes the group reader, making it always consider the version if it is present in the name of details file. The change has been validated by writing to a V1 group and then reading from it. The test group was created locally by Core version 2.14 and was added in `test/inputs/groups` (its size is tiny so it can safely be stored in the repository). --- TYPE: BUG DESC: Support reading V1 group details with explicit version in the name. --- ...362383727_a464f3a3a7e740c8856a476ee4c66ce1 | Bin 0 -> 123 bytes .../inputs/groups/group_v1/__tiledb_group.tdb | 0 .../group_v1/subgroup1/__tiledb_group.tdb | 0 test/src/unit-backwards_compat.cc | 56 ++++++++++++++++++ tiledb/sm/group/group.cc | 7 ++- 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 test/inputs/groups/group_v1/__group/__1708362383727_1708362383727_a464f3a3a7e740c8856a476ee4c66ce1 create mode 100644 test/inputs/groups/group_v1/__tiledb_group.tdb create mode 100644 test/inputs/groups/group_v1/subgroup1/__tiledb_group.tdb diff --git a/test/inputs/groups/group_v1/__group/__1708362383727_1708362383727_a464f3a3a7e740c8856a476ee4c66ce1 b/test/inputs/groups/group_v1/__group/__1708362383727_1708362383727_a464f3a3a7e740c8856a476ee4c66ce1 new file mode 100644 index 0000000000000000000000000000000000000000..48772da90954babe8aa0dabda8ba716ab5c1041b GIT binary patch literal 123 zcmWe+U|?_uVh}KePz)@LPzHk#n8C;Z5@2Kn5nvWn8l+wshy|d0kQ_{`f-yNIAt8Y! fNr2&EPtTkLGiK+)%%G~ky39t!WgHCIs_dBnM$Zh^ literal 0 HcmV?d00001 diff --git a/test/inputs/groups/group_v1/__tiledb_group.tdb b/test/inputs/groups/group_v1/__tiledb_group.tdb new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/inputs/groups/group_v1/subgroup1/__tiledb_group.tdb b/test/inputs/groups/group_v1/subgroup1/__tiledb_group.tdb new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/src/unit-backwards_compat.cc b/test/src/unit-backwards_compat.cc index 685c942cf35e..425e7afdf7f4 100644 --- a/test/src/unit-backwards_compat.cc +++ b/test/src/unit-backwards_compat.cc @@ -34,12 +34,15 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/serialization_wrappers.h" +#include "test/support/src/temporary_local_directory.h" #include "tiledb/common/common.h" +#include "tiledb/common/stdx_string.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" #include "tiledb/sm/misc/constants.h" #include +#include #include #include #include @@ -52,6 +55,9 @@ namespace { static const std::string arrays_dir = std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays"; +static const std::string groups_dir = + std::string(TILEDB_TEST_INPUTS_DIR) + "/groups"; + template void set_query_coords( const Domain& domain, @@ -1428,3 +1434,53 @@ TEST_CASE( g, "u64", TILEDB_UINT64, 0x7777777777777777); } } + +TEST_CASE( + "Backwards compatibility: Test v1 groups", + "[backwards-compat][group][v1]") { + Context ctx; + VFS vfs(ctx); + + // Copy the group to a temporary directory because we will be modifying it. + tiledb::sm::TemporaryLocalDirectory temp_dir; + std::filesystem::copy( + groups_dir + "/group_v1", + temp_dir.path(), + std::filesystem::copy_options::recursive); + + // Read the group + { + Group g{ctx, temp_dir.path(), TILEDB_READ}; + + CHECK(g.dump(false) != ""); + CHECK(g.member_count() == 1); + } + + // Add a member to the group + { + Group g{ctx, temp_dir.path(), TILEDB_WRITE}; + + Group::create(ctx, temp_dir.path() + "/subgroup2"); + + g.add_member("subgroup2", true, "subgroup2"); + + g.close(); + } + + // Read the group again + { + Group g{ctx, temp_dir.path(), TILEDB_READ}; + + CHECK(g.dump(false) != ""); + CHECK(g.member_count() == 2); + CHECK(g.member(1).name() == "subgroup2"); + } + + // Read the raw group details files + auto children = vfs.ls(temp_dir.path() + "/__group"); + CHECK(children.size() == 2); + std::sort(children.begin(), children.end()); + CHECK(!tiledb::sm::utils::parse::ends_with(children[0], "_1")); + // This is the file written by this test. + CHECK(tiledb::sm::utils::parse::ends_with(children[1], "_1")); +} diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index aa5a10edb51f..15895488cbe6 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/group/group.h" #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/common/stdx_string.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/query_type.h" @@ -775,8 +776,12 @@ void Group::load_group_details() { // V1 groups did not have the version appended so only have 4 "_" // (____) + // Since 2.19, V1 groups also have the version appended so we have + // to check for that as well auto part = latest_group_uri.last_path_part(); - if (std::count(part.begin(), part.end(), '_') == 4) { + auto underscoreCount = std::count(part.begin(), part.end(), '_'); + if (underscoreCount == 4 || + (underscoreCount == 5 && utils::parse::ends_with(part, "_1"))) { load_group_from_uri(latest_group_uri); return; } From 2e9231a569432f3b89b32ffa6d743a72c0b7e65b Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Wed, 21 Feb 2024 04:46:41 -0700 Subject: [PATCH 198/456] Default seeding of `class PRNG` (#4720) This PR changes the default seeding of `class PRNG` to use `std::random_device`. [sc-41075] --- TYPE: NO_HISTORY DESC: Default seeding of `class PRNG` --- tiledb/common/random/prng.cc | 76 ++++++++++--- tiledb/common/random/prng.h | 100 ++++++++++++++++-- tiledb/common/random/seeder.h | 6 +- .../random/test/unit_seedable_global_PRNG.cc | 18 +++- 4 files changed, 173 insertions(+), 27 deletions(-) diff --git a/tiledb/common/random/prng.cc b/tiledb/common/random/prng.cc index 842439b49b4b..e672e5e0b8aa 100644 --- a/tiledb/common/random/prng.cc +++ b/tiledb/common/random/prng.cc @@ -33,13 +33,75 @@ #include "tiledb/common/random/prng.h" namespace tiledb::common { +/** + * 64-bit mersenne twister engine for random number generation. + * + * This definition is duplicated to avoid having it defined as `public` in + * `class PRNG`. + */ +using prng_type = std::mt19937_64; + +/** + * Implementation of the random seed. + * + * This is a class template in order to use `if constexpr`. + * + * @tparam return_size_type The type of the seed to be returned + */ +template +return_size_type random_seed() { + static constexpr size_t rng_size = sizeof(std::random_device::result_type); + static constexpr size_t ret_size = sizeof(return_size_type); + std::random_device rng{}; + /* + * We will need 64 bits to adequately seed the PRNG (`ret_size`). We support + * cases where the result size of the RNG is 64 or 32 bits (`rng_size`). + */ + if constexpr (ret_size == rng_size) { + return rng(); + } else if constexpr (ret_size == 2 * rng_size) { + return (rng() << rng_size) + rng(); + } else { + throw std::runtime_error("Unsupported combination of RNG sizes"); + } +} + +/** + * The PRNG used within the random constructor. + */ +prng_type prng_random() { + return prng_type{random_seed()}; // RVO +} + +/** + * The PRNG used within the default constructor. + */ +prng_type prng_default() { + /* + * Retrieve optional seed, which may or may not have been set explicitly. + */ + auto seed{Seeder::get().seed()}; + /* + * Use the seed if it has been set. Otherwise use a random seed. + */ + if (seed.has_value()) { + return prng_type{seed.value()}; // RVO + } else { + return prng_random(); // RVO + } +} /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ PRNG::PRNG() - : prng_(prng_initial()) + : prng_(prng_default()) + , mtx_{} { +} + +PRNG::PRNG(RandomSeedT) + : prng_(prng_random()) , mtx_{} { } @@ -61,16 +123,4 @@ uint64_t PRNG::operator()() { /* PRIVATE METHODS */ /* ********************************* */ -std::mt19937_64 PRNG::prng_initial() { - // Retrieve optional, potentially default-constructed seed. - auto seed{Seeder::get().seed()}; - - // If the seed has been set, set it on the RNG engine. - if (seed.has_value()) { - return std::mt19937_64{seed.value()}; // RVO - } else { - return {}; // RVO - } -} - } // namespace tiledb::common diff --git a/tiledb/common/random/prng.h b/tiledb/common/random/prng.h index d03f5ceacf11..849c89e63afa 100644 --- a/tiledb/common/random/prng.h +++ b/tiledb/common/random/prng.h @@ -39,6 +39,80 @@ #include "tiledb/common/random/seeder.h" namespace tiledb::common { + +/** + * Marker class for a test-only PRNG constructor + */ +class RandomSeedT {}; +/** + * Marker constant + */ +static constexpr RandomSeedT RandomSeed; + +/** + * A random number generator suitable for both production and testing. + * + * @section Requirements + * + * This PRNG must support two very different kinds of situations: + * + * 1. In production use (the ordinary case) the seed must be _actually_ random + * so that the random sequences in different processes are distinct. + * 2. During most testing the seed must be deterministic to ensure that + * different test runs execute the same sequence of operations. This ensures + * that test failures can be replicated for diagnosis and correction. + * a. In particular, the seed in Catch2 test runs should be deterministic. + * 3. Certain tests, however, require actual randomness. + * a. One such test verifies that actual randomness is available per (1). Such + * tests necessarily have the possibility of failures, i.e. of false + * positives, but the actual likelihood can be made extremely low. + * b. Stress tests execute large number of test runs searching for defects. + * Such tests do not generate new information when run with previously- + * used PRNG sequences. + * + * This class satisfies these requirements with the following implementation + * choices: + * 1. If the user has not called `set_seed()` on the global seeder (from + * `Seeder::get`), then the seed is taken from `std::random_device`. + * 2. If the user has called `set_seed()` on the global seeder, that seed is + * used. + * 3. This class uses a global seeder in order to support Catch2. An event + * handler that executes at the start of the test run calls `set_seed()`. + * + * @section Maturity + * + * This class only has a default constructor. It does not have constructors that + * take seeds nor seeders. Such constructors would be useful for replicating + * test runs, but would also be premature at present. There's further test + * infrastructure required to replicate a specific test in isolation. As that + * test infrastructure matures, so also should this class. In the interim, in + * order to replicate a specific test with a specific seed, the function + * `initial_prng()` can be temporarily changed. + * + * This class uses a seeded PRNG to implement the random sequence. The + * requirement is that sequences in different processes be distinct, not that + * they be actually random. A randomly-seeded PRNG satisfies this requirement. + * The motivation for this implementation choice is as follows: + * 1. There is no standard hardware requirement for random number generation. + * While it's generally available, there are unknown variations in + * significant quality parameters such as the rate of random generation, + * duration of an RNG call, and randomness of generation (e.g. n-gram + * entropies). + * 2. In order not to stress a potentially inadequate RNG, we only call it for + * seeding and not for every number. + * 3. Qualifying a potential RNG implementation requires engineering resources + * that have not been committed as yet. + * + * @section Caveat + * + * This class uses `std::random_device` to seed the PRNG if no explicit seed is + * set. The standard library does not require that this class use an actual RNG, + * i.e. RNG from hardware of some kind. Indeed, certain earlier implementations + * did not do so and were deterministic. In order to validate that this device + * is actually random, it's necessary to run a multiprocess test to observe + * initialization in different processes. The test suite does not contain such + * a validation test at present. + */ class PRNG { public: /* ********************************* */ @@ -46,14 +120,25 @@ class PRNG { /* ********************************* */ /** - * Constructor. + * Default constructor. * - * Constructs an mt19937 engine for random number generation. - * If Seeder has been seeded, the seed will be set on the engine. - * Otherwise, it is default-constructed. + * If `Seeder` has been seeded, the seed will be set on the engine. Otherwise, + * the generator is constructed with a random seed. */ PRNG(); + /** + * Constructor for random seeding. + * + * This constructor makes an object that is always constructed with a random + * seed. + * + * @warning This constructor is only for testing. It must not be used in + * production code, where it would thwart the ability to run tests + * deterministically. + */ + PRNG(RandomSeedT); + /** Copy constructor is deleted. */ PRNG(const PRNG&) = delete; @@ -89,13 +174,6 @@ class PRNG { /** Mutex which protects against simultaneous access to operator() body. */ std::mutex mtx_; - - /* ********************************* */ - /* PRIVATE METHODS */ - /* ********************************* */ - - /** Default-constructs an mt19937 engine and optionally sets the seed. */ - std::mt19937_64 prng_initial(); }; } // namespace tiledb::common diff --git a/tiledb/common/random/seeder.h b/tiledb/common/random/seeder.h index e2807a146fb1..d24efebd4689 100644 --- a/tiledb/common/random/seeder.h +++ b/tiledb/common/random/seeder.h @@ -48,8 +48,10 @@ namespace tiledb::common { * default (set_seed) seed is set (seed) seed is used * but unused * - * Note that each transition may occur only once. - * i.e. A seed may only be set one time and may only be used one time. + * Note that each transition may occur only once, i.e. a seed may only be set + * one time and may only be used one time. This is an explicit design choice to + * ensure that a singleton PRNG is only initialized once, and to prevent the + * case where a seeming initialization is not the actual initialization. */ class Seeder { public: diff --git a/tiledb/common/random/test/unit_seedable_global_PRNG.cc b/tiledb/common/random/test/unit_seedable_global_PRNG.cc index b3d906207a84..3ec2c4153992 100644 --- a/tiledb/common/random/test/unit_seedable_global_PRNG.cc +++ b/tiledb/common/random/test/unit_seedable_global_PRNG.cc @@ -96,6 +96,8 @@ TEST_CASE( "SeedableGlobalPRNG: operator", "[SeedableGlobalPRNG][operator][multiple]") { PRNG& prng = PRNG::get(); + // Verify that a second call succeeds. + CHECK_NOTHROW(PRNG::get()); auto rand_num1 = prng(); CHECK(rand_num1 != 0); @@ -112,7 +114,11 @@ TEST_CASE( TEST_CASE( "SeedableGlobalPRNG: Seeder singleton, errors", "[SeedableGlobalPRNG][Seeder][singleton][errors]") { - // Note: these errors will occur because PRNG sets and uses the singleton. + /* + * Retrieve a PRNG object explicitly. This will cause the PRNG to use the + * singleton seeder, after which subsequent calls should fail. + */ + [[maybe_unused]] auto& x{PRNG::get()}; Seeder& seeder_ = Seeder::get(); SECTION("try to set new seed after it's been set") { @@ -128,6 +134,16 @@ TEST_CASE( } } +/* + * Verify that randomly-seeded PRNG return different numbers. This is the best + * we can do within the ordinary way within a single-process test, the only kind + * readily available within Catch2. + */ +TEST_CASE("SeedableGlobalPRNG: Random seeding", "[SeedableGlobalPRNG]") { + PRNG x(RandomSeed), y(RandomSeed); + CHECK(x() != y()); +} + TEST_CASE("random_label", "[random_label]") { auto rand_label1 = random_label(); CHECK(rand_label1.length() == 32); From 64e691285189bb8557e22c4242a8c77d698678f0 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Wed, 21 Feb 2024 17:42:16 -0500 Subject: [PATCH 199/456] Update REST-CI URL (#4757) Update REST-CI URL --- TYPE: NO_HISTORY DESC: Update REST-CI URL --- .github/workflows/ci-rest.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 4c69656c1de5..45de21e963c9 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -10,11 +10,11 @@ jobs: steps: # For easy access to lookup dispatched CI job. - - name: Print URL for TileDB-REST-CI actions - run: echo https://github.com/TileDB-Inc/TileDB-REST-CI/actions + - name: Print URL for REST CI actions + run: echo https://github.com/TileDB-Inc/TileDB-Internal/actions # If this workflow fails on the remote repository, this CI job will also fail. - - name: Workflow dispatch to TileDB-REST-CI + - name: Workflow dispatch to REST CI id: trigger-step uses: aurelien-baudet/workflow-dispatch@v2 env: @@ -22,8 +22,8 @@ jobs: # Skip if no PAT is set (e.g. for PRs from forks). if: env.TILEDB_REST_CI_PAT != null with: - repo: TileDB-Inc/TileDB-REST-CI - # Trigger workflow on TileDB-REST-CI at this ref. + repo: TileDB-Inc/TileDB-Internal + # Trigger workflow on TileDB-Internal at this ref. ref: "main" workflow: full-ci.yml token: ${{ secrets.TILEDB_REST_CI_PAT }} From bb13be6b706dae391f2fd2aee825032d9f33df35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Thu, 22 Feb 2024 10:17:24 +0100 Subject: [PATCH 200/456] Add unit_vfs to tests for windows/osx/linux (#4748) Add unit_vfs to CI tests for windows/osx/linux. --- TYPE: FEATURE DESC: Add unit_vfs to tests for windows/osx/linux --- .github/workflows/build-windows.yml | 9 +++++++++ .github/workflows/ci-linux_mac.yml | 1 + scripts/ci/build_libtiledb.sh | 1 + 3 files changed, 11 insertions(+) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 2b898a1a333a..6c2d693e3ca5 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -240,6 +240,7 @@ jobs: # CMake exits with non-0 status if there are any warnings during the build, so cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target unit_vfs --config $CMakeBuildType -- /verbosity:minimal cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal @@ -279,6 +280,14 @@ jobs: Write-Host "Tests failed. tiledb_unit exit status: " $LastExitCocde $host.SetShouldExit($LastExitCode) } + + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$CMakeBuildType\unit_vfs -d=yes" + Write-Host "cmds: '$cmds'" + Invoke-Expression $cmds + if ($LastExitCode -ne 0) { + Write-Host "Tests failed. tiledb_vfs exit status: " $LastExitCocde + $host.SetShouldExit($LastExitCode) + } $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$CMakeBuildType\test_assert.exe -d=yes" Invoke-Expression $cmds diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index c6c68362ed2f..0b9dd95d6096 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -191,6 +191,7 @@ jobs: ./tiledb/test/regression/tiledb_regression -d yes ./tiledb/test/ci/test_assert -d yes ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + ./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ################################################### # Stop helper processes, if applicable diff --git a/scripts/ci/build_libtiledb.sh b/scripts/ci/build_libtiledb.sh index f851a15830da..17adb2388b96 100644 --- a/scripts/ci/build_libtiledb.sh +++ b/scripts/ci/build_libtiledb.sh @@ -38,5 +38,6 @@ make -C tiledb install ls -la make -j4 -C tiledb tiledb_unit +make -j4 -C tiledb unit_vfs make -j4 -C tiledb tiledb_regression make -j4 -C tiledb all_link_complete From 2f8188e7682c285e441697a31892ed969fd1d44f Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 23 Feb 2024 00:56:17 -0800 Subject: [PATCH 201/456] Integrate std::pmr memory tracking for class ArraySchema. (#4696) Adding PMR tracking for all vector and unordered map variables of the ArraySchema class. --- TYPE: NO_HISTORY DESC: Integrate std::pmr memory tracking for class `ArraySchema`. --- test/src/unit-ReadCellSlabIter.cc | 16 +-- test/src/unit-Reader.cc | 6 +- test/src/unit-capi-array.cc | 4 +- test/src/unit-cppapi-schema-evolution.cc | 5 +- test/src/unit-enumerations.cc | 79 +++++++----- test/src/unit-request-handlers.cc | 31 +++-- test/src/unit-result-coords.cc | 2 +- test/src/unit-result-tile.cc | 10 +- .../unit-sparse-unordered-with-dups-reader.cc | 2 +- test/src/unit-tile-metadata-generator.cc | 14 ++- test/support/src/helpers.cc | 3 +- tiledb/common/memory_tracker.h | 12 +- tiledb/sm/array/CMakeLists.txt | 1 + tiledb/sm/array/array.cc | 2 +- tiledb/sm/array/array_directory.cc | 5 +- tiledb/sm/array/test/CMakeLists.txt | 5 +- tiledb/sm/array/test/unit_consistency.h | 5 +- tiledb/sm/array_schema/array_schema.cc | 119 +++++++++++++----- tiledb/sm/array_schema/array_schema.h | 56 ++++++--- .../sm/array_schema/array_schema_evolution.cc | 2 +- tiledb/sm/array_schema/dimension_label.cc | 8 +- tiledb/sm/array_schema/dimension_label.h | 5 +- tiledb/sm/array_schema/test/CMakeLists.txt | 1 + .../test/array_schema_test_support.h | 4 +- tiledb/sm/c_api/tiledb.cc | 29 +++-- tiledb/sm/c_api/tiledb_filestore.cc | 5 +- .../sm/consolidator/fragment_consolidator.cc | 2 +- tiledb/sm/consolidator/test/CMakeLists.txt | 2 +- .../test/unit_fragment_consolidator.cc | 5 +- tiledb/sm/fragment/fragment_info.cc | 2 +- tiledb/sm/fragment/fragment_metadata.cc | 12 +- tiledb/sm/fragment/fragment_metadata.h | 3 +- tiledb/sm/query/test/unit_query_condition.cc | 80 +++++++----- tiledb/sm/query/writers/writer_base.cc | 2 +- tiledb/sm/query_plan/test/unit_query_plan.cc | 5 +- tiledb/sm/rest/rest_client.cc | 39 ++++-- tiledb/sm/rest/rest_client.h | 7 +- tiledb/sm/serialization/array.cc | 34 ++--- tiledb/sm/serialization/array.h | 7 +- tiledb/sm/serialization/array_schema.cc | 56 +++++---- tiledb/sm/serialization/array_schema.h | 22 ++-- tiledb/sm/serialization/fragment_info.cc | 36 +++--- tiledb/sm/serialization/fragment_info.h | 6 +- tiledb/sm/serialization/query.cc | 21 +++- tiledb/sm/serialization/query.h | 3 +- tiledb/sm/serialization/test/CMakeLists.txt | 1 + .../test/unit_capnp_array_schema.cc | 9 +- .../sm/storage_manager/context_resources.cc | 2 +- tiledb/sm/subarray/subarray.cc | 2 +- .../sm/subarray/test/unit_add_ranges_list.cc | 7 +- tools/src/commands/info_command.cc | 2 +- 51 files changed, 524 insertions(+), 274 deletions(-) diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index 194f55cb9214..a9f5afc911a1 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -257,10 +257,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); @@ -331,10 +331,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); @@ -409,10 +409,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); } @@ -492,10 +492,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); } @@ -712,10 +712,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); @@ -898,10 +898,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); @@ -1097,10 +1097,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); @@ -1343,10 +1343,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); } diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 26d13cc93f6f..c7d78cc198c4 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -31,6 +31,7 @@ */ #include "test/support/src/helpers.h" +#include "test/support/src/mem_helpers.h" #include "test/support/src/vfs_helpers.h" #include "tiledb/common/common.h" #include "tiledb/common/dynamic_memory/dynamic_memory.h" @@ -38,6 +39,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/legacy/reader.h" @@ -250,7 +252,7 @@ TEST_CASE_METHOD( CHECK(dom->add_dimension(d1).ok()); CHECK(dom->add_dimension(d2).ok()); - auto schema = make_shared(HERE()); + auto schema = make_shared(HERE(), ArrayType::DENSE, tracker_); CHECK(schema->set_domain(dom).ok()); std::vector> fragments; @@ -258,10 +260,10 @@ TEST_CASE_METHOD( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), schema, URI(), std::make_pair(0, 0), + tracker_, true); fragments.emplace_back(std::move(fragment)); } diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 3da9120f9110..81f2e5b689b1 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -2586,11 +2586,13 @@ TEST_CASE_METHOD( 1, &buff); REQUIRE(rc == TILEDB_OK); + auto memory_tracker = ctx_->resources().create_memory_tracker(); auto st = tiledb::sm::serialization::array_deserialize( array->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), - ctx_->storage_manager()); + ctx_->storage_manager(), + memory_tracker); REQUIRE(st.ok()); // 6. Server: Close array and clean up diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 651b667759bb..ca85b4d5528c 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -31,6 +31,7 @@ */ #include +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" @@ -808,7 +809,9 @@ TEST_CASE( ase->set_timestamp_range(std::make_pair(1, 1)); auto schema = make_shared( - HERE(), tiledb::sm::ArrayType::SPARSE); + HERE(), + tiledb::sm::ArrayType::SPARSE, + tiledb::test::create_test_memory_tracker()); auto dim = make_shared( HERE(), "dim1", tiledb::sm::Datatype::INT32); int range[2] = {0, 1000}; diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 107e80053615..adaf63598f8b 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -32,6 +32,7 @@ #include +#include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" @@ -115,7 +116,7 @@ struct EnumerationFx { shared_ptr get_array_schema_latest(); // Serialization helpers - ArraySchema ser_des_array_schema( + shared_ptr ser_des_array_schema( shared_ptr schema, bool client_side, SerializationType stype); @@ -144,6 +145,7 @@ struct EnumerationFx { Config cfg_; Context ctx_; EncryptionKey enc_key_; + shared_ptr memory_tracker_; }; template @@ -1189,7 +1191,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Add Enumeration - Enumeration nullptr Error", "[enumeration][array-schema][error]") { - auto schema = make_shared(HERE()); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); REQUIRE_THROWS(schema->add_enumeration(nullptr)); } @@ -1197,7 +1200,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Add Basic Enumeration", "[enumeration][array-schema][basic]") { - auto schema = make_shared(HERE()); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); @@ -1212,7 +1216,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Get Enumeration", "[enumeration][array-schema][get]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr1 = create_enumeration(values); @@ -1226,7 +1231,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Get Missing Enumeration Error", "[enumeration][array-schema][error]") { - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); REQUIRE_THROWS(schema->get_enumeration("not_an_enumeration")); } @@ -1234,7 +1240,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Add Enumeration with Existing Enumeration of same Name", "[enumeration][array-schema][eror]") { - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); @@ -1246,7 +1253,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Add Attribute with Missing Enumeration Error", "[enumeration][array-schema][eror]") { - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto attr = make_shared(HERE(), "an_attr", Datatype::INT32); attr->set_enumeration_name("not_an_enumeration"); REQUIRE(!schema->add_attribute(attr).ok()); @@ -1256,7 +1264,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Get All Enumeration Names Empty", "[enumeration][array-schema][get-all][empty]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); auto enmr_names = schema->get_enumeration_names(); REQUIRE(enmr_names.size() == 0); } @@ -1265,7 +1274,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Get All Enumeration Names", "[enumeration][array-schema][get-all]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1.0f, 1.1f, 1.2f, 1.3f, 1.4f}; auto enmr1 = create_enumeration(values); @@ -1284,7 +1294,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Attribute with Invalid Datatype", "[enumeration][array-schema][error][bad-attr-datatype]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); @@ -1299,7 +1310,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Attribute with Invalid Cell Val Num", "[enumeration][array-schema][error][bad-attr-cell-val-num]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); @@ -1315,7 +1327,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Store nullptr Enumeration Error", "[enumeration][array-schema][error][store-nullptr-enumeration]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); REQUIRE_THROWS(schema->store_enumeration(nullptr)); } @@ -1323,7 +1336,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Store Enumeration Error", "[enumeration][array-schema][error][store-unknown-enumeration]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values, false, Datatype::INT32, "unknown_enmr"); @@ -1334,7 +1348,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Store Enumeration Error - Already Loaded", "[enumeration][array-schema][error][store-loaded-enumeration]") { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); std::vector values = {0, 1, 2, 100000000}; auto enmr = create_enumeration(values); @@ -1350,7 +1365,8 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Attribute Get Enumeration Name From Attribute", "[enumeration][array-schema][has-enumeration]") { - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); std::vector values = {"a", "spot", "of", "tea", "perhaps?"}; auto enmr = create_enumeration(values); @@ -1427,10 +1443,8 @@ TEST_CASE_METHOD( "[enumeration][array-schema][copy-ctor]") { auto schema = create_schema(); - // Check that the schema is valid and that we can copy it using the - // copy constructor. + // Check that the schema is valid. CHECK_NOTHROW(schema->check(cfg_)); - CHECK_NOTHROW(make_shared(HERE(), *(schema.get()))); } TEST_CASE_METHOD( @@ -1509,7 +1523,8 @@ TEST_CASE_METHOD( auto array = get_array(QueryType::READ); array->load_all_enumerations(); - auto schema = make_shared(HERE(), array->array_schema_latest()); + auto schema = array->array_schema_latest().clone(); + auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr"); auto matcher = Catch::Matchers::ContainsSubstring( @@ -1526,7 +1541,7 @@ TEST_CASE_METHOD( auto array = get_array(QueryType::READ); array->load_all_enumerations(); - auto schema = make_shared(HERE(), array->array_schema_latest()); + auto schema = array->array_schema_latest().clone(); auto enmr1 = schema->get_enumeration("test_enmr"); std::vector extra_values = {"manatee", "narwhal", "oppossum"}; @@ -2082,11 +2097,11 @@ TEST_CASE_METHOD( auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); auto all_names1 = schema1->get_enumeration_names(); - auto all_names2 = schema2.get_enumeration_names(); + auto all_names2 = schema2->get_enumeration_names(); REQUIRE(vec_cmp(all_names1, all_names2)); auto loaded_names1 = schema1->get_loaded_enumeration_names(); - auto loaded_names2 = schema2.get_loaded_enumeration_names(); + auto loaded_names2 = schema2->get_loaded_enumeration_names(); REQUIRE(vec_cmp(loaded_names1, loaded_names2)); // This is a new schema in RAM, so the loaded names should be the same @@ -2108,13 +2123,13 @@ TEST_CASE_METHOD( auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); auto all_names1 = schema1->get_enumeration_names(); - auto all_names2 = schema2.get_enumeration_names(); + auto all_names2 = schema2->get_enumeration_names(); REQUIRE(vec_cmp(all_names1, all_names2)); // This schema was deserialized from disk without any enumerations loaded // so both of these should be empty. auto loaded_names1 = schema1->get_loaded_enumeration_names(); - auto loaded_names2 = schema2.get_loaded_enumeration_names(); + auto loaded_names2 = schema2->get_loaded_enumeration_names(); REQUIRE(loaded_names1.empty()); REQUIRE(loaded_names2.empty()); @@ -2149,7 +2164,7 @@ TEST_CASE_METHOD( auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); auto all_names1 = schema1->get_enumeration_names(); - auto all_names2 = schema2.get_enumeration_names(); + auto all_names2 = schema2->get_enumeration_names(); REQUIRE(vec_cmp(all_names1, all_names2)); } @@ -2402,6 +2417,7 @@ EnumerationFx::EnumerationFx() , ctx_(cfg_) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); + memory_tracker_ = tiledb::test::create_test_memory_tracker(); } EnumerationFx::~EnumerationFx() { @@ -2672,7 +2688,8 @@ std::vector EnumerationFx::as_vector(shared_ptr enmr) { shared_ptr EnumerationFx::create_schema() { // Create a schema to serialize - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; @@ -2729,14 +2746,14 @@ shared_ptr EnumerationFx::get_array_schema_latest() { #ifdef TILEDB_SERIALIZATION -ArraySchema EnumerationFx::ser_des_array_schema( +shared_ptr EnumerationFx::ser_des_array_schema( shared_ptr schema, bool client_side, SerializationType stype) { Buffer buf; throw_if_not_ok(serialization::array_schema_serialize( *(schema.get()), stype, &buf, client_side)); - return serialization::array_schema_deserialize(stype, buf); + return serialization::array_schema_deserialize(stype, buf, memory_tracker_); } shared_ptr EnumerationFx::ser_des_array_schema_evolution( @@ -2779,13 +2796,13 @@ void EnumerationFx::ser_des_array( SerializationType stype) { Buffer buf; throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side)); - throw_if_not_ok( - serialization::array_deserialize(out, stype, buf, ctx.storage_manager())); + throw_if_not_ok(serialization::array_deserialize( + out, stype, buf, ctx.storage_manager(), memory_tracker_)); } #else // No TILEDB_SERIALIZATION -ArraySchema EnumerationFx::ser_des_array_schema( +shared_ptr EnumerationFx::ser_des_array_schema( shared_ptr, bool, SerializationType) { throw std::logic_error("Serialization not enabled."); } diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index f3378026adc3..5f134a292a63 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -32,8 +32,10 @@ #ifdef TILEDB_SERIALIZATION +#include "test/support/src/mem_helpers.h" #include "test/support/tdb_catch.h" #include "tiledb/api/c_api/buffer/buffer_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/api/c_api/string/string_api_internal.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/c_api/tiledb_serialization.h" @@ -72,7 +74,7 @@ struct HandleLoadArraySchemaRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override; - ArraySchema call_handler( + shared_ptr call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype); shared_ptr create_string_enumeration( @@ -94,7 +96,8 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override { - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = make_shared( + HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); @@ -118,8 +121,8 @@ TEST_CASE_METHOD( create_array(); auto schema = call_handler(serialization::LoadArraySchemaRequest(false), stype); - REQUIRE(schema.has_enumeration("enmr")); - REQUIRE(schema.get_loaded_enumeration_names().size() == 0); + REQUIRE(schema->has_enumeration("enmr")); + REQUIRE(schema->get_loaded_enumeration_names().size() == 0); } TEST_CASE_METHOD( @@ -131,10 +134,10 @@ TEST_CASE_METHOD( create_array(); auto schema = call_handler(serialization::LoadArraySchemaRequest(true), stype); - REQUIRE(schema.has_enumeration("enmr")); - REQUIRE(schema.get_loaded_enumeration_names().size() == 1); - REQUIRE(schema.get_loaded_enumeration_names()[0] == "enmr"); - REQUIRE(schema.get_enumeration("enmr") != nullptr); + REQUIRE(schema->has_enumeration("enmr")); + REQUIRE(schema->get_loaded_enumeration_names().size() == 1); + REQUIRE(schema->get_loaded_enumeration_names()[0] == "enmr"); + REQUIRE(schema->get_enumeration("enmr") != nullptr); } TEST_CASE_METHOD( @@ -394,7 +397,8 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize - auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto schema = make_shared( + HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); auto dim = make_shared(HERE(), "dim1", Datatype::INT32); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); @@ -414,7 +418,7 @@ shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { return schema; } -ArraySchema HandleLoadArraySchemaRequestFx::call_handler( +shared_ptr HandleLoadArraySchemaRequestFx::call_handler( serialization::LoadArraySchemaRequest req, SerializationType stype) { // If this looks weird, its because we're using the public C++ API to create // these objets instead of the internal APIs elsewhere in this test suite. @@ -435,12 +439,15 @@ ArraySchema HandleLoadArraySchemaRequestFx::call_handler( resp_buf); REQUIRE(rval == TILEDB_OK); + auto memory_tracker = + ctx.ptr()->context().resources().create_memory_tracker(); return serialization::deserialize_load_array_schema_response( - stype, resp_buf->buffer()); + stype, resp_buf->buffer(), memory_tracker); } shared_ptr HandleQueryPlanRequestFx::create_schema() { - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); schema->set_capacity(10000); throw_if_not_ok(schema->set_cell_order(Layout::ROW_MAJOR)); throw_if_not_ok(schema->set_tile_order(Layout::ROW_MAJOR)); diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index f86037a29df8..9dd75d266b7e 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -109,10 +109,10 @@ CResultCoordsFx::CResultCoordsFx(uint64_t num_cells) { frag_md_ = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); } diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index 62910cbb8138..bbac1ee67b2b 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -62,6 +62,7 @@ struct CResultTileFx { const char* ARRAY_NAME = "test_result_coords"; tiledb_array_t* array_; std::shared_ptr frag_md_; + shared_ptr memory_tracker_; CResultTileFx(); ~CResultTileFx(); @@ -109,13 +110,16 @@ CResultTileFx::CResultTileFx() { rc = tiledb_array_open(ctx_, array_, TILEDB_READ); REQUIRE(rc == TILEDB_OK); + // Create test memory tracker. + memory_tracker_ = tiledb::test::create_test_memory_tracker(); + frag_md_ = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + memory_tracker_, false); } @@ -191,10 +195,10 @@ TEST_CASE_METHOD( auto& array_schema = array_->array_->array_schema_latest(); FragmentMetadata frag_md( nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + memory_tracker_, true); ResultTile rt(0, 0, frag_md); @@ -301,10 +305,10 @@ TEST_CASE_METHOD( auto& array_schema = array_->array_->array_schema_latest(); FragmentMetadata frag_md( nullptr, - create_test_memory_tracker(), array_->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + memory_tracker_, true); ResultTile rt(0, 0, frag_md); diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 2ea31339a5e8..961b1e76f768 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -768,10 +768,10 @@ CSparseUnorderedWithDupsVarDataFx::open_default_array_1d_with_fragments( shared_ptr fragment = make_shared( HERE(), nullptr, - create_test_memory_tracker(), array->array_->array_schema_latest_ptr(), URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); fragments.emplace_back(std::move(fragment)); diff --git a/test/src/unit-tile-metadata-generator.cc b/test/src/unit-tile-metadata-generator.cc index b754f7fdac96..e254cf3537c3 100644 --- a/test/src/unit-tile-metadata-generator.cc +++ b/test/src/unit-tile-metadata-generator.cc @@ -34,8 +34,10 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/tile/tile_metadata_generator.h" #include "tiledb/sm/tile/writer_tile_tuple.h" @@ -80,7 +82,8 @@ TEMPLATE_LIST_TEST_CASE( // Generate the array schema. uint64_t num_cells = empty_tile ? 0 : 1000; - ArraySchema schema; + ArraySchema schema( + ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); schema.set_capacity(num_cells); Attribute a("a", tiledb_type); a.set_cell_val_num(cell_val_num); @@ -258,7 +261,8 @@ TEMPLATE_LIST_TEST_CASE( auto type = tiledb::impl::type_to_tiledb(); // Generate the array schema. - ArraySchema schema; + ArraySchema schema( + ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); schema.set_capacity(4); Attribute a("a", (Datatype)type.tiledb_type); CHECK(schema.add_attribute(make_shared(HERE(), a)).ok()); @@ -333,7 +337,8 @@ TEST_CASE( // Generate the array schema. uint64_t num_cells = empty_tile ? 0 : 20; - ArraySchema schema; + ArraySchema schema( + ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); schema.set_capacity(num_cells); Attribute a("a", Datatype::STRING_ASCII); a.set_cell_val_num(constants::var_num); @@ -433,7 +438,8 @@ TEST_CASE( "TileMetadataGenerator: var data tiles same string, different lengths", "[tile-metadata-generator][var-data][same-length]") { // Generate the array schema. - ArraySchema schema; + ArraySchema schema( + ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); schema.set_capacity(2); Attribute a("a", Datatype::CHAR); a.set_cell_val_num(constants::var_num); diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 251e6cfbd65b..7f2789d8bb02 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1593,7 +1593,8 @@ int array_open_wrapper( (*open_array)->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), - client_ctx->storage_manager()); + client_ctx->storage_manager(), + tiledb::test::create_test_memory_tracker()); REQUIRE(st.ok()); // 6. Server: Close array and clean up diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 81c5a74cc035..f06d3dc29ea5 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -110,17 +110,25 @@ enum class MemoryType { TILE_MAX_VALS, TILE_SUMS, TILE_NULL_COUNTS, - ENUMERATION + ATTRIBUTES, + ENUMERATION, + ENUMERATION_PATHS, + DIMENSION_LABELS, + DIMENSIONS }; /** The type of MemoryTracker. */ enum class MemoryTrackerType { ANONYMOUS, + ARRAY_CREATE, + ARRAY_LOAD, ARRAY_READ, ARRAY_WRITE, + FRAGMENT_INFO_LOAD, QUERY_READ, QUERY_WRITE, - CONSOLIDATOR + CONSOLIDATOR, + REST_CLIENT }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/sm/array/CMakeLists.txt b/tiledb/sm/array/CMakeLists.txt index c0b55afab26d..5aaebf284da9 100644 --- a/tiledb/sm/array/CMakeLists.txt +++ b/tiledb/sm/array/CMakeLists.txt @@ -35,6 +35,7 @@ commence(object_library array) this_target_object_libraries( array_schema baseline + context_resources fragment generic_tile_io uuid diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 7094cd3e6f57..dac76a6051e8 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1354,7 +1354,7 @@ Status Array::compute_max_buffer_sizes(const void* subarray) { last_max_buffer_sizes_.clear(); // Get all attributes and coordinates - auto attributes = array_schema_latest().attributes(); + auto& attributes = array_schema_latest().attributes(); last_max_buffer_sizes_.clear(); for (const auto& attr : attributes) last_max_buffer_sizes_[attr->name()] = diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 72615f674687..9d5fbae2cbf1 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -102,8 +102,9 @@ shared_ptr ArrayDirectory::load_array_schema_from_uri( // Deserialize Deserializer deserializer(tile.data(), tile.size()); - return make_shared( - HERE(), ArraySchema::deserialize(deserializer, schema_uri)); + auto memory_tracker = resources.create_memory_tracker(); + memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); + return ArraySchema::deserialize(deserializer, schema_uri, memory_tracker); } shared_ptr ArrayDirectory::load_array_schema_latest( diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 94b34c77bf70..272083ff20ea 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -29,11 +29,10 @@ include(unit_test) commence(unit_test array) this_target_compile_definitions(-DTILEDB_TEST_INPUTS_DIR="${CMAKE_SOURCE_DIR}/test/inputs/") this_target_sources(main.cc unit_array_directory.cc) - this_target_link_libraries(array context_resources) + this_target_link_libraries(array) conclude(unit_test) commence(unit_test consistency) this_target_sources(main.cc unit_consistency.cc) - this_target_link_libraries(TILEDB_CORE_OBJECTS) - this_target_link_libraries(TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 8e2aea539fe2..15971fd172d7 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -36,6 +36,7 @@ #include #include +#include "test/support/src/mem_helpers.h" #include "../array.h" #include "../consistency.h" @@ -106,8 +107,8 @@ class WhiteboxConsistencyController : public ConsistencyController { make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); // Create the ArraySchema - shared_ptr schema = - make_shared(HERE(), ArrayType::DENSE); + shared_ptr schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); throw_if_not_ok(schema->set_domain(domain)); throw_if_not_ok(schema->add_attribute( make_shared( diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 95fc59ace895..2522e5601568 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" @@ -53,6 +54,7 @@ #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/integral_type_casts.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/type/apply_with_type.h" @@ -79,12 +81,10 @@ class ArraySchemaException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -ArraySchema::ArraySchema() - : ArraySchema(ArrayType::DENSE) { -} - -ArraySchema::ArraySchema(ArrayType array_type) - : uri_(URI()) +ArraySchema::ArraySchema( + ArrayType array_type, shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , uri_(URI()) , array_uri_(URI()) , version_(constants::format_version) , timestamp_range_(std::make_pair( @@ -93,9 +93,19 @@ ArraySchema::ArraySchema(ArrayType array_type) , array_type_(array_type) , allows_dups_(false) , domain_(nullptr) + , dim_map_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , cell_order_(Layout::ROW_MAJOR) , tile_order_(Layout::ROW_MAJOR) - , capacity_(constants::capacity) { + , capacity_(constants::capacity) + , attributes_(memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , attribute_map_(memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , dimension_labels_( + memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) + , dimension_label_map_( + memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) + , enumeration_map_(memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumeration_path_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) { // Set up default filter pipelines for coords, offsets, and validity values. coords_filters_.add_filter(CompressionFilter( constants::coords_compression, @@ -131,23 +141,44 @@ ArraySchema::ArraySchema( std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, - FilterPipeline coords_filters) - : uri_(uri) + FilterPipeline coords_filters, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , uri_(uri) , version_(version) , timestamp_range_(timestamp_range) , name_(name) , array_type_(array_type) , allows_dups_(allows_dups) , domain_(domain) + , dim_map_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , cell_order_(cell_order) , tile_order_(tile_order) , capacity_(capacity) - , attributes_(attributes) - , dimension_labels_(dim_label_refs) - , enumeration_path_map_(enumeration_path_map) + , attributes_(memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , attribute_map_(memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , dimension_labels_( + memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) + , dimension_label_map_( + memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) + , enumeration_map_(memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumeration_path_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) , cell_var_offsets_filters_(cell_var_offsets_filters) , cell_validity_filters_(cell_validity_filters) , coords_filters_(coords_filters) { + for (auto atr : attributes) { + attributes_.push_back(atr); + } + + for (auto dim_label : dim_label_refs) { + dimension_labels_.push_back(dim_label); + } + + for (auto& elem : enumeration_path_map) { + enumeration_path_map_.insert(elem); + } + // Create dimension map for (dimension_size_type d = 0; d < domain_->dim_num(); ++d) { auto dim{domain_->dimension_ptr(d)}; @@ -192,30 +223,37 @@ ArraySchema::ArraySchema( check_attribute_dimension_label_names(); } -/* - * Copy constructor manually initializes its map members, so we don't use the - * default copy constructor. At some point this may no longer hold and we can - * eliminate this code in favor of the default. - */ ArraySchema::ArraySchema(const ArraySchema& array_schema) - : uri_{array_schema.uri_} + : memory_tracker_{array_schema.memory_tracker_} + , uri_{array_schema.uri_} , array_uri_{array_schema.array_uri_} , version_{array_schema.version_} , timestamp_range_{array_schema.timestamp_range_} , name_{array_schema.name_} , array_type_{array_schema.array_type_} , allows_dups_{array_schema.allows_dups_} - , domain_{} // copied below by `set_domain` - , dim_map_{} // initialized in `set_domain` + , domain_{} // copied below by `set_domain` + , dim_map_(memory_tracker_->get_resource( + MemoryType::DIMENSIONS)) // initialized in `set_domain` , cell_order_{array_schema.cell_order_} , tile_order_{array_schema.tile_order_} , capacity_{array_schema.capacity_} - , attributes_{array_schema.attributes_} - , attribute_map_{array_schema.attribute_map_} - , dimension_labels_{} // copied in loop below - , dimension_label_map_{} // initialized below - , enumeration_map_{array_schema.enumeration_map_} - , enumeration_path_map_{array_schema.enumeration_path_map_} + , attributes_( + array_schema.attributes_, + memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , attribute_map_( + array_schema.attribute_map_, + memory_tracker_->get_resource(MemoryType::ATTRIBUTES)) + , dimension_labels_(memory_tracker_->get_resource( + MemoryType::DIMENSION_LABELS)) // copied in loop below + , dimension_label_map_( + memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) + , enumeration_map_( + array_schema.enumeration_map_, + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumeration_path_map_( + array_schema.enumeration_path_map_, + memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) , cell_var_offsets_filters_{array_schema.cell_var_offsets_filters_} , cell_validity_filters_{array_schema.cell_validity_filters_} , coords_filters_{array_schema.coords_filters_} @@ -273,7 +311,7 @@ shared_ptr ArraySchema::shared_attribute( return attributes_[it->second.index]; } -const std::vector>& ArraySchema::attributes() +const tdb::pmr::vector>& ArraySchema::attributes() const { return attributes_; } @@ -1023,7 +1061,14 @@ void ArraySchema::add_dimension_label( // Create the dimension label reference. auto dim_label_ref = make_shared( - HERE(), dim_id, name, uri, dim, label_order, label_type); + HERE(), + dim_id, + name, + uri, + dim, + label_order, + label_type, + memory_tracker_); dimension_labels_.emplace_back(dim_label_ref); dimension_label_map_[name] = dim_label_ref.get(); } catch (...) { @@ -1250,8 +1295,10 @@ void ArraySchema::drop_enumeration(const std::string& enmr_name) { } // #TODO Add security validation on incoming URI -ArraySchema ArraySchema::deserialize( - Deserializer& deserializer, const URI& uri) { +shared_ptr ArraySchema::deserialize( + Deserializer& deserializer, + const URI& uri, + shared_ptr memory_tracker) { Status st; // Load version // #TODO Add security validation @@ -1392,7 +1439,8 @@ ArraySchema ArraySchema::deserialize( // Set schema name std::string name = uri.last_path_part(); - return ArraySchema( + return make_shared( + HERE(), uri, version, timestamp_range, @@ -1405,13 +1453,18 @@ ArraySchema ArraySchema::deserialize( capacity, attributes, dimension_labels, - {}, + std::vector>(), enumeration_path_map, cell_var_filters, cell_validity_filters, FilterPipeline( coords_filters, - version < 5 ? domain->dimension_ptr(0)->type() : Datatype::UINT64)); + version < 5 ? domain->dimension_ptr(0)->type() : Datatype::UINT64), + memory_tracker); +} + +shared_ptr ArraySchema::clone() const { + return make_shared(HERE(), *this); } Status ArraySchema::set_allows_dups(bool allows_dups) { diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index f41308eec537..b2e91fc31ed4 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -37,12 +37,12 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" -#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -56,6 +56,7 @@ class Dimension; class DimensionLabel; class Domain; class Enumeration; +class MemoryTracker; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -92,12 +93,15 @@ class ArraySchema { /* ********************************* */ /** Constructor. */ - ArraySchema(); - - /** Constructor. */ - ArraySchema(ArrayType array_type); + ArraySchema() = delete; /** Constructor. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + */ + ArraySchema(ArrayType array_type, shared_ptr memory_tracker); + + /** Constructor with std::vector attributes. * @param uri The URI of the array schema file. * @param version The format version of this array schema. * @param timestamp_range The timestamp the array schema was written. @@ -117,6 +121,8 @@ class ArraySchema { * @param cell_validity_filters * The filter pipeline run on validity tiles for nullable attributes. * @param coords_filters The filter pipeline run on coordinate tiles. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. **/ ArraySchema( URI uri, @@ -135,14 +141,18 @@ class ArraySchema { std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, - FilterPipeline coords_filters); + FilterPipeline coords_filters, + shared_ptr memory_tracker); /** - * Constructor. Clones the input. + * Copy constructor. Clones the input. * * @param array_schema The array schema to copy. */ - explicit ArraySchema(const ArraySchema& array_schema); + ArraySchema(const ArraySchema& array_schema); + + DISABLE_COPY_ASSIGN(ArraySchema); + DISABLE_MOVE_AND_MOVE_ASSIGN(ArraySchema); /** Destructor. */ ~ArraySchema() = default; @@ -202,7 +212,7 @@ class ArraySchema { } /** Returns the attributes. */ - const std::vector>& attributes() const; + const tdb::pmr::vector>& attributes() const; /** Returns the capacity. */ uint64_t capacity() const; @@ -468,7 +478,13 @@ class ArraySchema { * @param uri The uri of the Array. * @return A new ArraySchema. */ - static ArraySchema deserialize(Deserializer& deserializer, const URI& uri); + static shared_ptr deserialize( + Deserializer& deserializer, + const URI& uri, + shared_ptr memory_tracker); + + /** Return a cloned copy of this array schema. */ + shared_ptr clone() const; /** Returns the array domain. */ inline const Domain& domain() const { @@ -574,6 +590,11 @@ class ArraySchema { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the ArraySchema. + */ + shared_ptr memory_tracker_; + /** The URI of the array schema file. */ URI uri_; @@ -607,7 +628,7 @@ class ArraySchema { shared_ptr domain_; /** It maps each dimension name to the corresponding dimension object. */ - std::unordered_map dim_map_; + tdb::pmr::unordered_map dim_map_; /** * The cell order. It can be one of the following: @@ -633,7 +654,7 @@ class ArraySchema { * within this array schema. Other member variables reference objects within * this container. */ - std::vector> attributes_; + tdb::pmr::vector> attributes_; /** * Type for the range of the map that is member `attribute_map_`. See the @@ -654,20 +675,21 @@ class ArraySchema { * Invariant: The number of entries in `attribute_map_` is the same as the * number of entries in `attributes_` */ - std::unordered_map attribute_map_; + tdb::pmr::unordered_map attribute_map_; /** The array dimension labels. */ - std::vector> dimension_labels_; + tdb::pmr::vector> dimension_labels_; /** A map from the dimension label names to the label schemas. */ - std::unordered_map dimension_label_map_; + tdb::pmr::unordered_map + dimension_label_map_; /** A map of Enumeration names to Enumeration pointers. */ - std::unordered_map> + tdb::pmr::unordered_map> enumeration_map_; /** A map of Enumeration names to Enumeration URIs */ - std::unordered_map enumeration_path_map_; + tdb::pmr::unordered_map enumeration_path_map_; /** The filter pipeline run on offset tiles for var-length attributes. */ FilterPipeline cell_var_offsets_filters_; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 7b0c3d1e90bf..edfdfc7e70f2 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -106,7 +106,7 @@ shared_ptr ArraySchemaEvolution::evolve_schema( "Cannot evolve schema; Input array schema is null"); } - auto schema = make_shared(HERE(), *(orig_schema.get())); + auto schema = orig_schema->clone(); // Add enumerations. Must be done before attributes so that any attributes // referencing enumerations won't fail to be added. diff --git a/tiledb/sm/array_schema/dimension_label.cc b/tiledb/sm/array_schema/dimension_label.cc index 0d41a3ff200a..f4e439eb6a9f 100644 --- a/tiledb/sm/array_schema/dimension_label.cc +++ b/tiledb/sm/array_schema/dimension_label.cc @@ -128,7 +128,8 @@ DimensionLabel::DimensionLabel( const URI& uri, const Dimension* dim, DataOrder label_order, - Datatype label_type) + Datatype label_type, + shared_ptr memory_tracker) : dim_id_(dim_id) , dim_label_name_(dim_label_name) , uri_(uri) @@ -139,8 +140,9 @@ DimensionLabel::DimensionLabel( label_type == Datatype::STRING_ASCII ? constants::var_num : 1) , schema_(make_shared( HERE(), - label_order == DataOrder::UNORDERED_DATA ? ArrayType::SPARSE : - ArrayType::DENSE)) + (label_order == DataOrder::UNORDERED_DATA ? ArrayType::SPARSE : + ArrayType::DENSE), + memory_tracker)) , is_external_(false) , relative_uri_(true) { auto index_type{dim->type()}; diff --git a/tiledb/sm/array_schema/dimension_label.h b/tiledb/sm/array_schema/dimension_label.h index 3d91c8ffeda7..6a69adc7c969 100644 --- a/tiledb/sm/array_schema/dimension_label.h +++ b/tiledb/sm/array_schema/dimension_label.h @@ -48,6 +48,7 @@ class ArraySchema; class Buffer; class ConstBuffer; class Dimension; +class MemoryTracker; enum class Datatype : uint8_t; enum class DataOrder : uint8_t; @@ -107,6 +108,7 @@ class DimensionLabel { /** * Constructor for an internally generated dimension label. * + * @param memory_tracker Memory tracker for the dimension label. * @param dim_id The index of the dimension the label is attached to. * @param dim_label_name The name of the dimension label. * @param uri The URI of an external dimension label. @@ -120,7 +122,8 @@ class DimensionLabel { const URI& uri, const Dimension* dim, DataOrder label_order, - Datatype label_type); + Datatype label_type, + shared_ptr memory_tracker); /** * Populates the object members from the data in the input binary buffer. diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index e8a03ec32b94..efb004a35ee4 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -38,4 +38,5 @@ commence(unit_test array_schema) unit_domain_data.cc unit_tile_domain.cc ) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index 5728a96b4923..c4b18720f6cd 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -72,6 +72,7 @@ #include #include +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -313,7 +314,8 @@ class TestArraySchema { {}, // the second enumeration thing FilterPipeline(), FilterPipeline(), - FilterPipeline()) { + FilterPipeline(), + tiledb::test::create_test_memory_tracker()) { } /** diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 7bafd132752c..9da381e74394 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -52,6 +52,7 @@ #include "tiledb/common/dynamic_memory/dynamic_memory.h" #include "tiledb/common/heap_profiler.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension_label.h" @@ -257,8 +258,10 @@ int32_t tiledb_array_schema_alloc( } // Create a new ArraySchema object + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); (*array_schema)->array_schema_ = make_shared( - HERE(), static_cast(array_type)); + HERE(), static_cast(array_type), memory_tracker); if ((*array_schema)->array_schema_ == nullptr) { auto st = Status_Error("Failed to allocate TileDB array schema object"); LOG_STATUS_NO_RETURN_VALUE(st); @@ -3438,13 +3441,16 @@ int32_t tiledb_deserialize_array( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_deserialize( (*array)->array_.get(), (tiledb::sm::SerializationType)serialize_type, buffer->buffer(), - ctx->storage_manager()))) { + ctx->storage_manager(), + memory_tracker))) { delete *array; *array = nullptr; return TILEDB_ERR; @@ -3500,10 +3506,13 @@ int32_t tiledb_deserialize_array_schema( } try { - (*array_schema)->array_schema_ = make_shared( - HERE(), + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); + (*array_schema)->array_schema_ = tiledb::sm::serialization::array_schema_deserialize( - (tiledb::sm::SerializationType)serialize_type, buffer->buffer())); + (tiledb::sm::SerializationType)serialize_type, + buffer->buffer(), + memory_tracker); } catch (...) { delete *array_schema; *array_schema = nullptr; @@ -3766,11 +3775,14 @@ int32_t tiledb_deserialize_query_and_array( } // First deserialize the array included in the query + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_LOAD); throw_if_not_ok(tiledb::sm::serialization::array_from_query_deserialize( buffer->buffer(), (tiledb::sm::SerializationType)serialize_type, *(*array)->array_, - ctx->storage_manager())); + ctx->storage_manager(), + memory_tracker)); // Create query struct *query = new (std::nothrow) tiledb_query_t; @@ -4271,13 +4283,16 @@ int32_t tiledb_deserialize_fragment_info( return TILEDB_ERR; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::FRAGMENT_INFO_LOAD); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::fragment_info_deserialize( fragment_info->fragment_info_, (tiledb::sm::SerializationType)serialize_type, uri, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { return TILEDB_ERR; } diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 9c0a38342a51..7d4eb8cff8c6 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -36,6 +36,7 @@ #include "tiledb/api/c_api/dimension/dimension_api_internal.h" #include "tiledb/api/c_api_support/c_api_support.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -116,8 +117,10 @@ int32_t tiledb_filestore_schema_create( // All other calls for adding domains, attributes, etc // create copies of the underlying core objects from within // the cpp objects constructed here + auto memory_tracker = context.resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); (*array_schema)->array_schema_ = make_shared( - HERE(), tiledb::sm::ArrayType::DENSE); + HERE(), tiledb::sm::ArrayType::DENSE, memory_tracker); auto& schema = (*array_schema)->array_schema_; // Define the range of the dimension. diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index aa42ee99cfe5..6a37c8ee2856 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -841,7 +841,7 @@ void FragmentConsolidator::set_query_buffers( const auto& array_schema = query->array_schema(); auto dim_num = array_schema.dim_num(); auto dense = array_schema.dense(); - auto attributes = array_schema.attributes(); + auto& attributes = array_schema.attributes(); unsigned bid = 0; // Here the first buffer should always be the fixed buffer (either offsets diff --git a/tiledb/sm/consolidator/test/CMakeLists.txt b/tiledb/sm/consolidator/test/CMakeLists.txt index 3d97a4d2095f..fdc64b244be3 100644 --- a/tiledb/sm/consolidator/test/CMakeLists.txt +++ b/tiledb/sm/consolidator/test/CMakeLists.txt @@ -29,5 +29,5 @@ commence(unit_test consolidator) this_target_sources(main.cc unit_fragment_consolidator.cc) # Not actually testing a unit yet, but some things that ought to be units - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index 3e2f9d9e2fb1..1fb78d3b2778 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -32,6 +32,7 @@ #include #include "../fragment_consolidator.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" @@ -47,7 +48,9 @@ shared_ptr make_schema( const std::vector attr_nullable) { // Initialize the array schema. shared_ptr array_schema = make_shared( - HERE(), sparse ? ArrayType::SPARSE : ArrayType::DENSE); + HERE(), + sparse ? ArrayType::SPARSE : ArrayType::DENSE, + tiledb::test::create_test_memory_tracker()); // Create the domain/dimensions. Domain domain; diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 941b9313487a..7f089a84e5ed 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -1123,10 +1123,10 @@ tuple> FragmentInfo::load( auto meta = make_shared( HERE(), resources_, - resources_->create_memory_tracker(), array_schema_latest, new_fragment_uri, timestamp_range, + resources_->create_memory_tracker(), !sparse); meta->load(enc_key_, nullptr, 0, array_schemas_all_); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 65064eb4e2d8..0fc003ea5dce 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -100,10 +100,10 @@ FragmentMetadata::FragmentMetadata( FragmentMetadata::FragmentMetadata( ContextResources* resources, - shared_ptr memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, const std::pair& timestamp_range, + shared_ptr memory_tracker, bool dense, bool has_timestamps, bool has_deletes_meta) @@ -796,20 +796,20 @@ std::vector> FragmentMetadata::load( metadata = make_shared( HERE(), &resources, - memory_tracker, array_schema_latest, sf.uri_, sf.timestamp_range_, + memory_tracker, !sparse); } else { // Fragment format version > 2 metadata = make_shared( HERE(), &resources, - memory_tracker, array_schema_latest, sf.uri_, - sf.timestamp_range_); + sf.timestamp_range_, + memory_tracker); } // Potentially find the basic fragment metadata in the consolidated @@ -1316,7 +1316,7 @@ std::string FragmentMetadata::encode_name(const std::string& name) const { const unsigned idx = iter->second; - auto attributes = array_schema_->attributes(); + auto& attributes = array_schema_->attributes(); for (unsigned i = 0; i < attributes.size(); ++i) { const std::string attr_name = attributes[i]->name(); if (attr_name == name) { @@ -4661,7 +4661,7 @@ const shared_ptr& FragmentMetadata::array_schema() const { void FragmentMetadata::build_idx_map() { idx_map_.clear(); - auto attributes = array_schema_->attributes(); + auto& attributes = array_schema_->attributes(); for (unsigned i = 0; i < attributes.size(); ++i) { auto attr_name = attributes[i]->name(); idx_map_[attr_name] = i; diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 8321b248f856..a79ea251f965 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -91,16 +91,17 @@ class FragmentMetadata { * @param timestamp_range The timestamp range of the fragment. * In TileDB, timestamps are in ms elapsed since * 1970-01-01 00:00:00 +0000 (UTC). + * @param memory_tracker Memory tracker for the fragment metadata. * @param dense Indicates whether the fragment is dense or sparse. * @param has_timestamps Does the fragment contains timestamps. * @param has_delete_meta Does the fragment contains delete metadata. */ FragmentMetadata( ContextResources* resources, - shared_ptr memory_tracker, const shared_ptr& array_schema, const URI& fragment_uri, const std::pair& timestamp_range, + shared_ptr memory_tracker, bool dense = true, bool has_timestamps = false, bool has_delete_mata = false); diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index b4a8c6b0fb96..7c56e08278c1 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -38,6 +38,7 @@ #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_condition_combination_op.h" #include "tiledb/sm/enums/query_condition_op.h" @@ -61,7 +62,8 @@ TEST_CASE( REQUIRE(query_condition.empty()); REQUIRE(query_condition.field_names().empty()); - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); std::vector result_cell_slabs; std::vector> frag_md; REQUIRE( @@ -200,7 +202,8 @@ TEST_CASE("QueryCondition: Test blob type", "[QueryCondition][blob]") { QueryConditionOp::LT) .ok()); - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); shared_ptr attr = make_shared(HERE(), "blob_attr", Datatype::BLOB); REQUIRE(array_schema->add_attribute(attr).ok()); @@ -1123,10 +1126,10 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); REQUIRE( query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); @@ -1160,10 +1163,10 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); REQUIRE(query_condition_eq_null .apply(*array_schema, frag_md, result_cell_slabs_eq_null, 1) @@ -1309,10 +1312,10 @@ void test_apply_cells( frag_md[0] = make_shared( HERE(), nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); REQUIRE( query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); @@ -1568,7 +1571,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -1591,10 +1595,10 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -1629,7 +1633,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -1647,10 +1652,10 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -1731,7 +1736,8 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -1759,10 +1765,10 @@ TEST_CASE( frag_md[0] = make_shared( HERE(), nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -2280,7 +2286,8 @@ void test_apply_dense( const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2306,10 +2313,10 @@ void test_apply_dense( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -2344,7 +2351,8 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -2365,10 +2373,10 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -2450,7 +2458,8 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2476,10 +2485,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -2979,7 +2988,8 @@ void test_apply_sparse( const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -3005,10 +3015,10 @@ void test_apply_sparse( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -3043,7 +3053,8 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); @@ -3064,10 +3075,10 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -3215,10 +3226,10 @@ void validate_qc_apply( frag_md[0] = make_shared( HERE(), nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); REQUIRE(tp.qc_.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); REQUIRE(result_cell_slabs.size() == tp.expected_slabs_.size()); @@ -3815,7 +3826,8 @@ TEST_CASE( const Datatype type = Datatype::UINT64; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); @@ -3834,10 +3846,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -4100,7 +4112,8 @@ TEST_CASE( const Datatype type = GENERATE(Datatype::STRING_ASCII, Datatype::STRING_UTF8); // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4120,10 +4133,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -4454,7 +4467,8 @@ TEST_CASE( const Datatype type = Datatype::STRING_UTF8; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4474,10 +4488,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // For pasting into a Python shell: @@ -4776,7 +4790,8 @@ TEST_CASE( const Datatype type = Datatype::FLOAT32; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(true); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) @@ -4796,10 +4811,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. @@ -4872,7 +4887,8 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared(HERE()); + shared_ptr array_schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -4895,10 +4911,10 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, - tiledb::test::create_test_memory_tracker(), array_schema, URI(), std::make_pair(0, 0), + tiledb::test::create_test_memory_tracker(), true); // Initialize the result tile. diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 4c9dfb9c7254..4c807520382e 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -792,10 +792,10 @@ Status WriterBase::create_fragment( frag_meta = make_shared( HERE(), &storage_manager_->resources(), - array_memory_tracker_, array_->array_schema_latest_ptr(), fragment_uri_, timestamp_range, + array_memory_tracker_, dense, has_timestamps, has_delete_metadata); diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 23ce164c092b..af9790ce532f 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -30,6 +30,7 @@ * This file tests the QueryPlan class */ +#include #include #include #include "../query_plan.h" @@ -80,8 +81,8 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); // Create the ArraySchema - shared_ptr schema = - make_shared(HERE(), ArrayType::DENSE); + shared_ptr schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); throw_if_not_ok(schema->set_domain(domain)); throw_if_not_ok(schema->add_attribute( make_shared( diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index f69b55c4d234..2b9f6f758041 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -57,6 +57,7 @@ #include #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/group/group.h" @@ -81,7 +82,8 @@ RestClient::RestClient() : stats_(nullptr) , config_(nullptr) , compute_tp_(nullptr) - , resubmit_incomplete_(true) { + , resubmit_incomplete_(true) + , memory_tracker_(nullptr) { auto st = utils::parse::convert( Config::REST_SERIALIZATION_DEFAULT_FORMAT, &serialization_type_); throw_if_not_ok(st); @@ -91,7 +93,8 @@ Status RestClient::init( stats::Stats* const parent_stats, const Config* config, ThreadPool* compute_tp, - const std::shared_ptr& logger) { + const std::shared_ptr& logger, + ContextResources& resources) { if (config == nullptr) return LOG_STATUS( Status_RestError("Error initializing rest client; config is null.")); @@ -103,6 +106,12 @@ Status RestClient::init( config_ = config; compute_tp_ = compute_tp; + // Setting the type of the memory tracker as MemoryTrackerType::REST_CLIENT + // for now. This is because the class is used in many places not directly tied + // to an array. + memory_tracker_ = resources.create_memory_tracker(); + memory_tracker_->set_type(MemoryTrackerType::REST_CLIENT); + const char* c_str; RETURN_NOT_OK(config_->get("rest.server_address", &c_str)); if (c_str != nullptr) @@ -239,10 +248,8 @@ RestClient::get_array_schema_from_rest(const URI& uri) { ensure_json_null_delimited_string(&returned_data), nullopt); return { Status::Ok(), - make_shared( - HERE(), - serialization::array_schema_deserialize( - serialization_type_, returned_data))}; + serialization::array_schema_deserialize( + serialization_type_, returned_data, memory_tracker_)}; } shared_ptr RestClient::post_array_schema_from_rest( @@ -289,10 +296,8 @@ shared_ptr RestClient::post_array_schema_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); - return make_shared( - HERE(), - serialization::deserialize_load_array_schema_response( - serialization_type_, returned_data)); + return serialization::deserialize_load_array_schema_response( + serialization_type_, returned_data, memory_tracker_); } Status RestClient::post_array_schema_to_rest( @@ -376,7 +381,11 @@ Status RestClient::post_array_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::array_deserialize( - array, serialization_type_, returned_data, storage_manager); + array, + serialization_type_, + returned_data, + storage_manager, + memory_tracker_); } void RestClient::delete_array_from_rest(const URI& uri) { @@ -1265,7 +1274,7 @@ Status RestClient::post_fragment_info_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::fragment_info_deserialize( - fragment_info, serialization_type_, uri, returned_data); + fragment_info, serialization_type_, uri, returned_data, memory_tracker_); } Status RestClient::post_group_metadata_from_rest(const URI& uri, Group* group) { @@ -1525,7 +1534,11 @@ RestClient::RestClient() { } Status RestClient::init( - stats::Stats*, const Config*, ThreadPool*, const std::shared_ptr&) { + stats::Stats*, + const Config*, + ThreadPool*, + const std::shared_ptr&, + ContextResources&) { return LOG_STATUS( Status_RestError("Cannot use rest client; serialization not enabled.")); } diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 63d9044db9e2..1d9f476f320e 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -53,6 +53,7 @@ class ArraySchemaEvolution; class Config; class FragmentInfo; class Query; +class MemoryTracker; enum class SerializationType : uint8_t; @@ -66,7 +67,8 @@ class RestClient { stats::Stats* parent_stats, const Config* config, ThreadPool* compute_tp, - const std::shared_ptr& logger); + const std::shared_ptr& logger, + ContextResources& resources); /** Sets a header that will be attached to all requests. */ Status set_header(const std::string& name, const std::string& value); @@ -434,6 +436,9 @@ class RestClient { /** UID of the logger instance */ inline static std::atomic logger_id_ = 0; + /** The class MemoryTracker. */ + shared_ptr memory_tracker_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 6e1e4f04aa78..a010cd39ce4a 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -232,7 +232,8 @@ Status array_from_capnp( const capnp::Array::Reader& array_reader, StorageManager* storage_manager, Array* array, - const bool client_side) { + const bool client_side, + shared_ptr memory_tracker) { // The serialized URI is set if it exists // this is used for backwards compatibility with pre TileDB 2.5 clients that // want to serialized a query object TileDB >= 2.5 no longer needs to receive @@ -269,11 +270,10 @@ Status array_from_capnp( if (all_schemas_reader.hasEntries()) { auto entries = array_reader.getArraySchemasAll().getEntries(); for (auto array_schema_build : entries) { - auto schema{array_schema_from_capnp( - array_schema_build.getValue(), array->array_uri())}; - schema.set_array_uri(array->array_uri()); - all_schemas[array_schema_build.getKey()] = - make_shared(HERE(), schema); + auto schema = array_schema_from_capnp( + array_schema_build.getValue(), array->array_uri(), memory_tracker); + schema->set_array_uri(array->array_uri()); + all_schemas[array_schema_build.getKey()] = schema; } } array->set_array_schemas_all(std::move(all_schemas)); @@ -282,10 +282,9 @@ Status array_from_capnp( if (array_reader.hasArraySchemaLatest()) { auto array_schema_latest_reader = array_reader.getArraySchemaLatest(); auto array_schema_latest{array_schema_from_capnp( - array_schema_latest_reader, array->array_uri())}; - array_schema_latest.set_array_uri(array->array_uri()); - array->set_array_schema_latest( - make_shared(HERE(), array_schema_latest)); + array_schema_latest_reader, array->array_uri(), memory_tracker)}; + array_schema_latest->set_array_uri(array->array_uri()); + array->set_array_schema_latest(array_schema_latest); } // Deserialize array directory @@ -543,7 +542,8 @@ Status array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager) { + StorageManager* storage_manager, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -555,7 +555,8 @@ Status array_deserialize( kj::StringPtr(static_cast(serialized_buffer.data())), array_builder); capnp::Array::Reader array_reader = array_builder.asReader(); - RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array)); + RETURN_NOT_OK(array_from_capnp( + array_reader, storage_manager, array, true, memory_tracker)); break; } case SerializationType::CAPNP: { @@ -575,7 +576,8 @@ Status array_deserialize( serialized_buffer.size() / sizeof(::capnp::word)), readerOptions); capnp::Array::Reader array_reader = reader.getRoot(); - RETURN_NOT_OK(array_from_capnp(array_reader, storage_manager, array)); + RETURN_NOT_OK(array_from_capnp( + array_reader, storage_manager, array, true, memory_tracker)); break; } default: { @@ -708,7 +710,11 @@ Status array_serialize(Array*, SerializationType, Buffer*, const bool) { } Status array_deserialize( - Array*, SerializationType, const Buffer&, StorageManager*) { + Array*, + SerializationType, + const Buffer&, + StorageManager*, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array.h b/tiledb/sm/serialization/array.h index bef9d5143de4..a08b531b1e9a 100644 --- a/tiledb/sm/serialization/array.h +++ b/tiledb/sm/serialization/array.h @@ -74,13 +74,15 @@ Status array_to_capnp( * @param client_side Allows to specify different behavior depending on who is * serializing, the client (1) or the Cloud server (0). This is sometimes needed * since they are both using the same Core library APIs for serialization. + * @param memory_tracker Memory tracker to use on the deserialized object. * @return Status */ Status array_from_capnp( const capnp::Array::Reader& array_reader, StorageManager* storage_manager, Array* array, - const bool client_side = true); + const bool client_side, + shared_ptr memory_tracker); /** * Convert info for opening and array to Cap'n Proto message @@ -135,7 +137,8 @@ Status array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager); + StorageManager* storage_manager, + shared_ptr memory_tracker); /** * Serialize an open array request via Cap'n Proto diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index bc36b4ed970f..40a0418cfbac 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -780,7 +780,8 @@ void dimension_label_to_capnp( } shared_ptr dimension_label_from_capnp( - const capnp::DimensionLabel::Reader& dim_label_reader) { + const capnp::DimensionLabel::Reader& dim_label_reader, + shared_ptr memory_tracker) { // Get datatype Datatype datatype = Datatype::ANY; throw_if_not_ok(datatype_enum(dim_label_reader.getType(), &datatype)); @@ -788,8 +789,7 @@ shared_ptr dimension_label_from_capnp( shared_ptr schema{nullptr}; if (dim_label_reader.hasSchema()) { auto schema_reader = dim_label_reader.getSchema(); - schema = make_shared( - HERE(), array_schema_from_capnp(schema_reader, URI())); + schema = array_schema_from_capnp(schema_reader, URI(), memory_tracker); } auto is_relative = dim_label_reader.getRelative(); @@ -914,8 +914,10 @@ Status array_schema_to_capnp( } // #TODO Add security validation on incoming URI -ArraySchema array_schema_from_capnp( - const capnp::ArraySchema::Reader& schema_reader, const URI& uri) { +shared_ptr array_schema_from_capnp( + const capnp::ArraySchema::Reader& schema_reader, + const URI& uri, + shared_ptr memory_tracker) { // Deserialize and validate array_type ArrayType array_type = ArrayType::DENSE; Status st = array_type_enum(schema_reader.getArrayType(), &array_type); @@ -1069,7 +1071,7 @@ ArraySchema array_schema_from_capnp( try { for (auto dim_label_reader : dim_labels_reader) { dimension_labels.emplace_back( - dimension_label_from_capnp(dim_label_reader)); + dimension_label_from_capnp(dim_label_reader, memory_tracker)); } } catch (const std::exception& e) { std::throw_with_nested(std::runtime_error( @@ -1127,7 +1129,8 @@ ArraySchema array_schema_from_capnp( name = schema_reader.getName().cStr(); } - return ArraySchema( + return make_shared( + HERE(), uri_deserialized, version, timestamp_range, @@ -1144,7 +1147,8 @@ ArraySchema array_schema_from_capnp( enmr_path_map, cell_var_offsets_filters, cell_validity_filters, - coords_filters); + coords_filters, + memory_tracker); } Status array_schema_serialize( @@ -1201,8 +1205,10 @@ Status array_schema_serialize( return Status::Ok(); } -ArraySchema array_schema_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer) { +shared_ptr array_schema_deserialize( + SerializationType serialize_type, + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { capnp::ArraySchema::Reader array_schema_reader; ::capnp::MallocMessageBuilder message_builder; @@ -1216,7 +1222,8 @@ ArraySchema array_schema_deserialize( kj::StringPtr(static_cast(serialized_buffer.data())), array_schema_builder); array_schema_reader = array_schema_builder.asReader(); - return array_schema_from_capnp(array_schema_reader, URI()); + return array_schema_from_capnp( + array_schema_reader, URI(), memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = @@ -1225,7 +1232,8 @@ ArraySchema array_schema_deserialize( reinterpret_cast(mBytes), serialized_buffer.size() / sizeof(::capnp::word))); array_schema_reader = reader.getRoot(); - return array_schema_from_capnp(array_schema_reader, URI()); + return array_schema_from_capnp( + array_schema_reader, URI(), memory_tracker); } default: { throw StatusException(Status_SerializationError( @@ -1963,14 +1971,17 @@ void serialize_load_array_schema_response( } } -ArraySchema load_array_schema_response_from_capnp( - capnp::LoadArraySchemaResponse::Reader& reader) { +shared_ptr load_array_schema_response_from_capnp( + capnp::LoadArraySchemaResponse::Reader& reader, + shared_ptr memory_tracker) { auto schema_reader = reader.getSchema(); - return array_schema_from_capnp(schema_reader, URI()); + return array_schema_from_capnp(schema_reader, URI(), memory_tracker); } -ArraySchema deserialize_load_array_schema_response( - SerializationType serialization_type, const Buffer& data) { +shared_ptr deserialize_load_array_schema_response( + SerializationType serialization_type, + const Buffer& data, + shared_ptr memory_tracker) { try { switch (serialization_type) { case SerializationType::JSON: { @@ -1981,7 +1992,7 @@ ArraySchema deserialize_load_array_schema_response( json.decode( kj::StringPtr(static_cast(data.data())), builder); auto reader = builder.asReader(); - return load_array_schema_response_from_capnp(reader); + return load_array_schema_response_from_capnp(reader, memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(data.data()); @@ -1989,7 +2000,7 @@ ArraySchema deserialize_load_array_schema_response( reinterpret_cast(mBytes), data.size() / sizeof(::capnp::word))); auto reader = array_reader.getRoot(); - return load_array_schema_response_from_capnp(reader); + return load_array_schema_response_from_capnp(reader, memory_tracker); } default: { throw Status_SerializationError( @@ -2016,7 +2027,8 @@ Status array_schema_serialize( "Cannot serialize; serialization not enabled.")); } -ArraySchema array_schema_deserialize(SerializationType, const Buffer&) { +shared_ptr array_schema_deserialize( + SerializationType, const Buffer&, shared_ptr) { throw StatusException(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } @@ -2076,8 +2088,8 @@ void serialize_load_array_schema_response( "Cannot serialize; serialization not enabled."); } -ArraySchema deserialize_load_array_schema_response( - SerializationType, const Buffer&) { +shared_ptr deserialize_load_array_schema_response( + SerializationType, const Buffer&, shared_ptr) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index f0c2d7ee6666..2c7fe0904405 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -51,6 +51,7 @@ class Array; class Buffer; class ArraySchema; class Dimension; +class MemoryTracker; enum class SerializationType : uint8_t; namespace serialization { @@ -111,8 +112,10 @@ Status array_schema_to_capnp( * @param uri A URI object * @return a new ArraySchema */ -ArraySchema array_schema_from_capnp( - const capnp::ArraySchema::Reader& schema_reader, const URI& uri); +shared_ptr array_schema_from_capnp( + const capnp::ArraySchema::Reader& schema_reader, + const URI& uri, + shared_ptr memory_tracker); /** * Serialize a dimension label to cap'n proto object @@ -133,7 +136,8 @@ void dimension_label_to_capnp( * @return A new DimensionLabel. */ shared_ptr dimension_label_from_capnp( - const capnp::DimensionLabel::Reader& reader); + const capnp::DimensionLabel::Reader& reader, + shared_ptr memory_tracker); #endif // TILEDB_SERIALIZATION @@ -152,8 +156,10 @@ Status array_schema_serialize( Buffer* serialized_buffer, const bool client_side); -ArraySchema array_schema_deserialize( - SerializationType serialize_type, const Buffer& serialized_buffer); +shared_ptr array_schema_deserialize( + SerializationType serialize_type, + const Buffer& serialized_buffer, + shared_ptr memory_tracker); Status nonempty_domain_serialize( const Array* array, @@ -204,8 +210,10 @@ void serialize_load_array_schema_response( SerializationType serialization_type, Buffer& data); -ArraySchema deserialize_load_array_schema_response( - SerializationType serialization_type, const Buffer& data); +shared_ptr deserialize_load_array_schema_response( + SerializationType serialization_type, + const Buffer& data, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm diff --git a/tiledb/sm/serialization/fragment_info.cc b/tiledb/sm/serialization/fragment_info.cc index 5be515c7e91d..3c18e57d139b 100644 --- a/tiledb/sm/serialization/fragment_info.cc +++ b/tiledb/sm/serialization/fragment_info.cc @@ -276,16 +276,16 @@ Status single_fragment_info_to_capnp( Status fragment_info_from_capnp( const capnp::FragmentInfo::Reader& fragment_info_reader, const URI& array_uri, - FragmentInfo* fragment_info) { + FragmentInfo* fragment_info, + shared_ptr memory_tracker) { // Get array_schema_latest from capnp if (fragment_info_reader.hasArraySchemaLatest()) { auto array_schema_latest_reader = fragment_info_reader.getArraySchemaLatest(); - auto array_schema_latest{ - array_schema_from_capnp(array_schema_latest_reader, array_uri)}; - array_schema_latest.set_array_uri(array_uri); - fragment_info->array_schema_latest() = - make_shared(HERE(), array_schema_latest); + auto array_schema_latest{array_schema_from_capnp( + array_schema_latest_reader, array_uri, memory_tracker)}; + array_schema_latest->set_array_uri(array_uri); + fragment_info->array_schema_latest() = array_schema_latest; } // Get array_schemas_all from capnp @@ -295,14 +295,13 @@ Status fragment_info_from_capnp( if (all_schemas_reader.hasEntries()) { auto entries = fragment_info_reader.getArraySchemasAll().getEntries(); for (auto array_schema_build : entries) { - auto schema{ - array_schema_from_capnp(array_schema_build.getValue(), array_uri)}; - schema.set_array_uri(array_uri); + auto schema{array_schema_from_capnp( + array_schema_build.getValue(), array_uri, memory_tracker)}; + schema->set_array_uri(array_uri); auto key = std::string_view{ array_schema_build.getKey().cStr(), array_schema_build.getKey().size()}; - fragment_info->array_schemas_all()[std::string{key}] = - make_shared(HERE(), schema); + fragment_info->array_schemas_all()[std::string{key}] = schema; } } } @@ -453,7 +452,8 @@ Status fragment_info_deserialize( FragmentInfo* fragment_info, SerializationType serialize_type, const URI& uri, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { if (fragment_info == nullptr) return LOG_STATUS( Status_SerializationError("Error deserializing fragment info; null " @@ -471,7 +471,8 @@ Status fragment_info_deserialize( auto reader = builder.asReader(); // Deserialize - RETURN_NOT_OK(fragment_info_from_capnp(reader, uri, fragment_info)); + RETURN_NOT_OK(fragment_info_from_capnp( + reader, uri, fragment_info, memory_tracker)); break; } case SerializationType::CAPNP: { @@ -493,7 +494,8 @@ Status fragment_info_deserialize( auto reader = msg_reader.getRoot(); // Deserialize - RETURN_NOT_OK(fragment_info_from_capnp(reader, uri, fragment_info)); + RETURN_NOT_OK(fragment_info_from_capnp( + reader, uri, fragment_info, memory_tracker)); break; } default: { @@ -524,7 +526,11 @@ Status fragment_info_serialize( } Status fragment_info_deserialize( - FragmentInfo*, SerializationType, const URI&, const Buffer&) { + FragmentInfo*, + SerializationType, + const URI&, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/fragment_info.h b/tiledb/sm/serialization/fragment_info.h index bd69c37c60f7..42375f05d664 100644 --- a/tiledb/sm/serialization/fragment_info.h +++ b/tiledb/sm/serialization/fragment_info.h @@ -64,7 +64,8 @@ namespace serialization { Status fragment_info_from_capnp( const capnp::FragmentInfo::Reader& fragment_info_reader, const URI& uri, - FragmentInfo* fragment_info); + FragmentInfo* fragment_info, + shared_ptr memory_tracker); /** * Convert Fragment Info to Cap'n Proto message. @@ -130,7 +131,8 @@ Status fragment_info_deserialize( FragmentInfo* fragment_info, SerializationType serialize_type, const URI& uri, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); /** * Serialize a fragment info request via Cap'n Proto. diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 92414cd51d54..01755a141d9a 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -2284,7 +2284,8 @@ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager) { + StorageManager* storage_manager, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -2299,7 +2300,11 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = query_builder.asReader(); // Deserialize array instance. RETURN_NOT_OK(array_from_capnp( - query_reader.getArray(), storage_manager, &array, false)); + query_reader.getArray(), + storage_manager, + &array, + false, + memory_tracker)); break; } case SerializationType::CAPNP: { @@ -2327,7 +2332,11 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = reader.getRoot(); // Deserialize array instance. RETURN_NOT_OK(array_from_capnp( - query_reader.getArray(), storage_manager, &array, false)); + query_reader.getArray(), + storage_manager, + &array, + false, + memory_tracker)); break; } default: @@ -3189,7 +3198,11 @@ Status query_deserialize( } Status array_from_query_deserialize( - const Buffer&, SerializationType, Array&, StorageManager*) { + const Buffer&, + SerializationType, + Array&, + StorageManager*, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 1a711db6b7de..568faacf0f0a 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -142,7 +142,8 @@ Status array_from_query_deserialize( const Buffer& serialized_buffer, SerializationType serialize_type, Array& array, - StorageManager* storage_manager); + StorageManager* storage_manager, + shared_ptr memory_tracker); /** * Serialize a query diff --git a/tiledb/sm/serialization/test/CMakeLists.txt b/tiledb/sm/serialization/test/CMakeLists.txt index abd02d04503c..ac058b81d9ba 100644 --- a/tiledb/sm/serialization/test/CMakeLists.txt +++ b/tiledb/sm/serialization/test/CMakeLists.txt @@ -40,6 +40,7 @@ commence(unit_test capnp_array_schema) # Enable serialization target_compile_definitions(unit_capnp_array_schema PRIVATE -DTILEDB_SERIALIZATION) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) commence(unit_test capnp_nonempty_domain) diff --git a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc index f532d950a624..ad48184b1904 100644 --- a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc +++ b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc @@ -33,7 +33,7 @@ #include #include - +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" @@ -101,7 +101,8 @@ TEST_CASE( SECTION("Internal dimension label") { // Create dimension label array schema. Status st; - auto schema = make_shared(HERE(), ArrayType::DENSE); + auto schema = make_shared( + HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); std::vector> dims{ make_shared(HERE(), "index", Datatype::UINT32)}; uint32_t domain1[2]{1, 64}; @@ -153,8 +154,8 @@ TEST_CASE( message.initRoot(); tiledb::sm::serialization::dimension_label_to_capnp( *dim_label.get(), &builder, true); - auto dim_label_clone = - tiledb::sm::serialization::dimension_label_from_capnp(builder); + auto dim_label_clone = tiledb::sm::serialization::dimension_label_from_capnp( + builder, tiledb::test::create_test_memory_tracker()); // Check dimension label properties and components. CHECK(dim_label->has_schema() == dim_label_clone->has_schema()); diff --git a/tiledb/sm/storage_manager/context_resources.cc b/tiledb/sm/storage_manager/context_resources.cc index 999f0f7566b9..bd82b3c319bd 100644 --- a/tiledb/sm/storage_manager/context_resources.cc +++ b/tiledb/sm/storage_manager/context_resources.cc @@ -76,7 +76,7 @@ ContextResources::ContextResources( auto server_address = config_.get("rest.server_address"); if (server_address) { auto client = tdb::make_shared(HERE()); - auto st = client->init(&stats(), &config_, &compute_tp(), logger_); + auto st = client->init(&stats(), &config_, &compute_tp(), logger_, *this); throw_if_not_ok(st); rest_client_ = client; } diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 2cd6e3709f8f..6ea1846152bf 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -2210,7 +2210,7 @@ Status Subarray::compute_est_result_size( const auto& array_schema = array_->array_schema_latest(); auto attribute_num = array_schema.attribute_num(); auto dim_num = array_schema.dim_num(); - auto attributes = array_schema.attributes(); + auto& attributes = array_schema.attributes(); auto num = attribute_num + dim_num + 1; auto range_num = this->range_num(); diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index c8168cc65ed7..0459e8936359 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -37,11 +37,13 @@ #endif #include #include +#include #include #include #include #include +#include using namespace tiledb; using namespace tiledb::common; @@ -68,7 +70,10 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { make_shared(HERE(), "a1", Datatype::INT32); tiledb::sm::Domain dom{Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR}; std::shared_ptr sp_as = - make_shared(HERE()); + make_shared( + HERE(), + tiledb::sm::ArrayType::DENSE, + tiledb::test::create_test_memory_tracker()); CHECK(sp_as->set_domain(sp_dom).ok()); CHECK(sp_as->add_attribute(sp_attrib).ok()); tiledb::sm::Config cfg; diff --git a/tools/src/commands/info_command.cc b/tools/src/commands/info_command.cc index eeb028e904eb..e181dd6a9948 100644 --- a/tools/src/commands/info_command.cc +++ b/tools/src/commands/info_command.cc @@ -131,7 +131,7 @@ void InfoCommand::print_tile_sizes() const { // Compute and report mean persisted tile sizes over all attributes. const auto& schema = array.array_schema_latest(); auto fragment_metadata = array.fragment_metadata(); - auto attributes = schema.attributes(); + auto& attributes = schema.attributes(); uint64_t total_persisted_size = 0, total_in_memory_size = 0; // Helper function for processing each attribute. From 65615894f5f3da390b9bb3454160ea788a7da0be Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:43:25 +0100 Subject: [PATCH 202/456] Remove move from Tile. (#4749) This will help the memory tracking effort by removing all move constructors/assign operator for Tile, ResultTile, etc. [SC-41591] --- TYPE: NO_HISTORY DESC: Remove move from Tile. --- test/src/unit-Reader.cc | 21 +--- test/src/unit-filter-pipeline.cc | 15 ++- .../unit-sparse-unordered-with-dups-reader.cc | 14 ++- tiledb/sm/array/array.cc | 4 +- tiledb/sm/array/array_directory.cc | 16 +-- tiledb/sm/fragment/fragment_info.cc | 13 +-- tiledb/sm/fragment/fragment_metadata.cc | 56 ++++----- tiledb/sm/fragment/fragment_metadata.h | 2 +- tiledb/sm/group/group.cc | 17 ++- tiledb/sm/query/legacy/reader.cc | 29 ++--- tiledb/sm/query/legacy/reader.h | 12 +- tiledb/sm/query/readers/reader_base.cc | 3 +- tiledb/sm/query/readers/result_space_tile.h | 8 +- tiledb/sm/query/readers/result_tile.cc | 65 +++-------- tiledb/sm/query/readers/result_tile.h | 107 +++--------------- tiledb/sm/storage_manager/storage_manager.cc | 6 +- tiledb/sm/tile/generic_tile_io.cc | 15 +-- tiledb/sm/tile/generic_tile_io.h | 4 +- tiledb/sm/tile/test/unit_tile.cc | 80 ------------- tiledb/sm/tile/tile.cc | 21 +--- tiledb/sm/tile/tile.h | 17 +-- 21 files changed, 155 insertions(+), 370 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index c7d78cc198c4..0957023a0b86 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -278,38 +278,27 @@ TEST_CASE_METHOD( result_space_tiles); CHECK(result_space_tiles.size() == 6); - // Result tiles for fragment #1 - ResultTile result_tile_1_0_1(1, 0, *fragments[0]); - ResultTile result_tile_1_2_1(1, 2, *fragments[0]); - - // Result tiles for fragment #2 - ResultTile result_tile_1_0_2(2, 0, *fragments[1]); - - // Result tiles for fragment #3 - ResultTile result_tile_2_0_3(3, 0, *fragments[2]); - ResultTile result_tile_3_0_3(3, 2, *fragments[2]); - // Initialize result_space_tiles ResultSpaceTile rst_1_0; rst_1_0.set_start_coords({3, 1}); rst_1_0.append_frag_domain(2, ds2); rst_1_0.append_frag_domain(1, ds1); - rst_1_0.set_result_tile(1, result_tile_1_0_1); - rst_1_0.set_result_tile(2, result_tile_1_0_2); + rst_1_0.set_result_tile(1, 0, *fragments[0]); + rst_1_0.set_result_tile(2, 0, *fragments[1]); ResultSpaceTile rst_1_2; rst_1_2.set_start_coords({3, 11}); rst_1_2.append_frag_domain(1, ds1); - rst_1_2.set_result_tile(1, result_tile_1_2_1); + rst_1_2.set_result_tile(1, 2, *fragments[0]); ResultSpaceTile rst_2_0; rst_2_0.set_start_coords({5, 1}); rst_2_0.append_frag_domain(3, ds3); - rst_2_0.set_result_tile(3, result_tile_2_0_3); + rst_2_0.set_result_tile(3, 0, *fragments[2]); ResultSpaceTile rst_2_2; rst_2_2.set_start_coords({5, 11}); ResultSpaceTile rst_3_0; rst_3_0.set_start_coords({7, 1}); rst_3_0.append_frag_domain(3, ds3); - rst_3_0.set_result_tile(3, result_tile_3_0_3); + rst_3_0.set_result_tile(3, 2, *fragments[2]); ResultSpaceTile rst_3_2; rst_3_2.set_start_coords({7, 11}); diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index 107385645c5e..a059df46452a 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -103,15 +103,14 @@ WriterTile make_offsets_tile(std::vector& offsets) { } Tile create_tile_for_unfiltering(uint64_t nelts, WriterTile& tile) { - Tile ret( + return { tile.format_version(), tile.type(), tile.cell_size(), 0, tile.cell_size() * nelts, tile.filtered_buffer().data(), - tile.filtered_buffer().size()); - return ret; + tile.filtered_buffer().size()}; } void run_reverse( @@ -1221,19 +1220,19 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { key[0]++; filter->set_key(key); - unfiltered_tile = create_tile_for_unfiltering(nelts, tile); - run_reverse(config, tp, unfiltered_tile, pipeline, false); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile); + run_reverse(config, tp, unfiltered_tile2, pipeline, false); // Fix key and check success. - unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile3 = create_tile_for_unfiltering(nelts, tile); key[0]--; filter->set_key(key); - run_reverse(config, tp, unfiltered_tile, pipeline); + run_reverse(config, tp, unfiltered_tile3, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + unfiltered_tile3.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); CHECK(elt == i); } } diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 961b1e76f768..a76d879eff01 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -1527,7 +1527,7 @@ TEST_CASE_METHOD( auto&& [array, fragments] = open_default_array_1d_with_fragments(capacity); // Make a vector of tiles. - std::vector> rt; + std::list> rt; for (uint64_t t = 0; t < num_tiles; t++) { rt.emplace_back(0, t, *fragments[0]); @@ -1540,8 +1540,9 @@ TEST_CASE_METHOD( // Create the result_tiles pointer vector. std::vector result_tiles(rt.size()); - for (uint64_t i = 0; i < rt.size(); i++) { - result_tiles[i] = &rt[i]; + uint64_t i = 0; + for (auto& t : rt) { + result_tiles[i++] = &t; } // Create a Query buffer. @@ -1743,7 +1744,7 @@ TEST_CASE_METHOD( auto&& [array, fragments] = open_default_array_1d_with_fragments(capacity); // Make a vector of tiles. - std::vector> rt; + std::list> rt; for (uint64_t t = 0; t < num_tiles; t++) { rt.emplace_back(0, t, *fragments[0]); @@ -1756,8 +1757,9 @@ TEST_CASE_METHOD( // Create the result_tiles pointer vector. std::vector result_tiles(rt.size()); - for (uint64_t i = 0; i < rt.size(); i++) { - result_tiles[i] = &rt[i]; + uint64_t i = 0; + for (auto& t : rt) { + result_tiles[i++] = &t; } // Call the function. diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index dac76a6051e8..a30b30252583 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1478,10 +1478,8 @@ void Array::do_load_metadata() { throw_if_not_ok( parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { const auto& uri = array_metadata_to_load[m].uri_; - - auto&& tile = + metadata_tiles[m] = GenericTileIO::load(resources_, uri, 0, *encryption_key()); - metadata_tiles[m] = tdb::make_shared(HERE(), std::move(tile)); return Status::Ok(); })); diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 9d5fbae2cbf1..b57a030fd978 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -96,12 +96,12 @@ shared_ptr ArrayDirectory::load_array_schema_from_uri( auto timer_se = resources.stats().start_timer("sm_load_array_schema_from_uri"); - auto&& tile = GenericTileIO::load(resources, schema_uri, 0, encryption_key); + auto tile = GenericTileIO::load(resources, schema_uri, 0, encryption_key); - resources.stats().add_counter("read_array_schema_size", tile.size()); + resources.stats().add_counter("read_array_schema_size", tile->size()); // Deserialize - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); auto memory_tracker = resources.create_memory_tracker(); memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); return ArraySchema::deserialize(deserializer, schema_uri, memory_tracker); @@ -1323,18 +1323,18 @@ shared_ptr ArrayDirectory::load_enumeration( .join_path(constants::array_enumerations_dir_name) .join_path(enumeration_path); - auto&& tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); - resources_.get().stats().add_counter("read_enumeration_size", tile.size()); + auto tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); + resources_.get().stats().add_counter("read_enumeration_size", tile->size()); - if (!memory_tracker->take_memory(tile.size(), MemoryType::ENUMERATION)) { + if (!memory_tracker->take_memory(tile->size(), MemoryType::ENUMERATION)) { throw ArrayDirectoryException( "Error loading enumeration; Insufficient memory budget; Needed " + - std::to_string(tile.size()) + " but only had " + + std::to_string(tile->size()) + " but only had " + std::to_string(memory_tracker->get_memory_available()) + " from budget " + std::to_string(memory_tracker->get_memory_budget())); } - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); return Enumeration::deserialize(deserializer); } diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 7f089a84e5ed..8fa86a2fe7ca 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -952,7 +952,7 @@ Status FragmentInfo::load_and_replace( return Status::Ok(); } -tuple>> +tuple, std::vector>> load_consolidated_fragment_meta( ContextResources& resources, const URI& uri, const EncryptionKey& enc_key) { auto timer_se = @@ -963,12 +963,12 @@ load_consolidated_fragment_meta( throw StatusException(Status_FragmentInfoError( "Cannot load consolidated fragment metadata; URI is empty.")); - auto&& tile = GenericTileIO::load(resources, uri, 0, enc_key); + auto tile = GenericTileIO::load(resources, uri, 0, enc_key); - resources.stats().add_counter("consolidated_frag_meta_size", tile.size()); + resources.stats().add_counter("consolidated_frag_meta_size", tile->size()); uint32_t fragment_num; - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); fragment_num = deserializer.read(); uint64_t name_size, offset; @@ -983,7 +983,7 @@ load_consolidated_fragment_meta( ret.emplace_back(name, offset); } - return {std::move(tile), std::move(ret)}; + return {tile, std::move(ret)}; } std::tuple< @@ -1021,8 +1021,7 @@ FragmentInfo::load_array_schemas_and_fragment_metadata( parallel_for(&resources.compute_tp(), 0, meta_uris.size(), [&](size_t i) { auto&& [tile_opt, offsets] = load_consolidated_fragment_meta(resources, meta_uris[i], enc_key); - fragment_metadata_tiles[i] = - make_shared(HERE(), std::move(tile_opt)); + fragment_metadata_tiles[i] = tile_opt; offsets_vectors[i] = std::move(offsets); return Status::Ok(); })); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 0fc003ea5dce..f3ad74325aec 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -1504,9 +1504,9 @@ void FragmentMetadata::load_fragment_min_max_sum_null_count( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.fragment_min_max_sum_null_count_offset_); resources_->stats().add_counter( - "read_fragment_min_max_sum_null_count_size", tile.size()); + "read_fragment_min_max_sum_null_count_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_fragment_min_max_sum_null_count(deserializer); loaded_metadata_.fragment_min_max_sum_null_count_ = true; @@ -1527,9 +1527,9 @@ void FragmentMetadata::load_processed_conditions( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.processed_conditions_offsets_); resources_->stats().add_counter( - "read_processed_conditions_size", tile.size()); + "read_processed_conditions_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_processed_conditions(deserializer); loaded_metadata_.processed_conditions_ = true; @@ -2081,19 +2081,19 @@ void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { } auto tile = read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); - resources_->stats().add_counter("read_rtree_size", tile.size()); + resources_->stats().add_counter("read_rtree_size", tile->size()); // Use the serialized buffer size to approximate memory usage of the rtree. if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(tile.size(), MemoryType::RTREE)) { + !memory_tracker_->take_memory(tile->size(), MemoryType::RTREE)) { throw FragmentMetadataStatusException( "Cannot load R-tree; Insufficient memory budget; Needed " + - std::to_string(tile.size()) + " but only had " + + std::to_string(tile->size()) + " but only had " + std::to_string(memory_tracker_->get_memory_available()) + " from budget " + std::to_string(memory_tracker_->get_memory_budget())); } - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); rtree_.deserialize(deserializer, &array_schema_->domain(), version_); loaded_metadata_.rtree_ = true; @@ -2475,9 +2475,9 @@ void FragmentMetadata::load_tile_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_offsets_[idx]); - resources_->stats().add_counter("read_tile_offsets_size", tile.size()); + resources_->stats().add_counter("read_tile_offsets_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_offsets(idx, deserializer); loaded_metadata_.tile_offsets_[idx] = true; @@ -2502,9 +2502,9 @@ void FragmentMetadata::load_tile_var_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_offsets_[idx]); - resources_->stats().add_counter("read_tile_var_offsets_size", tile.size()); + resources_->stats().add_counter("read_tile_var_offsets_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_var_offsets(idx, deserializer); loaded_metadata_.tile_var_offsets_[idx] = true; @@ -2524,9 +2524,9 @@ void FragmentMetadata::load_tile_var_sizes( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_var_sizes_[idx]); - resources_->stats().add_counter("read_tile_var_sizes_size", tile.size()); + resources_->stats().add_counter("read_tile_var_sizes_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_var_sizes(idx, deserializer); loaded_metadata_.tile_var_sizes_[idx] = true; @@ -2547,9 +2547,9 @@ void FragmentMetadata::load_tile_validity_offsets( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_validity_offsets_[idx]); resources_->stats().add_counter( - "read_tile_validity_offsets_size", tile.size()); + "read_tile_validity_offsets_size", tile->size()); - ConstBuffer cbuff(tile.data(), tile.size()); + ConstBuffer cbuff(tile->data(), tile->size()); load_tile_validity_offsets(idx, &cbuff); loaded_metadata_.tile_validity_offsets_[idx] = true; @@ -2569,9 +2569,9 @@ void FragmentMetadata::load_tile_min_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_min_offsets_[idx]); - resources_->stats().add_counter("read_tile_min_size", tile.size()); + resources_->stats().add_counter("read_tile_min_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_min_values(idx, deserializer); loaded_metadata_.tile_min_[idx] = true; @@ -2591,9 +2591,9 @@ void FragmentMetadata::load_tile_max_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_max_offsets_[idx]); - resources_->stats().add_counter("read_tile_max_size", tile.size()); + resources_->stats().add_counter("read_tile_max_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_max_values(idx, deserializer); loaded_metadata_.tile_max_[idx] = true; @@ -2613,9 +2613,9 @@ void FragmentMetadata::load_tile_sum_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_sum_offsets_[idx]); - resources_->stats().add_counter("read_tile_sum_size", tile.size()); + resources_->stats().add_counter("read_tile_sum_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_sum_values(idx, deserializer); loaded_metadata_.tile_sum_[idx] = true; @@ -2635,9 +2635,9 @@ void FragmentMetadata::load_tile_null_count_values( auto tile = read_generic_tile_from_file( encryption_key, gt_offsets_.tile_null_count_offsets_[idx]); - resources_->stats().add_counter("read_tile_null_count_size", tile.size()); + resources_->stats().add_counter("read_tile_null_count_size", tile->size()); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_tile_null_count_values(idx, deserializer); loaded_metadata_.tile_null_count_[idx] = true; @@ -3607,7 +3607,7 @@ void FragmentMetadata::load_v1_v2( GenericTileIO tile_io(*resources_, fragment_metadata_uri); auto tile = tile_io.read_generic(0, encryption_key, resources_->config()); - resources_->stats().add_counter("read_frag_meta_size", tile.size()); + resources_->stats().add_counter("read_frag_meta_size", tile->size()); // Pre-v10 format fragments we need to set the schema and schema name to // the "old" schema. This way "old" fragments are still loaded fine @@ -3623,7 +3623,7 @@ void FragmentMetadata::load_v1_v2( } // Deserialize - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); load_version(deserializer); load_non_empty_domain(deserializer); load_mbrs(deserializer); @@ -3938,7 +3938,7 @@ void FragmentMetadata::write_non_empty_domain(Serializer& serializer) const { } } -Tile FragmentMetadata::read_generic_tile_from_file( +shared_ptr FragmentMetadata::read_generic_tile_from_file( const EncryptionKey& encryption_key, uint64_t offset) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); @@ -3958,7 +3958,7 @@ void FragmentMetadata::read_file_footer( // Get footer offset get_footer_offset_and_size(footer_offset, footer_size); - tile = make_shared(HERE(), Tile::from_generic(*footer_size)); + tile = Tile::from_generic(*footer_size); resources_->stats().add_counter("read_frag_meta_size", *footer_size); diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index a79ea251f965..600e55fa2a7b 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -2038,7 +2038,7 @@ class FragmentMetadata { * Reads the contents of a generic tile starting at the input offset, * and returns a tile. */ - Tile read_generic_tile_from_file( + shared_ptr read_generic_tile_from_file( const EncryptionKey& encryption_key, uint64_t offset) const; /** diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 15895488cbe6..3684135ea922 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -740,8 +740,8 @@ void Group::load_metadata_from_storage( parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { const auto& uri = group_metadata_to_load[m].uri_; - auto&& tile = GenericTileIO::load(resources_, uri, 0, encryption_key); - metadata_tiles[m] = tdb::make_shared(HERE(), std::move(tile)); + metadata_tiles[m] = + GenericTileIO::load(resources_, uri, 0, encryption_key); return Status::Ok(); })); @@ -795,12 +795,12 @@ void Group::load_group_from_uri(const URI& uri) { [[maybe_unused]] auto timer_se = resources_.stats().start_timer("load_group_from_uri"); - auto&& tile = GenericTileIO::load(resources_, uri, 0, *encryption_key()); + auto tile = GenericTileIO::load(resources_, uri, 0, *encryption_key()); - resources_.stats().add_counter("read_group_size", tile.size()); + resources_.stats().add_counter("read_group_size", tile->size()); // Deserialize - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); auto opt_group = GroupDetails::deserialize(deserializer, group_directory()->uri()); @@ -815,14 +815,13 @@ void Group::load_group_from_all_uris(const std::vector& uris) { std::vector> deserializers; for (auto& uri : uris) { - auto&& tile = - GenericTileIO::load(resources_, uri.uri_, 0, *encryption_key()); + auto tile = GenericTileIO::load(resources_, uri.uri_, 0, *encryption_key()); - resources_.stats().add_counter("read_group_size", tile.size()); + resources_.stats().add_counter("read_group_size", tile->size()); // Deserialize shared_ptr deserializer = - tdb::make_shared(HERE(), std::move(tile)); + tdb::make_shared(HERE(), tile); deserializers.emplace_back(deserializer); } diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 075c50259930..acf9d1327fad 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -492,7 +492,7 @@ Status Reader::compute_range_result_coords( Subarray& subarray, const std::vector& single_fragment, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector>& range_result_coords) { auto timer_se = stats_->start_timer("compute_range_result_coords"); @@ -547,7 +547,7 @@ Status Reader::compute_range_result_coords( uint64_t range_idx, uint32_t fragment_idx, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector& range_result_coords) { // Skip dense fragments if (fragment_metadata_[fragment_idx]->dense()) @@ -568,12 +568,14 @@ Status Reader::compute_range_result_coords( auto tile_it = result_tile_map.find(pair); assert(tile_it != result_tile_map.end()); auto tile_idx = tile_it->second; - auto& tile = result_tiles[tile_idx]; + + auto tile = result_tiles.begin(); + std::advance(tile, tile_idx); // Add results only if the sparse tile MBR is not fully // covered by a more recent fragment's non-empty domain if (!sparse_tile_overwritten(fragment_idx, i)) - RETURN_NOT_OK(get_all_result_coords(&tile, range_result_coords)); + RETURN_NOT_OK(get_all_result_coords(&*tile, range_result_coords)); } ++tr; } else { @@ -582,15 +584,16 @@ Status Reader::compute_range_result_coords( auto tile_it = result_tile_map.find(pair); assert(tile_it != result_tile_map.end()); auto tile_idx = tile_it->second; - auto& tile = result_tiles[tile_idx]; + auto tile = result_tiles.begin(); + std::advance(tile, tile_idx); if (t->second == 1.0) { // Full overlap // Add results only if the sparse tile MBR is not fully // covered by a more recent fragment's non-empty domain if (!sparse_tile_overwritten(fragment_idx, t->first)) - RETURN_NOT_OK(get_all_result_coords(&tile, range_result_coords)); + RETURN_NOT_OK(get_all_result_coords(&*tile, range_result_coords)); } else { // Partial overlap RETURN_NOT_OK(compute_range_result_coords( - subarray, fragment_idx, &tile, range_idx, range_result_coords)); + subarray, fragment_idx, &*tile, range_idx, range_result_coords)); } ++t; } @@ -603,7 +606,7 @@ Status Reader::compute_range_result_coords( Subarray& subarray, uint64_t range_idx, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector& range_result_coords) { // Gather result range coordinates per fragment auto fragment_num = fragment_metadata_.size(); @@ -678,7 +681,7 @@ Status Reader::compute_subarray_coords( } Status Reader::compute_sparse_result_tiles( - std::vector& result_tiles, + std::list& result_tiles, std::map, size_t>* result_tile_map, std::vector* single_fragment) { auto timer_se = stats_->start_timer("compute_sparse_result_tiles"); @@ -1580,7 +1583,7 @@ Status Reader::compute_result_cell_slabs_global( } Status Reader::compute_result_coords( - std::vector& result_tiles, + std::list& result_tiles, std::vector& result_coords) { auto timer_se = stats_->start_timer("compute_result_coords"); @@ -1719,7 +1722,7 @@ Status Reader::dense_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - std::vector sparse_result_tiles; + std::list sparse_result_tiles; RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); // Compute result cell slabs. @@ -1975,7 +1978,7 @@ Status Reader::sparse_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - std::vector sparse_result_tiles; + std::list sparse_result_tiles; RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); std::vector result_tiles; @@ -2057,7 +2060,7 @@ bool Reader::sparse_tile_overwritten( return false; } -void Reader::erase_coord_tiles(std::vector& result_tiles) const { +void Reader::erase_coord_tiles(std::list& result_tiles) const { for (auto& tile : result_tiles) { auto dim_num = array_schema_.dim_num(); for (unsigned d = 0; d < dim_num; ++d) diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index 057841d10a47..997acdcca22d 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -305,7 +305,7 @@ class Reader : public ReaderBase, public IQueryStrategy { Subarray& subarray, const std::vector& single_fragment, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector>& range_result_coords); /** @@ -325,7 +325,7 @@ class Reader : public ReaderBase, public IQueryStrategy { Subarray& subarray, uint64_t range_idx, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector& range_result_coords); /** @@ -347,7 +347,7 @@ class Reader : public ReaderBase, public IQueryStrategy { uint64_t range_idx, uint32_t fragment_idx, const std::map, size_t>& result_tile_map, - std::vector& result_tiles, + std::list& result_tiles, std::vector& range_result_coords); /** @@ -382,7 +382,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * @return Status */ Status compute_sparse_result_tiles( - std::vector& result_tiles, + std::list& result_tiles, std::map, size_t>* result_tile_map, std::vector* single_fragment); @@ -574,7 +574,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * @param result_coords This will store the result coordinates. */ Status compute_result_coords( - std::vector& result_tiles, + std::list& result_tiles, std::vector& result_coords); /** @@ -651,7 +651,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * Erases the coordinate tiles (zipped or separate) from the input result * tiles. */ - void erase_coord_tiles(std::vector& result_tiles) const; + void erase_coord_tiles(std::list& result_tiles) const; /** Gets statistics about the result cells. */ void get_result_cell_stats( diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 7f6eb29c5e80..edb84febaaed 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -152,9 +152,8 @@ void ReaderBase::compute_result_space_tiles( auto frag_idx = frag_tile_domains[f].id(); result_space_tile.append_frag_domain(frag_idx, frag_domain); auto tile_idx = frag_tile_domains[f].tile_pos(coords); - ResultTile result_tile( + result_space_tile.set_result_tile( frag_idx, tile_idx, *fragment_metadata[frag_idx].get()); - result_space_tile.set_result_tile(frag_idx, result_tile); } } } diff --git a/tiledb/sm/query/readers/result_space_tile.h b/tiledb/sm/query/readers/result_space_tile.h index 8f15945d4ecc..6225b2a6d103 100644 --- a/tiledb/sm/query/readers/result_space_tile.h +++ b/tiledb/sm/query/readers/result_space_tile.h @@ -126,9 +126,13 @@ class ResultSpaceTile { } /** Sets the input result tile for the given fragment. */ - void set_result_tile(unsigned frag_idx, ResultTile& result_tile) { + void set_result_tile( + unsigned frag_idx, uint64_t tile_idx, FragmentMetadata& frag_md) { assert(result_tiles_.count(frag_idx) == 0); - result_tiles_[frag_idx] = std::move(result_tile); + result_tiles_.emplace( + std::piecewise_construct, + std::forward_as_tuple(frag_idx), + std::forward_as_tuple(frag_idx, tile_idx, frag_md)); } /** Returns the result tile for the input fragment. */ diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index 7848d8a9da85..fe22258b9534 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -72,10 +72,10 @@ ResultTile::ResultTile( : domain_(&frag_md.array_schema()->domain()) , frag_idx_(frag_idx) , tile_idx_(tile_idx) - , cell_num_(frag_md.cell_num(tile_idx)) { + , cell_num_(frag_md.cell_num(tile_idx)) + , attr_tiles_(frag_md.array_schema()->attribute_num()) + , coord_tiles_(domain_->dim_num()) { auto array_schema = frag_md.array_schema(); - coord_tiles_.resize(domain_->dim_num()); - attr_tiles_.resize(array_schema->attribute_num()); for (uint64_t i = 0; i < array_schema->attribute_num(); i++) { auto attribute = array_schema->attribute(i); attr_tiles_[i] = std::make_pair(attribute->name(), nullopt); @@ -87,41 +87,6 @@ ResultTile::ResultTile( coord_func_ = &ResultTile::zipped_coord; } -/** Move constructor. */ -ResultTile::ResultTile(ResultTile&& other) { - // Swap with the argument - swap(other); -} - -/** Move-assign operator. */ -ResultTile& ResultTile::operator=(ResultTile&& other) { - // Swap with the argument - swap(other); - - return *this; -} - -void ResultTile::swap(ResultTile& tile) { - std::swap(domain_, tile.domain_); - std::swap(frag_idx_, tile.frag_idx_); - std::swap(tile_idx_, tile.tile_idx_); - std::swap(cell_num_, tile.cell_num_); - std::swap(attr_tiles_, tile.attr_tiles_); - std::swap(timestamps_tile_, tile.timestamps_tile_); - std::swap(delete_timestamps_tile_, tile.delete_timestamps_tile_); - std::swap(coords_tile_, tile.coords_tile_); - std::swap(coord_tiles_, tile.coord_tiles_); - std::swap(compute_results_dense_func_, tile.compute_results_dense_func_); - std::swap(coord_func_, tile.coord_func_); - std::swap(compute_results_sparse_func_, tile.compute_results_sparse_func_); - std::swap( - compute_results_count_sparse_uint64_t_func_, - tile.compute_results_count_sparse_uint64_t_func_); - std::swap( - compute_results_count_sparse_uint8_t_func_, - tile.compute_results_count_sparse_uint8_t_func_); -} - /* ****************************** */ /* API */ /* ****************************** */ @@ -174,33 +139,35 @@ void ResultTile::init_attr_tile( const std::string& name, const TileSizes tile_sizes, const TileData tile_data) { - auto tuple = - TileTuple(format_version, array_schema, name, tile_sizes, tile_data); - if (name == constants::coords) { - coords_tile_ = std::move(tuple); + coords_tile_.emplace( + format_version, array_schema, name, tile_sizes, tile_data); return; } if (name == constants::timestamps) { - timestamps_tile_ = std::move(tuple); + timestamps_tile_.emplace( + format_version, array_schema, name, tile_sizes, tile_data); return; } if (name == constants::delete_timestamps) { - delete_timestamps_tile_ = std::move(tuple); + delete_timestamps_tile_.emplace( + format_version, array_schema, name, tile_sizes, tile_data); return; } if (name == constants::delete_condition_index) { - delete_condition_index_tile_ = std::move(tuple); + delete_condition_index_tile_.emplace( + format_version, array_schema, name, tile_sizes, tile_data); return; } // Handle attributes for (auto& at : attr_tiles_) { if (at.first == name && at.second == nullopt) { - at.second = std::move(tuple); + at.second.emplace( + format_version, array_schema, name, tile_sizes, tile_data); return; } } @@ -213,9 +180,9 @@ void ResultTile::init_coord_tile( const TileSizes tile_sizes, const TileData tile_data, unsigned dim_idx) { - coord_tiles_[dim_idx] = std::pair( - name, - TileTuple(format_version, array_schema, name, tile_sizes, tile_data)); + coord_tiles_[dim_idx].first = name; + coord_tiles_[dim_idx].second.emplace( + format_version, array_schema, name, tile_sizes, tile_data); // When at least one unzipped coordinate has been initialized, we will // use the unzipped `coord()` implementation. diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index 192fcc3332e2..b7997b784b7f 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -268,26 +268,18 @@ class ResultTile { const TileSizes tile_sizes, const TileData tile_data) : fixed_tile_( - tile_sizes.has_var_tile() ? - Tile( - format_version, - constants::cell_var_offset_type, - constants::cell_var_offset_size, - 0, - tile_sizes.tile_size(), - tile_data.fixed_filtered_data(), - tile_sizes.tile_persisted_size()) : - Tile( - format_version, - array_schema.type(name), - array_schema.cell_size(name), - (name == constants::coords) ? array_schema.dim_num() : 0, - tile_sizes.tile_size(), - tile_data.fixed_filtered_data(), - tile_sizes.tile_persisted_size())) { + format_version, + tile_sizes.has_var_tile() ? constants::cell_var_offset_type : + array_schema.type(name), + tile_sizes.has_var_tile() ? constants::cell_var_offset_size : + array_schema.cell_size(name), + (name == constants::coords) ? array_schema.dim_num() : 0, + tile_sizes.tile_size(), + tile_data.fixed_filtered_data(), + tile_sizes.tile_persisted_size()) { if (tile_sizes.has_var_tile()) { auto type = array_schema.type(name); - var_tile_ = Tile( + var_tile_.emplace( format_version, type, datatype_size(type), @@ -298,7 +290,7 @@ class ResultTile { } if (tile_sizes.has_validity_tile()) { - validity_tile_ = Tile( + validity_tile_.emplace( format_version, constants::cell_validity_type, constants::cell_validity_size, @@ -373,19 +365,11 @@ class ResultTile { unsigned frag_idx, uint64_t tile_idx, const FragmentMetadata& frag_md); DISABLE_COPY_AND_COPY_ASSIGN(ResultTile); + DISABLE_MOVE_AND_MOVE_ASSIGN(ResultTile); /** Default destructor. */ ~ResultTile() = default; - /** Move constructor. */ - ResultTile(ResultTile&& tile); - - /** Move-assign operator. */ - ResultTile& operator=(ResultTile&& tile); - - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(ResultTile& tile); - /* ********************************* */ /* API */ /* ********************************* */ @@ -822,22 +806,8 @@ class ResultTileWithBitmap : public ResultTile { , result_num_(cell_num_) { } - /** Move constructor. */ - ResultTileWithBitmap(ResultTileWithBitmap&& other) noexcept { - // Swap with the argument - swap(other); - } - - /** Move-assign operator. */ - ResultTileWithBitmap& operator=( - ResultTileWithBitmap&& other) { - // Swap with the argument - swap(other); - - return *this; - } - DISABLE_COPY_AND_COPY_ASSIGN(ResultTileWithBitmap); + DISABLE_MOVE_AND_MOVE_ASSIGN(ResultTileWithBitmap); public: /* ********************************* */ @@ -945,13 +915,6 @@ class ResultTileWithBitmap : public ResultTile { return false; } - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(ResultTileWithBitmap& tile) { - ResultTile::swap(tile); - std::swap(bitmap_, tile.bitmap_); - std::swap(result_num_, tile.result_num_); - } - protected: /* ********************************* */ /* PROTECTED ATTRIBUTES */ @@ -983,35 +946,13 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { , used_(false) { } - /** Move constructor. */ - GlobalOrderResultTile(GlobalOrderResultTile&& other) noexcept { - // Swap with the argument - swap(other); - } - - /** Move-assign operator. */ - GlobalOrderResultTile& operator=(GlobalOrderResultTile&& other) { - // Swap with the argument - swap(other); - - return *this; - } - DISABLE_COPY_AND_COPY_ASSIGN(GlobalOrderResultTile); + DISABLE_MOVE_AND_MOVE_ASSIGN(GlobalOrderResultTile); /* ********************************* */ /* PUBLIC METHODS */ /* ********************************* */ - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(GlobalOrderResultTile& tile) { - ResultTileWithBitmap::swap(tile); - std::swap(used_, tile.used_); - std::swap(hilbert_values_, tile.hilbert_values_); - std::swap(post_dedup_bitmap_, tile.post_dedup_bitmap_); - std::swap(per_cell_delete_condition_, tile.per_cell_delete_condition_); - } - /** Returns if the tile was used by the merge or not. */ inline bool used() { return used_; @@ -1195,31 +1136,13 @@ class UnorderedWithDupsResultTile : public ResultTileWithBitmap { : ResultTileWithBitmap(frag_idx, tile_idx, frag_md) { } - /** Move constructor. */ - UnorderedWithDupsResultTile(UnorderedWithDupsResultTile&& other) noexcept { - // Swap with the argument - swap(other); - } - - /** Move-assign operator. */ - UnorderedWithDupsResultTile& operator=(UnorderedWithDupsResultTile&& other) { - // Swap with the argument - swap(other); - - return *this; - } - + DISABLE_MOVE_AND_MOVE_ASSIGN(UnorderedWithDupsResultTile); DISABLE_COPY_AND_COPY_ASSIGN(UnorderedWithDupsResultTile); /* ********************************* */ /* PUBLIC METHODS */ /* ********************************* */ - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(UnorderedWithDupsResultTile& tile) { - ResultTileWithBitmap::swap(tile); - } - /** * Returns whether this tile has a post query condition bitmap. For this * tile type, this is stored in the regular bitmap. diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 570c0e0f537a..c7e92b4b2a85 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1007,7 +1007,7 @@ StorageManagerCanonical::load_delete_and_update_conditions( auto& uri = locations[i].uri(); // Read the condition from storage. - auto&& tile = GenericTileIO::load( + auto tile = GenericTileIO::load( resources_, uri, locations[i].offset(), @@ -1018,13 +1018,13 @@ StorageManagerCanonical::load_delete_and_update_conditions( tiledb::sm::constants::delete_file_suffix)) { conditions[i] = tiledb::sm::deletes_and_updates::serialization::deserialize_condition( - i, locations[i].condition_marker(), tile.data(), tile.size()); + i, locations[i].condition_marker(), tile->data(), tile->size()); } else if (tiledb::sm::utils::parse::ends_with( locations[i].condition_marker(), tiledb::sm::constants::update_file_suffix)) { auto&& [cond, uvs] = tiledb::sm::deletes_and_updates::serialization:: deserialize_update_condition_and_values( - i, locations[i].condition_marker(), tile.data(), tile.size()); + i, locations[i].condition_marker(), tile->data(), tile->size()); conditions[i] = std::move(cond); update_values[i] = std::move(uvs); } else { diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 5968114d7866..50b8149d5470 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -67,7 +67,7 @@ GenericTileIO::GenericTileIO(ContextResources& resources, const URI& uri) /* API */ /* ****************************** */ -Tile GenericTileIO::load( +shared_ptr GenericTileIO::load( ContextResources& resources, const URI& uri, uint64_t offset, @@ -85,7 +85,7 @@ Tile GenericTileIO::load( stdx::unreachable(); } -Tile GenericTileIO::read_generic( +shared_ptr GenericTileIO::read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config) { @@ -106,7 +106,8 @@ Tile GenericTileIO::read_generic( GenericTileHeader::BASE_SIZE + header.filter_pipeline_size; std::vector filtered_data(header.persisted_size); - Tile tile( + shared_ptr tile = make_shared( + HERE(), header.version_number, (Datatype)header.datatype, header.cell_size, @@ -119,13 +120,13 @@ Tile GenericTileIO::read_generic( throw_if_not_ok(resources_.vfs().read( uri_, file_offset + tile_data_offset, - tile.filtered_data(), + tile->filtered_data(), header.persisted_size)); // Unfilter - assert(tile.filtered()); - header.filters.run_reverse_generic_tile(&resources_.stats(), tile, config); - assert(!tile.filtered()); + assert(tile->filtered()); + header.filters.run_reverse_generic_tile(&resources_.stats(), *tile, config); + assert(!tile->filtered()); return tile; } diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index 7410784df30f..c23193c7b326 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -122,7 +122,7 @@ class GenericTileIO { * @param encryption_key The encryption key to use. * @return Status, Tile with the data. */ - static Tile load( + static shared_ptr load( ContextResources& resources, const URI& uri, uint64_t offset, @@ -143,7 +143,7 @@ class GenericTileIO { * @param config The storage manager's config. * @return Status, Tile */ - Tile read_generic( + shared_ptr read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config); diff --git a/tiledb/sm/tile/test/unit_tile.cc b/tiledb/sm/tile/test/unit_tile.cc index 1c6b549d89b5..4ca36da42614 100644 --- a/tiledb/sm/tile/test/unit_tile.cc +++ b/tiledb/sm/tile/test/unit_tile.cc @@ -114,83 +114,3 @@ TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { CHECK_NOTHROW(tile.read(read_buffer.data(), read_offset, tile_size)); CHECK(memcmp(read_buffer.data(), write_buffer_copy.data(), tile_size) == 0); } - -TEST_CASE("Tile: Test move constructor", "[Tile][move_constructor]") { - // Instantiate and initialize the first test Tile. - const format_version_t format_version = 0; - const Datatype data_type = Datatype::UINT32; - const uint64_t tile_size = 1024 * 1024; - const uint64_t cell_size = sizeof(uint32_t); - const unsigned int dim_num = 1; - Tile tile1( - format_version, data_type, cell_size, dim_num, tile_size, nullptr, 0); - - // Create a buffer to write to the first test Tile. - const uint32_t buffer_len = tile_size / sizeof(uint32_t); - std::vector buffer(buffer_len); - for (uint32_t i = 0; i < buffer_len; ++i) { - buffer[i] = i; - } - - // Write the buffer to the first test Tile. - CHECK_NOTHROW(tile1.write(buffer.data(), 0, tile_size)); - - // Instantiate a second test tile with the move constructor. - Tile tile2(std::move(tile1)); - - // Verify all public attributes are identical. - CHECK(tile2.cell_size() == cell_size); - CHECK(tile2.zipped_coords_dim_num() == dim_num); - CHECK(tile2.filtered() == false); - CHECK(tile2.format_version() == format_version); - CHECK(tile2.size() == tile_size); - CHECK(tile2.stores_coords() == true); - CHECK(tile2.type() == Datatype::UINT32); - - // Read the second test tile to verify it contains the data - // written to the first test tile. - std::vector read_buffer(buffer_len); - uint64_t read_offset = 0; - CHECK_NOTHROW(tile2.read(read_buffer.data(), read_offset, tile_size)); - CHECK(memcmp(read_buffer.data(), buffer.data(), tile_size) == 0); -} - -TEST_CASE("Tile: Test move-assignment", "[Tile][move_assignment]") { - // Instantiate and initialize the first test Tile. - const format_version_t format_version = 0; - const Datatype data_type = Datatype::UINT32; - const uint64_t tile_size = 1024 * 1024; - const uint64_t cell_size = sizeof(uint32_t); - const unsigned int dim_num = 1; - Tile tile1( - format_version, data_type, cell_size, dim_num, tile_size, nullptr, 0); - - // Create a buffer to write to the first test Tile. - const uint32_t buffer_len = tile_size / sizeof(uint32_t); - std::vector buffer(buffer_len); - for (uint32_t i = 0; i < buffer_len; ++i) { - buffer[i] = i; - } - - // Write the buffer to the first test Tile. - CHECK_NOTHROW(tile1.write(buffer.data(), 0, tile_size)); - - // Instantiate a third test tile with the move constructor. - Tile tile2 = std::move(tile1); - - // Verify all public attributes are identical. - CHECK(tile2.cell_size() == cell_size); - CHECK(tile2.zipped_coords_dim_num() == dim_num); - CHECK(tile2.filtered() == false); - CHECK(tile2.format_version() == format_version); - CHECK(tile2.size() == tile_size); - CHECK(tile2.stores_coords() == true); - CHECK(tile2.type() == Datatype::UINT32); - - // Read the second test tile to verify it contains the data - // written to the first test tile. - std::vector read_buffer(buffer_len); - uint64_t read_offset = 0; - CHECK_NOTHROW(tile2.read(read_buffer.data(), read_offset, tile_size)); - CHECK(memcmp(read_buffer.data(), buffer.data(), tile_size) == 0); -} diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index 76bd8af3f06a..3923d0d44984 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -60,15 +60,16 @@ uint64_t WriterTile::max_tile_chunk_size_ = constants::max_tile_chunk_size; /* STATIC API */ /* ****************************** */ -Tile Tile::from_generic(storage_size_t tile_size) { - return { +shared_ptr Tile::from_generic(storage_size_t tile_size) { + return make_shared( + HERE(), 0, constants::generic_tile_datatype, constants::generic_tile_cell_size, 0, tile_size, nullptr, - 0}; + 0); } WriterTile WriterTile::from_generic(storage_size_t tile_size) { @@ -151,20 +152,6 @@ Tile::Tile( , filtered_size_(filtered_size) { } -Tile::Tile(Tile&& tile) - : TileBase(std::move(tile)) - , zipped_coords_dim_num_(std::move(tile.zipped_coords_dim_num_)) - , filtered_data_(std::move(tile.filtered_data_)) - , filtered_size_(std::move(tile.filtered_size_)) { -} - -Tile& Tile::operator=(Tile&& tile) { - // Swap with the argument - swap(tile); - - return *this; -} - WriterTile::WriterTile( const format_version_t format_version, const Datatype type, diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index 0bfcd906d4e3..84c38043ce88 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -179,7 +179,7 @@ class Tile : public TileBase { * * @param tile_size to be provided to init_unfiltered call */ - static Tile from_generic(storage_size_t tile_size); + static shared_ptr from_generic(storage_size_t tile_size); /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -206,12 +206,7 @@ class Tile : public TileBase { void* filtered_data, uint64_t filtered_size); - /** Move constructor. */ - Tile(Tile&& tile); - - /** Move-assign operator. */ - Tile& operator=(Tile&& tile); - + DISABLE_MOVE_AND_MOVE_ASSIGN(Tile); DISABLE_COPY_AND_COPY_ASSIGN(Tile); /* ********************************* */ @@ -473,13 +468,13 @@ class WriterTile : public TileBase { */ class TileDeserializer : public Deserializer { public: - explicit TileDeserializer(Tile&& tile) - : Deserializer(tile.data(), tile.size()) - , tile_(std::move(tile)) { + explicit TileDeserializer(shared_ptr tile) + : Deserializer(tile->data(), tile->size()) + , tile_(tile) { } private: - Tile tile_; + shared_ptr tile_; }; } // namespace sm From d6f8bfb9108cd903f17cd3c2ed43aae78a065b22 Mon Sep 17 00:00:00 2001 From: Dave Try Date: Mon, 26 Feb 2024 07:43:43 -0500 Subject: [PATCH 203/456] feature: trigger ci job in gitlab (#4740) Runs TIleDB-REST-CI on gitlab, from the TileDB repository. This ensures the result of this pipeline is displayed as a status check on PRs and `dev/release` branches. **Test plan** Triggers successfully when running a similar, albeit simpler workflow on a test project. TYPE: NO_HISTORY --- .gitlab-ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000000..0cb52e929843 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,21 @@ +stages: + - test + +trigger_pipeline: + stage: test + rules: + - if: $CI_COMMIT_BRANCH =~ /^dev|^release-.*/ || $CI_COMMIT_TAG != "" # only/except rules are no longer actively developed. Please use `rules` instead. + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" + changes: + - "!.github/workflows/quarto-render.yml" + - "!_quarto.yml" + - "!quarto-materials/*" + - "!**/.md" + - "!tiledb/doxygen/source/*" + - "!tiledb/sm/c_api/tiledb_version.h" + + variables: + TILEDB_REF: ${CI_COMMIT_REF_NAME} + trigger: + project: tiledb-inc/tiledb-internal + strategy: depend From f31fa3bc89bc6788a9dac1b08bf912ea38387bdf Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 26 Feb 2024 22:21:59 +0200 Subject: [PATCH 204/456] Use Ninja in Windows CI and explicitly set the VS toolset version. (#4759) [SC-41556] Since the GHA `windows-2022` runners [updated their version of Visual Studio yesterday](https://github.com/actions/runner-images/releases/tag/win22%2F20240218.2), we have been seeing errors of the form `error LNK2019: unresolved external symbol _Thrd_sleep_for`. After some investigation, the cause of the errors was found to be that the vcpkg dependencies were built with the new MSVC toolset (14.39), while the main CMake project was built with the old one (14.38)[^1]. After efforts to make the main project build with the 14.39 toolset (by passing `-T v143,version=14.39` or `-DCMAKE_VS_PLATFORM_TOOLSET_VERSION=14.39`) were unsuccessful, this PR switches to using the Ninja generator instead of Visual Studio. With Ninja the toolset version is specified by [a GitHub action](https://github.com/marketplace/actions/setup-msvc-developer-command-prompt) that sets the necessary environment variables. [^1]: If the opposite were to happen, it would be supported. Objects built with varying versions of MSVC toolsets [are binary-compatible](https://learn.microsoft.com/en-us/cpp/porting/binary-compat-2015-2017?view=msvc-170) with each other, but the final linking must be done with the newest version. --- TYPE: BUILD DESC: Fix linker errors when building with MSVC --- .github/workflows/build-windows.yml | 37 +++++++++++++++++++---------- bootstrap.ps1 | 11 ++++++++- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 6c2d693e3ca5..0fb5879d53ac 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -60,8 +60,22 @@ jobs: TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }} TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' + # On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory + CONFIG_PATH_FIXUP: ${{ matrix.os == 'windows-2019' && 'Release' || '' }} VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' steps: + # By default Visual Studio chooses the earliest installed toolset version + # for the main build and vcpkg chooses the latest. Force it to use the + # latest (14.39 currently). + - name: Setup MSVC toolset (VS 2022) + uses: TheMrMilchmann/setup-msvc-dev@v3 + if: matrix.os == 'windows-2022' + with: + arch: x64 + toolset: 14.39 + - name: Install Ninja (VS 2022) + uses: seanmiddleditch/gha-setup-ninja@v4 + if: matrix.os == 'windows-2022' - name: 'tiledb env prep' run: | $env:BUILD_BUILDDIRECTORY = $env:GITHUB_WORKSPACE.replace("TileDB\TileDB","tdbbd") # 't'ile'db' 'b'uild 'd'ir @@ -141,7 +155,7 @@ jobs: # allow double-checking path cmd /c "echo $PATH" - $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + " -CMakeGenerator ${{ matrix.os == 'windows-2022' && 'Ninja' || '`"Visual Studio 16 2019`"' }}" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } @@ -239,10 +253,7 @@ jobs: } # CMake exits with non-0 status if there are any warnings during the build, so - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit --config $CMakeBuildType -- /verbosity:minimal - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target unit_vfs --config $CMakeBuildType -- /verbosity:minimal - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_regression --config $CMakeBuildType -- /verbosity:minimal - cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target all_link_complete --config $CMakeBuildType -- /verbosity:minimal + cmake --build $env:BUILD_BUILDDIRECTORY\tiledb -j --target tiledb_unit unit_vfs tiledb_regression all_link_complete --config $CMakeBuildType if ($env:TILEDB_AZURE -eq "ON") { if($env.TILEDB_USE_CUSTOM_NODE_JS) { @@ -273,7 +284,7 @@ jobs: # Actually run tests - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$CMakeBuildType\tiledb_unit.exe -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$env:CONFIG_PATH_FIXUP\tiledb_unit.exe -d=yes" Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { @@ -281,7 +292,7 @@ jobs: $host.SetShouldExit($LastExitCode) } - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$CMakeBuildType\unit_vfs -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$env:CONFIG_PATH_FIXUP\unit_vfs -d=yes" Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { @@ -289,7 +300,7 @@ jobs: $host.SetShouldExit($LastExitCode) } - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$CMakeBuildType\test_assert.exe -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$env:CONFIG_PATH_FIXUP\test_assert.exe -d=yes" Invoke-Expression $cmds if ($LastExitCode -ne 0) { Write-Host "Tests failed. test_assert exit status: " $LastExitCocde @@ -311,7 +322,7 @@ jobs: $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api") $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data") - Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$CMakeBuildType") -Filter *.exe | + Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe | Foreach-Object { try { Set-Location -path $TestAppDir @@ -344,7 +355,7 @@ jobs: $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api") $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data") - Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$CMakeBuildType") -Filter *.exe | + Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe | Foreach-Object { try { Set-Location -path $TestAppDir @@ -389,12 +400,12 @@ jobs: cd build # Build zip artifact - cmake -A X64 -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. + cmake ${{ matrix.os != 'windows-2019' && '-G Ninja' || '' }} -DCMAKE_BUILD_TYPE="$CMakeBuildType" -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. cmake --build . --config $CMakeBuildType -v - #.\$CMakeBuildType\ExampleExe.exe - $cmd = ".\$CMakeBuildType\ExampleExe.exe" + #.\$env:CONFIG_PATH_FIXUP\ExampleExe.exe + $cmd = ".\$env:CONFIG_PATH_FIXUP\ExampleExe.exe" Write-Host "cmd: '$cmd'" Invoke-Expression $cmd diff --git a/bootstrap.ps1 b/bootstrap.ps1 index e9a7e7c3895e..66fa18b6ba30 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -27,6 +27,9 @@ Specify the linkage type to build TileDB with. Valid values are .PARAMETER RemoveDeprecations Build TileDB without any deprecated APIs. +.PARAMETER Architecture +Specify the architecture to configure for. + .PARAMETER CMakeGenerator Optionally specify the CMake generator string, e.g. "Visual Studio 15 2017". Check 'cmake --help' for a list of supported generators. @@ -128,6 +131,7 @@ Param( [string]$Dependency, [string]$Linkage = "shared", [switch]$RemoveDeprecations, + [string]$Architecture, [string]$CMakeGenerator, [switch]$EnableAssert, [switch]$EnableDebug, @@ -327,6 +331,11 @@ if (![string]::IsNullOrEmpty($Dependency)) { $DependencyDir = $Dependency } +$ArchFlag = "" +if ($PSBoundParameters.ContainsKey("Architecture")) { + $ArchFlag = "-A $Architecture" +} + # Set CMake generator type. $GeneratorFlag = "" if ($PSBoundParameters.ContainsKey("CMakeGenerator")) { @@ -348,7 +357,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake $ArchFlag -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" From 4c53f93c379faf25b89205ab2a9077e2c05b9158 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 27 Feb 2024 01:56:06 -0600 Subject: [PATCH 205/456] Instrument the Tile class (#4705) Instrument the Tile class with the pmr tracking code. --- TYPE: NO_HISTORY DESC: Instrument the Tile class --------- Co-authored-by: Luc Rancourt --- scripts/find_heap_api_violations.py | 4 +- test/src/unit-DenseTiler.cc | 435 ++++++++++++------ test/src/unit-ReadCellSlabIter.cc | 23 +- test/src/unit-Reader.cc | 27 +- ...it-cppapi-consolidation-with-timestamps.cc | 4 +- test/src/unit-enumerations.cc | 25 +- test/src/unit-filter-pipeline.cc | 202 ++++---- test/src/unit-result-coords.cc | 32 +- test/src/unit-result-tile.cc | 7 +- test/src/unit-sparse-global-order-reader.cc | 4 +- .../unit-sparse-unordered-with-dups-reader.cc | 6 +- test/src/unit-tile-metadata-generator.cc | 38 +- test/support/CMakeLists.txt | 5 + test/support/test/compile_mem_helpers_main.cc | 34 ++ tiledb/common/memory_tracker.cc | 22 +- tiledb/common/memory_tracker.h | 19 +- tiledb/common/pmr.h | 55 +++ tiledb/sm/array/array.cc | 10 +- tiledb/sm/array/array_directory.cc | 29 +- tiledb/sm/array/array_directory.h | 14 +- tiledb/sm/c_api/tiledb.cc | 10 +- tiledb/sm/consolidator/consolidator.cc | 4 + tiledb/sm/consolidator/consolidator.h | 3 + .../fragment_meta_consolidator.cc | 7 +- tiledb/sm/filter/test/CMakeLists.txt | 4 +- tiledb/sm/filter/test/tile_data_generator.h | 26 +- .../filter/test/unit_run_filter_pipeline.cc | 211 +++++++-- tiledb/sm/fragment/fragment_info.cc | 17 +- tiledb/sm/fragment/fragment_metadata.cc | 44 +- tiledb/sm/group/group.cc | 22 +- tiledb/sm/metadata/test/CMakeLists.txt | 4 +- tiledb/sm/metadata/test/unit_metadata.cc | 24 +- tiledb/sm/query/ast/test/CMakeLists.txt | 2 +- .../deletes_and_updates.cc | 6 +- .../deletes_and_updates/serialization.cc | 16 +- .../query/deletes_and_updates/serialization.h | 7 +- .../deletes_and_updates/test/CMakeLists.txt | 5 +- .../test/unit_delete_condition.cc | 4 +- .../test/unit_update_condition.cc | 6 +- tiledb/sm/query/legacy/reader.cc | 6 +- .../query/readers/attribute_order_validator.h | 32 +- .../query/readers/ordered_dim_label_reader.cc | 5 +- tiledb/sm/query/readers/reader_base.cc | 12 +- tiledb/sm/query/readers/reader_base.h | 3 +- tiledb/sm/query/readers/result_space_tile.h | 23 +- tiledb/sm/query/readers/result_tile.cc | 58 ++- tiledb/sm/query/readers/result_tile.h | 76 ++- .../readers/sparse_global_order_reader.cc | 3 +- .../sparse_unordered_with_dups_reader.cc | 2 +- tiledb/sm/query/test/unit_query_condition.cc | 39 +- tiledb/sm/query/writers/dense_tiler.cc | 7 +- tiledb/sm/query/writers/dense_tiler.h | 4 + .../sm/query/writers/global_order_writer.cc | 19 +- tiledb/sm/query/writers/ordered_writer.cc | 9 +- tiledb/sm/query/writers/unordered_writer.cc | 16 +- tiledb/sm/query/writers/writer_base.cc | 8 +- .../sm/storage_manager/context_resources.cc | 7 + tiledb/sm/storage_manager/context_resources.h | 16 + tiledb/sm/storage_manager/storage_manager.cc | 26 +- tiledb/sm/tile/generic_tile_io.cc | 16 +- tiledb/sm/tile/generic_tile_io.h | 6 +- tiledb/sm/tile/test/CMakeLists.txt | 7 +- tiledb/sm/tile/test/unit_tile.cc | 13 +- tiledb/sm/tile/tile.cc | 79 +++- tiledb/sm/tile/tile.h | 72 ++- tiledb/sm/tile/writer_tile_tuple.cc | 22 +- tiledb/sm/tile/writer_tile_tuple.h | 6 +- 67 files changed, 1466 insertions(+), 543 deletions(-) create mode 100644 test/support/test/compile_mem_helpers_main.cc diff --git a/scripts/find_heap_api_violations.py b/scripts/find_heap_api_violations.py index a0ffd80947eb..1bc28dd84638 100755 --- a/scripts/find_heap_api_violations.py +++ b/scripts/find_heap_api_violations.py @@ -101,11 +101,11 @@ # Contains per-file exceptions to violations of "make_unique". unique_ptr_exceptions = { - "*": ["tdb_unique_ptr", "tiledb_unique_ptr"], + "*": ["tdb_unique_ptr", "tiledb_unique_ptr", "tdb::pmr::unique_ptr"], "zstd_compressor.h": ["std::unique_ptr ctx_;", "std::unique_ptr ctx_;"], "posix.cc": ["static std::unique_ptr cwd_(getcwd(nullptr, 0), free);"], "curl.h": ["std::unique_ptr"], - "tile.h": ["std::unique_ptr data_;"], + "pmr.h": ["std::unique_ptr", "unique_ptr make_unique("], } diff --git a/test/src/unit-DenseTiler.cc b/test/src/unit-DenseTiler.cc index 785e07e8909f..f73f2834fed7 100644 --- a/test/src/unit-DenseTiler.cc +++ b/test/src/unit-DenseTiler.cc @@ -64,6 +64,9 @@ struct DenseTilerFx { tiledb_ctx_t* ctx_; tiledb_array_t* array_; + // Test MemoryTracker + shared_ptr tracker_; + // Constructors/Destructors DenseTilerFx(); ~DenseTilerFx(); @@ -86,7 +89,8 @@ struct DenseTilerFx { bool check_tile(WriterTile& tile, const std::vector& data); }; -DenseTilerFx::DenseTilerFx() { +DenseTilerFx::DenseTilerFx() + : tracker_(tiledb::test::create_test_memory_tracker()) { REQUIRE(tiledb_ctx_alloc(NULL, &ctx_) == TILEDB_OK); array_ = NULL; } @@ -211,7 +215,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler1.tile_num() == 2); @@ -232,7 +237,8 @@ TEST_CASE_METHOD( add_ranges({sub2}, sizeof(sub2), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler2.tile_num() == 1); @@ -278,7 +284,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan1_0 = tiler1.copy_plan(0); @@ -314,7 +321,8 @@ TEST_CASE_METHOD( add_ranges({sub2}, sizeof(sub2), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan2 = tiler2.copy_plan(0); @@ -338,7 +346,8 @@ TEST_CASE_METHOD( add_ranges({sub3}, sizeof(sub3), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan3 = tiler3.copy_plan(0); @@ -387,7 +396,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -396,7 +406,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(!tiler1.get_tile(0, "foo", tile1_0).ok()); CHECK(!tiler1.get_tile(10, "a", tile1_0).ok()); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); @@ -410,7 +421,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1 = { 4, fill_value, fill_value, fill_value, fill_value}; @@ -428,7 +440,8 @@ TEST_CASE_METHOD( add_ranges({sub2}, sizeof(sub2), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2( @@ -437,7 +450,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2).ok()); std::vector c_data2 = {fill_value, 1, 2, 3, 4}; CHECK(check_tile(tile2.fixed_tile(), c_data2)); @@ -454,7 +468,8 @@ TEST_CASE_METHOD( add_ranges({sub3}, sizeof(sub3), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3( @@ -463,7 +478,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a", tile3).ok()); std::vector c_data3 = {fill_value, 1, 2, 3, 4}; CHECK(check_tile(tile3.fixed_tile(), c_data3)); @@ -505,7 +521,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -514,7 +531,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(!tiler1.get_tile(0, "foo", tile1_0).ok()); CHECK(!tiler1.get_tile(10, "a", tile1_0).ok()); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); @@ -528,7 +546,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1 = { 4, fill_value, fill_value, fill_value, fill_value}; @@ -571,7 +590,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -580,7 +600,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(!tiler1.get_tile(0, "foo", tile1_0).ok()); CHECK(!tiler1.get_tile(10, "a", tile1_0).ok()); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); @@ -594,7 +615,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1 = { 4, fill_value, fill_value, fill_value, fill_value}; @@ -641,7 +663,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler1.tile_num() == 4); @@ -663,7 +686,8 @@ TEST_CASE_METHOD( add_ranges({sub2_0, sub2_1}, sizeof(sub2_0), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler2.tile_num() == 1); @@ -685,7 +709,8 @@ TEST_CASE_METHOD( add_ranges({sub3_0, sub3_1}, sizeof(sub3_0), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler3.tile_num() == 4); @@ -707,7 +732,8 @@ TEST_CASE_METHOD( add_ranges({sub4_0, sub4_1}, sizeof(sub4_0), &subarray4); // Create DenseTiler - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler4.tile_num() == 1); @@ -757,7 +783,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler1.tile_num() == 4); @@ -779,7 +806,8 @@ TEST_CASE_METHOD( add_ranges({sub2_0, sub2_1}, sizeof(sub2_0), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler2.tile_num() == 1); @@ -801,7 +829,8 @@ TEST_CASE_METHOD( add_ranges({sub3_0, sub3_1}, sizeof(sub3_0), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler3.tile_num() == 4); @@ -823,7 +852,8 @@ TEST_CASE_METHOD( add_ranges({sub4_0, sub4_1}, sizeof(sub4_0), &subarray4); // Create DenseTiler - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test correctness of initialization CHECK(tiler4.tile_num() == 1); @@ -873,7 +903,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan1_0 = tiler1.copy_plan(0); @@ -932,7 +963,8 @@ TEST_CASE_METHOD( add_ranges({sub2_0, sub2_1}, sizeof(sub2_0), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan2_0 = tiler2.copy_plan(0); @@ -958,7 +990,8 @@ TEST_CASE_METHOD( add_ranges({sub3_0, sub3_1}, sizeof(sub3_0), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan3_0 = tiler3.copy_plan(0); @@ -1021,7 +1054,8 @@ TEST_CASE_METHOD( add_ranges({sub4_0, sub4_1}, sizeof(sub4_0), &subarray4); // Create DenseTiler - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan4_0 = tiler4.copy_plan(0); @@ -1076,7 +1110,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan1_0 = tiler1.copy_plan(0); @@ -1139,7 +1174,8 @@ TEST_CASE_METHOD( add_ranges({sub2_0, sub2_1}, sizeof(sub2_0), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan2_0 = tiler2.copy_plan(0); @@ -1166,7 +1202,8 @@ TEST_CASE_METHOD( add_ranges({sub3_0, sub3_1}, sizeof(sub3_0), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan3_0 = tiler3.copy_plan(0); @@ -1225,7 +1262,8 @@ TEST_CASE_METHOD( add_ranges({sub4_0, sub4_1}, sizeof(sub4_0), &subarray4); // Create DenseTiler - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan4_0 = tiler4.copy_plan(0); @@ -1279,7 +1317,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan1_0 = tiler1.copy_plan(0); @@ -1344,7 +1383,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test correctness of copy plan for tile 0 auto copy_plan1_0 = tiler1.copy_plan(0); @@ -1410,7 +1450,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -1419,7 +1460,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0(50); for (int i = 0; i <= 36; ++i) @@ -1439,7 +1481,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1(50); for (int i = 0; i <= 29; ++i) @@ -1461,7 +1504,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2(50); for (int i = 0; i <= 6; ++i) @@ -1479,7 +1523,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3(50); for (int i = 0; i <= 1; ++i) @@ -1504,7 +1549,8 @@ TEST_CASE_METHOD( buff_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_0( @@ -1513,7 +1559,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0(50); for (int i = 0; i <= 21; ++i) @@ -1548,7 +1595,8 @@ TEST_CASE_METHOD( buff_a = {1, 6, 11, 2, 7, 12, 3, 8, 13, 4, 9, 14, 5, 10, 15}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3_0( @@ -1557,7 +1605,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a", tile3_0).ok()); std::vector c_data3_0(50); for (int i = 0; i <= 36; ++i) @@ -1577,7 +1626,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(1, "a", tile3_1).ok()); std::vector c_data3_1(50); for (int i = 0; i <= 29; ++i) @@ -1599,7 +1649,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(2, "a", tile3_2).ok()); std::vector c_data3_2(50); for (int i = 0; i <= 6; ++i) @@ -1617,7 +1668,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(3, "a", tile3_3).ok()); std::vector c_data3_3(50); for (int i = 0; i <= 1; ++i) @@ -1642,7 +1694,8 @@ TEST_CASE_METHOD( buff_a = {1, 7, 13, 2, 8, 14, 3, 9, 15, 4, 10, 16, 5, 11, 17, 6, 12, 18}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile4_0( @@ -1651,7 +1704,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler4.get_tile(0, "a", tile4_0).ok()); std::vector c_data4_0(50); for (int i = 0; i <= 21; ++i) @@ -1712,7 +1766,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -1721,7 +1776,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0(50); for (int i = 0; i <= 37; ++i) @@ -1745,7 +1801,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1(50); for (int i = 0; i <= 34; ++i) @@ -1768,7 +1825,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2(50); for (int i = 0; i <= 2; ++i) @@ -1790,7 +1848,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3(50); c_data1_3[0] = 14; @@ -1817,7 +1876,8 @@ TEST_CASE_METHOD( buff_a = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_0( @@ -1826,7 +1886,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0(50); for (int i = 0; i <= 11; ++i) @@ -1879,7 +1940,8 @@ TEST_CASE_METHOD( buff_a = {1, 6, 11, 2, 7, 12, 3, 8, 13, 4, 9, 14, 5, 10, 15}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3_0( @@ -1888,7 +1950,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a", tile3_0).ok()); std::vector c_data3_0(50); for (int i = 0; i <= 37; ++i) @@ -1912,7 +1975,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(1, "a", tile3_1).ok()); std::vector c_data3_1(50); for (int i = 0; i <= 34; ++i) @@ -1935,7 +1999,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(2, "a", tile3_2).ok()); std::vector c_data3_2(50); for (int i = 0; i <= 2; ++i) @@ -1957,7 +2022,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(3, "a", tile3_3).ok()); std::vector c_data3_3(50); c_data3_3[0] = 14; @@ -1984,7 +2050,8 @@ TEST_CASE_METHOD( buff_a = {1, 7, 13, 2, 8, 14, 3, 9, 15, 4, 10, 16, 5, 11, 17, 6, 12, 18}; buff_a_size = sizeof(buff_a); buffers["a"] = QueryBuffer(&buff_a[0], nullptr, &buff_a_size, nullptr); - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile4_0( @@ -1993,7 +2060,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler4.get_tile(0, "a", tile4_0).ok()); std::vector c_data4_0(50); for (int i = 0; i <= 11; ++i) @@ -2075,7 +2143,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -2084,7 +2153,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0(50); for (int i = 0; i <= 29; ++i) @@ -2100,7 +2170,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1(50); for (int i = 0; i <= 39; ++i) @@ -2152,7 +2223,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -2161,7 +2233,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0(50); for (int i = 0; i <= 34; ++i) @@ -2177,7 +2250,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1(50); for (int i = 0; i <= 9; ++i) @@ -2223,7 +2297,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -2232,7 +2307,8 @@ TEST_CASE_METHOD( false, false, 2 * sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0 = { fill_value, fill_value, fill_value, fill_value, 1, 11, 2, 22, 3, 33}; @@ -2245,7 +2321,8 @@ TEST_CASE_METHOD( false, false, 2 * sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1 = { 4, @@ -2272,7 +2349,8 @@ TEST_CASE_METHOD( add_ranges({sub2}, sizeof(sub2), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2( @@ -2281,7 +2359,8 @@ TEST_CASE_METHOD( false, false, 2 * sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2).ok()); std::vector c_data2 = { fill_value, fill_value, 1, 11, 2, 22, 3, 33, 4, 44}; @@ -2299,7 +2378,8 @@ TEST_CASE_METHOD( add_ranges({sub3}, sizeof(sub3), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3( @@ -2308,7 +2388,8 @@ TEST_CASE_METHOD( false, false, 2 * sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a", tile3).ok()); std::vector c_data3 = { fill_value, fill_value, 1, 11, 2, 22, 3, 33, 4, 44}; @@ -2354,7 +2435,8 @@ TEST_CASE_METHOD( add_ranges({sub1}, sizeof(sub1), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0_a1( @@ -2363,7 +2445,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a1", tile1_0_a1).ok()); std::vector c_data1_0_a1 = {fill_value, fill_value, 1, 2, 3}; CHECK(check_tile(tile1_0_a1.fixed_tile(), c_data1_0_a1)); @@ -2373,7 +2456,8 @@ TEST_CASE_METHOD( false, false, sizeof(double), - Datatype::FLOAT64); + Datatype::FLOAT64, + tracker_); CHECK(tiler1.get_tile(0, "a2", tile1_0_a2).ok()); std::vector c_data1_0_a2 = { double(fill_value), double(fill_value), 1.1, 2.2, 3.3}; @@ -2386,7 +2470,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a1", tile1_1_a1).ok()); std::vector c_data1_1_a1 = { 4, fill_value, fill_value, fill_value, fill_value}; @@ -2397,7 +2482,8 @@ TEST_CASE_METHOD( false, false, sizeof(double), - Datatype::FLOAT64); + Datatype::FLOAT64, + tracker_); CHECK(tiler1.get_tile(1, "a2", tile1_1_a2).ok()); std::vector c_data1_1_a2 = { 4.4, @@ -2419,7 +2505,8 @@ TEST_CASE_METHOD( add_ranges({sub2}, sizeof(sub2), &subarray2); // Create DenseTiler - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_a1( @@ -2428,7 +2515,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a1", tile2_a1).ok()); std::vector c_data2_a1 = {fill_value, 1, 2, 3, 4}; CHECK(check_tile(tile2_a1.fixed_tile(), c_data2_a1)); @@ -2438,7 +2526,8 @@ TEST_CASE_METHOD( false, false, sizeof(double), - Datatype::FLOAT64); + Datatype::FLOAT64, + tracker_); CHECK(tiler2.get_tile(0, "a2", tile2_a2).ok()); std::vector c_data2_a2 = {double(fill_value), 1.1, 2.2, 3.3, 4.4}; CHECK(check_tile(tile2_a2.fixed_tile(), c_data2_a2)); @@ -2455,7 +2544,8 @@ TEST_CASE_METHOD( add_ranges({sub3}, sizeof(sub3), &subarray3); // Create DenseTiler - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3_a1( @@ -2464,7 +2554,8 @@ TEST_CASE_METHOD( false, false, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a1", tile3_a1).ok()); std::vector c_data3_a1 = {fill_value, 1, 2, 3, 4}; CHECK(check_tile(tile3_a1.fixed_tile(), c_data3_a1)); @@ -2474,7 +2565,8 @@ TEST_CASE_METHOD( false, false, sizeof(double), - Datatype::FLOAT64); + Datatype::FLOAT64, + tracker_); CHECK(tiler3.get_tile(0, "a2", tile3_a2).ok()); std::vector c_data3_a2 = {double(fill_value), 1.1, 2.2, 3.3, 4.4}; CHECK(check_tile(tile3_a2.fixed_tile(), c_data3_a2)); @@ -2528,7 +2620,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -2537,7 +2630,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0(50); for (int i = 0; i <= 36; ++i) @@ -2559,7 +2653,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1(50); for (int i = 0; i <= 29; ++i) @@ -2581,7 +2676,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2(50); for (int i = 0; i <= 6; ++i) @@ -2600,7 +2696,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3(50); c_data1_3[0] = 0; @@ -2632,7 +2729,8 @@ TEST_CASE_METHOD( &buff_a_size, nullptr, ValidityVector(&buff_a_n[0], &buff_a_n_size)); - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_0( @@ -2641,7 +2739,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0(50); for (int i = 0; i <= 21; ++i) @@ -2695,7 +2794,8 @@ TEST_CASE_METHOD( &buff_a_size, nullptr, ValidityVector(&buff_a_n[0], &buff_a_n_size)); - DenseTiler tiler3(&buffers, &subarray3, &test::g_helper_stats); + DenseTiler tiler3( + tracker_, &buffers, &subarray3, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile3_0( @@ -2704,7 +2804,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(0, "a", tile3_0).ok()); std::vector c_data3_0(50); for (int i = 0; i <= 36; ++i) @@ -2726,7 +2827,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(1, "a", tile3_1).ok()); std::vector c_data3_1(50); for (int i = 0; i <= 29; ++i) @@ -2748,7 +2850,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(2, "a", tile3_2).ok()); std::vector c_data3_2(50); for (int i = 0; i <= 6; ++i) @@ -2767,7 +2870,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler3.get_tile(3, "a", tile3_3).ok()); std::vector c_data3_3(50); c_data3_3[0] = 1; @@ -2799,7 +2903,8 @@ TEST_CASE_METHOD( &buff_a_size, nullptr, ValidityVector(&buff_a_n[0], &buff_a_n_size)); - DenseTiler tiler4(&buffers, &subarray4, &test::g_helper_stats); + DenseTiler tiler4( + tracker_, &buffers, &subarray4, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile4_0( @@ -2808,7 +2913,8 @@ TEST_CASE_METHOD( false, true, sizeof(int32_t), - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler4.get_tile(0, "a", tile4_0).ok()); std::vector c_data4_0(50); for (int i = 0; i <= 21; ++i) @@ -2887,7 +2993,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -2896,7 +3003,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0_off(50); for (int i = 0; i <= 37; ++i) @@ -2936,7 +3044,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1_off(50); for (int i = 0; i <= 30; ++i) @@ -2984,7 +3093,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2_off(50); for (int i = 0; i <= 7; ++i) @@ -3015,7 +3125,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3_off(50); c_data1_3_off[0] = 0; @@ -3059,7 +3170,8 @@ TEST_CASE_METHOD( buff_a_val_size = buff_a_val.size(); buffers["a"] = QueryBuffer( &buff_a_off[0], &buff_a_val[0], &buff_a_off_size, &buff_a_val_size); - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_0( @@ -3068,7 +3180,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0_off(50); for (int i = 0; i <= 22; ++i) @@ -3222,7 +3335,8 @@ TEST_CASE_METHOD( add_ranges({sub1_0, sub1_1}, sizeof(sub1_0), &subarray1); // Create DenseTiler - DenseTiler tiler1(&buffers, &subarray1, &test::g_helper_stats); + DenseTiler tiler1( + tracker_, &buffers, &subarray1, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile1_0( @@ -3231,7 +3345,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0_off(50); for (int i = 0; i <= 37; ++i) @@ -3271,7 +3386,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1_off(50); for (int i = 0; i <= 30; ++i) @@ -3319,7 +3435,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2_off(50); for (int i = 0; i <= 7; ++i) @@ -3350,7 +3467,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3_off(50); c_data1_3_off[0] = 0; @@ -3414,7 +3532,8 @@ TEST_CASE_METHOD( buff_a_val_size = buff_a_val.size() * sizeof(int32_t); buffers["a"] = QueryBuffer( &buff_a_off[0], &buff_a_val[0], &buff_a_off_size, &buff_a_val_size); - DenseTiler tiler2(&buffers, &subarray2, &test::g_helper_stats); + DenseTiler tiler2( + tracker_, &buffers, &subarray2, &test::g_helper_stats); // Test get tile 0 WriterTileTuple tile2_0( @@ -3423,7 +3542,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::STRING_ASCII); + Datatype::STRING_ASCII, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0_off(50); for (int i = 0; i <= 22; ++i) @@ -3579,7 +3699,7 @@ TEST_CASE_METHOD( // Create DenseTiler DenseTiler tiler1( - &buffers, &subarray1, &test::g_helper_stats, "bytes", 64, true); + tracker_, &buffers, &subarray1, &test::g_helper_stats, "bytes", 64, true); // Test get tile 0 WriterTileTuple tile1_0( @@ -3588,7 +3708,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0_off(50); for (int i = 0; i <= 37; ++i) @@ -3628,7 +3749,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1_off(50); for (int i = 0; i <= 30; ++i) @@ -3676,7 +3798,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2_off(50); for (int i = 0; i <= 7; ++i) @@ -3707,7 +3830,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3_off(50); c_data1_3_off[0] = 0; @@ -3773,7 +3897,7 @@ TEST_CASE_METHOD( buffers["a"] = QueryBuffer( &buff_a_off[0], &buff_a_val[0], &buff_a_off_size, &buff_a_val_size); DenseTiler tiler2( - &buffers, &subarray2, &test::g_helper_stats, "bytes", 64, true); + tracker_, &buffers, &subarray2, &test::g_helper_stats, "bytes", 64, true); // Test get tile 0 WriterTileTuple tile2_0( @@ -3782,7 +3906,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0_off(50); for (int i = 0; i <= 22; ++i) @@ -3923,7 +4048,13 @@ TEST_CASE_METHOD( // Create DenseTiler DenseTiler tiler1( - &buffers, &subarray1, &test::g_helper_stats, "elements", 64, false); + tracker_, + &buffers, + &subarray1, + &test::g_helper_stats, + "elements", + 64, + false); // Test get tile 0 WriterTileTuple tile1_0( @@ -3932,7 +4063,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0_off(50); for (int i = 0; i <= 37; ++i) @@ -3972,7 +4104,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1_off(50); for (int i = 0; i <= 30; ++i) @@ -4020,7 +4153,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2_off(50); for (int i = 0; i <= 7; ++i) @@ -4051,7 +4185,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3_off(50); c_data1_3_off[0] = 0; @@ -4099,7 +4234,13 @@ TEST_CASE_METHOD( buffers["a"] = QueryBuffer( &buff_a_off[0], &buff_a_val[0], &buff_a_off_size, &buff_a_val_size); DenseTiler tiler2( - &buffers, &subarray2, &test::g_helper_stats, "elements", 64, false); + tracker_, + &buffers, + &subarray2, + &test::g_helper_stats, + "elements", + 64, + false); // Test get tile 0 WriterTileTuple tile2_0( @@ -4108,7 +4249,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0_off(50); for (int i = 0; i <= 22; ++i) @@ -4249,7 +4391,13 @@ TEST_CASE_METHOD( // Create DenseTiler DenseTiler tiler1( - &buffers, &subarray1, &test::g_helper_stats, "elements", 32, false); + tracker_, + &buffers, + &subarray1, + &test::g_helper_stats, + "elements", + 32, + false); // Test get tile 0 WriterTileTuple tile1_0( @@ -4258,7 +4406,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(0, "a", tile1_0).ok()); std::vector c_data1_0_off(50); for (int i = 0; i <= 37; ++i) @@ -4298,7 +4447,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(1, "a", tile1_1).ok()); std::vector c_data1_1_off(50); for (int i = 0; i <= 30; ++i) @@ -4346,7 +4496,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(2, "a", tile1_2).ok()); std::vector c_data1_2_off(50); for (int i = 0; i <= 7; ++i) @@ -4377,7 +4528,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler1.get_tile(3, "a", tile1_3).ok()); std::vector c_data1_3_off(50); c_data1_3_off[0] = 0; @@ -4425,7 +4577,13 @@ TEST_CASE_METHOD( buffers["a"] = QueryBuffer( &buff_a_off[0], &buff_a_val[0], &buff_a_off_size, &buff_a_val_size); DenseTiler tiler2( - &buffers, &subarray2, &test::g_helper_stats, "elements", 32, false); + tracker_, + &buffers, + &subarray2, + &test::g_helper_stats, + "elements", + 32, + false); // Test get tile 0 WriterTileTuple tile2_0( @@ -4434,7 +4592,8 @@ TEST_CASE_METHOD( true, false, 1, - Datatype::INT32); + Datatype::INT32, + tracker_); CHECK(tiler2.get_tile(0, "a", tile2_0).ok()); std::vector c_data2_0_off(50); for (int i = 0; i <= 22; ++i) diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index a9f5afc911a1..b84c0a58bec0 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -64,6 +64,8 @@ struct ReadCellSlabIterFx { const char* ARRAY_NAME = "read_cell_slab_iter"; tiledb_array_t* array_ = nullptr; + shared_ptr tracker_; + ReadCellSlabIterFx(); ~ReadCellSlabIterFx(); @@ -83,7 +85,8 @@ struct ReadCellSlabIterFx { }; ReadCellSlabIterFx::ReadCellSlabIterFx() - : fs_vec_(vfs_test_get_fs_vec()) { + : fs_vec_(vfs_test_get_fs_vec()) + , tracker_(tiledb::test::create_test_memory_tracker()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); @@ -163,7 +166,8 @@ void ReadCellSlabIterFx::create_result_space_tiles( tile_coords, array_tile_domain, frag_tile_domains, - result_space_tiles); + result_space_tiles, + tiledb::test::get_test_memory_tracker()); } void set_result_tile_dim( @@ -511,9 +515,12 @@ TEST_CASE_METHOD( // Create result coordinates std::vector result_coords; - ResultTile result_tile_2_0(1, 0, *fragments[0]); - ResultTile result_tile_3_0(2, 0, *fragments[0]); - ResultTile result_tile_3_1(2, 1, *fragments[1]); + ResultTile result_tile_2_0( + 1, 0, *fragments[0], tiledb::test::get_test_memory_tracker()); + ResultTile result_tile_3_0( + 2, 0, *fragments[0], tiledb::test::get_test_memory_tracker()); + ResultTile result_tile_3_1( + 2, 1, *fragments[1], tiledb::test::get_test_memory_tracker()); set_result_tile_dim( array_schema, result_tile_2_0, "d", 0, {{1000, 3, 1000, 5}}); @@ -1362,8 +1369,10 @@ TEST_CASE_METHOD( // Create result coordinates std::vector result_coords; - ResultTile result_tile_3_0(2, 0, *fragments[0]); - ResultTile result_tile_3_1(2, 1, *fragments[1]); + ResultTile result_tile_3_0( + 2, 0, *fragments[0], tiledb::test::get_test_memory_tracker()); + ResultTile result_tile_3_1( + 2, 1, *fragments[1], tiledb::test::get_test_memory_tracker()); set_result_tile_dim( array_schema, result_tile_3_0, "d1", 0, {{1000, 3, 1000, 1000}}); diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 0957023a0b86..a48a803b9467 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -275,38 +275,39 @@ TEST_CASE_METHOD( tile_coords, array_tile_domain, frag_tile_domains, - result_space_tiles); + result_space_tiles, + tiledb::test::get_test_memory_tracker()); CHECK(result_space_tiles.size() == 6); // Initialize result_space_tiles - ResultSpaceTile rst_1_0; + ResultSpaceTile rst_1_0(tiledb::test::get_test_memory_tracker()); rst_1_0.set_start_coords({3, 1}); rst_1_0.append_frag_domain(2, ds2); rst_1_0.append_frag_domain(1, ds1); rst_1_0.set_result_tile(1, 0, *fragments[0]); rst_1_0.set_result_tile(2, 0, *fragments[1]); - ResultSpaceTile rst_1_2; + ResultSpaceTile rst_1_2(tiledb::test::get_test_memory_tracker()); rst_1_2.set_start_coords({3, 11}); rst_1_2.append_frag_domain(1, ds1); rst_1_2.set_result_tile(1, 2, *fragments[0]); - ResultSpaceTile rst_2_0; + ResultSpaceTile rst_2_0(tiledb::test::get_test_memory_tracker()); rst_2_0.set_start_coords({5, 1}); rst_2_0.append_frag_domain(3, ds3); rst_2_0.set_result_tile(3, 0, *fragments[2]); - ResultSpaceTile rst_2_2; + ResultSpaceTile rst_2_2(tiledb::test::get_test_memory_tracker()); rst_2_2.set_start_coords({5, 11}); - ResultSpaceTile rst_3_0; + ResultSpaceTile rst_3_0(tiledb::test::get_test_memory_tracker()); rst_3_0.set_start_coords({7, 1}); rst_3_0.append_frag_domain(3, ds3); rst_3_0.set_result_tile(3, 2, *fragments[2]); - ResultSpaceTile rst_3_2; + ResultSpaceTile rst_3_2(tiledb::test::get_test_memory_tracker()); rst_3_2.set_start_coords({7, 11}); // Check correctness - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[0][0])] == rst_1_0); - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[1][0])] == rst_1_2); - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[2][0])] == rst_2_0); - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[3][0])] == rst_2_2); - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[4][0])] == rst_3_0); - CHECK(result_space_tiles[(const int32_t*)&(tile_coords[5][0])] == rst_3_2); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[0][0])) == rst_1_0); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[1][0])) == rst_1_2); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[2][0])) == rst_2_0); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[3][0])) == rst_2_2); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[4][0])) == rst_3_0); + CHECK(result_space_tiles.at((const int32_t*)&(tile_coords[5][0])) == rst_3_2); } diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 77aa4dedd6d5..43a1369a8d63 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -763,7 +763,7 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "9000"); + cfg.set("sm.mem.total_budget", "10200"); cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); ctx_ = Context(cfg); @@ -822,7 +822,7 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "9000"); + cfg.set("sm.mem.total_budget", "10200"); cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); ctx_ = Context(cfg); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index adaf63598f8b..f528b0b8712f 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1117,9 +1117,8 @@ TEST_CASE_METHOD( auto enmr_path = schema->get_enumeration_path_name(enmr_name.value()); - auto tracker = ctx_.resources().create_memory_tracker(); auto loaded = - ad->load_enumerations_from_paths({enmr_path}, enc_key_, tracker); + ad->load_enumerations_from_paths({enmr_path}, enc_key_, memory_tracker_); REQUIRE(loaded.size() == 1); auto enmr = loaded[0]; @@ -1141,7 +1140,6 @@ TEST_CASE_METHOD( auto schema = get_array_schema_latest(); auto ad = get_array_directory(); - auto tracker = ctx_.resources().create_memory_tracker(); // Check that this function throws an exception when attempting to load // an unknown enumeration @@ -1150,7 +1148,8 @@ TEST_CASE_METHOD( auto windows_matcher = Catch::Matchers::ContainsSubstring( "The system cannot find the file specified."); REQUIRE_THROWS_WITH( - ad->load_enumerations_from_paths({"unknown_enmr"}, enc_key_, tracker), + ad->load_enumerations_from_paths( + {"unknown_enmr"}, enc_key_, memory_tracker_), posix_matcher || windows_matcher); } @@ -1166,21 +1165,20 @@ TEST_CASE_METHOD( auto enmr_name = schema->attribute("attr1")->get_enumeration_name(); auto enmr_path = schema->get_enumeration_path_name(enmr_name.value()); - auto tracker = ctx_.resources().create_memory_tracker(); - tracker->set_budget(1); + memory_tracker_->set_budget(memory_tracker_->get_memory_usage() + 1); // Check that this function throws an exception when attempting to load // an enumeration that exceeds the memory budget. auto matcher = Catch::Matchers::ContainsSubstring( "Error loading enumeration; Insufficient memory budget;"); REQUIRE_THROWS_WITH( - ad->load_enumerations_from_paths({enmr_path}, enc_key_, tracker), + ad->load_enumerations_from_paths({enmr_path}, enc_key_, memory_tracker_), matcher); // Check that the fix is to increase the memory budget. - tracker->set_budget(std::numeric_limits::max()); + memory_tracker_->set_budget(std::numeric_limits::max()); REQUIRE_NOTHROW( - ad->load_enumerations_from_paths({enmr_path}, enc_key_, tracker)); + ad->load_enumerations_from_paths({enmr_path}, enc_key_, memory_tracker_)); } /* ********************************* */ @@ -2414,10 +2412,10 @@ struct TypeParams { EnumerationFx::EnumerationFx() : uri_("enumeration_test_array") - , ctx_(cfg_) { + , ctx_(cfg_) + , memory_tracker_(tiledb::test::create_test_memory_tracker()) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); - memory_tracker_ = tiledb::test::create_test_memory_tracker(); } EnumerationFx::~EnumerationFx() { @@ -2642,7 +2640,8 @@ WriterTile EnumerationFx::serialize_to_tile( SizeComputationSerializer size_serializer; enmr->serialize(size_serializer); - WriterTile tile{WriterTile::from_generic(size_serializer.size())}; + WriterTile tile{ + WriterTile::from_generic(size_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); enmr->serialize(serializer); @@ -2741,7 +2740,7 @@ shared_ptr EnumerationFx::get_array_directory() { shared_ptr EnumerationFx::get_array_schema_latest() { auto array_dir = get_array_directory(); - return array_dir->load_array_schema_latest(enc_key_); + return array_dir->load_array_schema_latest(enc_key_, memory_tracker_); } #ifdef TILEDB_SERIALIZATION diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index a059df46452a..412f9eb6adc8 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -68,12 +68,17 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; -WriterTile make_increasing_tile(const uint64_t nelts) { +WriterTile make_increasing_tile( + const uint64_t nelts, shared_ptr tracker) { const uint64_t tile_size = nelts * sizeof(uint64_t); const uint64_t cell_size = sizeof(uint64_t); WriterTile tile( - constants::format_version, Datatype::UINT64, cell_size, tile_size); + constants::format_version, + Datatype::UINT64, + cell_size, + tile_size, + tracker); for (uint64_t i = 0; i < nelts; i++) { CHECK_NOTHROW(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); } @@ -81,7 +86,8 @@ WriterTile make_increasing_tile(const uint64_t nelts) { return tile; } -WriterTile make_offsets_tile(std::vector& offsets) { +WriterTile make_offsets_tile( + std::vector& offsets, shared_ptr tracker) { const uint64_t offsets_tile_size = offsets.size() * constants::cell_var_offset_size; @@ -89,7 +95,8 @@ WriterTile make_offsets_tile(std::vector& offsets) { constants::format_version, Datatype::UINT64, constants::cell_var_offset_size, - offsets_tile_size); + offsets_tile_size, + tracker); // Set up test data for (uint64_t i = 0; i < offsets.size(); i++) { @@ -102,7 +109,8 @@ WriterTile make_offsets_tile(std::vector& offsets) { return offsets_tile; } -Tile create_tile_for_unfiltering(uint64_t nelts, WriterTile& tile) { +Tile create_tile_for_unfiltering( + uint64_t nelts, WriterTile& tile, shared_ptr tracker) { return { tile.format_version(), tile.type(), @@ -110,7 +118,8 @@ Tile create_tile_for_unfiltering(uint64_t nelts, WriterTile& tile) { 0, tile.cell_size() * nelts, tile.filtered_buffer().data(), - tile.filtered_buffer().size()}; + tile.filtered_buffer().size(), + tracker}; } void run_reverse( @@ -141,8 +150,10 @@ TEST_CASE( tiledb::sm::Config config; REQUIRE(config.set("sm.skip_checksum_validation", "true").ok()); + auto tracker = tiledb::test::create_test_memory_tracker(); + const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); // MD5 FilterPipeline md5_pipeline; @@ -154,7 +165,7 @@ TEST_CASE( CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, md5_pipeline); for (uint64_t n = 0; n < nelts; n++) { @@ -165,7 +176,7 @@ TEST_CASE( } // SHA256 - auto tile2 = make_increasing_tile(nelts); + auto tile2 = make_increasing_tile(nelts, tracker); FilterPipeline sha_256_pipeline; ChecksumMD5Filter sha_256_filter(Datatype::UINT64); @@ -176,7 +187,7 @@ TEST_CASE( CHECK(tile2.size() == 0); CHECK(tile2.filtered_buffer().size() != 0); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile2); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, sha_256_pipeline); for (uint64_t n = 0; n < nelts; n++) { uint64_t elt = 0; @@ -189,6 +200,8 @@ TEST_CASE( TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 1000; @@ -197,7 +210,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); CHECK( pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); @@ -230,7 +243,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { auto compressed_size = tile.filtered_buffer().size(); CHECK(compressed_size < nelts * sizeof(uint64_t)); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -244,7 +257,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); pipeline.get_filter()->set_max_window_size( window_size); @@ -254,7 +267,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -276,7 +289,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { constants::format_version, Datatype::UINT64, sizeof(uint64_t), - nelts * sizeof(uint64_t)); + nelts * sizeof(uint64_t), + tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -289,7 +303,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -312,7 +326,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { constants::format_version, Datatype::UINT32, sizeof(uint32_t), - nelts * sizeof(uint32_t)); + nelts * sizeof(uint32_t), + tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -325,7 +340,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { int32_t elt = 0; @@ -340,7 +355,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { constants::format_version, Datatype::UINT64, sizeof(uint64_t), - nelts * sizeof(uint64_t)); + nelts * sizeof(uint64_t), + tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -353,7 +369,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -369,6 +385,8 @@ TEST_CASE( "[filter][bit-width-reduction][var]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + const uint64_t nelts = 100; // Set up test data @@ -408,8 +426,8 @@ TEST_CASE( pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts); - auto offsets_tile = make_offsets_tile(offsets); + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); WriterTile::set_max_tile_chunk_size(80); CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) @@ -465,7 +483,7 @@ TEST_CASE( auto compressed_size = tile.filtered_buffer().size(); CHECK(compressed_size < nelts * sizeof(uint64_t)); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -480,8 +498,8 @@ TEST_CASE( std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts); - auto offsets_tile = make_offsets_tile(offsets); + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); pipeline.get_filter()->set_max_window_size( window_size); @@ -491,7 +509,7 @@ TEST_CASE( CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -514,8 +532,9 @@ TEST_CASE( constants::format_version, Datatype::UINT64, sizeof(uint64_t), - nelts * sizeof(uint64_t)); - auto offsets_tile = make_offsets_tile(offsets); + nelts * sizeof(uint64_t), + tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -528,7 +547,7 @@ TEST_CASE( CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -552,7 +571,8 @@ TEST_CASE( constants::format_version, Datatype::UINT32, sizeof(uint32_t), - nelts * sizeof(uint32_t)); + nelts * sizeof(uint32_t), + tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -569,7 +589,8 @@ TEST_CASE( constants::format_version, Datatype::UINT64, constants::cell_var_offset_size, - offsets_tile_size); + offsets_tile_size, + tracker); // Set up test data for (uint64_t i = 0; i < offsets.size(); i++) { @@ -585,7 +606,7 @@ TEST_CASE( CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { int32_t elt = 0; @@ -601,7 +622,8 @@ TEST_CASE( constants::format_version, Datatype::UINT64, sizeof(uint64_t), - nelts * sizeof(uint64_t)); + nelts * sizeof(uint64_t), + tracker); // Set up test data for (uint64_t i = 0; i < nelts; i++) { @@ -609,14 +631,14 @@ TEST_CASE( CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - auto offsets_tile = make_offsets_tile(offsets); + auto offsets_tile = make_offsets_tile(offsets, tracker); CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) .ok()); CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -632,6 +654,8 @@ TEST_CASE( TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 1000; @@ -640,7 +664,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); CHECK( pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); @@ -672,7 +696,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { encoded_size == pipeline_metadata_size + filter_metadata_size + nelts * sizeof(uint64_t)); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -686,7 +710,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); pipeline.get_filter()->set_max_window_size( window_size); @@ -695,7 +719,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -707,7 +731,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { } SECTION("- Error on non-positive delta data") { - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); @@ -723,6 +747,8 @@ TEST_CASE( "[filter][positive-delta][var]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + const uint64_t nelts = 100; // Set up test data @@ -759,8 +785,8 @@ TEST_CASE( pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts); - auto offsets_tile = make_offsets_tile(offsets); + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); WriterTile::set_max_tile_chunk_size(80); CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) @@ -817,7 +843,7 @@ TEST_CASE( encoded_size == pipeline_metadata_size + total_md_size + nelts * sizeof(uint64_t)); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -832,8 +858,8 @@ TEST_CASE( std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts); - auto offsets_tile = make_offsets_tile(offsets); + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); pipeline.get_filter()->set_max_window_size( window_size); @@ -844,7 +870,7 @@ TEST_CASE( CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -856,8 +882,8 @@ TEST_CASE( } SECTION("- Error on non-positive delta data") { - auto tile = make_increasing_tile(nelts); - auto offsets_tile = make_offsets_tile(offsets); + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); WriterTile::set_max_tile_chunk_size(80); for (uint64_t i = 0; i < nelts; i++) { @@ -876,9 +902,11 @@ TEST_CASE( TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); FilterPipeline pipeline; ThreadPool tp(4); @@ -890,7 +918,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -908,7 +936,8 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { constants::format_version, Datatype::UINT32, sizeof(uint32_t), - tile_size2); + tile_size2, + tracker); // Set up test data for (uint32_t i = 0; i < nelts2; i++) { @@ -920,7 +949,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { CHECK(tile2.size() == 0); CHECK(tile2.filtered_buffer().size() != 0); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; @@ -934,8 +963,10 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); // Set up test data std::vector sizes{ @@ -966,7 +997,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { } offsets[offsets.size() - 1] = offset; - auto offsets_tile = make_offsets_tile(offsets); + auto offsets_tile = make_offsets_tile(offsets, tracker); FilterPipeline pipeline; ThreadPool tp(4); @@ -979,7 +1010,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -998,7 +1029,8 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { constants::format_version, Datatype::UINT32, sizeof(uint32_t), - tile_size2); + tile_size2, + tracker); // Set up test data for (uint32_t i = 0; i < nelts2; i++) { @@ -1011,7 +1043,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { CHECK(tile2.size() == 0); CHECK(tile2.filtered_buffer().size() != 0); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; @@ -1027,9 +1059,11 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); FilterPipeline pipeline; ThreadPool tp(4); @@ -1041,7 +1075,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -1059,7 +1093,8 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { constants::format_version, Datatype::UINT32, sizeof(uint32_t), - tile_size2); + tile_size2, + tracker); // Set up test data for (uint32_t i = 0; i < nelts2; i++) { @@ -1071,7 +1106,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { CHECK(tile2.size() == 0); CHECK(tile2.filtered_buffer().size() != 0); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; @@ -1085,8 +1120,10 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); // Set up test data std::vector sizes{ @@ -1117,7 +1154,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { } offsets[offsets.size() - 1] = offset; - auto offsets_tile = make_offsets_tile(offsets); + auto offsets_tile = make_offsets_tile(offsets, tracker); FilterPipeline pipeline; ThreadPool tp(4); @@ -1130,7 +1167,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -1149,7 +1186,8 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { constants::format_version, Datatype::UINT32, sizeof(uint32_t), - tile_size2); + tile_size2, + tracker); // Set up test data for (uint32_t i = 0; i < nelts2; i++) { @@ -1162,7 +1200,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { CHECK(tile2.size() == 0); CHECK(tile2.filtered_buffer().size() != 0); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); for (uint64_t i = 0; i < nelts2; i++) { uint32_t elt = 0; @@ -1178,9 +1216,11 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { TEST_CASE("Filter: Test encryption", "[filter][encryption]") { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts); + auto tile = make_increasing_tile(nelts, tracker); SECTION("- AES-256-GCM") { FilterPipeline pipeline; @@ -1204,7 +1244,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { uint64_t elt = 0; @@ -1214,17 +1254,18 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { } // Check error decrypting with wrong key. - tile = make_increasing_tile(nelts); + tile = make_increasing_tile(nelts, tracker); CHECK( pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); key[0]++; filter->set_key(key); - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline, false); // Fix key and check success. - auto unfiltered_tile3 = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile3 = create_tile_for_unfiltering(nelts, tile, tracker); + key[0]--; filter->set_key(key); run_reverse(config, tp, unfiltered_tile3, pipeline); @@ -1242,6 +1283,8 @@ template void testing_float_scaling_filter() { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 100; const uint64_t tile_size = nelts * sizeof(FloatingType); @@ -1263,7 +1306,7 @@ void testing_float_scaling_filter() { } } - WriterTile tile(constants::format_version, t, cell_size, tile_size); + WriterTile tile(constants::format_version, t, cell_size, tile_size, tracker); std::vector float_result_vec; double scale = 2.53; @@ -1307,7 +1350,7 @@ void testing_float_scaling_filter() { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { FloatingType elt = 0.0f; @@ -1338,12 +1381,14 @@ template void testing_xor_filter(Datatype t) { tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set up test data const uint64_t nelts = 100; const uint64_t tile_size = nelts * sizeof(T); const uint64_t cell_size = sizeof(T); - WriterTile tile(constants::format_version, t, cell_size, tile_size); + WriterTile tile(constants::format_version, t, cell_size, tile_size, tracker); // Setting up the random number generator for the XOR filter testing. std::mt19937_64 gen(0x57A672DE); @@ -1368,7 +1413,7 @@ void testing_xor_filter(Datatype t) { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile); + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); for (uint64_t i = 0; i < nelts; i++) { T elt = 0; @@ -1407,6 +1452,7 @@ TEST_CASE("Filter: Test XOR", "[filter][xor]") { TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { FilterPipeline pipeline; + auto tracker = tiledb::test::create_test_memory_tracker(); SECTION("- DoubleDelta filter reinterprets float->int32") { pipeline.add_filter(CompressionFilter( @@ -1469,7 +1515,8 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { constants::format_version, Datatype::FLOAT32, sizeof(float), - sizeof(float) * data.size()); + sizeof(float) * data.size(), + tracker); for (size_t i = 0; i < data.size(); i++) { CHECK_NOTHROW(tile.write(&data[i], i * sizeof(float), sizeof(float))); } @@ -1480,7 +1527,8 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { CHECK(tile.size() == 0); CHECK(tile.filtered_buffer().size() != 0); - auto unfiltered_tile = create_tile_for_unfiltering(data.size(), tile); + auto unfiltered_tile = + create_tile_for_unfiltering(data.size(), tile, tracker); ChunkData chunk_data; unfiltered_tile.load_chunk_data(chunk_data); REQUIRE(pipeline diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index 9dd75d266b7e..a24a509193d3 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -167,7 +167,13 @@ TEST_CASE_METHOD( CResultCoordsFxSmall, "GlobalOrderResultCoords: test max_slab_length", "[globalorderresultcoords][max_slab_length]") { - GlobalOrderResultTile tile(0, 0, false, false, *fx_.frag_md_); + GlobalOrderResultTile tile( + 0, + 0, + false, + false, + *fx_.frag_md_, + tiledb::test::get_test_memory_tracker()); // Test max_slab_length with no bitmap. GlobalOrderResultCoords rc1(&tile, 1); @@ -194,7 +200,13 @@ TEST_CASE_METHOD( CResultCoordsFxSmall, "GlobalOrderResultCoords: test max_slab_length with comparator", "[globalorderresultcoords][max_slab_length_with_comp]") { - GlobalOrderResultTile tile(0, 0, false, false, *fx_.frag_md_); + GlobalOrderResultTile tile( + 0, + 0, + false, + false, + *fx_.frag_md_, + tiledb::test::get_test_memory_tracker()); Cmp cmp; // Test max_slab_length with no bitmap and comparator. @@ -224,7 +236,13 @@ TEST_CASE_METHOD( CResultCoordsFxLarge, "GlobalOrderResultCoords: test max_slab_length with comparator, large tile", "[globalorderresultcoords][max_slab_length_with_comp]") { - GlobalOrderResultTile tile(0, 0, false, false, *fx_.frag_md_); + GlobalOrderResultTile tile( + 0, + 0, + false, + false, + *fx_.frag_md_, + tiledb::test::get_test_memory_tracker()); Cmp cmp; GlobalOrderResultCoords rc1(&tile, 1); @@ -241,7 +259,13 @@ TEST_CASE_METHOD( CResultCoordsFxSmall, "GlobalOrderResultCoords: advance_to_next_cell", "[globalorderresultcoords][advance_to_next_cell]") { - GlobalOrderResultTile tile(0, 0, false, false, *fx_.frag_md_); + GlobalOrderResultTile tile( + 0, + 0, + false, + false, + *fx_.frag_md_, + tiledb::test::get_test_memory_tracker()); Cmp cmp; GlobalOrderResultCoords rc1(&tile, 0); diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index bbac1ee67b2b..fc38bbac6141 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -166,7 +166,8 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); tiledb_domain_free(&domain); - UnorderedWithDupsResultTile tile(0, 0, *frag_md_); + UnorderedWithDupsResultTile tile( + 0, 0, *frag_md_, tiledb::test::get_test_memory_tracker()); // Check the function with an empty bitmap. CHECK(tile.result_num_between_pos(2, 10) == 8); @@ -200,7 +201,7 @@ TEST_CASE_METHOD( std::make_pair(0, 0), memory_tracker_, true); - ResultTile rt(0, 0, frag_md); + ResultTile rt(0, 0, frag_md, tiledb::test::get_test_memory_tracker()); // Make sure cell_num() will return the correct value. if (!first_dim) { @@ -310,7 +311,7 @@ TEST_CASE_METHOD( std::make_pair(0, 0), memory_tracker_, true); - ResultTile rt(0, 0, frag_md); + ResultTile rt(0, 0, frag_md, tiledb::test::get_test_memory_tracker()); // Make sure cell_num() will return the correct value. if (!first_dim) { diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 582f59268ad4..3d2041608301 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -777,7 +777,7 @@ TEST_CASE_METHOD( // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "10000"; + total_budget_ = "10500"; ratio_coords_ = "0.30"; update_config(); @@ -1348,7 +1348,7 @@ TEST_CASE_METHOD( // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "10000"; + total_budget_ = "10500"; ratio_coords_ = "0.30"; update_config(); diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index a76d879eff01..b88b38e3f390 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -1529,7 +1529,8 @@ TEST_CASE_METHOD( // Make a vector of tiles. std::list> rt; for (uint64_t t = 0; t < num_tiles; t++) { - rt.emplace_back(0, t, *fragments[0]); + rt.emplace_back( + 0, t, *fragments[0], tiledb::test::get_test_memory_tracker()); // Allocate and set the bitmap if required. if (bitmaps[t].size() > 0) { @@ -1746,7 +1747,8 @@ TEST_CASE_METHOD( // Make a vector of tiles. std::list> rt; for (uint64_t t = 0; t < num_tiles; t++) { - rt.emplace_back(0, t, *fragments[0]); + rt.emplace_back( + 0, t, *fragments[0], tiledb::test::get_test_memory_tracker()); // Allocate and set the bitmap if required. if (bitmaps[t].size() > 0) { diff --git a/test/src/unit-tile-metadata-generator.cc b/test/src/unit-tile-metadata-generator.cc index e254cf3537c3..7854550999eb 100644 --- a/test/src/unit-tile-metadata-generator.cc +++ b/test/src/unit-tile-metadata-generator.cc @@ -32,6 +32,7 @@ #include +#include #include #include "test/support/src/helpers.h" #include "test/support/src/mem_helpers.h" @@ -106,7 +107,8 @@ TEMPLATE_LIST_TEST_CASE( false, nullable, cell_val_num * sizeof(T), - tiledb_type); + tiledb_type, + tiledb::test::create_test_memory_tracker()); auto tile_buff = writer_tile.fixed_tile().data_as(); uint8_t* nullable_buff = nullptr; if (nullable) { @@ -269,7 +271,14 @@ TEMPLATE_LIST_TEST_CASE( // Initialize a new tile. auto tiledb_type = static_cast(type.tiledb_type); - WriterTileTuple writer_tile(schema, 4, false, false, sizeof(T), tiledb_type); + WriterTileTuple writer_tile( + schema, + 4, + false, + false, + sizeof(T), + tiledb_type, + tiledb::test::create_test_memory_tracker()); auto tile_buff = writer_tile.fixed_tile().data_as(); // Once an overflow happens, the computation should abort, try to add a few @@ -295,7 +304,13 @@ TEMPLATE_LIST_TEST_CASE( if constexpr (std::is_signed_v) { // Initialize a new tile. WriterTileTuple writer_tile( - schema, 4, false, false, sizeof(T), tiledb_type); + schema, + 4, + false, + false, + sizeof(T), + tiledb_type, + tiledb::test::create_test_memory_tracker()); auto tile_buff = writer_tile.fixed_tile().data_as(); // Once an overflow happens, the computation should abort, try to add a few @@ -363,7 +378,13 @@ TEST_CASE( // Initialize tile. WriterTileTuple writer_tile( - schema, num_cells, true, nullable, 1, Datatype::CHAR); + schema, + num_cells, + true, + nullable, + 1, + Datatype::CHAR, + tiledb::test::create_test_memory_tracker()); auto offsets_tile_buff = writer_tile.offset_tile().data_as(); // Initialize a new nullable tile. @@ -447,7 +468,14 @@ TEST_CASE( // Store '123' and '12' // Initialize offsets tile. - WriterTileTuple writer_tile(schema, 2, true, false, 1, Datatype::CHAR); + WriterTileTuple writer_tile( + schema, + 2, + true, + false, + 1, + Datatype::CHAR, + tiledb::test::create_test_memory_tracker()); auto offsets_tile_buff = writer_tile.offset_tile().data_as(); offsets_tile_buff[0] = 0; offsets_tile_buff[1] = 3; diff --git a/test/support/CMakeLists.txt b/test/support/CMakeLists.txt index 935578a63124..96a59b151318 100644 --- a/test/support/CMakeLists.txt +++ b/test/support/CMakeLists.txt @@ -112,4 +112,9 @@ commence(object_library tdb_catch) this_target_object_libraries(seedable_global_PRNG) conclude(object_library) +commence(object_library mem_helpers) + this_target_sources(src/mem_helpers.cc) + this_target_object_libraries(baseline) +conclude(object_library) + add_test_subdirectory() diff --git a/test/support/test/compile_mem_helpers_main.cc b/test/support/test/compile_mem_helpers_main.cc new file mode 100644 index 000000000000..df5adeb2dc5f --- /dev/null +++ b/test/support/test/compile_mem_helpers_main.cc @@ -0,0 +1,34 @@ +/** + * @file test/support/test/compile_mem_helpers_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../src/mem_helpers.h" + +int main() { + tiledb::test::get_test_memory_tracker(); + return 0; +} diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 5157efacf9dd..65246947170c 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -50,22 +50,28 @@ class MemoryTrackerException : public common::StatusException { std::string memory_type_to_str(MemoryType type) { switch (type) { - case MemoryType::RTREE: - return "RTree"; + case MemoryType::ENUMERATION: + return "Enumeration"; case MemoryType::FOOTER: return "Footer"; + case MemoryType::GENERIC_TILE_IO: + return "GenericTileIO"; + case MemoryType::RTREE: + return "RTree"; + case MemoryType::TILE_DATA: + return "TileData"; case MemoryType::TILE_OFFSETS: return "TileOffsets"; - case MemoryType::TILE_MIN_VALS: - return "TileMinVals"; case MemoryType::TILE_MAX_VALS: return "TileMaxVals"; - case MemoryType::TILE_SUMS: - return "TileSums"; + case MemoryType::TILE_MIN_VALS: + return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; - case MemoryType::ENUMERATION: - return "Enumeration"; + case MemoryType::TILE_SUMS: + return "TileSums"; + case MemoryType::TILE_WRITER_DATA: + return "TileWriterData"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory type: " + val); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index f06d3dc29ea5..29b34fcb2598 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -90,7 +90,6 @@ #ifndef TILEDB_MEMORY_TRACKER_H #define TILEDB_MEMORY_TRACKER_H -#include #include #include @@ -103,18 +102,21 @@ namespace sm { //** The type of memory to track. */ enum class MemoryType { - RTREE, + ENUMERATION, + ENUMERATION_PATHS, FOOTER, + GENERIC_TILE_IO, + RTREE, + TILE_DATA, TILE_OFFSETS, - TILE_MIN_VALS, TILE_MAX_VALS, - TILE_SUMS, + TILE_MIN_VALS, TILE_NULL_COUNTS, ATTRIBUTES, - ENUMERATION, - ENUMERATION_PATHS, DIMENSION_LABELS, - DIMENSIONS + DIMENSIONS, + TILE_SUMS, + TILE_WRITER_DATA }; /** The type of MemoryTracker. */ @@ -128,7 +130,8 @@ enum class MemoryTrackerType { QUERY_READ, QUERY_WRITE, CONSOLIDATOR, - REST_CLIENT + REST_CLIENT, + EPHEMERAL }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index 3f12802d31d1..0005868f4fc3 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -64,6 +64,61 @@ using polymorphic_allocator = std::pmr::polymorphic_allocator; memory_resource* get_default_resource(); +/* ********************************* */ +/* PMR UNIQUE_PTR DECLARATION */ +/* ********************************* */ + +template +class unique_ptr_deleter { + public: + unique_ptr_deleter() = delete; + + unique_ptr_deleter(memory_resource* resource, size_t size, size_t alignment) + : resource_(resource) + , size_(size) + , alignment_(alignment) { + } + + void operator()(Tp* ptr) { + if (ptr != nullptr) { + resource_->deallocate(ptr, size_, alignment_); + } + } + + void set_size(size_t size) { + size_ = size; + } + + memory_resource* resource_; + size_t size_; + size_t alignment_; +}; + +template +using unique_ptr = std::unique_ptr>; + +template +unique_ptr make_unique( + memory_resource* resource, size_t size, size_t alignment) { + static_assert(std::is_arithmetic_v || std::is_same_v); + + auto alloc_size = size * sizeof(Tp); + Tp* data = static_cast(resource->allocate(alloc_size, alignment)); + + if (data == nullptr) { + throw std::bad_alloc(); + } + + auto deleter = unique_ptr_deleter(resource, alloc_size, alignment); + + return std::unique_ptr>(data, deleter); +} + +template +unique_ptr make_unique(memory_resource* resource, size_t size) { + return make_unique(resource, size, alignof(Tp)); +} + /* ********************************* */ /* PMR VECTOR DECLARATION */ /* ********************************* */ diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index a30b30252583..0a59f975d61c 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1265,7 +1265,8 @@ Array::open_for_reads_without_fragments() { "array_open_read_without_fragments_load_schemas"); // Load array schemas - auto result = array_directory().load_array_schemas(*encryption_key()); + auto result = + array_directory().load_array_schemas(*encryption_key(), memory_tracker_); auto version = std::get<0>(result)->version(); ensure_supported_schema_version_for_read(version); @@ -1290,7 +1291,7 @@ Array::open_for_writes() { // Load array schemas auto&& [array_schema_latest, array_schemas_all] = - array_directory().load_array_schemas(*encryption_key()); + array_directory().load_array_schemas(*encryption_key(), memory_tracker_); // If building experimentally, this library should not be able to // write to newer-versioned or older-versioned arrays @@ -1478,8 +1479,9 @@ void Array::do_load_metadata() { throw_if_not_ok( parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { const auto& uri = array_metadata_to_load[m].uri_; - metadata_tiles[m] = - GenericTileIO::load(resources_, uri, 0, *encryption_key()); + + metadata_tiles[m] = GenericTileIO::load( + resources_, uri, 0, *encryption_key(), memory_tracker_); return Status::Ok(); })); diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index b57a030fd978..6149f77de186 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -92,23 +92,24 @@ ArrayDirectory::ArrayDirectory( shared_ptr ArrayDirectory::load_array_schema_from_uri( ContextResources& resources, const URI& schema_uri, - const EncryptionKey& encryption_key) { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) { auto timer_se = resources.stats().start_timer("sm_load_array_schema_from_uri"); - auto tile = GenericTileIO::load(resources, schema_uri, 0, encryption_key); + auto tile = GenericTileIO::load( + resources, schema_uri, 0, encryption_key, memory_tracker); resources.stats().add_counter("read_array_schema_size", tile->size()); // Deserialize Deserializer deserializer(tile->data(), tile->size()); - auto memory_tracker = resources.create_memory_tracker(); - memory_tracker->set_type(MemoryTrackerType::ARRAY_LOAD); return ArraySchema::deserialize(deserializer, schema_uri, memory_tracker); } shared_ptr ArrayDirectory::load_array_schema_latest( - const EncryptionKey& encryption_key) const { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { auto timer_se = resources_.get().stats().start_timer("sm_load_array_schema_latest"); @@ -119,8 +120,8 @@ shared_ptr ArrayDirectory::load_array_schema_latest( // Load schema from URI const URI& schema_uri = latest_array_schema_uri(); - auto&& array_schema = - load_array_schema_from_uri(resources_.get(), schema_uri, encryption_key); + auto&& array_schema = load_array_schema_from_uri( + resources_.get(), schema_uri, encryption_key, memory_tracker); array_schema->set_array_uri(uri_); @@ -130,9 +131,11 @@ shared_ptr ArrayDirectory::load_array_schema_latest( tuple< shared_ptr, std::unordered_map>> -ArrayDirectory::load_array_schemas(const EncryptionKey& encryption_key) const { +ArrayDirectory::load_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { // Load all array schemas - auto&& array_schemas = load_all_array_schemas(encryption_key); + auto&& array_schemas = load_all_array_schemas(encryption_key, memory_tracker); // Locate the latest array schema const auto& array_schema_latest_name = @@ -145,7 +148,8 @@ ArrayDirectory::load_array_schemas(const EncryptionKey& encryption_key) const { std::unordered_map> ArrayDirectory::load_all_array_schemas( - const EncryptionKey& encryption_key) const { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const { auto timer_se = resources_.get().stats().start_timer("sm_load_all_array_schemas"); @@ -169,7 +173,7 @@ ArrayDirectory::load_all_array_schemas( auto& schema_uri = schema_uris[schema_ith]; try { auto&& array_schema = load_array_schema_from_uri( - resources_.get(), schema_uri, encryption_key); + resources_.get(), schema_uri, encryption_key, memory_tracker); array_schema->set_array_uri(uri_); schema_vector[schema_ith] = array_schema; } catch (std::exception& e) { @@ -1323,7 +1327,8 @@ shared_ptr ArrayDirectory::load_enumeration( .join_path(constants::array_enumerations_dir_name) .join_path(enumeration_path); - auto tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); + auto tile = GenericTileIO::load( + resources_, enmr_uri, 0, encryption_key, memory_tracker); resources_.get().stats().add_counter("read_enumeration_size", tile->size()); if (!memory_tracker->take_memory(tile->size(), MemoryType::ENUMERATION)) { diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 9f29563b981b..5509d1900d77 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -328,7 +328,8 @@ class ArrayDirectory { static shared_ptr load_array_schema_from_uri( ContextResources& resources, const URI& array_schema_uri, - const EncryptionKey& encryption_key); + const EncryptionKey& encryption_key, + shared_ptr memory_tracker); /** * Get the full vac uri using the base URI and a vac uri that might be @@ -350,7 +351,8 @@ class ArrayDirectory { * @return Status, a new ArraySchema */ shared_ptr load_array_schema_latest( - const EncryptionKey& encryption_key) const; + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * It loads and returns the latest schema and all the array schemas @@ -367,7 +369,9 @@ class ArrayDirectory { tuple< shared_ptr, std::unordered_map>> - load_array_schemas(const EncryptionKey& encryption_key) const; + load_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * Loads all schemas of an array from persistent storage into memory. @@ -379,7 +383,9 @@ class ArrayDirectory { * ArraySchemaMap Map of all array schemas found keyed by name */ std::unordered_map> - load_all_array_schemas(const EncryptionKey& encryption_key) const; + load_all_array_schemas( + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) const; /** * Load the enumerations from the provided list of paths. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9da381e74394..9f8c290541fa 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -515,8 +515,11 @@ int32_t tiledb_array_schema_load( return TILEDB_ERR; } + auto tracker = storage_manager->resources().ephemeral_memory_tracker(); + // Load latest array schema - auto&& array_schema_latest = array_dir->load_array_schema_latest(key); + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, tracker); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; @@ -606,8 +609,11 @@ int32_t tiledb_array_schema_load_with_key( return TILEDB_ERR; } + auto tracker = storage_manager->resources().ephemeral_memory_tracker(); + // Load latest array schema - auto&& array_schema_latest = array_dir->load_array_schema_latest(key); + auto&& array_schema_latest = + array_dir->load_array_schema_latest(key, tracker); (*array_schema)->array_schema_ = array_schema_latest; } return TILEDB_OK; diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index db2eba17e2c8..13186653aa09 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -32,6 +32,7 @@ #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/consolidator/array_meta_consolidator.h" #include "tiledb/sm/consolidator/commits_consolidator.h" @@ -105,8 +106,11 @@ ConsolidationMode Consolidator::mode_from_config( Consolidator::Consolidator(StorageManager* storage_manager) : storage_manager_(storage_manager) + , consolidator_memory_tracker_( + storage_manager_->resources().create_memory_tracker()) , stats_(storage_manager_->stats()->create_child("Consolidator")) , logger_(storage_manager_->logger()->clone("Consolidator", ++logger_id_)) { + consolidator_memory_tracker_->set_type(MemoryTrackerType::CONSOLIDATOR); } Consolidator::~Consolidator() = default; diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index b11ca6120b35..ddc1b8aea649 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -242,6 +242,9 @@ class Consolidator { /** The storage manager. */ StorageManager* storage_manager_; + /** The consolidator memory tracker. */ + shared_ptr consolidator_memory_tracker_; + /** The class stats. */ stats::Stats* stats_; diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 8b22b37c40c5..6103c3b8606c 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -124,7 +124,9 @@ Status FragmentMetaConsolidator::consolidate( meta[i]->write_footer(size_computation_serializer); tiles[i].reset(tdb_new( WriterTile, - WriterTile::from_generic(size_computation_serializer.size()))); + WriterTile::from_generic( + size_computation_serializer.size(), + consolidator_memory_tracker_))); Serializer serializer(tiles[i]->data(), tiles[i]->size()); meta[i]->write_footer(serializer); @@ -160,7 +162,8 @@ Status FragmentMetaConsolidator::consolidate( SizeComputationSerializer size_computation_serializer; serialize_data(size_computation_serializer, offset); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), consolidator_memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); serialize_data(serializer, offset); diff --git a/tiledb/sm/filter/test/CMakeLists.txt b/tiledb/sm/filter/test/CMakeLists.txt index b1fc76cc3d65..71844c964ea8 100644 --- a/tiledb/sm/filter/test/CMakeLists.txt +++ b/tiledb/sm/filter/test/CMakeLists.txt @@ -1,5 +1,5 @@ # -# tiledb/sm/filter/CMakeLists.txt +# tiledb/sm/filter/test/CMakeLists.txt # # The MIT License # @@ -37,7 +37,7 @@ commence(unit_test filter_pipeline) conclude(unit_test) commence(unit_test run_filter_pipeline) - this_target_object_libraries(filter_pipeline) + this_target_object_libraries(filter_pipeline mem_helpers) this_target_sources( main.cc add_1_in_place_filter.cc diff --git a/tiledb/sm/filter/test/tile_data_generator.h b/tiledb/sm/filter/test/tile_data_generator.h index 2f4bff73fa6d..61d30af6c1f5 100644 --- a/tiledb/sm/filter/test/tile_data_generator.h +++ b/tiledb/sm/filter/test/tile_data_generator.h @@ -60,12 +60,14 @@ class TileDataGenerator { /** * Returns an empty writer tile with enough room for the input data. */ - WriterTile create_empty_writer_tile() const { + WriterTile create_empty_writer_tile( + shared_ptr memory_tracker) const { return WriterTile( constants::format_version, datatype(), cell_size(), - original_tile_size()); + original_tile_size(), + memory_tracker); } /** @@ -77,14 +79,16 @@ class TileDataGenerator { * test data and the writer offsets tile with the (optional) input offsets * data. */ - virtual std::tuple> - create_writer_tiles() const = 0; + virtual std::tuple> create_writer_tiles( + shared_ptr memory_tracker) const = 0; /** * Returns a tile with the data from the filtered buffer and enough room * for the original tile data. **/ - Tile create_filtered_buffer_tile(FilteredBuffer& filtered_buffer) const { + Tile create_filtered_buffer_tile( + FilteredBuffer& filtered_buffer, + shared_ptr memory_tracker) const { return Tile( constants::format_version, datatype(), @@ -92,7 +96,8 @@ class TileDataGenerator { 0, original_tile_size(), filtered_buffer.data(), - filtered_buffer.size()); + filtered_buffer.size(), + memory_tracker); } /** Returns the size of the original unfiltered data. */ @@ -149,10 +154,10 @@ class IncrementTileDataGenerator : public TileDataGenerator { } } - std::tuple> create_writer_tiles() - const override { + std::tuple> create_writer_tiles( + shared_ptr memory_tracker) const override { // Writer tile. - auto tile = create_empty_writer_tile(); + auto tile = create_empty_writer_tile(memory_tracker); T value{}; for (uint64_t index = 0; index < num_elements_; ++index) { CHECK_NOTHROW(tile.write(&value, index * sizeof(T), sizeof(T))); @@ -179,7 +184,8 @@ class IncrementTileDataGenerator : public TileDataGenerator { constants::format_version, Datatype::UINT64, constants::cell_var_offset_size, - offsets.size() * constants::cell_var_offset_size); + offsets.size() * constants::cell_var_offset_size, + memory_tracker); for (uint64_t index = 0; index < offsets.size(); ++index) { CHECK_NOTHROW(offsets_tile.write( &offsets[index], diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 39b3ac171850..6f91dd7a35c9 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -62,6 +62,7 @@ #include #include +#include #include #include "../bit_width_reduction_filter.h" @@ -169,7 +170,8 @@ void check_run_pipeline_full( std::optional& offsets_tile, FilterPipeline& pipeline, const TileDataGenerator* test_data, - const FilteredTileChecker& filtered_buffer_checker) { + const FilteredTileChecker& filtered_buffer_checker, + shared_ptr memory_tracker) { // Run the pipeline forward. CHECK(pipeline .run_forward( @@ -188,7 +190,7 @@ void check_run_pipeline_full( // Run the data in reverse. auto unfiltered_tile = - test_data->create_filtered_buffer_tile(filtered_buffer); + test_data->create_filtered_buffer_tile(filtered_buffer, memory_tracker); ChunkData chunk_data; unfiltered_tile.load_chunk_data(chunk_data); CHECK(pipeline @@ -220,7 +222,8 @@ void check_run_pipeline_roundtrip( WriterTile& tile, std::optional& offsets_tile, FilterPipeline& pipeline, - const TileDataGenerator* test_data) { + const TileDataGenerator* test_data, + shared_ptr memory_tracker) { // Run the pipeline forward. CHECK(pipeline .run_forward( @@ -234,8 +237,8 @@ void check_run_pipeline_roundtrip( CHECK(tile.size() == 0); // Run the data in reverse. - auto unfiltered_tile = - test_data->create_filtered_buffer_tile(tile.filtered_buffer()); + auto unfiltered_tile = test_data->create_filtered_buffer_tile( + tile.filtered_buffer(), memory_tracker); ChunkData chunk_data; unfiltered_tile.load_chunk_data(chunk_data); CHECK(pipeline @@ -259,10 +262,13 @@ TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create pipeline. @@ -281,7 +287,8 @@ TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE( @@ -290,10 +297,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create pipeline. @@ -312,7 +322,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE( @@ -321,10 +332,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); // Create pipeline to test and expected filtered data checker. @@ -341,7 +355,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE( @@ -350,10 +365,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; FilterPipeline pipeline; @@ -373,7 +391,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -394,7 +413,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -405,10 +425,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; @@ -428,7 +451,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -450,7 +474,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -461,10 +486,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create pipeline to test. @@ -484,7 +512,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -504,7 +533,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -515,10 +545,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; @@ -538,7 +571,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -558,7 +592,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -569,10 +604,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create filter pipeline. @@ -595,7 +633,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE( @@ -605,10 +644,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; @@ -630,7 +672,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { @@ -638,10 +681,13 @@ TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create filter pipeline. @@ -663,7 +709,8 @@ TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -693,7 +740,8 @@ TEST_CASE("Filter: Test pseudo-checksum", "[filter][pseudo-checksum]") { offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -703,10 +751,13 @@ TEST_CASE( Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); // Create filter pipeline. @@ -729,7 +780,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } SECTION("- Multi-stage") { @@ -761,7 +813,8 @@ TEST_CASE( offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } } @@ -770,10 +823,13 @@ TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; // Create filter pipeline. @@ -804,7 +860,8 @@ TEST_CASE("Filter: Test pipeline modify filter", "[filter][modify]") { offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { @@ -812,10 +869,12 @@ TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. SimpleVariableTestData test_data{}; auto&& [tile, offsets_tile] = - test_data.tile_data_generator().create_writer_tiles(); + test_data.tile_data_generator().create_writer_tiles(tracker); const auto& elements_per_chunk = test_data.elements_per_chunk(); FilterPipeline pipeline; @@ -845,7 +904,8 @@ TEST_CASE("Filter: Test pipeline modify filter var", "[filter][modify][var]") { offsets_tile, pipeline, &test_data.tile_data_generator(), - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { @@ -853,10 +913,13 @@ TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); std::vector elements_per_chunk{100}; const uint64_t expected_checksum = 5350; @@ -894,7 +957,8 @@ TEST_CASE("Filter: Test pipeline copy", "[filter][copy]") { offsets_tile, pipeline, &tile_data_generator, - filtered_buffer_checker); + filtered_buffer_checker, + tracker); } TEST_CASE("Filter: Test random pipeline", "[filter][random]") { @@ -902,6 +966,8 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + // Create an encryption key. EncryptionKey encryption_key; REQUIRE(encryption_key @@ -947,7 +1013,8 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { 100); for (int i = 0; i < 100; i++) { // Create fresh input tiles. - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); // Construct a random pipeline FilterPipeline pipeline; @@ -980,7 +1047,13 @@ TEST_CASE("Filter: Test random pipeline", "[filter][random]") { // input data. // Run the pipeline tests. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } } @@ -989,11 +1062,13 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { Config config; ThreadPool tp(4); FilterPipeline pipeline; + auto tracker = tiledb::test::create_test_memory_tracker(); // Set-up test data. IncrementTileDataGenerator tile_data_generator( 100); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); SECTION("- Simple") { pipeline.add_filter(Add1InPlace(Datatype::UINT64)); @@ -1004,7 +1079,13 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } SECTION("- With checksum stage") { @@ -1015,7 +1096,13 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } SECTION("- With multiple stages") { @@ -1028,7 +1115,13 @@ TEST_CASE("Filter: Test compression", "[filter][compression]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } } @@ -1036,11 +1129,13 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Create TileDB resources for running the filter pipeline. Config config; ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); // Set-up test data. SimpleVariableTestData test_data{}; const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = tile_data_generator.create_writer_tiles(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); FilterPipeline pipeline; @@ -1053,7 +1148,13 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } SECTION("- With checksum stage") { @@ -1064,7 +1165,13 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } SECTION("- With multiple stages") { @@ -1077,6 +1184,12 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { // Check the pipelines run forward and backward without error and returns // the input data. check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator); + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); } } diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 8fa86a2fe7ca..0f0f42eae20a 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -750,8 +750,9 @@ shared_ptr FragmentInfo::get_array_schema(uint32_t fid) { } EncryptionKey encryption_key; + auto tracker = resources_->ephemeral_memory_tracker(); return ArrayDirectory::load_array_schema_from_uri( - *resources_, schema_uri, encryption_key); + *resources_, schema_uri, encryption_key, tracker); } Status FragmentInfo::get_array_schema_name( @@ -954,7 +955,10 @@ Status FragmentInfo::load_and_replace( tuple, std::vector>> load_consolidated_fragment_meta( - ContextResources& resources, const URI& uri, const EncryptionKey& enc_key) { + ContextResources& resources, + const URI& uri, + const EncryptionKey& enc_key, + shared_ptr memory_tracker) { auto timer_se = resources.stats().start_timer("sm_read_load_consolidated_frag_meta"); @@ -963,7 +967,7 @@ load_consolidated_fragment_meta( throw StatusException(Status_FragmentInfoError( "Cannot load consolidated fragment metadata; URI is empty.")); - auto tile = GenericTileIO::load(resources, uri, 0, enc_key); + auto tile = GenericTileIO::load(resources, uri, 0, enc_key, memory_tracker); resources.stats().add_counter("consolidated_frag_meta_size", tile->size()); @@ -999,11 +1003,12 @@ FragmentInfo::load_array_schemas_and_fragment_metadata( "sm_load_array_schemas_and_fragment_metadata"); // Load array schemas + auto tracker = resources.ephemeral_memory_tracker(); std::shared_ptr array_schema_latest; std::unordered_map> array_schemas_all; std::tie(array_schema_latest, array_schemas_all) = - array_dir.load_array_schemas(enc_key); + array_dir.load_array_schemas(enc_key, tracker); const auto filtered_fragment_uris = [&]() { auto timer_se = @@ -1019,8 +1024,8 @@ FragmentInfo::load_array_schemas_and_fragment_metadata( meta_uris.size()); throw_if_not_ok( parallel_for(&resources.compute_tp(), 0, meta_uris.size(), [&](size_t i) { - auto&& [tile_opt, offsets] = - load_consolidated_fragment_meta(resources, meta_uris[i], enc_key); + auto&& [tile_opt, offsets] = load_consolidated_fragment_meta( + resources, meta_uris[i], enc_key, memory_tracker); fragment_metadata_tiles[i] = tile_opt; offsets_vectors[i] = std::move(offsets); return Status::Ok(); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index f3ad74325aec..b9d1ecb5ef66 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -3605,7 +3605,8 @@ void FragmentMetadata::load_v1_v2( std::string(constants::fragment_metadata_filename)); // Read metadata GenericTileIO tile_io(*resources_, fragment_metadata_uri); - auto tile = tile_io.read_generic(0, encryption_key, resources_->config()); + auto tile = tile_io.read_generic( + 0, encryption_key, resources_->config(), memory_tracker_); resources_->stats().add_counter("read_frag_meta_size", tile->size()); @@ -3889,7 +3890,8 @@ WriterTile FragmentMetadata::write_rtree() { SizeComputationSerializer size_computation_serializer; rtree_.serialize(size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); rtree_.serialize(serializer); @@ -3945,7 +3947,8 @@ shared_ptr FragmentMetadata::read_generic_tile_from_file( // Read metadata GenericTileIO tile_io(*resources_, fragment_metadata_uri); - return tile_io.read_generic(offset, encryption_key, resources_->config()); + return tile_io.read_generic( + offset, encryption_key, resources_->config(), memory_tracker_); } void FragmentMetadata::read_file_footer( @@ -3958,7 +3961,7 @@ void FragmentMetadata::read_file_footer( // Get footer offset get_footer_offset_and_size(footer_offset, footer_size); - tile = Tile::from_generic(*footer_size); + tile = Tile::from_generic(*footer_size, memory_tracker_); resources_->stats().add_counter("read_frag_meta_size", *footer_size); @@ -4010,7 +4013,8 @@ void FragmentMetadata::store_tile_offsets( SizeComputationSerializer size_computation_serializer; write_tile_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_offsets(idx, serializer); @@ -4037,7 +4041,8 @@ void FragmentMetadata::store_tile_var_offsets( SizeComputationSerializer size_computation_serializer; write_tile_var_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_var_offsets(idx, serializer); @@ -4065,7 +4070,8 @@ void FragmentMetadata::store_tile_var_sizes( SizeComputationSerializer size_computation_serializer; write_tile_var_sizes(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_var_sizes(idx, serializer); @@ -4092,7 +4098,8 @@ void FragmentMetadata::store_tile_validity_offsets( SizeComputationSerializer size_computation_serializer; write_tile_validity_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_validity_offsets(idx, serializer); @@ -4120,7 +4127,8 @@ void FragmentMetadata::store_tile_mins( SizeComputationSerializer size_computation_serializer; write_tile_mins(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_mins(idx, serializer); @@ -4154,7 +4162,8 @@ void FragmentMetadata::store_tile_maxs( SizeComputationSerializer size_computation_serializer; write_tile_maxs(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_maxs(idx, serializer); @@ -4188,7 +4197,8 @@ void FragmentMetadata::store_tile_sums( SizeComputationSerializer size_computation_serializer; write_tile_sums(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_sums(idx, serializer); @@ -4213,7 +4223,8 @@ void FragmentMetadata::store_tile_null_counts( SizeComputationSerializer size_computation_serializer; write_tile_null_counts(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_tile_null_counts(idx, serializer); @@ -4265,7 +4276,8 @@ void FragmentMetadata::store_fragment_min_max_sum_null_count( SizeComputationSerializer size_computation_serializer; serialize_data(size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); serialize_data(serializer); @@ -4291,7 +4303,8 @@ void FragmentMetadata::store_processed_conditions( SizeComputationSerializer size_computation_serializer; serialize_processed_conditions(size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); serialize_processed_conditions(serializer); @@ -4620,7 +4633,8 @@ void FragmentMetadata::write_has_delete_meta(Serializer& serializer) const { void FragmentMetadata::store_footer(const EncryptionKey&) { SizeComputationSerializer size_computation_serializer; write_footer(size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile.data(), tile.size()); write_footer(serializer); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 3684135ea922..181b71d632dc 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -740,8 +740,12 @@ void Group::load_metadata_from_storage( parallel_for(&resources_.compute_tp(), 0, metadata_num, [&](size_t m) { const auto& uri = group_metadata_to_load[m].uri_; - metadata_tiles[m] = - GenericTileIO::load(resources_, uri, 0, encryption_key); + metadata_tiles[m] = GenericTileIO::load( + resources_, + uri, + 0, + encryption_key, + storage_manager_->resources().ephemeral_memory_tracker()); return Status::Ok(); })); @@ -795,7 +799,12 @@ void Group::load_group_from_uri(const URI& uri) { [[maybe_unused]] auto timer_se = resources_.stats().start_timer("load_group_from_uri"); - auto tile = GenericTileIO::load(resources_, uri, 0, *encryption_key()); + auto tile = GenericTileIO::load( + resources_, + uri, + 0, + *encryption_key(), + storage_manager_->resources().ephemeral_memory_tracker()); resources_.stats().add_counter("read_group_size", tile->size()); @@ -815,7 +824,12 @@ void Group::load_group_from_all_uris(const std::vector& uris) { std::vector> deserializers; for (auto& uri : uris) { - auto tile = GenericTileIO::load(resources_, uri.uri_, 0, *encryption_key()); + auto tile = GenericTileIO::load( + resources_, + uri.uri_, + 0, + *encryption_key(), + storage_manager_->resources().ephemeral_memory_tracker()); resources_.stats().add_counter("read_group_size", tile->size()); diff --git a/tiledb/sm/metadata/test/CMakeLists.txt b/tiledb/sm/metadata/test/CMakeLists.txt index a9c4c368edd0..c4c60b84bd19 100644 --- a/tiledb/sm/metadata/test/CMakeLists.txt +++ b/tiledb/sm/metadata/test/CMakeLists.txt @@ -26,8 +26,8 @@ include(unit_test) commence(unit_test metadata) - this_target_object_libraries(metadata) - # The dependency on the `tile` object library is suspect, but here for now. + this_target_object_libraries(metadata mem_helpers) + # The dependency on the `tile` object library is suspect, but here for now. this_target_object_libraries(tile) this_target_sources(main.cc unit_metadata.cc) conclude(unit_test) diff --git a/tiledb/sm/metadata/test/unit_metadata.cc b/tiledb/sm/metadata/test/unit_metadata.cc index 982902e779f6..bc09b5fdcfc2 100644 --- a/tiledb/sm/metadata/test/unit_metadata.cc +++ b/tiledb/sm/metadata/test/unit_metadata.cc @@ -30,6 +30,7 @@ * This file defines a test `main()` */ +#include #include #include "../metadata.h" #include "tiledb/common/common.h" @@ -50,6 +51,8 @@ inline T& buffer_metadata(void* p) { TEST_CASE( "Metadata: Test metadata deserialization", "[metadata][deserialization]") { + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + std::vector> metadata_tiles; Metadata metadata_to_serialize1, metadata_to_serialize2, @@ -74,8 +77,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer1; metadata_to_serialize1.serialize(size_computation_serializer1); - WriterTile tile1{ - WriterTile::from_generic(size_computation_serializer1.size())}; + WriterTile tile1{WriterTile::from_generic( + size_computation_serializer1.size(), memory_tracker)}; Serializer serializer1(tile1.data(), tile1.size()); metadata_to_serialize1.serialize(serializer1); @@ -84,8 +87,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer2; metadata_to_serialize2.serialize(size_computation_serializer2); - WriterTile tile2{ - WriterTile::from_generic(size_computation_serializer2.size())}; + WriterTile tile2{WriterTile::from_generic( + size_computation_serializer2.size(), memory_tracker)}; Serializer serializer2(tile2.data(), tile2.size()); metadata_to_serialize2.serialize(serializer2); @@ -95,8 +98,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer3; metadata_to_serialize3.serialize(size_computation_serializer3); - WriterTile tile3{ - WriterTile::from_generic(size_computation_serializer3.size())}; + WriterTile tile3{WriterTile::from_generic( + size_computation_serializer3.size(), memory_tracker)}; Serializer serializer3(tile3.data(), tile3.size()); metadata_to_serialize3.serialize(serializer3); @@ -110,7 +113,8 @@ TEST_CASE( 0, tile1.size(), tile1.filtered_buffer().data(), - tile1.filtered_buffer().size()); + tile1.filtered_buffer().size(), + memory_tracker); memcpy(metadata_tiles[0]->data(), tile1.data(), tile1.size()); metadata_tiles[1] = tdb::make_shared( @@ -121,7 +125,8 @@ TEST_CASE( 0, tile2.size(), tile2.filtered_buffer().data(), - tile2.filtered_buffer().size()); + tile2.filtered_buffer().size(), + memory_tracker); memcpy(metadata_tiles[1]->data(), tile2.data(), tile2.size()); metadata_tiles[2] = tdb::make_shared( @@ -132,7 +137,8 @@ TEST_CASE( 0, tile3.size(), tile3.filtered_buffer().data(), - tile3.filtered_buffer().size()); + tile3.filtered_buffer().size(), + memory_tracker); memcpy(metadata_tiles[2]->data(), tile3.data(), tile3.size()); auto meta{Metadata::deserialize(metadata_tiles)}; diff --git a/tiledb/sm/query/ast/test/CMakeLists.txt b/tiledb/sm/query/ast/test/CMakeLists.txt index 9597f820076e..44bd07de4d76 100644 --- a/tiledb/sm/query/ast/test/CMakeLists.txt +++ b/tiledb/sm/query/ast/test/CMakeLists.txt @@ -40,7 +40,7 @@ add_library(ast_test_support_lib STATIC EXCLUDE_FROM_ALL ${AST_TEST_SUPPORT_SOUR # We want tests to continue as normal even as the API is changing, # so don't warn for deprecations, since they'll be escalated to errors. if (NOT MSVC) - target_compile_options(ast_test_support_lib PRIVATE -Wno-deprecated-declarations) + target_compile_options(ast_test_support_lib PRIVATE -Wno-deprecated-declarations) endif() ################################################################ # diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index e871dd2b2a21..f5cdfdc5fbd9 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -153,10 +153,12 @@ Status DeletesAndUpdates::dowork() { WriterTile serialized_condition = update_values_.empty() ? tiledb::sm::deletes_and_updates::serialization::serialize_condition( - condition_->negated_condition()) : + condition_->negated_condition(), query_memory_tracker_) : tiledb::sm::deletes_and_updates::serialization:: serialize_update_condition_and_values( - condition_->negated_condition(), update_values_); + condition_->negated_condition(), + update_values_, + query_memory_tracker_); new_fragment_str += update_values_.empty() ? constants::delete_file_suffix : constants::update_file_suffix; diff --git a/tiledb/sm/query/deletes_and_updates/serialization.cc b/tiledb/sm/query/deletes_and_updates/serialization.cc index ad775b3bdd98..eb4721a743f2 100644 --- a/tiledb/sm/query/deletes_and_updates/serialization.cc +++ b/tiledb/sm/query/deletes_and_updates/serialization.cc @@ -100,9 +100,11 @@ storage_size_t get_serialized_condition_size( return size_computation_serializer.size(); } -WriterTile serialize_condition(const QueryCondition& query_condition) { +WriterTile serialize_condition( + const QueryCondition& query_condition, + shared_ptr memory_tracker) { WriterTile tile{WriterTile::from_generic( - get_serialized_condition_size(query_condition.ast()))}; + get_serialized_condition_size(query_condition.ast()), memory_tracker)}; Serializer serializer(tile.data(), tile.size()); serialize_condition_impl(query_condition.ast(), serializer); @@ -196,10 +198,12 @@ storage_size_t get_serialized_update_condition_and_values_size( WriterTile serialize_update_condition_and_values( const QueryCondition& query_condition, - const std::vector& update_values) { - WriterTile tile{ - WriterTile::from_generic(get_serialized_update_condition_and_values_size( - query_condition.ast(), update_values))}; + const std::vector& update_values, + shared_ptr memory_tracker) { + WriterTile tile{WriterTile::from_generic( + get_serialized_update_condition_and_values_size( + query_condition.ast(), update_values), + memory_tracker)}; Serializer serializer(tile.data(), tile.size()); serialize_condition_impl(query_condition.ast(), serializer); diff --git a/tiledb/sm/query/deletes_and_updates/serialization.h b/tiledb/sm/query/deletes_and_updates/serialization.h index 8f2d71846121..758ec69a5624 100644 --- a/tiledb/sm/query/deletes_and_updates/serialization.h +++ b/tiledb/sm/query/deletes_and_updates/serialization.h @@ -48,7 +48,9 @@ enum class NodeType : uint8_t { EXPRESSION = 0, VALUE }; * @param query_condition Query condition to serialize. * @return Serialized query condition tile. */ -WriterTile serialize_condition(const QueryCondition& query_condition); +WriterTile serialize_condition( + const QueryCondition& query_condition, + shared_ptr memory_tracker); /** * Deserializes the condition. @@ -75,7 +77,8 @@ QueryCondition deserialize_condition( */ WriterTile serialize_update_condition_and_values( const QueryCondition& query_condition, - const std::vector& update_values); + const std::vector& update_values, + shared_ptr memory_tracker); /** * Deserializes a condition and update values. diff --git a/tiledb/sm/query/deletes_and_updates/test/CMakeLists.txt b/tiledb/sm/query/deletes_and_updates/test/CMakeLists.txt index 70367c688b25..2b6b3e3662a1 100644 --- a/tiledb/sm/query/deletes_and_updates/test/CMakeLists.txt +++ b/tiledb/sm/query/deletes_and_updates/test/CMakeLists.txt @@ -26,9 +26,8 @@ include(unit_test) commence(unit_test delete_update_condition) - this_target_link_libraries(ast_test_support_lib) this_target_sources(main.cc unit_delete_condition.cc unit_update_condition.cc) # The dependencies can't yet be factored into separate object libraries - target_link_libraries(unit_delete_update_condition PUBLIC TILEDB_CORE_OBJECTS) - target_link_libraries(unit_delete_update_condition PUBLIC TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(ast_test_support_lib) + this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc index 60383922098e..9253c4bd67e6 100644 --- a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc +++ b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc @@ -31,6 +31,7 @@ */ #include "test/support/src/ast_helpers.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/storage_manager/context.h" @@ -46,7 +47,8 @@ using namespace tiledb::sm::deletes_and_updates::serialization; * @param query_condition Condition to check. */ void serialize_deserialize_check(QueryCondition& query_condition) { - auto serialized = serialize_condition(query_condition); + auto tracker = tiledb::test::create_test_memory_tracker(); + auto serialized = serialize_condition(query_condition, tracker); auto deserialized = deserialize_condition(0, "", serialized.data(), serialized.size()); diff --git a/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc b/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc index 8603780215db..ceeda833f689 100644 --- a/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc +++ b/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc @@ -31,6 +31,7 @@ */ #include "test/support/src/ast_helpers.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/storage_manager/context.h" @@ -48,8 +49,9 @@ using namespace tiledb::sm::deletes_and_updates::serialization; */ void serialize_deserialize_check( QueryCondition& query_condition, std::vector& update_values) { - auto serialized = - serialize_update_condition_and_values(query_condition, update_values); + auto tracker = tiledb::test::create_test_memory_tracker(); + auto serialized = serialize_update_condition_and_values( + query_condition, update_values, tracker); auto&& [deserialized_condition, deserialized_update_values] = deserialize_update_condition_and_values( 0, "", serialized.data(), serialized.size()); diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index acf9d1327fad..f99f5812dc0c 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -714,7 +714,8 @@ Status Reader::compute_sparse_result_tiles( auto pair = std::pair(f, t); // Add tile only if it does not already exist if (result_tile_map->find(pair) == result_tile_map->end()) { - result_tiles.emplace_back(f, t, *fragment_metadata_[f].get()); + result_tiles.emplace_back( + f, t, *fragment_metadata_[f].get(), query_memory_tracker_); (*result_tile_map)[pair] = result_tiles.size() - 1; } // Always check range for multiple fragments or fragments with @@ -733,7 +734,8 @@ Status Reader::compute_sparse_result_tiles( auto pair = std::pair(f, t); // Add tile only if it does not already exist if (result_tile_map->find(pair) == result_tile_map->end()) { - result_tiles.emplace_back(f, t, *fragment_metadata_[f].get()); + result_tiles.emplace_back( + f, t, *fragment_metadata_[f].get(), query_memory_tracker_); (*result_tile_map)[pair] = result_tiles.size() - 1; } // Always check range for multiple fragments or fragments with diff --git a/tiledb/sm/query/readers/attribute_order_validator.h b/tiledb/sm/query/readers/attribute_order_validator.h index a0c6c4672c85..77fb6b1e5aab 100644 --- a/tiledb/sm/query/readers/attribute_order_validator.h +++ b/tiledb/sm/query/readers/attribute_order_validator.h @@ -81,8 +81,12 @@ class AttributeOrderValidator { * @param attribute_name Name of the attribute to validate. * @param num_frags Number of fragments. */ - AttributeOrderValidator(const std::string& attribute_name, uint64_t num_frags) - : attribute_name_(attribute_name) + AttributeOrderValidator( + const std::string& attribute_name, + uint64_t num_frags, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attribute_name_(attribute_name) , result_tiles_to_load_(num_frags) , per_fragment_validation_data_(num_frags) { } @@ -526,6 +530,9 @@ class AttributeOrderValidator { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker to use. */ + shared_ptr memory_tracker_; + /** Name of the attribute to validate. */ std::string attribute_name_; @@ -562,7 +569,10 @@ class AttributeOrderValidator { std::piecewise_construct, std::forward_as_tuple(t_to_compare), std::forward_as_tuple( - f_to_compare, t_to_compare, *fragment_metadata.get())); + f_to_compare, + t_to_compare, + *fragment_metadata.get(), + memory_tracker_)); } if (is_lower_bound) { @@ -579,8 +589,12 @@ class AttributeOrderValidator { * @return Tile to compare against. */ inline ResultTile* min_tile_to_compare_against(unsigned f) { - return &result_tiles_to_load_[f][per_fragment_validation_data_[f] - .min_tile_to_compare_to_.value()]; + auto idx = per_fragment_validation_data_[f].min_tile_to_compare_to_.value(); + auto iter = result_tiles_to_load_[f].find(idx); + if (iter == result_tiles_to_load_[f].end()) { + throw std::runtime_error("Invalid minimum tile index."); + } + return &(iter->second); } /** @@ -590,8 +604,12 @@ class AttributeOrderValidator { * @return Tile to compare against. */ inline ResultTile* max_tile_to_compare_against(unsigned f) { - return &result_tiles_to_load_[f][per_fragment_validation_data_[f] - .max_tile_to_compare_to_.value()]; + auto idx = per_fragment_validation_data_[f].max_tile_to_compare_to_.value(); + auto iter = result_tiles_to_load_[f].find(idx); + if (iter == result_tiles_to_load_[f].end()) { + throw std::runtime_error("Invalid maximum tile index."); + } + return &(iter->second); } }; diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index 0c618d9f17cf..33d80255b61a 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -522,7 +522,10 @@ uint64_t OrderedDimLabelReader::create_result_tiles() { std::piecewise_construct, std::forward_as_tuple(tile_idx), std::forward_as_tuple( - f, frag_tile_idx, *fragment_metadata_[f].get())); + f, + frag_tile_idx, + *fragment_metadata_[f].get(), + query_memory_tracker_)); } else { if (r == 0) { throw OrderedDimLabelReaderStatusException( diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index edb84febaaed..8b449fdc988d 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -110,7 +110,8 @@ void ReaderBase::compute_result_space_tiles( const std::vector>& tile_coords, const TileDomain& array_tile_domain, const std::vector>& frag_tile_domains, - std::map>& result_space_tiles) { + std::map>& result_space_tiles, + shared_ptr memory_tracker) { auto fragment_num = (unsigned)frag_tile_domains.size(); auto dim_num = array_tile_domain.dim_num(); std::vector start_coords; @@ -123,7 +124,8 @@ void ReaderBase::compute_result_space_tiles( start_coords = array_tile_domain.start_coords(coords); // Create result space tile and insert into the map - auto r = result_space_tiles.emplace(coords, ResultSpaceTile()); + auto r = + result_space_tiles.emplace(coords, ResultSpaceTile(memory_tracker)); auto& result_space_tile = r.first->second; result_space_tile.set_start_coords(start_coords); @@ -1102,7 +1104,8 @@ void ReaderBase::compute_result_space_tiles( tile_coords, array_tile_domain, frag_tile_domains, - result_space_tiles); + result_space_tiles, + query_memory_tracker_); } bool ReaderBase::has_coords() const { @@ -1173,7 +1176,8 @@ void ReaderBase::validate_attribute_order( auto index_name = index_dim->name(); // See if some values will already be processed by previous fragments. - AttributeOrderValidator validator(attribute_name, fragment_metadata_.size()); + AttributeOrderValidator validator( + attribute_name, fragment_metadata_.size(), query_memory_tracker_); throw_if_not_ok(parallel_for( storage_manager_->compute_tp(), 0, diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index ca69ada01249..3200c272ce22 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -200,7 +200,8 @@ class ReaderBase : public StrategyBase { const std::vector>& tile_coords, const TileDomain& array_tile_domain, const std::vector>& frag_tile_domains, - std::map>& result_space_tiles); + std::map>& result_space_tiles, + shared_ptr memory_tracker); /** * Computes the minimum and maximum indexes of tile chunks to process based on diff --git a/tiledb/sm/query/readers/result_space_tile.h b/tiledb/sm/query/readers/result_space_tile.h index 6225b2a6d103..2758bb9da10a 100644 --- a/tiledb/sm/query/readers/result_space_tile.h +++ b/tiledb/sm/query/readers/result_space_tile.h @@ -47,6 +47,8 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +class MemoryTracker; + /** Fragment domain structure (fragment id, fragment domain). */ struct FragmentDomain { public: @@ -81,8 +83,12 @@ struct FragmentDomain { template class ResultSpaceTile { public: - /** Default constructor. */ - ResultSpaceTile() = default; + /** No default constructor. */ + ResultSpaceTile() = delete; + + ResultSpaceTile(shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) { + } /** Default destructor. */ ~ResultSpaceTile() = default; @@ -132,7 +138,7 @@ class ResultSpaceTile { result_tiles_.emplace( std::piecewise_construct, std::forward_as_tuple(frag_idx), - std::forward_as_tuple(frag_idx, tile_idx, frag_md)); + std::forward_as_tuple(frag_idx, tile_idx, frag_md, memory_tracker_)); } /** Returns the result tile for the input fragment. */ @@ -179,10 +185,19 @@ class ResultSpaceTile { "fragment domain."); } - return result_tiles_[frag_domains_[0].fid()]; + auto iter = result_tiles_.find(frag_domains_[0].fid()); + if (iter == result_tiles_.end()) { + throw std::runtime_error( + "Invalid call to single_result_tile with unknown tile."); + } + + return iter->second; } private: + /** The memory tracker to use. */ + shared_ptr memory_tracker_; + /** The (global) coordinates of the first cell in the space tile. */ std::vector start_coords_; diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index fe22258b9534..3872adb47931 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -68,8 +68,12 @@ bool result_tile_cmp(const ResultTile* a, const ResultTile* b) { /* ****************************** */ ResultTile::ResultTile( - unsigned frag_idx, uint64_t tile_idx, const FragmentMetadata& frag_md) - : domain_(&frag_md.array_schema()->domain()) + unsigned frag_idx, + uint64_t tile_idx, + const FragmentMetadata& frag_md, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , domain_(&frag_md.array_schema()->domain()) , frag_idx_(frag_idx) , tile_idx_(tile_idx) , cell_num_(frag_md.cell_num(tile_idx)) @@ -139,27 +143,55 @@ void ResultTile::init_attr_tile( const std::string& name, const TileSizes tile_sizes, const TileData tile_data) { + auto tuple = TileTuple( + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); + if (name == constants::coords) { coords_tile_.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); return; } if (name == constants::timestamps) { timestamps_tile_.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); return; } if (name == constants::delete_timestamps) { delete_timestamps_tile_.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); return; } if (name == constants::delete_condition_index) { delete_condition_index_tile_.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); return; } @@ -167,7 +199,12 @@ void ResultTile::init_attr_tile( for (auto& at : attr_tiles_) { if (at.first == name && at.second == nullopt) { at.second.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); return; } } @@ -182,7 +219,12 @@ void ResultTile::init_coord_tile( unsigned dim_idx) { coord_tiles_[dim_idx].first = name; coord_tiles_[dim_idx].second.emplace( - format_version, array_schema, name, tile_sizes, tile_data); + format_version, + array_schema, + name, + tile_sizes, + tile_data, + memory_tracker_); // When at least one unzipped coordinate has been initialized, we will // use the unzipped `coord()` implementation. diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index b7997b784b7f..e3949e9285f0 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -266,8 +266,10 @@ class ResultTile { const ArraySchema& array_schema, const std::string& name, const TileSizes tile_sizes, - const TileData tile_data) - : fixed_tile_( + const TileData tile_data, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , fixed_tile_( format_version, tile_sizes.has_var_tile() ? constants::cell_var_offset_type : array_schema.type(name), @@ -276,7 +278,8 @@ class ResultTile { (name == constants::coords) ? array_schema.dim_num() : 0, tile_sizes.tile_size(), tile_data.fixed_filtered_data(), - tile_sizes.tile_persisted_size()) { + tile_sizes.tile_persisted_size(), + memory_tracker_) { if (tile_sizes.has_var_tile()) { auto type = array_schema.type(name); var_tile_.emplace( @@ -286,7 +289,8 @@ class ResultTile { 0, tile_sizes.tile_var_size(), tile_data.var_filtered_data(), - tile_sizes.tile_var_persisted_size()); + tile_sizes.tile_var_persisted_size(), + memory_tracker_); } if (tile_sizes.has_validity_tile()) { @@ -297,7 +301,8 @@ class ResultTile { 0, tile_sizes.tile_validity_size(), tile_data.validity_filtered_data(), - tile_sizes.tile_validity_persisted_size()); + tile_sizes.tile_validity_persisted_size(), + memory_tracker_); } } @@ -340,6 +345,9 @@ class ResultTile { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr memory_tracker_; + /** Stores the fixed data tile. */ Tile fixed_tile_; @@ -354,15 +362,25 @@ class ResultTile { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Default constructor. */ - ResultTile() = default; + /** No Default constructor. */ + ResultTile() = delete; + + /** + * Constructor. + * + * @param memory_tracker The memory tracker to use. + */ + ResultTile(shared_ptr memory_tracker); /** * Constructor. The number of dimensions `dim_num` is used to allocate * the separate coordinate tiles. */ ResultTile( - unsigned frag_idx, uint64_t tile_idx, const FragmentMetadata& frag_md); + unsigned frag_idx, + uint64_t tile_idx, + const FragmentMetadata& frag_md, + shared_ptr memory_tracker); DISABLE_COPY_AND_COPY_ASSIGN(ResultTile); DISABLE_MOVE_AND_MOVE_ASSIGN(ResultTile); @@ -659,6 +677,9 @@ class ResultTile { /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr memory_tracker_; + /** The array domain. */ const Domain* domain_; @@ -798,11 +819,23 @@ class ResultTileWithBitmap : public ResultTile { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - ResultTileWithBitmap() = default; + ResultTileWithBitmap() = delete; + + /** + * Constructor + * + * @param memory_tracker The memory tracker to use. + */ + ResultTileWithBitmap(shared_ptr memory_tracker) + : ResultTile(memory_tracker) { + } ResultTileWithBitmap( - unsigned frag_idx, uint64_t tile_idx, const FragmentMetadata& frag_md) - : ResultTile(frag_idx, tile_idx, frag_md) + unsigned frag_idx, + uint64_t tile_idx, + const FragmentMetadata& frag_md, + shared_ptr memory_tracker) + : ResultTile(frag_idx, tile_idx, frag_md, memory_tracker) , result_num_(cell_num_) { } @@ -933,13 +966,19 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ + + /** No default constructor. */ + GlobalOrderResultTile() = delete; + GlobalOrderResultTile( unsigned frag_idx, uint64_t tile_idx, bool dups, bool include_delete_meta, - const FragmentMetadata& frag_md) - : ResultTileWithBitmap(frag_idx, tile_idx, frag_md) + const FragmentMetadata& frag_md, + shared_ptr memory_tracker) + : ResultTileWithBitmap( + frag_idx, tile_idx, frag_md, memory_tracker) , post_dedup_bitmap_( !dups || include_delete_meta ? optional(std::vector()) : nullopt) @@ -1131,9 +1170,16 @@ class UnorderedWithDupsResultTile : public ResultTileWithBitmap { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ + /** No default memory tracker. */ + UnorderedWithDupsResultTile() = delete; + UnorderedWithDupsResultTile( - unsigned frag_idx, uint64_t tile_idx, const FragmentMetadata& frag_md) - : ResultTileWithBitmap(frag_idx, tile_idx, frag_md) { + unsigned frag_idx, + uint64_t tile_idx, + const FragmentMetadata& frag_md, + shared_ptr memory_tracker) + : ResultTileWithBitmap( + frag_idx, tile_idx, frag_md, memory_tracker) { } DISABLE_MOVE_AND_MOVE_ASSIGN(UnorderedWithDupsResultTile); diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index c83d866ab158..58b4308c0c75 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -332,7 +332,8 @@ bool SparseGlobalOrderReader::add_result_tile( t, array_schema_.allows_dups(), deletes_consolidation_no_purge_, - frag_md); + frag_md, + query_memory_tracker_); return false; } diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index dcc13a574e44..ec9dc6d35acf 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -403,7 +403,7 @@ bool SparseUnorderedWithDupsReader::add_result_tile( memory_used_for_coords_total_ += tiles_size; // Add the result tile. - result_tiles.emplace_back(f, t, frag_md); + result_tiles.emplace_back(f, t, frag_md, query_memory_tracker_); // Are all tiles loaded for this fragment. if (t == last_t) { diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 7c56e08278c1..45bd9ada56b9 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1602,7 +1602,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { true); // Initialize the result tile. - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileSizes tile_sizes( var_size ? (cells + 1) * constants::cell_var_offset_size : 2 * cells * sizeof(char), @@ -1666,7 +1667,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -1781,7 +1783,8 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, *frag_md[0]); + ResultTile result_tile( + 0, 0, *frag_md[0], tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2329,7 +2332,8 @@ void test_apply_dense( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2387,7 +2391,8 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2501,7 +2506,8 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -3031,7 +3037,8 @@ void test_apply_sparse( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -3089,7 +3096,8 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -3860,7 +3868,8 @@ TEST_CASE( std::nullopt, std::nullopt, std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4148,7 +4157,8 @@ TEST_CASE( 0, std::nullopt, std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4561,7 +4571,8 @@ TEST_CASE( 0, std::nullopt, std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4825,7 +4836,8 @@ TEST_CASE( std::nullopt, cells * constants::cell_validity_size, 0); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4927,7 +4939,8 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile(0, 0, frag_md); + ResultTile result_tile( + 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, diff --git a/tiledb/sm/query/writers/dense_tiler.cc b/tiledb/sm/query/writers/dense_tiler.cc index 4b6fe9c5abb0..eaf83b388bbc 100644 --- a/tiledb/sm/query/writers/dense_tiler.cc +++ b/tiledb/sm/query/writers/dense_tiler.cc @@ -53,13 +53,15 @@ namespace sm { template DenseTiler::DenseTiler( + shared_ptr memory_tracker, const std::unordered_map* buffers, const Subarray* subarray, Stats* const parent_stats, const std::string& offsets_format_mode, uint64_t offsets_bitsize, bool offsets_extra_element) - : stats_(parent_stats->create_child("DenseTiler")) + : memory_tracker_(memory_tracker) + , stats_(parent_stats->create_child("DenseTiler")) , array_schema_(subarray->array()->array_schema_latest()) , buffers_(buffers) , subarray_(subarray) @@ -223,7 +225,8 @@ Status DenseTiler::get_tile( constants::format_version, constants::cell_var_offset_type, constants::cell_var_offset_size, - tile_off_size); + tile_off_size, + memory_tracker_); // Fill entire tile with MAX_UINT64 std::vector to_write( diff --git a/tiledb/sm/query/writers/dense_tiler.h b/tiledb/sm/query/writers/dense_tiler.h index 1e46927bb512..fdcfa6d5789c 100644 --- a/tiledb/sm/query/writers/dense_tiler.h +++ b/tiledb/sm/query/writers/dense_tiler.h @@ -146,6 +146,7 @@ class DenseTiler { * from `subarray`). Otherwise, an assertion is raised. */ DenseTiler( + shared_ptr memory_tracker, const std::unordered_map* buffers, const Subarray* subarray, stats::Stats* const parent_stats, @@ -207,6 +208,9 @@ class DenseTiler { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr memory_tracker_; + /** The stats for the dense tiler. */ stats::Stats* stats_; diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index a9110d240600..879715cbf265 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -207,7 +207,8 @@ Status GlobalOrderWriter::init_global_write_state() { var_size, nullable, cell_size, - type)); + type, + query_memory_tracker_)); } catch (const std::logic_error& le) { return Status_WriterError(le.what()); } @@ -969,7 +970,13 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( tiles->reserve(full_tile_num); for (uint64_t i = 0; i < full_tile_num; i++) { tiles->emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, false, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + false, + nullable, + cell_size, + type, + query_memory_tracker_)); } // Handle last tile (it must be either full or empty) @@ -1176,7 +1183,13 @@ Status GlobalOrderWriter::prepare_full_tiles_var( tiles->reserve(full_tile_num); for (uint64_t i = 0; i < full_tile_num; i++) { tiles->emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, true, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + true, + nullable, + cell_size, + type, + query_memory_tracker_)); } // Handle last tile (it must be either full or empty) diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 53e77377970c..aff7ea126352 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -182,6 +182,7 @@ Status OrderedWriter::ordered_write() { // Create a dense tiler DenseTiler dense_tiler( + query_memory_tracker_, &buffers_, &subarray_, stats_, @@ -318,7 +319,13 @@ Status OrderedWriter::prepare_filter_and_write_tiles( tile_batches[b].reserve(batch_size); for (uint64_t i = 0; i < batch_size; i++) { tile_batches[b].emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, var, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + var, + nullable, + cell_size, + type, + query_memory_tracker_)); } { diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 7dc5b57930f7..b2393373ce79 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -428,7 +428,13 @@ Status UnorderedWriter::prepare_tiles_fixed( tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { tiles->emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, false, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + false, + nullable, + cell_size, + type, + query_memory_tracker_)); } // Write all cells one by one @@ -499,7 +505,13 @@ Status UnorderedWriter::prepare_tiles_var( tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { tiles->emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, true, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + true, + nullable, + cell_size, + type, + query_memory_tracker_)); } // Write all cells one by one diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 4c807520382e..b54735a06a98 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -952,7 +952,13 @@ Status WriterBase::init_tiles( tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { tiles->emplace_back(WriterTileTuple( - array_schema_, cell_num_per_tile, var_size, nullable, cell_size, type)); + array_schema_, + cell_num_per_tile, + var_size, + nullable, + cell_size, + type, + query_memory_tracker_)); } return Status::Ok(); diff --git a/tiledb/sm/storage_manager/context_resources.cc b/tiledb/sm/storage_manager/context_resources.cc index bd82b3c319bd..ee9f4df9ecf8 100644 --- a/tiledb/sm/storage_manager/context_resources.cc +++ b/tiledb/sm/storage_manager/context_resources.cc @@ -55,6 +55,7 @@ ContextResources::ContextResources( size_t io_thread_count, std::string stats_name) : memory_tracker_manager_(make_shared(HERE())) + , ephemeral_memory_tracker_(memory_tracker_manager_->create_tracker()) , memory_tracker_reporter_(make_shared( HERE(), config, memory_tracker_manager_)) , config_(config) @@ -63,6 +64,8 @@ ContextResources::ContextResources( , io_tp_(io_thread_count) , stats_(make_shared(HERE(), stats_name)) , vfs_(stats_.get(), &compute_tp_, &io_tp_, config) { + ephemeral_memory_tracker_->set_type(MemoryTrackerType::EPHEMERAL); + /* * Explicitly register our `stats` object with the global. */ @@ -89,4 +92,8 @@ shared_ptr ContextResources::create_memory_tracker() const { return memory_tracker_manager_->create_tracker(); } +shared_ptr ContextResources::ephemeral_memory_tracker() const { + return ephemeral_memory_tracker_; +} + } // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/context_resources.h b/tiledb/sm/storage_manager/context_resources.h index b19cb7f310c9..dea51cfde416 100644 --- a/tiledb/sm/storage_manager/context_resources.h +++ b/tiledb/sm/storage_manager/context_resources.h @@ -119,6 +119,19 @@ class ContextResources { */ shared_ptr create_memory_tracker() const; + /** + * Return the ephemeral memory tracker. + * + * Use this tracker when you have a case where you need a memory tracker + * temporarily, without access to a more appropriate tracker. For instance, + * when using GenericTileIO when deserializing various objects we can use + * this for the GenericTileIO. Make sure to not confuse this with the + * memory tracker that might exists on what's being deserialized. + * + * @return The ephemeral MemoryTracker. + */ + shared_ptr ephemeral_memory_tracker() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -127,6 +140,9 @@ class ContextResources { /** The MemoryTrackerManager for this context. */ mutable shared_ptr memory_tracker_manager_; + /** The ephemeral MemoryTracker. */ + mutable shared_ptr ephemeral_memory_tracker_; + /** The MemoryTrackerReporter for this context. */ mutable shared_ptr memory_tracker_reporter_; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index c7e92b4b2a85..eb2d9350d66c 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -382,7 +382,8 @@ Status StorageManager::array_evolve_schema( "' not exists")); } - auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); + auto&& array_schema = array_dir.load_array_schema_latest( + encryption_key, resources_.ephemeral_memory_tracker()); // Load required enumerations before evolution. auto enmr_names = schema_evolution->enumeration_names_to_extend(); @@ -455,7 +456,8 @@ Status StorageManagerCanonical::array_upgrade_version( static_cast(encryption_key_from_cfg.size()))); } - auto&& array_schema = array_dir.load_array_schema_latest(encryption_key_cfg); + auto&& array_schema = array_dir.load_array_schema_latest( + encryption_key_cfg, resources().ephemeral_memory_tracker()); if (array_schema->version() < constants::format_version) { array_schema->generate_uri(); @@ -1011,7 +1013,8 @@ StorageManagerCanonical::load_delete_and_update_conditions( resources_, uri, locations[i].offset(), - *(opened_array.encryption_key())); + *(opened_array.encryption_key()), + resources_.ephemeral_memory_tracker()); if (tiledb::sm::utils::parse::ends_with( locations[i].condition_marker(), @@ -1263,7 +1266,9 @@ Status StorageManagerCanonical::store_group_detail( SizeComputationSerializer size_computation_serializer; group->serialize(members, size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources_.ephemeral_memory_tracker())}; Serializer serializer(tile.data(), tile.size()); group->serialize(members, serializer); @@ -1293,7 +1298,9 @@ Status StorageManagerCanonical::store_array_schema( SizeComputationSerializer size_computation_serializer; array_schema->serialize(size_computation_serializer); - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources_.ephemeral_memory_tracker())}; Serializer serializer(tile.data(), tile.size()); array_schema->serialize(serializer); @@ -1341,8 +1348,9 @@ Status StorageManagerCanonical::store_array_schema( SizeComputationSerializer enumeration_size_serializer; enmr->serialize(enumeration_size_serializer); - WriterTile tile{ - WriterTile::from_generic(enumeration_size_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + enumeration_size_serializer.size(), + resources_.ephemeral_memory_tracker())}; Serializer serializer(tile.data(), tile.size()); enmr->serialize(serializer); @@ -1371,7 +1379,9 @@ Status StorageManagerCanonical::store_metadata( if (0 == size_computation_serializer.size()) { return Status::Ok(); } - WriterTile tile{WriterTile::from_generic(size_computation_serializer.size())}; + WriterTile tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources_.ephemeral_memory_tracker())}; Serializer serializer(tile.data(), tile.size()); metadata->serialize(serializer); diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 50b8149d5470..4292999719b0 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/unreachable.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/filesystem/vfs.h" @@ -71,15 +72,18 @@ shared_ptr GenericTileIO::load( ContextResources& resources, const URI& uri, uint64_t offset, - const EncryptionKey& encryption_key) { + const EncryptionKey& encryption_key, + shared_ptr memory_tracker) { GenericTileIO tile_io(resources, uri); // Get encryption key from config if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { EncryptionKey cfg_enc_key(resources.config()); - return tile_io.read_generic(offset, cfg_enc_key, resources.config()); + return tile_io.read_generic( + offset, cfg_enc_key, resources.config(), memory_tracker); } else { - return tile_io.read_generic(offset, encryption_key, resources.config()); + return tile_io.read_generic( + offset, encryption_key, resources.config(), memory_tracker); } stdx::unreachable(); @@ -88,7 +92,8 @@ shared_ptr GenericTileIO::load( shared_ptr GenericTileIO::read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, - const Config& config) { + const Config& config, + shared_ptr memory_tracker) { auto&& header = read_generic_tile_header(resources_, uri_, file_offset); if (encryption_key.encryption_type() != @@ -114,7 +119,8 @@ shared_ptr GenericTileIO::read_generic( 0, header.tile_size, filtered_data.data(), - header.persisted_size); + header.persisted_size, + memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)); // Read the tile. throw_if_not_ok(resources_.vfs().read( diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index c23193c7b326..d99f4640bf5f 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -126,7 +126,8 @@ class GenericTileIO { ContextResources& resources, const URI& uri, uint64_t offset, - const EncryptionKey& encryption_key); + const EncryptionKey& encryption_key, + shared_ptr memory_tracker); /** * Reads a generic tile from the file. A generic tile is a tile residing @@ -146,7 +147,8 @@ class GenericTileIO { shared_ptr read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, - const Config& config); + const Config& config, + shared_ptr memory_tracker); /** * Reads the generic tile header from the file. diff --git a/tiledb/sm/tile/test/CMakeLists.txt b/tiledb/sm/tile/test/CMakeLists.txt index fceae274d674..6feeb4eafb18 100644 --- a/tiledb/sm/tile/test/CMakeLists.txt +++ b/tiledb/sm/tile/test/CMakeLists.txt @@ -27,6 +27,9 @@ include(unit_test) commence(unit_test tile) - this_target_object_libraries(tile) - this_target_sources(main.cc unit_tile.cc) + this_target_sources( + main.cc + unit_tile.cc + ) + this_target_object_libraries(tile mem_helpers) conclude(unit_test) diff --git a/tiledb/sm/tile/test/unit_tile.cc b/tiledb/sm/tile/test/unit_tile.cc index 4ca36da42614..35754dbbfe6a 100644 --- a/tiledb/sm/tile/test/unit_tile.cc +++ b/tiledb/sm/tile/test/unit_tile.cc @@ -30,6 +30,7 @@ * Tests the `Tile` class. */ +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/tile/tile.h" @@ -39,6 +40,9 @@ using namespace tiledb::sm; TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { + // Create our test memory tracker. + auto tracker = tiledb::test::create_test_memory_tracker(); + // Initialize the test Tile. const format_version_t format_version = 0; const Datatype data_type = Datatype::UINT32; @@ -46,7 +50,14 @@ TEST_CASE("Tile: Test basic IO", "[Tile][basic_io]") { const uint64_t cell_size = sizeof(uint32_t); const unsigned int dim_num = 1; Tile tile( - format_version, data_type, cell_size, dim_num, tile_size, nullptr, 0); + format_version, + data_type, + cell_size, + dim_num, + tile_size, + nullptr, + 0, + tracker); CHECK(tile.size() == tile_size); // Create a buffer to write to the test Tile. diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index 3923d0d44984..1a65ddf0690d 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -33,12 +33,10 @@ #include "tiledb/sm/tile/tile.h" #include "tiledb/common/exception/exception.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/storage_format/serialization/serializers.h" -#include - using namespace tiledb::common; namespace tiledb { @@ -60,7 +58,8 @@ uint64_t WriterTile::max_tile_chunk_size_ = constants::max_tile_chunk_size; /* STATIC API */ /* ****************************** */ -shared_ptr Tile::from_generic(storage_size_t tile_size) { +shared_ptr Tile::from_generic( + storage_size_t tile_size, shared_ptr memory_tracker) { return make_shared( HERE(), 0, @@ -69,15 +68,18 @@ shared_ptr Tile::from_generic(storage_size_t tile_size) { 0, tile_size, nullptr, - 0); + 0, + memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)); } -WriterTile WriterTile::from_generic(storage_size_t tile_size) { +WriterTile WriterTile::from_generic( + storage_size_t tile_size, shared_ptr memory_tracker) { return { 0, constants::generic_tile_datatype, constants::generic_tile_cell_size, - tile_size}; + tile_size, + memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)}; } uint32_t WriterTile::compute_chunk_size( @@ -107,8 +109,10 @@ TileBase::TileBase( const format_version_t format_version, const Datatype type, const uint64_t cell_size, - const uint64_t size) - : data_(static_cast(tdb_malloc(size)), tiledb_free) + const uint64_t size, + tdb::pmr::memory_resource* resource) + : resource_(resource) + , data_(tdb::pmr::make_unique(resource_, size)) , size_(size) , cell_size_(cell_size) , format_version_(format_version) @@ -124,7 +128,8 @@ TileBase::TileBase( } TileBase::TileBase(TileBase&& tile) - : data_(std::move(tile.data_)) + : resource_(std::move(tile.resource_)) + , data_(std::move(tile.data_)) , size_(std::move(tile.size_)) , cell_size_(std::move(tile.cell_size_)) , format_version_(std::move(tile.format_version_)) @@ -145,8 +150,29 @@ Tile::Tile( const unsigned int zipped_coords_dim_num, const uint64_t size, void* filtered_data, - uint64_t filtered_size) - : TileBase(format_version, type, cell_size, size) + uint64_t filtered_size, + shared_ptr memory_tracker) + : Tile( + format_version, + type, + cell_size, + zipped_coords_dim_num, + size, + filtered_data, + filtered_size, + memory_tracker->get_resource(MemoryType::TILE_DATA)) { +} + +Tile::Tile( + const format_version_t format_version, + const Datatype type, + const uint64_t cell_size, + const unsigned int zipped_coords_dim_num, + const uint64_t size, + void* filtered_data, + uint64_t filtered_size, + tdb::pmr::memory_resource* resource) + : TileBase(format_version, type, cell_size, size, resource) , zipped_coords_dim_num_(zipped_coords_dim_num) , filtered_data_(filtered_data) , filtered_size_(filtered_size) { @@ -156,8 +182,24 @@ WriterTile::WriterTile( const format_version_t format_version, const Datatype type, const uint64_t cell_size, - const uint64_t size) - : TileBase(format_version, type, cell_size, size) + const uint64_t size, + shared_ptr memory_tracker) + : TileBase( + format_version, + type, + cell_size, + size, + memory_tracker->get_resource(MemoryType::TILE_WRITER_DATA)) + , filtered_buffer_(0) { +} + +WriterTile::WriterTile( + const format_version_t format_version, + const Datatype type, + const uint64_t cell_size, + const uint64_t size, + tdb::pmr::memory_resource* resource) + : TileBase(format_version, type, cell_size, size, resource) , filtered_buffer_(0) { } @@ -261,12 +303,15 @@ void WriterTile::write_var(const void* data, uint64_t offset, uint64_t nbytes) { while (new_alloc_size < offset + nbytes) new_alloc_size *= 2; - auto new_data = - static_cast(tdb_realloc(data_.release(), new_alloc_size)); + auto new_data = tdb::pmr::make_unique(resource_, new_alloc_size); + if (new_data == nullptr) { throw TileException("Cannot reallocate buffer; Memory allocation failed"); } - data_.reset(new_data); + + std::memcpy(new_data.get(), data_.get(), std::min(size_, new_alloc_size)); + + data_ = std::move(new_data); size_ = new_alloc_size; } diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index 84c38043ce88..55804df42856 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -34,6 +34,7 @@ #define TILEDB_TILE_H #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/tile/filtered_buffer.h" @@ -46,6 +47,8 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +class MemoryTracker; + /** * Base class for common code between Tile and WriterTile objects. */ @@ -58,12 +61,14 @@ class TileBase { * @param type The data type. * @param cell_size The cell size. * @param size The size of the tile. + * @param resource The memory resource to use. */ TileBase( const format_version_t format_version, const Datatype type, const uint64_t cell_size, - const uint64_t size); + const uint64_t size, + tdb::pmr::memory_resource* resource); /** Move constructor. */ TileBase(TileBase&& tile); @@ -147,13 +152,11 @@ class TileBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ - /** - * The buffer backing the tile data. - * - * TODO: Convert to regular allocations once tdb_realloc is not used for var - * size data anymore and remove custom deleter. - */ - std::unique_ptr data_; + /** The memory resource to use. */ + tdb::pmr::memory_resource* resource_; + + /** The buffer backing the tile data. */ + tdb::pmr::unique_ptr data_; /** Size of the data. */ uint64_t size_; @@ -179,7 +182,8 @@ class Tile : public TileBase { * * @param tile_size to be provided to init_unfiltered call */ - static shared_ptr from_generic(storage_size_t tile_size); + static shared_ptr from_generic( + storage_size_t tile_size, shared_ptr memory_tracker); /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -204,7 +208,31 @@ class Tile : public TileBase { const unsigned int zipped_coords_dim_num, const uint64_t size, void* filtered_data, - uint64_t filtered_size); + uint64_t filtered_size, + shared_ptr memory_tracker); + + /** + * Constructor. + * + * @param format_version The format version. + * @param type The data type. + * @param cell_size The cell size. + * @param zipped_coords_dim_num The number of dimensions in case the tile + * stores coordinates. + * @param size The size of the tile. + * @param filtered_data Pointer to the external filtered data. + * @param filtered_size The filtered size to allocate. + * @param resource The memory resource to use. + */ + Tile( + const format_version_t format_version, + const Datatype type, + const uint64_t cell_size, + const unsigned int zipped_coords_dim_num, + const uint64_t size, + void* filtered_data, + uint64_t filtered_size, + tdb::pmr::memory_resource* resource); DISABLE_MOVE_AND_MOVE_ASSIGN(Tile); DISABLE_COPY_AND_COPY_ASSIGN(Tile); @@ -349,8 +377,10 @@ class WriterTile : public TileBase { * generic data storage. * * @param tile_size to be provided to init_unfiltered call + * @param memory_tracker The memory tracker to use. */ - static WriterTile from_generic(storage_size_t tile_size); + static WriterTile from_generic( + storage_size_t tile_size, shared_ptr memory_tracker); /** * Computes the chunk size for a tile. @@ -380,12 +410,30 @@ class WriterTile : public TileBase { * @param type The data type. * @param cell_size The cell size. * @param size The size of the tile. + * @param meory_tracker The memory tracker to use. + */ + WriterTile( + const format_version_t format_version, + const Datatype type, + const uint64_t cell_size, + const uint64_t size, + shared_ptr memory_tracker); + + /** + * Constructor. + * + * @param format_version The format version. + * @param type The data type. + * @param cell_size The cell size. + * @param size The size of the tile. + * @param resource The memory resource to use. */ WriterTile( const format_version_t format_version, const Datatype type, const uint64_t cell_size, - const uint64_t size); + const uint64_t size, + tdb::pmr::memory_resource* resource); /** Move constructor. */ WriterTile(WriterTile&& tile); diff --git a/tiledb/sm/tile/writer_tile_tuple.cc b/tiledb/sm/tile/writer_tile_tuple.cc index dcd330b99b6d..700705f7ef87 100644 --- a/tiledb/sm/tile/writer_tile_tuple.cc +++ b/tiledb/sm/tile/writer_tile_tuple.cc @@ -48,31 +48,37 @@ WriterTileTuple::WriterTileTuple( const bool var_size, const bool nullable, const uint64_t cell_size, - const Datatype type) - : fixed_tile_( + const Datatype type, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , fixed_tile_( var_size ? WriterTile( array_schema.write_version(), constants::cell_var_offset_type, constants::cell_var_offset_size, - cell_num_per_tile * constants::cell_var_offset_size) : + cell_num_per_tile * constants::cell_var_offset_size, + memory_tracker_) : WriterTile( array_schema.write_version(), type, cell_size, - cell_num_per_tile * cell_size)) + cell_num_per_tile * cell_size, + memory_tracker_)) , var_tile_( var_size ? std::optional(WriterTile( array_schema.write_version(), type, datatype_size(type), - cell_num_per_tile * constants::cell_var_offset_size)) : + cell_num_per_tile * constants::cell_var_offset_size, + memory_tracker_)) : std::nullopt) , validity_tile_( nullable ? std::optional(WriterTile( array_schema.write_version(), constants::cell_validity_type, constants::cell_validity_size, - cell_num_per_tile * constants::cell_validity_size)) : + cell_num_per_tile * constants::cell_validity_size, + memory_tracker_)) : std::nullopt) , cell_size_(cell_size) , var_pre_filtered_size_(0) @@ -83,7 +89,8 @@ WriterTileTuple::WriterTileTuple( } WriterTileTuple::WriterTileTuple(WriterTileTuple&& tile) - : fixed_tile_(std::move(tile.fixed_tile_)) + : memory_tracker_(std::move(tile.memory_tracker_)) + , fixed_tile_(std::move(tile.fixed_tile_)) , var_tile_(std::move(tile.var_tile_)) , validity_tile_(std::move(tile.validity_tile_)) , cell_size_(std::move(tile.cell_size_)) @@ -136,6 +143,7 @@ void WriterTileTuple::set_metadata( } void WriterTileTuple::swap(WriterTileTuple& tile) { + std::swap(memory_tracker_, tile.memory_tracker_); std::swap(fixed_tile_, tile.fixed_tile_); std::swap(var_tile_, tile.var_tile_); std::swap(validity_tile_, tile.validity_tile_); diff --git a/tiledb/sm/tile/writer_tile_tuple.h b/tiledb/sm/tile/writer_tile_tuple.h index 13364df1a669..244ac932114c 100644 --- a/tiledb/sm/tile/writer_tile_tuple.h +++ b/tiledb/sm/tile/writer_tile_tuple.h @@ -56,7 +56,8 @@ class WriterTileTuple { const bool var_size, const bool nullable, const uint64_t cell_size, - const Datatype type); + const Datatype type, + shared_ptr memory_tracker); /** Move constructor. */ WriterTileTuple(WriterTileTuple&& tile); @@ -223,6 +224,9 @@ class WriterTileTuple { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr memory_tracker_; + /** * Fixed data tile. Contains offsets for var size attribute/dimension and * the data itself in case of fixed sized attribute/dimension. From 22a71d477e6535e8b7d2811f0a7e89edd2026155 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 27 Feb 2024 09:57:56 +0200 Subject: [PATCH 206/456] Move `StorageManager::store_data_to_generic_tile` to `GenericTileIO`. (#4743) Move StorageManager::store_data_to_generic_tile out of StorageManager and into GenericTileIO. --- TYPE: NO_HISTORY DESC: Move StorageManager::store_data_to_generic_tile to GenericTileIO. --- .../deletes_and_updates.cc | 8 +++++-- tiledb/sm/storage_manager/storage_manager.cc | 18 ++++----------- .../storage_manager_canonical.h | 11 --------- tiledb/sm/tile/generic_tile_io.cc | 11 +++++++++ tiledb/sm/tile/generic_tile_io.h | 23 ++++++++++++++----- 5 files changed, 38 insertions(+), 33 deletions(-) diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index f5cdfdc5fbd9..1211d58105bb 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -36,6 +36,7 @@ #include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" using namespace tiledb; @@ -163,8 +164,11 @@ Status DeletesAndUpdates::dowork() { constants::update_file_suffix; auto uri = commit_uri.join_path(new_fragment_str); - RETURN_NOT_OK(storage_manager_->store_data_to_generic_tile( - serialized_condition, uri, *array_->encryption_key())); + GenericTileIO::store_data( + storage_manager_->resources(), + uri, + serialized_condition, + *array_->encryption_key()); return Status::Ok(); } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index eb2d9350d66c..4886c2c22b4b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1283,8 +1283,7 @@ Status StorageManagerCanonical::store_group_detail( if (!group_detail_dir_exists) RETURN_NOT_OK(vfs()->create_dir(group_detail_folder_uri)); - RETURN_NOT_OK( - store_data_to_generic_tile(tile, group_detail_uri, encryption_key)); + GenericTileIO::store_data(resources_, group_detail_uri, tile, encryption_key); return Status::Ok(); } @@ -1322,7 +1321,7 @@ Status StorageManagerCanonical::store_array_schema( if (!schema_dir_exists) RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); - RETURN_NOT_OK(store_data_to_generic_tile(tile, schema_uri, encryption_key)); + GenericTileIO::store_data(resources_, schema_uri, tile, encryption_key); // Create the `__enumerations` directory under `__schema` if it doesn't // exist. This might happen if someone tries to add an enumeration to an @@ -1355,8 +1354,7 @@ Status StorageManagerCanonical::store_array_schema( enmr->serialize(serializer); auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); - RETURN_NOT_OK( - store_data_to_generic_tile(tile, abs_enmr_uri, encryption_key)); + GenericTileIO::store_data(resources_, abs_enmr_uri, tile, encryption_key); } return Status::Ok(); @@ -1390,19 +1388,11 @@ Status StorageManagerCanonical::store_metadata( // Create a metadata file name URI metadata_uri = metadata->get_uri(uri); - RETURN_NOT_OK(store_data_to_generic_tile(tile, metadata_uri, encryption_key)); + GenericTileIO::store_data(resources_, metadata_uri, tile, encryption_key); return Status::Ok(); } -Status StorageManagerCanonical::store_data_to_generic_tile( - WriterTile& tile, const URI& uri, const EncryptionKey& encryption_key) { - GenericTileIO tile_io(resources_, uri); - uint64_t nbytes = 0; - tile_io.write_generic(&tile, encryption_key, &nbytes); - return vfs()->close_file(uri); -} - void StorageManagerCanonical::wait_for_zero_in_progress() { std::unique_lock lck(queries_in_progress_mtx_); queries_in_progress_cv_.wait( diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 50c374516430..4309f946e320 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -591,17 +591,6 @@ class StorageManagerCanonical { Status store_metadata( const URI& uri, const EncryptionKey& encryption_key, Metadata* metadata); - /** - * Stores data into persistent storage. - * - * @param tile Tile to store. - * @param uri The object URI. - * @param encryption_key The encryption key to use. - * @return Status - */ - Status store_data_to_generic_tile( - WriterTile& tile, const URI& uri, const EncryptionKey& encryption_key); - [[nodiscard]] inline ContextResources& resources() const { return resources_; } diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 4292999719b0..c2edc076f40f 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -175,6 +175,17 @@ GenericTileIO::GenericTileHeader GenericTileIO::read_generic_tile_header( return header; } +void GenericTileIO::store_data( + ContextResources& resources, + const URI& uri, + WriterTile& tile, + const EncryptionKey& encryption_key) { + GenericTileIO tile_io(resources, uri); + uint64_t nbytes = 0; + tile_io.write_generic(&tile, encryption_key, &nbytes); + throw_if_not_ok(resources.vfs().close_file(uri)); +} + void GenericTileIO::write_generic( WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes) { // Create a header diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index d99f4640bf5f..a177f875ab0a 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -120,7 +120,7 @@ class GenericTileIO { * @param uri The object URI. * @param offset The offset into the file to read from. * @param encryption_key The encryption key to use. - * @return Status, Tile with the data. + * @return Tile with the data. */ static shared_ptr load( ContextResources& resources, @@ -142,7 +142,7 @@ class GenericTileIO { * @param file_offset The offset in the file to read from. * @param encryption_key The encryption key to use. * @param config The storage manager's config. - * @return Status, Tile + * @return Tile */ shared_ptr read_generic( uint64_t file_offset, @@ -156,13 +156,25 @@ class GenericTileIO { * @param resources The ContextResources instance to use for reading. * @param uri The URI of the generic tile. * @param file_offset The offset where the header read will begin. - * @param encryption_key If the array is encrypted, the private encryption - * key. For unencrypted arrays, pass `nullptr`. - * @return Status, Header + * @return Header */ static GenericTileHeader read_generic_tile_header( ContextResources& resources, const URI& uri, uint64_t file_offset); + /** + * Writes a generic tile to a file. + * + * @param resources The ContextResources instance to use for writing. + * @param uri The URI of the generic tile. + * @param tile The tile to write. + * @param encryption_key The encryption key to use. + */ + static void store_data( + ContextResources& resources, + const URI& uri, + WriterTile& tile, + const EncryptionKey& encryption_key); + /** * Writes a tile generically to the file. This means that a header will be * prepended to the file before writing the tile contents. The reason is @@ -172,7 +184,6 @@ class GenericTileIO { * @param tile The tile to be written. * @param encryption_key The encryption key to use. * @param nbytes The total number of bytes written to the file. - * @return Status */ void write_generic( WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes); From b74a5592d03620376349d0724965727ff89c012d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Tue, 27 Feb 2024 09:49:36 +0100 Subject: [PATCH 207/456] Use CPack in release workflow (#4694) This PR changes the default behavior of release workflow to use `cpack` commands to generate source and binary releases of TileDB library. The produced releases can be seen here: https://github.com/dudoslav/TileDB/actions/runs/7757551437 How release look: https://github.com/dudoslav/TileDB/releases/tag/t03 --- TYPE: BUILD DESC: CPack in Release Workflow --------- Co-authored-by: Isaiah Norton --- .github/workflows/release.yml | 62 +++++++++++++----------- cmake/inputs/CustomCPackOptions.cmake.in | 13 +---- cmake/package.cmake | 45 +++++++++++++++++ tiledb/CMakeLists.txt | 18 +------ 4 files changed, 80 insertions(+), 58 deletions(-) create mode 100644 cmake/package.cmake diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbabae23bc53..bd5559d4469e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,26 @@ on: workflow_dispatch: jobs: - Build-Release: + Package-Source-Release: + strategy: + matrix: + os: [ubuntu-20.04] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout TileDB + uses: actions/checkout@v3 + - name: CMake Configure + run: cmake -S . -B build -DTILEDB_CMAKE_IDE=ON + - name: CPack Package Source + run: cd build && cpack --config CPackSourceConfig.cmake + - name: Upload Release Artifacts + uses: actions/upload-artifact@v3 + with: + name: release + path: | + build/tiledb-*.tar.gz* + + Package-Binary-Release: strategy: fail-fast: false matrix: @@ -88,44 +107,27 @@ jobs: ${{ matrix.cmake_args }} shell: bash - name: Build TileDB - run: cmake --build build --config Release - - name: Install TileDB - run: cmake --build build --config Release --target install-tiledb - - name: Archive installed artifacts (non-Windows) - if: ${{ !startsWith(matrix.platform, 'windows') }} - run: | - tar -czf ${{ steps.get-values.outputs.archive_name }}.tar.gz -C dist . - - name: Archive installed artifacts (Windows) - if: startsWith(matrix.platform, 'windows') - run: | - Compress-Archive -Path dist\* -DestinationPath ${{ steps.get-values.outputs.archive_name }}.zip - shell: pwsh + run: cmake --build build --config Release --target package - name: Upload release artifacts uses: actions/upload-artifact@v3 with: - name: tiledb-dist - path: ${{ steps.get-values.outputs.archive_name }}.* - - name: Archive build directory - run: | - tar -czf build-${{ matrix.platform }}.tar.gz -C build . - - name: Upload build directory - uses: actions/upload-artifact@v3 - with: - name: tiledb-build - path: build-${{ matrix.platform }}.tar.gz + name: release + path: | + build/tiledb/tiledb-*.tar.gz* + build/tiledb/tiledb-*.zip* - name: "Print log files (failed build only)" run: | source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh if: failure() # only run this job if the build step failed Test-Release-Artifacts: - needs: Build-Release + needs: Package-Binary-Release runs-on: ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: tiledb-dist + name: release path: dist - name: Test names of release artifacts run: | @@ -135,14 +137,16 @@ jobs: fi Publish-Release: - needs: Test-Release-Artifacts + needs: + - Test-Release-Artifacts + - Package-Source-Release if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - name: Download release artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: - name: tiledb-dist + name: release path: dist - name: Publish release artifacts uses: actions/github-script@v6 diff --git a/cmake/inputs/CustomCPackOptions.cmake.in b/cmake/inputs/CustomCPackOptions.cmake.in index 60f201003463..71d8fbea0686 100644 --- a/cmake/inputs/CustomCPackOptions.cmake.in +++ b/cmake/inputs/CustomCPackOptions.cmake.in @@ -1,18 +1,7 @@ -set(CPACK_SOURCE_IGNORE_FILES ".*\\.git;.*build.*") - set(CPACK_PACKAGE_VENDOR "TileDB Inc.") set(CPACK_PACKAGE_VERSION "@TILEDB_VERSION@") set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) -# Append NOAVX2 if needed -if(NOT ${COMPILER_SUPPORTS_AVX2}) - set(NOAVX2 "-noavx2") -endif() - -# Set output name -set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}${NOAVX2}-${CPACK_PACKAGE_VERSION}") -string(TOLOWER ${CPACK_PACKAGE_FILE_NAME} CPACK_PACKAGE_FILE_NAME ) - # Enable HASH -set(CPACK_PACKAGE_CHECKSUM "SHA256") \ No newline at end of file +set(CPACK_PACKAGE_CHECKSUM "SHA256") diff --git a/cmake/package.cmake b/cmake/package.cmake new file mode 100644 index 000000000000..08bfae9049c3 --- /dev/null +++ b/cmake/package.cmake @@ -0,0 +1,45 @@ +# Packaging configuration +configure_file ("${PROJECT_SOURCE_DIR}/cmake/inputs/CustomCPackOptions.cmake.in" + "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake" + @ONLY) +set (CPACK_PROJECT_CONFIG_FILE + "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake") + +# Not all options can be set in CustomCPackOptions.cmake +if (CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CPACK_SOURCE_GENERATOR "ZIP") + set(CPACK_GENERATOR "ZIP") +else() + set(CPACK_SOURCE_GENERATOR "TGZ") + set(CPACK_GENERATOR "TGZ") +endif() + +# Package file name variables can not be in config file as well + +# Append NOAVX2 if needed +if(NOT ${COMPILER_SUPPORTS_AVX2}) + set(CPACK_NOAVX2 "-noavx2") +endif() + +# Properly set system name and architecture +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + set(CPACK_SYSTEM_NAME "MacOS") + if(CMAKE_OSX_ARCHITECTURES STREQUAL "arm64") + set(CPACK_SYSTEM_PROCESSOR ${CMAKE_OSX_ARCHITECTURES}) + else() + set(CPACK_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) + endif() +else() + set(CPACK_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + set(CPACK_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) +endif() + +set(CPACK_SOURCE_IGNORE_FILES ".*\.git;.*build.*/.*") + +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-source-${TILEDB_VERSION}") +string(TOLOWER ${CPACK_SOURCE_PACKAGE_FILE_NAME} CPACK_SOURCE_PACKAGE_FILE_NAME) + +set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${CPACK_SYSTEM_NAME}-${CPACK_SYSTEM_PROCESSOR}${CPACK_NOAVX2}-${TILEDB_VERSION}") +string(TOLOWER ${CPACK_PACKAGE_FILE_NAME} CPACK_PACKAGE_FILE_NAME) + +include(CPack) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index a9222730871a..737cc50f7905 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -1056,20 +1056,4 @@ configure_file( install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tiledb.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) -# Packaging configuration -configure_file ("${PROJECT_SOURCE_DIR}/cmake/inputs/CustomCPackOptions.cmake.in" - "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake" - @ONLY) -set (CPACK_PROJECT_CONFIG_FILE - "${PROJECT_BINARY_DIR}/CustomCPackOptions.cmake") - -# Not all options can be set in CustomCPackOptions.cmake -if (${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set(CPACK_SOURCE_GENERATOR "ZIP") - set(CPACK_GENERATOR "ZIP") -else() - set(CPACK_SOURCE_GENERATOR "TGZ") - set(CPACK_GENERATOR "TGZ") -endif() - -include(CPack) +include(${CMAKE_SOURCE_DIR}/cmake/package.cmake) \ No newline at end of file From 23655a0abe3947ea56f151be5a0e9255e8596ab5 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 27 Feb 2024 22:10:31 -0500 Subject: [PATCH 208/456] Only run Github Actions REST CI job on dev/release/tag branches (#4768) Now that we have this job on Gitlab, let's reduce the GHA usage by only running the GHA pipeline on dev/release/tag branches. We'll run this in parallel for a while to catch any discrepancies, and aim to remove the GHA version entirely in a few weeks. --- TYPE: NO_HISTORY --- .github/workflows/ci-rest.yml | 6 ++++++ .github/workflows/full-ci.yml | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 45de21e963c9..08ba19d26a3b 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -4,6 +4,12 @@ on: - workflow_call - workflow_dispatch + push: + branches: + - dev + - release-* + - refs/tags/* + jobs: rest-ci: runs-on: ubuntu-latest diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index a710d9761fbe..7ef0d811ce2d 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -138,10 +138,6 @@ jobs: ci_docker: uses: ./.github/workflows/build-dockerfile.yml - ci_rest: - uses: ./.github/workflows/ci-rest.yml - secrets: inherit - # dummy job for branch protection check full_ci_passed: needs: [ From b43ac585b1ea95a61a27db762e2f236b6b60074a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 28 Feb 2024 05:17:43 -0500 Subject: [PATCH 209/456] Add v2_20_1 arrays to backward compatibility matrix. (#4770) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index df0dbbab047c..d7bfa7a884eb 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 027c98d6ddf519216018f888c92e4d6e8a7e2d48 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 28 Feb 2024 20:21:04 +0100 Subject: [PATCH 210/456] Fix memory budget tests on ASAN nightlies. (#4772) Some objects are larger on the ASAN debug build and causes us to load less tiles in memory, failing some memory budget verifications. Making the budget slightly larger fixes the tests. --- TYPE: NO_HISTORY DESC: Fix memory budget tests on ASAN nightlies. --- test/src/unit-cppapi-consolidation-with-timestamps.cc | 4 ++-- test/src/unit-sparse-global-order-reader.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 43a1369a8d63..4f6d614775ba 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -763,7 +763,7 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "10200"); + cfg.set("sm.mem.total_budget", "11000"); cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); ctx_ = Context(cfg); @@ -822,7 +822,7 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "10200"); + cfg.set("sm.mem.total_budget", "11000"); cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); ctx_ = Context(cfg); diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 3d2041608301..77580430b1e6 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -777,7 +777,7 @@ TEST_CASE_METHOD( // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "10500"; + total_budget_ = "12000"; ratio_coords_ = "0.30"; update_config(); @@ -1348,7 +1348,7 @@ TEST_CASE_METHOD( // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "10500"; + total_budget_ = "12000"; ratio_coords_ = "0.30"; update_config(); From 9883b340ff0d71396456b6e184ac10513f5e0b78 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 29 Feb 2024 04:24:47 -0500 Subject: [PATCH 211/456] Integrate individual filesystem unit tests into a single test, part 2. (#4714) This moves more tests that have common code to the same tests cases to remove duplicated code. --- TYPE: NO_HISTORY DESC: Integrate individual filesystem unit tests into a single test, part 2. --------- Co-authored-by: Luc Rancourt --- test/CMakeLists.txt | 7 +- test/src/unit-azure.cc | 386 --------------------- test/src/unit-gcs.cc | 476 ------------------------- test/src/unit-gs.cc | 575 ------------------------------- test/src/unit-hdfs-filesystem.cc | 139 -------- test/src/unit-s3-no-multipart.cc | 210 ----------- test/src/unit-s3.cc | 75 +--- test/src/unit-vfs.cc | 310 ++++++++++++++--- test/support/src/vfs_helpers.h | 10 +- tiledb/sm/filesystem/gcs.cc | 3 +- 10 files changed, 266 insertions(+), 1925 deletions(-) delete mode 100644 test/src/unit-azure.cc delete mode 100644 test/src/unit-gcs.cc delete mode 100644 test/src/unit-gs.cc delete mode 100644 test/src/unit-hdfs-filesystem.cc delete mode 100644 test/src/unit-s3-no-multipart.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ec67498ef670..7dc4cf539f88 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,7 +4,7 @@ # # The MIT License # -# Copyright (c) 2017-2022 TileDB, Inc. +# Copyright (c) 2017-2024 TileDB, Inc. # Copyright (c) 2016 MIT and Intel Corporation # # Permission is hereby granted, free of charge, to any person obtaining a copy @@ -119,7 +119,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/test-cppapi-aggregates.cc src/test-cppapi-consolidation-plan.cc src/unit-average-cell-size.cc - src/unit-azure.cc src/unit-backwards_compat.cc src/unit-bufferlist.cc src/unit-capi-any.cc @@ -180,9 +179,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-filter-buffer.cc src/unit-filter-pipeline.cc src/unit-global-order.cc - src/unit-gcs.cc - src/unit-gs.cc - src/unit-hdfs-filesystem.cc src/unit-ordered-dim-label-reader.cc src/unit-tile-metadata.cc src/unit-tile-metadata-generator.cc @@ -192,7 +188,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-resource-pool.cc src/unit-result-coords.cc src/unit-result-tile.cc - src/unit-s3-no-multipart.cc src/unit-s3.cc src/unit-sparse-global-order-reader.cc src/unit-sparse-unordered-with-dups-reader.cc diff --git a/test/src/unit-azure.cc b/test/src/unit-azure.cc deleted file mode 100644 index f0ad9fbdc361..000000000000 --- a/test/src/unit-azure.cc +++ /dev/null @@ -1,386 +0,0 @@ -/** - * @file unit-azure.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for AZURE API filesystem functions. - */ - -#ifdef HAVE_AZURE - -#include -#include -#include "tiledb/common/filesystem/directory_entry.h" -#include "tiledb/common/thread_pool.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/filesystem/azure.h" -#include "tiledb/sm/global_state/unit_test_config.h" -#include "tiledb/sm/misc/tdb_time.h" - -#include -#include - -using namespace tiledb::common; -using namespace tiledb::sm; - -using ConfMap = std::map; -using ConfList = std::vector; - -static ConfList test_settings = { - {{"vfs.azure.storage_account_name", "devstoreaccount1"}, - {"vfs.azure.storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw=="}, - {"vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1"}}, - // Currently disabled because it does not work with the Azurite emulator - // The SAS path was manually tested against the Azure Blob Service. - //{{"vfs.azure.storage_account_name", "devstoreaccount2"}, - // {"vfs.azure.storage_sas_token", ""}, - // {"vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount2"}} -}; - -struct AzureFx { - const std::string AZURE_PREFIX = "azure://"; - const tiledb::sm::URI AZURE_CONTAINER = - tiledb::sm::URI(AZURE_PREFIX + random_container_name("tiledb") + "/"); - const std::string TEST_DIR = AZURE_CONTAINER.to_string() + "tiledb_test_dir/"; - - tiledb::sm::Azure azure_; - ThreadPool thread_pool_{2}; - - AzureFx() = default; - ~AzureFx(); - - void init_azure(Config&& config, ConfMap); - - static std::string random_container_name(const std::string& prefix); -}; - -AzureFx::~AzureFx() { - // Empty container - bool is_empty; - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - if (!is_empty) { - REQUIRE(azure_.empty_container(AZURE_CONTAINER).ok()); - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(is_empty); - } - - // Delete container - REQUIRE(azure_.remove_container(AZURE_CONTAINER).ok()); -} - -void AzureFx::init_azure(Config&& config, ConfMap settings) { - auto set_conf = [&](auto iter) { - std::string key = iter.first; - std::string val = iter.second; - REQUIRE(config.set(key, val).ok()); - }; - - // Set provided config settings for connection - std::for_each(settings.begin(), settings.end(), set_conf); - - // Initialize - REQUIRE(azure_.init(config, &thread_pool_).ok()); - - // Create container - bool is_container; - REQUIRE(azure_.is_container(AZURE_CONTAINER, &is_container).ok()); - if (is_container) { - REQUIRE(azure_.remove_container(AZURE_CONTAINER).ok()); - } - - REQUIRE(azure_.is_container(AZURE_CONTAINER, &is_container).ok()); - REQUIRE(!is_container); - REQUIRE(azure_.create_container(AZURE_CONTAINER).ok()); - - // Check if container is empty - bool is_empty; - REQUIRE(azure_.is_empty_container(AZURE_CONTAINER, &is_empty).ok()); - REQUIRE(is_empty); -} - -std::string AzureFx::random_container_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << tiledb::sm::utils::time::timestamp_now_ms(); - return ss.str(); -} - -TEST_CASE_METHOD( - AzureFx, "Test Azure filesystem, file I/O", "[azure][multipart]") { - Config config; - const uint64_t max_parallel_ops = 2; - const uint64_t block_list_block_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config - .set( - "vfs.azure.block_list_block_size", - std::to_string(block_list_block_size)) - .ok()); - - auto settings = - GENERATE(from_range(test_settings.begin(), test_settings.end())); - init_azure(std::move(config), settings); - - const uint64_t write_cache_max_size = - max_parallel_ops * block_list_block_size; - - // Prepare buffers - uint64_t buffer_size = write_cache_max_size * 5; - auto write_buffer = new char[buffer_size]; - for (uint64_t i = 0; i < buffer_size; i++) - write_buffer[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE(azure_.write(URI(largefile), write_buffer, buffer_size).ok()); - REQUIRE( - azure_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - azure_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the files do not exist - bool is_blob = false; - REQUIRE(azure_.is_blob(URI(largefile), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(smallfile), &is_blob).ok()); - REQUIRE(!is_blob); - - // Flush the files - REQUIRE(azure_.flush_blob(URI(largefile)).ok()); - REQUIRE(azure_.flush_blob(URI(smallfile)).ok()); - - // After flushing, the files exist - REQUIRE(azure_.is_blob(URI(largefile), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.is_blob(URI(smallfile), &is_blob).ok()); - REQUIRE(is_blob); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(azure_.blob_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == (buffer_size + buffer_size_small)); - REQUIRE(azure_.blob_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(azure_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE( - azure_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - AzureFx, - "Test Azure filesystem, file I/O, no multipart", - "[azure][no_multipart]") { - Config config; - const uint64_t max_parallel_ops = 2; - const uint64_t block_list_block_size = 4 * 1024 * 1024; - REQUIRE(config.set("vfs.azure.use_block_list_upload", "false").ok()); - REQUIRE( - config.set("vfs.azure.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config - .set( - "vfs.azure.block_list_block_size", - std::to_string(block_list_block_size)) - .ok()); - - auto settings = - GENERATE(from_range(test_settings.begin(), test_settings.end())); - init_azure(std::move(config), settings); - - const uint64_t write_cache_max_size = - max_parallel_ops * block_list_block_size; - - // Prepare a large buffer that can fit in the write cache. - uint64_t large_buffer_size = write_cache_max_size; - auto large_write_buffer = new char[large_buffer_size]; - for (uint64_t i = 0; i < large_buffer_size; i++) - large_write_buffer[i] = (char)('a' + (i % 26)); - - // Prepare a small buffer that can fit in the write cache. - uint64_t small_buffer_size = write_cache_max_size / 1024; - auto small_write_buffer = new char[small_buffer_size]; - for (uint64_t i = 0; i < small_buffer_size; i++) - small_write_buffer[i] = (char)('a' + (i % 26)); - - // Prepare a buffer too large to fit in the write cache. - uint64_t oob_buffer_size = write_cache_max_size + 1; - auto oob_write_buffer = new char[oob_buffer_size]; - for (uint64_t i = 0; i < oob_buffer_size; i++) - oob_write_buffer[i] = (char)('a' + (i % 26)); - - auto large_file = TEST_DIR + "largefile"; - REQUIRE(azure_.write(URI(large_file), large_write_buffer, large_buffer_size) - .ok()); - - auto small_file_1 = TEST_DIR + "smallfile1"; - REQUIRE(azure_.write(URI(small_file_1), small_write_buffer, small_buffer_size) - .ok()); - - auto small_file_2 = TEST_DIR + "smallfile2"; - REQUIRE(azure_.write(URI(small_file_2), small_write_buffer, small_buffer_size) - .ok()); - REQUIRE(azure_.write(URI(small_file_2), small_write_buffer, small_buffer_size) - .ok()); - - auto oob_file = TEST_DIR + "oobfile"; - REQUIRE(!azure_.write(URI(oob_file), oob_write_buffer, oob_buffer_size).ok()); - - // Before flushing, the files do not exist - bool is_blob = false; - REQUIRE(azure_.is_blob(URI(large_file), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(small_file_1), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(small_file_2), &is_blob).ok()); - REQUIRE(!is_blob); - REQUIRE(azure_.is_blob(URI(oob_file), &is_blob).ok()); - REQUIRE(!is_blob); - - // Flush the files - REQUIRE(azure_.flush_blob(URI(small_file_1)).ok()); - REQUIRE(azure_.flush_blob(URI(small_file_2)).ok()); - REQUIRE(azure_.flush_blob(URI(large_file)).ok()); - - // After flushing, the files exist - REQUIRE(azure_.is_blob(URI(large_file), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.is_blob(URI(small_file_1), &is_blob).ok()); - REQUIRE(is_blob); - REQUIRE(azure_.is_blob(URI(small_file_2), &is_blob).ok()); - REQUIRE(is_blob); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(azure_.blob_size(URI(large_file), &nbytes).ok()); - CHECK(nbytes == large_buffer_size); - REQUIRE(azure_.blob_size(URI(small_file_1), &nbytes).ok()); - CHECK(nbytes == small_buffer_size); - REQUIRE(azure_.blob_size(URI(small_file_2), &nbytes).ok()); - CHECK(nbytes == (small_buffer_size + small_buffer_size)); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE( - azure_.read(URI(large_file), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE( - azure_.read(URI(large_file), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE( - "Test constructing Azure Blob Storage endpoint URIs", "[azure][uri]") { - std::string sas_token, custom_endpoint, expected_endpoint; - SECTION("No SAS token") { - sas_token = ""; - expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; - } - SECTION("SAS token without leading question mark") { - sas_token = "baz=qux&foo=bar"; - expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; - } - SECTION("SAS token with leading question mark") { - sas_token = "?baz=qux&foo=bar"; - expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; - } - SECTION("SAS token in both endpoint and config option") { - sas_token = "baz=qux&foo=bar"; - custom_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; - expected_endpoint = - "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; - } - SECTION("No SAS token") { - sas_token = ""; - expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; - } - Config config; - REQUIRE( - config.set("vfs.azure.storage_account_name", "devstoreaccount1").ok()); - REQUIRE(config.set("vfs.azure.blob_endpoint", custom_endpoint).ok()); - REQUIRE(config.set("vfs.azure.storage_sas_token", sas_token).ok()); - tiledb::sm::Azure azure; - ThreadPool thread_pool(1); - REQUIRE(azure.init(config, &thread_pool).ok()); - REQUIRE(azure.client().GetUrl() == expected_endpoint); -} - -#endif diff --git a/test/src/unit-gcs.cc b/test/src/unit-gcs.cc deleted file mode 100644 index e8065a9e665a..000000000000 --- a/test/src/unit-gcs.cc +++ /dev/null @@ -1,476 +0,0 @@ -/** - * @file unit-gcs.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for GCS API filesystem functions. - */ - -#ifdef HAVE_GCS - -#include -#include "tiledb/common/filesystem/directory_entry.h" -#include "tiledb/common/thread_pool.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/filesystem/gcs.h" -#include "tiledb/sm/global_state/unit_test_config.h" -#include "tiledb/sm/misc/tdb_math.h" -#include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/utils.h" - -#include - -using namespace tiledb::common; -using namespace tiledb::sm; - -struct GCSFx { - const std::string GCS_PREFIX = "gcs://"; - const tiledb::sm::URI GCS_BUCKET = - tiledb::sm::URI(GCS_PREFIX + random_bucket_name("tiledb") + "/"); - const std::string TEST_DIR = GCS_BUCKET.to_string() + "tiledb_test_dir/"; - - tiledb::sm::GCS gcs_; - ThreadPool thread_pool_{2}; - - GCSFx() = default; - ~GCSFx(); - - void init_gcs(Config&& config); - - static std::string random_bucket_name(const std::string& prefix); -}; - -GCSFx::~GCSFx() { - // Empty bucket - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - if (!is_empty) { - REQUIRE(gcs_.empty_bucket(GCS_BUCKET).ok()); - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); - } - - // Delete bucket - REQUIRE(gcs_.remove_bucket(GCS_BUCKET).ok()); -} - -void GCSFx::init_gcs(Config&& config) { - REQUIRE(config.set("vfs.gcs.project_id", "TODO").ok()); - REQUIRE(gcs_.init(config, &thread_pool_).ok()); - - // Create bucket - bool is_bucket; - REQUIRE(gcs_.is_bucket(GCS_BUCKET, &is_bucket).ok()); - if (is_bucket) { - REQUIRE(gcs_.remove_bucket(GCS_BUCKET).ok()); - } - - REQUIRE(gcs_.is_bucket(GCS_BUCKET, &is_bucket).ok()); - REQUIRE(!is_bucket); - REQUIRE(gcs_.create_bucket(GCS_BUCKET).ok()); - - // Check if bucket is empty - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); -} - -std::string GCSFx::random_bucket_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << tiledb::sm::utils::time::timestamp_now_ms(); - return ss.str(); -} - -TEST_CASE_METHOD(GCSFx, "Test GCS init", "[gcs]") { - try { - Config config; - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - init_gcs(std::move(config)); - } catch (...) { - INFO( - "GCS initialization failed. In order to run GCS tests, be sure to " - "source scripts/run-gcs.sh in this shell session before starting test " - "runner."); - REQUIRE(false); - } -} - -TEST_CASE_METHOD( - GCSFx, - "Test GCS filesystem I/O, multipart, serial", - "[gcs][multipart][serial]") { - Config config; - const uint64_t max_parallel_ops = 1; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the files do not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the files - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == (buffer_size_large + buffer_size_small)); - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GCSFx, "Test GCS filesystem I/O, non-multipart", "[gcs][non-multipart]") { - Config config; - const uint64_t max_parallel_ops = 1; - const uint64_t write_cache_max_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config - .set( - "vfs.gcs.max_direct_upload_size", - std::to_string(write_cache_max_size)) - .ok()); - init_gcs(std::move(config)); - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - !gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(smallfile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GCSFx, - "Test GCS filesystem I/O, multipart, concurrent", - "[gcs][multipart][concurrent]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the files do not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the files - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == (buffer_size_large + buffer_size_small)); - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GCSFx, - "Test GCS filesystem I/O, multipart, composition", - "[gcs][multipart][composition]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare a buffer that will write 200 (50 * 4 threads) objects. - // The maximum number of objects per composition operation is 32. - uint64_t buffer_size_large = 50 * write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - - // Write to the file - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_large); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Prepare a buffer that will overwrite the original with a smaller - // size. - uint64_t buffer_size_overwrite = 10 * write_cache_max_size; - auto write_buffer_overwrite = new char[buffer_size_overwrite]; - for (uint64_t i = 0; i < buffer_size_overwrite; i++) - write_buffer_overwrite[i] = (char)('a' + (i % 26)); - - // Write to the file - REQUIRE( - gcs_.write(URI(largefile), write_buffer_overwrite, buffer_size_overwrite) - .ok()); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_overwrite); -} - -#endif diff --git a/test/src/unit-gs.cc b/test/src/unit-gs.cc deleted file mode 100644 index c1cc1b367daf..000000000000 --- a/test/src/unit-gs.cc +++ /dev/null @@ -1,575 +0,0 @@ -/** - * @file unit-gs.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for GCS API filesystem functions. (gs extension) - */ - -#ifdef HAVE_GCS - -#include -#include "tiledb/common/thread_pool.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/filesystem/gcs.h" -#include "tiledb/sm/global_state/unit_test_config.h" -#include "tiledb/sm/misc/tdb_math.h" -#include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/utils.h" - -#include - -using namespace tiledb::common; -using namespace tiledb::sm; - -struct GSFx { - const std::string GCS_PREFIX = "gs://"; - const tiledb::sm::URI GCS_BUCKET = - tiledb::sm::URI(GCS_PREFIX + random_bucket_name("tiledb") + "/"); - const std::string TEST_DIR = GCS_BUCKET.to_string() + "tiledb_test_dir/"; - - tiledb::sm::GCS gcs_; - ThreadPool thread_pool_{2}; - - GSFx() = default; - ~GSFx(); - - void init_gcs(Config&& config); - - static std::string random_bucket_name(const std::string& prefix); -}; - -GSFx::~GSFx() { - // Empty bucket - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - if (!is_empty) { - REQUIRE(gcs_.empty_bucket(GCS_BUCKET).ok()); - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); - } - - // Delete bucket - REQUIRE(gcs_.remove_bucket(GCS_BUCKET).ok()); -} - -void GSFx::init_gcs(Config&& config) { - REQUIRE(config.set("vfs.gcs.project_id", "TODO").ok()); - REQUIRE(gcs_.init(config, &thread_pool_).ok()); - - // Create bucket - bool is_bucket; - REQUIRE(gcs_.is_bucket(GCS_BUCKET, &is_bucket).ok()); - if (is_bucket) { - REQUIRE(gcs_.remove_bucket(GCS_BUCKET).ok()); - } - - REQUIRE(gcs_.is_bucket(GCS_BUCKET, &is_bucket).ok()); - REQUIRE(!is_bucket); - REQUIRE(gcs_.create_bucket(GCS_BUCKET).ok()); - - // Check if bucket is empty - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); -} - -std::string GSFx::random_bucket_name(const std::string& prefix) { - std::stringstream ss; - ss << prefix << "-" << std::this_thread::get_id() << "-" - << tiledb::sm::utils::time::timestamp_now_ms(); - return ss.str(); -} - -TEST_CASE_METHOD(GSFx, "Test GS filesystem, file management", "[gs]") { - Config config; - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - init_gcs(std::move(config)); - - /* Create the following file hierarchy: - * - * TEST_DIR/dir/subdir/file1 - * TEST_DIR/dir/subdir/file2 - * TEST_DIR/dir/file3 - * TEST_DIR/file4 - * TEST_DIR/file5 - */ - auto dir = TEST_DIR + "dir/"; - auto dir2 = TEST_DIR + "dir2/"; - auto subdir = dir + "subdir/"; - auto file1 = subdir + "file1"; - auto file2 = subdir + "file2"; - auto file3 = dir + "file3"; - auto file4 = TEST_DIR + "file4"; - auto file5 = TEST_DIR + "file5"; - auto file6 = TEST_DIR + "file6"; - - // Check that bucket is empty - bool is_empty; - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(is_empty); - - // Continue building the hierarchy - bool is_object = false; - REQUIRE(gcs_.touch(URI(file1)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file2)).ok()); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file3)).ok()); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.touch(URI(file5)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(is_object); - - // Check that bucket is not empty - REQUIRE(gcs_.is_empty_bucket(GCS_BUCKET, &is_empty).ok()); - REQUIRE(!is_empty); - - // Check invalid file - REQUIRE(gcs_.is_object(URI(TEST_DIR + "foo"), &is_object).ok()); - REQUIRE(!is_object); - - // List with prefix - std::vector paths; - REQUIRE(gcs_.ls(URI(TEST_DIR), &paths).ok()); - REQUIRE(paths.size() == 3); - paths.clear(); - REQUIRE(gcs_.ls(URI(dir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(URI(subdir), &paths).ok()); - REQUIRE(paths.size() == 2); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Check if a directory exists - bool is_dir = false; - REQUIRE(gcs_.is_dir(URI(file1), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(file4), &is_dir).ok()); - REQUIRE(!is_dir); // Not a dir - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - REQUIRE(gcs_.is_dir(URI(TEST_DIR + "dir"), &is_dir).ok()); - REQUIRE(is_dir); // This is viewed as a dir - - // Move file - REQUIRE(gcs_.move_object(URI(file5), URI(file6)).ok()); - REQUIRE(gcs_.is_object(URI(file5), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file6), &is_object).ok()); - REQUIRE(is_object); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Move directory - REQUIRE(gcs_.move_dir(URI(dir), URI(dir2)).ok()); - REQUIRE(gcs_.is_dir(URI(dir), &is_dir).ok()); - REQUIRE(!is_dir); - REQUIRE(gcs_.is_dir(URI(dir2), &is_dir).ok()); - REQUIRE(is_dir); - paths.clear(); - REQUIRE(gcs_.ls(GCS_BUCKET, &paths, "").ok()); // No delimiter - REQUIRE(paths.size() == 5); - - // Remove files - REQUIRE(gcs_.remove_object(URI(file4)).ok()); - REQUIRE(gcs_.is_object(URI(file4), &is_object).ok()); - REQUIRE(!is_object); - - // Remove directories - REQUIRE(gcs_.remove_dir(URI(dir2)).ok()); - REQUIRE(gcs_.is_object(URI(file1), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file2), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(file3), &is_object).ok()); - REQUIRE(!is_object); -} - -TEST_CASE_METHOD( - GSFx, - "Test GS filesystem I/O, multipart, serial", - "[gs][multipart][serial]") { - Config config; - const uint64_t max_parallel_ops = 1; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the files do not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the files - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == (buffer_size_large + buffer_size_small)); - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GSFx, "Test GS filesystem I/O, non-multipart", "[gs][non-multipart]") { - Config config; - const uint64_t max_parallel_ops = 1; - const uint64_t write_cache_max_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "false").ok()); - REQUIRE(config - .set( - "vfs.gcs.max_direct_upload_size", - std::to_string(write_cache_max_size)) - .ok()); - init_gcs(std::move(config)); - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - !gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(smallfile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(smallfile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GSFx, - "Test GS filesystem I/O, multipart, concurrent", - "[gs][multipart][concurrent]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare buffers - uint64_t buffer_size_large = write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - REQUIRE( - gcs_.write(URI(largefile), write_buffer_small, buffer_size_small).ok()); - auto smallfile = TEST_DIR + "smallfile"; - REQUIRE( - gcs_.write(URI(smallfile), write_buffer_small, buffer_size_small).ok()); - - // Before flushing, the files do not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the files - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - REQUIRE(gcs_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - REQUIRE(gcs_.is_object(URI(smallfile), &is_object).ok()); - REQUIRE(is_object); - - // Get file sizes - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == (buffer_size_large + buffer_size_small)); - REQUIRE(gcs_.object_size(URI(smallfile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); -} - -TEST_CASE_METHOD( - GSFx, - "Test GS filesystem I/O, multipart, composition", - "[gs][multipart][composition]") { - Config config; - const uint64_t max_parallel_ops = 4; - const uint64_t multi_part_size = 4 * 1024; - REQUIRE( - config.set("vfs.gcs.max_parallel_ops", std::to_string(max_parallel_ops)) - .ok()); - REQUIRE(config.set("vfs.gcs.use_multi_part_upload", "true").ok()); - REQUIRE(config.set("vfs.gcs.multi_part_size", std::to_string(multi_part_size)) - .ok()); - init_gcs(std::move(config)); - - const uint64_t write_cache_max_size = max_parallel_ops * multi_part_size; - - // Prepare a buffer that will write 200 (50 * 4 threads) objects. - // The maximum number of objects per composition operation is 32. - uint64_t buffer_size_large = 50 * write_cache_max_size; - auto write_buffer_large = new char[buffer_size_large]; - for (uint64_t i = 0; i < buffer_size_large; i++) - write_buffer_large[i] = (char)('a' + (i % 26)); - - // Write to the file - auto largefile = TEST_DIR + "largefile"; - REQUIRE( - gcs_.write(URI(largefile), write_buffer_large, buffer_size_large).ok()); - - // Before flushing, the file does not exist - bool is_object = false; - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(!is_object); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - uint64_t nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_large); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - REQUIRE(gcs_.read(URI(largefile), 0, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Read from a different offset - REQUIRE(gcs_.read(URI(largefile), 11, read_buffer, 26, 0, &bytes_read).ok()); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - REQUIRE(allok); - - // Prepare a buffer that will overwrite the original with a smaller - // size. - uint64_t buffer_size_overwrite = 10 * write_cache_max_size; - auto write_buffer_overwrite = new char[buffer_size_overwrite]; - for (uint64_t i = 0; i < buffer_size_overwrite; i++) - write_buffer_overwrite[i] = (char)('a' + (i % 26)); - - // Write to the file - REQUIRE( - gcs_.write(URI(largefile), write_buffer_overwrite, buffer_size_overwrite) - .ok()); - - // Flush the file - REQUIRE(gcs_.flush_object(URI(largefile)).ok()); - - // After flushing, the file exists - REQUIRE(gcs_.is_object(URI(largefile), &is_object).ok()); - REQUIRE(is_object); - - // Get file size - nbytes = 0; - REQUIRE(gcs_.object_size(URI(largefile), &nbytes).ok()); - REQUIRE(nbytes == buffer_size_overwrite); -} - -#endif diff --git a/test/src/unit-hdfs-filesystem.cc b/test/src/unit-hdfs-filesystem.cc deleted file mode 100644 index 4715ccb098f3..000000000000 --- a/test/src/unit-hdfs-filesystem.cc +++ /dev/null @@ -1,139 +0,0 @@ -/** - * @file hdfs-unit-filesystem.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for HDFS API filesystem functions. - */ - -#ifdef HAVE_HDFS - -#include -#include "tiledb/common/filesystem/directory_entry.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/filesystem/hdfs_filesystem.h" -#include "tiledb/sm/filesystem/uri.h" - -#include -#include - -using namespace tiledb::common; -using namespace tiledb::sm; -using namespace tiledb::sm::hdfs; - -TEST_CASE("Test HDFS filesystem", "[hdfs]") { - Config config; - HDFS hdfs; - - Status st = hdfs.init(config); - REQUIRE(st.ok()); - - bool is_dir; - st = hdfs.is_dir(URI("hdfs:///tiledb_test"), &is_dir); - CHECK(st.ok()); - if (is_dir) { - st = hdfs.remove_dir(URI("hdfs:///tiledb_test")); - CHECK(st.ok()); - } - - st = hdfs.create_dir(URI("hdfs:///tiledb_test")); - CHECK(st.ok()); - - CHECK(hdfs.is_dir(URI("hdfs:///tiledb_test"), &is_dir).ok()); - CHECK(is_dir); - - st = hdfs.create_dir(URI("hdfs:///tiledb_test")); - CHECK(!st.ok()); - - st = hdfs.touch(URI("hdfs:///tiledb_test_file")); - CHECK(st.ok()); - - bool is_file; - CHECK(hdfs.is_file(URI("hdfs:///tiledb_test_file"), &is_file).ok()); - CHECK(is_file); - - st = hdfs.remove_file(URI("hdfs:///tiledb_test_file")); - CHECK(st.ok()); - - st = hdfs.touch(URI("hdfs:///tiledb_test/tiledb_test_file")); - CHECK(st.ok()); - - uint64_t buffer_size = 100000; - auto write_buffer = new char[buffer_size]; - for (uint64_t i = 0; i < buffer_size; i++) { - write_buffer[i] = 'a' + (i % 26); - } - st = hdfs.write( - URI("hdfs:///tiledb_test/tiledb_test_file"), write_buffer, buffer_size); - CHECK(st.ok()); - - auto read_buffer = new char[26]; - st = hdfs.read( - URI("hdfs:///tiledb_test/tiledb_test_file"), 0, read_buffer, 26); - CHECK(st.ok()); - - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - CHECK(allok); - - st = hdfs.read( - URI("hdfs:///tiledb_test/tiledb_test_file"), 11, read_buffer, 26); - CHECK(st.ok()); - - allok = true; - for (int i = 0; i < 26; ++i) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - CHECK(allok); - - uint64_t nbytes = 0; - st = hdfs.file_size(URI("hdfs:///tiledb_test/tiledb_test_file"), &nbytes); - CHECK(st.ok()); - CHECK(nbytes == buffer_size); - - st = hdfs.remove_file(URI("hdfs:///tiledb_test/i_dont_exist")); - CHECK(!st.ok()); - - st = hdfs.remove_file(URI("hdfs:///tiledb_test/tiledb_test_file")); - CHECK(st.ok()); - - st = hdfs.remove_dir(URI("hdfs:///tiledb_test")); - CHECK(st.ok()); - - st = hdfs.disconnect(); - CHECK(st.ok()); -} - -#endif diff --git a/test/src/unit-s3-no-multipart.cc b/test/src/unit-s3-no-multipart.cc deleted file mode 100644 index 2c973cf29b75..000000000000 --- a/test/src/unit-s3-no-multipart.cc +++ /dev/null @@ -1,210 +0,0 @@ -/** - * @file unit-s3-no-multipart.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for S3 API filesystem functions. - */ - -#ifdef HAVE_S3 - -#include -#include "test/support/src/helpers.h" -#include "tiledb/common/thread_pool.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/filesystem/s3.h" -#include "tiledb/sm/global_state/unit_test_config.h" -#include "tiledb/sm/misc/tdb_time.h" - -#include -#include - -using namespace tiledb::test; -using namespace tiledb::common; -using namespace tiledb::sm; - -struct S3DirectFx { - S3DirectFx(); - ~S3DirectFx(); - static Config set_config_params(); - - const std::string S3_PREFIX = "s3://"; - const tiledb::sm::URI S3_BUCKET = - tiledb::sm::URI(S3_PREFIX + "tiledb-" + random_label() + "/"); - const std::string TEST_DIR = S3_BUCKET.to_string() + "tiledb_test_dir/"; - ThreadPool thread_pool_{2}; - tiledb::sm::S3 s3_{&g_helper_stats, &thread_pool_, set_config_params()}; -}; - -S3DirectFx::S3DirectFx() { - // Create bucket - bool exists = s3_.is_bucket(S3_BUCKET); - if (exists) - REQUIRE_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - - exists = s3_.is_bucket(S3_BUCKET); - REQUIRE(!exists); - REQUIRE_NOTHROW(s3_.create_bucket(S3_BUCKET)); - - // Check if bucket is empty - bool is_empty = s3_.is_empty_bucket(S3_BUCKET); - CHECK(is_empty); -} - -S3DirectFx::~S3DirectFx() { - // Empty bucket - bool is_empty = s3_.is_empty_bucket(S3_BUCKET); - if (!is_empty) { - CHECK_NOTHROW(s3_.empty_bucket(S3_BUCKET)); - is_empty = s3_.is_empty_bucket(S3_BUCKET); - CHECK(is_empty); - } - - // Delete bucket - CHECK_NOTHROW(s3_.remove_bucket(S3_BUCKET)); - CHECK(s3_.disconnect().ok()); -} - -Config S3DirectFx::set_config_params() { - // Connect - Config config; -#ifndef TILEDB_TESTS_AWS_S3_CONFIG - REQUIRE(config.set("vfs.s3.endpoint_override", "localhost:9999").ok()); - REQUIRE(config.set("vfs.s3.scheme", "https").ok()); - REQUIRE(config.set("vfs.s3.use_virtual_addressing", "false").ok()); - REQUIRE(config.set("vfs.s3.verify_ssl", "false").ok()); -#endif - REQUIRE(config.set("vfs.s3.max_parallel_ops", "1").ok()); - // set max buffer size to 10 MB - REQUIRE(config.set("vfs.s3.multipart_part_size", "10000000").ok()); - REQUIRE(config.set("vfs.s3.use_multipart_upload", "false").ok()); - return config; -} - -TEST_CASE_METHOD( - S3DirectFx, - "Test S3 filesystem, file I/O with multipart API disabled", - "[s3]") { - // Prepare buffers - uint64_t buffer_size = 5 * 1024 * 1024; - auto write_buffer = new char[buffer_size]; - for (uint64_t i = 0; i < buffer_size; i++) - write_buffer[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - CHECK_NOTHROW(s3_.write(URI(largefile), write_buffer, buffer_size)); - CHECK_NOTHROW( - s3_.write(URI(largefile), write_buffer_small, buffer_size_small)); - auto smallfile = TEST_DIR + "smallfile"; - CHECK_NOTHROW( - s3_.write(URI(smallfile), write_buffer_small, buffer_size_small)); - - // Before flushing, the files do not exist - bool exists = false; - CHECK(s3_.is_object(URI(largefile), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(smallfile), &exists).ok()); - CHECK(!exists); - - // Flush the files - CHECK(s3_.flush_object(URI(largefile)).ok()); - CHECK(s3_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - CHECK(s3_.is_object(URI(largefile), &exists).ok()); - CHECK(exists); - CHECK(s3_.is_object(URI(smallfile), &exists).ok()); - CHECK(exists); - - // Get file sizes - uint64_t nbytes = 0; - CHECK(s3_.object_size(URI(largefile), &nbytes).ok()); - CHECK(nbytes == (buffer_size + buffer_size_small)); - CHECK(s3_.object_size(URI(smallfile), &nbytes).ok()); - CHECK(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - CHECK_NOTHROW( - s3_.read_impl(URI(largefile), 0, read_buffer, 26, 0, &bytes_read)); - assert(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - CHECK(allok); - - // Read from a different offset - CHECK_NOTHROW( - s3_.read_impl(URI(largefile), 11, read_buffer, 26, 0, &bytes_read)); - assert(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - CHECK(allok); - - // Try to write 11 MB file, should fail with given buffer configuration - auto badfile = TEST_DIR + "badfile"; - auto badbuffer = (char*)malloc(11000000); - CHECK_THROWS((s3_.write(URI(badfile), badbuffer, 11000000))); -} - -TEST_CASE_METHOD( - S3DirectFx, "Validate vfs.s3.custom_headers.*", "[s3][custom-headers]") { - Config cfg = set_config_params(); - - // Check the edge case of a key matching the ConfigIter prefix. - REQUIRE(cfg.set("vfs.s3.custom_headers.", "").ok()); - - // Set an unexpected value for Content-MD5, which minio should reject - REQUIRE(cfg.set("vfs.s3.custom_headers.Content-MD5", "unexpected").ok()); - - // Recreate a new S3 client because config is not dynamic - tiledb::sm::S3 s3{&g_helper_stats, &thread_pool_, cfg}; - auto uri = URI(TEST_DIR + "writefailure"); - - // This is a buffered write, which is why it should not throw. - CHECK_NOTHROW(s3.write(uri, "Validate s3 custom headers", 26)); - - auto matcher = Catch::Matchers::ContainsSubstring( - "The Content-Md5 you specified is not valid."); - REQUIRE_THROWS_WITH(s3.flush_object(uri), matcher); -} -#endif diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index cf05f8a9b3c4..aa2c68a593cd 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -102,79 +102,6 @@ Config S3Fx::set_config_params() { return config; } -TEST_CASE_METHOD(S3Fx, "Test S3 filesystem, file I/O", "[s3]") { - // Prepare buffers - uint64_t buffer_size = 5 * 1024 * 1024; - auto write_buffer = new char[buffer_size]; - for (uint64_t i = 0; i < buffer_size; i++) - write_buffer[i] = (char)('a' + (i % 26)); - uint64_t buffer_size_small = 1024 * 1024; - auto write_buffer_small = new char[buffer_size_small]; - for (uint64_t i = 0; i < buffer_size_small; i++) - write_buffer_small[i] = (char)('a' + (i % 26)); - - // Write to two files - auto largefile = TEST_DIR + "largefile"; - CHECK_NOTHROW(s3_.write(URI(largefile), write_buffer, buffer_size)); - CHECK_NOTHROW( - s3_.write(URI(largefile), write_buffer_small, buffer_size_small)); - auto smallfile = TEST_DIR + "smallfile"; - CHECK_NOTHROW( - s3_.write(URI(smallfile), write_buffer_small, buffer_size_small)); - - // Before flushing, the files do not exist - bool exists = false; - CHECK(s3_.is_object(URI(largefile), &exists).ok()); - CHECK(!exists); - CHECK(s3_.is_object(URI(smallfile), &exists).ok()); - CHECK(!exists); - - // Flush the files - CHECK(s3_.flush_object(URI(largefile)).ok()); - CHECK(s3_.flush_object(URI(smallfile)).ok()); - - // After flushing, the files exist - CHECK(s3_.is_object(URI(largefile), &exists).ok()); - CHECK(exists); - CHECK(s3_.is_object(URI(smallfile), &exists).ok()); - CHECK(exists); - - // Get file sizes - uint64_t nbytes = 0; - CHECK(s3_.object_size(URI(largefile), &nbytes).ok()); - CHECK(nbytes == (buffer_size + buffer_size_small)); - CHECK(s3_.object_size(URI(smallfile), &nbytes).ok()); - CHECK(nbytes == buffer_size_small); - - // Read from the beginning - auto read_buffer = new char[26]; - uint64_t bytes_read = 0; - CHECK_NOTHROW( - s3_.read_impl(URI(largefile), 0, read_buffer, 26, 0, &bytes_read)); - CHECK(26 == bytes_read); - bool allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + i)) { - allok = false; - break; - } - } - CHECK(allok); - - // Read from a different offset - CHECK_NOTHROW( - s3_.read_impl(URI(largefile), 11, read_buffer, 26, 0, &bytes_read)); - CHECK(26 == bytes_read); - allok = true; - for (int i = 0; i < 26; i++) { - if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { - allok = false; - break; - } - } - CHECK(allok); -} - TEST_CASE_METHOD(S3Fx, "Test S3 multiupload abort path", "[s3]") { // Prepare a large buffer uint64_t buffer_size = 100 * 1024 * 1024; diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index ae6fa1d250b6..b1c9b8b1f54c 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2023 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,8 +33,13 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/temporary_local_directory.h" +#ifdef HAVE_AZURE +#include +#include "tiledb/sm/filesystem/azure.h" +#endif #include "test/support/src/vfs_helpers.h" #include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/global_state/unit_test_config.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/path_win.h" #endif @@ -53,15 +58,25 @@ void require_tiledb_ok(Status st) { REQUIRE(st.ok()); } -void require_tiledb_err(Status st) { - REQUIRE(!st.ok()); -} - -Config set_config_params() { +Config set_config_params( + bool disable_multipart = false, uint64_t parallel_ops = 1) { Config config; if constexpr (tiledb::sm::filesystem::gcs_enabled) { require_tiledb_ok(config.set("vfs.gcs.project_id", "TODO")); + if (parallel_ops != 1) { + require_tiledb_ok( + config.set("vfs.gcs.max_parallel_ops", std::to_string(parallel_ops))); + require_tiledb_ok(config.set( + "vfs.gcs.multi_part_size", std::to_string(4 * 1024 * 1024))); + } + if (disable_multipart) { + require_tiledb_ok( + config.set("vfs.gcs.max_parallel_ops", std::to_string(1))); + require_tiledb_ok(config.set("vfs.gcs.use_multi_part_upload", "false")); + require_tiledb_ok(config.set( + "vfs.gcs.max_direct_upload_size", std::to_string(4 * 1024 * 1024))); + } } if constexpr (tiledb::sm::filesystem::s3_enabled) { @@ -69,6 +84,11 @@ Config set_config_params() { require_tiledb_ok(config.set("vfs.s3.scheme", "https")); require_tiledb_ok(config.set("vfs.s3.use_virtual_addressing", "false")); require_tiledb_ok(config.set("vfs.s3.verify_ssl", "false")); + if (disable_multipart) { + require_tiledb_ok(config.set("vfs.s3.max_parallel_ops", "1")); + require_tiledb_ok(config.set("vfs.s3.multipart_part_size", "10000000")); + require_tiledb_ok(config.set("vfs.s3.use_multipart_upload", "false")); + } } if constexpr (tiledb::sm::filesystem::azure_enabled) { @@ -87,11 +107,29 @@ Config set_config_params() { // require_tiledb_ok(config.set("vfs.azure.storage_sas_token", "")); // require_tiledb_ok(config.set( // "vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount2")); + if (parallel_ops != 1) { + require_tiledb_ok(config.set( + "vfs.azure.max_parallel_ops", std::to_string(parallel_ops))); + require_tiledb_ok(config.set( + "vfs.azure.block_list_block_size", std::to_string(4 * 1024 * 1024))); + } + if (disable_multipart) { + require_tiledb_ok(config.set("vfs.azure.use_block_list_upload", "false")); + } } return config; } +std::string local_path() { + std::string local_prefix = ""; + if constexpr (!tiledb::sm::filesystem::windows_enabled) { + local_prefix = "file://"; + } + + return local_prefix + unit_vfs_dir_.path(); +} + TEST_CASE("VFS: Test long local paths", "[vfs]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); @@ -99,11 +137,7 @@ TEST_CASE("VFS: Test long local paths", "[vfs]") { SECTION("- Deep hierarchy") { // Create a nested path with a long total length - std::string local_prefix = ""; - if constexpr (!tiledb::sm::filesystem::windows_enabled) { - local_prefix = "file://"; - } - std::string tmpdir = local_prefix + unit_vfs_dir_.path(); + std::string tmpdir = local_path(); bool success = true; while (tmpdir.size() < 512) { tmpdir += "subdir/"; @@ -113,8 +147,8 @@ TEST_CASE("VFS: Test long local paths", "[vfs]") { } } - // On some Windows platforms, the path length of a directory must be <= 248 - // chars. On others (that have opted in to a configuration that allows + // On some Windows platforms, the path length of a directory must be <= + // 248 chars. On others (that have opted in to a configuration that allows // long paths) the limit is ~32,767. Here we check for either case. if (success) { // Check we can create files within the deep hierarchy @@ -138,11 +172,7 @@ TEST_CASE("VFS: Test long local paths", "[vfs]") { for (unsigned i = 0; i < 256; i++) { name += "x"; } - std::string local_prefix = ""; - if constexpr (!tiledb::sm::filesystem::windows_enabled) { - local_prefix = "file://"; - } - std::string tmpdir = local_prefix + unit_vfs_dir_.path(); + std::string tmpdir = local_path(); URI testfile(tmpdir + name); // Creating the URI and checking its existence is fine on posix @@ -162,45 +192,21 @@ TEST_CASE("VFS: Test long local paths", "[vfs]") { } } -TEST_CASE("VFS: URI semantics and file management", "[vfs][uri]") { +using AllBackends = + std::tuple; +TEMPLATE_LIST_TEST_CASE( + "VFS: URI semantics and file management", "[vfs][uri]", AllBackends) { + TestType fs({0}); + if (!fs.is_supported()) { + return; + } + ThreadPool compute_tp(4); ThreadPool io_tp(4); Config config = set_config_params(); VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; - // Sections to test each enabled filesystem - URI path; - std::string local_prefix = ""; - SECTION("Filesystem: Local") { - if constexpr (!tiledb::sm::filesystem::windows_enabled) { - local_prefix = "file://"; - } - path = URI(local_prefix + unit_vfs_dir_.path()); - } - - if constexpr (tiledb::sm::filesystem::gcs_enabled) { - SECTION("Filesystem: GCS") { - path = URI("gcs://vfs-" + random_label() + "/"); - } - } - - if constexpr (tiledb::sm::filesystem::s3_enabled) { - SECTION("Filesystem: S3") { - path = URI("s3://vfs-" + random_label() + "/"); - } - } - - if constexpr (tiledb::sm::filesystem::hdfs_enabled) { - SECTION("Filesystem: HDFS") { - path = URI("hdfs:///vfs-" + random_label() + "/"); - } - } - - if constexpr (tiledb::sm::filesystem::azure_enabled) { - SECTION("Filesystem: Azure") { - path = URI("azure://vfs-" + random_label() + "/"); - } - } + URI path = fs.temp_dir_.add_trailing_slash(); // Set up bool exists = false; @@ -399,16 +405,142 @@ TEST_CASE("VFS: URI semantics and file management", "[vfs][uri]") { } } +TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) { + TestType fs({0}); + if (!fs.is_supported()) { + return; + } + + bool disable_multipart = GENERATE(true, false); + uint64_t max_parallel_ops = 1; + uint64_t chunk_size = 1024 * 1024; + int multiplier = 5; + + URI path = fs.temp_dir_.add_trailing_slash(); + + if constexpr ( + std::is_same::value || + std::is_same::value) { + chunk_size = 4 * 1024 * 1024; + multiplier = 1; + + if (!disable_multipart) { + max_parallel_ops = GENERATE(1, 4); + } + } + + if constexpr (std::is_same::value) { + max_parallel_ops = 2; + chunk_size = 4 * 1024 * 1024; + if (disable_multipart) { + multiplier = 1; + } + } + + ThreadPool compute_tp(4); + ThreadPool io_tp(4); + Config config = set_config_params(disable_multipart, max_parallel_ops); + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; + + // Set up + bool exists = false; + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.is_bucket(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_bucket(path)); + } + require_tiledb_ok(vfs.create_bucket(path)); + } else { + require_tiledb_ok(vfs.is_dir(path, &exists)); + if (exists) { + require_tiledb_ok(vfs.remove_dir(path)); + } + require_tiledb_ok(vfs.create_dir(path)); + } + + // Prepare buffers + uint64_t buffer_size = multiplier * max_parallel_ops * chunk_size; + auto write_buffer = new char[buffer_size]; + for (uint64_t i = 0; i < buffer_size; i++) + write_buffer[i] = (char)('a' + (i % 26)); + uint64_t buffer_size_small = 1024 * 1024; + auto write_buffer_small = new char[buffer_size_small]; + for (uint64_t i = 0; i < buffer_size_small; i++) + write_buffer_small[i] = (char)('a' + (i % 26)); + + // Write to two files + URI largefile = URI(path.to_string() + "largefile"); + require_tiledb_ok(vfs.write(largefile, write_buffer, buffer_size)); + URI smallfile = URI(path.to_string() + "smallfile"); + require_tiledb_ok( + vfs.write(smallfile, write_buffer_small, buffer_size_small)); + + // On non-local and hdfs systems, before flushing, the files do not exist + if (!(path.is_file() || path.is_hdfs())) { + require_tiledb_ok(vfs.is_file(largefile, &exists)); + CHECK(!exists); + require_tiledb_ok(vfs.is_file(smallfile, &exists)); + CHECK(!exists); + + // Flush the files + require_tiledb_ok(vfs.close_file(largefile)); + require_tiledb_ok(vfs.close_file(smallfile)); + } + + // After flushing, the files exist + require_tiledb_ok(vfs.is_file(largefile, &exists)); + CHECK(exists); + require_tiledb_ok(vfs.is_file(smallfile, &exists)); + CHECK(exists); + + // Get file sizes + uint64_t nbytes = 0; + require_tiledb_ok(vfs.file_size(largefile, &nbytes)); + CHECK(nbytes == (buffer_size)); + require_tiledb_ok(vfs.file_size(smallfile, &nbytes)); + CHECK(nbytes == buffer_size_small); + + // Read from the beginning + auto read_buffer = new char[26]; + require_tiledb_ok(vfs.read(largefile, 0, read_buffer, 26)); + bool allok = true; + for (int i = 0; i < 26; i++) { + if (read_buffer[i] != static_cast('a' + i)) { + allok = false; + break; + } + } + CHECK(allok); + + // Read from a different offset + require_tiledb_ok(vfs.read(largefile, 11, read_buffer, 26)); + allok = true; + for (int i = 0; i < 26; i++) { + if (read_buffer[i] != static_cast('a' + (i + 11) % 26)) { + allok = false; + break; + } + } + CHECK(allok); + + // Clean up + if (path.is_gcs() || path.is_s3() || path.is_azure()) { + require_tiledb_ok(vfs.remove_bucket(path)); + require_tiledb_ok(vfs.is_bucket(path, &exists)); + REQUIRE(!exists); + } else { + require_tiledb_ok(vfs.remove_dir(path)); + require_tiledb_ok(vfs.is_dir(path, &exists)); + REQUIRE(!exists); + } +} + TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); VFS vfs_ls{&g_helper_stats, &compute_tp, &io_tp, Config{}}; - std::string local_prefix = ""; - if constexpr (!tiledb::sm::filesystem::windows_enabled) { - local_prefix = "file://"; - } - std::string path = local_prefix + unit_vfs_dir_.path(); + std::string path = local_path(); std::string dir = path + "ls_dir"; std::string file = dir + "/file"; std::string subdir = dir + "/subdir"; @@ -542,3 +674,67 @@ TEST_CASE( Catch::Matchers::ContainsSubstring("Throwing FileFilter")); } } + +#ifdef HAVE_AZURE +TEST_CASE("VFS: Construct Azure Blob Storage endpoint URIs", "[azure][uri]") { + std::string sas_token, custom_endpoint, expected_endpoint; + SECTION("No SAS token") { + sas_token = ""; + expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; + } + SECTION("SAS token without leading question mark") { + sas_token = "baz=qux&foo=bar"; + expected_endpoint = + "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + } + SECTION("SAS token with leading question mark") { + sas_token = "?baz=qux&foo=bar"; + expected_endpoint = + "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + } + SECTION("SAS token in both endpoint and config option") { + sas_token = "baz=qux&foo=bar"; + custom_endpoint = + "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + expected_endpoint = + "https://devstoreaccount1.blob.core.windows.net?baz=qux&foo=bar"; + } + SECTION("No SAS token") { + sas_token = ""; + expected_endpoint = "https://devstoreaccount1.blob.core.windows.net"; + } + Config config; + require_tiledb_ok( + config.set("vfs.azure.storage_account_name", "devstoreaccount1")); + require_tiledb_ok(config.set("vfs.azure.blob_endpoint", custom_endpoint)); + require_tiledb_ok(config.set("vfs.azure.storage_sas_token", sas_token)); + tiledb::sm::Azure azure; + ThreadPool thread_pool(1); + require_tiledb_ok(azure.init(config, &thread_pool)); + REQUIRE(azure.client().GetUrl() == expected_endpoint); +} +#endif + +#ifdef HAVE_S3 +TEST_CASE("Validate vfs.s3.custom_headers.*", "[s3][custom-headers]") { + Config cfg = set_config_params(true); + + // Check the edge case of a key matching the ConfigIter prefix. + REQUIRE(cfg.set("vfs.s3.custom_headers.", "").ok()); + + // Set an unexpected value for Content-MD5, which minio should reject + REQUIRE(cfg.set("vfs.s3.custom_headers.Content-MD5", "unexpected").ok()); + + // Recreate a new S3 client because config is not dynamic + ThreadPool thread_pool(2); + S3 s3{&g_helper_stats, &thread_pool, cfg}; + auto uri = URI("s3://tiledb-" + random_label() + "/writefailure"); + + // This is a buffered write, which is why it should not throw. + CHECK_NOTHROW(s3.write(uri, "Validate s3 custom headers", 26)); + + auto matcher = Catch::Matchers::ContainsSubstring( + "The Content-Md5 you specified is not valid."); + REQUIRE_THROWS_WITH(s3.flush_object(uri), matcher); +} +#endif diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index c343f023d1a0..bce5c43d70ca 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -914,11 +914,19 @@ class GCSTest : public VFSTestBase { } }; +/** Stub test object for tiledb::sm::GS functionality. */ +class GSTest : public VFSTestBase { + public: + explicit GSTest(const std::vector& test_tree) + : VFSTestBase(test_tree, "gs://") { + } +}; + /** Stub test object for tiledb::sm::HDFS functionality. */ class HDFSTest : public VFSTestBase { public: explicit HDFSTest(const std::vector& test_tree) - : VFSTestBase(test_tree, "hdfs://") { + : VFSTestBase(test_tree, "hdfs:///") { } }; diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 52a076acdc3a..59ef5e87065d 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -452,7 +452,8 @@ tuple>> GCS::ls_with_sizes( } auto& results = object_metadata.value(); - const std::string gcs_prefix = uri_dir.is_gcs() ? "gcs://" : "gs://"; + const std::string gcs_prefix = + uri_dir.backend_name() == "gcs" ? "gcs://" : "gs://"; if (absl::holds_alternative( results)) { From 827be1c17abc4989e6b50d34decdbdd77e3880e3 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Fri, 1 Mar 2024 02:54:01 -0500 Subject: [PATCH 212/456] Fix yaml syntax error in ci-rest job (#4776) Fix yaml syntax error in ci-rest job. --- TYPE: NO_HISTORY --- .github/workflows/ci-rest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 08ba19d26a3b..0f5caab07213 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -1,8 +1,8 @@ name: REST CI on: - - workflow_call - - workflow_dispatch + workflow_call: + workflow_dispatch: push: branches: From 2fcf3718c2b53537dda1518b3496662ac51b1546 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 29 Feb 2024 23:57:12 -0800 Subject: [PATCH 213/456] Integrate std::prm memory tracking for class `ArraySchemaEvolution`. (#4736) All member vector variables of the ArraySchemaEvolution class have PMR tracking. --- TYPE: NO_HISTORY DESC: Integrate std::prm memory tracking for class `ArraySchemaEvolution`. --- test/src/unit-cppapi-schema-evolution.cc | 3 +- test/src/unit-enumerations.cc | 51 ++++++++++--------- tiledb/common/memory_tracker.cc | 18 +++++++ tiledb/common/memory_tracker.h | 3 +- .../sm/array_schema/array_schema_evolution.cc | 35 +++++++++++-- .../sm/array_schema/array_schema_evolution.h | 27 +++++++--- tiledb/sm/c_api/tiledb.cc | 9 +++- .../serialization/array_schema_evolution.cc | 22 +++++--- .../sm/serialization/array_schema_evolution.h | 3 +- 9 files changed, 122 insertions(+), 49 deletions(-) diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index ca85b4d5528c..b5bb38f0e8ff 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -797,7 +797,8 @@ TEST_CASE( TEST_CASE( "SchemaEvolution Error Handling Tests", "[cppapi][schema][evolution][errors]") { - auto ase = make_shared(HERE()); + auto ase = make_shared( + HERE(), tiledb::test::create_test_memory_tracker()); REQUIRE_THROWS(ase->evolve_schema(nullptr)); REQUIRE_THROWS(ase->add_attribute(nullptr)); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index f528b0b8712f..6f841fa278b9 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1576,7 +1576,7 @@ TEST_CASE_METHOD( array->load_all_enumerations(); auto orig_schema = array->array_schema_latest_ptr(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); ase->add_attribute(attr3); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); @@ -1588,7 +1588,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][simple]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1609,7 +1609,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][drop-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1630,7 +1630,7 @@ TEST_CASE_METHOD( create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values); @@ -1649,7 +1649,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-add]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values); @@ -1675,7 +1675,7 @@ TEST_CASE_METHOD( REQUIRE(old_enmr != nullptr); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); CHECK_NOTHROW(ase->evolve_schema(orig_schema)); } @@ -1684,7 +1684,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration", "[enumeration][array-schema-evolution][enmr-to-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1694,7 +1694,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-to-drop]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase1 = make_shared(HERE()); + auto ase1 = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1702,7 +1702,7 @@ TEST_CASE_METHOD( auto new_schema = ase1->evolve_schema(orig_schema); - auto ase2 = make_shared(HERE()); + auto ase2 = make_shared(HERE(), memory_tracker_); ase2->drop_enumeration("enmr"); CHECK_NOTHROW(ase2->evolve_schema(new_schema)); @@ -1712,7 +1712,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration Repeated", "[enumeration][array-schema-evolution][enmr-to-drop-repeated]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); CHECK_NOTHROW(ase->drop_enumeration("test_enmr")); } @@ -1721,7 +1721,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Drop Enumeration After Add", "[enumeration][array-schema-evolution][enmr-add-drop]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1736,7 +1736,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - nullptr", "[enumeration][array-schema-evolution][enmr-nullptr]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE_THROWS(ase->add_enumeration(nullptr)); } @@ -1745,7 +1745,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Already Added", "[enumeration][array-schema-evolution][enmr-already-added]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values{0, 1, 2, 3, 4, 1000}; auto enmr1 = create_enumeration(values, false, Datatype::UINT64, "enmr"); @@ -1759,7 +1759,7 @@ TEST_CASE_METHOD( "ArraySchemaEvolution - Enumeration to Add - Missing Name", "[enumeration][array-schema-evolution][missing-name]") { create_array(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); REQUIRE(ase->enumeration_to_add("foo") == nullptr); } @@ -1769,7 +1769,7 @@ TEST_CASE_METHOD( "[enumeration][array-schema-evolution][enmr-still-in-use]") { create_array(); auto orig_schema = get_array_schema_latest(); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->drop_enumeration("test_enmr"); REQUIRE_THROWS(ase->evolve_schema(orig_schema)); @@ -1784,7 +1784,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); attr3->set_enumeration_name("test_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_attribute(attr3); auto orig_schema = get_array_schema_latest(); @@ -1805,7 +1805,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::INT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1827,7 +1827,7 @@ TEST_CASE_METHOD( auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT8); attr3->set_enumeration_name("big_enmr"); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->add_enumeration(enmr); ase->add_attribute(attr3); @@ -1839,7 +1839,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration nullptr", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); auto matcher = Catch::Matchers::ContainsSubstring( "Cannot extend enumeration; Input enumeration is null"); REQUIRE_THROWS_WITH(ase->extend_enumeration(nullptr), matcher); @@ -1849,7 +1849,7 @@ TEST_CASE_METHOD( EnumerationFx, "ArraySchemaEvolution - Extend Enumeration Already Extended", "[enumeration][array-schema-evolution][extend][error]") { - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); std::vector values = {1, 2, 3, 4, 5}; auto enmr = create_enumeration(values); auto matcher = Catch::Matchers::ContainsSubstring( @@ -1928,7 +1928,7 @@ TEST_CASE_METHOD( auto old_enmr = schema->get_enumeration("test_enmr"); auto new_enmr = extend_enumeration(old_enmr, values_to_add); - auto ase = make_shared(HERE()); + auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); auto st = ctx_.storage_manager()->array_evolve_schema( array->array_uri(), ase.get(), array->get_encryption_key()); @@ -2182,7 +2182,7 @@ TEST_CASE_METHOD( auto attr = make_shared(HERE(), "ohai", Datatype::INT64); attr->set_enumeration_name("enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(memory_tracker_); ase1.add_attribute(attr); ase1.add_enumeration(enmr1); ase1.add_enumeration(enmr2); @@ -2216,7 +2216,7 @@ TEST_CASE_METHOD( std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); - ArraySchemaEvolution ase1; + ArraySchemaEvolution ase1(memory_tracker_); ase1.extend_enumeration(enmr1); ase1.extend_enumeration(enmr2); @@ -2416,6 +2416,7 @@ EnumerationFx::EnumerationFx() , memory_tracker_(tiledb::test::create_test_memory_tracker()) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); + memory_tracker_ = tiledb::test::create_test_memory_tracker(); } EnumerationFx::~EnumerationFx() { @@ -2762,8 +2763,8 @@ shared_ptr EnumerationFx::ser_des_array_schema_evolution( ase, stype, &buf, client_side)); ArraySchemaEvolution* ret; - throw_if_not_ok( - serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + throw_if_not_ok(serialization::array_schema_evolution_deserialize( + &ret, stype, buf, memory_tracker_)); return shared_ptr(ret); } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 65246947170c..833d7adac0b0 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -68,6 +68,12 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; + case MemoryType::ATTRIBUTES: + return "Attributes"; + case MemoryType::DIMENSION_LABELS: + return "DimensionLabels"; + case MemoryType::DIMENSIONS: + return "Dimensions"; case MemoryType::TILE_SUMS: return "TileSums"; case MemoryType::TILE_WRITER_DATA: @@ -82,16 +88,28 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { switch (type) { case MemoryTrackerType::ANONYMOUS: return "Anonymous"; + case MemoryTrackerType::ARRAY_CREATE: + return "ArrayCreate"; + case MemoryTrackerType::ARRAY_LOAD: + return "ArrayLoad"; case MemoryTrackerType::ARRAY_READ: return "ArrayRead"; case MemoryTrackerType::ARRAY_WRITE: return "ArrayWrite"; + case MemoryTrackerType::FRAGMENT_INFO_LOAD: + return "FragmentInfoLoad"; case MemoryTrackerType::QUERY_READ: return "QueryRead"; case MemoryTrackerType::QUERY_WRITE: return "QueryWrite"; case MemoryTrackerType::CONSOLIDATOR: return "Consolidator"; + case MemoryTrackerType::REST_CLIENT: + return "RestClient"; + case MemoryTrackerType::EPHEMERAL: + return "Ephemeral"; + case MemoryTrackerType::SCHEMA_EVOLUTION: + return "SchemaEvolution"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory tracker type: " + val); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 29b34fcb2598..e9f7e5632fe9 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -131,7 +131,8 @@ enum class MemoryTrackerType { QUERY_WRITE, CONSOLIDATOR, REST_CLIENT, - EPHEMERAL + EPHEMERAL, + SCHEMA_EVOLUTION }; class MemoryTrackerResource : public tdb::pmr::memory_resource { diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index edfdfc7e70f2..117dee9607b4 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" @@ -71,7 +72,15 @@ class ArraySchemaEvolutionException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -ArraySchemaEvolution::ArraySchemaEvolution() { +ArraySchemaEvolution::ArraySchemaEvolution( + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) { } ArraySchemaEvolution::ArraySchemaEvolution( @@ -81,13 +90,29 @@ ArraySchemaEvolution::ArraySchemaEvolution( std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range) - : attributes_to_add_map_(attrs_to_add) + std::pair timestamp_range, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , attributes_to_add_map_( + memory_tracker->get_resource(MemoryType::ATTRIBUTES)) , attributes_to_drop_(attrs_to_drop) - , enumerations_to_add_map_(enmrs_to_add) - , enumerations_to_extend_map_(enmrs_to_extend) + , enumerations_to_add_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) + , enumerations_to_extend_map_( + memory_tracker_->get_resource(MemoryType::ENUMERATION)) , enumerations_to_drop_(enmrs_to_drop) , timestamp_range_(timestamp_range) { + for (auto& elem : attrs_to_add) { + attributes_to_add_map_.insert(elem); + } + + for (auto& elem : enmrs_to_add) { + enumerations_to_add_map_.insert(elem); + } + + for (auto& elem : enmrs_to_extend) { + enumerations_to_extend_map_.insert(elem); + } } ArraySchemaEvolution::~ArraySchemaEvolution() { diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 24818c598ca4..bc2160698f14 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -36,8 +36,8 @@ #include #include - #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -53,6 +53,7 @@ class ConstBuffer; class Dimension; class Domain; class Enumeration; +class MemoryTracker; class ArraySchema; enum class ArrayType : uint8_t; @@ -68,13 +69,17 @@ class ArraySchemaEvolution { /* ********************************* */ /** Constructor. */ - ArraySchemaEvolution(); + ArraySchemaEvolution() = delete; + + /** Constructor with memory tracker. */ + ArraySchemaEvolution(shared_ptr memory_tracker); /** Constructor. * @param attrs_to_add Attributes to add to the schema. * @param enmrs_to_add Enumerations to add to the schema. * @param attrs_to_drop Attributes to remove from the schema. * @param timestamp_range Timestamp range to use for the new schema. + * @param memory_tracker Memory tracker to use for the new schema. */ ArraySchemaEvolution( std::unordered_map> attrs_to_add, @@ -84,7 +89,11 @@ class ArraySchemaEvolution { std::unordered_map> enmrs_to_extend, std::unordered_set enmrs_to_drop, - std::pair timestamp_range); + std::pair timestamp_range, + shared_ptr memory_tracker); + + DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); + DISABLE_MOVE_AND_MOVE_ASSIGN(ArraySchemaEvolution); /** Destructor. */ ~ArraySchemaEvolution(); @@ -184,19 +193,25 @@ class ArraySchemaEvolution { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the ArraySchema. + */ + shared_ptr memory_tracker_; + /** The array attributes to be added. */ /** It maps each attribute name to the corresponding attribute object. */ - std::unordered_map> attributes_to_add_map_; + tdb::pmr::unordered_map> + attributes_to_add_map_; /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; /** Enumerations to add with any attribute. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_add_map_; /** Enumerations to extend. */ - std::unordered_map> + tdb::pmr::unordered_map> enumerations_to_extend_map_; /** The names of array enumerations to be dropped. */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9f8c290541fa..ebf5888ea0f3 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -829,8 +829,10 @@ int32_t tiledb_array_schema_evolution_alloc( } // Create a new SchemaEvolution object + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); (*array_schema_evolution)->array_schema_evolution_ = - new (std::nothrow) tiledb::sm::ArraySchemaEvolution(); + new (std::nothrow) tiledb::sm::ArraySchemaEvolution(memory_tracker); if ((*array_schema_evolution)->array_schema_evolution_ == nullptr) { delete *array_schema_evolution; *array_schema_evolution = nullptr; @@ -3665,12 +3667,15 @@ int32_t tiledb_deserialize_array_schema_evolution( return TILEDB_OOM; } + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); if (SAVE_ERROR_CATCH( ctx, tiledb::sm::serialization::array_schema_evolution_deserialize( &((*array_schema_evolution)->array_schema_evolution_), (tiledb::sm::SerializationType)serialize_type, - buffer->buffer()))) { + buffer->buffer(), + memory_tracker))) { delete *array_schema_evolution; *array_schema_evolution = nullptr; return TILEDB_ERR; diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index c60b9882a674..66c18a7635be 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -156,7 +156,8 @@ Status array_schema_evolution_to_capnp( } tdb_unique_ptr array_schema_evolution_from_capnp( - const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + const capnp::ArraySchemaEvolution::Reader& evolution_reader, + shared_ptr memory_tracker) { // Create attributes to add std::unordered_map> attrs_to_add; auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); @@ -213,7 +214,8 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_add, enmrs_to_extend, enmrs_to_drop, - ts_range)); + ts_range, + memory_tracker)); } Status array_schema_evolution_serialize( @@ -275,7 +277,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer) { + const Buffer& serialized_buffer, + shared_ptr memory_tracker) { try { tdb_unique_ptr decoded_array_schema_evolution = nullptr; @@ -291,8 +294,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + array_schema_evolution_reader, memory_tracker); break; } case SerializationType::CAPNP: { @@ -303,8 +306,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - decoded_array_schema_evolution = - array_schema_evolution_from_capnp(array_schema_evolution_reader); + decoded_array_schema_evolution = array_schema_evolution_from_capnp( + array_schema_evolution_reader, memory_tracker); break; } default: { @@ -343,7 +346,10 @@ Status array_schema_evolution_serialize( } Status array_schema_evolution_deserialize( - ArraySchemaEvolution**, SerializationType, const Buffer&) { + ArraySchemaEvolution**, + SerializationType, + const Buffer&, + shared_ptr) { return LOG_STATUS(Status_SerializationError( "Cannot serialize; serialization not enabled.")); } diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index c1b1d8d25841..e48ebe37bc85 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -69,7 +69,8 @@ Status array_schema_evolution_serialize( Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, - const Buffer& serialized_buffer); + const Buffer& serialized_buffer, + shared_ptr memory_tracker); } // namespace serialization } // namespace sm From 47e70d82359571792ef68ea0552a6fbe8119ab63 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 1 Mar 2024 02:50:00 -0800 Subject: [PATCH 214/456] Integrate std::prm memory tracking for class `Enumeration`. (#4735) All member vector variables of the Enumeration class have PMR tracking. --- TYPE: NO_HISTORY DESC: Integrate std::prm memory tracking for class `Enumeration`. --------- Co-authored-by: Luc Rancourt --- test/src/unit-enumerations.cc | 135 +++++++++++++----- test/src/unit-request-handlers.cc | 3 +- .../api/c_api/enumeration/enumeration_api.cc | 13 +- .../compile_capi_enumeration_stub_main.cc | 10 +- tiledb/common/memory_tracker.cc | 2 + tiledb/common/memory_tracker.h | 1 + tiledb/sm/array/array.cc | 3 +- tiledb/sm/array/array_directory.cc | 2 +- tiledb/sm/array_schema/enumeration.cc | 18 ++- tiledb/sm/array_schema/enumeration.h | 38 +++-- .../test/compile_enumeration_main.cc | 10 +- tiledb/sm/rest/rest_client.cc | 16 ++- tiledb/sm/rest/rest_client.h | 3 +- tiledb/sm/serialization/array_schema.cc | 3 +- .../serialization/array_schema_evolution.cc | 4 +- .../sm/serialization/array_schema_evolution.h | 11 +- tiledb/sm/serialization/enumeration.cc | 22 +-- tiledb/sm/serialization/enumeration.h | 9 +- 18 files changed, 227 insertions(+), 76 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 6f841fa278b9..b8926f5561c2 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -159,7 +159,15 @@ QueryCondition create_qc( TEST_CASE_METHOD( EnumerationFx, "Create Empty Enumeration", "[enumeration][empty]") { Enumeration::create( - default_enmr_name, Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + 0, + nullptr, + 0, + memory_tracker_); } TEST_CASE_METHOD( @@ -174,7 +182,8 @@ TEST_CASE_METHOD( nullptr, 0, nullptr, - 0); + 0, + memory_tracker_); } TEST_CASE_METHOD( @@ -254,7 +263,9 @@ TEST_CASE_METHOD( nullptr, 0, &offsets, - sizeof(uint64_t)); + sizeof(uint64_t), + memory_tracker_); + std::vector values = {""}; check_enumeration( enmr, @@ -323,7 +334,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 0); + 0, + memory_tracker_); check_enumeration(enmr, default_enmr_name, values, Datatype::INT32, 2, false); } @@ -342,7 +354,8 @@ TEST_CASE_METHOD( nullptr, 10, nullptr, - 0), + 0, + memory_tracker_), matcher); } @@ -355,7 +368,15 @@ TEST_CASE_METHOD( "Invalid data size; must be non-zero for fixed size data."); REQUIRE_THROWS_WITH( Enumeration::create( - default_enmr_name, Datatype::INT32, 1, false, &val, 0, nullptr, 0), + default_enmr_name, + Datatype::INT32, + 1, + false, + &val, + 0, + nullptr, + 0, + memory_tracker_), matcher); } @@ -375,7 +396,8 @@ TEST_CASE_METHOD( val, strlen(val), nullptr, - 8), + 8, + memory_tracker_), matcher); } @@ -396,7 +418,8 @@ TEST_CASE_METHOD( val, strlen(val), &offset, - 0), + 0, + memory_tracker_), matcher); } @@ -417,7 +440,8 @@ TEST_CASE_METHOD( nullptr, 5, &offsets, - sizeof(uint64_t)), + sizeof(uint64_t), + memory_tracker_), matcher); } @@ -438,7 +462,8 @@ TEST_CASE_METHOD( nullptr, 5, &offsets, - sizeof(uint64_t)), + sizeof(uint64_t), + memory_tracker_), matcher); } @@ -460,7 +485,8 @@ TEST_CASE_METHOD( data, 2, &offsets, - sizeof(uint64_t)), + sizeof(uint64_t), + memory_tracker_), matcher); } @@ -477,7 +503,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -493,7 +520,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -510,7 +538,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -526,7 +555,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -542,7 +572,8 @@ TEST_CASE_METHOD( nullptr, values.size() * sizeof(int), nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -558,7 +589,8 @@ TEST_CASE_METHOD( values.data(), 0, nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -575,7 +607,8 @@ TEST_CASE_METHOD( data, strlen(data), nullptr, - offsets.size() * sizeof(uint64_t))); + offsets.size() * sizeof(uint64_t), + memory_tracker_)); } TEST_CASE_METHOD( @@ -592,7 +625,8 @@ TEST_CASE_METHOD( data, strlen(data), offsets.data(), - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -609,7 +643,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), offsets.data(), - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -625,7 +660,8 @@ TEST_CASE_METHOD( values.data(), values.size() * sizeof(int), nullptr, - 100)); + 100, + memory_tracker_)); } TEST_CASE_METHOD( @@ -644,7 +680,8 @@ TEST_CASE_METHOD( data, strlen(data), offsets.data(), - 3)); + 3, + memory_tracker_)); } TEST_CASE_METHOD( @@ -662,7 +699,8 @@ TEST_CASE_METHOD( data, strlen(data), offsets.data(), - offsets.size() * sizeof(uint64_t))); + offsets.size() * sizeof(uint64_t), + memory_tracker_)); } TEST_CASE_METHOD( @@ -680,7 +718,8 @@ TEST_CASE_METHOD( values.data(), 3, nullptr, - 0)); + 0, + memory_tracker_)); } TEST_CASE_METHOD( @@ -750,7 +789,8 @@ TEST_CASE_METHOD( init_values.data(), init_values.size() * sizeof(int), nullptr, - 0); + 0, + memory_tracker_); auto enmr2 = extend_enumeration(enmr1, extend_values); check_enumeration( enmr2, default_enmr_name, final_values, Datatype::INT32, 2, false); @@ -947,7 +987,7 @@ TEST_CASE_METHOD( memset(data, 1, 4); Deserializer deserializer(tile.data(), tile.size()); - REQUIRE_THROWS(Enumeration::deserialize(deserializer)); + REQUIRE_THROWS(Enumeration::deserialize(deserializer, memory_tracker_)); } TEST_CASE_METHOD( @@ -1397,7 +1437,8 @@ TEST_CASE_METHOD( data.data(), data.size(), offsets.data(), - offsets.size() * constants::cell_var_offset_size); + offsets.size() * constants::cell_var_offset_size, + memory_tracker_); schema->add_enumeration(enmr); @@ -1426,7 +1467,8 @@ TEST_CASE_METHOD( data.data(), data.size(), offsets.data(), - offsets.size() * constants::cell_var_offset_size); + offsets.size() * constants::cell_var_offset_size, + memory_tracker_); schema->add_enumeration(enmr); } @@ -1556,7 +1598,8 @@ TEST_CASE_METHOD( enmr2->data().data(), enmr2->data().size(), enmr2->offsets().data(), - enmr2->offsets().size()); + enmr2->offsets().size(), + memory_tracker_); auto matcher = Catch::Matchers::ContainsSubstring( "Enumeration path name for 'test_enmr' already exists in this schema."); @@ -2145,7 +2188,15 @@ TEST_CASE_METHOD( auto schema1 = create_schema(); auto enmr1 = Enumeration::create( - "empty_fixed", Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + "empty_fixed", + Datatype::INT32, + 1, + false, + nullptr, + 0, + nullptr, + 0, + memory_tracker_); auto enmr2 = Enumeration::create( "empty_var", Datatype::STRING_ASCII, @@ -2154,7 +2205,8 @@ TEST_CASE_METHOD( nullptr, 0, nullptr, - 0); + 0, + memory_tracker_); schema1->add_enumeration(enmr1); schema1->add_enumeration(enmr2); @@ -2450,7 +2502,8 @@ shared_ptr EnumerationFx::create_enumeration( raw_values.data(), raw_values.size() * sizeof(uint8_t), nullptr, - 0); + 0, + memory_tracker_); } else if constexpr (std::is_pod_v) { return Enumeration::create( name, @@ -2460,7 +2513,8 @@ shared_ptr EnumerationFx::create_enumeration( values.data(), values.size() * sizeof(T), nullptr, - 0); + 0, + memory_tracker_); } else { uint64_t total_size = 0; for (auto v : values) { @@ -2486,14 +2540,23 @@ shared_ptr EnumerationFx::create_enumeration( data.data(), total_size, offsets.data(), - offsets.size() * sizeof(uint64_t)); + offsets.size() * sizeof(uint64_t), + memory_tracker_); } } shared_ptr EnumerationFx::create_empty_enumeration( Datatype type, uint32_t cell_val_num, bool ordered, std::string name) { return Enumeration::create( - name, type, cell_val_num, ordered, nullptr, 0, nullptr, 0); + name, + type, + cell_val_num, + ordered, + nullptr, + 0, + nullptr, + 0, + memory_tracker_); } template @@ -2567,7 +2630,7 @@ void EnumerationFx::check_storage_deserialization( auto tile = serialize_to_tile(enmr); Deserializer deserializer(tile.data(), tile.size()); - auto deserialized = Enumeration::deserialize(deserializer); + auto deserialized = Enumeration::deserialize(deserializer, memory_tracker_); REQUIRE(deserialized->name() == enmr->name()); REQUIRE(deserialized->path_name().empty() == false); diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 5f134a292a63..287ea1ca949a 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -392,7 +392,8 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( data.data(), total_size, offsets.data(), - offsets.size() * sizeof(uint64_t)); + offsets.size() * sizeof(uint64_t), + tiledb::test::create_test_memory_tracker()); } shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { diff --git a/tiledb/api/c_api/enumeration/enumeration_api.cc b/tiledb/api/c_api/enumeration/enumeration_api.cc index d73759bef31c..1946969fe199 100644 --- a/tiledb/api/c_api/enumeration/enumeration_api.cc +++ b/tiledb/api/c_api/enumeration/enumeration_api.cc @@ -34,10 +34,12 @@ #include "../string/string_api_internal.h" #include "enumeration_api_experimental.h" #include "enumeration_api_internal.h" +#include "tiledb/common/memory_tracker.h" namespace tiledb::api { capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, const char* name, tiledb_datatype_t type, uint32_t cell_val_num, @@ -65,6 +67,9 @@ capi_return_t tiledb_enumeration_alloc( } try { + auto memory_tracker = ctx->context().resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ENUMERATION_CREATE); + *enumeration = tiledb_enumeration_handle_t::make_handle( std::string(name), datatype, @@ -73,7 +78,8 @@ capi_return_t tiledb_enumeration_alloc( data, data_size, offsets, - offsets_size); + offsets_size, + memory_tracker); } catch (...) { *enumeration = nullptr; throw; @@ -179,6 +185,9 @@ capi_return_t tiledb_enumeration_dump( using tiledb::api::api_entry_context; using tiledb::api::api_entry_void; +template +constexpr auto api_entry = tiledb::api::api_entry_with_context; + CAPI_INTERFACE( enumeration_alloc, tiledb_ctx_t* ctx, @@ -191,7 +200,7 @@ CAPI_INTERFACE( const void* offsets, uint64_t offsets_size, tiledb_enumeration_t** enumeration) { - return api_entry_context( + return api_entry( ctx, name, type, diff --git a/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc index 0b7139553f2a..d06771a93436 100644 --- a/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc +++ b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc @@ -32,7 +32,15 @@ int main() { try { tiledb_enumeration_handle_t e{ - "fooo", tiledb::sm::Datatype::INT32, 1, 0, nullptr, 0, nullptr, 0}; + "fooo", + tiledb::sm::Datatype::INT32, + 1, + 0, + nullptr, + 0, + nullptr, + 0, + nullptr}; } catch (...) { } return 0; diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 833d7adac0b0..6e81503d5628 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -52,6 +52,8 @@ std::string memory_type_to_str(MemoryType type) { switch (type) { case MemoryType::ENUMERATION: return "Enumeration"; + case MemoryType::ENUMERATION_PATHS: + return "EnumerationPaths"; case MemoryType::FOOTER: return "Footer"; case MemoryType::GENERIC_TILE_IO: diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index e9f7e5632fe9..bcfc512c4153 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -126,6 +126,7 @@ enum class MemoryTrackerType { ARRAY_LOAD, ARRAY_READ, ARRAY_WRITE, + ENUMERATION_CREATE, FRAGMENT_INFO_LOAD, QUERY_READ, QUERY_WRITE, diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 0a59f975d61c..0698944ec969 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -593,7 +593,8 @@ std::vector> Array::get_enumerations( array_dir_timestamp_start_, array_dir_timestamp_end_, this, - names_to_load); + names_to_load, + memory_tracker_); } else { // Create a vector of paths to be loaded. std::vector paths_to_load; diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 6149f77de186..48c0a65e49e9 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -1340,7 +1340,7 @@ shared_ptr ArrayDirectory::load_enumeration( } Deserializer deserializer(tile->data(), tile->size()); - return Enumeration::deserialize(deserializer); + return Enumeration::deserialize(deserializer, memory_tracker); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index c6b2ab9633f7..dabe1818a3a0 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -33,6 +33,7 @@ #include #include +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/misc/uuid.h" #include "enumeration.h" @@ -56,14 +57,17 @@ Enumeration::Enumeration( const void* data, uint64_t data_size, const void* offsets, - uint64_t offsets_size) - : name_(name) + uint64_t offsets_size, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , name_(name) , path_name_(path_name) , type_(type) , cell_val_num_(cell_val_num) , ordered_(ordered) , data_(data_size) - , offsets_(offsets_size) { + , offsets_(offsets_size) + , value_map_(memory_tracker_->get_resource(MemoryType::ENUMERATION)) { ensure_datatype_is_valid(type); if (name.empty()) { @@ -178,7 +182,7 @@ Enumeration::Enumeration( } shared_ptr Enumeration::deserialize( - Deserializer& deserializer) { + Deserializer& deserializer, shared_ptr memory_tracker) { auto disk_version = deserializer.read(); if (disk_version > constants::enumerations_version) { throw EnumerationException( @@ -224,7 +228,8 @@ shared_ptr Enumeration::deserialize( data, data_size, offsets, - offsets_size); + offsets_size, + memory_tracker); } shared_ptr Enumeration::extend( @@ -304,7 +309,8 @@ shared_ptr Enumeration::extend( new_data.data(), new_data.size(), new_offsets_ptr, - new_offsets_size); + new_offsets_size, + memory_tracker_); } bool Enumeration::is_extension_of(shared_ptr other) const { diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h index 950ca447dc92..3df183d65907 100644 --- a/tiledb/sm/array_schema/enumeration.h +++ b/tiledb/sm/array_schema/enumeration.h @@ -36,6 +36,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/types/untyped_datum.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" @@ -43,6 +44,8 @@ namespace tiledb::sm { +class MemoryTracker; + /** Defines an array enumeration */ class Enumeration { public: @@ -84,6 +87,7 @@ class Enumeration { * offsets buffer. Must be null if cell_var_num is not var_num. * @param offsets_size The size of the buffer pointed to by offsets. Must be * zero of cell_var_num is not var_num. + * @param memory_tracker The memory tracker associated with this Enumeration. * @return shared_ptr The created enumeration. */ static shared_ptr create( @@ -94,7 +98,8 @@ class Enumeration { const void* data, uint64_t data_size, const void* offsets, - uint64_t offsets_size) { + uint64_t offsets_size, + shared_ptr memory_tracker) { return create( name, "", @@ -104,7 +109,8 @@ class Enumeration { data, data_size, offsets, - offsets_size); + offsets_size, + memory_tracker); } /** Create a new Enumeration @@ -122,6 +128,7 @@ class Enumeration { * offsets buffer. Must be null if cell_var_num is not var_num. * @param offsets_size The size of the buffer pointed to by offsets. Must be * zero of cell_var_num is not var_num. + * @param memory_tracker The memory tracker associated with this Enumeration. * @return shared_ptr The created enumeration. */ static shared_ptr create( @@ -133,7 +140,8 @@ class Enumeration { const void* data, uint64_t data_size, const void* offsets, - uint64_t offsets_size) { + uint64_t offsets_size, + shared_ptr memory_tracker) { struct EnableMakeShared : public Enumeration { EnableMakeShared( const std::string& name, @@ -144,7 +152,8 @@ class Enumeration { const void* data, uint64_t data_size, const void* offsets, - uint64_t offsets_size) + uint64_t offsets_size, + shared_ptr memory_tracker) : Enumeration( name, path_name, @@ -154,7 +163,8 @@ class Enumeration { data, data_size, offsets, - offsets_size) { + offsets_size, + memory_tracker) { } }; return make_shared( @@ -167,16 +177,19 @@ class Enumeration { data, data_size, offsets, - offsets_size); + offsets_size, + memory_tracker); } /** * Deserialize an enumeration * * @param deserializer The deserializer to deserialize from. + * @param memory_tracker The memory tracker associated with this Enumeration. * @return A new Enumeration. */ - static shared_ptr deserialize(Deserializer& deserializer); + static shared_ptr deserialize( + Deserializer& deserializer, shared_ptr memory_tracker); /** * Create a new enumeration by extending an existing enumeration's @@ -359,6 +372,7 @@ class Enumeration { * offsets buffer. Must be null if cell_var_num is not var_num. * @param offsets_size The size of the buffer pointed to by offsets. Must be * zero of cell_var_num is not var_num. + * @param memory_tracker The memory tracker. */ Enumeration( const std::string& name, @@ -369,12 +383,18 @@ class Enumeration { const void* data, uint64_t data_size, const void* offsets, - uint64_t offsets_size); + uint64_t offsets_size, + shared_ptr memory_tracker); /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * The memory tracker of the Enumeration. + */ + shared_ptr memory_tracker_; + /** The name of this Enumeration stored in the enumerations directory. */ std::string name_; @@ -397,7 +417,7 @@ class Enumeration { Buffer offsets_; /** Map of values to indices */ - std::unordered_map value_map_; + tdb::pmr::unordered_map value_map_; /* ********************************* */ /* PRIVATE METHODS */ diff --git a/tiledb/sm/array_schema/test/compile_enumeration_main.cc b/tiledb/sm/array_schema/test/compile_enumeration_main.cc index d35fa1cca83c..e8ce6f2ae9c6 100644 --- a/tiledb/sm/array_schema/test/compile_enumeration_main.cc +++ b/tiledb/sm/array_schema/test/compile_enumeration_main.cc @@ -32,7 +32,15 @@ int main(int, char*[]) { try { tiledb::sm::Enumeration::create( - "foo", tiledb::sm::Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + "foo", + tiledb::sm::Datatype::INT32, + 1, + false, + nullptr, + 0, + nullptr, + 0, + nullptr); } catch (...) { } return 0; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 2b9f6f758041..0de78d0d4047 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -646,12 +646,17 @@ RestClient::post_enumerations_from_rest( uint64_t timestamp_start, uint64_t timestamp_end, Array* array, - const std::vector& enumeration_names) { + const std::vector& enumeration_names, + shared_ptr memory_tracker) { if (array == nullptr) { throw Status_RestError( "Error getting enumerations from REST; array is null."); } + if (!memory_tracker) { + memory_tracker = memory_tracker_; + } + // This should never be called with an empty list of enumeration names, but // there's no reason to not check an early return case here given that code // changes. @@ -696,7 +701,7 @@ RestClient::post_enumerations_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); return serialization::deserialize_load_enumerations_response( - serialization_type_, returned_data); + serialization_type_, returned_data, memory_tracker); } Status RestClient::submit_query_to_rest(const URI& uri, Query* query) { @@ -1616,7 +1621,12 @@ Status RestClient::post_array_metadata_to_rest( std::vector> RestClient::post_enumerations_from_rest( - const URI&, uint64_t, uint64_t, Array*, const std::vector&) { + const URI&, + uint64_t, + uint64_t, + Array*, + const std::vector&, + shared_ptr) { throw Status_RestError("Cannot use rest client; serialization not enabled."); } diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 1d9f476f320e..737e671ebc43 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -254,7 +254,8 @@ class RestClient { uint64_t timestamp_start, uint64_t timestamp_end, Array* array, - const std::vector& enumeration_names); + const std::vector& enumeration_names, + shared_ptr memory_tracker = nullptr); /** * Post a data query to rest server diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 40a0418cfbac..41c68be6c154 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -1087,7 +1087,8 @@ shared_ptr array_schema_from_capnp( enumerations.reserve(enmr_readers.size()); try { for (auto&& enmr_reader : enmr_readers) { - enumerations.emplace_back(enumeration_from_capnp(enmr_reader)); + enumerations.emplace_back( + enumeration_from_capnp(enmr_reader, memory_tracker)); } } catch (const std::exception& e) { std::throw_with_nested(std::runtime_error( diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index 66c18a7635be..36a3c329fec9 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -178,7 +178,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( std::unordered_map> enmrs_to_add; auto enmrs_to_add_reader = evolution_reader.getEnumerationsToAdd(); for (auto enmr_reader : enmrs_to_add_reader) { - auto enmr = enumeration_from_capnp(enmr_reader); + auto enmr = enumeration_from_capnp(enmr_reader, memory_tracker); enmrs_to_add[enmr->name()] = enmr; } @@ -187,7 +187,7 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_extend; auto enmrs_to_extend_reader = evolution_reader.getEnumerationsToExtend(); for (auto enmr_reader : enmrs_to_extend_reader) { - auto enmr = enumeration_from_capnp(enmr_reader); + auto enmr = enumeration_from_capnp(enmr_reader, memory_tracker); enmrs_to_extend[enmr->name()] = enmr; } diff --git a/tiledb/sm/serialization/array_schema_evolution.h b/tiledb/sm/serialization/array_schema_evolution.h index e48ebe37bc85..b75d873e5389 100644 --- a/tiledb/sm/serialization/array_schema_evolution.h +++ b/tiledb/sm/serialization/array_schema_evolution.h @@ -47,12 +47,13 @@ class Buffer; class ArraySchema; class ArraySchemaEvolution; class Dimension; +class MemoryTracker; enum class SerializationType : uint8_t; namespace serialization { /** - * Serialize an array schema evolution via Cap'n Prto + * Serialize an array schema evolution via Cap'n Proto * @param array_schema_evolution evolution object to serialize * @param serialize_type format to serialize into Cap'n Proto or JSON * @param serialized_buffer buffer to store serialized bytes in @@ -66,6 +67,14 @@ Status array_schema_evolution_serialize( Buffer* serialized_buffer, const bool client_side); +/** + * Deserialize an array schema evolution via Cap'n Proto + * @param array_schema_evolution pointer to store evolution object in + * @param serialize_type format to serialize into Cap'n Proto or JSON + * @param serialized_buffer buffer where serialized bytes are stored + * @param memory_tracker memory tracker associated with the evolution object + * @return + */ Status array_schema_evolution_deserialize( ArraySchemaEvolution** array_schema_evolution, SerializationType serialize_type, diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 16eb87916636..1df588442c26 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -71,7 +71,8 @@ void enumeration_to_capnp( } shared_ptr enumeration_from_capnp( - const capnp::Enumeration::Reader& reader) { + const capnp::Enumeration::Reader& reader, + shared_ptr memory_tracker) { auto name = reader.getName(); auto path_name = reader.getPathName(); Datatype datatype = Datatype::ANY; @@ -104,7 +105,8 @@ shared_ptr enumeration_from_capnp( data, data_size, offsets, - offsets_size); + offsets_size, + memory_tracker); } void load_enumerations_request_to_capnp( @@ -150,12 +152,13 @@ void load_enumerations_response_to_capnp( std::vector> load_enumerations_response_from_capnp( - const capnp::LoadEnumerationsResponse::Reader& reader) { + const capnp::LoadEnumerationsResponse::Reader& reader, + shared_ptr memory_tracker) { std::vector> ret; if (reader.hasEnumerations()) { auto enmr_readers = reader.getEnumerations(); for (auto enmr_reader : enmr_readers) { - ret.push_back(enumeration_from_capnp(enmr_reader)); + ret.push_back(enumeration_from_capnp(enmr_reader, memory_tracker)); } } return ret; @@ -306,7 +309,9 @@ void serialize_load_enumerations_response( std::vector> deserialize_load_enumerations_response( - SerializationType serialize_type, const Buffer& response) { + SerializationType serialize_type, + const Buffer& response, + shared_ptr memory_tracker) { try { switch (serialize_type) { case SerializationType::JSON: { @@ -317,7 +322,7 @@ deserialize_load_enumerations_response( json.decode( kj::StringPtr(static_cast(response.data())), builder); capnp::LoadEnumerationsResponse::Reader reader = builder.asReader(); - return load_enumerations_response_from_capnp(reader); + return load_enumerations_response_from_capnp(reader, memory_tracker); } case SerializationType::CAPNP: { const auto mBytes = reinterpret_cast(response.data()); @@ -326,7 +331,7 @@ deserialize_load_enumerations_response( response.size() / sizeof(::capnp::word))); capnp::LoadEnumerationsResponse::Reader reader = array_reader.getRoot(); - return load_enumerations_response_from_capnp(reader); + return load_enumerations_response_from_capnp(reader, memory_tracker); } default: { throw Status_SerializationError( @@ -371,7 +376,8 @@ void serialize_load_enumerations_response( } std::vector> -deserialize_load_enumerations_response(SerializationType, const Buffer&) { +deserialize_load_enumerations_response( + SerializationType, const Buffer&, shared_ptr) { throw Status_SerializationError( "Cannot serialize; serialization not enabled."); } diff --git a/tiledb/sm/serialization/enumeration.h b/tiledb/sm/serialization/enumeration.h index 85e5faaaf7a3..c92129f973cc 100644 --- a/tiledb/sm/serialization/enumeration.h +++ b/tiledb/sm/serialization/enumeration.h @@ -64,10 +64,13 @@ void enumeration_to_capnp( * Deserialize an enumeration from a cap'n proto object * * @param reader Cap'n proto reader object + * @param memory_tracker The memory tracker associated with the Enumeration + * object. * @return A new Enumeration */ shared_ptr enumeration_from_capnp( - const capnp::Enumeration::Reader& reader); + const capnp::Enumeration::Reader& reader, + shared_ptr memory_tracker); #endif @@ -87,7 +90,9 @@ void serialize_load_enumerations_response( std::vector> deserialize_load_enumerations_response( - SerializationType serialization_type, const Buffer& response); + SerializationType serialization_type, + const Buffer& response, + shared_ptr memory_tracker); } // namespace serialization } // namespace tiledb::sm From 916579576d896a330f5138f79c432f226e31cd16 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Sat, 2 Mar 2024 04:33:39 -0500 Subject: [PATCH 215/456] Remove Status from datatype_enum. (#4780) Removes Status from datatype_enum, returning `Datatype` instead. This also relocates `Subarray::LabelRangeSubset` to be public for use in #4685. --- TYPE: NO_HISTORY DESC: Remove Status from datatype_enum. --- tiledb/api/c_api/datatype/datatype_api.cc | 6 +- tiledb/sm/enums/datatype.h | 102 ++++++++++------------ tiledb/sm/serialization/array.cc | 3 +- tiledb/sm/serialization/array_schema.cc | 9 +- tiledb/sm/serialization/enumeration.cc | 3 +- tiledb/sm/serialization/query.cc | 2 - tiledb/sm/subarray/subarray.h | 94 +++++++++++--------- 7 files changed, 108 insertions(+), 111 deletions(-) diff --git a/tiledb/api/c_api/datatype/datatype_api.cc b/tiledb/api/c_api/datatype/datatype_api.cc index 854b49fe2a56..fd19e9b0e7e1 100644 --- a/tiledb/api/c_api/datatype/datatype_api.cc +++ b/tiledb/api/c_api/datatype/datatype_api.cc @@ -45,11 +45,7 @@ capi_return_t tiledb_datatype_to_str( capi_return_t tiledb_datatype_from_str( const char* str, tiledb_datatype_t* datatype) { - tiledb::sm::Datatype val = tiledb::sm::Datatype::UINT8; - if (!tiledb::sm::datatype_enum(str, &val).ok()) { - return TILEDB_ERR; - } - *datatype = (tiledb_datatype_t)val; + *datatype = (tiledb_datatype_t)tiledb::sm::datatype_enum(str); return TILEDB_OK; } diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index 8c21280b267b..7b84f00f18e9 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -235,100 +235,99 @@ inline const std::string& datatype_str(Datatype type) { } /** Returns the datatype given a string representation. */ -inline Status datatype_enum( - const std::string& datatype_str, Datatype* datatype) { +inline Datatype datatype_enum(const std::string& datatype_str) { if (datatype_str == constants::int32_str) - *datatype = Datatype::INT32; + return Datatype::INT32; else if (datatype_str == constants::int64_str) - *datatype = Datatype::INT64; + return Datatype::INT64; else if (datatype_str == constants::float32_str) - *datatype = Datatype::FLOAT32; + return Datatype::FLOAT32; else if (datatype_str == constants::float64_str) - *datatype = Datatype::FLOAT64; + return Datatype::FLOAT64; else if (datatype_str == constants::char_str) - *datatype = Datatype::CHAR; + return Datatype::CHAR; else if (datatype_str == constants::blob_str) - *datatype = Datatype::BLOB; + return Datatype::BLOB; else if (datatype_str == constants::geom_wkb_str) - *datatype = Datatype::GEOM_WKB; + return Datatype::GEOM_WKB; else if (datatype_str == constants::geom_wkt_str) - *datatype = Datatype::GEOM_WKT; + return Datatype::GEOM_WKT; else if (datatype_str == constants::bool_str) - *datatype = Datatype::BOOL; + return Datatype::BOOL; else if (datatype_str == constants::int8_str) - *datatype = Datatype::INT8; + return Datatype::INT8; else if (datatype_str == constants::uint8_str) - *datatype = Datatype::UINT8; + return Datatype::UINT8; else if (datatype_str == constants::int16_str) - *datatype = Datatype::INT16; + return Datatype::INT16; else if (datatype_str == constants::uint16_str) - *datatype = Datatype::UINT16; + return Datatype::UINT16; else if (datatype_str == constants::uint32_str) - *datatype = Datatype::UINT32; + return Datatype::UINT32; else if (datatype_str == constants::uint64_str) - *datatype = Datatype::UINT64; + return Datatype::UINT64; else if (datatype_str == constants::string_ascii_str) - *datatype = Datatype::STRING_ASCII; + return Datatype::STRING_ASCII; else if (datatype_str == constants::string_utf8_str) - *datatype = Datatype::STRING_UTF8; + return Datatype::STRING_UTF8; else if (datatype_str == constants::string_utf16_str) - *datatype = Datatype::STRING_UTF16; + return Datatype::STRING_UTF16; else if (datatype_str == constants::string_utf32_str) - *datatype = Datatype::STRING_UTF32; + return Datatype::STRING_UTF32; else if (datatype_str == constants::string_ucs2_str) - *datatype = Datatype::STRING_UCS2; + return Datatype::STRING_UCS2; else if (datatype_str == constants::string_ucs4_str) - *datatype = Datatype::STRING_UCS4; + return Datatype::STRING_UCS4; else if (datatype_str == constants::any_str) - *datatype = Datatype::ANY; + return Datatype::ANY; else if (datatype_str == constants::datetime_year_str) - *datatype = Datatype::DATETIME_YEAR; + return Datatype::DATETIME_YEAR; else if (datatype_str == constants::datetime_month_str) - *datatype = Datatype::DATETIME_MONTH; + return Datatype::DATETIME_MONTH; else if (datatype_str == constants::datetime_week_str) - *datatype = Datatype::DATETIME_WEEK; + return Datatype::DATETIME_WEEK; else if (datatype_str == constants::datetime_day_str) - *datatype = Datatype::DATETIME_DAY; + return Datatype::DATETIME_DAY; else if (datatype_str == constants::datetime_hr_str) - *datatype = Datatype::DATETIME_HR; + return Datatype::DATETIME_HR; else if (datatype_str == constants::datetime_min_str) - *datatype = Datatype::DATETIME_MIN; + return Datatype::DATETIME_MIN; else if (datatype_str == constants::datetime_sec_str) - *datatype = Datatype::DATETIME_SEC; + return Datatype::DATETIME_SEC; else if (datatype_str == constants::datetime_ms_str) - *datatype = Datatype::DATETIME_MS; + return Datatype::DATETIME_MS; else if (datatype_str == constants::datetime_us_str) - *datatype = Datatype::DATETIME_US; + return Datatype::DATETIME_US; else if (datatype_str == constants::datetime_ns_str) - *datatype = Datatype::DATETIME_NS; + return Datatype::DATETIME_NS; else if (datatype_str == constants::datetime_ps_str) - *datatype = Datatype::DATETIME_PS; + return Datatype::DATETIME_PS; else if (datatype_str == constants::datetime_fs_str) - *datatype = Datatype::DATETIME_FS; + return Datatype::DATETIME_FS; else if (datatype_str == constants::datetime_as_str) - *datatype = Datatype::DATETIME_AS; + return Datatype::DATETIME_AS; else if (datatype_str == constants::time_hr_str) - *datatype = Datatype::TIME_HR; + return Datatype::TIME_HR; else if (datatype_str == constants::time_min_str) - *datatype = Datatype::TIME_MIN; + return Datatype::TIME_MIN; else if (datatype_str == constants::time_sec_str) - *datatype = Datatype::TIME_SEC; + return Datatype::TIME_SEC; else if (datatype_str == constants::time_ms_str) - *datatype = Datatype::TIME_MS; + return Datatype::TIME_MS; else if (datatype_str == constants::time_us_str) - *datatype = Datatype::TIME_US; + return Datatype::TIME_US; else if (datatype_str == constants::time_ns_str) - *datatype = Datatype::TIME_NS; + return Datatype::TIME_NS; else if (datatype_str == constants::time_ps_str) - *datatype = Datatype::TIME_PS; + return Datatype::TIME_PS; else if (datatype_str == constants::time_fs_str) - *datatype = Datatype::TIME_FS; + return Datatype::TIME_FS; else if (datatype_str == constants::time_as_str) - *datatype = Datatype::TIME_AS; + return Datatype::TIME_AS; else { - return Status_Error("Invalid Datatype " + datatype_str); + throw std::runtime_error( + "Invalid Datatype string (\"" + datatype_str + "\")"); } - return Status::Ok(); } /** Returns true if the input datatype is a string type. */ @@ -440,12 +439,7 @@ inline void ensure_datatype_is_valid(Datatype type) { * the datatype string's enum is not between 0 and 43. **/ inline void ensure_datatype_is_valid(const std::string& datatype_str) { - Datatype datatype_type; - Status st{datatype_enum(datatype_str, &datatype_type)}; - if (!st.ok()) { - throw std::runtime_error( - "Invalid Datatype string (\"" + datatype_str + "\")"); - } + Datatype datatype_type = datatype_enum(datatype_str); ensure_datatype_is_valid(datatype_type); } diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index a010cd39ce4a..e74b17ae5afe 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -102,8 +102,7 @@ Status metadata_from_capnp( auto entry_reader = entries_reader[i]; auto key = std::string{std::string_view{ entry_reader.getKey().cStr(), entry_reader.getKey().size()}}; - Datatype type = Datatype::UINT8; - RETURN_NOT_OK(datatype_enum(entry_reader.getType(), &type)); + Datatype type = datatype_enum(entry_reader.getType()); uint32_t value_num = entry_reader.getValueNum(); auto value_ptr = entry_reader.getValue(); diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 41c68be6c154..cdeced731c0b 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -400,8 +400,7 @@ void attribute_to_capnp( shared_ptr attribute_from_capnp( const capnp::Attribute::Reader& attribute_reader) { // Get datatype - Datatype datatype = Datatype::ANY; - throw_if_not_ok(datatype_enum(attribute_reader.getType(), &datatype)); + Datatype datatype = datatype_enum(attribute_reader.getType()); // Set nullable const bool nullable = attribute_reader.getNullable(); @@ -614,8 +613,7 @@ shared_ptr dimension_from_capnp( Status st; // Deserialize datatype - Datatype dim_type; - st = datatype_enum(dimension_reader.getType().cStr(), &dim_type); + Datatype dim_type = datatype_enum(dimension_reader.getType().cStr()); if (!st.ok()) { throw std::runtime_error( "[Deserialization::dimension_from_capnp] " + @@ -783,8 +781,7 @@ shared_ptr dimension_label_from_capnp( const capnp::DimensionLabel::Reader& dim_label_reader, shared_ptr memory_tracker) { // Get datatype - Datatype datatype = Datatype::ANY; - throw_if_not_ok(datatype_enum(dim_label_reader.getType(), &datatype)); + Datatype datatype = datatype_enum(dim_label_reader.getType()); shared_ptr schema{nullptr}; if (dim_label_reader.hasSchema()) { diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 1df588442c26..197a2d8834d5 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -75,8 +75,7 @@ shared_ptr enumeration_from_capnp( shared_ptr memory_tracker) { auto name = reader.getName(); auto path_name = reader.getPathName(); - Datatype datatype = Datatype::ANY; - throw_if_not_ok(datatype_enum(reader.getType(), &datatype)); + Datatype datatype = datatype_enum(reader.getType()); const void* data = nullptr; uint64_t data_size = 0; diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 01755a141d9a..e3b0cd7c95ca 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -268,8 +268,6 @@ Status subarray_from_capnp( uint32_t dim_num = ranges_reader.size(); for (uint32_t i = 0; i < dim_num; i++) { auto range_reader = ranges_reader[i]; - Datatype type = Datatype::UINT8; - RETURN_NOT_OK(datatype_enum(range_reader.getType(), &type)); auto data = range_reader.getBuffer(); auto data_ptr = data.asBytes(); diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 04d0af954fed..b32a6af0ceaa 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -205,6 +205,60 @@ class Subarray { uint64_t size_validity_; }; + /** + * Wrapper for optional> for + * cleaner data access. + */ + struct LabelRangeSubset { + public: + /** + * Default constructor is not C.41. + **/ + LabelRangeSubset() = delete; + + /** + * Constructor + * + * @param ref Dimension label the ranges will be set on. + * @param coalesce_ranges Set if ranges should be combined when adjacent. + */ + LabelRangeSubset(const DimensionLabel& ref, bool coalesce_ranges = true); + + /** + * Constructor + * + * @param name The name of the dimension label the ranges will be set on. + * @param type The type of the label the ranges will be set on. + * @param coalesce_ranges Set if ranges should be combined when adjacent. + */ + LabelRangeSubset( + const std::string& name, Datatype type, bool coalesce_ranges = true); + + /** + * Constructor + * + * @param name The name of the dimension label the ranges will be set on. + * @param type The type of the label the ranges will be set on. + * @param ranges The range subset for the dimension label. + * @param coalesce_ranges Set if ranges should be combined when adjacent. + */ + LabelRangeSubset( + const std::string& name, + Datatype type, + std::vector ranges, + bool coalesce_ranges = true); + + inline const std::vector& get_ranges() const { + return ranges_.ranges(); + } + + /** Name of the dimension label. */ + std::string name_; + + /** The ranges set on the dimension label. */ + RangeSetAndSuperset ranges_; + }; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -1351,46 +1405,6 @@ class Subarray { uint64_t range_len_; }; - /** - * Wrapper for optional> for - * cleaner data access. - */ - struct LabelRangeSubset { - public: - /** - * Default constructor is not C.41. - **/ - LabelRangeSubset() = delete; - - /** - * Constructor - * - * @param ref Dimension label the ranges will be set on. - * @param coalesce_ranges Set if ranges should be combined when adjacent. - */ - LabelRangeSubset(const DimensionLabel& ref, bool coalesce_ranges = true); - - /** - * Constructor - * - * @param name The name of the dimension label the ranges will be set on. - * @param type The type of the label the ranges will be set on. - * @param coalesce_ranges Set if ranges should be combined when adjacent. - */ - LabelRangeSubset( - const std::string& name, Datatype type, bool coalesce_ranges = true); - - inline const std::vector& get_ranges() const { - return ranges_.ranges(); - } - - /** Name of the dimension label. */ - std::string name_; - - /** The ranges set on the dimension label. */ - RangeSetAndSuperset ranges_; - }; - /** * A hash function capable of hashing std::vector for use by * the tile_coords_map_ unordered_map for caching coords indices. From 619f3ae832f71a0964aafed8b5b7bb6884d6ff93 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Sat, 2 Mar 2024 05:41:22 -0500 Subject: [PATCH 216/456] Update crop_range to clamp to domain range. (#4781) While working on #4685 we found it was possible for a cropped range to fall outside of the given domain range. This updates crop_range to call `std::clamp` instead of min/max to ensure the resulting cropped range is within the domain. --- TYPE: BUG DESC: Update crop_range to clamp to domain range. --- tiledb/sm/array_schema/dimension.cc | 26 -------------------- tiledb/sm/array_schema/dimension.h | 24 +----------------- tiledb/sm/array_schema/domain.cc | 16 ++++++++++-- tiledb/type/range/range.h | 5 ++-- tiledb/type/range/test/unit_crop_range.cc | 30 +++++++++++++++++++++++ 5 files changed, 48 insertions(+), 53 deletions(-) diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 947aaaca2476..ea54b069e145 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -71,7 +71,6 @@ Dimension::Dimension(const std::string& name, Datatype type) set_ceil_to_tile_func(); set_coincides_with_tiles_func(); set_compute_mbr_func(); - set_crop_range_func(); set_domain_range_func(); set_expand_range_func(); set_expand_range_v_func(); @@ -107,7 +106,6 @@ Dimension::Dimension( set_ceil_to_tile_func(); set_coincides_with_tiles_func(); set_compute_mbr_func(); - set_crop_range_func(); set_domain_range_func(); set_expand_range_func(); set_expand_range_v_func(); @@ -355,21 +353,6 @@ Range Dimension::compute_mbr_var( return compute_mbr_var_func_(tile_off, tile_val); } -template -void Dimension::crop_range(const Dimension* dim, Range* range) { - assert(dim != nullptr); - assert(!range->empty()); - auto dim_dom = (const T*)dim->domain().data(); - auto r = (const T*)range->data(); - T res[2] = {std::max(r[0], dim_dom[0]), std::min(r[1], dim_dom[1])}; - range->set_range(res, sizeof(res)); -} - -void Dimension::crop_range(Range* range) const { - assert(crop_range_func_ != nullptr); - crop_range_func_(this, range); -} - template uint64_t Dimension::domain_range(const Range& range) { assert(!range.empty()); @@ -1591,15 +1574,6 @@ std::string Dimension::tile_extent_str() const { return apply_with_type(g, type_); } -void Dimension::set_crop_range_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBNumeric) { - crop_range_func_ = crop_range; - } - }; - apply_with_type(g, type_); -} - void Dimension::set_domain_range_func() { auto g = [&](auto T) { if constexpr (tiledb::type::TileDBFundamental) { diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 0e2a0014e377..6ca7f85f0336 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -487,19 +487,6 @@ class Dimension { static Range compute_mbr_var( const WriterTile& tile_off, const WriterTile& tile_val); - /** - * Crops the input 1D range such that it does not exceed the - * dimension domain. - */ - void crop_range(Range* range) const; - - /** - * Crops the input 1D range such that it does not exceed the - * dimension domain. - */ - template - static void crop_range(const Dimension* dim, Range* range); - /** * Returns the domain range (high - low + 1) of the input * 1D range. It returns 0 in case the dimension datatype @@ -818,13 +805,7 @@ class Dimension { compute_mbr_var_func_; /** - * Stores the appropriate templated crop_range() function based on the - * dimension datatype. - */ - std::function crop_range_func_; - - /** - * Stores the appropriate templated crop_range() function based on the + * Stores the appropriate templated domain_range() function based on the * dimension datatype. */ std::function domain_range_func_; @@ -1032,9 +1013,6 @@ class Dimension { /** Sets the templated compute_mbr() function. */ void set_compute_mbr_func(); - /** Sets the templated crop_range() function. */ - void set_crop_range_func(); - /** Sets the templated domain_range() function. */ void set_domain_range_func(); diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 139cca4ecac2..05b85cb385ee 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -41,6 +41,7 @@ #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/misc/tdb_math.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/type/apply_with_type.h" #include "tiledb/type/range/range.h" #include @@ -312,8 +313,19 @@ int Domain::cell_order_cmp( } void Domain::crop_ndrange(NDRange* ndrange) const { - for (unsigned d = 0; d < dim_num_; ++d) - dimension_ptrs_[d]->crop_range(&(*ndrange)[d]); + for (unsigned d = 0; d < dim_num_; ++d) { + auto type = dimension_ptrs_[d]->type(); + auto g = [&](auto T) { + if constexpr (tiledb::type::TileDBIntegral) { + tiledb::type::crop_range( + dimension_ptrs_[d]->domain(), (*ndrange)[d]); + } else { + throw std::invalid_argument( + "Unsupported dimension datatype " + datatype_str(type)); + } + }; + apply_with_type(g, type); + } } shared_ptr Domain::deserialize( diff --git a/tiledb/type/range/range.h b/tiledb/type/range/range.h index 7009283094a4..c3011983a764 100644 --- a/tiledb/type/range/range.h +++ b/tiledb/type/range/range.h @@ -38,6 +38,7 @@ #include "tiledb/common/tag.h" #include "tiledb/sm/enums/datatype.h" +#include #include #include #include @@ -461,8 +462,8 @@ template < void crop_range(const Range& bounds, Range& range) { auto bounds_data = (const T*)bounds.data(); auto range_data = (T*)range.data(); - range_data[0] = std::max(bounds_data[0], range_data[0]); - range_data[1] = std::min(bounds_data[1], range_data[1]); + range_data[0] = std::clamp(range_data[0], bounds_data[0], bounds_data[1]); + range_data[1] = std::clamp(range_data[1], bounds_data[0], bounds_data[1]); }; /** diff --git a/tiledb/type/range/test/unit_crop_range.cc b/tiledb/type/range/test/unit_crop_range.cc index 46dfcf055b55..8234041d29ce 100644 --- a/tiledb/type/range/test/unit_crop_range.cc +++ b/tiledb/type/range/test/unit_crop_range.cc @@ -89,6 +89,16 @@ TEMPLATE_TEST_CASE( std::numeric_limits::max()}; test_crop_range(bounds, range, bounds); } + SECTION("Test crop outside lower bound") { + TestType range[2]{0, 0}; + TestType result[2]{1, 1}; + test_crop_range(bounds, range, result); + } + SECTION("Test crop outside upper bound") { + TestType range[2]{5, 6}; + TestType result[2]{4, 4}; + test_crop_range(bounds, range, result); + } } TEMPLATE_TEST_CASE( @@ -126,6 +136,16 @@ TEMPLATE_TEST_CASE( std::numeric_limits::max()}; test_crop_range(bounds, range, bounds); } + SECTION("Test crop outside lower bound") { + TestType range[2]{-6, -4}; + TestType result[2]{-2, -2}; + test_crop_range(bounds, range, result); + } + SECTION("Test crop outside upper bound") { + TestType range[2]{5, 6}; + TestType result[2]{2, 2}; + test_crop_range(bounds, range, result); + } } TEMPLATE_TEST_CASE( @@ -164,4 +184,14 @@ TEMPLATE_TEST_CASE( std::numeric_limits::infinity()}; test_crop_range(bounds, range, bounds); } + SECTION("Test crop outside lower bound") { + TestType range[2]{-60.1f, -40.3f}; + TestType result[2]{-10.5f, -10.5f}; + test_crop_range(bounds, range, result); + } + SECTION("Test crop outside upper bound") { + TestType range[2]{5.1f, 6.5f}; + TestType result[2]{3.33f, 3.33f}; + test_crop_range(bounds, range, result); + } } From 79030b1ee2f134798c2bace30a05f79c5c2e1897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Mon, 4 Mar 2024 14:03:08 +0100 Subject: [PATCH 217/456] Single release CMake file that contains all required links/hashes. (#4631) This PR aims to autogenerate cmake file containing all required hash/url combinations so that other repositories can simply pull this file and forget about changing all url/hash occurences in their code. The output assets look like this now: https://github.com/dudoslav/TileDB/releases/tag/t01 Please let me know if the produced CMake file is ok, or should be changed to a different format. --- TYPE: BUILD DESC: Single release CMake file that contains all required links/hashes. --- .github/workflows/append-release-cmake.yml | 80 +++++++++++++ .github/workflows/release.yml | 8 +- cmake/inputs/DownloadPrebuiltTileDB.cmake | 128 +++++++++++++++++++++ 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/append-release-cmake.yml create mode 100644 cmake/inputs/DownloadPrebuiltTileDB.cmake diff --git a/.github/workflows/append-release-cmake.yml b/.github/workflows/append-release-cmake.yml new file mode 100644 index 000000000000..91a4e26e1c29 --- /dev/null +++ b/.github/workflows/append-release-cmake.yml @@ -0,0 +1,80 @@ +name: Append Release CMake + +on: + workflow_dispatch: + inputs: + ref: + description: 'Ref to be used as release' + default: 'latest' + required: true + type: string + workflow_call: + inputs: + ref: + description: 'Ref to be used as release' + default: 'latest' + required: true + type: string + +jobs: + generate_cmake_files: + runs-on: ubuntu-latest + steps: + - name: Checkout TileDB + uses: actions/checkout@v3 + + - name: Make release and output directories + run: | + mkdir release output + + - name: Github release data + id: release_data + uses: KevinRohn/github-full-release-data@v2.0.4 + with: + # repository: 'TileDB-Inc/TileDB' + version: ${{ inputs.ref }} + asset-file: '*.zip,*.tar.gz' + asset-output: './release/' + + - name: Render template + run: | + PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-([^-]+).(tar.gz|zip)$" + RELLIST="output/releases.csv" + MODULE="output/DownloadPrebuiltTileDB.cmake" + cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE + echo "platform,url,sha256" > $RELLIST + + for FILE in $(ls release) + do + if [[ $FILE =~ $PATTERN ]] + then + OS=${BASH_REMATCH[1]^^} + ARCH=${BASH_REMATCH[2]^^} + NOAVX2=${BASH_REMATCH[3]^^} + PLATFORM=${OS}-${ARCH}${NOAVX2} + + URL="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$FILE" + HASH=$(cat release/$FILE.sha256 | cut -d \t -f 1) + + echo "${PLATFORM},${URL},${HASH}" >> $RELLIST + fi + done + + SOURCE_FILE_NAME=$(ls release/tiledb-source-*.tar.gz) + URL_TILEDB_SOURCE="${{ github.server_url }}/${{ github.repository }}/releases/download/${{ inputs.ref }}/$(basename $SOURCE_FILE_NAME)" + HASH_TILEDB_SOURCE=$(cat $SOURCE_FILE_NAME.sha256 | cut -d \t -f 1) + + echo "source,${URL_TILEDB_SOURCE},${HASH_TILEDB_SOURCE}" >> $RELLIST + + HASH=$(sha256sum $RELLIST | cut -d " " -f 1) + echo $HASH > $RELLIST.sha256 + + cat $RELLIST + + - name: Upload template to release + uses: svenstaro/upload-release-action@v2 + with: + file: output/* + tag: ${{ steps.release_data.outputs.tag_name }} + overwrite: true + file_glob: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bd5559d4469e..8584e0d18709 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,6 @@ on: - 'release-*' tags: - '*' - workflow_dispatch: jobs: Package-Source-Release: @@ -175,6 +174,13 @@ jobs: }); } + Generate-Release-List: + needs: + - Publish-Release + uses: ./.github/workflows/append-release-cmake.yml + with: + ref: ${{ github.ref_name }} + Create-Issue-On-Fail: permissions: issues: write diff --git a/cmake/inputs/DownloadPrebuiltTileDB.cmake b/cmake/inputs/DownloadPrebuiltTileDB.cmake new file mode 100644 index 000000000000..608478abc343 --- /dev/null +++ b/cmake/inputs/DownloadPrebuiltTileDB.cmake @@ -0,0 +1,128 @@ +# +# FindTileDB_EP.cmake +# +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +include(FetchContent) + +function(fetch_tiledb_release_list VERSION EXPECTED_HASH) + # Local constants + set(UPSTREAM_URL "https://github.com/TileDB-Inc/TileDB/releases/download") + + if(NOT VERSION) + set(VERSION latest) + endif() + + if(${EXPECTED_HASH}) + file(DOWNLOAD + ${UPSTREAM_URL}/${VERSION}/releases.csv + releases.csv + SHOW_PROGRESS + EXPECTED_HASH ${EXPECTED_HASH} + ) + else() + message(WARNING "Downloading release list without SHA checksum!") + file(DOWNLOAD + ${UPSTREAM_URL}/${VERSION}/releases.csv + releases.csv + SHOW_PROGRESS + ) + endif() + + file(STRINGS + ${CMAKE_CURRENT_BINARY_DIR}/releases.csv + RELLIST + ) + + # Remove csv table headers + list(POP_FRONT RELLIST) + + foreach(LINE ${RELLIST}) + string(REPLACE "," ";" LINE ${LINE}) + list(LENGTH LINE LENGTH) + + list(GET LINE 0 PLATFORM) + list(GET LINE 1 URL) + list(GET LINE 2 SHA) + + set(RELEASE_VAR TILEDB_${PLATFORM}) + set(URL_${RELEASE_VAR} ${URL} PARENT_SCOPE) + set(HASH_${RELEASE_VAR} ${SHA} PARENT_SCOPE) + endforeach() +endfunction() + +function(detect_artifact_name OUT_VAR) + if (WIN32) # Windows + SET(${OUT_VAR} TILEDB_WINDOWS-X86_64 PARENT_SCOPE) + elseif(APPLE) # OSX + if (DEFINED CMAKE_OSX_ARCHITECTURES) + set(ACTUAL_TARGET ${CMAKE_OSX_ARCHITECTURES}) + else() + set(ACTUAL_TARGET ${CMAKE_SYSTEM_PROCESSOR}) + endif() + + + if (ACTUAL_TARGET MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") + SET(${OUT_VAR} TILEDB_MACOS-X86_64 PARENT_SCOPE) + elseif (ACTUAL_TARGET STREQUAL arm64 OR ACTUAL_TARGET MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + SET(${OUT_VAR} TILEDB_MACOS-ARM64 PARENT_SCOPE) + endif() + else() # Linux + SET(${OUT_VAR} TILEDB_LINUX-X86_64 PARENT_SCOPE) + endif() +endfunction() + +function(fetch_prebuilt_tiledb) + # Arguments + set(options RELLIST_HASH) + set(oneValueArgs VERSION ARTIFACT_NAME) + set(multiValueArgs) + cmake_parse_arguments( + FETCH_PREBUILT_TILEDB + "${options}" + "${oneValueArgs}" + "${multiValueArgs}" + ${ARGN} + ) + + fetch_tiledb_release_list(${FETCH_PREBUILT_TILEDB_VERSION} ${FETCH_PREBUILT_TILEDB_RELLIST_HASH}) + + if(NOT FETCH_PREBUILT_TILEDB_ARTIFACT_NAME) + detect_artifact_name(FETCH_PREBUILT_TILEDB_ARTIFACT_NAME) + endif() + + string(STRIP ${HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}) + FetchContent_Declare( + tiledb-prebuilt + URL ${URL_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} + URL_HASH SHA256=${HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} + DOWNLOAD_EXTRACT_TIMESTAMP FALSE + ) + + FetchContent_MakeAvailable( + tiledb-prebuilt + ) + + set(TileDB_DIR "${tiledb-prebuilt_SOURCE_DIR}/lib/cmake/TileDB" PARENT_SCOPE) +endfunction() From c76772426954b8e64c29303696a3f3eea849c40e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 4 Mar 2024 19:32:34 +0200 Subject: [PATCH 218/456] Move the GCS client header files to `gcs.cc`. (#4777) This PR moves including `google/cloud/storage/client.h` from `gcs.h` to the `gcs.cc` implementation file. Due to the current structure of the code, `gcs.h` is included by `vfs.h`, which is included by many files, resulting in the GCS SDK code being needlessly compiled. On my machine, the time to do a clean build with 8 CPU cores and only GCS enabled dropped from 104.22 to 75.4 seconds, a 27.6% improvement. This PR did the minimal necessary changes to the GCS VFS. Other VFSes that are susceptible to leaking headers are HDFS, whose header is relatively small, and S3, that requires more extensive moving of code. --- TYPE: BUILD DESC: Improve build performance when GCS is enabled. --- tiledb/sm/filesystem/gcs.cc | 21 ++++++++++++++------- tiledb/sm/filesystem/gcs.h | 19 ++++++++----------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 59ef5e87065d..b8b145f7c934 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -35,6 +35,17 @@ #include #include +#if defined(_MSC_VER) +#pragma warning(push) +// One abseil file has a warning that fails on Windows when compiling with +// warnings as errors. +#pragma warning(disable : 4127) // conditional expression is constant +#endif +#include +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/logger.h" @@ -171,15 +182,11 @@ Status GCS::init_client() const { if (!endpoint_.empty()) { client_options.set_endpoint(endpoint_); } - auto client = google::cloud::storage::Client( + client_ = tdb_unique_ptr(tdb_new( + google::cloud::storage::Client, client_options, google::cloud::storage::LimitedTimeRetryPolicy( - std::chrono::milliseconds(request_timeout_ms_))); - client_ = google::cloud::StatusOr(client); - if (!client_) { - return LOG_STATUS(Status_GCSError( - "Failed to initialize GCS Client; " + client_.status().message())); - } + std::chrono::milliseconds(request_timeout_ms_)))); } catch (const std::exception& e) { return LOG_STATUS( Status_GCSError("Failed to initialize GCS: " + std::string(e.what()))); diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index 9d57abecb25a..30cdc4268a44 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -35,16 +35,7 @@ #ifdef HAVE_GCS -#if defined(_MSC_VER) -#pragma warning(push) -// One abseil file has a warning that fails on Windows when compiling with -// warnings as errors. -#pragma warning(disable : 4127) // conditional expression is constant -#endif -#include -#if defined(_MSC_VER) -#pragma warning(pop) -#endif +#include #include "tiledb/common/rwlock.h" #include "tiledb/common/status.h" @@ -58,6 +49,12 @@ using namespace tiledb::common; +namespace google::cloud::storage { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +class Client; +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace google::cloud::storage + namespace tiledb { namespace common::filesystem { @@ -430,7 +427,7 @@ class GCS { std::string project_id_; // The GCS REST client. - mutable google::cloud::StatusOr client_; + mutable tdb_unique_ptr client_; /** Maps a object URI to an write cache buffer. */ std::unordered_map write_cache_map_; From ab30c444b1bc9432da11541219369b9a6c23fb3c Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Mon, 4 Mar 2024 17:40:05 -0600 Subject: [PATCH 219/456] Remove QueryCondition requirement for Enumeration values to be valid. (#4707) This change removes the requirement that values used in comparisons against enumerated attributes exist as an value of an enumeration. That is to say, previously if your enumerations were a list of colors, and you attempted to create a query condition of `value = "something_not_a_color"` TileDB would raise an exception saying that `"something_not_a_color"` was not a valid color value. Removing that requirement also means that we have to update query condition logic to handle various situations when a condition will never match or always match. In order to handle that logic, this PR adds to new QueryCondition "operators" `ALWAYS_TRUE` and `ALWAYS_FALSE`. These operators only exist internally and can't be referenced or created outside of the C API boundary. These operations are inserted while a query is rewritten which only happens internally to a query execution. When using a query condition with a value that is not a member of a enumeration there are three distinct logical situations. For every binary operator except `!=` the result is `ALWAYS_FALSE`. Obviously, a value of the enumeration can never equal or be ordered against an unknown value. The `!=` is special given the negation means *every* enumerated value matches since by definition they aren't equal to an unknown value. The third case is for the `IN` and `NOT_IN` operations. These don't have logical changes, we just don't add the unknown value to the set of values to test against. --- TYPE: IMPROVEMENT DESC: Remove QueryCondition requirement for Enumeration values to be valid. --- test/CMakeLists.txt | 1 + test/src/unit-cppapi-enumerations.cc | 71 +- ...nit-cppapi-query-condition-enumerations.cc | 1169 +++++++++++++++++ test/src/unit-cppapi-query-condition-sets.cc | 21 +- test/src/unit-enumerations.cc | 118 +- tiledb/sm/c_api/tiledb.h | 20 +- tiledb/sm/c_api/tiledb_enum.h | 7 + tiledb/sm/enums/query_condition_op.h | 23 +- tiledb/sm/misc/constants.cc | 6 + tiledb/sm/misc/constants.h | 6 + tiledb/sm/query/ast/query_ast.cc | 31 +- tiledb/sm/query/query_condition.cc | 106 ++ 12 files changed, 1546 insertions(+), 33 deletions(-) create mode 100644 test/src/unit-cppapi-query-condition-enumerations.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7dc4cf539f88..d57e8cac93d8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -229,6 +229,7 @@ if (TILEDB_CPP_API) src/unit-cppapi-nullable.cc src/unit-cppapi-partial-attribute-write.cc src/unit-cppapi-query.cc + src/unit-cppapi-query-condition-enumerations.cc src/unit-cppapi-query-condition-sets.cc src/cpp-integration-query-condition.cc src/unit-cppapi-schema.cc diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc index 15990fc00590..baf04bbea852 100644 --- a/test/src/unit-cppapi-enumerations.cc +++ b/test/src/unit-cppapi-enumerations.cc @@ -522,7 +522,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( CPPEnumerationFx, - "CPP: Enumeration Query - Invalid Enumeration Value", + "CPP: Enumeration Query - Invalid Enumeration Value is Always False", "[enumeration][query][basic]") { create_array(); @@ -542,10 +542,64 @@ TEST_CASE_METHOD( .set_data_buffer("attr1", attr1) .set_condition(qc); - // Check that the error message is helpful to users. - auto matcher = Catch::Matchers::ContainsSubstring( - "Enumeration value not found for field 'attr1'"); - REQUIRE_THROWS_WITH(query.submit(), matcher); + REQUIRE_NOTHROW(query.submit()); + + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = { + INT32_MIN, INT32_MIN, INT32_MIN, INT32_MIN, INT32_MIN}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Invalid Enumeration Value Accepted by EQ", + "[enumeration][query][basic]") { + create_array(); + + // Attempt to query with an enumeration value that isn't in the Enumeration + QueryCondition qc(ctx_); + qc.init("attr1", "alf", 3, TILEDB_EQ); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc); + + CHECK_NOTHROW(query.submit()); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Invalid Enumeration Value Accepted by IN", + "[enumeration][query][basic]") { + create_array(); + + // Attempt to query with an enumeration value that isn't in the Enumeration + std::vector vals = {"alf", "fred"}; + auto qc = QueryConditionExperimental::create(ctx_, "attr1", vals, TILEDB_IN); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc); + + CHECK_NOTHROW(query.submit()); } TEST_CASE_METHOD( @@ -572,7 +626,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( CPPEnumerationFx, "CPP: Enumeration Query - Attempt to query on empty enumeration", - "[enumeration][query][error]") { + "[enumeration][query][empty-results]") { create_array(true); // Attempt to query with an enumeration value that isn't in the Enumeration @@ -591,10 +645,7 @@ TEST_CASE_METHOD( .set_data_buffer("attr3", attr3) .set_condition(qc); - // Check that the error message is helpful to users. - auto matcher = Catch::Matchers::ContainsSubstring( - "Enumeration value not found for field 'attr3'"); - REQUIRE_THROWS_WITH(query.submit(), matcher); + REQUIRE_NOTHROW(query.submit()); } CPPEnumerationFx::CPPEnumerationFx() diff --git a/test/src/unit-cppapi-query-condition-enumerations.cc b/test/src/unit-cppapi-query-condition-enumerations.cc new file mode 100644 index 000000000000..872fedab7c76 --- /dev/null +++ b/test/src/unit-cppapi-query-condition-enumerations.cc @@ -0,0 +1,1169 @@ +/** + * @file unit-cppapi-query-condition-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for query condition related functions. + */ + +#include +#include +#include + +#include "test/support/src/ast_helpers.h" +#include "test/support/tdb_catch.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/serialization/query.h" + +#ifdef TILEDB_SERIALIZATION +#include +#include +#define GENERATE_SERIALIZATION() GENERATE(false, true) +#else +#define GENERATE_SERIALIZATION() false +#endif + +using namespace tiledb; + +/* + * The test fixture. See the first test for a basic example of expected + * usage. + */ + +struct EnmrQCCell; +struct ResultEnmrQCCell; + +using EnmrQCMatcher = std::function; +using EnmrQCCreator = std::function; + +class CPPQueryConditionEnumerationFx { + public: + CPPQueryConditionEnumerationFx(); + ~CPPQueryConditionEnumerationFx(); + + uint32_t run_test( + tiledb_array_type_t type, + bool serialize, + EnmrQCMatcher matcher, + EnmrQCCreator creator, + uint32_t num_rows = 1024); + + protected: + std::string uri_; + Context ctx_; + VFS vfs_; + + tiledb_array_type_t type_; + bool serialize_; + uint32_t num_rows_; + + // A fill value result. This is the value that a dense array query returns + // for a non-matching result. + std::unique_ptr fill_; + + // Our random source + std::mt19937_64 rand_; + + // Enumeration helpers + std::unordered_map cell_type_values_; + std::unordered_map cycle_phase_values_; + std::unordered_map wavelength_values_; + + std::unordered_map cell_type_index_; + std::unordered_map cycle_phase_index_; + std::unordered_map wavelength_index_; + + // The data in the array represented as a vector of EnmrQCCell instances. + std::vector data_; + + // Private API + void create_array( + tiledb_array_type_t type, bool serialize, uint32_t num_rows); + void remove_array(); + void write_array(); + uint32_t check_read(EnmrQCMatcher matcher, EnmrQCCreator creator); + std::vector read_array(EnmrQCCreator creator); + std::vector generate_data(uint32_t num_rows); + void create_enumeration( + ArraySchema& schema, + const std::string& name, + const std::unordered_map& values, + bool ordered); + QueryCondition serialize_deserialize_qc(QueryCondition& qc); + void validate_query_condition(EnmrQCCreator creator); + std::unordered_map make_index( + std::unordered_map values); +}; + +/* + * Test Schema + * =========== + * + * row_id - A numeric integer in the range 1 - $NUM_ROWS + * sample_name - A random string with the format [A-J]{4}[0-9]{8} + * cell_type - An enumeration of cell types, listed below. + * cycle_phase - A nullable enumeration of cell cycle phase, listed below. + * wavelength - An ordered enumeration of laser wavelengths, listed below. + * luminosity - A float value in the range [0.0, 1.0] + * + * Cell Type Enumeration Values: + * + * For the non biologists: Endothelial cells have to do with blood vessels + * and epithelial has to do with skin and other membranes. Stem cells are + * progenitors that can become other types of cells, and neurons are cells + * in the brain. Muscle and bone cell types are both self documenting. + * + * - endothelial + * - epithelial + * - muscle + * - bone + * - neuron + * - stem + * + * Cell Cycle Phases (These are actually real): + * + * Fun fact, G1 and G2 literally stand for Gap 1 and Gap 2. M stands for the + * mitosis/meiosis stage (i.e., cell division), S is the synthesis phase + * (i.e., when a cell is replicating its DNA in preparation to divide), while + * G1 and G2 are basically a historical "We're not sure what's going on + * exactly" stages. I'm sure they know more now, but this entire anecdote is + * the only reason I remember the stages. + * + * Also, this enumeration is ordered in this test even though it really + * hasn't got an order since there's no obvious first step of the cycle given + * that its actually the definition of a chicken and egg issue. + * + * - G1 + * - S + * - G2 + * - M + * + * Laser Wavelengths (Also real, but no, I don't have these memorized): + * + * N.B., the values are "355nm" or "552nm" for example. I've labeled each + * wavelength with their corresponding color only for reference for folks that + * haven't memorized the electromagnetic spectrum. + * + * Also, a quick background on the science of fluorescent microscopy and why + * wavelengths as an ordered enumeration is actually an interesting use case. + * First, the basic principle of fluorescence is that an atom or molecule can + * be excited by a photon of a certain frequency into a new state, which + * then after some time relaxes and emits a photon of a different wavelength. + * Anything that can do this is called a fluorophore. The important part here + * is that the both of the excitation and relaxation photons are set at + * specific wavelengths because physics. + * + * The result of all that is that you can detect fluorophores by shining + * one color of light on it and then looking for a specific *different* color + * of light being emitted. With that knowledge, applying it to science is just + * a matter of tagging something of interest with a fluorophore and then + * setting up various light sources and wavelength filters and voila, you get + * a useful measurable signal. + * + * So back to lasers, given that we have specific wavelengths that are chosen + * based on what fluorophore we're using, we wouldn't want this to just be a + * integer. Allowing raw integral values means that there's a possibility we + * end up with data that's not one of our lasers due to data entry + * errors and so on. However, they're quite comparable as obviously the + * enumerated values are numeric in nature. + * + * - 355nm (ultra violet) + * - 405nm (blue) + * - 488nm (violet) + * - 532nm (green) + * - 552nm (greener?) + * - 561nm (green-yellow) + * - 640nm (red) + */ + +struct EnmrQCCell { + EnmrQCCell(); + + uint32_t row_id; + std::string sample_name; + std::string cell_type; + std::string cycle_phase; + bool cycle_phase_valid; + std::string wavelength; + float luminosity; +}; + +// Used by test internals +struct ResultEnmrQCCell : public EnmrQCCell { + ResultEnmrQCCell(); + + // We're purposefully avoiding a copy constructor so that the single case + // we need to copy a fill value is made obvious. + void copy_fill(const std::unique_ptr& rhs); + + bool operator==(const EnmrQCCell& rhs); + bool valid; +}; + +/* + * Test case definitions start here. + */ + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Basic Tests", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto threshold = num_rows_ / 2; + auto matcher = [=](const EnmrQCCell& cell) { + return cell.row_id < threshold; + }; + auto creator = [=](Context& ctx) { + return QueryCondition::create(ctx, "row_id", threshold, TILEDB_LT); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Simple Enumeration Equality", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type == "bone"; + }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "cell_type", std::string("bone"), TILEDB_EQ); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Simple Enumeration Non-Equality", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type != "bone"; + }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "cell_type", std::string("bone"), TILEDB_NE); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Simple Enumeration Inequality", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.wavelength <= "532nm"; + }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "wavelength", std::string("532nm"), TILEDB_LE); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Simple Enumeration Equality to Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type == "fruit"; + }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "cell_type", std::string("fruit"), TILEDB_EQ); + }; + + // Assert that == invalid enumeration value matches nothing. + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == 0); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Simple Enumeration Non-Equality to Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type != "fruit"; + }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "cell_type", std::string("fruit"), TILEDB_NE); + }; + + // Assert that != invalid value matches everything. + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == num_rows_); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration Equality to Negated Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type == "fruit"; + }; + auto creator = [](Context& ctx) { + auto qc = QueryCondition::create( + ctx, "cell_type", std::string("fruit"), TILEDB_NE); + return qc.negate(); + }; + + // Assert that (not !=) invalid value matches nothing. + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == 0); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration Non-Equality to Negated Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type != "fruit"; + }; + auto creator = [](Context& ctx) { + auto qc = QueryCondition::create( + ctx, "cell_type", std::string("fruit"), TILEDB_EQ); + return qc.negate(); + }; + + // Assert that (not ==) invalid value matches everything + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == num_rows_); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration Inequality with Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell&) { return false; }; + auto creator = [](Context& ctx) { + return QueryCondition::create( + ctx, "wavelength", std::string("6000nm"), TILEDB_LE); + }; + + // Assert that (<=) invalid value matches nothing. + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == 0); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration Inequality with Negated Invalid Value", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell&) { return false; }; + auto creator = [](Context& ctx) { + auto qc = QueryCondition::create( + ctx, "wavelength", std::string("6000nm"), TILEDB_LE); + return qc.negate(); + }; + + // Assert that (not <=) invalid value matches nothing. + auto matched = run_test(type, serialize, matcher, creator); + REQUIRE(matched == 0); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration IN Set with Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type == "bone" || cell.cell_type == "stem"; + }; + auto creator = [](Context& ctx) { + std::vector values = {"bone", "stem", "fish"}; + return QueryConditionExperimental::create( + ctx, "cell_type", values, TILEDB_IN); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration NOT_IN Set with Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type != "bone" && cell.cell_type != "stem"; + }; + auto creator = [](Context& ctx) { + std::vector values = {"bone", "stem", "fish"}; + return QueryConditionExperimental::create( + ctx, "cell_type", values, TILEDB_NOT_IN); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration IN Set with Negated Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type == "bone" || cell.cell_type == "stem"; + }; + auto creator = [](Context& ctx) { + std::vector values = {"bone", "stem", "fish"}; + auto qc = QueryConditionExperimental::create( + ctx, "cell_type", values, TILEDB_NOT_IN); + return qc.negate(); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Enumeration NOT IN Set with Negated Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + return cell.cell_type != "bone" && cell.cell_type != "stem"; + }; + auto creator = [](Context& ctx) { + std::vector values = {"bone", "stem", "fish"}; + auto qc = + QueryConditionExperimental::create(ctx, "cell_type", values, TILEDB_IN); + return qc.negate(); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "RowID inequality AND Enumeration IN Set with Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + auto r1 = cell.row_id < 512; + auto r2 = cell.cell_type == "bone" || cell.cell_type == "stem"; + return r1 && r2; + }; + auto creator = [](Context& ctx) { + auto qc1 = QueryCondition::create(ctx, "row_id", 512, TILEDB_LT); + std::vector values = {"bone", "stem", "fish"}; + auto qc2 = + QueryConditionExperimental::create(ctx, "cell_type", values, TILEDB_IN); + return qc1.combine(qc2, TILEDB_AND); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "RowID inequality OR Enumeration NOT_IN Set with Invalid Member", + "[query-condition][enumeration][logic]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + auto matcher = [](const EnmrQCCell& cell) { + auto r1 = cell.row_id < 512; + auto r2 = cell.cell_type != "bone" && cell.cell_type != "stem"; + return r1 || r2; + }; + auto creator = [](Context& ctx) { + auto qc1 = QueryCondition::create(ctx, "row_id", 512, TILEDB_LT); + std::vector values = {"bone", "stem", "fish"}; + auto qc2 = QueryConditionExperimental::create( + ctx, "cell_type", values, TILEDB_NOT_IN); + return qc1.combine(qc2, TILEDB_OR); + }; + + run_test(type, serialize, matcher, creator); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Check error on negation of TILEDB_ALWAYS_TRUE after rewrite.", + "[query-condition][enumeration][logic][rewrite-error]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + + create_array(type, serialize, 1024); + + Array array(ctx_, uri_, TILEDB_READ); + auto core_array = array.ptr().get()->array_; + core_array->load_all_enumerations(); + + auto qc = + QueryCondition::create(ctx_, "cell_type", std::string("fish"), TILEDB_NE); + auto core_qc = qc.ptr().get()->query_condition_; + core_qc->rewrite_enumeration_conditions(core_array->array_schema_latest()); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid negation of rewritten query."); + REQUIRE_THROWS_WITH(qc.negate(), matcher); +} + +TEST_CASE_METHOD( + CPPQueryConditionEnumerationFx, + "Check error on negation of TILEDB_ALWAYS_FALSE after rewrite.", + "[query-condition][enumeration][logic][rewrite-error]") { + auto type = GENERATE(TILEDB_SPARSE, TILEDB_DENSE); + auto serialize = GENERATE_SERIALIZATION(); + + create_array(type, serialize, 1024); + + Array array(ctx_, uri_, TILEDB_READ); + auto core_array = array.ptr().get()->array_; + core_array->load_all_enumerations(); + + auto qc = + QueryCondition::create(ctx_, "cell_type", std::string("fish"), TILEDB_EQ); + auto core_qc = qc.ptr().get()->query_condition_; + core_qc->rewrite_enumeration_conditions(core_array->array_schema_latest()); + + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid negation of rewritten query."); + REQUIRE_THROWS_WITH(qc.negate(), matcher); +} + +TEST_CASE( + "Check error on creating a TILEDB_ALWAYS_TRUE QueryCondition", + "[query-condition][enumeration][logic][op-error]") { + Context ctx; + // TILEDB_ALWAYS_TRUE is not an exposed symbol so we even have to force + // the issue by knowing the internal value and casting it. + auto op = static_cast(253); + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid use of internal operation: ALWAYS_TRUE"); + REQUIRE_THROWS_WITH(QueryCondition::create(ctx, "foo", 0, op), matcher); +} + +TEST_CASE( + "Check error on creating a TILEDB_ALWAYS_FALSE QueryCondition", + "[query-condition][enumeration][logic][op-error]") { + Context ctx; + // TILEDB_ALWAYS_FALSE is not an exposed symbol so we even have to force + // the issue by knowing the internal value and casting it. + auto op = static_cast(254); + auto matcher = Catch::Matchers::ContainsSubstring( + "Invalid use of internal operation: ALWAYS_FALSE"); + REQUIRE_THROWS_WITH(QueryCondition::create(ctx, "foo", 0, op), matcher); +} + +/* + * All code below here is test support implementation. + */ + +EnmrQCCell::EnmrQCCell() + : row_id(0) + , sample_name("Uninitialied Data Cell") + , cell_type("Uninitialised Data Cell") + , cycle_phase("Uninitialized Data Cell") + , wavelength("Uninitialized Data Cell") + , luminosity(3.14159f) { +} + +ResultEnmrQCCell::ResultEnmrQCCell() { + row_id = std::numeric_limits::max(); + sample_name = "Uninitialized Result Cell"; + cell_type = "Uninitialized Result Cell"; + cycle_phase = "Uninitialized Result Cell"; + wavelength = "Uninitialized Result Cell"; + luminosity = 1.618f; + valid = false; +} + +void ResultEnmrQCCell::copy_fill(const std::unique_ptr& rhs) { + row_id = rhs->row_id; + sample_name = rhs->sample_name; + cell_type = rhs->cell_type; + cycle_phase = rhs->cycle_phase; + wavelength = rhs->wavelength; + luminosity = rhs->luminosity; + valid = true; +} + +bool ResultEnmrQCCell::operator==(const EnmrQCCell& rhs) { + if (row_id != rhs.row_id) { + return false; + } + + if (sample_name != rhs.sample_name) { + return false; + } + + if (cell_type != rhs.cell_type) { + return false; + } + + if (cycle_phase != rhs.cycle_phase) { + return false; + } + + if (wavelength != rhs.wavelength) { + return false; + } + + if (luminosity != rhs.luminosity) { + return false; + } + + return true; +} + +std::ostream& operator<<(std::ostream& os, const EnmrQCCell& cell) { + return os << "EnmrQCCell{" + << "row_id: " << cell.row_id << ", " + << "sample_name: '" << cell.sample_name << "', " + << "cell_type: '" << cell.cell_type << "', " + << "cycle_phase: '" << cell.cycle_phase << "', " + << "cycle_phase_valid: " << (cell.cycle_phase_valid ? "yes" : "no") + << ", " + << "wavelength: '" << cell.wavelength << "', " + << "luminosity: " << cell.luminosity << "}"; +} + +CPPQueryConditionEnumerationFx::CPPQueryConditionEnumerationFx() + : uri_("query_condition_enumeration_array") + , vfs_(ctx_) { + remove_array(); + + // This is used for asserting the dense-non-match case. + fill_ = std::make_unique(); + fill_->sample_name = ""; + fill_->cell_type = ""; + fill_->cycle_phase = ""; + fill_->cycle_phase_valid = false; + fill_->wavelength = ""; + fill_->luminosity = std::numeric_limits::min(); + + std::random_device rdev; + rand_.seed(rdev()); + + cell_type_values_ = { + {"bone", 0}, + {"endothelial", 1}, + {"epithelial", 2}, + {"muscle", 3}, + {"neuron", 4}, + {"stem", 5}}; + + cycle_phase_values_ = {{"G1", 0}, {"S", 1}, {"G2", 2}, {"M", 3}}; + + wavelength_values_ = { + {"355nm", 0}, + {"405nm", 1}, + {"488nm", 2}, + {"532nm", 3}, + {"552nm", 4}, + {"561nm", 5}, + {"640nm", 6}}; + + cell_type_index_ = make_index(cell_type_values_); + cycle_phase_index_ = make_index(cycle_phase_values_); + wavelength_index_ = make_index(wavelength_values_); +} + +CPPQueryConditionEnumerationFx::~CPPQueryConditionEnumerationFx() { + remove_array(); +} + +uint32_t CPPQueryConditionEnumerationFx::run_test( + tiledb_array_type_t type, + bool serialize, + EnmrQCMatcher matcher, + EnmrQCCreator creator, + uint32_t num_rows) { + create_array(type, serialize, num_rows); + return check_read(matcher, creator); +} + +void CPPQueryConditionEnumerationFx::create_array( + tiledb_array_type_t type, bool serialize, uint32_t num_rows) { + type_ = type; + serialize_ = serialize; + num_rows_ = num_rows; + data_ = generate_data(num_rows_); + + // Create our array schema + ArraySchema schema(ctx_, type_); + + if (type_ == TILEDB_SPARSE) { + schema.set_capacity(num_rows_); + } + + // Create a single dimension row_id as uint32_t + auto dim = Dimension::create(ctx_, "row_id", {{1, num_rows_}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + // Create our enumerations + create_enumeration(schema, "cell_types", cell_type_index_, false); + create_enumeration(schema, "cycle_phases", cycle_phase_index_, true); + create_enumeration(schema, "wavelengths", wavelength_index_, true); + + // Create our attributes + auto sample_name = Attribute::create(ctx_, "sample_name"); + + auto cell_type = Attribute::create(ctx_, "cell_type"); + AttributeExperimental::set_enumeration_name(ctx_, cell_type, "cell_types"); + + auto cell_phase = Attribute::create(ctx_, "cycle_phase"); + AttributeExperimental::set_enumeration_name(ctx_, cell_phase, "cycle_phases"); + cell_phase.set_nullable(true); + + auto wavelength = Attribute::create(ctx_, "wavelength"); + AttributeExperimental::set_enumeration_name(ctx_, wavelength, "wavelengths"); + + auto luminosity = Attribute::create(ctx_, "luminosity"); + + schema.add_attributes( + sample_name, cell_type, cell_phase, wavelength, luminosity); + + // Create and write the array. + Array::create(uri_, schema); + write_array(); +} + +void CPPQueryConditionEnumerationFx::write_array() { + Array array(ctx_, uri_, TILEDB_WRITE); + Query query(ctx_, array); + + std::vector row_ids(num_rows_); + std::iota(row_ids.begin(), row_ids.end(), 1); + + if (type_ == TILEDB_DENSE) { + Subarray subarray(ctx_, array); + subarray.add_range(0, 1, num_rows_); + query.set_subarray(subarray); + } else { + query.set_data_buffer("row_id", row_ids); + } + + // Generate our write buffers + std::vector names(num_rows_ * strlen("AAAA00000000")); + std::vector name_offsets(num_rows_); + std::vector cell_types(num_rows_); + std::vector cycle_phases(num_rows_); + std::vector cycle_phases_validity(num_rows_); + std::vector wavelengths(num_rows_); + std::vector luminosity(num_rows_); + + uint64_t name_offset = 0; + for (size_t i = 0; i < num_rows_; i++) { + auto& cell = data_[i]; + + std::memcpy( + names.data() + name_offset, + cell.sample_name.data(), + cell.sample_name.size()); + name_offsets[i] = name_offset; + name_offset += cell.sample_name.size(); + + cell_types[i] = cell_type_values_.at(cell.cell_type); + if (cell.cycle_phase_valid) { + cycle_phases[i] = cycle_phase_values_.at(cell.cycle_phase); + } else { + cycle_phases[i] = 254; + } + cycle_phases_validity[i] = cell.cycle_phase_valid ? 1 : 0; + wavelengths[i] = wavelength_values_.at(cell.wavelength); + luminosity[i] = cell.luminosity; + } + + // Attach the buffers to our write query + query.set_data_buffer("sample_name", names) + .set_offsets_buffer("sample_name", name_offsets) + .set_data_buffer("cell_type", cell_types) + .set_data_buffer("cycle_phase", cycle_phases) + .set_validity_buffer("cycle_phase", cycle_phases_validity) + .set_data_buffer("wavelength", wavelengths) + .set_data_buffer("luminosity", luminosity); + + CHECK_NOTHROW(query.submit() == Query::Status::COMPLETE); + query.finalize(); + array.close(); +} + +uint32_t CPPQueryConditionEnumerationFx::check_read( + EnmrQCMatcher matcher, EnmrQCCreator creator) { + validate_query_condition(creator); + + // Calculate the number of matches to expect. + uint32_t should_match = 0; + for (auto& cell : data_) { + if (matcher(cell)) { + should_match += 1; + } + } + + auto results = read_array(creator); + + uint32_t num_matched = 0; + + for (size_t i = 0; i < num_rows_; i++) { + if (type_ == TILEDB_DENSE) { + // Dense reads always return a value where non-matching cells are just + // the fill values for all attributes. + if (matcher(data_[i])) { + REQUIRE(results[i] == data_[i]); + num_matched += 1; + } else { + REQUIRE(results[i] == *fill_.get()); + } + // Just an internal test assertion that all dense values are valid. + REQUIRE(results[i].valid); + } else { + // Sparse queries only return cells that match. We mark this with whether + // the ResultEnmrQCCell has its valid flag set or not. + if (matcher(data_[i])) { + REQUIRE(results[i] == data_[i]); + num_matched += 1; + } else { + REQUIRE(results[i].valid == false); + } + } + } + + REQUIRE(num_matched == should_match); + + return num_matched; +} + +std::vector CPPQueryConditionEnumerationFx::read_array( + EnmrQCCreator creator) { + Array array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + + if (type_ == TILEDB_DENSE) { + Subarray subarray(ctx_, array); + subarray.add_range(0, 1, num_rows_); + query.set_subarray(subarray); + } else { + query.set_layout(TILEDB_GLOBAL_ORDER); + } + + std::vector row_ids(num_rows_); + std::vector sample_names(num_rows_ * 2 * strlen("AAAA00000000")); + std::vector sample_name_offsets(num_rows_); + std::vector cell_types(num_rows_); + std::vector cycle_phases(num_rows_); + std::vector cycle_phases_validity(num_rows_); + std::vector wavelengths(num_rows_); + std::vector luminosities(num_rows_); + + auto qc = creator(ctx_); + if (serialize_) { + qc = serialize_deserialize_qc(qc); + } + + query.set_condition(qc) + .set_data_buffer("row_id", row_ids) + .set_data_buffer("sample_name", sample_names) + .set_offsets_buffer("sample_name", sample_name_offsets) + .set_data_buffer("cell_type", cell_types) + .set_data_buffer("cycle_phase", cycle_phases) + .set_validity_buffer("cycle_phase", cycle_phases_validity) + .set_data_buffer("wavelength", wavelengths) + .set_data_buffer("luminosity", luminosities); + + REQUIRE(query.submit() == Query::Status::COMPLETE); + + auto table = query.result_buffer_elements(); + + row_ids.resize(table["row_id"].second); + sample_name_offsets.resize(table["sample_name"].first); + cell_types.resize(table["cell_type"].second); + cycle_phases.resize(table["cycle_phase"].second); + cycle_phases_validity.resize(table["cycle_phase"].second); + wavelengths.resize(table["wavelength"].second); + luminosities.resize(table["luminosity"].second); + + // Create our result cell instances + // + // Remember here that the default constructed instances are in the + // test-uninitialized state. + // + // The second thing to remember, is that this for loop is has two slightly + // different behaviors between dense and sparse queries. For spares, we can + // iterate over 0, 1, or up to num_rows_ matches. For dense, it always + // iterates of num_rows_ entries because non-matches are returned as fill + // values. + std::vector ret(num_rows_); + for (size_t i = 0; i < row_ids.size(); i++) { + auto row_id = row_ids[i]; + + // There are basically three states a result can be in. The first obvious + // case is when its a match and it should equal the cell in data_. The + // second obvious case is when its a non-match which should never ever match + // anything in data_. The third case that makes things weird is a non-match + // on a dense array which returns fill values. + // + // The logic below is dealing with each of those cases. Currently it relies + // on a bit of a hack. This is using an implementation detail to detect the + // difference between the non-match and dense-fill-values cases. We can do + // this because when we write null cycle phase values, we set the cycle + // phase enumeration value to 254. We do that on purpose to distinguish + // these cases. Core will return what we write regardless of the null-ness + // so we're abusing that for testing here. + // + // So lets dive in: + // + // If cycle_phase is 255, this is the dense-non-match case so the cell is + // a copy of the random dense_non_match_ instance.. + if (cycle_phases[i] == std::numeric_limits::max()) { + ret[i].copy_fill(fill_); + continue; + } + + // At this point the dense-non-match case is handled. So now all we have + // to worry about is match vs non-match cases. The following logic could + // easily seem redundant when we could just check one attribute and return + // a default constructed ResultEnmrQCCell or a copy of data_[i]. + // + // However, we can't rely on the compiler defaults for asserting match + // semantics here because of the nullptr ternary logic involved in the + // cycle_phase case. That's a fancy way of saying (x < null) and + // (x > null) are both false and the compiler can't figure that out for us. + + // A subtle dense vs sparse issue here. We're setting result[i].valid to + // true because the default constructed value is false. This gives us + // an extra sparse/dense behavior assertion for free because a sparse + // non-match will be false in the results. + ret[row_id - 1].valid = true; + + // Calculate the sample name length even though we know its 12. + uint64_t name_length = 0; + if (i < sample_name_offsets.size() - 1) { + name_length = sample_name_offsets[i + 1] - sample_name_offsets[i]; + } else { + name_length = table["sample_name"].second - sample_name_offsets[i]; + } + + // Make sure we're dealing with the correct cell. + ret[row_id - 1].row_id = row_ids[i]; + + // Copy over the sample name. Either the whole "AAAA00000000" id or the + // empty string if we're on a fill value. + ret[row_id - 1].sample_name = + std::string(sample_names.data() + sample_name_offsets[i], name_length); + + // The cell_type attribute is non-nullable so the 255 distinguishes between + // match and non-match for this cell. + if (cell_types[i] == std::numeric_limits::max()) { + ret[row_id - 1].cell_type = ""; + } else { + ret[row_id - 1].cell_type = cell_type_index_.at(cell_types[i]); + } + + // This is a bit weird because there's null-ability logic involved with + // cell not matching logic. One thing to keep in mind, is that we write + // 254 as the data value when we mark a cycle phase as null. Currently + // TileDB repeats this value back so we can use it to deduce when we wrote + // null vs seeing an non-matching cell in the dense results. + if (cycle_phases_validity[i]) { + // We have a non-null cycle phase. + ret[row_id - 1].cycle_phase = cycle_phase_index_.at(cycle_phases[i]); + ret[row_id - 1].cycle_phase_valid = true; + } else { + // A null cycle phase. The assertion on cycle_phases[i] here is testing + // our precondition that we know this should be null by the fact that + // core returns our invalid 254 value. + assert(cycle_phases[i] == 254); + ret[row_id - 1].cycle_phase = ""; + ret[row_id - 1].cycle_phase_valid = false; + } + + if (wavelengths[i] == std::numeric_limits::max()) { + // Cell didn't match, so wavelength gets the non-match value of an empty + // string. + ret[row_id - 1].wavelength = ""; + } else { + ret[row_id - 1].wavelength = wavelength_index_.at(wavelengths[i]); + } + + // This is a bit silly, but in the interest of preventing accidental + // matches, I'm using the nanf function to give a float that is NaN with + // the fraction as the leading digits of pi to help debugging. + // + // That is to say, if you start seeing NaN issues with this test, you can + // check the fraction to see if its a "real" NaN or a logic error because + // of how we're creating the NaN instance here. + if (luminosities[i] == std::numeric_limits::min()) { + ret[row_id - 1].luminosity = nanf("3141592"); + } else { + ret[row_id - 1].luminosity = luminosities[i]; + } + } + + return ret; +} + +std::vector CPPQueryConditionEnumerationFx::generate_data( + uint32_t num_rows) { + std::vector ret(num_rows); + + std::uniform_int_distribution sn_rng(0, 9); + std::uniform_int_distribution ct_rng( + 0, static_cast(cell_type_values_.size()) - 1); + std::uniform_int_distribution cp_rng( + 0, static_cast(cycle_phase_values_.size()) - 1); + std::uniform_int_distribution wl_rng( + 0, static_cast(wavelength_values_.size()) - 1); + std::uniform_real_distribution lum_rng(0.0, 1.0); + + std::string sample_name = "AAAA00000000"; + + for (uint32_t i = 0; i < ret.size(); i++) { + ret[i].row_id = i + 1; + + for (size_t i = 0; i < sample_name.size(); i++) { + if (i < 4) { + sample_name[i] = 'A' + static_cast(sn_rng(rand_)); + } else { + sample_name[i] = '0' + static_cast(sn_rng(rand_)); + } + } + + ret[i].sample_name = sample_name; + REQUIRE(ret[i].sample_name.size() == 12); + ret[i].cell_type = cell_type_index_.at(static_cast(ct_rng(rand_))); + // A bit hacky, but I'm reusing the luminescence RNG to make the cycle + // phase null 30% of the time. + if (lum_rng(rand_) < 0.3) { + ret[i].cycle_phase = ""; + ret[i].cycle_phase_valid = false; + } else { + ret[i].cycle_phase = + cycle_phase_index_.at(static_cast(cp_rng(rand_))); + ret[i].cycle_phase_valid = true; + } + ret[i].wavelength = + wavelength_index_.at(static_cast(wl_rng(rand_))); + ret[i].luminosity = lum_rng(rand_); + } + + return ret; +} + +#ifdef TILEDB_SERIALIZATION +QueryCondition CPPQueryConditionEnumerationFx::serialize_deserialize_qc( + QueryCondition& qc) { + using namespace tiledb::sm::serialization; + using Condition = tiledb::sm::serialization::capnp::Condition; + + auto qc_ptr = qc.ptr().get()->query_condition_; + + QueryCondition ret(ctx_); + auto ret_ptr = ret.ptr().get()->query_condition_; + + // Serialize the query condition. + ::capnp::MallocMessageBuilder message; + auto builder = message.initRoot(); + throw_if_not_ok(condition_to_capnp(*qc_ptr, &builder)); + + // Deserialize the query condition. + *ret_ptr = condition_from_capnp(builder); + REQUIRE(tiledb::test::ast_equal(ret_ptr->ast(), qc_ptr->ast())); + + return ret; +} +#else +QueryCondition CPPQueryConditionEnumerationFx::serialize_deserialize_qc( + QueryCondition&) { + throw std::logic_error("Unable to serialize when serialization is disabled."); +} +#endif + +void CPPQueryConditionEnumerationFx::create_enumeration( + ArraySchema& schema, + const std::string& name, + const std::unordered_map& index, + bool ordered) { + std::vector enmr_values; + for (uint8_t i = 0; i < static_cast(index.size()); i++) { + enmr_values.push_back(index.at(i)); + } + auto enmr = Enumeration::create(ctx_, name, enmr_values, ordered); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); +} + +void CPPQueryConditionEnumerationFx::validate_query_condition( + EnmrQCCreator creator) { + Array array(ctx_, uri_, TILEDB_READ); + auto core_array = array.ptr().get()->array_; + core_array->load_all_enumerations(); + + auto qc = creator(ctx_); + auto core_qc = qc.ptr().get()->query_condition_; + core_qc->rewrite_enumeration_conditions(core_array->array_schema_latest()); + + REQUIRE(core_qc->check(core_array->array_schema_latest()).ok()); +} + +std::unordered_map +CPPQueryConditionEnumerationFx::make_index( + std::unordered_map values) { + std::unordered_map ret; + for (auto& [name, idx] : values) { + assert(ret.find(idx) == ret.end()); + ret[idx] = name; + } + return ret; +} + +void CPPQueryConditionEnumerationFx::remove_array() { + if (vfs_.is_dir(uri_)) { + vfs_.remove_dir(uri_); + } +} diff --git a/test/src/unit-cppapi-query-condition-sets.cc b/test/src/unit-cppapi-query-condition-sets.cc index 76ba87d2918b..dadb33c08e61 100644 --- a/test/src/unit-cppapi-query-condition-sets.cc +++ b/test/src/unit-cppapi-query-condition-sets.cc @@ -260,6 +260,22 @@ TEST_CASE_METHOD( check_read(qc, [](const QCSetsCell& c) { return !(c.a2 == "wilma"); }); } +TEST_CASE_METHOD( + CPPQueryConditionFx, + "IN - String With Non-Enumeration Value", + "[query-condition][set][non-enum-value][string]") { + auto type = GENERATE( + TestArrayType::DENSE, TestArrayType::SPARSE, TestArrayType::LEGACY); + auto serialize = SERIALIZE_TESTS(); + create_array(type, serialize); + + std::vector values = {"wilma", "astro"}; + auto qc = + QueryConditionExperimental::create(ctx_, "attr2", values, TILEDB_NOT_IN); + + check_read(qc, [](const QCSetsCell& c) { return !(c.a2 == "wilma"); }); +} + TEST_CASE_METHOD( CPPQueryConditionFx, "NOT_IN - Enumeration", @@ -828,7 +844,7 @@ void CPPQueryConditionFx::rm_array() { } void CPPQueryConditionFx::generate_data() { - num_elements_ = 10; // * 1024; + num_elements_ = 1024; dim_values_.clear(); attr1_values_.clear(); @@ -1012,7 +1028,8 @@ std::vector CPPQueryConditionFx::to_vector( template T CPPQueryConditionFx::choose_value(std::vector& values) { auto rval = random(); - auto idx = static_cast(rval * values.size()); + // Note the `% values.size()` which handles when rval is 1.0 + auto idx = static_cast(rval * values.size()) % values.size(); return values[idx]; } diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index b8926f5561c2..a16b98955a4d 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -152,6 +152,11 @@ template QueryCondition create_qc( const char* field_name, T condition_value, const QueryConditionOp& op); +QueryCondition create_qc( + const char* field_name, + std::vector values, + const QueryConditionOp& op); + /* ********************************* */ /* Testing Enumeration */ /* ********************************* */ @@ -1946,6 +1951,81 @@ TEST_CASE_METHOD( REQUIRE(data2.rvalue_as() == 2); } +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Non-Enumeration Values Are Always False", + "[enumeration][query-condition][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + // This is normally invoked by the query class when not being tested. It's + // required here so that the enumeration's data is loaded. + array->get_enumeration("test_enmr"); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", "cthulu", QueryConditionOp::EQ); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + // Assert that the rewritten tree matches in the right places while also + // different to verify the assertion of having been rewritten. + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto data1 = tree1->get_data(); + auto data2 = tree2->get_data(); + REQUIRE(data2.size() != data1.size()); + + // "cthulu" is converted a 4 byte int with value 0 + REQUIRE(data2.size() == 4); + REQUIRE(data2.rvalue_as() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Non-Enumeration Set Members Are Ignored", + "[enumeration][query-condition][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + // This is normally invoked by the query class when not being tested. It's + // required here so that the enumeration's data is loaded. + array->get_enumeration("test_enmr"); + + // Create two copies of the same query condition for assertions + std::vector vals = {"ant", "bat", "cthulhu"}; + auto qc1 = create_qc("attr1", vals, QueryConditionOp::IN); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + // Assert that the rewritten tree matches in the right places while also + // different to verify the assertion of having been rewritten. + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto data1 = tree1->get_data(); + auto data2 = tree2->get_data(); + REQUIRE(data2.size() != data1.size()); + REQUIRE(data2.size() == 8); + REQUIRE(tree2->get_offsets().size() == 16); +} + TEST_CASE_METHOD( EnumerationFx, "QueryCondition - Rewrite Enumeration Value After Extension", @@ -1960,11 +2040,10 @@ TEST_CASE_METHOD( auto qc1 = create_qc("attr1", std::string("gerbil"), QueryConditionOp::EQ); auto qc2 = qc1; - // Check that we fail the rewrite before extension. - auto matcher = Catch::Matchers::ContainsSubstring( - "Enumeration value not found for field 'attr1'"); - REQUIRE_THROWS_WITH( - qc1.rewrite_enumeration_conditions(*(schema.get())), matcher); + // Check that the value was converted to 0. + REQUIRE_NOTHROW(qc1.rewrite_enumeration_conditions(*(schema.get()))); + REQUIRE(qc1.ast()->get_op() == QueryConditionOp::ALWAYS_FALSE); + REQUIRE(qc1.ast()->get_data().rvalue_as() == 0); // Extend enumeration via schema evolution. std::vector values_to_add = {"firefly", "gerbil", "hamster"}; @@ -2935,3 +3014,32 @@ QueryCondition create_qc( return ret; } + +QueryCondition create_qc( + const char* field_name, + std::vector values, + const QueryConditionOp& op) { + std::vector data; + std::vector offsets; + + uint64_t data_size = 0; + for (auto& val : values) { + data_size += val.size(); + } + + data.resize(data_size); + uint64_t curr_offset = 0; + for (auto& val : values) { + offsets.push_back(curr_offset); + memcpy(data.data() + curr_offset, val.data(), val.size()); + curr_offset += val.size(); + } + + return QueryCondition( + field_name, + data.data(), + data.size(), + offsets.data(), + offsets.size() * sizeof(uint64_t), + op); +} diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index 34b9cc66a1d9..39e5048a41bb 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -95,10 +95,22 @@ typedef enum { /** Query condition operator. */ typedef enum { -/** Helper macro for defining query condition operator enums. */ -#define TILEDB_QUERY_CONDITION_OP_ENUM(id) TILEDB_##id -#include "tiledb_enum.h" -#undef TILEDB_QUERY_CONDITION_OP_ENUM + /** Less-than operator */ + TILEDB_LT = 0, + /** Less-than-or-equal operator */ + TILEDB_LE = 1, + /** Greater-than operator */ + TILEDB_GT = 2, + /** Greater-than-or-equal operator */ + TILEDB_GE = 3, + /** Equal operator */ + TILEDB_EQ = 4, + /** Not-equal operator */ + TILEDB_NE = 5, + /** IN set membership operator. */ + TILEDB_IN = 6, + /** NOT IN set membership operator. */ + TILEDB_NOT_IN = 7, } tiledb_query_condition_op_t; /** Query condition combination operator. */ diff --git a/tiledb/sm/c_api/tiledb_enum.h b/tiledb/sm/c_api/tiledb_enum.h index a733847c8598..9fb10866c03e 100644 --- a/tiledb/sm/c_api/tiledb_enum.h +++ b/tiledb/sm/c_api/tiledb_enum.h @@ -82,6 +82,9 @@ TILEDB_QUERY_STATUS_DETAILS_ENUM(REASON_MEMORY_BUDGET) = 2, #endif +// This enumeration is special in that if you add enumeration entries here +// you have to manually add the new values in tiledb.h. This is to avoid +// exposing `TILEDB_ALWAYS_TRUE` and `TILEDB_ALWAYS_FALSE` in the public API. #ifdef TILEDB_QUERY_CONDITION_OP_ENUM /** Less-than operator */ TILEDB_QUERY_CONDITION_OP_ENUM(LT) = 0, @@ -99,6 +102,10 @@ TILEDB_QUERY_CONDITION_OP_ENUM(IN) = 6, /** NOT IN set membership operator. */ TILEDB_QUERY_CONDITION_OP_ENUM(NOT_IN) = 7, + /** ALWAYS TRUE operator. */ + TILEDB_QUERY_CONDITION_OP_ENUM(ALWAYS_TRUE) = 253, + /** ALWAYS TRUE operator. */ + TILEDB_QUERY_CONDITION_OP_ENUM(ALWAYS_FALSE) = 254, #endif #ifdef TILEDB_QUERY_CONDITION_COMBINATION_OP_ENUM diff --git a/tiledb/sm/enums/query_condition_op.h b/tiledb/sm/enums/query_condition_op.h index 919b65e03367..edfd40826e06 100644 --- a/tiledb/sm/enums/query_condition_op.h +++ b/tiledb/sm/enums/query_condition_op.h @@ -79,6 +79,10 @@ inline const std::string& query_condition_op_str( return constants::query_condition_op_in_str; case QueryConditionOp::NOT_IN: return constants::query_condition_op_not_in_str; + case QueryConditionOp::ALWAYS_TRUE: + return constants::query_condition_op_always_true_str; + case QueryConditionOp::ALWAYS_FALSE: + return constants::query_condition_op_always_false_str; default: return constants::empty_str; } @@ -105,6 +109,13 @@ inline Status query_condition_op_enum( } else if ( query_condition_op_str == constants::query_condition_op_not_in_str) { *query_condition_op = QueryConditionOp::NOT_IN; + } else if ( + query_condition_op_str == constants::query_condition_op_always_true_str) { + *query_condition_op = QueryConditionOp::ALWAYS_TRUE; + } else if ( + query_condition_op_str == + constants::query_condition_op_always_false_str) { + *query_condition_op = QueryConditionOp::ALWAYS_FALSE; } else { return Status_Error("Invalid QueryConditionOp " + query_condition_op_str); } @@ -113,7 +124,7 @@ inline Status query_condition_op_enum( inline void ensure_qc_op_is_valid(QueryConditionOp query_condition_op) { auto qc_op_enum{::stdx::to_underlying(query_condition_op)}; - if (qc_op_enum > 7) { + if (qc_op_enum > 7 && qc_op_enum != 253 && qc_op_enum != 254) { throw std::runtime_error( "Invalid Query Condition Op " + std::to_string(qc_op_enum)); } @@ -156,6 +167,16 @@ inline QueryConditionOp negate_query_condition_op(const QueryConditionOp op) { case QueryConditionOp::NOT_IN: return QueryConditionOp::IN; + // ALWAYS_TRUE and ALWAYS_FALSE are the result of QueryCondition rewriting + // which means they should not be available for negation. This saves us + // from having to have invertible and non-invertible versions of these + // operations. + case QueryConditionOp::ALWAYS_TRUE: + throw std::logic_error("Invalid negation of rewritten query."); + + case QueryConditionOp::ALWAYS_FALSE: + throw std::logic_error("Invalid negation of rewritten query."); + default: throw std::runtime_error("negate_query_condition_op: Invalid op."); } diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index cadde0492afe..7c575f77b532 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -337,6 +337,12 @@ const std::string query_status_initialized_str = "INITIALIZED"; /** TILEDB_UNINITIALIZED Query String **/ const std::string query_status_uninitialized_str = "UNINITIALIZED"; +/** TILEDB_ALWAYS_TRUE Query Condition Op String **/ +const std::string query_condition_op_always_true_str = "ALWAYS_TRUE"; + +/** TILEDB_ALWAYS_FALSE Query Condition Op String **/ +const std::string query_condition_op_always_false_str = "ALWAYS_FALSE"; + /** TILEDB_LT Query Condition Op String **/ const std::string query_condition_op_lt_str = "LT"; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 0cccd9f49987..4e23f45eabaf 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -333,6 +333,12 @@ extern const std::string query_status_initialized_str; /** TILEDB_UNINITIALIZED Query String **/ extern const std::string query_status_uninitialized_str; +/** TILEDB_ALWAYS_TRUE Query Condition Op String **/ +extern const std::string query_condition_op_always_true_str; + +/** TILEDB_ALWAYS_FALSE Query Condition Op String **/ +extern const std::string query_condition_op_always_false_str; + /** TILEDB_LT Query Condition Op String **/ extern const std::string query_condition_op_lt_str; diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index 94403e57aaa9..16dfad7c9550 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -199,12 +199,17 @@ void ASTNodeVal::rewrite_enumeration_conditions( if (op_ != QueryConditionOp::IN && op_ != QueryConditionOp::NOT_IN) { auto idx = enumeration->index_of(get_value_ptr(), get_value_size()); if (idx == constants::enumeration_missing_value) { - throw std::invalid_argument( - "Enumeration value not found for field '" + attr->name() + "'"); + if (op_ == QueryConditionOp::NE) { + op_ = QueryConditionOp::ALWAYS_TRUE; + } else { + op_ = QueryConditionOp::ALWAYS_FALSE; + } + data_ = ByteVecValue(val_size); + utils::safe_integral_cast_to_datatype(0, attr->type(), data_); + } else { + data_ = ByteVecValue(val_size); + utils::safe_integral_cast_to_datatype(idx, attr->type(), data_); } - - data_ = ByteVecValue(val_size); - utils::safe_integral_cast_to_datatype(idx, attr->type(), data_); } else { // Buffers and writers for the new data/offsets memory std::vector data_buffer(val_size * members_.size()); @@ -215,25 +220,29 @@ void ASTNodeVal::rewrite_enumeration_conditions( ByteVecValue curr_data(val_size); uint64_t curr_offset = 0; + uint64_t num_offsets = 0; for (auto& member : members_) { auto idx = enumeration->index_of(member.data(), member.size()); if (idx == constants::enumeration_missing_value) { - throw std::invalid_argument( - "Enumeration value not found for field '" + attr->name() + "'"); + continue; } utils::safe_integral_cast_to_datatype(idx, attr->type(), curr_data); data_writer.write(curr_data.data(), curr_data.size()); offsets_writer.write(curr_offset); curr_offset += val_size; + num_offsets += 1; } - data_ = ByteVecValue(data_buffer.size()); - std::memcpy(data_.data(), data_buffer.data(), data_buffer.size()); + auto total_data_size = curr_offset; + auto total_offsets_size = num_offsets * constants::cell_var_offset_size; + + data_ = ByteVecValue(total_data_size); + std::memcpy(data_.data(), data_buffer.data(), total_data_size); - offsets_ = ByteVecValue(offsets_buffer.size()); - std::memcpy(offsets_.data(), offsets_buffer.data(), offsets_buffer.size()); + offsets_ = ByteVecValue(total_offsets_size); + std::memcpy(offsets_.data(), offsets_buffer.data(), total_offsets_size); generate_members(); } diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 242babd69049..0b4bc63dcde4 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -126,6 +126,12 @@ Status QueryCondition::init( return Status_QueryConditionError("Cannot reinitialize query condition"); } + if (op == QueryConditionOp::ALWAYS_TRUE || + op == QueryConditionOp::ALWAYS_FALSE) { + auto op_str = query_condition_op_str(op); + throw std::runtime_error("Invalid use of internal operation: " + op_str); + } + // AST Construction. tree_ = tdb_unique_ptr(tdb_new( ASTNodeVal, field_name, condition_value, condition_value_size, op)); @@ -482,6 +488,22 @@ struct QueryCondition::BinaryCmpNullChecks { } }; +/** Partial template specialization for `QueryConditionOp::LT`. */ +template +struct QueryCondition::BinaryCmpNullChecks { + static inline bool cmp(const void* lhs, uint64_t, const void*, uint64_t) { + return lhs != nullptr; + } +}; + +/** Partial template specialization for `QueryConditionOp::LT`. */ +template +struct QueryCondition::BinaryCmpNullChecks { + static inline bool cmp(const void*, uint64_t, const void*, uint64_t) { + return false; + } +}; + /** Partial template specialization for `QueryConditionOp::LT`. */ template struct QueryCondition::BinaryCmpNullChecks { @@ -748,6 +770,30 @@ void QueryCondition::apply_ast_node( CombinationOp combination_op, std::vector& result_cell_bitmap) const { switch (node->get_op()) { + case QueryConditionOp::ALWAYS_TRUE: + apply_ast_node( + node, + fragment_metadata, + stride, + var_size, + nullable, + fill_value, + result_cell_slabs, + combination_op, + result_cell_bitmap); + break; + case QueryConditionOp::ALWAYS_FALSE: + apply_ast_node( + node, + fragment_metadata, + stride, + var_size, + nullable, + fill_value, + result_cell_slabs, + combination_op, + result_cell_bitmap); + break; case QueryConditionOp::LT: apply_ast_node( node, @@ -1440,6 +1486,34 @@ void QueryCondition::apply_ast_node_dense( const void* cell_slab_coords, span result_buffer) const { switch (node->get_op()) { + case QueryConditionOp::ALWAYS_TRUE: + apply_ast_node_dense( + node, + array_schema, + result_tile, + start, + src_cell, + stride, + var_size, + nullable, + combination_op, + cell_slab_coords, + result_buffer); + break; + case QueryConditionOp::ALWAYS_FALSE: + apply_ast_node_dense( + node, + array_schema, + result_tile, + start, + src_cell, + stride, + var_size, + nullable, + combination_op, + cell_slab_coords, + result_buffer); + break; case QueryConditionOp::LT: apply_ast_node_dense( node, @@ -2205,6 +2279,22 @@ struct QueryCondition::BinaryCmp { } }; +/** Partial template specialization for `QueryConditionOp::LT`. */ +template +struct QueryCondition::BinaryCmp { + static inline bool cmp(const void*, uint64_t, const void*, uint64_t) { + return true; + } +}; + +/** Partial template specialization for `QueryConditionOp::LT`. */ +template +struct QueryCondition::BinaryCmp { + static inline bool cmp(const void*, uint64_t, const void*, uint64_t) { + return false; + } +}; + /** Partial template specialization for `QueryConditionOp::LT`. */ template struct QueryCondition::BinaryCmp { @@ -2385,6 +2475,22 @@ void QueryCondition::apply_ast_node_sparse( CombinationOp combination_op, std::vector& result_bitmap) const { switch (node->get_op()) { + case QueryConditionOp::ALWAYS_TRUE: + apply_ast_node_sparse< + T, + QueryConditionOp::ALWAYS_TRUE, + BitmapType, + CombinationOp, + nullable>(node, result_tile, var_size, combination_op, result_bitmap); + break; + case QueryConditionOp::ALWAYS_FALSE: + apply_ast_node_sparse< + T, + QueryConditionOp::ALWAYS_FALSE, + BitmapType, + CombinationOp, + nullable>(node, result_tile, var_size, combination_op, result_bitmap); + break; case QueryConditionOp::LT: apply_ast_node_sparse< T, From 3be6978cfa7b49e4ce7bec60274647f4ac3a59e6 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Mon, 4 Mar 2024 18:41:25 -0500 Subject: [PATCH 220/456] Add memory tracking for Dimension. (#4762) This adds a MemoryTracker to Dimension and converts members and methods to use `tdb::pmr` containers. --- TYPE: NO_HISTORY DESC: Add memory tracking for Dimension. --- test/src/unit-Reader.cc | 4 +- test/src/unit-cppapi-schema-evolution.cc | 5 +- test/src/unit-dimension.cc | 35 +++++--- test/src/unit-enumerations.cc | 3 +- test/src/unit-request-handlers.cc | 15 +++- test/src/unit-result-tile.cc | 9 +- tiledb/api/c_api/dimension/dimension_api.cc | 10 ++- .../c_api/dimension/dimension_api_internal.h | 10 ++- .../test/compile_capi_dimension_stub_main.cc | 2 +- tiledb/sm/array/test/unit_consistency.h | 11 ++- tiledb/sm/array_schema/array_schema.cc | 7 +- tiledb/sm/array_schema/dimension.cc | 43 +++++----- tiledb/sm/array_schema/dimension.h | 63 ++++++-------- tiledb/sm/array_schema/dimension_label.cc | 2 +- tiledb/sm/array_schema/domain.cc | 7 +- tiledb/sm/array_schema/domain.h | 5 +- .../test/array_schema_test_support.h | 9 +- tiledb/sm/array_schema/test/unit_dimension.cc | 37 +++++--- .../sm/array_schema/test/unit_domain_data.cc | 12 ++- tiledb/sm/c_api/tiledb_filestore.cc | 3 +- .../test/unit_fragment_consolidator.cc | 5 +- tiledb/sm/query/readers/result_tile.cc | 8 +- tiledb/sm/query/readers/result_tile.h | 10 +-- .../query/readers/sparse_index_reader_base.cc | 8 +- tiledb/sm/query/test/unit_query_condition.cc | 86 ++++++++++++++----- tiledb/sm/query_plan/test/unit_query_plan.cc | 9 +- tiledb/sm/rtree/test/unit_rtree.cc | 3 +- tiledb/sm/serialization/array_schema.cc | 14 +-- tiledb/sm/serialization/test/CMakeLists.txt | 2 +- .../test/unit_capnp_array_schema.cc | 7 +- .../test/unit_capnp_nonempty_domain.cc | 7 +- .../sm/subarray/test/unit_add_ranges_list.cc | 12 ++- 32 files changed, 298 insertions(+), 165 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index a48a803b9467..4f17aa0013d0 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -242,10 +242,10 @@ TEST_CASE_METHOD( TileDomain array_tile_domain( UINT32_MAX, domain, dsd, tile_extents, layout); - auto d1{make_shared(HERE(), "d1", Datatype::INT32)}; + auto d1{make_shared(HERE(), "d1", Datatype::INT32, tracker_)}; CHECK(d1->set_domain(domain_vec).ok()); CHECK(d1->set_tile_extent(&tile_extents_vec[0]).ok()); - auto d2{make_shared(HERE(), "d2", Datatype::INT32)}; + auto d2{make_shared(HERE(), "d2", Datatype::INT32, tracker_)}; CHECK(d2->set_domain(&domain_vec[2]).ok()); CHECK(d2->set_tile_extent(&tile_extents_vec[1]).ok()); auto dom{make_shared(HERE())}; diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index b5bb38f0e8ff..0ce1c1921b56 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -814,7 +814,10 @@ TEST_CASE( tiledb::sm::ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); auto dim = make_shared( - HERE(), "dim1", tiledb::sm::Datatype::INT32); + HERE(), + "dim1", + tiledb::sm::Datatype::INT32, + tiledb::test::get_test_memory_tracker()); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); diff --git a/test/src/unit-dimension.cc b/test/src/unit-dimension.cc index 990790d79418..dd8027052895 100644 --- a/test/src/unit-dimension.cc +++ b/test/src/unit-dimension.cc @@ -31,6 +31,7 @@ */ #include "test/support/src/helpers-dimension.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/hilbert.h" @@ -47,10 +48,11 @@ TEST_CASE( "Dimension: Test map_to_uint64, integers", "[dimension][map_to_uint64][int]") { // Create dimensions - Dimension d1("d1", Datatype::INT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::INT32, memory_tracker); int32_t dom1[] = {0, 100}; CHECK(d1.set_domain(dom1).ok()); - Dimension d2("d2", Datatype::INT32); + Dimension d2("d2", Datatype::INT32, memory_tracker); int32_t dom2[] = {0, 200}; CHECK(d2.set_domain(dom2).ok()); @@ -161,10 +163,11 @@ TEST_CASE( "Dimension: Test map_to_uint64, int32, negative", "[dimension][map_to_uint64][int32][negative]") { // Create dimensions - Dimension d1("d1", Datatype::INT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::INT32, memory_tracker); int32_t dom1[] = {-50, 50}; CHECK(d1.set_domain(dom1).ok()); - Dimension d2("d2", Datatype::INT32); + Dimension d2("d2", Datatype::INT32, memory_tracker); int32_t dom2[] = {-100, 100}; CHECK(d2.set_domain(dom2).ok()); @@ -275,10 +278,11 @@ TEST_CASE( "Dimension: Test map_to_uint64, float32", "[dimension][map_to_uint64][float32]") { // Create dimensions - Dimension d1("d1", Datatype::FLOAT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::FLOAT32, memory_tracker); float dom1[] = {0.0f, 1.0f}; CHECK(d1.set_domain(dom1).ok()); - Dimension d2("d2", Datatype::FLOAT32); + Dimension d2("d2", Datatype::FLOAT32, memory_tracker); float dom2[] = {0.0f, 2.0f}; CHECK(d2.set_domain(dom2).ok()); @@ -398,8 +402,9 @@ TEST_CASE( "Dimension: Test map_to_uint64, string", "[dimension][map_to_uint64][string]") { // Create dimensions - Dimension d1("d1", Datatype::STRING_ASCII); - Dimension d2("d2", Datatype::STRING_ASCII); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::STRING_ASCII, memory_tracker); + Dimension d2("d2", Datatype::STRING_ASCII, memory_tracker); // Create 2D hilbert curve (auxiliary here) Hilbert h(2); @@ -517,7 +522,8 @@ TEST_CASE( "Dimension: Test map_from_uint64, int32", "[dimension][map_from_uint64][int32]") { // Create dimensions - Dimension d1("d1", Datatype::INT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::INT32, memory_tracker); int32_t dom1[] = {0, 100}; CHECK(d1.set_domain(dom1).ok()); @@ -538,7 +544,8 @@ TEST_CASE( "Dimension: Test map_from_uint64, int32, negative", "[dimension][map_from_uint64][int32][negative]") { // Create dimensions - Dimension d1("d1", Datatype::INT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::INT32, memory_tracker); int32_t dom1[] = {-50, 50}; CHECK(d1.set_domain(dom1).ok()); @@ -559,7 +566,8 @@ TEST_CASE( "Dimension: Test map_from_uint64, float32", "[dimension][map_from_uint64][float32]") { // Create dimensions - Dimension d1("d1", Datatype::FLOAT32); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::FLOAT32, memory_tracker); float dom1[] = {0.0f, 1.0f}; CHECK(d1.set_domain(dom1).ok()); @@ -580,7 +588,8 @@ TEST_CASE( "Dimension: Test map_from_uint64, string", "[dimension][map_from_uint64][string]") { // Create dimensions - Dimension d1("d1", Datatype::STRING_ASCII); + auto memory_tracker = get_test_memory_tracker(); + Dimension d1("d1", Datatype::STRING_ASCII, memory_tracker); // Set number of buckets Hilbert h(2); @@ -611,7 +620,7 @@ double basic_verify_overlap_ratio( T range1_low, T range1_high, T range2_low, T range2_high) { auto r1 = TypedRange(range1_low, range1_high); auto r2 = TypedRange(range2_low, range2_high); - Dimension d("foo", RangeTraits::datatype); + Dimension d("foo", RangeTraits::datatype, get_test_memory_tracker()); auto ratio = d.overlap_ratio(r1, r2); CHECK(0.0 <= ratio); CHECK(ratio <= 1.0); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index a16b98955a4d..e35cbc30d120 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2833,7 +2833,8 @@ shared_ptr EnumerationFx::create_schema() { auto schema = make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); - auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + auto dim = + make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 287ea1ca949a..f55f79231663 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -98,7 +98,11 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { virtual shared_ptr create_schema() override { auto schema = make_shared( HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); - auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + auto dim = make_shared( + HERE(), + "dim1", + Datatype::INT32, + tiledb::test::get_test_memory_tracker()); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); auto dom = make_shared(HERE()); @@ -400,7 +404,8 @@ shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize auto schema = make_shared( HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); - auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + auto dim = make_shared( + HERE(), "dim1", Datatype::INT32, tiledb::test::get_test_memory_tracker()); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); @@ -454,9 +459,11 @@ shared_ptr HandleQueryPlanRequestFx::create_schema() { throw_if_not_ok(schema->set_tile_order(Layout::ROW_MAJOR)); uint32_t dim_domain[] = {1, 10, 1, 10}; - auto dim1 = make_shared(HERE(), "dim1", Datatype::INT32); + auto dim1 = make_shared( + HERE(), "dim1", Datatype::INT32, tiledb::test::get_test_memory_tracker()); throw_if_not_ok(dim1->set_domain(&dim_domain[0])); - auto dim2 = make_shared(HERE(), "dim2", Datatype::INT32); + auto dim2 = make_shared( + HERE(), "dim2", Datatype::INT32, tiledb::test::get_test_memory_tracker()); throw_if_not_ok(dim2->set_domain(&dim_domain[2])); auto dom = make_shared(HERE()); diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index fc38bbac6141..7cc5518d882d 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -68,7 +68,8 @@ struct CResultTileFx { ~CResultTileFx(); }; -CResultTileFx::CResultTileFx() { +CResultTileFx::CResultTileFx() + : memory_tracker_(tiledb::test::get_test_memory_tracker()) { tiledb_config_t* config; tiledb_error_t* error = nullptr; REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); @@ -277,7 +278,8 @@ TEST_CASE_METHOD( exp_result_count = {0, 1, 1, 1, 1, 1, 1, 0}; } - std::vector range_indexes(ranges.size()); + tdb::pmr::vector range_indexes( + ranges.size(), memory_tracker_->get_resource(MemoryType::DIMENSIONS)); std::iota(range_indexes.begin(), range_indexes.end(), 0); std::vector result_count(num_cells, 1); @@ -417,7 +419,8 @@ TEST_CASE_METHOD( exp_result_count = {0, 1, 2, 1, 0, 1, 3, 2}; } - std::vector range_indexes(ranges.size()); + tdb::pmr::vector range_indexes( + ranges.size(), memory_tracker_->get_resource(MemoryType::DIMENSIONS)); std::iota(range_indexes.begin(), range_indexes.end(), 0); std::vector result_count(num_cells, 1); diff --git a/tiledb/api/c_api/dimension/dimension_api.cc b/tiledb/api/c_api/dimension/dimension_api.cc index c8a831d7a040..c42341e36e84 100644 --- a/tiledb/api/c_api/dimension/dimension_api.cc +++ b/tiledb/api/c_api/dimension/dimension_api.cc @@ -34,10 +34,13 @@ #include "../filter_list/filter_list_api_internal.h" #include "dimension_api_external.h" #include "dimension_api_internal.h" +#include "tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h" +#include "tiledb/common/memory_tracker.h" namespace tiledb::api { int32_t tiledb_dimension_alloc( + tiledb_ctx_t* ctx, const char* name, tiledb_datatype_t type, const void* dim_domain, @@ -47,8 +50,10 @@ int32_t tiledb_dimension_alloc( throw CAPIStatusException("Dimension name must not be NULL"); } ensure_output_pointer_is_valid(dim); + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); *dim = tiledb_dimension_handle_t::make_handle( - name, static_cast(type)); + name, static_cast(type), memory_tracker); try { (*dim)->set_domain(dim_domain); (*dim)->set_tile_extent(tile_extent); @@ -149,7 +154,8 @@ CAPI_INTERFACE( const void* dim_domain, const void* tile_extent, tiledb_dimension_t** dim) { - return api_entry_context( + return tiledb::api::api_entry_with_context< + tiledb::api::tiledb_dimension_alloc>( ctx, name, type, dim_domain, tile_extent, dim); } diff --git a/tiledb/api/c_api/dimension/dimension_api_internal.h b/tiledb/api/c_api/dimension/dimension_api_internal.h index fe96c866e5e9..03ca1657d826 100644 --- a/tiledb/api/c_api/dimension/dimension_api_internal.h +++ b/tiledb/api/c_api/dimension/dimension_api_internal.h @@ -33,9 +33,11 @@ #ifndef TILEDB_CAPI_DIMENSION_INTERNAL_H #define TILEDB_CAPI_DIMENSION_INTERNAL_H +#include #include "dimension_api_external.h" #include "tiledb/api/c_api_support/handle/handle.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/dimension.h" /** @@ -63,8 +65,12 @@ struct tiledb_dimension_handle_t */ static constexpr std::string_view object_type_name{"dimension"}; - tiledb_dimension_handle_t(const std::string& name, tiledb::sm::Datatype type) - : dimension_(make_shared(HERE(), name, type)) { + tiledb_dimension_handle_t( + const std::string& name, + tiledb::sm::Datatype type, + shared_ptr memory_tracker) + : dimension_(make_shared( + HERE(), name, type, memory_tracker)) { } /** diff --git a/tiledb/api/c_api/dimension/test/compile_capi_dimension_stub_main.cc b/tiledb/api/c_api/dimension/test/compile_capi_dimension_stub_main.cc index c407374aa149..d106ef1a5b1e 100644 --- a/tiledb/api/c_api/dimension/test/compile_capi_dimension_stub_main.cc +++ b/tiledb/api/c_api/dimension/test/compile_capi_dimension_stub_main.cc @@ -30,6 +30,6 @@ int main() { std::string name("foo"); - tiledb_dimension_handle_t x{name, tiledb::sm::Datatype::UINT32}; + tiledb_dimension_handle_t x{name, tiledb::sm::Datatype::UINT32, nullptr}; return 0; } diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 15971fd172d7..20e5c310f8f9 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -63,8 +63,13 @@ using array_entry = std::tuple; using entry_type = std::multimap::const_iterator; class WhiteboxConsistencyController : public ConsistencyController { + shared_ptr memory_tracker_; + public: - WhiteboxConsistencyController() = default; + WhiteboxConsistencyController() + : memory_tracker_(tiledb::test::get_test_memory_tracker()) { + } + ~WhiteboxConsistencyController() = default; entry_type register_array( @@ -97,8 +102,8 @@ class WhiteboxConsistencyController : public ConsistencyController { // Create Domain uint64_t dim_dom[2]{0, 1}; uint64_t tile_extent = 1; - shared_ptr dim = - make_shared(HERE(), std::string("dim"), Datatype::UINT64); + shared_ptr dim = make_shared( + HERE(), std::string("dim"), Datatype::UINT64, memory_tracker_); throw_if_not_ok(dim->set_domain(&dim_dom)); throw_if_not_ok(dim->set_tile_extent(&tile_extent)); diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 2522e5601568..f35ced27396e 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1372,7 +1372,12 @@ shared_ptr ArraySchema::deserialize( // Note: Security validation delegated to invoked API // #TODO Add security validation auto domain{Domain::deserialize( - deserializer, version, cell_order, tile_order, coords_filters)}; + deserializer, + version, + cell_order, + tile_order, + coords_filters, + memory_tracker)}; // Load attributes // Note: Security validation delegated to invoked API diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index ea54b069e145..2db81166179b 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -32,6 +32,7 @@ #include "dimension.h" #include "tiledb/common/logger_public.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/filter_type.h" @@ -46,10 +47,6 @@ using namespace tiledb::common; using namespace tiledb::type; -tiledb::common::blank::blank() - : tiledb::sm::Dimension{"", tiledb::sm::Datatype::INT32} { -} - namespace tiledb::sm { class DimensionException : public StatusException { @@ -63,8 +60,12 @@ class DimensionException : public StatusException { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -Dimension::Dimension(const std::string& name, Datatype type) - : name_(name) +Dimension::Dimension( + const std::string& name, + Datatype type, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , name_(name) , type_(type) { ensure_datatype_is_supported(type_); cell_val_num_ = (datatype_is_string(type)) ? constants::var_num : 1; @@ -95,8 +96,10 @@ Dimension::Dimension( uint32_t cell_val_num, const Range& domain, const FilterPipeline& filter_pipeline, - const ByteVecValue& tile_extent) - : cell_val_num_(cell_val_num) + const ByteVecValue& tile_extent, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , cell_val_num_(cell_val_num) , domain_(domain) , filters_(filter_pipeline) , name_(name) @@ -152,7 +155,8 @@ shared_ptr Dimension::deserialize( Deserializer& deserializer, uint32_t version, Datatype type, - FilterPipeline& coords_filters) { + FilterPipeline& coords_filters, + shared_ptr memory_tracker) { Status st; // Load dimension name auto dimension_name_size = deserializer.read(); @@ -210,7 +214,8 @@ shared_ptr Dimension::deserialize( cell_val_num, domain, filter_pipeline, - tile_extent); + tile_extent, + memory_tracker); } const Range& Dimension::domain() const { @@ -697,7 +702,7 @@ double Dimension::overlap_ratio(const Range& r1, const Range& r2) const { void Dimension::relevant_ranges( const NDRange& ranges, const Range& mbr, - std::vector& relevant_ranges) const { + tdb::pmr::vector& relevant_ranges) const { assert(relevant_ranges_func_ != nullptr); return relevant_ranges_func_(ranges, mbr, relevant_ranges); } @@ -706,7 +711,7 @@ template <> void Dimension::relevant_ranges( const NDRange& ranges, const Range& mbr, - std::vector& relevant_ranges) { + tdb::pmr::vector& relevant_ranges) { const auto& mbr_start = mbr.start_str(); const auto& mbr_end = mbr.end_str(); @@ -755,7 +760,7 @@ template void Dimension::relevant_ranges( const NDRange& ranges, const Range& mbr, - std::vector& relevant_ranges) { + tdb::pmr::vector& relevant_ranges) { const auto mbr_data = (const T*)mbr.start_fixed(); const auto mbr_start = mbr_data[0]; const auto mbr_end = mbr_data[1]; @@ -799,7 +804,7 @@ void Dimension::relevant_ranges( std::vector Dimension::covered_vec( const NDRange& ranges, const Range& mbr, - const std::vector& relevant_ranges) const { + const tdb::pmr::vector& relevant_ranges) const { assert(covered_vec_func_ != nullptr); return covered_vec_func_(ranges, mbr, relevant_ranges); } @@ -808,12 +813,11 @@ template <> std::vector Dimension::covered_vec( const NDRange& ranges, const Range& mbr, - const std::vector& relevant_ranges) { + const tdb::pmr::vector& relevant_ranges) { const auto& range_start = mbr.start_str(); const auto& range_end = mbr.end_str(); - std::vector covered; - covered.resize(relevant_ranges.size()); + std::vector covered(relevant_ranges.size()); for (uint64_t i = 0; i < relevant_ranges.size(); i++) { auto r = relevant_ranges[i]; auto r2_start = ranges[r].start_str(); @@ -829,11 +833,10 @@ template std::vector Dimension::covered_vec( const NDRange& ranges, const Range& mbr, - const std::vector& relevant_ranges) { + const tdb::pmr::vector& relevant_ranges) { auto d1 = (const T*)mbr.start_fixed(); - std::vector covered; - covered.resize(relevant_ranges.size()); + std::vector covered(relevant_ranges.size()); for (uint64_t i = 0; i < relevant_ranges.size(); i++) { auto r = relevant_ranges[i]; auto d2 = (const T*)ranges[r].start_fixed(); diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 6ca7f85f0336..c68943c4c4df 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -37,12 +37,15 @@ #include #include #include +#include #include #include #include "tiledb/common/blank.h" #include "tiledb/common/common.h" #include "tiledb/common/logger_public.h" +#include "tiledb/common/macros.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/constants.h" @@ -89,8 +92,12 @@ class Dimension { * * @param name The name of the dimension. * @param type The type of the dimension. + * @param memory_tracker The memory tracker to use. */ - Dimension(const std::string& name, Datatype type); + Dimension( + const std::string& name, + Datatype type, + shared_ptr memory_tracker); /** * Constructor. @@ -101,6 +108,7 @@ class Dimension { * @param domain The range of the dimension range. * @param filter_pipeline The filters of the dimension. * @param tile_extent The tile extent of the dimension. + * @param memory_tracker The memory tracker to use. */ Dimension( const std::string& name, @@ -108,34 +116,15 @@ class Dimension { uint32_t cell_val_num, const Range& domain, const FilterPipeline& filter_pipeline, - const ByteVecValue& tile_extent); + const ByteVecValue& tile_extent, + shared_ptr memory_tracker); - /** - * Copy constructor is deleted. - * - * `Dimension` objects are stored as `shared_ptr` within C API handles and - * within `Domain`. Instead of copying a `Dimension` one can copy a pointer. - */ - Dimension(const Dimension&) = delete; - - /** - * Copy assignment is deleted. - */ - Dimension& operator=(const Dimension&) = delete; + DISABLE_COPY_AND_COPY_ASSIGN(Dimension); + DISABLE_MOVE_AND_MOVE_ASSIGN(Dimension); /** Destructor. */ ~Dimension() = default; - /** - * Move constructor is default - */ - Dimension(Dimension&&) = default; - - /** - * Move assignment is default - */ - Dimension& operator=(Dimension&&) = default; - /* ********************************* */ /* API */ /* ********************************* */ @@ -158,13 +147,15 @@ class Dimension { * @param version The array schema version. * @param type The type of the dimension. * @param coords_filters Coords filters to replace empty coords pipelines. + * @param memory_tracker The memory tracker to use. * @return Dimension */ static shared_ptr deserialize( Deserializer& deserializer, uint32_t version, Datatype type, - FilterPipeline& coords_filters); + FilterPipeline& coords_filters, + shared_ptr memory_tracker); /** Returns the domain. */ const Range& domain() const; @@ -584,27 +575,27 @@ class Dimension { void relevant_ranges( const NDRange& ranges, const Range& mbr, - std::vector& relevant_ranges) const; + tdb::pmr::vector& relevant_ranges) const; /** Compute relevant ranges for a set of ranges. */ template static void relevant_ranges( const NDRange& ranges, const Range& mbr, - std::vector& relevant_ranges); + tdb::pmr::vector& relevant_ranges); /** Compute covered on a set of relevant ranges. */ std::vector covered_vec( const NDRange& ranges, const Range& mbr, - const std::vector& relevant_ranges) const; + const tdb::pmr::vector& relevant_ranges) const; /** Compute covered on a set of relevant ranges. */ template static std::vector covered_vec( const NDRange& ranges, const Range& mbr, - const std::vector& relevant_ranges); + const tdb::pmr::vector& relevant_ranges); /** Splits `r` at point `v`, producing 1D ranges `r1` and `r2`. */ void split_range( @@ -752,6 +743,9 @@ class Dimension { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker for the dimension. */ + shared_ptr memory_tracker_; + /** The number of values per coordinate. */ unsigned cell_val_num_; @@ -857,7 +851,7 @@ class Dimension { * Stores the appropriate templated relevant_ranges() function based * on the dimension datatype. */ - std::function&)> + std::function&)> relevant_ranges_func_; /** @@ -865,7 +859,7 @@ class Dimension { * dimension datatype. */ std::function( - const NDRange&, const Range&, const std::vector&)> + const NDRange&, const Range&, const tdb::pmr::vector&)> covered_vec_func_; /** @@ -1064,11 +1058,4 @@ class Dimension { } // namespace tiledb::sm -namespace tiledb::common { -template <> -struct blank : public tiledb::sm::Dimension { - blank(); -}; -} // namespace tiledb::common - #endif // TILEDB_DIMENSION_H diff --git a/tiledb/sm/array_schema/dimension_label.cc b/tiledb/sm/array_schema/dimension_label.cc index f4e439eb6a9f..06dbf28c2021 100644 --- a/tiledb/sm/array_schema/dimension_label.cc +++ b/tiledb/sm/array_schema/dimension_label.cc @@ -171,7 +171,7 @@ DimensionLabel::DimensionLabel( // Create and set dimension label domain. std::vector> index_dims{ - make_shared(HERE(), "index", index_type)}; + make_shared(HERE(), "index", index_type, memory_tracker)}; throw_if_not_ok(index_dims.back()->set_domain(dim->domain().data())); throw_if_not_ok( index_dims.back()->set_tile_extent(dim->tile_extent().data())); diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 05b85cb385ee..877774a184e2 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -333,7 +333,8 @@ shared_ptr Domain::deserialize( uint32_t version, Layout cell_order, Layout tile_order, - FilterPipeline& coords_filters) { + FilterPipeline& coords_filters, + shared_ptr memory_tracker) { Status st; // Load type Datatype type = Datatype::INT32; @@ -345,8 +346,8 @@ shared_ptr Domain::deserialize( std::vector> dimensions; auto dim_num = deserializer.read(); for (uint32_t i = 0; i < dim_num; ++i) { - auto dim{ - Dimension::deserialize(deserializer, version, type, coords_filters)}; + auto dim{Dimension::deserialize( + deserializer, version, type, coords_filters, memory_tracker)}; dimensions.emplace_back(std::move(dim)); } diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 80e98dd0cbf6..c8007c8d2aa7 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -58,6 +58,7 @@ class ConstBuffer; class Dimension; class DomainTypedDataView; class FilterPipeline; +class MemoryTracker; enum class Datatype : uint8_t; enum class Layout : uint8_t; @@ -186,6 +187,7 @@ class Domain { * @param cell_order Cell order. * @param tile_order Tile order. * @param coords_filters Coords filters to replace empty coords pipelines. + * @param memory_tracker The memory tracker to use. * @return Status and Domain */ static shared_ptr deserialize( @@ -193,7 +195,8 @@ class Domain { uint32_t version, Layout cell_order, Layout tile_order, - FilterPipeline& coords_filters); + FilterPipeline& coords_filters, + shared_ptr memory_tracker); /** Returns the cell order. */ Layout cell_order() const; diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index c4b18720f6cd..84f2e09ef536 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -181,6 +181,8 @@ class TestFilterPipeline {}; * Dimension wrapper */ class TestDimension { + shared_ptr memory_tracker_; + shared_ptr d_; public: @@ -189,15 +191,16 @@ class TestDimension { * or empty defaults for everything else about it. */ TestDimension(const std::string& name, Datatype type) - : d_{make_shared( + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , d_{make_shared( HERE(), name, type, 1, // cell_val_num default_range(type), // domain FilterPipeline{}, - default_tile_extent(type) // fill value - )} {}; + default_tile_extent(type), // fill value + memory_tracker_)} {}; /** * Accessor copies the underlying object. diff --git a/tiledb/sm/array_schema/test/unit_dimension.cc b/tiledb/sm/array_schema/test/unit_dimension.cc index 5ae3e3f12737..206f84168775 100644 --- a/tiledb/sm/array_schema/test/unit_dimension.cc +++ b/tiledb/sm/array_schema/test/unit_dimension.cc @@ -32,6 +32,7 @@ #include #include "../dimension.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/enums/datatype.h" using namespace tiledb; @@ -101,7 +102,8 @@ inline T& dim_buffer_offset(void* p) { } TEST_CASE("Dimension::Dimension") { - Dimension x{"", Datatype::UINT32}; + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + Dimension x{"", Datatype::UINT32, memory_tracker}; } TEST_CASE("Dimension: Test deserialize,int32", "[dimension][deserialize]") { @@ -135,8 +137,10 @@ TEST_CASE("Dimension: Test deserialize,int32", "[dimension][deserialize]") { dim_buffer_offset(p) = tile_extent; Deserializer deserializer(&serialized_buffer, sizeof(serialized_buffer)); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); FilterPipeline fp; - auto dim = Dimension::deserialize(deserializer, 10, Datatype::INT32, fp); + auto dim = Dimension::deserialize( + deserializer, 10, Datatype::INT32, fp, memory_tracker); // Check name CHECK(dim->name() == dimension_name); @@ -175,8 +179,10 @@ TEST_CASE("Dimension: Test deserialize,string", "[dimension][deserialize]") { dim_buffer_offset(p) = null_tile_extent; Deserializer deserializer(&serialized_buffer, sizeof(serialized_buffer)); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); FilterPipeline fp; - auto dim = Dimension::deserialize(deserializer, 10, Datatype::INT32, fp); + auto dim = Dimension::deserialize( + deserializer, 10, Datatype::INT32, fp, memory_tracker); // Check name CHECK(dim->name() == dimension_name); // Check type @@ -187,6 +193,7 @@ TEST_CASE("Dimension: Test deserialize,string", "[dimension][deserialize]") { TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { std::string dim_name = "dim"; + auto memory_tracker = tiledb::test::get_test_memory_tracker(); SECTION("- valid and supported Datatypes") { std::vector valid_supported_datatypes = { @@ -210,7 +217,7 @@ TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { for (Datatype type : valid_supported_datatypes) { try { - Dimension dim{dim_name, type}; + Dimension dim{dim_name, type, memory_tracker}; } catch (...) { throw std::logic_error("Uncaught exception in Dimension constructor"); } @@ -233,7 +240,7 @@ TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { for (Datatype type : valid_unsupported_datatypes) { try { - Dimension dim{dim_name, type}; + Dimension dim{dim_name, type, memory_tracker}; } catch (std::exception& e) { CHECK( e.what() == "Datatype::" + datatype_str(type) + @@ -248,7 +255,7 @@ TEST_CASE("Dimension: Test datatypes", "[dimension][datatypes]") { for (auto type : invalid_datatypes) { try { - Dimension dim{dim_name, Datatype(type)}; + Dimension dim{dim_name, Datatype(type), memory_tracker}; } catch (std::exception& e) { CHECK( std::string(e.what()) == @@ -342,7 +349,8 @@ TEMPLATE_LIST_TEST_CASE( } void check_relevant_ranges( - std::vector& relevant_ranges, std::vector& expected) { + tdb::pmr::vector& relevant_ranges, + std::vector& expected) { CHECK(relevant_ranges.size() == expected.size()); for (uint64_t r = 0; r < expected.size(); r++) { CHECK(relevant_ranges[r] == expected[r]); @@ -363,7 +371,8 @@ TEMPLATE_LIST_TEST_CASE( "test relevant_ranges", "[dimension][relevant_ranges][fixed]", FixedTypes) { typedef TestType T; auto tiledb_type = type_to_datatype().datatype; - Dimension dim{"", tiledb_type}; + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + Dimension dim{"", tiledb_type, memory_tracker}; std::vector range_data = { 1, 1, 1, 1, 2, 2, 3, 4, 5, 6, 5, 7, 8, 9, 50, 56}; @@ -380,14 +389,16 @@ TEMPLATE_LIST_TEST_CASE( for (uint64_t i = 0; i < mbr_data.size(); i++) { Range mbr(mbr_data[i].data(), 2 * sizeof(T)); - std::vector relevant_ranges; + tdb::pmr::vector relevant_ranges( + memory_tracker->get_resource(MemoryType::DIMENSIONS)); dim.relevant_ranges(ranges, mbr, relevant_ranges); check_relevant_ranges(relevant_ranges, expected[i]); } } TEST_CASE("test relevant_ranges", "[dimension][relevant_ranges][string]") { - Dimension dim{"", Datatype::STRING_ASCII}; + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + Dimension dim{"", Datatype::STRING_ASCII, memory_tracker}; std::vector range_data = { 'a', @@ -420,14 +431,16 @@ TEST_CASE("test relevant_ranges", "[dimension][relevant_ranges][string]") { for (uint64_t i = 0; i < mbr_data.size(); i++) { Range mbr(mbr_data[i].data(), 2, 1); - std::vector relevant_ranges; + tdb::pmr::vector relevant_ranges( + memory_tracker->get_resource(MemoryType::DIMENSIONS)); dim.relevant_ranges(ranges, mbr, relevant_ranges); check_relevant_ranges(relevant_ranges, expected[i]); } } TEST_CASE("Dimension::oob format") { - Dimension d("X", Datatype::FLOAT64); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + Dimension d("X", Datatype::FLOAT64, memory_tracker); double d_dom[2]{-682.73999, 929.42999}; REQUIRE(d.set_domain(Range(&d_dom, sizeof(d_dom))).ok()); double x{-682.75}; diff --git a/tiledb/sm/array_schema/test/unit_domain_data.cc b/tiledb/sm/array_schema/test/unit_domain_data.cc index e5b9968b7034..6997ca9cf23b 100644 --- a/tiledb/sm/array_schema/test/unit_domain_data.cc +++ b/tiledb/sm/array_schema/test/unit_domain_data.cc @@ -33,6 +33,8 @@ #include #include "../domain_typed_data_view.h" #include "../dynamic_array.h" +#include "src/mem_helpers.h" + /* * Instantiating the class `Domain` requires a full definition of `Dimension` so * that its destructor is visible. The need to include this header indicates @@ -112,7 +114,10 @@ TEST_CASE("DomainTypedDataView::DomainTypedDataView, null initializer") { Domain d{}; // tiledb::sm::Dimension dim{"", tiledb::sm::Datatype::INT32}; auto dim{make_shared( - HERE(), "", tiledb::sm::Datatype::INT32)}; + HERE(), + "", + tiledb::sm::Datatype::INT32, + tiledb::test::get_test_memory_tracker())}; CHECK(d.add_dimension(dim).ok()); CHECK(d.add_dimension(dim).ok()); CHECK(d.add_dimension(dim).ok()); @@ -133,7 +138,10 @@ TEST_CASE("DomainTypedDataView::DomainTypedDataView, simple initializer") { Domain d{}; auto dim{make_shared( - HERE(), "", tiledb::sm::Datatype::INT32)}; + HERE(), + "", + tiledb::sm::Datatype::INT32, + tiledb::test::get_test_memory_tracker())}; CHECK(d.add_dimension(dim).ok()); CHECK(d.add_dimension(dim).ok()); CHECK(d.add_dimension(dim).ok()); diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 7d4eb8cff8c6..482ccde4e2e4 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -139,7 +139,8 @@ int32_t tiledb_filestore_schema_create( 1, range_obj, tiledb::sm::FilterPipeline{}, - tiledb::sm::ByteVecValue(std::move(tile_extent_vec))); + tiledb::sm::ByteVecValue(std::move(tile_extent_vec)), + memory_tracker); auto domain = make_shared(HERE()); throw_if_not_ok(domain->add_dimension(dim)); diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index 1fb78d3b2778..f6f222324c08 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -56,7 +56,10 @@ shared_ptr make_schema( Domain domain; for (uint64_t d = 0; d < dim_types.size(); d++) { auto dim{make_shared( - HERE(), "d" + std::to_string(d + 1), dim_types[d])}; + HERE(), + "d" + std::to_string(d + 1), + dim_types[d], + tiledb::test::get_test_memory_tracker())}; switch (dim_types[d]) { case Datatype::INT8: { diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index 3872adb47931..ef7aae962f4c 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -874,7 +874,7 @@ void ResultTile::compute_results_count_sparse_string( const ResultTile* result_tile, unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, @@ -1050,7 +1050,7 @@ void ResultTile::compute_results_count_sparse( const ResultTile* result_tile, unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout&, const uint64_t min_cell, @@ -1170,7 +1170,7 @@ template <> Status ResultTile::compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, @@ -1192,7 +1192,7 @@ template <> Status ResultTile::compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index e3949e9285f0..b74ec9ee744b 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -569,7 +569,7 @@ class ResultTile { const ResultTile* result_tile, unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, @@ -613,7 +613,7 @@ class ResultTile { const ResultTile* result_tile, unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, @@ -666,7 +666,7 @@ class ResultTile { Status compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, - const std::vector& range_indexes, + const tdb::pmr::vector& range_indexes, std::vector& result_count, const Layout& cell_order, const uint64_t min_cell, @@ -754,7 +754,7 @@ class ResultTile { const ResultTile*, unsigned, const NDRange&, - const std::vector&, + const tdb::pmr::vector&, std::vector&, const Layout&, const uint64_t, @@ -769,7 +769,7 @@ class ResultTile { const ResultTile*, unsigned, const NDRange&, - const std::vector&, + const tdb::pmr::vector&, std::vector&, const Layout&, const uint64_t, diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 88f18e43beb3..c4afe018a2d6 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -618,7 +618,8 @@ void SparseIndexReaderBase::compute_tile_bitmaps( auto& ranges_for_dim = subarray_.ranges_for_dim(dim_idx); // Compute the list of range index to process. - std::vector relevant_ranges; + tdb::pmr::vector relevant_ranges( + query_memory_tracker_->get_resource(MemoryType::DIMENSIONS)); relevant_ranges.reserve(ranges_for_dim.size()); domain.dimension_ptr(dim_idx)->relevant_ranges( ranges_for_dim, mbr[dim_idx], relevant_ranges); @@ -627,9 +628,8 @@ void SparseIndexReaderBase::compute_tile_bitmaps( // there is no need to compute bitmaps. const bool non_overlapping = std::is_same::value; if (non_overlapping) { - std::vector covered_bitmap = - domain.dimension_ptr(dim_idx)->covered_vec( - ranges_for_dim, mbr[dim_idx], relevant_ranges); + auto covered_bitmap = domain.dimension_ptr(dim_idx)->covered_vec( + ranges_for_dim, mbr[dim_idx], relevant_ranges); // See if any range is covered. uint64_t count = std::accumulate( diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 45bd9ada56b9..70381200638f 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1584,7 +1584,11 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -1642,7 +1646,11 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -1751,8 +1759,11 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2302,8 +2313,11 @@ void test_apply_dense( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2363,8 +2377,11 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2476,8 +2493,11 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3007,8 +3027,11 @@ void test_apply_sparse( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3068,8 +3091,11 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3840,8 +3866,11 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4131,7 +4160,11 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4487,7 +4520,11 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4808,8 +4845,11 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); Domain domain; - auto dim{ - make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4912,7 +4952,11 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); Domain domain; - auto dim{make_shared(HERE(), "dim1", Datatype::UINT32)}; + auto dim{make_shared( + HERE(), + "dim1", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index af9790ce532f..eefbc92807b9 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -59,8 +59,8 @@ struct QueryPlanFx { URI array_uri(const std::string& uri); + shared_ptr memory_tracker_; TemporaryLocalDirectory temp_dir_; - Config cfg_; shared_ptr logger_; ContextResources resources_; @@ -71,8 +71,8 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { // Create Domain uint64_t dim_dom[2]{0, 1}; uint64_t tile_extent = 1; - shared_ptr dim = - make_shared(HERE(), std::string("dim"), Datatype::UINT64); + shared_ptr dim = make_shared( + HERE(), std::string("dim"), Datatype::UINT64, memory_tracker_); throw_if_not_ok(dim->set_domain(&dim_dom)); throw_if_not_ok(dim->set_tile_extent(&tile_extent)); @@ -110,7 +110,8 @@ URI QueryPlanFx::array_uri(const std::string& array_name) { } QueryPlanFx::QueryPlanFx() - : logger_(make_shared(HERE(), "foo")) + : memory_tracker_(tiledb::test::get_test_memory_tracker()) + , logger_(make_shared(HERE(), "foo")) , resources_(cfg_, logger_, 1, 1, "") , sm_(make_shared(resources_, logger_, cfg_)) { } diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 6ca42b686d73..5cdfdd7166f6 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -117,7 +117,8 @@ Domain create_domain( cell_val_num, range, FilterPipeline(), - tile_extent); + tile_extent, + tiledb::test::get_test_memory_tracker()); dimensions.emplace_back(std::move(dim)); } diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index cdeced731c0b..125d998d6a3e 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -37,6 +37,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" @@ -609,7 +610,8 @@ Range range_from_capnp( /** Deserialize a dimension from a cap'n proto object. */ shared_ptr dimension_from_capnp( - const capnp::Dimension::Reader& dimension_reader) { + const capnp::Dimension::Reader& dimension_reader, + shared_ptr memory_tracker) { Status st; // Deserialize datatype @@ -676,7 +678,8 @@ shared_ptr dimension_from_capnp( cell_val_num, domain, *(filters.get()), - tile_extent); + tile_extent, + memory_tracker); } Status domain_to_capnp( @@ -703,7 +706,8 @@ Status domain_to_capnp( /* Deserialize a domain from a cap'n proto object. */ shared_ptr domain_from_capnp( - const capnp::Domain::Reader& domain_reader) { + const capnp::Domain::Reader& domain_reader, + shared_ptr memory_tracker) { Status st; // Deserialize and validate cell order @@ -743,7 +747,7 @@ shared_ptr domain_from_capnp( std::vector> dims; auto dimensions = domain_reader.getDimensions(); for (auto dimension : dimensions) { - dims.emplace_back(dimension_from_capnp(dimension)); + dims.emplace_back(dimension_from_capnp(dimension, memory_tracker)); } return make_shared(HERE(), cell_order, dims, tile_order); @@ -995,7 +999,7 @@ shared_ptr array_schema_from_capnp( // Deserialize domain // Note: Security validation delegated to invoked API auto domain_reader = schema_reader.getDomain(); - auto domain{domain_from_capnp(domain_reader)}; + auto domain{domain_from_capnp(domain_reader, memory_tracker)}; // Set coords filter pipelines // Note: Security validation delegated to invoked API diff --git a/tiledb/sm/serialization/test/CMakeLists.txt b/tiledb/sm/serialization/test/CMakeLists.txt index ac058b81d9ba..23edbc74937c 100644 --- a/tiledb/sm/serialization/test/CMakeLists.txt +++ b/tiledb/sm/serialization/test/CMakeLists.txt @@ -45,7 +45,7 @@ conclude(unit_test) commence(unit_test capnp_nonempty_domain) this_target_sources(main.cc unit_capnp_nonempty_domain.cc) - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(tiledb_test_support_lib) # Enable serialization target_compile_definitions(unit_capnp_nonempty_domain PRIVATE -DTILEDB_SERIALIZATION) diff --git a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc index ad48184b1904..ccb4bfa5238d 100644 --- a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc +++ b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc @@ -103,8 +103,11 @@ TEST_CASE( Status st; auto schema = make_shared( HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); - std::vector> dims{ - make_shared(HERE(), "index", Datatype::UINT32)}; + std::vector> dims{make_shared( + HERE(), + "index", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker())}; uint32_t domain1[2]{1, 64}; st = dims[0]->set_domain(&domain1[0]); REQUIRE(st.ok()); diff --git a/tiledb/sm/serialization/test/unit_capnp_nonempty_domain.cc b/tiledb/sm/serialization/test/unit_capnp_nonempty_domain.cc index 6c61dc7c08ae..727f09812378 100644 --- a/tiledb/sm/serialization/test/unit_capnp_nonempty_domain.cc +++ b/tiledb/sm/serialization/test/unit_capnp_nonempty_domain.cc @@ -34,6 +34,7 @@ #include +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/serialization/array_schema.h" @@ -44,7 +45,11 @@ TEST_CASE( "Check serialization correctly marks nonempty domain as " "var/fixed size", "[nonemptydomain][serialization]") { - auto dim = make_shared(HERE(), "index", Datatype::UINT32); + auto dim = make_shared( + HERE(), + "index", + Datatype::UINT32, + tiledb::test::get_test_memory_tracker()); uint32_t domain1[2]{1, 64}; auto st = dim->set_domain(&domain1[0]); REQUIRE(st.ok()); diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 0459e8936359..06f757721d81 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -54,9 +54,17 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { // Setup an Array needed to construct the Subarray for testing // add_ranges_list. std::shared_ptr sp_dim1 = - make_shared(HERE(), "d1", Datatype::INT64); + make_shared( + HERE(), + "d1", + Datatype::INT64, + tiledb::test::get_test_memory_tracker()); std::shared_ptr sp_dim2 = - make_shared(HERE(), "d2", Datatype::INT64); + make_shared( + HERE(), + "d2", + Datatype::INT64, + tiledb::test::get_test_memory_tracker()); uint64_t tile_extents[] = {2, 2}; std::vector> dims{sp_dim1, sp_dim2}; std::shared_ptr sp_dom = make_shared( From d1355d2d2cc2f749641200d04b1ac6e2ef276af2 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:57:10 -0500 Subject: [PATCH 221/456] Instrument Metadata class for memory measurement. (#4760) Add `tdb::pmr::map` wrapper as an alias for `std::map`. Migrate class `Metadata` to use scoped allocators for memory tracking. --- TYPE: NO_HISTORY DESC: Instrument Metadata class for memory measurement. Addition of tdb::pmr::map wrapper. --- test/src/unit-capi-array.cc | 6 +- tiledb/api/c_api/group/group_api.cc | 11 +-- tiledb/common/memory_tracker.cc | 4 + tiledb/common/memory_tracker.h | 10 +- tiledb/common/pmr.h | 96 +++++++++++++++++++ tiledb/sm/array/array.cc | 20 ++-- tiledb/sm/array/array.h | 14 +-- tiledb/sm/c_api/tiledb.cc | 12 ++- .../consolidator/array_meta_consolidator.cc | 47 +++------ .../consolidator/group_meta_consolidator.cc | 50 +++------- tiledb/sm/group/group.cc | 38 +++----- tiledb/sm/group/group.h | 11 ++- tiledb/sm/metadata/metadata.cc | 77 +++++++-------- tiledb/sm/metadata/metadata.h | 49 ++++++---- tiledb/sm/metadata/test/CMakeLists.txt | 3 +- tiledb/sm/metadata/test/unit_metadata.cc | 40 ++++---- tiledb/sm/serialization/array.cc | 16 +--- tiledb/sm/serialization/group.cc | 26 +---- 18 files changed, 290 insertions(+), 240 deletions(-) diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 81f2e5b689b1..60f5907ce510 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -2289,8 +2289,8 @@ TEST_CASE_METHOD( // in array open v1 but with separate requests, so we simulate // this here by forcing metadata loading if (!array_v2) { - Metadata* metadata = nullptr; - CHECK(array->array_->metadata(&metadata).ok()); + auto metadata = &array->array_->metadata(); + CHECK(metadata != nullptr); array->array_->non_empty_domain(); } @@ -2340,7 +2340,7 @@ TEST_CASE_METHOD( Datatype type; const void* v_r; uint32_t v_num; - auto new_metadata = new_array->array_->unsafe_metadata(); + auto new_metadata = &new_array->array_->metadata(); new_metadata->get("aaa", &type, &v_num, &v_r); CHECK(static_cast(type) == TILEDB_INT32); CHECK(v_num == 1); diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 0191f6acad7e..74bf9f123007 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -540,14 +540,9 @@ capi_return_t tiledb_serialize_group_metadata( auto buf = tiledb_buffer_handle_t::make_handle(); // Get metadata to serialize, this will load it if it does not exist - tiledb::sm::Metadata* metadata; - auto st = group->group().metadata(&metadata); - if (!st.ok()) { - tiledb_buffer_handle_t::break_handle(buf); - throw StatusException(st); - } + auto metadata = group->group().metadata(); - st = tiledb::sm::serialization::metadata_serialize( + auto st = tiledb::sm::serialization::metadata_serialize( metadata, static_cast(serialize_type), &(buf->buffer())); diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 6e81503d5628..5abec0825de2 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -80,6 +80,8 @@ std::string memory_type_to_str(MemoryType type) { return "TileSums"; case MemoryType::TILE_WRITER_DATA: return "TileWriterData"; + case MemoryType::METADATA: + return "Metadata"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory type: " + val); @@ -112,6 +114,8 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { return "Ephemeral"; case MemoryTrackerType::SCHEMA_EVOLUTION: return "SchemaEvolution"; + case MemoryTrackerType::GROUP: + return "Group"; default: auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory tracker type: " + val); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index bcfc512c4153..492a3fefecd9 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -97,8 +97,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { //** The type of memory to track. */ enum class MemoryType { @@ -115,6 +114,7 @@ enum class MemoryType { ATTRIBUTES, DIMENSION_LABELS, DIMENSIONS, + METADATA, TILE_SUMS, TILE_WRITER_DATA }; @@ -132,6 +132,7 @@ enum class MemoryTrackerType { QUERY_WRITE, CONSOLIDATOR, REST_CLIENT, + GROUP, EPHEMERAL, SCHEMA_EVOLUTION }; @@ -431,7 +432,6 @@ class MemoryTrackerReporter { bool stop_; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_OPEN_ARRAY_MEMORY_TRACKER_H diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index 0005868f4fc3..6245737c269a 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -34,6 +34,7 @@ #ifndef TILEDB_COMMON_PMR_H #define TILEDB_COMMON_PMR_H +#include #include #include @@ -251,10 +252,15 @@ class unordered_map : public pmr_unordered_map { using insert_return_type = typename pmr_unordered_map::insert_return_type; + // Delete all default constructors because they don't require an allocator constexpr unordered_map() = delete; constexpr unordered_map(const unordered_map& other) = delete; constexpr unordered_map(unordered_map&& other) = delete; + // Delete non-allocator aware copy and move assign. + constexpr unordered_map& operator=(const unordered_map& other) = delete; + constexpr unordered_map& operator=(unordered_map&& other) noexcept = delete; + constexpr explicit unordered_map( size_type bucket_count, const Hash& hash, @@ -349,6 +355,96 @@ class unordered_map : public pmr_unordered_map { } }; +/* ********************************* */ +/* PMR MAP DECLARATION */ +/* ********************************* */ +template > +using pmr_map = + std::map>>; + +template > +class map : public pmr_map { + public: + // Type declarations. + using key_type = typename pmr_map::key_type; + using mapped_type = typename pmr_map::mapped_type; + using value_type = typename pmr_map::value_type; + using size_type = typename pmr_map::size_type; + using difference_type = typename pmr_map::difference_type; + using key_compare = typename pmr_map::key_compare; + using allocator_type = typename pmr_map::allocator_type; + using reference = typename pmr_map::reference; + using const_reference = typename pmr_map::const_reference; + using pointer = typename pmr_map::pointer; + using const_pointer = typename pmr_map::const_pointer; + using iterator = typename pmr_map::iterator; + using const_iterator = typename pmr_map::const_iterator; + using reverse_iterator = typename pmr_map::reverse_iterator; + using const_reverse_iterator = + typename pmr_map::const_reverse_iterator; + using node_type = typename pmr_map::node_type; + using insert_return_type = + typename pmr_map::insert_return_type; + + // Delete all default constructors because they don't require an allocator + constexpr map() = delete; + constexpr map(const map& other) = delete; + constexpr map(map&& other) = delete; + + // Delete non-allocator aware copy and move assign. + constexpr map& operator=(const map& other) = delete; + constexpr map& operator=(map&& other) noexcept = delete; + + constexpr explicit map(const Compare& comp, const allocator_type& alloc) + : pmr_map(comp, alloc) { + } + + constexpr explicit map(const allocator_type& alloc) + : pmr_map(alloc) { + } + + template + constexpr map( + InputIt first, + InputIt last, + const Compare& comp, + const allocator_type& alloc) + : pmr_map(first, last, comp, alloc) { + } + + template + constexpr map(InputIt first, InputIt last, const allocator_type& alloc) + : pmr_map(first, last, Compare(), alloc) { + } + + constexpr map(const map& other, const allocator_type& alloc) + : pmr_map(other, alloc) { + } + + constexpr map(map&& other, const allocator_type& alloc) + : pmr_map(other, alloc) { + } + + constexpr map( + std::initializer_list init, + const Compare& comp, + const allocator_type& alloc) + : pmr_map(init, comp, alloc) { + } + + constexpr map( + std::initializer_list init, const allocator_type& alloc) + : pmr_map(init, Compare(), alloc) { + } + + // Declare member class value_compare. + class value_compare : public pmr_map::value_compare { + public: + constexpr bool operator()( + const value_type& lhs, const value_type& rhs) const; + }; +}; + } // namespace tiledb::common::pmr #endif // TILEDB_COMMON_PMR_H diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 0698944ec969..cf8e9bea8622 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -60,8 +60,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class ArrayException : public StatusException { public: @@ -139,6 +138,7 @@ Status Array::open_without_fragments( opened_array_ = make_shared( HERE(), resources_, + memory_tracker_, array_uri_, encryption_type, encryption_key, @@ -317,6 +317,7 @@ Status Array::open( opened_array_ = make_shared( HERE(), resources_, + memory_tracker_, array_uri_, encryption_type, encryption_key, @@ -832,6 +833,7 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { opened_array_ = make_shared( HERE(), resources_, + memory_tracker_, array_uri_, key->encryption_type(), key->key().data(), @@ -1048,15 +1050,13 @@ Metadata* Array::unsafe_metadata() { return &opened_array_->metadata(); } -Status Array::metadata(Metadata** metadata) { +Metadata& Array::metadata() { // Load array metadata for array opened for reads, if not loaded yet if (query_type_ == QueryType::READ && !metadata_loaded()) { - RETURN_NOT_OK(load_metadata()); + throw_if_not_ok(load_metadata()); } - *metadata = &opened_array_->metadata(); - - return Status::Ok(); + return opened_array_->metadata(); } const NDRange Array::non_empty_domain() { @@ -1620,6 +1620,7 @@ void Array::set_serialized_array_open() { opened_array_ = make_shared( HERE(), resources_, + memory_tracker_, array_uri_, EncryptionType::NO_ENCRYPTION, nullptr, @@ -1691,5 +1692,4 @@ void ensure_supported_schema_version_for_read(format_version_t version) { } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 57bf6ef09156..6f304ae8826d 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -49,8 +49,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class ArraySchema; class SchemaEvolution; @@ -81,6 +80,7 @@ class OpenedArray { * Construct a new Opened Array object. * * @param resources The context resources to use. + * @param memory_tracker The array's MemoryTracker. * @param array_uri The URI of the array. * @param encryption_type Encryption type. * @param key_bytes Encryption key data. @@ -91,6 +91,7 @@ class OpenedArray { */ OpenedArray( ContextResources& resources, + shared_ptr memory_tracker, const URI& array_uri, EncryptionType encryption_type, const void* key_bytes, @@ -100,7 +101,7 @@ class OpenedArray { bool is_remote) : array_dir_(ArrayDirectory(resources, array_uri)) , array_schema_latest_(nullptr) - , metadata_() + , metadata_(memory_tracker) , metadata_loaded_(false) , non_empty_domain_computed_(false) , encryption_key_(make_shared(HERE())) @@ -730,7 +731,7 @@ class Array { std::optional metadata_type(const char* key); /** Retrieves the array metadata object. */ - Status metadata(Metadata** metadata); + Metadata& metadata(); /** * Retrieves the array metadata object. @@ -1050,7 +1051,6 @@ class Array { void set_array_closed(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_ARRAY_H diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index ebf5888ea0f3..c8adf5ebd270 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -4030,9 +4030,13 @@ int32_t tiledb_serialize_array_metadata( auto buf = tiledb_buffer_handle_t::make_handle(); // Get metadata to serialize, this will load it if it does not exist - tiledb::sm::Metadata* metadata; - if (SAVE_ERROR_CATCH(ctx, array->array_->metadata(&metadata))) { - tiledb_buffer_handle_t::break_handle(buf); + sm::Metadata* metadata = nullptr; + try { + metadata = &array->array_->metadata(); + } catch (StatusException& e) { + auto st = Status_Error(e.what()); + LOG_STATUS_NO_RETURN_VALUE(st); + save_error(ctx, st); return TILEDB_ERR; } diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 1677120d0835..cde5d10eb2c7 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -65,7 +65,6 @@ Status ArrayMetaConsolidator::consolidate( const void* encryption_key, uint32_t key_length) { auto timer_se = stats_->start_timer("consolidate_array_meta"); - check_array_uri(array_name); // Open array for reading @@ -86,30 +85,11 @@ Status ArrayMetaConsolidator::consolidate( QueryType::WRITE, encryption_type, encryption_key, key_length), throw_if_not_ok(array_for_reads.close())); - // Swap the in-memory metadata between the two arrays. - // After that, the array for writes will store the (consolidated by - // the way metadata loading works) metadata of the array for reads - Metadata* metadata_r; - auto st = array_for_reads.metadata(&metadata_r); - if (!st.ok()) { - throw_if_not_ok(array_for_reads.close()); - throw_if_not_ok(array_for_writes.close()); - return st; - } - Metadata* metadata_w; - st = array_for_writes.metadata(&metadata_w); - if (!st.ok()) { - throw_if_not_ok(array_for_reads.close()); - throw_if_not_ok(array_for_writes.close()); - return st; - } - metadata_r->swap(metadata_w); - - // Metadata uris to delete - const auto to_vacuum = metadata_w->loaded_metadata_uris(); - - // Get the new URI name for consolidated metadata - URI new_uri = metadata_w->get_uri(array_uri); + // Copy-assign the read metadata into the metadata of the array for writes + auto& metadata_r = array_for_reads.metadata(); + array_for_writes.opened_array()->metadata() = metadata_r; + URI new_uri = metadata_r.get_uri(array_uri); + const auto& to_vacuum = metadata_r.loaded_metadata_uris(); // Write vac files relative to the array URI. This was fixed for reads in // version 19 so only do this for arrays starting with version 19. @@ -119,20 +99,19 @@ Status ArrayMetaConsolidator::consolidate( base_uri_size = array_for_reads.array_uri().to_string().size(); } - // Close arrays - RETURN_NOT_OK_ELSE( - array_for_reads.close(), throw_if_not_ok(array_for_writes.close())); - throw_if_not_ok(array_for_writes.close()); - - // Write vacuum file + // Prepare vacuum file URI vac_uri = URI(new_uri.to_string() + constants::vacuum_file_suffix); - std::stringstream ss; for (const auto& uri : to_vacuum) { ss << uri.to_string().substr(base_uri_size) << "\n"; } - auto data = ss.str(); + + // Close arrays + throw_if_not_ok(array_for_reads.close()); + throw_if_not_ok(array_for_writes.close()); + + // Write vacuum file RETURN_NOT_OK( storage_manager_->vfs()->write(vac_uri, data.c_str(), data.size())); RETURN_NOT_OK(storage_manager_->vfs()->close_file(vac_uri)); diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index 589b067fe4c7..ba0a74d55882 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -64,7 +64,6 @@ GroupMetaConsolidator::GroupMetaConsolidator( Status GroupMetaConsolidator::consolidate( const char* group_name, EncryptionType, const void*, uint32_t) { auto timer_se = stats_->start_timer("consolidate_group_meta"); - check_array_uri(group_name); // Open group for reading @@ -81,44 +80,25 @@ Status GroupMetaConsolidator::consolidate( group_for_writes.open(QueryType::WRITE), throw_if_not_ok(group_for_reads.close())); - // Swap the in-memory metadata between the two groups. - // After that, the group for writes will store the (consolidated by - // the way metadata loading works) metadata of the group for reads - Metadata* metadata_r; - auto st = group_for_reads.metadata(&metadata_r); - if (!st.ok()) { - throw_if_not_ok(group_for_reads.close()); - throw_if_not_ok(group_for_writes.close()); - return st; - } - Metadata* metadata_w; - st = group_for_writes.metadata(&metadata_w); - if (!st.ok()) { - throw_if_not_ok(group_for_reads.close()); - throw_if_not_ok(group_for_writes.close()); - return st; - } - metadata_r->swap(metadata_w); - - // Metadata uris to delete - const auto to_vacuum = metadata_w->loaded_metadata_uris(); - - // Get the new URI name for consolidated metadata - URI new_uri = metadata_w->get_uri(group_uri); - - // Close groups - RETURN_NOT_OK_ELSE( - group_for_reads.close(), throw_if_not_ok(group_for_writes.close())); - RETURN_NOT_OK(group_for_writes.close()); + // Copy-assign the read metadata into the metadata of the group for writes + auto metadata_r = group_for_reads.metadata(); + *(group_for_writes.metadata()) = *metadata_r; + URI new_uri = metadata_r->get_uri(group_uri); + const auto& to_vacuum = metadata_r->loaded_metadata_uris(); - // Write vacuum file + // Prepare vacuum file URI vac_uri = URI(new_uri.to_string() + constants::vacuum_file_suffix); - std::stringstream ss; - for (const auto& uri : to_vacuum) + for (const auto& uri : to_vacuum) { ss << uri.to_string() << "\n"; - + } auto data = ss.str(); + + // Close groups + throw_if_not_ok(group_for_reads.close()); + throw_if_not_ok(group_for_writes.close()); + + // Write vacuum file RETURN_NOT_OK( storage_manager_->vfs()->write(vac_uri, data.c_str(), data.size())); RETURN_NOT_OK(storage_manager_->vfs()->close_file(vac_uri)); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 181b71d632dc..4aea136fb6ac 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ #include "tiledb/sm/group/group.h" #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/encryption_type.h" @@ -65,10 +66,12 @@ Group::Group( ContextResources& resources, const URI& group_uri, StorageManager* storage_manager) - : group_uri_(group_uri) + : memory_tracker_(resources.create_memory_tracker()) + , group_uri_(group_uri) , storage_manager_(storage_manager) , config_(storage_manager_->config()) , remote_(group_uri.is_tiledb()) + , metadata_(memory_tracker_) , metadata_loaded_(false) , is_open_(false) , query_type_(QueryType::READ) @@ -76,6 +79,7 @@ Group::Group( , timestamp_end_(UINT64_MAX) , encryption_key_(tdb::make_shared(HERE())) , resources_(resources) { + memory_tracker_->set_type(MemoryTrackerType::GROUP); } Status Group::open( @@ -471,22 +475,16 @@ std::optional Group::metadata_type(const char* key) { return metadata_.metadata_type(key); } -Metadata* Group::unsafe_metadata() { - return &metadata_; -} - -const Metadata* Group::metadata() const { - return &metadata_; -} - -Status Group::metadata(Metadata** metadata) { +Metadata* Group::metadata() { // Load group metadata, if not loaded yet if (!metadata_loaded_) load_metadata(); - *metadata = &metadata_; + return &metadata_; +} - return Status::Ok(); +Metadata* Group::unsafe_metadata() { + return &metadata_; } void Group::set_metadata_loaded(const bool metadata_loaded) { @@ -713,23 +711,17 @@ void Group::load_metadata() { rest_client->post_group_metadata_from_rest(group_uri_, this)); } else { assert(group_dir_->loaded()); - load_metadata_from_storage(group_dir_, *encryption_key_, &metadata_); + load_metadata_from_storage(group_dir_, *encryption_key_); } metadata_loaded_ = true; } void Group::load_metadata_from_storage( const shared_ptr& group_dir, - const EncryptionKey& encryption_key, - Metadata* metadata) { + const EncryptionKey& encryption_key) { [[maybe_unused]] auto timer_se = resources_.stats().start_timer("group_load_metadata_from_storage"); - // Special case - if (metadata == nullptr) { - return; - } - // Determine which group metadata to load const auto& group_metadata_to_load = group_dir->group_meta_uris(); @@ -758,8 +750,8 @@ void Group::load_metadata_from_storage( resources_.stats().add_counter("group_read_group_meta_size", meta_size); // Copy the deserialized metadata into the original Metadata object - *metadata = Metadata::deserialize(metadata_tiles); - metadata->set_loaded_metadata_uris(group_metadata_to_load); + metadata_ = Metadata::deserialize(metadata_tiles); + metadata_.set_loaded_metadata_uris(group_metadata_to_load); } void Group::group_open_for_reads() { diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 0cb415405372..2dfc5aab22fa 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -195,7 +195,7 @@ class Group { std::optional metadata_type(const char* key); /** Retrieves the group metadata object. */ - Status metadata(Metadata** metadata); + Metadata* metadata(); /** * Retrieves the group metadata object. @@ -208,7 +208,6 @@ class Group { * REST. A lock should already by taken before load_metadata is called. */ Metadata* unsafe_metadata(); - const Metadata* metadata() const; /** * Set metadata loaded @@ -386,6 +385,9 @@ class Group { /* ********************************* */ /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** Memory tracker for the group. */ + shared_ptr memory_tracker_; + /** The group URI. */ URI group_uri_; @@ -460,8 +462,7 @@ class Group { */ void load_metadata_from_storage( const shared_ptr& group_dir, - const EncryptionKey& encryption_key, - Metadata* metadata); + const EncryptionKey& encryption_key); /** Opens an group for reads. */ void group_open_for_reads(); diff --git a/tiledb/sm/metadata/metadata.cc b/tiledb/sm/metadata/metadata.cc index fbce14b864de..d5b585147e6a 100644 --- a/tiledb/sm/metadata/metadata.cc +++ b/tiledb/sm/metadata/metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ #include "tiledb/sm/metadata/metadata.h" #include "tiledb/common/exception/exception.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/tdb_time.h" @@ -55,43 +56,51 @@ class MetadataException : public StatusException { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - -Metadata::Metadata() - : Metadata(std::map()) { -} - -Metadata::Metadata(const std::map& metadata_map) - : metadata_map_(metadata_map) +Metadata::Metadata(shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , metadata_map_(memory_tracker_->get_resource(MemoryType::METADATA)) + , metadata_index_(memory_tracker_->get_resource(MemoryType::METADATA)) , timestamp_range_([]() -> std::pair { auto t = utils::time::timestamp_now_ms(); return std::make_pair(t, t); - }()) { + }()) + , loaded_metadata_uris_( + memory_tracker_->get_resource(MemoryType::METADATA)) { build_metadata_index(); } -Metadata::Metadata(const Metadata& rhs) - : metadata_map_(rhs.metadata_map_) - , timestamp_range_(rhs.timestamp_range_) - , loaded_metadata_uris_(rhs.loaded_metadata_uris_) - , uri_(rhs.uri_) { - if (!rhs.metadata_index_.empty()) - build_metadata_index(); -} +/* ********************************* */ +/* API */ +/* ********************************* */ + +Metadata& Metadata::operator=(Metadata& other) { + clear(); + for (auto& [k, v] : other.metadata_map_) { + metadata_map_.emplace(k, v); + } -Metadata& Metadata::operator=(const Metadata& other) { - metadata_map_ = other.metadata_map_; timestamp_range_ = other.timestamp_range_; - loaded_metadata_uris_ = other.loaded_metadata_uris_; - uri_ = other.uri_; + + for (auto& uri : other.loaded_metadata_uris_) { + loaded_metadata_uris_.emplace_back(uri); + } + build_metadata_index(); + return *this; } -Metadata::~Metadata() = default; +Metadata& Metadata::operator=( + std::map&& md_map) { + clear(); + for (auto& [k, v] : md_map) { + metadata_map_.emplace(k, v); + } -/* ********************************* */ -/* API */ -/* ********************************* */ + build_metadata_index(); + + return *this; +} void Metadata::clear() { metadata_map_.clear(); @@ -115,14 +124,13 @@ void Metadata::generate_uri(const URI& array_uri) { .join_path(ts_name); } -Metadata Metadata::deserialize( +std::map Metadata::deserialize( const std::vector>& metadata_tiles) { if (metadata_tiles.empty()) { - return Metadata(); + return {}; } - std::map metadata_map; - Status st; + std::map metadata_map; uint32_t key_len; char del; size_t value_len; @@ -157,7 +165,7 @@ Metadata Metadata::deserialize( } } - return Metadata(metadata_map); + return metadata_map; } void Metadata::serialize(Serializer& serializer) const { @@ -313,17 +321,10 @@ void Metadata::set_loaded_metadata_uris( timestamp_range_.second = loaded_metadata_uris.back().timestamp_range_.second; } -const std::vector& Metadata::loaded_metadata_uris() const { +const tdb::pmr::vector& Metadata::loaded_metadata_uris() const { return loaded_metadata_uris_; } -void Metadata::swap(Metadata* metadata) { - std::swap(metadata_map_, metadata->metadata_map_); - std::swap(metadata_index_, metadata->metadata_index_); - std::swap(timestamp_range_, metadata->timestamp_range_); - std::swap(loaded_metadata_uris_, metadata->loaded_metadata_uris_); -} - void Metadata::reset(uint64_t timestamp) { clear(); timestamp = (timestamp != 0) ? timestamp : utils::time::timestamp_now_ms(); diff --git a/tiledb/sm/metadata/metadata.h b/tiledb/sm/metadata/metadata.h index 7041223234a5..2e45614eaea8 100644 --- a/tiledb/sm/metadata/metadata.h +++ b/tiledb/sm/metadata/metadata.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/tile/tile.h" #include "tiledb/storage_format/serialization/serializers.h" @@ -50,6 +51,7 @@ namespace tiledb::sm { class Buffer; class ConstBuffer; +class MemoryTracker; enum class Datatype : uint8_t; /** @@ -81,31 +83,39 @@ class Metadata { }; /** Iterator type for iterating over metadata values. */ - typedef std::map::const_iterator iterator; + typedef tdb::pmr::map::const_iterator iterator; /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Constructor. */ - explicit Metadata(); + /** Default constructor is deleted. */ + Metadata() = delete; /** Constructor. */ - Metadata(const std::map& metadata_map); - - /** Copy constructor. */ - Metadata(const Metadata& rhs); + Metadata(shared_ptr memory_tracker); - /** Copy assignment. */ - Metadata& operator=(const Metadata& other); + DISABLE_COPY(Metadata); + DISABLE_MOVE_AND_MOVE_ASSIGN(Metadata); /** Destructor. */ - ~Metadata(); + ~Metadata() = default; /* ********************************* */ /* API */ /* ********************************* */ + /** Copy assignment. */ + Metadata& operator=(Metadata& other); + + /** Assignment via std::map. */ + Metadata& operator=(std::map&& md_map); + + /** Returns the memory tracker. */ + inline shared_ptr memory_tracker() { + return memory_tracker_; + } + /** Clears the metadata. */ void clear(); @@ -120,7 +130,7 @@ class Metadata { * assumed to be sorted on time. The function will take care of any * deleted or overwritten metadata items considering the order. */ - static Metadata deserialize( + static std::map deserialize( const std::vector>& metadata_tiles); /** Serializes all key-value metadata items into the input buffer. */ @@ -204,10 +214,7 @@ class Metadata { * Returns the URIs of the metadata files that have been loaded * to this object. */ - const std::vector& loaded_metadata_uris() const; - - /** Swaps the contents between the object and the input. */ - void swap(Metadata* metadata); + const tdb::pmr::vector& loaded_metadata_uris() const; /** * Clears the metadata and assigns the input timestamp to @@ -233,15 +240,19 @@ class Metadata { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker. */ + shared_ptr memory_tracker_; + /** A map from metadata key to metadata value. */ - std::map metadata_map_; + tdb::pmr::map metadata_map_; /** * A vector pointing to all the values in `metadata_map_`. It facilitates * searching metadata from index. Used only for reading metadata (inapplicable * when writing metadata). */ - std::vector> metadata_index_; + tdb::pmr::vector> + metadata_index_; /** Mutex for thread-safety. */ mutable std::mutex mtx_; @@ -256,7 +267,7 @@ class Metadata { * The URIs of the metadata files that have been loaded to this object. * This is needed to know which files to delete upon consolidation. */ - std::vector loaded_metadata_uris_; + tdb::pmr::vector loaded_metadata_uris_; /** The URI of the array metadata file. */ URI uri_; diff --git a/tiledb/sm/metadata/test/CMakeLists.txt b/tiledb/sm/metadata/test/CMakeLists.txt index c4c60b84bd19..326e66e796a3 100644 --- a/tiledb/sm/metadata/test/CMakeLists.txt +++ b/tiledb/sm/metadata/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,5 +29,6 @@ commence(unit_test metadata) this_target_object_libraries(metadata mem_helpers) # The dependency on the `tile` object library is suspect, but here for now. this_target_object_libraries(tile) + this_target_link_libraries(tiledb_test_support_lib) this_target_sources(main.cc unit_metadata.cc) conclude(unit_test) diff --git a/tiledb/sm/metadata/test/unit_metadata.cc b/tiledb/sm/metadata/test/unit_metadata.cc index bc09b5fdcfc2..dabe928194c7 100644 --- a/tiledb/sm/metadata/test/unit_metadata.cc +++ b/tiledb/sm/metadata/test/unit_metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ #include #include #include "../metadata.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/common/common.h" #include "tiledb/common/dynamic_memory/dynamic_memory.h" #include "tiledb/sm/buffer/buffer.h" @@ -43,20 +44,28 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; +using tiledb::test::create_test_memory_tracker; template inline T& buffer_metadata(void* p) { return *static_cast(static_cast(static_cast(p) + n)); } +TEST_CASE("Metadata: Constructor validation", "[metadata][constructor]") { + auto tracker = create_test_memory_tracker(); + + SECTION("memory_tracker") { + REQUIRE_NOTHROW(Metadata(tracker)); + } +} + TEST_CASE( "Metadata: Test metadata deserialization", "[metadata][deserialization]") { - auto memory_tracker = tiledb::test::create_test_memory_tracker(); - + auto tracker = create_test_memory_tracker(); std::vector> metadata_tiles; - Metadata metadata_to_serialize1, metadata_to_serialize2, - metadata_to_serialize3; + Metadata metadata_to_serialize1(tracker), metadata_to_serialize2(tracker), + metadata_to_serialize3(tracker), meta(tracker); // key_1:a, value_1:100,200 std::string key_1 = "key1"; @@ -77,8 +86,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer1; metadata_to_serialize1.serialize(size_computation_serializer1); - WriterTile tile1{WriterTile::from_generic( - size_computation_serializer1.size(), memory_tracker)}; + WriterTile tile1{ + WriterTile::from_generic(size_computation_serializer1.size(), tracker)}; Serializer serializer1(tile1.data(), tile1.size()); metadata_to_serialize1.serialize(serializer1); @@ -87,8 +96,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer2; metadata_to_serialize2.serialize(size_computation_serializer2); - WriterTile tile2{WriterTile::from_generic( - size_computation_serializer2.size(), memory_tracker)}; + WriterTile tile2{ + WriterTile::from_generic(size_computation_serializer2.size(), tracker)}; Serializer serializer2(tile2.data(), tile2.size()); metadata_to_serialize2.serialize(serializer2); @@ -98,8 +107,8 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer3; metadata_to_serialize3.serialize(size_computation_serializer3); - WriterTile tile3{WriterTile::from_generic( - size_computation_serializer3.size(), memory_tracker)}; + WriterTile tile3{ + WriterTile::from_generic(size_computation_serializer3.size(), tracker)}; Serializer serializer3(tile3.data(), tile3.size()); metadata_to_serialize3.serialize(serializer3); @@ -114,7 +123,7 @@ TEST_CASE( tile1.size(), tile1.filtered_buffer().data(), tile1.filtered_buffer().size(), - memory_tracker); + tracker); memcpy(metadata_tiles[0]->data(), tile1.data(), tile1.size()); metadata_tiles[1] = tdb::make_shared( @@ -126,7 +135,7 @@ TEST_CASE( tile2.size(), tile2.filtered_buffer().data(), tile2.filtered_buffer().size(), - memory_tracker); + tracker); memcpy(metadata_tiles[1]->data(), tile2.data(), tile2.size()); metadata_tiles[2] = tdb::make_shared( @@ -138,11 +147,10 @@ TEST_CASE( tile3.size(), tile3.filtered_buffer().data(), tile3.filtered_buffer().size(), - memory_tracker); + tracker); memcpy(metadata_tiles[2]->data(), tile3.data(), tile3.size()); - auto meta{Metadata::deserialize(metadata_tiles)}; - + meta = Metadata::deserialize(metadata_tiles); Datatype type; uint32_t v_num; diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index e74b17ae5afe..72c6a1c142e2 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -55,9 +55,7 @@ using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { -namespace serialization { +namespace tiledb::sm::serialization { class ArraySerializationException : public StatusException { public: @@ -205,10 +203,8 @@ Status array_to_capnp( // If this is the Cloud server, it should load and serialize metadata // If this is the client, it should have previously received the array // metadata from the Cloud server, so it should just serialize it - Metadata* metadata = nullptr; - // Get metadata. If not loaded, load it first. - RETURN_NOT_OK(array->metadata(&metadata)); - RETURN_NOT_OK(metadata_to_capnp(metadata, &array_metadata_builder)); + auto& metadata = array->metadata(); + RETURN_NOT_OK(metadata_to_capnp(&metadata, &array_metadata_builder)); } } else { if (array->non_empty_domain_computed()) { @@ -740,6 +736,4 @@ Status metadata_deserialize(Metadata*, SerializationType, const Buffer&) { #endif // TILEDB_SERIALIZATION -} // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::serialization diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index 3b5d59a27adb..de8611677273 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -58,9 +58,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { -namespace serialization { +namespace tiledb::sm::serialization { #ifdef TILEDB_SERIALIZATION @@ -72,13 +70,7 @@ Status group_metadata_to_capnp( auto config_builder = group_metadata_builder->initConfig(); RETURN_NOT_OK(config_to_capnp(group->config(), &config_builder)); - Metadata* metadata; - if (load) { - RETURN_NOT_OK(group->metadata(&metadata)); - } else { - metadata = const_cast(group->metadata()); - } - + auto metadata = group->metadata(); if (metadata->num()) { auto metadata_builder = group_metadata_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &metadata_builder)); @@ -165,13 +157,7 @@ Status group_details_to_capnp( } } - Metadata* metadata; - if (group->group_uri().is_tiledb()) { - metadata = const_cast(group->metadata()); - } else { - RETURN_NOT_OK(group->metadata(&metadata)); - } - + auto metadata = group->metadata(); if (metadata->num()) { auto group_metadata_builder = group_details_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &group_metadata_builder)); @@ -831,6 +817,4 @@ Status group_metadata_serialize(Group*, SerializationType, Buffer*, bool) { #endif // TILEDB_SERIALIZATION -} // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::serialization From 0a349d6070af2c9ffa73ecd3e9f8e7d83254c36b Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 5 Mar 2024 06:45:02 -0600 Subject: [PATCH 222/456] Fix missing memory_tracker_type_to_str (#4785) I forgot to add a switch case for ENUMERATION_CREATE. This changes the implementations for both memory_type_to_str and memory_tracker_type_to_str to generate compiler errors when a valid enum case is not covered. I've also added unit tests to cover conversions for all type values as well. --- TYPE: NO_HISTORY DESC: Fix missing memory_tracker_type_to_str --- tiledb/common/memory_tracker.cc | 8 +++ tiledb/common/memory_tracker.h | 16 +++++ tiledb/common/test/CMakeLists.txt | 6 ++ .../common/test/unit_memory_tracker_types.cc | 72 +++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 tiledb/common/test/unit_memory_tracker_types.cc diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 5abec0825de2..b7b24e17cdee 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -86,6 +86,9 @@ std::string memory_type_to_str(MemoryType type) { auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory type: " + val); } + + auto val = std::to_string(static_cast(type)); + throw std::logic_error("Invalid memory type: " + val); } std::string memory_tracker_type_to_str(MemoryTrackerType type) { @@ -100,6 +103,8 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { return "ArrayRead"; case MemoryTrackerType::ARRAY_WRITE: return "ArrayWrite"; + case MemoryTrackerType::ENUMERATION_CREATE: + return "EnumerationCreate"; case MemoryTrackerType::FRAGMENT_INFO_LOAD: return "FragmentInfoLoad"; case MemoryTrackerType::QUERY_READ: @@ -120,6 +125,9 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { auto val = std::to_string(static_cast(type)); throw std::logic_error("Invalid memory tracker type: " + val); } + + auto val = std::to_string(static_cast(type)); + throw std::logic_error("Invalid memory tracker type: " + val); } uint64_t MemoryTrackerResource::get_count() { diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 492a3fefecd9..8858fe07210d 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -119,6 +119,14 @@ enum class MemoryType { TILE_WRITER_DATA }; +/** + * Return a string representation of type + * + * @param type The MemoryType to convert. + * @return A string representation. + */ +std::string memory_type_to_str(MemoryType type); + /** The type of MemoryTracker. */ enum class MemoryTrackerType { ANONYMOUS, @@ -137,6 +145,14 @@ enum class MemoryTrackerType { SCHEMA_EVOLUTION }; +/** + * Return a string representation of type + * + * @param type The MemoryTrackerType to convert. + * @return A string representation. + */ +std::string memory_tracker_type_to_str(MemoryTrackerType type); + class MemoryTrackerResource : public tdb::pmr::memory_resource { public: // Disable all default generated constructors. diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 6e09d2fa3d55..552212e48a3d 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -35,3 +35,9 @@ commence(unit_test experimental) unit_is_not_experimental.cc> ) conclude(unit_test) + +commence(unit_test memory_tracker_types) + this_target_sources(main.cc unit_memory_tracker_types.cc) + this_target_object_libraries(baseline) +conclude(unit_test) + diff --git a/tiledb/common/test/unit_memory_tracker_types.cc b/tiledb/common/test/unit_memory_tracker_types.cc new file mode 100644 index 000000000000..e43a85d1952a --- /dev/null +++ b/tiledb/common/test/unit_memory_tracker_types.cc @@ -0,0 +1,72 @@ +/** + * @file unit_memory_tracker_types.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file tests the memory tracker to_str functions. + */ + +#include + +#include +#include "tiledb/common/memory_tracker.h" + +using namespace tiledb::common; + +TEST_CASE("memory_type_to_str") { + auto max = static_cast(tiledb::sm::MemoryType::TILE_WRITER_DATA); + size_t failures = 0; + for (int8_t i = 0; i < 127; i++) { + auto val = static_cast(i); + if (i <= max) { + REQUIRE_NOTHROW(tiledb::sm::memory_type_to_str(val)); + } else { + REQUIRE_THROWS(tiledb::sm::memory_type_to_str(val)); + failures += 1; + } + } + // Technically, we could eventually have more than 127 enumeration values + // and this test would pass when it shouldn't. + REQUIRE(failures > 0); +} + +TEST_CASE("memory_tracker_type_to_str") { + auto max = static_cast(tiledb::sm::MemoryTrackerType::SCHEMA_EVOLUTION); + size_t failures = 0; + for (int8_t i = 0; i < 127; i++) { + auto val = static_cast(i); + if (i <= max) { + REQUIRE_NOTHROW(tiledb::sm::memory_tracker_type_to_str(val)); + } else { + REQUIRE_THROWS(tiledb::sm::memory_tracker_type_to_str(val)); + failures += 1; + } + } + // Technically, we could eventually have more than 127 enumeration values + // and this test would pass when it shouldn't. + REQUIRE(failures > 0); +} From 9bd16ba97422fac995d1737c6b9783a790390bf6 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Tue, 5 Mar 2024 11:19:37 -0600 Subject: [PATCH 223/456] Fix segfault in ArraySchema::check (#4787) Previously we weren't accounting for when enumerations weren't loaded and a check was performed. This lead to attempts to dereference the nullptr when the enumeration hadn't been loaded. Given that enumerations are only unloaded when reading an array, that means these checks were already run when the schema was written so ignoring them here is fine since they're just enforcing a maximum size constraint. --- TYPE: BUG DESC: Fix segfault in ArraySchema::check --- test/src/unit-enumerations.cc | 12 ++++++++++++ tiledb/sm/array_schema/array_schema.cc | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index e35cbc30d120..15d874c31fa2 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1452,6 +1452,18 @@ TEST_CASE_METHOD( REQUIRE_THROWS_WITH(schema->check(cfg_), matcher); } +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - No Segfault on Check", + "[enumeration][array-scehma][size-check]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + // Schema has unloaded enumerations at this point. Make sure that check + // doesn't segfault. + REQUIRE_NOTHROW(schema->check(cfg_)); +} + TEST_CASE_METHOD( EnumerationFx, "ArraySchema - Many Large Enumerations", diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index f35ced27396e..e5bee621663b 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -586,6 +586,12 @@ void ArraySchema::check_enumerations(const Config& cfg) const { uint64_t total_size = 0; for (const auto& pair : enumeration_map_) { + if (!pair.second) { + // We don't have an Array instance at this point so the best we can do + // is just avoid segfaulting when we attempt to check with unloaded + // enumerations. + continue; + } uint64_t size = pair.second->data().size() + pair.second->offsets().size(); if (size > max_size.value()) { throw ArraySchemaException( From faa42b9d70695f0a94ae062a27f8866720bff23b Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Tue, 5 Mar 2024 12:47:05 -0500 Subject: [PATCH 224/456] Add memory tracking for Domain. (#4758) This adds a MemoryTracker to Domain and updates class members to use `tdb::pmr` containers. --- TYPE: NO_HISTORY DESC: Add memory tracking for Domain --- test/src/unit-Reader.cc | 2 +- test/src/unit-capi-array.cc | 9 +- test/src/unit-cppapi-schema-evolution.cc | 3 +- test/src/unit-enumerations.cc | 10 +- test/src/unit-request-handlers.cc | 35 ++-- tiledb/api/c_api/domain/domain_api.cc | 11 +- tiledb/api/c_api/domain/domain_api_internal.h | 9 +- .../test/compile_capi_domain_stub_main.cc | 3 +- tiledb/common/memory_tracker.cc | 2 + tiledb/common/memory_tracker.h | 1 + tiledb/sm/array/test/unit_consistency.h | 5 +- tiledb/sm/array_schema/array_schema.h | 1 + tiledb/sm/array_schema/dimension_label.cc | 7 +- tiledb/sm/array_schema/domain.cc | 57 +++--- tiledb/sm/array_schema/domain.h | 37 ++-- .../test/array_schema_test_support.h | 17 +- .../sm/array_schema/test/unit_domain_data.cc | 6 +- .../sm/array_schema/test/unit_tile_domain.cc | 10 + tiledb/sm/array_schema/tile_domain.h | 5 +- tiledb/sm/c_api/tiledb_filestore.cc | 2 +- .../test/unit_fragment_consolidator.cc | 7 +- tiledb/sm/filter/webp_filter.h | 1 + tiledb/sm/fragment/fragment_info.cc | 1 + tiledb/sm/query/test/unit_query_condition.cc | 183 +++++++++--------- tiledb/sm/query_plan/test/unit_query_plan.cc | 8 +- tiledb/sm/rtree/test/unit_rtree.cc | 140 ++++++++------ tiledb/sm/serialization/array.h | 1 + tiledb/sm/serialization/array_schema.cc | 3 +- tiledb/sm/serialization/array_schema.h | 2 + tiledb/sm/serialization/fragment_info.h | 2 + tiledb/sm/serialization/query.h | 1 + tiledb/sm/serialization/test/CMakeLists.txt | 2 +- .../test/unit_capnp_array_schema.cc | 8 +- tiledb/sm/storage_manager/storage_manager.cc | 1 + .../sm/subarray/test/unit_add_ranges_list.cc | 6 +- 35 files changed, 333 insertions(+), 265 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 4f17aa0013d0..6b053017984e 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -248,7 +248,7 @@ TEST_CASE_METHOD( auto d2{make_shared(HERE(), "d2", Datatype::INT32, tracker_)}; CHECK(d2->set_domain(&domain_vec[2]).ok()); CHECK(d2->set_tile_extent(&tile_extents_vec[1]).ok()); - auto dom{make_shared(HERE())}; + auto dom{make_shared(HERE(), tracker_)}; CHECK(dom->add_dimension(d1).ok()); CHECK(dom->add_dimension(d2).ok()); diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 60f5907ce510..123212deb8d6 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -79,6 +79,9 @@ using namespace tiledb::common; using namespace tiledb::sm; struct ArrayFx { + // The memory tracker + shared_ptr memory_tracker_; + // TileDB context tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; @@ -116,7 +119,8 @@ static const std::string test_ca_file = std::string(TILEDB_TEST_INPUTS_DIR) + "/test_certs/public.crt"; ArrayFx::ArrayFx() - : fs_vec_(vfs_test_get_fs_vec()) { + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); } @@ -2586,13 +2590,12 @@ TEST_CASE_METHOD( 1, &buff); REQUIRE(rc == TILEDB_OK); - auto memory_tracker = ctx_->resources().create_memory_tracker(); auto st = tiledb::sm::serialization::array_deserialize( array->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), ctx_->storage_manager(), - memory_tracker); + memory_tracker_); REQUIRE(st.ok()); // 6. Server: Close array and clean up diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 0ce1c1921b56..83a64f6e33e0 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -821,7 +821,8 @@ TEST_CASE( int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared( + HERE(), tiledb::test::get_test_memory_tracker()); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 15d874c31fa2..b96bb5a98069 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -141,11 +141,11 @@ struct EnumerationFx { void rm_array(); + shared_ptr memory_tracker_; URI uri_; Config cfg_; Context ctx_; EncryptionKey enc_key_; - shared_ptr memory_tracker_; }; template @@ -2554,9 +2554,9 @@ struct TypeParams { }; EnumerationFx::EnumerationFx() - : uri_("enumeration_test_array") - , ctx_(cfg_) - , memory_tracker_(tiledb::test::create_test_memory_tracker()) { + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , uri_("enumeration_test_array") + , ctx_(cfg_) { rm_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); memory_tracker_ = tiledb::test::create_test_memory_tracker(); @@ -2850,7 +2850,7 @@ shared_ptr EnumerationFx::create_schema() { int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index f55f79231663..8585cf1a9a61 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -62,6 +62,7 @@ struct RequestHandlerFx { shared_ptr get_array(QueryType type); + shared_ptr memory_tracker_; URI uri_; Config cfg_; Context ctx_; @@ -96,16 +97,13 @@ struct HandleConsolidationPlanRequestFx : RequestHandlerFx { } virtual shared_ptr create_schema() override { - auto schema = make_shared( - HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); auto dim = make_shared( - HERE(), - "dim1", - Datatype::INT32, - tiledb::test::get_test_memory_tracker()); + HERE(), "dim1", Datatype::INT32, memory_tracker_); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); return schema; @@ -340,7 +338,8 @@ TEST_CASE_METHOD( /* ********************************* */ RequestHandlerFx::RequestHandlerFx(const std::string uri) - : uri_(uri) + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , uri_(uri) , ctx_(cfg_) { delete_array(); throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -402,14 +401,14 @@ HandleLoadArraySchemaRequestFx::create_string_enumeration( shared_ptr HandleLoadArraySchemaRequestFx::create_schema() { // Create a schema to serialize - auto schema = make_shared( - HERE(), ArrayType::SPARSE, tiledb::test::create_test_memory_tracker()); - auto dim = make_shared( - HERE(), "dim1", Datatype::INT32, tiledb::test::get_test_memory_tracker()); + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); + auto dim = + make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); int range[2] = {0, 1000}; throw_if_not_ok(dim->set_domain(range)); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim)); throw_if_not_ok(schema->set_domain(dom)); @@ -445,15 +444,13 @@ shared_ptr HandleLoadArraySchemaRequestFx::call_handler( resp_buf); REQUIRE(rval == TILEDB_OK); - auto memory_tracker = - ctx.ptr()->context().resources().create_memory_tracker(); return serialization::deserialize_load_array_schema_response( - stype, resp_buf->buffer(), memory_tracker); + stype, resp_buf->buffer(), memory_tracker_); } shared_ptr HandleQueryPlanRequestFx::create_schema() { - auto schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker_); schema->set_capacity(10000); throw_if_not_ok(schema->set_cell_order(Layout::ROW_MAJOR)); throw_if_not_ok(schema->set_tile_order(Layout::ROW_MAJOR)); @@ -466,7 +463,7 @@ shared_ptr HandleQueryPlanRequestFx::create_schema() { HERE(), "dim2", Datatype::INT32, tiledb::test::get_test_memory_tracker()); throw_if_not_ok(dim2->set_domain(&dim_domain[2])); - auto dom = make_shared(HERE()); + auto dom = make_shared(HERE(), memory_tracker_); throw_if_not_ok(dom->add_dimension(dim1)); throw_if_not_ok(dom->add_dimension(dim2)); throw_if_not_ok(schema->set_domain(dom)); diff --git a/tiledb/api/c_api/domain/domain_api.cc b/tiledb/api/c_api/domain/domain_api.cc index 3a694a97c117..00238c143fc8 100644 --- a/tiledb/api/c_api/domain/domain_api.cc +++ b/tiledb/api/c_api/domain/domain_api.cc @@ -30,12 +30,16 @@ #include "../dimension/dimension_api_internal.h" #include "domain_api_external.h" #include "domain_api_internal.h" +#include "tiledb/common/memory_tracker.h" namespace tiledb::api { -int32_t tiledb_domain_alloc(tiledb_domain_handle_t** domain) { +int32_t tiledb_domain_alloc( + tiledb_ctx_t* ctx, tiledb_domain_handle_t** domain) { ensure_output_pointer_is_valid(domain); - *domain = tiledb_domain_handle_t::make_handle(); + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *domain = tiledb_domain_handle_t::make_handle(memory_tracker); return TILEDB_OK; } @@ -147,7 +151,8 @@ int32_t tiledb_domain_dump(const tiledb_domain_t* domain, FILE* out) { using tiledb::api::api_entry_context; CAPI_INTERFACE(domain_alloc, tiledb_ctx_t* ctx, tiledb_domain_t** domain) { - return api_entry_context(ctx, domain); + return tiledb::api::api_entry_with_context( + ctx, domain); } CAPI_INTERFACE_VOID(domain_free, tiledb_domain_t** domain) { diff --git a/tiledb/api/c_api/domain/domain_api_internal.h b/tiledb/api/c_api/domain/domain_api_internal.h index 76a427ce8ace..dabd36e402c4 100644 --- a/tiledb/api/c_api/domain/domain_api_internal.h +++ b/tiledb/api/c_api/domain/domain_api_internal.h @@ -35,6 +35,10 @@ #include "tiledb/api/c_api_support/handle/handle.h" #include "tiledb/sm/array_schema/domain.h" +namespace tiledb::sm { +class MemoryTracker; +} + struct tiledb_domain_handle_t : public tiledb::api::CAPIHandle { private: @@ -54,8 +58,9 @@ struct tiledb_domain_handle_t * `class Domain` is principally a container for `Dimension` objects. Domain * handles are first constructed as empty containers. */ - tiledb_domain_handle_t() - : domain_{make_shared(HERE())} { + explicit tiledb_domain_handle_t( + shared_ptr memory_tracker) + : domain_{make_shared(HERE(), memory_tracker)} { } /** diff --git a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc index c5959eecd7bd..6b6a726109e7 100644 --- a/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc +++ b/tiledb/api/c_api/domain/test/compile_capi_domain_stub_main.cc @@ -29,7 +29,6 @@ #include "../domain_api_internal.h" int main() { - // Domain is easy to deal with because it has a default constructor. - tiledb_domain_handle_t x{}; + tiledb_domain_handle_t x{shared_ptr()}; return 0; } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index b7b24e17cdee..344e8410633a 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -76,6 +76,8 @@ std::string memory_type_to_str(MemoryType type) { return "DimensionLabels"; case MemoryType::DIMENSIONS: return "Dimensions"; + case MemoryType::DOMAINS: + return "Domains"; case MemoryType::TILE_SUMS: return "TileSums"; case MemoryType::TILE_WRITER_DATA: diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 8858fe07210d..4a9525a4a904 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -115,6 +115,7 @@ enum class MemoryType { DIMENSION_LABELS, DIMENSIONS, METADATA, + DOMAINS, TILE_SUMS, TILE_WRITER_DATA }; diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 20e5c310f8f9..c0b56da2e643 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -40,6 +40,7 @@ #include "../array.h" #include "../consistency.h" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" @@ -108,8 +109,8 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(dim->set_tile_extent(&tile_extent)); std::vector> dims = {dim}; - shared_ptr domain = - make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + shared_ptr domain = make_shared( + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker_); // Create the ArraySchema shared_ptr schema = make_shared( diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index b2e91fc31ed4..8c360db43071 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -476,6 +476,7 @@ class ArraySchema { * * @param deserializer The deserializer to deserialize from. * @param uri The uri of the Array. + * @param memory_tracker The memory tracker to use. * @return A new ArraySchema. */ static shared_ptr deserialize( diff --git a/tiledb/sm/array_schema/dimension_label.cc b/tiledb/sm/array_schema/dimension_label.cc index 06dbf28c2021..b9138099dd83 100644 --- a/tiledb/sm/array_schema/dimension_label.cc +++ b/tiledb/sm/array_schema/dimension_label.cc @@ -28,6 +28,7 @@ #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" @@ -176,7 +177,11 @@ DimensionLabel::DimensionLabel( throw_if_not_ok( index_dims.back()->set_tile_extent(dim->tile_extent().data())); throw_if_not_ok(schema_->set_domain(make_shared( - HERE(), Layout::ROW_MAJOR, index_dims, Layout::ROW_MAJOR))); + HERE(), + Layout::ROW_MAJOR, + index_dims, + Layout::ROW_MAJOR, + memory_tracker))); // Create and set dimension label attribute. auto label_attr = make_shared( diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 877774a184e2..1b5883458b7d 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -37,6 +37,7 @@ #include "tiledb/common/blank.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/misc/tdb_math.h" @@ -58,7 +59,13 @@ namespace tiledb::sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ -Domain::Domain() { +Domain::Domain(shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , dimensions_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) + , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) { cell_order_ = Layout::ROW_MAJOR; tile_order_ = Layout::ROW_MAJOR; dim_num_ = 0; @@ -68,11 +75,20 @@ Domain::Domain() { Domain::Domain( Layout cell_order, const std::vector> dimensions, - Layout tile_order) - : cell_order_(cell_order) - , dimensions_(dimensions) + Layout tile_order, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , cell_order_(cell_order) + , dimensions_( + dimensions.begin(), + dimensions.end(), + memory_tracker_->get_resource(MemoryType::DIMENSIONS)) + , dimension_ptrs_(memory_tracker_->get_resource(MemoryType::DIMENSIONS)) , dim_num_(static_cast(dimensions.size())) - , tile_order_(tile_order) { + , tile_order_(tile_order) + , cell_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , cell_order_cmp_func_2_(memory_tracker_->get_resource(MemoryType::DOMAINS)) + , tile_order_cmp_func_(memory_tracker_->get_resource(MemoryType::DOMAINS)) { /* * Verify that the input vector has no non-null elements in order to meet the * class invariant. Initialize the dimensions mirror. @@ -93,32 +109,6 @@ Domain::Domain( set_tile_cell_order_cmp_funcs(); } -Domain::Domain(Domain&& rhs) - : cell_num_per_tile_(rhs.cell_num_per_tile_) - , cell_order_(rhs.cell_order_) - , dimensions_(move(rhs.dimensions_)) - , dimension_ptrs_(move(rhs.dimension_ptrs_)) - , dim_num_(rhs.dim_num_) - , tile_order_(rhs.tile_order_) - , cell_order_cmp_func_(move(rhs.cell_order_cmp_func_)) - , cell_order_cmp_func_2_(move(rhs.cell_order_cmp_func_2_)) - , tile_order_cmp_func_(move(rhs.tile_order_cmp_func_)) { -} - -Domain& Domain::operator=(Domain&& rhs) { - cell_num_per_tile_ = rhs.cell_num_per_tile_; - cell_order_ = rhs.cell_order_; - dim_num_ = rhs.dim_num_; - cell_order_cmp_func_ = move(rhs.cell_order_cmp_func_); - tile_order_cmp_func_ = move(rhs.tile_order_cmp_func_); - dimensions_ = move(rhs.dimensions_); - dimension_ptrs_ = move(rhs.dimension_ptrs_); - tile_order_ = rhs.tile_order_; - cell_order_cmp_func_2_ = move(rhs.cell_order_cmp_func_2_); - - return *this; -} - /* ********************************* */ /* API */ /* ********************************* */ @@ -352,7 +342,7 @@ shared_ptr Domain::deserialize( } return tiledb::common::make_shared( - HERE(), cell_order, dimensions, tile_order); + HERE(), cell_order, dimensions, tile_order, memory_tracker); } const Range& Domain::domain(unsigned i) const { @@ -610,8 +600,9 @@ const ByteVecValue& Domain::tile_extent(unsigned i) const { std::vector Domain::tile_extents() const { std::vector ret(dim_num_); - for (unsigned d = 0; d < dim_num_; ++d) + for (unsigned d = 0; d < dim_num_; ++d) { ret[d] = tile_extent(d); + } return ret; } diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index c8007c8d2aa7..5544169cb44b 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -35,6 +35,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/macros.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/common/types/dynamic_typed_datum.h" #include "tiledb/common/types/untyped_datum.h" @@ -75,20 +76,18 @@ class Domain { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Empty constructor. */ - Domain(); + /** Deleted default constructor */ + Domain() = delete; + + /** Constructor. */ + Domain(shared_ptr memory_tracker); /** Constructor.*/ Domain( Layout cell_order, const std::vector> dimensions, - Layout tile_order); - - /** Copy constructor. */ - Domain(const Domain&) = default; - - /** Move constructor. */ - Domain(Domain&& rhs); + Layout tile_order, + shared_ptr memory_tracker); /** Destructor. */ ~Domain() = default; @@ -97,11 +96,8 @@ class Domain { /* OPERATORS */ /* ********************************* */ - /** Copy-assignment operator. */ - DISABLE_COPY_ASSIGN(Domain); - - /** Move-assignment operator. */ - Domain& operator=(Domain&& rhs); + DISABLE_COPY_AND_COPY_ASSIGN(Domain); + DISABLE_MOVE_AND_MOVE_ASSIGN(Domain); /* ********************************* */ /* API */ @@ -508,6 +504,9 @@ class Domain { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory tracker for this Domain. */ + shared_ptr memory_tracker_; + /** The number of cells per tile. Meaningful only for the **dense** case. */ uint64_t cell_num_per_tile_; @@ -523,7 +522,7 @@ class Domain { * * @invariant All pointers in the vector are non-null. */ - std::vector> dimensions_; + tdb::pmr::vector> dimensions_; /** * Non-allocating mirror of the dimensions vector. @@ -535,7 +534,7 @@ class Domain { * * @invariant All pointers in the vector are non-null. */ - std::vector dimension_ptrs_; + tdb::pmr::vector dimension_ptrs_; /** The number of dimensions. */ unsigned dim_num_; @@ -551,7 +550,7 @@ class Domain { * - buff: The buffer that stores all coorinates; * - a, b: The positions of the two coordinates in the buffer to compare. */ - std::vector cell_order_cmp_func_; @@ -561,7 +560,7 @@ class Domain { * * - coord_a, coord_b: The two coordinates to compare. */ - std::vector + tdb::pmr::vector cell_order_cmp_func_2_; /** @@ -571,7 +570,7 @@ class Domain { * - dim: The dimension to compare on. * - coord_a, coord_b: The two coordinates to compare. */ - std::vector tile_order_cmp_func_; diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index 84f2e09ef536..2c751fe1bfd0 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -239,6 +239,8 @@ class TestAttribute { * Array Schema wrapper */ class TestArraySchema { + shared_ptr memory_tracker_; + ArraySchema schema_; /** @@ -263,9 +265,14 @@ class TestArraySchema { static shared_ptr make_domain( std::initializer_list dimensions, Layout cell_order, - Layout tile_order) { + Layout tile_order, + shared_ptr memory_tracker) { return make_shared( - HERE(), cell_order, make_dimension_vector(dimensions), tile_order); + HERE(), + cell_order, + make_dimension_vector(dimensions), + tile_order, + memory_tracker); } /** @@ -307,7 +314,11 @@ class TestArraySchema { "", // name array_type, false, // allow duplicates - make_domain(dimensions, cell_order, tile_order), + make_domain( + dimensions, + cell_order, + tile_order, + tiledb::test::create_test_memory_tracker()), cell_order, tile_order, 10000, // capacity diff --git a/tiledb/sm/array_schema/test/unit_domain_data.cc b/tiledb/sm/array_schema/test/unit_domain_data.cc index 6997ca9cf23b..b26174acefab 100644 --- a/tiledb/sm/array_schema/test/unit_domain_data.cc +++ b/tiledb/sm/array_schema/test/unit_domain_data.cc @@ -111,7 +111,8 @@ struct TestNullInitializer { }; TEST_CASE("DomainTypedDataView::DomainTypedDataView, null initializer") { - Domain d{}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain d{memory_tracker}; // tiledb::sm::Dimension dim{"", tiledb::sm::Datatype::INT32}; auto dim{make_shared( HERE(), @@ -136,7 +137,8 @@ TEST_CASE("DomainTypedDataView::DomainTypedDataView, simple initializer") { } }; - Domain d{}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + Domain d{memory_tracker}; auto dim{make_shared( HERE(), "", diff --git a/tiledb/sm/array_schema/test/unit_tile_domain.cc b/tiledb/sm/array_schema/test/unit_tile_domain.cc index 771c27726879..95676b338984 100644 --- a/tiledb/sm/array_schema/test/unit_tile_domain.cc +++ b/tiledb/sm/array_schema/test/unit_tile_domain.cc @@ -31,12 +31,14 @@ */ #include +#include "src/mem_helpers.h" #include "tiledb/sm/array_schema/tile_domain.h" using namespace tiledb::sm; TEST_CASE("TileDomain: Test 1D", "[TileDomain][1d]") { int32_t tile_extent_v = 10; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(1); tile_extents[0].assign_as(tile_extent_v); Layout layout = Layout::ROW_MAJOR; @@ -72,6 +74,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -115,6 +118,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -153,6 +157,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {1, 10, 1, 10}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -191,6 +196,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 1, 10}; std::vector domain_slice = {4, 10, 2, 8}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -228,6 +234,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {4, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -260,6 +267,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -297,6 +305,7 @@ TEST_CASE( std::vector domain_vec = {1, 10, 11, 20}; std::vector domain_slice = {2, 10, 12, 18}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); @@ -323,6 +332,7 @@ TEST_CASE("TileDomain: Test 2D, covers", "[TileDomain][2d][covers]") { std::vector domain_slice_1 = {2, 6, 2, 8}; std::vector domain_slice_2 = {3, 6, 1, 7}; std::vector tile_extents_vec = {2, 5}; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::vector tile_extents(2); tile_extents[0].assign_as(tile_extents_vec[0]); tile_extents[1].assign_as(tile_extents_vec[1]); diff --git a/tiledb/sm/array_schema/tile_domain.h b/tiledb/sm/array_schema/tile_domain.h index a89e498e9af3..1cc96404a231 100644 --- a/tiledb/sm/array_schema/tile_domain.h +++ b/tiledb/sm/array_schema/tile_domain.h @@ -36,6 +36,7 @@ #include #include +#include "tiledb/common/pmr.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/misc/types.h" @@ -81,13 +82,13 @@ class TileDomain { unsigned id, const NDRange& domain, const NDRange& domain_slice, - const std::vector tile_extents, + const std::vector& tile_extents, Layout layout) : id_(id) , dim_num_((unsigned)domain.size()) , domain_(domain) , domain_slice_(domain_slice) - , tile_extents_(tile_extents) + , tile_extents_(tile_extents.begin(), tile_extents.end()) , layout_(layout) { assert(layout == Layout::ROW_MAJOR || layout == Layout::COL_MAJOR); compute_tile_domain(domain, domain_slice, tile_extents); diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 482ccde4e2e4..f8c933659e01 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -142,7 +142,7 @@ int32_t tiledb_filestore_schema_create( tiledb::sm::ByteVecValue(std::move(tile_extent_vec)), memory_tracker); - auto domain = make_shared(HERE()); + auto domain = make_shared(HERE(), memory_tracker); throw_if_not_ok(domain->add_dimension(dim)); auto attr = make_shared( diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index f6f222324c08..b5e728fff267 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -53,7 +53,8 @@ shared_ptr make_schema( tiledb::test::create_test_memory_tracker()); // Create the domain/dimensions. - Domain domain; + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + auto domain{make_shared(HERE(), memory_tracker)}; for (uint64_t d = 0; d < dim_types.size(); d++) { auto dim{make_shared( HERE(), @@ -135,9 +136,9 @@ shared_ptr make_schema( } } - REQUIRE(domain.add_dimension(dim).ok()); + REQUIRE(domain->add_dimension(dim).ok()); } - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); // Create the attributes. for (uint64_t a = 0; a < attr_types.size(); a++) { diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index 5842f6351bec..cd6648a06cb4 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -40,6 +40,7 @@ constexpr bool webp_filter_exists = false; #endif // TILEDB_WEBP #include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" #include "tiledb/sm/enums/filter_option.h" #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/filter.h" diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 0f0f42eae20a..e83d42687a20 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/fragment/fragment_info.h" #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/dimension.h" diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 70381200638f..264aa3f5fb17 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -1571,8 +1571,9 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -1583,7 +1584,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -1594,15 +1595,15 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -1638,14 +1639,15 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -1656,15 +1658,15 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -1746,8 +1748,9 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -1758,7 +1761,7 @@ TEST_CASE( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -1769,10 +1772,8 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); std::vector> frag_md(1); frag_md[0] = make_shared( @@ -2300,8 +2301,9 @@ void test_apply_dense( const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2312,7 +2314,7 @@ void test_apply_dense( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -2323,17 +2325,15 @@ void test_apply_dense( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -2369,14 +2369,15 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -2387,17 +2388,15 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -2480,8 +2479,9 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -2492,7 +2492,7 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -2503,17 +2503,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -3014,8 +3012,9 @@ void test_apply_sparse( const char* fill_value = "ac"; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -3026,7 +3025,7 @@ void test_apply_sparse( REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -3037,17 +3036,15 @@ void test_apply_sparse( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -3083,14 +3080,15 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { const T fill_value = 3; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_cell_val_num(1); attr.set_fill_value(&fill_value, sizeof(T)); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -3101,17 +3099,15 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -3860,12 +3856,13 @@ TEST_CASE( const Datatype type = Datatype::UINT64; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -3876,17 +3873,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -4150,8 +4145,9 @@ TEST_CASE( const Datatype type = GENERATE(Datatype::STRING_ASCII, Datatype::STRING_UTF8); // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4159,7 +4155,7 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -4170,15 +4166,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -4510,8 +4506,9 @@ TEST_CASE( const Datatype type = Datatype::STRING_UTF8; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(false); attr.set_cell_val_num(constants::var_num); @@ -4519,7 +4516,7 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -4530,15 +4527,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // For pasting into a Python shell: @@ -4838,13 +4835,14 @@ TEST_CASE( const Datatype type = Datatype::FLOAT32; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(true); REQUIRE(array_schema->add_attribute(tdb::make_shared(HERE(), attr)) .ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -4855,17 +4853,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE( - array_schema->set_domain(make_shared(HERE(), domain)) - .ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -4939,8 +4935,9 @@ TEST_CASE( return; // Initialize the array schema. - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + auto memory_tracker = tiledb::test::create_test_memory_tracker(); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); attr.set_nullable(nullable); attr.set_cell_val_num(var_size ? constants::var_num : 2); @@ -4951,7 +4948,7 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); - Domain domain; + auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( HERE(), "dim1", @@ -4962,15 +4959,15 @@ TEST_CASE( REQUIRE(dim->set_domain(range).ok()); uint32_t tile_extent = 10; REQUIRE(dim->set_tile_extent(&tile_extent).ok()); - REQUIRE(domain.add_dimension(dim).ok()); - REQUIRE(array_schema->set_domain(make_shared(HERE(), domain)).ok()); + REQUIRE(domain->add_dimension(dim).ok()); + REQUIRE(array_schema->set_domain(domain).ok()); FragmentMetadata frag_md( nullptr, array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index eefbc92807b9..fedbc453a7f0 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -35,6 +35,7 @@ #include #include "../query_plan.h" #include "external/include/nlohmann/json.hpp" +#include "test/support/src/mem_helpers.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" @@ -60,6 +61,7 @@ struct QueryPlanFx { URI array_uri(const std::string& uri); shared_ptr memory_tracker_; + TemporaryLocalDirectory temp_dir_; Config cfg_; shared_ptr logger_; @@ -77,8 +79,8 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(dim->set_tile_extent(&tile_extent)); std::vector> dims = {dim}; - shared_ptr domain = - make_shared(HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + shared_ptr domain = make_shared( + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker_); // Create the ArraySchema shared_ptr schema = make_shared( @@ -110,7 +112,7 @@ URI QueryPlanFx::array_uri(const std::string& array_name) { } QueryPlanFx::QueryPlanFx() - : memory_tracker_(tiledb::test::get_test_memory_tracker()) + : memory_tracker_(tiledb::test::create_test_memory_tracker()) , logger_(make_shared(HERE(), "foo")) , resources_(cfg_, logger_, 1, 1, "") , sm_(make_shared(resources_, logger_, cfg_)) { diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 5cdfdd7166f6..6d81dd0c9499 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -90,7 +90,8 @@ Domain create_domain( const std::vector& dim_names, const std::vector& dim_types, const std::vector& dim_domains, - const std::vector& dim_tile_extents) { + const std::vector& dim_tile_extents, + shared_ptr memory_tracker) { assert(!dim_names.empty()); assert(dim_names.size() == dim_types.size()); assert(dim_names.size() == dim_domains.size()); @@ -122,7 +123,8 @@ Domain create_domain( dimensions.emplace_back(std::move(dim)); } - return Domain(Layout::ROW_MAJOR, dimensions, Layout::ROW_MAJOR); + return Domain( + Layout::ROW_MAJOR, dimensions, Layout::ROW_MAJOR, memory_tracker); } TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { @@ -139,10 +141,11 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { // 1D int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); + const Domain d1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); - const Domain d1{dom1}; RTree rtree1(&d1, 3, tracker); CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); CHECK(rtree1.set_leaf_num(mbrs_1d.size()).ok()); @@ -200,10 +203,16 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { {"d1", "d2"}, {Datatype::INT64, Datatype::INT64}, {dim_dom_2, dim_dom_2}, - {&dim_extent_2, &dim_extent_2}); + {&dim_extent_2, &dim_extent_2}, + tracker); + const Domain d2 = create_domain( + {"d1", "d2"}, + {Datatype::INT64, Datatype::INT64}, + {dim_dom_2, dim_dom_2}, + {&dim_extent_2, &dim_extent_2}, + tracker); auto mbrs_2d = create_mbrs( {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); - const Domain d2{dom2}; RTree rtree2(&d2, 5, tracker); CHECK(rtree2.set_leaves(mbrs_2d).ok()); rtree2.build_tree(); @@ -240,9 +249,10 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { float dim_extent_f = 10.0; auto mbrs_f = create_mbrs({1.0f, 3.0f, 5.0f, 10.0f, 20.0f, 22.0f}, tracker); - Domain dom2f = - create_domain({"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}); - const Domain d2f{dom2f}; + Domain dom2f = create_domain( + {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); + const Domain d2f = create_domain( + {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); RTree rtreef(&d2f, 5, tracker); CHECK(rtreef.set_leaves(mbrs_f).ok()); rtreef.build_tree(); @@ -283,11 +293,10 @@ TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { std::vector is_default(1, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -332,10 +341,9 @@ TEST_CASE("RTree: Test 1D R-tree, height 3", "[rtree][1d][3h]") { int32_t dim_extent = 10; auto mbrs = create_mbrs( {1, 3, 5, 10, 20, 22, 30, 35, 36, 38, 40, 49, 50, 51, 65, 69}, tracker); - Domain dom1 = - create_domain({"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}); - const Domain d1(dom1); - RTree rtree(&d1, 3, tracker); + Domain dom1 = create_domain( + {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -397,15 +405,15 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom2 = create_domain( + Domain dom1 = create_domain( {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, - {&dim_extent, &dim_extent}); + {&dim_extent, &dim_extent}, + tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); - const Domain d2{dom2}; - RTree rtree(&d2, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -451,17 +459,17 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { std::vector is_default(2, false); int32_t dim_dom[] = {1, 1000}; int32_t dim_extent = 10; - Domain dom2 = create_domain( + Domain dom1 = create_domain( {"d1", "d2"}, {Datatype::INT32, Datatype::INT32}, {dim_dom, dim_dom}, - {&dim_extent, &dim_extent}); + {&dim_extent, &dim_extent}, + tracker); auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15, 11, 15, 20, 22, 16, 16, 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, tracker); - const Domain d2{dom2}; - RTree rtree(&d2, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -532,14 +540,20 @@ TEST_CASE( int32_t int32_dom[] = {5, 10}; uint8_t uint8_extent = 2; int32_t int32_extent = 2; - Domain dom = create_domain( + Domain dom1 = create_domain( + {"d1", "d2"}, + {Datatype::UINT8, Datatype::INT32}, + {uint8_dom, int32_dom}, + {&uint8_extent, &int32_extent}, + tracker); + const Domain d1 = create_domain( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); - const Domain d1{dom}; RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); @@ -555,7 +569,7 @@ TEST_CASE( int32_t int32_r_no[] = {1, 10}; range_no[0] = Range(uint8_r_no, sizeof(uint8_r_no)); range_no[1] = Range(int32_r_no, sizeof(int32_r_no)); - double ratio = dom.overlap_ratio(range_no, is_default, mbrs[0]); + double ratio = dom1.overlap_ratio(range_no, is_default, mbrs[0]); CHECK(ratio == 0.0); // Check full domain overlap @@ -564,9 +578,9 @@ TEST_CASE( int32_t int32_r_full[] = {1, 10}; range_full[0] = Range(uint8_r_full, sizeof(uint8_r_full)); range_full[1] = Range(int32_r_full, sizeof(int32_r_full)); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[0]); CHECK(ratio == 1.0); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[1]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[1]); CHECK(ratio == 1.0); // Check partial domain overlap @@ -575,7 +589,7 @@ TEST_CASE( int32_t int32_r_part[] = {5, 5}; range_part[0] = Range(uint8_r_part, sizeof(uint8_r_part)); range_part[1] = Range(int32_r_part, sizeof(int32_r_part)); - ratio = dom.overlap_ratio(range_part, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_part, is_default, mbrs[0]); CHECK(ratio == 0.25); } @@ -589,14 +603,20 @@ TEST_CASE( float float_dom[] = {0.1f, 0.9f}; uint64_t uint64_extent = 2; float float_extent = 0.1f; - Domain dom = create_domain( + Domain dom1 = create_domain( {"d1", "d2"}, {Datatype::UINT64, Datatype::FLOAT32}, {uint64_dom, float_dom}, - {&uint64_extent, &float_extent}); + {&uint64_extent, &float_extent}, + tracker); + const Domain d1 = create_domain( + {"d1", "d2"}, + {Datatype::UINT64, Datatype::FLOAT32}, + {uint64_dom, float_dom}, + {&uint64_extent, &float_extent}, + tracker); auto mbrs = create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); - const Domain d1{dom}; RTree rtree(&d1, 5, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); @@ -612,7 +632,7 @@ TEST_CASE( float float_r_no[] = {.1f, .9f}; range_no[0] = Range(uint64_r_no, sizeof(uint64_r_no)); range_no[1] = Range(float_r_no, sizeof(float_r_no)); - double ratio = dom.overlap_ratio(range_no, is_default, mbrs[0]); + double ratio = dom1.overlap_ratio(range_no, is_default, mbrs[0]); CHECK(ratio == 0.0); // Check full domain overlap @@ -621,9 +641,9 @@ TEST_CASE( float float_r_full[] = {.1f, 1.0f}; range_full[0] = Range(uint64_r_full, sizeof(uint64_r_full)); range_full[1] = Range(float_r_full, sizeof(float_r_full)); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[0]); CHECK(ratio == 1.0); - ratio = dom.overlap_ratio(range_full, is_default, mbrs[1]); + ratio = dom1.overlap_ratio(range_full, is_default, mbrs[1]); CHECK(ratio == 1.0); // Check partial domain overlap @@ -632,7 +652,7 @@ TEST_CASE( float float_r_part[] = {.5f, .55f}; range_part[0] = Range(uint64_r_part, sizeof(uint64_r_part)); range_part[1] = Range(float_r_part, sizeof(float_r_part)); - ratio = dom.overlap_ratio(range_part, is_default, mbrs[0]); + ratio = dom1.overlap_ratio(range_part, is_default, mbrs[0]); CHECK(ratio == 0.25); } @@ -650,11 +670,11 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -720,11 +740,11 @@ TEST_CASE( {"d1", "d2"}, {Datatype::UINT8, Datatype::INT32}, {uint8_dom, int32_dom}, - {&uint8_extent, &int32_extent}); + {&uint8_extent, &int32_extent}, + tracker); auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 2, tracker); + RTree rtree(&dom, 2, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -856,13 +876,12 @@ TEST_CASE( // Build tree auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); - Domain dom1 = - create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); + Domain dom1 = create_domain( + {"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}, tracker); auto mbrs = create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -935,8 +954,8 @@ TEST_CASE( // Build tree auto tracker = create_test_memory_tracker(); std::vector is_default(1, false); - Domain dom1 = - create_domain({"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}); + Domain dom1 = create_domain( + {"d"}, {Datatype::STRING_ASCII}, {nullptr}, {nullptr}, tracker); auto mbrs = create_str_mbrs<1>( {"aa", "b", @@ -952,8 +971,7 @@ TEST_CASE( "oop"}, tracker); - const Domain d1{dom1}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom1, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 3); @@ -1033,13 +1051,13 @@ TEST_CASE( {"d1", "d2"}, {Datatype::STRING_ASCII, Datatype::STRING_ASCII}, {nullptr, nullptr}, - {nullptr, nullptr}); + {nullptr, nullptr}, + tracker); auto mbrs = create_str_mbrs<2>( {"aa", "b", "eee", "g", "gggg", "ii", "jj", "lll", "m", "n", "oo", "qqq"}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); @@ -1136,17 +1154,17 @@ TEST_CASE( std::vector is_default(2, false); int32_t dom_int32[] = {1, 20}; int32_t tile_extent = 5; + auto tracker = create_test_memory_tracker(); Domain dom = create_domain( {"d1", "d2"}, {Datatype::STRING_ASCII, Datatype::INT32}, {nullptr, dom_int32}, - {nullptr, &tile_extent}); - auto tracker = create_test_memory_tracker(); + {nullptr, &tile_extent}, + tracker); auto mbrs = create_str_int32_mbrs( {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}, tracker); - const Domain d1{dom}; - RTree rtree(&d1, 3, tracker); + RTree rtree(&dom, 3, tracker); CHECK(rtree.set_leaves(mbrs).ok()); rtree.build_tree(); CHECK(rtree.height() == 2); diff --git a/tiledb/sm/serialization/array.h b/tiledb/sm/serialization/array.h index a08b531b1e9a..e048510c5349 100644 --- a/tiledb/sm/serialization/array.h +++ b/tiledb/sm/serialization/array.h @@ -72,6 +72,7 @@ Status array_to_capnp( * @param storage_manager the storage manager associated with the array * @param array Array to deserialize into * @param client_side Allows to specify different behavior depending on who is + * @param memory_tracker Memory tracker to use for memory allocations. * serializing, the client (1) or the Cloud server (0). This is sometimes needed * since they are both using the same Core library APIs for serialization. * @param memory_tracker Memory tracker to use on the deserialized object. diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 125d998d6a3e..5f3bc76d3a7c 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -750,7 +750,8 @@ shared_ptr domain_from_capnp( dims.emplace_back(dimension_from_capnp(dimension, memory_tracker)); } - return make_shared(HERE(), cell_order, dims, tile_order); + return make_shared( + HERE(), cell_order, dims, tile_order, memory_tracker); } void dimension_label_to_capnp( diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index 2c7fe0904405..14b51d7b6203 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -110,6 +110,7 @@ Status array_schema_to_capnp( * * @param schema_reader Cap'n proto object * @param uri A URI object + * @param memory_tracker The memory tracker to use. * @return a new ArraySchema */ shared_ptr array_schema_from_capnp( @@ -133,6 +134,7 @@ void dimension_label_to_capnp( * Deserialize a dimension label from a cap'n proto object * * @param reader Cap'n proto reader object. + * @param memory_tracker The memory tracker to use. * @return A new DimensionLabel. */ shared_ptr dimension_label_from_capnp( diff --git a/tiledb/sm/serialization/fragment_info.h b/tiledb/sm/serialization/fragment_info.h index 42375f05d664..968a13d38a81 100644 --- a/tiledb/sm/serialization/fragment_info.h +++ b/tiledb/sm/serialization/fragment_info.h @@ -59,6 +59,7 @@ namespace serialization { * @param fragment_info_reader cap'n proto class. * @param uri array uri that the fragment belongs to * @param fragment_info fragment info object to deserialize into. + * @param memory_tracker The memory tracker to use. * @return Status */ Status fragment_info_from_capnp( @@ -125,6 +126,7 @@ Status fragment_info_serialize( * @param serialize_type format the data is serialized in: Cap'n Proto of JSON. * @param uri array uri that the fragment belongs to * @param serialized_buffer buffer to read serialized bytes from. + * @param memory_tracker The memory tracker to use. * @return Status */ Status fragment_info_deserialize( diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index 568faacf0f0a..a5ad85c03482 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -137,6 +137,7 @@ using CopyState = * @param serialized_buffer Buffer containing serialized query * @param serialize_type Serialization type of serialized query * @param array Array object to deserialize into + * @param memory_tracker Memory tracker to use for allocations. */ Status array_from_query_deserialize( const Buffer& serialized_buffer, diff --git a/tiledb/sm/serialization/test/CMakeLists.txt b/tiledb/sm/serialization/test/CMakeLists.txt index 23edbc74937c..832eaae79a9f 100644 --- a/tiledb/sm/serialization/test/CMakeLists.txt +++ b/tiledb/sm/serialization/test/CMakeLists.txt @@ -36,7 +36,7 @@ conclude(unit_test) commence(unit_test capnp_array_schema) this_target_sources(main.cc unit_capnp_array_schema.cc) - this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB) + this_target_link_libraries(TILEDB_CORE_OBJECTS TILEDB_CORE_OBJECTS_ILIB tiledb_test_support_lib) # Enable serialization target_compile_definitions(unit_capnp_array_schema PRIVATE -DTILEDB_SERIALIZATION) diff --git a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc index ccb4bfa5238d..0bd703210f48 100644 --- a/tiledb/sm/serialization/test/unit_capnp_array_schema.cc +++ b/tiledb/sm/serialization/test/unit_capnp_array_schema.cc @@ -112,7 +112,11 @@ TEST_CASE( st = dims[0]->set_domain(&domain1[0]); REQUIRE(st.ok()); st = schema->set_domain(make_shared( - HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR)); + HERE(), + Layout::ROW_MAJOR, + dims, + Layout::ROW_MAJOR, + tiledb::test::get_test_memory_tracker())); REQUIRE(st.ok()); st = schema->add_attribute( make_shared(HERE(), "label", Datatype::FLOAT64)); @@ -158,7 +162,7 @@ TEST_CASE( tiledb::sm::serialization::dimension_label_to_capnp( *dim_label.get(), &builder, true); auto dim_label_clone = tiledb::sm::serialization::dimension_label_from_capnp( - builder, tiledb::test::create_test_memory_tracker()); + builder, tiledb::test::get_test_memory_tracker()); // Check dimension label properties and components. CHECK(dim_label->has_schema() == dim_label_clone->has_schema()); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 4886c2c22b4b..984a7720a627 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -41,6 +41,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/common/memory.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 06f757721d81..040c7a129d06 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -53,6 +53,7 @@ using namespace tiledb::type; TEST_CASE("Subarray::add_ranges_list", "[subarray]") { // Setup an Array needed to construct the Subarray for testing // add_ranges_list. + auto memory_tracker = tiledb::test::create_test_memory_tracker(); std::shared_ptr sp_dim1 = make_shared( HERE(), @@ -68,7 +69,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { uint64_t tile_extents[] = {2, 2}; std::vector> dims{sp_dim1, sp_dim2}; std::shared_ptr sp_dom = make_shared( - HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR); + HERE(), Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker); uint64_t local_DIM_DOMAIN[4] = {1, 12, 1, 12}; CHECK(sp_dim1->set_domain(&local_DIM_DOMAIN[0]).ok()); CHECK(sp_dim2->set_domain(&local_DIM_DOMAIN[2]).ok()); @@ -76,7 +77,8 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { CHECK(sp_dim2->set_tile_extent(&tile_extents[1]).ok()); std::shared_ptr sp_attrib = make_shared(HERE(), "a1", Datatype::INT32); - tiledb::sm::Domain dom{Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR}; + tiledb::sm::Domain dom{ + Layout::ROW_MAJOR, dims, Layout::ROW_MAJOR, memory_tracker}; std::shared_ptr sp_as = make_shared( HERE(), From 0abb0b3551c5a5d5ae2039fa20c35436e62e937e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 6 Mar 2024 21:07:46 +0200 Subject: [PATCH 225/456] Enable vcpkg binary caching for the `build-docs` workflow. (#4782) --- TYPE: NO_HISTORY --- .github/workflows/build-docs.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 1bcc9623790b..31c15addeda8 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -28,6 +28,8 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 90 name: Build Docs + env: + VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' steps: - uses: actions/checkout@v3 - name: 'Print env' @@ -41,6 +43,13 @@ jobs: printenv shell: bash + - name: Set environment variables for vcpkg binary caching + uses: actions/github-script@v6 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - name: Set up Python uses: actions/setup-python@v4 From 0ddd3e88dc8fb97209fa94996852f0cf802a367a Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 7 Mar 2024 03:41:34 -0600 Subject: [PATCH 226/456] Prevent bugs in memory_type_to_str (#4792) My first PR for this either had a weird merge conflict or I totally whiffed on the commit. Regardless, the point was to remove the default switch case so that the compiler will enforce a switch case for every MemoryType and MemoryTrackerType. --- TYPE: NO_HISTORY DESC: Actually fix missing memory_tracker_type_to_str --- tiledb/common/memory_tracker.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 344e8410633a..3053c8ec84af 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -84,9 +84,6 @@ std::string memory_type_to_str(MemoryType type) { return "TileWriterData"; case MemoryType::METADATA: return "Metadata"; - default: - auto val = std::to_string(static_cast(type)); - throw std::logic_error("Invalid memory type: " + val); } auto val = std::to_string(static_cast(type)); @@ -123,9 +120,6 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { return "SchemaEvolution"; case MemoryTrackerType::GROUP: return "Group"; - default: - auto val = std::to_string(static_cast(type)); - throw std::logic_error("Invalid memory tracker type: " + val); } auto val = std::to_string(static_cast(type)); From 2a6c065c668f367ada45e67d1e182ea61f2bb7c0 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Thu, 7 Mar 2024 04:10:51 -0600 Subject: [PATCH 227/456] Add memory tracking for ResultTile bitmaps (#4791) This adds tracking for the various bitmap and other vectors on ResultTile classes. --- TYPE: NO_HISTORY DESC: Add memory tracking for ResultTile bitmaps --- test/src/unit-result-coords.cc | 16 +++--- test/src/unit-result-tile.cc | 8 ++- .../unit-sparse-unordered-with-dups-reader.cc | 4 +- tiledb/common/memory_tracker.cc | 6 +++ tiledb/common/memory_tracker.h | 3 ++ tiledb/sm/query/legacy/reader.cc | 8 ++- tiledb/sm/query/query_condition.cc | 24 +++++---- tiledb/sm/query/query_condition.h | 12 ++--- tiledb/sm/query/readers/result_tile.cc | 16 +++--- tiledb/sm/query/readers/result_tile.h | 50 +++++++++++-------- tiledb/sm/query/test/unit_query_condition.cc | 22 +++++--- 11 files changed, 104 insertions(+), 65 deletions(-) diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index a24a509193d3..b4cbe94daba7 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -180,17 +180,17 @@ TEST_CASE_METHOD( REQUIRE(rc1.max_slab_length() == 4); // Test max_slab_length with bitmap 1. - tile.bitmap() = {0, 1, 1, 1, 1}; + tile.bitmap().assign({0, 1, 1, 1, 1}); tile.count_cells(); REQUIRE(rc1.max_slab_length() == 4); // Test max_slab_length with bitmap 2. - tile.bitmap() = {0, 1, 1, 1, 0}; + tile.bitmap().assign({0, 1, 1, 1, 0}); tile.count_cells(); REQUIRE(rc1.max_slab_length() == 3); // Test max_slab_length with bitmap 3. - tile.bitmap() = {0, 1, 1, 1, 0}; + tile.bitmap().assign({0, 1, 1, 1, 0}); tile.count_cells(); rc1.pos_ = 0; REQUIRE(rc1.max_slab_length() == 0); @@ -214,19 +214,19 @@ TEST_CASE_METHOD( REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 3), cmp) == 2); // Test max_slab_length with bitmap and comparator 1. - tile.bitmap() = {0, 1, 1, 1, 1}; + tile.bitmap().assign({0, 1, 1, 1, 1}); tile.count_cells(); REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 10), cmp) == 4); REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 3), cmp) == 2); // Test max_slab_length with bitmap and comparator 2. - tile.bitmap() = {0, 1, 1, 1, 0}; + tile.bitmap().assign({0, 1, 1, 1, 0}); tile.count_cells(); REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 10), cmp) == 3); REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 3), cmp) == 2); // Test max_slab_length with bitmap and comparator 3. - tile.bitmap() = {0, 1, 1, 1, 0}; + tile.bitmap().assign({0, 1, 1, 1, 0}); tile.count_cells(); rc1.pos_ = 0; REQUIRE(rc1.max_slab_length(GlobalOrderResultCoords(&tile, 3), cmp) == 0); @@ -269,7 +269,7 @@ TEST_CASE_METHOD( Cmp cmp; GlobalOrderResultCoords rc1(&tile, 0); - tile.bitmap() = {0, 1, 1, 0, 1}; + tile.bitmap().assign({0, 1, 1, 0, 1}); tile.count_cells(); REQUIRE(rc1.advance_to_next_cell() == true); REQUIRE(rc1.pos_ == 1); @@ -281,7 +281,7 @@ TEST_CASE_METHOD( // Recreate to test that we don't move pos_ on the first call. GlobalOrderResultCoords rc2(&tile, 0); - tile.bitmap() = {1, 1, 1, 0, 0}; + tile.bitmap().assign({1, 1, 1, 0, 0}); tile.count_cells(); REQUIRE(rc2.advance_to_next_cell() == true); REQUIRE(rc2.pos_ == 0); diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index 7cc5518d882d..046cb1c8e7af 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -282,7 +282,9 @@ TEST_CASE_METHOD( ranges.size(), memory_tracker_->get_resource(MemoryType::DIMENSIONS)); std::iota(range_indexes.begin(), range_indexes.end(), 0); - std::vector result_count(num_cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_count(num_cells, 1, resource); ResultTile::compute_results_count_sparse_string( &rt, dim_idx, @@ -423,7 +425,9 @@ TEST_CASE_METHOD( ranges.size(), memory_tracker_->get_resource(MemoryType::DIMENSIONS)); std::iota(range_indexes.begin(), range_indexes.end(), 0); - std::vector result_count(num_cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_count(num_cells, 1, resource); ResultTile::compute_results_count_sparse_string( &rt, dim_idx, diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index b88b38e3f390..3f3192690679 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -1534,7 +1534,7 @@ TEST_CASE_METHOD( // Allocate and set the bitmap if required. if (bitmaps[t].size() > 0) { - rt.back().bitmap() = bitmaps[t]; + rt.back().bitmap().assign(bitmaps[t].begin(), bitmaps[t].end()); rt.back().count_cells(); } } @@ -1752,7 +1752,7 @@ TEST_CASE_METHOD( // Allocate and set the bitmap if required. if (bitmaps[t].size() > 0) { - rt.back().bitmap() = bitmaps[t]; + rt.back().bitmap().assign(bitmaps[t].begin(), bitmaps[t].end()); rt.back().count_cells(); } } diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 3053c8ec84af..ddc8cb870ee1 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -60,8 +60,12 @@ std::string memory_type_to_str(MemoryType type) { return "GenericTileIO"; case MemoryType::RTREE: return "RTree"; + case MemoryType::TILE_BITMAP: + return "TileBitmap"; case MemoryType::TILE_DATA: return "TileData"; + case MemoryType::TILE_HILBERT_VALUES: + return "TileHilbertValues"; case MemoryType::TILE_OFFSETS: return "TileOffsets"; case MemoryType::TILE_MAX_VALS: @@ -70,6 +74,8 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; + case MemoryType::TILE_QUERY_CONDITIONS: + return "TileQueryConditions"; case MemoryType::ATTRIBUTES: return "Attributes"; case MemoryType::DIMENSION_LABELS: diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 4a9525a4a904..c1413304f8cf 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -106,11 +106,14 @@ enum class MemoryType { FOOTER, GENERIC_TILE_IO, RTREE, + TILE_BITMAP, TILE_DATA, + TILE_HILBERT_VALUES, TILE_OFFSETS, TILE_MAX_VALS, TILE_MIN_VALS, TILE_NULL_COUNTS, + TILE_QUERY_CONDITIONS, ATTRIBUTES, DIMENSION_LABELS, DIMENSIONS, diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index f99f5812dc0c..b2292d98a3a2 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -453,7 +453,9 @@ Status Reader::compute_range_result_coords( result_coords.emplace_back(tile, pos); } } else { // Sparse - std::vector result_bitmap(coords_num, 1); + auto resource = + query_memory_tracker_->get_resource(MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap(coords_num, 1, resource); // Compute result and overwritten bitmap per dimension for (unsigned d = 0; d < dim_num; ++d) { @@ -1785,7 +1787,9 @@ Status Reader::get_all_result_coords( array_->timestamp_start(), array_->timestamp_end_opened_at()); if (fragment_metadata_[tile->frag_idx()]->has_timestamps() && partial_overlap) { - std::vector result_bitmap(coords_num, 1); + auto resource = + query_memory_tracker_->get_resource(MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap(coords_num, 1, resource); RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( *(frag_meta->array_schema().get()), *tile, result_bitmap)); diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 0b4bc63dcde4..5c2c8854a869 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -2385,7 +2385,7 @@ void QueryCondition::apply_ast_node_sparse( ResultTile& result_tile, const bool var_size, CombinationOp combination_op, - std::vector& result_bitmap) const { + tdb::pmr::vector& result_bitmap) const { const auto tile_tuple = result_tile.tile_tuple(node->get_field_name()); const void* condition_value_content = node->get_value_ptr(); const size_t condition_value_size = node->get_value_size(); @@ -2473,7 +2473,7 @@ void QueryCondition::apply_ast_node_sparse( ResultTile& result_tile, const bool var_size, CombinationOp combination_op, - std::vector& result_bitmap) const { + tdb::pmr::vector& result_bitmap) const { switch (node->get_op()) { case QueryConditionOp::ALWAYS_TRUE: apply_ast_node_sparse< @@ -2569,7 +2569,7 @@ void QueryCondition::apply_ast_node_sparse( const bool var_size, const bool nullable, CombinationOp combination_op, - std::vector& result_bitmap) const { + tdb::pmr::vector& result_bitmap) const { if (nullable) { apply_ast_node_sparse( node, result_tile, var_size, combination_op, result_bitmap); @@ -2585,7 +2585,7 @@ void QueryCondition::apply_ast_node_sparse( const ArraySchema& array_schema, ResultTile& result_tile, CombinationOp combination_op, - std::vector& result_bitmap) const { + tdb::pmr::vector& result_bitmap) const { std::string node_field_name = node->get_field_name(); if (!array_schema.is_field(node_field_name)) { std::fill(result_bitmap.begin(), result_bitmap.end(), 0); @@ -2744,7 +2744,7 @@ void QueryCondition::apply_tree_sparse( const ArraySchema& array_schema, ResultTile& result_tile, CombinationOp combination_op, - std::vector& result_bitmap) const { + tdb::pmr::vector& result_bitmap) const { if (!node->is_expr()) { apply_ast_node_sparse( node, array_schema, result_tile, combination_op, result_bitmap); @@ -2783,7 +2783,9 @@ void QueryCondition::apply_tree_sparse( // Handle the cl'(q, a) case. // This cases on whether the combination op = OR. } else if constexpr (std::is_same_v>) { - std::vector combination_op_bitmap(result_bitmap_size, 1); + auto resource = result_bitmap.get_allocator().resource(); + tdb::pmr::vector combination_op_bitmap( + result_bitmap_size, 1, resource); for (const auto& child : node->get_children()) { apply_tree_sparse( @@ -2805,7 +2807,9 @@ void QueryCondition::apply_tree_sparse( * = a /\ (cl1'(q; cl2'(q; 0))) */ case QueryConditionCombinationOp::OR: { - std::vector combination_op_bitmap(result_bitmap_size, 0); + auto resource = result_bitmap.get_allocator().resource(); + tdb::pmr::vector combination_op_bitmap( + result_bitmap_size, 0, resource); for (const auto& child : node->get_children()) { apply_tree_sparse( @@ -2836,7 +2840,7 @@ template Status QueryCondition::apply_sparse( const ArraySchema& array_schema, ResultTile& result_tile, - std::vector& result_bitmap) { + tdb::pmr::vector& result_bitmap) { apply_tree_sparse( tree_, array_schema, @@ -2865,7 +2869,7 @@ uint64_t QueryCondition::condition_index() const { // Explicit template instantiations. template Status QueryCondition::apply_sparse( - const ArraySchema& array_schema, ResultTile&, std::vector&); + const ArraySchema& array_schema, ResultTile&, tdb::pmr::vector&); template Status QueryCondition::apply_sparse( - const ArraySchema& array_schema, ResultTile&, std::vector&); + const ArraySchema& array_schema, ResultTile&, tdb::pmr::vector&); } // namespace tiledb::sm diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 327e48d551fd..33115e16f320 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -240,7 +240,7 @@ class QueryCondition { Status apply_sparse( const ArraySchema& array_schema, ResultTile& result_tile, - std::vector& result_bitmap); + tdb::pmr::vector& result_bitmap); /** * Reverse the query condition using De Morgan's law. @@ -561,7 +561,7 @@ class QueryCondition { ResultTile& result_tile, const bool var_size, CombinationOp combination_op, - std::vector& result_bitmap) const; + tdb::pmr::vector& result_bitmap) const; /** * Applies a value node on a sparse result tile, @@ -584,7 +584,7 @@ class QueryCondition { ResultTile& result_tile, const bool var_size, CombinationOp combination_op, - std::vector& result_bitmap) const; + tdb::pmr::vector& result_bitmap) const; /** * Applies a value node on a sparse result tile. @@ -603,7 +603,7 @@ class QueryCondition { const bool var_size, const bool nullable, CombinationOp combination_op, - std::vector& result_bitmap) const; + tdb::pmr::vector& result_bitmap) const; /** * Applies a value node to filter result cells from the input @@ -622,7 +622,7 @@ class QueryCondition { const ArraySchema& array_schema, ResultTile& result_tile, CombinationOp combination_op, - std::vector& result_bitmap) const; + tdb::pmr::vector& result_bitmap) const; /** * Applies the query condition represented with the AST to a set of cells. @@ -642,7 +642,7 @@ class QueryCondition { const ArraySchema& array_schema, ResultTile& result_tile, CombinationOp combination_op, - std::vector& result_bitmap) const; + tdb::pmr::vector& result_bitmap) const; }; } // namespace sm diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index ef7aae962f4c..2416c77309b6 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -608,7 +608,7 @@ void ResultTile::compute_results_sparse( const ResultTile* result_tile, unsigned dim_idx, const Range& range, - std::vector* result_bitmap, + tdb::pmr::vector* result_bitmap, const Layout& cell_order) { auto coords_num = result_tile->cell_num(); auto dim_num = result_tile->domain()->dim_num(); @@ -765,7 +765,7 @@ void ResultTile::compute_results_sparse( const ResultTile* result_tile, unsigned dim_idx, const Range& range, - std::vector* result_bitmap, + tdb::pmr::vector* result_bitmap, const Layout&) { // For easy reference. auto coords_num = result_tile->cell_num(); @@ -809,7 +809,7 @@ void ResultTile::compute_results_count_sparse_string_range( const offsets_t* buff_off, const uint64_t start, const uint64_t end, - std::vector& result_count) { + tdb::pmr::vector& result_count) { const bool non_overlapping = std::is_same::value; // Process all cells. @@ -875,7 +875,7 @@ void ResultTile::compute_results_count_sparse_string( unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell) { @@ -1051,7 +1051,7 @@ void ResultTile::compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout&, const uint64_t min_cell, const uint64_t max_cell) { @@ -1158,7 +1158,7 @@ Status ResultTile::compute_results_dense( Status ResultTile::compute_results_sparse( unsigned dim_idx, const Range& range, - std::vector* result_bitmap, + tdb::pmr::vector* result_bitmap, const Layout& cell_order) const { assert(compute_results_sparse_func_[dim_idx] != nullptr); compute_results_sparse_func_[dim_idx]( @@ -1171,7 +1171,7 @@ Status ResultTile::compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell) const { @@ -1193,7 +1193,7 @@ Status ResultTile::compute_results_count_sparse( unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell) const { diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index b74ec9ee744b..189631211192 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -549,7 +549,7 @@ class ResultTile { const ResultTile* result_tile, unsigned dim_idx, const Range& range, - std::vector* result_bitmap, + tdb::pmr::vector* result_bitmap, const Layout& cell_order); /** @@ -570,7 +570,7 @@ class ResultTile { unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell); @@ -594,7 +594,7 @@ class ResultTile { const uint64_t* buff_off, const uint64_t start, const uint64_t end, - std::vector& result_count); + tdb::pmr::vector& result_count); /** * Applicable only to sparse arrays. @@ -614,7 +614,7 @@ class ResultTile { unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell); @@ -647,7 +647,7 @@ class ResultTile { Status compute_results_sparse( unsigned dim_idx, const Range& range, - std::vector* result_bitmap, + tdb::pmr::vector* result_bitmap, const Layout& cell_order) const; /** @@ -667,7 +667,7 @@ class ResultTile { unsigned dim_idx, const NDRange& ranges, const tdb::pmr::vector& range_indexes, - std::vector& result_count, + tdb::pmr::vector& result_count, const Layout& cell_order, const uint64_t min_cell, const uint64_t max_cell) const; @@ -742,7 +742,7 @@ class ResultTile { const ResultTile*, unsigned, const Range&, - std::vector*, + tdb::pmr::vector*, const Layout&)>> compute_results_sparse_func_; @@ -755,7 +755,7 @@ class ResultTile { unsigned, const NDRange&, const tdb::pmr::vector&, - std::vector&, + tdb::pmr::vector&, const Layout&, const uint64_t, const uint64_t)>> @@ -770,7 +770,7 @@ class ResultTile { unsigned, const NDRange&, const tdb::pmr::vector&, - std::vector&, + tdb::pmr::vector&, const Layout&, const uint64_t, const uint64_t)>> @@ -827,7 +827,8 @@ class ResultTileWithBitmap : public ResultTile { * @param memory_tracker The memory tracker to use. */ ResultTileWithBitmap(shared_ptr memory_tracker) - : ResultTile(memory_tracker) { + : ResultTile(memory_tracker) + , bitmap_(memory_tracker_->get_resource(MemoryType::TILE_BITMAP)) { } ResultTileWithBitmap( @@ -836,6 +837,7 @@ class ResultTileWithBitmap : public ResultTile { const FragmentMetadata& frag_md, shared_ptr memory_tracker) : ResultTile(frag_idx, tile_idx, frag_md, memory_tracker) + , bitmap_(memory_tracker_->get_resource(MemoryType::TILE_BITMAP)) , result_num_(cell_num_) { } @@ -861,7 +863,7 @@ class ResultTileWithBitmap : public ResultTile { * * @param cell_idx Cell index. */ - inline std::vector& bitmap() { + inline tdb::pmr::vector& bitmap() { return bitmap_; } @@ -953,7 +955,7 @@ class ResultTileWithBitmap : public ResultTile { /* PROTECTED ATTRIBUTES */ /* ********************************* */ /** Bitmap for this tile. */ - std::vector bitmap_; + tdb::pmr::vector bitmap_; /** Number of cells in this bitmap. */ uint64_t result_num_; @@ -979,10 +981,16 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { shared_ptr memory_tracker) : ResultTileWithBitmap( frag_idx, tile_idx, frag_md, memory_tracker) - , post_dedup_bitmap_( - !dups || include_delete_meta ? optional(std::vector()) : - nullopt) + , hilbert_values_(this->memory_tracker_->get_resource( + MemoryType::TILE_HILBERT_VALUES)) + , post_dedup_bitmap_(nullopt) + , per_cell_delete_condition_(this->memory_tracker_->get_resource( + MemoryType::TILE_QUERY_CONDITIONS)) , used_(false) { + if (!dups || include_delete_meta) { + post_dedup_bitmap_.emplace( + this->memory_tracker_->get_resource(MemoryType::TILE_BITMAP)); + } } DISABLE_COPY_AND_COPY_ASSIGN(GlobalOrderResultTile); @@ -1020,7 +1028,7 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { void ensure_bitmap_for_query_condition() { if (post_dedup_bitmap_.has_value()) { if (ResultTileWithBitmap::has_bmp()) { - post_dedup_bitmap_ = ResultTileWithBitmap::bitmap_; + post_dedup_bitmap_->assign(this->bitmap_.begin(), this->bitmap_.end()); } else { post_dedup_bitmap_->resize(ResultTile::cell_num_, 1); } @@ -1036,7 +1044,7 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { * Returns the bitmap that included query condition results. For this tile * type, this is 'post_dedup_bitmap_' if allocated, or the regular bitmap. */ - inline std::vector& post_dedup_bitmap() { + inline tdb::pmr::vector& post_dedup_bitmap() { return post_dedup_bitmap_.has_value() && post_dedup_bitmap_->size() > 0 ? post_dedup_bitmap_.value() : ResultTileWithBitmap::bitmap_; @@ -1144,7 +1152,7 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { /* ********************************* */ /** Hilbert values for this tile. */ - std::vector hilbert_values_; + tdb::pmr::vector hilbert_values_; /** * An extra bitmap will be needed for array with no duplicates. For those, @@ -1152,13 +1160,13 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { * will contain the results before query condition, and post_dedup_bitmap_ * will contain results after query condition. */ - optional> post_dedup_bitmap_; + optional> post_dedup_bitmap_; /** * Delete condition index that deleted a cell. Used for consolidation with * delete metadata. */ - std::vector per_cell_delete_condition_; + tdb::pmr::vector per_cell_delete_condition_; /** Was the tile used in the merge. */ bool used_; @@ -1212,7 +1220,7 @@ class UnorderedWithDupsResultTile : public ResultTileWithBitmap { * Returns the bitmap that included query condition results. For this tile * type, this is stored in the regular bitmap. */ - inline std::vector& post_dedup_bitmap() { + inline tdb::pmr::vector& post_dedup_bitmap() { return ResultTileWithBitmap::bitmap_; } diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 264aa3f5fb17..60882000f871 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -2724,7 +2724,9 @@ void test_apply_cells_sparse( } // Apply the query condition. - std::vector result_bitmap(cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, *result_tile, result_bitmap) .ok()); @@ -2749,7 +2751,9 @@ void test_apply_cells_sparse( REQUIRE(query_condition_eq_null.check(*array_schema).ok()); // Apply the query condition. - std::vector result_bitmap_eq_null(cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap_eq_null(cells, 1, resource); REQUIRE(query_condition_eq_null .apply_sparse( *array_schema, *result_tile, result_bitmap_eq_null) @@ -2819,7 +2823,9 @@ void test_apply_cells_sparse( } // Apply the query condition. - std::vector result_bitmap(cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, *result_tile, result_bitmap) .ok()); @@ -3291,7 +3297,9 @@ void validate_qc_apply_sparse( shared_ptr array_schema, ResultTile& result_tile, bool negated = false) { - std::vector sparse_result_bitmap(cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector sparse_result_bitmap(cells, 1, resource); REQUIRE(tp.qc_ .apply_sparse( *array_schema, result_tile, sparse_result_bitmap) @@ -3300,7 +3308,7 @@ void validate_qc_apply_sparse( CHECK(sparse_result_bitmap[i] == tp.expected_bitmap_[i]); } - std::vector sparse_result_bitmap1(cells, 2); + tdb::pmr::vector sparse_result_bitmap1(cells, 2, resource); REQUIRE(tp.qc_ .apply_sparse( *array_schema, result_tile, sparse_result_bitmap1) @@ -5073,7 +5081,9 @@ TEST_CASE( } // Apply the query condition. - std::vector result_bitmap(cells, 1); + auto resource = tiledb::test::get_test_memory_tracker()->get_resource( + MemoryType::TILE_BITMAP); + tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, result_tile, result_bitmap) .ok()); From fb4c035402e6f6faafb21d787851a46d3b17d4db Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Thu, 7 Mar 2024 12:23:39 +0200 Subject: [PATCH 228/456] Fix crash in aggregates on dimensions (#4794) RTREE wasn't loaded when processing aggregates on dimensions. We use the RTREE when we aggregate full tiles so the fix is to load the rtree for cases where we don't have ranges and we have aggregates on dimensions. --- TYPE: BUG DESC: Fix crash in aggregates on dimensions --- test/src/test-cppapi-aggregates.cc | 32 +++++++++++++++---- .../query/readers/sparse_index_reader_base.cc | 8 +++++ tiledb/sm/subarray/subarray.h | 6 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 07c4ee4a5a25..7efa33159175 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -61,6 +61,7 @@ struct CppAggregatesFx { bool allow_dups_; bool set_ranges_; bool set_qc_; + bool use_dim_; tiledb_layout_t layout_; std::vector set_qc_values_; std::vector layout_values_; @@ -161,6 +162,7 @@ void CppAggregatesFx::generate_test_params() { allow_dups_ = false; set_qc_values_ = {true, false}; layout_values_ = {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR, TILEDB_GLOBAL_ORDER}; + use_dim_ = false; } SECTION("sparse") { @@ -169,6 +171,8 @@ void CppAggregatesFx::generate_test_params() { allow_dups_ = GENERATE(true, false); set_qc_values_ = {false}; layout_values_ = {TILEDB_UNORDERED}; + use_dim_ = + GENERATE(false, true) && !nullable_ && std::is_same::value; } } @@ -1431,6 +1435,7 @@ TEST_CASE_METHOD( for (bool set_ranges : {true, false}) { set_ranges_ = set_ranges; + use_dim_ = use_dim_ && !set_ranges; for (bool request_data : {true, false}) { request_data_ = request_data; for (bool set_qc : set_qc_values_) { @@ -1472,7 +1477,10 @@ TEST_CASE_METHOD( // Check the results. uint64_t expected_count; - if (dense_) { + + if (use_dim_) { + expected_count = 999; + } else if (dense_) { expected_count = set_ranges ? 24 : 36; } else { if (set_ranges) { @@ -1526,6 +1534,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( for (bool set_ranges : {true, false}) { CppAggregatesFx::set_ranges_ = set_ranges; + CppAggregatesFx::use_dim_ = CppAggregatesFx::use_dim_ && !set_ranges; for (bool request_data : {true, false}) { CppAggregatesFx::request_data_ = request_data; for (bool set_qc : CppAggregatesFx::set_qc_values_) { @@ -1539,7 +1548,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( QueryExperimental::get_default_channel(query); ChannelOperation operation = QueryExperimental::create_unary_aggregate( - query, "a1"); + query, CppAggregatesFx::use_dim_ ? "d1" : "a1"); default_channel.apply_aggregate("Sum", operation); CppAggregatesFx::set_ranges_and_condition_if_needed( @@ -1575,7 +1584,9 @@ TEMPLATE_LIST_TEST_CASE_METHOD( // Check the results. typename tiledb::sm::sum_type_data::sum_type expected_sum; - if (CppAggregatesFx::dense_) { + if (CppAggregatesFx::use_dim_) { + expected_sum = 499500; + } else if (CppAggregatesFx::dense_) { if (CppAggregatesFx::nullable_) { if (set_ranges) { expected_sum = set_qc ? 197 : 201; @@ -1650,6 +1661,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( for (bool set_ranges : {true, false}) { CppAggregatesFx::set_ranges_ = set_ranges; + CppAggregatesFx::use_dim_ = CppAggregatesFx::use_dim_ && !set_ranges; for (bool request_data : {true, false}) { CppAggregatesFx::request_data_ = request_data; for (bool set_qc : CppAggregatesFx::set_qc_values_) { @@ -1662,7 +1674,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( QueryExperimental::get_default_channel(query); ChannelOperation operation = QueryExperimental::create_unary_aggregate( - query, "a1"); + query, CppAggregatesFx::use_dim_ ? "d1" : "a1"); default_channel.apply_aggregate("Mean", operation); CppAggregatesFx::set_ranges_and_condition_if_needed( @@ -1698,7 +1710,9 @@ TEMPLATE_LIST_TEST_CASE_METHOD( // Check the results. double expected_mean; - if (CppAggregatesFx::dense_) { + if (CppAggregatesFx::use_dim_) { + expected_mean = 500; + } else if (CppAggregatesFx::dense_) { if (CppAggregatesFx::nullable_) { if (set_ranges) { expected_mean = set_qc ? (197.0 / 11.0) : (201.0 / 12.0); @@ -1789,6 +1803,7 @@ TEMPLATE_LIST_TEST_CASE( for (bool set_ranges : {true, false}) { fx.set_ranges_ = set_ranges; + fx.use_dim_ = fx.use_dim_ && !set_ranges; for (bool request_data : {true, false}) { fx.request_data_ = request_data; for (bool set_qc : fx.set_qc_values_) { @@ -1801,7 +1816,8 @@ TEMPLATE_LIST_TEST_CASE( QueryChannel default_channel = QueryExperimental::get_default_channel(query); ChannelOperation operation = - QueryExperimental::create_unary_aggregate(query, "a1"); + QueryExperimental::create_unary_aggregate( + query, fx.use_dim_ ? "d1" : "a1"); default_channel.apply_aggregate("MinMax", operation); fx.set_ranges_and_condition_if_needed(array, query, false); @@ -1848,7 +1864,9 @@ TEMPLATE_LIST_TEST_CASE( // Check the results. std::vector expected_min_max; - if (fx.dense_) { + if (fx.use_dim_) { + expected_min_max = fx.make_data_buff({min ? 1 : 999}); + } else if (fx.dense_) { if (fx.nullable_) { if (set_ranges) { expected_min_max = diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index c4afe018a2d6..9618125ff0ae 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -388,6 +388,14 @@ Status SparseIndexReaderBase::load_initial_data() { memory_budget_.ratio_tile_ranges() * memory_budget_.total_budget()) return logger_->status( Status_ReaderError("Exceeded memory budget for result tile ranges")); + } else { + for (const auto& [name, _] : aggregates_) { + if (array_schema_.is_dim(name)) { + throw_if_not_ok(subarray_.load_relevant_fragment_rtrees( + storage_manager_->compute_tp())); + break; + } + } } // Compute tile offsets to load and var size to load for attributes. diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index b32a6af0ceaa..bd36e8647700 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -1379,6 +1379,9 @@ class Subarray { */ void reset_default_ranges(); + /** Loads the R-Trees of all relevant fragments in parallel. */ + Status load_relevant_fragment_rtrees(ThreadPool* compute_tp) const; + private: /* ********************************* */ /* PRIVATE DATA TYPES */ @@ -1610,9 +1613,6 @@ class Subarray { */ void swap(Subarray& subarray); - /** Loads the R-Trees of all relevant fragments in parallel. */ - Status load_relevant_fragment_rtrees(ThreadPool* compute_tp) const; - /** * Computes the tile overlap for each range and relevant fragment. * From 8e5f815538b823118e312b2fcb89ae4fcc995a7c Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 7 Mar 2024 05:44:42 -0700 Subject: [PATCH 229/456] Fix defects with query channels (#4786) New `class QueryChannelActual` to replace the previous class that was not actually a channel. Added methods in `class Query` for it, and rewrote the channel C API handle class to use it. Correctly initialize the underlying channel object within the query field C API handle class, and return a channel handle with the right lifespan. (This fixes the defect reported.) Reworked test fixture to use `TemporaryLocalDirectory`. Reworked tests for isolation with SECTION. Rewrote the two tests with aggegrates to use correct calling sequence. Changed `ensure_query_is_not_initialized` to take a reference. Changed `tiledb_query_get_default_channel` to return an actual channel handle rather than a stub for one. Reference: https://app.shortcut.com/tiledb-inc/story/39123/tiledb-field-channel-returns-the-same-handle-and-is-prone-to-double-freeing [sc-39123] --- TYPE: BUG DESC: Fix defects with query channels --------- Co-authored-by: Luc Rancourt --- tiledb/api/c_api/query/query_api_internal.h | 7 +- .../query_aggregate/query_aggregate_api.cc | 9 +- .../query_aggregate_api_internal.h | 25 ++- .../api/c_api/query_field/query_field_api.cc | 14 +- .../query_field/query_field_api_internal.h | 8 +- .../query_field/test/unit_capi_query_field.cc | 193 ++++++++++-------- tiledb/sm/query/query.cc | 11 +- tiledb/sm/query/query.h | 96 ++++++++- .../query/readers/aggregators/query_channel.h | 61 +++++- tiledb/sm/serialization/query.cc | 2 +- tiledb/sm/serialization/query_aggregates.cc | 6 +- 11 files changed, 305 insertions(+), 127 deletions(-) diff --git a/tiledb/api/c_api/query/query_api_internal.h b/tiledb/api/c_api/query/query_api_internal.h index dcc24f111549..404c2bfae8ae 100644 --- a/tiledb/api/c_api/query/query_api_internal.h +++ b/tiledb/api/c_api/query/query_api_internal.h @@ -57,8 +57,8 @@ inline void ensure_query_is_valid(tiledb_query_t* query) { * * @param query A sm::Query pointer */ -inline void ensure_query_is_not_initialized(const tiledb::sm::Query* query) { - if (query->status() != sm::QueryStatus::UNINITIALIZED) { +inline void ensure_query_is_not_initialized(const tiledb::sm::Query& query) { + if (query.status() != sm::QueryStatus::UNINITIALIZED) { throw CAPIStatusException( "argument `query` is at a too late state of its lifetime"); } @@ -72,7 +72,8 @@ inline void ensure_query_is_not_initialized(const tiledb::sm::Query* query) { */ inline void ensure_query_is_not_initialized(tiledb_query_t* query) { ensure_query_is_valid(query); - ensure_query_is_not_initialized(query->query_); + // Indirection safe because previous statement will throw otherwise + ensure_query_is_not_initialized(*query->query_); } } // namespace tiledb::api diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index 786d663b8160..34e18d31dc90 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -161,11 +161,8 @@ capi_return_t tiledb_query_get_default_channel( tiledb_ctx_t*, tiledb_query_t* query, tiledb_query_channel_t** channel) { ensure_query_is_valid(query); ensure_output_pointer_is_valid(channel); - - // We don't have an internal representation of a channel, - // the default channel is currently just a hashmap, so only pass the query - // to the channel constructor to be carried until next the api call. - *channel = tiledb_query_channel_handle_t::make_handle(query); + *channel = tiledb_query_channel_handle_t::make_handle( + query->query_->default_channel()); return TILEDB_OK; } @@ -209,7 +206,7 @@ capi_return_t tiledb_channel_apply_aggregate( const char* output_field_name, const tiledb_channel_operation_t* operation) { ensure_query_channel_is_valid(channel); - ensure_query_is_not_initialized(channel->query_); + ensure_query_is_not_initialized(channel->query()); ensure_output_field_is_valid(output_field_name); ensure_operation_is_valid(operation); channel->add_aggregate(output_field_name, operation); diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h index 1c736f077b03..c33d27ae9676 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api_internal.h @@ -39,6 +39,7 @@ #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/query/readers/aggregators/operation.h" +#include "tiledb/sm/query/readers/aggregators/query_channel.h" struct tiledb_channel_operation_handle_t : public tiledb::api::CAPIHandle { @@ -71,6 +72,11 @@ struct tiledb_channel_operation_handle_t } }; +/* Forward declaration */ +namespace tiledb::sm { +class Query; +} + struct tiledb_query_channel_handle_t : public tiledb::api::CAPIHandle { /** @@ -78,9 +84,10 @@ struct tiledb_query_channel_handle_t */ static constexpr std::string_view object_type_name{"tiledb_query_channel_t"}; - public: - tiledb::sm::Query* query_; + private: + std::shared_ptr channel_; + public: /** * Default constructor doesn't make sense */ @@ -90,14 +97,16 @@ struct tiledb_query_channel_handle_t * Ordinary constructor. * @param query The query object that owns the channel */ - tiledb_query_channel_handle_t(tiledb_query_t* query) - : query_(query->query_) { + tiledb_query_channel_handle_t( + std::shared_ptr channel) + : channel_(channel) { } inline void add_aggregate( const char* output_field, const tiledb_channel_operation_handle_t* operation) { - if (query_->is_aggregate(output_field)) { + auto& query{channel_->query()}; + if (query.is_aggregate(output_field)) { throw tiledb::api::CAPIStatusException( "An aggregate operation for output field: " + std::string(output_field) + " already exists."); @@ -105,9 +114,13 @@ struct tiledb_query_channel_handle_t // Add the aggregator the the default channel as this is the only channel // type we currently support - query_->add_aggregator_to_default_channel( + query.add_aggregator_to_default_channel( output_field, operation->aggregator()); } + + inline tiledb::sm::Query& query() { + return channel_->query(); + } }; struct tiledb_channel_operator_handle_t diff --git a/tiledb/api/c_api/query_field/query_field_api.cc b/tiledb/api/c_api/query_field/query_field_api.cc index 6e396af2e35c..03fc140003b7 100644 --- a/tiledb/api/c_api/query_field/query_field_api.cc +++ b/tiledb/api/c_api/query_field/query_field_api.cc @@ -53,6 +53,7 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( tiledb_query_t* query, const char* field_name) : query_(query->query_) , field_name_(field_name) { + bool is_aggregate{false}; if (field_name_ == tiledb::sm::constants::coords) { field_origin_ = std::make_shared(); type_ = query_->array_schema().domain().dimension_ptr(0)->type(); @@ -72,6 +73,7 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( cell_val_num_ = query_->array_schema().dimension_ptr(field_name_)->cell_val_num(); } else if (query_->is_aggregate(field_name_)) { + is_aggregate = true; field_origin_ = std::make_shared(); auto aggregate = query_->get_aggregate(field_name_).value(); type_ = aggregate->output_datatype(); @@ -80,8 +82,16 @@ tiledb_query_field_handle_t::tiledb_query_field_handle_t( } else { throw tiledb::api::CAPIStatusException("There is no field " + field_name_); } - - channel_ = tiledb_query_channel_handle_t::make_handle(query); + /* + * We have no `class QueryField` that would already know its own aggregate, + * so we mirror the channel selection process that `class Query` has + * responsibility for. + */ + if (is_aggregate) { + channel_ = query_->aggegate_channel(); + } else { + channel_ = query_->default_channel(); + } } namespace tiledb::api { diff --git a/tiledb/api/c_api/query_field/query_field_api_internal.h b/tiledb/api/c_api/query_field/query_field_api_internal.h index 3e50160537e1..3ac79f9f4c09 100644 --- a/tiledb/api/c_api/query_field/query_field_api_internal.h +++ b/tiledb/api/c_api/query_field/query_field_api_internal.h @@ -73,7 +73,7 @@ struct tiledb_query_field_handle_t std::shared_ptr field_origin_; tiledb::sm::Datatype type_; uint32_t cell_val_num_; - tiledb_query_channel_handle_t* channel_; + std::shared_ptr channel_; public: /** @@ -87,10 +87,6 @@ struct tiledb_query_field_handle_t */ tiledb_query_field_handle_t(tiledb_query_t* query, const char* field_name); - ~tiledb_query_field_handle_t() { - tiledb_query_channel_handle_t::break_handle(channel_); - } - tiledb_field_origin_t origin() { return field_origin_->origin(); } @@ -101,7 +97,7 @@ struct tiledb_query_field_handle_t return cell_val_num_; } tiledb_query_channel_handle_t* channel() { - return channel_; + return tiledb_query_channel_handle_t::make_handle(channel_); } }; diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index 40e1042065e0..e9105f50fb97 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -32,25 +32,38 @@ #include #include "test/support/src/helpers.h" -#include "test/support/src/vfs_helpers.h" +#include "test/support/src/temporary_local_directory.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" using namespace tiledb::test; -struct QueryFieldFx : TemporaryDirectoryFixture { +class QueryFieldFx { + tiledb::sm::TemporaryLocalDirectory tmpslash{}; + std::string test_array_name{tmpslash.path() + "queryfield_array"}; + void write_sparse_array(const std::string& path); + void create_sparse_array(const std::string& path); + + protected: + /** TileDB context */ + tiledb_ctx_t* ctx; + + public: QueryFieldFx() { + if (tiledb_ctx_alloc(nullptr, &ctx) != TILEDB_OK) { + throw std::runtime_error("Failed to allocate context"); + } create_sparse_array(array_name()); write_sparse_array(array_name()); } + ~QueryFieldFx() { + (void)tiledb_ctx_free(&ctx); + } std::string array_name() { - return temp_dir_ + "queryfield_array"; + return test_array_name; } - - void write_sparse_array(const std::string& path); - void create_sparse_array(const std::string& path); }; void QueryFieldFx::write_sparse_array(const std::string& array_name) { @@ -179,15 +192,13 @@ void QueryFieldFx::create_sparse_array(const std::string& array_name) { TEST_CASE_METHOD( QueryFieldFx, - "C API: argument validation", + "C API: argument validation, tiledb_query_get_field", "[capi][query_field][get][args]") { tiledb_array_t* array = nullptr; - ; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query = nullptr; - ; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); tiledb_query_field_t* field = nullptr; @@ -213,22 +224,20 @@ TEST_CASE_METHOD( } // Clean up - tiledb_query_free(&query); + (void)tiledb_query_free(&query); CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); - tiledb_array_free(&array); + (void)tiledb_array_free(&array); } TEST_CASE_METHOD( QueryFieldFx, - "C API: argument validation", + "C API: argument validation, query field properties", "[capi][query_field][access][args]") { tiledb_array_t* array = nullptr; - ; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); tiledb_query_t* query = nullptr; - ; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); tiledb_query_field_t* field = nullptr; @@ -268,13 +277,12 @@ TEST_CASE_METHOD( // Clean up CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); - tiledb_query_free(&query); + (void)tiledb_query_free(&query); CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); - tiledb_array_free(&array); + (void)tiledb_array_free(&array); } -TEST_CASE_METHOD( - QueryFieldFx, "C API: argument validation", "[capi][query_field]") { +TEST_CASE_METHOD(QueryFieldFx, "C API: get_field", "[capi][query_field]") { tiledb_array_t* array = nullptr; REQUIRE(tiledb_array_alloc(ctx, array_name().c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -292,75 +300,94 @@ TEST_CASE_METHOD( uint32_t cell_val_num = 0; tiledb_query_channel_t* channel = nullptr; - // Errors out when the field doesn't exist - CHECK( - tiledb_query_get_field(ctx, query, "non_existent", &field) == TILEDB_ERR); - - // Check field api works on dimension field - REQUIRE(tiledb_query_get_field(ctx, query, "d1", &field) == TILEDB_OK); - - REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); - CHECK(type == TILEDB_UINT64); - REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); - CHECK(origin == TILEDB_DIMENSION_FIELD); - REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); - CHECK(cell_val_num == 1); - - // Check field api works on aggregate field - REQUIRE(tiledb_field_channel(ctx, field, &channel) == TILEDB_OK); - REQUIRE( - tiledb_channel_apply_aggregate( - ctx, channel, "Count", tiledb_aggregate_count) == TILEDB_OK); - uint64_t count = 0; - uint64_t size = 8; - REQUIRE( - tiledb_query_set_data_buffer(ctx, query, "Count", &count, &size) == - TILEDB_OK); - REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); - CHECK(count == 9); - CHECK(tiledb_query_channel_free(ctx, &channel) == TILEDB_OK); - CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + SECTION("Non-existent field") { + // Errors out when the field doesn't exist + CHECK( + tiledb_query_get_field(ctx, query, "non_existent", &field) == + TILEDB_ERR); + }; + + SECTION("Dimension field") { + // Check field api works on dimension field + REQUIRE(tiledb_query_get_field(ctx, query, "d1", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_DIMENSION_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + } - // Check field api works on timestamp field - REQUIRE( - tiledb_query_get_field(ctx, query, "__timestamps", &field) == TILEDB_OK); - REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); - CHECK(type == TILEDB_UINT64); - REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); - CHECK(origin == TILEDB_ATTRIBUTE_FIELD); - REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); - CHECK(cell_val_num == 1); - CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + SECTION("Timestamp field") { + // Check field api works on timestamp field + REQUIRE( + tiledb_query_get_field(ctx, query, "__timestamps", &field) == + TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_ATTRIBUTE_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + } - // Check field api works on coords field - REQUIRE(tiledb_query_get_field(ctx, query, "__coords", &field) == TILEDB_OK); - REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); - CHECK(type == TILEDB_UINT64); - REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); - CHECK(origin == TILEDB_DIMENSION_FIELD); - REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); - CHECK(cell_val_num == 1); - CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + SECTION("\"coords\" field") { + // Check field api works on coords field + REQUIRE( + tiledb_query_get_field(ctx, query, "__coords", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_DIMENSION_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + } - // Check field api works on attribute field - REQUIRE(tiledb_query_get_field(ctx, query, "c", &field) == TILEDB_OK); - REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); - CHECK(type == TILEDB_STRING_ASCII); - REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); - CHECK(origin == TILEDB_ATTRIBUTE_FIELD); - REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); - CHECK(cell_val_num == TILEDB_VAR_NUM); - CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + SECTION("Attribute field") { + // Check field api works on attribute field + REQUIRE(tiledb_query_get_field(ctx, query, "c", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_STRING_ASCII); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_ATTRIBUTE_FIELD); + REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == TILEDB_VAR_NUM); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + } - // Check field api works on aggregate field - REQUIRE(tiledb_query_get_field(ctx, query, "Count", &field) == TILEDB_OK); - REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); - CHECK(type == TILEDB_UINT64); - REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); - CHECK(origin == TILEDB_AGGREGATE_FIELD); - REQUIRE(tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); - CHECK(cell_val_num == 1); - CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + SECTION("Aggregate field") { + // Check field api works on aggregate field + REQUIRE( + tiledb_query_get_default_channel(ctx, query, &channel) == TILEDB_OK); + REQUIRE( + tiledb_channel_apply_aggregate( + ctx, channel, "Count", tiledb_aggregate_count) == TILEDB_OK); + SECTION("validate") { + // Check field api works on aggregate field + REQUIRE(tiledb_query_get_field(ctx, query, "Count", &field) == TILEDB_OK); + REQUIRE(tiledb_field_datatype(ctx, field, &type) == TILEDB_OK); + CHECK(type == TILEDB_UINT64); + REQUIRE(tiledb_field_origin(ctx, field, &origin) == TILEDB_OK); + CHECK(origin == TILEDB_AGGREGATE_FIELD); + REQUIRE( + tiledb_field_cell_val_num(ctx, field, &cell_val_num) == TILEDB_OK); + CHECK(cell_val_num == 1); + CHECK(tiledb_query_field_free(ctx, &field) == TILEDB_OK); + } + SECTION("run query") { + uint64_t count = 0; + uint64_t size = 8; + REQUIRE( + tiledb_query_set_data_buffer(ctx, query, "Count", &count, &size) == + TILEDB_OK); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + CHECK(count == 9); + } + CHECK(tiledb_query_channel_free(ctx, &channel) == TILEDB_OK); + } // Clean up tiledb_query_free(&query); diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 9e3880646dcb..837d487d60fb 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -67,14 +67,6 @@ using namespace tiledb::sm::stats; namespace tiledb { namespace sm { -/** Class for query status exceptions. */ -class QueryStatusException : public StatusException { - public: - explicit QueryStatusException(const std::string& msg) - : StatusException("Query", msg) { - } -}; - /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -113,7 +105,8 @@ Query::Query( , is_dimension_label_ordered_read_(false) , dimension_label_increasing_(true) , fragment_size_(std::numeric_limits::max()) - , query_remote_buffer_storage_(std::nullopt) { + , query_remote_buffer_storage_(std::nullopt) + , default_channel_{make_shared(HERE(), *this, 0)} { assert(array->is_open()); if (array->get_query_type() == QueryType::READ) { diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 520934c61df0..4b84f3313ddc 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -62,6 +62,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for query status exceptions. */ +class QueryStatusException : public StatusException { + public: + explicit QueryStatusException(const std::string& msg) + : StatusException("Query", msg) { + } +}; + class Array; class ArrayDimensionLabelQueries; @@ -740,6 +748,21 @@ class Query { /** Returns true if the output field is an aggregate. */ bool is_aggregate(std::string output_field_name) const; + private: + /** + * Create the aggregate channel object. This is split out because it's not + * at construction time, but on demand, and in two different situations. + */ + void create_aggregate_channel() { + /* + * Because we have an extremely simple way of choosing channel identifiers, + * we can get away with hard-coding `1` here as the identifier for the + * aggregate channel. + */ + aggregate_channel_ = make_shared(HERE(), *this, 1); + } + + public: /** * Adds an aggregator to the default channel. * @@ -748,6 +771,15 @@ class Query { */ void add_aggregator_to_default_channel( std::string output_field_name, shared_ptr aggregator) { + if (default_channel_aggregates_.empty()) { + /* + * Assert: this is the first aggregate added. + * + * We create the aggregate channel on demand, and this is when we need to + * do it. + */ + create_aggregate_channel(); + } default_channel_aggregates_.emplace(output_field_name, aggregator); } @@ -758,20 +790,25 @@ class Query { /** * Get a list of all channels and their aggregates */ - std::vector get_channels() { + std::vector get_channels() { // Currently only the default channel is supported - return {QueryChannel(true, default_channel_aggregates_)}; + return { + LegacyQueryAggregatesOverDefault(true, default_channel_aggregates_)}; } /** - * Add a channel to the query + * Add a channel to the query. Used only by capnp serialization to initialize + * the aggregates list. */ - void add_channel(const QueryChannel& channel) { + void add_channel(const LegacyQueryAggregatesOverDefault& channel) { if (channel.is_default()) { default_channel_aggregates_ = channel.aggregates(); + if (!default_channel_aggregates_.empty()) { + create_aggregate_channel(); + } return; } - throw std::logic_error( + throw QueryStatusException( "We currently only support a default channel for queries"); } @@ -780,7 +817,35 @@ class Query { */ bool has_aggregates() { // We only need to check the default channel for now - return default_channel_aggregates_.empty(); + return !default_channel_aggregates_.empty(); + } + + /** + * Returns the number of channels. + * + * Responsibility for choosing channel identifiers is the responsibility of + * this class. At the present time the policy is very simple, since all + * queries only draw from a single array. + * - Channel 0: All rows from the query. Always non-segmented, that is, + * without any grouping. + * - Channel 1: (optional) Simple aggregates, if any exist. + */ + inline size_t number_of_channels() { + return has_aggregates() ? 1 : 0; + }; + + /** + * The default channel is initialized at construction and always exists. + */ + inline std::shared_ptr default_channel() { + return default_channel_; + } + + inline std::shared_ptr aggegate_channel() { + if (!has_aggregates()) { + throw QueryStatusException("Aggregate channel does not exist"); + } + return aggregate_channel_; } private: @@ -968,6 +1033,25 @@ class Query { std::unordered_map> default_channel_aggregates_; + /* + * Handles to channels use shared pointers, so the channels are allocated here + * for ease of implementation. + * + * At present there's only one possible non-default channel, so we keep track + * of it in its own variable. A fully C.41 class might simply store these in + * a constant vector. + */ + /** + * The default channel is allocated in the constructor for simplicity. + */ + std::shared_ptr default_channel_; + + /** + * The aggegregate channel is optional, so we initialize it as empty with it + * default constructor. + */ + std::shared_ptr aggregate_channel_{}; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/query/readers/aggregators/query_channel.h b/tiledb/sm/query/readers/aggregators/query_channel.h index 62d87c15395f..e619d75adc82 100644 --- a/tiledb/sm/query/readers/aggregators/query_channel.h +++ b/tiledb/sm/query/readers/aggregators/query_channel.h @@ -39,7 +39,11 @@ namespace tiledb::sm { -class QueryChannel { +/** + * Original class is only used for capnp (de)serialization. `class Query` uses + * its own container to hold aggregates. + */ +class LegacyQueryAggregatesOverDefault { public: using ChannelAggregates = std::unordered_map>; @@ -54,7 +58,8 @@ class QueryChannel { * @param is_default If true, this is the default query channel * @param aggregates A map of aggregators by output field name */ - QueryChannel(bool is_default, const ChannelAggregates& aggregates) + LegacyQueryAggregatesOverDefault( + bool is_default, const ChannelAggregates& aggregates) : default_(is_default) , aggregates_{aggregates} { } @@ -85,6 +90,58 @@ class QueryChannel { ChannelAggregates aggregates_; }; +/* forward declaration */ +class Query; + +/** + * Replacement for the current QueryChannel, which does not work for more than + * the initial case with a default channel and only simple aggregates. + * + * Responsibility for choosing channel identifiers is the responsibility of + * `class Query`; this class merely carries the resulting identifier. + */ +class QueryChannel { + std::reference_wrapper query_; + size_t id_; + + public: + /** + * Default constructor is deleted. A channel makes no sense without a query. + */ + QueryChannel() = delete; + /* + * Ordinary constructor. + */ + QueryChannel(Query& q, size_t id) + : query_(q) + , id_(id) { + (void)id_; + } + /** + * Copy constructor is the default. + */ + QueryChannel(const QueryChannel&) = default; + /** + * Move constructor is the default. + */ + QueryChannel(QueryChannel&&) = default; + /** + * Copy assignment is the default. + */ + QueryChannel& operator=(const QueryChannel&) = default; + /** + * Move assignment is the default. + */ + QueryChannel& operator=(QueryChannel&&) = default; + + /** + * Accessor for query member + */ + inline Query& query() { + return query_; + } +}; + } // namespace tiledb::sm #endif // TILEDB_QUERY_CHANNEL_H diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index e3b0cd7c95ca..6e0a894e6063 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -1550,7 +1550,7 @@ Status query_to_capnp( // The server should throw if it's about to serialize an incomplete query // that has aggregates on it, this behavior is currently not supported. if (!client_side && query.status() == QueryStatus::INCOMPLETE && - !query.has_aggregates()) { + query.has_aggregates()) { throw Status_SerializationError( "Aggregates are not currently supported in incomplete remote " "queries"); diff --git a/tiledb/sm/serialization/query_aggregates.cc b/tiledb/sm/serialization/query_aggregates.cc index d3785e1ebd03..63ecf23d297b 100644 --- a/tiledb/sm/serialization/query_aggregates.cc +++ b/tiledb/sm/serialization/query_aggregates.cc @@ -87,7 +87,7 @@ void query_channels_from_capnp( auto channel_reader = channels_reader[i]; if (channel_reader.hasAggregates()) { - QueryChannel::ChannelAggregates aggregates; + LegacyQueryAggregatesOverDefault::ChannelAggregates aggregates; auto aggregates_reader = channel_reader.getAggregates(); for (const auto& aggregate : aggregates_reader) { @@ -119,8 +119,8 @@ void query_channels_from_capnp( } } if (!aggregates.empty()) { - query->add_channel( - QueryChannel{channel_reader.getDefault(), aggregates}); + query->add_channel(LegacyQueryAggregatesOverDefault{ + channel_reader.getDefault(), aggregates}); } } } From 899f5bcefca839569d867b77d8c3015019aaa01f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 7 Mar 2024 18:10:13 +0200 Subject: [PATCH 230/456] Move the Hadoop header files to `hdfs_filesystem.cc`. (#4795) [SC-42734](https://app.shortcut.com/tiledb-inc/story/42734) This PR moves including `hadoop/hdfs.h` from `hdfs_filesystem.h` to the `hdfs_filesystem.cc` implementation file. Due to the current structure of the code, `hdfs_filesystem.h` is included by `vfs.h`, which is included by many files, resulting in having to specify the path to `hadoop/hdfs.h` in all dependents, such as unit tests. A similar change was recently made for the GCS SDK in #4777. Thanks to @rroelke for finding this issue. Validated by successfully building all unit tests touched by #4793 with HDFS enabled. --- TYPE: BUILD DESC: Fix compiling unit tests when HDFS is enabled. --- tiledb/sm/filesystem/hdfs_filesystem.cc | 2 ++ tiledb/sm/filesystem/hdfs_filesystem.h | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/filesystem/hdfs_filesystem.cc b/tiledb/sm/filesystem/hdfs_filesystem.cc index 75eee420fa08..a4ce1dcde676 100644 --- a/tiledb/sm/filesystem/hdfs_filesystem.cc +++ b/tiledb/sm/filesystem/hdfs_filesystem.cc @@ -49,6 +49,8 @@ #include "tiledb/sm/misc/utils.h" #include "uri.h" +#include "hadoop/hdfs.h" + #include #include #include diff --git a/tiledb/sm/filesystem/hdfs_filesystem.h b/tiledb/sm/filesystem/hdfs_filesystem.h index 44ca198e4acf..899fbd884afa 100644 --- a/tiledb/sm/filesystem/hdfs_filesystem.h +++ b/tiledb/sm/filesystem/hdfs_filesystem.h @@ -41,10 +41,13 @@ #include "tiledb/common/status.h" -#include "hadoop/hdfs.h" - using namespace tiledb::common; +// Declarations copied from hadoop/hdfs.h +// We do not include it here to avoid leaking it to consuming code. +struct hdfs_internal; +typedef struct hdfs_internal* hdfsFS; + namespace tiledb { namespace common::filesystem { From 1e9154b4c502fe675cbf0384b320114a90c0acfc Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:30:09 +0100 Subject: [PATCH 231/456] Remove copy and move constructors from WriterTile. (#4788) This change removes all move and copy constructors from WriterTile to help with the PMR effort. It introduced a new IndexList container that allows to not have a move constructor but also to access items with an index. [sc-42467] --- TYPE: NO_HISTORY DESC: Remove copy and move constructors from WriterTile. --- test/src/unit-enumerations.cc | 19 +- test/src/unit-filter-pipeline.cc | 402 ++++++++++-------- tiledb/common/indexed_list.h | 189 ++++++++ .../fragment_meta_consolidator.cc | 15 +- tiledb/sm/filter/test/tile_data_generator.h | 21 +- .../filter/test/unit_run_filter_pipeline.cc | 24 +- tiledb/sm/fragment/fragment_metadata.cc | 64 +-- tiledb/sm/fragment/fragment_metadata.h | 6 +- tiledb/sm/metadata/test/unit_metadata.cc | 54 +-- .../deletes_and_updates.cc | 2 +- .../deletes_and_updates/serialization.cc | 12 +- .../query/deletes_and_updates/serialization.h | 4 +- .../test/unit_delete_condition.cc | 2 +- .../test/unit_update_condition.cc | 2 +- tiledb/sm/query/legacy/reader.cc | 16 +- tiledb/sm/query/legacy/reader.h | 13 +- .../sm/query/writers/global_order_writer.cc | 66 +-- tiledb/sm/query/writers/ordered_writer.cc | 13 +- tiledb/sm/query/writers/ordered_writer.h | 2 +- tiledb/sm/query/writers/unordered_writer.cc | 17 +- tiledb/sm/query/writers/writer_base.cc | 6 +- tiledb/sm/query/writers/writer_base.h | 5 +- tiledb/sm/storage_manager/storage_manager.cc | 20 +- tiledb/sm/tile/generic_tile_io.cc | 12 +- tiledb/sm/tile/generic_tile_io.h | 6 +- tiledb/sm/tile/tile.cc | 55 +-- tiledb/sm/tile/tile.h | 25 +- tiledb/sm/tile/writer_tile_tuple.cc | 88 +--- tiledb/sm/tile/writer_tile_tuple.h | 10 +- 29 files changed, 650 insertions(+), 520 deletions(-) create mode 100644 tiledb/common/indexed_list.h diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index b96bb5a98069..c7f5747d0e24 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -103,7 +103,7 @@ struct EnumerationFx { void check_storage_deserialization(const std::vector& values); storage_size_t calculate_serialized_size(shared_ptr enmr); - WriterTile serialize_to_tile(shared_ptr enmr); + shared_ptr serialize_to_tile(shared_ptr enmr); template std::vector as_vector(shared_ptr enmr); @@ -987,11 +987,11 @@ TEST_CASE_METHOD( auto enmr = create_enumeration(values); auto tile = serialize_to_tile(enmr); - REQUIRE(tile.size() > 4); - auto data = tile.data(); + REQUIRE(tile->size() > 4); + auto data = tile->data(); memset(data, 1, 4); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); REQUIRE_THROWS(Enumeration::deserialize(deserializer, memory_tracker_)); } @@ -2711,7 +2711,7 @@ template void EnumerationFx::check_storage_serialization(const std::vector& values) { auto enmr = create_enumeration(values); auto tile = serialize_to_tile(enmr); - REQUIRE(tile.size() == calculate_serialized_size(enmr)); + REQUIRE(tile->size() == calculate_serialized_size(enmr)); } template @@ -2720,7 +2720,7 @@ void EnumerationFx::check_storage_deserialization( auto enmr = create_enumeration(values); auto tile = serialize_to_tile(enmr); - Deserializer deserializer(tile.data(), tile.size()); + Deserializer deserializer(tile->data(), tile->size()); auto deserialized = Enumeration::deserialize(deserializer, memory_tracker_); REQUIRE(deserialized->name() == enmr->name()); @@ -2790,14 +2790,13 @@ storage_size_t EnumerationFx::calculate_serialized_size( return num_bytes; } -WriterTile EnumerationFx::serialize_to_tile( +shared_ptr EnumerationFx::serialize_to_tile( shared_ptr enmr) { SizeComputationSerializer size_serializer; enmr->serialize(size_serializer); - WriterTile tile{ - WriterTile::from_generic(size_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + auto tile{WriterTile::from_generic(size_serializer.size(), memory_tracker_)}; + Serializer serializer(tile->data(), tile->size()); enmr->serialize(serializer); return tile; diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc index 412f9eb6adc8..84e0060b5f22 100644 --- a/test/src/unit-filter-pipeline.cc +++ b/test/src/unit-filter-pipeline.cc @@ -68,30 +68,32 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; -WriterTile make_increasing_tile( +shared_ptr make_increasing_tile( const uint64_t nelts, shared_ptr tracker) { const uint64_t tile_size = nelts * sizeof(uint64_t); const uint64_t cell_size = sizeof(uint64_t); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, cell_size, tile_size, tracker); for (uint64_t i = 0; i < nelts; i++) { - CHECK_NOTHROW(tile.write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); } return tile; } -WriterTile make_offsets_tile( +shared_ptr make_offsets_tile( std::vector& offsets, shared_ptr tracker) { const uint64_t offsets_tile_size = offsets.size() * constants::cell_var_offset_size; - WriterTile offsets_tile( + auto offsets_tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, constants::cell_var_offset_size, @@ -100,7 +102,7 @@ WriterTile make_offsets_tile( // Set up test data for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK_NOTHROW(offsets_tile.write( + CHECK_NOTHROW(offsets_tile->write( &offsets[i], i * constants::cell_var_offset_size, constants::cell_var_offset_size)); @@ -110,15 +112,17 @@ WriterTile make_offsets_tile( } Tile create_tile_for_unfiltering( - uint64_t nelts, WriterTile& tile, shared_ptr tracker) { + uint64_t nelts, + shared_ptr tile, + shared_ptr tracker) { return { - tile.format_version(), - tile.type(), - tile.cell_size(), + tile->format_version(), + tile->type(), + tile->cell_size(), 0, - tile.cell_size() * nelts, - tile.filtered_buffer().data(), - tile.filtered_buffer().size(), + tile->cell_size() * nelts, + tile->filtered_buffer().data(), + tile->filtered_buffer().size(), tracker}; } @@ -160,10 +164,11 @@ TEST_CASE( ThreadPool tp(4); ChecksumMD5Filter md5_filter(Datatype::UINT64); md5_pipeline.add_filter(md5_filter); - CHECK(md5_pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK( + md5_pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, md5_pipeline); @@ -181,11 +186,11 @@ TEST_CASE( FilterPipeline sha_256_pipeline; ChecksumMD5Filter sha_256_filter(Datatype::UINT64); sha_256_pipeline.add_filter(sha_256_filter); - CHECK( - sha_256_pipeline.run_forward(&test::g_helper_stats, &tile2, nullptr, &tp) - .ok()); - CHECK(tile2.size() == 0); - CHECK(tile2.filtered_buffer().size() != 0); + CHECK(sha_256_pipeline + .run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, sha_256_pipeline); @@ -212,11 +217,11 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); // Sanity check number of windows value uint64_t offset = 0; @@ -226,7 +231,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { offset += sizeof(uint32_t); // First chunk metadata size CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == nelts * sizeof(uint64_t)); // Original length offset += sizeof(uint32_t); @@ -236,11 +241,11 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { (nelts * sizeof(uint64_t)) / max_win_size + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == expected_num_win); // Number of windows // Check compression worked - auto compressed_size = tile.filtered_buffer().size(); + auto compressed_size = tile->filtered_buffer().size(); CHECK(compressed_size < nelts * sizeof(uint64_t)); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); @@ -262,10 +267,11 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK( + pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -285,7 +291,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); INFO("Random element seed: " << seed); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, sizeof(uint64_t), @@ -295,13 +302,13 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = (uint64_t)rng(gen); - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -322,7 +329,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { std::numeric_limits::max()); INFO("Random element seed: " << seed); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -332,13 +340,13 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint32_t val = (uint32_t)rng(gen); - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -351,7 +359,8 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { } SECTION("- Byte overflow") { - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, sizeof(uint64_t), @@ -361,13 +370,13 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = i % 257; - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -430,33 +439,35 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); uint64_t offset = 0; CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == 9); // Number of chunks offset += sizeof(uint64_t); for (uint64_t i = 0; i < 9; i++) { CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == out_sizes[i]); // Chunk orig size offset += sizeof(uint32_t); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == out_sizes[i] / 8); // Chunk filtered size offset += sizeof(uint32_t); - uint32_t md_size = tile.filtered_buffer().value_at_as(offset); + uint32_t md_size = tile->filtered_buffer().value_at_as(offset); offset += sizeof(uint32_t); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == out_sizes[i]); // Original length offset += sizeof(uint32_t); @@ -466,7 +477,7 @@ TEST_CASE( auto expected_num_win = out_sizes[i] / max_win_size + uint32_t(bool(out_sizes[0] % max_win_size)); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == expected_num_win); // Number of windows offset += md_size - sizeof(uint32_t); @@ -474,13 +485,13 @@ TEST_CASE( // Check all elements are good. uint8_t el = 0; for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile.filtered_buffer().value_at_as(offset) == el++); + CHECK(tile->filtered_buffer().value_at_as(offset) == el++); offset += sizeof(uint8_t); } } // Check compression worked - auto compressed_size = tile.filtered_buffer().size(); + auto compressed_size = tile->filtered_buffer().size(); CHECK(compressed_size < nelts * sizeof(uint64_t)); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); @@ -503,11 +514,12 @@ TEST_CASE( pipeline.get_filter()->set_max_window_size( window_size); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -528,7 +540,8 @@ TEST_CASE( std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); INFO("Random element seed: " << seed); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, sizeof(uint64_t), @@ -539,13 +552,15 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = (uint64_t)rng(gen); - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -567,7 +582,8 @@ TEST_CASE( std::numeric_limits::max()); INFO("Random element seed: " << seed); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -577,7 +593,7 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint32_t val = (uint32_t)rng(gen); - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } std::vector offsets32(offsets); @@ -600,11 +616,12 @@ TEST_CASE( constants::cell_var_offset_size)); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile32, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), &offsets_tile32, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -618,7 +635,8 @@ TEST_CASE( SECTION("- Byte overflow") { WriterTile::set_max_tile_chunk_size(80); - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::UINT64, sizeof(uint64_t), @@ -628,15 +646,17 @@ TEST_CASE( // Set up test data for (uint64_t i = 0; i < nelts; i++) { uint64_t val = i % 257; - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } auto offsets_tile = make_offsets_tile(offsets, tracker); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -665,11 +685,11 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto pipeline_metadata_size = sizeof(uint64_t) + 3 * sizeof(uint32_t); @@ -677,7 +697,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { offset += sizeof(uint64_t); // Number of chunks offset += sizeof(uint32_t); // First chunk orig size offset += sizeof(uint32_t); // First chunk filtered size - auto filter_metadata_size = tile.filtered_buffer().value_at_as( + auto filter_metadata_size = tile->filtered_buffer().value_at_as( offset); // First chunk metadata size offset += sizeof(uint32_t); @@ -687,11 +707,11 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { (nelts * sizeof(uint64_t)) / max_win_size + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == expected_num_win); // Number of windows // Check encoded size - auto encoded_size = tile.filtered_buffer().size(); + auto encoded_size = tile->filtered_buffer().size(); CHECK( encoded_size == pipeline_metadata_size + filter_metadata_size + nelts * sizeof(uint64_t)); @@ -714,10 +734,11 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK( + pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -734,11 +755,11 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { auto tile = make_increasing_tile(nelts, tracker); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - !pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(!pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); } } @@ -789,30 +810,32 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); uint64_t offset = 0; CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == 9); // Number of chunks offset += sizeof(uint64_t); uint64_t total_md_size = 0; for (uint64_t i = 0; i < 9; i++) { CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == out_sizes[i]); // Chunk orig size offset += sizeof(uint32_t); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == out_sizes[i]); // Chunk filtered size offset += sizeof(uint32_t); - uint32_t md_size = tile.filtered_buffer().value_at_as(offset); + uint32_t md_size = tile->filtered_buffer().value_at_as(offset); offset += sizeof(uint32_t); total_md_size += md_size; @@ -822,7 +845,7 @@ TEST_CASE( (nelts * sizeof(uint64_t)) / max_win_size + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == expected_num_win); // Number of windows offset += md_size; @@ -830,7 +853,7 @@ TEST_CASE( // Check all elements are good. for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { CHECK( - tile.filtered_buffer().value_at_as(offset) == + tile->filtered_buffer().value_at_as(offset) == (j == 0 ? 0 : 1)); offset += sizeof(uint64_t); } @@ -838,7 +861,7 @@ TEST_CASE( // Check encoded size auto pipeline_metadata_size = sizeof(uint64_t) + 9 * 3 * sizeof(uint32_t); - auto encoded_size = tile.filtered_buffer().size(); + auto encoded_size = tile->filtered_buffer().size(); CHECK( encoded_size == pipeline_metadata_size + total_md_size + nelts * sizeof(uint64_t)); @@ -864,11 +887,12 @@ TEST_CASE( pipeline.get_filter()->set_max_window_size( window_size); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -888,12 +912,13 @@ TEST_CASE( WriterTile::set_max_tile_chunk_size(80); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; - CHECK_NOTHROW(tile.write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - !pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) - .ok()); + CHECK(!pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); } WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); @@ -913,10 +938,10 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -932,7 +957,8 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - WriterTile tile2( + auto tile2 = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -941,13 +967,13 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile2, nullptr, &tp).ok()); - CHECK(tile2.size() == 0); - CHECK(tile2.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); @@ -1005,10 +1031,12 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { SECTION("- Single stage") { WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1025,7 +1053,8 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - WriterTile tile2( + auto tile2 = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -1034,14 +1063,15 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile2, &offsets_tile, &tp) - .ok()); - CHECK(tile2.size() == 0); - CHECK(tile2.filtered_buffer().size() != 0); + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile2.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); @@ -1070,10 +1100,10 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1089,7 +1119,8 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - WriterTile tile2( + auto tile2 = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -1098,13 +1129,13 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile2, nullptr, &tp).ok()); - CHECK(tile2.size() == 0); - CHECK(tile2.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); @@ -1162,10 +1193,12 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { SECTION("- Single stage") { WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, &offsets_tile, &tp) + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) .ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1182,7 +1215,8 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - WriterTile tile2( + auto tile2 = make_shared( + HERE(), constants::format_version, Datatype::UINT32, sizeof(uint32_t), @@ -1191,14 +1225,15 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { // Set up test data for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2.write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile2, &offsets_tile, &tp) - .ok()); - CHECK(tile2.size() == 0); - CHECK(tile2.filtered_buffer().size() != 0); + CHECK(pipeline + .run_forward( + &test::g_helper_stats, tile2.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); run_reverse(config, tp, unfiltered_tile2, pipeline); @@ -1228,8 +1263,8 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { pipeline.add_filter(EncryptionAES256GCMFilter(Datatype::UINT64)); // No key set - CHECK( - !pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(!pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); // Create and set a key char key[32]; @@ -1239,10 +1274,10 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { filter->set_key(key); // Check success - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1255,8 +1290,8 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { // Check error decrypting with wrong key. tile = make_increasing_tile(nelts, tracker); - CHECK( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); key[0]++; filter->set_key(key); @@ -1306,7 +1341,8 @@ void testing_float_scaling_filter() { } } - WriterTile tile(constants::format_version, t, cell_size, tile_size, tracker); + auto tile = make_shared( + HERE(), constants::format_version, t, cell_size, tile_size, tracker); std::vector float_result_vec; double scale = 2.53; @@ -1320,7 +1356,7 @@ void testing_float_scaling_filter() { for (uint64_t i = 0; i < nelts; i++) { FloatingType f = dis(gen); CHECK_NOTHROW( - tile.write(&f, i * sizeof(FloatingType), sizeof(FloatingType))); + tile->write(&f, i * sizeof(FloatingType), sizeof(FloatingType))); IntType val = static_cast(round( (f - static_cast(foffset)) / @@ -1344,11 +1380,12 @@ void testing_float_scaling_filter() { ->set_option(FilterOption::SCALE_FLOAT_OFFSET, &foffset) .ok()); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1388,7 +1425,8 @@ void testing_xor_filter(Datatype t) { const uint64_t tile_size = nelts * sizeof(T); const uint64_t cell_size = sizeof(T); - WriterTile tile(constants::format_version, t, cell_size, tile_size, tracker); + auto tile = make_shared( + HERE(), constants::format_version, t, cell_size, tile_size, tracker); // Setting up the random number generator for the XOR filter testing. std::mt19937_64 gen(0x57A672DE); @@ -1399,7 +1437,7 @@ void testing_xor_filter(Datatype t) { for (uint64_t i = 0; i < nelts; i++) { T val = static_cast(dis(gen)); - CHECK_NOTHROW(tile.write(&val, i * sizeof(T), sizeof(T))); + CHECK_NOTHROW(tile->write(&val, i * sizeof(T), sizeof(T))); results.push_back(val); } @@ -1407,11 +1445,12 @@ void testing_xor_filter(Datatype t) { ThreadPool tp(4); pipeline.add_filter(XORFilter(t)); - CHECK(pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); + CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); // Check new size and number of chunks - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); run_reverse(config, tp, unfiltered_tile, pipeline); @@ -1511,21 +1550,22 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { // Initial type of tile is float. std::vector data = { 1.0f, 2.1f, 3.2f, 4.3f, 5.4f, 6.5f, 7.6f, 8.7f, 9.8f, 10.9f}; - WriterTile tile( + auto tile = make_shared( + HERE(), constants::format_version, Datatype::FLOAT32, sizeof(float), sizeof(float) * data.size(), tracker); for (size_t i = 0; i < data.size(); i++) { - CHECK_NOTHROW(tile.write(&data[i], i * sizeof(float), sizeof(float))); + CHECK_NOTHROW(tile->write(&data[i], i * sizeof(float), sizeof(float))); } ThreadPool tp(4); - REQUIRE( - pipeline.run_forward(&test::g_helper_stats, &tile, nullptr, &tp).ok()); - CHECK(tile.size() == 0); - CHECK(tile.filtered_buffer().size() != 0); + REQUIRE(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); auto unfiltered_tile = create_tile_for_unfiltering(data.size(), tile, tracker); diff --git a/tiledb/common/indexed_list.h b/tiledb/common/indexed_list.h new file mode 100644 index 000000000000..e3e87302264f --- /dev/null +++ b/tiledb/common/indexed_list.h @@ -0,0 +1,189 @@ +/** + * @file indexed_list.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2022 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines and implements the `IndexedList` class. + */ + +#ifndef TILEDB_INDEXED_LIST_H +#define TILEDB_INDEXED_LIST_H + +#include +#include + +namespace tiledb::common { + +/** + * Container class for data that cannot be moved but that we want to access by + * an index. + * + * @tparam T The type of the element used in the container + */ +template +class IndexedList { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** Default constructor. */ + IndexedList() { + } + + DISABLE_COPY_AND_COPY_ASSIGN(IndexedList); + DISABLE_MOVE_AND_MOVE_ASSIGN(IndexedList); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Emplace an item to the end of the container. + * + * @param args Arguments forwarded to the initialization function. + * + */ + template + void emplace_back(Args&&... args) { + list_.emplace_back(std::forward(args)...); + vec_.emplace_back(&list_.back()); + } + + /** Returns an iterator to the beginning of the items. */ + typename std::list::iterator begin() { + return list_.begin(); + } + + /** Returns an iterator to the end of the items. */ + typename std::list::iterator end() { + return list_.end(); + } + + /** Returns wether the container is empty or not. */ + bool empty() const { + return list_.empty(); + } + + /** Clears the container. */ + void clear() { + list_.clear(); + vec_.clear(); + } + + /** Returns the number of items in the container. */ + size_t size() const { + return list_.size(); + } + + /** + * Reserve space for a number of items. + * + * @param num Number of items to reserve for. + */ + void reserve(size_t num) { + vec_.reserve(num); + } + + /** + * Resize the container with default constructed items. + * + * Note: Only allowed on an empty container. + * + * @param num Number of items to add. + */ + void resize(size_t num) { + if (list_.size() != 0 || vec_.size() != 0) { + throw std::logic_error( + "Resize should only be called on empty container."); + } + + vec_.reserve(num); + for (uint64_t n = 0; n < num; n++) { + emplace_back(); + } + } + + /** + * Returns a reference to the item at an index. + * + * Note: This API will not throw if an item out of bounds is asked for. + * + * @param index Index of the item to return. + * @return The item. + */ + T& operator[](size_t index) { + return *vec_[index]; + } + + /** + * Returns a reference to the item at an index. + * + * Note: This API will throw if an item out of bounds is asked for. + * + * @param index Index of the item to return. + * @return The item. + */ + T& at(size_t index) { + return *(vec_.at(index)); + } + + /** + * Returns a const reference to the item at an index. + * + * Note: This API will not throw if an item out of bounds is asked for. + * + * @param index Index of the item to return. + * @return The item. + */ + const T& operator[](size_t index) const { + return *vec_[index]; + } + + /** + * Returns a const reference to the item at an index. + * + * Note: This API will throw if an item out of bounds is asked for. + * + * @param index Index of the item to return. + * @return The item. + */ + const T& at(size_t index) const { + return *(vec_.at(index)); + } + + private: + /** List that contains all the elements. */ + std::list list_; + + /** Vector that contains a pointer to the elements allowing indexed access. */ + std::vector vec_; +}; + +} // namespace tiledb::common + +#endif // TILEDB_INDEXED_LIST_H diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 6103c3b8606c..4afa82423b23 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -117,16 +117,13 @@ Status FragmentMetaConsolidator::consolidate( } // Serialize all fragment metadata footers in parallel - std::vector> tiles(meta.size()); + std::vector> tiles(meta.size()); auto status = parallel_for( storage_manager_->compute_tp(), 0, tiles.size(), [&](size_t i) { SizeComputationSerializer size_computation_serializer; meta[i]->write_footer(size_computation_serializer); - tiles[i].reset(tdb_new( - WriterTile, - WriterTile::from_generic( - size_computation_serializer.size(), - consolidator_memory_tracker_))); + tiles[i] = WriterTile::from_generic( + size_computation_serializer.size(), consolidator_memory_tracker_); Serializer serializer(tiles[i]->data(), tiles[i]->size()); meta[i]->write_footer(serializer); @@ -162,10 +159,10 @@ Status FragmentMetaConsolidator::consolidate( SizeComputationSerializer size_computation_serializer; serialize_data(size_computation_serializer, offset); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), consolidator_memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); serialize_data(serializer, offset); // Close array @@ -176,7 +173,7 @@ Status FragmentMetaConsolidator::consolidate( GenericTileIO tile_io(storage_manager_->resources(), uri); [[maybe_unused]] uint64_t nbytes = 0; - tile_io.write_generic(&tile, enc_key, &nbytes); + tile_io.write_generic(tile, enc_key, &nbytes); RETURN_NOT_OK(storage_manager_->vfs()->close_file(uri)); return Status::Ok(); diff --git a/tiledb/sm/filter/test/tile_data_generator.h b/tiledb/sm/filter/test/tile_data_generator.h index 61d30af6c1f5..1caecc356830 100644 --- a/tiledb/sm/filter/test/tile_data_generator.h +++ b/tiledb/sm/filter/test/tile_data_generator.h @@ -60,9 +60,9 @@ class TileDataGenerator { /** * Returns an empty writer tile with enough room for the input data. */ - WriterTile create_empty_writer_tile( + shared_ptr create_empty_writer_tile( shared_ptr memory_tracker) const { - return WriterTile( + return make_shared( constants::format_version, datatype(), cell_size(), @@ -79,8 +79,9 @@ class TileDataGenerator { * test data and the writer offsets tile with the (optional) input offsets * data. */ - virtual std::tuple> create_writer_tiles( - shared_ptr memory_tracker) const = 0; + virtual std:: + tuple, std::optional>> + create_writer_tiles(shared_ptr memory_tracker) const = 0; /** * Returns a tile with the data from the filtered buffer and enough room @@ -154,13 +155,13 @@ class IncrementTileDataGenerator : public TileDataGenerator { } } - std::tuple> create_writer_tiles( - shared_ptr memory_tracker) const override { + tuple, std::optional>> + create_writer_tiles(shared_ptr memory_tracker) const override { // Writer tile. auto tile = create_empty_writer_tile(memory_tracker); T value{}; for (uint64_t index = 0; index < num_elements_; ++index) { - CHECK_NOTHROW(tile.write(&value, index * sizeof(T), sizeof(T))); + CHECK_NOTHROW(tile->write(&value, index * sizeof(T), sizeof(T))); ++value; } @@ -180,20 +181,20 @@ class IncrementTileDataGenerator : public TileDataGenerator { offsets.pop_back(); // Write the offsets tile. - WriterTile offsets_tile( + auto offsets_tile = make_shared( constants::format_version, Datatype::UINT64, constants::cell_var_offset_size, offsets.size() * constants::cell_var_offset_size, memory_tracker); for (uint64_t index = 0; index < offsets.size(); ++index) { - CHECK_NOTHROW(offsets_tile.write( + CHECK_NOTHROW(offsets_tile->write( &offsets[index], index * constants::cell_var_offset_size, constants::cell_var_offset_size)); } - return {std::move(tile), std::move(offsets_tile)}; + return {tile, offsets_tile}; } Datatype datatype() const override { diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 6f91dd7a35c9..2feedb2d0ee5 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -166,8 +166,8 @@ class SimpleVariableTestData { void check_run_pipeline_full( Config& config, ThreadPool& tp, - WriterTile& tile, - std::optional& offsets_tile, + shared_ptr& tile, + std::optional>& offsets_tile, FilterPipeline& pipeline, const TileDataGenerator* test_data, const FilteredTileChecker& filtered_buffer_checker, @@ -176,16 +176,16 @@ void check_run_pipeline_full( CHECK(pipeline .run_forward( &dummy_stats, - &tile, - offsets_tile.has_value() ? &offsets_tile.value() : nullptr, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, &tp) .ok()); // Check the original unfiltered data was removed. - CHECK(tile.size() == 0); + CHECK(tile->size() == 0); // Check the filtered buffer has the expected data. - auto filtered_buffer = tile.filtered_buffer(); + auto filtered_buffer = tile->filtered_buffer(); filtered_buffer_checker.check(filtered_buffer); // Run the data in reverse. @@ -219,8 +219,8 @@ void check_run_pipeline_full( void check_run_pipeline_roundtrip( Config& config, ThreadPool& tp, - WriterTile& tile, - std::optional& offsets_tile, + shared_ptr tile, + std::optional>& offsets_tile, FilterPipeline& pipeline, const TileDataGenerator* test_data, shared_ptr memory_tracker) { @@ -228,17 +228,17 @@ void check_run_pipeline_roundtrip( CHECK(pipeline .run_forward( &dummy_stats, - &tile, - offsets_tile.has_value() ? &offsets_tile.value() : nullptr, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, &tp) .ok()); // Check the original unfiltered data was removed. - CHECK(tile.size() == 0); + CHECK(tile->size() == 0); // Run the data in reverse. auto unfiltered_tile = test_data->create_filtered_buffer_tile( - tile.filtered_buffer(), memory_tracker); + tile->filtered_buffer(), memory_tracker); ChunkData chunk_data; unfiltered_tile.load_chunk_data(chunk_data); CHECK(pipeline diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index b9d1ecb5ef66..8fa9007513f2 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -3885,15 +3885,15 @@ void FragmentMetadata::store_rtree( resources_->stats().add_counter("write_rtree_size", *nbytes); } -WriterTile FragmentMetadata::write_rtree() { +shared_ptr FragmentMetadata::write_rtree() { rtree_.build_tree(); SizeComputationSerializer size_computation_serializer; rtree_.serialize(size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); rtree_.serialize(serializer); return tile; @@ -3984,22 +3984,22 @@ void FragmentMetadata::read_file_footer( void FragmentMetadata::write_generic_tile_to_file( const EncryptionKey& encryption_key, - WriterTile& tile, + shared_ptr tile, uint64_t* nbytes) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); GenericTileIO tile_io(*resources_, fragment_metadata_uri); - tile_io.write_generic(&tile, encryption_key, nbytes); + tile_io.write_generic(tile, encryption_key, nbytes); } -void FragmentMetadata::write_footer_to_file(WriterTile& tile) const { +void FragmentMetadata::write_footer_to_file(shared_ptr tile) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); - uint64_t size = tile.size(); - throw_if_not_ok( - resources_->vfs().write(fragment_metadata_uri, tile.data(), tile.size())); + uint64_t size = tile->size(); + throw_if_not_ok(resources_->vfs().write( + fragment_metadata_uri, tile->data(), tile->size())); // Write the size in the end if there is at least one var-sized dimension if (!array_schema_->domain().all_dims_fixed() || version_ >= 10) { @@ -4013,10 +4013,10 @@ void FragmentMetadata::store_tile_offsets( SizeComputationSerializer size_computation_serializer; write_tile_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4041,10 +4041,10 @@ void FragmentMetadata::store_tile_var_offsets( SizeComputationSerializer size_computation_serializer; write_tile_var_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_var_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4070,10 +4070,10 @@ void FragmentMetadata::store_tile_var_sizes( SizeComputationSerializer size_computation_serializer; write_tile_var_sizes(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_var_sizes(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4098,10 +4098,10 @@ void FragmentMetadata::store_tile_validity_offsets( SizeComputationSerializer size_computation_serializer; write_tile_validity_offsets(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_validity_offsets(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4127,10 +4127,10 @@ void FragmentMetadata::store_tile_mins( SizeComputationSerializer size_computation_serializer; write_tile_mins(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_mins(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4162,10 +4162,10 @@ void FragmentMetadata::store_tile_maxs( SizeComputationSerializer size_computation_serializer; write_tile_maxs(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_maxs(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4197,10 +4197,10 @@ void FragmentMetadata::store_tile_sums( SizeComputationSerializer size_computation_serializer; write_tile_sums(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_sums(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4223,10 +4223,10 @@ void FragmentMetadata::store_tile_null_counts( SizeComputationSerializer size_computation_serializer; write_tile_null_counts(idx, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_tile_null_counts(idx, serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4276,10 +4276,10 @@ void FragmentMetadata::store_fragment_min_max_sum_null_count( SizeComputationSerializer size_computation_serializer; serialize_data(size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); serialize_data(serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4303,10 +4303,10 @@ void FragmentMetadata::store_processed_conditions( SizeComputationSerializer size_computation_serializer; serialize_processed_conditions(size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); serialize_processed_conditions(serializer); write_generic_tile_to_file(encryption_key, tile, nbytes); @@ -4633,14 +4633,14 @@ void FragmentMetadata::write_has_delete_meta(Serializer& serializer) const { void FragmentMetadata::store_footer(const EncryptionKey&) { SizeComputationSerializer size_computation_serializer; write_footer(size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); write_footer(serializer); write_footer_to_file(tile); - resources_->stats().add_counter("write_frag_meta_footer_size", tile.size()); + resources_->stats().add_counter("write_frag_meta_footer_size", tile->size()); } void FragmentMetadata::resize_tile_offsets_vectors(uint64_t size) { diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 600e55fa2a7b..f23dd1950623 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -1850,7 +1850,7 @@ class FragmentMetadata { void store_footer(const EncryptionKey& encryption_key); /** Writes the R-tree to a tile. */ - WriterTile write_rtree(); + shared_ptr write_rtree(); /** Writes the non-empty domain to the input buffer. */ void write_non_empty_domain(Serializer& serializer) const; @@ -2060,7 +2060,7 @@ class FragmentMetadata { */ void write_generic_tile_to_file( const EncryptionKey& encryption_key, - WriterTile& tile, + shared_ptr tile, uint64_t* nbytes) const; /** @@ -2069,7 +2069,7 @@ class FragmentMetadata { * retrieval upon reading (as its size is predictable based on the * number of attributes). */ - void write_footer_to_file(WriterTile&) const; + void write_footer_to_file(shared_ptr) const; /** * Simple clean up function called in the case of error. It removes the diff --git a/tiledb/sm/metadata/test/unit_metadata.cc b/tiledb/sm/metadata/test/unit_metadata.cc index dabe928194c7..35c3ed06806b 100644 --- a/tiledb/sm/metadata/test/unit_metadata.cc +++ b/tiledb/sm/metadata/test/unit_metadata.cc @@ -86,20 +86,20 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer1; metadata_to_serialize1.serialize(size_computation_serializer1); - WriterTile tile1{ + auto tile1{ WriterTile::from_generic(size_computation_serializer1.size(), tracker)}; - Serializer serializer1(tile1.data(), tile1.size()); + Serializer serializer1(tile1->data(), tile1->size()); metadata_to_serialize1.serialize(serializer1); metadata_to_serialize2.put(key_2.c_str(), Datatype::FLOAT64, 1, &value_2); SizeComputationSerializer size_computation_serializer2; metadata_to_serialize2.serialize(size_computation_serializer2); - WriterTile tile2{ + auto tile2{ WriterTile::from_generic(size_computation_serializer2.size(), tracker)}; - Serializer serializer2(tile2.data(), tile2.size()); + Serializer serializer2(tile2->data(), tile2->size()); metadata_to_serialize2.serialize(serializer2); metadata_to_serialize3.put( @@ -107,48 +107,48 @@ TEST_CASE( SizeComputationSerializer size_computation_serializer3; metadata_to_serialize3.serialize(size_computation_serializer3); - WriterTile tile3{ + auto tile3{ WriterTile::from_generic(size_computation_serializer3.size(), tracker)}; - Serializer serializer3(tile3.data(), tile3.size()); + Serializer serializer3(tile3->data(), tile3->size()); metadata_to_serialize3.serialize(serializer3); metadata_tiles.resize(3); metadata_tiles[0] = tdb::make_shared( HERE(), - tile1.format_version(), - tile1.type(), - tile1.cell_size(), + tile1->format_version(), + tile1->type(), + tile1->cell_size(), 0, - tile1.size(), - tile1.filtered_buffer().data(), - tile1.filtered_buffer().size(), + tile1->size(), + tile1->filtered_buffer().data(), + tile1->filtered_buffer().size(), tracker); - memcpy(metadata_tiles[0]->data(), tile1.data(), tile1.size()); + memcpy(metadata_tiles[0]->data(), tile1->data(), tile1->size()); metadata_tiles[1] = tdb::make_shared( HERE(), - tile2.format_version(), - tile2.type(), - tile2.cell_size(), + tile2->format_version(), + tile2->type(), + tile2->cell_size(), 0, - tile2.size(), - tile2.filtered_buffer().data(), - tile2.filtered_buffer().size(), + tile2->size(), + tile2->filtered_buffer().data(), + tile2->filtered_buffer().size(), tracker); - memcpy(metadata_tiles[1]->data(), tile2.data(), tile2.size()); + memcpy(metadata_tiles[1]->data(), tile2->data(), tile2->size()); metadata_tiles[2] = tdb::make_shared( HERE(), - tile3.format_version(), - tile3.type(), - tile3.cell_size(), + tile3->format_version(), + tile3->type(), + tile3->cell_size(), 0, - tile3.size(), - tile3.filtered_buffer().data(), - tile3.filtered_buffer().size(), + tile3->size(), + tile3->filtered_buffer().data(), + tile3->filtered_buffer().size(), tracker); - memcpy(metadata_tiles[2]->data(), tile3.data(), tile3.size()); + memcpy(metadata_tiles[2]->data(), tile3->data(), tile3->size()); meta = Metadata::deserialize(metadata_tiles); Datatype type; diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index 1211d58105bb..29e5c4a095ba 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -151,7 +151,7 @@ Status DeletesAndUpdates::dowork() { // Serialize the negated condition (aud update values if they are not empty) // and write to disk. - WriterTile serialized_condition = + auto serialized_condition = update_values_.empty() ? tiledb::sm::deletes_and_updates::serialization::serialize_condition( condition_->negated_condition(), query_memory_tracker_) : diff --git a/tiledb/sm/query/deletes_and_updates/serialization.cc b/tiledb/sm/query/deletes_and_updates/serialization.cc index eb4721a743f2..8a3a7467b025 100644 --- a/tiledb/sm/query/deletes_and_updates/serialization.cc +++ b/tiledb/sm/query/deletes_and_updates/serialization.cc @@ -100,13 +100,13 @@ storage_size_t get_serialized_condition_size( return size_computation_serializer.size(); } -WriterTile serialize_condition( +shared_ptr serialize_condition( const QueryCondition& query_condition, shared_ptr memory_tracker) { - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( get_serialized_condition_size(query_condition.ast()), memory_tracker)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); serialize_condition_impl(query_condition.ast(), serializer); return tile; @@ -196,16 +196,16 @@ storage_size_t get_serialized_update_condition_and_values_size( return size_computation_serializer.size(); } -WriterTile serialize_update_condition_and_values( +shared_ptr serialize_update_condition_and_values( const QueryCondition& query_condition, const std::vector& update_values, shared_ptr memory_tracker) { - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( get_serialized_update_condition_and_values_size( query_condition.ast(), update_values), memory_tracker)}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); serialize_condition_impl(query_condition.ast(), serializer); serialize_update_values_impl(update_values, serializer); diff --git a/tiledb/sm/query/deletes_and_updates/serialization.h b/tiledb/sm/query/deletes_and_updates/serialization.h index 758ec69a5624..dfd5c1791292 100644 --- a/tiledb/sm/query/deletes_and_updates/serialization.h +++ b/tiledb/sm/query/deletes_and_updates/serialization.h @@ -48,7 +48,7 @@ enum class NodeType : uint8_t { EXPRESSION = 0, VALUE }; * @param query_condition Query condition to serialize. * @return Serialized query condition tile. */ -WriterTile serialize_condition( +shared_ptr serialize_condition( const QueryCondition& query_condition, shared_ptr memory_tracker); @@ -75,7 +75,7 @@ QueryCondition deserialize_condition( * @param update_values Update values to serialize. * @return Serialized condition and update values tile. */ -WriterTile serialize_update_condition_and_values( +shared_ptr serialize_update_condition_and_values( const QueryCondition& query_condition, const std::vector& update_values, shared_ptr memory_tracker); diff --git a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc index 9253c4bd67e6..693b82344a80 100644 --- a/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc +++ b/tiledb/sm/query/deletes_and_updates/test/unit_delete_condition.cc @@ -50,7 +50,7 @@ void serialize_deserialize_check(QueryCondition& query_condition) { auto tracker = tiledb::test::create_test_memory_tracker(); auto serialized = serialize_condition(query_condition, tracker); auto deserialized = - deserialize_condition(0, "", serialized.data(), serialized.size()); + deserialize_condition(0, "", serialized->data(), serialized->size()); CHECK(tiledb::test::ast_equal(query_condition.ast(), deserialized.ast())); } diff --git a/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc b/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc index ceeda833f689..2204f99456b8 100644 --- a/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc +++ b/tiledb/sm/query/deletes_and_updates/test/unit_update_condition.cc @@ -54,7 +54,7 @@ void serialize_deserialize_check( query_condition, update_values, tracker); auto&& [deserialized_condition, deserialized_update_values] = deserialize_update_condition_and_values( - 0, "", serialized.data(), serialized.size()); + 0, "", serialized->data(), serialized->size()); CHECK(tiledb::test::ast_equal( query_condition.ast(), deserialized_condition.ast())); diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index b2292d98a3a2..0cf5015261a3 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -494,7 +494,7 @@ Status Reader::compute_range_result_coords( Subarray& subarray, const std::vector& single_fragment, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector>& range_result_coords) { auto timer_se = stats_->start_timer("compute_range_result_coords"); @@ -549,7 +549,7 @@ Status Reader::compute_range_result_coords( uint64_t range_idx, uint32_t fragment_idx, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector& range_result_coords) { // Skip dense fragments if (fragment_metadata_[fragment_idx]->dense()) @@ -608,7 +608,7 @@ Status Reader::compute_range_result_coords( Subarray& subarray, uint64_t range_idx, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector& range_result_coords) { // Gather result range coordinates per fragment auto fragment_num = fragment_metadata_.size(); @@ -683,7 +683,7 @@ Status Reader::compute_subarray_coords( } Status Reader::compute_sparse_result_tiles( - std::list& result_tiles, + IndexedList& result_tiles, std::map, size_t>* result_tile_map, std::vector* single_fragment) { auto timer_se = stats_->start_timer("compute_sparse_result_tiles"); @@ -1587,7 +1587,7 @@ Status Reader::compute_result_cell_slabs_global( } Status Reader::compute_result_coords( - std::list& result_tiles, + IndexedList& result_tiles, std::vector& result_coords) { auto timer_se = stats_->start_timer("compute_result_coords"); @@ -1726,7 +1726,7 @@ Status Reader::dense_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - std::list sparse_result_tiles; + IndexedList sparse_result_tiles; RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); // Compute result cell slabs. @@ -1984,7 +1984,7 @@ Status Reader::sparse_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - std::list sparse_result_tiles; + IndexedList sparse_result_tiles; RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); std::vector result_tiles; @@ -2066,7 +2066,7 @@ bool Reader::sparse_tile_overwritten( return false; } -void Reader::erase_coord_tiles(std::list& result_tiles) const { +void Reader::erase_coord_tiles(IndexedList& result_tiles) const { for (auto& tile : result_tiles) { auto dim_num = array_schema_.dim_num(); for (unsigned d = 0; d < dim_num; ++d) diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index 997acdcca22d..d544edee0332 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -36,6 +36,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/indexed_list.h" #include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" @@ -305,7 +306,7 @@ class Reader : public ReaderBase, public IQueryStrategy { Subarray& subarray, const std::vector& single_fragment, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector>& range_result_coords); /** @@ -325,7 +326,7 @@ class Reader : public ReaderBase, public IQueryStrategy { Subarray& subarray, uint64_t range_idx, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector& range_result_coords); /** @@ -347,7 +348,7 @@ class Reader : public ReaderBase, public IQueryStrategy { uint64_t range_idx, uint32_t fragment_idx, const std::map, size_t>& result_tile_map, - std::list& result_tiles, + IndexedList& result_tiles, std::vector& range_result_coords); /** @@ -382,7 +383,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * @return Status */ Status compute_sparse_result_tiles( - std::list& result_tiles, + IndexedList& result_tiles, std::map, size_t>* result_tile_map, std::vector* single_fragment); @@ -574,7 +575,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * @param result_coords This will store the result coordinates. */ Status compute_result_coords( - std::list& result_tiles, + IndexedList& result_tiles, std::vector& result_coords); /** @@ -651,7 +652,7 @@ class Reader : public ReaderBase, public IQueryStrategy { * Erases the coordinate tiles (zipped or separate) from the input result * tiles. */ - void erase_coord_tiles(std::list& result_tiles) const; + void erase_coord_tiles(IndexedList& result_tiles) const; /** Gets statistics about the result cells. */ void get_result_cell_stats( diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 879715cbf265..20d20785ad98 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -198,21 +198,18 @@ Status GlobalOrderWriter::init_global_write_state() { const auto capacity = array_schema_.capacity(); const auto cell_num_per_tile = coords_info_.has_coords_ ? capacity : domain.cell_num_per_tile(); - auto last_tile_vector = std::pair( - name, WriterTileTupleVector()); - try { - last_tile_vector.second.emplace_back(WriterTileTuple( - array_schema_, - cell_num_per_tile, - var_size, - nullable, - cell_size, - type, - query_memory_tracker_)); - } catch (const std::logic_error& le) { - return Status_WriterError(le.what()); - } - global_write_state_->last_tiles_.emplace(std::move(last_tile_vector)); + auto last_tiles_it = global_write_state_->last_tiles_.emplace( + std::piecewise_construct, + std::forward_as_tuple(name), + std::forward_as_tuple()); + last_tiles_it.first->second.emplace_back( + array_schema_, + cell_num_per_tile, + var_size, + nullable, + cell_size, + type, + query_memory_tracker_); // Initialize cells written global_write_state_->cells_written_[name] = 0; @@ -867,7 +864,10 @@ Status GlobalOrderWriter::prepare_full_tiles( // Initialize attribute and coordinate tiles for (const auto& it : buffers_) { - (*tiles)[it.first] = WriterTileTupleVector(); + (*tiles).emplace( + std::piecewise_construct, + std::forward_as_tuple(it.first), + std::forward_as_tuple()); } auto num = buffers_.size(); @@ -969,22 +969,26 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( if (full_tile_num > 0) { tiles->reserve(full_tile_num); for (uint64_t i = 0; i < full_tile_num; i++) { - tiles->emplace_back(WriterTileTuple( + tiles->emplace_back( array_schema_, cell_num_per_tile, false, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } // Handle last tile (it must be either full or empty) auto tile_it = tiles->begin(); if (last_tile_cell_idx == cell_num_per_tile) { - tile_it->fixed_tile().swap(last_tile.fixed_tile()); + tile_it->fixed_tile().write( + last_tile.fixed_tile().data(), 0, last_tile.fixed_tile().size()); if (nullable) { - tile_it->validity_tile().swap(last_tile.validity_tile()); + tile_it->validity_tile().write( + last_tile.validity_tile().data(), + 0, + last_tile.validity_tile().size()); } tile_it++; } else if (last_tile_cell_idx != 0) { @@ -1168,8 +1172,6 @@ Status GlobalOrderWriter::prepare_full_tiles_var( ++cell_idx; } while (last_tile_cell_idx != cell_num_per_tile && cell_idx != cell_num); } - - last_tile.var_tile().set_size(last_var_offset); } // Initialize full tiles and set previous last tile as first tile @@ -1182,25 +1184,33 @@ Status GlobalOrderWriter::prepare_full_tiles_var( if (full_tile_num > 0) { tiles->reserve(full_tile_num); for (uint64_t i = 0; i < full_tile_num; i++) { - tiles->emplace_back(WriterTileTuple( + tiles->emplace_back( array_schema_, cell_num_per_tile, true, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } // Handle last tile (it must be either full or empty) auto tile_it = tiles->begin(); if (last_tile_cell_idx == cell_num_per_tile) { - last_var_offset = 0; - tile_it->offset_tile().swap(last_tile.offset_tile()); - tile_it->var_tile().swap(last_tile.var_tile()); + tile_it->offset_tile().write( + last_tile.offset_tile().data(), 0, last_tile.offset_tile().size()); + tile_it->var_tile().write_var( + last_tile.var_tile().data(), 0, last_var_offset); + tile_it->var_tile().set_size(last_var_offset); if (nullable) { - tile_it->validity_tile().swap(last_tile.validity_tile()); + tile_it->validity_tile().write( + last_tile.validity_tile().data(), + 0, + last_tile.validity_tile().size()); } + + last_var_offset = 0; + tile_it++; } else if (last_tile_cell_idx != 0) { return Status_WriterError( diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index aff7ea126352..cc951acc7ee3 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -198,9 +198,12 @@ Status OrderedWriter::ordered_write() { auto attr_num = buffers_.size(); auto compute_tp = storage_manager_->compute_tp(); auto thread_num = compute_tp->concurrency_level(); - std::unordered_map> tiles; + std::unordered_map> tiles; for (const auto& buff : buffers_) { - tiles.emplace(buff.first, std::vector()); + tiles.emplace( + std::piecewise_construct, + std::forward_as_tuple(buff.first), + std::forward_as_tuple()); } if (attr_num > tile_num) { // Parallelize over attributes @@ -284,7 +287,7 @@ Status OrderedWriter::ordered_write() { template Status OrderedWriter::prepare_filter_and_write_tiles( const std::string& name, - std::vector& tile_batches, + IndexedList& tile_batches, shared_ptr frag_meta, DenseTiler* dense_tiler, uint64_t thread_num) { @@ -318,14 +321,14 @@ Status OrderedWriter::prepare_filter_and_write_tiles( assert(batch_size > 0); tile_batches[b].reserve(batch_size); for (uint64_t i = 0; i < batch_size; i++) { - tile_batches[b].emplace_back(WriterTileTuple( + tile_batches[b].emplace_back( array_schema_, cell_num_per_tile, var, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } { diff --git a/tiledb/sm/query/writers/ordered_writer.h b/tiledb/sm/query/writers/ordered_writer.h index 14887f26a846..1bd5943472a7 100644 --- a/tiledb/sm/query/writers/ordered_writer.h +++ b/tiledb/sm/query/writers/ordered_writer.h @@ -127,7 +127,7 @@ class OrderedWriter : public WriterBase { template Status prepare_filter_and_write_tiles( const std::string& name, - std::vector& tile_batches, + IndexedList& tile_batches, shared_ptr frag_meta, DenseTiler* dense_tiler, uint64_t thread_num); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index b2393373ce79..147f6c89349c 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -380,7 +380,10 @@ Status UnorderedWriter::prepare_tiles( for (const auto& it : buffers_) { const auto& name = it.first; if (written_buffers_.count(name) == 0) { - (*tiles).emplace(name, WriterTileTupleVector()); + (*tiles).emplace( + std::piecewise_construct, + std::forward_as_tuple(name), + std::forward_as_tuple()); } } @@ -389,8 +392,8 @@ Status UnorderedWriter::prepare_tiles( storage_manager_->compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto tiles_it = tiles->begin(); std::advance(tiles_it, i); - const auto& name = tiles_it->first; - RETURN_CANCEL_OR_ERROR(prepare_tiles(name, &((*tiles)[name]))); + RETURN_CANCEL_OR_ERROR( + prepare_tiles(tiles_it->first, &(tiles_it->second))); return Status::Ok(); }); @@ -427,14 +430,14 @@ Status UnorderedWriter::prepare_tiles_fixed( // Initialize tiles tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { - tiles->emplace_back(WriterTileTuple( + tiles->emplace_back( array_schema_, cell_num_per_tile, false, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } // Write all cells one by one @@ -504,14 +507,14 @@ Status UnorderedWriter::prepare_tiles_var( // Initialize tiles tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { - tiles->emplace_back(WriterTileTuple( + tiles->emplace_back( array_schema_, cell_num_per_tile, true, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } // Write all cells one by one diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index b54735a06a98..262a71331938 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -710,7 +710,7 @@ Status WriterBase::compute_tiles_metadata( auto tiles_it = tiles.begin(); std::advance(tiles_it, i); const auto& attr = tiles_it->first; - auto& attr_tiles = tiles[attr]; + auto& attr_tiles = tiles.at(attr); const auto type = array_schema_.type(attr); const auto is_dim = array_schema_.is_dim(attr); const auto var_size = array_schema_.var_size(attr); @@ -951,14 +951,14 @@ Status WriterBase::init_tiles( coords_info_.has_coords_ ? capacity : domain.cell_num_per_tile(); tiles->reserve(tile_num); for (uint64_t i = 0; i < tile_num; i++) { - tiles->emplace_back(WriterTileTuple( + tiles->emplace_back( array_schema_, cell_num_per_tile, var_size, nullable, cell_size, type, - query_memory_tracker_)); + query_memory_tracker_); } return Status::Ok(); diff --git a/tiledb/sm/query/writers/writer_base.h b/tiledb/sm/query/writers/writer_base.h index 8438ca75fd75..99be55ce2ca8 100644 --- a/tiledb/sm/query/writers/writer_base.h +++ b/tiledb/sm/query/writers/writer_base.h @@ -36,6 +36,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/indexed_list.h" #include "tiledb/common/status.h" #include "tiledb/sm/fragment/written_fragment_info.h" #include "tiledb/sm/query/iquery_strategy.h" @@ -57,7 +58,7 @@ class DomainBuffersView; class FragmentMetadata; class TileMetadataGenerator; -using WriterTileTupleVector = std::vector; +using WriterTileTupleVector = IndexedList; /** Processes write queries. */ class WriterBase : public StrategyBase, public IQueryStrategy { @@ -489,7 +490,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { template Status prepare_filter_and_write_tiles( const std::string& name, - std::vector& tile_batches, + IndexedList& tile_batches, tdb_shared_ptr frag_meta, DenseTiler* dense_tiler, uint64_t thread_num); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 984a7720a627..1a0cc587c89a 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1267,14 +1267,14 @@ Status StorageManagerCanonical::store_group_detail( SizeComputationSerializer size_computation_serializer; group->serialize(members, size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); group->serialize(members, serializer); - stats()->add_counter("write_group_size", tile.size()); + stats()->add_counter("write_group_size", tile->size()); // Check if the array schema directory exists // If not create it, this is caused by a pre-v10 array @@ -1298,13 +1298,13 @@ Status StorageManagerCanonical::store_array_schema( SizeComputationSerializer size_computation_serializer; array_schema->serialize(size_computation_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); array_schema->serialize(serializer); - stats()->add_counter("write_array_schema_size", tile.size()); + stats()->add_counter("write_array_schema_size", tile->size()); // Delete file if it exists already bool exists; @@ -1348,10 +1348,10 @@ Status StorageManagerCanonical::store_array_schema( SizeComputationSerializer enumeration_size_serializer; enmr->serialize(enumeration_size_serializer); - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( enumeration_size_serializer.size(), resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); enmr->serialize(serializer); auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); @@ -1378,10 +1378,10 @@ Status StorageManagerCanonical::store_metadata( if (0 == size_computation_serializer.size()) { return Status::Ok(); } - WriterTile tile{WriterTile::from_generic( + auto tile{WriterTile::from_generic( size_computation_serializer.size(), resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile.data(), tile.size()); + Serializer serializer(tile->data(), tile->size()); metadata->serialize(serializer); stats()->add_counter("write_meta_size", size_computation_serializer.size()); diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index c2edc076f40f..976f333291c5 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -178,24 +178,26 @@ GenericTileIO::GenericTileHeader GenericTileIO::read_generic_tile_header( void GenericTileIO::store_data( ContextResources& resources, const URI& uri, - WriterTile& tile, + shared_ptr tile, const EncryptionKey& encryption_key) { GenericTileIO tile_io(resources, uri); uint64_t nbytes = 0; - tile_io.write_generic(&tile, encryption_key, &nbytes); + tile_io.write_generic(tile, encryption_key, &nbytes); throw_if_not_ok(resources.vfs().close_file(uri)); } void GenericTileIO::write_generic( - WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes) { + shared_ptr tile, + const EncryptionKey& encryption_key, + uint64_t* nbytes) { // Create a header GenericTileHeader header; - init_generic_tile_header(tile, &header, encryption_key); + init_generic_tile_header(tile.get(), &header, encryption_key); // Filter tile assert(!tile->filtered()); throw_if_not_ok(header.filters.run_forward( - &resources_.stats(), tile, nullptr, &resources_.compute_tp())); + &resources_.stats(), tile.get(), nullptr, &resources_.compute_tp())); header.persisted_size = tile->filtered_buffer().size(); assert(tile->filtered()); diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index a177f875ab0a..ad75af43f81e 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -172,7 +172,7 @@ class GenericTileIO { static void store_data( ContextResources& resources, const URI& uri, - WriterTile& tile, + shared_ptr tile, const EncryptionKey& encryption_key); /** @@ -186,7 +186,9 @@ class GenericTileIO { * @param nbytes The total number of bytes written to the file. */ void write_generic( - WriterTile* tile, const EncryptionKey& encryption_key, uint64_t* nbytes); + shared_ptr tile, + const EncryptionKey& encryption_key, + uint64_t* nbytes); /** * Serialize a generic tile header. diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index 1a65ddf0690d..b678cdfdbbbb 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -72,14 +72,15 @@ shared_ptr Tile::from_generic( memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)); } -WriterTile WriterTile::from_generic( +shared_ptr WriterTile::from_generic( storage_size_t tile_size, shared_ptr memory_tracker) { - return { + return make_shared( + HERE(), 0, constants::generic_tile_datatype, constants::generic_tile_cell_size, tile_size, - memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)}; + memory_tracker->get_resource(MemoryType::GENERIC_TILE_IO)); } uint32_t WriterTile::compute_chunk_size( @@ -127,22 +128,6 @@ TileBase::TileBase( } } -TileBase::TileBase(TileBase&& tile) - : resource_(std::move(tile.resource_)) - , data_(std::move(tile.data_)) - , size_(std::move(tile.size_)) - , cell_size_(std::move(tile.cell_size_)) - , format_version_(std::move(tile.format_version_)) - , type_(std::move(tile.type_)) { -} - -TileBase& TileBase::operator=(TileBase&& tile) { - // Swap with the argument - swap(tile); - - return *this; -} - Tile::Tile( const format_version_t format_version, const Datatype type, @@ -203,30 +188,10 @@ WriterTile::WriterTile( , filtered_buffer_(0) { } -WriterTile::WriterTile(WriterTile&& tile) - : TileBase(std::move(tile)) - , filtered_buffer_(std::move(tile.filtered_buffer_)) { -} - -WriterTile& WriterTile::operator=(WriterTile&& tile) { - // Swap with the argument - swap(tile); - - return *this; -} - /* ****************************** */ /* API */ /* ****************************** */ -void TileBase::swap(TileBase& tile) { - std::swap(size_, tile.size_); - std::swap(data_, tile.data_); - std::swap(cell_size_, tile.cell_size_); - std::swap(format_version_, tile.format_version_); - std::swap(type_, tile.type_); -} - void TileBase::read( void* const buffer, const uint64_t offset, const uint64_t nbytes) const { if (nbytes > size_ - offset) { @@ -285,13 +250,6 @@ uint64_t Tile::load_offsets_chunk_data(ChunkData& chunk_data) { return load_chunk_data(chunk_data, s - 8); } -void Tile::swap(Tile& tile) { - TileBase::swap(tile); - std::swap(filtered_data_, tile.filtered_data_); - std::swap(filtered_size_, tile.filtered_size_); - std::swap(zipped_coords_dim_num_, tile.zipped_coords_dim_num_); -} - void WriterTile::clear_data() { data_ = nullptr; size_ = 0; @@ -318,11 +276,6 @@ void WriterTile::write_var(const void* data, uint64_t offset, uint64_t nbytes) { write(data, offset, nbytes); } -void WriterTile::swap(WriterTile& tile) { - TileBase::swap(tile); - std::swap(filtered_buffer_, tile.filtered_buffer_); -} - /* ********************************* */ /* PRIVATE FUNCTIONS */ /* ********************************* */ diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index 55804df42856..cfd1bea986c2 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -70,13 +70,8 @@ class TileBase { const uint64_t size, tdb::pmr::memory_resource* resource); - /** Move constructor. */ - TileBase(TileBase&& tile); - - /** Move-assign operator. */ - TileBase& operator=(TileBase&& tile); - DISABLE_COPY_AND_COPY_ASSIGN(TileBase); + DISABLE_MOVE_AND_MOVE_ASSIGN(TileBase); /* ********************************* */ /* API */ @@ -144,9 +139,6 @@ class TileBase { data_as()[size_ / cell_size_ - 1] = var_tile.size(); } - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(TileBase& tile); - protected: /* ********************************* */ /* PROTECTED ATTRIBUTES */ @@ -307,9 +299,6 @@ class Tile : public TileBase { */ uint64_t load_offsets_chunk_data(ChunkData& chunk_data); - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(Tile& tile); - private: /* ********************************* */ /* PRIVATE FUNCTIONS */ @@ -379,7 +368,7 @@ class WriterTile : public TileBase { * @param tile_size to be provided to init_unfiltered call * @param memory_tracker The memory tracker to use. */ - static WriterTile from_generic( + static shared_ptr from_generic( storage_size_t tile_size, shared_ptr memory_tracker); /** @@ -435,13 +424,8 @@ class WriterTile : public TileBase { const uint64_t size, tdb::pmr::memory_resource* resource); - /** Move constructor. */ - WriterTile(WriterTile&& tile); - - /** Move-assign operator. */ - WriterTile& operator=(WriterTile&& tile); - DISABLE_COPY_AND_COPY_ASSIGN(WriterTile); + DISABLE_MOVE_AND_MOVE_ASSIGN(WriterTile); /* ********************************* */ /* API */ @@ -488,9 +472,6 @@ class WriterTile : public TileBase { size_ = size; } - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(WriterTile& tile); - private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/tile/writer_tile_tuple.cc b/tiledb/sm/tile/writer_tile_tuple.cc index 700705f7ef87..9ce07d20f95c 100644 --- a/tiledb/sm/tile/writer_tile_tuple.cc +++ b/tiledb/sm/tile/writer_tile_tuple.cc @@ -52,63 +52,35 @@ WriterTileTuple::WriterTileTuple( shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , fixed_tile_( - var_size ? WriterTile( - array_schema.write_version(), - constants::cell_var_offset_type, - constants::cell_var_offset_size, - cell_num_per_tile * constants::cell_var_offset_size, - memory_tracker_) : - WriterTile( - array_schema.write_version(), - type, - cell_size, - cell_num_per_tile * cell_size, - memory_tracker_)) - , var_tile_( - var_size ? std::optional(WriterTile( - array_schema.write_version(), - type, - datatype_size(type), - cell_num_per_tile * constants::cell_var_offset_size, - memory_tracker_)) : - std::nullopt) - , validity_tile_( - nullable ? std::optional(WriterTile( - array_schema.write_version(), - constants::cell_validity_type, - constants::cell_validity_size, - cell_num_per_tile * constants::cell_validity_size, - memory_tracker_)) : - std::nullopt) + array_schema.write_version(), + var_size ? constants::cell_var_offset_type : type, + var_size ? constants::cell_var_offset_size : cell_size, + var_size ? cell_num_per_tile * constants::cell_var_offset_size : + cell_num_per_tile * cell_size, + memory_tracker_) , cell_size_(cell_size) , var_pre_filtered_size_(0) , min_size_(0) , max_size_(0) , null_count_(0) , cell_num_(cell_num_per_tile) { -} - -WriterTileTuple::WriterTileTuple(WriterTileTuple&& tile) - : memory_tracker_(std::move(tile.memory_tracker_)) - , fixed_tile_(std::move(tile.fixed_tile_)) - , var_tile_(std::move(tile.var_tile_)) - , validity_tile_(std::move(tile.validity_tile_)) - , cell_size_(std::move(tile.cell_size_)) - , var_pre_filtered_size_(std::move(tile.var_pre_filtered_size_)) - , min_(std::move(tile.min_)) - , min_size_(std::move(tile.min_size_)) - , max_(std::move(tile.max_)) - , max_size_(std::move(tile.max_size_)) - , sum_(std::move(tile.sum_)) - , null_count_(std::move(tile.null_count_)) - , cell_num_(std::move(tile.cell_num_)) { -} - -WriterTileTuple& WriterTileTuple::operator=(WriterTileTuple&& tile) { - // Swap with the argument - swap(tile); + if (var_size) { + var_tile_.emplace( + array_schema.write_version(), + type, + datatype_size(type), + cell_num_per_tile * constants::cell_var_offset_size, + memory_tracker_); + } - return *this; + if (nullable) { + validity_tile_.emplace( + array_schema.write_version(), + constants::cell_validity_type, + constants::cell_validity_size, + cell_num_per_tile * constants::cell_validity_size, + memory_tracker_); + } } /* ****************************** */ @@ -142,21 +114,5 @@ void WriterTileTuple::set_metadata( } } -void WriterTileTuple::swap(WriterTileTuple& tile) { - std::swap(memory_tracker_, tile.memory_tracker_); - std::swap(fixed_tile_, tile.fixed_tile_); - std::swap(var_tile_, tile.var_tile_); - std::swap(validity_tile_, tile.validity_tile_); - std::swap(cell_size_, tile.cell_size_); - std::swap(var_pre_filtered_size_, tile.var_pre_filtered_size_); - std::swap(min_, tile.min_); - std::swap(min_size_, tile.min_size_); - std::swap(max_, tile.max_); - std::swap(max_size_, tile.max_size_); - std::swap(sum_, tile.sum_); - std::swap(null_count_, tile.null_count_); - std::swap(cell_num_, tile.cell_num_); -} - } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/tile/writer_tile_tuple.h b/tiledb/sm/tile/writer_tile_tuple.h index 244ac932114c..850a17b9ca90 100644 --- a/tiledb/sm/tile/writer_tile_tuple.h +++ b/tiledb/sm/tile/writer_tile_tuple.h @@ -59,13 +59,8 @@ class WriterTileTuple { const Datatype type, shared_ptr memory_tracker); - /** Move constructor. */ - WriterTileTuple(WriterTileTuple&& tile); - - /** Move-assign operator. */ - WriterTileTuple& operator=(WriterTileTuple&& tile); - DISABLE_COPY_AND_COPY_ASSIGN(WriterTileTuple); + DISABLE_MOVE_AND_MOVE_ASSIGN(WriterTileTuple); /* ********************************* */ /* API */ @@ -216,9 +211,6 @@ class WriterTileTuple { return cell_num_; } - /** Swaps the contents (all field values) of this tile with the given tile. */ - void swap(WriterTileTuple& tile); - private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ From 4486f54eec1af1cf18e7d61350bcd878d3333a53 Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Thu, 7 Mar 2024 17:00:07 -0500 Subject: [PATCH 232/456] SC 7826: Convert Dimension function pointer members to subclass methods (#4774) The `Dimension` class has 20+ function pointer members which are set to dynamically dispatch a template instantiation of a function based on the data type of the dimension. The `Dimension` constructor sets each of these function pointers, switching on the type for each of them. But really the same switch case applies to all of them. It is a cleaner encapsulation of functionality to nest the implementations within a subclass, which also adds a small benefit of reducing the object size. Hence this pull request introduces `DimensionDispatch`, which takes the function pointers we previously had and makes them pure virtual methods of a base class instead. `template DimensionDispatchBase` implements most of these virtual methods, and defers any exceptions to the pattern to `DimensionFixedSize` and `DimensionVarSized`. And the `Dimension` constructor chooses an implementation with a single function call `set_dimension_dispatch` instead of the previous logic to set each function pointer individually. Closes sc-7826 --- TYPE: IMPROVEMENT DESC: Convert dimension function pointer members to subclass methods --- tiledb/sm/array_schema/dimension.cc | 441 +++++++++++----------------- tiledb/sm/array_schema/dimension.h | 263 ++++------------- 2 files changed, 237 insertions(+), 467 deletions(-) diff --git a/tiledb/sm/array_schema/dimension.cc b/tiledb/sm/array_schema/dimension.cc index 2db81166179b..61fc54fcf984 100644 --- a/tiledb/sm/array_schema/dimension.cc +++ b/tiledb/sm/array_schema/dimension.cc @@ -56,6 +56,143 @@ class DimensionException : public StatusException { } }; +/* ************************ */ +/* DYNAMIC DISPATCH */ +/* ************************ */ + +/** + * Dispatches Dimension behavior based on the physical type + * of the Dimension + * + * Subclasses DimensionDispatchFixedSize and DimensionDispatchVarSize + * handle the few scenarios where the dispatch is based on + * something beyond just the type (namely, function arguments). + */ +template +class DimensionDispatchTyped : public Dimension::DimensionDispatch { + public: + using DimensionDispatch::DimensionDispatch; + + protected: + void ceil_to_tile( + const Range& r, uint64_t tile_num, ByteVecValue* v) const override { + return Dimension::ceil_to_tile(&base, r, tile_num, v); + } + bool check_range(const Range& range, std::string* error) const override { + return Dimension::check_range(&base, range, error); + } + bool coincides_with_tiles(const Range& r) const override { + return Dimension::coincides_with_tiles(&base, r); + } + uint64_t domain_range(const Range& range) const override { + return Dimension::domain_range(range); + } + void expand_range(const Range& r1, Range* r2) const override { + return Dimension::expand_range(r1, r2); + } + void expand_range_v(const void* v, Range* r) const override { + return Dimension::expand_range_v(v, r); + } + void expand_to_tile(Range* range) const override { + return Dimension::expand_to_tile(&base, range); + } + bool oob(const void* coord, std::string* err_msg) const override { + return Dimension::oob(&base, coord, err_msg); + } + bool covered(const Range& r1, const Range& r2) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::covered(r1, r2); + } + bool overlap(const Range& r1, const Range& r2) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::overlap(r1, r2); + } + double overlap_ratio(const Range& r1, const Range& r2) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::overlap_ratio(r1, r2); + } + void relevant_ranges( + const NDRange& ranges, + const Range& mbr, + tdb::pmr::vector& relevant_ranges) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::relevant_ranges(ranges, mbr, relevant_ranges); + } + std::vector covered_vec( + const NDRange& ranges, + const Range& mbr, + const tdb::pmr::vector& relevant_ranges) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::covered_vec(ranges, mbr, relevant_ranges); + } + void split_range(const Range& r, const ByteVecValue& v, Range* r1, Range* r2) + const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::split_range(r, v, r1, r2); + } + void splitting_value( + const Range& r, ByteVecValue* v, bool* unsplittable) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::splitting_value(r, v, unsplittable); + } + uint64_t tile_num(const Range& range) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::tile_num(&base, range); + } + uint64_t map_to_uint64( + const void* coord, + uint64_t coord_size, + int bits, + uint64_t max_bucket_val) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::map_to_uint64_2( + &base, coord, coord_size, bits, max_bucket_val); + } + ByteVecValue map_from_uint64( + uint64_t value, int bits, uint64_t max_bucket_val) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::map_from_uint64(&base, value, bits, max_bucket_val); + } + bool smaller_than( + const ByteVecValue& value, const Range& range) const override { + static_assert(tiledb::type::TileDBFundamental); + return Dimension::smaller_than(&base, value, range); + } +}; + +template +class DimensionFixedSize : public DimensionDispatchTyped { + public: + using DimensionDispatchTyped::DimensionDispatchTyped; + + protected: + Range compute_mbr(const WriterTile& tile) const override { + return Dimension::compute_mbr(tile); + } + + Range compute_mbr_var(const WriterTile&, const WriterTile&) const override { + throw std::logic_error( + "Fixed-length dimension has no offset tile, function " + + std::string(__func__) + " cannot be called"); + } +}; + +class DimensionVarSize : public DimensionDispatchTyped { + public: + using DimensionDispatchTyped::DimensionDispatchTyped; + + protected: + Range compute_mbr(const WriterTile&) const override { + throw std::logic_error( + "Variable-length dimension requires an offset tile, function " + + std::string(__func__) + " cannot be called"); + } + + /** note: definition at the bottom due to template specialization */ + Range compute_mbr_var( + const WriterTile& tile_off, const WriterTile& tile_val) const override; +}; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -69,25 +206,7 @@ Dimension::Dimension( , type_(type) { ensure_datatype_is_supported(type_); cell_val_num_ = (datatype_is_string(type)) ? constants::var_num : 1; - set_ceil_to_tile_func(); - set_coincides_with_tiles_func(); - set_compute_mbr_func(); - set_domain_range_func(); - set_expand_range_func(); - set_expand_range_v_func(); - set_expand_to_tile_func(); - set_oob_func(); - set_covered_func(); - set_overlap_func(); - set_overlap_ratio_func(); - set_relevant_ranges_func(); - set_covered_vec_func(); - set_split_range_func(); - set_splitting_value_func(); - set_tile_num_func(); - set_map_to_uint64_2_func(); - set_map_from_uint64_func(); - set_smaller_than_func(); + set_dimension_dispatch(); } Dimension::Dimension( @@ -106,25 +225,7 @@ Dimension::Dimension( , tile_extent_(tile_extent) , type_(type) { ensure_datatype_is_supported(type_); - set_ceil_to_tile_func(); - set_coincides_with_tiles_func(); - set_compute_mbr_func(); - set_domain_range_func(); - set_expand_range_func(); - set_expand_range_v_func(); - set_expand_to_tile_func(); - set_oob_func(); - set_covered_func(); - set_overlap_func(); - set_overlap_ratio_func(); - set_relevant_ranges_func(); - set_covered_vec_func(); - set_split_range_func(); - set_splitting_value_func(); - set_tile_num_func(); - set_map_to_uint64_2_func(); - set_map_from_uint64_func(); - set_smaller_than_func(); + set_dimension_dispatch(); } /* ********************************* */ @@ -276,8 +377,7 @@ void Dimension::ceil_to_tile( void Dimension::ceil_to_tile( const Range& r, uint64_t tile_num, ByteVecValue* v) const { - assert(ceil_to_tile_func_ != nullptr); - ceil_to_tile_func_(this, r, tile_num, v); + dispatch_->ceil_to_tile(r, tile_num, v); } template @@ -296,8 +396,7 @@ bool Dimension::coincides_with_tiles(const Dimension* dim, const Range& r) { } bool Dimension::coincides_with_tiles(const Range& r) const { - assert(coincides_with_tiles_func_ != nullptr); - return coincides_with_tiles_func_(this, r); + return dispatch_->coincides_with_tiles(r); } template @@ -321,8 +420,7 @@ Range Dimension::compute_mbr(const WriterTile& tile) { } Range Dimension::compute_mbr(const WriterTile& tile) const { - assert(compute_mbr_func_ != nullptr); - return compute_mbr_func_(tile); + return dispatch_->compute_mbr(tile); } template <> @@ -354,8 +452,7 @@ Range Dimension::compute_mbr_var( Range Dimension::compute_mbr_var( const WriterTile& tile_off, const WriterTile& tile_val) const { - assert(compute_mbr_var_func_ != nullptr); - return compute_mbr_var_func_(tile_off, tile_val); + return dispatch_->compute_mbr_var(tile_off, tile_val); } template @@ -376,8 +473,7 @@ uint64_t Dimension::domain_range(const Range& range) { } uint64_t Dimension::domain_range(const Range& range) const { - assert(domain_range_func_ != nullptr); - return domain_range_func_(range); + return dispatch_->domain_range(range); } template @@ -392,8 +488,7 @@ void Dimension::expand_range_v(const void* v, Range* r) { } void Dimension::expand_range_v(const void* v, Range* r) const { - assert(expand_range_v_func_ != nullptr); - expand_range_v_func_(v, r); + return dispatch_->expand_range_v(v, r); } void Dimension::expand_range_var_v(const char* v, uint64_t v_size, Range* r) { @@ -419,8 +514,7 @@ void Dimension::expand_range(const Range& r1, Range* r2) { } void Dimension::expand_range(const Range& r1, Range* r2) const { - assert(expand_range_func_ != nullptr); - expand_range_func_(r1, r2); + return dispatch_->expand_range(r1, r2); } void Dimension::expand_range_var(const Range& r1, Range* r2) const { @@ -461,8 +555,7 @@ void Dimension::expand_to_tile(const Dimension* dim, Range* range) { } void Dimension::expand_to_tile(Range* range) const { - assert(expand_to_tile_func_ != nullptr); - expand_to_tile_func_(this, range); + return dispatch_->expand_to_tile(range); } template @@ -492,9 +585,8 @@ Status Dimension::oob(const void* coord) const { if (datatype_is_string(type_)) return Status::Ok(); - assert(oob_func_ != nullptr); std::string err_msg; - auto ret = oob_func_(this, coord, &err_msg); + auto ret = dispatch_->oob(coord, &err_msg); if (ret) return Status_DimensionError(err_msg); return Status::Ok(); @@ -524,8 +616,7 @@ bool Dimension::covered(const Range& r1, const Range& r2) { } bool Dimension::covered(const Range& r1, const Range& r2) const { - assert(covered_func_ != nullptr); - return covered_func_(r1, r2); + return dispatch_->covered(r1, r2); } template <> @@ -552,8 +643,7 @@ bool Dimension::overlap(const Range& r1, const Range& r2) { } bool Dimension::overlap(const Range& r1, const Range& r2) const { - assert(overlap_func_ != nullptr); - return overlap_func_(r1, r2); + return dispatch_->overlap(r1, r2); } template <> @@ -695,16 +785,14 @@ double Dimension::overlap_ratio(const Range& r1, const Range& r2) { } double Dimension::overlap_ratio(const Range& r1, const Range& r2) const { - assert(overlap_ratio_func_ != nullptr); - return overlap_ratio_func_(r1, r2); + return dispatch_->overlap_ratio(r1, r2); } void Dimension::relevant_ranges( const NDRange& ranges, const Range& mbr, tdb::pmr::vector& relevant_ranges) const { - assert(relevant_ranges_func_ != nullptr); - return relevant_ranges_func_(ranges, mbr, relevant_ranges); + return dispatch_->relevant_ranges(ranges, mbr, relevant_ranges); } template <> @@ -805,8 +893,7 @@ std::vector Dimension::covered_vec( const NDRange& ranges, const Range& mbr, const tdb::pmr::vector& relevant_ranges) const { - assert(covered_vec_func_ != nullptr); - return covered_vec_func_(ranges, mbr, relevant_ranges); + return dispatch_->covered_vec(ranges, mbr, relevant_ranges); } template <> @@ -928,8 +1015,7 @@ void Dimension::split_range( void Dimension::split_range( const Range& r, const ByteVecValue& v, Range* r1, Range* r2) const { - assert(split_range_func_ != nullptr); - split_range_func_(r, v, r1, r2); + dispatch_->split_range(r, v, r1, r2); } template <> @@ -1061,8 +1147,7 @@ void Dimension::splitting_value( void Dimension::splitting_value( const Range& r, ByteVecValue* v, bool* unsplittable) const { - assert(splitting_value_func_ != nullptr); - splitting_value_func_(r, v, unsplittable); + dispatch_->splitting_value(r, v, unsplittable); } template <> @@ -1089,8 +1174,7 @@ uint64_t Dimension::tile_num(const Dimension* dim, const Range& range) { } uint64_t Dimension::tile_num(const Range& range) const { - assert(tile_num_func_ != nullptr); - return tile_num_func_(this, range); + return dispatch_->tile_num(range); } uint64_t Dimension::map_to_uint64( @@ -1098,8 +1182,7 @@ uint64_t Dimension::map_to_uint64( uint64_t coord_size, int bits, uint64_t max_bucket_val) const { - assert(map_to_uint64_2_func_ != nullptr); - return map_to_uint64_2_func_(this, coord, coord_size, bits, max_bucket_val); + return dispatch_->map_to_uint64(coord, coord_size, bits, max_bucket_val); } template @@ -1148,8 +1231,7 @@ uint64_t Dimension::map_to_uint64_2( ByteVecValue Dimension::map_from_uint64( uint64_t value, int bits, uint64_t max_bucket_val) const { - assert(map_from_uint64_func_ != nullptr); - return map_from_uint64_func_(this, value, bits, max_bucket_val); + return dispatch_->map_from_uint64(value, bits, max_bucket_val); } template @@ -1210,8 +1292,7 @@ ByteVecValue Dimension::map_from_uint64( bool Dimension::smaller_than( const ByteVecValue& value, const Range& range) const { - assert(smaller_than_func_ != nullptr); - return smaller_than_func_(this, value, range); + return dispatch_->smaller_than(value, range); } template @@ -1577,200 +1658,24 @@ std::string Dimension::tile_extent_str() const { return apply_with_type(g, type_); } -void Dimension::set_domain_range_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - domain_range_func_ = domain_range; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_ceil_to_tile_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - ceil_to_tile_func_ = ceil_to_tile; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_coincides_with_tiles_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - coincides_with_tiles_func_ = coincides_with_tiles; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_compute_mbr_func() { - if (!var_size()) { // Fixed-sized - compute_mbr_var_func_ = nullptr; - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBNumeric) { - compute_mbr_func_ = compute_mbr; - } - }; - apply_with_type(g, type_); - } else { // Var-sized +void Dimension::set_dimension_dispatch() { + if (var_size()) { + dispatch_ = + tdb_unique_ptr(tdb_new(DimensionVarSize, *this)); assert(type_ == Datatype::STRING_ASCII); - compute_mbr_func_ = nullptr; - compute_mbr_var_func_ = compute_mbr_var; + } else { + // Fixed-sized + auto set = [&](auto T) { + this->dispatch_ = tdb_unique_ptr( + tdb_new(DimensionFixedSize, *this)); + }; + apply_with_type(set, type_); } } -void Dimension::set_expand_range_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - expand_range_func_ = expand_range; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_expand_range_v_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - expand_range_v_func_ = expand_range_v; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_expand_to_tile_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - expand_to_tile_func_ = expand_to_tile; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_oob_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - oob_func_ = oob; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_covered_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - covered_func_ = covered; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_overlap_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - overlap_func_ = overlap; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_overlap_ratio_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - overlap_ratio_func_ = overlap_ratio; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_relevant_ranges_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - relevant_ranges_func_ = relevant_ranges; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_covered_vec_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - covered_vec_func_ = covered_vec; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_split_range_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - split_range_func_ = split_range; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_splitting_value_func() { - auto g = [&](auto T) { - if constexpr (std::is_same_v) { - assert(var_size()); - } - if constexpr (tiledb::type::TileDBFundamental) { - splitting_value_func_ = splitting_value; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_tile_num_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - tile_num_func_ = tile_num; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_map_to_uint64_2_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - map_to_uint64_2_func_ = map_to_uint64_2; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_map_from_uint64_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - map_from_uint64_func_ = map_from_uint64; - } - }; - apply_with_type(g, type_); -} - -void Dimension::set_smaller_than_func() { - auto g = [&](auto T) { - if constexpr (tiledb::type::TileDBFundamental) { - smaller_than_func_ = smaller_than; - } - }; - apply_with_type(g, type_); +Range DimensionVarSize::compute_mbr_var( + const WriterTile& tile_off, const WriterTile& tile_val) const { + return Dimension::compute_mbr_var(tile_off, tile_val); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index c68943c4c4df..45665f6e3ea7 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -68,6 +68,8 @@ class FilterPipeline; enum class Compressor : uint8_t; enum class Datatype : uint8_t; +class DimensionDispatch; + /** Manipulates a TileDB dimension. * * Note: as laid out in the Storage Format, @@ -738,6 +740,58 @@ class Dimension { return cell_val_num_ == constants::var_num; } + class DimensionDispatch { + public: + DimensionDispatch(const Dimension& base) + : base(base) { + } + + virtual ~DimensionDispatch() { + } + + virtual void ceil_to_tile( + const Range& r, uint64_t tile_num, ByteVecValue* v) const = 0; + virtual bool check_range(const Range& range, std::string* error) const = 0; + virtual bool coincides_with_tiles(const Range& r) const = 0; + virtual Range compute_mbr(const WriterTile&) const = 0; + virtual Range compute_mbr_var( + const WriterTile&, const WriterTile&) const = 0; + virtual uint64_t domain_range(const Range& range) const = 0; + virtual void expand_range(const Range& r1, Range* r2) const = 0; + virtual void expand_range_v(const void* v, Range* r) const = 0; + virtual void expand_to_tile(Range* range) const = 0; + virtual bool oob(const void* coord, std::string* err_msg) const = 0; + virtual bool covered(const Range& r1, const Range& r2) const = 0; + virtual bool overlap(const Range& r1, const Range& r2) const = 0; + virtual double overlap_ratio(const Range& r1, const Range& r2) const = 0; + virtual void relevant_ranges( + const NDRange& ranges, + const Range& mbr, + tdb::pmr::vector& relevant_ranges) const = 0; + virtual std::vector covered_vec( + const NDRange& ranges, + const Range& mbr, + const tdb::pmr::vector& relevant_ranges) const = 0; + virtual void split_range( + const Range& r, const ByteVecValue& v, Range* r1, Range* r2) const = 0; + virtual void splitting_value( + const Range& r, ByteVecValue* v, bool* unsplittable) const = 0; + virtual uint64_t tile_num(const Range& range) const = 0; + virtual uint64_t map_to_uint64( + const void* coord, + uint64_t coord_size, + int bits, + uint64_t max_bucket_val) const = 0; + virtual ByteVecValue map_from_uint64( + uint64_t value, int bits, uint64_t max_bucket_val) const = 0; + virtual bool smaller_than( + const ByteVecValue& value, const Range& range) const = 0; + + protected: + const Dimension& base; + }; + friend class DimensionDispatch; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -746,6 +800,11 @@ class Dimension { /** The memory tracker for the dimension. */ shared_ptr memory_tracker_; + /** + * Handles dynamic dispatch for functions which depend on Dimension type + */ + tdb_unique_ptr dispatch_; + /** The number of values per coordinate. */ unsigned cell_val_num_; @@ -764,146 +823,6 @@ class Dimension { /** The dimension type. */ Datatype type_; - /** - * Stores the appropriate templated ceil_to_tile() function based on the - * dimension datatype. - */ - std::function - ceil_to_tile_func_; - - /** - * Stores the appropriate templated check_range() function based on the - * dimension datatype. - */ - std::function - check_range_func_; - - /** - * Stores the appropriate templated coincides_with_tiles() function based on - * the dimension datatype. - */ - std::function - coincides_with_tiles_func_; - - /** - * Stores the appropriate templated compute_mbr() function based on the - * dimension datatype. - */ - std::function compute_mbr_func_; - - /** - * Stores the appropriate templated compute_mbr_var() function based on the - * dimension datatype. - */ - std::function - compute_mbr_var_func_; - - /** - * Stores the appropriate templated domain_range() function based on the - * dimension datatype. - */ - std::function domain_range_func_; - - /** - * Stores the appropriate templated expand_range() function based on the - * dimension datatype. - */ - std::function expand_range_v_func_; - - /** - * Stores the appropriate templated expand_range() function based on the - * dimension datatype. - */ - std::function expand_range_func_; - - /** - * Stores the appropriate templated expand_to_tile() function based on the - * dimension datatype. - */ - std::function expand_to_tile_func_; - - /** - * Stores the appropriate templated oob() function based on the - * dimension datatype. - */ - std::function - oob_func_; - - /** - * Stores the appropriate templated covered() function based on the - * dimension datatype. - */ - std::function covered_func_; - - /** - * Stores the appropriate templated overlap() function based on the - * dimension datatype. - */ - std::function overlap_func_; - - /** - * Stores the appropriate templated overlap_ratio() function based on the - * dimension datatype. - */ - std::function overlap_ratio_func_; - - /** - * Stores the appropriate templated relevant_ranges() function based - * on the dimension datatype. - */ - std::function&)> - relevant_ranges_func_; - - /** - * Stores the appropriate templated covered_vec() function based on the - * dimension datatype. - */ - std::function( - const NDRange&, const Range&, const tdb::pmr::vector&)> - covered_vec_func_; - - /** - * Stores the appropriate templated split_range() function based on the - * dimension datatype. - */ - std::function - split_range_func_; - - /** - * Stores the appropriate templated splitting_value() function based on the - * dimension datatype. - */ - std::function - splitting_value_func_; - - /** - * Stores the appropriate templated tile_num() function based on the - * dimension datatype. - */ - std::function tile_num_func_; - - /** - * Stores the appropriate templated map_to_uint64_2() function based on - * the dimension datatype. - */ - std::function - map_to_uint64_2_func_; - - /** - * Stores the appropriate templated map_from_uint64() function based on - * the dimension datatype. - */ - std::function - map_from_uint64_func_; - - /** - * Stores the appropriate templated smaller_than() function based on - * the dimension datatype. - */ - std::function - smaller_than_func_; - /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ @@ -995,65 +914,11 @@ class Dimension { /** Returns the tile extent in string format. */ std::string tile_extent_str() const; - /** Sets the templated ceil_to_tile() function. */ - void set_ceil_to_tile_func(); - - /** Sets the templated check_range() function. */ - void set_check_range_func(); - - /** Sets the templated coincides_with_tiles() function. */ - void set_coincides_with_tiles_func(); - - /** Sets the templated compute_mbr() function. */ - void set_compute_mbr_func(); - - /** Sets the templated domain_range() function. */ - void set_domain_range_func(); - - /** Sets the templated expand_range() function. */ - void set_expand_range_func(); - - /** Sets the templated expand_range_v() function. */ - void set_expand_range_v_func(); - - /** Sets the templated expand_to_tile() function. */ - void set_expand_to_tile_func(); - - /** Sets the templated oob() function. */ - void set_oob_func(); - - /** Sets the templated covered() function. */ - void set_covered_func(); - - /** Sets the templated overlap() function. */ - void set_overlap_func(); - - /** Sets the templated overlap_ratio() function. */ - void set_overlap_ratio_func(); - - /** Sets the templated relevant_ranges() function. */ - void set_relevant_ranges_func(); - - /** Sets the templated covered_vec() function. */ - void set_covered_vec_func(); - - /** Sets the templated split_range() function. */ - void set_split_range_func(); - - /** Sets the templated splitting_value() function. */ - void set_splitting_value_func(); - - /** Sets the templated tile_num() function. */ - void set_tile_num_func(); - - /** Sets the templated map_to_uint64_2() function. */ - void set_map_to_uint64_2_func(); - - /** Sets the templated map_from_uint64() function. */ - void set_map_from_uint64_func(); - - /** Sets the templated smaller_than() function. */ - void set_smaller_than_func(); + /** + * Sets the dimension dynamic dispatch implementation. + * Called in the constructor. + */ + void set_dimension_dispatch(); }; } // namespace tiledb::sm From cfe5d972157257e708410eaaf8976e67f2cb9de4 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 8 Mar 2024 16:28:39 -0500 Subject: [PATCH 233/456] Instrumentation for memory measurement: FilteredData and FilteredDataBlock. (#4779) Add `tdb::pmr::list` wrapper as an alias for `std::list`. Instrument classes `FilteredData` and `FilteredDataBlock` for memory measurement. --- TYPE: NO_HISTORY DESC: Instrumentation for memory measurement: FilteredData and FilteredDataBlock. Addition of tdb::pmr::list wrapper. --------- Co-authored-by: Luc Rancourt --- tiledb/common/memory_tracker.cc | 4 + tiledb/common/memory_tracker.h | 2 + tiledb/common/pmr.h | 62 ++++++ tiledb/sm/query/readers/dense_reader.cc | 267 ++++++++++++------------ tiledb/sm/query/readers/filtered_data.h | 106 +++++++--- tiledb/sm/query/readers/reader_base.cc | 21 +- tiledb/sm/query/readers/reader_base.h | 18 +- 7 files changed, 294 insertions(+), 186 deletions(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index ddc8cb870ee1..c7d9071bdc20 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -56,6 +56,10 @@ std::string memory_type_to_str(MemoryType type) { return "EnumerationPaths"; case MemoryType::FOOTER: return "Footer"; + case MemoryType::FILTERED_DATA: + return "FilteredData"; + case MemoryType::FILTERED_DATA_BLOCK: + return "FilteredDataBlock"; case MemoryType::GENERIC_TILE_IO: return "GenericTileIO"; case MemoryType::RTREE: diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index c1413304f8cf..4bda450af2c1 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -104,6 +104,8 @@ enum class MemoryType { ENUMERATION, ENUMERATION_PATHS, FOOTER, + FILTERED_DATA, + FILTERED_DATA_BLOCK, GENERIC_TILE_IO, RTREE, TILE_BITMAP, diff --git a/tiledb/common/pmr.h b/tiledb/common/pmr.h index 6245737c269a..a5d15a6f0bba 100644 --- a/tiledb/common/pmr.h +++ b/tiledb/common/pmr.h @@ -34,6 +34,7 @@ #ifndef TILEDB_COMMON_PMR_H #define TILEDB_COMMON_PMR_H +#include #include #include #include @@ -120,6 +121,67 @@ unique_ptr make_unique(memory_resource* resource, size_t size) { return make_unique(resource, size, alignof(Tp)); } +/* ********************************* */ +/* PMR LIST DECLARATION */ +/* ********************************* */ +template +using pmr_list = std::list>; + +template +class list : public pmr_list { + public: + using value_type = typename pmr_list::value_type; + using allocator_type = typename pmr_list::allocator_type; + using size_type = typename pmr_list::size_type; + using difference_type = typename pmr_list::difference_type; + using reference = typename pmr_list::reference; + using const_reference = typename pmr_list::const_reference; + using pointer = typename pmr_list::pointer; + using const_pointer = typename pmr_list::const_pointer; + using iterator = typename pmr_list::iterator; + using const_iterator = typename pmr_list::const_iterator; + using reverse_iterator = typename pmr_list::reverse_iterator; + using const_reverse_iterator = typename pmr_list::const_reverse_iterator; + + // Delete all default constructors because they don't require an allocator + list() = delete; + list(const list& other) = delete; + list(list&& other) = delete; + + // Delete non-allocator aware copy and move assign. + list& operator=(const list& other) = delete; + list& operator=(list&& other) noexcept = delete; + + explicit list(const allocator_type& alloc) noexcept + : pmr_list(alloc) { + } + + explicit list(size_type count, const Tp& value, const allocator_type& alloc) + : pmr_list(count, value, alloc) { + } + + explicit list(size_type count, const allocator_type& alloc) + : pmr_list(count, alloc) { + } + + template + list(InputIt first, InputIt last, const allocator_type& alloc) + : pmr_list(first, last, alloc) { + } + + list(const list& other, const allocator_type& alloc) + : pmr_list(other, alloc) { + } + + list(list&& other, const allocator_type& alloc) + : pmr_list(other, alloc) { + } + + list(std::initializer_list init, const allocator_type& alloc) + : pmr_list(init, alloc) { + } +}; + /* ********************************* */ /* PMR VECTOR DECLARATION */ /* ********************************* */ diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 688f981c80fd..331de145b9cb 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -54,8 +54,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class DenseReaderStatusException : public StatusException { public: @@ -433,13 +432,13 @@ Status DenseReader::dense_read() { result_space_tiles, tile_subarrays); - std::vector filtered_data; - // Read and unfilter tiles. bool validity_only = null_count_aggregate_only(name); std::vector to_load; to_load.emplace_back(name, validity_only); - filtered_data = std::move(read_attribute_tiles(to_load, result_tiles)); + shared_ptr> filtered_data = + make_shared>( + read_attribute_tiles(to_load, result_tiles)); if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); @@ -448,17 +447,17 @@ Status DenseReader::dense_read() { } } - compute_task = storage_manager_->compute_tp()->execute( - [&, - filtered_data = std::move(filtered_data), - name, - validity_only, - t_start, - t_end, - subarray_start_cell, - subarray_end_cell, - num_range_threads, - result_tiles]() { + compute_task = + storage_manager_->compute_tp()->execute([&, + filtered_data, + name, + validity_only, + t_start, + t_end, + subarray_start_cell, + subarray_end_cell, + num_range_threads, + result_tiles]() { // Unfilter tiles. RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); @@ -911,139 +910,138 @@ Status DenseReader::apply_query_condition( tile_subarrays); // Read and unfilter query condition attributes. - std::vector filtered_data = read_attribute_tiles( - NameToLoad::from_string_vec(qc_names), result_tiles); + shared_ptr> filtered_data = + make_shared>(read_attribute_tiles( + NameToLoad::from_string_vec(qc_names), result_tiles)); if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); } - compute_task = storage_manager_->compute_tp()->execute( - [&, - filtered_data = std::move(filtered_data), - qc_names, - t_start, - t_end, - num_range_threads, - result_tiles]() { - // For easy reference. - const auto& tile_coords = subarray.tile_coords(); - const auto dim_num = array_schema_.dim_num(); - auto stride = array_schema_.domain().stride(layout_); - const auto cell_order = array_schema_.cell_order(); - const auto global_order = layout_ == Layout::GLOBAL_ORDER; - - // Unfilter tiles. - for (auto& name : qc_names) { - RETURN_NOT_OK(unfilter_tiles(name, false, result_tiles)); - } + compute_task = storage_manager_->compute_tp()->execute([&, + filtered_data, + qc_names, + t_start, + t_end, + num_range_threads, + result_tiles]() { + // For easy reference. + const auto& tile_coords = subarray.tile_coords(); + const auto dim_num = array_schema_.dim_num(); + auto stride = array_schema_.domain().stride(layout_); + const auto cell_order = array_schema_.cell_order(); + const auto global_order = layout_ == Layout::GLOBAL_ORDER; + + // Unfilter tiles. + for (auto& name : qc_names) { + RETURN_NOT_OK(unfilter_tiles(name, false, result_tiles)); + } - if (stride == UINT64_MAX) { - stride = 1; - } + if (stride == UINT64_MAX) { + stride = 1; + } - // Process all tiles in parallel. - auto status = parallel_for_2d( - storage_manager_->compute_tp(), - t_start, - t_end, - 0, - num_range_threads, - [&](uint64_t t, uint64_t range_thread_idx) { - // Find out result space tile and tile subarray. - const DimType* tc = (DimType*)&tile_coords[t][0]; - auto& result_space_tile = result_space_tiles.at(tc); - - // Iterate over all coordinates, retrieved in cell slab. - const auto& frag_domains = result_space_tile.frag_domains(); - TileCellSlabIter iter( - range_thread_idx, - num_range_threads, - subarray, - tile_subarrays[t], - tile_extents, - result_space_tile.start_coords(), - range_info, - cell_order); - - // Compute cell offset and destination pointer. - uint64_t cell_offset = - global_order ? tile_offsets[t] + iter.global_offset() : 0; - auto dest_ptr = qc_result.data() + cell_offset; - - while (!iter.end()) { - // Compute destination pointer for row/col major orders. - if (!global_order) { - cell_offset = iter.dest_offset_row_col(); - dest_ptr = qc_result.data() + cell_offset; - } + // Process all tiles in parallel. + auto status = parallel_for_2d( + storage_manager_->compute_tp(), + t_start, + t_end, + 0, + num_range_threads, + [&](uint64_t t, uint64_t range_thread_idx) { + // Find out result space tile and tile subarray. + const DimType* tc = (DimType*)&tile_coords[t][0]; + auto& result_space_tile = result_space_tiles.at(tc); - for (int32_t i = - static_cast(frag_domains.size()) - 1; - i >= 0; - --i) { - // If the cell slab overlaps this fragment domain range, - // apply clause. - auto&& [overlaps, start, end] = cell_slab_overlaps_range( - dim_num, - frag_domains[i].domain(), - iter.cell_slab_coords(), - iter.cell_slab_length()); - if (overlaps) { - // Re-initialize the bitmap to 1 in case of overlapping - // domains. - if (i != static_cast(frag_domains.size()) - 1) { - for (uint64_t c = start; c <= end; c++) { - dest_ptr[c] = 1; - } - } - - RETURN_NOT_OK(condition_->apply_dense( - *(fragment_metadata_[frag_domains[i].fid()] - ->array_schema() - .get()), - result_space_tile.result_tile(frag_domains[i].fid()), - start, - end - start + 1, - iter.pos_in_tile(), - stride, - iter.cell_slab_coords().data(), - dest_ptr)); - - // If any cell doesn't match the query condition, signal - // it in the space tile. - for (uint64_t c = start; c <= end; c++) { - if (dest_ptr[c] == 0) { - result_space_tile.set_qc_filtered_results(); - break; - } - } + // Iterate over all coordinates, retrieved in cell slab. + const auto& frag_domains = result_space_tile.frag_domains(); + TileCellSlabIter iter( + range_thread_idx, + num_range_threads, + subarray, + tile_subarrays[t], + tile_extents, + result_space_tile.start_coords(), + range_info, + cell_order); + + // Compute cell offset and destination pointer. + uint64_t cell_offset = + global_order ? tile_offsets[t] + iter.global_offset() : 0; + auto dest_ptr = qc_result.data() + cell_offset; + + while (!iter.end()) { + // Compute destination pointer for row/col major orders. + if (!global_order) { + cell_offset = iter.dest_offset_row_col(); + dest_ptr = qc_result.data() + cell_offset; + } + + for (int32_t i = static_cast(frag_domains.size()) - 1; + i >= 0; + --i) { + // If the cell slab overlaps this fragment domain range, + // apply clause. + auto&& [overlaps, start, end] = cell_slab_overlaps_range( + dim_num, + frag_domains[i].domain(), + iter.cell_slab_coords(), + iter.cell_slab_length()); + if (overlaps) { + // Re-initialize the bitmap to 1 in case of overlapping + // domains. + if (i != static_cast(frag_domains.size()) - 1) { + for (uint64_t c = start; c <= end; c++) { + dest_ptr[c] = 1; } } - // Adjust the destination pointers for global order. - if (global_order) { - dest_ptr += iter.cell_slab_length(); + RETURN_NOT_OK(condition_->apply_dense( + *(fragment_metadata_[frag_domains[i].fid()] + ->array_schema() + .get()), + result_space_tile.result_tile(frag_domains[i].fid()), + start, + end - start + 1, + iter.pos_in_tile(), + stride, + iter.cell_slab_coords().data(), + dest_ptr)); + + // If any cell doesn't match the query condition, signal + // it in the space tile. + for (uint64_t c = start; c <= end; c++) { + if (dest_ptr[c] == 0) { + result_space_tile.set_qc_filtered_results(); + break; + } } - - ++iter; } + } - return Status::Ok(); - }); - RETURN_NOT_OK(status); + // Adjust the destination pointers for global order. + if (global_order) { + dest_ptr += iter.cell_slab_length(); + } - // For `qc_coords_mode` just fill in the coordinates and skip - // attribute - // processing. - if (qc_coords_mode_) { - for (auto& name : qc_names) { - clear_tiles(name, result_tiles); + ++iter; } - } - return Status::Ok(); - }); + return Status::Ok(); + }); + RETURN_NOT_OK(status); + + // For `qc_coords_mode` just fill in the coordinates and skip + // attribute + // processing. + if (qc_coords_mode_) { + for (auto& name : qc_names) { + clear_tiles(name, result_tiles); + } + } + + return Status::Ok(); + }); } return Status::Ok(); @@ -2256,5 +2254,4 @@ void DenseReader::fill_dense_coords_col_slab( } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index abb2f8884ac5..4b66a38dd3a2 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,21 +34,18 @@ #define TILEDB_FILTERED_DATA_H #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/storage_manager/storage_manager.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** * A filtered data block containing filtered data for multiple tiles. The block * will contain a number of contiguous on-disk tiles and the data is identified * by the fragment index and offset/size of the data in the on-disk file. - * - * This uses a vector for storage which will be replaced by datablocks when - * ready. */ class FilteredDataBlock { public: @@ -63,13 +60,26 @@ class FilteredDataBlock { * coming from. * @param offset File offset of the on-disk data for this datablock. * @param size Size of the on-disk data for this data block. + * @param resource The memory resource. */ - FilteredDataBlock(unsigned frag_idx, uint64_t offset, uint64_t size) - : frag_idx_(frag_idx) + FilteredDataBlock( + unsigned frag_idx, + uint64_t offset, + uint64_t size, + tdb::pmr::memory_resource* resource) + : resource_(resource) + , frag_idx_(frag_idx) , offset_(offset) - , filtered_data_(size) { + , size_(size) + , filtered_data_(tdb::pmr::make_unique(resource_, size)) { + if (!filtered_data_) { + throw std::bad_alloc(); + } } + DISABLE_COPY_AND_COPY_ASSIGN(FilteredDataBlock); + DISABLE_MOVE_AND_MOVE_ASSIGN(FilteredDataBlock); + /* ********************************* */ /* API */ /* ********************************* */ @@ -85,21 +95,20 @@ class FilteredDataBlock { } /** - * @return Pointer to the data at a particular offset in the filtered data - * file. + * @return Pointer to the data at the given offset in the filtered data file. */ inline void* data_at(storage_size_t offset) { - return filtered_data_.data() + offset - offset_; + return filtered_data_.get() + offset - offset_; } /** @return Pointer to the data inside of the filtered data block. */ inline void* data() { - return filtered_data_.data(); + return filtered_data_.get(); } /** @return Size of the data block. */ inline storage_size_t size() const { - return filtered_data_.size(); + return size_; } /** @@ -109,13 +118,15 @@ class FilteredDataBlock { inline bool contains( unsigned frag_idx, storage_size_t offset, storage_size_t size) const { return frag_idx == frag_idx_ && offset >= offset_ && - offset + size <= offset_ + filtered_data_.size(); + offset + size <= offset_ + size_; } private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** The memory resource to use. */ + tdb::pmr::memory_resource* resource_; /** Fragment index for the data this data block contains. */ unsigned frag_idx_; @@ -123,8 +134,11 @@ class FilteredDataBlock { /** File offset of the on-disk data for this datablock. */ storage_size_t offset_; + /** The size of the data. */ + storage_size_t size_; + /** Data for the data block. */ - std::vector filtered_data_; + tdb::pmr::unique_ptr filtered_data_; }; /** @@ -159,6 +173,7 @@ class FilteredData { * @param validity_only Is the field read for validity only? * @param storage_manager Storage manager. * @param read_tasks Read tasks to queue new tasks on for new data blocks. + * @param memory_tracker Memory tracker. */ FilteredData( const ReaderBase& reader, @@ -172,8 +187,16 @@ class FilteredData { const bool nullable, const bool validity_only, StorageManager* storage_manager, - std::vector& read_tasks) - : name_(name) + std::vector& read_tasks, + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , fixed_data_blocks_( + memory_tracker_->get_resource(MemoryType::FILTERED_DATA)) + , var_data_blocks_( + memory_tracker_->get_resource(MemoryType::FILTERED_DATA)) + , nullable_data_blocks_( + memory_tracker_->get_resource(MemoryType::FILTERED_DATA)) + , name_(name) , fragment_metadata_(fragment_metadata) , var_sized_(var_sized) , nullable_(nullable) @@ -184,6 +207,8 @@ class FilteredData { } uint64_t tiles_allocated = 0; + auto* block_resource = + memory_tracker_->get_resource(MemoryType::FILTERED_DATA_BLOCK); // Store data on the datablock in progress for fixed, var and nullable data. std::optional current_frag_idx{nullopt}; @@ -251,19 +276,28 @@ class FilteredData { // Finish by pushing the last in progress blocks. if (current_fixed_size != 0) { fixed_data_blocks_.emplace_back( - *current_frag_idx, current_fixed_offset, current_fixed_size); + *current_frag_idx, + current_fixed_offset, + current_fixed_size, + block_resource); queue_last_block_for_read(TileType::FIXED); } if (current_var_size != 0) { var_data_blocks_.emplace_back( - *current_frag_idx, current_var_offset, current_var_size); + *current_frag_idx, + current_var_offset, + current_var_size, + block_resource); queue_last_block_for_read(TileType::VAR); } if (current_nullable_size != 0) { nullable_data_blocks_.emplace_back( - *current_frag_idx, current_nullable_offset, current_nullable_size); + *current_frag_idx, + current_nullable_offset, + current_nullable_size, + block_resource); queue_last_block_for_read(TileType::NULLABLE); } @@ -274,6 +308,9 @@ class FilteredData { current_nullable_data_block_ = nullable_data_blocks_.begin(); } + DISABLE_COPY_AND_COPY_ASSIGN(FilteredData); + DISABLE_MOVE_AND_MOVE_ASSIGN(FilteredData); + /** Destructor. */ ~FilteredData() = default; @@ -364,7 +401,7 @@ class FilteredData { } /** @return Data blocks corresponding to the tile type. */ - inline std::vector& data_blocks(const TileType type) { + inline tdb::pmr::list& data_blocks(const TileType type) { switch (type) { case TileType::FIXED: return fixed_data_blocks_; @@ -378,7 +415,7 @@ class FilteredData { } /** @return Current data block corresponding to the tile type. */ - inline std::vector::iterator& current_data_block( + inline tdb::pmr::list::iterator& current_data_block( const TileType type) { switch (type) { case TileType::FIXED: @@ -516,7 +553,10 @@ class FilteredData { } else { // Push the old batch and start a new one. data_blocks(type).emplace_back( - *current_block_frag_idx, current_block_offset, current_block_size); + *current_block_frag_idx, + current_block_offset, + current_block_size, + memory_tracker_->get_resource(MemoryType::FILTERED_DATA_BLOCK)); queue_last_block_for_read(type); current_block_offset = offset; current_block_size = size; @@ -554,23 +594,26 @@ class FilteredData { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** Memory tracker for the filtered data. */ + shared_ptr memory_tracker_; + /** Fixed data blocks. */ - std::vector fixed_data_blocks_; + tdb::pmr::list fixed_data_blocks_; /** Current fixed data block used when creating fixed tiles. */ - std::vector::iterator current_fixed_data_block_; + tdb::pmr::list::iterator current_fixed_data_block_; /** Var data blocks. */ - std::vector var_data_blocks_; + tdb::pmr::list var_data_blocks_; /** Current var data block used when creating var tiles. */ - std::vector::iterator current_var_data_block_; + tdb::pmr::list::iterator current_var_data_block_; /** Nullable data blocks. */ - std::vector nullable_data_blocks_; + tdb::pmr::list nullable_data_blocks_; /** Current nullable data block used when creating nullable tiles. */ - std::vector::iterator current_nullable_data_block_; + tdb::pmr::list::iterator current_nullable_data_block_; /** Name of the attribute. */ const std::string& name_; @@ -591,7 +634,6 @@ class FilteredData { std::vector& read_tasks_; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_FILTERED_DATA_H diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 8b449fdc988d..d5948d81170d 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -55,8 +55,7 @@ #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/apply_with_type.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { using dimension_size_type = uint32_t; @@ -74,6 +73,7 @@ class ReaderBaseStatusException : public StatusException { ReaderBase::ReaderBase( stats::Stats* stats, shared_ptr logger, StrategyParams& params) : StrategyBase(stats, logger, params) + , memory_tracker_(params.query_memory_tracker()) , condition_(params.condition()) , user_requested_timestamps_(false) , use_timestamps_(false) @@ -586,25 +586,25 @@ Status ReaderBase::read_and_unfilter_coordinate_tiles( return Status::Ok(); } -std::vector ReaderBase::read_attribute_tiles( +std::list ReaderBase::read_attribute_tiles( const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_attribute_tiles"); return read_tiles(names, result_tiles); } -std::vector ReaderBase::read_coordinate_tiles( +std::list ReaderBase::read_coordinate_tiles( const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_coordinate_tiles"); return read_tiles(NameToLoad::from_string_vec(names), result_tiles); } -std::vector ReaderBase::read_tiles( +std::list ReaderBase::read_tiles( const std::vector& names, const std::vector& result_tiles) const { auto timer_se = stats_->start_timer("read_tiles"); - std::vector filtered_data; + std::list filtered_data; // Shortcut for empty tile vec. if (result_tiles.empty() || names.empty()) { @@ -613,7 +613,6 @@ std::vector ReaderBase::read_tiles( uint64_t num_tiles_read{0}; std::vector read_tasks; - filtered_data.reserve(names.size()); // Run all attributes independently. for (auto n : names) { @@ -637,7 +636,8 @@ std::vector ReaderBase::read_tiles( nullable, val_only, storage_manager_, - read_tasks); + read_tasks, + memory_tracker_); // Go through each tiles and create the attribute tiles. for (auto tile : result_tiles) { @@ -1368,5 +1368,4 @@ template void ReaderBase::validate_attribute_order( std::vector&, std::vector&); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index 3200c272ce22..d9a859d13e81 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -35,6 +35,7 @@ #include "../strategy_base.h" #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/tile_domain.h" @@ -49,8 +50,7 @@ #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/subarray_partitioner.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class ArraySchema; @@ -256,6 +256,9 @@ class ReaderBase : public StrategyBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** The query's memory tracker. */ + shared_ptr memory_tracker_; + /** The query condition. */ std::optional& condition_; @@ -561,7 +564,7 @@ class ReaderBase : public StrategyBase { * `ResultTile` instances in this vector. * @return Filtered data blocks. */ - std::vector read_attribute_tiles( + std::list read_attribute_tiles( const std::vector& names, const std::vector& result_tiles) const; @@ -577,7 +580,7 @@ class ReaderBase : public StrategyBase { * `ResultTile` instances in this vector. * @return Filtered data blocks. */ - std::vector read_coordinate_tiles( + std::list read_coordinate_tiles( const std::vector& names, const std::vector& result_tiles) const; @@ -594,7 +597,7 @@ class ReaderBase : public StrategyBase { * @param validity_only Is the field read for validity only. * @return Filtered data blocks. */ - std::vector read_tiles( + std::list read_tiles( const std::vector& names, const std::vector& result_tiles) const; @@ -799,7 +802,6 @@ class ReaderBase : public StrategyBase { std::vector& frag_first_array_tile_idx); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_READER_BASE_H From e6f79dad6e15a6080388ddd945ba093e53969029 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Sat, 9 Mar 2024 04:06:31 -0500 Subject: [PATCH 234/456] Remove trailing slash from rest.server_address. (#4790) A user reported errors accessing assets over REST after setting `rest.server_address` with a URL ending in `/`. This updates `Config::sanity_check` to remove status and return a validated config value for the given parameter. --- TYPE: BUG DESC: Remove trailing slash from rest.server_address. --- test/src/unit-cppapi-config.cc | 9 +------ test/src/unit-curl.cc | 49 +++++++++++++++++++++++++++++----- test/support/src/helpers.cc | 8 ++++++ test/support/src/helpers.h | 9 +++++++ tiledb/sm/rest/rest_client.cc | 7 ++++- tiledb/sm/rest/rest_client.h | 4 +++ 6 files changed, 71 insertions(+), 15 deletions(-) diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 609ece4c7c0e..7be58bb59d32 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -33,17 +33,10 @@ #include #include +#include "test/support/src/helpers.h" #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/cpp_api/tiledb" -int setenv_local(const char* __name, const char* __value) { -#ifdef _WIN32 - return _putenv_s(__name, __value); -#else - return ::setenv(__name, __value, 1); -#endif -} - TEST_CASE("C++ API: Config", "[cppapi][config]") { tiledb::Config config; config["foo"] = "bar"; diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index 7ae68275e38d..9241941ad2fb 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -31,13 +31,15 @@ */ #include -#include "tiledb/sm/rest/curl.h" -#ifdef _WIN32 -#include "tiledb/sm/filesystem/win.h" -#else -#include "tiledb/sm/filesystem/posix.h" -#endif +// clang-format off +#include "test/support/src/helpers.h" +#include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/rest/curl.h" // Must be included last to avoid Windows.h +// clang-format on + +#include +#include using namespace tiledb::sm; @@ -91,3 +93,38 @@ TEST_CASE("CURL: Test curl's header parsing callback", "[curl]") { userdata.redirect_uri_map->find(ns_array)->second == "tiledb://my_username"); } + +TEST_CASE( + "RestClient: Remove trailing slash from rest_server_", "[rest-client]") { + std::string rest_server = + GENERATE("http://localhost:8080/", "http://localhost:8080//"); + tiledb::sm::Config cfg; + SECTION("rest.server_address set in Config") { + cfg.set("rest.server_address", rest_server).ok(); + } + SECTION("rest.server_address set in environment") { + setenv_local("TILEDB_REST_SERVER_ADDRESS", rest_server.c_str()); + } + SECTION("rest.server_address set by loaded config file") { + std::string cfg_file = "tiledb_config.txt"; + std::ofstream file(cfg_file); + file << "rest.server_address " << rest_server << std::endl; + file.close(); + cfg.load_from_file(cfg_file).ok(); + std::filesystem::remove(cfg_file); + } + + ThreadPool tp{1}; + ContextResources resources( + cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); + tiledb::sm::RestClient rest_client; + REQUIRE(rest_client + .init( + &tiledb::test::g_helper_stats, + &cfg, + &tp, + tiledb::test::g_helper_logger(), + resources) + .ok()); + CHECK(rest_client.rest_server() == "http://localhost:8080"); +} diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 7f2789d8bb02..f7192213ad9e 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -57,6 +57,14 @@ #include "tiledb/sm/serialization/array.h" #include "tiledb/sm/serialization/query.h" +int setenv_local(const char* __name, const char* __value) { +#ifdef _WIN32 + return _putenv_s(__name, __value); +#else + return ::setenv(__name, __value, 1); +#endif +} + std::mutex catch2_macro_mutex; namespace tiledb::test { diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 283f536ea446..ddf3bc581a7c 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -52,6 +52,15 @@ #include #include +/** + * Helper function to set environment variables across platforms. + * + * @param __name Name of the environment variable. + * @param __value Value of the environment variable. + * @return 0 on success, -1 on error. + */ +int setenv_local(const char* __name, const char* __value); + // A mutex for protecting the thread-unsafe Catch2 macros. extern std::mutex catch2_macro_mutex; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 0de78d0d4047..47a5f809fbf8 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -114,8 +114,13 @@ Status RestClient::init( const char* c_str; RETURN_NOT_OK(config_->get("rest.server_address", &c_str)); - if (c_str != nullptr) + if (c_str != nullptr) { rest_server_ = std::string(c_str); + if (rest_server_.ends_with('/')) { + size_t pos = rest_server_.find_last_not_of('/'); + rest_server_.resize(pos + 1); + } + } if (rest_server_.empty()) return LOG_STATUS(Status_RestError( "Error initializing rest client; server address is empty.")); diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 737e671ebc43..e3e67ddb3b71 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -385,6 +385,10 @@ class RestClient { */ Status post_vacuum_to_rest(const URI& uri, const Config& config); + inline std::string rest_server() const { + return rest_server_; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ From c0be5c84c11b9adc11f901e84be04e12465a6413 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Sat, 9 Mar 2024 11:07:41 +0200 Subject: [PATCH 235/456] Fix throwing exception on remote incomplete queries with aggregates. (#4798) [SC-42786](https://app.shortcut.com/tiledb-inc/story/42786/remote-incomplete-queries-with-aggregates-fail-with-a-vague-message) Validated with a locally built REST server. Before: `[TileDB::REST] Error: Error submitting query to REST; server returned no data. Curl error: Error in libcurl POST operation: libcurl error message 'CURLE_OK'; HTTP code 400; server response data '{"code":2208,"message":"Error serializing query: C API: unknown exception type; no further information","request_id":"a0f026a8-8507-4c54-b592-5166c7d4f0b6"}'.` After: `[TileDB::REST] Error: Error submitting query to REST; server returned no data. Curl error: Error in libcurl POST operation: libcurl error message 'CURLE_OK'; HTTP code 400; server response data '{"code":2208,"message":"Error serializing query: [TileDB::Serialization] Error: Cannot serialize; exception: [TileDB::Serialization] Error: Aggregates are not currently supported in incomplete remote queries","request_id":"c6638565-d80d-4c0c-88a6-52c1a748df1a"}'.` --- TYPE: BUG DESC: Provide a clear exception message when performing remote incomplete queries with aggregates --- tiledb/sm/serialization/query.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 6e0a894e6063..7d98392eddaf 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -1551,9 +1551,9 @@ Status query_to_capnp( // that has aggregates on it, this behavior is currently not supported. if (!client_side && query.status() == QueryStatus::INCOMPLETE && query.has_aggregates()) { - throw Status_SerializationError( + throw StatusException(Status_SerializationError( "Aggregates are not currently supported in incomplete remote " - "queries"); + "queries")); } query_channels_to_capnp(query, query_builder); From a477ca6e10fc6d344be0ab019bef9b6dab58294c Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Mon, 11 Mar 2024 08:53:20 -0400 Subject: [PATCH 236/456] Support VFS ls_recursive API for posix filesystem. (#4778) @shaunrd0 previously added `ls_recursive` support for the S3 file system. This pull request implements it for Posix, in a hopefully-portable way (`std::filesystem::recursive_directory_iterator`) which can be recycled for Windows. Some notes about input/output: - while `LsObjects` has a `std::string` component, the string is a URI. - for posix, we include empty directories in the result set. The recursion is implemented so that if a directory does not pass the directory filter, then we do not descend into that directory. I have added a unit test to check this behavior. Something like this could be useful for S3 as well if the "list objects" request is done in hierarchical mode. --- TYPE: FEATURE DESC: Support VFS `ls_recursive` API for posix filesystem. --------- Co-authored-by: Luc Rancourt --- test/src/unit-cppapi-vfs.cc | 37 +- test/src/unit-tile-metadata-generator.cc | 4 +- test/src/unit-tile-metadata.cc | 4 +- test/support/src/vfs_helpers.cc | 18 + tiledb/CMakeLists.txt | 1 - tiledb/api/c_api/vfs/vfs_api_experimental.h | 3 +- tiledb/common/stdx_string.cc | 7 +- tiledb/common/stdx_string.h | 10 +- tiledb/sm/filesystem/CMakeLists.txt | 1 - tiledb/sm/filesystem/ls_scanner.cc | 33 - tiledb/sm/filesystem/ls_scanner.h | 57 ++ tiledb/sm/filesystem/path_win.cc | 6 +- tiledb/sm/filesystem/path_win.h | 2 +- tiledb/sm/filesystem/posix.cc | 16 +- tiledb/sm/filesystem/posix.h | 32 +- tiledb/sm/filesystem/test/unit_ls_filtered.cc | 607 +++++++++++++++++- tiledb/sm/filesystem/uri.cc | 26 +- tiledb/sm/filesystem/uri.h | 22 +- tiledb/sm/filesystem/vfs.cc | 7 +- tiledb/sm/filesystem/vfs.h | 37 +- tiledb/sm/filesystem/win.h | 24 + 21 files changed, 835 insertions(+), 119 deletions(-) delete mode 100644 tiledb/sm/filesystem/ls_scanner.cc diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index 52bf25ac41f2..0a0c78bced4e 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -63,6 +63,7 @@ TEST_CASE("C++ API: Test VFS ls", "[cppapi][cppapi-vfs][cppapi-vfs-ls]") { std::string file2 = dir + "/file2"; std::string subdir = dir + "/subdir"; std::string subdir2 = dir + "/subdir2"; + std::string subdir_empty = dir + "/subdir_empty"; std::string subdir_file = subdir + "/file"; std::string subdir_file2 = subdir2 + "/file2"; @@ -71,6 +72,7 @@ TEST_CASE("C++ API: Test VFS ls", "[cppapi][cppapi-vfs][cppapi-vfs-ls]") { vfs.create_dir(dir); vfs.create_dir(subdir); vfs.create_dir(subdir2); + vfs.create_dir(subdir_empty); vfs.touch(file); vfs.touch(file2); vfs.touch(subdir_file); @@ -85,15 +87,17 @@ TEST_CASE("C++ API: Test VFS ls", "[cppapi][cppapi-vfs][cppapi-vfs-ls]") { file2 = tiledb::sm::path_win::uri_from_path(file2); subdir = tiledb::sm::path_win::uri_from_path(subdir); subdir2 = tiledb::sm::path_win::uri_from_path(subdir2); + subdir_empty = tiledb::sm::path_win::uri_from_path(subdir_empty); #endif // Check results std::sort(children.begin(), children.end()); - REQUIRE(children.size() == 4); + REQUIRE(children.size() == 5); CHECK(children[0] == file); CHECK(children[1] == file2); CHECK(children[2] == subdir); CHECK(children[3] == subdir2); + CHECK(children[4] == subdir_empty); // Clean up vfs.remove_dir(path); @@ -503,13 +507,18 @@ TEST_CASE( } } -TEST_CASE("CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]") { +using ls_recursive_test_types = + std::tuple; +TEMPLATE_LIST_TEST_CASE( + "CPP API: VFS ls_recursive filter", + "[cppapi][vfs][ls-recursive]", + ls_recursive_test_types) { using namespace tiledb::test; - S3Test s3_test({10, 100, 0}); - if (!s3_test.is_supported()) { + TestType test({10, 100, 0}); + if (!test.is_supported()) { return; } - auto expected_results = s3_test.expected_results(); + auto expected_results = test.expected_results(); vfs_config cfg; tiledb::Context ctx(tiledb::Config(&cfg.config)); @@ -534,14 +543,6 @@ TEST_CASE("CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]") { include = [](std::string_view, uint64_t) { return false; }; } - bool include_result = true; - SECTION("Custom filter (include half)") { - include = [&include_result](std::string_view, uint64_t) { - include_result = !include_result; - return include_result; - }; - } - SECTION("Custom filter (search for test_file_50)") { include = [](std::string_view object_name, uint64_t) { return object_name.find("test_file_50") != std::string::npos; @@ -553,21 +554,25 @@ TEST_CASE("CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]") { }; } SECTION("Custom filter (reject files over 50 bytes)") { - include = [](std::string_view, uint64_t size) { return size <= 50; }; + include = []([[maybe_unused]] std::string_view entry, uint64_t size) { + return size <= 50; + }; } // Test collecting results with LsInclude predicate. auto results = tiledb::VFSExperimental::ls_recursive_filter( - ctx, vfs, s3_test.temp_dir_.to_string(), include); + ctx, vfs, test.temp_dir_.to_string(), include); std::erase_if(expected_results, [&include](const auto& object) { return !include(object.first, object.second); }); + std::sort(results.begin(), results.end()); CHECK(results.size() == expected_results.size()); CHECK(expected_results == results); // Test collecting results with LsCallback, writing data into ls_objects. tiledb::VFSExperimental::ls_recursive( - ctx, vfs, s3_test.temp_dir_.to_string(), cb); + ctx, vfs, test.temp_dir_.to_string(), cb); + std::sort(ls_objects.begin(), ls_objects.end()); CHECK(ls_objects.size() == expected_results.size()); CHECK(expected_results == ls_objects); } diff --git a/test/src/unit-tile-metadata-generator.cc b/test/src/unit-tile-metadata-generator.cc index 7854550999eb..8ededa58c32c 100644 --- a/test/src/unit-tile-metadata-generator.cc +++ b/test/src/unit-tile-metadata-generator.cc @@ -348,7 +348,7 @@ TEST_CASE( bool empty_tile = test == "empty tile"; uint64_t max_string_size = 100; - uint64_t num_strings = 2000; + int num_strings = 2000; // Generate the array schema. uint64_t num_cells = empty_tile ? 0 : 20; @@ -362,7 +362,7 @@ TEST_CASE( // Generate random, sorted strings for the string ascii type. std::vector strings; strings.reserve(num_strings); - for (uint64_t i = 0; i < num_strings; i++) { + for (int i = 0; i < num_strings; i++) { strings.emplace_back(tiledb::test::random_string(rand() % max_string_size)); } std::sort(strings.begin(), strings.end()); diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index f478b2d30953..8b92b13f9b0e 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -748,12 +748,12 @@ struct CPPVarTileMetadataFx { std::default_random_engine random_engine; uint64_t max_string_size = 100; - uint64_t num_strings = 2000; + int num_strings = 2000; if (f == 0) { // Generate random, sorted strings for the string ascii type. strings_.reserve(num_strings); - for (uint64_t i = 0; i < num_strings; i++) { + for (int i = 0; i < num_strings; i++) { strings_.emplace_back( tiledb::test::random_string(rand() % max_string_size)); } diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index 14bde730bd1a..98d831136cd5 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -497,6 +497,24 @@ LocalFsTest::LocalFsTest(const std::vector& test_tree) temp_dir_ = tiledb::test::test_dir(prefix_ + tiledb::sm::Posix::current_dir() + "/"); #endif + + vfs_.create_dir(temp_dir_).ok(); + // TODO: We could refactor to remove duplication with S3Test() + for (size_t i = 1; i <= test_tree_.size(); i++) { + sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); + vfs_.create_dir(path).ok(); + expected_results().emplace_back(path.to_string(), 0); + for (size_t j = 1; j <= test_tree_[i - 1]; j++) { + auto object_uri = path.join_path("test_file_" + std::to_string(j)); + vfs_.touch(object_uri).ok(); + std::string data(j * 10, 'a'); + vfs_.open_file(object_uri, sm::VFSMode::VFS_WRITE).ok(); + vfs_.write(object_uri, data.data(), data.size()).ok(); + vfs_.close_file(object_uri).ok(); + expected_results().emplace_back(object_uri.to_string(), data.size()); + } + } + std::sort(expected_results().begin(), expected_results().end()); } } // namespace tiledb::test diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 737cc50f7905..e245c55109fc 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -193,7 +193,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/uri.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/vfs_file_handle.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/ls_scanner.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/win.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filesystem/filesystem_base.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/filter/bit_width_reduction_filter.cc diff --git a/tiledb/api/c_api/vfs/vfs_api_experimental.h b/tiledb/api/c_api/vfs/vfs_api_experimental.h index b70f97629c7d..dba16811ae40 100644 --- a/tiledb/api/c_api/vfs/vfs_api_experimental.h +++ b/tiledb/api/c_api/vfs/vfs_api_experimental.h @@ -58,7 +58,8 @@ typedef int32_t (*tiledb_ls_callback_t)( * on error. The callback is responsible for writing gathered entries into the * `data` buffer, for example using a pointer to a user-defined struct. * - * Currently only S3 is supported, and the `path` must be a valid S3 URI. + * Currently only Posix and S3 are supported, and the `path` must be a valid URI + * for one of those filesystems. * * **Example:** * diff --git a/tiledb/common/stdx_string.cc b/tiledb/common/stdx_string.cc index 147017c6ae3b..1ad50a473798 100644 --- a/tiledb/common/stdx_string.cc +++ b/tiledb/common/stdx_string.cc @@ -31,21 +31,20 @@ namespace tiledb::stdx::string { -bool starts_with(const std::string& value, const std::string& prefix) { +bool starts_with(std::string_view value, std::string_view prefix) { if (prefix.size() > value.size()) return false; return std::equal(prefix.begin(), prefix.end(), value.begin()); } -bool ends_with(const std::string& value, const std::string& suffix) { +bool ends_with(std::string_view value, std::string_view suffix) { if (suffix.size() > value.size()) return false; return value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; } -size_t common_prefix_size( - const std::string_view& a, const std::string_view& b) { +size_t common_prefix_size(std::string_view a, std::string_view b) { size_t size = std::min(a.size(), b.size()); for (size_t i = 0; i < size; ++i) { if (a[i] != b[i]) diff --git a/tiledb/common/stdx_string.h b/tiledb/common/stdx_string.h index 2f2cfb9436ca..7a9f4d37e9f5 100644 --- a/tiledb/common/stdx_string.h +++ b/tiledb/common/stdx_string.h @@ -43,7 +43,7 @@ namespace tiledb::stdx::string { * @param prefix The prefix string to be tested. * @return `true` if `value` starts with `prefix`; `false` otherwise. */ -bool starts_with(const std::string& value, const std::string& prefix); +bool starts_with(std::string_view value, std::string_view prefix); /** * Checks if a string ends with a certain suffix. @@ -55,7 +55,7 @@ bool starts_with(const std::string& value, const std::string& prefix); * @param suffix The suffix to be tested. * @return `true` if `value` ends with `suffix`; `false` otherwise. */ -bool ends_with(const std::string& value, const std::string& suffix); +bool ends_with(std::string_view value, std::string_view suffix); /** * Returns the size of the common prefix between `a` and `b`. @@ -64,7 +64,7 @@ bool ends_with(const std::string& value, const std::string& suffix); * a[0..n) == b[0..n) (These ranges are empty if n == 0.) * n == length of a or n == length of b or a[n] != b[n] */ -size_t common_prefix_size(const std::string_view& a, const std::string_view& b); +size_t common_prefix_size(std::string_view a, std::string_view b); } // namespace tiledb::stdx::string @@ -72,10 +72,10 @@ size_t common_prefix_size(const std::string_view& a, const std::string_view& b); * Functions forwarded into the legacy namespace. */ namespace tiledb::sm::utils::parse { -inline bool starts_with(const std::string& value, const std::string& prefix) { +inline bool starts_with(std::string_view value, std::string_view prefix) { return tiledb::stdx::string::starts_with(value, prefix); } -inline bool ends_with(const std::string& value, const std::string& suffix) { +inline bool ends_with(std::string_view value, std::string_view suffix) { return tiledb::stdx::string::ends_with(value, suffix); } } // namespace tiledb::sm::utils::parse diff --git a/tiledb/sm/filesystem/CMakeLists.txt b/tiledb/sm/filesystem/CMakeLists.txt index fd0e1f34a40e..121829379347 100644 --- a/tiledb/sm/filesystem/CMakeLists.txt +++ b/tiledb/sm/filesystem/CMakeLists.txt @@ -38,7 +38,6 @@ commence(object_library vfs) uri.cc vfs.cc vfs_file_handle.cc - ls_scanner.cc win.cc filesystem_base.cc ../curl/curl_init.cc diff --git a/tiledb/sm/filesystem/ls_scanner.cc b/tiledb/sm/filesystem/ls_scanner.cc deleted file mode 100644 index bcccd82870ec..000000000000 --- a/tiledb/sm/filesystem/ls_scanner.cc +++ /dev/null @@ -1,33 +0,0 @@ -/** - * @file ls_scanner.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This defines the LsScanner class and related types used for VFS. - */ - -#include "ls_scanner.h" diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h index 4f925c55aa0b..92bc435bd9e7 100644 --- a/tiledb/sm/filesystem/ls_scanner.h +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -37,6 +37,7 @@ #include "tiledb/sm/filesystem/uri.h" #include +#include #include #include @@ -73,6 +74,10 @@ class LsStopTraversal : public LsScanException { }; using FileFilter = std::function; +[[maybe_unused]] static bool accept_all_files( + const std::string_view&, uint64_t) { + return true; +} using DirectoryFilter = std::function; /** Static DirectoryFilter used as default argument. */ @@ -350,6 +355,58 @@ class CallbackWrapperCPP { LsCallback cb_; }; +/** + * Implements the `ls_filtered` function for `std::filesystem` which can be used + * for Posix and Win32 + */ +template +LsObjects std_filesystem_ls_filtered( + const URI& parent, F file_filter, D directory_filter, bool recursive) { + /* + * The input URI was useful to the top-level VFS to identify this is a + * regular filesystem path, but we don't need the "file://" qualifier + * anymore and can reason with unqualified strings for the rest of the + * function. + */ + const auto parentstr = parent.to_path(); + + LsObjects qualifyingPaths; + + // awkward way of iterating, avoids bug in OSX + auto begin = std::filesystem::recursive_directory_iterator(parentstr); + auto end = std::filesystem::recursive_directory_iterator(); + + for (auto iter = begin; iter != end; ++iter) { + auto& entry = *iter; + const auto abspath = entry.path().string(); + const auto absuri = URI(abspath); + if (entry.is_directory()) { + if (file_filter(absuri, 0) || directory_filter(absuri)) { + qualifyingPaths.push_back( + std::make_pair(tiledb::sm::URI(abspath).to_string(), 0)); + if (!recursive) { + iter.disable_recursion_pending(); + } + } else { + /* do not descend into directories which don't qualify */ + iter.disable_recursion_pending(); + } + } else { + /* + * A leaf of the filesystem + * (or symbolic link - split to a separate case if we want to descend into + * them) + */ + if (file_filter(absuri, entry.file_size())) { + qualifyingPaths.push_back( + std::make_pair(absuri.to_string(), entry.file_size())); + } + } + } + + return qualifyingPaths; +} + } // namespace tiledb::sm #endif // TILEDB_LS_SCANNER_H diff --git a/tiledb/sm/filesystem/path_win.cc b/tiledb/sm/filesystem/path_win.cc index ded0bcb25bb8..49ee7c4adb15 100644 --- a/tiledb/sm/filesystem/path_win.cc +++ b/tiledb/sm/filesystem/path_win.cc @@ -69,11 +69,13 @@ std::string uri_from_path(const std::string& path) { return str_uri; } -std::string path_from_uri(const std::string& uri) { - if (uri.length() == 0) { +std::string path_from_uri(std::string_view uri_view) { + if (uri_view.length() == 0) { return ""; } + std::string uri(uri_view); + std::string uri_with_scheme = (stdx::string::starts_with(uri, "file://") || // also accept 'file:/x...' diff --git a/tiledb/sm/filesystem/path_win.h b/tiledb/sm/filesystem/path_win.h index 1ea8a1e5e9a5..b7f5c429d5db 100644 --- a/tiledb/sm/filesystem/path_win.h +++ b/tiledb/sm/filesystem/path_win.h @@ -61,7 +61,7 @@ std::string uri_from_path(const std::string& path); * @param path The URI to convert. * @status A Windows path. */ -std::string path_from_uri(const std::string& uri); +std::string path_from_uri(std::string_view uri); /** * Converts any '/' to '\\' (single-backslash) and returns the diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index 70d4e8320bcd..58b4f0ab1d5f 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -355,7 +355,7 @@ Status Posix::ls( return Status::Ok(); } -std::string Posix::abs_path(const std::string& path) { +std::string Posix::abs_path(std::string_view path) { std::string resolved_path = abs_path_internal(path); // Ensure the returned has the same postfix slash as 'path'. @@ -392,7 +392,7 @@ bool Posix::both_slashes(char a, char b) { return a == '/' && b == '/'; } -std::string Posix::abs_path_internal(const std::string& path) { +std::string Posix::abs_path_internal(std::string_view path) { // Initialize current, home and root std::string current = current_dir(); auto env_home_ptr = getenv("HOME"); @@ -411,15 +411,17 @@ std::string Posix::abs_path_internal(const std::string& path) { // Other cases std::string ret_dir; if (utils::parse::starts_with(path, posix_prefix)) - return path; + return std::string(path); else if (utils::parse::starts_with(path, "/")) - ret_dir = posix_prefix + path; + ret_dir = posix_prefix + std::string(path); else if (utils::parse::starts_with(path, "~/")) - ret_dir = posix_prefix + home + path.substr(1, path.size() - 1); + ret_dir = + posix_prefix + home + std::string(path.substr(1, path.size() - 1)); else if (utils::parse::starts_with(path, "./")) - ret_dir = posix_prefix + current + path.substr(1, path.size() - 1); + ret_dir = + posix_prefix + current + std::string(path.substr(1, path.size() - 1)); else - ret_dir = posix_prefix + current + "/" + path; + ret_dir = posix_prefix + current + "/" + std::string(path); adjacent_slashes_dedup(&ret_dir); purge_dots_from_path(&ret_dir); diff --git a/tiledb/sm/filesystem/posix.h b/tiledb/sm/filesystem/posix.h index 958a4df51a1d..1c397e8e21ce 100644 --- a/tiledb/sm/filesystem/posix.h +++ b/tiledb/sm/filesystem/posix.h @@ -35,16 +35,21 @@ #ifndef _WIN32 +#include #include #include +#include +#include #include #include #include +#include "tiledb/common/logger.h" #include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/filesystem_base.h" +#include "tiledb/sm/filesystem/ls_scanner.h" using namespace tiledb::common; @@ -259,6 +264,29 @@ class Posix : public FilesystemBase { tuple>> ls_with_sizes(const URI& uri) const override; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + * + * Note: the return type LsObjects does not match the other "ls" methods so as + * to match the S3 equivalent API. + */ + template + LsObjects ls_filtered( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false) const { + return std_filesystem_ls_filtered(parent, f, d, recursive); + } + /** * Lists files one level deep under a given path. * @@ -272,7 +300,7 @@ class Posix : public FilesystemBase { * Returns the absolute posix (string) path of the input in the * form "file://" */ - static std::string abs_path(const std::string& path); + static std::string abs_path(std::string_view path); /** * Returns the directory where the program is executed. @@ -289,7 +317,7 @@ class Posix : public FilesystemBase { static bool both_slashes(char a, char b); // Internal logic for 'abs_path()'. - static std::string abs_path_internal(const std::string& path); + static std::string abs_path_internal(std::string_view path); /** * It takes as input an **absolute** path, and returns it in its canonicalized diff --git a/tiledb/sm/filesystem/test/unit_ls_filtered.cc b/tiledb/sm/filesystem/test/unit_ls_filtered.cc index 8af2ec8f225a..b8a2cd8b9a05 100644 --- a/tiledb/sm/filesystem/test/unit_ls_filtered.cc +++ b/tiledb/sm/filesystem/test/unit_ls_filtered.cc @@ -35,6 +35,18 @@ #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/vfs.h" +namespace tiledb::sm { +/** + * @return true if the URI represents a regular file, false if not + */ +[[maybe_unused]] static bool accept_only_regular_files( + const std::string_view& uri, uint64_t) { + const std::string path = URI(uri).to_path(); + return std::filesystem::is_regular_file(path); +} + +} // namespace tiledb::sm + class VFSTest { public: /** @@ -52,7 +64,8 @@ class VFSTest { , vfs_(&stats_, &io_, &compute_, tiledb::sm::Config()) , test_tree_(test_tree) , prefix_(prefix) - , temp_dir_(prefix_) { + , temp_dir_(prefix_) + , init_open_files_(count_open_files()) { } virtual ~VFSTest() { @@ -63,9 +76,38 @@ class VFSTest { } } - /** FilePredicate for passing to ls_filtered that accepts all files. */ - static bool accept_all_files(const std::string_view&, uint64_t) { - return true; + Status mkdir() const { + return vfs_.create_dir(temp_dir_); + } + +#ifdef __windows__ + std::optional count_open_files() const { + return std::nullopt; + } +#else + std::optional count_open_files() const { + const std::string fddir = "/proc/" + std::to_string(getpid()) + "/fd"; + + std::vector ls; + const auto st = vfs_.ls(tiledb::sm::URI(fddir), &ls); + REQUIRE(st.ok()); + return ls.size(); + } +#endif + + /** + * @return true if the number of open files is the same + * as it was when we started the test + */ + bool check_open_files() const { + const auto maybe_updated_open_files = count_open_files(); + if (maybe_updated_open_files) { + const uint64_t updated_open_files = *maybe_updated_open_files; + return (updated_open_files == init_open_files_); + } else { + /* not enough information to say otherwise */ + return true; + } } /** Resources needed to construct VFS */ @@ -77,40 +119,573 @@ class VFSTest { std::string prefix_; tiledb::sm::URI temp_dir_; + std::optional init_open_files_; + private: tiledb::sm::LsObjects expected_results_; }; -// TODO: Disable shouldfail when file:// or mem:// support is added. -TEST_CASE( - "VFS: Throwing FileFilter ls_recursive", - "[vfs][ls_recursive][!shouldfail]") { - std::string prefix = GENERATE("file://", "mem://"); +/** + * Represents a path used in the test. + * Encapsulates absolute and relative paths, and can be extended for URI + * if we determine that `ls_recursive` should output that instead. + */ +struct TestPath { + VFSTest& vfs_test; + std::filesystem::path relpath; + std::filesystem::path abspath; + uint64_t size; + + TestPath(VFSTest& vfs_test, std::string_view relpath, uint64_t size = 0) + : vfs_test(vfs_test) + , relpath(relpath) + , abspath( + std::filesystem::path(vfs_test.temp_dir_.to_path()).append(relpath)) + , size(size) { + } + + TestPath(const TestPath& copy) + : vfs_test(copy.vfs_test) + , relpath(copy.relpath) + , abspath(copy.abspath) + , size(copy.size) { + } + + /** + * Create a file at the test path. + * @param mkdirs if true, then also create each parent directory in the path + */ + void touch(bool mkdirs = false) { + if (mkdirs) { + std::vector parents; + + tiledb::sm::URI absuri(abspath.string()); + do { + absuri = absuri.parent_path(); + parents.push_back(absuri); + } while (absuri != vfs_test.temp_dir_); + + parents.pop_back(); /* temp_dir_ */ + while (!parents.empty()) { + REQUIRE(vfs_test.vfs_.create_dir(parents.back()).ok()); + parents.pop_back(); + } + } + REQUIRE(vfs_test.vfs_.touch(tiledb::sm::URI(abspath.string())).ok()); + std::filesystem::resize_file(abspath, size); + } + + void mkdir() { + REQUIRE(vfs_test.vfs_.create_dir(tiledb::sm::URI(abspath.string())).ok()); + } + + /** + * @return a string containing the way this is expected + * to appear in the ls_recursive output + */ + std::string lsresult() const { + return tiledb::sm::URI(abspath.string()).to_string(); + } + + bool matches(const std::pair& lsout) const { + return (lsresult() == lsout.first && size == lsout.second); + } +}; + +tiledb::sm::LsObjects sort_by_name(const tiledb::sm::LsObjects& in_objs) { + tiledb::sm::LsObjects out_objs(in_objs); + std::sort( + out_objs.begin(), out_objs.end(), [](const auto& f1, const auto& f2) { + return f1.first < f2.first; + }); + return out_objs; +} + +TEST_CASE("VFS: ls_recursive unfiltered", "[vfs][ls_recursive]") { + std::string prefix = GENERATE("file://"); + prefix += std::filesystem::current_path().string() + "/ls_recursive_test/"; + + VFSTest vfs_test({0}, prefix); + const auto mkst = vfs_test.mkdir(); + REQUIRE(mkst.ok()); + + std::vector testpaths = { + TestPath(vfs_test, "a1.txt", 30), + TestPath(vfs_test, "a2.txt", 40), + TestPath(vfs_test, "f1.txt", 10), + TestPath(vfs_test, "f2.txt", 20), + TestPath(vfs_test, "d1/f1.txt", 45), + TestPath(vfs_test, "d1/c1.txt", 55), + TestPath(vfs_test, "d1/d1sub1/d1sub1sub1/g1.txt", 33), + TestPath(vfs_test, "d1/d1sub1/d1sub1sub1/d1sub1sub1sub1/b1.txt", 12), + TestPath(vfs_test, "d1/d1sub1/d1sub1sub1/d1sub1sub1sub1/h1.txt", 33), + }; + + SECTION("Empty directory") { + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + CHECK(ls.empty()); + } + + SECTION("Files only") { + testpaths[0].touch(); + testpaths[1].touch(); + testpaths[2].touch(); + testpaths[3].touch(); + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + REQUIRE(ls.size() == 4); + CHECK(testpaths[0].matches(ls[0])); + CHECK(testpaths[1].matches(ls[1])); + CHECK(testpaths[2].matches(ls[2])); + CHECK(testpaths[3].matches(ls[3])); + } + + SECTION("Empty subdirectory") { + auto d1 = TestPath(vfs_test, "d1"); + d1.mkdir(); + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + + CHECK(ls.size() == 1); + if (ls.size() >= 1) { + CHECK(d1.matches(ls[0])); + } + } + + SECTION("Empty subdirectory and files") { + testpaths[0].touch(); + testpaths[1].touch(); + auto d1 = TestPath(vfs_test, "d1"); + d1.mkdir(); + testpaths[2].touch(); + testpaths[3].touch(); + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + CHECK(ls.size() == 5); + if (ls.size() >= 1) { + CHECK(testpaths[0].matches(ls[0])); + } + if (ls.size() >= 2) { + CHECK(testpaths[1].matches(ls[1])); + } + if (ls.size() >= 3) { + CHECK(d1.matches(ls[2])); + } + if (ls.size() >= 4) { + CHECK(testpaths[2].matches(ls[3])); + } + if (ls.size() >= 5) { + CHECK(testpaths[3].matches(ls[4])); + } + } + + SECTION("Empty sub-subdirectory") { + auto d1 = TestPath(vfs_test, "d1"); + auto d1sub1 = TestPath(vfs_test, "d1/d1sub1"); + d1.mkdir(); + d1sub1.mkdir(); + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + + CHECK(ls.size() == 2); + if (ls.size() >= 1) { + CHECK(d1.matches(ls[0])); + } + if (ls.size() >= 2) { + CHECK(d1sub1.matches(ls[1])); + } + } + + SECTION("Deeply-nested files") { + auto d1 = TestPath(vfs_test, "d1"); + auto d1sub1 = TestPath(vfs_test, "d1/d1sub1"); + auto d1sub1sub1 = TestPath(vfs_test, "d1/d1sub1/d1sub1sub1"); + auto d1sub1sub1sub1 = + TestPath(vfs_test, "d1/d1sub1/d1sub1sub1/d1sub1sub1sub1"); + d1.mkdir(); + d1sub1.mkdir(); + d1sub1sub1.mkdir(); + d1sub1sub1sub1.mkdir(); + testpaths[7].touch(); + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + + CHECK(ls.size() == 5); + if (ls.size() >= 1) { + CHECK(d1.matches(ls[0])); + } + if (ls.size() >= 2) { + CHECK(d1sub1.matches(ls[1])); + } + if (ls.size() >= 3) { + CHECK(d1sub1sub1.matches(ls[2])); + } + if (ls.size() >= 4) { + CHECK(d1sub1sub1sub1.matches(ls[3])); + } + if (ls.size() >= 5) { + CHECK(testpaths[7].matches(ls[4])); + } + } + + SECTION("Recursion") { + auto d1 = TestPath(vfs_test, "d1"); + auto d1sub1 = TestPath(vfs_test, "d1/d1sub1"); + auto d1sub1sub1 = TestPath(vfs_test, "d1/d1sub1/d1sub1sub1"); + auto d1sub1sub1sub1 = + TestPath(vfs_test, "d1/d1sub1/d1sub1sub1/d1sub1sub1sub1"); + d1.mkdir(); + d1sub1.mkdir(); + d1sub1sub1.mkdir(); + d1sub1sub1sub1.mkdir(); + for (unsigned i = 0; i < testpaths.size(); i++) { + testpaths[i].touch(); + } + + const auto ls = sort_by_name(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_all_files, + tiledb::sm::accept_all_dirs)); + CHECK(ls.size() == testpaths.size() + 4); + + if (ls.size() >= 1) { + CHECK(testpaths[0].matches(ls[0])); + } + if (ls.size() >= 2) { + CHECK(testpaths[1].matches(ls[1])); + } + if (ls.size() >= 3) { + CHECK(d1.matches(ls[2])); + } + if (ls.size() >= 4) { + CHECK(testpaths[5].matches(ls[3])); + } + if (ls.size() >= 5) { + CHECK(d1sub1.matches(ls[4])); + } + if (ls.size() >= 6) { + CHECK(d1sub1sub1.matches(ls[5])); + } + if (ls.size() >= 7) { + CHECK(d1sub1sub1sub1.matches(ls[6])); + } + if (ls.size() >= 8) { + CHECK(testpaths[7].matches(ls[7])); + } + if (ls.size() >= 9) { + CHECK(testpaths[8].matches(ls[8])); + } + if (ls.size() >= 10) { + CHECK(testpaths[6].matches(ls[9])); + } + if (ls.size() >= 11) { + CHECK(testpaths[4].matches(ls[10])); + } + if (ls.size() >= 12) { + CHECK(testpaths[2].matches(ls[11])); + } + if (ls.size() >= 13) { + CHECK(testpaths[3].matches(ls[12])); + } + } + + /* all tests must close all the files that they opened, in normal use of the + * API */ + REQUIRE(vfs_test.check_open_files()); +} + +TEST_CASE("VFS: ls_recursive file filter", "[vfs][ls_recursive]") { + std::string prefix = GENERATE("file://"); + prefix += std::filesystem::current_path().string() + "/ls_recursive_test/"; + + VFSTest vfs_test({0}, prefix); + const auto mkst = vfs_test.mkdir(); + REQUIRE(mkst.ok()); + + std::vector testpaths = { + TestPath(vfs_test, "year=2021/month=8/day=27/log1.txt", 30), + TestPath(vfs_test, "year=2021/month=8/day=27/log2.txt", 31), + TestPath(vfs_test, "year=2021/month=8/day=28/log1.txt", 40), + TestPath(vfs_test, "year=2021/month=8/day=28/log2.txt", 41), + TestPath(vfs_test, "year=2021/month=9/day=27/log1.txt", 50), + TestPath(vfs_test, "year=2021/month=9/day=27/log2.txt", 51), + TestPath(vfs_test, "year=2021/month=9/day=28/log1.txt", 60), + TestPath(vfs_test, "year=2021/month=9/day=28/log2.txt", 61), + TestPath(vfs_test, "year=2022/month=8/day=27/log1.txt", 70), + TestPath(vfs_test, "year=2022/month=8/day=27/log2.txt", 71), + TestPath(vfs_test, "year=2022/month=8/day=28/log1.txt", 80), + TestPath(vfs_test, "year=2022/month=8/day=28/log2.txt", 81), + TestPath(vfs_test, "year=2022/month=9/day=27/log1.txt", 90), + TestPath(vfs_test, "year=2022/month=9/day=27/log2.txt", 91), + TestPath(vfs_test, "year=2022/month=9/day=28/log1.txt", 20), + TestPath(vfs_test, "year=2022/month=9/day=28/log2.txt", 21), + }; + + SECTION("File predicate returning false is discarded from results") { + for (auto& testpath : testpaths) { + testpath.touch(true); + } + + /* + * This also shows us that the file filter is only called on the leaves, + * since "log1.txt" only appears in the basename component of the test + * paths. + */ + auto log_is_1 = [](const std::string_view& path, uint64_t) -> bool { + return (path.find("log1.txt") != std::string::npos); + }; + + auto ls = vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, log_is_1, tiledb::sm::accept_all_dirs); + + /* directories appear in the result set, we aren't interested in those, + * and the callback doesn't (yet?) have a way to descend into a directory + * without also including it in the result set */ + std::erase_if(ls, [](const auto& obj) { return obj.second == 0; }); + + CHECK(ls.size() == testpaths.size() / 2); + + ls = sort_by_name(ls); /* ensure order matches the testpaths order */ + + for (uint64_t i = 0; i < testpaths.size(); i += 2) { + CHECK(testpaths[i].matches(ls[i / 2])); + } + } +} + +TEST_CASE("VFS: ls_recursive directory filter", "[vfs][ls_recursive]") { + std::string prefix = GENERATE("file://"); + prefix += std::filesystem::current_path().string() + "/ls_recursive_test/"; + + VFSTest vfs_test({0}, prefix); + const auto mkst = vfs_test.mkdir(); + REQUIRE(mkst.ok()); + + std::vector testpaths = { + TestPath(vfs_test, "year=2021/month=8/day=27/log1.txt", 30), + TestPath(vfs_test, "year=2021/month=8/day=27/log2.txt", 31), + TestPath(vfs_test, "year=2021/month=8/day=28/log1.txt", 40), + TestPath(vfs_test, "year=2021/month=8/day=28/log2.txt", 41), + TestPath(vfs_test, "year=2021/month=9/day=28/log1.txt", 50), + TestPath(vfs_test, "year=2021/month=9/day=28/log2.txt", 51), + TestPath(vfs_test, "year=2021/month=9/day=29/log1.txt", 60), + TestPath(vfs_test, "year=2021/month=9/day=29/log2.txt", 61), + TestPath(vfs_test, "year=2022/month=8/day=27/log1.txt", 70), + TestPath(vfs_test, "year=2022/month=8/day=27/log2.txt", 71), + TestPath(vfs_test, "year=2022/month=8/day=28/log1.txt", 80), + TestPath(vfs_test, "year=2022/month=8/day=28/log2.txt", 81), + TestPath(vfs_test, "year=2022/month=9/day=28/log1.txt", 90), + TestPath(vfs_test, "year=2022/month=9/day=28/log2.txt", 91), + TestPath(vfs_test, "year=2022/month=9/day=29/log1.txt", 20), + TestPath(vfs_test, "year=2022/month=9/day=29/log2.txt", 21), + }; + + /* create all files and dirs */ + for (auto& testpath : testpaths) { + testpath.touch(true); + } + + SECTION("Directory predicate returning true is filtered from results") { + auto month_is_august = [](std::string_view dirname) -> bool { + if (dirname.find("month") == std::string::npos) { + /* haven't descended far enough yet */ + return true; + } else if (dirname.find("month=8") == std::string::npos) { + /* not august */ + return false; + } else { + return true; + } + }; + + auto ls = vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_only_regular_files, + month_is_august); + + /* directories appear in the result set, we aren't interested in those, + * and the callback doesn't (yet?) have a way to descend into a directory + * without also including it in the result set */ + std::erase_if(ls, [](const auto& obj) { return obj.second == 0; }); + + CHECK(ls.size() == testpaths.size() / 2); + + ls = sort_by_name(ls); /* ensure order matches the testpaths order */ + + CHECK(ls.size() == 8); + if (ls.size() >= 1) { + testpaths[0].matches(ls[0]); + } + if (ls.size() >= 2) { + testpaths[1].matches(ls[1]); + } + if (ls.size() >= 3) { + testpaths[2].matches(ls[2]); + } + if (ls.size() >= 4) { + testpaths[3].matches(ls[3]); + } + if (ls.size() >= 5) { + testpaths[8].matches(ls[4]); + } + if (ls.size() >= 6) { + testpaths[9].matches(ls[5]); + } + if (ls.size() >= 7) { + testpaths[10].matches(ls[6]); + } + if (ls.size() >= 8) { + testpaths[11].matches(ls[7]); + } + } + + /* note: this should be true for POSIX but is not for S3 without hierarchical + * list API */ + SECTION( + "Directory predicate returning true does not descend into directory") { + /* + * In the test data we only find "day=29" beneath "month=9", + * so the `ls` should throw with this directory filter if and only if + * we descend into directories with "month=9". + */ + std::string monthstr = "month=9"; + auto throw_if_day_is_29 = [&monthstr](std::string_view dirname) -> bool { + if (dirname.find("month") == std::string::npos) { + /* haven't descended far enough yet */ + return true; + } else if (dirname.find(monthstr) == std::string::npos) { + /* not august */ + return false; + } else if (dirname.find("day=29") == std::string::npos) { + /* not the 29th */ + return true; + } else { + /* it is the 29th, throw */ + throw std::logic_error("Throwing FileFilter: day=29"); + } + }; + + CHECK_THROWS_AS( + vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_only_regular_files, + throw_if_day_is_29), + std::logic_error); + CHECK_THROWS_WITH( + vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_only_regular_files, + throw_if_day_is_29), + Catch::Matchers::ContainsSubstring("Throwing FileFilter: day=29")); + + monthstr = "month=8"; + + /* now the result should be the same as the first section */ + auto ls = vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, + tiledb::sm::accept_only_regular_files, + throw_if_day_is_29); + + /* directories appear in the result set, we aren't interested in those, + * and the callback doesn't (yet?) have a way to descend into a directory + * without also including it in the result set */ + std::erase_if(ls, [](const auto& obj) { return obj.second == 0; }); + + CHECK(ls.size() == testpaths.size() / 2); + + ls = sort_by_name(ls); /* ensure order matches the testpaths order */ + + CHECK(ls.size() == 8); + if (ls.size() >= 1) { + testpaths[0].matches(ls[0]); + } + if (ls.size() >= 2) { + testpaths[1].matches(ls[1]); + } + if (ls.size() >= 3) { + testpaths[2].matches(ls[2]); + } + if (ls.size() >= 4) { + testpaths[3].matches(ls[3]); + } + if (ls.size() >= 5) { + testpaths[8].matches(ls[4]); + } + if (ls.size() >= 6) { + testpaths[9].matches(ls[5]); + } + if (ls.size() >= 7) { + testpaths[10].matches(ls[6]); + } + if (ls.size() >= 8) { + testpaths[11].matches(ls[7]); + } + } + + /* + * Note that since we throw in the previous section, this check + * demonstrates that all directories are closed whether or not we return + * from ls_recursive normally + */ + REQUIRE(vfs_test.check_open_files()); +} + +TEST_CASE("VFS: Throwing FileFilter ls_recursive", "[vfs][ls_recursive]") { + std::string prefix = GENERATE("file://"); prefix += std::filesystem::current_path().string() + "/ls_filtered_test"; VFSTest vfs_test({0}, prefix); - auto file_filter = [](const std::string_view&, uint64_t) -> bool { + const auto mkst = vfs_test.mkdir(); + REQUIRE(mkst.ok()); + + auto always_throw_filter = [](const std::string_view&, uint64_t) -> bool { throw std::logic_error("Throwing FileFilter"); }; SECTION("Throwing FileFilter with 0 objects should not throw") { CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( - vfs_test.temp_dir_, file_filter, tiledb::sm::accept_all_dirs)); + vfs_test.temp_dir_, always_throw_filter, tiledb::sm::accept_all_dirs)); + } + SECTION( + "Throwing FileFilter will not throw if ls_recursive only visits " + "directories") { } SECTION("Throwing FileFilter with N objects should throw") { vfs_test.vfs_.touch(vfs_test.temp_dir_.join_path("file")).ok(); CHECK_THROWS_AS( - vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, always_throw_filter), std::logic_error); CHECK_THROWS_WITH( - vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, file_filter), + vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, always_throw_filter), Catch::Matchers::ContainsSubstring("Throwing FileFilter")); } + + /* all tests must close all the files that they opened, regardless of + * exception behavior */ + REQUIRE(vfs_test.check_open_files()); } TEST_CASE( "VFS: ls_recursive throws for unsupported filesystems", "[vfs][ls_recursive]") { - std::string prefix = GENERATE("file://", "mem://"); + std::string prefix = GENERATE("mem://"); prefix += std::filesystem::current_path().string() + "/ls_filtered_test"; VFSTest vfs_test({1}, prefix); @@ -118,7 +693,9 @@ TEST_CASE( DYNAMIC_SECTION(backend << " unsupported backend should throw") { CHECK_THROWS_WITH( vfs_test.vfs_.ls_recursive( - vfs_test.temp_dir_, VFSTest::accept_all_files), + vfs_test.temp_dir_, tiledb::sm::accept_all_files), Catch::Matchers::ContainsSubstring("storage backend is not supported")); } + + REQUIRE(vfs_test.check_open_files()); } diff --git a/tiledb/sm/filesystem/uri.cc b/tiledb/sm/filesystem/uri.cc index 0c7823387850..a0efc94aff0f 100644 --- a/tiledb/sm/filesystem/uri.cc +++ b/tiledb/sm/filesystem/uri.cc @@ -63,7 +63,7 @@ URI::URI(const char* path) : URI((path == nullptr) ? std::string("") : std::string(path)) { } -URI::URI(const std::string& path) { +URI::URI(std::string_view path) { if (path.empty()) uri_ = ""; else if (URI::is_file(path)) @@ -76,7 +76,7 @@ URI::URI(const std::string& path) { uri_ = ""; } -URI::URI(const std::string& path, const bool& get_abs) { +URI::URI(std::string_view path, const bool& get_abs) { if (path.empty()) { uri_ = ""; } else if (URI::is_file(path)) { @@ -132,7 +132,7 @@ bool URI::is_invalid() const { return uri_.empty(); } -bool URI::is_file(const std::string& path) { +bool URI::is_file(std::string_view path) { #ifdef _WIN32 return utils::parse::starts_with(path, "file://") || path.find("://") == std::string::npos; @@ -142,7 +142,7 @@ bool URI::is_file(const std::string& path) { #endif } -bool URI::contains(const std::string_view& str) const { +bool URI::contains(std::string_view str) const { return uri_.find(str, 0) != std::string::npos; } @@ -151,13 +151,13 @@ bool URI::is_file() const { return is_file(uri_); #else // Observed: semantics here differ from sibling - // is_file(const std::string& path), here is missing + // is_file(std::string_view path), here is missing // additional check using "://". return utils::parse::starts_with(uri_, "file:///"); #endif } -bool URI::is_hdfs(const std::string& path) { +bool URI::is_hdfs(std::string_view path) { return utils::parse::starts_with(path, "hdfs://"); } @@ -165,7 +165,7 @@ bool URI::is_hdfs() const { return utils::parse::starts_with(uri_, "hdfs://"); } -bool URI::is_s3(const std::string& path) { +bool URI::is_s3(std::string_view path) { return utils::parse::starts_with(path, "s3://") || utils::parse::starts_with(path, "http://") || utils::parse::starts_with(path, "https://"); @@ -177,7 +177,7 @@ bool URI::is_s3() const { utils::parse::starts_with(uri_, "https://"); } -bool URI::is_azure(const std::string& path) { +bool URI::is_azure(std::string_view path) { return utils::parse::starts_with(path, "azure://"); } @@ -185,7 +185,7 @@ bool URI::is_azure() const { return utils::parse::starts_with(uri_, "azure://"); } -bool URI::is_gcs(const std::string& path) { +bool URI::is_gcs(std::string_view path) { return utils::parse::starts_with(path, "gcs://") || utils::parse::starts_with(path, "gs://"); } @@ -195,7 +195,7 @@ bool URI::is_gcs() const { utils::parse::starts_with(uri_, "gs://"); } -bool URI::is_memfs(const std::string& path) { +bool URI::is_memfs(std::string_view path) { return utils::parse::starts_with(path, "mem://"); } @@ -203,7 +203,7 @@ bool URI::is_memfs() const { return utils::parse::starts_with(uri_, "mem://"); } -bool URI::is_tiledb(const std::string& path) { +bool URI::is_tiledb(std::string_view path) { return utils::parse::starts_with(path, "tiledb://"); } @@ -341,6 +341,10 @@ std::string URI::to_string() const { return uri_; } +URI::operator std::string_view() const noexcept { + return std::string_view(uri_); +} + bool URI::operator==(const URI& uri) const { return uri_ == uri.uri_; } diff --git a/tiledb/sm/filesystem/uri.h b/tiledb/sm/filesystem/uri.h index 35974859d18c..a9e2e6e8af98 100644 --- a/tiledb/sm/filesystem/uri.h +++ b/tiledb/sm/filesystem/uri.h @@ -79,7 +79,7 @@ class URI { * @param path String that gets converted into an absolute path and stored * as a URI. */ - explicit URI(const std::string& path); + explicit URI(std::string_view path); /** * Constructor. @@ -87,7 +87,7 @@ class URI { * @param path * @param get_abs should local files become absolute */ - explicit URI(const std::string& path, const bool& get_abs); + explicit URI(std::string_view path, const bool& get_abs); /** Destructor. */ ~URI(); @@ -125,7 +125,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_file(const std::string& path); + static bool is_file(std::string_view path); /** * Checks if the URI is file. @@ -140,7 +140,7 @@ class URI { * @param str the string to search for in the URI * @return The result of the check. */ - bool contains(const std::string_view& str) const; + bool contains(std::string_view str) const; /** * Checks if the input path is HDFS. @@ -148,7 +148,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_hdfs(const std::string& path); + static bool is_hdfs(std::string_view path); /** * Checks if the URI is HDFS. @@ -163,7 +163,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_s3(const std::string& path); + static bool is_s3(std::string_view path); /** * Checks if the URI is S3. @@ -178,7 +178,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_azure(const std::string& path); + static bool is_azure(std::string_view path); /** * Checks if the URI is Azure. @@ -193,7 +193,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_gcs(const std::string& path); + static bool is_gcs(std::string_view path); /** * Checks if the URI is gcs. @@ -208,7 +208,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_memfs(const std::string& path); + static bool is_memfs(std::string_view path); /** * Checks if the URI is mem. @@ -223,7 +223,7 @@ class URI { * @param path The path to be checked. * @return The result of the check. */ - static bool is_tiledb(const std::string& path); + static bool is_tiledb(std::string_view path); /** * Checks if the URI is TileDB. @@ -317,6 +317,8 @@ class URI { /** For comparing URIs alphanumerically. */ bool operator>(const URI& uri) const; + operator std::string_view() const noexcept; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 4d24aad32f20..3b3295731922 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -118,12 +118,13 @@ VFS::VFS( /* API */ /* ********************************* */ -std::string VFS::abs_path(const std::string& path) { +std::string VFS::abs_path(std::string_view path) { // workaround for older clang (llvm 3.5) compilers (issue #828) - std::string path_copy = path; + std::string path_copy(path); #ifdef _WIN32 { - std::string norm_sep_path = path_win::slashes_to_backslashes(path); + std::string norm_sep_path = + path_win::slashes_to_backslashes(std::string(path)); if (path_win::is_win_path(norm_sep_path)) return path_win::uri_from_path(Win::abs_path(norm_sep_path)); else if (URI::is_file(path)) diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 08798df4c1ce..1281ad82aead 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -332,7 +332,7 @@ class VFS : private VFSBase, protected S3_within_VFS { * @param path The input path. * @return The string with the absolute path. */ - static std::string abs_path(const std::string& path); + static std::string abs_path(std::string_view path); /** * Return a config object containing the VFS parameters. All other non-VFS @@ -514,7 +514,7 @@ class VFS : private VFSBase, protected S3_within_VFS { * the FilePredicate on each entry collected and the DirectoryPredicate on * common prefixes for pruning. * - * Currently only S3 is supported for ls_recursive. + * Currently this API is only supported for Posix and S3. * * @param parent The parent prefix to list sub-paths. * @param f The FilePredicate to invoke on each object for filtering. @@ -530,11 +530,42 @@ class VFS : private VFSBase, protected S3_within_VFS { [[maybe_unused]] D d = accept_all_dirs) const { LsObjects results; try { - if (parent.is_s3()) { + if (parent.is_file()) { +#ifdef _WIN32 + results = win_.ls_filtered(parent, f, d, true); +#else + results = posix_.ls_filtered(parent, f, d, true); +#endif + } else if (parent.is_s3()) { #ifdef HAVE_S3 results = s3().ls_filtered(parent, f, d, true); #else throw filesystem::VFSException("TileDB was built without S3 support"); +#endif + } else if (parent.is_gcs()) { +#ifdef HAVE_GCS + throw filesystem::VFSException( + "Recursive ls over " + parent.backend_name() + + " storage backend is not supported."); +#else + throw filesystem::VFSException("TileDB was built without GCS support"); +#endif + } else if (parent.is_azure()) { +#ifdef HAVE_AZURE + throw filesystem::VFSException( + "Recursive ls over " + parent.backend_name() + + " storage backend is not supported."); +#else + throw filesystem::VFSException( + "TileDB was built without Azure support"); +#endif + } else if (parent.is_hdfs()) { +#ifdef HAVE_HDFS + throw filesystem::VFSException( + "Recursive ls over " + parent.backend_name() + + " storage backend is not supported."); +#else + throw filesystem::VFSException("TileDB was built without HDFS support"); #endif } else { throw filesystem::VFSException( diff --git a/tiledb/sm/filesystem/win.h b/tiledb/sm/filesystem/win.h index f4732d304db8..e5d3ded22069 100644 --- a/tiledb/sm/filesystem/win.h +++ b/tiledb/sm/filesystem/win.h @@ -42,6 +42,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" +#include "tiledb/sm/filesystem/ls_scanner.h" using namespace tiledb::common; @@ -163,6 +164,29 @@ class Win { tuple>> ls_with_sizes(const URI& path) const; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + * + * Note: the return type LsObjects does not match the other "ls" methods so as + * to match the S3 equivalent API. + */ + template + LsObjects ls_filtered( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false) const { + return std_filesystem_ls_filtered(parent, f, d, recursive); + } + /** * Move a given filesystem path. * From 607d5ad92783dbaa094990ec420b672109fbf730 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 11 Mar 2024 14:16:25 +0100 Subject: [PATCH 237/456] Implement dense dimension aggregates. (#4801) This implements aggregates support with dense dimensions. --- TYPE: IMPROVEMENT DESC: Implement dense dimension aggregates. --- test/src/test-cppapi-aggregates.cc | 716 ++++++++---------- .../query_aggregate/query_aggregate_api.cc | 19 + tiledb/sm/array_schema/domain.cc | 8 +- tiledb/sm/array_schema/domain.h | 5 +- tiledb/sm/fragment/fragment_metadata.cc | 3 +- .../aggregators/aggregate_with_count.h | 39 +- .../sm/query/readers/aggregators/field_info.h | 37 + tiledb/sm/query/readers/dense_reader.cc | 140 ++-- tiledb/sm/query/readers/dense_reader.h | 7 + .../query/readers/sparse_index_reader_base.cc | 8 +- tiledb/sm/subarray/subarray.cc | 34 +- 11 files changed, 545 insertions(+), 471 deletions(-) diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 7efa33159175..5d10e7c7e6b6 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -62,8 +62,11 @@ struct CppAggregatesFx { bool set_ranges_; bool set_qc_; bool use_dim_; + std::string dim_name_; tiledb_layout_t layout_; std::vector set_qc_values_; + std::vector use_dim_values_; + std::vector dim_name_values_; std::vector layout_values_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; @@ -75,6 +78,7 @@ struct CppAggregatesFx { // Functions. void generate_test_params(); + void run_all_combinations(std::function fn); void create_dense_array(bool var = false, bool encrypt = false); void create_sparse_array(bool var = false, bool encrypt = false); void write_sparse( @@ -122,6 +126,7 @@ struct CppAggregatesFx { std::vector& dim2, std::vector& a1, std::vector& a1_validity, + std::function aggregate_fn, const bool validate_count = true); void validate_data_var( Query& query, @@ -162,7 +167,12 @@ void CppAggregatesFx::generate_test_params() { allow_dups_ = false; set_qc_values_ = {true, false}; layout_values_ = {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR, TILEDB_GLOBAL_ORDER}; - use_dim_ = false; + use_dim_values_ = {true, false}; + dim_name_values_ = {"d2"}; + if (nullable_ || !std::is_same::value) { + use_dim_values_ = {false}; + dim_name_values_ = {"d1"}; + } } SECTION("sparse") { @@ -171,8 +181,34 @@ void CppAggregatesFx::generate_test_params() { allow_dups_ = GENERATE(true, false); set_qc_values_ = {false}; layout_values_ = {TILEDB_UNORDERED}; - use_dim_ = - GENERATE(false, true) && !nullable_ && std::is_same::value; + use_dim_values_ = {true, false}; + if (nullable_ || !std::is_same::value) { + use_dim_values_ = {false}; + } + dim_name_values_ = {"d1"}; + } +} + +template +void CppAggregatesFx::run_all_combinations(std::function fn) { + for (bool use_dim : use_dim_values_) { + use_dim_ = use_dim; + for (std::string dim_name : dim_name_values_) { + dim_name_ = dim_name; + for (bool set_ranges : {true, false}) { + set_ranges_ = set_ranges; + for (bool request_data : {true, false}) { + request_data_ = request_data; + for (bool set_qc : set_qc_values_) { + set_qc_ = set_qc; + for (tiledb_layout_t layout : layout_values_) { + layout_ = layout; + fn(); + } + } + } + } + } } } @@ -775,6 +811,7 @@ void CppAggregatesFx::validate_data( std::vector& dim2, std::vector& a1, std::vector& a1_validity, + std::function aggregate_fn, const bool validate_count) { uint64_t expected_count = 0; std::vector expected_dim1; @@ -903,25 +940,50 @@ void CppAggregatesFx::validate_data( CppAggregatesFx::STRING_CELL_VAL_NUM : 1; - dim1.resize(expected_dim1.size()); - dim2.resize(expected_dim2.size()); - a1.resize(expected_a1.size()); - CHECK(dim1 == expected_dim1); - CHECK(dim2 == expected_dim2); - CHECK(a1 == expected_a1); + if (request_data_) { + dim1.resize(expected_dim1.size()); + dim2.resize(expected_dim2.size()); + a1.resize(expected_a1.size()); + CHECK(dim1 == expected_dim1); + CHECK(dim2 == expected_dim2); + CHECK(a1 == expected_a1); - if (nullable_) { - a1_validity.resize(expected_a1_validity.size()); - CHECK(a1_validity == expected_a1_validity); + if (nullable_) { + a1_validity.resize(expected_a1_validity.size()); + CHECK(a1_validity == expected_a1_validity); + } + + if (validate_count) { + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["d1"]) == expected_count); + CHECK(std::get<1>(result_el["d2"]) == expected_count); + CHECK(std::get<1>(result_el["a1"]) == expected_count * cell_val_num); + if (nullable_) { + CHECK(std::get<2>(result_el["a1"]) == expected_count); + } + } } - if (validate_count) { - auto result_el = query.result_buffer_elements_nullable(); - CHECK(std::get<1>(result_el["d1"]) == expected_count); - CHECK(std::get<1>(result_el["d2"]) == expected_count); - CHECK(std::get<1>(result_el["a1"]) == expected_count * cell_val_num); - if (nullable_) { - CHECK(std::get<2>(result_el["a1"]) == expected_count); + // Call the aggregate function for all expected values. + if (use_dim_ && dim_name_ == "d1") { + for (uint64_t i = 0; i < expected_a1_int.size(); i++) { + if (!set_qc_ || (expected_a1_int[i] != 4 && expected_a1_int[i] != 35)) { + aggregate_fn(expected_dim1[i]); + } + } + } else if (use_dim_ && dim_name_ == "d2") { + for (uint64_t i = 0; i < expected_a1_int.size(); i++) { + if (!set_qc_ || (expected_a1_int[i] != 4 && expected_a1_int[i] != 35)) { + aggregate_fn(expected_dim2[i]); + } + } + } else { + for (uint64_t i = 0; i < expected_a1_int.size(); i++) { + if (!set_qc_ || (expected_a1_int[i] != 4 && expected_a1_int[i] != 35)) { + if (!nullable_ || expected_a1_validity[i] == 1) { + aggregate_fn(expected_a1_int[i]); + } + } } } } @@ -1176,11 +1238,11 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { } else if (set_ranges_) { // If we request range, we split all tiles, we'll have to read all instead // of using fragment metadata. - expected_num_tiles_read = is_count ? 0 : 5; + expected_num_tiles_read = is_count || use_dim_ ? 0 : 5; } else { // One space tile has two result tiles, we'll have to read them instead of // using fragment metadata. - expected_num_tiles_read = is_count ? 0 : 2; + expected_num_tiles_read = is_count || use_dim_ ? 0 : 2; } } else { if (request_data_) { @@ -1199,9 +1261,9 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { // we read 2 dims * 4 tiles. For the attribute, we can process 2 tiles // with duplicates and 1 without using the fragment metadata. if (allow_dups_) { - expected_num_tiles_read = is_count ? 8 : 10; + expected_num_tiles_read = use_dim_ || is_count ? 8 : 10; } else { - expected_num_tiles_read = is_count ? 8 : 11; + expected_num_tiles_read = use_dim_ || is_count ? 8 : 11; } } else { if (allow_dups_) { @@ -1212,7 +1274,7 @@ void CppAggregatesFx::validate_tiles_read(Query& query, bool is_count) { // Arrays without duplicates need to run deduplication, so we read the // dimension tiles (2 dims * 5 tiles). Only one tile for the attribute // can be processed with fragment metadata only. - expected_num_tiles_read = is_count ? 10 : 14; + expected_num_tiles_read = use_dim_ || is_count ? 10 : 14; } } } @@ -1495,7 +1557,8 @@ TEST_CASE_METHOD( CHECK(count[0] == expected_count); if (request_data) { - validate_data(query, dim1, dim2, a1, a1_validity); + validate_data( + query, dim1, dim2, a1, a1_validity, [&](uint64_t) -> void {}); } validate_tiles_read(query, true); @@ -1532,104 +1595,63 @@ TEMPLATE_LIST_TEST_CASE_METHOD( Array array{ CppAggregatesFx::ctx_, CppAggregatesFx::ARRAY_NAME, TILEDB_READ}; - for (bool set_ranges : {true, false}) { - CppAggregatesFx::set_ranges_ = set_ranges; - CppAggregatesFx::use_dim_ = CppAggregatesFx::use_dim_ && !set_ranges; - for (bool request_data : {true, false}) { - CppAggregatesFx::request_data_ = request_data; - for (bool set_qc : CppAggregatesFx::set_qc_values_) { - CppAggregatesFx::set_qc_ = set_qc; - for (tiledb_layout_t layout : CppAggregatesFx::layout_values_) { - CppAggregatesFx::layout_ = layout; - Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - - // Add a sum aggregator to the query. - QueryChannel default_channel = - QueryExperimental::get_default_channel(query); - ChannelOperation operation = - QueryExperimental::create_unary_aggregate( - query, CppAggregatesFx::use_dim_ ? "d1" : "a1"); - default_channel.apply_aggregate("Sum", operation); - - CppAggregatesFx::set_ranges_and_condition_if_needed( - array, query, false); - - // Set the data buffer for the aggregator. - uint64_t cell_size = sizeof(T); - std::vector::sum_type> sum(1); - std::vector sum_validity(1); - std::vector dim1(100); - std::vector dim2(100); - std::vector a1(100 * cell_size); - std::vector a1_validity(100); - query.set_layout(layout); - query.set_data_buffer("Sum", sum); - if (CppAggregatesFx::nullable_) { - query.set_validity_buffer("Sum", sum_validity); - } - - if (request_data) { - query.set_data_buffer("d1", dim1); - query.set_data_buffer("d2", dim2); - query.set_data_buffer( - "a1", static_cast(a1.data()), a1.size() / cell_size); + CppAggregatesFx::run_all_combinations([&]() -> void { + Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); + + // Add a sum aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, + CppAggregatesFx::use_dim_ ? CppAggregatesFx::dim_name_ : + "a1"); + default_channel.apply_aggregate("Sum", operation); + + CppAggregatesFx::set_ranges_and_condition_if_needed(array, query, false); + + // Set the data buffer for the aggregator. + uint64_t cell_size = sizeof(T); + std::vector::sum_type> sum(1); + std::vector sum_validity(1); + std::vector dim1(100); + std::vector dim2(100); + std::vector a1(100 * cell_size); + std::vector a1_validity(100); + query.set_layout(CppAggregatesFx::layout_); + query.set_data_buffer("Sum", sum); + if (CppAggregatesFx::nullable_) { + query.set_validity_buffer("Sum", sum_validity); + } - if (CppAggregatesFx::nullable_) { - query.set_validity_buffer("a1", a1_validity); - } - } + if (CppAggregatesFx::request_data_) { + query.set_data_buffer("d1", dim1); + query.set_data_buffer("d2", dim2); + query.set_data_buffer( + "a1", static_cast(a1.data()), a1.size() / cell_size); - // Submit the query. - query.submit(); + if (CppAggregatesFx::nullable_) { + query.set_validity_buffer("a1", a1_validity); + } + } - // Check the results. - typename tiledb::sm::sum_type_data::sum_type expected_sum; - if (CppAggregatesFx::use_dim_) { - expected_sum = 499500; - } else if (CppAggregatesFx::dense_) { - if (CppAggregatesFx::nullable_) { - if (set_ranges) { - expected_sum = set_qc ? 197 : 201; - } else { - expected_sum = set_qc ? 315 : 319; - } - } else { - if (set_ranges) { - expected_sum = set_qc ? 398 : 402; - } else { - expected_sum = set_qc ? 591 : 630; - } - } - } else { - if (CppAggregatesFx::nullable_) { - if (set_ranges) { - expected_sum = 42; - } else { - expected_sum = 56; - } - } else { - if (set_ranges) { - expected_sum = CppAggregatesFx::allow_dups_ ? 88 : 81; - } else { - expected_sum = CppAggregatesFx::allow_dups_ ? 120 : 113; - } - } - } + // Submit the query. + query.submit(); - auto result_el = query.result_buffer_elements_nullable(); - CHECK(std::get<1>(result_el["Sum"]) == 1); - CHECK(sum[0] == expected_sum); + // Check the results. + uint64_t expected = 0; + CppAggregatesFx::validate_data( + query, dim1, dim2, a1, a1_validity, [&](uint64_t v) -> void { + expected += v; + }); + typename tiledb::sm::sum_type_data::sum_type expected_sum = expected; - if (request_data) { - CppAggregatesFx::validate_data( - query, dim1, dim2, a1, a1_validity); - } + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Sum"]) == 1); + CHECK(sum[0] == expected_sum); - CppAggregatesFx::validate_tiles_read(query); - } - } - } - } + CppAggregatesFx::validate_tiles_read(query); + }); // Close array. array.close(); @@ -1659,106 +1681,66 @@ TEMPLATE_LIST_TEST_CASE_METHOD( Array array{ CppAggregatesFx::ctx_, CppAggregatesFx::ARRAY_NAME, TILEDB_READ}; - for (bool set_ranges : {true, false}) { - CppAggregatesFx::set_ranges_ = set_ranges; - CppAggregatesFx::use_dim_ = CppAggregatesFx::use_dim_ && !set_ranges; - for (bool request_data : {true, false}) { - CppAggregatesFx::request_data_ = request_data; - for (bool set_qc : CppAggregatesFx::set_qc_values_) { - CppAggregatesFx::set_qc_ = set_qc; - for (tiledb_layout_t layout : CppAggregatesFx::layout_values_) { - CppAggregatesFx::layout_ = layout; - Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - - QueryChannel default_channel = - QueryExperimental::get_default_channel(query); - ChannelOperation operation = - QueryExperimental::create_unary_aggregate( - query, CppAggregatesFx::use_dim_ ? "d1" : "a1"); - default_channel.apply_aggregate("Mean", operation); - - CppAggregatesFx::set_ranges_and_condition_if_needed( - array, query, false); - - // Set the data buffer for the aggregator. - uint64_t cell_size = sizeof(T); - std::vector mean(1); - std::vector mean_validity(1); - std::vector dim1(100); - std::vector dim2(100); - std::vector a1(100 * cell_size); - std::vector a1_validity(100); - query.set_layout(layout); - query.set_data_buffer("Mean", mean); - if (CppAggregatesFx::nullable_) { - query.set_validity_buffer("Mean", mean_validity); - } + CppAggregatesFx::run_all_combinations([&]() -> void { + Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); + + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, + CppAggregatesFx::use_dim_ ? CppAggregatesFx::dim_name_ : + "a1"); + default_channel.apply_aggregate("Mean", operation); + + CppAggregatesFx::set_ranges_and_condition_if_needed(array, query, false); + + // Set the data buffer for the aggregator. + uint64_t cell_size = sizeof(T); + std::vector mean(1); + std::vector mean_validity(1); + std::vector dim1(100); + std::vector dim2(100); + std::vector a1(100 * cell_size); + std::vector a1_validity(100); + query.set_layout(CppAggregatesFx::layout_); + query.set_data_buffer("Mean", mean); + if (CppAggregatesFx::nullable_) { + query.set_validity_buffer("Mean", mean_validity); + } - if (request_data) { - query.set_data_buffer("d1", dim1); - query.set_data_buffer("d2", dim2); - query.set_data_buffer( - "a1", static_cast(a1.data()), a1.size() / cell_size); + if (CppAggregatesFx::request_data_) { + query.set_data_buffer("d1", dim1); + query.set_data_buffer("d2", dim2); + query.set_data_buffer( + "a1", static_cast(a1.data()), a1.size() / cell_size); - if (CppAggregatesFx::nullable_) { - query.set_validity_buffer("a1", a1_validity); - } - } + if (CppAggregatesFx::nullable_) { + query.set_validity_buffer("a1", a1_validity); + } + } - // Submit the query. - query.submit(); + // Submit the query. + query.submit(); - // Check the results. - double expected_mean; - if (CppAggregatesFx::use_dim_) { - expected_mean = 500; - } else if (CppAggregatesFx::dense_) { - if (CppAggregatesFx::nullable_) { - if (set_ranges) { - expected_mean = set_qc ? (197.0 / 11.0) : (201.0 / 12.0); - } else { - expected_mean = set_qc ? (315.0 / 18.0) : (319.0 / 19.0); - } - } else { - if (set_ranges) { - expected_mean = set_qc ? (398.0 / 23.0) : (402.0 / 24.0); - } else { - expected_mean = set_qc ? (591.0 / 34.0) : (630.0 / 36.0); - } - } - } else { - if (CppAggregatesFx::nullable_) { - if (set_ranges) { - expected_mean = (42.0 / 4.0); - } else { - expected_mean = (56.0 / 8.0); - } - } else { - if (set_ranges) { - expected_mean = CppAggregatesFx::allow_dups_ ? (88.0 / 8.0) : - (81.0 / 7.0); - } else { - expected_mean = CppAggregatesFx::allow_dups_ ? - (120.0 / 16.0) : - (113.0 / 15.0); - } - } - } + // Check the results. + uint64_t expected_sum = 0; + uint64_t expected_count = 0; + CppAggregatesFx::validate_data( + query, dim1, dim2, a1, a1_validity, [&](uint64_t v) -> void { + expected_sum += v; + expected_count++; + }); - auto result_el = query.result_buffer_elements_nullable(); - CHECK(std::get<1>(result_el["Mean"]) == 1); - CHECK(mean[0] == expected_mean); + double expected_mean = + static_cast(expected_sum) / static_cast(expected_count); - if (request_data) { - CppAggregatesFx::validate_data( - query, dim1, dim2, a1, a1_validity); - } + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["Mean"]) == 1); + CHECK(mean[0] == expected_mean); - CppAggregatesFx::validate_tiles_read(query); - } - } - } - } + CppAggregatesFx::validate_tiles_read(query); + }); // Close array. array.close(); @@ -1801,118 +1783,81 @@ TEMPLATE_LIST_TEST_CASE( Array array{fx.ctx_, fx.ARRAY_NAME, TILEDB_READ}; - for (bool set_ranges : {true, false}) { - fx.set_ranges_ = set_ranges; - fx.use_dim_ = fx.use_dim_ && !set_ranges; - for (bool request_data : {true, false}) { - fx.request_data_ = request_data; - for (bool set_qc : fx.set_qc_values_) { - fx.set_qc_ = set_qc; - for (tiledb_layout_t layout : fx.layout_values_) { - fx.layout_ = layout; - Query query(fx.ctx_, array, TILEDB_READ); - - // Add a min/max aggregator to the query. - QueryChannel default_channel = - QueryExperimental::get_default_channel(query); - ChannelOperation operation = - QueryExperimental::create_unary_aggregate( - query, fx.use_dim_ ? "d1" : "a1"); - default_channel.apply_aggregate("MinMax", operation); - - fx.set_ranges_and_condition_if_needed(array, query, false); - - // Set the data buffer for the aggregator. - uint64_t cell_size = std::is_same::value ? - fx.STRING_CELL_VAL_NUM : - sizeof(T); - std::vector min_max(cell_size); - std::vector min_max_validity(1); - std::vector dim1(100); - std::vector dim2(100); - std::vector a1(100 * cell_size); - std::vector a1_validity(100); - query.set_layout(layout); - if constexpr (std::is_same::value) { - query.set_data_buffer( - "MinMax", - static_cast(static_cast(min_max.data())), - min_max.size()); - } else { - query.set_data_buffer( - "MinMax", - static_cast(static_cast(min_max.data())), - min_max.size() / cell_size); - } - if (fx.nullable_) { - query.set_validity_buffer("MinMax", min_max_validity); - } + fx.run_all_combinations([&]() -> void { + Query query(fx.ctx_, array, TILEDB_READ); + + // Add a min/max aggregator to the query. + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = QueryExperimental::create_unary_aggregate( + query, fx.use_dim_ ? fx.dim_name_ : "a1"); + default_channel.apply_aggregate("MinMax", operation); + + fx.set_ranges_and_condition_if_needed(array, query, false); + + // Set the data buffer for the aggregator. + uint64_t cell_size = std::is_same::value ? + fx.STRING_CELL_VAL_NUM : + sizeof(T); + std::vector min_max(cell_size); + std::vector min_max_validity(1); + std::vector dim1(100); + std::vector dim2(100); + std::vector a1(100 * cell_size); + std::vector a1_validity(100); + query.set_layout(fx.layout_); + if constexpr (std::is_same::value) { + query.set_data_buffer( + "MinMax", + static_cast(static_cast(min_max.data())), + min_max.size()); + } else { + query.set_data_buffer( + "MinMax", + static_cast(static_cast(min_max.data())), + min_max.size() / cell_size); + } + if (fx.nullable_) { + query.set_validity_buffer("MinMax", min_max_validity); + } - if (request_data) { - query.set_data_buffer("d1", dim1); - query.set_data_buffer("d2", dim2); - query.set_data_buffer( - "a1", static_cast(a1.data()), a1.size() / cell_size); + if (fx.request_data_) { + query.set_data_buffer("d1", dim1); + query.set_data_buffer("d2", dim2); + query.set_data_buffer( + "a1", static_cast(a1.data()), a1.size() / cell_size); - if (fx.nullable_) { - query.set_validity_buffer("a1", a1_validity); - } - } + if (fx.nullable_) { + query.set_validity_buffer("a1", a1_validity); + } + } - // Submit the query. - query.submit(); + // Submit the query. + query.submit(); - // Check the results. - std::vector expected_min_max; - if (fx.use_dim_) { - expected_min_max = fx.make_data_buff({min ? 1 : 999}); - } else if (fx.dense_) { - if (fx.nullable_) { - if (set_ranges) { - expected_min_max = - fx.make_data_buff({min ? (set_qc ? 6 : 4) : 28}); - } else { - expected_min_max = fx.make_data_buff({min ? 0 : 34}); - } - } else { - if (set_ranges) { - expected_min_max = fx.make_data_buff({min ? 3 : 29}); - } else { - expected_min_max = - fx.make_data_buff({min ? 0 : (set_qc ? 34 : 35)}); - } - } + // Check the results. + uint64_t expected = min ? std::numeric_limits::max() : + std::numeric_limits::min(); + fx.validate_data( + query, dim1, dim2, a1, a1_validity, [&](uint64_t v) -> void { + if (min) { + expected = std::min(expected, v); } else { - if (fx.nullable_) { - if (set_ranges) { - expected_min_max = fx.make_data_buff({min ? 6 : 14}); - } else { - expected_min_max = fx.make_data_buff({min ? 0 : 14}); - } - } else { - if (set_ranges) { - expected_min_max = fx.make_data_buff({min ? 6 : 15}); - } else { - expected_min_max = fx.make_data_buff({min ? 0 : 15}); - } - } + expected = std::max(expected, v); } + }); - auto result_el = query.result_buffer_elements_nullable(); - CHECK( - std::get<1>(result_el["MinMax"]) == - (std::is_same::value ? 2 : 1)); - CHECK(min_max == expected_min_max); + std::vector expected_min_max = + fx.make_data_buff({static_cast(expected)}); - if (request_data) { - fx.validate_data(query, dim1, dim2, a1, a1_validity); - } + auto result_el = query.result_buffer_elements_nullable(); + CHECK( + std::get<1>(result_el["MinMax"]) == + (std::is_same::value ? 2 : 1)); + CHECK(min_max == expected_min_max); - fx.validate_tiles_read(query); - } - } - } - } + fx.validate_tiles_read(query); + }); // Close array. array.close(); @@ -2068,84 +2013,75 @@ TEMPLATE_LIST_TEST_CASE_METHOD( Array array{ CppAggregatesFx::ctx_, CppAggregatesFx::ARRAY_NAME, TILEDB_READ}; - for (bool set_ranges : {true, false}) { - CppAggregatesFx::set_ranges_ = set_ranges; - for (bool request_data : {true, false}) { - CppAggregatesFx::request_data_ = request_data; - for (bool set_qc : CppAggregatesFx::set_qc_values_) { - CppAggregatesFx::set_qc_ = set_qc; - for (tiledb_layout_t layout : CppAggregatesFx::layout_values_) { - CppAggregatesFx::layout_ = layout; - Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); - - QueryChannel default_channel = - QueryExperimental::get_default_channel(query); - ChannelOperation operation = - QueryExperimental::create_unary_aggregate( - query, "a1"); - default_channel.apply_aggregate("NullCount", operation); - - CppAggregatesFx::set_ranges_and_condition_if_needed( - array, query, false); - - // Set the data buffer for the aggregator. - uint64_t cell_size = std::is_same::value ? - CppAggregatesFx::STRING_CELL_VAL_NUM : - sizeof(T); - std::vector null_count(1); - std::vector dim1(100); - std::vector dim2(100); - std::vector a1(100 * cell_size); - std::vector a1_validity(100); - query.set_layout(layout); - query.set_data_buffer("NullCount", null_count); - - if (request_data) { - query.set_data_buffer("d1", dim1); - query.set_data_buffer("d2", dim2); - query.set_data_buffer( - "a1", static_cast(a1.data()), a1.size() / cell_size); - query.set_validity_buffer("a1", a1_validity); - } - - // Submit the query. - query.submit(); - - // Check the results. - uint64_t expected_null_count; - if (CppAggregatesFx::dense_) { - if (set_qc) { - expected_null_count = 0; - } else { - if (set_ranges) { - expected_null_count = 12; - } else { - expected_null_count = 17; - } - } - } else { - if (set_ranges) { - expected_null_count = CppAggregatesFx::allow_dups_ ? 4 : 3; - } else { - expected_null_count = CppAggregatesFx::allow_dups_ ? 8 : 7; - } - } - - auto result_el = query.result_buffer_elements_nullable(); - CHECK(std::get<1>(result_el["NullCount"]) == 1); - CHECK(null_count[0] == expected_null_count); + CppAggregatesFx::run_all_combinations([&]() -> void { + Query query(CppAggregatesFx::ctx_, array, TILEDB_READ); + + QueryChannel default_channel = + QueryExperimental::get_default_channel(query); + ChannelOperation operation = + QueryExperimental::create_unary_aggregate( + query, "a1"); + default_channel.apply_aggregate("NullCount", operation); + + CppAggregatesFx::set_ranges_and_condition_if_needed(array, query, false); + + // Set the data buffer for the aggregator. + uint64_t cell_size = std::is_same::value ? + CppAggregatesFx::STRING_CELL_VAL_NUM : + sizeof(T); + std::vector null_count(1); + std::vector dim1(100); + std::vector dim2(100); + std::vector a1(100 * cell_size); + std::vector a1_validity(100); + query.set_layout(CppAggregatesFx::layout_); + query.set_data_buffer("NullCount", null_count); + + if (CppAggregatesFx::request_data_) { + query.set_data_buffer("d1", dim1); + query.set_data_buffer("d2", dim2); + query.set_data_buffer( + "a1", static_cast(a1.data()), a1.size() / cell_size); + query.set_validity_buffer("a1", a1_validity); + } - if (request_data) { - CppAggregatesFx::validate_data( - query, dim1, dim2, a1, a1_validity); - } + // Submit the query. + query.submit(); - CppAggregatesFx::validate_tiles_read(query); - CppAggregatesFx::validate_tiles_read_null_count(query); + // Check the results. + uint64_t expected_null_count; + if (CppAggregatesFx::dense_) { + if (CppAggregatesFx::set_qc_) { + expected_null_count = 0; + } else { + if (CppAggregatesFx::set_ranges_) { + expected_null_count = 12; + } else { + expected_null_count = 17; } } + } else { + if (CppAggregatesFx::set_ranges_) { + expected_null_count = CppAggregatesFx::allow_dups_ ? 4 : 3; + } else { + expected_null_count = CppAggregatesFx::allow_dups_ ? 8 : 7; + } } - } + + auto result_el = query.result_buffer_elements_nullable(); + CHECK(std::get<1>(result_el["NullCount"]) == 1); + CHECK(null_count[0] == expected_null_count); + + if (CppAggregatesFx::request_data_) { + CppAggregatesFx::validate_data( + query, dim1, dim2, a1, a1_validity, [&](uint64_t) -> void { + expected_null_count++; + }); + } + + CppAggregatesFx::validate_tiles_read(query); + CppAggregatesFx::validate_tiles_read_null_count(query); + }); // Close array. array.close(); @@ -2613,7 +2549,13 @@ TEMPLATE_LIST_TEST_CASE_METHOD( CHECK(sum[0] == expected_sum); CppAggregatesFx::validate_data( - query, dim1, dim2, a1, a1_validity, false); + query, + dim1, + dim2, + a1, + a1_validity, + [&](uint64_t) -> void {}, + false); } } } diff --git a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc index 34e18d31dc90..65912d21cfe2 100644 --- a/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc +++ b/tiledb/api/c_api/query_aggregate/query_aggregate_api.cc @@ -187,10 +187,29 @@ capi_return_t tiledb_create_unary_aggregate( tiledb_query_field_handle_t::make_handle(query, input_field_name); tiledb_query_field_handle_t::break_handle(field); + const auto is_dense_dim{schema.dense() && schema.is_dim(field_name)}; + const auto cell_order{schema.cell_order()}; + + // Get the dimension index for the dense case. It is used below to know if the + // dimenson to be aggregated is the last dimension for ROW_MAJOR or first for + // COL_MAJOR. This is used at the aggregate level to know if we need to change + // the dimension value when we move cells. + unsigned dim_idx = 0; + if (is_dense_dim) { + dim_idx = schema.domain().get_dimension_index(field_name); + } + + const bool is_slab_dim = + is_dense_dim && (cell_order == sm::Layout::ROW_MAJOR) ? + (dim_idx == schema.dim_num() - 1) : + (dim_idx == 0); + auto fi = tiledb::sm::FieldInfo( field_name, schema.var_size(field_name), schema.is_nullable(field_name), + is_dense_dim, + is_slab_dim, schema.cell_val_num(field_name), schema.type(field_name)); diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 1b5883458b7d..0aca71521d9c 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -532,16 +532,14 @@ Status Domain::has_dimension(const std::string& name, bool* has_dim) const { return Status::Ok(); } -Status Domain::get_dimension_index( - const std::string& name, unsigned* dim_idx) const { +unsigned Domain::get_dimension_index(const std::string& name) const { for (unsigned d = 0; d < dim_num_; ++d) { if (dimension_ptrs_[d]->name() == name) { - *dim_idx = d; - return Status::Ok(); + return d; } } - return Status_DomainError( + throw std::invalid_argument( "Cannot get dimension index; Invalid dimension name"); } diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 5544169cb44b..856a97918727 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -403,10 +403,9 @@ class Domain { * Gets the index in the domain of a given dimension name * * @param name Name of dimension to check for - * @param dim_idx The index of this dimension in the domain - * @return Status + * @return Dimension index */ - Status get_dimension_index(const std::string& name, unsigned* dim_idx) const; + unsigned get_dimension_index(const std::string& name) const; /** Returns true if at least one dimension has null tile extent. */ bool null_tile_extents() const; diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 8fa9007513f2..fa2f2691911c 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -1971,8 +1971,7 @@ TileMetadata FragmentMetadata::get_tile_metadata( unsigned dim_idx = 0; const NDRange* mbr = nullptr; if (is_dim) { - throw_if_not_ok( - array_schema_->domain().get_dimension_index(name, &dim_idx)); + dim_idx = array_schema_->domain().get_dimension_index(name); mbr = &rtree_.leaf(tile_idx); } diff --git a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h index 70ed41adccd5..cf7234fbc754 100644 --- a/tiledb/sm/query/readers/aggregators/aggregate_with_count.h +++ b/tiledb/sm/query/readers/aggregators/aggregate_with_count.h @@ -93,7 +93,17 @@ class AggregateWithCount { // nullable. The bitmap tells us which cells was already filtered out by // ranges or query conditions. if (input_data.has_bitmap()) { - if (field_info_.is_nullable_) { + if (field_info_.is_dense_dim_) { + // Process for dense dimension values with bitmap. + for (uint64_t c = 0; c < input_data.size(); c++) { + auto bitmap_val = input_data.bitmap_at(c); + auto value = dense_dim_value_at(input_data, c); + for (BITMAP_T i = 0; i < bitmap_val; i++) { + agg_policy.op(value, res, count); + count++; + } + } + } else if (field_info_.is_nullable_) { // Process for nullable values with bitmap. for (uint64_t c = 0; c < input_data.size(); c++) { auto bitmap_val = input_data.bitmap_at(c); @@ -117,7 +127,14 @@ class AggregateWithCount { } } } else { - if (field_info_.is_nullable_) { + if (field_info_.is_dense_dim_) { + // Process for dense dimension values with no bitmap. + for (uint64_t c = 0; c < input_data.size(); c++) { + auto value = dense_dim_value_at(input_data, c); + agg_policy.op(value, res, count); + count++; + } + } else if (field_info_.is_nullable_) { // Process for nullable values with no bitmap. for (uint64_t c = 0; c < input_data.size(); c++) { if (val_policy.op(input_data.validity_at(c))) { @@ -166,6 +183,24 @@ class AggregateWithCount { return AGG_T(); } + + /** + * Returns the dense dimension value at the specified cell if needed. + * + * @param input_data Input data. + * @param c Cell index. + * @return Value. + */ + inline AGG_T dense_dim_value_at(AggregateBuffer& input_data, uint64_t c) { + typedef typename type_data::value_type VALUE_T; + if constexpr ( + !std::is_same::value && + !std::is_same::value) { + return input_data.value_at(0) + c * field_info_.is_slab_dim_; + } + + return AGG_T(); + } }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/field_info.h b/tiledb/sm/query/readers/aggregators/field_info.h index 0a6d6071736c..df415c10659e 100644 --- a/tiledb/sm/query/readers/aggregators/field_info.h +++ b/tiledb/sm/query/readers/aggregators/field_info.h @@ -50,6 +50,8 @@ class FieldInfo { FieldInfo() : var_sized_(false) , is_nullable_(false) + , is_dense_dim_(false) + , is_slab_dim_(false) , cell_val_num_(1) , type_(Datatype::UINT8){}; @@ -71,6 +73,35 @@ class FieldInfo { : name_(name) , var_sized_(var_sized) , is_nullable_(is_nullable) + , is_dense_dim_(false) + , is_slab_dim_(false) + , cell_val_num_(cell_val_num) + , type_(type){}; + + /** + * Constructor. + * + * @param name Name of the field. + * @param var_sized Is the field var sized? + * @param is_nullable Is the field nullable? + * @param is_dense_dim Is the field nullable? + * @param is_slab_dim Is the dense dimension the slab dimension? + * @param cell_val_num Cell val num. + * @param type Data type of the field + */ + FieldInfo( + const std::string name, + const bool var_sized, + const bool is_nullable, + const bool is_dense_dim, + const bool is_slab_dim, + const unsigned cell_val_num, + const Datatype type) + : name_(name) + , var_sized_(var_sized) + , is_nullable_(is_nullable) + , is_dense_dim_(is_dense_dim) + , is_slab_dim_(is_slab_dim) , cell_val_num_(cell_val_num) , type_(type){}; @@ -87,6 +118,12 @@ class FieldInfo { /** Is the field nullable? */ const bool is_nullable_; + /** Is the field a dense dimension? */ + const bool is_dense_dim_; + + /** Is the dense dimension the cell slab dimension? */ + const bool is_slab_dim_; + /** Cell val num. */ const unsigned cell_val_num_; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 331de145b9cb..60a3d5a18ba4 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -418,27 +418,28 @@ Status DenseReader::dense_read() { // clear the memory. Also, a name in names might not be in the user buffers // so we might skip the copy but still clear the memory. for (auto& name : names) { - if (name == constants::coords || array_schema_.is_dim(name)) { - continue; - } - - // Get the tiles to load for this attribute. - auto result_tiles = result_tiles_to_load( - name, - qc_loaded_attr_names_set_, - subarray, - t_start, - t_end, - result_space_tiles, - tile_subarrays); - - // Read and unfilter tiles. + shared_ptr> filtered_data; + std::vector result_tiles; bool validity_only = null_count_aggregate_only(name); - std::vector to_load; - to_load.emplace_back(name, validity_only); - shared_ptr> filtered_data = - make_shared>( - read_attribute_tiles(to_load, result_tiles)); + bool dense_dim = name == constants::coords || array_schema_.is_dim(name); + + if (!dense_dim) { + // Get the tiles to load for this attribute. + result_tiles = result_tiles_to_load( + name, + qc_loaded_attr_names_set_, + subarray, + t_start, + t_end, + result_space_tiles, + tile_subarrays); + + // Read and unfilter tiles. + std::vector to_load; + to_load.emplace_back(name, validity_only); + filtered_data = make_shared>( + read_attribute_tiles(to_load, result_tiles)); + } if (compute_task.valid()) { RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); @@ -450,6 +451,7 @@ Status DenseReader::dense_read() { compute_task = storage_manager_->compute_tp()->execute([&, filtered_data, + dense_dim, name, validity_only, t_start, @@ -458,29 +460,31 @@ Status DenseReader::dense_read() { subarray_end_cell, num_range_threads, result_tiles]() { - // Unfilter tiles. - RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); - - // Only copy names that are present in the user buffers. - if (buffers_.count(name) != 0) { - // Copy attribute data to users buffers. - auto& var_buffer_size = var_buffer_sizes[name]; - status = copy_attribute( - name, - tile_extents, - subarray, - t_start, - t_end, - subarray_start_cell, - subarray_end_cell, - tile_subarrays, - tile_offsets, - var_buffer_size, - range_info, - result_space_tiles, - qc_result, - num_range_threads); - RETURN_CANCEL_OR_ERROR(status); + if (!dense_dim) { + // Unfilter tiles. + RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); + + // Only copy names that are present in the user buffers. + if (buffers_.count(name) != 0) { + // Copy attribute data to users buffers. + auto& var_buffer_size = var_buffer_sizes[name]; + status = copy_attribute( + name, + tile_extents, + subarray, + t_start, + t_end, + subarray_start_cell, + subarray_end_cell, + tile_subarrays, + tile_offsets, + var_buffer_size, + range_info, + result_space_tiles, + qc_result, + num_range_threads); + RETURN_CANCEL_OR_ERROR(status); + } } if (aggregates_.count(name) != 0) { @@ -499,7 +503,9 @@ Status DenseReader::dense_read() { RETURN_CANCEL_OR_ERROR(status); } - clear_tiles(name, result_tiles); + if (!dense_dim) { + clear_tiles(name, result_tiles); + } return Status::Ok(); }); @@ -507,9 +513,12 @@ Status DenseReader::dense_read() { // Process count aggregates. if (aggregates_.count(constants::count_of_rows) != 0) { - auto buff{make_aggregate_buffer( + DimType unused = 0; + auto buff{make_aggregate_buffer( + false, false, false, + unused, 0, subarray_start_cell, subarray_end_cell, @@ -1262,9 +1271,12 @@ Status DenseReader::copy_attribute( return Status::Ok(); } +template AggregateBuffer DenseReader::make_aggregate_buffer( const bool var_sized, const bool nullable, + const bool is_dim, + DimType& dim_val, const uint64_t cell_size, const uint64_t min_cell, const uint64_t max_cell, @@ -1284,6 +1296,8 @@ AggregateBuffer DenseReader::make_aggregate_buffer( std::make_optional( tile_tuple->validity_tile().data_as() + min_cell) : nullopt; + } else if (is_dim) { + fixed_data = &dim_val; } return AggregateBuffer( @@ -1861,14 +1875,28 @@ Status DenseReader::aggregate_tiles( const auto cell_order = array_schema_.cell_order(); auto stride = array_schema_.domain().stride(layout_); const auto& frag_domains = result_space_tile.frag_domains(); + const auto is_dim = array_schema_.is_dim(name); const auto attribute = array_schema_.attribute(name); const auto var_size = array_schema_.var_size(name); - const auto nullable = attribute->nullable(); + const auto nullable = !is_dim && attribute->nullable(); const auto cell_size = var_size ? constants::cell_var_offset_size : array_schema_.cell_size(name); auto& aggregates = aggregates_[name]; const bool validity_only = null_count_aggregate_only(name); + // Get the dimension index. + unsigned dim_idx = 0; + if (is_dim) { + dim_idx = array_schema_.domain().get_dimension_index(name); + } + + const bool is_slab_dim = is_dim && (cell_order == sm::Layout::ROW_MAJOR) ? + (dim_idx == dim_num - 1) : + (dim_idx == 0); + const bool is_col_dim = is_dim && (cell_order == sm::Layout::ROW_MAJOR) ? + (dim_idx == 0) : + (dim_idx == dim_num - 1); + // Cache tile tuples. std::vector tile_tuples(frag_domains.size()); for (uint32_t fd = 0; fd < frag_domains.size(); ++fd) { @@ -1912,7 +1940,7 @@ Status DenseReader::aggregate_tiles( // If the cell slab overlaps this fragment domain range, copy data. bool overlaps = false; uint64_t start = 0, end = 0; - if (tile_tuples[fd] != nullptr) { + if (is_dim || tile_tuples[fd] != nullptr) { auto&& [o, s, e] = cell_slab_overlaps_range( dim_num, frag_domains[fd].domain(), @@ -1925,11 +1953,18 @@ Status DenseReader::aggregate_tiles( if (overlaps) { // If the subarray and tile are in the same order, aggregate the // whole slab. + DimType dim_val = 0; if (stride == 1) { + if (is_dim) { + dim_val = iter.cell_slab_coords()[dim_idx] + is_slab_dim * start; + } + // Compute aggregate. - AggregateBuffer aggregate_buffer{make_aggregate_buffer( + AggregateBuffer aggregate_buffer{make_aggregate_buffer( var_size & !validity_only, nullable, + is_dim, + dim_val, cell_size, iter.pos_in_tile() + start, iter.pos_in_tile() + end + 1, @@ -1941,11 +1976,18 @@ Status DenseReader::aggregate_tiles( } else { // Go cell by cell. for (uint64_t i = 0; i < end - start + 1; ++i) { + if (is_dim) { + dim_val = + iter.cell_slab_coords()[dim_idx] + is_col_dim * (i + start); + } + // Compute aggregate. auto start_cell = iter.pos_in_tile() + (start + i) * stride; - AggregateBuffer aggregate_buffer{make_aggregate_buffer( + AggregateBuffer aggregate_buffer{make_aggregate_buffer( var_size & !validity_only, nullable, + is_dim, + dim_val, cell_size, start_cell, start_cell + 1, diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 37eb8a08d5ea..dcbf60f3fa63 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -257,9 +257,12 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const uint64_t num_range_threads); /** Make an aggregate buffer. */ + template AggregateBuffer make_aggregate_buffer( const bool var_sized, const bool nullable, + const bool is_dim, + DimType& dim_val, const uint64_t cell_size, const uint64_t min_cell, const uint64_t max_cell, @@ -280,6 +283,10 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::string& name, ResultSpaceTile& rst, const Subarray& tile_subarray) const { + if (array_schema_.is_dim(name)) { + return false; + } + // Make sure there are no filtered results by the query condition and that // there are only one fragment domain for this tile. Having more fragment // domains for a tile means we'll have to merge data for many sources so we diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 9618125ff0ae..49d0e7bcb369 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -494,7 +494,13 @@ void SparseIndexReaderBase::load_tile_offsets_for_fragments( load_tile_offsets(relevant_fragments, attr_tile_offsets_to_load_); // Load tile metadata. - load_tile_metadata(relevant_fragments, attr_tile_offsets_to_load_); + auto md_names_to_load = attr_tile_offsets_to_load_; + for (const auto& [name, _] : aggregates_) { + if (array_schema_.is_dim(name)) { + md_names_to_load.emplace_back(name); + } + } + load_tile_metadata(relevant_fragments, md_names_to_load); } Status SparseIndexReaderBase::read_and_unfilter_coords( diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 6ea1846152bf..d535b75e4900 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -527,10 +527,8 @@ Status Subarray::add_range_by_name( const void* start, const void* end, const void* stride) { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); - + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return add_range(dim_idx, start, end, stride); } @@ -586,10 +584,8 @@ Status Subarray::add_range_var_by_name( uint64_t start_size, const void* end, uint64_t end_size) { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); - + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return add_range_var(dim_idx, start, start_size, end, end_size); } @@ -697,9 +693,8 @@ Status Subarray::get_range_var( Status Subarray::get_range_num_from_name( const std::string& dim_name, uint64_t* range_num) const { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return get_range_num(dim_idx, range_num); } @@ -720,9 +715,8 @@ Status Subarray::get_range_from_name( const void** start, const void** end, const void** stride) const { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return get_range(dim_idx, range_idx, start, end, stride); } @@ -732,10 +726,8 @@ Status Subarray::get_range_var_size_from_name( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) const { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); - + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return get_range_var_size(dim_idx, range_idx, start_size, end_size); } @@ -744,10 +736,8 @@ Status Subarray::get_range_var_from_name( uint64_t range_idx, void* start, void* end) const { - unsigned dim_idx; - RETURN_NOT_OK(array_->array_schema_latest().domain().get_dimension_index( - dim_name, &dim_idx)); - + unsigned dim_idx = + array_->array_schema_latest().domain().get_dimension_index(dim_name); return get_range_var(dim_idx, range_idx, start, end); } const shared_ptr Subarray::array() const { From 7753f3360b42a837f2e18275a5b23691aa86157d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 12 Mar 2024 10:09:25 +0200 Subject: [PATCH 238/456] Update libmagic to version 5.45 (#4673) [SC-38521](https://app.shortcut.com/tiledb-inc/story/38521/update-libmagic-and-use-the-upstream-vcpkg-port) Split from #4553. This PR updates libmagic to version 5.45 and switches to using a vcpkg port closer to the upstream one, which we can easily consume with find_package(unofficial-libmagic) since microsoft/vcpkg#35274. One complication is that the upstream port builds libmagic with its official autotools-based build system, which is significantly slower on Windows (on Linux it builds pretty fast). I tried to upstream the CMake-based port I had added in #4119, but the PR was rejected. Apart from binary caching, there is unfortunately nothing we can do about the build performance regression. We could maintain the CMake-based port for our own use, but it will split what we build and what a future user of TileDB from vcpkg will build, and that port is not without its problems (it failed for example when I tried cross-compiling to arm64-windows, because it tried to execute the arm64 file.exe on my x64 machine). --- TYPE: BUILD DESC: Update libmagic to version 5.45 --- .readthedocs.yml | 11 + doc/dev/BUILD.md | 1 + ports/README.md | 3 +- ...-Use-pcre2.patch => 0001-Use-libtre.patch} | 2 +- ...-zlib-lib-name-to-match-CMake-output.patch | 39 ++ .../0003-Fix-WIN32-macro-checks.patch | 10 +- ...ude-dirent.h-for-S_ISREG-and-S_ISDIR.patch | 9 +- .../0006-Remove-Wrap-POSIX-headers.patch | 2 +- ports/libmagic/0009-No-fcntl-in-magic.c.patch | 25 -- ...y-check-for-the-presence-of-bitmasks.patch | 2 +- ...ve-pipe-related-functions-in-funcs.c.patch | 12 +- ...Convert-MSYS2-paths-to-Windows-paths.patch | 24 ++ ...k-for-backslash-in-argv-0-on-Windows.patch | 42 ++ .../0014-Define-POSIX-macros-if-missing.patch | 38 -- .../0015-MSYS2-Remove-ioctl-call.patch | 8 +- .../0016-Fix-file_famagic-function.patch | 40 ++ ports/libmagic/CMakeLists.txt | 170 -------- ports/libmagic/config.h | 401 ------------------ ports/libmagic/magic.def | 20 - ports/libmagic/portfile.cmake | 87 ++-- .../unofficial-libmagic-config.cmake.in | 71 +++- ports/libmagic/vcpkg.json | 50 ++- ports/pcre2/fix-cmake.patch | 334 --------------- ports/pcre2/no-static-suffix.patch | 33 -- ports/pcre2/pcre2-10.35_fix-uwp.patch | 10 - ports/pcre2/portfile.cmake | 73 ---- ports/pcre2/usage | 6 - ports/pcre2/vcpkg.json | 39 -- tiledb/CMakeLists.txt | 5 +- .../util/tdb_gzip_embedded_data.cc | 19 +- tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 | Bin 322448 -> 410953 bytes vcpkg.json | 9 +- 32 files changed, 356 insertions(+), 1239 deletions(-) rename ports/libmagic/{0001-Use-pcre2.patch => 0001-Use-libtre.patch} (98%) create mode 100644 ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch delete mode 100644 ports/libmagic/0009-No-fcntl-in-magic.c.patch create mode 100644 ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch create mode 100644 ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch delete mode 100644 ports/libmagic/0014-Define-POSIX-macros-if-missing.patch create mode 100644 ports/libmagic/0016-Fix-file_famagic-function.patch delete mode 100644 ports/libmagic/CMakeLists.txt delete mode 100644 ports/libmagic/config.h delete mode 100644 ports/libmagic/magic.def delete mode 100644 ports/pcre2/fix-cmake.patch delete mode 100644 ports/pcre2/no-static-suffix.patch delete mode 100644 ports/pcre2/pcre2-10.35_fix-uwp.patch delete mode 100644 ports/pcre2/portfile.cmake delete mode 100644 ports/pcre2/usage delete mode 100644 ports/pcre2/vcpkg.json diff --git a/.readthedocs.yml b/.readthedocs.yml index 459b768ef7f1..0c2c9b0fa841 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,6 +12,17 @@ build: os: "ubuntu-22.04" tools: python: "3.8" + apt_packages: + - autoconf + - autoconf-archive + - automake + - curl + - git + - libtool + - make + - pkg-config + - unzip + - zip python: install: diff --git a/doc/dev/BUILD.md b/doc/dev/BUILD.md index 1717cb0fa908..d0c8140d1454 100644 --- a/doc/dev/BUILD.md +++ b/doc/dev/BUILD.md @@ -12,6 +12,7 @@ title: Building TileDB from source * Apple Clang 14 * Git (required by vcpkg) * curl (required by vcpkg on non-Windows) +* autoconf (required by building libmagic on non-Windows) ## Downloading the source code diff --git a/ports/README.md b/ports/README.md index 7faf08053d33..09981c7650fd 100644 --- a/ports/README.md +++ b/ports/README.md @@ -30,7 +30,6 @@ After copying the port, add an entry to the table below. You should also contrib |Port|Reason| |----|------| -|`libmagic`|Updating to the upstream port deferred due to failures.| |`openssl`|Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024.| -|`pcre2`|To be removed alongside libmagic.| |`azure-storage-common-cpp`|Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221).| +|`libmagic`|Patching to add features explicitly enabling compression support.| diff --git a/ports/libmagic/0001-Use-pcre2.patch b/ports/libmagic/0001-Use-libtre.patch similarity index 98% rename from ports/libmagic/0001-Use-pcre2.patch rename to ports/libmagic/0001-Use-libtre.patch index 2bcf73cd2bc6..a1080d5a070b 100644 --- a/ports/libmagic/0001-Use-pcre2.patch +++ b/ports/libmagic/0001-Use-libtre.patch @@ -44,7 +44,7 @@ index c548e97..299ac0c 100644 #include #include /* For open and flags */ -#include -+#include ++#include #include #include #ifndef WIN32 diff --git a/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch b/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch new file mode 100644 index 000000000000..5e070cc07fe0 --- /dev/null +++ b/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch @@ -0,0 +1,39 @@ +From 2fa43ece9ec7564e1fbb9867bb5852b834643aa4 Mon Sep 17 00:00:00 2001 +From: Long Nguyen +Date: Sat, 8 May 2021 19:36:11 +0700 +Subject: [PATCH 02/14] Change zlib lib name to match CMake output + +--- + configure.ac | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/configure.ac b/configure.ac +index b05c334..dd4063c 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -173,7 +173,7 @@ AC_REPLACE_FUNCS(getopt_long asprintf vasprintf strlcpy strlcat getline ctime_r + + dnl Checks for libraries + if test "$enable_zlib" != "no"; then +- AC_CHECK_LIB(z, gzopen) ++ AC_SEARCH_LIBS(gzopen, [z zlib zlibd], have_zlib = "yes", have_zlib = "no") + fi + if test "$enable_bzlib" != "no"; then + AC_CHECK_LIB(bz2, BZ2_bzCompressInit) +@@ -193,11 +193,11 @@ AM_CONDITIONAL(IS_CROSS_COMPILE, test "$cross_compiling" = yes) + + dnl Final sanity checks + if test "$enable_zlib" = "yes"; then +- if test "$ac_cv_header_zlib_h$ac_cv_lib_z_gzopen" != "yesyes"; then ++ if test "$ac_cv_header_zlib_h$have_zlib" != "yesyes"; then + AC_MSG_ERROR([zlib support requested but not found]) + fi + fi +-if test "$ac_cv_header_zlib_h$ac_cv_lib_z_gzopen" = "yesyes"; then ++if test "$ac_cv_header_zlib_h$have_zlib" = "yesyes"; then + AC_DEFINE([ZLIBSUPPORT], 1, [Enable zlib compression support]) + fi + if test "$enable_bzlib" = "yes"; then +-- +2.29.2.windows.2 + diff --git a/ports/libmagic/0003-Fix-WIN32-macro-checks.patch b/ports/libmagic/0003-Fix-WIN32-macro-checks.patch index 44ab2ddea2af..e90beb4d4097 100644 --- a/ports/libmagic/0003-Fix-WIN32-macro-checks.patch +++ b/ports/libmagic/0003-Fix-WIN32-macro-checks.patch @@ -12,7 +12,7 @@ index 299ac0c..2c365a6 100644 --- a/src/file.h +++ b/src/file.h @@ -82,7 +82,7 @@ - #include + #include #include #include -#ifndef WIN32 @@ -31,13 +31,13 @@ index 299ac0c..2c365a6 100644 #define PATHSEP ':' @@ -103,7 +103,7 @@ - #define private static + #define file_private static -#if HAVE_VISIBILITY && !defined(WIN32) +#if HAVE_VISIBILITY && !defined(_WIN32) - #define public __attribute__ ((__visibility__("default"))) - #ifndef protected - #define protected __attribute__ ((__visibility__("hidden"))) + #define file_public __attribute__ ((__visibility__("default"))) + #ifndef file_protected + #define file_protected __attribute__ ((__visibility__("hidden"))) -- 2.29.2.windows.2 diff --git a/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch b/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch index 76cbc1222c8b..afcb658d8890 100644 --- a/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch +++ b/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch @@ -11,14 +11,15 @@ diff --git a/src/file.h b/src/file.h index 0332506..4aa9f60 100644 --- a/src/file.h +++ b/src/file.h -@@ -88,6 +88,7 @@ +@@ -88,7 +88,8 @@ /* Do this here and now, because struct stat gets re-defined on solaris */ #include #include +#include - - #define ENABLE_CONDITIONALS - + #include + #if defined(HAVE_XLOCALE_H) + #include + #endif -- 2.29.2.windows.2 diff --git a/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch b/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch index 56efac248157..169561d24372 100644 --- a/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch +++ b/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch @@ -161,7 +161,7 @@ index 3ab52d1..fc48d84 100644 --- a/src/pread.c +++ b/src/pread.c @@ -3,7 +3,9 @@ - FILE_RCSID("@(#)$File: pread.c,v 1.2 2013/04/02 16:23:07 christos Exp $") + FILE_RCSID("@(#)$File: pread.c,v 1.5 2022/09/24 20:30:13 christos Exp $") #endif /* lint */ #include +#ifdef HAVE_UNISTD_H diff --git a/ports/libmagic/0009-No-fcntl-in-magic.c.patch b/ports/libmagic/0009-No-fcntl-in-magic.c.patch deleted file mode 100644 index 47bf16165ef8..000000000000 --- a/ports/libmagic/0009-No-fcntl-in-magic.c.patch +++ /dev/null @@ -1,25 +0,0 @@ -From 6d10bef865b69764f6e0ddd2b0f6a84e484cdb37 Mon Sep 17 00:00:00 2001 -From: Long Nguyen -Date: Sun, 9 May 2021 13:25:14 +0700 -Subject: [PATCH 09/14] No fcntl in magic.c - ---- - src/magic.c | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/magic.c b/src/magic.c -index e9aeafa..382bd96 100644 ---- a/src/magic.c -+++ b/src/magic.c -@@ -462,7 +462,7 @@ file_or_fd(struct magic_set *ms, const char *inname, int fd) - rv = 0; - goto done; - } --#if O_CLOEXEC == 0 -+#if O_CLOEXEC == 0 && !defined(_WIN32) - (void)fcntl(fd, F_SETFD, FD_CLOEXEC); - #endif - } --- -2.29.2.windows.2 - diff --git a/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch b/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch index f0d8738e465e..e241f5e49c09 100644 --- a/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch +++ b/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch @@ -17,7 +17,7 @@ index 5204f20..7244841 100644 #undef HAVE_MAJOR -#ifdef S_IFLNK +#if S_IFLNK != 0 - private int + file_private int bad_link(struct magic_set *ms, int err, char *buf) { @@ -108,7 +108,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) diff --git a/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch b/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch index ae2720ca3617..13c24faaca0b 100644 --- a/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch +++ b/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch @@ -11,21 +11,19 @@ diff --git a/src/funcs.c b/src/funcs.c index b926625..b585486 100644 --- a/src/funcs.c +++ b/src/funcs.c -@@ -809,6 +809,7 @@ file_print_guid(char *str, size_t len, const uint64_t *guid) - g->data4[6], g->data4[7]); +@@ -888,5 +888,6 @@ } +#ifndef _WIN32 - protected int + file_protected int file_pipe_closexec(int *fds) { -@@ -827,6 +828,7 @@ protected int - file_clear_closexec(int fd) { - return fcntl(fd, F_SETFD, 0); +@@ -914,5 +915,6 @@ + #endif } +#endif - protected char * + file_protected char * file_strtrim(char *str) -- 2.29.2.windows.2 diff --git a/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch b/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch new file mode 100644 index 000000000000..728fe77539f2 --- /dev/null +++ b/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch @@ -0,0 +1,24 @@ +From f0139ced57c1579450f3d09b6e3ae0159aae031b Mon Sep 17 00:00:00 2001 +From: Long Nguyen +Date: Mon, 10 May 2021 08:43:28 +0700 +Subject: [PATCH 12/14] Convert MSYS2 paths to Windows paths + +--- + src/Makefile.am | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/Makefile.am b/src/Makefile.am +index 34781b9..26f853f 100644 +--- a/src/Makefile.am ++++ b/src/Makefile.am +@@ -1,4 +1,7 @@ + MAGIC = $(pkgdatadir)/magic ++if MINGW ++MAGIC != cygpath -m -a -l '$(MAGIC)' ++endif + lib_LTLIBRARIES = libmagic.la + nodist_include_HEADERS = magic.h + +-- +2.29.2.windows.2 + diff --git a/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch b/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch new file mode 100644 index 000000000000..77777fc61e8e --- /dev/null +++ b/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch @@ -0,0 +1,42 @@ +From 3f10c7d31627b64b068b84ba72e706991f672560 Mon Sep 17 00:00:00 2001 +From: Long Nguyen +Date: Fri, 14 May 2021 08:14:05 +0700 +Subject: [PATCH 13/14] Check for backslash in argv[0] on Windows + +--- + magic/Makefile.am | 2 +- + src/file.c | 5 +++++ + 2 files changed, 6 insertions(+), 1 deletion(-) + +diff --git a/magic/Makefile.am b/magic/Makefile.am +index 0eb5865..170bbab 100644 +--- a/magic/Makefile.am ++++ b/magic/Makefile.am +@@ -353,7 +353,7 @@ ${MAGIC}: $(EXTRA_DIST) $(FILE_COMPILE_DEP) + @(if expr "${FILE_COMPILE}" : '.*/.*' > /dev/null; then \ + echo "Using ${FILE_COMPILE} to generate ${MAGIC}" > /dev/null; \ + else \ +- v=$$(${FILE_COMPILE} --version | sed -e s/file-// -e q); \ ++ v=$$(${FILE_COMPILE} --version | sed -e s/file${EXEEXT}-// -e q); \ + if [ "$$v" != "${PACKAGE_VERSION}" ]; then \ + echo "Cannot use the installed version of file ($$v) to"; \ + echo "cross-compile file ${PACKAGE_VERSION}"; \ +diff --git a/src/file.c b/src/file.c +index 2889f8a..12a604b 100644 +--- a/src/file.c ++++ b/src/file.c +@@ -200,6 +200,11 @@ main(int argc, char *argv[]) + _wildcard(&argc, &argv); + #endif + ++#ifdef _WIN32 ++ if ((progname = strrchr(argv[0], '\\')) != NULL) ++ progname++; ++ else ++#endif + if ((progname = strrchr(argv[0], '/')) != NULL) + progname++; + else +-- +2.29.2.windows.2 + diff --git a/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch b/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch deleted file mode 100644 index 09ac7a11926e..000000000000 --- a/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch +++ /dev/null @@ -1,38 +0,0 @@ -From fa0e11f36bb0e322250e1e488ced9f2bf166874f Mon Sep 17 00:00:00 2001 -From: Long Nguyen -Date: Fri, 14 May 2021 18:11:39 +0700 -Subject: [PATCH 14/14] Define POSIX macros if missing - ---- - src/file.h | 15 +++++++++++++++ - 1 file changed, 15 insertions(+) - -diff --git a/src/file.h b/src/file.h -index ccfe0da..98cd37b 100644 ---- a/src/file.h -+++ b/src/file.h -@@ -100,6 +100,21 @@ - #include - #include - -+#if !defined(S_IFBLK) -+#define S_IFBLK 0 -+#define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) -+#endif -+ -+#if !defined(S_IFLNK) -+#define S_IFLNK 0 -+#define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) -+#endif -+ -+#if !defined(S_IFSOCK) -+#define S_IFSOCK 0 -+#define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) -+#endif -+ - #define ENABLE_CONDITIONALS - - #ifndef MAGIC --- -2.29.2.windows.2 - diff --git a/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch b/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch index da8ef79df29a..13fe3cf0c550 100644 --- a/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch +++ b/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch @@ -7,20 +7,20 @@ diff --git a/src/compress.c b/src/compress.c index 33ce2bc..f172eda 100644 --- a/src/compress.c +++ b/src/compress.c -@@ -378,7 +378,7 @@ +@@ -407,7 +407,7 @@ sread(int fd, void *buf, size_t n, int canbepipe __attribute__((__unused__))) { ssize_t rv; --#ifdef FIONREAD +-#if defined(FIONREAD) && !defined(__MINGW32__) +#if defined(FIONREAD) && !defined(__MINGW32__) && !defined(WIN32) int t = 0; #endif size_t rn = n; -@@ -386,7 +386,7 @@ +@@ -418,7 +418,7 @@ if (fd == STDIN_FILENO) goto nocheck; --#ifdef FIONREAD +-#if defined(FIONREAD) && !defined(__MINGW32__) +#if defined(FIONREAD) && !defined(__MINGW32__) && !defined(WIN32) if (canbepipe && (ioctl(fd, FIONREAD, &t) == -1 || t == 0)) { #ifdef FD_ZERO diff --git a/ports/libmagic/0016-Fix-file_famagic-function.patch b/ports/libmagic/0016-Fix-file_famagic-function.patch new file mode 100644 index 000000000000..5eaba925a589 --- /dev/null +++ b/ports/libmagic/0016-Fix-file_famagic-function.patch @@ -0,0 +1,40 @@ +diff --git a/src/fsmagic.c b/src/fsmagic.c +index 7244841..2c553c1 100644 +--- a/src/fsmagic.c ++++ b/src/fsmagic.c +@@ -66,7 +66,7 @@ # define major(dev) (((dev) >> 8) & 0xff) + # define minor(dev) ((dev) & 0xff) + #endif + #undef HAVE_MAJOR +-#if S_IFLNK != 0 ++#if S_IFLNK != 0 && ! defined(_WIN32) + file_private int + bad_link(struct magic_set *ms, int err, char *buf) + { +@@ -108,7 +108,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) + int ret, did = 0; + int mime = ms->flags & MAGIC_MIME; + int silent = ms->flags & (MAGIC_APPLE|MAGIC_EXTENSION); +-#if S_IFLNK != 0 ++#if S_IFLNK != 0 && ! defined(_WIN32) + char buf[BUFSIZ+4]; + ssize_t nch; + struct stat tstatbuf; +@@ -122,7 +122,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) + * Fstat is cheaper but fails for files you don't have read perms on. + * On 4.2BSD and similar systems, use lstat() to identify symlinks. + */ +-#if S_IFLNK != 0 ++#if S_IFLNK != 0 && ! defined(_WIN32) + if ((ms->flags & MAGIC_SYMLINK) == 0) + ret = lstat(fn, sb); + else +@@ -290,7 +290,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) + return -1; + break; + #endif +-#if S_IFLNK != 0 ++#if S_IFLNK != 0 && ! defined(_WIN32) + case S_IFLNK: + if ((nch = readlink(fn, buf, BUFSIZ-1)) <= 0) { + if (ms->flags & MAGIC_ERROR) { diff --git a/ports/libmagic/CMakeLists.txt b/ports/libmagic/CMakeLists.txt deleted file mode 100644 index a6248bdf871a..000000000000 --- a/ports/libmagic/CMakeLists.txt +++ /dev/null @@ -1,170 +0,0 @@ -cmake_minimum_required(VERSION 3.12) - -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/configure.ac CONFIGURE_AC_CONTENT) -string(REGEX MATCH "AC_INIT\\(\\[file\\],\\[([0-9]+\\.[0-9]+)\\]" _ ${CONFIGURE_AC_CONTENT}) -unset(CONFIGURE_AC_CONTENT) - -project(file VERSION ${CMAKE_MATCH_1}) - -option(FILE_TESTS "Enable file tests" OFF) - -# Get library directory for multiarch linux distros -include(GNUInstallDirs) - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") -set(targets) -find_package(PCRE2 CONFIG REQUIRED COMPONENTS 8BIT) -if(WIN32 AND NOT MINGW) - find_package(unofficial-getopt-win32 CONFIG REQUIRED) -endif() - -set(LIBMAGIC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") -set(LIBMAGIC_TEST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests") - -set(WIN_COMPAT_SOURCES - ${LIBMAGIC_SOURCE_DIR}/asctime_r.c - ${LIBMAGIC_SOURCE_DIR}/asprintf.c - ${LIBMAGIC_SOURCE_DIR}/ctime_r.c - ${LIBMAGIC_SOURCE_DIR}/dprintf.c - ${LIBMAGIC_SOURCE_DIR}/getline.c - ${LIBMAGIC_SOURCE_DIR}/gmtime_r.c - ${LIBMAGIC_SOURCE_DIR}/localtime_r.c - ${LIBMAGIC_SOURCE_DIR}/pread.c - ${LIBMAGIC_SOURCE_DIR}/strcasestr.c - # ${LIBMAGIC_SOURCE_DIR}/strlcat.c - # ${LIBMAGIC_SOURCE_DIR}/strlcpy.c - ${LIBMAGIC_SOURCE_DIR}/vasprintf.c - ${LIBMAGIC_SOURCE_DIR}/fmtcheck.c - ${LIBMAGIC_SOURCE_DIR}/magic.def -) - -if(WIN32) - set(LIBMAGIC_SOURCE_FILES ${WIN_COMPAT_SOURCES}) -else() - set(LIBMAGIC_SOURCE_FILES) -endif() - -list(APPEND LIBMAGIC_SOURCE_FILES - ${LIBMAGIC_SOURCE_DIR}/apprentice.c - ${LIBMAGIC_SOURCE_DIR}/apptype.c - ${LIBMAGIC_SOURCE_DIR}/ascmagic.c - ${LIBMAGIC_SOURCE_DIR}/buffer.c - ${LIBMAGIC_SOURCE_DIR}/cdf.c - ${LIBMAGIC_SOURCE_DIR}/cdf_time.c - ${LIBMAGIC_SOURCE_DIR}/compress.c - ${LIBMAGIC_SOURCE_DIR}/der.c - ${LIBMAGIC_SOURCE_DIR}/encoding.c - ${LIBMAGIC_SOURCE_DIR}/fmtcheck.c - ${LIBMAGIC_SOURCE_DIR}/fsmagic.c - ${LIBMAGIC_SOURCE_DIR}/funcs.c - ${LIBMAGIC_SOURCE_DIR}/is_json.c - ${LIBMAGIC_SOURCE_DIR}/is_tar.c - ${LIBMAGIC_SOURCE_DIR}/magic.c - ${LIBMAGIC_SOURCE_DIR}/print.c - ${LIBMAGIC_SOURCE_DIR}/readcdf.c - ${LIBMAGIC_SOURCE_DIR}/readelf.c - ${LIBMAGIC_SOURCE_DIR}/softmagic.c - # ${LIBMAGIC_SOURCE_DIR}/strlcat.c - # ${LIBMAGIC_SOURCE_DIR}/strlcpy.c - ${LIBMAGIC_SOURCE_DIR}/is_csv.c -) -if(NOT APPLE) -list(APPEND LIBMAGIC_SOURCE_FILES - ${LIBMAGIC_SOURCE_DIR}/strlcat.c - ${LIBMAGIC_SOURCE_DIR}/strlcpy.c -) -endif() - -# replace the version in the magic.h.in and write it to magic.h -FILE(READ ${LIBMAGIC_SOURCE_DIR}/magic.h.in MAGIC_H_CONTENT) -STRING(REPLACE "." "" FILE_VERSION_WITHOUT_DOT "${CMAKE_PROJECT_VERSION}") -STRING(REPLACE "X.YY" ${FILE_VERSION_WITHOUT_DOT} MAGIC_H_CONTENT_NEW "${MAGIC_H_CONTENT}") -FILE(WRITE ${LIBMAGIC_SOURCE_DIR}/magic.h "${MAGIC_H_CONTENT_NEW}") - -add_compile_definitions(HAVE_CONFIG_H VERSION="${CMAKE_PROJECT_VERSION}") -if(WIN32) - add_compile_definitions(WIN32_LEAN_AND_MEAN WIN32) -endif() - -add_library(libmagic ${LIBMAGIC_SOURCE_FILES}) -set(targets ${targets} libmagic) - -target_link_libraries(libmagic PRIVATE PCRE2::POSIX) - -target_include_directories(libmagic - PUBLIC - "$" - "$" -) - -# 'file' CLI -add_executable(file ${LIBMAGIC_SOURCE_DIR}/file.c) -set(targets ${targets} file) -target_link_libraries(file PRIVATE PCRE2::POSIX libmagic) -if (WIN32) - if (NOT MINGW) - target_link_libraries(file PRIVATE unofficial::getopt-win32::getopt) - endif() - target_link_libraries(file PRIVATE shlwapi) -endif() - -if(MSVC) - target_include_directories(file PRIVATE getopt) -endif() - -# Following is the compilation of the magic file -file(GLOB MAGIC_FRAGMENTS magic/Magdir/*) - -# Prepare a temporary file to combine the magic fragments: -set(MAGIC_FRAGMENTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/magic) -file(WRITE ${MAGIC_FRAGMENTS_FILE} "") - -# Call the "cat" function for each input file -foreach(MAGIC_FRAGMENT ${MAGIC_FRAGMENTS}) - file(APPEND ${MAGIC_FRAGMENTS_FILE} "############## ${MAGIC_FRAGMENT} ##############\n") - file(READ ${MAGIC_FRAGMENT} MAGIC_FRAGMENT_CONTENTS) - file(APPEND ${MAGIC_FRAGMENTS_FILE} "${MAGIC_FRAGMENT_CONTENTS}\n") - unset(MAGIC_FRAGMENT_CONTENTS) -endforeach() - -add_custom_command(OUTPUT magic.mgc - COMMAND file -C -m magic - COMMENT "Compiling magic file" -) - -add_custom_target(magic_mgc ALL DEPENDS magic.mgc) - -# Include module with function 'configure_package_config_file' -include(CMakePackageConfigHelpers) - -set(PORT_NAME unofficial-libmagic) -set(TARGETS_EXPORT_NAME ${PORT_NAME}-targets) - -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/magic.mgc DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PORT_NAME}) - -install(FILES ${LIBMAGIC_SOURCE_DIR}/magic.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - -install(TARGETS ${targets} - EXPORT ${TARGETS_EXPORT_NAME} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) -install(EXPORT ${TARGETS_EXPORT_NAME} - FILE ${TARGETS_EXPORT_NAME}.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME} - NAMESPACE unofficial::libmagic:: - ) -configure_package_config_file( - ${PORT_NAME}-config.cmake.in - ${PORT_NAME}-config.cmake - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME}) -write_basic_package_version_file( - ${PORT_NAME}-config-version.cmake - VERSION ${CMAKE_PROJECT_VERSION} - COMPATIBILITY SameMajorVersion) -install(FILES - ${CMAKE_BINARY_DIR}/${PORT_NAME}-config.cmake - ${CMAKE_BINARY_DIR}/${PORT_NAME}-config-version.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME}) diff --git a/ports/libmagic/config.h b/ports/libmagic/config.h deleted file mode 100644 index 3a6e492f7c21..000000000000 --- a/ports/libmagic/config.h +++ /dev/null @@ -1,401 +0,0 @@ -/* Define in built-in ELF support is used */ -#define BUILTIN_ELF 1 - -/* Define for ELF core file support */ -#define ELFCORE 1 - -/* Define to 1 if you have the `asctime_r' function. */ -#undef HAVE_ASCTIME_R - -/* Define to 1 if you have the `asprintf' function. */ -#undef HAVE_ASPRINTF - -/* Define to 1 if you have the `ctime_r' function. */ -#undef HAVE_CTIME_R - -/* HAVE_DAYLIGHT */ -#define HAVE_DAYLIGHT 1 - -/* Define to 1 if you have the declaration of `daylight', and to 0 if you - don't. */ -#undef HAVE_DECL_DAYLIGHT - -/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. - */ -#undef HAVE_DECL_TZNAME - -/* Define to 1 if you have the header file. */ -#undef HAVE_DLFCN_H - -/* Define to 1 if you have the `dprintf' function. */ -#undef HAVE_DPRINTF - -/* Define to 1 if you have the header file. */ -#undef HAVE_ERR_H - -/* Define to 1 if you have the header file. */ -#ifdef WIN32 -#undef HAVE_DIRENT_H -#else -// TBD: will all non-win32 xplatforms we want have this? -#define HAVE_DIRENT_H 1 -#endif - -/* Define to 1 if you have the header file. */ -#define HAVE_FCNTL_H 1 - -/* Define to 1 if you have the `fmtcheck' function. */ -/*#undef HAVE_FMTCHECK*/ - -/* Define to 1 if you have the `fork' function. */ -#undef HAVE_FORK - -/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ -#undef HAVE_FSEEKO - -/* Define to 1 if you have the `getline' function. */ -#undef HAVE_GETLINE - -/* Define to 1 if you have the header file. */ -#ifdef _WIN32 -#define HAVE_GETOPT_H 1 -#endif - -/* Define to 1 if you have the `getopt_long' function. */ -#undef HAVE_GETOPT_LONG - -/* Define to 1 if you have the `getpagesize' function. */ -#undef HAVE_GETPAGESIZE - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the `gnurx' library (-lgnurx). */ -#undef HAVE_LIBGNURX - -/* Define to 1 if you have the `z' library (-lz). */ -/* #undef HAVE_LIBZ */ - -/* Define to 1 if you have the header file. */ -#define HAVE_LIMITS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_LOCALE_H 1 - -/* Define to 1 if mbrtowc and mbstate_t are properly declared. */ -#define HAVE_MBRTOWC 1 - -/* Define to 1 if declares mbstate_t. */ -#define HAVE_MBSTATE_T 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if you have the `mkostemp' function. */ -#undef HAVE_MKOSTEMP - -/* Define to 1 if you have the `mkstemp' function. */ -#ifdef _WIN32 -#undef HAVE_MKSTEMP -#else -#define HAVE_MKSTEMP 1 -#endif - -/* Define to 1 if you have a working `mmap' system call. */ -#undef HAVE_MMAP - -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - -/* Define to 1 if you have the header file. */ -#define HAVE_STDDEF_H 1 - -/* Define to 1 if the system has the type `pid_t'. */ -#undef HAVE_PID_T - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strcasestr' function. */ -#if defined(_WIN32) && !defined(__MINGW32__) -#define HAVE_STRCASESTR 1 -#else -#undef HAVE_STRCASESTR -#endif - -/* Define to 1 if you have the `strerror' function. */ -#define HAVE_STRERROR 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if you have the `strlcat' function. */ -#undef HAVE_STRLCAT - -/* Define to 1 if you have the `strlcpy' function. */ -#undef HAVE_STRLCPY - -/* Define to 1 if you have the `strndup' function. */ -#undef HAVE_STRNDUP - -/* Define to 1 if you have the `strtof' function. */ -#undef HAVE_STRTOF - -/* Define to 1 if you have the `strtoul' function. */ -#define HAVE_STRTOUL 1 - -/* HAVE_STRUCT_OPTION */ -#define HAVE_STRUCT_OPTION 1 - -/* Define to 1 if `st_rdev' is a member of `struct stat'. */ -#undef HAVE_STRUCT_STAT_ST_RDEV - -/* Define to 1 if `tm_gmtoff' is a member of `struct tm'. */ -#undef HAVE_STRUCT_TM_TM_GMTOFF - -/* Define to 1 if `tm_zone' is a member of `struct tm'. */ -#undef HAVE_STRUCT_TM_TM_ZONE - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_MMAN_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_PARAM_H - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TIME_H - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_UTIME_H - -/* Define to 1 if you have that is POSIX.1 compatible. */ -#undef HAVE_SYS_WAIT_H - -/* HAVE_TM_ISDST */ -#undef HAVE_TM_ISDST - -/* HAVE_TM_ZONE */ -#undef HAVE_TM_ZONE - -/* HAVE_TZNAME */ -#undef HAVE_TZNAME - -/* Define to 1 if the system has the type `int32_t'. */ -#define HAVE_INT32_T 1 - -/* Define to 1 if the system has the type `int64_t'. */ -#define HAVE_INT64_T 1 - -/* Define to 1 if the system has the type `uint16_t'. */ -#define HAVE_UINT16_T 1 - -/* Define to 1 if the system has the type `uint32_t'. */ -#define HAVE_UINT32_T 1 - -/* Define to 1 if the system has the type `uint64_t'. */ -#define HAVE_UINT64_T 1 - -/* Define to 1 if the system has the type `uint8_t'. */ -#define HAVE_UINT8_T 1 - -/* Define to 1 if you have the header file. */ -/* turns out, v5.39 file/src/buffer.c does -not- subject inclusion to this define */ -#ifndef _WIN32 -#define HAVE_UNISTD_H 1 -#endif - -/* Define to 1 if you have the `utime' function. */ -#undef HAVE_UTIME - -/* Define to 1 if you have the `utimes' function. */ -#undef HAVE_UTIMES - -/* Define to 1 if you have the header file. */ -#undef HAVE_UTIME_H - -/* Define to 1 if you have the `vasprintf' function. */ -#undef HAVE_VASPRINTF - -/* Define to 1 if you have the `vfork' function. */ -#undef HAVE_VFORK - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_VFORK_H */ - -/* Define to 1 or 0, depending whether the compiler supports simple visibility - declarations. */ -#undef HAVE_VISIBILITY - -/* Define to 1 if you have the header file. */ -#define HAVE_WCHAR_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_WCTYPE_H 1 - -/* Define to 1 if you have the `wcwidth' function. */ -#undef HAVE_WCWIDTH - -/* Define to 1 if `fork' works. */ -#undef HAVE_WORKING_FORK - -/* Define to 1 if `vfork' works. */ -#undef HAVE_WORKING_VFORK - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_ZLIB_H */ - -/* Define to the sub-directory in which libtool stores uninstalled libraries. - */ -#undef LT_OBJDIR - -/* Define to 1 if `major', `minor', and `makedev' are declared in . - */ -#undef MAJOR_IN_MKDEV - -/* Define to 1 if `major', `minor', and `makedev' are declared in - . */ -#undef MAJOR_IN_SYSMACROS - -/* Name of package */ -#undef PACKAGE - -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT - -/* Define to the full name of this package. */ -#undef PACKAGE_NAME - -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the home page for this package. */ -#undef PACKAGE_URL - -/* Define to the version of this package. */ -#undef PACKAGE_VERSION - -/* The size of `long long', as computed by sizeof. */ -#undef SIZEOF_LONG_LONG - -/* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS - -/* Define to 1 if your declares `struct tm'. */ -#undef TM_IN_SYS_TIME - -/* Enable extensions on AIX 3, Interix. */ -#ifndef _ALL_SOURCE -# undef _ALL_SOURCE -#endif -/* Enable GNU extensions on systems that have them. */ -#ifndef _GNU_SOURCE -# undef _GNU_SOURCE -#endif -/* Enable threading extensions on Solaris. */ -#ifndef _POSIX_PTHREAD_SEMANTICS -# undef _POSIX_PTHREAD_SEMANTICS -#endif -/* Enable extensions on HP NonStop. */ -#ifndef _TANDEM_SOURCE -# undef _TANDEM_SOURCE -#endif -/* Enable general extensions on Solaris. */ -#ifndef __EXTENSIONS__ -# undef __EXTENSIONS__ -#endif - - -/* Number of bits in a file offset, on hosts where this is settable. */ -#undef _FILE_OFFSET_BITS - -/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ -#undef _LARGEFILE_SOURCE - -/* Define for large files, on AIX-style hosts. */ -#undef _LARGE_FILES - -/* Define to 1 if on MINIX. */ -#undef _MINIX - -/* Define to 2 if the system does not provide POSIX.1 features except with - this defined. */ -#undef _POSIX_1_SOURCE - -/* Define to 1 if you need to in order for `stat' and other things to work. */ -#undef _POSIX_SOURCE - -/* Define to empty if `const' does not conform to ANSI C. */ -//#define const - -/* Define to a type if does not define. */ -//#undef mbstate_t - -/* Define to `long int' if does not define. */ -//#undef off_t - -/* Define to `unsigned int' if does not define. */ -//#undef size_t - - -#ifndef HAVE_UINT8_T -typedef unsigned char uint8_t; -#endif -#ifndef HAVE_UINT16_T -typedef unsigned short uint16_t; -#endif -#ifndef HAVE_UINT32_T -typedef unsigned int uint32_t; -#endif -#ifndef HAVE_INT32_T -typedef int int32_t; -#endif -#ifndef HAVE_UINT64_T -#if SIZEOF_LONG_LONG == 8 -typedef unsigned long long uint64_t; -#else -typedef unsigned long uint64_t; -#endif -#endif -#ifndef HAVE_INT64_T -#if SIZEOF_LONG_LONG == 8 -typedef long long int64_t; -#else -typedef long int64_t; -#endif -#endif - -#ifndef _SSIZE_T_DEFINED -#ifdef _WIN32 -#if defined(__MINGW32__) && !defined(__MINGW64__) -typedef int ssize_t; -#else -#include -typedef int64_t ssize_t; -#endif -#endif -#define _SSIZE_T_DEFINED -#endif - -#ifdef _WIN32 -#include - -#include -#endif - -/* Define as `fork' if `vfork' does not work. */ -/* #undef vfork */ diff --git a/ports/libmagic/magic.def b/ports/libmagic/magic.def deleted file mode 100644 index f286a62ae487..000000000000 --- a/ports/libmagic/magic.def +++ /dev/null @@ -1,20 +0,0 @@ -LIBRARY libmagic -EXPORTS - magic_open - magic_close - magic_getpath - magic_file - magic_descriptor - magic_buffer - magic_error - magic_setflags - magic_version - magic_load - magic_load_buffers - magic_compile - magic_check - magic_list - magic_errno - magic_setparam - magic_getparam - getline diff --git a/ports/libmagic/portfile.cmake b/ports/libmagic/portfile.cmake index c40c3e2bac64..34ecdb150d1e 100644 --- a/ports/libmagic/portfile.cmake +++ b/ports/libmagic/portfile.cmake @@ -1,53 +1,62 @@ -set(PATCHES - "0001-Use-pcre2.patch" -) - if(VCPKG_TARGET_IS_WINDOWS) set(PATCHES - ${PATCHES} + "0001-Use-libtre.patch" + "0002-Change-zlib-lib-name-to-match-CMake-output.patch" "0003-Fix-WIN32-macro-checks.patch" "0004-Typedef-POSIX-types-on-Windows.patch" "0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch" "0006-Remove-Wrap-POSIX-headers.patch" "0007-Substitute-unistd-macros-for-MSVC.patch" "0008-Add-FILENO-defines.patch" - "0009-No-fcntl-in-magic.c.patch" "0010-Properly-check-for-the-presence-of-bitmasks.patch" "0011-Remove-pipe-related-functions-in-funcs.c.patch" - "0014-Define-POSIX-macros-if-missing.patch" + "0012-Convert-MSYS2-paths-to-Windows-paths.patch" + "0013-Check-for-backslash-in-argv-0-on-Windows.patch" "0015-MSYS2-Remove-ioctl-call.patch" + "0016-Fix-file_famagic-function.patch" ) endif() vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO file/file - REF FILE5_40 - SHA512 d76bfe5326e1b40368e055c2e049a24b4ffdbd727371f4f3aa1dd3f53787d16b88550b3cc71ecf02151e2fb3e567eb2598e4707badab8c391eb71113c2dcc319 + REF FILE5_45 + SHA512 fdd4c5d13d5ea1d25686c76d8ebc3252c54040c4871e3f0f623c4548b3841795d4e36050292a9453eedf0fbf932573890e9d6ac9fa63ccf577215598ae84b9ea HEAD_REF master PATCHES ${PATCHES} ) -file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}") -file(COPY "${CMAKE_CURRENT_LIST_DIR}/unofficial-libmagic-config.cmake.in" DESTINATION "${SOURCE_PATH}") -file(COPY "${CMAKE_CURRENT_LIST_DIR}/magic.def" DESTINATION "${SOURCE_PATH}/src") -file(COPY "${CMAKE_CURRENT_LIST_DIR}/config.h" DESTINATION "${SOURCE_PATH}/src") +if(VCPKG_TARGET_IS_WINDOWS) + set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS") + set(VCPKG_CXX_FLAGS "${VCPKG_CXX_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS") +endif() -vcpkg_cmake_configure( - SOURCE_PATH ${SOURCE_PATH} -) +set(CONFIG_OPTIONS) -vcpkg_cmake_install() -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() -vcpkg_copy_tools(TOOL_NAMES file AUTO_CLEAN) -vcpkg_cmake_config_fixup( - CONFIG_PATH lib/cmake/unofficial-libmagic - PACKAGE_NAME unofficial-libmagic) +if(NOT "zlib" IN_LIST FEATURES) + list(APPEND CONFIG_OPTIONS "--disable-zlib") +endif() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/${PORT}/man5") +if(NOT "bzip2" IN_LIST FEATURES) + list(APPEND CONFIG_OPTIONS "--disable-bzlib") +endif() + +if(NOT "lzma" IN_LIST FEATURES) + list(APPEND CONFIG_OPTIONS "--disable-xzlib") +endif() + +if(NOT "zstd" IN_LIST FEATURES) + list(APPEND CONFIG_OPTIONS "--disable-zstdlib") +endif() + +vcpkg_configure_make( + AUTOCONFIG + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + ${CONFIG_OPTIONS} + "--disable-lzlib" + "--disable-libseccomp" +) if(VCPKG_CROSSCOMPILING) vcpkg_add_to_path(PREPEND "${CURRENT_HOST_INSTALLED_DIR}/tools/libmagic/bin") @@ -55,23 +64,37 @@ elseif(VCPKG_TARGET_IS_WINDOWS AND VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) set(EXTRA_ARGS "ADD_BIN_TO_PATH") endif() +vcpkg_install_make(${EXTRA_ARGS}) +vcpkg_copy_tool_dependencies("${CURRENT_PACKAGES_DIR}/tools/${PORT}/bin") +vcpkg_copy_tool_dependencies("${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/bin") +vcpkg_fixup_pkgconfig() + if(VCPKG_LIBRARY_LINKAGE STREQUAL static) file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") endif() -set(UNOFFICIAL_PORT unofficial-${PORT}) - if(VCPKG_TARGET_IS_WINDOWS) if(NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/share/misc") - file(COPY "${CURRENT_PACKAGES_DIR}/share/${UNOFFICIAL_PORT}/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/share/misc") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/share/misc") + file(COPY "${CURRENT_PACKAGES_DIR}/share/${PORT}/misc/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/share/misc") endif() if(NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/debug/share/misc") - file(COPY "${CURRENT_PACKAGES_DIR}/share/${UNOFFICIAL_PORT}/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/debug/share/misc") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/share/misc") + file(COPY "${CURRENT_PACKAGES_DIR}/share/${PORT}/misc/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/share/misc") endif() endif() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/${PORT}/man5") + +include(CMakePackageConfigHelpers) +configure_package_config_file( + "${CMAKE_CURRENT_LIST_DIR}/unofficial-${PORT}-config.cmake.in" + "${CURRENT_PACKAGES_DIR}/share/unofficial-${PORT}/unofficial-${PORT}-config.cmake" + INSTALL_DESTINATION "share/unofficial-${PORT}" +) + # Handle copyright and usage vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/ports/libmagic/unofficial-libmagic-config.cmake.in b/ports/libmagic/unofficial-libmagic-config.cmake.in index d3671f89b479..5ff3b0380aa7 100644 --- a/ports/libmagic/unofficial-libmagic-config.cmake.in +++ b/ports/libmagic/unofficial-libmagic-config.cmake.in @@ -1,12 +1,69 @@ @PACKAGE_INIT@ -include(CMakeFindDependencyMacro) -find_dependency(PCRE2 COMPONENTS 8BIT POSIX) +if(WIN32 AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") + include(CMakeFindDependencyMacro) + find_dependency(unofficial-tre) +endif() -include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") +# Compute the installation prefix relative to this file. +get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +if(_IMPORT_PREFIX STREQUAL "/") + set(_IMPORT_PREFIX "") +endif() -set(unofficial-libmagic_VERSION @CMAKE_PROJECT_VERSION@) -set(unofficial-libmagic_FOUND true) -set(unofficial-libmagic_DICTIONARY "${CMAKE_CURRENT_LIST_DIR}/magic.mgc") +if("@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") + add_library(unofficial::libmagic::libmagic STATIC IMPORTED) +else() + add_library(unofficial::libmagic::libmagic SHARED IMPORTED) +endif() -check_required_components("unofficial-libmagic") +set_target_properties(unofficial::libmagic::libmagic PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include" +) + +if(WIN32 AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") + set_target_properties(unofficial::libmagic::libmagic PROPERTIES + INTERFACE_LINK_LIBRARIES "\$" + ) +endif() + +macro(add_library_config config prefix) + set_property(TARGET unofficial::libmagic::libmagic APPEND PROPERTY IMPORTED_CONFIGURATIONS ${config}) + if("@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") + set_target_properties(unofficial::libmagic::libmagic PROPERTIES + IMPORTED_LOCATION_${config} "${_IMPORT_PREFIX}/${prefix}lib/@VCPKG_TARGET_STATIC_LIBRARY_PREFIX@magic@VCPKG_TARGET_STATIC_LIBRARY_SUFFIX@" + IMPORTED_LINK_INTERFACE_LANGUAGES_${config} "C" + ) + else() + if(WIN32) + set(library_dir "${_IMPORT_PREFIX}/${prefix}bin/") + set(soversion_suffix "-1") + set_target_properties(unofficial::libmagic::libmagic PROPERTIES + IMPORTED_IMPLIB_${config} "${_IMPORT_PREFIX}/${prefix}/lib/@VCPKG_TARGET_IMPORT_LIBRARY_PREFIX@magic@VCPKG_TARGET_IMPORT_LIBRARY_SUFFIX@" + ) + else() + set(library_dir "${_IMPORT_PREFIX}/${prefix}lib/") + endif() + set_target_properties(unofficial::libmagic::libmagic PROPERTIES + IMPORTED_LOCATION_${config} "${library_dir}@VCPKG_TARGET_SHARED_LIBRARY_PREFIX@magic${soversion_suffix}@VCPKG_TARGET_SHARED_LIBRARY_SUFFIX@" + ) + unset(soversion_suffix) + unset(library_dir) + endif() +endmacro() + +if("@VCPKG_BUILD_TYPE@" STREQUAL "" OR "@VCPKG_BUILD_TYPE@" STREQUAL "debug") + add_library_config(DEBUG "debug/") +endif() + +if("@VCPKG_BUILD_TYPE@" STREQUAL "" OR "@VCPKG_BUILD_TYPE@" STREQUAL "release") + add_library_config(RELEASE "") +endif() + +set_and_check(unofficial-libmagic_DICTIONARY "${_IMPORT_PREFIX}/share/libmagic/misc/magic.mgc") + +unset(_IMPORT_PREFIX) + +check_required_components(unofficial-libmagic) diff --git a/ports/libmagic/vcpkg.json b/ports/libmagic/vcpkg.json index 63944114b565..e809501e900c 100644 --- a/ports/libmagic/vcpkg.json +++ b/ports/libmagic/vcpkg.json @@ -1,26 +1,52 @@ { "name": "libmagic", - "version-string": "5.40", - "port-version": 1, + "version": "5.45", + "port-version": 2, "description": "This library can be used to classify files according to magic number tests.", "homepage": "https://github.com/file/file", + "license": "BSD-2-Clause", "dependencies": [ { - "name": "vcpkg-cmake", - "host": true + "name": "dirent", + "platform": "windows" }, { - "name": "vcpkg-cmake-config", - "host": true + "name": "getopt", + "platform": "windows" }, { - "name": "dirent", - "platform": "windows" + "name": "libmagic", + "host": true }, { - "name": "getopt", - "platform": "windows & !mingw" + "name": "tre", + "platform": "windows | mingw" + } + ], + "features": { + "zlib": { + "description": "Enable zlib support", + "dependencies": [ + "zlib" + ] + }, + "bzlib": { + "description": "Enable Bzip2 support", + "dependencies": [ + "bzip2" + ] + }, + "xzlib": { + "description": "Enable liblzma/xz support", + "dependencies": [ + "liblzma" + ] }, - "pcre2" - ] + "zstdlib": { + "description": "Enable zstdlib support", + "dependencies": [ + "zstd" + ] + } + } } diff --git a/ports/pcre2/fix-cmake.patch b/ports/pcre2/fix-cmake.patch deleted file mode 100644 index 93d2f7196957..000000000000 --- a/ports/pcre2/fix-cmake.patch +++ /dev/null @@ -1,334 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index cec7dfb..84d1769 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -100,6 +100,9 @@ - # 2021-08-28 PH increased minimum version - # 2021-08-28 PH added test for realpath() - # 2022-12-10 PH added support for pcre2posix_test -+# 2023-01-15 Carlo added C99 as the minimum required -+# 2023-06-03 Theodore used standard CMake constructs to export the library's targets. -+# 2023-08-06 PH added support for setting variable length lookbehind maximum - - # Increased minimum to 2.8.5 to support GNUInstallDirs. - # Increased minimum to 3.1 to support imported targets. -@@ -136,6 +139,7 @@ INCLUDE(CheckFunctionExists) - INCLUDE(CheckSymbolExists) - INCLUDE(CheckIncludeFile) - INCLUDE(CheckTypeSize) -+INCLUDE(CMakePackageConfigHelpers) - INCLUDE(GNUInstallDirs) # for CMAKE_INSTALL_LIBDIR - - CHECK_INCLUDE_FILE(dirent.h HAVE_DIRENT_H) -@@ -705,7 +709,9 @@ IF(PCRE2_BUILD_PCRE2_8) - VERSION ${LIBPCRE2_8_VERSION} - SOVERSION ${LIBPCRE2_8_SOVERSION}) - TARGET_COMPILE_DEFINITIONS(pcre2-8-static PUBLIC PCRE2_STATIC) -- TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC -+ $ -+ $) - IF(REQUIRE_PTHREAD) - TARGET_LINK_LIBRARIES(pcre2-8-static Threads::Threads) - ENDIF(REQUIRE_PTHREAD) -@@ -718,8 +724,9 @@ IF(PCRE2_BUILD_PCRE2_8) - VERSION ${LIBPCRE2_POSIX_VERSION} - SOVERSION ${LIBPCRE2_POSIX_SOVERSION}) - TARGET_LINK_LIBRARIES(pcre2-posix-static pcre2-8-static) -- TARGET_COMPILE_DEFINITIONS(pcre2-posix-static PUBLIC PCRE2_STATIC) -- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC -+ $ -+ $) - SET(targets ${targets} pcre2-posix-static) - - IF(MSVC) -@@ -736,7 +743,9 @@ IF(PCRE2_BUILD_PCRE2_8) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-8-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-8-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_8_MACHO_COMPATIBILITY_VERSION}" -@@ -749,7 +758,9 @@ IF(PCRE2_BUILD_PCRE2_8) - ENDIF(REQUIRE_PTHREAD) - SET(targets ${targets} pcre2-8-shared) - ADD_LIBRARY(pcre2-posix-shared SHARED ${PCRE2POSIX_HEADERS} ${PCRE2POSIX_SOURCES}) -- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-posix-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_POSIX_MACHO_COMPATIBILITY_VERSION}" -@@ -786,7 +797,9 @@ ENDIF(PCRE2_BUILD_PCRE2_8) - IF(PCRE2_BUILD_PCRE2_16) - IF(BUILD_STATIC_LIBS) - ADD_LIBRARY(pcre2-16-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -811,7 +824,9 @@ IF(PCRE2_BUILD_PCRE2_16) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-16-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-16-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -848,7 +863,9 @@ ENDIF(PCRE2_BUILD_PCRE2_16) - IF(PCRE2_BUILD_PCRE2_32) - IF(BUILD_STATIC_LIBS) - ADD_LIBRARY(pcre2-32-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -873,7 +890,9 @@ IF(PCRE2_BUILD_PCRE2_32) - - IF(BUILD_SHARED_LIBS) - ADD_LIBRARY(pcre2-32-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) -- TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC ${PROJECT_BINARY_DIR}) -+ TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC -+ $ -+ $) - SET_TARGET_PROPERTIES(pcre2-32-shared PROPERTIES - COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 - MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" -@@ -1075,9 +1094,13 @@ ENDIF(PCRE2_BUILD_TESTS) - SET(CMAKE_INSTALL_ALWAYS 1) - - INSTALL(TARGETS ${targets} -- RUNTIME DESTINATION bin -+ EXPORT pcre2-targets -+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -+INSTALL(EXPORT pcre2-targets -+ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2 -+ NAMESPACE pcre2::) - INSTALL(FILES ${pkg_config_files} DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) - INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/pcre2-config" - DESTINATION bin -@@ -1089,11 +1112,12 @@ INSTALL(FILES ${PCRE2_HEADERS} ${PCRE2POSIX_HEADERS} DESTINATION include) - # CMake config files. - set(PCRE2_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config.cmake.in) - set(PCRE2_CONFIG_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config.cmake) --configure_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} @ONLY) --set(PCRE2_CONFIG_VERSION_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config-version.cmake.in) -+configure_package_config_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) - set(PCRE2_CONFIG_VERSION_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config-version.cmake) --configure_file(${PCRE2_CONFIG_VERSION_IN} ${PCRE2_CONFIG_VERSION_OUT} @ONLY) --install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION cmake) -+write_basic_package_version_file(${PCRE2_CONFIG_VERSION_OUT} -+ VERSION ${PCRE2_MAJOR}.${PCRE2_MINOR}.0 -+ COMPATIBILITY SameMajorVersion) -+install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) - - FILE(GLOB html ${PROJECT_SOURCE_DIR}/doc/html/*.html) - FILE(GLOB man1 ${PROJECT_SOURCE_DIR}/doc/*.1) -diff --git a/cmake/pcre2-config-version.cmake.in b/cmake/pcre2-config-version.cmake.in -deleted file mode 100644 -index dac149e..0000000 ---- a/cmake/pcre2-config-version.cmake.in -+++ /dev/null -@@ -1,15 +0,0 @@ --set(PACKAGE_VERSION_MAJOR @PCRE2_MAJOR@) --set(PACKAGE_VERSION_MINOR @PCRE2_MINOR@) --set(PACKAGE_VERSION_PATCH 0) --set(PACKAGE_VERSION @PCRE2_MAJOR@.@PCRE2_MINOR@.0) -- --# Check whether the requested PACKAGE_FIND_VERSION is compatible --if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION OR -- PACKAGE_VERSION_MAJOR GREATER PACKAGE_FIND_VERSION_MAJOR) -- set(PACKAGE_VERSION_COMPATIBLE FALSE) --else() -- set(PACKAGE_VERSION_COMPATIBLE TRUE) -- if(PACKAGE_VERSION VERSION_EQUAL PACKAGE_FIND_VERSION) -- set(PACKAGE_VERSION_EXACT TRUE) -- endif() --endif() -diff --git a/cmake/pcre2-config.cmake.in b/cmake/pcre2-config.cmake.in -index b313d6d..159669b 100644 ---- a/cmake/pcre2-config.cmake.in -+++ b/cmake/pcre2-config.cmake.in -@@ -5,11 +5,17 @@ - # - # Static vs. shared - # ----------------- --# To make use of the static library instead of the shared one, one needs -+# To force using the static library instead of the shared one, one needs - # to set the variable PCRE2_USE_STATIC_LIBS to ON before calling find_package. -+# If the variable is not set, the static library will be used if only that has -+# been built, otherwise the shared library will be used. -+# -+# The following components are supported: 8BIT, 16BIT, 32BIT and POSIX. -+# They used to be required but not anymore; all available targets will -+# be defined regardless of the requested components. - # Example: - # set(PCRE2_USE_STATIC_LIBS ON) --# find_package(PCRE2 CONFIG COMPONENTS 8BIT) -+# find_package(PCRE2 CONFIG) - # - # This will define the following variables: - # -@@ -23,70 +29,42 @@ - # PCRE2::32BIT - The 32 bit PCRE2 library. - # PCRE2::POSIX - The POSIX PCRE2 library. - --set(PCRE2_NON_STANDARD_LIB_PREFIX @NON_STANDARD_LIB_PREFIX@) --set(PCRE2_NON_STANDARD_LIB_SUFFIX @NON_STANDARD_LIB_SUFFIX@) --set(PCRE2_8BIT_NAME pcre2-8) --set(PCRE2_16BIT_NAME pcre2-16) --set(PCRE2_32BIT_NAME pcre2-32) --set(PCRE2_POSIX_NAME pcre2-posix) --find_path(PCRE2_INCLUDE_DIR NAMES pcre2.h DOC "PCRE2 include directory") --if (PCRE2_USE_STATIC_LIBS) -- if (MSVC) -- set(PCRE2_8BIT_NAME pcre2-8-static) -- set(PCRE2_16BIT_NAME pcre2-16-static) -- set(PCRE2_32BIT_NAME pcre2-32-static) -- set(PCRE2_POSIX_NAME pcre2-posix-static) -- endif () -+@PACKAGE_INIT@ - -- set(PCRE2_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX}) -- set(PCRE2_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX}) --else () -- set(PCRE2_PREFIX ${CMAKE_SHARED_LIBRARY_PREFIX}) -- if (MINGW AND PCRE2_NON_STANDARD_LIB_PREFIX) -- set(PCRE2_PREFIX "") -- endif () -+include(CMakeFindDependencyMacro) -+if("@REQUIRE_PTHREAD@") # REQUIRE_PTHREAD -+ find_dependency(Threads) -+endif() - -- set(PCRE2_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) -- if (MINGW AND PCRE2_NON_STANDARD_LIB_SUFFIX) -- set(PCRE2_SUFFIX "-0.dll") -- endif () --endif () --find_library(PCRE2_8BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit PCRE2 library") --find_library(PCRE2_16BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_16BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "16 bit PCRE2 library") --find_library(PCRE2_32BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_32BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "32 bit PCRE2 library") --find_library(PCRE2_POSIX_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_POSIX_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit POSIX PCRE2 library") --unset(PCRE2_NON_STANDARD_LIB_PREFIX) --unset(PCRE2_NON_STANDARD_LIB_SUFFIX) --unset(PCRE2_8BIT_NAME) --unset(PCRE2_16BIT_NAME) --unset(PCRE2_32BIT_NAME) --unset(PCRE2_POSIX_NAME) -+include("${CMAKE_CURRENT_LIST_DIR}/pcre2-targets.cmake") - - # Set version --if (PCRE2_INCLUDE_DIR) -- set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") --endif () -+set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") - --# Which components have been found. --if (PCRE2_8BIT_LIBRARY) -- set(PCRE2_8BIT_FOUND TRUE) --endif () --if (PCRE2_16BIT_LIBRARY) -- set(PCRE2_16BIT_FOUND TRUE) --endif () --if (PCRE2_32BIT_LIBRARY) -- set(PCRE2_32BIT_FOUND TRUE) --endif () --if (PCRE2_POSIX_LIBRARY) -- set(PCRE2_POSIX_FOUND TRUE) --endif () -- --# Check if at least one component has been specified. --list(LENGTH PCRE2_FIND_COMPONENTS PCRE2_NCOMPONENTS) --if (PCRE2_NCOMPONENTS LESS 1) -- message(FATAL_ERROR "No components have been specified. This is not allowed. Please, specify at least one component.") --endif () --unset(PCRE2_NCOMPONENTS) -+# Chooses the linkage of the library to expose in the -+# unsuffixed edition of the target. -+macro(_pcre2_add_component_target component target) -+ # If the static library exists and either PCRE2_USE_STATIC_LIBS -+ # is defined, or the dynamic library does not exist, use the static library. -+ if(NOT TARGET PCRE2::${component}) -+ if(TARGET pcre2::pcre2-${target}-static AND (PCRE2_USE_STATIC_LIBS OR NOT TARGET pcre2::pcre2-${target}-shared)) -+ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-static) -+ set(PCRE2_${component}_FOUND TRUE) -+ # Otherwise use the dynamic library if it exists. -+ elseif(TARGET pcre2::pcre2-${target}-shared AND NOT PCRE2_USE_STATIC_LIBS) -+ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-shared) -+ set(PCRE2_${component}_FOUND TRUE) -+ endif() -+ if(PCRE2_${component}_FOUND) -+ get_target_property(PCRE2_${component}_LIBRARY PCRE2::${component} IMPORTED_LOCATION) -+ set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) -+ endif() -+ endif() -+endmacro() -+_pcre2_add_component_target(8BIT 8) -+_pcre2_add_component_target(16BIT 16) -+_pcre2_add_component_target(32BIT 32) -+_pcre2_add_component_target(POSIX posix) - - # When POSIX component has been specified make sure that also 8BIT component is specified. - set(PCRE2_8BIT_COMPONENT FALSE) -@@ -105,41 +83,5 @@ endif() - unset(PCRE2_8BIT_COMPONENT) - unset(PCRE2_POSIX_COMPONENT) - --include(FindPackageHandleStandardArgs) --set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") --find_package_handle_standard_args(PCRE2 -- FOUND_VAR PCRE2_FOUND -- REQUIRED_VARS PCRE2_INCLUDE_DIR -- HANDLE_COMPONENTS -- VERSION_VAR PCRE2_VERSION -- CONFIG_MODE --) -- --set(PCRE2_LIBRARIES) --if (PCRE2_FOUND) -- foreach(component ${PCRE2_FIND_COMPONENTS}) -- if (PCRE2_USE_STATIC_LIBS) -- add_library(PCRE2::${component} STATIC IMPORTED) -- target_compile_definitions(PCRE2::${component} INTERFACE PCRE2_STATIC) -- else () -- add_library(PCRE2::${component} SHARED IMPORTED) -- endif () -- set_target_properties(PCRE2::${component} PROPERTIES -- IMPORTED_LOCATION "${PCRE2_${component}_LIBRARY}" -- INTERFACE_INCLUDE_DIRECTORIES "${PCRE2_INCLUDE_DIR}" -- ) -- if (component STREQUAL "POSIX") -- set_target_properties(PCRE2::${component} PROPERTIES -- INTERFACE_LINK_LIBRARIES "PCRE2::8BIT" -- LINK_LIBRARIES "PCRE2::8BIT" -- ) -- endif () -- -- set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) -- mark_as_advanced(PCRE2_${component}_LIBRARY) -- endforeach() --endif () -- --mark_as_advanced( -- PCRE2_INCLUDE_DIR --) -+# Check for required components. -+check_required_components("PCRE2") diff --git a/ports/pcre2/no-static-suffix.patch b/ports/pcre2/no-static-suffix.patch deleted file mode 100644 index 7f41bcd566cf..000000000000 --- a/ports/pcre2/no-static-suffix.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index fa2181e..3bf5317 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -711,8 +711,8 @@ IF(PCRE2_BUILD_PCRE2_8) - SET(targets ${targets} pcre2-posix-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8-static) -- SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix-static) -+ SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) -+ SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) - SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) -@@ -777,7 +777,7 @@ IF(PCRE2_BUILD_PCRE2_16) - SET(targets ${targets} pcre2-16-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16-static) -+ SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) - ENDIF(MSVC) -@@ -829,7 +829,7 @@ IF(PCRE2_BUILD_PCRE2_32) - SET(targets ${targets} pcre2-32-static) - - IF(MSVC) -- SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32-static) -+ SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) - ELSE(MSVC) - SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) - ENDIF(MSVC) diff --git a/ports/pcre2/pcre2-10.35_fix-uwp.patch b/ports/pcre2/pcre2-10.35_fix-uwp.patch deleted file mode 100644 index 476dde0f6a4c..000000000000 --- a/ports/pcre2/pcre2-10.35_fix-uwp.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- a/CMakeLists.txt 2020-05-09 16:43:10.000000000 +0200 -+++ b/CMakeLists.txt 2020-06-03 20:57:17.026182500 +0200 -@@ -619,6 +619,7 @@ - - IF(MSVC) - ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS) -+ add_compile_options(/wd4146) - ENDIF(MSVC) - - SET(CMAKE_INCLUDE_CURRENT_DIR 1) diff --git a/ports/pcre2/portfile.cmake b/ports/pcre2/portfile.cmake deleted file mode 100644 index 5d6c5c39b6a6..000000000000 --- a/ports/pcre2/portfile.cmake +++ /dev/null @@ -1,73 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO PCRE2Project/pcre2 - REF "pcre2-${VERSION}" - SHA512 3d0ee66e23809d3da2fe2bf4ed6e20b0fb96c293a91668935f6319e8d02e480eeef33da01e08a7436a18a1a85a116d83186b953520f394c866aad3cea73c7f5c - HEAD_REF master - PATCHES - pcre2-10.35_fix-uwp.patch - no-static-suffix.patch - fix-cmake.patch -) - -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" BUILD_STATIC) -string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" INSTALL_PDB) -string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" BUILD_STATIC_CRT) - -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - jit PCRE2_SUPPORT_JIT -) - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${FEATURE_OPTIONS} - -DBUILD_STATIC_LIBS=${BUILD_STATIC} - -DPCRE2_STATIC_RUNTIME=${BUILD_STATIC_CRT} - -DPCRE2_BUILD_PCRE2_8=ON - -DPCRE2_BUILD_PCRE2_16=ON - -DPCRE2_BUILD_PCRE2_32=ON - -DPCRE2_SUPPORT_UNICODE=ON - -DPCRE2_BUILD_TESTS=OFF - -DPCRE2_BUILD_PCRE2GREP=OFF - -DCMAKE_DISABLE_FIND_PACKAGE_BZip2=ON - -DCMAKE_DISABLE_FIND_PACKAGE_ZLIB=ON - -DCMAKE_DISABLE_FIND_PACKAGE_Readline=ON - -DCMAKE_DISABLE_FIND_PACKAGE_Editline=ON - -DINSTALL_MSVC_PDB=${INSTALL_PDB} - ) - -vcpkg_cmake_install() -vcpkg_copy_pdbs() - -file(READ "${CURRENT_PACKAGES_DIR}/include/pcre2.h" PCRE2_H) -if(BUILD_STATIC) - string(REPLACE "defined(PCRE2_STATIC)" "1" PCRE2_H "${PCRE2_H}") -else() - string(REPLACE "defined(PCRE2_STATIC)" "0" PCRE2_H "${PCRE2_H}") -endif() -file(WRITE "${CURRENT_PACKAGES_DIR}/include/pcre2.h" "${PCRE2_H}") - -vcpkg_fixup_pkgconfig() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${PORT}) - -file(REMOVE_RECURSE - "${CURRENT_PACKAGES_DIR}/man" - "${CURRENT_PACKAGES_DIR}/share/doc" - "${CURRENT_PACKAGES_DIR}/debug/include" - "${CURRENT_PACKAGES_DIR}/debug/man" - "${CURRENT_PACKAGES_DIR}/debug/share") - -if(BUILD_STATIC) - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") -elseif(VCPKG_TARGET_IS_WINDOWS) - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/..") - if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config") - vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/../..") - endif() -endif() - -file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") diff --git a/ports/pcre2/usage b/ports/pcre2/usage deleted file mode 100644 index a8e97871bd52..000000000000 --- a/ports/pcre2/usage +++ /dev/null @@ -1,6 +0,0 @@ -The package pcre2 is compatible with built-in CMake targets: - - # Each component imports a target: - # TARGETS: pcre2::8BIT pcre2::16BIT pcre2::32BIT pcre2::POSIX - find_package(pcre2 CONFIG REQUIRED) - target_link_libraries(main PRIVATE pcre2::8BIT pcre2::POSIX) diff --git a/ports/pcre2/vcpkg.json b/ports/pcre2/vcpkg.json deleted file mode 100644 index 3dd6b9455d55..000000000000 --- a/ports/pcre2/vcpkg.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "pcre2", - "version": "10.42", - "port-version": 1, - "description": "Regular Expression pattern matching using the same syntax and semantics as Perl 5.", - "homepage": "https://github.com/PCRE2Project/pcre2", - "license": "BSD-3-Clause", - "dependencies": [ - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "platform-default-features" - ], - "features": { - "jit": { - "description": "Enable support for Just-In-Time compiling regex matchers", - "supports": "!emscripten" - }, - "platform-default-features": { - "description": "Enable default features", - "dependencies": [ - { - "name": "pcre2", - "features": [ - "jit" - ], - "platform": "!emscripten" - } - ] - } - } -} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index e245c55109fc..2d59b59cd809 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -431,6 +431,8 @@ target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) ############################################################ # provide actions/target for preparation of magic.mgc data for embedding/build +find_package(Magic_EP REQUIRED) + set(MGC_GZIPPED_BIN_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/..") set(MGC_GZIPPED_BIN_OUTPUT_FILE "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}/magic_mgc_gzipped.bin") set(MGC_GZIPPED_BIN_INPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sm/misc") @@ -449,7 +451,7 @@ add_dependencies(TILEDB_CORE_OBJECTS gen_mgc_unarch) add_custom_target( update-embedded-magic-data - COMMAND "$" < "${libmagic_DICTIONARY}" "${MGC_GZIPPED_BIN_OUTPUT_FILE}" + COMMAND tdb_gzip_embedded_data ${libmagic_DICTIONARY} ${MGC_GZIPPED_BIN_OUTPUT_FILE} # need to work in 'local' directory with no prefix paths so no paths are included in archive WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" COMMAND ${CMAKE_COMMAND} -E tar cvj "magic_mgc_gzipped.bin.tar.bz2" "magic_mgc_gzipped.bin" @@ -618,7 +620,6 @@ find_package(LZ4_EP REQUIRED) find_package(Spdlog_EP REQUIRED) find_package(Zlib_EP REQUIRED) find_package(Zstd_EP REQUIRED) -find_package(Magic_EP REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE BZip2::BZip2 diff --git a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc index 6c15ad3d66a4..23ac215c97a8 100644 --- a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc +++ b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc @@ -75,14 +75,25 @@ int main(int argc, char* argv[]) { // output stream. if (argc > 1) { - outfile = fopen(argv[1], "w+b"); - if (!outfile) { + if (argc != 3) { + printf("Usage: tdb_gzip_embedded_data \n"); + printf( + "If neither nor are specified, they will be stdin " + "and stdout respectively.\n"); + } + infile = fopen(argv[1], "rb"); + if (!infile) { fprintf(stderr, "Unable to create file %s\n", argv[1]); exit(-2); } + outfile = fopen(argv[2], "wb"); + if (!outfile) { + fprintf(stderr, "Unable to create file %s\n", argv[2]); + exit(-2); + } } - auto closefile = [&]() { fclose(outfile); }; - tiledb::common::ScopedExecutor onexit1(closefile); + tiledb::common::ScopedExecutor onexit1([&]() { fclose(infile); }); + tiledb::common::ScopedExecutor onexit2([&]() { fclose(outfile); }); #ifdef _WIN32 // need to be sure in/out are in binay mode, windows default won't be!!! diff --git a/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 b/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 index d23e06bc82aca192b974977acb1ae32599d28094..9c821f153b8244291ea1a350aac770069f561993 100644 GIT binary patch literal 410953 zcmZUZV|N@1!**laZfx5pw#|ucyC=5oHlEm=Od8vna5Awmand&JdEWc`d;N#K*4}Gf z3Kp&s0`lgphUTmlUB6LJUTw1V2;>s}Jww1iK>5EKfF25hebzrwF$a6OO&mm=Mz>Lx zIv^15b?-yy?K);q-$&5GZZ*xWqK?XG#L&S@9;{UVZk;a}n$&J|ONu3f-nz9`vHW2d zF@No0Xb&o7+^Fyg;MxH3t_1I_Ub$K0hHv6^>f<{%-MUr4x9;iy_wyb#D%v}NtDH9=w`TD z3NW-?H8>@FGq|&^UPGohC^frAG|z$k)v2>NB7At&*qZ-7r_Xp7*0kU`nctfWl&NsN zm%bc0n6at`Mf87{y!E;V4cr{SMA4v%-RiUw5<6Q<=3)T&$54TSPN#wwn^jLk+x?Yz zL+}YyyA(*ZwR4MNDKm}oE&|vatSsk@S1K?9Zv~c_U%+`+{BG++U8}b4An~~iGH`2f zM^b-HL%Z?b6v|tJuJ@I~ee(98^K~1vzR7|r1JncVVXNVMvxC*DG)K7ORX`(#Kschp z22Iy(yYSmC2IWmR!u|#&s16j$j;bwS)TY82A>v`!LSj&cEmFKjb-mHbSeq1a5Z=fY z0vcPC*$ww`e(Kd*P;lR=P6&U4`S=7Fr~w)4f=MnkydNGk2v_y$i%~%P2u@_@xMw_Nx(ze2(po;!p zw~cMRrJyABKG>%G8dtM-cT2ZBi>j#@3TZP>gG;bxSpeu@n=iO&h-3O=h>vt5!#VM- zvn#33O$sUae!c-ZbsN6PJL&}+%&(K`w^}S9`6Z$YxbO&H@ltq)2hHc~EcN15?1tEt zjl4HMcr6Y_`Df?cSMIi}Tm~}`dBZb%YE1bp8A`|F0*u%S)tX48Jv22g1xY}whP4DSvGyZfN4c<}S_zw3L23v*R?w{@hb$up>*7{p1_aP=0r@h}4Be zd%eM8hrFjO?Kj%`e#=}oU~HZ3c7q0`Bo^uJ2XC{2K_o+iFsVVys0V<>$llMMRL#{r zYM+n@Shum^!Twv4cD;Tu;*r_ z7+Y?ABm(5$Sk-UZ>4xnx43vI8;QXBoVY;VaH+t%kC21kA)URc zeC^h^){9i+LKb(h{)pfYW2udBxVz5c!G!jL<^Cp%l#>XbcRPS}4mJR)Kx={Zb?w61 zBO0wu^7hLNgC3F_qKUB@Mg9kjvNQswzR>{?Sq>pJlu|yoDH;vK5yfyiMtIIx4jDxd ziQ_IhP6MbC9UhHfV2mF}?gczGs-b4bR8&+7MoHa`ykrIGmW z;rRIf41#nu_}>3JNicn4`Y8V@L=E$Kx(~@N+RG?8i!2VQgA$3uP!u!XI+q^(QbDh! z8QNM-tW_#PpKaX%!0#Ig*#5R8s}XybNf|j_MA#wg4dD^$wD_SaOXwS1JW`QgYdh)$ zj(`?GvDIBPx^)GsdNXa74YMpuCn-mmpB9D5FM*ta;T~oFuQgwJ%sXs}JBghyg*;6v zv4gTjPwg57lE5t2B|T41s>85r!;L7h4>a+3d<7(IykF0=6mOs3h|W)uz6W~#avUtl zCBgWFh`RGtpuO}@D~9`?U(xeH59FeJ{NJ;$XX=^o%oR!xHVetR*3uUlY^jpd)7v+_w!4?s(ad{JFO&z*0nV&g1%keb5aZI=uAqlf(-oAtN_RhMgOh&wy-sR< zQQpl<`Pdi$zpZ&D738L(jkPIA%5VI#)4z6VFjr5*+cm(!9VvoK>e}mR!X|GRI`iC; z)O-I%jc|mob3K~2mI0>uI#YQpYw;!}&VLo13g-__T?%^x;qfs#ky9AaiXIX?Aa8q5 z$o&_GrbR`OWB>{?P$ciQ@h-+vqP4G7xN63E415bbIlC~2MT-I{j&t$Omml*A9CB3I;99vXkc zVe;d=DRXf954mCz_lNC!Hv&XL&l7{TG=$g5MjKx_Z*}vt+cLW9tO4p68{X?eTJl3Q zrb6#@spoAbVbl6_5azOlj>oy*7GnLT2>+xKHt39g&4Kj)%XSOO=BNyeGNyOM^m zTv_J-dMn%cP`sq_uBX7~n~1fgJDg-C)?DQ=#n32kY7fn*NvWi31H-a zVc!)oRT4f=Tih75$nm1J=jh_4P;r;aC`6Bc6a!vpyGH1l?tfmzN@+TNYx=9N7=fiM z`R54FXJhm;veF~soPu+=LDY+$eW+fZA{(OI;C(G<@)99w?+O$;fYu_7!wXZM57LN+ zzL@QbLa;2x)!`sTE3~oap9I$}nu6V~=hG89c=6@mW@!GEN`Xn+nfn6cf1x(=bL{!7 z|CQ8uVOqtwLY^lx@>NY^wH6oV;PJtt7s(awx;O`LL^F3DYU@mYv;h{&rgh_{cX!%{0|Ur04=x z0A)wahtZCZ{SscVb>_S?gVW|J6Cc(F7tOLYgkh(VLvb%ju~TT+u9yMnt&_2lRoss> zC@?%7GdhvlT(34v_JsAddq}7dK|W4Ono|Nb`J8ns`h2QaQv1f&nyyWKSQ z*Vea}jTu}p3w_mF&QX^$ zTC@#EU3($C*=tz9HKH&yS`w;(cAv@`H>r3|cxH4(w4sju4dpQw*`fGf*m%S|C(1Ha zS54G3rj7poI)@eMN*tZ`@P#F4ulStvQy@mGV5~A!ar%D7HvEM^&14T&Y^If2RLXwR zuDBtiHfDAfLi$n^*(~K!E9S|_iOA*8CYm<hM`QLy?t9CPmZpKpKUA$H;3yDDx}8mK6vtf#G=PS?S(h29>WJJ|W&8Nk%tS06c*Q(t2~VyZQ*O@4?Rvg zN*1F|f^Xt-PDH9zS;b)?S4+h_@FUBkDEaFvdmQ|zNw51O_F}4jt(Z-fvYyvn_=C{= zpYdb{)-29(L7trSM{hxQ3DM!u82O(KyQ%Mdi?dU)E72T&--+u#{?i}^Bplp z!0f0CZ6m_cam2{1WbZ@ghY3M6Hzabb6O?rOWWHbwn%GHne9qsql_rBiI|?OqFp#ajIAB?gYXh zaiF7N(CYcok+YFCC*Z+THBwH`olKmxu}grvwJU9;1^Q&F30(w&*a+=ElRG4P8ptF! z5dzB32I4|_bkOm_eJuI(u3OoSdXMG_w--!@04jnx;Hv1{-H8=b2e!6Sjg_MKmUy{* z3!^lbO8z9ASpa!C9o#5~@`#X#@xX3REg84hA~@hoL%(ylFWq((gzE=xe#HKIiSar= zn>Y7X`p4Zieb{!~2Tz85kD=K4dx9axNpc_4Osy|K!R{h(k4VwsC{vQKPmD9X`z4ex67w@krh*v^IgWnmCDuY!CZi)#~ zxfVwWc?bV7AMJY8#aU^pAwVa-!`RM5;WvCh`(1v<`b~?v-c5S0#9sr>4qYN{cT3lM zluDRW4bTjLW(V53B*;?omvQy;apLUc{^NWdffD0gn1RAh7G3Qtoy(kJEyN@?Z$-M4 z9X|}UxIBK>ke2r>KcV#kNt_&U!icAoDI-IdXP2wt>{>w!8Y*GlW=sH)JA*#~^OqL< z(@LWQy~Tx?U!&L{9f$nQbWD~Jg#~@*DiE(6xUe5uWLUlAe+Pa{^~3xz>Y1o2V!d6j zl|h49)vJIDtnuH;lX12x{>pocJy9m&+79;ALcJ>IKdgsn>(o-kXAK>_)yrf=HNHmB?G=PX z3%yXy+wHzStzLSY4K1eKD*yw7pil93>G#H_GbP73z&AmnaY0Oy-!wn;OfvYUw4%}+ z+}`12+%iIHV)hq_#=vVYq$h>jpG3-B$j!J^M`>ria{v7G2Ne(3$y=wh?P%fK=*=B{ zUIZ2Q%;-79NXjKEN)PjD)_ImmoNh=Ie9&`9jJt%J)!gP@HfyUrPxedEH0`de8}Kvv z3zULTyhxO>DSA*ds94|_;~d<;5I&j_@O?REu{}hq0`aBqIY<&~2{P65* zuWu878-B#@A-Vu%+kxGhfpQUuZEJ4=Pca3+6jcLgGCM}HEoMZ6UeG=PdqGJtEU_Zf za~pc{eUoSqvx}0AD7G6bc4ReCUD9iK-(r%#fLozQY}tII4OvNJ(uy@FShI=@o}7*Wxx42NET2et zuk6GUk9qlj_-iLhjX84X0#N16+d?J?+2nsu^I13cfpLM_)!bP3izgRuXK6D0kGkgiBIbCKm*Y zmh`I@{p~c{FT!kNcAC|$F2%L$i(>v`0q*%9i&zqczE#eW!#6~QDLaO>)LNlF2nH;e zIA_1VhE}`|aZ$5l6%_f==*pQ28_WZKkk6i!AsdVn0c$f9S3?Nq4C=s=_v>d ztEjY)V-;22qT4Uj`Q6hOI|(ccoO{5l*K1LtA5-H#ToyqL%`%BhojodMopmT0zTsja zh)tWyk9Np6tE82tHLg1UNAr&`pBS_V*no(%eG8+$;dE-o!2A~86n3WFc##EMI^O)g z>Z(xRtnz7x3tt-!#02D=rWzI1&};js9dqj1nnr(QTHNoUtA3khVSb?yps4EyvZYIGObPJFDq%KK>vxLmYI?8~IdhY!g!zW@~g) zK4#F`biMIi=J|bY{#ZkYDW&Xk#t6$A;(Ic+(AV#xE*qW4dRy9@hH_HM7g5_@|01w{ z>IR^3i?skIB(q;c8G*=6ZfWWKz3Io0$>X_o9&<}I7}jBU>@Qx;Ps3Q#RQANd8(5D9 zlJ4J?xMdC!t*NG5@r|^xt%qr1yw(;^8mUNku1BzY|#}H2Tl4)^<0!%4kD(+hz_k3-)`@HSdrt>{#)N34BEax@4 zG|PmRW;zr`LLzottR7BoJ_J?*VA~Ed`0q=uu8{-W(xq4d0pYFm!a5p3oUObN$lwQ) z4q_bufYfB(siNVEnQ~{pzDPM*!c$9>rk9?Y!b z8Dbtk%y!LaIohJf?w;({77up&H%xKu=WeV^`%0pT^kQ6|YDkkvi<(HA+iDAck(ntT zRm}Y))qOx+YoLd%6cri1wi>9mRw159Yyg1ia&go+w_Ge z3gxgBd#@4FLyt}9jPqNQ!`5Mx0=0cy#p(AZ#q<@d=0bF9FP17LqX1a>s%*;kHw}Du z=ldnCge4Swv+gRoe|v@um!_Jl9?fT*R(~=MvcccfxKisP`N7)u*k3)yp6yM#q)bte zfDivB6W%7sGuRX7r}hxZ`O1V`=NOgud&aMpm!xZ=q^`k*YsRYJo?6m&e>LW)}IyMvMn> zjYD4$8}TB#Nqb^DL-xBSmDt@NZL=9FVd6!al22!KhSfi$X;@!d<>h=~ZWT!>&v7zL z_9TyS+0;XK<0zW5j0m`Ca*pD1k@)zYY_Ax=@89ME5UO{BIW$6-Ddl6A7>s^?6>grW z+1JGltf{RtM<6mN;%X%)&KMb!b5NXCB>?Bs{=O*_s#j~Qi?qwum@F>}&hK+I`XjmT zc8gnoy|bI`o#4;$(NoH6{^Yd!D-hp7{r!kf1(%jbb%Io`m6Zbf2I3z-XoyINlzHZ3 zC_2>{HZAwakKe0mPttkiD_%z)ftjgvXg&VOyg`T<;T&0GjqZ$X1O+z2i)j-1J40QS z@x7cZqwr3*hLzEcpQf(QldeN2IZ38e$F18-nuG)k^Xprin!DfV}gZQtDcRXAZ% z@paG$3LybD6?4c4&KzUFSl$%JTs7(%1*t)1mJ9YFN}%0*yp>atPYL_PK&(c#&m2JK zg<*^UWQ7pTda|#g=khDgZi07Kr<;ZStw?z zNZ4^%<$dQ~oY~oG^-<1>er`>%XyRpmb{5+p4hXmSbsvu*=$^QT%ZmSyJzHVzX8!87 z!V$v^o-2t{bHlieQ~i8RJzRS))3tbUX#iXL4@6ReH&FR;ZoPUt9m%pl*Nm@Yi8b0{NX-^U|QZE`la13Q?xl0bZN)}<8-;9eGFylR*v`*}j@H3P<#M@05! zo*XlmEm1s!Ta32}X=>-ntj#K9C&rvf#OD?F|0wQGWspZ>i3?b(LS8hQl{qn>K<E9AMHEAy4+V1hBMkwL_OT64yUx6rAN&4e zauza9P8Z&{HP#?n3>*;b+5nOjGTcNwEQdpe+;RgPITPZhf7|^xc9{EDzvYTG1vkXq zlp9ER%#YvcgO_%yXfnW_S%vOAwTmcV4zZ&Ba9Io2Q23&NY!x!`McpSrxV?~fb0Vp& z$@g|H7V`qRj6g!7NdI3=r+qNqcPmy_AtH)2Ix=C9^TN~$`Ipm|lh9JQ0%qpG=^CBJ zqa73nm``tFAy?M{;(q38ZoRKBn+(+|?aar+C+q6>Nv1QA)YWSeoy(h7bjU5sCqNyF zV8~+-6lPElD9V0iyaS}&;_SC5;n|x#c~ATlcf~sr+#TPZw`?F4a~S{1O-yWHO`gyn zia9Z!88$>2UG^2Jom&9`T^nyJvPMJ(Vqs8=Vd~r!qd>Dck%y;YNfLX~!QSED_lsei z!zd6kZdm{@oSGDcn5gCVh%T|&SKg&!QwyI}c~GTawa|GwcZM5`>Xh3Aol)B$h7K%Ij)(M~z@sdTx|PB9;1W0{TQ?7NpN)cC zf9zw6fmOzqBGU<$BN4a^7v3x0Jaf6x_AyBe>(EfE{NY|?46FmNfUgpaZ6aJS7N*SI zDT6_yyaTbw$L8d)3NtsI$UvWe+A>#DHqOe711?X16yPB#j2c36d}{d`DlVgfVe;zHXR zEn(z+$Ih@m)T`Z`Q`fL`44YJMuywet&XLtzdh4~s&4 zzCQf?NbAv8{bR;@A2ovF5fLsNUlk80O3;-267M>2w2AW&@(lFKL~ z3qJV5OryAt^ppFcnvI*9R&9^S+_cD+8{W6>E4r-AAW%mn&@P|O->C>Ktjw7zIz$a1 zl2{J??7Ew;tYPZ~`H4O->J9r*xhGu#*1V+zE#G5*4%CG9CbR&b@0cYf zi_nK=vZkB@@tCuW9=NVOUK7#QUyF8gdF~DDRZ{F(dN$8Cgsz2{jAL(ZWi=!jJVKk# zk`kNzMLWpMn)e>al_cKb1G%Fvq1qk8A}0Iu5=dwwC_5V`Asxn)AJr3Lf~!-c$sF8E zi^va!c6X(Z_i^D3ih6uj# zGq}M3w+3~$n-x7y&(hV?FvNdLhhSokEr{D_&NWJ(gE$!w8R{LKTvoF;lS<3qE2vTL z{d{09f{2SV6xwrARb{zMtbXnSl9yA?k=5E0iY-g2C>0eww6HvU*p(PLRHTeRfcKT2DGfH%d> zu_ZhSMA}xXnC7?dhzB_uUpXLVr<*q}2xUH&Mwvul*{NsLB7?hGf)c!zx%0j^b)bA9 z5^!@vy7U`hJ=bV@o#Emc!|-t}I+kpEJ8ua6SYFcz^M`2wzL9d9s|UimVJ3&6yD6gZ z_DtpU3O89gN_4N?thneSQi?22^-MF8i_AUQ*IxP&P4V~!GfYZqx@tvjSC0b9%}=Y< z!cT>Nhq6Hv0;%&{1?!=f2h|Q6HmbL zqSt!t+nJ)HyQpmgtO#zFjSpK3j z9;``RI(?G%K`DQ|bFYrp{K**`eY`CR2G^(OBkb&;8XBMW)dJ!X0w1-lhIG9z6j7TY z)4o;Plri&$oP={-C?sZ#BE@6SX0;FgkS*OD!D%H1@X`A7Ia34HISkLTLXw_%-ZEP; z=Gy^jfQQ98doW6it)4@!)*4Put@J;Bd;F?B~)=)qUeBKo*=p45IaX^Kk97xz2oJ8jJmzw z3q!5n| zs#er|3?I9SHQ1|xSvNPXGMLb3tQw2C66AIm5q=@*RtmB}$8WR-R%BA-3l<1v1M{|L zVr9zB+k(g^s!$bW?i;bAqKzmmPRIub9uE^&KsKPF|0 z$mwick$?Jk-O-4Jxjoo(iBn}B2NCg?C42^uA{}w7@#*#oq^nW6#Owz`Nq~=Dt4iK3OOf(I=iJ(7MKB`dERv(qnJ*Ih~Od z-cHHTG9uBI$be2v6Y(G6xVq&(lq;1q~uxp4oq;vcQc^*S{h%MUfH$>j}_o8iUI(`99 zJ#5W?YAu%xb1_rtZ()pZROZY8xw$DQsCRl?SQQSHckhB>b?F~^PNYKcZXn1EDcQ2w z_LqTJq>p-N=)##^T0?=> zX8O=YQ(PP+`h2s5pA^gQNti*F6B+OU%i;NE*LW5^C4ACv8g>lTWuHFT4)&4elVRH% z-$kA?ll390g?A51f6=2k&>M&ZRz>To+7E0^WJY_dL0D;#fw3;H+F0+W+8bA*ia+() zgt66cqUo9gZ@ZnmcqGg9ezN_T@(gGa2`FHyCf zfW#;mdBler7l&SPF6`Bf{CCs5-WV}74&eD0F{Lm~Yc4Aq35Q$loLK;KPGj_LgUR!uMy6DxI; zD-OQq%36Od*NMrZKECL%pX8tSNok8a7iE)fTl?oW;a^)bNgc;V59&gR4ILN`_I3-p zzW=!BYoci5U_gfQ;&58ci4K`7i#kv;KG|GqG*-C-?hlOxi7gW-v0993BN&k;MO)>j zv>&njhXJmYpQT@la%vMkUDw;|M%xHb(`QbQGWj6|q9=$P zfRf6`JDXiFO5-bTakhIju7*4%Be|V%=~X7aAXV;@*GyjhqMwSpJke;Q{B3A{Ump{* zm9GcSJdbC7q(q1<-TzXV&6peIiFRxMi>d{i(THEBf8<|d+=B8x9g?t6XI(S9e;Sz6Wg)o`HU4%De$BpZ0l<*}FI^%lj6 z<|7s#-v+iBW$R|=obr491%DHg1sit|XIf$dEhmcsUcaO0u~HS#s^Sn}#Cj(Wg7_wp zIN62PBX7;zF&SZa&5&D)=&JiHtFGVv#uwJ7aw?8XYfYCfw?`{PP4pKT=oIWhn9NJ4 z+WiXdZn@eRhivgFq>rEW(whD=jiUhj2lHpOHpxWtcJnj2g?&9jedVzAUv!#rSfME@ z)ZSS0049rwlzKA`H6FQ2xB!etHNwV{70m~SIM(TgOpS2!UrI_3{}~h?+|NaF_lg_Y zNoxj|o;Pmi_Uu{Z0tv&O7R6+unz=xv{i9sd^s|zhtj$eAzqqdLhFbl&!i)m9@wT|5 z{H}>2v~lk!l8!&Vl$hxG@>MnNIv!<hU2;r%THGpJchO@2wE4>D(-jFGFc6GxCo` z$d20W&a-F?V%%aR%DftR(!8VZYova-QNDJDb2O8o&ke$OxvI7I4aK4x z%Wl`t5?DpP7Qb`I{=Re^2Q3fqxv`|VJ^I*n{?9_!9FUR$?F?t!;^mahU`zENQuhOL zL!g#hkPzp|RUt3Sy8MhDv92FxYI=J?$dm_u#qOvOrFkRa(1?gm_BRw#4U4Qnn3m!? zNXR>^A0<73V!Y&TOGs@nE)S;Yp4D#;`QF$FkBWrSH0m%Br#mUh$haJIJ;u6 zK`NI2l8tlD@v2F`4ASijDAG21rs3nu?J0f3)Xlk4QOP%~phv{Kw69@F!-9<6I(&bb zbmnvzL>1z}rz-ufT8rkyc+vEj^2*7!uvk(`;()%05&F(oBd5!VBYMskEX{Q&`IYSLx{$t{f6FPcqqB)ats*IMZqN| z_WQ8pL|uv3iuVnt7ws>mRW!X2B1yoU&_f$1d0D)!@hP=D!I&~8NZzo%py*3XM2h*1 zCwA*e0?ehQGxuvspOLb6AW|E z5_S8QfGv$SR&Gh@z~F$|6rn@_=iO8@jMF9IV#j54y~$W79S(S^GKp={{#iO}MOD5v z-jywHVe%|6x{b5?s*O-8fOe-spoRW0+^C$ zRd^p<1#mM49G6e&#S2O3_Ew?4Oc|8YWmI3N)hHpvUD1Dzi&`DX*lzDO(hW6qMKpZ? zf)e=L*cUYVdsWmvn@8wFra|Z=tFL2grVbFHFrUV(_Q~J2cAX9$@+%WyQncNI3(!Kq zSTS81MUEdK`hojGV;CB$>|Sz>ibCNEpV;ziF=}jA!WB7ZONy|(4%Q9zmG-YyVfUsi znR&+;9YS%+5)8Y^P(b7r zUU4HOn)-OmzM8u~zjNxg2iki-Ul)Dkyq!m_J>iG&oFd5^3Kt^ayQQ{heVW7tvM=@S zN|;OkT&lm1tBawjH6p3WufHPF&w@x4?T`;V;shL!Z`C=3K7xy&DoXyB2^kUsea&M4 z5az%kfk7GA*!shZPCa4ht;#73Ky`>_*++dPV^6avcM$gM+(5pnL=V?{?NzubN7W0z zz11GN!|Teu7o6FA$*U(cj-yq!KyNy86k?k3b40S*aPSi$!;^h_G1}P-wc!}c`MvH@ z=zV`Xlw_kAFjbsThE12e0Lr)zK5;X#^Ti9%*fw+KD97;K18Tx~^DQoe!=FZfMP$x< zQ_u(r&54Gulc&m!=7kIekCKT82jbP0r#A&FwYWpi`t%5W4Wdg z{Oz_O1C%rAbpChb%gZ-dkEX0Yx0!3stR87C!J)jGVSQCZBEQ?pNg26u@H7a<8<2S% zHi1}oFU9VgYTlQ-z9IzAPP#&DD2Cc+mw}Br^s7qGMe2uTu|DDQ7H!lV#0(LO)Al$d zsdeNQ+P8rinXYTwv*em*C?^yUyAm}~Cs%F11A^z~#4@^!5gkcEhYF&+g=Fjv4{oVF0OJNlGUtrl^u+p2i*@>IWS^w|IO!!HUS)dtPv{Yl zK;F2@JUSoc7(Ke_spHB7V7w-yihXrdlNq#QtU}Da40@JvHLgYPSCAp4s+q44Ls#{x zII9mc5Fda~e^IRo8AEcWLA7;SxKK>4m0_uE6WCeqsO+4qCrbKPCRb-UfI}8D7s{AW zT%L!*)FAv9Eu@NvX!q&{;!(S$IELC!Ew}*r;rXybeoJypHu(^s8l7i36@Ma2uM=>- zdaV3(id6ITv-oqEvJ)bfNT6PcW&*gO@vsf)zNuT>SEj0S41&WK&zi+JV~?ta9nBqW zc;YSB0<&fFe$B(UcB8)dHUe>EnpAFz zO)YtK+hZU)&1)!&ZhfPydbr*V)DZ6SmY;BBF{LK>w4*$Z?HsNfe{2^>&&568|@=Inp1>2tG_fPz# zEdqWTXJOOfzmw{%n7C~!3wV%9uhyOuT1(M1p(NJzKkhzw$l^;X1ue@-GI0|8VfqmKDQ5TwtTkt z4?b`ISmbtzlvvt&b>2tm$na|UyZV%7ly0HdC2u6(gmz*x&tY}MpL}|hjzc2^^sL8h zCItrk=bLpAbJFmJCkM?G#^d;`_R?xgd1ltp?~8o~sOwusO zqN#~KoS)92L@&Vq0nghL-ZWEM@SCI1_xP(LjQP}<1$DR3BtmcNL-`%hz-Ot_JeQ}W zVdiNlr}mhXQhfqB1RHtrM@W~oOp7y!qv*s-v(^`cqzJzXxf|jgw@(!=eJGPSH))!t zIYqgy+XYPo-Nry<5`!d7>gx&g4a)Lg#3>o{6-ur8TdSn2V7(z+o;VEy>kl0ct*9{} z2u`4xufY{>3RLNC4$Ltp$uRZ`X8wO@Ngkr+jClC3r$URyT!1ShNH1IXQq&TV43uVv z7W~0x+I(-7f)8ogdF}Ptie%Uy-!X_vjF_a+mwyJnTZpLPk^2=Lddb&C|9MgBzApXC z;a+GuFmsNTUqQhnm<0Br%%GE34$gC(M0C7oq7{gFNdTFhCmFhX{ao*Ao8O&bYMSo4 z1vue_9#%f*$RKs|M6Lsw#c#gitz|svf{P?kH?Oa z%HJl_KwAE){A4~gTZK?5+FNbO5=&Fwy*3PDg9-}%)^S!Tic;HM*GMO)v1iLxwvf+< z#$Y!g{_iC>O%*`m^2OM%@z0f7Ipq%TmM{hxoPEL-z_v~nrG05ITnzVCURyn+L6vKsE<4h z4%j;(SN&N#QE58WvQY{AJz0?()+Dng7djc=mLxEZZRc1d&rrak*Do(t&H!HOr{a<1 zB~(q53!+p!%Sor=arEm~!-yN@6TNWitEYD%X=lbZw`6T^8hV?BMF4PyuT;eIFon5- zv!L?Ll{l{Ft~{gM#}go&4&frTlO<~jXuGVrGV>Uc(0KBaPSrQ3oam3;6#%BvURJ`8BMg&P3`NX6t6p! zqlOMRD}$8khtuRCeH0q1czgVh*R>7V@5L8nMqV+B|CUl7KlAH+ZeUU#%3U^8*&`i0 z!U|~9>;c?ZtY3R}|D@9K$zx`=(Mq*gv%-mn0$;B~)LeO0UADDmmQ%XM`s^lbq^LQV zfVI?kK5;vgKtUEPGT-%S5$u9sTMJ*kd}V#r=fVM(z&0YsYWZ@@7H;18S>BUeQAFn4 zxFGfoZN4IsKeYXev}y;UaQQeQ%(lS8$O- z_tCrR!JWX^%|qBs)b1Ln!!`>6Xqe(_e2>A(-22GcNCfSGJGH^Xk zW(Kk}INKp|!Erq`;#Z;G$n${BTpnIvQZO(#97f!&$EVBLSCvKaIejcS)wfi5CA)rR z5$)j0RPv-~y@?ePPdiyID6aSQAWwA?Q)_!3$-`UB25}!bFSNlBy6x;Pc66RexXw6! zhP+|D=9~gW!KGR7I#KzsmwxM@BWbkL&LiPH(GI`8rV|8(U+RS-4TdcL;4(Y;hTFeG z>)VI0Te3t$W@3>iw>BLgpVsY!Gp{-#c8f|87buH!3@A#pa57K3>@FXlpf2v?ce!b9 z7kF5(B-=cO_dE)c(Ejp_@;mwP7&)eKHX!9d!`cWwSUP$Tpvf)< zh{QoiOGdO{{_RCx7XI@P%UUmmQ(5=d#LjXGQ-$KIV?e1Me`2BN4XMkR0}|({aPO%p z4rxVm5n+Ljg&ZNCTL00lg*I8{7)CSjo#8Vjk{Cyxl2V!SDw?q1wDGk2MAAGL)wTh3 ziKyWou$Yn-!S}Z(4ya-NOO~x&LwxH(TdjHH4Z8JKhAnrcS?8`1gnRkCrf4 z&up>LbUrt(5s6ZK5q>+Tkn4?)c6Pv z+flU!!PVsvOvT|*CW{Dl{JiZ7gQEQC=j`U~M-uFmm63A<;8Ql;{&haN8fnR#HP)t(KxzyWeK)nJJJ3Zpj%zATai!FI%GS@jH zh-fv64#vhr1yp|SmQ{^iyx7%PF9JWQB3iHg+*>~CC#i0uHKF>eJL=-To^IlaCl52> z;&BLV^}RCGEn=5*qtIDmNFz~_v0$^NiO1v(hCzEULr7-y7?{Fyi2rkBGyxDav?mDl zzCH>!Eu~OEx>mj9qwbAWrOX+KY>R5`_Z)?uC+Ss+B>A`3+)3W$V-7&mcPHrwA=k?B zmTBeomWMm+Wb0aWHbF`RH@ek)i&jhY@EqN592mYiZw(545HYJ(D14RUTVC>+_Zm|% zIV;d%MC66kGFET@x!B3NH;HHjiG=u5G1D%qRoAs^B|59>?`T)BR{spf`xh6#}HGL0HyQ+;ekA7Uqn`TjB!o{Q7*3B2E1Nb^EaSr8)f2Q1k?FuVyeN( z=t8xVp?UTvwQfBvKWxwLHFQ}x-CbL7jP2j>$vv=tG-d+wKagD)MtgGVM4eWNAA^-r z)U3HCai_=Yib#?L*M|&Pe!6`BU-qHy5Y7i*oX2D#7{e1YDpQKwG1}HUFlml{Av8(* zr@GTR%$DFa&+@XbIJd>jeXX;~z|kW7Muv~rYNMbqQZ$T=FK|ivcJwiq(Wb*tRLaA6 zfb1aL_Qh4s8A_iw?}>ISynjTmy-*%_yhJDS>~-_=iuP_LVth+&D*}aW#b+^NBQ`Au zdDFCc>INBJ4apvg5irJrN-53K*IzjnNbt~L4Pyp+UZ`b~kkc`+$%z*7;RXs3=FJK? zWhHwD|6w4hN+4Fg3`v>%@ZswsmQh?)ewx9R4!ltV#f1*VaK`v*mW4&ynHdwxeHMe_ z|C7bWCJ$YYw|0majvwj!qo1q>d4%X+tn$Q4SZc=t#{VAxML@d0)Me?QM2=?p*He6< z&W`nh_(3O}+qvACkKR8r#JLsvK=6&|_67V6sH(D40cbqx6q>fAVD9&>ossF85#yUG z+}o^n4%GN!9CYF?O{!=)Yc9(zKd>Gg%@x=Re@iN}RXViVEvs+Ru(vCqzv#)LxB>`o-0#CE( z`WO@5WAvwG0e2v|*^%2$K-eUR*iQqL+WYR|uriesj{|oDF5FPfl9HzzfjFGVs`lg= zb;~zUPK$ZuB6$nSOJ=zts_YSggymL?B-q+Ii4Wn(Tdp1)+kDWLtpan>1_bfe8tual zE;BHrM!Su^^JH#${qqODVdpunp>4p z`k1`jd?BTAvB^DCgxjRYL=NgXM!2orRU7IQ#KgYQ8X?ixGM#WS1`LY40 z6qzLM^PasE9%mFlh3%)Kj8dag&hZpTWyNXt07fD>iH2#xb;#&Q6R=K8-!D73zP&Rp zQ7*Z@#FJy@IfO zBZ4O-^TN%qTMX1q+a*r-H<8y7 z*2tV(>`S~i1kk9CAzHVun|zKT=vC|AXk57Hp@O;y7;@nR7hWM~o(5XFPhiooS=KHQs1=w(NjBFn z6JEr)GXRmxCh`=JIxK9rVv0!gbwUh;rs`PZOZFAm{4_u^OHZ)U-XbKXidx`&Q25laE6Y`98rR-b+9d>_VLcMzdQl~1O3>m%$3t0!XP&=cb3;Y@ zNS{%0!n)#Sea-0g6qeZ*KL)9u9L5~4)HZ(mi9ntxS!7?uRUpoI7KM#_p~TZTOw1;i zB>O{SGfm0I_gKxQ+Ewd>%rbg0WQ#4suz>U2?{+?AQV_ufkbMTzmD_kp%e-3>RiMex z(x)S>w%GJ-2BQ5;ma>${Zj8vLLS;>t7y?E^!a4*3>_+vH!(!qB8_&DsO6LruM4MkF z!@Zm(y;Yh*DN0X>=PbiR@3BZMD`i-Lp4FO+n)AId;T!wQ9cFBBJW99RxA z+Qjq2q>AJprc7w~t|2JWHRTG?gj>xDx>WMs;5HcOT2?HMYDx`j2Wx?xU3XiZL{nUV zD|a+wnT^!be6kfEQwmj7LrDghdYxF}#xYNT6STtM*r8hPYKfd8v76e$l7NP_8jQNz zHPAr>BRprD$-^vP4w8+w2Eu*?Sp_nce3%Se3im*U1@W4Xl8nZ46Ffk}rEO*7NNF%t z)^?4_%of6`1&54a=~tf7nr6K}C2LZy`&M+9+^0soGPAI$a7k$V2C`HmLx)(c`_A40 z`?#Pr2s4SZaqAR8OuE3tJE-_nxl2B(nizM6MB9$lf{NmSXV4MG%@hw<7@$+^p3OD-{mT4!lO)@m9x> zBdTTOovTxx{pS+A7E2C3*HrWadi~2*(+Q8m=$NsJZy$EfYb$pzOHls!oeoXM z*X(k{f0b7`j^0W9FLT`@q9I;@9$#f;MGvoua|UMyxg)2}4c2R|?{*^*WV$Hrg%%A` z?{(X1$ptt&UVY{}n^3Ug2;|HzD(doYaKdXZ6_dM#X?w+3+Z$Y0G%(pD+wG6l??EW>gz5Hd^n`)V2?}6 ziark%r1pnEaunCT#VJ?!lnE>>?+O~_?g|XVn)|nGJXannUB!VmgRuh&uul}$97g39 z);J)MDT|V&^ks(Z#PjB+ts&nUsWw4r^DYViVsNK})-k6$ZYs`_ZXiLYDcU*>xOX1*yUrLz8b@w!j0G+z&_Gn%AaRROd=cY211;z*n=1QsqdeZT zzUk(C#KA+mn~aXL1EEy?|0?;{)vER2mr?H6GyC zA9Ot5lMYTkubbJ~J-7YO3N~BXU!uPpI~FqfAv&1@`jrva6gsoL(8p z@RV!wZh}!f3RFR?{DNca$_gk@>Pzl8If^&o4`q5O6%B?KrA&I=Ww9Psq4zu;Fm!`5 z6hKwku=B-zUQL4R=EJ!SMqLnCT0-?hQF@O~+4%@?ri6yvx-jU{GoAu~?kfit-R9%B z5m~m3vUd4p8Zt^-MvSk7ijLjuo|X08V3iGR*TJXAct#$k0F9dcirRK#41Y$MGJ;?jVP!q^`O=pGlFW9iE^v%CF=S%tR?YWk{luwJBe5q zn^IQV)+`;Ia@dSFy)YCY*1f=N%fLbA;sLwZ%hD*Ztl&;!np)+Eu93bo(suK#g^!0H z1Tg&M;Tn9TZXOk(Zj#O57P3>?QJA(OGBjnQK!x)&GE$is$DM}h&>j40c1l0cZIz*b9ZN_F%HoH|&o~K`D-4Di=$&N|}a}qc`5HFj9uF;OsHNNufch6f33lO~7qNv&d#FLKP74Hgh|0s0xL)Q_7%8=l#Z$ia#??K^ElX`}bH#9A|L<*W*4P%AgbL{P~$ zivkMF0)sqo8X=EIR9>A;S@vmJdfgWaI@Y=&M6_wq@R&uHXdsfoWXM2K&Mr+wW$qPN z7zMAFM8>x|JWhIdDwS{rWim?{i=mNRww;D@Q-kD9%eLjGCoNi)l4mn24lCrUhlsS?^WVVzJ5vgsAmhlyG1Q>(v2Xt_X-xG_iawG`@q3 zE!XK1EV60_=Zg(>z+BE@<|~F8nTEiQ`t&!1kpj#*i&1M-$4KSW7KlVj{^{aG)r*l1 zI3Y6Cy5pZ3Wmk|n`r8%HpyzDygV7!{RxYJ(iZMc{lF2(bWDeJyR>418 zxC0F^i49TEg9fDqQAoG$W5Wv3SW@9aF9k~cgz6Y^pAi9(m0fm|!doAynN+0~%xFwlA;@Rr-JBIVSK2_tBiRORoGcQ_+GRzRjxL51oZ zi-5U$8H1y?=SEie*&7ebchU!KNpVOgWVZu=$QvYa!b^$^;+|C%%PzymyX<;%7P2Um z4>#0`OKeg_t0C2=JC`7$iw$68_p8Giw8Pmd3|A7oxu0cIQ;WABOB1wy_~iEJZZ$N- zJ?N%vb*2>|I~WRL;^9L?ni807LgNpX;qXpT^+}%C%U(Q&3$0eQTtab)Hh8+2{j$PZ zgQEN%>OO8$dBF%FtV)HppFldryCn!=+-d*e5; z-JdEuPBVreKL(k!Xr*b-7Xa9mnKmhU1sz@kaDnwNqqk!!3U(`DO&64c{o#;49dj~c(4+@YzAL$ zMD(dRyNS~R8E`HsH`zjF1#NG3^^xXds1)H0WwRJ_E=gsKVZ_L*si1d38u3;A?J3Jy zD+zKx7(?K7CLSJw#l09d@CZW-onS5tzcGqp)F1D-E{J_KSCsY~?hurS79s*YF?YwxI2;&8AwJ&(U0Fl|xX_*tH;xdj+`$4WI+NWxJ=g9} zx4n=A6$DA_K`sfizLOY9gftAL(whwEn*Gad-OnNhpCH25(TVRJa_!afwQ)#Qq0ME+ zuS9wv5`vH%pU@QZVm63H%L&+L<&{CcN{Wjw0I)Tmy;Jv< z@dnsL+1j}}cT{Xc4Xzdf+eIY!aC@B2oDU7lVrUlFj4E$wFvfyC8YW>4VwNFWO9(KO z?{llPVUNkcg3E6(^<$vL(@j%~1?T#VgDzvsp)7dh3sxP9)}>4;9@~puq-fT45E*+I zPkT8AX|d{qWgFXfWW{LaE1a^I99+@j+0_KAgc{$sC={z=&UUd1J&1%OdvM3iaJ&r> zqxQ`;IJTmsHN6=N5&GrCvoYO1`ikW+11HQ+2lU!<#L32EzB zy%n^%GvFl96LiW@vgg^(%i0(cdOhTekdEL%QO$-05UNi?Q!_RvW{6th=&A9Q@b)uztiC?w3rMl28`necLMOWBloex=4m3r@@m9Z{yKu*ftPScRfrFCNM!AQ?GpzZy9p zI|vA!@l~ydSr-XxXR~wJ6C?{rOTj~V?VXL7#K}!wI)`~KV&gmV*RgIsA8yNK?At(N zkU@0$u1|s>9xnm904UV-~1z|^Ht88zDoWo{ar?jlc1l1+<9d|lnh#V$i0|#oXHlbCL2oV-o1&xs@8?AvR zi}n-{mJeU8TNQ`i!s8sM#O-7$G{y~Lq_?#(lkzm-Shzt=aIc1JJ*2B1uskxd8@W6} zl1CptGnI~*+;(X)vI(o)m!XZg*lQLuz=16(5=*TZH_jni1g3>KtoL zlzzKIm||qyeka5w`4ob@j{7x6v&uXY-kdtr+Q_r9I&OUn_aUgbAlWJ$zJRi@T|l+o z7r1Y=L$_NhFR1*-p+RZEtTS?`6NBc}qZs?fYMhjpdLgy80$warX$YXUgLO4%Ss5~` zXblNhO&S+oE4LxG`qzUXmW_DjfaTEr^x1im32He*C&`R>uANfUbz^=>GqR?Afh0AD z@HEh5ZggbZ6+oJ#l=V+JJP)|_RTExUNnWgT?kZ$`;}L zSrm^_tAkjb;mwZcj0@&JY6G%Mo|w746DJsXs$zG*J_gQaPUhK~8;A|G_>WwUx^0X%hb z7}==M$>W*cM>fL}1Ces$U8zvbCuQyJPM?twT7=g@X500QTUf_V8QZ%OWrn>D2 z3Tbr;Zpsm)Hg>Wdte~p!5N5_r=M2>wPsS#V|!G4(nB~u9@o|$_)wOQmd&A zeQ34u+USG0Aw^o0z5QtUXl2?|xWzf6^wpG6k+|?-M1ux-6DwU;OQM^^58|WYilsL` z)73GTAZ{>Lo(9N^3A$2QDr#nIBclP!Q_*o}yKkx&WCATTgC0w$3vj^(mm1li|*bo(?OtACFIHmxgHFeY1W?xohaqyx1-4-!SwH}Kp}F*MHGOIK&dY4i>P)s@uiAmW9G$sm8+i&MO!RS2og?ncGr75 z#_x9HPb|SegYh^%TiPhAS5&kf{6#=*=TqI$QznpD9%TDL9B!WnZL~r?>MmV8Un|4B zvIarTw{(ILTjyqd&pCJE@)`r`AuxzbI&C4>Ep*5!J?;t|W*+7cem+kL0vX+^k;w#e zJ}49KFA-%oS+3~_iaNe)*$xS~=f9Fk(h3lj5r zXVO&~Cay_EIjm^EHN{;QEd&7Wd|w%# zePld(_oxAY#F#zfl8(#_uQ9EI3t-hqJzj?m$#!DB?)6s|hF7g=gclcmlpHHZu$Hqo z0P384Aw%A1@$JQ$xHQJw0kC83wjSsgd1)6tdZVFw10_S+UeC4e7-O$hke;osxEy`P z?#p4qhSXw?&IQ@gU8y3f32OQX$~CUkDQRh`MiVo`-nB4F*;+{>uZ1qaL&ifBWZCGn zG!%59C&-FQOvVnnzBm#0QFQqU)=k~2t4yZ@p#!Wt!NT6xV=}_(7a{p&R1rs15}~FIf`JK z>`fNDlvCHC!GLiNC$7#zqs&}!QJGTEFmCOP7V(%(Qr6@m_~*+!b?S8 z_C`T4Ck+M;q>li#%9Q|l?>8x)kx9?%eN4ZawO~$q=axUjbw6+Y_YRiS!H%q2$VC3J4T76tmUf^^!I$Ydtpn1j{FiHMrT@WRmf?J?7*w~Ob7W3cE1h-d2N(h+f~M%1`vT8 zh}=q7pQU}f6nmAxgk&X$*;cQEBG0}J>A2MD#uvEWLDAjxkBf5Q&o`ePB;(RTs4F$z zXM5SCPV5<@LGYTi+C9Dp4ZH4W*p~u$3L9LDEL5Ayw;2M-A~9xYAXDxN3K5qDhJz)p zCgaNC#VLsO?0JwU)}jX?p`qc%mghv~T%`mSO;ovvRkPzgq9AD#sA1nqRuRdg^sz#W z_%amO2}g>9EwvP>Q0TF7j@&*5TWCj{jJY^3DkEvBN_kzww$V2v0IYTp^(}oP<@o9S zwHY@!UE|<1^8bh=m@!J6qV@Ug`KT z+^N`m-wu1VJ3c#J%Y z+9bWTyznlldqcV7lv}a@|*r!jfhYRK$TM(+hK_O*dB9 zjYM_4P_$vRZVa)SQ_lTJDiH>}+(n(QrwoHhD;Db#Y%`c>HdB;ojtEm*7JZsZ+YI@p zD~;gC@f&gm7~+P}dpw2365&mdBg&3c?G>6<9a)|$Mka+T1+O31fd_fgrO&;ZDrn8J zok;rsWurf_(W$PIUm;P=DgpLD`>JUwSSXT>@LcOAfo?xDbf%l%cwd?VP^^Ak@wF9sMFs>4#!KY`w9(y#bo?I5iDtTWA5LHFX8x z^l}bphM$SM2PPWh0D{MbRY3s@R#;(!4qoD@lxm1gQrpKN?g_`*tKe6cA8QXaBHPfX zG!eKKMb;WCV2bp)fYjD%-oZfnkG;+29|7Y#TB5~_&&V7UOQ!TkQH20?gHbtV0*_a;Jt2c!*jKI9|qG(>$<8bQRo`=Nr zJp>a3+!o{4s7x{42gjMM;#5WqUyiK2qXsL$kgnYO%8KC-N6uC78l4HIrqu`HQDd(~wt}VKKdZxD%#t zyxQujmby0Pxr{{bc6XZZA|)l27>S8dtFD7(mt)PJ0+q?JGxX@eX4OIf#nHV}GmJye zsoKPFNRDFqv?!9Up@%QX5=y3g?xjGftek~3whkV6u9lbainvh-2*)=Uf3bWQW4udthh!G zH6V5_vG#1@7J*oiZj5<;%)QcWEPOmRsYSWAGDNMPvA}Z1NmnhtEP00Qu;$@+_NdC( z6O#!(KSSC-z+)5Kov#@Uw<_Kfr%q|aiwkk+Y+-oxT+whE;Dwh_lKbkr77nrVA+hYP`kbJpj zd3wkDT-%FKw6?D`vj&2|#uWiLIp-VmR7TZOIK*Fa}Bm`mmGZ6*rO+woY?iKMMt z;9XsxR4m)9miP}OE=9ZMEtgQ@yaEjg9p{DHtXo}|Y|@S*Ea469H;m%0$TzU@l`ZmB zNCn4^1Y2^X6bbFJ#O57cu7(>+?pEHHc30lb4%NEb|A z7<-A*f=K79HIrmBhoQ=|3|GGwt({XIp9H7Dx7wP(siZ*sJ!uSWMIv-XMsKiz)L~}A z-j5HtyJs7a<{fMW%C!S|Jav3FCLkygfTPu^7AEb_;=A4RSDV}SH9W<;3XbDz7R7wx z5;~+QYtLKA)k~=mGcsflls3e>Vu3++_{={dzBolnF&zP5EF^bq{C; z7cI%NB17Z9@i(O7Uz&>+-vS*~xi&#lhjcACg{;Hol8 zM?kj>Gkz5@i{^Y}>els`?}jN9lJD9LDNQ_)TVfU{NF& z6;`h5`BffABS8wIV|IZu*H1CtXR@10Z8O?Ulhid?R^YsIx@184pDd9)taBnOy$IOf zuy&t9Wb0P;SQpE(viNRwN};LautY852VgM~nTQEv(tIZYe}9yd~8mbU1<#F-FOIAuVi^K9He?@*hq0tcDU$) zvEeSv4M2VZk=cxM6msB7<5-4TsW=f~kQ+B{3Ja$c7NzsaT`kOw`x@;BtrXDFNSy~9 zLo+%cpIH(Jd%*l)q3p%+;=B4SWY{~eG?}tpGbl>oJw(cRws%~t z?~FT}FsVBm{vj3c7AaUIvhwbTfK!xKtY!R)k%M8kr+UUs87rvcF zQO1;4y^6EXG-!fgXj3FD^+o$vJs!%Ng>%NW#&_qW&i+QNzHa=+LL|`&<*f!;rPcGp z3eb%%r^F}_KFC1?p~G~WwT`mT)fW<&B9x4RGu3Kmnmk#01udv}t7(ZC_4h!X?;0Lm zD-(=L^Z@q0MiN;lhp2}N8bu{Y%Hbi9tpIaUJ@aN4vsw6naAmnyW*D7n*-m8*V|>)~ z8U;~o0MAA!;n*?RRYtWeu{KCjo0W>HXL}AgL$2L9l)o)kAi2D7Cc2f(^^bs5&Yk8L zFUDs4)CM&#a@cgn1C|`M>gJwJ))QlrInI=fUo85wF6<|jxe&*}9$4Y$ewTdY>vQic zwu>aQQ2gMdfZ0jiTn4$#e+?GlzBB z_QP*o?--#ZT6T1=8Z8zS#$7Hngt;P2@ToA`EwXrA0^N!jEDwymIdEB%R@m=MY=Dd@ zqG13+K?klg<5X``$~=>hG1_%b>dSTD0ZS2t!k5cBnkpMj0kQ_qLCQmAmoBQWrqnn{ z*n~yGRdmRH@rc*bW)7NYfE7(3E4JLRT56qc;PBAZbZrzDljyV7RZd*ONc2Z9GNoBn zg$eYF;Z!ALa20FM2X#bGh}n4*&VaDrE=OwEYT04^*l>5Xh{$J>!BC8kFA*rN$%-yK zmxLBsB1w|*NHo=B>+N=t6gJ&R^5(+=}n2}<*DiXgkk^P;K}R8t%ELOYIj*H=0pdrs8%>RGyd2<-U`^)F(9$3ES&jBZFumG=%hv>*!1JRrX`p;(`(}8s^J5 z^(jqBch(UTu*km%^o5u(!4OFZJfV+VWsvV;48AntjiA#rWc$}MbzgcUEd7*{%$5U& zXG&R6nnPM4l-a7wOo%*%@|=l1=;eo#$XOdm?YP!oMHP$718iD`)6y-kFUBE;FGH-< zmQ$GSHvaLeF%^I`T|Yy{?aL5#KQYQA)E|T>TP2`fgH4UZG_TaK`U7KYZBMi$S(#9* z#YNP-YCGcJq@tBhcu7@aXmAH3YHZ@>^al*e*)<9w=X}QEV*Et)nGm3fWLZbnF%nXP zH;_3aGVdyid2f4aAfc!`FEOHRN9wD8TUyx>!9jrnI;Rz^URA4*Hp|$uS06z%F+HFx zPp#*xV{s6aXu{@6qY%uVuCQteG4ak<5m>Dg$^==W(n!1FDwz?U0$pFULDSU2P=!Jx z^`J9(c}yG@tqH!KLG7=k*{o-?E*sw~6c`gB{gzFT3ah0?Q$U)CSOrf=QF?Mos(`3W zU}NBjCS_2Wqqx%ZM|&FZ@7i&f z9@>A2wmXNMxyt=k!ySUqlOp3ND5$$Gp+L_0kxUET`ZtcG8`HMFVSbS%8TPsjsn-Zp zU9m}}!PkJ0S_$+H%><-5o@Y0^g$jVej0!{(i$G`w+E*$%kEpP1{XC_}nx#e$5DJ*% zE`2LcW;3E$cIoL2GTf*hMQPfgym$*H$u9{>4m6@&FD}eHlbrgk*93;I*nw1@Np(Ci z2%sP}2C|u;n6Wi(yWo06_VQ?yQOq2s*fFW#@aYGk*c!+O)6I0=3R=sdd@5@fC|yHq zd=y(-PVA70Zk*Kz##fWA&k0wd}`}PBPbsAurUkb5($1z;0b-gjX32O<~9_SW($*>i$z8kYk&Ui)LGGEoG}Z^_mU zrS6&DCQ`Arrm(nBD+ypd?>MgzL>#ZF44Jcc(ULo7-wbM3IPhxe-kuPBGG5pf95|kV zktaUxI3pHlgXbLL5BI0`WrpgNOx|yjpk& zFc&n|P4LSd+$;(V7{!d}+}NzbjJ-^el409j#>IHR_M5fpcade^)u;tt>v+AfxK;&r z(v)g~?BZe#pH=rhlU=lVIc3ts%X=9yQ(Z;CsZ1^-5xXQFf zJ%GG*OFaChwRGhgGN9{w>s)nGg>&=5V~)9t@tOZrs9a9Ep`_IRn@H_kwjr0dK=qi7wRDzY*|ira5`jg&FP?_5P?uTn|xaN=b;GU2xU!yg;tqQJzjZ?esKXE1}HdDz2 z6y!u7O?E`K35(e~g0tb^k9w0e9>;?8qP&@~psz58j{O8zYtAi}7DI9O z53kh^*Gh8{bt}+5zG$*cFzPqWvi7Ls!A_y`mC1*JbRcc?8G&1rkUll7Qy~r~6VrN> z3>UMxCOSt1qB+~j^mB=x3T#g{u-O@27WEt@ffSRJzmqgptGUg`8+^7y<}jympdyh} zbB?rsHm)fTN|-c9F>g5y52Uj2*JnCKgpYJW8*U)U`DSW(9WK{Mn3l?bsWVywNQ8xS zDZADz)NE|+v7mG-d|hUCIb>+my&h+7z$7W)j|LAynkYP-a=hHCxeH(@nZ{W5aHtv7 za9h#fldBqd6xIhyAhv#LrDP6XI$Z(Ba2MLP>Y;+e;Nn~!;nLeY_C#r_kxNz0P(`aw z@0G4fg#k-&$LiPM9%K?(&*R) zbxNY!2yQHNF1ABqx06y--1*N$T3Bq9AyzOpwUV?EgLCtqylqdx$Y zO6}#N^m*YC`@d<{!W4_~ef=o&pu1w;BEW3dAdRTSS*5;bNzjV2{(kZ8@K&}`%uV@a zkp$kDIkW;1om38TkaZpgQ#UVsw%0I-5Vh>>yrnYBC5tss(W}L#7o_S#B^D2dS?TW8 z5W3X{&aROj(=qlhUVAY|q#N2~BQ@PMrI9M0ZH#t4MI;B+XRL-}5p~dlq&C1we$**9 zD2}ymJ<7$OOKiLZ(2|Q?50056#=?xgDGdQKtRo#^t7Q@iDzvkvEztv>)NpSA$%QDn zqs6ZCKh`}&BA}rhB;{+E4Mu%2q7>Z*xOgXYxXF!`tLHVE=N6RJXd7Okoxy3yGzv3| zODMlY%bXJccnxN6M#>8D(PSJCEpY747+ftndSVfaZc6Dv%`}CKyN0cfmpIV2%EuL8 z-F_b?S6k3J#}VzAfjEZi=0nwKsPOZ){DwAi?o7b<*bk;w^?FeAwkRx%xzD#O92Izq zxy8369u@)9+z&zoibZN)5mQ1@eS2PO#2E=LUu1fs97!fd?_ew8vC`xaJ;SAdX;%q2 zaN+bkTTm@`sUY)uS}YxKz|?Hx$ep)^Bd79&%4K1C)^78~=i+qIUo}u)mqhVwfpZBUc6dUBNncW-C6{OeixYG78C1?KQscsOX3!g3JP}68I5>4&Vplu zNRCg4uheNFN-(+#XzlAIAebdVMTR~(HvwSljM$f+rJU3T(IA~Sh3My5?{zvc$*48R zWwKd8^R>?SRnq{=AYmITmb?*7J&cQ@gdFpC%#MCV_4z7!h;3K~?1e0y9H{;f8--I+ z1*yp@P>_^AJ@m7Px_5Cz!^LEg=n;phwB7rT^13OBux2)=pWK<`kp>LUNi~BAoDvZ@ zQDLmT3(Gs?`;6DbY0wrdPYAh|TF!3pu3c__T-K=5)!yagzAni2)t z6I^$l1CLssN`{H!5Q^!RN62!zJCsTS+jpv3Q3}t1_Buwnk4Y=%Ubg~qh!n`V)@GWZ zV`PB_dPh?X&{IJaH4$CjaTy9MxQCE?N{gQ_*j*_W)?qR_!Lj29E|6t;u?4~vRD&Tb za8OxX>v*nSn!xus(oBj3RkIZqU3Y%QF?9-8ZJ^x&N`U2#TYyU>A z*4|baDvzF2P0>v^H6SM%$Onp{;*)F~?1r~=8P9d=JeL>>lWXpyOc9Hquoip}r<;lm z2`KazTh+)nb#2N%B=8Kkw=YqYAt(s#F%QVNONG&M+oys#MdsUClbxPm5`c_0C&`&r zhGO9O?hvB3j3_H$s2{%xg0+>ThoPZ0Kr5f7f$pbU zv=q=m?|8;i4?_kvUxnjcoioA)X>pS)q z6^rh>bk;+k7B z0y>zS^=jA&pOlPi3)ibjYCC1+%b!FDuPp#I2&V$vz!{W9HYm}FQ152fr+8Q>=!}Z0)=cwP zO)5-uD22WtXHb{(=v`rGr#qv1EjIMCd3&VlJfnU#Mz=S#Oa)UWU6P#}XR+e&y_btR z4`J=H!)GmT* z-9}6+&B#Pl(AcMBQ9P51hKdr$cQ=R-&#*Y-4ND8HL6VkGcjpu^9x3U zM$y&47=sqWZQXFW0?DJrSrZ2%rTE@SZ9-~^YHhCv(*rKoTvCczLNL^fb4e?MSzPcI z_qzQDZqlk|*2(_Q+L3G>4eN{{BGR-( zwsvTnpCtOMYE3G9y7VUkmU30^bv;VJ8;?sN$Yml%^&fKh5G*G#-uKftk1)BM=z>dexeK;eQ{yC@my5PZS`<>2{ zLxN`9S+#5J1=5ajkmZ`Gsm{6x^b2}WvwgBuNgA=S~&b;{0} zPf9~9`28AAZ)4PC>1o+va=_^socz4iJ=0V>OqE-k>eSs%1CSBLWxOaf4N&)D`e{Oe z#@+fD*T(2ISBR;N0MpHhJyc~$DahiaD^Gn+I4OkLZsSpca2Boi6>6!tjCNomOCS-K zP9d^xbox4A3q(~d0A!)!OijN64V22kI zeH$>NLIa#6R1wlZ;N~2615egKi&oyd6D;NDthZvRAX;q{y~}>|qhY{_9^!*F=U{XT zcl1SbYIiBc5?nG{4UY}=I|-+g0jBFahe`_g2+dh*DE+(g;sv*jEGuDe(Igp|290x2j`J!RCp+@xnlXThZE$$0ISd}EfyBPQ zJxtQxs|;BnAezwyX+I9MS&LKvs<;g3I-+*V1)49J`Ij;3xZ`pCD1iK4tJo+ss~Sl6 zFbdzUDa6qv2C1#q0DV_}S}4a|)UQ1Pu@A3;*W0LBX^waWVm3x2PU|;Aoa5YLMZC9c z>eLZ>cA~SrOz7vAQ67&brEz)<Pq`?U4#d7` zV&_u%ULGmTJ=eCuJyx>Dh%#~&*I|h(jQFJEe64o?!1u>ISD5@sDEYcc>fim z%Mxa$ZRQJ`>3;zP78k~Z4HbUiB2XDO5Y)s-Z*=v(bDhs!g92%!>goZt&YxsPCWyvD z?4*aR9Ez!AwzH%!CK`E;jFovIdwK}>E0b?r3=OzR$fOTQChwOn;Ah%KbY^S~S`&w^ zmQw!MuHi=%M+E@o_>@tiI0d=`4^%6Imko`qj9pqEikIU}y|kBzB3AGXD;I#K#Z!?f znSBtI(d9h#GVxCLU?Qkxqm!%w_=pn6U<^p|C@GJ06#S?6q8@KT@5Jx2wio4y38M5h zkEZD4&BBA$D#}r!XUDzeo)K7TAV3 zlgm$C$0aGBxFw&H>I>$X1M*XtknpuQ>AJiwjp3^zEi9?HrJ(@2ix^$& zsES=RgOtZitIW9#RuT)_2L5qaI}|XB*eg6hWY3R|?->jjjt0!#ne%CC%G`Xe7Qigg zgL+D66VNo4>U__)9T9#9(nwP#eIXSE5q5%#%iZl}z{&LSxT=9}xwl%`+CJn)D_(=D zkW#@FHtD|<8tA5*V_nZ$x*^ZBU zI9!0khrIsF4yW9NP0A+6it8_H^a8JpbGr?S+UAIF&ts5;PDa>VD)e>+H2#rUh5{%u zFvd9-!>P|;5H|EY`Tg2ger|r{oB6Wbix7c|oY?G`4@f)SyL`}@2ak8lkS3NtM`a$L zfJR%BFGhD>-l!3iEvd0Vc5GOyNn7IaOOlJVy$6{~W47dwCDytb5j#|M8%8lCD(Vzf ztd0<0Q9;ofa?#2eMqr-;mBcf%eA^`wLV-FDT`1&4KC{J8vRHqJN7>Vf4qI|&ASVN< zTgfHs4{wl5wv@J!XY99&<59tyA=`tY_rfo}r*~G2Tf58N73YhfC2*0Xx_JZM-;V@` zTo6uO-B^P=j8B{_tI0NA3rF-Hw9c%gNnTFSFPzPdp1;90S}dnjee2#TOn8B3o$}ee zAkq!d&Kg5~uQRWOB}%7O8lxXRrf;BUs6|a9*w2YJ&DEFZ2(V;}WW6t6d$iQ+8Gv_O zOHtEp<)vxuriW!dl5ut-K?Q8pa^=Jp8!3<`;R?q@d=!KyS0*kObqckHvNXJhS~ChS z0;tFs2pO2jP`C7~41cN1=H*Dlf-1;$?r z_|(|KD8&a(^Im=DL#ytMwcbie=_Kx7X16Iw2~7{$wJh?N8-^K!RyDU~7K8%D=T5Ux zs|&ax@)?prP@EZ_awWPWrExMpmk+bAR_smKX5Ke}#13S9cbK+EL^I6S<6TD_KJJR2 zZnkzbb&xcSfZ9XXbGapge&EFfBsuWE3b16&rI9ZvfN-O4Kp76qhZuw_<=9>M_D6Dp zkV5dn0E7rAm0|g@97aAe&2)GpOq&$UPo3{`NOKpSK$N$XMj6Z%@0+oZmYQIf;U~gX z!*#@Z-o4F)!(F+-lwy!$qB{#y$9-DR?UM{Kua(Zp18=m}q6YH8Sp=H|;<2g}P2zkq z1UTD+WGAIZI0zfs3pU}9iO0TqJ>mvjZ$(54=7VX}0X&0SXzZTA_n9d@0K4HR80 z8DOT~OD3E~JN46I1&?o!xc8K7gr6Dib%oS8-iH+-ww*vbS!~$Fa}Hc+;@T|91?8@ zyfMo0*bCqeS&s7{k{Oe$u!O)3%86I$S$1WG}Sreve5Gu#DV3!2`AQ6b?<0tcik zSf-?K-zr-P)8s{Mu6H&EBT zpR-%un>Hi35fi|R?&KhAJy`Go_lw65SffixtBs4NuXUm*dJ^-dC(yi+9<-83Fpn2= zm8r7f-7S@IcsmeV*9Yxkv2DtW8=E2T7u9~++M8N0V=6rF8~FPKv88Pw2Py)gM_zY4 z&%buXpzRe_nvJ&ZF~BvJ_AO@GEK-3jY3qxKzC2dS8Ic$?VwWqtHr~~M7M@QVk6?(P z5C${BJqnvSM5QV9Ikd>c#Ow)E)Y%eRiFXjO^)nZD}}74W&;z*GldPT?NNw zDe}bU8-OiVXO~v6ND_}@{liQ{Meza83S9+Aob!0l1){@E%6(CUy7A)p`y<)nlx4RX-^w?!|6)4dp+er|u8Ow8{z1@d|MQvD(1B&z4rXqVm z)qtiUZ~_OwU5;KmVxOp4N(nh^*n5{8ns!>)Vq${`#eGhKCe*Hl2`|T3LBc4##I176 zMKf2_H1Z67wN>bhD-)k(2pu|)SWsWAEKWqqR=K9edui9+wq6;1VW6|cyR9f64`_x! z$`%&xTF`oJav2vlJ7b^0f+1`vry1oY5F?Tku$ec5~P#I8zpeXR4qY$eN0|Qr5b{?g+zRh z12hZxuBeStxzgcksfai5E$8HpO+-!8_AQP9Nzvw;0vBYiW*b=Ssn?x%%u@kmj1u0- zVOm!hr0#{Z@iqs8(x5TST9Qqp9eQ4tbiV3!H6tU@E@(9~o4%Lp!VH*AEs1iZfH<|aCui9YaHx!bH`kOu{iiRlncUowAM#TQW@$)N?xzd~=4riP!^Ugv$CdJn3ybQq#C;*m+V3fz? zuJxryMnq5^V$;ACOe;2~t1XBS;KAlUGC5fe;o9gzRY6krkiulr30WJCw)-g{&|NEE zq!L1u;EaK-^Tm4%fYVeONZe}VIHYThH;4H5od*sqAD$Khq9KNVwbw$AQrI-dF zU3E!XtY++eYrPBa78>$BfUc%#>Xaz&mif~A#WG4v?9_LT38-FDB?CFe&jOp)*9ymA zzL^N+HrsPbZn8nDDHE5pDBhNq3Um)T%sj65xk2>wMlu~NiZBJX6@$tW#Zs(2ce z+y{*+`4IqPa>}h1ULY)t`^?%aa-d+SrwB|;n8|*)j;d<~T)U7=BsH}AroH7}GH0g@ zpv@?pE$=is_CGLVA9zP>5VA|qz$boYFlhs-2Y?%-yjb;zg7~<-?rGt>BWbrwiM|Gx zF^h~gakbSQlygTczr#Zk4+eHC09E5>pURtF-k;gaV{-3BRLu`dU_^D*a&(|X6_W>> z6OGhpg=CRQ&M(_!BcoOtjH)r%1ms660#y|wPeFzPMW{5Mbi3rGgrW+-INgHAX6%Vr z20CRt5F$?*)S+|*%J-gMe8mS-?FSA{i9AJ7Rr2qOZHRHl-6BKbln#Bnikq83hs~z( z_+}7~BC(SKrDoO5qWVI4IFPGtM5h&I<*3YTPs`n?);3b0R&nc92I^WVnRqqOPY6&> zg~gdu1*xYthH)s+#S;o)l$*h@Gpxt@w*(@g8)mFX8Qzk7`g$1%X z!9 z-c|GVHQ@npWwV7*hFR>RH$-dGbjTeGtOsmjUoxk?Eyk4lnsU4-Ig~J-ofPM&Y)>VRlPY1WY7&t`t_U!acDUfw^=h)8pkD zvzj3cSnEzRrRgoF-*%4VnB?}gwT2w2ND8>CoQE|u0-*||7G0j=z35Hj@I7NOBx~QK z6&BfG5|Z$gkR7yX6vf?@@N(MYba_VF&%@BSqPdq%WVTz-W;490$k0KAwc<}?5YquN z@xMK^rCa^lbSG#}xzsKs2aoIRg-y}aEQ3HSiGW!3iU@;L^&@+Q?mOqeR)C1`5Nj7^ zVpxjjEl{;hgJPm$ylb$7ER>Q0&r>dilq2Q&uG={RT^l<&1}-#3!gQxddvPnpZLfvs zPS%mvR0&Xqen(~BUjjg5C`5DHR=bb8six1UbtQ87QsMU_gsi9C9Jfe1#N)Rz!@x$F zYjYkXKBE>yW)#PcFH)(mB&NY>Y!qQmI^RR!?kYKEJ5>dEfZBp>M@*Z{h@f430t6&_ zLKjwWS41$r;_Dwb%rgdFN4`S52sFv!l2#_@nigEv%wH9PhOwbj#7g9Rc~q{rN_l|e zIBt4g%JkTz=RMT4pbC`40;B-j(po|U2=j;CEv*y@RVCmS=d^UU-7Z39EeJqZeY4#L zmQ+i1vySB%d-tWaT~r(h4^BD_@)Of67dd@)rNfkKw>E{!@VsuE|7WKGY_HjsuxIWI?=O8iW>z}JXS|Wo?;FpHR%FF_ z*KL8LIRw5w5^n`DFqwjP?{jQO?F3}H=88lZLvY;PgyE7&e3eAbBf<%rd6iY82_>jf zEolP6Z2+uob{rRr3xY)$rCz@s&n?u!?7W|GK>)j>(=ZM1gXTD z3p?Q#b~FLhpoLDT=p^Nb>MXfHz>UUOgv}dr%L87|UPhLJbZ{R_wOCSbu;l9|v|)** zMx4~BpPJE^pC=ZtJ+h<(A7Zvx#k zeUU}xLKyDk)gJ0Mr+U+cFBl53NQEO3D;@n?=z$q}3MRHFC#;RX9KKM4s|1)@=T{qx zXy7d1gzPgoF@e0``x}ErL1~fPA1&^0BBV8u6K(Z%>Z+(PBNr`OM1dKur9E4UWS7=f z1JRLwq@cJr>@kPT*a2|E*EiE@nTf_t4p5~y7N`0tzS+{V9GAU8LFwYkIU9%r#uDp; zJ}TW7FmQuK&16K|KwdJH=6X{H;=s=@PlisT1Vrr*EvHmoGt(F@L0~WnYY$?Lg4Dro z8_hy5VPc-fq+cWpEvOYb^wsI>*Aa#UkZ9XFjPrPThUg#eUt)4y?;ZhazoJJupz?ek z(0QSUA#Xbvp^L^{VWdgGcWljV!`dLq=GGOZMGf2(_D~31C_q5EdSNA~lm%vk5{M-& zfv=Y%L~Xw^@({@*x5Mu+lybgz*ao-Wj)kH6ZcD z8g)v*HuR(8s0`iW51zVsuSA^zFsU@gL?-Xs^rXb#T4js1T8{89dn!|npi&ZzJ~06q zCbbX{n|f;wKw5HiLb66V7AXj{-mt%2nD&-Vd!B4h%P%s%n!95ZOz_wlJ$Y-P4X@9N~pXD)G7>& z1{{c7nmfj}8ezv(L#)OTS?FCInfsbb@llIaFkOqEvSA+ZhqqiHQ`<-(Nq7KDp?%}d z>vd~!?VP`Y7vyeAqW5*gesPkQYuYJncb>>N4@JC5T!oVjc4A`pb4p8bmv5jOB(?yi94KEa}un6gVMm~A0$~jh>%TYVRW7-Ft0l~p;Pe3veE{n{X_<&1$oAz z^0yxd)3xu=uOi;BL?igZvC>ZAZ-SvmLr3B+@Sixs_ZLEe!!UW}q_%MIln4&C{+6!2>I98~vGq&N zT1@ z##wLeOo;%Isbzbhwsv_B8KJ}p9NpGic+C;Ft^5MgWYlwB0Rwm46KfIRzXr8--ey+O zfx?hmg7RCfXHT%8%mitu(A$=4B~X(@fGdoH7W)x8djcg*Cky_Bgh?U37FZ$dsEgHP}t>1h0OGMHZN;jH3-w27|?;T=!l%d&9$Lh!6j)!sT3KDxl*+v z;eb^N1I8ZlINgOb#6I_)$h)RoEYcTrRx4$%29wZDsfT*#NEKx7IN-`GvA2|OnH1Er z&9lAJcXbwy2%?i17adZ*mKl#-h~AP&ix(06mcKYq;Y3%;4rFj5DUC}TS{K6rtoEcb z366e8Et$)J1P8IFtHym-ekCf$W@6pfZC>f~n~M*W8YCk(cJ9=7#Kg_a2!tUbRud73 zBePg8aS7-0j@TvN5!MmR-=+{q^RVp+<4%mfOww|PsW>bQ?R0yFPWrOFQiWiV*fplo zuAhmqq}V%|$#$nBildGaRgR0ODHLqsTGGRE>h?5yR$X~Adon{4F+7*iUKbBs)vyap zTq(iWUIwMOux1&QQTzq)ShsEucGH(|$^W{xobG=&0NkP~~o zP!*;QMom7Pm#&NF4-Tl-l`T>TXTS%I?C9R~p;pj=hP)y$ZAI7WSa#f;xJQRL+}cY~ zb0Sjug=FgXlr|;q#tfV~6BSJbDyT6ceYS7b>M!RacFNLn0oa?GsqeRTG(r*kQu1!pu%^<`lH-&&wmg1uA zM-WYokeZyhUEfEbJo4(PiRWch<=`66RrBs?VQO4ODA9_sMCPJ;i99^l*pyf_9AsgD zME-kmFj4b*l{nxwaeHk!x%?Hgh%3r-k8u!NNmPQc`BNS+62RY{=n2Z}7bj$HZn1i@ zEVxuE^rO$VP%9}q%SP?wXR<;^q)P%I=U9o}2AsprEHWwXQ`MBjMq zhbI`EDAZh`a?2~-HQH;RdICXhOy#kJWc48fD`AMPt&CZ{xrQCzR?_02Do~dAWVj$a zB)%ss3ZY;yqe0g&W@<;=wN$%%>%l{Ic2u~E(R*9AStIl%sK(UpH`Cp&Sc>F9VC4dKv*ua*cGr533r{rNBGu3}j6%FdIHXu8CCy9Lx_6d{Yoyt} zvD`B}KJ5-ruDb}|790bD#qSsxrySII=X_~PuMr9%RK_Tflos;-85Sw#D(_4P@sQrf zjXWA0O)^gw@1<@PL>wU_iq}w984O+%fVF>S$=$?MBK?GiS_ZF!SKPT$>>=A1G;8FL zdx(yU^Them^-j5uTZJ-5!17DOtp-_9RGmA!&V6G6-oRovFv5_9~4*+0i5#7;dePng94*qV}1nAbVVG|y} zTw2Uz&Gbe^(LU`q`S4$jgW2vZSsoXr>y#QZ5G)F5z9NDN?4V)r?MZ;SkF{<>Bil)9 zfqf&S(0FdPN{eqFWE{Nw>N8VPQ@GOWH z={JwZz#VX{Ixtu;ysJ+TILge*Yyvf8VhT$ov2ncHRd*_CAYxN@g4XLGTbwD5b0ts% zA4og9^>Teq3kcf5ab^#KzUdku&E0JQ2OVUW!JJbZ#DNIMX+v?yRwP*6;IvuGvNw9k z0(o^&Y$OyF#6WpsGVw=h8ASFg#cbWTFl!Q*pPuYIi^6j2(kba+T?(AnacT-igt4cz zG82MZdh`myqF|WV(Fm5oo8Jw2{lRW z3m4Q(q9E;`u0-09Z2IGe#12(At`=<(robWwr#E)a(8_D46Iv7sQQT)JIQ|6*$2o99 z2bIEff}6pB5a8anMC0oyp(kt*DP0z&K)A^stqEme}ulYIIIxuLGaU{YL}4~NfQ~$3^*$T6?;T)v4t{j*bM`0HzqHF9+_0^ zivtVO}cE(}@+H#XRG%V7*D%2aXKYmvdT2$R+3kmk!f3!?>RM-zAu!jZB| z1zyt;_#}@E#Y}cy?u|2E2s%o=Lq6kiO59yKI*L?ILk@~Yiy-Q2c~U5!6j=iDvAuO| zjKRl>^oA6@MlB368V(~oj0|IyC)hzsc%lkX6^R*qySh=T^5fQu6Jm)E#}8KcBR0w2V{zS8&g^^vqRTm7jJ2^tFCRf z-xi1|2by9QG5#$)#A41;&Gbr&h2p}B#WY2}*vt#nSZ5XKAPbGiDDk=2{Qj-!Z3D`k zV2LpHEX_(fN;2ze1L|2|AYC?1%|NjOtb{<+tLj+oa8MX44v=J`RwQND+IhrLr+PtW zrU^B8q2U1Tn)H??_Z{YX%Uqw}ae!D>^+0geyh+I2FQ#(q0XmOVJVd3bV}a%@c5G7D zgS+pt1D_ZLYG0^Q=0vQ~7>BG0cfss-Q4oo_1YEP=gA&_6b@ZrA1LtYyAhfTj3OS#( z5tGGnqJ7VIdP;JXW!1)K_kwmtnVmva@3k@n1B+Gm`;+^Q5i{l(vacM#@8<#k) zgpnmsQ^lKpH&Wxp2?r1mYr*v)AUeyyokUX*Dl%rHwpbCsO?s3HL9FRKm7q4-P-dWZ z(|)qZkzMF+T_K`kWJzrFEJF~pSu;zj@+kN%j7y_Ewd!AaNeCq>n-bF`=NK4V9o+M~ z#5dJ$nm#cgpOH+>%2tOrZBF@QJruw)1px_ZXLSIC^-_6?RrkvS)#&a>vhtMeX))~! zRZ`2klGKrC_- z50fOXv3YG0AcrHdCL}2EwHRfbSXfX?ygpsKBX3(M5kB+0tMms?XMIF}Sl&6pp9$37 zxs1ZcnW@(-{A(tMyoV_W%zMQkPlk3j$!QUv!BE@H@mmQ8+h!oG;HCXm%bU;(JxZ~7&dP1SPG-qqZ zVtW~jR$9dQN6CabHTwI?9_Hz$Bqv7u3U5hzn1WQ(Q_k~3Zq&X;J196ORtYPGLkFeMhm~S|I|dSc+`mct+@ZUi+Qv)U zHP^k$)q5;y)_&SRTwePXB8~0Oe5y>iZB%;+&2&goe)5|c;>3@h084HSB{r6vhghy8 zW^t&^j^-hFK*wYR*0laL&s|4wil$(t0u?&!SICe`EUh^#sIM84rVTd_YVK8*0?7v>Y2pxvw~ad#(T_!-ys!z~jUBgrRRp48yRg zy~p7`RC0T^G6_@Kd6R6H9F0n>cuq3a>&cLL3$Wd$?GsHb!U&xWEQb>9$17vDX1EF_ zCyRW4EIGie24%K7NH(CLc5qJO6`h8zsVews`M$fq1g@2FKsAB)<2DTC&fhK%uaS}& zA9!3j4upZI?6@=w2G*#Vyv=1*<_WT^PH^k7PY^nj)mo?=EX860U~GRIMRueR5^&{sTgk&+It31}t!{7B5;HEhWa9S2V%mv)b5}kY=`)u6 zILYn$u2}ifnAmJ}a;kp8c^oUj6B*%SOk8$4yCv0*g$9(B!sUJ-lmq##^O-Vpu>F0?@&42ecl> zJiBh!kijt8yxF{mA5MnV^=utZ*A}T>6GyvaI^Fo#fI!(m?@&Prt3`fxJX|~Ef$rSN z7JKEm@q`AcOufFKBN=2PvjQcoJWkb_d59>Znaru}1K*jI1Xg~SiorL93a7%ge( zu?|IzDT-00NJ!&;w5n=}Eh`Q5KM7C)sbRwsM9tq`<|4X)%V#+7065& zI{C3I;Wu_forjwoIiV)NOVDak*}Sa`1BJQmcM)2}Ve94Bwkcea2D*te%iE!-o@XZ4 zwCj{H4ig8C_v;9aZ+;6AHc{&_j1gSdN~2iNL1+hI7RLYGh!B`wQ8tA-nDe* zPyx6ycu^$Y388HAtErvwiv?S|Qp7O}3Be^F5uWY|DV|x$9U|gE&Oug_iC9Q2*BWU1Ru5kp z_>GHtlHFbfX2$M#EMO%1>&Je{G*lzqs3N_`kZ&ya$#2sJFX=K$trhOqXqnT|D8T`j zkT7k-=MnOXT2Eb?ah%hGYO)W=I{JY`){_R?Cd~6HH8ZtchKaC`eHg;laDYyjgjp>@ zE6|%MmJSy>gNQr~q{1eJ$7hc`UU|Y7OA`)@DLoCelT|#rj?(wA#WU7j%-L2}573gl zYwOJ{wxJv+4|kxYL&UCe(acs*p?nQ#QiUCW+Q~Y}kU4u-2R5hI!o;ahw&4~CX3Zq&!t^Qy*+3Uc`F z(cD@IyM#QbWoS6^v6F*~&X1P5h$eatNo9E313P0NW^hzFU)tBbMdkM`p74pAZ&-(< zfJM97ViBc!Cih zz>WxZc^V)+(|=hSUQ0AzKGQN#A+4FiZ4z%I(aQev^Tl9;d^7~L-ys)a2$eBy&QKFi^R>0^m_OP2h`}* zJkLX>fFtbc1w9Hw9-<81aAFJ9)KkE2OoiuOoC8D6?bG!(Q3az_E$=diEtIubi^qoA zFwdtqZ9^tQk+{hhd@eaiuUe^YTgzpjkyS4xZ;x= zkquzFYUvNE=KKhLL|Wo4B$6zX`PUR1K!wEa^{FNV;|N`PpHkzD5~QF&h6s#k5)N-Z zmYqmoTG@fH;S8o54rm768_A}cg3g2nhW8en5#IsrYjrHyF4MUidkrr#^y2{FowK8l zH<6U|pz!Y|DHC|>3@3XTDbq~tDc~&tfb)7tN$EXM+GLAC?oN{M5W}7ZW znNkggH!#3Qpl(30YW zv}r2jLbq4v4`u2tSw{k@tCIoq={a|~l0mR0syavrt_wrQ!@_WCK4o`hbQvcpu-87X zT)f2!OH(Z_y~{ozwrONKt*m&8hvGGh>Ip33V!Avpa<5pGh%^XLf`VcwYQQM4hOMGG zJSDW%zB6toG&nvJQZWnj0SbuQW}?uz>9|`xOh3Nl>gqxdWTy5-&G#7cMH%{iI5w?W z3lQEUc6c1j*>pwdwpZ?F!}yqk49_5>^6SC^jJZV5maQ$=j912{JSuV}r%=wZiCATw z-d#b@p#WjCT=ZLwDtS~rm>y2)4>w(PY`f~OB!!*pcEl#5vIiScCQUUWv4xu1)b%M; zcJgvi9iUiJE$_oTJeG0L46t}Tix)m=+YY&qK;)%-?ID&u-kkOiD5uCin=`|xQf?jkOF{C3i#T(iyOAJt>q7PbNKxh=1*R}c}$RI zCxQids(^NT$y2$6BfOnJx3%Ks7RGlv;sdxqjyEUSs`FY%dh|&fgGWZs%PQ1yICi2A zMrj-m#uySu_qv%Scz_~z!$r;ZVPqs)AS+4@3e}=62*}%rS3Nj^BszMPc#v;* z3E2&$7KH^TUk>fG2H$3D!V6yBD5xObR=8G?y-8>%Y?YV6zKth}a#rMU+)_1ehe!>& zkUe~Tl(`Tv@x;l*4Ui|arL_sx@s;5nS@1h=sf;soZHQ`!5NTY_dd0U9ygXiXk_(75 zAi0jNWF(5GN56e}8OxU<*H(zKgcGv04KxBbsgQ&%MzD#@wN9?LUboi9T>(1r#xQYF zW6gPryle%_d1MfAa_%=Xx9d?k^!U$7B>=?q_p7}nt*^uu#B!pMZ;f~dMQ5=aK8;C< zwIgXNYFDwdQwtoxA2n4ph#fnE zT~3j(Qd3`fjkXcy4;j#AK$~0T<@Ss$o+4Vl+BYV^q7B^;c@F!HfbJUY=agnxzn>^*b zt7auZeZE-~Ut}q-GU$XFL1AN;Kxy{xC0(sR&?aXg#9xDZ3}z|M-Dw^WsOty^I)=Q1EhLC|qAKUP4Q5XV+U&9uVY6X;reU?`#n1q`RYyP!d%R>3pB zf-9iMMvGmP`_}Lyr-4$TlDlk7c%B31m)5YS$ zj!N^)#lTVam+EWC37poBf)35!i@ zi>0~WI=ndWgLQ{cLQX@d2nsUVwtq5^(y64D7p1S5r#|CMQnm5dbVUVgCU$=}3 z`_dB*l^s!OwY7gt;_GTuJB6L84$ceT;mi%Sg&$sMHL+J{b)l@`%3`(^EfHgBQ~W zfaI(1L@s2VrFv>xC#Gb4jHmln!Fz?*xtFUKq?&=N?HF$wV3Fy>0)Y4;m z2CqPItMvk*=23gD_WJnP)q~c4ln1e}W%V06a|}4R7RMhiZE!tuIqBVVU9Wc=UWU>L znq-c+f*A-4A9|M*of?cD&7~iRj$?(!>W?Uip}_%knho0_uvrZ{bRUM-Y8a31I;Ls5 zcoO&6b(Iq6d;|*;H?!qbceI+LX3-sifo`P-i!XRoJ?YmArvc2wy9l!g8>EnsZnF`H zbG_-rMwE|ZcG2K9uf^{YIVgF3JjY-ZBNcs~xYsC0F~sb!8IHoWE;)4uVeoT20L~M1 zIN*j`GmKsDrOlZni@5AOZqV8XYZ}3x=SCpUVtOb-F0@;0j^1zFA+K9qPc|7PGQ|kseEF? zTiyw9sNOVM1-gW~u}Iq5w8Qpdkf8~ugFfhI>ww%pKNbSBC0Y=OqE~$!3?r169(Te4 z>W6yd+_49G%^T(g@+x5=kQmtUdF)8H#HRc&mjWY;mBO`_oFx$AWT2TWtam=zd-HDO zG+dDwl?WmTAso7mihbl2_gwg1eYlGf@6`ec8Q8E2yQSBK^wJ25xJ%?&dgOXL35&=+ z1qFuL;S6>mIk0ukoph@$*dr#CgUk(#>LUh5Q@$4u?*Kh~Y1o0^c(XtxhOO78c(t6- z;D)Hm)A&{O3@YqnE!-bcL1=~jD>aG&P?Nz4op{8n0TQEErR}FntA)sKp|QrmL%Q1A z;jIN47Gw~atZB^J^97|{w!duTMy@aH;ZXEm%lQ$d;Jd@lD2@L2v3RcpDD;W!RKvWN~#Uu}qBpAX1 zjB?)2Y^ybPSV=rVXus)A@lpp=`4^%pfk$ z&{cl8Qi{yiqD<_fv-hnPUo36ZNuUT0elaoCi z?78y+cpA3ll9-vqUAAZ0a2TZ)1DqXiepwlr9!2j->PcjX<9ROav!u^PZ;(5knizw@ z#?F5o)-cF&aa{1dA`!LlZqX{mY6oQ++>Vr>-oabF#9otI+Hut1H|@MORsz9YeQ9h^^M@M2gDmiqIw^ae!yu|S6#U4+dd4Em>F*5) zYdfal-m#Wmky)mM{4EO{rcqb7EUyGLr)pjQb!;>O~t3tmUqtpZ>RglP5 z_o;CKdCpxIJwjk2rsnb!k82D(b267lA(O$qbjbM&VUeK)LWDCKF!S_SWv$}I0ck}w zxv{i+lMr3kufa>wCGPn?`TQl|OYclA6I^x>auSwseJeqAkj5JtGcZN9G^Oqdi)Af$ahA3 z*4mUNQ`+qUB$5+1VzZg-xHKLUJHW1p;Xo1QFS7=n2XSq#rh6(8E?vaX8_c(ZWy{2 zr3gwOI}2tUquz{C=9_RIV7EswS-8#0F~Af<+TXoPaQ*_Py)usI<-HPeRX&&p5J1H| z_rBY~QP&t5)lDNG2ElH)l-?*FPbJs6vmP|y%etAf}uz) z7>+SNdKY;xsQ0rFh1O@F9)g@Zqz{`d7Va2_T=I%0N{V~lNUVAp0lJ8YCwu0ypgRXE zAABJecfd_%_=kPUrc5+1AUc~(Aw``E!DrkOMhoq_T65r~31sN;TGAOCmTYJq9!+C8 zGX;SA60!+fGdKws6?q;5FQj;Ui1E%WX>@mL%(3GPvS_i-YP1pIDi}|%-*Yu~7~8ka ze7+SyYtf3oGArl12)m|8pQg{SSW7RMPh;Nw#A;hG_6L*R5m`1!@(>I z=MGtANJ|ER8hirK^_J&V`Ed(Vm}04pfQwZOro~2x2ah!K*6^MRcj&&9J1Gl$$SEQ- zgy`&HSonE}RC{F3bg_c7vym%gt=f<{=7F50r&B_kOXPN&u|L-25*im#|rDvc)Ya;QVx+bvB(rLxY7^J}d?X^RT9scSJ41DWz6 zYt}kZq{ZDwQ$Xg4AGnt>9cKILAYG)_4`vdzlJaaC8gF-%lof3IAJ1SjdP<{=EYi*7 zfqrLrNm$ZpqS%o|EoLI!zR#(P*cWA%@pUM74W~w}RHm{!nMUcS29|znfi!bLjX~T* z-q2# zpL76K7-y6T$jDmM-B=MkZP_ihW)BQ?Z9>MNfO!3u3eH0|YeP0Z;)4l6=qrK> z?%qJ#MV{0?m`m8hdc_e2D2s%(P*>#4FcY?HJ~D-8_F(EUpan9aQ_zu%Lyxr>8G^<6 z$H=d{vN?5Wf=Q|keTXJYZSgdSlQRj3hFW&w%W(?s;%25o>MD$xLKdnYn>GcIyQ0>) zYdi`DMN8Qaq@R@4=!w>&$;&5m|^|c#QA6*Rlh$1}y$Sv<44lEGenLBXtsY)C_BZH=F36}8+Vu}S%=ZRQxIK1`N~8di&tIw58Y8H6k)g#wxCv5q>FiX&8Nnf=m)sM>bDaU9l}tvD)m}NfqIT$Gq#V2ZTY%< zC$NKuz{B7&U}2uhP6A*azYvFngF)s*fNW!lhQLB5=%suIN{KJ7=d;pp?qjp{;D`bU9_&g|V>Z+N5@N1loeF7#;x7sY01sXj`M zObBe)UOC(_@yr7sg>x*#bSC%}z?ys$C`fqX5Q2s(RkGp^#q>uaNMQDek;AC;aM27V zmxhEFBdALPl$wkcglsd0jAfQWgNm{%k~C{fre&=_w^OdK4jlN)B>-JOqQ7Z_iAmND zTE>m7`N*G3OQvRD$Sq~-r$eh|A~AvWJM{;#hmOd^Vu{u*>`M`y$w&(NELvp3rJ8Bb z+?zbPVBah)14-ePXyZ-B!DycCDk>5mtBQH{S>-p4ykd0lHFpjL5YPhWWggE_^0`{p zH7Od7BaHP&te(e!sFm|OLdea{Pf$imG`>g#?jjZCp3EuWrK6;fVcWrS`h*rRhhx5n zkl}aQMv#xInI?#>6`y?%bE)lNp4d{RqKh3d zZW2kzlTyL)p+;9)37ilQ&PQD)2K_mhTGT*ez{Ry=y^fJa%jIqaM02Y~p^ z4;uP|i^q_$L`^BJQAIBlFmc{HvEiNDjAkWT4%^=nF>PGllHuB=CDRU|d319pMWh&V zr4O}$=WUeF9T{{DXtl=jl{`4^NPG~TE)Lrmd1=nzci%H&wTN5RHjjIwQ9CV~LCN)7 zgS>)7Bah9wTCyH>vk9;k;cL{+4LilFz=0)Wu2r147Dz~zrzUGy%6Mvx+i~8WQFztW z$N`utwqmzVK5X+uC{1SfMy)ZHw8NElDlvJiYp_h7`_vC34X@9#ic1+zt_o0^CtW=) zWJy$HhBkA!Yn(T)$GFNbjK@n}X7^z@G!a1@$c)D6!MHrki;WR=O- z(2k}7Li^4;ZEnZM;J9#z-ZnlpM0@k=bx@F)a)HOvv?eAAoWjzp z9lhxA+b!-clKXrYPtjKVz%o((y`Nv7ZTHCXz>+p%-t-f!Aa>PB46{w}N9a5SM_V+s zyzOAc;JfyiqP5#`2aPC6#T4VJt}vCwWbkXV$KD|KVEDm$$%r-I*lm-^BMjleO6Ou& zzSZ(a9b1Q*Xsu*5!%-+{GKw;2{ns+vb6o;MB)mi85E!=L6&fVjWiLep-gP&u*=@07 zgpVb?D8P(eZ_c5klNN(}FtJN;R$FBxP?Gg=uRn!;)oSl$a0CGhp_vs6yiA1f(248hH|Zh3fpxtIurc;YtH(gcz= zp^>Y7;TJw~6$44K@PSJDAcA9^H=5Kf?`u{xsSGN-WtCP}rJb~*837{qLqyk6#Z;;! zo+}u|CaY>U5U8q?aR92a0K>a>6p9RL9F{@?H~H(O5wFfw-NV?&;9b(X2d!c|Fr-z} z3fgfQOzG?JGq@@w+k_^JyJ+^Gq||Jd_IZq>F;u+aY}Q?yB17;9i4>DDY}is0(K}-c zQjb+#&iBsM$$|kIWaat;L%x> zAR(x%?b_#$NXHstytT@wt`6{KGg_}2-jvFcV)+nY?p9~FoRfkx$lh`Ww6(i++7AV; z%pWnwmV~J3H8z1YF$JGKt7uY`hAJ4qNV(;8!dGC+i^c4w!tNUTI;%9wF#v!D1p&CB z#|LM<$Zua)iVXXkfLqWVqeZCLb;f{-nap+VpbXykUSK&wfstB+%0RUrMP7{7jNvjR z1Hg$}%oJSkU9K68cY;Y6RKnA#UPUY_Op3=GFT@qy*A>4SO&SP15vZ{rSqHV9 zAQq#wyv)G};-X9-!(eZd`BvR!CJC zdp%J6Ls#m{hD|h-TO%7!N(7Gr-8RdR&`S>e%2M3>$LMl$l~_ExAon|xA$XK}z%79u z9f0Atz{9FSoK1*4sy;VtJL^y(NQESLbm~cvnBgl7zjAH>(&fv=kAwwp%4!sy>~X1j zI(4ud*_*O5n=O;~jIGb37B?7fP0)E`oZmBC1Z1y>_q^?RP>(05Y8JMH5HQrU_on4l zUJaJnWk3egBD$IpCwvsXX=N2GZ z0UsX1iJ=FMl3wB#<2_Bw-!hz3AS3NaT$qDZrsB}bVRkQvj_KEpeGDj_gf3r0Ny*Jy zfat2MS*kX@Q&=V^6y3IGR-bM8OzvuB#u8XB9@Vor;n74R!bDN6ubP-td@yXjH90G( z6jE(OB+-wChw#)c45FKx^*zCu?ogFT^L*J|MlCiv0!FwkIUt0FNN;e3JSG5*YoUT+ zxnW@9idv<$9ExN=O>Cj|(s0@O( zf=%kn1q6CX;9j*!jwp`fdHy6YbBvHDzh#PegS0h6l)}QTiyKKv_D_$rii2Ts0H{XC zDt(;L{fBrrhM6v^m?d8G&|Cd2B{VB~wO7KYZ`)8>*wl+oshNu8$-EawfKh?otLkKq z10bB}h4+)q!Z_ar_pS7uXDlA&0?wmmNWDXAkvjLpXUIi6Z7g1BD2KX+!H8RLKe{$$ z%&;qJbSqlW)8 zcskAC@jT?g4{E)W*^ZZvFrbv(>8yx?8x#`R6b9A|*Be6`IcUQ+z6zDpgS+uD2`aT? zk&`k^2$jz!9$CC+ShBZa;TNm3Y1&GubEB`Y4|ygmz}Oav0A}AZV)}l|e**hQbP$`p z8tGTAK$`=LlGz^s%2_n zW-|xp_K}$eLPYk63pdKk13g1_tz~#ueN5WnTm>&|ZG#G0A^4N-zt)st6t<~HN8TbQ zP&%>WIC5)uN1t>G#FoXmUaA*WRc^0!AbX%owXbz&n59i99>v*=qRY41mZrPuxgASF znikK5-^ue)VEvuJi25WI+~Lq>WwX9XaL_t3BR=0$N>P^A0Z|}^b6^!X;_q0l%U0fY z$6148H!%8ejBi^NZnP#J7FTAwy#)#fUBuO3aF4>eNPDh0HK|wIFGw=Vs2Al<`T8aH zuPNZ+ObGN>Tr0z=LBWn`=Ws`*4R>hv>BOjpo2rHnx_iOOtd<#)YT~GrsbN@ zv<>j47)w=I}#f6jSCyK_Y}a%?~5?DyNGOjHtr`TfLWP+ z$`@h8=%|VyR>@oC!+vx4aFp*_CDiAP!vd4H86o>lP{ddl?OhP)*(#sMA#Z8cb2$>eugHX?S@5lwZNdR%o*|}rKs@yw3h`!V>B_LUiWeH zIVd)5+U0RETPQe;jVo&kZx_gMGFGU0WdJ z5wYJI%{B+3eSA~2mRIrNEd71)$nADRdc+=;f$cJio%>?=G43Jg{qQFh30X365A%> z`{AU{xo7iBeX&=I3R*ByPhtUz#1Y$4l{#om*CVZ%luA-gA_UYAAw%Ib=0q8$1${&d89)~3I z`u%-vM53YuA%^@d_A1CfG8fJ?4k%c|AnqP=#_$O6gmgF#ZOVru_%eN6YOAccVA4jW z`cT_DT#mxj+7_~tbUwnrG=+9rB%D=bO6HwxTaSz7R`{cAX?R_3L5fplheZvYTKxJalXh_=~$qzE)SltiGUa{vs%xtG85_RS9iG&dQy1IQf zJO<(BUo5F03pAOBFkFeCs#F}D(q306T&1Rhqo<_c?(aKWnccP|dqe`;NyzL<_^yF4xp)~0dE6!%BYCg@0Eq|ka!K$ko|t1(pVo_*Q&C{hzPh| zL?!GIuw#=9;1!Az}73B>|HUSspu=h^f`h0vtXHE zw_FkS>}BNMu@wY3+nIs*=sYiQ4QK^WQfnH_Aj-+mwZC|B3A(jdE!Lo5oNKVzx%v(J znD<4|%=fi4c(XBwDwdfwZx#CYsQb8;Lqk)z*zPpgEoCID9U0*h5Y6`~; z?@HYEK}p%rg6P&G^98&v+jbsLeN}0-wE+vs>4R5?b{1G+uIhR}sYe*d=fqPayzOoy zCuEUiSX(spj3B`cAS=oey=j4vYvXHszcG%hHAu*?t59PaOuWklj<$v|<+;5B+TntA zX_tpkhnvQxUTNS>ob)kD@s+m534(f5@k6YyMV5xD&V!{bKoE78JP@XJ;zo33;n>rQ zFthZT=Tjb;Pq>jzS%_Gf~8No*$=y zgffUI8H7RtG$it_ZmRWd^EySL!EB*?@is#DffXiri=~|P&J1}ZXx18-*k%ktKG^#? z#pK&8o~-*m;5N`ftrj^caSI^Di3snD(%D`?eHjK~`1P=17_H6$+h-3;NzFW@L$ak- z`7_dI-ipNU8}!+G+`#UZQBO2xr@^vUE_-0QJ6we#!0o`#?{#oxw0RrW?p0Kt5KhUG zOwEThzOD83Hx9>jh2cu(-7d2y%s1_1kp63mcS~AabIO@KO`I+PjJVpg<+zYZ1!rFaWiPcIBB-&t|qs|+H`n5!yUD_Ed(z`s*S zIKyi>jFC1Kd_o0BQF?9$SuW{c1q#gHFhvpfWF`X>Mox+b; z!pl}76dwuYBsztN;_}u5@p%sbEfd>IJ1l0bT(DC5e-LzDELfc6W&!xQV=JGnj|R1i zs7ZiF>~*cCR}L`ry8!F=ve}r&-EqEc@Rss4aKI`LxyxB$aj27&Q(JVN>RX}IuP$85 zzqq{!>#77U(9+Ay9YA}EpO%QQd5p*>pyTNa$KxW=e<1q29N zz~UHiySRRa=5U(k=Q|ZH@EQRI`;KfFA&i9HD11ng7Qq}FebuAlf`XM<^*vasA*Zzb zJS(T7PLRobUUs_-@trpLBE1E+o)OMz%ey)=ju4hM7MZRSKtH{(QPixrIHJb zArUgJ2<4_5QeE1z$|PsPm5qW*_poM-m2TCU*oB%!sgB%6h*Wnp96exFg=ueP8%fjS zZzf$qU$K~U)o-in3qkkIrC6L}lLO9acKz=>%9O+wW+nAVWHzT?ZnYRm#QZj29^VPJ;v*trv{;7u=52`{+>Tu0!B23T^?g zE->t82DIJUGduN9o!GT8-kxU2tJw2l0f8lcG!=X)DtTLA5S0co1Yq8wD}}B%;P9KW z`=Kd`NP;C%bJ-I-1>M1*WT34x=PT%bT{{Nx&@q=a@1L-(`&rU$W&WY8TZc`|zrA4d zmKDt>$Mir6+HAL4Oj0^);48+zs$_uRshA7S788uGH&-b1kUF+7#qx}wtBM1kKn!LR z(W(<$UUC-vT^OBri)fxjToS@U9Jgy}Uo~6i%NER29t>}|BPMfEfbC5L?-KXj^(60% zC26>;(DFd;eYB)lVsuENe0?ai7=h&ME*nfZ9z{2xdkhRnA5b7~9LJ{mA<^Di60DPe zmon6^n&vpfE3=am#fdteM+0gIM$*RFbN50^y5taHf?JwHL;#00)z7-6yVoTP#jw2apr#~Yi8cj?Eq&7KT zwfl_ecBs}eyH;;xsN2>w2rZTHB5Pj;ia8&SNb??BPYkT+;v+|yS|;^q)Psi0*L3xVv{TZ8**mc_9N(7Y83NX)oRuwhSy?NnD7s=4qN_aZ0!L)(Glfyv+uF~5cwDT6mYK=*Oric~! zZ6ep>mC#tEvJ*Cwwy%t&185m0Fx3c`k;eX^8khyc6k0v7~1hPL_73fuRR2 zXL2i7G+k>h0xj^6CjvbS#hJVvaEY5UxeG0WC8y3Ky3}{oR0`cG%ydzK7Uy>?t{!^I z#o+Dk1;z6~wo{{m9^reSd77eE4D&cWNnths;3}1roPt8>TM(rdeD`=_7E*U8Y9UoQ z4+<94wJunCQAy#U&6*uu>|7?Bl(_dJJ)Qu<%i&j_N=1z$YbWOjJu z-0JAfO~4~4p3(S=RM0PX*zQaemjLm4s>R(cO)|Q2TviI?hZDURov648ixp%=rL^ZGUq(1 z9_spr$V1O77>F<>WULP=LFIJ#hox%Bk8Zfbeti8I4;bX$z}NOAb}OvboX)9u2L+c}RfKlVw4L--K5X znXP9_MU)6ESgzhJA15)pJcrkB?3uBv&k?^|Y2yZ{d%ZP$3wniegkjn`2bF5>E3Lsl zTj+YK{k$|N_YpzkS{fjSGn9Nkhh{af@xfd@(AMJ?&_l9wov^2rrB7q>9K#qa9sy$ZeWO-OVQl z)a$TT1lytPp$JTbwy7cBfOArxOnTV-+JkPxHJTRg1k~^_ntEzwZ<(gv?t!JQq!iN* zphW{Ge(F%vy_;Fef;^pGEni?}OVoKuJVsno&dZ}-0tUGjMicr7$49tHEAMLh@9B%F z3IRW~IbmdkiBO^#TX3yB4d6W=uA_p1bAs9dtWscXrx3?t?N!J*AVt&UZc9-?f!Ol( zG8(h218L$Xr#zKBhWap%F)_P*x6-;gXlGEM!M&{yMh(;5C7Z6Vl5T*m-*Iv}OU%Cj zn)c!~)BxZJ=^sRT8kU$%9SSuRdBWDF7;&f#w-|XJyKhr*tQE;3Pm_Cka!~ILvSpFc z-@59=p|jzXv(6cyAsE|8GCFG5dWbC%88=KC=v4foFJ~CTnRIyjDp!S3y4{7-ta!xQ z&<2yDWI&`FmhRrza$&U1D8)%i9Y@jd0PgY8fqMG(fJiEtBthgyNu8 z_E9LoAa~LYdouC)ZJZqH9*2?4=p6z(H7+(QmfB}*H(1;xx9cN3&Zm&6hL4!5=!3;F zp%B!fWlIKgK(qnDbU-^3C9ag)++-oiYvltS;#X6%@mSnh(1wymF{i~3ChjF58sY2# z)Nw{ocakBjdv7|r1B@kdpf*x2K-V?q4uWk}#&ezE!JTHeDGsGf8&0{fBxG!gHZ>`U zn4rzox>s7NzHMdLn67wPZ~{V#pc2{nETWu{v(6y#mcw&=xJ zHO_1~C90!Kq2B~t*(=`juusR|O!lBTNnEQOmN!m>3Ub@?3A2!G!LKWOj(1`Fj89959K;yy6>2Ay# z)Tf7S5vCa;(4N!^D2!d)VBjFkAR!95=t;Fk2*(~m6g&ss#ft7AMh{n_oX$X1 z1JWY4c37SmuOz|KWCo)Mj6<_U}PM*iP?Q; z)Fdi`6=Eh3uXb`|ps`Rwjf@RBP6RGPc~&N96U*Wrw<8g~TuF8k$X$&V8X&=`9$mcT zoXgQQ@f%{}z-hsO7!?q*9J4@-ZDQ`>=xHf?jNnf$5gG>gQR`5 zAga`ooDJ``F#C?4T;tOxRRhT@*FH%J1BGRbYE~y$xfCNKd!Ar?oR?}R0e4nS&0t(a zeGbgI!Kz*ofNLYtT=uK>_LwGiyK)SI329yJ4kQ9=u0;-<)~m*^T|tw(=K_Z-cq<0X zV&!gTWq#lhP<%E)vlHMxgSqmJPzfc>Mm2eSIm68pp@OABsmNA2F;_Y*%xdOhu&5!z zsl`kQTc>)rcO?8N)&!QA@eJ&RaMt~jOYHZh+KTFfu6avj*F6aL9-<;3=ImT`gjj4@7AN}4LlsT^IHECecrOMW zZGBTpnn&@7po9?dIfSZEa8up2T)^{;!i8yK!A%XIQW8a@qfPT`_n!5K#(9Bxe4Nyvbu2&$fDoQu2IEUDA4&XQ;LX;k~g>q93T<$Pc z_1lzH=m^kDjU~wSS)KM#RM82=!6bZ4?L7WXqiKO__8ksbE;nKmHm>f?D`46Q$e%=W zfXO~jJ3v)Gvsh*Gk1?#Y;08*0uR7M)k7sNqdsD-$5a7YrrFh7MmB)#|pn?!;@3>sj zJYRTik%mB}$^tQKX~*r=`X1FB^pt@~zdK}8!wfUL5KqchRNTT?P}e^NBfQN%w`YA! zm?knoZ6gC9^OaDBkBsMx0 z)b)cAjJy!e>Xk_I%2TNJfmIO}5omzYc0j3=tIC0+LY6KfTMqhMx2C&xb^Xk0^EN(&94 zYz0kw%EI;zb1iT>lZKasWpa3M%7g@*UW1oA%FE0-y%9l%GzE1STxC}ZppM7nPT z_qXg4o9ib}Si+GeJ*7q)v=NHtd0J|Pzc)KsoVK}~6n=)V!_ejzZCN&%pB7SWp150b zu)F}{%b;LO>L-I}L!ndF+sNGd(*Yrwvr||vDEwNw;TrhKaikA^92ACkF6cdWoHv%6 z7e-39MzB#8*;JB~`=-y@jO+u$UKxTynZ@=R0rWRqv8@S=24Ttggm&^5ZA~)Fkpry{ zDNr4WpzRUfB2+LWy=-w~!SvI}wcV3J0+^-EzdOAR3qVf6)s;{|G8VYrMJ{O%B2}JM z%`$fZvd1B~Y&b9(tgHA`*;XTh)~>+ANUzyBegd^Ld83Bwe2ckgu@-Sni5rY5n5+fMTU!-auT%=^EBn<4n`kFG z2Uyx`5Y0^VZQWa#J2gS{MCrMYq-eRlPMSheOw8?o(SVZhgskO>0#2*&qY%2iED&qc z-l*#WR!&q-krY%25rxDRmTB2NHU^;-$;d!9KU>hi1*KQPi+r~1@dD>VP;&z~I{^8x z(bn-$+*H794)j-y`JiF&`oNXC)nBL#b}zPj-vQ?3w&XCEY6RWO)@?N_K_59Z*GD zLd83_g`W)7E@jy4Vk>EiP)!+3C~IM~usgj+^vJ7VtwGzu*}kfmM7LHZxFhGg5!oD+ zvfiWDo6N>R=wF0N0d_#<)gK6sPzkkxLao|4&@?@PWf7~h9^q|)yel9D5`%h3sJ52# z`_z8~(Mg4zI@`X~wZ(YExzMfF5Q5$oDfhZJ-m(>>exV{(`FuBsc)`}QH&S~Du7Sfy ze0P%fjh7KIlz_d|9&7V`C-l8Qkc|W_zLDYIcZIbo&b!6v!H9;Jfqs_Jqi+OY;J5AZxT`mY3EeMW~iXh zoq~1-5`+m$IK7wAk}NLN(U{i@lPg}zPinJ?+G}&cD~=2SqEzOeAkmhXb?%L*nV`-J29@GTvRxu%ey%4>zlK1czG7p z-;6kV{khYY_+^=EwbO!(q%`Ne$UPUH#6Z^o?yfr%)^~xL5nis0$IZ3rZP3Bk2~lfF z@OM>$s=p9mz{~;DVfH#ciY@F+ozY45g*)S|tMEs&L!7|=z)2q9d>d|SIZd|oy9ZFZ zQ3+@XTB$||Fz^I2s3n@g1UFpZZKDUE*eno@S72c6eVDL#3PKN3ia>jFYhk)YXA%&_ z#P6{n=a1 z)N&;%q_}`ucsJj$wD>ljEUfo%Qzw>10|set7|q>(&Ax1q9j zdz-$duA9ou;^wiwjt%JYi_qSzzCCd2MY|+Vy>(F{ zvd(MdIZ6c<1}R0C#(x&_$$nn6AqoiWTCVKE6BQx?C2okSnunP1>f0Y!&hz+` zdD&RwM5&}As&vqBr-kUJ410BTpH?Wy?s<$@LfKh34TvqUgwt|WRQObcwf?JPhsD@W zKx(tlP_hR(ok*ls);2TH+{|h!;2f`&;`if<%2t%}>)0-1s%*yE+6IsyyqNP#qZ!#= z%7ji950elM^<`6sTq$6Z86VX`3Z0?yd9##$D?t!xP}2c1un|dOZ7fBlXj>X+-9{IHp(*P^5I>*Byynk*kvRu- zN$K2n6zOW1?sr)3$8#n)0lR7V_c;zi!?tj(ai3(qbzSFX1c2_V(+cT3^IZd2oQIWz zx3|`9Wq5OyD_*HPqb{sFd5!S{(n(aI=ND&L%4jp%&`D0a>n}| z`#SG?F5-kOs33aGeRGh!_6)t_#V5Su1;|FOe!S>3?(ic!3)=}~UyGui-dW!*j7W>k z(X`Jx5>ON2&4B7E$0vLYm-4kB)c`K*X>JwLIyqnzT?Rm>Xe(7cQ& zeJCJPfzr6J2PA2}^=_q$h1`jW+B!(L48haDZwrQ$nh^B&TuhCi(XfL~TN>H^<9#Kfc6kXe$l6g!B_Qd}ycLEy z+4paum878KKJbJbASBL@>?~3fV1p#Dcy{gX4}h;&&e%d_fcc8nhXBorpg15*ea#i6 z*)waq7Q;r)D4p1MBA3$ep=Ji)@sNl_k!%kMy3^%k!BE0Jm|g92pG8cakt^RHV%mW) zU7*uLJgWMTg?r_Dul4fs#VV9yp-p&s+es0jZ1KYY_=!?P+^p%_6c*6ju;Zqs8%Lt2 z5K04Kl&p8Wj_Ol#?Jp6-aCf>|W@tU+wFy8oqbli>s5~QB$*zHZjOQhXt;oH&ttci; z_S?d57LtvO%0$X2gGfr|J1!uK21~x8qtVY5bGp>V@GKMvdK-5K&&WxxVw=}` z0V%eJegifLOo=U9>+>6)A{!l83WZqde)E>NS}~X7hiu$EGvK;$M6O5yd4P>qTUjhq;zUyX%iSvp_aHh?71JUlNdfdUHzp1UN6lJf{!264nO zq^$;-6^A#p8CJ_s39Z=7r^;$DC8%Oic=QZ=*OE(emKnKt z)k`Su{UODlXj9jwM@HHYw_n=06$C_hCkSD7_4GIxm~1H!FbGKZdW^R*{lQ)6?Rb^+ zyRCZ^a}TAgFODh&iO1nrx)6r%VP`WXdfLOtJF_F^Ezcf_>XM(CI0`$l;_SgTnXAcT zZOk2W(^z_43jBTdZmZxX8smc#g&~UylMw}?_1N3i0wtDvO#wkjm1^!5-sMLw5lj#d z(Me{!wkGDvnD2Pond&;0*+TGPl?h6U;FmenUSa*|zBn=BVXI~KQKiMR z$}!}nC7)WEQ6-A_T7c#?H=)ux^Tw2EO&>{Gbn30rd!8A1ts$G%^>`*~sf8%j1s#mI6Cm@Mj;>f3~>= z^hyL@J=K|Z9s*^6oaXOhD+(D3ABMX39P=51CX@(^4|e#N!`z_^2*Ko$x)XI<2Y%ZR zG`F)|YG4SYChhb%*8P}|>EdYD--mn@wS1QK&8wB}GZF|CK1G*IlosBeUwD-(bf_(y zT|kUM5q&$D@jP`!rJ^S>2`XtIw?>agI3%)M*QxJ!p`eEbCeYiVx`m)Xmcyt)XQrpm zQ^~k_PQ^(t!Wk0IK&CcEtxr_`8SIB}Ui&iUjhu_ms+>(=&sox*C9AM+EkS z!30dWaLQ1VKz(z0lPYKNY+bc*_9Q8Dp|b`2i|Kz0jzUe1X?L3VGW9bb**EWy!?|cp zvm})YSvcDf^#p*dFKeHDTuZ%F81_ys?pZ4*#fPU#q^GjlemRDb(+ZsH7^kvPsl&ws zQNAnrb}Agycaj3^R!*pm&AohH0G+DZxE=9^9qdxNgS@pl)9Qqs!(L5=Zz~8rKrgX% z8(k|4yh8K%IC}u1@^0TE$OncFZ`fEYE^2_i%^`B?YEo41CJ3u`gw2a4flw*x!XU+? ztpsjZkyhNYTrD?ISILw&;!JVT0Y(#Q5l~ohehcxUyKRr@zsao zb@yc^->S~$^2(+_yFsOh;P)nM_~!?~%_C~`mxI4zzh)a1h{^~J(D#|2reO=!bdLWl}MtrHqjwCuoCA^eD&+?!ldR z_^A!TVp_?p}JaG1wyuLIQKM_Jph} zR-di3%8gctb9}7|@^B#W7^=rY-h35klQ=C0Atub|1`XvXAl16G+PYDb8fwO!f|it_ zg6*6_tuf|q5ofH9h0}ZhtHk!Y&N-bs2GQU)IDu5!YYsLvhGqba4Z1tfj;N||J`mu| z6mk&(*IBdHM7Nl#ZB-_U!w=fDAd^B3^dGq@<56X>ImFo6)o*Agg+<60NF0N5l{AKC zT3^h7Ef#M}44#|2J+i7??&Fl>xZ$@7Egx?aqX?K2Qd)2v+C;=uD8GRC0?zojK zC}wwg#V;_5fZKh*t)hx)n@Z51RhQD3U17Beb#iVqUJ5AZx+K1diUT5Z)y27Hdf>Hi z?7)=qPcrV6x4d#HmrEwf1>7lbZI z1cy&&XkeB#aE9O*f+653CzuMvJe4S1w7#wav^B`x+vpxTQP;~JRdntO?S(}J1EVVAxUAHR*9q0^2lesD!vMilZ-we(YscE09!eB(gd_g0< zv_g_IK$vtoHd8A`RJ!me^r(x)_CdRKF9wV4o5#0n0Vb?thl2oOLe8L!RT;yK%?YST zP}GeSm>!NcG4YUBP^k9t+QmyFeMCA#TK65J$ya0ysm((<=qKJ(A3e-Kn#|sH#!ySb zvfl&;F@+G7JLdV?5UUZ!^gG60Wwlk;OPGqjj zf_!n#FrpMlT!f`_(`=5z?^_uBY0{#|tbXU>3cbF+2`j8!%caP>6K1=J5}K6#2v;D% zv4Ar%AVmVWtzJ<*6>POPrzRd(J$Uo!Cvse14VK1QiNpC#wniQ`z{`%ekFt3B#4l|t zan92Au}IurO?bi@c>M2_P9n>&o4LByj67h{i;@*hLAVwRKR>TZK|8c`J$Tj#3!2@{ zG33}~KqPAxjJVonhQgNODZ9(GM%U!aK+XqK34LJ{kr40ss^VcahOt53>e4A z=B(uKExv@l-daHN%qc95Rm1zfY^-ML{c zA9gbU4){4MO|*L>R`euv?Um3<2f5?Ms&o@uInMzSBB35T|(y+HfaEg(bXq*;N!Q4an!-@U}a8W+imMh8wno&t>OtQuI5% zfNh^iQ+_@QhBzHM*U?eqacT|`#`mMB%8;2kbfGU3AVx2qTz1NZrw}A-S|WaFI>!%=ieGN0p1E zE=Bm&yU=J8V}sBTE3Vz`nr*lw^GlAKI$udDwyOp@j0%pY%T=>l_7NEZPj0#jr=ZDf z?KsF)FrwX_h-##^LP{vZK2#>PeO@3w5iF{6!2{WdyfM0;S#${1zh~Xt&~4ES!Npe< z17v8>*IAU<6s*{w!$h;3Pbwg}lob8LijcA1mc!JqJSAfHXq6XT+x6$E+6z*KO=yTV zI>fhjGmW&jcjK-0hi=1oC#Q(v zK;Tu#3Z9Y$-5A?O5bSX+ zG?`bUk4A4T-h?h_vd;d!h0DX}3$BikJBeHh(>9iz5(7y!>Pl;{rV3D@Sh+%1McY*` zI!c8Q%WTuwO*j5;nI*Rd$8pNC@onvY{{f}wTu=@>UZYp<1>&h9EZl8HAs)?>T7 z*EY8Dq&aUz<{bivbRa8;sJ<@H9Xz`USZpJr#A*6WwrW|0RXWCYqW0~!K_Pq@+Dk>x zmJaqLYhnrnuBx_cku}45mr#DNOP6l9H0HeA?{0LV=yYu5`Ya5m4*Waz3)(AdHEb=T z5Dg5>3%S^6h=MH#Nyk$_%`1njR_CVypcA(IC{MHo+^PalV~XQXSnz~!-p6nvMGIQW zNwvL6Zxmil;?p~u2O5M(dNfZAWv&J)(#_tZW@?z6;ftCYsR`Q>liw`F*cc;FDI5+p zj4=@6B&i!7I8lK`2G*#vW%$9j*!gCIJ#a06qc#@2eWn9jPQItY7$e7CzS|^D1TG)a z!0`fuk2_Jz&hq+oeIbV|;kyjx8DDaoL4fl8FUlYI6!UL zE9{uOH#><{C2I9m6hH~biwtC;$XSx(Egi##W2o<7`x!-6+I?xducJKh1GVTHsh6y|$MKBAaeG!N?nn zdY`m$Rupk$;7GB0GeD`!G#VW}^h6PQ4$`oZm=SW%Zn<;DIX*Ye?3TNgH~+^`mkJb@@G#X%=~zd?zrGGKG#WmL)|tG7?gj zbyXiiL5`t>CZj4^YFT$w0)T^I(SEXs>kRh^095Jc*>X*SngUD*ERu3>spSC}wG4n0Jn@H|~dG9Kwi` zj#_NbF80d=5u$Bukrl;yx_?9W%oyZ{K7Dulv1S0za>b8WN}!Z za|^W|e^u7PRCm`@cOo=Ukv#ir2>!F5HZsA&HX|VY+mpDZ!fO6%x3~My^nNDm%B9SX z%nGFQmB)dJeP-8KXdk1Y$NIE^siK*4BBW&T2lh?;!xfcASLw_YTo_{rg9rliQJE%% zX=@N2VA^_X0W@vJ$gdb<0@p{u4dGxY2vZETgo-tqAYtT%aOf+-l-q!13asI=aI;K1=a(iR|^D6mNZw6JLcDg;EB zWVSZv6lnRHl<-fW8UeyU1dx9L(SxePSWjCopoqKpCrWy;JB7Lc`sfzFbad$o3M}FY z?y~4=MdoBcX#E)GwuCVVpgBy$2LghYzzkvfs6sfyT9FS5m5-+Hr1j(R&|k_#*ZZJq zdxd(n{7C(*C*X8wf|ch1N=zC%n%3ajXomIH%+j^eITSLd7>E!Pwa zC*p$vp^<1K9|#c;3@ia~0~=xB9ue2B(s^(X1l-fuljR1jg&BG3^q@+ht#vJK=$u9% zJroHQ>PhTq+A!SW#nhlt1;7+c4aVXb^c6q?2a9F4HiA1^>)l%XP;gDSWsBRh-cni{ z(2377Hm#6n<`jz6L0kNg=oDxI6bWqfYV5My0>KS22&nd!mYbCishdtu(+aADU?ryx znAuGNd7cpl27qyW;Cv=69yzsL_&pFDU{Wj6q#4}Hzd^S}-OC4q5NtsC*8!F&b2y(^ zK~K}FSQ1mSdy2BAx%o4C>~D{R|q$N z1jVochUr6dN_ukYpn@1AssvcK1%hd#@q6_#EgsezT%9I`IjK-wHxWNLL>q?P0p*`V zYkA`L8e+Jhq@KDfW;3SDAPfjvv!&Ao5l&kH;<_;`l)xA^)X_Q5Ad>|^{7wiry557l z5dwy0!oV;H7=RDa=>v7yt-GvZV=anm0j!J6ha*0DKT( zlS~xNv)@@l$(zBQ)>8msh>O1`~z}1GW50H^>|LH^4oGC|Vb|5P8kBlQsjNJN2IZ z3T|kE)Llo{VT6G-s8nbj91m^@(0R5L!8%YMx}^<8Ziyji{fdFBW1D^DXoM30qU zCPo{|^&Gvm(ArKQ!yLr2$gdG<08Q+$=n+fygBXfsLO}m-KVS*8pqF<`cLS=Afm@=1 zbr`YYrVVhXPFImHkYKIF*Q=p5ZDtCEM4QxLN?=Xzs-#*1`5K6d!H>RRtU77LT5b-0>Hrt!So>-FtOAg05P!M z&jMP~OWjq70Y1lc0l*NI*|eeM0}%$MKAQs#Ay@#=U>adU5PC23)&i3jR|t8QCwV&1 z0{E@@z@;RCHv?6%?nlX*Qd^EVLbQPEh%n-{*fzWAd^{xR+&IjDRnS5HnYL>~>IH>pD19cI~@DTah|LmPWVJWx#wrEA4oT_yNriAiw4cY+U9ji@Xg+&44YbG?0PZwFSg%>HH=|+tm}Ix-#Zz~|xyhdu zE!N^kp339;ERZ3D3kZT50+G)~U28=$tT(L-bu>-ExO&Z(b-JDd8io%w#{j4yycjr0 zEHiB-ZG+(Xy`1JPlLiQw)DQW*Wt0GE&^gU;d=n(5-VY43a(j-HW{3g929U4U%B{6& z9YWQ#VbE3YY0!mi%?E(2H}JbsA#^CZ){BPi?OWvc;{)SEHw~ga7|i zyW#WpVB`ZAE?p>cg=n|9inwf9LKA?L9BYCN$cxjv*}Iy-OTCne_kA&V7Fk zLI1fH*^7In%|j|*p>@lkj1M<2{NGS&`toE|8ncmXtbW-|Kxa;U*d7_Y@l;jw6U3sr zj*^PL%0hFp50N}9d5zGB5S8!D)Sb?Lih3&|YkC%J&MDy_<0{|iaz1B)EwVR)H5ok0 zmt4AnAlM@PM62!@#+~kE-Z+Fi{ltEx&FNhz36h#72ZZ|Q@Py?b6Xmu9gr-Fh)=r}} zXzglYf*>uirDOXR1N#kzg+%&-GK*|k!9ZscnY3agFB*il!Xcr-CYb;Heew#)#KvDh|zPh<`*0ghVPZ&{k{|QE7 z?l_|Hob{hdg|@hM7U|$p5E1-DBpXupCg_h&4Y|t`TUfOMzsQ5K`B#4JuuMsh<1kYD z2yfN~_o(UWe7<0JqRSZb5RyhEP5^H=C>yFyr3t1F*eDbf*0Fw6q_Y2)cMbmyFVorf zqowjE&<70JvYbW8b`wH>Dh%W73Ly;Xb3Gj52V)_iP;xcMCc~)S^@|S`ec1Q-oogLF5+H}g*5&g4y%&~d!0?EzrgG@Y$Jca} zzG0>cT3#^a*sp=Drk>LS;k|rU`}eatdr^m>@S&8fAIf(`PdoA?ma9&~dmN6nPxn{? z$(nj&FD&ht6L*+r`FhlXVrmqVvm`S`x?D;7RslL!pEO!Hb63qSr9G$p0-kx)wK9kh zWuhI3E3oE|ebLW#Mia~-05MlSS&fbp9VN`_t`3nDI`KW*v7CZ5Hs$PYnLkkkRo6{H zqj6E1%!QTL8w}%Kt1Pvsmm|YM&x-g2+{-L++#rhS#`!BY%#n!!bzE$B9lELYHHE|U zj@ec)IHOn{EE_>=fp=DZiASwvflZV}H_sb(K5wwCH|JRmJO4wm1 z;v|`ZRQ)CFkGZ z-3m@^*Z5+;Z|yP@a$@=BtQ2&`mFCsKrL<}L1J#d{i}J4973RZyvaIoCXOtSvd2_Eg zGZ8Zlp8mUb?kZj1vR*a|Hye)g0P%2+T-7tdA5J@BxF>!=a}4bzkGeFjCjM|~WB&eq zAdP`8{V`r#5oa^t>eTB30XXJh3mN`iy)ARhCqQKKFz-+A0=lI;iKyHqELIUFU}uP1 znVjn{E9Wo$=@sg#fr7@lbk1vYCpPN=ybSE0ExTgYeyCPr$%RujdaYcN_IvmoypN*( zkT}tDWftr;#(TA(|CQ@XFXCFW=?}$Us3PuR>6zf{M8TtZaA~XE%|0`UkMKMRA$h2O zMLbT^AN{L!eDBgsYW}1#7d6}sRH+%s<;S-ivTn=1d~Eq?O7`sGI>y_UXYPl=ffXaQ zjH{w^(8JUU{9(2F!;bl$Rrx`Btn(adx6I9=9R|Zo>D!%^|H+HfGFRikC>A19 z_p8>FuRH#pZ1$8oPV~@SmLrd;OmYpVJ7vJ}Uf5Kezw|lr>56_TbNU(+%@j21e0^kK39_@~tx%HXr2zfdiO+sqYl)#Let@Jf-keUdodc|=<&FYwZ|GdDp zrP!f`_-k_nNiMTQwrw4uaioz<=w@r6yGvvfI=5J^49ybWY?5wBPz%Q z86&u1FQIxN{f5LnkK3GCb5%z{_p>s!6$+!r^kFfrfqP`Wh$ss0zDxbb`aLdevQ-jN z4{bG7z-l$(Wa)6OC=q3fn-(^uZ7E?M4+dzX${*T#c^qYuP*e4y&=<+VfA&fRHVr4s zUP`2r(h4Rq&HsqX@khj*N30L@5M{mu{z_I|^B7ud-cBOww5mtlKz9H;gcm3A7i=$Ei+XUjyoF$ z6ePCY5o@XSMNj$u>0}~f4ot$KlJvU|HzGmBqR_-TZi*=EQzxK4ElXwDj7EB$0wab9 zf7zNHn|0q*f4+;4-o4LpwQK5t$qx}wPgn^RiG~%9guLI6 z?oEEt*Prq7#}ipZcJ|=b7n9ky$Mat5BOLr;1i?<;b%Je+G3qufs-o8z0qvb8ZlWcA zb;%_Z%h}uXc$v*jyh$@=;j5+AkW~fQal?AVujHSwazc(059Rq#jZMoba+XoIKco6n z$Dgcn{5*9zLaM(GOB*v>3f!QBA2qOJR_M?U3Off(+=2%i?Nixg8`C4|lqoF01L2t0 zpZNl0R{y-(vle%q$=!pT1}i*<p zTkI!ewSBkDGZbb^1Q3i^oHOlYoHV{(cnDJ4rVFg^n>{Ea#I@J+yN!V#evh^NV=nWW zAhwvwK?OyWsF>dqD@hJtb4?NolEDEG~;XutHp43X}~N5ics-uYOjpu#|BsJwbF1jX%cI$C2j$TK`Y4 zLuZflF=&+Q2^1vJ%%ntKyy5kAfTfofu2J2@p1h!o4W@tDl3@jLYFx>e8$+gU(AzgE zeWc3k&#DTeZK!-%K*GR5`iu{$6>a=wwscg3goRW6Y6(+QdvB%esX6q7dS9X-I#NR!q zBVyK!R@s&0QMaCJbh#muk7V?Aip^W`12839y1XNO{hkv$Yx%V(k?V2Ae0xWBqnRrv zK}HZYi}u!>ymp)l(~a^BU0+o(SNKu-Q#^}gcr{sSXI5QRd+7))WU*}y+pm~Eyw+rx z*Vqs-3AkMGl;6F>;y{j5l?&D|1JX;#TJCB>e{v&Da5ql9ehDVt zfbO!0H;E9NG>clSB|AP!6@MoD?>Vnde#Efa)EfiAgB1&W+4Fi|{BhD}!k+ycVM+QPsq zg7AvF=$;v|vXU9O1CNghCbf7o_TQuZJN$Hgr>rKMHatx{$k$qF(&f{%it z?GlQ(P6tmzU)})om@3B)G?!1Ep!MRdO7OX9Wu#t?78K5U^w^($`~AdwQ09 zvce)~BY5A6S;=!Ec$NsP-6mz)G$njftgA;dsugehi>0fYWix!R)_{Kx^2j-|!2Co1~pB0W4^pCwW-DSiU_b!Sl0 zX<8Y^CYczQ>Ed@J_By65WMSF|rKBPD(M+aG=MQ~pQhVJ~%-;}AAZf~ZV>wnlO z^z;E*2L)mvdJMesP3xp_iN?_pnLpKgcHX)0$#MlQ{&Zr`>~AFBRT^j?2UVkS3FFra z@M|Lwj3=*o0Wq^luZ;G?6=YOV^@!cBbcS-zv=kVsIG4(bz^!KX)5%+inj{xtSWhA` z6+5c>D`#L~d^IqQy=%9Nu%&OE=flr@M`<=MShE~o_j@{y{CF5t%kqQvQweaUaw;_E zGKC=}@}(%iyZVb`SMItyGC;D@4tCv!vFb5XIf))jyQI^bsQaAYl7<{@#nQ(oG%#&Z z0Pen|)i$GSLngeVyVXC?cl4nNoH-}WEm|QT*nU^*2GCBRk^pEcXCzFeZC|@+Jl*Eu zs(i7bKB7(|P?LAcoN?%_p&=2*4Svki#(&8;=t^oH<8blVQ8lBE*ze-t8jhXWQC?>_)}T-GFYl_WD2837!`lD z?|?J5!@k&eYZ49*04rcA4o-PCA)Gnw_NUFi`(ab!e@|D|;r@2G_GS%w{GiG0rJT1) zLk!empRC_0wcMUje0Um20g0)4vEVl2=#`+}g0gyinJI+i9XJ|sRf7#hC%=3+X+#bx(ggWo|uW4J#TI9KzFCTIUubTV+Ve;D=~`QW&a zOZBglzbUV%3CV2m_V;WYr3OcEi1c@WlH?K35YeUxI{4CzIT5m%s;T?F=zwsmWS^Nz zjoJ6_18XAB1Q`yMHomjmT7NmF;=6mS54Xd0OwT8h=38!?(no+kJkJwI-lb;$9mf`w zA#x&aPtG~+-H^sv#M)8yvU+4aKCO}%lA7WLF0RM|C|i91?1m->OWmEt^u)3ESTYqd zHZI58Y(5``HmP4T{v5Z}V<}g(JaHBzdxF&*rpDE@z&!loiYCt{g+KVb!8#HZS?IJ$ z3i*DlLv6rWmrjo74*3FWg`SPJR$R7eKY-5ox|?4iR%4s4eO<%Pqsht6|NAD-g{qIW zPl+6~)&a`g0Q_Bsm&2S9#;*oN^`?v82okwh*@QlVe4M3x!v>6>*& zsV93f5YOcrx-25_@E^jG@8l3A-LBFkOKomdoD5G{cnT+%>zqa?D;MRpJt|Tzmg-&M z>IXc=bfMe9o^X-p`DlKvYhR{R4o>9Tix_fA;9+=g9zf+^7G%3>~6$UKXrxq?s5_UIe3Vl1hH z(lWRzfcyu2m-E0Myb-|qzX;H$z`YzC-lr5tYPvE!JOT2$gjVk~^;YuMCX*y1zTrO1 zM~YtqjW21xspkVra3To#Yr-qOa^bwTEHNEco|6d5^AK?Cxh*q1mH=7-K+JKZd%s&9 z?QC+%t)=$t!YroLa&=1aM_=Jqj=ckRFP-SQNEdJ)F3e0*ecPmy)BIXgknm|nkbPzN zDbPl$uTk_Ur&QS@7Kp6ON)p!DbNG=S9Ia$*UmbMADdz*bm;0S zxZlMPuW&v z;bsULH=LsHrK%BUZ1GA084=?Vxd>sri2-Yp=8L|#!zM|BYe<;a>9GAkgR%5KfFR)S z`=||6B(IG^m7m@G&(CbC+i~;|RpQtSQx{EZDKWl`5FA&b0K#v=`!Bq6u&C6hu+~NL z&kL9PCKLqlAId23*hnuqnl+AV2hN^xKBK0nMm47RPNf@VZLVt-s|t6&9P*bIr8lQD ze%A-9H@9>#B{k^lN`&brkp0yldO6=xuEl!3=C!BRsJDQ}s&bg!%tMUDidZox5GY1; z;~TY^pC00E3Q0HDs#xADlB@;Rn`4*$4QDZk!IyBFv5EjLrZCv#iDEa0)8pqD4kI(jT*NCl)D|UU%8hN%7CwhC%( zr@{lBWJY_Y={qz`y|rQ9rGg2t3mw$Q{QhWnWMs{EuRYj9ndnb467kw$t>m0X2lrie z-IA*nZWu1EabtP4PAAn32046ELvMz2mEY_h7iBbMK06qcB(t{MvkY0+6$60&V%U;b zuBSE6#Yq-oPhV(4taO9#SBgFusr{O@)eeE<3Vu^6YdYB45e(5vAHBKxQ9IjX{YST4 zL>M{dlaNPIBDKARBw3z=7(@o0+xZk`TL}m6fnYoXCq1+$pByXe!`faIsoVmDBj*|? zQ_|Kx8XtbEVrNs_uy4AkReW3{!FOFgnh6z>`s((W6r09bus(86>^X4-2WjgaOYG$5te;fa78Z-SOC?e| zN-2#sw;hESY4#?$ZPkpJPk%|soJJ}M*_KoI#toe{^yPOuaE-dX@re}h7r7Us6TE*6 zWTB`bojzP+SIIn! zaHN~)Z^TahoAVg^NkDeVFBKbLk*<6P%yB-=AD9!GIaqB#O0TMve4j#B;Py ztMcrvY;u&6#T|m|Tn=iLhm<+#$@F$7rL zvf~@^3hzH!K2jBIL5ayK0vLu$bh2$qo%lamd|AX;^GP+#|AI)reH5!+p`$vy;NRNg z0$Xg&r)5WZE)mk$;$sa3Fgq$yQBsSRCJ7h_3i&R+K^ln4R^fXJflh4YwbFSF%Bf~| zA~@ge1;?>&ma6|GpdfID{$0?<9_g@!IsA zuiMD$D~>X<*D!kIpCb2Jh0C;c$*A(T8dHAEOh^?sCR`3DToK*m+C|Z@m!0ZHhNa(R zX^==E{FWTG(k_WUvw*6!C^m?E3eqVQ#rGJ@M9&fWtSgWHf@~J^RY^Ryq6s`+5rJm8 zTww=fYkyI{b(_P7#qN8_vkKIrHybh=CCa8v8YVX*wq|FQN~b$T*<0oI2_OxApB(zc z+1|L~y%nhPxBBc5>@*!hBjBY^w-g_h7*{*=*$8Ra!b`~E%$6kwS9+J#HBUwM3*Ho& zuOjW@u}3qEOe+2Lo~n++qQ6dPjsNJNutG1|ZG*bofnZo`OY^5U{TF9>P50-f$2!6h z#CKvkoNm?xdfF64gdW6Qj1lrEaB3XyhTjv;0E{N(U1P&Sw+i@;0OAp$N0N z*oIDz^3FP7-4E^WcM#_lu(_GoZlNic%jl>3E_9Q3f0=x@@&C0u#y!Jga!%umox~Mw zbchuK<7B+9OJG@n+P(xB4bRT)_iD^j@ar-sHaGc=YMbjZ5Q;OKUsn*-3ptr)(h=%J z+)TLqkeFkftFJhyi7aAMjph0Y%&lOU4-ACUa;0J;wnmitTpx!e5o&q<0w6G|DG`0ugWyZ6pU+egkN1j1-2Zp z?7o^FWcfP#HF8B;i6QwEoW%2$97Tmvi9&=B`S`gGZ%x*A z+cg@0T0GkGf=Hd!Q*XL8R!5Q%DjSU7MK;%}r~>}nED_L*C?z~L>$8e8@5Hh}rH@W` z-}fw5pWk0*`naF=m%t|#4<60;+6h4N}Ka$4F)tng@sWP;BjigPmTT}kyX~R zeDDh@msk0(Fk$FFxgWWC)~x5sD76v^)u*{gc?RK!O7fqJh4~1I4)BSdKSnd&xse+E zgu4q71*2)AsaQRZkYE{R#ioMk$;yY~Y3RCnn5O1B#Tjg6>c5{;gTB^$+L`IP72h`B z**HY$`>HR8lWG5l*9lF&PPN*s`%*DYwsEl@mNg7+0ceo9L3@jwTWJe67`&B*iQ@G1 zu5^$g^C6SyW)Z01ZWhg5#c`pn1n=r0z1c_r9wFNohGG-O`eWn2AMo z6G>>Lq#gJPOvJha@+e!Lok~HyC6LC==sXH|Ir7GddE3*5=EQ}zo2>tp=?c@U6mo&y z%ccdQz5qSmx%wbJp>!oOiiapOX51VGfwayn{{+{jd05Hc{q3Dy`t-MY7BnZEe{Emt z5pP-5EHJDLJb6iO8D6+3fEfjSk-$zJ9I+`?&^Q&YVSj&%$^?kYZ20xbLo7dP;bDtF{)>+!oq+C#!9k)0R+aH&@nOakVth9msfPaq z=s?ckm0wEuTsS|Q`YVD{;ED$zHIBwd>N1Am|7tqN4mFqW48GXc^eY+iUz%AHlDe^^ zxEa|`whPf_m`$!LHkXt13&VSG1lM#!YEUBDf1%cj?(s({;>@D6nR{NNy+7#evAs=? zY)#uzhVeXVzgtNs0`9Y}_rZKBir7c&2Fh;n$|TD%NlnumxxB?7o$_X0iP6;rSEnTJ zg@wzF<5pWhxB$Hc?*8_^KPx^)1SfG>Y|aU_vk;O0^U{7S&RNw+2-x+5_zyR(Hr85T z)A!;AqL;1$w-dFAQPbrCKXgF(&rGMBwqtrhn0GqD9Kjrp;D1*+WxdIwC@^7+5L30d z=`21*obim`oRK1)ny)h(`z|)zYxIIYsVvD+yb*WfQUpC|T)l`?dHFW5C~i?|r$F(Y)FFigjKRzN(Oba!Z+wWbTa)?(tAxcFzSF_m zK#G_zaO_9Aq%>u-{dk!DgUYJOM5@A-)2yLZ8`|4fLQiUFB6)YQj_<~<^c&t9hLzG- zfwO;?KKW~A5=ooies~_!o z(7`q+iT=@_=!cl(c+7>JdXDChzYW=gyd+Y!?mwA~mE5ZmbLmgg)0ARlZ4UK6M=AWY zf}=mH*vhoeHwm=K6N?-Wfk{xSPD~bLBFkhJUN3R%0<1En_s3k^JMvFNCgHf>?K?@o zVWVwb&}n0FHxp3-Y}7s{Z-l;+E1RSPu0?;W^2u-?& zY6jAHBtxO2PU&P$>^23r5UA?3j#cbou!6IY1R1|dBcsZ(XuO+NBKtejS&dsoBGQ%td@jd+9E}4O@98A^C+w}m?u1U zu@ZYGz|48`$K@`oQAXBQp;`*ifELO(l1LAhW9~uIk{SuQ@O`_3YGQe9kiJS=`A28H z7MJG*q<}I%1^H*mFtDvjLv}$L%bFrD7;T%>-FG~R$ycRc_j3brru{kHR@%Voxp*n3 z6>IgLv&b5vacq*BK6LxXIvU+TDuR1$9`Q@#r?DZ-xUo+L^4qdgE7;)sbejXMLqDq7 z#Dm)_3%3PRw($ig@A{`{4tgHHfJu%=g-2nM zXnl{*ArA&j`Pc6;QF%~?s{PlYeh(F_(FPQ^fgWLGQEh+5h2zY{tjpuJBnwQj;{SC> zC-!e_xl>Z&oyM3HJz24m0WK6LGJ7THMLo|{^;~lrQg%{>3OvslGf`mvWkwN@QG#jy zn;oi9(c(}<7%_ML;<^cXTQY6Ujm?kF9UIq7Sk?#|<4cdFF%hZD z*Pkn6EG~5*oGUi^u&J@3+5E4%c^$CaF1DIw)t0wAs&H+Onw%<$5*WeGuwb8m76k~B~ z;{x=#=QNdNI_kYE)-w(RX%$g95%F0`xYG^T2_^|`e3QZrDG@uDF~KQw-xUzVTwX$j z+WXld4BRA!IIYXHMIH+}U>i^G4-difv8i(OHs?%95Uv^OJ^%(7!s)kOHrzh) zg$H~s2L%^t>F}9hj|TSRX=hwTH+HTJQn}VX&atKFtB#i@IDN=oT*Z;&Ok{sg_t>pI zy{b|Bu&i+9sM0{p#kNMB(_YcYx;$PXAy&FSqr#RI=CX0lpg-ZZV$yDqP9ADf~$_m0h?h6tX zA2mx&S$@fUm7ku8;D6fzi+8&s6VmMP_^(Xh!khOR^O0`t^&}rgxRSft%v}uo+d#+d823u3^4;Qo2UJu?MobASk{QP)g z==ck)F`u>fh=qk4dy_@>&W!9m(+j%cGgo#6_}?=;Dd&&x36eN&HoD|DUPR^u3haJJ zBJEzdj0D^iDwKzLnIF9epN_CIUN;QzEA?7wV9AOhGq}Q6Y@Ab6iVL;x*8g~12gLgM z{r-nDYVLzLtUEn9!_q8jH{^%gQELVnIi*}l`zMMrlimC~ipAHqA>R`&2cmwwBh@I{ zIeu#iifY>h)0h*kk4=#M`}0u(-cT82_$h7ptwURFvE5Y(&sWdBmJPod|Ge>qxSO$B zi~gF-XTBL2FB*gA@(pje#PXp% z+ez;u-wS$^Ye~sQl}1=uMF`<@DLt16c z>WY8@xv79UYcJk#fcI3ERtmnyWZ!-V%u?C;>L9#Nb;NLYwfigtPKih^-bcOu4MFtr zJB0`D{^XroBuGg2Akx<8jkjHrx+dVm$Gwt_uSpdg&S?z8-el?JTgr{h)tx1P4e-Ag zjcq#3vLBhhU$(TEG&8pBEZCQzk)Fuk`OqpBfwI4=X%V?i_P22oEo8HK-(H8G%jZE@5iMkeHcX)g2d#VR( zQ>2&zYldHel6QZO+uzlbaZWRh)h1!7uD48+p^<3h82f!u~^btyIr)>4JfhL zBe?(e5cc6O+o)Ra)Ea-E*O4D#s>fTJq%)eO8&lLgQ5JB)v&2!`Sj89rAl zQm+XbN}E%^8-L8wj+GR9ZEWAAdvnl{irdCQ!rx`bbVa1a`0-kmad5|L2oq6sU~kJA zr)#BwXx1KEfgoO2Qpwl>R7@v+HNKpF+QFkHZ7g{#^Uy>BjBZF-*T`Meb=54%3>xY6 z3|_`@SI}%6e>tucWTnz8xa+Zf+j52Ets7EH@9&r#*rtxH$< z7BVT{MLkrYJ0&S1DV^vEVWDed5PR%IcT~Z0REzWVtIYIRxjX_k0}Buy=X5 zyhoRdY$`Z{7I|*?^>u>t^i4E#eoxt;R+HUy9?KewZEm)vhGUFiD%d^;<60fRB;lx* zThPu0sJCSkAxT02lg-MyRw=}+I=$FB5e;Q!(f=6Z?WEM*dg=99#EW5RtI364?Tw&R zYppf}KJb(S7y2#U(DlTC9)x-+8f>I3navr$4%rD)Eq`)yC#8#d{EQ}7!b$jDTS)eO zMgrdnPSv~WbDlyHrmV#O9q%S6!_GcUyxik$l)r9_epiHDq=I~@-lu(PuXa<`q}$*> z1qq$@-zDor`blv8ru!zn4l|8$-$3_p&y4stA#X#BqDCaO{S+gfKn&rlmU4c5;VKnG zOo2-+^3y_7dowfCIJHX#l3&A~kYY#naH!bPGqGY_Vd)=yCfps`Z?j1;lUlTCmV&^W zg>AVVWOWMF9^lE6Gv0LGA|$!mEOSWFunr{;I_hyxZAT5x9e?AIvK)P|C|F^mA2F!3 zFcmqMEenMVeQo}fY72Q&BVoy-x>K%@7BO~!aG!jcV|icC!3m$EjZ&lg*Q{pdbK1$S z7ZnM30f6vk9}5{V&sOn0kGe?Jh_5FDqs$avR95m->8?~SWnZ5cy%BSeWWnRo!$l_d z?nZw?A+9?oNK<0j%|~n}L+S%h|LKZ*(9XF?w+SUGm|h>OSp;frpG$R9Fn*}#dZoT} zOpi<)hN}Oa6Ra~lRQ{rYnm{m+Z;7#WgEX`x$?K#-+p7e*8 zkdyvvTOERB}5N^aqdk$0W}1w&tIOGe14m@K6lj>GsvX5ZA)HV z`>s8*G*WZBO1bL7u*>kr*OLh>-tByEZlGDhi(m6f3I_Q7>h%hCYM`hYb@RBPZ!PqS z6BjS@iG0IWx&M4&SlH3!wCL@!ll{@t_NPctGPp z>82~0@LGMi!ofI>7!S=<2q!Alw5-+e$MG-qpClKtoOz8`!8?L!9|(wU#1ECvR&T?( z6IR3N(-klFBJSki6>D>>SzE1Iq}(5SgZL+~MtL5bDHQe05Xw^NHXBq!SXPhTt;C-6 zJaGFw@f$Jdy{}GLS4Cs0-)r9E_hf2(*~>3*r*C6;J`Xd}JwV2dJ}#N;a!B!I#3&E+ zY^OUl+KKRBJ70a|x>sM7;X5N=4G+x<(z6wdV>`|M5HV-rPPMPJ+g}+Re{bmFul_4R z*b`YW_tOd?z!{Ewr#4l^xs!c4pzO0MHO`$ETe{mCfJdw`n{=#&9X1!uVv_~x5p3g- zP-&L^QmG<;ov^r&KV+{FLYlQwznXi!(YO?P56UK!llwHeZK52LILGYg7Tr4&6LhM5 zTnq|&Bv3K^j}5FXb7@Xf+LHW_iA(AYr5>l9qxWQ}P_xyrbv1m#+iY|oR?7S?Q5Iy1 zQMstNbuBc{^A_5cqvjf`iHWzpe0y}mZ4|bripaQy_uQy$d1=vgy^AX6eA%^UG{Xh} z@z>l)Xsl6+zj;3H@EbcRexNO)|9#{fz~x}rx1hFn)^q$PjCn?^Q<(Qkff#_l{8m>; zuesS*X~|Whl&dlTPBb4&`Ia;^|8IdSSE>U2+3BfU-*!(RCl5(t!5w(J;Ol zPD0>%NJymCsV*L{@v2sq^M0a;bECeSI3b0fb~yX^R;W6<{3GSqGMB5eJohYGIIxp# zkZhkpy}nJOd}c@L%#L8w^)i1h&Xo6NeDihxS~Jh2(vJMOYQX81J!c(-Ul4!yZPGr$ z*8maJp~yQ$ICrAt5z{FxX_qI`II;dstp5U(;a_*1nHMyPv|Abc;h7wvvO_Y0<;5<% zYfLr+JmCQ5IxVIpd-_WR?nk$s+aj6Giper!2LnNO;8LBJ95T1uMi#1<4)BtDBZ3!rmlWh6BgVZY`b?i zH(K+^cf}nehh>U0HSU?isrAe;tm<95&*)>lAQnoW(T-V@iHD`*I>ic(ix564WoLHJteA^y0QO9@)TE#U6(#$uIM@@}v!Nt$@O&?lT z)lU*6-d@i35)V)?s|!+7?-MI=39}RXGD@+yXfTWoE(-WHn(;p=m`|;oGaM8ba&yqF zzBzMbzp{R_Bj^3A{lg;^b%h0J7!0xFxz%Oq4|;GU`cleZNs#U$6(wXek3H0)a~g2S zb3xU?#2b967Ax??0G~rxAX)3P^0gy=@C1|(!}_V`tJr09i0fQ{oF@XF1nc z);lVKxs;6EO30aU#gyG(OJQKE#Rhq<3_5n%XftMGh_~VFX6Vc~W>Qe}=8Ipg&-p=r zrcWvuoA^jRRQCG2g-*)BcBE`J_TTu8x+I$nq`JV`36ntH$TLSpI5r<%@v)rM%+*=t z%ZD}vKC^|X?we%dRT7yLYDHx8Px=nBmbMFZyH1rcAXqKR#M3X$45ySl-9-s^p8tOU zb@Xcf-?|9vH;jXIXscAbo~%6v7%5gUfdTy zI*jt$e#%Meuc2@WBSD`(A1EtC57_XMtBtFpeg+&2VZd{RQiM0X&T$9y>V1UP^c;*j z5z<(F9BmQtoD4@kadkD}R>BRlYQiQ#dQ|;^?-kLHm9Da@8+l6PwzjS<;a1u zzX_i-rI0qju1%~hR^pc}|ITO@yAHm$a;+bes0q2BWdJ5xdTNZC)lX!df9z%a9JzfZ z(%z7b#~{*|6)Et-kjN3;b$cZM*Na69&f3CjYa@gjaFVPHORgGS2VeBl~vLe@x=}&Z;pf5(}WCJ))m93+`zj2If3@Z}HEoTfOVS8)j zPZAEF6m9&Ogx={?t~v6nA;~3x%rq%;8!|>bYDaN@ItknbsvcR8x zxi}lK!tVwRu$jJ@jFXlh*TGPb7xuxTP*7kgCaVX#QoKrnksI%--4vwyV>PSKwXRkn1SL6`3y~KvSs;~^4(78QoHag7 zDDKryX9Pus){e6>n1m_kTGRotuOUE1_oNe2 z$u=&{8<|L{nDqQo+urdMW3LKD)9np7TH9!@p&)g`>?SG%2*nGfH#gn(!amo@zzWIc zLC>o8yN{J|0})OJ;m;m{(LjPGxztR--$5HJJ~A|OFI#XHt9 zmMG2*6*S%I_ft5`H43O(+;m12ca?>Zg>W#&LU!U55*{6{ozWGaweWzl@a1eSCnu5G z{4bOij3X9CPsBJBXuxI>BWaBLr(9yj=+1|{6WMBu9vv!BA%~Hw5tftPc5b(_FBgz3 zJb`H>cS;<(jvXuo>;>{PkoIik!6?bJn@<+f=Qu-Q!C9#|cubziNsK1V@-7BH8AV zVjy?zuwb<6YL`b!`~tz=@T3t!=JO?08!4wLrval#2vdi9lrFB2+Y^Jr$(PPX7Bn&= z%tleAYIpNmw3b*Wv98G&uA-sYdRxam01+yKU_&LL6z0a|TRploPrEWv<-b7eT?<-P zR4!VZH1;`WBmzHT)|K2nKt|AW@U3n3u&xgKt!!Lj2^<_$C!!a)WlU&UbH@g*Mkvn$ zq6)xNcgl^7qhW`waG7-|mqW)qzQN5>baj^8o4T7DY*zA&-~)&egF)-Ti0H)uG1!L) z#)HQaCCQMkci1Pm8M5Jnq3zSVLPzWU1X?2 z$rmMoZUO$6b3Qh@j?$^#=XSsb*;)j~D#m%_7`oKHUOx+A@|!ku4np(JmLiHbG1?Ux z?kUr@ICV2y1rw0_c&LVhIn0Hn<2eUe+QG1cGJ^vNSFQG+YF%5#F%P5$Md?e#dF_JSB0i!!=wV|H^Ap%AzstfR9)5Ry zMFqL41O|0?P-$Ide&s8s0x2?E8_E*%D1%Q{1(O@gpdZ;L@THM!F83)9g+XjC)DaY< zIv6k*PS2302CORrtVlkUaO^muVepET_nG>WEPdwGo`o@kq!0!ZTNGrbR&rbH7))T= zzPE$hM`jQ{?VKv-5yV;tyLGs38ghe zU=Bu14~8tD8_=ROfO%w7cB0@aPK32+s~#Q_MD@Ad+319+08^XWqeI~4bO60^hN7Uf z(LM0F*Hvi~$ze5Np_q+XfUprhOWh<)@vjSVx(j)@q1Z#62c4`-F0j0&wISRp3!oB` ztMG-Xta7R#dmr5oOlAi7bkojPEF_#vSJK`KS#x=cR{IOm?1aQ9AS1y1#4h5~+V{N_ z%EG0K1F4x^V;K^P|5$o&x1Bhl9_xVtN)A$2;)h`M6_dPv7ot2- zbIG(>tIN}Hb3uHJQxXU+?UeIFcPX`{l%33hgLoaK*8Wiyl6~a!a8!Dhx?pu^#hoH5 z)o-GZnKIgETihX67r2gS>3xDbNi%lHi7{S zV|RKvoO=_IHAHHPoV}&?(wuQIgw4!kIB{CAtuPpnSic>mBp5-H8Mdd3Q`Jk|lNZf$ zTd#Po*4X9PG-r$ldJ>8d4i+2|8l;!U<3N@pZ1p3VEI|q1C{!VmoMKauK+S^fBaUUM z*Dz6_L}^w5aoff}(p-rKXy|vd)fgh{0vQo*(!po*i6jgzBZ`I|1e51WVBC0(qoLm)z+NuU@z1e?C?0BDM ztIIOe>*NiULXI$}S*C+ZXPi0|yXSr;9X!~VA}ZbAY*ZYZiVT2)L$_lI)uOFvpi5=} zFm<`zXtVhw1@U+!u8S(nr;p}*Fsl=E0__!d5%M0daaimGj8xE}v>vHQ+2CYQpGqIB zDQy*z=F!t>4q1H|!+z^9MgWM*eRn=zRMX**Q5%##cBxKCd+Vdvb#oUZfjyVaCyike z7*bu|9$sk;E%=am8A%AtDihOc#j#Gf9->cZAir-qW&|U?U7$2%)bv@|>jNpdqelFZ z@esI6?6tzz9Wi(agb#w{fyyEqlp)IjvvR3vxQ(DkVtZJL^25R1L(@ zQg!4V=AqltQF>~=1XfYjZfeHaqoDh#scg*eo^EQ@`~I=+k)R=>!y*wYu3+#)h~E4C0|={|26g}913sl}Gd zrd(C*3kGQABC(OfPoN53`EH(B5dt68iX3if937^jSEr!HmIk65e&hfVg3uv?wue`w zqZ~Soum%rk~B%nhDh$S@QEjWB0uY5@fUoe6v6 z<-NOGz~(AELFeOd2bJ;2BpU6z<*EfiOTSFtU55SS2QIh;!oW(98`5$5>%J4yzy!d< zX;3NzFrOI!wB-hRg6b~?LX2;LGrc0l8(E-0i_UVP5)ed?%_3sjk( zI8;oGMNzg63ZbtW@t@Fvo}*B#eOdy-aCML^!bKso#zjVsI&g z%M20VKx8d0e1bWe87^R-Y?iyuZ8nYp2eVHy@QljK4PZr6qji7m6D|UBk9ore`Bo%rc zU|bXoGuY+`)9|eeWL}7*bkjB+7c`iutezMv39Kdg0cNwnMams#h84^Yw}=-41&vdM z3KdlAlG(*KbCf7J3_f8kMx#;$@wP3!fhKWg+lP$1;wuCB)S$^Igmh}W{GB$HosJ&Z zg|5EaDu7njcH$67LVgmVQY1pt{E=ItGn&1bRL&qnrY$0uH-@k2QU#VVGkScWx=~7U zkSvfva5B2AEKQSKF@a-H3$&{-V}x9yP?h2tQ>+253s6;eXJNE6bz^zDl`Y(( z1FwQkI~DL zR!sn!IDTbG7fcnGijv;_cYdDUdh+GZ*RLG-ZyoqJ?w=$VT$Be*06zQR`9wYd_y7a% zx}G~+_Mh?3hw-c^6;GNPzaAUvocsZjAcE78&q%4Dk;u0si>XHnvPEqxX=^P^J@1}h z0dZN#;C+F#LxILBgJ+e|gm*c-SO^UFtaiC&x>;;*A$~A4&F)0U=L!!SF}2^eq@<{h z`!F5AK}4Kez4muX?)+LUXp50*7D}v@I)<{cB;I9!U4)aNR=zXMCcbDRrFe} z%{#uT(Zd{&GmMoW0Qx)Gh`~(>(J#rMgS2FAFLBu;l6U3+4z&nzsw}ZTzp@FCT$UdyqlZtJL=c z%bv+)L@r;oFj^)nrzx4-H4)q_5x1589i%DTO)lNU!rLlp+!-zCP3lUYC^jBA;UEA!l2XVmB!GwJc*~y4hh;Y ziK{KfPqi3>sJzoOQzYYJAhbrrQg%$&iY9n$?@vWf05c`4T>|5q@e(rCYMD#=3VZ?@z_92VY{mu2bB~&gg4N0rCmKN9d`R44bz3E zGFb3r3owSLPZ6^P2PvxIX*A~Ty4Nkc>4z9%SXj3CLVZxi%MCjncJ(;iu72$@Vn`dJ zR!?bLFJpB>pnHn#WLnm&76VZoO^nBwvKH0A#Fv-a)B^;}8PCLP;n9}dr7v*CA!x9hV~L&FHl^d73b>vo#~a8$ zL1G#R4&*a8pfz&)uy}^)GlSH1)OU0fLyArXC{?wn@CQ2NzPW<9Bs-Ka0}bJNQ0O6h zV%@(xClkTm%OFp z?Cs8Ij#@TeBW>VPm64L`omdD>N3q3jKAlS=jYoaDPP%L(K7vcDG|(y1kwIpP`=ELA zxD>oG+*tsI7)ip&MVLZ?qh4r#3c96hA7cvC_z`jX+TM(&Z-w2GnUAn@K&gknwB8qgU=e=dVNXnz3`#$yDKzRiGIGa+n>$4?bHqL9%yKodz*k z0CxBhR?wzfEOsR2?nyaJ&%#@V=Xp=++wDE7ccP~FEQx9w znJWC^LPW-M)K)&=h_7rU;udn>lpF|)iIVMAv<JtGX zYM!TVsX(Eu1DJTPLMT8$HWe!0 zV*{ZB>rZQ00#hmj4wDA1rZPuf1C?sq1N_PD%TW-llR>j7>`WK3n=fz|2?+C&i+M0Q z53U7z$EVHq_FB5PVplJO6ud=8mcc^EOVe)K$O^94+%=V)m**p{e*z{3#w@9Zd%d2D z*4+VG=yz>kcW_Oz5Ym?t>o6Ryv)uTsN+?t3z2wI}b79e_G1)$Nd?xVxkx}K*2A^A~ zAH;cPg+|lTOa7jmGWOXZjqdJ z@vIJ90H{#CWU1oR^zSdTS7)%n9L{cl?Ow!1i#}xn!=aIhE@ku_ z$xSouvK9;n5#I-b+B9go0Erin#xy!shM-R3T#|>;Zt^~zE@stN5EssD z;HBX+RKtp?C&?nyt1yBjn!DomFDiv5A0nkuHpyo!?HezpltQkjcYMs;jS_{K&<>$* zi!m$iSnX4_$Pr#j>KVI#z1}QZ%OD>lh5(zul2hP1Q#CglbLo2gOERi_gub#aRzEOb zhq<$cSbZ?^tj-8& zS=ljo!W^`P9;;;OQ};EB1?&4Ss-^_xgvd>{tzv{)#tLgupfCi5!i?<6R2Y5QZMkTW zBts)`JJ(RLt_0j*%Py^SH_sKI>hjATpT{_7Ebq=K=-cM8;t#s_J~f>K15) zBS6R-OBf3rRq@#JbetxLB`QUp4hszWtuzwrrX6~diP>$FS?6D+ zE<Zl^SPb^{4-x$#|e$^?_5_WizN zo`fzEORye73MX|P1SDgrOreJoR)NYWYnp~gTEvxb_XZT>Ln2S-mx!~SWKDj-k>s{r zA1G6*TEI~Yv9Q&~`N2G%kn5s5J3LTeD|h&bGPonjVlUSe9yiiOo7)`DGLkWHr}rj6 z9w!jHw~?KA)|Nhd!0$GDxj=0m&iomxeV}9p9wHHu`=yQD*Hw;=h9aCHopB#2PT!NK zF||d5xkFGP4+WkXirMR-_7AF_cEV@C)|A3$A`N##=>cL37FJ{qLvg7F2o%1PFivF} zTb{b>snTI^H9qn$hdRv2$q~c5JxipTnBP;tn+Qpo;~&l+6J+bRzaDDv)Drq*doDSd#&ruX(KqkA2X` z1T%P64=T6@w&09$Kq6vL9LFjNnEHamJ$Vgd z!BQ10PupOU1U|-gi8u*}6Pvj5{PgWni90;`C*Z*8mhK)tQhPlRN7dD{KUx}!)Ho#2 zvOwsO62L%%rN(V^^t3vK)#oT(9r9gVpP$;c64q()HD3-J^Q#SJS76)YG? z!HxYGz}jIr6%LncRA4-l>D^dM4Fc!7yT&W6wTayz$g@x}M##8xHzEl<-H4=r6V-{1 zMd!!@1qE6Xmy8?KFcN|&DZ^^&A=%|qaI+{Q3>x9@n;%N~TRPU;ip3Qy1ym}=i!_IC zl}H~dDq@A~6o`_r5f}jdKT5i>lNp>FeTEH8rjw->r0QvDRSOX-WO^>*np7SP0sgaK zno6^}v@C*?YSK|=*KYiM@CV-mmMW{~C@wiF1LMDrop%?{J>WgclTIUuZ(T5@o9U{r zej22_emxxDq#ZMmZI0vI6g-x@z@@ zBU#**IT4Ep4uZ%|RmMNI$BGJb0}UHYTj8H>EeyA{k_M9X?9HzI&zM6kBi8vS1RUot;`Zt5}B=gBU~eu zN#O$0!-O&AuXkPt8bqvnj(Qd}vKPvUFR;>tjKr z9+2fj@mrk`^17B_4q27IOWHNOcfmY^6i-%FMjE4NABC4YiR+jT5Q3}}{Rg|K28BcC zakF1rF}S|k=o*xqPUFOhYUi+mDvH8QqhZ)>L?U!eqv zkDR6;_-~l}rPvJx)_uc-{<9~T<*Ry}mVH&XIA%M^ZXJIQD|6y|k848xbDUnqCf=BU z?75kqu5fy`16Zlq9b{WYDh!0StxBlgOusl);R;11&4uy- z1WyHm+XyJ?S1lX*T{2t^4U^C#=8GwEE9;E4*aCnMVYjFEY4z#4#`9u2uLN@EhZy5c zFLzKA*CyZ|+S7%|4)GMGQRz|*Qi-^OI+tGBS;jwY@=f0ot%nC+UWxeyy8QoVeRArY zBEzVz0HQ+|$2i2K`Di&I1iF7K$AyJNxhB)o22wpD)T67iZbu1J=gn417VTy`f@Fp77GX*TcC8&tOjfX<#t+xc-j$Q95^{BO% zb@OL6>r;8zF1Qy2@ZE4l$v}HT*{|I^=`7%E^IirHDFjSXA$Js-_XdqxieUovn|aj( zZo#N+(sCpYDZESWn+CEt3FKpY4Ct~W$ZS&*yl0kk=Jf-RC}Q%4TpClRnCD)=@kVnd zk5jpk!1w1%f3}vbY_8R5e9Gca%apT@y^RlZqd94G!Aiw6fo;({?wp4E+EySYW*%8w z^}Qt`TH2;NGGrJJVo}f&#%1710yYG)5{9I}98%!|4{o|hg5p7=#h)$Tq#wDEDvV7nXHgE35M!Ey6#9@%P&8g zRF3>l1UGCQ?$`nZFttk1Hp00^3$?l(64+bEYEjxMG&avAj+mQjTk!Le2yyKed0eU@q~+Z@1QD_Jc2f{SsxaU$Ap(l=V4pr zkU?4Wr!PvW@L`Sw8zJ7_Ybz6womVt?)N7#PT{qiIxl5Oka(66TP-O6u{ck%UPbv23 zObFy}Htsif%$$#BEurf|LgT+183akmSqoK2b zo$KX$xsv`cXmN*sWqoRTgJ|gp?7GAv?rW45!RzCW)lo?+!wZC9o>bXbUr(%DWuc)9 zRBY55Nsx-WBKYyPwgO;#tmc%{*ShHlV8stJa@A@?VF6yEbw(dS7#pJq8!vV@!$SO) z3G5RZfSUGTsC#?unB&ShP%z|U)UB_8Xg?Q%WG38GS1?S@q}5L2NlyBsMM&su5z%je zvX)uy2_dvP;LgZIJ(JgYstpUO+Z{G_Pd+%@G(aMDh}pPMF`g{)d%Rfh6bRoh1rRIA z=`U6gasqk8NucGR+)2rXc)HLbD&qt&*LR^dFOb`Y#k|61nB7t4AgjPs_(4I-ECja3 z+gA@b6nrm*1bI^q60il`Fda`fKJZ5)YWhBq#5%GTKoca~r2+xNpdf5c5Yl48HgS>i z#!OXM!PWD%eO{IEv4Fb*_vS4CIYa&OZ8y~N!&095ztV5$WVoIrTKm^pz`a&Z9{PB8dY z$pNROs3aRG{HxuBp>)I@-Mrbw+i1^(av)0OZFgI8Uc-nF7G@3%j!eyZkRG)>XNQY! z;S>?@-5e0MRV#+3AkL;qUXHrWPERl1zk1y<1WuiU(DmlvOMy2_O=TlCp1wi`0+Qmr z?>>0fK+{$PgHqu(GGG%ygRj@58^qyuanhZhe|sNnS*wyq(wDqm!S<$FKo)Fw4qIHE zIQNQWV`m1A)}$gfjfq3H*al+`{`CHA=%U7+=APfpxqmGjiKYDo?d!5=Y=q~ne8b*DX9DSMX#05D_9`Q(!j5QV2@c-{b9>^CqucK*!*!zlDGgI0 z1y<7(?yNGZbEpNboOmr9j`v&>wAAg9rAC;QL_UI5XT;8SU66-KX=|j)!PY(|J+_5Z z6j5RE6_;VZw3tvi=OWxP+b(^NM{#88Yyy+k0vH1~tfnj@H;zW)-Zw0-BRk()<#I(m zo*1|UK1J+X*sQmDwois$RwN+KMSQw8DJ(vl2ghP~9AX0Ly#Tr>*@RKGAk%~Tz(Mj6 zVKpd#qJ2H4{Wm^NR$V;P-dTYt%6==QlfrpOxZK_a&5t3E9=E7$Xsq>#s#i-o8&^^q z27NU}JKk~)Zm7oVHHeo7u!EstWOvmXIPoyq6X+FSi;bXk@#e`8QxMariu5Z&>?{^j z0T&boWwTpew;1vJ<(?#8zO+Ifq0@1$rZVUS98iSFK$+uq4#Uu(tX4&(UL;Zw3V9Wy zJh`AKKDxag>U+h<_q!o-$A{N@q7Brp#>3>`&<3cFungTOsSnEPLZV+(UykuU+X4sR zSDge)_tZ1xj)u1(kq7BfLyC14I$Kq7d@NapCOQGQV0JPhf$MuR<&cutw9hL@IQZ8B zpK>;@s5<5p(%gKj4}B)H0=)xna^b7JK5UlazUJOWxUN{bkmXb9YxT?5aTTPRUs*!4 zg+WC0SYf@Z0k8>cB6?WT(GG|2iq-sG$iY%t9t+!Kl8@{qA9(j|X!vAOl!h^ad3VoB z5>+WsdkRXR)A`?XqB-eBI4z@XLYoeAF$62_O?M#5=W4b4XyY6P|~LTzj{Z>69nKFZ0l zVDfPqkfziDoNJ{PB~@yPoYpcc*o246u%VUc?ojDgUJ^r0ryz2lAmqXBrh|2D%;wX< zvW5#*?Q3e9U!a75K3G|{E2Whxwm$jrZS#!-`r3L=QyO z3>`V>@Fy*fNfp{RPCPvJ@7t$GE}VV$$cc5*CC4Bp)hTiK;Zi{F-S6|$*G?ZE9ep`C zcH!HPZ*Lyl{lmlfACYriHtm*)MK>EKT(C0RVJ z&RwXRLBtqf*QVr_meJ?L0XEuxj!x=FS8mROP=O?cDw-T=fIy(q^#ZNfK1fLn_X+ir zB|Jko-#4i(7zKhfol9hZ*Wub6t`emv&2}XiH=-Qpn3A?sKPqB`?lcQ5qj-9+Ok!RV zk(Ou3-f)D#);Wn(gCg(83MeJTsAZ9DIQc1 zd+;nlE0J(YJbzAF^^Kq*T#iGiA2uZbxRAD}CRVLULcKOy2m|4O&A251RuFe+Wxv-1 zsuhgF)YYgNcdS8%Q#3GIsbYu-6o9~_BB)8d>;O0Fey9v`u(UuyhjxK&D%#5dZ#LQl z#9|>&x-dbbiMX`sP4aDSOBlvFolwhb#Ce^P4Avy!Z+!dXkiaFx2tcPae_1k@J;#Ud zxjcxpH1P{BjTl(?a{-Ng;vzHp^N)aZeh!C_SBFJ-Fj)X|9E1o{T$qAK1+5GsyPFPn zV9;=7b!ebI0cEZ?n@Cx`gtsOV^PIEXMDq;1>%eGkQ^F&vp__b)MLf34igocr1}~-= zvG(soFWS9RQ>`~#CIE#QP>%t&%5_u=dZL2fmAmZ@GnkN_f3$o_-#1g|#zG(zU7ps< z@IV3+7KpG10u+%`?Ib69xs=S~5LPcv%_;V?zF~7^jMKC=`Oxt6RrGE}AplSwBko4c zX~yjkbor^qRzQNOT>4}Lo!g4&0j)p@07A$-Ai@HCs0CGi0@gvSlEfbEVqbW3h!oT@ zM1bD6el(LjPJv5L2N~q%1$~_0I+{yb+pS%&5N+_dPC1P3)Vqc=tr4nA$UeYZP7O-J zeqtSlI5L7#%jk91Kv5wWBi;4SRGEeLPa;yc9)%AGFB?5D6Os!Fc#dhUQz+wO0?Vh{ z$ZdT6sY$Tqj9Fi9$FUjgUR2~#n-66lQbH3@D-bPY^Dn@20TRmMvmVw`=J(Z=+YRDQ zL)|r)Tq=<0PiSrv_hH;kMxQrIJqE+Ri`ba3_QctTv_lS_>Osi^g}@|Mv87ZrSnblB zlte)XxH^&$-f!at-%C=E!AFAO z0Kk785#HO!W}8PzX}ltelxkwOy`a0Ox9rWS^vAkNNFl}Xk(xoea6DKiE)pqB|(J< z2tjV)8biMN17np@94}C|_*i4@owZl`UvMRxukBc_YbkNST|^22A)$eB;)k`t#~>(ETQ6E!;uV>k zvEV#YNxC?DDE@a0R%CtTTjqUxsM-OoL8}X~)kKduusF77o^MQ+*hO76BZqLkCQkXB z!9wi9qtIUWLRA7UCEwL86I(@69Pq=iQ^rhqP%hK8<8{eaXsa0zFA4#R zMN6oRUrE7y!$J}(CG;4U6=}4=RcV=#j{rYjdh&Sk`r&~Mz=2o{;R|rJ+)hCtX3`MX zFz~?E{d$#ZtZg!=_@*AAE<--1_rf76BV~~uw_Tbl-b8s2iK8eFMXxEd363!Xd141I zBrS?-f*`G1MkK1D>ao|OK_cHL)V!2MRmFwJspiN7TvQYh1=DcSK~Hi>i_zVw+`0~w zj|fi>M4@Hq=rXwDxV$s;TN(%g;5_B(`XH8h?Y>Se3_CuWh2bj7O8BOgZ`@rD)I4MySm&1cyA3RhI=hvJbu6GYG$qS?{*EY=D|d1l3aJmW=oYM%iP30YSACn7 zz>_J`uT3F;QO_Dl=Ie~YA!*pZy2!xE&q265xd6Dd$p%}8Labwjv1mQ9kb=Yk!Npn@%HpK)hu@8?F_Kh={W?ylxkS55G zs3s-Om=Org4aCHCas`2*H+EPEWg@h4qXT zdgCUXIF1(0@~EmshukjvF$8PG=D=CXOAmY4@>(rbN%%*H)@4Re2+WZ?=nSaG5T;H8 zls=--#Vnx1DPjc*L&&PWHp^VI?ebbP-DcE`sQWwd$U)npxi>$jlLiQ_(t?Dc#2I7k zBAmp~o*?-)$h~U`Xkqx8TF)w-Az_X%mNl1x6C?$zOgqlV(HtJkMz^@IqQebA_XaO9 zD#AxKo_U0U^Gt0$xN0K3XElK6KE;`)X(%A$gHakRQsg*DC48|4e8lhwyctRLXf9qz zYW-#l9>5-3c~lU-dR7rZ zxW`N`T=Ha5*DPY1;>s$wo-U5K)W;Ju1H4n~4TbEkOStL6JzmRvSTn=Lai2nynB=yrkyKvP8i+ql&vb^0L9a^OEMILX7G=GX6K+x! zgjcCZ)2Pb;X7LFF5C~|L*7L_CGh4BF0E}=4afVT!GbljouNoC0DfIY2XGUQZI@PJ` zu0Y|~hIYT-Iwj^PRYDAS6AjrZYNawfbwZ<|e(1a5K5!s&-7*v>W)dS|_cK$O9Ass+ zJ@j}CRVcZpWmbqbY(97kW?&fBF`*CUBO~WLQzMm@>+88jkX0QNnELhvdafz=z4bE` z*wD`1@@$V+2-z#>GQqmIWIaA{y9>)8VV>9W;UV|6XatYN+txy^0s<8~-3>%j>+)>6 zPf8rgjNH*eD>zQ_8?vmFA$7iGS1vX(&`6uuL|EQFM!AvgKQ)*%aYB%IK^kl@p|azu zARtqVn5p6h)BsIQW7%pjK*Ql_qZJVW&QldM;+%go;DT~Ttnju(ODUli?QsA|T<^+& ztXL|EuObu^)R+Z)lm-HVjK+ zLv+NboYG87+UX+INgF6U{Sc)%gkT1^ag;V9hCb5aVF-CXABRBRgdC9=j!bB48kxqlD0_InJ{BqKbPB z8iWaFvCyx&imU0PX{HKM4}wo>#*!W~W*>P;Lekt4J5t9BP}XBm`GFLLuzj7?(c(Q1 zP_*3jm@p9h_%tgf^0vXpIzmwFcGE?i)!x>B8toO(fFTW68MN-ubfTrTxh(SK)VGmY z%S^q3TpuuT=I^~^SCC+};t?OHhM&>T$~T6KKQI~snV+`KIGV9(k!VartwW{+U)CgV zdBniCtx<7~h2lzKY(YzhKx>mO8V83r4Q8|4Lcu}{3((##M;X%!=>fx^+fJt0UiSmh z2v(z{)1|mK1IV;ozl{~g9yskskL>DnZedNxa**n!4!Nhs91{%q5hGNmGA2X2YhqUo zY^_L6Bzy_9MlRG$W7gg#2iR<30&a$(L>3=>J0;E)%W=(j7fx|QcWPiI6FZv-$H<=C zL(?(($g!Cy3yXr+)C<-xKr9|fp0X^~NRgX2b@woKEpInU)U7nWjcKohNOjh4wVoPK zEqs)zLt-AG4N)L6vrUDked?231S==t>x^(m9LYAk-t9&K2`+u`q*I;@PBbrgT1Y8w z*8B(y1p`+s4loE!;>FZQ_=>uku$7tTV^cy^YR#l^-e9ErYjXpfsFw>jS3&yR`y79C&fwd z2A-c~l)2)USp}q8yTZSb$SXx1x?rCc^)`?A(Y3bHpo=(i4jdZMO|J z6N}|^UE3rpB(8vUHnhPNM{QQ&Eckvr`kzD%T*V8G4)AAv4n;Jtg?iw^a#RT}#VN1= zXna3TX6U$NwRw6{ucYRz!BG1SW8SkMiqh5M?S=;eNicQI7+Ct_Iig1yFVRdypOG;|p~k#Fve1k>~Adfy%< zZdnMfs3MS|>^~Ko-Z!)Ef*+%vkh(eE7$~UC9kp#KUTsOSMQoBurj~a7QiY(;6>KU^ zfMBNcc@gJUl|Nz%T-!71fuxh9TfWgn_78)@G$k>4ij4W}NhbulI#9L_Gzw~k!xHt( zJYPuM$}^K%e6*4#%bGFupM~V;CVVF8t$M%-MOEb(O~ci|#|7G7zS$x>mwTu+Ffmh+#KkVdckFL5EW3re|i}=R|*Bk`&t6 zeb?z48Yd22TRE#CQ@%q&v<5a6-v<~!Wa&?q!_(BW0=Wbej+&)qR+=Rpn@StTwTNq5 zVGHfYwhwOihlV|53N6G16WS0=c#Nw3lp$8Yi>)n-)?8QP=3+w27)}>PbWt}V>mvv$ z7HC#ibZ6v3dY00mYCcy6K!rky$G%K1FipmnjaXReqfG;mTYbdL3jCTV9Ke2#<@W-1 z&&HxXhkn{-q8kaq7Av3Rix|A&MBbljkQfDwh~|!y9psM=;$0DbzaOItO!v^!fzR)OQmIU_)=dI2ZU`X z;!N0}^CE_xpNqNPVhXzYh)FOlbXRUSdp1^RZfIo$5P@SbB3-~nWq58*c92@$po*Ob zTLzz0X3u%TfnW%Lo^f|5eqcUQ!!C+8CB$av7zs*Ed^QuKt-P_L(kAEIowU2)i0M~qMl#y|V;dm1buv)n#eW!SwJMwagrUMV) zOKDqMRJU~K59EMH=UyDmmy&JmtR3vBUXgcMe*uPwg6bt*m?>94O8Z>GI_NnUr0ITc zXwOqW3wz4Jg*7i-XDh;Z4yOwtj+e`g9h@;VmK9Ql{al#?E`$0_re}hKpsQDguNOfj z5$m-banB@-ze|;JctG(HAf`4yz4q7On^bke*G}s6J&+3wQW`z$j&+|>C$~I5YX`O$ z6}~X~080n~dJKFOTm*$y#9C|0%*}{?aJG|AH2bls#@UeSQ>>Xnr456-k|8Icw+d8C zqJZ_Xo)b~R&JA1X_KSGl43U-jz)+ZW*`_u@dm1Szp7=W`h^JCz)wQ9ZSa7sxVE( zd?iEkc#P!z0fKe}E9hUmNFFok`#GjmTJ7tC_pX(tk-8f4WdODD=_B}=C{^^&xvBlc zPzi!3CzF*@ksi-X=A>lk3rb>mP$GF%ezj#Pl}4o3p2nS1;~>*x;4o@F0ep+M1_J@Y zjTwEFu+ps-AE}HfX01;G864lVqtLdGQrC*W;2{$Vcp(9F)i|D@8}Pt3BVIXy&U6}! zQf-f9JeWGAuxnpgrjM>fpz+Y(FJzE@YA3;5C~z$+c=1sXCy`s4?m@CR!O(lWh)zZW zy9D;m-Poz@+Ce_31b1WT%Ot(i0fZyCAcZj7E;i)#AWONWO+22+Za0{4O9|RbrP2P; zhc$(dln!R161V3lSScI%JU8h&5WR*M?~-^BEI>dF6~$os3A$tbshv!YH*hAmNQaL1H|2uO(0IT(2Y*1d|%3J`sCbfP*eqv*M!KSgp? ztU;oCu%!yZgzO6259P>vMH>$*{aNlWsQLaDwA+xFJLOC9q!fYJ8%Hmw6}4+A<>|7o z$#dcdx}qx?q8PBKHD@AtghpDNnQ^hPVDU4R1#^bhZCl$-3!}5y13EJ=xGBp8N7j|6 z>OrUXs)@*5;YvwG&r)RDKqm{yO99rwqGA!pK2bem942E?MPH)R%w%!s>mq!>?#(7{U@EViCMqaLRY?#wQDP(T zTzvmcNIlg`&wA`5*c`L~K|sF0kw&KKS$ygQ@$V{>#bJHhG)$$2ikR@l*&>x#JGtP5 z3kyOeE&=YuiTSU1jqC*nx~HE7yHeBL3%+r5kvCx9Tj^Z&qbil}EeXTsx^&NKAn@?`e|`2VAh)z3aqD^M+diq^n8dWG@FIDkbZ!_3MMiK)}5`ub&rf-}Y zR?6L2L6@GwYCA9`Qrno&_cs_zN^a_~4xDyY%Nx%8#_QE}7Ky$Ft7_>#9(P-udn=t) z3^aZPakIfJ$G(Sw4ZSuV7yF(v#F#WMeYqf@nb!v+SQYnLo8%CF z&}MZz(USA}Mykfirs{-pFcNSgExGV{i-wm3=7W*Kg&DhJ%A_SwsRDPKI)_(5N#x%; zS5ABsx>zJYO`g4XIp^x|g&vSquSobp0BOKdp{*Efk#NdmU*$P=We^;w#^~JBhUw7| zi}Iyb-mL=)V)Q=dHZVhgo>g~pLJN3IYu?XkG&pq>i3x}aEfJ)7RDct33d5rOA6?BV zhr9GYD&d`$S=&6*5Ts8V@FMO24a7=n3d2I8G|3>n*4BfFj6j5cVVRPdIJ-pI;1*euBgb4t z32JFo5CZoR>-GliEGTcMgH$_sUXY520BM_EJZEBLmLC4p_>}brM?&>Sbv2Ix218xr zL%J$hWF76(n#BkkKojx#xhqDEk>tRY6a^q13pT(pVIZl%*klS8B+n@cRl4)fk0ppr zQtBDjEQAFSD^d3))mr3o1jR%!$ON`yk<6_j1@86XKET1FWV;&tA%{yKJPH@6k` z2ToDkt)Y?HkSY+cr2x$N0EaWgsJ>nwLnTE5C5Y4(bsw{RCbB~`F>}GyR)V#mOLjE{ zF4b;G*ReO~McU|vI!&PKpe@%Y=6FOt5C-`mqrq!3tV9<>MC*7&7IRAg*5L36ZWPxV z-N3lJ${P)!A`S)=smX?!()N5fe6N+eG2vZB8ODBXhBMDOp> zsZ2V3WG5pcF#$w5oGf9%;ui=Q(wMxY=rBKoO$z)v33_I?`~#ej_GWwqi+`!ndb% zMcQsDi7|j+Hu*_GaG9_VLyi<5LCLyd$R(7!7TwpT64Ys9e-id-NT5cJ3?f&J6NwS(d4rK&bW^(mN>jR3JV zk{54^C38w~v|GelV!p+c(cf#g^0NCBM@)Pa0|p`>n9)Q6?_?c&*4F#wKcHB|%oMWWs#{&l9)JxMJPW-ees4a;n&?>}q_O5<%4|hy zWMD85+Z*(vG|I=Q_)i7{F>aDZYS^CdQ^u>6PpqImc3L}Pq_PZLR4kKo1RbIRG)Jle1M5rL~n{r#RLTh z>Jl95-k?x-_^Q&Dl@Ho*nlvgRyC@*O=_V6d%Y9cFTIX9jf!ot+m+|2gDMbtGQ`k`0 z3HCaR9jvtMg!q%7{xB9F3;iB_ZV^qcI1`t^BJ^Pn50gH#K`aPiyA7c$3@t@JAaaj4 zlHkn_!7RyrxLJWlFjagW_aOQ7A3M7YSKtd$N(a8&9 z2$HAe_^>b$$Y4vN7u-akkZ90smR;KhJ#@6cM2zx?HITr#+jQ1UwJeqr`iLTTf`K_a z6g}k`Frl*SIjMW(A@<6!kUoAQ{1MSv6+Tu%LE}ohBC5=!GEtVCX}k}dA*w(COMnR$ z)XmtQ1!~ziJs9BcdW(;-p?m~5`Pc3cngxR|+wCK$<^eIb0x=EH0BkbnIGXY6q|X;s z@?xSTs(j(EjdIQ&{4Sb3bKrX1x5oJ4=a0-GOSgV<`bDRi3WdS=`Oi3x>FL!$P?9+> zM1FT)4z=|Rl1-IW#QNbGZA5^F8LOPuVLCC8-5gWJBMh7++}JA)A=Q=yHyu-vDd}15 z@VPWww_u~`rp5aQ;Tu;xAF_a?Lf1Jxo{s90%K2rP5(f!o_wJfl$Q~^&k|T)1WwCkU zfrBP3A$*7!ggDc`4OGOA`^~AuDqFaMz*D%z(~lB>SaHwLYYtWN)ZbzSx3QJTDv6PS+Y(b)BAY{izSwc8X zWbV|pv5AU0l&WpLL^CFrHv!{l6EJ+z+*Ym8NArVHOM1L%W-1!tcQK_o8G$$K>kv_j zAs1n}61*nw063s$V-r`L`QLZbm*XyuiV+wtG6yqT^>8zBhn7vR)<0Tp>A{aF+wd<^ zVl6>-8gkA*m(d+QFj+t`9H!O!0xcQ=t3@(dGS|ZV@#_1cIqcA~`qH7wy#*2!8yhB% z1L8ZLv|Q*t)r*VcbVYqhl=?;|yL8dqUp$jE?Ed4bZB&}DSGh?Znz9Bv5YhRW29P(M zE@ZtmrA0b?J9Cyiz9C!dN4lXu%E?++iFq$aU8&%rXOwR^Zn( zIc1{5A_$q5ti{*khR#lY5(Pt?x)64`tKSz~R4@DSNq5VHi=ER;;EY0ejwsRRBtIWo%@_|WA9Wjw z!&$-j>)jT;fh`lhyowwOSAIC@S}S2LB<52PP!S68VDmoKNq{7(WtGoK863U33W2|Q zHP+^H5ZEl@)%J0&sV2M&YlE;VtXG`PqD8p!tQ%TYa##XW&QQdXL@1X7)=bXAC~+|D z_QsrvFon5|3h@2mcAyjFN9G`55KyH;+SMCf)>eIJL5Xh8+a3ltjsO7sxnT+?XqXvHzuHm#;^snlCBkN9R%#oNsDa@?m`xz2=V-n zBrmoJmSZMn=e>*W9I81#A3I4$9#;*D{76|VV9_f1i$L@1@-fdGX2>zfo56m3-3qF~q6Xq17_RQ;poh_`4{&%a|Zz47&54O)<4_frmY&(`E;hI9A$NKQV3++b^)l z&sv(|MBI5XyDPo0SX@gl6plv5eT~3~Q-@0ckOs7UM3x~36+X7Nb!cJOgy_*rD#VQ^ zI5cv#=yMZbGCca}1PR77+^jnyfqL0hmYOpcpC`4cRIQ{sXB^SJajd4TO1~_69UJZzNVhK1)^aN0~Uxx z^PuSr5vN(s*;>6w`>v90oQz0>KqU=v(87V4{8=8#s2Z}LrpvC6O9pB^(zCIyCD&2LkuMYj28Vs z$s=7@fkk}9$=Rp!(Yb;u7>1aQi>RK6LPi*c*MRy=rta?6IF85@R$yxAIl<#_U{NINvB?mNO`%jq-Ly z5#yEMagJ5%veI$s!<2}=3ktFpZ$d$pAsmWGd!s?(=*sr$l}PYOwMBrL0{K{Kx6L$R zqMOorBVCG-piy`NsZEN?Z$Okr4UO(}zGwFEMX#C)^)&3#AT6zg73fF0b9%C%RY%ZA z@i2n|f*WkY)C3Bo_Ms@g_o5Pvvqjn{1!PorZ2un4nk4d+SiMXpHhBmmLAApLbH2r$ zD5C77X}NByvGZWiZXRT!o`we{(xNlH=Y>MmqZyw?BDF|<$UJ#0l5t=`b&8`VAP^#< zV37J*JOkHfTd`Fj3U2dN9i* zfPk>!lcz!`Ayy1YZM%GgG@UNpAbB|Ox-NYkFkyEbXC}jpBkFC4XXcz?DP|s7JOp`# zS|gHBl8CZnW?E%N;Cm_P^&Htmkn@At$ci{t>zXXM;+#!n;GPZ)e+)j-}`gj5FS z)-~PM=bUUw0>F%dheHR(R^^JIfia$T>_LZB%=*RrL^nw2s}$nDQ>8-W& zt5VFA@amAhWadS)9U}$w7P19m zpka3015`LL#gR9wJE3BmY#pzG+QsS;!W)obW#0~n?|I%5094XI8>^|XvhtMq9{A?> z$kQT3PPKagT@m0?)vFFHPNlaaik!%j4{VZ^^Lf5Y`WO8G>%+`1(e+gg~J-p+m zI2}!2ybYuT+rWh^a)%Ec-MA#|*$Lo749<>Jy*?uOb;0{qsnw?$Va0J&I ztpkdbmPT~grgLbPq04KuG`m-+`PCuiYEcL<^+IZ9)nx!8D}-8s@RHV#uH%=Za3W;n2!TJC!?TPsnySqGwn|x8+{C zdKFR+Pdcm~9A|2^afyY4`S!Ymdx{jFJP%mLO;Q4pJg`BOl&{S8?BtJ(I+=`lPuqsJ zsOqnCx80>QmopvSpH(i0hFgJ@N0mBJu^uw#YWm04;p72)D64B9kfrU@5%(||@Y)iX z#Lx|+Ij)eiP#`Bfq~`uLtZq#d{R*zS80NiRw;(&`MgRl0#!X? zbY|fM_=j?Sy`XExOZFndQ8m6mdBKE}E&++t6{PBFK|R%PL;(SNDiI;24MvL&)M!a6 zftj31-MU;uanbBLn9~xH-shw4;I*7;DT_a6|-%wqzJjeeyvn`@b}z zlB>gqiR7T@HCb*FQ?~K`aeb|$s*BRqq^fK6l&CkTo#A;<~+AR#k>)RRhOGQ9wdONHc=tRPbKgXK!vri5D&1=CfSDS zoFF2ok8wtD#T}1_BRYDAp}p2ZH0FFMkZr{1K~S>Qw@|5NBHsa}PQLJO0r52MaXnn& zv4wYMw57@oCzN9aMMZxLPA?;9uG}}=@Q5OksBGH}!(fLAGCb@ZNRVu*u4JrF5hReiK!QT* zVDlxg{%)o**CBF!soM9s`G`KJf1}uaspzj@!P#d=q zf%wM2l{ST!DQM^uk=Fr_;u9Uo;buadK94mTB;}*Cat)s{#p6kFYN`)oltBD{E*p1~}#g;PPZ z8F5YMHa|7=*DA!(2Emc)TCZAdNik}6=COEdjw+Zo*+KJKB;a>VUX1rpWLPVQfwRX6 z5tm9*ZW`09%nyM^3D)%xxmr=~r&2H@_kiSeQDUaOt?d4qJIaWw$zDK@oR*vKOhXx)@DK>_e5UVBu zrE)Zp>yac>=%VR_ItL>8%d$4Q;Z;78h zdhfRrtz>6!#Mgz^C&m}HR#D=ShHAV?l~(U$LzDxBq$0Ed!7{ZPS)n|A9SOsh(CfZ; zJF0vxxj~e$GfLR@bcHJJiknqzQm|gvaK@^`rxm@GZOR9G9S*&CE`sgY3XwjKT^i}q z0Jw}oieYAORx2}(>aTpwL&$zO#vyoskvM?R@hytY>Ai|=$3lv9dVhjM9b(uKHy+QF_0`v5?3oS?fQ zj@RR{Nx^wYEL$)oJef_1kP$w)wHk>_7|mBz&}*)rDq{$RO&Z?M8bon48jS z8)6S2YCt)1-OjlwkZ(Hp>)W3xo?v%3B^uyeBc#BvGlCG7GAsP``#c$FOTh3Sd;9fQyDEE1dFt3dy5W;Z6^p=@C+}Y zL#(n^0W45*I(4XEn3cRfQ@n3mw9Bcsf>J*EQhnLn_6yH_9VkU)Z-)rYBRqVh`v+tb zOiPHUQ3SrdSk}Px=9bJ?V-@Yj(9&xGbEbsWq)`D|J7-&C%E+szMs|{ko!0zZ7)-3> z)43s8vexL8Og_Z5VId7$6ycJX&|Jj@aG>ODMU!zVO9Z|*hNcEz*UBStLK)*DkrWO* z7aw%n+o#_;W(<^*=v6o@7drU?`om%XZ zrBgZJN5Eo)_QmtKyu%(LfGtv$HHGSQyMKWD4H0$6%|Sr6egqU0mse>?jrc$-t2>?N z(5QB{^+t3Nnzskr;~ObGc`AjJPrP<80(BTSY^ng8A}i&1Ek=%p0Gq2Z!BBQ9Cg;k> zJ7K{XOYcqECC4iz)|E?Ih@YYK)WDO$%=&L8F*zHW4nqfZ<@K>w1YA>lCR|zqr_>2m zU@EdK{E>GZD6N6%zS}BGxIJ`S0j9L!c?SX{j9M{fRw1Fgb`@S>*1Yp<$c~+o%@7Wo z*#aAu%$x@=ZTWPg!UavYHIBW{?FMx^)yMLHE;FJ4p{2JDu&h;hoS8w~cw;A-Q1@CP z%w+S5JGLGLa~TXr8xta+EsdN#S!~m(!mMz%Ve9LODu$wuY=FciT{zNpj>cb#kvhQ4 zC4^`&aay1VAVE%WVADlwF>LN`uIj)DDzPCrS%=hXMtE~6)SQlx2D#{?GYdH5+|~~V z2O)NNIe5HR)YpO46~Y$AomY12W!#v}c>}?DN2#<+b~E0Yy@sWV2!aVBjZhpB0X;pX zq|z{=)nIy|NFd)oUTY_4Uo=2m5n4t10oltRDMwy-Jk8fb=*vUhQ0Bl;Uu_+kFWdt$ zgx^DOVh1nU{=6n0dX$qmxHi1ij-PO|7h{)bEXd#vp=a3X3k@-KZw=xk5Fz<_P~1Y3 zx{PQgt5XN#-MM(FGxWMAgoF%;9Apm!DGj0DVBMUy*{S>xB2*EGW@lwUWmjRt=B&F5 z5EuF8@*9tcP7KU%spAFQd{i)VdqoY#(Gfw@@&!~IP!zpdDNz*@>yzx?qPlmGbVn-}f@JV<~Tlt~3w~Uggy!XzgV~P^K()~b%p{c8?33opo7we>qI?i7*{NqCyMV*VK55oXWNjIjw17n zU`8C)bVKvML~blLqUlSqjOPue(ZooT>M5{F`y?H%%7GN12k&7tm(#A0psgKNHIHT5 z-%FujZm>FxM1599CLBS*WDT*`ch4CBtL~NxfiM**pJBA)-sIhqVexRYxu~aG30M)T z$ScTPUV7>}t;UB>yb5CNL1%Nwyl0{mqk#2+FQgEH5%;nY0IL16vWbk&f@@;*<1Yc2 zaE-EgBin3|cW~jlQWv&e24=Td3(e9-WgHz1mmsj9x=U_e&lL?Yn1 z?_EWGaD6g@@P(xb!0D_JTe_epz*sEZw9&Fnnc@dPvocACpkh7Ym zt(t`lA<)GofQVw1F4dS)TY-U zXt5@06ai|>kw)2dqaJK^9t<@^xh-QfOd!z^G6ey0&UKMu+L=-taB&A>?D-gE$R9S9 zEWIt0hRjt)O48yxWB}Mu&IALiR5L_e7Bgst9)|Ec%@FG99hUe8-WLT=4mE*cDCAUz zvyi7%2cr+sD~8cb1v5iK+X|lYknuwFP%0sj_@IEbK;E>ITqL>`bJ!s`QH|WMEK>@B z=36;qSd`u#)NINaz>C3#IFah4s*S^5>$MnVFyAT_VJkeU*aEmm=yipbq}`leI}kdxjkD-n}~-hV)HRf z3-Cx(=Ruz(l^P8gI4K^6t?#KQQ(Zfx9*y_7EGT}ph=&H*qwr7?v4=q@!!fdjVo?1H zhY3(cqgK>$TEugFdYf8yZ4o;!#Qw#KY8qXKFTjWv9uaiOGu!j1f|_U2nC5THafi-v z$dcfl61jk-Ec9y(lo0MpUCM;jtw@_|@zb}L(v^ULbu`?&Ixx&Bi-f8X@imeS_H$Ix0YE@_5bB#k&K?kF=~g3S_!JnU z4mhl!<;iCuT_BDIomJ1ptH*Wult~~+UPXA|_{H8D78$P^oRy3!$Rur4T!usie5MuV zv3e`myCCRJg#>G$rHcY}L((XdQ*AcBx;ajasq?P@L@2oJ&{~xe z8cUNn$+<#GqVRlnvt_G}h^0ud zR==Joe4!~^XGpw<2emB4EOMl(T1{Lqj-D9x;HqGDnT;=)8k>_cc{Z{i9O)$an2C5P zg`3Vc?{nbrgNe@IBp1hF&JzyF2~7@5-lBpnyP8uko#UjIRiUmRF>gOYi6PFgAUsXT zo~p5>@QFH(G!tXQ^75uvGBX4j7=#YmHa7Y`k%?2jj%F=h^7yHGNIaD`4|&sd#fISd zD$InzvXWSQ;N}{hkTxLZ`26mF1fo+YSQgE@T`?E{rN?eA`13=S>Ah7sj4@&sohDf{ zs!a5vGrEchdMJd>@7oM>OI~84$WN;S5>3TsPDl`u9>p*bZM{#sv~R{4z$~yy=x?F_%llv3r{@nV1 zS*{rv=@3-iB;qIO7KUNqdu~hfi%wuiM!Zc`>sXjP>O5KyEOm!^giI*fi6NDsj1q^n zt%E(UeF1c10mHXmM-Xveh=N~&lEgl75G{NG9ZGxd zTmxXd$$b|GRWX4>tTrWeieq@XG4mM~7rIN=q}1vI^-i4K%#32Mp**LKa=B?sG$-q; z6X;qhb=p$~CI~FBK&E@O_av4c?G4E8^YZV6ThTIWD4BW|iS)$SW> zqX_4!R67oq%PF8FP(J2!p~AZ80d4A3z4p-Q66Wc<#iRwZ3~b+2%lRZ>c^Bw9cmvVybhclmrA`p zco3|IO2H8om8wyQlC;aLR)=LtxFcYF+{!38b7M=)T3Kh8n`T5tLj~oWI`Bj$ z7$}sGb`^uK6?C1F$P!8@l(4KvfydRfP9P#RwyO5NIDzrw$i&8mVgenYG;El|fy_BM z%gU6g2$i3kW>x0C=Mj6z?_6t>@_m$b3zaF0H+$B`!#CAJ_~z5q8697H)(5`}4ayeA z^!z6^f#Knnu_f_)MFLq5f<}sDcp>jd9m{a}5^It%UAQXh0c(80I@_vOAg4yc6yjIU zV#I~OP{1Vyp|~;6+xjNOEGq5R0<|doJEwXS@Xa7FaO_ z*Rt@=u|n}0jGYY45w6RHo{(|t+Scn(YSgN+M!vdcD1go&Uf_cta6ne`j^jh2#KI;B zff{n1UIjT&fe2S-F)gMRAV7sE^&$yHnGeyuq6h{80S#WU;68t+YkJ(m;L`k0S(`MH zXPIi%pawI1j^zkFgu8el}M_*{&OG~&QRm=G_v zw{S@n60p2#59P?0OTRiT3D+cH==U;3GQoaAJ@qLre1j$G`68pnGHEr}kj^D2N0DY# z&3tG-4l9L00U;ez42Q1|YJpxd{!G!LDjx_rX@&IB4aZ`TNT&10y6-@4BXQj{<92V; z=ZjYsEC#$ooCXL&2FI~p@$`7eNK=i{JKeLwz6OCMW8#Vr2&35%(-u1lQcO$i$@%?i!g`Rj36Z?43}6H>Kw?Lh-`CYYwmYEvPQPgcXe>%>$O?oh zmPwdm4o-hgF-|KSh%pBOd&Ktux1ioz4c8bQpzXe*q_iZl`Tj5>8xsKJ?wg=}8usB8 z@xp@FPHAtz<1 z=W{~O%iC;$su15KwH{3ivywzbrp#Mk%Ea7NN@hOvyhEI$Lr+w0`PXQJjfpEB#%px% z2yBiNHuhTS{kMhmw<@xOaATD7H>{NWkgCSPqOMn$dp&@hr)=e1FcYFet1K4T^0K(} zpD!c=;rlc)>lmeELhSf!HT%a?b1@We6WC9&8WXy6X11$ae> zd8yJNrv~K+Ld!?MAT4IWPztN61;kMugcRa=I>A^@hkc_f3l0Kg5yG>>FG?8LEd$CH zcP@|WLd?S?aAg-UV_^4#yP8q=O+uB;g7AdKR*|x-dUbqrAqPkdwtUQqokiK_9f5b6 zqBY;@>>!rAvH~1p z_laQ)H;f&xdKxn|=o%CbMw+Q;dQu=AV$dMtpwOpl2-2~Z$Yh*ILadh8&H$EY2Q|T7 zA4Fm1WKZZc~9MdoO!7%@<1cCUE0tSw7lV&9L>ePT5jh z^jBosKSJED!sjN9gn`Ru!yegaNfHHpR!Hq=%10$_1#}jVAqtZfkK#SG`#ow4wx2LH z1ZYPFyz}cwas!HJrF_3TZUo*Wn_!a+)!8N%o(@@kEv?3)z7Xlr1Hai z6qqxVT^EE@z(^HS;zXm{qpAl2)*wP#r<4)_PTi^*ppZl=R5Jpjs6~BuxtKuU(2MYi zD7ZSRVOPf@`o98#&%slSGi9B4mh7IMlwYTC&|vA}o8OvFvxY$Ogf)`Qtr^~6wyV&@ z1;g+(TbsI?N2dh9$8!&OBtN~;QL%pxjDIOT6TKbU1L}tQQy&R2A1a4-S(Ml~ zJrL9nA&R46W)Z}URpipjh?qhKg_dv|RF;JZD+M$agk$)!twiyqZ3AhnA8@3U;2B&^ zSl8SP@>(U&s8t?ou8oy06JnPTM7%nf65;oXhr>#m=d6RiZW*iPOAG1F=3)nhNNTKt ziR>Zl6GWlkYuac|j#StSdSv*n(7r)=B4BnxP@`oCA{s%85m=Z$UbF*I^a5vFb4li3 zlP3kuy!!4tbAD9OHq_h?e=Rht3OXX@czL<6+qwhyw&Pzsj%?6^Y2QeQJ15{it%bI8F=Rtj^41hMVvBcvo`moPYY z0M0fzUK0z%W|D1*x>HQ$hLfF10SkmF&qA*{v?NV=FF8izL^?q!;ahwLHN)diH1#^) zliu{(tR+aK9Np8DU4~_}4Jom!YwVY%{cZ@pKNCw(L=tQ%rj4^^ClnvG8;JX1i-9P3 z%~A{?7hzZ@E}DdyGZjy_ScOLL>{ZYa!L7i5bCCU(N#IPG@?Gg? zDauYgaGrQ>6<3O(j5sEhL?x09vCE+Kh17Y8y27W)3w-x)q7KSufyP`OBKYmZN18ra zKJ|j){DGwO*NQ^7OESj>o+bsmKwF&mxk4*LZ%cTm^_os5q(B5*9P%61E}eUs?g2PId0l5gR25GU4Np zve4kS4n>qufC006M@n83op$D-tEr>0yNjrAojL)zjS!&AKq@WpP4r2|csqzbP&|9~H+CuD$}+de++N<1PqM4dN>yG=NBpNW@^k zEf&h3+LP7E^~QU#xMjlUXi!*~atQ5xvil2?<2a;*+Vd5PMjB+rt7Gv~u%Y_0I-{z? z>D9|<{LykuhdcovbncYI$7qa7IRj42{iagp%jNen$)dxSSR3quy0Z*q8yopk2}_K? zWRNF)4Q3MhO-TUn<>ckJv%#XmQm+E3%9)EBw%UNqNYi!ZN%wmyx=t4${pc>p zXVxXt1(i{`f=PU2kPlfFU$zp=; zvYt?2STMFVwqaWm1NdvseC&KlrFk!BNNgOeW+!N| z~5YDj!KsRTq(1fw!UP~EC&dzy7SzZL#u)3c-=fX4-RQs z-4F-`hS=4q(l9WH;wm62_>WMD<_0dj zx{GZvrz7LqEWd152otWp-!O+Mlm=27Vp(~e)15K)lf>BhIRO$wEy-r)MRsvfmD4j^ z4*RWB7rgNUOKD;*3Ijz30b5kOETAI91jU4x%QQmxagGE7TO-_<*|GuYQ%QKI=whl) zU$B||Y-T3Opz$D_rFQ|&xr8neq3@5IfQwwj)~2rwO8(GMdC5}AQIp15@n!E+1yV?x)Fmphdy#^>PBcz_64w3er zLQ1tcXheBlB6}(9-pJ5~l?Vo4>PjrAUO9s^s~^{j5o!mS@wLtRj>&;9T}KvP(ET=hQC_~? zW3>)uQSOxq*MXY#j$t>8Q4z0TB5ua2enpUJ>tYlNRuiOx!AE3KJT)|G7r0kp(w@&SwJ zMK(HvDPfm-tGkU$4Q?{4#yFjvD3Qbf5H~-h{=@s{%r^~9@qj((mw9svlD{(bQ9!#W z+F@{XPGHo_VLJ~1tWN+vb+;u^-n1xpaY92)h#S&DRRc(U_f|m z&lQINk@ltSgrh@vP^c%djf`t#XA~hWixd*(ypKA&$5U~KT8c@mQ|FPn=2(-NFxm}f zoxHIz*_Blz9`oDr@kLmlVyt(HHXf`n!bibe(eEI8opr3az)kUuBN9UtON~t0te-+N z$@tgf*?4Ml2{q-9jqR~lsm7}i0yvr`h_-LA>6C`O^-n9X6wsMiyq!RC2w8o2yghe# zc!Q4J6q!L=Qt~TrLU-+zTJZDBiYsb&tA#^e#1jPeIbpT_ES6^X2D}Nj7hu*lfL4O4 zoqRG3mydzW1yUvRh(1|^kJ2fj2(Tg3o7B8WKm!mwMQ*|`HjXs<>E3w7sa4sAV^27W z4s|EaC$NN+A~?uX+i=1oz3k)=Dlxl=Rmsqxi%}MLfaG2jub`G=G|OnnonMO*e4*?n zH$G$uXXM`k=^Tj~^4a^oS$mkGx9iP`iG5nqnv$a1z4=8g5iN=>y1Z3&XtA> zgkaWlQ+U?#Dz2m$g<=8el|r&~0{ex;{f$VzUhA~pcW5&aqui8xdU=4ST-NHZ(D>}z zme|FtV3<&u0sTp|xGCj@MZp}@LOKN5rbG(%DX>vzV&E$m0YId+vjFNiQ zpi6`wV5(xrQM9n%6_Y`Qa51H)dStQ%0t`W2vq@&pVGQUinO)l;;?-q@Z`x9Yn%*KB zCFo@rQ!D_ub*EiZCJ~biV9rrNReU7AFRap|+aqkRiv~@tos8(sp_NzErq~ZW3S5QZ zhatI}sG7B$e22hrCeTcryatk^E6f@Pg7j&m6EB3RoDRe9IONEMbyQs-!O_J>bRK}o zz!g!@4iL(ZR;E@R6p&^a*#dkfn8ikMCLUTO)kbvJB|Bv{uAEype3qVF8dBcl%&`aa z9n_F2y_q;N2FmTi>{Y=m2X-+-=V>w?L#lzX1s)G!nCRI30u?6c0We4$KmsmsZU=4b zm?jd}Iks@6vrKq!b;oWl);g3x63CJGtGk$Wig(Jr@Ed@Jkk%6T4q|i5Eq$meyZX$ueyUmleZ4-35*zspJR}`v9lT@sT?h?Sp+pi zNmuN}RLYtH0&KnwfP$gIfX0TKuAfr?d|6JTM^&=RyGs zUH3)cAn^ra)Q*#U9i7SDG{R445;TnMae)IUyqmbiv$s>rhFm%{LjhIKaRJhnA89bb z%Nklsj}hNJzQJjuzRU8wj*yclzfGt9Jf@ z%(P#<(XL_?iUUf5P~Mq(+#k-%dU#ya$jOc7YnZriO9>StP{=M4nWkp&>9U zB`nm+6Uo;-LSM1eS@g0kCGLW@UV0)agPn8ztleQLL6pzRRGYYmsznCkNpAQ?X`T6Yi92T)z7xT+yl!@F*w zJhRaH6jolQ5qB97qjyHGCwb?7bnGA?6A;Ywk#1Xs33a3=zRKuI<}wT_HE!5&JXNm+ zmLy_YUs{hZ+6xUQXqXEt&q&&*^@ncUdAmwxJkZfZtAb@I3wrYBH!+^1TRZNlIs(bm zeYW71-t?r4vqQ|KUIspf@w#y3m)HWvw<~!%Q%||ME<(o99LQ@}E5cTiccMsiOk`82 z*X_LFQTdQwN)2ZV+gd`xG>cYIEY|Vlh>`NB6UTIAdUZT}^yCue(XpQ+(AGgJ-9m?S zDMId{veSTM27h(Xh`qGYyMtMVo2B!Nmqv{AS z17YxhTU4M)+Qt_I8eFMITrj71u$LDUv~Le z&&v8@F_K&uhxHTiq8WlUAKmWdWAkbwVJj5~(VJdc1Ir!^4;`PwafN~bWw~^WHr`OR zn~303`{W{^BAspE^!W$D>r`8Yx5wc=6_4vjr8g@M&rv*eP5KwBDwR(NAo z#$QFeoQNLhvtpr_sm)*Q!exjy`%s%aayM zH7Fn&v-pG*LBZ$^(DupGaAIYfnQ(;^Dx4W~iqaVJ3H(|X&_M1Th%Oc&vk?rz)ZV<0 zEXYdr9bbi-i_^Z2qFKS5f!%RLA=FY$%X=unv6tG{;es`J%8Wi46Gn3d)Zm}&9SE9= zMh6%PP(2x;=T!zDjHn8-?ModbQk1z(#IC|MJA&6r2<3k9@vNF4QoHu0;Y_xf8v5Bo zhGsW`VEGH)5La(RohNq<>%KOz>3?*h*lGraVDost1B{tv*=oMK^Lv1=8Vm$nY$rx> zQTW%6lUYhsg3l#ExTXVlUer7uH4->ez`wXhPU)c~Ch4`0+LHfiQayq6%!bhEV@7$P z2kjUFyIMJXd`%V5zlt$0TN)DpR>0 z8yggGDV{~_TomlBt@8NvENMD|;hh=EF;YUhr&u?T8d&4Ec)p{hSEQF}mX~2su>*8N zRU#|lHVemQ1uR$Ikk1pp#HNbu&cVPlTrr}}ECV6=&v(Jgb&G)w z$r}O&H=JqGbVG_<@Fx^xq1I!=yz`GKux={toM3m0m4R%TCx$JsI{~4fkqG*F^=V!a z$aw0*O}!{8aBwN?HrkD*Ldslwb`x~3HkBJL<&RHs+gCXy)eyODj!+6iy2d|>8!$>? z&=r@ELGb9`Etk&^WMLp4E=0t!MqU~c(gqzqNd~2mx)AJ@-OErYPX~f})T!QhS8|?A z9Knf8g%1see5$n3y9Cu6m5gV)2AuGBoVFFoq+l$wEYRgnz(K3K7@SSeWC?p-OBV?% zRuLSV^D$bzSe{}Y%HY9#yb7>?lZ)&lDN%Xy+xOezZ;{#>1kiRysB{%L^!!0`!7hRE zQJFSBm1HS%q$gBto1#bRu08{armJsIfIybI>zp zkYsMx&dX#3WlwwTbJqNtzE`q5;dp6dWvXmQK+jyEi$%xe z-E?<}9|~UZ?`glCuFN_MKduszY0FIUtMB+fyUd7Elug4oCGF#clTF?|AKbsE(`_ z?eXgn>VQ*Gca~k0%g9cfBNBmw zJc=>X;KNCahbdUs6#DC~8s`q? zduYiW`GYPHQty|;ue9XzTgrKBAs$}l9e9|(6nno5I#Y|nwbgD;=_86dq9TtGZQk1;|g+_0M{W-s~`nZu7Tbj2qV5M z88DIE#3?L#sW4pe@+#jT;*S9OF;Z;7PNFRAo@hPsNtq~}3*40Kc^o2`>jub#W`&H^OcXE-~)SD`yv zQlctUo7d@jH{vx=utw7Z;&dRfR3P>rWZ1X3)AhW>TdrNz**(+KL6JI;7G46@FBd5L zhBzYpIBrwjSxZeaxa&m!!PJ~$zqWy?M+`q}b zRnM0pTBZP1PcHWzMU4c6!b_jLh<_ac_YwkK9hK0$sQ9KwwYAb7fyBP{cTI4M?cS#_0 z6#2#Xh+nMwn#5n;J#FWKN174D96gQK)E=FtO8l4U8Xi?Kw~W$OFh;y31(90v-HH}` z<{G=t(GDr?Q?1sV+in}KD5ithz{-a-lz3ew_@l$zHH|xF3%IG+gH$}beZz#B!vucA z)GKD&NYp?Upml|}-<9r?*d*YaY9EpJ%Rs8lItAc5ChyIz6qQaFCpT?Vcad2A1T-HF2b&{F6o_g%^CCBDwJAxcNs+jh z3ru>4;ZoT4<|2gGSobk9@hTwS8IGqWB@h>-DrY52rM}hi zC#)XG2rLTxd{VLPq%2zwBz3J}l5igLq(0v(MFv^!w%?g?=>9;{#i!e%mEx7_DD_13 zIc&(+L_s-_h}|F6N)gH0&VV67rB&m{G9#2HKxL$iC8j!PD3!D*6a@_2%Wo{DlV)_Y z!eEQ>?+zsdv2xK~E={E20eHS{n|=oH%MeRiz*O?Mruk7;+MvorGp1F%s~i=e5~dY? zwh1W=VMF+1O6%IUP@`DBPtKr_t42fq7H~_onx3`bI!;4Q{v^2FL_47@qVL-K1 zQ)!VVHGuiA5+*q}EY*qO%sV74nvh`JfXFt|#L*Ht8Kr@w4Hw5R^xgwx|Z5>vCCb%vjAk*32u9?i3=DvGl^Y_X+++cHJ z8DBc3uZHwA&2kkY)nQ5_u#+mJWM@ahDJ~IftnhZoX(Ry2#@bm|Qo*+45|)Y=!-#1a zieccU3=>Yw*@s}-136}?yv!(7(zVUjZqAyP2|sVl`phRTdo+=|etfqTKMmUSiVfqi zR*_GU0cTlx4Y9oS25|M!hcz!K%Et`Od33BeUr>fQYM}B6VSMmFCkB6A`kH{AL&s=D zL1Dq%*an=1i)I=lxMRB>L%unwC?y_j$B73-{#CiUsbGe$P)xOM3%6Dhs5epOrfs(N zHsVggiCQVSzco{;EeStLsPb6QG1p!@YO;iFRsNYo(wvoRde={YTo>GFj7tgbz7fts z8Fj7_>Xbylt0$MIW!-0Zn}n|(%bY7*z%U<5w!vRN1EkrK~cD@0X&*e@K$XnNAG zZR@j*(G}?xR3)&Y7v(OZI;|0$Z>LZoNb%{jf=MNxIVy|ZE5(#u@=z&cl11&`G@<5{ z$SVO9u%W$80m14nM2kdm^tTJr6;Bh*aI?o8#NC)WfK@uN+1t8o8~UN4NKZgWNJC5$ zKMVs`5b)`{R|stT8F<$07l#Z$Tjw}VV7f&*roASiAZrJ~!SlJ9;HVWThKx1JDKZTz zw(T581SBo$BH*u+rU3Jm$f$etuM;W*m9#9^}9f-Dmzn% z$J=t9v0Omn7r#`KTUHqZ?If)vSZaDf?keik<^bi1?9)5xuh1DG3PPtV0U$i|Po!PV z2AhSK6oA6Sh6EnkFHh=~2)Dr{sYiynnB3MXV=ITW%WqLKjoy=R{0=xEsIm^IvXw@r z5W98Syo;18S>-GkRkT>zenR^Z^3+qO%g?!f>aQf?Unpf9l;P^zEYXvU0U6nmjb9N3 z)QBgu5FJK{R6r`_hC(*d73%~rM9hw~=#bI)i#emry2&Ano>p61thEClzS$1cDlIIc z_^D4gaL`(y`SGAFA6{yI2NM)e7NeCKkTi{T4Ph9p#!DT7Ndd{*6;Gy#)?HSDY`{LF zxN9(!Q}UmwObBlXF8orKQQ^@%hZDmhPYQaR(z6$cGrj`n3I z9J&v@J)MR1aGCF)TuF9N7SYU9HzMED!Q!W+cKB_8_mg0O0EdAs)uoo1r+Y#7#9*TU zvC0OnzCAj@4#D+B7+Li&rLV4ceA8OcN%cg&4ym#;L*D5ymm5pw)g>UEu-Vj-TX zE(OMjO1ZnAgOAsH2JgOE_fDhioFDn49KBK7=Qc1>a z0w(iVu-Ik@nBZNy8KO8!M=(MQq09Liq$_EgR%oye-VcSQ;wTz1LGn+Y7At_*cAhz2 zbHcAgCbo>HvQ({;L)>G2H4rzG#4U_rdnp@ZoEoCwYG24!|vA@3m|rb07D4vwfdsLfI)yc{e(XXsnvI4T^q zCh}KMfR;f~^iPb`AY}#s?qO2HY~enL7iP#qyqSeMDlj`jZ-jeUW!KI!GDE}Lw?P7O6tQ@9gkd8~g?s9G`PD8&+p$u7ayx=&i&08; z?a2Y1uHhN47+`5ViH4{(KMQ-wiUi^*YHm2J6tp#5D{}^CDg!zuC=Ll=F?R>y<+D21 zN@0OCJ!T-7PUgPSM}b97sv@Z1=K+zZS}&3rtWv|*O0%G7J+1g3eeiq%_rL(hH6d}X z`$5fS0z6}ODpknuM^REfPL$8YI7q5bdk1A|?OvH~0SsaM>yMzOfxZ?yq!jg230nma z86dAM8Um*ys0Te;WiJiW+s|h#&k~DG%)#I{nZ4wFMaf6$lO_?`kYg&-3+q|SfB|!} zYywn!Qm&slK^7#Fx~xb!TIBZGGppZsudt2b*>I)M@#Om9 ziRNE{&pF`JG7(4MY8tI2L);VFRUQ^qF=84U>G^xqmaZnKHx_DlQPvYKlH#YemSdcf zL5RkbI>3hx0svz|^(hGJsRB6;G{UHgJt`8yfMHGpMqKq)>5!#nKyoyfSL#%`0mZpn z%jEIXq4pIyLa-#Y=}s5bB!ZCoz%R+1@(x+XY1#y*c{xL>739#>vu!sfh=|+bI+Ue_ zOIEKj33`mh%miFphExlkp9`vu(C)c(e1!K1#eL~kuoy@4_|@Dr1vz-~45SQEY#}lZ zFgLUYuF2K-bdIY)^MFBa=ga4qp)4q+O`9LJJiQz%Jk;#kDqOayy|N?JH?vwqiuMnTdt>JU~swVWAmK z%%Bl_f$0@LF-$D^i0mmNU@l%bhsY`{c!AEKMFmC~Tg;X`d(jx_p2SnbEhunJ8KRc7 z8)RYOEecW~s5B@UFj*PcnvD#jVt1$S5ex)bTcE<5XVu$Q*!J{oWSz&sdlMMx1*^Gm z?IIkq@u2vO7O>+I=#r153B?H#(9=bc%0=opZtDM@qA}L%8W3SfhLF3yJjKF?e>t_1zC$EP4|o8MvZ|X7Nv! zUnvk&XSUf;d@Y49h$ncshG$9gP$fOa|gZ#0kem`+KPkGS_Fz#^B&#FPaLTRJ)kzGdYVOA+iT|7 zeh#ji#kb;zH^vgE><|M1c<##Ns{9os8Ikj+gRGBJGz!~86n7u+ft}uK zYW{aY6>=reeMnU&r<22BAT?$o0>~A;d}bPsv-G@O+QMHwo2?RwjA`(WGT6y6`obtAJ0@Ri4O+8g_azSf zQ)o~-1sKcdi?n@%WXtMjHM1ENt{sOKOENkrPB*8km6&xY@zOm)stxT#^;T4F(`lDk zbdX}Uf<;O7D6t!5a;8IGvcpO19g4Mc3_f@?dM%R^s!6L1`qRF&l$@@^{Nop+o70FC zD(%DHv~AQc9zwzA?vm{?tAAXgxeAvw{9UNGbyBGwf%*+Clylj%9Fl}7)cOf~p7|G< zE%UD!ws_w(-*JTwP7R@pklz;FE7Q(P?28*A7b`p>u=Zm+5h47Uw1c;v3Q#M(2KdQW z)Tk&1w9uXutRr)!sFKwv0}!KwmW6ZMAd7lH7FMFL@xE|YQ9OjQJ!Q8nIcdH!4eiM? z1jf%ghhIK{1q7+`=_DD*0jYI3Ayg~_4hHxXaLA}G!i8mML5~qDr`L~YZ>{k7HH4D}ba;k@wq=DW zx>N{6Ro+aZqRyGS*&CXYehp5Aqm3*Xv_QLnJ|l;C>0)fZlFMln?b51vv*0;HeI;Si zfjL)@*rcc)TIpgok4lr`=?yzs@iDAfeUw}7k+u{GINZ-eZBravV^rL%A;J#4$lLoT zxkISFdRgwhF;|#W=AdXGVKN*jKLsVGXpSl=}hD~=YX)XLSXN?g0 zI*KtN=?5HVwW3Nk7{&D|gHch>LN*xHWF)_M2~1`b@g9;!z&vp0L-UrWUkcvXoCI0A zipPrfunts5CphwL)=tP6guC5#kcW{;9!$#dT**lmYsat)ox`h1md3$6e$bL3!n`}j z9hTYwIMmnyUJpY71#tek^g(fN2pYdclLoUTBx%DA7@RLCkuO>%p~29*>XcOWH8iYy zg%L_{wm$Y@i>lOw(RBAA^_^~?rm5r`eC9&|haD)I&W;A!8>gJHb3!ZwRusa7WNKC* zq6wBjq);HdQY%0*0tIS~I3AtG(ZUE+>dPpl?!6+m%91+ZNSOsoE*N&$R&DeaI3nE8 z`V__7N`yw!)=+8+|1JF>ZA)BB=6G0NAUKQnZxWjlgz?f@`a^vctEM@9VUp>Q*raMm z)IF>Kse4$ybOa%FdeecqB6=!GZhqecoQFn3O-c}J)3PcgAdkJQoz-cNnV6(QG8@fgHmf_U@b!SwYIU+M<`-E5|ga%q2if?uJaRpf$nCLQi{n%RRvZGu;i((z-KYrN6&4lwY@24bqt=Xc$BX)VP@q%q}3{$krj>wsiS+E*|3^l3T@<51fxUz?Y4ex4t6_x)b}5*bfiIrv=9n?a zS}7cr+<|q0!U<;!J+gxJfw*^^w*|TD!aAxGnV+9F26@ow>7Q2c|lq^>n zdaohQ3UYJgYEzUmqtd&A4` zx#^;1dm&idOCSjsvQ14QyVWKl)-qv(c9Y_E`|nmQ!k#PH>e!N8&|+9XfV2S^ZZb~^ z1_;F$fYcKBTfq2Mi?7~Xu?ob=p_lp4@77gER{^4V8KWWIEeV?>^7lF;*HuYq*jvqZ;La@19VL0m!DR0u~x4KRl8lWNR4Fr}B#fzMvY9#np- z+gpEEq4ZE~Umpb;<#MOxXzNfO;->XuTshuF88{!dJudod&-Y#Gz$ZwX6 zXo+gC>DpH+Ih6+wg@T&X>Z!_V0SOU+G)FU6xU_LzIJ*s1wWrk}hK?vGe_M5SxLOq@ zRJmtW6Dt6>N~-u+{^)Gy<>4F)Vsa|EGkzdMe+Hd{y#V$Jd0cXo5zJKHc4Gzn1!8xM zqKb}IIykuP44zbJ0WpF)Hmt8%MI=UWJ_Kx9kf(X5+7J+udPWjogc;Dri3PwyR0#7u zlX!C3T+^)T17bC<&LZgvQ+|Eds5#=WviSx^M4#+)AYM=?eqw%Eu5KL8K|n$&y|Z0l z5rP>i5b%LVZ?j#p_EZBk_SE7|GXjy~Nb`wj4zi>N3P_P7k|Sok|2I|E0L+INSDM@F zV_`660ILYQ3CU}3eO=@sx1cotQhGXBa7w)kOAIO~5btZ?+Ca-h@o|7N|N@gisjaN$4(N*#5HRM1xVoA3T>vrQUj{6|4h{Tr1h>R zS;te{Y?ORqXj3fLKs|+0ZQzfw2xRKE18`U_kFfaoaHtOqGbAM#*1-NT(RDLASv}=t zmUULGh)SAz=;hO^rtz5i+R%tEs}a!U=-x3#2jY*XESfLAF>^bye>o@u48)&PmL`Za zwxu*QXmWMauEuEEwN zGC^VLJ+|i{EW8Lf%G&wqiHpd-cPzvS-KjKl!=0+`>6T!ci0bXYR1ix}^g+=v1$dL;%87*( zvx%LneO>Bc7Y04ry4&lfQ!pz-`3pel5G=8phe4EV(-f9oQR1uWqO7y^=s?= z4d#!N!UhRLJjSdbCuY$5fDXN7_~dfE+ILY-HqEcQ#It19T6aFsyeVb!(`UY^=#G9c z*B>dD2}dc)GRc8US!vb^t3N58G9KA5$qJC^wSKGP$;VD$r-k)4)gwaVu2~8wh#f}x zA|OBwt%g_T<1<#BM4!3E)YlBkoR)-{~J42%q>aB7u3VI;VikOA3h8OEt8(~^H5V$?HE|PJv z`5&r>m{|=6Bt_8!NG!v$z^pr?E_#Yz-j7yebF{0q@`r&hw6h9?G&2kk6kZcM5<`UR zxq`a3@Xk!c8>G$+%J3z@ue_`17|L?c!+5zHUID{`@pMilLW=lfYG}ZTelbop;$y!- zh*CwMDrJ}g`6AlqFOnjDu@es~wXZ{qyPgo6fSz7i+}4v_Y=v(+3qhXV;suV*8usER za~aI?pfI-$>b7GF3@i=EDu@;#-VUPP>yq+Jt-H@7$sntIB_ROtx%pK1+FdN@nZspQ zvs{!M39iMB)u9+Ip~q2Awoj;g1sq16@{9H;C5)LM`fGdvOG=0$J(dhLs@L#>`wLr} zv}d77GAkUVHq=BW7%tjK+aOFkP?lbUWq@bHTVnpdTMC3&VC?8Fv4b^}?7Df1trO2P zH7liD!?CP!eBg(*nHiGO;{6Wo>*B`50r9}AdiDYrV=Y2Og%3CoHwTrBGBz(DD7ft# zi*b=yiMkz6)Z1R^CHYGk3*7fCRYmWGX&8#s#`U!+z^;%U*4Gc*iP}@L`1Bj>Qctc<~4Y_dh>yIv7Jn{4AkAIO>75_4bk6&(Hc=h@5&)XSUJPhK{-TH+F zurh?=e!Km4L9f^k-D(_Tym+Qw-y2Lekm{P1r8%RMw1gmdch)J?lu)+ErtYiTd>F>W zqTjc3-p_Tz35A~nkp!C<#GVXpAtH^vmcdc8S7lWxP*};89~VaRb%mYU%DPA*Q2|)% z>J3$7WDG{Mz-shyQIoDp@x9rfE+O5WCuA9)I1UvQZ+u1Ms9c~B67K39#RWmozfy+H zuHQmk_&gfAB1u+q91Rkj{HegLnlrFh2M??^R0CO|OUY6(LnaboZh@onG`xylP5Skcu-J=dJ_cz6DCGcEe_Yfkn0M)9za^Ha5y4S^@FU{IL^U+Q<1DrB~OSg z@i=fN68fJrA9_1}T?dmAzLa$^;Q)CIilL^C%d#wvrY_ zl_0te7lTV769{zf%HCe-C^@jD8ywAUC`2(m!NDmU5_he7zZ@SEI~0HFE+OD&`TJ{ph_kdl|-W6 z(hI9`bGMMGF_rkfaQANkZ$e@S5X^=7>y~q?%MB>e5DiEi)+8ZWxOj13< zqskr40>%~bTAF+ujc$tX7rc(w0_`(7<~7bQS>wkg4p?E(QV8^)wnn~$TcX}K3UVsp zF~e4susDw#2zf=~E+iOIUQ41?4AXIdLg52NK9YdYYi4x{$=St>QCt>zzoiB|ZLta9 zrjuTML-gQF=Ln7WoTo$;D$((QNlq#%z^b@6g{&d=0=feH!Equ7;`1yZEhDnJX5x-> z6g{fduTbEx*d&S$5`j~@G7QBT2VtD4=sp1~O`caR(V)gKxAwYe{ltccVcYcCBXM>v&1j0S4NK?&7OD8 zRs9Xg0qnU?psrAknno^ASXZGcN#*0gZUp=}D*BdMo07%N zYov_uf+BDjX$doE6P#TW8Q75~(tYFgFNbD!l0neSl*r^mrCtdHC_@;5mDo9kH2a3QK(iB+9`3S%uiA0(6Ji0W5N-R_%Z`SgcVh|c#;dgp&}s&gpTl#2Zc6wH&#rCYun^_BmB!c=_l1-jk`?OXi`m}H zTzwO#lLb~F9USLAHW32YPe{?s(M;m}9syzbYv%*ySUVq}SVFvH2YoUVwNw>ANM&e= zJXSuRKgpw`LA7X-3}hlBq87Etp^S89`Vz)1I?;K)y0)7MYvJsXM2P0sEU++5ikuJ z99q&>Qmx6D=>qVGS7QKCLn5sw!v2tVjA6ksHMD~Znt;J++9R8H_Z0LWgiaiyTWZ0% zM~O+!sSeTfl*eS-0f)m=pvY@p7rnH6${O~zJwoOlR& zdNlC?haS=hx+oK&G9wj9ZGo@HVwTW41Jz01u1wEVF(#e!Pc-GZpj)YI+FB5eLk$^T z0Wlqp>^BuKszJRWSzt$irU6tWj#=j^Loh;sIen*Q7N>@fgq7JET z(nm1QVxyz;s+f!S5^*1&qb12R!Y&KQW2JhQT%l2@0v80nc}{aA<4_SeZGbYG0|vV8 zlV$_hThrxfou@lxx)f1(ce(eodU#p3VL3c5c;uY|xmz#ZSg{guYlB?oS}^e&yH^nA za32V1Q;nFBQ#xox?e#X5Mcxg=7Z6Zxn#ylH=V=qErrQ?~K?f2bOLQKvF%KfyE|aBD zb>O*C5c@I$lqRm39-b|aiR2)etlY943f=KhZCgvx!$EPd`SLe*C^I`+gN^X-FKhyC zfxYK_*DU~|K0+M3?J%VZX=H>N;1zNs;vugZT_SAB&elj8iB&;iz3ThTL&(BQ>FY+7$n8k!R6wpc7EZHX{!}@T_WAlPX`di41JwY zR^hx>Q#oZ_I%iOHc*e2;j@)DQMbxCjxh7Pt3sg8GC!3i8@kY{Wnk7{j3f#a(vPFZS zrY$g$bPxh(Ggpu!~SveO#4iMuHs-ok4us=X%82 zv88#`lJk)1jMb|e_qYma(XJUOEP=u&slX%7t5U}iMV+^RW z3mq9$#}WKyU0DNpSN2yISdb`p#E^Quy;M%Ha`9`ig4%%SRPL%>10C0`u5KN}59m> zg-1mz%%{1m=+L4WQyN+Ms^J^a6QrA{2Q+YkDVkPYYBv!Wyr*K@MdV`CuK1q;@TgRH z4FQYGc*e7)m=z=9bG^0nCCfd=s%VFnKGudq5+65(SJVYtR~BW&q>pfJ*HnAz z`avhh@C8CdwE5Pk&=M|AJ_5SJfVCOH6UZXChs4)7=^xqHt{5e>X_Af-8!k>3V%!( zaI~^bsX?6KK8aNenAEO5Q9w{oEV&xYEUs@Pdz3&xoMOYJf*>2r0-?$U#tE;Ul9Sz3 zh#klvV7CGcM~kw|yAq6{Ff!ZA(%%{`=dwt$f)lqT4q`R5_+Z19#%Y{H1^M*N)rIZw z6$Yh{d!F1d28Sq#hC^1xb3BhV$V`6@G0fv(N*T1OpcSueo5G!g?og<(J{;MbkhrVr z2$06BXzKlvaX*C!?&xHKG8U%w3XIu3 z#eJ2RN_1fvqm9eSTmaFE)qq6es|Q-wExkeql(+1-z*PIzTflJs6ru6WQ*3i%^DbL4 z-He=zrZ$K#%|!$9xP=IITny#|SUh0mP;T2rlhI%fgg{y^8^-g2P*)ILfRTN8mN8^I zz_x`SOn9aTTS~P=mPR~F(2yldOy_Sd z0!q4C3Rj*U<9%*?@*jY+?t)F*-Ftvwa`=L$uE5DeT$T0P8MRAM4J3`92El-Q9v3zo zg04ExSJiOOn{4T|LOLfp9%MHg^ZjRlvsSOf8_OZyM;TZa_)e~5$0Ts5COzOVs4k@? z6X_?~&DCKglUBJ$!}bSdMwLYf#nFm@6~&AN7G-Wp!t$-ub$PH3!8th!?-HTH$&zyE zB~Qx0tl8sTVUF2;3`Sge;Bj`&(@94mioZnSVZCB81qOyYEvRCsIaq@r$1?9AHQy5i z#a!EQ8#Im_?X3k3EyQCiOGhRAPGHYnd9IpN-9iGU7Uv1WQ!BTQT96Z^lE^z+KtvtM zr>Vjvo387$L=6h{7IfpCZkQ!s35E?7?O@QZ*(X$=1S5y-^_M$>46FKWe4)1qCi9Y0 zMV&N^Wm@HjZJYs=*eI=1Nd@8SH3%iK7+SS&6b2Yh5)4c2tuvR6tqTn8MB{yN9D2ET z_5GK|;ZQ^%*85Rp1I5jeVHe`LcM#XZ5Fe%}yp+mlBr)C)JvYrhMS~99a!awhjr%HN zmBd_^$=xEiFQgGSvt?REs%bc8*@ZQf(Z2+OFfNZ3=7qNk<>aXQFeu~@*@$GOlu%$A z*IN6q3ci>2vIbwguOTu?_Vl;9|S40aa?4<2J1Yx zOm6(wia9VeIIOAEF1vNtYz?s;e+{M18X zK9(X>*)>RDU`sCHkz2Cxf={;68bQ_Vx-9dHV|aq9Tg#dbg4E)W%hw(YSd!F06_ zC0@Mf?5G#fYy%sxC>?M|%}200U`OG%?xUItcN76Pgu*^ga*;23Jz+2mRkSv}#vs-d z?tq>el#6)s=x1PXI{1JaAj(r&nfgL@{BlD=6=H>QItUNkEZUjOy_5lCcV( zH%k`#vW041&zz4i{X-52gRl?{?1Qp+ae-c_%Vc5|iZRNBK%T%YYvjQvBC3~n>Zes>b!=*CyTv8E~=x(mg0H9PC{c|Uq%VL@ppPKfx4Lj zf`lbo2M^-zYx6iWwk2F5j}=QhwvzA~FBH^qqgl1{0vWDNrY>#*RNbP+n}9<7dEV|yfiTPP8K<{X zI8+2gQ5#h1-ixDE+rUC5 zvQR2Q-b^ym8!6ees+$>`V9`C>sYjidNi*eagk?o_3O&5jhK;VS{ckiq6&_IjIxb!Y z%}3Q^^Pho~&I&2dd>L-4JQYPkn%#n$`gU{PWVYfjs637;%UHfvRW@ioV;cc9fb<6Y z*4A43#nH#t^NNp-D4uu`8tsP{MWIz!h)mGpo|`s)Q7jxZ&7{Q(cm`-gVo@2@Rg1S* zN=UFgdg-m0o^orIM*?ie7MunP##GH1H(RN=vMP$6^8|cdOqVI|O_Z$IG&KgcnIA^B zP==s92?#c#M6P%$ie`K2Ora=zu_(vh?_J2pn}alr44g4xt|G35*o?#nl8oJ~hR!%| zX{zcL_&8k`+AORZ?xZV)pab6C*~Fo-xHLp8WatYOs6tvxg^ z2`LlpvJOi`ZB%grZiV<%+)>^(UL`tsuBR>%kZt<9)e#@W5txQtzBtOypH$L(aWF4R*NnW3j~UU0d^t;rntm0i+$!z>8(vbq=L<5KJB`E(FIi=g@el}kkl z0Z~(_KB2kQPFl)m(~cQ4&7x(ybwIC@lv>P4%F24I9P7+j!&5dQz?|+%L|Ha81Mj4{ ztRPv3$j{@xC$K7hHbBG6qXK~7J6JTNYYh+~xGrsz-s|C;lO{I0xoT+VIRINgq`%=@ z5jv_>NDPgUPRRy#R}Agsf*}IXb_uMen|aeYR$0!ybpgCYVFhE9G3plNtt67ONuoI1 z#wX^aW6jf3@!c%L=g_7xR+pi`4F?BWd~VBHN@`tH3U?`sWP!WXi zY$fn$1}z@&N9GV3BjphUEj1fY4qedU#j-q0;CX(oAf0$l0RhG%UkImmw}C=k2w1+? ziR0&i<^;rd>q_#?V z#kd0Ys?%bv%(I?nfcJuP=r zwo`{yx4m;h_SO>jwu+$TdS|n3sV<$%w`^?8%dYLDy7y`9T=#YAPkR;bYbU#U)QSOn zJ=%A>v@}(%An9$&&2@#=MBeAGOJLQlR;<-QyE>KYRIO^d^yzlgU9vXHs8L?EQf%F~ zBTeq@ZSHW9?DpMTWmOky%a)}y#Zv3tm9duD+SsZtuFg+;b=}K%6K&SDg1v7;N2W#V ztvY)2)o!f2YTevQ2U_0X+-08K0vBHGdNkRWZQi-sZqG%#sZrB%XiqS2b-EtzZmdd5 z9J$%k+}@Z5-t0B1+X-`Rnnm2_6;eb%fJ?)0iZt6;jv)mrL-0Qc!&Mj`+);+AFD)!Fy-KjbN_qVETiA!yx zJ25WC(|fwNXcgAx?#*XdLMp4<-Jl+Zh3%ain0vb961A$K5$&Gwo}TRK*4mqER@>XT zX$re@wNkHMc7Z*cY)Wl!b5#j#&h+iwyGLyRD)pzWt=N0DUhUmcnQqv*b=7-m*{YqL z>s!4~ZL6SawkR62d3txN*Frlc4tI9DYT1<|+j`P>z2NW~#7DQUdrxKo3VUkzyQ&k` zmw9(RJ=@z9ik0or)$4j;z0Y_mc5iP-qur+3yqGFd9=wC8dhUCf*KX+@U>5D~ZoN52 zO=`Uz1fAJVi__EBUcED>y_v1+RK2#XL%r&*j`r_+jeFTe*`@5(vDTH(ZcRJ2v%8(w zX`a~DOD$f5JqEY9b9Q^p?AvOtnbSKqjJLNm)veY^rR&_I%7lxfQU%c7wd~8Svvs2| z_E#NL4OXc3YAS-KW)!3i?DW&4-JLpVtF77VsV8lFvEAM900000000000000000000 z0009s*V}#WU?C)LNz!mN&+1Gb>IlXdQ zYO{3p=Gl90_UpB1x>k*(+G-Qhjdf;wXFYYw&AQG8T6=rl=JTxXy&G>&Rc~j#dvUfN z-tFzG+Q)j@c<#@8yK~*%t=qeuyL&gW)bV>eYrW0w<}~c|xz`fCx_a`p*R#i3-kO!u zRqfrb-P(EYJvW&L9kDI%dwaX4ySuj;y{*07&3MzU_Q-jb?srk!b8S~yQ@P%~Et{e1 zsqMYDHaD2NY%^`h?YqrNqdRKs?$Y+_+_$Z@R9>^J_V&51-MT7|Z0V}jCEITAPZd<2 z)EjNrxZRhNcV%wg?)SR%#^mF!XPnoLd)}q$$GW?%TBO&yY`R+`wGBPp&F<^lWsPX) z-j91*UOhWww+~Nx^mliAXOX+NQl`ZBHf`Fy+$HPRcH7u+JZdUoBG*)?<3;<`=U)pKb$cHN6wY2No2z3BDkjlFiaY^}BLH?@y? z+Xr`dJv`$&_r2ZT&h_VZdY!!h9Io|uvvu0;?Du0@?6XO4Fg@cfv$omjyKi={Uav)a zr@K<6x2GOniECd^cG`M;-QK-tn|Aj)yE+}zque{0s?lg`PkY^Wy6xL`%5P?uIyvlh z-tE11?^~-*z3Ec3Yr5Xvt*O0Tx_jJrUY&c~)TrmNdscFm72^(E%_x2kmatGV8u!`GwTy7K2S z?RMO!z2mpJu9X~DihCd+ z(y5_>lByn)A*R$cPrf2AzWeI^{{NFdtyG8NNH+Zp|L}49`X*f9@AWVHvU<>;jfm3J zpWIb_q=a5HNd-oYICTc+H#4_DL*b|2QzF!d8{R_dEPCQ@_lVsB$6A%dSyap#;x{L- zU8+iE0G{*=hQ#9!Xy4Ynw?LdT;EEBj*gZb=Hs&2L@ar@od{3okl>zLjgC26U8?_@d$A&o+b&kHw~~8 z!Xea8v1=TavYm<{q&@eFoCzOgD{0syBwtUr!1?KmqdexZ4*@%T%E0XYuM1kQkF*Zv@D0$NB8lT(Gg0Ydg(#d%@>I;Cp84h>!s|g$ZRia3 zm}fΝZUvqS53P_662gTuz8Rv!4#DO+wm% zCs!XIcriwM5IFCNq3#(L21et--tAX!Xg^r;Y2PHLlGxY5uL03fxyC^EewIn%(T17z z@#YO|$V-}`#B(hYw1WzX&ADZo#GJL!;XFVv_BXZ}nnR`-rh<2Kv(T?<=dv|1(E+mQ zVYn!mJr~M9NxALh#!HH%=*ZK+78YSjz^aDGb3%N}1g-GLzLS8eLo)!L#)O$4E=*#z zO+r5l`rswgk!IOJwxv!t7T|zKi`b)VqF~`v2~vnyn07u;>Nu#10!sXzMYSr4OY`j` z6Sjn9O0Z;D2q^;wjyVGXbJB>ed?O?HKtw=9L_}x!{yz-vIXwU-l+axDqS z(%(z^8xR%`cNw3FoC)juIt6`)^-mG*auvi$q4RcF6%Z68j3_XO@R)^Gkl;J~k%AY9 zut#_kq&3+=9SPB+-xeI0v|5w-Oo5j|DiRqRnI}UGFJ@dWuXrr!@}=hc z+?np1DPpTZ7iBd2K#V~zr3X*02ceXic|yb#%hqIcdnvGEm&rpnY6Rj5=mCPYJkhim z_{E;hK>%2qKz5@?Ob8HiP$BZ7mi!Y!_Pw7&?08`2iGz6^0d<)`U9&Rcg6knIO^&Cl5 zpDqJTh^wW=s|fq4KQ-(k!lM^QP&76MfN&j8ruXOPRJ$X`ZJzyy0@Dw&QF1W@hf0IR z6B-ffOs>3PO!8h@81N`y*xlrL-^rw@PQ4bM3hf38a#V#P<>*2r6O?jQkXI{Tw=no- zX0aNd6i|u=fXUZW!<%yDV*c=5=ZAPQTbg=|D(EA!ML;Pyj!9@QuI{ZVv^{<`q1GpQ zIp=`tQA#mPymam_DL{pb@qMT>S%nFD5$X>NiU>EsP^@zmj30NImPW4#e;!q-3hH$8+9~k8oK@jB&IK6p?T+rG?UAFPbhO{ zvUgKArr5%@N#g52lz8paC=vuN<^WX=T5t>r5C zK%+v)UZIkHEt+H8PRYNsKVpgeYs%?za>ydad6-78;~Aj4SYTfC0NEBcl(S2t{Y}h# zrRNls0{aZUUf6+k87A}#15;W&STSUu5Fqu&;9|l+4~y4?)}Py@-8qKc}qN6Jg~6b>gZ`< zWxS1CUkckS0bNe?)z0(??yp5;>(bmf4MQ6)>xKYeg1ZN>l!=ra7^X;FgJZ9AHsE4b zt{Hgx*UF$H@YMGb)8I)R>XyVlZY zG!H2#UoZ%}r7urYUK&j>GI*F)Tsmq^V^j>sCUwfyb1T*rE$*vm2f!4xytgi9OfI1( z==O^gNVQ<}12IGTJ5D!?0D0>|qTDpTEi@HaHc>=5vcF_tVJ+qTAX*l61f-{F3rjmd zpxdMDePZDOB4*ag3Uaw3WZfKw*N?jk;20ttd}ajzu?^zwgX6;p~26zF-!WRb`{o>yN;`K!GMk)6MkCrs{a9;qoL%i)36YO`p-06Fm7bKW# zh2_bgOy82Hv4VNqCV`F!JM-Y_ZhNqS#g?9R4NcMz>UVlq#Diijc(ygFz=Y-osc`#^ zMr1N@u2;ONQNNvc)BsB-^pyey{HtLiqDFBLts|8gEVol62?4KKGji}ic1`YWB?Y8#+GuLGZ{i* zbtN#+No6Y}Ab#1gde0PEVngpjgXxyySrt=nPsfF8nI07VmfaJ8u?6sLQd5EDrnr(( zmSZT&Odq{L)#{U5Rx|kCWoYcK83;JEX$Hov;|5H?@S1z6nZD({RBGWeoEY!8PEI`` z&T>MH6-j)(27uQ+b;K@N1->(&`1puyT1X)<$~DPdX4#h0I@0v!Mr1n7pfL*JePI32 z3|aSzVjTB8fv9i{2sw2Xm35_Yf`0}!>Dyvq@0ea5gau0_$sYH5{foqzjRNExwxhp# zqAf?;q*F}d?wz^RHoEk$mdUi*hyd1!NcYj3L~H0vEV9FHqX72n$GcI|&hFV1?w9Hp zDPka&V6vJADeo$aO!6mJz zSbirf3xOt!kP49LPoDN;xcwbtR}e`6r5%hS;$3P%$|ium>&z9Z3!)!(+t#8GOIqkr zMLHs9w5z5k82O99Dj-U;$oAfS1txhg=pz9Ofknf1*3P30%N*kJH=hV=h83{J_j;RA zu>-0q4T93Uf{-!YO8k>0QE|GZ>{J?_|KbdhK|YSm+R7T02@RvMW;YPo)I!9O09EWv53OL z)*Dgb8QXCDWTSL;K~sCMoa-!ckGLP$5u0aC-9BFh1fv(wgonJ76TCo7o>z$HFS z>9r6->&k;t4ymB=AxQ3=9>D2X-v)+@*G+6&{D#{!(5p^?!PBL}T1YG2v$fDc;i%f_ zb#dby9WE7ZPW|TwYBf!&cQPD$pA`#y0+nr3`=LxR_GQS?EvZ?CfOpuAp_G|P^De!P z`3n=fvXw>rA!)<5+;{1u$#Z$*8FcL}fs^ z+B>F%EslF@M%rk2ZWhXnk_Di`Aty@ZM6p`>Y64|@-L3-}e4wz11?OlJDGOLOQ^u3H z#VH1z5vnF?Lbirac*kc@HVhYfOAzI{yY3xlMYAPfLu)m<}T6_;; z;BsW9)|7+7d;u;n>uPURXUTbI9%|nLB(NZj5C#t5OG9u> z&`p8pSF*uM(J`hy3{|@h#j^6@kX)`Li)-h|ER5Y;-e8qtuwpM(q1nEL)> zUP>fngL=#u5zON;T{{~Kd2w>8Ku*uHGms3~-uUS}*>&jgt|ko1ApM7d=WG z6KC5fjEFGdnM^9v38nX#6JKqyH0oKheEf?E5koO{OCjkAD3Q69g?>Av>+6#zIhYC2 zPAT^Y9n~XVc~>ZVBnFPtKIwdpI@rR3I#gr88+{$x=a){#$?+RlMSO-LW#;lxt(=vh zS(RP)Mf=W5?hsw9%BK>8M_-7_uMKgsMz2KV&TQSVi>>s!cC|A2$VT2CNYZa(5M7rc z_&}=WK&#G{Jay8dlEN7ub~|ajBtEe4+hD4Qf1&U<;pe;}@i+NC{>L*8*#YUjA!ben6Qw zI}ryVGNo$h_V^@n(A$}3Ao+=QPFV0Rt`CHBd3p(IPd49539AB@bZ^QcBTAtFsx$JW zMGW^GWJ&v*2d`$*U6sSOx`kF%<9gLK;3yDD(u63<7Z_{nV`!mYFeqG|q!0*4K`;)$ zJ(Uk9Od%YY>2ZK&;~SBUQ{-Oy=1c(tJ5;nmgp=OTcvPa%*Ri5hI5c7F!OKwaTv^;0 z=XV+G!A?5PdxiA`2vNMeQjR<|<(j8g%O7OROg)mPPy)u&u)1*>plxoJizNcfNqe2$ zl87_8-rnG^#eJBz@>tBs*ar5-84O^bK`Vi~LUBBX#CsxH{^?`2*Ry_h^IrY#6hsIr zLq*P}4$09EdxxMseul_F`>(bq96e)}Eb9wmI3lt!(n(JRV5-3ssJT0Z*vp&-{31enc83;7Ru0Py(+5$|LqYTF9;7R99%#W|ZLpA4hpZBF+ONj?X95er$ZPZc)nyQ6+2*K= z?@r)U6fllYOFc&Qsg@8h6^B=MY_}GLE>~eedhAYpm#$S;;|rMzEU~mA z@!7)WH(2v9uCuvw8j_5Se+X{5BDv((Ql@;+HrbK5@uhem;>F8McZky{=-85_eu$+U zP3dGU;`ZKXj7yK+T_}E~X){)-PNb`MBM^&uR4c7QW4JydRaXbCtw8aWgs(uXQ+qA|DaVoL$ zW_Iq@R!a{&ljPLtBnbC(d3OPWe1IK1H_qNoKZLC}4)^G{GYR7-RN3-mIJTBwARZT4 z{bvC-O)czma+Y8+Dk~h}qnjo$BMxMw0umWIAsHp0M251Spqy(vwJPINGqJ#ZSN>}Zh}PhGT)78yO1@YySsZ)Uqa=6Gw#KL(Bp6_^u(I+vtfn~~LbCM{Ab$6Bt8#LYK^igsSAa?@i6 zoI?3&;7p7qn4c(-jm8nSAat=h@V$#t#+(cdpn43zJV~CkcBOz&!2!2I-p+#uBHlro zA;q|8?qs^EFA)rd2blN47h}Q|(dB1d4|D2lWx%!h?P1E#k=Agilh~l}kvD|7&B7UI zWo=mL2YI_mDf9|b(4pOE@FN$Ax{;jnwOmPK;wNY)TfA#ELJq_wAsX0MxzWWchkmXk z^7*2U2zNrVqfN$R#!%;M5Sn4S;DelurR$Qk9wfb~0E&AXc~Q5E5Rxe+0PW{m6d~t; zclXk!o&Dz5@M@d$wFae=3gHqtmCN!x)yuZ5Etn$6Drb(CUPL)S0JXn9iif-pX~K~6 z_37`*$)QL>s9|9+4MCe{SCB%}Ox7;*dSsx*T36as1kDad09YGa9Dyemd)KMKPev$L zxzQU$4Oc`IPKGwRf+7Nf8Z1H5OPnG6TR^6PwZ{sUA~Q_f>H^z~Z37b%I}IlB09b%-!af z$h1Bn&ElvuG1CvPofpSNV9d{ApAwRU|r z9-PtDMDh-i(+Kr|bSjmpb|{$Ijb0o;P6E-Mor|;0#~YIZmpBlmJ_)gMp4xFYN-AkN z;Nf(oJP5_dwqz=c;k8CsP6IC(nNJ7bM(zWzC1`Z`mb+#}s6%BuwHv6#T$BDrsX)nhTJmq1i79Z5x^luWTt9zc@TX&KEpT}pkyc0 zYnGF|U0xUCYzz-Ygwkq3Ohnz_D(N0_bxW^yAxYvRrTLOsBW3EcEY7Iajs_Je8|u}% z?^=m`1fc2ljwJY!UQYuI4NjMO!HeD% zwpl()Ik?5GOI{%#YD*2_dS5Da^9AStnA+id? zpyJQIge%6>HrdIw$x;G#WX97X$S-yTy142?1dpV8OxY<#3G%+-r`1ga_KrzxLDE49lR2|QXHtasMQeS|(>+8sjlRKs z5yQN?-VA?YP-i&`kjr3(pPPh0STQbfW?)1|Ftg3Um+;!Rrs7!PXFLeC_Yb$Kwd51s z%`zqfcrkSsnbB6voOXbzv5ZF611c@WX;zRRAZ@HTBVxh=2YjR)0E5VL*i!CDj`;oq z=D_*ihkM-{s};pr2>sa`L$qI5$g0<}?vi~0FaS#nV8WEW#K#9N7(=(!K?ksoec~HX zrw6qL-Z%{#slTi@c}2IN=7C6MD1;AuXB00~RLx@dZvJvhvyTetuFywsQ6}*b(!yH6 zLREpnloIs6PDRpI`jM+Jg-ZNG-dmjllU#&$2NMlSySL{=!~zn?oCw7^2o@d#fV^%4 z8(i--9|$*+NKtmp5jNpE@KdqMij?h%uBHokE4GU8y>K<#gQ%g;Lig)@+F|(DqFJNp z!w{BIr@2N}0m8*TGJDu2Vm2=XG25mik{1npK~G-@~ATF`DktQx(g)+XKF(Li%4N+_!Se>? z3cfgBPbW6`+fkW{JqX!*?k7AVk0yf5H%1^E;Y(Ub!}CgMatP>$r=q(hn+hX}hN$PN zccJlQdDn+5vn<82e4W^8;Ts2J&o0ZIgnN$$y%IsOmgdpnl<0c0BtIFd z)B+Gti5B&^*{?5->pKd@OCy!$pG_NdBZp97kcPW z%&?{icVkZmHSKd30u>zs(Nb?t`_mrE-;WhZx-bWdr)ftCm_okt+`GtDqMOz;o^*j0 z-O?=%&~zC|<|o<5mzXuh zbEgzCX4yEmxlg=g)v<+-l}sm8r2gnQVH)WfdVFYEuqHu#4h7M8gQZhZdOHnKe|qp$ zdPWG{TR4evdr;kgivDkV_gv)4_u)HkXyR?x${Xn$!MmLm^QFwZ(4#+q;wtwi1{(8% z7BybuAH?DvWqaAq?c_R~1C$;I zvNfdR3ivH`Ddrg)XP0u&ACayq4$Q8WB)1q;kU``pTP1ukE|f)n;no* zTysubRwY|!mSOCG(h?Lg8`?3wu+o5DfLz^73wVv%V|z}I3`DU^Xsa-jvhjk9hMUvC zU&`!oG!**)r+L9jYuX*9$8y5^ECL}C_IP_N9?C%Bl=xH>N=32);RaR>-p%pNL$SvK z)I{erO03jWfK}GFR8}C!)Lm_~1)s$N6IRI6v55VsxP#1b;9qsbQ4MV@O$ZIzTF*`S zGXc&yQOn{$QGLEw3MuAA997SD+cBu%Jx?M?&hC1v54He0x1Qf3OKFW-H9WWGPxKGs?IZrzMjeY zDH%aCg11kSIxrWi*5_oP$Lh@s3>WG%5a6#pv$b5sFnBu^7k0-T?s0Q`=R*dM%sS>o zJyJeKavQQ9hYF$FcV=`B(}1g!xtA|{iD>TU#7@SVjh6Pdz&1ZfJhbfMl-EG#9hMRxD?*u@dQ}L^ippxflSw|O!FI`{a*MRW*`~Vd z6^UolRmHIZckiOeH8Y(?_S5!|L0i=^f{{+Yc}EsWd3{iJ!7y-+@Wh3KLxYhz2ZjnF zRFpD@8sbJT_cwOjX(F9HhoE0=qM0wW@(gdp(9)@_5)9i#Ka35Ql z(Ypbb7BS=zrlM{>oaLIN$x@G}F6p|H8u(E~CxjK$MC})^GCi4tDCk`%=O_#fUVw<~ zg54tUu+@&cO@&zmOME;GJu9gK4o*2|>_S#ZX>(jV-$1a+UL1S2%N<(O<~c>7;0y}k z-pHw(X6P2nQ%!317omiL*1W)t-i0|*F_D%bK)YoDN>#%tsN$W7nr1Iecw z3he6vb1Pon1tf-oV|=uj!pIH@^KInC!k>dkWgY0seB5DeM{U+rVdRdWPHTzup*1To zDDwj`H;rHwuq%^$hWRSBJ42W<;2qN!5t+M=SFQ%)3_P_p@+MLH$-K$SD0O| zr)LR6&aN!v$VXjufRE;C%bOfR9%mt4g$j(Kpt^p>`Xwh2%W1J;oq6`aQ zWW)rOkLggMbqYR7p*w%DBWqHo5%eQ++fMM*8brj}17?StsO@|T%LWLQ_;6Ou@ME%N z?ksC|I>an#7M-$N;a0n50r0~r$d#cw@e;|}@`8}`zB2pbVr6(*J_8326FVbuCmRJ2 zaO>_3#RUV_X4-saZMjiLE@}dHIY!Lar0rKnyGG6a?*3Sg*9rl zy1?oISdp8Mz_O(E3Ryt5-b7DZoe$K4I_V$ zmCJ);(=55sBFLK`AzTdOUtciMo0Ah21Iod7n4CamxlIA$J30$oO)b}OOqd7Ax8l20 zM#8|m9-4%D1P7;}%W6cGqt-3=wQg!q62qg6g9vmtS#oSlCLhEC`^lBcoc>jWx9BVJ zBi=1DoyO|pu%D~0ET7y#*AcW=mMrXh;LLpSnsHM>0Cw&zGkXvi#V3^lghDOKaaM{< zEy;rG#st#ig-ebXwPlY`4y=}wAsU>j5t2DhToWP666QJVuQm)Jn^n6$;x+l{}q@xIhjIR$cyBz%qdjWT_zEu;0(VkqsG1+&dDN%P+z~qz`E#h^|O%qez zr-uuocvsI%FZ4d4h;}rmOch7hkW4J&Y0=#M8jliv{amgB){eH{|D?A*n9s zs?l60#_c$iVI&rkR5}AZG{Q-mUgT@e_7fd%i*D>oJlZ9Ce2? zh$VF_u7a2947AjQj#&rF)bfUr^G>bPJtXuKAv_=@K2#ja|rde+(0M+>4K*2!~(ldac5VhlDk;tzdOB~QG= z>AsHNN4a5BoZ?j?>{4c8x;M}6Cr8PoWXsLb2a>QSnp(4=RfFKzSPKPxHy%U27M7j4 zT2`MT2D1=%TCjnM$Q7@E(`@$AixV${NDoQYg3UoG$yvn$%45_!&Fg-s(R-w`5F#Vn z6W?J&LrvSJT*F;%0F~?!jKxFTG?uq ztr@{?bRScKcUtGiY^eGI&1Uvwu9krZxiTR7XtS8&bxMBB z7t%ih%aTUxB z+b6_1&8p;=(%8VQS92=vU@>m0)mFqvdJGxM#X~uWey>8E-yftOu27^s96&(i%il?6 z0mA&0#dBvIRW8mX#Pjp#ASTQl9Az!oo1qR;%=8eUscVl-t`0z|iJ>S~VuTPL$?H^~ zTQOmoP6Y`Q$!2K}I+mf&D)x8XxKa~FTuna1?$~)F+0eY{h0rjTcFMuj?e=ToNECQd zNsh&DU+;JiQfpRPxgIGJ^6rifL}?DXy$pB-KJ!Nb2a$@f_?;~npMkpbuF)uiJ+_D9 z-jn3RVd!&l5wH|Rba{ogsaQ{J%=0cHUfA6^=|c3p76W%-pkn$J3e|O`jhx@rTB+!4 z-ckw)1~HhByX{p;QRYh?R%SG$WjyxGr_ai~=}CVDmPZ;Yg476Ky!QGYTq?sb60r|OiOBKYDDZ1F zDJf@p{YN930y8V)3^#rE$bgz$HKxcA-!` z=m*GajPrCzAP8}3u=V)qmar%~nB}CmDF?5FRgHaE6zxA&Jd;)|;GWGB?wB;26H}*t zO)qraM={8NhoiT5!$_^2hP^;3HOV$>Boa&O=LPG7(DNul_~(L^s2B`BN)Tmm?M)`J z40TX=e_hyk*A=$`f!)J3P{%Co)S*bQxpo{-F}vP7f;7zvwoGZ$wJ_x@*IZ$dBi5`? zpdGC6_ywa4K8NE5pE;*yG9p(@vW&!9gRpABAYYlVhONvbM+S)~6O>9nf~ZUDJk&0# zye)g)n#jSb-CEsSEVfK|Cz}G~B2Rlf{t>G>1MWjggc)Ro0?Ngrzi2&bBB2Z9MHt3;TFr&52rHyjWEC>EOPqq+9m)CG&d+AFZ)+0~OBEV&JDC zhom^lz*l$(D7TbrE{z1{R@7zcb!(_sn+O-vkQM+`CO{|#7i|g#&6VPFRZe3F5Ih_t zgUYV%_0o#J*X}AoOF)77ZabW-G@tKn1s9Z>si5p0CF67<=3kU@-5eS@9rM}BAa$uN zjW9_5Fuy$N;_SK{&hAm@C|S-oBW2_}-i}$&Pemplybi6UnrFw|X(@o%*mXx7vv`}a za1nA)(G3NsPm?2@P>W;|5W=Y~r9x4vHg@+E)`+8*0zVmu;+*KD+BJq)@34lei*S&< zAO;E4nn8>XIEL8i(0*$+z!Wk`y1EwgodWHZUB*-jCeznC+~NRR!*Jc60(eGTRF)ZR zP&pjdPq-0m9l^Wfe2g$#)UMQ7i0_RG)Lg}m^TaOZhI;maWGWX-fs0t-Z00gsb>(5R zdV^&{H@PsNk4YpH9}M~o82BFWo@njC(bRJ;QA|fY1~xxKV(d%E{AJBaDxrM}harcg zKx%5xipWhH-Em=0SacarN^fqWLIQgAX|lnovS~Kxpx(tD2&757hKLLvC0Qkz+X*Z@MNM4DO;x24Q&DEZp{$8>51i(Fq5Sb8ba;E3Ock z)m@T{txhRWAoD1Xfl|IuyE2rWMg~;CZ)N6}YARrz-M=a9N+5N8UCcxmy<2h&@dZKQ zH>LI2Xe}kJ0pv_J0}r&VUZ^L01`N1xYZh^y87YrXSyx30i+tUKtF;)x)lO_0%%1j7Di0S`$PgnQc+%kc z5ozAjS{WlxW%3IfB%|8p)PqHQc|RUza}QKZ5D^X~qH!)@9R=xOSU9N_rJKPYPsE_< zCd$oa3oQFlUDf6F9jWrDE4&6U_@q=iIZK-n&@zff;<#MpXb;XeY`Ro>gxy{{I9qZe zW#Y2G3vY*(82lpdd~c^Xv#^tT_UK3NWTw}5Dh^t)I`7UCki&&AaporDwCo39=@_Z& zQV(25M8>>i&z>-PH??ib^@2oBH>C05`&-l**Y2Zgcjfgbtc>xe?K91FHFrTwL(Hb= zwJ!Cezbq}+j>Tl`qWZnNQswHiDF^qS$;*KZl!};IxSKmBx7NU~oNEhuLyPdmEOzUx z_R7Jzv)s$C2e{-GNTtRBGG6DzH+H8N)>0#hSqpiy)VZswHRB8H-;nCuUOEB9enr9wz063&MO3l~t4IZa=T%PE z2VfU7EcIRjuc90c^N{QzG*CliD5g}OPPv=gR~Q}OGULG^V-Sx~>_0K1r6CseUY4ih znH8Gpk-n2psWvwudB$z6sJ67YtWJ|~$-9*uc%`{*#^maxH#=t79%@gx0*|i7-t?)R z&>YVWx24CHmq0}aXEu<2$3uD3I}Dsk%^uDW2IldVryA+gygExKQ?C;M>O#7=Kt;yo zmA)zjjy;D`tUW*|?zLM^$_R>AC-)|;FH3vI0|+(EqZ-bi53iGEFR#5*t|#lm@F`-8 zSt6yBt~R#%L9dP^G8DoU7TnI2&UN~vrEvFVB=OsTIP+}Sw!Qd6jJRaoXTvzqH=YY9{cf(7Rvv*_N%bvWTHnR)*lwlSt zkx+5BS)@u_=1X^c0Hvtx2Tr>{HD27-k<-wpijE#O!4D|NBtcbksF*@=;)YIp)2+4| zg&eKqJR}_1 z4GQ*FX;N18H9`4FC>okbQ?1E?(TdSzgmT&$$@2SDa|hkfhd7T7?6oix>OoXfc8o4Q z^+t&t=eyrrnskg1DV3jsrtDW%a^2QeStI6&Kt2mM%)^vMS-h1^aPEp{dx%m`Pg!CI z5?`L&_(J;J+}lUiS9jsVhCvD4<2K8~;)O4hycP2L(E$2?4nRThW@|GoXVZiQ5!I7H zcA9=$kFYJz%&Q?Vwhf_u71c)pN-CVfp@;>P8V^ZZK;1`1lXYc9X3<&V!d-Bcl)9i= z_N1oAg-#ZR#JMhR%F;Xrp6c;5A4N#nbSRx&wL{rLoWm0%FgJl1JkxQrC48r|nuM%< z6&s9vJeK#B--@KdHM(X~sZwh2G0wtzGaHA7Zv0tXlkD5@cMyHhP8dE2p#d#-Xn+%c zX!*f8Xth0?!623yYgbiy!<)H-B`0APayW1tK64Pvv6u!n_D17$DPUlb<+rgXmSA1ABSn2gpB&06p zk&42mW3N{z_Cs(@MEy+z+&M8Urj|jJzDa9yXRxM=EW#f#^RWgb-g$Reg1&qK#Leqo zDkP;O>V=FXDxAFFx+>&LiuFgRJg-~Ysx^xC!GccCEfZ-_!v1{%TyV9Pdd%s%CZ%6r zSq#+~p#@>Wn(=zblb5|hC%2q61WF;DHQKoo)eJ}!4z0bh=JTaNaDpa1sq- zx_w0s87?3?y;hK97p7A=407(bwp`YjiQU%Mz8 zO%HiG)i9R?tQX0nuDKl~69L<73!!6G4X+1BL*#njrl#Y0?M8z>qTIN{Ti>-%6`IVZ zIQ(;jj80~7v&_m{Pi+*46d_&#O!3ETB?W|I+q8oS3%8CNf(vRfXFc7RaUhenaz>fV zh-P6%J~?<+`MBFHHiScBQ5Kkq%bYsSq*|vpWstLF%!!J^gU^C2J|0ks%CBn8ZB5x7 zVp};CUD&4~yN+f&X?-kleLh6f=}w#UrEupMLG6o$3%-fku_E-I2Ay2abh zWv{v@gsKlzR$~!%vnowszGD}RH3~BjFGqJ^IFa8?@*rdp)YRllMmf?eddh;2c(%ah z(H^e_GM@bLl{A=#m$SFK&coK-9i8ZocSu1B=i~P<*!s(FYuzT3Clm?qTh85uNQ@S2 z?kq9_U8K1ZV4oS94@*$lD{n6=q5H>%cA5)^v+WWlcXWVJd>=Au^C~%#53`+OuQpbf zVE`)rhrvz(Co3+O%kD9XkdiQvi4sOamtT=Zc~ zgF~^5X2xbSAs)Bbn-edXAwY5yuWT@cYXsV$&2Y#8zTEOx52C`J z9S}s4GG||}qo|G{PIs-$rE~F6T-X zK2vf%U_T3wh+HwqLc~E{G){K%d!1SO@dq(*_3NUjgTBH z%|2)W=2C0axOtUE^rv@nd_d7c7_K8y<;~vf6uxwkmw;V>d7Pb*pmGKW7*8xPc1L0l z6IV~pOovE0ap;R@UAc5_iBw*XqP{#O81|gc4n)mpE#VE$m)6jkvuZ&@QbwrWHOa}3L%#yc|@aWDDVO{ zK6!33sg^axSyeD>$FZxCGa>sIF`ODp#@ZxF5VJnrD+uyx=+|C6`8%#}IuqCwoP^o2 zYOU!=el8`xa(7501sSz9Az~Wm1aWOm5IaSftlq4^;k%N0)_3@Wy_@E>?i@PMyipws zyw0RjWGp^@x(1ml4`KJoQ*z%?BOY4JY6d+nJky4w@dN;|?sTsn8_RKxWBvy6}3uwtBBh;ZLA=>m}zlrc<2O&%*ad zf!7TnkOIc>OyIaebTp*i4g)0ZnszNN4Y7o8btaL@4@&wl7Kb|*5&L2Oh>3T2J`_*1 zyxiA7I`S39siy#=)+;WRiX8H~UawC90Kl<{*H4gBAG8=C$m)b-9mU>smgEJPQ{|)B zixQrxevs76*w*ZtwvH>@k7m~%)t_MKoLf`hvn~P#SE~ww8^r+^9iwZWvml%9bF&#M z-?)nUA70nRy?s>fk4r^jeBCauTOs<_bZe#dyq!LevsD66&GQCzh5 zhklOLpq@ZR@6?Ts)C)#{N7iV)B_~~0x}C~6K5eYQUQtDniaGmKYI4Aq68qX+Zr&T+ zBzs`NVsU;yY|?W)aqa>LX=%MpMT?CSkQb3E`L8G85HOeHi9`0ws$k%I*+uZ9b+EHh zRoxw~m%1guAaG;E6+#8=d)g{MCd%J;PZ1Qn>udzzkAB8|B2y(tw52W5oN5VtIYSO) zTyh}HwR^K!mXgrIfhM43O4*l&ndoT}xW-_<4I7)MXY6$|gWW-Av|vJOXEs?);qi-S zjN>p*y=*a7Ez+|QohrG=8nVFcT2+CrSM+v!(wPf0d#OYn3sl-U+l4X!w zp|I+Sm>_nZ0wUdA)YTL=?sgZLsN8FEyRdDkHOkh#JTC>2qKO|{d8FTiN?4PIK-)Rp zkRAA+Jz>pP8!p|wJ3^#QLfn#5lCt=YBa-G&F>x;m8nI zIi?~Bicf#jK^A67K$5< zr+uvDnvr380^T!_AczF(Th`rVcoDEUfhzF0=dL$U`ueGc{lRt7rDeHhH;AyZ0s;z4 zAcf&;DyIc^))E zxa{jsteOP}cd#=rABwFjgfz(L0?#>qMX*0-y|7PZ%fgbkONVI`rF(4uQVTv_)R$Y} z+xq6;8ERB~Tv4c$6`VFMfjPB?HR?4w?nCEv&mGgtgSt@!Z4Wv*lFfd-`O&e$ed*0C z(a#R|T;jx$7{f1&2Jk_=Uet@S0=o-tRE@o|+i1jPe0k8$)kQ|oBXAV10!kX(NPaZG9V}tRF(lleZyawOZg%W|z#MeO3?wqj{lf2a z#OvF;49l!>I2OPw*RNcLH7q2cWD>C1}?m@Kp#8@A@fQaj^u<|Kwcq+Y5 z&T(M;EGBa_7+QGinf1(rm?|Z8D6k!fWR)0$f~X$|q>6!A9=_SBch z;8snqu=a>nFnuH_^?ayymIUpKtMK$SUDbJn>NOBowIgO~4OkbIb2e$<8n&h(+QrJR zl{R?%d$!@!jzV7YE;k4p;$}L78bVDSbeyq0xRzWIG@Kpp2t*V!khuo)bHaEwJPwI_DxU{RB#bJy`C9k%%{BNx)GG?Yj}oZmPTdB!+R zH$)n#L!9&=8r41>ajC~(83iX%wOpLVP_W1_C%=phH{$^* zKX=qpoU4GWruD*y-AUw(D&>|n#Ur^A=;1C>bzy1)GNC|JxT&MUg(`sIvF%ab&^*Wm zb}b@7^c2Z@0+vd--VU8CN*L#Q@0Bnn}C;F84S@QgQb{i#->@NEv#4CnQo zPdpCI=ga7yB@k{_qP5;)?pQ^o7>V4*VF1%-ZfW}GJ4;~tJrF+VZNhPUGW`J~iop)I z)*iRe^RdXQYB<#>0L*Pbcs3?1AX}8+CQv_G5Vl`KEsCd>X<97LKm`u`(MWEd}MYZR0gf9 zsK$W-TXCGkEq-YQ8V+9{R%7iemT!iSbT+aPPrB@c85mCJ8k>8rP&Av}glr{^qZu1h z!($8BxyuP%kI&RegD_du5x57TXw%K;Xka7~K-k`!3_IXK4W7@ip7Yv?q5>HKt%|@n zCr>OAmuZKwfHE;Qv=!cb_K+OUcXDM=fnp6Wo+hfPUgNa9UV4h}7TAs^?}LdsbvmBU z%$D6`F(RgEA~C|EvB<-TyltO(QFl9qo}l!IEWP!GxQ`f9^z?aE)E%?ltY5O1&;^_1 z7)5^IK`={-h4Kj8raW4J-MBvifssR}fbI8Yy2z!?wGUXeIC8=8)vqK@3RZ%l0{9g7 zDuZ5OFHf?d`a8VhoM)%GXy+%G;{etXWI-OF*fW%>(c|IS6^qc-OgeH6yuR7=Iq{?G zF&6~7-Z0zMZ9jx8mt89JBJ%@y$ZmKKcXJMBA9-I39PzYw?&%cawO~pM;~vL0qOQwZ z?teH9+efWZx@hcuNh7{93EV+gRWdjXYnp!q1!jE_%;%bCe4REySZZSlkKv6w^rvDkyH@MaqP8e6hM@K-u5D2+M#eg z7}ep*;u8gA##hs;+&IU*fl3x}9u-`?Y>;01FqA>zjC$CMscuy*sUVmbB>hhm*PD0sO``@A)vbHnxpLwIqja66dGV0x`Xpw zyuIm=Q%5r-9vu(P0x({!S}r?RVYgtZNm21{a5WgN=U9nm)Lg15Cr_kS*;r=0`YoKn z)D;<;UA*(i`*Sq#HFLNm>qjTBubadje z-wy)e)BDNGC?ILhduNe`dT{$q?1jJ( zfJ9`4b~7Oqk%M{@l*iWY#?xsDb15w|c6+U5u$abUcerIr$ziST+7<}-;$Z>Ige-f| zA$;k4Ck1d?iro^lx|e!MVX-8GFPWB?#40vI(t|*|a1ermP*Ew!vQtCqnUKxG9Z(&g z*dwHbG&_Ck#ddz`LO>QCo2lT^^@#&1aE4ZRa3|CYYY!Vk@<_l1av1KEZNER9NSh!xo(?+Bv}o;fp9`A!yR*+GfKmushh5x`xaf<7K~~>TgxHl zl!0{g8Wx;oXbv5&+L>Px)wK(UjKoGB{eU%W4FOys=u&M^m_GC;T)fMD-IOj*agW#3 z#z?G<+O$dN6so9W_1_0I7e1z^di)|4L4ib0JjY>QXw%B-yLT-IrZi|NA$vn!(iV^} zDtpBPyU8)$@}!+u9&m^xkK4+Pb-Z@bJq_LC0)_81#8eD5d6he4S)+yajhs6Mk}hR8 za@uJw5=Fqa1d0c8QGq!}D<4XHdfe!aQ3GNm+ca09yMs+M%pwJbXI4opK+PQ5zZqW1%wTdt#B>o7b7zGl6S0K_^lm zrnik4ggY19JICQ}gl(IeJ|R!bI@8QjJw@hdEpug+DKKLT zCz0exMI`<$Cq&l14b_9cXP{XCF6B>^gi43j5B2dmEgRHPFp9e) z<;|$Z43-N;V2#%%Ad6UqCCJjb95c17r39r#o{AW0F;9;R-h$$`o?@&f^RaKT^}s^R zAIh&cY7nI^{v7VWo?apPylohk@?P?;S|O2;m)DPs;buuUy(|tD8k4x)W_dLN*vDwV zf}~cN2@aHBmC|e(h_pS2I8^4nJ6&lM40&+%YW6R-sK?wROV(B9BEF6DaRe;H0-zTQ zD3_$sEZjr1Pr8!yAQN|-t$t@&mvl9hg;soNLGPNv6!LwQkK0D+2D*y+2fmRA8Fe%x zw$C*}qf;%`1DJE+5j9S^vjWLWLV=Fi`h1x)(})+q@6vxq%QL5>}gq;M({_-uQoNF zt3WTN8#GnKTnWmsq=C*P&>AzB9tkr12D6+mDlR(#o_tmx#XdN+V->BrhJo7nGSB|HpDqpDKU0@@BtpfEY+$K z9;^|9WGGXcUO9E|e57JG*D_Qo3kwJ`AIJ)Wh#>eR*saG0!Ac4sOg^$>d2W}+Yw?32 zzBE5kG1kZyd)|dLrso0<%b86*i;&xIBo8f#Mad|Ei0LL0tt+ljuaSN4c23jyG?lz!4<*ZH?lNti2%%Q&HExKF+O4tMZ_q4Vzh+U0#~?>neb?F zA;qyIbwJ(@DoRr?XC=#?s=Y{R*PTHy+@Ahk)KN-RKDDsvjY)+Adq&0i?ADPO6J3T! zVDdMB0Spc*g`FMM7LV)If+ zN!-8;vIML5cS z$pd$jZc<7-o>r?UWQ9JmSWD$ixq>M2K0WVCgb}tDX1BKp8@)%&k+Pv)e2b2f#JLgy zH!RsvmZ^e#FDq7^hd2@S%!xsHnzU$oCh*X72FYvJB%EM&Q@2!Tt7!=o+aP`xuCoz4 ze6KQlmss##7DU&iEA3iTM$8F?kmOIZ&$XO92bK}dAOCyX1 zGuVMP#lV|Eut5SVvV2$eAD63u_bhz@PfBMi70K(q3yAGD2K9}e0NE|2A;6XOzR9WO znp#D4jW#bcTwRrw$F6EZkc7(1l;c73YrR}(YGxi@Pkl}#V8dszu`fe#rx22PuLdoa zD-e_M`<@I#XWd?LA&W=}bf26W;UiA~yzR!uTmauwjOe|&*2akAytLFNvXE6cjuzZ; zhlYg6K0ivE;}Y`dq)&h zC>nQCEfOQ{mg;FUFFTO%&JU((g>b3loIqsGEoK*reu_t zmrhF%H6Hn?NPIrmvw3z)-6TC~9vT&{E1xx+_oT+exzCq1ZQBBV5&)H|PKl8Y7gIoTj!Z(6I!ItGDg#7tk>Eh>lA zyJ8sO#8^%*yWD(_!dhM{T5%R!EeYQxh!2kVZC=NA{B##NTWQ2c@>#>c+|M5x1AFT1K*d~2>%0WI_ctP z*o6!wn=Aw%H<;NRi{1?}E)z3q;HjBze5_@D>OzO5pKX!BnlF1J7HvY%Ol-C}W2+0+ zLByQua8(x9E2l{T%g~fo9%bEI!6m|*Su#oYiFhvg`<9_L-EO3e4#~8kSBO;78hl2I z5^l7l0bEiD6tEd+pdC?Q)IoD;MAj+b&CKkDhsgxX$5N&^@dz|YDA01LY2G`ct-r%s zUJLDb!>KeWV&d0h5=aEnuR7E}-IeMKe+EOZ7MUi|GeLW^?Pwdkx?^sNH@24_NJV;? zG>EW=#}@)X_#r3={4@pDR0c00kB!xM_rr~;R|49(fXGShh+28B2u^PzP&XLcuIWI5 zwg(|v;l?(xkW)Z6I_3<@_SDF84(`G0vi-+lcby8=EXm9TAYw=ET;O9U^_w^#mQ|D0 zo=@B^I}phN0#L__ja&#>Czb%T5oS3#hb+9CdA>60=(NuDff0|uX!rG;8y6PQaXXa+;mq5828n%zCLB@T#h38 zZ>@m52kCv$r*ayNKt!h|o2W5xyI$4AXXxWHj5^+M!k3xV!DOtaZtLmoBR4F{czC08 zC^pu?A86a6j?okPXJ=X^$-{)j99S*<`M-Mv$4O>!7cT8L?-#Ecbvdte39YV}4QelXzB9rEyL-w>x^N8>` z%3o)FAth$h11g-{^=i`YE+Uc6jG6Z1WWD!p7OQJWK6#?eJz9{C)GBDw5jnEoUux?1 zYQWES4d-qiTPn1t5lO8kF$V<6La)7s12}!f6is{AR(moCtH5Cg+huZA*11Ha?^jTi zA2uqvd^ycl%sZjk*cJw!)6F;!Toq2y+flMDA%)cm*}shr45VyPMO>AF>r5?%Q+S(% z`oMcSm&dx5cG{poF426bH0tc-@zRSs&~X_E7@6A10+5P}TS#I+9;+z0dh0DpfP%Nz z0Go0Z>M*asRk=si1TrV8TK=z!a$~7TCu6|!v-Sm8@7v@ z^a*L_!>_#^UOz}8k0=?W3Kq058e+~CPT|c5NKvvcd}5*0(G9vh4D~fM4s$J+yk%g% z0Z%wU@?(x;oPgR}k-GRA*HbPcrvELcl1u zH%u|LvB_k)U1a#D4!UZ1p^hYZ8B!fBJ0sCbQ(T%>uD;IZEOoGJg<0nz z1pNY?m2ILgLY7AWFhZjCELQ+MG=z9EqAf0wKYPX|WhJ6^eGwbnyqBcC%GYvz=ie!9 zhAI@;yeSW5BA+ZzA3F^9~hZyp3IjyBjp8iRSh&Qbc`cC%v3sa^qs9BG4&Sv zttw&+!cj&?g5J{V(Ht7K;?(djtv1?UDfFP#xwT=JH6JC-sp)xf0>E`T%fbSn=29Cp zLzo-sSaG1K>3Dc2E6|f%YX(LLy~0k5Wh#QvhKfx$Ro3i*I_7Tm z$pUze@L#208#w@>9%N+7QW0QP*VID;u*^HH)h44dk71u8+7;isJx<9kCZP+W7~yBz z9@e;(gsMSEdU_7T>E9_pKX7+Wxm;gkXLNW@tWeA~cwcG&iiXDEJSf)!bn8fM=#3lR z%Ah!W6~V5B_LvDaeEG~%!NoqIMf5QdHLtD3tD7B+3I)1C}vn>8)6@=0D|P@xQKWRSE-X6o@^<+MI7x-I{hnJN0>E1 z!8xufmx&^?n@|Ghc(E#yE5c0^J3<4l96pXPIqcr&T~P^6A~^8zyr>r31U?X=82dMJ zKAfUUD_QMsTQqO6F=l-27~cVyoaPJwom62k%^b(opq>#GD&c7oz~0v&aXewR>pMt7 z-dNDZzCsZJ+j2*G_fy3&U2Moi(u5mZX~B4LJsrW|HePY`WsBKsG$^RT7I8>k3`(V; z`JROt1iF0o8Gt1|seoAQxing=qgNpq!+HyYIkoQD6KbZhp}9w$VqBv4t)Et_oc9E; z9>Yrf^Hbd9JylFtUd|~Dhw5*_BF#zZ14c7W-QcBQh8R6BoR-xXQLF7|Rp~HSc<1YC zX{2nVuS~o3c6I55r-Z17)-s9D#T+BuoyWfRXNsFq6H(8fm2sydePZ!wHt=1x2t0e> z?(=vjg%UmKP@sG7o>c7^6P2n|aw9W^3SQkL75C@Tg9W>#qeiNS^?RweOfuMee9bUw z8k~6lbKPK!l0|L<`n!5{oQ~}lYs2w`%rz&V9%lH55AGQTRiQ9D5X*$t^Oa*_A*?aY z%dkM-y1QY}Zg?cd<(I`MF=1EDyJ3o%>hWYL5h=%Nr;{ECw@n&ZQ;fE_4(#zH6+vY0 zMN#-PrZREX!HIgi6KFSQ8GS!Gl;b{h7?Q?fxKYzr9=V zstR^ZxOdk|)9e8r+H zoVhly2D=M~fUjA`q+yRTU%S)Ch{wW)`Lf!?3%#U!BG#=OU7AUVg@Sx2#>+~7MqFko zJSwiS8e%sc5uaRO4{+0aAwX!L20Z3-qW=hV-iN)FMDlzvk(TN0b(jm+fn(SZb+xTU zA{wxPbrI?fd3CtFFi!O}(>=gYz`hq@Yi}QXMdNxV2{yUEJw}Q@$J7`4N6mIHDkNx0C5%e7jQ163h3y$z>9DWq$5hD-}XkQ85G8)hyE!=nf{_?CJ;mQpE4cC91Jz!5Y!xL`-71K5!DR!yMk^xf zen>Ars}O@8{a70^85MC@A$DVaxCxug29@mBn+6!=f{kfX6A=p9i%Sid;9=8aV^DSK zZfBwjY(?x#8Ks!{4D8RT*F0lra27|uL%&$zH1bUW0x+F;(pWGMINK)^l>>!s&Zeo< z%KAX9`yZW0vYfzZPON4^57vAK8l{C1hx0Vb+FmP!IxCzKS4tQq;3B zQWUnT@p@|>6 zfcAQ;YO(m6+kU!-D|=;0<9&hWV?8wyo@1uEVWW>*?=lgHM-*M&ATgP6J?AaJ6J5Y> z28ut=7@O!Zsd_c6n(ub>`7i`JbfUm{_gEvQ#jO>5QcOv=u~Eb|>8XI-BM%8kBJ8gS z>h&J>MV2)HsYK!#QT0`*xO3R3GY6Acn+FOF6ph&W)JEIN!N^S9JgzayHFO)`d&9PF zn1{h$^10FhOi`x`HnDgFgB5M7o27bAhKCeEF0?5u7@hf}HQ@%vSptG~bxf>yfiKco zH+Fq4BlQED;y4maJG^;Uw(;s+3^u|9Jgr#bZ`d-Hq~c9FIt!Op_88UnO8WJSlB>hR zVFBRA-3cI63gqND9cQw=sUf$EaexO9?~q~Z(st76hwH&7tX1S;wakUi^0;1-mt;g7 z2FBtuJL=bA>3fHTZMmCkC!}$V#atRhLyWf9o}smKz2vX2fP>ZM)=QHia9X%v zn8WSuW6{_ru+r~3bfkIuPzZ_DMzM-}>(&relWc`DMW?ki-O7PhjE>YWY06w^Z^N`*=+Ikd93z}6%|M1&b9%6MWn`9S3ftHm!htHx5cI87 zt;USwoR#H?Rc~QjTqG21t5oLJWcp||COR7G+zc>uv3sF7^#OdO1dtvEQ{C4*7j3Uv z*{em(_=H?AB)N-QUCYKF3o2CG#k@^KguKtAfTB%(A}d>NOtm;9dICzS%iS4HXp4=qKHrJd%p3x*NbwK zM8e{gep5mYr^|@5okpxwD@nnTkqa3edyCfbUV`A@t#UQPXS2+#Njstn8H8SY9kp=S zL*oG!$6uB_&fIvd!G|gdmQ(8Wm_3eQr#vWla9k=W9uk*|^OVnB=>kL+lv913K!0eMJ8++xFn);)`` zHroo&O}P(?14vc94MJuVI;1E&b$FIKNsDZGY0ozBI833`>y0*|QQ$gC#5_i6Nwku^ zyl@tx%@<%aRerEwSfy~pCpXI<8DI#DInB-L%7Xbks|{OsuZYP7EpYgw-4faSpaSvC#sPHw*0cvmyX&f8t>Sl3OSWyA@Fj##`^7FP~m0Ph!# zApHFOw1-u5#7iw~?})0_FyOcBt~QD#oQ77>Q^R4`Diu1*rR_(}bOk6^*MDw|WHzK` zPkYV5*cGG&3Z*eOdJz{IO1!~Rk6Ukw{~4*vF@V-8r-H$hD^)9|-4ox9^QGr3Q}SsUy<5B-UhxfrRuXXqXs50Wxabu~5uQwmUFsSXnVzM-5z9&W477K1uB+ zC2CzGBk{g9j)RqH^f)ElIqZAwXIOv)VC{e%VdGddErIxtCLw!KJNhKAH<`YHTlnA65jal=x{h%1BZt*brxvhBFr|*J_?G!yy2KvF=hXiSve|8&aAP{l$Ckj&iO;Zw`K_U` z-uTBGm}WZgN6D9@9-ed#N6d50SCq@wuS z+oTGNs&tI`d48@K^1EbTa1CFnZd#6H7h5!T`E1QMoRa=&2O1oB3}kX;~h9hPl{A@4V>MCuJ|<@ zmpy|znB^|Y4+JeWM8ep8qrugY9YABF5$OhSON~`;aZ=%2w9IL|Cl`Kdj@rII-A{>A zt!$Yak6(f!!u~b|mObO3;L(8eO`UcnI1LCcYcPYBTY;ogw*(g!J3#a%e{5o=n(M-KFl8!=WY8}+bF0^nB0qQvuhVpFz z1MBcvCAhvnJ@V_pm&xw9Wj4tQEK&(!A0n}ZahQ!Osb1a=hn4`3bbrF+z~s!%H9srTOjfLlu# z>9*ArU{gl*O2shWP@;IH?nHvOI0&{fM&YwF6Q%d*$Fdl?Jjyemad9Ed69RbiWszM) z2_*3I+iOY%0!cnKPK;?eYRbgR&mK1M*n)G(?i7uuS~WRrOVi19BN=gRv)FPStsjoN zlCiW|F1esr$7$C7TL>Snq|aYU0mVS#1I@G@DV0p31i(*LW0lH_#y~5XQ+SfAK`-SP zbz!W;KLx@3@?=+cmBL0e;gTLh*6{V&tyze+3lsCwK?vUB7Kz^d74Da#GDc@rpE-lD zcjf6CR~o>vdzqFfdS~d9e9R!SWWKWl%BR!Ka=n#q##t*hdWt_s)HLGv z?Kss4sUjFL#0G*^bDmwx5dtZWnwP!iad}+f8xSZjj=IB6Eluu4$&aClr<~@MEO3Oz zD;)T71P(|!b=D5|GpZ-TqZ<{OK8Z1<-z8)(U6WtLKX~5>${^WG;50(1Ssh_S;rkSw zV8hLBAVq0&Oya@2#{-UI^#W8pwhtMZo2a1 z&iJMMv%w0hmFE<)7R_f{#pRV??bIy^hhK1*^8^RB%zf5Kr2Wmsy|qj^#Ry1#)PQ4b zvtOhMn;(JZ!qtk0eraK2K3$~oauATk?_;Q)xZqSfQRI`{%ibL1OiR%w)@wMcyw9jk zb?Pi*=2Hp6p5s7CIK7j3H6sx-n^G_$H9M6GJ6P#v&41z|QIX)nxddj0ndpQ_`W%rMNb z#?e%$*xz_g(m3PSuvuAGgoVY^Hn~14AWTScsHcHo5l1dw#xZdSu#2c>!6>*PcQZ8Q zS>=P>@L6pd>f%tC3_`?CZSAFqWoHIh8wZ0`)?GXZ@ZG1Gev4Go(n_EQ6|!?tXIypH zuyyI-0)epPmbRcHt#h6e^}_7)PUD%l>Vo18^a}^F5*(sLjfcoiAvFQgqASHDyt`B= zoQ-gSUd^3feaT3~Gi&c`M%kWm&H6guj07^sL}-!6K6e!5HY?HE?!C=jHqd2Z082ret6o6#A55k5N9z3wjx z-66zTrU6fy=T`%usdRavew1@eSiHqPxl+thmRH4X@^fmD3i{4#FrZ%*;i}a{qIG13 z;qFdy0~Tzsi_{w8VvfR?gj(tzm9aE;g@??|h}%xxGT>LtRANS@#io8iLc?fm)!Xq(ird*wA$v7iqA|!N4)L(-8 z-ibp56$cIWmQxia*0h<0P=Nc>C1bpIq@o82Z3ZGzOu7^6TuVOlHbQ*-g*-vnalAq9 z_3Yp;raHQ@l7RiJF(*^iqMDxgz|x;!w-RNj!H=pqM~GTTg6Eu2vN8Rz~_q>TWW@8S7B=;w9T+_gT5ls^K6KxaTO@7Maz4+k;&k} z8B7&``rF#0<7$ztHBzcV=iH<%@|e&&hF}pUF^sQ-%J#c!P^Y- z_^chRJO{IS6DMc9U~b9A7{vHCWs{EB2vw>QsKm92#|ke}QevVN;>{`ToTeC3rsHau zR}PFTC@YE}sBa2(7TB$CEirY%#I>aEIzWcEM*6O!e>zJs_1c;*O_Hpaa?CFk;kZwz zDmP;_0!#@gd!fU%2s@NLE^IAjgXW?lB`6;(o^7i1Q!z0l^_3IpvfSA`F3Q4|5F`5b z`^(iPFc~N%8$ywORYePkRQR*3re5{gFqLocM=0oOUM^{Y$>*UF8;Sw2wGHPg>sw>j>AU ze0xbDCUfS5gY|Y$G>CM?-m;@Mc%JP0XM4M0eGvn3HWU-(pq%v(N&x~Rm~{dksc_%` z@6fz7)mZv1<)&D5lSn^3?5u1BKCzpc+GTP(dARFd$d57Ns0`c5w`LQ{(lv7a%c!o+v@*;S?9iaTLM-x473GfQn=b7EDARmJ zxIj>PgDu?ZECViXkRf>yS2~snAYsOj%?R}-O;SXKx~{z*mA7M93Nl35h`S--mN$8 zQ5TsjmU6;`bwiQtX@%k_aDEJTk#o9|XoNaTW=8G4KT$_;Z&u^G79&)Q&73GOg@_cn zqZNXa8u2}7`NLuQovCoU>aoJ5r6q92$nK|m|zhoL+&3uEua>DQRE@|E8{yi>5i0HEy#3(=#e7P_3dV*ISNaF+JSTF z+sIo;I27uNwuD9C;m%>4WMf&>1?$&v70y|pwjqH@NkFmlrmwEg#V#{&5|ioc(6k^@ zFLfruG&L#+%ayEPCBj!}^JMkVRQYiz#;arb$f+cSZ5yiS^?f_qLGU>d(n3)N+2+<+ zT%v@9K(fIK?N;IWN=Br7@9vj$a?p*N$sMQx7Ov~Oe78bCpu-sde50EGnc8b|dJR4f| zs2v0N(%uZI53DTfeO4e)avmN=R-}IvjbO#XN;MVJ2#Rvm0g-j!ljpdXk=d1b2|D_4 zK}RTEvqU3KR{>cRs0?DEbG+b~!d^rhm5xv&1``|P2^r!LlISTFB%#G#N5PgoAZAw9 z&uZToOgw)pQ9JPR+JCdgERxJ)Gp7O6$;rk=Wp%fSTIsX2Q|6K0W7IHy z6A0X`M;NdcU4j|l6db|71V&n$Fx1C-bWriV8Lh}nZug)QXX}E;VmwN4TRjX5%sNEM zHC_wN+(05b2wpSY(3ddq*_UTxkg&k_BHcK>MY6%fIRt>rn7Qj8V(GSuoqaeg#vcFdq`R$|0Z`^h7K9Ltl@exl=1S_alxIh!rF=xDh z${V)05-r&*;GR5IJK*5SFB4w?ix$xgNdSlw-)5eQi7-7=)Nb5dgjNINXTV!lw8}#q~s{2a3$Z+ z3s%xDeqmj3exgK`evKVRDm+sADvtr~y#z^mXiD9MGLNa#af-698R=yadAz02U%rW_|gepI0HK(hD-31lvO+}zDaUU67~YEYaLrXnN(&dE87`%y~fGg z_P&UDR&tes!wU4EG_JA{5vt9;MQc+YUQy`kI^DFW-wXj%0YK2rJzO6mc$h(Q8uDck zJW;SA)u&;&>$YTBj>YZthc-C@rt0>aU!CW$)|QI>r|a&tI)*`(%8xVYp-tpkdtNZY zJoJG&gd9I$(s2XY%#C zUFn8wH4P6#^3fMRuPt;!I3yket93VM$i=;P+u*72`C;sG+<@q&Ctpl%rdC$Q+C;Qtfqk|igg`^Db-}b5-9V;Gp63X1 z!8xlFR6|JijhQmyA45eb{ z7LyFu6y9{m5@tN$b*afSiW6x>EMUoFkji8sMCJwJqM1!GAUxK!C6V6#^tz(W56#26~}-+!$01)3**$iVoy#A)fcwr=r>jD-HT_J2ZgPb_vE&8-=Ge4J?cx+$$KLJw27G(EjuY`P=r9i?c0>D zo|<~AOjB^eQ?RBDWgW|#@EUr!4DiT!-)rs1WLZ;jVPKwOJ+O~7uz_uxCzLn4&8Lr^ zqEY@hG^FeFFRwO<-+JZl{ZSU&%qdp|k<4)P{k+A%H@MN(z&p*If;HC^hOD3>C`4_y z*vC~3x$Q(Fx}=)~_AdK1+U$v-5X@z~hE)DOoTBcb+pBLU9^A$N>}dez=t+)nfRRjd z-(j3Rqa`MV$7z{~^6*~76#JOCdDA2nJ=AEb=%GM8{XBeC1L#gR)>5wO4GDtw9YDSc z^MbtBmlT-;K2Nb};(6qRKscdYUKVTT96?2m=gg|_I#htc^y1s;{W)iM&d^?yLZR&M zUceu&^TL26^_~G{VLu=&JC5=7hC76%rc!1<8wJXAi1c-xOtfJ-9%rqZX;B_}F_QRd zA!s{v*K}}!D_Z#j+>vL+4zzK=TW?r2FhqG!D$WX4;o?KPlw9FKi?him7bk_87bKzj zD2f2uItbCW{ zvWqCXp%=b28As}TYD}n=CXuZ}bSU)2`x%uwue_n%eG+TMm(_T2rh*`cfYPR*L>e?( z5lzb{B`T{mCAe08VcM0<9IfntXl_9Ph?q|E8XX#~@LPv&Kq;WqA|*_`8577vj&X9U zWL;FT9C{H8Lx(X+_=+4Z>!r&I{m9~JAA89T$&~Qu9hAw61$-~;opV)Q`T7l^j9|U1 zd0J)+7e$-C((sn8t3#n@*Qbgf`cA4W#+G*tnV8j&Oj@ZK!E zwBF>XuGo91Bu+M!uyI+1dfym7aJNkI1I>!1>@9G&GrIM7S-R0HoM8=aN7O_!VVn?R z$ZKf~v%*8L!BA0q#$)$;$u2Z{Q-P4H5JnT%-Ey}08ZQ_~?xyCiq;v(Of(ZO24j4<+xiG#)QIe> z1{((IUpLa-Ssj88UL^`!oFBC4#%}VMcJiN!VrRL9#ufs|9^%au$Gq-|2u`soTW$?G zVP(adjN!AH?v)dzr~$Qg5~`QhnJ%{MhfC`V0gzP!^KAN@hN?~(1<%kPi&#l~YSw@( z+g0kE^AAZ4;LnnvK!~#V=1!@NOw(Ii8>(@)?X5u+<1>eF`E`ZaRk`u%M$bjohN6T6 zq^Yo}3sVwZLj1bYwmC^u!QZI$;@N_p*p++(7*2hRz`)tuH z{pUTw_a(mXtCI%vlUh4GonEZ?RWRKahtcjLD6p_+9t;f}if3cUNx!|}<6f;GY^q;E zq=se>DS>P|b6ugp=hRK*PBk~QEr)Q>OG;M(sivu42Rz9Jj>{*O1yQG|$W$9inFiCW zxb8Ay$X7_^_1HFI4|9#uocBC_W35|B+3H}+(#WNcRDwlVKW99q7h*Y^WdfbuEUGrK z;=+l(V6P7pQQRo)T0o$Y)y_FWpPx96^8xxWp;y}2z38*UyN@PBy#h0@EIkd8=v`7? z9{XkfUfD#BpGn&goFOc}B$HSzpLMM-OO{+6^Xuz_aKjZWkU=T%+pBtcFg&+#HWL|b z>MO1jw)V|%@p`ByRIW(YqfnK_d|8Klps|+mXg*#SQ0fQE47VU<0Lm^od35Wp(&YMPTP48y%Emq6hSbP_#4#RYFzXtV%#*f@SoEYmMV0{~ zvYuO#@$jxIXI09ku}WYps##W^lO1u^24X>CnK>-k8$6#0YeP#p3~H@m!fM&@W2R22 zl5}j|h)LHoXInk)%(r)qqP#XQmSZho&ecs8!i$2g(}46LXGs<$c1qqu8*W##@z<(U5Fw9>Y_OHH2uG&U<<2-oAI?Fq zypRMl-8RHUP=L*f!(iGGVzy?Zh&qIW;;FajirA7LQ9H2o44_Ulyl*0FAr;x%&gwlB zjU*cTy~*Z?Ub;X{MpRv&IL+QohWmKLjdOdRJqZl*&Bfl4WsR09iM~axfD>D*PmO$V zjvKBQb(EFpv-ixtqT2HVUR)X;J1s#*3!IEW14rE8iAhrNWK^#@EP;0IUF-5e_q+A^A!eROe7Xo+JS8lCWOptLG z2B1p8Ye${MoqQmtFkwq!>_+XI25uDYFf(hU$TE!uz?sFfCZ~=}Gg>4duQu)?J?A!f zfmo~o+(^EM6c~JpjbWiYN>#WZ5dgjoEvvB0XmwY{6~#BeP?72mQ0rS|AB07x2~gKk zjyP8oZGggn!I3+|=H)o;6ymX^J<3OUp*#bd2lFnAM$X(ZRy|I7%zUM)8)b5??*}O* zFmmg`8Kgq8dy3T=>_;oDEN2ppQu5qoaETU5x;?I}f}wzC6O3UK-&kbu8MLDMmt?bZ z3HV(+-0o0jnettvn0HKa`4Sm;*h>b z=C@USYuvZAE!eYNjn8nwFv)>p3PyU8IfxzQxsLYab7lL$X*R91wiS#OERyNv8#od$ z5V8!gV{P-5&VcE}=DOw@KthC(jT0b@fLMY9dy;WSe;Wy%lEDAonK6tqy+rENi_IhqW$5BIda0htx%eRv&kuwsVUHZXnPlP~3 zwxq)1OJCqNov5!7c(3+_*PvP>CSMaSiWj4Zu+oAqmCc9myKTxKy(bf3KSIbve<= zWgA7-;y1-JeVLT4mmx>Oo94`hI4`s&6`Fde2)U*2M&kN8GPHv07ul~+)Ps__m<7+9 z>wtpippNpAt3lAS*##*An@^87xo%CiOA|Y6ODwaJUQ&GvX7HkoZNrV*wYET({dhu)6l`5OxQSuY1h15y3K{s4Fd5 zYs!40Vg{YZ7`&QF>ii%p4gv{Ki{^7N=HR=vxyd(DV<2t_C!HAA25{n@HR3NrkF0_3 zGIJd(5=JXFQVJQGY*;)ts6lWC({T*SCO;c>d(f8(*#IaJ7s*_^8%5GIqqzVjV}Ly2 z>$TkaM({@<_>W>*b>5%1HS;~J9jsH*dUM{rr#Y+Em217|b9#&JiC+m`EP~_)mD`a>#Hl_J z2ng%FrczX=b2$5sZF0$8VpmSlt5|0edPzEJd51lHAYWdt1M*1dCRyXM1o==r%CqK% zEPo|$u$CO=FLqnGUvRDq&ji0xI+>d(a zQ`vhAA&K0q2%&*qF=hzsyC07RHV(?i+XYWuakFOE{m0|i)nqAzPC`6y@-zP(dBZ>k~G-f zKRkz-z_80MS9*q;*f0fERK?1w;`_8;lA-|C?|^}9oJHckmUkSHbTB~C#-@8xevTNdmH_xqia!7+!ffizSc!1^(@vs3KQsq=Cn|ys#1q9i%*thA|jyl zD*ek|PQJthA#*I3Lz_mxn{ip&-B6%4A~=)F?q@+MqV$Ah2aH%K&0ZKCgJpr3pNXmS2V{2_Hey(yf za1M}>(~+_Gi$LSiV>6no%P!{}{7FTho>~?~?WJ(it@czsugc1X$`?(G z<5TL@p#t-W@qxjP6HJH=xVZF#Y*hu{7=tn=4}qzAy$!jLFUyjVDp^W1bMjGik-`pK zh-B_rkujYzhiG-USSzN4jcE21_od-zLh+u;BYn3PKsoBVAk;}6 zMP4QxMoobt+hu6)iNxB?pz-plX?s9cr5cmeO?Y-*W!eX>(&{-4JPusPvUS)+Tm37<5t`Nrb>xXH4 zcDi_xxOzI;sLh3340!cEFxd??bhXESTkoJNoVD5>MsCHe0~1SlEn~bX9n)d9yb}XN z5uCX9A9UNffS%r+B7!t{1_bV<23;*m3DV0d@#a;5g@OWh$)V?N&12;S2Z}B2XOYIE z*i|~4LR$uwUefKbPf!us@*vyFbNO`u+IKmyNuWeY6<}=JV%36Du`jEIz{5Kz-9bbs zT}X}HMaaA%xcQ>*0{V3p`bPV&>Ox9OWVNbb%xIzNC6eHCCVCaELI%G{OSn4VS;6AuhZ0WWmE*P6V zDwA_5qLG9T35dZ_aP3MU`tub9U7vE0IONBsIS$F1M$VpRCuL@inmABY1h~?*hVU2+ zW14(B?iZ>$0B-9>p#ryW1~#+X7VjUCgv%c{t&*~9ai+<_hp{N9-pqS~Zr=iXomsPH z%w5yJ4wg1K+RWHJOCG2|kpQ`E@VH~0I8#1E0zgT8i%JBFxW_maoHZxtGHgy6@(^ul=IIRu)2dh( zJ`x;3dC!anW-P4laDJ32gO6@j!c6%&Fs^${NuZY4gIbWC!e+$4XqvlwqG)D0Ju>z1YYKQbErS_HFS5^SBYAzK&l~JUfH-NP#0XJ@n_JcbRD7!HJ%| zTDo397jhfgzU;Fa$Sjau?xg@R?<=fUqfBH8hT>;>?SQh z_4j>qBXAjY=T2_rX>jfOM+P3KmONw8zKqXDdE}Sl;DzK7m9G63b;qE)vOplx6hSW; z<^rW06`hTXKqz?^gs{NaEDWkTlms%J(`qxMQR1$$ID#= z;8bq>(}?}5?hM6>UOInnh=Cdrr>Pe1E`uH?L~+>d+3$j-Cl@Spzzx1`w*{zjAV5z8 zYM{COZf6N70=_EJrLiD?PjOuWMN|pxGZjq>fQ4R%rn;EX7ex4&wHdw-bgxZLgUr=3 zGm@@a2B>KztGcgF`Y;zlkmNqVUrGf-NLsfE&TEp*VMH8!`qN)X#wfSI@~oYw&1Db? zU?@3f7Z@a(u5lw0!Y2>~jqxQ|R^t{w0#x9n9pR;@Hd7w@mk+v$K>6nz~{mK{o8%0I;c{($AL5A|FvkTLvXSlJm`NpT{6G7 z2nsT(_^%%Wh{t!yG}!J3DCqjAW^H3{VWe!OmwD(`Al3>VloM-vDr9@AzNNx{{IMd`(?`5oZe64m|IEwCoydb67 zOx6cIZ2I$KA_~`&f;*R53x6LTN-YDhMw4-IY28%lx-0Jlk!cPZ1=Do`h%!#V4 zBSD4=sclQO)(b_p8p%tt!6MT-IZEc?>te>Fi>f6VC(_J0ZKv^t*MSK!J{j9DKexx; zu^Ah_zB-|wmJpvZn;1^7x7C%z!YE2JLcgCo3+6+sS+_z534S5Whs@h7l8alXGqrWy z4;CWp_H)Wg$lr2b%S1*6wv>ILY=R>lAbv`?M=SH8J7u~MX!`cZJm(=Wbp4LK0Vp=I z<$*vUFlb8RNFAqY`7v45hG|lXk9k%R(KPJTM0+Go_#m|E=Ng0vmMUb{=8(3M79{d2 zF`{hU*b04|99#bSv zvSmadcK9pXA+A)SB*u0*4rg$Ru#8S{6tx*qP&%`SLkL$Hw5R zt<1QJQNCYqCFnx5__eQu)O{sNr9L%GBY@n0Sae($#8>nMqR;AY+$*iK5O^z!>!T}9 z(T84wi_H|$KEfDngnKJSY!%@LyY`tn(8x#;G}&3wR77wzhZYONcT3TWu^>(eKw%C!&@wrR&k*URY2BqH&BwkWQ3!~D6_6_t>t-bpUiGafpDcS3$39C!bY2f!dAzYwg&jadQwJPZX;nbl3g=RO$t%t1ZS>T#8%}g^u^u?0hS3%YFgrjut zWR@YL$zPNcHsac;j<(4|A2>V?ipcXG*YVt`M<1Jx>=sxp^oU+5yJ8gJ4~1V9yiiQj z?hCQP16Z#YNWc-c(|V9KYi zOfS{0ljRcafQ&pu1f&U=oRYe&KN>T__$047o0NVtCSnoDvQM|a~o_;+sq5&}J zZ!F@no77jrFym!)ksR2aP3CkISEkv*cLwE~&d!caC8I4v9omGN)n`vMz65KE;wTCc z*)s~m+P=6dArvOUKHBaf4d^Xvg=O|9hq)kYSUm4+unS?f)T1-v=wS1`pQQ^~7kU<{ z?v8FSDewm2iVC#V1=|H(q+=PlI#RgHprn@AO+&Aw5#bCO#vcrPJgq*t1hJ{?8*FHu z%X=6fp8-02h(3o=8X!cvGCo(z1MU{t`kSy-Z)rM7nq%QX@yIhQo4y=)u)G&t<{mlZ zAutex#12HCxae)`*ijKuxZ0hkfLUujEP;MVLmM*9@frLTvCDo4RsoZAyllz_?#US~ zuD-g7W~cE;4OgvclVMD#5_Y~mR8)rww+e8xkJT(j?F#bDc{9W#p6583>s|8k{^DGe zrmqFe3TqiRO<*W>dxZi=t*9q1FyEP{(-xMP{_I_p32Ker&eah2zIKemNcgqfpaRWM7jD}_z46n<3+ciX1L8_6C zQ7js`?Bdpsb&=SYpp7a=q=wlt-nlsR0fflY46qZ}AN%4U}aoUCa z^e*r&vw%qIBkkxU-dMwADM%{RWxD25h71BlysY$uIn=DUgZq`5HI^5Af^_O=2+wUX zISN^}h|S|y&Ujvsj*6aOXJ-g4D3uy5>q!dQ$Q%MnQnp!8KIrMdx+0s>uoD+h!8(^j zYqTL2#!m?Vh=J53v8StCja?C>ZsEhr9G9vTviIrRkkRKjJ+MTD$A$CAl9*sKcNlLX zWxKg5$=h&}sQ^A!^terzI^=wuVzAJiZ3Qa(r{X<{RnWL1JsVWWzPic6m}>zyBks>3 zsiero1HmyKD}ll z#m-nX0|8*cExrN+N3q_XB?oDvXw8Q1nM6o{gX9zYO{)&%!|`MxlVd3)AO?X! z8H8YU(SKCcoQbtXdjT9Xy-8FmVqpsfU|oKHIryJM3CZ|bHCpj$!jHCP5ad}U`G&sh zz4`@XOTMxx#D%{7SbbQAp>3n@Tp zAS^J1FPF3uAT&?9>tk(a46({v-#u1)#j|A$XCZBMODhCpe_Tpw8Y=rR2FJsp=XW_r z)b-8FyuiI!gsG&FI_5n=AYS*fyI}kj8$(Ni!P*-T!F&m3BYF%A)e?-<7^b-eIj!7- zBch=;=vLyc9@V{?Y&z&Cm5%Kd*k5Ckm?=0nfLKzp-*FOUCT`~4(1U`)6GUrVz+b6- z_K?1ci|h7SG6Il*Q6sSuoQ($3gZZjhx)pBoaeG-GeNV{IaWgM9Zjv`Q$%3f|3-JuL zf(fh^lhk)j1neeri*!lcj~+b7xlH@bGdHipHU!h=PW!vUEaEfT;JEDFL9slCz{C|S zezeW(ar2Ifjo}&FaowmexpS|U?8&@0%PIp`^P(LmOVB%`nS%Jr?P>Hoo0X0m7k2m* zUYx4fv9T9#E7%lvDByV5PdXZN2(Fl3mVYg^-$@kr6s{x495z-CRw|sB%WLpQ69-)* zD4?1pPqqNLLdwr=1Zq}eB!JCw;MAzAcSlD&-0|8cm|7ag$v&GaE6{^f#i%FZUbgt- zb@Xt(N*zNo$16Lq8{za5*I%m^Jq>$y3oXumgd^R#I=Xv!_ek7;2A*{-lbX4Ecw%y8 z@++G!wDBvHwzM?H=Ujc{n^298_C9pGKNMS7-#T=l9XZ>?u1x6K5Dc5K$#=e2Ve)hD zwFi#Ivf_;)qfHIDB$DlKFV19}p@SO=kKnh3sVX$6mr zt&9c)5`o_W8;f3fjqWrf%2+)@s~q1iiQhDL&Sru|CV0b!c%G$8TQscXU9=9%;1ZK2 zhAA2pZ##6Os?~IYvoIB`XmF0$;YqW-;QVKs;i%U}?hUlEVm1*0ru3>1$n3Zah}&Fs zgakE9Z$;dOFv(szSjeU!XDfx7q+RZL0bNgC&?D%~7iBNHjy3$fr+1K4PDu8!rLqDR z7<(UtZ(8>fDz#TV@g}FaYDiqI#4x3cdkza!mOjbl^w#YFzgoY>1Op70P2>dVI>Ndh z8|FD`@<6N#6Ru)xZ=QIxhhl)!is-erxGB%vr5PM25180y{{xjV-#2U8ZVVGG|?{mrE8Qr&)&>0Ah)CcCnnq@m4c zVQ8wfu@$`Odp`2v$f??cIj6{?V!`c{Q7)ZbuLCy3r=m%C%4J{>!5g2vBZTZ`0p~I3leOa-j?F)USBy9W=@s${yfo;THEyiOB&?o5)&cU^ zNXKSb<}Gf0)Sx;c`cz79Y6t;GA_2xK#a?>t%;x2J zwQsiV)t%EyVvQ?Q&L=T-xK)jIj?=Q;@LrOU=j-7i`+k!6m_td7OE4x#x>cTL$KIIA zrQXqc$oQV-@EJwd8B`I9N){M0Ij3~Dm2PLnQ0yeJa3FJ&E|`t>pkXq;!f_#zEMa*j zs=-c=g5|HyM!=o*w_jQd;0GjN!$TK*GD__#&ht9E0N9}LQj?KN8XmVtcat{76xUek zF05JCs=5IXRz|npk*MlHFcly&W<_4^?*^Ss;yJ&J3a=*g)G<_?27**8uf*~Wg!|!H z<5@v-!vQ@Zu%h~(ASImMu)q<(ga)a0Y`0~#+k2!D=s~h4bJ%>K4k_01vX{m|Czdai zd#4L*3>Q|2Oc|PeD17L?QC8e*R4Ns`jE9{S(8&{f(ZG)blL*`}N0kzONJbQY8tM*6 zS@F?XkQ22jk|jWgbYQ{@>`QB6_^Oz-69qt(AP8}g)|BCt8#~&62tpP5odGBVu5JLE z#{s`quuWs5o%*4fu-_{z$?OCT~Ap)09LizzWDd28cO8Q0ZT^UO0y?05IDX8H@f zVNg;=4&p{63PK=&ptU5a=~a@tvf!(s?g5K;GkXwR>;t^5QuURA`!D0$IR;~U&X}Ho zGMN-pq^99=-o3aPfH(BPW4ECiV|i`ZBoU#fV*E=F+r`Toy?%a9u)Mo!j>uC5s0cl? z4?UEFGma|czTc=BrklYlczH`*1Mg>4s1(|ZwzW;6+2ICS^vAi+9NJ*3(A0EV9vx`@ z;VPE%Z^}o72H2=D42t)uBw(0es`kUVs1Tno?GRh8EWe{DYFDe5#M_!_r~}rT#yVz6 zJ?_128GWVp)q=TKrDlqgN#w%L=H_urf{gbG*`(_**V#aF^?8mK=Yq}57u^GvqF6yA zm^`KDTwDU%G@*UG*7yi!KyH}I7MWPzS%G|_8w`Wgs`6^w6@i1d1)(>uKEM+9S=5kL zhKeVTW(i-6UwY~I$9CKs%>aT=)1--fd)>}@4O!9Y+q^sW50cS~cy0i`*Rh+T9;poXnCf?$?jdt0i#wNjnr87tGn+ z$W0A7=@kbtJ%X=Jbtu#G*99ihb`-vgUvL%o&Xx5S7YvuU4i<%Au0Wa$0rs>l^^QxMsIC-D7a1;_;^fBU$T(58GhJ{z(-h^{C_L;vi-}a+(Eyd_ z%St+Mi>stF6kKfL4(Jm@PWMu9i!E9RIHa5NcU$V95;z2DXf8TdQc*+ATSMOY!6MN8 zT!wZK^C2ib!EK2lAZhHQGUwy?L^Ta}JG`uV71hIa2sDSUp>}TKuDanI*9+);IS?)8 zA!Sp<@UPmWGcwq%#FJNbI)NItAT3tYR*QpWh5BToq>qj=0zjV@iHh#d3W|>q;yFcR(Q{*sCiKSFw$Xe=IlBx#h_6E$BB179U;BJ4>smhDdi(Zs;0Wz@qn(#TXjLW=0&X@J<-xMXc(d zQR<%9VZCnEwR2@%7!6vW6MuRpHKedpk^r`;1x+TCG5@2pd$hx;2EP|n;;t`_=xgg_h<#yy6ukk zr>O2&7<|?uB!)#iko%+Z2nc3KW5RNX104s7m44)77HI1PvJkF!qa^~6d}!k4H6_qS z4MAK+q6qgY&MTKnmIe7#9zep%iIFo9{Y#1qIt8EXUlubfT&079zL{-$k9T&bLT@Rydrw?kl}(- zL}RE$X<}9Zh3&t=S-o;|*kCh+#*~8=5>eTAb}_!w-6tkXhzpP%_{zLKc0Qb) zp}Rf0+DQu_^MW{sAm2YO`-uxcHlg0;fke!>Zh$0WrN zl)qgC@>n@`yp&c;cHv=)dDc7W8ojrtS<(p`ZUgNPIomOmSPukyui@YcjOhyzVmeGYi2!MT8A4f43hCX+fxtZIFPbFS_LqYG>+SM z8g>gU_H;Ap!k|0(_03m<-sX5b5YB80ySm3U%p$#RR<9U_o>L}H1mdU+zds7nb&-%z zr53XZk<=tUYYSZ>KOfP+cJrv{qF z#nx$wC-yE-K0fmqrFlCPPr&lU;HGZjlo~#zKXJb_A+?vHPDTli!(!?f@4Yfx*c%5t zy`IQrx#f%$x37k5U8Hm<2vG%;g?s{u7%$RyslxQ%vS5`F;)dmD`iQ}HUCGYtM%tj2 z$HqX*H~}W^>;Z`xdk`_Xb1h4cLQ)7r?%qE$-TY#1tVsx5_2g$hmY~#i${U{BEf1en zeBq^X6RVZKklJ`R9>lAdXlcz`P_Fm|q=*~20rQ70Z6O>3(eUJ2M^_T9n2U%?zFil@ zo9WXo_dIfN8Mr>F!hu;);Wa3IWZqJ2EcA+pr?|G`qjF_P2%oSeig>MIyz4|pG`)VC zt?1IEC?M=&*zVrUCm8E@8Q8e?_c5c_w(v+=d(H6X2_W;cmSR}3AxoL0GTop($hz4m zwmXVJj8YIiFczj(Os@PQG)<~eYwCXkI^{s*fyY^*vd6f^%z^E@l+#rp*bYQ&K5lz5 znh=Am2X~wTGYpuGRTPlTp5r#lX5L)dhi0NyVY+lXGfvy_J5gdTEXs1yR;Q@le(^pG z7P-bqrPp_9><33YrEI1SYJ_+~gG(8UkEf0ar_t{^MxO+v_W4Av68EHFwNB}gJprRP zeX_pJDl&-~)%fAHBC^k9⪙NC@9R6x7VBzAaoGg%Iyg!r86{V-t>7&?>B0S)eO+H zZ&uO8FC?Zly4h|Z50bNO#@NqHa0NUeZ#)!ayK;QTaM!{qOVe%5>s&fBWU^irT=IFj z!h)bil$v_AL=+fvg4)3QmszUwfOiN~gA}U*8)C%estWq9Xs6y0#PHKx*jQg;DRm_n z=QglBVvv%{)^H>dtr|B(-xz9*pokeDh4K%^TOpB^y8z<%nU>3^1iQLAabHow9eS0D#EH#suQpJsYT>1pBqY z9oe2Y69=NKyccsmMulQEn;X` znEfuu5qg)L`~y5f$(Qe13Q-X`s_6L=8Pn25)_kYcYD%OY!bMl~gL6Wbny82`S89l3 z^>!)AVCZ_IpRXwcSm@2{xopp|`L6{mfn*T`E0l^n;WNrY3SnZVejSbL=yMm*^BPgQ zW>)*GrM1V+#OrNb!UCrWf6R-E=HrO+Od;O{gq}VGZ61pKKQde;+ChRLF(86hGA_LP;}x{p16^-Q`E~5 zo8{3uHq%EVxvPg>EU0>EenSvpQc8-&YUhwWq4GYwT25hlV)Kn8%dEc90g|xw*$JcB zhoDPjAbN&MlEnnnMmWO5iUoIXwN8}{(xuePuJ=xPccMH_5_7Qv?VnQqg2&kb+c6I;AOS$`z=fbVm*WrwYFF{>Iz+) zIt;VRa790Z2kbEtY*ll5jBD4F^)G-uG_%zD&ub1LC7qmf+5`g0I?~Yrs-I>`3hL{C zk?tuOJJg`FRH5iYd0MyUQ|6&Nxva&>ODUdIV+t58#z;UD6NM@PrtoVgCvDNY!vdTr z*5(P+LK5K~GlogQ_>G$(A!$|i+&N-OSUgZJbpg;N z4yVpMF+#H0LrfwpDKvLicqbmQqrNrlo z5zeOQuQW{!yI3wamgUv;xsXh2*x8!*xUe3%IJ44MRxqP=ZbFCcls+qOCttZpJU;Jo z8%DcD--N+DBspVsaqOfX`F^qno_<=UM44zLIODG8R!}^C%QfijE(K4o0oUQv9#riM zig#`DU!uA8{aa*?*KzVAXeXf$vJ)03S^3=Ic^fDa%v-?eHTq6v}hM-iL!hwr?;zkFH$zC0rbcxBLcc|*jq$7%%;c->7H`lvrS$c znV6+57diMrx^iw!#AP6XNHEa`P18?PZ*m(qb&Nctbjm%NdwPn^&ddjsig;sajZECR zK?(<}A$d>(r+YG0i@RoW6V%^@+pV&u=!j%?zo!n}+7Z?yZG@yK=P>@hcvR}N6VUPcqb!L~|!kk(lO*;eZ_8w`7wgMy-jF8dJc3My$6WV5Nh8Ss+ z-P)d%3%f2)l%bZ;0c=$6w^<{poZUnQ=XA3daOAFO*?e{63x2mXKzV6AorhE_GWD*yR(l z&T+bM#Of*447O++UpdqUGD}KtS%G}DWGf9gC1CcB1 z1LhySIdwS{ExCNSii>4Y{ zVVlC)v`v_O2=hA$-ibxDZK2tVB1K1*&%5|5@wT249p!q;n|`}%ZbPc}y3Zjc&X*(} zlXRJAdPF=t6ai-EYWyX59{y*7gC z_3R#E{JYCZGPKZ^J9~H;-mMzz^l7dQWo@LK1D@AhTbG|Y#**{~9*^UM^0cR?`6qgj$nZ;%``R5svc_9*US&3T;#ffR9)xU=0?C zP(8LyfPq(a@)zAkBxB z+&EVjCLxB=LNAXT9eO>)u7hzddyV2?Ul(mnV*yK8SB-n9D@6UhZ+%tiAiahfmEyl% zwiQIAd0=v}#Ci`$R&3mXU)O>=CbR@&?t3+RwpiJj2mu^>4z)*gTSMwLX&3D8i6?yl zn~hFwy8Qz(tAuS9OKjYlP$ENQ0}tJ#FE1(W5S!0Z8>u#6$6s?XB6DkI*+C42X~;J_U#psVC+ow4+#%@A*J%5E}))V$Dx4pbXVj5-vHc{KN# z?#>tsLB}>Wd4q5%`9`w!X>e2CT$QD__Z@w-oXwC~adNb&uSnfIy1MB3=rAAA%cr<^ zDxTmmX^B=m=crjzS0*DUcV3-FQbyih31IpI9&tYKG zLZx{l0lJ0EZk9T|2D4UDP}qSmyAiq2kqtWZuT{KB$GbXoni`DiKu)rSiad}@4Y!1w z0y#cMNYv{ws*YEA)kD*or4XjW$V?fAq`B2ao|4TGy1Iq$Pm7-z?FI7e8Il5j>xRwQi0eOXIs?kdU`UX%+Cv~rm> z0+x#&jY?rC>@i|)Frnk+5!*PZrWFWztnWBsd{@9k;%;IZ5X&7D5l!Z)*v``hz@6+P z6zcYBJ6uRxa!OksYg!m7&WVk`d4|!RM{l+xJ+>xWn#HB^g(`yMw0T#z-KsMIWMpc^ z*m?EbjHBL%dlgvR>wU?|1j}>+Pwha2NFugVp$nW_Um1>+Kpk8nSwp)fJks`fDNo*d zc}a-qZaMma4zHYddXiJHjFF*9FqX(v`HM7ODj6ZcE6H?7vM4Ozty@Ef3L>s>sN!&; z755omu{4W;m@+3==}6@;6rd z(XtBKp#^8xNY#RuF5_>}gpMT!15iwQBm-`IB$oCbKV5GqGA||5MD^u_Iy8CjvphY@ zb!lFG&2EnL(in(D4%?Pdh&Mq8OyfN+20r}nM?mb?fq+3eC65>jj=W;BP`ncp~0u$#EtB)`?` zW-8PpxkMs|EuMA2)zpqtpdJ8k2ffoWei=Ulh#qnbyha;#$64nl;{x5R6}cs&(KYw5 zgzj$`LW+(|uwWgiOYKYcHOaY;Fo0Vyc#_KO%!+&*4iz>QB!{;u@VAanJa1Yk3Dcrc z*;%$VXL;Ihea1{Ub<1FM2^N8S??+nJ?`1x=E|wL94G30&xYB0i$O_!F-O*z`d3n*3 zvvmX_+m*)C8Q3l%=A0vF+NZc(4HVb9}PY z$=D*jRm+MMIkEUhRb&r4SIuy?OtkFV6>My=?x=E*;uPCcbDP;VUb*d4Vq+Mz7a19q zM(G${!qkFYIOyExwZ!tq-`-))kz;o?6n^tz7-yjYFndS1iDzRu6(Fz)c|8lpRq1fz zeKneOvG%y*8)@S*R^bG;P>7ojOw(Z)a$c0V!|d~jJ88|zHidMwmA`1U>;d)UlS1vJ zJa{xa&U%m-L*TJYB?ht6Zl1WolpzNW*oD4) zS+!~rgHdoaQYf}jn7TRpFB|lym!pAAjJ~U6Q}P;H5h6i zvxd}e^fMrieg*-{%p63fAutPAC zI%Sn0MZb7OB&xAl1?HliD9d$fLpz4MWYB$b4F{7OXGWY`_nQ|j_YtaJZ!(vC9qWDa zC%UfgOSxPSgr8YK81=E4=Cl?g0{d&Y)`dlC$h{e90IdgdHrpPF(3Wsyb7y3kwW29) zxb-!dOp#|=*k#}_XGe({1bWR{%_v`(0?@O~zQA{b5@5ptjic6sO)26_3IZAOTTK+U zR|S-aso|bx$Odb)6&o@3l z6_QKuhS7dy)nVi+Cd*@9_-J6P%Ybu`DhLMJ(E#r_MvRvU$-v-c0AxzEsKpg_D&3nO zl1-8-<%t6v3&b6LQ{a-O55nQ^Z_Wux$($Wt2VyU2M1cXXrOEz&r?stxMd_=C{vk*{c zlT|t5-VU-ikq#&DD=J>|stq4~N(T)%=*AsSgCz|7elc-^ln5nvg_l7$*TK&BMaq+T zGuNSGp-Mr5in;6nm7~H#szdmKBjL)U`h*ixD)-Vmjx$?|LH$+cnZ_d1-6r26y9X-= z{>rnhQsRDKj&4?j-Zb zc_+JDe{ixp0G}#&$ds+zGmg}C#~Zft37hOYb^K&Xv3M{jb!BCV3@tz<9K?aYNu$2K z2@Xrn4v!=AT(Mm&A-lP-2t(P9O~01b%=)$lCNrnO0IFC*y-1>9Da)mAVd}7>MHkUu zJ*x0+As2X?a!YqA`_TA#S9B{U=E8JoK+R6hP@syquNsLbp#!j1gvgktlhU2n)~8U-*KK`Om1UOLgHwT^H#{X)WD07Ev=XEp_dVA_L(6te7 zFQL~EyS!UGGU=ZsfF2LFOTIKAo^8O?+q#ApbG4@;k0Pr63oUd>R^_JGk6&PzqpRH$ zULUt}Y-)nB$3uu4jj5&%@;4?tN0FhnV*Y$&ag{U!JgbKHUIc(HJ^|?wb;Z}i>K?47 zIJ8*X675k49Xmz~?s9J&@ugyLSgn|_F6uUja(m}oRLgM8GZwpv2lo>@u{BR?L&OAQ zGz)~ozX(W3Zr%quAia2wM?oK#)WqBcHJ-Mn=t66y3&@m^iJI$Fk`cA06^f5BYU)Dw za>VQ|$fv5LgmbL+=GZ$4omsX5$s?ZaO;vikU4@Bx2}#zMRV_4PXW^>oHh#Cn`|%9XbS6v5kE(#U2Sp)8=Z8TO-TH_f{(JDS~SOv3X|` zchX%wgh;hlw}Q56-ab>&aS%Di!Ge;J3n&ByQxqCvp{*X|A7zagO91HIYD7q~bEK}p z+S^*LG~Kw@x-^eCNj-3!UXW4AA5HYus`nL8d21P%LeWmd&w@$4_&UzwJ(5<39m`2@ z*d<=-K+wMpNU^YR?%0$LKcdK5SRav{3feon$w3PnpoeTe^j1VxqF6|}yJ3~I+~29+ zU<)#ytqaO-sa2bylT-wSIJ?=(;Hh`-3Js4!dLNh zItLJg-4ALCzC+)bi@?=QHeTVw3p4i&Fkii+ z>#PNf(*VSnIKnr*S=CO_bcC5+m=Xk@z0puQX1{1Ifo@`Z*HCAL&6z>AmHNZjz z!b6&{mbCG`6DM!lJ#J^9YSbLpAi~3WcX*GspiAWfKXuCxto*Wq8r&^a$;mz%v4YPQ zJD(=%S23vrnS$rMqG2MPie&z0aV8R#DvZ6#l55x_n`NQwMG26s67rX)p{iJp5@Fcc#PR3ia@iou82vV-+ z02Ut1F|ap6WXa3q_!phQChG+QMrP~85jnEersbg=xJ@x zTm6b9?d~WkO{Y;I*^3 zbW+8NV1TI@c+9UeHZ_HThDD@>n@Oousa-PZfg{G1bFoh61`}lYh@2j2jL^ zG)r(3HojnEbr6rNs0&PbwhoJp95Zf2vrzHnGlna4p1-YviFPjHi-aZ!CyHM>c>Bhk z1`Q^fnjpKAQXSiClQcRV(|foYQW6*8a!OeXRU3Sh@dVV?oZM(_?M-sKgNwjA6+uUp zSZu-jC1T3+You`OJy<=jVr7hIlOn>gn`j2ThZiO2fV;H~XNYXsTN?u3Z@O zhf&XT{p-t9rsp$ffhM+QP#?BS)6pDYu6DLv-asS^TUHM)9*{!~cbBFz6h_N46>2?% ziXq&WY;?y!BQWL-atB4DIcIvc;3+Y(SO)SN>+(XdaXR#qqmK!2aMkZ_4De}WT(G_% zp?cgC1;Qe61YAM5)~>cU$;@~@j3wl;y&4ST-i~)RCq#HUQ>Q%6MeZ*qA0z=8*_?IX zh!1XHz11QDcA6qZZ$f7n?0lqUk;0c~CB_*PNt2?`H(;VPu{k8BYLqJNQ1QakhK4vj z%JWDhDYEp}gq{sH3XIa1tA5i)EWZ-*hGv6$+zihHiOq7vBzhyNO}cx=BZVDy{!4Th zi{nN~vXCfv-Q_OX-vo!bLcFH7Mhk=tGO@Ql7Y2pknkcu!3AOZd+))6OUCFX#^%Y;LqMmZ(5>_72S(Co@ZuD^3^HBhum%^++vbrFW9>Q4WG)7(ERw6Xspum5mB$pj6N{)W91BtpMfu`B=gq^9A)qEri z^C_i>a~t(DJMy@waPs}b$0LE(zNOOBil^ciz7Gb?`No4Heii$q;U-C(`zLDI0?$u+ zJRxQ&mrnek<%uk9DyEqc)Oe3&6Y59HB~{9PWjpc=V4$aKF$;6bf-jLH7Z)!e*xsA1 z>{x-@-le0&G>@H$pNDdZOvR&%eLmnvh~r~FMrl%3?HHKDEah;Jb+)g1m4bDJ1XlLV z0xZ}O-Om)t%o)=bZN`p%_bB>lCY~^i&7xbr2yN}c36gf=?}5!fwb1UHrP(C;bDae>H=etf&07e@@#uwhX!=a$-v z&tGD4g|ST5vRl#G@!PeN;-7>S7Q+*QsQ2f6u^c+&*8mV)>K`-$T&Eq!Q#XekC1d*Z z5X&6`^cP`zg-rUqTN&)s5#ydy1U$TxEt)NMkj=dI3p8nTu8SToK5cfO&Xb^@(gV09 zfl+b1%ynq@DCweiYmBjG!8tce$a-CuJ)MEd3a+q!fe5bETuz~l@CqH z+KCy1#1=;35Vl%bb8;uoH!D7|US^su)J1KPH&c;c0XC$#QLfo6(m7(avrJlNoD52> z`styD#){w%6Lj`gRVud{P(YLipr}$d4#>jyk`_`S!bNKG{oA6>-2o{u52aEjn;fyK zm?yXr9vI?!%G8X++wl$p@HrP&Q`PBPdVq#kr+9almy_Cf`c0@sWp^*F47pWg5m)nHr zfX0GTvg`y|qdQYON9jA{(M=f}u{PuNXGp%%6kZdZ3(^|LXmyKxAYkua^uS=;E*xt_ zknneftS1SwGkLZM!vKZU1qvS?lDJ(oIu?;ng5A_xE=x2;NcAltY$a_JVnIv#XEz z^;9FC&>o+3KMiVHch2~tk&)h$QbAe3hs9xE*Um}0_P!TiP`0R~F?+Qx8GMYA4wGrz z{yy<>1EBm|cINGHFqM#r0mv)9=p*i*C~(IuXeSlsd$GHe274(L7&KIdnxM2$eHLw( zur8OrS)CHtd0BCU8zu>UURMWT@v_miy9RM=j*(;mM%j0c*cM>afJ6oegc|CrP|$ID zD%#DMua zX7!BnuobAhG`ibKw=HmSZqw10S(6Y}KR*T1%>Y z<6uqs6w+T6i1*z)Z>sX)(batFS-PQ8B4NVsXk*Rj*_X|^W$Gv0^%i9`6ijLjhXh|k zJoR}lwxncEw-B;j1B`uYIc#j(6`5z?BNI0XOZGm-C^XF!`91K&72s|y>WE*0ZL-iY z!+BNJ&xY_KE$Vhw$Hew8I7#WrO}I9!hC{lrdY9myAY^MC2YwGcDP^h>UP5oNm)MwO zWRRX6p|&XsQ*`Ub@&oG;4zwj`?c(+C5EX&uAgV;zYG(j}+AibG;4{#*+JLOhi=YUc zFkb|5$s2_eIy#^Res}wkD3c;90knkcZybwg+~Ys<{5Aki?%~`(n-jT zmu{I?z&q~>{Je#$E?kyfx57Mgb1~(PqXL3@=|TEgpRvKJ6syVy+jpBnKtks}UU0Xr z>-SnF2RT+E+U1yLP)(p440JmLbd!3`&RvFy@jU-zx}qqS$Osg=?X}2t*$cix9bxEZ z*e?V*5)s;P=~v?@yP+_;9+vE&xaJ z#uF|>n%Tadqc2LV040e8nlvzJjE6NjGCloXr~yMhD3g6k-v zoSw<-yx9#(Gw>1LzIFmu>&m_QQ!RrxP{L;}#c9EP5dw&k8kSDGrF^%k*)||qJ4GgD zW{8G@DWal6bu!`m6!TF!=#9H+c72KTcn(WimQ18q2riOC16V{vETkH3>HE&mRnLW3 z4&zT!OZGdgb4mzN?SW(s8R~+Xh=!xsA46%I)7}<@n@>3+nuq$6( z^3th22Lr@m28v^mObhz$LQ;L&b*uxF+0RQ{et_$CAUKxw!fAZ`)O<~Tj@p~Nozm}rIpAnb1+5+#CP5 zi?g@eWzLSJth_t|Jq+=|)C0yDXn?I^2p^1=;o})?0idFT(p+k+Hsl3ip@J#Vot19P z!X+&vyAm%?(xeJBw$xOG3P{W_V;$yt3C<4?Ym;mHcG-XwhElIrhotlJ%wr(I9xUGm zaIqXK+{UPLM2bav~8LuWF1iKAQ9+Op)R0+-D+^TC+8-Oy2h{G^UVijM$6 zx=Oii1deZcK88pgre05*H!8~md`~YSswue(5bfJ9sdTIr1PFVMY1}vPz{dhzWm?ig zRLfQkJIv~$G)3WBoQ7%6is2zj<@OJqJaD6oi_A(i#qDky1JQ164lvq{iVkkV4@bOF z^$IMzRj}VvUGsT5)Zb%U?bvHe<-UuSM9&G4409haz>pwf$zfLQ1(%~MsyiLQd;&`n zIzkqevex&#?0zh=r#%?=P7)JXLaEGy*76?buLzIpEW>^oPNz_sIO@8G9QccL)ie#20O7aL z9taT4KR6~dIW&);`&b!n&e^(ApMY6wO+4lhL6$eEPnaEE&^w{!YmS+sJ8RP$&hv3r`c(bH_mSIZLViiMtjK#SEu?ipi{p}uHe(sW)c{C(((%1$r2DuD90G;2wa_NTZ&r6v01+uFTBb|lPX#q^m< zMIKNq)d{1c=FpfJ?5H&u4@ZcUlzghesYSm_L#p(YnJ%FcTFrvzf@rgND=B7y$RzBG z+Ae`}oZzX%e)mx)Mr8yp2CbG3IS^Peg{Iz1EG1joZce@9Vr0>uEsQZPIlUqD8`$Q> z9hR{i;gfre(+&q+ey)e-=7r=UtIBXRi&#-ps;YUS^`lu=lA7oYW)O>s2PFmc;d`9x z$WGzzPC~#z*##l%1|x4<69ikZHg|_d0x!#$gGD1{O$?)jV-IGsK>Fv?1&N|tm7^?) zBB`cA{hlk3N{+Br%ULsJfVeLhJFSvOdkpBd85rwc1zBGdrI(?1$`xj035=S~R_+#Vu;L){xf(`Q za&4;!Gv}YE8JPm@&z8MA(%WiCbn;|@a0ZIiq=?t1`p!K8;Q_wTS3(2xM;8x!htY?# zvg&S*7*3HRVAG=5p(7;>qn0{}o50!#xd(5k**;e0xmDsj4kfNu7nOSS9=s^|Ip!{+ zs%YNMh{Nf3nwF&O1JREfP__xNC#kGVNwOAdfzG`0N=YesKIWjbX{))BVD9u@YjXs7 zV9DwMPdCK;N#-W*&YP5%2Dx;5cDD14hBGUK&@Y%6>vY;3nF69TVLU>Nbh4H`Udse!r@h!b6y6|lv+$$zsa;(UhOtk2cx--8D{qBa(7rjE z`WfEwtzE)sD+qVxrkTG6v(TDI(uZ6R2d@3&foT zP!cx}P2=4AST(CwA(9|+PbACUX1Kn!|NUHk)D4>oeDi0@9`?v`T$?_V!Aci$5m!ok7$2&nGgO9Du z-1cvFC4n1f7pZhcIqWloGw@32VPY34IXr{9KFJ!h7%q|6v}qPtIqg+5w?c9S#`Qml zc~FExRDjVGZ%SDOTe3tLeB!-*K0Z;aq8pgyX&@1#M-Ch6B#!~$lzB)~BXv%Cm?7{7 zBa>^uHn@Y`N|Zi8Q-`Uy+$4ptu^#eZmr_Mxxl3g6>y25gdn$A*`S&0y-$_b?y!9_U~C_bP&|5w82WFpsD)d> z79ACPomXlREMf|pn||@&Xp(IMRA^|io;r)ODoiyx z;-Ps`qmCH0yyxK&(+$SAFf41^9}L|O8-jIEHq5gGLdVQFG|^^B9Jfy3k5wJyIi zxM#>>OiTv&9JC(Z0j1klT>v;F`mY=q>;>mXM2=BTSdl&4Ag#`|(dRd9cPKRQBl(y#ukb~ou+jBu-B+9EQ;XyLSu4K}=6s;&VvX6}sZ- zc%oM6U!>vSn7#b>tLH28;UMypfXW>2+0aQFPk2*wx0%w|n%471kl_)W;#%_kCfGUK z;x={8YX>mhdhW%_atkIS=wCFbK)&I$V03iPUNC_&q)VMkTnTSaW)O9{z{aL|`NaXP zRPlsbrH~QLUadPGk#yEC9_8PYN@PfPuqUeTC~YF=vh|idHaB%19!ft#5lkpLtmq_@ zPr}JH!D1Cfto|bdO!5HTyYUw=v7>H+Uk#+Zw%FBu7MmEFgty_bR!5%z3ZI?~V)Lu& z%o)xbDb*Bv?{q;y??9Y2op`dqlMHNP4IGhoi_^{u3X-1sk#0<~ss!@p4&614T(r2e z=Y^h!l&>YHGJUSyMdJ*y1D(&*eSRO>xaI!`4NUOXFSh??# zFU<-;+BB|00-W+a%ZR6Fq!c<5Xg!bH`@>H%~UuuvMcHEJeGt+kiR)i_#NYI{) zJTljF_p5zw-052jeNZ{bY+!}G#-Lf3$KsY;JiKW{x47{18L&PG$grUS`qNcJ$dKd8 zUULR)A#TH3<@=HFSETCIF3@mSty0k-8$~Q!xjh7Jw!U_1D5U3z={l^AOIhJ{a`ps) zUKPhW32tdh_GO#{S_wFTI2bBcB?uAQ+~0?)7IF$riwMQhZ5t6p39u?)Y`KWG%vSS~ z4el!B#56o2gUL+HiKB|imlfz35xsjFP#gF_XO8x72uc!CwQg38TZVNGKnHbtT1-QD z6*qpyd)ho}L|-^?@5W=vz&C+X&!9h5;Wc}_#VXIoni2zzL!rbmP^C_pa*IAk%Q<`D za-Q~69;m21om$`v6A(ihTqGZRUko@kYdp)YyP>9+SZhbO4cLbr!L;11Yb$1@Vj~G7 zoUX%NE+%05y|o^MbUdh>(KBx_#6tf$O_hwk}(X zOc5SEk6*7_F0{|Ygiw&5L@ngbb-LKv1Tdw^%^c({=%Hg-^&DsvQFOFk9>ZsLIteTh zdy6lTmopOl(7nfq&y;O4%6LO#JduOG24lK8i_4*z0!y%fHb^WL$<8qXOz1(7T#+Gh z+8C_CL8K!r7`Ji`mAS&g&D?wP0IR@IGcbWYS>iawO=_;S+_ zGeXwDrvz67QNc%&8<|-mJx9qCe)Q4uhi{&~TZ82Wx(1v+6Kys6&51*$wffOVyKyWE zv*WNUWO-ml2e-5_E+K*Ix*WUM!n7DN`v~yYZ2;Ibq@6e3ITQ^FspEap)ws~UU>)xv zb>&N$@1_zLZ8%D&k2?B>SRvxsg+h6@_#KaW`QV{Gz2?x~FW|8n1v=utMF1wHA~QwP z7X5RoMa1^yg&^p<-Bn&cy5Lr4W(%PT$dq0oIG*P{YtBol<;3#}9A8cA>dUzZ>BXm* zn@~9#NxEZVmgQlG7>o{xo(Cc4+3@TIKGWpw;!*;c)UeB#J(&)HWXyx!YtHfk&Q+Wzhny>PE6k8A^)n(ak9@D*TT*WADKzzzNXHt{wV;AM`U&Zv z6c|M6W1gM^Ez+WnLB!X%B=L@4;U?NXa(Bk9$r3;|`1k4P^qql>CgHeFb-=~Msn7~~ zbw}K&im&Ed>U_ku?@K`4ABGv?y ze7~R`@DzKO z4jxM_T3)63qv<4G0fjzICndjIduZJ}COLt9C;}AfgL1P_F(F`wbQK6*Uv|OGoek)3 zyWHVx5K42G{o=wuHpwY1+&XA|h2qYMX!%L*baO<>_TDccsabCRjw;|2Mp9HwD z)}|rg4d=$uWUgw?j+7oNP3@itV9UxG-o%T@_hNVkYeHly)N- zh8VEpl7a!U^N!<3LsNoBaV{$)%HDX} zEWSOgQ3>gPM8cTQ2za1bTqM?=Ak(7CzB&!6)vK;A-ihmM1!Q#8IX39hJJkdhdVoo8 z2PNK7+!lbEGE+yXgmAg&3u-fmnb(NU%J4FQE*l2|GY6+!Rz4Q7p|z_>qbBlgl~1`2 z-d)fQw76?fh`VDpfoph$fP^zMhp~B_3U^p6I>*`_(@owOp0&IY!f1>0=rtjLr($Lc ztIwQYiFSS>T{H#!CpHSFrSz$~)5_7Y9 zycuURNt`W1)+E>wp%Zj!iYbtNZ$Oz;F@5IZ&B2xdZGi-M$Ru`RV@@ifTZIjGVj!Ly zYUvGQ67$LIw)-r_JKUF;Y4@b`J+bsoJ1`Y5MGWJcptHO*+ zta!)qCp_I2qel*wlfDr(@4VaOK9ThVPz`gNaoU+C)dT~G6z>Qk^zXG-Kzz3K3TF3c zq>*Bk3bJMwq++%Pq()WyEi;9I45$G%2@SvXlzaZx!`o*itqTz-t7B2R80 zRZuYBw4NRp6-<=K1qhrMeQwS^;Yr#){Dg8XAurTZgX^O?r}fU?u%)v?jVCq0q)Ztu z(eA3*YP`J&D+0ObuGK{sh4%}2uG`H+A(y>;hUnnLa`FuX5o?TiXCAp~$dHx1%0sa- zJmygq?)8i~8E`aHz5zEaO=74VDCdVS&>ApdSgO#OcW!iAui9v9dx}KwM3{wGEz(4f zAsWI?>A@9XhRJlWn-Z&j3ir6v?qk?Dm*RQD3GSKKzd>GbRD0dGd1z-|R4&P9@kH)Q z89H0_aJMN9PoP#59CT~mHIn4omzu)_prr36s!Z5=+)YZo&iVB2>&$mOO{Qlo(hry| zR=c`rWe#)Fr+L&0ZNqOzra(fLDmx3;eljjOHk>_PxMmBud{ADA%XtoA%AI5v*aQvd z@FE<^!dBf+cm!ZP*4&Z|GZqGNtnGkjtTRC!G6Hib62{CtXW%? zGvKFi9+}xu!r!VQUyBLK!Os*1#qjVfdL@A}zCF5g#Taql< zFL*KQxJ+GpqHKJ7F*YroZOZz%k@JuyGcyBU9w=ol1m(lMtz=^F>`Ow`Oy45~+ZTuk z!UyK!R-ZRU&De`UAWO9|LZV6yzz~^to;qR*+jk!g6-vDZCFHsZj?Q=|ek&#i+hS0okI z3Z-}m<~Ahtl*v90npHK1dWV)`_Kx0OJ`z#&v?}XZJ1QD%rBKH4>uGY);l&VA**wJu zC^IPP3$PM5_YKHuTiEAx_c)%$U26N*i~`<$^?;k(w^&2yL#DdA?3YkuhhsR(y11m!`)Smc);_Doaxw?6 z9>XkfsBmk)aa5nHy{R^g-tdI2ts!f^gHy4+6LrBDwWQh%iw>Wy4R}$}hX`F>LDshd zNCj7{{CmK$?DmEPhsAMEL)(-@qF$h|_)ja{-WfbFIxSq^WM1B$cgV@UfW@&NK{Hmy z;S|p(g?UxW(bF8Fh`e1sp!c*c`L!h9H#>RX=aUf#6c|orw%@PBEUO}SIJz}T#LM8A zdfO~~8GAz2ZALpRig|WD^$5_y_KF~dEZGlumlK+$rKUvf;|IB^s2<+xp4kf)HfH^pI1L!6r()o(*#oZeZ-`w16Mt=GL(W>xNkU1|?A_|U@< zUGQ>QQuU&2X9nhSq!vU9{GCP`>zYp%!vH5pRcA4Y%SmP5m#R+?NqUY!?aK0KTf${1 ztRmM*P3|K?$7-3$5PR0qm+kYHC6stP+&zLoVNEhl+#H*JI<^P4i^F;{lH$F|6BRhs z5u5tq#+nR5O^KHlJS%F48KZeh_J)=QF-Oif#^ymY?c0|T;YD%ia7`dbkyaVO3|h-K zl3z~UH!_TT!?6dbz^d7$hes(K^T(Ll95v2{gf{Lt_Xy0Ok$B(d@($z>;g8QhQ zc$;c-KDI=e)lq;VgOb^hRXj}=Kjzy2H=-+t;@U&}s z#`%B{`*(>(`=SO#U7s9Q_df0LA&c!-ygt`dC=MtI+xDdM%ua;l&`%6JU8s;=J}H>=_t4yCFqS9gn)+n-*dFSAH!nx7|-W^NjAUwcmCS_Ng% zi#%BqSQUroocT8LJS9O(K474C|k<`#9+@p27Ml1cajpxmr#SY zk5EQF^@#>-$&bvRJ-v-gc3Jog>*RyWL|3;JWA^LR#Hd^RTZI0$U#fH(U%AV#tJ#|jfWl+9VLC_ z9$@}2Wj@;rbK&iong@17XT(6zVKd8uUB!fbSF=me=-HaO1?Z)d;`gl;Q@&-^Dl$>Z z+uul?`ENsmP8|2qi{-^zbqCR~k(EEG6bRs=L)5|Ca$(DT5-I}uVH_}q@jq%4EZ1kABs=U~y4QUPYalx0l3xdN+nvA&D8f%9VISL zl>V@{BZ0UuzH0HB2%s^M_FB8Sq2X~#216eOP=bYKj3|V($W7m#*}Qp%+zOLi8S7Fg z)@RF`H$AH=rpV3*#{nYYMzM|+xe^Lc60o_X9<7Hc2ZTv8xao+BCvo)Ql!HzdP4l+^ z?FEgYl?&qm*{*7|EMH$@)t*sPGAX*+tVFK~@q7~gRFx745QL5_u~T4+Dmvm5!4--X z-Z$SaIfXh$p0@yYYs@o|4V?BemExkzdB*K^6EMLhpO?*{YI$KYiWnTd3MZZ-D5E5} znkMqiNN`%rFMGI<$fz|T@Qf7v({d%`8Q_uNyKj4GD#*x87O`E@VQ*R_I8Kf3jG) z-Sx3-iR-MHPpzt9@%AWpH2j_1cr(FQ(N2b!+!`nsFm;So?-#k>!-+JPSiv0(T&ruP zGTe+aou}I9JETlcd*dEz)3!qT*F-?Tlvg`K4V1Ns?_}-N86y}>eO^BeFK<}e$#K~A z(Gol2&8m@QAy0{TqOSO0Da!0`*yP5(QdyV34&zOb(Cm??8Q-}%_IYNr5VKz13c%j@ z;7-8D?JTw;DK=j7?&-G!)zh3)h+95hjocx_7E60&Q+dhw-q>dCN+jv;f$;I%zZDRW(IMdyoSvCcOMAH=N0 zB@(OLo;F?^BsMsWki=wOxVd0cAPItk0-6pR&kY@E;>#wvo(Vvu|VYVx+h_MJd%DeB7@q(3#Zl< z7{m@6jcCNN1j3UGrO9w_Y?ZlX#8wFs=+I!&q~TV|Bo2{u)kY@t$vAt~&Yl1<78L4} zceS>pUNm>YoNaTF>=&>VF-0l|vf4MHUahT69%5-JH(DwT#H2ENIfe1*vgxHNvs^@& zc{AF4bqQL=J!F){YNoR^yWnD$ndZYW6d+95ZoxZLd@S+Dc#4FY$KJH=Nj?Tm2=iA= z1umWs1-v@WAJw++ywyQzZL4@=HLYR7$WA0T)KgRDn3##BDyu*uS09utY!N-PJ(hLd zx)i+Ami+{NMOi&5G*Mad3oY$LBO!nl_v;(4a?^;jEw1{Aqcj~p5@++s7S*weW7eIb z?Vm6~Y(xsvveE|93!#zh5mB9n$9YH0&U+@_?e=ps3*9F+YFOLRMt6qvo5yIygyF^; z;fGe0W_HVLNYcJNZ3vc%s}ib@NL4%)?m$&0HEpV5VD+B&fwe}K^pl#Fa4B}Sb`f!f zDE!thGwNnmccH>)t2}8mC7*T*zaY^Ea`KoNnU1T$!WKAtfj~q}>@7pFL)8`(Z20;= zRS^XAEhBeqEK>_#WNjfO57rh0P`mE>sjD4niX_jZ$=osEu=$#7WqQOCh^Sq09NXL7 zmAj#dq6(&WzQlH_=yM6(RiJsxzh$+$m^w(?b56aT*h*TE_Tw@)(=PD|QOB%xtjQi< zPEFES(lj>f`WKjxGnv^QwKY`bfN0FYY#wI@^+%uHlmlSNl)2?io*QCszOKR23vFmr7d3ACf-G+UFgd4uE4)mu>y7h^3af=`|ELsz=7 z_q6Y#RKaqoLZ_g~A!s{ZJ?}%Pb9xzBUpQXHZxn(t!$Xngj9#;XHd*+Ak#Z8N0_I8S zdkVK%BKMG>LKQ;7XR_T0Gmn7#So)_k%81-Bj@@!Idn5?SK;;^i8hjbg4Z%z0Go=}c z31KK_>&YtRiYOY(y0%l=QwfV5S*4uR!Haz0n4vKak}R1l8^ARQq(cj;OOJ98v0C=f z>XJDKixP72UX}-D(_KfTdJ?!%29xJiaFT>VSMvjc&xg%1Q@ zpOe;jhl(9QiiYXx73*u3G4}@e`&G?KD*XA4r9OF?LnQ(@Ucpq97H6B45-P+XqKw>Z zmSzvRnpkJ8xng$vomyTFjtjBh4~(^_O&o%w%gI9BJa3LJ=(J=TcX`pcn1-fXScxN; z+<21*wMfo<>UO?hG^&L}Dh`W-0)Ix@?arbml96ANI{3BxR(p>Tg6*bPue0XtB7sUO z#$dV~UXYXMuzJsLSXx$(ite*{fMy63U1eBXU9-hqf;+`MNDHB8aS0B=rMM=zwm@-r zrvZvf@BqOnTHGxJDN<gNnUJee=(}>%rRc& zVkwLO;~3x}fJT{N!iiX(1y6SCL~O`q_3PL;H}y$N+fZKx7l-GqEYG3;I5)&MO@%n3 zQI4lNg`#v*Z@S)BDnx$une{LhwGM=`zoz(HzARf9dNIwY{f@ns;ygQ-Nw_P8Jt(HM zI}CX4Z^cyjEU#xlEoarLmnL;xGVD)_+p%p%){Ofl@b`O-w70e#T{GCJSl zKTls)!Qtg^D~oZVThgsryRoa=4HTANB5K8Q(+L52>WJ(JNmJMinO`0J&3D zF3G!m6uKUWM4c+0P3lc3GbU^q_K|wa$LXiNOtX5Y+{!8^|JI*!4uKJX>UfwUxzOoZ zXpC}~AhOC4*RvUso(!!4@jUr#jl;mGJgA8aT^j;}cp8TX;sI58SQBBm(4buB^jJ@8 zy@oDY7%`HwEgOljR&_92H)^M!O3$_5(pcAVL{Ve%_2AaVpnPEV)883&sapdRR`@1_ z9rUU}h=gTpJeVnq*-F6_0{!&-*_;fj$Iy6IqdD945SUm`$d<5!i3hM6MKED4>;M8| z1TcD8=;SUi8rM_UvjzsNZ%!syW8p-qx`T2s2$TRw01b-uV9|z|BGW5T#u#vO&XZCb zfr7w_Y$a z3I$BTpemu{2o|;GYHKw(u4AL45GK{J$`huCKvhGjx`6S(2ACFH5M;PT>Wl(oDxlQp z2ABXe)g5g%!R^jBm5!)#Z>}Ln01@QTpnOm{%yO$NG2i9*i{8S}1NEEkcbeAcp8cx4%)F$j`jbkmSmyguK)O3PkV0_~& z#%RG#RS&%+l&*3QZV>8Ff};o>Ef@os@kJ17mYC5SgaTq#es%L(i**f^CNCj(5HB{S!-b`koavZ zf?dF*EeNOqm<5MCr2+v5H*^xLL!cV#K*ST;5SVfd{j77s`V*YeQ&2HaOX6&Q+JEgj zu)^Bd8Ug}f=&W&_8y&&uTn`BPsWP7`4~(j5v!>IS(nB=ovv65MtWBJsAk@%qO$P!c zw}a<$-BaDo)}ZLTr@(Bfch-RNtWEV0TOb777z$Uj#slTE7&oV&QnpN=t}IwFn|X?g z3XH?msOxE=NDwcKWDCC&F0|G5G{6r4%Cd}tLqOI%D5Wjsb_j-G#B3!!#TxPiKJsLw zj)yfZoCJelA+Kbih6%ydVW#a67{N3hviT|27#IYk7X?M2cu)XL4K%qezMaJ|J!XqM zPOtuH5ilbt1eF3aMr(Igc@TFRV>mn0(;Gmkj_RG(#MXl3h=e6H!TJ;9@obTLsv}K2 zOuMu&s*WHosH%OVr(j$36D#sSQ4lmV{)sy}bD+8I^%%YSCvrmab~beB#h{X5B-T{z zq>fL(UDn9WNQJqaluUO@(V6>qjRcz!K`OBjK)1-1%)!3NzVgup>#Y1 zwv4xc`7GE-9miD9oGs2SzI7q=T)_;gb_Fg3YS4>$qGJe5)tS7?g#=AMlj{gKhM8)t zSaZSj8erh&Bxri7i;iPFGS)$39a)bt?Xorn)<02HbL|@36xg8ST)oBFZsG_Mf@xu@ zw=`A~CZ6(8k7~3}uY6+cnl{9i+LpCIbB@_42K2pGq_ooL=JzO{yS zvuW~Zn2s~OCvRKAiZz&igvN!1^C^33&=j};iUZ1mZ#L^_84n}%tm#tQ(%3dNo7Ri( zOo28!5Ugk{6-TZx>7jgKE%A*>)tDLghp0Ksuh}$z6ib zgb5u7kd7ma2xGRcY7c63hiaiw+16CA42daUY8!>W4maKvd;9ad!# z(#gMnIHXrF-HYydVBA*APZ6)x85F0(MK3{sl^^-rd)YasASSQ|e#NR_ceP_U8Ga(# z1K&LDzaxTiiAIhJif+YIqu*p)i)U3{Qd7`U1DpD!b_?+ybcCfxjB(~ESPOAH@C!nB z*m%P!^R@tbQ=%p>qH##2Fi!Sj|8qpB9jqskN>o|rQ-F-wB~hNyb)M92&b-uU61$k~ zn~3T8)88coS5taI@%sHtrb=HCwzpSi%Q>4Tki&9mAE7-@*52NR;(wnx9egXi(VL!R zfjAED7-32N?~|Smh5xD51&bzYylk&ZPF!qBQbUbLBpwo+Z*s5QXaTua2~EoP%8>uu zg(9jM%J+XuXGb`L&#K~L%akWsC++>7vxJcEvb$Zk$Xi;;g3KgB*f!D`Nz@3t8`kym zKSWqGL0EZl>IGSpIBXe|Uesdnzut2tqmn{~So~;u4e5R{WA|Mcs@41o)z8cPx@UAN zBlEJ&!bvc}SoWM5Mk7kiz>(x|Mq8r=6}PJMnyj~&&l`mp{+x&mEoyV!w4i8wR=&SB z9qw3HT6-wb_toW>^KVodoI*uR=D3EmL$;c6h67KDNJ=J$>TWfufZbF(rHlO#J4gcD zw3JO)lsME0e3iEINl;11j#BCIzjyWP>OjHZ4u1yU2IdZV_SeOW#e6uSc+KsmkNH&X zzXwJ?$YHfWbbd6(?9Q*MN{;n!|IvK@0*zgFri=|3p<+MVoq;y$TJ{G#sQ&%2Q(ua4 zFiyqn5AH=RA*iv9L<-(xx{T2+;a@tuT(j7d)^r)gLl<%$%Vqae>Tvqkh9WDoy^!4# zu>N*!(GTuY+Z37cN-IE0I3bio@;?ju6rv^bXxws9OjdjH!qeeXfmHS2;}M~?Z`Iok zKgwGMwSu}F^F+xdnuV(EstY8gT|FghEg6aY0d>X%V3%Qv^o~ao$f!m+@-42?-@jG| z84qp@poUU;4>~TRlh01N!f1dG^0ct|jW`EmRE(GuyPDDPf01f9{-6U1Y7Tk;o}eTx z)1wRPbo`Sv>MUi>@FN|GQcpwAoVS)(S2=;b#rK>u;Z7k{0HI7cGd(t14ECjG#(}8c zRCH?{Ej-F2dU5E`}6vT*7~|y%^<@44;sh2FvG5= zkuE^G!RwX325xd>;3#}eI-lNgfh~v~)Ko^gYluvOrrjEE* zt|M~jABmr>5&!N;fZ@TSuW=Iv##e#n z#+*wX(J2`k*wtYM`K0?|7Dck7d9A)%cyU?2h3%V%sj7;mO56l_F6YJMCbuS3UDgDr z?aU-*+imoWlFjE?!F5P?d=xMr_4@g}Di$PYw8Csmr$hq7e`^mI*gFz+&r62@=QHe`K=I&B-MOhtQUJ?FK$*B7B}t}} z%xFdiQOZ_$f9U10aCyD5635#%y7Je^R;0JgPqc>H%790x{5rJOrZ^WAA1(b+uDmPctf9y{ASoD;HS&=OxHI_e@ni= zR&hrYp(MbQ7>+|#F8*gaxxnm7d{*_IJlW4O+n)EQ^o~40(XP7vUN5fs=d({Pjn36> zDYBBW9&d7WKSZ+LslB3}noj%AP`bKi|NTS3ZwiVm@2-vg8uNI(yWRVejiwS6111q= zylkm>f+>5x=5*G2Q@s*}mNuz34bJ`$`}mCMyewV-`1+u~PMP&TQ=0oe&0=2bl)Uhf zE3=}F^WDb0k4{||^0@PH_bF+wM@%MvL~NegjV=iIkj^u3>o%?2Iu{yG^qxJ(yFU_I z!%F2562x4Hrrk*wP`L~}W??nz60q46%->t33BF~CKAUR7&5E$#eTp-%9XX=k**@lg8@Qk?Z@97ZCiqCd)9u3AbY z%k4}Y;`+5Vbk`StQ3ayR15{eedZ575P*gRXIOro5PG!r{uFJeun1iHdTa&6YlYuVD z?|2(~UDYvfWjRfhA?I16>i+x#E|y{J<$eklM^e#`#gdZdQ7luEvZUA5c$sC2L^Gdh zZOBm*w_t{MadtzGtJ!a~d(kzqXDz0|fj#KPkb9(0{w`X^xR?!Vow zNt_kay-&r*G$g(t^qvu-4p-G-h*s6Yza$1PN^SoPMD7dRGVA*Cr?m8&wvaJU(i@v< zu|6~f({4rk8l1$SuD{hi>)fpxWY}p4nulF!u}Q3c?ergkFZtW|aSY#lk&$by)=En3l{v;^QadB{J?g(W5+7+JIhYHk5u!>T6TQjemP}r## zNlVfk8%Yc*5n>ZXNW|lQ%oRzYx@w}D(&D8Flp7G1Z}Dl*^zqdV;bw97kR`oIdYPc# zL&CvO7b2r5Vz1`AZvru{V&I2l2iTh z+kC7`7~MaWlo=oaW$N@$O$?>4?cW(k>R$;{^`$CpI^nZV()ImZf!(7F86CZ}%#t_v zhQdFSurx<6^sdvWDJ;piWGCfJ@@kV_#u81{_cAlgNZ%zn-fVe!%isOEax zObVW3cr79}2+R4;X=uf}af=d{>mF*`Rreu}CG$|$A-*S;Mz8g?jb%AUcxl;{UGOc+ zfV!H*%ED8E;8nl}VOfajW6h)&<0Yer7-?FuiRR8c?m)Xu4x_5vyNNiaF>6bXU|_C-}Aipe=mCTgGO+&-m?zhr43YbDImqX zP8%%|f&7ZM>|CMtEHsjWx~9hHRTwKk{6(}Som1L`2bT=RO2O62sl&-I?RU;*DY0|g zN}Tni0RuM-cI@rc%^$YYf&eLn_LH+@(Rr-E3QZSkW&HO%Dx*!+OgTUHqhmB`-_vBp zi;!Rku$eIdbxB)Fg#yJ$Xd#Eq1eU=|i7t+ke`HWd5TY2J1pq{K} zJV+`bdk%FiA}S~40c0-mARqR8Qi$ih(Kx=R!dUp`u^H*3jDSfEDe$El zZOGbvF^y;6vxv0G8IR0~?AYg4Ii{YA45xm$`(#N|YnfCeexe|kq%?3Zrsmm_hW^{H zC9g`2%~O)FFn`Y{itv+Okv!uD%imPcti=AH`9AcBg{4?2pZYK0w|Kx%+IZ?Y zPyVY1A{O1QKZM;cUX?LFb0pHKNTW%f_H6{leejUQe}zuY?N(!ltw8L$DgY#tICLOla782rDpt=*V&G^|w)`Df(+hFxMT^ zAa)gb8okLUto9>XjFE*n)PL@h>=E*|*q212Sm7cA*jlbi=`TQCUP*US9KP@+v@TM-}za;F|(jto_*|%V?z9rFe-#h?993m|q&A|62?5n~r42MDARb zarCx-Q9ME@Ub=gNG_~g@rWE2%><h zjNI%*Af{PKtvcoCerX_#YXlB7I}3gVDZo#phaZsFc6@8;?*G)Tj71H2;5ocR$2)k$ zuW8-LW$UxTxQ`cbz0)snXVFy~hR$4jqJI*8@GT{tw$fIHbGLHMs69mDO1mP7%5sp&t3fZ3A3#0$;F9?A|Id>TIOR-4XeQ^|9x)~ zp)U0uiQM7`5Kw!|jF1NY0?I`oOTm(O;ol`7$dF2b;Cn{#bLdK8rLmUtD^T^dajJs< z)Xp1O)i~l=u)F0Pq&qy?94elx5)${n=M2M9(s5ceUFnv*yPjJT|KnKB{`0oMHt};n zHWAV&kjlemSW0))L^BCfDy|;hEt2|9;;0*K891VmhUY87TCjh&nh+hhU{NU(xL`Ix zGO6Ro?m@IeWb1fNXtwE1zK>%^lpk5WxfpxAr7-Ue;ay)$j()tP2N%TMHHnq8K zcp!ATr8d|%`s#O{z&GV%K9ATym49>KkzP6Iu#vsR2VmSdIv!?Jmw^3u=0S5lX+}M6 z=Iuo-1RFcBW6v_X?7;dCKb;XCk5k`ARo#ZGGv^_tYohmEFQi6{d)x0m75S&&%V?GW z2~B)f+8w-vjiQw+C^*B5EQ^tsdw#(w-iHvSslTy~@P6ur)8wc03q<&je4$X@ik)!y zfpT`m#vfrsch+yIndAvTPvgib9zj=H1f0z}dC`*9*{*Bs94Z$t-;&11D{5>CNew;S zvW#Fp&O<(p{PC1oQ3SSguoBHRl*+Jh9gymYO)@W_xP1 z9EL~oH`us7)Mv43V#BtI3-JJ_7DaXRE9%to>3z5&{vA1Xc;>}LuBSibB%5T)GaUpn zT3JPDtS9us-OjpO5~H`(#a!-9(Ti7*82Dlf%s2FLE@e!PbBt zB8z@!&e_--p^y|Mg~ZF=7l2{P)aaQn%q29v^d;v39k1ASy~SL3d_+=?%w4K!lcU|# zlocxmm>5EMJXU0sGwXXr{b1H0Tl&lB=`xkqk*|qfrEb)N!rLt$EJ82XeaQd-wcOk- z$)Gj`u;nHor77bhT`H5A3HXCYqyd#@y2MQKsNx3lI%Q$G|*uj3|Y zD0FGUM4DM`d$BZJu6^?&JcPr6N9e^C{Xhj9%NOBY?BxrQ*9j^W86?ifU2l=YhWksF zdcFOdv>!M4`fr;uxevSx?of&Kg_VnQMs?2$5(A*m zVl%{xqG}>y7pcCZy4;muCX%{+Jc;BLh|U$Sz*2oAJ$IA6*T?VtR&6`}Lc2wa#dJWPfp}84}~R)MXWezhqABjcP4% z;E;-oHr&tM_WyXTMD{rhth;chB>R%D+L{iB{V&xqDrd@;m>eP{wbSdT7!E7qU zc#U+e;R87_uHSEQV4OrBy?(2gkJ+c_!ZfFOEM*2CQF|xaNzw6hSJjz31ck=taRnAJ z!-o8IP<~(@sTa6>Wm-uuX~_I6Ac$8+?i}}bNsD?Fo`|l`Q>!MnwzFatI7KFNBX!ND znH8VE>VfaaeljPdqIi%#g6|u{FqA0)jAr&N>ls;7fL71XkY+|Q z&%FF3H8%hig@;SAqMeZaFfbI#5}WtY{sbKH(sk@U3F~_)gwgC}{D9Zhs1de@;DeC9 zNVES1&FQnrVST{<^Kk>yj)?I_0v<$~G?~e-r3|_a6U)a`oNF$prRZcg3-N8NHFnNM zQfLm+i~WFz9x4Ig6Vtmr>`*UGBwA}Imj+Bae0VF*G32<;#(rN?O83Vp)-e#>j?L>) zSnz^%UaF&BrE!54WHml%aeqQD6h+i9QR6LPOv{Tj$*h(hjxeU%B8J5vag>@@5#pv~ zI%lT!dF3F&UXg48TFkiOK}$@X93%-JEb2U|uiI6^Of4PhH>;5;PG{GkFk zS>_iQ3llu}t=?qjY}1yI5l1%^QT42-7^!Cpc{>UZ*foD;e;G~k&o)*a&#+}W-iK=^ zmV_VWH7X>&Ei_nb!|EQSh`r`JcrI;U#?Q*;*NjJYDmk4`!mNKc>)BR_C{k%u7@Vx; zZB~m-P0`f(!offSQ4pFmFX4fGS7A>!kYe_4Vc;m^%U4``L%=m^jLh&cter~O?GKo5#A7Ca^AH&brS9?jM(l-yD>E_y71G-Ur@X1vjyCailm41~(;kHT2OXkCY6!N%*T-U2r8f%v; zlg{i+r7w_p+vb9FG$P#;rS?^{6vL3=J|FrOKoqXviq6aB3g@^i8RiaKIX6bByoIkM zn2tp@Mgn4QIOX_NajEQC)ZXiA-PBcAHt#FXnN|>jMO_G!^i5U1^d!=sOEy$cFqLMg zV2x#;g%;rBicDa=={pm5_mxX)j{3f%lS-#Y(FW0ZF#TNHQ!4{9kfOga2VeR#Dks7c z0;X-3PM$<=wlDi-sQ1%&`iO~+?uefRp%c+FG3~^iw0#aVN6zD=^B2D@Cdd-L;wU@e z#}J1Dqq8vsYd@Ww3yd0OIt0?{P!1|J{^&;jo$Lo;%S&_8M(Mn6jfPfHJ-gRl$F&!@ zb~_I^c;xX+tvo9w@Ot-!MHeM~rJgL6^-m6X3~Eeb(cBP5*|Uj}r56=XE$@hOnRG^V?!qww|zni5x?T-O$3Aq zsM@T#J~60ANS#d1C(;h(Wcepd2bqqn-y4@goEP9NvA%Y}eP)Ed;hf#{XqC!K&24NW zI`eXO65CjmQNrbw>SS2ELcY~!rrE{_T~-@>-BdmB!|YpKhxx(bxDN{RLO+AZUV&YX z?4+26GkzM+_!^1;xW4=v*h_G}70pA5zX2R1f2Y9I&4ShOjWqE>L2~<}Uk{E4;0w|d zNa7*sJ{^^EXW^0?uR98__aS-jMF3F-0@ur}Sp)?F+GKe?CC>|OI#qJCO(gI%oC6PA znaaeq8)Ah%;eV-$Rl-Nk2|?d13hu3Ci!cH|?5(+6dZIY03#jfL9+~O)<>wey+9Ve; z+p+9Sb9cX|AN~&AOUVlOI(nA8r((#YJ~yloR%)g9_SgH0HP3A)BlF6;g zXW}q?t=yN038Fbo;U&da5EEJroi-^;e%)t)N#jkZ<12lwrUlay*-Bsu1=9R(QUf8E z=I7FBA;#|a6(P}%wE1PULjVu-U+ePg{*EKWpbLL?K?H{VEx!4WtTV3yH!)YOl^ zTogZ&D0it{lT)b7J?b__toewMcZjtfJbD_gLx8MzXkbEIgX#|bl`8lfH;c8|%?mqf znmDnNZb9NnkmPF-HmtlI4-xOw@1dlC+dLA|Q_-L}&T9S@zB*ujwC0abQm+7g{vG&9 zo-T+I8;+M{;Occosp9JQPJ1@ql|=<=$(Py1;=g3gC5AVTn5Y$j!j|}o z;(4Uw?KeSeK5ZsW{lCLs4n|K{W)uuh&?zzYTl&HooYY3rhs6F zCw(3>*Rp!DpB8+5FwUeMO-SJ@LB$vISh#%F$EGpq$Y>Wx3OkwMv@cS+x3?Ju%iwM? zC2QFf#K%9*i>yd&ovC$5XLr~I`g%eK4w069mP$5*02X+gg1z{U1c~jomX|{8_z(QV z^jv37I*4e_0rl#3!>P0XzMw?~J&I{yqi!5yZJKU_simb&Z7sv!KbY~n*whNJ=#TEr zYltnsUQER)kMOX0cj3<+< z59IXd5tXU*6uPK*R_PNVCmpJVoQ~o}H{FABWSUZ4t}Nf+^NhgqkRUVVd{A==WQu2yaJ8-i?yuJv!b?DWYg+U2WHp%o&`N+%A9cZEl?1YXT&4eKSx4=$~@oJ9)wMea&WL z{pcm|*V#AVz#u4IRC%5LT8>kUQ4fu=Va&y3I|lQ4B`7Yug3DfK`Zx?{;M<7x2SZ|= z(6nanyMcAHxgO^|&S)+sZ>-0aF6>HCVg@FoX05PZQvISiJ_&`smK-XXw=kqCbYc1OrC8 zd_>zF)rtk@s_aX9%y!TN+!)RlQqRKdK0FPMN*Nsmck`@wtobX4auO20{!_m75-*jQ zEv(!<;*3c=bip^h)paM!UJ$a%6J5Vb=FaCd;X9H8^bR&9~;GZ8KJ3cW_FKP$PO8 zd;{MzH&0wGJ&%=?!}81-4}{hVEEYd2RhhK`1gh-rvUu*S|LkL@7o_?sJ>cCwqa%0H z%*i;mdtyxzw8hMHzk<3uiqDd(R*cJY``{KY|H5i`^1QL(s^VJ=jW7`R;-x`y-AA?B*4_t~m$eUiiLoz#=_YfZ7aFoG-+O$+m1H`zLHZjY~p=5J!*r*Gyz- zz0sSThU1Q2+USKO2t{V*Rii@Z^Mr-bfUQ7-I0xFPf|V)x#-B%@=ok=u3O0o$sMZ=H ze>#0GXIe*HtZ|_~L0w*~(Uvo^(wK$tFA-zsGjun1W~w)pOj*35xD{)3OaYLu(8TWQ%P3B75F zJ*q;MJA?B#e!)^EZh$IEt=>k~A=$5DU*7-X6GzRw1K;+j4m)R`ivQ6TpipL2{cojc ztrsLnghMH7*X>3X82P6;G}`w~);Uix=f7TIw|g2hG4gLTc&|RV;*C->H94?7E9W5o zd+ReclFds!91LE~00L0_*hiJ7 zsEhZ4<4Tf{p90N@U!S3wgR~qBX?tTh@x9lY<(so_-p;^cGgfi+Xu3V5R9tt^L4N0t zP(w3vrTtT)Ye7d?Li8f-YPGv#Ue1nrFoef1N6_nO{zAkH%EO)3kfW3uXE^E)PDPn~ z{ta~erGQS&cWkl*#!^P9urUnx;;}wxVdLr8QqW7SA4IT4tU9-B9j7B%4q{-&1E(@u zyykB4`W`nYaRyE|FwxITrOTMT#&Y0p7uM3>qfZ48&YieQrovBJus$?n%M)2K>f^P? zAvU)t%HK{k4z_&8@?_TQ9BDjp!#Kdg`;Utz9wK}1pu}3^WZIG}EF7K|spqn4PZJ+6 zoIj@HIs2=?mJPB{QMO%8{K!-srKWzGkMpVq&Lj3B=ASr>a&K>^sQKa*EbBRcFXIdR z&Mmy5AF=HF8yvhd$uE;SPVn*Sv;Hb4v6Y)0&SY!D{OW|KiS#O zeq644U%8sAaD-iu#-(w!+h*JFvLWkp7!v2-&xA7_LAE!l0iMf$3HruHsNKn526L;FiqWCz|S^s-OF+4_hwF3Qn@Z7Jgm(O)cygc zJwHp(9#a;qmih521UM24)QkNf>3l{zgsYS_GG*giOR3N%lVums`0Ir1^>zTZwoV(} zgyQbg#KH&%Q=+<)HsUJlRK_f80yBWc8BHP5igZS?Z{lj_&0F6;Z|I$W=`(75UG3nk z%)0HXuGa0os@U$NvSg^``1zyG-F>>3TzomF&Jvfn0Z30d5!nyW#0P{60}jgLd9C)o z5Wf9Ac_?-9f;+zXAtZIAaH|hYuI~MhvDZ$ERo_eHH_K;-i^Ra3Y4BHS9bn8rXTAyP z&y;yAcOC_N(%qS=pHU-9J)GD^FQYujH*k3fm694s?|=>EDHSuT_*hiCf8$M#UvvFU z4#yr#<;6*LC_0rc{{Ti@hRBOl;qogC3Pk?-9{YWWGD@)zkWVVRdiThfE}z&|y+5h1 zMoC7bn@m9nnACoB;ug1BSonTVyLsWOj$0Aq%`Rb0E^(j1Drv9# z&S`z%%9C9f|9rtchuEgWw)ZsYPX-ebZnbXH_1_XnKRld(u~>MP+TV@jt`hcWEEe9% zXGl9u>$vO)7e{IH{`8A;eXFYZY0?$udmL4Mi~CT)iP|+D9^26up3L{odEh80X0rXd zLeul|3UNqK7agX~_P*!?3)wevb;^~#g;?3P=qQr~F z%DDt_>ZX@pcwwn;K<#+X#mu$TXl)cC={DZ)hH_I={OZvD7B3i|^@ds5Lj<#D^0+*Y zY~%%82VrL)0^i4ko$1uQp7~=D?)xI!r|GlEl>-yVXJxunZ1m{8Zb!y1OHN-=t0@(M zjz#+4z1+)p#M&0%)Cn=V7avTC(hb{x?3~?`^@{riRcV^MKzkWoLGj#Tm!kh_oZUFB zAE((0U3Tez;wv0zuh}BGB5lm`d>xZFnp=|0s_Wbcd0R2SPLq~>Tth+$h`tG>R0(wl zHBxU^k2~q)HI_y5<2Nr?IIVK&BB=98PC>3pARjtn=BJTae@+WBjOu!6#z+O5LLzu$ z2(E}&*gE{Km9?|^1n~6YuqwxN1|nxBX_mu|~E{VI^D(JZn7v$gX+@9Gd{jZq;ESt4|~ z8PPNOX%Euugba29q%DnBULaGyh=!h4*2@n$h;!HMuSM$idQJuZp@GMA5vOD;(iKNv zB5^8-r9^RP*YV%szX9V`-g)mdCr;Oh4iU4O0_6TG~0?~X@!;J6|1+?OT-G@YN| zNBX1@VIBFj&LL5O*O_O@mP5S2-`~_HLG^-HK+)82mV>S=C0KXwdII+Slc`mJORCTT zNwXcvPTO9P&I|Mgsh`%Ru)1IMW_I_MLf>mFc*RIy!7HYOQ;|a=i%<0|so@f>!jonJ z@v6yohBk$dYC)Z29YmYa7oK0;Fs-DbkL;qWMEEU()}7va#ypGM3zA<#*f;RhS@w-+ zGI8507f$7thoGwkNk9#@<2%WZWj8;@NjuH94SOWSPt`$JSeAMkV7xRb=X_kl zsdLvOm*rf8Hitp^Uhn&MPm2udag%~UVge~21|5l7<`2>UtHOdceL}C{3HLxh=GC5W zw37PpL73VJV8okrhlL6MZRoO8*OrkGg99M*BX80{LXnjjdDDSA`$2GS@3qF&Boc`e zn5Y9DIf%z*Od|t)5P?{kzuh3!;&eG0KVb^~j7!f8h|89#x!if?m`VKU8PjXDgqo;P zdJ$6^g^LwC!y57T)#(hKweD~NH#R-zro`FZ(1Y7jm8dJ`Wm>ZQ1F?;28N7Ey8M(Z9 z2Rjp!I5Ya4^C+o->YHMdiYaj?D&$&}2Kc>JOkE*B>=Z8g>AGBoDVf8bfoI*^ohVi-KjGaE>V z=DZEieQBFNl_C|OqpFefCmy}^OVW|&vX{EZQtnmr@#!DURr}|&eXF!d?N}5+VTIow z@+&L=(eB_XhlJGmjlZ@b{7riGd zzW-#?U9oWy@n%JtnbM_L=yEn(N)PmBDWZut2}7V8ca)-;_TuVw%D)1_WXU-50das2 z??W;i;(vdNNP+6Ck!Tv!PO@SBh$4wiokBHDIXIjKN)IXYQQT~V%1B&!5q^UxYY9`= z51N@D#b`3h$BNor`4BavH62%q&nc2?TRo0i0^skj{5W9vJk8LCf3TZF1uznAZ%C`WU#+N$=Snbj*cBG~_Y3l{#K|CT7Z8n!$oh8q2I z)tmWGyTG6_YcupeDy&4GG(}BLI(_=GYyj{5%C{emiyt6ICtNpwu&BuIX;Ze`6FU** zm%s0CTbe^hIzBKo5|@v9pNz+cS4qw8S%3CX%#pv2nbCAgVSZy8qmoC=QVI?~k#|)n zX2lQK7tXe5RdiPM2_z^d=PL#6z{;bBJpLKgdb2hU+4g2Z^K|G*>By75ehjkxd&7d? zYsoL`_$Na4$K2fJxmb%<^Wag3xP`v@!6&WerRD-I1Zlw=?4@bi-e;C!@-?hOrEv)X<-^N<9s`lVK`C!xQ%tw!b@XhU&Tcp7miXM zWAsz_1Ky5cQ+Dtic(db@Q=!k(%JNt~$+6|t-V!DX3DF?i4kxN;TG=&VTQhBX(4ZdQ z?sJDNJREzSQPmeB>2PLT3C7*8q#M#QDf`jBEFg(OfbM8y6_mfdgPFoW&irN9Rgt=H zIrug9#h_XIa3w1-e9oJRZj`@Z9;(!}!(ky0dvggv2D^4Pec-Ks{M~k|tTL~~AJ-5W z@oPTsC5o8uG1}5IgHSFZgp&Mk1Ea7q)?vnpI6+54&*woYnX9eHmywOiKxT@*wJ6XR z2#(YUcDM}(pZ+U;RxFMm^p*sz+c`sqc7*acKPLQ=Vw6KNhFkR{c-|+stYuXG$z`oN z6PICc>4Xsp$N0R>z*Q9)gvvtv^sg~}*m4GvX?9}5TEbZxOW7deD_>{Fi|c5AOFc(A z3oQF_<*<9^ow!J&M@pqwhW7_5;#r0UgQVKwio}p@&~@>|`!_mbq?5oT4l$#l2!9VE z4o|-{rx{&4$M)pSR9(fuZ#SI(LbT_JNzCODOmusW3C48Phlf8|i>UM|tX@U`sa58z zr7Y-)YlF6ZQ*->h=8MZRAUATQxB0`vJHSNIW_Sl@>6BvOrqL$N-EPy|hngiFZ^}6? z)Ah7ObZFfmR)S&bg!o-tC7+5Jo4%J(Ph z#lR1QyT|Chs_j$~;5aKs$uMGTb0SZGj7Tefl(C*drB>12HFV@QK~K{dZ?t1x4!xCv zUn@VPIk52RnIK_KR!!5)tcjeJLhc8#Wjc9MqF%~Qe1rP`bD-Y&ImBr zufdFDVSiq@&F~8LtPL9U_Dq_6%4PD~O_JH`)To9b925@vGuBaq?yX*GUh-zeof^5% z^4?FvB(mIP-HlGUNKr=X95O}w-Ji`>-t;eke5~Ww2T_@ZP~Kj9W-`I)*Qsu1V@#6; z<3guN+i2G7XiQ#;x}PU2iZ7ItOuJu=ER>aP>7E%6t%254^4O@LnxC;24&M2-!p#%B6ca5gQNDU zxQS@(RC>Xxr;8*f#T-5 zvUjYof%*Eh|Jf5G}St^Q}LP=cq%N(bghtgu{--eAQ7YA z>C1(koP`jLyh3Zz62g0De}t1V6SjWbnOCREyeWKXlPB*|plf=-sJ%^Ai4&IF1|Ze1 zGArcy$QwQf`DODo9?k#wXc-=ql!+w$)e3OeQE7#mf0_!L}{#RE}>|;awP$b}4 zP5$ZAhqfVX-8AapT@EP8){TCM->Oq$; z{5(5bCWz|9nS?iyaHWQo<1x|pW7MyHG%TH6VwT88X`Cb2pr_15`5^smlV;@>8TB8x*ofkAKG%4nX$nei*AD6i5Cpua`xmMn!B4aF zXUm*3w&7)QTmB2_o^8EU0-%aB@@K8BIyOg#T3+g~KXKeI!v=MG3m?P8a}E3jxr96} zam$7`hwr-;=jPP9t3xRkaUARvdIW0D`}^Q*|MF&jn+A=(u4}}~?fqu4g~!W5KddN? zCHN7nyb7hh!;nWSb8&`R!|FasNrNO`G?Qp73fhrD{9WJ^DDJkFyk}(3$|P~(_%O$W zjnV$m3ElCYCqNfJAM`?wZP%X|exE(}#BCX!hmlEmc|CXCPPFcdDR*Rt)fldWL4Y^2 zu#NM6o`l^thnRcW>y$w*0n=YJ!Pw8ZXQPSc4JH9i#&a&U)+SVw^q!T5-R#4Vhat6m zk|aoh=0);vRIvnQx6v4{0rO-3pZ!1kSbv`6n44v*P2HarEHfhc#AXi{{ta9>NTUb6 zu;N3k0+eAHxwk2kcznw0J&7d*jS$@BK|(QAH^nl#J5KEVIn{w1UX*|3v$w=is-hGL z4Xo@KBF-qJCV`;VHSgCp64DE+yo0xtwnFQylo6xYT$Svkl+1>6h{9@_V>J#g;`zF) z-w)GQx8}z7(-#E4bEg(X_`QP01^IBl#fM1P(Cw6%qK6~X>s#qn3oa~!AVFu_%PoRJ zToPqi?qk_c_nFP3#9xEB6m1=u);6+VTrV2}i$R9r9`0j-!J9Q5X!Y;j-$;8}l0Q)R z*n1is%$%(+kfkRJy`{`~a!7OJI-LF0(TgS$vvwTSWc!dk-LjN~%QP$JHHMw&lo9{D znG(YNVdVtt^5}QmqB;jC0*w=~t(3z%N$k&BaZ82WA$2R(Q=>`!W1(?pLB((73Zk%4 zqdYP#|JwVWDV_UFe>C_)8Tiu^)qvn2ClSutV59Z0KfJWxZjY z3}3!D0VB{JpXQR#iBahTbu1$Wah(*7w8`i>45R7)=<&JGLxU##X6&2p=EKZInp$e+ zR&oU}3ZjipZCTBRPKGUsYHa3Uv2WO6`&6(U6c0d=?;&Yq|7=pbv-?@riP_s!0C$@?a_9`{O>Uv~K(=hry7Zc+_ z_Ipz!Usq8K&rW97xm?{Od*c_DM^-C)1$4pOiW+0Ofs%ux+t2{hLkmI2`>)*LDbq?EcGpn46ay1HuCM$)BZJLfs(On*v;=HR>eD1`` zZfo*9O;e>}d8s#BljC~fgGE?|sRm1Kp8OPlqMEF`2wtFj!ki!Cnrh7G>!RqE{L3;w zu`2QZ17tv(zlz$r*YoZ&J%Ve$H+|c9HG9b zq~N;jtp;>kSW>ce)HBFa&~p6Zj&(~y7=WOxOM?*Qhw$arYVz$1*i9mLQj2BGKgFpL zEz20H{&J3uSXzeR2GcjT1%$z36FV7})2p)jn$>P@!2G5NNj`&UO6X!?-6|7=B7BU- zb+FP`4g)V7KMC$pz_(T2oC1eUlD?CxUz}3P9*js}O zM~KQgR-me01v~-g>EPyOIy+Mm`jqj3;7^osr_do`V}~oJ$#A6{m+2J~es|$|L!y)0 zvj!j^tZdUni#4td#NLn-$Yu~WFq6&E?x|^x^*&PxXzJoyt8VVQQRwG{k*+Tag+SX| zgA}ZUZ{we0XAls3v1r<2ytHF*4tk8urbs~p8QOdxy&Tp(=E1eOTC@oD@*S~RQq4&$ z`{Oxc_7g!%iv)`fWyiU0mf$-AW~uOA#dQaa38E5r8|INKlgU!IG}V@Uk&6+Yoa1`I z#C^7F^tu|}=qIiGoz2+g3MZ}F$sFd~f^bHmuW#ty!}=LzO>@_*GW|Oza%*rEgyXGe z6^@|O^jCXU!3YanYO+Z_F%bmvU?Rh57s=QHObC)kukJ86ZMokEqV;Yx=er-qdcB0~ z1->#aw1fk3FG%Y;2Tgfp_pi@{v>XLPg(LHqG29h95_iM`sik2u58;N8>dZPAA z4YjHueqxhgW5wRDK#XI8jH`lvmws~{yIfLdG~C|CCEuZ#yfkqLc3c{H0o9DFz;D#Y zh)WxSLR)3lw}c4(YUD6IUsC~3ywLoW+h*{0=L*9wCqSBgBO)EMGhjgAN!~SKvQ`U$ zDM4HI@}eLd*`DTrB3_}$gbZkBn8Y@ubQMBD#?0=-J@K=obYXk9ho1c*V^*xi03}bb zk&zQKkFXNi$O|$;b~t@9E_4d>GqqMKNH4!t*6iJ8N7TWAm6~AHzD!EuY;7S%AF~oN z0Ke zNZ7HKmep`rZQ4M1645}U>i~p0R16eZU@TWloGjKQ zMRC|AXf+X7c8>?z1MaCE+@`k9?|q=Si}1A0%n`9&z}I4LEx_-bfyK*`;UAV#&9HcM zfR9VB(*T)V*5M)&92TkE-&YcNvKiQnP#6`U&CA;)n3{A!VQ)tHsN;*SteBrzCxNce z8fzX8fvB*;pi@qX7WYt$CL1NO;{gyT&cb*u1cwH#Wb7AI#Trku5Lx(T;maZB9UrMy#$a;IiAw^y$|_!+!o&~-J*KEq+Qcgz zmL|4032|Bh6&Pz@GX1uUsi#Jjk?T+zgiX|UwxvEey<@vVYcLUVX+@^s!115F&Uxf$f-?^PV^op8x%)A6NJs*L^6kP1qQ)RW0<1)Iux2X^Vxmo z(j!EKu~=g4Sl}XhYqm+o4D3gRqB<0BAvJj6_i6a(Zft#vZ!vPoEXH>7ljJzeQS#o{ zRbkOjM%A>L4!Gh%;2(&bjjg@}m`-w^64-AXr63P}OL&8o8rw@WEpy2Oc!X7`%2Cq+ zFuUxzE~!HrnuwH@;BsI_p<)5XYT`idoW;Mzsag~6)Oxv-enZZNRqr)aSd(E0-znHe zlsRz%bcQolwM=^1R^83`^$V5CC)H{6kS)cu#MD}1_3U8#&EBxv)Gm&Mz;{uPg|f`& zm>(-5Zx^M!#s&mtL9d&@ehVZXOm+6r0dw1}B}iTM0W4fl6J{@(Fmg@=7V|N(!{=?5 z?~r9dfR@Lpx7#Q66f%uMlHsy3-j<4rx!OzNz%cAvId+SgyPXM6nYoQTZ!#FTT0gd} zVfDeO#2NXM+O2B6qqq&y|GWADsVe{#GT{CpM0BnQ-RlTR=q&4q6M2=z-&-{ zyJ{6+O45WBAnoL~pnA8dxl8KsE|>!4mwpMnCXG$uO|f1e+^LX(Q?mxhR|RM_44|67d6R9H%N(3e zMWo9lTD`0y>r_H3S(Gp5(H4<}M#FKgSXoe(x!Rcft%PWTd=MOZJ0`JmsG0(vHM0f~ z7ceXE@cQv2IwA@Aj6HL0w|?(Yge%IYH^x~{+UpbmRT`fH9dhwol!`56XdAY$scJfL z9o?7Jr2t!F`j;i;0$37^s#UZJiRXnjE#4<<^wWewX%Z~@V}NAax^-n8GFLko$P7uY zx|;56qY!Fx(Kn8RCDtZD!nc~hSQ8M&4W!_xuMQ|tFfIbV3&pggt`-c{?%@G}^C0^U zKcRFfykQkBjkaaVG)Y0i%f%Idl?t^%e7xy?Rt^IvQS)!QS}?p{y=XpRcq9Q4AFl|+ zaos%)Y^GBoN^A0-_8Nc=(H@VbATp9ZfJ(1`PjD32`=m+)P{b-WiU?@$q;{2fDC4|k z_y7ao```fj=06r8oA{Px`}SFfDqI4kev!(({JIy6+Jm`mZ?Dr1>emGaq$b zaNC$+-akWe)-Pn_tJR^03^ka2mmA}Y;Wq{cAF)@W-TSj|L&vCO+q{(-)g~=uMmM3< zTmZ^U1x3F>j!Row@|V{jD=xlU%V@%ba)rg31>hy%HR!lS+_X#-xm90Y6?v{fM<9iL z-qk?8xj@PgS515dQ8auGMWA;i2TP&!l!!)Yju#d<Y0JgVmB-h0Ul=K1Mlpb_t<9O`Ew@j!^4 zOLqD&xkHyt(>H!-;3nfao#tTxZNQ9&IWqEu*sT>44%0i(Y}LrH5=zeLBGYJLjNl_V zh^uY7$K49FIGIKD`p)4Ujz`;asAVlF!t7`xq1L79t zlRim>g~qjhk*RJZD7l75K`&JP;EDb4(tWi zATc^>nZ(W}b_Pds-wJS$2FrUxCx&M5#8FBSnhMTF`XGcma$c@y$KXNp?Rn5$pdDU_|L(*f^?I+x$?@Ey+}9TXNN59 zf$t|hTsI*X3At<{2n(`hv%Sd6COQ$$`J9l`hqELFI|+f=FnVw4uDU-}IlrKmmCqa^ zX@@IQOgcg|KPfO|VepQC%OkoeZGQ2{D1Bk9jw#rjO(;I`<%+yqS0zikGbNR&ff~|X z%aTy-yMgdKfWi@St@A8@%Hbj$U-}G_@Nt`$rb=RR`8i;_vbZB4T8bV4b;Z zlG2U5syd1Uh5?gJ4WhPCV=xSpWPvw@60@*8TZc|PBowP_Nv=heMvbN;H5wb*D*3^b zSN3q$w|61ZOiEucz>Bm?`V0)`q%{`sDt6RbnN^j7ZadG9fmQEP4e>D@K+ja^%9c0~ z1`g{WaJztrJ&g(05%{g72NkqDI4U8XrbxSuZe?i4F78YO;M=)MmfUo*aVI={pymmH z!d^IJT=v(<1gp2u;PUmO1i)^4-0uSW10$i}u$;_@)|!m!zKH=@-*G5VQMFVlu^kb! zB~dR8`xjMZ%GNP>!@8BQ))eAEn$6Y_!Io3im6C@6WPxSQxJ2o(fSkS?lo2$n`OfLy z@cBY&PRCxOl3MW~{hvMEoZ571!g(n?qZRH{gUS05UkHc}JZw~rkYx=2F1W7A+6 z)>4WBm0|Uu&C)$$;8M{O$qB^jr?A_pTSMjFC(=~ZK{5pQD`v^QyQ-N_v}9+Vi81N` z;Go6W-;0zZcg#n&2){0D()sa}NU4=^pzBWT?~A-6HF-9hal>er1<2VPtPtc?oD;r2 z5aPmt*SDouK504FWv+bYT7jjtn7xSSji<=JV39rYhmcfz%)xti>@qZ z<{|XXqO;eott1Pd^*x>rbB*!g(?(gOcs+7#g&?+_zjM>tiRN zpod^JP{LT)e=oHJjyP5hGwolk;L~buI!l%DK78~M=SEz zQmy8iCcsa8pe7l_9q2lm8)x#9p<9H?1Aw}RBN!mc1&m=RgnEG3K>bIdVY(`|zUrFO zieYR{2*}8+nE+X4a*iw>k@VJ-y3-c>v-Z%|zL29f9dDED2qKHZ!`-xKrOl_IZ_w5x zBci!%fwJ)&WO?(axU|LvD0Z*jYa4Sx8i6BFzUYb6N|Drnj*!iYaB;}A6`XSAv*tr4 zuzALmRs*>};gqd3e(-ph(>|^shTTG!vm0w(;>B|@WZ`mJ2?i0~@LBIda3nkhFENlp zo?Es7-1l3NkE%N2VS4QX#dM?<$~rwiX>H}rzNA25j|3&#gMw9ix?>m|K!EK%IJ}K0 zL0;w?0qVBIGX)AUoO<=kcC{Ic<{%&{^t;(*1u=THiIwbd(4g^Ot{8y#kBSEDMl$Tk zUL0FdIb|CM7wAT7x_0#SPXcITgghGU&8QUFS;6FT0*K@IE}CN7ce%!L$b4+tmbt?W zh8&t#C1=4?6K{iQbxB}LTkfc#0G zN9BIT2r&&6zMgB6=S2$dE0E74(2@nn7$GkXEtS$k6)ZXddqIN>h|91Z)}h044%(iC?AugO4AyG1wk4{o?!gT!b6-_eT0B{v!|D-m2ScUJT_s_ zDjM{M9l$3D)a-yRsX^=U-`BE27$a-9iy)QH%0+kgb6I5N!10dt)1hrr*pPV*iVQS) zysiRDrbJeQspoTAs~pHMVUE;sCvGdh4#N=6q#>-nRqRN?EUx36J*a@Ttu)hhh8Ps$Kg~#BM8R|>7hbh=@ z#0G-aiIG0Z&ShD&gow{baULPiAsG9({jBNl0vKQ^A9WxH9)^R-XRT+ai1Kl39&=-L z(D%LS!$JCi8c?pv*fzz0jp##VqcX2AqkgG^SiViF#!Q5(9wr{!a-W)=_Xr+dh$IABjcC?-`i{PH(RJ1GfjnJl6|ZQm6@z zlUpuuu=VMNuPP`Du3^O}yGOXT9y|x$`>6X^!<%1ZUl&ipvi`R;32GEP=S*(kJ0Br# z?h4W@8uZ%oOF?!`%c3iEsSv^2@~S>aw4JX>f)~L>zQ|p+*Gd(~;o;A5ZHp3XZ zVDU*SlTnPLMX zQ1UO=n#NM#q;8lzrzV%h-mA1;bG@lM}-cYzKpbs&AW)somUQrD$|nLAr^o8_QkXPfHuW(h-~Y4Ht= zLuhh%^AyN>Eae+^j8Te8Si~~aYV|uKXcX?OmY3vfr&|RmMqRVKgKSr;I#@16bFVsq zy%A}{uR3&;fn+C(HE(>cjPg99ywF767=?O9eUz&vJH6Ygf5n-!+oMPcheX z7l=~L$1DgE$Tx!vMLy~`hak#$i31-}g7A9CB+=m?N1;s(w5>pJxLRqSb7C}LMuW_K z=N|KoB$+m{$F7?p?=JP+DvstF4f#)G*q~DOWf#s&1`?r8Sm8E}vkkV_T?84gz5S^wVJhdK={LB&eOy8c{rApC4#?<>!(6Hi83J6XBGJgZR;1Y?Il;&Cpu$@ zOK9q=i&D0ZGYSdEy&;o)1?VbrhYXogIYEk}_5#$Lw`n6&J&`nKH-$hdRa&u|mst&( zawj6zR1kiGxvQdFaK5Ng=0|a1CHV}2h1aYZ#f_^mZiGs`+%5Z+y;J$BT#M#yEH$9~ zq7BZfBA9&y+K!xZ1)5q4^bD({SloB+$<|CTEF{nmc6kI?RGrOI#1@G&8CSUk z5dF_*zPpyfz*RG$AXZ@(S}4wuDfZ#rM7iitY6d8wVDIFkE zJ1A%&DlBXy7GDx)rt!2+TCEqB!?~{tJl}a?2BEzuVg)Y!c=Kq%v|0*qUqBK?CMp2n zhq^MzJ;j4_ffwE-Yl>6}0zgZOG6q&RqneXu#YBysdZZq+JcTMZWbi@?mf|}LF0Ou9 zQQs8YAhi+BqYY;nHG*#xBJ6aaRryl~2wj?H@}^@NU$DB&1ny6^yN|87x%5uFrBE?R zFu!=zzkEbreen@}^C<6RSWMJ-LHKy^M&ApHXFh66AaVKJMH&^a9Yy!@Bw9ep?6wch z=~;&GrTGOA!Iac*8?(4Os~Ts6RO4$3PYrm|><>ep8bM6FQ!yx$Q@Ir{j+fRc?VP7Y zRGNg<(9u`KiX(BTHJIBudr^Zu44%<=?L%A8rFp07=Ka%$w};TzMnD~Gy#pbZKVwK#fk?q zh>+uv=?JLnGA00>y~K+tQrpPtg{4@pcI5Tm+NVkVMI2inLSEkaIn$c*rBaI0$3F&I z_5%QMCx||$^hHYtHD(r zeV*b=wxCU>cpNUwSzwhTZEpj>J`h*JLJ5B1G`68Tsnja9vUj#!mXcA`EF`Fn5VX%W z%v7(GJ423-F||{(D;mWN zJ6r_U70OyUqinDHXbO7!!PZrGWy0fq={xEV&SG^SwBzJm*i!QfKxrJ@e2TB{c32T( zq03#dgRVF_%V(NLOl-m_viev+aU{J)z_Tkn%da$!kLxB3in-Xx zx(E#bh@dsMVzI=PArL&CioHr&h3rDNDWy-)V+OlCQf={ZP2h1T>QpX5G>N$2GVIL( zw8I4Ns=I2!za~fk!;DI#_ z!RbqvZ*_&Tp(thCW<6uVFCIr;_HM|;Y#l+qGU4ky+>FN!k6Op1D+ zS{XBq`89Py+Mm6!F@l9-d1n%=r; z)a$ZWH|1O!wzbxLd#gC}L-CP%%@Z2AQwKAqiq-@ew}N!UpupXJSX%}PJ*5~OeRu@- z+6kb>OKNZ>l-A4*lrCBFjH!HZmT%oB>Iu7FYguT`Sj3h41`4Z?g5jU0V`Vy1dn)hU z6bdKRlKu9qAog$=} zUStCzoq`FT@wtVE72#(vK`WkK?CN##+5I^r;?>Tq8#-l@g9ZwTP@Nb)eWj7^Pg4@X zy#bze)PBcz&w+(;u9h+v5pJv5tRX!Bs1EQ`@YIcN_< zg%E{KtL9Oe0xPZM@oE^FH5m8V;kR&#bKzKGjPg4VA{x+Bn!5XzoqBG9fS z==A5)DJ^QVik^cqCadSl^_Zo3Chmn+uSaGQ&ld@0Kd81w=249a5JgU4f5VIb83FsW0Eq}UeJzPD6cNh%ot<6GTRpN#(dNbxR)CSD3 z5R8_#L9BQoASzKb%Dhmh$f(n=aVhs~!oW==@bp_qXom3Z2$F>sWLxVHwx0&k^ltK) zs40MAq+b_YB8k>zK!(j4xiG$hxiK)?z88YcC!eDCI%*p}Vmqq$ zxdu39w_1GULcw z+3C-SAH@4{b=_L<|+o|eCY_m4u0JwTuFx6)@dz>{xdZ>Fb7*{A|LTsd(t{DC`5zuucnW;-u$c5cFay;LzkYbl3>A$Tu7PkD>ymwt7raEWr$HN7JPNlw4 zY5GnEIR*DTU~4FKjKU=NCKKIZ19o#J9A>=Fc@)+;_r^n$vCXTcn{sc#2As!)XO_V3 zgnMueugxlI7`%ejDY)TTD(tF+$CNc(2jF~rq~^ssyk zLt(WT%n>0b#jrw;W6;}X<{T4wH#Xfj=mr;9NKHuf5OAK^7?*`EvjDZJ#qcK0)gAVr zRXNZHhBVkAnGQXT>CMBOQW;=4j)b^jw2fDEkUn`eKV=x><`5EB8U2=mfXS9r?OB$Y z(P|XY;4V=JY9=N{FJGSr07#EWIKyoh`PNi8nciKXChWaA#9u2aX`jT>XJ(j1 z+A_V$pt17mRoEf;aYTfw*VU#t9y$dxeI*VIhU4buYs`wVN}A-O`F41y53QALOFKT+ zN=k9UNMNYK0SIfCAWD=cgeVAoXH?HzPnmGF}+hqP!Xo6|%(I*`wb4SX+w&QE2)hgJi!;HLtwZHM^1Zh|dIg zLD74poW2>giH!6x4^;TZpI)6Uf=qyr#nc7T=fXgIB~pXWQ@Lk&Zbu!0y@L)I3Oe&n zedq>>{ZfH0Yu+GNZf~OWC$3*BzV#H`wK-u6G6(@mOTMevAv%ZmaDc^+cZr?2bi(=v z$?0XiBmrUNCjsveHqwqEzV(c`G#`@#OYNt&v`eCN1!?yC<^u4b0U7K1Pqf^D)PCNGL2gSPBa;! zp_mQNmy56&15|!e8B^6)2apQIFJXMH`ezi* z;dD4!$zBqjF3wg^>(nl5AFFQ6ZMZg07fCG8!I~NwnMs31gI9EncU=S$v4I&1qRB}l z(}dOU*W1lmo}jx*z1XIsJ+#-dr@|nd8GzxM5#v|&l%1=mCVI6wbS95yq?4%n^OIVb zWW~<1HfF3#B=kH+g0wT(sD1#Qc$1Ow4xe8%qs`r++=FhtuehaLDB6`&R4c$xBB51P z;11-wl=8t;?&?I~$wxT6CmWe>#ZN2BvPHE;c=8~bG5t=4+FCTK2kkrIQ>SIcWiUHo zLB_CjB7HjTMHQMMg~R|?o@-Z4VdKkhEQbqMt(8uJk*}hyN_gwg(7--Bt8QNw>bBT3 zVx9{D50Yqt(VYR-`0(cwQwatLjFOaqfifqYR8nrd1x@JsxKdNOyzEnNh?io_DrkYe zZ&ct6VhAg>AV66HkH9-Ca=nlkJ`5-mCp7~FM*tN{uqxj#lSM$f;lZJFNq99adu;m* zuj9*yVE~0zj(du4TmUFva2HNKMb)D6zVv&5yX5Rt8%`TY$XVS=_b@iSw4_;qYe=uO zad&0LFfd1))H$8Q%e7Uz$bQ`Qj_8LLZd^Jjk%)+M5CK+gOa-gLHmstvJo0sWtI|zH z3=NILH$D}6lh$!JU%F=%Jo5q30oH=&!<2+&463{m;d>&Baw8?lbsfRJV57^~KzXjN zKDY-!@+NzSx>@VoE0qv;a*KTzer|4pB2{s#8WpKRq!ulJtX9}XwlYkq6?_5#M}12~ zr}sai&B-2zf{y{AFoUz0Kw@LGY}v74{zY{Q70YQ5f;~~B#Uq4RXY6DsHaqM&O2>=_ z*qp{|Ue|8p^vo}4oEJ9Vx2Yha{+abkEjvMn7ebD#|SUTEYfLpG`|I3ys84=2&%q= z)(Q7jcpOD#swWhv_`29 z5?ZJv(p>hCIjCAz=g{EED3qF3{edRQw%suilm zX8HtGMUf!jK?FvYeFO|9<#9%f_fn3P4PZDMjnasq_$gvjyOz=}o6iblL4GT==Q41A%ZWSfcI3xzA30h6FI^Su3YTHrFFc9<2;@&3(hx56Ok&)4LytA$+&YipfVdwdDKI7Q~A- zXFi&-fnFiOVIO!-)r|s7FNat$O9odN?*6e&>MK7so8#4-^GAk-gQugs@R;YWm3JM=OkkO3`)5&5D&W)n6KyV*MMZ!xM(Duwj=$Om3zHljF zWN4pK>j^84%LhrvPEf(l2(w}Lwk}#dwsbn3F6Iu%g$$43ZifRVDoX2#o5Gtmo9Pdj zdk6>E?sFA0&LBWE&v;-|;YWv>whenw9YhKyKN)z!A9-Ewm~y$x;hiSV!+OO(Zt>7C z@VG>LxHe`+cp$?igQ%cRV1hJ3VtPRQByEPUQ!|pQWtLS2A(A;bWhc)kWdWFx;c!tH z!e12y_&2W&8h%^TEjSbB<0p-?Me)^DZA#j~2dysXhHuTu3DP)88WC-1_QWI59S6xn zzH&PkT5Sv6lRz zITO<*2C)JIL-~)E3gDDDV^vByFklB4;gb!e3vj3M*pd|!nYh03eRsCgog4}eK`NdU z>wZbbC=%Mny}(woMIPEMv4;s-c5XNy_+d7nDdCrA3l{TsLM;J^UO>@ zt2^w2GVz|)6KzS@n{T;ZmV;^_hP&X6w%SQl&pacIkq^=)byK-4q7cf)B7E+bp@m{; zC>inA6L+2rz6jN#eY#FOTqqt;_=S}lXDYJ93Jvm}oVPwF9D)OzIQa&jw~yi*w+*1d z?F@>q2D{}=wU#f zWu4b-?!~5J{V3ZSRB-F`NPCPaYgiy!1s`~51&q#zB#S|eZ2{~yZ<0~E5R^9|SIOQO z*KklILHo<2EEKLF{mbDHPrIw#?BX2*Q`+H6R7??35vv0PU?6eNT%z~T=rl;334vB8 zCD9q+m5xe@{IMAdH7PAnuYU0RifFfmYPqC%JJm4WN& zRlrb|2(fbluofDsWY!=%S0~4}&N|hs&_>&l#Ho!Ip@)ma<2a?o!ON2lpL6eLMJQZL zxdY2^TS}ioTN9cHdMa1B>;{v zFF0l1@_T7o`wyg`AT6;1)sD%`3J3`K4DC|<=wQJG5kcL5Egr4aZ{YM1N)@F{U6^c@ zcX~*Z#zS%2y^kg)9LWj95HweT*`^mSuo^gkuRL9mN+@2E=81H;^)>=N7}B{@#eX-Qh3xxj1Q={>woQWtOT_r} zcp^Rh?V&* z3@hgDYqCZ=C$m4TSG(QbhfpFAPXel30N5YZ#vxZx%CWid5UDJz&d@%pq1*-|t)SpF z>)rzhA`rFu*&2noG%g=?G?v)*Fe`~G3=^u%!EEb;S^WGnQ(8JBK2oSh0NEx*G>z}D z+@ojX>Z)`-S{NF`q|;)9I^$d6g)jp;gK?Hav7__rvR(dIC%}71F4g zArB3Zj9pv~>Yd;bu6eHPoU4Lu7E-kfHO+^4Ztlb@X|aHEvfKE!9Rl3EH&t3IC+LSd z@>~wkb{y0$uR**K@RJZ2_)7?oc2E_<;}?fBX%6c=3uY2%bvxcl=dNK~4f%V^?3jX) z)R;M0MDjKLtRNgyMIfp}8nF!yj?a#-y*OHPhlPl9L@t;JsoAVX85ZF+HCPlj2lJE0!6Yr@l|vQjsZi5 zx40ADM6u-QW(Q8xhn-a?5EO*d!@n~S#6p{;pcM;`YaZVz)4aUtnAE%GIzzBHEAK*y zcxTw(e3x*de7`X>WywqV6ZV|>w_*+l`mcK@0i>I$B6@F&)}cCizKXM5`Ee? zaQbi11ojia>m7B0<&P>b^=R$N1`hAdS?VL~xO26RWGy;eURw4re9BS?C>d!BnrP%j zzcHTDUWr-xiADNYPRc#2(}=7LlZ!F-$adQ)Swoy}Zh7xh**#w$kFNuRHX7K(5wn<7 z#3K1}-2q>TSXKfR*awJYx)?R861h_Kz;MZ~*M5LxLdXcJ*=^QDUOJ;yyRRd_0)=F0 zg-AgFY#x|wm{*olUIaz47h&+bl z^|=m{!42rP`_7me22WKUS&k|oUQQa{2rI=r)!75XF6j38-J-F#W&$&o2(LtT)#>^Q ziA+=lBY3f?`*kM#+MqcuJ&D`vJknclpzFQ9Zp^XRNK8QKowzJacjWCQG*vZ&><(N7 z4Du<|W0g?gK3wllJ& z;n8-GdCai#wn(QePtv}T?N}JG;IO@-@RxkmIY&^uc#ZKve=O3|qlu_D%}yUYTH!AH zo*oRGF<8kCjMk;x?}hsH0vohZkslDiD@eJ7JL{0?J{`s*)Cjyp<^9C6UMd#5s?}+u zaldx@4+ z8EM?y;-)^zc?jB@;e!SE!`3used7VGeso*LGzTlOAlm9vpoSyG{xN3$HRTpmFi(mU zo>?moQ0Z3o3e79wm=MV8s@f1A(y3;>rm&a67keIJ?sokuxblJ*y2dwUrE1!KRvtCG zVe#+^QN{-rz507~aQjJ;pFJ_rK9xi@>h8jm zd&4t3Q5yA9MXqFE!hs(Mddg!xE1@LaSY>+Z=&ex!6uKyXR-1O4vzi#tHJ%Mp624!a zT3?w8xDjU8=*DYG31#OB*1(qHz9k#aIDRW)yg#|9!(rTkUO_U>n*&fWae&_BJ^|ik zKqNBKRW#aq}vOsQlKpW|KarHXG4 zIqDJG9g*=Bhzc(%`yri_1+M}NWrEmIC@B!plgbk7Ri+_|qSJ+3C>a#_INggxCyh=Q z))9oYxQ$5gLcj!$bX$@Wnn0;bh<5Ip!|)(Cos&>Q$I));3EH;uwDQ4rXFVo^V#P3x zki}%{GJrxIQEA85vkP`LUOE>?Uf`)*Kzt$~9a#avK{VEOZ=OWOQ82pQfz%~eB6Ze< zlnE!c*BE+bpdQH)f=M;lS;8a4wpF^nUiXrXj^%h|;I1irOLTGNBdry}@T;8NjmLJa zE-GQ_^QTy3uQD~3y*Ib!TQ6Dwk4Vy}rgjVTF6dYRErR7RY_9EY!ksKghX^&;6C)LF zBizCOfe=#Swjg34h!+rXvA$j$!vN4h*nyD*VAkPGMPT&fp!=T`!V74!zLVTXtEwq; z3Yr>QCt0+eJH@XE9uUJ+Uq`7kIeQ}-XJ!%fAjC9j4>6E-_z>yxC7M7*`DI4J+RTAB zsV*MHDI_P)KR_VH4H&hgf(FH)Gf`Lwq$@ge4foW2q*(VN8OWtT07%G##)N5_+g-Az z%+vsi-HTjEwp{e^lVnMC-=?l(E*Z-7%}&U>&m2rQdLlc}EE;8IA$&Y8Y{Otmt&5vK%fY7 zKpK5cR;nR%kdY2y294!+ZLm{msR`&wc#gasx+qt$Rc}4R|sghE&8!qa899NVgbm-$#nXSri{onsiY1!Z9WxNn?8PJU+*!v zOxt_K4>U`!xVgOvGbjK*a7V6|O^>;0^W>mn1R{S%e$v zmbb}$_NC`)W{X6N=MfTgeiMR(4xz&^`Gr7baO#cx*blHfc@jF)j|-RigasN;1}Q|6jX-Pjyy#%rpie(ij3`!rZW7Kb`__)5T)HNcv4dv zqMP44RfNJA^sW@Qz{C}nN)yq2E}*y$)8sMjS83Lycq!YI93{FV!*IfnORom-XN@)? zpWfD(?^nJ{X!FKtMcNZ-3f2TV;uvOe7}ew@vh>99c=fwhHW!+KHm3!G?mm4BA~iPL zy9M+zjHbCn0S;R}C{h7>@7Hdep~hxH@ab}eN0m-tmTOcuEN$j|kCDM2wIC|+X)@ut zI{e9)hGm^V!oI87r@Ud7rfRf%1(|HNLQOY@!@|Iz)J7FRIHF#LHXc?<3((LKInb>- z7nO0%h@(>ij}QsnhpSkr3h4Q91X-jPXpf@-F`1HCy~Q32I)GT1@$boIrP0+VDgeu= z@Y!y$AskOxnb~9m#z}L*=DW6V)kz1iFCo|x8###8CsM@cH@4~!y+=3E? zF>ZvT%1V-orE(v-?LjWSf-9s4@nqq@!{!KwR))0QOo>L5Cp7Ac*eb_v+H`b;rP^en zNKuMa9f~TSlZo!?ne*5WQ2d`Y#I@O|)o_rM(u zn(>Ylq#X@0V)&>BUSj%s#LlDkurO)V6J5icMji5j;7{XLLF!GOizSnbAkt41K1VB8 zB3TM{`0YCvFBvOVXM|ud>U83~I^km|v#`T;z6PF)UIecr2&y&Mf-*iKybTR8bUim} zuMi{Q?PZ}@O?Wo@&}S)Yf=EFS;>nTQhuA@UUpTRhn;p8~Or6~rh@E?Vw^ch!@OIh0 z5J-SJk{JNz%7-9<3&e*MVX#8O=5@h}n8-on(h-z|AwO?BN?3>pJ?dhKS{rne9N2NQ z+W6GVt0nl>yX*>3E~9C43AY(&n;$}y0S5M`imHt%2L^V~4sHcwl<#hLJctgM*2~sP zuLxKTAZxkcLIf>?n1~btZEzvQZmGp0eUkg-yIZZXHB{)=8e))aoH7wQYb`Def-sMe zY#JW&QUNhwK?Ve;6!4jVgfKxsehUym#Z^c$0O14Hbgk^*Tq^GsGJ7jx%){8OCKs|1 zE{P23YZO=sezr*&5W}!y!KV_uV+6wY6E?i?Lj%*$oLpN(YEP?g#_{kV3wazMlIPtnW*^iXIU z-pO+!Kp$==uOn>@_pZnUYy>`$5#d`R7ZAae45O78F61a?>O)AMov238_}s=rEM7@) zLhdb|8qugUdyv4#!XF@j)EWo^kH^WPhU97?hAFac4?5P?Sw(7nL@aGtd_lk<1+ifw-5b=}oMphfxD=L1UD%fgQ!c^b2@k-tX2G6*4p2uO&7 zBVl?NEG&N6R9eU$z0$ur>odt0UCT$+g_H?-2}{pro`h{y;Y}#_fyrVju*j`D`Fmi` z9A*|IWOnddK*f*?heV|fM)!2t<(Qt#y@Pl+!RtB7S1KGWZ*(wW7ZK$8fqsj{wM8DL zf|(tJ1q7q|jgAk)e?=%2%`q7JB{#~oZ79WLoPm%~r9#RXWnRVo=%g$*eDW{C_kqct6iUoOE@g_~J2RfmONl`?B zbw*oyFXj&DMOhS;6d)n^Kwz*zq_bqf07NLvx~va9V-HBEJ;wrXMm{V8@Q4^P4!Exe zH?d}z2K&`r69B&Z1S-U@wP&T+V}+=@hz%MqrR~;dtK+X!&9~mrOKj*7qCTTkEBic= z5N880QzuI;E?!w1zN&hFV0HgexH`io3H8aIW|xViDU#r#Y(2%8(|U@G*W6 zwIRYuKZ_?K&*oz+|4>>tY=$sMv*&IeSf{w`pWIvMe~86L*%AKwyPqxCQMstV4{4#6a(0fg(2$ zko@8mkxDDC-v#*m&O5&+2G16wS39K0C+My_m1o(-u8{4}=Pts8G}H56rIhiLi@IxZ z)?!k+arZb<*@%yQ9AglJbnAT&JXO`SQU!*g1uo@&DKwM^#Yx%XDzpNLrB_DLMU=l- zM@PqPam_^W%E@iJqd-mRMS#nuZV~A11_6vUkQ-blYez*-1Eh3uGikTRDKntm!*yYU(L zed_EWbiZ3Mk+XA-)~(W-utlBtuDSfnsgqzdJn8JqaA?!JAb|r|<4PRoHoMsE-M&`9 zmkjo~yM#FdVz6jU;TkN3+}F$9Z2cfD(M_w!yFuwg=0T$UF)FN`@`kH6jh_&Nh~~YL z{UQb+YbLJeZi459VX(>Dps$$0P<6B1HThWj;{iCH$mVLD9iGgfPOIm1?8|SRAWOzQ54-= z15E_OuR&wSlbvER%9SN%gUa!El<+SFgor2B#ED_W5j57Casg0)K1UvKV?lypO{kPq+{`d)BWrk_{+A$6>H4w{B1vQv-q}!sbXzF$PWd zRlciu2y{9Tn7wf{fY(a6>FUAYpjwKw&vFW~`sAVcSXF@X7SiF#-y6Nx3*AfL3k?c3vn-L^y+B;&lv8za5XW+q@I{-o2-68m z>2;acoz_CbvXFr_?33J-kZ%r8&qN<8fV3Z3lyFH=#DVLLPh7buqDQ*-4qI0dpw~Lm zU&uMsswa11C2ocry74&9m&LV1Alu|U`dMR)jG50zlPcyE*f^ocvX3L#Du^ZxVW&6_Ja3rg(?atc(t zlx$L}qx;pMcVUwE#HUM&Qv&N~w%o`S#&B7d+4dsB=~ejFWzS#2pwu5g8f9iiWJP3U zc68FeB0?jZ8d_9t(ec6;u0GPBce+<78`p;ucd(4ySbw~QL`^Azu)#(JqjMmT11XPe zzXOUJVDu9Q3xlT&gURHlYZT$MV(wx-=TB|xE73cx!&pHWLOdrBGhyHxY*NW$GzYO* zm;@_=I@Vn7sofrYfPJaKTn^M8O9z=vbwbi(62C@PiPlOjW!oUx>wP!OL*<)$)xO$3 zYSu1hGvv{aOqX2*2mt&zKPZNe&17Z-DeoEsFpwbKYA==a)kO{({eJ6=W=FMJ3JaL5 z3DtE9_Tqd(U@D1pI~u)IH9w{MJhHBh2hSGesz)YorFaFF3tElR#DXJXp`F$-ykn?n zeOz(q+1uNhH)4fhn0|uTv$?=3UrQM9C=!!FO%8(eW+AG&f?CX*WE%6^3Mcy4&Gc zz}8f9io(ed7`+A2#d!En8&}0E%5^Pgfb#=A6yOW+xke`0LWPKx4>E29FjA(+!uDgK zWCR1H8C42xesw~qwZ;trJWBVYj>uJ3$ICzvDN6BK!-!;M08;W}UW+c)^_+UVB#A*j z?ivUw%D`-i8r>jd*^IS?C6hwX<|=Zl<3czf&zS=0={M50?#!i9GkPrq6~#{|;;61C zm6F190RcF#m{Hs$4wK|MAY0&uzF_$3ttQkDP!Rrct1B;8Ss{V$4O1pP`AcxHmvT+U}$1w4*ID9X2yrS-SS7x0CsyJ zTsP=5G>6D_xv!**gw=kbMe^dzA)v{pZC)(%SWr~j1p36~cee)mtz%SvQ5gWmO#|f# zLwubqD1exvSjRB0%(T63tD|h0QBF*cnO+cj5HbW!?4JB+esH8PxTp6FD}1mJ9xK$l z`l!R4zsPP7m>uQpd6}X z0c8+`M+ouKFS2vi{F!KBmDXU_ZfxcpwrHJ;qNQXhFus68z?Iyfdsyltmvk=0=<>3+ zmH$+T+0sg%Nn#~ZY3y1mE2cc?z5?vcTemfLDT-E|LpWFp$Xd74&yQ2fit?7qc(z-B z4&2QN++|Ct9|T3!ghISfjJXVA3@l;8LWA~@uB?>BNE%O@EJ6k>zNkTk ze;pu05FId=qLi^%U&Xv#tIdlYEI{lgg+PMFp=)O88b57qC5U?fw{dc4Z!5cm)ZQbb zFPX`XM^V_D-h4t$qND;IqG5TkbtqJCIcfCJSsUGn|yvd z^~|6x6yD?D)taqXyLx2}-vnvGjUj78hE8LMS<8Kg1p_r2N+6z%+$bTyc;_+d=c+r? zir^sRj%c2CYeZzT*UtJe{H*%+FO2=q0ysF+T+lw+Ny;gYns#1d@w0~sMZ*W;Q~O)y z@3>l`WMB%#M?p7$eF$+~remGz_h%EE3-U#Rx3mzRbJiIFWom)>vYVb4N!%twYSqrZ zP~4eYT<{$N#qNgBF@X!Y{5~MzrE&${48x7e1(7ijyOaW)yt?6`kV=0yA22|8dfj65 zB~Mg_uiaCiLs|=d?T-m&-zZeb7&P{p;z=5fO8rf>X|6CHc`mFh6I8NYQ?!q z&T$G88lP;`4++MLp>T89qGX>I`*c(yoZ^-9$d`L+TvP_zTA+aYr<^h?VlS?gq#=BV z;2XP%7K31M^Tfwk5|djntbo!Qk?+3tIo0QJ-BdY9r$%+|>J1W5l6g2G&nq@vo*CtPJ z&>kP&AD}=$Y`!GNflr{gaXvJ24~$TA8=yMHs9D}Pj?5blG31v6G5G9_^_7E_&czY} zO|&dL|4mFy9hC_0S^6e9I(B`r@ZtEthV0kV?biLb?%*vZFm|&B`HWdMP zdfs9q7$)9w2O|+sO=K>+%NeTny`WlGMME4A2pF$iM{^QHy%QRmG8~GnZc5%E+xhFf zGhgzo^+kMx&RsMxV>N5?axI&kgm%mN?e6+N6{P$N9p__l%wh`)%PhZ9#P?QB*$?b zxig7L4KQY!tmj0&-PbS#o;SBXe;u&NA5JKR29?_S3Z(jzZS$@uxjE}P=H5<2kJEwU z8J^;;yDgk73K$dzwUNIK>*PUVOH#bT2r#mXzrFI*aB3+P8|>1g-S)THImOQkn+H#z z+tL{2J6^zc6WBKoNMP@@{&mZ$%cAk-uNY zpV8>WIT9DtXR6V4ZN3vZ%v(D7Yf+q0Bp2i4v&}sM#%jP;AU9~B^Hm2Yt@KsAa$@{v ztCNPmQ4KdzkfktmSQ)pPno2q&#luC!c$)g?844FE(cPCS!jc$gKAapa1T=skU8Q@= zcNT2*W!d59PhRW=C7Rn|8T3weo&`^m1)lv>>jap{Zew}L)k<^uaYtw~;A4!tFNNBVhk-o%j}1|mMG_dV&sK`cV$O0A%X4tv zS0>f4&?8hyLSQ~0J{eYN65lEc^twD_$PlQDr??(bsj{?*TAL7t!LNK3#!OP9dUU=% z=y_}VwgPcOy6A%ZOT?$FU8rb}Mtojco z6W#X^AH_pzfbV+YcvyX;vU4Rp>(@{xHJGDx!a0*Cs)CsmBF9VtWW#7&^%4?knOeeB zk6TD?-4qXZ_ka_(+CuGX<|wX4oduJ9zZ{0_${60M`T;$`F@o86j2b{gl3^lmC=0X7 zFHAfGJ%H1^Nvkmsr-LLqN9TvON0M`;k2>T7@uW1I2spwC`hj2|8b`D*O#02Ta9l4d z)i1EyJmW+LuNRn8mkx%#v*6&sMFfc?QNGbcaHCM-op+0Q^2nGk8ghl7&SAn47+@;l zv#mCAcBs1H%vj@98cpF#@W2ZZi~!YvYkB7aGb}88PvQ#~mUNc`zTntg9BV^V4lgkh z3jJmbiYPV4jWMUOHjfkbjhda0tK3O8gWqQB{2%t_Rgt*8x0Hu%EYVP+V^xZ_+o_9a zqH%{c*j_4H6xSUGf%%iC+}(+#`5lETsx-j_AyjoLs;kvVki=|riCgsBWix>B*8$H> zUMvx7W2_JqFlO%`5J>Ne1Q=hmkBc=q0y;u{0#=;p1Vj$TsC}p&=?YJATV;u7kM1qh zh@%LSFtQc;Cu8Y40pWEn-WrTkvKiP4m(mU&qcajECtG_GMZL@_`fKtbBfJk^gUy3J=C;gBB! ztruJNSF>9uAhJlb_@QM@2?Y#YnYr;4z)uW3Btu&hme+pV%tzQ|f`T)y4c{H1O`(A4 zXn}|#x}jNM{c|jk9`T-B22}<7g#DX$H0rFw_4c~t4izxP+jtN(2nI9OJh-)TkE4N0 zf-%n&1>Pj@B|i9#Xn_Z~TP!YjGb!xkpd-~M?Xf53H+8eBX5C2RRUAiM%W`EgwKjnx zrogg=V*z=PKq9|&HO^Yyjzcjls)H@j(4)$5`~>`#Mq)sOdUy&RX-*N=+njNEcR*PI zVY#&q(t`3D=(^A>#M&{LIE+=!J?~t?+wC+7?J}ga@MBvqS#5dN@+ajSL{J-}?j>Xa z0~YQpKqW%(W;SLa`5RicyOrdCrnTldD$$En>o-A2H-6#OKkm=cBTz#|;YT;vIG zAkwE`<5Zk=-cb}&lg#UopA7ND0f>0a;$2Lurz9VH7a2GeOvx4Oa2BT=HVP5i4nAF=62Zwlb!^3nQmlM#uOvuO{ef?{~K==$_fa;@nMyOdC&vc7@P%*3W# z?3H_GAq|+fG6l7#TiCqnRrJTcqzoa^>BphWM^vN=HRTRTnf$7d*XGNsT;8SCG*NZy zonX3~Av+mWF`39c@z4;h>tynIMx3FFw2ZmJ&?20e7WCmeFNX-8i%8iJb)PXVFF}sH zLjC!b^jFZf?wf*kCiCLA1(&m$7#Ivd3d@3*%nTjm!>(Fn4_tlFaa5l?+ChYYRN2kaWPd=df5%F zYKe>ApUvSSHPbaAR-qb4ho3~B8jX1eTy1NfT$Yfo`!a--J-MK)8wuK}+$ zpMhqVByY3>Z2&B^_4hjKoS8mz_~>~CJ`>K8?)Lf(qCtXzx&i6EZXTf@ z#TxZ5A^+%#hpY}xgR}w<9M$i>(4p}ot8u9dZ5lK`tA1U8FH)0p?Et4Z!|Bw zU^pKMuMihl)oJvb|% zG}Mo$>|5J>R;=l&FOXZGwJ`zMV~ z{X5QY^B))=EANlagJ>bZ>EtA*YZNpYYHME)!2;WaL0JyJN*Ydl;3HQIcrly315gXk z%!Q|FfJiVA^r&C2kT~ITRhHfsC?Wy$dc0IU-N8HrLvaMWKvKcQjOAPAKE%s6zAeO* zQM$~OVUYSe>Rs~6!|Q8T9pm5-WGfkVuKQb7B^atEeWJxv71c-AF;5TZEOuwI6?8{# z0-|yxU|DFn4`SDNlCo0bs1{#Tm(@gjB{wp=v<1*o5ZdK2PguL=w1G$oKIMqrQPo;V z)cm=q@x-B25UZB>49Jwj_oTjXZ7V5byk@EVk+C*LqCn#CeviRWHrJ1@% zQv~sWPJ$7sTUgpuLrFJzqR}sgDrR|XEYH5^l&^X8VMv^Y?!+jQ6bml`36({KG8>^S zN;;jX=zHp?JlxfTVQ0}=eKC_wuG8RBsd6~xnx3&$U4~U1?9Hc332pZg={mf4TYoev zx1)1GEoymm19T+$Vb}(wjkM%8*~EEBK+6i)3B{KQH9Q+}rHP?TJ1lpjL4uN^6%jnE zY7cdqf03qKm4(hlWtyFCn<94QK*FbdkV(W7oI~S4Uyy>^F#I|=nX@U zvX^{SgX1!J9H^X29sbvbF z_y{ozwv1vX3A8q>8Cjq-deGW;5lxA!j1@SBd9^TR$U2X|R%{ycE)_2@=jZa!n=}?mQZWgzzuCW9Cd%#$4Ak^XipHU*5A$hjL<2 zRm!Ae)IylTHz?HizWkCX;dLYK?(D2C>|$#q&`}|-qh?~A2f&#YeGN;J3&D)_n$t;W zQ2}%btl^MY3x-BrHh8*8oV7D_Abzm9%D#BFTtXJGq|u2Md7LHaFg>`122oCMrMMVV zNltBTpjS^HN49(3MGvx&!8U1wBs~(csHuTW4j)zb)csnsyU2wi_lM%5D#>w!BE~l@ z5XVEiTPx7dA%?rPFq}>kt^9d<9H8)pd%&whEgN7f!gms0^Ih=AOQaA4DNis#se|10 zD}-lVA*a`?UbkY_y75-976n93l!W*qiB8Dg;X0XP;V|Z!u(Az#%XR6(Nb=t*rv7o+#{ze-V}sthP%7Y_*q6EL_?IVW=u;ft$#-Q9Bcv>sME&R5AL zZZ{5QNIbDhWUa^-vgLvAiSn!HK>dDSoP=e5`k^^6F=sHQxCB$C)vSo5K(;${Fjsne z(7Z!L12<@^UzlQhxk-lZ>F+)fvP>sSls(G2%z&;0s0(!pB{}v)8>#IZr`aPB$}lEl z^)qJmLR`w*#|NSX+YQ*=JOf!~;ugz!am-_c#kxeV3i^AITT6p0DLko748n??2GprQ zi|4rsT7dkiDov;<+QvG{0R{DV41s|~0E~#`kyFBrSgn<0x5hUzQlzaNvLxCXz={bnyb5E2$~~S9C|=A*7iLTA~1v0i_5p zD!2#%jR*@Cj~Jwr$Q)H2GH2NVRu(4KsIn1IcAvO;fx>S6B5&8NxkrNwM~>MhAY78j z>OsxyDxR&~oH*b>9#55`GaDOCbWiQA86SD z@>Llx$XSfh~#fdZ65nio{IAdC1T`)~KS}+rnH#se#+e^2VkEw;JA<^p< z$39IxUu)%$X`5#4UzJ-gB?27Tcb{4>x{RXw_j>Nb-roapg8-+ZEp@TR1Tworg2{ks z96&H3=FkdnF=xV#j+_&?i{erClIv9}qQhsU`NZawnfY(D2w_(_s~*?wU57_F)~v>n zlYN4Gsad37YR;~Oq!io>bIc0NWQ&6#@ehinpu|%XQ&8I}Yh&uiQfJ1iMDz=uv1{dI zHkMo!;`I!NQIvLo<-#6y66l*=Q#C|=!|_xB;YfWPwb7+4X}30K*I9)i*LXrw4%JfR zv$%{=H)|0(GgPd&s`cB`yS`wT9*!ZR%m}kCI1{%Q5zzg?%J1t*0lFaW(4LINhn1tZ z-rPYquCBQ*Cfyem035i3Jzo`#7`oKm%hOb$>h)xWz=0DY5ay{OA+?BJOsEJb&N-JO z2M-)5mi59|znFfzx~wRChCNmI$QmM2@vX;%%dsB@-C3DK0P4ayY`R|^4-a&e%n@K7 zXln3-1wszHrK(itKMTUmg4}X|kUhu{n*}Fi16MnTe&jl2TX?aw-+#6JzbIkM46 zK@%BKZ059ol1xH+of!dSChz*Mel0~n+&t<@{(|H_mM4>#NPWr z#p9NJn-y!Wv`MxRH|8!Le{GSCYPG2EaF>`XES~B0lT9o_koUFrCyZ9J%9MNPow5p&DG>W@z7f7X9BOMv%ZXCOj>{J7^CVNpuf-mP zAn`RWh%j(lNknMHLYTK3>3Cc(Dyqaa4xVeenOd7W&osGx>u*xrBk`Z!9p?BkOPxDz zj?zw{f3#cBK&;HyQwN=xGr4?u6Q&2(9ieuj zOKdIYH0$((^QbBYk2W?JjzgVL{ckYP2aCg*nN&j#+>DCmdVT;-1A4YhB+um_DyO+j+-#$xOl!#AO%XwrXjs`+F zKwd@RnNAHSYo>I}rQl*34t&(2pbqQCx42fBhtSBB-d$d!F8a71#IA+KuCp;C%+-Ck zs;s2m^dQCcZqZgWrDf8wZ+t1RSyG{c3dz$4(q-osS%ntz3|?hrm9Hd5oT9ChsdfMu zMLZFZPbsA`_|@o0s48giUanE)EcBku3jk20g$eYa>4!Kz7E;hz-L5olhwlL#Kw*teH9oi$KF;ARl% z2jQrs8CF*FS!A&Mp5SN>T*?VRaTXxk8*dP4Gt)+3PiuhLQ1Yfa-T zw)Tg7C4N^4h(u5gwV{_n5uF}17vfMG3xn4N7~v6KNE~KCVbLcLDx8+9Piv^u z7OvzPdrhIF(Q10={t-?fm9+@aQV2%^9I0s5$tDy^4uKN^T6Q=QXXlJuD8KSepw;9o+%r-=-t1q!kGYeO%0WD9S*t0_bdwl!ja0 zZ^*i)WXOkFAmJlCM zV4F2?=x*dFC|pQKlI^aJOV&ykSdK`42nkzYt0A%&hzJ9K=&bTOmXL1{zj9lbJ*0beEmCaE zSGBk(d)7u#watH$bj1=aC8@g9LhpGgT9mR$45+Oo1d4L`1S>$;3b>$gAo}KxzPu$1 z$B&gq>;aQy;TO~S=f|Dk=O-5UtaGm|+g{SQP7aooG)L{S7nS-9sG7^itQp&o#LVOn z93#Gwp-g8RNtF#eHKO>&cYQ>+A}Kv((wNO!)zjNX3`81aizaHzj<^m-U1H%*7RPcp zL?L9+!gRb7%gjCG(c&Orj1+i2bIJ=6LWNlqUrx!S_@w4~2Ov0hCDMhLSqK*LQTfqA z>v*}x+)Zp?CFhX?Sb`1iw|Dt4F_O!#ZNSLK4Sq!_2yvMdzjd|gDL3CwXhxKIT)gi) zk}81~Dg!0C1`$-hgf3x$h+kOMa6sc00z+A#T6(gT2Jr73{z-AE8@pLQ74sJ zP&+nwgKd|#(Y!I1`p;R1jj`=DpwWf?I`0ZCdOoib$dV}vtN+H{&Z*Eb(sZV zocFH7rtz5T)><@`ewjcZF(8S$08tBBv^@@(i@0@CV@<&u;u!hc>iwvVe?MwmgbSP+mR8Y&7S|Q)waxJ4&g*j9V@DA={2%7c~ z*ptg|UgX+AB2IOx0)YC%4$neqoQ-O8xk^;R9-|zSJuLX_3ncE>hp9`>#uscfGq#A! zteo|t`DDe?d=LtSc9wiHBW0t0Ay6CerVP!l5SI#@327dY1CdX!je@tLwUu911VZqa z2NDR(B$ZMb*cC*g`cvb*JQf=rA4V5+W3v#W@5t-OtLpICVU6g7GvSK{ev1okr(rh& z_-bE1zqW%iLXH=N2~y-*@T+6dYgDL4!h{@l`326OL4=2K_i0&49`+u|H#v zr^gxKP)CZP@u?FqWUq-H_lS79_+Q6@rVe95cTi!cDppjUOE&lkLiZh`c_Zto7Xhn@ z%;VjBZ$Naq%GnOM>7Zc>5D9aeXJ9yIBHlC#SOH<46hD(5xqe{L?onR{m)(Ir8SsEJ z2@MtTX5 z()NggsGPxQ-CuzNbB70oM3i#Zgw>e{1Y;EmF;bgj#eROirLhvc1Ws_JRTRxyaMtER zGD}Vjk<{G-+5n$18tbtb3u)!mw6pGzL0%>TprttsISy5`4-Kp+fXeYWcmv+y(-2T^ z5m?Hlf`ZpVmUAmm8HSMzg)+(bn};leiZ^jtz+T81C_!${JV%}`(R3DZuZ~AM&Bb!* z*VjFSz6VrP8d?1q=F=8dk^En#~$X?qakeQH(Cs6P$z zRqOctT7lRI)JU4$FqTKW=Oby#ow>k)4gw(@*R^sMsKMS4ze1z{hYN_(f^q}+oEsC8 z{JqlM=t|*4s^VNuBR2;|rQYFN?WeOGzYlq<&ibeeol^MBPw=={34;2zHbgQmGklFZ z0Dxv{G(SyZoei2%GSdy;S=cY7kaDDp&T)F);(mIEwb2#XOhGPFL!Pv>6jr+73AgWN zLcFrgCQ4T9d9UYP5sP!WFyRPYq$3KDpz~pZ8p)e!5&+}V&?u>cN>;kC(HWtLg8R^s zDmBF$vpsU-Q?f#J6r8fDxD2LStd;RJ@AqTfZO@(^4u%HdY+~W89`+|9Rk~|<^NH_m zu}}i2b|B}7yu``E-$;U0AKKh-H|8|y_W8l9Pp^XxB+$z`&D0lS1~`Mg&0{w_VQ?!( zCt&6{>}qN!1@Pmlz{AHf3GLY{W99H~6alXDTYJ3CDCb?3Us8oKoDm4bSPS-)9~7$% z(v-@z=^s39`n!pWSuL_Ybe_d2$;4= z(cECVUschkuWIxp8gPc?+G~xb=<^TRL1=lr4MChXVmH??r*P{SDH7Hq^rR3vu;avE zbu3kugv)$Y>Wffr{z2zl!B49!RA@(H-l#TQGw2pk9U2_)3yg~#l0u`!qstI2Jr9&%~Q$baMW-_Y2 z;+?+}Ko`0eY?5tA&Vt2M1=mC#M^1uhcM2lW0vJk+3Pcw$%7CGYMX7CzdJcoZUv=?= zjE^k)!s8>x5fq+j|Ea5d-M7tHaQ3>Snr&`ciQ1oPF>B^VJjC; zaMw@IE47o{+>06Q3xx-3 z4o7~Xfwe;{oFy$~0r=fI0E)sDU8S*Kfv3hN%samI)!d5IA!%Xfz-dyU)qxd4^Hw2m1fl=pxsTzE0*c1(t% zX75+`)h>eqT~Se~mU>-8&m`SPC71Ln!ZA-B4sEVR3T!+ESWQI}NV{BJY&^ytzI|+- z-BlXEdmI|#vZ+OhYsNmmvPCOTzAV7@><8*^K*&mCr@yro!|orta)@Uz@7jP|oaaWs%{VC0raU?HM@_?^I%cIvc@GWzluGQDS3 z+&-9cChVThS`bDLN*UiYgEDT3!P>LYBZl=WYaUfSOGb@ph2>VBSd?d=gk5jb4Z-gQ zpok9jdCnZoqVvhlA_AAw*~+h%jWDkBD=^ZHaQosZDx`G^ft4&EppN9ZT*UJsIt76K zo>cZgw!|_#*j|N_R|dEN+uLUAlw7iv|lz@ zysEfA5NO-(+JcDq{I>m+>e`ES`ANE?Sg<(c9k<>*J3L!b@uGmr9Ac^Ej)CtR?LbK{ zgbv;Iy<@yoc2HRf&>B&^3zpwel%2YWOK=%m-GDqT8A+1^IOFky^1um0@aC>6%HadT z&G|0s;;=k7VJ0K8n<7~d-Sc)%2lS$bsD-Y!PGT$LhEI=B==!0gNIHcoMqTFAM`5dmh-=E5*Ou1p2zehFs5({NO$U1qDhm<#kIB$T zqyp?lYhXuW&fcbK6rRaY^^p~0CiI$RlvakFXF0-KPtCpR_kG`&?3096H%kDbb`b4nDWe& z^-)>g+W5LK?(xzT4BC>E${8LLVFxSphwK%}Gx6vw1@R@{hV~q}?d_i`0UD(4pF;Uu z6rSa}>wwuNq*cW6^PURZ!OSe6=#agY4zWlu8z9*lczzkmsBVzGZ$hce0*-uE`aKZf zf+8g4sf*Sv=-74EB+ID=iC< z-8AwD$X$6AZt`B)CO8fAnDCecO-yx60X#c-GPdw$@J1UNBWNnN-Jw1cITxf}C|?&8 zTs0s!Ea-)sdNBEz((-7D92;U0!hw)%jYY+7(00txYf!jvz-b{Q9YOhEa62?WL%1hO?HJC5Y!`nK&#bb^1A#N+$`TZmFDE| zhXEI0!q%P|=dy!uq*V4g6&3?th*n#Tph=oDgbr_7i`e-cPA4MGNFp1mqpB&!bq$N0 z8{_15{J&~l!s~%l-H%j3)lRkCgkJ>;UC1dlA3V`(@=&oXzXuUY*4j%c6|VSAHffYR zW-!|O380M1!?>d*;^$D-O~Zz4=5d2@?aCAId>;UO9|yn-)>`@|dYja9G7Tl37=|By zFWP36NG&4j>(-r3R1lPCp#pJCC7k)M2Sk~I*t#i9?oe_`0$A4gN}Cir0*aXTR)jY{`} z$6-p6s&JsW=hX^XV{2|;?&yN{>1j_(&JGy)>{4tx7D;n`+^5$2h%aP1G_@)^{F{}I zW14wXi$N=PBE+Ob`MlCgw|eOS5-F$yArSo#ONi= z0ZtAKZufA6Nv*_&Qi*aOjjyWrj37Kp1Iy2bt~kF=>%r;)-x4@7Ea+uYp=iRH2pseS05Y&dIiP05p}2n`1v2qQ817A*YUnFS*z+Kv*`YUSM> zeJk_{o=m$?TZZJ(O#4@ujpEG=6&g#Eg8A~a(a$h7kX9k5-47T%28=uuPq;!5#-|?c zOtm(!^4w*%=DiGvTQWSbK9ggc;?r8yj%l7%u zKr3o>m19?qkkrN=;(R|DSRF%g7r=_Ixr(Z1T@T5#(6wg=GtKaHo)b*5tkET8jBmxr zftEa5Z5$zwqG9RbWVj-JggA~2rK=Hjcu^i8lJ>Qgq_qt+)q?_3QBb>rZq?^;NS_>T z`HL!?LI}J$QKEEb;bGzesUw5}03jcx1(a)g0>U2cNXdfca9Ni--fj?4uB!YK{ z#tAbVlKnw=6PWC%!=l`*9uQZ#7Xu(YX4yQVAi{i$OCjXokb8{fBpLZoqPow$@lFND zF@_ka0MO?`m~GhDM zCX47hLIX7 zF~h>Q;doo2gKIPanm!UAUYadpMu&1LRjdSWXkOh&YWuZx4=^NVwOTix#iMcppkJM& zNNhzsrI9>|OGEdWmVC^uXxj&%c4OROxA0S>;(8DrA8&3DS$$n^2vn-M1+~DRyf@ya;86p7of&X*3s@oXA&`@|x?df~g?R9B z7sfbkWlWstghN9n=^hJ;I7}~q-!MZ>R6#GA?VH1bVj(0<1-52RfHceEsUc3d8BJ6H zzZqPOfh=E$EL=gy&4L+xU}K18_Etd_bs=(`sJ$pF7f9H}f_P@Fn!xH~bIwn=DO500 zg1jtgH!9Lw_pLhBbQxo019i0{6bO_G^bKz>)`na?hYK)<;kZH;kjxp9H-jYMl9i_c zm5XWulrzXSz2=KwZ30oS`HtG}ZwP?SMfOdMO7A5F&;{k?Eqv9wCS*%V@|4ciWX#?AgFL}PQB0&M=V;ccY;_G zquJFhDCe0}v>9p;Awx4+=P6rwVbucq;DObj&D{^9@OQ>|YQpT%;T5Wk4zuv$*jzOb zsD;DG@{n?JBv*#Z2pDn>?3KJp!x28V7J$V=78r2SJ=ijnmwJP%1D5jJ_nGi`CGpKf zbuP65aLz}V5$T~F!GNspFc*;S2%H!(IA*}SM{->^cP7UIrmTZrVwObTb*IXsR;0Dk z*8~Kq9OIK8C^S;n-$>2|!kg$mn`D$!t)NY<@(iR9&k~$kBD&hm*h4W5RF;E}h+1w` zq;UHQHCXB337AU^viIJ{Dc!(b@=waCDx)y#N-T<7HwNSuzr{Ofi&?A0F86hxlIWbaE!Bn zmdLV6&=eLR>Yy(I8=~5=+tT9W@FFX~PH1kp>Xipm2a=mY(Y;wtc;nw25N9(8^e8ZS(^*_YMS2hCy~57Bu430d57=c? z)hQ_Y|0H#gt_G#z!$)|JG)A_;?o%6z5E-q7;NYrM6s}21J8qS`8&v~d0%tB@J2)Bv z5CE%S>?)?DD#wz9Bsv54O{QJxwkj_HPA0ZeIuA+tz>4?9gHrqu`Z~bxlRA+c zTc8hDO*PtDo4*UI!kq!vU{Gc+m~eunlyg}0N1}iq#qbjs4W&DqCf4Iwl^HiUXNFch zOTLv!%O~A~)dmoP({WUw`4gMdeHP^emF9vt)w5Cv3&k&K^>@xIYRU2-eIYFPPx23p zlOWAiQ;b*#$L}hE7K-60*np5K?ae~4_w9=V$P|qol^4pd=WtOJv+SVv!(PRs9&Q)>{C5zcL~yz zfd(Wi^n6t&Wqj3P9*|cvZSzp-qhjV#>@ulmHRVNd-v>8n;{FmIu@!CUH^wFvQecXk z^au{Ar+Db)u! z#F|}Rv7dHmp^{{SHq5ye(_iLo$}4`uMuP;a!fq%#;`Qa|kOq@_&pA*_Yts8iz{~^e zSKKjjr-@T2VPJ(GRzdJ@nR;*rngP)F2CCOSG?1ae`qs2LtSpf1j|`f1(=B&SB(1BT zT8Hk8VWqzDfDkpCu8?dNQ@RR~sQm{h%u3emWL6c3>=m7W+~gObj>%3Tv_OR@z2aen zg3PMQvMZ9DqyVFX4M$YP%hT6Bk`e2J$Y%M96xcIsI{g18L4?+QET03nZnC622M}YLp*r zvWR6Rq$cO78tcC3a|S3^w7V0SHcJBLd0}!TboenwY$1<@)+O$p{yiyNFg+$;k6elo z;JOJ|f|FCxoE1SI8_gCGwcE%0XJUUS{H#bXnvDiC$JdilHlj@O{ZT88k_+~=kuTWA zR)J12zFHRIE}bmiIjbqnbnNG?LZzhL*(0*d_L(vWca+*}bZxkqS;NFVM61jJ#P)6+ z4n7=VWE(i4^Z_|4swQDvv8eh&ae+%Y5%P6WF=1b!6_iGEE|Npzqe^8Wi8Oi&LmaI` z7&Vqn=N~=NQczyt(x#m%KxCD*U3d9QrkI6D_~}u5_hB2dZ8#=S_(NiGg(*_-qEK=n zdz1!cW zkYi&m?@3>$FnDMpdJ;K! z(aEjEOW1fOI9G_uflrH6cae|h`^HkqM*$3}9KouiC`AE&R1~(u>SEX(A3!f46-rJM z#|Ts?&k9sTF2Se?B^^FNJH_&=5p$dk)3Rd$l!dK#V&UPG&ag0%N>oZ6NYE-1*eP2-1JB%ySC?W$bcnveI zkx`IY_{zM}PIw^$E-$BfTBnW6ySe>@w#MLD)jtV7DLFG^=V5}O+IYtK;j2!b=~-0a z?>Z2>+))Nsg0dJdXB-qoE+{!Xj5ZK4YLF9S2D03<5)$}9M%vb(QJ7R^ts-Z00Iu5e z*S`5RX1jC2HtefCR9FWJQ^wW>6w4m)qs9obk z8e)$JIEMH`aF7>D^=HUiAhm&*_?XRL&J@&l1s@QEnqVS?MAEML` z2r^~DMX$`*if>`odW?tx^ADa%dt0F(v1wF>l^=Z|;?Yi}x5LI9Aj!^@P)5QU<)H6{P$V}OK8jbYDwq*5_J&L|FVZ)@ zI2^;)S2@}A;{E2d*9=(hCrcBk6sT?C5?f%%GLz({oa4nrF>We#G1D;UV|+gPR9rH9 zPC>&lpGX?Em*U9Fs%(`@0_Hl?7AzXRJ-uIJ0vDvzi3;UPn(@un^8Rh*;4x-#eQ?-{ zuVwpJBjh(hF$C!Ck0GgHI#*9aFAP30c5g2VECR5_shp-NLj0zjHODp~R%h(Wi=daP z;-k6tY$)6CuVtDQ`Yl;GlR;UP>kLqkl2=ZfzhlEPWABbD2~6yAETM9b)UC;$&%yn+ zeEOXaAZF;M^7rVAIeyt>=8f1WsI1<11(z?2o|Fe{NA6H_cdyb3SlNbuCDZvktE|~_|>4jN$3pp=*#5gp_9ss zxh@tRl5J-is*;zUGYcx>F<%rY=zI*|Kv9g=7T2ak95F|uW*h^%G($gQb9q3!l>7yL zWL6Y!t6GM~lHSk`iCzRc=_cpXVZo~aTtqkw9laiW4HfkhQZRcI!lYoA5gZyoluWD> zT(yATH5wOHHJcPpFpMRXY;jgNP5{vjthp^x97Z!1?~Zd_`9_=%z&8SjJCn2`Eb|IV zKILz*Zm4$wETIo7t{XjQNR=OSkCcYygc$R>p0q9uN#*_|3lOU(VuK*9$nuY1a2HrM zsfx? z6}HI9pkwD1Y*-k^3Kkk0G!M$Ca?jdywa#i4EzW{q2kdkJbopsWVPwMYdRT8t0fb7T z!V#PAHyoRYa18Bk1G^bgSaW!iVg(0?I|R)I6~vdBV#~pU9a_j^K@lF1jErhK7BMYe zIuo7uT}|jvP_yM3f<$2tuui+lnONAncSpfc;Q?W@ad69Mbv8&TUWj&>+??t6jdy+WNfIu;GYd&)`Jd6N97E;O%jmx zHgr5GTTaprv0uAYl4KIBpnll5#MTC|dIcf?m#Ibt5LQDISv?|PqcPSkZOVagXr;PG z!R3&9Dz%ogE737lh%8oEP1aZSw-FjMfp@$CgDBaVUC>npEx z(rAkBi|BZCV9r2RE5Av{%&uS`!&2q(c~7AytcCDPC^kJMWMVB=63KZX=-|#>iZEGd z&7r|jB9?+GLb3ECQ`5sHvfmZ-uNRhsqVDzGx3hX$DTKqkf>dcpZEAJ$Ixp}4XU0NwAIt$$rs!=Md5hsVw5GXcY9v@9cYpGeJ9(p?g zj_A>dM_gXq@|jVSXhb4Pxp?#=Tq`kfMI-TK%hYr2KUdYZwl42MAymS z1HQ;9F|&##9EKz|=WkZ7*nxBf6RbR@Nt$7V4lE9;!Vqi1>GF+0K#lPTU0VoL+{(9v zqt!A^b#OqxQZ_2H(Zf`jhNgQCts6NkOa(zeTzfMs2(+y0)j^EhfVNUAk6_SFa2kcS zT!G}t_^^5|EV?pol%yrD#q^1o`T9r5l8&J^44(9e#>MJ)!f?wKOrap|W<;l_J9abT z8!X!6=!{1u4a?&7Cb5f8F{CvhV>$WEf|IXhb@PGpuNUUf+P{Yv?~h~@?4(0Fq%cQl z^O#Dg;8z6QLE#``2tIiQO%U$Rp`20`t10D|C9W z35t7Ka{jL4w7-^w$`A&H;Rw1&h$2G8t+0fA-j@qujpQ{c9%S(ctX*sJOx$YX`JSc+z>5~ao z%Z}rP*M^R&7q6X1)_GK|(TRC^?E+&{PDjA5GvxswsrRKqNd$MJ8(A|fxKtv>og4%w z2!Uy=&_cM#aUs{4K|>Oin1|+Dkh)WFsU09;fI0+3F`CME?=TtA4!14IMl}fp(Bv38 zgw<+&pq;~g3gEK!9>D+)-Np1?^&wkBuCWZu9{VV)bEqr^k^Mh<`AJ(%F!_lHhgKAs zQU~QL<4fK{j&Z&6;;Msy#{r=?yYx&nd`*4bt$TwB3|%wVeugRZ8Rt!OSXFV!L2e4Q z>fl#Og9htw*=cD5?YE#zBFMZM8$O+bLxD=5L52{~4J8gZovF)XYfleUFwEr+jiL3- zCyFIxx&X@NR5rWjZJ(pyzXCp02Gwg#QN`A~;=<+7k>5m zZ>{Wy73G+|b=O&~X54}y>M-G?{>=9%j^-YL>Sh9}c<3G(8>k(CSu>FK^FqSi_+r$m z6CXVzi{1zloi6ss{abJ>N^LVu^vJ@EtL?!)(~j}b5w!dQj}W={LVU<XL(Da}Vzs?qW>ZxAk(2q?gT8Ojxdb1fh&mv5|7y#0YN^r6CinsddSY62X!oV7z2srfi{KdjqNTv?%M# z@KmwP92>>BdAps7SQzGjgH|h!R2$uu$^&{O92MMHn282uyHq7F8Qxny%W_pzzP%Bw zQSz|E*E>!f9wi_zRHX)aRYOJ|?kx6NjsU#u&~^mju~)2$Sekefq6abtxYR5K0LHkG zF<|?MnUmM(&|P`pD3hxze;6lK1PBj_EE?F^j{$^jt%KJu%c+c~0fSpZz^FDreC<%; zKWe@%wh0u&^^rYn0odP?5N^nvBvcJZ_zUb>3o8(<=MIIwT99xQjTuERp60>jIp@8r-QC=jewFRG_jI0gV#Ssf5-iRB>M^ z`GqF;y@+68HXJ%O;>3YHHlhy6V50;MO9TiDMDqP-t+mw9869O9ccl@cvQ4GDw_7gW-N4t-SHw+O37jud%!0pPJalcZz7im!V7%nPAb$SW!Z z_3=#69#TO*Jkm;@N--D7>&8z0UOFqJTbFgU3$Ot`0Zwy)inAQeuc1%}W$rm)&@@&D zr_eSX1u#`6AKlS37MIS&pFCDTMt(T!=P`Ra__L4tvxw3fhaKh>QgvniQkk<9t2pqIL!V#7qbC#{bconTyw6@KLLwkb)SGG*^8Q+t!EbQ=9 z@sxxV0Xv3C;8voXtqn`vlxa#r=!1>*p>!W$+?JG&t0yci>wrCHW0hJEC$^9+UfxRr z3y_r#@XqOO-KF`AD0tLFs5iV}yjuK#2TE#GY2qPacIIuyYMb7soxQZg^8>LWQpP24 z<6B>YN!TxJ4}MYQQzUi4hwPO!RC(WEJamULwpro^M~%pnpKrXg!CZS#1n||V2 zB^%E?2;j(2a7pNEUNL~I9S^RKe4;`5jM@EWLc5`w_@aO^n}-~sJhBa<93jwzw@RH( zcYuQmP)4SlTo%-4h9=&3U*XCypkYOtR1=R){Y) z>I4ek=`ybw4u=xx<|CxOm|g|FXbSbb1+qmSDhX}6D`^0$!O0i5ioij(6>wfY1ZkAY zC6!Gbw3^MzG9hV{lV3x9V&i~=nYSn?EPjIN4h4^miUh2sm4a1>2TQp`q}R7!JPxZO z{F1*1y|OyKbYHxMEUcxS!W5aLP_Bm*vG3mzefPvfUlAsLZ8wQGbLoY9j>IpY)Qc<@ zucXD`M@IZ%m25G<$Tkm4fh|yt?hnc`eb%P9;FL3WJ!h~oHR?_>s8T>vz@@1adBuUxkTpVcPB2)|BX;W+;oLAR?@A!cv1UhP{Sx7SO<-H#ZYff{ls& zQARNi9?1#3$+G9qW{mQ+h=L!X1h110@IAVk?IscvZ$8o2qBHI5g5U^0ks_*QQ97~sMnCdP)K7C#fJeRQ{ky|-o}7f94p3jS+fpB8mM-0;)jSKPqK3B0(9PEObbD~1E$b8F$7Np#tz=*f-NY4qJ?k+teMN^x zExyCCL}bD9&Nmsh=G7%xc_`Qu<=T@YU>f23*X+Z{;+UP&I%3*eG%Kd6t4!gNzCroM zTn-W&Ar~g2sQ{3z*2AfXuSmCB5DZU}s&K*9%Hs zvS%HhA=s>Vn7FuqG^SF$pm*~o?!P)U$xITV)%`PLVZ!{hpJi}vE2*|azt)oWq3)tE zHRx&ux%inv+e6$Q2C9WDmZ@y7O(Xx?u8*j+6yU;?BFCdX`sL)MZC?O zmtf6^TC5?8sGzieC56<(`Pkbep-I|-HGoQ?wQSW%lf0mBm}=yh6rM5xIv|l4*hE`x z%)mQWh1p*DBRckpV}_lbi;bF83t}p-=S&84IKw5^#SIC^XWtnJM6Q-djqM+3N?Zbp z^tw}bEsB+*U_2}xau&_18ia19=4>-#qHN(tb!xoNBJzxdFRb(&x>c8p9_rKacC3=i zGx6jgDlB$VZX==}D0>JbL#Rj>)+d5gRzC#bF@i#kizzf}WXXX;44GtS99~o@^#$Ss z)2Jm&dRR~d&6(T;eE5KAitN{aBItnGAcABVc|k(8gs8ypBow$vB~6phMK%;6Fk6RB)Q4jKHUyq?$Ors#=65JS9dw9dz(>PUh&626x()&ttH0QGhyBk$E7G*(JM^-^Z=B+@0{Wfxa?@Fi+-?mGOzj= z2m>OGk;G}VI;;Wq3%s&~xc%%PgozJhV*KJu+vBITSPa76sE3GXTXW*D=T)^l$vmyy zs%wQ}6C{#xdM`g3kKD0AlVn)o=|{`n+p?dmV0hH%n}0#Am$6oEilGD6LG8A%DHg03}aNFw-HGMHdHX|e`9p`Wy;34%nZlu zhOStJa8RJ?us47^Ac4aoz*R>du}vt(Pc?G!Jz^9D7(7<&CGQ#Wa@xl+vsy1VSgLr< zOE}1$@eM8l&%S=HeK3iPV469?HsozeLkLubwejNN^H6YA`6dItL^wIQylrNbzj9EkG;j0drYM*xOKW$ zC{B~Mn1S43#B}UaHiFX~K+hBk2W6WcIp?Rvwy`&}JtuonWIVN*X>=SAKtby(38zt- zR<>mVTPlrjJ z_mp&@0mNS=;PIt}_`rq?%Wl3`Rry|&qpEsQdTP-7O&>+2g$dsE_fUgtC3GaAY3#MB zTN ztzyCZluLJtlqh)z1jI5IWqzb*KpN`wH)8Aoh)*L+FZ9P$4ye-WsOZkV_-Xx+3*M!vd#orb2R#hED2V&AvD>@g> z9f6)H?SBIn7l6>ushR2t!m%ro2&gHU_G5~u2Ean9PVOaG7$x`)Eh*%F2f7QQ*-j$z z`uHb|xDs2`5`kUGo(Do5t`bw|HrO&dRP9y$)rBkuoV<9fqQ2WKZ+;<71mdMMX2|~N z-?CgUvvler@C>}fRO%!hNophNP39rbd2;;uRPaq$=EGH?@0_vN_qlW&qc+i*L}v*S zHvyphNn?Nh( zQ=*%hheqM$Ka|eHZV-Z#3ot#JI!GQHU?H$X&SXN;;oI+XG!2G@o6D?*&reQWG;t|L zFgR2qNR=-q*1oE&TnP|9Bdav#q+}6m^5v@1m|qVG?*PkE1Q^(ONGzYoYg8ymg}4A9 zwD>#cViE-c3z_97R)&_f=kr~*h|DYZMT+@7@J^8di5`fAF+n0C>h&n9-x)g{pRJO5 z;!?5^{U+y}*GJDZsz5)2UF>qqw=i-Y0SM6F86LWCH;Q+y1IAU$HOAc#9UnpJf)kJ+ z5j8dfLbZZ;P~k)z?v-rLyJ`GXvSDjtMrLYAC4A3~=!Zz)yxazWc@!|UT@9ZvgET3) z6pa=-Iq+Cc@~M=|A^UcSF?x&43viN|tt&jFu4>SVFUYid&OWh==%;`lzLAfOCW!B?j1m|6o3 zXGKBC_U#>a>wILe-e3w&l=Qj5H3*Q|jEu7AOp5^V-8>^ji$C}YYfOQ8H4%Os4BC-^D8)A5mE*KOArwJBtxVWKq8o?v|`bXVOmHb>FbloI+9xt zLfU}V1zx{68O+*IfV4A(ie~zAwP=AykH$i31oc~t*+NrmX_!>kQo3HgCO~N-8Zu{& zRb;lYIu;FeR<$t(^O{FzqEU(H3hT6*Mcpth*V#>>&y+@xmwjf{k6s*8iCYn&euYCL zCDV{)atL(u2Hm(W83iLKWrbEpXI8+zCHO=}TKPfy4v7&a;gAZK;^;jRxG=jW6|%s4 zch#gk)SO{K_{S?5n*^YjZ7s&!CZUnQL_}P=%N#h#$r@1UPXh28S3L1XZ0PUY)1Kj} zupf*-6l^$xMRn1>Ss;O07Bt*{1ZH@SBym9u`;WpHxNbYliO|p)oifLHWE{dmgt)6l?IM_-J(UOm>xJd`fI5K39-eyAE25lxnByhWs=-J!{le zMk&R@<7PteIpCD%SJN9Yf{Jj$pge%Ppeoo>b8KM@W;^25AQ+zra9|^C{1jT*>phk8 z^n>ozy)G%B8PG|~u-T_)jRu>OnI75?l=`y?08`L-Nk(?azg5a3GG0Y-+41nAiU(dp zAuAf!craFkEyLNZjw;_Ovwoyn#z_jHp0u7`VaGfmJm;futLNV2p@{XFjkIJ#V58zF-3_-V(?rj;!gX1RrFp9gVTvZpCQq7 z2%QQk6LtIpdf*#v#iYM1SGS9&fWjWOYbrSoE&L@HU?0Csag*yQH(JLh9f}}`rr8A& zC+3Y8nR5k-df!A*cbf@~sV_1e! z8(Qa$?%0rmGFsm)Bd=_qONB$aXd~%W)612XrB(_=p^40yg{cXJ4ZL+XB9$+kkew(aBI*!gYL%(>2v@A^p!ohOrhFDJ9L<(~o6dXaT_`gD$^FCAxUo4t`QVo8Js)_Urb zyHu2*=HOSAnnT^Mwr+utj;i&?`BTl2QNXw06s(^v(t=w^@mB()#X|a4sl09VO7=#p z0o~?1asWcTpz@;e*_CO29#+L_^ZF4>rc0gCgOv!Gl992n-NO$T3Of zB3}d{;;dMRE7lJf;fg%5vKZYEAE9>qJ1CVfXLLKyMCBQxPlLjp2&5--sW>k(AmgZm6xF+BNI&IKLn}-Qqm*u1&AgyidyV@#%sr1mu<9RX5P7qBV3e$Oe%&3TNz8T zfOvu%TWgk6yIeIng%O~G-$HeBM;Q&m?ZQpMN}2)Wasq3(;aRxOB5>zsqY>G>)ue7$ zDe;ozUzXH@sQnz;)COw4uD4>mpoh~A(hgVGk9NvXDSgA|>)?$>_~JoMZCACfK4{Bq zbgRJf%hw7Domlt^hq4E?ba4ubC@MEH+lOKdEg*oQj)RiGrybTgH43vjRiocZBNvh&$%bNTO}y9(P8}(sXSBM} zl4qlk_I^ar5TSc~a0}%aywd8O?e0c$X^ZB13Q@ojfnGJ6V1+0Ep(|`OW-0-h7ght9 zJWaH06I8OI~DDDib?$m7$}<(*!!UIq=+NyVUvRM3pz|Lj>%a4F=PMNQM@Yfe(fINgjf0i^vbud$}0!IWH1p!Jj=aYu|!FJWF9fzVY3?C*n%8BHw2A805 z#70a2WV2`p9W5LYGAvx>A%_iOj-cTP2fpNGA$aV!lH5?tu25!jLroZXXST#j{A)sb z_CjDt8VP>1G){GFj=Hq=xSb<-%lnkZ-U7kX5-;Jb%lJnSJ#-Ho#ww&+&{R0DJv+wB zwRIW8x%7=0<@IEw=!R`0b6gIa@b2Mev()|5 zEz?CN4V|ricpKBu@=*KQcy@ZyKslTbCE1!Iu7*U}F&-dA!9lh{-!{)jYwv>@0cvpE zvqop*00g1H|k77IG3(f@_9*Z?3!s7?2=CTU)Qqjc;H_uD_ zB_5%P)^(9kP)HXf#RW0s_2Em2Z0^yOI1+WgGG7e?o^A8__h$iFX1UKd;}Ny3iI z=-?=9?d2#!EOuGUZz%#lh}@J~-c!Q2R;>c~g-E+Z0dz!T2%m1IahPYXECx(X#f6kL z$&ARng9b|VcBOU%fl@+-Yb3GiNFAEZ>x>}l5nSjW2Y}&J8A6*P85sifO0*98LPqzu zc&Ft_P8E8rSv7cgI9|uK2Eny^3CXXD>H=jHrQsXN0t}1`j2^(ID?$qqan-0m@B?f8j6Q2qdk+G zb5yLM1lLrGNzrs2bSUP|4OPs{jo_3G_2;Ih=8%Vw$P|+006`U@0(q7GKoujfpA+H& zqv*ZMB8*c>RO1t;3*AsN0WSnGV_KGGyrVN@O;;>P+o)Pgnc@su+Xgl6K;>1cw_-D` z3{^MHz|{1ed_g2??Vn$ZH`zF>d4^GzUzemMpcE=l5Fx88Li1wYLsa935U@ZWLBakL~bv3JVZTj0a#ud2Ao;nCY?v z+N2Z(mt9{kNZat?F~KMZD6|lQ-LtJYpSxbHHS|@fmJ!x6d7f8vMkkLleO^ zlB5mrH9i@on}vuS1K2&Mi{x#hrXt1b;8$5@8=Zv`A^5@?!4A#a#R+14??5QJF>?~s zi@zIXN1SDtQWocAb5t7~Az=FhQj^2*;`@tKtVN2;&O_Yk1VK}WWI<%umgD25I}|w( znt(8bojiPtW20!sMpaicby42|HQCHzupc(3Slrm!!gq|ZIsIC$dP89we{5XhznhCE zH+R-n5olq9@D~~PDyDrm*j@0)HmJ*CTFkvbE@2ZiF#zKmzAj*xke4nSS1tu>EhbQ2 zOxL$a{si52(qlBkL|f2k9pKVm3Ra2mRnOqDLsrRkj8QQgEw`A zR3l{S!+}S&Tc;vN0Eni&!e#*6fMj1g`UhB+@rZ|&$p}2+VFObMBTf~J@6C7ys zvvGZSjQhY<)j_yNIjhMuqKcuQf~2D6y-4vlNW2EbE-KV&G{J1=>3hkLG01dTEVdPC zYa8n&w`uEb43?di>RRyh%!)h1Jk93;+~jfb8HSMejht|?U{mVFIb}FFOBNWCAg#eb zSt}7`=84GJ`-O+w_Vp}uR8iXE{Uy8iye8SWCQVqK_M!Z!h!?fHQhOnqOgyG##pvsj z3Z#QZz%>}RI-zkmviH6qjJ=u)8?~v`H|_&_H}4|i8I2POOHKi2GMnihG!&>tUhmAi zm|X-V^V=e0z=W#^L2SI)mn}J4TT-HWCRBw@qjgXr?VoReOYdFPjT%^m;7-VsxdHfC z!hr0|Owi^TT|(qw-JWbfMGTVRPo&@IA&iv{%B@7~>!aBcWMj!zC`9dia2wn*OmrRI z_`2JM@pKZ;$V{b749AlL92yBbmO37>xJd#%n*Ez0&ab+JAuy&OV?Js3kEV>hDO`nM zt#+mtK8+LmF^roH`c8palnwKfpQxHWUN!To4C^Oikun?eO1Obu?-7n$=r4<8LOkPN%3Gh7OUFN|xZY%cX% zJ)&^*76?!F?4BmvJ-uadV2Sx!lgd44PJ$&9FLLH%RbqUeoe;jJenC{Vkk3cPnEZTR zD`9bD_2@YPdSeEhc3Y}4$WVRE_ zh%!3CPy+h%ySG>GZjj!RO(pUMsFg8rP^Mt=I`m;}ISoPEwgrZi^KD zgy<+S8%!B>uHH<$03s0phX@EaGF(}cP){eTC^#qATPG_Oqe>aMF$ENZJ>%DHvUZnY zm7)dsTB;Uuze=lsun=Lp!LmN@k?|u^5$R%5QQ28Qf|Fbl{@kjrfJtTZ8I%YZSg=AC zWo##VUZ~T6g}glbWg%p5QgxJ0?o3iaD_Se0)V#d8$-i3wj{Pqb4MUTBhV#C&W(UHs zo@>tpiP*N%DBzGs&YNI73Tt{3Y+FYg)vCFO>(d-x-r+|5g*L+t?CXL?f%7hFV6?6I-oA_Oidq;6X{&)nWlPkacN(e{oin$Q%#JIiZtbG4f> zdnycm@VF0*!;8?Wm3uk}MR5pseNgyoNCFj*osj35j4D$cskCm6Q*w{6N=Tva9bLhQ z2xG?#T=C{CVRdn`koAKC;*W;|gnHb=#f%Of3EshH7HB>!bGO7mU|#@Rma9&PUOjuR zX(9MpL6_JG?b!#Xp}IQr->f8fQI{3FmVklMxV~k+;F<;k_HIOIaKq0*Os&lpY)c_)Lachw{HAT5#DnCiPfTimLe%S|>V z(cPSO%TbR82>gOXDZ>C}GqY41jnw}nwaSvAu)#Cxapz<(@KTmz!xYC~RD172s%mrB zEtect0=u%DZ!GEM2M8%4EjaNkdmhRs-FF_t#3-;B05@qNLFYf*WId%o8YX%Z#WPho%LW#&SY?M?EtH%$7Dgz3N6kKuLy<~B<%KsP$~r_d0$ZN z>n6ak{9#748^Ot*Yxb&6bDfYJsjmV~(t@~NeWYRC=ne`bw?{+h?rY8wYV3_LsPz4JqQLi=cUiiYlVjCMnk2VjRxerwBy`V32`EB1R@1v zg4ui(>029I6u?o&FA|OcUJ&h8s*KGa#x1B_7R`Q@a>!fJRRBuLvlGNC(dTXHeNoY3 zJDS=Au+*q>*{fz61F-LtnmTg>X)ZV8sB_R%UUl#%B||im@tGNfM`6>0_XLsRuqxvX zPLx4q%L^l|(bmMYI<8reKY2Tsn~0^Z}nca$D5B%vxK)K=_8 z4y&D^ug(a-zJPI(ZW&i+i*o*>J$Uir1A+mSuw+Uoh4#H64a2^Dc(p;>pwE?)p0I%c zCVbjO9c&?6WR?oF9Yhd4pMahqY7w|SpEV2S_H6GKeccRN?g8stdN%uajiMiW66jJC z&#N~|q1?p1w^nqAicLYpmZKJ~Na(i!Zu=(>5*BE<)vyR|Pi)W4kRvf`f}l+I)1NR( z*fT_0j{VBMiwzFWdHB9bBL@+o?RtzezQMRY2Pi}gxvx@4cV)I?#fBrn44A>O0mZuS z7-xc85Z02RHIq*^9{VaXOb%QZs7CDrHMPmCMAYggvcwo*{Sw4MRfw=k32>Jf||rnzOQ8RtwY`>d4f;I}j;U%nm=B;v zelg|S$r0jTQ_`he*%}P_!21i@;J90rDInF{)C1EMDs)d<&P%_p?>Kd`o^X5{w`n03ViPZ-tRmJID z3ye1f6%^ni8&a&ebDY~-)dmj~k{7LI=YZUNQFiw{O;M zg(TKA;$fH#bfvXH8NvFq*==V~Ix;(I6iGF&uB^eW;<}a;YYedC?4mi^Y)Jzi5vhQH zQD#H2z_*kRQsGwT^|{|D5KEPg$vY#ESy%=FrV)Y|fRHMvE(P&>8Gu{V0QF`cE$}0h z!G|<;4RIwt;FOvuF5A&Z-$5GinENkR<|SSP7>Rl(OCcY6TP7{bVs#PQ9XPcNoZxf_ zz7BO|15NLhHBcX8$3pnlttnfcSBsK%PQ7#SSn^VY)pdhaVK0+l>L8 z-zLJA@!NX4lu+5k)OuD|ii3S2bU{sxkx)_4OxtCQ!K0fHcp#|-LKf;qNtdKO5P+yr z8f?ZwkECRVD}}9B%iEoK7M4US0ie2EhI0=|71gW+eTA3k^eUzSakzx6l!$~kZxB7~ ztfR^MuX3|V9pucOhe zLyx=(s)DMF%NCMdshp=>sK`2iX2-~NMJZ0pJ@YH+j;Ute3J|dfQuAqnKhWwBz|2L( z@rIQ$$g_*xT0I!>{q_&VjJQ76p(~h35P^Z3gEt5>*4Anp>1zt$?WFRV(ja(Y+0q}M zpX}wf0|Lj4RFHU`y+}Wc=%B@lvvJK;fJlF&i5J7%9StERO!kXX8|do!FxcQlKtcAO zG`aGvFvE#l1Dj8%Uyy8EEf-dEB1lwCx(IZc24WrQ%Rrilwv;T6no0?Z{m68ol^Vaj28^gPLMF-F2~kTEGXTU zUAl2jv|(N)JZG$7C?PwZ0`E#7;bxiY5(1z#2X@>mzc zt$NP)IBk$npoYy*Y~1N=k8e6;YWA%P(^TEvRi(DIbk%EnozSmZ?9STb9opTv%XY*Q zTB>c7Kr7wdM(%Ld)@XM!g|BRvuWQ`fYDqiY#Zn#J#oIlMy=noscRMckakh5rf>XDy z+dFC_)r)Jkt$BO9y_tx1TSAf1-Iud=Vw2s@I`!4t+}+!i>}0B=w(o8-YWKHQcFAo$ z-LG@5dw0F-)TK=uSSmeq?yXv`+q~UUwr<5~QBJpPotXA_ZQaCE%XX)@YPWD4N4s^s z?)2=-*7mB-v|7~$+9zjj=WLB`%$99UvZ^}md%Evs6jg1CCig3KdJgpU>ho4+tY(`U z?X76;?hf_~d)}LOS?ivSJ=>MYGTYZf)t$ZbH-wXALIJy)aK8Eo~{M`LVvZ#ML_w|lyXr$)VdtxZ{OP2TOzF1>emIqmJa zLVGeRW+F=4-rkZ@5Gt10SC!q#Q%y?tuJ3odZKRT~O!j&@^eZZRY3ok2(-kW9gc};7 zb==bEZrZ9VI}JV7cXyt>I_-=t9_ebUu%DM8 zZp2H*So1Dgb=P&$omK6i zlIwQ2r&qD>Z!J$<-EU*NJ>9$9XNxBp(B06@i?vB)%5Qsn(yLpqdwa7w&qEiz-2o)o zmu7n2%|fDj&_qs~*wU5Q?{9VP4Hj80vwL|J-oo|e?=1Ib)b2X#rMFDj<;L}JZ+B}W z!l~?PDiV74ygRpdT(0cvWZFkeJ$t=&%>^%JLX=lUvI={#()RTD000000000000000 z000000007e>7Q=xK2G0r-nHg#>#g0Hc;Mf7&%9-AYVFkJ*4?{rYtH(8@4fH4fPCsf z?SZ$|Yq#5#v)#KGdQ02Adb+jFJ?q-_z4i{f-0u5!z0~h+UM`<=d4oM`yKC#$y`J6Y zn`XVe*q5=bn)Ir7UfZgzK<2k~EG?Prb#mLi?&)+itnl zNglU!8}0Y!N$RCtG(t?5ekBw=LfHR<$0oU1Z(5hcJ7F^KO>AgSy>5`+Lr^JAK{Tl{;>u zmfCl2SG!HEUhU@9>$&aeoxI-J!?Cs36WL~^9qTx?-L`dY-K^JlbJ)?j?%uJsyT0E0 z+cva)T*-Uc&sDDNy^mE=>^yU5_Zikln^V2o-rim9B&(-)bl7tp+I4pws}8Hao$n>1 zv>mS7cAsAFW}~iij`z20dzsg}cE)WMytL)r*zH}6_3wITyS(RbW~9Apsv@QC?dx~9 zddBkmz8`&n;j0&x4{ZPTY+7<002)pNRc zR=0L;-n-YLq}MB~>hrxl?Csfo?)usG74ke+Zo91J+uY`|(96B99lNPFP2AmXIkoS1 zv(ejGz3p~Io3{4eS8lgEdyQW1=VY}jHD|jXdB)rqOWx0UJ*$%UyVmw@_(ePbOjM}B z6z~8gDXK9QKLaT<6;v2SJZK2XN=-2o_CO$21_dQT0ECnXMe!HkefIxwzq&xdQJVi3 zOwjl66Q-~IRCb#8KV-{)Km5wm;dX~$eoA~9tFlQu7)ymydG5%MWZO?z)pFk@M@+Fj zoUvfVXmPNuHi(sXaXC4V!&o#f1X{g{2&xmubDpQ>tuYpYrddEPhPgPN;D+Z+-06o{ zVWAr%r0_w2%|u6^K)#m1op1v(&}W$$+>AVq2!Xk5}EhTc`u@%Pbhui6aa z($Xv-a=_@+o**i`#XY=7!q6WTvT?4H>xjR!4e&CV!mh&H;qWhcVuNz0HX6-f4v1%8 zas{2lCL?@4oZy?w*j)`><5;#TfP-|b)u3BtI&5xfLfxL%rpazvy?xfs5oGpm1m`Q9 zcn;mud#X9NA^4Ir(*?Dbp_Y-fV?1}ax3#}4ZPbBi;tYGL>1RR1+vo|0&Nv;(9u1`( zC>B@+MZ9K3!|Cx>M)bVjM{z)w4%j^}i@c}F zrTG<8GPhT5lv|4I`(c^7K&|Hm5Gt?U0d}I7id9iqAX1@hQ3{dnbr}&|2A+80`OjA_ zoN?!Ivu9sFI^%VmcH-^2XMgT_*sDy+s;Hzoablt7%3Vt9A(I}!_wD;3xEitzjn2_; z*r$*W2A7E~r-Ct7HeW39p!sGi=opv>4G59=dVs(|%>FT9zBL@w-Ir-FEoOWsYs?BI z0aU83C1x!HNeF-*sSl_?8i`dH1d19(R^-ALzH#j+c+(BQy+G3-O{iBOHJBM8{Dyvk z)+oxGI8gmOGGW(n?ng*j=eB-w>utMn&Rn(Mac1wsgYNOWx37HlcUMrS0kDpK2p5pf zte+~P{8W6(3Z^NA3J09*^gsUKJ?HU*LhmZGwoGLI-fv?b`eZvufeP0s}# zNR)!enYfpNpi>_3G51$F{8ulCNc$Fn(@VBJN*0;w0di+`JI0M&`dJT(v*kvjcXnnt zG19krgy2tza1Y+DnL&{Qk_!PRHRgb>)|{9DO3k4$iH%8zLD)l}N2p`+{)3GFt!sPhQfg+v*raA>NNWi(J~7J%sFupW5N zbLyfpc|IN$b*HL=x529*lOj|$+(&tLG~&e$pxCM@ zI)D(+zc`g9R$5a^*IgW6kp3q!GgBxK9k;o)?c93X&%WN>y5n)DJMTHy-_JkZHt)7; zH+{vojy?#go?ulKep4#RxB5txL|>`rJr~zk?xPtF58y7+L_Vh*CC$2`*gr9>pL*7Q zg0njhaEoQVxJ;8DCu)@gebopBnxs*_ilPF*9{ULm1qaZhmTPbd=p3qWfjudpdu3i8$hjsw09B8&jb z-4BH!5?Z|wrER1t`CKSHMb60sugREIPc5*B3Z8J>gk?nXpj1i@`mF`M-yYFUF6+fE ztlNiSbGr&)3A^%uj8eDAfb0rD4>8iTmLkvH%w%po319?Ng+X>psr0==n%&j!+ucn_ zq#<*LW)zCAQxvwat3FM7@6x7P)-{laMl_0gcB(*))2)OxaZ)iat$j|1s1DLs2Z2WD zDA0bJ+^q6d=T0m|Er&xE>Fz3)gaG)a67t@Jq%nx&x2Y))TrvYp`hxt*+KaWm7n&Xe`vf$Y z{)#O#FdJF7M_+-vtjUVHqA9H`w^w@F3yIDwwP;^l=Gl#CT;^YKQ4JArWE^z*c5Tr- zr-=`j?HXPjQ}G)q<5$Q-lIXUhy;5=xRm3=dwq8n>L(|1@IqlOCOKOl-b=l+R1ft|N zu!ll~1nCf{*(I7`7(&v=V-$?fQ9_iJ-4dCS$hs>)Z}(5)Ei>z^sL*N%OlUEO6bo^l z1j}O(ZBGa)T;-qG0>a#WWYPeVNOi}Oo4wV{+Tm#1!(>XLk(}AZz}7PfD)`|%jyO#H zc$;NZ@mKLsuQzNimX1mpNdUg(gZXEz^pkuIK%BoUzVulLk>12W; z^~vH_jwR_3l1FfqC~~FQsqwb*Bf+q4HW2LNMj$EdG)5pMK?HH3U?&&ts#y%bTGfuK z_q8kp#S31zdJ#-8;mdanQmE(~Y9&l`pp9BV>oJi9B3S90L!_q?&^HEH1>+SBY)TUd z^hp+pW&ugrg7|o8v$|wfcq1t5bJ%gLYu?=R&aH|R5oq*=Gmm9jQjKe0sg*S;yXo1{ z&Y8Zxzj7iXL6KR*h=B{{n{{6ec4^csaV`f^Fcz_Wd|8E<#5BmbM8k}H8R{s)A_V(l5xL4o;|7X=_X}h^JjZ%QMoe6>0jUhJa@a1wn9&0|3J@dZ!e6dJdA0 zjv8P}jkJQN*O@nN&;V9Kur8}I{42~4f|BENaDq0OBEf!@m~HhNbQ;JvNjG%)JJyHj z=VVol7~Ur^d8QQ7oj_-<@OFHCv$} ze<=6O7nk=n8WWpP3Q+dmU~fIJbVm3O4>&!p1a50I&c;9?0i~!*Pu9L3j4ibbk5_<_ zVe`czYVmF!fa3Y(7>^xV(feyPXq zjw?{nbOjXz@oE%G%~ZyFHaO(!38nZ5!H2w9Z;7?BN~R~NR+!~tD#yjoO7I?l@`JcL62iD zgPu~rG~?vKZ#2D5H8&<~JIHOsgwScA8s^!8tikIDMz)H{s+}ryd7#jNmdiA%GSc|B zG_yBg?hYme6z_*wE^Ay>zFdH~@&O%0xxJ50h3ys)hTbv&oY`!>5R6VwLSi%)3X7>A zw_cHEjRcYqc5^5%KZL*#e#7aY&@Gta(ckXuN$ZIu3KzV!M_pTd>CxK#4$p^7E<7yOGSc%M`yE4X2X>)!n!)bLm-a zDOy4Z`bJ7NYg`id&wbUgs2C|Xj2*-dM_1BNlvf1&Se+^AD$s5VDh_Ogrq7-`u|qfk zVCWrVy<%(hG|k^V(McSuU3H3)0*avIDuLr9^^{po1Uc;8c;a8MkkXwgiu$O20+Q>p zzk4;}i!~dQNQ0`dti2teRf5KTf^}aP9&Z++NpCLc29GQ1Ue#J@!f=>`csT5Q4JNna zk-gF5NwfyK0%V4=#yKKd#H_j_;(f1Ft})%?%_*Z6CVSJrTU%rq92M1B*TlWCk20sQ zB?6OU&U;2Cpdm_fgrPMN#h9XeWbWIj1I!&u1dhHnVA+c5Tp<@(gE7*?nnxn&!3dkc ztY&&yc-xa15_ZTJtAsEkaVLe~@Ds=kpQ`Rx-c{OC&!VUr_96%oidQuu3yd4m)%Cq~ zCs+{-qba=iha_kx(XRe5aC8Z_WEGBS`&-9=dqv0QwLS)y`bWAs=x{tY?31D`WQtY5z`!5P}B{np2g2&M87?|VZ`Q2#6 z_Ct9K0J@O9P0Q%%Vx_|#0fA|v3$2e7hQ2%1T2{$wvXG_umx6TL0xNTlh9h`TEJqBw z>Ey3g=y~BU85Wco?h_=^=BZ#l1q->;P~Ix1jKs%RF|`v@<q9uX(SZg&NAbpzba^T0Xk(SF0fJe5-MowhslU zxT_nhX(^Gyi?)Kq61}q8>NM%8gH8}c(#Ox2@fJrfPq$a+CHn$_MR-%s;POyYuWYY8 zBFAD{vAjmA-l>fURAf75`>FOO-IPj*6%r*GPao2ltj#FBp3{72R6kJU<_KOn{HuZK zRpo@3T0H6J4ZCsqqFZiU>%j;WET}k6VIL=hj2|4#a0`fwn}V{j4rz>& zgIn`I2HEB`5mzJzK;cgkJv;S9L~DVC71y}-92Nzz>pXru>-db)yjj4zkekjfY9d6m z5*Q51{BTzsMf8^|`W>`YwfvkjWU_)4ZHH+xrLrnvV_Dt~=hYGOMA5`v8Y(Fd$TWCo zaO7ybc%diOP7!&pic2y)MYy>hOF;G%J2q7&tTn*Y~ zDiXq1QWVUTrWh3h@RziTQHj?DZ*jFEphknXq;t0&+KS632Y%TldL}Gk_O2Q&hFU?G zeXFTiR&9Dt9_i>z?A^Dc-1!CSCs zE&FR&yi`V~^-HZhKL*2bTaq+xcRS_$BiB$%iP2)JFsKW|=zu2dgh&ZvphrBlxPPsYbY}SiKJdvmsd#AvlaK|M z*>rk2lJ%S^i?3?w=Rgt1a{=>^K_4+cyVH?f=5TJ%MjCmIWCh;K&3#hwu?>3Oqn<&= zeFdYs3ttT?DEmdDtOkbVr!eWgf)f|8H{(Aq9!72?AC|Yd`nAA|j2Q6bPmNkael#mG z?v27_?8@+iG9JIRE+?C5m^t?&A~VR_NSW^FF|n*_zr77~o+-^uO5{Va=5F&|HRp3) zPVd}~CgJSxbSOQhk=Q68!m=g{ijSyw*L=XGJlgp+w%XG2?lSbMp%~rcU18`TMkFd7 zegI+un`jnM@`+wwP9|m7q^HhgBb}q*)%iL~D}&Ws_N?$=s4mLg&JnqZt3n%K_9}3? zJ=wzs54K>^e7t#?lY2O|mRN3Wikog}Gl8w?1$Q+T-puioS&f@C#8>ds)+qx0!Znb2J}9Ze%fqRWKRcqEAaE zm$D3(M+I%jkaUbdN3xE==~gX`-+Y!9a7QAGq;XQdQuCpOeB~e94Z5ZUO0HYs+|;}9 z=+8RS3rZtY4ok2sIRA!R(lne3l23VoF=NG&?i zb%lGsY1dW{YuS}Xbzhh*X>k{)u+fOn9#&bn%PQb}K`ntH9ON3C@KZCX3Ni|fn9T*) z4P6ADK*a^Lo{lZ94@Gsm8FHzf_{u&Me;S93N9FtGhZIG0tRPB#YqfRN8cs4zJ`|nI zW*Bc5Q=9~*kyJ@Ev-F?*xaF8%87mjUEB$7~Q zwre5XyR;Lo--S)6XMogI?4B2-xNCrbTbFCzuKNXtUWIoD1gKPgAr8Zm^cO;%bxw2+ zywK<^D-FA2P7v539)|QK%*osO)dx4t7V!@XD{VT|x3>@K_eUsfqd1~1;WymQo{^_^ zQ*_2^2VJbNf`F-5$9Y1=j#QzqGMxMF?cU-}9JJLG_6s6i78$MO3`9;#cV&_IS&UnKz8EXyz+zHc*1yV@F!+Y;LCqm`&NsKWL#}q3NQmDizL;2c?GjkE} zDq7%b9vF928m8rfrbH$r@fFCs0T~Y?0b@MKVJ{AcreZtg?d~%gBU;s4sKq8&#sXHG z+S_4T5XZX#W|vf335&W`5vsy7Z$?6<)|OWdl;m@~-1FNss#qUWC&vS1XH z_JC|t3EAWl_U2Aswwc|iY7VN+kxgc)!&KK}L*=eBEGw@uVWGvRWc>_7#UQ~m*<&&` zOI%k%5Z_$!(S3Ox)EQ0F@scSs06+@%N|n~lSeDTzGoG`5FfOEV2vM?VPMWQvJ~##7 zqw@ADQ5e|-dUWgLQPhz4GUT{`q^p@Jd@ zEU*^gdrnR4xJrlbYnxpfbnA|owsxUKD8Ub#u>$8=?OL9@)~>7yRC+8Z z8D-3za*E#4Yz!C_#PQe&=HIj#IrU%|nq6R3t8nxx91Pa6ZyC^%NRQdinyZYG0M`fx z7hIbtCS#~DxbtJ3?TrayPS>XH|1jR+G5zw~i(5YS1|44UOoz(L%VNWAN5z?go*LC; z1q^O3*Nz)e>9V`V15s{x)9#{H620_36IViw3G2nM#N`6A2yV~=QFo+ASmELGkwgad z74kxe;qnG1xDE{6^8=)t9vZ_tpr}*^&}S-&zbb{SYgElK(U3fK^$?@*N3z#VYAYzc z(cJVCisA(PCij93`wJ%0(PYbyqckL>~^WShQ-C2nkCA6s8ho)!a~7$jdBxTRD~A>A{44 z$w0+)tURLFH0NL-w21C`5(N)mKrp|-tCxIq0Om}zkS3z)_(9_X4h{rlFuRTPTsYaf(R8yakqO3a^@w1G9d&{YOE6@z zw&Vj>wyQnZIu=VR4&KOu&vHBqHo_~j2X0wg+U2DQLKfYLD7g(6!@EO-^^%JM#-cjD zS&H#v-g*O#O2-UpX~Q`UBiNXBavbf4>WF0&rOE|Sc;z5*ymYjrOQ~NY{F`nD)|;gGOUs2f#Ak75(Hw-(&!7DvS9&I z<6J%+K3^4%_@grcScFVXmO5J_5DG*S%?JyoNEvgjp~RS0#8OOg4dE6X>aZ0z7XzH$ zTon&|L##Vq&lYIG&G)eQ+O(W>2!Y0Hqe=|<0jOt2-_tkd`ZwK@yrrS#nU2)%3N^xt zis_Rem6gYueaW z^O&fYwU2^Sp$&~%9ep~4tb|hr26|OuA`Kr?0iU{_xoH~l zRd?#+JsZCC;nlXtCNk?Gkq>FdO)@` z!vZ2L9mM0v`tL?>YI7GXG;*fxbM?8g#!eBWz~mNU5X?Be+na%IxT!0~K~ z;E>^%cD5JoEJxdZEOH}Sf_~+>UcW=D2Dp=B!iNbBJ~9@Ihr)xx-L` zi9(q`SN3|2{al@+Fork5lb>4i_-`R1nw_49G=v;*dRGFcZ+jqM99$zdmk;T&%!&4( z+1IgapNtpSXUMNAOdUKKj6K#|cIG13eE^#BcDjdO7itle-^XQ%pp?~dXyPxJ#1~I( z;3|>_Q_S}X#)yLEH*M%cU@a9_#5GnbA(j^g0s^hhmLjPpI>YD*xGA9Tt88tPDD{L~-inv(6{>?EOhT@= zTk{TGW<(cqH#;(9%T!4e+?zj|X=3i^%O4hm5*sKXsTWM_azbMU%1Bje0d|hF#wS&a zoc`3%PaMtB(PTq2V%%YGp6#ET;&4H)OZ0lf5WpM+Q9wZ_PHr=J=O*mRTaYCELuAKb zAU-uQ8xzvQMbSrrNe*J~o!jmuv@!tIxS^Av$B*iaTIOUkE7E3s!6bnvtT>n12T2hD zH>>6IhrT`Mn!Pv#m10g!$}mOp-WZUPU8Setjo^jugpi{ix9sQ=Q#Ho(ni;8iuAqzc zWqI^yT7x6A&Jk7E?j8)1ndeV|lo|mO9;C;^i`Veo8+w#E^`x~SqFiXHLWeC?^>{_{ zkww|3Dm7!2R@4ebW73R)=Xi}cF>ZWUv(uii_&L0aZp+YFW6r+o195_(Uuk}&(31FX z#aj@fX!~SQGuW9Lvwo0KQ~^m%dri9h7+W!V4YI%!Sh&0+7oHBIlt&7S$D`LX!`(k319wpe{qm zTQH`}`u)a^)a{kv{F}1th%zb070&gdh)k76_H!|w?a~Xw6l%y%GeC9=;+M`}L{>ca zp#2%60g^OHhqmm4cvt|XJBN2fXxLTZc01|^FF+|jcN&g_F;s|^b|2o07lWA|gJFP~ z99XBEstOQ`Mwe)2HDg$V0wudk^K#@pojiE)){~|-H#2YzIRupRd9}-NEp?}NE4;iMP3k4P+#5%kq`nZ|d7 z?3{x`U(64kkyRKI?<9{khH#KyN4DMi-Ic*jP0hU0V0D&t*)0P#X4D4aEQgHzGkW!% zldp4EL`v|3(1C-u(Wa^+quC!ex5y?T&X!I_>J9N?uROhy;3Tc8d0b`ZcAk~T=71&b zBtfv3e0EalFw#)3O>#i*qY~ua+H-*xuap~EBxvPT*VWQ{W(|mTiYg3R`?0jF^`VYY zG&jaG;hgdIDb_fJKreyfCOI_0cWS~7vqj}Cxv*$>QfaMzvJ)0Mz-c2z&l#xRN>wD~ z(a$kzaA6=4S>K~7tOlu1GwW%G_b+p}RSy`NM)ARB}b`pQ+t9YQpe6AeE!SsU)NZ$K2aw-Wao7U&j0u{vP_px~Z3 zWx!&=jHg09O}smTgs||u)jZw}MRkG5hd^$hz7IKOGs=ji8Z%-Gm>Y2)fRIeNb!Sk= zr(${ia%-h*i#z` z;VRwVBL<0V7)+w+Z|)UG2o5zktJD%TeV%&~E`boVl6Pc2v6L+D7n!N)Omq|q3$P;) z8tyKog(N&x%_?1`=hr&tFzp>l#!!m^?jVF#ssb`Fm6LjYJ8NFb^~-tGvXVhc+HZW$ zYsKA}+%>*Dc{bRD9I<|cq8|q4mkm25%k7s~4Nrrc-WMn&R?KX=9j4lb2K0Aeh6}ef zG%DVKqo(ZXgZaL}`{tU=Dho}K89<(aqN0*wf!jf+-78K8o01cp63oGJ{!aUK+4|fkC!rfaaov#nV6eT}*Uhg*v`|HZ zp%5B!WK@*B2J#XTWinC)`jF-B74 z4P8|eoa#ff37!aA4*@%Oqq>6weULwvJ6xWv(mi%njw)LKT=^8l((AtCFe>~2imjRD zosQYk9dxc#QydhmBz|&9Jf|Gj@-L+DfS*MRrc~HQN8E$?X!Hgn%*y+-nlF zwi@oU+;KIKlU34iXG;Vb%oBw}aSTBBWrecTrOb(FQdpFxPo&m`ggB(KaZ%zPascCz zO9q}*{W2lF?tP+i%y!fi%I4=yk$tLUN$0YBMl&gm;RQw3A9)TTue~{u=p`>-&8UEj zTGkFC9l%4XZZbF64g%uNT`v=PR62Tvrp-)eF(#wgP?@Chdv*(M((fhS&q(^hDy5B{ z#v4~zU5#o{$b@Y7X7W1|t7R0Yx{q=1c1}hy)@Gfwxp(7B7E`HGm?nj9e2fk%Ypch^}6{2+AaE>oHU zx1uhUyWXLqW8ZDfrMX;iTvFzma5)TP@edf_oxJ+3R9-yi+9fQ8Ci}4cM#tcZaYv-N zJ)(?5?*T<*c=^K#DXB;>|~hJ?o`C;pyCpD2{GA7X`<+Ury^o6t#A9jzEvI3dlFLV97NN z5&_PS3($Vy#xnVdN#Z(*1fMSyrv_#KK-wraWV#@Dd&+yh&0d`=1!)2mnlkKP1H5~A zr&*~OPGkx{rBn#CC8)!?l5Vi<8_0RSGkP|$L?pBiWvW9_66lRaLQn^qT-{$B|tANBob?k4h6J&Td zy-B=E?T#;?b$&_teFdk5!^JKr2of2JcE=G2Eg2Yg0 z+{O>6V+%&Gq%inT!^RYnmL9U-9HqDuC)f#wt*mm`ZrkeW#fBHJKx&O8$8V1we&{yWT<0qZe=ig+Rp;68Ce^C_`GI;Fv6uKR(Kxm z+_;=89X2jF!DbN~(JQ=4Zq0qx0z0UWupgmR_Gy6H&QEwxz=^bYkysB$vr~{#r4y6O z@-v|+>%R{+S0OJ`ZghaphWkE+)D4Wr!3pUBY|pm=f)8Cs~FTiR+(v-x{CbUIpuX6v>_{%wjMJ&+2uV*1^lF{GxsTqEH~ z5+NzWM>$uFu~rGz*n@(=I6K*rX_DE8&`W`)96A9zg28J?LCg{gDrBmJ(_`V!a2w)g zGkdQ^AM0@s6)kVvH_M^IUcDa+g>kJF3+M2E`pa7A?VcF zjLIh5ar(;M*EA{vg~S)Oyf_x*yR(`a_mQaGKCw51QRM5$nR`!Cg+O>fE?o<>dtiq! zA0T}bA$G6Fk8DDsfaQtqVFh8RrjsY*ew(T-EUJOtz z96rdd@^@#nwGYC;}rhm~!^4MV50_>+t| z?XEl9kCx03Lkg<+>zsD5d&V@SeFV3Yz`Y4gL@kb5z2OIur8a5;N|sq zv}NgEWiwS(KeZxAw<0$o*c4kf&!Xv+Ue3t-zKvEenjk~mvN+Qs#Tg{)iotWE^_<~r z<4KdxRMdfr8qR2Vop!D&blcp@G2L`7c$9qAx)nD;3d0)e$gf61;tE(Q3xXWUe=?=n zFcfY?KMW^Ut*;F5-*PH5Wx5X`nWtx zK!swqo{KI~lP$=lr;yggPbi6<9w!NwwaM~>T3FVWu5lniu!<|A0aR1wH6n}yL?n!ORZ5Q~bG)j7={mWw1vBs>$#Vp$Ns^lE7mscl% zz=nzO##nX!dM}JXz*QpI-Izc^Yoe|(j90@+*GJ1kFq!HtyKKIS1}DP^nbo>CyE3Pt z#zHCtYGt9z!RoDyoxDuZyj>&-u9BV=L#4b-qJm{u0oFB0!q#_LUWZdqnw{ zY;Ktwv4HwZLL-`zEcjIQ(o7`|lPx69@JQ=58!4Srh`5y${@-dPATEn%4|lt0Ff^za z;!Ba+z>liA7R`AJj|-0%9!adc+%Um}Azj|&oA^iu5#UQ^&mqT;WZN#9=s09_PDQS4 zK>!81rh}?u_8R0Fx$<{&yE3TP_bISiEv&W~1ST`l_YC_gXJm;oz)BEP7$4k@cC2BJ z0sH_(cyfyB*E43PA#)=xpVR`sNZwT^%tb8bd+p6`-x=~f_ zLVY43LmHzQ++Ho@uXt?SLKUEnaS%zrzDzwDsmZ31;W8toK>qS zf1Kx1go0^f&%MnahD8GG7_DQoFHG(SbL z2u`m}X291NVF6rXZ*oI8jhH3D*>*+3`U9M0Z2*A50=RUGINhFQ_tY?eqe`Gx2%~SP znvBaQdTllXMZ{Th4UMi7SpFHpV@lyK9W^Y=rDIRk2ZHorp)w*}!q&Xw!KP?MIDww{ zLiOdVMsQJUP&zIo^kmub+DMp$JzEnHv32L2 zIpaL4yaVu{+a%KaRH~K=hT~5-ldz1=?xT={IV88HA86YWO>BHkJ~v?5Tp?%!&+P=6V4cuLDjc=B}j_7 zv~>~zc=qUOOY?H9@kQe7C=H(rnj%hN-H;cq8Izwy(MIC>9f1kDXPm&!ge^9w+jkMh zrPRR6ISYY~$=q~LVv*(L&N>zXh3~TNu&4?j+DWESCO1+BJIp4Igfc5&R#yqR3Cc*O z!975<-bqyaD*lS;h^(J2NC{4&S3>#ZG-?K<5)?!!2!t3`9R9d3s%u|3`W`$CW$rDw z8MlGUb>#@p6zKg22busx{c}SsEcL`7fC($c%6$T%;EX%Uz`elKnUwD;O?hFcb-`bk z7jXb|n;XAx4;;ks@> z%W3^N;z{nCJ2c0!t^q>~LzoiLE4+>cH(YO!hUd1nDv4TpaT;a93O2bLTX<`bF_V;K zv+q4nI%|`j)Z`Y*4cL0B-lU59TwfEyIJk2lDEm6ed%Tq_rmc_Go50T}M8!Vk;O`Ox zlo(8>lPHF(v|M>DADN~=s}%5`Miaq~;W`xPFzcb<^HNq(CvI~#-L9UdqPQw0k0>ud zf}N``W-mvC=#IRas9I}kVIl>&q~T$lDv>jr>6L-kUkmcRyBCIrNl9H3*kibqYa#ug zT<|`e|z`@*!1FY^>xrLhbMC*$*I9AM`%!^?T z=Mf$}vEWWXC_Xpbq|zRoo|Vi;5`tu@5J!6oa@wlZiFes31JlXTZw8%FcA3y`X{&ksZf|(3I1j?LAp1J(7LNB$m=oKs z$~($4__Ei!s>u9 zK{)q{VLdp$*ahO8kr0`O29sdVMr1Hmb*FOS(u5zO)U$^Dd3JX`VVuJevj`z%x7@}h z1PM9o*D6nE4L0V`^_#f}Y21ph5WeEEsU1!40fn{F@%RO(Fxci8XAc{P0l|Q69Mus{ zP;A-2&Fg~F8A1Jb(k1pFpJGo#Y26gO2`oGa<;^$D<|P!!sFwE`*2%d%P@DanLP6A# z*>Eu5GhtyVVasPAF08BzP|)=C&q9!S*<6AY#Q{n!w)n+7=EJ;gqnXUhq?5^D!YUX7 zQbuivQfI~xi*ShH!*Q$TG?O6|*;mo0!B8m^@^I=goAS}Tj7VfqM z?WjPA#&@Sd_uQMI&(LzBSE%>8wGZYIhYem#GglJ|!V3s8xZgi+4=T#x=@E~pL&h9e z>?hQMLm}<61qUhyu`ud8S)dpaq2rXPDZsT^rFh!nJ#-KrG?mMSl>yTGUU^+ey>ttd zz1iUh7M?t?$plz{@ z^7m%nlqh+!LZ}l2&chg-)F*Ijbtr4rWVo>p!Z`GTm}D5RBfAV>hBMvXSpnRutbL7u zcK;HS*pjFo0D|!!i=>)ok;fSw=M9HcKw`7e&rLw57h|zO!YpOhSKc*7zAw&b3Q3AZ z_t%>!!MrXkTeX%QdxuOqI(%2A5lSDuNl5Z|2cAb3d)Y~;>!*6c*UAXO$M32eo?lcK z2VW*!sbl5%N+qgs*_r@HK)AnJ+T!n)!d{&3XB4N<%SE%k+h)^kN~H)S*%fxl0!(;> zn_Gr?)@~B{X&`}dl>i`zA2-JIHOg+&9(;U=ZFt9i&S;e(WcV-{yuk0BNRAF3W@ikx z;V(4d@C(}r#R%1^u4j$7I~|EWO!cpuT37DY&7-iAosOXE`%f-6sw*0Wy!z7CE(h+y zT$EuSRNJiFoxswPnY%}>&w~Z45JQJP9&P9cgqe{&R;26nuxbm}A-5i!39Rx=Y4?rf z*PRCP_igGaI3q8ZBMGHKn=sP`R;ARwXPqE2LJ@vTUi4Igp)Xfmw?eX27_$$Jid2Tg z_{o>dtrS_2IL`=JHGV$ogroZE*J1*hWENs2T}hcW0}L=fNXU350V9x;)#t~b7193*wseU zHJ}r(Yi<;neN@mcr?vz^)eIg21}y80Cl5q=fQ3CtDaSaIMf+E!k1RKQNyImEMd+}NWU zX6YE&nzAocicjDInf#tqV{PLSVcuj!P?ekX%8Q? z*(P6b5Vi~~mvgZy6CJW@z@P~4n%A|HnUVDdp5)%Av=)T6$ov|KEWLYBN)iKuc9=V; zf@2N|;2Sg+8mfq0S+SzqsX1xU>zz9l;_=kIWMQOmrOv=;ewR`?)Pe&VmM6MO!L&f9 zw@pNGS+qT`0@`AkG~A`uwoQ!@`21Hj}+v$+zu?=7a^kRiasU%4RiYyX`bF>W8 z7e(1~T-#4_4%iG?n}~tmQhX;Mv?%lJ3f|n=cavIyAuGLt69;@&`XnvaC0Pc%gNJU7 z^g(H!zWL^Kj^=4IZ-IHTifKt%GbP-H}HIHE$!I~JYL`X?b7$QCx zu3A;6R&Ppj5u#9U#nFgW&vg$aEttdC&}@;Itu;J%HStFX!ILdi^fxEOMlE@e+O+~~`-o>h9lBlC+hQmy54>FfLc> z0fWz10ykE0f~dlQCe0w+<>R7zO&&puGOX~gq5C1+8P+nTUK^x6je_^;SVIx{2=bQq zxPZpQG&-@EO*lm?j^sy0#p2xC(h)~z;%f&e2p!p{h!(pB-Wo1RX7u&H(732@dIl#= zRDkQ76EA5AoKIJKyHtR5HjWqVv+jfnmITTv>A+{|B!^06*gscg%hqc?adlbl-rLVJ zN!{PUypyQwP`@aHfv9qrVlFq!)^z40V!>FBF_0?C z^HV6`ZZ%`kEZ-)%YplpoLRJMwh!58rU9_1erIi=dZJLTr(11Bc7^exjVJNt)8U^+_ zt*5T+`0+H0F-xyO7Qv$VMK^&s7`x6s5QnJmx!6uMVD^}1=3QY zX4ZnV;yta>zS8m7cf4!$l>JM4rKI)1uqq75mg&jNhZgggFmU|R+^7NdfWa~8=`G6p z&M;M3HV({7(A*5;UW2_>I#vs|IVgc+z9isdBM#`q^*lJMNV0adP43)z%w)EMx3zQ} zMlS^TSjhT#yq&U)U9>lW=hqz0XB?O*)I9|)rK3g?j0HBeY2Ki2`5}qfb2-(zB9R?+XtC0YGteq9G_3HpOd{K)4~GeddRxCEe z$R^B2L#18|wDsgh)0GvWRc%b1TmmS=sX(0KW# zP!b;4Pl8X!FWYJq>ESD|Ia7KM2XV%x1se86DDJT$EBHr!bEXsM)h#_95`3^4LXBi2 z!7nP?8$E+rpekYL;$y5IY``Hk=mskF%L*`8*1}L1)~bouJHkQWkE?A%oBc_G(q^a0 zs%ye?)a~VGpzhpd21}nTZf`vf~Y%=-ahHF zgXsgMCqvb8GouJiU8YZT$&BT!KOr36H`RFh6kiGHx4UT^)S^w?{t$s10c`OJx3sOw zdY-y)R@HEuc=#1MvOsnLs#^24l6@QpDWn2mSOwvW5nlIKT)U5n?~v;Kgxa!L!Gv(x zE}$r2YTDy~#uUe{dK;TCs8TUv>+4jiC&dby#HUo?HpT!na-mw}UWM>;f;HRjv~5lyQ;=UZY1oq1XN^wmuged%-r9d#XHe1xq>K5*O4+$gLTkNG~uX{WP zl?mNWd1T_bsQ)4pV{bC?_%Nwmz(dltPQ7vB;x2=>xp{o<4pKQ&(jHNEHM3>DF1lSY zpDctZ#Voc|2xO|s#T)vGhED2eT9C^EXQ#glk zty)frvEd=_R`gsz&;%JRb)CPh_PKEgP^Bbw8Jc!DVJGd29R|Rz7U|pcMIfEKW3^#5 zc|vZ=T?Mm{Oz2(63h9jRj511THp!Lfl3kasXO!JGJXtC6kc=-LZqKQoA>Smf>d!RM z;d@meN%M&-5{o)ZF(F9-6cg`fBe5L-hfUWJL`p>$z%9+t%dxL57Cvgc7EFS4aTMD; zfDY2toqMe6B2!H9XkZSEmkgrV;{9t*<&bSW&CX4JzJWZSDphg;sNsoUldc>CGXTeL zNK9E7167XWMFa%TDS-xEu|at*Eyaln8wZ#P1hIOM>V1LKd2O10KNk~9UK{;$26`C|a7ADH~;)SVqmeaEPWFdk) zwP~6*cCu{)aLcz(%p-kt^Rk#r(jb%?N2@;4aM2mntSlbcA=@?&tVYsr(2@*6xk>x<1wjIIj0){vq zw8~IVBc1BZyE`DlY|~^_NrOu;?;MkFN%L{+Fmyx3;*VCTxb>z&YBZmj+dgvL+Smm0 zrJ!U6=IH>2j-$?fC37PVsrRh=CVMFmEb}+CLzPaGDIap16z;L!52?l~u)Lap$+wF1 z`PjpWTb(%&BR7u3D}iXLI*JQg#Pob!0~kJ=I_0*M=%i$3RfzWk6Hyq2O*2}kRdxnf z%_Lie+4s9`vCpV#zP}iVGA`+48IpvhPn~NA3TQ;x_ON3DsWp3ARfWU=eAEg>iqR{f z%-brnU0=a_tU-AHoUzWh?Nd|ddL=GE%N$(0O(nw_#nY1*VUr}9nf-1 z0x_&~GNh59JUQ5j!2ZuF{VoQ*?MJHhu=BNJ6~g8OkwYuV#8lu=F}LHW9D#EmUpc8M z6$p#28<-Q4GDx)B4}gzP^OM0qTQ0^uS3`SNN|7~!)LU%%-^v^jb&gsOJs!UVn%i0J5*2TQJB1;SIZkANj!(K!LBpCbEc(2Pa6vIqKDOx zn0kElj}3!13Fn0)5g}pGW6dLxMJ1xBQaX0k5hR_n$KYEEO1rlQ!un7&ASw5O1xT18 z+EV9MXuS+26N0SDe7La8TN#q|z;$1fIKj7l z-t76i4F&fl;9uJU)(=#36^9N*X@k~o)|0X*skHAV9&F~_Bfk+<12RsQ@Jz0e3DOh0 zRvU!k10h=7*=WKBZE6@;=cwJ48uM|yM;|{vUsf6!RF|J8XN}bM$~OqmX*Fy<(!B6d z z_rT~k)`Ue1X^!u8M1*v}nPIu^49RA*fW_8v7lw=wg!nWyu-sYE-hv58*jGw~_Bd&R z3R_}0(CtSTm_9=-%~><4pD(ehBdfPO&59fegO<%n66k|1I8Z^A5#&KRvp_-Cz)Bbn zl%*bGoy~DD%n8eyc^vB{P1Vr|`VBVwn2eB>u#HRX@awG*u-}i>RBnl|bIP{Gu_xMP zsQ~Ss>vPgt$F_e(xtK`nrdMA1P5CY*09&_uy;6!y>wb3QtHqchc1ngo?;op6#3>+* zF>THj<~86cY-RSb&0LH-7sKwNZ1MRrSyyfY z@@1*UvLFiE#nK!-Hw0*!K9vw(8z-U8aBFizFUswMfnXX*!NcTM437^|$HXgHJ|as|Sw|Pb+ftOn1yp6|tgYr$%_L^2bv_|{+atnayFLNh zYsd>j+aOtsQpqBH3Gj*o)<;0Ci_mVg3T0`mx}ale@aIF^C{VtTd!i6R?)O`3@w z$V?@RRqA9Sw$bGZ=q8aK1F{8k+d!&T?7@0Z$b1#uurOhuKVXRlLyPuWNY;f+$QKpE zG-+dJuLBhbFNKOvl7z0bFDbKH$aBy=wvvb#<#r6@2SPVrWWx4}lil$yZQ^AP-=j_= z!k9ot&|66f5wW=<^pSBYCrGzRXYpTmzjBW&>{1wl5=v;ip6mSz6m<{@S3P%FIBlE4 zI0*?G&aub{3gDJ$jcmvRnnyl3BIZ+z)t>}pOv`jlJoO@R9w2w3vtv92kzZ97_E=M>Ia%l3*jhAjwL zfHM`|&TLygmX3<>kWDj8T~sWFN=k(HM@pGTp|GKLn(PVcQ)eF5sH*m1Rcd{T_98Y) zfb7FU{8_xPx+=Cxl4a{T9T^`vRT9m0-WuncUr%=pqcOmeAJiJmYz6Uyv-BQtz#Xks ze)p}5=P??u>Jn}=YMYk1n7Y4jRzIw81W*P{Zz|njm24fB@LYcD-!|ZgTO?P|&!Se- z53!b9^&))cPG?>GEuhS$YqzbAmU7KzN(cc%uMFx z1*kR3iJ(B->S57v2&hWzu6o8M>dF;orrvZO_Zue`*H9!Mx|Ck@s2s4hB3yk7hV*oi}joS`asFQou#e z*WkLNiz&}|U4TX5;j^C}#}ykF8qCdoLar`%SZaPWssmUoTwBZ8psxX4MjJ6%KikX; zw)&W6y_U-bmOTTJV;bGnSYBSil2Y0RUBD_vxUobvW5lvD&vfScoD(zt^2EYRh|G3#UNBuvo{ zWm@a-M#lZGN#T_mNvtPQocS0byPPge);Z6H0`zsj#t?2{Y2k zU<|?Zz@9+WCc%TmlNzr~49*`c9u}MSCt@UxM#JTn&jvxbh}N(fNO}8va&h!Jg-RF? z-lf{t3kfMS2c=MvC7!8@0*|AZuo*sK0g`hR$zi%0H=ZxF-4v{F>~yV@H#Vbbwhctp z5H#nX5krc*CALskbd-Bc-TBp5V6D_ObV7m=LG}3=*=p{~#UdOG+;}Gqz3d)tX+JcG45RZ+ zdIr5Hgjz{ic;^(or<18*i4+_&8bbGqIutn}Ge5o|O-H*fr^9HyOs^n-CVI;?e71SD zy<(-pm&@dZY?V6$<;f-V(Q-BGN4zB=R?iWEl3f(=DAuV0;)08Z^v;H(z69I5Ms(;V zf*5Mz+`nB|)#MXsBXTk~drGiENR(=A%2d83NK{d+e6~j*X!13WdhRT&{Hc&QuX)Xb zoSEgEP4H>1h2kCCM)mB|I_Pi@Ety^&1R>Hjc^3<cJUm%1vtMh80)~+;VvN9%Y!E!<>+dUbI;5nm$v3h3D(A!4`dGHfoKODBC0X0 z{Rnh7-AaN&a=5sT+dom3Z``2Tudf0?JGI6F27v+Nk)gX|$@s98Qw_r&vOd8(EKfKT zW!%mPEtE%54ZA0&KmEgGtirX^qV3?OLOalX2_MpIcyq=yM(#>a?F|HEZGXWD2 zljDf?gc%cnMYb=QS*f&_(c9S-PVI&>L3Q~X*51N-&7f#vG3r1W9d>pcm$j^wS4Y`u zFU8@Ia7g4Gobi}*-ZKMakC$bXC5d(2bQkUNtYyes(t0#<1lGO+9AO2t46*TLp~2rv zKuX?Y2P6#P)2gGYM#8gg{Tdzh91O9XU5ZP|L zQ6(@eORW>!HrRAjN|n34?AfQFTR3r|LLYakQj1c-m4flZg!hi4I?oVxthFk!2s~j4 z#x0mnfjFWRV+4Ts$p>lNK#>KEFj~c}#Y-p~vgHFILK0<|zPQ5Ui-VxCyf8r=t|nhk zhr_*Z%f5l{vO|zu8^{#|`-YG{qs|s^zIo1`ary;7h-;14B%c+YI4@6cvVe`q#wb$T zSc&xb)gx-d4LD6KrVV_qfl#nqNp&@0o^<##_Hfw)O+7? zrR3H$vtf;`oUsKCOt;h#AA@S7$)I{lR}c}my%HTA@gf%xrdy+WE?4r6dqzz zuOtIVFu9W#k@B`EFpW7rhTWJd=$pg$DQ8Uij}mAVkj&m<>SNnv z0vCZ@Hhk)#-d{40acB*Yi|62r8_H_xux?+6lv;Y2HlsDv6uK5_Mia^Dg7@o$V1 z2yO?Eofo3#*D_rSN|rWYJ$vlQgGWF}@cU|$YqO-U%_Z@APewZqdI4Jc6YrM&`6 z6UwUF<&9$!NGG#q`6B^qtc?A~m@R30O{se_ow*xcA5E^41@`9R`&b8wCh_q;T@lTf z@LK$M8fY9?BTyU%YD4b`xZ9RJnYgGS+muY*fqg*sNHRZh?>dsGYA_YoizL`R!#c^m z?Y>(vT^HBx4|WQI(TV}r8Bp^mD+~@>wbRAQ()qcyMp_9&WY^v^>=S^ZPacbD`ci&a zETUzWerW8{(8GzL#fqvRb^|-35KQ+nwRqbRXl#KyFpYp1atWCLp-Q$h3aYngiNc&x z0!Ot-0>uGpvCc03=jld>Op#cF+lr&`loFf@bL^DSYs9HYITa=22K`Q@FKYGAou=1q zA!9wLfsI%#daBF?<*APKkZCpq2L=!?*&2#8%frvAr#9n)Mujdfz_RfAvwM}2iO7|4CZGdz*N~}clWQPJ7_1##q=_b5)I}hR|7ABSv$CBZ| zblVJHYgwDk?6y_6ITOKQ-hT=Sl;IGn7J&4w1?1&^C4xt?3;i+ z>Y^e?cGbgA4@8Y!K#v0oucT}0#RHe{okd?I+5u>~`;tJ&x2OvP0Z$`#iJI3-uIp|# zacdM|V|P~&1qevmZL@SOIqMzV{8^T}7z@^IGP`GQD4!)U!#ULp41_z7@MdzqD>8QI z44DXpE-$XOV7MuW?zU3e;_2gKzc1SGBJBYKTz${zbVef|#G_IRb0EuTC z%a0T-7~x%8>9rvp@sFS{pu&l$8{~Pev1CY`U&wqBu4OjVF! z{RmwY*7@<))_7D&L#tC$iNx>75PAs7!;x6P6Km zyA2Avg@bKEb)~+G*{tK(9!ko)35ydd%?77@sZ_XJuoi(Dxv& zkf|#PBy-xXP5E3lK5T@TNlkor%Ek)CFbt~^?82~Q41jsBjk@jN39Um1LK4rYuMt9L zw7O+do$07PgztvR5#71aNF+>G>+u;ujF_!c=(f2e$h_u4G$Q6YR6HRAFnCOrg(0%= zQ`8Ul4F$MXD-cD-T(BEHK+s2G`)KJw+jvtxxXjO^f z0zJfqiGz=n@ZtoaVvqwq3s!>S`Q#E#gB?wcgJ2MHm12tngFkTJmsT|Fyd9h?)woc2 zJ#pUk4SKy`^i8qQmup3HNMI1v>4WzCY2Yxhu|>S4pB8z8C`v{5ebG=(v(X(#}h4>)x zX!iHhMh1AM<%2Qr6b*!S;yfElDGpWfA>5K%eD_&x!O7YX`9X69?=E2rM$QKZg_i{( zrX)>F;K3!|2Z@AznOua({aDNtDRKho;J%Vet`~j8d6gw21iOD^3~;4(j_Q20M}{4A zW#?)Uz502!?veKCLA41J8wi#?8-_(2qRW3kFP%=W;HOX;1-{)KD0;9y_N&CmSa6KQ z_S-nzB|FFx0G}POTCWL2Whm+poSO?C93WJP{Zh8*%?WyhwQ8~3W!$J1j#fPZf~uS- zM+!~d`NvN<3WW7*m$Wy;9qh?M%Lu!Z06e~FZb!W-;A?Au#i~kW+(^{y&zdL1hD95Oo;fHy z=ARWThoH)D*fe$46grEeThNAZLgMzYRWs=u14Pqav))DOP7XoQ0*_=Gg&-SYHpuM0mcXr{qP~e`dzsPKzHvctg=a4>gAOCf1zRA(_xb z0q*f+T4O8^q!@v1@~a3dD$TuQ=G$f%DPAGdKex9gZtpgk$oydKLNo$dO<;Dp9jIyc znP`Aq*+cF>I8+Q4dt0J>s6Pi=xNx;gmgDT>vwlmkAMS< zo&ux{4tzPl-Vsa~#5l+HPJ&NMjcg=sc`Sz(_trpALYFQ<`t!>h+J%KiDHCFxlBa$U zt>73{fNa3DxEK^QLSqQz(z0Sa3Km}2(ZGG%HS&5Hv(vb0A=cxSp%%H8fYKl*VR=!R zAw3aH1Qt{z41I|W9#y$gWu$XTt*$h{Q*G%|R~QV|$DI0vh08*b!eACMR|n|8=92nDpp*I-Ik zuOXM`4GkTg5hxXl1Sz~=y&o*v>sdJCKUyt)!7Nt?XNUliu{xzDPY8$fjPOrNRf^ck z0CR(JChI4edTaStiiL)@<1FGji8*ptda3Q_pyQ;=1~Q8;3%w|A(~`yqhD6>TAh|tpXQ=&9`tlGre0DaDL3l{QHftOFNJMlB1suh1&}y|f*`JO*;T#Tf$an1{Be{bK$%upExedmzYu_Ab%{ApNo&V^uH~Pw+HGw5d`@W#$22 zg#audW4YF9-+HzT!`|!`j-tp9cKAIlOb#sL+r3c)bm%I3R*IXIcQGoBcoE%j8NluC z!ftM&J$yGb20-qxh3*2|D&dF1^-;H3N>J5;V~H|vZjM)EEN7Iy4g-@s$rc-=9Ea?I zIAH<1GH;$5y994dwX|KD4MENa64UP~GL$I&vUXYXt z%sfwQIaj7>>7TnXQv;FS#hW$I4f1{w`s;gLK^bqom<6ULG#3OtfgMVMsU}ei!wjN( zp_vY|7(4sAqj`}Da!%sAOHN4Me_fr(o}RzD!nHcdGq;uZp(f4N&d!n$B!`h67QsNS z#_q(`*nB$00XZh4B3~PTi$O!sfME0&caZ`<%{2xXYVBxU?^sS6?pa~ArmW4HtA-N>osY85xHPP2jtUfBfH3gmzEnqm~7@97KBI zX>I84QMum-A6EfFSmZMxyRpf!-u7JJ!mw`{MAAo?5u6**M(!sM1U0__X;j;dop37iuh?prP}fNOB>5z z!`mA<&uC{S;->Dfd`Z*?XFk2 zfwD8hI~GQ0tR1*!j|#zSu5M@==vEaWk7PnK@!MJWcF(B{X$D72rVuI}(>Ojx^I7x8pgTAVUS}WgnD`cDphnxYmsN; z=-j;rR!Zf}30)5&g&{f$MR9iMMNrpRc(<9`!1>sdWR@dd38wS%-q$sD(Yvhg$YJaZ z;ZtHLfIfQgZ0C9<0jUefdC1NJ#->%=WmrIsm`XG7A%0s2HS~{^s zT8foVlG!stPu3kmhKzl^B5e6clK#fv9T1{{jvCrF_RPi1E1CISEVpQ+Nm8=8`b#wr zWdkBO^abWGV^jc&wwcZqsp?X0NmwIIWL=cgnJ59cmhc+i+%!eZ-pPqL$W6%?m^I46 ze05O3i-O_N#x94ZN5f-T8oe^E#kKKM<-KRy}RzV(}&UHV#ydKItMX|#ggX0l0R!TXA3 zM&fW7Vy$8`7OK8SVnwX=%=>-V`rbD)IIwu+s&2f9;Xygc=kfrn?hI+Cy@N1JfXE)O zVF7492k`$mL(uR}%K-u&qLpVg>3c9*IDK4!m(u5aPAl2eoK<;}LKx62rJF%0*vl1?4RiGG9gL@lE27@RT^r8~&s|<{0 zohE#S>$eI!lv#LhhTkQ-je9_MqT;(BfMmSB9F{B9W-#(Kx^|Ha8L%^z+>fL{>-vaF=ongKNY)iIF%QU-Qh*PX9W7}w5rEJtB)29Yj(uRPnZ zw8}cwGX#dZ5O`RN+Aet~M(azLkpMb&`$rB3WE3_KO0wp#tvh&VAuSb4Ug>WYtA%!y zTw*}qxmnVad^Kw)2KxB5@!7Z+-ixynO3Ain$C-giO`g1^lj^S_VvV7z=;XcD(1m1r zn$EJYE-%{x@7{>R)@o+!&{V7)IvtY&6N!xrCuI4cL`95&)c}f04T}y((7ehHzUvD~ z-r<_lnVc$eD>6l4Cy+hWScqFT?BtBmLY++XiDSJEdh6&OA|bbkxru>9B38BxoAPgO z>KZW+Z)Y|^5h1goeIk|70u7R-BSuJTp59uERSQ>agz;d?h-kZSBC6YVT@Q7)z03E# zi%A(pUpQmesuw+{v1Ps3Na7KtTghAlA|heNB*aR7hg_!0vL9h&=^)TZGY#N*SE{rQ zs-J$qw;>GGlcg>NB1D}Za7x$J0yBcfoFWei6Vi)J+}t*Y>QZvHONz+>z?Cw$0iJDA zhG+{94zyUclie|@aNXy5#4^W>a>C#>0TuGdXlrZ+3slIECE}DU9-N4WECpO0#sHeg zY`WO^B&_lrk=&MVDULCw?GgdR9drd}7~-_jg~y%jT&&g@N7R5M3Y8+h?q8ameHju1 zcEM<|=|RC&NzC4#VD%bQ?@T&IlP^enYs+yMLfYQ631^g?)F6uAqWhLG#K3 zPoox6!yeq=;_!P+H2Tzf$aX~E1`X#9Z5ZGoQP-LfOk1NZFAcXG?I4WiUCiLqjV`_) zmGm5b6sF{}ir`*=cks&cn>sP5oQkB7DTSDDkGOTHWjioa?8o-^iSy2cbd_}FaI|Ls z8m8_YS3AhAr4{Ic)Go-&oQV*k2uAm;T0pOxUw~C1?&?% z3KCV7jY5YTvd#&^9M=rIqT*+sxfI-hlc5fl3uXD~Yq>&xL}C=$v? zM@fN`5ex2l$PkfRwb)GI$5v^C>gbQcYo2nM)K(r&ezzvYnpc;a+&+xB(|Dm48;Niq z$#|PIo7=->=#81uQSX$yzC;;px)-QL79vakg8?8dG+e_uG<3^-V3#kx6Fl{V=#gqaf7ZMp{YQ2E~ukVe; z!*Id}xTxBv$Ln(n==o+iZyh_{cvMT#!s>i|EwZ*csnHys1V%cb_!RS=Y+g~+aA3+Y z6`}V7XvDmSsaE*2;1;$@g+7*4Srwa81MfE&q2Xp9HB3{_lXsNG13il310jLYR4t(4 z1t7%xek;O!#ia|;NJm>`-7yDA6MBp<7NY8b1sBkl@{im_R9>D<>!gD)UE@RIc6GZ3 zXdoo_!GQw^@CdfY0;AlJM6{w{b$bU%cjS$>luW-H4$`E@z*Ct0k58PIk~bR6`!OQ0 zOwXbO{dmB}ZnT7o1T?Y+>r{b*_j>LXq4{A5slwexK}6h7ksDeXNYK$N8p|HTRyujg zS}P?^F1)dV%1>e9=e_L0`-RWMM!mC#Ia=>_J-FVu7L8u(*R}X!xUxex-IFmZ^cEUb zcLg7(J&ogj1>rWb>~0PC$_MDqpulxr3XG@)wsu0IyW2uXgITLw9-Jg2wdf1uJ7?`F&L5!=6 z8^&7ACfD{JzH$p7f;I?ZQA`)fDcd5<`f6*osq zr7L%V-2@HCph8EEJ`JfaTbeRb7l!+Z{HA_;a)*w)I@W%o^Cnff4mLvwpmgm)7vi6Ci3YF;if6s$KJT1KU4}0VX;5FO!#u@26RD2oPR; z33Oe0Z|Z@JDGHQEcNn5=PeKk!jRO5E=StFAa5iy@TRatoyosrF>HX=lB?k(Wh7ZUH3xg!dS8-HvxKq?mBrrnwrBQ3F3XE)baVuSrbjn`W zr?4fx;A{0DNPYP-;&cXs&H5ap$W&I4f;Tx{jhVKKEW zdA{2NRBB6yb_!-uNYVqfQfKjyncd5Ux!qWzZ{oFh)>DoGGSa=;NgzI(T^X zw0oq;Q4W2!h(crR39&9RX@N{Mt!Eu9DPf-qrsj5NUQq-ta63{8uNgvIU#EQEZLv|| znO*dAOyz0#4COX)2A|4fqZ605DA6EIz36N98^~Km2uSZeB77iPfl{SuTDG3z2OQ?@ z>$-NehV?CCm&7J`O6BWf0W5RnrEM%yR=rya-c(OI50}8$bxj%Iq#+BUhO4*jHU<`w zArlgVB5AVAmeck2`gE|i@El1f*g-K*g$I{8)`<*Qgt$PWXtn_ucG81jNCa?XNXT{2 z_9^F|0wvJ(GDh`pI6)l#qC+tmpo}Jio~jQji&W}1v%2gzHDwBfR8|DvHfr@>HkTXa z&K^%79pik8_GE;DfG3B-f+*Cq z9a~j%NHP~vgrIqguRX_yPG(d(%XZdC4-kdhsON>2@Qbs4hIB}mOl!1jKnZ#!DAkVS zaxq~9F|g4}{XA*us;AkG-KBjVw=&w{EL zlfa`nl8OZj?84M3{kqyXhu5L5j$VwToD~Zx?`=mGBUU_Iw$WrB%O3|S>ytRiB{qC( zTr8tH4*4sd42qE;H%r1SU#Q3m>Aj;|tj%@CyDXr!(uS1W6L~P7jfFTB!kXLS@oDa_ zy3vQ0?F+{*!8w}L33d}r*EUs!5PoFLkD3`f&_ZnLrMyp~O99L6x=9j7tf$+Pw*vWL0}l z+Cz7+xr~}@r4EL5jUQO~bo^#p&6bxd`j?I{21QHpI2{`8qay8ZKjQa$knz*QK%`r%u z-mcYESA_Q6Eo*ttctu?_IH8Hy2vL6dbHP1};5}{yJC)}k6=4_wnn_f$S;PZVw;i{! zHK^=h+|F7vc(^I(!!vA$Dg=^o>aOA&1-pjz>J&w5=|0(qdzcPpS&Bz1l#Sq`9f>oe zE=3c*qfKLm4Z*q)U~ORmW8)CP!7xqE z&AAhD)1@@}#m~$jF3$K1gSh>Qr3J?rK1vbrx=c06h%gZL23;*G{S0k18YerWQ#jm5 zwsBFBXTx#`NCHBJdU^Mn{k(cG4b|hfC9?HRwpCc&j|=mnAXxnT*zz1h0`YpK))f(T zL8Jtpl1zSmV{>Kp$UCX2I!N#t`YchOLPJtn(lb#L{oX-Fh*`B4-Q#-Te?g5hZ7UXKmpu-;n8{GOSONNyP+e~z7>AdAmgE5_CmOp3@hT)cKOWmq zm!!LFOI))~l|E|$Bni^Owf1|W1RJoVdh#mLpCdd)Apqli!x>@5QSu#F*t$C8cLs}Ci0nUdUmHtF@YxF?#cx9 z_!}e~S`ra0kb@JC)t4qf@G?h3gW*dIqu$ke1=={SedSqI+W7{dYSKGN>bII6-@hy% z@V$m|N=rNQumf;D%2GT-M3o|!MzOhHqzUcEaqz5NB%u3%qlg_}pgYi`Wlr-uwdoFq z>KnlFV851AA1sWWDVxzFF?{%#@bd71Qm}E-hMJOC+GttK`#% zJCJwgL&es_G@_YFNF9nrd=%iy<#ml};rj~YM)Gobx=VFV5I3a6>Q115UT)2qYX?@W!=OQ7ILrnAU(A+fB&n2a0-p$#%Y5X>#D=%mR6Q1T<|Lr=)ue# zmf#S8b*W;x^k^q(-d{NltnO(j8X9N5KN~?cxxURPXkQZ#MwjCcBPVRPZfr{ zWy5@}N`^;F%q%Z8EtO~=%&_d4TkU7We{`v^Spj$Q$hu>)$;wF5g45MJlB+IXYrUi}-{_(qCm zD{rVzToJyW7F}M5$%=K??wuN5yX1V~@PP87Y!PmXqhf6=tn>TDD`{OP8q%(V+P;3N z$fe3sxVR$|$Q3K~*f2P)@@p=>Y$wJux>*NrMd$I!{OR9g9lT+UQ)}8lw3lr?I6@8! zURWN@aH9(Zj|J>l>9}d@>{_Vg6~io6g3c@2K_SwdNNAn{p#u2BJ#&i{eYXWt%pANA z$WZ?D3+n5)jUi4W1WlxcvPYuk^+P`3`Nh>%Sy4|Lx+^f6TpzG=fjL2Y9a8fqgfepTsl#ZbT_ZTit=z;B z`Cwx3aO2&!M9eYh!%j@HdA9pcAb0G!^#jJ1i-%1gY_|zS z$7Bysw+xB4nXWBF&s}CbZ-)!-%un)!Hh5P=fN3FBL=qG^)t8kqsO>ol3{rhE_cZ z*vy@l_1${3O~+ro#Saq!(8{V5KzgXYVM{9@=KIS6LT$ZVb=a1jrQE;TY0$uaj0!20 z6cm``{bENmF2P*Eh*mY%<$>&>KKF1q(1LG47Qc$=rhtPaT;i1c&#I;>6vmBQw}}(x zyC5RP@>%8?zoroOF>AY|CKbM?@0AO3AVqizIS04sXA)S)y*X8_i*#(!MGWk!lwL(v z;h-{y1lK-k8g*NGnEl7fScl{tiw!6{CqWX9M^?9*#o=So!xOOx2Dp$`wIkGHW7EHr&Y+*1eN!^(7!lnp)e&l95SuW%F)rgXM~%ZF{YW2M&0#a#&r`CSu= zS!;^PO8vM*dt!j@F} zx#JPHCkMr#vdA;lX|c=o(Ze1H&jf@H7u*x8hXJJ>9m%?dmQInBJIQOFYX5c=*45rRl*5Fcr zk7(>fJwIj1yEUWn^K>vdlk@mO;ZkQ+x5FE%^qql|ha& z=*9ww6MZ8I_a@wOo0a81yp%R^GX&Vl3Ux>^nZn@32P0-9n1s`-cjPWekY2-GO-!5J<6)xQ7EMwL4Fe^9d5MP@Hgxv!*ER8^qWy|$ zX

(X}8!?spK67qipdjniQPEP$K;YI>ft&FW1nGamC@GToy!52@QGo5K~KO>9v$; zbIheX zV+3jBa~)KXp-8se5R%_2HZaNh*dr-~iuI}hRAX>8y4bn++}0`(#;c06)sWtp6`M!L zK&*p6wvIMtbS^NrYM*DPd=@e-!veVPgDs_;T2~Q-?w6f5fcnjxdqbFk3; z?t5v>8O^0)gDXv;M!S~#>2t&crUZOZ=2Z=$l)4ptmSs5CTAT?Tn=3lytE=OaKJAcT zm3fSFN#6=IuI8)8+96TeT;y;s@ljogpjej?J~O!NpI}p~Alx$KzdIbnZlFwctRi1z zVaihWT^dV>ZlQ=~M2VzX$PB7DmQ7xq2+QGXuJHC08f3mPGDKy_>-x|jV)g5jS=9`f zH+LHoDLD5pmu@-O?Fya5y1EoRbWDDV-tX{c>f|4_nKHWMhY6qS#m{66;jVmRT#^u^X1! zXzJy)zT#T;Y_n6I!P1wfF;MZ&F=SQ3gwP1YdO4Pxwb7W>an{!I^u#PDY%$g81u4a( zV??n_ypEcBhB7d?QRO4n;)dsIV_%$7RhZoMUBC`N{&#=m_T>IX)=Q zLaVpD8^@f}7@xs8)Ff!wh%MvDefL6LEuiiLuS7UE;B_kO9iaDc=o(cMhB)DN;m5PR zv#^5_V_pUc6++)lf?yYLOf?>$sf_gK-W?M{Z^%2CWtb8Ft7}#0v1)B zgcU$!O4&drOl3~zw4khVY-o0Bl4+@+jiZb*1i*XA1_dfZmE*Q%i@-71aj95AtVE{{ z8Bv2JzCQ`LFW?w=g_kwy&kl2kE*1h+fae2I?(?Ltw&h5{oN*(<71yE`%93ucd8KhB z(B-rWh~k2!2xXn)8TLaEg6W?Jt^QyT+~N>2;2l~e#(L_$;tGu-5nOm_gkpg2QH0Me zx3W5xlh`kzlo@i_39-tO{`Iw6>6Tkxpww9?b8C~x6BZq}mCtcY#h`y_RXRt*f;}`R zQcL?Bd7+qUqeSF%t)*-EKI^!1tLJ9PZuf>88Q z8|h`nZXj|VG5t%T{lne*YnjzG&~U;OU$!y3a~5g5(S4=LAm_M(8OEO5S|rH7K0XXo zY$$hA!LJ2`tW_M;3@OM*o$d4#vyqLb?w!a?e15ROAavazTKROa;?$eMd!+E0-D;J_vw-&gwS1 zA5>;CJN6v~lbCpH?ATJrPL+e4YII0&yO_5mJYBX+Q2TfLqc1WWr}L$Ho@DkmQOn%~hgCz)woP}E%D~b|1v(16;GusSScE>|>&g<-h@*(n zac)?}5H-!nlat=2PX~1b;xu$3VNJ8#zQU~)2CY zn437e@0~jhrrbY#{{WfRT>zgT#U+FV%M1Y@w21hhS)ToSQJgilK-gjdB4ujM(F?8E zql^S$;Z>U?s&>(uVO2KXZcrbJb?-w*Lam5qEbXF|pFPeYm!uGyeibV5Y*lXX z(Xxs9ZJ*xY&YD8$XIHxyO>*meetTM3Nb#`rM!9&aJ+U0pm<57wHM+dmnx?b6TGN{E ztV80K5D5{|QOHfwVc%{{6ti{XA%e&bXVC4IfMUI&eQ!M=qallR9ta>G1g^e7+7;Cq zEr@7AFI8L)bRbe1gJpCEW1v zbMkk&<4B%C=NgA_9s%@v$8(q9Ta9xu-&OlK3$GNd0DOkRDj_S_(<|lbADNfbN#R9o z&@Z(DWyfew4!sZo{iE09tS={*N%|(jLAY0Zv?i+ER87O!2a5IMCr%R_K_6`IUis0U zsEed(nB#Mgk-Q=6fI7FsSi^pgb;(`~V39a;XL=>GC4Lm2m`F(ui7M1j6wkJlQ;*@( z62O8Cl-;Dk5xFOXBBV@>i*z>+<#gA8JfC|fM9totw_u{xdAjTEe$o{PE+2c5W^(ye z0Fw)_09f#Irnyd{scWuI~|fR}RE4eQaiuS2sm~ zlwL$=OzH?rjiNQ3;CZ%drJ0}Bbj9o8uGp>y9I+J=S`kQBnxvqfq!KrI^DLBjqIY+) zE4tUgYB_G%5OZZRNtjMGWY7c%dIrbq7iXzH$J%dAiW-|qP^uB5i4upd})9fuRRTI@8SXfs;N4;YDyl zb#>eWI4?d$foSO<)k}I_s7IDgYBF8(rOdL#+7*BchrpvO+0EAwvs_`Ysky*8by;47Zt1QiWZKRQx9pv znx|nYsEBSa3c3vWOKj;!w9~^(5z+_HI{vBauv247c8g`l4L$>)82mOFgegDtPPFu!aGdm-5B?}-28J~d4;?x zdk&p)w0oAvj`GGKQ_3<@W;Rox&_Io1Ld7MJL3t0ouqFcU!q86&A;w zKRY=}zzl$F4H2*-G&fX_pbTXMiPK_s7s6@Xbjm&gmLzwhvGiUWs8gz#R3TRtdz>&y z%==)uEJfU%b8t#HJ5R;PptwRTeTGIo?W~dzQsstz?TUXnX|;;X3?5@L_L*f#7N$gE zvGDl9s!PbANt`dlL1A+Gq!@dPA<96TrZZC|)@99UUlnmg!!E~6Hnn;P1s&d?J-wdQ zYAHf-0|nIFE5Pdpx5^(OWyaM3nzy6aUd$j7q_Oy{1;a^C%~>`>#Njy-e->M}F0gQ% zIS^$S2oi-L#<(7pfdLqhfa(|lh*(~}Q0a1Fu*RVA=9RnsAhJ9LdbD_`CLI`MFX2d& zJc4x_eePDBQnEGm8#1MWbBlnZy;WhrSD3Sx+j-eE81f*il*S(Sd8B2)>NZ@(xmKRI z_|gnVq5E_;Uq0|I&UpUyovvTR&9(tiMeAm>JKafE^bVVdj&mc?iHN*q#LI?5Ce?Uo z1Yp!jlqAS97$w83r79S73Zr3fE<1 zltRsqnYNBbhm8lCNV(;x7JUwx z%cMoSh1fur`by26*U)V>%jKhE2JW}*9;3Q>IlIP!oFa_we+WAh@6G83UGNIa#_?Df zpw~rDo(|GF0^ps&u@kw5Q$^L;o&A${a*Gz0uCd&B1x|gpjaaw|O z-VEIdMEKBMR0XuHb6?ClJPC--`WqD*FAN#&>Eo_H%SyVVy0As5lhwF@zFHFbTNTo!!F$+bB%qSx8jbY$AJF z4+?jfmDjhOpM-^IkI4iHr37Ai65T|=DwQCcS?fDGpyk!uDD`T-7i%<2BQ_2z4a~ZD zn(=T*A(tVi&k!5T?`X?BOOO}BLIK3(<+hdyGTc;z_&mxHo>lAdvtaQVX5WZ%9 zUuA15*A0ygEtG)f%;j}wki^&UEMMc^6$&oQ~^MHQPR z8aR*9h(1@|Jh#pQ^x1~?PfGaDbq03#ZIElG#{+zgZZ_7_jV+E!K_-}3P2`3X@LcG2 zI*A^RK??E6NYj_q+GKE){-L!zn@r{mQGgzD9r2fwxfKJ;&V)TzW`y_=y)L7+ig3`B zIJ%F^f~B{1HY3a4(c@G-9M%*m8!6}`U8e%T@ZRUTwoV?eD&tbYUk7rIkbG~9`N`EY zVR<}71il`@en-N2TM$e1LV+&J%KC7PKf zdd=}-i2AxJM!kqr?S#u2$hA|`pn>%iAfWZLR@_3!feAg=cW)z#W!lUK0x-LpqTnY4 z-n@t|C%sdPw@Jcw#05r5HCi`ym^xVlayM+~Bs5uKIyu!qhLohMmX4;Lfk`P-(KGb&`Y8!Y#%gGvj<*M8 z^+D5^b)!Ma>8#GJqX zH0qHhJLQxI35w2|>PqhaY7*?d?$GAN#1YPB_YNOckN15Z^+@e&2aTFyP{AFIpl(>v z3l~U?Q}mB{OM!0a%K63xtkrIi*>63jyian#!;Gov!w#LQZ3iXjT*C@qeGY|6mzd&q z7G#LL{>*A9p#ySAmqzFHsCw#M$*vI@BOJ;EnN;t{H}ns#yS3~_>W((kdycglHj_M( zsrJ6nR}B`Frzx#-*`?v}X3%WviKZ#L)fxq&qc%Yh#*i-nYDryghhc}MoFHnt7K5Qg zn|F3wp`H;Q)WaMuVr>IkF~0GkBYNAlvTqI0@Z8>mmQON56M)#}Xqe0@i7|VwdtM~oRVH&6=7G8f^rf7d7&o9*LjbOWp%Sp*etn)yFEf?wrzaf z)CO^>x{ZX^o)DnmJ zf;>BYH$BCsEErr((YQUFvAbf2z|D@(k)v0M;7YZj(0V@dC1c`NO*$Yx}AbdENotg9@y%WO=r^^v|Brr&}M8h2J{yI z+Sp3acu7r*vdT0(x>BkgT09#AJ<;Qi+4t7U!8Vl$YEwk?T7%A|zFWhon(q?v@QueM zM-wX3xn=PaU}wx07eQJ+`oNlJRMO89IjFC_2uy`ere8x~$n$7e!r=lPYdrJ?#(64m zz@kye-K67Q@_I@MYW&tJKDmchFkCJWuo59;4!B5nsL)Z2!# zSiy#BCOgvT9Ye-OIJ}HF!BkM71TWhKmqj(`7^$S&z~^SJz@{ZNu@QGaiJag-s;w0ycYw0b#OMg5F^)@ z+Fj&0&IkSZE`AFKSrUxs&=OPVJw_UGh9&l=&7S6#(H`m+V;_^fS{&a~bn*2O1Q@H)+Q2Lz1#X;% zY}nZ~L>(en5_#%V^zi`Wt(ZnfTzY!gtmI-Z$`1^zw&`s;J#U~d9yV;IDrlNf7$iWr zBq%w}GBG9~fQcCPFjBfjyUU)EXNWH}@sxXSojAHNG6J3`f`=8}-5}61f(-js1(d8p z1VpeDKtAUssUWu@G1;D3Nt-H_HAf)fy0Ts@KG1el*KAqP<=MP%RKrNOBm+k)w}>@} z+PkT^Z`fZCqYdQT5U4>?3|_UC+oZ%<;oC?z0;bosMyR)}r)W*CTxUG&8b`-3&?8N9 zAo#XMaJDW&&e=f$ML=`ip&p}?q@ZmIu!d-Hh?OU|Gu6jLLMYKO%2dKmmX|)PP)fxI z_a8C57&r#}mMZcwF_?9z4w>mjSg#;HR)OXga5R@SQ@P&x=z!^#N^ag~oz(3i(Hv`W z943&s2ZPJP5(DkAHvthCf-o1NZOfD?Fyr)IotCacgPSGmUW7hn)|!@TB`DjZ+6dX~ zz~2-OXyv(x2tOkl1sVI)E(<&}t1eGwye&CoQab=!=i$Ory_2!c>$bShUvc|L40xYC zY#vAo2~)YL0j3T!+^$Br6*ucftT{WJ9511cEh-^=DuS9!2$Fn99Yq!vg~9HXd2X0{ z?`d4NHE+GswaWxXF(^3;rw}F%<>O$pL)oOe2rCtwBsl4kUEWib1K3^&yN25EExM?C ze2fqbGxYKq)CG#0gmOfduC0t_?2_!42J51k;M5Htr7-I8&iIO?Dm`Zl(y|4vUaD}W zqn$F{nl7N@lG06Q}1D_mG~O8Pln@lWrM-$=tWZC<=$=R zENyyyi0TFm72#uIS~#}MdtHaexUnh;M+-sKvJ)&jQAHLMt`P9#^>$lAOanm_P?y~O z6MfMY;8g(y*`r$MvfD|%STQq&L5ApdF;QcVo9@K>mVNGsqXBu8a#E<2po}ll-4-A| zXW{J%$nLmieiYR3=LD1!S3*ANnkETVa|T-B6yB#+$SszP@L#kN1uIT^#~>B_h05ka z?=(h=le^2YsvXMA*6QFn<*XTi64qodMA;E?Iu%bOSKa{#Kb)TX7FIE1r}-_cceW!Wc#N_kr@~~ zMB}vJNr8U#S}U>dNX-;16MJ)W804tUL7uxTxoJ03$M1^Z#8{0XnTue?^E!0Xk)4@u zRYh1gPz}{tJ}%zo`E3|=dh)X#hgweU$``3x6+Wv68=LxB@A;i>1ZFt}?0**(jx{)G|UTb_&IyU2VqTd@>-Ti3`YKIk*)_iqh2W9+mS{PeT~_ z8y+u0Erb}uJY;;$DS&x%VD=iw3lgkW4|90u4OPZLJun`PA>Ulb&eWootCL})Sl+3`WZo~|jRQzg_(IJB0XU%0kaE3#$~5!(8LhfRl+VQV%# zi%JKbl6Mobw>@!0?e4~0piz%OGuc}Kg)F>ON#P3@S`CCr+@-6pKuYm?9{9S$z_w@J z;O1ndrC-wO63vVW74{wYV8ZtF5FI7s&f#=Yh(rW(0a47Bq5!5d2wFs#7^r}lxpE(6 zyeQ65fneO<{6D4#;u=L=vn4y$!!bi5UJb&&F4vLWNucmuJ_Z788# z%5Y{P8I+e6ZpMKNR~sGZ2=<}+PVF5{$J9vMs>wD+2Ic4KhR3>1KI^B%I0UjHvpw~l z??5@ySTwi7hD%p95>1VJ3Ao? z_bK@j850#EzynjYJyW49i^Y3q9wt-}^!##xfFbi;a;*t{u?8*1^}283K~@T$A!zW4 zH@Gi}JMUa!N!jc&~W2dQ2|Pb%6ByBULMS; zI>6C}2e!h24+`udrC1#{V*`R!RlwVftSlR{b_Dhh(raUM+j(uRz!*Wp7Hf{qJ=fkN z%tGJ@Ep8%sC=afccKcYVje2EvNLF8jOhrP`^3~zF0mVkV)3@u4!NTPk5g9oZ%x>sq zd~?j0PBEDBTqWV=Ez}L@)#tNCbIFnK7%Y)ESdRtg2XY*aa>`DW3p=}G8z^=;9XN?W z194l8gZr-3l)h>_qg$J&=?Np&;=rLgdys4axyBt%Cpykv9~ZOu(Cps1%zJ1)Nk3=Q|b_|yUA~L7^R=4!Zyj7^qktV*kY{_ zAKdtp66BQrRH0M}`!*u*3rY&Xv(Ta53WO&WtIBq*A_~iS$H{7%j8nL3uWK%#Bo|hw z$iGr=Zv50(Y2#Lx`U1V7LZ*fyVi0Bxg|;OVPrg4@;NsXnVvl)W6z4P3p^v>Klu=um z9uI17POxtxcD+O>Hxc{o1)J#=Nbwm2Tg5kteDChhiQS;=L5B@Lly(-ETZWdTsD+l@|t~yI#4pa zqN+0{%B7-)HcT{u>l95err0~+(a7q#*#nGfzDEGgXgh=SvoLMT)uDAfwIDuU95bt_ zQ}RV0UI__qdzQVVxTrb?K&-aIy!FT)?0yqkT=lB<2)Gg8YC|ialELHg#<1xIT#(z6 zh)SZbT@BXjM5aYr3T2c3okTzEmTGMt_E%5<1;|clV0x?7z+~OMLeByrx~IK7R-?XjqO=^wiuTcr zXo0HjFiFplOb;To@pv|#+;LMnm&pB(UKj;y-VTI9=a5g4a|R_1+`znC5#~pJJAVUa z6Jw}_Rm=H6NDIQN*-57omQVHd9gcWwf!>sQfEY3#j|7;+Aq!MU;1GgLFJ9P$)hHJg zg2y>^*2cq5M~~L-eF-n7Ro+Sw3i8~#i#{h}tRJRuoEB+JCNrplyam5VCfBxwdGIgB zbll87zVL`R*?bV!FAyc8*co9V&XS23h}-F^*jWYT6};h`$3BA@9^CV?5$_T=ECcel zzE!Y=ZF8ZzS07j-=!Gty1gaF?;@!AHtJ}#u`7oaM{mUPL zMGT(X1d-&hyI45&DTW1Xj!w1%eDJ=H@{LK>i24n=>a3^uJ2+4}7 zQ{drgK|^^Sxh@-gz*Lq-Ci4U;@PQ$HDhHFdkRxN8SlN@*vyO5Ic`_hO8}Z=6cfE)f z&u!)FlTNeY!9jJKj6EwKmUPxt<4uA)M0iygFs;EdByU|+S_uUw>*u35F6Qq$q*{hY zS7VSkc&e`neqBOJu-ux3NmXBh8Qat$jT(Zfl9do3;J|?!3aHMfZ%eSo;=EI5bU0}F zH?c`}EX2!qS2{A}_-4Jqf=xrxgAVvXCG3N&CK5YY=I!?$3*x43sv>w)8kz1e$;*07 zi#(-)he%MY7QpEXuZ;l`&b8GNgnc`pdhf%cU4zt2f3W z1H>i{@wK8_NgB_nMLrH2B_5-m`_4jcJ;iW+PI`G=NVwj+t_Z7{LMS%h45wvCik}S# z_l2}I=U3A;92k~r9i9kwNH!UyI>!l|Z5Iq9`yt7VG!x+p8;+1$eh3iUxib^#r6EW? z2hD&jk;yMvor_cyV2LUf%Lff?Megw5SDaSc#tJx%*vI!Ua;2d^0*`BcM=SDuqS3v$ zHYQrvQUd3B>x*99UNaJ*ZEaBX8TQXUFS>3E-VK!F?PFQ8oiTkot zc3|PXJX{hs(d#x0?=E_LrcW$WH`kQfw5dmO;! zdVtd$5V*^WuRRPH$RVDq@x-ubZxAVHP{npnI4YMSZXD=`w$Bu<>5)aBZ|_R5uqtZG0=SqH%{gYjQo6fcS+zCg zUxRF?;t0!--yYj6Q*%Cp+=Js?!NW}9r8KfD0ZlHHMV#(rXH>&Ll}fvz=&I8Wsjn1$ z??L-rzh;a`af_S0>7}JOc4P&x91J#rNOAL6lCMC=tPN4gwDpB^hRsiacPM&Q7Bc5g z9fKPzQMnz(kBlr?kc_eAV?xo}7?d*=W&>^7eKDT#B0X8_J>mJUR)H5flB<3Z;~S7< z^$%OsxPYzgKXCm6wXOt4qa47P8xXa4%aLm``ec7ZvB`|mE8c1>1qVr69jb}qL-8i% z;PG?0X{HEs9Mxg=(vK+^5xr0oz^R+~ zU~kM|*KX-#-9lY!0G#d5cF&Ki4z+hRX>U|4;m}ssteljAmhE&BJER%(7@FLRI>ifU zv9&OVLTHC#-f_!4Ww+c~#V;bLxR_?X&Vth8n-;I&%4Gr0={*3u}Xd&8Sd>Y+1$g0bj+K%_YZ3bv3yEfTDV zc@N1{YCi{>>5h_Ft6r6(!xJVmJej`{7*o+51U=QW;WMA9Y$V@GTM`6r-fX~X41w|J zX~=8&1jr#JsjkRoNDL@V86DeqZd=g!o~2~3M9hX|QQKxU?PFB+C9RKg=ku4NvG#sCG9`Tf*qPOwX1 z9eg{9sKvpY@8||ABcLAJnj`f~BBch@avrB4Rf~<(h`qX`&LwU&t1w#br?IeB9J3y& zZ^3>MEzq8u2#7LLb05A3--beowC~uoJ}eH?q1q#`a^#V~f@I9#9AOY^)9JGjF{{+Q zZ(wLMExC3&hOO>~5}x&O?cH`VB;CNZLiDjGx{Eku#*dMjx>IuK5I=y)iJekt<;geG zB5cpM%VM7M-NU_(VrY>Y=OS0TiB@49>(1s>DbLWh)+m8dRR6l2cG_24mk8)udSVB%^&wcNK=i z^V^&0_m>g3Xfx;IxYEV9T#V33__P_yGdO$~&vH;IN#Q#pyWWuwX>dY+M7ItH53?dt zpj1T$g@u|kBE)j{j3T;4tn%;{Sp#f*fun9QU}_x;twFd1n8A8 z&J&UI5cxuf3B|ZAT91h)Vwxz%hK{YY=sP{s+f7ypVs~&ba_ubSi6y#&2n7{)G)nE} zFI(0crH5t#2|n?;h>IFX=)xrpFuZw}U)7Kp*g?V9lB6PnL{b@2>=?xMYKBW8lo?@S zw#c_@^N2sfSeLpqOiLskd~b%9Z2$`n?s0coI{~24v%-r(aSKPu!AiA+% z-NP=y;l-fHaC2Q8xD2_hnN3`_;6>O9gN_1y)ne#!@9u<)2uNZmO(l4tXtop3%!roj zDp5J#3l7AKUF0UZcNZbx3i$D-Zv>qc%o_cvrgJXTq{raFVkR9PJexd#B);&r_IL$F z3Uf595mb6%uY9lLcgoi+3wiQB_+r(N1VJ%@iS7?{?;Ggrq!8_Zdb&XoSOM;nzb{u( z_2J})scy&C9S|HZmTtW>_FL$!dpubR$Ios9Nz6RmCyzMPrgyo%6LWI3TQ5 zj}E&@$$rv$F-I&X&0s1GxTH%sXCZ~zjKO(Dc48yx zakZrf07*c$zez^O8wl{5DUPTY2`#jQ!Gy3x^{s=i#}sNRRiQh<+#pB5cx>g5q27Dv zaVoqvx>+n-Mb;kM%A@vmbgj#C_ToK_7kdbNPKb`tOl#>^=`qOcyuTeXEyLYJnXwMb zI_V0|+DX9^4XUs}GA=TyXoF4ovH{8~TqTggX1Bw$C6jv|=Ea;1!O=ByhCfW6l5@nN zAG6)K>T2YFN~5ce@?w<|`7TSW)!qP3u)W)*>*tibl~dJ3GEKuSAyl~P<0Ej8iz44e zk;=kbgNyFjq~@uwic=%+y!6uC+togG0FK43d)@kids?_&aPi zquW+b-Fro61lczic|D`|y$*ZB2!7WA5&^DZgy_WxvV+3-JW)<|1&pG1Eaep0^n+!LmO(+E z(m?sn-Uq`N4BdK}ycbn@-re&pSE%=F7Wa~CtcHBe(Wx5|qEu?8A=z4wfzDpAMl{jx zo7~Asse3LXdkh(YR)`sQqANa$=6=%&FrYyM7qjwB`Jh9px{_dqT>}O>@@Pob%Z7DT z341<%hMR~VB*gOETA1IxD^AKDyr|;AbiWhXrz@lGb;y=CJl^;WMn`Re$5WO|@h+vM z@>%$fGit6=!Ft!W6(TMSE}(qeL&tFiB|8#=S15Oh=#Hk*P$4IXbYn&HnxTiK^cyZz zLDyQE*t%$*OeHGir)`BwSiIu6qBi_zqIl~;#)XaPXp7A!R z4ECAHuIT7UbL)QS*x`*Mz#Po0skMn=T|WX8_|M8Fc_2;#vPloxE)KX8Y|)PIEVEb` z`%CIC9x|OLjtYi@HjuUFg-2aO4?GS!^TC!czgwi@7s{dMl@_G^v#nK#<$h!N^uI zHLkjVp#xJv7Ow+U&Dcn<&0-P)Bljz_Ox!Yp@Uu}B9L02A)B*x{K+`Yd8ppPw=|G|_C1W1ca$JG8Bn)?mJF3Ka@qwgo z0RzO8f5UIyQA&MuYI=}^cTwF=) z@k5bA11DCdEfx%u2DHOS{-d+6PkXTTBC6{MubNRILp0X#Ioo^9;v$`S#ZRdo7%1QU1Dwa|Z`N{{1c{^_s$YV23BNfm~J6CgXU6by@Snd|jeel=dU;L#|}e~dde(!AcFsC|dL zt4QM8N^3txd#;HQa+fjNaG~d*w1pTqj#r8znpTiNo)?y!}GH_EU-IjaBdLXhq1P9(q&Ad`^wJ#2h5O0+w>yjD4zx6*IdTm9VNzXl-c@r%0R2u z>;!;Zn9)+muX^6nQBsJZ3&Zg~LQ=}sGSUZ&0gnP?iF+ff?xmqF`L+Y(XC3uvL#Y)h z?tq3`r0s!u({fXMeY(7tH@?ITL<`u?SBz`Q^GC_=rO-gq6735<;3U-E*G`Jo4T7zh zbIxu__z1nqfX3kZ@x2J*+f(qX%>>pMmu7c=}HV;Zl#De>Pg8>Y&td z&%#+0Gal&5J*4UU>e(h6-JPD$uSA!XL8$ty_g28d;OXKe4dFcUAS3F9qHcFEP8W%D zlJR>C*lpN~#h0p!6;%^mU)XZJH3A$m1C&7lRQ8wC1EXq|9GtJ___GHnJSwrw78*o$ zx=E7{qe}OJS(Wy6QcNQb;w@IcSQeTQiP**iu8Zij<_1YBjj1QQsbqY3NVl#R?wdQY z0mBqH3LzJM&}1}Ow<#>Ch`~;2*(rI>-B|Qogu_W#B8&}zf^|0Vs$!DJ$P!LlAe=i7 z$uXBrRC zaNkJQpfLbg-7jVJip=}DTe-aPW)q5M%YGT9MjfoBmzRc(%wn*6FqC2ss3q5w7riYl znCLOUc;s%ZUVL?y=$|PYg$Nytlz}AKeE#;hfe7%ITB)M-%0yOTI1+IkJu?x3!NtW8 zbDd(f1|=gG2zzYvnC{aH;E@&g(?#Qd1ZTKTMni^7ZhBP;`UYOdgxS|do=dnh zDm_Nm#4FpCpf1(Ra2oI8uVrKQyo-R6?)=%pDZNAN@XabRis}89M+k!l)n*t51-5OM zqOX&8dFX24Of5bO)6uFOW-|g!-nO5;n0i4}TV=y(14T6lxD>NBgyQqEA&o??A?Sl* zM<$ibNorT)&3P(Qg|o@aRjO5%FHeewQM<1gR)GFSPV5yUAFyWHXf8U|C%E5Y9FrM2 zDLD8I8P>d1LT_R+{07TWjD1M%^MnIRt5VfVeFka_ zYZVdcL>@M>GtW>)_PV{4_PC{1KDv~3$Y_OCN&-q!Wy#C=OBZ_Yn8hiTXR*NF8RDDA zL!$yj06`?dNee@@q-8f@>}w1hhgJarVP@QuPC5bB98v<1^LHgTV=m`e! z{Qib8LeB$;{J8X@A<*o>RrH(cjy2uLVN$ZnpU$MV;fTucWunb?kTap@7sSv&I+KVr z-#Av{dQ>mv-K^HVJkjM6CDG}f)oXUu`!T(`mWL&V2;SRms#v54W$6wod9bm!LVdT? zl4i9*c)sz#mR#rw_5@JpRE?F>o%)EwOjH#|X8r9Ca~aRm%7POD8DLx8>Y*O7&0f0MI{hlD-C0egt*l2m4*r?$}(5D zTenLTX-FFdH*7=~NPu6gwrul7kho_v>N&r7)x<%iW|qS)p>`;MgN(@=-5*BE@3vIq zAp}1LuDIRRdGoQ|4128R<2I)#g%`l$2swkB3uWFLW(3nFWj6TEv>i}8X5H!R_5C6} zWl_Yb5YIHDWz5*pl1bo`xugz#sMzsF5V_}Z43EmDZd=3-Xt9diTe(i2RcL1_WsWoX z8Ul(snBk^OJ*4}RYm|v>R6_T!L?kh~jgX5ZW}7EC&rvjRkIWZim%?P2KPMh$l{%J^ z%azGW^4&NY)=yTH!?h(J!#C(@;sXaEbQnpX;p}!~v?6!6jwZ6$G-najStJ$%O&g*j z^pq7hYb$4ro;|&J?A0 zT`OU*cM>GW4)D1^v(x##E1|)y-%cnBwI22}_Pir-NWnNzzmMv9u``n`<2~$fu+sO& zkLO~S?`Ww*QI%QDY=da4qA3%yA{?Tk?01=0=@V>eTU-+5KrK6dR1`0LG!*}1W67V8 zsA9uYSdJkQ`!4DM-i$;zSuN7(1tbj&YI#T}m2O=ac@DCXWm{>Ms+lxOF13t~=nzVU z+)(ZaV@aRiPz3R%)5D#_TT-nXj$k5)y>^dF2PuKl~$Hq`W z33g;0)>4B#te54Y&H=^+=Sq2QI9_Wf?G8R62D%>MP-l-o94S`jmeVBx;rR=@m1vj- z_QOir4I*)CCo03#MO52?#75DMx?!m-GAXAKPE#N!4uk1a_*8;!`?6smtT25bw#rj^ zXguM?3a8mIVVE90%2f{l(Bo*;uB-X&DxnZ7uE;!tf?MpO=eb@zcj z?ogV4Oz0zs(hH{B1J&cWy+4)Uv(qzHUCBtdfv86SeK{Z_+-^zfFI+Ex_5PV`SFs_k z(LhbUV_d=p8Yz{!JCJcwL`8$w?UTbKg2}hT8maOgWmXFe%>+il=2IqfZj9;Mh@=bk z(w3ZFpPaUCl|gTfMypYIS1Vu}5=Oo_+r;-1``>lPoHLhuifMPKpa#Ub&BEjfRa68V zz|cm#JGAs0D+M#{R1Ms0wXCPab`n}q5K=hVT(?u?niQ+I0(8~H2Q}daXgUcwQ|DmZfP5yEfkmmIT={-5M!%du z!qOQ7oDud)j_5s^Ku1J4U?|a;(88U->SZBM2QpA`iEvd{I$8s4_pR_2^5x2NMz|Nu z_@rY?KCqlLDt2q>efpio4nA&q;%Os9%mi3MsC+sHtVOBRv6KrL-#UX95e!_{i8=>*3~aU9df_h% z*B{iKjbUNywH8Nq9^DnHcIs!y@dfFUy(*y;Pp4XGP&P#@hwLkf4`?{sp;H?7XK$o@ zn7OUsiw2;PDefKhI3DL(xpO%t!)a6>aSrCWjk_0RiUB7Ifuq@!Z_Gc0NpK9h}MD!99h;FdMCNZ>z2yOgGg~T#>w?I8=%>p`B_?7`{;#BZ{=Vi)T zGrM+>i@U^jCGj+kWWKbO_LsBv!VywczJ^E>d3FI@re~^5+|Qc+#!FZEJ&I$6YdnoY zb1Zt*Mu_>Bh#l2}NvKvc6+@?jF4ABGdw3f@H(l1A@4nJRc`+EgFv&i*6kcR?nbTFl zhLt67ZgWl*>guxZyV0*t9x)7>y5YvPt&dVwRp&2dqQ-95aZyz{DlHR*G1chm!~( z8S2>T$bjK_3}QsOXbz%WAd>`xa<&0M=>2Nf~Lov55cSEt5Iw9UoFC9`TqN;FD{4Y+aW z?5uSTurPDRqu?X8!Dh!8X8ktLG|oNXOf}J6MK4xOqsv)frAV{Fvt|Y^xSS8PrIm2a zd4r{wAR*d5oq}V|Q|a{^T5lS!P+Nz+jhYIs!}CdP9QK(udyl|K$sIEa2@al%oQ`js z4J2bWgbO{*Ef$X=65EfE{_iwvi>^nlyJ#qtfRMROvVpA+-jS}x;tHplxFcM|DisJI z#G$cM@lT`X*0du{tP1pJxVia5FFubCvvEmLn~t{C=bA5etH(r+O8~J^q(WDQ-tQ`b zTsi!lqL%HZ6Q4?8l#mz>1KgbaxQ;L|#I9f}FGB#O>8|pEouFyx1)U_LCq!s|cLlBu zP9j}5qtH&1$fi2mN}?@=iZFiS0}uugG6LDVnHkV8WrvT;oXPBcIfHh^79MP5a?+{v zddJ*xXnV0t&AFBk)iW!Dinb){8v$-~7e_%No4z~oJRNR#S8||umNk8ZKqrZYRw$WS zj-;`CIl&%c$&Cnd71tu_tM)BdgQr!!?TYPMLOR*mPgr|Y*ch8Cnl_X1l4Gm2r{PRR zNdOIpYFcdSm&vxDFC*^KEJAZ~GZ^b{PNGmm6NghU@tN`a*kX0;2=#uQ~Zb__4F;|?uMR*)|rNm1JaJU0;C z-PhB_3ZcD)(-)mVL5Mm^gn-Kfg#pk6OiBEk-fa6e8`7n0NJr_c;)a-0h#(%BK}1r^ z@P-eM*xh&)!$}uZ#TJ-V)X+}m49Ezl+;%e|oMysMBp@NuKv-#A6H^?g^HUnag7gyQ zJ`D3N?HR|9>OASDhz-TrEZSEK3IwIJoJR_bsPDxF#a)l*nP5zi!3YheprPT@u?y85LUHIGH%4gJLPYkA7-28AJ^Pe3^Ws8JrMzY?lY{Jr?SmG_DG#H8n zfo8(zy2Hes-)V)2LlBJ(DcUV6pf8e=CY1Ur6uR^V2$B2uzt9iP~gqQf5%8N?a{C_H3n{5f?|o`6d@M zBIV((a%r0*2@i#Gj`@6`vn`{rwq#X%lD#a)xmWZY7%{Lm+HVr_0g2#%>$50N1C(Kc zS#6q2bOBoLIZvE|{WacIa3`f4>THYQnpEf_L$rC@>(F|jj&5H2DNVD(TQyoup+2M^ zhk#^sp=Qf#AzyKH5nnr^3lTtYB5Y44xS@BhSf-D~CP+)xV|vhnHb%&Iq>c@US7G95=+e()LtfVWJOhh68!_tN>z#>c>%`AI^ zPfGyi6Ju@NsFguy6H8_N&OerER1LKTWZudjucOH$REmwEX;pZ)G?mXj=0R2F%lC?G z?DxYPDJ;%Bp~PBL^C5zGSV8#mDVzv#O;;3zd9a>RFpA^vB0!O(j?)K*JT&>)?#vv? zUuBvQh=9EqB!#pr&JJ7mv&NfA!l&87^%AMbaefiqZR_-ry#_;`&B?Q{d}0wPr4x+5 zV7ndw`SGUsR$TcB20RMbr;M%Hi)NK)hltYJV^#%WOt~kF4A6~F=nPBr;0_H>Rr2G# ziYHf&mkG)ZWg|6+Y@Z_dn!wsEeBhH~mt0BUw9FVDWAIaOI8IE;m}=8j4r?Qer6usB zaQ#d7piU&g#k9%!o$ zO&N^C2|;4cns;(PT=BjyyWMt?JfRL>Ya7jz=!WIoweOv1%Vc-S*0?hY1~)jGAd)Lq z`1EmjJZ2QOkeYesMx7v6XjjUgG~;L@>0|2p{b78@7`?J1q$N&;CxVPU7?PIav0dsu zIO15#kY)*PA3^JuX|E#5Ub1&eB1n!k=6ABXvf^jSk|3rGOC&{*08WSkj zoTaT(AyZ~VMw)}eCTz_CSEqi$P*4}4XcKVv9d?hC%%$iQn~Pq}YuN|k2p!Bsdvlbr z*Y;wsv0UZR^t;1YvhoB{0|jE~)jEa?kd-fXu{b8#~Z1dY4GR8f~A^LGJCB*#} z?k-s5)us%*7x4|4h#?4-XSw3+{UHWeKOB*3)~TpFYo zlnLsPZKf2goh6*3&0*HBX=mq%Qa%!Ix5 z*zVxI%6-yPdCh1ic!iYU)F+uiTF8>FrX3jRUHZ9r(f4xR>Z(n0E>wb0FO|F4FR0v5 zB%aL98(W`K0frL+p8RZUdEo1$ZX?1g!42fUIU84z-kJ8*JcLTfQghdriXupezGa=` zeq|6MROEgon{w&{gtHa({Mf0rKWQvQM;W=d6{Ac;$BLXKn=gpK@$YyLK?lhgFAUG( zQ@|6k2I%W+auWPCOmepJ4fV5*Vsi+s-rXlVeZ#F)bu8nH~o${QVa znzM2RE=CujpqoXLW+-`*wC)&i;^alZQ;BXILZ#pmn70FUttfn$Am&)$-o?Vz@{c`} zdRqO9CK97Exo$J%I1Ab}1}Df#hzZ0zk@ZG$%`zq?kvbI$u9=(BuHvhCad3;LFT@0j z-M2Jt4z+ZY*!LH_8O$^0bi7I2SMa+shKY})B`8APxE^gBGW=)FC3x#(>_txD&e>0{ zf(y`kLLOpV%UL!^+w#f&MY;+-L2Ux>#Z{mr!vOT~dSHlZJ z=}$usPH1(-v1I0REt6_bQV9$3Frfg(bzO#ok%^0O9izCKaAx;FF64UlqT%e{yT3;u z9BdVsjSEVLd<1$&b?$7?w03h8NPzMo7^DmYKtY z$-wK-!*bo)>2UDSq+{sa*Rjy{_0o=Z1my5#YrM-MEMqM?8yX}FJ3?jQ;GHlND6I|=UAdsI0_T9wM{B(!wKp;i&9rkDr4{BZ5 z6Bu*V)(Nykcd^^u*TZ%az7v!u&qj96+RXNm?+&coQ7EwB{N{GLC~dV(%3Am*qhJG8E_`$yPpDm#JZlm*{G+ zk>ktW)8J%EJI9`c6ky#AfLtb8PmE4X?@np5e7$h9!KodBt~FAO+EVMDxxL3n33q!P zy2l8YDWu&HMt4JwJHTMG?m)fT+{iQZ?6?cZ+lF%4M1qzU+%iYN`Dp||Mpo3DoDhrY zPlREO#I7Wmx1TUiMHQ-%>=;U!$3##niv&1y#2;(CuV@Can^>D&r8zIwkP@Sym6{Rl z2#ke)2AM|?Ia~;5j>dAmz}M7#%cU%2X>{1Ynp8@InG*{^#E9y8w)j9(sdajGk3s1! zl)Uds16M6{{0a-RkAxR|7b=zOOfZX!3X?gOg)rgpIP|>DC8CFg2c- z-qLCA+e!{PRGCDdo4$os!GoWd90s(DZo!3CCjkt{S0F-(E`5vygbA4X#w)(vdj!}_ zrY}CqV_Fe`4f%HX9OY}4KxY`mh;TxWRroxVW~+B;c5RoW;B=!5yg1_y=BYTChrOLb zYQp1v(NpSZz^)kL7xGZ$-QC$lT4?w74Sj{JT;E!R6p**;_KWKz!G~rV#-)b;m z5eSf)!=3xrEfqX3%1WeXf`K6zb)zmE(U`*(*0Q*Blqrk2(*7{L(0FklJ_?|T#9p?+ zYxydmqGs7fDT_U|vr~$tndEx$;x!c!O@pi2+YXC$o)AT;FH1aU42*QdrEq6AD0In! z4|7HWXk)3)@*Y|$=r|1m$A*|EHWY2$X9G%)(Z-L(*o5wFreSaK$m0QK#Mll{Y?MJU z&RAwmx{l$9yQr3;ipD9O0N=et;UVMGQexXyQwF>=f#mPvh0<)dlwEe-`9N%$3;~p z6BmRxbTn?XjP^^`j{?G(d^@B?*T7F%I)NKF;#qna!srf~E9%LrfhgU%VOTT8#d|_f z=60P)x_2SXz#R-zR-CD;8#d??X2>;UXK*MlHwuN}hU|B8GlED6KOGn_3`(e0l-L%! zV~AMUx>QQY4@-9G4npq6c%;>NEaA4yejkZCj5I}`FrU23@_`-FFg4^N?7@V!l?#eA z#3)k@x~$o^o$U}5)=;5NZkd)s7WVHYRB7L$$gokq?Nz$+lwQXqrxl1}rnOjL*MbMd zH`+mL?v|h=v}TW(p^t(;b{dq0HmG1@kEm-3`f%=?)q5tJ)5;hF?RAW;rGV4OAwf7` zg^)_gd0Hjl1RKnkUwFH+3GB-S2gpLCqi{e@ELbl|fLTR=@@rI*@KW;)?_vm2WE&5V zOFGwg!7jimssj2Yq-&8)6d_5b+1FUu@c8N}k{1}1{%%lK5aNKV*6L%y3`zJv--EdV ziPTlNsKf*%*geJ4dRlI4k>|T*^gy9>$igUAXubn{Tko=H*u+CvmEqNr-NCz6)h%nb ze?cQk3`yz`8lpYQ66F3hhc1?%#uhmnS6_=HOBPy3*~$nC;+{z5Am_X~w661TpF@D) zsH&v(0R-y;s%V$i-aee3HU#eXh%D`D0lGy(Wfb;unS@-6-g3hR5n!z7VBfQRE89WW zb21+t#4J`rWdakindgl%>ToC2V`@Of7mk40){k$JqkLZZ0~#K~O2EX`W(xAqPOV}CRVGlU2*Fmssw@MN-G-SSg+gtS^jTgiZ`ftQa| zRI#2-aoL&#sA4)e=)4(HlHd=S-T6fb5`s|5B6+BQ?-!gt?WbnD12?Cj9~Sd)T>4GM zJW>XJXbNR+%HnuIkSF!m9eOQts;;&Qp`*k*TjcJj2#_(5%M>4FZw1lPF*wayq7B9R z+}M~;H2v8Tg>w;KO^XD04~n4`9k%x2Jf+<| zR=F&p?+!0!>y8t6+3vSj0zMA4*5dc;S)X^sTG4`TCvgElBX4T%j`8fyYuYNYGnVQ? zGAzhxWVmSv`9&3WxW1||=Z`*k7lAO+W|k#xgO?(i0x2?p<(VUxJ9GzAJB|csZ5(UHc+!Q1fkG1ZE1C^~Z{lGxmdI-ulmqDm5Ua(GsllKU zQkJkE2RE(OdudYUSsc5vMkI#oX5n&Tn24VJ2>qV2U3)jS9p%gJSyb>PF=YZtY{@gT zjB(FuQHT$BT3J0Ba{%_uz4b1aNo~(|JIK=;BheCVm0%mGUQ-qo#ctk+YE5T&{);Z& zL@VEm;bq zhMzi*=%zKYS z8HohMOCFY=4V+!AB2dVt5D&vAL`m}T{p2vly?FKA(LINrf|V95T1h>z3*CfVJj>W^3*26tI_ zA@&`c&ok2S>M1dSGKdpmfrE>==*$ZgYJ6?R?_m9+S+-Lo6NwqmoxinYv!z~C@uLv|${YL`0xn7t^I;si|9Ccr6X^26>(!Lr3h!xe-87;am9!H$= zQ9+ZnEF5i3jcsAHU<|XZ(gfckD zDDIUy3pJza!B2f)fog`&t*p8OC?wPfkyurM#);oLs}m2vLzH>w!!omrx8F;qYTM~b zo*1Yci>3r}GK_&B`;e30gAwYX?5OLlEa1@0(zG4`4=PhC3bd-#63Av>k(;4+m4;+X z2Iw6JpU|^-{-zN0Rg43?F@W>Gy0-S%4Goa0`ZgS_aCi$RSXSfQBYC{Njh5Kg=W^o% zy6fDAxdo>M5W)q~_{KTp#}8>ax#bPC`p7^In=Bn+;Dcy_Yk+)d1-AiF8(c2-xDU!; z=pf~~e|etoC3fE4;7&m6NHn1NmL$UX zkfd7Zcd3okbgSy|knDhu6?rXEkBM&`-@As!$`nIOND-us7%BWV9ZA#MvhmJG3;VPp&8)gjE0?_s zSVF!<3Z`UaMs8+c)b%K)prmP5^@&%9uNmc@b!44SEeKwdP!{&$AX|8#!$aOTvGyR) zGVTO|51FTsGm;}-n~`E#`U^*}uD9c+nM5%?k?1X;w!-vI<2WPv$k__QQBYJ?|TsjaYDg|n_5 zWq~~JcjR7#mJ!6BTo^qrsckeU*7UO}KO#L3ieIUT597S_aMigK9_r`-iHLgCCA(c^ zLR@iHT79KZhg3M$8+=;43=iggvLOKDG2&rZUEalRdf^%n&~^d~%^VQMSZv4ULOI{Bmlp8F0e7p+kDwy7r2snr?x>c6j zc3}?z)s*=sjI)<9iOxeJZ+tvV%2ExOcW9yuY}xq~d*+{mxg&D!dSHP)=<2YEK5v1( z)|87pW^XgGw_`y82H%Y?qK@2X1q`Evmwiig%KTtnLn(Jyp4N%?@y`(Q{ z?!c?U$%=g?Ze!}zN5*XHu@kPBcpVD$x&Xd_U;+593G@k!Ca4c7=w&($y$@l9R!Kp` zp@1@~=s96Sak11&1~B+xqUoS*Sd~6Han4PGWudNzpd`8Iw`x_>5my|714^aU2-w%6 zsO{+*Pk0?54I0IqFHRfg&?y}^#d8-@gaqS?kC(8H#V*!zC)gjj*8RR9@|2PWRRU7l zrQ=#BrLHWhGctLi!0MCk(rr9`nP|&m%Vun+R!zT-vef1td)<;0ncM8fwS4d%PN7xi zyG*kuA`1s_j`H2J*{;xs3Q5`o0yI^_6-U?0<>^L#+qb8JK;ws`D07I$$}{y{dQd#P zd~y(8<&J(x1qCkNGwZ`oH1ng!aMb9#3QhHaU_N(+-9Aj)FxQkJTRc8d>gQSSm}98$ z+?01^$j72@+frW692ayMO{YhpsJ4C|6YOqei1rkbAoI#pqjqBGyze-^&}-}}?ssbA zuLfZ-``JhdMU`Yd@iBbkCFb_J%rfoA7jE(Gv|Qj)wa=v87zipz;6j?wtJ{MQ#$&k; z>pIN$m4gYd$1&s1#wV7!B>5y~PGB>E!FSb(9S)hI#pzMmwt|enuO*p@mg0!<%H%OA zteMBF3=nO(EjHv7y@%`@eykZ7-w3P7>1p+Zj#Gg^VvxiuN%N@>T<4US(O~|u# zxQ}JW9~)@c7QO-bbVD0^)?|lpE=MyFiFI{6x;*tEV_0Vw9&iUf8GCHHLtgWwT}5HH z6arfUs>S-~#8#`eRt`nzZ>t!VHFw9ZV;>|SYO6#HzD#A%TjpQ3ae5q5&Oz4gl5Ykv zbxzz2-uag@d)&qBCqI4h83`*1iWGu0k>O@1X_6H}rScBF*h6b9PN9oP-m&Fm?xx*! zby}W+kRDUv-54;svP-Wa0l{4oZg)xr8i%-01|Fh3XC%$8L-dQQ9?4MSPWNNck`X}2 z8i2}K!f%GP;z6T9ZtU6_+s7+|Y%5Ht8NS3Jxd3&s5tw zo`O(B_dR;B#dJ3qopG#XU0vX7N%4ExIU+~Toy6xvwo%VzcXaw%t_ob`nA4L6Y#q6r zVdit!4Pq@a=GDs825G7ln#lrOCjn=A-1T`6i^v-C7JdXktWBPjd$N{+t}j!={bF(muIJ)5+w-wEmIjU-Vj#Mo6XIwYxSvcW(PFs=>UjU zM{{lQF}QZmPq9!jB=OJ}iM=uIELH+}SIO5!J1Qhx&G`+B<%7!X!y^3TR3#W{#t_fiu+hC2jRN>~x-37?e@2x0|x@>D! z`+HQenmDw`EU8+ygw4}#{dcR?NPuhX9&4~vz|Jv z;DH@)nzCr3)(Ix{620?k1NzvvueY9(U0ASM0@i0>4;{oceOc_uUJ6BIF0o=G8crPC zB(Br5h~tt{L7PgBEiYmfJ{Mbk$1)a-(+NKsWS_N{dzq_-Oun8e5QQl|Bo22r1jiC6 z8=M>-9YQ6c2bHS8vwL%}A_V&UNOhiihy$!L8xpLT5D=-?7#0ZhV}Z2@S#0ZkyGZwr zx>7{}*4}Z}ixy1pU}a9K!d^-WGQeER^^K0V6=>WlEq+iUXw7V7nqDuq#zlC4(Nk&#WkZ z*k{jj0#V5b*ru619#S}vYP9#+aQ`0d=@WBtL)*+Em^e)m_XP?&cTT{&ffRb@#|Q-$ z{Z0pz!j?WNj}gWk>jv4`P=&H2%kYYhF-gj-4sM=7eL_gl3>gY(*(Am5G3|XumzFuS z)K{?BIR?evhj3bXjlo_nH)~H9#cC#{OCsVHp=ad7%C}O3%8_+Sv`Ydm9DqC)a!xRZ zI5$33PhE=3j>tGQ)!cF)cHtFPi#^mQx)e;&E}28Po2S-FD;=bmg-m7WcQ{L2;Ejz` zy!IYmuv*Jw(+1=F3{$NvTs#8viDGz4g+OSxU8lsViB@<+I6h%P2Hd)w(0+iRYa265 z;1I$`HCn~d!2#hRkw!&@n4}yox|^2EgB|1zP(wmT_cgNaV_fpdiq52MWF!ktZ!r<3 z%l5?65N6r=#gc_37Zr|!jZfUt!6$r++@YRD>#)|_k7tl5Ww_W?yA@44v9^bv)H2LL z1v?~Lx3+}|aVi%?aSIH50tDZ2gBX+nIm{skX7+PN8+fK~?ldqyqH@kzW|CIg7D+2L zWDhv#I3_SNc(`H=8qk7OQ{9--;jBmOF_3R3zB%IWa^3YUtiWbsn65HVhJ=)WHum8gMZCo(A?z%-FYKc2e&nQ!?rTwXM0f1iBYzlW=aA z6&D_^0pFDeNuk10;xPQNRN?357+_!_jzc{@Mu5MJv`rMcf*I!yuw*-4fP=Y{agkH# zhHkJoRRK!eIqm)Dxl#q|(glz`g3+SmyzG}CnoAN=>}?^t^}KpZZbqeValso^yt=W) zXQK+K*n|X&4T{SnyxHCZORCQ#1~&27h(Tti0i^=Ps|s7hATt#{sSWX=l9=0~@z9mW zvTmy-?#X(ky1sZ(i|oHvkgXv}U2G{(Om{hu@hS!o`dx2N&>4ZD&y{4j21J$?tiFtx z7NI>0C_=lof3~Nug zYeoc7i_G@qFto76N@;#>f)m^XE}e!U2P&lRW?@LbFo2DFDTQ#Bwe3E$RWrqTy>mdE z(_}L&otlVt2~sl~8bp14?;leh<$|f3pjWw^4Xc&}fIx3t*eS{1U z$b4l%hM%qwmeG^DJl*HRJZIQ8Qq0<5DR&c%+ASOQf;?bT1b$0X+Uf%OZyrzMrZazOG3@BvVhyaWhFy7v7Fn0%EsWZbu*rJt}s1PG! zGaJ29|8JK1N$)#fYgAg0eT(AJE)jXzl%sgl)Z@VbrZr*37?)_lC2w86>?U2MmXIU| za<#mvw29-h1NR!R;le4QW4N>3F%W63S#8-x6NC!TTwln8&M%PM_lfGusEAAphnem9 zMQKXtPO)yXz+soFiVGBovpv2j;w|)si@$((40F+$jgA9e)6=J_un{c`h+Woq2Yq9R z8G9Y`tn(V~Y^qRg;Zo9&phK_<506z$kV8xk*^}m>fek|LqvPU=$15{&LG?=~g(dfg zD0s>Hjd4}FH*ZhchgOTaORdR<#t`KLjoC7n-9Qzq5x3=p*z9-4U456?D+?g?^J8$L zXoruson_Ti96ZGY!I{CEl}yRZ8f1eI);z0P4Rt+$yb`F-y%E|_%Sgp&0_!^)*LlI& zF{TNiaXw|sT*6!zd*iTjS@dN>^m?93cuSOE;#bE+;Nq(N&eWpRUfY1BP<3dXq)ypt z8rd)!(b%^fR%uiE$#S&PgFbcX&dzc678lNO4T~k1b{9=UuiB<;V594%Dw%52Bdo9y zixRBG(eoCWX5TEpFgTgPLm^^x_hW3%p=Fb@BwHFoX(_fcJNHbBwq4_CL2C!NX{?IA z7Ae?fW7w)Ul_2b=?=!j{H?MC91^Ocb?QyG;H@AGtsO_;XM$$J-QC1P@^X!7og(C2j z<30-3Hd~PaV>`%ToM8j$@CY=Wh!L1CG|_}#T5GuIF(}xu&iUV!5J+f-uCyl$TTG)- z8MgrWouq-s4NXc19TR%njWxFujFDmRw+gA09!Aokn1ahOM92*k+^?5dp}e;Ht4jHo z1oqaexm^VofmmoJZzxH|mqm{xj&i(5a)7VSXYP!uYn&Hlp@db|z_Ca&4M;VMT@GH| zt8{wZR_J*0gx*ZMC`-CRCUR3mRa^${^vWNRHVJZA&~UO-Es6|+K5>Hs!8)zjo-UqE zIJ!L8B}+#gVPMK!IjnH_QeV4`OuS%O6ue23NHY+BDR_CYDdwMDY&Od7MZ*Myv`JbF zFV}O?$ZC7KsuFz}sVqB&*6dJZn7Nj$&uL(Tx`J&nYQ`JOtRVQOVMFl*8PsJEgyEpj zs}%_<mFk@^!JbC)WLK1sv{=m`UX;=}kgTdfs1Ed4mX1d<17+pt+AxSJ)d{K4k zwsL%kCe}k5h2`ztNfNtI%7_xazi@J$X#=v{_H48l@nj1t$lc~wpW-KIsFWAAHuT2S zkoQ!OP^H}1@mg(VuqeVFS5;`JR99COo6w*vT0_CUS5E^DZ>mQj?F{BBl*9JRTF~J; z?{=mpXpT^nss?FHO0pfCB)3K!CseKYRipAARuZmc=cq zn>GsCQ8ub=32i3TTS#ms!VQaM7SKl2ro@{h+HGu=Hj!IZwMtt9V4FzVD{4)Wt(8rn zn?+kv8&S0-wXj2BBVv}wCd#%^Z6Iw@wxt_NHYseN*+AM`2Fj+zOJKHzY^2y*MYK|E zLA6bkHb&H&MGc7AD%vc_menh23urd2s*R9LujVREtE@T zO_8xA*(+k36q`y`%A;V|LuyNEp|qv38wFcQY)Ne@*i&R|sMuD;7RehDW|6f`inf$Z zs#_&i%AvHX+M;Yyn6xyn6QLNZCkjB(@ZrP_~gap|q-PRT~9On*p*mQf(&2(QHC(QZ0&0X-jIAwJnHjjjE}# zTR~RRQL;9M(3>D^6xk^@s+&bt+eJ1LWg}>t18Ef66J=FwNwAi}wwolagqte1jioJ# zwMsUiY!S4jvJI*Yl&Wl`wuv@|!COOQO{$|}OJP>ULux465w$C5Nwh( zuxzB-2Ef`?u$vGzB{o*o7Sy4z8!2riuxvtY4Xu%~gK9|GO^P4Tw#omd3&jg&P7JL~KE|Lt@$# zwh(Q#Euz^AYLT@{+LGEzY@*navMq>~#TLX?!bZY2l-jD=sb)sf4TW1)%VLt+p<7!- zt&nX&vZm6O$yUO)idzKP4XTZzwjj1t+E&{N*;dHeN^C0F+KIH0ZB$z&TM}(l+LaqE zq}mqJQ)yJ$N^DYWM#_}hiZ+djTNcr6L~J%vwk5Gf)gxpk)lk|aWwecHT$fdPN*dt`vk+dsh zmeNCLmdPf;Hk51z)PrJH$fndSl~ipCZBEr!&qXq!@ORkWhmY>8|OVQf~`)J=sA zibl|*VNHa#qMKE0EtMNet*Y878%VY&woz5I6KPSj1+a$7Z7SInvR1(+)iWb%rpTKS zuq$CLwko!&rpiUMt7zFaq}eT!lVVMZjfGQUBVvuIHWOuR*ofLvn+dY3Vy4hGlC6@q z2GZDw*r{z*8!1M~EumXf8&oT1)SD?bM#x6h6||t(Y^!LEqD_dcs#`6UTLrL7VOs!g zDJ`~w+ETVsn@SrAHkDS&Y*g6@Y^t`EHmOams7&dO_JJ@wx-f;RHn_6t7uzWQjMZ)EtG7E*w~WVp=~JGL9#YYsy0oMwy3sKWR%%W zsGCr25wTXv*nrr!rEC=1g4mN}2GmitO_ZBaY=pLjvQ@Gu*r>Kf)LSUnO^R%kwneoC zvZUCXDvgybjf-lnp|MkB3uEwRZ3?y`*b8M= z(JiQKsMXts&4q}o$xHj%Q}t7xIILtz^t*$UWJ(pJ*8 ziA|$uNwSLAs%(v@qiI!b6J)jvWY|%%OJoMfY^vE*+P12j6*j3$Xq!PB1x=||)TOYY zvJHu}lW2y*TO?Z>Hll4JYy{a`Dw|QXOK937YEiIlCACb#Y=pL@u|sJiX$^@@l19>- z5NuUVq$_Hs%57B}QcZ}qLTwGIR>W4#khV(MNwOBfv{G9_n_EiSNNkC;qS~t1hRCMU zEvUAtp|Ys9lx(QBRj{q9mdZ_%Y@1PS6KHKjwn4JlN^D6sRBa)(O4&-bsBEROuh0BR zDJ$=urtC`3_wVqh^tArS_w@~b9{*4h3U@4wucsTSAS*3Ikj2IYYoBoJ5FjjT2~;|$R7qzpD(m1kLWM&0Dn-|dvRQkA0~-DBI>H9k%S3em z#L58q+#zES=?VfxN3h5$Uo5LOj1k57rP1U$(}m173+NzbTfRlWGfBaw*GT$Aw1nIw zg2C0K6teO8;ib`#*xnHIUI&a^m(Y;|oxN93+V1|!j?@Ki`LcgP)(SZ$4O z>cI@M2;)wze#Pm*a>s0n*;x3#0zic&XfxcMf@|QGny`T}`FS(X?&Y)z!vu2#+@o%# zPk}|uGT6_p=8SKbJ!V7iLlqL~JrRJmGl}s?G&o6=R#YX}-S`U<*sdYvLbjCMTvTX1 z1xZlBUF@t;4`4iQvXSQNrhe)E-R7^Ip{H&UF`Be(xEXaDHI*KLx zNV1_Bl`p!X2o*!S7(%7`iw>LRL~Kb=fRFFKgC*||%~lNy0uhp_V+$WSm(C2OemzgLL8oCFdR2Z4 zP!&@cW`;{seAE`eMfz1SG*10zzWh9A=Kk%RJofv_e{JK34DNU1mO0vn zszRgg{H^pOSxlk!Dli5t1{#{j(jXR2U?*bDSpNiDTqq!bQ#?Vb9$@YmBb|zqLF=5a zV-3)NU%h#I_qishHX?2p=dATtx)K5dt<%8*soga^tN{f@;X*>5J-MI6*D6dnrFtQd znMoChNyE9|aO7M)CT132fMRazI4a#+yxnFMz6HT~EAJ{cIgzzEz_xB2_oLYRitvV> z;+QCz1%1t;KRieXcthe4aryv5V;m3|@C;ieyF3UzWGqt(h1>up@EYP!FzUbnV9Aj~ zqEnhA{e2yDjA)ezhpH8mOAL4I7;@8?+|94h9&=El#-}|hVUxkgxv~-zk|slO#Zuav zg*doii(vppj6$Lc#i6VqJ(>3QLmDqYJub{k@rM+889fqPallfVw%2H)P@-BYIZ)_o zv?>EC`KX{tQSvIFRaY3N3$i~ONXrJzkt&WtD@RC2L>O30z$79>JlN>@@IXe=fVK;r zlZX)1??7dce+m%VcE)LW@%IzB4BiI+!8C}vfY(BMh>RIkBZ zzno5V_QQ{y4|v#m^~WAu`snYTetiD->Urme&l`+pwsWg*8}Hsa!?%7LIo!D8togZl z;OFmOXB)o#)%~B@>s@1KuHL-=Yi=CeoA|{k-usF8M_9{<>tfC&03eBsATs(08VCv{}PZ5lp`q!r&X0OV5dg z&&zKdr+hus>A{f(Cg7+@<2`D^+wD4Cdc7gVWN{3%U4&{)aD{nH7WRpw=KD^X3Cjb8 zSMP|O(+3Ofs-3^I3SXmCan+d}rJODyRasq>?^5S!dI7jK@C#AdFE(WH+P64jerJdz z1ovA+UMl;Zd*PQ<0n(zHJ!nR6s8?rwVZBU29HJ>Oj~1iz3u{R2t2pWda8o6z*|KWd z7zxDv<;Q7Gjvm$A^~Y^lRZ*g4rmt84Iul6Ky%nmU6K^KcBjfTfugt2tU!H<2@sXO5 zrqsF4MxW~_>Wl~?%&69hudGh0Jt55S@PLwYe&I>D-!(3g!cYg%((C9_Au=ai*0eB?%TC$h=>lAzYqdqs@bu z+wOFx>@;jT3;~QEXuhUQPlRi5OR7RK?gwWq`_oaiyjmAh@r@mH?gY_GQie^7sfA52 zP1jvn5wIkW)63|FU@&)`>O&_m;1WPFOhL1Q#;65h!RSn6E84g+QWMv@S39@CB-2$? zUbNoqnn}GV=R^7g`>R7-j9P&8LKxHMRARNY=Uque8WF?-8>NAb7(m6Nr~?a9nk8%p z7fpDzmJG`ustwQ|#HVKXgbjAY&6;8ob@zDS-8s`?0w&cj5&4gs2%DNRua{Q;I{lwe zh5Eud&JE9j!OvtE9)u200~O1)H2J_;2+>Y_~sP^;lgJnG1lsGN&K z)Q&<0jHw=Bh7ZO9KL{*K^8IR1KRgV8>%jTGiT1eC=4q*5pp@uk5ip%2I%Y=7i~!a6 zj$UG3giICEiAcDFEojX)a8eMEIvloCx;1gm+)#W-Y-bQXl=OvPWje;EGStZ2@x;17 zpcJrn##nk68J+^I0|iB%xM^4Ed`mLJM+`%T!Im?F+dD{@W!F&{0#&FVMXx>4dxRN$uaX+KNpkJKPlfd2*tz% zavblwP^&v#Zpz6mi8Lsk-ipSj?a36d>%AJ%kdl5eQW9rIwH{G%iKi9cfkG&ppu$Qz zQTkvwX9nBF1M=Tv-n_cZon%fjC~QodCl(2pFFwyz+VHuP)Bk*5j7k zyX%Kraodi&Vr_yZkwUM*lFp=gWV^p3bd&9uOV*0?Bo@u=m3M>yUyk7(VB{lu9^glL z6s*ZU#I+Vxjs_0z?PviB!U@1Ecs{zC7{6dY+8{tIGJ&2D*U!^TDSD|(Fpgm_&o%gH zL+_vjos!@$nBRzwfD1jsNH%ktji`&*p`BTRjUe54_rJaM_0z5AJm-CU@5SGi;d7p| zmhZ=0@w;-A0Wdyh7{sG2Nck08L+N|AD77x}7s_=&m4kPXbO8a@>uVrz2fQH168e6^ z_>M6E*qzlB88PEo9fo2SQFJTl;dx505X|)YeKTI3zWVjoj{Iif^L*#$M_%}D9ABS# z@9V$McE;0b<3F^y-08;J<2A<=@^^6WuD7imG21qG=bg{yzUOWEymQCh-}{cU-(Nm< z?b}>u=Nqi~oKJg<=a24v^Tyt|_q*=8;m5z6GcUckoNB#j>qm|_<5xKG`@^-KGV!@} zZJlm6TjuYU+dq2Y`t!rS+Wpt|pSb?dzkG7q&DrJGU4HS~p1I-qzg&NK@!O7eZ26aL zuFZ9~93Qyl*5jVOZhHOg^RHig`2Xc17w@jUf4%eL$+n=`5TN%|gNNb#GvjN*Y2Zoy zwgtHpOhU_A33)yja@cIcWWaIzDh!9m7#=Ij=3qR520GM`_Ujc22o6huF3iAnJydKg z!ShYNYM5hJJG?f12c8jIQ~ITjy^}yC4nBLXUg=CBQ-sl=e=41DD_VNH4#Y-_P%U0V zb1lP9<`t4*(Uxu$@dM`S{e!J1$RR1gt40$&8}nK5)3l>7tv!;khT&iw!?EmL*r1AQ zK~0rmY1^noo{5EQ%7(Fp*xQiQzVX1+#9nrsA9=gkP*G%BV8C!Lq_* z6{vBpcMll2y%N-}&wF`MS zpo#>O-fg5WU_l_;VK6)Cos(u{&(gZ+jB>dvkPsxHnt^GIGLT^Ue;=7%+GJ?`EC?Ng z7XG^qYjt7Al3yQJ?z~Z#eo(=12C2q5L9gwxfOnlVfK}oFK)TqBVbB9 z=N^%qpw_rcxef|AX!Fgghq~x}_4lOHmr+M=*PT{z!2KFg!P=3@aaZB%(zQdvu+OwI zj)%A&l?#dGDx6zi(!8~l%Rf;XNzuQCUn88g*x*9>&*uRL_c8~#q(sO*)phmMAa)9s zImLvo$d2O`LZA(0X>H!^);9v=n4U0g9B~ovwQ!iseXTd7*S4${-rCD_fajgs>w!rnSZ zmrHI3WCdPwBrO|fN#j>;$v3@Y1oQ;vhtWgoT2!yKu*VBfwCWE~3uR3Qa6w|q*sud7 zWTW(T$ccz2g`f$q#$*tJdH_A^41vA4+6`zN$lG2nrBrgRAZvpm&aUP|zqlb>hyk$e zhYFB6ZF35}dc416(dB9mpBIfZxO~s0#zSUB&M)CwMHs+-GY?w^xnmdI^^ko3Y2vRz ztGqH)l`hy?g&|8)v|9z}s4jA+WDZL`a5cuGaakj-vgq_Hy}M&9_4etCf~b1~i|iyj zqFjeOEGzLPl6X%u-7J2uBA?94*aK@HAwc~3g)#^(ZEQ|sW*!hxvG16Hg~nw_d$*S% zfceKPx#80q2c`X+_g?*h?%YmY>D;pmwceofAVp|ocj>72J23Q>2{U;{1{ULp1^c-j z1lYo|r9N=0V>#p@{Z_@%Ba3gVGe-EDCX&<3xeT!(+B4P$C)$Gav zHph$*XTXsHxZ%i$6ORq~&wy)?m)0Of!IH9TNNHAd8B&3@l(6!s`XEvma*)K)a21XR z6~}C~RH(?0+{y6M)e8m+5SWEr1r7)>(BjkZ6g-6C;+>QSwIvela)c_Pr^@);gygck z0{zl=hdD$VNPSB7t3;^Q>_Go=@G!)K4?5swI7<7j?RRM4!?2wBDj~=Y1hPI*YF#Zd zvA&0iNu{3I8b55R@lsaQDk4h#W=bWOFF-m;YzPG>6&W|QDdd#*g(K%NG?BLC!M-4X z5V)}Y3iZI+`a(sWZ-N;9k`oO3@J==`!4Cg#~o6Eu%HEHg>t%n!kgmdm@^S1E4ei$9-A>c z9DDbS#0N92`%NcE_IJVq*b99v$_8Q%ixuRJ;kt{tk6|bk@ebc!C1+RD@=NsY3ce==w(Sju%T&7gc1TOaG^rp);yDyQu1!2^yRE4 zlqIiuTrzCKqiho9vJ~7z%DFLH+HB!v0suCXr|3*YMqQ|0f|D5hE=uu{I&J7wGmx4P zweJI;+d)VmuNPhqh(S#&y*e|%i?K|`=8fRT1?n83w-Rrt76~rO@L_@tE1@TYCzT2P3mC@ByzrG;y3)p@vy$@eZTcVy zN*B&$G{t~VP_#KH#IoYYZZx1wn~4-DSV%A0kD)&HdNjs|+fgJ|iv<_U$??ZA=YyR? zCZ93gGf&F+=|wO1kzX9rg;8IfoFOH<&Saj$%GAls8Wm5Qy>u$SWtJfo0+be219T|v zkVgijkPZy|OBjBN#JS?j<}nDa!D_UAYH|opVD2oUDiDJ}kiH(Uj;OY?rt2#3@2j$m zH$J|fCHO)_k&`KEVFTq!($tSsIM6(HY1U||blLR5&$X%BBH~yGw*Hn52_vYHZt<+G z<3A)zo0|wt!|=in&Ob%i{9s8R0r*4&3*fxbs#-EW%w-`%~vV)?Gk=ePGizdQ2+Bq(5c>4){c36Ea7l20-wQSMcOEszWw zEfpr{|A2I_?Ce~LspF9}1TeHu1Y)`as$e7X z9c6LTinB`Sn_!QRMYIxT8Cg)*?>c+T0OTrCGSVcAD!jDNknBgFOMO|Gj={xiOCn|v zOSdNNG^nYHlg*K!n6p5u#0g1l69>wAJLzJ`W#PHh~mSv0#! zgKK+15fDJaIRS-%eh%fci{+E;Xokx^Swd92C(N%hmV`F_2oT2D&_XlU*+9-&Dq*+? z6DA82rpH_ah_EHYRDAeE_&;pAA7ePE+rx4%BT*=6#X%Ll6B)97%~1YI;&rONzG0ah zGho3hR=!PWXGN_E0WxQyCbJS*OQ@bzoWcx6f-d3#M1D#Ghk0|k=uwPt)E?@lhV;%i z#6JK=maeuyt+N3fbB{y-NQnq~KKjh`grOG@!zVGxn3hpIA^J+E9nm&%YHw(2H+o>4 zO;AudjNnSIkt&LCRzfN-AQFp%AAQ{cc365K5t>2>U|K1*vWp0^O4?VfWh(vjs!2eC z5Pb~n&+8P*0YyYHIIPzc&m|oeSGM!>=oS}$1x{VPrsIlJ0~A239*L7Nt2O6DoVJNb z_zV0cB*Qw8DgrKv)vWk{ZETNcz|vX*BoD?J%c_ojG7fSy3mPiJM(bQA9(S)@x|*rInFLP*aRb;B?NWl+G8sHcCztN^umNdsIESB zP%Rf`3L#Xv4gyiru+JGM0mqjrwOx6ntvg^~R~35YS~Ry+?^( zH%JRdL6u&tt~yYvH-O3SWdM?HWfpSdL;BRz(ci?N%gNVBHZCF_99Cc`9q7Eh(ZqT#9Udfz)5AKWH(lkzM577BSCb?|W+u;0 zoFmtAat?#vSYG*>=Dl%+7A$ zWi?wvYs)Q`g_y)s_zjASJtFJYqM~Ng`>bcgBwiI0d)^ViABXHI z7%1jNAfnJP_D-c~NYATtvtg=}CQ0k++FW~rHYRQexW4VT3EmSgSUe!2gl3je2~wKC zP!H>T(<)w5y6(jD#c+X%r8!=4MYFmwEgeknYO;7og8ri|M#80mBV5sY(C%vBcahy@^I_8zn`yea zYPXA_>)yypiop&nPE{~W*6>mjd1P+KgW^=}B#~B;@#BjYi)~8ewqqp)1aZ2ay-k;n z{EK^d>hNS)3tD%bhpdx+e2+|Dge7v=DQOZydoMQ!|;VD-OuPQ@e`ksleGPb>`Lcxrx3`VS* zvP{Yi=SWYd4(T-xQ5^Lbg#@bT$R75ek~FbEivYkw)aZ2@ASAOGQkp)+aj$swxHzZ> z2%daknL#K@zKmePKr4m$MZ_q8xj>9F>sNfuv?jhyMQrX%>Kahg@{YpXVpXe!zN2na zd+1&Tv~z6`-)l*W!8I&7?dsgc(8#AFGpSQTrqd%H%SUS=gjR2CkFp&x2+WF0hSk~v zMzDw$q%RcdebAh(V>05b!!iZ4PkNfyS`-bQFJOn!xEbP`woQjRmI7ar(({Kit2$0y zOvbM4qx27-mgLIOKw(@{wK+sm#Ys_op=zdz1Zvkh^|-*XGU$~aNuXSez~FFxt2qY1 zp-3rd0tWVzrdu?HmwXu{qni;)4Ff1ls=gb(%@$fzwfWbUDN*j;5+&uA%eY!6gucp# z4y1qx$&g@%h*M5^LTSz%f_rbQoT>W+Iec^@*Umz07prDNepFHMCsm~@GY;sS@^<@j zM#U5`Lz(EQKczFA zlFQby8%mBMyYI5s5Zi+;vubFXjG2krEImRgRe({!yRsshm@C(<2DtdWO&4sZfU)U} zAka5*yCop5ly;kN6Uz4ogm_&qtqq==sb8u1h|CATi%jRagASd+I#ybXxH-;IMJ89E#idsNRD-Oy&7MGw9RRjuQTLgi{<<81TWup%!INF}v z2C4zUc@^zC&xO)}z~J#8rkGY*Rn?{~0~y1O=ylc962)tNoAk)}OedNj!rKF?PMwd^ zQShR;J+`u7E5^Wp%n!bFs?`eh&5I4;pThiwk%JC3j>8UMrs6stvDQhJh#Q+sQ6Qyd zK*mQ6xCawppF&Y~pgKRguV!3pkbE*K6JE*|Yklog-&yst=$xaaq8)s4dz-s;g~?bu~Vt@CgLIH z?#TK(?Bs{q*9yAlp|pz*6*W;Cd5CGp zc#>YzE)gx-<~LEnWsXuz4`x4*jTZEo7~Jip#lXP&)fqjPddMXlja-{KtMZ1#WP{4O zo^p#VR?zv1$juKg-)8th2#VU9ib{hOt&(5ZgstKGt0mK%6dYHaed0Z8Q|xrw4t91v z>1VX|T`;LqgO=3Bh=73)B^=79U3^j%2VkovL4ZBC|0U+L|ye3Px1)&=f z8O`pnQ`N_g)a`j>qM>K-O@mCv(KjrUv>c6HT^O#z;0P4>)EW0sCM`s(^X&U7x&j&- zP5_ZJ1&-Y26C4M<1o3 zn-!q1q@#NfRM&}VTNv;ixX6ZO(WN=;0U#&62XAgDJY*xQY^L>s?Ewr}KtQGI?CS@S z7LsG6b!M1O*%`pxdh>C+%vi!lSn->Z85|=LuqAcU_W+F|!38LSjo&X+(&iK}%ZQ-h z3`IX=IIbz4PloDt&y!aQ$+6AOMD0v*@Gy*;FV2jv?`t$RBXSE|EnE|`V8D@9%-!f-j+R)pUULA%j0Zhf`3@LwK&Tk(+kMZpjyDt1PllZ z-6!mme7Y4-ECV(ppnAD2}#8{6Oge3-i2H4?07PPiU zSq|F}Wb}y@_8@B?o*;@Gou1jWKEVtjED%BX^t?N>>uQBN#75Bi*3~V2Og1Icncxdz z@VtdeT`u9vAge8gP!#LS4(@mj%fR)s-P2TrjE{t;v4dL>l$n?`R6*=UPSVY|VJx<; zEguRb(z*Rm_T`pJ4L(f}MM+fadP=lT@ufLIVy{`RP^^%o#)`W@KD9TF38xDpp}9&*{OVumc3vzq6nfKpyC4oLHFdWH`KE$9TB0A1W3 z^Ar~;v24AhAT};as(0yr1tU11Jn}w8dytriS&Hk3l<~l%=wkO1s}b=S{cN{hS+w{^ zuHo*n^=L{mx!N5DV=gTWQP)A~fU3>-?6`|-Q8|wZUZ2s-cLX$f7o}jntT!<3-j2|WQ5xDK@C4< zI|GKrC2)%7UaO+~Tqch*6r-0yd~$_3Ekhk`ERV31<)QZm1x)z#+ zSxH-b^MirRsbCxGs z#SfrOTeMNQ4vSNzp?eDa2DbvSMISkH@A5((1>p5QcbMTw!{t&4Pb~`N=a=hX^kDBYUu%6(aDwOkRlMoqlaH`xFlQX z=dYH00ujOw_FIo1Sc<7Wf{X{zm$stC|{sMDmXDmsK!R^7MEMJiHY4JjHvdQFb& z1*!%4DO8WMPIa9>LjA=}keAnsFJ>L(${(wk(Qt@&vr6j`$AGC^&!Ib%- z92U-u2%9neb%7M%hIk2YaKyZQVx25SXxks*A6B}earfG`Q zt_9K6-poDDcX&mwS!9W``i`&Xe8dF~8+C-q+DnP3kg^{V&#<5{fEvJ)Im{tsB408! z@zDIW@bXCEHsdd``77*eWzEfR3h`7Bx=i3QhU5^N8_gk615iqZSZ)M58T1_|E5mgq z`X)gv(y`yQQk8Lv)?d$38V$wHjXN~s1hfn7sU`ShNqusf*`4=p4+q6!p{a* z2xFINP3@2$Zd>&&%CARWV8sfCBBkQBE>My7_n!&W2p(Mb?6f%4r$5~dU-I1(mlWVJYThI~>E-Dn8Gixi2 z<<|gg_3?vL--otu#pD%JKyZ{tXS&KCFkWSdd)}HD7AC|MDRo_U35;UQ5t-tW;vH9? zCaZ85lz5lX*<{J!Hr~y>`cUgq%UdZKyQNH@tMjz%-AZDX@hA zkD$!h$|{ASQv6t3x!Nv~BpMz%D6}I_yo$q)A}E<%L}N zZWSxYpnwLYqWTN695IFXL*3)D)Tys|4kajNX9YUy_i@%!QW zsHVqa*_7GO(?yR~g&8c#!mEeqE|@S(C4w0OnJcoK1d~2)`rTD-qA2hqOl`5?8GL;x zfE(r$O(CLQvt2OIuQ(r%C%Re>yEq&>k*qFU`j3_~?jV?CEOjOzd2SYXVpq8*>tA#- zSV<#)EA_p@H+gglB$9S&>IMS%r7%D$(vsoc+X4{@wNIm|)OZFm&w>idvB(XQGR!$` zswzXf!3&vWe3(a_Lxf2@%nF5^hj0!kqJ*!0aSLyzS|PQ(ED6?^PLJJ_veSV4{!%IV zA(Gn9&t7&wE(LxjHS&btw}70gE(I>=$Z)&r@0r2Z^+8h)j*A?aRZ7H$TCBlk}3(lrqF^EG+ zd9eA&Imj;~6I`rbG|Z6L*#|DY@(T+HV1P*VIRXND>V-a00K+{%vEE}$e>Mjg2zt$1 zc-S$N>OK`GnV8Ub>U3Dc3h1(C=lFYw-$z=t)5EYZ!V?*d6v{sb$wAu96 zj#az1iZREd8kjN8t;|vZx3-LaT~!}s%JgxpMkrJ=sU`?YJi(r^EsVcNPk4=&UiOGv zTMYY2gNTP;9BWE96$^6E5<2=#tr~|+HlW}Dwq%jI7wW0n3oN1Pz_;*^tyxg)2G1F6kIATcmMEsJJrs#S1Ut>2+o%<-8SHdkv!q8NiPg|wURmEi79k^9yvY;QV1 z&Y3KY^_s86*ig5thdCuFoG;1Aql*;G9xhJVuy(5tBlW83PGOZE? zu7bzBrwL~pMmz+nH3gC|Wx=Iy+7HKSmY8Y*w3J<|5Z4;D+{hh0sT7-ijl_k(Ym9=~>x@b<}WuC2c z4SGr9r|&G?qQV}AH>~d#amfC+37teS+rkwnIrb^U4-(2aJRByKs~4ZW4&~j;xo{&# zFXP__WDpd`NuYR=d#GDj1`a3fx4c)Fjk(bA!N%fYj_Mm&`0NCE(7?bSo#|wBX<71; zg}%_M(+H1o2QrP$7z}>+80Q#g2oC{q{w+8>XH#gbD(-3aO`D!! z>aL;!sL*i}2n`8bT9TNAAUKG>Gq~Vs)Ho7eL&MHZ*-vTT)oQ7tI2gND5XZ%%XGLD( zs=i)w2F8&CxJ8c0*<&^|7zGizt1$}KoT(()OFG*ZxL%nyGwxk$>~v&1+-A4HUv)Z) zq@m3U`3uu&fl{9(t%L-@V1SV&mkg~LA!GC>!Y(8Or>%)ku%BLrbWI1jaa2z^1OyO4 z$W#?FmEcr?&z3X`Lg!QLk9qA-olByWYoxlm0^qJ#BM@Sn<)RZ+1U2_ZO6-Q34Ruy_ z;l59i@&E~4E(zijWrsAGHgZE{miR!1i?qE|CLka)%^0E%m_D^#J;)wfNwFUo9HAo| zVa%cksFxA%X0l-R0ID3rGjnD5Q`6*OyWNVzxKLG%;w3&Te)cG-MZX%ydbUBNfl9>N zy=ZJ+l4DptoIdP~zN|mmhRU&R|5fw%IG+ z8troJL3cu;s`z@A{i2apR4D!B0562Tb!dy28Wb!ru+<)_R| z7S0r7aJ6~Ab5Xv~rX}0n>(5&5_B=AN!jgQ=Jv7Zfcj5q|A^amN;ABFOjrrhE#oH7R z4ZCGNu4x%$1b;DwL1Gnhos}>kxDwAzcLhNrM35vWw<80#*-75SstEEVg)6*MyyN-G z!@pR=S^m9=08MQ*;K1j!t(XcExlDiFAjI*9#<7+^lnEjv3F z$~Dt7>jB%YgXi*E}pdQ0Vf?%c*G?qSY9uzKOkXKWREH^vxePQlw=uF5`Y&;OU z(D?l$faF4?abla4G&NkQzV#@Nw08Wa8f3k0&A~FrD^UgrAz_0AhKLs8@#@ni9>UtP z6Y)7FfXaj^VUSqL`%YoCJELDiz8UbXlqx9Omo!4Tds;c-qRU=elu&3rH$#PS$G zM%9QGAQB+rL-UJQloa>8BT9Cbxt0}`&jF>Mm0%Uzek!o&NSaMjd42{7@IyYMy;&lXk}GE%Eh7B_u+m<5>^pFaz1S*mpb`}LEd6KL&i;X(AcPoLW1b>QbhFe=Qro0H@O^# zIO`CQtA%L*QNT@o?I}FEIKpKRQEh-D1%baHm7}GPAc}I(GW6K9Yz6;yvI0&#+t-cX>CP8JvQ|7to5RcVS}+W5V>;( zmXJrQSgMd`LM?chDB;LbJgF-7wojTLIXa}>IJB&>x7iA&1v3inkxGELN*^mA0D8TJBxf_{x+W#= z4CeD6JWPmp(Z*Kn5Fyi!suOZyhexGIYJurX1Tu<5Rdm7$%gmyFP#G|?`5N4X&vVxwl9-#f@+=d zi_b<8;b@-k6-3S;$+BQE#ybzf0tgfQNGxAbg zI)o9u7)|w7H#=0U*pz9|*O~%{%mq4hjz+d%Py?)(*|*(-ft_K&s1m31B#ywsb3`HI zHhWQCFR!X?N@s~8$Y{kgvVg0+CSbI5xusgX3XZpUU?D5g)G7rco_M`O2{Dx)0Z@hJ zrgWrR?n>2u3I-wI;J#dQK=GmUQ}WM(DP~*;bUB z(9yIDxg>s4V4atgT_0)bzgGtEc*h~{gtkEg5-Lbwt3iO7af6pxC?@3?se^Z-9oGyy z=LQH`LK7|!>auJ=&6kb+N8&87q$W>|>H}h9FB(^D8=zf1X2x-MAu%lRA-bwZTOiY; zUCW{P^H=aCqXeldmxmJZCTPrRdiAU=gqEI@oQ+yU6F((5E+bvwHbEY3P!|%tbSc)6 zbz3#|h_FASmir~0USb=N()){%0;Nr8{3b)oD{te#+XV>_8jNxzUOme;;w>{zEai32 zWf23JX{&|{CS>G^UTwbAma0T#RG}Y*EFgZRXXDC}0|b$+xuKx6ezG@YcTnzB78V4+ zpBUG%oVJz*K~&#dFN=A|7L8qLBP6szY;2$@4%tCdNy;f-X*@HV>pm*WIpofDtz)a$ zIlWXs{ZX9GSqf<|EPDyioe>v93jv#zMAmaZywh@8Zu$_kQ`V#kDh3;{B8xeD_X}-x zhzN1apzCc~gA5@TK;{4{K-Iq(Eb%$i6RA2jotSN42vuV1TDT?@y;+X(cSAl_o0q_REHqJfpTZt_8L)!B}9-J$lVdri9#Qy?Q7$}ftNz3jGBXO$N< zlOV2=VK>2VVFnOeM!KAbs2;M4BdCXtgC%kbIsAY+Cqm3w^Z|X^L4rOIA%u@KJ+D!a zkGb`i60JE_hgi@jzZwVJd#pxHY^1J}x>BSdSU~|50Epa6?ex;%>?3Nymxd{HBKx_Z zjuEn|5+}MM! zz&>F!Kv!kl_S<1$FBw4Fs5C49Bz0c#CRm-3$4;RzsJ}bCud#$ruChIqIBc*w!KT3( zI4jU9%fj5DoYAkbRBP^8;U6^iT+&US$2w!=1L6u6QZ6NqZdX^vZzepxs9yNwxg0sn zh7bzW1iRSbeq9LAV*vWn;!_R{5oa+*d?Lci2q29sv7p4X{lg}~-nDSb!ykq~km$oE z0+BIvj7NI;P~Jv|LDTA5?Wqt?aS)K4=@CFF&Z3SiL#7OT_WfEelV3eaEvHWB46_AP zb_Hu=vBVnF6jL(epu6|hW}6a1R!*-rN514YmB%(8^tx{=b0vfp< zbW=svz*VBh?(44JZXS)W(k|uLs(H`fGP||}oH>Y6u!mYhupJT2V2aBe;>uF=qgJY> zpQvj2jtcmg7Pe3tCkbO0(U?VW(rYlwvPDk_c2cO%$kFF& za*jy7mq*<%A%_?hS*6tyGO*lfxIters19*J^**85Kts7n%k)k`?sn{C0$;tgWWg*i z#&xd=(f5S?z>xRQ#1_S!;3EtcS{}fGzH6KYhstz^1)E2IUM+eGkznD|Hsi823eq{; zS(}m=e5#EubDUpY#@bc5mBI{ntEOwI>y^DB;}F;Bi14Z9H?&{2R#Ob9!U@N7+}?sj zz=rLf#~KED0X>s^pEDyG9pEonI>pQcPnmt$;6q@$%Z^MYOh9LS3LQ0>4yJ)|)c7oB zv-}9#$=Y-6Z(?wo3FeX6_)r*Og?=zG!<)Vut+3+-I*<($*1LuyND|05!>Qm++L!Zy z^le%v368%I`LwfAxxX=uoU@80HG2b_)_R;`1HfC3mQQgbfhH4YwQ5yYv_kAUy9^DE zQNluKp=cGybx2iRFoYxTYR!6yFJtwZoM?IrFrti4CFXpNeC*QZ{#IHXYbI zGk4V}#~7=U6=+|;JmoO0jW9m?F@8=w!i>mBeojA!7TJu5qRAwr+0JD89K#Ay^LP9`y$ zY%=;uh;?;jG7-w-W$w)x{M@L*y|=RN=4JEU2pBUM9mgp58rf$@cb{}B-a$z7y5ZGS zq%_oLgYjX}VK@UYRapkawsb1bfdeeaAs3-o=QR__V*g8FxiCnxkFIzZ8lr81$!W5; zo?%jxMjYjFp(icGoJFWSn5BXYr0k8RbB8@Rcu5FhwIJh%?^K}Nj@4hC=P9wwuEk3; z$I7s3&4~>OPAnbIP#9GF-VIz>6DJY`i_})ez(El57LM1I%Zp7hW5(<0k&b1YW5j5! zEI^YiVOI0__a?B|`$uM0r8WB?=IJiz)Rb zvK1-SAGF13;W-dtuObB9*r0=PnVRZ`2yVfx+faC&Q<$_7WiBQ<^790Mi<=N;s)D3^ zDHcYpF)z!BHb@CT#UtECtvKp1lBAiitM0A&3Mb)U&_ae)-`iDfDo#2Q>+juVO^(WIvTL5i0G)w5K0~@LjA*U zZi^;)s+bpK4XHI~kB6MU2j6@E0q_US-=3m@t>^P8qoT~>H24){(P%HE?Jek~U%yng z#i#6^)E;0e8;Cg_r(%wNk~0W@Ft}zzGKZj2NkzNH$dD#FUCWH9n_NxUJ`x)C`)i0R zj7Qd{7z_^nfD^#Y&k3B-*n;&6H1uHL2_w?ar0qns+)VrvXuzj6j^N#4LjW1Qgy4o8 zUA4ruJeFV}B;K`2;}%qN#E#3Io~)fT>nic0MI>$CaPXjGaEt863BsloQ(_m$1sW(Y zP+71#3wp!BCD^tdsK-_sCF;6SZ%3;`9FjWFWQz~5Z+?cj-n*T6hXQ~e%?jT#aBfr5 z@TyIHOpBPii@Tv>&PS&B_~^DoAsYkf4xAU{4o@snqfM|pXdKN~3BSIlkxS9Qo#qkf z)IyUJTjzoU3g62h0i`cX1V$j#oY#@jIjiWq@+?4+USZPQbDDsqgPab z3yJg|U=jrhDy3S5vmCrbAdfG~q1exR#}n*J=+50K)8O1FoCz{9U%gMTr=*%+6nGMq z^yyj_?>l!z$0QggcYIIOJ9P`kcekF7^4tLkR|_IJFObEliJ;!c=kwY@I4#8F=>bJL zu%)F227@5M*jsK2Nm}{NNME5PkZVr~?Lpui%ELHEp;H_S9K;1`(-lI9Kx$8vEs95S z)EPOe38{h>TVg8qVNj$@ptpHa&C^sj#C{dt<(PDV;rdCUsx&OULofE5CU(XRj^LF` z&NnSJBPbfW6;=1GOdQZqd|LN%^a=kQ|`QEvbntpcOP=XE9U8 z3X{|$N(&ubd#mkoP7n~@avs@>>VO^q(P`IN31Yt7>B5gMpTSPh^ zQpyHTmk@-s&8F(IPgAiWIXqMa!IiL=b~1y9<3U zx~YhFQvwu-VC55veu|$4I8fEzb$0Txg9juyZf!Rro=2-fEgF%o-MI+uJ5ZcpffpY` z3Z&r7+IhClinDl^OPvQT{aags19%J%B>4DVRWUG0F%ZXVQtaZmOfXm@GBP=7M-fdLkN8u>&)ClaW+djM)vzzyxeeXm@b08 z797&y1YqS`7NRCBGVmF&=U6qNgLt9iXKGC=xhvXw8;1Ed2KgQFdLnqkCVT;hChzMD zCA>ptq!obUm1FX0KJvP*O;al5Mdn$mvHqk(Jo~?C27A&+$WgS3D?gI_b*&Z6+o|`RWN?h+cOm}4dEk;&8F@a=)pmZ2}j|%owBBi8(qw{(8`qnpX z%eGtI<6Nk7%P7Km67@@+>zt3ZJf_9$%d9#+8XkkC92|NV5>ALUS+-6!z!E$Pg%{`^ z^$FZl+X1&-pUy6fayhE&xF;{iTC2XNfwdB#qCo*|)}}7+NDlocHEp7^9qT6`_IJiA zP~q?iZueUXAU4o2-CRMSDv&4BiJGGxsJ5)xedJh-a&VnPy!CJPRWAVkF;?X%)m$ZR;1R%m?f zykVs+uel~_t=@=(B@WoB^X~GZ#>hz7*3XgSK`a=?`KD+f;3q7eK${v9;09fo0}ZUr zng!E_d12Zcsvp9)S#Zk1B=ZO@fR&9T}=cz z@45V3$sq@pU^h%?_;UOqt{LG++^OBGdsOML70w$|aIAqby1kalE>W9SE zK#d)M&57GX~em?S$}3KuJ5G9*?XC24+R znhb;jkja-36p;5Z)IJcZ(1Z+=fKavJxGP)EvWaGmOda^z<;l8vAQAY;c!0cQ*S-X! zKiFA5#uE&>W<4a=-&p34Lo!f+{#dc5iE59w-!1H)|RjzBHLS8CM{HB@1d6s;DQ z=U1f&h1ol!C5r zFN5L9a^sl$=-xSDECB~+?R5|^L79qpM6+WNOPUE_^7wdE7H$Ody+^rsMB)&e-b|Zv z0wvqLvfbDQh%tzV@SbxQEa7LW5TGf%+3N5(Bk!OBvu*F$>CH#r&~2H3d9s&tv1qkG zKqc~aUQvh;RcwHF}-7}h;r?|^AMq|Ourphys0c$CKNP`D= zwsgr1ZD1btUzpLnc>=8i*d?T(Jf+Ea@QwqiD`oBMZmH}gX!$iNB=9R@rohYjl=AM7;$Pq;U&QvTtOH|zYWwi^#X2IXM#qZynaY^LTNX5y2*fJEV6v*fDN+W~AMhxiy}6_H03~ftt8h1mh)#u$d=WDeGsM zOwD_Ad%6~qeMrEwW*b0NNja8mzKR>LeW^%ZTQ-vm~JhM29Y0e(f}9Cg)`D9F9W|H6S4=cdHSqNSAD&s%F7Sb&K1w zS`hs_Dm_33kQd^=17k@})Uv>)znqw{+3Ql#hc!JX!msLBi-iuFgv=>-l!~%{WMz=5 z4)%2&!oDL(Stb!Wf!HT~p)uy1oO{7ue(;e7n26cH$b&Nh%CmFc%qR)OVtoh+MjS<| z|36XdlXVfZQX{a2Yzx8&t*V6!GG(aX6U5z_62OF>!rd7xyS^yJjrN98axxK8#b|my zeTFG6HR+7Yzb%5b@RSS0&nV0vf0Y?$!zX#Vk+tesD-fthEnNc3#>;fayYfs3JX82aMTgHwj!3YymeYvUFe@y%QXq`F9N9|; zo1q7}h#9 z;;pjkMM`;24PcQ7lK_q&QdZNzdeFo{0jh%Sg7B^ukcG3%fy2f^0@zO>t@V(r(yhOU zBKVA>(=sw4o{u>4Xe>2if(jXJvpSP(TW3RjL>!0|!1Qq!%qvs2X>}M?GpM#&=TnT} zI5Q);G3X`2#0DcJ2-Z?Upu#318LJ)Y^e8>WW&!mwojbA~3`t*- z8YkR($_h%Z(_g&BSWiJ(^@JKIJ*7Y}!_AliUDV+qczR^Eb}aUyFJh7iTgnc=r{X?E zW!&WHAa@Y-c@Pj-SO@M?I})KwzA`k<^J=$}%dG=jC_N-GVD+KdO0iYv2JmeI$Dg?X zeoTU$RajYjhE4t9#rjsHAywFyw54w-bO0YrWfW~WD1(-hE1v;peLh} zo}g=y6&<~_hLB&a*XtNrQ+DseL@-rL_`+`p<>lGo-xD; z*+nJC#?n`?7KEa}QRK{Dh)5`(XLGtj?(LlIgGE`Lp$A-L94ZW>3t@)v5I9@kV3cuR zcEI?cZl*-uzwaq&?M~>qMn(ldOUMjkoe}1so&@4GRFzMy;AYe$LT5}gj>3b?PRd|H z0z?=*Ar!C8jX|fD7oy|PfK*H)I&fo32QyP_$<^tOf^w_7X!eZ^VV?J6RyhVLxdmYv zg9()`j^a0{-`OpKq$Hc9IS>$T$#doD+R};;fO<{pwr23qF#T!3rjZ`lU>f+$w97nK zrCz|B;RBS_&H9-ZnOUO&fJu6#< z6a?w^Fx)OU=&%os;jaM_Fl;VJq8^#YbdEI0ekI{x5FrSy7GDUU;DiJs>AnI3LMv~X zUnekuML4^z@K59lXiAj!Od&ofc>xJ!Zn83<@*P8)KW_YYJ$+SNOdklE*Qr&o5@r@i zL??O%=J^~7(&sJbMa+-{OnQv-?%ZHC$4!9sN3OPJIC~C|M8_71H3)~?7F^4GEcRVP ztoN)&2K$#k6-_UfQ7^2oStY%#_lZud!ex~x$F5(yi?l&S7P%mIEsNHFVlpabPkRYx zeuW@JI1hlE!UPf+;$9Tsod6dOv7tC6~>TgRc^7zA*A#79oa zrB(%=!_06SrpAuvTT2&`aQxI`?D|=V@Kz&QC!gsV2xjyb#snb-*pZp9o6(w>#m@gV z*_RZj5{A&75K-l1P+uaDWjryO3-}$@K$|a<%s(o>jjGUfOy=M-WF2TPLaJ#U$4WP0 zPF+OW8)%af_K3r9kYcSmA|_9PMprXqa0+Z7M+hq&CGB zb}&<{C_7j@8VC$!w6j)Gvpxop$apF`{NRy!HpkdfBw}_*c{j%sfu4f$`KmXL2-(S% z;pazdW}IM!kAkTQ&hDhtKZNbF+7`L=n=!b=eri*&gB1ckC@nha$_y53Ud`0G+SRE@ zfiss5L>u!-z=jNJY(*pi`CR9FJU1%QMX|ZKTg-DHjlr@f3`B9Zce6aHE!guDw&EF_ zSE1~z8baI$ zoZ$GOC&Xb(k7yCLIWia+gW}7VLfSKB&fGGG$xuQ|bqD4J@fAcV=ohD0Cpah+ME19i ztBA%eM19RiCOBIiFPT)Jj6Mk=m6icbMaTxJ zCbFatbkpRm3m)J$0{De+dg-7H{fw*`8tp!xvTg?KhE@QP!K{u&6{U$7JGWFZY|to2 z;iBmKRvWX@>EzBRAF_**lnFP)cMG#PK)Xz(h-^tfG`sYp=_~pjVcgI}w zw%qd-q#w$&>>Tj`8cvmY50?MxH=M|f`abz(FBuzW-Cdq(8^%v#v0cMJA+SF z(b<tAah~)1)k>81^TnWHKV?_QeN6Xfa24>_KRDlS4b#Q7iSi*ZV-f! z0jUT+!%;0rl=%sl_c!P3r)0tPSV;Sl_Wv0Qwkkm{PxZRBNX%quCK8d+Uc(s1DdI7fr z>$;6ZwHC5*4pu)#Du|W5l=l#(9ynV}qmO$`-vl)Hi$HI;NktZI8Vp5+Vo>NG5cVCo ziJr{8m-3Jn5J80uNqyM)+GFi@SkYC9Myvuc>I3mQtO<7lPSUgI7-LAMsAoyMp!{-e z;mz>!CACr07afv9FpZRxbze}i$(pgs>nJe%9@E7$%hQBfR;9fR2@QltL~xfC*VJoS z*saJTcc4CD!N8V(3E0^i1Az0icY1M*D2#krA(3%5a8Wxsg~)ahbW-_3dAUHtfgOX!DK*kg=1u2* ziR9eOG_x5QkczNN#8WstVHKkZ0xEh-tcDlA9Wp{tXEQlBa}yYk1F4k_#!RhyM7%0S z`iVZ-W3v;{0QAfo_+$6;I$%`K4cb|b>F?p>68Bz1CbK4MY%huxSXE~Ho^hf*AQ zk39XnT}1dYz9ToyYzQyL93k%@9r*PNF^>cw$>L;|aMj+im7<2bm@euQA(%Ey1P|WY z#A0qs=y<{?boA@>q)=x#Ma|(L&6@g1s?UDqodk3~$50`4Eg3scIHfHGxzC_$y;6D{ ze+u$(%QN{27^PT#b&S<8x$&KOw=E*2`1b?NH!f%!xPU599b8~GN|YCb!0Do!;u5Hi zQ9qr&4M& z;#Tb-+ei0^usp|!q-7JUhanjQ1QvkVOG#}w73iqXO?i43RlzV!%Mo#P9Qa%w#cJ5) zJgPv77YxBeHbB{BI1Klw*^^Og>;er53T=b^>0dZcjB zyxpDIJJw;ZAR`%2;Q&8HhzJpjg;NFkvCyy!=1n-_4A^+ewCB_H+>aX4;xZ5@xK>Ib zO8qxzry*EDk(CL@yU1vuo{16Lh2M5+8iOipz!C7%#jFX_(GxdMBQ7lcDn{K>j4*iv z+DfxzX0Ekw0%NZ3nOT=DGp5ou(=@XQ2w4I7{tCVj?6f6hF-(ZQ^B?SVO|^ia^HCGE z2u#w`JPH7VlCcwl^67=&sjNsua+zNkBI2HC16chCp_NjVrW+O$TV*t22&`n;Y^Q2S zNJesdWNp`gk$v*;OyGc^(N6naJ}eRh<1)rp@%1?7JnSQlZ;)yizQ7EatBRvn9u%Ut z?GunC7H)c(S4b;g4ou}XHc@%}fuAu~tzn7LleG%6;i(Dmfsdp&oEtk z+&C~9tb9BoH5tq*VIl6gc_{ZNUC+>6QQpp5od__}Gml#O4b|nI zi!m`I>7Mn@Xb5A_!VweBl3M3!ql^L|L~JvU4I$`0kwLdY1skm%L(ypx=JM?<+QibN z93ORl4bO@sx0Z4K&aINaVzRN$22GgVJM7su#$g=7Md;ag9l%5HtHBIHfmE?apFBV= z&h6MXW>+~QS8(bf18rx17t-VeTq04lvObB^vH_6L$U?3=ds05G1_XKDx_(p;RI&%tus%}MLy32JZDE-as*%Op^*DsiwZl|y% zFU+C1O89-A9=N8n*%_k^azL(7@p0-*4{Pgx~Yhg0E7>Wr4ijIOP4O+9c@x;6*n znNeAYY0M{rJtKS7W~Lk$`{4WF4}d-otUr1GR%WAq41s+lSr#AE+@Ne*1(Ge_+8V*3 z3tOsI=MeP@d%_AwK>(ifgrNR}H8CDQ<2q8}E+QHb%(_`z5H?;z-jS~e4&z4pk^l_q z!xroaK&DfHPsC~TP)=Y+fJ&Sd(@p+~0Nl+xAub^!Vg=-0LQiEmD9rDUuC9!;=7P0A z6zk1nq=sNVn}Z7)kt?5zyg_?`wyT0ImY_c2j zsgA2BdRjDf8MBn~xLOcclb5QZfI(f0Kx9;YedO$%T1He%je?E>G=X7KgvuWT03>mnM9E=o$>48{P41_mD!Cd-MX@#QU1k=oueHQ6{xk?sNN_aE zF%nrJS`bw!dlhp!XSCt8kJ=xN(^}BF`Q^3Dy0hC+(m{yH`;oH zgSEKJ=i>GX$S4l;fs3-4KpZ8Q_mFA>nd3 ze5*<_R3_DxmWcQ)DrQ7-pS~@UG2oK1V4@J)#0dcM9=0ZGATbEtpJnvFH9&5uf{(8h z$jk=WsBlFJZK@A$0bm};oEYGKglz;Fsq{PG57DFyXd8K`FF+z6C#QbJlk}xh0kBnR z7-}*5wZDb7fqYQZ>pSaO_6E)~3t+CKAvtLz;StP&4tktK9B}?ZPaw^x`u9CXRO{13 zl>t!u;-DMmo6WawZeoasKtq zBKFWs2V0K}#hTt2vCP5BSI4U%)FLfZ3CH$&Sw|&LJ6ZLZ(#Ecf2eH@WZ%In4REq0K ztvYBcl~e!~Z3b{QrSECzgm&Nt54@VY9d34*HBQLGaEoZqvr|8R1p#Q zMyC%Xspx6lB+l>$KE%XTs1Du9OTC?@g>&6Bu7n9#7`j>xiY@%odJAPSMj(7Q_pQzA zix5~qcz1(a5bC>1w9BeHRBd<*9P*RZlyq!PE5SE}y6sY>Dtal=*Ann7IGnV0z-20% zBx6TCE{Wx>K0ei27P1^vph2YTD}8Ch5{#o|1%@>;a?HsWP{fcuP8tCo9Zwqz4X{2Z zh6X~sUBpw<8_1c6j8=(B6zcyM8DtDaEh%6{u6U%~) zZ5UxJAe(07K{B0%zH(gfGu{|%4!1zF5qwg?2qDV8r&IFd)Ky(F22;k4TkL}|2{=Gt z?$T_6D#RgGy7ge#Jb2)SOo`Tj%rf_s4PXH22!*nd)U%80YOxm&qhD~@9GLs&S%HyfgoGQKW>OiEM*D|1Z+*|}c~H(+q}T~4C% zE*&`LL=x*=qMB4)r2441^Aoi)s8lRK9jVcDODY}L21T!EcvYYqTU@r}@2;!}`9_&` z71o(NMDUf4$z@=x3JNv5B}n^|9DqPNS(hl^dwj^i7XJu@frBVy(UAONieDl+=_!K+ ztdfA#%*n68i$?LiPXrYVj`3Yg#pAf_8|;D&4YTmZEv2D%SQ^3RYe?;GBWZnJqVy@G zXUH58@K~W1X94^jvjca4rAVYSoS}kgML>4vfJ<59GbJvV`-Zhf)#+53KI}}9 zI1vT3^SL>_;}xYLGALhE_l0{aqktinJBNizKE;k z659P}=&buG<>9|B;cSInk#=9ZMF;89A^4E7c0G)^yIqwMk&7ETMc&Ib(4Cjq4fNvl zpz(!^FC|0UzMjnF zM8!8!x5))2`T<)$nyv>GRfKs+Wa4(m#``RZHTD@R@Ch~;z;9b#0_=zl%=b_sR~hYC zR6^FG(;O2<7SB*KqS4fhEv}VOOe@;w$9O_r3ydg@aLl^gX5~1$DnbZk?3~-g#zzps z3zq|@l){2*-7v%G=8y&XouIt1$+Yjr5vqF?QK}7-8PBJw%3D?}qc-4E^inNdLV97k zU~X0nf%F48fLQ(Ym#x$|c8lm3h~Q6wH${P2A57ff9ZTZknb9c2To{q1V9uxEoF4$F z66ttAfZhK01`mNRZ3Oy@w7|e_0(!+PBkH{aCZ$Y8NF(j4aZ09^%a~YKkVs40nQTqB zk?XlL+}>HBrIVMF;Q+@8=}@V%q>#JLPz}>fTaGlDU#(XxjVZ^HWOIw}0S;wDDX?0C z2*x-fp4vk(k1{+z!RCC$Z6V<3L6Me%OKJ4PvGt_24g@0%uo9*?9m7|o1m`Wik-&3E z{xF^I9Jpp?RxGCDbC zT8Q}9&Iuw8ZCI;B3wOs<(Q5PEd}O9PEbC<>tL%{Gs`?efq;W)V$!WAMs@a;BOq1SZ zscd!LjLi_->5f_$Ywci@F*D(Y!*f`@mBx%my5xshe ze;1*8W}~Au$s{Im?_LNrY5h0^7jFXsMwjFbc2{$FcZVnXxCVK8mR8Z>D6-)>FLw21 zfC!uFW+Cs1$dG10!&>z>X3mwIAlO?sESBFk6$apBJG>qU+pVHFHF$oGh^Ze5N7A%$ zd2T8~QA914o=AW}nkLRnJeG+bXOk~wzDFOrG5kp!py{i*>4;=Up6Dry4?P5EOf9&V z1{YalNm665xC_ma#rAoA-sYvjgRG^06Qw{UXe(NKUEL=SD`6kSkbNK&oOr*X9;uU! zrsPwmDkKOux)l1oUTUeUkw-R&Bv9tTRANJg<^^VSNPAC$DeW05#BXZ@7DCK>2zhkj z%pTjyr|9l+(`rsijX-0?#}dJl&oGS6e#I;K0UT2Cy zLCDN1?vAV^SfKC~CMIumlhj-){=Kwm?6lJES;r1pDD;U%2c{35vF!*R!c-wv^kITE zLHvqT5{H4_15O3M47*lLxd5a-BfXi%z-fU{l^JsNFz8x z&qmqMDnb@oL+H_y*_W~GDWHk!7Zsik_}a%zu}!*0+bdJMKwn`f`#&=A^Ij4undwb} zQ}Ksm#j6kB0*r_VCsw~t9m_8!5I0)ZJ;aN@ZQe-_T>wylgVEi*QHTn*7)CNKA2Dps zyd`WuQh9$d8<}vioL3WDyQVhFqT=`_8VC#-l=5L7ydAa*t~*65A$D0;w~aMc zglf$cnh^*naI)9tK=7B2l&b_%BLx-lFSdUlKg@iZzG7JL`zB}-$P7EfBs3s&C<@qs zOKb=rCsRue>q(7K3l4?={-o@)*(BMm z?1T&#c;}Eb;i~W_M=yc^J6Aj@%|+OX$v0sE-HC#TKB%h~V*+ZDr&(cuh+blem@g>I zty~3&P|UH2DW=&P$Dd#Y#o4t=$T?ln>nf*AJd*JB1sFgG zpooGBfY=pTR`aUFok$RK4O*IDx$>}<9QXJ1Y6qAAr7D-@RMA+5iQZ9aG}-X4KNusgNxHypyPP}S@2wP3OpI< zpd)czs#5ggq#k`Wuh%AxI|1m@FRe(J$EHSvf)S3i7=;SHUOQC;!AImK;Q?E>CHdIR zS96VXK~Xk=wYFIdOj=`%A>_&yZ8tw2B(b_$CFx9&eU?FC?QvBn$RmP zhd_B);o|S1aZ}If0$f;1bz=n!RY95Lbq-Qaxpa{TMi-G;YwTtZf$++6g&9-MfpJ|T z$_nfr&`i)O-3L(2U<_mwa@@F}tY{ULzojK7UPkh>?3I;vFPwg{Rdq3?DC&!R0Hmq) zmhVfp1yxI5mzO@4j_&>>Sw++aM^;QL5mCvFFtwGwQH0e6j0gtnSP>MI;^Bec9OO`Q z2ra4oionhHu@Q7rTq&4?U#&umJG;>W`M9{sD1oElMOWnvJCL6>s;md8<0eW8QKeB3 zF>-xQuk}&trKrU)v1*Y-b|~pe&$d}Ca1F9Dqhbkm#aVL!)G|F|{8w{F)H9kislMo@ z^fQxow{$IiyQg2~(QMd*p6yTr_oT0f9hu7^#!OjV5~JZt%8ii&RwtOSh_*iB5K>*}leU5^nxVZ70M1AQ zgD?V8_f)wQ=qO4YF(wbgmGE@N1OpFj_oe29Yb6;#f=;dN5U3cLaqCaLvjAUrqGrxd zVp&)0?_m-pcX&F3z{4>*i5`dnACcmLdRBG7>8s$CJJCN3t;q7F1{^x;JJS{SV4hgG z4>=M-5ou+)h^4=sx{5Ie6C?*)sk6kx<7sP1PY^MIZQDAsrRgW`5Qqm(pAT&URVYp0 z$w1{n@}wo*Tc=^l5Wu>-q?D}3cr!+QoN}xK7W(Wtz3Pd@x{po*3fxbm2ZNBuC*csR zc5dKp7jlowg%(}5Dt#Ml@-&BEn8K8eRgAP(V=UhJiL?e|V?tU&4o0|R6DU;>3RUeD z3x#Ts?otuGLb7U3&AA@-t;jThEl;k}x-JUP9r9Xb%EWeyEMNvoON6&WE~CT&i>3#= ziS{$nFt`Z9ve}6kDW?y@7LX{{fdh9C=kQK{X`K$o@kRP;}kOADIMVqK>~V&C!VfY!pxAl zK9Jov8IUuM5WfSomNa|gaa7XGo~RhKsjG_NA}A+ZB_L!}?p5N8p=S@o2dsddSUIFD z1EQ@vwkmWCbK|dN_9-vY!qkwMsXcIpfHI*ZCF)CRAx_|!3Uqd?jESp1bBVsAX!yd9DVfh3kLNn67 zfJgE1g=mmE;^ zyF}Xj}4~0wULJ)8eebKrNqSMi(Q0gPHQbOQq>WhI;Ysb zD}_=jhjMM$9EMZDDcjtF8{%P44X|6MOGr3S%W$QpbDfVPM)M+UwAz6BLNe*cPQwQ^ zfWxg-1oJ5-&>+D8G9UyH zvQqi#1JcGICV>~IG1nHUlsty%z$z-fb|e(YJ060@by17(3SHni7->}ip>CbZwV1D@ z-<4o>uvia_0)#gz^a{v^#0*RYL;!>gi)(Mg)dumbKu2bMSu3q~_VA`YIFq326)mZp z*p%t^R)li9n_dmGT3tEwbCOgD>H2rIhqJ6fDq_J@fZ)h}b@cgbNH@mB1pFJTSb?@s z-WdHzC7&0RTVTjEQ!|zs8QG$u8!QR^!T_Kk#mr5{oRi-BZ`Pc*oIdugh1_tZMMz8X zCV&mX7vyADBa68}g!g`4YW)6%Ni-uD3fZj0xa+(I zfvKU>&wFbPMeee)Nzfw2^Z*qMfupT6F)QOC@CUyR`F85y@q>cS{ipE}AsUdB`HD<3 zs#wu`@lIE-ExFp%zhE!RX;8}J1o{}RwV)lrsYW38IVT2ZW2>YBw~H0!>#s?@C)Hk zqGON*c^ncn$GKKyR92ad^kE_c8NpJFnFE2ZJ0v}pp#&Ni%Z+kERCiG7VrsZ7KO09u z3Fo4^O=>K2y`nMRXuuaO5zXL;KV8k`=m3Eh2?fRlD-0@hD2Oz{vdf?>zOXO~+_lg= z4@ZukY@qWAEE*H2Z!{nYTtff1kXMO}OYe<(UpS`!XoYp)tjjS22RU}=te+~T8K7V& zLm*&O68FLak?!fOP*@2e5l$4fRZM`3g4NsG&4Z-(OF%*65c11#=a%MdxeOYiA>?O_ zlxLiM96IV&vs3R*mdN&%3q0v!3-jO-*tFRrs{-t|D!xx|fdT-A4#gTddKttd5$!q> zfc<0>8aj30Sw*KNSj-$w4zOrj;y7c~F-Id{deO+C&N#PW=vli#Z3DELeqmDXI-8{N zGQxa_6&-DZxiS!+p;={7*FIFplDi3=r`XczP=Sk1c4)lG8m@Klk&VgrE(ea92hla-{$b0w~4ydJG&(8suP=l-u1q;P9rM7%J>MpZ5 z2f7i`5I!l>js^)IfRgCLg*+xdGad2KSI)jy(!41n=U6a>6;;&ByvoGUbpQ{M!|gY1 zLk_9p)ivX>a!W!EGpO}dZh|C=q5FH!WM;L`7BCvtLVd+Y{ zZh(bCH^P$0-jI~4*@{a(d3ULS}{ z9p(CMvG!F$jjmflgU+_-JTJk@q*JKlLIeWFaLZ=w4dLr;vy@3(pLVAf!yj4s zA%2_*zI+PHQR?~9JhF@qeD=RxnqDIzxhX40+(@~y(3PMDn-24%d-Xk?cc?*TyU=y` zBC2j9CKX#2<(?V3lpUoS)r5z`w;SkTB9URmg$PbGN(1P=tL4&TnTQ-zAZIh)=YgkB zy2vs{yHqR?oUok;Tijw9rgR2pWz~lf){G7|m{$V~zd&rjB5i{XZ#JyKXqwRzdAndY zg71xLovW~~OU^o^+A&UQDqyCDAOUtrE!Zda3Uz#cXJ22MmBpBX0qI75-$`iMkaj;u z$p=B63CU-zM(|`(6e5<-FN@5*&#*X6Z&+|-%3@Qu$OdxdOLVHDsegz~_g#c_^6i|B zglJHJRJ2ln;MuoovEF2vRbFP-OsOchNaxB;>vr?VUBU3433!2ydEO}%hLpNrw^^kE zj}&I;7kiaxoc16$sQM^8FXAeZuQ(avihcI{rS}$4V(Q*+nuH@UD?Llp+nI8haoWtl zj~v%9m9Vr=O5Q=RG}DgA5}Ru*d5=V1fWRH%ER&Kv;}s=S>htvs`q1OcrOwqCmk$3) zVB_bradSLKe8aX_!hT)+5w}GfHg}%JfeBT}tHyjQ#h*YvQi} zl&uK~0j^c<&|~Q%Qa)4{z;JyUb!RQV0thiJVZg<=3yotT6No6ai*4^(;-*zj1CVcE z&Ws((nv(&Z%*NhTIp~&&W5Sv4n-SsD1ulb*#jxTyCc8k`D3DBnV+64R>65CYAYh^l z)5DKmPAaG!w24=X7Nwjb*9F*c8Ao*_i$|?YJHULRaLplLVk$P0XyDQFR16O;|DQW@ z>owocXD)9%IQhlb-T+uYr@!6$u#0WkR@W}j3#qq?ESo6M3@>F(+G9q|D z!pR5g*#a;jTniZx5OD<%(QTLpQ2H=Bab62lyaN^XQh4TtYpLRk^n~lR5gk+z$U|#y`Oteg2^=g1rWI5E%-v#ZxIV z&>1(K_-SLz(zIVgMEGoWoKHbOtLo86RC-e5ODh>$Wz<@?+ijSfCRJo?HxQSYA4s81 zbXIJFB>?Jqth?`FttOw{j5YdKr%9<4%+{33SMOlN*R4HyRDmDlLa2yxVzY7|1J;;R zhZ#o^{5~K7-bIb*_{ZT=2N;_We9#mQei|^to(UL_dSF`UA{6!aCSJ^HVD&}wv}eUu zI3r8Vg&^UhyQU~X?gWu#JDj8~+=9p?DzKQ3Q)e|nIQ;NMi@1OfVxQ4lfmegn0yE2| z3Oskhkds{NWIa&XrzLy5ih*Qs7|Iu3&k0+@ge|^^A&js{ko`ugg=8CsW{J5+4Cxm};hz8{iFop9Ps16OL zh z3CiD}o1c`EuJRYo53{ah0_J?HYe!XBy59n{vqfdd{6`Lk(K!cwh^oq9spfq`@Ecvz zu6fCT_T_c!#zYqd!730yGC*=5!eWauHSY%D>vW!&<|54l;A3L#V}ec3ddv{MO6z-L zw*>8>w=;YK%VUs!82agS)*M}&gG}CrU{GTj6(=kQ(4Jis= zWu5g2Q0O&K$jb5%FDijX3z6qdE}29vphhCP(CE}4GQ@Wsq6tfB>=U_`AUFA>&P5KH zYFtf>Ya^35khQCHJ>=s))U%qxwQDKbAuXTWqvHu%yW1h9{qjws-=wW9tq9jcUiAt_ zMq3!xZ-7u_3asac7!+!7W`c{!K(NCi3o(917&WO-s~86cEm2EAYURBvAuW|b5FmjL z;0u%T=SSD@wQ78r113(EI;w171|VeWCW^smd{ys-l{B)k4X1TjRa`<;(Oq`I z)@E?R`Y;J`06sAffE;0OQ|lTK5YnLvOdwero*rosuYMkm^vV4fY!++w=c@|fL;fv!^CoB zP_by+R`tpO07oLCQ6PX6*+9{Y)n&#lqF~Ps>2=ZPg`#kYxRvdVF2|C{24E@}5=ye1 zA*@U(IR$dcy!g(q+Lp2G$iZgJHfu7YOwcR2e(wYLTqXl$mI~R$_T2k_) zF&gNkkhWd&l)eGUMA52`1Lvs|Y@8ZC1z@Q7_!O+Q>_d)RmG~V4G{J=H38GmCL^_QZL`$a)Mowz)-B>fcLbl{4g5Mu7yh@+73GUQ?x;_C$#rg5c

ap?^dW#gV_%1cRpXyBC7ue9 zY=CkP>J4QIq^&@~Q***|!Mzd8_X$f)!vkNTE#Pl%kh~nJ@V;_=#V>KUq#!VanQDB* zs4z^kZ)3>b?#1mS>Yx!t;4pK9lf@9BmT@|GyPeIoiA0&De$A{zBcTklr|IMZLIGSm zjWCud;#Xcpyq)!Xh%U0a=%$3i48D<30~sHLe#}M|H8=+OSyDrOMa;PC<+5bm(}Mk# zll86K8oL08Gh^}-7EWlOvgNFU#1?zT7&f)in1oUT!;IYd)S5y>p!86TeXST`qV#*4 zrNVyZP7%4BNHjHcna4aD8iI~5m zVqeTqD{YOH#jb`_Gouujx*Rzv6imGQqv`_8*EQ^62wK5peVJ_cPiLg3twV#QMoXsj zQG*W|Jb?y}X8?hk&jFjUTy|ho<4Ye=$!RnYgrqZrRdA5VG{9i#z(jKtxpg$&y$(Fj0m+OkgJwhcYoP zB?^`b?G}<;1cvtQ3f_fv^Kq+(4y5^3YCK^>wIM@G!b};bunn=nweSg;Xqiql_IDfJ zNL$2cZCZ;C^b*#5!0n_<>RmQP?6uf$cX&F=Ec3b$RCQ&_E6iIUTtL`0zN8 znzVGStCWcc$^qiGS;g#w1vrtQ%QP-LkSA=*7*+?deZNbTS;=mM+3Y&;fk_>sE;)}~ z^MZnahk9_5U4-uDJ0)qsBU#ss0eXx7Nc8y4IKMXsV@{F?J5jz($qTIyY-c zFb~|eVC9mei1CPJnYueZjyA30#rUKs{~Mqf*(fmh2v&P!iy5X$FrT- z!xBfgAS!4jCp7TQJuU3G#j)|%#$|0Lk>XHQBkW7IxMMY?a94&D4IIC>EGkqOuzm6` zRHwEC=c8G$hMvg?i>V8(Cfy+bV1tBO@8tDxSb28`l?~;yPo2mhgG3BgiEoi>8&4s| zn9?=+NF;v1;5bQHEqX6J`5d1oGnhl8C!86F=B!Q9YiFb(<_fSXf*v)1HfzFr5xuij z`!^t-#M?Bxx)WTUNHEYiJS&w;5hPWdx=-!~S(7+2mQwhIa1vu$w19j;LxLJ1o_HIk znh>-d2l9=G-I#eH%K7`unED9jvm{ujJ$ofb9vzFHY(h9*74YGPbhz0}gKTjmd(eh`N z9h_)a$CqwurqL4PAt-|36DF?gx!axX+hT2`K~;n|&vBiM^BHTWv!jR=a>2pYh_p69 zMV9@K^Th(gv0<>T>(Mg z1Qg{Ql~CHwY6^D+B{luIx*f^2I=2#gTS)Unrd#E}%BNuGrs$6-C34!UTj62H|s`Ois-+3_>Uv z)wmlyAs4%7b|iz^hGXu7&1rK;Y5!5KoxI1#GyQE;unWTq&qF39TBWk4~kQfjk&k&D;nx$1vW1pp=tMFooF zIXnznE-+Yx#+!LF(XtK>;$Y#!pp{t~3+fO=(r%ztW*G|D;<~ao@+X^X+FWGLvXtP?8*r((l4HItL|3TNJnv2d>#9K<2EY)NHG+c~bi&qL1o?Tl${`EI;uP@L zQ%ZDJ9r9R_?|Q>$jgeMT#p3&`Z!q?9--En{=rB67+T@14!V5-c!+AwL$Xb3s$9(j> zHXdsAAJ*LJtWA~_T*!57lNwj?r0HmCv}nW8`>Q#aXx@YE{SXW@5+*N6(2 z>%ANck;v}nkFh$O^(abY+@lF-F}{llCzOO2!Qsg7C4}z^U(k`Ii4PKSkwQH^j!eX+ zL{KUwFUXj&X+~#jEEZH=wa3{Ek2D}-BhS8HY~-?fCSdh-xVbC@= z6ZPY}{BIIOXHMXjR5CvcZ(Evs_|_F^%t($8lBD1*j?FQZkZ>?Rdos1((|Qx1x|| zI8|p5MUj-=KLzpkBouiL+}!~A-#k2&4EW3k8HY=EpHar`VMZ3@4~S z=94OSeT>$)AVDnU2eIu^j+i!q!qviH*hi*J8E9(U>p5_d!l@Gib@l{BXE3oS7UT>V z7~q#Y0hoBt0+?o@aM~TDQ(u+mMQ~DUut0#tU!W{hm%w|r6!{9f@OKCEZ*(** zs8*EOu#EwJjS;+DhI@Z;fb@ZmF~ShShqeUNBuTPXfO$eKY7rz$f^ai*rJguO8BZ#Q zz#Smej%mr%=&DA*p#>E(N^O}Xk%(nc zpmQ0m$GV&Mtf^f^%QaD=tzgu!%8v@(&93JOC7dhd`hHh!r73&Gs8^?HDfhL9nfY;N zEt3$#kOiF9E(9WF7FOMl>8cDecy$p3pdQr*Y9z+1}8{dbSj>>^>^zAo}&6&NV!19*yYO^h5 zrhH8{$}A1MA>KbO1iy15vgf>?7ry$3f~4Nb1%+jbCy5pA;q9p1__``C5SYlb@J}M3 zWmO5#5Ez3R=D^>K201r)fxd&5eoS)-H)fpJ@ldKDUZZohzzf1(0py+iRwd4CW{>fE zCIfUl3>N(WqQ%81v(BSuUV)gQk`SQ^XJdalgNdsLNSAUwzW91<$UtD7lSMr*LSm{k z20jOiSFsc(het5Di4V}#5HaJ#t_M7(nys|c`*- z@e-729x3UyjV(DV_{c3?HM}xg%FZOV{H=$xg0v3~fc|MmJgHsKj`dash-r~UjI8Jk zAz&U44Hr&{RMp(K7Pqn6M{ac16@`Q_3V}n5a4xA2%#|f6QS{S3cvMX*qJ&h@#amFG z+caVLreP_7OtBYi-bgHA3C9j-G0v2>aXkkpCGC+0&67^M<*cJiJ~0ACWEKDcT|2YP zMJ*g-Hm1h$jHBx)NWMJ7PnRL zSyO(PIsiw&Awmon7ZI}4y#(m$%se)I2nT6nbI2Y;xI`ZM$clVz; z=hqx{<=@}0-aUVF;{Q=qEB;9-D!lgIJ$~%_+n#%WcmIFRx930Jy*lIl{o8-HpB{3@ zKh6b}0CGhD@|d>k{_&ad<&5WGFv-~i?v5B3Zb%<#R9~BRf%kBRZmB3k13vexHLT2< z!c}NwYo*5`>KqTUJx9tBd1zXxmP<0*eeBgL+?g^JhNQX6cGsZDbEiuAPOOieoQLU_&yCB!UeMv(kT*I5ILZ()LV@!-3!SZTNCnXM7?6aYvtI4 z*Fyg^)2CZ>x*_(yt3FEXbJt;R>sx+cy&!fV9lv*@O5q&>I^00EY2EH3Il`}>UFIZiU0>lg7 z`*ft=PrImR7Lr0yV3G+Pjp#Y)$yOvGbl&GA;el$wgtn%!yQID_y)YVpP9d;=J_DXZlMsY*Sdn_qrSqJov2VZp1i=ey)r#Ow7 zjTlf^uZ^qF^$aURxT>$RG{WH`r%uFN*!N)2#ON+&x9)0Z3feUl6&l19a7K`CbO^-W z4$HKHzLl6Ma`ujEy@GT!cyGKEa@)?ED%(581)i`MH-pK0+$w32?9l_uK2(oniVuY3 z2HJ!~VxfgGZN!9Ht$Yu#i?qPt_5zH-xwj3K0dF$vbdT3)5+*OCqk^Qhpk2;y8!s(|;K}I8#+g{nQSraw6@S-gd_Ts@z)tJXlo zst71PS(=3z9tH(#an;m=x1J2Jb+CX5E&iin^1_vXJI7__D&MIeG9+gyc^Wxz9 z01-$I0Sq_2E~CsBfJ1y$56D%M;?{oE=`+s~YW9db)j+7EGiXPCqUzsV@Z)U` z@X}E5;Mvt0pKn;0<9gR=XG72+-6LL`7O6W0oWtZWndmM&2Ql%JpTYdBcJ4(5G*)if z_`A5LcEBNDSPkYh5UhvB*r~MMu;c_YNvozR8fNf zqW<$9cS;f>cwKrGEog6Cgcow1=M3y%WD0&)yiFqA63zn)Dq9F5klZ?|(%m^c;U5gt zAJfc@^_PU|Ey7F`o z2bX}m>WJVz8g~U;VBiv-HWg7V3Is}qyg?9EA=Jk$pL+_z0~f}$dQ4tBV_HgaCHLJE zwDkqhd$K5M_*vX%tuLpWpJFx!R%OTpG$(UaL^krYbed9R*4c{Zr#S{S){B^-o^%WX zqB@>jTY!Xsh$zYukKYRGIk#+l`#wTheXjO#V}Ln^=yy+hgwG3hDYW3;4rZz_*eA`c zMdpwzRwLfoX|yh?nTt~?%cSiar`#Ixc~35RoT=Ko+ah=x6%wF}y`uI`4ViPh7e|ec z#6CFj zm=$XWseG%gP+|4y6c2vo0y z5iXjkTG%-y=wC2LgOd;o*2H=8&1vmFCXjKWk2@-T|mPueM3 zaw<-3a*lW27Id(z9bKSmm2)xbDhY~O2(!iGxLc$U&bm8Ybo(sWhpa%_=`LK2oV#hb zRB0|l3Zlz3brC)h<{D|k_JkKlb=wL#0TPDAg1e@VGLYJVXAY@8vNA1Qlto??)fFja z9vUVL0~s6}-oDc?(T6>rZS(qz3ucGFIY@h9l8*K}yDxsC>mPCXI@oxVPk3B_t=eo2 z6b0|BDceb(t_}T>Mr5n`sN({TXtDP%F35~n8&@`)xdH&Qb9MavZHoO>4lyQ4)M zU~yoh<^Cq_Zo?Du+S_RNkZFr^c{^JlLk!uzsk=z+abP(0$j9D{?#~=|l=wA0qAWrbmee zX*`}FRgYF|=?udR-1hR5FuRHJb%euG!A4D*ZWzgDbrqF~NVE({nEB(tQnz|w-P%CP zHZWa5Hv2tjjf8dCZl5}x-)~R77|wdXdg z1_Mn7#l=8)Vai3w++Fm7CK=6);RrvALGfgAVdFv}JzX3MxZX@_F(VJTms^DDWcw-= zn$os7CQ3m;_Xh8)=B{B}AceN~#nsTSgh^4|ZOj{mTaC2`dJroF6D9D|=!G}PwtOb! z1*Pr8!oV0sxf~4mO2`e8Z(>cr4wC}Ui=a~8R!=j^F0k?DzA0i79m`4=s`5+7xCJZ(Gj$UbJx!F@o?&H$F zrX3J*Dr`V#C#$2f!tRAMJPSbAd(>d(H8q}o3K+5&@Zl8Bj1lmZ9}@Go0BT}Ah!|Jh-ghynW6!ArWb96~ z3Fbo!9C9TCU=W3cG=s5Rw&WyPu(bs%M~Y^tUggNo#jej&3BxWA*bQo7$W}#pCpZ$P zS$l)*C)vABk2~Elxp0SuL>X=%0=5HsyM#4yy#qu<3kW9uqx$|^4vz`UeQKJod!>R{ zP4O_5*G!L0I8xrs(uL*HWjaAkx$f|#d)QhyJ|N#VhfYY7F#?7z&wD0d+Y?TCB|Yj_`qCh?9Sq%9Fm4* zX;@>`A{@<$hzh*jV%hDAuoi@z7O;xd-6Yrqr8XHuyqku5&{0a7W2X4tLrrRwZ`{iB z+7w<3WZwC0FVz;bp=5y8MM6Rq)94d8-Hk_7DVnn8dwF>nPeqj}Cmku;0Q(QfRTY!V zdpbbkdUP*^#F9W<5Zt5UzWXlkhhW}$>0hfRy%{XMsNaORMl^8fpwLU4#C-PaaL>}* zTGoAt-gC=;WWJK!O)2%JlPJK5q(r3Qde@R{WyaZA4K;kNkB6>|o;wdsp1-yuMWuc3 zsa^sM4kic_(9l&-I`Gtxm0EsKEnEu_^)Oj$aoK7#BpB*RhK6KOWhZTgnV+ST`Ntx`X>PlC(-CiO@D_A?ozN70^SY; z3wb3Trz{r93NyizV~McDO}3S2Jin;Y_LY!ERG(=u?^CZ$KthVyR!qXOG*w8M_U@gA z<~VY3lff`O5QVNd%@D6 zN2jm*1}JQ%@|dHP$|Ez-a6r{PMjq=U1J0%kdsb%))H}1{uyC_yRf;;u6AEZB;aQO9 zx@8wb5@8E~Rf$4D_ZVKO*>xtbkYTDaRpWOV;L&7&RqHok3^3r;D`m$p1^Pi!Q{@+i ztz2nK1xF*qhGrUa(fLQ@TqE5(Zzwq6DaViJUeCDeUwGnq-aUKfzW#gX`^)qDuDkW? z{wk=xFS+^2|8^^9p!~$m3XpUU_Jt23S>BuX6e<`I0V1E4 zu@0UCN7@5-!I7=pXku5k#Ftc89*1{zuZ3#fAHcghb5?^L9H7w4wrxSoLKIM6iHgvu z*Di6<(G-kpJCdE!DRi`&wR-d_zLxAD_R$!ZISbwgxs=Uso(G1`ZpYRH6d{uek;W{P z8@ofuz+w)pZmcI0eJUBUWS$B=Q?Tw%Csu7LEFhX`_RJ&t3k;XH}>ea#EQ;e=}J_XS_ z_mKo4Wnv6^;PP7#Je{0y5;*>zHpJF-Vf5kZ1F`0qFWO#P`;BtU70*b(91`oO*=Zya zi+6{Km#OYZ&5t9>_Z7abMucYlmE*rMACu)pRCko}HDIOQ8e@=eBs82DT!4bO7Ofy; zux`FT+L-VzK*!3_P36{LhF}*l@(q+kth)~wF|BUp>~Lm#VXVhZ*i0`MkGy&{YviMZ z#-<->6x(}}S-7b#<0d^73xS;#P}h?fC9IW1B~_u}T{AdERL3EKudk=nujyy$B~iQ7 z-q%*aEUeGB_0R&%ZY^|Gw-dDn6;XbG2=>RTfJ=umsQcEpFsNSyITF{Pr=Tnjeb$vu z@jWR!(GBJY$&Bz%93rO5;$w4M=ZwjrL*{w- z?*k)js=DiEJ8|J0Z+4yrY7C7D9{Hx#oChW;RfofhgDCdCREJU2Mpc&>svAqDJ94-u z7NP2U%GD|}gXeIaZ<<4DncX$95vEbL6tCG9qtcr*;k7Zzo0g$glL9fXBB0}Ijxv@$ zV@_QZ1eu>(va(c46-D{eg%2!ZrA(4Snc1WsXC7#7y{)8qJVNu`4Xv7n3ZNHm8w$*y z(x9!-j0ejTYRZ58wLn?Oj`^aI4gn>LVG%8j6{g)=aP5LwGT3w34SsP zv{eQ}!N8#7jm9T&zHd!5}yt@;-vy?+RfS z0F7ao9)e(jRJj_{^SF3)FDJ%I8t6-TxmTEekR8}OQMF?YYh9Mwm)BMt1 z<4Fcj2fS*5g@a^5cSa3RBv7pf&7PaPd`UIiefvdDZbQk8FwPrf+bnWdSosDw*@IY{ zexrni;`mTt1pvMqFPi05A)6PxDhrbTFbS$Zka&uRJ%J)P)h9ELWRN}Fmdpn$QTNIj zAe;A2Is}y>@*fsc6;)EPR`ch{caGx5CD00wU<}Xhfxb#WV?AtuAjKRP^-^X z!vK!a1?=?mCQ=5QxGd3UBen=3gn-6G!c4z0{CU}uxp1;1V!&a4PZDR1 z#KxXY4e?kMJmw6CDCgWal6j!y=|q6p1LB8mmw7ALQMnrV6s^ANc@Y^=DHdrcZ1qBe zV((g_!b8$VyAVAd4E3H0TWFD*?Mq|WNA52m9hjXM8~1R#iShbfYtR8eE;J_V+cK?1 zXV#n5#XK2B@}WY-*z5-a5!bR^h z>T7zl3YJQd#$*asJU*X1TEr!}DCSPxtvq(F{oE$=ng%WTFrT$l0dPvujx+MhFB*>4 zPo;Nbspx9$s@0c>S6Zc&7uWm08+a1=BCI zmUrePB*Z`}-Q}t%!@IURJ>#&!Hk|^HA)STksSBE{Zjox)xQ-c?1Y;DzE#j;9mb5&K z5nYeSpiTe?=Ne_D2^xh$7gfkXDb?R}`^yd!x0lz756v{Z;KWm~Q)4*$fmFgReRHZj z-WIhALD*WC++%SRan-Y68&&kMpSLv(IBH4+qY;uYB3KdsX?7x6%|o5Zv{Rq(8}N)6m4NV zdwlXs$v`NomR(|isbG}BRgUrlNc|^KXaOI`^0yU^47ku(u!ys5i7mJfv$;c#XA;L z#8u0wY;gOJ5YbLb$zD`;yFTMGwm%HXg~94LJr_!-Ppl`^LFXvpE6-`OY;6N6_YtcC zCH8}*E*d?H))7;4xe4RSUqnRu=P(Ew<5a4YoaO+KSj04Xda2Ajra+f-dyL)_Wy-d6 z5N9sJ;)xw4Rd)KDai?YrKIv7_3W`z&5SDOwV^RVX72_2pOGaqSU1S3ZYR8ATnC~0& z?qpG1YCsHwmg70a;Wg^2_zn9|S7W?M22X^8i!RO191~KD$KeO>WqL;O;tE6ZX6bW8iuXv}sX+~&y>=;ajXoEzm0C`q7il8{S*Fg+M>CLVZjX+hQkKL-G z5%i+W=SWpLNQe<~+K7tfaYHdmyeQ5*; z>FJd#+3Q z$ufNo3pW8zvlL-#!DpIrxDSvZ@QuYCN|Yna9_)e%5QyK5I^v})D+1C63kJNjJxVvR z9BpV`j{B%9^(v#4Gp=m?ibO`chp@pyyHFGbAcd6b+DY&&LBi6VcCu3_j1ZC!mz^E& zI^-SFgoSIGm7&^QJtQJHd&Mr#e=^S;;UpQWq=Rp%<@V}9hitU^lkmrw}z1jMT4 z;FE?NRN@B8+LaI0x~cJDLWD9$hzvvr2~vdRrf_H3c9sn^;E&I?DyB4e*d{sdHrNSh z%g$kL^w637sJ8Y$_^|| zPnhtOb40=#h}UY6KqJAWY~fK=tPuf+_ZOOGV4C^~4ljIxEaIPEqmx43yw1k`gSjv! zv|Q&H@!k+7uO*(n9Akk_Dh{{_wE|uJ6jLQ4RIXAkBGa~SZ_RA%SnfC%P!r~wXoGzt zU{8=hdxfdxjCV9?>IWqOz;y$8f^5M=N*u-)xQ#+6-?1V|uiqT0t2sir$S7i_)?;53 zN?if<+dtWvD$$GBH*%!*5G=KKD{!n#$ehB1v?Y1ntGO#TK)Pl|5>w?s3+%coX)_Ka znB5k9uIm!0GBP>XsF^qV!<6anIAF06BY$B;7!;Rib@C(j}L_E z+#j=05VegX3cOIKf#qeYN)gB%1kGqwl%h=B!fcrjR4=7i&Fo14>_bKp+4!2LIZYHb zf<4`X#NGkbF*qGgtIURx&@E^qtBh#-6%w9s30^8sC@Pc}bgXYkONq}_>Ep+%Iyo%+ z)nXe|R1>oyXA(ROiW(ErUSyy=9osY>l~^P94oG?TP4NsP!!tx-&OorR*4plB;hbR@ z^?}yH^@5#LR&hPyRcINdzgi($HC zbwqW2Q7yYbn^a$6RZlo~IhQD98QNL{E74&IeQiNQ(E+W(`5Lo9FJzJq@H;LS5Sl%6 zsLROtuXY_rm{F++l>41MDI6EhjKVs z*I*APrZ(_+lJI#9$HPStEx~olv0PE1AuP{A7z4@IIV+ZUXgdX@P9KTGCa0}BI4o;R1!5xtx`V)5 zk*Y=ZZfK`W#WbnJGK0kO8&?%cCuHcb8`NaZWzD)I)706U_DCt67<5Uyfi1#d`39^x zN!O+fJtq;ajk52#82!pHO(kRvWpCz`HEzYx>6Ud09g1}dRcF5yS)eoac`anxGD<34 zzn`J;jFO}*+^jgM-Gu>ggdWoug7^(Dsgm$K%45p-f-XSF;&v$w*Hwnx)cwk~5*DMA>X=!O%3`8l}eshe+fwVz`5 z7=xJ7(gZ^e_6_}@OF9a8uEStNE2iX5dm(`!|WYk!WU@VlYCm4awoBM&JRt&g-h4T`#GrgJ20RjZ{La^p{jfDaSGK z!>78Vbjhk^$B1B+r>9mLP*7(X8CnO&1FT2RKd+j@KfEuc%7Xkv*%ZLs1=6l#Gu<<%|>2A-aY#+LMTOJi*JxY2$Ho5bW0 z-x!Pn!g5t4S0iu1r75gmUM_%_@ktUPYJ=!j3rR$U6K6wgy9hbh3=<3GnS)IZtHsmx z)gD^Z!e9?(vi^1{5Qa2Fi)JV7QQ1`XM-2q1b` zE}6X5%ZNQfz&~udr^+Hh`}DYRCw+OZSXKzVUh$RMlxiJV>~2~<=oXXGllvDol+w(2Erbt;y$19OK3 z5dAGSbOx(Fslz4G`h`crSs*toT?)#J0oFn#V(38YOdZM?pT0l8y8GkTf1f%&{_*Sc z-(Gw>{&D`2Ab;AG3*kO{)CC}$0#fz&(Zs|ErIcfiMD2;Z}dU*qSOe} zpT%N%aD+>gaVDsQJ)c~$JDktgO6Rdkfa?j;&V zE_-njAdwgvak(dr@}}DZPp|y~PgYliR1BbTY`WV6j36M)V->R_zkw@*#bK z1T)593^BX9{PN;KUW>`$4{>k|iBX2hzq84Mjh7W8NbA1We^qW{b|%Gb^D+TR7zhu> zEMf)-1nmIG>wP0{JKZ4-mQIPGUo$bk<5wT8gJ8y-2fJRv9wFNAc8z(Jov5mZZ_O8BbEmFKL+?KauANz*K>1wryEqXeWOf#^X;lEgsAAW2cw+9bfmAJA;Umri4r|(t&fm& zMQKroM&oF0G7IX@wM8$}i&Da10Ql$Op?sh-Li_D=yjvWS3>SqPd3=K`;4)*rH&_Qs zGds=%Q8r){oL=&1Qd2_rrTx=n%9vOJ{{Y?Wdft6m= ziOWAfYjb7jWUGXT#JUGalVc@1VI?WY7Ra2a5UPh>6%QyrN?9??XV zR53|cZAQls?ZjKL$-v4Q_gE;{RT!K@TY8W-i1=*K$tw2uMymq5e359e?|O7? z&?|_FpnKK@QsZ+DuQ7<6awZ~BK|-`I=Dd)Kio86?M}VP|ek!E5!}Sj+1PEx2Qc`uc zL*xKq1#-r#@di$h%aRo4hz58rCpB&(LSG1>d=$GD;-sb)vN{qX011frzXSr1Sm68? zQn$SeKFSs~aF*dB)}!nT zycf_h$46V^Fk=>U$H+Sd#atp%{JnWJ(z*F*LxQ-Xm6@pkjS20LY=e8S*q2uTY!cx8 z)t5pe^pb<=mqbup&0TZ(H2fxH8{_^qm9k9pqWRTuyl2vcTuv1&9T za8}@LrAkCCBf>gyo=A;8$WC!6>%pX23S0qt)+in{bIoH|l0_=l*}ijilTUC=jY2g? zAX-DlsL_!QduN^#UBsqxoD@VY&WZO4m&pDcupGE@)5*+@>MAr;Liq5>T$l_>OwV#M zrRY|8p?*MJxUfSD2BOhlX&oCu~SpS)h!)*bFA!s-l*a-@t>`r!&Zk zpDS(pI247;tX_hl82vz^%jn^Ikb-P=4%DvCIfo%;k&qW>B9qSwCaK$V?)$pua^wdJ z(-zQ_IM0Pw@0_PrJWx&KMsPO(%&nM+BQx`i49v`w@m)y_hN&-^!2~-;I|qNcrI5&y zQWM6sd3^e*zitL~c*A^Wl5oVd5XukpUC$XvlDIRpOJL6p@fDZTgb>4L$;0v18Q|1> z0U{tjAcKXFbSpC1xq*BOqAP3ylj=h#==9O?UcjF{s1RWWTY0kB{#7jP@+%N;r$XOG z0)-#Fh`ne!PLlZO{%ql%qY~w_(1(D)32S50Eqkpm(7!^(3tmXp>$$b&Ra2a#MlMhR zcQfesL**AT1`r3pkzG`B3GmG1)j5u7FYTW*A}XTn-#x9FAk;wHYvsxfdr_)YEYup8 zQbj;iE7rL+01*oi3F;|M!n!76^&0tjL!ctYh5^I4ZHjyipJaMw0r)}*qn@I$ujhcz=+P3Ev0a)DS^O1ULOElqP5VCKL zA!w_yBWTe?Uxs1yXzDXMQ2LeFoBQ|(hQrgP%07l z+C7K>^r4k00v3{aMFmS|WXb}<^gY!9D==hnA)b?Zxf71cE)K~|oWh|DKw!y~8qxUS z8?83|>|*)Q_s3Eq)7M>ph)4(r!yr{U7y9K4LNx&Q0K++&)VyCn)m$pfG+l~2g)}Kp z11f}i)kI556=k$iR&chre&52-S@}C7i@m!c@qdw92gge*Tb;C!g!N6mxQ9Ua9hw#N z-CmZ9s*RF$H+`<;e_%;V{PwAnxPc)$Me*K6(;y)-n~A1)eKzny79e_Y3$CG#>$|wV z$|~BbrYn1e)ItO>x&*LRa8Risyn6hvL~?6&%(P;LgWwNKyD&a!@k4G;pLnBv=xHEgMUllDoMUAV~w*`aO z?6n*-pzaCU8m!&jlKlO@lbZ)GytTZS5C+(fkfGwAjKL8n?RO1CI>fxf!7#f}J@rg` z%dQ^tv2S5w&Zlvt(b zjq3&wmPM!M`uQHHteg8;oEQ8?YLp=Xbrn-cK(CG1mU&lMd@+enJAl;B*|G+I}J zSpI)WV#6kUwjT|5P#fGr0!;FjvJ(;UcnHtoW|kBcDG#qAEWyeA{!R@R-&{@tvBWt; zI#ucepwTZ@YPSvYX&@pR)x!HnrsA#b*%4XfA_PUX?}|gjAX!DVP@p(&0wkg7_qf*% zYi-2q_&U8|e?4>K#y?mV;{f@U6Ou}ILr$SKmGj*eI{Chnqfux$oPvY3gyPPhESfjo6X~otno37 zI)gb~DpUhm@miG9HXg>X zkJ4a1$=9Ic`O1)im*x1xu`r?4ONp71>njD=th7Ctgc2o{CJOoxC*G9S!243~#A8T{ z;oMaDEfo{sRV7v~AL?x#mAJ4ND0Y=lVJ#GxGkiaZ3Lmi$88@?!5v$JYad@0|DnDy3 zbMu66uxG=o7@m4fbGOkxgz(d+U{fAYP9>pnaM1PfguLA>3TYq==nmj2u2KV~Pii3W zy@(+})u10K9>qFUAVr8kOXn;IB3!vwVw1ahC#lw#grO-C)N^eEg;7*^65%GDDqi+F zu=sM>_V^B)Aa&MQv_%;%h3FTKQ{+2cm)g0P%xq{^9>|wTT4WP4^$JAkOAQ}qmRSZW ztHhzwQoH76-Fxkz>@)+VBFAbvrNqmHdvu2~W79j5R5Fo74jSMZ9%3aEZG@R_FvopS`#GjQ*r_U9|SbH(?~d*3_%bH?w-ZhCLs z_4DU${dV=&JNvh7{)JcHee?ePbNLYy_wBg+CO@3?KR7E^BcT2@&TPe9P&wX4>|j~^ zM|+v3TkopXdexkU%FV(ABn`?dbA%4vz zL?isaZMedp;+qPn6HbM3cGFvzE4fC8nWMW?8)A?h^?4x25!z7u*^pA&*~w&@(G;XH|&!7=FA&WVc_+6l{y zgl*N658$QGBdLlAs*>ANKL)9DfdzyO?ZJ&z<4vn!L!^KvM~4#LM@isXn7G#!Sf@Z# z+VR;;Z_dc?dA5J4dbtj@vW|gdt02;@1>F~*{bYXowg;Iv54%pm!O&p~NOvmX8Winm z0f6a9jjhxSz-cH?Y!bgV_Q@ z6)xB)MT4_ie65i7uQK?=J7Dx^Tm&!Jn&HsMJH0OX&d+Q(##IQgpGakM+%;cAF=Ha~ zueJdZ!@92DgglN*szAaJhsYilBH?tQKODHCvT3+=6|5Lt@j5dQ2*REjE+~jq)XO6$ z91*3Y1n!Xah`Xt-amA}YY7pkg-CMNaTCuH|(|n5pJRw$Jd5Ef3W9yCJW1|Ps zX_=fLV5r4pFw&?H=uGZkbpKL3_%+X12J{+oa7@PM#&g@u?D0roM^7G{Y!z_BpbJtT zoMj87%5Sd( zji%nQ5V@nrC5a}vJ_8UYgO=4f$8vOTl2jD#$UK^#t+7IzS)z|C9ttpV>{_4|m!l}k zFv~`q;JmFspxqAfV0SQFb9mP52|0;ZJz_e6z{*n!4(vLbnpkyb zc1#b;FUZf%DO~YNNQy>XZt}=TTykG(7uqx4gq`g3S?AWPHa|QNNUZG`_;WKzGfEB# z(}Z%Dt28Ma!PzD;i~|2kNO6W&HPHcIwI_G6TGg86#=)>s2anthG-^63ai-u8C0?iL zKS)rH1nrx{77f}z4J0eQEDn^Rl$*uz22ZGi82DykMp@%S&)Hy z>~x2hXz~GpA#_%Rh$+3f%cBg;Ge43<)eac}8iTh{-xDPP^n)8fun-IiNJI@V56P!< zlMtUf^qL}*q$1UYJz@~9g|Pf!LuR9iT8a4PMNkQT$povpl^&^_?LTXNq1T7dzAH9Z@UTfd_$E%fV3(sPZEs8s0to7 zM=j^o+<9{h&`BPW%MsnIC2c2NCn3wY_B<}mb{vZzF7=5q^FNfvNl>QL7S*IG?C;G8 zvJuDE*7CEKp2x=FwCg{6JUwf%$oODn5LH3N6`r6M_Mx0ABr1lVQxdi`IG08$oes@zUt$2J%`sj=9xtloW^<2&~D-Fd=q@HVp;^C{`1D4lp2J7F6t_ z&W@c+W}=AeQ>Zv*AWp^}B`@kss|O?j2FO?J&02mAuLDUr2yw;HgE5JQ8mR+MM~Lvz z*FA}K&{liTX;t>@N|R%iE(32dASOZ9w2b~#hggt-P(Ir8n9UDgYIa-DQB_fxJvm20 z?)polKBR&zQz4*;V(J#xQ2AW~F9rl1=xVRm*nm8Q@WCY1!9l8Z&<3Ac5Gs%hhdqvM zY@C#gt|TikjTpP8O@tu0Ds1u8J{n8RdllNxuMs_<&terMeBgaUCrV3+`t9d!kc zw*(NzArYqoAvG7Aw`bB=Rlpbny&e!A7#De}CdL3!1J*meN)JR1%$gW^gDiywfZZRC zT!)A&Wmmx?4C93z%&&M#bbOX;L9!jOQL^90p$B;?&I^w>2VN_P0WEzF4Mne5b5;0y z?<_iVqk?d!?J^aBy`GJfk=eQiCxHE(oN9f-fuVZ6C%)C`w}~VN^)tIKpeq=3vpz62 znx!0pz~D{pF;7GY>Z6I(X5INT-Fm~c^^Y&>LfFKyx9C%^Aem0!MK>)Ssx7R#fiQ@i zV){hI#O9YC4RdDtGy`S`Wne{iV8|Wr@?mIz#+phK*}!pP1Rp#aZ)1yEw0f0X%6`jS zRH!iFU4m7#U>u3dNf*_*@YTumgvp3}1bDY|&k_PCj-c%1G71gql z0ufLZ`*@APoCXdtZg1m+mVJD7vV~xJ^nuR(aj!BOL{i62G<(|_JH~`b#hmVxw3EiK zdM%qs=KTV6ju8t%XN9_C<%}|imN9wSUxYQA?oAI8uFtapEz&NrZ6%rYl(V|vn;jIL z83$GL8~`UF=(cye?_p(K%Ea6~6`9`XY&baYY2quQ3$AKdkO*kWP@l=g1KSV`wvm$^ zl#&?^A!~siX)(HKhZFcDLeFR|q}&XK4Fd-hVzqcu1IeAbYV{fiXDScSS|B~Vhr&yU zKSsLn(&NThRiht}k(u==d1<)X6Rpa7O+d5`ECCQaV|xcn?IBUzL#eH+<5Q+maM9l@ zR)S|1KM!}tXzG4t5CNo=rkRZntc|jHAr^Q>J_`~9x^jH?zR1DU+2D^IF{gvP^zF*o zha~Js23mOq3?g<7VnRx{eAF~lT4V=+|2CrMg;E1y{glldh)3C99VSNu0!VfyuNpGe zhJ+Cj-f04&Sv8_ziE=f?!k1gMK1QqB>f1qHI0a`<2Th=v*8hrxbg=MBuGmp>K57t7G&^y1YGJ<{^U zZ>MxsGm$WRR7($5w8#)hVjsl>0AeBH?tZvzjHU1?Di`FHn6`*Xjbl1E`AKwAtcnSE zt8n0BBn~1VVb(Dc%w*!WW*i7TK>?>8V%%m>Cs4?A#tPj5^)41r8<3&4x12Gz+qZuW zD~H!7d84p;jKdWfJk+ksNdK7wx+1%{Un&(tVI5je!_uAAUHk3Tx{SvHE^B9SMnoEm zetZyaP9`Y=UMt6x2oM<1!y0ztB)iF>T=&QnU?l)0>vlyPVrk45rqTJtD1LoN>X{u0 zFl0&6Ln+p!G19KmOe?D# z6b48U^N~25&QjDQx{X2M{#BLpG2tDPw_-Th+H#occ-u@+fwfjhMJU5yXhK-IC7caI z1JXO*YG`!z=XpDb<49F*oHL!>DA!gVJduWlLmZqWE+H2=Th@CST<(LQ78lsPZzRF@ z2TV%91$*Up^-It=6Q+Zn0YMjJ8#2i95^)Ga?GjyEL|(ZGMs_m<5!5dyWO*{+-a z-E?@?ABAVaVnLmR}qZJRou0kSo1x^E^hIR2?Kl z*N51i%shEC;;`X)k;wA_uNj-RVpBN1NITa;T(FPGi$?6~!la&0JduZpH8GZCsI7-A)6y47@Gp#m8mRJ>F` z%VZ`ct7t&dt*!H*S|P4mdeU2ACoFuORGmqcMjQnS@4&x~&o~rOwUj|nboZ>~x#X@v-K06ZxZV*$9q?66a)?FF5XiLyX~!kem6}_dWV4!J z&WjDapv=-)Ws94_zj*Wkj>tbo3+oj7jHTYL}ot8E)5uhO}TwyXM=zjM^pA*eq<6!q5NMZRRW!OU? zTy~V@Us0ge-FoPSDP%^BSJu{`?<7aH1bNNj`oU`vRTK`$DQ!x|wu1CC&dvkEd~>25 z#QrY%p~jzAKHv7mMee0OjN$i|~X@wgFV-(!ln!gYr`qaD!{2uDHcK<6$2 ziTNrK4NlvV6q1!`88_SZpr&0rENQT5A}4Wx(z-hQtP0^;TZf{Cp_?DUp()BJ9g*aZRhiS6LQQ^0lI2r8zMPFE=!z=^$f^(245UE6LIwM=V7yi5Xm+GK43FJ1{c9bhRG+a1FtO zJ;;|Vevr1amyl77R|L9hGftu!Or!O~H@+D+EY=DlUL6Yd<;BgHQ9%GEk?aJuq z24Mi>r36LFPoF+0jMUL=3k&6TB>H`sF8?VXMX@}z+C#gPK{xi5)K*#fJN}& zyunI*V;joTPoBr+IVZQ11bSa$rWi{iwXBs2sSTBs0=L4#|9a`t@b+ZZ4H3kMfV6uS z$L0jji0@rMDZ{8#n=poiE%dh?iNw8lsUhp+5ME%!i9%xQ>Z=Eq_^pqH!#WxTlp3)s zAi~B~xkqkdZhD1S=KQ3DTAq*?s&!KNOk>h!rpt_XCs!SIQ3fW92rNz`7DK$r(-9Cw zmL!RHU!phd`X4y@k!LD2+8BZ~8k>_8fdF0vP6ZXgrwg`*=71r@$clNC2O;v1DMxH` zS_DbpFfg-BxJpofaEG9JkV3(+RdT#aFf=~_kem$TV4`O-jP4*{bqTB3og)w}^LWIy zBZa+EL~V;Y<56j|tqL3k2n)%&B=gB#0Z>0aB5lxB0oVi!kVzjQx5EC{%BOor)>K5S zx1YKQ)ABlb85D`;<}yYwVgdVn0c8pdzeQ~Li^K6L$G_pD&Wrbj;>)3Y0N>D|oHBMia2ADTY+*k#!uLm0Zvjg;+ms29@$QnKCr~_yroQtR+APjZ` zh>Zvo<0JjyMi`vBnjyiZZb>V`OJazQleu>5(04g70P4v<*pcvUegbJ@ZYD!E1vf^!xJc!u}M> zD1M2nP=>tRCUKz@ETLm@A$BE%cZCi+BmUP%JXKFiD+UF6|`RKl;QcdJC$aB9^f=w zQTd8JJkp~GF=5pl`lXd&_;+EI8TL_2G}8T}w#YywT#^GQF%2+@Jr=2 zFhv9@3j!D}AkxC4LNR_irE^OW9&-4!GSu?v!Z6-RrWqDQEqXATSJzw;6=CHiSvReq zL*EA|vLQ7nwVEK7Qeg97ju2fbxRJNlZDO21F%j(%fMA0e0DAXnL~WzPlwQQ($`_30 zKSjWo;)bP8sKz7qu~jBI*q7b6999k+ci@?~NMkTC3=81{3m-W(msyl-QY+xE9u)cI zGP&49hvM{%qpH?LxpgyaADc;W&B<+j?wZc+v*feZaI+`}$ivDK6exjuE3B}g8|sSz z%f(n05s(KN*y&79OZRjIV}Y>{9Ui%M88R7qkbSmfuXw(i*SsXfbv6bRtWlKohM;J* ztwsS09j(qYsWJ!wSVTE?t1l*XwhP@3u0Ym#;;}8KDhF(-VnnDo z<7aTKOvu6c_MKl=05x?#4q&Mc1z8VK2MzUR1V_dX@Q@Ys6sO9`OsI;Kf{E7=dVsl? zER!%do`lg`Brl8UtoV|tgRB6ifg}k4MpPQ{phimg3bM)uM1^sQgCMmNwpwM^HePm7 z(IvUzy}Y0!ZptD=vOb1$T!u2*h5+maPzhkuw+ye#IY>8gr zk9N4BpS3$Qk#LKKP@Gw-xfdeOEsM$x`dlp5#_v_2iUGlDQe2PLN+>?!=xhk!p?gp> zBSB{&?2GL04dpHZRMgm}t0tAZ$?$-=14fsTXEK)sVy(4q+dh&9Uh+x5pFoQpDUQi9 z^McTH)I~Kj8E92*6yvH7WdSxratBI_M-AO%fpHMEv8;t3w%D>UjY3Gi-%v}(#+#4? zD6iX$9vWcpwrdhRXIW5k+z1bdscRCN;6jWU^qS}1jbKOE8e&IvA`%2WAFM+J3&Rkr zeenf|aw)w_TtQXt$yCz1cX-~|5F-12aGdy0FhO)_WI80**9bZ~9DW0HHFb;J&S|o) z+OYLghfRnG7hjn7W9QRSQ!ksg!j1Aig_4he95rH4+s@1XeEAx5g420?P&#?6Hq^lMT9xdI<(A9!QwefhC}Ik za{mq(=4hOSyn_Off-0&2j9)DHn$rX!iw+=6*RpX0Y>3X*Onip}w&8ntEC!O0LYxwb zF42u9N^(b&-vMPul;g&0wFELa|0b&FrM`Sg5FlT9ekQTY@;rAN-5C@T6n)1QkW~ZOsK)dTeWcmPyH}vN)qiwnIk|9VX<s zlKH9BKzVlZ7D~gXEdU5gofgX6G*yXf*`%u@IRmOe%P&aq_u?RS?6Mz`AmVgC(z{pK z-$g#Q-yS5tl%1_6;m8SA@>GnyV6nrX!(3UIBwI<9DZt|1_Z#991G>x)$(<&r>m%V% z^4j+hv}R@v>2(g*fbEb-ccdsL3g~3m^W$h0=37i7tdKhz`PbXqf{z^1{$|BNo4MyI zDfG?b00<8_G~p41iixp%Fw(FXa;@b_tHx7o4%QF1()(ap@D`XKt~O zCJ};1Q(qTRr4T?wY(oeS??y**W8IW++5ty`)zI9KEm(_No(e{c~LYxTJx@GnIiyqMCV{mC#WHU zL`c|7SfbtY8w%+VwaKg3INo*f7?B0QUpu#zIVqL`)DhBHx#E(4DK@a`GAzFkWnn8=%-iA4^^hRhdIsbf5x$=e|f1c~P>mP7bnJh#|!xK=*EuIrD$VNBKl zIk=314CqV((zzRzJ1fM-Hy5kOQ!%oWBkh`ww(%uMe)ScR#l%=dv*eDK)q<&lS6hyV z-*JLEisbEwFF62qka$2WY(cOjO~gK-J}o~hjfdYgk|LIQzQ|hX+iq+riVl@lU2!^t zy9>n;rUzgs1vCTsL#Kjgas*B5K9bUXv=0 zP77LhE>lonoGIEyTyH((?x1>8{Kgl^o>Z~)#spLpJvFRK!8Z)RXe^3o}dkP)*d!ZnzTuqeHL$d~TxZZc>%cflKCV6vub zu&mQ)u5TA3^`e&;jCHtoUrC2fNR0XbDI7^k?q@BAl6aL=gNT$^XLF*%KqBw4}8A z>zl6=z6G_jwKB`KRy72IQvO)K5#on3DUmhCx$y;}>;s;MM{+85vr1|cW*Nsdy##8? ze0&2C6reC5Blw0;KKOgcj4aX*fpN@{Vm1yPY-l!AA(=IU1KG>(Hu~TbDnWI?_O**v zfbkIhAzpuK@YV+JG@Ja$zVXaHQWkNUQ8Wt)#EdvhaQ#kuhaiANUXm~eAgvK)6PSi` zUvYrScKUunBBqv#77*~+yf{n<63kyRV3LjqSr27!#gzlIXm~QPSG1ckLWzO$c z)(n^~;F;gOJcdlXYwxis5UvoHv5WMo1!6VOJrEBQhNmXxUg?Q7;aRzqh@4Y*-RLhM z`#~z0Sx-RY@!KLFECGqG--j~+s8RG}IZG@{wm=)Cj8^1@E^-kpK;#(1A{H0)pu zKySQc{W>*QijxC&BsWBB)_+Eo*65mZJWq_l8aqXYY*T~=ATnPeAgEPy*87KsDYe6_goyT=i}S}#sS-l{Qi;h} zYh-;b8`F+kO)xeb@0kJ(C`JrJtj$AFeiEWAl|^$%ymRgEa}J?yu1A;T2D?bW! zRi{nEx7ykS{@xQY=uZ^4HkSp2pTe#^eS#>uIIy)ySrD2`PRfWwA{!V22Mjg>m1rJZ zUzw^oJbu(-j^TCVs!kgiLA_+c8<5*6vdtouxa)`&07zKpltsD0^D!j$Sd>TAx|S$T=>8xs7{5KJQs!?!R|l|NB6hygpo%h%b!5VZgAg~v+pCe5Yx{l_p*#;9q${+D+rf=-l2R>f zr$ec8ac{DbZ3-(O?K!HZ3IZ6QO}E><+bV^-Tdf71<+({@abjvi)shPUx$>BzR%kPi zZe4mw`H0ck=O^X##^>pwR-NOMjpxUu%S{}cbciK;4i6cl928uy~WiUwX!o!Mb^IExeI&LcB2xrC1GutMB$gUi&gjf|dRycY3oik7Lz zAQ2G^!utC=pzsaMOv#U{8x(z8Gm9>`KWU>xkARIt2Jw92{5JMZ9POjS1wzSv+8*Us z(h%CGTx1R(Ll!kNv*r_{0lq-ZyJC|ZU8G{2L6GAWicv%O(^hu{t2p#+K^Dma1f7b8 zh2$haMc5~YY#9`;W44C+w3N%`J711HBE4&eTuL-}B2r2Yp}YYIC?48~({Vg}#|hCm zCZdyNp@Elhb42@zqSqX}>#WSJ5L&VzW8iuc%U^Tmjlx#x5#2nh zqiR=RPZLaHj_7R&{AXfXuS&mbf|xfBOvbmFY)$jR5-rm)gz2%c5Q9;URT3%zd7{HC z)Ig+_K!tv&f+Mv$(eb-~voIxW+`V1M+=4RW3tXRUYJ(abt00OfZkKJ!M_rhyKnby6 zks%8(`!m&p+x39|J<*7Tukd{)`z({Zi>R>-&uTeR$NLx~4nF(n2h@VB@@X zDs~opG$Ugp%eNqq4IAnmw%nj_k3jT$7WT-8#dt6r)qGQ&T@D^g7T&9?E)H1`B{&4E zKYQ1F=)BsZq`zZI_mWsi6#!s?TyFPYfedpGgi>jE57nKN>7@Cn{l{2p6!+GBc*&j6 zgD%e#=t^r~&7;$iPb+oGD(SoXhl3PLc)dQ$YG5fT=G9)FI~1ioI99Ba$y2PSM<@&k z(jvu@k|#HD{G-F8hK?{HXE?8w;V$fjPQL$UKL{t*3NBHy3@Tc^~0vcc{0ZCKn zMY*63RGuETi$p_h3Si7+Z(QGO@=8~^WkEwuq{m$((f~0wBkkeViW1NTJX7`%3>9Z$ z0<8q{T78i6kdogpUIe0pxgGi?cSezC;;vRw*+#o+B!?>}FN0;VIZY^;#m8oOXGL96 zPM4CB3G%Q`Xid#w8aRg!rT7S7!cT^66a>VWvkTJ_3yg;ixEurOAabs``Jk8$(ql30 zTS=<=^2u?`nw=NIOQ4+H8Bw-aivt9blc96&2&yw5y3)@32k3$($Rv|aQ(-QVPDtS^ zS0U#w2b(%DH-j;5k8pgdA*O_S3g?(5G6)Y($iFhNAH;PY^$O5s(76m3sDg&dyQ3M# zuyA6MRN@F8#n;&CS*lSI@Fz_@La^qlIEU8y6OWJ0h1QXk!b~u zR;PwYZ;4nhE4Q?gXTIho_QMrON&yv zYl);3lvk|xFjOM|lphLT%K41Y&ovJm{znD~`ozR+$jaWhvKJ&RWTfo*+}6i6n!YrX zcb-z}7cXoVui+Uno*Y2vd65BB*fS!Ck$cdC0x4z??6q=v(~ej>35*zlS+2!xpeJ~XdFjVuTa~OqGm_zu`6Hj|maRf5=cfgs-(y`x-zq-_^>Rc?~JS*0=Py&*k5nClGCk`vh z2q7I}M;t!)sHkPzLx@LOmldKL4ZKnx2q1(5Ow$Y!21gKQIUNPKZ9A%)WupujYYZ;m zV+;*sTVP4x(W?znNZ>Hkw@6PVMEia{`j+<9fge{yOwzJikQa)K^{JlBdw8g1=HdBd z_?9R~d>6J2wGLvVi($jbW`~F79G(UON3qHvjZmI^>_M~!8xesU z@cdB7#j5f%&%|mGuaxg9`o+n@L*1Cls)iCGJ_~TCu-qA4Jwd}3pc5kR9a7cqZ+r5$Rn3$Q5>g9LhxZ^{a0!b7%G?tl_@n)Ue-p(1@~>H(mD z(Z^J%%q8aAn_^2J8_JofQc=;2F5-m`QBRnt+Dj;1Gf}wOSHkMS}>|)vt zOCN|g-E!ZJL+2|4m6?gJ67j7vK9JXB5}Ga-p+`CvNgqfh?22Zy_IotSbsxIdXKwmj zk6I#KSW{>0!DFF!sX$`GV+$(a#>|*HT?;`ST2q7pn8em2;KTbwJrV~}WZj#{lN0UJ z@hy~yl{t4KQ|EP!>4ag$Q|rxm`RHLnqYQ_uP0^7*IO>ykrs(R=X9>I7-WrRVd`IXe z=!EM7ZoTMXu&+<>VNtfR?2H@J#oq(8(Kf3(a{HrCaiN5olo`meSCJRPs?!xSFr9(- z+J3Ai9CGA@^x<}phtSx60SE)5_XJ(63D_D4P87QzOWnQC6PDGa!*v6g`S23gD$&QE z>w*H6py+XgaT`wpak#l{mlT?pTKEskF90pcZMh~9#C!+-lgXZg52!YQi&sSiBqYLlWkvRVE5&D#J;J{sU2JnjWTU6 zt zZuNxmMm7fFdMSX=3M3*N6keZ)N{8;^VIsb@1A@VSR6>(KFYmC-vTOnCF9`h$E;NBI zJ<&ec^5$cz)54}M)(ZyTKuCcU#ddmAcF3(I3O`qgE)4F*EhVbwl5wjNKnJn$ti)FH z3qfX79Oc`Pr?n4JqS$!k?{e9>e_BVGamAh*mc}F_V7*u|&XiOXdaMEZLETRqnSSd_ zXmtXe0p84dgYL2M4qMcnkm1IZ^7FB?S{R#jD4%fHD)jDV%tA_HE4GihV&wV!4q+Z! z^x!!PqKGk)gL3CS#RGAJbhFQFTPN7EZ7+emB|#yKtH&y9#4_)^`J&@qpd%T>gx4NMO<6f>k0;pL>}}2pomE9gx_Om9~U}Q z;=^{;@2o4ap`}GNrD%ck{dl1Y17CS9&ZA0obh0N~v8n1#aqK#K+cqIngecW#R5xUM zFcHJElrk{FMoP$S>~+EL>7_mDpoX3K-PE4ynN$T;M(>IOLQZ*PHG_N$I2N?y#B@%; zS(N&k0q<4)D()S*Q@KPlq9?glo(W3_IW?yDW{-!Q^y$;MFXQY<2HeRdd_)k$APXOo zOf+=5L2n4f?nuPQ8w~PmQzR;ttA-<1#N*Dui_je>S<4EK;+VsN`Nh({VF;pvWaF0b@drc3l`Jxd6dOgkppX zIw=DRqh39w&)4b6cr{07JNA4DCl&-S8RqIH3K_6&meI`Tm9+MVYOu@_G6)#kmJ$e1 z#Zz^^Xl>uyKKCM!(qiC(pw6lIw8_HxLRcG2)#OdY;-fb_fV#eQtxk8LplrtoM-TG) z>ei8B+P8AmSv^V!%!z=XXBHvx5`Lx?*9}HVC1`~B^N$BODlTfEbW>=iq`$jeOup)}O~(q3W0 z6AS?YloL2eY2bNMypb(=ispSxXm385EZh*$S-EP+1Uin#LM%bwyFeVrS-8pupnTd`=k*zH`~edQdhbkF69mW8ij^|$M1nM$+8Q|HUv*S z1-x`=1nlS?73$~eUcfEd>$eO)bTj-Ez|2}doU>>AVKC+U;b{4Y`xIGmo(_OlRG*Wi>fSkbp~8X3EKv zCwXF$Nw&_;>%pe$CT>)rN(dGWPCIX@fZd}y{9Qdb*~SVv;yy6HED&LNI`6>9N#$V( zLK^KA&Sxul??PL@X-5KVrWECros1zOVFDXpRs1~JXzlKo0=@LfLdpdLYm%!s^P-s;o+1H^xkLHyKv%7p z^E*ABd`33ACsAh`DpElJ;RXTpDDmPo%U8+`(t%^;Jse^4YsomXA4%sGBZb%`!=n2b z`JHo!0fc=;A#F%81oABdp2RaPuNZCXQpbMuI+b=3mu$V*g3Au_j^Rl31oy01m5dMu z_RFKhQ}@g67BnUYj?yuuBjeS?5)GpV1LvT3$RnrcTF|Yjs$xQ+s+R|DD>k74wI#?j zH7k;2fkzR%j~pm@7sv7PdFJP3%eoK(eZW}D(q+k{A@H5bqI?Sc%v z3cd}&@+9%GDj&K*ULF$1Er^_)!q8Z)BA;{L1l{NL0;1yP*e`COfG%jrdM*erOiS-yOXlc?Q<>l1(pKpV{Bhr)Y<^)H?GE97{+{hRM3kY=!-mg)A4s`QV#iOSU)i3FvBi8k4rSIqRgg)I6 zJR@Dwt0_~CZB9+Fos8br2HzjuYXq`kdO>;!PC6j4J`43IuoYaA$#P{%x^neR#ja|Z z(Q;G?CDHozRIz0#ye^Iiv~7#`j*&!?sC->~0Frv!k%+(>SLoJXu@9O%w|J zB7{N;A3MFUs4gyQ0^Zm^#wlD%h#+GV+|&9?aY%vX5RaCrLBet}K2zfSM{+$EcTYK^ z?d@$>2JEvDly$k4i+0E1P{(8^19CL!a+Dx0$fW{P+10yv)+b<8XFpY#y*Xvlpna!` zb|Q;*&;nDiEs&@Z$f&#Xl&O(zwvkLBm_8gLVzCK;%k+zh%#YB-dU5g&AecXypL&r4 z!o%gu9?9-4w57#Yfq)bsptr?@#o!<*G?WPjClJw!Ftc|K0rdo~K$MaIU@xf-oP9u@ z7J||2fg)B9b*Ri`9!PcbqjR65Gl0a}b5Y8E4IhErZEe;80*n$|M?#`j4*w?A zcry1*Gm`WPJqb14M-Xd*dJ85y3yQ&UfXyL%voj(S9NY&S>DBI6cL4`qQD$ltmsXNd zRoyt`!A%chU=tQeb9;*PzWfrZl^yRVh`X*lIx?3-vkr>urC21Z3RO-gSt(V<1tPdH zR>YO?Uu(~~gfMO7A0MVp@*i1~OPfoaj4iev2dWIuMuxW7k>?BOU zTL=)SQ@OT@V1R{vHhFUbnv1Oll)y%a$F6EL--NKhbN{Ow|1~P3ow#`R1T}q}=!Kqbe6X!9epd&S54_$stB_CAg>T{<= zNOiur(~P;HNkT`K!JJ6kfT@Y*o%ZwEJvnRkfKF|^&Uie|s+7{DT{wap=f~a{A*j}< zD*`Qs^`qNorOBK!5-|jDfT9=-K)f-+aAvot65iLWD^{5;l!UCy z3VKdgn#mtZbC79YGsSgkG^!J%bqICCJRI?*IFC9An$$!?!ULuwz?w7(OG41YY(-(G zhSKSVrNtX_W8B#0|HbiXZ8s{6~Yd#C1j3T&^4KFSGk2z9H zdV>{N)J4@2!%$AHF}6c$PB&lKds#iUolcRG9`Tb_PA=e;LJ}0`5Hh)WeX-9Wo=kCG z9G7o!UGIqq?B*y4ofL|eh=UbDQhwlcF+`d*`OJC;g6BO~t(g#hDWZa9#qT$-3Fw0O zwuecz8xzN`y=Q9=@K_)r2o~9qm0RL?UuCb{x*>==$z?>?e;}8R4~;2U{EO3N7~JfDE{PSU>@y2xAz4a8X0D>z(oZNGc~T>bmvk(Q-CFfm=N9ALV@qD96g@Ci-j`z zOd}b1AsH2u?>oZkDmodcNR{HE715~^;uEy}1$`2lVI_;Y34}3|YiSUQw4DU3_@nzb zGBjG3bi%^Y3@s&hw6j+=TFUIW6vi(W6$oOTn+z0`_d!4?@o@~~c}NgjSF%GVW=HG1 z8Ia7-&9^af_t5!{aVTrqQIK4ZVn!fCr=m#g0Ha}*RX6~F#YDxlR7genQ(}mX2@nD> zdQX}xB-vKTBxrJqkSCl~M{_M#pHen10)0~W=?gb*R+ivsy$!hRcE(y`rpwH7FG93aYuL}uq3YPe6K%~-|@Cp;0E zEki+tP>yCn1U`}?kC|i?ESaBse){9L=O4Y@dgGqFbJu@;djCMFPyR_MD!=^21!S2` zW{U^QSOFW4ZUX>70uld!0Fe9u0RR9YfkFLteZ2VkChgumc1h75>z1=4-p_XC+2y>K zdwY)!X3uln$Gv;A+O2iv>20)kI_&qnRP}>^k<`O^vHg+IOyV zEn87Fwkpou$4_>9X}aT0atUv3&9&||*VF(2HmvUT?+L13eV5-p==4#^yIUG&u}>{` zXQ8ibYO6yb?T1Y@+cxTG=9;Y(uHNre)9brp_Aw-O-)G*>8{F!;cWZ&6F6#PJdQlJ3}P)#u!RP{;dpZPNcHBn#se}CWh>%ZFTr2oJ7{MR+kG~eEz{pvM1 z3PC@@{HwDkU(6-;UEmIWkV2;>SH|?8wg!%~bahstu`~o9EN$^yK9U2neAkb&ijBLL zJ1}~^VWp>=>uIl-UG3Ga!xiO~p;VdoZ#mz4e_5r^dOG)>p!F3^t~h?@+@%bNB+yx}9NrJ^Rb&VprAd_0`WTw_r6ZnV;gW#DK}+ru3NMYtd_@cP4g*~Z zy-a-$UL)iDTvDjlbi@P!tvyo05oC9Iyir^<+J>wHLq9za#1A$74g0oC%HQMKpVsOC znxp{kaea893gYTwkyfZ*ye?)GbKX3B$OEdQwWl}K!$w$rC9vsJi?Q-qVX`C(jR+k( z3-__S$b{ubMonj%tZ6o-Y!h%yN$LvvEl$amJ%izgAAbw!A2t_6~Pmsf2w>el0tup2ev|^fT`sv34n!zjVzMwv|X)V7I_O0DYWJf)baTVX7O0zKx&N)exXox;O}Drkb;`%lQo zKV>fs1Rm%^$v!7Svp{>#tKRlyEW)~N*^Y)Blsr1`d8ZMh7GePkqr-mZ_oVJ_UAhB- zZK#NJ>Gi?Ew0fO^DvTpXT@Rk1eA={?h*^L+%}53}(4drlECEz-@UE=_m5RRskdES} zjflzNvDH^j74Wj=PK&nQghz{`thgkgi(%;pJyu7v5@#e?DHkXo zx~j%jHguOPqupc=IJoB|5uYwalEappDm1{XYtQopN>NNby~$c9%~%h2*E-MO`Y~wY3&1r+bldgH^K$3JuwPh{ix*n-ZN? zJYC)l=>`RBXe zXqV~pq)_66>K3vXb2U>Oc+4bf5k4u_!j$>1sNAnulv(bFVYmn%pgq^Q)URTk)@@Ik)4Xx#T=LC9erAzCKe75cx((UN5GFyaNz;86i%D(55Ll z8!|mbS<@-mpy6m2<$=mcSPu4CI@FcRBb>lME_)}dTD0$L%dEVy88#Yu0tTh-$zV|` zlh;|b9c< z8Z@7kg{&4b`DvDVk!aiAoM?KlRQ8RE5+2qyq+x5Uso8gAj);rkZPXO^*tbU$0~hLE z*K?%ER%5Czdb4mjqIFQ&@{-P79M;WUxl3_Y0Zi_hQ+n~4YPD?1BpF1VXHkkhNq7-X zq$k8y`7pgMTqGUV@(7S6OgSY`>NU*l5eig`#W&~k2CW=4Kqg|DbH;OzxT%FC12HM* zS`Ht*Ow_`iy@ZOgaJIYl__}AmDtt~CwYlS12%3BW(d46sOVZR~!RU`YM9-77=U&SK zzA<7lr$c`L%)lqH7Ru%I)7;s0=}TMSU4I%}v=Nh8!F3}T^gY6qUDu;LWyL8E7rH>s z@7=&Q{3-|^F&BbOgAF|;p^Q9+&Kv8ov9**7c_hZ$sQaZwOF8+ZUW`2JMxRK&NshGIvlr6~R#wCl!B@&_W4%^c zHwYG2B(SaXJS=^kJzP#n7)w1ifL*9-9@0Kf6&kvZ=Qz&mX|hw$ljn+(nMbj!EH>^D z+=DX96He8csK*ePof?P+ti5}INwcBD%E?OH?xk%YOXTEZa(Z3J#QVg8yiZR(2CkG! zQcF^sN(GHwoZhJDq+URS%%v`U;z&J!@D~Be%yw~I>^tGO+NX7V(OQ(yeMm)F%8H19 z&KZSo3+M4_V@SGLKy31+iP#h7-P7El!zX}riU;%1Zk-d|rL!I?2G&qpt%yTB*YYj}Mq2W%L& zzP}ekcXiWRYVxi$8R-b{u%D+J?U&I4$vE2~F%wn9oMVrz25yCBNX zeK9sdDlk`U!Z;maq}sT{szHti6OkKCrl_1pfPG2nao1Fh*G#~Z5LlGtL4T~YuIW}) zt^5V4FF!REZ55G0BzWLjk(9KdA2LzL4gTGFUki*M1<>XPDe^<&(Xx zRP(6#6%Ag&L!@dBVzQ)ys(oX0d@sr?der$y;KOF{7N2*834t$<15>IfQ*~r%QlhKU z3JbrlmlzAqFp6^S{H`dKx6JlR15f z8g+-GrNGBApRm}407tr~iOF7{wO?g6z})ERj~#v6cg7=kBySoT*jC)O; z1Jgq?r~|D?8K~KP6^h&CmM+mYeI58vEK%2OUPG{W!ypc3fF`fp7~YL+Ex;j$DLA$a z^AOV3k_n;ElXoMN8f?a3nF8~7qHe~^;TZy|5{7#An@i?py*SH~4R#<9lJ%*0uR8fc@4h{q6BcbP>P&_@PNuCT@64G!1$%jRSp^cQ)&R5n;gs%)EK zp!@8*q%P6<`0#8@NEaP#j`HVA>=w#nCFWnPyQW3ex+Ur$#VWe%17lqDLct}nl5%J( zDYtER?7ia1Cq8~%)+}%t01Vp9S$u645Onre{Vh3spH#%-P6<&wVRX>4qlhV0LOjC3 zL1N}^RSvDv20A#lckEhuE+OiB%hhv*>xPx(Yc7 z;+sSg-O`zZ^(}y`O4P`1`QOh&)a2ny$){ngyVskDVcFWXOI=%VOP+uy4ClKXR;AtIl;wLGr$x z;n9V!X54{Y=*2--$!|%oE}T|ua|RL9FJUX$gANY~pgrvKn4eFAbUgntgz?R6)E3)> zNbaob*HVet#E$o|6nx21Y%+G&x7zXGO)8!g&JP81NSDJ?iy=*#_GiF@eGlfNi%v=- zBSnPKQ>D=rbPC>k+qvo92+^IV6yEbV!Plc`shxmRI+@9-UA+`eM&5;Umx}+}6dPg9^aoB!BIMZ{W*K)B%80S+g&qZFc)@m$zm7p$< z$aI9*qz601LUIc*o*-G4Lh7U(U{dIW>9L|E4vA`c;j$!!?Sm)?a2h~T1jMY&0vv$z zL~!r3rl;LKh0ULJ)mgprgqOn=T#G5tO^Jq`byS(u2~jRtI#Q9x*ALC}p5gUt!(Rd_P8 zm<*mGQ2b3HOgwyJF-Cu0jiIqp>WcZjw6%ogu+A55y=GT?4 zwb>o0-+Wm77#cRfQ;Tz|W?ZSG1xz78#A2xuN0o8hAeKeMmDJtx+;diKZxzC)lZVCh zi`{b%PeT)k=d6~(h>;*h87j7@*VNICU4|{<4EtA*I_+Ko`c^VsB%r0Of-uwUd{ z+gm$q^;?);i5jm&Kchn4_H|9=J*M2g@rH?u@RtvqCHqRF90sbMRvO&M*9?2E7{v*z zDDBiaQ=d(JDtmRpeTk;(QntTu*ooPk&RROv#iHz|OQ1v|ds&9qCn%QGPm1|0&mk>G z2+m8@EUvgXA3aJ6UP@9?KBmU1fx@7X>5Yu5Zg>c$LP;m!VKnTJW+6uN1l3EV#<_#? zWZty}dqV|#M!kyxCt{|_4l-iSMg^8sy{WG*NVjOSsT|o$a_LD87Z?zT4-^Pf0*pw(?4OkG>L0VryAa;w_GqwY^NSmi>C zg2rB}w(unf^F)Oj5o^;_({J;{wViOYtrzQJC#Y%#T|rIHA1aJ-Bz{^ljXzu2)#bgM z@$g%nI^9*5O{!`wogQh_fJ!FY_Ij8fUi&UcG>=gD@yVK>0p>L1(3=EWQLiyoTq2Ob zT~IcvSKTZ;Sy5_K_ky<50Mz!{U zB?ll)LRMJ_1|^W{{*9-5Wy!OIRmZB#AUKnUt-ksf&a^hYkx8)mB8PZjfRtHwzZ042 z&QrWm!A~m@dP3t# z%sD!zP4VB^uCcCO;;@D(ug}LD49cjxq>~P0I0_^%;A+6eCz6#wqql{?LocQ=UUbtG z=#c{*rArt1BAdQ%J6<1A$Eq%ebgiRo)8y0W#$7hOD7oyismTs6xNs|6mIRO+QfTs-v97$|h=^eEjqR~G@(6;UxJ7Q5SeQ>d^y!sL zVs1*sR5XPGxtBSLPZ9ak+M)g>8(S_JU3lQ@IB4o=(B>8jgf(o~HIhydEt4Aa!=3hG zTJgY9?gYdw7a$a>E?#lK2Mt|LjBGDQpR6<*xuJyyiGA6>5`^<{qMj^mrIsHUEnCZr zfDtiWR6FULqZCL}qc(drD!$>8O3_U5*FmE`2eQC~EY;u8;O~?es$SBkg|(n;ap)y6 z-nbB*)FPJTkP8vPM1rBC(un3~Dq7%l!|*uWm}K_p`v$k3)D+u6E$h%+yI`Wzo~mG? zK!xa4o?#aYOk))B?X`wgkF%)4_9)*+1VJ%W>M^QOVzY!qezpsU-%*jNM>Xz2d9~)t zQMwCji9Iu+krhyy^2vh*FqWGnA(ykQz}O3U)d5M!O{Lh;#3;S8Y9T|@)b6SZ)pGo6 zoz7CTcs1bMENsN4@Ghy0`W^%fZ>;U(3F_|7vrU3X?Irw6kW375r? z#)90Y0y)O!n^H2bMy3e)}9mTSjjvzqDZwpW3D1AfB8R;yN#D@H`p~GW_mX_0+^qfX7gbLCTC_ zca8;103gNU=fb31h~ZhB`F}NrDusO{~fOItzy}b()b@rSSVu>fMP)Qt~j>9)a@I?GE-AY zxGa<#RVLU7W)?_k3gfK80%X#}Ry#(} z*#z$fI80AOQdBg^rXddpG4>rqvHHCDHYFtGIQGoV9wg70o5B<`V$yfCbLPHQn}^cXdXJnt*OS4tc}%S zAp|5bMXj3eJ4`FNs^4o;vDYev=pYbzFoF2y2+Fu&p6jqWWcOAz*%t>Vi9` z>lm^V+mjU$8c5>-^UMqr5Uf!yC%fJW4CEs*nt&R^iP-iK)viYDTQx#9eS8|p;@F3m z=VZ+$bGUTmj+oriLkx&FgVE{4*KVeOw#aK;hIMOs11$kkek;jGgRNJ6M!uHz$SUT+%Yq2o72(gKN|~dxQQ&3mvGv%TaNxcG%PzJ~1yThmz$^J(a@b) zsHw?{NU^H|Ehr#t>@aWxwuI<0aDcJ)XKIXY4UbPk1C^BZE6@kd>*;I?5n z?53-vs=#=MCP%JA3jI9ZeY6adxisH~sl`mNA@mQenV?;68>ZY6giX;089G@+7`m2@ zdIX}+0sZAwOt(-KWekfSbAs)C&o;!6>&K7OW-PD45u|cp&{~)kMC%BA*JEQ>aCHLu z%VssTal|KJ#ZXfu=#{1dP8qR)X)M|TnRxL{nlp_>2pYH8Swz_3)z7%Qo~dYH#XY{r zLM2eJTC^Gxy*{?7;)ZHA1&Aslkt{;+2$mdm)fK!j_y|i^ zh&FH3eqJQPx=&dwMhz=W8zA{qFCfFwU~VCI3LLzH7bwgg>^TN7dJ`Fi0}!pWc*{mj z;t3f;bfwAlU+@$-99+Kou=OZGy53u`*vE9%1k?Rq0+lP>Mx(2oQU;$uy43b1axI z3W`at93Rr*7xK@@7U)5wl2hYavD|5BptMI4>FE|1%g7ovE{rT)mZh z{rBH|9{_(zKmkaIQmRm(0ZJegC z2}wv&D$*#>iqHv2(g08@G%G?QLI6+*&=e{oN~uDjN(j&(w4zd_A^|`vNChAb3JF0# z2$X_>KnT!-KmkBB1p=)Apwg%`f;1^Yp-DiZl|UsZtqRg9O38 zBS5VpB><%&(uxHlp-L%8fY6Fi4FZJ<&?v-!(zGc5@CYgg`wEJvpFHpPKi~6j>&&&^ zkMoSpH@*GP{@mF2)`s12rTzW^eh^RwL<@mM zM7*_X&ej&@$aoHXqT!%oTdDVJpP8oBzdn8WksEqA@4rU(+k24W+vf4yL|$4MiIJZ9 zyz_m%`ZV+Im!nPgH+c`2!(OfFLpVZTmp0tilW?7hVX&Pm&nWSS@SY^Jtcbj^*@aCo?*4Swj(LohN z(@k~Yde;ESIk%U)vEYhJEOqu`p&FWv9mnXrr>tg2xGND+T? z97>p({xNktfK;qX!eASt+ep5Z@2o(ojxE5b8%23vGu}CHvsNA+_sS(GHFLHSPgVW5GVk-6~P`Q7>)NgnjmiE~$w8Vywb?5bKJAA%OIrAgAgN9zdT! z#_oXdFL^B(;H_7o=K6=;Fs@%VS+3+m%~~koR(?$~#Mhh4TnH-?u*=^hl$TmVhZ|!` zfU-xqEx`vi-af)Fr6IOs%j>3-q>DZPxE$?}NS!&M?w~Vz;u|Jz8O&uY6}-dW{ac`=JLXb_iKC>y3en z?0nP8Fyv=)j?^>4L7V#=9{N_%a;7vM2L@PL4i(JgjF7!i&Nsw1MjKXnah?gu52 zUKFNilr0v_uQ@4NO#{f5&DYVw)yom-2eJoX9I`%8$OrX3F#!(P<+l9-1`Gj~Ko~{5 zXU%*J`KskNt}K~AHh0Jq_EwV6fDed;1<(Y-;7@eE%5T3g)2{w|?DIT5;kMLX$6ucN zvY_vk;yuo<Dr8*BG&Jxw3C`&l%J9ClBi;CU?j#|pEj3{uCw8>yog zTAy$hs+A>+L5=)wLx4(ex!CtcQ(dTb^W%XPO1lwR8}ZIo1LrP18o3k!$(FL&2$A2ZN?BJ(w{CC}glCpc2&QYP5l@IQ z8MId{m67e)K5?=7%`_wug}RjD!&AIFqz;>TJ+=ycjbrK4A;?1VB%lxy?skU0FO6n# z-hMyb*Vb&ME6Z&q6C}i$oh3B-=2*Nyoha;nK?N{C5HZ76`Yr&#K-I-sbh1ZeEbPIV z9PsEoTbCY54G294Uhfv7v-}POqA~bB0Qdv%ywtJmKRcsiDwpp!>n&eLxg1hU0KS7Ba>NkATchfUT(kd0ea&|uOYqu8JB3_s3!(^_I0UtRIemG>njVIT zPK=#lxqold-lo?szVN~EOv#{lsA2X5*AZS?F+kH{l5}YCp8a!N@!c_~KI=E_55I{C6 z1L)lW<5_zL5G-mim?*Lf6h(;OrVS5#b;WVkn8xIP z45G{fq0kXhSP1Z73?^uA@vb8ep@>94#Az1?dF5c z5O9QH4S1*wUt1nzcef7l#(g@yc>&^=Yz08b&oTSv`{w)R`aB$gZYKTi4w1?cx%_kW zE!se~6Tl>x@q$$$H`;V=>`((aF}Hv#yZSg0z%TM$N4$)JuDhrL=3Uu6DZ|a zTJGBPz&rZ-k43KzNKxL^7&ZPy=os}qmI&Zo=%wW57PFApXI@i9+Mizg?@4?D4@T@- zH4SgpMnJt=L1I3nu>wrx=bl9t4`5jr(q0r=0-=lAYpm+3sw%ezafVPuVYg z=|FmZoD#4f0D*iVlZY^@6vb24uuS)mvY#Rl5cmr1$E;?`D7Lx6rZJ7e4DF~aS%9P8 z-!KQ?d>>+jX6dG2n={Dhjs7L3>GBIVgq(S(br}O}UF;whS)rjmVu=v3FTlM4Sa{xa zknwgR2|d(aR)OpK3yqd#J7nbuuwN0xI!f^I?&lS9sze)6FjY(5I1RW?p`ttqx@T*8 zkiq%@Kpar;vzdyB0bPnk`%UbAk&+NWT`bdWwE<7dH4pS71*v(CRf(&sH+bo*s2*fcNa=iDm_(H#XT*b8;{v!#m(YqpPWJAm zwNvFm0E}6+U8O^!ePALrY&P@u0O6D?KshjNVJx>Zbu^^&-*2dgg` z@%VP>=h@RZkY}rG|4?b?Q%?Aq2^eJ*zSEQgPhDlia{Gq;-7G04Iw-|ZNTr1V2*fXr z%3NilCz9C>am$2azDd zn_AmQqRT;Db?d{CgSnv!b!r6$u76_gxJ;he>IJc3S_hA$P@_Pp7_xv*I%LVq;<4OS30B8AjC(W zlOIZ-aUp#C$F{Jh)iGlQDe-B1B*P+aj0L+{2H4P{JWNG05-3O{PEkKng@rf3W+jfG zMzdp$ej&P75HQneVb|GRZWD$}Xi`U#!cUBylCYjI5;LD99fPy56b7awKZmWpS}heG z=<;+51P4Vp4-0RVa>On#(tH=}KA$_7mrC}(y(lqJJ z4nKToL+msH7&};?D*JH>wLrw=C6#PYNTYfvk4nH$x2BHXx&37ps;=&ohd}%5y5o=i9+-d{1h&&XB0f+ zM}yl+9Z-=F4Nxz>UNE3cLfrG#Tt4s}v>|b85uxAK3130OIV`j)905nT#bTwq&2&XZCVyl75SbN%sRPRfM}lhXwK4dKQqmC!c*>a| zgV7Rtz_`m*)9{D`dGGH1->mujy{9|p?Y_TyI^(8b{3iF0rOpOCAyt4k`oR2EhCC`E zSnUne<=+T3pgt7^5DNH?pnUDl_o@O$95>foa}&8lz(SW=24K+>mOmh=wL1zgxG&qp zI-6D!YypPAzUS0#(;UimJDLv3&q$mI3=9Z*i3_frM@}rXMYKZ@vjA1pAQGVk`i^|8 zjo^JcSY9*ljjY@CY*@yIXStK}C~^Soxc7SxHz6^s+4U zZ<~<58W<54Bmob(WU&H{?x9TIyTzNy1`R=lOBn8A&v7z}V)zE*G$>w{RPK5%l%1FH zhCr^Rz2?I<`(&G-VHm<6FotS^5%+2Z!OF#ks6l zs&(`cEnN5#m<&=DY#Q*|gbaAyP4amsSXcr^Y-p5E9y8-HvihhBviBC>qW%p2D!yk^l2BT|&vM`!g$Wq21BnVBEGNY$^`b(oH~+2ej& zRXSWBhs>mb?K==EwBrG*CQPBPbQ-CBqisbrg2n|a@Z+@)a#ZMUjM+1p>EDG=U-$S? zMHT+r-g`dj$A4XZe}A9ddHY^D{u6A{gSSTlzrt76iC+=Ua?D$ZBt8;0V$ZvV>&eJ^ z8!EgyvNS0{WY%1EZpiCpvjdo9dLkM%1$MegLJ_40(q1~Lbd=)0v}7!zP|UGj^Ag2s zv<2FO#-g{-7A_z+SOy_)N&^*?cWMu?al_)0OkYe`uX?m24Z5%H5X#$rf)5IFo0B|t znT)_oY>4ADC)9&m#2yB!((|wKmsB1&A9YAY64xXaC_8&m>cqtJx)AF zFKI6{Ei#I1P6~VAGgY#l?&vB^0y8RH7C|NUy~tH-)pJ^8h@$@J3F|4njR8b^B| zb_6OEyYugWJ`aQB1?q*+3!Z*46%GgMYR#|9vwZ5+Gnog%izyXzp`oL-#s#7Hb&SdO z&V2R&D3m1i0uC3cVF*W%Ar|4v<~5iaQXt@;Xvl`4ugPU2VvO=TBV8%IGe@1{bvP)V zGkO#uk5#3hp8^sO#W$CvGxlvDGhZDdy*w?7P)%R1F&jEuAf#Em9}Z#8)Zg2*^dKJb z`0K7TK=n;iM;wwnhX92#eSH$1y1NMVPOA|N zG{8Zf3AmF?ffYQ;6!c0t5@c}LJIhPLivKTg2d-52;P)d@I21ASqVoL~nan0~G@8Umt!XgneB@#M3B6>?3?Y1Oue+BO zOWeo=fNhYw5wXq+VRL~Wh~o-y`n<9jB&6s#&B9MQrNisO91JQGZa%E45=GeuM$GTs zq-KVPA2Z)C+g&v4Ji5Oi0*1Wz^R9a1a8TIW{PQ2Zf3?-N^YH}mg2OrpVhGq82ZI3c z5C|(P(4}#hzCARp6+OL2`-C@P2bM}NArllfDblzk@%cLtLA|A*`ph&gLXA9dE zeXW>nLc^)AgGk;Nc*}}*AfZd8q&|ZSAX*@stU?$O5MVJzGW9O8WJ{F#n6U-`V?c0h zCE+FH{jgjB;(OD8=R_?zTwara3mz&n(g|(AummT=$Qc$nwl@bgr3e>fX=VV94<(DR zLV-{-bUXu`owt&41n(O?}<1*W26}&&v!MzOG4AL6=KPZvz_~80&jH=n# ztFtW6i29jY9a;6a)-dZ8EPKl6NxS(nYky?b)o}(I`*immsAR>}*9`b$q2a_M;UM~y zsMWyQt#i`^#l>w-fGA@Vn@?HqO*G)}@N;a_Oru+CTV1N>sGYdiy6v+|7j32)=Z!&X zvcu>up+~sZ(~pmD=E2r2-g;rsUa*m>ulcF@;eeq?^bJ}tu33DkG> z%>uW$Vm*>2xm@kkFLl#*FV4g8R+)bjC~K9{eNuni3E&dM%jA&pncpz4W)hgF{8`}+ zw4Gd0du=K!VK2sJ>F@DNVErrBg*AmhnGw^ds;Ud?KP;iT$@uK*C~6Gyq(#J?;hrBZ z?MzExU`WGtMAC(L6Cq30mqm1!%}tAEJ+{DPF~FX(6BC}7#NuCNjzyzQUOeheC_EOn z;_j9SLWpJvfrP#@x0!Oyav$g96Hh#=Rn~trRa}tII zv)i?X?AR(OB%uTPRx*2cd^OFD?LC)*P?D9enD`-4`3T3{UnM0saC{C16{KW)C*rqL zOh*;d0*HpBEcDaRi4oanE^7@Jql1_P z%SRvEmKwEZyW#^bU(BL3y6>xA#xE?pzS-~^ZNVnf-s0jiLK)2&Wh#M8L>_a91Qq!eTPWm`m2y%QNY7pI7P&Qf zbT0S>5X+noa=vX3Lq7c?2bh|I;-|~8stNJam_{@f{n2Wegr?+a&2VX@W`+!u&rLk% z3wGUm5ubzaz5ow^1#L7B!96S$VV_S_p;~$MshmM))X80Jun;jsgGsW$j4&W9Flx6P zxvbd&$TYG0S6CqZMa_2vSw95J0|%H_dnJYg)CqrAtYS)tK{!G^>BLk)pL)fdvfYX> zoWAOxA2+bbRAs%9)$d4ApsAfDHm*i$<1Ai9U=UT33tDMQaB{ z6yIRvGG@bq5`BbIB3cRK1Hcsn!VS9M2Va-nIQBsV6(SO^FBvzak#>nUv|9dYHfTNx zlkv{&$8f!lJhzT*v`+Vp7$;_jHg%dKkaGWn7m zApR@k+Lm-*A_1Zq#+2-i1mhiO(4hTWWEC1KLHt@phCFQ0V|i(AZLf_dEp-660aTZn zE}k4m0CXut&x-DA6f+InFIa=j9YiotQ#lO=488}jQTOdo&yri{7PFd>XY zrU>8HG3ktuh|9>>W5}j8LI_2xlH{YDGK4BJ)g!2OGB+|p;7yQynvFFMNRQMnV4OMjIOFzyvraklr%_6@_&IK>FZHVs zg@FuMfULE}Yts{%@1BflgUsnvf!&OZuc~J8^;vtDf=nV~CXlJpj!PA;jskRMMu|`9 zATJW8sWar-(Z@o{4Cb^K3(o}cI-PL&`Zeqwl-w) zCxen7mfDwt+Ad+oZvS-V3jzi4nuPNp!y~ECZ32+`K3LN|%FAS~#MIh1o?G1ak$#}BMW;cqKoY9T5h>ZiB{Rde*UOoFCUvvS zacU7bRiUmRUIAo~T?T4o&RLCvybR5YI0DY=ATnhj=Ef>#6DNkTDtqo_5xC<4lcbmR zn3s9`W;v+tO9hi-f}F+OKf>DRUM6Aa~+y z`+4gKA|aIl2UG=bK}0G}>k!JuMyR_Oq8ylIiD>qU#R-=1;2esb&ndbyObDk1t{0#& zdUnLTb!hAm1Ue0kGad>KZ7~tyjWp=vlM2$z*FA22PytW;D58r0#Q*VkBvXY63OWRC E@bQNTNdN!< literal 322448 zcmV(@K-RxPT4*^jL0KkKSs4*;o(1CcUyx7`K!tzg00;mC|D#?|1OQ@=Uu{aFsZU; z=xS2aB(;zd6jxro-sRREwziDvhz9R_+P3Y{-r9F#sZ^HQ?&Fc(+@|$zqq7BSw6AvP zyR+B3-P^e4vJ|N++j{G#tzGN8vE8VuyA5pHx!r20y^n6!QmfXLvn{uK4m|eu#@=_k za+Edic6EZ1Pod$l>T(Vc3E ztca?f#qH}SdFbWs*0?)84c_;hJz!L*D&5|mG+-r0&tAOuy4}*HI-q4PZRwGAumOd)<3gF4&Jc&Qxi($7OF?D$(7gB$W2|XPa%M>e#zA z+b?!?do*3P+hsRefYW=jy7#v(#)5BlUb*K^+uf~iTUiN3Rd!dZ_p+*q*SN~Pwc5-t zcXPXy-JR_9J+-HmIeJjiS~vm1lFFy*=zjw(Z@5-JEN7&i8L;uASZ47q)ev?9Q_7 zx4WS4XM0^a)vZwC$FmjNO?2v-G|;wtdg@(n$EO{ds+H-Qvvkr4JG*lfLv6PAyIcxO zuxe3MRqtKhuvKzrO!VE`i)U`#m3uHLBvm>$Oo$4t4SHv`Pi9+ZcREUmF0-Yp9@wNR z6yDc6-uGtR?>%R|j(52D000000000000000000000005)ddcopWy}I^yHJe*}>$>#owTv$r~%Hruvt?(XiLcL$w9~&?7hx5+hXmu?qTWN^Nj+xx!sz2 z3eLk-+1+<;_1^Q`JIwCgYj9bQO_N*O-JLw_G`oAN1+A}K z9?g5b**)34X@;rx_pE#GbY9wn-R@-b(=FZEmuuc`(9~HdQ%iTe_1CFx?t+%y_V>N- zc}ZTqb6vJvcIfX|-p=;RR$$wmS8lZ0WZk%zrm1_qpk5ow)eg2*;4ej?cTe^dCy+Ya^<}pt#xXlS?(8iz21(UUhVGAz1DWO zd(^ug#@pSIt<7$`ZQGvR#=JEfV!Fp}bDf-b16tQCJ%+p3-u8C)yn9b|?Y9bYsarBG zJ9Ocq`gPd*xSVslr)S>xt?kR+d!&x$aJ8j#d$VV{?r~RZJ#Nco*SoFa+C6tSymu<@ zo!#1Q`_D%bdr{Rowzk^Z<)zu3lIwSO-qW+z-N$l`(eBhc*E)7{Cw1-6?!#WG?7Fn= zw63kAvEALfI?s5^s`k4a_jeADRoezTy5(RC^0B>mILW^Lsb0 zyLki9BWpg z({FCw*S)RHT2+O#z1!2*yPR~_SFUen?=`#ECo|sg=eF~=EzYgNM%Qn9-FG<-t*!3f z&v$LFZtr?DrLnu7TW)%LUE7`8z3n?U+Bm2z3$zL_045-Oi{@24-{`@dE{=+R008~@Y0ukf6_ zGs}FF{-iu@{cfP7uM`?fRz>hDYoe-L{1K?`?_03KA~W|f6Yck4=m$Y-ESnx*Z0+cI2hRw=%E0c!vN*3a+=6iB=`A*e zp5jeuY8%81W*&!Ctxd~zCffq9ByGPKY9-U8mIq?6--^xs)drUv_X7CyNt;v$z+h?` zW3snFnao|$7=sNV!RQVJ2ziyu)lD<1LS8{GxGXY#)0VbL^CLoCX72Rllb5(*_Q6nR zEs%UXOD)JkCxJb~Y1fI^!`$O@u&_};t0=z5hr*IXOOk6lYd!BHf)G~rg)Ks1v&6Zc zodYYbAi0rq8}KcuE5W?;SnqNuj@NE&u;~F|hiM5_Yed`(6Js6AGRC%voJl+yHPRW@ zRT^s_L_lb@Fg4A#2suLc8evnpiB_14zO;B5povj~ngl5zQ=N^dHhg@g$e5}k8{9to zBzVP`5G<)P|evep%WMc(m&CffA<)!T7H_6O{73aO8CkBFAiWD?)kP?M_Z8l8_uoXRT~wv+Q(& z>&iU#wCQmRNJ<6a+_XeA+wiZT_#UT(7i2m8?ei4KI!oILPY*cHsyXIJZBR^1UkpiPB{Cgk6~t<TX=hC(=7|Bm zIbe4dZVQ`MJ>Z=4qoC=Ait#4F&TIQfZunw!iMX{o$Jliz3r`#p>5xbQ|7pSRjL;G6 zgGZv%xvUSWxU{ZX+S@#1wBjw?7Vs#n1^CP-$Dbwb?~HlVst3sot>7u1FhGT~u6FV{ z?~KSo%W2heBCQ+3_F>D0My%A%0t0<$cK%_7cQCFjcsG#q!Gm#Z{Pw*}S(4^2?~{ZY zR@?|d5n51o$neQ*)GVHEojj%r8MjYLeKw&uA-NLRT-)IY)NHd}$A`k1^BVqx+8#(e zSz=BeeWnpKCng*%(JX`gxoJzIE9nTJ?1rJ_CQn`Y9Km)y-4QxN7UX&>d?L%)>1pV+ zE6C8S@I~r1$&26~WNDD;LSVjHcSvyjzVoD537{enU?BkCUQQQUA}mFyBza5bQdd)? zA0?FltMjYhXe{?C)_r5=^YNTu&!Ko7(g-44@JGa9gW0;lITP)hgiUDCcEpPMX^>nN z?RfKCkBCdO7sp-8uPSZ^1rqkPo}TOl$R#s*9b?{en3YI|e792KOi z96duKrV;8}FpPPFB@pB$D`9cyFsTZWw=znuMVMmY>OWokQB9?YR16FZ_$YXw-<<4O zEkUk#nNqW*!F6_a2=2hCbZ6j);s9&~%G32HR?}Doz=vSinvCYyTWZ5uN2_$3+B_zl zGfvkv_U zQ1wAVpKU^L5`HAp-wePjCY^C;{@x@Rf~&?sLA`hth+)#&?yKK5Fq2c$lSmwDJLHBg zy>+b;oXfoNs%Qbx=gkgQE5cGVYz82>Hb83v?J_}#;5PIWax;E{8|uYuSP)jcLX}9M zoOx<;HF>q_VeVFMcicz6nm$)kNDB(ol`?L9aTXnQUk*PKTaIx2EGZz;}i|%VOVUr?4P4lAfdn*EHf$GL_O77)6a`j^hE-YDh6$+DU00@ zD{aK>d~ks#2988R8ccDz-*TIE?7*?}IDMv#zYmsso>a6Esv)=!lR!h?3GD_8(KIi;oIOGNgc}F9uY#^6#4M12|5> z$y@<3_aPbdprGZPvJ+_ox9PUruXg_0#VIFQ?wKk~Vzp%abWG_Nx*(r5S4QOm0em#`TU0^ov@=V(8$ws3&}Q zkcqM=#vdbD(XpJ6V_9-JwZ}2F#31&juO4B!3VvDV-(50^Si>E~j}!JsCJ@QqFH(|4 zfr?n%wu`7b8JHpr@Rm~&+YQl4$wdi;5nYsp85)~?swX^f4Gw3l6pYaj%TXjEwwtNf z)J{hFhQb0UGUQQLjuDr;ykvBvcl4ztLW{47SSmP zBcnz_vfUK7!p=~XkAT(!2!qcbejOGdl#hOtO zAnIdSgD#B@A_3M7SkuZ=mum~we+U(OPmk@y;y9|n0 zs)XqXq;R18xb3~-*yP_m%Zpb@Nj1dTL(1*vBdYeIG>*KU@P!~NL>C>gGl2bZlZLNg z%)ld=Pf6jq60E-AVu=I>rUgtX@qw#K_-w5SkrC))Yh>}$Cba|-48``W#z$gOAqhJz>@K{VHJQ*}^WtH>Xu!GL3wW;zjFYr_xq#-l5R%^W zj`USuDQN3S3>vLwhPa@>sG+RKlaM`H;sn&vVLD#Hu@0?mH!;RBkSkIG0pOmYO?f<= zbHAxUsUx9f94QEA3|=*)4jw@(LDta99R9&$u?^d)9hou)8Ca*Jjjy51vrh)+jnFTU z7l>p=1~3<$lIT%KD%NNtLqIx{6?@H>5CWAtD80 z@yMLWON#5K%B0p+y8)0!XR`6(uo!HnPKsU$UrWft zp$;mT-hBPG2L$O|3+Z>dA}>eDrrPIa&WO1)^eCxVLFH_61Egh*V=)N$)>EAqN!p`!lgX`j2Sr0j0&?^enSxlsvKz)$6 z#buV(+>R&V&ROvw8A#a(NT~>8k362FOU)rnA>oAkDmW*->Ar-Uc$$w2Gmb>m;<>*7 z)meqaXSywv;I8cCj2A%ZAWyu`u6<0Ea%BjqLUV|_PE9r1C3sYpK}tng*8Y_TyJHJY zlPOj-6GfBdpuP*izi=rkSwi|e3FVV(v}jc^6+<4mz8oCW^^DN^a~}z$tLNYi&UF)C z-hhVz>yk|AbLP5J>4ZXjHN0LY+e>oVZ$epgPb+U`J5LWPBCpz+5XpGLQ~ zH|T+8$e8VmXYyf|pCqt?nD-=Qg_5ldeV1@xxP;dsbgXR#^{LGa1A87xwQ`0jFS7XO zVt7hzxI|NnOjJQkI>8;cUMlT0WjQXG{-V3U>A_jmO2t+Zgy&qJaMSP(I{1-vq-qH} zfQ_@0wL>h`k3>saQ*@kZ(Vss1gpg76=o0I6dtEtjJ=*l^WZXvnej2_ zpL!))#M#PJ%ZC>d*Nw?Eopa{(NyL7R)P2* z-IZBpi4K}eu2Z{?r<(+HYhfOQ|ensE%j0p%k?aaN*U39JQvZ7g?6xom@8E z9duGUK?T({&%RB_o%Jl|fb>>0p48(mlt&CSr9w-SbTxdSKvJM3yQjAIi{l>lhP5+r zkg1DGmDenbN(v5Y3RhKMj$u(s3WwYn$X=V`sJ$=fhPfPRy^KdHgD#RDBr?V$b&M`}Zp1nr`@%Rb zlj0y{5tgONh$B*y9w<#_GLqpdes+NLI^Nvq5HGAUM1V`hLWW;h2q&fyJ=l=;Hu6() zl_M?79&;PO5`z)nuW3h#DeGs-zJMcA=ZcNE)A&fFQt4=iTr0-$L8AWx(O1e>D$t~J zk88vWfiB}D)h#%(vJ?%dEDU@LhcAInJ-(9(?`4(R1d~F!t}vS~yQF=3SwQ&7RSmdj zKM3lwT*m={)8>0>oUR``Ra|di+7eqgvxJG<5kkoUPN85ersc>9#^WJWP^wCJ@D`3* zBH*dB080&toVo66MqR0cQ0vzZIN85VC8%1?hn-2evGAy=++?k>J{zFqu+lZ`LMy)| z?JQOku_}>G)|*p<9-L2D5d_Qlek>R5G3t>u?nQ`ZY#wQIUt+O&_M9|?T8%|?Hl2~W zkQ}$SBo8v}GGy)#Ex*^2F+P(#+BG6n1H+4@WKz2vxQ{E1-MUL*~5l z1>a8StaIa1cZt$sy~NQVm_tEgUg(v;-CucPcJ#4r0CvyGewheLXGh^2BWNh&yOyK_ z)6NeFI(f#eVHxcMZF^1Cj3!&J3Q_X7jaiir3Xo^MJj8lbv^mO>XuCfJg*z$5C_Tnn z)KtM~kt)O$hoiWLGd59V6-^q4WKCwES|-~li%%m5B5ZwagWDksb5QI(Dn3-%n@X4| zbENQ~LTSEm!P$a!$QaPhc~2?H&TP5eol05P#$14tjyjtzf;J%opM72p7_xQ)%B{E# zV5afugPl~{EL_kv_h*EEX+8!Qo{a35X_7ZMGkg4lH~n1sM5YIfg7$=9o`V4&FX_4cqdFmS zpqMdhI&f4^L-egww<*Su>quzfA*Dlk1T{=s&KnKgk`G1^?mj(6-qK9E@*4UX82#)^ zk`2RqOIuBvl|AkSQY2mseK5JAuU(_itrg0KVCyH(hG)Y^YryAhA#mI+bo~sXH>GW| zLAq?wPHGz~-wS;-TBj4ue!EC+CbS+Yy$?0*wDM@kKQv{n_on0~6B%vtDcC`XG(u(C zgm`@5>N2>e!b>7sZ4#}P2vLyZEk#RQ;;mv%AR5R_3zVFA-N}6NIHu7C0t|rl zurc{W?}f}HTU6XF8gk{spc9uDBa>9_NTodx4TO8u?1O}j8o`Q=Iv1uAq{zc5aIwsp z8&Z)$1sJ~3^s;Wa%#dEV#ZVsiB?LhQikU4}0z(s_d}2!m4x~(m0)WG=EW?@4WyCcL zTWQb-v3~ai+70GCM6V~k;Jm}4fI(}awdv|>c7kMk5&X2pA|Z zx%B2Tj{*UBc5pijUxiRGdD?CLEs=T6FXg=tH?KuZE}_IOzyw4bo>@`&H=g4iEH&HA z_l{&`K;|^C1ky#zRG$MjS_6S}75} z#Fvk)bBjKC6!VXcV-gO!CMGHj0>K=q({+$uobx(TK#Oev%&a826FPhC7E$T4=IWKuqNS4&1fk9u>qsAP7b z2<8uHNZ#8ALol@2U)@XC6HXEjj~5~W{HXKd-F&N&v8f#-;n?kPAp=A*^+pl!!Hn3( z2dPhbmB^0|j2`vEnhL5kpf8x4#ur@Iyhy6PC$#tE0`%qxg!e8u<@~gqXgO4r-_-o+P^Bz8xIN@@p_Ft|8d7nWrQ9|v7Sp*-r|<8V0MWL5X7H@IwOGNwTiB8AJhK%gqu1o2GM>ftEm z)uA9(Tm-%4OFy3;jC`L&@dVf2yj&aK9H3h<~=OPVwIACZ%yMV_KHf4EObIQ12JuulQ#15~SAm1jaOqLR=l4U$G;v?KHqh!4;< z?V>2PeJ89QS(7X@4${Y=TqCgqqPs(XA)Oss0O2eW*#(4+TJ5yBcvwJ6x?@(F63wN^ zhTvNI>S&ZUi&eadQN0U8-XJ-8JSJoW7tazrrWY&B&Xb3_&j_u$?QhfQ%f4$@MBqPX zaC@xju{vbBedN9HWwfm#X)--3$m4)G+8!|^Lm5XY{RoHcDK)+8Rj6xJP$-n3av0~< zdXse$$EHhDP(Wle_fcpJu9Z9v5EknkM(y_K5fhhL>x=^}@rSLDS1{kMU|FEO99yK~ zt`E28^IT*IG{f0J)UZRBYD#qI5oImx-?iti6x9%e+eL28LubO&+t!n7=s|i@4vCdq zVCm`w2eNfF24juDA5Ka?R67*YQ=1}N zr`njlXjQc${x&-sRFTf9TZNSgV3AAlvbHoft+?{ZuuV$jyCetB69INF&K5mKE%OvYnT#fxB>sHVdD0JRZ_dEe#3MCMP zJ-CCbf+WVEWuWRSkRW&mE}Jnp&i`K0Nb_(qVcryL!fO zuaqw7p;eY>-Vlm}SRrUlh7^@$k_2~WklGAxA49f`aSjeo2~{bW+)DC-kgdO45vPgL zH3=&Q4ZC4k5uy?*o5KS~r9*La7ZkR0}?#(3;OK*+I2SkK^sO-i}2w^0n%{G_ZTe# zDcRqMaaxpa?--|?5#G+YVbDxO4Dmf)e#X3@KUaXjNHpo74CtT>C=$Ldm5Jz0!k-~P zj8H|>*{j@044v}Zhbzn3 zv}0V`4KL0su5Wv1F9zyt#7MXS)F=W_994 z;8ytr3Cl0-yr+zHl%z}rtm{IOht%t{<8;L*J7cCrEY^ATEk{*VyXogx>$FE@Xx;c| zH0^|&9}yR9+w@nxJI}m&Y$fyCxN+!EV-}Jx1ntje0u{~Ay#WDh>bn7#s>Ea-U4zlf zF^cjCc{MJ&MHfuc*t)p(M)r8%xIJ*jAjNH~P-Lm#m3M~dvD5D@aQ#F@WWl7lurCZJ zSH0&IlGWg17@H-iyBZ>4qzjK$W>h6N6=0dFudPt8qMGFF)d$_?&rIM7wAINHv>0)y z56`e6D#l9lL>j3I_W<8O_F;sZ2Mh3mkr@GkTA{Gs#LN)u$Fjpk0$jN}w%@ng$3o%fFz2Ba z0_gdVFo~>|O4PV21>V%aYHc?ptP!qP&6(0-1??6~Ydd7;%In3V<#mA?(L1$=4>S=3 zxYGj)a8!VYUV`k1_(H zn|bVvseqcQq}` zKw-0$8VZSVv6d=$HQcJ|*AkfWc20}hJeW>aK*QvJwEz*1^l+il`psy&#I z@Ph~_8E`Oy+{Y}qgcyX{EqI``2TUn=_}-HKo@k=!Z70*&p(lqH=R%szAQ3$hZnQbe zciRoc*0l<7=&Ks&+o7E(BC_L{+;@oz^ipQ319?Wab;wYp0%Kxa4^J1F+Ypc@52|;& zT0mGR3n1hPPiH%~PbaUTe87tochSHqCL*;l zK%DuCghU7d%{<3#_bo?UN}wD9CzlB?BioNqZF7N|fSGQh>}7&t_mdt0Hbd=o9`96t9!?d z>VOu2S@h3}hK1PFV3Ozr7F#YC1kG$DOf@;t-Sp_)t~D->>52>5T`8|o>^HS$)UpQ5<06{TFPgBUFp(>9hVR9r=1 zxPn2Ov|0F3Jf7&;hVf*@k$CW$FM0ehCCr+&dL5TsIe}e@w-KR4z_B-=duCq5&aptC zmdNGmVV@eVa7)(t~coV3}WND+FR*Wf>pN{cZtuL%d+3O*h6jF$w8;au9m z(E2`bHPz72rY=0;H3%Zutc~)J&5$7guKFJJmg=;HY_Eo4@%hWHg7MnoFNl7eV|tYA zQ(DxH0-ZH7^(aWflgd3KWVuKb(feo1!6nHO7;_j^x}(_`T*VM&S{G?{J@p*6z7m40 z2#$0%S6cBVv(^>PTQ%f{xI7OFpDC_M=HS^|-mQ?UAeYpjbh*LcP;_B1skwxR6RY09 z$EE6NHY$Y$v?WHdB+!n^mU8Oa=qFFhKBu5FoT`-B%?L>+$L&foU6;y14g&&uqVS&w zZ(TUEcs)W2MyQ;GQ~1TIa6sg<%-uT@11#JxUfDiUztU%_aec#mD?!v{r*R<5bmVX5 zBr`l383Q+aS*hLILNd58lAyd;WDSZ%svB-rlUFcU#_FKbuBWml9!_gV_TCT^bt_IQ z>$w=J508+GKP`uS91cZn2G-|lPaS9knv@hCrz z&vT>-Qdx#a=)@3c>DBOFojs#GKMJ7j_Uh36eDV3XO#+KTxzuBk+J!A9S^%rghDJfw z*-o=mx<|C-YYrGo-6n9csD;qPz0oL=ZlHwPnM%`CycQw0tIdV>sZfrFr7ppXJ(ke~ zS0{DD;)MKTdLX93LY7T*;hwLWwwk&wSLawrv|dLM(})v^m=h;W+EA!!T``KLc6QV) zQN0|3`~wc%q#JM|LDJ2{)!fxO9%HWQf$8C2Xo0!qO}=%J(Vte6bWM_RzPdMHX6%$~ zC0$JVxKk3Kkdc6*?r5eC%~L_Ri^4}KTm;fsxQcA=UfBt zv|Tk>1@8sXh&taClv); z@&$Tna;oFkJ7^!6A_2Rj^>xw_i`b@9QJk3uyfQsYbDFe576mK$cB`Z&Ms`eE&dll8 zm2@!?cusH1=b1$2lA%F0U?*3~gBLkI2(6v8O@2~y>xl`any*}zU7Fabw3GaA>Af$$ zp20I7Y=xUZC7%J&`Nin_B{~v^D_INO*;HN5%a%e;Rw5ouflhuvpzY-Lu9fK)W#oB$ z=U`kpFQHD$9$@R`vEBuRJ;zlvKJX^Tw*ZDmYr1nl)`P-YU1C&lmfb{Bj<6x-fn8ez zvAu4M+VW6*@(3fNr;!F_p6DT#Wf=~#Dq^GUG)q~&aNEX|84>gN5tQ-OX4MKAIn#^c zuqwE!;Q}sZ;p`2+xL{1PZjM-Uo2;fr!%$nm_kP9~bG@ZH^#@^XOq(gRgI8hzxx_o( zF*4YBaa=K_1YWO5aN8QgKAr0tw~|^_mRk0h4SD-l*gJl%?o%GXj}wX_%PX`@jlN@4 zW5*7W%-j-zSPu`ywZj;=Evon!WVgDLDlrMH~#JmE$pn zma@d$F=AiSw+0a?g`);kT2N*kz;+~?z2;NUIiVzUS9ib*hp zs4$ZJi3X-ry<^M|ZrK>?8CZa56#NYHShRFE zW+yfz>=JMz9=Behdl920Y()_}LRe&LwUwUWlpDobv*e z2Ny18xLDY3c!SkJ$7no`B`;0ONp3`yTJ;*ZE!$i;<9y{pu6rYezD41j`3huNM1Nfc z+z6eHXE?&^dQ9YnTYl742Q_#O zxipVWl92eljv;;K-oEDE*11vVet;6sHJj+r+oQ^sSDO|7tL;A9RAh*)SJndK-NjSW zeCOoRjeexBKez{W>`87 zM|*Z&-Ypd$i^AC1V(~|)`bg~rl?+AeIDtHV(Au@wXgc8m9;!V1>Qp2WgVP4cBzBn}UQkMrcuSu6l+_ z(VG$g;MbBom(soB^_a$8Q`VnTmefwjMo})k=5P}O*sY$#IzfH2$0b`la+rXb$>wc; zHF7t0RLB~NolXR%dqQGshV)+SuXMM9KF!W$VbmU=yvdpAg1|`_PA=IH6mv3|`qz)U z()}3?Tq^TCf})tARY||Tq&HM(ARmId=gf?{3Jc>{cw3&*3E{t(b6PcEAzE7DVkA(& zd0Mng`b>;A&~3rY`RuB-uV*hy0?eha;pA~M%<#>;Rx-94)go>wrS3}E+fZwfJCBm^ zK}~$P+3v~C9-npBqOY^Z$&WfNxHSgWgB+#qv9f?^-5zc~2$thpKFW3hJ(ieUlgEU3 zZNiur=nz-}nm7|XQCzm15Wqyfg$wxk9V6Co1J*-;!#S~roQG;cC+|uev){Ox+b?zt zLdT?1n*{Q%COtG{)7@R-5L}ch8zz)Gx>;W|69G6(edf6)1ft_*yavYuf>jP_ZTGhi z)N!s6$h~|Wwy6W*;7s}DHzD*Tu_gP7ws?CkBNvd;-4_MLiN#@)5YdVg>=l?r?9Mu}NNO*1 zkWD6?UKAG)TJJVaUa4)&+FgeM#gri@#OB>^Nh?L|KFOERL~mk_^-%G|cR#CwFf)(B zjW)pb!+c=46qKjP{sx%u3hP&&yAFXLYX_QaLn?Idn@wk}AS29Jumc>*1#R3f<80Fy zd<7JFZTH(mLU?A4R-<5Jx=DU@DHY43y4rJ5{n(1Bq4CUgBjmllNW<0{M1dUSj;wB3 z+5!|duF!Szij@8FK zhq$U*(S;V|bhzXQ>U%*>`T_4Ai9Bsnv5|xZ9J55p?gSMC0`w~izNGXUJF&>22wdU? z>d2a#+KBrB^&u>cd>0b?<}o@5Ec8CmBk+@yO3idPVsZ^jeEYUnohtds?mVlty$6hs zL1Gm2V+hl6d3^2BJv#6Z(zQHbXYpyXyl$hm0j*PA=?`Aa^ zXdFQoMvV&!zNum0rat<9tbGpph&SFUH9y4s;7&eY`@QsM% z@?)8!^^Zb)sLzAQDQh5tM#**(7CDM|amD6_d{RQsQo4rJywHsw2up8QrbtD$gPjik zjSr;jZr>P%t$%9FBS404S7poV+NU3mLEfd1b%Muy9{6BMZ4CLfGE(1wJuDmkA!LV5In)_cQ20Z5`xHcroZ0zn|QF<1rdNjXdwe#O{L6BnVJOjsJ3=^ge3iXmm-8?)N2b` ziXde4F5dAVv=ea1kg9hXeFcarhBqMxzMfyr;hcPBB)0JU@bO#>{G-EbZe8O$;?SFX z2Hh-aLXIw;kvo4_UmtkI*<|FZrXA{hD~MY=Z+p2^H=0e9%#il;&wBY3HPbszFsp5s7qErqd>T^J>dYuL|Y;O*mk(=vQ zfl8eE4^XaO8y+q+uGeEVy}`)HJ}heo`t@iqHuk}WtELis&?mG|67Id!8g;pbw0n4D z8hd)a58|Hq*z$Zrc0+Iu&UtovsMc*@=Ep|HXfV+ICU1P~?Eb_6$Y`^KyFS~KoT(*u zhtdi(u4wILj!qVeS1`t>o+$M4Xh;(E(Lv!mb+fWMuqsVk^F+vLM0t&qwzvoxy?yRvDFc#nd3CFzOc>nc*!i1u+pBuqcyWn3hMsjuM-mT3Z4X8wOT|j#@XuoBh~22 zd5vD8aNCjx9Y>3bR|c|ZP#gA|DYN6vltQPO3nX%JdspaaATYPSDDW0_UDuw{Al$%_ zDRa|<*<-=FvQV$n8`MdqF>jBH{FN=D%!22*u1WKj874Llor@0 z-gOZfD93A+zj{b(8%fma7AIRQyZUu8bp$PSkwHu~wS>b?29{o(BUh#Z&3Y3}%a65C9vB>ie?4^?la84v#I21`o}&JG zf0S}3-7#a*IQ_J@sDSnfqM^69P8F7ZdiM(WYtp-*MXOv3J{L_10q4k@vMPd_E+ixr zt6h;1$dL0npA&DO979l{gx6NLX*l#4NmkLN&V!t-Tc}}FbI|M5gqx{nu+V{ee7pHD z%3j1*4Yvm8gP6=+OE{M+m{%BS_PN2|#Gzsu)~`|zm1pms>L41^CG0$hPct5dSAJ4v zLT+|b;)LpnBfF2N66BYc=pju|?X^e^FAPgASmlf!>`mVa%8Sj=T6#px=V=bBpc`p` z*b!hsiUtL(+}&ZD3F@wd?_tMqK2I~ct1BvcnF>`k_X#?eD13|G(YDPEz6oBdi!A{R z9QgIC%OYEIr)k{j$C$=Ur)WiyS8LVlXj9g|hNCX_xDd6N)I!scE!3gaMHXsQ@%S*aw3-z5eLeyVWXI{LrA_?a3&5z9vRouImt0}n;p%- zI{W7?G=o6(SC^(rG>E0I%i_j%I&+8v5kqH9Okng;KU&&9ydljydD8N3rGWuEh!J7b z#Ed8EJ|~LFd+lhk#$Zr`6NMt=0VQv3cq+$+@f#2^44jVnyT}PsMpH5v6euXkBYq z*`vmRZ33_*NQPa?EnqPRLK1uob={UdYy)dUC0>~91vBWC2F+IWK|6J~q-@{Yao8E) z+ugDSosQQkaO!49S}bw$j^65#7hyeE~O zeXT|A8p%{E-X7y%9X-vFAj2CHI3V!E_M=H5nViLAo`e?alC`qp*l5X6ptkcx8ZxdZ zD{`|T!e_0J!3u|I9*=r#n3AGbf-&Amsz$-mnb~>$QU*fS?oT3SjMm^oxwho5w&j7} z6yRlu7pP@u%#M`h)Gw!AA$O)Qq(-8KYK@g`oZM z=#ZP=O$s@Icp-a~SWJ3Up+TEt9tzx2P^EkC7W(FO;sAHg!V^bxG>n*d*fRq|)xyp) z4^M|2%3_)Ks*{bL!OQX%m zGR}L^y0EDo)aJaFO>ZF<^>d4Y>ttuy;n3^2D1NzV!~2VSZR(SLY}k&5!bsD%5X zJvSvy6S-?-81^M++&*l_pcAOG>$0-(@sSlL79FqBm!e78dqE9=ubVl+YTz@VCaaFT zJ+k+mC~I&*uXM8YGq?DImdvuacBt0C>9#L9^O+o22X%862g7IlFtti)Q_T`mSZnQiCSloSE8)S0pcG)VoSo+A66i4Gy@)R(6f_E*BY=EO~ zbv)E07#ua}8+8YNe4Bt`x5kqpwbKS}@yB5ImXI%Pw~Ib)(o<-zE>a-%h4@bAG|X+- zORyQG&LFVD0bNs$>H@&t%;*}io*bXAJbj*O5yl0p@LB~ZUpBQV_fi?lh-F4_^o=CL!3Ut8&5_URGnR=NJnahZ>3D;K!jaZm0&ivXYmOvWZ80 zwGiQEMSB6)p&9ESRj1kNyz+7N|5CPn)qKLHmswju3LOAn#-ZNGV2Y#2S^{bpL4<|l z47eyZia4$vL|~TuekG8*gzdS;u8;3 z?$?5HUbyPaT9!PNQROqE+)ndV)?lP?0!c{YLg**e_C4Qg>2OjNJ8PK-b{9T_@W)`n z>$*?43QwTc`Q7Q1>4cUPh*2v^Udo6`BWsHmvR>e0E-tVZpyx})B%-Lu0f6k`=SNT+fSxWK@kyIPsN z*yq!@qwRGPYFfDPmirEn7YhB^mM5Q1){Z&FyW<;+)PyG9p7+^G;MlCQr1`CIT@z=a z#18q=a5g46yOTx9>ON6;4;2ovtgW03rK@bKl=6Xv2r!U=17!32vz29ixmk?9Wd|1s z;s#Bdbm_wJA=id@_6&m%ZhjR*4#G>ZgAJu@<33q4G%PS{Dw(@duvsrJGN^_S7VrtU zGj(zZFbtPg#5hjNrI%zqvn6IpeK#CV!{#LgR}a9K+@C}efejKCpxT-VL1Tvj3b%ky2n`zS|%9IS!JK$$wy{e4k5 z6@)&uH>$A}vZo5W+>0+hgosZ*4?y$Sj^XZ;>Z&>lP*Hn@eSG3{^nh zWZI`qBY9;IhmTjDZ90Nvzd<2S=49I zdJ@OSn{6h}Ud>*dH=@yZ1wOja>k=NDNwH0-I7gNZ?-gU6T~4dB=3A`na++W@`w8G2 zDtJ*3l6}zjcMG~WI%K5?mQ7DLeR7OtLB69B;E+yo-NlgL7pM}ptYgXI4z$v;t;3;` zexug3jta)E&AW z=xBjdPMN*%8el;^p4*6+4UQDe8jJV^*{8bos6~i@c=~{%HUQJ?A(R|+&TMZKP1LWw zqEuf;KHbLEDTHRWFcn84p7>ZfJ><9bzQu^Nvgc`FkVgqNsnE6^0Z?f92-%wnQ}1lN zP(87pjK;A_`rv?q!8gG15^hU07YHc@&C9dio%BuCm{=lnss+BoyFMJ6wanhqYc`i7 zpS>vE;$!ZexDO7?F%j6)@bJJyW)%!>k7i3bwi?!0`?>6!xX(2~7|e#v7&tO3ZMMRF z?ki&KyIJmKYFkdc@X)x?c2mM^lKx`SeE_$JGO6?ft=Hqfh4^zBZ8}uFi*&v3e7|RcKZ{??TfrBM#$+-Be|S z&$*Di#SO5?u{{kby%M^*1RJk3IVj4VyFjh5K(x~A1cW657`O?9iM zYE&pdHE7gtE5#&3n%!zo8uzKD7*5_e<;RPnn6A{BfXbIjnvSq-daz>95>;84nW>qP zJ*E?b9F`Y0aUL}`$82U+G|t^$F&&*JBXP$UhEptN!Z?-dBME}g*bzcUdxH}?YL;cd zn#+>b$QFwbaS65grPv&sU)GU3YHR97R?A{C9vlu7e3&l65F}Ov?d4<1Hwr~KD7o89 zZSv_WsF`)2bv4GNDvV>;;kE;FA4x^NHk@eJLGJ8kxOGzYsnu@8w^drSjRqKOZlJ3Y z6moNNAA~uBFssJsJ~Nxsj~xjU2EP{sEKB`By#`5AK>`Xq;+HYp@s5W#E0+kV+%-j$ z4W<$4mp;)l0;fna_&tmmW27Tn9OgfHcHcRYk-5=-3)fA`3E|fPJx6v6>Pg>HT;Vsc zV0ay?Uh$^bBtqL15g!|#%*}7a1{Yp2=iPR>R-Mttfa<<{xPMQc)a9wrcCQ0XcT2o} zvWDa3z??4gc3N0o#b`Db-P(H{{CGm(b;0Qh;a{AE_^zj;U!2JELZV=@(uI>rxd>|c8uFfJHAmqa@l8q+BN5xVuJN|1BQoc(+H$vCiN`g< z-e5$E8y1ky25w9l)tHskc3t*^hn2C*xg<#xH<7h>UshgJ880if5Qa(}nmMk(NM2}AR*IJb z5-Z#%)CSeBsjNm2Sbbfx#~AW%EF~nEcHS(y9Rzpb=*AiQ1|(!zO?ZBed@3D|E-H7M zhVFcIL>R`Uv1#3ag7wzF(3Y1e8>zNFBx3ba+-Hsaj);mb_^<<{>Fc zV&Bel?M}4RETP`iZlcSZrz_Y37LAnOj@TL0#KelCfWGncg89 zhK-6HK;j-GbFQ3-I-C&giMJJZ5H1yj1z7Q$OPG7+E9!4hS*w&l7$$FFu=B1Q-5bzIVF85lpu0rSm2(*H99b9!gbH~z@_2!x2+3VZE}Cmo z9{id^ZIbs=#u_@M-4Hzph&GN)U{{w@d!sx&fsUI9l=u;BQVPC-@>)ZXW@HyD?!gLZns#Sj;!07(Dt&uD@DqJ6aGQzb%Eo}rx?z!|7z+ERzBV`$1aee?;Oo8h29#Zf?r)eRIb+0# z>NemeSBMzmItaZ{pz~+XXyA;jWTP>K%7VKQOlpg>bOzWDP~jP2lC_~(WDr$% zIkJ~j+|*M3oLKPpGxv|15_ql6QEn$Dq!bHodlcAwTYoHhQ-qT**`%b{MeSf zYS^VNC%5X?DsJ>L&S5`(QzHsHHB?B_`)oL z*fPW}w4ouo({Gg|W33s5Y>F_c0)Sw|yy<>mKXi1bMjMOc^O6>@lC#5DYHjXkgh-#S zhkm0Gt`|JB#Br@w9$;N`-zn6_wCp8@y=y3|j|?)Pmz)*`5;fw$Bk)qH4{ry1t$fGM zd=yV~i5=CPoQZrl+|hRA(#0pv{iA5i8kcY!X?D?YJ~SOo;;D+CCA{2H;StI-Bqx{Z zzBfZ=9(iSR9hvR|?FM9Jqj1z-7W~gO<9&}l?)--Qca#b4u$o!&zcK)blTmXAHn_U@ zR6yIek}hu|w}R>w)vYrQ4r`W(tOUA4lEN;7aL*;86>dzfo5TTB)qHrUASGqU6^3De zD=&EP^sBTH<|wk09PmdRKWN$>?;TNASGhJg&1`hDyy`Dq!YxhYkT}Rq^PX6WmaYS^Ieg;vZvnmKpKDiK z(;No(O~TiS12>lxm|pzv`7!A$Hf-efEsxjet)=m%0^K?6*xrSErHe@ep!R}Z{02n& z0>Wv57%}5$zY~(!!pzI>w^9al&ib4&mq)W9X{9XD3Uu3LjUZ=|3NX`0DsBd+Mso?c z7Q#4@#yXtEH!VnZ8Tp2DvZ-@w*fr3=_7{caCfo)20P*23KJ^%Zk1YhL3tuQ>vZ#vj zZng1Ad#vdBY^^YApxeU0e6IN48W@-(JXNP=PJ_|4ff z6ig7jJoS^dX>I2|i`@l|6oUqu9kwxGm8FCuHd^kS6YJjt!;zuK80Kfy4PuY(xSxep zs)~Y-3|SlPbLU}QM0OR)gC8t;H5p1dS1d^4X8q+LwtgcC9>kvld4g8ikS?*iixId_W4%;K%xZ-i01m?7HDNG7p}X$WwN5-Rpo2rEL? zyu4Ed={OVbR7lGzC%;foaHTYWc9Eah_$=%)fG?I-jiIvwGbrGrAvWP<=C&YNW*46F zUTp!n;L^~H z2Zg65!JW+qa%a(ZW`n?0d_H-eOL>Jr9w>hKP1rXMM{i`oSg@MOhndHd)LBF#@;yIX zW5dYMO~HzILypjcAn(cEfKGTs*}Z|=h+ZYQ#tt*M#$Flb;%*rR#<$NCdD3FIG%v)8 zy0yu33YsCdctN@Z6p;BnA47?crBp1h9-AtxH`j2BYR7G-A|_TGD#bAY+C^MiF`kS; z!x%ZAKPT7cLKTNDqnK(K@5>`d)pB&<}gJHppuSJJZ=L}wz<~>;GvDnHMM6B${pvY z1V2lWt6>}2Q;P}{6OfITCVy#Ky1-NQ$b{hFJtS_FBg7QyO8E^^&csBxf(oqw#5=3O zx=n%=muy6Z_C{}eDM)lBHD7KMM{{d_H`W0u{e9y5hEARspgj4I5Ouh@S}$?x`s>xy zm~Ha8Yvf64iHFreUFcOLj52DVb}Je}5XDY2J1Z5$P6aV9vHCQH2I*tr`W$u6LXOL; znK~l|B)81PG>1N&`#8v@O4xQ$h z;5;zw!$6vj%cX|)#PE?pWLq-Y-`nMM0}`Z#c!=e>#0o`bXqymW`Tl5eF82B8?}V&Z z0YvHH1QhAOgsy|(N*(PfaH!Hq$XlpHJ-Xc0-gyN&XbVw?>YHCtaGKm7UZqhFs!{sGxRw~}oyUr7YcJ3S z!;Tz+G+K!0r{IBDyM(sEe-?FFccI$^1n73sB%IUQA65cZf@X#cT{W#_n)|20>)Z}Th>?0{}Ci}cTiR45Y#0uY~DYDE4CQb1&(`}d!Ej)az z6cfm3E_3R6J+WJMj*^-%noo(!R@{MHs4q=ant^ z_lbBLEbF%*%FBn+;p!}~*ai8~dWaf*cTPhiOr)iTSa5+2!GR;V=*Jr-XbY>P3ZuZI zO2*i$7Z;GqM%Ibi=XR$+H4y+gNvYM%QCl!u7tXbrXEf2B*{XK)aikNV9MrQZ7D_l+2YSLpSLy>IMu_V4lTX9N-i^K0~yMoZ*JS(F)$q?wJ9^ zx$sXp!q_;6v=H$Qxa_$-rJZcdnpyTFskP(H1Cp!8e<>X)RBq%Bf&^*m-8nCkbii{O zAO&{t!tYi%#7)7s+`t;^=pw_9g0O#|EF5fxYHEYtn#$cw-xg|7Zo=qZ2Y3PthZXq?M@u^*|2|9P7+j(xY!=zIy$**^Oig0V) zl+M1?0}AxQjIeJ^ZOzMzM(*{BtOL-rTRQN16|i0g2hba`?0GAh9k+)g%i=}&Y&in! zuI9=#_~;uE9LBKOX+65n4$Z_`&GKIC4Dr>J-si`@Y7?h#72zQ;b-n7PvG7t6n0!N-AUu3AXKZH(#EF-8bB_dV#Hcf~r{ylK zlv4KIxSFqKO&eL8-K5`$9dR{Z7g-s#le2Y7Uoybe^+5VxeSZ-6+0<`+!B?rmDQZ!~ z&|e#G7Bd_Um{@nPpVslJ6-EHMbR@D5Y7Z%0-Gkz>4w<=;cokSBF}p}$y=Y4kf+t#r zu%9!?Kn+wDib&wg#i(~Zyh}1T{j^e|hL}qexqIkS+H)R^ZUc=zkuNZiN>%165v0jp z6)_2#!Kf{#qap|bpA;(KG%qEhcQ`2I91$==1w=!l1G$oDhl0d>Y#GHwygY|Njxyfb z1<6*6z7qwMq0*7CDZ?vs!y7sKt=9V%X2AsRfY;E$g9YK$48DLU z1VLs2=oSbuM?KP9dJEl3 zx#snngrV4Ta2mF>%+H!AZ{zY3j~0b6@CxZgj6&Z~yDI!f$76;K>w@5$-P|B>hq)zb z`Wd{W`j{S3#yFl3_Z>=|wv#`T6tuSnHpO-jv{qIG`&LBmr=#qehZ8aKBs z9=Pf0xF|hRGT>}qaqzr-y&iui(R`Y%Sx5TS#4)f&)%5a7M8tOF*F*?JZ4z&dNT@;Z zOm^Roy(~=VfgMZp9nf2ni(Y~v8VTgDGWDm9)((s{&QzkGgEenp2|%J7v6*y>j}vU5 zt{iORa7b7VR70)B7NMZB{x$W;^7+P!tTWIOouTYivyf!g2wt=haZsGoF2XKE3f4Is zfFKZ{^>p5@X4Gy^t^AL=8U3MbpefU4_o03&QDoBs)0LTi!~8x5#fmd4sg~(FcjP zp9`Q?N2R+DV3~dG)b|?M#9qFMVf^8a9uBVNx?Ck_NKldN4b|T&X`i2_*Lx?;#z}L` zJ;&vF(t09akKr0XVjtF_5Rf9@*!^NRBxP(DlhzT~v}w=i4<I5GH$pF~}Y$E#KZu$FM(h5>~ zd$XMaK#x;9q_@;cI|w``bP`uJD=>>!&Fh;29eBW;aq|3+NV-ir>3V>~h!x|w3A%0M zg~-ruvYwB7p;_{5y=hLPb-ctO$G&!7Xxo!o?N4^Y!+S$*`P7|bs~ZmtGn{kdS0E}K z>@F}wY&95zF%gO&-J91SORVZ9BcQ46N-YI?rONww=x4>ou)}5anbR17_!(E z0*t3l0t?G>;f%G2HdT%(#*kNQ_0d^k5T7d?PHNj!k7C&PnjGd=5f&GP0^%rui)Mbi zN*0_&Pf_9LpNq+ZEG=FcOlHnlgluje^n6Gj`x@@+t*1N?hwfo{?-zL=86%rSLe%kX zG!oZVhGC1e&8;DFB>Q#WV6%bs+yV?7g`o+%ziKXV^zRro`is$Zb@Y@ zy7su=BB^l{L!?$ovw>uiM9%iDfsiwZV5NG5SOZC>~9&Oh5#^ul3b(cNF*!2F-WKBjEakfKHpc%{Vq|%a`{wAm_OuX0uDU zf?SO~?wC|A(ge}P`T{63&+jG6z1)l3@Fs~o<1`84<7ZO^(pJ^Se;vIsVP}?Z-T3>9 zV$906tMS~Tf%bzHAV;l&vJ|4Z*iCOPS<9NV2L=PzDo89Nt=e%tK9E+(fF^?>(%&XT zC3x-Dm*rvLDL}A~cU5XJu)K4|{Ld$l1${A?hT+l{W+8Y@)0G5*V@EJ3F&koyV3wETecVw9c;8gHW!R11`6cJNve->l(cdebKmVsiYl8L^?)# zh0pD)aEq#>APcslQ%6ogGQ_t63|1~Wz}PcVi(|wC?Kgau;x<1`*u-4$JFgz#Ix#r7 z>mqB_uTjHbWskIiR-gzOflXx`2Z7xx8Ow*_B{}BO1en_wjR7p2b`*?gdyxnNq4GQ< zVS_8#c{N1Sq!Q1CaB<>S&IC&TN-c%7rb0hmp zy>Z=fH4DChph?ME@fCJln9;ASK?@6w$KE2DlXR zD8v)t99D0cc&B^mxj;>r)xGwKx><`k24;*v&}blK`AF{fIcv8wYA2~T-g&;dLeq55 zb33-PORtaSfT$?x`(i91ZY#&zTJujoOo{s4Py&*vB9hjI5mu1&R zfuSXd0zIV8kry#0k{%5c@Q7NJVCcDvzh=wU13E0q109Q4dmL+!STvBmhGx@A*dkLC zR1I@>zBf`4Dw^kQG6K+Fi53GiE!a#Oa6W$~9`&`o?CRV7m}kIB;=z$F}tKuaS^O#;+|bxg+PWSiWZ2%-xpVZJiFxt};0 zZD_54G57NxrE=3 zB3UF0c7bcVV|*LGGjwwmhk+xLlM!=LGg${?(H{2f4ao`e>F~L}((A7Wj*WJnBV^ex3FK=*)z?0u-c-TO6gC`yev>*=*M4s~cnH!?HGV#RKSfXl2% zaUU<&gAKTFkF-3G53XN;957%nfp|xdc zxO6~BaXssQpcoV>Wld!yfHu%+M_VWO4dc3l_MPc#R#pbBFjSEG<^Tjtdc0kZHA8Cka`rb_`G zFo5-(gZnLKjH9}3k0gyF5U^!h2tD56>jrv4x`4$Va&kcvYQ>ky;U3lkJMKnx;N>e% zH4t5~hPvKn+!<=FN{iLm?lUFlj8*2a%eCfZ5_w7{5X!k$sdZ}DXnq2=J*C~a2!OOg^=(BV!_w7s z?pj&!XbRHpU}5HvDdXM8FQAaVqB(Fi9lGd!JL@&@rbR@xC;=pi4oS~=rQd#-$%b1d zx6V$f0HlCSnNCI+gjF8WtEA-5 zofV)CF7Hb55V|}AHSH5+D;lK|Y{v7U79}0M-hm}b7IwL~>|^9+O1yt2wvJW)_GsNJiP zCSb1${d~|jeD@$Ne#*zwLgcKj@`R6@P+uH@T^K+t_^raY?PJBSgm5u7{wvrftQjO{cP=j#b;_ zKVQ1@ZHXEITCh|YW@icAh!9-Oi*v#g**nda{sbl+a-{n-$ZJKSl-LEPE|w zXJ5JCAo&Tia^y7l+E1Az;BNrU3L@cYK&0U2&J|D=CQ!1)mtthq>oMFIN_zVgG+;?P zp3!RNZ`tUi^RWF~i*)dSuVGS$P%8It3MoWn>M`aO@pY87q&|2shV5xrWtz;PeCh~g zY@Xji7V$)?ogvq?tQM5%1E!khJ3-jcAfB~A!gVP7Dm_mxdRczbmm2;&ThrMTp z14U_gylCYDr=FWx#~`K6gHxi@);y}tDG|gXPnigHTfRBcp>G8&jxHQtJK1T^!BmD~3cP%WG6N*jgs1>3<|NMb6o z5T0Ntkd=-2`26ggIM20i%f4-DvM1v6>+x<2aXV%a6n zTycmU6AJgEhvw2*0s)e^C}OImHJ{BOJok^rMKbbT)rqra)^{KS3E=`3 zRmusPrXJ;FcytRWCJjeZ?<&sRE>LL(_ZlYa`z5EVsbu$Epv*_c91sFh(mS#RsXQ3L z#?4E%=xd?xqqE(aZhFxjJ|W%(dz%Gul)|`fg=Og+g%eGi6$S%v-;ydC`W(Lf#RHw= zL}r$B!w61ilLWR_)O*~?9#PxFYZfnWitAaaRoA%vUi4qHi{sdR7V?9_5Vwp!Mx35z+!rqBqaU}zeD&BuXuS(dsCCtQ*nlxrQgaZ^8nI4kJ!Vc@7s27ksdyj() zV{gu<&j@Z&Grn(Dk}^)Zhq*2{Qm2?YVe>naxwq58LVc)x(N2MT0py{5sY>k#L)XP> zj+odS1^+xQKPW-I?d3zbM0TGUyB+HR>-wUCr3PWWVYIl*i}PfgmG>fUy?)PSg?^%! z&LBMYa7VhLp^j9jjO;o)JT2z|&wyF+cDGm?>0220Ysm4VM-mfZ!kD?DTo(8jIc2H@ z#>_@G{R)i3Mbg${wZ^Wh?SUelYNP2{6{QN|0bW7Vy7z|RY39~`F&8AU5%d-`ZLc&% zxXhG=o2L01bt9GzY%mVB$g~Y*n#t%)OmGUt9@f^cn5YdnF+DTEU)DnB)I+7jI$N&sBA`K;mDuOY40oQJ6=o0Fap zu4f_ye0BhWI9hHkEp;VM^(1K)7M%;2C+*xOFg~SsZ-#=6UXv*7-4b6(#{kooq{q_` zkgOx$N*Ccuwxw^XibjZArZ9ExUmu!{H)Y2u8jjY=Lv~33cuSTG%WEu_*ec5NJb<;u z;vue|bx8n7&}?!h35MuaSbE))rH?Ks+r96Y~uT(^DCf(z7#`T^Mv5o5x_qV|}+a5>v3>hX?q%6Iu z5?hW2^ZQQa(z^E=EHbL55X^yx0r;gyGca*dsb4mP99SxPSRWmkSqqxC*_C7&lQ%k- zes;MCtp+aaC_=0S92pH*+a9wqzm}bA&XZ$Xgb>XSo#W$%Y}71icbKqn$0O{nvSVO} zP)%nQ9(+~ONZO>kt>0_ZIO7WsO9G7yg6%{S@xaLyyXHl(+~JHcR<@~Wc)JetcoJCP z(*5ry&ONi2GGBpdMS0Vs5mUj#-iO=>lKVQZDtUK&)(P!~v-Tv=Si6Dt@ERvSC=F|{ zrP9rSsS9>tD&@yb7%RJgmP-evG15{%rpf6$irSz~tEB+o#o(Yag|Zg;oLV!xXs*bH zwP4G-(mWfz4opQ>UompM4bk@K&s>vF5ZnaMXQO ztCJM$242;|4jPo9m8ijPwSF$E4UEqaUdzwXVC%2jTie1IJ(diP-Yp4|Q1l_3tlUsA z;-bZKg(jkDG#!%OS>YxGt$?r>ZK4Q2%*ZvpP#$YMv&}Bwaf~XOTOO!7!_jyPohsp+ zPt(g1X{X+jK;tRn>y)0icUT%$$q|;M&9y^}N!AXchaAqDMQRCLreB@@aC6|@;2cb+aeWceiCJ}_v?2#FLFo?s8 zR%jtMn%*amBXW)t%^RIU=(k_3*eMF7PU8Dy%0e-WNS@}_GLm}ic16BNU%8Uc!|so% z=x`vTlBy2zE%!_gBq{|~FL3I%%CMHS&E?6I7P0gMJbAda1h!`SFMOo`R)ZzP;HJUj zWW60WqUB++`UFb@PA{;XzAoK{busXP3&GS~>lsFB!Ep&pagJ#z&EVVuZikLh80LwE zrL!O?6R72#0>lVXAFg)+E<|L|goPC&te9>XW-<4GMQFy`&W)!1x4eXhrA&$o9Yw{k z+*o|+(~w3=ebfs7O~!SE$ZLaH;te29E0JxUhl0Q|9H;A}t-BWVfU|){YZA^7XaX}B z9l#313dBJDf|G(gRX! z9LeOOwIadrT^tOtcu-^w#}3S51@>Kgu7ipjyWB7bR#)nY;36wmw8oyqK2Lc=PRsys zVMA}z%b}c$-)c!8H7=7V#S6VP+X+^JpLb+V^iopFX%(_0x&1PJny`@EL| z5=TMWq|MiQs3k}G8|B=is46UBeWbH(4+EpB$yAkGU0%+fPwu6i zFs>8mCBJRWVZv`Q?!Y{K5*a&44q={@Qt zs8dyxjO2;GWdbDodHndcYho`4XaSEm8mxPKChvwjPsggk@! zM@8-uJs#J7>1GSMF)AgHJKhAv%MP(*5IlCBT5x2-eDW%4jnG5rc|;a`!&*{P(Cr{O zSy(kj%c-(CV~uLW1jv9wO5tb8qVw#!v#{wx5;4N;XNobH zX3v`1O}m*^K>1cnd{OR5&oobVG>oYqqhz*daa_azk!bc|YRb&cpy!c`mFkej;i~V9 zVvQQ;9|VB5EC|?uZ&z9CjjDxy+_H8^0)7F*e{Lf{M;``U5MnDtL=U-S7AjLT4Ed7C zHXw3ye|aC!M}%pds&42w8T)rtG&&0~TB1_~&BQNUZ$F$2LZuTSc>qMEsiA+V2$b4kSY3zlTLt?&AF&Gkz?mRW6Zb9HML3uTE*a@4O zz4?-;=MWz5=divgi%mS|Fz_y6rzr*p1%%BNS|d_H&VfE)o7QET9LmFmVAvLl8Db>_ zd~Gu@#306&d${)vuWJkvNNsYCWV<)IPfE_FXza7XT<9M;@~y~+y5mb{51a0B%0$#2Fgj;Fv1PU}h`KE(}!+Kw0VEK~0)I4Ds%C)?- zqjqtCP9ptU6!u+nUwWVj`K-b1Cv$4j7m?O9B2G;BVk)18ki1^PKY&z^F@jO{_4DR{ zeDZT0B<6&~hc)lkBh< z4D30Bs9U}HfMYd|+$AFch&mn{*v){tP5P3U!S~8f`85 zHTyXU8R78(l*J*6t}s|7HfDa?1Q?v`=&MT(m)&L%3nM=Ex}zkH0N~G>N4MN*859Xu zQ2ioK#K{qOGp>QE$epL-3fX2^~54m!r!wVKrCISk4heseEvqBb9E_L|3=r z-sovF;vmClwp6IE5+swa*pL{@CG;VCv{GYdmokB4!py+%PhU^e6#OH}lG*ij>}3no z*z4Fsk+qfIdVx0@lc?6VVAOpZRWH-&snSG{rs>=E1EytLrBirJgb^-s*kSA@QdSMe zVvpWIj{~J^-keBv6&^bv{5~ndQM$_0s1gOoEZXD3C*bSK2^2K2=tzDitRfsIA*35c6~bz%ZLsxx!O8 zC|J0tMl=g=XhcP}R)N@oRBiL+bP1Yp=Q?Kus^ITPAvtc3>`U6?h)Hw&BF3 z+l6cZXU#K5^5S}*-6nmvSoaFBx=*aU=w?-j5wsyEx&Z)eQ+CzEyC`ctCkWRMelw6E zB`67MYly>amKl6mu?qXb{>bem3dWJoRR}I)PZ?xXY z8&%{@eEL%=`lE9&mN1b0V)rI$PS6BlH?wsBILq5zP#)MrB`jU7HB>AO0gzBOMIhE% zOHwSpQ;b)V z&s&$gCJlB72}Wk>6xmf2SYe5{mtO?oNpX#^Sfh=3H^_~7-O)}^#0@s2>V@VzLQ7Cr z)Z*#8Bf3+&Ai}+r8*9p#6O>pG^a2K`B&GyBs5zX9%ME3j@Cn;IYtgGsH|x7@vV-Z_ z^U*5DQ!-rmmfw%$E5c>lMI`uo)cG%@UJ7jy3WvGP&dcJlmI>6A4n>x;APz2AM& zy%%UuMq_JBpR~Op?uRbS)ubi^AwP7jI{@*dn%3W9LHpa7(6t}}8oRU3rHNlo>uJ(X zz1Bs^(3Kb0**ii77l@>@*xh^zr=;j919u9}9+ZT+?uHf zRqFyPg_WN>i4w~7PuQWzs?V?+?q!Y}pi!g0An7O%C)QS)YwuLdH3oWgVkQn6DYmbXpnA=*$ zv=Qnv%4#u361>YPRq8AEU^lcWttPQYLz0r}U9z40NJF=C0;4+fYT!Ga8Vtz&e!I+ww%~O{g->*LVQUgo;ymL*9R&UY)0bonr`6HB z7d4UTB*!iS*V``~J#l8vD_MpFZ;Pddr<%^}5t`Gg=v5L2*8sF_ER^yVbss#QO|D;+ zQg(PqIehA;Im>gDy`02xkz8vm6GneMTh?3%>m;Hj1n$gHDPEexdQX&>q8^xxF2rmn zvlyxZc8eQm^xmgh`Ml+##HqqdyJXulM6y`8!0^_Y*R->mI{^dCu~%LN?6ftkk02pbV4d^%pN zh?2zi8uo%f(o-}R*t#@LF4eq+co-b1)j{*;j)Rj6`KvRj0Z?}7iqgIH({y?(Q(t0) zd|4CB168^V;RzTaoZ}BdOVD9Aopq}OT!MSgCpCdj0KGm8D`XMW=j?U7z#VdJHB5?v zcQy3XUbd~I$Et!SEjdUPjMyATK-R>IQ{rYY!2s*jNW}&{GPdn*H+%DjCdHHZw|4dJ zNfjPKmk1b~j_ntCqQYJF-5_!gRpxLMvMhLmHXp;U)Yg~?)!2Hfap?(J!7Z^_czvg| zjfc_uUUKy7ToGBWc!S1hiHLt_ls(?ZLfUIoiOs2Cbq`t`E2gca>Q@1we27zJ;duE<=J@_WekGB za%Rz<%q5R;ZcuTTqs`Ep=Bq1SL%FqNm`+#rRNuH1g)-xIu(Z07ltSYKNpd1x-No8D!{pYt!TbfLKD2;jtx}tjfO}DwW#s4hayFKA}ZByXCwV7QuF15OQgoI{uwx zffg+=X|+amG+o6?=pqYEE8*YD8TARXI!+ zwnBt3u|d9Tx@eopTkgM0CfAz#&yu_oa?^knD&7^Gi!qbbA05~*u~ikzeG)1b7vh_7 zoCykiWDwHXxAK!rmkL;EZXyyjIdIsFDz=2GcM_WOO2^*^DIsWxiMY?>p^?;WN?Tzl zoMkp{AV#mE5157X2JR1xF!YR>&W<=TL4uZIGZBNB^|D=3jyUuxcd_JKo;n&F=6Xs< z79}dAPAYV4$-sT<+j-I$eNcg~ee^9`kGCzMok0BhtqOgmCoD$Ze6|_johO$e7YS3a zx9WO0lwi(Qq~{19q3=LLX-2YcsH$;E{7`PS^$jQAiK7)5b6@lgMdd&N^m%B?hEVqLM2b51=-JP0AbjI z#TVOlJ=_WJAa)9Pxg}WDeKkF4sk%HA649zHyTkW}OIo}!A-wU+o`%}-rUYzZS}?(I zF!wln+Jq3ASZ*G*D8?=*679o#+lH4X06{>$zqK$%@o8@ z+3xU&VBTmb!6X1?V9D3I&%@#RFD)yF(dsLLp>yqvEz40DYN@bXQs}QOxqFL!#$-#Q zZ7Xl#5jv|w>uRo{Pl^biOOJq8 zPi!2$vLfwLsU9x9tMPQ18d(BAB%{TvHFuNGfWJ_x(CcZ3J6;LeD9HHEh+b_Syr_lI zdBLo39~;7q1$IG*YbAL#@cs>`e7LA7i`i&pLr$WbVI+#Y$9pA6oF0!6Ue) zM8Kfg6<}piolvyJ*wp5ta9}d<&AF*OO0INnmW3EmmB-re)#kCk2K6L%p*kGRcb1qW zHtg-TUUv14acHg(lS8Z)5PfE9BG@l_-hlyvWIMg}W>7J`Er7}`qu}%5)V|n`wu8oG z-icL+d$_FPn!+?AOen*ZE0u$go1-Vad(biE){U?QUxzkq|+aIRLAv{5y?#KVR16Eb08uia};&mdSK=F!QS9_S?D zaD|pjW*7F<}F{r|OeDj`(lVn!znvm#8j^f52s3W&95tHPuE{*c%yd!%ImKFpXEDt9D z;~EC_3thAI?lF)%!g5mJ^#jpH?_rEJq0G!Plv0&Wc}=Li3DDbef+CMfN}>2!+t>MS1aRef#I6(_$I>7}nD>yhR_PH+wxxt&&gb4br!IF^|U9_a6-p8QY z@3Pz_KzK$2#!X}_qV)Yx4Ex}fiU#(SFKgLw`If3a=ezRckuqM1SgT%?_cHj;W1;gB zhznrK0$4k?9CkQIVZAg{XLn`RxWVD)+v8IOW*)AKT!b15)9r3OHy0S9J~Q!c43l-w zoh}d}08u?;&|hB&focq~SES<7FdKP%q#<0^9~QDQ#csXc;gFHwh_?lB(E7LBCCxVH zZWFr<_sW@w1<7~bXC+);P5S}Ga?-uU-U`1JI0}mwU4*tJt)phral@%GjTFpIk&N_g zaiMoi+kz;YsLPL^(hkrdLt!Gq6wV}Mo!zAvuvBdYA$(n+#R9b?!#b1OKEUK_R};bl zbmTy3E{IQ~Gp$T9iBhSdMN}<63dV^y*NxZ zqy!;fL;+rAD%o5nsvuKyc=ep9nOLeS5EPdq>5Jf>f-NLi;fAzw0UwJ*UOmz1&DeU+J)+(T} zx}4{F$Bqu$Z4-a|5=G*>`k!+hQSR zioFi(92_ygRz0|(cjeE7@_N`VW9RBmnK)_z?HLl z5O?Hz8+vwfV37t~9&)Oip02DoVl_ercR`!lO0fqque|hSN4tRtJAqe+?h?TiB9wLJ z0L`!^pch*av^~#?+d&|b99RXo%hU-KqQq+f2UySNrr_i*GZI2amG_NbC3OO{LYd{~f(!6p2wg}V)@4_~w7f*QND*NG_U7tx7DUlgGkC}j z!b_FYFHXKcyN(_fHBh91JjWe_>>Aa^)3Q>_>IF_ySyB>m`9gHF4*kR^-9tR40@d0F z-I;ASltVPR%AksEXY20O5|ei{z(TObyAjJ-==3bVJ+o#fx4_tj`4;uBtdUpFgGx7S zA(yn~i3>|C9f9k!(9VV0HkV6P^Mfh3l>l37{p^T8~3a}`i! z?W_i}OoPpWA;;GdngvFcCF|BX?|3-Gn!s_in^-JmY$oU5aMTD+vHJ`W-)*Rxs;azH z#V$EM5N_o#hp~IU1WWV5!&!2lygiX?T{}&^jt``G_wPlTaA9zSsO)Q|yU9U{t6)fq znoROH3lar+U0ou?tVEXvv^XvziBQ&bb=wpblx*@gS6&}+;7FExc+Ut z7xfU;;o7lH)|&ClS?wxcGX`c)qxM_0l2?HKRHBre_e9ceDz8ym_*yt? zHm|oYdCv@3Z740nU?Kk*S&}f1YR;S?(@p< zxeITQ2Hc{y(3LM=w|eFG+WaqmiQ@kSXBdrSaz{I67~H zYJ8pI*{mIa-zOe7%iknX4b`?!m)c+2oSzQlJOOy{(jt0&Iu92v^-dvsM%4mfIkkF+ zGM}skC>~3|ydL2cdc$9e&XH(cJIL`q8qhtUwo`Ory0r`mKUgR`$1-s(uH`!xrs~@&=O=volu;#ibS{ z_Kdo&l{^F%VNw^PajxGdld8~GD6=aT$RgV^sDdBf<_QUxwQSHq_3Oe)}pH zL)z$758?N*^1PPoCy)hCPHtUz{V4s*%_$y9zEVMPwm#Qkqa6!<7nQ9VA*>}cTZOjtnb z)zl0HdOXTN$L-j#N{$)Yj64Os=|pZ}L|b;vhJppKSU(ugCvNH;c zfZ_`ate1gHMfRm9XH{}?<2f%?gbo)+dpPo`G>9M&)dNz^3D(UNw-2)M z)3Mmo3$ck4!x1TrD`onEFSi%QaeyG@MFt*mD;&@6wHe6g1Ur9PTnYYl#o@9n-`oOj``@+{q1Kz7?KUw zUDYuffWdI51+A8E64XrQ_~m-$w4n0o%rg(#V5V|yE>IVKQCd(8-)3ECG3{@RtK+dfZ6(0sE6I4KkAmy*5`p0FJ_mE2^3 z^Uq{CQkEHdsbNYr2Dwhhc1su@XN(*`2FHtWdITI1a1MK9=F&dB2~%Bf=R1x*mX3xUjV}s24|{Qp3uJ7PVN1jVwT%+}+1| z3g-0`gF^J@pA}njPi`R7txl0_ahU@=;+UsRQ|V||+XDOF3dCh+2usO@fiw@E#ic|vqph99Lh?vP=;2}$tjfK`t)bLAk!*%LuSYiLUBd%wGV;3 zV&&ap7!8bcUKu2c4aqsEi9rxk(gi6wz=fGqH|!a^D$sl8P<$QkqIu1X1VY2|zJ*xm zhevoN?{(Q<26?x!W{Mj&FmcU5S(LpD!)XHO3IzV0nFvXxa1r;WrtXoID99CV!cPr7 zUcQAUQpT^jpQ&TgpZnkUTz{Hfx$xz5_%QWsZeCqvK zXyT4|duM~Fo8&SdH?bJ0tn5U(CTl>pEx9r%;HXGt>EON(Gx+rAvcm-Jy=+y3@W&N8 z*e2zjLVS7@`e?59VS?*QiL7RW4eaLzAB0xl-F%NC^vKEq4|pa>e$x3VTqYx9kHOs( z+=fplxXt*Iu=!CVo@Km)xwugmMM7g^c9I-WLvV%M1#W{Tkv6ETcMKk~d4<(govlo> z0at=;*mn^*ReiWg0ea%MfY@DBp($g@@4~yjen?PO8AM zr&~e`u`77DH_BS}ph@tC(Uws=bt^jPvRmfz7LuXr#U4rF0U?k?KHs{RhtZmMR$K0J z&3U!P?<7pq&v;}@yy};xqrq1qCCc1z*K!VMo(#WdQO}H+-z{yJS~Y4g<#{ghF|*!D z8Byl9nIQ=c@Ce&x4hZH>%GJ89hVO-cN#Dq*tP{)F88`Gu&SUa?1+jIV-6TwoM&U!fVGi#$)Rb<3$4MvG3bMQ zglXfwJa`36)=A}OIaT_wG*a-1^QbE2m1}Ic4NYNGDPu1Eo{mGNwpYJ^*|{c~8-smg zE~KcrI_~^89Y&^t0pt#HaaY`cYsK`^@@W}rh6bw4mEc8(HVC^+Wi5aR;hpITQ^mhV zNqCZwz*YIg<_Pw0tz|^;7Scz_V1~~?5zXGwGzqfH-l#649=DUxAqwoy5^0LYv|nc1 zL(>wJpCtkP6%mL{8zY9E@PN?;A1g|{n}K-{X%lR6=IhOrpvUI+ub=il$oW^ zQM~DbAp>JuXyQau$Ztj5Kz{czY$fASRgOS!*MU|9s6Pz64r3tL&@_VCKJPNka?pB7 zMSqE0U9AOJ4xvp#y{_(xpd4(s&1HPzP&w4<`SKT6>bHIZAUeuYpxH-gr{#&gHaGDE zXAX#i>t`x)x&nsq9AJeXL=}OMuOX<06%xh5mD!0U1^MJmB|{ERjwp8>CZ(-Zx}6WL zOiF%-b&}R2^UUv007TxUwGsiHUReR%hYt&HK-$HtXFK`=9RgNWtxF}@{pe+<3_Y%K z6;!e3eu?;MEzvjZy%=7=_@|38Ek#Mqe$onyG6NJ`v#!j&SxX0;UZE)QoRvXzUq>Ey zX~~vpXB9(YiAh>o9<99dJ6d)IT`bcU_Q4qvK>-GA&@X3JQ_5?!v^Qu&&_T_y-ZJ}T zp9~m$Ab#Tt-V-b77v03aV;~QK4v^ky@|TwbK737+M&j>F7M_+2o*jwm>+0Am#Mnrk zwv8lE--=h;%nICc_TZDW*6u2PtWe&jKt)9@Z?}EFSha|C?e!KLb?70&F&iQ~&1He{o z)dCUL3(a1EqR$l$twC8=V5l#>{Nw$Lw~dW?o4&<4r6b1nctj{x|O)TkY- zNi=dj&xl;c(;HWc_$SX)8U+Gm9pKk|;?$|@rQ|B^^`SNfZbo*E-QIy0%nO%z(P>w{ z+727u2V3WUGLo`C8Pbuzfu5H|ffX&fL zix+0z@|qjVZL|aSf(nv-vALRRGKSvau5ht}6C0W2>tXgY;3^9hqPD1Sb0?YAJn;dB z9T-rPt(gjl2A*6%@UY)cC=H0Odf!PoR_4p+0x>Zc5G>``6vT$azCu}^sFAs5zpfk8 zODn?mbBi*vGLSRn%JYU&F~!_jowVTPft7%jO7)C)oanPyu2B)xJ zr5xPt0M9_=@$SzTuhQCAjAKb})fTjjnqv%n%(qxkGOK8?QgP`lzA#rZ_SSq7Gur2J z<$RLO&1%SvNouboI&hmm4yi1hPBq1hpeU{Csv5RaNHyBS!NuAuEqQ6!d}xZf6`Ibg zgCpmyzc!f~a@|Xjw}@L*IRo0de&HFHedE-%y?4{@WOuINWB~+{lW7Es3-tA@!2;W0 z=@$vncoL*NECUzo+BVTG^7#j@qRBTXpmn<&SQt8;<6iQHofl)nvrwga; zj_W4}Ah$%6=#~)}aH{)U?Evgf4+aA*QzKL5ktyIHUmQO~#4WZ}xQ(qaQJ3_~$t%X6#BBv!*8{W8{XiDSG zy`)>()m{cB3vOBIU|^OM2+0Hl1?CG(0*lImv3p3cibfswCCwO*c6r+jLU<_#OhK|6 z7Vf$l2e{0X>qJ3n<#cYq`7lUb*HLaqAlBnfOODQ&!-3c6!494_!m;+GT>QwecBO*O zV+8l2BML*WED~%Mfi9SrkFira%i=UDKA9(x)*_dnIx1WZfRKAZsJBES?IUP`aZn68 zF&{$~VhTm}yP0jsPX^*h^_0zOuUpz{bR&Af&oFTW76D`A!h4u-pp%oP<1dU(Sgu) ztw`tat;^kE*Zt~MTODg{lvm?UrskXHth&Gl>{6x zIB-^4tAuHYl3q4wRo0TBT&^&#{S&}#1CL5Ccf8HH=DC$WkVrJ3;VDloYlnjwAbRoSmC@Cnz` z9HeMW=u1{)Htw|e3);Q~^`cNnAE-I?uD63Fr+|bV55dM8fH56OV7m6cRkUX!(5y`5 z-A*-q%t|Y2e{Nl~NrQ0CNp|mk*I?#QG4}5*N3~ z7j22oyV*Pl&`!7+(h22Ewo~6MXigprdyUHR5Jq~Z7l+HvS>mE& zD}FLDCpIJSWnI=#GTygCr&8Z0kb3qScO8kl(guT|6zByh9b?ek&I8X0nF$F4;xOa2 zA#SOzyIKj#05jGnPd?$t%+N$H+3i$TS%RMVeE|@vI*Fx53>Du;-8@q;RmauNaJlKk z9k)|#ZFMsRA(>kgJr1xgS_Y44t`~5b2p6H5l2wU9e40cem7Qw#T{q*++%tuZ;26=5gXnbM0v#Zg1tn)I5{KPwsK1yxN*vYxBak|9#p46fD& zDMiRISCxFHh|f4~DrT!j53|s%f?jkGfaQ9mJ~EmN*t&DR;q`N*_&X7KCE z<&Y5b>DWyoOfa{xs_RdPVtd#=-FH1uP82sDx$doy?vm{vE-t39z>q%@j(THfyxZ3F@VKm3o(H7gZ_|C6Mh1{A~p}x9pWwyh(pq~#` zTV427ImF7=uqBvtw1)4Io?oFOL-@C2*ml>Kyorq?#akHXJyH}MHRz1=(yQ#)=^C>d zm~fQ?7Ql_qt4XBS1*EGRtB`6qOWLhj!LHaG9p>-bU1KVz) zpvf{z+D2I?vX@a7u@};SDt#b+X-aYLHB>oUWu32^lLJ7!pqmT48_lsKi%%fzAqyRU zcZ;@P2e%A5w1|fS&{^>Cg`OMf;FA)x=VTQv8y? zYn_&EqMwPl-68wIF05Fs96LLPIB|`00F*+~9k+s)Ily|je$m+JP(X?flH8-OT!&YM z!KEzyb3HN8PCRQO1kcEldf7%T>QmK~BoMKuPaD^PbYMXQ?eh3=C#l+MilzzIYaVfw z(IMz^{SlqG^oEIsHoP4jez^om$1Vp`a*?~3WvS$Ni+!t`T|XE%rjR1D>_D38K3vV* zbB7#)g%q=q=^N47Z!eCX-=4Fnq3V*;98zbTw-`*ggSmkExOsR~pn+ z4BL+5&iu6g%XPuj=#@3dvBIDo7H=BIw~F8aUOHx5(sH@3)9_YScZ5{+50tn-s2LOC zu!xIvLLg?`MAGQw>#WMm!(bkCZVuHn$uptx7+c+-boDZ+&L|S~ku)RGIrj_(*PQ4V zqpLS7X{{jw3sEX5ez7)FR~bMm>XC|cTU7BeLx$(6!tHO^In6PG*~Qy_FR|GaI!l9A zP{2uQRYtQR4jfmCSdOKZM&LxvWl2`jC0e_T$rAEha%UuKq7luxYT=B$epPq8u$bQ| zX(vFd;RVUuSPQqWyidc4b*wsQFv9m9P`glLP~`&6l2(=Nuf6F=w6Go#7ON^*P%0ag zY<3hirSYDe?9;vVqe2u|LjsXzE4JIl)L1uQ_~mpCfOlFYgncvf+#lVdCNrTSl_cz4780C5mD!}PGhw?B=EroV$|U6z*-ZCD905Tf#ldteo)AXlb{+3lG743IkBi2JtBcSHlkID>brHZ*QA#X>gT>`xggE$OMriQ#(WC zx4o0_@Jx>`9eB_ey)P|iRV4~I_D#02jMnakGYamto`paegP!Dmz;?MvazPdP={$x8 z&uuVg7y^z=Gt~aHD}=LC_e_<}dFH$gGLw|rGs!A6?kj@yDixO~t?3fAcUL}BoKrhp z_S4WI9L;1sxGK1yAS41fw>4L4}0mZMNIUqMA5mG)LcfQ)C#m% z?3H!{rKWt^c_4A6Gw?`MmpL*gIam*rUst3tj4_-nE0{2=xr52~4cz+e`aC4Bkr5`7 z$x1ioDZZ%!G6mtf0wKZJH17+6G^mK3VYV${%CCmyk6me!ts6+@GT7~MoWDM{t6wN< zbo7&VvU68&ZPeBVpWLw3cdk!-_i2vg8Q{x;-d9^fCnRm1BTv|CT1k8vbrw0me?emIc z=9cze2U{P{8dSpsFo3&}l~B^%AD#xq^y6W!&x*)KpBo!g4Aj5~Wn zX>x`}6aMqQtTgXyHN%EdJ4r8Revn2ttECk)q>9YkVHLaZczob@`vE!@q z)a2aLcL39+z>kDt4%mSd&yFLiFneyXY1a>#K)TF)I0uLxak6=kU~HX1Ok_)K@v{@V zrG$3|5=CBr}C9I+Di^#3AxNik5 zL3l{slBi}0ZicAIrC9Qrl#05B9YkIxa_J4F-xZtYp$8D5gf2M)UNDP;UyPOP(Y$WD zfZL$;BJ`{V!(`5Vc+faK)oRfGKY2K%A5Kc2(r2=$oM7itsX29{u$;(M?jIEwgkyUY z*obL|jy@%==i3Jj*_9@!L=o0ARi!%!9LzkW^(OOWtUdVfBf*a+2|D{w?RhIA5LXBp z4Ya{$$MSo@jTknt6Md#b<;iXbw$L`~o>z&z=0~l;;Fncp4mYtmmZ4~EvK#CM^+aQB z#8(Z@8p7ig$EtHXO^#)}E1okKYvLEeP8NXbZV6?$eu&1;LN&FcUE*|QgR+pHBdO+t}F-Yt3qLLSPnWC{5U&5&Ts5k#0fWL-`5Zj7QyWych|`v7(u(ZEdbB z&otr>H-&msQW6_7+!!(D6ziKRE8$c1dMD_hFGKC!=bX#IC08zW&_y~uBL?w7b_fb3 zo=c6=Ba-G#Fpsom4N?w*{rj;_@g^2R$@ zTDT93DO3*7!a;a#fL}_>jkf~Atvt*$;e7+3VhibcPdR5ihdk{sa+R*hAw9zSQ+aK! z7~EC1+;h_DrdM+}J2)V+L8!MAV^MDaTN<$^+BE))!G#qS>lHHb1-y!$)?`!8<{rN4 z0q{_$2=Ky8W_hK6SXJ!yJsH-hGmae+tnRHodgKlz{Iw4v!)So+G6f+19-V zCD5*Ucilu|=?QRj?t_?lWQ_L6+Nu~NiO>=Q9$hJ_*(lmUGmH0l^n_h|VvDJJe%^@F zo!lbeSp#HowQ0LIgC_}jEN>|jd#b9>(qQYj&8X`|Z{gRtxQ?#Q_GOVJ`K+OL#LnlC zUDg*c$Y?|tGZsv;Dq@p+$QKup3yujo*1F;EQ_*N20l6l*J20z1R&jw2&D;=pp?w~5 zE8Ps@H5uLRq5Axf#}KuP8aS&iZlIg2a8Ta)Fto9J>NQYvF}l+j>rS@0)x581caNou zhT<))IgFdC11wxH1>|fEu=g0SyR2%{=_E1u7uKC1<$&+UX;(@XTOdPeGADOniMP#I z#@BntCOfqG$#VG97PoCdXqtqeQFE(cgVKr@n`3FjguE&# zO>3pN!&36Ls`S)!y_T4YX2{SnmIKn(`_fz&e$M4uuVYLZ2$6DL8q6CBZh@_pj56aN zB2evztf2q}i|?!6_A6Q^ zU>NO%Lk{FKk-7FG-k(TRNr z^-(Zjl0*W8K(lbpX$ldb;1dqrZns|Z<3i>KtBsKvF44_=WXXXT%q?5k35oAJ zWFd}dDR+i1oEDh-2Zsgsz+;yiD4@n0g6QjrE1>Uus5rpsXW08UI`RoktkYF8`^4wC zO^eS7diUIh3#T_&DvqR0+_;X`4L!~W0THFfxtc9?Ng!M;nUe?yQi}zVr(o+!(>r*u zw+Ss8HR2YG;s_dCuw_TUUkNZS@nNGwXxb#U5Z(D6Oqkv17noIaG;GZb4yY+vR-pXb zM!g8?J>iJvow-UJc{~iDJ%R&4F2vo&O=(9*5S1kxT9`hShIT~olz_{WT(~~RVwF~r zYoy))om_+IRk}3dik2k5InFvyc}>`G-M@N z-d;qQ3jp8XWlW$lEuW9Q6hX)a!vy`z zJ{Z}9fuVeL&EU~N8i%tTASHfi_18sEzmq9IL{t*@2Jv`vYP?9mMMX*z9}B+*7OMzt zN@{7_7M(brA2b!C4|FB{jehmla4?%rfsBF1;Sw;xAK99q1it7MV;j z@Lsb``<+)EbOpPBoz(>W$2z?nJBlQW+3h+Qokmv1R?`WeZn&b-1)IJ`DqF1bf^Y(< zc7o~#IvKxYoTvf?PW6?(jYwC0Jgl#1)E45`8LJ{4vbqD3b?VnwO9~(q-!f#un<#Wp zZ(^pmNJzK5n6qX?*HE6c;^4Upa~>39QZ`A|=7?Q4pE+Brt~7-4z=YcA9;D#aUYg)V zXPEN*!({E;@u5s9M3}_+Z59_A?I6PFT-YEdI2t#^a!&@A#9d=bldmO;SRnxE*~6&u zu4LiFl$mG~Nnp^EASGimH1kyS(Hgg_0n!i)*lM#PV0!gy(!L`gCNkr`cJZjuB1Xbr zqqpVAZLwGb5^?Dq^AidPr&ZVOpiTG^HRh4$r6 z8wtU{dGLHB6n6{@-C>0Y%SQ(R~nGb`qUU@xaUv3s<;=~A)o&|?cy+gLRC3ji| z<%m-$iJjAk`+Dz{PC~I}gRf=53=Jf{vk1NBjo^V{jypAdP$AkI@Tvmvwy9`{7#E$P zHFn_0%KaKU)ah$N6uKO}oE@0C602hynucx=b`{YG$(%MdcGku+s?1fJ$bgRmvrl@w zN_K0_0HlRaQY4qGipwXpWdhypmD+-DKQBrzy2Ode=af18H|lt>!n75IpJJ+8E7kdt zmsfEMERQ&0ktnDP!$uQY_LTvpp0cAb{Fcqh1yN&;n4#N>JYQGVRTw5i zGy~;B@aCbFuAJeAbX{elu7S||)RBs4?l)HY}T*>17i=}{q#Ll zHK-KQJD6s3cdJl5q2uh=-1O8~Eb*J#$UyB`$_X{)#f58~im1T7Fgzo_P&!wfX&lcr z#i^#9;Az)mu&OJPX;rdLi+hc5dh@OLL|TSxp}}t66S!ob!>$z*;A`V=K60?lkyEo3 zWxNe6;iJ}Q(I|=xl*Ey*#f}XMvCgoov}`TR&1;M6me#%AQfn6GWh0%V+|kxvKG<;V zv{XIvl^~k8o(>X7Dm>I}Z+HUF$4TQ#+cP3mDP6Y1?}fe_`=}K+3{*c5(a^@kkZ-fE zC^#N252-wB;lu>kM$Ev<0>16USXboXEUoIm_IXp;Rj)3acstu$<-mSHX=?B(Im0he zRf*X>h7ZahZ)Z9-Eg_{w7<=|sn#e5&uAFOl78pqq*7>TGH`mnn0CJVIl7qT0-CGj} zY(O=~m1w-y@USK3vaqII-~!?<2ySa7OHrGYyw0x!*92ud9X`L1W7b0oDv+m7&_I+%kh2Dp|QATqbeP{7;E%QA;)C!taT-m zz^Q71!^`D2UyzBVDu#za2TRT4?MM#C#7nH*MHCz>t*)-^^{$vFA+t-pP9J&oK54e5 z-?IEZ=t8?;3ik>(Z1dG5WUFJ>*NcdE-4nS`Lg#h+vH*OlZ!B~n_-}(U?>eG6WSdsS zsys_^(LwuWG5=j*QVv7W&vD~^o(#uzzMr%#BPYlh33nwgRk=ng>aSi`SJ#H-667~l zB(Q+o(Y7^%Rb8#r(X75!7q z1V_6V^C3q>tG?np&r4?%RmsUDVxnj}gG`l_7R@I2WFkjxl~f9{zSTJd3W}_pvOG$i zq0UjiUvN{qWCMELC|oaej-`zsGA{jHdO~S)D0%h!i*;7FsYJHvvTo5!-xn4?0xC9X z(r{m?77Q$o6U2hug_QwE_M_LRPr4>j?ur0QE%TocMK<7OY6uJ~&EE2s9kG(zyc6AK zI{ff7U^0zxv&q3DfY`5JmHQemW`6d7sBB8rMEbsVT|!qwd4Fa&?xjQENY0juoE1wd9%ES0C{+9mg~4a zP>Z{1Q}SQ49x#;tR(0q@(+=D{)iz6Iqj&3APf61SQ^q3rPaDBuq{0=6o<{SZYTJ;8 zpJhhv5Ohv;jwiV^?qWlU0@0&wquvM^@vE0OF4>{|r6-6R6*39D@j&e_aHR`U7~8Rg zPxAamPI0u0rpzYJnXv`eyySENPum4khrq_uA~)V$i>rBw&pqyR+WOaz(dig~<0!nU zhfj7&*#}TbZra&+MhVUNJ;pn(%{vYr(B`R$OXn{UVDjIu&n;;zP79B-m}2Ug@JZSr=YSWpx=-RviPYi* zl9hPnQE+E;<&RPEcvpP!>}*DTOfR8+LM2y5=s3ih3j09$BCnwCrLO0KROD)`6q!|1 z1RXR~S;J=R$5WB-MK(aNOj0n%$Ci-P<;z#-+c1C~-0on2Yz<&~ zAd2MHQ00wQOwc;KD~O!sNo-7!@_8I&!pUr6Ymr9mm$rg%3SB0-8vTSl>vCU{ivq+1 zO+noo(cvc<3{6WB1zjjuJ%a1tt{UF7@QPW+qJ{Mmfs)SdtVeQf)*!Bf+0QKQ&)CK} zLV)9JIFZyjA`jOgID_t=1f<7$IHg>G`-%QdwY6ZphaoGrjG)&-eW8(@BO^D?DicYR zzHn1(P&OcGeYBMX z6|jK{4O6|J7X%dzTP+qD97<}pp~fnFlbb1Tt^RU!nzVWMHM!s+HE0oiJP!QZ-& z+Chhq@L8zMm?@2jhZ91qhT^P1CPKQ1$tX|G6b;cF(`0Ev0#V9OuT90`-6HFNlmW38 z_(9EJex~^nk6?y`io(^{L4bvbB>ZGGrSg(fH*Tn`U1sa!UTF#lHnRr7j^eMh@84bg zHG{BJyvnuoNU`On4#B1ttpeE0g?RQs^BJtUjLu|l!+%B@2{WOK)$Tm?qmGRbyL74o zK|*%Xq}&eIt({s8qgj#MhV_p-yd7}?EV7kp5D_5L2(2R4+gj~~o7AFx><%kevm|#u z-a_a~0$CeD!k$$`n1x*xEs%1uYufrmQ%44Z79SBER1OS^NMg`e40r?-8J{Q?Z{!&0 zOJ}Ny^4G2&06O-L6=k2R6B-kF;S4`2-$Bgu65(FAkIIC5>f?CG5kr7lV$VlqQddeT zRm>HEaA0ME4{eoI#@Dvh-#(T(GF*7xlS~M_pevD%K4adyey1?AtsX}tAmzu`QCyzO znJ=tbH1r0KM@c^5ZKlM&c;w$uKgNT5IU0%9#k)5nHzUBfyG0d5vdt zAUnNNk{s_{TLc5$1Tvr`nN~ifj2s`R2P@hn)^ZH*7Cg5MQ@lj9q5x*ix3wG@-Qu>z zcJD-%k9%JxpdlfXBlRB!U6}XQfO%>8NfNK)Dc7UKD8-vl_!8U>10(yTKK4vR-0Dhz zpI3_d0r1I-#VuguObw2D7=5zBX~9Wj)oGbNYK0pqUYWyP^N2<*N{=&~bcduT%&-zh zN19NS3tP8Jzc@152-e%Ddu|NgfKq^^s4X{T+}mK}RhNSvS_&*>;!NAXM5~<26L5v7 ztdUOwSRm(KS>~sK>!J0kQOCAw&pGQevZ(7^_)Vu=SSJbBEdtYftM?YrL?_#ut%f&( z*LxM%%HE~V)i+LgY%oef0>|DP>s&cdu44DMao5X2RBffK=k&{TP0rwg{C#AwK>8})Wekgf=dGK{Q)V(GL_aV$ zb}5!j977p~zB3?*fUXYjk2z0O0$CmK-w&U_OSJS z<9n$uIF-7K`q&{w0ZyJO7|KAXBZqek?u*5KzSbpnxStC2R6zT0buI3#YZVv-gh9dw z0YdRs1q_MNK}YP84xkyLb+yuOxO3{mF%eYT;oV#gaJ+=%I591IxuEAo*t;$~#AjAK2ThyU zC7RNBD%g1`dz|-R5CQi&lP4=>z&~qzN>&?~1kxOpG`pdrm{1WjCDSQqQ#Po*GH*fQ z*(k(r%qfEGR-!y@;dqd+i#tZ4;Mk2n2}bp7n0P|8>B`|^gfEC?VC^)JRAP7(SF|lj z?gfUdF>EuAYObspTW$A2lU7#}v<&llVZxh;2g6cT7Gc~X_7O#_PfS`#_a-g;$ZgDU z2B_?eQOp8Q!UKJsB;Ix=lt;aM(#Nz#L6<5v^5-Ooq_h=Uc5G<87EKODn% z#R(O?5yfYi)DgTZZR9nVbVrI-^TAl_6EZ}Nf~o0J7@I3znHr1nZ=1MSypAlPV`srR z0XRkMy$>#pM-uCq_A+sER#y-tpB;ki>_~Gy&bLy@lTePM_Q7z1Sw}yXX~DW9JF6vp z###gq(9F5S0*7IVZ&!;R;sbhmW(G1*g_5)KoYPheM+gh7;njLLYXYPB>XIQZA&9T_&LWY@8PSE9CB=Kx*n znzA++Ob+$GXXVdfr_S0KQi@doW#qbOm<`S)r*sZ^7gbk+qo_BNGSvj-VFuvZO5_TB z=c5qvdTtk$89DRI*Gn!Z3Fv;3pMg2vk7IN)u;s7KL{85X9n8~-w(RNa;6r$~o&p`(KdFVzd z@{!mwp-+Xk&A1h7Lz1>wSAR_=!agPxQYlRro=48?c`RvE2q*UPx^JFc4U2-_=Zidb zd-j@#(@53C&7oRri`*VuDr0`x2~~~jK3m)UF4b&dzdNZT127{Awbz)8UEh`*KM0-}pLAIf2)lh^L z{49ksPN7ZX!IKTXo^i9V?jW*`VCEseSo0mEvb4uf#Y0u2x{d86Z=miylf}rT0FZM9 zUG5E$Oz08S+5z`WC%hF2T&pfDKvsWhsf4J?^cBI;TQG#1k6^bI?#_5Y655r?j5O>T zx0khQQpge}@kMp~ht&Qa#Rd&RP*Kp^@c5B-NWx+U!FKxA^~)R$U!}2DYZK}7SulpP z?detmXBl}@DT#%6i>8TuC=7Z6HByb&tmbKG_Q`jWkXGEHe2NF8gp9-DgC+nsK*+zQ zp~#hxjkeco$>()hjv!P9v6E3j_K0=stGFlD-QH~-E4R1;kbZ*YYM&OXC=}Z%68#gt zo3T2Pn zl%H2zU0r~LVayP-5W)7S8%Q(Tki3Re-G@3?jRu^E7Xww)=U+69R9Nw-*vp+4&caS-*C!j-nFM;kEZ5;eUMP&lM&wBzm2FT)}& z)^&o5>e;$I&fLB^y}TX_sDEXCWGjuR&26U?^>M)SD1@ch3JH^*J#ml)^dZc#ba9Sw;$C1E zlPTuA;K4Mv2S$Mdm{yb<&07%O>mhv`%alPpB)*`_oF51=1kNXdx0HEZq_GQ@61Kp= zd){+|XE`*25RGEWB=OYW$J%GSlR1$Nx#{$BKEEMY7!hw&B@u; zVi}jcOJScVO~5woq#%&?7X02|JxMOZ@#)c}EhE>n4#uxd1@}SWb_z~(lUg|Lv%iu!_nR4^QWf? zqP@5_*KJ`yxC-7Z@@!|_EGMWiIpVu$Du5OPB&a30I(e()s56>}5|lELRWWRzR?9>x zmToixapk{@d7qC_MndCpw+GJtYMwCzhq$$=^h2{>fO8084&;kCdhRA1{;{$WZx%3H zjF(VmIlQ1wp;Kk0Rb^4xz=0B(fg-6O+0@O-qEuc{?yCq5_86!^RpJ=Xxj>b2;F*_rE|ob7ppruVTULU5swAK|G*_IJtW&v}fafe8nU+Go?IDI7h} zTGq!&MmJzXOn4>X9wZ-Ok)SDKFx@Qe`Aq0?AH$`K&PG*(N4~vaxnrYuIdh~lI?W=h z!jo>ZrMnTUV3H^HJV-NTP6;6@g)UGiGvsq83Jv4nP`0o72ImEd^&bQRFDHz1X^5;`1`uUw5#3&&q6us{1rF zBG!4z=AMlz%n6r>hvrSoT75J^%^A(^9JDMlgi?C2jAyNU0%K=EtCawK;Z7hrB}Gc< z7f}g#BD-2`;Z5g@GYsZcdAA@SmNB%bvfAa|O}V}|-Z>&=v~9Ps13JtnF3QRN7}DBg z1e5t4{h*iyWg{#bE}&)Y&NswMa8Vg)+1nt~#nOaKorSCBv2{gwxw)+oL*Da&IK}5G zJC$TiAA*k;CT%*$PdR|ngy~-nIXBdnF5ow^%j(J^@p$2EbhS3&`b&QF?^nJY@|Mft zxn_D7CIVu{j1il~av0_JyJle}!fT`!t$j`@a}tX^oQbRk!9EUuF5j?nTIdGdQQwd# z_T*|81rJTu7}!9~~IgU&;o3TjeZj-m8rY@kmTd%+%aoGMJ^eW!#=HDyIe1uMukM7R$0^?{cOsGR`7+ ztQE+eMC5nu0W!6GX(W6K5P+a{ks!`dVCvG$w8D)jRjy$d>r+!|jAS&f)U4~W+3WDL z5o^o5<*wzsbV93LlZ6IGVGF=|>6Tp0tmhoGZR3eO34YvPZj3ehe>esKFslvsJLvy<90u9^^U-Kvh_Qt>C` znCu`HMBA#fCh+XbYvqCnu;et_<@6TrA9!~h?}CXdprLD+@;gHNI1z^9NV({_XdTs9$p#&6 zNAXT>8IUR5V;x2B2o?=cEk4 zz_#vi0a}n8t%4kQC^@@xOScQ`Pd&2(5$T|CK~E{L(l^WkaofNZPaYSqJ0kaBTM>0} z3#9$7;u#a+d?98pu)r)W2-&-ABRh^K!#j^RZI2Xvd2<`EJo0cgL0q6J*f_xq-&6X1 zfmq=QPG>&cE4iHmEZu)gw#=VmVyeA9Sl0=Q+N@@&Lue@a5|udeHGa_1$AXi+Q|n#N zL#LnJSpJv#-Kr568|~N;j@&vJ>oVgVWpqi~OfOCGM&0$;N}4z;xg=mOmipL`C`tHM z`L*+MJhwYd5q4!9N=$@>?%GT7?Q|Q1j~bOs%MG|P1XgZOGser6aC72!nUd<%5Ip1t zk*7kibEM3yS(?Sx1>r=i_^d;8392cNaNN8ZS4**{WKo-Zhd_y7Qc4Ihi{^e(Oj^5a z-8!fjA`v)CdfEbQ1Gbu*yz1oyR34kdkyXpktUJhAi4nPiq&8Kt2~*7i`P@5p%B5R7 z9O2}KPF^d~X_EH{Ti2-PbDFi~jz1MEIG;;}K)o=3P-m%=@DZiP)Q6>Z#1WROOS!{rScZwb@799W&<37x(wMh{N%$kF@s%aJQ17N zXlFoF3kt z0iO_iK&%#W#b3MGx1AazX2?O~;p0c`=BmrjqG(!dz`o$Ck-Z`|7(-gXef3mublsWYckHzVSEn_5(dut-iB2d4GMjZ7jHYODcSJYgspb>% z${gp~{A)~r%~4GcI;p`?NF_T+*0!!-<+xn1$N-Hw=uS6wEPUF%1c*U z(ruu5i@X`nW)Lz~joL;#hb&jns>KTY;_aZUhUin;NP0ea+m-l$ZT&T))Cz(d5Iy1$ zj|2?`u*>$*1+E5~7$|mLF+l(=cIdXqDaVh7Oyaij!@6=5JAG#ZK(6VD6|N;2T=+*I zjv&P{*(Cmgn!8xv9EO!)5=`0}JER$8ig;|J3!0_|w3d&>!W{H@fMMo{Sq!V~^-lMiDoYOgq=t?M`e7sL-x4g=E*V$__1tBm?f_$8aT*` z4?5Ok;e=9-Z9#zIjGC=_vP#i+FL^$-6=-6j`>W!sQ#jx`T(AIE-)Fala+G-4#TW zyk*y5@j%T=F3J-cPhr}#eX68Qa-D63Esd!{#)dH^vXsQXGrywl=R$MCdc%;f>p7w4 zys=BgFJb!@?NTV4E?J(hF2r>^1SP&HNDLC!8s}N@GEE;|0P&;HyM(d}QS&aebzEcx z0ZG}P*%{94r2LoXH8EaaRG^tX;{8w;mb9hPihI5lsu6FFq?ChU2k}#xOJOj3q6j4; zjzPHUbshx^#0TBzemB##m*iFs9OR2?J6@lQI&bYxF;tEdgrP(!VZC?+1xJ@oCUo*1 z*+6JmtO&aLX^N}W&4;@SY&V-S5LMxn+G!nOy_uo6kQF`7Cad)2apHvItqb+d6t!xx zV6cfg4T-1-C~TjBOf#@xLBT*zea@*;?!M=w*FZZ)T~X5I>QAjNi+wDI0@z-Tn9S7~ zhpKpn>oWIQfrPCxOR{i>1eqSTO29Rn0ZP%OyuD_cUx{fBq;?ASwk6!9E!xIc5l++_ z$S=v}3|ls%$nNWfe~E$jK@R6ya75`vx~qZUVVxAMS!fXJZY3vMCGszH=MLBlD)SH# zsd?7BBfFmsFgCM+bT`fRK)!jQ5;S?W^Q?BUhcnW(#`uq9A$oQb{;1HW%$Blc7w@y@VbPco68dHz>?yX zjpaq7julIR0}qI?-1=WP>zv zXYFdwhqG1anzE+m1585>lrsN4axb@gmXfvv=mai?A>yt~d+1;~rzWY4+HA&t*OQMsD4Up?z ztv==ipw8)Rv^&}2g1tay4Zk2Ophb63H!2GHA5yb0rzn1TID9Wk@1QUg?~j$i0{G&EdLpLNj8E)TCY_ z!ipAng4M^_%rAOvg|Nv{!AZ})=cKc_#!@KtU#o=$NqZgLP!9}5#aLEi=ET*x-uMTZ zq0?ik)qQ$4yf2;(+)_NNTrW%n9pW<^L0}MW_?Lzn9?XOgGgd9U@}>wJ8$(Wj=uYFM z_^MyY1y6r) zN6QxIB+pV5v@IGgG$r*V%o`c@*Q(UQ5~gegUfeaJ{f`9V_!}xTWL>4~l{44&!{Qix zeLGC_EJcyeymd+l0>{PN9X$leq*q#~zCdNH2f>Rg*eQ~dRzo0UtzG7+xN+DvzYV$= zqn#;UA`QxM*^bW1KntgWxM}070!*_cWTq{)jcDEk9Ihx7GeZfBDk{$`y84v$&TaxB zqZolkc^`+up);7xspnPJ%4_5ooRfmeF@cU1hks5};(d|EqjEwt)a)7ey3EFNOfkF8 zupuxGv=|oN%;zWzzjB#}hl3$27pa6pCj$XXdd8Aq3CUYVV?^~j!-HBKP!`WT$-}E% z{OW;bub%-IG?2S-!Ky6~FP%FF4(byjXBrL$EJB8l+|0|Pe3q2LUSN_YEDD){CJMqjl@KYE_2YG7&q$ne+H`Xq8wuCfxCfld#?qhXAg5L?5BH8U4~QLeeyJ z-eH<73HMF>woG+kT^n91{s5HROZPgJNrx3&Zkkpe(iNKxB9XXPR0%kjcZIFX6G#Oa zquN$)uPgvH?t^MtjyFpYn{TawAk!usDv`-d{O`90ioCTbhXVSkC*)CD+O0i-g>?~0k84N-#8YSY_>-FRmNdqv6D31Zw`tnEJPpOny#Z{nuZ(0m?|C#2x2?Z63v*l0W!RSN+dCV(`8SBDKeDt%Bh zGgAY~LR@KN$*8NPXjp(O9A+y9;Kgz!SfC}D&S4kz&v9WUyq{{PRWs}?dgo`=slCE9 z{7|}_?L$$48D?R|b&_&sI5Zj38L9UuvR)#?c4HA8Jnba9LbvTP`|&?#sc*zj~=Nldmi-=3!okSaw|1;@bYinJqVmRPNB z{Ayd=w>VDZF^B`f(_nbC_Bf@+&WrD*m5{5e+m1(v&Dq;XTyHAlcaZ0y`PsA-VbYtRt$VEox&_Xx7| zqj%JVf?hZTjmC24Pq)BcqIV~867I7pjaP@hyrGFK1?D;h&qjqbHRp|j%RvxY1JF`> zIasK*-(o{W&99H|y(v=CeDx zh>0!xyfngVilvEFQCzkc51)J`9qe%0-07y64xdF`RGfpIz{y>UA{IacuPFJbLD-I- z%q`WcjXcACHfAU0<)Po4BwY6lXN$2{#;w*%tTC{(p%HTdWnR}nq#CRN@NHP;dhw>) zmQs1(HcoJ#Yy{l0X*`^p9(4Cy-9e_1H>SKmDj|9xp}Aag8im6{w}}>GmY-1h>?k1c zAc&SquQ*++npmBO2ban~UqqER9CPSLj@baCx;935*xY?SDlwNC8|c$8*eo4BxWZbx z72?NRdKCTMD>;%i5!nllP3@Z!q0^Qp;fka=awj}RCFQ1cL*Mk zzI>PNxaM3aH3HvUmeaMiXd}Jj;#puL!DYT9f`zI06p|#Nb*-hO+ z0Stv=7lUt&I$M`h;}Y6Hjns2xtLhe|*tHWRJ;5O!tMJC0;J-PX!k6jv#a?`mX2%C6 zR*}$PO0x99Jny2$@{+U&qwRM+hNHJ9$GD}p1SqAKCCk9P5D^HOYVJ&GdUp+XPJ2s6%iq|S+e15c+c$K-p9>w20JB`0c(V=PxQRD@@|9L1&)nED{l-rz?7S@|gzqyI9%o04h73S;{p2bI+5TB(Jj& z>-ynwrt~_3@b_Ks+~N}5!efbx=Nrrg7FP%hmY*dwL!OJ&#ftVBC57kZ_j>(*U2 zx8bO@P*CUea!Ao`lJkXxgc;;ZwL5tATDuF0IgL+g zFDTQq&7pc^xEjWReN_BCR{ZoJMoFf=AG}FQs`1D>xv^Y)^x3;)_F1{>4`)};U7avs z&mvaRf@Ua_2w!4AKRiZg6#KPn#<;3k%)}fpmm!+r4AqwLvAB-T#hf0QKw5BOTwFv51|E8{p|kKRR5?hEZv{Mpz9%w^rfCi=FeAE`2@27-DcuhA zE0=c(FAmpwrJ=!*##&3*(YlN2##^;qH?%6w`9Vx8%Xq-^v>4nzp>U&W<{pqhr{g)+ zu*`gNWT8w;?-$Z*Drkq;%Zm3P)h|0IebJ=2fir|JFlw$eB3QU zG2j-?TK11)=ryE&M>E^WvQY0u4>K~&Jl80k(vu494Ag*{uuS)P(~Qk9Eal^jMD$Kk z>J_+K3ep)-ZMz#B!TuAbHOO$>0;@%`@>Z!mg0e474($`~PJuBB&J}f}0|gMMgWErE zL)*C*gz(ak^gY0J5S&mP=i3`Dnw6T|M|wb7je8)x$sbZj>gYT4qIpMiXC0SKSoo)S ztCwo>J+@=!doyydc2>Y+d3<7@I`oEN-J!!Sp%>GHfD5 zAQXbQM?4iM6tMbtW!%9Nvg%gA-o4~mth~=^Cb=~}r*TUMzJ@+P*c3Zf7uY$=D#U+I z<`YIq!NNTAmbec)l|X5a_E)RQOK#ci12V)*FykTrSwHx9f1qj7Ox8 zEnRvqhX9Qcm9~f$9Zf}&n!#2(l(y??{JR49-V^Res_a8Y8{$lYCAkTSR1PN}8%eE9 z0?>tN+l0enmPqUwRstoZtQQF`CK*I+=^FJ35}VYNj>k4;FuGG*&w6@p@f1s4#dse{ zn0MBJSS$*SmL;1b+=hK9${p>VQX-~j?x$me)^I_w;(HQ7*=+T$^z{8H+)EM7kUbS_dyh~``xy}EYefwA** zI9~V?>}Y)ktyWLV!#8kfWaPG}o=ymn?vZpctiCaRg?uk#iz+lPc>{6Vs-$daQ)6!M z1*;P{cvueRF0F4x85~GV@1j**`?xvN+6#RR+153i-l>)_^U)O~MOtNm#+4VWN!fGV zaAa0u^VpaqTBJ!AYf@#8kPxG{FVBax`Ml$(gjDXe{0Ds$kle)$iwGT!lP#CpwUJoh zwC5FU$KW_(%dZ3hzN?dsrX7mgr3M;-TEDgn!9A>4uB}eMfPl^=Wgz^pK}Q3p*!0fz zSwY7z3DH1XK8!p9mgx=!3Ukcyg%?c8q}pTOf(b~43=;u(D|mpJ>_q`e@jyjGJzef) zb#v$!r&7YYKv}4fqmk=(=d=st_x2b;fN^^CT(wBg$@9Y4;ei(AP9vJm`-tk z3~i%8Q2G_$nJdMeqA-ea6Yp}dhm$xWyr{Hf+$uSs>)z^?-3KR=9Yu@pUm`vgP!Tdq zu*3mYFH>V8xO$4aUy2jy!S{AuLyfl6F8uU_#QEtZHi8*JTuV){-qO{WGjZRZ=s@3> zMK~>7BT1;}f~`fcrOnnIGk2ai(v&5Pd*@fgUlnb&ipkM3@~3!joaQ_jQ!rdyd9Rs? z@*j^H`MpV=;;0=(NgJc?a7+U!Pn*QqK7{i2@Zotv0tNB|$aX!Qun<7L=e`Es5am6w zZCzQA73&J(Z@f+jR|!zh4KH2iD6-AnqDxOtcyqxgvqe@DLyq4iX$xJCE+eb2TueP_ z2MUv^b7Fu`9&0O`_UX7VAn>-WXEhf+-up8yR`aPu9R~5b0J|*UK&adw#>%VTLKX=i zLbFN=3(Y5S*f94T-&`;xn3YsCgD}wFg}9ThNmR?Wp{Y0#mn|!`B9~zorle3YIiG@6 zX#}Xdu!JKc6hs_$Esm=1m@cF(RQOMpaRxWe%rf`kDz}|qUu%qv3i|1I9n-V$rv#?I zaVHqdTKLEc!Nb>QX2+{nGKvDv8DMqpp}C`m=H#o?HKm1)OuRyHnksgK^DH?+@070u z?+xP(Um{$yIFPkt1+Nlx;wW3FouztWuRc1bo>1;?SVL$lq}C@IVe?IKDng)vRu7yq6@n2vz5coWglGDzdq;H8%OiOOw>7piX}j)H@X(2TI*+y+mO$ zd|1yLyu1qz9nZ3!y}mn_12Z;=XH7p|Ftjcut6XXjP%gyHX(|@f&xV*Rq@j3HA8LB+ zN_$l($7U9DK2U)n7ifxIjSpquSl*d%9YjDv1Q|zi67my>B9KH<=M;Mvrr&opJY(M1 zBo7c#!xZyEuqoG4TQM~7&E51;(ZWKIjn(2+CK4;sgBL3|7uBlrVB00S8LdAdb@SmX zI9r%$lH*s>B6pqT!1JR;sO>O|QNsnDQE_=XWSdhE*4abr^Idh5-JY_6IqNvWZWX)0 zPZS8;-*$y>fEkVA+Pg;6^c|Ogb90Rb;deq8*B#I-LA-W#aa`Rzec)j7 zR!={s-_|H$453eS06}?!(54W5&&>^_(cS~R*1sqr3MK29@uxs#AdLunt`XJAh`Q~- zvDF8)rzDc$QO?WQ8I-I_f!;_eXyaxwqa*`ctnR;K#jwv?h)woP(EWYP=DtY9$+scF zKQtrLY#`hmiZ%y;QuW#|BQU_bgbaP5f>oLIdwiEt>kxf`XSutUfK0-OSU`fRNHB#> zLWJ)L0 z?ch(4{K-j!XIfbBt^xuD-AM^L{+U{3OI0v4s&|oV;>t%8kg6Ok`pGVz9nO4_H< zI1xJ;p-tsySYD7w>m{TMuVu(t98W1E%1%z14y;Q!q{kfjE<;OaF)z(VvK%*Y8yR@7 z2ZMD2oZhW+@$Iy7GWc3h9PgU+g7{ytLTtk45(KKwvqbAW78&yD!Ru*JoVc*Z&Tq*G zWnO!si^)s}VMnU;=Fp18IolPOE6F%%A>HaV0gyT?H%ipK3p#@)1h@x;s+$4VVQ5%E z-ql%)x0?pd81DvZE&Tcr^tF@<5$4!e-8o%U;Y4vuC-JoU-i=-Bl9jVGx;KcdwVZ%s z*-#+gK)852LMMs@5pv?PHDbzbE7xnnx8e+^ZbPwQx@KN)0buCwPYB4fr?^jAgy45V z>%HESn3|iz)JJ!}SWxyb#aUI(?;kYF8IW!2iVoNyrl`J3_#&;8Ooih*)6S6R)YwR- zBcQ73H`f(dPxioC$j7DZ546!|f@VYvJ`ob~L0X`nU34|8+_kWgJ71JEmeRHJ?rh}R$~YuPdZOXyQw6rvzq@a)!p#;70j&Lq{I>c;VRjETYkHI zXECx-r;eXf$v1XYk-9)Yr+SBV8{XQaC#J+KKykx>=&Q)(bn?VZ3f$8UVaeA zCicQIi4euvcx!~=rs4!k(+o}ajtFap1`=Tfv=DeQJv+Z4?6=tFJKHdc~5)H@XCzO{8SmHh`HnyWYHJ(~>F+;Zm3| zSHRuKJCREsn~>e@11y{@6QUGHO5j!s78#+n&^5R_owI(5Y^+JDV6K%OIU`DH78y2F z$w8T@H$Lj9P!YE6MUCb;QzFdR%V$+ntkHr4ybAQ6w)3({(+g?OUJ&d?l@K7QoTta$ zG3@3@rM}8Jn2Og8u9D0pbXKW1mOJk-U5Bl-;G4WI<02a0#HE9B#tGxEH_9W?nR%`w zdP&7w%>A06q7$}A!vfmdMd8-ywc*H~3t2EYP@jW?xL_I}1rimNF4s$PW^u2w*pk;`_w+3ZKTl+_2He4~dyC z>S+m}<3*6JNWGUGW8+;rIccy;@d5pCCN{oTrD0lT;75z!I1W90s~1?B_vgh_prm+y zQdO1;Ta7iGd6nq&C519NV8uKs%Ue`nB~}l`gUj6DeArGFe)VULEb`1aE*~lgM>Ev9 zh~=s0A52j6Xb|kDxiZT7$Ba2<&L38NaX2_2*wLC>aR@>foe(4TfTqm%otETXj)+k^ zd1l13NmzZu5l(r_;i8ub6vkThu4Xuf&vaZkPDvuY0y;Du+6sp%LEM!K%~J zO+u)-eIPVn9rMpckjbzc(+_gXgV>0}^F3{!y;jSlK?u(`z7Y`g+Tr_xp_PP4WtbKy zeT7noyV?;^ni9(*pe~pZYGY{Z;746eJrURqd)iR5SW0ha;v6j?s!fo3KX{RxO$==Iq7c4hrSuHsGkX^Lwx~~U>Ky-I_ z`o4HsH;`S*lc*{;@?2RhJGGPNsisz}LxVQgn5qz?Q66%{YtBexdeK_czIA5S;y~M2 zI08iz)tV{6ap+juZetn6eZx^*SoG_&l}K>4C{Yk-GoJfaL0lf|yI!m=`fc!8W{(pP z1_@bghU&e22xZC&P3G7%Zu57jE>ZEKWjKy&hm^XTuDG%9+-;pNMG7iOdPTF=)KIqx z9%A<74rulUijgL16NZn+w`dpMDkyaiCX5qS2t6Lw@Rsj_-EAliqA=NZqy|8+{y6wM z;fc(x!Uo}C2>Cv2EDfe_8AvM;dbO-kZ;Vr1))FTs%705=dCpDe4+tM}hgJ+CfxSH6Y z+6p{2tJEC9SyzKWdDorc4BJc12|jn!76tm334Qi~62k zN?%IWhSaUKOIUz_y)vXV(F)WdnWnyx7iN?}5c`rT*~F`+?lIBiUx98E6?{4)#uXkT zIF&j`N9h3@LR42)C$z9KQL3hYg!Qklcitc;DOL~|6pn$ea*bB;ZmApBZLK1fU$lqL z!3`txjg{2JGCV$H4{y{|#Ankx@5%*VbCShvsmy->t-WcaTbnawNtR|9$C83 z5;j8BC?{+c?|_O41Hvqgwjv5rCOjNxQZNMR-EH7SR2BGUj`e*QcaeaL5ov`JJmKIs zkk@IRiuCQQ7>{{VqJ|y^&g#=Lnmf=en&{O9F%l0vC5nk@1N|*z9AZ_g>b5YD9u&Bg z(h9^GajbO1nmNEgZ7_qG4`(g$s#_nqF(0IC#4p>j9(|s$PaUXn?^4wUjHK?d@dI2M zlb>qz*h~t$i)Q8!dN6?25$H4vv<#rVXE`4zxtA-mv*XQyH8>W`7-ONl ztB_S;c3n7m(9PL_O$=YE@nW{kb#(M+NtJLx=ReZ zP_d+{ED|ZeC(;4x1L6v}Z?GWdg7|nNn>n z7*<2sjuk|a*s#J%luv9GK^_Q4J3ev^3RXGeC7*KA2AJa3=Z#iN)#LGII1~nVR~Ybf zSPR#NmhF+cauL;sxUS!CiBSH&{85QUeO8rR_~U7{zW}R9ibg!M;v89v-cMQ#KCe2r zixCAGe8&RKo2qi;x{*N8K<9=yT%@DLr7(6I>EX~4-GIY$r#G9C-MLAD2?%3>kw(fs z@K;v`deFBR*sc8VQ&32inUHzaK-utId7#I_(-fyZp|YCH0s%7QZHmyrBPq6QVl zcNzC()b>DTUbiMB(AithZFjIKgg}-}cFpRvr~NwQ%?vher~Sby!}X8 zsrq&jhGZsF)a)e3Q~|GMLLw5A=2UaQ;Axer=*4H^N4n#xirA;61}_L*aG$!#5nC+~ z{i&W1HLX2J3Tu_EZw;}hS9I*|Zl#d)%X1L%g3?T~`6Rr9&kK4~@h7==m>n^fHzg0V zTEvLLWKhrAtJV#TJK|O04A;FBDDsX18*GCps7NvU&22)frJ9>B(QDMzHYGfZma{M7 zdH|*&#H!{ImrTNJC!t!ZIx@n0w|)yp%*CK83v3+o6_u#mB*%c~?J?|sO+&7Sy9=2$ zB2oD7uiI=M^`mxHj?hAF$RxD-LSTD|=M3J(IJHdj8qq{dVwvgvbSD6=*f3yfuef&( z>&*qZ5Th;$QB1St?a=AZf|%Wwgq3d|6FI!5Q#yrGPQ%G*UiZErJZ~xQcc2~LgX%9k zUt4toLfpA>Ee}3x?a5Krx1%@Pn`*&maM@nA%=LRC*5rc-vW+sK$>4~{?M2to1r?Nm zRUyw8pSG>KOid(0TFH!11gVMq*3O%71vuDLF?Acg(|Bt$-L*if+*}9;?mJh>6IVO| z%)mvvhNTLCCgVW?#IK!r)h%jFDv zj1u2NNF;{J+q_Sz{W0vAo@BEO%Z)YhM~^_4UoTU%=5Cf%u`%DjWkz*LGL$bTUE9^*-+f#2Ej>mZeDsAEO*3K z-Ue=dgmg1?TQSvSJL18V;5vZpx^ouf6YbL=Am|Gfs4(EQrb$zW4B#&zJ$04=rWZxS zyZ5gf&>)i%PF=hcHHgwn5KdMgfmBo|!Fdd2PNOafsk#Fvd9j+N6%>01V{IqB(GO>L ztB)S4t2Z-k8+4%aKcequ>2}##-O}+M*iYH%-sJbozV|$*a#bKquVlQ1FLIFQN+wx~vIbGo2dVo}w3Ij-(R-2g%MVec=xNS-tSvqd3O2z!JK8F|f;2$Bc z0F7^#CX)K!2g6==^|dh<*n%)XO$$~@YkSKJkd-b4!&Ai~E8+{yB5lq&YLk`XuIU-zCsli+jnPD!mevc-Nngy<4`e9HIaqpHN~$$xn=a{Znx!J zyDMnCWxy6iI_7cD2ROqZj?_}ZZfJw>;~ho}Fde)sO>X$423X`K*PemHqRv@7RULZD z8yFGE+=q1wCuum2T`ngypz>$YE#$xR@+}=kVR4 z?~R4;H@e3D?eG$9Zz?SM8kA*MH$^p;(r0f*q9eR%lP{`99Y{wrwyudXU9yEFoQk1$ z5obthbf_%UpDST}zbcC^Jom4?JJ6TzVptWo5}X<3JSwoRXrpDYoron)odJCtZB>_? zIvcr=9@Dd~0t+Qls8)MrpD`?gkC3-}D?OZZ9Znx+Ahw5)J$pAwZff|=iA+i?>!?l0 zw*H}P3h~_`#N@EDngn~fCIw9)Ep(9%taf#c%9^{Laxf;~Y?-SD#^b5(pagJq5kXMw z=sDZJ#EP>T&ckc#M$?`;BZiFtz736grIz=1BVPZc@w#FRRHw{fHi3R7f#S0R^|)X{Qg+IaBl_hMr$`MWdO{k^C>+xRk5G+l-Jec8 z%4!iNPiU9LcniG>E#bfzs6SdPqSkQ@Q+l1;V%)EA4Y&iwJPtH6`b0CK0#i6tC4_2l z{eHkbiFE~n)b_hk2u=us1(T#=v-rzyCybu0mdL`kI50cJg#?NSS(WJH-S!|RUD8*< zht(TzVT{enhTt}UyEP%*WQP`oB5H5RMqJl6reOSf5~{%=M!|OTCr?f+Igsq4c;xg6 z(hnXmGv@MZ1o6ue1tPDAEt(#Mjd*lC#`8L6YCYlsq8Cm(LA;TzMVs6cvF@|?Y^fe& zlGv2HaCX-S=Yz8}AFwg`z^MkVN$uK|$R2}OnD?ofe(rD_(5-xR+-oa4Yth8&seer2 zD^~aTa1A`kRFBdUnSc$OPY#`Ki3N$lIxNL98MTNNODoKyP~x!cFwmT zG+H{+kWYcj1h_6eDoYv#0M}C*bUk|%j`VJ| zPefoV1zQb=;xT0~ImxWan}S#NoOSA7eEJ4(Yc9PPa(Ig&N$w<;GfR*Q*Uh@Xy+j$7 zr`=*H1sN^LW(Be?k8`(DwQUuQ&dqJv&3L!5DN5~Mp*%wfp>&XW0R@QgKAu%+NHBO| z(&YleeggU+hOe-c+@)*;b9)7m$}SC6&Wh?%1TeutM|g@)y+Kc99O&{k7$%ZA3^k~k zQRIv$Qtha>1#mnm!gY8g-0Bl}Cg@sKw71a?G^BaSu-Vq#=rnM>=Bs<`=z*ecGgOkN z!WmiST2tVuCJ-Y|HM`3#P;gu@v1Q2_mAbn1B4&q+N9AqOAq+Pz=F$`g+2u1SS;EG> z%?S#A8<%7b_Qss+6ee~zxoS&`k6MM3#fHj$-tuw;%!_p%>SBgQ6!ZZeGERt+$dK1L z#nQKM+=IZdh?CB3a#q5+XlZHeC$)w}?2}6##wBnvn(xIFAQR}pc4lO^KNSG$!Mnsg)#lI7`GB&;ULe(^L*1Oc8pT`Ymg9r6O*u8yL;@~*BbW< zC#%Xd8js$-O~KA~ITLmVHd$Rf;VlQHoqsQ|vQ`@4&xV&?+!J~OK`#;aG<%Y3z7}2_ zvvlvUt)pOvhD3@0?CR;C9HR(UGE9d3xQ|QrHw_((GEG#zLIv}}UV9VFGwO7*L&=z- zH1=Erip^Z*jymgV&{5i1=6cUi0UYkmc*n-B%kSBOy81eJByx^*uP$t$)9!a$vrFq{ zjgo3J)x81>&SfsUb7K!0o*C@#c$K!uCC5-}dWHO68tMrYJZ^O6B=q0WjKzzoi=CPk z?Boe@#%b-tVO9^VWGkw12P>4qmJyqFdrr)^U{0v{TU1xz7OCDPid2F?x^1Hsf`5UQ6O>jE-m$Fkxnila)Tiq}ug{ zQWL@#MuG5N%(r1PSY_>Ol#41SYO{a@OL8dj&z>n%3T+Dc#Y6yBAZ2&dE~O)?6RoL@fcA>iq_|5nNN}(_9=YITkbj4mC>E9O}C8RtgkT~ zmBgAD=X$6tM3!pga%LJ{Ej9fxmeuhVr>rp!6fmnbAF*+r7TwO9foem3P$5Qqky(t_ zD6_sh6(<-e``dI2Vw3}QAgHfN7u-T)qDd@l^JjXZ;JsuE0hv%5%c+vikmHsEIz8v1 zhT}m>-eOS)rV!6TD%IsDC%E?$$S>_t;OcdmWd;v-qyXfHMPRoVlazedR%woF<}4l! z*aRX=o=9jaNg6FuNL(6kvSScgZCTnEhqs!GrP+@KSyRqNjKbq^Kge>|sBnA~>SA#< z4KQA@jkNTZ-Ax&ePE5y%BsIdZoL_wx&SJjT7~bQ{?ct&o5qtK+q2M;}AQ080BS;!` z0WlB@PG487wD5w~2M%sQK`#ilKwmexMq)1;QEyCSsw7^?hA??YjzZBUTiyYCeQW7Y zBjJIqn0We@K4DR}ac2zM9!=+JNgs&7v)oP!*}S6^$5#ws$QB`cQHlbD_^Wb90ul5PozOCV?-fM?3MA~$sA+iEfjX4piNj3OhJ(tPJBb(gug-f(;gOhNa;qsB?4trj)OlUlFOL(lBKdHqd*Y|n$Zg!>k&5j@gWMu$l)Wkk#O@{_SudL`z)oOo7!xmPR=3B;z`<(dMUyH`m&+0 z1aA@ukZ}(~c{$AWM=qJ2H4+E0NDdstt(OKuL03Cr%!hEuPS-6RSTlz5gG5f{?hYOy z^60HfoqVi=8jL=>lDpXD z^LdY-i+M z7C{QI6jn@creJC^1SA?zZR$#UFm4WBDF4I<$q(=ppyhEG27q0=KRQ!hMMoG<7= zoLVW0FA?Z1)>E%>2_=qrTMkrURY33{E2llkA%2=ySOuziluaVnxIZh-bs0*j-e;~? zez9w^HDQM#bB!y+n6}H-Ol3|`&z)RQk*4wN(<_x|{g9O%8 z5m;_-LgTflq~(TzmJ)0?yMu}>4>yn@A|ZUR&e+BoUi&Unsuz`9n^}Mv*G949ijEm& zt|g;P1$O{xK$gF}iV~q;S*}kJ#zzn^tGuh`@W8ZY6FP^FY>@WSA$gLrjf0)6OJB^z zQNF`$kQpn@phkBTi$=Z(phYP=Cf$Y#(DY9-v|K<~c08C(QL>bV)PgigaLp*`Z4wMS zYY7BerL%T>)EXcv74+ajKO%C?*m4#M$37i9rjSN>!$y<@(n2Mc<2d)Uj~%;N>b2l@ z#aXlkLR5EOc==EUQf#P2^m4MhuSagK2ak!F$+G$DgQdGo=XkFsT=5w*#L+7zpNCXd zIda9rvP|wl`FH5Y<|KsYSj_}5jRwU@TVP4XTz?`~OR56di4CF;km}PrqeSM_s}MAd z>J5mM+-ae0&|=SoU?y3yW$lvpu3nw{ak={Q@#~Yrx5uyDQvNzaWKD!NQM4OKn-Vsn zwy7911Z3|+XL`{gcRcuzkt)!N~ zHc_&vw1a3{RTjdwq+3K=Rkp!fB^yPwDA`2Xs%-Ip=_jVl59e4LuE$QP}wSN39&}V8!1~vwpPU^)k?`UwxnB9HiB#mYMVs1C^o6JNZ3nh4TZFeWhHG!(XmL{sM&25wFcE& zNwSr(3u=yiRV}Iu zX|*M7A~qGZY$VxRB`vB3)T3%v)HalDNUf@ks-d+;)eWSLs@W~1TSaWDrqNB2ZL*t0 zEtM+T3uzldHlntbHY;Lmg=|$elGz(Ihn*|ebAsjxO%0NSeB zB{ov6g(k?_rL?Hp8$)GQ(v6a>pskeJp|u-S8&hQqWZIIpl`X2a6||_@iM2r5MQkR- zt)pnO0=A0SrodZEYMUDcHj8AI)l_Vh*+sOiqAh@JB{n9+Hib>8O_G~Y8#Y@jU^bB1 zL9~IjR@AnL*lZgVEt^)*Hh|hyZB;f%+9=qzQ)(jGfwHQW#WqzfsJ5z>&|5`YNNl5I zjj6Jh%B8VZ)Han3gJemtjiEMFwxn#J+EK8!Luo5v6KILD*rM4sp|XV78&svRhSFP7 z4T^0=TPTLbQ)O);+DdICZ4tFCr7eN6Lt+i8NwHP1O{Cb9Wkt1BZ4$Psnr7e^rWZAM)X-jHVHUP~^8v|)Z z)TYojRBc4s7SyX^4WyeP8xd|H*$tw$MX-j%2FX)uq}Zm(MX@Sv6xxlGYKgQ-vPrcqt(2D4Q)-)3n?$w} z*c&P~ked=lWks(7RXJZEu|YRg|wx#P;Dq|6|pAOR?5 zwN0p|(pw_h6J-sl7ST&#TWl#C3R^0xVM}TwWD8`QLN=Y*5%y zwI;|zYLy#REtRqr_|-=kiLy45Z5G5CiEM$iQrJeon@F~*TM2B4whGuuwHDEBAlj_T zTPUrF*e$9Jk+n&&64@%YsHW1JQnsizqiHKODz;Td!E8%qM#k34Y$({Nu-cPkn@UZB zHc>W$wuH8n*;Lw%g$!IzxxM{8kK zBt7O#8m2;CjiGA(!SHz}r61nzhX8z4gvc7$8Jc zmi7IUbe{m2G%*zw#1SjYT=&ykm&*Qll#Zq{_I86u9Ha#1Le1o5@?Q19OI}|JEKh-w zvG5IQWdrtRpWB1o+=pxE<*b)Dxc$&62b7ZkX)!32oJ(PGYJs0;}YL& zH*@;-ywerTcJN#Qh!`?;&0-!l?Vj>tRz@4RDJsc`xQe}u7su&75MPrTxv#s2>ht86 zqtUDi$T@n+GNJtA+rvGeK?fU^Ke0-^#sC6HMhHIy)rk6&unw#?UCJ}A2_2H| z$UHwy4kQ)`--La(;lV~wnE~XDevde)I#C5m>7UBM;OcV?uH#casd4L!(H6ec%$r@*WrLux|ADWjORVi5jwVbaNZ=|-MQ7>z-; zD;gDY;X1ISodbd5B=1$@SZ*zjsz`!)R{D?CR*%iAXcyd@Los4-U@5-_ft^WRA@RnAU%vS4oBwurr4+7LxI%3+ngk=Q|p%U+W# zqPwMcS~wv%QebqsiPuvEJf5LF@IJK$uRWhsoSgbx*<%!;(m~36rEPK6XCK%^pQboC zM+?#7!n0~A?%hl0JL+=t4~t7O;1s&Q$T%YY_7vslSz~)ijmI%WgDkzf!*Q3_IDmB}+2{lx;J<$Ap zE87X12?Hp{my+4HkHN#m^nBt2#d1U;B))YGKq09LI-*}(Hyg25fh7dISvN{JYY;Yj z6;DW-5}CG_rg1%8mC(#7&U^#sZ9AQH|yC9Xm2^pSya_fjR{m1>T(wyCA!k;iF*gWm$YIa<9Ef>8W|0FY84!T^Ke;rbml zMTI`AAOjzae)z-%Wsm_<%ou$3G7VHkfngM@rf6VZnFGECn-_l=BT^CiH`lmZSvT^Pq+^Fjv`R zTC~++fK#atesm1L73C0zToYOyF6$-V1)>}E9tDrj)$Y_rFzW~pjq12n1h6TlaKqMs zNJb!xlw4KpsBjxDE~Tv0RfVWd?P zM5FVKGL$itO!6i3sDDLec5j>kxqCEYPT#wqUe;H0s5?oBrf4$O6eT8AK+P_?fjIj_ zKtz_GuV5f7EUff#{G|ISb8bIZ3)!;R$29e~JdfMvUqhd~(=6!q?)GC39jc!M2BFB+ zalVg*Sqof1@8w32Sio(4&#Lm#sVJ<3`MvJU0=iC__1Vs9h9c;ypJTyCn{I-? zGlrY!P7Fge0VuL9#DHBFC=tXMQ<1E>BFl)xH!Msosjfzqo0wz#d2`l*`9~<&qR&ATL zbG|yAk6e)vu3VT~@s8FEIK&vUKnVywqNmvo#qiHLZvdf===|!5si$M-c%)TG z@irMjv6UGIPl&A8$+0ZST?voiwt7m0ak{RaWpU@k@N*m?L;d5x#X4Z@CJke_W{ zh3mPWOXfW~+o#oK5A0~whlhu?Z^U&7h12-6&XZ*PfL(YN9eoAH_-3kmGmsG}d(|;w zUKEW2k-!}b=e3+Hs}A!7RXPrcD{iT=sRN zngua)z%1G}8y9I{EA23uhm%j8suR2DZ&85HX102!>@}3fq?(%G8X*k3IqC;2CwF&5jL9-YJ5<5CTA{o>WY(FP=u#E}s7rXCGL;{R zqkD)i?gzHS_Hocig*#GJg%Hhnc-Dd)JmAF`C1To}+8rA&Nxd!Y@FoNf?f%kMT9QRW zl(HFIk9~zKg}Wy6n6T+>Aqq0LA{?Jswtg_B*Yn)858K7Ejo%1B?`Zr!f};EbIjyEj zJT9t^zo5;+FU%|RqP|3+gQNK*tLWV{BGWG!5HLvqfd?WgRPTVK-l@WX#n^)4&a!hE4n1hH&zzfg{Bi;Gb0~b zW7Ollo-{lW?%&9$wyS=)=9YC5WF@$SRN@P`1NRU(W;Ghrz=A)VVr!?KtR<1hGzlQ{ zamuT)F9e~)pdd(E@9n%{^`lU6+#(Z$cp=|d%cOX~N3SQ|@yq~~a`=$2=8k!*gTtUa zi2F1qhwOOs4!j35^e~PC72~GMWsP=nDVr7q82mMB%U8aD$<4m*1spyrnHdCSB9qFe zhLi9kBP4cPCm)?vavibk&y|rCn3#q_ir)_{w+fPxPE7tnMKBYRpDU)CD!$M%1vl6D zHjv~aF+vu(0vO!Ui2MW;g;%0{1ErN-Uv06v{bLOSSI#SVDzSOczcE{**JF=0V4+*) zcICNo6ES_|p$-IsG7OZkSb)yixEwo2b*s*OU99vx5OAfqFqEwp7{KzyskN5Cu@sEr zaxnCW7$=FE20Nf;7z7P0=Hlb!$$6F>h*2H%Ku%+xEjnWArwv_je2|0X}I&+30k{*a^>hh8t}y3@PXNa56oLX-spnxz$!`wC!RWw0W$Z4KJ^rYO

@I_g8pdMpw=w7wA1)l!BtV)fQT7uQwgD<_`eC;E^P0x_s z!c0poa_T#NtO;~a0?e3OcbV1AXqZ*{<|V+KIXMDESiU9)!yx3f!oGNGR;Vc!bLo`f z_-TaHl~OB&{O)dKu1{~dmq14ijjYZX5?8gCJ5uS?4On}72sBO`EhA^XvD{a6!RXK; za*i6iWCR9^GubdtJnDTJOb}4n$ID>=;K_Np*^tbTgt#U~LQ<{Z9~%UU zZ`2k=z_cT`)v7D?JL5^>8E1qq$w=_3OR_es6+d{I{8IS>d{k(DFbHKUPm1P2rq~TG zLP*@bFG}7W$ij4{UCJ+vSG`8agT1rQfD|!pLdT$TG^^NOb_S{aB&P|d4k$s37--${BA0N zW5xDH9ZOe}yX59yM)aP1a@Xl^47go0YD2NZ;nD{QEx7Xn3@2ZKP31c*F=+IFT$azt zBqS07dl7L9jY*L!XU#Zc@(gC(x_O_jghZih#ST88m1!-c`%SHjkAM;#p)(*fG&1r0 zNM4otOo!ERXLs<_uKl*cp&mk-J1#r}ppia$nUEgR%JR)9zVf?bI0j*0B``=(Dzqx? zI3+Qm$>-6zZh$MPec~E27re`$o$=~KpvXd&Ob-2Ur!!60#2OV=Lm}CKqIS_BwB$}6 z>QTDe1EK<9IE`pVPSdJyTC=cy!0Nzs{82cb`NB06Jec^V@mK*@w@X6W(TC*1`34d` zKZAgX;OCc-N}I|@br!1c9jKMQ&J+wLUEhf6(R&u=8uj%`QLkip8XWm$*2)$NhsG-W zRxJrDO1#o~B!xho&X%46_ju>9^g*j@qQ_=rZBOHTw3Ti>`e}<$2U?IOI-?+NKkfG; z~Y1m7*V@80tuVkW=U=BgHY>;~QS=%VGBG@4 z#_B6n#Qhj~s*)xUfMx{MKzx#@}hy)vr}r%s>7-cU(q=xSHhEa`VCQ~(gbr66xN}s zN^l{mM6GG=JUPfezN02+Qqi(=8=~q%NfE4>F9BW+yzJoaebQ1TAxVc411&tUb8rcKTOAIZA&F^Z zimb3yZB7Hjpc*d?Te?9ibp6n z!j*4gv(BiofGZ5gvL1kOP~_j2Si}TqinxJ~Ly(Ohq!~Ka;(4ONqwEWH><))J?QNb6XIEd&VG3-Nex^>9NZ1UExH@n^3nvjLo z)D%1>P{5v~-%}U|n!_qx+!QI45YlI>uS$BuIx9Z&UXRz@kP}MA$_*~viuOCVDSJ(5 z{^{2}C3Um5L**YWN?1m=7o_C|`S4sQ7&5UjOmWW7Rzx+#C{TOUV8iEdM7IL^z-m|= zZf^B?h;$i_#8ruE)(kh#%vsA`1SYjC9Fo!RZx0Vt50E7V*w!qXEL-`OFPQ{b&Z0_Yzz{jpTGB5N z6lIc>(^OgDk?rQ-Xdc>c))cEAGPvkGD%EMGF=Vh z%b#3i8|TQaecMyNCJp-3fhV6=Grgsv)1@}jmnl#i;|u8PV7-GHZT=EecJy=DvPyvtuZgent( z^<#jDj@#F^MgpbE{uST@mrsmI+rt7Ucovmh+X%c^XZ14-{1Q$0D zf!08mQ($aW4dDwYFiUc9v@VkFb-sg_ca(Vlqz(CR+AH9G>ij(N+5tvg>tPxlUM7CiJ@VWd{K{@VA>oAlcv2g{`U>}~uenRY5(e?30yvQggU5Q&%#tKLgCeMZVnh)D_;7)ZpAe>S;3_3#` zE|E|!qri`@h0NztwShW7oVvGuOLe+1^IMa}izP_B#}pN{ZW&8gOK&z&+}USAt%lO0 zR6@#yt>g&MzJUetZu8QHP8&HmxX{Asg9w{$Rca$7s~#KCGV<5FQxorZmV_`_gm5Wb z>R6bB3bii6>SZ0H@%GHWX~gOGhRO`RiG;lwdFp=_h}o^76+Qds%wh@W+V-fRi3N?x z-?f}ULeIECkzu*Mg*CD!rsQ}czhWcFJ9INTJi%x;-dc5|!7B@ZQQ{qSgo!iR?XNBNqCzR^;Yvp-41l9YQk+^SQZvM%?Er60>5E70_8V`^wczPf(8fwT+r5sCo4jR6^s^ZZ{(;wVR8= zmZ)ChWo8Vwo}(6nc(?B5Iy_e`Swe{)aVPcbaUC0w-=WnSg!X3vlMrR#BpJ!F!}7ALD^>!oxMB+pTF}w6?z-O)(;+|%evENZ`dL|2 zAb0wz453nrXYR-_a zPa`9snaoD$0`KCD38ye5qBpKEh~VW_b1abSxTT8M^e0LrKTdk&A;H`wJL(CJT8PxC z%UW^Rxq>tHJ|I)V31P2?A>6{nZpIicjQZTL`5~nK)hMumu`tezJQVRv2_k9xe!F++ z)K^-LFHw%mZUMJ_e&dfG^nhR8^8GI=9pM^uae z)X9gHWgUIckO;V#q9&PTb`?pv~dR*q7!<=gGfFL@o0w+$|2$T zkySoUFQZVT$a+2@2ugfEoXDul;6l~6x=8>{g>N9DP0W-}?j)$BJ-`4E&Y zNPO4D;hWQQ5rmWPGJ3RN;^L;)CI4M> zk+>BDRs-aP#Y#-AUit(kG7)^@R3%b1hG)Xz?7=71&GVTQ(Wg;+NpG|`w@yuAb~evC zsPDRM5v{X=L?-G!dxq0{H0UQAK37(e;lUttVT^=J9vto#VI}yI*;=>5mX=0*-aoT0 z<0X27oQhds9un+FRJ_2Bswa4J?JkazO%~oL{)h2 zcCi-4VGnzuiJLvfj0E9Y%CAwG*`V9XJbE1A&f{^CiJZ0A#5{Co}#Z+%~ zQ#T>SK;eVw&1W9)#*Dy3QC3v~cs10LXq{F=}2Sb7ohp-SaP_hwtI?gnrrLYKIZ{I}ij8t#D z#YJ6wLqkc*6n5j7nhVV0bjZ3!hp;Ay++2f>Diwv#5We==yopbc)qfh`sUk&FdAqZ(j_}+acCFVLo2; zaw8XR0|2QEKEdJ9QHLWYHW?TMCFwB43S=^vy85__?#Q-1lo!11<>$XBm zV?GqbXU0IR=oCtGk4%f06y$J8-fNxA(@nl$sD`z;HnOi!kDkDQoCFYKl2i+7^s)d4 zpa&lT=FJtO#ACwLf3_(b`$>t7wUuJ@CXOljK(NiYp@#vjqBPC_x-0N(aC?I*r zq>3MOb#l!H8>@D*@5)&-Av{y5UrBIu{DlaaW(L4RWo}N#mP=yz8r#sTz?C7b1RWZ8 z4{D4A3LLtT8Y&=56bbgi4>2^v_#n4vEsEBFLd#PTlf?J$uhSd^lH?S2!>RebNJ<4% zXOfZ1Np#_nRL2U@Vf5pGXlhmE1YP92>`0j4dS6C@5OvlPK*m zFnk1YW$zXVMFFhPO}8{#-NiNMg(h zAWtJcq=?k2jPN%IddWrG6}U2%axSix^xptk7O}?fs=e8wl~oh*48ft4!--x4l9)CC zp^MUid~dql=F8L;)Lzel{Q|Cvjps6ymf$@_PCYblTNy5a>I$lb`1!0HnJ^1WR5e1j z0o&DnUIgUSlVbYPh$Yr>O@~FF63IZi_4|Sr1gex-Va?t{Vc` z%a98G8C+wYw6uk!@WbyQcx|-Z=tt}$UgXXAFk?k#N{8P@Muyz z(!mtgRl_jWgE|@zZFD4n&)Mj2hZ5Yktza>|k=k+)BnQw-VHJ=ba&0cXh9VLXq=3F( zBL|1zrmn-J#6A{As$&+3X(?9sLhf|4dw3WC!dC0<+X~LtO^#a5$#>82-;~-Reri-r z=n+37ki&0gKWB^GCU*rYP!uWb@1nx06z-b2bdjh!-O`MR~`419W0WC1N<*N6Q5`?2<(u zk$*MA>`UXBaBa#G1=@h&lO}WUWz8d1`qfX!F#wI$&~l>fGU92ZS_w5VAl{X0>lmLQXD`~6D{!u zNm18I&G^NvQ#tE#y!sM|&U~Q(MD#|zmgJlUz2x4-?`Thlay%P2B?I#fi>>ENS^UEN z^YIb+$zs7tsDM>qVAE~mTF=Lt5d=(X3qgD_HAf334yuq#k;&!z9y6B(TGjA$yajNQ zLDDD&#lkfR(Z&Ti#KlVHN04#9c+1~UJwTld$!?Yj-Y;00W?k5zYEk)KxxLTfgzMbx zlrh4U!rRVyH$hsUq4ug4E5HkeK38o__NGruFH%XMDpjPT%*#PAS_42ALj@(tVDdfD z3$apA13P4(aBM{nQv<|OSY8A)VAhvt4LSSfwY6bJ43r7szJkvbjLa61Q;*jEW#WjTP{JX% zN8{Wm)j;6GLtTZSALil6Z@lRVO1=A}P)!TcRa?y|%=+sjF91~HyKCAy^-1D@@ste+ z8cRpF4DON4dBAtyV1fxvnZ}2ru_a%F zBbVVWgorPcMd#g2umV)nvB;kISWG9lz=D~OhJ-xxAj8^hYr3PU-Uev_Z#4!`Ii!U) zYgpm?3PeeP*LB!9C_C5Ns~#(4m+2rL0WByG7Sz5!Yc*ULpN8PN>nEvCZ^j+wr)_H0 zEfn&UDvf}FiHqz*h@Wv9Xe)XJQ(wVmgH6g9N>qo0hg_rs!Zi zj3SV-I=~Jn3UsxaN?3SBWF~j=IV>W^$b0&rpwNN1fTwoG_iH1{XUdtshvY>A>6Kx? zABNc(5XGN8HzQ>~M`K5ij)!zbqL-F}g&0FpzBCz^IKLmDW*U3j8}CcyPzu2Uz1&iK zpv9tKg072yG1+#s;$tVDWtXWi2uq_VI29bHWz@$uo!WpD2Mq7WV(PBPAd8XPSd^9G zmQ}16p1VL38<5+GQ+6_~xvwGuZHQlwdP2|2G|am;x^s8}ETtG%$t`o)INzOrGWtTI z*N5T=inHC+`Z-pxW{$C?HUQ*b?iCB06~Y-^b6NwOeSGrq%bT}f&%w(`zG1fVGTQJL=R!i%k%(Rc zY6Pkx0>6fy+$xZ3`c+G8(AaeA)wt3)C{wX-b-HO=I1>G{?0gt_l^%BqSzZE$+Y{pD zB+-~4>IvYuVj1w(MN}TvIeTU`P?%!U$5nfNUaTA_5 zvSbbN=pH7fz8nlif_2t_;CVs3S@*$?Js>fR^#h< zUy}1Qz0uS`B?PTbn!p*x>Z&t(S* z%p;%9huh0N8s^E?gQF}8JIh|bY>vFgJd6r;WJ!oW6QpKM74wH9tuB>1s#lj0bPwK% z6SK71Z)4HqP%!EgeXZ`A(){(n1FGs2=Zh)?Ud$f&eh@ZQcM?k98XlL_K1CmE-;`QY z5akJU4h>g9U}X^^r|JP8}q<6{@fpDiUin?&~arwx@XcEl9)y-C5&F=QjO z1?3GDl9wVCyY*DK4y+zqMLGIZ@bZs~7=-|^+3k9%h~3nZhN|pw+r~SBl)qbnZF*ga zf_$A@6E7t&`))FNGGX0-XoSNm9CpJ&N^>(<-KEkB{&VtxC)mnuje?YG_NuEBsiF4< zD5(7YI`;;A)0gjpaB;Dc3|M0HXj{!sZ+3*ZBLmu>7%#m5YiS{khqX8Y7hIz3-$SBs z%x4bdcwoTmqEh+7U)-2b$;Imx&a`Nv^tV$46;;-v-<&z}3Z%?dL8VLHa1LCm&nCnz4GgJ04qNLWcJc|0 zmMai*aM-@B&_QYS zrm_13WvU`#Kw*pS?ga3}CuEJigaIE>V$srjQM%q~1A*=0)|DptQ4Q~-8!_S(RtRd$ z4DSs<(W)Aqsj>=Tsnr+GRl6gNoDr$x@g9%QWEl_SEh#XgF#;1(3OpnrkZsX&H}$S5 zfYxbdfrz(hK)Sj~RF`I{0PezvQ;V3R%HKpL4|0f$FL2oVLtsG^oE6AoMDgD66MRw~ za5r3Y%=WosSRcG;%F71oLcB3cPl_q+3`4x7)xJ*K?spl6;haeqKCOYIVow1zKLFRv z+=kvGe!pKLF9li=eboS7o`8P^btdm@tU0tmxdR`vo1oD31%Bad=gRB?%qZ8>3YN(= zE{|Aoj}k$A+IL9OQbQ4 zZ7n}~%%PAn!k~kpOCnuXt(JTtU3Cl7khbD7&j(|?!KwrEQb}11JMVSsiuy$wj3`U% zG1Hf$k%AzQ*!gO^HFWwme2Ei;#;Ui+8nx9f43pa?Qnv)-PbqSmGmC_yLCo5|Ih9yj zpL$tjXB{}?92|s@`ZX{!5l8K`v|->x+`lw&n8VgIENPhn@^zE|MyxnwNzbTqz_pyQ zy&HMn!o!%Wmi!0;AsBiM4u}|P3A-D3E=3fJ3&NxIs5iXGUqN*tE3peHCp`k~SJAEr zS?Hq&(isqDZJ-r=8r`z0`U0blP`+}3#a7d_3A(poX@>Z@kcT?dJz4gft~d#H#VU;< z$b#ow^Af7I#h*@ZHeP0yriU}7b5Bb}H9NT62|q2E+trk}0x<%YIzx&Y8$y3Q;m8VM zmaD;9Wlvd=4;iUK}^DhzHagdDu@!cO?)Y@}R|;`wNdtAfWy} zjb=9{grI|(?cYN}pzJG<5vXSIoyoXLg=eYA3@05P=H!*A>X)=R$&*e$SOai*2&Q17 zi>y7ht^jKBS9IH!u8(CW=Rs-ldsKI62zD()dlHUtMek#(G8<%mAlCS=&&v+_@PX`6 zy&SWyNmYt&TrHtV1Ka_%E?W7y&AJD}HRaA!t1y9a3eJM-EXBCx2YqK*`4BFyDJ6qa zL@*);)`hX;t?cIRpGUJAMhl{4b5_YzR?zCTe1*5+k?Cz6cQrS#G9I^4UzBPhS~gxt z5$us`CqxWF(Q#7rxa)CBg;hj#LcZ%t!u~SRN54wTv9baX=j=nE>o8=b4T6iYC33VR z+s9Nq3N8sUr+%Z8fsyPdvgUO#~!}rK!q;h{L_?VaF@%#X>c#v#l zW=YacfN?k%x0|n#gEM>63zyVc1MVLPgKu~*kzs5Hy3p)$u^UZRSoPqHELr32*U%9! zqt{dEdl2fU%y&MC6q{`20z&+UGy(%c@b8WfxQTqr3A%H;cU|MxDQGGX7uZXM_tFE| z`RmCYV!UHYVJ^FugHs8T2nZTmG_O1+ftN8uS7R%5`WqGf2Ur!~ZwGwtLj|K#ARx#= zRC_A0S~M5D-!U^^GO$u0fBbFbOutjdU^LN90 z=(9vI&C`Mc8?n0*B;$4RbU1pL4qPR0yq z1Tx}OM6Vj3IF3IL0L9@z%W_d;Lh-TW>UT&Nu2ylHQ8%MUBC^E&>SSgkH`3)O?|s?& zh)wIIaVJpcZ?A52*KSaJX#SI|SBTDlq%<&{9OfRE$T8<&+vkGP<$ zADJb2M)MF(E0jj<)vG!~+YBzE_KM9muWaC$5Dlr~+(KiLSt}Q%fMu4*O@JeRa0*x! zr^Hmr-Qx-H{GC)qRRgN7LUoqOMhV4qZKhP@!}$^dE({!k;&Raa*g*mkA-z?0W(ZlY zVy^q%Cs#b!7&3m}m=hkH&RAvnOQ5Ga*!i)I%y>hMLb=n0M@?MklbdHHe}Sar*s za4(pVv1Z(oLf2(DL>Jo{#1K?u606KxR|Xm_mS{KG71?M#_OKmYK;BKjdq>Zz9E2Vj z;c3&|fbpP<%KcWmvIS6o3>3?poLfASdLvbo&lo z6r0+st`Pn=ac!~be{AY~r|0M!p7TI(zoF#>B`OUQ(44Srix z^}@H$v@lnic_6HcfRABfFaibfU6HB86 z`NhWZ7&vU&9K`pBrZ>qEix(g&1XMld?qSQY%YKs`f}FC*46kP)VHhU7l211IATU56 zz^O}#=SS019p+t2p!26kB{lK)pwB_pCUYO0WO9hTn@aS1Yhd9S(QV&|{l~cG@=4{X zUx5!ms9Hz}Ze9^b;1_{59Tlu0DG|=?#Ay}al)nsheeIWj&Wnh&eWxvl~za0aB7f*gdj#AGeQg0r<(9-m6Edx~Q4II_bNIZMI8 zH$vE?f{kC2wz|Z~sz@PGP|hERCOPn?BBVRdyrno+8_KuneF8WUuAyPVsFH;_9Bz>d zwUb0vCn%EH`K=V)U>5~XV)KD*G6Lf0T3WzQBm|am6rW^gfUSrNAhcBSG3;8 zIk;WJtSChyD1r~B=+0MQF{oFqxOiX4GdhGIj!A;%_3 zHi2IDz*IwWvS{=ojcB;uEA+BOO~G@Mh7*Wzvlf%oYjA}M2xJ+V_(ISN3*bYOB$VG- zM(xDYbxkgf3*}41h!gUQtm?SxcO0;Guc-i|d|BNSkSr0qRqJsf-6&#=KZ~p({vebp zV;`(hprP?wiyT(2J2kPn2Dx&IinYgATA~bR)MnWn1wT7h3OKs;-@zft3LX}(kVpqG z1?eT1P?Ev!{YV9B%6s1QRCQyqh2pO&^7lY(rh?_yRRc?#+Q{2_Wx;P7!~*ek_NAl- z2xMva)J&xt8x1IUq`h7u>s>F2xBz2mL)X(~H0I{83rsK2&AO)WrHxmdRcCiHtb-*w z3g(gPUp2(gwC=)@eH`G0J-H8phNCQ4sg_$+OGLEyRI@g4VnX~#VlcRIVyQ|G@PzXX z+-`S`#2k4Xia@U}k6pl92pUL5L)*qVY@o5IcMfUCfmV<276=;mDW|3%oTxDx@|o^t z(b|ia@}>Y)cec*?<3rWMUKzL4`8Ee}`8D_~fU%L>mHIw;X*4v$mxaaYcNfY<+GR(H ztam+vJ=p8mh;9}e0XYrx&}56xi@`A4fC95GaS6_Wq#$#}*Hb3V!;90M=iWj6L%E4S967M2dy~=h;6yj$o)=|8lGLt8XQ{=VTW06jp$zpR{DtGq@bV6e(>3?XDzy3VYMq+Id>h_)EW;%5%I zA4)J~EHxFu_jSJoB93v{wDWl`#)tz|Ziv4RQ=T>UzUkq~t71XiDheVL;qv0~upHq8 zZuV>Uj1qSe;}V>tD5T-^-Nuk6Fnq45!C}Gc#9T%Qn|^0r(z^Gg#G}S7%p#D0?3#vw zA$&{NBiDF^H!XN81|`%6D0)QYVUgexS`p28SzE6?nmcYS5x*=CR73-hOIrOl__V1@ zx%Cl9yt`YmneI)d!NYq^_8{2Ns~t;1Q=L&`$lkAEnn3OlK#A-Mpsei;)iBumzhjj? z0nA=e+CwF#@Q@K zr@Z4!yispPkhZMoyI?GY-Syqh$-kq*Fr?hywe>+`l9hL0Az@w_^Nur@wI^f-;Z< z)Yy-=a>s|7_($J*R?MVWfFx^uYKk4^s?=zs_MGy4=k&_M=qH1983?q4UFgo+=c0O4 z#|sV=Km>D9!ElDmIJBUn8muZ5JTQ6K;S(Le(K5?D{*}b;g_rmE#w<0IYs9r z<*jqfIgNg=hf~J2u-1#BuMjKse(wVI8?W95h6P51p>AQdHeZ;ci<}Nt(!wf8X!%$NozJFyA<&~eutXWB^Ds-+5o8&sodesF4uIzZ z!?{S%{?qz)m}wvAFJ1W$P#^@ID5ED!pC+v08?zrF)2BmvS2-rysp(>` z6g(Uzfwim&12m(87ez)P3w++DHecT0>h}tqxIxvc14AVJ=V!IC=>=Yf*0@9BWqbwW5~*y2BKoi_<9zc|%YG6(16lOirhbzGbKGIL4n1%a%7LzDSEM4Hr4{O&uDPLWnL9t{hdT~&3 zlG9k@b`B{{w#T>0bE9(N{=_3YNDOdK(^v_NT~;O?<|bd8*~J`6LY;xWDH1x{no`00 zgweh~67cfu`do9ss0T1?dlfnlcVWZY(~#5P_xV0$K#9u;F{x3Cpj1>^h$T{bT{S6Y z5==ZKX}&lW#)Cf-vEG>>mqMyieZvN3Vh%mksmNxER5COmq9g1jAD}~u8|iX)_)KVC z4;4Ww&g4pa$0}>m!6WdB1i8MISUMA;>igY|&iX6gUzfOnMdn4(+~kM!AjB(x`RGQ^ zm^=u*TX~Aoa(rrVtFzCc&MFUg&*pO<5mzr#Nkk@7`%0qJMoDWJCZmHTu#Zk&(Nh}C z8WpLzR}4A|6$Y=~KOO1lvZWGTHJ-N6>Rd;&ov!>%-EBJlBH(o1Fe6$@`my@nyw%Zr z5#vjidL|Ag!&s%G&FS&l62zCnm^_|I<8EXg8Q|IKTScZ9aPSUZW6QboQ)6412ksq>X3JfyfwbAUgR+C$VBiez#8H>Keqr)~XLlnGGn4pm#85vu_wn z7LZ^Eg^^3{Zcq@uWt)YS`1ia=AuyPXRE3m+hPm*tQcq?j(6z+WYa6NciK(<2N0W8a zb!skT{fP|i&OBX*;E>HDZC{)B){wY3mu7>V@lyE=kw10$n|AlHo@~wr=(`FzJ<~jB z$Gr_HwbK)GZUx_YU@O!^JI+nC-;YjmVNOUIH+E^6APXMk2NEL$#9OzyTqe84LAz%j z7}%0GEe!|Md?$D}@s6c>MEK1REM`SGGuHR<1rg5?L6L$Ad{a>sxbfl#>-N`!V(*$q z>wQG>SVdBTkGRQ7rSNm??@oGH3p77fAgPod8O!*lLdsQ}6Yy&(+=C;|?Pw0)g#?+y z3P2Q5goIR4nEteds1Dck-F+KZLbMu09?yHLtD;Z@E6%g?YqbBAkIcU?fnM8$t^5^H z08q^j+`*0%vo?@BvQ7vNp$4i1cC*2Z$(6 zcmYT>>Mn@t!1g*QoE$fGTH8x_ozNL()hxp7dlAH68ihDZ z2{IUH;zNUZvg(21T}IG;Mqfd6HxG@bGQ;F>pO6`a6J);<@Yy@4wvKszFTx!Vnd^XrL*6)NdJuhQXgqtJ@ArNMA#FY-GH!;{3vP=O4( zs%F$!C24rASzU#_^;`SWlVtVHL3PR5p7C=j=s6%SKm-fx&H~kFpyrLyTSz?DPl<#V z&KQ`f6_3zE1y2jjA%Pxnh6GTmc=iw&A~-aSiE%M-?d;21mtTl9QqtbHnc8a(mt`lX zFBTnU9ri-X+wLQV=BbUA7TsQivW^IO_fynd6PH4?Gu>u3gY^{~DO1Bw15kmu)}Av< zDl$3c{W9t8b1pE^_{w7f(e2xa(4u*t-iHL-#S1}a_fD1$+X<&z*X@Z3_lN2s6IDEO zEuZ$UR+58K2Vv9Xm$wkyFE)CY;T!?+SdAp&#}Vb&IuNmkD3axlMFfhC^yAv2o-x;{ zcrir^5X(}jeHO_c$)5sodPvN0O^Yz1+#P3AjRKbK$C(`70PGBk>S#5YSv(^YnM-gj zBks(8hjXE)h+IUC`LxN35$wHvU#YQNBdeAljJK9oC`D1w0ZB$;3aDO*R$wGY<}MMD zmH!|gxTO%NR0Ht&Nk}RWLNutNEn9&)P%I%7j1od55v+zE6hEd~(}Fv9z52PB{}I%1 zbWw=AOrpU;%809~Z5UzgX3@qe27O9gAQNRwwo%aO3P8mQ;f1cl zg%`O8+>gk$rT~mW;pM<1h(hg)uT2-3I42*Ndj(wa%VL1v4?kx zHlDCZA_G*$`mhtYBqwgYes3NpRrqdbB{b)?8ka?(t#V^-c|T(^^X3UAq2=z>nW+yT zeC-h8ma@2`@T`iOzD>#$Fn-2Xi<2m1o_HIBMVR<%w=AO}w8$4wpQA~gGc!@y(gOp7 zmMM(UuDH5*E7slM-&cB>n^i7vW3gcdd|w&AaNllGTOwE+a~+^LJWbiWXoe_a2W6i? zp%e+CRYqh(S8!NT`dsr^R!@b#jLSB_WTKY3=5;xV*}|g&+lfOx@%ot*n~X2Ca_X5J zv&#ss(A_kZ5m_%#1Wg#4G*G8rsfvPu(2Z_VyW#B&n2f=ec!*6saIp|Wt~xJLw`5+_ zYUTtCFJCLLPecY^UpfP@lHyM8i}7)KGKr}Zgn@mHC@gC zk=C#-tG3Df{6yge(ofK#-Kt1lvK@AD%&pfo4+5ZEO&teqpKOPhpBX}Ey6~9Nyc@+j*9={WM5Eqdj zEZ^#MY#zsgii()OKX*k5K-s>jkd@dfpyQUN7{T+hCX}sWME$b->Xcl$x_(5eGZqLb z6;zaTha>DCL?vL|3Ovp8JtwQ&CX@{#V6D$KDSb2@AC{P)OGZ(T zN^*mK&3};MgFlT?A%wXB$$4a-A|#Q$t#|hNv$MbL2Rn zmWis3Noncv%|9AszXk24L+B3eO20wNndgYDLQR+@%Go?aKW+L9wMU=4KI}w?PTtCE zHY(P;M^e@$(KF_t)CXYF3OSHm?L#IU0yrC-K)43kzDmF+Mm;6s%h6L#oyNLGdJhLH zI4A|!qlG(J-a}$LOH%UwexoO}2iI!@$Pkji!lo&d4tOUWxM{p;LS>*lx<#{cZZ}Dt z%?2HC@bATlEdwyU=s<~K`rF#;GjtbYdqOci27eXT!Nvq0hLKmICK`@J^2moF=*nRz zsM#_E%SzobFQcv0Pa;h_#+ScGS+&);XvWfnRI~&@)0a(*3$q!Dycnr30x#V_EjT$K zd;{`d5oeFPXz2i(_58?JI?&&%UMTu^sFLP=Hw<++R2d22xdb?9MiNLBgQtS33rJXB z2r77|mL775JXswreR;2(@-UU_<-nHiao}hwJ)DdMOwz9!ngm6;0&k@43hhF#%CyYP zP(o}&u}RlluVI7IfP5;;?OpN-w%D-fyZr8)nDFl8u?5|p!w?AT0MlxXOM&K zuFri|{C|LWw!;}SP!Ol4BS4$U)XN-9M|?^T3n_}U(nL5n!oN`v14wEG4#BhHc_1&j zhtFeZm=~u7&jg}V5!X}5DRROU=EZi$GO5(tMg)HRzUG3QgH-Iwu-qQxw0KWHVaJO4 z9c?dVs!v;J3bE@X`+8lKSsoHCmksFzm7VW|nP$9Tyr`cyPClwk!SGyFd7SMpbW*RY=;;ad*sdZw4a2xc%SI+t=hNnxyJ@Vnzd` zsB{^l=V7X$%7OAGl0zR^DLUiwEmZK#>F;L7t}Jsv?XW)BV+{?Zm%O&(IicW0)guun zs*VRqbwQo3;KK={m5P!gDBzBNMTXm3n^VybczYlzDz+wMkY zjO>uCcKk-TWwL>5TJY5&8<+oUr1=CKww9EybPN}Wnt6Nz>8JU zS)D~e^tfXj%0ajDk z!4|quU60PacT}wpc-u9qJG$4Ri~}|kINlJP2qM}*#P05<5f?&$jp* zH7;UPZb7I?xbkgfPOci{lP3g=s`J2ZexP=F`pALyql0^Go!$4Lx_+1|weXX9e$d0;MDUKBqu+V1QmVG!bK;R@NMAT`;Y^ zx6*jxZN3=e!VfIlyGt?n(PX>n;k-Jgy9dPe1 z#JLsG30Q)(byLX_=riVJbl_(p*T|v}w+T6Hikh3@$_Tp#3;t`;IA>NMT`|kfJK)56!4*klAmpcnKDen7^xYyFT=u| zObsD7KxXg{JU<0GUP|aPm{{$F zvcqdcNFGd}JCp{HW#`CKZ_e%w-n|Y+gZ1L+xHlZ!Eo}OIr?FPdJf$IyCWJP+8v#tI zN=-n3!*on562^dVw-&@x13=bnRWi?wr&U|Pb6&1AMrItxf_l{sQnn3ZsoO!IDG<;D zAEXczbqG;0(#h%jj-RrDeQ;M(LsojVlIU+zeV?jz%Hw&h-ecy4UapXk9eI-1_|g~A za&3Hw3GCy1$cjKHXC5+)U@{@0&qb{CGMG?6?OeCbGkT02VN{zi+pTn5`Ym-Yer+0a zg0RqLl%itZQMU`VOy)0_bULZn^kWTuE{5Q4!!^3svn^w9Dad{$^h-sloMyYr*ERDM zb(Z7CGj(g7KY&?M2rNQ%c9IweiD=C3Mg~@2g>*isQ|?&eH8EnR%mOuVD1=qb=u(HK zg6q#lOAu@MMkl#V@}!!>oWimvoQ^W07wdsVs%ix8*j;&Z9F|0V4yj3GCM}N6+S#Mn77uc*n08=d_`~SD z9v#szz}RNQ>lgBSB&>wY!!J&QO1aMLb`nHf=OHr=W?1g*3~{F0ih^?AX`?W$BiMO| zZqsom(9P$mUZVS5v{s8B7T$_3egk-!(P9Hs>sn2SiZIWKQXpA#5NP+y95;)kN~3(L zjVSU^P<5Mef&@9o#Lv^g;!9 zEu8m$Rx$$>msH7L0vS(@SfWsYzpfM`8JJsSN&`^}t13Z=?=6{D`gb%;-)NrWg2>qX z5fxKSGb=9-A|f3V;tH<5+Q}In|m5-@c5nj;;~Ah+?(^w zFOej12gpLizjv=CyrLjYv3;P8N!NK?WVhwP>zEyvcPZ!Si|QRNq>F6zg^v5RzFcm+ z1;RrHwb8Y>xO9I()M3Ea^jBf1+#S;#jYv|S2J*zZ+`C(quxOJxZ~_h~5P@*&y9?jF zvB5OF*gFcWA`QpHFOS$2u9*+U1VXC}c%K8-3xj&oVN{+z)?j-j7z>dwnsrzfM2anu zbL;RSA=kUr8}!nzo)jOf`OqkZor!6PeNiCXjwZl$z(Zmvg)izK2$N(dKDSTjb;eHsK;JjX!Hbk8)V<5M6yX ztfWTjd^$tb-FA(=R&fh(=*?WRLscbmBxt!v@=e3=#nNdMRwCxo!xw9qD2e2}fExWj^pNm!ur2WpA$Tu-^rsWI^Wa zfUUvOU>GN%iBwl?7kNIpM!pcI_frx_%aL<&=2SWDQhnqqH3NX~SPTJ=bj8ZK$BdL^ z>g@AIT@D07rH?He?6OfA5xJ@ZNMo5)(fU0zY`UI0qCsK}lPSdd;Qi#e<~~oG-5b{X zcsI?Dhs6Y=ylZu^S0>?40jn1>W1`~h$LPm}8l05JafZEfgR}yuxSQtT`*Yz~=1RQ)`+^`4V#{3B6BN#i6fERuCa(CTNwO`J+0+1S*V)7X|oc*bHzlTov6_ z4NB!Fd=iQ+Lm*?~y4l~NR96Qa!sln3hcJn#?MfPzE{6#FBGf%^stk9Xx~tKTMAh0m zZn+NKG^ZeW(DlqK3+)Vo*qTT2N{E7)d2|^UDYa)PFWW&(km*8JSFO zIaUZ=poru<>XqC%LSHreRE&n*A9|Mg2_za2f_mq4Y&cUr^cPt9Lp4l$xjC+iRl1<~ zLTO^62bynH+^8-)LPg$NSFXGhQzo$rcRBQA11ApDgTf+OGYF%=q$`g>EjSEmEEiZyR}RcxnB~M`E5)=pTKWestU$ziK%|f(2i&Sw`$aDFlR%m0 zks#TAMl~C3OTrt(o5$QEmVi6`ElgtscyN~1!n8Lj-e7S(Yc~s1jA(wG@K_25U;>?F zd%E*fb9xXCw>3hi&ZVMhK4)IQVlu*21>kxgpHrO4ViAad zNMegPLqU6)G7?*rsLVmyLki~PTI}b6>uah6(}Xo5oiZi$m!uv~9KmAZH!e2u7!t9R z=$9lTvrtGtWbmC!Fr+F%>eB%c)7IU{i^i$RM1Fnq!UHX*aEwDds+dGphmGx?V*yHx z3*MS|446Ez7fK{oyNM_F9ndr38jd3`-NZgFSMo5~ai$})TRf;4Bg5)(_6RhVE<{I= zU&z3i=RG{&JZG5!8`eEhj^v~9sYqyoHG=DVNN1^L`KhNk!ZHX ziHkX&Xfb|yd-JZ#(`E2@y@73R9wP&uU7E0z4{Acqz%^GGis8RPch?T67bF=DC6$GJ z=5JzZKX!(SgXjZ$ra}YvcZV5Hizh~Dw3LKasHTQoF2!uOk)q%rAR-_lgd8DpNR$+& z1r}w@Ln(sP4>8H0FMwZghlT8d$276Bz}JmpX$2nfM3zy`xq_*1F19oW4&4ds@>m@OT!H1ZK|DEfg9a<_YPDPlY>P{g!Wu z?G#k5d;}-C-rH}ZjD`U@#R&bFVA+ew^hnvBCQv5))znD3c2(}8+cAJN$*G4i(aBV* z1!T?X$ZQdTG7e<+d6B1}L9NB0m7gvt_9tYy81Qt;)rR|Jn|z7`yATZoUW?w&4iI~j z(nA0@V{Zu9VvWV|B2oN+!MO@=_aHRy(>El;fSvjz$CLhWGT6vcwx1S(YJzfL|`z$EqQ8_oGi3}bZnX%Ury?M>m zU0Jo6YEm0TMj-N4kbS(2nLH)#VMPbsK?$(}>?bz+*LEI$Q1O8XgakNVjN9;o1NfGS zdi3oHhDc(IpUR%gW`R$ZLtarTn+nX^V!YCgmVg*&$kgJNbUpLkj0l3>HF zW!dshKw8*0BMYv`)m`pR~dz>{EYLxkXj zWF_JHD_%H&zJk~tJ7N=+>#L4;7Lk{QUm66MMJ|Bn(_kRF0z@yj(cTG7rF5Zg>gN{P zN!pKEVO2o`fe)aZ^!|*GL9=UuiHIa~>oIJ*35bfdv4=4k$2JS71YM=Xr1TD52Tw8H z1@=*n4qqm=istx~sE34Q(XVxD>8ZOhd4ulDw8WPc6C4L+y^b@Cr=wCncwa zWRSN2go*Uq-#5#>#}H5$7|sBK^LJIYOg}*gJeMX&AyLud7rTuHU(y6R^X=Z!p%{p5 zM_tHRZWJ&bBnc_Uk7)vgJ89)bhT|A0^j5BH3JK|q+C8Ib@$VOs9XOe8+iL5Wd-gR-H*2+Q$ttYCS zxoe9Zb93jp2pWKne9%kyuGcHQ9ZClT)TFCsKzgXs zU9^C*?46!rW4>VC)SEMr8Pa}OQzIw_Pvx7S;kaPRo@(`7i;duE(*JqW5@$Z?ayz@t#kn zl2h%410@iM3GQ`U*Ea;7Gw?XNMy_%-LxT(*dSf|bF4g>6e7Jx<7n>)j@&%qxa3S4uO6Wt~xzmj8J}|A)%fxAoC*Lv-yl@M9+iCPr!XAqq+>( z!wYHSWhDnV718gzn>Sw9Mz@%t_sYrVr228gHQurV(w``a65|e6!ch3Ip;Dm;pi46) zKv+PZD_zN8Zr)lJkUB{Vq}}lL6#;y$i&cIpsa5x!SV%WMF}SPcandQErV*A813wSh zM83&Pnmr?1VS;dUAUl3ZfqUt1gi<#ShZHoVxj}<&kf%nBCe0?E442zd#*BPZQ`L4^ zG74aD7g394>w4xcK&+YAZ`MW&bqV5Ol6Yqmb_fImcb?E(E*Nh)TAB?D?x)jrlt?@R zRL5CNxiZl0#XV%7OzAqxl$WTgPz)Bz$4dNxaJoRQ=KRI<8=2dV^xbSE zL$3qLd+C2hz%Fz-nXTeedwEot7EGkcx>ZKSN;&9K_lS^sgMDP8QUTSu zpQ!qj?IFxCv0EuZ9MF3LS$KFPDlNt?G%mvO#NeX63@XYi!)dVaI2ThV&c6mzc3A*4vq-fm&#Nx zv5V1ntuDn-Dk`dh^X<0V_b2g(?0Q3|N3Nvf(enLdE81+ltS;zKj8ajNnVaRlrr5`M zdFV74oyiv7$W^Z89UZKRkXC4RsJ+H)hS(^dAvtOStF=#OF{ka949g&_-fh7Vy`V}l zFbMif8s}<_t=b&h!6$MIzDop=p3)97uY-Nf&_3K!*M;fmCLV3mHgKM%hp0N) z&^xD+k?pWTm4Ugv4ZY|B_5`{^{3{Z0uI>y(qw$N-jTne1sPK%CTWPR`5UAGa{nasY ziUu!J&Jp^djFyNJTf_<|W&oz3M#bLD2By>$n6u7ort2pV`8!vU!Lprc|&vsqnrRLAIUU zpvK`JXVY_>ix(Qop;kn;x;fTbbfh5y422RFHz$1XGL#xR>QsiDT;^n>iW~TvG~>Ho zg@=Z;P$kMihouhX_cs~erZIgEPNnJ>M5R47SV7QIBhF7f%L z2P;=IR3sR1x*2MSs)WF-W?HsS*=2xdPeyQk4KgMfaV=+9&yv2(Nya~$AucD` zhmC~Z_NE@)XaV5pQ_2zDOQeYBs~o246(K28?rcuyo1A)`XlyyNVZdOa=fjJAK!c7bBtsgWsrIhm6#iaX)V+QrEg(S zt@XGbmYLD_Uxc_$SU~Ri=o$&)=JA_82c3IfoMI!ef2y^iL$dG`bD&acb}Vplbndad z4ZL#mqL~jP>&ompQ>hcw_P2*FA+>{=*dwt;E#{_om);9YRbQ9CBBB@p-^>P`j$n{N zp<&K75J(59RVxWkkw~0#ZM@|qGCrO)Pd_{ph%Jr)^&pYO#onKeUA=0aVcFUqCyJ1c zj-p=f4=ylBg^Vs7>_?%M9#fcsw&PX7ct%q#E9pN<#{={BqEX$?k9= zw5bo(<&JVfwF6Hn>II%E+Flu`91QQELfgSHXEX>inlEz56K?2u+@;B5PY2{Br>Y5^iCfQgT{apy4cbBFqUm<{1d>Q8=MY3^}d_6e9K8 z%l4fGtjs}|Fct!ikWh^AgY#4Z$yjzuKEN1p6Xzo|)Ov({)c9KcXhK7`LWV`eiaDGT z!xjTc_!h8oDKs)V7J>5tULwlG5m^IE0N2=~Zc?7^wD=c@-nguL0F*_EC#pXrsf@T} zZRb38db8qze!cXROg%d2B>^Y#Pn9`_*&3cBd8xf_R&p++MA=E?#syx`DoZ729%-3Ykhh1PM8LC$8Y&|(^r^Xy zJ|rnf&sgrBFl^MsuPiT=ACXa=su&pTu1a*HB3v=OPDjY8vri&7vq|L=5Pid8G&ZWQ z#q61HOGi|OF~*-a+TJ-0?T2mb>UoHbuRlu_q!^kNfb$_$`JnRXET`=}>M(LCFL4Fi z;O(ikiVcblL3#B0^yqS9dK!mB_rc}fqpUT{o1PHKe@oaKnmo{JATSiD$T?tQ+s84O zn=clbc*iepCCQ^ys7}v>#mTlm8`c7|17s51LC+@bUN$q1fvhERozEivkXVHZ5aW|M zvE}=4X7wXM8*Lt@J=D?Z=Z#oVi_&dQF~owkA-^^o3sHTpHY0k;j$O(yXaco<7#XL^ zA&@m!8wx9=A8T~w5N=jM&$?$sJUA&_ z_@viBI7-t#U7^eP)Z=X~PEZL}BkmR#ygd$J6_eu#G0lND!B>nA;F> zaE4Q0YEf!bSisH0< zbxr2&q8{x6>Pl$C28A2}@CX}mLJ;zj1d04KuOQ)%0Vt8Z%)$Wob9qCSCBy{lHi11g z#f}-2Y4t8ZSXY6sU(*dZGjxl`P*dp8RAyCRO1{_*MS6+eRbMGZc{68<^iS?Ugo=0I zZa5dYv(-(F+TN%IX%BHLsRsv$wc=O>h7jaw$OYsS;3=Qb;UF{bq1PtmJq+Mz2T(s< z#GgK1atovtT^B8lI5f7vXpo`7{Ru|6sW`HtrGqkxNM-XuqwlJ!*FchDb@#%(Hzj9; z8rqsTFMfVZ3}A_@FgOSwgdqcsH85)kFbphnaEYLFnP{YRno)hDDNpVO@ema49Pu&x zy~)pkWs5w3!A>WNLhx+H619$Kz28D63(FFxxn5HsK~3-kY6Jmx$^&yHl*? zb!tU-!K@<@j+&&u+;~=jG7nW!zLoe;}LCkqh8PeeFw zB9MoDv3Ix*KW#l^u26bIAv^GY(P9EtscZ#s`FNK8kCa8#1H)@@2bq}SIC&QwM8^VA zM;zZ98jPFMtU#rgViXp!Ln$-asz~DQGh0eU@;aQL?g?0l>Pf#lUFhLd-~~O~BBZP6 zs0)JRopzI?Bw$%n^R$ZfJj^$*HEUoZgeb#oG;1uT?$6bip19d?Razp^?3+)jTgzSi zRd%Q?s?-u>xb>xWUe>VBw+ydG0x+2+L3lwL25C8_v(KkRj$1 z9wJX!oK_Z83XB-QyG4gSB1-y2`z(B=(kDI3ywkyra}r-Y(7Am!;D~z$;tW-acTp39 z5}1r4a{zFqiz);e_5uf}lHSs5QC_(Dh#D(uP9l)g%$5(v_7yCuD*S+xV+t5yH#n+4|0sz4M40ytyz7{dmL z5Qiu|gd+&_NCfyr$-cWPJ6I2?lLEn=h+-C6BSry5prmSOHWEpuCn^2n?5^<-WQIQICfG2KbIT6E?Ugo2UJcl&y~;@Av`5~uj8};zn9Oa!} z9oj(r1=~`pHeYY)1q7q7>!Wy+OID$ZJf^@Zi6IzK|%r(42zLWdd&wM!ZLu1UGzZ=t2k**`MldITe6euVV zRQUEF9tBz8nMjV!rlEqWOAfa(2Xwa`O1ts(8Oe}(Pv!@i@ZfOMp@?OvnvxKZsRP>> zASjTqq|iT2Jtle6bZ!GTnFt}Chlv;~c&CY)2L;=XgbyCweypR9mD#MjSP?E-T|L*c zuvUd_wL}2F1-tWo_H~1}LUja-xes~nqXbYH<(tSL{Zf@gkfa`gmUkmxH78x?i1CDT zIqg9KUgZizG7Fq##8LSBPV>-#P?mK^a#l@*6^xb3lh?uQH$lcn zKQS3MExLuO#~^;rQ0`UkC2r|LpN4rA8yxet_nSBJ|1jqX$@k8&?GGNzv32b2}eskXUi;-@`2 zUZGr3TGN(hm;;K8z>s+hrWN&pZf0iu zyegfnMAVw}yl>?sf^S3+cxx6jQohRXT|VjKM0VGymx1^S%ea*gBd7J!!?Q;zIbwK< zy9}XQHcGHE18_+(S0ncbh6xmn!k!@`k|us~VJqh=bq&o%>n5U>U(uXvN}m-0&{ctI z3OQbBSBwdDD|)Gv<9;A0Ew51@qk(3X6HHRr?*Q`n)Y!@>4ePFkSYq7ACqTtbXJL!? z2m(n$(})h!8yTG$A5g@DtOmFn4LciS_D#HpI`IgVYC8O8XoPoH<@rX(cTekb(_MbW zhG#&ZGEq(TYUrTc%1K0V&CA^iG~dL5*RbG}!aObFy+&8St@l@hoRqYWBBdvY7o${6 zFc|S11fWL*+mQq*XKi|NnAhx!>`q3})et~gMe@GN7WF*En zVk{VXGzdZt0z-pmM!13)>f%J+>KYX0k~zU8*HNh+4@KFelZHbHILt$XgsNnAN7YaU zhj(=VjMCDG5@9E>6ofS=4D@5ORMf(y*W|ZnM}?)l4TW4n?dNqD2PB80810Nr;q?@p z1-wu1kX1Z9wVku}T>ZvW@%);;R0 zn3%s0G84_O-eVjvM`%<9ei425#6(|wMd|$2G-c~B{QNJ>ou&-YeYsuen=pGov*Ym! zyai~a?n-PT8@*e8A)*}qF7X4%O^b@JhFLasrpyT1=ty0e5Dgd|oNd1b^&@$T{vO+~ z=6x&r67;4J(P6@EK!F|XOH*Xx)piS@)j~f3ouL9|Hgi9VTrMyZn%TmA=)Ca?KG@ls zQw*RGfzdUW!AtZEX2Cmx2LoEZS`mO~ga|AYaVX}K2x@1L6g$}f=RA31r!#l!G*IU2 zV)9PIln7tgz7xucuR>V?QQc>pxx~cErvZUl`Am8;_!e!Cr#$8Cz{FR<0aWV{2&J~X znld?Ls9P_nz`G7;aTPsTYZV}ahhR+E^q_a8M43c6;~_*9+rU)1oC>^V42|xhXDGYu z30zT?CE3(z)I#VYOdLFENeFw|Zch!b80MvJEPRID?-tSABIUp?t~VFrlYo>Q*W!L5 zY5jf$!SIgyT4%nN9aK?+dgf;$^q6`?RE>^;`5Nv=LB^?6NLqY=x=QfIHta3a=khOF zS0SK1r-in;DeP;;c$A3W6a2NjoFLi+489^1dm0sTQLl?yOYtc}9?WlOHp{dA(T=IwoW7~zXxJ_ImaOHY_gAQ`SE zqv0q-Xe6=5lI4g+WSmdIxw6k&siO#5%yPQCkxl+3j`MUtEQy0`ZEQWn%Q` zyU8fCX+j5&vJ2^RI7b7xca)=#-pdeNhlW~Qj!!7Nd@UZl9Oi70PSp)ofn|EV$n`v^ z=zyJlzEL#hx>k#0dO%_NZxIsop$g03sI?g66=AL%Qe;QL8B6+1z(x>^WzT6Im*sFB zG)CUMI&W-h>DcHl zE@z$;1?JVpYd1!I9+XfO7xJO?2~@=f0|TPtNy-vSiSkRNnL8e_YUy$yEy-D+LDp`D zc%#?J$-G?$V)!1U$=Wb^kfAqVN5o_9>KYj81)|?qoIJpjhNI)7_fE4kNvx@v??a)e zxCjy#)-BCqY1?M?_BV1tlkQ zNt2g*AnS}DY5XW}O)DM~`2xx&AGM|}%HC5$7FH_kn=AlMK(W6`4^S_JlAa}P;gBD= z?(Le&ERZvieAz)_6UjiUkwFF&p=q=;8u~f+lmw8#tZ+QYYfoW*&E-~n3Y?H`t#Hn9 z&urPUoUhaEy<-oat6c}aYrfEpxNARM!x@~zag1@^O?FSZutvTjPir!--1`3`=E}=> zLM)&}MIb<~gg~g7lGos0C{pdS)SfiPmN20+D~A}#fdHyMN2e-RSgq_KP5JyQN`hcv zg~{s(+k)$U6ABG1qIN>rqa9A{8pK%f*ei4?F+jkAoq;M?f`~^%PC~_Xox6%Ey*MJf zh4*izfsrj_9)Yb|hhr$x$xHd7Ir8E`OC!qvqOhv$zKfv4o#O%S(*%@C<1LH=l@tum zaQRW1Syf!QoJ{T^rCs;A^vj;GsgI6j%%~&UQQU1hy}|^Vrb{wmTTr|!o~%1@QB%Mc zEU1%m?ehK_9e7B9zn}MXZ?-Tjv(gCQD0_DalG0H~FoB#idD>ECSQzlk#Hmd1U}J}3 z!d|5mM%-(=uw>1#+w0O3Fs=7!%e;^9s7i|FFJuOSFlj@ zPF)O7vfG#tNkoi@9@~!n(P&#ZwKMR%W;Lq>>McT;hLn=pqS{DyHAHzv3A`!XH|?0{ULEpxMIyl_G~WFKVMebMK{Ebi+Wyh}^J3 z@Prx-gNUgp;|&nHR1RV0S6J4oGEeGVwEDD=1Uy6z)=ii33*c63k_GRE1#xWQqsl0Ru zs5G@Z&W>2Ll(pqjc=&9W4AVMR9$|vf&;-ij7H2|S2S{Gr`fU0|9mys1sC=QutgR{x zeCbzztkVz4z!g$eib&-vVu-a`J4=1aQbQWBvqe`ZO`TRJdo)xc;kXwFvdYhpIlnx7 z57IU4NzS?O5>42txN=(3!);~tZ%a68NE`$t$^p-Tg~KWjB##)myjE9}Cft0b%~{2c z4&pMhl=y;>4#xqar-cSEO(1awVstVT{pl#7=V#D6UR%+^>uRvnJVi?gfC<|bYuM(RaK~df5_LJsMZZqTi$tNA>t}D zD+N2>ON$1=L7)MO7nWO_yizn=c(k6Tm6cRf?@Cd)MGX6`dKNKk-|NWImrlKk^@3!% zs6AvvvgmFBk_HW!acf1{r&WhM{tZ(Q197waBes|k#5V0oi2-4NVf)OJ7cy;e)Ixpb z3f1o;utyOaMmbfNeMIO9Q3Q48nrdxU$>jq+J6I(iw3{dy5rTUL!j|`nk8OhOm%y{D zSlkV1(@Py1*gIGv^WuQvxbH+aks7UMl616ho)hPbOk;C}1W1Ti$vy`}CxOwppBKn_ zWhe1zt&sqlU|Y(b3-09C7^EfiN1N#s_kk`Rh{|(CgEZLonKKYaz>8A&$g#d(BPbxfwA~qm0mN7yPBJ#!5 z99-lXMSI{oQa^NEmxNh&NNcO+d@)r9hD&}Mf(#SkF9nX`WLcF!LkW54UABf;{%Z8f`#J~) zeZfU$)Au5j90B#9&Bf$f;JGiTrYatY=mI)R-7y1RnbAR40`a~wRm|+lf*sn!W2&v%xORtA?)9+CblmoWj(Ggn1TM48EB| zKZt4X%ryR!DKafdwMT_*=^A!41%>w}Kez>IsyIARE2^(thGUQj?%J-Hd(IaeNb0pm zo~Z_D72w&Fj#CI>GXf@&!UqE#1Tk>aK@ShX67i0VjyX?h?0}o#A_v5*NwETdh(Kjc zc7aS{ZpPFcg!Wl(d*;|!cgpuLRL^Kg3nP(}&j&BWFva5v*t2Puq#_IVnKr<{NH+Rt z2dUhv$_Ub;0us_wZ^U+_P9agarA`B~fl=HpSs>skbrG{v!<=xL=wWtG428S2iyGP& z+!!UI1ueXveZmZ^TpS=F8wMB5yi7NCKE?93f{wlcATkx=gj8S3I8YP{s{I8~w3Jy; z$;kk#56X_K1zFRXN}TAG-zk+wMOX+Fy%t6YURfX^m4ab7u{TM64)sLOx&;2N8n)Nvl`bEZJ6qQxX2ejOm7$;J~G8{0_Sx5z2iscHrDi6e+wr|j`yCkYBc%fAy5!n}6 zy6Qr|P_ao-3}7!p5?L6cssI%g7GgelkBT~`H%Vg*ip>9%hE$76?^G%>F|kp8n0^Bc z`&w?Ivz*b5p9RBVX;l`2?SxDSJsN7b1Ovh&NLfm)C93}TmHqi@V6qBL2pN%B1aN^7 zRgq}^C-*=q!>Xh-lD(n%)eaBTsT{{Mazv%p%l^w0&Nd1jlvn(YqliBwsuU~uC(EFi zQ9qO8Av{Z^)c8@igGW;8&LD@jZ;Zo7@44%vzZWdJXPvXw`SsJ|zn;8&{gefI^W)#& z)Z5wJ!!TmM1fVY8&DLseu|d1VtZPo~W%K2q~Hr z0bB6>&wU_;#b!uqVYFK0T-kuo&N8(}fbHqK)q-&{xc78eu`ZHzK<2FL9dfG0)n6{r zV;xbbr0N8Sq}NgF6Rr7!jtM`V{fy0qg6(Tt#Y8xWGlSW17oL4^QQN!YogEjG;_%Gz zd(G4aim6-cB_PK2zN~8Yw59v3J~qC78|AH@z3C>cUNCe1-(Iik z?k|k6Ot)kaIQmpN-qs{T&ZvV9*t#j&!!HlMQaZgLUaqW-7(xKZvolshaFs+55rbX@ zTj3#JmvTjhGk6DjRbIV#H8BPjxH+W;;6xC7ppt>6hbpCmRSJ_Pm%N~mEG-635XX7# zv>i+m4v)TzH%w~8AP(qW!jiG|xLYh*J@>dR$fYG1#H|4S4GJrhY+5+y!KX7>@@c%~SjR10$6k=|OWRQnhemmWTjH8MaRk4m86zFMhy ze0J{cren)IW1V(%o_%341fPgIS?+{9@aL+k4z^hZx)^~|cVo`QhRv)f)5?RxkRE74 zEU|BWB`6p_M7$L;ctO=E0u(@6HrPc~q%F6(fi};*xMReh72i_9bn;ZE=-EA?40{ADX+F$AF;@smuk#Fpj>8zNA79l=Ew zd(+-5@p+pV&_M-oRr-gI(GM45`M^kl&}jm9EYwb?zj=h?Z5XA9 z3bxTgeb8mS2V)Ua-gZ+CKTpGIGT6?*T^Yi1B{lfoPQRO zD!bUIBGL*Lc?P|BuzL1(WC~v54(QaZ4G?CH$LKevn#^p9NUz_Gj zLo&yf7)q4r`Km_v^&V-bx-~L3dsd_g$h~J&D%$2Dj&};b$gT-YRh;u$l&Oz-GYJ6- zpaK!75UB7ws0#K1=KP1Iw->u>>eQXKI0(4(E_Ps6n7zOH?-4i z%A2jwkGrG6L+T;|>z&H%FC@!}y68mIQx)*1B`YO!K)b_cZedN9#fI$Wvn1RmmCIS? zo!GOhvMk_YdS*vr0hn|gQcmt!d?QllH;ll4A({_`oaR8FIWu7 zYfJYjq{F3A4L_F{_-K}?F1vN%yIq$+u!*7)Ws7FdY)WsEUV`G&!tcuqxHIQFvfsjW;eTL$;Pgi3v&ox#tBx%Lc z#Lgj2%qMKE({4eKN-j8wX(F5^c1r_~2W?33+oRGWUimGS4$cS>l?rSO04V5yC2lEpkavYrUg+jo zBZG~|Zkk9U&>>`kkj4!YTaq+J%34MdV|Yl$0y%O0HU;R@+hQLm z2Op*_(V8lk1Ug8Xt$be7@)5xO+LJ)h7a%)jeSRd3zcJLsgpItS`?j8oZu;%&uLR0P z@a7?@ReNZYkkN2%0R&GrflpNMxO<1*oEE|O&Cf*;*cH+jK(>v2a3`m^G>qC|jP`Mq z{qzYW@-sG6Vdclfmu#c%p_P$wwAF7dJ_eWoCWWW*og^uued=1*Qh6jPaVN&`W(Pe5 zqG2F^JSl-1xMSX|3ys=FbOx^QH-bDd8j01X=O{xk0Q=NJVN8LX4emY(z9RC3-@kmJ zDTYJxE+iW97t%ingFTIqnu>u5344| zt(*3viw=@ZS25g^u`pQpPles3RL+e>A7jRptMW!Y`*?7vP^&u{U_`o@d8!D;Z9oOl z?MOf~3*-}W2znV(P+KpC?;K&ka-lAUinvW|@Kp#rE+mX*!)I71k;9IIW5rNQ0A+!i zhMhjb&$eZ&pqJ|4{1H!(6~EQ0xjEvN}Y26-&ScTbpYEInO%vy+js%}|Gi z(``_d2BEl&z2I4t@^l2zXoF=XDFr0@mut~eZcu7Owo-(@SV-H{-1=w9M>)bckc5sK z@Fq8m|9l4mM4V5?0tAS{hCF^J#k#WW%dmX}TM*;{d@EX%O>iOzerff?#7C`C^E=5I zj#O=pV21eJNQ$1NF*$ph{WOUWPbBHcMoM}|0!55~(i$u$4}oHy852XLBrz46!|8({ z@C?WaTu;aycD}}|@*Y#E)e6_kL!tMGq3luno})8h>>esqtp@rG$6Jw*kBYm~FK4CX zz6~^?VnTt$gKO zQ8DM2sq-7iWrvgDJ9)EUeP?!1(O#k2_~;DYqS-{VkGa^TTBjOXyLC2=j3Vtv5T}MF z9Ks$xh3ue7!A;^G)XJ2vQZJOlqCwoQ2Z<1lRgJ?K{@5AX1{Bi^rPW&jGly!R!{kLo z8k#aZS??8G3c(OBaO`?+>n8A)!D!abBBDw97{n|;Sh)4g`NHe8)x7cm$=Xz59$AjKXDs2JXOaauZkx_Kt`wL6&H^z0#$~ylqiHV6fLtt^< zGH<$B`A3LgzCpUf`>+*gm%a^ZwY=pt+U^w2^k)j9`2yD#+mFj)q(H#(oZQxdVFl1) zkg&5vsIL_4^f^J5DbuLu$D;M`8_)5@o7n~ zrJOILuy6|6*<;{1EEPJRgBu$W9DEZ~= z!a$6eRAdUz>m2YdL1IEs-O}>EIAW-46dt)SPZZq5P$Ovk^=%8NJ*(>LUGnIXQ;0m2 zXlO*#xE(VQD&0+t=6lQSvLu(a5+rnN6z*Or-ykUA+snCB6^U(P1E7t_AQw12`{FqW zUN$mso)m$F2x^9Z6zZw{CWwd(lx|$g)fF58lyr~B!2(uys4i{SHA`4<@2%- zZfi$m0(L+UnEs0}_bZ|fW864bF0Z@BluH*7)dvK1$UAf0IR!AWHRq;KFIJ78U@@_* z3C|8TLrI-OpcS@ngRI%cSozraOBaOSMEUh(lDjPqH;=^snTwX<&^L>OVRt8}>*nvRI}? zg{qi<{3}n_l53Q1vT$cp4&^Fz@4(uO;$z6PAS4s5pWey1N->0yXy*i};2jX;!jaHs zL9^L}Qche+L-59(H!OL9YPGh9@~07kxf-aA;F*co-Q@3yMOajoA-J83T@4W#inq;< z;85ZCMTi)~Sv(U=20nW5>*I^#b>r@#O22#aN+~M6IOp^0mOY5x{q6oDseaGue-a$I z%f**&;`P4vK2+A6c7&#-Ajyq{xa!5U-bcGN&G%#?QK-lwSr>-TGjEz?P35R~x==Fp zqJ{Yi-z=|^=SD&-R?3~}BN!fSx;5_O4mOrdn3%0BGgap{)!`JhBDq$ozBuFd)s8?! z{j^4!sD37ec^!x3p5QKSK&{(N>HwKGTS zv=-+2Oy|PcW>v99YY3ufNv6*r*@k704q;e2J0mrzn;ap9K5(?hOMt9a>N^;PbW1?+ zJX)}L)i|N7bbTj^dme<)i5_oSz~grsK-7Xc5jDu#I<+smls%0|RdZqQBc$&oh_P0x z%y(Ei2R`1Vdxci9@pHB>L{@l2(ie*RI$eZh530P#ZEiNC8R3mne(cMo29a7iDH1#* z5(H$QiNM6a#BleX=Nx2MWWNA6(t?D9v}&eDDBTSUQ65eWNaq1ZmLTFbWcSkHrHcv< zEFJA3(~T>QyBP%f6IR=_b3$(W^zSV!BKH?;glBEu0oOJ9kg_qEaqN^Z*r&lKx4|fd zNeYMIiz_4;vN3T}Fm&BU2GNXealAU`39cmQG)RtVCgseb4Z9BWZvc2i2e0kdtlXD| z)Gde%M!1ORL}mm(zMVZjXBx>M93UPlk`2cO+Ry@qp0b)hobtt=2n`u7P>-mjmc}q{ zOh7L`i6%;|w{PX09i~~S2N`+NwGREW@ay>mKnr>BqSuHTMB)!|LiU$om#=852V3&<3)VuK10yme9uQ#8*6o>ZG&1oHkeyPZn}H9n$FtY4T$W2a63XIk zI(cpC&@;sC&kMf|l4-V$hunf^_C)EhF21a!8DSYUa0k?B^l>y|S6hUG+2+Mh5qFQC zeEkWbVRmY((#@nIF^DF?M(BbxHnf7XyBtvB9XJ(mc?%&2S$OJC?PeaVKp$yG7JAAI z_$bd+6Vn?!44m|Dg-#9$Lkby1t3oj(At*{`sx1%YaYO3jF-IRGlv(sDLp~UpCo|LkFBZb-cTa|$zJA$?trt@$8__?KuNN$Af{3_ zBRF9B76=x)S=4_s5p#{8$$2k5b2qpNNtXffVkm2cQ8sB!8y)O1nQ)Ou7g#fDRgJy9 zB!X%1@G4>u4ESx<{?`q`MYUDFqclKxknj&tb?tQt~aX zW1OBFi*5S-lWY=Du{Medvyy|8DB=Sx0Yj||P6>3*SBG#_qrkas*fNaM`~?5 z0ZNw)xV)z(?S|E(1{PJL=@IC7;npeDAS>MsgQv7EqC`G@d|f1dx~z5eDLLMRIV`q% z;hdyHeQ=LU>j=L;GaK6{PaKE><;$0igD9<^YGNlK3&~#OiaEf1zg-HzEGe|>#rDb7 zAToizYp5rAd#41df&(;^Ja4o$jAMJAmB(nWo7O0z@F$R#bKbagW{5E1(D&VhhCBQ<{fsxoCMYQ`lJu-n+e`*1A3<m=jkP*U=B>er~7g-p_ZSLNAJ z4#>R$9E%qnxLXqtHMua?0G#e9hY=BZFxueo5nG`46Qoxd+j>#hq5b3c1PjA9d>R9F zHU_Sn2e+8)S3Kw-qr8Xk-P~g+g#hv^$YDr??!Zwxoc}|EI@@a-yfJ&#;k1@27&5#vvo(czcJb1>982Bn`&|x?Z1gl9U-3VGrL<*?EpDMlNnJwPibCw-YEVbKC=J-JPSbHrY zdxG9QqcoGa!%EaQ0YH{Q)9O0Y9jQr6QBrb+)I}I0V!>&8YNsQn5JAJL=Ue7EzO|h& zcGt@fG3P8|^S5$QOOA|S_;H{R#M{Ob*TX1kgHQk_CPE>H4NhgS7|J5C0~df!_bo!+ zs{)l}Xd>2iTEW~N5{$*7zHn$$B%IBue*`Pt_2VdlC6Ea4 zAc>v6TQ|79?Cv;WWOw14%16a9$ILm>l8fwvtdX;!UUDhF%zy*y5#H$IwpyoQRdGXFpC64)Vxdvc&5$q1ye|zu zoH001d+bpLhJ89VcIM-aMFduCTj{z=$%gBalfaIv$$L$9Pl2+CJPtNI@t; zksU~WfggwAiMvSWBE*qFR}g{WxET|1sfTj|u1>53HH+*UPG_NGd9WQpGfV0k0%X*D zU6^9}dh2k)!@xS|ggHtKzFy0H6_lH2ZLP*G&6KmA6P_sx&z1JY_$sahZ%*T7;2i)+ zkx)&7$sm;>N%aeQ&3;#fW&(4TOkYJkm?Sa=WPc&WN^t>ZlqVKTfj8NU+JY@z*-_ei z1>)fY1A;=ZgsaCCiG-ESYZW;Nv6WDusE{D0CX;mx|9gs&R)BPaOti^L-HC$?B^K@HYH|Y1F4HWKc(+762xzs4TLNUToa?2haM$%a!&wYz}56>H=*89f>(4ZBv*A*$r`Vhc}&pi@j&w57DJBE;3yut-Nn zUE#Hys+~~P3iLdYx+xYf3*v>Z}wR;L}d&OcI;rrkG9FLGw6r zsgAZCqQb5i8e?N&NjH}nuzEq{lFPMwwVdGxUgb{iHNC8S$g+r*a}4+7PNM1Pc*GUy z)_;!TEWo|OS0t2LEA~8e@#U7x-lg^G??a&m zg)yxiT}sEuuT=<2-f(hnD)dSGmq z0S*Oo@qRbvWMC8&EDj);1I~Xc>i!GGM_TAG$3i!+9F{ru|3UYgAFV zDe}b%T6rz*C`a=n{EEupBOwKcm{FMZZnqG52u60!e7zRmD`6Rm;UJK)3G#!8h$#IM zgv5PRasq!(#rNNQUlD!x`TvUC-yk71aG`X!M$KCRgRkh|Lq8qP!v9vfp$!lEGky@v z#2N|6f}@n0Xw?p0C<^McpLUw&2p*!1Xh9;C^28R@ChRYWf(!&BVN3~lwg9%F0M35B zx8S%2nQl{S#w6&+ApB+%j!RJCjsBk^D*Cj}<Zxe6kcQM^~6gA1DR-DtEQfhtsy9!f@QUAGlp++C5A?f9oY1V#up|vI1G*FiB*7t zsC`Rn!9O#H2pZ5334pGj%piebrerv1mA`qaY`16_|GzOjaoSvoTrR`o^- zs=p7{=;Dk{lNwuxJ3wN9YPv=n7-$rFE|brtR^syC#3VpPaGxM{2^<#Bk_2c{!uFw@ z&q!9OK-a-sZpTN;@V>*97^W12VNXyvzM6AZ+4w=ED>%!nHz!BT(i+5WBN~91HuIy( zgx#kbwgH0q0B)oLA$by!Bo+1>oEwI2&ko(tRhYIK8OKshn2v=m}D*+ z?MzZJ=fVxec2rnl*jejt?)vUipa$7(*t+36^f%Hxu>4T4tg2VWj$o(+=!hxu@{ZsJ z=T`8Ii8gkFRG&1!Yme8`)&^Tb4axfO&>&tRc}kDz0M-)Zbj!m60t1v>2-Z=Wa{&&5 zHa}pvQN5XSXMq`Eql-C1rlNi3s#1h0SoNtFb4v&~k+xbb*c&~4yX-vvA0guTd_WKj zFFnks53y@>g%N&me3JkZ8Nw`379xiR?CrKGeUjc(zTlf&o3I+QAX8Nb3@fNG7CsyT z9gJ}#u5VoB9j6UY(6`LL9=ek31PE~`4UT4Enha7G!;_%3H-nD-y5*c)rV@EfXc7i@ z<%e)B?#coKGws*v3rdcJ>0L9%8$%0{`V)B9Rf1cW&m$5=5t+;^C4W+Rqbyza|`!Qd3wN6$I9AN#6mW=o;N0_;c1QW zD~T$&kmWDrnnm;`vR?#Ghgn(8JiI5BA#p4!6dX2o@rBGze*2h?wRs*Z7VSpfh^0q( z&3to7@uU5w#m69K%q%-*9_5QzH&zuLLzjMZIv1!~YIRC9W78#@GFHAXa-N;y;K*Jm zkt*V_P*#<9oN4VBhOQ;&BXOm%F=rwDKLXg#&FgTvd=xy=3%nOB5DMwrm+zoFa$DmV zd2^(bEL6FUaPe1Hi@3hoh`fOjFLlNq7RgjYJUE{#1PAWHX&4OmYG6C6c zdp=v|no;Md8kQB0L%^r z9f7se&q2y+0lS%TkUE;78(K2EY@!<*4kl}dmEG0ABX|{RVIn;&9L2qC~9?&pu5Pfjm^qv@8k2>H; zsSAAK)DAj=Agb`&QnHB&N^dBQpmjcgoGYDY#7LkU>RtH1e0%ZAii*wJqe-7%PpPY7 zeH`X*+!s=U)ul$OqdE?AtQ!wV%*0U9iXPerQ(||=sOhwT0|2}yxz6Z7@V3iv;O&gY zfz&y((6+FOPu^f*wb_axS0pIsNqWeJ!9GJYEcG#=(7H?*kU21CqnTjZtJxwk9(6Q< zmNu}``RE@M9hy8Q81(`2tV8SYjLm6Sc*~Qv7^VE6$1Kb+UoowI>tkgSa{vOBK~3{2 z@M;y?ur<8%v|SpwB{|@c=<-^l+3fs_DE=Bf~wwtFUk+Fby_0iY*3A zPOv82Cb1BOsjMpeo?#bc>L8V{Xp^c{7w~3&3Ctq{OOou_P7zz~F94zDo zB!rL=jGoLzd+oj-j`CyF1`}>H_dn~)j-ARL|-qn3rvWi9?wa&u)N~Hu0cDg2wgDTuq4ZME>NuY3k;Q)4wIk? z8ALT$fwn<_h(}DjlwJ28dsEe6W|}x?^J(=TFUg+PF`_<~3kT^>mJx|`&RLj1`*sf{ ztTro!7J|VB*8u{0k=Jb{LWcF#{mpiGzAM%$6y>>e@x>E3Q=ah)*m~v$SgWjX;%y5W zt(s;lg9PG(k{)5*c?gHcCZ=1q^L;C^JqA^tV~38jzlE;Kiej4Vr_M9gTN~?zRyCN4 zBGBnMnmSmDMBf1Sn-+n_c!5uqBpda-H$HKq#$l~}Y$1ERXLqCaAY|yvsUJy@&_97U z)D%J_CMn%k8Zs!3&AcrD$EQJRQ6Mcuqw7r`)&_o!z*8anOAwQ>O#J+d`HE`n55-Z< zS1vp|G%Y}Q!Z&^*CrTLRHPZM%q%Nu!!r%$1K0p#AKd}V}6h=ZoKb2mScx+uZ&{tZ?L0U3P)0CU%C%6Ed}iLgCp;*7WDe5OxHK2+8(WHV?^G{l znm}sW#O0_Ul}A>Ow~RXDGc9?u2-qB#fqc;>y<491UO*j^9S+jvbUg6r!hLUsV8t$6 zi8)~B?(RRQF952juDOg`Oqk$-0?GS!4L#<6Yn$s?eT!y$Chu9hIJLz;cVy%sg+&gU%?$473%t+jNV*Xmqf}RnS$@WASZTNVy7*X^2jTc!f=iE3 zp-F{=gcMO#K$GhjcrrO-oV)c1L%>f3v81r!2*(+#K_g@MTNFJ-g)_rZeb?pVy~#!7 z>%5kElQy%(OCXC!h)|!k#UcfCce5xisRES-SxF;n*fl^vE)>&ya{05zYDM7*f_2j& z3BjQU59*y?6++sG@WXQyj~rFji0p5exKsr4>r|GiXSA|>HRe?jS$wkRWyA36biJtxg-B5WAhO*tV9wdsQ)vvi9W zbbiEhrusF#4VElf-MtdzY!vEcrvu*2QB-pU5Df$8b7K0K%Mt*$N-&vuUVK~^$Mk_+ zQrM)egC)WPO)9%Asfl<)XO~=MKTSdNrO{=NFPt0@vEwo)p=`y+@u#^ zzFI#?%*h?FpIaPrNul41n1cIA%j*gQRux897a$j@(NSxo){F4130qLfWUP=t73c~C zkBDUHPG#6I)#Jn?x)hi8Za}TS7I*hyh^aicVhm^h1WAA$A6q5ZZ^JEaQ49 zX;x+=!Ird92>Af3YW+|`L?RyH%iRzZ=BN~XWhw}C2EMK+U}4IeLWk=hdC#HHXr75h zge3{;YggbV5&^KaVb%dkxdkzAPd7`SF=MD$j-vj4*}B&KO*paIy5D)Ym_w&9sE2^z znifLtk`trW^>ma=wLcpdtC=0}2D#2=l0Y+zz-c!ZF9`m9XF{jiP^xQH6BTfCFJ&T1 zp;Zs4b-A{-5+xnq80?-+wmGl~e0T1ZMi`o+@wvAXpkBsmrB<&EdQS(28yP0rN#GOV zgdM^@SKjLO&`9>WNWS~uKVFfCo!ue6d2o6e0D+e(V>&0Qh(k0Esct_SEqho>f{Em4 zhL0A_`vBN=HX3*4;wP9!9yK9Brizh_GcGDh?3rKbMCRc4mr`}9nTWr@9V6a5zeVf_MSKnx^4CexuBZ17X zB;KcQ=~Azc?~PpM6_F* zqw6aws)}5dq$px;1W-`wbT3&jKrSGS9V?3qy$+AaAY;R0BdBTPRr`tnp#src#L_hW zxgCbiA$U`e<=U6H33HQ)2uW!^8L3wUrJ|J^74Ig->U%IfvR|^ERfQNX#Qd2p5V`Hv zV}@WKaoo4dQcK0LksNoIi-8`3D5R=#WB}z94*c+jTNEHeW&FyMpBRlQkRGTSR1+Xs zfk9=BO=-a1kmHb<6s7u|4HgIrP$>J}4tN6%>eibW2g2C9g7Uu8_*s!rdC=h*=S324 z2Ea~9FshsYrU=A3FlZoIs4Y3`TQsuOmE!diy{4?+zhq9BBOu(CcQ!{o za+%zkI&5fvJfmTYM7cE1uqC*LEEgQM-x6_%)2ZUMeqmkp-lJ z#c-@yQLcjY9RUSkfcwTsud@aY0JaA0)VaP2*1lRgo#F0BRC4BeNXy)ENmQ7@JcNb7 zdllYrER$7G6bEhiI*gC42S-?#23+zv5bIlK*?7=VT!&308Mp#)oycY7Xatb%AYo`P zfy7g=kbq=V%GiR*mj>!4sXD&-7OnY!+L*&59jP%^M#uuPAJL@<{_EvER&=Sf80%Zt-wBh8(v>$fPUHaK(tfvXw z%&LbsjL4}1@&wPJ9<5T<6HQb)zhFX;5IG^fS=WS&I@g&yM&~a$e0J4jJC7 zZL=qaOLzLPwXEcR@CXYK1ek$%&}j;*(w%tmej%%|4jtY}pkX4&;Zza(1i4jgSnQ0^ z{AkuzG&DaU!173kdsSV7O(g?h^efts8A-h#&0x*Eh0eN?>hu+; zsj|kEa73m^WDZS@9f?jrNmN}K;|;%7Hm{#oQz=^YI<~8vH+cg1K*ehsa+;m#_m&w2 zsRYn4=G$0Ms0Pdkks^a_0=%=19>NZ*L43Xp&b~DY;6!d5NCbmCs12YdIs-g|(fcHm zTrf~?Raji|xbPUS;Rb3drq%;=3A%R+W1WwlL{bWD3Px`J3kDcIbPZfaaA1o^zA=a) zgo(638_*6k2lEg}X;Pqt_N6fx`%IMVj}N9%QDdoH>h#94narcIIOCbDPNAh8liiq* zq0udso*jf`-U`)95`}@jiqnM1qg5_G#0jKa5}_c4KPb*KqP(GvcZ(rJY+SfnTk7I+ zr>zTba%&RwoF7aA*Ar`S4b1b+wxb7wuaCo!K|}e?k(?`qiIPfQ``w_>>7D0OXVpvJeSlq{RV4ggLGEA`FLM z^AF8!RQn;t261$)SsojlWEL4RJ4s+MUZG$!1ID-o!gTwSm8krK9JN)6xQ0WW_&0b$ z?e6Pafy0O1HAWR=@RfS2YWdw*o+yYwgm)*0wIa*)4mQtVm`s_22TkRC6`JO zM^u`NXqI8FKzfq9NnxYwYdE-T8KpZzJNPgVq1W9l0{tqi#TGXDa3;lqH78;W@70_j ziM!y>SE%W$u!nfGWv(-G>%m-Z*ub$W0*Bs>O`l=hgw}D2U5Of5;+6% zu7O{arM~y-gb`V3=d$v8vedEXM#K&nEIdC~u02kOxa<}doon^quWzuRR7eJWNN&c$ zq9ih5hIFjMOsiD)80>nyw?~lkJvB}pz<^D>v>PFeL>pAUOh~@z%Q^{D&J-Z4uA4bp zECus+F7w0qZZ)NTp@!^i;J4d&Fl<-qSOdVoFS=jVZd_kRka&ylgk)%RcXUh~Tv136 zMl#aLfPK*leCrK#gmS*s3jy?#;ImFwQ7BZLv1aY4b3l|upxqeA60kZ{Q*o$a$r$p0 zBbV6FLQ4M>0SV|3h>#J92f{c&K!w>UEME(Bg$t&l#NJ$CJLr(bHXh5c)^%zVMq_a0zXu74WZ#6xVVb1YU}{Z zCAr3ZY~utE)=TLeIu!Epi5|_g1&wvW127!aFiAYJkkn_isNNmtbyOqwV11*KNxQA>mf3W%QBLjMoJAG@@D;bj4SpZzVEJeEU{`rf~cTU|Ap(cuf!tw zAn9=Zw|GfY=kfPoP8C!`aw{gm2zcK;+oeuw6f=~H5(4Xft-5R=F;T`<1En#cQG4}U z#of$EYfEwwdj^jgS|m0C$pZt6dc5ssctLXSNejI&O~-H8mGj(&->YRT&J3ZG)WoT>x-^#2oFQ8S<)f_&|as zQP({Y=oT&Bwj=G-9XjLyr7wUmSc~TnvB@JEFOdG>?;-A&K`xV96!|0u?2}CBDn-Ca zx|x`f@2LHu@-*#?N3T{ouCcKR^sNEeE|f#L_5f3(>cr?-e)q6#+EKcfd87Bv3Dopf zKL=>&D%)3_uo8+TL4a6_ZI!KwoceV_fI`|m2p&!L8b3L}`er+B?K<$&iLCH`S1$c;p$zR22!3f2R2)H4bwmpf3eh;#)uLP3{A4g2XzEi#1z zN4~sZ!TSg(!vxccY|f8i(4I#XJ}v~KMaL>8Y`I)Le}KKcx{E(@O?@!cjEX?I=I!R0 zc%G7(8D2>7PHVkG5MKbh_V>K>2}J2>CA?C>SgnXak9X_@5IP{b0ss*Ug31Rrrk?`* zq6IkpB%9h2Lwc&h(z9hij|-yL8G!bZ%LOK|te#4?DB?s~Q&^WrdLwl4t|QiHo*~(3 zNyZ-ylyw6;cn=x_fIdV`Ibk-5XC4xE0jj_#a4_|au~E1P&K~uq8$*=WzC)CYD$~~X z1g>e{p^_d5PIVMwRNry+(8EDKR8=_s6tiEb z3y!2O!L!`wO>ILdnzXcf!mf5-A;l8;xz#*MNkzDAW9Y6QWL9~Ckmh};S`<)Kk+kc|cVDJ~-ra*|{gXEd$}g<%<$X#{~$q?gxp zUTJ(N1Mz9`G;aa)dAY&s)cPCqGjLl#6XmueLOw2l=FC7*Q|Z163fo`LY9IA)_rj;Wvw3KOo{Zi8_WsiR^T^=%k10~SuNm~&IPt>SYtkBGi5S! zVq^7=T2Erdjoag)w4kg|^tNSDN;*h^WYaH0aPJ-`;GWF(8^yen@P+dqV|Oiw2)pyg zhJ+E5)-2&^C!9E-L-0t^x`&kJSDEZKdi%qU2&yALk=kAw1SP^Rzz4kIwKmlESEUs# zUl}U6g5`?VcC9Un9(mbkJ;surINHWX4xLoDeKC62gk_rUJ|s3HGBwjro*Ii77y@{x zQV2e_OYr5R-cSw;3=UlYDIZn)C4^~}ip8Qe;AlFg4)+sJ!56R=-tLtHnQQ(TYuQpoPSfHXb*XXIZVK&PO1)cZh~0q-Ti0)2>yN zQNj>CK$7^(sl%1IY`m0P%LOd7c zzis|003vz%-3g`0)^p8m7c&BGzx04-S}Mkb^_r75jk0D zDPXmg4hD!CQ`4%HDmx;-bYZF0!ZocIOS^@gBWYnd8F+;id0h_M z244_s5DA2lGn(3}IN6wTcS+d1)5^9uMA`Xh)Yi&WO69QvjUESS_%A%j5+Z%gQl&2H zk(BzWuLVN7=1vz)UfR`goal9Ypp?xJ&#Cf&ib#fNkf>HH6Nh*N=36pOMl)0~rq6bn z%?*+Pl{TCktxFt=4b&wp1#j{PsyU#v-wav4KFcqVC1v`vx5E1%PYJ|*A zWxoo?Fq>U@Zc?AWG4Q+21H_?WS+bc{7~Th4s*d?bYHdw?l<%T#(@kAKHPh)yFj08h z*ucmQj`JiG902k5aJaQb+O?2IR2M$tXErS>&ZnBktsD;r>o(BHptlU8Xi%;p@-T>m zQJct};JPA2_1X)Z_loh<)cjH@WK=+a!_ThDy<_AHtudVlAaPW3CbjxCvWplRYhA9! z&V=y$9UYBvT+mboq%3T1`j~HMDi!G}4FOydI9+Ao7oG3Kro^s72ujP~u@>O^u2u%( zq9lYH%cT2kK?HTf^2wQ1?tCj%Djj}7GnbAS^n|sq0{K`L}v`KX-Y{v1M3S;6V7d!&bhQyZ;q|R{w zta3*dT_w{*zLA+p3nUN{x-yPo;;2=aQ3f_MiW@gla2DhrC<57Y4A5yFuwb$M(_1O9 zhi8zgWN>UoIu*{U4pfuPOfczG3~ZNl>0bxpq~<+rnq9Tw#fk?TmOWV&Nj;@wumohhFY7lwL(dGr*TGB}58(m7a^kU;8DdW$ zgj#}~EmSz_VX|1t9L9~amwm{b`Bp4hU(3p!`Sp9FW^<{9H(e+pSg1%|z^HjnRQeB?ibZj2 zKrg%AWQFZxz(OuQx7PZkh@^o6Jwz2BJxCFP)vufzQZ+JJ?efrlG8?*I4nlbM*RqVw zG!yRRzLLJ6PD0H(rJ*vyBOZ+k8TeSZhwVNvlc}VJb4K7Am$zVh-gz2*9BdSlg&{@{HL> zh=T;a!jXst226KQadlu3GFnwZ0CR!*`gcO<^Ti~TUK`asn5yT9e`b-Fg}4+}1cgE~ zt!7QbrnjvyU<}0rAn(zAit6!#ZS`mB2&S-{QQGL(Ll%7?+#&IG^Zav6vc&{945C)L zoI9mw+|zZ^kEh^iQaR7Y3Y^`v?(8Va=fYg)37@%U^sW%k(lPOYG9C3j;S+^%5c-46#Z zhkJ9!GD*(>!K`+LEB63eXC;GOIWu^Q-ZFb8VoaqKb5x$4!bzhbLL|D>W^W$JA*o(v zxHe*CYvyUMUFD-!s72m+q0A{It3<)e!YIsvl0w-iS%wf4fL{^I4bwtf0vwj3hjD<7 z`3VNmUdJlf&4_}VK;(E~w&q`GjpS}N@~#*guL0PLDb)f-`93^OWlNnuB(ZlPfZ0QQ ztAAnF%i>dWvGT;!d%7c>>3-T!DZW3XAB-m?jY56B|@}Lu#9_h79O<-rnT9`=V+Xa&>p<393=8)9=r03 zb2AN1M4om_NIVz8%ZFX}4iigQCxqy;*TSHt4*I&~H(4KhFy=rR%@U5@8n-uNwnC9i zrzJ6fxFFz%=$E2Hu#hXA6p0u$;)(0Ng&seZR~$6~HDAgI;M&H1hnHZWokS$E(|OmA zkQtXw#ZMxGM~w`v|ibQQ_7VMp2$2?`#DJ_gwevj$}R^5jA5gGmC{PP)>}V~%K} zrdI_U0+Z|ay*2`1HmMifMmkH~7HMu?kVzt`L)WH#y}*~W9j84}8tqb8`H+5L{UEiLlFk8Ze^z=XKgFeWFptdKz+S8EWJzA_}z*w51rP z;>t71(7v^Tm>1nj<9yXTUj{vDQoBJR8vs?ma9$-@R+kT2xKSUjt6ys z?)+eXbMIqkg;6uLY>~D}k8-uSyiHn>NY%Yt!3{AHcR<xNpvg8FC_*FAxXv?y!4n~GeInRqJc5RGy6z5ToDDv2P*x6>7y{M$>bP%(Ja^_fw^@g=TjtX zRv0=T%F{Z{2zR`&_#I99WK)+h3LL(QwdfA$IJZ#gdPe5x~1t7Jm` zXoPJuA*01z;C5;Vp21_b95@LzlUalIQ=tZFrB~EEUnkRwh#)`XmH`z3Z&?1 zT#_EuxDu*k7#|CF9LsfG>&Kg>Msy)8BWqCjF_|0=rRN27hYIe-y51`($@t9DCBze2 z?uf{)f=f;hU~y^ez9gb@KwUsk;o1>Xu`aWNrRwwim0o00u(#K_Gz&>3eVYA z<%AVWDzhsTteJ(mNy|9R>bh~16PILDXw%#tTRlDxg_sLEOvFKE58?ww<#d3qv5{YF zbuV2|*Mi-k#*!c+hD!*08@X}k9zL~esZn}0TIC?2cw*XLLL;jCOiH?@Zl{HUTsU({ zd6a@w)sMiCj;PU;IACFKV{!+XJxl3*M53=7v3IT+eSwY5&X~AytqNGd3PP16AgW*@ z52WN!FEB2FIX3NSILb~CGgHgwyS&EV8VZ&<$h5r; z-YJkF%BFQ5_Syr8H5Oget#d$H!u_~yKI8Cxi-x9Vx)$gv5QR@k7z*M{rdY3*lsY3% z%s`DD0TefrVHY#f#H#O#C|GXoDv2D%6L8Qv!K+D{veV7u&UQl8BbaOmG+d8k1d?5( zN3s^Y-K5xYNj)s~_)Tua-GK+}lr=)2cz{@GvxaIK+{}yRx3t?%zKzQ?^4jc_+U+Ve zB$o@1)&td$qM_mV*B6<-5ysPzkaKti5UW|xdJu5UArKHp`;lZ!ToLGPWFQUQ>8;6! z#4r;rLN@BY$Y!mc=N(?m>=5vh=^qGhE_++T<`r7PR^a|Fqqp^fO|@bVKozc$pnKie zU&3B1)ius9td~q)4EYu&I<~v%zi$-tirZtx>7^JCY|s$7J(~}7E!;EFxH;RmtoZI? zRj`}q(A6*_dH}lyM~Hmtdd^$X`nxE*pv-e^PH&US`qR>;4JFK!v?s*wA~0q1M6@f6 zjvV%=K1#NfzEz`{sJ%FX1C~XZ8t8%ri$um-Vg`r@b;$ud6l-e6->CQn)Z9{UhX5ds z-e;lMC_3G<*0Sv3K=HnSGAr>7DU*f3O6tDm5s7zkmAV!(z)nR-MENQ8EnejOO)N7{ zgtaE6Oaj6{RUa`eWgg#N{OLE(Xo=34BqLDv0%Z3V!vw`Hlpn);=%!hFQ*Tgxjky*J zEW%oIr1$r6*f?A`ISOCPnmTr_D_Lo6qcJO*{||(;*8R`?0}G1PDNX0aq)T?I{IjN_ITVft9NiVo^3($Rz@i@-I1d9nE1m+?Y!cbCM1@)H71kD>TIN zyfwIJ+?Bc zI=VJ?lGypSNH024_R9zn(7ln;siD*I0#S%V+g_h+#EGT%70dBkz?zB>hDt;?kA&35 zeiK`u$N*;5;g(VP{Inz^l5HC5hz4~R$jn{R)oAjNx~_p9lhbFN4$?E5!?#CLx;vd5R?Z& z18#v=>JfQ%W(jLd9G66fVz9oPwKV}gL|=Iq(RH5nq`kRVF_1BD!LU;NaGrHgs$~k zddrQY&_}Y|b;J$=56i)-$cZNgvU-}%?9KGXx?oEe4uG9KYN>`nO%#>lHkCs_A&a@6 zW%H2_R%q)G;VKlf$95g2XQPJFn1%c!z^s5dOorUO7;mM`N5t=IM^R^~lu8VeCb@o! zlK4jqEFo7U&=cO%kDTX)KT0w-nJqCZvJ!ahjS~2v*qs1=REDvSMM(#URLaom&$%G9 zZmUXEAkbW{;xN1+VgwmdfkMIwtV#*+!*VW-iG>f&Aj8kh3UoJwj0Ym%M(gc@?n>TFhCb`ncQO`ha5;|Y*iJ%%q^AYaPT zQk*3}3xhda^I18gU=5`v<3g`%QT-jyL$u%|_Lhl3s@^N5Ta=k#SpJ6OxzmYfqS34; zO=7X}nx95lk-Lr+9f+w4Fmh2)G6LoYQvF0k;qeqIS&Oo>J+4I|9pNjFBK)z&St?LX zp&3H4B&ZZ94JyW`l==?2hLs;^I7a+J;MXC=;O+x)#X6w2)eCYC-g}M@Cz;f*Obf(= zqM{B?xmw~r`AjIBOUU5;nz>6ogfnCUzmx(U&?4F#&X z4iF!d1Ot{2m65+jb`c1;&W*?x_b$QA1p^XE3Q9VYaLV?)2w zFyC+z5!$)q(c>K%i$U3%!{@JS>3BsDX0Mub{|$us?=)Ad!gRQ zPEVm!mKoyGr}XN->R=psaD0~*(ZHz)#P_@~$&#V;vU{fY?Anpz@}r7|Ac`BAPJsDwkep6@&1JR`j+gj-ym8xF&8=O;dE`Y&b`35~>sJ{bk!RbaL{CyPe}L27BmmdxqP<;TNnhX)DHoy1sd8<)8$ZgB2XCH zunF=k(tBuHct&2NB{D7;5W4Vb)H_h(K_Yex8%}wV3KABRh7Ds+wVYl6OH|@Jpg>q6%Bx0`>Ub&R=ddu37*K_hS0j$n?ts@^nQGGOCDOUltIq@_$dWI7 zv%8>mdNQ)1_U(PpsY>FS<;G2-vcwY)z1B=1WyH$hokhLS1%IpPnR1Z7ngxFK5@ z9!nLiStD-37NEib1UGHvxl6*mZBormBzCf(QKrtSZH`odTs}C5_+Mi25U~P@dVaU3EaVF5g^jb{9wDZxdX6Z-He|0H{ zXo$C3D+q;kqYW8a8l;Y{;RRqXcE-mNvD}gKu&X74_pz*PeEi-JcEq_Xstd}ti+=gY zWviuKlZYPt8N?$2s4OlMdew5^*_le1b;_eMg1!!j85tVCCHgu-`<@Ye1Ms_m;PZl3 zwV@mroyt0>n4y+=2Ggap1RCSgBD^S2b7k}l56d=t=tYe_)-`5VrxVkgyFoKoO|`WptnjgJ2L3bf?#;O0R~_pr}Ca5PPc5j_;vXfJa#kL0Mr|q}+&>KI7?Hln*a)%0D%nDFHro?%XV2?#lu#kw zeDtM3Np89dPW1hZ8g`#)p7+xMqeBj1etG~ab4qBVwX!}iQ-xGh=Ot>o>Dh$Le#wTJ zs7ERY3Fo*<8gVs>p0sBrNisST2;udeXL@4ki9sS|45x#Gpl+wSDOS`GccJanIu<({ z4FTw+LwRZ;TpK6`@>4*r$-XR^x`=YIQuRv$p(w2uhun(nplR-cxip>1hhT6}m~igR z)5=#OqLP8saSYs@C1)$e0uyxePvSVd3BqcSxcbMMe4s0KE89cluNl=JjlmE=D;*le zMU@Yph*Y<2cxDdKgWM!I%Cg5I#b>cmq(!4#G1B|1sTWPh(ld5jnRoQD0%2nk)Y&2+ z+stGgCkQzTdio6gVN&WMWj?Q!K?oobcVsnv4L88_(b7GzHqWA%FYKONsaxn?uvAr5 zR87i-aILTk2&E-^SUwUyyD)a??bPC@%Mq8Gtkxn0NpuSk%o^7gXZw^*k9O<`SyAhQWE+zf~{P z*f>4Irw@;oj)CJWA<}c-Y(v{a*z1rwSpg1!Z8VKi1u%qGg5p@*OMVtK-`Q9 z#ycU@G+UJq<-8ps7IddgSATM1r1ap@tvoSN%dM5m2zhkIpz8E0X=D^5YDjcM-7pX_ z1UmuJN965Z+VwGsI(98RnFiF-LQ>-26dz9uw}ixbvsJ}`MgUoXge?KR00K~84$`Jc zY#8yLcavKyV6nYktkwtSjj{owLP8?&zHV)(a}>G5(R09*m(cDCl2uIyH(oD09~h#A zz=jM6f}n9F51qF{^sfbmfK#db#TkuI^%nQ5q==i4zB5sDNq#7?|0W}EtHqF z!Am%x=HH|eDG85QoeT^P>RBC+3`P)&sa8yuQI2S~7pBI@3G}7$5uHf^^x9#8M*Q3` z4Ral*ZDqc7nyJ9K^2v($*|L)BGX`djC0EjM@m&d4tnpn<0jXf82DJIThf`;AC^RGp z*~LxQ5gX>srb0`cl>VTWj<4c#Qn{}XiB&6AXYpa=P z8Hg*=Hzxk6SpKvqtXtJ=W$6&cW|R{V_=-7kFx9UCTn&O z`K=X-k|&gysF&}+w!0mxsg+tT5ANIh-6;e zGndhYeM^{(?TgCaV(sDzkH|`RflYyKG-Ve&A>K-^NYuMqMOfQFlfMYgY{3cVvE9lB zg~AbZ5t9!HShW>d)Y34C0J2&nz3l3>K>$IRnKSztoc5J{!>Da`+Z_eDP$d$;RlvUG zj+D}VYp#dTtMpaxKR|lpok5NXvXWvg~ANDd4fW@8))9#b4G(y@N7(VZ%vxeW*!hS zt5(F{KnjVYo|H+1LkKMgjNN#hnv4zd$>e}?-6ywo(5%OYMMZ?Q1dxd`N&sr$Ov0jS z%QDTau4ep?Rm|*qZ%Qi9R+qfNT<$~dV7!DSrd7|OLiymGZqRUzYsWxS$xb_*l%!JE;MWyGAcMCx!A|94R{#HkZsE4Wea5&S9N}V zyX^0hl-xrD2suK0Fn;F*T8rIwbdFxJv1o-EQ&wTnQ+ftu6O{O_2p`_mTG@+u7FwO~ z#ujAhgGV*zhv;W4CIv9gDa;)C=6NK$Ma;t0_L_}hNg;NY<&e_>#dPRZ97=DDkjTA` z#T2`US1OH2#bjlE_M-h0ecBBHR7YiKJ#q*U>oCWQ^tlTIWR`eD7*0pS& z@^%49-hPG*VD#Tatd-p}=`4WqqS&H`IJPBsX?p~F(<2n@a}w$-?MA1V$=P8HVAyxg z*`Oo2g%^e_U6OBZK)iP#K?6h3=wGPo=49q$i>2Z(xK%h*EvuBf6~4l83UZ)E(3<1=P8eD^tub3bItKRIXf`dBsDW5 zjwTwr9++O$j0?3K^t};df?}-P;?(t>C1R<`mH6n&6;6Q!FL((WTJ>_&x`zQFNG^h4 zw6u|i27|;(c-^tOiM|jclN(i_I{n$zOm;0*s->3F$XCRzXk~x3SO+w+?FuKVkJV)> zc*w{(t8oZc)I`;OmZ>s_FlJKd=q;t^vtoG#BDH;RnP*G6ZO@pNmwpQ zEeem`yc^@R_N6PzrSWc*rKyp-W)bg5S@X^p7$MR^O+|)NquN6!pHeLGv6|x*KqEbc zBR586x^!b>2=N?rqVo*)CJI-zxi4AE5{ZXm+qt$q*t`OKH|~)7wUjm(9IvOdy zCA>(Kl?zzz@T=Ad2(-GTDL8Recd2p{zH+X`JnF~ifZ)f@;d zre{5ibgU&)RzKG%uH{FQstS{AR=`FRH}9fJK?sh$FAW29XcbJjXh?b#p$DR{?}o+A zz-lf#0eyj@@YeIInl|sP>r*wPK&M^@WDCBD;F|_09S}&k3q?_Y^fIa!Dc?1*1Tc+i zJNHT2O<_v$APeSu8ny_`?MRZWQwkgvD8WO5Q71l2VAKL82y#zegD8tgD>Q*3)XkKf z7~`-~?3j8;V%i{8^5kN}Wwg!rx(&m8UVxCUqV&N72>L^>s`7qVmF+tW3t7ure3UkP zAZT32(6N6!IeTq+k4NY zy_CZX`ebir#$AeqMh)i;AU1D-kxc;o7#Bs3Dn8r1m|$7w>u-wLpVm&IJOns0VXA@8 z)Ix@%numF|NV`j(FowvJFAH4QQwNW`R;h-q+v+*pg5O~~mjw1Nv{d}8x3+`Zuj(-5 z_iqs7flB)`gj`TWjySVyZ5EdNd}+`t%JRZ1^;nGfP@lyomN$##@QHm|@x%o_nNR^f zbJAHI-BBlIHYc<gdlK0!E(HK4iV)LvRcw^cckf=gX&oGWGM1J=C zMdy~ax%HHY7o*|JV$w#A&?Ky2y7!TGvxR{w=Ji(wN zystJHfmqmrr2A(io^nDPAYy9Wqx z;MCTiR+^DLWH^CLoZ#gUrg&p3phorruATusW}#CQ#CjA1h@sFuYCz5phQT%(Y^Xhc zJnaN~*JY+OB{8aMdUTplY!-B4i)Mw1aXfXb)l7o7mKhbB9q>X28|M)6GOH0KB2{p| zqQ={_V|-3_Dm+XDbY!uc>(Y7^vZkzPC}TXpb(ij8mVgUvW3lO=jOW@&&IO_k#4y8T zh_hft)_Jo>y$vH;9hrf!3lJ#yhraSZl+@1$WLELzB}?Fc21PDK_9C|`!|5kEo|UGT zK#OS{HyD=7ESuU9k_Sry*kGyPi!uB-R#prgwz=_(iZTe;CU}+#@;2MwFHkQl$*d0y zE-0vu=X|!(+A%p6L{bxD?+~tm`ro2u8uxO-p9B*wk`bxIyG$xo6?IBfuvK%PRU9}w z9nUq`4?dNgXjI>^Es=gSq@dpHFfwD$5>6I!U zku@58a7hjarp!Nj?oc*Lgvwf27Zp!>@3E$_Y~#pHkBCa-l%Kl4JqR97$D}>s$g{o! zjD6Cu&|kC4>bi=tXnELd=NJ5UL?4+q9-h^m*?6wH2?`yDmR zF1l@=X0M5b^F2!w@yC^TBWbtQYcN75#0oFYu+h@kMrPSu)txaSiR_tqjfT%3A6~R( zh+Ta~K0|G7KZ^UJa3a(TWVdf{Xt0pk6K)$1(7FiaZv4}>8g-_t8!67vA-q6%LRMdT zK(zAlUn_yv1vsjNm64TIGdWaMO#LAebCfg>c8r&bJ>Z^8QB@e(rbem}yYH{BwUVNRB}%=17rM8Dx3SDx1#PhcjtfJ!O`q2rsp+#e|mcK6v7% zY@&`mJi?BR>u`g35Z6S^VtTGfCc{gOPf<#%Qb%Zp%tK?_@iT}J%Mgmf@U%3vAZTa2=2+H%p$}|Ev#v@(dvX(Zg~x%7EZh{vBzoj;ERqpB96 zoSwn46GaTY0O0MwEmpG`C*^X$#CCOke~SCnni$7u$Rqb@2zfq}Vm!i#3-eg@fiy(m zGbsH-B8ddLnkb~Gt~GmRjMplqN~f66zSjfr^9OpC+umeEQpTcd`|9MdaX|%8#sf?X z!8MMY;f^4%o!-#0hmmHRQYIsv2Z8|w9i#{ZGPHA5XR4^DX2APG+WV@|r6PRM0E4#} zgl4Rtgvc+Xq2011!fCvO>w+acDb6(QHCMO9v*p`1K86S9VLFC!Z2<;pmNf-|5aT81 z$rBd7Qq^)!dP=HfLuQh>7tXBSlWh$*k>2qgD|%IvhHNT6@r5+NcPg|h9~&!IwfZ3% zM?CB~z;W1%qr?WJxuU2e+zdCuZ+k9`g-*k5>4;SNk5 zMPieZ?F&{K=%^nJDK%sFQj;d7{VT1{Zo{Qbz39E~#K_%g34=vs3Ux@A{yu^1;i#=CI# zN$OkScWe^A=nH`gFg;&BKoAHmRb?S>WqR5II^I3hBPPe@`sv-6@h<4=fLdJ^ywY)2 zdV*qZh0;LQDn|~-t&4F#6J1C!&j>NI$=t6A6mhYJ6o*5$0J8{X22+3tmDz_$2{m+& zfM2B5xBnhm7P(~P0DKeM}tB&lUeY|{}Ch;-@ zlm~mKNCRe!XKaVfuuwUvXK*(}e#f+zr93m~1p%Q93w3+M%1OW(CP*;atPd~_B6AAr zcW`>2ioAzDc~XIqYY!I5ti`V82##A~uaAf=B4C*Y0%3a|3xi4NXTc2+8d8-pxooR~ zk3ri;q;R`va1v%lwC{k9e3z*1rP>xYS1K`58!EX-lA!pj+~6489?njz+j>!}fQDv<(DS1ZE8hh`b0j!}_y zjrB?zjYe*tde)WR6k2doRoqhcXA%ln;5;a*jAy}W3(eAivyF=t15BdvXaxc&%sQN_ zPkw>IOLCpdl|LKW8dR7!hiAJMc6WFV-@=CON|#)KRawvT?WGXZw2ZMYA!AEQ@?kQ; z3Ai4~6!)RnWQO?j&dccH=x|M! z5|Dw`WUhZZ^Dj-|N3B4{n;8~d(`wyGBoz^5kcLPUtvOwTwjT0X!mn}4ugjrYx|yz~ zXIVp9nb7L7`+C!gb7p6_HaYT;89E{=w#`Z9$t(MhBLN z!p-8wn7Sw2NJ`lm?NNaw!l|fy7?Ue4Qjm#%KzRWz^h=|d=%PsD9St_I-dPdsvrR)mW+s9_4`=LGnH(i~o|!Wb+(P6XA2;$D)F$$~zQYFp=R zxQyG&Lt$gZ(bOtT%=R>($mu*$j$y8?X$jG}l;E;NViPQ5lM`!IzcmOG+3137k_*_} zlZBzWh%f-uMRdUT8Bq~jKay4XG46iY3^T=mqyndOr)GHqM%ZA(Qvz_AGs`x@Z?3<6 z3h4So=amN^yj?=0q;|Gn5g?hibDa$h)lt>#(y42OQKHCMO>W6E8Mm*@+<(D!{Jts0 zz?08I`%+U>6X?X1l&(22CpjUhLp#4q!v#hIs)DJbll7eU8H#tRmw9e}r)6@c49cm6 z3O@oSTX6}2%Kg^zhgsfOSTAv2Y#~^Kfr}KO@?^>)i!I_4gsm*SFVr5+YDV&Dxm`Z7 za(2Gy1%^yD4Q@DS9y+6+Krxvxu|^%x5c0`lkfN||-T*qDF^@p#_j$euWo%Rp#@Xg|CJt zTf0xJpXFA-IxDcKvDfxY94#0Cqn?#^#>gZBE3^{u=)tCFsf5+CiAsb@_V+5BSd|{K zv!-(<#;q{R9iCqv$H15CpE7zNu?)KW`fR0zni=kanADdI!M0e1+Zk{zE zoY2Ugi;3bKH5aLgD`=>y(HRw#<$gjO6E;G4Y6&WODW7P%we~H_0WaIq?(d?tzqP?TXTC5YIYljdWmifI&j9VSz_m#bB|RK~d8QF*kKzYgaTcITz`idw z6w4?GZy%Abt$|UTx#+?v3Ir)7hWZnnW35F37Mq6K%~sCD85k@i@L_9Ace@yX zg~Hj;n-Cd<0Uc&&F0sVnFeD`k>~y+9FTo3f{!1zA# z%)F3^(}2M}~K zPj-7z-ncIoxdU$%NH`=m>Twqap~j=>S!-Ll96sC2&Dcd&%BqAy zDyZw$cezq2aL+_px}QY@$mg?bC^EREu*>e?rs1KftX%BhV=)gH7^XR?!`ag+rC>Gf zof#!oN)VK*SUcMyh$Q>HMUN_Rx_(z&bP?cr9Zou|s#qJWRM=9N^cqsmgwhXD0*SA1 zKoBrH=N-BeapKyeq^UTEa)K$oQouq%leaEt@eiyNM2A8p)D^m4#I4(tZ>s_+ph9D? zP}x0$4dY1@2Xm#Z@1U8ro2Fp9ktzLD>F2+!!`O=^di- z`iWvxwe-=oZZHR94x_Lo%Cm8xV_C@MUH6N9hImfq=99!hfm)109kA{rdm%ciElsOv zSQjCao9{M0b8C+pf{WqA>wYp;bu>ezaeI12gg z>ktq+I7+~_FHpo+4h(n(->$u7?|4#zjZv{#9++<)%5gK5^!5T3MG8EoR%tdBJpI*# zTN3O~M=}jR$qz=gP?U#A8HmVkxoqE5(!ZC3fV=6R%Trw<J!B(;hd0cz=cT|msbiEEYU#xuKf$VH~>jNw!cxUw3D_#IY(w;&8C$? zK@P@EkZIHj0;4CP9wKB+^rI^XScgL#SrE#hAyMVd z^z0O|9lzW&5;e^Yg*73=`|U#sTQZL7+L3O-l-Lk|Axo47V1#IVMS6b6!cECs?<#zs zS|!L1X5kdZ#C+VXR@FfU-8xBDq%s09o?>O5Awkl%Gg1;SiHHI}F#*FR`}5)=_=t<* zpU3O{{(b)+uRqiD-M=4?$KfcXs`&oj{KW-inN4IJm9}gOSIKSw06+o}|9}7x`~U#~ ziZ1(`o`Aylce(exS3cU=yO{0WsaD<1yPWN&+p$l)_W*k{GXjJWExjGq^`&g~?z9@U zy*C#04M?uF>jVfDbYdWFt6HS(*Dq!gCG2{2>qX{jy6e-|TIrQu#-g5>bkw1Fu~92* zHe`|)vDYQB=;DW?RVaw4rCK#03btPEsa<=Rl=jjAwkvxc_OC+}Pc2UN>XGd8wQSJP zwT|81u_^A(BBayPtJg*wxLV?P+SND?6N?+t_bf zkan+kZuZ{nG>Z4PRO~VZCbUxNqCMX6#-XQqPJ~zz1*5rJ;tupwqQs~G|^tnde^p+Cq^hbXwKR= zgWBd7XzNZTb?M}E-MuGUr@Xz?;wlrfM5y)+0H{)@YuvjY&Z*bCk6u@8ZJQ+ebmDI7wy-+%%X8eV z+f=FBYZGg+)w4xXyEQ=`#;d&Nc6xCms_tp7>IZW@y>j-)ZC9)oZF^yMGLlJ4Z7*3$ zq)X7!od5@3hlyUzMN->U)~<@SmF=5qhel9zYEf%T){yq;bJKvRt(&7t^l3>bHp`~2 zvRgN8)-4*TU6|e6uIzF)Y1UgcQ7H4=>UuYA4Vy)}=Jp+}RU=yMq>A3%>CO!;o}6Bp zLXF#zAU&0fv!`@v4YRY;q^|dF%95LFrd3o<)tdKYuWXgOh1#O2HK>a9rEQR{ja3So z>)vXoY?N*5wTGdvY&OT3FsWX*w?WBwxNx*$YeQ`G?l){Br?zrrHM2dtnOemr+Lh}d z_HM9uz2QU8ZtqT#G-mbBfB*mh0000000000000000000Mk9xD6xw_u-h4X#$&Z=JP zx8HBCZL&LS?2)Iw$Gpw5@wHOPuYK<3j6L_qT;FWFVIn)XS60g1?d#gxOWV1%)i`Ud zmuXr%qh{N?M|0ff^IcY~lxw}6=I-vQG0E7VRkn(~z2}an zx$iZ2`)6f|Pn~x=Gq$4L+_Ph+*F^T!dlRnQ!`H7pomlr5b!xixy>-a!cQg~NJ7o#F zZ@YT(*yWaO}H@)PCySl5Y(=To)^`qWt-Pf~coNsIGx6Ti&2!^v} z`|Z0D+jj2t?;X|ecX`&7^zPl=u97A1XA8Tz)^*j*ta^3c+r4|b$>mp}lWkqu&1FSN zJ8ppPP22$0)-}6ZOzd^bMzOkXj>gTGbFSLb*1KX|CdS*j>w8AUvAp%T3cdGq z`g`f!>3Og`(|YS#_6t+p?(X*N>>9*lY}ebBc4wQq`@3fJuRYv6r~}uocJ^lHy`HLM zYOP^e7T(>nt+MT@r%T(>+jq5g_Zz-l*`9s5+j`!{D|fv1y>wa)UfH*Hd$(S*+1~c; zp4?q!7VWz>*|jQewRKsuZR_64?(dcCNx7-5&%D*+=-tS&7 zTWYJ_!>+*IhkMP<*`U{_k9%;djcuE{UG1%pce}eakzaOl({E>TYLRE$bJx3jxOwMk z-9&Bf-rlXf&aZo&Z+-Kgs~Tzd+n!#X)w=EMd#B%d+P>Y&zM2=iId12@rDpPQoxODR zX7sV{?^fH+R?XYjU26ASYr8V7?!E7CHSc@0D7(7nr(1Vx-tTW5*1ER2cVeBg_U_v^ zyQ_+-ZsF4BUCv(ajQ5A0w|7q0H##dqR^+c|VyDhzK?RUHE zS)qA&IZCuZLygeax z%eyyT-sh{Wt~Xqji=?`9;wj()Vx>k1r+@(|CzL=g86A)lZp=gLT=AS<)n#O% zX#|&ErLcA7GnmteO<%mM z6((TfDOl#X;C7u7n)kDCg&%2rcZFNV8QJRc=P=)s(#>rIEOPHRj0!6WK?-T?uq@j~ zZa13^Ifge^&I8^~4H&ImwnNr&(KZ9`}wy zQUr6sac&lNf||C7g0dh`3b3OuQ8hQ%hMhZGeY#JDL!)+Svz&8G49o^miPOi3Np94I zIFbTwtKRX)pQB1v$BBq?_B*Ox$f3=KK~E?qNJOx-Oy0TbTnmXW%{W`&c8xwOdPy$j zKX#4Q9l&a`aONXam@(qMN0JAb!Y>UeuhC297q}C>Z83JIuGs?yxCIIxVl@zyFzWYv z1BH-N7c*Kkg{Gn8cUdr6>Eb5jbJF!Kn1)Z50s*dKu#a;gMZF@UNPJAoczkW$%W%y=fG(WSUirAQ-dCxl<_{<^jZY9%FhB?(2$v8Y#RMV~a&e*c zDU``0RgHJ;A)%}tCa({{dU26!w-rlJtSx(1`HxJA9Z~i#-w;l$dx%RTfXX4d{w@_h zIMC?iDupV8<&YI0oQ3RI*K4`?v(L^ry628FHyv@;J9Wc@m);jSx#6>Mfq+V6r3gl0 zk(iU`98yobeq_Vxp>ePfEWVXMN^)|L)8wBMo26F(KXrP&3?|xrZ@?#6F2qB3LR}r9 z3(cqD4NLSmF}?K;3yA>{2!IM|9y#H<)tIfZPH5P!IZeOURnLtNHYS$OXma?(NO1); zrj9EDViPqdj!N0BGQ=R)cT3mCAO;{K=`XBkLa)Q>U{#B1MUoK^TcD0H(UDP6mB>ym zkn2TX=6;a-BrcZVGQy&TK#~MF2C(RK^%kBU%~g>~6bjxutWJ;(QOS9m9> zYDfrjR0EObM2jCL0RDzkjKj>XkQZ6((k6xU_-i)10GtGdJx&Iu1k+qBbYDjjJgJHt zd3W-!1ud2ueF3-%oHwo)M)-kcS4JWN7b?Z*DfKU?$&)}pG>8~NyzX?lc_UUy`QxEo+n|_&BJqb(6NGbeJX`bMh#SB|WJ0Ft?nPiLRKi)A zH{P3tWS@hWyfp+IFf!+go-LGkkp$a|Q2`Ml1f?r#K=}eH{UHJ;7T4}H@%yH1gUxM* zb6}b-52pEzi7ce+HyK!%H@Q(GPJ|yctGKkio{f8Fbp?3xk7ksEyoeH%39X~2q-3W> zX;DPdp>JI9w0GRe)DS$yy9T5l(y|A-` zIro(**2i+Q*YuZCI}<;#V71n{s@g1rixJ2wD45a(6;9cyfMh_%rK4XF>V=VVgD&xw zwTv)K->e&m5uq0|uC3qd5K-`eV229<2#|>tPN-`X3;{mX!0DJ^-kBBZTt4xjn^G6x$)lzv zILQ85-z3LWFQ%x;a?_1p5Mq`a4N0U5SC4PnZIIa~! zWKlxDR0Q0F?$BD2fiQ4hIT4Zut8yq43eG)ZQS63xA3b42mvJW%EkYiU6t*YQTJyT0 zOl%Ci(WRBLXhD^^AO!6cXf$AoH;aBARD+==7!kbCyVQ4@y*|Y4AB{GAN~GaqRqk-% zhuyF-IfdTnj@*8u-f8Aey=HwyxNQs$T}Uo~Q^N66ibO_{AR6?yHv<^CrcZLOF>nO| z>EYE(&vRxgrmaPT%3Y4=JWm`r!HIM^j^=zGl`!^St#;O;1}jCvS}6!J@oI{*BvABD znqCF;85az|evw&=w0L0x+ccReL>B_Xy7gPeKpYLt{9c)bnhmLR+w zWqPI1wwxsgTev5D_s0!iO=}$6ut(W69AQ4F3(PBjB6Cs+4=SqiVvHmC*=3p6seLVU z#g@PfT$(XZ5(|xosxjn=Ljm-zJkFY23Ej9u*X&%vpLU;t?dQ6d9tnmAv&8X0LCis7 z87OHH+V6ThdrZ0V{2gTtuYiTbM&&q0p2R1;Kp8B|s88FbLaIU_MM(m)-+F|&+CEm# z$`+8PN?OiIN7R-Krr0I8eH!3vbee%Pl09P8m}}zlIG=BV`>3cb2mml8*eD<`l{LJI zS8CDjyQUoAm`w{VfDyxP7)N)&wqH_G6->XCT?K7GnYs&OaFtU!S1wJ3Z!8qZgsJi2 z9L6M?ZZ$}0?Cc4x=n>kW4@{fOjxemGrYi9u9s?&YxR2(ct^oCZtE5r0~Fyrx16l!+!(( zB0mfe5Q2yjmIG-Qwk7WRSp9Z=*pkr4-F-w0l?#BFVl!$}LD51&0dB`Zv1D;R6j}EZ0da#Ut_N%y zR#}UDaH-OFvDhngu`CuPJfLjAMPqEW9_|*Lng>!TUJTqkrLKUBfy1h7Nr+U2!8M*K zS`sYG>y*z=W|dLK#-|td6M1VNM`J-j1MxuV!5a5HOU4Zt*T)Q15SdSt*a409*uvn2 zo^zCZS;gQp+H21b%qj!1iO8b>g)d}q9N|RnzF-i1rY36j9N?98AEiKPIE;4)WoLai z;9|z8>~zbi!I$mhkAvB=NEuZ15)F|fqb#uc$d0HRu&sQ|N*ffOb8^CF8c>PU1hbA& zK2&^mpQz2JZp9|cmZ+sH!iAJV2NS_4I9V}49@c)5VFJlngNbz?Hsl#Lu_UM??6ho7R1=NzvCX)~)9kQ9P5a{(5E;E$k)7GB7$6SR1c)rP8@?-y zgN21K@Vuh6fZGz~1<)hThJpeR5|dXFEsAB!ED$7=F$ai;rqJpP1xyM(sBS|#u(?l$ zbqe?QN+w9dw-0>FtL_r6l`vY=>&dWaFfoxOVFcPKZ%U)^k_~4EW^NKk(EoJ0FU0np{JXj zzP^xJ0IM?F$%EB!s;SAN?-5PU7nM5rsFKXUY6DIAlHa*<@y5p8{3VQXc_5}GGA_kf zD-w_!6m8MF7;Sc%Jv0!R=X_E%r_+bTteme1Ed_uk6LSy~C1D7{J^0#P$CbB6cUFPS zt~(78e8UIo2)n2gEzTuqs@;XS8E(oV23!My?x0I-?^D@xM-s8F^tQ5#FReGcD|ha1 zE;Cw*IeDfj5EN~}7tP;1r~zW0SuezRC;-MzW^bq9mV*ULDNFZzY)b&T@*a$vh_zKt zO+RFuQLK<3gJWXkHn+B$+^Ld-XDWt4)9Mxfmb!nMb;DC zSG6rdC@|zBb2N*@!Eii$NM){V@RT7X$vuab?#81$_1b%~b-#+~>D1_1QT9q%gP(`X zww+`HRyq?s?h6u>lRXH)EA>^5A#z;;y&8#b7&2Bw*c`dG_{TKgIM1t#BgxuJ2A&k) zgwpf`f^drY_n$n=P~;RmjiB3m;p|vG?SaC*U5n)7D4~q#LJd*_a*)N&HBS~yGU~Ma zwdPur6^zNiz60w-NU6P*nA_2zfM5g=5daXfSp^7nc_4Iw0ysc~q*TpEX;Kmqeqa$* zA;NBv%1?XDvPvrA%1YpePWeD=pExHJY@F|Ob$OMjZ3&&?p#8g^okwVbIy5+$$GPSK zoi~_dIBLyj8hEoO1?uXNbKP9VS);Uvky}u|7EYtCoc2;`Yk@5BTegiwe#;N1zAv<( z>Ch?MqKXW!erRCV(!eAM0SWciOqC_oJ~3Ge^zqG^hg^=x(e|`%=yaaJljNa|*`U84 zCoOHn=BwZ)dHuo+0t54bbz0%82!T@Q@+EJR(*Z+E)ZP`m-*{k%<(D$WWPMSN#Q?8o zcH12RIeWUybE=RbBLcNyIs;SDMzfD)=*-}imm{*_H!K8VzfkYH#SMJ%MyTSx*VCD0 zBps2#*Ex>wBVn8$Zr$^B6n5Y>Ah0(?1cwqeU@D=jP5JDqMWBNE+l<_yPD8avnZ~Di z=D3Lx{ZDmJs3e;subpHmOlGw~*dp^i$8F?`=J!d+0pLT7zfeF6oo2fskmlgVLxSu0 z{BAuiNW8t_V!Bn9$la?9?M?D`e+lxheoP0IEKjG*+OXo z(>$4c>=MwCks9@%F(Z+5V4sKtEIi$Q&RZmc-l7Wk*M$)Zjd*m08aOD7^_GNfxuKJZ zvH;H#sxj_dl_>itDuRN`4rW4jo@`gZU_VIpIH}6g@W^D7ou&5dqXzsS4NYWTU~9?* z#n5>|r^rux3>Y9G+eWRJc3E_U3VteHjzTM7-FfCyR9S`Gc2fQea?p$p;bLLYg1|7e zTVq$Kh?8{1m39imTk$<{6?A;DC5Y1QtG@`_lzrNS+?MNtF%LTN9i%B9P@SzYFlgv7 zA(yTIPdNlOl;%4zprHDP;g!GO8oCRgtY#ApnJk+~1tj1Of?#qb@JSLQ8Q1E9LnpPK` z>0giTEL^{PcY-=7>L+ARi+TCPvIecw!{QTUwkAR+WTNEWZ9<$>DCX9y@KdhMmGz0J zQNg~bB|SVb(ZPgkh`Hn(shlYqUkX{#lX0+bn%8n+uGDhZKA{vd^>${5sQMifUJHy~L;A0x9uwT%nfFLg- zCs*bjd|f@<&tVt4Nyra%2n2!0>l75sKe@HGrtogY1cmv7jhsUKU6r>Gy4Ri7gnA~P zCg9m)Q?q{-+PORBs0#Om#CYoywnc!U6uc+k)mGKedu$(S(!6kq32u8BoH+v4DOiBu znJjF_gd%$jnWNRD!Wcx02V<#q9|k=is8}zc2}vgNN33}o^y{gD8qG7!kx6t1yQrl~MH% zECgslizmeER!RchE)qD+o!sI;f&vwlt8kE!Xf~L2sZE(YIqmang&(T?;8TuH`Yg(h zS{f#iU42-?4l!liemsenNhn1)#O0io^PiF>bDS((W7na8(~izi!g7oWallv_qG%T# zUk`|mS*&`KkXY785*=K8RjJ#?6^s`o=+@hs2C2qIJW7lN0~+XQy|}bUX+(1 zKJdr8M@N&{z0uplM~>l%Gz{u>md~m~7XBkt;%pZ>F-Vu$GI8XOQzR=#dZwm;o$C(^ zH?BSp4;B!3_|>`MfNLbF9PAg-9S=7mBVOi?e>0m02!e>Pr!^3S>3t^J@9Th7xWw)k z`e+XU9kNR9V7J>d$R^gppJgr*+cd+?W>pdh`r^q-0?j_kCJ>=ULy|;ex_Qc+8xzW| zf^gXjc>6Wd2z4BQY^Pg5)@)0$7|khHt~_+o`yN_k0U74Sz(==&zfK|*rLr|n=Dtj@ zNMscn!*`Q>#Prc0xESGB zJ*Kp%W~JS4hpkC8g)ShZ+}Djet}z7`dBhT!%7!KhV4m?xGS?Yg0yz?5qvcwH@sb~| zpu&WNt>vbRXA~dY=0VgZ)GUPX_dJ?+@V2f6%oC&F1}|Z@is>+6tf9Wh`I!nG^n=E> zAdE=!&D5Bul|;rDfO+5}?8(9nH`B@~ibP2;%Po=}h`U5AuCSGJ#Q4n!IJKLc^TBO6 z4qIlDF9Y3LOp&32w1*P(jMfj$bEd9rCyecvF=Nb)g;<5nDIlnyC@iZ8!d^x(KTLJj zmrlNri`b3T@Ji~bw3~If0o}-x+9+|X=o%I zPN&?MZqTa8P$;IJB=uQ+_GM2DE%j-kPe^i7c!lMCE8x*1GxFzlPqueFMn}4M1@qiE zrK6kX-N@e!MTLm)&Io|Vk&XH3+&jn>M8P!KH-%L0h8a2M-#8P=Tyv%6nR(Z;h9OL& zg2`|t;s~0I8^i;U)T5H;>+$AIVn+O?FM3eQX#`?;P)6$_6f*F}BdD}3atz*|vI(0y zeyH_AflRi{$i_R7rQJ2-43bk*xO4fCZnJtHGDqB+6C5sp+00k{cERHm#b~cMl-3@bIJ1a7nwo8R91?lv^S_!I@Lp({ZtrlgT zAdyV|!RzMjF>&oX)Xv+)Up;{IOsLxhsVMU|j9}f<%xRDu8PLgsF~_H+alx774A9xF zC|w249_P%Xn1S|GxDq2ftgk9emEI+@eSHWMY~>;|+rp|-p%&^?q3+JP!A!)C>=hG~ zhw;9EP6TkNJou~4LWDIBX-8p@eyr`W+Pn| zMuH}JeubVc+6+My#wj3qe4=P1?f12?GK%2eBpDNeJPsF+$kgj#VURPgLP;X(5-w+0 zM(kE#b=@~HrPvw`c`%o>5yt4Shdl!6IgovcR67;vtY8_{8o6`uk&<@c=$#1o*&l~p zNNd?j!NJC6WavWqIONrM=Kck@0YQ6QzjGeJE!^JAhc){1w?*^}V{fU&Jeq0?u=s7` ze9^b_o1xpNgDu^c?%vC)%R7*9p^r|z3Z?;*N2Tl=#)E)Kc(3v4g6v-V&XbfvohPFe z8-n+%vB`&&<9+KrUQLH4MNbSOn$sg-gSF`3zbjKuYo2K7Ph&E0(yhWHbK6PC*y}F4 z_VJHB5f?9Xolav;=+>_=T8*4oEse}$R}OUI?uH;CoIPt$W0AnE4rZJ z=i(e5sW9p(g8MnL8e-R8GoIs5LG$c zaV_HDI3TNa3@A>>exV$0rMG;rcZ4N^?LLEx%THFyO?{D7ojf;Hhga2a^$~Ea&9@00 zvKFhNL!{E28ebAF@&vm)6shK}B%6Wn5e=SQhQ_on3{&6=;Je9q3y>9ah((p^FDh_H z0L318tLGpUe4}{k+-EUVj3o-v4TiCLb*A}#tc%SmX%miybQF!V9`9jd!@C$iy{rrqHQM;nRq-t3ED?lM>L*l<;cnfMOH#JWQ20z~&C)d9EI zq*8sM*de7AYkW~liGs5Hu<6zZi4`QfB7)kmt-!X7+9#mfc|gWrp5`*fJe?KDr>os4 z(U=-ekzro}VVp~{yCR>9R<4F86F}&RhqoOcVj)jS_fna1yJ|w{mo?@6cK3P6h`UDF zApC~D=C^bShUhxx@NH=zWs*Qu)rGlB#@uUD^(j`#kX_z_hS1_O9WN#3%B@0j!@r%c zKyZP`fVQ0FgRSOFdaQ6?aWfpafuVbR{9Od_z}B;{f}s{%^wxTxJ(B6gQy@Xgv;v^D z#Lx@#(GAQqe7(;gb4b=M7WShXSRO#;1J>Puj=+(61t0~GPIsx6EoC-chPy8^vAfts z3DPGQEj4|nxIBo7@~YWe2+F@?bv0HA90x(3)5uSNfusTLxi@>uAyJH37?0f`V4?%@ zIG(EoiVPG=A0?h;LwU6@iedM4Owt4^P)v-Ege~zRalHPuLtA5De4d$OmrpLl_Rn5i zZ%9X(2iGsnkw^1Kh_Jc{&+X2I1BWF#+oCK7$&p}ijJ%s)P0EId6#N9uyc}7WbA*lN znv-Vp5EHg&bZqrcguoSig^S*`sTn5%c4@Rep>by}?`?cfr` zJy~MM#V9+_U{8H3@wk{oGm3{GRgMPJv0;w2mdW%jF59*G#XwpDLpgHz6iDMQycR7z zy!JCUEs@zvs}r;?iDki0YByYJIHfySfMw7HCoMKb@?i*8bE4^$%RuTypdQ)gXMoBx zJ$r?c>{`i-i`@$Imv;mf1`1;CouSWkv}`>cr|Z}rZG6fPRE}Jgd_#ocOUs)V*1*gs z+t5#Z96l7CCU~>+6|FT?CrJ1z1xMFBy*Bjm2O?yR@fCWTGx@-wlfqEt5$?qr5gn`P zAQZ9{ok&yBouSwQt2+7*Z+^tCC@zG(acM_$jDR$<&30wLDvfrB(=N;Ih9j{W+vBJ5cQ|SRG@yz zK=x^o2adH_c~Xc^rB*c#&zNS9P+V9mXnBqD2E52A&b?TGHjP9tE3Bg%9%{4($^*Gb z3mG>AdVvkay0W}@cp+B={;-iqABo6-jXRV?W{z&6h#xW};9l=+ttVA5gKD|3!HYg} zuX-JsQ22=!6~d)f!8|XIhY>k}eQpxnZI=pm#n*UUMe#Zs3MEZu<~%-arX81B^4&MWGz z)9{`nje})pf&Ib_DvP%c8f$okhPuhkDh<5bS6D`5#<7L7q|FYI2U%pm=(uw7oTj*X$Rw&6PI}WRTOwHmh!lwNl0(8^cqG;gwU3^jLD+G|N$sjL ziGvLZ&l9`+G3^-yos#8yEH`@_ewhLpX$u3|%8*|Q91_I#&Vu#k20)^SP{ux1R}#Uw z3+Rtfzjfc)Cb(@d>&gNJibj6vRhk~wkxOh-t4I$4ba?rsbrRjTj5H6(6?+}oFat~k zSEh9k_gXbmN$O~8K^$`QySE+(G+$0Y68jM+jTkkLS_2F-W0jUK*3~YqKVp)Htrmnr z_83zfY;TUzW9JqQr#Xhe!LBSIcSQo)2%S>lx9o!h1>;%jYJ3^r8a;zp))K5{8Ipr~ zxMqpsjNZPSsCQA$Y5K!oNhxp#F9PVnKylX_=G@<~9}DFTpBf+tN0JQcuvYu#pEU?5 zEYafm{gar(TyzV*FJNhkaQBd7w3-0D65cO2R~g}N8k|N?tf>+N=dSz$wxoP-49zXc zO&Mxc-X}xKu^0^FXS^Q|Vy&+_hQ2~gtOE#9Inj_v=!?m0%3Pr#G{`h+gF1(?O(<%f z0976?R=*8;??`P9D7Q0TC=^`t2M=60%e}ud&{?g^5Z^=PEK_ZQg3?-dNgL>*ihhe2`RU2 zs-R~h5}$eF39o!Yji};bZAz2{G`)URAGCHE2L@%c0jcA8n!@1UmV}D=f|77=NW4nS z#FGNLj7uvUN`@74EO=jS%k(P< z!km|C0dZ)-2vwQURuEI$%(aTTuZ?&Y0UVSr&`Ae+j&@t6f`x;Lf-e<-YAk1^CME3v zTE*EEb>Xq*f?j-enVNEi?-};)?l0afd}-9UvCx^0Bge3TlMbw6oU{TKeGE?cWy#37 z(Bs+^iG{0Y(U}QRSm45wl{_)9v+Z9d7qO?rbG$%HXl|DH$+dcs3CVMn zHZ2gw5<-PnMm;6O;{yhBjP-9lf>({X3$>Ct1fsq2`0-k_ww0Lkh$|>Itx%sJmL2mI91fk4=RCfE>Pu9; z6+vVxAi)X_PN4PlVICl4LR>+j*u;^9zzakOFoSx4aN5HU!C;k_f|ORS)mV{cI3iG| zdu>P)JtWK^xO#WW_n?&b1sg%XBM7{5RWP31smiVH9CG_NlZ7G`TC%&xlDK`WrPath z?(@v?lF8o4IbF(uP(ZnZSiMV0u~)XGUo$ZUbR$$@aZ4clG8oR4ydmQp=X7+p80Tbf zjTuBEJMI;&3T`20S}uVfUDLWKiD9}WhC?^pffISrt$Jx5sT`9C6wk!*Q`- z-K6_jLV%3l#(hjSlYt>1UR5~D;hb`S)!(usTC7r@^jsxC;xc(j)|8Cw)b3KnXp7K9 z{$5(!o&x(pr@pDY^s9x(Df@g40v)v#&%6#rM&xyhhbmJKUp!VKlf1w>U^`NxR`* z;>85?Hwn8Ryv5s{VVa_15=lq(2tiLT552fx_;OM)CxD7CmWojkRuZRvwd>@9u0^5} zay3mTV08wFy0%xq@D@p1&fum~@`PxLae*MVbQ}*;CgNy5qFYBzPdMI2a8x6q52i6( zi-<%zL6%&*W$1W9tm(H+i#)O79LTNoS=)=;23O7Zm?}(I?3~=TA$vW0nBNd7$eA41 zY(JWa*}!6LHFzP`vrCzn_%Y7(1gK&)smzsz?A-$tYn`4^EwETTb1WsQ0=(3WJ>s^- zOyVNGpcgTK%)yId?yxQKYB-Mi-MqF21;N}NZBn7z0kFzcZ5y^71La>+=&N}X9SUYb zw&B*hv-9;Rm@u+Dm)ia5G=l)SI7EyQtTJb_ENyfP3%b^)eL?qF?ip~-P0f?G^b-@! z)zA$PB6-G+EQ(v=DQ|bN8<4qO8QjKQVs}@7)SRA5rAqM@0N7jGYOe$p#8shCY1)Lf zwLny6p%vK6)hCYWc2d#m95V3ZuN2xs2CHkf;+18~)RGHBAz1x31I1lXLjXP_83Ji# zu*$X-ERE(VE#?eU>b;3%3quyvD#d#2Qx*E-ObutjYWy--2}mqW!$N5Kx5!F-(@Nrv_ChsP6i`u>I5v8rn#)zNig$Q+!-i>#F8QqpB&6Z!LY~20Hv-H|qo=FHj}&1K zM||n$OIyVVm+w1tNt2b$C?UbmVcZQ#4U&0ht3+P2nC)>NLmy1GJZK89m8@v8fV!`2 z<5M1XJw`*KxkDCsm5g2lo4`sv8{TuAQ-_Cb7ivCYL|nDLkrc@W@ppDi_69D_f~%2% zhM0>R8=Ib_$(ckDtzcXwwMR%(J4Kx44LMobmv1{0VA`s#4YljANY`>%>D^(aO2|gw z4aNxenBWnyF$yiFW=aXHVRU278fgR$syY#eDgpY>okOj!5JvMtra^Z~ydwB$<7<}y z;O2I+MmYc*$LTxmbCD670o_b!qoOPm^t_8WYH5Yd_suYg7fyuq6EyOfw|c)bPhNcS3d5CbW5 zNb%N)gDwS48{xX@mji@bYeXwKBbK1#Mf@!>ZKHX7s`Qxk27-R!$@d&(DC|NNChe7E zb~|L&bJIS&*kUl!#sPx~uu|5qtvk5$%;F5lE*PuQ3Sh;_Pg>FC>Tn`M!sfP=p)u>v zIQGUYJES&vUO)W2qlPNmEynY9`!iSoaPQe73$E z6}Zj4miC?3)|xuv3)dZ&o*zeHa*Rj}c#Qzp%&e?HH!Uid@jTflfl~`14he#yq$23! zuBJB_6du%S?R3$~i3?}CqpccZ-karLb%<(-+tExSZD>kSfcS@lGtm*9j~@)cYDKQv z4G@hgaVYc^*yR>XZZn=#`#Tq5f1 zn}V#tNs#S^FxrM3C4<9!^iQPw!ICtq~+DcfwzxzRRoGa zPqOB4u?>7cs)8du0nFfBIleG~A0|201)#t^b33IZ-)E7ei_oYcJ3^J^J4%PjxT#VG^{=MBpnNtqCP~XeY!K!Du7LLoDNeaWg$l;O0`jdog@YW&&%)oGjMM|OL=NI7Osf8N{^Y@D3vw{h-Wld+Gga~yGw&B|y z7>b*=Y(s7yG6&W?9fzQfUvE{iJk?6?9Ew1X0g9E`$7$;~s;r6)Z!Q_OGZ@!9Sfe!d zH)FTNEQe5*X77E6BG9ejCVX09Kv}-Qkp)9eyE*}W0k9d{cO@+3G|+hD29U7|ws(2l z@ddk=2+Mewx5sJN_XJUEi3n`FO`IO`*f4X!95mu|PHeV84c3V}{SRK`^Dh9PERCZp z0S>TWwjvN5Emm@ON$Jl067TDArz)R&D)?9ySJqmqI0*aPnvUkpwyu!tC5u*Y!N#EBdOB`r^)Gu8s9w>MjEsytp-2lg+?!kj z!%6QGBg&hgr)RZ7?G}Loz{8=Dh=rA|V5f~nJgHnI*%R&_a|9ifQG)|iG<`#MdPRE3 zE%U(BQ9w%0U>exmAw@Q)6T^3m2p7vXM(Jq$7Lw^8;__iZveku5%GnmqoW$JdqtC#2 z0x`nG1!J^j)ybyzndQ9BOOAm;@VU zl|-x@k01>J>9IM=x6zzhgLz+3^-wFopkb0|=#1>kd4_ymxtQ*J4YAio&rxU9t*Y_~ zd@{sfKzy)ak+WlRz`&Rf?*P=q08VNZ!0KJ=t6Mx&1tww=P=)$@(%cZTh@r~%=GL_r zSP&>ng3J!r45o1hy+c1|380fDtCdyKTtS*nfcf=Ls_HWEeu1<`71!g%DaNuud6Q!z zR!clv94&_A1P@9anwM_`L!Fccia?a2Ud;2r3_J|J={J~@Up_KxjWVqD+R3U8gyxf| zP9gYxJ@8%G8KZG)=ysu>DuK$yEf(CJ8#zaMI9>|e*T8BVeyI5%a4lt3W+h*!GX3NjZf6nOJiv3*MoV!2r9xI>HAsLSaK+G*xKK z3eH!oVeCR*&<4Ez5u|Sfd1RgUD17;N{!Zm+HAQyi=5u4cF#o9KjT#Z=md3d#;m|QaHgIr$t4Nz5(8sM`*q`6fW zsb5Guys69rqE)MU_P*ZAi6ZS^ulXF5e3ZWbQDsix)e?G z_PnK->>nAIbBhF*ZSdJO`Z^Hi3)*2dR~TX z8&Wsp^YOahyWznh_P?HApQ@SXIk;aMIi0m-L7I4h+Bm#mICPw8>s@eI=Y7hEu%^VU zZ8Y>BYcP?)4X%C0b%fkh&_G{Tyx)AxhY-x@``Z!|e_^ixCA7!uUMQ zZs5;sAltLG)eNKz<)2+Y6{<9B^R#Ny$Eiy4v z6-FGegb^SlcL&vn-g3p42FS|G%E@0fokqsvZNTYStBES0`dRWI6S0$+tF|6>0cikRyxlUUb?rIWOd;*|G7>OSrWhY9ZPCND4(AtwBt2bzD)3(vIoXH< z1az?g*C&SPkD~JfZ3M*Aj|9b~mpT`@P?4ODT8psTM#IDbE!iuSu}n%7I}40-r5o84 zPA!Pt+6)rqiP=7sd^6I2M6ReeiEiJ8YE^_gr<1~f-!UN|O45;FKRzB3Tn%edz)lViSLz|GkmyPjp-TJ%;Hz4QT&XbZ5 z?X3CU&27>{E!K3g$4hzc@s^ zmPwm%I^vD(V$Moayv1wjg*LL>qiZ{Ct2^3;1#zGTgDcBs3p9xG)0zY+ub8mwtj_3; zjWiz@5itOUUJttyF6NL>GB-%-Qz2}+KwDV#15G0d3qZUN+$*W)a`gBK$tJatI_fM9 zy47{ZiCDnF#htd2!#gNvYW`MmaNUhEYS67t#!5=DggWL^_0I$0IKtV2=RL^5niU8x zHs*Z5!B8Xz%~I0)0Hk2GtK3dN(DxEF{Jy)XRh6=1Qr?rB*p6dTdywK4XlODQ^3hNT zn6+<;NVj(0HtO3!yA(&Kgy?+$Oukedt&&hQA~|B~!!bF;H%>RSz`*%g)SusF_{ z&wCfRGDwt`b(7M>?IN1wEA6-BAj8q5%LFhSluR-LLbxnQ1_<>uxDBYzZNPGv$|pTA zJes;#J@6f;2LyY%&5AQGtos7-=w9@#iTM*0A=k>tNN0Eh z?v%!!?>T_ykqHZ`DV~;63eV1Zp;qY(mJuV*xq8=)$QpKa#VD`kT3zrXQk}~uBUTlt zdNJNPXA2dp_(MY3fcSOSekw<|&xRz0L6YdJsX7t6%n+OdCM~%RpmeABidfzy!`#-* zYz#oQ~oeLn; zf-M&fE@d1WETVa2I-fN;@`k<3=I+gE77`b*qw0@BdfUR6m=zyx`k`z<729XU-Xt21 z0Jc6)5D*9m{hOy&uvRBj&dDc27K1FEJVwduo9n8jPcm#pH%r?HS>k$osjnt=%s50k zM5D)Hism@E#GUZRO{=8GvZ4epkuTBZ<>iaa4)v|C3<3;^fVmcMnj&|Edp!B!F_Obl z+2=vbNCigLm>2`YbH?1gykzLHL1&)5h~R$O@vjYEg!1HEjW>aWFj!wrPL%u&nIQE+ zdrF3nO;@q#5E&gHT|zrcb|Ix7i|Ujo4qdB*Xxc_z6~K5JxaJ2@VNMmOP~8!aqT-8s zZ;6Vb>(eMmZJ3^1FEBn;^~9*AdN~OTy9|}Y7+7Xs58WuD9Am@g@KQ2f!R`XEh`5jm z?3Ls|A!D$kX#_7Mk6cTbsSUGwd&)S`$2xh24ulbGs6#SLBnZ%roErMm%TZbx6%sV6 zlMwA9BA9ufZh)*l5s%4yYgkt&fEYLm89YAZsWGP5jBVlwr zJq<7$67JWkze?6`%^c5|%+Xd=6bW0E)UtI!B{NMDs*%s^b0(hcNz|XfuhnNXVXJFOMN6FY*Px_&43b^a8@J3 zm}D-IbVz$O+iL3oJ3z$0v{Dc?!G^xQ@@>zp3GviuV?kiYoX7i8m*SHS`s7Fm4%5A# zIS>jijVWCi_6GCjDTa;+0g4H6KlO?U{IO^8D{}Bvo0a}{zFl0 zCG%DH(Os8A;Tf*_lzD3A{-bqbt;oUX6<0#QaD}o85W7Z5T}xUN*HiQ6!DHL1Ne1=0 zY1Sp_Sjnf8H%pEev8eV>E8fFaIlD8&@J1JUdR!%}(T4C|-Hnot3L4i;4T%>EYuV9D zUMsn@N8ZLCsV;`mkQUql*;qHMX$=3kT> zZr95wL(z!wY}O6H-qI2rmR4BJ1`*aNADk8DLY+=1q!#>CXD+tPX{TZjE6C$Y1Py!2 z(`UP?xODipF7HcTdJw|3^1<#1*wm-b7KF@!xTC{;d%}IYgHlPs+8!oDKeB406yUY; zLRU`05o>~zCO?hMUCnuIwi}+l$P%Hp?da3XPy{{{*eQ2r0}piL5{YZaIjauZ-q&kW zF2T#(Y%rbS$`FU^mF1{FQ;?mDBt+D8B2m5W&qIkJ2Kb`EnjPHNWY95ir7fCQ z7NP`$YzYgKMrcyf7ZjxmY&tl;eXyAAGGhANGDO!37fw)t_&Oj7C1a^KCOq>5KD9-g z$m;O>2VYY#H4I*T?-Ri}zGoL<_h}e0&zQr*)>+bQJ-m5@0xcMwth7Rel%+7$mA0~C zcjU|KYfF|Qs2UcM39XN`b-0wI{B{oDJ6iToJe9gMp1wR|8v-E(y{(N$3dUrBwY#Ui z!RsLofX`kq=PV#l{o$RGsY%2d)_c@&)v@;FKd)D++~LL93Z`>+U8;gdo~MQi^5rYq z;p=lTVNHO?663Js0urlLBc+D%74>pBtri&(j#tiihY+h`h0bB>_j(?R`rkNPHdB$? z&MwIu+?^5?>QWhD@KT5w4A)Zci#%Q!8%G|H^H^^bw33-NL08-{W(no6g-h>6Eb@?f zpfM$wZ{)MSdziB@;^ymo@J-g;?2L~M-Y120XclW)$3$z!b*Bej!Bev0 zFttVn*mFy7WAwAnd)&*}OjfHhLL|ea+Tv5a&mAs{l?t1o7@z)WBewi|~h zc9wm&Z!zX*CFPR6q$#tRlAYM0&~MdkiYbI(SZ0^7K{Y1LEmz6DT{at@dUhe3V%;_I zs#505s48aPCF$n|dLcTkFr0%6!{i{4Mim(|#OKSF+7xw=aGR!i4}D_sq?+^@1~YE7 z_f7N-J?#ef3q3C65i30TY9Ua_lA^MEvqmS$yfSO1cJP6i(`_hoY*#9L^~7mdx>jKC zTFxG_Gq7f1Zjkkt(in+Z47c3Efy?d84Di(x;lO#DgqVA#ev!dUp9pFP8COVVmD|6R z&eEuDQK-z-U1u&~cTWcL0RaJ?7nTb-!0dhNYa-qmEGos|-&&X@l)K`R#DanajtwP? zVM-*md!}49Vp=cksjOwKgL^T4A3$S=3)}D$25*Wv3E}&OU%Wot|IwLVqA_CGO%YNM5(eQx($Nhvrp$3Xd@kQd$i! za9S^0V;$5pwa(i&^ z1z4lz!iWjN~1BTH36kCjT8T1T{j z2I2%AcFJX_@~~{0NF2nZ@=u-7njx0bg%RpEFeYIjfTZ`vlp~VWf*VDLffP8>R|Fzd z)XxP22uHUep|*0;%#L8}Vxj!oUOu0V9tDUa4JOtxgWM%ayw)Ns=B251lRIfSalV79 zc8M^a=~<;-Shc!P8tI|}om{jrwTzczi1ErfYdi#x5yGOZYFXn-$7g}D6?N_mc52XK zy5~b%DMI+@OC9VSJ--_;T>(5SGS`N&m~F)&m*8_Pz#~f|N)TC9COakV3UG*Y;mJ#Z z66szaMYv!Vpigkb$loGK5OC3(+r(8PTq7Haba{w$XLhVk6V~joh6mHq^c8(y=Lp2m8H6pfSub^ zBSa09#vSO#pcphxjzkzuh!o% zq7odN5q?frvD_8N9JHLa7*$J`&mjF+VFX$ZCt_8QV##&aC(~YAizAj|9D|j-HgqVb{nTf2%O>Cm z$zyMA+aw5GI|Q2LHhatl4z?Ax5Z^BLauA8zhM<5e4BuSfb)nKaTQU`h-eC(kJCbxQ z-N`@GNoZg+&oI1TlI(AGq>_Yb7Nu5#FLe+bE<2RrE#y#1I6>m-!k98QoCV&~X2h2X z@g@wrf#CSbsnOQzZuE~(zNpM)?kM(8PE~mvF4>;>?L|O5NE1+wAlBlX8i%OKxFkng zOt7{vHr!GW!Z%!s!p}-?#g>xq6v>X+RV&9M939onf@=GC5V!1gwE;@skide`iwi)d z`!z?W(3Vha936r(9LAdAfO9PCbpl2R(L>2ue2G9&m*z0^@S2${Hv?={q@sjl4l^3h zj-*Y_Gv!q}!WmJ}TlXaiIJ139Xf0o6CNk#DJrNqZ8Ksie;W+dbj?C&M*?My$5sk1N zD8z&tu&bR&q(Iq*LrVu=v~uq7(B;Ll*vq<3InOfeSJo%!v9!M=m@Rth`rI#3r(q5* z+29@URWmh>OK#-}YU9;s+YKmDvQRWTiSdGoNn1kDta}$s^<|t(JmJJ>mob96I?3|d zVo?3=?vpQMN{LPeg3(}>u!*cOeDG<9eYY;s1sB=5cC2E(X=zzPkrXrqh->bs#92+(v5I2mJ_4m zW#DD&vYi4%UZO=7JOg=~LGi<_%u?Vk`^vW4YcY>Et})0&frcPjta=4AS0F~}J!k1@ zlZ@GcNEK14iH%*8B|MjR3T+&lZH{^+U|Ozg*4~WkZnvG{%6Jx?v>5h8nt{T!^ksSp zCi4p?lyQCD)AmW)ykl@@t|V$h(J2Ya4_T*x_%-ti4FbWX~3nJ|;m8Pk#YB#M)iDMoL9JZ*lb;i2| ztD7LFGlSr_o)nUKn;R?aDWt=@i(tE0(;If=GNzot(5`t*Fzz2euG7iLgD@F_V4O@j zoV<4$x)33-a`mzYY$$US7-ilh(i!j?vc9|zKW5m@50nZi^#mgxr+QJO>C9a32_MCj zn075Bd~10#vwqNH8)@*cS8!qVi}#N`nzC(1SkrWyE>Bv}s(U07hP9I6R7?;wg0gA1 zaQ4O+G9>e)<574%%xUrmL@TwAR=WL$qicqSkSR2Z>`Ri;j%^c{YIY8Uq*uvcp3k(|}ivr0&7IbqrbD73A zA%%D(uL^ID*r>}nL{u}}9$a!sA1k#eQ9i6RvW80;|kxn@fPW;owCRckWc~ z#s((@+0~+I{x2lSmnlw5oiR;hpsCyu!VJ81v_8u+4jGntL2jvXrzvTu?-|58C*Mtk zV#!1v2?Mg_DK3n{=~@#A$?roiv&xgHFW$oW5a?iNUTg~w(c9R}87&>!LQa^>3RNj` ztx~NguMCnPQeshnOO0v`k(iZ&e17SC!afp#P`?8}7~#+q~2mJGo>Qg5XvVqykX5tcZGy=p#hY zvRF_iwK8*jEhwRY=&7Ti;=e---9eV3FgAQI_mTmHtL;8 z5fx=k^AN)a4BZbqpM`SE#ze`wHq6v~wS>|kj_ApzoO+yyR) z9hBe#n5OU!Ow-)H$GY>Hd*a74*2eIopK0DmTZwoDUf~BoN6V_{K%LUFBdJmC>1=m- z9_M!-Mel|6xCyQAt-1!M>$u`>{KAIY9IwHsbGO-DD>pvu;sLz5p~XQhiLYD zQ18K5rEq(yd{aWb9#0VJ6ng_8hRsYx9zFw$ENSv6{OwATz6pyoD|4jThRb-9gZj#^ z)vXIT#2EfQw@PU$8nHZ!MTx@;+I!%9PI?#@4MzfeDv%Qwq70p~B%suUPc~PFFjseZ z_1GZWiQ`0UX8P^Oxj@Ur#soFi&rl{+sScT3yJCz$+@3fY94)vtazi%k$mbi3$7sTp zYa@`U7tT?-T3Y5IO;)CTfO){lUrAhqI$ZE1MT+zJwd&6bbX8=!L?nuf19Pr>(4MPp zADHup3J&38wVaUmlKIqbN8MZU3`}I;92E~}jida}UDAzH#tae-p?3iiEMUb;lT>xd ziV>s|i*MysBqgI}Woi}D7J(*6WYUUAK6wyu#ohGSm$oie0lKLQ@@D25`*%zWFQNia zYKFu~kV`zxK~-t_6(Z0R7$b0~6}93W0`r|-`_>wPM&zrwCH<1?dF~u;xDorfJEm}Z zkkRL%q!npPttnl>Qqa^Xd3)2&XscjObUH7Ridl^c9s-KqrX4RL2%5NC=m?aNCAoDT z@7-=jEB3)bKH43O0pQg_C#iM>%9Cl3%9VD;@OeNtV=ZZy$DaFku$D>rMN+MYZ3DS; z%hi;yFo&_2+@u~&E!a?D0zrse$Fd^*aiR^{PXYvW$O<|~rQX}!q|xGF_q#D6vZ8%4 zv{*d6s01f{h$AH&o_0}mH;5XH%k2c|(^AdZn!INNcIUS!pn8Ys907;T9)D*j_irQ& z%R*5f5uxqO=~KBDAuKG9t6MMzedQcmE5=$j3bs;cp37__35Tk;&pK43qjxr$BiW`b zB?n9>v)4$`?oDNZAZswBrNk^>mv)1CHO|62i@p=O^CHunuiW*ZO92nD?zqSEa#Dm}mkh!q z+nXgljJy&gvMVns%2-chK8cgV?ock{e|R;j%yp^wZQXO;k~l$x4Hei5A=Gku4d(FQ zKJRd5qCAxB>0hdLU>4eW8sw(EsJPj!49M6ssh1wq z&JL>J%G_;%jvG}1F?$*56~_-Afv;X&frbZ&!tm%f6k?OatQ`YEkSmrh$^cFjrODgx6E+p|F-b@Y5M$glxwtI`x&LqSw0?j;a^1 zbPDgtvVDL7&xp)=>OCtMoKEo z(zUJEIf4;}zT5_y4mc*qs_@TjO}Q`Av9(7J#=Iew@d)nuvOA-YJ!iocB8>{7C79zO zfJV&?;Bd@{*DW8Yo&A`@&}%72phB7KQ}NeFBV#V9VqKqxYznH|G&{xSIKv?wdUw1! zoS~al!hw8>IqwDFtXIevWNaeTylbNXT(wglFkvU(44?=YFxE@x>+2Advz%sX$YQ}u zM)GsffR9ZugbUrQSE&5z7N`mW^;J5vML@*8n4W+vWIGlPa7v!!U2fD`@m2eTR5V3KmF>O2;X-zr z93&nNVDLkh0R*fG3=vXT&e9GjYp^UW(b#39Gmcv*=dA&yjWCLp!S;j80-)LizV>6X z5?3>DqJ~m%Vem4l7g&bJv6{l*X8;;*TT&fDmLc$k=BIJ-0}&S z^J~Ojd}S_oE}TR{Kx2{GZo{z$ATV3^Gw&t5Xd1vlTPK8mlq@~nc>J-q`y(e-IES&bS{WT}S%2X^Ca6wK0t0ZWBr_K%x;8LHrXAwzt8Ly$tE z{TpiU<&D5_s^2W08QQ)YD2b|vizbj%$*w|syO&u;A@JUq4O)W}aAK6wNgK(&kh!~1 zU$loVONT3xXV6pv^tUX~7%hOqTuT|{-SeqCmkSewj6+M%`o|!!2WG<&SS4KF3iPwI z!x2YGk;)W|a4YR0FRCx7730Q(UMdG>j6*q|`Th;Qc^M+}$IHpUA-0(OLE2mJ-PdRs zIq3775dpr?oN9m?5{FJPdj%b)ZuD8R&jbcs_f+DA)rN17kcQZhYV2T@Ylv~X=}ztP zvIyqfa72O&uS+C2s;UXun}YBpdN+kvb2u6{c6X1v!<+|TlP5W{wK2t136mukX%H0D zElDmj$y)7P3eJ$$e`XFu5?ewV>|xPzOZRGfLVR&K?lH@;YObVlUv?Yn!&`?l;gLSS zb~3jy!lyeAm(ZJ5j|3M2CDMa~Agv~&JA)I>Zce!PE=7*3wF(Mm2k7YbNzs*I7`lY+ zdTE}{QStS`0eXVpoT5Lf%KOkvo80jtiG%WWz1O$}bAapx<7`B+`w-zB<^iLrt*qBL|US?U^xWs1#_q%|&x&fgXE)OLf$>|A% zLPQSmIlgg;=77xaJmn9IyaT+|z}dWIix)ZDmf-pDJtAbArKg74EVx-o#6I4!W#u^f z@#i0~AWg~bwr+e|zU4fgGqD1_t%iAdYK7{-m1ai~3S{8w27DTsIOkTZBw)Usgx1?glUk-6=Ajn@H(T%c!jYO*cg>2RhRSV&*Jip^^4F zsS(NwR9^f>g^-n#=CcyPQ`zT^1S4WWD1?hp;@R3L!ePMVdyr*Za)Mx_AU?^e>e}Mb z6})D}cyK~N@po}yd78UKA<)Yu$ZWHnk-4)a^3vT`6~+V&SZ+(JFcKW?UgkGhNqgkW zJDagYGS;MdBSf8Rrl002J~*kj=YVwawMRtq)wax8ayzq)w@ZS8Byo~q&DS{- zHbIzT;f~Yl?^=p&sN~U(2AB|oz!It-rf@rr2}C`FOsbpB zvFM?I5>^YgZhOdjT$W6#?h}<1I`V?9O9~x7wH@|7In=#`W$DKOxr^|)btQ9@5DiuJ zv3QpPA)Oy>GKymeMulY#LPoQ2tb;aw@W#c=%CzV48DcYo^r!n47pO6up)M{UL zi0SkmRl(`~6@YYBY~OI1wG#VT0mzp8bSp5oHN;B`0wCwXL2Del+T@{X(c+@Kcdc(w zI1jOt+KtW)Fxc7}fg^FEjkyic^l))qCV1c39yxn@6Yo>R2qS^yOhlh4I~NyYDtdVD z&6%K_E4EU;*$!&!_Esha+U7K!pQR8k&_o%|TFJm^8PL7dFK8N$aZ0O)cXPZGlbJBP zED`7%25750J#E`0F1j6;H`#6=lN{(z2!iB_<8W`fajf`6!``LgJJoru*itI_ASQ-+ z0-qZ9Ua$qxN7!o~3G~ep+Y+j*Bwqw&w)biZg+V^9BMZ_Ofp@!#)conzmG<}$>`BV$ zBIBaj-(1gK!*=}R5|wtm42E>II1id{$T_BP!Owff!iiy!l_4b^g>X>e6H0gW zo1DUL*Q14IQYy6aumGU2Ap46uQ{hKvwR90S;&LA0V@=l~24}TRv3f|tqV>)M!?o@K zC}3$(YOd}Rx{ivx-oE7vW5o*o9GD(Q>s?ODBxNPz85oRxrx~WGEEV-Q=Yh>*)FaF- zy@0Ok3VwS%3|evmSF9AmDQ#+``_P6nAObcS(|TKOHrD!wm|2zzu1c}VF+z(03&C9t zxho(qJz)k?ntE2Gt!r)zb#iJ4_iD}|v+8`MLMUAz*SsACFM95@c%<8AAmWz7K{F|r zh)C5er^xT;8{7=SkbA>QI-e%Iccra8;Hk~jY8E*T<_HBw=76oH*nKh9} z4zuT-fQOBLYmjre!5~580G=%_k+GS%@0J8Di#vo5(mSm5^2s1;A*?L(z*8L6&N?_VjyW3;S5uF+F-@33f)Biuu+XkBAgW%3I4 zN5xyNP!MDSfus}*09%2v7g9P%juW=qn2npbu|rHW7mFdx9R4zzGs3R2@n;htW|##C zY7XqDB6Wd00#8vh&Q=zbM-q;~IY!@8ItJ`@WDyZ1%{6ZC5# z0LXAvNm;h!``gPEQ4LFw4`!M?LE&)e4usa=65cr=Z(M4CkRPF0sx%wd+`u(KNTh*b zqco6wC%c5*sP1S$E43wIpq=KhrJ~xHCLsVt_XTHSK%qJK;nc~<&Vk}|L^-EZW`%6K z6gM@|P@Sz`(ETZ+WY1O@uvE8Z1SO;tCV_&}a#iy!re)x%EY~E>oB)jzbu$Aqno3$W zPVq~dU9A|_3!??NK(}?*Cg|u^nj%_qOi}a!wG_9C85nY$T0uB3J_UgH37Iga7u~ez zCE^rjn9lYeOqf@Wo3gP(LOvBokg#5crZ3_i#Y6fVR6&K3IB2D-u4O_V+ zLLCmVmeND)mUw6*4%d%Ez3km@(V!@yL7?pnLao$?5-EYZMee*-p|r#=1?F7~;s;I- zSeudg4viRCOj(OLD@fLsa;7eg&k^3Mdc}}DQLwizEu1WL!>I~;!&b)wthF4g4kx~3 z-EtayAzV{fC@EQSeT$6vX2`O)r_o-}?!d$}P4W)XPLH^6>9u985a_%|TWv|)MO^9B z$_dbDq8a(B{R2oD_1wLyTV2X$dhMaX@6sIVAt0m#Cwcfoz&GNwU;j^?=d&IV*&46Yn zqs#I;+JPt*b1OK8xbW89yqhG+$vA;aYHI}(AXRRaD!$dZQOubhIaZHZM2Sg~$TeJf zI#QLC`87}HFqUdm)p+fU&tH^GiUW9;jqxDPoUfrgF<4G2J>u2b;OaL5-10_|y>6Kc z%3)F=5{HFvbT%kWWz~R#gR|8t{VBoS=|ES-vu$E<4TT0)Nzbv!PcmY&&uY+DM^dEP zdBvf2z{R1R?{$diGO9B^cKW*q-D<4|G#DE=Zh%(O$H-(ieuH`6o%t45A2=w-!-rFL zSim&YtyE&9nJ{Sdh8JcTS?Ls?C1m?ibioBJvc&+tR%~`UA<*|%s++R8bn$d5VS}t# zQjtV;&d8kRxk*x_OIciE=CWQ_BBE77*xkvot! zK?<05ag;1GP3@HJX50hE;3TbHaOvbbWrS)yYN*l9S~5eP@wng-zUbVMN+oijkm}uI z?M@DnOmsjFUfD06@rs)17RXoHuGD{nJT_;GUu$M)hOEfX@#LHtJU$-D4uuvK?NKs`Qv5>Zs=4 zAW;g>s@Z|;vqe2^n-o?hdgp7wg*%FfLGtyDw}=QR(arb|1e|S2tUP|9NpT|c3~VqB zCS;G3&~8#%WnqtqCwqo;M(lqgr-ejRSQ9*(2p>*l@G`d0w?4m=$}e#=apT9a(j z^5J4Rk?Ed~W-KuQ9zY`TiwFjy2!W2c*S1yn#gR`KpK{|y8!vKKQ5NB1O-G<>+KT68e1zu@cfRV7%M|SB~;1?5r%OVX1mCjf%Tn@0MoV! zIy}*^SC_8D2TB;ea=MqAJ(J&#$xB%Gm`yJ+emCLt?l^^c@Kt?&c^4%z94)}Cj}N0u zA#viYZxfRzstGArED zW0u6~N#bUl+LRzR2k>XKK?qPvQ9-HWTMk1yuGWec#b8BLEki{NVH~eHt+-Mp(9_-C zho7I;1i`YR67FHS`l8%2_$^L#)hZy#r*Ct%EvlEK0&KY_SvJ@xt}Ag-#C6|6;HGey_#ukK2q}XW2fC4IrYqO2f}kQI&Lszs{C_7$g3FH3M;0-* z8S|EPNXNtOJ26Z>H%(j+;sOo=HV0J9^Q7WZcdH#TSHolDnobB@XX1Di3}@g)$HioZ zmQiRa5Dy4>Dc!`%%O^ZMQ>R_nNzM(G;CJ7U%+(%LV@I;;@Q~VTR6RjM=5(0B>&;i{ zzE?tFm`vm1WJ$HsW8x(QUh+Lm_Lk*>^$!ad7)`_ieA`{IeL2^!HpcXjSvaH4JQiU+ zJipt;l_eyMamL9>neQ?ztNN5gcCm(GJ)^wU#sHySjw&ynS^>laOntnX>q_Td4M&)f z=w)QG4yD6W8@XCK@*OnR*cqhyjD?_vxU_!Hho{*u*dYcfB32~64oB``>~{u!KM%W4 zB&^9aQP!ZQO=cAcg5@)QD~!x&qqiCmA2YU~*qd&3lODbgUaAyVE~CQ^(&GZf zjRN!q*lUawr=f><<2rfy6ptKtq}>jQ7)h)^D07L#;d#7(o4c{@fKrEtkmM&!NftjJ zEoqFch=#^t)Am#okCtbUbb4$n(XRLno#7Tu;je19PJPD?PltZN@_0v^JyoxQpYe_(_ zi8xF#vqYNm`J9TUiI^X(!w%HPIGNg4CIZMo;hWfI>Bxv)yPfH$7bpg~fd+y>ZQ9#M z4pH1J>dV5q&ml>IOVwV9XF5MnaP&QJX^uOWTDTqv8p8OmV#2sRc>p}bwDh$$nE70B zrGPf%D0#CkNLLUPe5{mg1?$8&wt)mxfUyu>!}B!5h9#g6vXeDxMPtPmtdaZjVs#K-cOlE;gW7{!r+psYXeo&&}fyu~n7-97A-geG;JQh=dteX!kwo850_ zWAm(Yce9s3s}PcKA_m(53(G|3)7*4swwgNXeP%$Zm|1TxUqe(QRX}1tu51r#{gnO= z#1>oz4D#vQ4lsoP=Dg7t#&ezK2`-AziRggS8XvP6qk9Bt9gPgfkfJE=4X74va*Qdc zQqwNYk8{w(JDED>^fbyCX&6UnPb&!Pc9Icf22Q;;)>65$fkm2~I#tcry~5*!1A1f9 zxy>)(L6#bc=qA@m!)DWWLif^|oyE)6HJZY~t(DT_59NYf74U{2gxgn@Y8wTuuDcq& zfH~X(=R>H(V4X2}>K=FAd+@5I9uJmteG{WSjZLjg99II7u4b4Dp)Sx|8D5bXKb4w% z29Je?{R?p@oEEqdp=NopEv*xoG2nGqea@@9ivB< zajuw5VaVCLRmm@g#Q;pYVfDuWCPHBVd}nsKi-h{YkmF{_=7LU)NxbF>TEy^;N$PMV zlGEtmoiwB#9jvHc2C43GDgFuYD6)c!A6U4Bkj{oU`F#AR49?sht+cl#D+cvaih^%2 zX?8A9!1nrG`QS`SuOFdh?}pObwzt?IZU;0W*9&8YEsbC$MF@#mVZvmnAbDza4Lp3f za8FPP=A2x_wSat8C0!|L_hO}0oZ?hLkab%4vqNj!QD=~loTQ=&cr0#7sH6i!wM!vF z@Y=>bWq6Iz=rFY6>V{3>1|qm7S(f|3!2zY8Z8)2y_ap4flS5*AGN^Yd-h4iU+ZD-_ z+puyH;kLXD#qhhh=`B}yMX{5J;1a@o#F(ukDwdpm!&gn>mB5Oz!Z)FWYkF?P=yZ^z z-eNMF-8!riCm%4Kb%uuc_1U(_NxM#<>j)G$i2=fS-&@fsZsXf?s7q;Uu`o>=l8PXdsCA#^&47tiZz(=ybFvwuPj@B}$Bs!j!kg;1RGL;<)PX zL(!Q8h?r)I-Hx{hPT}NY`K44aH=n5B*A|i$2Y3~J$Mk6|wbi=G= z*k`-PL|}3^BF{3_4p(CM-LVK$EZl2YSi)3bDqc*0;;rfSGRx!Q_IEJ$^7Jng{7P*Y zSl$t+^gJdacUNeg<5Ge#pSbZ@Ms75zQ&U3w$;(ZFB)e3Qsa^*SQ|Ya#7N}DQa&ol> z>{55)g{<-LNObiLO5N&^VT+5*vcj331S_DUKT*)hwhOVq?1=h`Zi3&49ndq<**@|I zDT+R-uGw#Bzqj1bd?6w-q>Q$Ep#Ah8VB3NlM72gLE3t-cLlOnF;{Y za|*m=sEl-G77KVVlj=NE2(*37>ncF2tFpAea2gL^yK|Lzq$Cm(hf&`L?xOl4>ZLlZ zR5{mh=gu6ZLR&HGVmi@uSP?LfK!Q2)<~31qkE7M)EKaad5XQq}Hh-0~)p-#$ja80l z<`~W_gfSu|j#MGmc0%Svh(o)TL5P)e?KuwNn^p#x85gwUx{K=XE|$QVpn@v|gB&~= zU35~VIX?5oTqsrzp_O|x(*>E z>(+7XHKYt$2SU|Nvom8X(&fFy1Nn@7AmGd_qF=oaX<+rXt5fQ36>O;$Me*)Q?lH6n%!^3lFX3GcS-g(>0L&0tG90tNJ zP&*-hXL9uRhAIj2PNr@;U<(^eUO`$B^VqwbETCX9vKAbslOD&VvI%tz*qt6hemW8m z@E8lfR6giNUSvjWAg))+Yjewr z1z;dnKosEOhD>rfY|jYCXpr%cuD2Rc?8@#+vHs6xpzOIg*HHWd%I}oyN_|SeG$6p* zay_>17xj7TRM|UWlo5^ka28~#BQ~o;JTKKgd260$uxZ5?QvnVrEWZI&{i6gu@VJ)D zsnX7Nh(DCFE$pZcgGhYia2YR&XemIu=6FTynhpp?kb_SY0_NJ)md+ofZeS6!wP4q) zU=|u@yh*o!y@?*Jq_~IS!7ajPTd$zWutZdeTb(2&7z{@y^^W^>N*kRv{255(+oIy0 zjtA{>9Vn~G@tBU0AWk^0-EbcQIb(~p^bEcRVd~aAf!w8dr4AX9&0yTBc3a4IjxjYu z0q$EN%!!(uJYhP}px`t-9}0((!ZZL0wAZ$zF@A_}S5^0Yt4Hx1y>qQmaTrU2%p@7^ zax>EeUuVqnRlcCxtVjcKUV@7xj&?N=*&!gJJB7`|!Eq0z!Uh+$e}1u!1VIwUa5 z2#Sg>g1Bcs_aheC+{Sch)XApKa^oLjU|3}P&|!MUY9SPOb%4Slr+P4@?>ARRXP-Ln zDCzD`wB)w~AJCm#oF~%~I4Qi#=tT*?5MOS-@n6ik7A*rmIP}wvvKkRe1PB>;i&XY! zhly|$b|&i}lDxLhu{jb@1&rcN#I#xi-(n6T<50eo#f+vPWk{XCfv!I(sA*5Gbs?D6 z1jL*?jm{K}B2G75?y9cu8cC>bP`0b1u8grNKW8<7<^`9an`gsfMaq$@=}rxo$dUy; z*G;}56$)Qna6XxbJQO~tFxv=HkT`)DsIWSXyYx;iEm=yjyYkC7IWedqK-ufVIGag> z>>H9Gbf>+vT|O3AHWq=AfLTZnGaB%Cc-uj_;|Tnk7#cht2nlM{bkfkpux~4mVE4lGX=*KiH;#Ed!`&?s4N6`$O21JLtg(zHn#Bd?0KjVcr8F``^{s8S_h zp#;S)fkt2sw({f;CAGg}@aQ%apQCaYFhxwwisJc>y+Hz?@MztA)KV#|1?{+@JcWKV zR%Ran^cH~?Lx}~;X)iRIEpsOw*=Vffc-dq+_QS!?!egV}K#3j}LzYQ}0S392tpcl{ z!E;;Nn50u}74bv{^}BQ#He1JPSr(nByM+7b*=S9&@Hkts=LQbADtte3)~wRL?QV2W zMUpwhyZJO*c&K11;%@?Z-EI3YE$5hI#P(2z+n|+G1rJ?~WV_XCV%tNH(8_ytmniDO z&YFWd-Z%-CS6mY}GSX)b){U_PL7h}=G$2{*a8(~sFzL<{sZy!tcIFZRAS0bNQ&8IbZr&=P^fBz(Bt26hjaoRGa6YoZ*xEiLC1XaZ7eFRU_d9EY zu2KTjl(NSLbr|CYfm`26f~mhB=;&T8)l%?1gR48pL$PL+e08=9V%r6mKjW=Arvh;fIyoNj6` zYJf4Fr!#cn80adI4isma=0Qnl?v&`e(il?bx|FF(9yI1VJ_L%pKZu7fpt;EwyHTuU4n)S(z`UPDwY(Q7 zKOC6o+v}%2o8Gh??v6{Blmp^W0kf!Onwf;4LVBH0VPZ_%8DQ6X@46hTsKz=_`=XBL zvlj0@yFz72<7sWS?j~#4_1(4cpGaG4_MfY$E&MeC!N|&HnKnf<5pac4^J9$ag26Yi ziU&;{RKXTgHgXQUf%r&A?5@J2JDOnkO0soJWzaczK>YV;cNJLTO`c^oa~?=@PT1Ls zdjPk?7MvA{&9LD(fJsYU_NH!}@uWKR_2$&cm_zoOW|DgV(54k3$wDJO8b6y1`YjYh zoRlXhiqFSdePJR=lc8)u!r3vYqaT&E=Y&B|d)T_Z(?0mgblQ&^VsdA$qK^`UuBDr& zkGI#4ZGUihr3n1=b_6$iuNFHDnFdrZlVKDyVM1MpLqUu}2y0(fOuedCi*Wi4`*kZD z@Of&ryeU#?aR^clz$(C#fso>ZdL^JCvMN}|3%olNGu$DeS~b+&vBY#$^#Xk@6Ta|o znDNjYKVw z?C~?c9tP_{h~Az=j@6?M8ENw&hM#P_GH!KA#^f={wMODRUM+Owleh>xfNOxJdKNrw z_gv6Vu@>6ONEtEZW#5wVv?x70TY~rl)fZG3ttU}H2FXHFkH(pCWYi#+4;gp7V2!yS zHd^2nt}fk-a@}Ofe77~I#Ie!01diNEApQ?~CO5Q#ZTh&_2@0nv=h;9)D&mVOLaIB9 zRN|hU+<4>T-@?r<{I40@v(89+(R8EhI)hOUYfpMp!OVo+Q>TFL&_GN~gqCRE7h*6w zv_(i3?bFs)hWSFy1hP?nb^2Fv89PpPQ+7a(3KiUD#|5~bP%yWB-WtZM$Iy&k;5yf0 zN=0IXmX4*d0ZD0ERa~Zu62{mXIdBz{K*xLx8)@1m7~AV1PeoMo6k;B?RV{Ik4c;zY zD`6vfZrBjWe8o1#g)tEo(VA)uaY#@?2t#PzTcP_%8E|{6%teXgQ4l5EQ$rhnF*epj zmLYM2z75{-^MQ+m*j`c^rxmVG(YDTX2X1zQ|Y+fe0I(Ten^)9eMsnCj+kc|OZ>D#Yb3>W2+gk~zbR5j>!5nMV`XMhBwl zFQ}f)Jx;J} zH01&AAZR6AS1K@oAvr+MVm^ZMC5F_+i>J5?B_? z3^SpE>yE9H#CE0+u0UQH_O&iy)iEWEHfJmVK|sF0ERZ-+S* zOhs7qge7n`=RpKV<{E*dGaP8_=|qE@T@3Vk{gio=(Rjh9Kp}7d!)PE^_VPzM!U(Bb zt33)IfxUqTjZvfovSVbEBLECE9MnoI9*)Z!(5anv7X&XRPPMTU$FWjZ-J1gB(0dte zPUlXfA`-h)bM8gpU%IT*nqJTi81p<%!yKF?d1dAaQ*6(0mVG!VtkrYJsJSX2wuR4Z*2@GeqMB@v87hjE-So9fv((^967u)8?1Dd>WI|0v; zM`Ceds-i5#s6cs?%PIjCh?o(X#L2XTnuV`>uCV+FH31|Na9SaL5(?xg)M0>*A1Tpl zdJn2+&2=uF<~i43Zy(b~&+e2Zl5SEoykhY2KalMBEh(A9NYYrD>TP@pVDZax`8jSD z$yC>8)^4?^c<{{NjB!+rjxYlZ$9;KX3)P0cX-<%1c8y7}VPFOc zB2r6n2M`$>3<+D74BDZ##>%W+Nj3;LR&n!s4?TTSP&=JGjs@%$;#In8#z=T;ku{gy zJugdT?+1}2(TwJ1CYQCOL`~7c*_zd}4^%$>a7bOhgH zWw!E8(&V>-B&!kXMqgMY4f$KwhCQpi!YIpT1j>XydUrRX(R|+HRvsIj(!5OO9!%wG zhI$xu4M`FMA;dk845wIV$mfqUTT;mK?)=Y4$J>!^>DyuJxMnr0_o|?)h*kPWRZpuq zRcIC>z@FRFM)b8kR6MwJz)_zUt`Y_&D(=v$3=4HRE`b@<@xn0aS zFJrBauy|5cg35=k=E84-HJTTL$_EWWnm2w95uS31Dp?h?@UorVy9p!|5eA+n!uly= z!#yk5SeD_L&J5>1Q5#|spVLp`HCGO{T%{0ci;a|I*rKp);5+jrZ+yye+0{0QOCVI_ zfUfD!Wkfa|zDD-o0;F?^kxN3hL-0+|L)r_->`D6k&AFP^%|+0lh$QUcjr1@Fyi^3y96v*2v4mR?JJTXjs87X?4;*xGStRyHrtMg=gfG&d< z-O9+w)OBjeHnG*F=;+epUvy^RHba>y82BGkd$F)YZud?JjdxX;}7#G1r z3SLkYKPC`m+W@>aXE^Z4R+nE6DkwfrszJD*&;$n%hXk9kB30lpTs$X@o$7#VqCu%u%Ufg0F*)5R!an zs5V$^K9x`?R1lCG=OyoM=VYop+wJS?1xX7`)#0ENMcmnJ6q_>EqtXg_-FlcJ?hp6DbKtDDZ|bSq?a!bGK_$i2%4K&WjdTHgun)cJNeNYac& zI}DBt4#Rn3R7`XhLK5QIOZS{7vA%7W9^?!$ZFj(-obAQ4*EnP0z~RG>hm@!T7l3eyo8cD5M2-j> zH@ssK^-LLKTjv2Sd^Fsv@y6!8LX`ybdRcwi#ZHQ=;63pMkPOBJWJPY@i5!%1c_cm3 z0$~#fheG2adJiMhCz6c-K%%{eGfb0w*OEncl%ne4+f~40WD2gn2VU%dH*U)Obg?P( zy963(vVzbqZuRAbXFYaKpk{G52g{b1VLbAOh!pW`=}}GM6q}a0yK^{+HwZL*3@%TZ zNu^fDj8zx%T)L~ig&^?;;;7G~RuBX<%sAoD7`jPIUKU}DePgwrTfAXd$Cd&TSW*KZ zHcm;_ECsdA&1u}?w+POH=2&gxo36V)=(Y&LxqISfGqA9W5aVZrqX$HR%ybd*^g`!Fm3kB{DKtoVw#%15X)DH_bSRG+K1s)1%UkC52W-T* zAfLASPA!GT7Kd}oYTowrnOm;Eg1e9%1}%1gY5ub|4pH|iSQccnH5Sn*+&Vi2_s;bB z&2oG|QnMLvyaOIBwzgD<&c6>Yfe^+6ds02pKG8!gIT*}6 zc}Dq{H3fT55bmkUVHAcRPJ;vrVuAMHB4h{_Da*4HJk`pI`QR6m!7+p`knb)rP}8cB ze01_FZ@o|uoGAX$gsu;b%dWjJodQJshu*i;?^}0m`uYb6s{w*T9Oe6{@n}u0*eI)T z;biwUtF30AgSW}01a?ciHjd%v)*1vrtz;;O4e>VzZPkDq3n;7FZP^YVM?J|G8Vwy= z-o-&C-NX%ui5i5K%dNEmP_^{A1Bvk=rfFvT*eM_`89G3Bc-Pfs#Xm@2OLK}L2J zAlyOEF<#{(AZ@+OJ;AYE)|E;`BCIRk^V~>j77+{xKnvF#^P1gAr#3K5ydmyd+K%($ zp&oh;b=jJ+giy6rm{ta`N<5-0;fF3}bh8nC3}TUCDhi|24QFT))bd97vt_d*)tUY1|fw6`R9y0~p!Y%t~~f*TDTo#}A!^ucCa`Na_JwRHAm&Q<6x@UwL| z%8LOe5R~^lTx#5jFqv{TvD9D`2(0CDsQp>>w6AFcVo)=Abg&ZEqYeiwS_M(OadMp* zgdD6PQ==*hAb>U+-P95+msMaY))S^8rr^DW=Sf=vE5+#%IRE1awFA4z2 za{(G2RSUSv@g0j3qP#G!D3PMc=rG~$6{>)oK4%|8KZN+9J&-F)1{+yEFf&+~H-T}G zPP2@YfyndPlMW*18%tzdfmTc4Xst?>%|a`g!U*|o8Tgn+-aIrrA%?ahcbt78tQnYk z3u-lFMc&8f&o%jIQ^I;taT1(}_@OS&qkWS(#m{EjPWRxrO13K;5*z_Xe5~F340{g2 zIJD@viQU+kIBOLRmC%nVdG9!jbA^wy2M!~oi#K*t1Fb$+4QeI}?k3%~WFsss%7-=_ z>hG{^p*KkNNd8Y3A_O3f5>e`1PPS(pg{Cd!FpzdAF5#KmI*%Q<-BBdyj=4n7>U(ua zJ1BblBnCvleIg1kKucmi3dz1$ByDiV6pC1aW4(PWOUT-4M>^J`u_GmC#8-{S*sNYd zY#d@^N6VAk9Lo-vC_~T_Es3h<1=Ry*gTAz7NB7BGoc|-S7Wou$TGR{5WU#3A9I1i| zBs`qCq+I(s&-p0K^CY|8{1YB1ZAw?S<^O)|C|eU$JgIbc}OW?k`+42*JLAgy?6ZbhJ? z*$->UR8_aM+SbSI?gWQ!lmP~g>CPUgGbW+JJJ=1`AEqiAn1&+rxH{tz)7-ZzQ1>K8(VEiTRU)+rU)4DOe|Qq*RJzf z6V_2pMb5x&Vp0}-z2xrQ<)P=0=8h#zNZY%qjXRdLEy<1f@iQ#?921zBqOXy+>YlKj zKUmb%anHD_s5n7LV>uIy^djemi6XsWDRnb9y7H1Hc^OL9f|ETxscQoSJ?-uvoGL-p zbIBsr5-_b1mxW*Do|z!EmMmx~z}T}aW%9PAf+M`_luiQH+$+&{`1Uv6?u(P9D;^&`UE z#Jvjky3mzpiHNxXG2T7U(l5*Og`+2Z^fq559@r`!%meYiRC9}L{<$<@#e1Q3hkvd$ z3cN&jiPB$So4`g<6e8JfP@zfR4alg(46~~Xj26;a-Kg(R=R+Zur-n)%Kk(I}!`7$@O_o&o2RX$gr$Z{bhSj z1Vz8m67|Zv2EAn6&o7KL@wL7L|;1%YMY(R17#`QUGE7*1t!V5 z)1PSA=CKu1(9&6<4De~(cg7a35pP?G1b6Ey6!M|b-xxtJc?Xlg-j#u}l^H6##{1kT zcCCd-Jz&JiG<9qj8Bdbb^)h$ot;VFxfRJ;?+w&QA_96L0Y;jnIz=`54O}hPw4F`i@ zw;YeMGYU0iOIHo-U4=1s2qiuqmO*)kJLGr*_aZZD02ESc^n07p%FO(B(jIO-Z&u7- zuW0lzMwV4>4b2JC>m@=9`xxuBdnv^8E&_X<#APt9K$Y<~mhwG)5cZ%rW7Oz}N&^@+ zzQ2RL%ekN@?(@ftm84kEBN>s)W1jfH__I*Ec|{=MqTzsaT?D38)M9g%=_LAlH%%kb z@0o)iApOEL#{&Mam>UFiPOZ@OF;drH%N|#mV>)UlEgGtN*f-^&5FRNT3bIv6BqN=z z6DU&)-jkJNNNiOP7~=+#Gj?^byzKhm9c@_lt+>0DL;5;j#DHjMRhsL6CEg3DSTT>g=Z6>!(87j#?!>rbQP%`+PeK5`SGeJ zsCb)pf)O`C+V6_O<@Q2a#=uS^c=lr7oKR@@Fqg4EXS9}UNggbEJyHEaB|>`gs)w{VzHQ2BsObZ+3jt{ z^j|Nw2oy?|Lc)Vdv;{aWOX%8|-qX@`vdr&Vc!B}GVNJR(H&v@R>L53WFfm7jlo~T< za=lT$8;cvXk-FLxI6En$Jk&XmN$ORS@RUXZfg z-WV!Empqk?(ugjr>uA2}5eC^LXQpnq#Ume15^G+W#i7vT_Bh)@P8u1($a^69iX_Wu z=zVPR&IypBIn7D3JOPL%EW_7AD*JJqo}GBhF8z=NcPkfzU9Kiw%4Ar`t7O1fz#ZmR z#y-hQP(O&q(7?BK?_>mx%-n%1(wVy+-vU@o^|p4U_8WZ_o023lH;mR za0`0VLK1#Y4IU9Pi>u>OMB|D$yK7FPbpp9{IOul5Z7I3FqwI=DSynMwj(2y#uLVKL zQMK_WHuj*x#CUUV;j(2%oVRC7xt)*fEYs;3%;6dP&>fkWF)LBQI3e#jU0KoUx|97= zQ)AWX&xd5yxj~7@6^nT+yt%HyYDx`}8*8Oi7CH!55Dl=qShJWiHSD{#!CS|?p?N`i zawkf5b9EvOEWwpM2-E34Ss;%RpOYDxC$@_cYJ6Lz;+KBK*%cQfA6sg_D9$$pz zM5+ciQJ7dWCO+PnR!kR!d!u)9K^ugJTq3N<@iHey-o)zDk0KT0wysB}5o+0HR-8o?GGQO=p2qYfuhXr*Q0K(8{me(;|VYeoUt% zmgt&&@3?R@Z+RoZ4=c5?EUcZ6aON0gUO%$Bn$HAr!iN3_tZoLrH@!#O^Vr@jxJE9E zYzYG3uN9D?CHBkH;^0?u8^QxoP4K>D0oA-8UG*lA;W*ldQkwPNHC0ym3{;GOdU8Ww z(5)XU;K`PVzf$;aDY(zy1kfZB#?+sHHoKD#(JtnRTPvz?==#=iDs{b@HD_Qk_icp; zi_|DC5LX1F%n9B_EO9_+#JbTS-UNOoEaM02))!r3*BHn=EgRsMk74v@ z%cL9WMVByD2#TN+kyi-s3ghho+Co|=RJ||G!G6jrqu?kg@G=7P&OA)@qR|GXTw0PC z*^Nv=tqbV{Xuo57z92;nmP$9}s%!#LmpkEjp0}#< ziL4vfHAbg+@T#@m-kbqM<494h3ZJ90l)(St9Gt_9^4EUqqaoZ=T$J zx)ir5%Fyi~V_~K1DsJ*KD-NJrMU3UR)FQVXo@5cF(BYT@tA>i%`+?r6MXmhm6jB7F zUO-O|LuYj2E;jI+i!Q@dqp65+-8Ax91VaS-6sex|JJ~v-K~Qv>I2_dP9UOWIXuxWr z)Cz{StT&3j(UD`#+d~zcnhJ1n4H*y{wlaj3X{FcSN6A&%cI00$M1!yzn?;y(=H(1_ z1EF+`1fVITI|X4ju^)JkVA!&Ig`dc?#G8++>zUlwE0vCXMMD=&(G4n8Unwvr2VZoH z)@haH*Ri5b3_zr)4Q%GQyz!yz4H_!>>1_@#2@2qFB0%<}dE{wJ0IrYCkYrKkFuTIi zRhExwXPBe0L~vRg&CXaVu&#dq9UUSU;{|(E=YL5~4)fO&5(W%rJ%&6`V-m*_W%%n0 zg|#z|T_`T#mvb_yeC@Y{ni_l#8O7!Ka*}JoDzS0#pAV$HDRfrcmGNw}I@kr5=^L-{ zU~$<9PDV*YDYJ3v9eFT3fUl*FmTAwH*KxcUCjPgJOGkF;5U8TZ9}Z+zXBohQO8UuE z77fMBIYJlOnMAt4kJtCGk}skDD!@5W&gTzy;y4KRv3<68-w^p;WHb%G65z2 zKM)=pA3*0_!qteFES-UlwKTb!bL9~a0jiqevaz8(P;#{eKg~8e2CO?X?j@NYYAzCq z1Pv-*O9H1J^FS2(Ogsk&Wi7kG%{bUE9MDh2KKAE-j#5Y4qre>@ECPi z3Id>yqhGhM(9Ae(3nX$Dz<6U}>@=L_p!l8Ri~&M;)bm@1=5Z^#sij)O;cmxGtSo?q ziY2xkN-PT)6u4&P4(aKsre8Use~6FmUQ0StSi zk4+5_dTL&VkZ!MDzN}*G2^=%Ucd?rew-pT9#YgHH$|7%udfmc%7<0~zAYKkR_k($2 zZ61SKWFmCwZ;hc@=JaLPSDx4pt0(G7>@nUoYzo`;A?#M4JHjT481v$7W6Y;chhd$A zqWhgJkz>7b7vu5itnX;ubnwB3q3QXFhq6Vd3hMai%gqDBi+F<@i+)0A&=oO4j>w}o zlLc^U@of?4N`pm1^Jd#eoT8hGoK{kbb=+-SldDFHL4dl1FOs1^of?NLJz>05(~N7) zS_cSjzP+!i+5lzWw1u-ljZ3QzOj4F!0Wz++JaNl=n8mTI5y8fb%kO)xPEt=0j`sm# zJ+%HOF!nn>;|&Y39U(0U34`0EJME?QWZ2TL&5u`0;Tc#1ArAGqm^|E=_0k}rTM2Jh z6K4vFd)Tom4Y%E4b~wpJf>%*Uj#8%NP^5Nz+Da7;NbCc4X(}gV$JdNj<8EXbd29AH z$UiiMVFli8l*lQQSTxG!5)BfjJCpK;lhX>S9@hpwgFYHjs_46g=NIGhpPIxD^B$R-*b_fr`*fv}tT z6E+(Iu-=*XjFE$MVRDkv4XEu;39{{{Q$i4QP{P>)nJiUcod*FvG6lTFVlmzAlSnH& z%ymUZDu?*A?dSu*RVkY*G-3$6kcGk*Uc0>I7t0xFe)Zbf5LAGmdL>E_xwni%J1Z0B z1q5hB_FPKIaZGlWGB*A_jFl_wjn^~t@XR#UX%&!< zLL0y(29#TqV6xC-Q+c@0)+~#*ns|BT-znjRhIV=`3k`wdMS$(BRl}6(x{Mj&0!n!= ziee1ry<2@ub?cLucDYt=c3}$!z)A&4zEM!;71Ow-e}qg37Er5lO}Pbga@8v^XXuW2 zP1)@6la+MqxMl^EUPej*y(HY~4yI^dRIxKNyDdZQJQUv`cZ*0IP4>m=!N<)+ed;dO z`PZrh8$>GaFc45X)60ZcxX4u%6Yn^Y+6DsIS71g#z{n9@BArbH(=^j@9Qbbea&+W@ z`;(oZ-8Lr6r9lMul}Vb$*MaLzJGpY1uAxM*J ztqwPoSjy@RqfIPQ-dMrniXUMGbt`h9rd!&d)C=8=x}#`xU`aI_R9=TvlRoB) zG@xP1@+uRH{#S8-}3R4rH*zMm!4k~}6`Q97TL zhpugndpR29jif?wGY@AeCF_z^RFb20pxP>JPA3I{i+c;u+(OoSa>d{X%ry!Z6kP5re z-CJhU=oLxZyUX+!D3J1WjVMftksG11ki3(pX>D^hj9jKbbX9rL2+Na`1I@T6MX&;k zI;$%;bZ;6r!NjvrT;)U=e{JQ{)P~AaL}N-hLn?iWoJB73kU4eg&|C;;nt|nXxNY1@ zaPX$6Q)60!h(3~Z+to%391qx$eX@(!$&8jS9%atga~LD9Dvc;RPmW7CqvN{90=J)X zzIVH&@DpVL?#R5?viLGsjRH5O+SRK>gT6GAg=ChE9PH##Vd*1FBDxTOfk@Kl82p9pyqZ-`bX)QL$d_L1hRef^z-SFZu z$H|{A+4nMCfqO%#F{5Kbqv68Xb_X{@v`#UtRXILYE^z82_mdn%Uq^1Id6I-Reb5sL z!rB_%$?~f^lv4nPC8xzlG#HyRQijqV#&*&rbblbgT8mI}n)uf5A{SQ;r|34LRl&6& z&U^Mo$y>}SO6euMGHE*fi|>~yR~#V zN7bQ%ER%N5R1>K^GTvt&Dk-TM_139pk+wv7GGsBdt#>HZ9QH8!FsXoUYksrd9`52~UU~ zHa#OKsnAXb!4x1pfKZ;3pHv4=4Njmh1at{gPWibz5(RzTGT8<~T4I5(z3 z8w{MOq;l%&MvW)GFZ>p)$EAI=VdMC7 z1{in9xm-6X&ec-*1U9B%E>tx1aL0>O!9L^J!b389!?)?^mT=?L(*nz=fvY^qG3 z51Lvu2q1uODeX#tw@lSnNm-`&+NOmzvk<~l+?X0 zx%mN~1_@jW9q5!u#uGM-AJ4SKWIg%fq8nsc(lZ3JbnqN^nP1 zWxOEjqa>3rCE<=;o{!b?;S{|VeEy^ybp&u%6EQa`G*MHqvFEN1c>-t&@zSr{Rq6nu zE)n0x{+Kp!u$~K!rDJku4!Y@5<7FGZy^9YF6e3HT_@D>w2%T`C)25fpx=>Qn71WOs zcu6N4;o{O zkh7iF0_QX%51!4u`)QcOvh-6iIL0QqBw z*9_?`C3$4vtB%QLbPI#aOzrF*53aM33BqY@mnzpWQ`QHas%d7;ePNbP>QWHpBRUI3 zz_Al4%tVqH%Ec9Kafhihfn*@zzHJ|$uSmlC84ji`ZP(r8l3xtvO>j)Oz;fjzrdgpt zkzPBr0x*|CfTG-@)2CRiDGP+R$k=iGo(rDV2LewA^G$kw?JR4h;O z>O=z@e2bul9m`eia3P~46x3&2aK31bABn{MyUs;p5ie?m(`PuoQ!v3pMfqGmGn64( zoT?Q|YNcgNTk&I6Mw$zuydlooa^3y5ki_}9z#$k@R^3{aS^&Y0^@w5*0(E-%PGQ30 zQAozez6X~f=j(!K?JW)2`fa>3iH&QyOX4iL81ilenU%Q5YKxdJI*G#)V9iJ%zK()v z-iTHpg5w3Zx~Tad7H`3@;0*UGB1e779MF9onA&{puBmXi6js|T=bnko4zOF7&ssUD zFw*!O&B0yonsLD7qup+3?Nb<)9UQqx@}Tt^Dsrov8SYDXG_>Ft@-6|ANV&;eDtoC3 zkM5EHeI&`U5Kon?gWFRS>UJEp>c*e22ja=?5aAghsVP~-u|iTP`ze%AL4*M%%qQEb zIhY=h!*Hq-?h6~5=fdK$77J=Pp?W$6ha>?oSlt}!A3!kh8ib|X4^|+aWiZit#x^i+ zob=xmd?@yK=6s+}CG;3bPe_Xaa<{0^T?IYdubh)gAWltPTH;HWzbFU~Rx4rDUU{UY zpD$AVyHKT(@<%c762=+}u}g$4T`YB<3}m$NR9;-s5pTJ~LlQXT-&7Xs_NNHo9>n_gVIuJf75DbV&(<+qRzhH`_-VOvU; z%GT$zmFqWPQ_J=^4wl;lVf4P@u3(u5i|roVrcZeIDS?2jX8}bmSyRO*yo$XSMzOcq zjO7&OKtl6R<{+7js6ys9K^%AOj7wvSyu9c+ymYNUL(DZ4Z;~$%G6Bk*CDj&GDi1u@ zBO8!l-Ec6OX5riCttn@TDWog%&<-9D;K$Ax*j#~Ws%em?#0cUQB+~_1q8g!vo7xB= z6PWv;o{*PE)-bh93EcY{q>l89#-uKkU?WtWG2)Lloa0^Syn-#I&hcc!De2%VME3zv zCxxEQ*pNYtI~?LE;uCP7jG*Y3(dQUA)Im#o?t|7k99fbL8Zxgqf+PF)F$XMm8oZ+EmUcyL)X219a%S`Y{b_4 zRz)FBZ=9U39xkT!n551kV^&~ha?d;cNuJeen*oah1Fx(mI}x7wJ#LnxOadBEK@`Iou!^Meddb~O%a1q0bWJsK+GX{PH%nJ$Yf zkm&35oSzziv$v)ajW014+@z>*n79>&g_`GM zV|okCPCE2TV~@4MsYNBiAE@2)t?IgKtC?90_<- zTELt`oGY_Zw|qT>qW$3w*KQ0t=WM31NK9WQKJcq-fm)8&0w{S&a16D@m~khxFs9R~ zQC^}v@1xcpdKH%3Z?}u>OU=VTiIW7*@iumJf%9NO>I{#RUkE2WWKxjXgtzgLNzCV_ zNS)~~y~^pN50actku1*vz(RWDATtEmTpr54Hi{gUEy(9bB)L0LC_?cJQ+qVhYJfv@ z8vt-u1tgUzd0Hc<8DtS0RoZyC3p0OY5k6ls*R5UjP+U8HUjA12k#8tj zBAT}E&?0j^-icE9lAX$jaG$Gei^8^44nIUJvgjHe_YM@#pQJhOfV$VO;Ue<}tVfXq zhlNaFPzgct&^xb7Ye(jD9X;{{QSD8arz+k%rTl^>y@GP*a}n++@qm=&O$!F&AmY#& zRTzhmYI)y>P=ghs(i0}E_Plq*WS*eqJ%Z=xf}%SQWigv{U8~|p%_XR3)o>QRNt=*&r^_H|*jChc;vQ!OKJ?2XFmNPRk>t3>V+41n zTs^`Ai-C6&1j-oM`McKx0n?b~>houK5LdBfQcP zA2nnwlN5KY5TQv{r7OKjAliicx`!Cfl$!Jw!R?SN0ktF)w6mO}V40unAc6^khy z7QbfOeOb%TV!BX{m_?im^vO_{)TQ^Gn#pjmf|=254s|+#TX*F6aFD=8csM~u!_c(9 zZ3Hrf%4D`Ssl)WD1Yecos8gJ1;6GO3w;VnQ*~ebKx;NJYvfI+cDZ|zBBpCwA&>FBV z211BW&0&14&9qC~P=&sPMr`ny+I`h5RN`WtE9HPS$8}jDDln2Ue2iYQ_X&#f@N8VL ziEj^2aGjB(k)MS_62>5-y9hleD!m+V9`sLWTMLPdS-v*ezN7VDHZ&=4n8W$!*(fdEXCcQdfguj z-YaS(p+6=p&@WHZ$#M}y1}s`AfO5&j+1*D6^eNCpO|vh1(mWf88c*a^BQbl`updPi zB|%Y;V1kd{NduKG!ooCeLHRYp--ppSjW;HsqtPM>Vr~`LVBE}adA6NVVkjO*BXWG^ zf!1Kr>CH>5DKH=ANQd^Uim7~WrJJmz=e9GWIG`p|nQ(w5~kWIPBzqQWjl$ znPUlW2JrDX#?~^RWWnTV4kXvk3iL;ktJnrZMh?@e2JgLMNbogWlJfVX1Egd!!Dvhl(EDQ`q^Me7pVM8upOz_|(8(!vn zPjs7e(95)nUT_r6H`|(w(&JiSeeDXtp}2TxFnw@6J;-bM^;Jjmx_3KE*S*Qc(^$S z-!PMfT1tBPJQd1b5hXda@#~IZ9f>H=Ubm~wgCYlHpw*?r!z*Ktd9g;y7cxY1KaPh#7P+sR|=G(ZQwRkZ2`sd zh@H;T>e*u?JH6=(NNl|ebxKq)Tg%~jeGo(kbmc}FL-CFKp>czEECr0ralrP$#dfeF z+lVFRE8S8GJHoLnMj|63H)n4?Fsp&FSkkU6QA)5(AQ66|QtW(oDmfy@iyPUR??H|^NzB*%jC817tHp$ez8SYUD0?p`g< zIR>>gt~0!#J39EJCm865tOn~7nGdAAE1;-qp2(MQg=}B{!u(k<|GN}%+?si8?gevhw2gbS4QE}H@6)a%lD2hD?h$xSKq}#->E5UjQ zSqBW75}+Mtj(1>^`FW)0-F^*V6x*nv{NQmzXe9N*GE%in79G8)p!qL}?=|$SarrzG zB=4LV3?2fc5iv-NQM&LM115r5^y3vq&4tHmq%F^kojmeDTd6?us~X2w)UN|F6>xKd z+H4cZJ}C^cQ_K_;pqMWPq8}sP4`OPFGE3oS((Y%p6-${G%+Ax?lU-JTritPt)~&9D z9w2t2ak?d0WK;_S@Ue$FWzG+^yltO;Jl*-p_*zMXQ!mv+3KI2J?RAnBscbRcrruFf z6+Ol_fv>5n0;wAk$8rknJ?w1P&|TPG#v_CGY(&^__Hx?}=#%&FLpBF#bX{1&R zh1gM|&C-zP!YPxnV^_VQISQFWLvef;a~|H#s@57`CAK?*$ms-`!Ub0A9-VpFBx78P z__FgJe1}iK!Gx=;NqaPKAi4~r+>|xOS+(NT@{Y2&&?`)?-!h_r`=hzB%Qw4d@)IW> zgfI|dJQ~<<0{?q}-MmoqYqMeLp7CvhQz9bc5`l*@Sc2%Fanb11i|VEcBV+K|i0F|| z!BV+^k&-v+4@ZS+aHf+1Zx~(fUN3ZFj!;BcUdl`v!y<7!z{$%BNaNzGhbOGKHXA+l z;$u+}uz|@rh})W`%iB*2UKR;tz~c~RL=8HSun@(IHiMlwi7(*SNfZ z3)a?boQ{@YYZkvbJ^cVVjvf{4&CbgfADKI;<;BF!Vnl}ae9VTYg{pY&e!2F0^FyBOFqNdnO=6P^!3kf0VaNA$^FcGXUG;;V~VOCXNRjz+S8ao$`;=(@yFNWDTZb16fKwjaOo@$2|5g zPU1K+u3U=haIEp=-LuolQq zBzgTA3*OEPV+n3Z-%&oIw68t^!DXUS;4$M8i8~P?#=GsQ@%RRcW>hl%TwW_2l-|_z zX0;4iy4h}^{4Yvd1k0TK32X-RS69L?6b%n^cf~uM4d;D4&pt7nbHeB-{kCo zn80*ro=i}TDJ5s|_Cnw(8n71<`qe;DChkgTkcZ>1*@hrCJ0|PfP;T#8LT6 zOGF&Tuu6D6UQ;8dR+%WlvYybLj66j6!y zh;^r>dF|oy=I5a-_Cm76=P!xp#sl_5C|WgKJHT*b2zW&s;R}4L>-Yw(<9n198S~ab zK7HBT&lz`hm6oj3R!!y|xN*aVb>s}-FmiX9Meoav*vR!=ZGw`0=BH7(8DN0si@hK< zXW9>1y&xld1KFf0nGTZ0y)*|^m2QSLQuewLIOM>X)RhB`78%>qmgIaNdjQv% zyPSZ+o^2R4Z*gt&Fk zE5$DUYQk3`%C;dzm0s1&Ar9Gj!8hfKi_0^QeTteo~KI?U23gQQo z9oJT|4JBnoOkOPa$Ep|v1SxyyEiCTRZQI zg-Wy>*+(;aeM+fZ+N0qNa5x3#4zZengiGP_N+h_MS@K5TE3yxPm2Be5J8UD!CvRII zSw(f}YPUN{97!v_Hc+2QDP=Ei{u47o2NScM!Cy-?QgCxQ*ys+TSp72;5;-H6RD28=g5JAwdV>}z+`K7>mK!QaA{8%F4 z;cq+6MA->oX_t_?VbB!@;`9~_HP-E^b72bt*;IYea@pg-a>!il+rDgCZL7a84f?x+ ztItQCR6$70uohv+LL~zAcL}c8T!6~hF*PP@Ny%C;N(EHlsn+g5yV}5IIqs}`4PB1D zU8pMPJ8BeF?0s8#%}0^gBZiz;s}hoDiDC~dkXR40qTR*B$0FSdMXmi`jS%3fn&Y7j)W zWdl1&hKr^ZLm`G_iff3n>C-R1q``kjuy0kc!yg_jc4I)^lH~AaB~e-Kpxe_od2VJA zU^%)9n!+pK#3$a)1&q7EpcXqcVSi@0q4F#CE2cg%s5_i`iYPgr_S zaxM`?0oYC$u>G^wfML_Janr>joO4nOkE407er6>w;uu{X?-RR3=Zn{cr2ll(yQv&U zOGg8eP$JknH>|Y`E(nacFSu!N_BpAYy_EbDiYsbJZv;!H0P_eJue`6PDz|K5W&m11 zrN55pWDji-y?bNF++$W~SF;J8qIckWz-7J+q90F9Fd-BHOW9DVVZ8KqVfzB*vX_~@ zNKj3PuSo!&G?Hl{ut6+h`Je9QDrwD-P3zaU$|m*K#-Si!_-w}7z(-M};6^G8UO$?d zcI4Z1fxVvY9dnYd0y#wY?s`ac9p>tjX^vGZOG5c&(iICm0u#-%-S*>ov&P+sBEhA} zKJizdPBEuiMsMAsq)8zjiQsCz!H5?aJDm#9YUM9D=~d;cTXigExt#~%-pp{PR?b;`ow0bVm9>_HMZ=B@4Iy2E|EQxXnDARsd9cz%j4 zk{iFUfWw-)03r*hvxJio4(O*mq`GoW1dlfYK+2a*>*AGaTyYli zqB;fScnFV@sKOP_R5yh^MGUUCj1BdKVyvo+xIZ1)8|}V9>fzc9yyBL)cXWCQGO9y! zp=oM;6}7@S)*>o%;;^;1ur_)v9*E6pD)8s2lm;@xSAcMP7wx0tAcj7-(<>G=KV9^v zYUwdQQ}KBlB+RUPk=drWc27i8jI78gPh7pyN|wEVR^8VtBR4V?;X3f@L6yB^y89aK z)bLYR(kx8uOs+15(n)J|!2W(Yz!4LCDr4i2n!{V%G8&l%QldL$$J>Oa2jqn2_dhT- z0GT&*1$mPSf)K)Oo~paray;dgNPHW>Dp+f_qANhaZ7;<~=E132=)hi(&3VR9tM-30|t(EW*30yC4v=cKH+pxD_T{wIO=X0bjnbX+$}P2ZaQUZpz9#vPN#?l{?;_pt{W4oIX&6d zF+w_*e1CMJZRx5*2Md?Pm_c5pFtO}da}8DOj2+7BxbwB;=+LXqs~Avwo=H%-q`kXf ze?~3|r*&MouTI*UIJuXnv{l-%C1j*Wm0wpu0C$$n(>mHEL(m%llaA>$aKS*6lURkE zB}@!zGMrp@+@#ueHBaf;th%~B!$q{S1Q%%CiEJh!Akg?mOK}|6s@WmPOoo^a)0%xzen*oiQg~Mz3bE5`Fr^kyK`bl4%1N3e zW#hnsMXr=OS=X|{AeqgyYtq2^C6@CqhpI@-Wl=A_-h11ilxq zP4usCl4BeUrX=KR3PM}2R|8U)!md?) zP79k?*w&^p%{}!d(XqvxNxpUI4_XP{WuuilWY-QcT>zGb?B{Oj)VHjg%ZSHL&ZO&; zD8v=S6`{GxNai*I+LxWzCKCYs(<8xofFO69Nfp81UoUr}>y8Ahc~7bEVc4{7X)0|@iT$=bUCSqmTXmR@0VvfIX)5+D&^2K&3N-! z6UB8BP`Kw8B^r56LrTtZwkN2=bhWE(w^P8h=R@gCiz8(4H#BrPUWoiXVj75iL>b3) z2C8Qn*^O5em!~9oAgo!-HqcE?9dRKXN7*4>(JY!nC%N78G5uZboGd`2TaH_gaX2MV zo~lTntjHP0v^_S&MwIa;yMbz|aocv*m>zrzPFD28b!9`GtDc6$$v}#1(+RavIFU#j zp=?OR#ya@D*2D-p^fk3aiMoAUrel7C9Z3QTc)I*B#wVue>piv)eHXrD6vzj3tSs4W zLxFXM(WyTe><5K5Lusw2W^p9aJ(!JW&&(W-Go#Lg?pt^WrCH)5%VnyQ7Fu&Y=t}C&Qj2}b= zECn?in%+Bj6IxIhLDND#Al}0pC0HgINVl!^50uU4QQ3I9t4q*+Y9*sc24+Ifib;n) z!<;A-4*ZAgq&ueW(`os)sy&FixzSiE0G=6PjA=3>vvD?Umci=4o3VbrjmzqxOjq^L z(hePq@Uri8<8+p~$)y?M6i*u!&yFSVJ!>7<4^XUXKDV=BC9%A!5LPTQkmNfS1;_~Y zdE9(PrN|UxuqItVn-9zl?^K_YqMH$*Ma}mN6PFtP>VQr~$aq!@aT%=BwI`@X_0tV> z6NKBR9sEZ<0TJ1cPRmKg0*blvPPMc~^0LnFU=J}Np$nZH3|(W^TnaC1E2{LRpmSLj z6MF1Yp^zZPp6s)YR%kpbqMJtG#Yn&d%Kh_#S~^;b zBp+8mL*+yw)rG3bGxpg7Db)ql3FBGiaX9MKRIfRz$#CW|5*qsM0;3SiaSs^Iv0uxK zW4Y(^UA+emlTF`3(Y|ElA$s3nIXPZJ&%rDm+k==ki&j#>wl zLW6GNhvmgpkQLF)zaMP5%d9M9+fC%3V`Bq|H7qi`dnzTCK|Rlk3T_1}1Fg6nHpj=P z>{D%EfXFq2wx$E*AiQe{vO{2T2a5?B?lErJMl8wmy`0eaydiKCltD?5kjg&HWg4TG zjP!s7bATQK1bFk>oLmcn7Fl=c-ZHeF({P7uo-X(RBI|c*qhd_Uc~Nxr-nSl{xI)8< z@YCBEl9g=Qj=n194#Z5zCzTq`e3)$2wEa4m9eEtoax5h&h+`5m?M%n*6@r+aqV!du zxlUlPw@&E4x48lR;9iX()o%lRsPRBB8@1zp;yA%U60q^H#&KU|IFae0tAbFFZBOar(WnIcER4bHg zW=2rME>6G`ghV?vXAH&m3G5}#b3yh3eZp1ZFv%ggaUF%sPbh@XiP1K~(&3SAI-pmA za`@&!^9gQH@+Mcbsvb2!=1Zcvwd z?~IR*j7&P*y?sT}B0deOiqW=s(_eF##frsqllT<@+zAachoW9!ZFZGc$55r3t2T$Y zo^Tu~>aCXO@`BN0E6PM0bJQ7#R{Z{UZ7$`Oo4jo6TlM6X=5QL*82*QRK+fe_Re_eS%x&YB>iOT_N(%@a*wGX z;H;(3MIr`}55qva9lY(-8Hygv1QG<=A#Wd9T$2d_b#x?FvNxruLt+<6Bs!uYf@#Z? zBy#yHuU>$zuM?1=qz^7LWC;q*aBD@=1fGWSuZ8yw&_9#imm7lVM`6k-Sr zh1gVcO9F#dmliVDtn3tr$ZkT3?SKzou)5ud-XS{e&{1BXM}R#MsyQujUIocx+Jl!- zQX68AYE+t`zr7+MWQGlD!={Cd)D2LCf}=o8p}6q_e)aL~!nh=x5=u3`9b?uw=vbh1 zFSxLIAqR-uZ#-6SD;Rs;85ScL(FQI)2fgDO&Z>Ii0YqhKFc@)eMiy4!BtU*CKp+oM zb)Hju&dRd;uIS4|SZOOtW1UIp!+8We}TgW2^fdpUlR&%J!)CL+ErI zT1=eG;XR%lsoabT+LgexF3l)q#jy^92ZoP3q!Xe$69QR`5{)vcxJSyTnBIyYi=A=6 z6}B3snB?jgurPO^e3d*TXnl?~m?as-E`Zzjco(b+w$QA27?3uw?vQ&%`8UHcgRjd@Wp3fGx&G-ueO)*K6?wW9iS)!vn13w->h*~mkhhe^mrC6c;={Wf+m1avKqqugB#6Qb zWi%mr%yRiovaD`tigQ+!qOf*g<= zB>}E%Kwwy{R-`2|CEm+vkTie+!I2KcE-ETQoIrlI(%*^lW*r1)s%V5EyeDl3VNFK( zjJHTV*`dyNNc~zDz%$a?EKeWkCvJmvk$@#5ciZAC9*Adw;sFM`K-2Nx3~+g=w+(d zMuC=JrX_rGt<$-$nak4hCS>K!yBDL_+--`J0PVtTXohfgU6*cnl#1wa`jr~k93mV- zZcYb%siq=8P{dukdPswze%ZB$b-e@p~~952?iPPvgF~GVhM&JWW==g z#*-i2%o3zKt6JGj48fO&MP2QaX_|ZR)dgY5@6O`$?PYR%VpVaY*LKf#~qHw_FuSd^8>d@T0 zs+?qU-iV7_)Uj)1m%eXsl`>5{ayjlThm^uT<-G?s5_NXv>f-_#DtFM%WuO%g4tQY4>}B+3dpT`M7H7MVd=O!4 zxCP4FqOC(f;0B@sad`C(8eMlm#JoH?qIm5ia!J}KP$K<>>!s7_?WmGdLISeN8gZ(5 z?Lw!pie;nYKP~{2$3^)CsxLuxwDHOzfv*o(jaXoCeT5viNhZW@$Cu6&dEU108PcW4 zI|W900b(429^o#Pa?wZ&!U#Fg2M<29>U>I4nm9;-gK!ovY*NHxE&3QfbZvZk>#gqS zVc4MR&YiG?0-@*7r$*Lc+Tn$p9`56#$yIM2->Ya!C}zlakx%3XUP@_(SJu;dz-xBC zt4ZGZaYlUN!U?2k3D*3NCSv)Qj*}M zE$gME5ZP&_zBgcsVq6-c2N&t%sWpM59vA48-Mq7IiYhHbKL)8zePiRKRNya1)s)TK zgR{qaFE$_>>#9a|X^b0C73y@a3FT0b6E6*NFnH+};Lb#pm$B-)=_g1C;Gwzc4$6Np z)GelCRa#?>W4Tit*OhB4`OqyKWSG~ySMK?SV&w5%0V-_$;S&0TvigksU zs_a`hTVRbqi^?8KxzyAUu~VL^PM#Zl`pj3WV+1u(ZR=f0afEOZFhP1d3=9qA?KZ@F zvKkI+?G>m~&vCd5L-eU_#kV*>(?<_m3qn+( zaA;vJ-kweaj6*q99=uEkhDgmebEZ(8eW})j$<%uVI$1=Ge6OHF95xpSdjq9}R)mea zk_)qfRZ;`RszpU%7tv`vsWacQj-+@Aq>P6QmqO>?-%VDb*RfQ(>n zeUW4_l-E&nH%UC_a5=S%yTmN*XcHLD6cbLyp;+!TeB(udRfFP3%(J!}6t)C3@-QL# zLqxr1%o3X>!Lr!C8mAR3!53Z{Pbg5K)kw9+MMC^yxWvo!n#=j1$V?D z9e`4x3|_`jfqRwwKG$f>v7)pRQ3DJy#0HsI5HJ+W=zfN1Zz6OP4mg+?C5}3ZH4JYh zfe+rF5StVPXr9#Wz8b$2OxqlAJK=UKOe;)lFfFJdq08J|7WcVZvAsc#e!%YH6cz0g zN&#;MAp}NVF2v>6qL#`tBRADnTlWH9Nra6G{>LuBXWrZihU0pRd+ORVzzU5PZonJ}Uf zRV|IdP*6R%ElI~>^o;n=^M@hAa;SE?n5ZKUOWA_T%rrUz^;fgr;*N`Yx4J5tS&P0x z323f9#CPWiR$^r^eibSd7-p1NoS1S{`Lv?dZ6pN==z zroVjvX5cBYKp+-S24k>EK{dio5Tm?_Qy4zU^FIldHB^w`9KC?KlO>QwZ8kn177-&E z28JJbG*RU#&h9B4v9>%7HfRnYUJEWr7S@2V=uEFzw5DjQ#~7i+Pz%T>+#a1$yVn>q z5rP|!F)tw2%5w`RIxUb}w})+|v?(Ri+VH2SW@E2D;(N=`ED=bNZG#3%9JLMO3?x5P z#S~cXlCBMfB?6YfyHq$okzRm17*P~v(^}Ri3=zmusEk%WgP;JYSB+YI=yQvW=TK6H^lw1 zoRnTwRRNzg@n^6(28HXppbfbn?QlO4aC{|I5sRwQUR3ZQFIgut zW~&XksdPv@YJ<8>osc3iP(PC%Bx)e0{%J0XosHE6JUAM_z2{u-)~xYH4%{AW^q7sj z$YD<6;d;F=x^8sWm!Z#7t;Ev2a_Ua?C&6dbwxLnu>U|hKptWuyWnH8doUMkPmm=_( zY62P8n}@;{Z;xFM$f5z`yu&m-C6>uInHnH_80$P-wGiP!e;%lBo%I%IMW!P=skn8n zO&iw4R|WTjg6bB5g^}1n23xf|ZH+iZS&YjZ2so9})K3~b?U-L0M3`(%FxjvG{31UH z_m0U;1*X1DS&8Mc4I}r21cfv87oK!Y&_pK9*_!%7uGJ! z#_Hg1)oTnXqRT=uQY=NV)nrOa**WP|!ih8lxz~kB;A5cy6%sDhWC4o~Ndn!pwLoFD z!U`6J5ECF#0|D4CyxuraE>~B+ssR$+MZLin#$7SUA-lG(V(AYG-Vk3Azh>Hnrk3td zqX|hcpV0%UE00IZj8)z|HK}4d`@)E`An9n!&-A>aT}!FFlq|wFCq8SyRa=VrkD;L9~bz95yw6!g40gRY zQYx7mQPqpP=4pK>)B7cOOU8=rdIt$?xQF6(I8jiHO)pkVRylEe=JnyKKHm_)Yo8Vc z9@AL)1SZ!Wo>&Jvh5&VU4WsOZcnqd=>^bnn#Bvu!M&$ebqvfCaR zRVQ*h@EAbWiA-pFUd&0+;;Oe_DIf|d1;+^j^4{d!jNs8>26k?d6ubGue2wskaSl=B zIt>U|Lxc;i9gLTWP%QC-s9&re`_T6UZ2)^vIN_2bzNQ zFRI{(N!wLc0dQv^pa;T1R8KlmKKsis0XQ_?6AkNbNA5mkp~*{# zb^&`q-72-oT^MW` z!}hnMWp7OZJxQw8un+?JW;~<}Vx#Hp_=E?Fv&=fr2wu5+N8mkM8_%dojMEBda6u7> z@MvOfu4_Zkqc!gGLTMAw49jtf=5XuLm956uLk8)6gecuU-b2ErW<)jLZT7|IVlCcG zjR3>|xVDI>VNeb#$B$H&FAb&6NZmpK+uNG33pc7dCJamEp}(?Hek6B36#@EZCP+xb zX4dscs4J2|#eF=Zw{P7xL!PE|=NreG7|`DG!K|YTDdT7k5x7B=xz1u3*5|JEw+vRL z0x6uws>;TMjPml~tRXK9XkVo&6hO=u57ZrxKy?gNMjO>sty|1DIHneu*Lq(ohGSICy9UqaLl(nIR z0rS9b2w~Z1MLQ)5k5}HCKSukKXq%Z5)E%^MK_(Rx7F3+Xa_<>YS*;kiA+e6YtN;hl z4talB>|dx#3W>!GDrnGg(bg_-J(1Fv4$oj%gW*=DnbJMJWjYE;15HTA&W-RGAj`Y* zR(S7Ug&U;qEIqQ$E~v|f)5QRU<5ldK&Y-sU7tosDGs#pgnK7pvgdvnGm zAi(83t?Ta7dMU?HMi=o9r^o_=?3;lblp#6o84c$QTkjcATDHl;m87u*rE}VmA*C4$ z;M@>GIhO^mc(JTGdRHGB`%JIp^Ta7E4h;p4=GCkC8)}WuI~0aYa-~;j%orRyN=8ga zIm<84>~N>KlfM`&pu9I_3AD33;7kn4gPV~D1^H-6XXz|aF$`w`Yonpgu3Dgu2j)Mt zaML=dGsBG(k(O%RIWL(RU~{Bc6hlL^9Ty3BaVQJ#Qx=T3Ook9l%AB1QDJA08M_ak# zftdw#=a(iG-NQ{2-HPCKAaz1fhHLQUWQ=-shJg)=uQX^dUD6;E9mLYdVA9G?IN;E zN^lJwaAbv;$JJoZ>sHabG#qBmNMFGkT8rCW16>}c_0l=aeew@9 z!5}jepDcNCya^aE2usYFa?^o2AmRo^HB2p1LwB36>Mf;#>Bx| zZ40DN8vW$FtWLvugYVPF;2>$kv$!t`!FW_YxVUcQTF0S0s`{k+HklYVK4QGZ5nwB& zFGyyx&q7k25{eOe5-X)g%$G$-t0inex6-MX-Ah#A3z*JalPU^Q1zER3TPDI;RM}ag z+i`3m)Wdm^Ie|_IR?xhs>8A@`L|!NjHB3_ZeC10eaSBV3oWYO`Hv`AXJuhR;2!2iS zHb`I@4A?BXUv!4^xtX1msUO$`K10Qaj?fg5uVZ)leqp%UuVBbCpFbU`jPC(ZPoGV7iXP)PYE z=YWc<#%{Dlta|>;%$M%JS}+=c@Q;N9mS*`W(el21D{)XS4VWWPRNESzUeBIamC`f- zMuK;!E3&~b`%)rjyGXWre^rjXJ>Oq$a(Tc)*U#uG4BU{W%EYYp8?**?%2%YSb8R!3 zQ6`R1*z$C{mB5CHw<^7G)tlLQN)DABc<@CAIN+o8dKsX6cB0s!VmrxA%Ww-^KOP?Y zc0Bjm1t$>fjQ)L5?SbXQ^`0H=_E1J+OzjE>JSUF~=)72T+|k5AU5DCPURh!GBWvv9 z_n1vjE>E|uvWsEPN;>X&@UR7?WsQ0i}78>%54MgUb5j=ui z%^3|2mzy-GX7(`2d=%lvL*sHKO)T`)w$D0CV-s@AwPj0XyjZhlJ!4o$ZpB;qjzwi1 z#G{9>`sbWj6%lwmN#xlc8;TL0D`|L#$MmHj6WU^~;OtvAMZvmzf_}*tIrVcQQ%=o> zmF78Uq3u(s-b(2RNdu~4>#`{+Z(hiR2TPjBj{^o-V|TqhVj**X4a^X!H#_x6meLqJ z?Y2utqsY2hQX+Be2kLn`o$a-W-Me5Xt2fC(yMK}X&UoYiMdC9Htj?Z60pj)?bK46q zw>(hi>&H=Ej`CnBl9Zk$aG%h_Z*KxK<}p(RGzP9rs|Qsev0a)47Kj8@AmiB*ZMk4V zwiFYkng%fvm*EI@rYWyRtUWzwyfIckm}YR~Ixg}PIxn;l{w7w7m`DY@xHl0z#avc7 zv4kQv>6n|9plJ6w_i$?qR$R6XCfg%P>C7aZOpCH(XH923b259UOV_Iyod)6N;}RW2 zxKYRykg@k_s~KGM2s(M+O$_9 zBWbNlqsj}{pMxtk+R*FEXTiYOs=yL63!RE9q`P2P(Ls~g58%Oq2=W3=jD`$FmtB?1 z!4o!hyz|~DS;VwEs*ORzkp6nCP-KMYR+JuD8|GO!#5<}mQBjCH8L1(>4taQ@TyzW) zaED)>TMi+p{Zyt9JY*fKy9JM>8XGx8gqNbm;ESgmxDyhl>X#2>w`hPWe=W6$Dfpho zS!2#qDrjaLG8oQOsM0n{TOiZfdrzChsn;H%t7&4P=6!5r;Ebf$Bdm*NzH8|ssq^{S z-H54xLyR@2Hc4TE^cD8|BwAFKHJ(Y|X?L}DOwS%NWWyQ>g1@q%nPhr}4oj41OE{r1 zR$i$OiS(KVpD);*0D&r^>xzl-ICNxPu8m)N=Itjed7rn3SI@A#DTRXrmUdAA;#=Z8 z?~#2B1DO%np9wfZK;?juLG?{to_6-O&mv8#2tg!qEzSdmI!lv6a=i8Gh_8u`YTJ6S z-Dv(BEQ^M{k)+aNVTKDf1S>T%(oPj#ff*Ac)0$XNg9Lz@*!Du3Wm{XQ4qWCSK#ny` z9*fULcF{7vf}rTVlnq1~I3r#_LuBn00kGU~aS`vtw3HP^Wqg4zDkb$8oz*b8Tj>h& zb~L1Af~E}Tx>(UO>R=`)Dy1yw)8cr{bL)F|i-sX|3Y+aVF9d0Z+^FN1vD{%gZzjsz z1VVM&BE2;9z4Ob)~3?3*oqNz#I}p08$3?;7egN`4l<4A_>{yTqqjn z1n?+}^{N)AsGKT^f|lWgX+ev3#cZQ+Y`5B&q6*`&<{lvK5&J&SeyA|<#i{H4jBQsC7Byhr<_=x&J{*gXFbR9NVtw0E1#LeP?=XziJ7V1QLm#3K95}FgHkx6 z&cv+SW&)9Lw6YVO14`3ZFQ0Ffrg{Wm6ONf%=Q^mAM?RBQ&|{|yavDxhuWx{f%KFLX z0hB1TqPxVlQ#Tma4mTnjkV^A3BlWR$rpoJ5SIi@MyY5HYs(E*uTB!fOo}b^M}RpeTEauC-ZOy`Q^oxSCEsq-o5; zL$zgQhI^MX56;4W052164_kdE4?Eaz;s`rAcVXpCh;gV}=}?Q0i(nqaKBCf_2~&k% zIdj*@>xg}r55??M4DVKHC2!WqoD2kN;tg!g6P~5xK`~&n9;LV^6{7Q)??NPr_Yc^X zXkyW61_7-Ps9ED%B4NknBV2sLqwaUoOYbigcUbo8n4bC_Vo)G7ymWpbgKkM@Zx z$H+OEGu*+}O*Fc3OUsrW?&7sXnQ4EmsDo2EvI41kZCd@Dc>U@U@*s?#lebJdK;?E; zTS|`eHT}FcE#liebyw9Y5YU3nVlO(p(Q2wh9v{4YQ)D?tu!Bp2;e!gpFgj^7%ce!8 z<4~}isfD;;2ZV;A`&NY8KQzN(av}X@DKpHe2)?`Eb&qI7^gk3{4>6@XO?CsASV3aP zxefcY@$WI`rC9dHpUVwp3$*lvz--(H%?czeSlNwIt`~QlRh^c*M-4nTIEqY_?&~o+ zH4Q~J>O@Q})m+whajTl_JloC#wRTKzv!Dy5SB$o8!6RtMOsze>__oj1md=u4fZBzf zz^SLQ{LMP^6_>T?$!sZN=c5!JvanR9XAaIz_m4g{xa@aD~1BhjJC=EpKB{KD4ut@tuvSm_wtiCA+w)=c6h1;PNDi_V; z#EC`R^Ve}Pnoy7}eyz*#5>nfJ17j&w12~5QBhy;Qm^dmT9Y}FkEHs$LSUPokB*#@z?3XKM6gawH_OtTBXqJ?~?T8K4kx<$Q)KpxtXD8~0@mZ@4iu zjx${&yl0j0IJJ0#xV~JjC|DuT+dECDUQFAG4lJs70TVhHiMl@Np+dSL66@g=4O%e3 zo6iOCh98Ys8ceNUA%?=D4p_4A+ze{k-JX%Kt=dhaly*_>860~o`sz!ay!N*YnT#ok z>j@)h_1ot_h$5+~@hy_ylYHlh<|EE4KEXx@feH7G*CCG>Qxn{wQmBzw=2afk*As5E=Y;AD5Q}6B4KudqL+|p zc2eYeOrSx++yz1+F{Bk`i)C;S6foT2xnU)q`4xubp&ul+?CHIZxQ#fDhBoJmL&^+g z%B}IifFD9A#Rl`61bu)iLe$Ht3NVAg>uF*+hR@1IO3E_ zP&BgEu}7rTbN50Gvl+jS(4TeCjxH1TA+4A>E&=j+Bhst`OGx@qGJ;l|zRP zj7C0sMu9U^r4DG?SyygUuWQ7Qb$47PplTCc(yfYZ)eJ2Ik&z5HtD#Yx?fk^8fNr9v zIx22KHS^D@>8GoyhSWP%W#Iig631>5F94lQ#$J2_b?>T}i`1P(Qhq}2?wUUezeJyy zxB|sF(1~>FLx_xY60%ng`bl8rnOhll)!3peNMPAuDeD*u$bNqgU5LnNK-L(zJ-uJr zg;?qv?Wjt=E9EkyHXP7a8(i$5}d#na0pD*~thM zO0LuuNuE35V{=}5*pXOB6DwgiE-EyS7}kpv=BK>5nQ}7h-!<+Z*MLH zgB-%xLiDMsS%y+_;Ves(!!U-rFrLijiXD*%NlIwJKJvi6Bzw|N4@#X^RXJ3I3?1hL zuwJz4G4^oBfzD)HpPl}c1;YK^>3Mfs1Zqq(@_etVY%vfj?`hk+w)n*IjT9x=5*M?M zPiSXHOSQ0x&k}^`TTUhAHi(M7t%iJFwuMX%qE6Lxb5A+N&uQkkHzU2zOcDXRUijgw zWMVq=Fm!52yE2f)oQm6vgHV&!N9`KtQ z_l+XOFcEFHcyO~VhjbNjfa{z@B5!P+ftOEUv`$f|9nh>t7=w$-7e56KC@%p>HlB!0 zge!L5j3MA4XuqAcGj&@{qS)=CJ=Rwwb==J;yFn3d;7E30LET2BYf}ND+gQG-P)uH&f$!RBjKI90VJiMT{Cy0=CZdf?#{{^f^5wCfb5V`0eISctGtTX<&GFa#y7VRV$Uu{m!s&K z8Fw-yys#1)QB5mNTo@Th!>GqW4z_T~77*6kIvTT*79(FT2=EygJSrUy@EzmBr$M#n zdsKY%#G7$8p>DU+bVQ=C&a>-`rBmV@nsI8)$F?h8Z%0~;P)zmM%Y#0QK24tGu;P&! z%qepeg?EE#MJ4fZ2*ZraYhbT-n!7F{8-@|>%)KXj95H-aLl8kB?mxSf{>TXkcH}O+ zNXXc|OM2n0^brJnRCcFNN9B>)UWE6D6RW&H;9M7dC#%tt#}jg40@&9Hcw22jTBD1`C5 z9dq{@ryp*3M}ZsNOJ-#}Mm5J&JtH+PK|z8PV9OnIC2AGSimOF)#;K3T%gBK_-duC* zk#%JPQtLkrF=Ttv`ygp@VE4!ecxRG%=^k?RFlXGkc?<>S`N9!*XV>`cr%aqHYDLSq){YQkU^AP{bbPB}K7S^1_@Qmiy{ zdNfafeC6P@>Ay8h3)6&=s)myf8@wQr3hLD?hI_El3hst?%*rbHo<#-BOENQKx6u#d z=!1JkGU*FH2VBq%(7>PsKonoEwt%b`m>^QSv1q|VcH)5WYGVqJCGLxBkVeXVW7_z0 zGaB?2$wCg3P2La@U=-U<_rRhPcoFGSXfjsAh2yFrnTB1+8yj3|j|<{4fjzM#a24rD z#^&V}VEB5Q(=t3~r9ft31YYp;tv6x}Xmo=J%)blFoeu}BROi7JAqHF?(?Ch0K;l*q zl%ijx&R|SGRJrRWE<2X!N_pb=Za!H!-j0f4qe#&Es`W=mEs)|IFScN~X1CrVPq5KsJ3^ig^ z(tyWjLlI=sg{6q6Nw~}1;tiXW(d%_0mWC)_GLRKU;0uX-&R#7+r3+I%61H%50g(VY zBb9RVv%7dItP;FOCIStX zrJKYzr9CT?jlMq+N3%_#c(cDT?%=j)Y8AmjDpmh9O z?q70&S?4_zD%3efdK=Ou1KXepU71fs>#)<|0@?2->_@r@yv9kwo-enk6-c*f;7+>| zkdhLiix@3O+9C#gw*ebVKKT6ZFdlkPaGRv)-s3|Ont`zb0k4R+2W!DxUeF@? zgW)mMqyc157Ap1Zb=Z2%k-T zOObtuu8x_?Hr+eYqGsp?DtC)nUB3wjawmO~@%yNmZv6?Ls{X)jyONk1RJbZ^cfw&|$>` z*4ZeQQr3punNu8N(tgXdn_cL;?Z9j8ER^qd4#x%SCXK+E)Lmy<4Mg)Bo3sRcne&9v z3W*CLl|?czpdyH;t^S6G!3?9tZ)O0FiZ2S^0tIFChA!WSs{5x$A{bAQq{{~>M}TAY zKIK0OBH0qPNk}O1!Ftr#26NBBRCO_^YTN{NQ20=|Y=^>Lgs!cH@n;vQXi&uOq$JI` z!3BwoBSg4JJChqeTQ9ev=z_>=IBeH(-lNz(xIV4eT5N@Fz&uib!9SGM(7E}Q`+Wj_ zzA1<&VKcg}cpH7(lsL&LnaNLKo2-khnPSI-!2+;rr87Kfo_Hh@w56L7SCT$wpUn~8 z=;60xMC*%#f>)y?Ze5j$Jp=&@z>D_w0`beKQw9m@`ByAa4`hzTpXM7njuCTGTBmO2 ziq{<;g(Fs^JkD}OPXcq^hA~-*O~*H3YL+=H47lp2goL81>bLBWheZr?dqPNjif7HZ zUmp&10R+Hpn(9!WccOA>U~Lcs1P@4(468hT@#5l;RA(lO`5YgMV*F3lJs3KK2R%Gn zHF$_RW#;oUkDkrCUZr&#P}FaqP^Emzj53#+Wfol#8v&0CI&VD+6UKL(cUnW5P&|I~ z*AHqyrU$hZFE~7R&JHnLo)d+%+2Fh-vBOC`@Lb4G1~iT!9!XO69{JzJp!q;jw?HP zQ1eAK=)L&MIBC@Avu&8^$EAv#f#~m8sG`oUs%4&*TEAT?092$(L8)!&zQIX*a z6~8rgPLMrwUP*dvw3i0jfmjAhuW!{kB5gD_uEjK+tEQP9oq$kP7XAu>%tpjjUtDnu zN#@^o-lq=R9wxXK$3%$_;R0oYg5{MlO=l~ZhpeD=rEUuSjqQT)hL_7pf=tmemjFFM z2qbl*+_~lxo-DUotbVyrd9OoG=^Hdty~==!2rp!O?iuK`#Be@dGG#m5!0Q6h@irNc zF~ShnmpJ${j(nSzO|f2^$$HpmWV}5AqJO=S9RVr=KCf`MF$*+G#@Y4(=C_L}06>o` z439&^r57s#6IumDYdt&!55pKPlOUJaLJK5z!3*X~oHuHLHy8JaWDAcIxbm+dHeoY` ziC@9snR83JOV=5fh^QV~S z8c_ZR6FSrhsAdbwj>+3xa=~3?hfGIVZV{^0nc+sd_x7p<(UNWPh;@d|{pzi!O)W_A~}et00Y21m^@jg51Rj z?)vTU<(tJA3QV(XS4!^rDK7TuTm2xS7v6nI7QDQ|=roOyX)l;nV1mxVq5_u$Lj`8tLJ4gVG=6OlvAct6wyg56Xc5s*1ioZg^6NAxw{r{4m5O@~fRfgk{-@($jp)9lmct zjflyl>30s5&5{cDX^%E9%u2(TdU5mmU({G2Q30^Z@Kq*)d5Va1r3(kD4w^y)s%#wP zcH?!(q9pTWj((~e$efB6@YzK+N^Z}L%y97p->|iH{a(KNLp0xA!jVG6^zG|56ph*HxLW$L* zdR9e;1RSf(DqVq_(EJcv?=(O?;yPE=m9t%(n9z)wuycg?DdjwahWdqU-aS%s6e^sZ zNum<3)l`}$7VM1;kJ*Yn7D8ra-lx7Dy-;hx5HO}Ah6P%#8V6ROsKd4C(({)wE7k9_ zfQm02@e}M-7ksBZ9h6s@el(td{)ZSi&D z4v#sY)K<0)D@ubwiU9TQwOJ#eln&nCWo^8#gmQ$fnLJwPW7%N_jyVam7-u`Mv~KZk z@PTZ>Zgjs+M-Rp>oBUAfN&*a-#SM~eu|~-z#Z94Xwk5SoWwlh=*(GgI zY!R?l#0`qsL2OC1fv}qaY@oJQ)E30rs@kfys2c*(T zZ3fhZw1Z_*+NzC#O{qrGCe*8CHiX#|YK@3Cl-i`(RkcfGNwBt{*-L3FWQnqp+9|TO zgti9MR>@7OEtN@Z*-L1tu$8eE)fUPvh;2o(Ce)*3Eu`9$Xf}bcMYT(0meQ@NnVlQkznn2{xqID`i`40kWfM7S&q?u}zfO8%mo>wxl*nqiCXRk+9e( z+NEriwkkHDY*cL$wyK*cEvj2m*w}2U*+H}`X>BF6t*F?AvZmD*(k+o&NLxxxq?;uh z3O1l^D`=ZVR@AmCt*WW9QrRP7HlVg6XsX(V!nQ4vHUY2}#j>i|4S?DxHVJJdY^tr5 zR>HPbY)fReEuf8%wl*SdR1KA#Y(cbE#Iq4?RLn-vR?}fiX-Tp{wN%=vu&t^aO`~X|WUY#AR?;TP zLtxs5%55pKQf85`O^T+}hRI86N;Xw3h&GL?Y>{k|Y=dP@s20g9YNpDj#G6f(O|6M+ zRc#Am*=&hy1+*=JTNSVt%Bi(dY>?QKX>CcdTPkfUWkYJIv?AIhY^d6mY>~BFRJNr} zlBU&DX>COWl6N8+E&m_ zir9l_Ccz_Un^Id9R?URAky}x5o8&$LeWi6^xWo#&I z654G6uvWoaK&_(KlVNQUu@h+xg*K9HC9<0a#ce6IM%37YYNKi<#5RetrpmTSHcD+1 zWhyo)Q)I1@wyLeFt7#3XErDAqn@Vj`TPmvB7R5HITS}(MCc!pJn^HD|WUY#uDK-Yn zX$smUw1&ba)mub1l}g$xY9`2)v9gre6J*$%RBcfJR?6CJgJ?rwHmS0uHma?W z8%t!3sHVwQz?Ri3H5&ppr8bHiCe=-pZAi8wYEs&jHd1Vs#TLm?vIVpYV%ntIq}f)= zro@{>wkFCOQ7xshRkTfnrqxE$t7?|SY^a+gZA95iYExx4rq#6~*htt*VwTjV)SCjf zl(r3(ZC2EV)Y?ThC9qc1hR~+arfRklHX_+l+JxFbwn=QMEvi!4DA=L3D%gW*Ce)_Y zOJZ$BY!z%lvL?c)*b8YU)V7p1f^7wCErlk@O{&@mZ6?Zvu&s$(RU1*Vs@kb+DB7Uf zM7D)ZsT)N$pxQ>zY&NKyD%!1zRkbBHC~Sz?6|#uhsM<}S*p+Qkmcq6Ywi4Q^*&9){ z4U|o)Z6j(MQZ|ENnYoCe4y%#in*}ze z$X3)=!i|L+61EYr4T##QwN0gMTPU_vn^h}nl-erVt)iO|wAia8%ZsZ zO@%h3wpBKTY)z0=v=Z3^XtqeUs@k?C8zQ!;OKRAQV6BR6N;XEyQ)o@9me89@EsdZy zD`J~LY(r>lfZA1CRGSHGm29hI+NuqTY*S*DY)INF|KgG=kNuTZ3Z1|2zt8*qamO9M z-<|sF{&VO1`>%Wc|M$Os`tkohZu|ZkMRS3FxB_zumbZ(c18N+@T8;<>&5&y&tz+Im zS{;j0hQ3)(c{5qx40XcI@r3|LkeIza4O*RcOX6mo98n)uf{enamW}bJSQ?XbgKgQ1$OZZFkUX=p#hnP zOlUc%>L5uh9h;ET;fMH)xiR)r1}rP`oW2%ycD>1X;3fD+8! z%iuu(mZ$4qH%?vc(4t5VwqLfL@z^hT$xVoHSU{C0R%+#kBDm3_oM|xWxTGGD##9YJ z`4p6g?aPYtRjziq(duT`H!q3LrbRflLb}sO(t|2@lgZH#8e$E8iq@ti#|@0EV-`EJ%WrMQ^reY`$9(gkHgzd`{$D^TWm* z<72O0c;3SBM>N0#6^9UImcm>gZ(Z7!=u z5R`Jr@q>n%R#Znq%$YPaAR`17xfa9=!u)J47Mxdj!GI)|eK%f&lrkL_4Ro#1n`p9t zRi4sb%o)jqUL?f}8zOd|#1XrnB@1|&XC?KH3)sPrFYbo|5IP2MoPO&7X)|u~3Vi|r z#gd-_%8)4if+7sW3_lOFjubNemnea1fOLC99qVsaM(@iDLIsFhwg^0Bl(^m)Z5!^xsd_89TfJ}*wO5F~y~V4uN`y+7en96cqwI5Fe6kqP z;4S4^rIK`}bF{;w!V6DG8hp-O1%NNb*kGH9@M_X4&L3Q$m7~eNif+5E~v8$2%9!`v?FV5dH3qKpKN1)`Q@HvH|1h<*dS=oz`MiM5zf0YqO z1V>CViO%zg-(OX4sA?;oG{Gq$JP;$nlWS&{X{6#})L?`xgdidz;RRob2|F8^KSH7_ zG18><>(XKm+z%zWPwP)F^Dx< z0+N=>)O;7sqUKqxv68$h9phXRh2A8Y%NrkMBPucx>sea9N3YB0xX^{AL3<0cIE|2o z>CUOCekov)Mf$`H$f+#^My~Yb9Hx$b$T`<(z1vXQ>JJnV6_^iRKV`HxgBXmDb3Pem z0fkMfZBsD^)U^OaZ(n$(`(^5qjJB;izVC=ToD4<+1H=v*ClzPP8;6s%iPbqSnSJC= z0!VCRa8+j?7MB}=U=tS{0>fD`=aNn>h%s^XKoy4g9h&jZ z?u-{M+_5xN2T^p1LasjNVAFh9ho!%knP<|wR0(Blhs`_)L1J0ApL`S`;4H$0Y6=|& z14N&Eq5wE}u}+*e+{)m*nUM03J#xJzn^n~F z8o~z^V#XR0N%MEokvh0Vwv-Hr_JuSZbkY)|l7$J}VPo1Wtq*!2oI<6&T-Muh#h}Dj z>Js?JpkJBBM?}Q+p3yY~EF4rwL#{YP3Cj1@_i3DFzF3saModi<6BGq~x(M{TLGW}- zh6*|zA!U9G@@a0k{v3YE1?Euln8DAPGg_-RYt-Ez6n24t?uV<~3GoW)G!EvQ3yN01 zF~PHRYj%XNWH|3j1Q5@G)%iJw zE<}|vl+dQKbFqaBy?Kld%$`ZnvFX5qC{lWKuX;sHML`FJd^*`JTD{=YFS@M$XLAi2 z0R{y4hqU-`${)++FHpDCAQ;_!W1Ht98YnQf(tViu*W|_21R(>j$vFk&3iGLhyf`eYESYCmEECOLt6?=h+`?O*Hc z9stDLWK*gUETA&!pm;LRt4)_RH(^sZTRtgj^;4YP$v%fm_*qOH$=6I07zEDKvS?gF z7!-i8$BYCyuzUlxer(706Qv=^+{V;^i1v}hDTk2Xt<|OC%N38{1rR`_uG}3To#&(p zLkOTCqDV2x0fsBk9b*~e78?jX2J9)K1(xFPwm>1}btf6qX#>>lvyf3+V|;QoM#OV_ zEMy3$6-4X}*yA5Mv3&l04aG(#o{BN07v`kUeK1k zl#hB3q0xJB0R0IKhL$E|q(9FLLF-0MKu#Hm5$QklJ2Uu8N zkgv#?;H|hSiVEP;N+ETRl|sY}L+_BJ!VI|pmhTvwP*84*oMY&z!4G^H?zL270|J0| zUHY2b7DpzS-hCl?PzT4gKR0@Zqg)|rJtfu|r56WR(0dix3r%t+0)3eUMt(7DR;U4Q zB()KcwIEG6rc-p9;D|LLxEHK_6jihGO!U_csrrN=fVfR(?d|9I9!3S+x{g~*Wt$Q9 zU`!Ci@Uw(H$G%8RS>jYQuq_99YsNAriS*0NRvtp_cuNn(M_`82nN2#C3DcqLYY}waa?wSJD(vW zK1)zHHNc^(QmXq9V*QumRjEHMhAx&=;^c>5CHVBMJS$IgX3-_NeH38yw$(OUq=Y&O zApB?IK@SNC0QJM1am%i4`Nh_AZNIk~@wnHH+-&j3ZhFrA1t&8yGjEO67$F&?1!E%! z`3z6<69Cr4oo=?*T3@flpag6OB-jCd5VA@ja*(L!$%6y;IUDholmO7oH?i#BLb%d} zlb{S9PB4df;M><6HoR+%ZMNKQzVU49*BpA`chmDY*IBk}ec@2vN^alCZlzwi?0eVFzj^zxKoQHo@4f&44}b>`>k8{nNXl#pS962J<$Kbk!*2!Dr@xk{=BE zpe+6}3+$PRXm-q*!cl;JtB52fhhP(kZaYDr6sYhK`!%(wM?DRFkEU15X37qUNuHK$ zZ4DoDz;L}oQ7)cxX<@qh?L#gB4#fvqhH*evtt3to?ffxwJOT{RudWz$VPxxbJ6z*H z6)$7kiDh(sUW#y#k3b(5KUVEy%j3H;2L;e%eX90S!-aGw>+R2#U*dR#8-&cpspcT| z8GJD7frBw27`#K>B|^mN&l<&eG(RB78Fek^mNUvh=942Rn-=8`XMo*QmN5@O_I9g2 z+aW~?8CScS5$GaE9`{sOLh`E-5>-LpFF0eg%^iw z^AsK@akX+%C{=l_!es-yEPCxs=bZ>jE%qrDRx%XfZ)r=s31|=CKV}*+sXk&nq+GKP zD(EA>r2;jb|I)NaNnl)bGJ((nD-$Rc&p;L*X-Wn9y zTcAHycnjI^76*>opC6~g5t0IkVI4~9{dh+`eU`XBdzx*O51{6O8XDpXqNonu@a$g# z^}Bqa#x8Q@UV=QAv78ADUIH~6LQ_+JH$sAdoQV6A?j|Wa#xGK)NEPPzdwNK2vSlk+ zGdAS3!^_}0UE;lYszO@FW65G*^F>|;_N(Cue8(zaB11}nTaES7(}nvchC~o0uO5D* z){qkIni5bqGKK6O-IfQnZEhs)>0deb!ngyWAs zkTve)(b$B1{vgf~IBr~~QLgCHlpry~32E^RsPOnNkHpoSlbE0( z1>O`06%wPBOWCObn8!JwP7@VrFE|81uP_H}!q9A1GtuwGeQn00gUrhw;=VG(18=w5A56N%o?I5*_2N&>r!N)We1k{>m#^vE?e zivSd7YsuhT(~ct3%e<}7Z@WagCh-FubP$VeOE3yh8%S%`uKNDz5#$kd;_1wjJiX5~ z2nmE^B(jR2=75dnsd@oB+Qx8UnZs#MMjn)GRBUD99t_|+nb7Fk$-KnO2sQE*3T(v9 zOne{#!W~4OZ&`~>_WH9xqNobB3)3!Mq(>i5?@9M8Yme|AZqGCX(Ti%nb7s?Z2Am@EXMcK^uqo~C6Sj?%I(U=5@Jj@TzC^yh&pGoM2y8pJ zigK#IIB8_Fb7LQYn1Mxjs#M%YmI`S&EbrNv?cmMW3)q@bmJ9eFAZoPMNQg?!ro)C_ zv~qVBd1*JIoZTv)3|{#x46uG7NTkAN_#St?mu0oopa?D9+!q(&VP4rR@acNtKqd?C z3FsHdAmbS5t5q$?po&03S!qtk?6fJW0*cKO5J6Q>l~E^Sipw(=nty}A!{W&UQ`u#u zM0NNmefE#cAP=ifXGneM%w)3%svaZa_BeY)V+Tf8=Gv<{C^K-?f;P5Xp#qXf7pyY*n{5Vpr9k5wDis3~Gxrn?pMM^vaGR%VI6WtRgNeX37BZD%8? zk6MW*KFM6)fe@G??Q0yx^>pEpUbdP(E`vh?k`nFvj*_YhE3g6hrQw6F``p9Ur{7bI zAmy36?H9U&>M3+RwS!1ZiV`pqK2d_#y zmjpoEcB_E|FkVYM7b+@w+$Du)lw0G@i=w1wz1}}$I$5uZZVrnZ1L<8VsYd}2uKLmQ zbRjn4=P51}NxM6D7|++WLdT*6(_+m{g!Cn(FtY%-6`)o*5-LDh1t^5x!ff|yDwt)( z%f@ze?06sH`9Xk8hw4|`!z3;+AQlPycq#8Y5T zLh_uD7laTjqYp3)qzZP(kyKwst^I({yg}sQ%mw;Wv?)RLO;tAlK^>0->}U(8LL*tZ zzQ}CzPELqv5ZRa1f>D>`1`1O-m@6c>;Y6P)&4dLy`)~&*F@;>6J@s8>HyU|#<`@TX zT7gL}u;WXm!-Q1vkbdDiwsAkMV8Mdav_%<co!y@&RwV&)0b8Pj&e=NdKFnrNr;BuM-h>sC(xm{?=E^`Uri5VAVKD0p zj$sQaEl}$i5Zi4PQYP+sVHM94U`7f*QjJV0!L-z`G_UV)at&~0rh}5RoKzo865W0% zVeW$Z^M=!M#;omhY|I+?Q-!wMF;K2cb^SwX9$}#c%ujStkUEsZMK~++o7TC2WOp7$ z2vpDBHaX};#npFc1=-1sOCZA5`>@HopIL%>lLnC0*VHd8A=%mDoUgLD!>qh(XdXgk zZdi)ay%=TGC|cgTTAO+R6cnSEcN$PLgJA)QHUt{OUQ&|F6wDmu!xwQMdP2pCg6Z2t z{nYfArOi*H<@SmKP~Olo%Ff8$B@M7_suFTrT(63WekdlgoLhh(;mDn^SdxNZB*CuW z&UHf;(4_>^aG`x9?jXgn!^HHo2ZH+OLW6m8q)VNXG}d(dJLRk}Owy+BH_s+hMK*xd zFA^<%C9}c@5Hr$D$x|Y3KBrEOT{-foln~SHBR&0MP>;;V!8ekdR;k#G%LME)u5KfJZia99vD3*5hGn1RUKl=OIHDT;8E` zBS^}7TH8p2l=gc-l~f?|sLXoyp;*wD!bz9r1rYAR_e*Dzl4ML!S!n_HTBOf3Afdd_ z25m#hLT(oYH=sO3WkB`^p(OJUX(-JPYV4PVaalg>~kjvkJ41W&5b&n*^bX%b&z6@ zDZ62PG>1#*c_C3OFz9gH1dohJ?C6k$2+9ouF=B{I0U#V7)W(%hb|4?Dyf~_BWN^oF zI|&OC>`a8|A+FpNdI?5D!Cep%0`IPmkq#D+$7s=g-wDUWNTV%i@{elRfC}7rJ z9T*U)DK5c-MG*&<$vnZ{nr;ZK8hya@@}ckW|WKXEY+Q9!c@%+?YWQAZdsmRk?L47052dNF81@BCj;^sNo!{ z0%C)jZPClIPiDy&<>#VikV?=NMq-k*kS`XtK4q$nuvAq{P^nJ{dm$B9U}^!%s)vXc zKZrmG%k3YLDE_hd1b$iWwVWWRu7&xg~ zd;#!$A8grv{rZxXpYpwM_{GdWZ55d#Vt!k(E_PXQh0AGbC#}Y`OZ+&%q}bU&ew9k> z=E|ozU|(VUJIG{!vR*7sN=~6$;AA8>iJz6hOh^LgSd}LCtI!8>x6;TS;&=hHo&_&L=s`$7Y+Un~?QH z1Z4a|2iSg|U}(sS8-l8SC!n7w9My9YjQcGssX+W)))(b_In5AS< z^EV{Kz^KDe$zOv`jHlEU8i>_~=fu2yea>1riS*!bq~sXe(0*|No7T!&)Hyy~Be zT!Dd9921kzy--6;<(*IRx<(U?ZEsFle05mH*(kLr?FT!pK|AV2vnoQThPlfcBcF8N zsqhKNZq}dJ7YUdS^=w0*5P%fHFJF%c&jl^G5C@+`HaNotG_sgT!`nRLlE>CjJPqMl zGzZHO69RLcRxnO~P3ipMh*jDfkBTc!6WYC59j+R%%AJ#u^uSFUp%1iC*sT1%DnvBP zB8noW>2h$_-lGA0pwqPo5?wH}_=O&{EG30>fbuQyFFnN$lFZ?VNfgQhOAW;!{2unV zKot!W2#9Hs2ZEsiYG(0<`OuHbmV|0c0*Gf#1f6n&3CQ-#aO(|PqrTmX=NN2iTZ=T9 zLV9&9!M%w8C;<$-7}Es&Z;hPnxRM1hB3xrDRkI>2 z`LYJM^dTvMGOH(i#>1l!jp-Jd7UM-_L##OKeDGytLi&xa7$%Ejq2^nbVPa$}Vc~)l zHF>xZ_(ceQ`iJH~&DTCMb3CH^idyA@0j?^qoT=oJFVn(}rp!Rg$kxeCC=M$z5FZ@|hf)mq-GuhZ$0QpW ztUU^M86lt}R0*PnAg2}OsjLLS;8vO~4_hD}UNLeCh;>M9FVh4g9aY+5kvcRqJ~zC| z8&o~O@48_<1Wdq>1uO;v!WVeBo8o2W7I{YZo1KCNE`5aSMz{~UKAY~Kh@1u-3VVs7 zD1Syj24i4?fqA1xwxyM3+*}phiH#)2L_^PJat*K`*hPEnOK!mz!3^im7+PZ9JY)Jc zBWZx63*LkH94gg|eH5tTJcj-EXCb(2U{m9mYz!g&L6 z(dqbAQTi^-9LLj+?-7&Im~IQ>ToSB%9oHfib(G`=kw$LH%LJko=OXLb*8`1&&XX8; z8h$lj7X%c60M(ePNp1|V0amc^$lH)#xck_!Gqp=8;tU6aD(FE_EaLZ9kz+~vP(hzo zO6@+uk+7Y}wzk3uT&|`NJpuqwP;YJEfgyIJi6?aF#@YK`UTAlO7OXfzJ{+{^8(bfB zpiWA4r4XyIM2gMBIdg02SOKfZzCD4g8rO%=d_Y}m=B%wZxI}I&oH}bvO=o*gh z3O|JQ?YBaHzV__F$|s?m^m(^OzEf^?u^0iTG?TN!hp=1;G%aYSGl;R z9>|lJ2^!j8T-`C_r=E(m+sboQx_%2#n;8$pFCaj+zQG9+mDiPubzdoQj$b8A?BVQr=qCEGLs9prQoVl5Qeb;P#XCva|Rb=-~*L#oej0xkGS zcyh4VNN{yvup&a{?@C3EF`S4B#a>SM{gu~&SzHt31o1^-0g5tUOM}SU_f9C%glv4_VzVH~27hk!9t50JH^65Nc=G3k6gP<52%sh z1b0kFS567(+42<&l*@$IE9EpBypVAo!7R6=-8=CrNJ=~GXvjLPF-6hR1(9dQB_GYt zy-2Ill#6_KD~rWgLg8&?N~B9tqawW*r=4-Ur@VNlfNF9BS8CH`fhwA_rQ&Vg7mke? zi9@vUW)&`!#p3i9DCdXL&X*dATLXZ_vFf4LwWMQUAi@!+5MtFd3?EvX!JaD9%NGi0 zm`g8@ud$^)Y0;dQ&E`?R2qZk&w}Y3SmKb0f92%IHT{Qu)^VezuxDYgGbp&CcIHRde zg)?OS*R*0Fa%7{3W)udq`=*JMN-(U!xog{Tb&;hNjF`{wOb1pq@frp6gl2VV6Bz0dFiGJ>0sK~TP@x?7V&Mks<_SuZRBw6=?V|p zB>5lbfK>fk1f|H^H$E59gfJ%oRE7%Tsds?P5(`n-T`0_~Hbcl=Fc;UDQ5*xL4ky_>~~^4qKc2|Drk?DZBM1=;nMi{6P zRophMG4Y2d+2Jfr2yn2bl|}^GHd^jbCLDle+ay|cc+vB*lbV^lM2_Elsc7 z63NM#rvwRLB^enX6vZAT6ekw!S4=nFyro^H3wu|pT6kWuRraa^%h)t}!1bRr6;jg0 zsRM2-j;;G~g6*^zbfzzoI*zQ64A^vNf}%=?P5B(J)VftSIQSN8SG8o%LYgQM z#0RYI<_ELM73NQYA);H#1$v66918-hYIr(@3_=aZ$RVeUIgtfwT-U4mme8Ib~oH<8riEHAcs6Ng@0nYhw zoER%<$kT-824S{Xc1pMmn!G%Vg?8ddfSD9md*ocmG@A;c*?RFEySclSK_c}z1`ft7 z);d@ah=8Y6Yhz2YH3}PVF$A&>HtCDha8ZOdov`$r=E~p>cJbUy4m2x4Nv|ZgvUhR} zAd1D+5jz6R&?_Jyu)Lkj@jGE)XLx(m1NHD&{>caQ2#hhb z0S=N{JPr-_BwVz;*s1O{D5*mUX#CsOJ4!ohCLwPzk9}5f)>%q&Cfig?j737IysFZDAH7`PiVL?98vN0OspGM)q|*hznX|RzeEWbv*NYX-#VL1ST%- z%@Y~My!&s1*c+y!1QCs|9!-o7 z`UiDw=!HrKvFRLO5T82n(TXN)I+1+S3kW_K$I@TA#5j)oJnJpankDzeg{$U~BB8{5 z5Gxb|hC0FPqWMMn5`4DTELdP(UZ^BTN5KjK9Fr3m#eV_m(zwhAG9t)4&g1V}x#(wk zC?IMmYMYg|S-Xv`Rd*A*i|e|2#QB6P&?ZI#6$ibTFLZ=KY%==64!+UT&rz_hH`7bW zVtKyA*#*>xQucPP^H@UDsS>mJne~9pRHl*%s)5cP=-W*itar!%!j#EezKv zpi(wo+s|N0>K*S2Rvm(r&u1;LV+pXzYZq&c25g4#OBD_Smr|9dz$>{KB}Q-o zUpo9(HMSNmS+2b+J$sKw(;KFAuzEOns5L|4@?H4=FJ+o1ei`~tl!YtNgoJcpo)Y`k z;#1ZQ+7v7At)m#?mjef`uB=7mtuj%cPKH8n&==%akyKej!OIlpEUmDK@}W1mdQC=W zHbUNIl$s9D*x5Z#S{7^Ku^^{Ix>Q!+YLu-*-Y~m~X`u#n%7|3UR~1UVViMyT?PN-H z#~*J|?G0)4w7M#hbXv^bh;#0UMtI_bxK~HQ#!@?mc?Tx(7&>n|wW$+~sCq#FILgsv zB}kN@N!m;Y7Z%*6>yp4Yv3iJ}JquEquRdd+atyQQ7$}*CH6uA{mx?t~L6q49@7*EX zpv&v>O&`OmydM|`m(uLQ)bI~{+4@DZwRit$t z5*FacH%|b{nu5dXz&U{f!EEjt(&Wp*X`%D?1qYz`PDGolSh;Ds4H7H`Jb`-+2p+j1 zJIx-dK?xJJR7;k13v(|jPc-QtAq@q1S%IYv${7ZGlmm#DF;JlcTH?Yz z61;7h(==JCMMCyd%2(m(S*REg(ShfbezUei>IHI2(k>QL06YRA!}y#qgVwqNY6Ljd zK#XO8S{7STlJh8W>PRIaDe3xD6)j{T@n$Us^T*+a>7fvs7cm^BE^b*Dx-ez3C_+vK zG1Y`RkQSyr-MS&1qT~)u4S@yPl~sUOIMW6g+s7gwH5rimnR`XkV73U7o&y%gC)ua$5ya`mP@|-G zN5Soch)K7+lE}-_hYNOTXXz#O7&Mw?2?R9*?^Iu?hXVYdRU5eKu|G*j()Pwu6;AW4 zObFR~nw}!$Qduac6d5iQvZ8?<1vd1vyb!_C!*eHIPzxb`6Pi}kg&Q72;pGado))&H zXF7D%q+{zs2&RZ#z91eAsgO&sYX{=t@;fv)WW&Ob8&o(2GYp{SeI33U>4$Nct8J-G zoo7-qUVF((m??fuvDFy#fV}n4&oNT4ZF==9Z;B6;R|p;ir>`B^T137}!s8qdYEmK6 zgc$>*VLf(%kdDlPi9=JagPJL@?dskG@esFQNm3=LSS$(Mn9@9ZbD}V+6SN_Ti_Hl9 z7J$AMIoYE{<=I8p$n$rBkK4}xY;Yj&!InV%8l$T7Gox2Bh|Uv^%n00$uf^mO23-ql z=~-c@xXRvRh$D=@P^X38pw=-Gj0wW_U#n(IjH7J%4mmUz=L1)$Ft;};IUXIfX5nXpDk?APl5zeCnCMX16I^fB*xEA!72F{7S}GNpiE;Bp|#c&XOhLfuNBP z^#MN;3uSXL*}iL1_b$;xC_cV=-va-x5UyeUu_px68x$U z_GJ-41cl^{>E;3#bKdamevG^*sj~F+u*=Rfa4|=M=IPv(2e%L(L}Ubtrfp%LY@{8I z_!I?{As)9== z3F+$cjNIAhi{*6n@Fg5jfdaF%>IWrK>t^dsjiJ`hZ}wg$8i z8F;-x(H`)wHCE=|Fa#%2((B@e=%@>tv%w+Pxe8V$RvRHNmh-LmCmEDKoXuxLkxNkx zh;KUI5!O)sMYq#ili{#UFb84?Kp2F-(%EY&u{ zqV7bLhfdOU%RJY+uvI`o_V@ehh z>1Ixcx@wjJTI7!N9grOE7A(x=zKC{&4pb+YvsHqeas z6Sn|iLu}RA$m$471C57Yu&Mp$@F{Bhm#s1dh8?j!36aO+wgb0Pt)781q$|rL1hVQw z;BN7t^D&5J6`O|Vjkn92DJ$tCZOFXjp#d!MEJP&(!y2bGq&6x{rc0kmFJY;K*6Ee$ zzJzPl+fQ}{CW6W-6qcyPwI>P)z&saHowV5;<(aw^B!Y<8pldQlc{p>u${r>WbPfyk z_KukuNYY!DQ^Q`k0D#bqZbAo;7agh6nQVP~oUDK~%M6zRJZ>0aXnq56acoBeyCo|+BWpo|`S4;Mx5}pE2jev|VE$a-56&{9Vk}@1Y@bb=5VKGF(#u$}~ z-83duG~3ZA6d^aKj0KEGoi>s;ai|s-BRu1vhJ_l6t_kUj%KZNRj07 z(tJUu>F3l-t<~L)ol$>bxV}WHpPEyzo4m}CCBUN&x5PfP0yNxwVpVl^m>|e9IUewH z+u5^{Uq<9^U?Jmb30)j^4?%;C;E1k`$RTP;#W6UFNMUn5wDS1JT1JmVA{5@#!P1VE z>hhO8X6}uLJdBZLm?Z5Vy0Pg2NYqe8ZiF?Cx`Q!&!>4mt-(vX_H(apRa}1%G7PDa* zwx$=!#vsZ#w+cU7MI&-+%51(`HpNX ztZkp9n8=9=I6mwNzR89^9Tu{pW$W23-@?V6;at!f#R9v>1j>fttS}gOxr^=)O_P z4a-UxJ^{+Bq{%B&V`ajP;>8E5aS{=*i?e}evD3q41;%v1+La?hB`0DECpLD22BD## zaz{{AF6UUem6PCaZcKIxhIzd7%AM?Rz=7>z_h9M!DUp>Lgv}1Bo&4?o!s-@iL^y?gxp-+w)J`~FHrU7zpKr&h0v zbNdZ zZW(NQ$`NH8DP&Z~GamA`Ui!$yAvC)o^aLW1Qstq7Vfv7`oV7T(ms=m+(q^*it`LO0 zEQ#uDI|-a{n*nA&qu%SrU#J6Vqm)kPY`UT%I|o5=1@-1@W7BCM zQi;W{S>L4>6{`y}o@q;xAzIMLQZyF5mAue%W402@EDNN%z=SwcgB%4s@5@M`YCXQ| z^r?_aEDX3omz|spb%2i!$$})r$yWa`II$G9nD@^)#ol;*1_kvg)*KZ=Gcu2ag!+ZN zH&g|tT*Ps70)U9~JgBbk^*_8X%n`VHHf4ZLh(c{~Y~5`j#-Cj?9i#0)#k3|T6DF3E zMqWRwNh)K2AU3IH>oYWzsb(;pjvj!nIK>!D0+ zz3;Vu8`ko5wo;0H&n{Heh!SXK0p<-LKebAxEVgpjY#Ievd<+xTTB6&~Xn5!;?TBa) zN^I#O+efL3$O$eQ8Vu_JnFqwvj#io#TOsg68^^h7A?pUm6f@tDvR221L+LoVRb@8Y zO?gX17~!cYvD`d?XSi^lDvHB9?|Nfh!`2W-i3NQp5Hym{ULFB=&tZWi5`@LT+sJm7 z_SIa;R#-F&v>bz#Cnaxsya|NNxL{yoBBsOy84gN_8O@>tU}OTmt(DpjRh?nr?qZn3 z6T};k(b3|O=P2`&U1`886%b%#7cMqLdEJ{Gx6{+oJ2B>;QZze_5!!1p3ebMx5);s3 zIEFg$V=fh=G%$ZaF?@tnBMQ0%LxULKS{H{=16#G(L{jR;>@x#gj+zue$)s|fgywt| z_R2s3Ty~sr-+J`8hl3e?dKOq#JTn&#vBT0AOqPk*N)SsF9|(k9K53jVQn`Q(%#}h0 z?-o$aCslPaJp|@5swT+20h6?ZZXtt2_OT32OHa&4kXcq8t1 zrkAin%-heiVOvHxfw+cdQjr^t7^8>M&kv9SUu`TPX;>A1G&}2)q%e<4Q8DC1l;bS-137dQz0xHQ1qX z#AUbaZBpq+aQVGXdET4_xIkDCWUZvKbb1e*R!TMqFLSAk8(`+J4R#tge)h#!HA6|J zrgR>CZ+pT*JO>_g36-wKHyK!AS|MNqn=~!F1Ra}&F3Tm+Z6p(bi^pP(@ z$i}4n?`6U^7CRj5{T%V27N^U_X@x~#u8-&1(poW4x2A$GB-{&H?+$x%o$&|-_<$aR zz+q$Y3uz#jpncB&dKJa50nY)7h-V&RE=IE`%wqhOWvs)p8O&kc4OF+y?^#jNh9r-I z^xBT{1*|*E%$bSfX;jxiNA|%dIaS)3C|tm!Vh0sSb+?{egka~X2aic<+!SJtjvC~J zh-iYf8`#8;VB2;Cj3cczN^YoO*O{+(WrIMc#9|ddrd1&{y$9vrIj%d1Q0Zick1sn3 z*eeAin+Rr|5<=@Z>sr!Y;pf)gBgqU}rnReeaPGGn-3in^BKw$9*X%9i;~;Fa(5+_{ z$BN7v;b_{n!DxZ>n4FOxb# zGXc!9n;kyj(HDv)omNi(T0;XBO7V_>uF#**Heg6pQiv_E`0sv^o$N`>1AYnNVwhMC z$zka~k8BtQ+d&zmz`1M=w*0{zVjh5QnRou!@n;L&^5&*hJE&EM(kIt54M^t%n zG))Bq3(v~c!+=9aarht>nuy&Gy-&)f%{@zca@si6ZKV;K`0CgUJ7y8N zi&d7)(_nW4ZmRMYPPpeD$`Qjl0TG7wre~*DPK0YuK%YMO1ST-LRuv3}R!tr_MrF(% z(mB@~X6#(WOL)d}Xvncm!AXA(0l+N9NPC;On4~z3R_V_M*t;pvH`k5pbGYuT+|lN_ z+sEURHy)i?r5Za1Ay@8e@R=6C_1Jc5(z%za)$;U?=rfgF*###nG;7h;F|=pTw~-8r zyRNZWHlpsWx3o)dj7@)T+aQkdONOCnOCyhQ`MMPlOj`O9-o$r2o2VQj44LBO zI)kfa*!Km~30*24z-Q(klZ#so#q@%&(V@jG-!${gzLJIJ_1?cCQB6FV4vA)=F z*P#(JGXgfa<6wV=&D4bT>yGp+>ki%;64Vq2;p8D4Sx_Q0GS@&D7-UYi#T~##_0$%> z4Xzkd(dg2GfT=1B105w7=ui=i$2%F_K9N6}sRh-XD<%4o14jTt3PMehU10vPAn7Nk zHj_(HKB{hlg-$EWUbmybd;#**OSP1rHc@)w&;o+XjiQW_@i=dOAmCFCC-eR2TT_Sy zV(yZ%`sA^7fc0)3@Vak&K7=vk;%$`1n0Cj=-f-Zk&?u;@Su6*N?|EDjNT^M{CUe?M zW8Gy@4bx{^CIFk}tIaGKH&lnvG+vj_2ifQ}qy;IRuo@VcdfKdWE!UA%w$>hf;mG`R!u%4g|RiGE|(^ zVT=)BIciEnPR(uS0eJj)Wz9si#nrKDX2!?UD)JO@Felkr7&=oHHZe%3%Mso|ZC$5K zAF!fjY`+Udmw0lOu*;5`JMN6Pa@~QurQO(;yekscth1CMsH$M+^hkNIS`?14LXtBF zhE6IG@tr0k6vnAEY`6$3LPjnKI>NKL{63}v#f!XZeuxls2n@0>0^8AzyA^rqLB5_q zHk_6T#!Fw51Yq#7@!~<$2S7s)7#nvw3pPYz_lL@KHBBuC{0u)cu7jAYl zd?e~d{6^T`oe;)t{YSALF+Of~gk|fS#8@Dj<21tsLtkpd_E1!D`Pna|pjDNkHxUPQ zfmbbJ44VjeB)*_fAQm{A9I~_qsz)veN=eNrGbaI=rUbW(F0UFLUFVQhxV=m(47fy! zkb8EmqR=^|Do8CrPn%mf6AL`ygFb(R!1a)LL?D;%q7G_8`4pY~rVC|H(ps<=pol5i zA?>y@E*?(`8URIabU{>=wPi$zM|o^l=zzRi-sm50T&wQcRq*4qsxQpWQPEu-{DoC4 zBN_<3Rd<+y_@(SZK2VYH^3m}HEwei&a8Mv$fd?bt_zJ51A_^+_qnhh{Q=<3{Iunjt6-VQB$5$@fhIQAjeDlV$XK$Q5 z_1|^YZY#BxSq#wz3x zqEWW!HS*z(%vbl=kIY3Z3&9WI6ASW#P%W#fEg7+UnM*YuY4L7gJ2bG2lH zu5IyzB7ZKWi7g;WfV(!J88Ni<7o~g686^Cd@ZfupxqPv8XStXmZzcrz(mrm+3gCIU z95Chd!06AjHx<;%WYp2SV?I1Q6v??h2b5;OW$Lt}w;ZpAD*@fAb z(7ps+l?zGoX|1VQ7ja*)tz4hafMAtRBzC|iD|&rIN{e#%dz_E8H|(AFs$gFah6;jk zz#x69ZPfvgg^}SZkxYvk%pkuerFr&GlSw4YXsn(*5Y??~A}9)H83)Q&7Znvz(3v!9 zFvJ9Eh1>4KQx)C#SsxSbgj#3DqRUTkWym=O?)q|2oUm-uz^53Ln4>)cNDElvo?~!* z5)_`Pp*T+>ESoCq5T1^LWou+&zWGCLNH_*nln0k|@g>*DkJU!e&6|dw}Gz#Rp`Q#U>~Kgv9Y9?X@s8roD=h3-WJ;en z@qUD5PSK9#I;mDzRCHTgx8w`4eP1EQF?(?PG>=ox7aeI%r-F3O7CupR)*3U9a$AMX z7_v3Y_5<7oAVhn52bQ<5O|TG{y4lI(Nu_?jhHgUzY0o7k_cajA7;Dd*a7?_rH;t1t z)=pi%Ua%z^%H@?^KGq=XU`?GxE%J=`P`ip#zOmddtp$oCN@DHf8!RP3Y42UAS{<=5 z_Zz1$FP(4@Lk94^1EvB#4hxA;$j8;hw|*9HD<$Jgw%+^%;#Rw=0J7NZ_Wvq~uslq% zLs6nod5&vq7`Z}#+|9iIWg{|zwhwfXfVBgKZ7dSdL$LE`P7`>1H$0Sq+%I{H=jG(C z4f6SIL3Iz5>D%8lt9Y34DSFKzUh94-58FRtdf>enc+EjA@Y&ZPLHk!^(lgKK2GgT} z;!JER3wy|+G22TK#MqNp^}GTfsY<~OXyTPjXhHbk3Bozwd4||gxNWVVwd6@%<;@=y zTFXLPRVBj^J|39XZUhE&@a#ZR88fCyIEDOiZBu49}cCI7{#wSAEs# z+eDKe*gpqw2@az50|6e^rT3o;UunlEj2uRNk6`MIlP9gtMdrr@7HB#O6DykHgG}kCE&SrdaTu=W=uVd2^!L2<8D_Xue6)7xOGvygU}39vkHSC`$GO!{dH?PoxPqW#^? zZ)M{t??p0)euBMKUjli%>_LE3gf|!pDPj&FmO-NThzUt+DWv&$3p{&>ev}MU3=FEF zi>Trx-sYOx!9q+K;L=N-qdf6t8;wyGJ^3V5*Inio9!i{SfqXzR+TDhdaG^yuS&XlR zPj*cRy%QF?H53MjRpCN%NXs&uQmUT7irBYO>$pfVo-}U|=H5oxm(Re6FM{|Wz_^nM zm2r=>9@YiL07t_^Hm2~C&@fS7Er@zHhA^d95F}AZ?!iwt0?=ik8N<=b4XpV#{%*=d zs&2M9;taw&O-3avLdV9Gq^%l^9mq$QO{^qjpfkeYYWa8Ty43IK)m(uF3 z_6~IzloKS?m5UqOD28lY8%&%+(F+$*Ek0TvNw6}JpOTW+ASrf2j;W>j4xAyC`WzpY zdCz-xZ(7>aj8ZMs>4zJqJbq(K5)5ruiYN|uLfRWy)&bJUHx6xUfeQ3|+IO#yYW*>D z5tfKpRvAjYRz%B`xr|y2n}$(Yxc{pfX zI^NAg*U#%T1Pf4%Iz$K;WCQftB|RuJFgVU+yI0O$2@&{Lj5B1fIcJ9FSPcSk@iN8T z0EZz8of0F*Ya!PgTIUVxq;`d_y})#~dLme^#P~Hupfh=KeLP+`zV}RIIL%uwFSjPz zz-U|%l{6FtVJ~}gHtLWl@_Ng%#Z)j9g^r_pN0gkexK`TMMoS^+H#6KX&WnJL%@+af zH$fa*F<{(L>H&w^M&Z~qo3>(I+4ei`9&dreZH)oQPPH^^ZuA%zi^@svm(J&rxu0?k z6R4lMlG2jB5R+}*I}CoG2cy}~6bU^7{~xnLjK0A>;BOO_W?48Se-BhA7pK$C9hl?{ z)(M&-k_BI$mRm!4{VOn1lm8~u> z+&e~Abw?w$$w$3(*~845+4k3`EC{W^#SSzjSkejknU@biTZLO@dlZF^7uJwq_|(KD z1{uYaPD<5^1TIH%c`kCe9xF-_yhAyXBNWm{T50SmI9=V*=p)ML`dGTFL!}H@>1LGB zL1bd9Fhi)qUdb3y2)Gj+BXpHbQ_bWVpjz?(Hh`Jd!waf{p$u~ohP(@)h)4Q0t><`chBl_`$l^*?5eUB{OuFLK1J# z5HO8LP^mqo%b}R4jSkZ8Q%Y)M3smgaQFtYlC`qp;b*EtQ5Ym}@yfJGM1Pajuii`;D ztwIe@fTxuy%RYcxr+0g16J%>A5oZ(@6vHryQGBWpJcQ$A6?eGMd8hAL*j{RcDC?Ho zx%l8ma*}77xlD&yh}3a&eF6xHGHcAj6KB?`$_j)SSY+(J(3C0-&8D4__A!KJU5gn3 z)roauM4>~|ZS27L*Wm|oT(!R0tU9gE23B{(@1u3Iq4ly{Pgkk~>R4hX3f&t9IM{VJ z#R3IlCtzh(Zu~m-=R`w679sByKwa&agGam*!ryp5ryUTK6LC0T!l4+ZJ8Lu~`%T4@_c(5; zYDq$(yv+kNmA+ujGs=YxyyE0Wkado7S;?dnip{Wz3y*9V`rN{tc!=>3z(kpgacmHB zJQ%Lc!)YnCwi>4g##e^gakk=I3g@_FmN`CBHGcS64@5dLR!L44l~z?iSo!*Mljd|Kj|L?j%W_~)P4&)Ys8zs6(qM|F>8}@(QjXqm*rN`s zc<(vn2t?6w91yKwf*Xo)keS`3s#QZJdmuR3gal6*Thy{yDm8_2^T~-SjEl+!yVzdI zc@RTu9VaW?76y=Z9SR5Ka1iMngGF+nQXWh@;a?-mtLGUrKcUe?#ccp^XSEc!opEyz z!Gto%OIn3^Zb)5YC&J5N;}_(vVr%l^Fv|Bx7s!O}YmLzQg1hKwh_oKw@YDgN6?UNu zHFX&ofw=5gcf~{Rke}KunFq92v=Nvgf%LD6oRR-Xxo|Qw&6shKELU!5%M2foW@hq5iBFvK6j;x= zpHlYPL3q~WGs2Kbs^d&WogCL&a-got7N@u|O2x*}IF_L5PSwJ*Lx5a3>8Wc7A@%sT zCOX4vvq6q0*N@uz+`;W_xb;|ST3CYdQXZ(NJqf?yf#O03>u?xWt1q999O4qd3RHG(o^-q=OyE1gxe# zBlGXf0!(sdZ;7}QYWY2@RNr{@BhKHa2X~Yk(RZglRlh zY7?^)N^&9cgJ2gQqdR=D9(N<4m&(h=^5U&_QS}OxLFMiUgQ1USG>ftam5yW-p zU5#_vFsNf+wL_S)6K@k{x1ZW86jB7Y%&e5k@SwI=eI$#ej~G|&dmo7%fdaV)F&O#A+b2x*xM_j@v9>w&s^{L?6w^2v#U#XQ%_1l# zkf6g^*{zhv_UtqWAvoO_PZA$j&oR<~``I%G)IO;^m{5^vz+?bzy5pD#!jQTvA74|j zyQCHC>Oi{}&?*8I#BEA7R>$tsk;B}(s_#tGt~+^_K_<3&omUoRy^et3i3(Au`1rQ>VpjwWi45Qh25}~e7>G0G4smv+N2QH3*+pq zjuh~-FxZNX zlp!C@MX~L-(z?LJ-csi>p-~4`uSmp)HezBq!gj=@5R?FENem5;6PSf^rM2O3f!|uY zBhqyJ0Z)UFY%gHG19xE`)iE!C_&(Cyi)~ z(EPI+arXfR+pR<~ySH^;3Z)X&K9LJdAjAOXJT2OwQ1!08^)OcIXCQ1*>Sj3o-fp3^^gj!q~Sa@=; z1nCqeavMlW(GGJ#11xZ`WP8itq*K;0TnV29%4-uYT^!_?2x>`&D0Uw5GKBsXY9{Z=eTzJrB1!QqcavZIkn^kVjDx&Y= z5}-+8POEVwm0l!pfMrq}3mm$#4H_IsihENavGAQxks_E0t#t9(rJ2pbY>Hk-18h2Z zAg+#!A?8v^DE)j?;I^PC0V9KxWLs!TOmycy9;`_cY!pWXIUB_5CKf%Sx>YOSEmpGC zVQXUzG;6G8z!Oy=sjwFUi7JC$72WMkCIrB*tP0XI89g6>sXG77?9y?|(2>860KBjIUj7@i< zOau>8t2qqtp^U{~KXFd4k48BFWSr>G9ejGdb3$hGquN5ITQ!bE=jv#e2mY~I;b3)SP(AGXFob+4!lx|w> z-65Uwu}{e2>2i{|x=N+AYfUoZbernk&XgG_cqbvKE8II6L$7a{euBwC9FSR<*8$X( z1%Z^F2l~gIqWowmj~rojehGP`MtyB6CdTRL4@5a0sa;#AraF?CLu3iqX%Nrl@z*%@ zmi4;cH)YywbPlFq&jEA>q%AQyQ0V_RdO~zo*OFfgB^7 zMyG_6Qz(kVCpls;Hp!AwLpJazU8`g)%}t z49$+<@(`F?u%#7?San3#%D@?5aZ^aOW88Bw4dKwkAlx!kFuZg!?XrC+szFfv>{84e z7%BjzostFZ)wUya@Ht@Qn)p^^C2Gv?7n{)ndU%l#7U4j#3tC5fQ!#X5KZG>hYu-Oa(#e4?5 zoe^4A8W~v;&y)o*RhBv!frRXl^tk;bUoHWC0t7HGA!gJ@KVMqfptYqqJ7Z#$6E!kf zVvcR&&Tg`r{c@d0AZHQm)9p8@0x(TGo3yXYQN>|@wE zPD9UdtIm!Zy8*~*rziutRFDs8R_413vEjIujCl3499g#|+X$>;fRQn!f?l78=fZ$C_N^&#N#?yhKi3C=%1MH16AcMLQ zD8u1idN|*&q8Ep=#MMfcCqSOXTQ4H#HUdemLKP*WdYh2wlILxOtzUXZGoSjk z`e$DdSCVzwPfCJ$B;QM;tBf8)_NAXaf1uP-=ASaD7>>~zfbw4Z(3jJX9 zEz0dgLNl;y4!T(;`gSBd{_$i>8Jm^n-CTpX zq%MWAs*{y@tQtrGLK+gVUlrL78`9hh4l<00JOP_LU|98cN_|;a>pf?F(A(Ncx91o> z6vY^?0UcTD+y^O}$+k`-s_jg{!SRnw;di%F$L zK-&7@YTbj8zyLwuSw>b!Fho&d%@-lK8j_S~n#}Da7m8fGaW)3>Te*XEt>hf`R&KF; z#Kl4vI89Za%x*2sRy9!ulnvTop$iWYKgHr0;~`;I$IRj)x&f3IR9X&f_A?F5g>#Od z6gam=2XEEuTQbq?nN#4faN2_j7je-%DMDp+gQgL4_@@Tzn*D~CH;dAUfIu=kCQ_hZ zau^76FzDXqyzPJwET=*5qq~b$ihh(eA|XLxLm9rtEi1uFz51|W>qQW^ zgQa1;=v5jWu*Q`U^k`H0jw zt+mI{j3|6Vm+CRd`67%h3Wrw-dg0JOYh72ELb+5IWW$eMdKri+SYx#s+FtRY@2nMR z8sz=#9$&4wVzeVrf_)gyI8oFRe7`^+c1ClOns9^(7O9~}PUZC>OxW_Wu;;Ag-@bq! zzNzg8U#0RzM+mJ(DeF2yD{H5Jqj6Q9~FWV*8+T(z`Uu%<CutoqX$y^5I9(ykr;#+5y*@M zYssCQfV$AsXI)XeZhl%Qd^a?LY>IE&ZUu%IAhDz-)2QX=ycaU%UrMl8!ZY8Q9__%B z>Ja%thWt`-5({SM*Vvd3eiFly&eylIHOJJx4`afM06au*^hF^ji+ zvpAhq=(t-K9g0oVzI&dh$1NJe&B6u9;3}emol=d&ef30_(srn{$;_DHNlc%|TpT)^ zi<*JC1cKd^Hy{AQ6{#xa(jbYbnV6&IrB90aXr3|0B9Yu48>=34(xfm!JUrWapRg5p zDmYQqOCa>Aa7w1fj+T0H{Js_-j+==mMBc4Z3L!lJFNpe{Ynrq@=;!+eF4?j4!c7*bge$QC)XkLEgxo0 z5REIYx)G1+uW8^M0fnv#ws#}}H<%W<$+NySZImu5GSk{u#%^SICIi}mKWj+9 zF@FbCGA3J%&6CO&^M6!zwU;gBLzlu^)~d;LC)jwM(D`wSpmy7NdaNMyTe2U%B&^u< zda~D&jSQMC4Oh`q0vYfw52Knsrf_UfsD@ZS1&N)&f=XB-1p&SP!m zR)a0D1-MQco@IfIykcH9UR|Ik)9rac!o!n!Yr2s!9vASIPRU-aygpWExFg+q^IY?Z zS^!X~me-BjBnLs$934=`&jsyit{p&2)-D>f$X&dAEaB=Ouu%SD0u>Yx5zrE;woa6X z03&U_T}3mZ*lf)tRoMa%y?Ne+y5#uY&ob)g;3fcEphfwGB$A_j&i&E41*%1xA`J_u zwR2r8w%Vq}M7x-Tr+P39Tt*3)?q_8g#%D2QB8Z>ly+uB(}x@{xmDzfhCTVR2DzGz9E`7;%dr2!{N3 zn8Jv7V6K6E!ANK8JQI(T^EXhELXxXHLz0K+(9W%L`MG*w%1HcwL-HM=9=?WFe>0qN57^5&%W<@+B1$Vxj z1Gxz?tBEy_DO>`^7*;|Z89j#}th-`Uw>+C+jw9*W5@R8eBV7n6sT*|WOXNqQEJE37 zcy8|S;E^l`>|l$`QaE# z5P=?A7<6dD2U!ga{9+Sl;E>m)bD4{q7#0w-B+%m?iy23dfWzD;vH=AkGXz>1NBi0* zW~nSzrRF4GJezhd1Z6$2g<->i1TA|;vTv|$V%J(WPR**GAbE#?gdoj*1tAia4O9x^ zyu(nB4e3e{HWZ7dqE}}6P6c92oqM9hdG=vKXdE40JdpX*N&u5>C_ZnS=?UglmQ}}? zh9I(635NSJ-uus82BIyIujsZrpq@6q>iKH2C5&EoiBTatmjS{Y3W}V8(j77BU4tfy zO>$qD8L#L$CP@{fOv>!^-Yqc|utX1SRaZ4lo;(CR`jaw%wnI`xCOS2q{bQuO)Uc#Y z7M21Vqta*&Ynm{2YKZ<>$T1AkNc2t&g@p=OjzCaAU<((mEklfB9A!W)vA{Cyd{br{ zPbUki@F9u8wkOV4?MhBbnQnEQ14n;OU zHGG$(mbSP4<=3deb0vqMHavRE;jmJ`TW1bJ5pr%31D9mQE~o zjmYJJHif!&{F!92a)K2{JEVCyEt1zU%d(p#;!IPox*NX@E7-BHX+}* zIIXuklo4=41U=XaMUriAR~*7*GI3lF7wH7tOA1%g-W|$3wl|*kLYPvpl&II6U#(Sw z{O8%$0pQIljj@II7z2oSPsJcY=5lWjBO%Wfk_&r2PF+!FoQHV z!upfSU>v#OLVRa?Qs$9D+*uvzz9?^Ne6;Fp>xdJI?=Gwc<6gaELU2>K4<(+pduU1{ zb)_*AHNvVJf<>0&06@vsRrzb)qvfe5m4s;163a7$E1NNKEZ*~F_ui~9Um<~p6%Vx4 zn1$8Tt>iJA)p8FPrbzWQtFncJ6d?jbK$c$bv^gwR+kk1KD%bh)4+*jSlTbeh-HLR!x@iNP?? zQM{*0mwi%UyWu52git(j$hMqMm^wQF|sY@+ zml|R;*QOyky0H|k>^jZT`6|*W_c%iWzDXuj5X|SkSv!T+wuMo!X(_koLaLaG9d=V| zHXVd{l0Zgfe%8ZuOv@xDf}{+6Nj@}RImp`z9RiSjV03E*Jl<6@_gA8Z1-qr0p4Xj7 zIR3Do2vn{nI1DuU*qri|I;r@|9NQKk7si7ZxoIw@eM0_f>VxpXh9VeH7eOA7;M7S~ zzt-KJnkwL_qLAl)B4GtQ!wY(eg}PG-7c50IHxwUo*bIYeF=tnYXKf1RJ0l}Aq+VoZ zA>Dw?9Y_)vNFzpDIv}jl zLO#G%Xtw5`Q@|6`1`41Y#D4?AUQoPz{N8r4PJ;(SPhlU&_MuROrRqNtF&!gUCZ*SH3_R-qR-!*Ls@aNiF=5)s%X~2ae26ZRFQvlv zSRtcB6NWkkKp;{NiiM|#qbjQ|n-`}^RxN&k3LGpCXJ?=zTWC(=HrYOlsBxV#-+kf_ zU+78P;p_64Z-*UTy+PG})r1LbXUq4JFha|^3g&?V6LS2&XEH%C+^_~&rf#Jn$-N1} zM1s*n6snX;kT`KzrHO=Xy_la6(;>Z7BrwkyMJ=aHGGrT%ISQiz0t6&+-cvC#?%O)o z!XOdsEK2w!KYecWp*X2!M+T)rKz*isx8rmomD|aqyY0Wy&r6t&?e9 zf>h>aj}Ry+m2@%P;}@tii;imPz0sMUM*1k=$^&+y4YFM)bg6B?6*WiP$_rriDD6n~ zTR8kXx2&KU>j5?fNQuFs9Fx@KlHmwlfvM#!ycQB2rpu zXhJ#Lqpl?k=awmytdcBbF$l}rSU(Wr+4r3JgB8xa`U1_e2%#xHAaG&tpGwr1k{S*% zdKBtwSS~?>6rv-QuvtOF<3K0GQuz-(q7cKKo2?O^{lODdE zo_bW^Ei=|iJv+d=vk7PhZr2XxI<=k;U3rFrObduuWy++fY>zvs4v38@#SDPk7Y=L9BFEcPDW+k-Y-!` zE@Tiqj#YD7IvsFAvh;{u5gW#??7-ll{c8pq!=7)RYoqIQ`-V<-eH5fXh53h%u}GCO zlVZ3HR5}@esXK|W>JDvHi@gWkmse1AJy#4raWOtC;RN=?sss2}X)@|kn?nmoOUps! zBRlp)gS}wt81~ck_Z*wkn7`dJqIj!a>M>F!RM0z~;QiW)*r?j8>wvx`Ilp zV>D zN+p1_*)A~n5|hCCgd24Q6|-=?1JblmS9JWfRF!h~EU+o!LKcrrvUpdWfCHPE0oc9? z4D=c^2$2Iw^t$%>$%3%EmQ+?cWCuqzgG^_2lcOIzxXwc$qgzKx+^y!1yL(Aw6lP{- z(mvop%VDB-BH0#FlM6a75#Z42J@Nqrhh`Tfw9+6rqJuutbH6G5@ysqYUkg|+E08`h z(Sc$EsFx9?K0Qlaf@q#@5v(B#@hz9aLb47jTR}8hVafx~f)!r6ww+-;8?LM*yVVX} ze({H^qf`}7gOtm1cdjj`#k__MXT2`rE>vYaG1J)CB=M%M<`aOkt zlQ`CKs4Oo?OE_V73tN7>hcCh48uGk8^}Qby9McnveY|Amvym$^q-EbO-QeOmu#C8h z7A25chnrjFNbe>g?jI}8__n_KuNN%B)88T6quYOed8hv10YHWXa^mi4jV5BLk7wSI0Zu)tl^vHZLilFrwL?W zg)*^1l-#?cB$T*T+RFhH!Nc-sS5+k7AWhFJ5SWGtIiMja3^Myxc zWQ_3)gkvKlfvA>)58rYZ(Y9fZhP21$FS6+V5deXaD?<<*2?&u9x0LkIG|FzN1_z6~ zL$;G1(!%l>uUo~5)uGvJo{yz{-ecFNE;YOib`K0#V{aN}`UQIzTPRpqu+@;lay32J z5}#?gQvnh4HDsNPf{+!u!;EbX^-tGHN>!30{DRYxmj*mo?5gCa3RzO7HZn2 zw|HOg0P^hg=eWY4IGgdJk}kg0rJ*Nr0xlLiR#Mw37u@IF{MFxV>v+r7?dSKMyY0Jo z`T6s&E`Im@ckSDn_XQDw`u<=C>Pu$^F=S{!gD8SE8(EuM%S}{%A|{{PkiIRT((LP)eaKNjC%*u4Bcdx73LE?`{DZh8{`u z$gq@^nv2LN;0U|EEG!y$MNj2kPatR4Gkj}E>C&+Ra zZj-W0qIZ1vYE(h=KcEyVZ9E4YL1Uav@;|Sp(uCtz3Bet#!%Vb}{qh!j!X+>@mPt9t}qA)P@@-AUZ2J+*Qg$!Tddl54XitHW}48n%=lzRM-QE$s^exu$DW!gYl*+8qz z;e`(vQMys2FBfg+ss$8vhp&FyJnc&|IzwP~-SPn#fKmZag&?tjCQSP^m0fM`#0Hr} z4Ji+Ihh##Lov5~Ez5~RFl*8RKvH;PcTzmw<`Fz!K(ZvCDv9}U_O%j(RjyY!CcbMU>Y>(Byd+ZV8ma*GCI%pv3=7Vm;`0KiA@r$=pe zzKL=cmz4vg7#VOmK_6P7=g}z0Q+Oi0NFW1>W;WmeML@d0Ko=wuJBD!@7=gkf3t%K7 z3{re?2SoU+Wofn+DvsGNVkB>xC6=(R#9%T%GFDXstpbnMnSV*&JhUNLldS`DCV7(R zxVxl^f)nd)yCvx34FmB&Jfw&ShX{B;!5pwCCqm(a5dI?(`|+Q=+MXXn-&nfLy*Y2h?V)1mnanLBW=2{Q#YVqSWY9 zE6lzeT=6Az36@#WJt;LF(D8*}5O{tJ5gk&VJkih)?vsmuan}jpU=~`lMXo};5nCdu zjrTEARcPp9*mmhXv|zqP=Kx&;rQqe2_f)P(L=21RWuA%xfG>8bTOAR>7GHx^dXE+{ z_EYYFP(V%Vst_%62?X)CiraCqrR+Mo0z+`fN+M%n$5~Jcfl86=)x`q+9F%?}qrQY& zOBm_NQ+#XR)#~R!>XK*;6*^8IVrU~7$T0cwU+W`SCV_BkgKv^ zEc_egKFSKo3lV8OL_yC?mGv~o)@mDdR5L+l{oFKy#o)ibMjwOR(as>*A9;cmvJs}C zXM4o+%(}Hq&};Ni7^5tZq0qOrVB#0nZ4KHUd~CWag!3<&vS(B#(31^(yMDqt{VIVK zmDK4`vrqGJ$RgC0MdT*QR#myBS=8Ou5MLHS0dy%G>ebI4WS={RLL8M?wl6+MFC87| zmUS7+NshNS1OaCee9$RD5EJc@(;-p{MudEP$X2d77RVdG= za6IwKwuY=D<7{>V*H}A|sDZ-KrUI3EaI7W?VRcbX8((VULN378mr zBV^G(yG=liTaS^F#L>iY#j)_4F`tL#-^46zowwGN)kKlz%EGFvzpgo$y$KI zPq@7ZE_Ua~cO%mMT-I*ohn8U(b@o||`G9!n!%FGB&MIaSJRrzlNf0j{`9s11eC80A z3zR-Z)ln8yI`b%eFV2BbL>$MWX&Vayq@mVIKp+M8VYO!~5}1G#4=M!*Uv}e(BbC_- zAubhzcVg;c%|@cd4X(5|g!1(;l<#!=$p*^CZMP=bt94+40$~iv)xJ6w_7jk$ zc8umA6FCDvuLE39qz;W!H{0qISMd9?<&_wGs7kL;x*i z6>#*?+<_)Vyt_f{nbMR#Kz1ZvOc%xm1AurTa3ZM4Mwzw{u&{uXQYC^jDk?JsYO@Gk z{$Q{|?#nVsFc}+Rk?e=E3N~i&G%lO*+I?yv=sUh|h(i>B zsDhSIkQik45x#~FfQJE@1fj*<&i8P^n_b+Q+tYG=np|pit;e5TeciXO-rT+Ktnt48 zl!-6zaFT>$B3w}_NGKtSzo7qgyvQ22L;h16dXIYYRy*@oOc7!nMg=)Chqe2(iG+;I zJwyP}1|7ZH9T2eE)3%HV2_vwlkd%s~H-{McQecYPGitdZT!e8UEz*(UZf{>3&$oO* zxG#su*Tcu}z9<12kTLS=2{O>ON(X>%x7zpir|w)PSx*wi1LGdQ(@w?vRXwG z`o1OfajeZy;Mrh`kR+$4w@63D9|(Y+nYc6O&aV5h_MFd9qbuR& zPZkddrk!cBfvLdF9jJJhw>@bAgY_qYof9g=X4ZbTf^`(R03}fGI${Juf?9<%JudLq zX`Z{)QlAKHq+GY2l`e7ESzKscP1)4_Dl z)vipE3mKX`gnawKg~Uy8gqlU75Muq?RdLC8>)}1i-S~Z^Y3Hug#R9yii#R8c{VF1g zZ!l+=6%ppE^QN=dxIk`f(lYt+)>+FHeTsF&xFwYhO@jWF1F;L(a(wm*Y)PI8)7M-s z!I)oZ&9#RH`^od6x?uc5!>d?_1!Qg#x)?!>F&ZH6YGpi=w-V-hfuaHk!0RbI+Rp3f8oXg4W+1}vwA#w@^r zAwn0k9j|IhOi5ySfu1G9JB|`kOP^>sX3kBc_nMVX;=}z;I7cW*ItbII8;gkFyLCo2 z03x7K=XppR(WLD}=RlQ0fvIiJlF&_hgzmfYPnin1h>Kl13G$trX1aJU1#cFCQfUtP zVV9m9vDBj**G>(jY~oaJ?W-Z8+E%Nh;8{TMHnvPWDk;2BKU@0qL&`8iu0`e);EPBB z>fw|FHZ>C#nK^ZdGWI!OlPi|WIga;BU?XAi--Jj? z37(w9Z78KI@VIDDJfeScLR$_Zy*L&L=ga0UQE3D7BFeU&%(679mbeusu}4(A)VM2h z*cF@{gcVM5Ae_Qlf?)7S!sC~VRIfGd8u!0j;z(h!Fhj zpL6;Qa4ojLOB<+C0|S|Xa|m0oAEb=DK;=*64lSdO%%;Wlm$b=kR0+A>S_|wHC1OF@ zgb>9EhT+Cx`T&L}V<%z2mvHLZO)>0M`_VUarWy!j(m?^7oo1mCa_vhFv+1&qejifuUO$WNbHz8aOtcUQ@ zC#Dp3Qw3TmGu#IJ;o)029pX|dlf z@#3T7M3~2mg5Izl*xI$(OoAahY&vaoU^%Df(H2yooFf?(oZB3V9rA}}8{{8^k*I>5 zkRYgXhkQITg_X$c73^;%U|NO_y~*v(G3v)|jO9<>H&~K1Y?g5*A`gWVoPNP-e3BgW zD0mZL;rLIyd{I)D8VoTBqO<6x#jYkH~?tf|pHEzF|3Bu7)Ov(qz56&2z^^z?;K z3xywFTNhBd0YlG+_KN%@JBR|_1vT8(33_KUs*Gb{_&zuw0%G$<9IX%1G9khiAmg&> zGo0XF%w~zZM^rK3$OwY&BWb7$;A4RRLRnx?vGYLfssM(`2^^h2Iw5MF7-1%^h^sii z4znZSU`B<}+-VQout4M8;Wp+Z*0%S@cu0Qy4T7XdEf-AS#Bs76H_$!G=8B!9M28rzoTgmM zS_!}$U?dzkrHny?ox~t>QBoD<*O79Cd9jQVWbk>Ov`f~TV&F67B<{;@ZS4|K)oy|8 zVAsi;OCD;$h){_s2lJt3!iLjQQ0sH07V--cn8xNghMSw|#l*!O_F!Ntn>$K6W)-eH zKaP4PRa93`?p5PW9b_^V02rvW0O1A`AA#od^eM)xmPcNJ?yz#Y0DFG4YEIbX#G6p7 zS;-3oqvJt}AQ}%a72wX|a|sm5vLmjIgkKipO_nTRLwpokY_ndlv#}Pw3Q3jw!VeO0 zu$oZR8Gehi9u)%QdBrHs&smZamBSt)GtmRba?tFqjPhXZ3b(K@8Lx#nGQ)-ds`Ny= zR|#>t`(ASJ#P!fmE_IH1q>ShYLj~8tX0$59@|x8`nJ=COczQsLI2HAD4FfXsPWrJ11GMHSMPik#Wz0kG~thH~R1J;X9pK}SGceWU}#Q8>bT!|y2Q zlrs@pT49Bb4Go_zJ$p!lT5Sfrf#7oaEl}y94WUu4Sw>DT zZJg;H0JACx?yVh_Q4~64KDnyoU&IoYE>Lqc7m=w9-0(hBWFGNajWfHor4XbmGbMp#ej7e-FK^4uRA)6q^#gQWtv5nK|YaoA`_ zf@m=m!Q0C^^`^8SJ;;qAjn9sv&!kI)9dcQB^D3g^o^0`PWazrj z+fW!E><1qUqvDQuQJZWqCf`bN8_*&O5=q-{>3t9gEaS6de8Vy@P++Fn0SSh~`->6? z(g;e}9}kfboX85=bF36q5wYjDne-hjA#7Cn1Qj?21fkQc+y_0hH!&+~kt;h4XuV&h z>Qd5G6ptsO@)&2Fe4t8CjZR1l(e<}DQAP23ywPs)j*0V%*#QLwsz{JTRVGq(V)OUF zKY2ofaxju>^pSMN_Gf*4Cn?ogiL_JNU2%LB1c6nr0MFEk?sC6wc6eSQBLxlofKxqleLC<|PZ{1oorGOR8$N4B9XkuM<)-f_v}+yWsNV5>zDhN+iyhZq7U5s>kQBwqo*>yT)=BW8DWm}OxQ z7e!ckk?=thIOz!y(tIy)7$jUt*x4m+TbpyZ@t`;eDw(+sEbkO8NL9<5-yYrS=!Ff? z5~ofR=-3$^n!ZS82NOYQaQvHCb)6>S6vW*NFw2Da*hn)iLlzHvogq{t)|~kGc8BXB z^)mbim5OrW;w~YggU&(Edci7<5*t_ut$EgCGnlY#2C)ttmNHJ&k6UMGp3hkY2Rr$t zZ+Z3*!ND01X-M?-`ucTM03b|5R0@^uBI$CVJpSyX8qoC5AxD}eMFWo|1B(tp(Rs+) zr=JM&QA{>ws8<5(*27p*7Lj+N{rN0)Y{IO$tc}IT(pLO%g;)ejJ#7u-zZqOR09=qw z&rZ^V548{EkwUXWP*mB!T}h;;O!OaF0?>2lXZT7lvZo18brmHK?Y~a#*r3X0BR9-m zys(0|C0vsNH&_UPC-@5E?iLcOZqd#TkPlfy?c6mvqy+llZbEr}6(lbqhYtktFGenM zAg27HEA555>yv`ju?vlJ92%V{QHauvQyQhC-DP6!Nwfs)IhJ+5u9-Zl$O8%pI&=`P7>UqLQ0iG~gY-oWy^3cS}P1#@CNx!_g; z?gq&>8&&ifY)-dVtIxYTTKX)a#5dDyJC>b`>c0VqXzEO%7o|vZg%nm06+NA^5LQqi zY1rvYj8dm1gM>R095$lq=Y^S_i)bzVf$JWkD92ard z;N#M023^VIAS_i$=0su{h+uq=Q2~y|=c)qVZQZnYbX&_D(J!E*q_!Y{6K7tx39d~> zr$AzoI<92~N>y>mFBtifHWEXxNo|IiK;jo%G#gWs3DFO$p2C_a`r_`aC%G+E8nE$! znkx(3wbsktj3oh-!gxgJtRRN&!J4og0&rO%DPWx1fhPLyYl@Bz&siRD%SVo+!1COT z_H9>xZB;McE^BpsxUB)6lc&oB3{!qA8IiWI2n2`13^t06#w-P5A2S@0sM6mCF!p&h z%DVUC`>qZB-<|i4et!IW^WOK@p8WOqI_=~9l1jhjB&4eO_ut+*@$1jW9z1`$Z|}R; zjr;5KulM}**In^KoZw3Af!I5)ZB1$ze=S$`-@}m8!rx#^mfft@D1#wf?umrm8YCSEl_&2Qy}GiZgnPh5}l#j?vMRMXd@SDx+ zh_kjs#bl1|M_?_mJd-iAeTPOpfk|IP1NIFyIuyPRnPmBGep`1CQOZTr03-+5iG=45 z5}Xwkk&3V@Kz zMn=R6Y{H+$hn-iigW)25`7p{}d&%k$oGiKP&i5WZYN!>Fz?Ka zrc?|UUq?-MhSvaj^xlIfwHwc78JKd+cfjB`48jDV_m}AeCX$!n8<@gTqkNSH$yCje z328!#65LP##k`i64y>Nez^P<0!9zK>=hK$*3maoZvPPiNf}pxHi6;PLT}<7r;Yre9 zUD1$9dP5;B(gL6ieR?vqUE+O8u$wli&M21CiKm-)R=wP`$KHIh;j5w@R2|$#GOFl@i7e6(usJ>&VkQGa8_C`AqUchyXxpl#*IFldrVkOFTLzY=suHCxsPShE;yMBYx*bs2_#dyQQ1QH-dMYMRt7tX2PHPZOU!tq9+=X9l{ zIZXo0mheVIz`PTGQS_&hIb$MH3(wHf$;>lo(5gk=A)ShH9-Imm#nMZzMBg3_ZL}PN zTA1K@;^j%xK%E#{eu?+Hu}nPAwrC~qDy7M~728Z;t-P}{AYW2-G5K=>Oc&8gAk7F} zymx-U#a!J+Bqpw$^1!qR6B}w4=%VyKQxJF_dl$1ne4WZ7Vv*{QCTWz3SB%EF<16kX z=oZZbfe(F=TPwjn+qYNCBxzA6L^;}5>Ik%J_84kmQs$%hV%xynUu)-$JDnJ$QJr5Y zDN4$n1V@{pj9l)BDj$vjMIbsJd!oBBJBn9Jf~k>8GHj3;GGlEv`?G?QNG{eaCLb99 zYD==A#?YKm23SF?TH5NNb-kkT-X3bWJI5 zLgz*p;jo8?ed3kTVVY%08XRm?14HXymS+6S8YBts!7)i9>X8EywZPrsL&s@DGnxsr zWcXprNdreH5Y4bN|KtK`qLZ>NJ zrI}COnEQ^|IvCM49dozTeJXPlv% zO57q1DFj~_7=;Mcg5zG@o_f#|T9Tex*9VSX8ATY!T)#!MB#x8C+p+x^5budlJaBcE zqlF#p!j5`Srej6nhT}XF>X?Z$ZCbmUKP^tyhBB&wjEd>zC`+tgMI%Up2Y6M}=#YRf zAv&MPLKXscEvqCo))5BAbiCJid@uvcwrn??Ds(uOJ{2r7IU2)POa_}b)ywe?R3B$$ zXdYc*TJx{C<9*8^Bs2=}QdpeLBG}4^GpBoEG=crJ5xL%Kfwmq4#ZDkJRWPSi=oup2 zu@5VYyWT*(5Qcb!WP-;u!hYkKHsNH}laiU4j6k11B4E{kWq)mhw!cOQtHVh2tY|pA zIBG>tVZ7<}cVC`O1kyyb!-*y5t`wf7=2ee%>x6?P`!8gDa{W$!zbC2jTITj(+TETGbS@SBci@~j0XcqfS;4uL1a1$ zmW9%ul|`U95T?LYg^p(0G5Wj&oDwGNBp9Q7m zECRhZ(b5zvSiW&0q$0_;HS|k`V%NC19T4eURr}cnbQ(U1#%d$IB_CTAH(}tmun-=V zNg*XndLUXbmjcak)?(Z%ZpJ)5m&8b|CfTREZl#w2;YacXpcruNGt6F@2$_iuw8P?v z-))D76bU$EIN}(`v%Jl|U<-2L7`!3KLTX%gIy92YsprD1jRNS^LB1gypfqDwt2^<% z&M(-Gkbwu0;MEGxm^@G|lFU(I5CE-$1h;}L4aSS;q_&~#Z7a+M9Z1x1iJE%kBK2W{ z&rdveW=4#cgoQDYaib*rvzuU}F}cEUOBjYwAVjbz%T#VnaJ=BXuFxR|j%!mg^YJ@@ zfR7l}GoKSZ72;5VXST1#_Q?G@wgqjQ>@?Xs;j08o;6+7|K(GjYW6OL}s#9($H3HxSHjT3=<4E0Z8o zZwo72JS{QZbKx8X7BW%qJfN|Yi~?z2H$+Y%LsKnSmE*V)do={6&$E|{-tV!oqvgb4 zIkM@3w?q^Osykz4!d)=S@DS-}*=36>E@NOr5g=2mpmu41aAe9DFob!Kgj&oG#n}d+ zEv{=ieYpIZ{`#QprM7_zZ+?eOhVWG4jD;)RRSSr~7(|UoGam7L;WfS3oh8<0Ayli^ ztdmnO493WcG-+oSux=_`_(6!^$~;n((vUROYNR!@%o2@7H9aA8GgjV5^O7(rcV zYC;a>wz8agVwnIVL#Be*Mv!QwmAiemTk-)2*VwU;4K^MHnk`|jNj>OfOX$&@q>x0# zH)4y%&@2^+P3^{IcMwb?(YIy=p)wT zEofr%TTi>s?c#(f6*A%mY-J5M+{Ge@6=06xHmbrG>~Ym0P%PB&rvgMEV2i=t1m4&~ zVlEh(OU6r-v_Dwt*_mEf!rr#q%}WVkSsr6c8`1Cyac~32wO%{ZD#N|u)B=?wFM-k6 zd0n`ApoC7XzkR3|T;|-6Yh#7gS3+(CYMwe0$vh_XiQBFxR6*934BP72fV$6ljwozspxhIOUfu>oZW00}3`@urtZ z6DEl0iF2J3;zPzz%dQ;yq@v0QK}U7F%vnMSC!5^NB$TT(s)7Vq$(|t^!TZKUEP8z} zb88<3DweyknapjXA#m>Hyb(62(ln{N5J&d$-{R}GITnS;t{h$v@Cwjmo$FqBV!agBc775$kdXjR z&S1d^-jwPwlHwsfm-Y}%kAXW0m_yrGh#+Ga*+!x-wUX>vM~7RtFX;mNjDY?Zvw^h) zG>!*tA;qP1-ltE<2(jnB$8-?5bpR3}o6q6JMSlPS!2}3hNs4lyXI)VB`!^j;3`oq} z*mBbLNwYOP8t-|gZ&C^2P`pGH0;tb&dk8Vh73%=}nD{=%C4gJPKxM5XAvBTE20((O zHzEg%VZVWq%i>G(obg4T=sfWcwfZ`=g+^6TLK;AzI+t}z#N&jbLJH=VOBfIz zO@=;No3x)?GbJBojr=qMrgiXxgpgto1q&%55KzBS(+KxwF~ATUfqert^*kWGtwsqU zJhrCW)}rLJpfzdngeQ#nxL&>{6%-Ggo`QLR2GS6@EmLs%YCgTeN^Pn_Y*LzCit?V+ z(n9^Ao(h8-shM4O1}$TMA@%kXxn5ys0%6Cpq(M;@a|aRa`Pgw`$OtGn+uW}EpeO76R_b|S0{tux`{`(WK0tV5va^MqiEt*oniK`Lmzo5ot;9saX1(t zl66m_UuMPIZ@tM!o80*6k|$xP zP`@9e;AJK^REwQXubdIrTU+XZ48#KGn|9_S#T^#X+%C;X)^{yV*VXhGKy^iyKp@n$ zvxK-4q40ov8kt^et9M>DV1vbp*?`hcnc1DEWK@K{mnKGW`d%S;=!!JupiuH)HOke5xMbz`0RWsPk~L>!5M2kbU9*H-G_Xfws%9!Ue~^!|_)A3)o+E?h2zAp#_oyY!(s zA4Q|H6k6k$Lc#wv%)y(90pKC>(vO#D*tuO95A%gA^@Ym71y75Xx( zu>8d%bRLP19|3#_BhgCU-SY0`#6;~|3M8ah+chdRj;`o1pk!9(DgL@*Dg@&-%LZ;| z1R{H1V1njo?970w?Qw=n?_vWoYzwp0@^LU zX85Xiup&u>yCav8zK@Mxp5oOGK^YEMTTexi2Wt@bwz>DOPKJ?0=7#JxHiN< z#~Ff-jD}Z=fzL_uAfLS@tj_);?rTxv2ca*sNH;7hsF+>OhS%u;0MHXS?79~#FMJ1h zr3hkcl5NuA3Cxj+*JK<^p)zQ^_;PHy)Kw3F?w3vJ%K|(C(3VQ?qB1mSMVUNEDp>A) zyc|wBU@xo0B%|wtF}lK{!({TU2p2OPKS0G>a8r{y5?mmwBS;b;n*0Zn}zi#rq@>(B%mB6@m$3*9>gsPV*is6*>i8(yY`$y=EW zpaoUf!&{bVbaS1eIpqFPyWV z+9YObNkt^aS8-m55%$`A4k;YUR7AFWB!4E*)#aTC1W#EDSUVl0c!BVLn$LAF&D$h_I+o1;Yz1 z5NN@k5=qM#q;&080rr_(;B06(LGT(?ZA7Gz0Y1$kPAQksDenZBIfSaJN`QjT0mfBv z^jrPJU7pMZOQ7w!Wt8q%&JK}os5XhfRS=pPX^1^Iq$>gfMg@BDnl?VQ`6YhfWFS3Z zacx$}1&FL^z22wHd8G-iKxF1NMWt}>5CP4A4ngWta73ePZ z{Y`JSR0X{&E6sSix#D7W(u&_z5#6HO#R8@$=m&xMJy2H2Skv?ZQ^1!3ExkyLqo&p2 zCTd-TGm|wLOK5D^L0;WKDFPZLu6Aa~QZf|7^078kN|FuJG$wi=I6$wF!h(3XOaJx?u z&h#*Xp%E?vEmH&47gxkbdXBMDH|T^i&|WkVeZLXd(+|`|eR%k+VgQNb$%e6E#8YE? zmHH;k81rxi3ai-w!GZ~iCCdT>{OBalkPgw`nF&y4^_k`rKg1!SDIuNF( z2OM%oCn_<>25*ztS1VV2Vnv~=H$XVRK@lD#0U}Q#qH3R*qS@|KsY8*5g>mjnsvCsD z0j{~72P4aRF&MPCFLBk)1T z)o0Qo$I2uKA-Ma_&H_3v0&pO5K$76scL}i~EFsq*s4{?h_+hO*7y^2p(K~jm`)rAnPU+_v7Uy9x$$u9UWe8ls>a6Y@sD_}+ zNryr`jmm3d>@{5bDgLk*2Rww6SP_xNK}kU_yxXI(aoKG^Z&#J0rdg z#4qiTCAL&y9+iE@oe$h-sChf2kmmVUIHcmBZjj(-*S=L+nnObujJ}~IHDQ$y;tUYj z9`lDhhdUe0`uQnDaWPfc_}vWgLSM!OYe5awR3{!Lk4Er3v4qpV8?Onetw@B~MqQMH zvQ-({7i-gmV@a{(z!a}Zz16R&)y>F90*MnXG3B=Jf3DyEkJ;@Y~jXoinT zesy6g{3D~fpDQ>M3nLUl+z8W|>OG6oLXQLnmuMHv%Xks#(I#^8n<`g|YzanBcZF9tCZ-`Msj7BXd@CtT{GO>I?;>`tY;Z0txJ^ zAYE{70#okBVDVdY_{AKZ2HacOR)Lh9f2j|&=}{1`s$NKyF8+NrQz1RjDX4*nHoKQD zG1I`Ck-eZW3nI`MDu*3>Zl5Uz2ty+}m4OA68_&HQqlsxyymWF?m3&*y1egO7K+eRk zgR%t5x$PfyCi(JC(N7=4%dIfVa(LLMsH`J#Lr|IF!eHvmdmta86PYkN9n;~l#T`7Ha>|3#*+6PM$!pbg1-H{9+{sG=nm0O#q#B2X z2g9jb2-a04FbPXTl`AF_mX%Li^_I@2;|74z3~Q0CZXGZ_YOiVHdQslxIAjNz#_j(5 zERJm}IRkP{f%gmtm9oX>k6sv;SrwmaPp3S>eZle%Pq)>WXX^KCC{ zk16UZz}G;G<;{waA8aZ06)6iI#&2+2t^tu_$AF{T zR19WwGw)4>3kFf$91Gr7&Gr~Dh|U(qSLbDAtcsaJ zsA1M60*Als@>38M1hnN^akBO%t=@&RV5H4oX4 z;o+0vdnIyPCD2{(wX(UHw(^;-#vG0WvToE_sN)^^xCc+Kd&nxLE;FV~74DUUHVL3L zI(Z+k#@3Gl$!oJwBJb@>PK(z0Kwt_pK zOEsw)#|O~1X0%>-78i*Nj$^@VqFo^2(e~$;Wt>=z{?(igTZ#_eeEZvvee~^{ao_Xr zPu91s#rK{kZn^UJo%bJ{ZMIrqmy^IXNLQCI%7Bg}jpnw51f5+JvtN)j?8s<9^;U6J zrjfqf$J=P86S~+WU0Av?b^*I`Aq^O~rn7;X2^ovLmxDaGLT`Zt6ha;m2Z_cVjEeB7 zTq_ZK;=D`0EA0p#^civiPrF4VE&)u~u){DBkWQ(a#YDKmn<8Cg0Yi(`oF>`lRQNmv z(8dBGQ9x^+-cJjLrIJ;LlYCO+8C{5_L~{1`>+{~Be&YO#C`+OQ7y&BIRIEXS1B{JH z$KbSDN=R14Wt1J&6^P;Qcp-X+JDhWo@j7MSeop|0PPJx=cX%ttr-wu$06}n!qV)D1 zhlhIy4R!@E1}aY>iaWlOI?@vSxh{!ut7Jiex)W-MU=%W=BJq(N1@N+9NNvOqopF6- zAZKe7SFWkblH^wmZVyT7D9~yvh%W~2H&kIP)+U^Q&Y+lOkQ)qQ2;Ddf-;q*waTrwE zw6RSrP(_F!SO!EE$i-;8K90Wu+(;Tdql$`#Fcb!(9S1&2lt9XeAaa6zypwD9%BinX zmR_zUAy8R)(h#f+Q!nEx%gpubfLLAmeb>sEJ);yeH}|I^%`(aA>V2vdc__|QJXB^q zJoIE?X`*aU7cKz@ z@k&Y67tcaK08@G1p&rV$vSz3d-w-%ayNUc|v{~UTKTtq`b&xlpb3x6`XG0qlbpDQk zYLUyv3y9(j37NOU=IPK;9%9lcJ|$jttXWeE0RW76OaTL}q!B>}5a59VxiZ>f6$UG* zxQ(*Aszs2)BIfbdvM#ty<&-v#H$u^uIl%0oAG3`WHB8(aHG&C7xLP7aBtwT3NbWm| zplMe7ixygMq(ddgev%@Q19eC9?g+vxCpin$st?AH`Vb6(c|Du=nXiVZ0J69%{ADW@h)pn|JTkEWGHO{ANfd%s-ibEXnh z;a0CY6h3>cwmVM?EPXNaTU?RuUt8E?*|7o{ifH))U~uGd#AN=iE1=J+h&~Y!JRJl0 zEnlGTL>%74{`kwzp8|_EQOc?%4o;bY;Ib?N3|lh0*Qa^JMApM;8_+v|&;?57si3Zd z(c{a(>K#fdBqLFq_?uS&p@LawqZmxq+nIi;BcZ66X269dTz7_fK=4L)rXDVRor|#L zz-SyW?aWE3m_#5?d}%i*J`3?Eh2ean??;~q7`WF~);|NA454_gk;~k9JL)QhL@ULq zD=L@Uge%B|)Hz(D{&*r#ewEt5*98d(K~$4Jx}C?e{1?fbn9HoQ%-7y~LA2>3XA}jq z_NLNXM+QZCV&$d)>ZA!$m+!YN>STqW*5%P8Uy9dN)q=8adn3yIHOB14xO#FWoo`|$ zWk#45NoRsHrSaXTah~Q(5eYA*qF3M$6I_iaZ${e4Fg83GP_N3dPztqRpwN7g#^AXo z6>BCXH(-6mqIl0{G8LVVhwm z1s+azH@rU8HRujtE~uDO#=A~DDIbGZ5AS%8h$LIbiP#t>_CAK@>k&V7wgBar2`bZ` zWeo7CxZ#Kp_VDPv_)P``-|GqxG*RIPvS+h=X|igwwL>$YRp!vC{+R4H4xljxM zf@qbLiTofk!5MBU4ZH{98d#Mj|52u2l8XV|Q&?YAKu+XGyY zOuYM~{zB;&R(cR4P(lgIA69X;Y^;L-gK>$lSWLk@svIPauVq>kKL-vW5U_&S`%0W& zFW5v%7;k{#r8@{C?(Sg4wc|BovFUQa+fD6g`=I2XsUu^1Rgl028G<=#5RVkg9-L zeZ80g5WV}#wn@a6_Cw#bsCeMmOnip#l@P+C7qJP+nfjrS2d$JBiO3smPRokNBaL!@ z5Tqhmo)2}L6PYRfz?6PvL-#}aALf|}3{SbNEv`QZ_MQ!HCwV8XX!OWsGOJ-{Wl--g zdJ^j~gy64w^H>~QUE?k~nivnwOVc>>oDxKB!l>{~6@E9q>%iY1HtB{+goXfg{OF|oEX4eo$2@G#0uEeOucsy5Wkue>))3i4f^cupCiE zR39Ruz(i8+Z1XxkW|}FP0Dz6HnR1_n$3z20O6sCR>Qtpfb^FGmO^CgREQMXy=@Hd= z;_@V+t=G-(KX;t#-xJNhH-9dS<;L&lzhB+IIqRA2_n>Me`RIzz6i}E7oMXy}b&bgr z)aLM9tI=L5Xz~zSMC~ox)b`s}?sru&K*2(gCIu>SLf>&OEBPJ>9WlvXb4F6vd2Ua5=S7!qMq`^f<1N*F5P7F~n!QdscXg|$UmRO0)O zfxfjVBZ@ZqTn%%;mJxPFcKQ9>?24v~4*PAKG`Lq;kVR@57$+kMUELHaHHjc5@twZ6 za_Y3zwn@5+P_ck|CmSwz8EL>)IK7>*>np{;#`uDG*nVQ~S@GkW(z|GLt~&!w3-KV} zg9g^IfXx&%&WNl_SA|D`FE}KJA?-mi?Upi2ncLSFs+o*@if>7C+GZ3Yg7v4c83~X< z;F4hreU#o(ZJ0}UhP&fc0Yms{LI~~7g{7<5w0kMT3*jAJVWKU0qi_}ubiIMFp&y(; zaJ0#UIj)o95)=ExXid>IYysPXA(@_^Ja$)-_N6l8d!RW{2{?Kp6r_CpgQ}`7b~wVg zx$hr<;di1qwj0jE}xh;%C`p7WTc5l%H9IxAK3PzFS;@*yZ8 zDGW)CljxCorN%^#Yx>LJ;SB+23oJ;ob}xYrM_KA(ik&`z0r+t^Z9e<5!e!gq1fsKx zt-3k`;7FomP}-xY6*KOi0hNz7V9bP=n`YG!-sqBF%mv)<`i|mQBNsu zmGla7!`n@LG&;MBC0Ow2Hak9XCj7@-^n4n1i1}0-1k1AyW5$!9Tw>Rox*J9;WDLZa zEAzL{Kh7@3RGXlRGLwc*o}M0+TPK(aM~BA&gF|9=1K|N2ZidHHBMZjYpt=SautXTY zcBCGeO&OQkAcKP{Kp7PD9j4&Cv!RXP6wSaZEo7;+J2;0;R8_v`#)SmVf%D+n{mMab zARGFj}6L&9sgzOM2BI7Wq)uM_I+$Su}a?<9UA;2f44otE#DdBYzu;Ctq zdt-K9=(F5Eu&O%zgk1vPWCY5rTtAUON(k_A*jw82g>!aK%NPUCumL8bwwke)&VcN& zWfnbqJe!&}1Iosvi6Lm^vzIOUxrTs>;=#&|?*+9-L!YiWMZi1a$_LhLc5S5U|WEwp9MjXM=a z3!;#luo62(`21kSV0`u}i?N5OffNa|hcd8kTgk_#4IC{JP2qVy!vaQX^=ME&IkyzU zK(A9xNy-6>H&O>Nr>@cBiUY7vA4h={{GBt$sBd^FyL-QduLG1wt z8y-~N%tF%xNK)uQM79cnJMtyGzPnI_Qc~V1^%Eb?0wDFWH?TssC@2@8q^Lp_ke|4r z6&Mc>V<%3L+Sc`DG_H?ek@7-Fii3P}!#Lhs_~X;RrR!X0XZIUk|9IoJzc;PA z{pSA^0Dbr0Cqa96phNNJci}ZC#LvL6)mMgE+Hl`9QCYBz-T8*=8`KTon(>B+u6McE zA&24zRA4eJ>$qzeWPu@}aS@>hmXa_;A@TGr5OBtw4$M~ENY7KsUk`S_LKm`P7p&{^ zcPUh}iKyYZ?>W&H8=(Ao)Ic+Qls*^>xL`83c8DkCT_`eAm=#3CWV6X~78O0{s%w1% zeL&VtBnBMt0c8x70&N?b(E9#bmzz7bz??sL^r`_ej&KIHWLP}&^#`Mq^t_lt&$E?I zVb{(f?HJzAX)0w7gCh4_C@|9@>Zg*KPLLaptZVj+F@(*D1go^N4}=s~1_f-mSVTq! z%!aacs{sz4(H*+{z%Eh@p!1=M;((|}W&6xkG{h7-Cz3kY34hYG4&#f zg_q1x)(LDRS2ScxEDx%xQJitu_;|nw8_z58=A-j3fa$%Eq3b!t#xBNQFpB846)CB7 zp;`&^C>)iDYg0?#i$H|{tV4N$Tpw8M7=*=kLf%K|~( z;nIXI7ZI>>3kcR^8B-!d5OrN~wm)pufi+~B0I?QJGHpS5NG=>Y$!s*^fmF2u1Q~$M zVmt$@5*AM5MyXbG|?-zZahK^<_sZ6&`K0xcD;~hF;az6j5u&?^iFhcH*vDC zG`*6aWBRS4BUu%XSFsT%-O1HdJ0Eh#GI45DG*RcJv-bB(R@&o=x= zo~xirAVpe`4Zw*JhQ#tm>3s-8il(V zirRb5%H0;_823cjNWXweQaI~#DfFyGT3cySskT@Lf!OH&0Wgg44gz3M6Fg|*wZI7w z#3*^VWIW{@phGcH3XSZ$kQULT&RB0IixcF-qjzBTv6VnN8XIx@4ks^~KrI>sZ} zh3{*k#(Gg`%=v}#VSF$qBO}Ecm(g2yu)KgJt&k+rMrhBn$1wp9rekgJ6O5KSVUl1Z zBrXnzf-QrMyOamC^myTfwRskx!a*hQm4>7kbrj+PJERzIwY2VQVq~Z?ta2wb0)+bo zSnzjp_B8Je(Zs@y3q#;yqQ$P*$0^n^Kb zGi*TLy8PAUbyaGLZ5z8_)Hi08S;bKTLwz!W335>a6!V5HZjdFc{NOE}*vMf6vx5W) z=f%l+VqJSCPMYsaM7nl>+R#cbCXmY0I`}7L9O`e}0m5L!La|KnHg=K{8fknpTb)|? z`mk?g3Isb{IV64QT_gDXmgI)7HNeYozd=9^qLVFfA+$p#!?jIu`g}J<%z>PVe9t|j zHeiV?8(3Ub#<9;_+qxs;`vEtZHCRRG$t_unV^%lq_HE2INr_CLJ2e0YF%A^kTN;*e z=Qo$d%&!m2*EOIm=TEyisA^YgEF*A%HoV5`r{7chObGCzNFh!i3Rzc&AyYj!EAfV- zmx_E&%TE(wsi1VlO^-<~o0J)f5#Uavpo(GIaMplvP>_iU4bDyFeYvzfKTTre0*@^m z;?})U*61cW#o{XQ3zZ#i(<1H1l^v{O1H<%$ovluKI8&OFpxSxM*-8px41%WoUrM^P zK_ju$=ZMyr+p!bPZVpQfkf20$O|5B_$_$51_iV`b zs8gqSp9X(S-`r^T0yBzlI2x{Czhp@v;FAQl5(Po&a*#<}^@{FOrxxhz^!tyE%{v;_ z0HV2#9HF>t6wn8i!{l(Jw^zch6t9R2$MlGAm9{dw*cl}06MeT7FHKtOVzD$P;7O*u@|^?_Qqg$NZSv6*{B#k zwYXN%645;%FNQ{pvsyqf!XFZZeQ$l7x_a4kb=t`x_5nHk7`bl;))gdV%0^s_JYgbn zmHn}!tEFkOoF75aF>9fn zXHj~&BkFX~Kd^T*2Eq4`u+YI)&QOO}XH{-v)o+oP#u_?FvN{akwI?lV+&~%}klk1y z^7Q{gL^Ew-67tl%_l1nO3~<*lk8CR(3`B~EB{LsQykgU$l*{be?>f*Sl8uiiUe`)# z8o5yj52}P6;`WC<6=`4*QuRz7>7eZR!?e`>gR442TLgQ6YnJFO*yCsgW+Xn~HpTO4 zgBg7tE-=_A5J9w&5ID1I6ZK$Bo-ePb#52<>lvXGiS7RlL-K{}n zHOuH6Cx(?n~-M` zvt<#K#2-bm1!Y6Ew)XLy$<=;h5uma2hJy#Wjy@xOM#bXnCnKJ-d`Ha*H1Z^5_t4%-8svkp=||sxs~OgVwxKWUp^k1rY9&} zo;_Vdse%3C4r+ilZF>+Q0MJz`CGlDzyOT-@LGZ_!US#59gx7q zLAeQLC8L;80tRHGL-?jZ)P<%uPI4-YX!%>b?Qh}G-*+I~gOK#v8A0r$ zgjp~hZ=;d%QIeX5F7UfN#10%*wDWW|`p_3+E$)g9oixI5AScE`tr7xJ07@mBT2Br#mR%OZ}nGQJnNXW^!i8V+SEnnAvs) z6Dkib^@dumWoOKn(7dJ1qCjBpO42%3fjPww!6d8>Azunq+2E}_1MVXPhz+c%lud(~ za2W$)@73xdvdDB06w7{Og1Md+w{u{E8qh!b(GSyz;LNAxqNAKfW|td%jk z_)gsiPU4ijbsM-5?#$I##O%L8NAg2N?kpbZPC&Q{zOM?V9vD9o1$!}S7c+;0JmN=j zj1SBa&KNTH2WrW3aga?`7a}0Lz;y5?LIO(yk4Nh=a2#9^p4PF`z*4h2G-)agqpDG|Q&(qY2)+ z7Kb^JgywY@1l$kzyMQ}@lHi6Fs}`n2ymS*d4qtb#D=3bRC;lSdVn)lP(+G3jWP}&gJXE132jP=KK_wJ$vhz`c??^zj0EM>} za?-Gqv9Zr!qi>(vok#Rz6FKs1;=;y!sY;;eSxQX?Rh4?SrQv(x8->5W-vLzEg?tqfy&b_%77F zA|V5WR8NMZf>7Zi?=W{bNvH#AsX^tF1_w(Az$XppGYq;#(3BDfDl--Mf>Xn(_KrCN zj0-m1uH;~Z7hB_qCpf9X;UEnW8hODz%Yb&m4zeMgV3?$0IhHDmL_5anij~gIgxO3% zpLh(X=N~VcxotM~z@aKZ!<(vmy6*LEL_4#EZ(FWPdz86zmg<*)w2+ZTGV*BJR$0)F zJAlavNWVvCC8Ft zRR#K3Y-=CQcA$k1+8{j{ewR3jMA{aE)o2;{3XA3yT%IWfX9l=gdW(K51G7eDEQGrp z@d(?JJOQGpZ%a2*a;Uun`RPIWv#B+UanfW6J!8}%92mE75hUwIQQf%DtnD0%+fa>E zwOr+P{^|`}CNzY5GF%inCM;|zz~`p=#FUjz<{}pyp4)kP3d|5mP{m|dQWoW~gSs)d zEy=-oq6^#l$@5^0WR;$ZS5%eo(M0mf{A?gq>UIS-r3v;dMN}IHl$xS&CH4-L%N6{! z*nHJ<=Y*d3B?col;Z^kb)90+K~?I<&8E%`L*6BS zRFUXOsD}rl$r<<=S_;>&zTs&QHA^qN&v?uLSA3BQZPwbR)e%-!)fX$6&UNvX_KUS` zAXc~D;pPCJI}~KYqY@*U*RJcpEXN4?!AqQsRGE4ctoPc5M{jE)q*r_;Jya>O*TgsFk+azxWP&p z08%9o+cIv7a4Xbnx7(3;8dH-OCagq@pzh(Wmog0ketJq=2oNw6Rj9J-3{D)WqRp!o zUn!RSWc_hqH^7-l4zsv3(}FOB7{szTHE?gGZcHK|(7z|wE4vPyVPdqbA&`#(5H=o^ zX!F6=FIk0mQGpOggGj*LNYE9$L!6|m?_(QRb43W`?{Y6#f}R262?U6QxjLiq)R=?` zbOg60>4W;vA!Stxs;jF&94lmo>g=v81yBTx^~TFpVv9JNa&2txMWllY62z(*1yG{t zL*%17rqej+*6hiOGx#3|!S~+)d;nf+v+HV2ve_Zo;*m!E^Ds!jAkAXh_>z>B>fG@4 zHOq#q+IEM3Q_JBGDPXUxfu6v7uPkT}m2T&kb;d{U1DI<<2_Trk!!g&;7q?9nhN&)w zQgPHPMTjH@Z-4~Gzccu9SeU`5Qk3MNy`@BkPsC?JGmt^UpTjlXHRvp^sn-|*hRQAE z&4}(k9;S%T34|6R#Xhx3Yrva+G5X4Swv6TNg0Ts5GtX1*Ec=)n7L2h(DSa9 z8X6eND4_6r)@lzWW(jJLt>dL`8sXb@3=l}lTk;B%b z8f4@20k-=Zg{sBrX(ZC>~EdJ(+CJ2;_{G) zjcG#6@F{*)GMXUmtT6OZjK)C6{7YW~T zXD&o86D8O*P(s2Ba#%d1VDHQGu_Q5dFQk~8Uzb~l?oz{UVji~v#pMB78h1qymeDhS zL(Zyu($Ex9OvG`55c+;ecp<21JOD|^_7f2hEsc1Y>Z3I+VKi(PdGtuzI$YIVKd(i@6|yVtr;WVaA&Ly=_?C=%Jd zyo>vr0$J)pOJG6?P!+8qZ1X5P^J;FVW7JS7Yc@LP-Z3v>fFb}Yx~&KctHbo+W$FD` z>rP)y_HjgkTZp}_m zk>t^A!%Qg2{tQ6@Gm(1&teiQaYG!80+@6;M9`L&e0wH}u#zO={LPFmK&E1lQJ_l3> zhM`h+AX<@)hPsRau;_yaY!kCk3$28pVjY8MW(6eRBmf^sJCZp)=P& zXTKm&kuL0lxzRmKyz#Y?w{bv&jtjZRmG!*BEYLO~+!BND9rj_vR9t*=kjY1=6k&3Q zHD~Y*c=bpEl$>NG1~5D|Oc$I)N|$+d`@7+$ zOI1?|4*(GBwk{HRAB)cZRdS&!AQj=L#i7uIOAx^fa%K|ga&4Dfgv0wP?tWBhz)XR5 zG-^?YWTBJu1fB)C58Ete>PCrT2$~>+Lh6RL!D7=4v3EwwZ&j5h3NYpDz657@`jNe@ zhCt)m739MN)bQOfLl0Ka3woVNIDr+hI75VD0(!ZRa)uBJfG~7-uy`73DbOVWXpg*|(+yVlNc- z9_B%>k>Zl&iW1(=l*sT@?hTT|he76NUKEl*s?XvYkc z#Khn#tg5Sua_Ib*Qx2o%kcO#qBVN5Goc&eZLKFec*TCx&9^c%^GczqU@=fnTZ7Ryb zb}CH;In-a1+k@zX1_1^e#>dtM0fe1NEJ`L}^!Y%t!m?qB)NvBM*>TargFHm)LE#F% z2G?U7WV0MC@FKDB1hW00Yt{;#jIXZ6e1t@CDs&rGUQW-Jc|tg%jqj}Dh0eo7pLAMO z1=va;R9e_>w+YbQbQfbOtc{}u>f;9vYPi>l?4xn<22|zA+&Bz(0kldAnXi=+6+R58 z(z``C*u1u*_b*kU06)tG>jSN5BuZX0+&)t!4iCW)ccY?2qDs)ArlSHB98pB^Qe-S9 z(u%qjF3HEHh52o@c^Xtgom%z?UBm*5sHiHUD5K^qL3~*LSsfn5QGk#!`!zkcFgRL! z7jV2U1)>1OZqUdrA}HTYCAcsOKC>|V3wSkZa|gpCFh&D(a+p9lh7YhBftFTjRJ3$9 zX#(#cX|=WSW?wa^wMo*#DhP+W#*3iP?EDx$!O?p2penr89Jo+u;P;nv*1R!r2TWv9Q5*-*cYo?NyyO%4+H79HFgXOnOMfqyX^$Rqx zbeZOdu6}~dvXueh#vlP1%S77Tn)d5kVHJ{KAnOd=s#qvrT7_dlbr9SiS{Wtn_1B%& zwgbJZA}YFqNDy9$;$@dob}{U!I=rNrJdiW-^LNs8TW=P~=#n zq43)22uTDB_`Wg75=^T3#(ihDYLn#y!2%%7M}cPfzI8;y%hEZ+D0n6%MD1axXpm`< zLNZ!#VNl^ah|MvE#zQS-eN0eqYRloM?gM)RoM$m}Odpz^eb{c}N4bQ!(4*mUQJvQB zJ(V1Q{d!OLu_$()+g!r;+>CH2bTCUQPi$$=B{K(FT6J{T2*3h7&|QS^d9AsAVZ_c? z(u*&EH9XYTWG6KbDdO1UrQkKs5iFB*&?3M>&xs3ADZm|;<=bMNTfs*k6l6TITZU5F z>?{|Su|acf$?TsI;j~?Xa)PVykmPJ+=1pymi_AnVx?b)0GwFM`0<(&Y^(^5E9_Q<6zniKZ*c^1?s6?m2o9(pm93o=gF}Ap9tk(HIU? zdU3rO#LZTrz`y_-l(fRA{3_Y@B#lT(HM7v(+Lg?n<%JYUCkQtX0tO);926%8WTzPq z&Jyiz)-58Aa$hD~1E4kNJ(QbnEjWi;gKZOdVjpZk+=x`93sVD%g0)SBB7%BZya~CC z=PR6>YcVAd3Zc@D-3_YAzO;BMxVIO)xRj+=4~QJXJvn8}l0Js%wN*7}mi#&9eS_9t zj`w62K|@l}Dl5`QVyS{Q%>f}sd7amSBPm68b1NO#r+3eWnhCJ2n-(umIqJk~1C8Z= z_p>CF=Lx>Wq>}Ej!1DXL&}Z$ZG*Juz^Cpvv1q?9~LD9@>$>_e8)tPC?9@c6gf2%`Jh8tQdw)@E(< znoeF_IEisBWf~C3hh~wjlGx!`v))3B@ql?I6Q79@-t*1+g8pV6Gw2WbF$0)FYBXJB@g^34#;Hm)?2;f;orb0MXzN4(^XO z(aJLunu)$TQl+vrRYDsvFg1s|_4tr%tZ^(=`t%bAL)n6Yx9f)lE5O53=0-p%?|`k! zkPR_Ye0coR)@=HI+PA3!b0Q3juwRr0MZujdt_NPCfFMP1ITZ--gp&+6+HKjO>xKe_ z?7wmDab{Xn1xqKX-AY3rq)>MOBi!$ctDs66Oo8V(NwL-ThGL(1hX`Y=7aEzgE|OP3 zJY1bhJWXIaD>sVIjH}wQ_k8JNt3&N_fl;~fS9?3AkRfQ7AXpgCA%mRhRJc&kDE@l| z0!a~q`N@dKrP25Z#IgA7oCS%NB;qN>Ypl6U&hA zS$>0TLpH(6BT|KQ#PMg^0*!FPM-CZy_8{YK9v9-F<483NN{E13Xt2grQVIkm6g52L zMWZ$i?6fOTJV9U@?dPAD-g0`Qq1Hf!Nk<6KqZj8NX-~yb9XO$=9>`r&qE<*A@iGT| zluE^S&?%8bw4gV%T){j^uY$502&PlX5scj^uvY=#J3KH-f;AE#11&2(!yLfm9@ZuS zCq*0gW<1ldM9611C5u^mRS`_!qI{LqechuBM2Dxm-nK?plkA@|YMQ(*Cl-~^Ngc(p z$28R`OGv_NLXL$EK=&LBEYw_V8RIy10*l6a) zA&Q4ML%_qndgmixa$~$8H!suRbM8NSd{cetLB9Y~?2k)&i& zCHpX{SD+r*WPRRBydeXotRyf^I7$d1ey!^1p*uefSw$)eKTpXIk*uWWvKD*kByuRm zv9|0yIv~UH8oz>^?#jh_6?aBCtEqgqDF?Vq2i5{rs++yKMw}=siML2pN)~(GoyLuw z5W`5oF41jfQpjDZ`&(@tlUPcGTW)m>;pSGJ+>!x8vLbCLOcork zUQXEn=M|wls3vU+P=bJZ!PXWHK{)(z$e1U1Ha(>}p(wa1)TbDhf!I>G4hRzkiD-O5 zVbH|I!ASDiaQ&{m^4VpKT!b)?+2>% z<#8l@R2(P3aIf_0#-I@BQ98gytfrlcZXXk%Aje@C*hzuaV0lzyn8;LxtmQ+UgKh42 zKBmBz^kn@5iEu}Tagf+prSA$CnCmSoC=8Q3SR`m%NgVH%8+zs75QdH%#%RG@?Ejngv{C z;m&ZT728I2X`qj47o`1wM*G`up`k(=svRHFWr7C7JP<-`+Z3=g9R&`EBnn&{OdnL) z#mkE{vo$YJ?-cH>AK}*1y;s3L6HkTT?GZY=D>5sT;BN8+tw2JO)bphd3>K8Mjbb8T zx!p3tT%J`7t`iiHpw-r~1m`?D>7=fF-;ix1kcQRF3$(+hPlZf@Dq3^f1i_vKBmRsn^oVd7d+LmVA_%@nLtbn(c3AQJBeMrbn5P#X58pRi*($VsPe1q7A1s zRBXt=fvOEvl?M2p0I*$N1T;~u*LIjjsK|V3wWvpBr+b%2rG2deSbz_-H9X`|NTyMW zBkBX^FJxRxiCRL%NwCAx5-s^p#RpwTWzWN>$IXl9@in^& z5wX(>LU22tSG(<&S|;kG-Oo3@$uhpU?2x;%5?!raz^rPi9vb{(a0 z5)imBS(gE?G^#Tzh--s`k_-GIaQx~R{-%m>ngbeFv_0$!To@4>f>ii@tFevG4K5*X zlX`*?NIq!3A@xBP$Z?|*pr`NhYL{v=f#_kX+h{`>p$uYwqGe2`qO z_le(+OI#y$$WH{T0{;}pw-97C%VPMIQo8_HLUGf8QH@-kH%*^{OjLp}Gr&7pFOuK;fXNy>tdj@sJ?~$-|IEA$+Ble6k`LnX2&8nCrmv@TGNxZ77In zKKSslh8*l@ht8MFol2+(TTN(*ZQ~u%%;k12h$ujRAi{{ULD(KL@z$fR9ta~OKZDxV z*d{96c-$#GGts+pp~OL+=oJqC5)VP<(XO|pNQq1Pse8QVeAr1$E>;o*1|5M$Om&U4 zs1;$j6F9nAwGE;$Qzu{pl9n>7*8%a_P4 zS8qZCy_Y9aK_PWDK`>M?x=Thb3@t#});s6Gk}4UrXQW2qgY{BN7R7E11e?bCxbaI) zaUCqfuEb}Jp@_k_KPnia*RlYGEj`trVo;*Fgc1)=f z);^OH0Ga|ZvZ-8mR^ft2md4sK%(`u{qMLm)BWxuHK=u)B*G3xCwH44PWz+_a8yYH53V_#zgUVfRLzw3B3}Q~1DLh$K zbyn3#8!q2%&=plSucT2}g>NovNM;llwS3TwWA|j+0~w9ML+lF5uQ=;3nm4*S%qjAy zmfP<@8b%snOzP6-am%~1`XJqgKC`m9azYGojAzmFwT*{@rA$Sdgu&-0Lu#(m6tufl zp{^D&=m~6KZM0l^LXql48%-ApDiVw3;k)X=C4_*JC{h6pG6BlQ#;OMvI_&DzzpR~D zK0Xun$vbeoBgwCTFgIX%VeD7Y+Tnb3DuyGjR2@xPOpHyK5a`BGfj4MPUNVX+1y8HO zEMcRGih+x^C-BAT82IMx?sQxOPoyHtiuoP5+ zpkz?u(d9{3k_#!%w}nnDTp5yWsa-TWv>y?5Ep|{9%?uavzJ`znD~f+cF&G8XeLC*NEsX~+n31{ zO1kURS7uJPVUiNHU8*YxxKL_lwvOES!ub9+?77XUVGY8I3TIk2 zGw`w5I7xc=zPOR{-4MAN0XJ+tEj@r_2F_^XWPSy&i0!U}_PoeOy- z0z0D85tRzYC{F`*4+R8-KW$l-aX?)fnX!CW>&&(?N5$No5WR>$OTvH^!!0aG`4Ni5 znwnm)bkJ*>EhMj|R9>`)R@A)pP9C*-@HEFLP48u^u+dyEn6bngSeHr@WPsIH<*p3d z!=8=WlkJZbRaW=Dr99LC}6>$=0Yot#$BSX(|VEhu*#B+;c(Q$RkWaIK)zJ=@7cq&5K8fiFgEPXLWIz^ zg~8#&LB|ny0~{4=P)3DR!6#}Z49oRVk)7Qhju$}ZZi4aAGh1b3UeGBUO~8eF<(hbK z^C%R=Kf5=un2H^Wl?`<>p^HaTUk9nh5ZsZGY%`VK)^JK>6f2Ca0g!>s=#kSY-V|

!gs45JGV@a1yR!ASaze5^$V( zx`L6$x6{YEmML4xB`4y#AjC^5a~`5Fp^)<=^lr3SOeWa?pRv(m(8vXN$CiVmYLz~& zq8`WUfX5eHaq$`!i|Hm6TPejiD6U@cJ(H@;fR#aIkJr9r;{`?)Ei*>;b)|#$O@gd( zvUnP;MorHQkU(g`utjT9=LjmG`LnRt1elz>GPNK!rW=lbUHn0&Rmyi|QMrJzl&FQ= z#Hxf8aG8dk3dW0EH|Dqb#7KY%fa zrc=ww5bQE0fV?3yStSRDVHZI-N+R^YLF`*(buA`bm%7M>jSVc{U}W8KyU|2q+B(M3 zmfc{dhdBgt(Bx|6^<&6pL??g>vPig8wmU(lscLuHg^vA++`C}L#xyRVOYTVtZy_Md1#@im#MZJXS@#1Jq#0Xc zKNt?nY{1(ulVwjh`^{2eDxhK6cD`vjNLs9p zT{pJ;JZ(X#t1BioTX+w2v?poxdcXV>~0ByvN2Uu)kc&N&ja78iWWtta&{@;tU=9HJ=cj^3UOBQ z3E|UDP$v7rqHALPn-;7>9y2_5lVL6`M`fPMwJ0THHW?Y$*g{w8E2~>ya> zOVJ^+k*h^MjWx45o)}t{XOMQrbZ@0H`fg!1=kmT!zK8r$aVp57r zO~ZWpU!l|1Cw*ilOtavkKG}qv$Od%-2@3fWZ9`9thp-=l7gyX6yn#X9EZ6WlGzJl^ zc+#(iX3s^smj@z<_mQES-td~sqFMx45MI2e+-mOwsh!BM* zQQ&mLG2>gnR)lK?L2IhHL!iM)le(ouRD65^5D2p#DOKo}Lp)UGD7H7Urwb^2T@}?8 z*~}H)C3Og5v3MoCm!x+gQ9>v?X$c1#BsE@;eC3uT)=Vidh*LrLvo`jgT|iI=AHqzEu3$8PgJZBjeDn|_A{Ws(?Nl!|>8r_FR_M?H zkC)Y;;(9m|y}=Q%as)${8uHtFk={3j7Fnh?2-61TpKN=qT7!$JkCGdNTID8PW0k0< zC*|-k)Mck7GIcJjr+YR&uLzLVfxwrPAz^~Egj}yVS5LbbjHnwM%r=&!8RXKN z6dLTpVD^!c$gv;@i9pPPZKd>P9Z;k0JFV*cbg&dvAkA;-Na6a-m}uxZNh84mZnb=d zuOU@~Y``IfKBy4TG@Q67=Hwq@=L|d@6b@uCAXsH0zD8u{tVN+q#8MzwtjZVY#WEg{ zSyG>=&0In%Y+f2`G3`bGfLJ5LZs1|!!N>xfIywvxhu*eDB-is1J<+w!z^UjMZ49GM zA5~>eIhVRO%oQ_4PAMcUzQjye6DB)Xc!1Ljs4y5rlBCB9j|rT1)lP)#FS~PFIFXs6 z$mU_sx?D}?LwZ9zag$RqHorzDL3p;Q!htE_HS?6JMq6PT2jx957a<;68>R1Of|nFr zbwN`kTb^EJ(Jv%8QjMRSxkAuq&VHf?ccxJliFY*ma}PS2R75a9`5c)Uat{NpU3;)O zRA(-F0e~pM1P&<*XWkONQoKAdjJ@tX3x{6v<&7eaPk;zxc;oZ$RtwySByrjb^+UH> z?+&EvEA49p?!7}dx3aa$dN8*lvC$O>>9$zbX2RU@)V8JboRSd>^_uAc()9N(hPNQJ!PVtPthT4h$)l5h!!_WlbBlzjUwd@k`bx45DvwD{zZr6SAp088vik1(J4xL6?6<`$t&6ostxFL( zaycb!fpFw28Wa#wB`Fee$-ZOheYa|VnMsaCTZjo@Uf(f$E=0l7)z+SoWT~Mu7TFA# z4M0@W;D+G^p`gZiTChH~iEg(SIBJxJu$qo;ir?j(1nwu44Ko(<9ERp_G8L$1qjYCX zE)6z4Qit4D94#RaH&-_A+kUYN8Vah#dU zX$w}2I)3VSZhWxBMx677^c|FyRRiK=ORKK?sAb2NO1SbDh<&$;TUwl+S`cvy><)M6 zSGpn8M#ZP-)IoMgDN>s5$vvB3tFuQO=3&XcDmyyvlrsq<2cW7}u8O9Ifj+>y=tyUS z@o$>nooHXQa|!x5kY=!q9Mv=vfpD6$M+o72agY)s;o&PN1t2w)e{gpE!@&P{(d7;x zt2ejcu4+s9aFgU5o3!xL0F|e?`*All%IMC@;3Ufmcrd{Nrh$UB-RLEZA>S~zN#k(a znuGxy6mIj-V0j>YU2{HW@Gxa0b&3eEF_Fdz$hPF}L!&_nf`~9&_ueoLrcB^k_jQPb z`x4`Y=2A8Xxe&axws33hWe3+<3qU66X_F#`4RVXY#KPho z)@xtfj`U3u)$xZ3p(v!A_F^;K42+dOgaSHL+YWM`vhMVUA&ce-#pAZUB%WOw_=q1f%pI|yPZ;p3^rxz^S zW~YJ|cGr?j1NCZE324}ijOW(o6RildNZuR@DaO^lVXFK=dc1UtI&xWV1IeE$ac>-V z1+9mstOXsJ@U6%h4ab8ojSF9NBilN*2OQS-NQW`*AGwka-(q1iu4)r*=TfQ_KRD=RgDP(Ir9B2$4jZ5q;ni8W5`p4G`mytZTvg+4xynI@`rjDEG_Q zIX6`m9a@IQJO5Owm?&uF^MWL`A7}>1(OR9za%8>nI!u* zqU=&IlkvL^1?en%PF!L`QV%B>vu>EP(7}jh*SsXcpQdO~RE$<@0QUf^e$mTqLm} zEkQ|g%!`yk1tk0^)^dh%2Tv-&2f+x$FBqw9N#JW0kc7}04DxQ=@VzbU{fypOn|xoH zQ$eb3p$_AKTFR<(@XR|b-sJWJ*BlTJ>NdbS}vI8G4A{DVObpBHXQ%!Nvmswh66 zUjBGx1UN`|Le|fw&RwAg^tuKi!O2gG4K)baIROHot(H#&(l5+_AdWjBxjaz$E|2p% zj*;$|Uz{7lh!qOGY<3|{(=(kTif1ZqMn_ubsp?Xm#*ga((B!4w&~luUGCoS_k&d_ zReJ6<9knaluJ*N-W3b-C+Z^|DxT1FZZ>QgD(Y>oY_7`r|t6qBtd(~H`mwVm3-o0tr zdOh3P!uQ*Y-sgR5T&~jAt!A;aI>}A$be-+F+{brtH0z!2=cm2Al8!SvuJgT6YsJjo z_G7(tXCtS3Y11mb?AJ=!o7<`?Tdl8eI@?zFy7um)k@bv&hK)b){HG7<^ zyK*CA5Ve!DP?xg@z0T0}Ztk|TZ+2a?+f^FcWwX=X%7?6L)86&gw(Zrnu5Cq3ZF_Fb z9?eg0cJ?w;+KfHdb=xU+x31mKUE9;UrNK%I%FSNb9&VGp#@2gzy?g)w00000000lN z?$Y`0``g(lQ&8=00)hE&iyZoS*C>$~aKw|l+r*H69W&28@W_nYmKy>-5B zt+L+lz3*>pw?*6So_WZ09=ES^2j0(5cW-uj`y1}H?DJc9ce}mi?)IMQre^ay?a#ND z-)<*9+vlyhH@V*C_j}bn?{{*qxlKB&ySjVb-l@Cio4n-cb$6caXMNv&Z@%u^tv%tR zZhPHcx7JnLyT12Jvf1un>p_b5y_Y=Y-MhZK+uvt;^=|97>U-WU-t6w3b#ea7t-P?CQ`R@BU^||a<&eG}a z-5K-U?TfqXsoT}=_mW+ytx@ggw{4B>RP!CpuYBivd}g_-p5Ae8@~piw_p`gW>)D>&E8Ew(*KGr4_hYWFd?KCz zm@1JO1XI8Ol&AoTpMjDxnyMoK6!D-VB`RnjihCeTf~brF6%Yi)DrlKM@M(o)KwbLx z|G&%4ajp0J!Rzg|cP%|$9#+P6vGPuhPYZfaF7_50?Cb-5it(+5s zNW~RV7BF>&1yx{D9g!~9X&~ib(3Kz+KadUOyauFKX)I_*0$qZk%>n@0(vRvpG@W0P z8B2$!W@hVto@X$cd_BtkkR z_7(~JA{}T6i%)i718eVC*CQBrVz{0xm0m9bXG8hi-;^-rYuvoNj7Za?MeS8n1vrOkkBzE6WE|n6zP>Mg=)6&bg=d z*(W(h*Kj@xxD%a?)8s9Mr_P9ttyBd08Brf(;8IX3DyN{^1$AbJyhai+NqUel?M1>$ zHO%%Ss;w=NG>y?CmlJz;IPTqO>Q6;hYOqr0e7{mr*C!sLdPGJ69jLx5== z%NPo2l}5qdl*=H@y&(=jkHO7m0bn>F0x0O?C`T%GK;d%8g=rV_EUtz8yWr`_I9Xv+ zrq(9rxRo+vuw1hP2{t((!hLXeo4V9jZKyU_z2MBwJwWP(*JxZjSX>a2IEb~2aiR;$ zzJ-l0=eaj1v3)e0xs+2<)EGQQJyCe)MX6NCPE*s3BUK4ClIdO9mK4*JpBYF8uf?3$ zX_WY;2t>=pyN)0tBTx@=Xf#@QR~?r15aC0*{k6 zm5)C3qRGAE5z9R-I7MKWEUGva(aXY?%>3NDG4vuK5?1r0Z${!g9#hi`p z7$%wVl-xHurUQCy`cSN&ElsMqRCx0u%jFz)xq%3L1fK0*v;zW&@7F`Zay0c!9zZY@ zaqJh$K_EDhkZu5cerEQ%W8#Ht_?qb85997sIKV;GyR5aS2wok3K)$@3(E+O&h-igG z*I|z@K~)=^KB}1X-Ah^=vd9%aZr55O-t-ZrtgDbdzg9xjbAPEN| zK01i_EX8PuN0V=?fopIafF-(vp8{!;B|iCYz#X`pn+DsyZa(W3^9zo36zN+TKFB+O zZ!+SC5S?H$!C`@qPAMtWqRbGZ6UB$FvZxRE zlz&;_op$XetX!z?DT14KHt7qS%&z*tK)`inJ|aM4Dssg(7eIqiN*~)Ra6=3D=F9 zg=&5h39(V^UxD^!Le^__LV+MjQS!N3_Kbp%s^FQT3osOkjwPsv+d(9RC9#HnM9gPl zx*ic-i?xAuyMqTW9n6LuXeqyq=N_RT$B?Q^?!YVV2B>=;=S)$XqO8YbLjn=26(m#{k{Sdfx^)49e=O;Ae1+G7)C>a5y|D`1WkLmEogwAUR)dovBz zl(;pm=^I?;RB2%f;9ms^3Ub}2D7RdjIjX0)wMNsR6+wdiL z4TwHe+Z~N9g!ZjaKmsYVq`lf(`pONhVY3Pa)#ql-sqPX}R*u&zpIV%v5@|GU3=*s1 z=?M2&DwEzU0D#Rwh^H8N8IX;GSpBrFaL+*3*o==MD0zj_qvo7Ne<0X4Wo;KrL3$pa z-5cqRcRRbY>k?2+9#rg#Rg1U{cNU=`Q;>{a+r)Q`Pae(_-n8WKNJv4Tf}7`3Fe46w zY?d7b#3j=V!35ekhGp7$AOt~pf;D5!K}e|M$G73OhzK;$*&?MeX>FNPyh*EX39VV{ zOoqi-K>X~dF z-FC+^&$Y_Tof~pmuBMEXI%`AYezO3@Ndyv#5g%!8nbkW6n9!uks(yUD8dxEzQ1QwE z1A9Buqn;;7rRxm`VNvNcBS@2h-)T$i3rOo8! z=7L#GuRVtw;BAoe3gjDc&5m>0cePItEF<*ZZAsdWPMI`9D)Rj@^6z4SGh))TRf{mK z83i0c62@eHgTn6aT9ra*=CEpUnT!Nd346E-%k?X{Ajucwn{Yv5F@ao27!uUy3N+4g7)OAn$Fb+$9gdU0*b~5RHb) zI8$@eP;EW3gE5t&2!jYX{S#_AYqQI^Fs_DRSI+c16RC&cd9Him!aU>3dL4_0$VO>M ze#U9J5U3|@rdPhA!HdYAP)+ZL7@3VP0m>mulcAy4+{`I_w@OQm;l)>GaGlynnqa={ z)s*O^lOh)u2_yDHEy*%+UNW$|A`3FuX|n^l;)PvT8pSn&u?+$+QKkhj8_{QOB@DN$ ziMhc@d)5+7$4`mApkniXw70k;z3bSYbwwP3X~aRePoXW>@K!z$de6@J9M5vX{%Lz zu3U;uE$Kn-WOz~QuocVD&7?THKq7E~zUx;>Fi^{R!JMU&Q%s2nNY+cn!P%@evqSNZ zovd-)qw(GsV zKBsaZtS#X&tF=C#6w}BZSaAW<`4CkBFidsb%P7sLcyLs9ru`lBqdFO27Cb2`dOSe@T zeVv}`Ce#4H!4?-*Po!=l4Y)=!R#?8n$op4V^61Q~F;!JdxuO-D-FHL`Sy=||gArn5 z zauER!QXt{?$s)RstwTV)duD62<1Ph*kk^Bdy^USnULPuxwR|MezG9W`%J4uWmFab_ zbT$NNpMfC4Bj-iYWf2cb_PUdkz9Bsxpg0?X<59bJ7|>T6Xr>=4Ci|$w{HX>!4eQ2U zS3{LB-c0HVmqLl5#~V%OP}HEDOj~e5Gl2A%7|7g7l7)F6=@(A!60qTKRkv7f6|GNo zWVBQTu4!=2dJ}nuNtSMrx@fww%KX{8QIr2qf`@nk)Rm9cr`1> z5y%M<>?3`xeLK`9gBDBKRvL}D$4Q~rbFvxGCMK0-kX|e(#ZV`rsD{@Sre()~*Ojs~ z>S3uca)_|*!gW#=tb-#@LU~*;bQ-&T!>|vxz3S7Aqe~7sAa|GIl3#8I9of+Al&K`% zVI-5{vHI#t?)=lyYE&Wbdf8yiv)dsbfYWB8uj}x0^o1$88ZBWX*%RK1-FfOQ{&e%Q ztDOfoLw4tsiuuF4(^?dWJ|*r}!nD>(DhDpa7s)yvmHXhe$44^WR-$1RbXM;(2ZWLF zF(5KsD?ul08OIKv7u(J*(&)-4*2B^=&Tp!CIm{50KDBxWl|Pl++-}q%AT0tnoc25f zs=-4;_pS!HK>UfQDWKVHd<9z*Ej5i-JG97Xrt&O`z}eH7_Bd634VJvTO@XMLWJ{1Q zT1lhisimmc`SRV_@09NLy^eL&H&Yx%wyBGqEH?yIMSFL%ILf)(j*hA5WOCh5rd!0F z!_B&DaGV0HP$Pa1?o8R^byL}ZcAJvweyj9VrgOz3ERQ(NM27sZF98Xq!86-dQGy3fs>$1vd-xP zzj?1VRQ2`Lo-Ql8+)W$`XJDx76nqQ2nzr?ssy0q`dwf5<49< zo$nvM0A_>u_H^<7Co9An&|h&wkp_g)lO}66yes`fcz0N)KF|t)T&@%Kz3-o!g`F^W zH_2bG0%LnONr*I?JRBa3-D)o5S5P34)I1{-aiN_BkubWktt1{^nMPttpD}P0Ffc*- zgf#3A)4=Tp4yTQ2D^E?r=k(;p7!%{xO0w_`ZKL^KMMEb>G<>9@kQidCx$80%HV%%B zy-A0&ZoN~`KFLIPsJh4&DiG{;wxHMDt0WzA6L&E-=xqZl*g9<$3)|5z^Yx4PBI}FT zsQ69D4ptn^dHBFujHqP@Fl1DI$~gv~7raR!1{S|48g#H2F*g!UDg{S4!>(R6TT}Mr zv-hE=twrZO;rL-Ra11%unkmtYJpwg(axFlpkc6hB0v!UTLIIT+4kQ}RHY=W2b0kD* z=+YkP?G|%ba9iFEns25}F?eKajcWD*_$0?`%YO7-7>?5s(e`R}L{~=nvWHFWuJIq} z>fg+PDo!z>`PUj&5_(X-x;xRx;W&K>j1u>fzbQgbBJ;@kidcLy!zVwCzQ<4oYdJ8m zfHGmJqi3(%3hg7}w|X%ORi^c&M?}PpCJ2@~wlyU07keY-GJmE@o;D`t4ff-Y1tInt zpXAXg_E|-vN`-}C{#1EvV$Q%^-R$kIARb>MGP4}3PZ3n?t-uzDBF!0ZyG zuIpu>G9H(-U_dbyN6em^wuV92l=D*I^ysqW;&suj`8v(W72wt0Np16ac)SV`OGG^b z!0m}^M7x_}`i0#?MNf>$P?6>9d<2ESP|2*5=nNw)3Z&0MrEbrTIacXs6!;HgCAy8U zVLj68AxCWT>n~E0xb&U?5is?nT>LcvM=%cfIk8ppMf`PuNr2%{(qzwXS#nZmz$XFw z>jEp1TOm6XzT9eJyv)kzp@(uAjwCdPg#acZ(rL>Z)O7Ib9|o2Y*KI~3K2%Grbbym+ zG~|P4$NTA$+=8`GsETAk!OE2L?Tq)-yDP`;JHmK(ZAON!8<2|P{hl0yX3M9c7Ukop zpg{DArwPotI2%VUxYctCsMn7m%NBIpRQrn$Xsp)&QiC#0J6L1M{mD%hTwJ zWosPj@xbXpCv{uh50pWPt4Ld*!J!b{xla#mW(9#mpDn7aQZY4Da+hLQqImT8FDjT| zYp0EM0q>}E#vH~x zmJKP*njbOB zd>dp8s+&YLEOD0XrJOBMDG^I(;_%fIb?LG7O%e}!*viB*j{GZd1fbj5s_$BWP4)1+ z;i|uuopH*GF%Lfo9wVD|O-j3&n#}ogXG9MIL@wQcV0JBM zCm)a*3#OI}WZex|#nF`RGRLUO1`pycy`ghy2c)>hbF%+QmgN#D3{-o)2BRi>P> zQsRcYS(l(#lZ}ore0+AERYp+{M)hfC(D#Xrf*y&JuuxySNu39ga)e?F5l_zGr|O`_@R6(NO6HIf|SwH$N2`l0@_mwRyQVB{AQ+5$LSs=ufJF)Nw`O;U>$;K9P>rJpi{QA z?5offr;=jASse#QqOyh0L~?-u?h+dSosPPa~( zixwb*Z)*5>R7#A>U0QFCt;Bd`V=_@!u1cJfe6;bdSAAHrsUfeE5ba#0MHW$PV>;}!E5>>03lYS9 zUu7OFSIQ$G*nlT0w^;n2+Y;B`$9InrEkU63}RBdtgMu6~Ve8 z6-q%ai|_|@{&z^1T&rbPE83oXt{wJ%B~)F2d`M*-EF7{K1*a)TWH}v^G7mV*8Q3f<}f@6c`Pd=quAzfw~8QUag>5%4&=e z=5tsm;gb`>1j_+i!k8RCEnlMM+Pj`ESrYM5W)^6i#-M`SwZQ|Wt~SXyEjA_;k3z3j zQXqp^Subh*FK&mEUwBtGN9tdb8{O`9KFEKRNvQ}iNm4y(!||}b?AEa*%Ati5o;$@4IdJ) zFA^NNi7l>(R!pz8?7sUhDKs0PB?@_+ zupFXc-i#to2X1nxY=)_SU$}xIgR8WQ=+@jJoZ6=5%Vf@dxAn9;KMzG288aYqJJg14 ziiM?hjgA{xI8IGKoaCtHo;SKs5iC-Gg?FvB8h#;581TL=sTqK|slV3Q(i{zG0m1KW zPT_7I5zhqtVRbFKU(KQ?emKG3lk)54w!`A`gLUzUJ% zTqh+}y@YaWO+r#Q(V4S_UJ4JixSLS_bYsb+*L5o zrCjn4Iiz}R+@jXdDk6cJ2~;V|($TT56pY1G2LXDmjm5-PzbN;qJ0&6RN)r6E44#P2 z&2~pKb{sm8T!7cgjI)~(9`$m#-nBxFO_PM52dT%4VDHSxD4Hd3cUqS=E47PV=SWzgQ zK$77?g$`e{5o9RqB(g;NYh2aL58c5da<2Jl#lu?O1RrGK$G**0ZW(!mWdoW8?JLuJ zzi6!EUE!CJ^E6jILO1Q`BB>XxNJW+SJ|F=PgbJ2Ts!DDTBAnSL6fra820wcbPhNVD zv#)`#c|G5ge>mc=)9$(G-p9YUH97D&H}2!Poy*M>>a+ISTj_Hb-O#Kp<#^|HLt%w6 zyxFExn@zEyptx5&N|zVZYR}7)F=Or%=mH_$wKJvZZu(OYq@a;rbJ)0if`YtSjUB?p zVyT5@fL~03!njhxHf2H&J9VuM8krTqr^Dwr6d4D?$0HXwcQ<4ba>YxM+z%i)|MiksZ3BXX!m5UGl93bX{n<4(^jg{=GOPZLC0$TDB7*) zkqsULK~E@z^{;4in^T0i*vD6=0UX8;C#$unY|Kf4 z2|%n~)y4O!UFHLxo$n5x+6@O!sEd^*B?Gq(gZOlL2;3gTS70M)RtE&uRiqqifrOW- z_)?OWWGU*zEfV=GcPL1yLR8j8j>a>>R7E9l(zS08>w+t5meYG9csHjhx`Bdis+>h> z6w+C)E16vtQ+q=s==pHeWEwSt9*yivBem7q=Uxn+na8OB>v5chZ=_fv2+rnR=VEF}*p8VCGV_&9 zfS)>?E0*bYdIvaVpeh4t1H22vV~le^ebD35ku<~hHnKiMdXbj^v?6>Pqimj?;@RT&E_eV$!a0;F@=NAv7_Y0 ziXlO)_*W0W^HRhh%Yt*$eLKVE40V2AQuNjgqoC`M6?VeP13m!Hch6`YK4)72}> zuQ8t4)@oA9+_JrK!Vf5F1t@M)sIk2c3RZ*05Eu_2%}q;2hBn058oF3E5?3Q3P)jhB zvn5>y!(2_a*xMc-rEm*#)s8tTQa$f0^UCMay6(k`*a(z0G|d-P*7`=_`lQ>Ain$(G zg5k>?yH1FBvtN77Bukc5R?}DJ^OHAi!BznoFde94%S~!iLbE-m)#4S^Z4j~r*&*+M zMnx*|bY~_u!8ArnLZOG^zj>*GimvueuIG=pADoBrm93Slq?7C(ix?=wxK)C-4os9c z8Jfx4ifFHCjNUMU9eVuI^0(Z_?G^1uy^~I?yYL-VIC()Tj5p2Rp&^WDTw}3_0JBhp zz~&(MbsQxRYH1{?4KzR)SdF6W#o7_nQ!xV6dqv?W@wtZ7dVwy4Q|}PVvou%zbz@^k~}qAgD`YThg-IHRZT1Q~72cI5{;FpfD;fu=BY)<8EJfM@5< z4v=6vr4+w+f>rr!*04Z|a@|#dDoOP=j@>13wejwtMEQkg`VMo`-qOyj)(=t&uzK9( zfI!(Nl0n*JF*RO%GO4Zi(;yCAYt2w&JvwFPDhFoGRJin+J)G&7bG{Z*c`#GW!A|hw;PB!QzTMX z!b5K`kJ4dR8Wr?BmaM608t0~RbeocKpk>i-Wylkx0kAyP!mr4)fRQE9ORVlDK zi-R2bLaqwu+HKID%L;^PUZjA=omX_pq`-U(Hp?m~n?Dtt1-_o(KwRy8^0 zN~a8Wc2FY{?43=Cf*}-=G_ur**7seWGJa&)fSWIPAJTm4K7b_}BY@{s(Urt^ptfkNx0KMv#mJ3As{e=?$8ei6ivgRs9`k5or^@|Qzy?UZxA@|c~$1r zcSli`6*C=FnGg|fLcG`uL%$d4d_a@*by0i zD7Ms2Mz}5ZK&WieRX~bUeezqR9xtrkib`ovV+d?Ua?#U;mvTzVE?>$LAsGswtgtXH z7e95=9Yva0k4O$7W(jkwK#!E}9QoB>z63$V3quVFGJ%q~+!e+lvTwo_lVeb4ou#n` zs}*+_<7LlGLx59Y&H)1su#VhFa8I+FDG!~sYvRRa)4PUzB7!BuIw|Q(g>Dskm%dzu z!sH5<`@nsaK<-`gG#}f6OV^*e97UemWRQx`NDsYR6ZGcOI!{jkSkEZZCs|%}?TJjfpDWPaFABk2b+n zuQEJ0zs18X3A)*2*8&S?8&3MI3pL|jf`!tc5aso1gF$H6ltT&uJ) zsVC#S-krj7U?3BTa9U7mD&sV~IT#ruMkQyRlSf9ir-`nC=+2?n>)Qx?QPLDyQf-)8 zZ5~eGAhITBe|M!;+tkKug0Tb}J51h{tY?)%m$d`c%M{vM+=Z^PgkpfE{%Sejd7~Wi z^S;5D_SYH5h7GmN=U01Y_Z)Gjhov5!n3aWf+BC)E*(!r}cA{5=$d@=FCq#S#>MMr< zu22PUDY?aPvjfr?A#@PQwRj8c&xGI?1375I{oFNd1S9qCcm_I=GB^gqFOs=Af@!c4 zfq`M>>Dw7KC_PS*@reREUC*h-#0@yldnhKHn zqO6NQBjh&zb^GLmAr4~{US;jwB;f0uh(9Kq>0(!;QKsF29?MP`9b5O!tyJXD@5u-b z^e@)Vk(@x(g5}+Qz(d=zAv>s`t&v!?43nZ!WNP!+w;C%|Ogvph>txS%@<*;2Abna# zH<@=;bci*?!R|Fy;X}fGeyu}FoVnC#+q!P`p{XZ^p*^{tBt!U0}NeNE!o zR`FhWp%RGNY7m6ELx@0e#p|P$y=ijrD)NeFV)JnOuDWxSdPf^BcvKuZ5Ef0~scXCB ztiwy;^+-KmSU65z=(=V+6Ji76wFdJYinlnS`XugQq}T)qjVm&o9=wM(=4*+D?Uw@P zDy1S5H&K2%x`ZPSacyqXs4ys+0uz)3$DaDx-Z=(Yrf*{S_ZMU@ZM=wFE^8ucU}_2sMNfw0%R)gqzs!|C@(YDnQVGd*&;AkmXrhPGqG{E;RX!!!M937 zM&YP>T%$zg3~w1>S)VzStqS&~k_={*CVE1ao4VD6Vy|T?j|DSu>}6)c{7^r15G{cR zsW5B2$y8iEBMFPLq4{UM4k0wEvNDtJCV_K4y^~;CuF6p1N-NS{WsOYCFf+N+tr*Zc zjaM8%yZWysy%H(+6vJ%`F(-IP4h+2S@u7>}J9v|tr3PlA6YW-!e6$1EMAt?hdg6*U z$0!2ckdv(*RfXfb5Qod05>bB1R_o{49cpWO%u5%eq_dh-AI(F zeG!4G&Qehxu|HlBScBACdHqu)Kyd#WDBu>dx*ucB~wZJcP#tsV|GJ1OzW3 zSEXm9=!HwI;v|KJ+F(?!MuaIKZz(O=3e5H`B2(FOXsZ5!DZdBv%z{o+N)YU#p)z=$ zA-p#qJ@_Fi%3;=sr@=~!GaP}yKMIGWQ)6R2=N4tA$bhE|>u|gvM2in|cX&OWj*8nc z;OFWppoI?y4pU%R51-2qkU-w%&y}AJhjl9~Md^^95QCofA6p4sMHk*ldX2%pYodD8 zg648tW)mL}ZQbI_-i?nTQU#kq{TS}y)GG{Rs(y~Z_4Fj-h?i09xv;A5Vkxph2!+Fx zS_L?0G&%Jq8Z6vdIkr~&Ru&}hiHkEylj*)BJ6iL{YFX2Zi9p>1m6r;tzjSyTV%%x8 zZ5}@*(^+|_Ibwy45!4CgDm|4WGSzOPU9(Htw{@dr#C8ZMFN!{9atnP1R5ok+q!|ND zh6lt7LQGYBw`%H|yv-4*4y_|l369~kuFK<_>3VxlbuyEHfpZYH>}`f}ce^W#b85-G zFe1ExhtucfyIk77A~2M=FjQ_|J~SlD%b+MKgCTjfCGKWrk~Jzr@JS9S9|MFpL?Ehc zwTK&A6*lCCuWG(neU`i>mJl{{rd81a7d{ey7**U2z$t3|JgIa+1g+ZWmvD(MC1>cF zB@xE$K#&m2xr!`5Zx^Er3`WV#G&sW#jg^o?6yH)GRM=kPS(fW{^Y4P?8RSb%J&LDZ zd-Q_dbLeq!Wx8{5;bEtPjUb3JY+LQh!WIq1os&8elYN)ldZ`z=uE@TW=!=|sec# z2UQgBy93fNvK+V@XlOv*2o6$=)1e_>D1q@m)N^$kwKtHLa`S<=gu6?s)b>}+c~tq# zBPx8lj|EUXHPh%L1r{H2T((K+;G&EoqWM@DaOAV2lBFu#F@@Q}#i4lFiQj$MHr~sK z8d!pc8f?i*#OTYH%J!SeGf5vexP)?J2UYQiQp^;>5QT~qN~d0~@F114;>VTx(mm_K zdvOf1yDr%Ek82L^(ner2)ka63tlAvN!~{<$u& zFKz>4!Aa|5p)!$9t9-7xd98Q4);{Y|u{snbge(+_;272=n-$6enIVz) zLYav!aA@f0N%(1yd02jpM;wd}``dvLuAwRGnS;>IV>P&iM0Rrq+0DzjdVEsry~A5% z*24q6p$U%mrGO>Q3EEgoYtn{cH8Wf} zOF=@eW$2CdEF=cjU2vYlmNrwcEzZt{g-OMfzE(lEkCm#RW=TqbxNv>}_&6KbIHbfH zcM0%RO+4ey1{g5nv2E)pqq&8ld^SNar+7#mt9v^Ih;4vD!?avKC6uva!LosT7Ij?s zlFCw!zN>|=Tjc5XgxIVdlU~ItB(x{q#MWfD#I({-((bi!E_g2FDvpyP6au*K8q&Rp zfIHjP-4@~K25p`aMToZFLBe1x`V(+`2 z7|-2Ce%bVnUc^l`)tq5qjBQOI%BZi2j(~zjfXWBL9hE_pZQ*Gt6%cXqG5yzMei67b z$7^14QEvoV+SbucUYX@)TYTfaQi@G-xOn13fN!SMUOBk8T852WDCL+S+D)N(jzVtH zi5}XFW$(EYE6tEDrFlL~AeY%V7L$r)k@G*A0PeWna2*jc8D?)gs-bqW^F8#94mhu0 z45_KM9(y*O<&}RC)L0J54&Hzf`uafS(4uYm1R5Ess_?mkpg7IuW|7K8uSGi`eLa@o zMRlj6ML(3+tN@$VXtdShlBhRCEWL@fhqX)9e&o_d8I%o*>ubHhG)HATzmP<1MnQLu z-MVKs_`yI>fFy=BMhA5Q3Xf!mP_-5ccDZ3D}9QPjzAXe=)MP^zu(D3-_ zuq+_`Ajc*kAzQs@x!j>nsOm{sRK$_0hE`u>PiheADEfo-pd(!Rx2R4(ytuYtA_Oa! zCX#8Ti_XLI68WqV@LhU%&J4vWLx&z|fT;@brCT(z>XwBm8jlBM-SO%+TiZ+f8V9Qu zchg`({1w!4_^qpnO94=#^b~N8QA3Dx;|Y^LGa!~?<`QKV9_<4aj%(7sL$gx(cadp#7uP>Yv0E$h%$s4ARh!^x0O3WM&oYbc{! zV*-V%fL^?T8n-we@}RQs2e>>$9uVR32;qYt#2Qp#9KfA${fphMCw>eO79l@BWCPXW zE@uIe^0u}Qnqk!$I452@CZertsOd{UpdcBA!_}u=5Hpc6hpNM@DmhgTW-xWMU^&V;@|Qldal&x;joqSnY?Z6!iF>E(QIaJqL@ZihFet9j zz;X1`iP#V$H`Qz7;%b!^?Um;hpv%+M^%&IAor5nH@456D1K7@+N7W#(8~ENRZ9;O4~i3UZ_Dw`9P?0D`*>Quz)PF*inR zTLVstD;4g<&1~(*ojfNDsKN_NS3b@1FDFE5|z^qO(&_H}s>h|DzWst&6K4JJG$SgzCRJoVF zsGgvxhzp}`v$?W<@z_3Q8R$x-6t956-vbPZ<)4rF7d6fNx_zDnzyQB;FU)P%8pt*@xa?@;fXgs=FvT1#`&6jYvZ%_JLRqK}I5Y2C(Roy0Pr06u*#;hcnsG8wPkA}Ue@`|WV7I13|5Ke=08HR>F$sF}o z+Ac36EaSk8GWc8*lBQ+&FVdv4XT!-DS6Lz;6LEXD zl_cI#o0!>bRgQSA?M>yPsxOqIMNWq!!)u0>S9y`Ar3cKlJ?XBT5tiI}eo&Mr)Lnc# z4mW9-JRKevL5Z^SGtFWN=odu#A|tSrL2GNM=V2FF&VvmIki@MI4&}=B`&&Fj1 zWpGkdm@)c2>E+GPwfOk7!jLSmaBOu=wuN0>UaVj=5fr)i3Zebemb_F#M@|kVcdyq# zNwNw>BV&Bi=xl?7ya5X<4nDU%7va3&IJpBOL_r2IzV~$ACN^QRBt})?ZZTJk8KmQQ zQjVQ=Jbeoa4hbw5J&@5S^D?v82-@oRs~viYGsBxuVRYI{!3UNQQca4OM>8X@$bQA>K3gQr3gjkyzg5!B*i1zdxkscj)6tY{3q3vuQiv?nBi($}(hLvhms z+1c=$<~6O47(&Uy%)a-fut*PQn$cQ+VSOuhvv+G8grw;~s8!M-yg9XUB0Ag3h0PAC z%~vs`M$9;f+S;#xL(5Q#aF8%2Q>}e)Um4(?7cLiLBTRYB`H;?E>wJ~tYib{KbZ1u1 zh(hGL`>3luA_67Vo)TQe<=l{rrmngcM`#~(8ee&qJ?*u8juIK>mdPG;%3ix0UrJV} z52Qmb|<4WWvN8(kG=?*;9om$Tlc zB02M=WrAKf<+n5f%IlJK9U9eTDRSGUax)q;0c|6JBj$7g=0x+vO|%X!(&-npQv=b{ z+<$yS17W04Wo;VkqFH6M{%#ca5H_?O1>S7zU5!*SQk?*rM+&HmD7-tNcl4zCj^WJ} z>k-nIi!=dlQu?1}jJdwUx>-VK`{kvjNw@YyH`-H+Xb|b)`awbKB`-gCsYd)U+Zo?1NlzK-XoJw&Sy1*AP4~KEV z2Me#22CXVX58J|fh?IEqSZY;Sh)ewbjJGG{3yj00ar1j_fr~tg}!(ncRES7;A91;NW=-pifCt`DrL_ z2>mfYA1k*f7Y6n&OT~3*?Mdw9J;6}FUyoINQo=1kd(H^BBg7d*eG`}<>&cPlcR`Yq zB=0V)UfBi(EIajfut^XK38%Aak3EiT5nc&iu5uN20<8sZZ6UQz3*KM??? zqj_gg!)kgS_Wr)5pdA{8Q!B^j;2n}E({*yxC5i~j@$329*tnBzFpSe9WGdybGU;O~ zw<66Tc<&^*-W8B^=*LL_c0h^0DKV#v0K$(4a633V?Bd7~mu4}t3j*m-`{M_p18y|+XsR;suVY04 zV1TuSO(Gv$7*1EUtbS6H?IwMI6R(9b@Dy4~)R{#teXIfjKp!OglHZqEU!R(WcaILA z6}td-7FaWA^BKC_?D0IV&6xF8@2?_K6R&7s+9qTzZ-|FYfItC6A!$MMz=tKMVL+3e z_XZGD1yp z?^w21MW1+?CZ!By7%Y@d_SwV>Ogg%?oDPm5_(i}1y+__ zt_^Q;r2Ed3O!ZJAsXVvNB}!0ckE&hD&Y){StSAIs%zfSHF75?*yA$R6V*(A8uz1vX zt`elJVXSqE7cCasAos8na~O@M;iEqtBRKb{hb_Ifs(PO63f5p60Io2R23~X6+$^DK z#(k4qqsPQ7C^i*jFIx;rH3~8EQgL}`uj<-o?hRh)9}fapv)JIIVe1FsNTKm19`Y)= z3c(7On(vuAS)s7Z!6zFj@;H;L!o%$91Q7Al{4tT9E|`qzRP~Doy}@IFAbaxreO!^%+%oEUrnVJ%pv1K|Z6KtDr>Y<$TylRX#Stt+TBK zdI9l`x82%0Fm#5LHPd0F{DK2w~4eJig zhg+CJe5AX%q6~g}R^7s=&U^J|3iJ2cJo~M@S2d=zX3DC)Q%JSB? z@JlJ>B~lr+iOT4%sOO5G%w>pMPAwQN49$g=IFn&*#HRHW2&w{dZ#j9M25yLf-NVfB z{UKbeicUvPkKHOfHSW4;VDJ@pTgllp70;^dV|>iE=1b(X>=1)#zHqbZfU!9OT&Dz_ zIrbRM+f`tbM;2o{M2JwxvXiOe=`Uz=QN+4z^KMIqDTerhI#1nz2xEpa*>5nRS+mNQ zZBUz;4RySrs6Mhpv5;#$Lfi}j34$YW&E$b9$IWd^H^y_aQ{ko*6ASIP4xWdBCob=i zJ!lhf@j<7Y-mMF?VVa>r=Zz$w^j7dbV8x7aNDop&JJBSlv*<+^3YKK6%$kMEc?tOz z$igH+Q;8Q0Xc@q$3%E7vnf9}k4)GI^Cd5c~gAe8K7qJ@|yHR4mhNccHV<{5SrCUz& zkKS(1uDK6($bms6HbFRWOdM3IXdMeQ9`YO3TufP4V$RuBr7nfTP6CiY4`WywRvMWw zxcs^4z3Y^+px%vUXr-u(qqC+5l6ty|@RvQG8@GPL1$0K*DlFQntG#365G7#?!|&NSdeau+IchrQuGY9+cOKZfTyetGGuFG2gU47y z*NjW`DO_w~UVD^hQbUYuMg(LE)^cc5(Wvo(!ZjhDSJAK(p(WOj7@uN_vJ1cHZTHj6 zB4(60NCDa1b^ER8iNB@X#{pa0_@yQAhw7U1ys+f0J?6GMt2ujBmUJ2;I&B8-Icex) zfuU70!b+nS^F$jO4hs!s;E=uMXpd7s;x}l=kH31+h{*9lfsV{8&3h%#U9tut!L#ZV zGVQur3Ww4W0!KpPO;vnyZ_o6*;+5`-yb=PRr@2ISD9{|io_u7e z)~7-f-Qe>TQHb?!jr(cyb&v<%@I5aY{X|X^)4p&yTFA&9!t~8tXDG<^l9codsavp! znEQq7Nv(5_-6D+c1pVo2HK5014PJK{naAOQvpelW&EyA?!NW8k0=ryvjVd%7j>|h^ zWZvn~RTY14wc$-FMj6rfjj-Hx_3GKD&W(Ys2<}O3CAV(gveBCN9Mo9uc054Fn1x_v zT;DgZAxH<{hGwa6F0#8`KJI= zCqxh24igxQH+@y(HNw?Dn@?$PhN{Y*^A-{`j7&2aMi4EaRBg|st%wAKHibhhx*i&n z?V>d)6D`o4wG~ZVtc9WOT3n9bbYa*e9j^nNAzia;A}DD0?^+c|K#L-sdoJ%36IJB~ z`UZyUOEekdNvFaSqeR6#Hvb(yEop=}(D)T#syN4m>cy=DI>#M}c}1?F(wHtAYKxeb zF_^KT`mpYzcg1`hI1e$$?!bHjYY@K71!CbfOZ#fOxs(TcJZ=Rm@U@qt zV2zmTCD;cAO|X#Rfh>`D;;gEIjrNi35Ws?b!-K86pHzCu1a~2NjHd z)GsU12Cp`i;Lj$rQHwZH@WE<^Yih#`YNMhHpgjR7&@WxMf}=Bd_+QHPhg&R4)3L(L z;OiQ+vu`M!ILEtSGAi*L*JalU3mdJHgG9VMo+T#>0P!xLn8fKQ=L$Spx#zdkiQ-F( zZ49Keb8qA-$;8LZ9_<>zQQqof6;2oW_qVRelrBU^5)6$xL+Yaks_4yEDRF5q1kLqn z7gm*Xb_2u$KEg^@yUq+;Ga0uI-R(NZx{(d$Mq`&b6E{PEAkAQHa5oMj;RGw78P>tR zG;vvgW!qroUIhj%mOooFEfQsl3J%Xw7-Sf!ERHE{1ZFjRoGPaE^m;%L>_*?ZhSzG( zvSyhPyr9zeRD`a6#RTra<|@=S-EBrMQLKXPwYbDFgVSn)fUO2AR7Fg5`DVSmy_39D zDJ%oqL(pr>ebeKKN6oTP6_+G_HgQPV%jic1^VkZ|oaSZRO^<@&ZwD|aim?Z(q-WtA zu<4qhFl;Fz4D#2^YBIdW2 zkA3ECAsXivveApAZKX{LF%&MBVasPX5(+lKOU9st$H*5OgW?8m#a{KcnyHakt!_Kn z6tGX4kbr}c+mwVUt<^}Eq5NA>;YouA##(B~iXB!GKvJGdKkf@r_Ry`C$z_LrWveAM}_};5& zCQ&5%LFpgW*SV)Bw`+r=y>QVI$238v5ZbaUkr_Ni-XR+dk$A8e(Qhf-8-bB(w?{ee z2Z@mzb)N6nrz)d@LWApH0mPM_G3we)G)?)qsHi=N!=@=S+H(tr4f=DfJfpo`jO|#| zrUn|Om$w#EPfd8_k5R2OT{ZQP2%yx|85>8G&&*BT6c2#Uo0J!Z`(qepzOI^@D$x@= z2v_L?JPIu`nD4B|yq@s0%S&`mK0-J8%{#7~35|XebPJ{)m4)bfs7$iLK|_uN;Z03bgI$BJXxI$)_?3u4^sZygIOjuXrbk_M!Wa*HpzAr7E|)hAm3 zuU~grRIv`Q)rD(-fPoNfry01&b2Z(21#BlUmA4Wy0D!S6`&KiWjA3AtQ!u8SsD^kB z^+jcn_|i4YLNVcIZA3jnr5?}BB`c0W%0Xd% z;9cPHLFb!0*`Rbao9P?{lrB8XADiI@vj!HPA8wQg`AR*V))C29VK7-L)S=2v3Y;HT z!BM^B%x!UU5P>Ebdu~w zU~XB&CX}7ORKMD|$Qo4{0Sq=f<*d?eQPBxfk6>#tP`|MaTaa=S7DdDh3~K5EtH~zP zpVPqaWgvAdpfq#!j#f@oR|{!-ag@EQau=_gdqB%A<5~eR5#G2lC$Jr96-6X~N%v(# zyOyagr>cvX9Yb$~8?y-j+$Rd$h`us3Zhk+Qe&$mP78h#JNfA;G^5W&&Wk^IShM_1u zoZjA7e6=;qLRB=*(Cm|Uri;p%Ut*atbS4zsP=i?Z7lP#|#qdGnnSgpAhlDrW3>*== zNRWXrl-4y$J$?yqWEP+yA=RP;;xMo-4};EKW$&FC+0u?H7L2N10kmgKRBK4` z7o4WLIUhMcNeI5S5nuKIEVsau$3*x z?@A4BosUrgaUlh=v2A+%@Kxyqb-bsGf;wmEhNX0iFjvCy)WV*UilrKitMj51;=jtg zDA3U>rnrKkG2+{Br`NqQQCV-U4q&`D?|8*)fgI06A-xtwsh(x#7!9E!vbB5FtOIQ6 zy9{|xV^D!Z?r?;-M@Ar=jV;@u546hn87S{voPB~+Ilx7-<|a^}ebO`ZSc!;RAxOrK zt;2{I*BS49Wr10mU?6rehmBa3h|DOa!qBEdE~%hbs;TpxEz8+@zPJ0JGEhUEU$TUX z?7rFJKVQmDuZg0`htXr1j6Gnt6^wvVHKA-3!Jflcqn<@k=nRpm)B58#TFlga?hchbq!1iJ z^72Fm57DR`JE z>a!CT(u6j6C(xs{vVd!H*ySsHR;71{TIF17Z%=M+m2<2y*={vLIrG}5ao$-k+U96$ z4He96`C{86g(2)92|)ADRbO(f+3+AWOVG=?Y3hX$#?90SxK(d*jby8mD_b-f3A+u# zp&ml)^(oK8I-txbKa9zYgQc5oz535Ij12PB$hedRC+bbO3rG>^EeQ8M^He(3nXf7* z7FQ5@GV$EOTc+aCZYNnYJ;Ro-7;Q3GPZBk~msQnYt!`bEo0*?V3#bd^OR7Eadj_lv z83AabhwI%EPNb-q(3N?5wi;YCf_oKbk=##v+6r&9(UwGcMifN3Iu*;2aDW8)lG5ZM z3yhwi$X3$_(5n;F)>_Rmncj{9grP?AhOK9~ctRJW(8RMdhSh2+;l)-H(UsIuz7 z+P!Bs+KbWkG6WA-(p#wk8OoOss1B{>@gW7gjT=p=468$!prO?*5%l3vRI_SgK#c+x zrtH!*KI>mB*Q1wQLSVl0Hb|mYfhy23bn?~@)G!hq_>~h4K{Tl+N$IT!OOXR%j4o+} z>_Hfxs=QRrcN(#_*~AR*YFZ0$Qz;)@?#rAJ@}?MF2zRYBvdP$R*vgZ$mF2kAX?1%e zi}+$&fEyJdiVz|s_ZFUnpbJ;!jO;UL)uW2_T>{q!s=U`JMVX?wrI@?`Xpi?GvorDi z?Krx3vB}RB3z@Zvwf%K`%Hy;wNZTQ7g)F`8iv9fFX834an8flAXJmzb;EmPyQwe)@65>F6134UhF-|l zi<1=wh&x09sdiDTZCA(It*(4uo$+OHG}>Itk6>^O^OuT<(Wd*C>}YzHXOje_Xh}$h zK|-taoykfHmyOiBrfu*KYsW$+ocae_>DM|`P%b78vJbwDm4;cNm^vd1i5rpTlhM=P zYV~2~iVF}0WSz~)VY{w*l=$#`{h%bFidl*qmZkPr|k-CovRl$$yj z=t4CgMgAD!h>HYTzX z1PdmNJ1Mtx6X)+P&g}h01Hi(4mN8?rSsX$4RESw5-g&>t8`5PXt``;O0YNhe6 z4FVfZln`%S_Kl+MnsST+&Qkx7j;LD0#S2OD<^}~rcyL6 z1O|eedB%0uUpIJp2tJj1gzm_j?+LMbX|B4#CJaJptq(c!KHYRL2P>zsqISu*dNoW8 zYeaz%iwuNrxNfObJK9)Xnpl?k#`KgI__prJ{C>+hyJB??unMDJFaBp7WM>3|s=;USFflYIe7#xQ_aVSGQej9?l>{ERIfQhy%4E z3gJqQfR8?EGW3``As}su^y+d!wydcm0OtY>EwOl7Utz>W?t-6~N$uwMmV=}yxWy3$ zml@Zr^|WrA1hpU-3{I~~+;1a&HDz#b1fnW9pxiT&upH%`zH|{_iFc&Yi2>JITLsu0 zmjdQC*PFdE5?AJaY_Ne9Wxj|w1~{rVbk7zjDlr(vfxKUruTg9{+XgWfp zZ2asUjA*<#TY~nLJ_SD1qe+cYhct8<=vEtE7#oYI8z%7JNUX}&n8%m9(kSgD9u6Sq zs+q?=;v>IIPB0mYrR~FM zW<;UMh}qsfgVmcCWQaZF_mhAt+z`O4HO^t+a=o0t8A*!|aLfc(EaMJ%Obd2}2+=L= zjUR0iY1{FEc{kc_AwcY~YLvaSBse2APXXa5P_c@AOoHoykX0q>3V!lf_CETG^VkHg zTj}>B436wdmvqOG@Pe#|s&rNWxNVe#DH}IpHlpxg%f)3xUM}-+b7WA+p=37+cbwo& zjzt$O#(6K0Nl{b~sy>X`Q$U_CXnOHc`gJFIge;Xo;F-9k=e<%M=P2QVo-TJx@N^{m zvVd{Ma1c+bNJZ z-Rww(sMG;j_13M{!V*TMoIr(w%x$rd=Ypq06pOjN0=PCciA<@;YNQ||TAhDkQ@HS) z*cOf}CIXb_CSYl(=K{0U9Ol}078VgZ3sa06r2?%3Rv^xEn?q`?a@azajNy84t~6X6 zvcPrLRjyrvhQ&`g*mUYE@2D0{DHA}_4Si9&jZdBf6j_w^f_XZzR)evSnRs*712Aaf zRY9u2)B%HYMD_a;ZJZ3}1JNy{TxURc!G~Lu#i?C6Wi(oc1__)h6~*Z?V**4bH-ANp zsnU>6nq{03bAb%UvkZ?Fokrj@a&2}lE;X%lEoiA3xtx-NBndA|4S;lZqhj``k^``9 zxyMn=1_Ju=kM5((tvvE^A1S}6LA>tTVjQv+LcNZP#v{?M*ymDXCX-cZ4Ce(`h5S0Q z5R$vDJ=6wLEd$~_8rD`Aw`Q1eful$!6|gGL2u;}td9=uHBt<-ytM5J@p z3qw}rN85ua`D=EidEu#uVK*Tfof7LfQ(BhluY;+i(- zt7P*6M1?d|i0J~>f1nVNIILL{V8!e3CX8JL10~LRYP^@d(F$@|uHH%$Fl5kc)YBae;ocoy9fZ5nGrCvZ zzB>lLJZHN?4EdoBZUIR`GTTS^(4T? zgp=U`eQw*jEjh0h#hpW@KaRYJ z=9X_G$6X6S0uy+oTGlEnC>pj4CSOu^D)439uGXtL4w}e^t+2vynCLQ;gcx8!M%0A7 z%eeDnLT(|o;bdtte$gI0U4JL4MXl9Ewr;k#D^bHDfj$xF(LJ-)bmzcT;Rs$A5FTN) zAkQTzWjdwzVGqYs*%hgBhw9aKUYg;(@p+~?2pMCOqE$FuTQL%>Rk>D%{v&t3Q7)bu zMPDP5p~rZ^#HCYIBC(f35`Iv%;Sm60u=SkcU%7js$~4J?dyQ!eoC=`5Pt)kOgjnpU z2Uyauw@Szza3p9;x!Ro9{jFBNid$OW8M_8kR4EYDzNhZTQoy^+XAA+@M$FmCMO6dU zgEA}0#rSL?e4f^*=U1G>7~%Olviv6*SWo`*)u>8M!t!PQ--m_@nMy z3;S=uTzds#v5|KoSGm<3UsO*LIW%trPcY!XTZV9cjxomK`NxZN0lkzI)89=2M!+=Y7lx@F2Xkd|GzO z3xrNEwJan-dl2&so;-R^J~jv6ZmRb*4i&KUyQ^TRQ)#lj8zYdPW9w2XHdE#|2FEQ6 zQ!rAMQIr7vG9z{iqj+ipV{Pp#z<_6>@QM``RP{aR$s$zBU%ka}#n2G`Dv7Ft10ZHz zQWgf(d@G44)_^ao?{V-ofR4g^UdxWX1J)c-Y=n4tbm_IknKEwz?f6x_ctmO;)5meO zm$M#z%>fy%aXt7#1$LIDx!4u{iWbc)XalgBVAb}*hlWneg+P!?o8U`qF_VS?kU0#6 z8d{XBNX5PZ4_(U*AS4b0qQ2-Y)5O9-Vx>)STI-T3zv;{O%{fGAqkjaf| z2buJcrW4R>0->(h=#e^dkZI=2Fc9sEnv_mndV`<>8B5fF8lZ3#FzJ)-l~)4$%T8|O zUvw@PMt*lhCwRH(1WQ8GR7=IAkx5psCO*eRMm$*bd6b0~GPQPT?93ZTx-APT2LaTj zdivsLh;2iYkoAV{!pSJ&A(FZz=q{PFR|j2(joDJU)j-`F-v_mJJCiX^J@WOX(*+oV z(lI85Ha|kGzOjANvT3xhVo-7xWy3=|`=b^fFU?>R3UpoY(ylJGCvAb+5*M*Z(^wxQ zV!d7~+i#MT`d8PF2(VvD+xL*lT6>r&x|nuf2?TluSH?CyGqtz$^U6Q!SKx6e$7hriVji1GPw$)IubE2dwZHQJ+Su){%A_5)(0ZUj= zM;F^ma~QE!e3P?hhg#oAr0<0V&iIWI*%19QaODpE(roXjHCi8oc?X4^^?_8d5KT;(O* z*9Ni%rEchnC_OfcfjPVivaCLb1-9YRR9Vplf$@Nv9<79eS95>nB zD3-HqXB|11LJ>ok>QU_y9YgjRLZYrY(5*GmZR~w15p%2y1qXW}T9{(PGxW4j;&u_2 zUwClnq+6V1SU&RCG{v9Ljj~X29g&$7;ABD(Es{Y!=E&<|fmt538Iq zzXuk>%!OePAx=8nNc6Y5QfIt7q2v3=mU&{FniW%uNgrO+_rO7rNft~~E{N8|I>o<2 zolUu=Stk&{aLfzcc4Av~cYF$zLKlZ!n&bs-32XMR)YLPu9Kd->1*4@TGHM4xcI)n* zG02FJn6#g%j!kEC?xoUXY@1nsgSfE>NQ6qXB3p&$Whh|mry-@;oYGAMtc!fd`7C{! zQx-%;jBM+rAB{nNOv`|T!F^m`J1M9MfX70wa+(pnfUMRBJ&ZS#&^y?&k3KSX35B@k)@Vr_A1md7nIMv9-UogYOc42!ULmm@{-b zZN(*VQYH0XDexzy)Li4{wn^CPqY&XjS1lpKIs$df8`5?$86>zuISPzTv<`_iLtC+V zq@X4lSMgg_V9LXl93f3r+kB%0Rgft=O~KP5nc8Pi)mG^aLZBj{LE*@Q4!BI#z{YE3 zmOY#X3-SqZfRA9jfnl8orQQp$XqbGP98|rS^We2?g^<@E;-a| zOWbIolmvsYWi`R`tW&5+^RSN&n$h4m4G=%EZX?dDdeo2w!|LOv9u-CO)?W{x(y~6< zXGX?vc}lp!^Q`zGnLMWzZ(#c2Y3Zy0 zgi@9WE;;2Ll?P*RCh1;(J_6 z-Dd1=sX?GzKw*?>n)B09tAaOpb0~cobqK>x)Bpi^#FwH%(nRrsQe}J`eAx zUNmrJIOlXL8i)j8^Vd(-Aep_@JieP7;un#`Yko&CbH(a~OkI$v`v&}Eo3y+VRWS?a zB?5pZcN`jGR$H23Fl9^h&c!*5g$^Tcje0pj;x(R8xWl}H6b_@M@DPOf;}UXa-mArE zoow4Y{g;KDqd<=K`lY(kwueTz1yCOH`~cGz?y{_k8}<8hX82p_TQc z?9D|3fl8k1DhpK!CU_z|4|MmR|HSM~Gj1sbFKDlzC+QeuF zATG=B4q9zZBHv|J``rp*r&-lwXfl0bv;&H1GH|r?OrorVHe?15;5Ml!9-%QpL6ahE zyg^MR|!txw%f6bMmU z6YsW=RubZN7bPCgECR$e%0#DfEa@i?o+zMn4;hP0nrt67rd@r>nM0vPV-)5KI&dsg zh*Am&M@Lf;NPK>VJ7!;u!t!bZxpkl2ubSxyQMptIg3xD1xa5pk;7s+WP>+X~0mD%D zp>NvqoK|XCvnjDB``W;~K@r!lYaA<*1t;qqS5QdYUAMn#Zk+;o=5J7x$$8 z3>r=L5R)EF;WaoqLeA5sxl0zO4}}X9ric__zK$Xi905JIp*w9~q?a<}bl_C{tjTvu-&xhs*6-cK`qu~Z zk_=!`2UM7gHz?StKx6`l&o+a{%{qdD6M{*xQ$wKuc9G;a#A`R}v5Ds7-6q&qB#okv z9Cdl`19V{`FzlwT`7gXE2F!~ZB9H~eqNK@}7#q0Zb|nCyKATsh;mPuJ>v7Eb0ao-? zq~O5@g`G?}EX$f!m_3Bv86^bgVO_?71@S`VHZCu!eOY66W0cArXGzjX?}6(CZDj-$$s5@+t#4_|U!g(yud(E;Z}ffxsN7@X$lNT;vt-1W*|E; z%1ngk%g!e6kkHx=2e380-ZT3=TItEd-u12pgf`~B*GPW)TSXskY22P@Xp31L>?)O> z0yjm*hTpmJ$7}1->K*Vz5E%T+(`q!X@M;afb@hW4F$Me2qLY0qOpxYnT|N%36lXd1 z)?9JjR|WceCdDZ=Sv$_WX4OM#q#l%@F1!UN;5+F#whDR{p$s8=2UfDbbBJNxMoUE_ zDh~{(Z)H}{Q>Hc)4cewlQiydJZGqO=!`xf@v~pl89fF*Xk{*u}erZ;6E2YnR+I5)GUM^v;2W18<~aaMCZj~@F~ zjq5?QTSt$Y6-VI^b}cd~yF?8 z96+I3)nkRDu3Fhs<~ni*-ltrj-sH0zuGHFWa@YNAiwnw#4A7+7R;* zs2g^aR3UK66|}n!wfd*R+g@>^BLxp^fqE7KhTqejRYoXko-0epEmLTS*u4?)%Za5P zCwrfQE&F{@k&jB#{tiSnMtrr|dQ^bBG=KqxPBTm)be z7Q_rP%t}W)ib8PM%#x(qPkwn0A#rIA+qDk>qUy~lRKtcKr`Gb~E!hT>kCe(XEQ&8L zL+OZ?HmI1k{p&pB$V_h+&Fm%0T8NfbS07_IRE)&I*z@m86PJ~}Z6rxj>1RBCaL`Dl zW+SYeDj<)b=9aJ`Vf{xEwPv&!`azkvR&`~*_(_D#w-JI>S6?F8We)mN-BJqX$Kfl} zMY@7)w8&pL6n&GR$9O+)#y@BIK>u=u3XLjKq7KqEp> zDii=vDpDv?14soRh)^m}f)oV;3rdPm&@CV+Pz@@IQVBqUP>2*F0YE5}f{iGpBBD}& z6o3r^B?yHgg#r`;Ql$zs0sv5oP$<$0(kN1cKmZhgAxNq;4Js8vr3$nn1prcl6oEiA z3dE~GgGwtx2}%f3DwF_?AguriKqUYJN(o2>2#Z233ZY636rn;OPz5RtD2*zkKr{l2(1F4K#CL%C@msTAxKh_qD1``6h_p1V0*wHT z2q4m}AR|JJA|pzaBA`-aX5$TDkPtF~E#pV?C4=Hz>n-^o-n zI8Zm`Go*^2BRWlsiSzkMjSYl`FK!U}<0uGrzZ1VZ6Q2y=r`h&+%sn;V? zLt8f15$CQx!R4jwB&EdyZGCVGFQlOoTGiaUjU3{x&6e*Uq`y*faJ6 z3HtB$*~YlQ>AzVIXL(FWX(9zj*3zr2u8>_i{1+_&pHmS=e0v@T;B<4)!Q@-eG$K7aqr$1=%I>%Sg8k_Z=xSzAPc8BeF$+r7zlp8WIr*p9e zgtQJxh`zvb&R{5#3QqD2s4<4Q2$zO)PvZ>|2u!|W*el*Yc9rvSSIa*EtQA4#F!SE# zO*X<9;{p0^jXH2nvvgrN05_2C(-`&(>@V8=$eTdE-F2_J99#2^{?rX8*-dxF9?Ql$ z??cbr{_NX+E_Bc%#h(tHHU`@mK7`=< zKIh!y>>KsInjTMb0#Ix$A_&LwnFgB}URDf;bJ#QMp2tj6o(6xak~YD4j`4%VO-j;3;5Q4G45*d5mv-{mA0nfL=Sv9Roc`~>dwky;>())7+d#m3 zhaO-KZMCtr*&fIpSk4c09ND;?xxw$aLbF_L_H_G9ckk{0Z^3xX_tD4jH3~_^s2ryg5n8JA{426i0XIg5f@gg97V9DwqXGSRA z(br^C@e|EX`0rl(-+j@gC^(Ek+%)R}qy}XA6JM}85u9Obc*S3L$IG2z-SR`JPdfKFxw(Ub{)ix{-}krfU)PVBfxcqT$soX> zcbwh$`t}CA3zX1?5r8{I&A{SJD^||=CKY(HtO8nm%NvlveG>JyIe|Snc-CLY$*`p;cBl;jvti!Q^xvWD)%6y(;+X139JVn9nL*DY# z)B-__j`bp1#eK~*D=?#VMmVBwEQx zDfWkn1oS(i$n=hi?=PUba|lP{MNPH)p!^GZ7qLK2ER>Z3sN)}~ z)Yi<|B>ES`BELT2e6G_N8te&eaKmq$Z2ct&bz|eaH7Lhms9DDHKDwS6icGhAgr~}x z>Ni}td#fuD)%G(1>cR2GFRRRXUNT);xtCs)$$CbwB%&+6%f+(=-X&zvHg8`*@}^+? z!*Dkkkxv8ky~68yf)mNEhG&&C`pE(PhFH!>WQT@l?AzSCmdw`)dDD_V9cm+nBf14s z3@ZAO9PP#~)sMPS7$I4Hg9I7`S^x3bCZQr&X^jsj669fA58Bx?Xd>eb*|pGjzf29N7= z7Gw=EPYw|w1H|P~IpP{`A1hushcMCNkP8k4L@)p%7sz!126lZkRz1+8!oHL>AbMKl z>H?u=lN&Fd8r~(1k4n%S@UHARc(m z=5n)_5FA?!si2kdf=?b{&kFX}99`I}>X6dn+}~g^S%L^aEML*4G@}udSP>NfwNzZO zmTxQ?1cF)LLbB%_hjU6YjIxF)2)?e=5*8O>)$Lhp0NkIaH%+vik;o@J7P%3TQ8V_c ztfc%Mi?52%KBuL?8iDKN)Mo-aOVg_qx{kV1v>}jH5okqhm;n`Z6;RM=6-R2T>Plsg zTe&|e=B-KI>N`oX(KM&2W@LB+fPDRX+&M~@maZ1@gaR(od$7VSz@}MlS}`QVUg>~G zs;P18kO3ctfUJlJ$xWzSv>#?n4ksB@vhl(MA60eDO;(c%Vj;jf_uIYo)cdx3bi>to z$<4&flAi(9nL+UN`{0@nfboa`J;dVqajNI@nW;w{9=txFj)1G<1Sn(lVGV^oq_Nd{ z7E~3Z4w;LRYqPe#s^%wDjM?Ctm$v}VtXIwv`?u3-^MnN5U;<3lZ$A)y9Wc&Gxo;sB zLs`ip5cJ_MrlXQnG$u1r-B%jg1b)IRG!x?9@$p}r1Yd|38`h24t#CSlA3I_YE)dxX zv-aEWI47vP>wn){a&_J|$@|#zzcO<_ZG_s+rk9!rdSlDE)-*V!bONbRtBL2Srt`ul zb_?1ci-%uzi#G^-lxS;YSU99b^IJ+>AgO z6$ccNMh^ILH`a53nMXm1*4(uxa>iz%)mCdrp*>YWbx8PNBHjQ~K&-#L={0FMuVIpb zf#%eG8)W>6v5>FUd^zuHlQkaere5Q}I@Hmu6i`~pwmkjUTO8Zm^Me{-(*f@K5o;eu zF~@bre%~5roc=FOtdy0nh(Rm`zo9l$MpbKzt>(J~*J{?x7HZWq2aS7DqtK^<9ispK>WS$S7XezjSmS-Zczz?UvM+~giTTkWdRyms+M zzlm!(c|?8>M)lkaz__IF1`C|9YJrcq!>!U9O}1@)%4JteMQ(e15zt?5iwtGEo?0qh&0L+LWAjlWoNF&LKXrUc-gVvB1NmV=?XK*AaBWcOb49*Dl#{lX=W>c-9xY|M$z~*6 z6(||HvqrcXac<{S(-6+#3`XXJXXjhFyig^#Vzj7D0E4uguw0-iws8z zp3~(Nh3jb2=#yJj;W8Y{QV$J7G_QnymU#kTSetG~s7aDfIhcpk&4+vqP&*$ITJX6d zhRW>+wVRKbN84k5@Q&@x=EyJy8JKtm3urtN6R^6*AXJzq@Px&cq1H(Ok|@Z=W{%5% z5iv)Bl-4q)wi4rC6{(f=hdU?ocw>Vi2LoCkQB~)($;|cQbY%IX&O8K51_Z()l7?5` z1Smq;Iv*=al_%8Tw{vb#eP*C)uugiV!G}++gB~HVe-J#T(*43N36Y7Pce(ek$DZ4F z&pg;PFlOR(sOXb>jDFWnbI*!W%Y~SP9pU2z5H#z&(^1n9E1$ChIAo>6u?7jvOKEE< zG7BiCp|~6mODrWa)fr{1iAww!&LuT2?@nKxED5s6>IJKdQev#f>tb=3I*9xyDPUti z8y25N1;4r%_H`Y~U?>mv3PL4xP*!el7tcG#*PB8Beoojb{OYf_^YJ57Fog(1MRRa)M|moPvOTFbX;w`?6d-~(3u=1ld3);*?p-v^lACdHa8L*8-Cj&fscXl8kI<2q}` zPP)!M;Fm~KoSYHya`0&Mrbv%=^()9)l2{XC8VL6D+h(4eZ6I9mLm?_6%~R==B9gpE z_#u{jMdyYYVaUtMR%x+fP^xzizLRBw7sAx~VP2koWXMpv0Iiaik+HGi^vwqS%wIr( zS#XD{Hong}!Hq5X_iuB*?TlZ(!@0J3n4^vIWW}%Vz0QyuM*HC3Y#LbSjF;~0&hef8 zGl;a)IsR2dQ})K%ZSlUn*fWD0UHQI!eA|CM>wR?VUkZOb69a@dCkZCWZF`9nr68IBSUCc+b$BYlU#)~^W;@qjk$Yf5fF7(mE1VM?_C zbkpoc$IPo;snG`x)t1R86haP>#w`CIMr7m&*lQ6d1q&`pL>EGwHu&mc+(370JCfE~ z>TD!LYWwRTS0-K8`W=#NZD8ZzIrSL$-N6nWPlq}l6bgn>mm&(Xsk8L_`+Pjzp1h_! z^bZ^f0Lfwq4Dy263N4BwWrqVmz9i_nuy|CkFd+DRywlrmBH~m;ft$|yE;iYG6Kju3 z7U7fXKUyHD45Ks7cV30ZC|$0$kqvGht!h4Coz72E z&hZVJ;Ddpdd1wegaL6PGFBl4e1;Dxi;gO&c${9kbDIxe`6eCc?Z(r;TtR89i6LBLw zK5a4+n`v^0ZRo^!57w_Q#OZBjsxE;Rqa2{3i6aMn)Uyre)sig(zZv*d`gzRd_XYB& z5!&-i)p12)cZ#^Ed1l+DWMcQCvS7nkgrMEU_zQfvL*xrbYu|M>=pR7<-yPXo07fOp z8w`tJIC#`onGLtC>O@>bXX%=4-mTYZr)^10wPx+%f3$f`n}e7TD_R%FhhFg=h`Y~0 zkCwTn^JAwm*|m&fuolKv6I{U1Fh0^dBE_C%*dTMWMXp z0|LPCS0uezRblBSDs=&LddaxN_L0{T1xmKKEXcP!LsQ_rjE;d#3uh`kcGMFwYJ_|~ z(1{>$@IGPO=f#e5h@doNlaaLGDwtbAZzi7N==BG zf{D%ba+cdMKAKrUA7D}Sgi%W14FNR`kHd-ML{0H2JCPbn4erZyOtu5`>Y${p#O_@t z;B}K2PkJ7KdGBpp{AWOcK)PZ8dM9K)OB51Ls)9>_7f*7L$?urvABO=*b{MUEGQ+s0 z3jOt30|%xL=1;JXDTfoX?F{xy-BpE2ZUgNIXSN<0;xzc(NT`pdk8QGp# zAa9P8$3nXW!2+er7R^V%VIw2+2q=Y;nk{h-9v#kE;<{^q!&LEt;6fM_ZY1023J|Wu zfn>!8civ)&3>};wF^Crwn<1ldBa5jpjL{o_gl19312;+^w(pUrM_*H@K;NAWJwU?I z%_MO<>b{S)jtgWvy@BeDLEn$5Ap_GR%#`jW$L8Fyo@@$;tNZ3j`KYZ@)u4VXHs~4e zNG?5KvH>Q2#lLTulya_c<%yI9Lntocdn|$BLeLjtT#ZA?f*I2s)`Rt^6t1>2s2U7O zXi}DFy(4t0F!@4|eY+c$H+nDiZ(j{qmVnQ^6WzHz$Yt+Ea=R6Wc^O1yeM&G@JV2(K ztpQEbDxHr69lDY1drCbWf{)x)-i=dCM$b`#u2mAkqszfm)Lcd$$6u1+WAQ_Pm{fQRVQkxzr_&xuP-h)m&v zS@;P%MJTo;Z7#}31GG<@QGW4)+9N6GI{+uHo0gI1gGj0Co^p&RBGW~gtm*Nq?1QY( z421gvsmEB3AG$d)ZAFWEa5EA(qT#JE**BysAe57>FiQhz3e&WDJA7dkueyraDUBSy z!2=Zp5XPcFTG_XucBhz!n+%jXF4mbrLLa^n`O8cYR6~VP&*x-1;tj6nii?aKTX&Cn z#_(0ux4Y-weeVZXnA;j)7a+_7i6w$-j$SVib&V-^-#}ad^#`Ag+&&wgl(=%W*`Y@R zA{NA{!bUuu4Y`AlK9R_ATq7d-3OcJj8CG{tw}8}X<#ZxK{cH_kpn;1BikFN6L=`k2 zj9??Q&>(o~I?uBNHswdON_qpfSFa~JUY|^_OtOe8IOzfwQcb*>K3{6c#Td;>k=c|D zN-@d@Od)mkuYp(R(54A_@WLX=bvCfvk=R@u|dh) zJsff+5hpEP1!2(xzPK$6I2GX!Oj$settm}??GSX)byO1T!x0H1TUdTRo(w0m>DL2&iV64-mgnZi&&1G>R(li1pk< zbobGIlEn%;CDktrCNaKcK=HhMypX>3<%R_7cjNTOH0BI|<##@Q0l=&m5QLtMWvn+_ zt;`2%6pJ-5yh0&BJECo?%c|R~*eI{M0(b;g5^8s$l?Au1v~b*ys6t}7oOH6ojA>I9 z68g#2jp442N1Mz-rq`;34;WLzV`IA9g@>_;GdAMhU^2Y$lZsk&haAqW+q$p`nfcd& zay8Oss*OY?(}u}IpH-{eWn!9^I|@jTUJ-z3QI}R(6*0oAE114jK-RpsPj@n>vYSwP zKo8nCRkH#n#Wjtd!zn1Tw&a$O+rfipAZf06OWLCtidK?HT~60qf*N+?LUF_l7K6qR z^Rcn0K4Uj9rMMu~fVH9PtY(pH(EXQ}WqI9!S(%-K!XoU;UiZ@?jGqdMNm zK!tB?hDM{yx<)`J?MX>#p2kD!SZ#L(@i-276?M7W>RE{&n61eJbF`Q4U@O?f$*)<{ zAGz$`tnM1xappg5=GZo{agIsb>6euSfKXxWDCLkWZbQ^&N_y*b)5^H46$bF?)3hWa zZI9BDOxfBj*Q(j?3ndd!D%h(N>jON+cqMPDniMJ%-FD6oz_!|(e_0SA(DROZ4PM*W z9+EaBpB%c1rYobBn@6pqj>hS1ZD(%-{F&;dT^w=%5-*5ALxXZrXf<1hRbs}wf&19h zGiG2M9tEm_RC#I9@E=fc>)3*3(Kd(`Sa^d|&+Z{fh7B97NpCY)QqrlK;Matnx+d6A zFo4)daU5t+^V5ml9XWkTK{~o2WM(57b1cUc%h!g?N0B}>jxsVTYZ?KXMKsS9GT@w3 z5=)rNKu!)2ZxH2~E+`V6z>V30uPEgZ`WB}p)Y@l*0+^zrrwqZSDE?K|vD$H(jf5e7lQmznLZGGNUR$d8GQ2Z zN{&=ucUEHmHFF4CKl3j}F|r64MU( z>szD5fT>Z+7$_6-Nr@;V#R7D|wOo)e5K3p2)#dBrL}Q>cnv_`B#`;wiiUh_-l^MlJ zmc>AI;?7BOqw;6dn?VyGa9SY@vTE$>DiWyMZ(9l7%a#nNO`H{!f{U{!D5FB?>hE)7 zAR1qILOB@Q4Smk-cPZ1wAgK{O8~~`TT-7-A1}M-#K*vICB_mCRYmf=U%F?&R5nONO z(PU;`gr~S^NAXnH!-r?9_SrO0Xe{+as%dFQkrMz!Q-f&=ipKcipJ|a()k*U}EwxDL zJ+^G)J$Sg4YZ{^Mg=FfwV@I+WD0C2}&1qGXE__lFA~{-SkXB6Nrv;cA(Am`wAFK|* zxaJp3e0FGgI$Td~A15S*SR=Y)K==f9IZmc>C7pvW9muN*#g7Qu+A6GotD-*mmGlu^ zd7ie{jSOv%c+#YE* zi&e3Y|HL^1{h($?}Y5m#v4OO)U$I?kVNs2178NdYcreEVZ?9J#IaK6*tzT zjBkB()e;JewLxw5cqlw-+X~H_?!x#D#QNBC1A9n$oj!>MLuA&agHC6GhC9@dxP;pn zd*Ic?tMUp!u9v_Tx>4^6wUgBJsh#1v)?S@sE9+9IFcI%BGBZn{5vlo^fKa>A#Jv5S zpAQiyowC)P&PQ-AS3_`P4THoDOr?Swe=cSq38g%FkCdVaAZRss)W~sxaud+x4{|cb z3JpiX1|%24f}2D^tm8D{+Bp-n)Ks7%1m#qFXS!jhmPftgHY322XR$$Tw+LK`bX&y`g}!UR(1eZ~)BSep#(fL{})DOhsY&^(HCr zF@V7DFWT?LXi45=IE;bXWm+43mtq}Istu*ed_0$s(HfQ~if9Ehl`pkk2j$TMd?XNW zOp9Cc6BSWoAci&QRHC$6oC>~k@2GGRP`>Y~c@`E`sF|m@NwtE^rGuc#VC@`zgYVjL zaSl|QicZdnb5f!V7v}+T&EqNYu)?vWb&4s>mK?IF+{d+tB_(`B5I+$YgjQ`pAwnk; zy1JZ_O2!7aOSPmTnFX(v)m6~YC|A(X#5fF$kDY}>hfFEg?y%y8xL&l-ghwhl+C@i{ zAu=J}?mHIW?zF-Q-a&ezJsFj5GOa{b5Eu4v~xPXiP2$!4_O*1ZD5p$tC5NB`bdxsV1vAZ(>=qT#1E>-*nh$#>j@;BAPQKbFid*tpErdTz#qHCesrU_pd+LYE5o zn9V|YwVjHa$?J}y00KtB%^nNB*}SwH+Pdwvy~Brqiu^$UAVpgeaeY4ceXMDEyB6qnM zhk7`}tPJI-87hdU%D9HoCIV1stxp|=5r9Ji1_Ra5{T<+$Ft-<=OCh~W>&MYaa{4ED z=z~6#gSx!FaveQkUB_ML?YI1CeFy07a|V$0rZ(HWai)FWZ^yUYTO*({zg`Y=UL+f5 zz{ucF1(X&^bjpp;ogmPNPA9_8_Y zp{QfM130YxOh++MHQ|t_S4xK-5?;Es1~^gHDu9}mK0y?-9PYrRu)t|Hy{l^M zP1agJBqQrm-hB~OrA39@JkC$gW5gNX6l6d%h9oGF(3ip?)b}Vco|GFt_BQjEWl?4% zFbIqCkxJ3ZfZa}?3}>L(OV8GT(ko)8d%b@m!H3q8g6~CM^QL#-j(E+TPjjvBeQnnN zX!CD%`~oN!)biODql)5^+wvh#>sW(E691#wl<9w+8RTE^r;;TzfoLOxyvs7aNb@~q zhf>oDjLP3);6CZefsS&be4-me!ZSqPF9X^l>3yL~i-ncOL$v%xrvyyEG$|1~)$CUTn5)$;>@--GG72pr)#BiIloDrXsY5*yIVc6pD(aS;8D0J`UoXuP$Hzz_azlN ziG2G+%NTjzHWFh^V7b7Y0=KpCFMgr7x3RKc5gS-vL)2UulPY81Rf0zW`2)ImBv|I! zCPnNW&o+Uqt*S40ji4&fb2LR`GWg*fB0?(HA;Nhilub+K!!nW_Lt8T|HU}B((qo?c zmkhglBz~O8$17V9eb9(u$Vzs~OR{S!>34fh^Reqnp3Ait7`FMr!-zfOdIFU>ooRI-N#EMNpLFq#^)9TH|?QGXa?D3v~>75j5=$K7}yIKFUe zH>16Ur~w{S5l*+eu55m9Y+??cpV-8Mak3P0S$#@0A5TxwO+SG{{D z)@y@4GZ*6AwL#;N=qtCbhv43Ptc90{A-QZwwX~N(p&ip?*4S&xcUnB&FeNG|?TZS{ z7ZBsZ>USaJ>5s2qC%cm(0-rOtdPC9#q?{9HlAOy{yc#?6s81a(JmRlP>PF&%e#d|@ zW#hL$H6hJ;PVob*Uz}v3@W!qn{N)NysY~oE(mSNAECMEYWR#lZVqFLo1kkT%IP3`g zWmWAK0eAw|P91Ep+NJ&PN3n8`5B{SCn{mX<{^SJI|KG!^ek? zXhTiGvbt{4mEg2dY2s|A_E9Zq#E~A}$6TvP7!kV*0+4b7`*ymA zA4bHq8{u$lMO+-6su2d#aS2>@NXn#ywA^W8er?L|3hj4H1r;H8ycbe$sLX>*gcV`v z1Xf#Og`=l2r0*}H*P&JH*89{`5A6(dt6-l9IWjK-hf+2Z#E;Y-M^D zt%oCgUiNy1l#L^)8QK3+qxW2m}T{CsLIO*#Un zG;u3KOD9fCnt-U01*!F-zYeh(zKOLlk%N=(yS=17#M6A|L)8Wlg`hqnzTaX#g4N@V zD6>IAIrI&Bx5qxl`oEnHrx;TZJS=_~&5rodey9$)u;YT zjdv&Xy8)BW%BMpgyfl&uq~eWnnba~;kZum$Rpvs3s-lH%#sssGrJ9gYnIWME8W0LF zZzsZK!h?t7wF~kiT^g-)jH@csgr=Da*-~{{{h);OCZ`avQ2S%PcYv80HoZ=9(Mvsw ziM8;L)d0YqNAOYS{rVNcKTyf^Ix``CDHw&>X8wm`9xqd<5^(&0;2?B5RDj{cLIU>6DS?RK(oQ2J zqi#5;{15f2Sx-b7PKY8|C(1bIko=PS?r8vtL?Y!MhlU4rYE=}zi|tEmH4}$Qt;;L~ zAsj3V!EuC$Ky%*2P-~^&Zwugp2?>n|DC(q&vVqvLrQ8)%(cuZ2(PZHB6XbPl_Oy8M zHj4<;dU2DlQo2;GhEX}N%9g3@7XbRJsj)W5-2lyR%OKKv%7E>l1S&Qw=N55NF`FUD zyoDKhh7HYDnF(F%QM0h2%MDP-X$0Wq*PR?lfOUb@L`2lazV8Y%Z!9PD!0ZWQ4rL1sWMXRIeR@ zmk!9nWkE(vTef**^=zcbUb;D=*=zu2dWwA!C<(&sNGE~U_i7STml%pxlu$}oT>yIb z>b2aBdiz|=DN&GQj`4#~y(k9yTKYZ6FOIj=YleF)7~IxVp(wP_}vTUl4HIX#584 z;5$(f5l1j)d@diTb`YhJ?j{=6PsR;NS3T^)!VQn2fUQ@6SdXNGWP>Q(s}~{%&Qu`% z(R3KK zEa_^6b%Pbx6tRlYw$E~^g+n6mz|7_ue4LH)3rPTxnGa#$@H4*0+<1<{DC4HqI%On! zG8uJ|7l|Mm!d!4DNYX{QfDW(-{qoAaFf$G05wcH~MiXI*&wD)N{Y&MwOj1eLd3KIB zqka?%wBo7T+a|?a4a1i-p|2Arv0p>#rO8qh1Kla-Ha>tEg@j4h6j&P}f-Ds+2!Ov~ zWc8qU#G3%x+=SDF?`7Gbe9LdXFRB&HqWsHAwKzoBe1RtSVQE<6S*c$KQXnUW=q`-M zQr+Onl*NPa;0UVfs=deT9(xC((IeD)()@J0&Ud`)opNdVe^2Q&pWNe^H{yPXVD>mx`fs5izfYpK?p+anmrxLVIW(5dLeSEZ~Q)pfx>yi~vDxpRjLJ$L81jB4CGEuE7CTY zMeaJ)P^LkwO6YJkuO;&P})d@r|pSOEz%e z+y&DOvL2`~9ckycpGJ>Wkq(^58dUe4Ptt53P{8#PX#w(L2M8|hS?WfY(1OLs+DslN zS}Hc?``tzW!BT%|m%yPuKQtai6L(Gg1LeNhbv<#XU?62M>+V4Lyg*~~t^7h5qs&9} zpvljTSC@$qnz@8fe)W&WI3Y1592;OryzLeBv#RGPBtEW6v6_5&Rcdg#;I`>ZWFn9XbG+^(ofzhvYg9&oo)~=fZfppG+rSso-u5jOjcGNkn z$P}Fu?||Y7eLOqjcFvWIHxsEa2i@t0c`I=gI;D1?pl@W$jVgxIl68@qLF#{k51TmY zMn*}5V0i4Uda%G9q@oLD2rMdtZtM-&zhMlLrO}wOs2rMvYN`r_az|B`ClA;@asz5% zA@?n+T!A1O5Exk6+S6L0nsWQv0upj}vZxEhr#~g#rZr)TdNqR&H5ML(aR7uB&43`N z`T{suM@pc?AD6rA5he#E;mT}Nc=pr2c>4***!`2;13G)Ir#Nk~sUFGB`e)wx*McGm zNtJ#+p%6S|-8=*1l`O9YHYWCZ8rVY-AI)l^hBF)S5b8Shy--!;BR-h~vs$M{45-U8 zykN+(>M7kB!5fBL5UQ&93P2f8ndyP@JBF-!gM}A5@hytOb4-!26)!Z5ln&=!{jl;Othl?doON*e);Sa%Nky-F!>H2dC=R^Q*S{N={?OK zzW3dS=#OqQde7@*0*S&Je^H7apA?B|CPLlMP9T z$Ft-MhZ9`#)MF!eVZe(Bb$h6hE?fZyFuoQbv`q!344^L$E(wC7{m>D;jTmt(PXl7# z(HuDr&*)E`QI!$(7Hh%f91Qfurdd2zQ-%|@I$J>z4%rJw4J05iZBD@1`Q_ueS%&XZ ztal@j9_U;p4QKpwj&a!*oMEG=Ex&WWyEO0 zdEt51`Wk1wy#2S|oTbvhNU4<0e2%%Nj8*6{qZ3IR!6HTJ(SB8yd18NJuX4OQ%iN+P z5(FU))icKQ&n)6Y4Ob$H2`*T{Eo1~NUTV0i?Lzl@HB!R7wdIM)CEpZ5^=?RnaS}|# z!_{w{dHV)OpU&=&B^}y$ffV=gIE2y3c2M!2c+8-q?d#GU7s0{+SZtz0{Dekf>#zuHs$1bPSUg| zNwYAef=p`?W3>3~iPeo68%w9RI1vD#D9{Blx51mL8#0M@8{cjy=JhP7S}EH2gHKB$ z8{-=@Fa@`^odS_(F{DG#q41objqRN1=Gi*{cVi|9W*LBU(fvqGSz{EQ6#H!KbGJ?* zl2o4y+ERl;4y9$&JY$j!*pG*}3YDZV9YOB)+qi`OU$FiKz?2;=!7wC6`AA_y{7a33 zk;&mlfNA+zQG1LSHgt~cCQ8UzP(nm@12Uv3ZI@bdt`$-gt;7+}7_bplDJv4PB_u&z z?bS+v5d!&5Fg1wRlL;^KaekHLYsrrK_Q$=6#AX7WGv1ni3Mi?^U$^ILWW)+$-_^1u zD{eoT|107ch>DT^n^B!HgVjuS{Iunua;k~a@V+?!w(^uEDfB7}jf9*|8Ec=P=@C_q ze&PkO`o!I8+f8Xgu_F&)d(yrA63 z&L+|`6E&o8HaWJIBk-*ejoZkE%+%8cukk79Y@gLGzE*`uWJS(Bx?3);S#wOZ!H+9y zkT=9fM@XQ|Z3C-bk5vt+$%X6ilgLaDwZtV8u-0gGK2#;+IaI>{KG|NR&|S0{QzzQD z=%t1b?4sz=!tKx{h*y1}ELzGaYv31Rmi4ElV72%nmAEWJ7R1=iyjV`=9Hk}@c)4t( zw_=l@P?a*;?;Bm)Hx`U4I1>h10mXbL#*+iHW2h%rpj4Me-N%qkvjkDiIpaF4tOiJ2 z!@H@=DnUj!&m21sfI(3Ict>a(JzC-pDV3grc=SA zR76us(@Gpv6ujlU6KMqD?h=@i8)r!#gX4@qOjEiLBq>^bn6L*@Rvi00cs)oUGYwBO ztMOWzp{Oc_(~}ciodLKV`IYcH?EQ&Oc^Oa>jXNC7327P@?-v;uA)Nz?jO_v9W68`EJNb&$OcK=#Ol@Q>`%dJ+up?KLnF@sWA~*%;?w>Rj@H(fQq34 zgFhUerYt1UR9=IFjrh7)?wO{^Dj12=xi0>EOv6p6F7ZFQKwLE4(`{CfaZ;(qja`65 z>XiL(VlRM}J%fT2Izw5Jo<PP=<2zKll;>UhbjVB2-QgS1a@|pBqhcd$0EnM2yup78*LLGnqQW)p zi{abU`QwTz(mG@$g!%9nCXj)*8hI5mBv2@CCTYqSDab{kq*b6~o!*N&rtWW3Qob@X z1JU=JU4n6-EyMt25IMR9{h7&r3h6*K?cUvZ6w_CwGGc|Ru(3;OCQoV)oFl=5Y1a5d zTo^bY1U*p+@i_P|mS$fZu+&j=DR>!*75(0!0?aMiRGRlu->Z+#%ME;RJAqhMtX{fA zOYn(vJ;aPjl?(R2G4sD*krBonC^VAP*1FVS#S2 z2Hbgo)q?bHsM!Fls6a}@lcoCz`(=07gS6p-m70?EGAPguxK9Hv$%>Hn968u zagbrw*VG^uDXe#dz{q%ns;a*UC<{2OBlb|LtDkk<-9RwZFzbqtbQ_v=g#*p_UujO^ zLaKOz0md}24M;&$UkpKaF~WMG-tF_mUaIu=l;N&ZVsQs)PUz<*Z**B55mh?5IxGbM zn?^X|)Oxgy?Xq^uqbe;|&Dkr-XkXA5t>tWp8d(G9D4Y_pryG})_z=+Q4L)inWe_-1 z1++#{fqhwtxiYV-r_$#g+WD9WM^IG)PqtR^lMG{ zuPR{Ecu|Y%>aQS&g$nNcFuA8vCOIAfk#2q$+Fkp?z_%I8Tamo;X|$z>O3W$Qj9L}l zwsIyC`yDB~`AW*4aJ2`(jXBRxC|>G@}#@y8CxZY}r%p1?&jYB;6b5K&5zkh2F5GOk{+6T3%+7R%s=ABtGQ z6qX$+5d<%YH+cT}R%^4*;rro12meSa3;rM?s0;jk*Y|wm{QaiC=kysruiyoK z{T*_P{&hc%Cv27T5JqHQx!O^)pu8QW2ry;fYw{b8Zi9@)eRt)-N2UmyL0*00pL+~s zF$#~LaS18QHwAOHP%k~kddsX$6O90WFv}ka}y2ui>XQEjvA;h&;E0CG8(}Dyk z0Q;QXq=<|5P!1zKK+}y5wgJ*Hj|I9db>pEsmmpH`mJM~fEjdao0MKg}F>NTb)u2kc zSxm_UVsPFbJMxJ-gLvWg=1_QcNI%DPRSv|2)JM(I)3RF9ly#P61vEvj6OU=`t%UDk z9i{Qkby>y$C|qv?j+uBmz^bVc)*INY)>rKz+?(>y%(rW&>k>1(yV<>Z0E9+Wbu3=5 zYdqr@*$ua#MysQzrpm_vR!%62q(&fo_%kxg!c9P_v(~?G_Ds;5*Wn6~RF_6xBQ21o zPL7OdQ)&t-1R+E@E9n@uaXcOho5v|NjWDa6tUU?9dVvKoK??*`Q+@ORm~H_)inq_! z&uR_t6L@U>s7xFTXG^#33KQ#oFyHRK#XvlU>x*X0Bz3ZXJ-%TeKDt!~Hku0d_ zC67`V1KJb|Pq8s=y0_8I-TLEdLHE8=C47h672CoMilB8GWV27K@ zuLBloW_;ofu-5g$pN3x?Epx(%Gx7Zu4_!qm9a=g$j=P|}^J$sPAr*NFA{QYa4t!%v z*!mt?6$W4zAVG9$P_kfy#A-#!hgPlh1KBDS58Fpwz?-*b(bTtlYk(yS0^Cwd8MNL3 z@Li4BdV)23&u_&mbeZ=L$RrHO8^uU?GiJPNjp4vzVCU0qoRZc;u zDypjmC6QuFM)S_~rTItTu0$UWHz~AlU64;9UR$LwiVU8>Z=VJV*!e!@5Wl*#M3v zG^q(g^mYKABzl;7k`%hlRC8U`;za`x7=*jyr@p`TBZn5(WB$ypKKaETT@)#LkQ|D2t{@Om)B{am@&Ekk6WwD3Xm6 zxf6Mw5|wtLn;}*y5032MMO+VwNp92L+C_#5k2>3{nDcCL_6~Hd&%_#D;=^Mf=@k`C z6~S>ZG*bejv+=mKneH+_Hp;N>QtGSs?B5dP@Jl$#w;te(=r5-9V@zGx5UwX80;5sM zL|wXvuolIsK!gy91zJGIKnmX#q{I404QTP5^S0V& z>b*U|!4`GMefR8_?S9w4dcnusoV{S;aE_-eXpoPH;(0c}YMmDYZqZ^~#*`q4r^h!? zbPdn(%FH97Va(+>UzK2jj)nlx{u6}2UUU=t(nIVlRkyo!`VWqQt!Lv<9!q8MZc>Kh zS<4Pv2kJXx>lQ{RIIl3R!_@pmXd3 znLnclmQ++hLzq2Wns5l=*wzUn6?_6Lh)ag15$*#p5eu1ECPt`5Nf0Rz_RKV?NMJP> zRo_dcL_bi-yNCl~%Sb{maPjisL&kl*EI3_UD*k3Y4F!AhmYbXD-=56Z=2Sq!#1g`a4FG%nwy4Hg#di40RU*O z+&Zt-!CauZfjnfNu`nz3EoXGv0erepc40s)$>mH%=4}0@K@VnJz0S-MN>+Rze8&hu zgD#&9$B`B*6~*&A8g_K|%Tv{5IOh^{&@eu@j>eNGX5@Iv-{rTtq7 zB{f?5-%1Q2BgfCQ!HTCQrXb19^f*twWJscw_XX_*;XAmy<688kj15_kd zrg%=xS5$N*qlr9;=xjpLJ1Nv5=x&9yQ#S}7!ZVB!mr6QHFCdlLzzEHK44zm%y(-q2 zIihU~ty#xvj7xqIoO+X1h*BPHc$Y9Qo`em+fMl+a>fI7wq~od7ai(D@n=O`&+>cUY zW*k6Jymi>4p@i?*GBe)>>88PkZr|3psOs-=q&EpV4!SQN3Xit78j(V6J)VlM2|#O- zeen6igc5^%e|lt5 zL!=jDXaH}HU?86IzKuS=jjTvK1Y#Q`@*+)#Af>@!pne+x6tF_(WxB=hdyOeFDWTuD z>-Fu!C@!949f2->n!S_(<=c6pu7v^8gxHWcxgFnFpE_uuR4`Tx-#FMnuNY?W z#8JusY;QKXC@E=p%x%Ksq4NBYsCO*!UgXk%E8r6OCL$9)+EfD?S%V%@E-*&ZVKO8G zggFO*+-ue8C1*M8gs1d2K$z2w1+@4otSLb=Emw37g$`t*Co^$K%Kd>6$s~YM_lRlzF4lQ zbX`>wnjwEDG5^SV<#%epB_fqn~lN;EppB4Z?hN)g`AEXN3RJnVuK zgLXO1@ZyALa%<^X-V}cVMEPXS0-kJZh%53ctOLS!s8t`!N{`fZi)TpKR<*~9iYU)T z?W+)4h8`nE9v(Ul53b*>_f-!^1WQV^=~r|QMHdLqZ>3CL z614ej8(Iukb1AAXdq$%-ONV1tq9K@#`c8nR)q1gHc*d}1Mr<*0g_u_g1DPpRAu*Pq z8N**N`)W~mH@&(-uL)@=!OzO7sg4K|Am}d=-U4UJ0t?S$#d~<^7_5G52I&?&{qmv+ z8hLxqcHC>NsgK?M?>t`Vp|-yDgFfr~zBl|N(NI$rNFZV|;U>MGt>1D-&xreuCXg&Y zV4)YlQC$~7hyddsK;w8w2dUl5abg!)5S%w7R~_jtC$jy^7fa z%ez{)oZv)IDAM~^C|UhBB1b2Ro+&0goCnRP_}I=z>}6|EZ-RsZ*>uDlCIF}so@$2x zc*1@32x!PzE;5#LHV}0ILY#~upNvoDJ*zj#inf6~s2qKq5!X)ljb0=x2rnDr*@o3u zoOJY*74}DJFqs(v2;~h2c(i(9bPlPTg4(PgUYmk`tb>j4_kl)7;h%;F({Z>=H7{15 zBrm}%Qi7oX6Vh66)u=F=OmywqpzNItRglok4wD?3jqKZ&v@I4A(ai_Ue&AHl2?R-# zikIAT%RmrW_@E&ol3}6YsDNnv0D@qW zJ;3+rwaqR%&$LZ55IR9f5jx3QnoNO}&1eUwo8&O0ij~6Z7)_q{s747Dh#Z=sJnnoK z$gf4QiI>MSMsHG1GX5Bu z5~rU|mu$Fiy<^mqr}IJIfOGJ_bwc zuOPQ?edINPmQjp}wns-Udp zLJCZe;xqc;EW#l>i^AzK<{j;)Nc~fsp(@)up+6As96T0>1$6T71I8@mVt&xV86#+&mt& zS(XF@MV@$u5Ns)xA+r8_6&<@b&vQZ&^O|R0p zw4__leC`bWI|=n}G!n!DVI=U!>!l8h74h19^wKlX{bpdG)WE&%^$53UiXC=vAqOcy zcVIKahb?c8ht2oB!bV6CYj^e)7*N=mpbI_?UOwkUVokgdTWI(|E5v~00k%|bpzbwN z+u~lX)|5J#FJ-x9r4e|_wm}_@9 zMhIn*0f+#)Ca4F6M*2ilNL1911XO|az>iM*R2v}#FYkT2B~d(!1VS-3JZ3El2UMVe z6|)qo%kN)Op1Z`);-~Q+jBk|XC)ZMF%Bik z))PqfO>m)saRSH7&i*@ZXOxPKB|cB8=g4jimb@v++N1?)y+g$C13>*|$VZCV!`a&k ziH?}VBO|&%5QKNqJ_7D_G1tFaAtx0g4z^tC4_#@Gckk2PHuE^^tx^p)=Q`l~?lX(X zxuabB;n$4?#I2vf?kxc=Bg4r^X%+#F_T+QfyqYPn(S}HQ`>yGir{ELgdI(A?%#aFo zA*62K0`N*GQaSDk@Ut zT=~sYL61d>RpEI9rY7Djz=rL60mQk8@d>l1JENhcLMZ~H)do#-%V?sBUxoOiC2XW( zvJ)?i=oEcSRE0`kkn~W2znhBF6%wX41?O? z)%!C^V>EQ-*&60bcLmZXc0LLfJgDj%by5j-S4IYlpNjoascu&W=(O60g!C$v)+qg{ zonu}4^GzNvEb>}n=gb>xaw z^v~<3p$`E~<+FxU(?2;sG1IQx@UPi-fD@n&GunzYK&UT!GpkHnh-C zn2b5CO(Cud-C>jHAL6w2=NT!f9@8ffiYb77n7-E3rKE5rT)!1KD=qU{%o4d62DkvA ze;A3;@(C9`9>CHgI$W1T45@?yTpGy(FpZ#Q#F=N^A_L(mcuMMUv{J@U;3rxE#LmaY zBC?qco+r=XlPcUdsoZwJ5Tt3vLJ6DCO6;s^G~?UGzHaMnst0IfF=jwC|;q9Mms)tbrAb4>aG7TwGr z3mT120fr6^V!(40^7>VXP!xRWo+>bbli*2ul|h=T5T0gh>QwhVA~buC(C141CGQHO%IxEjlfe+vTq+Ci*x4zNj{<3Q&|>Ul%3_@=|K))sEJjG1R2$L5*Ti zF;Pq|Ek!|U!lJG6q^=X<3M7E{X0S#NM*xcPY6qRihC~Ey&p?I6W+*4)4q_raY{*J4 zjdI=DB*4Oay+o!rb{MowkP^(oWCOt635A%ueGaG0psB)*jNPDL0y$BxZJ=?Roj?e4 zFfu9#3BHyO%pI+?%mQOB2WC&_ikgPn0wch-#L)w@Fz^bHaYk}5mwcpuIHG9GkSvmO z-6k!3%Vk3BW{NN*AvGin)_@=?EaP>xc_q-L<(C=1kLVq)1cK5~TNgHz%77df<6A1N zu-S}W2bAFEOvFO|Rd36TNEBk{@EUs#$7LTCjsq8~jLt}8bOb@+VaXAGbv%^k>43mt!mnZEokk&;Pm#{)0Gz7^ z(O7BrrQZjkG^1WMU8KVu&iM3F^|+`h-#NIxH(_6V$TgB(8ZG;J)Vii%MdUSy6RYF{ z;;@FAUf;ga0Qq$({WIA^P7|8Qs|t#umx_C{jNxfMl_8F0e1N6ucLmgNnCPA5mvFt# z3&hQ#Y-}UYVdNt>oc3lzgLsdoRQOH=Oe2NCOuIU}Q zI=n4$)=wfHDmtKN3-bJX;pt=U&r+^8yFUO+H(LFLhuO(NY6Uo0@#8jQh@Ko*LQ=Xo zF{$({JocUuN7Xio<1-*-wJPMK(KH^=v1G|j4>c-0T?>Mc;j_hWcZWr=;OMOG0k26( zPPJM{Rup!S_PMB;g^9WFIraaf6eyo9< zHYS$Q*y}pLi`L3AcgX{dg3a!(!A_*g(P>^iNNO6aF5wM9P(}JF)Cq(O>1IyVn8M&3 zg)y?Au30jUzOl$gvYQco9#-ZKWjZPA%3oEN55Q9?fR4>8XhngNiaoVZiPD1aMG{?l zJmM8Jxve2w@!Jo)v=s~h8=Rb~VFB}*cUGvV^iF8C&y7xW_AGA(<1D){>bf5mMOB$c zcKTHU>Q1n{ScK%0xL)$Zp1!=QKwn-Ei`rTa13QC)_`;eMyNIvi)7ZuJKg{x* zc;80mk5(070c3PjYiZZYC#vBLR+$uyY1cgnp4*uoQGg>@y=K#b<5i zQI+9AcP+4$hgNpvuV%ftl6T5G|68BpQ#X@R+FvQ{*6ECv^cEi3XQjt`Yi# z_W@!+%~v6!CqWNX67;RlwWiFHwR;xXUcH=OY|iU5wL?1vyRoG^=UGbeK=lCo1DQ@s zRr#2dSH-~7%`@&wxkIs)EKB*+&Va`+K~>Y$$ABgW8MA#lytV;BOErh+xPa*@rM+|gP^Q7%hWC!HlUgRn8>sRy_s z`51loO^Ht%Dam2Cx|+zMQ2<_K~dBv z)df=D6adb2!U7@Q4~6}Np{EWRmMrpaA>xgDzJ_v)>$uJ{<&$h}jkxokG|wIBiLcg^ zn8%oP<=|Fp1L>Ntn1d9$9(?i{zL=-{kJ>uo@3pjTCd8W?yww(`R3$d zI3|Fvlr&+}WT8Qb>^AEw#S?Uu2eRB*qbX!5_>IF11YDTb^zwXnez;3Ox7u#{h=^S|3F;CudEt)r_1uGUA0%0UN;lp9byF zgpq!&cT$8>T5_liXqJmWm^I_d_ARp9&!}B8$s`DDR5O<1E0BC-WI5RqDfZk3O$!8s zhCq1H%yo1`(D5gI6qM^m?q87Fk#?h$^C2a$O~4)wuo9-SwW*baNn5k>F$s#wSJQ<& z+nEn=XQP9tQ$@@~X@SZKq6U1>klN~u{N@vJNiWBcp&c9!DSgJ;lt-3rE3J5%IgF7l z1E{*H0{huX^Sem|No|4qL7v-ttqul>#Xu{G2Q;nL&@{*q9ogxCK>j3?4WV7kzQ)SWnY38tCsC|p^25dO&Y%Y z(r*ATxUR3!F=K{REK#BHT2yb1J8z>TfKhmfQ}Q}hx~!$pxE!0D(kmvNUk)kx*W$v? zh+9e?wkUOZar7!;qsZau#8onq&~P+yV~IYCy^LdZwE#*zV>R9oUO_xWZgXIMCaGcz zAe1mEiRTY44@ApfQ~6v&WWyu~7$_Mjp;>3fUn_~!8s21FaF#7zTK7rvDLpbM8mXJL zP;2~vGAkS?fGD~Z1XH!qQV0kVK*(MR28*D@=T6fF>_ZMT0um98o&}yM-vT8A>2T>S zT^KmV;cecT@>9I*XNl%G z&gN?48c&}6!OlfuNlXvt8n7G&%uX#hN4RFjLl9IP} z@LgWy)!*GR1|lv*$P)%YY--iQ zOj7wVRQv-Gzb#AB%;f&c3hBP4Z32#?Nfrtasdlrl7+OxBHDKI1*$VWDq<5~`+K_^v z>*I~kI5@<4?z(fnIi2X^Ok(#Ne$y^+QQuvib;kbDSAsAi+H5N|)Cet76s91VY@&pf z(H_;)=~*KtEZ1$|K^e*;>CRW6DO#e(ykENrfJNKs>x za8FT&1Vg|hLkbN+iYbkERY~!n+C;s8a8mn%k!gqnZ6OFMChY8=r^SP^q4(G-2@gz_ zvr_Az=2R9U1k`Rt<#G!Vtrf&q)B>yxQ#Km)91O~X!;+6kM|e@z%*O%n*x~J2McRWT zg{pPTzDZ6x+@uF^owL)*>*YL{Npq5b>|QI8EI)F%=)Ti2t1xKbWk@+$y!&Z#gj8W?F&mzJ(A7 z8YjzUk_B!a7c1xa!d`Oyp00x81I$cBc9$I^-7TKMm=DXwK><*z(1v)2wa7TVR)k_z zJ|n)SxwYHp3-NbT^j`wHjci$mO4I<4!-era8b&O(%;>*X4{A&D0p5mci1RNI8DQ)t z906%Ik)rf1G@c7I`%`HYT7;})@3|IwS&$)A<`Rbjb#RGPD7dMyy)q!QCeOhvt*$Ve z%YJqt{nv+-O&2f(rgET@v_~`FTf&prr11RE->$M2OyP?0Ks#I;7kA5VOyJ|91Qch8 zB*XMsJT&=bJ5T3Q=mVjP4lCA1Xq#K3M|K4h0I@mcfk#XVSy#KH$%{r|<>LHqU1HFO ztfaWE3%oRQCbL4bC#R9Dg^7u!CJsgdiOutgj2%G7!pKAIWg(xOI;a5UL+MN-1s)JC z(+KeaP~m`85NLpmoVe4Hm~87WG;aY^~{pG z(s{8OqB}R0sHj7swt>be&heaD*l=VGQApaWPn0e|DNqxTfi$jH*~B$>?qS7RRFR<$ z4e|Wi=7-qhL1Rfw~T`dLm#q7<3A%P@qMW$u#S^P zw@z9f>VZ)~%=sq(lsE-MbH>62!yH0OXk#h>(V8oA9!LtZf!9_P0m2J~PMB>zf_XP3 zO>)yq8H}lp#aM+vg%xQ{nfIIZtE$UCH5E z?>~LXpS|PG_qO}3xaS*do_)+g2XmRJYCb~}yP9?2^PYwQrJD*RNGuSE8&+`9Q2tIU zEP2IGNXJPdzuMj*kz=iKZ$jc2;V;MDQ!7#}yit|M12CM{YUg~U&YxJl6yVF2`cvl` z1A+n%a8qa&GO}wyP3x-|Kqg({rMiaU*}17o)eRwcV~&A}TdOxTZl%__Vdjz#F;Vo@ zz%e46{DKZVVIz2*5Ldi#=auH++u`B+#2q6l06^)+DbyZ-;}c)Km zo9)3s%cY>q5Hc8~4zyV|k6WHoI;vk#L9xMyW+1zFK&v1MRL=oLT4baOTH%F6PMDmN zh|wtOh06(|AU()TiTu@Yp5VMM~5osPSA#k&@BE*@q!4Z@E^aCNZ8Ppzmk};sM zQ603dMM2>1qv<(MQ~Mc7%6No*IYrMHo1m?GhH7+a1~rG6={v=V`@1XA`M{4 zRR%K{H@Pd-Z$vkIwi+BlZf{BrU!qk}K^{jIhS1cNw$|m@ID&S8bcB$u!M^gl4U|&a zk3&~#1de0XEfRzvHKQ}3g+$ZNl&(xooCijI z<8fi6nN5*-&Ypn7K(q>)u!DgH8Jtn&Y6c=HmPloplz&lH5uV@~zl6F-EXUSN}&)<<1Qv;nW5QF2xthTE$wljH^ zHu;A{vZvO@8u_e3Rjk(CWPnba#5Cw3gOmpNfha;!JiuZJ;8Y{vTpgg=+N8kw*;Yq% z6MlQvn2(>wa*tF?p)AO!=TfLO#(kv5l|6)el?Ylsr^U_@T=e5;J53X}FvFdC-br{@ zLOnIg4%j`FCRuRDHq%KUuSQy;8*p}dXPez3Uii&O<=LcFg-FayEmeLKL&JR?g;>yl z=cV_#Hz+B!^b;Cae*41if>uhfZF(cz1w{heL{d`3=@1};+LK);HYJT{B1l%2Gma!+ zl?J9Y(B6XsC1#}v46e0k!NC~YWXgr+>H2&diB>qxNy#8AvV2oGmY5fkZ9;-0ZUZ_f z>V@Ufj3y0@;)Lk{wR#x&dnwJ+%&;7hv3%p8LZRy$mKRYvu!F4Pu<@Ev z@y^$Tcf|rX&9l`@v`GwjogJRKj)dDKgW-2ntZGd1s}p$~ZrBb5*Ouvift)9QA8oQ% zn%8{Y6f7@z9mNb}3S27eiHZo}a%SDmyt8i=P4!D}%ba0@Cs}Q@qLew0qq|O~WzWE} zknsh$uMZ}I323WhD@eMbo!J)P@7m^WYak7#VILf!%zp4LOy6R-mguI5+N+*7DpT1y3*{d2B1%D##4_2PF^2Sr!BoTvh72E zR*8HPAONOZDE!BVDnldV(RQOp0>MlnaENws%ATuiJ-2@@0S_ZBaHHCQ(&9AKdDOep ze=3g+SB=Ny6X<3j$0u-LwSqmmWmy*qMt$gJKB8uiQFz;7--w3Emv<4mAJp=&kF?;W zNQqE!+}GF%)G3V@pMSungxRQ3K*Qg!$M;LHjz-> zOgh7O{(d3jj+zjkNgL}#3n*IV+`40y+kowY6sDC`ScH!wlbo$Vo%zUe0(k**!<|$K zKw}icWRG%8($uQ|5ntakv`5W(Ss`H7#_Cc?g5SQK;JF<~W-)?4pp|^EDD-ahZOzB{N`y zC1W0|04TwM0vSErZAUOr6yd_Ax_fvwt1D;+3!x?HzTc6`$TlsX35|78XMlh!_AZm# zQ5?NWaP2S*PbDrqoc5#A(^s_kXj%JPh>*s4ciFYngz+kiy=fMo6qmIL?8xFUDe^+` zbZC;96Rkumt5~?Xjvz?l=?oYttZmG2tSz~s`nz;bx^@S*gQdBFS8%CE*ht{mL_V;T z$q;vM`y zJ##3WL3FZg!F!|`P@*UaN3q>R*GI>Q0ydG#aqCo(=_5b3weU<2aX3B9*a9ZS%n~6m zLF#UviS|Iml?bH;sRx*&=fJ*A9x73;p=R{gT~B~`8JlK1QOp@4!9 z9Hi!3Ht78t5Au~yZF%BbwkR)<9TdRlg!{Y_t5xr1n-9j#+a_iLnH!a&6E4+A^XvM& zTG3P*H)?$zDGy`aS1}YWBtIN4RRzD2JeIQ6;?^kxG2#s(_RC7etHu+0gcNvw4l&vU z`9vCw`=I_Yvo0RqAjjTjK&!g&fsh*LsJcQVZK?%QkEbuQoWSpy2f`8@GA+{OL_~LNqU(x>Wo{xms}$$qk(+8R zLY^><0t{lt)A;~nOjDF3aDc~SO|WB2F;4A`4UUkrEYDtPY;aX7RQ}x2P}6V}5P5Z< z6||ecb=DYzjFF864?17F2+IIQ;%~jCstE%KlqhfU&`4uVKh5q7YWZ1xby(G;{59{_ xBHCE1TNVb+zE+x_tZTND_AqVEz5B27iXtceKt)g&{-^(oxgwk>NLc*h3V^#Ix3d5M diff --git a/vcpkg.json b/vcpkg.json index c8b9831f013e..eccafa372cac 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -80,12 +80,5 @@ "libwebp" ] } - }, - "overrides": [ - { - "$note": "Remove this when the custom port for libmagic gets removed", - "name": "dirent", - "version": "1.23.2#3" - } - ] + } } From 2e25ad2976bea2755e1119b36f442795a56f3727 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 12 Mar 2024 14:53:54 +0200 Subject: [PATCH 239/456] Query Plan REST support - part 2: rest client (#4519) This is part2 of https://github.com/TileDB-Inc/TileDB/pull/4516 , adding the `rest_client` part. The REST route needs to be added on the Cloud side (the actual route needs confirmation/specification from the Cloud team). --- TYPE: FEATURE DESC: Query Plan REST support - part 2: rest client --- test/CMakeLists.txt | 1 + test/src/unit-query-plan.cc | 421 ++++++++++++++++++ tiledb/api/c_api/query_plan/query_plan_api.cc | 7 - .../query_plan/test/unit_capi_query_plan.cc | 246 ---------- tiledb/sm/query/query.cc | 4 + tiledb/sm/query/query.h | 7 + tiledb/sm/query_plan/query_plan.cc | 20 + tiledb/sm/rest/rest_client.cc | 93 +++- tiledb/sm/rest/rest_client.h | 26 +- 9 files changed, 553 insertions(+), 272 deletions(-) create mode 100644 test/src/unit-query-plan.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d57e8cac93d8..10cb67d82fda 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -182,6 +182,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-ordered-dim-label-reader.cc src/unit-tile-metadata.cc src/unit-tile-metadata-generator.cc + src/unit-query-plan.cc src/unit-ReadCellSlabIter.cc src/unit-Reader.cc src/unit-request-handlers.cc diff --git a/test/src/unit-query-plan.cc b/test/src/unit-query-plan.cc new file mode 100644 index 000000000000..f3fe5ec7fe02 --- /dev/null +++ b/test/src/unit-query-plan.cc @@ -0,0 +1,421 @@ +/** + * @file unit-query-plan.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Fuctional test for Query Plan locally and via REST. + */ + +#include "external/include/nlohmann/json.hpp" +#include "test/support/src/vfs_helpers.h" +#include "test/support/tdb_catch.h" +#include "tiledb/sm/c_api/tiledb_struct_def.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +using namespace tiledb; + +#ifndef TILEDB_TESTS_ENABLE_REST +constexpr bool rest_tests = false; +#else +constexpr bool rest_tests = true; +#endif + +struct QueryPlanFx { + QueryPlanFx(); + ~QueryPlanFx(); + + void create_dense_array(const std::string& array_name); + void create_sparse_array(const std::string& array_name); + + // Vector of supported filsystems + tiledb_ctx_handle_t* ctx_c_{nullptr}; + tiledb_vfs_handle_t* vfs_c_{nullptr}; + const std::vector> fs_vec_; + + std::string temp_dir_; + std::string abs_uri_; + std::string uri_; + Context ctx_; +}; + +TEST_CASE_METHOD( + QueryPlanFx, + "C API: tiledb_query_get_plan API lifecycle checks", + "[query_plan][lifecycle][rest]") { + create_dense_array("queryplan_array_lifecycle"); + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_c_, uri_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_c_, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx_c_, array, TILEDB_READ, &query) == TILEDB_OK); + + CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_ROW_MAJOR) == TILEDB_OK); + + int64_t dom[] = {1, 2, 1, 2}; + CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_OK); + + std::vector d(4); + uint64_t size = 1; + CHECK( + tiledb_query_set_data_buffer(ctx_c_, query, "a1", d.data(), &size) == + TILEDB_OK); + + tiledb_string_handle_t* string_handle; + CHECK(tiledb_query_get_plan(ctx_c_, query, &string_handle) == TILEDB_OK); + + // API lifecycle checks + // It's not possible to set subarrays, layout, query condition or new buffers + // once the query plan got generated. + CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_ERR); + CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_COL_MAJOR) == TILEDB_ERR); + tiledb_query_condition_t* qc; + CHECK(tiledb_query_condition_alloc(ctx_c_, &qc) == TILEDB_OK); + int32_t val = 10000; + CHECK( + tiledb_query_condition_init( + ctx_c_, qc, "a1", &val, sizeof(int32_t), TILEDB_LT) == TILEDB_OK); + CHECK(tiledb_query_set_condition(ctx_c_, query, qc) == TILEDB_ERR); + CHECK( + tiledb_query_set_data_buffer(ctx_c_, query, "a2", d.data(), &size) == + TILEDB_ERR); + + // But it's possible to set existing buffers to accomodate existing + // query INCOMPLETEs functionality + CHECK( + tiledb_query_set_data_buffer(ctx_c_, query, "a1", d.data(), &size) == + TILEDB_OK); + + REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); + REQUIRE(tiledb_array_close(ctx_c_, array) == TILEDB_OK); + tiledb_query_free(&query); + tiledb_array_free(&array); +} + +TEST_CASE_METHOD( + QueryPlanFx, + "C API: Query plan basic bahaviour", + "[query_plan][read][rest]") { + create_dense_array("queryplan_array_read"); + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_c_, uri_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_c_, array, TILEDB_READ) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx_c_, array, TILEDB_READ, &query) == TILEDB_OK); + + CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_ROW_MAJOR) == TILEDB_OK); + + int64_t dom[] = {1, 2, 1, 2}; + CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_OK); + + std::vector d(4); + uint64_t size = 1; + CHECK( + tiledb_query_set_data_buffer(ctx_c_, query, "a1", d.data(), &size) == + TILEDB_OK); + CHECK( + tiledb_query_set_data_buffer(ctx_c_, query, "a2", d.data(), &size) == + TILEDB_OK); + + tiledb_string_handle_t* string_handle; + const char* data; + size_t len; + CHECK(tiledb_query_get_plan(ctx_c_, query, &string_handle) == TILEDB_OK); + CHECK(tiledb_string_view(string_handle, &data, &len) == TILEDB_OK); + + // This throws if the query plan is not valid JSON + std::string str_plan(data, len); + nlohmann::json json_plan = nlohmann::json::parse(str_plan); + std::string array_uri_from_json = json_plan["TileDB Query Plan"]["Array.URI"]; + + CHECK( + json_plan["TileDB Query Plan"]["Array.URI"] == + tiledb::sm::URI(uri_, true).to_string()); + ; + CHECK(json_plan["TileDB Query Plan"]["Array.Type"] == "dense"); + if (!array_uri_from_json.starts_with("tiledb://")) { + CHECK( + json_plan["TileDB Query Plan"]["VFS.Backend"] == + tiledb::sm::URI(uri_).backend_name()); + } + CHECK(json_plan["TileDB Query Plan"]["Query.Layout"] == "row-major"); + CHECK(json_plan["TileDB Query Plan"]["Query.Strategy.Name"] == "DenseReader"); + CHECK( + json_plan["TileDB Query Plan"]["Query.Attributes"] == + std::vector({"a1", "a2"})); + CHECK( + json_plan["TileDB Query Plan"]["Query.Dimensions"] == + std::vector({"dim_1", "dim_2"})); + + REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); + REQUIRE(tiledb_array_close(ctx_c_, array) == TILEDB_OK); + tiledb_query_free(&query); + tiledb_array_free(&array); +} + +TEST_CASE_METHOD( + QueryPlanFx, "C API: Query plan write query", "[query_plan][write][rest]") { + create_sparse_array("queryplan_array_write"); + + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_c_, uri_.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_c_, array, TILEDB_WRITE) == TILEDB_OK); + + tiledb_query_t* query; + REQUIRE(tiledb_query_alloc(ctx_c_, array, TILEDB_WRITE, &query) == TILEDB_OK); + + CHECK( + tiledb_query_set_layout(ctx_c_, query, TILEDB_GLOBAL_ORDER) == TILEDB_OK); + + std::vector coords = {1, 2, 3}; + uint64_t coords_size = coords.size() * sizeof(uint64_t); + std::vector a = {1, 2, 3}; + uint64_t a_size = a.size() * sizeof(int); + std::vector b = {1, 2, 3}; + uint64_t b_size = b.size() * sizeof(int); + + CHECK( + tiledb_query_set_data_buffer( + ctx_c_, query, "a", (void*)a.data(), &a_size) == TILEDB_OK); + CHECK( + tiledb_query_set_data_buffer( + ctx_c_, query, "b", (void*)b.data(), &b_size) == TILEDB_OK); + CHECK( + tiledb_query_set_data_buffer( + ctx_c_, + query, + tiledb::test::TILEDB_COORDS, + (void*)coords.data(), + &coords_size) == TILEDB_OK); + + tiledb_string_handle_t* string_handle; + const char* data; + size_t len; + CHECK(tiledb_query_get_plan(ctx_c_, query, &string_handle) == TILEDB_OK); + CHECK(tiledb_string_view(string_handle, &data, &len) == TILEDB_OK); + + // This throws if the query plan is not valid JSON + std::string str_plan(data, len); + nlohmann::json json_plan = nlohmann::json::parse(str_plan); + std::string array_uri_from_json = json_plan["TileDB Query Plan"]["Array.URI"]; + + CHECK( + json_plan["TileDB Query Plan"]["Array.URI"] == + tiledb::sm::URI(uri_, true).to_string()); + CHECK(json_plan["TileDB Query Plan"]["Array.Type"] == "sparse"); + if (!array_uri_from_json.starts_with("tiledb://")) { + CHECK( + json_plan["TileDB Query Plan"]["VFS.Backend"] == + tiledb::sm::URI(uri_).backend_name()); + } + CHECK(json_plan["TileDB Query Plan"]["Query.Layout"] == "global-order"); + CHECK( + json_plan["TileDB Query Plan"]["Query.Strategy.Name"] == + "GlobalOrderWriter"); + CHECK( + json_plan["TileDB Query Plan"]["Query.Attributes"] == + std::vector({"__coords", "a", "b"})); + CHECK( + json_plan["TileDB Query Plan"]["Query.Dimensions"] == + std::vector()); + + REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); + REQUIRE(tiledb_array_close(ctx_c_, array) == TILEDB_OK); + tiledb_query_free(&query); + tiledb_array_free(&array); +} + +QueryPlanFx::QueryPlanFx() + : fs_vec_(test::vfs_test_get_fs_vec()) { + auto rc = test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_); + if (!rc.ok()) { + throw std::runtime_error("Error initializing vfs in test set up."); + } + + ctx_ = Context(ctx_c_); + temp_dir_ = fs_vec_[0]->temp_dir(); + test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir_); +} + +QueryPlanFx::~QueryPlanFx() { + test::vfs_test_remove_temp_dir(ctx_c_, vfs_c_, temp_dir_); + test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok(); + tiledb_vfs_free(&vfs_c_); +} + +void QueryPlanFx::create_dense_array(const std::string& array_name) { + if constexpr (rest_tests) { + uri_ = "tiledb://unit/"; + } + + abs_uri_ = temp_dir_ + "/" + array_name; + uri_ += abs_uri_; + + // Create array schema + tiledb_array_schema_t* array_schema; + int rc = tiledb_array_schema_alloc(ctx_c_, TILEDB_DENSE, &array_schema); + REQUIRE(rc == TILEDB_OK); + + // Set schema members + rc = tiledb_array_schema_set_capacity(ctx_c_, array_schema, 10000); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_set_cell_order( + ctx_c_, array_schema, TILEDB_ROW_MAJOR); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_set_tile_order( + ctx_c_, array_schema, TILEDB_ROW_MAJOR); + REQUIRE(rc == TILEDB_OK); + + // Create dimensions + tiledb_dimension_t* d1; + uint64_t dim_domain[] = {1, 10, 1, 10}; + uint64_t extents[] = {5, 5}; + rc = tiledb_dimension_alloc( + ctx_c_, "dim_1", TILEDB_INT64, &dim_domain[0], &extents[0], &d1); + REQUIRE(rc == TILEDB_OK); + tiledb_dimension_t* d2; + rc = tiledb_dimension_alloc( + ctx_c_, "dim_2", TILEDB_INT64, &dim_domain[2], &extents[1], &d2); + REQUIRE(rc == TILEDB_OK); + + // Set domain + tiledb_domain_t* domain; + rc = tiledb_domain_alloc(ctx_c_, &domain); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx_c_, domain, d1); + REQUIRE(rc == TILEDB_OK); + tiledb_datatype_t domain_type; + rc = tiledb_domain_get_type(ctx_c_, domain, &domain_type); + REQUIRE(rc == TILEDB_OK); + REQUIRE(domain_type == TILEDB_INT64); + rc = tiledb_domain_add_dimension(ctx_c_, domain, d2); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_set_domain(ctx_c_, array_schema, domain); + REQUIRE(rc == TILEDB_OK); + + // Add attributes + tiledb_attribute_t* a1; + rc = tiledb_attribute_alloc(ctx_c_, "a1", TILEDB_INT32, &a1); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_add_attribute(ctx_c_, array_schema, a1); + REQUIRE(rc == TILEDB_OK); + tiledb_attribute_t* a2; + rc = tiledb_attribute_alloc(ctx_c_, "a2", TILEDB_INT32, &a2); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_schema_add_attribute(ctx_c_, array_schema, a2); + REQUIRE(rc == TILEDB_OK); + + // Create array + rc = tiledb_array_create(ctx_c_, uri_.c_str(), array_schema); + REQUIRE(rc == TILEDB_OK); + + // Clean up + tiledb_array_schema_free(&array_schema); + tiledb_attribute_free(&a1); + tiledb_attribute_free(&a2); + tiledb_dimension_free(&d1); + tiledb_dimension_free(&d2); + tiledb_domain_free(&domain); +} + +void QueryPlanFx::create_sparse_array(const std::string& array_name) { + if constexpr (rest_tests) { + uri_ = "tiledb://unit/"; + } + + abs_uri_ = temp_dir_ + "/" + array_name; + uri_ += abs_uri_; + + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + tiledb_dimension_t* d1; + int rc = tiledb_dimension_alloc( + ctx_c_, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1); + CHECK(rc == TILEDB_OK); + tiledb_dimension_t* d2; + rc = tiledb_dimension_alloc( + ctx_c_, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2); + CHECK(rc == TILEDB_OK); + + // Create domain + tiledb_domain_t* domain; + rc = tiledb_domain_alloc(ctx_c_, &domain); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx_c_, domain, d1); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx_c_, domain, d2); + CHECK(rc == TILEDB_OK); + + // Create attributes + tiledb_attribute_t* a; + rc = tiledb_attribute_alloc(ctx_c_, "a", TILEDB_INT32, &a); + CHECK(rc == TILEDB_OK); + tiledb_attribute_t* b; + rc = tiledb_attribute_alloc(ctx_c_, "b", TILEDB_INT32, &b); + CHECK(rc == TILEDB_OK); + + // Create array schema + tiledb_array_schema_t* array_schema; + rc = tiledb_array_schema_alloc(ctx_c_, TILEDB_SPARSE, &array_schema); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_set_cell_order( + ctx_c_, array_schema, TILEDB_ROW_MAJOR); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_set_tile_order( + ctx_c_, array_schema, TILEDB_ROW_MAJOR); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_set_capacity(ctx_c_, array_schema, 4); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_set_domain(ctx_c_, array_schema, domain); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_add_attribute(ctx_c_, array_schema, a); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_schema_add_attribute(ctx_c_, array_schema, b); + CHECK(rc == TILEDB_OK); + + // Check array schema + rc = tiledb_array_schema_check(ctx_c_, array_schema); + CHECK(rc == TILEDB_OK); + + // Create array + rc = tiledb_array_create(ctx_c_, uri_.c_str(), array_schema); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_attribute_free(&a); + tiledb_attribute_free(&b); + tiledb_dimension_free(&d1); + tiledb_dimension_free(&d2); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&array_schema); +} diff --git a/tiledb/api/c_api/query_plan/query_plan_api.cc b/tiledb/api/c_api/query_plan/query_plan_api.cc index cde57b7f9340..47c5ecdc4b27 100644 --- a/tiledb/api/c_api/query_plan/query_plan_api.cc +++ b/tiledb/api/c_api/query_plan/query_plan_api.cc @@ -48,14 +48,7 @@ capi_return_t tiledb_query_get_plan( throw CAPIStatusException("argument `query` may not be nullptr"); } - if ((*query->query_).array()->is_remote()) { - throw std::logic_error( - "Failed to create a query plan; Remote arrays" - "are not currently supported."); - } - sm::QueryPlan plan(*query->query_); - *rv = tiledb_string_handle_t::make_handle(plan.dump_json()); return TILEDB_OK; diff --git a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc index 8a794b1ddba2..3c60adf798c0 100644 --- a/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc +++ b/tiledb/api/c_api/query_plan/test/unit_capi_query_plan.cc @@ -31,7 +31,6 @@ */ #include -#include "external/include/nlohmann/json.hpp" #include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" @@ -108,71 +107,6 @@ void QueryPlanFx::create_dense_array(const std::string& path) { tiledb_domain_free(&domain); } -void QueryPlanFx::create_sparse_array(const std::string& array_name) { - // Create dimensions - uint64_t tile_extents[] = {2, 2}; - uint64_t dim_domain[] = {1, 10, 1, 10}; - - tiledb_dimension_t* d1; - int rc = tiledb_dimension_alloc( - ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1); - CHECK(rc == TILEDB_OK); - tiledb_dimension_t* d2; - rc = tiledb_dimension_alloc( - ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2); - CHECK(rc == TILEDB_OK); - - // Create domain - tiledb_domain_t* domain; - rc = tiledb_domain_alloc(ctx, &domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx, domain, d1); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx, domain, d2); - CHECK(rc == TILEDB_OK); - - // Create attributes - tiledb_attribute_t* a; - rc = tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a); - CHECK(rc == TILEDB_OK); - tiledb_attribute_t* b; - rc = tiledb_attribute_alloc(ctx, "b", TILEDB_INT32, &b); - CHECK(rc == TILEDB_OK); - - // Create array schema - tiledb_array_schema_t* array_schema; - rc = tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_capacity(ctx, array_schema, 4); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_domain(ctx, array_schema, domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx, array_schema, a); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx, array_schema, b); - CHECK(rc == TILEDB_OK); - - // Check array schema - rc = tiledb_array_schema_check(ctx, array_schema); - CHECK(rc == TILEDB_OK); - - // Create array - rc = tiledb_array_create(ctx, array_name.c_str(), array_schema); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_attribute_free(&a); - tiledb_attribute_free(&b); - tiledb_dimension_free(&d1); - tiledb_dimension_free(&d2); - tiledb_domain_free(&domain); - tiledb_array_schema_free(&array_schema); -} - TEST_CASE_METHOD( QueryPlanFx, "C API: tiledb_query_get_plan argument validation", @@ -200,183 +134,3 @@ TEST_CASE_METHOD( tiledb_query_free(&query); tiledb_array_free(&array); } - -TEST_CASE_METHOD( - QueryPlanFx, - "C API: tiledb_query_get_plan API lifecycle checks", - "[capi][query_plan]") { - std::string array_name = temp_dir_ + "queryplan_array"; - create_dense_array(array_name); - - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); - REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); - - tiledb_query_t* query; - REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); - - CHECK(tiledb_query_set_layout(ctx, query, TILEDB_ROW_MAJOR) == TILEDB_OK); - - int64_t dom[] = {1, 2, 1, 2}; - CHECK(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); - - std::vector d(4); - uint64_t size = 1; - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a1", d.data(), &size) == - TILEDB_OK); - - tiledb_string_handle_t* string_handle; - CHECK(tiledb_query_get_plan(ctx, query, &string_handle) == TILEDB_OK); - - // API lifecycle checks - // It's not possible to set subarrays, layout, query condition or new buffers - // once the query plan got generated. - CHECK(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_ERR); - CHECK(tiledb_query_set_layout(ctx, query, TILEDB_COL_MAJOR) == TILEDB_ERR); - tiledb_query_condition_t* qc; - CHECK(tiledb_query_condition_alloc(ctx, &qc) == TILEDB_OK); - int32_t val = 10000; - CHECK( - tiledb_query_condition_init( - ctx, qc, "a1", &val, sizeof(int32_t), TILEDB_LT) == TILEDB_OK); - CHECK(tiledb_query_set_condition(ctx, query, qc) == TILEDB_ERR); - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a2", d.data(), &size) == - TILEDB_ERR); - - // But it's possible to set existing buffers to accomodate existing - // query INCOMPLETEs functionality - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a1", d.data(), &size) == - TILEDB_OK); - - REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); - REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); - tiledb_query_free(&query); - tiledb_array_free(&array); -} - -TEST_CASE_METHOD( - QueryPlanFx, - "C API: Query plan basic bahaviour", - "[capi][query_plan][basic1]") { - std::string array_name = temp_dir_ + "queryplan_array"; - create_dense_array(array_name); - - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); - REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); - - tiledb_query_t* query; - REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); - - CHECK(tiledb_query_set_layout(ctx, query, TILEDB_ROW_MAJOR) == TILEDB_OK); - - int64_t dom[] = {1, 2, 1, 2}; - CHECK(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); - - std::vector d(4); - uint64_t size = 1; - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a1", d.data(), &size) == - TILEDB_OK); - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a2", d.data(), &size) == - TILEDB_OK); - - tiledb_string_handle_t* string_handle; - const char* data; - size_t len; - CHECK(tiledb_query_get_plan(ctx, query, &string_handle) == TILEDB_OK); - CHECK(tiledb_string_view(string_handle, &data, &len) == TILEDB_OK); - - // This throws if the query plan is not valid JSON - std::string str_plan(data, len); - nlohmann::json json_plan = nlohmann::json::parse(str_plan); - - CHECK(json_plan["TileDB Query Plan"]["Array.URI"] == array_name); - CHECK(json_plan["TileDB Query Plan"]["Array.Type"] == "dense"); - CHECK( - json_plan["TileDB Query Plan"]["VFS.Backend"] == - tiledb::sm::URI(array_name).backend_name()); - CHECK(json_plan["TileDB Query Plan"]["Query.Layout"] == "row-major"); - CHECK(json_plan["TileDB Query Plan"]["Query.Strategy.Name"] == "DenseReader"); - CHECK( - json_plan["TileDB Query Plan"]["Query.Attributes"] == - std::vector({"a1", "a2"})); - CHECK( - json_plan["TileDB Query Plan"]["Query.Dimensions"] == - std::vector({"dim_1", "dim_2"})); - - REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); - REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); - tiledb_query_free(&query); - tiledb_array_free(&array); -} - -TEST_CASE_METHOD( - QueryPlanFx, - "C API: Query plan basic bahaviour 2", - "[capi][query_plan][basic2]") { - std::string array_name = temp_dir_ + "queryplan_array"; - create_sparse_array(array_name); - - tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx, array_name.c_str(), &array) == TILEDB_OK); - REQUIRE(tiledb_array_open(ctx, array, TILEDB_WRITE) == TILEDB_OK); - - tiledb_query_t* query; - REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query) == TILEDB_OK); - - CHECK(tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER) == TILEDB_OK); - - std::vector coords = {1, 2, 3}; - uint64_t coords_size = coords.size() * sizeof(uint64_t); - std::vector a = {1, 2, 3}; - uint64_t a_size = a.size() * sizeof(int); - std::vector b = {1, 2, 3}; - uint64_t b_size = b.size() * sizeof(int); - - CHECK( - tiledb_query_set_data_buffer(ctx, query, "a", (void*)a.data(), &a_size) == - TILEDB_OK); - CHECK( - tiledb_query_set_data_buffer(ctx, query, "b", (void*)b.data(), &b_size) == - TILEDB_OK); - CHECK( - tiledb_query_set_data_buffer( - ctx, query, TILEDB_COORDS, (void*)coords.data(), &coords_size) == - TILEDB_OK); - - tiledb_string_handle_t* string_handle; - const char* data; - size_t len; - CHECK(tiledb_query_get_plan(ctx, query, &string_handle) == TILEDB_OK); - CHECK(tiledb_string_view(string_handle, &data, &len) == TILEDB_OK); - - // This throws if the query plan is not valid JSON - std::string str_plan(data, len); - nlohmann::json json_plan = nlohmann::json::parse(str_plan); - - CHECK(json_plan["TileDB Query Plan"]["Array.URI"] == array_name); - CHECK(json_plan["TileDB Query Plan"]["Array.Type"] == "sparse"); - CHECK( - json_plan["TileDB Query Plan"]["VFS.Backend"] == - tiledb::sm::URI(array_name).backend_name()); - CHECK(json_plan["TileDB Query Plan"]["Query.Layout"] == "global-order"); - CHECK( - json_plan["TileDB Query Plan"]["Query.Strategy.Name"] == - "GlobalOrderWriter"); - CHECK( - json_plan["TileDB Query Plan"]["Query.Attributes"] == - std::vector({"__coords", "a", "b"})); - CHECK( - json_plan["TileDB Query Plan"]["Query.Dimensions"] == - std::vector()); - - REQUIRE(tiledb_string_free(&string_handle) == TILEDB_OK); - REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); - tiledb_query_free(&query); - tiledb_array_free(&array); -} diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 837d487d60fb..94c7e15575e5 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -2147,5 +2147,9 @@ void Query::copy_aggregates_data_to_user_buffer() { } } +RestClient* Query::rest_client() const { + return storage_manager_->rest_client(); +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 4b84f3313ddc..c02cea963581 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -54,6 +54,7 @@ #include "tiledb/sm/query/readers/aggregators/query_channel.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/query/validity_vector.h" +#include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/subarray.h" @@ -848,6 +849,12 @@ class Query { return aggregate_channel_; } + /** + * Returns the REST client configured in the storage manager associated to + * this query + */ + RestClient* rest_client() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/query_plan/query_plan.cc b/tiledb/sm/query_plan/query_plan.cc index d52160cb2aa3..12df72b6dde5 100644 --- a/tiledb/sm/query_plan/query_plan.cc +++ b/tiledb/sm/query_plan/query_plan.cc @@ -35,8 +35,10 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/enums/query_status.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/query/query.h" +#include "tiledb/sm/rest/rest_client.h" #include "external/include/nlohmann/json.hpp" @@ -49,6 +51,24 @@ namespace sm { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ QueryPlan::QueryPlan(Query& query) { + if (query.array()->is_remote()) { + auto rest_client = query.rest_client(); + if (!rest_client) { + throw std::runtime_error( + "Failed to create a query plan; Remote query" + "with no REST client."); + } + + rest_client->post_query_plan_from_rest( + query.array()->array_uri(), query, *this); + + // We need to transition the query status to INITIALIZED to mimic the + // behavior of getting a query plan locally + query.set_status(QueryStatus::INITIALIZED); + + return; + } + array_uri_ = query.array()->array_uri().to_string(); vfs_backend_ = URI(array_uri_).backend_name(); query_layout_ = query.layout(); diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 47a5f809fbf8..86b0b57ea1b4 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -43,6 +43,7 @@ #include "tiledb/sm/serialization/fragments.h" #include "tiledb/sm/serialization/group.h" #include "tiledb/sm/serialization/query.h" +#include "tiledb/sm/serialization/query_plan.h" #include "tiledb/sm/serialization/tiledb-rest.capnp.h" #include "tiledb/sm/serialization/vacuum.h" #include "tiledb/sm/rest/curl.h" // must be included last to avoid Windows.h @@ -129,10 +130,20 @@ Status RestClient::init( if (c_str != nullptr) RETURN_NOT_OK(serialization_type_enum(c_str, &serialization_type_)); - bool found = false; - auto status = config_->get( + return Status::Ok(); +} + +Status RestClient::set_header( + const std::string& name, const std::string& value) { + extra_headers_[name] = value; + return Status::Ok(); +} + +bool RestClient::use_refactored_query(const Config& config) { + bool found = false, use_refactored_query = false; + auto status = config.get( "rest.use_refactored_array_open_and_query_submit", - &use_refactored_array_and_query_, + &use_refactored_query, &found); if (!status.ok() || !found) { throw std::runtime_error( @@ -140,13 +151,7 @@ Status RestClient::init( "configuration option from config"); } - return Status::Ok(); -} - -Status RestClient::set_header( - const std::string& name, const std::string& value) { - extra_headers_[name] = value; - return Status::Ok(); + return use_refactored_query; } tuple> RestClient::check_array_exists_from_rest( @@ -709,6 +714,64 @@ RestClient::post_enumerations_from_rest( serialization_type_, returned_data, memory_tracker); } +void RestClient::post_query_plan_from_rest( + const URI& uri, Query& query, QueryPlan& query_plan) { + // Get array + const Array* array = query.array(); + if (array == nullptr) { + throw Status_RestError("Error submitting query plan to REST; null array."); + } + + Buffer buff; + serialization::serialize_query_plan_request( + query.config(), query, serialization_type_, buff); + + // Wrap in a list + BufferList serialized; + throw_if_not_ok(serialized.add_buffer(std::move(buff))); + + // Init curl and form the URL + Curl curlc(logger_); + std::string array_ns, array_uri; + throw_if_not_ok(uri.get_rest_components(&array_ns, &array_uri)); + const std::string cache_key = array_ns + ":" + array_uri; + throw_if_not_ok( + curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); + std::string url; + if (use_refactored_query(query.config())) { + url = redirect_uri(cache_key) + "/v3/arrays/" + array_ns + "/" + + curlc.url_escape(array_uri) + + "/query/plan?type=" + query_type_str(query.type()); + } else { + url = redirect_uri(cache_key) + "/v2/arrays/" + array_ns + "/" + + curlc.url_escape(array_uri) + + "/query/plan?type=" + query_type_str(query.type()); + } + + // Remote array reads always supply the timestamp. + url += "&start_timestamp=" + std::to_string(array->timestamp_start()); + url += "&end_timestamp=" + std::to_string(array->timestamp_end()); + + // Get the data + Buffer returned_data; + throw_if_not_ok(curlc.post_data( + stats_, + url, + serialization_type_, + &serialized, + &returned_data, + cache_key)); + if (returned_data.data() == nullptr || returned_data.size() == 0) { + throw Status_RestError( + "Error getting query plan from REST; server returned no data."); + } + + // Ensure data has a null delimiter for cap'n proto if using JSON + throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); + query_plan = serialization::deserialize_query_plan_response( + query, serialization_type_, returned_data); +} + Status RestClient::submit_query_to_rest(const URI& uri, Query* query) { // Local state tracking for the current offsets into the user's query buffers. // This allows resubmission of incomplete queries while appending to the @@ -770,7 +833,7 @@ Status RestClient::post_query_submit( RETURN_NOT_OK( curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); std::string url; - if (use_refactored_array_and_query_) { + if (use_refactored_query(query->config())) { url = redirect_uri(cache_key) + "/v3/arrays/" + array_ns + "/" + curlc.url_escape(array_uri) + "/query/submit?type=" + query_type_str(query->type()) + @@ -996,7 +1059,7 @@ Status RestClient::finalize_query_to_rest(const URI& uri, Query* query) { RETURN_NOT_OK( curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); std::string url; - if (use_refactored_array_and_query_) { + if (use_refactored_query(query->config())) { url = redirect_uri(cache_key) + "/v3/arrays/" + array_ns + "/" + curlc.url_escape(array_uri) + "/query/finalize?type=" + query_type_str(query->type()); @@ -1051,7 +1114,7 @@ Status RestClient::submit_and_finalize_query_to_rest( RETURN_NOT_OK( curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); std::string url; - if (use_refactored_array_and_query_) { + if (use_refactored_query(query->config())) { url = redirect_uri(cache_key) + "/v3/arrays/" + array_ns + "/" + curlc.url_escape(array_uri) + "/query/submit_and_finalize?type=" + query_type_str(query->type()); @@ -1635,6 +1698,10 @@ RestClient::post_enumerations_from_rest( throw Status_RestError("Cannot use rest client; serialization not enabled."); } +void RestClient::post_query_plan_from_rest(const URI&, Query&, QueryPlan&) { + throw Status_RestError("Cannot use rest client; serialization not enabled."); +} + Status RestClient::submit_query_to_rest(const URI&, Query*) { return LOG_STATUS( Status_RestError("Cannot use rest client; serialization not enabled.")); diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index e3e67ddb3b71..fe530239b4e5 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -54,6 +54,7 @@ class Config; class FragmentInfo; class Query; class MemoryTracker; +class QueryPlan; enum class SerializationType : uint8_t; @@ -73,6 +74,15 @@ class RestClient { /** Sets a header that will be attached to all requests. */ Status set_header(const std::string& name, const std::string& value); + /** + * Check if use_refactored_array_open_and_query_submit is set in + * input config so that rest_client chooses the right URI + * + * @param config Config to check + * + * */ + static bool use_refactored_query(const Config& config); + /** * Check if an array exists by making a REST call. To start with this fetches * the schema but ignores the body returned if non-error @@ -257,6 +267,16 @@ class RestClient { const std::vector& enumeration_names, shared_ptr memory_tracker = nullptr); + /** + * Get the requested query plan from the REST server via POST request. + * + * @param uri Array URI. + * @param query Query to fetch query plan for. + * @param query_plan The requested query plan. + */ + void post_query_plan_from_rest( + const URI& uri, Query& query, QueryPlan& query_plan); + /** * Post a data query to rest server * @@ -420,12 +440,6 @@ class RestClient { */ bool resubmit_incomplete_; - /** - * If true, the new, experimental REST routes and APIs for opening an array - * and submitting a query will be used - */ - bool use_refactored_array_and_query_; - /** Collection of extra headers that are attached to REST requests. */ std::unordered_map extra_headers_; From a0e82bd5e4dc5ef969c04918ddeb18f7165d0723 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 12 Mar 2024 16:43:25 +0200 Subject: [PATCH 240/456] Consolidation Plan REST support - part 2: rest client (#4537) This is part2 of https://github.com/TileDB-Inc/TileDB/pull/4534 , adding the `rest_client` part. --- TYPE: FEATURE DESC: Consolidation Plan REST support - part 2: rest client --- test/src/test-capi-consolidation-plan.cc | 85 +++++++++---------- tiledb/sm/array/array.h | 5 ++ .../consolidation_plan/consolidation_plan.cc | 16 +++- tiledb/sm/rest/rest_client.cc | 49 +++++++++++ tiledb/sm/rest/rest_client.h | 11 +++ 5 files changed, 121 insertions(+), 45 deletions(-) diff --git a/test/src/test-capi-consolidation-plan.cc b/test/src/test-capi-consolidation-plan.cc index bce48e219d1d..11e6ea64e6bd 100644 --- a/test/src/test-capi-consolidation-plan.cc +++ b/test/src/test-capi-consolidation-plan.cc @@ -30,6 +30,7 @@ * Tests the ConsolidationPlan API */ +#include #include "test/support/src/helpers.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" @@ -42,17 +43,13 @@ using namespace tiledb; using namespace tiledb::test; -struct ConsolidationPlanFx { - // Constants. - const char* SPARSE_ARRAY_NAME = "test_deletes_array"; - - // TileDB context. - Context ctx_; - VFS vfs_; - - std::string key_ = "0123456789abcdeF0123456789abcdeF"; - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; +#ifndef TILEDB_TESTS_ENABLE_REST +constexpr bool rest_tests = false; +#else +constexpr bool rest_tests = true; +#endif +struct ConsolidationPlanFx { // Constructors/destructors. ConsolidationPlanFx(); ~ConsolidationPlanFx(); @@ -69,20 +66,39 @@ struct ConsolidationPlanFx { void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); void check_last_error(std::string expected); + + // TileDB context. + Context ctx_; + // Full URI initialized using fs_vec_ random temp directory. + std::string array_name_; + + // Vector of supported filsystems + tiledb_vfs_handle_t* vfs_c_{nullptr}; + tiledb_ctx_handle_t* ctx_c_{nullptr}; + const std::vector> fs_vec_; + + std::string key_ = "0123456789abcdeF0123456789abcdeF"; + const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; }; ConsolidationPlanFx::ConsolidationPlanFx() - : vfs_(ctx_) { + : fs_vec_(test::vfs_test_get_fs_vec()) { Config config; config.set("sm.consolidation.buffer_size", "1000"); - ctx_ = Context(config); - vfs_ = VFS(ctx_); - - remove_sparse_array(); + REQUIRE( + test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_, config.ptr().get()).ok()); + ctx_ = Context(ctx_c_); + std::string temp_dir = fs_vec_[0]->temp_dir(); + if constexpr (rest_tests) { + array_name_ = "tiledb://unit/"; + } + array_name_ += temp_dir + "test_consolidation_plan_array"; + test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir); } ConsolidationPlanFx::~ConsolidationPlanFx() { - remove_sparse_array(); + Array::delete_array(ctx_, array_name_); + REQUIRE(test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok()); } void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) { @@ -115,9 +131,9 @@ void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) { schema.set_coords_filter_list(filter_list); if (encrypt) { - Array::create(SPARSE_ARRAY_NAME, schema, enc_type_, key_); + Array::create(array_name_, schema, enc_type_, key_); } else { - Array::create(SPARSE_ARRAY_NAME, schema); + Array::create(array_name_, schema); } } @@ -132,16 +148,13 @@ void ConsolidationPlanFx::write_sparse( if (encrypt) { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_, TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp), EncryptionAlgorithm(AESGCM, key_.c_str())); } else { array = std::make_unique( - ctx_, - SPARSE_ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp)); + ctx_, array_name_, TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp)); } // Create query. @@ -152,28 +165,12 @@ void ConsolidationPlanFx::write_sparse( query.set_data_buffer("d2", dim2); // Submit/finalize the query. - query.submit(); - query.finalize(); + query.submit_and_finalize(); // Close array. array->close(); } -void ConsolidationPlanFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; - - vfs_.remove_dir(array_name); -} - -void ConsolidationPlanFx::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); -} - -bool ConsolidationPlanFx::is_array(const std::string& array_name) { - return vfs_.is_dir(array_name); -} - void ConsolidationPlanFx::check_last_error(std::string expected) { const char* msg = "unset"; tiledb_error_t* err{nullptr}; @@ -188,11 +185,11 @@ void ConsolidationPlanFx::check_last_error(std::string expected) { TEST_CASE_METHOD( ConsolidationPlanFx, "CAPI: Consolidation plan", - "[capi][consolidation-plan]") { + "[capi][consolidation-plan][rest]") { create_sparse_array(); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); - Array array{ctx_, SPARSE_ARRAY_NAME, TILEDB_READ}; + Array array{ctx_, array_name_, TILEDB_READ}; tiledb_consolidation_plan_t* consolidation_plan{}; CHECK( @@ -231,11 +228,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationPlanFx, "CAPI: Consolidation plan dump", - "[capi][consolidation-plan][dump]") { + "[capi][consolidation-plan][dump][rest]") { create_sparse_array(); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); - Array array{ctx_, SPARSE_ARRAY_NAME, TILEDB_READ}; + Array array{ctx_, array_name_, TILEDB_READ}; tiledb_consolidation_plan_t* consolidation_plan{}; CHECK( diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 6f304ae8826d..380deef7e9c7 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -840,6 +840,11 @@ class Array { /** Load array directory for non-remote arrays */ const ArrayDirectory& load_array_directory(); + /* Get the REST client */ + [[nodiscard]] inline shared_ptr rest_client() const { + return resources_.rest_client(); + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/consolidation_plan/consolidation_plan.cc b/tiledb/sm/consolidation_plan/consolidation_plan.cc index 051306b5db90..63f439fca95a 100644 --- a/tiledb/sm/consolidation_plan/consolidation_plan.cc +++ b/tiledb/sm/consolidation_plan/consolidation_plan.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/consolidation_plan/consolidation_plan.h" #include "tiledb/common/common.h" #include "tiledb/common/logger.h" +#include "tiledb/sm/rest/rest_client.h" using namespace tiledb::sm; using namespace tiledb::common; @@ -44,7 +45,20 @@ using namespace tiledb::common; ConsolidationPlan::ConsolidationPlan( shared_ptr array, uint64_t fragment_size) : desired_fragment_size_(fragment_size) { - generate(array); + if (array->is_remote()) { + auto rest_client = array->rest_client(); + if (!rest_client) { + throw std::runtime_error( + "Failed to create a consolidation plan; Remote array" + "with no REST client."); + } + // reach out to the REST client to populate class members + fragment_uris_per_node_ = rest_client->post_consolidation_plan_from_rest( + array->array_uri(), array->config(), fragment_size); + num_nodes_ = fragment_uris_per_node_.size(); + } else { + generate(array); + } } ConsolidationPlan::ConsolidationPlan( diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 86b0b57ea1b4..5c1a758104a3 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -1598,6 +1598,48 @@ Status RestClient::post_vacuum_to_rest(const URI& uri, const Config& config) { stats_, url, serialization_type_, &serialized, &returned_data, cache_key); } +std::vector> +RestClient::post_consolidation_plan_from_rest( + const URI& uri, const Config& config, uint64_t fragment_size) { + Buffer buff; + serialization::serialize_consolidation_plan_request( + fragment_size, config, serialization_type_, buff); + + // Wrap in a list + BufferList serialized; + throw_if_not_ok(serialized.add_buffer(std::move(buff))); + + // Init curl and form the URL + Curl curlc(logger_); + std::string array_ns, array_uri; + throw_if_not_ok(uri.get_rest_components(&array_ns, &array_uri)); + const std::string cache_key = array_ns + ":" + array_uri; + throw_if_not_ok( + curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); + const std::string url = redirect_uri(cache_key) + "/v1/arrays/" + array_ns + + "/" + curlc.url_escape(array_uri) + + "/consolidate/plan"; + + // Get the data + Buffer returned_data; + throw_if_not_ok(curlc.post_data( + stats_, + url, + serialization_type_, + &serialized, + &returned_data, + cache_key)); + if (returned_data.data() == nullptr || returned_data.size() == 0) { + throw Status_RestError( + "Error getting query plan from REST; server returned no data."); + } + + // Ensure data has a null delimiter for cap'n proto if using JSON + throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); + return serialization::deserialize_consolidation_plan_response( + serialization_type_, returned_data); +} + #else RestClient::RestClient() { @@ -1789,6 +1831,13 @@ Status RestClient::post_vacuum_to_rest(const URI&, const Config&) { Status_RestError("Cannot use rest client; serialization not enabled.")); } +std::vector> +RestClient::post_consolidation_plan_from_rest( + const URI&, const Config&, uint64_t) { + throw StatusException( + Status_RestError("Cannot use rest client; serialization not enabled.")); +} + #endif // TILEDB_SERIALIZATION } // namespace sm diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index fe530239b4e5..25eb29507ddd 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -409,6 +409,17 @@ class RestClient { return rest_server_; } + /** + * Get consolidation plan from the REST server via POST request. + * + * @param uri Array URI. + * @param config Config of the array. + * @param fragment_size Maximum fragment size for constructing the plan. + * @return The requested consolidation plan + */ + std::vector> post_consolidation_plan_from_rest( + const URI& uri, const Config& config, uint64_t fragment_size); + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ From 2a780bd4855290efe67ced06eef294aebb379bad Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:33:18 +0100 Subject: [PATCH 241/456] Update dev HISTORY with 2.19 and 2.20 patch content. (#4803) --- TYPE: NO_HISTORY DESC: Update dev HISTORY with 2.19 and 2.20 patch content. --- HISTORY.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 1eb09d8accd7..bd326298fc93 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,13 @@ +# TileDB v2.20.1 Release Notes + +## Defects removed + +* Support reading V1 group details with explicit version in the name. [#4744](https://github.com/TileDB-Inc/TileDB/pull/4744) + +## Build System Changes + +* Reintroduce the `TILEDB_STATIC` option under a deprecation warning. [#4732](https://github.com/TileDB-Inc/TileDB/pull/4732) + # TileDB v2.20.0 Release Notes ## Breaking behavior @@ -44,6 +54,18 @@ * Fix regression where release artifacts had 8-digit commit hashes. [#4599](https://github.com/TileDB-Inc/TileDB/pull/4599) * Fix importing TileDB in CMake versions prior to 3.18. [#4671](https://github.com/TileDB-Inc/TileDB/pull/4671) +# TileDB v2.19.2 Release Notes + +## Defects removed + +* Fix bug with new minio behavior. [#4725](https://github.com/TileDB-Inc/TileDB/pull/4725) +* Support reading V1 group details with explicit version in the name. [#4744](https://github.com/TileDB-Inc/TileDB/pull/4744) + +## Build System Changes + +* Fix regression where release artifacts had 8-digit commit hashes. [#4599](https://github.com/TileDB-Inc/TileDB/pull/4599) +* Fix linker errors when building with MSVC. [#4759](https://github.com/TileDB-Inc/TileDB/pull/4759) + # TileDB v2.19.1 Release Notes ## Improvements From 163172a8e7316422b315018323d90537027f81a7 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:34:17 +0100 Subject: [PATCH 242/456] Bump dev version to 2.22.0. (#4804) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.22.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index b75e2e30713b..1535c996190d 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.21' +version = '2.22' # The full version, including alpha/beta/rc tags. -release = '2.21.0' +release = '2.22.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index 91b07fb241d8..4b32ddb980b3 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 21 +#define TILEDB_VERSION_MINOR 22 #define TILEDB_VERSION_PATCH 0 From fdb56746accaf4254d2bc023d77477f44a9cba79 Mon Sep 17 00:00:00 2001 From: Julia Dark <24235303+jp-dark@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:04:10 -0400 Subject: [PATCH 243/456] Move filter pipeline tests out of tiledb_unit (#4797) * Move unit filter pipeline tests from tiledb_unit to unit tests in tiledb/sm/filter/tests. * Rename file with remaining C++ integration tests. --- TYPE: NO_HISTORY --------- Co-authored-by: Ypatia Tsavliri --- test/CMakeLists.txt | 2 +- test/src/cpp-integration-filter-pipeline.cc | 263 +++ test/src/unit-filter-pipeline.cc | 1814 ----------------- tiledb/sm/filter/test/CMakeLists.txt | 8 + tiledb/sm/filter/test/filter_test_support.cc | 235 +++ tiledb/sm/filter/test/filter_test_support.h | 157 ++ .../test/unit_bit_width_reduction_pipeline.cc | 497 +++++ .../filter/test/unit_bitshuffle_pipeline.cc | 198 ++ .../filter/test/unit_byteshuffle_pipeline.cc | 196 ++ .../sm/filter/test/unit_checksum_pipeline.cc | 92 + .../filter/test/unit_encryption_pipeline.cc | 103 + .../test/unit_positive_delta_pipeline.cc | 286 +++ .../filter/test/unit_run_filter_pipeline.cc | 166 +- tiledb/sm/filter/test/unit_xor_pipeline.cc | 327 +++ 14 files changed, 2364 insertions(+), 1980 deletions(-) create mode 100644 test/src/cpp-integration-filter-pipeline.cc delete mode 100644 test/src/unit-filter-pipeline.cc create mode 100644 tiledb/sm/filter/test/filter_test_support.cc create mode 100644 tiledb/sm/filter/test/filter_test_support.h create mode 100644 tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_checksum_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_encryption_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_positive_delta_pipeline.cc create mode 100644 tiledb/sm/filter/test/unit_xor_pipeline.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 10cb67d82fda..c725e1f444e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -177,7 +177,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-enumerations.cc src/unit-enum-helpers.cc src/unit-filter-buffer.cc - src/unit-filter-pipeline.cc src/unit-global-order.cc src/unit-ordered-dim-label-reader.cc src/unit-tile-metadata.cc @@ -204,6 +203,7 @@ set(TILEDB_UNIT_TEST_SOURCES if (TILEDB_CPP_API) list(APPEND TILEDB_UNIT_TEST_SOURCES + src/cpp-integration-filter-pipeline.cc src/test-cppapi-dense-array-dimension-label.cc src/test-cppapi-dimension-label.cc src/test-cppapi-subarray-labels.cc diff --git a/test/src/cpp-integration-filter-pipeline.cc b/test/src/cpp-integration-filter-pipeline.cc new file mode 100644 index 000000000000..61c9730a8896 --- /dev/null +++ b/test/src/cpp-integration-filter-pipeline.cc @@ -0,0 +1,263 @@ +/** + * @file unit-filter-pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017-2024 TileDB, Inc. + * @copyright Copyright (c) 2016 MIT and Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the `FilterPipeline` class. + */ + +#include "test/support/src/helpers.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/filter/webp_filter.h" +#include "tiledb/stdx/utility/to_underlying.h" + +#include +#include +#include + +using namespace tiledb; +using namespace tiledb::common; +using namespace tiledb::sm; + +TEST_CASE( + "C++ API: Pipeline with filtered type conversions", + "[cppapi][filter][pipeline]") { + tiledb::Context ctx; + tiledb::VFS vfs(ctx); + std::string array_name = "cpp_test_array"; + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + tiledb::Domain domain(ctx); + float domain_lo = static_cast(std::numeric_limits::min()); + float domain_hi = static_cast(std::numeric_limits::max() - 1); + + // Create and initialize dimension. + auto d1 = tiledb::Dimension::create( + ctx, "d1", {{domain_lo, domain_hi}}, 2048); + + tiledb::Filter float_scale(ctx, TILEDB_FILTER_SCALE_FLOAT); + double scale = 1.0f; + double offset = 0.0f; + uint64_t byte_width = sizeof(int32_t); + + // Float scale converting tile data from float->int32 + float_scale.set_option(TILEDB_SCALE_FLOAT_BYTEWIDTH, &byte_width); + float_scale.set_option(TILEDB_SCALE_FLOAT_FACTOR, &scale); + float_scale.set_option(TILEDB_SCALE_FLOAT_OFFSET, &offset); + + // Delta filter reinterprets int32->uint32 + tiledb::Filter delta(ctx, TILEDB_FILTER_DELTA); + + // Pass uint32 data to BitWidthReduction filter + tiledb::Filter bit_width_reduction(ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION); + + tiledb::FilterList filters(ctx); + filters.add_filter(float_scale); + filters.add_filter(delta); + filters.add_filter(bit_width_reduction); + + // Apply filters to both attribute and dimension. + REQUIRE_NOTHROW(d1.set_filter_list(filters)); + domain.add_dimension(d1); + + auto a1 = tiledb::Attribute::create(ctx, "a1"); + REQUIRE_NOTHROW(a1.set_filter_list(filters)); + + tiledb::ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.add_attribute(a1); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_ROW_MAJOR); + REQUIRE_NOTHROW(tiledb::Array::create(array_name, schema)); + std::vector d1_data = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f}; + std::vector a1_data = { + 1.0f, 2.1f, 3.2f, 4.3f, 5.4f, 6.5f, 7.6f, 8.7f, 9.8f, 10.9f}; + + // Write to array. + { + tiledb::Array array(ctx, array_name, TILEDB_WRITE); + tiledb::Query query(ctx, array); + query.set_data_buffer("d1", d1_data); + query.set_data_buffer("a1", a1_data); + query.submit(); + CHECK(tiledb::Query::Status::COMPLETE == query.query_status()); + } + + // Read from array. + { + std::vector d1_read(10); + std::vector a1_read(10); + tiledb::Array array(ctx, array_name, TILEDB_READ); + tiledb::Query query(ctx, array); + query.set_subarray({domain_lo, domain_hi}); + query.set_data_buffer("a1", a1_read); + query.set_data_buffer("d1", d1_read); + query.submit(); + CHECK(tiledb::Query::Status::COMPLETE == query.query_status()); + // Some loss of precision from rounding in FloatScale. + CHECK( + std::vector{ + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f} == + a1_read); + CHECK(d1_data == d1_read); + } + + // Cleanup. + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } +} + +TEST_CASE( + "C++ API: Filter pipeline validation", + "[cppapi][filter][pipeline][validation]") { + tiledb::Context ctx; + + tiledb::Domain domain(ctx); + float domain_lo = static_cast(std::numeric_limits::min()); + float domain_hi = static_cast(std::numeric_limits::max() - 1); + auto d1 = tiledb::Dimension::create( + ctx, "d1", {{domain_lo, domain_hi}}, 2048); + auto a1 = tiledb::Attribute::create(ctx, "a1"); + + // FloatScale used for testing different float->integral pipelines. + tiledb::Filter float_scale(ctx, TILEDB_FILTER_SCALE_FLOAT); + double scale = 1.0f; + double offset = 0.0f; + uint64_t byte_width = sizeof(int32_t); + // Float scale converting tile data from float->int32 + float_scale.set_option(TILEDB_SCALE_FLOAT_BYTEWIDTH, &byte_width); + float_scale.set_option(TILEDB_SCALE_FLOAT_FACTOR, &scale); + float_scale.set_option(TILEDB_SCALE_FLOAT_OFFSET, &offset); + + tiledb::FilterList filters(ctx); + SECTION("- FloatScale filter accepts float or double byte width input") { + auto d2 = tiledb::Dimension::create(ctx, "d2", {{1, 100}}, 10); + auto a2 = tiledb::Attribute::create(ctx, "a2"); + filters.add_filter(float_scale); + CHECK_THROWS(d2.set_filter_list(filters)); + CHECK_NOTHROW(a2.set_filter_list(filters)); + } + + SECTION("- Delta filters do not accept real datatypes") { + auto test_filter = GENERATE( + TILEDB_FILTER_POSITIVE_DELTA, + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA); + tiledb::Filter delta_filter(ctx, test_filter); + filters.add_filter(delta_filter); + // Delta compressors don't accept floats. Should fail without FloatScale. + CHECK_THROWS(d1.set_filter_list(filters)); + CHECK_THROWS(a1.set_filter_list(filters)); + + // Test using FloatScale to convert to integral is accepted. + tiledb::FilterList filters2(ctx); + filters2.add_filter(float_scale); + filters2.add_filter(delta_filter); + CHECK_NOTHROW(d1.set_filter_list(filters2)); + CHECK_NOTHROW(a1.set_filter_list(filters2)); + } + + SECTION("- Webp filter supports only uint8 attributes") { + if (webp_filter_exists) { + tiledb::Filter webp(ctx, TILEDB_FILTER_WEBP); + filters.add_filter(webp); + CHECK_THROWS(d1.set_filter_list(filters)); + CHECK_THROWS(a1.set_filter_list(filters)); + } + } + + SECTION("- Bit width reduction filter supports integral input") { + tiledb::Filter bit_width_reduction(ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION); + filters.add_filter(bit_width_reduction); + CHECK_THROWS(d1.set_filter_list(filters)); + CHECK_THROWS(a1.set_filter_list(filters)); + + // Test using FloatScale to convert to integral is accepted. + tiledb::FilterList filters2(ctx); + filters2.add_filter(float_scale); + filters2.add_filter(bit_width_reduction); + CHECK_NOTHROW(d1.set_filter_list(filters2)); + CHECK_NOTHROW(a1.set_filter_list(filters2)); + } + + SECTION("- XOR filter interprets datatype as integral") { + // Datatype byte size must match size of int8, int16, int32, or int64 + tiledb::Filter xor_filter(ctx, TILEDB_FILTER_XOR); + filters.add_filter(xor_filter); + CHECK_NOTHROW(d1.set_filter_list(filters)); + CHECK_NOTHROW(a1.set_filter_list(filters)); + } + + SECTION("- Multiple compressors") { + tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); + auto compressor = GENERATE( + TILEDB_FILTER_GZIP, + TILEDB_FILTER_LZ4, + TILEDB_FILTER_RLE, + TILEDB_FILTER_ZSTD); + tiledb::Filter compressor_filter(ctx, compressor); + filters.add_filter(bzip); + filters.add_filter(compressor_filter); + + CHECK_NOTHROW(d1.set_filter_list(filters)); + CHECK_NOTHROW(a1.set_filter_list(filters)); + + // Should throw without FloatScale to convert float->int32. + auto delta_compressor = GENERATE( + TILEDB_FILTER_POSITIVE_DELTA, + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA); + tiledb::Filter delta_filter(ctx, delta_compressor); + filters.add_filter(delta_filter); + CHECK_THROWS(d1.set_filter_list(filters)); + CHECK_THROWS(a1.set_filter_list(filters)); + } + + SECTION("- Multiple compressors following type conversion") { + auto compressor = GENERATE( + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA, + TILEDB_FILTER_GZIP, + TILEDB_FILTER_LZ4, + TILEDB_FILTER_RLE, + TILEDB_FILTER_ZSTD); + tiledb::Filter compressor_filter(ctx, compressor); + tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); + filters.add_filter(float_scale); + filters.add_filter(bzip); + filters.add_filter(compressor_filter); + + CHECK_NOTHROW(d1.set_filter_list(filters)); + CHECK_NOTHROW(a1.set_filter_list(filters)); + } +} diff --git a/test/src/unit-filter-pipeline.cc b/test/src/unit-filter-pipeline.cc deleted file mode 100644 index 84e0060b5f22..000000000000 --- a/test/src/unit-filter-pipeline.cc +++ /dev/null @@ -1,1814 +0,0 @@ -/** - * @file unit-filter-pipeline.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. - * @copyright Copyright (c) 2016 MIT and Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests the `FilterPipeline` class. - */ - -#include "test/support/src/helpers.h" -#include "tiledb/common/common.h" -#include "tiledb/sm/array_schema/array_schema.h" -#include "tiledb/sm/array_schema/attribute.h" -#include "tiledb/sm/array_schema/dimension.h" -#include "tiledb/sm/array_schema/domain.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/config/config.h" -#include "tiledb/sm/crypto/encryption_key.h" -#include "tiledb/sm/enums/compressor.h" -#include "tiledb/sm/enums/datatype.h" -#include "tiledb/sm/enums/encryption_type.h" -#include "tiledb/sm/enums/filter_option.h" -#include "tiledb/sm/enums/filter_type.h" -#include "tiledb/sm/filter/bit_width_reduction_filter.h" -#include "tiledb/sm/filter/bitshuffle_filter.h" -#include "tiledb/sm/filter/byteshuffle_filter.h" -#include "tiledb/sm/filter/checksum_md5_filter.h" -#include "tiledb/sm/filter/checksum_sha256_filter.h" -#include "tiledb/sm/filter/compression_filter.h" -#include "tiledb/sm/filter/encryption_aes256gcm_filter.h" -#include "tiledb/sm/filter/filter_pipeline.h" -#include "tiledb/sm/filter/float_scaling_filter.h" -#include "tiledb/sm/filter/positive_delta_filter.h" -#include "tiledb/sm/filter/webp_filter.h" -#include "tiledb/sm/filter/xor_filter.h" -#include "tiledb/sm/tile/tile.h" -#include "tiledb/stdx/utility/to_underlying.h" - -#include -#include -#include - -using namespace tiledb; -using namespace tiledb::common; -using namespace tiledb::sm; - -shared_ptr make_increasing_tile( - const uint64_t nelts, shared_ptr tracker) { - const uint64_t tile_size = nelts * sizeof(uint64_t); - const uint64_t cell_size = sizeof(uint64_t); - - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - cell_size, - tile_size, - tracker); - for (uint64_t i = 0; i < nelts; i++) { - CHECK_NOTHROW(tile->write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - return tile; -} - -shared_ptr make_offsets_tile( - std::vector& offsets, shared_ptr tracker) { - const uint64_t offsets_tile_size = - offsets.size() * constants::cell_var_offset_size; - - auto offsets_tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - constants::cell_var_offset_size, - offsets_tile_size, - tracker); - - // Set up test data - for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK_NOTHROW(offsets_tile->write( - &offsets[i], - i * constants::cell_var_offset_size, - constants::cell_var_offset_size)); - } - - return offsets_tile; -} - -Tile create_tile_for_unfiltering( - uint64_t nelts, - shared_ptr tile, - shared_ptr tracker) { - return { - tile->format_version(), - tile->type(), - tile->cell_size(), - 0, - tile->cell_size() * nelts, - tile->filtered_buffer().data(), - tile->filtered_buffer().size(), - tracker}; -} - -void run_reverse( - const tiledb::sm::Config& config, - ThreadPool& tp, - Tile& unfiltered_tile, - FilterPipeline& pipeline, - bool success = true) { - ChunkData chunk_data; - unfiltered_tile.load_chunk_data(chunk_data); - CHECK( - success == pipeline - .run_reverse( - &test::g_helper_stats, - &unfiltered_tile, - nullptr, - chunk_data, - 0, - chunk_data.filtered_chunks_.size(), - tp.concurrency_level(), - config) - .ok()); -} - -TEST_CASE( - "Filter: Test skip checksum validation", - "[filter][skip-checksum-validation]") { - tiledb::sm::Config config; - REQUIRE(config.set("sm.skip_checksum_validation", "true").ok()); - - auto tracker = tiledb::test::create_test_memory_tracker(); - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts, tracker); - - // MD5 - FilterPipeline md5_pipeline; - ThreadPool tp(4); - ChecksumMD5Filter md5_filter(Datatype::UINT64); - md5_pipeline.add_filter(md5_filter); - CHECK( - md5_pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, md5_pipeline); - - for (uint64_t n = 0; n < nelts; n++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == n); - } - - // SHA256 - auto tile2 = make_increasing_tile(nelts, tracker); - - FilterPipeline sha_256_pipeline; - ChecksumMD5Filter sha_256_filter(Datatype::UINT64); - sha_256_pipeline.add_filter(sha_256_filter); - CHECK(sha_256_pipeline - .run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) - .ok()); - CHECK(tile2->size() == 0); - CHECK(tile2->filtered_buffer().size() != 0); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile2, tracker); - run_reverse(config, tp, unfiltered_tile2, sha_256_pipeline); - for (uint64_t n = 0; n < nelts; n++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile2.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == n); - } -} - -TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 1000; - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts, tracker); - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - // Sanity check number of windows value - uint64_t offset = 0; - offset += sizeof(uint64_t); // Number of chunks - offset += sizeof(uint32_t); // First chunk orig size - offset += sizeof(uint32_t); // First chunk filtered size - offset += sizeof(uint32_t); // First chunk metadata size - - CHECK( - tile->filtered_buffer().value_at_as(offset) == - nelts * sizeof(uint64_t)); // Original length - offset += sizeof(uint32_t); - - auto max_win_size = - pipeline.get_filter()->max_window_size(); - auto expected_num_win = - (nelts * sizeof(uint64_t)) / max_win_size + - uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - expected_num_win); // Number of windows - - // Check compression worked - auto compressed_size = tile->filtered_buffer().size(); - CHECK(compressed_size < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Window sizes") { - std::vector window_sizes = { - 32, 64, 128, 256, 437, 512, 1024, 2000}; - for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts, tracker); - - pipeline.get_filter()->set_max_window_size( - window_size); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - } - - SECTION("- Random values") { - std::random_device rd; - auto seed = rd(); - std::mt19937 gen(seed), gen_copy(seed); - std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); - INFO("Random element seed: " << seed); - - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - sizeof(uint64_t), - nelts * sizeof(uint64_t), - tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val = (uint64_t)rng(gen); - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK((int64_t)elt == rng(gen_copy)); - } - } - - SECTION(" - Random signed values") { - std::random_device rd; - auto seed = rd(); - std::mt19937 gen(seed), gen_copy(seed); - std::uniform_int_distribution<> rng( - std::numeric_limits::lowest(), - std::numeric_limits::max()); - INFO("Random element seed: " << seed); - - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - nelts * sizeof(uint32_t), - tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint32_t val = (uint32_t)rng(gen); - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - int32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); - CHECK(elt == rng(gen_copy)); - } - } - - SECTION("- Byte overflow") { - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - sizeof(uint64_t), - nelts * sizeof(uint64_t), - tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val = i % 257; - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i % 257); - } - } -} - -TEST_CASE( - "Filter: Test bit width reduction var", - "[filter][bit-width-reduction][var]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - const uint64_t nelts = 100; - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - const uint64_t offsets_tile_size = - offsets.size() * constants::cell_var_offset_size; - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts, tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile->filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile->filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - out_sizes[i] / 8); // Chunk filtered size - offset += sizeof(uint32_t); - - uint32_t md_size = tile->filtered_buffer().value_at_as(offset); - offset += sizeof(uint32_t); - - CHECK( - tile->filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Original length - offset += sizeof(uint32_t); - - // Check window value. - auto max_win_size = - pipeline.get_filter()->max_window_size(); - auto expected_num_win = out_sizes[i] / max_win_size + - uint32_t(bool(out_sizes[0] % max_win_size)); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - expected_num_win); // Number of windows - - offset += md_size - sizeof(uint32_t); - - // Check all elements are good. - uint8_t el = 0; - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK(tile->filtered_buffer().value_at_as(offset) == el++); - offset += sizeof(uint8_t); - } - } - - // Check compression worked - auto compressed_size = tile->filtered_buffer().size(); - CHECK(compressed_size < nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Window sizes") { - WriterTile::set_max_tile_chunk_size(80); - std::vector window_sizes = { - 32, 64, 128, 256, 437, 512, 1024, 2000}; - for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts, tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - pipeline.get_filter()->set_max_window_size( - window_size); - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - } - - SECTION("- Random values") { - WriterTile::set_max_tile_chunk_size(80); - std::random_device rd; - auto seed = rd(); - std::mt19937 gen(seed), gen_copy(seed); - std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); - INFO("Random element seed: " << seed); - - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - sizeof(uint64_t), - nelts * sizeof(uint64_t), - tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val = (uint64_t)rng(gen); - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK((int64_t)elt == rng(gen_copy)); - } - } - - SECTION(" - Random signed values") { - WriterTile::set_max_tile_chunk_size(80); - std::random_device rd; - auto seed = rd(); - std::mt19937 gen(seed), gen_copy(seed); - std::uniform_int_distribution<> rng( - std::numeric_limits::lowest(), - std::numeric_limits::max()); - INFO("Random element seed: " << seed); - - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - nelts * sizeof(uint32_t), - tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint32_t val = (uint32_t)rng(gen); - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - std::vector offsets32(offsets); - for (uint64_t i = 0; i < offsets32.size(); i++) { - offsets32[i] /= 2; - } - - WriterTile offsets_tile32( - constants::format_version, - Datatype::UINT64, - constants::cell_var_offset_size, - offsets_tile_size, - tracker); - - // Set up test data - for (uint64_t i = 0; i < offsets.size(); i++) { - CHECK_NOTHROW(offsets_tile32.write( - &offsets32[i], - i * constants::cell_var_offset_size, - constants::cell_var_offset_size)); - } - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), &offsets_tile32, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - int32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); - CHECK(elt == rng(gen_copy)); - } - } - - SECTION("- Byte overflow") { - WriterTile::set_max_tile_chunk_size(80); - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::UINT64, - sizeof(uint64_t), - nelts * sizeof(uint64_t), - tracker); - - // Set up test data - for (uint64_t i = 0; i < nelts; i++) { - uint64_t val = i % 257; - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - auto offsets_tile = make_offsets_tile(offsets, tracker); - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i % 257); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 1000; - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto pipeline_metadata_size = sizeof(uint64_t) + 3 * sizeof(uint32_t); - - uint64_t offset = 0; - offset += sizeof(uint64_t); // Number of chunks - offset += sizeof(uint32_t); // First chunk orig size - offset += sizeof(uint32_t); // First chunk filtered size - auto filter_metadata_size = tile->filtered_buffer().value_at_as( - offset); // First chunk metadata size - offset += sizeof(uint32_t); - - auto max_win_size = - pipeline.get_filter()->max_window_size(); - auto expected_num_win = - (nelts * sizeof(uint64_t)) / max_win_size + - uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - expected_num_win); // Number of windows - - // Check encoded size - auto encoded_size = tile->filtered_buffer().size(); - CHECK( - encoded_size == pipeline_metadata_size + filter_metadata_size + - nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Window sizes") { - std::vector window_sizes = { - 32, 64, 128, 256, 437, 512, 1024, 2000}; - for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts, tracker); - pipeline.get_filter()->set_max_window_size( - window_size); - - CHECK( - pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - } - - SECTION("- Error on non-positive delta data") { - auto tile = make_increasing_tile(nelts, tracker); - for (uint64_t i = 0; i < nelts; i++) { - auto val = nelts - i; - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - CHECK(!pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - } -} - -TEST_CASE( - "Filter: Test positive-delta encoding var", - "[filter][positive-delta][var]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - const uint64_t nelts = 100; - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - auto tile = make_increasing_tile(nelts, tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - uint64_t offset = 0; - CHECK( - tile->filtered_buffer().value_at_as(offset) == - 9); // Number of chunks - offset += sizeof(uint64_t); - - uint64_t total_md_size = 0; - for (uint64_t i = 0; i < 9; i++) { - CHECK( - tile->filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk orig size - offset += sizeof(uint32_t); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - out_sizes[i]); // Chunk filtered size - offset += sizeof(uint32_t); - - uint32_t md_size = tile->filtered_buffer().value_at_as(offset); - offset += sizeof(uint32_t); - total_md_size += md_size; - - auto max_win_size = - pipeline.get_filter()->max_window_size(); - auto expected_num_win = - (nelts * sizeof(uint64_t)) / max_win_size + - uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); - CHECK( - tile->filtered_buffer().value_at_as(offset) == - expected_num_win); // Number of windows - - offset += md_size; - - // Check all elements are good. - for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { - CHECK( - tile->filtered_buffer().value_at_as(offset) == - (j == 0 ? 0 : 1)); - offset += sizeof(uint64_t); - } - } - - // Check encoded size - auto pipeline_metadata_size = sizeof(uint64_t) + 9 * 3 * sizeof(uint32_t); - auto encoded_size = tile->filtered_buffer().size(); - CHECK( - encoded_size == - pipeline_metadata_size + total_md_size + nelts * sizeof(uint64_t)); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Window sizes") { - WriterTile::set_max_tile_chunk_size(80); - std::vector window_sizes = { - 32, 64, 128, 256, 437, 512, 1024, 2000}; - for (auto window_size : window_sizes) { - auto tile = make_increasing_tile(nelts, tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - - pipeline.get_filter()->set_max_window_size( - window_size); - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - } - - SECTION("- Error on non-positive delta data") { - auto tile = make_increasing_tile(nelts, tracker); - auto offsets_tile = make_offsets_tile(offsets, tracker); - - WriterTile::set_max_tile_chunk_size(80); - for (uint64_t i = 0; i < nelts; i++) { - auto val = nelts - i; - CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); - } - - CHECK(!pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts, tracker); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Indivisible by 8") { - const uint32_t nelts2 = 1001; - const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - - auto tile2 = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - tile_size2, - tracker); - - // Set up test data - for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) - .ok()); - CHECK(tile2->size() == 0); - CHECK(tile2->filtered_buffer().size() != 0); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); - run_reverse(config, tp, unfiltered_tile2, pipeline); - for (uint64_t i = 0; i < nelts2; i++) { - uint32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts, tracker); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets, tracker); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Indivisible by 8") { - WriterTile::set_max_tile_chunk_size(80); - const uint32_t nelts2 = 1001; - const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - - auto tile2 = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - tile_size2, - tracker); - - // Set up test data - for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile2->size() == 0); - CHECK(tile2->filtered_buffer().size() != 0); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); - run_reverse(config, tp, unfiltered_tile2, pipeline); - for (uint64_t i = 0; i < nelts2; i++) { - uint32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts, tracker); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Uneven number of elements") { - const uint32_t nelts2 = 1001; - const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - - auto tile2 = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - tile_size2, - tracker); - - // Set up test data - for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile2.get(), nullptr, &tp) - .ok()); - CHECK(tile2->size() == 0); - CHECK(tile2->filtered_buffer().size() != 0); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); - run_reverse(config, tp, unfiltered_tile2, pipeline); - for (uint64_t i = 0; i < nelts2; i++) { - uint32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); - CHECK(elt == i); - } - } -} - -TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - const uint64_t nelts = 100; - auto tile = make_increasing_tile(nelts, tracker); - - // Set up test data - std::vector sizes{ - 0, - 32, // Chunk0: 4 cells. - 80, // 10 cells, still makes it into this chunk as current size < 50%. - 48, // Chunk1: 6 cells. - 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. - 56, // Chunk3: 7 cells. - 72, // Chunk4: 9 cells, new size > 50%. - 8, // Chunk4: 10 cell, full. - 80, // Chunk5: 10 cells. - 160, // Chunk6: 20 cells. - 16, // Chunk7: 2 cells. - 16, // Chunk7: 4 cells. - 16, // Chunk7: 6 cells. - 16, // Chunk7: 8 cells. - 16, // Chunk7: 10 cells. - }; // Chunk8: 12 cells. - - std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; - - std::vector offsets(sizes.size()); - uint64_t offset = 0; - for (uint64_t i = 0; i < offsets.size() - 1; i++) { - offsets[i] = offset; - offset += sizes[i + 1]; - } - offsets[offsets.size() - 1] = offset; - - auto offsets_tile = make_offsets_tile(offsets, tracker); - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); - - SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } - - SECTION("- Uneven number of elements") { - WriterTile::set_max_tile_chunk_size(80); - const uint32_t nelts2 = 1001; - const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); - - auto tile2 = make_shared( - HERE(), - constants::format_version, - Datatype::UINT32, - sizeof(uint32_t), - tile_size2, - tracker); - - // Set up test data - for (uint32_t i = 0; i < nelts2; i++) { - CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); - } - - CHECK(pipeline - .run_forward( - &test::g_helper_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); - CHECK(tile2->size() == 0); - CHECK(tile2->filtered_buffer().size() != 0); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); - run_reverse(config, tp, unfiltered_tile2, pipeline); - for (uint64_t i = 0; i < nelts2; i++) { - uint32_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); - CHECK(elt == i); - } - } - - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); -} - -TEST_CASE("Filter: Test encryption", "[filter][encryption]") { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 1000; - auto tile = make_increasing_tile(nelts, tracker); - - SECTION("- AES-256-GCM") { - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(EncryptionAES256GCMFilter(Datatype::UINT64)); - - // No key set - CHECK(!pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - - // Create and set a key - char key[32]; - for (unsigned i = 0; i < 32; i++) - key[i] = (char)i; - auto filter = pipeline.get_filter(); - filter->set_key(key); - - // Check success - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - - // Check error decrypting with wrong key. - tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - key[0]++; - filter->set_key(key); - - auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile2, pipeline, false); - - // Fix key and check success. - auto unfiltered_tile3 = create_tile_for_unfiltering(nelts, tile, tracker); - - key[0]--; - filter->set_key(key); - run_reverse(config, tp, unfiltered_tile3, pipeline); - - for (uint64_t i = 0; i < nelts; i++) { - uint64_t elt = 0; - CHECK_NOTHROW( - unfiltered_tile3.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); - CHECK(elt == i); - } - } -} - -template -void testing_float_scaling_filter() { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 100; - const uint64_t tile_size = nelts * sizeof(FloatingType); - const uint64_t cell_size = sizeof(FloatingType); - - Datatype t = Datatype::FLOAT32; - switch (sizeof(FloatingType)) { - case 4: { - t = Datatype::FLOAT32; - } break; - case 8: { - t = Datatype::FLOAT64; - } break; - default: { - INFO( - "testing_float_scaling_filter: passed floating type with size of not " - "4 bytes or 8 bytes."); - CHECK(false); - } - } - - auto tile = make_shared( - HERE(), constants::format_version, t, cell_size, tile_size, tracker); - - std::vector float_result_vec; - double scale = 2.53; - double foffset = 0.31589; - uint64_t byte_width = sizeof(IntType); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution dis(0.0, 213.0); - - for (uint64_t i = 0; i < nelts; i++) { - FloatingType f = dis(gen); - CHECK_NOTHROW( - tile->write(&f, i * sizeof(FloatingType), sizeof(FloatingType))); - - IntType val = static_cast(round( - (f - static_cast(foffset)) / - static_cast(scale))); - - FloatingType val_float = static_cast( - scale * static_cast(val) + foffset); - float_result_vec.push_back(val_float); - } - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(FloatScalingFilter(t)); - CHECK(pipeline.get_filter() - ->set_option(FilterOption::SCALE_FLOAT_BYTEWIDTH, &byte_width) - .ok()); - CHECK(pipeline.get_filter() - ->set_option(FilterOption::SCALE_FLOAT_FACTOR, &scale) - .ok()); - CHECK(pipeline.get_filter() - ->set_option(FilterOption::SCALE_FLOAT_OFFSET, &foffset) - .ok()); - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - FloatingType elt = 0.0f; - CHECK_NOTHROW(unfiltered_tile.read( - &elt, i * sizeof(FloatingType), sizeof(FloatingType))); - CHECK(elt == float_result_vec[i]); - } -} - -TEMPLATE_TEST_CASE( - "Filter: Test float scaling", - "[filter][float-scaling]", - int8_t, - int16_t, - int32_t, - int64_t) { - testing_float_scaling_filter(); - testing_float_scaling_filter(); -} - -/* - Defining distribution types to pass into the testing_xor_filter function. - */ -typedef typename std::uniform_int_distribution IntDistribution; -typedef typename std::uniform_real_distribution FloatDistribution; - -template -void testing_xor_filter(Datatype t) { - tiledb::sm::Config config; - - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set up test data - const uint64_t nelts = 100; - const uint64_t tile_size = nelts * sizeof(T); - const uint64_t cell_size = sizeof(T); - - auto tile = make_shared( - HERE(), constants::format_version, t, cell_size, tile_size, tracker); - - // Setting up the random number generator for the XOR filter testing. - std::mt19937_64 gen(0x57A672DE); - Distribution dis( - std::numeric_limits::min(), std::numeric_limits::max()); - - std::vector results; - - for (uint64_t i = 0; i < nelts; i++) { - T val = static_cast(dis(gen)); - CHECK_NOTHROW(tile->write(&val, i * sizeof(T), sizeof(T))); - results.push_back(val); - } - - FilterPipeline pipeline; - ThreadPool tp(4); - pipeline.add_filter(XORFilter(t)); - - CHECK(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - - // Check new size and number of chunks - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); - run_reverse(config, tp, unfiltered_tile, pipeline); - for (uint64_t i = 0; i < nelts; i++) { - T elt = 0; - CHECK_NOTHROW(unfiltered_tile.read(&elt, i * sizeof(T), sizeof(T))); - CHECK(elt == results[i]); - } -} - -TEST_CASE("Filter: Test XOR", "[filter][xor]") { - testing_xor_filter(Datatype::INT8); - testing_xor_filter(Datatype::UINT8); - testing_xor_filter(Datatype::INT16); - testing_xor_filter(Datatype::UINT16); - testing_xor_filter(Datatype::INT32); - testing_xor_filter(Datatype::UINT32); - testing_xor_filter(Datatype::INT64); - testing_xor_filter>( - Datatype::UINT64); - testing_xor_filter(Datatype::FLOAT32); - testing_xor_filter(Datatype::FLOAT64); - testing_xor_filter(Datatype::CHAR); - testing_xor_filter(Datatype::DATETIME_YEAR); - testing_xor_filter(Datatype::DATETIME_MONTH); - testing_xor_filter(Datatype::DATETIME_WEEK); - testing_xor_filter(Datatype::DATETIME_DAY); - testing_xor_filter(Datatype::DATETIME_HR); - testing_xor_filter(Datatype::DATETIME_MIN); - testing_xor_filter(Datatype::DATETIME_SEC); - testing_xor_filter(Datatype::DATETIME_MS); - testing_xor_filter(Datatype::DATETIME_US); - testing_xor_filter(Datatype::DATETIME_NS); - testing_xor_filter(Datatype::DATETIME_PS); - testing_xor_filter(Datatype::DATETIME_FS); - testing_xor_filter(Datatype::DATETIME_AS); -} - -TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { - FilterPipeline pipeline; - auto tracker = tiledb::test::create_test_memory_tracker(); - - SECTION("- DoubleDelta filter reinterprets float->int32") { - pipeline.add_filter(CompressionFilter( - tiledb::sm::Compressor::DOUBLE_DELTA, - 0, - Datatype::FLOAT32, - Datatype::INT32)); - pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); - } - - SECTION("- Delta filter reinterprets float->int32") { - pipeline.add_filter(CompressionFilter( - tiledb::sm::Compressor::DELTA, 0, Datatype::FLOAT32, Datatype::INT32)); - pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); - } - - SECTION("- FloatScale filter converts float->int32") { - pipeline.add_filter( - FloatScalingFilter(sizeof(int32_t), 1.0f, 0.0f, Datatype::FLOAT32)); - pipeline.add_filter(PositiveDeltaFilter(Datatype::INT32)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::DELTA, 0, Datatype::INT32)); - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::BZIP2, 2, Datatype::INT32)); - pipeline.add_filter(BitshuffleFilter(Datatype::INT32)); - pipeline.add_filter(ByteshuffleFilter(Datatype::INT32)); - pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); - } - - size_t byte_width = 0; - SECTION("- XOR filter expected output types") { - byte_width = GENERATE( - sizeof(int8_t), sizeof(int16_t), sizeof(int32_t), sizeof(int64_t)); - pipeline.add_filter( - FloatScalingFilter(byte_width, 1.0f, 0.0f, Datatype::FLOAT32)); - auto byte_width_t = - pipeline.get_filter()->output_datatype( - Datatype::FLOAT32); - pipeline.add_filter(XORFilter(byte_width_t)); - } - - SECTION("- XOR filter expected output types large pipeline") { - byte_width = GENERATE( - sizeof(int8_t), sizeof(int16_t), sizeof(int32_t), sizeof(int64_t)); - pipeline.add_filter( - FloatScalingFilter(byte_width, 1.0f, 0.0f, Datatype::FLOAT32)); - auto byte_width_t = - pipeline.get_filter()->output_datatype( - Datatype::FLOAT32); - pipeline.add_filter(PositiveDeltaFilter(byte_width_t)); - pipeline.add_filter(BitshuffleFilter(byte_width_t)); - pipeline.add_filter(ByteshuffleFilter(byte_width_t)); - pipeline.add_filter(XORFilter(byte_width_t)); - } - - // Initial type of tile is float. - std::vector data = { - 1.0f, 2.1f, 3.2f, 4.3f, 5.4f, 6.5f, 7.6f, 8.7f, 9.8f, 10.9f}; - auto tile = make_shared( - HERE(), - constants::format_version, - Datatype::FLOAT32, - sizeof(float), - sizeof(float) * data.size(), - tracker); - for (size_t i = 0; i < data.size(); i++) { - CHECK_NOTHROW(tile->write(&data[i], i * sizeof(float), sizeof(float))); - } - - ThreadPool tp(4); - REQUIRE(pipeline.run_forward(&test::g_helper_stats, tile.get(), nullptr, &tp) - .ok()); - CHECK(tile->size() == 0); - CHECK(tile->filtered_buffer().size() != 0); - - auto unfiltered_tile = - create_tile_for_unfiltering(data.size(), tile, tracker); - ChunkData chunk_data; - unfiltered_tile.load_chunk_data(chunk_data); - REQUIRE(pipeline - .run_reverse( - &test::g_helper_stats, - &unfiltered_tile, - nullptr, - chunk_data, - 0, - chunk_data.filtered_chunks_.size(), - tp.concurrency_level(), - tiledb::sm::Config()) - .ok()); - std::vector results{ - 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; - for (size_t i = 0; i < data.size(); i++) { - float val = 0; - CHECK_NOTHROW(unfiltered_tile.read(&val, i * sizeof(float), sizeof(float))); - if (pipeline.has_filter(tiledb::sm::FilterType::FILTER_SCALE_FLOAT)) { - // Loss of precision from rounding in FloatScale filter. - CHECK(val == results[i]); - } else { - CHECK(val == data[i]); - } - } -} - -TEST_CASE( - "C++ API: Pipeline with filtered type conversions", - "[cppapi][filter][pipeline]") { - tiledb::Context ctx; - tiledb::VFS vfs(ctx); - std::string array_name = "cpp_test_array"; - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } - - tiledb::Domain domain(ctx); - float domain_lo = static_cast(std::numeric_limits::min()); - float domain_hi = static_cast(std::numeric_limits::max() - 1); - - // Create and initialize dimension. - auto d1 = tiledb::Dimension::create( - ctx, "d1", {{domain_lo, domain_hi}}, 2048); - - tiledb::Filter float_scale(ctx, TILEDB_FILTER_SCALE_FLOAT); - double scale = 1.0f; - double offset = 0.0f; - uint64_t byte_width = sizeof(int32_t); - - // Float scale converting tile data from float->int32 - float_scale.set_option(TILEDB_SCALE_FLOAT_BYTEWIDTH, &byte_width); - float_scale.set_option(TILEDB_SCALE_FLOAT_FACTOR, &scale); - float_scale.set_option(TILEDB_SCALE_FLOAT_OFFSET, &offset); - - // Delta filter reinterprets int32->uint32 - tiledb::Filter delta(ctx, TILEDB_FILTER_DELTA); - - // Pass uint32 data to BitWidthReduction filter - tiledb::Filter bit_width_reduction(ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION); - - tiledb::FilterList filters(ctx); - filters.add_filter(float_scale); - filters.add_filter(delta); - filters.add_filter(bit_width_reduction); - - // Apply filters to both attribute and dimension. - REQUIRE_NOTHROW(d1.set_filter_list(filters)); - domain.add_dimension(d1); - - auto a1 = tiledb::Attribute::create(ctx, "a1"); - REQUIRE_NOTHROW(a1.set_filter_list(filters)); - - tiledb::ArraySchema schema(ctx, TILEDB_SPARSE); - schema.set_domain(domain); - schema.add_attribute(a1); - schema.set_cell_order(TILEDB_ROW_MAJOR); - schema.set_tile_order(TILEDB_ROW_MAJOR); - REQUIRE_NOTHROW(tiledb::Array::create(array_name, schema)); - std::vector d1_data = { - 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f}; - std::vector a1_data = { - 1.0f, 2.1f, 3.2f, 4.3f, 5.4f, 6.5f, 7.6f, 8.7f, 9.8f, 10.9f}; - - // Write to array. - { - tiledb::Array array(ctx, array_name, TILEDB_WRITE); - tiledb::Query query(ctx, array); - query.set_data_buffer("d1", d1_data); - query.set_data_buffer("a1", a1_data); - query.submit(); - CHECK(tiledb::Query::Status::COMPLETE == query.query_status()); - } - - // Read from array. - { - std::vector d1_read(10); - std::vector a1_read(10); - tiledb::Array array(ctx, array_name, TILEDB_READ); - tiledb::Query query(ctx, array); - query.set_subarray({domain_lo, domain_hi}); - query.set_data_buffer("a1", a1_read); - query.set_data_buffer("d1", d1_read); - query.submit(); - CHECK(tiledb::Query::Status::COMPLETE == query.query_status()); - // Some loss of precision from rounding in FloatScale. - CHECK( - std::vector{ - 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f} == - a1_read); - CHECK(d1_data == d1_read); - } - - // Cleanup. - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } -} - -TEST_CASE( - "C++ API: Filter pipeline validation", - "[cppapi][filter][pipeline][validation]") { - tiledb::Context ctx; - - tiledb::Domain domain(ctx); - float domain_lo = static_cast(std::numeric_limits::min()); - float domain_hi = static_cast(std::numeric_limits::max() - 1); - auto d1 = tiledb::Dimension::create( - ctx, "d1", {{domain_lo, domain_hi}}, 2048); - auto a1 = tiledb::Attribute::create(ctx, "a1"); - - // FloatScale used for testing different float->integral pipelines. - tiledb::Filter float_scale(ctx, TILEDB_FILTER_SCALE_FLOAT); - double scale = 1.0f; - double offset = 0.0f; - uint64_t byte_width = sizeof(int32_t); - // Float scale converting tile data from float->int32 - float_scale.set_option(TILEDB_SCALE_FLOAT_BYTEWIDTH, &byte_width); - float_scale.set_option(TILEDB_SCALE_FLOAT_FACTOR, &scale); - float_scale.set_option(TILEDB_SCALE_FLOAT_OFFSET, &offset); - - tiledb::FilterList filters(ctx); - SECTION("- FloatScale filter accepts float or double byte width input") { - auto d2 = tiledb::Dimension::create(ctx, "d2", {{1, 100}}, 10); - auto a2 = tiledb::Attribute::create(ctx, "a2"); - filters.add_filter(float_scale); - CHECK_THROWS(d2.set_filter_list(filters)); - CHECK_NOTHROW(a2.set_filter_list(filters)); - } - - SECTION("- Delta filters do not accept real datatypes") { - auto test_filter = GENERATE( - TILEDB_FILTER_POSITIVE_DELTA, - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA); - tiledb::Filter delta_filter(ctx, test_filter); - filters.add_filter(delta_filter); - // Delta compressors don't accept floats. Should fail without FloatScale. - CHECK_THROWS(d1.set_filter_list(filters)); - CHECK_THROWS(a1.set_filter_list(filters)); - - // Test using FloatScale to convert to integral is accepted. - tiledb::FilterList filters2(ctx); - filters2.add_filter(float_scale); - filters2.add_filter(delta_filter); - CHECK_NOTHROW(d1.set_filter_list(filters2)); - CHECK_NOTHROW(a1.set_filter_list(filters2)); - } - - SECTION("- Webp filter supports only uint8 attributes") { - if (webp_filter_exists) { - tiledb::Filter webp(ctx, TILEDB_FILTER_WEBP); - filters.add_filter(webp); - CHECK_THROWS(d1.set_filter_list(filters)); - CHECK_THROWS(a1.set_filter_list(filters)); - } - } - - SECTION("- Bit width reduction filter supports integral input") { - tiledb::Filter bit_width_reduction(ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION); - filters.add_filter(bit_width_reduction); - CHECK_THROWS(d1.set_filter_list(filters)); - CHECK_THROWS(a1.set_filter_list(filters)); - - // Test using FloatScale to convert to integral is accepted. - tiledb::FilterList filters2(ctx); - filters2.add_filter(float_scale); - filters2.add_filter(bit_width_reduction); - CHECK_NOTHROW(d1.set_filter_list(filters2)); - CHECK_NOTHROW(a1.set_filter_list(filters2)); - } - - SECTION("- XOR filter interprets datatype as integral") { - // Datatype byte size must match size of int8, int16, int32, or int64 - tiledb::Filter xor_filter(ctx, TILEDB_FILTER_XOR); - filters.add_filter(xor_filter); - CHECK_NOTHROW(d1.set_filter_list(filters)); - CHECK_NOTHROW(a1.set_filter_list(filters)); - } - - SECTION("- Multiple compressors") { - tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); - auto compressor = GENERATE( - TILEDB_FILTER_GZIP, - TILEDB_FILTER_LZ4, - TILEDB_FILTER_RLE, - TILEDB_FILTER_ZSTD); - tiledb::Filter compressor_filter(ctx, compressor); - filters.add_filter(bzip); - filters.add_filter(compressor_filter); - - CHECK_NOTHROW(d1.set_filter_list(filters)); - CHECK_NOTHROW(a1.set_filter_list(filters)); - - // Should throw without FloatScale to convert float->int32. - auto delta_compressor = GENERATE( - TILEDB_FILTER_POSITIVE_DELTA, - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA); - tiledb::Filter delta_filter(ctx, delta_compressor); - filters.add_filter(delta_filter); - CHECK_THROWS(d1.set_filter_list(filters)); - CHECK_THROWS(a1.set_filter_list(filters)); - } - - SECTION("- Multiple compressors following type conversion") { - auto compressor = GENERATE( - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA, - TILEDB_FILTER_GZIP, - TILEDB_FILTER_LZ4, - TILEDB_FILTER_RLE, - TILEDB_FILTER_ZSTD); - tiledb::Filter compressor_filter(ctx, compressor); - tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); - filters.add_filter(float_scale); - filters.add_filter(bzip); - filters.add_filter(compressor_filter); - - CHECK_NOTHROW(d1.set_filter_list(filters)); - CHECK_NOTHROW(a1.set_filter_list(filters)); - } -} diff --git a/tiledb/sm/filter/test/CMakeLists.txt b/tiledb/sm/filter/test/CMakeLists.txt index 71844c964ea8..edfc14b9068a 100644 --- a/tiledb/sm/filter/test/CMakeLists.txt +++ b/tiledb/sm/filter/test/CMakeLists.txt @@ -44,9 +44,17 @@ commence(unit_test run_filter_pipeline) add_1_including_metadata_filter.cc add_1_out_of_place_filter.cc add_n_in_place_filter.cc + filter_test_support.cc filtered_tile_checker.cc pseudo_checksum_filter.cc + unit_checksum_pipeline.cc + unit_bit_width_reduction_pipeline.cc + unit_bitshuffle_pipeline.cc + unit_byteshuffle_pipeline.cc + unit_encryption_pipeline.cc + unit_positive_delta_pipeline.cc unit_run_filter_pipeline.cc + unit_xor_pipeline.cc ) conclude(unit_test) diff --git a/tiledb/sm/filter/test/filter_test_support.cc b/tiledb/sm/filter/test/filter_test_support.cc new file mode 100644 index 000000000000..ec1b2f4803c4 --- /dev/null +++ b/tiledb/sm/filter/test/filter_test_support.cc @@ -0,0 +1,235 @@ +/** + * @file filter_test_support.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains helper functions and test data classes for running tests + * on filters and filter pipelines. + * + **/ + +#include "filter_test_support.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +SimpleVariableTestData::SimpleVariableTestData() + : target_ncells_per_chunk_{10} + , elements_per_chunk_{14, 6, 11, 7, 10, 10, 20, 10, 12} + , tile_data_generator_{{4, 10, 6, 11, 7, 9, 1, 10, 20, 2, 2, 2, 2, 2, 12}} { + WriterTile::set_max_tile_chunk_size( + target_ncells_per_chunk_ * sizeof(uint64_t)); +} + +SimpleVariableTestData::~SimpleVariableTestData() { + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} + +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Filtered buffer data is as expected. + * 3. Pipeline runs backward without error. + * 4. Result from roundtrip matches the original data. + */ +void check_run_pipeline_full( + Config& config, + ThreadPool& tp, + shared_ptr& tile, + std::optional>& offsets_tile, + FilterPipeline& pipeline, + const TileDataGenerator* test_data, + const FilteredTileChecker& filtered_buffer_checker, + shared_ptr memory_tracker) { + // Run the pipeline forward. + CHECK(pipeline + .run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp) + .ok()); + + // Check the original unfiltered data was removed. + CHECK(tile->size() == 0); + + // Check the filtered buffer has the expected data. + auto filtered_buffer = tile->filtered_buffer(); + filtered_buffer_checker.check(filtered_buffer); + + // Run the data in reverse. + auto unfiltered_tile = + test_data->create_filtered_buffer_tile(filtered_buffer, memory_tracker); + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + CHECK(pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); + + // Check the original data is reverted. + test_data->check_tile_data(unfiltered_tile); +} + +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Pipeline runs backward without error. + * 3. Result from roundtrip matches the original data. + */ +void check_run_pipeline_roundtrip( + Config& config, + ThreadPool& tp, + shared_ptr tile, + std::optional>& offsets_tile, + FilterPipeline& pipeline, + const TileDataGenerator* test_data, + shared_ptr memory_tracker) { + // Run the pipeline forward. + CHECK(pipeline + .run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp) + .ok()); + + // Check the original unfiltered data was removed. + CHECK(tile->size() == 0); + + // Run the data in reverse. + auto unfiltered_tile = test_data->create_filtered_buffer_tile( + tile->filtered_buffer(), memory_tracker); + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + CHECK(pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); + + // Check the original data is reverted. + test_data->check_tile_data(unfiltered_tile); +} + +shared_ptr make_increasing_tile( + const uint64_t nelts, shared_ptr tracker) { + const uint64_t tile_size = nelts * sizeof(uint64_t); + const uint64_t cell_size = sizeof(uint64_t); + + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + cell_size, + tile_size, + tracker); + for (uint64_t i = 0; i < nelts; i++) { + CHECK_NOTHROW(tile->write(&i, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + return tile; +} + +shared_ptr make_offsets_tile( + std::vector& offsets, shared_ptr tracker) { + const uint64_t offsets_tile_size = + offsets.size() * constants::cell_var_offset_size; + + auto offsets_tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + constants::cell_var_offset_size, + offsets_tile_size, + tracker); + + // Set up test data + for (uint64_t i = 0; i < offsets.size(); i++) { + CHECK_NOTHROW(offsets_tile->write( + &offsets[i], + i * constants::cell_var_offset_size, + constants::cell_var_offset_size)); + } + + return offsets_tile; +} + +Tile create_tile_for_unfiltering( + uint64_t nelts, + shared_ptr tile, + shared_ptr tracker) { + return { + tile->format_version(), + tile->type(), + tile->cell_size(), + 0, + tile->cell_size() * nelts, + tile->filtered_buffer().data(), + tile->filtered_buffer().size(), + tracker}; +} + +void run_reverse( + const tiledb::sm::Config& config, + ThreadPool& tp, + Tile& unfiltered_tile, + FilterPipeline& pipeline, + bool success) { + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + CHECK( + success == pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + config) + .ok()); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/filter/test/filter_test_support.h b/tiledb/sm/filter/test/filter_test_support.h new file mode 100644 index 000000000000..e1a93081d95d --- /dev/null +++ b/tiledb/sm/filter/test/filter_test_support.h @@ -0,0 +1,157 @@ +/** + * @file filter_test_support.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains helper functions and test data classes for running tests + * on filters and filter pipelines. + * + **/ + +#ifndef TILEDB_FILTER_TEST_SUPPORT_H +#define TILEDB_FILTER_TEST_SUPPORT_H + +#include +#include "filtered_tile_checker.h" +#include "tile_data_generator.h" + +using namespace tiledb::sm; + +namespace tiledb::sm { + +// A dummy `Stats` instance. +static stats::Stats dummy_stats("test"); + +/** + * Original variable length test from the pipeline tests. + * + * For this test target size is 10 cells per chunk. Below is a list of value + * cell lengths, the cell they are added to, and the rational. + * + * target = 8 cells + * min = 4 cells + * max = 12 cells + * + * | # Cells | Previous/New # Cell in Chunk | Notes | + * |:-------:|:--------|:------------------------------------------------------| + * | 4 | 0 / 4 | chunk 0: initial chunk | + * | 10 | 4 / 14 | chunk 0: new > max, prev. <= min (next new) | + * | 6 | 0 / 6 | chunk 1: new <= target | + * | 11 | 6 / 11 | chunk 2: target < new <= max, prev. > min (next new) | + * | 7 | 0 / 7 | chunk 3: new <= target | + * | 9 | 7 / 16 | chunk 4: new > max, prev. > min (this new) | + * | 1 | 9 / 10 | chunk 4: new <= target | + * | 10 | 10 / 20 | chunk 5: new > max, prev. > min (this new) | + * | 20 | 0 / 20 | chunk 6: new > max, prev. < min (next new) | + * | 2 | 0 / 2 | chunk 7: new <= target | + * | 2 | 2 / 4 | chunk 7: new <= target | + * | 2 | 4 / 6 | chunk 7: new <= target | + * | 2 | 6 / 8 | chunk 7: new <= target | + * | 2 | 8 / 10 | chunk 7: new <= target | + * | 12 | 10 / 24 | chunk 8: new > max, prev. > min (this new) | + * + */ +class SimpleVariableTestData { + public: + SimpleVariableTestData(); + + ~SimpleVariableTestData(); + + /** Returns the number elements (cells) stored in each chunk. */ + inline const std::vector& elements_per_chunk() const { + return elements_per_chunk_; + } + + /** Returns the tile data generator for the test case. */ + inline const TileDataGenerator& tile_data_generator() const { + return tile_data_generator_; + } + + private: + uint64_t target_ncells_per_chunk_{}; + std::vector elements_per_chunk_{}; + IncrementTileDataGenerator tile_data_generator_; +}; + +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Filtered buffer data is as expected. + * 3. Pipeline runs backward without error. + * 4. Result from roundtrip matches the original data. + */ +void check_run_pipeline_full( + Config& config, + ThreadPool& tp, + shared_ptr& tile, + std::optional>& offsets_tile, + FilterPipeline& pipeline, + const TileDataGenerator* test_data, + const FilteredTileChecker& filtered_buffer_checker, + shared_ptr memory_tracker); + +/** + * Checks the following: + * + * 1. Pipeline runs forward without error. + * 2. Pipeline runs backward without error. + * 3. Result from roundtrip matches the original data. + */ +void check_run_pipeline_roundtrip( + Config& config, + ThreadPool& tp, + shared_ptr tile, + std::optional>& offsets_tile, + FilterPipeline& pipeline, + const TileDataGenerator* test_data, + shared_ptr memory_tracker); + +/** Legacy test helper. Do no use in new tests. */ +shared_ptr make_increasing_tile( + const uint64_t nelts, shared_ptr tracker); + +/** Legacy test helper. Do no use in new tests. */ +shared_ptr make_offsets_tile( + std::vector& offsets, shared_ptr tracker); + +/** Legacy test helper. Do no use in new tests. */ +Tile create_tile_for_unfiltering( + uint64_t nelts, + shared_ptr tile, + shared_ptr tracker); + +/** Legacy test helper. Do no use in new tests. */ +void run_reverse( + const tiledb::sm::Config& config, + ThreadPool& tp, + Tile& unfiltered_tile, + FilterPipeline& pipeline, + bool success = true); + +} // namespace tiledb::sm +#endif diff --git a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc new file mode 100644 index 000000000000..ceb2262c7494 --- /dev/null +++ b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc @@ -0,0 +1,497 @@ +/** + * @file unit_bit_width_reduction_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the bit-width + * reduction filter. + * + */ + +#include +#include +#include "../bit_width_reduction_filter.h" +#include "filter_test_support.h" + +using namespace tiledb::sm; + +TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 1000; + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + auto tile = make_increasing_tile(nelts, tracker); + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + // Sanity check number of windows value + uint64_t offset = 0; + offset += sizeof(uint64_t); // Number of chunks + offset += sizeof(uint32_t); // First chunk orig size + offset += sizeof(uint32_t); // First chunk filtered size + offset += sizeof(uint32_t); // First chunk metadata size + + CHECK( + tile->filtered_buffer().value_at_as(offset) == + nelts * sizeof(uint64_t)); // Original length + offset += sizeof(uint32_t); + + auto max_win_size = + pipeline.get_filter()->max_window_size(); + auto expected_num_win = + (nelts * sizeof(uint64_t)) / max_win_size + + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + expected_num_win); // Number of windows + + // Check compression worked + auto compressed_size = tile->filtered_buffer().size(); + CHECK(compressed_size < nelts * sizeof(uint64_t)); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Window sizes") { + std::vector window_sizes = { + 32, 64, 128, 256, 437, 512, 1024, 2000}; + for (auto window_size : window_sizes) { + auto tile = make_increasing_tile(nelts, tracker); + + pipeline.get_filter()->set_max_window_size( + window_size); + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + } + + SECTION("- Random values") { + std::random_device rd; + auto seed = rd(); + std::mt19937 gen(seed), gen_copy(seed); + std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); + INFO("Random element seed: " << seed); + + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + sizeof(uint64_t), + nelts * sizeof(uint64_t), + tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val = (uint64_t)rng(gen); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK((int64_t)elt == rng(gen_copy)); + } + } + + SECTION(" - Random signed values") { + std::random_device rd; + auto seed = rd(); + std::mt19937 gen(seed), gen_copy(seed); + std::uniform_int_distribution<> rng( + std::numeric_limits::lowest(), + std::numeric_limits::max()); + INFO("Random element seed: " << seed); + + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + nelts * sizeof(uint32_t), + tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint32_t val = (uint32_t)rng(gen); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + int32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); + CHECK(elt == rng(gen_copy)); + } + } + + SECTION("- Byte overflow") { + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + sizeof(uint64_t), + nelts * sizeof(uint64_t), + tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val = i % 257; + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i % 257); + } + } +} + +TEST_CASE( + "Filter: Test bit width reduction var", + "[filter][bit-width-reduction][var]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + const uint64_t nelts = 100; + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, new size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + const uint64_t offsets_tile_size = + offsets.size() * constants::cell_var_offset_size; + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(BitWidthReductionFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + + WriterTile::set_max_tile_chunk_size(80); + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + uint64_t offset = 0; + CHECK( + tile->filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile->filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + out_sizes[i] / 8); // Chunk filtered size + offset += sizeof(uint32_t); + + uint32_t md_size = tile->filtered_buffer().value_at_as(offset); + offset += sizeof(uint32_t); + + CHECK( + tile->filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Original length + offset += sizeof(uint32_t); + + // Check window value. + auto max_win_size = + pipeline.get_filter()->max_window_size(); + auto expected_num_win = out_sizes[i] / max_win_size + + uint32_t(bool(out_sizes[0] % max_win_size)); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + expected_num_win); // Number of windows + + offset += md_size - sizeof(uint32_t); + + // Check all elements are good. + uint8_t el = 0; + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK(tile->filtered_buffer().value_at_as(offset) == el++); + offset += sizeof(uint8_t); + } + } + + // Check compression worked + auto compressed_size = tile->filtered_buffer().size(); + CHECK(compressed_size < nelts * sizeof(uint64_t)); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Window sizes") { + WriterTile::set_max_tile_chunk_size(80); + std::vector window_sizes = { + 32, 64, 128, 256, 437, 512, 1024, 2000}; + for (auto window_size : window_sizes) { + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + pipeline.get_filter()->set_max_window_size( + window_size); + + CHECK(pipeline + .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + } + + SECTION("- Random values") { + WriterTile::set_max_tile_chunk_size(80); + std::random_device rd; + auto seed = rd(); + std::mt19937 gen(seed), gen_copy(seed); + std::uniform_int_distribution<> rng(0, std::numeric_limits::max()); + INFO("Random element seed: " << seed); + + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + sizeof(uint64_t), + nelts * sizeof(uint64_t), + tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val = (uint64_t)rng(gen); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK((int64_t)elt == rng(gen_copy)); + } + } + + SECTION(" - Random signed values") { + WriterTile::set_max_tile_chunk_size(80); + std::random_device rd; + auto seed = rd(); + std::mt19937 gen(seed), gen_copy(seed); + std::uniform_int_distribution<> rng( + std::numeric_limits::lowest(), + std::numeric_limits::max()); + INFO("Random element seed: " << seed); + + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + nelts * sizeof(uint32_t), + tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint32_t val = (uint32_t)rng(gen); + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + std::vector offsets32(offsets); + for (uint64_t i = 0; i < offsets32.size(); i++) { + offsets32[i] /= 2; + } + + WriterTile offsets_tile32( + constants::format_version, + Datatype::UINT64, + constants::cell_var_offset_size, + offsets_tile_size, + tracker); + + // Set up test data + for (uint64_t i = 0; i < offsets.size(); i++) { + CHECK_NOTHROW(offsets_tile32.write( + &offsets32[i], + i * constants::cell_var_offset_size, + constants::cell_var_offset_size)); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), &offsets_tile32, &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + int32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(int32_t), sizeof(int32_t))); + CHECK(elt == rng(gen_copy)); + } + } + + SECTION("- Byte overflow") { + WriterTile::set_max_tile_chunk_size(80); + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT64, + sizeof(uint64_t), + nelts * sizeof(uint64_t), + tracker); + + // Set up test data + for (uint64_t i = 0; i < nelts; i++) { + uint64_t val = i % 257; + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + auto offsets_tile = make_offsets_tile(offsets, tracker); + + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i % 257); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} diff --git a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc new file mode 100644 index 000000000000..181b892b22eb --- /dev/null +++ b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc @@ -0,0 +1,198 @@ +/** + * @file unit_bitshuffle_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the bit + * shuffle filter. + * + */ + +#include +#include +#include "../bitshuffle_filter.h" +#include "filter_test_support.h" + +using namespace tiledb::sm; + +TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 1000; + auto tile = make_increasing_tile(nelts, tracker); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Indivisible by 8") { + const uint32_t nelts2 = 1001; + const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); + + auto tile2 = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + tile_size2, + tracker); + + // Set up test data + for (uint32_t i = 0; i < nelts2; i++) { + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); + run_reverse(config, tp, unfiltered_tile2, pipeline); + for (uint64_t i = 0; i < nelts2; i++) { + uint32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK(elt == i); + } + } +} + +TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts, tracker); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, new size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets, tracker); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + WriterTile::set_max_tile_chunk_size(80); + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Indivisible by 8") { + WriterTile::set_max_tile_chunk_size(80); + const uint32_t nelts2 = 1001; + const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); + + auto tile2 = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + tile_size2, + tracker); + + // Set up test data + for (uint32_t i = 0; i < nelts2; i++) { + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + CHECK( + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); + run_reverse(config, tp, unfiltered_tile2, pipeline); + for (uint64_t i = 0; i < nelts2; i++) { + uint32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK(elt == i); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} diff --git a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc new file mode 100644 index 000000000000..c21631bfc4c8 --- /dev/null +++ b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc @@ -0,0 +1,196 @@ +/** + * @file unit_byteshuffle_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the + * byteshuffle filter. + * + */ + +#include +#include +#include "../byteshuffle_filter.h" +#include "filter_test_support.h" + +TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 1000; + auto tile = make_increasing_tile(nelts, tracker); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Uneven number of elements") { + const uint32_t nelts2 = 1001; + const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); + + auto tile2 = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + tile_size2, + tracker); + + // Set up test data + for (uint32_t i = 0; i < nelts2; i++) { + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); + run_reverse(config, tp, unfiltered_tile2, pipeline); + for (uint64_t i = 0; i < nelts2; i++) { + uint32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK(elt == i); + } + } +} + +TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts, tracker); + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, new size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + auto offsets_tile = make_offsets_tile(offsets, tracker); + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + WriterTile::set_max_tile_chunk_size(80); + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Uneven number of elements") { + WriterTile::set_max_tile_chunk_size(80); + const uint32_t nelts2 = 1001; + const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); + + auto tile2 = make_shared( + HERE(), + constants::format_version, + Datatype::UINT32, + sizeof(uint32_t), + tile_size2, + tracker); + + // Set up test data + for (uint32_t i = 0; i < nelts2; i++) { + CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); + } + + CHECK( + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts2, tile2, tracker); + run_reverse(config, tp, unfiltered_tile2, pipeline); + for (uint64_t i = 0; i < nelts2; i++) { + uint32_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, i * sizeof(uint32_t), sizeof(uint32_t))); + CHECK(elt == i); + } + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} diff --git a/tiledb/sm/filter/test/unit_checksum_pipeline.cc b/tiledb/sm/filter/test/unit_checksum_pipeline.cc new file mode 100644 index 000000000000..c7e667abf8dd --- /dev/null +++ b/tiledb/sm/filter/test/unit_checksum_pipeline.cc @@ -0,0 +1,92 @@ +/** + * @file unit_checksum_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the checksum + * filters. + * + */ + +#include +#include +#include "../checksum_md5_filter.h" +#include "filter_test_support.h" +#include "test/support/src/mem_helpers.h" + +using namespace tiledb::sm; + +TEST_CASE( + "Filter: Test skip checksum validation", + "[filter][skip-checksum-validation]") { + tiledb::sm::Config config; + REQUIRE(config.set("sm.skip_checksum_validation", "true").ok()); + + auto tracker = tiledb::test::create_test_memory_tracker(); + + const uint64_t nelts = 100; + auto tile = make_increasing_tile(nelts, tracker); + + // MD5 + FilterPipeline md5_pipeline; + ThreadPool tp(4); + ChecksumMD5Filter md5_filter(Datatype::UINT64); + md5_pipeline.add_filter(md5_filter); + CHECK(md5_pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, md5_pipeline); + + for (uint64_t n = 0; n < nelts; n++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == n); + } + + // SHA256 + auto tile2 = make_increasing_tile(nelts, tracker); + + FilterPipeline sha_256_pipeline; + ChecksumMD5Filter sha_256_filter(Datatype::UINT64); + sha_256_pipeline.add_filter(sha_256_filter); + CHECK(sha_256_pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp) + .ok()); + CHECK(tile2->size() == 0); + CHECK(tile2->filtered_buffer().size() != 0); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile2, tracker); + run_reverse(config, tp, unfiltered_tile2, sha_256_pipeline); + for (uint64_t n = 0; n < nelts; n++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile2.read(&elt, n * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == n); + } +} diff --git a/tiledb/sm/filter/test/unit_encryption_pipeline.cc b/tiledb/sm/filter/test/unit_encryption_pipeline.cc new file mode 100644 index 000000000000..ed0ed9a1ed93 --- /dev/null +++ b/tiledb/sm/filter/test/unit_encryption_pipeline.cc @@ -0,0 +1,103 @@ +/** + * @file unit_encryption_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the encryption + * filters. + * + */ + +#include +#include +#include "../encryption_aes256gcm_filter.h" +#include "filter_test_support.h" + +using namespace tiledb::sm; + +TEST_CASE("Filter: Test encryption", "[filter][encryption]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 1000; + auto tile = make_increasing_tile(nelts, tracker); + + SECTION("- AES-256-GCM") { + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(EncryptionAES256GCMFilter(Datatype::UINT64)); + + // No key set + CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + + // Create and set a key + char key[32]; + for (unsigned i = 0; i < 32; i++) + key[i] = (char)i; + auto filter = pipeline.get_filter(); + filter->set_key(key); + + // Check success + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + + // Check error decrypting with wrong key. + tile = make_increasing_tile(nelts, tracker); + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + key[0]++; + filter->set_key(key); + + auto unfiltered_tile2 = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile2, pipeline, false); + + // Fix key and check success. + auto unfiltered_tile3 = create_tile_for_unfiltering(nelts, tile, tracker); + + key[0]--; + filter->set_key(key); + run_reverse(config, tp, unfiltered_tile3, pipeline); + + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile3.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } +} diff --git a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc new file mode 100644 index 000000000000..d2239632ee1a --- /dev/null +++ b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc @@ -0,0 +1,286 @@ +/** + * @file unit_positive_delta_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the positive + * delta filter. + * + */ + +#include +#include +#include "../positive_delta_filter.h" +#include "filter_test_support.h" + +using namespace tiledb::sm; + +TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 1000; + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + auto tile = make_increasing_tile(nelts, tracker); + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto pipeline_metadata_size = sizeof(uint64_t) + 3 * sizeof(uint32_t); + + uint64_t offset = 0; + offset += sizeof(uint64_t); // Number of chunks + offset += sizeof(uint32_t); // First chunk orig size + offset += sizeof(uint32_t); // First chunk filtered size + auto filter_metadata_size = tile->filtered_buffer().value_at_as( + offset); // First chunk metadata size + offset += sizeof(uint32_t); + + auto max_win_size = + pipeline.get_filter()->max_window_size(); + auto expected_num_win = + (nelts * sizeof(uint64_t)) / max_win_size + + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + expected_num_win); // Number of windows + + // Check encoded size + auto encoded_size = tile->filtered_buffer().size(); + CHECK( + encoded_size == pipeline_metadata_size + filter_metadata_size + + nelts * sizeof(uint64_t)); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Window sizes") { + std::vector window_sizes = { + 32, 64, 128, 256, 437, 512, 1024, 2000}; + for (auto window_size : window_sizes) { + auto tile = make_increasing_tile(nelts, tracker); + pipeline.get_filter()->set_max_window_size( + window_size); + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + } + + SECTION("- Error on non-positive delta data") { + auto tile = make_increasing_tile(nelts, tracker); + for (uint64_t i = 0; i < nelts; i++) { + auto val = nelts - i; + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + } +} + +TEST_CASE( + "Filter: Test positive-delta encoding var", + "[filter][positive-delta][var]") { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + const uint64_t nelts = 100; + + // Set up test data + std::vector sizes{ + 0, + 32, // Chunk0: 4 cells. + 80, // 10 cells, still makes it into this chunk as current size < 50%. + 48, // Chunk1: 6 cells. + 88, // Chunk2: 11 cells, new size > 50% and > than 10 cells. + 56, // Chunk3: 7 cells. + 72, // Chunk4: 9 cells, new size > 50%. + 8, // Chunk4: 10 cell, full. + 80, // Chunk5: 10 cells. + 160, // Chunk6: 20 cells. + 16, // Chunk7: 2 cells. + 16, // Chunk7: 4 cells. + 16, // Chunk7: 6 cells. + 16, // Chunk7: 8 cells. + 16, // Chunk7: 10 cells. + }; // Chunk8: 12 cells. + + std::vector out_sizes{112, 48, 88, 56, 80, 80, 160, 80, 96}; + + std::vector offsets(sizes.size()); + uint64_t offset = 0; + for (uint64_t i = 0; i < offsets.size() - 1; i++) { + offsets[i] = offset; + offset += sizes[i + 1]; + } + offsets[offsets.size() - 1] = offset; + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(PositiveDeltaFilter(Datatype::UINT64)); + + SECTION("- Single stage") { + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + + WriterTile::set_max_tile_chunk_size(80); + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + uint64_t offset = 0; + CHECK( + tile->filtered_buffer().value_at_as(offset) == + 9); // Number of chunks + offset += sizeof(uint64_t); + + uint64_t total_md_size = 0; + for (uint64_t i = 0; i < 9; i++) { + CHECK( + tile->filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk orig size + offset += sizeof(uint32_t); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + out_sizes[i]); // Chunk filtered size + offset += sizeof(uint32_t); + + uint32_t md_size = tile->filtered_buffer().value_at_as(offset); + offset += sizeof(uint32_t); + total_md_size += md_size; + + auto max_win_size = + pipeline.get_filter()->max_window_size(); + auto expected_num_win = + (nelts * sizeof(uint64_t)) / max_win_size + + uint32_t(bool((nelts * sizeof(uint64_t)) % max_win_size)); + CHECK( + tile->filtered_buffer().value_at_as(offset) == + expected_num_win); // Number of windows + + offset += md_size; + + // Check all elements are good. + for (uint64_t j = 0; j < out_sizes[i] / sizeof(uint64_t); j++) { + CHECK( + tile->filtered_buffer().value_at_as(offset) == + (j == 0 ? 0 : 1)); + offset += sizeof(uint64_t); + } + } + + // Check encoded size + auto pipeline_metadata_size = sizeof(uint64_t) + 9 * 3 * sizeof(uint32_t); + auto encoded_size = tile->filtered_buffer().size(); + CHECK( + encoded_size == + pipeline_metadata_size + total_md_size + nelts * sizeof(uint64_t)); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + + SECTION("- Window sizes") { + WriterTile::set_max_tile_chunk_size(80); + std::vector window_sizes = { + 32, 64, 128, 256, 437, 512, 1024, 2000}; + for (auto window_size : window_sizes) { + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + + pipeline.get_filter()->set_max_window_size( + window_size); + + CHECK(pipeline + .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + uint64_t elt = 0; + CHECK_NOTHROW( + unfiltered_tile.read(&elt, i * sizeof(uint64_t), sizeof(uint64_t))); + CHECK(elt == i); + } + } + } + + SECTION("- Error on non-positive delta data") { + auto tile = make_increasing_tile(nelts, tracker); + auto offsets_tile = make_offsets_tile(offsets, tracker); + + WriterTile::set_max_tile_chunk_size(80); + for (uint64_t i = 0; i < nelts; i++) { + auto val = nelts - i; + CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); + } + + CHECK( + !pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) + .ok()); + } + + WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); +} diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 2feedb2d0ee5..2c225aff553d 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -79,6 +79,7 @@ #include "add_1_including_metadata_filter.h" #include "add_1_out_of_place_filter.h" #include "add_n_in_place_filter.h" +#include "filter_test_support.h" #include "filtered_tile_checker.h" #include "pseudo_checksum_filter.h" #include "tile_data_generator.h" @@ -92,171 +93,6 @@ using namespace tiledb::sm; -// A dummy `Stats` instance. -static tiledb::sm::stats::Stats dummy_stats("test"); - -class SimpleTestData {}; - -/** - * Original variable length test from the pipeline tests. - * - * For this test target size is 10 cells per chunk. Below is a list of value - * cell lengths, the cell they are added to, and the rational. - * - * target = 8 cells - * min = 4 cells - * max = 12 cells - * - * | # Cells | Previous/New # Cell in Chunk | Notes | - * |:-------:|:--------|:------------------------------------------------------| - * | 4 | 0 / 4 | chunk 0: initial chunk | - * | 10 | 4 / 14 | chunk 0: new > max, prev. <= min (next new) | - * | 6 | 0 / 6 | chunk 1: new <= target | - * | 11 | 6 / 11 | chunk 2: target < new <= max, prev. > min (next new) | - * | 7 | 0 / 7 | chunk 3: new <= target | - * | 9 | 7 / 16 | chunk 4: new > max, prev. > min (this new) | - * | 1 | 9 / 10 | chunk 4: new <= target | - * | 10 | 10 / 20 | chunk 5: new > max, prev. > min (this new) | - * | 20 | 0 / 20 | chunk 6: new > max, prev. < min (next new) | - * | 2 | 0 / 2 | chunk 7: new <= target | - * | 2 | 2 / 4 | chunk 7: new <= target | - * | 2 | 4 / 6 | chunk 7: new <= target | - * | 2 | 6 / 8 | chunk 7: new <= target | - * | 2 | 8 / 10 | chunk 7: new <= target | - * | 12 | 10 / 24 | chunk 8: new > max, prev. > min (this new) | - * - */ -class SimpleVariableTestData { - public: - SimpleVariableTestData() - : target_ncells_per_chunk_{10} - , elements_per_chunk_{14, 6, 11, 7, 10, 10, 20, 10, 12} - , tile_data_generator_{ - {4, 10, 6, 11, 7, 9, 1, 10, 20, 2, 2, 2, 2, 2, 12}} { - WriterTile::set_max_tile_chunk_size( - target_ncells_per_chunk_ * sizeof(uint64_t)); - } - ~SimpleVariableTestData() { - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); - } - - /** Returns the number elements (cells) stored in each chunk. */ - const std::vector& elements_per_chunk() const { - return elements_per_chunk_; - } - - const TileDataGenerator& tile_data_generator() const { - return tile_data_generator_; - } - - private: - uint64_t target_ncells_per_chunk_{}; - std::vector elements_per_chunk_{}; - IncrementTileDataGenerator tile_data_generator_; -}; - -/** - * Checks the following: - * - * 1. Pipeline runs forward without error. - * 2. Filtered buffer data is as expected. - * 3. Pipeline runs backward without error. - * 4. Result from roundtrip matches the original data. - */ -void check_run_pipeline_full( - Config& config, - ThreadPool& tp, - shared_ptr& tile, - std::optional>& offsets_tile, - FilterPipeline& pipeline, - const TileDataGenerator* test_data, - const FilteredTileChecker& filtered_buffer_checker, - shared_ptr memory_tracker) { - // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); - - // Check the original unfiltered data was removed. - CHECK(tile->size() == 0); - - // Check the filtered buffer has the expected data. - auto filtered_buffer = tile->filtered_buffer(); - filtered_buffer_checker.check(filtered_buffer); - - // Run the data in reverse. - auto unfiltered_tile = - test_data->create_filtered_buffer_tile(filtered_buffer, memory_tracker); - ChunkData chunk_data; - unfiltered_tile.load_chunk_data(chunk_data); - CHECK(pipeline - .run_reverse( - &dummy_stats, - &unfiltered_tile, - nullptr, - chunk_data, - 0, - chunk_data.filtered_chunks_.size(), - tp.concurrency_level(), - config) - .ok()); - - // Check the original data is reverted. - test_data->check_tile_data(unfiltered_tile); -} - -/** - * Checks the following: - * - * 1. Pipeline runs forward without error. - * 2. Pipeline runs backward without error. - * 3. Result from roundtrip matches the original data. - */ -void check_run_pipeline_roundtrip( - Config& config, - ThreadPool& tp, - shared_ptr tile, - std::optional>& offsets_tile, - FilterPipeline& pipeline, - const TileDataGenerator* test_data, - shared_ptr memory_tracker) { - // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); - - // Check the original unfiltered data was removed. - CHECK(tile->size() == 0); - - // Run the data in reverse. - auto unfiltered_tile = test_data->create_filtered_buffer_tile( - tile->filtered_buffer(), memory_tracker); - ChunkData chunk_data; - unfiltered_tile.load_chunk_data(chunk_data); - CHECK(pipeline - .run_reverse( - &dummy_stats, - &unfiltered_tile, - nullptr, - chunk_data, - 0, - chunk_data.filtered_chunks_.size(), - tp.concurrency_level(), - config) - .ok()); - - // Check the original data is reverted. - test_data->check_tile_data(unfiltered_tile); -} - TEST_CASE("Filter: Test empty pipeline", "[filter][empty-pipeline]") { // Create TileDB needed for running pipeline. Config config; diff --git a/tiledb/sm/filter/test/unit_xor_pipeline.cc b/tiledb/sm/filter/test/unit_xor_pipeline.cc new file mode 100644 index 000000000000..51ea0c0e6df2 --- /dev/null +++ b/tiledb/sm/filter/test/unit_xor_pipeline.cc @@ -0,0 +1,327 @@ +/** + * @file unit_xor_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the + * xor filter combined with other filters. + * + */ + +#include +#include + +#include "../bit_width_reduction_filter.h" +#include "../bitshuffle_filter.h" +#include "../byteshuffle_filter.h" +#include "../compression_filter.h" +#include "../float_scaling_filter.h" +#include "../positive_delta_filter.h" +#include "../xor_filter.h" +#include "filter_test_support.h" +#include "tiledb/sm/enums/filter_option.h" + +using namespace tiledb::sm; + +template +void testing_float_scaling_filter() { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 100; + const uint64_t tile_size = nelts * sizeof(FloatingType); + const uint64_t cell_size = sizeof(FloatingType); + + Datatype t = Datatype::FLOAT32; + switch (sizeof(FloatingType)) { + case 4: { + t = Datatype::FLOAT32; + } break; + case 8: { + t = Datatype::FLOAT64; + } break; + default: { + INFO( + "testing_float_scaling_filter: passed floating type with size of not " + "4 bytes or 8 bytes."); + CHECK(false); + } + } + + auto tile = make_shared( + HERE(), constants::format_version, t, cell_size, tile_size, tracker); + + std::vector float_result_vec; + double scale = 2.53; + double foffset = 0.31589; + uint64_t byte_width = sizeof(IntType); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dis(0.0, 213.0); + + for (uint64_t i = 0; i < nelts; i++) { + FloatingType f = dis(gen); + CHECK_NOTHROW( + tile->write(&f, i * sizeof(FloatingType), sizeof(FloatingType))); + + IntType val = static_cast(round( + (f - static_cast(foffset)) / + static_cast(scale))); + + FloatingType val_float = static_cast( + scale * static_cast(val) + foffset); + float_result_vec.push_back(val_float); + } + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(FloatScalingFilter(t)); + CHECK(pipeline.get_filter() + ->set_option(FilterOption::SCALE_FLOAT_BYTEWIDTH, &byte_width) + .ok()); + CHECK(pipeline.get_filter() + ->set_option(FilterOption::SCALE_FLOAT_FACTOR, &scale) + .ok()); + CHECK(pipeline.get_filter() + ->set_option(FilterOption::SCALE_FLOAT_OFFSET, &foffset) + .ok()); + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + + // Check new size and number of chunks + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + FloatingType elt = 0.0f; + CHECK_NOTHROW(unfiltered_tile.read( + &elt, i * sizeof(FloatingType), sizeof(FloatingType))); + CHECK(elt == float_result_vec[i]); + } +} + +TEMPLATE_TEST_CASE( + "Filter: Test float scaling", + "[filter][float-scaling]", + int8_t, + int16_t, + int32_t, + int64_t) { + testing_float_scaling_filter(); + testing_float_scaling_filter(); +} + +/* + Defining distribution types to pass into the testing_xor_filter function. + */ +typedef typename std::uniform_int_distribution IntDistribution; +typedef typename std::uniform_real_distribution FloatDistribution; + +template +void testing_xor_filter(Datatype t) { + tiledb::sm::Config config; + + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set up test data + const uint64_t nelts = 100; + const uint64_t tile_size = nelts * sizeof(T); + const uint64_t cell_size = sizeof(T); + + auto tile = make_shared( + HERE(), constants::format_version, t, cell_size, tile_size, tracker); + + // Setting up the random number generator for the XOR filter testing. + std::mt19937_64 gen(0x57A672DE); + Distribution dis( + std::numeric_limits::min(), std::numeric_limits::max()); + + std::vector results; + + for (uint64_t i = 0; i < nelts; i++) { + T val = static_cast(dis(gen)); + CHECK_NOTHROW(tile->write(&val, i * sizeof(T), sizeof(T))); + results.push_back(val); + } + + FilterPipeline pipeline; + ThreadPool tp(4); + pipeline.add_filter(XORFilter(t)); + + CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + + // Check new size and number of chunks + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = create_tile_for_unfiltering(nelts, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + for (uint64_t i = 0; i < nelts; i++) { + T elt = 0; + CHECK_NOTHROW(unfiltered_tile.read(&elt, i * sizeof(T), sizeof(T))); + CHECK(elt == results[i]); + } +} + +TEST_CASE("Filter: Test XOR", "[filter][xor]") { + testing_xor_filter(Datatype::INT8); + testing_xor_filter(Datatype::UINT8); + testing_xor_filter(Datatype::INT16); + testing_xor_filter(Datatype::UINT16); + testing_xor_filter(Datatype::INT32); + testing_xor_filter(Datatype::UINT32); + testing_xor_filter(Datatype::INT64); + testing_xor_filter>( + Datatype::UINT64); + testing_xor_filter(Datatype::FLOAT32); + testing_xor_filter(Datatype::FLOAT64); + testing_xor_filter(Datatype::CHAR); + testing_xor_filter(Datatype::DATETIME_YEAR); + testing_xor_filter(Datatype::DATETIME_MONTH); + testing_xor_filter(Datatype::DATETIME_WEEK); + testing_xor_filter(Datatype::DATETIME_DAY); + testing_xor_filter(Datatype::DATETIME_HR); + testing_xor_filter(Datatype::DATETIME_MIN); + testing_xor_filter(Datatype::DATETIME_SEC); + testing_xor_filter(Datatype::DATETIME_MS); + testing_xor_filter(Datatype::DATETIME_US); + testing_xor_filter(Datatype::DATETIME_NS); + testing_xor_filter(Datatype::DATETIME_PS); + testing_xor_filter(Datatype::DATETIME_FS); + testing_xor_filter(Datatype::DATETIME_AS); +} + +TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { + FilterPipeline pipeline; + auto tracker = tiledb::test::create_test_memory_tracker(); + + SECTION("- DoubleDelta filter reinterprets float->int32") { + pipeline.add_filter(CompressionFilter( + tiledb::sm::Compressor::DOUBLE_DELTA, + 0, + Datatype::FLOAT32, + Datatype::INT32)); + pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); + } + + SECTION("- Delta filter reinterprets float->int32") { + pipeline.add_filter(CompressionFilter( + tiledb::sm::Compressor::DELTA, 0, Datatype::FLOAT32, Datatype::INT32)); + pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); + } + + SECTION("- FloatScale filter converts float->int32") { + pipeline.add_filter( + FloatScalingFilter(sizeof(int32_t), 1.0f, 0.0f, Datatype::FLOAT32)); + pipeline.add_filter(PositiveDeltaFilter(Datatype::INT32)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::DELTA, 0, Datatype::INT32)); + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::BZIP2, 2, Datatype::INT32)); + pipeline.add_filter(BitshuffleFilter(Datatype::INT32)); + pipeline.add_filter(ByteshuffleFilter(Datatype::INT32)); + pipeline.add_filter(BitWidthReductionFilter(Datatype::INT32)); + } + + size_t byte_width = 0; + SECTION("- XOR filter expected output types") { + byte_width = GENERATE( + sizeof(int8_t), sizeof(int16_t), sizeof(int32_t), sizeof(int64_t)); + pipeline.add_filter( + FloatScalingFilter(byte_width, 1.0f, 0.0f, Datatype::FLOAT32)); + auto byte_width_t = + pipeline.get_filter()->output_datatype( + Datatype::FLOAT32); + pipeline.add_filter(XORFilter(byte_width_t)); + } + + SECTION("- XOR filter expected output types large pipeline") { + byte_width = GENERATE( + sizeof(int8_t), sizeof(int16_t), sizeof(int32_t), sizeof(int64_t)); + pipeline.add_filter( + FloatScalingFilter(byte_width, 1.0f, 0.0f, Datatype::FLOAT32)); + auto byte_width_t = + pipeline.get_filter()->output_datatype( + Datatype::FLOAT32); + pipeline.add_filter(PositiveDeltaFilter(byte_width_t)); + pipeline.add_filter(BitshuffleFilter(byte_width_t)); + pipeline.add_filter(ByteshuffleFilter(byte_width_t)); + pipeline.add_filter(XORFilter(byte_width_t)); + } + + // Initial type of tile is float. + std::vector data = { + 1.0f, 2.1f, 3.2f, 4.3f, 5.4f, 6.5f, 7.6f, 8.7f, 9.8f, 10.9f}; + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::FLOAT32, + sizeof(float), + sizeof(float) * data.size(), + tracker); + for (size_t i = 0; i < data.size(); i++) { + CHECK_NOTHROW(tile->write(&data[i], i * sizeof(float), sizeof(float))); + } + + ThreadPool tp(4); + REQUIRE(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + auto unfiltered_tile = + create_tile_for_unfiltering(data.size(), tile, tracker); + ChunkData chunk_data; + unfiltered_tile.load_chunk_data(chunk_data); + REQUIRE(pipeline + .run_reverse( + &dummy_stats, + &unfiltered_tile, + nullptr, + chunk_data, + 0, + chunk_data.filtered_chunks_.size(), + tp.concurrency_level(), + tiledb::sm::Config()) + .ok()); + std::vector results{ + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f}; + for (size_t i = 0; i < data.size(); i++) { + float val = 0; + CHECK_NOTHROW(unfiltered_tile.read(&val, i * sizeof(float), sizeof(float))); + if (pipeline.has_filter(tiledb::sm::FilterType::FILTER_SCALE_FLOAT)) { + // Loss of precision from rounding in FloatScale filter. + CHECK(val == results[i]); + } else { + CHECK(val == data[i]); + } + } +} From 0ae48188ebc394adaeb02dcbe55eef39d5f0e369 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:43:22 +0100 Subject: [PATCH 244/456] Revert "Update libmagic to version 5.45 (#4673)". (#4808) This reverts commit 7753f3360b42a837f2e18275a5b23691aa86157d. --- TYPE: NO_HISTORY DESC: Revert "Update libmagic to version 5.45 (#4673)" --- .readthedocs.yml | 11 - doc/dev/BUILD.md | 1 - ports/README.md | 3 +- ...-Use-libtre.patch => 0001-Use-pcre2.patch} | 2 +- ...-zlib-lib-name-to-match-CMake-output.patch | 39 -- .../0003-Fix-WIN32-macro-checks.patch | 10 +- ...ude-dirent.h-for-S_ISREG-and-S_ISDIR.patch | 9 +- .../0006-Remove-Wrap-POSIX-headers.patch | 2 +- ports/libmagic/0009-No-fcntl-in-magic.c.patch | 25 ++ ...y-check-for-the-presence-of-bitmasks.patch | 2 +- ...ve-pipe-related-functions-in-funcs.c.patch | 12 +- ...Convert-MSYS2-paths-to-Windows-paths.patch | 24 -- ...k-for-backslash-in-argv-0-on-Windows.patch | 42 -- .../0014-Define-POSIX-macros-if-missing.patch | 38 ++ .../0015-MSYS2-Remove-ioctl-call.patch | 8 +- .../0016-Fix-file_famagic-function.patch | 40 -- ports/libmagic/CMakeLists.txt | 170 ++++++++ ports/libmagic/config.h | 401 ++++++++++++++++++ ports/libmagic/magic.def | 20 + ports/libmagic/portfile.cmake | 87 ++-- .../unofficial-libmagic-config.cmake.in | 71 +--- ports/libmagic/vcpkg.json | 50 +-- ports/pcre2/fix-cmake.patch | 334 +++++++++++++++ ports/pcre2/no-static-suffix.patch | 33 ++ ports/pcre2/pcre2-10.35_fix-uwp.patch | 10 + ports/pcre2/portfile.cmake | 73 ++++ ports/pcre2/usage | 6 + ports/pcre2/vcpkg.json | 39 ++ tiledb/CMakeLists.txt | 5 +- .../util/tdb_gzip_embedded_data.cc | 19 +- tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 | Bin 410953 -> 322448 bytes vcpkg.json | 9 +- 32 files changed, 1239 insertions(+), 356 deletions(-) rename ports/libmagic/{0001-Use-libtre.patch => 0001-Use-pcre2.patch} (98%) delete mode 100644 ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch create mode 100644 ports/libmagic/0009-No-fcntl-in-magic.c.patch delete mode 100644 ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch delete mode 100644 ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch create mode 100644 ports/libmagic/0014-Define-POSIX-macros-if-missing.patch delete mode 100644 ports/libmagic/0016-Fix-file_famagic-function.patch create mode 100644 ports/libmagic/CMakeLists.txt create mode 100644 ports/libmagic/config.h create mode 100644 ports/libmagic/magic.def create mode 100644 ports/pcre2/fix-cmake.patch create mode 100644 ports/pcre2/no-static-suffix.patch create mode 100644 ports/pcre2/pcre2-10.35_fix-uwp.patch create mode 100644 ports/pcre2/portfile.cmake create mode 100644 ports/pcre2/usage create mode 100644 ports/pcre2/vcpkg.json diff --git a/.readthedocs.yml b/.readthedocs.yml index 0c2c9b0fa841..459b768ef7f1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,17 +12,6 @@ build: os: "ubuntu-22.04" tools: python: "3.8" - apt_packages: - - autoconf - - autoconf-archive - - automake - - curl - - git - - libtool - - make - - pkg-config - - unzip - - zip python: install: diff --git a/doc/dev/BUILD.md b/doc/dev/BUILD.md index d0c8140d1454..1717cb0fa908 100644 --- a/doc/dev/BUILD.md +++ b/doc/dev/BUILD.md @@ -12,7 +12,6 @@ title: Building TileDB from source * Apple Clang 14 * Git (required by vcpkg) * curl (required by vcpkg on non-Windows) -* autoconf (required by building libmagic on non-Windows) ## Downloading the source code diff --git a/ports/README.md b/ports/README.md index 09981c7650fd..7faf08053d33 100644 --- a/ports/README.md +++ b/ports/README.md @@ -30,6 +30,7 @@ After copying the port, add an entry to the table below. You should also contrib |Port|Reason| |----|------| +|`libmagic`|Updating to the upstream port deferred due to failures.| |`openssl`|Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024.| +|`pcre2`|To be removed alongside libmagic.| |`azure-storage-common-cpp`|Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221).| -|`libmagic`|Patching to add features explicitly enabling compression support.| diff --git a/ports/libmagic/0001-Use-libtre.patch b/ports/libmagic/0001-Use-pcre2.patch similarity index 98% rename from ports/libmagic/0001-Use-libtre.patch rename to ports/libmagic/0001-Use-pcre2.patch index a1080d5a070b..2bcf73cd2bc6 100644 --- a/ports/libmagic/0001-Use-libtre.patch +++ b/ports/libmagic/0001-Use-pcre2.patch @@ -44,7 +44,7 @@ index c548e97..299ac0c 100644 #include #include /* For open and flags */ -#include -+#include ++#include #include #include #ifndef WIN32 diff --git a/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch b/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch deleted file mode 100644 index 5e070cc07fe0..000000000000 --- a/ports/libmagic/0002-Change-zlib-lib-name-to-match-CMake-output.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 2fa43ece9ec7564e1fbb9867bb5852b834643aa4 Mon Sep 17 00:00:00 2001 -From: Long Nguyen -Date: Sat, 8 May 2021 19:36:11 +0700 -Subject: [PATCH 02/14] Change zlib lib name to match CMake output - ---- - configure.ac | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/configure.ac b/configure.ac -index b05c334..dd4063c 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -173,7 +173,7 @@ AC_REPLACE_FUNCS(getopt_long asprintf vasprintf strlcpy strlcat getline ctime_r - - dnl Checks for libraries - if test "$enable_zlib" != "no"; then -- AC_CHECK_LIB(z, gzopen) -+ AC_SEARCH_LIBS(gzopen, [z zlib zlibd], have_zlib = "yes", have_zlib = "no") - fi - if test "$enable_bzlib" != "no"; then - AC_CHECK_LIB(bz2, BZ2_bzCompressInit) -@@ -193,11 +193,11 @@ AM_CONDITIONAL(IS_CROSS_COMPILE, test "$cross_compiling" = yes) - - dnl Final sanity checks - if test "$enable_zlib" = "yes"; then -- if test "$ac_cv_header_zlib_h$ac_cv_lib_z_gzopen" != "yesyes"; then -+ if test "$ac_cv_header_zlib_h$have_zlib" != "yesyes"; then - AC_MSG_ERROR([zlib support requested but not found]) - fi - fi --if test "$ac_cv_header_zlib_h$ac_cv_lib_z_gzopen" = "yesyes"; then -+if test "$ac_cv_header_zlib_h$have_zlib" = "yesyes"; then - AC_DEFINE([ZLIBSUPPORT], 1, [Enable zlib compression support]) - fi - if test "$enable_bzlib" = "yes"; then --- -2.29.2.windows.2 - diff --git a/ports/libmagic/0003-Fix-WIN32-macro-checks.patch b/ports/libmagic/0003-Fix-WIN32-macro-checks.patch index e90beb4d4097..44ab2ddea2af 100644 --- a/ports/libmagic/0003-Fix-WIN32-macro-checks.patch +++ b/ports/libmagic/0003-Fix-WIN32-macro-checks.patch @@ -12,7 +12,7 @@ index 299ac0c..2c365a6 100644 --- a/src/file.h +++ b/src/file.h @@ -82,7 +82,7 @@ - #include + #include #include #include -#ifndef WIN32 @@ -31,13 +31,13 @@ index 299ac0c..2c365a6 100644 #define PATHSEP ':' @@ -103,7 +103,7 @@ - #define file_private static + #define private static -#if HAVE_VISIBILITY && !defined(WIN32) +#if HAVE_VISIBILITY && !defined(_WIN32) - #define file_public __attribute__ ((__visibility__("default"))) - #ifndef file_protected - #define file_protected __attribute__ ((__visibility__("hidden"))) + #define public __attribute__ ((__visibility__("default"))) + #ifndef protected + #define protected __attribute__ ((__visibility__("hidden"))) -- 2.29.2.windows.2 diff --git a/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch b/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch index afcb658d8890..76cbc1222c8b 100644 --- a/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch +++ b/ports/libmagic/0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch @@ -11,15 +11,14 @@ diff --git a/src/file.h b/src/file.h index 0332506..4aa9f60 100644 --- a/src/file.h +++ b/src/file.h -@@ -88,7 +88,8 @@ +@@ -88,6 +88,7 @@ /* Do this here and now, because struct stat gets re-defined on solaris */ #include #include +#include - #include - #if defined(HAVE_XLOCALE_H) - #include - #endif + + #define ENABLE_CONDITIONALS + -- 2.29.2.windows.2 diff --git a/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch b/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch index 169561d24372..56efac248157 100644 --- a/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch +++ b/ports/libmagic/0006-Remove-Wrap-POSIX-headers.patch @@ -161,7 +161,7 @@ index 3ab52d1..fc48d84 100644 --- a/src/pread.c +++ b/src/pread.c @@ -3,7 +3,9 @@ - FILE_RCSID("@(#)$File: pread.c,v 1.5 2022/09/24 20:30:13 christos Exp $") + FILE_RCSID("@(#)$File: pread.c,v 1.2 2013/04/02 16:23:07 christos Exp $") #endif /* lint */ #include +#ifdef HAVE_UNISTD_H diff --git a/ports/libmagic/0009-No-fcntl-in-magic.c.patch b/ports/libmagic/0009-No-fcntl-in-magic.c.patch new file mode 100644 index 000000000000..47bf16165ef8 --- /dev/null +++ b/ports/libmagic/0009-No-fcntl-in-magic.c.patch @@ -0,0 +1,25 @@ +From 6d10bef865b69764f6e0ddd2b0f6a84e484cdb37 Mon Sep 17 00:00:00 2001 +From: Long Nguyen +Date: Sun, 9 May 2021 13:25:14 +0700 +Subject: [PATCH 09/14] No fcntl in magic.c + +--- + src/magic.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/magic.c b/src/magic.c +index e9aeafa..382bd96 100644 +--- a/src/magic.c ++++ b/src/magic.c +@@ -462,7 +462,7 @@ file_or_fd(struct magic_set *ms, const char *inname, int fd) + rv = 0; + goto done; + } +-#if O_CLOEXEC == 0 ++#if O_CLOEXEC == 0 && !defined(_WIN32) + (void)fcntl(fd, F_SETFD, FD_CLOEXEC); + #endif + } +-- +2.29.2.windows.2 + diff --git a/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch b/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch index e241f5e49c09..f0d8738e465e 100644 --- a/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch +++ b/ports/libmagic/0010-Properly-check-for-the-presence-of-bitmasks.patch @@ -17,7 +17,7 @@ index 5204f20..7244841 100644 #undef HAVE_MAJOR -#ifdef S_IFLNK +#if S_IFLNK != 0 - file_private int + private int bad_link(struct magic_set *ms, int err, char *buf) { @@ -108,7 +108,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) diff --git a/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch b/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch index 13c24faaca0b..ae2720ca3617 100644 --- a/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch +++ b/ports/libmagic/0011-Remove-pipe-related-functions-in-funcs.c.patch @@ -11,19 +11,21 @@ diff --git a/src/funcs.c b/src/funcs.c index b926625..b585486 100644 --- a/src/funcs.c +++ b/src/funcs.c -@@ -888,5 +888,6 @@ +@@ -809,6 +809,7 @@ file_print_guid(char *str, size_t len, const uint64_t *guid) + g->data4[6], g->data4[7]); } +#ifndef _WIN32 - file_protected int + protected int file_pipe_closexec(int *fds) { -@@ -914,5 +915,6 @@ - #endif +@@ -827,6 +828,7 @@ protected int + file_clear_closexec(int fd) { + return fcntl(fd, F_SETFD, 0); } +#endif - file_protected char * + protected char * file_strtrim(char *str) -- 2.29.2.windows.2 diff --git a/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch b/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch deleted file mode 100644 index 728fe77539f2..000000000000 --- a/ports/libmagic/0012-Convert-MSYS2-paths-to-Windows-paths.patch +++ /dev/null @@ -1,24 +0,0 @@ -From f0139ced57c1579450f3d09b6e3ae0159aae031b Mon Sep 17 00:00:00 2001 -From: Long Nguyen -Date: Mon, 10 May 2021 08:43:28 +0700 -Subject: [PATCH 12/14] Convert MSYS2 paths to Windows paths - ---- - src/Makefile.am | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/src/Makefile.am b/src/Makefile.am -index 34781b9..26f853f 100644 ---- a/src/Makefile.am -+++ b/src/Makefile.am -@@ -1,4 +1,7 @@ - MAGIC = $(pkgdatadir)/magic -+if MINGW -+MAGIC != cygpath -m -a -l '$(MAGIC)' -+endif - lib_LTLIBRARIES = libmagic.la - nodist_include_HEADERS = magic.h - --- -2.29.2.windows.2 - diff --git a/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch b/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch deleted file mode 100644 index 77777fc61e8e..000000000000 --- a/ports/libmagic/0013-Check-for-backslash-in-argv-0-on-Windows.patch +++ /dev/null @@ -1,42 +0,0 @@ -From 3f10c7d31627b64b068b84ba72e706991f672560 Mon Sep 17 00:00:00 2001 -From: Long Nguyen -Date: Fri, 14 May 2021 08:14:05 +0700 -Subject: [PATCH 13/14] Check for backslash in argv[0] on Windows - ---- - magic/Makefile.am | 2 +- - src/file.c | 5 +++++ - 2 files changed, 6 insertions(+), 1 deletion(-) - -diff --git a/magic/Makefile.am b/magic/Makefile.am -index 0eb5865..170bbab 100644 ---- a/magic/Makefile.am -+++ b/magic/Makefile.am -@@ -353,7 +353,7 @@ ${MAGIC}: $(EXTRA_DIST) $(FILE_COMPILE_DEP) - @(if expr "${FILE_COMPILE}" : '.*/.*' > /dev/null; then \ - echo "Using ${FILE_COMPILE} to generate ${MAGIC}" > /dev/null; \ - else \ -- v=$$(${FILE_COMPILE} --version | sed -e s/file-// -e q); \ -+ v=$$(${FILE_COMPILE} --version | sed -e s/file${EXEEXT}-// -e q); \ - if [ "$$v" != "${PACKAGE_VERSION}" ]; then \ - echo "Cannot use the installed version of file ($$v) to"; \ - echo "cross-compile file ${PACKAGE_VERSION}"; \ -diff --git a/src/file.c b/src/file.c -index 2889f8a..12a604b 100644 ---- a/src/file.c -+++ b/src/file.c -@@ -200,6 +200,11 @@ main(int argc, char *argv[]) - _wildcard(&argc, &argv); - #endif - -+#ifdef _WIN32 -+ if ((progname = strrchr(argv[0], '\\')) != NULL) -+ progname++; -+ else -+#endif - if ((progname = strrchr(argv[0], '/')) != NULL) - progname++; - else --- -2.29.2.windows.2 - diff --git a/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch b/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch new file mode 100644 index 000000000000..09ac7a11926e --- /dev/null +++ b/ports/libmagic/0014-Define-POSIX-macros-if-missing.patch @@ -0,0 +1,38 @@ +From fa0e11f36bb0e322250e1e488ced9f2bf166874f Mon Sep 17 00:00:00 2001 +From: Long Nguyen +Date: Fri, 14 May 2021 18:11:39 +0700 +Subject: [PATCH 14/14] Define POSIX macros if missing + +--- + src/file.h | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +diff --git a/src/file.h b/src/file.h +index ccfe0da..98cd37b 100644 +--- a/src/file.h ++++ b/src/file.h +@@ -100,6 +100,21 @@ + #include + #include + ++#if !defined(S_IFBLK) ++#define S_IFBLK 0 ++#define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) ++#endif ++ ++#if !defined(S_IFLNK) ++#define S_IFLNK 0 ++#define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) ++#endif ++ ++#if !defined(S_IFSOCK) ++#define S_IFSOCK 0 ++#define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) ++#endif ++ + #define ENABLE_CONDITIONALS + + #ifndef MAGIC +-- +2.29.2.windows.2 + diff --git a/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch b/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch index 13fe3cf0c550..da8ef79df29a 100644 --- a/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch +++ b/ports/libmagic/0015-MSYS2-Remove-ioctl-call.patch @@ -7,20 +7,20 @@ diff --git a/src/compress.c b/src/compress.c index 33ce2bc..f172eda 100644 --- a/src/compress.c +++ b/src/compress.c -@@ -407,7 +407,7 @@ +@@ -378,7 +378,7 @@ sread(int fd, void *buf, size_t n, int canbepipe __attribute__((__unused__))) { ssize_t rv; --#if defined(FIONREAD) && !defined(__MINGW32__) +-#ifdef FIONREAD +#if defined(FIONREAD) && !defined(__MINGW32__) && !defined(WIN32) int t = 0; #endif size_t rn = n; -@@ -418,7 +418,7 @@ +@@ -386,7 +386,7 @@ if (fd == STDIN_FILENO) goto nocheck; --#if defined(FIONREAD) && !defined(__MINGW32__) +-#ifdef FIONREAD +#if defined(FIONREAD) && !defined(__MINGW32__) && !defined(WIN32) if (canbepipe && (ioctl(fd, FIONREAD, &t) == -1 || t == 0)) { #ifdef FD_ZERO diff --git a/ports/libmagic/0016-Fix-file_famagic-function.patch b/ports/libmagic/0016-Fix-file_famagic-function.patch deleted file mode 100644 index 5eaba925a589..000000000000 --- a/ports/libmagic/0016-Fix-file_famagic-function.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/src/fsmagic.c b/src/fsmagic.c -index 7244841..2c553c1 100644 ---- a/src/fsmagic.c -+++ b/src/fsmagic.c -@@ -66,7 +66,7 @@ # define major(dev) (((dev) >> 8) & 0xff) - # define minor(dev) ((dev) & 0xff) - #endif - #undef HAVE_MAJOR --#if S_IFLNK != 0 -+#if S_IFLNK != 0 && ! defined(_WIN32) - file_private int - bad_link(struct magic_set *ms, int err, char *buf) - { -@@ -108,7 +108,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) - int ret, did = 0; - int mime = ms->flags & MAGIC_MIME; - int silent = ms->flags & (MAGIC_APPLE|MAGIC_EXTENSION); --#if S_IFLNK != 0 -+#if S_IFLNK != 0 && ! defined(_WIN32) - char buf[BUFSIZ+4]; - ssize_t nch; - struct stat tstatbuf; -@@ -122,7 +122,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) - * Fstat is cheaper but fails for files you don't have read perms on. - * On 4.2BSD and similar systems, use lstat() to identify symlinks. - */ --#if S_IFLNK != 0 -+#if S_IFLNK != 0 && ! defined(_WIN32) - if ((ms->flags & MAGIC_SYMLINK) == 0) - ret = lstat(fn, sb); - else -@@ -290,7 +290,7 @@ file_fsmagic(struct magic_set *ms, const char *fn, struct stat *sb) - return -1; - break; - #endif --#if S_IFLNK != 0 -+#if S_IFLNK != 0 && ! defined(_WIN32) - case S_IFLNK: - if ((nch = readlink(fn, buf, BUFSIZ-1)) <= 0) { - if (ms->flags & MAGIC_ERROR) { diff --git a/ports/libmagic/CMakeLists.txt b/ports/libmagic/CMakeLists.txt new file mode 100644 index 000000000000..a6248bdf871a --- /dev/null +++ b/ports/libmagic/CMakeLists.txt @@ -0,0 +1,170 @@ +cmake_minimum_required(VERSION 3.12) + +file(READ ${CMAKE_CURRENT_SOURCE_DIR}/configure.ac CONFIGURE_AC_CONTENT) +string(REGEX MATCH "AC_INIT\\(\\[file\\],\\[([0-9]+\\.[0-9]+)\\]" _ ${CONFIGURE_AC_CONTENT}) +unset(CONFIGURE_AC_CONTENT) + +project(file VERSION ${CMAKE_MATCH_1}) + +option(FILE_TESTS "Enable file tests" OFF) + +# Get library directory for multiarch linux distros +include(GNUInstallDirs) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") +set(targets) +find_package(PCRE2 CONFIG REQUIRED COMPONENTS 8BIT) +if(WIN32 AND NOT MINGW) + find_package(unofficial-getopt-win32 CONFIG REQUIRED) +endif() + +set(LIBMAGIC_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") +set(LIBMAGIC_TEST_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests") + +set(WIN_COMPAT_SOURCES + ${LIBMAGIC_SOURCE_DIR}/asctime_r.c + ${LIBMAGIC_SOURCE_DIR}/asprintf.c + ${LIBMAGIC_SOURCE_DIR}/ctime_r.c + ${LIBMAGIC_SOURCE_DIR}/dprintf.c + ${LIBMAGIC_SOURCE_DIR}/getline.c + ${LIBMAGIC_SOURCE_DIR}/gmtime_r.c + ${LIBMAGIC_SOURCE_DIR}/localtime_r.c + ${LIBMAGIC_SOURCE_DIR}/pread.c + ${LIBMAGIC_SOURCE_DIR}/strcasestr.c + # ${LIBMAGIC_SOURCE_DIR}/strlcat.c + # ${LIBMAGIC_SOURCE_DIR}/strlcpy.c + ${LIBMAGIC_SOURCE_DIR}/vasprintf.c + ${LIBMAGIC_SOURCE_DIR}/fmtcheck.c + ${LIBMAGIC_SOURCE_DIR}/magic.def +) + +if(WIN32) + set(LIBMAGIC_SOURCE_FILES ${WIN_COMPAT_SOURCES}) +else() + set(LIBMAGIC_SOURCE_FILES) +endif() + +list(APPEND LIBMAGIC_SOURCE_FILES + ${LIBMAGIC_SOURCE_DIR}/apprentice.c + ${LIBMAGIC_SOURCE_DIR}/apptype.c + ${LIBMAGIC_SOURCE_DIR}/ascmagic.c + ${LIBMAGIC_SOURCE_DIR}/buffer.c + ${LIBMAGIC_SOURCE_DIR}/cdf.c + ${LIBMAGIC_SOURCE_DIR}/cdf_time.c + ${LIBMAGIC_SOURCE_DIR}/compress.c + ${LIBMAGIC_SOURCE_DIR}/der.c + ${LIBMAGIC_SOURCE_DIR}/encoding.c + ${LIBMAGIC_SOURCE_DIR}/fmtcheck.c + ${LIBMAGIC_SOURCE_DIR}/fsmagic.c + ${LIBMAGIC_SOURCE_DIR}/funcs.c + ${LIBMAGIC_SOURCE_DIR}/is_json.c + ${LIBMAGIC_SOURCE_DIR}/is_tar.c + ${LIBMAGIC_SOURCE_DIR}/magic.c + ${LIBMAGIC_SOURCE_DIR}/print.c + ${LIBMAGIC_SOURCE_DIR}/readcdf.c + ${LIBMAGIC_SOURCE_DIR}/readelf.c + ${LIBMAGIC_SOURCE_DIR}/softmagic.c + # ${LIBMAGIC_SOURCE_DIR}/strlcat.c + # ${LIBMAGIC_SOURCE_DIR}/strlcpy.c + ${LIBMAGIC_SOURCE_DIR}/is_csv.c +) +if(NOT APPLE) +list(APPEND LIBMAGIC_SOURCE_FILES + ${LIBMAGIC_SOURCE_DIR}/strlcat.c + ${LIBMAGIC_SOURCE_DIR}/strlcpy.c +) +endif() + +# replace the version in the magic.h.in and write it to magic.h +FILE(READ ${LIBMAGIC_SOURCE_DIR}/magic.h.in MAGIC_H_CONTENT) +STRING(REPLACE "." "" FILE_VERSION_WITHOUT_DOT "${CMAKE_PROJECT_VERSION}") +STRING(REPLACE "X.YY" ${FILE_VERSION_WITHOUT_DOT} MAGIC_H_CONTENT_NEW "${MAGIC_H_CONTENT}") +FILE(WRITE ${LIBMAGIC_SOURCE_DIR}/magic.h "${MAGIC_H_CONTENT_NEW}") + +add_compile_definitions(HAVE_CONFIG_H VERSION="${CMAKE_PROJECT_VERSION}") +if(WIN32) + add_compile_definitions(WIN32_LEAN_AND_MEAN WIN32) +endif() + +add_library(libmagic ${LIBMAGIC_SOURCE_FILES}) +set(targets ${targets} libmagic) + +target_link_libraries(libmagic PRIVATE PCRE2::POSIX) + +target_include_directories(libmagic + PUBLIC + "$" + "$" +) + +# 'file' CLI +add_executable(file ${LIBMAGIC_SOURCE_DIR}/file.c) +set(targets ${targets} file) +target_link_libraries(file PRIVATE PCRE2::POSIX libmagic) +if (WIN32) + if (NOT MINGW) + target_link_libraries(file PRIVATE unofficial::getopt-win32::getopt) + endif() + target_link_libraries(file PRIVATE shlwapi) +endif() + +if(MSVC) + target_include_directories(file PRIVATE getopt) +endif() + +# Following is the compilation of the magic file +file(GLOB MAGIC_FRAGMENTS magic/Magdir/*) + +# Prepare a temporary file to combine the magic fragments: +set(MAGIC_FRAGMENTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/magic) +file(WRITE ${MAGIC_FRAGMENTS_FILE} "") + +# Call the "cat" function for each input file +foreach(MAGIC_FRAGMENT ${MAGIC_FRAGMENTS}) + file(APPEND ${MAGIC_FRAGMENTS_FILE} "############## ${MAGIC_FRAGMENT} ##############\n") + file(READ ${MAGIC_FRAGMENT} MAGIC_FRAGMENT_CONTENTS) + file(APPEND ${MAGIC_FRAGMENTS_FILE} "${MAGIC_FRAGMENT_CONTENTS}\n") + unset(MAGIC_FRAGMENT_CONTENTS) +endforeach() + +add_custom_command(OUTPUT magic.mgc + COMMAND file -C -m magic + COMMENT "Compiling magic file" +) + +add_custom_target(magic_mgc ALL DEPENDS magic.mgc) + +# Include module with function 'configure_package_config_file' +include(CMakePackageConfigHelpers) + +set(PORT_NAME unofficial-libmagic) +set(TARGETS_EXPORT_NAME ${PORT_NAME}-targets) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/magic.mgc DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PORT_NAME}) + +install(FILES ${LIBMAGIC_SOURCE_DIR}/magic.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +install(TARGETS ${targets} + EXPORT ${TARGETS_EXPORT_NAME} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) +install(EXPORT ${TARGETS_EXPORT_NAME} + FILE ${TARGETS_EXPORT_NAME}.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME} + NAMESPACE unofficial::libmagic:: + ) +configure_package_config_file( + ${PORT_NAME}-config.cmake.in + ${PORT_NAME}-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME}) +write_basic_package_version_file( + ${PORT_NAME}-config-version.cmake + VERSION ${CMAKE_PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) +install(FILES + ${CMAKE_BINARY_DIR}/${PORT_NAME}-config.cmake + ${CMAKE_BINARY_DIR}/${PORT_NAME}-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PORT_NAME}) diff --git a/ports/libmagic/config.h b/ports/libmagic/config.h new file mode 100644 index 000000000000..3a6e492f7c21 --- /dev/null +++ b/ports/libmagic/config.h @@ -0,0 +1,401 @@ +/* Define in built-in ELF support is used */ +#define BUILTIN_ELF 1 + +/* Define for ELF core file support */ +#define ELFCORE 1 + +/* Define to 1 if you have the `asctime_r' function. */ +#undef HAVE_ASCTIME_R + +/* Define to 1 if you have the `asprintf' function. */ +#undef HAVE_ASPRINTF + +/* Define to 1 if you have the `ctime_r' function. */ +#undef HAVE_CTIME_R + +/* HAVE_DAYLIGHT */ +#define HAVE_DAYLIGHT 1 + +/* Define to 1 if you have the declaration of `daylight', and to 0 if you + don't. */ +#undef HAVE_DECL_DAYLIGHT + +/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. + */ +#undef HAVE_DECL_TZNAME + +/* Define to 1 if you have the header file. */ +#undef HAVE_DLFCN_H + +/* Define to 1 if you have the `dprintf' function. */ +#undef HAVE_DPRINTF + +/* Define to 1 if you have the header file. */ +#undef HAVE_ERR_H + +/* Define to 1 if you have the header file. */ +#ifdef WIN32 +#undef HAVE_DIRENT_H +#else +// TBD: will all non-win32 xplatforms we want have this? +#define HAVE_DIRENT_H 1 +#endif + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the `fmtcheck' function. */ +/*#undef HAVE_FMTCHECK*/ + +/* Define to 1 if you have the `fork' function. */ +#undef HAVE_FORK + +/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */ +#undef HAVE_FSEEKO + +/* Define to 1 if you have the `getline' function. */ +#undef HAVE_GETLINE + +/* Define to 1 if you have the header file. */ +#ifdef _WIN32 +#define HAVE_GETOPT_H 1 +#endif + +/* Define to 1 if you have the `getopt_long' function. */ +#undef HAVE_GETOPT_LONG + +/* Define to 1 if you have the `getpagesize' function. */ +#undef HAVE_GETPAGESIZE + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `gnurx' library (-lgnurx). */ +#undef HAVE_LIBGNURX + +/* Define to 1 if you have the `z' library (-lz). */ +/* #undef HAVE_LIBZ */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Define to 1 if mbrtowc and mbstate_t are properly declared. */ +#define HAVE_MBRTOWC 1 + +/* Define to 1 if declares mbstate_t. */ +#define HAVE_MBSTATE_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `mkostemp' function. */ +#undef HAVE_MKOSTEMP + +/* Define to 1 if you have the `mkstemp' function. */ +#ifdef _WIN32 +#undef HAVE_MKSTEMP +#else +#define HAVE_MKSTEMP 1 +#endif + +/* Define to 1 if you have a working `mmap' system call. */ +#undef HAVE_MMAP + +/* Define to 1 if you have the `pread' function. */ +#undef HAVE_PREAD + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if the system has the type `pid_t'. */ +#undef HAVE_PID_T + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strcasestr' function. */ +#if defined(_WIN32) && !defined(__MINGW32__) +#define HAVE_STRCASESTR 1 +#else +#undef HAVE_STRCASESTR +#endif + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the header file. */ +#undef HAVE_STRINGS_H + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +#undef HAVE_STRLCAT + +/* Define to 1 if you have the `strlcpy' function. */ +#undef HAVE_STRLCPY + +/* Define to 1 if you have the `strndup' function. */ +#undef HAVE_STRNDUP + +/* Define to 1 if you have the `strtof' function. */ +#undef HAVE_STRTOF + +/* Define to 1 if you have the `strtoul' function. */ +#define HAVE_STRTOUL 1 + +/* HAVE_STRUCT_OPTION */ +#define HAVE_STRUCT_OPTION 1 + +/* Define to 1 if `st_rdev' is a member of `struct stat'. */ +#undef HAVE_STRUCT_STAT_ST_RDEV + +/* Define to 1 if `tm_gmtoff' is a member of `struct tm'. */ +#undef HAVE_STRUCT_TM_TM_GMTOFF + +/* Define to 1 if `tm_zone' is a member of `struct tm'. */ +#undef HAVE_STRUCT_TM_TM_ZONE + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_MMAN_H + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PARAM_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_TIME_H + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_UTIME_H + +/* Define to 1 if you have that is POSIX.1 compatible. */ +#undef HAVE_SYS_WAIT_H + +/* HAVE_TM_ISDST */ +#undef HAVE_TM_ISDST + +/* HAVE_TM_ZONE */ +#undef HAVE_TM_ZONE + +/* HAVE_TZNAME */ +#undef HAVE_TZNAME + +/* Define to 1 if the system has the type `int32_t'. */ +#define HAVE_INT32_T 1 + +/* Define to 1 if the system has the type `int64_t'. */ +#define HAVE_INT64_T 1 + +/* Define to 1 if the system has the type `uint16_t'. */ +#define HAVE_UINT16_T 1 + +/* Define to 1 if the system has the type `uint32_t'. */ +#define HAVE_UINT32_T 1 + +/* Define to 1 if the system has the type `uint64_t'. */ +#define HAVE_UINT64_T 1 + +/* Define to 1 if the system has the type `uint8_t'. */ +#define HAVE_UINT8_T 1 + +/* Define to 1 if you have the header file. */ +/* turns out, v5.39 file/src/buffer.c does -not- subject inclusion to this define */ +#ifndef _WIN32 +#define HAVE_UNISTD_H 1 +#endif + +/* Define to 1 if you have the `utime' function. */ +#undef HAVE_UTIME + +/* Define to 1 if you have the `utimes' function. */ +#undef HAVE_UTIMES + +/* Define to 1 if you have the header file. */ +#undef HAVE_UTIME_H + +/* Define to 1 if you have the `vasprintf' function. */ +#undef HAVE_VASPRINTF + +/* Define to 1 if you have the `vfork' function. */ +#undef HAVE_VFORK + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_VFORK_H */ + +/* Define to 1 or 0, depending whether the compiler supports simple visibility + declarations. */ +#undef HAVE_VISIBILITY + +/* Define to 1 if you have the header file. */ +#define HAVE_WCHAR_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_WCTYPE_H 1 + +/* Define to 1 if you have the `wcwidth' function. */ +#undef HAVE_WCWIDTH + +/* Define to 1 if `fork' works. */ +#undef HAVE_WORKING_FORK + +/* Define to 1 if `vfork' works. */ +#undef HAVE_WORKING_VFORK + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_ZLIB_H */ + +/* Define to the sub-directory in which libtool stores uninstalled libraries. + */ +#undef LT_OBJDIR + +/* Define to 1 if `major', `minor', and `makedev' are declared in . + */ +#undef MAJOR_IN_MKDEV + +/* Define to 1 if `major', `minor', and `makedev' are declared in + . */ +#undef MAJOR_IN_SYSMACROS + +/* Name of package */ +#undef PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#undef PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME + +/* Define to the home page for this package. */ +#undef PACKAGE_URL + +/* Define to the version of this package. */ +#undef PACKAGE_VERSION + +/* The size of `long long', as computed by sizeof. */ +#undef SIZEOF_LONG_LONG + +/* Define to 1 if you have the ANSI C header files. */ +#undef STDC_HEADERS + +/* Define to 1 if your declares `struct tm'. */ +#undef TM_IN_SYS_TIME + +/* Enable extensions on AIX 3, Interix. */ +#ifndef _ALL_SOURCE +# undef _ALL_SOURCE +#endif +/* Enable GNU extensions on systems that have them. */ +#ifndef _GNU_SOURCE +# undef _GNU_SOURCE +#endif +/* Enable threading extensions on Solaris. */ +#ifndef _POSIX_PTHREAD_SEMANTICS +# undef _POSIX_PTHREAD_SEMANTICS +#endif +/* Enable extensions on HP NonStop. */ +#ifndef _TANDEM_SOURCE +# undef _TANDEM_SOURCE +#endif +/* Enable general extensions on Solaris. */ +#ifndef __EXTENSIONS__ +# undef __EXTENSIONS__ +#endif + + +/* Number of bits in a file offset, on hosts where this is settable. */ +#undef _FILE_OFFSET_BITS + +/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */ +#undef _LARGEFILE_SOURCE + +/* Define for large files, on AIX-style hosts. */ +#undef _LARGE_FILES + +/* Define to 1 if on MINIX. */ +#undef _MINIX + +/* Define to 2 if the system does not provide POSIX.1 features except with + this defined. */ +#undef _POSIX_1_SOURCE + +/* Define to 1 if you need to in order for `stat' and other things to work. */ +#undef _POSIX_SOURCE + +/* Define to empty if `const' does not conform to ANSI C. */ +//#define const + +/* Define to a type if does not define. */ +//#undef mbstate_t + +/* Define to `long int' if does not define. */ +//#undef off_t + +/* Define to `unsigned int' if does not define. */ +//#undef size_t + + +#ifndef HAVE_UINT8_T +typedef unsigned char uint8_t; +#endif +#ifndef HAVE_UINT16_T +typedef unsigned short uint16_t; +#endif +#ifndef HAVE_UINT32_T +typedef unsigned int uint32_t; +#endif +#ifndef HAVE_INT32_T +typedef int int32_t; +#endif +#ifndef HAVE_UINT64_T +#if SIZEOF_LONG_LONG == 8 +typedef unsigned long long uint64_t; +#else +typedef unsigned long uint64_t; +#endif +#endif +#ifndef HAVE_INT64_T +#if SIZEOF_LONG_LONG == 8 +typedef long long int64_t; +#else +typedef long int64_t; +#endif +#endif + +#ifndef _SSIZE_T_DEFINED +#ifdef _WIN32 +#if defined(__MINGW32__) && !defined(__MINGW64__) +typedef int ssize_t; +#else +#include +typedef int64_t ssize_t; +#endif +#endif +#define _SSIZE_T_DEFINED +#endif + +#ifdef _WIN32 +#include + +#include +#endif + +/* Define as `fork' if `vfork' does not work. */ +/* #undef vfork */ diff --git a/ports/libmagic/magic.def b/ports/libmagic/magic.def new file mode 100644 index 000000000000..f286a62ae487 --- /dev/null +++ b/ports/libmagic/magic.def @@ -0,0 +1,20 @@ +LIBRARY libmagic +EXPORTS + magic_open + magic_close + magic_getpath + magic_file + magic_descriptor + magic_buffer + magic_error + magic_setflags + magic_version + magic_load + magic_load_buffers + magic_compile + magic_check + magic_list + magic_errno + magic_setparam + magic_getparam + getline diff --git a/ports/libmagic/portfile.cmake b/ports/libmagic/portfile.cmake index 34ecdb150d1e..c40c3e2bac64 100644 --- a/ports/libmagic/portfile.cmake +++ b/ports/libmagic/portfile.cmake @@ -1,62 +1,53 @@ +set(PATCHES + "0001-Use-pcre2.patch" +) + if(VCPKG_TARGET_IS_WINDOWS) set(PATCHES - "0001-Use-libtre.patch" - "0002-Change-zlib-lib-name-to-match-CMake-output.patch" + ${PATCHES} "0003-Fix-WIN32-macro-checks.patch" "0004-Typedef-POSIX-types-on-Windows.patch" "0005-Include-dirent.h-for-S_ISREG-and-S_ISDIR.patch" "0006-Remove-Wrap-POSIX-headers.patch" "0007-Substitute-unistd-macros-for-MSVC.patch" "0008-Add-FILENO-defines.patch" + "0009-No-fcntl-in-magic.c.patch" "0010-Properly-check-for-the-presence-of-bitmasks.patch" "0011-Remove-pipe-related-functions-in-funcs.c.patch" - "0012-Convert-MSYS2-paths-to-Windows-paths.patch" - "0013-Check-for-backslash-in-argv-0-on-Windows.patch" + "0014-Define-POSIX-macros-if-missing.patch" "0015-MSYS2-Remove-ioctl-call.patch" - "0016-Fix-file_famagic-function.patch" ) endif() vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO file/file - REF FILE5_45 - SHA512 fdd4c5d13d5ea1d25686c76d8ebc3252c54040c4871e3f0f623c4548b3841795d4e36050292a9453eedf0fbf932573890e9d6ac9fa63ccf577215598ae84b9ea + REF FILE5_40 + SHA512 d76bfe5326e1b40368e055c2e049a24b4ffdbd727371f4f3aa1dd3f53787d16b88550b3cc71ecf02151e2fb3e567eb2598e4707badab8c391eb71113c2dcc319 HEAD_REF master PATCHES ${PATCHES} ) -if(VCPKG_TARGET_IS_WINDOWS) - set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS") - set(VCPKG_CXX_FLAGS "${VCPKG_CXX_FLAGS} -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS") -endif() - -set(CONFIG_OPTIONS) - -if(NOT "zlib" IN_LIST FEATURES) - list(APPEND CONFIG_OPTIONS "--disable-zlib") -endif() +file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}") +file(COPY "${CMAKE_CURRENT_LIST_DIR}/unofficial-libmagic-config.cmake.in" DESTINATION "${SOURCE_PATH}") +file(COPY "${CMAKE_CURRENT_LIST_DIR}/magic.def" DESTINATION "${SOURCE_PATH}/src") +file(COPY "${CMAKE_CURRENT_LIST_DIR}/config.h" DESTINATION "${SOURCE_PATH}/src") -if(NOT "bzip2" IN_LIST FEATURES) - list(APPEND CONFIG_OPTIONS "--disable-bzlib") -endif() - -if(NOT "lzma" IN_LIST FEATURES) - list(APPEND CONFIG_OPTIONS "--disable-xzlib") -endif() +vcpkg_cmake_configure( + SOURCE_PATH ${SOURCE_PATH} +) -if(NOT "zstd" IN_LIST FEATURES) - list(APPEND CONFIG_OPTIONS "--disable-zstdlib") -endif() +vcpkg_cmake_install() +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() +vcpkg_copy_tools(TOOL_NAMES file AUTO_CLEAN) +vcpkg_cmake_config_fixup( + CONFIG_PATH lib/cmake/unofficial-libmagic + PACKAGE_NAME unofficial-libmagic) -vcpkg_configure_make( - AUTOCONFIG - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - ${CONFIG_OPTIONS} - "--disable-lzlib" - "--disable-libseccomp" -) +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/${PORT}/man5") if(VCPKG_CROSSCOMPILING) vcpkg_add_to_path(PREPEND "${CURRENT_HOST_INSTALLED_DIR}/tools/libmagic/bin") @@ -64,37 +55,23 @@ elseif(VCPKG_TARGET_IS_WINDOWS AND VCPKG_LIBRARY_LINKAGE STREQUAL dynamic) set(EXTRA_ARGS "ADD_BIN_TO_PATH") endif() -vcpkg_install_make(${EXTRA_ARGS}) -vcpkg_copy_tool_dependencies("${CURRENT_PACKAGES_DIR}/tools/${PORT}/bin") -vcpkg_copy_tool_dependencies("${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/bin") -vcpkg_fixup_pkgconfig() - if(VCPKG_LIBRARY_LINKAGE STREQUAL static) file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") endif() +set(UNOFFICIAL_PORT unofficial-${PORT}) + if(VCPKG_TARGET_IS_WINDOWS) if(NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/share/misc") - file(COPY "${CURRENT_PACKAGES_DIR}/share/${PORT}/misc/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/share/misc") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/share/misc") + file(COPY "${CURRENT_PACKAGES_DIR}/share/${UNOFFICIAL_PORT}/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/share/misc") endif() if(NOT VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/share/misc") - file(COPY "${CURRENT_PACKAGES_DIR}/share/${PORT}/misc/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}/debug/share/misc") + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/debug/share/misc") + file(COPY "${CURRENT_PACKAGES_DIR}/share/${UNOFFICIAL_PORT}/magic.mgc" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${UNOFFICIAL_PORT}/debug/share/misc") endif() endif() -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/${PORT}/man5") - -include(CMakePackageConfigHelpers) -configure_package_config_file( - "${CMAKE_CURRENT_LIST_DIR}/unofficial-${PORT}-config.cmake.in" - "${CURRENT_PACKAGES_DIR}/share/unofficial-${PORT}/unofficial-${PORT}-config.cmake" - INSTALL_DESTINATION "share/unofficial-${PORT}" -) - # Handle copyright and usage vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/ports/libmagic/unofficial-libmagic-config.cmake.in b/ports/libmagic/unofficial-libmagic-config.cmake.in index 5ff3b0380aa7..d3671f89b479 100644 --- a/ports/libmagic/unofficial-libmagic-config.cmake.in +++ b/ports/libmagic/unofficial-libmagic-config.cmake.in @@ -1,69 +1,12 @@ @PACKAGE_INIT@ -if(WIN32 AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") - include(CMakeFindDependencyMacro) - find_dependency(unofficial-tre) -endif() +include(CMakeFindDependencyMacro) +find_dependency(PCRE2 COMPONENTS 8BIT POSIX) -# Compute the installation prefix relative to this file. -get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) -get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) -if(_IMPORT_PREFIX STREQUAL "/") - set(_IMPORT_PREFIX "") -endif() +include("${CMAKE_CURRENT_LIST_DIR}/@TARGETS_EXPORT_NAME@.cmake") -if("@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") - add_library(unofficial::libmagic::libmagic STATIC IMPORTED) -else() - add_library(unofficial::libmagic::libmagic SHARED IMPORTED) -endif() +set(unofficial-libmagic_VERSION @CMAKE_PROJECT_VERSION@) +set(unofficial-libmagic_FOUND true) +set(unofficial-libmagic_DICTIONARY "${CMAKE_CURRENT_LIST_DIR}/magic.mgc") -set_target_properties(unofficial::libmagic::libmagic PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${_IMPORT_PREFIX}/include" -) - -if(WIN32 AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") - set_target_properties(unofficial::libmagic::libmagic PROPERTIES - INTERFACE_LINK_LIBRARIES "\$" - ) -endif() - -macro(add_library_config config prefix) - set_property(TARGET unofficial::libmagic::libmagic APPEND PROPERTY IMPORTED_CONFIGURATIONS ${config}) - if("@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") - set_target_properties(unofficial::libmagic::libmagic PROPERTIES - IMPORTED_LOCATION_${config} "${_IMPORT_PREFIX}/${prefix}lib/@VCPKG_TARGET_STATIC_LIBRARY_PREFIX@magic@VCPKG_TARGET_STATIC_LIBRARY_SUFFIX@" - IMPORTED_LINK_INTERFACE_LANGUAGES_${config} "C" - ) - else() - if(WIN32) - set(library_dir "${_IMPORT_PREFIX}/${prefix}bin/") - set(soversion_suffix "-1") - set_target_properties(unofficial::libmagic::libmagic PROPERTIES - IMPORTED_IMPLIB_${config} "${_IMPORT_PREFIX}/${prefix}/lib/@VCPKG_TARGET_IMPORT_LIBRARY_PREFIX@magic@VCPKG_TARGET_IMPORT_LIBRARY_SUFFIX@" - ) - else() - set(library_dir "${_IMPORT_PREFIX}/${prefix}lib/") - endif() - set_target_properties(unofficial::libmagic::libmagic PROPERTIES - IMPORTED_LOCATION_${config} "${library_dir}@VCPKG_TARGET_SHARED_LIBRARY_PREFIX@magic${soversion_suffix}@VCPKG_TARGET_SHARED_LIBRARY_SUFFIX@" - ) - unset(soversion_suffix) - unset(library_dir) - endif() -endmacro() - -if("@VCPKG_BUILD_TYPE@" STREQUAL "" OR "@VCPKG_BUILD_TYPE@" STREQUAL "debug") - add_library_config(DEBUG "debug/") -endif() - -if("@VCPKG_BUILD_TYPE@" STREQUAL "" OR "@VCPKG_BUILD_TYPE@" STREQUAL "release") - add_library_config(RELEASE "") -endif() - -set_and_check(unofficial-libmagic_DICTIONARY "${_IMPORT_PREFIX}/share/libmagic/misc/magic.mgc") - -unset(_IMPORT_PREFIX) - -check_required_components(unofficial-libmagic) +check_required_components("unofficial-libmagic") diff --git a/ports/libmagic/vcpkg.json b/ports/libmagic/vcpkg.json index e809501e900c..63944114b565 100644 --- a/ports/libmagic/vcpkg.json +++ b/ports/libmagic/vcpkg.json @@ -1,52 +1,26 @@ { "name": "libmagic", - "version": "5.45", - "port-version": 2, + "version-string": "5.40", + "port-version": 1, "description": "This library can be used to classify files according to magic number tests.", "homepage": "https://github.com/file/file", - "license": "BSD-2-Clause", "dependencies": [ { - "name": "dirent", - "platform": "windows" - }, - { - "name": "getopt", - "platform": "windows" + "name": "vcpkg-cmake", + "host": true }, { - "name": "libmagic", + "name": "vcpkg-cmake-config", "host": true }, { - "name": "tre", - "platform": "windows | mingw" - } - ], - "features": { - "zlib": { - "description": "Enable zlib support", - "dependencies": [ - "zlib" - ] - }, - "bzlib": { - "description": "Enable Bzip2 support", - "dependencies": [ - "bzip2" - ] + "name": "dirent", + "platform": "windows" }, - "xzlib": { - "description": "Enable liblzma/xz support", - "dependencies": [ - "liblzma" - ] + { + "name": "getopt", + "platform": "windows & !mingw" }, - "zstdlib": { - "description": "Enable zstdlib support", - "dependencies": [ - "zstd" - ] - } - } + "pcre2" + ] } diff --git a/ports/pcre2/fix-cmake.patch b/ports/pcre2/fix-cmake.patch new file mode 100644 index 000000000000..93d2f7196957 --- /dev/null +++ b/ports/pcre2/fix-cmake.patch @@ -0,0 +1,334 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index cec7dfb..84d1769 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -100,6 +100,9 @@ + # 2021-08-28 PH increased minimum version + # 2021-08-28 PH added test for realpath() + # 2022-12-10 PH added support for pcre2posix_test ++# 2023-01-15 Carlo added C99 as the minimum required ++# 2023-06-03 Theodore used standard CMake constructs to export the library's targets. ++# 2023-08-06 PH added support for setting variable length lookbehind maximum + + # Increased minimum to 2.8.5 to support GNUInstallDirs. + # Increased minimum to 3.1 to support imported targets. +@@ -136,6 +139,7 @@ INCLUDE(CheckFunctionExists) + INCLUDE(CheckSymbolExists) + INCLUDE(CheckIncludeFile) + INCLUDE(CheckTypeSize) ++INCLUDE(CMakePackageConfigHelpers) + INCLUDE(GNUInstallDirs) # for CMAKE_INSTALL_LIBDIR + + CHECK_INCLUDE_FILE(dirent.h HAVE_DIRENT_H) +@@ -705,7 +709,9 @@ IF(PCRE2_BUILD_PCRE2_8) + VERSION ${LIBPCRE2_8_VERSION} + SOVERSION ${LIBPCRE2_8_SOVERSION}) + TARGET_COMPILE_DEFINITIONS(pcre2-8-static PUBLIC PCRE2_STATIC) +- TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-8-static PUBLIC ++ $ ++ $) + IF(REQUIRE_PTHREAD) + TARGET_LINK_LIBRARIES(pcre2-8-static Threads::Threads) + ENDIF(REQUIRE_PTHREAD) +@@ -718,8 +724,9 @@ IF(PCRE2_BUILD_PCRE2_8) + VERSION ${LIBPCRE2_POSIX_VERSION} + SOVERSION ${LIBPCRE2_POSIX_SOVERSION}) + TARGET_LINK_LIBRARIES(pcre2-posix-static pcre2-8-static) +- TARGET_COMPILE_DEFINITIONS(pcre2-posix-static PUBLIC PCRE2_STATIC) +- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-static PUBLIC ++ $ ++ $) + SET(targets ${targets} pcre2-posix-static) + + IF(MSVC) +@@ -736,7 +743,9 @@ IF(PCRE2_BUILD_PCRE2_8) + + IF(BUILD_SHARED_LIBS) + ADD_LIBRARY(pcre2-8-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) +- TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-8-shared PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-8-shared PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_8_MACHO_COMPATIBILITY_VERSION}" +@@ -749,7 +758,9 @@ IF(PCRE2_BUILD_PCRE2_8) + ENDIF(REQUIRE_PTHREAD) + SET(targets ${targets} pcre2-8-shared) + ADD_LIBRARY(pcre2-posix-shared SHARED ${PCRE2POSIX_HEADERS} ${PCRE2POSIX_SOURCES}) +- TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-posix-shared PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-posix-shared PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=8 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_POSIX_MACHO_COMPATIBILITY_VERSION}" +@@ -786,7 +797,9 @@ ENDIF(PCRE2_BUILD_PCRE2_8) + IF(PCRE2_BUILD_PCRE2_16) + IF(BUILD_STATIC_LIBS) + ADD_LIBRARY(pcre2-16-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) +- TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-16-static PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" +@@ -811,7 +824,9 @@ IF(PCRE2_BUILD_PCRE2_16) + + IF(BUILD_SHARED_LIBS) + ADD_LIBRARY(pcre2-16-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) +- TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-16-shared PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-16-shared PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=16 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" +@@ -848,7 +863,9 @@ ENDIF(PCRE2_BUILD_PCRE2_16) + IF(PCRE2_BUILD_PCRE2_32) + IF(BUILD_STATIC_LIBS) + ADD_LIBRARY(pcre2-32-static STATIC ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) +- TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-32-static PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" +@@ -873,7 +890,9 @@ IF(PCRE2_BUILD_PCRE2_32) + + IF(BUILD_SHARED_LIBS) + ADD_LIBRARY(pcre2-32-shared SHARED ${PCRE2_HEADERS} ${PCRE2_SOURCES} ${PROJECT_BINARY_DIR}/config.h) +- TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC ${PROJECT_BINARY_DIR}) ++ TARGET_INCLUDE_DIRECTORIES(pcre2-32-shared PUBLIC ++ $ ++ $) + SET_TARGET_PROPERTIES(pcre2-32-shared PROPERTIES + COMPILE_DEFINITIONS PCRE2_CODE_UNIT_WIDTH=32 + MACHO_COMPATIBILITY_VERSION "${LIBPCRE2_32_MACHO_COMPATIBILITY_VERSION}" +@@ -1075,9 +1094,13 @@ ENDIF(PCRE2_BUILD_TESTS) + SET(CMAKE_INSTALL_ALWAYS 1) + + INSTALL(TARGETS ${targets} +- RUNTIME DESTINATION bin ++ EXPORT pcre2-targets ++ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) ++INSTALL(EXPORT pcre2-targets ++ DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2 ++ NAMESPACE pcre2::) + INSTALL(FILES ${pkg_config_files} DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/pcre2-config" + DESTINATION bin +@@ -1089,11 +1112,12 @@ INSTALL(FILES ${PCRE2_HEADERS} ${PCRE2POSIX_HEADERS} DESTINATION include) + # CMake config files. + set(PCRE2_CONFIG_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config.cmake.in) + set(PCRE2_CONFIG_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config.cmake) +-configure_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} @ONLY) +-set(PCRE2_CONFIG_VERSION_IN ${CMAKE_CURRENT_SOURCE_DIR}/cmake/pcre2-config-version.cmake.in) ++configure_package_config_file(${PCRE2_CONFIG_IN} ${PCRE2_CONFIG_OUT} INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) + set(PCRE2_CONFIG_VERSION_OUT ${CMAKE_CURRENT_BINARY_DIR}/cmake/pcre2-config-version.cmake) +-configure_file(${PCRE2_CONFIG_VERSION_IN} ${PCRE2_CONFIG_VERSION_OUT} @ONLY) +-install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION cmake) ++write_basic_package_version_file(${PCRE2_CONFIG_VERSION_OUT} ++ VERSION ${PCRE2_MAJOR}.${PCRE2_MINOR}.0 ++ COMPATIBILITY SameMajorVersion) ++install(FILES ${PCRE2_CONFIG_OUT} ${PCRE2_CONFIG_VERSION_OUT} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pcre2) + + FILE(GLOB html ${PROJECT_SOURCE_DIR}/doc/html/*.html) + FILE(GLOB man1 ${PROJECT_SOURCE_DIR}/doc/*.1) +diff --git a/cmake/pcre2-config-version.cmake.in b/cmake/pcre2-config-version.cmake.in +deleted file mode 100644 +index dac149e..0000000 +--- a/cmake/pcre2-config-version.cmake.in ++++ /dev/null +@@ -1,15 +0,0 @@ +-set(PACKAGE_VERSION_MAJOR @PCRE2_MAJOR@) +-set(PACKAGE_VERSION_MINOR @PCRE2_MINOR@) +-set(PACKAGE_VERSION_PATCH 0) +-set(PACKAGE_VERSION @PCRE2_MAJOR@.@PCRE2_MINOR@.0) +- +-# Check whether the requested PACKAGE_FIND_VERSION is compatible +-if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION OR +- PACKAGE_VERSION_MAJOR GREATER PACKAGE_FIND_VERSION_MAJOR) +- set(PACKAGE_VERSION_COMPATIBLE FALSE) +-else() +- set(PACKAGE_VERSION_COMPATIBLE TRUE) +- if(PACKAGE_VERSION VERSION_EQUAL PACKAGE_FIND_VERSION) +- set(PACKAGE_VERSION_EXACT TRUE) +- endif() +-endif() +diff --git a/cmake/pcre2-config.cmake.in b/cmake/pcre2-config.cmake.in +index b313d6d..159669b 100644 +--- a/cmake/pcre2-config.cmake.in ++++ b/cmake/pcre2-config.cmake.in +@@ -5,11 +5,17 @@ + # + # Static vs. shared + # ----------------- +-# To make use of the static library instead of the shared one, one needs ++# To force using the static library instead of the shared one, one needs + # to set the variable PCRE2_USE_STATIC_LIBS to ON before calling find_package. ++# If the variable is not set, the static library will be used if only that has ++# been built, otherwise the shared library will be used. ++# ++# The following components are supported: 8BIT, 16BIT, 32BIT and POSIX. ++# They used to be required but not anymore; all available targets will ++# be defined regardless of the requested components. + # Example: + # set(PCRE2_USE_STATIC_LIBS ON) +-# find_package(PCRE2 CONFIG COMPONENTS 8BIT) ++# find_package(PCRE2 CONFIG) + # + # This will define the following variables: + # +@@ -23,70 +29,42 @@ + # PCRE2::32BIT - The 32 bit PCRE2 library. + # PCRE2::POSIX - The POSIX PCRE2 library. + +-set(PCRE2_NON_STANDARD_LIB_PREFIX @NON_STANDARD_LIB_PREFIX@) +-set(PCRE2_NON_STANDARD_LIB_SUFFIX @NON_STANDARD_LIB_SUFFIX@) +-set(PCRE2_8BIT_NAME pcre2-8) +-set(PCRE2_16BIT_NAME pcre2-16) +-set(PCRE2_32BIT_NAME pcre2-32) +-set(PCRE2_POSIX_NAME pcre2-posix) +-find_path(PCRE2_INCLUDE_DIR NAMES pcre2.h DOC "PCRE2 include directory") +-if (PCRE2_USE_STATIC_LIBS) +- if (MSVC) +- set(PCRE2_8BIT_NAME pcre2-8-static) +- set(PCRE2_16BIT_NAME pcre2-16-static) +- set(PCRE2_32BIT_NAME pcre2-32-static) +- set(PCRE2_POSIX_NAME pcre2-posix-static) +- endif () ++@PACKAGE_INIT@ + +- set(PCRE2_PREFIX ${CMAKE_STATIC_LIBRARY_PREFIX}) +- set(PCRE2_SUFFIX ${CMAKE_STATIC_LIBRARY_SUFFIX}) +-else () +- set(PCRE2_PREFIX ${CMAKE_SHARED_LIBRARY_PREFIX}) +- if (MINGW AND PCRE2_NON_STANDARD_LIB_PREFIX) +- set(PCRE2_PREFIX "") +- endif () ++include(CMakeFindDependencyMacro) ++if("@REQUIRE_PTHREAD@") # REQUIRE_PTHREAD ++ find_dependency(Threads) ++endif() + +- set(PCRE2_SUFFIX ${CMAKE_SHARED_LIBRARY_SUFFIX}) +- if (MINGW AND PCRE2_NON_STANDARD_LIB_SUFFIX) +- set(PCRE2_SUFFIX "-0.dll") +- endif () +-endif () +-find_library(PCRE2_8BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit PCRE2 library") +-find_library(PCRE2_16BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_16BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "16 bit PCRE2 library") +-find_library(PCRE2_32BIT_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_32BIT_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "32 bit PCRE2 library") +-find_library(PCRE2_POSIX_LIBRARY NAMES ${PCRE2_PREFIX}${PCRE2_POSIX_NAME}${PCRE2_SUFFIX} ${PCRE2_PREFIX}${PCRE2_8BIT_NAME}d${PCRE2_SUFFIX} DOC "8 bit POSIX PCRE2 library") +-unset(PCRE2_NON_STANDARD_LIB_PREFIX) +-unset(PCRE2_NON_STANDARD_LIB_SUFFIX) +-unset(PCRE2_8BIT_NAME) +-unset(PCRE2_16BIT_NAME) +-unset(PCRE2_32BIT_NAME) +-unset(PCRE2_POSIX_NAME) ++include("${CMAKE_CURRENT_LIST_DIR}/pcre2-targets.cmake") + + # Set version +-if (PCRE2_INCLUDE_DIR) +- set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") +-endif () ++set(PCRE2_VERSION "@PCRE2_MAJOR@.@PCRE2_MINOR@.0") + +-# Which components have been found. +-if (PCRE2_8BIT_LIBRARY) +- set(PCRE2_8BIT_FOUND TRUE) +-endif () +-if (PCRE2_16BIT_LIBRARY) +- set(PCRE2_16BIT_FOUND TRUE) +-endif () +-if (PCRE2_32BIT_LIBRARY) +- set(PCRE2_32BIT_FOUND TRUE) +-endif () +-if (PCRE2_POSIX_LIBRARY) +- set(PCRE2_POSIX_FOUND TRUE) +-endif () +- +-# Check if at least one component has been specified. +-list(LENGTH PCRE2_FIND_COMPONENTS PCRE2_NCOMPONENTS) +-if (PCRE2_NCOMPONENTS LESS 1) +- message(FATAL_ERROR "No components have been specified. This is not allowed. Please, specify at least one component.") +-endif () +-unset(PCRE2_NCOMPONENTS) ++# Chooses the linkage of the library to expose in the ++# unsuffixed edition of the target. ++macro(_pcre2_add_component_target component target) ++ # If the static library exists and either PCRE2_USE_STATIC_LIBS ++ # is defined, or the dynamic library does not exist, use the static library. ++ if(NOT TARGET PCRE2::${component}) ++ if(TARGET pcre2::pcre2-${target}-static AND (PCRE2_USE_STATIC_LIBS OR NOT TARGET pcre2::pcre2-${target}-shared)) ++ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-static) ++ set(PCRE2_${component}_FOUND TRUE) ++ # Otherwise use the dynamic library if it exists. ++ elseif(TARGET pcre2::pcre2-${target}-shared AND NOT PCRE2_USE_STATIC_LIBS) ++ add_library(PCRE2::${component} ALIAS pcre2::pcre2-${target}-shared) ++ set(PCRE2_${component}_FOUND TRUE) ++ endif() ++ if(PCRE2_${component}_FOUND) ++ get_target_property(PCRE2_${component}_LIBRARY PCRE2::${component} IMPORTED_LOCATION) ++ set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) ++ endif() ++ endif() ++endmacro() ++_pcre2_add_component_target(8BIT 8) ++_pcre2_add_component_target(16BIT 16) ++_pcre2_add_component_target(32BIT 32) ++_pcre2_add_component_target(POSIX posix) + + # When POSIX component has been specified make sure that also 8BIT component is specified. + set(PCRE2_8BIT_COMPONENT FALSE) +@@ -105,41 +83,5 @@ endif() + unset(PCRE2_8BIT_COMPONENT) + unset(PCRE2_POSIX_COMPONENT) + +-include(FindPackageHandleStandardArgs) +-set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") +-find_package_handle_standard_args(PCRE2 +- FOUND_VAR PCRE2_FOUND +- REQUIRED_VARS PCRE2_INCLUDE_DIR +- HANDLE_COMPONENTS +- VERSION_VAR PCRE2_VERSION +- CONFIG_MODE +-) +- +-set(PCRE2_LIBRARIES) +-if (PCRE2_FOUND) +- foreach(component ${PCRE2_FIND_COMPONENTS}) +- if (PCRE2_USE_STATIC_LIBS) +- add_library(PCRE2::${component} STATIC IMPORTED) +- target_compile_definitions(PCRE2::${component} INTERFACE PCRE2_STATIC) +- else () +- add_library(PCRE2::${component} SHARED IMPORTED) +- endif () +- set_target_properties(PCRE2::${component} PROPERTIES +- IMPORTED_LOCATION "${PCRE2_${component}_LIBRARY}" +- INTERFACE_INCLUDE_DIRECTORIES "${PCRE2_INCLUDE_DIR}" +- ) +- if (component STREQUAL "POSIX") +- set_target_properties(PCRE2::${component} PROPERTIES +- INTERFACE_LINK_LIBRARIES "PCRE2::8BIT" +- LINK_LIBRARIES "PCRE2::8BIT" +- ) +- endif () +- +- set(PCRE2_LIBRARIES ${PCRE2_LIBRARIES} ${PCRE2_${component}_LIBRARY}) +- mark_as_advanced(PCRE2_${component}_LIBRARY) +- endforeach() +-endif () +- +-mark_as_advanced( +- PCRE2_INCLUDE_DIR +-) ++# Check for required components. ++check_required_components("PCRE2") diff --git a/ports/pcre2/no-static-suffix.patch b/ports/pcre2/no-static-suffix.patch new file mode 100644 index 000000000000..7f41bcd566cf --- /dev/null +++ b/ports/pcre2/no-static-suffix.patch @@ -0,0 +1,33 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index fa2181e..3bf5317 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -711,8 +711,8 @@ IF(PCRE2_BUILD_PCRE2_8) + SET(targets ${targets} pcre2-posix-static) + + IF(MSVC) +- SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8-static) +- SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix-static) ++ SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) ++ SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) + ELSE(MSVC) + SET_TARGET_PROPERTIES(pcre2-8-static PROPERTIES OUTPUT_NAME pcre2-8) + SET_TARGET_PROPERTIES(pcre2-posix-static PROPERTIES OUTPUT_NAME pcre2-posix) +@@ -777,7 +777,7 @@ IF(PCRE2_BUILD_PCRE2_16) + SET(targets ${targets} pcre2-16-static) + + IF(MSVC) +- SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16-static) ++ SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) + ELSE(MSVC) + SET_TARGET_PROPERTIES(pcre2-16-static PROPERTIES OUTPUT_NAME pcre2-16) + ENDIF(MSVC) +@@ -829,7 +829,7 @@ IF(PCRE2_BUILD_PCRE2_32) + SET(targets ${targets} pcre2-32-static) + + IF(MSVC) +- SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32-static) ++ SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) + ELSE(MSVC) + SET_TARGET_PROPERTIES(pcre2-32-static PROPERTIES OUTPUT_NAME pcre2-32) + ENDIF(MSVC) diff --git a/ports/pcre2/pcre2-10.35_fix-uwp.patch b/ports/pcre2/pcre2-10.35_fix-uwp.patch new file mode 100644 index 000000000000..476dde0f6a4c --- /dev/null +++ b/ports/pcre2/pcre2-10.35_fix-uwp.patch @@ -0,0 +1,10 @@ +--- a/CMakeLists.txt 2020-05-09 16:43:10.000000000 +0200 ++++ b/CMakeLists.txt 2020-06-03 20:57:17.026182500 +0200 +@@ -619,6 +619,7 @@ + + IF(MSVC) + ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS) ++ add_compile_options(/wd4146) + ENDIF(MSVC) + + SET(CMAKE_INCLUDE_CURRENT_DIR 1) diff --git a/ports/pcre2/portfile.cmake b/ports/pcre2/portfile.cmake new file mode 100644 index 000000000000..5d6c5c39b6a6 --- /dev/null +++ b/ports/pcre2/portfile.cmake @@ -0,0 +1,73 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO PCRE2Project/pcre2 + REF "pcre2-${VERSION}" + SHA512 3d0ee66e23809d3da2fe2bf4ed6e20b0fb96c293a91668935f6319e8d02e480eeef33da01e08a7436a18a1a85a116d83186b953520f394c866aad3cea73c7f5c + HEAD_REF master + PATCHES + pcre2-10.35_fix-uwp.patch + no-static-suffix.patch + fix-cmake.patch +) + +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "static" BUILD_STATIC) +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" INSTALL_PDB) +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "static" BUILD_STATIC_CRT) + +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + jit PCRE2_SUPPORT_JIT +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + ${FEATURE_OPTIONS} + -DBUILD_STATIC_LIBS=${BUILD_STATIC} + -DPCRE2_STATIC_RUNTIME=${BUILD_STATIC_CRT} + -DPCRE2_BUILD_PCRE2_8=ON + -DPCRE2_BUILD_PCRE2_16=ON + -DPCRE2_BUILD_PCRE2_32=ON + -DPCRE2_SUPPORT_UNICODE=ON + -DPCRE2_BUILD_TESTS=OFF + -DPCRE2_BUILD_PCRE2GREP=OFF + -DCMAKE_DISABLE_FIND_PACKAGE_BZip2=ON + -DCMAKE_DISABLE_FIND_PACKAGE_ZLIB=ON + -DCMAKE_DISABLE_FIND_PACKAGE_Readline=ON + -DCMAKE_DISABLE_FIND_PACKAGE_Editline=ON + -DINSTALL_MSVC_PDB=${INSTALL_PDB} + ) + +vcpkg_cmake_install() +vcpkg_copy_pdbs() + +file(READ "${CURRENT_PACKAGES_DIR}/include/pcre2.h" PCRE2_H) +if(BUILD_STATIC) + string(REPLACE "defined(PCRE2_STATIC)" "1" PCRE2_H "${PCRE2_H}") +else() + string(REPLACE "defined(PCRE2_STATIC)" "0" PCRE2_H "${PCRE2_H}") +endif() +file(WRITE "${CURRENT_PACKAGES_DIR}/include/pcre2.h" "${PCRE2_H}") + +vcpkg_fixup_pkgconfig() +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${PORT}) + +file(REMOVE_RECURSE + "${CURRENT_PACKAGES_DIR}/man" + "${CURRENT_PACKAGES_DIR}/share/doc" + "${CURRENT_PACKAGES_DIR}/debug/include" + "${CURRENT_PACKAGES_DIR}/debug/man" + "${CURRENT_PACKAGES_DIR}/debug/share") + +if(BUILD_STATIC) + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") +elseif(VCPKG_TARGET_IS_WINDOWS) + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/..") + if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/bin/pcre2-config" "${CURRENT_PACKAGES_DIR}" "`dirname $0`/../..") + endif() +endif() + +file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") diff --git a/ports/pcre2/usage b/ports/pcre2/usage new file mode 100644 index 000000000000..a8e97871bd52 --- /dev/null +++ b/ports/pcre2/usage @@ -0,0 +1,6 @@ +The package pcre2 is compatible with built-in CMake targets: + + # Each component imports a target: + # TARGETS: pcre2::8BIT pcre2::16BIT pcre2::32BIT pcre2::POSIX + find_package(pcre2 CONFIG REQUIRED) + target_link_libraries(main PRIVATE pcre2::8BIT pcre2::POSIX) diff --git a/ports/pcre2/vcpkg.json b/ports/pcre2/vcpkg.json new file mode 100644 index 000000000000..3dd6b9455d55 --- /dev/null +++ b/ports/pcre2/vcpkg.json @@ -0,0 +1,39 @@ +{ + "name": "pcre2", + "version": "10.42", + "port-version": 1, + "description": "Regular Expression pattern matching using the same syntax and semantics as Perl 5.", + "homepage": "https://github.com/PCRE2Project/pcre2", + "license": "BSD-3-Clause", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "platform-default-features" + ], + "features": { + "jit": { + "description": "Enable support for Just-In-Time compiling regex matchers", + "supports": "!emscripten" + }, + "platform-default-features": { + "description": "Enable default features", + "dependencies": [ + { + "name": "pcre2", + "features": [ + "jit" + ], + "platform": "!emscripten" + } + ] + } + } +} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 2d59b59cd809..e245c55109fc 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -431,8 +431,6 @@ target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) ############################################################ # provide actions/target for preparation of magic.mgc data for embedding/build -find_package(Magic_EP REQUIRED) - set(MGC_GZIPPED_BIN_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/..") set(MGC_GZIPPED_BIN_OUTPUT_FILE "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}/magic_mgc_gzipped.bin") set(MGC_GZIPPED_BIN_INPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sm/misc") @@ -451,7 +449,7 @@ add_dependencies(TILEDB_CORE_OBJECTS gen_mgc_unarch) add_custom_target( update-embedded-magic-data - COMMAND tdb_gzip_embedded_data ${libmagic_DICTIONARY} ${MGC_GZIPPED_BIN_OUTPUT_FILE} + COMMAND "$" < "${libmagic_DICTIONARY}" "${MGC_GZIPPED_BIN_OUTPUT_FILE}" # need to work in 'local' directory with no prefix paths so no paths are included in archive WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" COMMAND ${CMAKE_COMMAND} -E tar cvj "magic_mgc_gzipped.bin.tar.bz2" "magic_mgc_gzipped.bin" @@ -620,6 +618,7 @@ find_package(LZ4_EP REQUIRED) find_package(Spdlog_EP REQUIRED) find_package(Zlib_EP REQUIRED) find_package(Zstd_EP REQUIRED) +find_package(Magic_EP REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE BZip2::BZip2 diff --git a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc index 23ac215c97a8..6c15ad3d66a4 100644 --- a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc +++ b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc @@ -75,25 +75,14 @@ int main(int argc, char* argv[]) { // output stream. if (argc > 1) { - if (argc != 3) { - printf("Usage: tdb_gzip_embedded_data \n"); - printf( - "If neither nor are specified, they will be stdin " - "and stdout respectively.\n"); - } - infile = fopen(argv[1], "rb"); - if (!infile) { - fprintf(stderr, "Unable to create file %s\n", argv[1]); - exit(-2); - } - outfile = fopen(argv[2], "wb"); + outfile = fopen(argv[1], "w+b"); if (!outfile) { - fprintf(stderr, "Unable to create file %s\n", argv[2]); + fprintf(stderr, "Unable to create file %s\n", argv[1]); exit(-2); } } - tiledb::common::ScopedExecutor onexit1([&]() { fclose(infile); }); - tiledb::common::ScopedExecutor onexit2([&]() { fclose(outfile); }); + auto closefile = [&]() { fclose(outfile); }; + tiledb::common::ScopedExecutor onexit1(closefile); #ifdef _WIN32 // need to be sure in/out are in binay mode, windows default won't be!!! diff --git a/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 b/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 index 9c821f153b8244291ea1a350aac770069f561993..d23e06bc82aca192b974977acb1ae32599d28094 100644 GIT binary patch literal 322448 zcmV(@K-RxPT4*^jL0KkKSs4*;o(1CcUyx7`K!tzg00;mC|D#?|1OQ@=Uu{aFsZU; z=xS2aB(;zd6jxro-sRREwziDvhz9R_+P3Y{-r9F#sZ^HQ?&Fc(+@|$zqq7BSw6AvP zyR+B3-P^e4vJ|N++j{G#tzGN8vE8VuyA5pHx!r20y^n6!QmfXLvn{uK4m|eu#@=_k za+Edic6EZ1Pod$l>T(Vc3E ztca?f#qH}SdFbWs*0?)84c_;hJz!L*D&5|mG+-r0&tAOuy4}*HI-q4PZRwGAumOd)<3gF4&Jc&Qxi($7OF?D$(7gB$W2|XPa%M>e#zA z+b?!?do*3P+hsRefYW=jy7#v(#)5BlUb*K^+uf~iTUiN3Rd!dZ_p+*q*SN~Pwc5-t zcXPXy-JR_9J+-HmIeJjiS~vm1lFFy*=zjw(Z@5-JEN7&i8L;uASZ47q)ev?9Q_7 zx4WS4XM0^a)vZwC$FmjNO?2v-G|;wtdg@(n$EO{ds+H-Qvvkr4JG*lfLv6PAyIcxO zuxe3MRqtKhuvKzrO!VE`i)U`#m3uHLBvm>$Oo$4t4SHv`Pi9+ZcREUmF0-Yp9@wNR z6yDc6-uGtR?>%R|j(52D000000000000000000000005)ddcopWy}I^yHJe*}>$>#owTv$r~%Hruvt?(XiLcL$w9~&?7hx5+hXmu?qTWN^Nj+xx!sz2 z3eLk-+1+<;_1^Q`JIwCgYj9bQO_N*O-JLw_G`oAN1+A}K z9?g5b**)34X@;rx_pE#GbY9wn-R@-b(=FZEmuuc`(9~HdQ%iTe_1CFx?t+%y_V>N- zc}ZTqb6vJvcIfX|-p=;RR$$wmS8lZ0WZk%zrm1_qpk5ow)eg2*;4ej?cTe^dCy+Ya^<}pt#xXlS?(8iz21(UUhVGAz1DWO zd(^ug#@pSIt<7$`ZQGvR#=JEfV!Fp}bDf-b16tQCJ%+p3-u8C)yn9b|?Y9bYsarBG zJ9Ocq`gPd*xSVslr)S>xt?kR+d!&x$aJ8j#d$VV{?r~RZJ#Nco*SoFa+C6tSymu<@ zo!#1Q`_D%bdr{Rowzk^Z<)zu3lIwSO-qW+z-N$l`(eBhc*E)7{Cw1-6?!#WG?7Fn= zw63kAvEALfI?s5^s`k4a_jeADRoezTy5(RC^0B>mILW^Lsb0 zyLki9BWpg z({FCw*S)RHT2+O#z1!2*yPR~_SFUen?=`#ECo|sg=eF~=EzYgNM%Qn9-FG<-t*!3f z&v$LFZtr?DrLnu7TW)%LUE7`8z3n?U+Bm2z3$zL_045-Oi{@24-{`@dE{=+R008~@Y0ukf6_ zGs}FF{-iu@{cfP7uM`?fRz>hDYoe-L{1K?`?_03KA~W|f6Yck4=m$Y-ESnx*Z0+cI2hRw=%E0c!vN*3a+=6iB=`A*e zp5jeuY8%81W*&!Ctxd~zCffq9ByGPKY9-U8mIq?6--^xs)drUv_X7CyNt;v$z+h?` zW3snFnao|$7=sNV!RQVJ2ziyu)lD<1LS8{GxGXY#)0VbL^CLoCX72Rllb5(*_Q6nR zEs%UXOD)JkCxJb~Y1fI^!`$O@u&_};t0=z5hr*IXOOk6lYd!BHf)G~rg)Ks1v&6Zc zodYYbAi0rq8}KcuE5W?;SnqNuj@NE&u;~F|hiM5_Yed`(6Js6AGRC%voJl+yHPRW@ zRT^s_L_lb@Fg4A#2suLc8evnpiB_14zO;B5povj~ngl5zQ=N^dHhg@g$e5}k8{9to zBzVP`5G<)P|evep%WMc(m&CffA<)!T7H_6O{73aO8CkBFAiWD?)kP?M_Z8l8_uoXRT~wv+Q(& z>&iU#wCQmRNJ<6a+_XeA+wiZT_#UT(7i2m8?ei4KI!oILPY*cHsyXIJZBR^1UkpiPB{Cgk6~t<TX=hC(=7|Bm zIbe4dZVQ`MJ>Z=4qoC=Ait#4F&TIQfZunw!iMX{o$Jliz3r`#p>5xbQ|7pSRjL;G6 zgGZv%xvUSWxU{ZX+S@#1wBjw?7Vs#n1^CP-$Dbwb?~HlVst3sot>7u1FhGT~u6FV{ z?~KSo%W2heBCQ+3_F>D0My%A%0t0<$cK%_7cQCFjcsG#q!Gm#Z{Pw*}S(4^2?~{ZY zR@?|d5n51o$neQ*)GVHEojj%r8MjYLeKw&uA-NLRT-)IY)NHd}$A`k1^BVqx+8#(e zSz=BeeWnpKCng*%(JX`gxoJzIE9nTJ?1rJ_CQn`Y9Km)y-4QxN7UX&>d?L%)>1pV+ zE6C8S@I~r1$&26~WNDD;LSVjHcSvyjzVoD537{enU?BkCUQQQUA}mFyBza5bQdd)? zA0?FltMjYhXe{?C)_r5=^YNTu&!Ko7(g-44@JGa9gW0;lITP)hgiUDCcEpPMX^>nN z?RfKCkBCdO7sp-8uPSZ^1rqkPo}TOl$R#s*9b?{en3YI|e792KOi z96duKrV;8}FpPPFB@pB$D`9cyFsTZWw=znuMVMmY>OWokQB9?YR16FZ_$YXw-<<4O zEkUk#nNqW*!F6_a2=2hCbZ6j);s9&~%G32HR?}Doz=vSinvCYyTWZ5uN2_$3+B_zl zGfvkv_U zQ1wAVpKU^L5`HAp-wePjCY^C;{@x@Rf~&?sLA`hth+)#&?yKK5Fq2c$lSmwDJLHBg zy>+b;oXfoNs%Qbx=gkgQE5cGVYz82>Hb83v?J_}#;5PIWax;E{8|uYuSP)jcLX}9M zoOx<;HF>q_VeVFMcicz6nm$)kNDB(ol`?L9aTXnQUk*PKTaIx2EGZz;}i|%VOVUr?4P4lAfdn*EHf$GL_O77)6a`j^hE-YDh6$+DU00@ zD{aK>d~ks#2988R8ccDz-*TIE?7*?}IDMv#zYmsso>a6Esv)=!lR!h?3GD_8(KIi;oIOGNgc}F9uY#^6#4M12|5> z$y@<3_aPbdprGZPvJ+_ox9PUruXg_0#VIFQ?wKk~Vzp%abWG_Nx*(r5S4QOm0em#`TU0^ov@=V(8$ws3&}Q zkcqM=#vdbD(XpJ6V_9-JwZ}2F#31&juO4B!3VvDV-(50^Si>E~j}!JsCJ@QqFH(|4 zfr?n%wu`7b8JHpr@Rm~&+YQl4$wdi;5nYsp85)~?swX^f4Gw3l6pYaj%TXjEwwtNf z)J{hFhQb0UGUQQLjuDr;ykvBvcl4ztLW{47SSmP zBcnz_vfUK7!p=~XkAT(!2!qcbejOGdl#hOtO zAnIdSgD#B@A_3M7SkuZ=mum~we+U(OPmk@y;y9|n0 zs)XqXq;R18xb3~-*yP_m%Zpb@Nj1dTL(1*vBdYeIG>*KU@P!~NL>C>gGl2bZlZLNg z%)ld=Pf6jq60E-AVu=I>rUgtX@qw#K_-w5SkrC))Yh>}$Cba|-48``W#z$gOAqhJz>@K{VHJQ*}^WtH>Xu!GL3wW;zjFYr_xq#-l5R%^W zj`USuDQN3S3>vLwhPa@>sG+RKlaM`H;sn&vVLD#Hu@0?mH!;RBkSkIG0pOmYO?f<= zbHAxUsUx9f94QEA3|=*)4jw@(LDta99R9&$u?^d)9hou)8Ca*Jjjy51vrh)+jnFTU z7l>p=1~3<$lIT%KD%NNtLqIx{6?@H>5CWAtD80 z@yMLWON#5K%B0p+y8)0!XR`6(uo!HnPKsU$UrWft zp$;mT-hBPG2L$O|3+Z>dA}>eDrrPIa&WO1)^eCxVLFH_61Egh*V=)N$)>EAqN!p`!lgX`j2Sr0j0&?^enSxlsvKz)$6 z#buV(+>R&V&ROvw8A#a(NT~>8k362FOU)rnA>oAkDmW*->Ar-Uc$$w2Gmb>m;<>*7 z)meqaXSywv;I8cCj2A%ZAWyu`u6<0Ea%BjqLUV|_PE9r1C3sYpK}tng*8Y_TyJHJY zlPOj-6GfBdpuP*izi=rkSwi|e3FVV(v}jc^6+<4mz8oCW^^DN^a~}z$tLNYi&UF)C z-hhVz>yk|AbLP5J>4ZXjHN0LY+e>oVZ$epgPb+U`J5LWPBCpz+5XpGLQ~ zH|T+8$e8VmXYyf|pCqt?nD-=Qg_5ldeV1@xxP;dsbgXR#^{LGa1A87xwQ`0jFS7XO zVt7hzxI|NnOjJQkI>8;cUMlT0WjQXG{-V3U>A_jmO2t+Zgy&qJaMSP(I{1-vq-qH} zfQ_@0wL>h`k3>saQ*@kZ(Vss1gpg76=o0I6dtEtjJ=*l^WZXvnej2_ zpL!))#M#PJ%ZC>d*Nw?Eopa{(NyL7R)P2* z-IZBpi4K}eu2Z{?r<(+HYhfOQ|ensE%j0p%k?aaN*U39JQvZ7g?6xom@8E z9duGUK?T({&%RB_o%Jl|fb>>0p48(mlt&CSr9w-SbTxdSKvJM3yQjAIi{l>lhP5+r zkg1DGmDenbN(v5Y3RhKMj$u(s3WwYn$X=V`sJ$=fhPfPRy^KdHgD#RDBr?V$b&M`}Zp1nr`@%Rb zlj0y{5tgONh$B*y9w<#_GLqpdes+NLI^Nvq5HGAUM1V`hLWW;h2q&fyJ=l=;Hu6() zl_M?79&;PO5`z)nuW3h#DeGs-zJMcA=ZcNE)A&fFQt4=iTr0-$L8AWx(O1e>D$t~J zk88vWfiB}D)h#%(vJ?%dEDU@LhcAInJ-(9(?`4(R1d~F!t}vS~yQF=3SwQ&7RSmdj zKM3lwT*m={)8>0>oUR``Ra|di+7eqgvxJG<5kkoUPN85ersc>9#^WJWP^wCJ@D`3* zBH*dB080&toVo66MqR0cQ0vzZIN85VC8%1?hn-2evGAy=++?k>J{zFqu+lZ`LMy)| z?JQOku_}>G)|*p<9-L2D5d_Qlek>R5G3t>u?nQ`ZY#wQIUt+O&_M9|?T8%|?Hl2~W zkQ}$SBo8v}GGy)#Ex*^2F+P(#+BG6n1H+4@WKz2vxQ{E1-MUL*~5l z1>a8StaIa1cZt$sy~NQVm_tEgUg(v;-CucPcJ#4r0CvyGewheLXGh^2BWNh&yOyK_ z)6NeFI(f#eVHxcMZF^1Cj3!&J3Q_X7jaiir3Xo^MJj8lbv^mO>XuCfJg*z$5C_Tnn z)KtM~kt)O$hoiWLGd59V6-^q4WKCwES|-~li%%m5B5ZwagWDksb5QI(Dn3-%n@X4| zbENQ~LTSEm!P$a!$QaPhc~2?H&TP5eol05P#$14tjyjtzf;J%opM72p7_xQ)%B{E# zV5afugPl~{EL_kv_h*EEX+8!Qo{a35X_7ZMGkg4lH~n1sM5YIfg7$=9o`V4&FX_4cqdFmS zpqMdhI&f4^L-egww<*Su>quzfA*Dlk1T{=s&KnKgk`G1^?mj(6-qK9E@*4UX82#)^ zk`2RqOIuBvl|AkSQY2mseK5JAuU(_itrg0KVCyH(hG)Y^YryAhA#mI+bo~sXH>GW| zLAq?wPHGz~-wS;-TBj4ue!EC+CbS+Yy$?0*wDM@kKQv{n_on0~6B%vtDcC`XG(u(C zgm`@5>N2>e!b>7sZ4#}P2vLyZEk#RQ;;mv%AR5R_3zVFA-N}6NIHu7C0t|rl zurc{W?}f}HTU6XF8gk{spc9uDBa>9_NTodx4TO8u?1O}j8o`Q=Iv1uAq{zc5aIwsp z8&Z)$1sJ~3^s;Wa%#dEV#ZVsiB?LhQikU4}0z(s_d}2!m4x~(m0)WG=EW?@4WyCcL zTWQb-v3~ai+70GCM6V~k;Jm}4fI(}awdv|>c7kMk5&X2pA|Z zx%B2Tj{*UBc5pijUxiRGdD?CLEs=T6FXg=tH?KuZE}_IOzyw4bo>@`&H=g4iEH&HA z_l{&`K;|^C1ky#zRG$MjS_6S}75} z#Fvk)bBjKC6!VXcV-gO!CMGHj0>K=q({+$uobx(TK#Oev%&a826FPhC7E$T4=IWKuqNS4&1fk9u>qsAP7b z2<8uHNZ#8ALol@2U)@XC6HXEjj~5~W{HXKd-F&N&v8f#-;n?kPAp=A*^+pl!!Hn3( z2dPhbmB^0|j2`vEnhL5kpf8x4#ur@Iyhy6PC$#tE0`%qxg!e8u<@~gqXgO4r-_-o+P^Bz8xIN@@p_Ft|8d7nWrQ9|v7Sp*-r|<8V0MWL5X7H@IwOGNwTiB8AJhK%gqu1o2GM>ftEm z)uA9(Tm-%4OFy3;jC`L&@dVf2yj&aK9H3h<~=OPVwIACZ%yMV_KHf4EObIQ12JuulQ#15~SAm1jaOqLR=l4U$G;v?KHqh!4;< z?V>2PeJ89QS(7X@4${Y=TqCgqqPs(XA)Oss0O2eW*#(4+TJ5yBcvwJ6x?@(F63wN^ zhTvNI>S&ZUi&eadQN0U8-XJ-8JSJoW7tazrrWY&B&Xb3_&j_u$?QhfQ%f4$@MBqPX zaC@xju{vbBedN9HWwfm#X)--3$m4)G+8!|^Lm5XY{RoHcDK)+8Rj6xJP$-n3av0~< zdXse$$EHhDP(Wle_fcpJu9Z9v5EknkM(y_K5fhhL>x=^}@rSLDS1{kMU|FEO99yK~ zt`E28^IT*IG{f0J)UZRBYD#qI5oImx-?iti6x9%e+eL28LubO&+t!n7=s|i@4vCdq zVCm`w2eNfF24juDA5Ka?R67*YQ=1}N zr`njlXjQc${x&-sRFTf9TZNSgV3AAlvbHoft+?{ZuuV$jyCetB69INF&K5mKE%OvYnT#fxB>sHVdD0JRZ_dEe#3MCMP zJ-CCbf+WVEWuWRSkRW&mE}Jnp&i`K0Nb_(qVcryL!fO zuaqw7p;eY>-Vlm}SRrUlh7^@$k_2~WklGAxA49f`aSjeo2~{bW+)DC-kgdO45vPgL zH3=&Q4ZC4k5uy?*o5KS~r9*La7ZkR0}?#(3;OK*+I2SkK^sO-i}2w^0n%{G_ZTe# zDcRqMaaxpa?--|?5#G+YVbDxO4Dmf)e#X3@KUaXjNHpo74CtT>C=$Ldm5Jz0!k-~P zj8H|>*{j@044v}Zhbzn3 zv}0V`4KL0su5Wv1F9zyt#7MXS)F=W_994 z;8ytr3Cl0-yr+zHl%z}rtm{IOht%t{<8;L*J7cCrEY^ATEk{*VyXogx>$FE@Xx;c| zH0^|&9}yR9+w@nxJI}m&Y$fyCxN+!EV-}Jx1ntje0u{~Ay#WDh>bn7#s>Ea-U4zlf zF^cjCc{MJ&MHfuc*t)p(M)r8%xIJ*jAjNH~P-Lm#m3M~dvD5D@aQ#F@WWl7lurCZJ zSH0&IlGWg17@H-iyBZ>4qzjK$W>h6N6=0dFudPt8qMGFF)d$_?&rIM7wAINHv>0)y z56`e6D#l9lL>j3I_W<8O_F;sZ2Mh3mkr@GkTA{Gs#LN)u$Fjpk0$jN}w%@ng$3o%fFz2Ba z0_gdVFo~>|O4PV21>V%aYHc?ptP!qP&6(0-1??6~Ydd7;%In3V<#mA?(L1$=4>S=3 zxYGj)a8!VYUV`k1_(H zn|bVvseqcQq}` zKw-0$8VZSVv6d=$HQcJ|*AkfWc20}hJeW>aK*QvJwEz*1^l+il`psy&#I z@Ph~_8E`Oy+{Y}qgcyX{EqI``2TUn=_}-HKo@k=!Z70*&p(lqH=R%szAQ3$hZnQbe zciRoc*0l<7=&Ks&+o7E(BC_L{+;@oz^ipQ319?Wab;wYp0%Kxa4^J1F+Ypc@52|;& zT0mGR3n1hPPiH%~PbaUTe87tochSHqCL*;l zK%DuCghU7d%{<3#_bo?UN}wD9CzlB?BioNqZF7N|fSGQh>}7&t_mdt0Hbd=o9`96t9!?d z>VOu2S@h3}hK1PFV3Ozr7F#YC1kG$DOf@;t-Sp_)t~D->>52>5T`8|o>^HS$)UpQ5<06{TFPgBUFp(>9hVR9r=1 zxPn2Ov|0F3Jf7&;hVf*@k$CW$FM0ehCCr+&dL5TsIe}e@w-KR4z_B-=duCq5&aptC zmdNGmVV@eVa7)(t~coV3}WND+FR*Wf>pN{cZtuL%d+3O*h6jF$w8;au9m z(E2`bHPz72rY=0;H3%Zutc~)J&5$7guKFJJmg=;HY_Eo4@%hWHg7MnoFNl7eV|tYA zQ(DxH0-ZH7^(aWflgd3KWVuKb(feo1!6nHO7;_j^x}(_`T*VM&S{G?{J@p*6z7m40 z2#$0%S6cBVv(^>PTQ%f{xI7OFpDC_M=HS^|-mQ?UAeYpjbh*LcP;_B1skwxR6RY09 z$EE6NHY$Y$v?WHdB+!n^mU8Oa=qFFhKBu5FoT`-B%?L>+$L&foU6;y14g&&uqVS&w zZ(TUEcs)W2MyQ;GQ~1TIa6sg<%-uT@11#JxUfDiUztU%_aec#mD?!v{r*R<5bmVX5 zBr`l383Q+aS*hLILNd58lAyd;WDSZ%svB-rlUFcU#_FKbuBWml9!_gV_TCT^bt_IQ z>$w=J508+GKP`uS91cZn2G-|lPaS9knv@hCrz z&vT>-Qdx#a=)@3c>DBOFojs#GKMJ7j_Uh36eDV3XO#+KTxzuBk+J!A9S^%rghDJfw z*-o=mx<|C-YYrGo-6n9csD;qPz0oL=ZlHwPnM%`CycQw0tIdV>sZfrFr7ppXJ(ke~ zS0{DD;)MKTdLX93LY7T*;hwLWwwk&wSLawrv|dLM(})v^m=h;W+EA!!T``KLc6QV) zQN0|3`~wc%q#JM|LDJ2{)!fxO9%HWQf$8C2Xo0!qO}=%J(Vte6bWM_RzPdMHX6%$~ zC0$JVxKk3Kkdc6*?r5eC%~L_Ri^4}KTm;fsxQcA=UfBt zv|Tk>1@8sXh&taClv); z@&$Tna;oFkJ7^!6A_2Rj^>xw_i`b@9QJk3uyfQsYbDFe576mK$cB`Z&Ms`eE&dll8 zm2@!?cusH1=b1$2lA%F0U?*3~gBLkI2(6v8O@2~y>xl`any*}zU7Fabw3GaA>Af$$ zp20I7Y=xUZC7%J&`Nin_B{~v^D_INO*;HN5%a%e;Rw5ouflhuvpzY-Lu9fK)W#oB$ z=U`kpFQHD$9$@R`vEBuRJ;zlvKJX^Tw*ZDmYr1nl)`P-YU1C&lmfb{Bj<6x-fn8ez zvAu4M+VW6*@(3fNr;!F_p6DT#Wf=~#Dq^GUG)q~&aNEX|84>gN5tQ-OX4MKAIn#^c zuqwE!;Q}sZ;p`2+xL{1PZjM-Uo2;fr!%$nm_kP9~bG@ZH^#@^XOq(gRgI8hzxx_o( zF*4YBaa=K_1YWO5aN8QgKAr0tw~|^_mRk0h4SD-l*gJl%?o%GXj}wX_%PX`@jlN@4 zW5*7W%-j-zSPu`ywZj;=Evon!WVgDLDlrMH~#JmE$pn zma@d$F=AiSw+0a?g`);kT2N*kz;+~?z2;NUIiVzUS9ib*hp zs4$ZJi3X-ry<^M|ZrK>?8CZa56#NYHShRFE zW+yfz>=JMz9=Behdl920Y()_}LRe&LwUwUWlpDobv*e z2Ny18xLDY3c!SkJ$7no`B`;0ONp3`yTJ;*ZE!$i;<9y{pu6rYezD41j`3huNM1Nfc z+z6eHXE?&^dQ9YnTYl742Q_#O zxipVWl92eljv;;K-oEDE*11vVet;6sHJj+r+oQ^sSDO|7tL;A9RAh*)SJndK-NjSW zeCOoRjeexBKez{W>`87 zM|*Z&-Ypd$i^AC1V(~|)`bg~rl?+AeIDtHV(Au@wXgc8m9;!V1>Qp2WgVP4cBzBn}UQkMrcuSu6l+_ z(VG$g;MbBom(soB^_a$8Q`VnTmefwjMo})k=5P}O*sY$#IzfH2$0b`la+rXb$>wc; zHF7t0RLB~NolXR%dqQGshV)+SuXMM9KF!W$VbmU=yvdpAg1|`_PA=IH6mv3|`qz)U z()}3?Tq^TCf})tARY||Tq&HM(ARmId=gf?{3Jc>{cw3&*3E{t(b6PcEAzE7DVkA(& zd0Mng`b>;A&~3rY`RuB-uV*hy0?eha;pA~M%<#>;Rx-94)go>wrS3}E+fZwfJCBm^ zK}~$P+3v~C9-npBqOY^Z$&WfNxHSgWgB+#qv9f?^-5zc~2$thpKFW3hJ(ieUlgEU3 zZNiur=nz-}nm7|XQCzm15Wqyfg$wxk9V6Co1J*-;!#S~roQG;cC+|uev){Ox+b?zt zLdT?1n*{Q%COtG{)7@R-5L}ch8zz)Gx>;W|69G6(edf6)1ft_*yavYuf>jP_ZTGhi z)N!s6$h~|Wwy6W*;7s}DHzD*Tu_gP7ws?CkBNvd;-4_MLiN#@)5YdVg>=l?r?9Mu}NNO*1 zkWD6?UKAG)TJJVaUa4)&+FgeM#gri@#OB>^Nh?L|KFOERL~mk_^-%G|cR#CwFf)(B zjW)pb!+c=46qKjP{sx%u3hP&&yAFXLYX_QaLn?Idn@wk}AS29Jumc>*1#R3f<80Fy zd<7JFZTH(mLU?A4R-<5Jx=DU@DHY43y4rJ5{n(1Bq4CUgBjmllNW<0{M1dUSj;wB3 z+5!|duF!Szij@8FK zhq$U*(S;V|bhzXQ>U%*>`T_4Ai9Bsnv5|xZ9J55p?gSMC0`w~izNGXUJF&>22wdU? z>d2a#+KBrB^&u>cd>0b?<}o@5Ec8CmBk+@yO3idPVsZ^jeEYUnohtds?mVlty$6hs zL1Gm2V+hl6d3^2BJv#6Z(zQHbXYpyXyl$hm0j*PA=?`Aa^ zXdFQoMvV&!zNum0rat<9tbGpph&SFUH9y4s;7&eY`@QsM% z@?)8!^^Zb)sLzAQDQh5tM#**(7CDM|amD6_d{RQsQo4rJywHsw2up8QrbtD$gPjik zjSr;jZr>P%t$%9FBS404S7poV+NU3mLEfd1b%Muy9{6BMZ4CLfGE(1wJuDmkA!LV5In)_cQ20Z5`xHcroZ0zn|QF<1rdNjXdwe#O{L6BnVJOjsJ3=^ge3iXmm-8?)N2b` ziXde4F5dAVv=ea1kg9hXeFcarhBqMxzMfyr;hcPBB)0JU@bO#>{G-EbZe8O$;?SFX z2Hh-aLXIw;kvo4_UmtkI*<|FZrXA{hD~MY=Z+p2^H=0e9%#il;&wBY3HPbszFsp5s7qErqd>T^J>dYuL|Y;O*mk(=vQ zfl8eE4^XaO8y+q+uGeEVy}`)HJ}heo`t@iqHuk}WtELis&?mG|67Id!8g;pbw0n4D z8hd)a58|Hq*z$Zrc0+Iu&UtovsMc*@=Ep|HXfV+ICU1P~?Eb_6$Y`^KyFS~KoT(*u zhtdi(u4wILj!qVeS1`t>o+$M4Xh;(E(Lv!mb+fWMuqsVk^F+vLM0t&qwzvoxy?yRvDFc#nd3CFzOc>nc*!i1u+pBuqcyWn3hMsjuM-mT3Z4X8wOT|j#@XuoBh~22 zd5vD8aNCjx9Y>3bR|c|ZP#gA|DYN6vltQPO3nX%JdspaaATYPSDDW0_UDuw{Al$%_ zDRa|<*<-=FvQV$n8`MdqF>jBH{FN=D%!22*u1WKj874Llor@0 z-gOZfD93A+zj{b(8%fma7AIRQyZUu8bp$PSkwHu~wS>b?29{o(BUh#Z&3Y3}%a65C9vB>ie?4^?la84v#I21`o}&JG zf0S}3-7#a*IQ_J@sDSnfqM^69P8F7ZdiM(WYtp-*MXOv3J{L_10q4k@vMPd_E+ixr zt6h;1$dL0npA&DO979l{gx6NLX*l#4NmkLN&V!t-Tc}}FbI|M5gqx{nu+V{ee7pHD z%3j1*4Yvm8gP6=+OE{M+m{%BS_PN2|#Gzsu)~`|zm1pms>L41^CG0$hPct5dSAJ4v zLT+|b;)LpnBfF2N66BYc=pju|?X^e^FAPgASmlf!>`mVa%8Sj=T6#px=V=bBpc`p` z*b!hsiUtL(+}&ZD3F@wd?_tMqK2I~ct1BvcnF>`k_X#?eD13|G(YDPEz6oBdi!A{R z9QgIC%OYEIr)k{j$C$=Ur)WiyS8LVlXj9g|hNCX_xDd6N)I!scE!3gaMHXsQ@%S*aw3-z5eLeyVWXI{LrA_?a3&5z9vRouImt0}n;p%- zI{W7?G=o6(SC^(rG>E0I%i_j%I&+8v5kqH9Okng;KU&&9ydljydD8N3rGWuEh!J7b z#Ed8EJ|~LFd+lhk#$Zr`6NMt=0VQv3cq+$+@f#2^44jVnyT}PsMpH5v6euXkBYq z*`vmRZ33_*NQPa?EnqPRLK1uob={UdYy)dUC0>~91vBWC2F+IWK|6J~q-@{Yao8E) z+ugDSosQQkaO!49S}bw$j^65#7hyeE~O zeXT|A8p%{E-X7y%9X-vFAj2CHI3V!E_M=H5nViLAo`e?alC`qp*l5X6ptkcx8ZxdZ zD{`|T!e_0J!3u|I9*=r#n3AGbf-&Amsz$-mnb~>$QU*fS?oT3SjMm^oxwho5w&j7} z6yRlu7pP@u%#M`h)Gw!AA$O)Qq(-8KYK@g`oZM z=#ZP=O$s@Icp-a~SWJ3Up+TEt9tzx2P^EkC7W(FO;sAHg!V^bxG>n*d*fRq|)xyp) z4^M|2%3_)Ks*{bL!OQX%m zGR}L^y0EDo)aJaFO>ZF<^>d4Y>ttuy;n3^2D1NzV!~2VSZR(SLY}k&5!bsD%5X zJvSvy6S-?-81^M++&*l_pcAOG>$0-(@sSlL79FqBm!e78dqE9=ubVl+YTz@VCaaFT zJ+k+mC~I&*uXM8YGq?DImdvuacBt0C>9#L9^O+o22X%862g7IlFtti)Q_T`mSZnQiCSloSE8)S0pcG)VoSo+A66i4Gy@)R(6f_E*BY=EO~ zbv)E07#ua}8+8YNe4Bt`x5kqpwbKS}@yB5ImXI%Pw~Ib)(o<-zE>a-%h4@bAG|X+- zORyQG&LFVD0bNs$>H@&t%;*}io*bXAJbj*O5yl0p@LB~ZUpBQV_fi?lh-F4_^o=CL!3Ut8&5_URGnR=NJnahZ>3D;K!jaZm0&ivXYmOvWZ80 zwGiQEMSB6)p&9ESRj1kNyz+7N|5CPn)qKLHmswju3LOAn#-ZNGV2Y#2S^{bpL4<|l z47eyZia4$vL|~TuekG8*gzdS;u8;3 z?$?5HUbyPaT9!PNQROqE+)ndV)?lP?0!c{YLg**e_C4Qg>2OjNJ8PK-b{9T_@W)`n z>$*?43QwTc`Q7Q1>4cUPh*2v^Udo6`BWsHmvR>e0E-tVZpyx})B%-Lu0f6k`=SNT+fSxWK@kyIPsN z*yq!@qwRGPYFfDPmirEn7YhB^mM5Q1){Z&FyW<;+)PyG9p7+^G;MlCQr1`CIT@z=a z#18q=a5g46yOTx9>ON6;4;2ovtgW03rK@bKl=6Xv2r!U=17!32vz29ixmk?9Wd|1s z;s#Bdbm_wJA=id@_6&m%ZhjR*4#G>ZgAJu@<33q4G%PS{Dw(@duvsrJGN^_S7VrtU zGj(zZFbtPg#5hjNrI%zqvn6IpeK#CV!{#LgR}a9K+@C}efejKCpxT-VL1Tvj3b%ky2n`zS|%9IS!JK$$wy{e4k5 z6@)&uH>$A}vZo5W+>0+hgosZ*4?y$Sj^XZ;>Z&>lP*Hn@eSG3{^nh zWZI`qBY9;IhmTjDZ90Nvzd<2S=49I zdJ@OSn{6h}Ud>*dH=@yZ1wOja>k=NDNwH0-I7gNZ?-gU6T~4dB=3A`na++W@`w8G2 zDtJ*3l6}zjcMG~WI%K5?mQ7DLeR7OtLB69B;E+yo-NlgL7pM}ptYgXI4z$v;t;3;` zexug3jta)E&AW z=xBjdPMN*%8el;^p4*6+4UQDe8jJV^*{8bos6~i@c=~{%HUQJ?A(R|+&TMZKP1LWw zqEuf;KHbLEDTHRWFcn84p7>ZfJ><9bzQu^Nvgc`FkVgqNsnE6^0Z?f92-%wnQ}1lN zP(87pjK;A_`rv?q!8gG15^hU07YHc@&C9dio%BuCm{=lnss+BoyFMJ6wanhqYc`i7 zpS>vE;$!ZexDO7?F%j6)@bJJyW)%!>k7i3bwi?!0`?>6!xX(2~7|e#v7&tO3ZMMRF z?ki&KyIJmKYFkdc@X)x?c2mM^lKx`SeE_$JGO6?ft=Hqfh4^zBZ8}uFi*&v3e7|RcKZ{??TfrBM#$+-Be|S z&$*Di#SO5?u{{kby%M^*1RJk3IVj4VyFjh5K(x~A1cW657`O?9iM zYE&pdHE7gtE5#&3n%!zo8uzKD7*5_e<;RPnn6A{BfXbIjnvSq-daz>95>;84nW>qP zJ*E?b9F`Y0aUL}`$82U+G|t^$F&&*JBXP$UhEptN!Z?-dBME}g*bzcUdxH}?YL;cd zn#+>b$QFwbaS65grPv&sU)GU3YHR97R?A{C9vlu7e3&l65F}Ov?d4<1Hwr~KD7o89 zZSv_WsF`)2bv4GNDvV>;;kE;FA4x^NHk@eJLGJ8kxOGzYsnu@8w^drSjRqKOZlJ3Y z6moNNAA~uBFssJsJ~Nxsj~xjU2EP{sEKB`By#`5AK>`Xq;+HYp@s5W#E0+kV+%-j$ z4W<$4mp;)l0;fna_&tmmW27Tn9OgfHcHcRYk-5=-3)fA`3E|fPJx6v6>Pg>HT;Vsc zV0ay?Uh$^bBtqL15g!|#%*}7a1{Yp2=iPR>R-Mttfa<<{xPMQc)a9wrcCQ0XcT2o} zvWDa3z??4gc3N0o#b`Db-P(H{{CGm(b;0Qh;a{AE_^zj;U!2JELZV=@(uI>rxd>|c8uFfJHAmqa@l8q+BN5xVuJN|1BQoc(+H$vCiN`g< z-e5$E8y1ky25w9l)tHskc3t*^hn2C*xg<#xH<7h>UshgJ880if5Qa(}nmMk(NM2}AR*IJb z5-Z#%)CSeBsjNm2Sbbfx#~AW%EF~nEcHS(y9Rzpb=*AiQ1|(!zO?ZBed@3D|E-H7M zhVFcIL>R`Uv1#3ag7wzF(3Y1e8>zNFBx3ba+-Hsaj);mb_^<<{>Fc zV&Bel?M}4RETP`iZlcSZrz_Y37LAnOj@TL0#KelCfWGncg89 zhK-6HK;j-GbFQ3-I-C&giMJJZ5H1yj1z7Q$OPG7+E9!4hS*w&l7$$FFu=B1Q-5bzIVF85lpu0rSm2(*H99b9!gbH~z@_2!x2+3VZE}Cmo z9{id^ZIbs=#u_@M-4Hzph&GN)U{{w@d!sx&fsUI9l=u;BQVPC-@>)ZXW@HyD?!gLZns#Sj;!07(Dt&uD@DqJ6aGQzb%Eo}rx?z!|7z+ERzBV`$1aee?;Oo8h29#Zf?r)eRIb+0# z>NemeSBMzmItaZ{pz~+XXyA;jWTP>K%7VKQOlpg>bOzWDP~jP2lC_~(WDr$% zIkJ~j+|*M3oLKPpGxv|15_ql6QEn$Dq!bHodlcAwTYoHhQ-qT**`%b{MeSf zYS^VNC%5X?DsJ>L&S5`(QzHsHHB?B_`)oL z*fPW}w4ouo({Gg|W33s5Y>F_c0)Sw|yy<>mKXi1bMjMOc^O6>@lC#5DYHjXkgh-#S zhkm0Gt`|JB#Br@w9$;N`-zn6_wCp8@y=y3|j|?)Pmz)*`5;fw$Bk)qH4{ry1t$fGM zd=yV~i5=CPoQZrl+|hRA(#0pv{iA5i8kcY!X?D?YJ~SOo;;D+CCA{2H;StI-Bqx{Z zzBfZ=9(iSR9hvR|?FM9Jqj1z-7W~gO<9&}l?)--Qca#b4u$o!&zcK)blTmXAHn_U@ zR6yIek}hu|w}R>w)vYrQ4r`W(tOUA4lEN;7aL*;86>dzfo5TTB)qHrUASGqU6^3De zD=&EP^sBTH<|wk09PmdRKWN$>?;TNASGhJg&1`hDyy`Dq!YxhYkT}Rq^PX6WmaYS^Ieg;vZvnmKpKDiK z(;No(O~TiS12>lxm|pzv`7!A$Hf-efEsxjet)=m%0^K?6*xrSErHe@ep!R}Z{02n& z0>Wv57%}5$zY~(!!pzI>w^9al&ib4&mq)W9X{9XD3Uu3LjUZ=|3NX`0DsBd+Mso?c z7Q#4@#yXtEH!VnZ8Tp2DvZ-@w*fr3=_7{caCfo)20P*23KJ^%Zk1YhL3tuQ>vZ#vj zZng1Ad#vdBY^^YApxeU0e6IN48W@-(JXNP=PJ_|4ff z6ig7jJoS^dX>I2|i`@l|6oUqu9kwxGm8FCuHd^kS6YJjt!;zuK80Kfy4PuY(xSxep zs)~Y-3|SlPbLU}QM0OR)gC8t;H5p1dS1d^4X8q+LwtgcC9>kvld4g8ikS?*iixId_W4%;K%xZ-i01m?7HDNG7p}X$WwN5-Rpo2rEL? zyu4Ed={OVbR7lGzC%;foaHTYWc9Eah_$=%)fG?I-jiIvwGbrGrAvWP<=C&YNW*46F zUTp!n;L^~H z2Zg65!JW+qa%a(ZW`n?0d_H-eOL>Jr9w>hKP1rXMM{i`oSg@MOhndHd)LBF#@;yIX zW5dYMO~HzILypjcAn(cEfKGTs*}Z|=h+ZYQ#tt*M#$Flb;%*rR#<$NCdD3FIG%v)8 zy0yu33YsCdctN@Z6p;BnA47?crBp1h9-AtxH`j2BYR7G-A|_TGD#bAY+C^MiF`kS; z!x%ZAKPT7cLKTNDqnK(K@5>`d)pB&<}gJHppuSJJZ=L}wz<~>;GvDnHMM6B${pvY z1V2lWt6>}2Q;P}{6OfITCVy#Ky1-NQ$b{hFJtS_FBg7QyO8E^^&csBxf(oqw#5=3O zx=n%=muy6Z_C{}eDM)lBHD7KMM{{d_H`W0u{e9y5hEARspgj4I5Ouh@S}$?x`s>xy zm~Ha8Yvf64iHFreUFcOLj52DVb}Je}5XDY2J1Z5$P6aV9vHCQH2I*tr`W$u6LXOL; znK~l|B)81PG>1N&`#8v@O4xQ$h z;5;zw!$6vj%cX|)#PE?pWLq-Y-`nMM0}`Z#c!=e>#0o`bXqymW`Tl5eF82B8?}V&Z z0YvHH1QhAOgsy|(N*(PfaH!Hq$XlpHJ-Xc0-gyN&XbVw?>YHCtaGKm7UZqhFs!{sGxRw~}oyUr7YcJ3S z!;Tz+G+K!0r{IBDyM(sEe-?FFccI$^1n73sB%IUQA65cZf@X#cT{W#_n)|20>)Z}Th>?0{}Ci}cTiR45Y#0uY~DYDE4CQb1&(`}d!Ej)az z6cfm3E_3R6J+WJMj*^-%noo(!R@{MHs4q=ant^ z_lbBLEbF%*%FBn+;p!}~*ai8~dWaf*cTPhiOr)iTSa5+2!GR;V=*Jr-XbY>P3ZuZI zO2*i$7Z;GqM%Ibi=XR$+H4y+gNvYM%QCl!u7tXbrXEf2B*{XK)aikNV9MrQZ7D_l+2YSLpSLy>IMu_V4lTX9N-i^K0~yMoZ*JS(F)$q?wJ9^ zx$sXp!q_;6v=H$Qxa_$-rJZcdnpyTFskP(H1Cp!8e<>X)RBq%Bf&^*m-8nCkbii{O zAO&{t!tYi%#7)7s+`t;^=pw_9g0O#|EF5fxYHEYtn#$cw-xg|7Zo=qZ2Y3PthZXq?M@u^*|2|9P7+j(xY!=zIy$**^Oig0V) zl+M1?0}AxQjIeJ^ZOzMzM(*{BtOL-rTRQN16|i0g2hba`?0GAh9k+)g%i=}&Y&in! zuI9=#_~;uE9LBKOX+65n4$Z_`&GKIC4Dr>J-si`@Y7?h#72zQ;b-n7PvG7t6n0!N-AUu3AXKZH(#EF-8bB_dV#Hcf~r{ylK zlv4KIxSFqKO&eL8-K5`$9dR{Z7g-s#le2Y7Uoybe^+5VxeSZ-6+0<`+!B?rmDQZ!~ z&|e#G7Bd_Um{@nPpVslJ6-EHMbR@D5Y7Z%0-Gkz>4w<=;cokSBF}p}$y=Y4kf+t#r zu%9!?Kn+wDib&wg#i(~Zyh}1T{j^e|hL}qexqIkS+H)R^ZUc=zkuNZiN>%165v0jp z6)_2#!Kf{#qap|bpA;(KG%qEhcQ`2I91$==1w=!l1G$oDhl0d>Y#GHwygY|Njxyfb z1<6*6z7qwMq0*7CDZ?vs!y7sKt=9V%X2AsRfY;E$g9YK$48DLU z1VLs2=oSbuM?KP9dJEl3 zx#snngrV4Ta2mF>%+H!AZ{zY3j~0b6@CxZgj6&Z~yDI!f$76;K>w@5$-P|B>hq)zb z`Wd{W`j{S3#yFl3_Z>=|wv#`T6tuSnHpO-jv{qIG`&LBmr=#qehZ8aKBs z9=Pf0xF|hRGT>}qaqzr-y&iui(R`Y%Sx5TS#4)f&)%5a7M8tOF*F*?JZ4z&dNT@;Z zOm^Roy(~=VfgMZp9nf2ni(Y~v8VTgDGWDm9)((s{&QzkGgEenp2|%J7v6*y>j}vU5 zt{iORa7b7VR70)B7NMZB{x$W;^7+P!tTWIOouTYivyf!g2wt=haZsGoF2XKE3f4Is zfFKZ{^>p5@X4Gy^t^AL=8U3MbpefU4_o03&QDoBs)0LTi!~8x5#fmd4sg~(FcjP zp9`Q?N2R+DV3~dG)b|?M#9qFMVf^8a9uBVNx?Ck_NKldN4b|T&X`i2_*Lx?;#z}L` zJ;&vF(t09akKr0XVjtF_5Rf9@*!^NRBxP(DlhzT~v}w=i4<I5GH$pF~}Y$E#KZu$FM(h5>~ zd$XMaK#x;9q_@;cI|w``bP`uJD=>>!&Fh;29eBW;aq|3+NV-ir>3V>~h!x|w3A%0M zg~-ruvYwB7p;_{5y=hLPb-ctO$G&!7Xxo!o?N4^Y!+S$*`P7|bs~ZmtGn{kdS0E}K z>@F}wY&95zF%gO&-J91SORVZ9BcQ46N-YI?rONww=x4>ou)}5anbR17_!(E z0*t3l0t?G>;f%G2HdT%(#*kNQ_0d^k5T7d?PHNj!k7C&PnjGd=5f&GP0^%rui)Mbi zN*0_&Pf_9LpNq+ZEG=FcOlHnlgluje^n6Gj`x@@+t*1N?hwfo{?-zL=86%rSLe%kX zG!oZVhGC1e&8;DFB>Q#WV6%bs+yV?7g`o+%ziKXV^zRro`is$Zb@Y@ zy7su=BB^l{L!?$ovw>uiM9%iDfsiwZV5NG5SOZC>~9&Oh5#^ul3b(cNF*!2F-WKBjEakfKHpc%{Vq|%a`{wAm_OuX0uDU zf?SO~?wC|A(ge}P`T{63&+jG6z1)l3@Fs~o<1`84<7ZO^(pJ^Se;vIsVP}?Z-T3>9 zV$906tMS~Tf%bzHAV;l&vJ|4Z*iCOPS<9NV2L=PzDo89Nt=e%tK9E+(fF^?>(%&XT zC3x-Dm*rvLDL}A~cU5XJu)K4|{Ld$l1${A?hT+l{W+8Y@)0G5*V@EJ3F&koyV3wETecVw9c;8gHW!R11`6cJNve->l(cdebKmVsiYl8L^?)# zh0pD)aEq#>APcslQ%6ogGQ_t63|1~Wz}PcVi(|wC?Kgau;x<1`*u-4$JFgz#Ix#r7 z>mqB_uTjHbWskIiR-gzOflXx`2Z7xx8Ow*_B{}BO1en_wjR7p2b`*?gdyxnNq4GQ< zVS_8#c{N1Sq!Q1CaB<>S&IC&TN-c%7rb0hmp zy>Z=fH4DChph?ME@fCJln9;ASK?@6w$KE2DlXR zD8v)t99D0cc&B^mxj;>r)xGwKx><`k24;*v&}blK`AF{fIcv8wYA2~T-g&;dLeq55 zb33-PORtaSfT$?x`(i91ZY#&zTJujoOo{s4Py&*vB9hjI5mu1&R zfuSXd0zIV8kry#0k{%5c@Q7NJVCcDvzh=wU13E0q109Q4dmL+!STvBmhGx@A*dkLC zR1I@>zBf`4Dw^kQG6K+Fi53GiE!a#Oa6W$~9`&`o?CRV7m}kIB;=z$F}tKuaS^O#;+|bxg+PWSiWZ2%-xpVZJiFxt};0 zZD_54G57NxrE=3 zB3UF0c7bcVV|*LGGjwwmhk+xLlM!=LGg${?(H{2f4ao`e>F~L}((A7Wj*WJnBV^ex3FK=*)z?0u-c-TO6gC`yev>*=*M4s~cnH!?HGV#RKSfXl2% zaUU<&gAKTFkF-3G53XN;957%nfp|xdc zxO6~BaXssQpcoV>Wld!yfHu%+M_VWO4dc3l_MPc#R#pbBFjSEG<^Tjtdc0kZHA8Cka`rb_`G zFo5-(gZnLKjH9}3k0gyF5U^!h2tD56>jrv4x`4$Va&kcvYQ>ky;U3lkJMKnx;N>e% zH4t5~hPvKn+!<=FN{iLm?lUFlj8*2a%eCfZ5_w7{5X!k$sdZ}DXnq2=J*C~a2!OOg^=(BV!_w7s z?pj&!XbRHpU}5HvDdXM8FQAaVqB(Fi9lGd!JL@&@rbR@xC;=pi4oS~=rQd#-$%b1d zx6V$f0HlCSnNCI+gjF8WtEA-5 zofV)CF7Hb55V|}AHSH5+D;lK|Y{v7U79}0M-hm}b7IwL~>|^9+O1yt2wvJW)_GsNJiP zCSb1${d~|jeD@$Ne#*zwLgcKj@`R6@P+uH@T^K+t_^raY?PJBSgm5u7{wvrftQjO{cP=j#b;_ zKVQ1@ZHXEITCh|YW@icAh!9-Oi*v#g**nda{sbl+a-{n-$ZJKSl-LEPE|w zXJ5JCAo&Tia^y7l+E1Az;BNrU3L@cYK&0U2&J|D=CQ!1)mtthq>oMFIN_zVgG+;?P zp3!RNZ`tUi^RWF~i*)dSuVGS$P%8It3MoWn>M`aO@pY87q&|2shV5xrWtz;PeCh~g zY@Xji7V$)?ogvq?tQM5%1E!khJ3-jcAfB~A!gVP7Dm_mxdRczbmm2;&ThrMTp z14U_gylCYDr=FWx#~`K6gHxi@);y}tDG|gXPnigHTfRBcp>G8&jxHQtJK1T^!BmD~3cP%WG6N*jgs1>3<|NMb6o z5T0Ntkd=-2`26ggIM20i%f4-DvM1v6>+x<2aXV%a6n zTycmU6AJgEhvw2*0s)e^C}OImHJ{BOJok^rMKbbT)rqra)^{KS3E=`3 zRmusPrXJ;FcytRWCJjeZ?<&sRE>LL(_ZlYa`z5EVsbu$Epv*_c91sFh(mS#RsXQ3L z#?4E%=xd?xqqE(aZhFxjJ|W%(dz%Gul)|`fg=Og+g%eGi6$S%v-;ydC`W(Lf#RHw= zL}r$B!w61ilLWR_)O*~?9#PxFYZfnWitAaaRoA%vUi4qHi{sdR7V?9_5Vwp!Mx35z+!rqBqaU}zeD&BuXuS(dsCCtQ*nlxrQgaZ^8nI4kJ!Vc@7s27ksdyj() zV{gu<&j@Z&Grn(Dk}^)Zhq*2{Qm2?YVe>naxwq58LVc)x(N2MT0py{5sY>k#L)XP> zj+odS1^+xQKPW-I?d3zbM0TGUyB+HR>-wUCr3PWWVYIl*i}PfgmG>fUy?)PSg?^%! z&LBMYa7VhLp^j9jjO;o)JT2z|&wyF+cDGm?>0220Ysm4VM-mfZ!kD?DTo(8jIc2H@ z#>_@G{R)i3Mbg${wZ^Wh?SUelYNP2{6{QN|0bW7Vy7z|RY39~`F&8AU5%d-`ZLc&% zxXhG=o2L01bt9GzY%mVB$g~Y*n#t%)OmGUt9@f^cn5YdnF+DTEU)DnB)I+7jI$N&sBA`K;mDuOY40oQJ6=o0Fap zu4f_ye0BhWI9hHkEp;VM^(1K)7M%;2C+*xOFg~SsZ-#=6UXv*7-4b6(#{kooq{q_` zkgOx$N*Ccuwxw^XibjZArZ9ExUmu!{H)Y2u8jjY=Lv~33cuSTG%WEu_*ec5NJb<;u z;vue|bx8n7&}?!h35MuaSbE))rH?Ks+r96Y~uT(^DCf(z7#`T^Mv5o5x_qV|}+a5>v3>hX?q%6Iu z5?hW2^ZQQa(z^E=EHbL55X^yx0r;gyGca*dsb4mP99SxPSRWmkSqqxC*_C7&lQ%k- zes;MCtp+aaC_=0S92pH*+a9wqzm}bA&XZ$Xgb>XSo#W$%Y}71icbKqn$0O{nvSVO} zP)%nQ9(+~ONZO>kt>0_ZIO7WsO9G7yg6%{S@xaLyyXHl(+~JHcR<@~Wc)JetcoJCP z(*5ry&ONi2GGBpdMS0Vs5mUj#-iO=>lKVQZDtUK&)(P!~v-Tv=Si6Dt@ERvSC=F|{ zrP9rSsS9>tD&@yb7%RJgmP-evG15{%rpf6$irSz~tEB+o#o(Yag|Zg;oLV!xXs*bH zwP4G-(mWfz4opQ>UompM4bk@K&s>vF5ZnaMXQO ztCJM$242;|4jPo9m8ijPwSF$E4UEqaUdzwXVC%2jTie1IJ(diP-Yp4|Q1l_3tlUsA z;-bZKg(jkDG#!%OS>YxGt$?r>ZK4Q2%*ZvpP#$YMv&}Bwaf~XOTOO!7!_jyPohsp+ zPt(g1X{X+jK;tRn>y)0icUT%$$q|;M&9y^}N!AXchaAqDMQRCLreB@@aC6|@;2cb+aeWceiCJ}_v?2#FLFo?s8 zR%jtMn%*amBXW)t%^RIU=(k_3*eMF7PU8Dy%0e-WNS@}_GLm}ic16BNU%8Uc!|so% z=x`vTlBy2zE%!_gBq{|~FL3I%%CMHS&E?6I7P0gMJbAda1h!`SFMOo`R)ZzP;HJUj zWW60WqUB++`UFb@PA{;XzAoK{busXP3&GS~>lsFB!Ep&pagJ#z&EVVuZikLh80LwE zrL!O?6R72#0>lVXAFg)+E<|L|goPC&te9>XW-<4GMQFy`&W)!1x4eXhrA&$o9Yw{k z+*o|+(~w3=ebfs7O~!SE$ZLaH;te29E0JxUhl0Q|9H;A}t-BWVfU|){YZA^7XaX}B z9l#313dBJDf|G(gRX! z9LeOOwIadrT^tOtcu-^w#}3S51@>Kgu7ipjyWB7bR#)nY;36wmw8oyqK2Lc=PRsys zVMA}z%b}c$-)c!8H7=7V#S6VP+X+^JpLb+V^iopFX%(_0x&1PJny`@EL| z5=TMWq|MiQs3k}G8|B=is46UBeWbH(4+EpB$yAkGU0%+fPwu6i zFs>8mCBJRWVZv`Q?!Y{K5*a&44q={@Qt zs8dyxjO2;GWdbDodHndcYho`4XaSEm8mxPKChvwjPsggk@! zM@8-uJs#J7>1GSMF)AgHJKhAv%MP(*5IlCBT5x2-eDW%4jnG5rc|;a`!&*{P(Cr{O zSy(kj%c-(CV~uLW1jv9wO5tb8qVw#!v#{wx5;4N;XNobH zX3v`1O}m*^K>1cnd{OR5&oobVG>oYqqhz*daa_azk!bc|YRb&cpy!c`mFkej;i~V9 zVvQQ;9|VB5EC|?uZ&z9CjjDxy+_H8^0)7F*e{Lf{M;``U5MnDtL=U-S7AjLT4Ed7C zHXw3ye|aC!M}%pds&42w8T)rtG&&0~TB1_~&BQNUZ$F$2LZuTSc>qMEsiA+V2$b4kSY3zlTLt?&AF&Gkz?mRW6Zb9HML3uTE*a@4O zz4?-;=MWz5=divgi%mS|Fz_y6rzr*p1%%BNS|d_H&VfE)o7QET9LmFmVAvLl8Db>_ zd~Gu@#306&d${)vuWJkvNNsYCWV<)IPfE_FXza7XT<9M;@~y~+y5mb{51a0B%0$#2Fgj;Fv1PU}h`KE(}!+Kw0VEK~0)I4Ds%C)?- zqjqtCP9ptU6!u+nUwWVj`K-b1Cv$4j7m?O9B2G;BVk)18ki1^PKY&z^F@jO{_4DR{ zeDZT0B<6&~hc)lkBh< z4D30Bs9U}HfMYd|+$AFch&mn{*v){tP5P3U!S~8f`85 zHTyXU8R78(l*J*6t}s|7HfDa?1Q?v`=&MT(m)&L%3nM=Ex}zkH0N~G>N4MN*859Xu zQ2ioK#K{qOGp>QE$epL-3fX2^~54m!r!wVKrCISk4heseEvqBb9E_L|3=r z-sovF;vmClwp6IE5+swa*pL{@CG;VCv{GYdmokB4!py+%PhU^e6#OH}lG*ij>}3no z*z4Fsk+qfIdVx0@lc?6VVAOpZRWH-&snSG{rs>=E1EytLrBirJgb^-s*kSA@QdSMe zVvpWIj{~J^-keBv6&^bv{5~ndQM$_0s1gOoEZXD3C*bSK2^2K2=tzDitRfsIA*35c6~bz%ZLsxx!O8 zC|J0tMl=g=XhcP}R)N@oRBiL+bP1Yp=Q?Kus^ITPAvtc3>`U6?h)Hw&BF3 z+l6cZXU#K5^5S}*-6nmvSoaFBx=*aU=w?-j5wsyEx&Z)eQ+CzEyC`ctCkWRMelw6E zB`67MYly>amKl6mu?qXb{>bem3dWJoRR}I)PZ?xXY z8&%{@eEL%=`lE9&mN1b0V)rI$PS6BlH?wsBILq5zP#)MrB`jU7HB>AO0gzBOMIhE% zOHwSpQ;b)V z&s&$gCJlB72}Wk>6xmf2SYe5{mtO?oNpX#^Sfh=3H^_~7-O)}^#0@s2>V@VzLQ7Cr z)Z*#8Bf3+&Ai}+r8*9p#6O>pG^a2K`B&GyBs5zX9%ME3j@Cn;IYtgGsH|x7@vV-Z_ z^U*5DQ!-rmmfw%$E5c>lMI`uo)cG%@UJ7jy3WvGP&dcJlmI>6A4n>x;APz2AM& zy%%UuMq_JBpR~Op?uRbS)ubi^AwP7jI{@*dn%3W9LHpa7(6t}}8oRU3rHNlo>uJ(X zz1Bs^(3Kb0**ii77l@>@*xh^zr=;j919u9}9+ZT+?uHf zRqFyPg_WN>i4w~7PuQWzs?V?+?q!Y}pi!g0An7O%C)QS)YwuLdH3oWgVkQn6DYmbXpnA=*$ zv=Qnv%4#u361>YPRq8AEU^lcWttPQYLz0r}U9z40NJF=C0;4+fYT!Ga8Vtz&e!I+ww%~O{g->*LVQUgo;ymL*9R&UY)0bonr`6HB z7d4UTB*!iS*V``~J#l8vD_MpFZ;Pddr<%^}5t`Gg=v5L2*8sF_ER^yVbss#QO|D;+ zQg(PqIehA;Im>gDy`02xkz8vm6GneMTh?3%>m;Hj1n$gHDPEexdQX&>q8^xxF2rmn zvlyxZc8eQm^xmgh`Ml+##HqqdyJXulM6y`8!0^_Y*R->mI{^dCu~%LN?6ftkk02pbV4d^%pN zh?2zi8uo%f(o-}R*t#@LF4eq+co-b1)j{*;j)Rj6`KvRj0Z?}7iqgIH({y?(Q(t0) zd|4CB168^V;RzTaoZ}BdOVD9Aopq}OT!MSgCpCdj0KGm8D`XMW=j?U7z#VdJHB5?v zcQy3XUbd~I$Et!SEjdUPjMyATK-R>IQ{rYY!2s*jNW}&{GPdn*H+%DjCdHHZw|4dJ zNfjPKmk1b~j_ntCqQYJF-5_!gRpxLMvMhLmHXp;U)Yg~?)!2Hfap?(J!7Z^_czvg| zjfc_uUUKy7ToGBWc!S1hiHLt_ls(?ZLfUIoiOs2Cbq`t`E2gca>Q@1we27zJ;duE<=J@_WekGB za%Rz<%q5R;ZcuTTqs`Ep=Bq1SL%FqNm`+#rRNuH1g)-xIu(Z07ltSYKNpd1x-No8D!{pYt!TbfLKD2;jtx}tjfO}DwW#s4hayFKA}ZByXCwV7QuF15OQgoI{uwx zffg+=X|+amG+o6?=pqYEE8*YD8TARXI!+ zwnBt3u|d9Tx@eopTkgM0CfAz#&yu_oa?^knD&7^Gi!qbbA05~*u~ikzeG)1b7vh_7 zoCykiWDwHXxAK!rmkL;EZXyyjIdIsFDz=2GcM_WOO2^*^DIsWxiMY?>p^?;WN?Tzl zoMkp{AV#mE5157X2JR1xF!YR>&W<=TL4uZIGZBNB^|D=3jyUuxcd_JKo;n&F=6Xs< z79}dAPAYV4$-sT<+j-I$eNcg~ee^9`kGCzMok0BhtqOgmCoD$Ze6|_johO$e7YS3a zx9WO0lwi(Qq~{19q3=LLX-2YcsH$;E{7`PS^$jQAiK7)5b6@lgMdd&N^m%B?hEVqLM2b51=-JP0AbjI z#TVOlJ=_WJAa)9Pxg}WDeKkF4sk%HA649zHyTkW}OIo}!A-wU+o`%}-rUYzZS}?(I zF!wln+Jq3ASZ*G*D8?=*679o#+lH4X06{>$zqK$%@o8@ z+3xU&VBTmb!6X1?V9D3I&%@#RFD)yF(dsLLp>yqvEz40DYN@bXQs}QOxqFL!#$-#Q zZ7Xl#5jv|w>uRo{Pl^biOOJq8 zPi!2$vLfwLsU9x9tMPQ18d(BAB%{TvHFuNGfWJ_x(CcZ3J6;LeD9HHEh+b_Syr_lI zdBLo39~;7q1$IG*YbAL#@cs>`e7LA7i`i&pLr$WbVI+#Y$9pA6oF0!6Ue) zM8Kfg6<}piolvyJ*wp5ta9}d<&AF*OO0INnmW3EmmB-re)#kCk2K6L%p*kGRcb1qW zHtg-TUUv14acHg(lS8Z)5PfE9BG@l_-hlyvWIMg}W>7J`Er7}`qu}%5)V|n`wu8oG z-icL+d$_FPn!+?AOen*ZE0u$go1-Vad(biE){U?QUxzkq|+aIRLAv{5y?#KVR16Eb08uia};&mdSK=F!QS9_S?D zaD|pjW*7F<}F{r|OeDj`(lVn!znvm#8j^f52s3W&95tHPuE{*c%yd!%ImKFpXEDt9D z;~EC_3thAI?lF)%!g5mJ^#jpH?_rEJq0G!Plv0&Wc}=Li3DDbef+CMfN}>2!+t>MS1aRef#I6(_$I>7}nD>yhR_PH+wxxt&&gb4br!IF^|U9_a6-p8QY z@3Pz_KzK$2#!X}_qV)Yx4Ex}fiU#(SFKgLw`If3a=ezRckuqM1SgT%?_cHj;W1;gB zhznrK0$4k?9CkQIVZAg{XLn`RxWVD)+v8IOW*)AKT!b15)9r3OHy0S9J~Q!c43l-w zoh}d}08u?;&|hB&focq~SES<7FdKP%q#<0^9~QDQ#csXc;gFHwh_?lB(E7LBCCxVH zZWFr<_sW@w1<7~bXC+);P5S}Ga?-uU-U`1JI0}mwU4*tJt)phral@%GjTFpIk&N_g zaiMoi+kz;YsLPL^(hkrdLt!Gq6wV}Mo!zAvuvBdYA$(n+#R9b?!#b1OKEUK_R};bl zbmTy3E{IQ~Gp$T9iBhSdMN}<63dV^y*NxZ zqy!;fL;+rAD%o5nsvuKyc=ep9nOLeS5EPdq>5Jf>f-NLi;fAzw0UwJ*UOmz1&DeU+J)+(T} zx}4{F$Bqu$Z4-a|5=G*>`k!+hQSR zioFi(92_ygRz0|(cjeE7@_N`VW9RBmnK)_z?HLl z5O?Hz8+vwfV37t~9&)Oip02DoVl_ercR`!lO0fqque|hSN4tRtJAqe+?h?TiB9wLJ z0L`!^pch*av^~#?+d&|b99RXo%hU-KqQq+f2UySNrr_i*GZI2amG_NbC3OO{LYd{~f(!6p2wg}V)@4_~w7f*QND*NG_U7tx7DUlgGkC}j z!b_FYFHXKcyN(_fHBh91JjWe_>>Aa^)3Q>_>IF_ySyB>m`9gHF4*kR^-9tR40@d0F z-I;ASltVPR%AksEXY20O5|ei{z(TObyAjJ-==3bVJ+o#fx4_tj`4;uBtdUpFgGx7S zA(yn~i3>|C9f9k!(9VV0HkV6P^Mfh3l>l37{p^T8~3a}`i! z?W_i}OoPpWA;;GdngvFcCF|BX?|3-Gn!s_in^-JmY$oU5aMTD+vHJ`W-)*Rxs;azH z#V$EM5N_o#hp~IU1WWV5!&!2lygiX?T{}&^jt``G_wPlTaA9zSsO)Q|yU9U{t6)fq znoROH3lar+U0ou?tVEXvv^XvziBQ&bb=wpblx*@gS6&}+;7FExc+Ut z7xfU;;o7lH)|&ClS?wxcGX`c)qxM_0l2?HKRHBre_e9ceDz8ym_*yt? zHm|oYdCv@3Z740nU?Kk*S&}f1YR;S?(@p< zxeITQ2Hc{y(3LM=w|eFG+WaqmiQ@kSXBdrSaz{I67~H zYJ8pI*{mIa-zOe7%iknX4b`?!m)c+2oSzQlJOOy{(jt0&Iu92v^-dvsM%4mfIkkF+ zGM}skC>~3|ydL2cdc$9e&XH(cJIL`q8qhtUwo`Ory0r`mKUgR`$1-s(uH`!xrs~@&=O=volu;#ibS{ z_Kdo&l{^F%VNw^PajxGdld8~GD6=aT$RgV^sDdBf<_QUxwQSHq_3Oe)}pH zL)z$758?N*^1PPoCy)hCPHtUz{V4s*%_$y9zEVMPwm#Qkqa6!<7nQ9VA*>}cTZOjtnb z)zl0HdOXTN$L-j#N{$)Yj64Os=|pZ}L|b;vhJppKSU(ugCvNH;c zfZ_`ate1gHMfRm9XH{}?<2f%?gbo)+dpPo`G>9M&)dNz^3D(UNw-2)M z)3Mmo3$ck4!x1TrD`onEFSi%QaeyG@MFt*mD;&@6wHe6g1Ur9PTnYYl#o@9n-`oOj``@+{q1Kz7?KUw zUDYuffWdI51+A8E64XrQ_~m-$w4n0o%rg(#V5V|yE>IVKQCd(8-)3ECG3{@RtK+dfZ6(0sE6I4KkAmy*5`p0FJ_mE2^3 z^Uq{CQkEHdsbNYr2Dwhhc1su@XN(*`2FHtWdITI1a1MK9=F&dB2~%Bf=R1x*mX3xUjV}s24|{Qp3uJ7PVN1jVwT%+}+1| z3g-0`gF^J@pA}njPi`R7txl0_ahU@=;+UsRQ|V||+XDOF3dCh+2usO@fiw@E#ic|vqph99Lh?vP=;2}$tjfK`t)bLAk!*%LuSYiLUBd%wGV;3 zV&&ap7!8bcUKu2c4aqsEi9rxk(gi6wz=fGqH|!a^D$sl8P<$QkqIu1X1VY2|zJ*xm zhevoN?{(Q<26?x!W{Mj&FmcU5S(LpD!)XHO3IzV0nFvXxa1r;WrtXoID99CV!cPr7 zUcQAUQpT^jpQ&TgpZnkUTz{Hfx$xz5_%QWsZeCqvK zXyT4|duM~Fo8&SdH?bJ0tn5U(CTl>pEx9r%;HXGt>EON(Gx+rAvcm-Jy=+y3@W&N8 z*e2zjLVS7@`e?59VS?*QiL7RW4eaLzAB0xl-F%NC^vKEq4|pa>e$x3VTqYx9kHOs( z+=fplxXt*Iu=!CVo@Km)xwugmMM7g^c9I-WLvV%M1#W{Tkv6ETcMKk~d4<(govlo> z0at=;*mn^*ReiWg0ea%MfY@DBp($g@@4~yjen?PO8AM zr&~e`u`77DH_BS}ph@tC(Uws=bt^jPvRmfz7LuXr#U4rF0U?k?KHs{RhtZmMR$K0J z&3U!P?<7pq&v;}@yy};xqrq1qCCc1z*K!VMo(#WdQO}H+-z{yJS~Y4g<#{ghF|*!D z8Byl9nIQ=c@Ce&x4hZH>%GJ89hVO-cN#Dq*tP{)F88`Gu&SUa?1+jIV-6TwoM&U!fVGi#$)Rb<3$4MvG3bMQ zglXfwJa`36)=A}OIaT_wG*a-1^QbE2m1}Ic4NYNGDPu1Eo{mGNwpYJ^*|{c~8-smg zE~KcrI_~^89Y&^t0pt#HaaY`cYsK`^@@W}rh6bw4mEc8(HVC^+Wi5aR;hpITQ^mhV zNqCZwz*YIg<_Pw0tz|^;7Scz_V1~~?5zXGwGzqfH-l#649=DUxAqwoy5^0LYv|nc1 zL(>wJpCtkP6%mL{8zY9E@PN?;A1g|{n}K-{X%lR6=IhOrpvUI+ub=il$oW^ zQM~DbAp>JuXyQau$Ztj5Kz{czY$fASRgOS!*MU|9s6Pz64r3tL&@_VCKJPNka?pB7 zMSqE0U9AOJ4xvp#y{_(xpd4(s&1HPzP&w4<`SKT6>bHIZAUeuYpxH-gr{#&gHaGDE zXAX#i>t`x)x&nsq9AJeXL=}OMuOX<06%xh5mD!0U1^MJmB|{ERjwp8>CZ(-Zx}6WL zOiF%-b&}R2^UUv007TxUwGsiHUReR%hYt&HK-$HtXFK`=9RgNWtxF}@{pe+<3_Y%K z6;!e3eu?;MEzvjZy%=7=_@|38Ek#Mqe$onyG6NJ`v#!j&SxX0;UZE)QoRvXzUq>Ey zX~~vpXB9(YiAh>o9<99dJ6d)IT`bcU_Q4qvK>-GA&@X3JQ_5?!v^Qu&&_T_y-ZJ}T zp9~m$Ab#Tt-V-b77v03aV;~QK4v^ky@|TwbK737+M&j>F7M_+2o*jwm>+0Am#Mnrk zwv8lE--=h;%nICc_TZDW*6u2PtWe&jKt)9@Z?}EFSha|C?e!KLb?70&F&iQ~&1He{o z)dCUL3(a1EqR$l$twC8=V5l#>{Nw$Lw~dW?o4&<4r6b1nctj{x|O)TkY- zNi=dj&xl;c(;HWc_$SX)8U+Gm9pKk|;?$|@rQ|B^^`SNfZbo*E-QIy0%nO%z(P>w{ z+727u2V3WUGLo`C8Pbuzfu5H|ffX&fL zix+0z@|qjVZL|aSf(nv-vALRRGKSvau5ht}6C0W2>tXgY;3^9hqPD1Sb0?YAJn;dB z9T-rPt(gjl2A*6%@UY)cC=H0Odf!PoR_4p+0x>Zc5G>``6vT$azCu}^sFAs5zpfk8 zODn?mbBi*vGLSRn%JYU&F~!_jowVTPft7%jO7)C)oanPyu2B)xJ zr5xPt0M9_=@$SzTuhQCAjAKb})fTjjnqv%n%(qxkGOK8?QgP`lzA#rZ_SSq7Gur2J z<$RLO&1%SvNouboI&hmm4yi1hPBq1hpeU{Csv5RaNHyBS!NuAuEqQ6!d}xZf6`Ibg zgCpmyzc!f~a@|Xjw}@L*IRo0de&HFHedE-%y?4{@WOuINWB~+{lW7Es3-tA@!2;W0 z=@$vncoL*NECUzo+BVTG^7#j@qRBTXpmn<&SQt8;<6iQHofl)nvrwga; zj_W4}Ah$%6=#~)}aH{)U?Evgf4+aA*QzKL5ktyIHUmQO~#4WZ}xQ(qaQJ3_~$t%X6#BBv!*8{W8{XiDSG zy`)>()m{cB3vOBIU|^OM2+0Hl1?CG(0*lImv3p3cibfswCCwO*c6r+jLU<_#OhK|6 z7Vf$l2e{0X>qJ3n<#cYq`7lUb*HLaqAlBnfOODQ&!-3c6!494_!m;+GT>QwecBO*O zV+8l2BML*WED~%Mfi9SrkFira%i=UDKA9(x)*_dnIx1WZfRKAZsJBES?IUP`aZn68 zF&{$~VhTm}yP0jsPX^*h^_0zOuUpz{bR&Af&oFTW76D`A!h4u-pp%oP<1dU(Sgu) ztw`tat;^kE*Zt~MTODg{lvm?UrskXHth&Gl>{6x zIB-^4tAuHYl3q4wRo0TBT&^&#{S&}#1CL5Ccf8HH=DC$WkVrJ3;VDloYlnjwAbRoSmC@Cnz` z9HeMW=u1{)Htw|e3);Q~^`cNnAE-I?uD63Fr+|bV55dM8fH56OV7m6cRkUX!(5y`5 z-A*-q%t|Y2e{Nl~NrQ0CNp|mk*I?#QG4}5*N3~ z7j22oyV*Pl&`!7+(h22Ewo~6MXigprdyUHR5Jq~Z7l+HvS>mE& zD}FLDCpIJSWnI=#GTygCr&8Z0kb3qScO8kl(guT|6zByh9b?ek&I8X0nF$F4;xOa2 zA#SOzyIKj#05jGnPd?$t%+N$H+3i$TS%RMVeE|@vI*Fx53>Du;-8@q;RmauNaJlKk z9k)|#ZFMsRA(>kgJr1xgS_Y44t`~5b2p6H5l2wU9e40cem7Qw#T{q*++%tuZ;26=5gXnbM0v#Zg1tn)I5{KPwsK1yxN*vYxBak|9#p46fD& zDMiRISCxFHh|f4~DrT!j53|s%f?jkGfaQ9mJ~EmN*t&DR;q`N*_&X7KCE z<&Y5b>DWyoOfa{xs_RdPVtd#=-FH1uP82sDx$doy?vm{vE-t39z>q%@j(THfyxZ3F@VKm3o(H7gZ_|C6Mh1{A~p}x9pWwyh(pq~#` zTV427ImF7=uqBvtw1)4Io?oFOL-@C2*ml>Kyorq?#akHXJyH}MHRz1=(yQ#)=^C>d zm~fQ?7Ql_qt4XBS1*EGRtB`6qOWLhj!LHaG9p>-bU1KVz) zpvf{z+D2I?vX@a7u@};SDt#b+X-aYLHB>oUWu32^lLJ7!pqmT48_lsKi%%fzAqyRU zcZ;@P2e%A5w1|fS&{^>Cg`OMf;FA)x=VTQv8y? zYn_&EqMwPl-68wIF05Fs96LLPIB|`00F*+~9k+s)Ily|je$m+JP(X?flH8-OT!&YM z!KEzyb3HN8PCRQO1kcEldf7%T>QmK~BoMKuPaD^PbYMXQ?eh3=C#l+MilzzIYaVfw z(IMz^{SlqG^oEIsHoP4jez^om$1Vp`a*?~3WvS$Ni+!t`T|XE%rjR1D>_D38K3vV* zbB7#)g%q=q=^N47Z!eCX-=4Fnq3V*;98zbTw-`*ggSmkExOsR~pn+ z4BL+5&iu6g%XPuj=#@3dvBIDo7H=BIw~F8aUOHx5(sH@3)9_YScZ5{+50tn-s2LOC zu!xIvLLg?`MAGQw>#WMm!(bkCZVuHn$uptx7+c+-boDZ+&L|S~ku)RGIrj_(*PQ4V zqpLS7X{{jw3sEX5ez7)FR~bMm>XC|cTU7BeLx$(6!tHO^In6PG*~Qy_FR|GaI!l9A zP{2uQRYtQR4jfmCSdOKZM&LxvWl2`jC0e_T$rAEha%UuKq7luxYT=B$epPq8u$bQ| zX(vFd;RVUuSPQqWyidc4b*wsQFv9m9P`glLP~`&6l2(=Nuf6F=w6Go#7ON^*P%0ag zY<3hirSYDe?9;vVqe2u|LjsXzE4JIl)L1uQ_~mpCfOlFYgncvf+#lVdCNrTSl_cz4780C5mD!}PGhw?B=EroV$|U6z*-ZCD905Tf#ldteo)AXlb{+3lG743IkBi2JtBcSHlkID>brHZ*QA#X>gT>`xggE$OMriQ#(WC zx4o0_@Jx>`9eB_ey)P|iRV4~I_D#02jMnakGYamto`paegP!Dmz;?MvazPdP={$x8 z&uuVg7y^z=Gt~aHD}=LC_e_<}dFH$gGLw|rGs!A6?kj@yDixO~t?3fAcUL}BoKrhp z_S4WI9L;1sxGK1yAS41fw>4L4}0mZMNIUqMA5mG)LcfQ)C#m% z?3H!{rKWt^c_4A6Gw?`MmpL*gIam*rUst3tj4_-nE0{2=xr52~4cz+e`aC4Bkr5`7 z$x1ioDZZ%!G6mtf0wKZJH17+6G^mK3VYV${%CCmyk6me!ts6+@GT7~MoWDM{t6wN< zbo7&VvU68&ZPeBVpWLw3cdk!-_i2vg8Q{x;-d9^fCnRm1BTv|CT1k8vbrw0me?emIc z=9cze2U{P{8dSpsFo3&}l~B^%AD#xq^y6W!&x*)KpBo!g4Aj5~Wn zX>x`}6aMqQtTgXyHN%EdJ4r8Revn2ttECk)q>9YkVHLaZczob@`vE!@q z)a2aLcL39+z>kDt4%mSd&yFLiFneyXY1a>#K)TF)I0uLxak6=kU~HX1Ok_)K@v{@V zrG$3|5=CBr}C9I+Di^#3AxNik5 zL3l{slBi}0ZicAIrC9Qrl#05B9YkIxa_J4F-xZtYp$8D5gf2M)UNDP;UyPOP(Y$WD zfZL$;BJ`{V!(`5Vc+faK)oRfGKY2K%A5Kc2(r2=$oM7itsX29{u$;(M?jIEwgkyUY z*obL|jy@%==i3Jj*_9@!L=o0ARi!%!9LzkW^(OOWtUdVfBf*a+2|D{w?RhIA5LXBp z4Ya{$$MSo@jTknt6Md#b<;iXbw$L`~o>z&z=0~l;;Fncp4mYtmmZ4~EvK#CM^+aQB z#8(Z@8p7ig$EtHXO^#)}E1okKYvLEeP8NXbZV6?$eu&1;LN&FcUE*|QgR+pHBdO+t}F-Yt3qLLSPnWC{5U&5&Ts5k#0fWL-`5Zj7QyWych|`v7(u(ZEdbB z&otr>H-&msQW6_7+!!(D6ziKRE8$c1dMD_hFGKC!=bX#IC08zW&_y~uBL?w7b_fb3 zo=c6=Ba-G#Fpsom4N?w*{rj;_@g^2R$@ zTDT93DO3*7!a;a#fL}_>jkf~Atvt*$;e7+3VhibcPdR5ihdk{sa+R*hAw9zSQ+aK! z7~EC1+;h_DrdM+}J2)V+L8!MAV^MDaTN<$^+BE))!G#qS>lHHb1-y!$)?`!8<{rN4 z0q{_$2=Ky8W_hK6SXJ!yJsH-hGmae+tnRHodgKlz{Iw4v!)So+G6f+19-V zCD5*Ucilu|=?QRj?t_?lWQ_L6+Nu~NiO>=Q9$hJ_*(lmUGmH0l^n_h|VvDJJe%^@F zo!lbeSp#HowQ0LIgC_}jEN>|jd#b9>(qQYj&8X`|Z{gRtxQ?#Q_GOVJ`K+OL#LnlC zUDg*c$Y?|tGZsv;Dq@p+$QKup3yujo*1F;EQ_*N20l6l*J20z1R&jw2&D;=pp?w~5 zE8Ps@H5uLRq5Axf#}KuP8aS&iZlIg2a8Ta)Fto9J>NQYvF}l+j>rS@0)x581caNou zhT<))IgFdC11wxH1>|fEu=g0SyR2%{=_E1u7uKC1<$&+UX;(@XTOdPeGADOniMP#I z#@BntCOfqG$#VG97PoCdXqtqeQFE(cgVKr@n`3FjguE&# zO>3pN!&36Ls`S)!y_T4YX2{SnmIKn(`_fz&e$M4uuVYLZ2$6DL8q6CBZh@_pj56aN zB2evztf2q}i|?!6_A6Q^ zU>NO%Lk{FKk-7FG-k(TRNr z^-(Zjl0*W8K(lbpX$ldb;1dqrZns|Z<3i>KtBsKvF44_=WXXXT%q?5k35oAJ zWFd}dDR+i1oEDh-2Zsgsz+;yiD4@n0g6QjrE1>Uus5rpsXW08UI`RoktkYF8`^4wC zO^eS7diUIh3#T_&DvqR0+_;X`4L!~W0THFfxtc9?Ng!M;nUe?yQi}zVr(o+!(>r*u zw+Ss8HR2YG;s_dCuw_TUUkNZS@nNGwXxb#U5Z(D6Oqkv17noIaG;GZb4yY+vR-pXb zM!g8?J>iJvow-UJc{~iDJ%R&4F2vo&O=(9*5S1kxT9`hShIT~olz_{WT(~~RVwF~r zYoy))om_+IRk}3dik2k5InFvyc}>`G-M@N z-d;qQ3jp8XWlW$lEuW9Q6hX)a!vy`z zJ{Z}9fuVeL&EU~N8i%tTASHfi_18sEzmq9IL{t*@2Jv`vYP?9mMMX*z9}B+*7OMzt zN@{7_7M(brA2b!C4|FB{jehmla4?%rfsBF1;Sw;xAK99q1it7MV;j z@Lsb``<+)EbOpPBoz(>W$2z?nJBlQW+3h+Qokmv1R?`WeZn&b-1)IJ`DqF1bf^Y(< zc7o~#IvKxYoTvf?PW6?(jYwC0Jgl#1)E45`8LJ{4vbqD3b?VnwO9~(q-!f#un<#Wp zZ(^pmNJzK5n6qX?*HE6c;^4Upa~>39QZ`A|=7?Q4pE+Brt~7-4z=YcA9;D#aUYg)V zXPEN*!({E;@u5s9M3}_+Z59_A?I6PFT-YEdI2t#^a!&@A#9d=bldmO;SRnxE*~6&u zu4LiFl$mG~Nnp^EASGimH1kyS(Hgg_0n!i)*lM#PV0!gy(!L`gCNkr`cJZjuB1Xbr zqqpVAZLwGb5^?Dq^AidPr&ZVOpiTG^HRh4$r6 z8wtU{dGLHB6n6{@-C>0Y%SQ(R~nGb`qUU@xaUv3s<;=~A)o&|?cy+gLRC3ji| z<%m-$iJjAk`+Dz{PC~I}gRf=53=Jf{vk1NBjo^V{jypAdP$AkI@Tvmvwy9`{7#E$P zHFn_0%KaKU)ah$N6uKO}oE@0C602hynucx=b`{YG$(%MdcGku+s?1fJ$bgRmvrl@w zN_K0_0HlRaQY4qGipwXpWdhypmD+-DKQBrzy2Ode=af18H|lt>!n75IpJJ+8E7kdt zmsfEMERQ&0ktnDP!$uQY_LTvpp0cAb{Fcqh1yN&;n4#N>JYQGVRTw5i zGy~;B@aCbFuAJeAbX{elu7S||)RBs4?l)HY}T*>17i=}{q#Ll zHK-KQJD6s3cdJl5q2uh=-1O8~Eb*J#$UyB`$_X{)#f58~im1T7Fgzo_P&!wfX&lcr z#i^#9;Az)mu&OJPX;rdLi+hc5dh@OLL|TSxp}}t66S!ob!>$z*;A`V=K60?lkyEo3 zWxNe6;iJ}Q(I|=xl*Ey*#f}XMvCgoov}`TR&1;M6me#%AQfn6GWh0%V+|kxvKG<;V zv{XIvl^~k8o(>X7Dm>I}Z+HUF$4TQ#+cP3mDP6Y1?}fe_`=}K+3{*c5(a^@kkZ-fE zC^#N252-wB;lu>kM$Ev<0>16USXboXEUoIm_IXp;Rj)3acstu$<-mSHX=?B(Im0he zRf*X>h7ZahZ)Z9-Eg_{w7<=|sn#e5&uAFOl78pqq*7>TGH`mnn0CJVIl7qT0-CGj} zY(O=~m1w-y@USK3vaqII-~!?<2ySa7OHrGYyw0x!*92ud9X`L1W7b0oDv+m7&_I+%kh2Dp|QATqbeP{7;E%QA;)C!taT-m zz^Q71!^`D2UyzBVDu#za2TRT4?MM#C#7nH*MHCz>t*)-^^{$vFA+t-pP9J&oK54e5 z-?IEZ=t8?;3ik>(Z1dG5WUFJ>*NcdE-4nS`Lg#h+vH*OlZ!B~n_-}(U?>eG6WSdsS zsys_^(LwuWG5=j*QVv7W&vD~^o(#uzzMr%#BPYlh33nwgRk=ng>aSi`SJ#H-667~l zB(Q+o(Y7^%Rb8#r(X75!7q z1V_6V^C3q>tG?np&r4?%RmsUDVxnj}gG`l_7R@I2WFkjxl~f9{zSTJd3W}_pvOG$i zq0UjiUvN{qWCMELC|oaej-`zsGA{jHdO~S)D0%h!i*;7FsYJHvvTo5!-xn4?0xC9X z(r{m?77Q$o6U2hug_QwE_M_LRPr4>j?ur0QE%TocMK<7OY6uJ~&EE2s9kG(zyc6AK zI{ff7U^0zxv&q3DfY`5JmHQemW`6d7sBB8rMEbsVT|!qwd4Fa&?xjQENY0juoE1wd9%ES0C{+9mg~4a zP>Z{1Q}SQ49x#;tR(0q@(+=D{)iz6Iqj&3APf61SQ^q3rPaDBuq{0=6o<{SZYTJ;8 zpJhhv5Ohv;jwiV^?qWlU0@0&wquvM^@vE0OF4>{|r6-6R6*39D@j&e_aHR`U7~8Rg zPxAamPI0u0rpzYJnXv`eyySENPum4khrq_uA~)V$i>rBw&pqyR+WOaz(dig~<0!nU zhfj7&*#}TbZra&+MhVUNJ;pn(%{vYr(B`R$OXn{UVDjIu&n;;zP79B-m}2Ug@JZSr=YSWpx=-RviPYi* zl9hPnQE+E;<&RPEcvpP!>}*DTOfR8+LM2y5=s3ih3j09$BCnwCrLO0KROD)`6q!|1 z1RXR~S;J=R$5WB-MK(aNOj0n%$Ci-P<;z#-+c1C~-0on2Yz<&~ zAd2MHQ00wQOwc;KD~O!sNo-7!@_8I&!pUr6Ymr9mm$rg%3SB0-8vTSl>vCU{ivq+1 zO+noo(cvc<3{6WB1zjjuJ%a1tt{UF7@QPW+qJ{Mmfs)SdtVeQf)*!Bf+0QKQ&)CK} zLV)9JIFZyjA`jOgID_t=1f<7$IHg>G`-%QdwY6ZphaoGrjG)&-eW8(@BO^D?DicYR zzHn1(P&OcGeYBMX z6|jK{4O6|J7X%dzTP+qD97<}pp~fnFlbb1Tt^RU!nzVWMHM!s+HE0oiJP!QZ-& z+Chhq@L8zMm?@2jhZ91qhT^P1CPKQ1$tX|G6b;cF(`0Ev0#V9OuT90`-6HFNlmW38 z_(9EJex~^nk6?y`io(^{L4bvbB>ZGGrSg(fH*Tn`U1sa!UTF#lHnRr7j^eMh@84bg zHG{BJyvnuoNU`On4#B1ttpeE0g?RQs^BJtUjLu|l!+%B@2{WOK)$Tm?qmGRbyL74o zK|*%Xq}&eIt({s8qgj#MhV_p-yd7}?EV7kp5D_5L2(2R4+gj~~o7AFx><%kevm|#u z-a_a~0$CeD!k$$`n1x*xEs%1uYufrmQ%44Z79SBER1OS^NMg`e40r?-8J{Q?Z{!&0 zOJ}Ny^4G2&06O-L6=k2R6B-kF;S4`2-$Bgu65(FAkIIC5>f?CG5kr7lV$VlqQddeT zRm>HEaA0ME4{eoI#@Dvh-#(T(GF*7xlS~M_pevD%K4adyey1?AtsX}tAmzu`QCyzO znJ=tbH1r0KM@c^5ZKlM&c;w$uKgNT5IU0%9#k)5nHzUBfyG0d5vdt zAUnNNk{s_{TLc5$1Tvr`nN~ifj2s`R2P@hn)^ZH*7Cg5MQ@lj9q5x*ix3wG@-Qu>z zcJD-%k9%JxpdlfXBlRB!U6}XQfO%>8NfNK)Dc7UKD8-vl_!8U>10(yTKK4vR-0Dhz zpI3_d0r1I-#VuguObw2D7=5zBX~9Wj)oGbNYK0pqUYWyP^N2<*N{=&~bcduT%&-zh zN19NS3tP8Jzc@152-e%Ddu|NgfKq^^s4X{T+}mK}RhNSvS_&*>;!NAXM5~<26L5v7 ztdUOwSRm(KS>~sK>!J0kQOCAw&pGQevZ(7^_)Vu=SSJbBEdtYftM?YrL?_#ut%f&( z*LxM%%HE~V)i+LgY%oef0>|DP>s&cdu44DMao5X2RBffK=k&{TP0rwg{C#AwK>8})Wekgf=dGK{Q)V(GL_aV$ zb}5!j977p~zB3?*fUXYjk2z0O0$CmK-w&U_OSJS z<9n$uIF-7K`q&{w0ZyJO7|KAXBZqek?u*5KzSbpnxStC2R6zT0buI3#YZVv-gh9dw z0YdRs1q_MNK}YP84xkyLb+yuOxO3{mF%eYT;oV#gaJ+=%I591IxuEAo*t;$~#AjAK2ThyU zC7RNBD%g1`dz|-R5CQi&lP4=>z&~qzN>&?~1kxOpG`pdrm{1WjCDSQqQ#Po*GH*fQ z*(k(r%qfEGR-!y@;dqd+i#tZ4;Mk2n2}bp7n0P|8>B`|^gfEC?VC^)JRAP7(SF|lj z?gfUdF>EuAYObspTW$A2lU7#}v<&llVZxh;2g6cT7Gc~X_7O#_PfS`#_a-g;$ZgDU z2B_?eQOp8Q!UKJsB;Ix=lt;aM(#Nz#L6<5v^5-Ooq_h=Uc5G<87EKODn% z#R(O?5yfYi)DgTZZR9nVbVrI-^TAl_6EZ}Nf~o0J7@I3znHr1nZ=1MSypAlPV`srR z0XRkMy$>#pM-uCq_A+sER#y-tpB;ki>_~Gy&bLy@lTePM_Q7z1Sw}yXX~DW9JF6vp z###gq(9F5S0*7IVZ&!;R;sbhmW(G1*g_5)KoYPheM+gh7;njLLYXYPB>XIQZA&9T_&LWY@8PSE9CB=Kx*n znzA++Ob+$GXXVdfr_S0KQi@doW#qbOm<`S)r*sZ^7gbk+qo_BNGSvj-VFuvZO5_TB z=c5qvdTtk$89DRI*Gn!Z3Fv;3pMg2vk7IN)u;s7KL{85X9n8~-w(RNa;6r$~o&p`(KdFVzd z@{!mwp-+Xk&A1h7Lz1>wSAR_=!agPxQYlRro=48?c`RvE2q*UPx^JFc4U2-_=Zidb zd-j@#(@53C&7oRri`*VuDr0`x2~~~jK3m)UF4b&dzdNZT127{Awbz)8UEh`*KM0-}pLAIf2)lh^L z{49ksPN7ZX!IKTXo^i9V?jW*`VCEseSo0mEvb4uf#Y0u2x{d86Z=miylf}rT0FZM9 zUG5E$Oz08S+5z`WC%hF2T&pfDKvsWhsf4J?^cBI;TQG#1k6^bI?#_5Y655r?j5O>T zx0khQQpge}@kMp~ht&Qa#Rd&RP*Kp^@c5B-NWx+U!FKxA^~)R$U!}2DYZK}7SulpP z?detmXBl}@DT#%6i>8TuC=7Z6HByb&tmbKG_Q`jWkXGEHe2NF8gp9-DgC+nsK*+zQ zp~#hxjkeco$>()hjv!P9v6E3j_K0=stGFlD-QH~-E4R1;kbZ*YYM&OXC=}Z%68#gt zo3T2Pn zl%H2zU0r~LVayP-5W)7S8%Q(Tki3Re-G@3?jRu^E7Xww)=U+69R9Nw-*vp+4&caS-*C!j-nFM;kEZ5;eUMP&lM&wBzm2FT)}& z)^&o5>e;$I&fLB^y}TX_sDEXCWGjuR&26U?^>M)SD1@ch3JH^*J#ml)^dZc#ba9Sw;$C1E zlPTuA;K4Mv2S$Mdm{yb<&07%O>mhv`%alPpB)*`_oF51=1kNXdx0HEZq_GQ@61Kp= zd){+|XE`*25RGEWB=OYW$J%GSlR1$Nx#{$BKEEMY7!hw&B@u; zVi}jcOJScVO~5woq#%&?7X02|JxMOZ@#)c}EhE>n4#uxd1@}SWb_z~(lUg|Lv%iu!_nR4^QWf? zqP@5_*KJ`yxC-7Z@@!|_EGMWiIpVu$Du5OPB&a30I(e()s56>}5|lELRWWRzR?9>x zmToixapk{@d7qC_MndCpw+GJtYMwCzhq$$=^h2{>fO8084&;kCdhRA1{;{$WZx%3H zjF(VmIlQ1wp;Kk0Rb^4xz=0B(fg-6O+0@O-qEuc{?yCq5_86!^RpJ=Xxj>b2;F*_rE|ob7ppruVTULU5swAK|G*_IJtW&v}fafe8nU+Go?IDI7h} zTGq!&MmJzXOn4>X9wZ-Ok)SDKFx@Qe`Aq0?AH$`K&PG*(N4~vaxnrYuIdh~lI?W=h z!jo>ZrMnTUV3H^HJV-NTP6;6@g)UGiGvsq83Jv4nP`0o72ImEd^&bQRFDHz1X^5;`1`uUw5#3&&q6us{1rF zBG!4z=AMlz%n6r>hvrSoT75J^%^A(^9JDMlgi?C2jAyNU0%K=EtCawK;Z7hrB}Gc< z7f}g#BD-2`;Z5g@GYsZcdAA@SmNB%bvfAa|O}V}|-Z>&=v~9Ps13JtnF3QRN7}DBg z1e5t4{h*iyWg{#bE}&)Y&NswMa8Vg)+1nt~#nOaKorSCBv2{gwxw)+oL*Da&IK}5G zJC$TiAA*k;CT%*$PdR|ngy~-nIXBdnF5ow^%j(J^@p$2EbhS3&`b&QF?^nJY@|Mft zxn_D7CIVu{j1il~av0_JyJle}!fT`!t$j`@a}tX^oQbRk!9EUuF5j?nTIdGdQQwd# z_T*|81rJTu7}!9~~IgU&;o3TjeZj-m8rY@kmTd%+%aoGMJ^eW!#=HDyIe1uMukM7R$0^?{cOsGR`7+ ztQE+eMC5nu0W!6GX(W6K5P+a{ks!`dVCvG$w8D)jRjy$d>r+!|jAS&f)U4~W+3WDL z5o^o5<*wzsbV93LlZ6IGVGF=|>6Tp0tmhoGZR3eO34YvPZj3ehe>esKFslvsJLvy<90u9^^U-Kvh_Qt>C` znCu`HMBA#fCh+XbYvqCnu;et_<@6TrA9!~h?}CXdprLD+@;gHNI1z^9NV({_XdTs9$p#&6 zNAXT>8IUR5V;x2B2o?=cEk4 zz_#vi0a}n8t%4kQC^@@xOScQ`Pd&2(5$T|CK~E{L(l^WkaofNZPaYSqJ0kaBTM>0} z3#9$7;u#a+d?98pu)r)W2-&-ABRh^K!#j^RZI2Xvd2<`EJo0cgL0q6J*f_xq-&6X1 zfmq=QPG>&cE4iHmEZu)gw#=VmVyeA9Sl0=Q+N@@&Lue@a5|udeHGa_1$AXi+Q|n#N zL#LnJSpJv#-Kr568|~N;j@&vJ>oVgVWpqi~OfOCGM&0$;N}4z;xg=mOmipL`C`tHM z`L*+MJhwYd5q4!9N=$@>?%GT7?Q|Q1j~bOs%MG|P1XgZOGser6aC72!nUd<%5Ip1t zk*7kibEM3yS(?Sx1>r=i_^d;8392cNaNN8ZS4**{WKo-Zhd_y7Qc4Ihi{^e(Oj^5a z-8!fjA`v)CdfEbQ1Gbu*yz1oyR34kdkyXpktUJhAi4nPiq&8Kt2~*7i`P@5p%B5R7 z9O2}KPF^d~X_EH{Ti2-PbDFi~jz1MEIG;;}K)o=3P-m%=@DZiP)Q6>Z#1WROOS!{rScZwb@799W&<37x(wMh{N%$kF@s%aJQ17N zXlFoF3kt z0iO_iK&%#W#b3MGx1AazX2?O~;p0c`=BmrjqG(!dz`o$Ck-Z`|7(-gXef3mublsWYckHzVSEn_5(dut-iB2d4GMjZ7jHYODcSJYgspb>% z${gp~{A)~r%~4GcI;p`?NF_T+*0!!-<+xn1$N-Hw=uS6wEPUF%1c*U z(ruu5i@X`nW)Lz~joL;#hb&jns>KTY;_aZUhUin;NP0ea+m-l$ZT&T))Cz(d5Iy1$ zj|2?`u*>$*1+E5~7$|mLF+l(=cIdXqDaVh7Oyaij!@6=5JAG#ZK(6VD6|N;2T=+*I zjv&P{*(Cmgn!8xv9EO!)5=`0}JER$8ig;|J3!0_|w3d&>!W{H@fMMo{Sq!V~^-lMiDoYOgq=t?M`e7sL-x4g=E*V$__1tBm?f_$8aT*` z4?5Ok;e=9-Z9#zIjGC=_vP#i+FL^$-6=-6j`>W!sQ#jx`T(AIE-)Fala+G-4#TW zyk*y5@j%T=F3J-cPhr}#eX68Qa-D63Esd!{#)dH^vXsQXGrywl=R$MCdc%;f>p7w4 zys=BgFJb!@?NTV4E?J(hF2r>^1SP&HNDLC!8s}N@GEE;|0P&;HyM(d}QS&aebzEcx z0ZG}P*%{94r2LoXH8EaaRG^tX;{8w;mb9hPihI5lsu6FFq?ChU2k}#xOJOj3q6j4; zjzPHUbshx^#0TBzemB##m*iFs9OR2?J6@lQI&bYxF;tEdgrP(!VZC?+1xJ@oCUo*1 z*+6JmtO&aLX^N}W&4;@SY&V-S5LMxn+G!nOy_uo6kQF`7Cad)2apHvItqb+d6t!xx zV6cfg4T-1-C~TjBOf#@xLBT*zea@*;?!M=w*FZZ)T~X5I>QAjNi+wDI0@z-Tn9S7~ zhpKpn>oWIQfrPCxOR{i>1eqSTO29Rn0ZP%OyuD_cUx{fBq;?ASwk6!9E!xIc5l++_ z$S=v}3|ls%$nNWfe~E$jK@R6ya75`vx~qZUVVxAMS!fXJZY3vMCGszH=MLBlD)SH# zsd?7BBfFmsFgCM+bT`fRK)!jQ5;S?W^Q?BUhcnW(#`uq9A$oQb{;1HW%$Blc7w@y@VbPco68dHz>?yX zjpaq7julIR0}qI?-1=WP>zv zXYFdwhqG1anzE+m1585>lrsN4axb@gmXfvv=mai?A>yt~d+1;~rzWY4+HA&t*OQMsD4Up?z ztv==ipw8)Rv^&}2g1tay4Zk2Ophb63H!2GHA5yb0rzn1TID9Wk@1QUg?~j$i0{G&EdLpLNj8E)TCY_ z!ipAng4M^_%rAOvg|Nv{!AZ})=cKc_#!@KtU#o=$NqZgLP!9}5#aLEi=ET*x-uMTZ zq0?ik)qQ$4yf2;(+)_NNTrW%n9pW<^L0}MW_?Lzn9?XOgGgd9U@}>wJ8$(Wj=uYFM z_^MyY1y6r) zN6QxIB+pV5v@IGgG$r*V%o`c@*Q(UQ5~gegUfeaJ{f`9V_!}xTWL>4~l{44&!{Qix zeLGC_EJcyeymd+l0>{PN9X$leq*q#~zCdNH2f>Rg*eQ~dRzo0UtzG7+xN+DvzYV$= zqn#;UA`QxM*^bW1KntgWxM}070!*_cWTq{)jcDEk9Ihx7GeZfBDk{$`y84v$&TaxB zqZolkc^`+up);7xspnPJ%4_5ooRfmeF@cU1hks5};(d|EqjEwt)a)7ey3EFNOfkF8 zupuxGv=|oN%;zWzzjB#}hl3$27pa6pCj$XXdd8Aq3CUYVV?^~j!-HBKP!`WT$-}E% z{OW;bub%-IG?2S-!Ky6~FP%FF4(byjXBrL$EJB8l+|0|Pe3q2LUSN_YEDD){CJMqjl@KYE_2YG7&q$ne+H`Xq8wuCfxCfld#?qhXAg5L?5BH8U4~QLeeyJ z-eH<73HMF>woG+kT^n91{s5HROZPgJNrx3&Zkkpe(iNKxB9XXPR0%kjcZIFX6G#Oa zquN$)uPgvH?t^MtjyFpYn{TawAk!usDv`-d{O`90ioCTbhXVSkC*)CD+O0i-g>?~0k84N-#8YSY_>-FRmNdqv6D31Zw`tnEJPpOny#Z{nuZ(0m?|C#2x2?Z63v*l0W!RSN+dCV(`8SBDKeDt%Bh zGgAY~LR@KN$*8NPXjp(O9A+y9;Kgz!SfC}D&S4kz&v9WUyq{{PRWs}?dgo`=slCE9 z{7|}_?L$$48D?R|b&_&sI5Zj38L9UuvR)#?c4HA8Jnba9LbvTP`|&?#sc*zj~=Nldmi-=3!okSaw|1;@bYinJqVmRPNB z{Ayd=w>VDZF^B`f(_nbC_Bf@+&WrD*m5{5e+m1(v&Dq;XTyHAlcaZ0y`PsA-VbYtRt$VEox&_Xx7| zqj%JVf?hZTjmC24Pq)BcqIV~867I7pjaP@hyrGFK1?D;h&qjqbHRp|j%RvxY1JF`> zIasK*-(o{W&99H|y(v=CeDx zh>0!xyfngVilvEFQCzkc51)J`9qe%0-07y64xdF`RGfpIz{y>UA{IacuPFJbLD-I- z%q`WcjXcACHfAU0<)Po4BwY6lXN$2{#;w*%tTC{(p%HTdWnR}nq#CRN@NHP;dhw>) zmQs1(HcoJ#Yy{l0X*`^p9(4Cy-9e_1H>SKmDj|9xp}Aag8im6{w}}>GmY-1h>?k1c zAc&SquQ*++npmBO2ban~UqqER9CPSLj@baCx;935*xY?SDlwNC8|c$8*eo4BxWZbx z72?NRdKCTMD>;%i5!nllP3@Z!q0^Qp;fka=awj}RCFQ1cL*Mk zzI>PNxaM3aH3HvUmeaMiXd}Jj;#puL!DYT9f`zI06p|#Nb*-hO+ z0Stv=7lUt&I$M`h;}Y6Hjns2xtLhe|*tHWRJ;5O!tMJC0;J-PX!k6jv#a?`mX2%C6 zR*}$PO0x99Jny2$@{+U&qwRM+hNHJ9$GD}p1SqAKCCk9P5D^HOYVJ&GdUp+XPJ2s6%iq|S+e15c+c$K-p9>w20JB`0c(V=PxQRD@@|9L1&)nED{l-rz?7S@|gzqyI9%o04h73S;{p2bI+5TB(Jj& z>-ynwrt~_3@b_Ks+~N}5!efbx=Nrrg7FP%hmY*dwL!OJ&#ftVBC57kZ_j>(*U2 zx8bO@P*CUea!Ao`lJkXxgc;;ZwL5tATDuF0IgL+g zFDTQq&7pc^xEjWReN_BCR{ZoJMoFf=AG}FQs`1D>xv^Y)^x3;)_F1{>4`)};U7avs z&mvaRf@Ua_2w!4AKRiZg6#KPn#<;3k%)}fpmm!+r4AqwLvAB-T#hf0QKw5BOTwFv51|E8{p|kKRR5?hEZv{Mpz9%w^rfCi=FeAE`2@27-DcuhA zE0=c(FAmpwrJ=!*##&3*(YlN2##^;qH?%6w`9Vx8%Xq-^v>4nzp>U&W<{pqhr{g)+ zu*`gNWT8w;?-$Z*Drkq;%Zm3P)h|0IebJ=2fir|JFlw$eB3QU zG2j-?TK11)=ryE&M>E^WvQY0u4>K~&Jl80k(vu494Ag*{uuS)P(~Qk9Eal^jMD$Kk z>J_+K3ep)-ZMz#B!TuAbHOO$>0;@%`@>Z!mg0e474($`~PJuBB&J}f}0|gMMgWErE zL)*C*gz(ak^gY0J5S&mP=i3`Dnw6T|M|wb7je8)x$sbZj>gYT4qIpMiXC0SKSoo)S ztCwo>J+@=!doyydc2>Y+d3<7@I`oEN-J!!Sp%>GHfD5 zAQXbQM?4iM6tMbtW!%9Nvg%gA-o4~mth~=^Cb=~}r*TUMzJ@+P*c3Zf7uY$=D#U+I z<`YIq!NNTAmbec)l|X5a_E)RQOK#ci12V)*FykTrSwHx9f1qj7Ox8 zEnRvqhX9Qcm9~f$9Zf}&n!#2(l(y??{JR49-V^Res_a8Y8{$lYCAkTSR1PN}8%eE9 z0?>tN+l0enmPqUwRstoZtQQF`CK*I+=^FJ35}VYNj>k4;FuGG*&w6@p@f1s4#dse{ zn0MBJSS$*SmL;1b+=hK9${p>VQX-~j?x$me)^I_w;(HQ7*=+T$^z{8H+)EM7kUbS_dyh~``xy}EYefwA** zI9~V?>}Y)ktyWLV!#8kfWaPG}o=ymn?vZpctiCaRg?uk#iz+lPc>{6Vs-$daQ)6!M z1*;P{cvueRF0F4x85~GV@1j**`?xvN+6#RR+153i-l>)_^U)O~MOtNm#+4VWN!fGV zaAa0u^VpaqTBJ!AYf@#8kPxG{FVBax`Ml$(gjDXe{0Ds$kle)$iwGT!lP#CpwUJoh zwC5FU$KW_(%dZ3hzN?dsrX7mgr3M;-TEDgn!9A>4uB}eMfPl^=Wgz^pK}Q3p*!0fz zSwY7z3DH1XK8!p9mgx=!3Ukcyg%?c8q}pTOf(b~43=;u(D|mpJ>_q`e@jyjGJzef) zb#v$!r&7YYKv}4fqmk=(=d=st_x2b;fN^^CT(wBg$@9Y4;ei(AP9vJm`-tk z3~i%8Q2G_$nJdMeqA-ea6Yp}dhm$xWyr{Hf+$uSs>)z^?-3KR=9Yu@pUm`vgP!Tdq zu*3mYFH>V8xO$4aUy2jy!S{AuLyfl6F8uU_#QEtZHi8*JTuV){-qO{WGjZRZ=s@3> zMK~>7BT1;}f~`fcrOnnIGk2ai(v&5Pd*@fgUlnb&ipkM3@~3!joaQ_jQ!rdyd9Rs? z@*j^H`MpV=;;0=(NgJc?a7+U!Pn*QqK7{i2@Zotv0tNB|$aX!Qun<7L=e`Es5am6w zZCzQA73&J(Z@f+jR|!zh4KH2iD6-AnqDxOtcyqxgvqe@DLyq4iX$xJCE+eb2TueP_ z2MUv^b7Fu`9&0O`_UX7VAn>-WXEhf+-up8yR`aPu9R~5b0J|*UK&adw#>%VTLKX=i zLbFN=3(Y5S*f94T-&`;xn3YsCgD}wFg}9ThNmR?Wp{Y0#mn|!`B9~zorle3YIiG@6 zX#}Xdu!JKc6hs_$Esm=1m@cF(RQOMpaRxWe%rf`kDz}|qUu%qv3i|1I9n-V$rv#?I zaVHqdTKLEc!Nb>QX2+{nGKvDv8DMqpp}C`m=H#o?HKm1)OuRyHnksgK^DH?+@070u z?+xP(Um{$yIFPkt1+Nlx;wW3FouztWuRc1bo>1;?SVL$lq}C@IVe?IKDng)vRu7yq6@n2vz5coWglGDzdq;H8%OiOOw>7piX}j)H@X(2TI*+y+mO$ zd|1yLyu1qz9nZ3!y}mn_12Z;=XH7p|Ftjcut6XXjP%gyHX(|@f&xV*Rq@j3HA8LB+ zN_$l($7U9DK2U)n7ifxIjSpquSl*d%9YjDv1Q|zi67my>B9KH<=M;Mvrr&opJY(M1 zBo7c#!xZyEuqoG4TQM~7&E51;(ZWKIjn(2+CK4;sgBL3|7uBlrVB00S8LdAdb@SmX zI9r%$lH*s>B6pqT!1JR;sO>O|QNsnDQE_=XWSdhE*4abr^Idh5-JY_6IqNvWZWX)0 zPZS8;-*$y>fEkVA+Pg;6^c|Ogb90Rb;deq8*B#I-LA-W#aa`Rzec)j7 zR!={s-_|H$453eS06}?!(54W5&&>^_(cS~R*1sqr3MK29@uxs#AdLunt`XJAh`Q~- zvDF8)rzDc$QO?WQ8I-I_f!;_eXyaxwqa*`ctnR;K#jwv?h)woP(EWYP=DtY9$+scF zKQtrLY#`hmiZ%y;QuW#|BQU_bgbaP5f>oLIdwiEt>kxf`XSutUfK0-OSU`fRNHB#> zLWJ)L0 z?ch(4{K-j!XIfbBt^xuD-AM^L{+U{3OI0v4s&|oV;>t%8kg6Ok`pGVz9nO4_H< zI1xJ;p-tsySYD7w>m{TMuVu(t98W1E%1%z14y;Q!q{kfjE<;OaF)z(VvK%*Y8yR@7 z2ZMD2oZhW+@$Iy7GWc3h9PgU+g7{ytLTtk45(KKwvqbAW78&yD!Ru*JoVc*Z&Tq*G zWnO!si^)s}VMnU;=Fp18IolPOE6F%%A>HaV0gyT?H%ipK3p#@)1h@x;s+$4VVQ5%E z-ql%)x0?pd81DvZE&Tcr^tF@<5$4!e-8o%U;Y4vuC-JoU-i=-Bl9jVGx;KcdwVZ%s z*-#+gK)852LMMs@5pv?PHDbzbE7xnnx8e+^ZbPwQx@KN)0buCwPYB4fr?^jAgy45V z>%HESn3|iz)JJ!}SWxyb#aUI(?;kYF8IW!2iVoNyrl`J3_#&;8Ooih*)6S6R)YwR- zBcQ73H`f(dPxioC$j7DZ546!|f@VYvJ`ob~L0X`nU34|8+_kWgJ71JEmeRHJ?rh}R$~YuPdZOXyQw6rvzq@a)!p#;70j&Lq{I>c;VRjETYkHI zXECx-r;eXf$v1XYk-9)Yr+SBV8{XQaC#J+KKykx>=&Q)(bn?VZ3f$8UVaeA zCicQIi4euvcx!~=rs4!k(+o}ajtFap1`=Tfv=DeQJv+Z4?6=tFJKHdc~5)H@XCzO{8SmHh`HnyWYHJ(~>F+;Zm3| zSHRuKJCREsn~>e@11y{@6QUGHO5j!s78#+n&^5R_owI(5Y^+JDV6K%OIU`DH78y2F z$w8T@H$Lj9P!YE6MUCb;QzFdR%V$+ntkHr4ybAQ6w)3({(+g?OUJ&d?l@K7QoTta$ zG3@3@rM}8Jn2Og8u9D0pbXKW1mOJk-U5Bl-;G4WI<02a0#HE9B#tGxEH_9W?nR%`w zdP&7w%>A06q7$}A!vfmdMd8-ywc*H~3t2EYP@jW?xL_I}1rimNF4s$PW^u2w*pk;`_w+3ZKTl+_2He4~dyC z>S+m}<3*6JNWGUGW8+;rIccy;@d5pCCN{oTrD0lT;75z!I1W90s~1?B_vgh_prm+y zQdO1;Ta7iGd6nq&C519NV8uKs%Ue`nB~}l`gUj6DeArGFe)VULEb`1aE*~lgM>Ev9 zh~=s0A52j6Xb|kDxiZT7$Ba2<&L38NaX2_2*wLC>aR@>foe(4TfTqm%otETXj)+k^ zd1l13NmzZu5l(r_;i8ub6vkThu4Xuf&vaZkPDvuY0y;Du+6sp%LEM!K%~J zO+u)-eIPVn9rMpckjbzc(+_gXgV>0}^F3{!y;jSlK?u(`z7Y`g+Tr_xp_PP4WtbKy zeT7noyV?;^ni9(*pe~pZYGY{Z;746eJrURqd)iR5SW0ha;v6j?s!fo3KX{RxO$==Iq7c4hrSuHsGkX^Lwx~~U>Ky-I_ z`o4HsH;`S*lc*{;@?2RhJGGPNsisz}LxVQgn5qz?Q66%{YtBexdeK_czIA5S;y~M2 zI08iz)tV{6ap+juZetn6eZx^*SoG_&l}K>4C{Yk-GoJfaL0lf|yI!m=`fc!8W{(pP z1_@bghU&e22xZC&P3G7%Zu57jE>ZEKWjKy&hm^XTuDG%9+-;pNMG7iOdPTF=)KIqx z9%A<74rulUijgL16NZn+w`dpMDkyaiCX5qS2t6Lw@Rsj_-EAliqA=NZqy|8+{y6wM z;fc(x!Uo}C2>Cv2EDfe_8AvM;dbO-kZ;Vr1))FTs%705=dCpDe4+tM}hgJ+CfxSH6Y z+6p{2tJEC9SyzKWdDorc4BJc12|jn!76tm334Qi~62k zN?%IWhSaUKOIUz_y)vXV(F)WdnWnyx7iN?}5c`rT*~F`+?lIBiUx98E6?{4)#uXkT zIF&j`N9h3@LR42)C$z9KQL3hYg!Qklcitc;DOL~|6pn$ea*bB;ZmApBZLK1fU$lqL z!3`txjg{2JGCV$H4{y{|#Ankx@5%*VbCShvsmy->t-WcaTbnawNtR|9$C83 z5;j8BC?{+c?|_O41Hvqgwjv5rCOjNxQZNMR-EH7SR2BGUj`e*QcaeaL5ov`JJmKIs zkk@IRiuCQQ7>{{VqJ|y^&g#=Lnmf=en&{O9F%l0vC5nk@1N|*z9AZ_g>b5YD9u&Bg z(h9^GajbO1nmNEgZ7_qG4`(g$s#_nqF(0IC#4p>j9(|s$PaUXn?^4wUjHK?d@dI2M zlb>qz*h~t$i)Q8!dN6?25$H4vv<#rVXE`4zxtA-mv*XQyH8>W`7-ONl ztB_S;c3n7m(9PL_O$=YE@nW{kb#(M+NtJLx=ReZ zP_d+{ED|ZeC(;4x1L6v}Z?GWdg7|nNn>n z7*<2sjuk|a*s#J%luv9GK^_Q4J3ev^3RXGeC7*KA2AJa3=Z#iN)#LGII1~nVR~Ybf zSPR#NmhF+cauL;sxUS!CiBSH&{85QUeO8rR_~U7{zW}R9ibg!M;v89v-cMQ#KCe2r zixCAGe8&RKo2qi;x{*N8K<9=yT%@DLr7(6I>EX~4-GIY$r#G9C-MLAD2?%3>kw(fs z@K;v`deFBR*sc8VQ&32inUHzaK-utId7#I_(-fyZp|YCH0s%7QZHmyrBPq6QVl zcNzC()b>DTUbiMB(AithZFjIKgg}-}cFpRvr~NwQ%?vher~Sby!}X8 zsrq&jhGZsF)a)e3Q~|GMLLw5A=2UaQ;Axer=*4H^N4n#xirA;61}_L*aG$!#5nC+~ z{i&W1HLX2J3Tu_EZw;}hS9I*|Zl#d)%X1L%g3?T~`6Rr9&kK4~@h7==m>n^fHzg0V zTEvLLWKhrAtJV#TJK|O04A;FBDDsX18*GCps7NvU&22)frJ9>B(QDMzHYGfZma{M7 zdH|*&#H!{ImrTNJC!t!ZIx@n0w|)yp%*CK83v3+o6_u#mB*%c~?J?|sO+&7Sy9=2$ zB2oD7uiI=M^`mxHj?hAF$RxD-LSTD|=M3J(IJHdj8qq{dVwvgvbSD6=*f3yfuef&( z>&*qZ5Th;$QB1St?a=AZf|%Wwgq3d|6FI!5Q#yrGPQ%G*UiZErJZ~xQcc2~LgX%9k zUt4toLfpA>Ee}3x?a5Krx1%@Pn`*&maM@nA%=LRC*5rc-vW+sK$>4~{?M2to1r?Nm zRUyw8pSG>KOid(0TFH!11gVMq*3O%71vuDLF?Acg(|Bt$-L*if+*}9;?mJh>6IVO| z%)mvvhNTLCCgVW?#IK!r)h%jFDv zj1u2NNF;{J+q_Sz{W0vAo@BEO%Z)YhM~^_4UoTU%=5Cf%u`%DjWkz*LGL$bTUE9^*-+f#2Ej>mZeDsAEO*3K z-Ue=dgmg1?TQSvSJL18V;5vZpx^ouf6YbL=Am|Gfs4(EQrb$zW4B#&zJ$04=rWZxS zyZ5gf&>)i%PF=hcHHgwn5KdMgfmBo|!Fdd2PNOafsk#Fvd9j+N6%>01V{IqB(GO>L ztB)S4t2Z-k8+4%aKcequ>2}##-O}+M*iYH%-sJbozV|$*a#bKquVlQ1FLIFQN+wx~vIbGo2dVo}w3Ij-(R-2g%MVec=xNS-tSvqd3O2z!JK8F|f;2$Bc z0F7^#CX)K!2g6==^|dh<*n%)XO$$~@YkSKJkd-b4!&Ai~E8+{yB5lq&YLk`XuIU-zCsli+jnPD!mevc-Nngy<4`e9HIaqpHN~$$xn=a{Znx!J zyDMnCWxy6iI_7cD2ROqZj?_}ZZfJw>;~ho}Fde)sO>X$423X`K*PemHqRv@7RULZD z8yFGE+=q1wCuum2T`ngypz>$YE#$xR@+}=kVR4 z?~R4;H@e3D?eG$9Zz?SM8kA*MH$^p;(r0f*q9eR%lP{`99Y{wrwyudXU9yEFoQk1$ z5obthbf_%UpDST}zbcC^Jom4?JJ6TzVptWo5}X<3JSwoRXrpDYoron)odJCtZB>_? zIvcr=9@Dd~0t+Qls8)MrpD`?gkC3-}D?OZZ9Znx+Ahw5)J$pAwZff|=iA+i?>!?l0 zw*H}P3h~_`#N@EDngn~fCIw9)Ep(9%taf#c%9^{Laxf;~Y?-SD#^b5(pagJq5kXMw z=sDZJ#EP>T&ckc#M$?`;BZiFtz736grIz=1BVPZc@w#FRRHw{fHi3R7f#S0R^|)X{Qg+IaBl_hMr$`MWdO{k^C>+xRk5G+l-Jec8 z%4!iNPiU9LcniG>E#bfzs6SdPqSkQ@Q+l1;V%)EA4Y&iwJPtH6`b0CK0#i6tC4_2l z{eHkbiFE~n)b_hk2u=us1(T#=v-rzyCybu0mdL`kI50cJg#?NSS(WJH-S!|RUD8*< zht(TzVT{enhTt}UyEP%*WQP`oB5H5RMqJl6reOSf5~{%=M!|OTCr?f+Igsq4c;xg6 z(hnXmGv@MZ1o6ue1tPDAEt(#Mjd*lC#`8L6YCYlsq8Cm(LA;TzMVs6cvF@|?Y^fe& zlGv2HaCX-S=Yz8}AFwg`z^MkVN$uK|$R2}OnD?ofe(rD_(5-xR+-oa4Yth8&seer2 zD^~aTa1A`kRFBdUnSc$OPY#`Ki3N$lIxNL98MTNNODoKyP~x!cFwmT zG+H{+kWYcj1h_6eDoYv#0M}C*bUk|%j`VJ| zPefoV1zQb=;xT0~ImxWan}S#NoOSA7eEJ4(Yc9PPa(Ig&N$w<;GfR*Q*Uh@Xy+j$7 zr`=*H1sN^LW(Be?k8`(DwQUuQ&dqJv&3L!5DN5~Mp*%wfp>&XW0R@QgKAu%+NHBO| z(&YleeggU+hOe-c+@)*;b9)7m$}SC6&Wh?%1TeutM|g@)y+Kc99O&{k7$%ZA3^k~k zQRIv$Qtha>1#mnm!gY8g-0Bl}Cg@sKw71a?G^BaSu-Vq#=rnM>=Bs<`=z*ecGgOkN z!WmiST2tVuCJ-Y|HM`3#P;gu@v1Q2_mAbn1B4&q+N9AqOAq+Pz=F$`g+2u1SS;EG> z%?S#A8<%7b_Qss+6ee~zxoS&`k6MM3#fHj$-tuw;%!_p%>SBgQ6!ZZeGERt+$dK1L z#nQKM+=IZdh?CB3a#q5+XlZHeC$)w}?2}6##wBnvn(xIFAQR}pc4lO^KNSG$!Mnsg)#lI7`GB&;ULe(^L*1Oc8pT`Ymg9r6O*u8yL;@~*BbW< zC#%Xd8js$-O~KA~ITLmVHd$Rf;VlQHoqsQ|vQ`@4&xV&?+!J~OK`#;aG<%Y3z7}2_ zvvlvUt)pOvhD3@0?CR;C9HR(UGE9d3xQ|QrHw_((GEG#zLIv}}UV9VFGwO7*L&=z- zH1=Erip^Z*jymgV&{5i1=6cUi0UYkmc*n-B%kSBOy81eJByx^*uP$t$)9!a$vrFq{ zjgo3J)x81>&SfsUb7K!0o*C@#c$K!uCC5-}dWHO68tMrYJZ^O6B=q0WjKzzoi=CPk z?Boe@#%b-tVO9^VWGkw12P>4qmJyqFdrr)^U{0v{TU1xz7OCDPid2F?x^1Hsf`5UQ6O>jE-m$Fkxnila)Tiq}ug{ zQWL@#MuG5N%(r1PSY_>Ol#41SYO{a@OL8dj&z>n%3T+Dc#Y6yBAZ2&dE~O)?6RoL@fcA>iq_|5nNN}(_9=YITkbj4mC>E9O}C8RtgkT~ zmBgAD=X$6tM3!pga%LJ{Ej9fxmeuhVr>rp!6fmnbAF*+r7TwO9foem3P$5Qqky(t_ zD6_sh6(<-e``dI2Vw3}QAgHfN7u-T)qDd@l^JjXZ;JsuE0hv%5%c+vikmHsEIz8v1 zhT}m>-eOS)rV!6TD%IsDC%E?$$S>_t;OcdmWd;v-qyXfHMPRoVlazedR%woF<}4l! z*aRX=o=9jaNg6FuNL(6kvSScgZCTnEhqs!GrP+@KSyRqNjKbq^Kge>|sBnA~>SA#< z4KQA@jkNTZ-Ax&ePE5y%BsIdZoL_wx&SJjT7~bQ{?ct&o5qtK+q2M;}AQ080BS;!` z0WlB@PG487wD5w~2M%sQK`#ilKwmexMq)1;QEyCSsw7^?hA??YjzZBUTiyYCeQW7Y zBjJIqn0We@K4DR}ac2zM9!=+JNgs&7v)oP!*}S6^$5#ws$QB`cQHlbD_^Wb90ul5PozOCV?-fM?3MA~$sA+iEfjX4piNj3OhJ(tPJBb(gug-f(;gOhNa;qsB?4trj)OlUlFOL(lBKdHqd*Y|n$Zg!>k&5j@gWMu$l)Wkk#O@{_SudL`z)oOo7!xmPR=3B;z`<(dMUyH`m&+0 z1aA@ukZ}(~c{$AWM=qJ2H4+E0NDdstt(OKuL03Cr%!hEuPS-6RSTlz5gG5f{?hYOy z^60HfoqVi=8jL=>lDpXD z^LdY-i+M z7C{QI6jn@creJC^1SA?zZR$#UFm4WBDF4I<$q(=ppyhEG27q0=KRQ!hMMoG<7= zoLVW0FA?Z1)>E%>2_=qrTMkrURY33{E2llkA%2=ySOuziluaVnxIZh-bs0*j-e;~? zez9w^HDQM#bB!y+n6}H-Ol3|`&z)RQk*4wN(<_x|{g9O%8 z5m;_-LgTflq~(TzmJ)0?yMu}>4>yn@A|ZUR&e+BoUi&Unsuz`9n^}Mv*G949ijEm& zt|g;P1$O{xK$gF}iV~q;S*}kJ#zzn^tGuh`@W8ZY6FP^FY>@WSA$gLrjf0)6OJB^z zQNF`$kQpn@phkBTi$=Z(phYP=Cf$Y#(DY9-v|K<~c08C(QL>bV)PgigaLp*`Z4wMS zYY7BerL%T>)EXcv74+ajKO%C?*m4#M$37i9rjSN>!$y<@(n2Mc<2d)Uj~%;N>b2l@ z#aXlkLR5EOc==EUQf#P2^m4MhuSagK2ak!F$+G$DgQdGo=XkFsT=5w*#L+7zpNCXd zIda9rvP|wl`FH5Y<|KsYSj_}5jRwU@TVP4XTz?`~OR56di4CF;km}PrqeSM_s}MAd z>J5mM+-ae0&|=SoU?y3yW$lvpu3nw{ak={Q@#~Yrx5uyDQvNzaWKD!NQM4OKn-Vsn zwy7911Z3|+XL`{gcRcuzkt)!N~ zHc_&vw1a3{RTjdwq+3K=Rkp!fB^yPwDA`2Xs%-Ip=_jVl59e4LuE$QP}wSN39&}V8!1~vwpPU^)k?`UwxnB9HiB#mYMVs1C^o6JNZ3nh4TZFeWhHG!(XmL{sM&25wFcE& zNwSr(3u=yiRV}Iu zX|*M7A~qGZY$VxRB`vB3)T3%v)HalDNUf@ks-d+;)eWSLs@W~1TSaWDrqNB2ZL*t0 zEtM+T3uzldHlntbHY;Lmg=|$elGz(Ihn*|ebAsjxO%0NSeB zB{ov6g(k?_rL?Hp8$)GQ(v6a>pskeJp|u-S8&hQqWZIIpl`X2a6||_@iM2r5MQkR- zt)pnO0=A0SrodZEYMUDcHj8AI)l_Vh*+sOiqAh@JB{n9+Hib>8O_G~Y8#Y@jU^bB1 zL9~IjR@AnL*lZgVEt^)*Hh|hyZB;f%+9=qzQ)(jGfwHQW#WqzfsJ5z>&|5`YNNl5I zjj6Jh%B8VZ)Han3gJemtjiEMFwxn#J+EK8!Luo5v6KILD*rM4sp|XV78&svRhSFP7 z4T^0=TPTLbQ)O);+DdICZ4tFCr7eN6Lt+i8NwHP1O{Cb9Wkt1BZ4$Psnr7e^rWZAM)X-jHVHUP~^8v|)Z z)TYojRBc4s7SyX^4WyeP8xd|H*$tw$MX-j%2FX)uq}Zm(MX@Sv6xxlGYKgQ-vPrcqt(2D4Q)-)3n?$w} z*c&P~ked=lWks(7RXJZEu|YRg|wx#P;Dq|6|pAOR?5 zwN0p|(pw_h6J-sl7ST&#TWl#C3R^0xVM}TwWD8`QLN=Y*5%y zwI;|zYLy#REtRqr_|-=kiLy45Z5G5CiEM$iQrJeon@F~*TM2B4whGuuwHDEBAlj_T zTPUrF*e$9Jk+n&&64@%YsHW1JQnsizqiHKODz;Td!E8%qM#k34Y$({Nu-cPkn@UZB zHc>W$wuH8n*;Lw%g$!IzxxM{8kK zBt7O#8m2;CjiGA(!SHz}r61nzhX8z4gvc7$8Jc zmi7IUbe{m2G%*zw#1SjYT=&ykm&*Qll#Zq{_I86u9Ha#1Le1o5@?Q19OI}|JEKh-w zvG5IQWdrtRpWB1o+=pxE<*b)Dxc$&62b7ZkX)!32oJ(PGYJs0;}YL& zH*@;-ywerTcJN#Qh!`?;&0-!l?Vj>tRz@4RDJsc`xQe}u7su&75MPrTxv#s2>ht86 zqtUDi$T@n+GNJtA+rvGeK?fU^Ke0-^#sC6HMhHIy)rk6&unw#?UCJ}A2_2H| z$UHwy4kQ)`--La(;lV~wnE~XDevde)I#C5m>7UBM;OcV?uH#casd4L!(H6ec%$r@*WrLux|ADWjORVi5jwVbaNZ=|-MQ7>z-; zD;gDY;X1ISodbd5B=1$@SZ*zjsz`!)R{D?CR*%iAXcyd@Los4-U@5-_ft^WRA@RnAU%vS4oBwurr4+7LxI%3+ngk=Q|p%U+W# zqPwMcS~wv%QebqsiPuvEJf5LF@IJK$uRWhsoSgbx*<%!;(m~36rEPK6XCK%^pQboC zM+?#7!n0~A?%hl0JL+=t4~t7O;1s&Q$T%YY_7vslSz~)ijmI%WgDkzf!*Q3_IDmB}+2{lx;J<$Ap zE87X12?Hp{my+4HkHN#m^nBt2#d1U;B))YGKq09LI-*}(Hyg25fh7dISvN{JYY;Yj z6;DW-5}CG_rg1%8mC(#7&U^#sZ9AQH|yC9Xm2^pSya_fjR{m1>T(wyCA!k;iF*gWm$YIa<9Ef>8W|0FY84!T^Ke;rbml zMTI`AAOjzae)z-%Wsm_<%ou$3G7VHkfngM@rf6VZnFGECn-_l=BT^CiH`lmZSvT^Pq+^Fjv`R zTC~++fK#atesm1L73C0zToYOyF6$-V1)>}E9tDrj)$Y_rFzW~pjq12n1h6TlaKqMs zNJb!xlw4KpsBjxDE~Tv0RfVWd?P zM5FVKGL$itO!6i3sDDLec5j>kxqCEYPT#wqUe;H0s5?oBrf4$O6eT8AK+P_?fjIj_ zKtz_GuV5f7EUff#{G|ISb8bIZ3)!;R$29e~JdfMvUqhd~(=6!q?)GC39jc!M2BFB+ zalVg*Sqof1@8w32Sio(4&#Lm#sVJ<3`MvJU0=iC__1Vs9h9c;ypJTyCn{I-? zGlrY!P7Fge0VuL9#DHBFC=tXMQ<1E>BFl)xH!Msosjfzqo0wz#d2`l*`9~<&qR&ATL zbG|yAk6e)vu3VT~@s8FEIK&vUKnVywqNmvo#qiHLZvdf===|!5si$M-c%)TG z@irMjv6UGIPl&A8$+0ZST?voiwt7m0ak{RaWpU@k@N*m?L;d5x#X4Z@CJke_W{ zh3mPWOXfW~+o#oK5A0~whlhu?Z^U&7h12-6&XZ*PfL(YN9eoAH_-3kmGmsG}d(|;w zUKEW2k-!}b=e3+Hs}A!7RXPrcD{iT=sRN zngua)z%1G}8y9I{EA23uhm%j8suR2DZ&85HX102!>@}3fq?(%G8X*k3IqC;2CwF&5jL9-YJ5<5CTA{o>WY(FP=u#E}s7rXCGL;{R zqkD)i?gzHS_Hocig*#GJg%Hhnc-Dd)JmAF`C1To}+8rA&Nxd!Y@FoNf?f%kMT9QRW zl(HFIk9~zKg}Wy6n6T+>Aqq0LA{?Jswtg_B*Yn)858K7Ejo%1B?`Zr!f};EbIjyEj zJT9t^zo5;+FU%|RqP|3+gQNK*tLWV{BGWG!5HLvqfd?WgRPTVK-l@WX#n^)4&a!hE4n1hH&zzfg{Bi;Gb0~b zW7Ollo-{lW?%&9$wyS=)=9YC5WF@$SRN@P`1NRU(W;Ghrz=A)VVr!?KtR<1hGzlQ{ zamuT)F9e~)pdd(E@9n%{^`lU6+#(Z$cp=|d%cOX~N3SQ|@yq~~a`=$2=8k!*gTtUa zi2F1qhwOOs4!j35^e~PC72~GMWsP=nDVr7q82mMB%U8aD$<4m*1spyrnHdCSB9qFe zhLi9kBP4cPCm)?vavibk&y|rCn3#q_ir)_{w+fPxPE7tnMKBYRpDU)CD!$M%1vl6D zHjv~aF+vu(0vO!Ui2MW;g;%0{1ErN-Uv06v{bLOSSI#SVDzSOczcE{**JF=0V4+*) zcICNo6ES_|p$-IsG7OZkSb)yixEwo2b*s*OU99vx5OAfqFqEwp7{KzyskN5Cu@sEr zaxnCW7$=FE20Nf;7z7P0=Hlb!$$6F>h*2H%Ku%+xEjnWArwv_je2|0X}I&+30k{*a^>hh8t}y3@PXNa56oLX-spnxz$!`wC!RWw0W$Z4KJ^rYO

@I_g8pdMpw=w7wA1)l!BtV)fQT7uQwgD<_`eC;E^P0x_s z!c0poa_T#NtO;~a0?e3OcbV1AXqZ*{<|V+KIXMDESiU9)!yx3f!oGNGR;Vc!bLo`f z_-TaHl~OB&{O)dKu1{~dmq14ijjYZX5?8gCJ5uS?4On}72sBO`EhA^XvD{a6!RXK; za*i6iWCR9^GubdtJnDTJOb}4n$ID>=;K_Np*^tbTgt#U~LQ<{Z9~%UU zZ`2k=z_cT`)v7D?JL5^>8E1qq$w=_3OR_es6+d{I{8IS>d{k(DFbHKUPm1P2rq~TG zLP*@bFG}7W$ij4{UCJ+vSG`8agT1rQfD|!pLdT$TG^^NOb_S{aB&P|d4k$s37--${BA0N zW5xDH9ZOe}yX59yM)aP1a@Xl^47go0YD2NZ;nD{QEx7Xn3@2ZKP31c*F=+IFT$azt zBqS07dl7L9jY*L!XU#Zc@(gC(x_O_jghZih#ST88m1!-c`%SHjkAM;#p)(*fG&1r0 zNM4otOo!ERXLs<_uKl*cp&mk-J1#r}ppia$nUEgR%JR)9zVf?bI0j*0B``=(Dzqx? zI3+Qm$>-6zZh$MPec~E27re`$o$=~KpvXd&Ob-2Ur!!60#2OV=Lm}CKqIS_BwB$}6 z>QTDe1EK<9IE`pVPSdJyTC=cy!0Nzs{82cb`NB06Jec^V@mK*@w@X6W(TC*1`34d` zKZAgX;OCc-N}I|@br!1c9jKMQ&J+wLUEhf6(R&u=8uj%`QLkip8XWm$*2)$NhsG-W zRxJrDO1#o~B!xho&X%46_ju>9^g*j@qQ_=rZBOHTw3Ti>`e}<$2U?IOI-?+NKkfG; z~Y1m7*V@80tuVkW=U=BgHY>;~QS=%VGBG@4 z#_B6n#Qhj~s*)xUfMx{MKzx#@}hy)vr}r%s>7-cU(q=xSHhEa`VCQ~(gbr66xN}s zN^l{mM6GG=JUPfezN02+Qqi(=8=~q%NfE4>F9BW+yzJoaebQ1TAxVc411&tUb8rcKTOAIZA&F^Z zimb3yZB7Hjpc*d?Te?9ibp6n z!j*4gv(BiofGZ5gvL1kOP~_j2Si}TqinxJ~Ly(Ohq!~Ka;(4ONqwEWH><))J?QNb6XIEd&VG3-Nex^>9NZ1UExH@n^3nvjLo z)D%1>P{5v~-%}U|n!_qx+!QI45YlI>uS$BuIx9Z&UXRz@kP}MA$_*~viuOCVDSJ(5 z{^{2}C3Um5L**YWN?1m=7o_C|`S4sQ7&5UjOmWW7Rzx+#C{TOUV8iEdM7IL^z-m|= zZf^B?h;$i_#8ruE)(kh#%vsA`1SYjC9Fo!RZx0Vt50E7V*w!qXEL-`OFPQ{b&Z0_Yzz{jpTGB5N z6lIc>(^OgDk?rQ-Xdc>c))cEAGPvkGD%EMGF=Vh z%b#3i8|TQaecMyNCJp-3fhV6=Grgsv)1@}jmnl#i;|u8PV7-GHZT=EecJy=DvPyvtuZgent( z^<#jDj@#F^MgpbE{uST@mrsmI+rt7Ucovmh+X%c^XZ14-{1Q$0D zf!08mQ($aW4dDwYFiUc9v@VkFb-sg_ca(Vlqz(CR+AH9G>ij(N+5tvg>tPxlUM7CiJ@VWd{K{@VA>oAlcv2g{`U>}~uenRY5(e?30yvQggU5Q&%#tKLgCeMZVnh)D_;7)ZpAe>S;3_3#` zE|E|!qri`@h0NztwShW7oVvGuOLe+1^IMa}izP_B#}pN{ZW&8gOK&z&+}USAt%lO0 zR6@#yt>g&MzJUetZu8QHP8&HmxX{Asg9w{$Rca$7s~#KCGV<5FQxorZmV_`_gm5Wb z>R6bB3bii6>SZ0H@%GHWX~gOGhRO`RiG;lwdFp=_h}o^76+Qds%wh@W+V-fRi3N?x z-?f}ULeIECkzu*Mg*CD!rsQ}czhWcFJ9INTJi%x;-dc5|!7B@ZQQ{qSgo!iR?XNBNqCzR^;Yvp-41l9YQk+^SQZvM%?Er60>5E70_8V`^wczPf(8fwT+r5sCo4jR6^s^ZZ{(;wVR8= zmZ)ChWo8Vwo}(6nc(?B5Iy_e`Swe{)aVPcbaUC0w-=WnSg!X3vlMrR#BpJ!F!}7ALD^>!oxMB+pTF}w6?z-O)(;+|%evENZ`dL|2 zAb0wz453nrXYR-_a zPa`9snaoD$0`KCD38ye5qBpKEh~VW_b1abSxTT8M^e0LrKTdk&A;H`wJL(CJT8PxC z%UW^Rxq>tHJ|I)V31P2?A>6{nZpIicjQZTL`5~nK)hMumu`tezJQVRv2_k9xe!F++ z)K^-LFHw%mZUMJ_e&dfG^nhR8^8GI=9pM^uae z)X9gHWgUIckO;V#q9&PTb`?pv~dR*q7!<=gGfFL@o0w+$|2$T zkySoUFQZVT$a+2@2ugfEoXDul;6l~6x=8>{g>N9DP0W-}?j)$BJ-`4E&Y zNPO4D;hWQQ5rmWPGJ3RN;^L;)CI4M> zk+>BDRs-aP#Y#-AUit(kG7)^@R3%b1hG)Xz?7=71&GVTQ(Wg;+NpG|`w@yuAb~evC zsPDRM5v{X=L?-G!dxq0{H0UQAK37(e;lUttVT^=J9vto#VI}yI*;=>5mX=0*-aoT0 z<0X27oQhds9un+FRJ_2Bswa4J?JkazO%~oL{)h2 zcCi-4VGnzuiJLvfj0E9Y%CAwG*`V9XJbE1A&f{^CiJZ0A#5{Co}#Z+%~ zQ#T>SK;eVw&1W9)#*Dy3QC3v~cs10LXq{F=}2Sb7ohp-SaP_hwtI?gnrrLYKIZ{I}ij8t#D z#YJ6wLqkc*6n5j7nhVV0bjZ3!hp;Ay++2f>Diwv#5We==yopbc)qfh`sUk&FdAqZ(j_}+acCFVLo2; zaw8XR0|2QEKEdJ9QHLWYHW?TMCFwB43S=^vy85__?#Q-1lo!11<>$XBm zV?GqbXU0IR=oCtGk4%f06y$J8-fNxA(@nl$sD`z;HnOi!kDkDQoCFYKl2i+7^s)d4 zpa&lT=FJtO#ACwLf3_(b`$>t7wUuJ@CXOljK(NiYp@#vjqBPC_x-0N(aC?I*r zq>3MOb#l!H8>@D*@5)&-Av{y5UrBIu{DlaaW(L4RWo}N#mP=yz8r#sTz?C7b1RWZ8 z4{D4A3LLtT8Y&=56bbgi4>2^v_#n4vEsEBFLd#PTlf?J$uhSd^lH?S2!>RebNJ<4% zXOfZ1Np#_nRL2U@Vf5pGXlhmE1YP92>`0j4dS6C@5OvlPK*m zFnk1YW$zXVMFFhPO}8{#-NiNMg(h zAWtJcq=?k2jPN%IddWrG6}U2%axSix^xptk7O}?fs=e8wl~oh*48ft4!--x4l9)CC zp^MUid~dql=F8L;)Lzel{Q|Cvjps6ymf$@_PCYblTNy5a>I$lb`1!0HnJ^1WR5e1j z0o&DnUIgUSlVbYPh$Yr>O@~FF63IZi_4|Sr1gex-Va?t{Vc` z%a98G8C+wYw6uk!@WbyQcx|-Z=tt}$UgXXAFk?k#N{8P@Muyz z(!mtgRl_jWgE|@zZFD4n&)Mj2hZ5Yktza>|k=k+)BnQw-VHJ=ba&0cXh9VLXq=3F( zBL|1zrmn-J#6A{As$&+3X(?9sLhf|4dw3WC!dC0<+X~LtO^#a5$#>82-;~-Reri-r z=n+37ki&0gKWB^GCU*rYP!uWb@1nx06z-b2bdjh!-O`MR~`419W0WC1N<*N6Q5`?2<(u zk$*MA>`UXBaBa#G1=@h&lO}WUWz8d1`qfX!F#wI$&~l>fGU92ZS_w5VAl{X0>lmLQXD`~6D{!u zNm18I&G^NvQ#tE#y!sM|&U~Q(MD#|zmgJlUz2x4-?`Thlay%P2B?I#fi>>ENS^UEN z^YIb+$zs7tsDM>qVAE~mTF=Lt5d=(X3qgD_HAf334yuq#k;&!z9y6B(TGjA$yajNQ zLDDD&#lkfR(Z&Ti#KlVHN04#9c+1~UJwTld$!?Yj-Y;00W?k5zYEk)KxxLTfgzMbx zlrh4U!rRVyH$hsUq4ug4E5HkeK38o__NGruFH%XMDpjPT%*#PAS_42ALj@(tVDdfD z3$apA13P4(aBM{nQv<|OSY8A)VAhvt4LSSfwY6bJ43r7szJkvbjLa61Q;*jEW#WjTP{JX% zN8{Wm)j;6GLtTZSALil6Z@lRVO1=A}P)!TcRa?y|%=+sjF91~HyKCAy^-1D@@ste+ z8cRpF4DON4dBAtyV1fxvnZ}2ru_a%F zBbVVWgorPcMd#g2umV)nvB;kISWG9lz=D~OhJ-xxAj8^hYr3PU-Uev_Z#4!`Ii!U) zYgpm?3PeeP*LB!9C_C5Ns~#(4m+2rL0WByG7Sz5!Yc*ULpN8PN>nEvCZ^j+wr)_H0 zEfn&UDvf}FiHqz*h@Wv9Xe)XJQ(wVmgH6g9N>qo0hg_rs!Zi zj3SV-I=~Jn3UsxaN?3SBWF~j=IV>W^$b0&rpwNN1fTwoG_iH1{XUdtshvY>A>6Kx? zABNc(5XGN8HzQ>~M`K5ij)!zbqL-F}g&0FpzBCz^IKLmDW*U3j8}CcyPzu2Uz1&iK zpv9tKg072yG1+#s;$tVDWtXWi2uq_VI29bHWz@$uo!WpD2Mq7WV(PBPAd8XPSd^9G zmQ}16p1VL38<5+GQ+6_~xvwGuZHQlwdP2|2G|am;x^s8}ETtG%$t`o)INzOrGWtTI z*N5T=inHC+`Z-pxW{$C?HUQ*b?iCB06~Y-^b6NwOeSGrq%bT}f&%w(`zG1fVGTQJL=R!i%k%(Rc zY6Pkx0>6fy+$xZ3`c+G8(AaeA)wt3)C{wX-b-HO=I1>G{?0gt_l^%BqSzZE$+Y{pD zB+-~4>IvYuVj1w(MN}TvIeTU`P?%!U$5nfNUaTA_5 zvSbbN=pH7fz8nlif_2t_;CVs3S@*$?Js>fR^#h< zUy}1Qz0uS`B?PTbn!p*x>Z&t(S* z%p;%9huh0N8s^E?gQF}8JIh|bY>vFgJd6r;WJ!oW6QpKM74wH9tuB>1s#lj0bPwK% z6SK71Z)4HqP%!EgeXZ`A(){(n1FGs2=Zh)?Ud$f&eh@ZQcM?k98XlL_K1CmE-;`QY z5akJU4h>g9U}X^^r|JP8}q<6{@fpDiUin?&~arwx@XcEl9)y-C5&F=QjO z1?3GDl9wVCyY*DK4y+zqMLGIZ@bZs~7=-|^+3k9%h~3nZhN|pw+r~SBl)qbnZF*ga zf_$A@6E7t&`))FNGGX0-XoSNm9CpJ&N^>(<-KEkB{&VtxC)mnuje?YG_NuEBsiF4< zD5(7YI`;;A)0gjpaB;Dc3|M0HXj{!sZ+3*ZBLmu>7%#m5YiS{khqX8Y7hIz3-$SBs z%x4bdcwoTmqEh+7U)-2b$;Imx&a`Nv^tV$46;;-v-<&z}3Z%?dL8VLHa1LCm&nCnz4GgJ04qNLWcJc|0 zmMai*aM-@B&_QYS zrm_13WvU`#Kw*pS?ga3}CuEJigaIE>V$srjQM%q~1A*=0)|DptQ4Q~-8!_S(RtRd$ z4DSs<(W)Aqsj>=Tsnr+GRl6gNoDr$x@g9%QWEl_SEh#XgF#;1(3OpnrkZsX&H}$S5 zfYxbdfrz(hK)Sj~RF`I{0PezvQ;V3R%HKpL4|0f$FL2oVLtsG^oE6AoMDgD66MRw~ za5r3Y%=WosSRcG;%F71oLcB3cPl_q+3`4x7)xJ*K?spl6;haeqKCOYIVow1zKLFRv z+=kvGe!pKLF9li=eboS7o`8P^btdm@tU0tmxdR`vo1oD31%Bad=gRB?%qZ8>3YN(= zE{|Aoj}k$A+IL9OQbQ4 zZ7n}~%%PAn!k~kpOCnuXt(JTtU3Cl7khbD7&j(|?!KwrEQb}11JMVSsiuy$wj3`U% zG1Hf$k%AzQ*!gO^HFWwme2Ei;#;Ui+8nx9f43pa?Qnv)-PbqSmGmC_yLCo5|Ih9yj zpL$tjXB{}?92|s@`ZX{!5l8K`v|->x+`lw&n8VgIENPhn@^zE|MyxnwNzbTqz_pyQ zy&HMn!o!%Wmi!0;AsBiM4u}|P3A-D3E=3fJ3&NxIs5iXGUqN*tE3peHCp`k~SJAEr zS?Hq&(isqDZJ-r=8r`z0`U0blP`+}3#a7d_3A(poX@>Z@kcT?dJz4gft~d#H#VU;< z$b#ow^Af7I#h*@ZHeP0yriU}7b5Bb}H9NT62|q2E+trk}0x<%YIzx&Y8$y3Q;m8VM zmaD;9Wlvd=4;iUK}^DhzHagdDu@!cO?)Y@}R|;`wNdtAfWy} zjb=9{grI|(?cYN}pzJG<5vXSIoyoXLg=eYA3@05P=H!*A>X)=R$&*e$SOai*2&Q17 zi>y7ht^jKBS9IH!u8(CW=Rs-ldsKI62zD()dlHUtMek#(G8<%mAlCS=&&v+_@PX`6 zy&SWyNmYt&TrHtV1Ka_%E?W7y&AJD}HRaA!t1y9a3eJM-EXBCx2YqK*`4BFyDJ6qa zL@*);)`hX;t?cIRpGUJAMhl{4b5_YzR?zCTe1*5+k?Cz6cQrS#G9I^4UzBPhS~gxt z5$us`CqxWF(Q#7rxa)CBg;hj#LcZ%t!u~SRN54wTv9baX=j=nE>o8=b4T6iYC33VR z+s9Nq3N8sUr+%Z8fsyPdvgUO#~!}rK!q;h{L_?VaF@%#X>c#v#l zW=YacfN?k%x0|n#gEM>63zyVc1MVLPgKu~*kzs5Hy3p)$u^UZRSoPqHELr32*U%9! zqt{dEdl2fU%y&MC6q{`20z&+UGy(%c@b8WfxQTqr3A%H;cU|MxDQGGX7uZXM_tFE| z`RmCYV!UHYVJ^FugHs8T2nZTmG_O1+ftN8uS7R%5`WqGf2Ur!~ZwGwtLj|K#ARx#= zRC_A0S~M5D-!U^^GO$u0fBbFbOutjdU^LN90 z=(9vI&C`Mc8?n0*B;$4RbU1pL4qPR0yq z1Tx}OM6Vj3IF3IL0L9@z%W_d;Lh-TW>UT&Nu2ylHQ8%MUBC^E&>SSgkH`3)O?|s?& zh)wIIaVJpcZ?A52*KSaJX#SI|SBTDlq%<&{9OfRE$T8<&+vkGP<$ zADJb2M)MF(E0jj<)vG!~+YBzE_KM9muWaC$5Dlr~+(KiLSt}Q%fMu4*O@JeRa0*x! zr^Hmr-Qx-H{GC)qRRgN7LUoqOMhV4qZKhP@!}$^dE({!k;&Raa*g*mkA-z?0W(ZlY zVy^q%Cs#b!7&3m}m=hkH&RAvnOQ5Ga*!i)I%y>hMLb=n0M@?MklbdHHe}Sar*s za4(pVv1Z(oLf2(DL>Jo{#1K?u606KxR|Xm_mS{KG71?M#_OKmYK;BKjdq>Zz9E2Vj z;c3&|fbpP<%KcWmvIS6o3>3?poLfASdLvbo&lo z6r0+st`Pn=ac!~be{AY~r|0M!p7TI(zoF#>B`OUQ(44Srix z^}@H$v@lnic_6HcfRABfFaibfU6HB86 z`NhWZ7&vU&9K`pBrZ>qEix(g&1XMld?qSQY%YKs`f}FC*46kP)VHhU7l211IATU56 zz^O}#=SS019p+t2p!26kB{lK)pwB_pCUYO0WO9hTn@aS1Yhd9S(QV&|{l~cG@=4{X zUx5!ms9Hz}Ze9^b;1_{59Tlu0DG|=?#Ay}al)nsheeIWj&Wnh&eWxvl~za0aB7f*gdj#AGeQg0r<(9-m6Edx~Q4II_bNIZMI8 zH$vE?f{kC2wz|Z~sz@PGP|hERCOPn?BBVRdyrno+8_KuneF8WUuAyPVsFH;_9Bz>d zwUb0vCn%EH`K=V)U>5~XV)KD*G6Lf0T3WzQBm|am6rW^gfUSrNAhcBSG3;8 zIk;WJtSChyD1r~B=+0MQF{oFqxOiX4GdhGIj!A;%_3 zHi2IDz*IwWvS{=ojcB;uEA+BOO~G@Mh7*Wzvlf%oYjA}M2xJ+V_(ISN3*bYOB$VG- zM(xDYbxkgf3*}41h!gUQtm?SxcO0;Guc-i|d|BNSkSr0qRqJsf-6&#=KZ~p({vebp zV;`(hprP?wiyT(2J2kPn2Dx&IinYgATA~bR)MnWn1wT7h3OKs;-@zft3LX}(kVpqG z1?eT1P?Ev!{YV9B%6s1QRCQyqh2pO&^7lY(rh?_yRRc?#+Q{2_Wx;P7!~*ek_NAl- z2xMva)J&xt8x1IUq`h7u>s>F2xBz2mL)X(~H0I{83rsK2&AO)WrHxmdRcCiHtb-*w z3g(gPUp2(gwC=)@eH`G0J-H8phNCQ4sg_$+OGLEyRI@g4VnX~#VlcRIVyQ|G@PzXX z+-`S`#2k4Xia@U}k6pl92pUL5L)*qVY@o5IcMfUCfmV<276=;mDW|3%oTxDx@|o^t z(b|ia@}>Y)cec*?<3rWMUKzL4`8Ee}`8D_~fU%L>mHIw;X*4v$mxaaYcNfY<+GR(H ztam+vJ=p8mh;9}e0XYrx&}56xi@`A4fC95GaS6_Wq#$#}*Hb3V!;90M=iWj6L%E4S967M2dy~=h;6yj$o)=|8lGLt8XQ{=VTW06jp$zpR{DtGq@bV6e(>3?XDzy3VYMq+Id>h_)EW;%5%I zA4)J~EHxFu_jSJoB93v{wDWl`#)tz|Ziv4RQ=T>UzUkq~t71XiDheVL;qv0~upHq8 zZuV>Uj1qSe;}V>tD5T-^-Nuk6Fnq45!C}Gc#9T%Qn|^0r(z^Gg#G}S7%p#D0?3#vw zA$&{NBiDF^H!XN81|`%6D0)QYVUgexS`p28SzE6?nmcYS5x*=CR73-hOIrOl__V1@ zx%Cl9yt`YmneI)d!NYq^_8{2Ns~t;1Q=L&`$lkAEnn3OlK#A-Mpsei;)iBumzhjj? z0nA=e+CwF#@Q@K zr@Z4!yispPkhZMoyI?GY-Syqh$-kq*Fr?hywe>+`l9hL0Az@w_^Nur@wI^f-;Z< z)Yy-=a>s|7_($J*R?MVWfFx^uYKk4^s?=zs_MGy4=k&_M=qH1983?q4UFgo+=c0O4 z#|sV=Km>D9!ElDmIJBUn8muZ5JTQ6K;S(Le(K5?D{*}b;g_rmE#w<0IYs9r z<*jqfIgNg=hf~J2u-1#BuMjKse(wVI8?W95h6P51p>AQdHeZ;ci<}Nt(!wf8X!%$NozJFyA<&~eutXWB^Ds-+5o8&sodesF4uIzZ z!?{S%{?qz)m}wvAFJ1W$P#^@ID5ED!pC+v08?zrF)2BmvS2-rysp(>` z6g(Uzfwim&12m(87ez)P3w++DHecT0>h}tqxIxvc14AVJ=V!IC=>=Yf*0@9BWqbwW5~*y2BKoi_<9zc|%YG6(16lOirhbzGbKGIL4n1%a%7LzDSEM4Hr4{O&uDPLWnL9t{hdT~&3 zlG9k@b`B{{w#T>0bE9(N{=_3YNDOdK(^v_NT~;O?<|bd8*~J`6LY;xWDH1x{no`00 zgweh~67cfu`do9ss0T1?dlfnlcVWZY(~#5P_xV0$K#9u;F{x3Cpj1>^h$T{bT{S6Y z5==ZKX}&lW#)Cf-vEG>>mqMyieZvN3Vh%mksmNxER5COmq9g1jAD}~u8|iX)_)KVC z4;4Ww&g4pa$0}>m!6WdB1i8MISUMA;>igY|&iX6gUzfOnMdn4(+~kM!AjB(x`RGQ^ zm^=u*TX~Aoa(rrVtFzCc&MFUg&*pO<5mzr#Nkk@7`%0qJMoDWJCZmHTu#Zk&(Nh}C z8WpLzR}4A|6$Y=~KOO1lvZWGTHJ-N6>Rd;&ov!>%-EBJlBH(o1Fe6$@`my@nyw%Zr z5#vjidL|Ag!&s%G&FS&l62zCnm^_|I<8EXg8Q|IKTScZ9aPSUZW6QboQ)6412ksq>X3JfyfwbAUgR+C$VBiez#8H>Keqr)~XLlnGGn4pm#85vu_wn z7LZ^Eg^^3{Zcq@uWt)YS`1ia=AuyPXRE3m+hPm*tQcq?j(6z+WYa6NciK(<2N0W8a zb!skT{fP|i&OBX*;E>HDZC{)B){wY3mu7>V@lyE=kw10$n|AlHo@~wr=(`FzJ<~jB z$Gr_HwbK)GZUx_YU@O!^JI+nC-;YjmVNOUIH+E^6APXMk2NEL$#9OzyTqe84LAz%j z7}%0GEe!|Md?$D}@s6c>MEK1REM`SGGuHR<1rg5?L6L$Ad{a>sxbfl#>-N`!V(*$q z>wQG>SVdBTkGRQ7rSNm??@oGH3p77fAgPod8O!*lLdsQ}6Yy&(+=C;|?Pw0)g#?+y z3P2Q5goIR4nEteds1Dck-F+KZLbMu09?yHLtD;Z@E6%g?YqbBAkIcU?fnM8$t^5^H z08q^j+`*0%vo?@BvQ7vNp$4i1cC*2Z$(6 zcmYT>>Mn@t!1g*QoE$fGTH8x_ozNL()hxp7dlAH68ihDZ z2{IUH;zNUZvg(21T}IG;Mqfd6HxG@bGQ;F>pO6`a6J);<@Yy@4wvKszFTx!Vnd^XrL*6)NdJuhQXgqtJ@ArNMA#FY-GH!;{3vP=O4( zs%F$!C24rASzU#_^;`SWlVtVHL3PR5p7C=j=s6%SKm-fx&H~kFpyrLyTSz?DPl<#V z&KQ`f6_3zE1y2jjA%Pxnh6GTmc=iw&A~-aSiE%M-?d;21mtTl9QqtbHnc8a(mt`lX zFBTnU9ri-X+wLQV=BbUA7TsQivW^IO_fynd6PH4?Gu>u3gY^{~DO1Bw15kmu)}Av< zDl$3c{W9t8b1pE^_{w7f(e2xa(4u*t-iHL-#S1}a_fD1$+X<&z*X@Z3_lN2s6IDEO zEuZ$UR+58K2Vv9Xm$wkyFE)CY;T!?+SdAp&#}Vb&IuNmkD3axlMFfhC^yAv2o-x;{ zcrir^5X(}jeHO_c$)5sodPvN0O^Yz1+#P3AjRKbK$C(`70PGBk>S#5YSv(^YnM-gj zBks(8hjXE)h+IUC`LxN35$wHvU#YQNBdeAljJK9oC`D1w0ZB$;3aDO*R$wGY<}MMD zmH!|gxTO%NR0Ht&Nk}RWLNutNEn9&)P%I%7j1od55v+zE6hEd~(}Fv9z52PB{}I%1 zbWw=AOrpU;%809~Z5UzgX3@qe27O9gAQNRwwo%aO3P8mQ;f1cl zg%`O8+>gk$rT~mW;pM<1h(hg)uT2-3I42*Ndj(wa%VL1v4?kx zHlDCZA_G*$`mhtYBqwgYes3NpRrqdbB{b)?8ka?(t#V^-c|T(^^X3UAq2=z>nW+yT zeC-h8ma@2`@T`iOzD>#$Fn-2Xi<2m1o_HIBMVR<%w=AO}w8$4wpQA~gGc!@y(gOp7 zmMM(UuDH5*E7slM-&cB>n^i7vW3gcdd|w&AaNllGTOwE+a~+^LJWbiWXoe_a2W6i? zp%e+CRYqh(S8!NT`dsr^R!@b#jLSB_WTKY3=5;xV*}|g&+lfOx@%ot*n~X2Ca_X5J zv&#ss(A_kZ5m_%#1Wg#4G*G8rsfvPu(2Z_VyW#B&n2f=ec!*6saIp|Wt~xJLw`5+_ zYUTtCFJCLLPecY^UpfP@lHyM8i}7)KGKr}Zgn@mHC@gC zk=C#-tG3Df{6yge(ofK#-Kt1lvK@AD%&pfo4+5ZEO&teqpKOPhpBX}Ey6~9Nyc@+j*9={WM5Eqdj zEZ^#MY#zsgii()OKX*k5K-s>jkd@dfpyQUN7{T+hCX}sWME$b->Xcl$x_(5eGZqLb z6;zaTha>DCL?vL|3Ovp8JtwQ&CX@{#V6D$KDSb2@AC{P)OGZ(T zN^*mK&3};MgFlT?A%wXB$$4a-A|#Q$t#|hNv$MbL2Rn zmWis3Noncv%|9AszXk24L+B3eO20wNndgYDLQR+@%Go?aKW+L9wMU=4KI}w?PTtCE zHY(P;M^e@$(KF_t)CXYF3OSHm?L#IU0yrC-K)43kzDmF+Mm;6s%h6L#oyNLGdJhLH zI4A|!qlG(J-a}$LOH%UwexoO}2iI!@$Pkji!lo&d4tOUWxM{p;LS>*lx<#{cZZ}Dt z%?2HC@bATlEdwyU=s<~K`rF#;GjtbYdqOci27eXT!Nvq0hLKmICK`@J^2moF=*nRz zsM#_E%SzobFQcv0Pa;h_#+ScGS+&);XvWfnRI~&@)0a(*3$q!Dycnr30x#V_EjT$K zd;{`d5oeFPXz2i(_58?JI?&&%UMTu^sFLP=Hw<++R2d22xdb?9MiNLBgQtS33rJXB z2r77|mL775JXswreR;2(@-UU_<-nHiao}hwJ)DdMOwz9!ngm6;0&k@43hhF#%CyYP zP(o}&u}RlluVI7IfP5;;?OpN-w%D-fyZr8)nDFl8u?5|p!w?AT0MlxXOM&K zuFri|{C|LWw!;}SP!Ol4BS4$U)XN-9M|?^T3n_}U(nL5n!oN`v14wEG4#BhHc_1&j zhtFeZm=~u7&jg}V5!X}5DRROU=EZi$GO5(tMg)HRzUG3QgH-Iwu-qQxw0KWHVaJO4 z9c?dVs!v;J3bE@X`+8lKSsoHCmksFzm7VW|nP$9Tyr`cyPClwk!SGyFd7SMpbW*RY=;;ad*sdZw4a2xc%SI+t=hNnxyJ@Vnzd` zsB{^l=V7X$%7OAGl0zR^DLUiwEmZK#>F;L7t}Jsv?XW)BV+{?Zm%O&(IicW0)guun zs*VRqbwQo3;KK={m5P!gDBzBNMTXm3n^VybczYlzDz+wMkY zjO>uCcKk-TWwL>5TJY5&8<+oUr1=CKww9EybPN}Wnt6Nz>8JU zS)D~e^tfXj%0ajDk z!4|quU60PacT}wpc-u9qJG$4Ri~}|kINlJP2qM}*#P05<5f?&$jp* zH7;UPZb7I?xbkgfPOci{lP3g=s`J2ZexP=F`pALyql0^Go!$4Lx_+1|weXX9e$d0;MDUKBqu+V1QmVG!bK;R@NMAT`;Y^ zx6*jxZN3=e!VfIlyGt?n(PX>n;k-Jgy9dPe1 z#JLsG30Q)(byLX_=riVJbl_(p*T|v}w+T6Hikh3@$_Tp#3;t`;IA>NMT`|kfJK)56!4*klAmpcnKDen7^xYyFT=u| zObsD7KxXg{JU<0GUP|aPm{{$F zvcqdcNFGd}JCp{HW#`CKZ_e%w-n|Y+gZ1L+xHlZ!Eo}OIr?FPdJf$IyCWJP+8v#tI zN=-n3!*on562^dVw-&@x13=bnRWi?wr&U|Pb6&1AMrItxf_l{sQnn3ZsoO!IDG<;D zAEXczbqG;0(#h%jj-RrDeQ;M(LsojVlIU+zeV?jz%Hw&h-ecy4UapXk9eI-1_|g~A za&3Hw3GCy1$cjKHXC5+)U@{@0&qb{CGMG?6?OeCbGkT02VN{zi+pTn5`Ym-Yer+0a zg0RqLl%itZQMU`VOy)0_bULZn^kWTuE{5Q4!!^3svn^w9Dad{$^h-sloMyYr*ERDM zb(Z7CGj(g7KY&?M2rNQ%c9IweiD=C3Mg~@2g>*isQ|?&eH8EnR%mOuVD1=qb=u(HK zg6q#lOAu@MMkl#V@}!!>oWimvoQ^W07wdsVs%ix8*j;&Z9F|0V4yj3GCM}N6+S#Mn77uc*n08=d_`~SD z9v#szz}RNQ>lgBSB&>wY!!J&QO1aMLb`nHf=OHr=W?1g*3~{F0ih^?AX`?W$BiMO| zZqsom(9P$mUZVS5v{s8B7T$_3egk-!(P9Hs>sn2SiZIWKQXpA#5NP+y95;)kN~3(L zjVSU^P<5Mef&@9o#Lv^g;!9 zEu8m$Rx$$>msH7L0vS(@SfWsYzpfM`8JJsSN&`^}t13Z=?=6{D`gb%;-)NrWg2>qX z5fxKSGb=9-A|f3V;tH<5+Q}In|m5-@c5nj;;~Ah+?(^w zFOej12gpLizjv=CyrLjYv3;P8N!NK?WVhwP>zEyvcPZ!Si|QRNq>F6zg^v5RzFcm+ z1;RrHwb8Y>xO9I()M3Ea^jBf1+#S;#jYv|S2J*zZ+`C(quxOJxZ~_h~5P@*&y9?jF zvB5OF*gFcWA`QpHFOS$2u9*+U1VXC}c%K8-3xj&oVN{+z)?j-j7z>dwnsrzfM2anu zbL;RSA=kUr8}!nzo)jOf`OqkZor!6PeNiCXjwZl$z(Zmvg)izK2$N(dKDSTjb;eHsK;JjX!Hbk8)V<5M6yX ztfWTjd^$tb-FA(=R&fh(=*?WRLscbmBxt!v@=e3=#nNdMRwCxo!xw9qD2e2}fExWj^pNm!ur2WpA$Tu-^rsWI^Wa zfUUvOU>GN%iBwl?7kNIpM!pcI_frx_%aL<&=2SWDQhnqqH3NX~SPTJ=bj8ZK$BdL^ z>g@AIT@D07rH?He?6OfA5xJ@ZNMo5)(fU0zY`UI0qCsK}lPSdd;Qi#e<~~oG-5b{X zcsI?Dhs6Y=ylZu^S0>?40jn1>W1`~h$LPm}8l05JafZEfgR}yuxSQtT`*Yz~=1RQ)`+^`4V#{3B6BN#i6fERuCa(CTNwO`J+0+1S*V)7X|oc*bHzlTov6_ z4NB!Fd=iQ+Lm*?~y4l~NR96Qa!sln3hcJn#?MfPzE{6#FBGf%^stk9Xx~tKTMAh0m zZn+NKG^ZeW(DlqK3+)Vo*qTT2N{E7)d2|^UDYa)PFWW&(km*8JSFO zIaUZ=poru<>XqC%LSHreRE&n*A9|Mg2_za2f_mq4Y&cUr^cPt9Lp4l$xjC+iRl1<~ zLTO^62bynH+^8-)LPg$NSFXGhQzo$rcRBQA11ApDgTf+OGYF%=q$`g>EjSEmEEiZyR}RcxnB~M`E5)=pTKWestU$ziK%|f(2i&Sw`$aDFlR%m0 zks#TAMl~C3OTrt(o5$QEmVi6`ElgtscyN~1!n8Lj-e7S(Yc~s1jA(wG@K_25U;>?F zd%E*fb9xXCw>3hi&ZVMhK4)IQVlu*21>kxgpHrO4ViAad zNMegPLqU6)G7?*rsLVmyLki~PTI}b6>uah6(}Xo5oiZi$m!uv~9KmAZH!e2u7!t9R z=$9lTvrtGtWbmC!Fr+F%>eB%c)7IU{i^i$RM1Fnq!UHX*aEwDds+dGphmGx?V*yHx z3*MS|446Ez7fK{oyNM_F9ndr38jd3`-NZgFSMo5~ai$})TRf;4Bg5)(_6RhVE<{I= zU&z3i=RG{&JZG5!8`eEhj^v~9sYqyoHG=DVNN1^L`KhNk!ZHX ziHkX&Xfb|yd-JZ#(`E2@y@73R9wP&uU7E0z4{Acqz%^GGis8RPch?T67bF=DC6$GJ z=5JzZKX!(SgXjZ$ra}YvcZV5Hizh~Dw3LKasHTQoF2!uOk)q%rAR-_lgd8DpNR$+& z1r}w@Ln(sP4>8H0FMwZghlT8d$276Bz}JmpX$2nfM3zy`xq_*1F19oW4&4ds@>m@OT!H1ZK|DEfg9a<_YPDPlY>P{g!Wu z?G#k5d;}-C-rH}ZjD`U@#R&bFVA+ew^hnvBCQv5))znD3c2(}8+cAJN$*G4i(aBV* z1!T?X$ZQdTG7e<+d6B1}L9NB0m7gvt_9tYy81Qt;)rR|Jn|z7`yATZoUW?w&4iI~j z(nA0@V{Zu9VvWV|B2oN+!MO@=_aHRy(>El;fSvjz$CLhWGT6vcwx1S(YJzfL|`z$EqQ8_oGi3}bZnX%Ury?M>m zU0Jo6YEm0TMj-N4kbS(2nLH)#VMPbsK?$(}>?bz+*LEI$Q1O8XgakNVjN9;o1NfGS zdi3oHhDc(IpUR%gW`R$ZLtarTn+nX^V!YCgmVg*&$kgJNbUpLkj0l3>HF zW!dshKw8*0BMYv`)m`pR~dz>{EYLxkXj zWF_JHD_%H&zJk~tJ7N=+>#L4;7Lk{QUm66MMJ|Bn(_kRF0z@yj(cTG7rF5Zg>gN{P zN!pKEVO2o`fe)aZ^!|*GL9=UuiHIa~>oIJ*35bfdv4=4k$2JS71YM=Xr1TD52Tw8H z1@=*n4qqm=istx~sE34Q(XVxD>8ZOhd4ulDw8WPc6C4L+y^b@Cr=wCncwa zWRSN2go*Uq-#5#>#}H5$7|sBK^LJIYOg}*gJeMX&AyLud7rTuHU(y6R^X=Z!p%{p5 zM_tHRZWJ&bBnc_Uk7)vgJ89)bhT|A0^j5BH3JK|q+C8Ib@$VOs9XOe8+iL5Wd-gR-H*2+Q$ttYCS zxoe9Zb93jp2pWKne9%kyuGcHQ9ZClT)TFCsKzgXs zU9^C*?46!rW4>VC)SEMr8Pa}OQzIw_Pvx7S;kaPRo@(`7i;duE(*JqW5@$Z?ayz@t#kn zl2h%410@iM3GQ`U*Ea;7Gw?XNMy_%-LxT(*dSf|bF4g>6e7Jx<7n>)j@&%qxa3S4uO6Wt~xzmj8J}|A)%fxAoC*Lv-yl@M9+iCPr!XAqq+>( z!wYHSWhDnV718gzn>Sw9Mz@%t_sYrVr228gHQurV(w``a65|e6!ch3Ip;Dm;pi46) zKv+PZD_zN8Zr)lJkUB{Vq}}lL6#;y$i&cIpsa5x!SV%WMF}SPcandQErV*A813wSh zM83&Pnmr?1VS;dUAUl3ZfqUt1gi<#ShZHoVxj}<&kf%nBCe0?E442zd#*BPZQ`L4^ zG74aD7g394>w4xcK&+YAZ`MW&bqV5Ol6Yqmb_fImcb?E(E*Nh)TAB?D?x)jrlt?@R zRL5CNxiZl0#XV%7OzAqxl$WTgPz)Bz$4dNxaJoRQ=KRI<8=2dV^xbSE zL$3qLd+C2hz%Fz-nXTeedwEot7EGkcx>ZKSN;&9K_lS^sgMDP8QUTSu zpQ!qj?IFxCv0EuZ9MF3LS$KFPDlNt?G%mvO#NeX63@XYi!)dVaI2ThV&c6mzc3A*4vq-fm&#Nx zv5V1ntuDn-Dk`dh^X<0V_b2g(?0Q3|N3Nvf(enLdE81+ltS;zKj8ajNnVaRlrr5`M zdFV74oyiv7$W^Z89UZKRkXC4RsJ+H)hS(^dAvtOStF=#OF{ka949g&_-fh7Vy`V}l zFbMif8s}<_t=b&h!6$MIzDop=p3)97uY-Nf&_3K!*M;fmCLV3mHgKM%hp0N) z&^xD+k?pWTm4Ugv4ZY|B_5`{^{3{Z0uI>y(qw$N-jTne1sPK%CTWPR`5UAGa{nasY ziUu!J&Jp^djFyNJTf_<|W&oz3M#bLD2By>$n6u7ort2pV`8!vU!Lprc|&vsqnrRLAIUU zpvK`JXVY_>ix(Qop;kn;x;fTbbfh5y422RFHz$1XGL#xR>QsiDT;^n>iW~TvG~>Ho zg@=Z;P$kMihouhX_cs~erZIgEPNnJ>M5R47SV7QIBhF7f%L z2P;=IR3sR1x*2MSs)WF-W?HsS*=2xdPeyQk4KgMfaV=+9&yv2(Nya~$AucD` zhmC~Z_NE@)XaV5pQ_2zDOQeYBs~o246(K28?rcuyo1A)`XlyyNVZdOa=fjJAK!c7bBtsgWsrIhm6#iaX)V+QrEg(S zt@XGbmYLD_Uxc_$SU~Ri=o$&)=JA_82c3IfoMI!ef2y^iL$dG`bD&acb}Vplbndad z4ZL#mqL~jP>&ompQ>hcw_P2*FA+>{=*dwt;E#{_om);9YRbQ9CBBB@p-^>P`j$n{N zp<&K75J(59RVxWkkw~0#ZM@|qGCrO)Pd_{ph%Jr)^&pYO#onKeUA=0aVcFUqCyJ1c zj-p=f4=ylBg^Vs7>_?%M9#fcsw&PX7ct%q#E9pN<#{={BqEX$?k9= zw5bo(<&JVfwF6Hn>II%E+Flu`91QQELfgSHXEX>inlEz56K?2u+@;B5PY2{Br>Y5^iCfQgT{apy4cbBFqUm<{1d>Q8=MY3^}d_6e9K8 z%l4fGtjs}|Fct!ikWh^AgY#4Z$yjzuKEN1p6Xzo|)Ov({)c9KcXhK7`LWV`eiaDGT z!xjTc_!h8oDKs)V7J>5tULwlG5m^IE0N2=~Zc?7^wD=c@-nguL0F*_EC#pXrsf@T} zZRb38db8qze!cXROg%d2B>^Y#Pn9`_*&3cBd8xf_R&p++MA=E?#syx`DoZ729%-3Ykhh1PM8LC$8Y&|(^r^Xy zJ|rnf&sgrBFl^MsuPiT=ACXa=su&pTu1a*HB3v=OPDjY8vri&7vq|L=5Pid8G&ZWQ z#q61HOGi|OF~*-a+TJ-0?T2mb>UoHbuRlu_q!^kNfb$_$`JnRXET`=}>M(LCFL4Fi z;O(ikiVcblL3#B0^yqS9dK!mB_rc}fqpUT{o1PHKe@oaKnmo{JATSiD$T?tQ+s84O zn=clbc*iepCCQ^ys7}v>#mTlm8`c7|17s51LC+@bUN$q1fvhERozEivkXVHZ5aW|M zvE}=4X7wXM8*Lt@J=D?Z=Z#oVi_&dQF~owkA-^^o3sHTpHY0k;j$O(yXaco<7#XL^ zA&@m!8wx9=A8T~w5N=jM&$?$sJUA&_ z_@viBI7-t#U7^eP)Z=X~PEZL}BkmR#ygd$J6_eu#G0lND!B>nA;F> zaE4Q0YEf!bSisH0< zbxr2&q8{x6>Pl$C28A2}@CX}mLJ;zj1d04KuOQ)%0Vt8Z%)$Wob9qCSCBy{lHi11g z#f}-2Y4t8ZSXY6sU(*dZGjxl`P*dp8RAyCRO1{_*MS6+eRbMGZc{68<^iS?Ugo=0I zZa5dYv(-(F+TN%IX%BHLsRsv$wc=O>h7jaw$OYsS;3=Qb;UF{bq1PtmJq+Mz2T(s< z#GgK1atovtT^B8lI5f7vXpo`7{Ru|6sW`HtrGqkxNM-XuqwlJ!*FchDb@#%(Hzj9; z8rqsTFMfVZ3}A_@FgOSwgdqcsH85)kFbphnaEYLFnP{YRno)hDDNpVO@ema49Pu&x zy~)pkWs5w3!A>WNLhx+H619$Kz28D63(FFxxn5HsK~3-kY6Jmx$^&yHl*? zb!tU-!K@<@j+&&u+;~=jG7nW!zLoe;}LCkqh8PeeFw zB9MoDv3Ix*KW#l^u26bIAv^GY(P9EtscZ#s`FNK8kCa8#1H)@@2bq}SIC&QwM8^VA zM;zZ98jPFMtU#rgViXp!Ln$-asz~DQGh0eU@;aQL?g?0l>Pf#lUFhLd-~~O~BBZP6 zs0)JRopzI?Bw$%n^R$ZfJj^$*HEUoZgeb#oG;1uT?$6bip19d?Razp^?3+)jTgzSi zRd%Q?s?-u>xb>xWUe>VBw+ydG0x+2+L3lwL25C8_v(KkRj$1 z9wJX!oK_Z83XB-QyG4gSB1-y2`z(B=(kDI3ywkyra}r-Y(7Am!;D~z$;tW-acTp39 z5}1r4a{zFqiz);e_5uf}lHSs5QC_(Dh#D(uP9l)g%$5(v_7yCuD*S+xV+t5yH#n+4|0sz4M40ytyz7{dmL z5Qiu|gd+&_NCfyr$-cWPJ6I2?lLEn=h+-C6BSry5prmSOHWEpuCn^2n?5^<-WQIQICfG2KbIT6E?Ugo2UJcl&y~;@Av`5~uj8};zn9Oa!} z9oj(r1=~`pHeYY)1q7q7>!Wy+OID$ZJf^@Zi6IzK|%r(42zLWdd&wM!ZLu1UGzZ=t2k**`MldITe6euVV zRQUEF9tBz8nMjV!rlEqWOAfa(2Xwa`O1ts(8Oe}(Pv!@i@ZfOMp@?OvnvxKZsRP>> zASjTqq|iT2Jtle6bZ!GTnFt}Chlv;~c&CY)2L;=XgbyCweypR9mD#MjSP?E-T|L*c zuvUd_wL}2F1-tWo_H~1}LUja-xes~nqXbYH<(tSL{Zf@gkfa`gmUkmxH78x?i1CDT zIqg9KUgZizG7Fq##8LSBPV>-#P?mK^a#l@*6^xb3lh?uQH$lcn zKQS3MExLuO#~^;rQ0`UkC2r|LpN4rA8yxet_nSBJ|1jqX$@k8&?GGNzv32b2}eskXUi;-@`2 zUZGr3TGN(hm;;K8z>s+hrWN&pZf0iu zyegfnMAVw}yl>?sf^S3+cxx6jQohRXT|VjKM0VGymx1^S%ea*gBd7J!!?Q;zIbwK< zy9}XQHcGHE18_+(S0ncbh6xmn!k!@`k|us~VJqh=bq&o%>n5U>U(uXvN}m-0&{ctI z3OQbBSBwdDD|)Gv<9;A0Ew51@qk(3X6HHRr?*Q`n)Y!@>4ePFkSYq7ACqTtbXJL!? z2m(n$(})h!8yTG$A5g@DtOmFn4LciS_D#HpI`IgVYC8O8XoPoH<@rX(cTekb(_MbW zhG#&ZGEq(TYUrTc%1K0V&CA^iG~dL5*RbG}!aObFy+&8St@l@hoRqYWBBdvY7o${6 zFc|S11fWL*+mQq*XKi|NnAhx!>`q3})et~gMe@GN7WF*En zVk{VXGzdZt0z-pmM!13)>f%J+>KYX0k~zU8*HNh+4@KFelZHbHILt$XgsNnAN7YaU zhj(=VjMCDG5@9E>6ofS=4D@5ORMf(y*W|ZnM}?)l4TW4n?dNqD2PB80810Nr;q?@p z1-wu1kX1Z9wVku}T>ZvW@%);;R0 zn3%s0G84_O-eVjvM`%<9ei425#6(|wMd|$2G-c~B{QNJ>ou&-YeYsuen=pGov*Ym! zyai~a?n-PT8@*e8A)*}qF7X4%O^b@JhFLasrpyT1=ty0e5Dgd|oNd1b^&@$T{vO+~ z=6x&r67;4J(P6@EK!F|XOH*Xx)piS@)j~f3ouL9|Hgi9VTrMyZn%TmA=)Ca?KG@ls zQw*RGfzdUW!AtZEX2Cmx2LoEZS`mO~ga|AYaVX}K2x@1L6g$}f=RA31r!#l!G*IU2 zV)9PIln7tgz7xucuR>V?QQc>pxx~cErvZUl`Am8;_!e!Cr#$8Cz{FR<0aWV{2&J~X znld?Ls9P_nz`G7;aTPsTYZV}ahhR+E^q_a8M43c6;~_*9+rU)1oC>^V42|xhXDGYu z30zT?CE3(z)I#VYOdLFENeFw|Zch!b80MvJEPRID?-tSABIUp?t~VFrlYo>Q*W!L5 zY5jf$!SIgyT4%nN9aK?+dgf;$^q6`?RE>^;`5Nv=LB^?6NLqY=x=QfIHta3a=khOF zS0SK1r-in;DeP;;c$A3W6a2NjoFLi+489^1dm0sTQLl?yOYtc}9?WlOHp{dA(T=IwoW7~zXxJ_ImaOHY_gAQ`SE zqv0q-Xe6=5lI4g+WSmdIxw6k&siO#5%yPQCkxl+3j`MUtEQy0`ZEQWn%Q` zyU8fCX+j5&vJ2^RI7b7xca)=#-pdeNhlW~Qj!!7Nd@UZl9Oi70PSp)ofn|EV$n`v^ z=zyJlzEL#hx>k#0dO%_NZxIsop$g03sI?g66=AL%Qe;QL8B6+1z(x>^WzT6Im*sFB zG)CUMI&W-h>DcHl zE@z$;1?JVpYd1!I9+XfO7xJO?2~@=f0|TPtNy-vSiSkRNnL8e_YUy$yEy-D+LDp`D zc%#?J$-G?$V)!1U$=Wb^kfAqVN5o_9>KYj81)|?qoIJpjhNI)7_fE4kNvx@v??a)e zxCjy#)-BCqY1?M?_BV1tlkQ zNt2g*AnS}DY5XW}O)DM~`2xx&AGM|}%HC5$7FH_kn=AlMK(W6`4^S_JlAa}P;gBD= z?(Le&ERZvieAz)_6UjiUkwFF&p=q=;8u~f+lmw8#tZ+QYYfoW*&E-~n3Y?H`t#Hn9 z&urPUoUhaEy<-oat6c}aYrfEpxNARM!x@~zag1@^O?FSZutvTjPir!--1`3`=E}=> zLM)&}MIb<~gg~g7lGos0C{pdS)SfiPmN20+D~A}#fdHyMN2e-RSgq_KP5JyQN`hcv zg~{s(+k)$U6ABG1qIN>rqa9A{8pK%f*ei4?F+jkAoq;M?f`~^%PC~_Xox6%Ey*MJf zh4*izfsrj_9)Yb|hhr$x$xHd7Ir8E`OC!qvqOhv$zKfv4o#O%S(*%@C<1LH=l@tum zaQRW1Syf!QoJ{T^rCs;A^vj;GsgI6j%%~&UQQU1hy}|^Vrb{wmTTr|!o~%1@QB%Mc zEU1%m?ehK_9e7B9zn}MXZ?-Tjv(gCQD0_DalG0H~FoB#idD>ECSQzlk#Hmd1U}J}3 z!d|5mM%-(=uw>1#+w0O3Fs=7!%e;^9s7i|FFJuOSFlj@ zPF)O7vfG#tNkoi@9@~!n(P&#ZwKMR%W;Lq>>McT;hLn=pqS{DyHAHzv3A`!XH|?0{ULEpxMIyl_G~WFKVMebMK{Ebi+Wyh}^J3 z@Prx-gNUgp;|&nHR1RV0S6J4oGEeGVwEDD=1Uy6z)=ii33*c63k_GRE1#xWQqsl0Ru zs5G@Z&W>2Ll(pqjc=&9W4AVMR9$|vf&;-ij7H2|S2S{Gr`fU0|9mys1sC=QutgR{x zeCbzztkVz4z!g$eib&-vVu-a`J4=1aQbQWBvqe`ZO`TRJdo)xc;kXwFvdYhpIlnx7 z57IU4NzS?O5>42txN=(3!);~tZ%a68NE`$t$^p-Tg~KWjB##)myjE9}Cft0b%~{2c z4&pMhl=y;>4#xqar-cSEO(1awVstVT{pl#7=V#D6UR%+^>uRvnJVi?gfC<|bYuM(RaK~df5_LJsMZZqTi$tNA>t}D zD+N2>ON$1=L7)MO7nWO_yizn=c(k6Tm6cRf?@Cd)MGX6`dKNKk-|NWImrlKk^@3!% zs6AvvvgmFBk_HW!acf1{r&WhM{tZ(Q197waBes|k#5V0oi2-4NVf)OJ7cy;e)Ixpb z3f1o;utyOaMmbfNeMIO9Q3Q48nrdxU$>jq+J6I(iw3{dy5rTUL!j|`nk8OhOm%y{D zSlkV1(@Py1*gIGv^WuQvxbH+aks7UMl616ho)hPbOk;C}1W1Ti$vy`}CxOwppBKn_ zWhe1zt&sqlU|Y(b3-09C7^EfiN1N#s_kk`Rh{|(CgEZLonKKYaz>8A&$g#d(BPbxfwA~qm0mN7yPBJ#!5 z99-lXMSI{oQa^NEmxNh&NNcO+d@)r9hD&}Mf(#SkF9nX`WLcF!LkW54UABf;{%Z8f`#J~) zeZfU$)Au5j90B#9&Bf$f;JGiTrYatY=mI)R-7y1RnbAR40`a~wRm|+lf*sn!W2&v%xORtA?)9+CblmoWj(Ggn1TM48EB| zKZt4X%ryR!DKafdwMT_*=^A!41%>w}Kez>IsyIARE2^(thGUQj?%J-Hd(IaeNb0pm zo~Z_D72w&Fj#CI>GXf@&!UqE#1Tk>aK@ShX67i0VjyX?h?0}o#A_v5*NwETdh(Kjc zc7aS{ZpPFcg!Wl(d*;|!cgpuLRL^Kg3nP(}&j&BWFva5v*t2Puq#_IVnKr<{NH+Rt z2dUhv$_Ub;0us_wZ^U+_P9agarA`B~fl=HpSs>skbrG{v!<=xL=wWtG428S2iyGP& z+!!UI1ueXveZmZ^TpS=F8wMB5yi7NCKE?93f{wlcATkx=gj8S3I8YP{s{I8~w3Jy; z$;kk#56X_K1zFRXN}TAG-zk+wMOX+Fy%t6YURfX^m4ab7u{TM64)sLOx&;2N8n)Nvl`bEZJ6qQxX2ejOm7$;J~G8{0_Sx5z2iscHrDi6e+wr|j`yCkYBc%fAy5!n}6 zy6Qr|P_ao-3}7!p5?L6cssI%g7GgelkBT~`H%Vg*ip>9%hE$76?^G%>F|kp8n0^Bc z`&w?Ivz*b5p9RBVX;l`2?SxDSJsN7b1Ovh&NLfm)C93}TmHqi@V6qBL2pN%B1aN^7 zRgq}^C-*=q!>Xh-lD(n%)eaBTsT{{Mazv%p%l^w0&Nd1jlvn(YqliBwsuU~uC(EFi zQ9qO8Av{Z^)c8@igGW;8&LD@jZ;Zo7@44%vzZWdJXPvXw`SsJ|zn;8&{gefI^W)#& z)Z5wJ!!TmM1fVY8&DLseu|d1VtZPo~W%K2q~Hr z0bB6>&wU_;#b!uqVYFK0T-kuo&N8(}fbHqK)q-&{xc78eu`ZHzK<2FL9dfG0)n6{r zV;xbbr0N8Sq}NgF6Rr7!jtM`V{fy0qg6(Tt#Y8xWGlSW17oL4^QQN!YogEjG;_%Gz zd(G4aim6-cB_PK2zN~8Yw59v3J~qC78|AH@z3C>cUNCe1-(Iik z?k|k6Ot)kaIQmpN-qs{T&ZvV9*t#j&!!HlMQaZgLUaqW-7(xKZvolshaFs+55rbX@ zTj3#JmvTjhGk6DjRbIV#H8BPjxH+W;;6xC7ppt>6hbpCmRSJ_Pm%N~mEG-635XX7# zv>i+m4v)TzH%w~8AP(qW!jiG|xLYh*J@>dR$fYG1#H|4S4GJrhY+5+y!KX7>@@c%~SjR10$6k=|OWRQnhemmWTjH8MaRk4m86zFMhy ze0J{cren)IW1V(%o_%341fPgIS?+{9@aL+k4z^hZx)^~|cVo`QhRv)f)5?RxkRE74 zEU|BWB`6p_M7$L;ctO=E0u(@6HrPc~q%F6(fi};*xMReh72i_9bn;ZE=-EA?40{ADX+F$AF;@smuk#Fpj>8zNA79l=Ew zd(+-5@p+pV&_M-oRr-gI(GM45`M^kl&}jm9EYwb?zj=h?Z5XA9 z3bxTgeb8mS2V)Ua-gZ+CKTpGIGT6?*T^Yi1B{lfoPQRO zD!bUIBGL*Lc?P|BuzL1(WC~v54(QaZ4G?CH$LKevn#^p9NUz_Gj zLo&yf7)q4r`Km_v^&V-bx-~L3dsd_g$h~J&D%$2Dj&};b$gT-YRh;u$l&Oz-GYJ6- zpaK!75UB7ws0#K1=KP1Iw->u>>eQXKI0(4(E_Ps6n7zOH?-4i z%A2jwkGrG6L+T;|>z&H%FC@!}y68mIQx)*1B`YO!K)b_cZedN9#fI$Wvn1RmmCIS? zo!GOhvMk_YdS*vr0hn|gQcmt!d?QllH;ll4A({_`oaR8FIWu7 zYfJYjq{F3A4L_F{_-K}?F1vN%yIq$+u!*7)Ws7FdY)WsEUV`G&!tcuqxHIQFvfsjW;eTL$;Pgi3v&ox#tBx%Lc z#Lgj2%qMKE({4eKN-j8wX(F5^c1r_~2W?33+oRGWUimGS4$cS>l?rSO04V5yC2lEpkavYrUg+jo zBZG~|Zkk9U&>>`kkj4!YTaq+J%34MdV|Yl$0y%O0HU;R@+hQLm z2Op*_(V8lk1Ug8Xt$be7@)5xO+LJ)h7a%)jeSRd3zcJLsgpItS`?j8oZu;%&uLR0P z@a7?@ReNZYkkN2%0R&GrflpNMxO<1*oEE|O&Cf*;*cH+jK(>v2a3`m^G>qC|jP`Mq z{qzYW@-sG6Vdclfmu#c%p_P$wwAF7dJ_eWoCWWW*og^uued=1*Qh6jPaVN&`W(Pe5 zqG2F^JSl-1xMSX|3ys=FbOx^QH-bDd8j01X=O{xk0Q=NJVN8LX4emY(z9RC3-@kmJ zDTYJxE+iW97t%ingFTIqnu>u5344| zt(*3viw=@ZS25g^u`pQpPles3RL+e>A7jRptMW!Y`*?7vP^&u{U_`o@d8!D;Z9oOl z?MOf~3*-}W2znV(P+KpC?;K&ka-lAUinvW|@Kp#rE+mX*!)I71k;9IIW5rNQ0A+!i zhMhjb&$eZ&pqJ|4{1H!(6~EQ0xjEvN}Y26-&ScTbpYEInO%vy+js%}|Gi z(``_d2BEl&z2I4t@^l2zXoF=XDFr0@mut~eZcu7Owo-(@SV-H{-1=w9M>)bckc5sK z@Fq8m|9l4mM4V5?0tAS{hCF^J#k#WW%dmX}TM*;{d@EX%O>iOzerff?#7C`C^E=5I zj#O=pV21eJNQ$1NF*$ph{WOUWPbBHcMoM}|0!55~(i$u$4}oHy852XLBrz46!|8({ z@C?WaTu;aycD}}|@*Y#E)e6_kL!tMGq3luno})8h>>esqtp@rG$6Jw*kBYm~FK4CX zz6~^?VnTt$gKO zQ8DM2sq-7iWrvgDJ9)EUeP?!1(O#k2_~;DYqS-{VkGa^TTBjOXyLC2=j3Vtv5T}MF z9Ks$xh3ue7!A;^G)XJ2vQZJOlqCwoQ2Z<1lRgJ?K{@5AX1{Bi^rPW&jGly!R!{kLo z8k#aZS??8G3c(OBaO`?+>n8A)!D!abBBDw97{n|;Sh)4g`NHe8)x7cm$=Xz59$AjKXDs2JXOaauZkx_Kt`wL6&H^z0#$~ylqiHV6fLtt^< zGH<$B`A3LgzCpUf`>+*gm%a^ZwY=pt+U^w2^k)j9`2yD#+mFj)q(H#(oZQxdVFl1) zkg&5vsIL_4^f^J5DbuLu$D;M`8_)5@o7n~ zrJOILuy6|6*<;{1EEPJRgBu$W9DEZ~= z!a$6eRAdUz>m2YdL1IEs-O}>EIAW-46dt)SPZZq5P$Ovk^=%8NJ*(>LUGnIXQ;0m2 zXlO*#xE(VQD&0+t=6lQSvLu(a5+rnN6z*Or-ykUA+snCB6^U(P1E7t_AQw12`{FqW zUN$mso)m$F2x^9Z6zZw{CWwd(lx|$g)fF58lyr~B!2(uys4i{SHA`4<@2%- zZfi$m0(L+UnEs0}_bZ|fW864bF0Z@BluH*7)dvK1$UAf0IR!AWHRq;KFIJ78U@@_* z3C|8TLrI-OpcS@ngRI%cSozraOBaOSMEUh(lDjPqH;=^snTwX<&^L>OVRt8}>*nvRI}? zg{qi<{3}n_l53Q1vT$cp4&^Fz@4(uO;$z6PAS4s5pWey1N->0yXy*i};2jX;!jaHs zL9^L}Qche+L-59(H!OL9YPGh9@~07kxf-aA;F*co-Q@3yMOajoA-J83T@4W#inq;< z;85ZCMTi)~Sv(U=20nW5>*I^#b>r@#O22#aN+~M6IOp^0mOY5x{q6oDseaGue-a$I z%f**&;`P4vK2+A6c7&#-Ajyq{xa!5U-bcGN&G%#?QK-lwSr>-TGjEz?P35R~x==Fp zqJ{Yi-z=|^=SD&-R?3~}BN!fSx;5_O4mOrdn3%0BGgap{)!`JhBDq$ozBuFd)s8?! z{j^4!sD37ec^!x3p5QKSK&{(N>HwKGTS zv=-+2Oy|PcW>v99YY3ufNv6*r*@k704q;e2J0mrzn;ap9K5(?hOMt9a>N^;PbW1?+ zJX)}L)i|N7bbTj^dme<)i5_oSz~grsK-7Xc5jDu#I<+smls%0|RdZqQBc$&oh_P0x z%y(Ei2R`1Vdxci9@pHB>L{@l2(ie*RI$eZh530P#ZEiNC8R3mne(cMo29a7iDH1#* z5(H$QiNM6a#BleX=Nx2MWWNA6(t?D9v}&eDDBTSUQ65eWNaq1ZmLTFbWcSkHrHcv< zEFJA3(~T>QyBP%f6IR=_b3$(W^zSV!BKH?;glBEu0oOJ9kg_qEaqN^Z*r&lKx4|fd zNeYMIiz_4;vN3T}Fm&BU2GNXealAU`39cmQG)RtVCgseb4Z9BWZvc2i2e0kdtlXD| z)Gde%M!1ORL}mm(zMVZjXBx>M93UPlk`2cO+Ry@qp0b)hobtt=2n`u7P>-mjmc}q{ zOh7L`i6%;|w{PX09i~~S2N`+NwGREW@ay>mKnr>BqSuHTMB)!|LiU$om#=852V3&<3)VuK10yme9uQ#8*6o>ZG&1oHkeyPZn}H9n$FtY4T$W2a63XIk zI(cpC&@;sC&kMf|l4-V$hunf^_C)EhF21a!8DSYUa0k?B^l>y|S6hUG+2+Mh5qFQC zeEkWbVRmY((#@nIF^DF?M(BbxHnf7XyBtvB9XJ(mc?%&2S$OJC?PeaVKp$yG7JAAI z_$bd+6Vn?!44m|Dg-#9$Lkby1t3oj(At*{`sx1%YaYO3jF-IRGlv(sDLp~UpCo|LkFBZb-cTa|$zJA$?trt@$8__?KuNN$Af{3_ zBRF9B76=x)S=4_s5p#{8$$2k5b2qpNNtXffVkm2cQ8sB!8y)O1nQ)Ou7g#fDRgJy9 zB!X%1@G4>u4ESx<{?`q`MYUDFqclKxknj&tb?tQt~aX zW1OBFi*5S-lWY=Du{Medvyy|8DB=Sx0Yj||P6>3*SBG#_qrkas*fNaM`~?5 z0ZNw)xV)z(?S|E(1{PJL=@IC7;npeDAS>MsgQv7EqC`G@d|f1dx~z5eDLLMRIV`q% z;hdyHeQ=LU>j=L;GaK6{PaKE><;$0igD9<^YGNlK3&~#OiaEf1zg-HzEGe|>#rDb7 zAToizYp5rAd#41df&(;^Ja4o$jAMJAmB(nWo7O0z@F$R#bKbagW{5E1(D&VhhCBQ<{fsxoCMYQ`lJu-n+e`*1A3<m=jkP*U=B>er~7g-p_ZSLNAJ z4#>R$9E%qnxLXqtHMua?0G#e9hY=BZFxueo5nG`46Qoxd+j>#hq5b3c1PjA9d>R9F zHU_Sn2e+8)S3Kw-qr8Xk-P~g+g#hv^$YDr??!Zwxoc}|EI@@a-yfJ&#;k1@27&5#vvo(czcJb1>982Bn`&|x?Z1gl9U-3VGrL<*?EpDMlNnJwPibCw-YEVbKC=J-JPSbHrY zdxG9QqcoGa!%EaQ0YH{Q)9O0Y9jQr6QBrb+)I}I0V!>&8YNsQn5JAJL=Ue7EzO|h& zcGt@fG3P8|^S5$QOOA|S_;H{R#M{Ob*TX1kgHQk_CPE>H4NhgS7|J5C0~df!_bo!+ zs{)l}Xd>2iTEW~N5{$*7zHn$$B%IBue*`Pt_2VdlC6Ea4 zAc>v6TQ|79?Cv;WWOw14%16a9$ILm>l8fwvtdX;!UUDhF%zy*y5#H$IwpyoQRdGXFpC64)Vxdvc&5$q1ye|zu zoH001d+bpLhJ89VcIM-aMFduCTj{z=$%gBalfaIv$$L$9Pl2+CJPtNI@t; zksU~WfggwAiMvSWBE*qFR}g{WxET|1sfTj|u1>53HH+*UPG_NGd9WQpGfV0k0%X*D zU6^9}dh2k)!@xS|ggHtKzFy0H6_lH2ZLP*G&6KmA6P_sx&z1JY_$sahZ%*T7;2i)+ zkx)&7$sm;>N%aeQ&3;#fW&(4TOkYJkm?Sa=WPc&WN^t>ZlqVKTfj8NU+JY@z*-_ei z1>)fY1A;=ZgsaCCiG-ESYZW;Nv6WDusE{D0CX;mx|9gs&R)BPaOti^L-HC$?B^K@HYH|Y1F4HWKc(+762xzs4TLNUToa?2haM$%a!&wYz}56>H=*89f>(4ZBv*A*$r`Vhc}&pi@j&w57DJBE;3yut-Nn zUE#Hys+~~P3iLdYx+xYf3*v>Z}wR;L}d&OcI;rrkG9FLGw6r zsgAZCqQb5i8e?N&NjH}nuzEq{lFPMwwVdGxUgb{iHNC8S$g+r*a}4+7PNM1Pc*GUy z)_;!TEWo|OS0t2LEA~8e@#U7x-lg^G??a&m zg)yxiT}sEuuT=<2-f(hnD)dSGmq z0S*Oo@qRbvWMC8&EDj);1I~Xc>i!GGM_TAG$3i!+9F{ru|3UYgAFV zDe}b%T6rz*C`a=n{EEupBOwKcm{FMZZnqG52u60!e7zRmD`6Rm;UJK)3G#!8h$#IM zgv5PRasq!(#rNNQUlD!x`TvUC-yk71aG`X!M$KCRgRkh|Lq8qP!v9vfp$!lEGky@v z#2N|6f}@n0Xw?p0C<^McpLUw&2p*!1Xh9;C^28R@ChRYWf(!&BVN3~lwg9%F0M35B zx8S%2nQl{S#w6&+ApB+%j!RJCjsBk^D*Cj}<Zxe6kcQM^~6gA1DR-DtEQfhtsy9!f@QUAGlp++C5A?f9oY1V#up|vI1G*FiB*7t zsC`Rn!9O#H2pZ5334pGj%piebrerv1mA`qaY`16_|GzOjaoSvoTrR`o^- zs=p7{=;Dk{lNwuxJ3wN9YPv=n7-$rFE|brtR^syC#3VpPaGxM{2^<#Bk_2c{!uFw@ z&q!9OK-a-sZpTN;@V>*97^W12VNXyvzM6AZ+4w=ED>%!nHz!BT(i+5WBN~91HuIy( zgx#kbwgH0q0B)oLA$by!Bo+1>oEwI2&ko(tRhYIK8OKshn2v=m}D*+ z?MzZJ=fVxec2rnl*jejt?)vUipa$7(*t+36^f%Hxu>4T4tg2VWj$o(+=!hxu@{ZsJ z=T`8Ii8gkFRG&1!Yme8`)&^Tb4axfO&>&tRc}kDz0M-)Zbj!m60t1v>2-Z=Wa{&&5 zHa}pvQN5XSXMq`Eql-C1rlNi3s#1h0SoNtFb4v&~k+xbb*c&~4yX-vvA0guTd_WKj zFFnks53y@>g%N&me3JkZ8Nw`379xiR?CrKGeUjc(zTlf&o3I+QAX8Nb3@fNG7CsyT z9gJ}#u5VoB9j6UY(6`LL9=ek31PE~`4UT4Enha7G!;_%3H-nD-y5*c)rV@EfXc7i@ z<%e)B?#coKGws*v3rdcJ>0L9%8$%0{`V)B9Rf1cW&m$5=5t+;^C4W+Rqbyza|`!Qd3wN6$I9AN#6mW=o;N0_;c1QW zD~T$&kmWDrnnm;`vR?#Ghgn(8JiI5BA#p4!6dX2o@rBGze*2h?wRs*Z7VSpfh^0q( z&3to7@uU5w#m69K%q%-*9_5QzH&zuLLzjMZIv1!~YIRC9W78#@GFHAXa-N;y;K*Jm zkt*V_P*#<9oN4VBhOQ;&BXOm%F=rwDKLXg#&FgTvd=xy=3%nOB5DMwrm+zoFa$DmV zd2^(bEL6FUaPe1Hi@3hoh`fOjFLlNq7RgjYJUE{#1PAWHX&4OmYG6C6c zdp=v|no;Md8kQB0L%^r z9f7se&q2y+0lS%TkUE;78(K2EY@!<*4kl}dmEG0ABX|{RVIn;&9L2qC~9?&pu5Pfjm^qv@8k2>H; zsSAAK)DAj=Agb`&QnHB&N^dBQpmjcgoGYDY#7LkU>RtH1e0%ZAii*wJqe-7%PpPY7 zeH`X*+!s=U)ul$OqdE?AtQ!wV%*0U9iXPerQ(||=sOhwT0|2}yxz6Z7@V3iv;O&gY zfz&y((6+FOPu^f*wb_axS0pIsNqWeJ!9GJYEcG#=(7H?*kU21CqnTjZtJxwk9(6Q< zmNu}``RE@M9hy8Q81(`2tV8SYjLm6Sc*~Qv7^VE6$1Kb+UoowI>tkgSa{vOBK~3{2 z@M;y?ur<8%v|SpwB{|@c=<-^l+3fs_DE=Bf~wwtFUk+Fby_0iY*3A zPOv82Cb1BOsjMpeo?#bc>L8V{Xp^c{7w~3&3Ctq{OOou_P7zz~F94zDo zB!rL=jGoLzd+oj-j`CyF1`}>H_dn~)j-ARL|-qn3rvWi9?wa&u)N~Hu0cDg2wgDTuq4ZME>NuY3k;Q)4wIk? z8ALT$fwn<_h(}DjlwJ28dsEe6W|}x?^J(=TFUg+PF`_<~3kT^>mJx|`&RLj1`*sf{ ztTro!7J|VB*8u{0k=Jb{LWcF#{mpiGzAM%$6y>>e@x>E3Q=ah)*m~v$SgWjX;%y5W zt(s;lg9PG(k{)5*c?gHcCZ=1q^L;C^JqA^tV~38jzlE;Kiej4Vr_M9gTN~?zRyCN4 zBGBnMnmSmDMBf1Sn-+n_c!5uqBpda-H$HKq#$l~}Y$1ERXLqCaAY|yvsUJy@&_97U z)D%J_CMn%k8Zs!3&AcrD$EQJRQ6Mcuqw7r`)&_o!z*8anOAwQ>O#J+d`HE`n55-Z< zS1vp|G%Y}Q!Z&^*CrTLRHPZM%q%Nu!!r%$1K0p#AKd}V}6h=ZoKb2mScx+uZ&{tZ?L0U3P)0CU%C%6Ed}iLgCp;*7WDe5OxHK2+8(WHV?^G{l znm}sW#O0_Ul}A>Ow~RXDGc9?u2-qB#fqc;>y<491UO*j^9S+jvbUg6r!hLUsV8t$6 zi8)~B?(RRQF952juDOg`Oqk$-0?GS!4L#<6Yn$s?eT!y$Chu9hIJLz;cVy%sg+&gU%?$473%t+jNV*Xmqf}RnS$@WASZTNVy7*X^2jTc!f=iE3 zp-F{=gcMO#K$GhjcrrO-oV)c1L%>f3v81r!2*(+#K_g@MTNFJ-g)_rZeb?pVy~#!7 z>%5kElQy%(OCXC!h)|!k#UcfCce5xisRES-SxF;n*fl^vE)>&ya{05zYDM7*f_2j& z3BjQU59*y?6++sG@WXQyj~rFji0p5exKsr4>r|GiXSA|>HRe?jS$wkRWyA36biJtxg-B5WAhO*tV9wdsQ)vvi9W zbbiEhrusF#4VElf-MtdzY!vEcrvu*2QB-pU5Df$8b7K0K%Mt*$N-&vuUVK~^$Mk_+ zQrM)egC)WPO)9%Asfl<)XO~=MKTSdNrO{=NFPt0@vEwo)p=`y+@u#^ zzFI#?%*h?FpIaPrNul41n1cIA%j*gQRux897a$j@(NSxo){F4130qLfWUP=t73c~C zkBDUHPG#6I)#Jn?x)hi8Za}TS7I*hyh^aicVhm^h1WAA$A6q5ZZ^JEaQ49 zX;x+=!Ird92>Af3YW+|`L?RyH%iRzZ=BN~XWhw}C2EMK+U}4IeLWk=hdC#HHXr75h zge3{;YggbV5&^KaVb%dkxdkzAPd7`SF=MD$j-vj4*}B&KO*paIy5D)Ym_w&9sE2^z znifLtk`trW^>ma=wLcpdtC=0}2D#2=l0Y+zz-c!ZF9`m9XF{jiP^xQH6BTfCFJ&T1 zp;Zs4b-A{-5+xnq80?-+wmGl~e0T1ZMi`o+@wvAXpkBsmrB<&EdQS(28yP0rN#GOV zgdM^@SKjLO&`9>WNWS~uKVFfCo!ue6d2o6e0D+e(V>&0Qh(k0Esct_SEqho>f{Em4 zhL0A_`vBN=HX3*4;wP9!9yK9Brizh_GcGDh?3rKbMCRc4mr`}9nTWr@9V6a5zeVf_MSKnx^4CexuBZ17X zB;KcQ=~Azc?~PpM6_F* zqw6aws)}5dq$px;1W-`wbT3&jKrSGS9V?3qy$+AaAY;R0BdBTPRr`tnp#src#L_hW zxgCbiA$U`e<=U6H33HQ)2uW!^8L3wUrJ|J^74Ig->U%IfvR|^ERfQNX#Qd2p5V`Hv zV}@WKaoo4dQcK0LksNoIi-8`3D5R=#WB}z94*c+jTNEHeW&FyMpBRlQkRGTSR1+Xs zfk9=BO=-a1kmHb<6s7u|4HgIrP$>J}4tN6%>eibW2g2C9g7Uu8_*s!rdC=h*=S324 z2Ea~9FshsYrU=A3FlZoIs4Y3`TQsuOmE!diy{4?+zhq9BBOu(CcQ!{o za+%zkI&5fvJfmTYM7cE1uqC*LEEgQM-x6_%)2ZUMeqmkp-lJ z#c-@yQLcjY9RUSkfcwTsud@aY0JaA0)VaP2*1lRgo#F0BRC4BeNXy)ENmQ7@JcNb7 zdllYrER$7G6bEhiI*gC42S-?#23+zv5bIlK*?7=VT!&308Mp#)oycY7Xatb%AYo`P zfy7g=kbq=V%GiR*mj>!4sXD&-7OnY!+L*&59jP%^M#uuPAJL@<{_EvER&=Sf80%Zt-wBh8(v>$fPUHaK(tfvXw z%&LbsjL4}1@&wPJ9<5T<6HQb)zhFX;5IG^fS=WS&I@g&yM&~a$e0J4jJC7 zZL=qaOLzLPwXEcR@CXYK1ek$%&}j;*(w%tmej%%|4jtY}pkX4&;Zza(1i4jgSnQ0^ z{AkuzG&DaU!173kdsSV7O(g?h^efts8A-h#&0x*Eh0eN?>hu+; zsj|kEa73m^WDZS@9f?jrNmN}K;|;%7Hm{#oQz=^YI<~8vH+cg1K*ehsa+;m#_m&w2 zsRYn4=G$0Ms0Pdkks^a_0=%=19>NZ*L43Xp&b~DY;6!d5NCbmCs12YdIs-g|(fcHm zTrf~?Raji|xbPUS;Rb3drq%;=3A%R+W1WwlL{bWD3Px`J3kDcIbPZfaaA1o^zA=a) zgo(638_*6k2lEg}X;Pqt_N6fx`%IMVj}N9%QDdoH>h#94narcIIOCbDPNAh8liiq* zq0udso*jf`-U`)95`}@jiqnM1qg5_G#0jKa5}_c4KPb*KqP(GvcZ(rJY+SfnTk7I+ zr>zTba%&RwoF7aA*Ar`S4b1b+wxb7wuaCo!K|}e?k(?`qiIPfQ``w_>>7D0OXVpvJeSlq{RV4ggLGEA`FLM z^AF8!RQn;t261$)SsojlWEL4RJ4s+MUZG$!1ID-o!gTwSm8krK9JN)6xQ0WW_&0b$ z?e6Pafy0O1HAWR=@RfS2YWdw*o+yYwgm)*0wIa*)4mQtVm`s_22TkRC6`JO zM^u`NXqI8FKzfq9NnxYwYdE-T8KpZzJNPgVq1W9l0{tqi#TGXDa3;lqH78;W@70_j ziM!y>SE%W$u!nfGWv(-G>%m-Z*ub$W0*Bs>O`l=hgw}D2U5Of5;+6% zu7O{arM~y-gb`V3=d$v8vedEXM#K&nEIdC~u02kOxa<}doon^quWzuRR7eJWNN&c$ zq9ih5hIFjMOsiD)80>nyw?~lkJvB}pz<^D>v>PFeL>pAUOh~@z%Q^{D&J-Z4uA4bp zECus+F7w0qZZ)NTp@!^i;J4d&Fl<-qSOdVoFS=jVZd_kRka&ylgk)%RcXUh~Tv136 zMl#aLfPK*leCrK#gmS*s3jy?#;ImFwQ7BZLv1aY4b3l|upxqeA60kZ{Q*o$a$r$p0 zBbV6FLQ4M>0SV|3h>#J92f{c&K!w>UEME(Bg$t&l#NJ$CJLr(bHXh5c)^%zVMq_a0zXu74WZ#6xVVb1YU}{Z zCAr3ZY~utE)=TLeIu!Epi5|_g1&wvW127!aFiAYJkkn_isNNmtbyOqwV11*KNxQA>mf3W%QBLjMoJAG@@D;bj4SpZzVEJeEU{`rf~cTU|Ap(cuf!tw zAn9=Zw|GfY=kfPoP8C!`aw{gm2zcK;+oeuw6f=~H5(4Xft-5R=F;T`<1En#cQG4}U z#of$EYfEwwdj^jgS|m0C$pZt6dc5ssctLXSNejI&O~-H8mGj(&->YRT&J3ZG)WoT>x-^#2oFQ8S<)f_&|as zQP({Y=oT&Bwj=G-9XjLyr7wUmSc~TnvB@JEFOdG>?;-A&K`xV96!|0u?2}CBDn-Ca zx|x`f@2LHu@-*#?N3T{ouCcKR^sNEeE|f#L_5f3(>cr?-e)q6#+EKcfd87Bv3Dopf zKL=>&D%)3_uo8+TL4a6_ZI!KwoceV_fI`|m2p&!L8b3L}`er+B?K<$&iLCH`S1$c;p$zR22!3f2R2)H4bwmpf3eh;#)uLP3{A4g2XzEi#1z zN4~sZ!TSg(!vxccY|f8i(4I#XJ}v~KMaL>8Y`I)Le}KKcx{E(@O?@!cjEX?I=I!R0 zc%G7(8D2>7PHVkG5MKbh_V>K>2}J2>CA?C>SgnXak9X_@5IP{b0ss*Ug31Rrrk?`* zq6IkpB%9h2Lwc&h(z9hij|-yL8G!bZ%LOK|te#4?DB?s~Q&^WrdLwl4t|QiHo*~(3 zNyZ-ylyw6;cn=x_fIdV`Ibk-5XC4xE0jj_#a4_|au~E1P&K~uq8$*=WzC)CYD$~~X z1g>e{p^_d5PIVMwRNry+(8EDKR8=_s6tiEb z3y!2O!L!`wO>ILdnzXcf!mf5-A;l8;xz#*MNkzDAW9Y6QWL9~Ckmh};S`<)Kk+kc|cVDJ~-ra*|{gXEd$}g<%<$X#{~$q?gxp zUTJ(N1Mz9`G;aa)dAY&s)cPCqGjLl#6XmueLOw2l=FC7*Q|Z163fo`LY9IA)_rj;Wvw3KOo{Zi8_WsiR^T^=%k10~SuNm~&IPt>SYtkBGi5S! zVq^7=T2Erdjoag)w4kg|^tNSDN;*h^WYaH0aPJ-`;GWF(8^yen@P+dqV|Oiw2)pyg zhJ+E5)-2&^C!9E-L-0t^x`&kJSDEZKdi%qU2&yALk=kAw1SP^Rzz4kIwKmlESEUs# zUl}U6g5`?VcC9Un9(mbkJ;surINHWX4xLoDeKC62gk_rUJ|s3HGBwjro*Ii77y@{x zQV2e_OYr5R-cSw;3=UlYDIZn)C4^~}ip8Qe;AlFg4)+sJ!56R=-tLtHnQQ(TYuQpoPSfHXb*XXIZVK&PO1)cZh~0q-Ti0)2>yN zQNj>CK$7^(sl%1IY`m0P%LOd7c zzis|003vz%-3g`0)^p8m7c&BGzx04-S}Mkb^_r75jk0D zDPXmg4hD!CQ`4%HDmx;-bYZF0!ZocIOS^@gBWYnd8F+;id0h_M z244_s5DA2lGn(3}IN6wTcS+d1)5^9uMA`Xh)Yi&WO69QvjUESS_%A%j5+Z%gQl&2H zk(BzWuLVN7=1vz)UfR`goal9Ypp?xJ&#Cf&ib#fNkf>HH6Nh*N=36pOMl)0~rq6bn z%?*+Pl{TCktxFt=4b&wp1#j{PsyU#v-wav4KFcqVC1v`vx5E1%PYJ|*A zWxoo?Fq>U@Zc?AWG4Q+21H_?WS+bc{7~Th4s*d?bYHdw?l<%T#(@kAKHPh)yFj08h z*ucmQj`JiG902k5aJaQb+O?2IR2M$tXErS>&ZnBktsD;r>o(BHptlU8Xi%;p@-T>m zQJct};JPA2_1X)Z_loh<)cjH@WK=+a!_ThDy<_AHtudVlAaPW3CbjxCvWplRYhA9! z&V=y$9UYBvT+mboq%3T1`j~HMDi!G}4FOydI9+Ao7oG3Kro^s72ujP~u@>O^u2u%( zq9lYH%cT2kK?HTf^2wQ1?tCj%Djj}7GnbAS^n|sq0{K`L}v`KX-Y{v1M3S;6V7d!&bhQyZ;q|R{w zta3*dT_w{*zLA+p3nUN{x-yPo;;2=aQ3f_MiW@gla2DhrC<57Y4A5yFuwb$M(_1O9 zhi8zgWN>UoIu*{U4pfuPOfczG3~ZNl>0bxpq~<+rnq9Tw#fk?TmOWV&Nj;@wumohhFY7lwL(dGr*TGB}58(m7a^kU;8DdW$ zgj#}~EmSz_VX|1t9L9~amwm{b`Bp4hU(3p!`Sp9FW^<{9H(e+pSg1%|z^HjnRQeB?ibZj2 zKrg%AWQFZxz(OuQx7PZkh@^o6Jwz2BJxCFP)vufzQZ+JJ?efrlG8?*I4nlbM*RqVw zG!yRRzLLJ6PD0H(rJ*vyBOZ+k8TeSZhwVNvlc}VJb4K7Am$zVh-gz2*9BdSlg&{@{HL> zh=T;a!jXst226KQadlu3GFnwZ0CR!*`gcO<^Ti~TUK`asn5yT9e`b-Fg}4+}1cgE~ zt!7QbrnjvyU<}0rAn(zAit6!#ZS`mB2&S-{QQGL(Ll%7?+#&IG^Zav6vc&{945C)L zoI9mw+|zZ^kEh^iQaR7Y3Y^`v?(8Va=fYg)37@%U^sW%k(lPOYG9C3j;S+^%5c-46#Z zhkJ9!GD*(>!K`+LEB63eXC;GOIWu^Q-ZFb8VoaqKb5x$4!bzhbLL|D>W^W$JA*o(v zxHe*CYvyUMUFD-!s72m+q0A{It3<)e!YIsvl0w-iS%wf4fL{^I4bwtf0vwj3hjD<7 z`3VNmUdJlf&4_}VK;(E~w&q`GjpS}N@~#*guL0PLDb)f-`93^OWlNnuB(ZlPfZ0QQ ztAAnF%i>dWvGT;!d%7c>>3-T!DZW3XAB-m?jY56B|@}Lu#9_h79O<-rnT9`=V+Xa&>p<393=8)9=r03 zb2AN1M4om_NIVz8%ZFX}4iigQCxqy;*TSHt4*I&~H(4KhFy=rR%@U5@8n-uNwnC9i zrzJ6fxFFz%=$E2Hu#hXA6p0u$;)(0Ng&seZR~$6~HDAgI;M&H1hnHZWokS$E(|OmA zkQtXw#ZMxGM~w`v|ibQQ_7VMp2$2?`#DJ_gwevj$}R^5jA5gGmC{PP)>}V~%K} zrdI_U0+Z|ay*2`1HmMifMmkH~7HMu?kVzt`L)WH#y}*~W9j84}8tqb8`H+5L{UEiLlFk8Ze^z=XKgFeWFptdKz+S8EWJzA_}z*w51rP z;>t71(7v^Tm>1nj<9yXTUj{vDQoBJR8vs?ma9$-@R+kT2xKSUjt6ys z?)+eXbMIqkg;6uLY>~D}k8-uSyiHn>NY%Yt!3{AHcR<xNpvg8FC_*FAxXv?y!4n~GeInRqJc5RGy6z5ToDDv2P*x6>7y{M$>bP%(Ja^_fw^@g=TjtX zRv0=T%F{Z{2zR`&_#I99WK)+h3LL(QwdfA$IJZ#gdPe5x~1t7Jm` zXoPJuA*01z;C5;Vp21_b95@LzlUalIQ=tZFrB~EEUnkRwh#)`XmH`z3Z&?1 zT#_EuxDu*k7#|CF9LsfG>&Kg>Msy)8BWqCjF_|0=rRN27hYIe-y51`($@t9DCBze2 z?uf{)f=f;hU~y^ez9gb@KwUsk;o1>Xu`aWNrRwwim0o00u(#K_Gz&>3eVYA z<%AVWDzhsTteJ(mNy|9R>bh~16PILDXw%#tTRlDxg_sLEOvFKE58?ww<#d3qv5{YF zbuV2|*Mi-k#*!c+hD!*08@X}k9zL~esZn}0TIC?2cw*XLLL;jCOiH?@Zl{HUTsU({ zd6a@w)sMiCj;PU;IACFKV{!+XJxl3*M53=7v3IT+eSwY5&X~AytqNGd3PP16AgW*@ z52WN!FEB2FIX3NSILb~CGgHgwyS&EV8VZ&<$h5r; z-YJkF%BFQ5_Syr8H5Oget#d$H!u_~yKI8Cxi-x9Vx)$gv5QR@k7z*M{rdY3*lsY3% z%s`DD0TefrVHY#f#H#O#C|GXoDv2D%6L8Qv!K+D{veV7u&UQl8BbaOmG+d8k1d?5( zN3s^Y-K5xYNj)s~_)Tua-GK+}lr=)2cz{@GvxaIK+{}yRx3t?%zKzQ?^4jc_+U+Ve zB$o@1)&td$qM_mV*B6<-5ysPzkaKti5UW|xdJu5UArKHp`;lZ!ToLGPWFQUQ>8;6! z#4r;rLN@BY$Y!mc=N(?m>=5vh=^qGhE_++T<`r7PR^a|Fqqp^fO|@bVKozc$pnKie zU&3B1)ius9td~q)4EYu&I<~v%zi$-tirZtx>7^JCY|s$7J(~}7E!;EFxH;RmtoZI? zRj`}q(A6*_dH}lyM~Hmtdd^$X`nxE*pv-e^PH&US`qR>;4JFK!v?s*wA~0q1M6@f6 zjvV%=K1#NfzEz`{sJ%FX1C~XZ8t8%ri$um-Vg`r@b;$ud6l-e6->CQn)Z9{UhX5ds z-e;lMC_3G<*0Sv3K=HnSGAr>7DU*f3O6tDm5s7zkmAV!(z)nR-MENQ8EnejOO)N7{ zgtaE6Oaj6{RUa`eWgg#N{OLE(Xo=34BqLDv0%Z3V!vw`Hlpn);=%!hFQ*Tgxjky*J zEW%oIr1$r6*f?A`ISOCPnmTr_D_Lo6qcJO*{||(;*8R`?0}G1PDNX0aq)T?I{IjN_ITVft9NiVo^3($Rz@i@-I1d9nE1m+?Y!cbCM1@)H71kD>TIN zyfwIJ+?Bc zI=VJ?lGypSNH024_R9zn(7ln;siD*I0#S%V+g_h+#EGT%70dBkz?zB>hDt;?kA&35 zeiK`u$N*;5;g(VP{Inz^l5HC5hz4~R$jn{R)oAjNx~_p9lhbFN4$?E5!?#CLx;vd5R?Z& z18#v=>JfQ%W(jLd9G66fVz9oPwKV}gL|=Iq(RH5nq`kRVF_1BD!LU;NaGrHgs$~k zddrQY&_}Y|b;J$=56i)-$cZNgvU-}%?9KGXx?oEe4uG9KYN>`nO%#>lHkCs_A&a@6 zW%H2_R%q)G;VKlf$95g2XQPJFn1%c!z^s5dOorUO7;mM`N5t=IM^R^~lu8VeCb@o! zlK4jqEFo7U&=cO%kDTX)KT0w-nJqCZvJ!ahjS~2v*qs1=REDvSMM(#URLaom&$%G9 zZmUXEAkbW{;xN1+VgwmdfkMIwtV#*+!*VW-iG>f&Aj8kh3UoJwj0Ym%M(gc@?n>TFhCb`ncQO`ha5;|Y*iJ%%q^AYaPT zQk*3}3xhda^I18gU=5`v<3g`%QT-jyL$u%|_Lhl3s@^N5Ta=k#SpJ6OxzmYfqS34; zO=7X}nx95lk-Lr+9f+w4Fmh2)G6LoYQvF0k;qeqIS&Oo>J+4I|9pNjFBK)z&St?LX zp&3H4B&ZZ94JyW`l==?2hLs;^I7a+J;MXC=;O+x)#X6w2)eCYC-g}M@Cz;f*Obf(= zqM{B?xmw~r`AjIBOUU5;nz>6ogfnCUzmx(U&?4F#&X z4iF!d1Ot{2m65+jb`c1;&W*?x_b$QA1p^XE3Q9VYaLV?)2w zFyC+z5!$)q(c>K%i$U3%!{@JS>3BsDX0Mub{|$us?=)Ad!gRQ zPEVm!mKoyGr}XN->R=psaD0~*(ZHz)#P_@~$&#V;vU{fY?Anpz@}r7|Ac`BAPJsDwkep6@&1JR`j+gj-ym8xF&8=O;dE`Y&b`35~>sJ{bk!RbaL{CyPe}L27BmmdxqP<;TNnhX)DHoy1sd8<)8$ZgB2XCH zunF=k(tBuHct&2NB{D7;5W4Vb)H_h(K_Yex8%}wV3KABRh7Ds+wVYl6OH|@Jpg>q6%Bx0`>Ub&R=ddu37*K_hS0j$n?ts@^nQGGOCDOUltIq@_$dWI7 zv%8>mdNQ)1_U(PpsY>FS<;G2-vcwY)z1B=1WyH$hokhLS1%IpPnR1Z7ngxFK5@ z9!nLiStD-37NEib1UGHvxl6*mZBormBzCf(QKrtSZH`odTs}C5_+Mi25U~P@dVaU3EaVF5g^jb{9wDZxdX6Z-He|0H{ zXo$C3D+q;kqYW8a8l;Y{;RRqXcE-mNvD}gKu&X74_pz*PeEi-JcEq_Xstd}ti+=gY zWviuKlZYPt8N?$2s4OlMdew5^*_le1b;_eMg1!!j85tVCCHgu-`<@Ye1Ms_m;PZl3 zwV@mroyt0>n4y+=2Ggap1RCSgBD^S2b7k}l56d=t=tYe_)-`5VrxVkgyFoKoO|`WptnjgJ2L3bf?#;O0R~_pr}Ca5PPc5j_;vXfJa#kL0Mr|q}+&>KI7?Hln*a)%0D%nDFHro?%XV2?#lu#kw zeDtM3Np89dPW1hZ8g`#)p7+xMqeBj1etG~ab4qBVwX!}iQ-xGh=Ot>o>Dh$Le#wTJ zs7ERY3Fo*<8gVs>p0sBrNisST2;udeXL@4ki9sS|45x#Gpl+wSDOS`GccJanIu<({ z4FTw+LwRZ;TpK6`@>4*r$-XR^x`=YIQuRv$p(w2uhun(nplR-cxip>1hhT6}m~igR z)5=#OqLP8saSYs@C1)$e0uyxePvSVd3BqcSxcbMMe4s0KE89cluNl=JjlmE=D;*le zMU@Yph*Y<2cxDdKgWM!I%Cg5I#b>cmq(!4#G1B|1sTWPh(ld5jnRoQD0%2nk)Y&2+ z+stGgCkQzTdio6gVN&WMWj?Q!K?oobcVsnv4L88_(b7GzHqWA%FYKONsaxn?uvAr5 zR87i-aILTk2&E-^SUwUyyD)a??bPC@%Mq8Gtkxn0NpuSk%o^7gXZw^*k9O<`SyAhQWE+zf~{P z*f>4Irw@;oj)CJWA<}c-Y(v{a*z1rwSpg1!Z8VKi1u%qGg5p@*OMVtK-`Q9 z#ycU@G+UJq<-8ps7IddgSATM1r1ap@tvoSN%dM5m2zhkIpz8E0X=D^5YDjcM-7pX_ z1UmuJN965Z+VwGsI(98RnFiF-LQ>-26dz9uw}ixbvsJ}`MgUoXge?KR00K~84$`Jc zY#8yLcavKyV6nYktkwtSjj{owLP8?&zHV)(a}>G5(R09*m(cDCl2uIyH(oD09~h#A zz=jM6f}n9F51qF{^sfbmfK#db#TkuI^%nQ5q==i4zB5sDNq#7?|0W}EtHqF z!Am%x=HH|eDG85QoeT^P>RBC+3`P)&sa8yuQI2S~7pBI@3G}7$5uHf^^x9#8M*Q3` z4Ral*ZDqc7nyJ9K^2v($*|L)BGX`djC0EjM@m&d4tnpn<0jXf82DJIThf`;AC^RGp z*~LxQ5gX>srb0`cl>VTWj<4c#Qn{}XiB&6AXYpa=P z8Hg*=Hzxk6SpKvqtXtJ=W$6&cW|R{V_=-7kFx9UCTn&O z`K=X-k|&gysF&}+w!0mxsg+tT5ANIh-6;e zGndhYeM^{(?TgCaV(sDzkH|`RflYyKG-Ve&A>K-^NYuMqMOfQFlfMYgY{3cVvE9lB zg~AbZ5t9!HShW>d)Y34C0J2&nz3l3>K>$IRnKSztoc5J{!>Da`+Z_eDP$d$;RlvUG zj+D}VYp#dTtMpaxKR|lpok5NXvXWvg~ANDd4fW@8))9#b4G(y@N7(VZ%vxeW*!hS zt5(F{KnjVYo|H+1LkKMgjNN#hnv4zd$>e}?-6ywo(5%OYMMZ?Q1dxd`N&sr$Ov0jS z%QDTau4ep?Rm|*qZ%Qi9R+qfNT<$~dV7!DSrd7|OLiymGZqRUzYsWxS$xb_*l%!JE;MWyGAcMCx!A|94R{#HkZsE4Wea5&S9N}V zyX^0hl-xrD2suK0Fn;F*T8rIwbdFxJv1o-EQ&wTnQ+ftu6O{O_2p`_mTG@+u7FwO~ z#ujAhgGV*zhv;W4CIv9gDa;)C=6NK$Ma;t0_L_}hNg;NY<&e_>#dPRZ97=DDkjTA` z#T2`US1OH2#bjlE_M-h0ecBBHR7YiKJ#q*U>oCWQ^tlTIWR`eD7*0pS& z@^%49-hPG*VD#Tatd-p}=`4WqqS&H`IJPBsX?p~F(<2n@a}w$-?MA1V$=P8HVAyxg z*`Oo2g%^e_U6OBZK)iP#K?6h3=wGPo=49q$i>2Z(xK%h*EvuBf6~4l83UZ)E(3<1=P8eD^tub3bItKRIXf`dBsDW5 zjwTwr9++O$j0?3K^t};df?}-P;?(t>C1R<`mH6n&6;6Q!FL((WTJ>_&x`zQFNG^h4 zw6u|i27|;(c-^tOiM|jclN(i_I{n$zOm;0*s->3F$XCRzXk~x3SO+w+?FuKVkJV)> zc*w{(t8oZc)I`;OmZ>s_FlJKd=q;t^vtoG#BDH;RnP*G6ZO@pNmwpQ zEeem`yc^@R_N6PzrSWc*rKyp-W)bg5S@X^p7$MR^O+|)NquN6!pHeLGv6|x*KqEbc zBR586x^!b>2=N?rqVo*)CJI-zxi4AE5{ZXm+qt$q*t`OKH|~)7wUjm(9IvOdy zCA>(Kl?zzz@T=Ad2(-GTDL8Recd2p{zH+X`JnF~ifZ)f@;d zre{5ibgU&)RzKG%uH{FQstS{AR=`FRH}9fJK?sh$FAW29XcbJjXh?b#p$DR{?}o+A zz-lf#0eyj@@YeIInl|sP>r*wPK&M^@WDCBD;F|_09S}&k3q?_Y^fIa!Dc?1*1Tc+i zJNHT2O<_v$APeSu8ny_`?MRZWQwkgvD8WO5Q71l2VAKL82y#zegD8tgD>Q*3)XkKf z7~`-~?3j8;V%i{8^5kN}Wwg!rx(&m8UVxCUqV&N72>L^>s`7qVmF+tW3t7ure3UkP zAZT32(6N6!IeTq+k4NY zy_CZX`ebir#$AeqMh)i;AU1D-kxc;o7#Bs3Dn8r1m|$7w>u-wLpVm&IJOns0VXA@8 z)Ix@%numF|NV`j(FowvJFAH4QQwNW`R;h-q+v+*pg5O~~mjw1Nv{d}8x3+`Zuj(-5 z_iqs7flB)`gj`TWjySVyZ5EdNd}+`t%JRZ1^;nGfP@lyomN$##@QHm|@x%o_nNR^f zbJAHI-BBlIHYc<gdlK0!E(HK4iV)LvRcw^cckf=gX&oGWGM1J=C zMdy~ax%HHY7o*|JV$w#A&?Ky2y7!TGvxR{w=Ji(wN zystJHfmqmrr2A(io^nDPAYy9Wqx z;MCTiR+^DLWH^CLoZ#gUrg&p3phorruATusW}#CQ#CjA1h@sFuYCz5phQT%(Y^Xhc zJnaN~*JY+OB{8aMdUTplY!-B4i)Mw1aXfXb)l7o7mKhbB9q>X28|M)6GOH0KB2{p| zqQ={_V|-3_Dm+XDbY!uc>(Y7^vZkzPC}TXpb(ij8mVgUvW3lO=jOW@&&IO_k#4y8T zh_hft)_Jo>y$vH;9hrf!3lJ#yhraSZl+@1$WLELzB}?Fc21PDK_9C|`!|5kEo|UGT zK#OS{HyD=7ESuU9k_Sry*kGyPi!uB-R#prgwz=_(iZTe;CU}+#@;2MwFHkQl$*d0y zE-0vu=X|!(+A%p6L{bxD?+~tm`ro2u8uxO-p9B*wk`bxIyG$xo6?IBfuvK%PRU9}w z9nUq`4?dNgXjI>^Es=gSq@dpHFfwD$5>6I!U zku@58a7hjarp!Nj?oc*Lgvwf27Zp!>@3E$_Y~#pHkBCa-l%Kl4JqR97$D}>s$g{o! zjD6Cu&|kC4>bi=tXnELd=NJ5UL?4+q9-h^m*?6wH2?`yDmR zF1l@=X0M5b^F2!w@yC^TBWbtQYcN75#0oFYu+h@kMrPSu)txaSiR_tqjfT%3A6~R( zh+Ta~K0|G7KZ^UJa3a(TWVdf{Xt0pk6K)$1(7FiaZv4}>8g-_t8!67vA-q6%LRMdT zK(zAlUn_yv1vsjNm64TIGdWaMO#LAebCfg>c8r&bJ>Z^8QB@e(rbem}yYH{BwUVNRB}%=17rM8Dx3SDx1#PhcjtfJ!O`q2rsp+#e|mcK6v7% zY@&`mJi?BR>u`g35Z6S^VtTGfCc{gOPf<#%Qb%Zp%tK?_@iT}J%Mgmf@U%3vAZTa2=2+H%p$}|Ev#v@(dvX(Zg~x%7EZh{vBzoj;ERqpB96 zoSwn46GaTY0O0MwEmpG`C*^X$#CCOke~SCnni$7u$Rqb@2zfq}Vm!i#3-eg@fiy(m zGbsH-B8ddLnkb~Gt~GmRjMplqN~f66zSjfr^9OpC+umeEQpTcd`|9MdaX|%8#sf?X z!8MMY;f^4%o!-#0hmmHRQYIsv2Z8|w9i#{ZGPHA5XR4^DX2APG+WV@|r6PRM0E4#} zgl4Rtgvc+Xq2011!fCvO>w+acDb6(QHCMO9v*p`1K86S9VLFC!Z2<;pmNf-|5aT81 z$rBd7Qq^)!dP=HfLuQh>7tXBSlWh$*k>2qgD|%IvhHNT6@r5+NcPg|h9~&!IwfZ3% zM?CB~z;W1%qr?WJxuU2e+zdCuZ+k9`g-*k5>4;SNk5 zMPieZ?F&{K=%^nJDK%sFQj;d7{VT1{Zo{Qbz39E~#K_%g34=vs3Ux@A{yu^1;i#=CI# zN$OkScWe^A=nH`gFg;&BKoAHmRb?S>WqR5II^I3hBPPe@`sv-6@h<4=fLdJ^ywY)2 zdV*qZh0;LQDn|~-t&4F#6J1C!&j>NI$=t6A6mhYJ6o*5$0J8{X22+3tmDz_$2{m+& zfM2B5xBnhm7P(~P0DKeM}tB&lUeY|{}Ch;-@ zlm~mKNCRe!XKaVfuuwUvXK*(}e#f+zr93m~1p%Q93w3+M%1OW(CP*;atPd~_B6AAr zcW`>2ioAzDc~XIqYY!I5ti`V82##A~uaAf=B4C*Y0%3a|3xi4NXTc2+8d8-pxooR~ zk3ri;q;R`va1v%lwC{k9e3z*1rP>xYS1K`58!EX-lA!pj+~6489?njz+j>!}fQDv<(DS1ZE8hh`b0j!}_y zjrB?zjYe*tde)WR6k2doRoqhcXA%ln;5;a*jAy}W3(eAivyF=t15BdvXaxc&%sQN_ zPkw>IOLCpdl|LKW8dR7!hiAJMc6WFV-@=CON|#)KRawvT?WGXZw2ZMYA!AEQ@?kQ; z3Ai4~6!)RnWQO?j&dccH=x|M! z5|Dw`WUhZZ^Dj-|N3B4{n;8~d(`wyGBoz^5kcLPUtvOwTwjT0X!mn}4ugjrYx|yz~ zXIVp9nb7L7`+C!gb7p6_HaYT;89E{=w#`Z9$t(MhBLN z!p-8wn7Sw2NJ`lm?NNaw!l|fy7?Ue4Qjm#%KzRWz^h=|d=%PsD9St_I-dPdsvrR)mW+s9_4`=LGnH(i~o|!Wb+(P6XA2;$D)F$$~zQYFp=R zxQyG&Lt$gZ(bOtT%=R>($mu*$j$y8?X$jG}l;E;NViPQ5lM`!IzcmOG+3137k_*_} zlZBzWh%f-uMRdUT8Bq~jKay4XG46iY3^T=mqyndOr)GHqM%ZA(Qvz_AGs`x@Z?3<6 z3h4So=amN^yj?=0q;|Gn5g?hibDa$h)lt>#(y42OQKHCMO>W6E8Mm*@+<(D!{Jts0 zz?08I`%+U>6X?X1l&(22CpjUhLp#4q!v#hIs)DJbll7eU8H#tRmw9e}r)6@c49cm6 z3O@oSTX6}2%Kg^zhgsfOSTAv2Y#~^Kfr}KO@?^>)i!I_4gsm*SFVr5+YDV&Dxm`Z7 za(2Gy1%^yD4Q@DS9y+6+Krxvxu|^%x5c0`lkfN||-T*qDF^@p#_j$euWo%Rp#@Xg|CJt zTf0xJpXFA-IxDcKvDfxY94#0Cqn?#^#>gZBE3^{u=)tCFsf5+CiAsb@_V+5BSd|{K zv!-(<#;q{R9iCqv$H15CpE7zNu?)KW`fR0zni=kanADdI!M0e1+Zk{zE zoY2Ugi;3bKH5aLgD`=>y(HRw#<$gjO6E;G4Y6&WODW7P%we~H_0WaIq?(d?tzqP?TXTC5YIYljdWmifI&j9VSz_m#bB|RK~d8QF*kKzYgaTcITz`idw z6w4?GZy%Abt$|UTx#+?v3Ir)7hWZnnW35F37Mq6K%~sCD85k@i@L_9Ace@yX zg~Hj;n-Cd<0Uc&&F0sVnFeD`k>~y+9FTo3f{!1zA# z%)F3^(}2M}~K zPj-7z-ncIoxdU$%NH`=m>Twqap~j=>S!-Ll96sC2&Dcd&%BqAy zDyZw$cezq2aL+_px}QY@$mg?bC^EREu*>e?rs1KftX%BhV=)gH7^XR?!`ag+rC>Gf zof#!oN)VK*SUcMyh$Q>HMUN_Rx_(z&bP?cr9Zou|s#qJWRM=9N^cqsmgwhXD0*SA1 zKoBrH=N-BeapKyeq^UTEa)K$oQouq%leaEt@eiyNM2A8p)D^m4#I4(tZ>s_+ph9D? zP}x0$4dY1@2Xm#Z@1U8ro2Fp9ktzLD>F2+!!`O=^di- z`iWvxwe-=oZZHR94x_Lo%Cm8xV_C@MUH6N9hImfq=99!hfm)109kA{rdm%ciElsOv zSQjCao9{M0b8C+pf{WqA>wYp;bu>ezaeI12gg z>ktq+I7+~_FHpo+4h(n(->$u7?|4#zjZv{#9++<)%5gK5^!5T3MG8EoR%tdBJpI*# zTN3O~M=}jR$qz=gP?U#A8HmVkxoqE5(!ZC3fV=6R%Trw<J!B(;hd0cz=cT|msbiEEYU#xuKf$VH~>jNw!cxUw3D_#IY(w;&8C$? zK@P@EkZIHj0;4CP9wKB+^rI^XScgL#SrE#hAyMVd z^z0O|9lzW&5;e^Yg*73=`|U#sTQZL7+L3O-l-Lk|Axo47V1#IVMS6b6!cECs?<#zs zS|!L1X5kdZ#C+VXR@FfU-8xBDq%s09o?>O5Awkl%Gg1;SiHHI}F#*FR`}5)=_=t<* zpU3O{{(b)+uRqiD-M=4?$KfcXs`&oj{KW-inN4IJm9}gOSIKSw06+o}|9}7x`~U#~ ziZ1(`o`Aylce(exS3cU=yO{0WsaD<1yPWN&+p$l)_W*k{GXjJWExjGq^`&g~?z9@U zy*C#04M?uF>jVfDbYdWFt6HS(*Dq!gCG2{2>qX{jy6e-|TIrQu#-g5>bkw1Fu~92* zHe`|)vDYQB=;DW?RVaw4rCK#03btPEsa<=Rl=jjAwkvxc_OC+}Pc2UN>XGd8wQSJP zwT|81u_^A(BBayPtJg*wxLV?P+SND?6N?+t_bf zkan+kZuZ{nG>Z4PRO~VZCbUxNqCMX6#-XQqPJ~zz1*5rJ;tupwqQs~G|^tnde^p+Cq^hbXwKR= zgWBd7XzNZTb?M}E-MuGUr@Xz?;wlrfM5y)+0H{)@YuvjY&Z*bCk6u@8ZJQ+ebmDI7wy-+%%X8eV z+f=FBYZGg+)w4xXyEQ=`#;d&Nc6xCms_tp7>IZW@y>j-)ZC9)oZF^yMGLlJ4Z7*3$ zq)X7!od5@3hlyUzMN->U)~<@SmF=5qhel9zYEf%T){yq;bJKvRt(&7t^l3>bHp`~2 zvRgN8)-4*TU6|e6uIzF)Y1UgcQ7H4=>UuYA4Vy)}=Jp+}RU=yMq>A3%>CO!;o}6Bp zLXF#zAU&0fv!`@v4YRY;q^|dF%95LFrd3o<)tdKYuWXgOh1#O2HK>a9rEQR{ja3So z>)vXoY?N*5wTGdvY&OT3FsWX*w?WBwxNx*$YeQ`G?l){Br?zrrHM2dtnOemr+Lh}d z_HM9uz2QU8ZtqT#G-mbBfB*mh0000000000000000000Mk9xD6xw_u-h4X#$&Z=JP zx8HBCZL&LS?2)Iw$Gpw5@wHOPuYK<3j6L_qT;FWFVIn)XS60g1?d#gxOWV1%)i`Ud zmuXr%qh{N?M|0ff^IcY~lxw}6=I-vQG0E7VRkn(~z2}an zx$iZ2`)6f|Pn~x=Gq$4L+_Ph+*F^T!dlRnQ!`H7pomlr5b!xixy>-a!cQg~NJ7o#F zZ@YT(*yWaO}H@)PCySl5Y(=To)^`qWt-Pf~coNsIGx6Ti&2!^v} z`|Z0D+jj2t?;X|ecX`&7^zPl=u97A1XA8Tz)^*j*ta^3c+r4|b$>mp}lWkqu&1FSN zJ8ppPP22$0)-}6ZOzd^bMzOkXj>gTGbFSLb*1KX|CdS*j>w8AUvAp%T3cdGq z`g`f!>3Og`(|YS#_6t+p?(X*N>>9*lY}ebBc4wQq`@3fJuRYv6r~}uocJ^lHy`HLM zYOP^e7T(>nt+MT@r%T(>+jq5g_Zz-l*`9s5+j`!{D|fv1y>wa)UfH*Hd$(S*+1~c; zp4?q!7VWz>*|jQewRKsuZR_64?(dcCNx7-5&%D*+=-tS&7 zTWYJ_!>+*IhkMP<*`U{_k9%;djcuE{UG1%pce}eakzaOl({E>TYLRE$bJx3jxOwMk z-9&Bf-rlXf&aZo&Z+-Kgs~Tzd+n!#X)w=EMd#B%d+P>Y&zM2=iId12@rDpPQoxODR zX7sV{?^fH+R?XYjU26ASYr8V7?!E7CHSc@0D7(7nr(1Vx-tTW5*1ER2cVeBg_U_v^ zyQ_+-ZsF4BUCv(ajQ5A0w|7q0H##dqR^+c|VyDhzK?RUHE zS)qA&IZCuZLygeax z%eyyT-sh{Wt~Xqji=?`9;wj()Vx>k1r+@(|CzL=g86A)lZp=gLT=AS<)n#O% zX#|&ErLcA7GnmteO<%mM z6((TfDOl#X;C7u7n)kDCg&%2rcZFNV8QJRc=P=)s(#>rIEOPHRj0!6WK?-T?uq@j~ zZa13^Ifge^&I8^~4H&ImwnNr&(KZ9`}wy zQUr6sac&lNf||C7g0dh`3b3OuQ8hQ%hMhZGeY#JDL!)+Svz&8G49o^miPOi3Np94I zIFbTwtKRX)pQB1v$BBq?_B*Ox$f3=KK~E?qNJOx-Oy0TbTnmXW%{W`&c8xwOdPy$j zKX#4Q9l&a`aONXam@(qMN0JAb!Y>UeuhC297q}C>Z83JIuGs?yxCIIxVl@zyFzWYv z1BH-N7c*Kkg{Gn8cUdr6>Eb5jbJF!Kn1)Z50s*dKu#a;gMZF@UNPJAoczkW$%W%y=fG(WSUirAQ-dCxl<_{<^jZY9%FhB?(2$v8Y#RMV~a&e*c zDU``0RgHJ;A)%}tCa({{dU26!w-rlJtSx(1`HxJA9Z~i#-w;l$dx%RTfXX4d{w@_h zIMC?iDupV8<&YI0oQ3RI*K4`?v(L^ry628FHyv@;J9Wc@m);jSx#6>Mfq+V6r3gl0 zk(iU`98yobeq_Vxp>ePfEWVXMN^)|L)8wBMo26F(KXrP&3?|xrZ@?#6F2qB3LR}r9 z3(cqD4NLSmF}?K;3yA>{2!IM|9y#H<)tIfZPH5P!IZeOURnLtNHYS$OXma?(NO1); zrj9EDViPqdj!N0BGQ=R)cT3mCAO;{K=`XBkLa)Q>U{#B1MUoK^TcD0H(UDP6mB>ym zkn2TX=6;a-BrcZVGQy&TK#~MF2C(RK^%kBU%~g>~6bjxutWJ;(QOS9m9> zYDfrjR0EObM2jCL0RDzkjKj>XkQZ6((k6xU_-i)10GtGdJx&Iu1k+qBbYDjjJgJHt zd3W-!1ud2ueF3-%oHwo)M)-kcS4JWN7b?Z*DfKU?$&)}pG>8~NyzX?lc_UUy`QxEo+n|_&BJqb(6NGbeJX`bMh#SB|WJ0Ft?nPiLRKi)A zH{P3tWS@hWyfp+IFf!+go-LGkkp$a|Q2`Ml1f?r#K=}eH{UHJ;7T4}H@%yH1gUxM* zb6}b-52pEzi7ce+HyK!%H@Q(GPJ|yctGKkio{f8Fbp?3xk7ksEyoeH%39X~2q-3W> zX;DPdp>JI9w0GRe)DS$yy9T5l(y|A-` zIro(**2i+Q*YuZCI}<;#V71n{s@g1rixJ2wD45a(6;9cyfMh_%rK4XF>V=VVgD&xw zwTv)K->e&m5uq0|uC3qd5K-`eV229<2#|>tPN-`X3;{mX!0DJ^-kBBZTt4xjn^G6x$)lzv zILQ85-z3LWFQ%x;a?_1p5Mq`a4N0U5SC4PnZIIa~! zWKlxDR0Q0F?$BD2fiQ4hIT4Zut8yq43eG)ZQS63xA3b42mvJW%EkYiU6t*YQTJyT0 zOl%Ci(WRBLXhD^^AO!6cXf$AoH;aBARD+==7!kbCyVQ4@y*|Y4AB{GAN~GaqRqk-% zhuyF-IfdTnj@*8u-f8Aey=HwyxNQs$T}Uo~Q^N66ibO_{AR6?yHv<^CrcZLOF>nO| z>EYE(&vRxgrmaPT%3Y4=JWm`r!HIM^j^=zGl`!^St#;O;1}jCvS}6!J@oI{*BvABD znqCF;85az|evw&=w0L0x+ccReL>B_Xy7gPeKpYLt{9c)bnhmLR+w zWqPI1wwxsgTev5D_s0!iO=}$6ut(W69AQ4F3(PBjB6Cs+4=SqiVvHmC*=3p6seLVU z#g@PfT$(XZ5(|xosxjn=Ljm-zJkFY23Ej9u*X&%vpLU;t?dQ6d9tnmAv&8X0LCis7 z87OHH+V6ThdrZ0V{2gTtuYiTbM&&q0p2R1;Kp8B|s88FbLaIU_MM(m)-+F|&+CEm# z$`+8PN?OiIN7R-Krr0I8eH!3vbee%Pl09P8m}}zlIG=BV`>3cb2mml8*eD<`l{LJI zS8CDjyQUoAm`w{VfDyxP7)N)&wqH_G6->XCT?K7GnYs&OaFtU!S1wJ3Z!8qZgsJi2 z9L6M?ZZ$}0?Cc4x=n>kW4@{fOjxemGrYi9u9s?&YxR2(ct^oCZtE5r0~Fyrx16l!+!(( zB0mfe5Q2yjmIG-Qwk7WRSp9Z=*pkr4-F-w0l?#BFVl!$}LD51&0dB`Zv1D;R6j}EZ0da#Ut_N%y zR#}UDaH-OFvDhngu`CuPJfLjAMPqEW9_|*Lng>!TUJTqkrLKUBfy1h7Nr+U2!8M*K zS`sYG>y*z=W|dLK#-|td6M1VNM`J-j1MxuV!5a5HOU4Zt*T)Q15SdSt*a409*uvn2 zo^zCZS;gQp+H21b%qj!1iO8b>g)d}q9N|RnzF-i1rY36j9N?98AEiKPIE;4)WoLai z;9|z8>~zbi!I$mhkAvB=NEuZ15)F|fqb#uc$d0HRu&sQ|N*ffOb8^CF8c>PU1hbA& zK2&^mpQz2JZp9|cmZ+sH!iAJV2NS_4I9V}49@c)5VFJlngNbz?Hsl#Lu_UM??6ho7R1=NzvCX)~)9kQ9P5a{(5E;E$k)7GB7$6SR1c)rP8@?-y zgN21K@Vuh6fZGz~1<)hThJpeR5|dXFEsAB!ED$7=F$ai;rqJpP1xyM(sBS|#u(?l$ zbqe?QN+w9dw-0>FtL_r6l`vY=>&dWaFfoxOVFcPKZ%U)^k_~4EW^NKk(EoJ0FU0np{JXj zzP^xJ0IM?F$%EB!s;SAN?-5PU7nM5rsFKXUY6DIAlHa*<@y5p8{3VQXc_5}GGA_kf zD-w_!6m8MF7;Sc%Jv0!R=X_E%r_+bTteme1Ed_uk6LSy~C1D7{J^0#P$CbB6cUFPS zt~(78e8UIo2)n2gEzTuqs@;XS8E(oV23!My?x0I-?^D@xM-s8F^tQ5#FReGcD|ha1 zE;Cw*IeDfj5EN~}7tP;1r~zW0SuezRC;-MzW^bq9mV*ULDNFZzY)b&T@*a$vh_zKt zO+RFuQLK<3gJWXkHn+B$+^Ld-XDWt4)9Mxfmb!nMb;DC zSG6rdC@|zBb2N*@!Eii$NM){V@RT7X$vuab?#81$_1b%~b-#+~>D1_1QT9q%gP(`X zww+`HRyq?s?h6u>lRXH)EA>^5A#z;;y&8#b7&2Bw*c`dG_{TKgIM1t#BgxuJ2A&k) zgwpf`f^drY_n$n=P~;RmjiB3m;p|vG?SaC*U5n)7D4~q#LJd*_a*)N&HBS~yGU~Ma zwdPur6^zNiz60w-NU6P*nA_2zfM5g=5daXfSp^7nc_4Iw0ysc~q*TpEX;Kmqeqa$* zA;NBv%1?XDvPvrA%1YpePWeD=pExHJY@F|Ob$OMjZ3&&?p#8g^okwVbIy5+$$GPSK zoi~_dIBLyj8hEoO1?uXNbKP9VS);Uvky}u|7EYtCoc2;`Yk@5BTegiwe#;N1zAv<( z>Ch?MqKXW!erRCV(!eAM0SWciOqC_oJ~3Ge^zqG^hg^=x(e|`%=yaaJljNa|*`U84 zCoOHn=BwZ)dHuo+0t54bbz0%82!T@Q@+EJR(*Z+E)ZP`m-*{k%<(D$WWPMSN#Q?8o zcH12RIeWUybE=RbBLcNyIs;SDMzfD)=*-}imm{*_H!K8VzfkYH#SMJ%MyTSx*VCD0 zBps2#*Ex>wBVn8$Zr$^B6n5Y>Ah0(?1cwqeU@D=jP5JDqMWBNE+l<_yPD8avnZ~Di z=D3Lx{ZDmJs3e;subpHmOlGw~*dp^i$8F?`=J!d+0pLT7zfeF6oo2fskmlgVLxSu0 z{BAuiNW8t_V!Bn9$la?9?M?D`e+lxheoP0IEKjG*+OXo z(>$4c>=MwCks9@%F(Z+5V4sKtEIi$Q&RZmc-l7Wk*M$)Zjd*m08aOD7^_GNfxuKJZ zvH;H#sxj_dl_>itDuRN`4rW4jo@`gZU_VIpIH}6g@W^D7ou&5dqXzsS4NYWTU~9?* z#n5>|r^rux3>Y9G+eWRJc3E_U3VteHjzTM7-FfCyR9S`Gc2fQea?p$p;bLLYg1|7e zTVq$Kh?8{1m39imTk$<{6?A;DC5Y1QtG@`_lzrNS+?MNtF%LTN9i%B9P@SzYFlgv7 zA(yTIPdNlOl;%4zprHDP;g!GO8oCRgtY#ApnJk+~1tj1Of?#qb@JSLQ8Q1E9LnpPK` z>0giTEL^{PcY-=7>L+ARi+TCPvIecw!{QTUwkAR+WTNEWZ9<$>DCX9y@KdhMmGz0J zQNg~bB|SVb(ZPgkh`Hn(shlYqUkX{#lX0+bn%8n+uGDhZKA{vd^>${5sQMifUJHy~L;A0x9uwT%nfFLg- zCs*bjd|f@<&tVt4Nyra%2n2!0>l75sKe@HGrtogY1cmv7jhsUKU6r>Gy4Ri7gnA~P zCg9m)Q?q{-+PORBs0#Om#CYoywnc!U6uc+k)mGKedu$(S(!6kq32u8BoH+v4DOiBu znJjF_gd%$jnWNRD!Wcx02V<#q9|k=is8}zc2}vgNN33}o^y{gD8qG7!kx6t1yQrl~MH% zECgslizmeER!RchE)qD+o!sI;f&vwlt8kE!Xf~L2sZE(YIqmang&(T?;8TuH`Yg(h zS{f#iU42-?4l!liemsenNhn1)#O0io^PiF>bDS((W7na8(~izi!g7oWallv_qG%T# zUk`|mS*&`KkXY785*=K8RjJ#?6^s`o=+@hs2C2qIJW7lN0~+XQy|}bUX+(1 zKJdr8M@N&{z0uplM~>l%Gz{u>md~m~7XBkt;%pZ>F-Vu$GI8XOQzR=#dZwm;o$C(^ zH?BSp4;B!3_|>`MfNLbF9PAg-9S=7mBVOi?e>0m02!e>Pr!^3S>3t^J@9Th7xWw)k z`e+XU9kNR9V7J>d$R^gppJgr*+cd+?W>pdh`r^q-0?j_kCJ>=ULy|;ex_Qc+8xzW| zf^gXjc>6Wd2z4BQY^Pg5)@)0$7|khHt~_+o`yN_k0U74Sz(==&zfK|*rLr|n=Dtj@ zNMscn!*`Q>#Prc0xESGB zJ*Kp%W~JS4hpkC8g)ShZ+}Djet}z7`dBhT!%7!KhV4m?xGS?Yg0yz?5qvcwH@sb~| zpu&WNt>vbRXA~dY=0VgZ)GUPX_dJ?+@V2f6%oC&F1}|Z@is>+6tf9Wh`I!nG^n=E> zAdE=!&D5Bul|;rDfO+5}?8(9nH`B@~ibP2;%Po=}h`U5AuCSGJ#Q4n!IJKLc^TBO6 z4qIlDF9Y3LOp&32w1*P(jMfj$bEd9rCyecvF=Nb)g;<5nDIlnyC@iZ8!d^x(KTLJj zmrlNri`b3T@Ji~bw3~If0o}-x+9+|X=o%I zPN&?MZqTa8P$;IJB=uQ+_GM2DE%j-kPe^i7c!lMCE8x*1GxFzlPqueFMn}4M1@qiE zrK6kX-N@e!MTLm)&Io|Vk&XH3+&jn>M8P!KH-%L0h8a2M-#8P=Tyv%6nR(Z;h9OL& zg2`|t;s~0I8^i;U)T5H;>+$AIVn+O?FM3eQX#`?;P)6$_6f*F}BdD}3atz*|vI(0y zeyH_AflRi{$i_R7rQJ2-43bk*xO4fCZnJtHGDqB+6C5sp+00k{cERHm#b~cMl-3@bIJ1a7nwo8R91?lv^S_!I@Lp({ZtrlgT zAdyV|!RzMjF>&oX)Xv+)Up;{IOsLxhsVMU|j9}f<%xRDu8PLgsF~_H+alx774A9xF zC|w249_P%Xn1S|GxDq2ftgk9emEI+@eSHWMY~>;|+rp|-p%&^?q3+JP!A!)C>=hG~ zhw;9EP6TkNJou~4LWDIBX-8p@eyr`W+Pn| zMuH}JeubVc+6+My#wj3qe4=P1?f12?GK%2eBpDNeJPsF+$kgj#VURPgLP;X(5-w+0 zM(kE#b=@~HrPvw`c`%o>5yt4Shdl!6IgovcR67;vtY8_{8o6`uk&<@c=$#1o*&l~p zNNd?j!NJC6WavWqIONrM=Kck@0YQ6QzjGeJE!^JAhc){1w?*^}V{fU&Jeq0?u=s7` ze9^b_o1xpNgDu^c?%vC)%R7*9p^r|z3Z?;*N2Tl=#)E)Kc(3v4g6v-V&XbfvohPFe z8-n+%vB`&&<9+KrUQLH4MNbSOn$sg-gSF`3zbjKuYo2K7Ph&E0(yhWHbK6PC*y}F4 z_VJHB5f?9Xolav;=+>_=T8*4oEse}$R}OUI?uH;CoIPt$W0AnE4rZJ z=i(e5sW9p(g8MnL8e-R8GoIs5LG$c zaV_HDI3TNa3@A>>exV$0rMG;rcZ4N^?LLEx%THFyO?{D7ojf;Hhga2a^$~Ea&9@00 zvKFhNL!{E28ebAF@&vm)6shK}B%6Wn5e=SQhQ_on3{&6=;Je9q3y>9ah((p^FDh_H z0L318tLGpUe4}{k+-EUVj3o-v4TiCLb*A}#tc%SmX%miybQF!V9`9jd!@C$iy{rrqHQM;nRq-t3ED?lM>L*l<;cnfMOH#JWQ20z~&C)d9EI zq*8sM*de7AYkW~liGs5Hu<6zZi4`QfB7)kmt-!X7+9#mfc|gWrp5`*fJe?KDr>os4 z(U=-ekzro}VVp~{yCR>9R<4F86F}&RhqoOcVj)jS_fna1yJ|w{mo?@6cK3P6h`UDF zApC~D=C^bShUhxx@NH=zWs*Qu)rGlB#@uUD^(j`#kX_z_hS1_O9WN#3%B@0j!@r%c zKyZP`fVQ0FgRSOFdaQ6?aWfpafuVbR{9Od_z}B;{f}s{%^wxTxJ(B6gQy@Xgv;v^D z#Lx@#(GAQqe7(;gb4b=M7WShXSRO#;1J>Puj=+(61t0~GPIsx6EoC-chPy8^vAfts z3DPGQEj4|nxIBo7@~YWe2+F@?bv0HA90x(3)5uSNfusTLxi@>uAyJH37?0f`V4?%@ zIG(EoiVPG=A0?h;LwU6@iedM4Owt4^P)v-Ege~zRalHPuLtA5De4d$OmrpLl_Rn5i zZ%9X(2iGsnkw^1Kh_Jc{&+X2I1BWF#+oCK7$&p}ijJ%s)P0EId6#N9uyc}7WbA*lN znv-Vp5EHg&bZqrcguoSig^S*`sTn5%c4@Rep>by}?`?cfr` zJy~MM#V9+_U{8H3@wk{oGm3{GRgMPJv0;w2mdW%jF59*G#XwpDLpgHz6iDMQycR7z zy!JCUEs@zvs}r;?iDki0YByYJIHfySfMw7HCoMKb@?i*8bE4^$%RuTypdQ)gXMoBx zJ$r?c>{`i-i`@$Imv;mf1`1;CouSWkv}`>cr|Z}rZG6fPRE}Jgd_#ocOUs)V*1*gs z+t5#Z96l7CCU~>+6|FT?CrJ1z1xMFBy*Bjm2O?yR@fCWTGx@-wlfqEt5$?qr5gn`P zAQZ9{ok&yBouSwQt2+7*Z+^tCC@zG(acM_$jDR$<&30wLDvfrB(=N;Ih9j{W+vBJ5cQ|SRG@yz zK=x^o2adH_c~Xc^rB*c#&zNS9P+V9mXnBqD2E52A&b?TGHjP9tE3Bg%9%{4($^*Gb z3mG>AdVvkay0W}@cp+B={;-iqABo6-jXRV?W{z&6h#xW};9l=+ttVA5gKD|3!HYg} zuX-JsQ22=!6~d)f!8|XIhY>k}eQpxnZI=pm#n*UUMe#Zs3MEZu<~%-arX81B^4&MWGz z)9{`nje})pf&Ib_DvP%c8f$okhPuhkDh<5bS6D`5#<7L7q|FYI2U%pm=(uw7oTj*X$Rw&6PI}WRTOwHmh!lwNl0(8^cqG;gwU3^jLD+G|N$sjL ziGvLZ&l9`+G3^-yos#8yEH`@_ewhLpX$u3|%8*|Q91_I#&Vu#k20)^SP{ux1R}#Uw z3+Rtfzjfc)Cb(@d>&gNJibj6vRhk~wkxOh-t4I$4ba?rsbrRjTj5H6(6?+}oFat~k zSEh9k_gXbmN$O~8K^$`QySE+(G+$0Y68jM+jTkkLS_2F-W0jUK*3~YqKVp)Htrmnr z_83zfY;TUzW9JqQr#Xhe!LBSIcSQo)2%S>lx9o!h1>;%jYJ3^r8a;zp))K5{8Ipr~ zxMqpsjNZPSsCQA$Y5K!oNhxp#F9PVnKylX_=G@<~9}DFTpBf+tN0JQcuvYu#pEU?5 zEYafm{gar(TyzV*FJNhkaQBd7w3-0D65cO2R~g}N8k|N?tf>+N=dSz$wxoP-49zXc zO&Mxc-X}xKu^0^FXS^Q|Vy&+_hQ2~gtOE#9Inj_v=!?m0%3Pr#G{`h+gF1(?O(<%f z0976?R=*8;??`P9D7Q0TC=^`t2M=60%e}ud&{?g^5Z^=PEK_ZQg3?-dNgL>*ihhe2`RU2 zs-R~h5}$eF39o!Yji};bZAz2{G`)URAGCHE2L@%c0jcA8n!@1UmV}D=f|77=NW4nS z#FGNLj7uvUN`@74EO=jS%k(P< z!km|C0dZ)-2vwQURuEI$%(aTTuZ?&Y0UVSr&`Ae+j&@t6f`x;Lf-e<-YAk1^CME3v zTE*EEb>Xq*f?j-enVNEi?-};)?l0afd}-9UvCx^0Bge3TlMbw6oU{TKeGE?cWy#37 z(Bs+^iG{0Y(U}QRSm45wl{_)9v+Z9d7qO?rbG$%HXl|DH$+dcs3CVMn zHZ2gw5<-PnMm;6O;{yhBjP-9lf>({X3$>Ct1fsq2`0-k_ww0Lkh$|>Itx%sJmL2mI91fk4=RCfE>Pu9; z6+vVxAi)X_PN4PlVICl4LR>+j*u;^9zzakOFoSx4aN5HU!C;k_f|ORS)mV{cI3iG| zdu>P)JtWK^xO#WW_n?&b1sg%XBM7{5RWP31smiVH9CG_NlZ7G`TC%&xlDK`WrPath z?(@v?lF8o4IbF(uP(ZnZSiMV0u~)XGUo$ZUbR$$@aZ4clG8oR4ydmQp=X7+p80Tbf zjTuBEJMI;&3T`20S}uVfUDLWKiD9}WhC?^pffISrt$Jx5sT`9C6wk!*Q`- z-K6_jLV%3l#(hjSlYt>1UR5~D;hb`S)!(usTC7r@^jsxC;xc(j)|8Cw)b3KnXp7K9 z{$5(!o&x(pr@pDY^s9x(Df@g40v)v#&%6#rM&xyhhbmJKUp!VKlf1w>U^`NxR`* z;>85?Hwn8Ryv5s{VVa_15=lq(2tiLT552fx_;OM)CxD7CmWojkRuZRvwd>@9u0^5} zay3mTV08wFy0%xq@D@p1&fum~@`PxLae*MVbQ}*;CgNy5qFYBzPdMI2a8x6q52i6( zi-<%zL6%&*W$1W9tm(H+i#)O79LTNoS=)=;23O7Zm?}(I?3~=TA$vW0nBNd7$eA41 zY(JWa*}!6LHFzP`vrCzn_%Y7(1gK&)smzsz?A-$tYn`4^EwETTb1WsQ0=(3WJ>s^- zOyVNGpcgTK%)yId?yxQKYB-Mi-MqF21;N}NZBn7z0kFzcZ5y^71La>+=&N}X9SUYb zw&B*hv-9;Rm@u+Dm)ia5G=l)SI7EyQtTJb_ENyfP3%b^)eL?qF?ip~-P0f?G^b-@! z)zA$PB6-G+EQ(v=DQ|bN8<4qO8QjKQVs}@7)SRA5rAqM@0N7jGYOe$p#8shCY1)Lf zwLny6p%vK6)hCYWc2d#m95V3ZuN2xs2CHkf;+18~)RGHBAz1x31I1lXLjXP_83Ji# zu*$X-ERE(VE#?eU>b;3%3quyvD#d#2Qx*E-ObutjYWy--2}mqW!$N5Kx5!F-(@Nrv_ChsP6i`u>I5v8rn#)zNig$Q+!-i>#F8QqpB&6Z!LY~20Hv-H|qo=FHj}&1K zM||n$OIyVVm+w1tNt2b$C?UbmVcZQ#4U&0ht3+P2nC)>NLmy1GJZK89m8@v8fV!`2 z<5M1XJw`*KxkDCsm5g2lo4`sv8{TuAQ-_Cb7ivCYL|nDLkrc@W@ppDi_69D_f~%2% zhM0>R8=Ib_$(ckDtzcXwwMR%(J4Kx44LMobmv1{0VA`s#4YljANY`>%>D^(aO2|gw z4aNxenBWnyF$yiFW=aXHVRU278fgR$syY#eDgpY>okOj!5JvMtra^Z~ydwB$<7<}y z;O2I+MmYc*$LTxmbCD670o_b!qoOPm^t_8WYH5Yd_suYg7fyuq6EyOfw|c)bPhNcS3d5CbW5 zNb%N)gDwS48{xX@mji@bYeXwKBbK1#Mf@!>ZKHX7s`Qxk27-R!$@d&(DC|NNChe7E zb~|L&bJIS&*kUl!#sPx~uu|5qtvk5$%;F5lE*PuQ3Sh;_Pg>FC>Tn`M!sfP=p)u>v zIQGUYJES&vUO)W2qlPNmEynY9`!iSoaPQe73$E z6}Zj4miC?3)|xuv3)dZ&o*zeHa*Rj}c#Qzp%&e?HH!Uid@jTflfl~`14he#yq$23! zuBJB_6du%S?R3$~i3?}CqpccZ-karLb%<(-+tExSZD>kSfcS@lGtm*9j~@)cYDKQv z4G@hgaVYc^*yR>XZZn=#`#Tq5f1 zn}V#tNs#S^FxrM3C4<9!^iQPw!ICtq~+DcfwzxzRRoGa zPqOB4u?>7cs)8du0nFfBIleG~A0|201)#t^b33IZ-)E7ei_oYcJ3^J^J4%PjxT#VG^{=MBpnNtqCP~XeY!K!Du7LLoDNeaWg$l;O0`jdog@YW&&%)oGjMM|OL=NI7Osf8N{^Y@D3vw{h-Wld+Gga~yGw&B|y z7>b*=Y(s7yG6&W?9fzQfUvE{iJk?6?9Ew1X0g9E`$7$;~s;r6)Z!Q_OGZ@!9Sfe!d zH)FTNEQe5*X77E6BG9ejCVX09Kv}-Qkp)9eyE*}W0k9d{cO@+3G|+hD29U7|ws(2l z@ddk=2+Mewx5sJN_XJUEi3n`FO`IO`*f4X!95mu|PHeV84c3V}{SRK`^Dh9PERCZp z0S>TWwjvN5Emm@ON$Jl067TDArz)R&D)?9ySJqmqI0*aPnvUkpwyu!tC5u*Y!N#EBdOB`r^)Gu8s9w>MjEsytp-2lg+?!kj z!%6QGBg&hgr)RZ7?G}Loz{8=Dh=rA|V5f~nJgHnI*%R&_a|9ifQG)|iG<`#MdPRE3 zE%U(BQ9w%0U>exmAw@Q)6T^3m2p7vXM(Jq$7Lw^8;__iZveku5%GnmqoW$JdqtC#2 z0x`nG1!J^j)ybyzndQ9BOOAm;@VU zl|-x@k01>J>9IM=x6zzhgLz+3^-wFopkb0|=#1>kd4_ymxtQ*J4YAio&rxU9t*Y_~ zd@{sfKzy)ak+WlRz`&Rf?*P=q08VNZ!0KJ=t6Mx&1tww=P=)$@(%cZTh@r~%=GL_r zSP&>ng3J!r45o1hy+c1|380fDtCdyKTtS*nfcf=Ls_HWEeu1<`71!g%DaNuud6Q!z zR!clv94&_A1P@9anwM_`L!Fccia?a2Ud;2r3_J|J={J~@Up_KxjWVqD+R3U8gyxf| zP9gYxJ@8%G8KZG)=ysu>DuK$yEf(CJ8#zaMI9>|e*T8BVeyI5%a4lt3W+h*!GX3NjZf6nOJiv3*MoV!2r9xI>HAsLSaK+G*xKK z3eH!oVeCR*&<4Ez5u|Sfd1RgUD17;N{!Zm+HAQyi=5u4cF#o9KjT#Z=md3d#;m|QaHgIr$t4Nz5(8sM`*q`6fW zsb5Guys69rqE)MU_P*ZAi6ZS^ulXF5e3ZWbQDsix)e?G z_PnK->>nAIbBhF*ZSdJO`Z^Hi3)*2dR~TX z8&Wsp^YOahyWznh_P?HApQ@SXIk;aMIi0m-L7I4h+Bm#mICPw8>s@eI=Y7hEu%^VU zZ8Y>BYcP?)4X%C0b%fkh&_G{Tyx)AxhY-x@``Z!|e_^ixCA7!uUMQ zZs5;sAltLG)eNKz<)2+Y6{<9B^R#Ny$Eiy4v z6-FGegb^SlcL&vn-g3p42FS|G%E@0fokqsvZNTYStBES0`dRWI6S0$+tF|6>0cikRyxlUUb?rIWOd;*|G7>OSrWhY9ZPCND4(AtwBt2bzD)3(vIoXH< z1az?g*C&SPkD~JfZ3M*Aj|9b~mpT`@P?4ODT8psTM#IDbE!iuSu}n%7I}40-r5o84 zPA!Pt+6)rqiP=7sd^6I2M6ReeiEiJ8YE^_gr<1~f-!UN|O45;FKRzB3Tn%edz)lViSLz|GkmyPjp-TJ%;Hz4QT&XbZ5 z?X3CU&27>{E!K3g$4hzc@s^ zmPwm%I^vD(V$Moayv1wjg*LL>qiZ{Ct2^3;1#zGTgDcBs3p9xG)0zY+ub8mwtj_3; zjWiz@5itOUUJttyF6NL>GB-%-Qz2}+KwDV#15G0d3qZUN+$*W)a`gBK$tJatI_fM9 zy47{ZiCDnF#htd2!#gNvYW`MmaNUhEYS67t#!5=DggWL^_0I$0IKtV2=RL^5niU8x zHs*Z5!B8Xz%~I0)0Hk2GtK3dN(DxEF{Jy)XRh6=1Qr?rB*p6dTdywK4XlODQ^3hNT zn6+<;NVj(0HtO3!yA(&Kgy?+$Oukedt&&hQA~|B~!!bF;H%>RSz`*%g)SusF_{ z&wCfRGDwt`b(7M>?IN1wEA6-BAj8q5%LFhSluR-LLbxnQ1_<>uxDBYzZNPGv$|pTA zJes;#J@6f;2LyY%&5AQGtos7-=w9@#iTM*0A=k>tNN0Eh z?v%!!?>T_ykqHZ`DV~;63eV1Zp;qY(mJuV*xq8=)$QpKa#VD`kT3zrXQk}~uBUTlt zdNJNPXA2dp_(MY3fcSOSekw<|&xRz0L6YdJsX7t6%n+OdCM~%RpmeABidfzy!`#-* zYz#oQ~oeLn; zf-M&fE@d1WETVa2I-fN;@`k<3=I+gE77`b*qw0@BdfUR6m=zyx`k`z<729XU-Xt21 z0Jc6)5D*9m{hOy&uvRBj&dDc27K1FEJVwduo9n8jPcm#pH%r?HS>k$osjnt=%s50k zM5D)Hism@E#GUZRO{=8GvZ4epkuTBZ<>iaa4)v|C3<3;^fVmcMnj&|Edp!B!F_Obl z+2=vbNCigLm>2`YbH?1gykzLHL1&)5h~R$O@vjYEg!1HEjW>aWFj!wrPL%u&nIQE+ zdrF3nO;@q#5E&gHT|zrcb|Ix7i|Ujo4qdB*Xxc_z6~K5JxaJ2@VNMmOP~8!aqT-8s zZ;6Vb>(eMmZJ3^1FEBn;^~9*AdN~OTy9|}Y7+7Xs58WuD9Am@g@KQ2f!R`XEh`5jm z?3Ls|A!D$kX#_7Mk6cTbsSUGwd&)S`$2xh24ulbGs6#SLBnZ%roErMm%TZbx6%sV6 zlMwA9BA9ufZh)*l5s%4yYgkt&fEYLm89YAZsWGP5jBVlwr zJq<7$67JWkze?6`%^c5|%+Xd=6bW0E)UtI!B{NMDs*%s^b0(hcNz|XfuhnNXVXJFOMN6FY*Px_&43b^a8@J3 zm}D-IbVz$O+iL3oJ3z$0v{Dc?!G^xQ@@>zp3GviuV?kiYoX7i8m*SHS`s7Fm4%5A# zIS>jijVWCi_6GCjDTa;+0g4H6KlO?U{IO^8D{}Bvo0a}{zFl0 zCG%DH(Os8A;Tf*_lzD3A{-bqbt;oUX6<0#QaD}o85W7Z5T}xUN*HiQ6!DHL1Ne1=0 zY1Sp_Sjnf8H%pEev8eV>E8fFaIlD8&@J1JUdR!%}(T4C|-Hnot3L4i;4T%>EYuV9D zUMsn@N8ZLCsV;`mkQUql*;qHMX$=3kT> zZr95wL(z!wY}O6H-qI2rmR4BJ1`*aNADk8DLY+=1q!#>CXD+tPX{TZjE6C$Y1Py!2 z(`UP?xODipF7HcTdJw|3^1<#1*wm-b7KF@!xTC{;d%}IYgHlPs+8!oDKeB406yUY; zLRU`05o>~zCO?hMUCnuIwi}+l$P%Hp?da3XPy{{{*eQ2r0}piL5{YZaIjauZ-q&kW zF2T#(Y%rbS$`FU^mF1{FQ;?mDBt+D8B2m5W&qIkJ2Kb`EnjPHNWY95ir7fCQ z7NP`$YzYgKMrcyf7ZjxmY&tl;eXyAAGGhANGDO!37fw)t_&Oj7C1a^KCOq>5KD9-g z$m;O>2VYY#H4I*T?-Ri}zGoL<_h}e0&zQr*)>+bQJ-m5@0xcMwth7Rel%+7$mA0~C zcjU|KYfF|Qs2UcM39XN`b-0wI{B{oDJ6iToJe9gMp1wR|8v-E(y{(N$3dUrBwY#Ui z!RsLofX`kq=PV#l{o$RGsY%2d)_c@&)v@;FKd)D++~LL93Z`>+U8;gdo~MQi^5rYq z;p=lTVNHO?663Js0urlLBc+D%74>pBtri&(j#tiihY+h`h0bB>_j(?R`rkNPHdB$? z&MwIu+?^5?>QWhD@KT5w4A)Zci#%Q!8%G|H^H^^bw33-NL08-{W(no6g-h>6Eb@?f zpfM$wZ{)MSdziB@;^ymo@J-g;?2L~M-Y120XclW)$3$z!b*Bej!Bev0 zFttVn*mFy7WAwAnd)&*}OjfHhLL|ea+Tv5a&mAs{l?t1o7@z)WBewi|~h zc9wm&Z!zX*CFPR6q$#tRlAYM0&~MdkiYbI(SZ0^7K{Y1LEmz6DT{at@dUhe3V%;_I zs#505s48aPCF$n|dLcTkFr0%6!{i{4Mim(|#OKSF+7xw=aGR!i4}D_sq?+^@1~YE7 z_f7N-J?#ef3q3C65i30TY9Ua_lA^MEvqmS$yfSO1cJP6i(`_hoY*#9L^~7mdx>jKC zTFxG_Gq7f1Zjkkt(in+Z47c3Efy?d84Di(x;lO#DgqVA#ev!dUp9pFP8COVVmD|6R z&eEuDQK-z-U1u&~cTWcL0RaJ?7nTb-!0dhNYa-qmEGos|-&&X@l)K`R#DanajtwP? zVM-*md!}49Vp=cksjOwKgL^T4A3$S=3)}D$25*Wv3E}&OU%Wot|IwLVqA_CGO%YNM5(eQx($Nhvrp$3Xd@kQd$i! za9S^0V;$5pwa(i&^ z1z4lz!iWjN~1BTH36kCjT8T1T{j z2I2%AcFJX_@~~{0NF2nZ@=u-7njx0bg%RpEFeYIjfTZ`vlp~VWf*VDLffP8>R|Fzd z)XxP22uHUep|*0;%#L8}Vxj!oUOu0V9tDUa4JOtxgWM%ayw)Ns=B251lRIfSalV79 zc8M^a=~<;-Shc!P8tI|}om{jrwTzczi1ErfYdi#x5yGOZYFXn-$7g}D6?N_mc52XK zy5~b%DMI+@OC9VSJ--_;T>(5SGS`N&m~F)&m*8_Pz#~f|N)TC9COakV3UG*Y;mJ#Z z66szaMYv!Vpigkb$loGK5OC3(+r(8PTq7Haba{w$XLhVk6V~joh6mHq^c8(y=Lp2m8H6pfSub^ zBSa09#vSO#pcphxjzkzuh!o% zq7odN5q?frvD_8N9JHLa7*$J`&mjF+VFX$ZCt_8QV##&aC(~YAizAj|9D|j-HgqVb{nTf2%O>Cm z$zyMA+aw5GI|Q2LHhatl4z?Ax5Z^BLauA8zhM<5e4BuSfb)nKaTQU`h-eC(kJCbxQ z-N`@GNoZg+&oI1TlI(AGq>_Yb7Nu5#FLe+bE<2RrE#y#1I6>m-!k98QoCV&~X2h2X z@g@wrf#CSbsnOQzZuE~(zNpM)?kM(8PE~mvF4>;>?L|O5NE1+wAlBlX8i%OKxFkng zOt7{vHr!GW!Z%!s!p}-?#g>xq6v>X+RV&9M939onf@=GC5V!1gwE;@skide`iwi)d z`!z?W(3Vha936r(9LAdAfO9PCbpl2R(L>2ue2G9&m*z0^@S2${Hv?={q@sjl4l^3h zj-*Y_Gv!q}!WmJ}TlXaiIJ139Xf0o6CNk#DJrNqZ8Ksie;W+dbj?C&M*?My$5sk1N zD8z&tu&bR&q(Iq*LrVu=v~uq7(B;Ll*vq<3InOfeSJo%!v9!M=m@Rth`rI#3r(q5* z+29@URWmh>OK#-}YU9;s+YKmDvQRWTiSdGoNn1kDta}$s^<|t(JmJJ>mob96I?3|d zVo?3=?vpQMN{LPeg3(}>u!*cOeDG<9eYY;s1sB=5cC2E(X=zzPkrXrqh->bs#92+(v5I2mJ_4m zW#DD&vYi4%UZO=7JOg=~LGi<_%u?Vk`^vW4YcY>Et})0&frcPjta=4AS0F~}J!k1@ zlZ@GcNEK14iH%*8B|MjR3T+&lZH{^+U|Ozg*4~WkZnvG{%6Jx?v>5h8nt{T!^ksSp zCi4p?lyQCD)AmW)ykl@@t|V$h(J2Ya4_T*x_%-ti4FbWX~3nJ|;m8Pk#YB#M)iDMoL9JZ*lb;i2| ztD7LFGlSr_o)nUKn;R?aDWt=@i(tE0(;If=GNzot(5`t*Fzz2euG7iLgD@F_V4O@j zoV<4$x)33-a`mzYY$$US7-ilh(i!j?vc9|zKW5m@50nZi^#mgxr+QJO>C9a32_MCj zn075Bd~10#vwqNH8)@*cS8!qVi}#N`nzC(1SkrWyE>Bv}s(U07hP9I6R7?;wg0gA1 zaQ4O+G9>e)<574%%xUrmL@TwAR=WL$qicqSkSR2Z>`Ri;j%^c{YIY8Uq*uvcp3k(|}ivr0&7IbqrbD73A zA%%D(uL^ID*r>}nL{u}}9$a!sA1k#eQ9i6RvW80;|kxn@fPW;owCRckWc~ z#s((@+0~+I{x2lSmnlw5oiR;hpsCyu!VJ81v_8u+4jGntL2jvXrzvTu?-|58C*Mtk zV#!1v2?Mg_DK3n{=~@#A$?roiv&xgHFW$oW5a?iNUTg~w(c9R}87&>!LQa^>3RNj` ztx~NguMCnPQeshnOO0v`k(iZ&e17SC!afp#P`?8}7~#+q~2mJGo>Qg5XvVqykX5tcZGy=p#hY zvRF_iwK8*jEhwRY=&7Ti;=e---9eV3FgAQI_mTmHtL;8 z5fx=k^AN)a4BZbqpM`SE#ze`wHq6v~wS>|kj_ApzoO+yyR) z9hBe#n5OU!Ow-)H$GY>Hd*a74*2eIopK0DmTZwoDUf~BoN6V_{K%LUFBdJmC>1=m- z9_M!-Mel|6xCyQAt-1!M>$u`>{KAIY9IwHsbGO-DD>pvu;sLz5p~XQhiLYD zQ18K5rEq(yd{aWb9#0VJ6ng_8hRsYx9zFw$ENSv6{OwATz6pyoD|4jThRb-9gZj#^ z)vXIT#2EfQw@PU$8nHZ!MTx@;+I!%9PI?#@4MzfeDv%Qwq70p~B%suUPc~PFFjseZ z_1GZWiQ`0UX8P^Oxj@Ur#soFi&rl{+sScT3yJCz$+@3fY94)vtazi%k$mbi3$7sTp zYa@`U7tT?-T3Y5IO;)CTfO){lUrAhqI$ZE1MT+zJwd&6bbX8=!L?nuf19Pr>(4MPp zADHup3J&38wVaUmlKIqbN8MZU3`}I;92E~}jida}UDAzH#tae-p?3iiEMUb;lT>xd ziV>s|i*MysBqgI}Woi}D7J(*6WYUUAK6wyu#ohGSm$oie0lKLQ@@D25`*%zWFQNia zYKFu~kV`zxK~-t_6(Z0R7$b0~6}93W0`r|-`_>wPM&zrwCH<1?dF~u;xDorfJEm}Z zkkRL%q!npPttnl>Qqa^Xd3)2&XscjObUH7Ridl^c9s-KqrX4RL2%5NC=m?aNCAoDT z@7-=jEB3)bKH43O0pQg_C#iM>%9Cl3%9VD;@OeNtV=ZZy$DaFku$D>rMN+MYZ3DS; z%hi;yFo&_2+@u~&E!a?D0zrse$Fd^*aiR^{PXYvW$O<|~rQX}!q|xGF_q#D6vZ8%4 zv{*d6s01f{h$AH&o_0}mH;5XH%k2c|(^AdZn!INNcIUS!pn8Ys907;T9)D*j_irQ& z%R*5f5uxqO=~KBDAuKG9t6MMzedQcmE5=$j3bs;cp37__35Tk;&pK43qjxr$BiW`b zB?n9>v)4$`?oDNZAZswBrNk^>mv)1CHO|62i@p=O^CHunuiW*ZO92nD?zqSEa#Dm}mkh!q z+nXgljJy&gvMVns%2-chK8cgV?ock{e|R;j%yp^wZQXO;k~l$x4Hei5A=Gku4d(FQ zKJRd5qCAxB>0hdLU>4eW8sw(EsJPj!49M6ssh1wq z&JL>J%G_;%jvG}1F?$*56~_-Afv;X&frbZ&!tm%f6k?OatQ`YEkSmrh$^cFjrODgx6E+p|F-b@Y5M$glxwtI`x&LqSw0?j;a^1 zbPDgtvVDL7&xp)=>OCtMoKEo z(zUJEIf4;}zT5_y4mc*qs_@TjO}Q`Av9(7J#=Iew@d)nuvOA-YJ!iocB8>{7C79zO zfJV&?;Bd@{*DW8Yo&A`@&}%72phB7KQ}NeFBV#V9VqKqxYznH|G&{xSIKv?wdUw1! zoS~al!hw8>IqwDFtXIevWNaeTylbNXT(wglFkvU(44?=YFxE@x>+2Advz%sX$YQ}u zM)GsffR9ZugbUrQSE&5z7N`mW^;J5vML@*8n4W+vWIGlPa7v!!U2fD`@m2eTR5V3KmF>O2;X-zr z93&nNVDLkh0R*fG3=vXT&e9GjYp^UW(b#39Gmcv*=dA&yjWCLp!S;j80-)LizV>6X z5?3>DqJ~m%Vem4l7g&bJv6{l*X8;;*TT&fDmLc$k=BIJ-0}&S z^J~Ojd}S_oE}TR{Kx2{GZo{z$ATV3^Gw&t5Xd1vlTPK8mlq@~nc>J-q`y(e-IES&bS{WT}S%2X^Ca6wK0t0ZWBr_K%x;8LHrXAwzt8Ly$tE z{TpiU<&D5_s^2W08QQ)YD2b|vizbj%$*w|syO&u;A@JUq4O)W}aAK6wNgK(&kh!~1 zU$loVONT3xXV6pv^tUX~7%hOqTuT|{-SeqCmkSewj6+M%`o|!!2WG<&SS4KF3iPwI z!x2YGk;)W|a4YR0FRCx7730Q(UMdG>j6*q|`Th;Qc^M+}$IHpUA-0(OLE2mJ-PdRs zIq3775dpr?oN9m?5{FJPdj%b)ZuD8R&jbcs_f+DA)rN17kcQZhYV2T@Ylv~X=}ztP zvIyqfa72O&uS+C2s;UXun}YBpdN+kvb2u6{c6X1v!<+|TlP5W{wK2t136mukX%H0D zElDmj$y)7P3eJ$$e`XFu5?ewV>|xPzOZRGfLVR&K?lH@;YObVlUv?Yn!&`?l;gLSS zb~3jy!lyeAm(ZJ5j|3M2CDMa~Agv~&JA)I>Zce!PE=7*3wF(Mm2k7YbNzs*I7`lY+ zdTE}{QStS`0eXVpoT5Lf%KOkvo80jtiG%WWz1O$}bAapx<7`B+`w-zB<^iLrt*qBL|US?U^xWs1#_q%|&x&fgXE)OLf$>|A% zLPQSmIlgg;=77xaJmn9IyaT+|z}dWIix)ZDmf-pDJtAbArKg74EVx-o#6I4!W#u^f z@#i0~AWg~bwr+e|zU4fgGqD1_t%iAdYK7{-m1ai~3S{8w27DTsIOkTZBw)Usgx1?glUk-6=Ajn@H(T%c!jYO*cg>2RhRSV&*Jip^^4F zsS(NwR9^f>g^-n#=CcyPQ`zT^1S4WWD1?hp;@R3L!ePMVdyr*Za)Mx_AU?^e>e}Mb z6})D}cyK~N@po}yd78UKA<)Yu$ZWHnk-4)a^3vT`6~+V&SZ+(JFcKW?UgkGhNqgkW zJDagYGS;MdBSf8Rrl002J~*kj=YVwawMRtq)wax8ayzq)w@ZS8Byo~q&DS{- zHbIzT;f~Yl?^=p&sN~U(2AB|oz!It-rf@rr2}C`FOsbpB zvFM?I5>^YgZhOdjT$W6#?h}<1I`V?9O9~x7wH@|7In=#`W$DKOxr^|)btQ9@5DiuJ zv3QpPA)Oy>GKymeMulY#LPoQ2tb;aw@W#c=%CzV48DcYo^r!n47pO6up)M{UL zi0SkmRl(`~6@YYBY~OI1wG#VT0mzp8bSp5oHN;B`0wCwXL2Del+T@{X(c+@Kcdc(w zI1jOt+KtW)Fxc7}fg^FEjkyic^l))qCV1c39yxn@6Yo>R2qS^yOhlh4I~NyYDtdVD z&6%K_E4EU;*$!&!_Esha+U7K!pQR8k&_o%|TFJm^8PL7dFK8N$aZ0O)cXPZGlbJBP zED`7%25750J#E`0F1j6;H`#6=lN{(z2!iB_<8W`fajf`6!``LgJJoru*itI_ASQ-+ z0-qZ9Ua$qxN7!o~3G~ep+Y+j*Bwqw&w)biZg+V^9BMZ_Ofp@!#)conzmG<}$>`BV$ zBIBaj-(1gK!*=}R5|wtm42E>II1id{$T_BP!Owff!iiy!l_4b^g>X>e6H0gW zo1DUL*Q14IQYy6aumGU2Ap46uQ{hKvwR90S;&LA0V@=l~24}TRv3f|tqV>)M!?o@K zC}3$(YOd}Rx{ivx-oE7vW5o*o9GD(Q>s?ODBxNPz85oRxrx~WGEEV-Q=Yh>*)FaF- zy@0Ok3VwS%3|evmSF9AmDQ#+``_P6nAObcS(|TKOHrD!wm|2zzu1c}VF+z(03&C9t zxho(qJz)k?ntE2Gt!r)zb#iJ4_iD}|v+8`MLMUAz*SsACFM95@c%<8AAmWz7K{F|r zh)C5er^xT;8{7=SkbA>QI-e%Iccra8;Hk~jY8E*T<_HBw=76oH*nKh9} z4zuT-fQOBLYmjre!5~580G=%_k+GS%@0J8Di#vo5(mSm5^2s1;A*?L(z*8L6&N?_VjyW3;S5uF+F-@33f)Biuu+XkBAgW%3I4 zN5xyNP!MDSfus}*09%2v7g9P%juW=qn2npbu|rHW7mFdx9R4zzGs3R2@n;htW|##C zY7XqDB6Wd00#8vh&Q=zbM-q;~IY!@8ItJ`@WDyZ1%{6ZC5# z0LXAvNm;h!``gPEQ4LFw4`!M?LE&)e4usa=65cr=Z(M4CkRPF0sx%wd+`u(KNTh*b zqco6wC%c5*sP1S$E43wIpq=KhrJ~xHCLsVt_XTHSK%qJK;nc~<&Vk}|L^-EZW`%6K z6gM@|P@Sz`(ETZ+WY1O@uvE8Z1SO;tCV_&}a#iy!re)x%EY~E>oB)jzbu$Aqno3$W zPVq~dU9A|_3!??NK(}?*Cg|u^nj%_qOi}a!wG_9C85nY$T0uB3J_UgH37Iga7u~ez zCE^rjn9lYeOqf@Wo3gP(LOvBokg#5crZ3_i#Y6fVR6&K3IB2D-u4O_V+ zLLCmVmeND)mUw6*4%d%Ez3km@(V!@yL7?pnLao$?5-EYZMee*-p|r#=1?F7~;s;I- zSeudg4viRCOj(OLD@fLsa;7eg&k^3Mdc}}DQLwizEu1WL!>I~;!&b)wthF4g4kx~3 z-EtayAzV{fC@EQSeT$6vX2`O)r_o-}?!d$}P4W)XPLH^6>9u985a_%|TWv|)MO^9B z$_dbDq8a(B{R2oD_1wLyTV2X$dhMaX@6sIVAt0m#Cwcfoz&GNwU;j^?=d&IV*&46Yn zqs#I;+JPt*b1OK8xbW89yqhG+$vA;aYHI}(AXRRaD!$dZQOubhIaZHZM2Sg~$TeJf zI#QLC`87}HFqUdm)p+fU&tH^GiUW9;jqxDPoUfrgF<4G2J>u2b;OaL5-10_|y>6Kc z%3)F=5{HFvbT%kWWz~R#gR|8t{VBoS=|ES-vu$E<4TT0)Nzbv!PcmY&&uY+DM^dEP zdBvf2z{R1R?{$diGO9B^cKW*q-D<4|G#DE=Zh%(O$H-(ieuH`6o%t45A2=w-!-rFL zSim&YtyE&9nJ{Sdh8JcTS?Ls?C1m?ibioBJvc&+tR%~`UA<*|%s++R8bn$d5VS}t# zQjtV;&d8kRxk*x_OIciE=CWQ_BBE77*xkvot! zK?<05ag;1GP3@HJX50hE;3TbHaOvbbWrS)yYN*l9S~5eP@wng-zUbVMN+oijkm}uI z?M@DnOmsjFUfD06@rs)17RXoHuGD{nJT_;GUu$M)hOEfX@#LHtJU$-D4uuvK?NKs`Qv5>Zs=4 zAW;g>s@Z|;vqe2^n-o?hdgp7wg*%FfLGtyDw}=QR(arb|1e|S2tUP|9NpT|c3~VqB zCS;G3&~8#%WnqtqCwqo;M(lqgr-ejRSQ9*(2p>*l@G`d0w?4m=$}e#=apT9a(j z^5J4Rk?Ed~W-KuQ9zY`TiwFjy2!W2c*S1yn#gR`KpK{|y8!vKKQ5NB1O-G<>+KT68e1zu@cfRV7%M|SB~;1?5r%OVX1mCjf%Tn@0MoV! zIy}*^SC_8D2TB;ea=MqAJ(J&#$xB%Gm`yJ+emCLt?l^^c@Kt?&c^4%z94)}Cj}N0u zA#viYZxfRzstGArED zW0u6~N#bUl+LRzR2k>XKK?qPvQ9-HWTMk1yuGWec#b8BLEki{NVH~eHt+-Mp(9_-C zho7I;1i`YR67FHS`l8%2_$^L#)hZy#r*Ct%EvlEK0&KY_SvJ@xt}Ag-#C6|6;HGey_#ukK2q}XW2fC4IrYqO2f}kQI&Lszs{C_7$g3FH3M;0-* z8S|EPNXNtOJ26Z>H%(j+;sOo=HV0J9^Q7WZcdH#TSHolDnobB@XX1Di3}@g)$HioZ zmQiRa5Dy4>Dc!`%%O^ZMQ>R_nNzM(G;CJ7U%+(%LV@I;;@Q~VTR6RjM=5(0B>&;i{ zzE?tFm`vm1WJ$HsW8x(QUh+Lm_Lk*>^$!ad7)`_ieA`{IeL2^!HpcXjSvaH4JQiU+ zJipt;l_eyMamL9>neQ?ztNN5gcCm(GJ)^wU#sHySjw&ynS^>laOntnX>q_Td4M&)f z=w)QG4yD6W8@XCK@*OnR*cqhyjD?_vxU_!Hho{*u*dYcfB32~64oB``>~{u!KM%W4 zB&^9aQP!ZQO=cAcg5@)QD~!x&qqiCmA2YU~*qd&3lODbgUaAyVE~CQ^(&GZf zjRN!q*lUawr=f><<2rfy6ptKtq}>jQ7)h)^D07L#;d#7(o4c{@fKrEtkmM&!NftjJ zEoqFch=#^t)Am#okCtbUbb4$n(XRLno#7Tu;je19PJPD?PltZN@_0v^JyoxQpYe_(_ zi8xF#vqYNm`J9TUiI^X(!w%HPIGNg4CIZMo;hWfI>Bxv)yPfH$7bpg~fd+y>ZQ9#M z4pH1J>dV5q&ml>IOVwV9XF5MnaP&QJX^uOWTDTqv8p8OmV#2sRc>p}bwDh$$nE70B zrGPf%D0#CkNLLUPe5{mg1?$8&wt)mxfUyu>!}B!5h9#g6vXeDxMPtPmtdaZjVs#K-cOlE;gW7{!r+psYXeo&&}fyu~n7-97A-geG;JQh=dteX!kwo850_ zWAm(Yce9s3s}PcKA_m(53(G|3)7*4swwgNXeP%$Zm|1TxUqe(QRX}1tu51r#{gnO= z#1>oz4D#vQ4lsoP=Dg7t#&ezK2`-AziRggS8XvP6qk9Bt9gPgfkfJE=4X74va*Qdc zQqwNYk8{w(JDED>^fbyCX&6UnPb&!Pc9Icf22Q;;)>65$fkm2~I#tcry~5*!1A1f9 zxy>)(L6#bc=qA@m!)DWWLif^|oyE)6HJZY~t(DT_59NYf74U{2gxgn@Y8wTuuDcq& zfH~X(=R>H(V4X2}>K=FAd+@5I9uJmteG{WSjZLjg99II7u4b4Dp)Sx|8D5bXKb4w% z29Je?{R?p@oEEqdp=NopEv*xoG2nGqea@@9ivB< zajuw5VaVCLRmm@g#Q;pYVfDuWCPHBVd}nsKi-h{YkmF{_=7LU)NxbF>TEy^;N$PMV zlGEtmoiwB#9jvHc2C43GDgFuYD6)c!A6U4Bkj{oU`F#AR49?sht+cl#D+cvaih^%2 zX?8A9!1nrG`QS`SuOFdh?}pObwzt?IZU;0W*9&8YEsbC$MF@#mVZvmnAbDza4Lp3f za8FPP=A2x_wSat8C0!|L_hO}0oZ?hLkab%4vqNj!QD=~loTQ=&cr0#7sH6i!wM!vF z@Y=>bWq6Iz=rFY6>V{3>1|qm7S(f|3!2zY8Z8)2y_ap4flS5*AGN^Yd-h4iU+ZD-_ z+puyH;kLXD#qhhh=`B}yMX{5J;1a@o#F(ukDwdpm!&gn>mB5Oz!Z)FWYkF?P=yZ^z z-eNMF-8!riCm%4Kb%uuc_1U(_NxM#<>j)G$i2=fS-&@fsZsXf?s7q;Uu`o>=l8PXdsCA#^&47tiZz(=ybFvwuPj@B}$Bs!j!kg;1RGL;<)PX zL(!Q8h?r)I-Hx{hPT}NY`K44aH=n5B*A|i$2Y3~J$Mk6|wbi=G= z*k`-PL|}3^BF{3_4p(CM-LVK$EZl2YSi)3bDqc*0;;rfSGRx!Q_IEJ$^7Jng{7P*Y zSl$t+^gJdacUNeg<5Ge#pSbZ@Ms75zQ&U3w$;(ZFB)e3Qsa^*SQ|Ya#7N}DQa&ol> z>{55)g{<-LNObiLO5N&^VT+5*vcj331S_DUKT*)hwhOVq?1=h`Zi3&49ndq<**@|I zDT+R-uGw#Bzqj1bd?6w-q>Q$Ep#Ah8VB3NlM72gLE3t-cLlOnF;{Y za|*m=sEl-G77KVVlj=NE2(*37>ncF2tFpAea2gL^yK|Lzq$Cm(hf&`L?xOl4>ZLlZ zR5{mh=gu6ZLR&HGVmi@uSP?LfK!Q2)<~31qkE7M)EKaad5XQq}Hh-0~)p-#$ja80l z<`~W_gfSu|j#MGmc0%Svh(o)TL5P)e?KuwNn^p#x85gwUx{K=XE|$QVpn@v|gB&~= zU35~VIX?5oTqsrzp_O|x(*>E z>(+7XHKYt$2SU|Nvom8X(&fFy1Nn@7AmGd_qF=oaX<+rXt5fQ36>O;$Me*)Q?lH6n%!^3lFX3GcS-g(>0L&0tG90tNJ zP&*-hXL9uRhAIj2PNr@;U<(^eUO`$B^VqwbETCX9vKAbslOD&VvI%tz*qt6hemW8m z@E8lfR6giNUSvjWAg))+Yjewr z1z;dnKosEOhD>rfY|jYCXpr%cuD2Rc?8@#+vHs6xpzOIg*HHWd%I}oyN_|SeG$6p* zay_>17xj7TRM|UWlo5^ka28~#BQ~o;JTKKgd260$uxZ5?QvnVrEWZI&{i6gu@VJ)D zsnX7Nh(DCFE$pZcgGhYia2YR&XemIu=6FTynhpp?kb_SY0_NJ)md+ofZeS6!wP4q) zU=|u@yh*o!y@?*Jq_~IS!7ajPTd$zWutZdeTb(2&7z{@y^^W^>N*kRv{255(+oIy0 zjtA{>9Vn~G@tBU0AWk^0-EbcQIb(~p^bEcRVd~aAf!w8dr4AX9&0yTBc3a4IjxjYu z0q$EN%!!(uJYhP}px`t-9}0((!ZZL0wAZ$zF@A_}S5^0Yt4Hx1y>qQmaTrU2%p@7^ zax>EeUuVqnRlcCxtVjcKUV@7xj&?N=*&!gJJB7`|!Eq0z!Uh+$e}1u!1VIwUa5 z2#Sg>g1Bcs_aheC+{Sch)XApKa^oLjU|3}P&|!MUY9SPOb%4Slr+P4@?>ARRXP-Ln zDCzD`wB)w~AJCm#oF~%~I4Qi#=tT*?5MOS-@n6ik7A*rmIP}wvvKkRe1PB>;i&XY! zhly|$b|&i}lDxLhu{jb@1&rcN#I#xi-(n6T<50eo#f+vPWk{XCfv!I(sA*5Gbs?D6 z1jL*?jm{K}B2G75?y9cu8cC>bP`0b1u8grNKW8<7<^`9an`gsfMaq$@=}rxo$dUy; z*G;}56$)Qna6XxbJQO~tFxv=HkT`)DsIWSXyYx;iEm=yjyYkC7IWedqK-ufVIGag> z>>H9Gbf>+vT|O3AHWq=AfLTZnGaB%Cc-uj_;|Tnk7#cht2nlM{bkfkpux~4mVE4lGX=*KiH;#Ed!`&?s4N6`$O21JLtg(zHn#Bd?0KjVcr8F``^{s8S_h zp#;S)fkt2sw({f;CAGg}@aQ%apQCaYFhxwwisJc>y+Hz?@MztA)KV#|1?{+@JcWKV zR%Ran^cH~?Lx}~;X)iRIEpsOw*=Vffc-dq+_QS!?!egV}K#3j}LzYQ}0S392tpcl{ z!E;;Nn50u}74bv{^}BQ#He1JPSr(nByM+7b*=S9&@Hkts=LQbADtte3)~wRL?QV2W zMUpwhyZJO*c&K11;%@?Z-EI3YE$5hI#P(2z+n|+G1rJ?~WV_XCV%tNH(8_ytmniDO z&YFWd-Z%-CS6mY}GSX)b){U_PL7h}=G$2{*a8(~sFzL<{sZy!tcIFZRAS0bNQ&8IbZr&=P^fBz(Bt26hjaoRGa6YoZ*xEiLC1XaZ7eFRU_d9EY zu2KTjl(NSLbr|CYfm`26f~mhB=;&T8)l%?1gR48pL$PL+e08=9V%r6mKjW=Arvh;fIyoNj6` zYJf4Fr!#cn80adI4isma=0Qnl?v&`e(il?bx|FF(9yI1VJ_L%pKZu7fpt;EwyHTuU4n)S(z`UPDwY(Q7 zKOC6o+v}%2o8Gh??v6{Blmp^W0kf!Onwf;4LVBH0VPZ_%8DQ6X@46hTsKz=_`=XBL zvlj0@yFz72<7sWS?j~#4_1(4cpGaG4_MfY$E&MeC!N|&HnKnf<5pac4^J9$ag26Yi ziU&;{RKXTgHgXQUf%r&A?5@J2JDOnkO0soJWzaczK>YV;cNJLTO`c^oa~?=@PT1Ls zdjPk?7MvA{&9LD(fJsYU_NH!}@uWKR_2$&cm_zoOW|DgV(54k3$wDJO8b6y1`YjYh zoRlXhiqFSdePJR=lc8)u!r3vYqaT&E=Y&B|d)T_Z(?0mgblQ&^VsdA$qK^`UuBDr& zkGI#4ZGUihr3n1=b_6$iuNFHDnFdrZlVKDyVM1MpLqUu}2y0(fOuedCi*Wi4`*kZD z@Of&ryeU#?aR^clz$(C#fso>ZdL^JCvMN}|3%olNGu$DeS~b+&vBY#$^#Xk@6Ta|o znDNjYKVw z?C~?c9tP_{h~Az=j@6?M8ENw&hM#P_GH!KA#^f={wMODRUM+Owleh>xfNOxJdKNrw z_gv6Vu@>6ONEtEZW#5wVv?x70TY~rl)fZG3ttU}H2FXHFkH(pCWYi#+4;gp7V2!yS zHd^2nt}fk-a@}Ofe77~I#Ie!01diNEApQ?~CO5Q#ZTh&_2@0nv=h;9)D&mVOLaIB9 zRN|hU+<4>T-@?r<{I40@v(89+(R8EhI)hOUYfpMp!OVo+Q>TFL&_GN~gqCRE7h*6w zv_(i3?bFs)hWSFy1hP?nb^2Fv89PpPQ+7a(3KiUD#|5~bP%yWB-WtZM$Iy&k;5yf0 zN=0IXmX4*d0ZD0ERa~Zu62{mXIdBz{K*xLx8)@1m7~AV1PeoMo6k;B?RV{Ik4c;zY zD`6vfZrBjWe8o1#g)tEo(VA)uaY#@?2t#PzTcP_%8E|{6%teXgQ4l5EQ$rhnF*epj zmLYM2z75{-^MQ+m*j`c^rxmVG(YDTX2X1zQ|Y+fe0I(Ten^)9eMsnCj+kc|OZ>D#Yb3>W2+gk~zbR5j>!5nMV`XMhBwl zFQ}f)Jx;J} zH01&AAZR6AS1K@oAvr+MVm^ZMC5F_+i>J5?B_? z3^SpE>yE9H#CE0+u0UQH_O&iy)iEWEHfJmVK|sF0ERZ-+S* zOhs7qge7n`=RpKV<{E*dGaP8_=|qE@T@3Vk{gio=(Rjh9Kp}7d!)PE^_VPzM!U(Bb zt33)IfxUqTjZvfovSVbEBLECE9MnoI9*)Z!(5anv7X&XRPPMTU$FWjZ-J1gB(0dte zPUlXfA`-h)bM8gpU%IT*nqJTi81p<%!yKF?d1dAaQ*6(0mVG!VtkrYJsJSX2wuR4Z*2@GeqMB@v87hjE-So9fv((^967u)8?1Dd>WI|0v; zM`Ceds-i5#s6cs?%PIjCh?o(X#L2XTnuV`>uCV+FH31|Na9SaL5(?xg)M0>*A1Tpl zdJn2+&2=uF<~i43Zy(b~&+e2Zl5SEoykhY2KalMBEh(A9NYYrD>TP@pVDZax`8jSD z$yC>8)^4?^c<{{NjB!+rjxYlZ$9;KX3)P0cX-<%1c8y7}VPFOc zB2r6n2M`$>3<+D74BDZ##>%W+Nj3;LR&n!s4?TTSP&=JGjs@%$;#In8#z=T;ku{gy zJugdT?+1}2(TwJ1CYQCOL`~7c*_zd}4^%$>a7bOhgH zWw!E8(&V>-B&!kXMqgMY4f$KwhCQpi!YIpT1j>XydUrRX(R|+HRvsIj(!5OO9!%wG zhI$xu4M`FMA;dk845wIV$mfqUTT;mK?)=Y4$J>!^>DyuJxMnr0_o|?)h*kPWRZpuq zRcIC>z@FRFM)b8kR6MwJz)_zUt`Y_&D(=v$3=4HRE`b@<@xn0aS zFJrBauy|5cg35=k=E84-HJTTL$_EWWnm2w95uS31Dp?h?@UorVy9p!|5eA+n!uly= z!#yk5SeD_L&J5>1Q5#|spVLp`HCGO{T%{0ci;a|I*rKp);5+jrZ+yye+0{0QOCVI_ zfUfD!Wkfa|zDD-o0;F?^kxN3hL-0+|L)r_->`D6k&AFP^%|+0lh$QUcjr1@Fyi^3y96v*2v4mR?JJTXjs87X?4;*xGStRyHrtMg=gfG&d< z-O9+w)OBjeHnG*F=;+epUvy^RHba>y82BGkd$F)YZud?JjdxX;}7#G1r z3SLkYKPC`m+W@>aXE^Z4R+nE6DkwfrszJD*&;$n%hXk9kB30lpTs$X@o$7#VqCu%u%Ufg0F*)5R!an zs5V$^K9x`?R1lCG=OyoM=VYop+wJS?1xX7`)#0ENMcmnJ6q_>EqtXg_-FlcJ?hp6DbKtDDZ|bSq?a!bGK_$i2%4K&WjdTHgun)cJNeNYac& zI}DBt4#Rn3R7`XhLK5QIOZS{7vA%7W9^?!$ZFj(-obAQ4*EnP0z~RG>hm@!T7l3eyo8cD5M2-j> zH@ssK^-LLKTjv2Sd^Fsv@y6!8LX`ybdRcwi#ZHQ=;63pMkPOBJWJPY@i5!%1c_cm3 z0$~#fheG2adJiMhCz6c-K%%{eGfb0w*OEncl%ne4+f~40WD2gn2VU%dH*U)Obg?P( zy963(vVzbqZuRAbXFYaKpk{G52g{b1VLbAOh!pW`=}}GM6q}a0yK^{+HwZL*3@%TZ zNu^fDj8zx%T)L~ig&^?;;;7G~RuBX<%sAoD7`jPIUKU}DePgwrTfAXd$Cd&TSW*KZ zHcm;_ECsdA&1u}?w+POH=2&gxo36V)=(Y&LxqISfGqA9W5aVZrqX$HR%ybd*^g`!Fm3kB{DKtoVw#%15X)DH_bSRG+K1s)1%UkC52W-T* zAfLASPA!GT7Kd}oYTowrnOm;Eg1e9%1}%1gY5ub|4pH|iSQccnH5Sn*+&Vi2_s;bB z&2oG|QnMLvyaOIBwzgD<&c6>Yfe^+6ds02pKG8!gIT*}6 zc}Dq{H3fT55bmkUVHAcRPJ;vrVuAMHB4h{_Da*4HJk`pI`QR6m!7+p`knb)rP}8cB ze01_FZ@o|uoGAX$gsu;b%dWjJodQJshu*i;?^}0m`uYb6s{w*T9Oe6{@n}u0*eI)T z;biwUtF30AgSW}01a?ciHjd%v)*1vrtz;;O4e>VzZPkDq3n;7FZP^YVM?J|G8Vwy= z-o-&C-NX%ui5i5K%dNEmP_^{A1Bvk=rfFvT*eM_`89G3Bc-Pfs#Xm@2OLK}L2J zAlyOEF<#{(AZ@+OJ;AYE)|E;`BCIRk^V~>j77+{xKnvF#^P1gAr#3K5ydmyd+K%($ zp&oh;b=jJ+giy6rm{ta`N<5-0;fF3}bh8nC3}TUCDhi|24QFT))bd97vt_d*)tUY1|fw6`R9y0~p!Y%t~~f*TDTo#}A!^ucCa`Na_JwRHAm&Q<6x@UwL| z%8LOe5R~^lTx#5jFqv{TvD9D`2(0CDsQp>>w6AFcVo)=Abg&ZEqYeiwS_M(OadMp* zgdD6PQ==*hAb>U+-P95+msMaY))S^8rr^DW=Sf=vE5+#%IRE1awFA4z2 za{(G2RSUSv@g0j3qP#G!D3PMc=rG~$6{>)oK4%|8KZN+9J&-F)1{+yEFf&+~H-T}G zPP2@YfyndPlMW*18%tzdfmTc4Xst?>%|a`g!U*|o8Tgn+-aIrrA%?ahcbt78tQnYk z3u-lFMc&8f&o%jIQ^I;taT1(}_@OS&qkWS(#m{EjPWRxrO13K;5*z_Xe5~F340{g2 zIJD@viQU+kIBOLRmC%nVdG9!jbA^wy2M!~oi#K*t1Fb$+4QeI}?k3%~WFsss%7-=_ z>hG{^p*KkNNd8Y3A_O3f5>e`1PPS(pg{Cd!FpzdAF5#KmI*%Q<-BBdyj=4n7>U(ua zJ1BblBnCvleIg1kKucmi3dz1$ByDiV6pC1aW4(PWOUT-4M>^J`u_GmC#8-{S*sNYd zY#d@^N6VAk9Lo-vC_~T_Es3h<1=Ry*gTAz7NB7BGoc|-S7Wou$TGR{5WU#3A9I1i| zBs`qCq+I(s&-p0K^CY|8{1YB1ZAw?S<^O)|C|eU$JgIbc}OW?k`+42*JLAgy?6ZbhJ? z*$->UR8_aM+SbSI?gWQ!lmP~g>CPUgGbW+JJJ=1`AEqiAn1&+rxH{tz)7-ZzQ1>K8(VEiTRU)+rU)4DOe|Qq*RJzf z6V_2pMb5x&Vp0}-z2xrQ<)P=0=8h#zNZY%qjXRdLEy<1f@iQ#?921zBqOXy+>YlKj zKUmb%anHD_s5n7LV>uIy^djemi6XsWDRnb9y7H1Hc^OL9f|ETxscQoSJ?-uvoGL-p zbIBsr5-_b1mxW*Do|z!EmMmx~z}T}aW%9PAf+M`_luiQH+$+&{`1Uv6?u(P9D;^&`UE z#Jvjky3mzpiHNxXG2T7U(l5*Og`+2Z^fq559@r`!%meYiRC9}L{<$<@#e1Q3hkvd$ z3cN&jiPB$So4`g<6e8JfP@zfR4alg(46~~Xj26;a-Kg(R=R+Zur-n)%Kk(I}!`7$@O_o&o2RX$gr$Z{bhSj z1Vz8m67|Zv2EAn6&o7KL@wL7L|;1%YMY(R17#`QUGE7*1t!V5 z)1PSA=CKu1(9&6<4De~(cg7a35pP?G1b6Ey6!M|b-xxtJc?Xlg-j#u}l^H6##{1kT zcCCd-Jz&JiG<9qj8Bdbb^)h$ot;VFxfRJ;?+w&QA_96L0Y;jnIz=`54O}hPw4F`i@ zw;YeMGYU0iOIHo-U4=1s2qiuqmO*)kJLGr*_aZZD02ESc^n07p%FO(B(jIO-Z&u7- zuW0lzMwV4>4b2JC>m@=9`xxuBdnv^8E&_X<#APt9K$Y<~mhwG)5cZ%rW7Oz}N&^@+ zzQ2RL%ekN@?(@ftm84kEBN>s)W1jfH__I*Ec|{=MqTzsaT?D38)M9g%=_LAlH%%kb z@0o)iApOEL#{&Mam>UFiPOZ@OF;drH%N|#mV>)UlEgGtN*f-^&5FRNT3bIv6BqN=z z6DU&)-jkJNNNiOP7~=+#Gj?^byzKhm9c@_lt+>0DL;5;j#DHjMRhsL6CEg3DSTT>g=Z6>!(87j#?!>rbQP%`+PeK5`SGeJ zsCb)pf)O`C+V6_O<@Q2a#=uS^c=lr7oKR@@Fqg4EXS9}UNggbEJyHEaB|>`gs)w{VzHQ2BsObZ+3jt{ z^j|Nw2oy?|Lc)Vdv;{aWOX%8|-qX@`vdr&Vc!B}GVNJR(H&v@R>L53WFfm7jlo~T< za=lT$8;cvXk-FLxI6En$Jk&XmN$ORS@RUXZfg z-WV!Empqk?(ugjr>uA2}5eC^LXQpnq#Ume15^G+W#i7vT_Bh)@P8u1($a^69iX_Wu z=zVPR&IypBIn7D3JOPL%EW_7AD*JJqo}GBhF8z=NcPkfzU9Kiw%4Ar`t7O1fz#ZmR z#y-hQP(O&q(7?BK?_>mx%-n%1(wVy+-vU@o^|p4U_8WZ_o023lH;mR za0`0VLK1#Y4IU9Pi>u>OMB|D$yK7FPbpp9{IOul5Z7I3FqwI=DSynMwj(2y#uLVKL zQMK_WHuj*x#CUUV;j(2%oVRC7xt)*fEYs;3%;6dP&>fkWF)LBQI3e#jU0KoUx|97= zQ)AWX&xd5yxj~7@6^nT+yt%HyYDx`}8*8Oi7CH!55Dl=qShJWiHSD{#!CS|?p?N`i zawkf5b9EvOEWwpM2-E34Ss;%RpOYDxC$@_cYJ6Lz;+KBK*%cQfA6sg_D9$$pz zM5+ciQJ7dWCO+PnR!kR!d!u)9K^ugJTq3N<@iHey-o)zDk0KT0wysB}5o+0HR-8o?GGQO=p2qYfuhXr*Q0K(8{me(;|VYeoUt% zmgt&&@3?R@Z+RoZ4=c5?EUcZ6aON0gUO%$Bn$HAr!iN3_tZoLrH@!#O^Vr@jxJE9E zYzYG3uN9D?CHBkH;^0?u8^QxoP4K>D0oA-8UG*lA;W*ldQkwPNHC0ym3{;GOdU8Ww z(5)XU;K`PVzf$;aDY(zy1kfZB#?+sHHoKD#(JtnRTPvz?==#=iDs{b@HD_Qk_icp; zi_|DC5LX1F%n9B_EO9_+#JbTS-UNOoEaM02))!r3*BHn=EgRsMk74v@ z%cL9WMVByD2#TN+kyi-s3ghho+Co|=RJ||G!G6jrqu?kg@G=7P&OA)@qR|GXTw0PC z*^Nv=tqbV{Xuo57z92;nmP$9}s%!#LmpkEjp0}#< ziL4vfHAbg+@T#@m-kbqM<494h3ZJ90l)(St9Gt_9^4EUqqaoZ=T$J zx)ir5%Fyi~V_~K1DsJ*KD-NJrMU3UR)FQVXo@5cF(BYT@tA>i%`+?r6MXmhm6jB7F zUO-O|LuYj2E;jI+i!Q@dqp65+-8Ax91VaS-6sex|JJ~v-K~Qv>I2_dP9UOWIXuxWr z)Cz{StT&3j(UD`#+d~zcnhJ1n4H*y{wlaj3X{FcSN6A&%cI00$M1!yzn?;y(=H(1_ z1EF+`1fVITI|X4ju^)JkVA!&Ig`dc?#G8++>zUlwE0vCXMMD=&(G4n8Unwvr2VZoH z)@haH*Ri5b3_zr)4Q%GQyz!yz4H_!>>1_@#2@2qFB0%<}dE{wJ0IrYCkYrKkFuTIi zRhExwXPBe0L~vRg&CXaVu&#dq9UUSU;{|(E=YL5~4)fO&5(W%rJ%&6`V-m*_W%%n0 zg|#z|T_`T#mvb_yeC@Y{ni_l#8O7!Ka*}JoDzS0#pAV$HDRfrcmGNw}I@kr5=^L-{ zU~$<9PDV*YDYJ3v9eFT3fUl*FmTAwH*KxcUCjPgJOGkF;5U8TZ9}Z+zXBohQO8UuE z77fMBIYJlOnMAt4kJtCGk}skDD!@5W&gTzy;y4KRv3<68-w^p;WHb%G65z2 zKM)=pA3*0_!qteFES-UlwKTb!bL9~a0jiqevaz8(P;#{eKg~8e2CO?X?j@NYYAzCq z1Pv-*O9H1J^FS2(Ogsk&Wi7kG%{bUE9MDh2KKAE-j#5Y4qre>@ECPi z3Id>yqhGhM(9Ae(3nX$Dz<6U}>@=L_p!l8Ri~&M;)bm@1=5Z^#sij)O;cmxGtSo?q ziY2xkN-PT)6u4&P4(aKsre8Use~6FmUQ0StSi zk4+5_dTL&VkZ!MDzN}*G2^=%Ucd?rew-pT9#YgHH$|7%udfmc%7<0~zAYKkR_k($2 zZ61SKWFmCwZ;hc@=JaLPSDx4pt0(G7>@nUoYzo`;A?#M4JHjT481v$7W6Y;chhd$A zqWhgJkz>7b7vu5itnX;ubnwB3q3QXFhq6Vd3hMai%gqDBi+F<@i+)0A&=oO4j>w}o zlLc^U@of?4N`pm1^Jd#eoT8hGoK{kbb=+-SldDFHL4dl1FOs1^of?NLJz>05(~N7) zS_cSjzP+!i+5lzWw1u-ljZ3QzOj4F!0Wz++JaNl=n8mTI5y8fb%kO)xPEt=0j`sm# zJ+%HOF!nn>;|&Y39U(0U34`0EJME?QWZ2TL&5u`0;Tc#1ArAGqm^|E=_0k}rTM2Jh z6K4vFd)Tom4Y%E4b~wpJf>%*Uj#8%NP^5Nz+Da7;NbCc4X(}gV$JdNj<8EXbd29AH z$UiiMVFli8l*lQQSTxG!5)BfjJCpK;lhX>S9@hpwgFYHjs_46g=NIGhpPIxD^B$R-*b_fr`*fv}tT z6E+(Iu-=*XjFE$MVRDkv4XEu;39{{{Q$i4QP{P>)nJiUcod*FvG6lTFVlmzAlSnH& z%ymUZDu?*A?dSu*RVkY*G-3$6kcGk*Uc0>I7t0xFe)Zbf5LAGmdL>E_xwni%J1Z0B z1q5hB_FPKIaZGlWGB*A_jFl_wjn^~t@XR#UX%&!< zLL0y(29#TqV6xC-Q+c@0)+~#*ns|BT-znjRhIV=`3k`wdMS$(BRl}6(x{Mj&0!n!= ziee1ry<2@ub?cLucDYt=c3}$!z)A&4zEM!;71Ow-e}qg37Er5lO}Pbga@8v^XXuW2 zP1)@6la+MqxMl^EUPej*y(HY~4yI^dRIxKNyDdZQJQUv`cZ*0IP4>m=!N<)+ed;dO z`PZrh8$>GaFc45X)60ZcxX4u%6Yn^Y+6DsIS71g#z{n9@BArbH(=^j@9Qbbea&+W@ z`;(oZ-8Lr6r9lMul}Vb$*MaLzJGpY1uAxM*J ztqwPoSjy@RqfIPQ-dMrniXUMGbt`h9rd!&d)C=8=x}#`xU`aI_R9=TvlRoB) zG@xP1@+uRH{#S8-}3R4rH*zMm!4k~}6`Q97TL zhpugndpR29jif?wGY@AeCF_z^RFb20pxP>JPA3I{i+c;u+(OoSa>d{X%ry!Z6kP5re z-CJhU=oLxZyUX+!D3J1WjVMftksG11ki3(pX>D^hj9jKbbX9rL2+Na`1I@T6MX&;k zI;$%;bZ;6r!NjvrT;)U=e{JQ{)P~AaL}N-hLn?iWoJB73kU4eg&|C;;nt|nXxNY1@ zaPX$6Q)60!h(3~Z+to%391qx$eX@(!$&8jS9%atga~LD9Dvc;RPmW7CqvN{90=J)X zzIVH&@DpVL?#R5?viLGsjRH5O+SRK>gT6GAg=ChE9PH##Vd*1FBDxTOfk@Kl82p9pyqZ-`bX)QL$d_L1hRef^z-SFZu z$H|{A+4nMCfqO%#F{5Kbqv68Xb_X{@v`#UtRXILYE^z82_mdn%Uq^1Id6I-Reb5sL z!rB_%$?~f^lv4nPC8xzlG#HyRQijqV#&*&rbblbgT8mI}n)uf5A{SQ;r|34LRl&6& z&U^Mo$y>}SO6euMGHE*fi|>~yR~#V zN7bQ%ER%N5R1>K^GTvt&Dk-TM_139pk+wv7GGsBdt#>HZ9QH8!FsXoUYksrd9`52~UU~ zHa#OKsnAXb!4x1pfKZ;3pHv4=4Njmh1at{gPWibz5(RzTGT8<~T4I5(z3 z8w{MOq;l%&MvW)GFZ>p)$EAI=VdMC7 z1{in9xm-6X&ec-*1U9B%E>tx1aL0>O!9L^J!b389!?)?^mT=?L(*nz=fvY^qG3 z51Lvu2q1uODeX#tw@lSnNm-`&+NOmzvk<~l+?X0 zx%mN~1_@jW9q5!u#uGM-AJ4SKWIg%fq8nsc(lZ3JbnqN^nP1 zWxOEjqa>3rCE<=;o{!b?;S{|VeEy^ybp&u%6EQa`G*MHqvFEN1c>-t&@zSr{Rq6nu zE)n0x{+Kp!u$~K!rDJku4!Y@5<7FGZy^9YF6e3HT_@D>w2%T`C)25fpx=>Qn71WOs zcu6N4;o{O zkh7iF0_QX%51!4u`)QcOvh-6iIL0QqBw z*9_?`C3$4vtB%QLbPI#aOzrF*53aM33BqY@mnzpWQ`QHas%d7;ePNbP>QWHpBRUI3 zz_Al4%tVqH%Ec9Kafhihfn*@zzHJ|$uSmlC84ji`ZP(r8l3xtvO>j)Oz;fjzrdgpt zkzPBr0x*|CfTG-@)2CRiDGP+R$k=iGo(rDV2LewA^G$kw?JR4h;O z>O=z@e2bul9m`eia3P~46x3&2aK31bABn{MyUs;p5ie?m(`PuoQ!v3pMfqGmGn64( zoT?Q|YNcgNTk&I6Mw$zuydlooa^3y5ki_}9z#$k@R^3{aS^&Y0^@w5*0(E-%PGQ30 zQAozez6X~f=j(!K?JW)2`fa>3iH&QyOX4iL81ilenU%Q5YKxdJI*G#)V9iJ%zK()v z-iTHpg5w3Zx~Tad7H`3@;0*UGB1e779MF9onA&{puBmXi6js|T=bnko4zOF7&ssUD zFw*!O&B0yonsLD7qup+3?Nb<)9UQqx@}Tt^Dsrov8SYDXG_>Ft@-6|ANV&;eDtoC3 zkM5EHeI&`U5Kon?gWFRS>UJEp>c*e22ja=?5aAghsVP~-u|iTP`ze%AL4*M%%qQEb zIhY=h!*Hq-?h6~5=fdK$77J=Pp?W$6ha>?oSlt}!A3!kh8ib|X4^|+aWiZit#x^i+ zob=xmd?@yK=6s+}CG;3bPe_Xaa<{0^T?IYdubh)gAWltPTH;HWzbFU~Rx4rDUU{UY zpD$AVyHKT(@<%c762=+}u}g$4T`YB<3}m$NR9;-s5pTJ~LlQXT-&7Xs_NNHo9>n_gVIuJf75DbV&(<+qRzhH`_-VOvU; z%GT$zmFqWPQ_J=^4wl;lVf4P@u3(u5i|roVrcZeIDS?2jX8}bmSyRO*yo$XSMzOcq zjO7&OKtl6R<{+7js6ys9K^%AOj7wvSyu9c+ymYNUL(DZ4Z;~$%G6Bk*CDj&GDi1u@ zBO8!l-Ec6OX5riCttn@TDWog%&<-9D;K$Ax*j#~Ws%em?#0cUQB+~_1q8g!vo7xB= z6PWv;o{*PE)-bh93EcY{q>l89#-uKkU?WtWG2)Lloa0^Syn-#I&hcc!De2%VME3zv zCxxEQ*pNYtI~?LE;uCP7jG*Y3(dQUA)Im#o?t|7k99fbL8Zxgqf+PF)F$XMm8oZ+EmUcyL)X219a%S`Y{b_4 zRz)FBZ=9U39xkT!n551kV^&~ha?d;cNuJeen*oah1Fx(mI}x7wJ#LnxOadBEK@`Iou!^Meddb~O%a1q0bWJsK+GX{PH%nJ$Yf zkm&35oSzziv$v)ajW014+@z>*n79>&g_`GM zV|okCPCE2TV~@4MsYNBiAE@2)t?IgKtC?90_<- zTELt`oGY_Zw|qT>qW$3w*KQ0t=WM31NK9WQKJcq-fm)8&0w{S&a16D@m~khxFs9R~ zQC^}v@1xcpdKH%3Z?}u>OU=VTiIW7*@iumJf%9NO>I{#RUkE2WWKxjXgtzgLNzCV_ zNS)~~y~^pN50actku1*vz(RWDATtEmTpr54Hi{gUEy(9bB)L0LC_?cJQ+qVhYJfv@ z8vt-u1tgUzd0Hc<8DtS0RoZyC3p0OY5k6ls*R5UjP+U8HUjA12k#8tj zBAT}E&?0j^-icE9lAX$jaG$Gei^8^44nIUJvgjHe_YM@#pQJhOfV$VO;Ue<}tVfXq zhlNaFPzgct&^xb7Ye(jD9X;{{QSD8arz+k%rTl^>y@GP*a}n++@qm=&O$!F&AmY#& zRTzhmYI)y>P=ghs(i0}E_Plq*WS*eqJ%Z=xf}%SQWigv{U8~|p%_XR3)o>QRNt=*&r^_H|*jChc;vQ!OKJ?2XFmNPRk>t3>V+41n zTs^`Ai-C6&1j-oM`McKx0n?b~>houK5LdBfQcP zA2nnwlN5KY5TQv{r7OKjAliicx`!Cfl$!Jw!R?SN0ktF)w6mO}V40unAc6^khy z7QbfOeOb%TV!BX{m_?im^vO_{)TQ^Gn#pjmf|=254s|+#TX*F6aFD=8csM~u!_c(9 zZ3Hrf%4D`Ssl)WD1Yecos8gJ1;6GO3w;VnQ*~ebKx;NJYvfI+cDZ|zBBpCwA&>FBV z211BW&0&14&9qC~P=&sPMr`ny+I`h5RN`WtE9HPS$8}jDDln2Ue2iYQ_X&#f@N8VL ziEj^2aGjB(k)MS_62>5-y9hleD!m+V9`sLWTMLPdS-v*ezN7VDHZ&=4n8W$!*(fdEXCcQdfguj z-YaS(p+6=p&@WHZ$#M}y1}s`AfO5&j+1*D6^eNCpO|vh1(mWf88c*a^BQbl`updPi zB|%Y;V1kd{NduKG!ooCeLHRYp--ppSjW;HsqtPM>Vr~`LVBE}adA6NVVkjO*BXWG^ zf!1Kr>CH>5DKH=ANQd^Uim7~WrJJmz=e9GWIG`p|nQ(w5~kWIPBzqQWjl$ znPUlW2JrDX#?~^RWWnTV4kXvk3iL;ktJnrZMh?@e2JgLMNbogWlJfVX1Egd!!Dvhl(EDQ`q^Me7pVM8upOz_|(8(!vn zPjs7e(95)nUT_r6H`|(w(&JiSeeDXtp}2TxFnw@6J;-bM^;Jjmx_3KE*S*Qc(^$S z-!PMfT1tBPJQd1b5hXda@#~IZ9f>H=Ubm~wgCYlHpw*?r!z*Ktd9g;y7cxY1KaPh#7P+sR|=G(ZQwRkZ2`sd zh@H;T>e*u?JH6=(NNl|ebxKq)Tg%~jeGo(kbmc}FL-CFKp>czEECr0ralrP$#dfeF z+lVFRE8S8GJHoLnMj|63H)n4?Fsp&FSkkU6QA)5(AQ66|QtW(oDmfy@iyPUR??H|^NzB*%jC817tHp$ez8SYUD0?p`g< zIR>>gt~0!#J39EJCm865tOn~7nGdAAE1;-qp2(MQg=}B{!u(k<|GN}%+?si8?gevhw2gbS4QE}H@6)a%lD2hD?h$xSKq}#->E5UjQ zSqBW75}+Mtj(1>^`FW)0-F^*V6x*nv{NQmzXe9N*GE%in79G8)p!qL}?=|$SarrzG zB=4LV3?2fc5iv-NQM&LM115r5^y3vq&4tHmq%F^kojmeDTd6?us~X2w)UN|F6>xKd z+H4cZJ}C^cQ_K_;pqMWPq8}sP4`OPFGE3oS((Y%p6-${G%+Ax?lU-JTritPt)~&9D z9w2t2ak?d0WK;_S@Ue$FWzG+^yltO;Jl*-p_*zMXQ!mv+3KI2J?RAnBscbRcrruFf z6+Ol_fv>5n0;wAk$8rknJ?w1P&|TPG#v_CGY(&^__Hx?}=#%&FLpBF#bX{1&R zh1gM|&C-zP!YPxnV^_VQISQFWLvef;a~|H#s@57`CAK?*$ms-`!Ub0A9-VpFBx78P z__FgJe1}iK!Gx=;NqaPKAi4~r+>|xOS+(NT@{Y2&&?`)?-!h_r`=hzB%Qw4d@)IW> zgfI|dJQ~<<0{?q}-MmoqYqMeLp7CvhQz9bc5`l*@Sc2%Fanb11i|VEcBV+K|i0F|| z!BV+^k&-v+4@ZS+aHf+1Zx~(fUN3ZFj!;BcUdl`v!y<7!z{$%BNaNzGhbOGKHXA+l z;$u+}uz|@rh})W`%iB*2UKR;tz~c~RL=8HSun@(IHiMlwi7(*SNfZ z3)a?boQ{@YYZkvbJ^cVVjvf{4&CbgfADKI;<;BF!Vnl}ae9VTYg{pY&e!2F0^FyBOFqNdnO=6P^!3kf0VaNA$^FcGXUG;;V~VOCXNRjz+S8ao$`;=(@yFNWDTZb16fKwjaOo@$2|5g zPU1K+u3U=haIEp=-LuolQq zBzgTA3*OEPV+n3Z-%&oIw68t^!DXUS;4$M8i8~P?#=GsQ@%RRcW>hl%TwW_2l-|_z zX0;4iy4h}^{4Yvd1k0TK32X-RS69L?6b%n^cf~uM4d;D4&pt7nbHeB-{kCo zn80*ro=i}TDJ5s|_Cnw(8n71<`qe;DChkgTkcZ>1*@hrCJ0|PfP;T#8LT6 zOGF&Tuu6D6UQ;8dR+%WlvYybLj66j6!y zh;^r>dF|oy=I5a-_Cm76=P!xp#sl_5C|WgKJHT*b2zW&s;R}4L>-Yw(<9n198S~ab zK7HBT&lz`hm6oj3R!!y|xN*aVb>s}-FmiX9Meoav*vR!=ZGw`0=BH7(8DN0si@hK< zXW9>1y&xld1KFf0nGTZ0y)*|^m2QSLQuewLIOM>X)RhB`78%>qmgIaNdjQv% zyPSZ+o^2R4Z*gt&Fk zE5$DUYQk3`%C;dzm0s1&Ar9Gj!8hfKi_0^QeTteo~KI?U23gQQo z9oJT|4JBnoOkOPa$Ep|v1SxyyEiCTRZQI zg-Wy>*+(;aeM+fZ+N0qNa5x3#4zZengiGP_N+h_MS@K5TE3yxPm2Be5J8UD!CvRII zSw(f}YPUN{97!v_Hc+2QDP=Ei{u47o2NScM!Cy-?QgCxQ*ys+TSp72;5;-H6RD28=g5JAwdV>}z+`K7>mK!QaA{8%F4 z;cq+6MA->oX_t_?VbB!@;`9~_HP-E^b72bt*;IYea@pg-a>!il+rDgCZL7a84f?x+ ztItQCR6$70uohv+LL~zAcL}c8T!6~hF*PP@Ny%C;N(EHlsn+g5yV}5IIqs}`4PB1D zU8pMPJ8BeF?0s8#%}0^gBZiz;s}hoDiDC~dkXR40qTR*B$0FSdMXmi`jS%3fn&Y7j)W zWdl1&hKr^ZLm`G_iff3n>C-R1q``kjuy0kc!yg_jc4I)^lH~AaB~e-Kpxe_od2VJA zU^%)9n!+pK#3$a)1&q7EpcXqcVSi@0q4F#CE2cg%s5_i`iYPgr_S zaxM`?0oYC$u>G^wfML_Janr>joO4nOkE407er6>w;uu{X?-RR3=Zn{cr2ll(yQv&U zOGg8eP$JknH>|Y`E(nacFSu!N_BpAYy_EbDiYsbJZv;!H0P_eJue`6PDz|K5W&m11 zrN55pWDji-y?bNF++$W~SF;J8qIckWz-7J+q90F9Fd-BHOW9DVVZ8KqVfzB*vX_~@ zNKj3PuSo!&G?Hl{ut6+h`Je9QDrwD-P3zaU$|m*K#-Si!_-w}7z(-M};6^G8UO$?d zcI4Z1fxVvY9dnYd0y#wY?s`ac9p>tjX^vGZOG5c&(iICm0u#-%-S*>ov&P+sBEhA} zKJizdPBEuiMsMAsq)8zjiQsCz!H5?aJDm#9YUM9D=~d;cTXigExt#~%-pp{PR?b;`ow0bVm9>_HMZ=B@4Iy2E|EQxXnDARsd9cz%j4 zk{iFUfWw-)03r*hvxJio4(O*mq`GoW1dlfYK+2a*>*AGaTyYli zqB;fScnFV@sKOP_R5yh^MGUUCj1BdKVyvo+xIZ1)8|}V9>fzc9yyBL)cXWCQGO9y! zp=oM;6}7@S)*>o%;;^;1ur_)v9*E6pD)8s2lm;@xSAcMP7wx0tAcj7-(<>G=KV9^v zYUwdQQ}KBlB+RUPk=drWc27i8jI78gPh7pyN|wEVR^8VtBR4V?;X3f@L6yB^y89aK z)bLYR(kx8uOs+15(n)J|!2W(Yz!4LCDr4i2n!{V%G8&l%QldL$$J>Oa2jqn2_dhT- z0GT&*1$mPSf)K)Oo~paray;dgNPHW>Dp+f_qANhaZ7;<~=E132=)hi(&3VR9tM-30|t(EW*30yC4v=cKH+pxD_T{wIO=X0bjnbX+$}P2ZaQUZpz9#vPN#?l{?;_pt{W4oIX&6d zF+w_*e1CMJZRx5*2Md?Pm_c5pFtO}da}8DOj2+7BxbwB;=+LXqs~Avwo=H%-q`kXf ze?~3|r*&MouTI*UIJuXnv{l-%C1j*Wm0wpu0C$$n(>mHEL(m%llaA>$aKS*6lURkE zB}@!zGMrp@+@#ueHBaf;th%~B!$q{S1Q%%CiEJh!Akg?mOK}|6s@WmPOoo^a)0%xzen*oiQg~Mz3bE5`Fr^kyK`bl4%1N3e zW#hnsMXr=OS=X|{AeqgyYtq2^C6@CqhpI@-Wl=A_-h11ilxq zP4usCl4BeUrX=KR3PM}2R|8U)!md?) zP79k?*w&^p%{}!d(XqvxNxpUI4_XP{WuuilWY-QcT>zGb?B{Oj)VHjg%ZSHL&ZO&; zD8v=S6`{GxNai*I+LxWzCKCYs(<8xofFO69Nfp81UoUr}>y8Ahc~7bEVc4{7X)0|@iT$=bUCSqmTXmR@0VvfIX)5+D&^2K&3N-! z6UB8BP`Kw8B^r56LrTtZwkN2=bhWE(w^P8h=R@gCiz8(4H#BrPUWoiXVj75iL>b3) z2C8Qn*^O5em!~9oAgo!-HqcE?9dRKXN7*4>(JY!nC%N78G5uZboGd`2TaH_gaX2MV zo~lTntjHP0v^_S&MwIa;yMbz|aocv*m>zrzPFD28b!9`GtDc6$$v}#1(+RavIFU#j zp=?OR#ya@D*2D-p^fk3aiMoAUrel7C9Z3QTc)I*B#wVue>piv)eHXrD6vzj3tSs4W zLxFXM(WyTe><5K5Lusw2W^p9aJ(!JW&&(W-Go#Lg?pt^WrCH)5%VnyQ7Fu&Y=t}C&Qj2}b= zECn?in%+Bj6IxIhLDND#Al}0pC0HgINVl!^50uU4QQ3I9t4q*+Y9*sc24+Ifib;n) z!<;A-4*ZAgq&ueW(`os)sy&FixzSiE0G=6PjA=3>vvD?Umci=4o3VbrjmzqxOjq^L z(hePq@Uri8<8+p~$)y?M6i*u!&yFSVJ!>7<4^XUXKDV=BC9%A!5LPTQkmNfS1;_~Y zdE9(PrN|UxuqItVn-9zl?^K_YqMH$*Ma}mN6PFtP>VQr~$aq!@aT%=BwI`@X_0tV> z6NKBR9sEZ<0TJ1cPRmKg0*blvPPMc~^0LnFU=J}Np$nZH3|(W^TnaC1E2{LRpmSLj z6MF1Yp^zZPp6s)YR%kpbqMJtG#Yn&d%Kh_#S~^;b zBp+8mL*+yw)rG3bGxpg7Db)ql3FBGiaX9MKRIfRz$#CW|5*qsM0;3SiaSs^Iv0uxK zW4Y(^UA+emlTF`3(Y|ElA$s3nIXPZJ&%rDm+k==ki&j#>wl zLW6GNhvmgpkQLF)zaMP5%d9M9+fC%3V`Bq|H7qi`dnzTCK|Rlk3T_1}1Fg6nHpj=P z>{D%EfXFq2wx$E*AiQe{vO{2T2a5?B?lErJMl8wmy`0eaydiKCltD?5kjg&HWg4TG zjP!s7bATQK1bFk>oLmcn7Fl=c-ZHeF({P7uo-X(RBI|c*qhd_Uc~Nxr-nSl{xI)8< z@YCBEl9g=Qj=n194#Z5zCzTq`e3)$2wEa4m9eEtoax5h&h+`5m?M%n*6@r+aqV!du zxlUlPw@&E4x48lR;9iX()o%lRsPRBB8@1zp;yA%U60q^H#&KU|IFae0tAbFFZBOar(WnIcER4bHg zW=2rME>6G`ghV?vXAH&m3G5}#b3yh3eZp1ZFv%ggaUF%sPbh@XiP1K~(&3SAI-pmA za`@&!^9gQH@+Mcbsvb2!=1Zcvwd z?~IR*j7&P*y?sT}B0deOiqW=s(_eF##frsqllT<@+zAachoW9!ZFZGc$55r3t2T$Y zo^Tu~>aCXO@`BN0E6PM0bJQ7#R{Z{UZ7$`Oo4jo6TlM6X=5QL*82*QRK+fe_Re_eS%x&YB>iOT_N(%@a*wGX z;H;(3MIr`}55qva9lY(-8Hygv1QG<=A#Wd9T$2d_b#x?FvNxruLt+<6Bs!uYf@#Z? zBy#yHuU>$zuM?1=qz^7LWC;q*aBD@=1fGWSuZ8yw&_9#imm7lVM`6k-Sr zh1gVcO9F#dmliVDtn3tr$ZkT3?SKzou)5ud-XS{e&{1BXM}R#MsyQujUIocx+Jl!- zQX68AYE+t`zr7+MWQGlD!={Cd)D2LCf}=o8p}6q_e)aL~!nh=x5=u3`9b?uw=vbh1 zFSxLIAqR-uZ#-6SD;Rs;85ScL(FQI)2fgDO&Z>Ii0YqhKFc@)eMiy4!BtU*CKp+oM zb)Hju&dRd;uIS4|SZOOtW1UIp!+8We}TgW2^fdpUlR&%J!)CL+ErI zT1=eG;XR%lsoabT+LgexF3l)q#jy^92ZoP3q!Xe$69QR`5{)vcxJSyTnBIyYi=A=6 z6}B3snB?jgurPO^e3d*TXnl?~m?as-E`Zzjco(b+w$QA27?3uw?vQ&%`8UHcgRjd@Wp3fGx&G-ueO)*K6?wW9iS)!vn13w->h*~mkhhe^mrC6c;={Wf+m1avKqqugB#6Qb zWi%mr%yRiovaD`tigQ+!qOf*g<= zB>}E%Kwwy{R-`2|CEm+vkTie+!I2KcE-ETQoIrlI(%*^lW*r1)s%V5EyeDl3VNFK( zjJHTV*`dyNNc~zDz%$a?EKeWkCvJmvk$@#5ciZAC9*Adw;sFM`K-2Nx3~+g=w+(d zMuC=JrX_rGt<$-$nak4hCS>K!yBDL_+--`J0PVtTXohfgU6*cnl#1wa`jr~k93mV- zZcYb%siq=8P{dukdPswze%ZB$b-e@p~~952?iPPvgF~GVhM&JWW==g z#*-i2%o3zKt6JGj48fO&MP2QaX_|ZR)dgY5@6O`$?PYR%VpVaY*LKf#~qHw_FuSd^8>d@T0 zs+?qU-iV7_)Uj)1m%eXsl`>5{ayjlThm^uT<-G?s5_NXv>f-_#DtFM%WuO%g4tQY4>}B+3dpT`M7H7MVd=O!4 zxCP4FqOC(f;0B@sad`C(8eMlm#JoH?qIm5ia!J}KP$K<>>!s7_?WmGdLISeN8gZ(5 z?Lw!pie;nYKP~{2$3^)CsxLuxwDHOzfv*o(jaXoCeT5viNhZW@$Cu6&dEU108PcW4 zI|W900b(429^o#Pa?wZ&!U#Fg2M<29>U>I4nm9;-gK!ovY*NHxE&3QfbZvZk>#gqS zVc4MR&YiG?0-@*7r$*Lc+Tn$p9`56#$yIM2->Ya!C}zlakx%3XUP@_(SJu;dz-xBC zt4ZGZaYlUN!U?2k3D*3NCSv)Qj*}M zE$gME5ZP&_zBgcsVq6-c2N&t%sWpM59vA48-Mq7IiYhHbKL)8zePiRKRNya1)s)TK zgR{qaFE$_>>#9a|X^b0C73y@a3FT0b6E6*NFnH+};Lb#pm$B-)=_g1C;Gwzc4$6Np z)GelCRa#?>W4Tit*OhB4`OqyKWSG~ySMK?SV&w5%0V-_$;S&0TvigksU zs_a`hTVRbqi^?8KxzyAUu~VL^PM#Zl`pj3WV+1u(ZR=f0afEOZFhP1d3=9qA?KZ@F zvKkI+?G>m~&vCd5L-eU_#kV*>(?<_m3qn+( zaA;vJ-kweaj6*q99=uEkhDgmebEZ(8eW})j$<%uVI$1=Ge6OHF95xpSdjq9}R)mea zk_)qfRZ;`RszpU%7tv`vsWacQj-+@Aq>P6QmqO>?-%VDb*RfQ(>n zeUW4_l-E&nH%UC_a5=S%yTmN*XcHLD6cbLyp;+!TeB(udRfFP3%(J!}6t)C3@-QL# zLqxr1%o3X>!Lr!C8mAR3!53Z{Pbg5K)kw9+MMC^yxWvo!n#=j1$V?D z9e`4x3|_`jfqRwwKG$f>v7)pRQ3DJy#0HsI5HJ+W=zfN1Zz6OP4mg+?C5}3ZH4JYh zfe+rF5StVPXr9#Wz8b$2OxqlAJK=UKOe;)lFfFJdq08J|7WcVZvAsc#e!%YH6cz0g zN&#;MAp}NVF2v>6qL#`tBRADnTlWH9Nra6G{>LuBXWrZihU0pRd+ORVzzU5PZonJ}Uf zRV|IdP*6R%ElI~>^o;n=^M@hAa;SE?n5ZKUOWA_T%rrUz^;fgr;*N`Yx4J5tS&P0x z323f9#CPWiR$^r^eibSd7-p1NoS1S{`Lv?dZ6pN==z zroVjvX5cBYKp+-S24k>EK{dio5Tm?_Qy4zU^FIldHB^w`9KC?KlO>QwZ8kn177-&E z28JJbG*RU#&h9B4v9>%7HfRnYUJEWr7S@2V=uEFzw5DjQ#~7i+Pz%T>+#a1$yVn>q z5rP|!F)tw2%5w`RIxUb}w})+|v?(Ri+VH2SW@E2D;(N=`ED=bNZG#3%9JLMO3?x5P z#S~cXlCBMfB?6YfyHq$okzRm17*P~v(^}Ri3=zmusEk%WgP;JYSB+YI=yQvW=TK6H^lw1 zoRnTwRRNzg@n^6(28HXppbfbn?QlO4aC{|I5sRwQUR3ZQFIgut zW~&XksdPv@YJ<8>osc3iP(PC%Bx)e0{%J0XosHE6JUAM_z2{u-)~xYH4%{AW^q7sj z$YD<6;d;F=x^8sWm!Z#7t;Ev2a_Ua?C&6dbwxLnu>U|hKptWuyWnH8doUMkPmm=_( zY62P8n}@;{Z;xFM$f5z`yu&m-C6>uInHnH_80$P-wGiP!e;%lBo%I%IMW!P=skn8n zO&iw4R|WTjg6bB5g^}1n23xf|ZH+iZS&YjZ2so9})K3~b?U-L0M3`(%FxjvG{31UH z_m0U;1*X1DS&8Mc4I}r21cfv87oK!Y&_pK9*_!%7uGJ! z#_Hg1)oTnXqRT=uQY=NV)nrOa**WP|!ih8lxz~kB;A5cy6%sDhWC4o~Ndn!pwLoFD z!U`6J5ECF#0|D4CyxuraE>~B+ssR$+MZLin#$7SUA-lG(V(AYG-Vk3Azh>Hnrk3td zqX|hcpV0%UE00IZj8)z|HK}4d`@)E`An9n!&-A>aT}!FFlq|wFCq8SyRa=VrkD;L9~bz95yw6!g40gRY zQYx7mQPqpP=4pK>)B7cOOU8=rdIt$?xQF6(I8jiHO)pkVRylEe=JnyKKHm_)Yo8Vc z9@AL)1SZ!Wo>&Jvh5&VU4WsOZcnqd=>^bnn#Bvu!M&$ebqvfCaR zRVQ*h@EAbWiA-pFUd&0+;;Oe_DIf|d1;+^j^4{d!jNs8>26k?d6ubGue2wskaSl=B zIt>U|Lxc;i9gLTWP%QC-s9&re`_T6UZ2)^vIN_2bzNQ zFRI{(N!wLc0dQv^pa;T1R8KlmKKsis0XQ_?6AkNbNA5mkp~*{# zb^&`q-72-oT^MW` z!}hnMWp7OZJxQw8un+?JW;~<}Vx#Hp_=E?Fv&=fr2wu5+N8mkM8_%dojMEBda6u7> z@MvOfu4_Zkqc!gGLTMAw49jtf=5XuLm956uLk8)6gecuU-b2ErW<)jLZT7|IVlCcG zjR3>|xVDI>VNeb#$B$H&FAb&6NZmpK+uNG33pc7dCJamEp}(?Hek6B36#@EZCP+xb zX4dscs4J2|#eF=Zw{P7xL!PE|=NreG7|`DG!K|YTDdT7k5x7B=xz1u3*5|JEw+vRL z0x6uws>;TMjPml~tRXK9XkVo&6hO=u57ZrxKy?gNMjO>sty|1DIHneu*Lq(ohGSICy9UqaLl(nIR z0rS9b2w~Z1MLQ)5k5}HCKSukKXq%Z5)E%^MK_(Rx7F3+Xa_<>YS*;kiA+e6YtN;hl z4talB>|dx#3W>!GDrnGg(bg_-J(1Fv4$oj%gW*=DnbJMJWjYE;15HTA&W-RGAj`Y* zR(S7Ug&U;qEIqQ$E~v|f)5QRU<5ldK&Y-sU7tosDGs#pgnK7pvgdvnGm zAi(83t?Ta7dMU?HMi=o9r^o_=?3;lblp#6o84c$QTkjcATDHl;m87u*rE}VmA*C4$ z;M@>GIhO^mc(JTGdRHGB`%JIp^Ta7E4h;p4=GCkC8)}WuI~0aYa-~;j%orRyN=8ga zIm<84>~N>KlfM`&pu9I_3AD33;7kn4gPV~D1^H-6XXz|aF$`w`Yonpgu3Dgu2j)Mt zaML=dGsBG(k(O%RIWL(RU~{Bc6hlL^9Ty3BaVQJ#Qx=T3Ook9l%AB1QDJA08M_ak# zftdw#=a(iG-NQ{2-HPCKAaz1fhHLQUWQ=-shJg)=uQX^dUD6;E9mLYdVA9G?IN;E zN^lJwaAbv;$JJoZ>sHabG#qBmNMFGkT8rCW16>}c_0l=aeew@9 z!5}jepDcNCya^aE2usYFa?^o2AmRo^HB2p1LwB36>Mf;#>Bx| zZ40DN8vW$FtWLvugYVPF;2>$kv$!t`!FW_YxVUcQTF0S0s`{k+HklYVK4QGZ5nwB& zFGyyx&q7k25{eOe5-X)g%$G$-t0inex6-MX-Ah#A3z*JalPU^Q1zER3TPDI;RM}ag z+i`3m)Wdm^Ie|_IR?xhs>8A@`L|!NjHB3_ZeC10eaSBV3oWYO`Hv`AXJuhR;2!2iS zHb`I@4A?BXUv!4^xtX1msUO$`K10Qaj?fg5uVZ)leqp%UuVBbCpFbU`jPC(ZPoGV7iXP)PYE z=YWc<#%{Dlta|>;%$M%JS}+=c@Q;N9mS*`W(el21D{)XS4VWWPRNESzUeBIamC`f- zMuK;!E3&~b`%)rjyGXWre^rjXJ>Oq$a(Tc)*U#uG4BU{W%EYYp8?**?%2%YSb8R!3 zQ6`R1*z$C{mB5CHw<^7G)tlLQN)DABc<@CAIN+o8dKsX6cB0s!VmrxA%Ww-^KOP?Y zc0Bjm1t$>fjQ)L5?SbXQ^`0H=_E1J+OzjE>JSUF~=)72T+|k5AU5DCPURh!GBWvv9 z_n1vjE>E|uvWsEPN;>X&@UR7?WsQ0i}78>%54MgUb5j=ui z%^3|2mzy-GX7(`2d=%lvL*sHKO)T`)w$D0CV-s@AwPj0XyjZhlJ!4o$ZpB;qjzwi1 z#G{9>`sbWj6%lwmN#xlc8;TL0D`|L#$MmHj6WU^~;OtvAMZvmzf_}*tIrVcQQ%=o> zmF78Uq3u(s-b(2RNdu~4>#`{+Z(hiR2TPjBj{^o-V|TqhVj**X4a^X!H#_x6meLqJ z?Y2utqsY2hQX+Be2kLn`o$a-W-Me5Xt2fC(yMK}X&UoYiMdC9Htj?Z60pj)?bK46q zw>(hi>&H=Ej`CnBl9Zk$aG%h_Z*KxK<}p(RGzP9rs|Qsev0a)47Kj8@AmiB*ZMk4V zwiFYkng%fvm*EI@rYWyRtUWzwyfIckm}YR~Ixg}PIxn;l{w7w7m`DY@xHl0z#avc7 zv4kQv>6n|9plJ6w_i$?qR$R6XCfg%P>C7aZOpCH(XH923b259UOV_Iyod)6N;}RW2 zxKYRykg@k_s~KGM2s(M+O$_9 zBWbNlqsj}{pMxtk+R*FEXTiYOs=yL63!RE9q`P2P(Ls~g58%Oq2=W3=jD`$FmtB?1 z!4o!hyz|~DS;VwEs*ORzkp6nCP-KMYR+JuD8|GO!#5<}mQBjCH8L1(>4taQ@TyzW) zaED)>TMi+p{Zyt9JY*fKy9JM>8XGx8gqNbm;ESgmxDyhl>X#2>w`hPWe=W6$Dfpho zS!2#qDrjaLG8oQOsM0n{TOiZfdrzChsn;H%t7&4P=6!5r;Ebf$Bdm*NzH8|ssq^{S z-H54xLyR@2Hc4TE^cD8|BwAFKHJ(Y|X?L}DOwS%NWWyQ>g1@q%nPhr}4oj41OE{r1 zR$i$OiS(KVpD);*0D&r^>xzl-ICNxPu8m)N=Itjed7rn3SI@A#DTRXrmUdAA;#=Z8 z?~#2B1DO%np9wfZK;?juLG?{to_6-O&mv8#2tg!qEzSdmI!lv6a=i8Gh_8u`YTJ6S z-Dv(BEQ^M{k)+aNVTKDf1S>T%(oPj#ff*Ac)0$XNg9Lz@*!Du3Wm{XQ4qWCSK#ny` z9*fULcF{7vf}rTVlnq1~I3r#_LuBn00kGU~aS`vtw3HP^Wqg4zDkb$8oz*b8Tj>h& zb~L1Af~E}Tx>(UO>R=`)Dy1yw)8cr{bL)F|i-sX|3Y+aVF9d0Z+^FN1vD{%gZzjsz z1VVM&BE2;9z4Ob)~3?3*oqNz#I}p08$3?;7egN`4l<4A_>{yTqqjn z1n?+}^{N)AsGKT^f|lWgX+ev3#cZQ+Y`5B&q6*`&<{lvK5&J&SeyA|<#i{H4jBQsC7Byhr<_=x&J{*gXFbR9NVtw0E1#LeP?=XziJ7V1QLm#3K95}FgHkx6 z&cv+SW&)9Lw6YVO14`3ZFQ0Ffrg{Wm6ONf%=Q^mAM?RBQ&|{|yavDxhuWx{f%KFLX z0hB1TqPxVlQ#Tma4mTnjkV^A3BlWR$rpoJ5SIi@MyY5HYs(E*uTB!fOo}b^M}RpeTEauC-ZOy`Q^oxSCEsq-o5; zL$zgQhI^MX56;4W052164_kdE4?Eaz;s`rAcVXpCh;gV}=}?Q0i(nqaKBCf_2~&k% zIdj*@>xg}r55??M4DVKHC2!WqoD2kN;tg!g6P~5xK`~&n9;LV^6{7Q)??NPr_Yc^X zXkyW61_7-Ps9ED%B4NknBV2sLqwaUoOYbigcUbo8n4bC_Vo)G7ymWpbgKkM@Zx z$H+OEGu*+}O*Fc3OUsrW?&7sXnQ4EmsDo2EvI41kZCd@Dc>U@U@*s?#lebJdK;?E; zTS|`eHT}FcE#liebyw9Y5YU3nVlO(p(Q2wh9v{4YQ)D?tu!Bp2;e!gpFgj^7%ce!8 z<4~}isfD;;2ZV;A`&NY8KQzN(av}X@DKpHe2)?`Eb&qI7^gk3{4>6@XO?CsASV3aP zxefcY@$WI`rC9dHpUVwp3$*lvz--(H%?czeSlNwIt`~QlRh^c*M-4nTIEqY_?&~o+ zH4Q~J>O@Q})m+whajTl_JloC#wRTKzv!Dy5SB$o8!6RtMOsze>__oj1md=u4fZBzf zz^SLQ{LMP^6_>T?$!sZN=c5!JvanR9XAaIz_m4g{xa@aD~1BhjJC=EpKB{KD4ut@tuvSm_wtiCA+w)=c6h1;PNDi_V; z#EC`R^Ve}Pnoy7}eyz*#5>nfJ17j&w12~5QBhy;Qm^dmT9Y}FkEHs$LSUPokB*#@z?3XKM6gawH_OtTBXqJ?~?T8K4kx<$Q)KpxtXD8~0@mZ@4iu zjx${&yl0j0IJJ0#xV~JjC|DuT+dECDUQFAG4lJs70TVhHiMl@Np+dSL66@g=4O%e3 zo6iOCh98Ys8ceNUA%?=D4p_4A+ze{k-JX%Kt=dhaly*_>860~o`sz!ay!N*YnT#ok z>j@)h_1ot_h$5+~@hy_ylYHlh<|EE4KEXx@feH7G*CCG>Qxn{wQmBzw=2afk*As5E=Y;AD5Q}6B4KudqL+|p zc2eYeOrSx++yz1+F{Bk`i)C;S6foT2xnU)q`4xubp&ul+?CHIZxQ#fDhBoJmL&^+g z%B}IifFD9A#Rl`61bu)iLe$Ht3NVAg>uF*+hR@1IO3E_ zP&BgEu}7rTbN50Gvl+jS(4TeCjxH1TA+4A>E&=j+Bhst`OGx@qGJ;l|zRP zj7C0sMu9U^r4DG?SyygUuWQ7Qb$47PplTCc(yfYZ)eJ2Ik&z5HtD#Yx?fk^8fNr9v zIx22KHS^D@>8GoyhSWP%W#Iig631>5F94lQ#$J2_b?>T}i`1P(Qhq}2?wUUezeJyy zxB|sF(1~>FLx_xY60%ng`bl8rnOhll)!3peNMPAuDeD*u$bNqgU5LnNK-L(zJ-uJr zg;?qv?Wjt=E9EkyHXP7a8(i$5}d#na0pD*~thM zO0LuuNuE35V{=}5*pXOB6DwgiE-EyS7}kpv=BK>5nQ}7h-!<+Z*MLH zgB-%xLiDMsS%y+_;Ves(!!U-rFrLijiXD*%NlIwJKJvi6Bzw|N4@#X^RXJ3I3?1hL zuwJz4G4^oBfzD)HpPl}c1;YK^>3Mfs1Zqq(@_etVY%vfj?`hk+w)n*IjT9x=5*M?M zPiSXHOSQ0x&k}^`TTUhAHi(M7t%iJFwuMX%qE6Lxb5A+N&uQkkHzU2zOcDXRUijgw zWMVq=Fm!52yE2f)oQm6vgHV&!N9`KtQ z_l+XOFcEFHcyO~VhjbNjfa{z@B5!P+ftOEUv`$f|9nh>t7=w$-7e56KC@%p>HlB!0 zge!L5j3MA4XuqAcGj&@{qS)=CJ=Rwwb==J;yFn3d;7E30LET2BYf}ND+gQG-P)uH&f$!RBjKI90VJiMT{Cy0=CZdf?#{{^f^5wCfb5V`0eISctGtTX<&GFa#y7VRV$Uu{m!s&K z8Fw-yys#1)QB5mNTo@Th!>GqW4z_T~77*6kIvTT*79(FT2=EygJSrUy@EzmBr$M#n zdsKY%#G7$8p>DU+bVQ=C&a>-`rBmV@nsI8)$F?h8Z%0~;P)zmM%Y#0QK24tGu;P&! z%qepeg?EE#MJ4fZ2*ZraYhbT-n!7F{8-@|>%)KXj95H-aLl8kB?mxSf{>TXkcH}O+ zNXXc|OM2n0^brJnRCcFNN9B>)UWE6D6RW&H;9M7dC#%tt#}jg40@&9Hcw22jTBD1`C5 z9dq{@ryp*3M}ZsNOJ-#}Mm5J&JtH+PK|z8PV9OnIC2AGSimOF)#;K3T%gBK_-duC* zk#%JPQtLkrF=Ttv`ygp@VE4!ecxRG%=^k?RFlXGkc?<>S`N9!*XV>`cr%aqHYDLSq){YQkU^AP{bbPB}K7S^1_@Qmiy{ zdNfafeC6P@>Ay8h3)6&=s)myf8@wQr3hLD?hI_El3hst?%*rbHo<#-BOENQKx6u#d z=!1JkGU*FH2VBq%(7>PsKonoEwt%b`m>^QSv1q|VcH)5WYGVqJCGLxBkVeXVW7_z0 zGaB?2$wCg3P2La@U=-U<_rRhPcoFGSXfjsAh2yFrnTB1+8yj3|j|<{4fjzM#a24rD z#^&V}VEB5Q(=t3~r9ft31YYp;tv6x}Xmo=J%)blFoeu}BROi7JAqHF?(?Ch0K;l*q zl%ijx&R|SGRJrRWE<2X!N_pb=Za!H!-j0f4qe#&Es`W=mEs)|IFScN~X1CrVPq5KsJ3^ig^ z(tyWjLlI=sg{6q6Nw~}1;tiXW(d%_0mWC)_GLRKU;0uX-&R#7+r3+I%61H%50g(VY zBb9RVv%7dItP;FOCIStX zrJKYzr9CT?jlMq+N3%_#c(cDT?%=j)Y8AmjDpmh9O z?q70&S?4_zD%3efdK=Ou1KXepU71fs>#)<|0@?2->_@r@yv9kwo-enk6-c*f;7+>| zkdhLiix@3O+9C#gw*ebVKKT6ZFdlkPaGRv)-s3|Ont`zb0k4R+2W!DxUeF@? zgW)mMqyc157Ap1Zb=Z2%k-T zOObtuu8x_?Hr+eYqGsp?DtC)nUB3wjawmO~@%yNmZv6?Ls{X)jyONk1RJbZ^cfw&|$>` z*4ZeQQr3punNu8N(tgXdn_cL;?Z9j8ER^qd4#x%SCXK+E)Lmy<4Mg)Bo3sRcne&9v z3W*CLl|?czpdyH;t^S6G!3?9tZ)O0FiZ2S^0tIFChA!WSs{5x$A{bAQq{{~>M}TAY zKIK0OBH0qPNk}O1!Ftr#26NBBRCO_^YTN{NQ20=|Y=^>Lgs!cH@n;vQXi&uOq$JI` z!3BwoBSg4JJChqeTQ9ev=z_>=IBeH(-lNz(xIV4eT5N@Fz&uib!9SGM(7E}Q`+Wj_ zzA1<&VKcg}cpH7(lsL&LnaNLKo2-khnPSI-!2+;rr87Kfo_Hh@w56L7SCT$wpUn~8 z=;60xMC*%#f>)y?Ze5j$Jp=&@z>D_w0`beKQw9m@`ByAa4`hzTpXM7njuCTGTBmO2 ziq{<;g(Fs^JkD}OPXcq^hA~-*O~*H3YL+=H47lp2goL81>bLBWheZr?dqPNjif7HZ zUmp&10R+Hpn(9!WccOA>U~Lcs1P@4(468hT@#5l;RA(lO`5YgMV*F3lJs3KK2R%Gn zHF$_RW#;oUkDkrCUZr&#P}FaqP^Emzj53#+Wfol#8v&0CI&VD+6UKL(cUnW5P&|I~ z*AHqyrU$hZFE~7R&JHnLo)d+%+2Fh-vBOC`@Lb4G1~iT!9!XO69{JzJp!q;jw?HP zQ1eAK=)L&MIBC@Avu&8^$EAv#f#~m8sG`oUs%4&*TEAT?092$(L8)!&zQIX*a z6~8rgPLMrwUP*dvw3i0jfmjAhuW!{kB5gD_uEjK+tEQP9oq$kP7XAu>%tpjjUtDnu zN#@^o-lq=R9wxXK$3%$_;R0oYg5{MlO=l~ZhpeD=rEUuSjqQT)hL_7pf=tmemjFFM z2qbl*+_~lxo-DUotbVyrd9OoG=^Hdty~==!2rp!O?iuK`#Be@dGG#m5!0Q6h@irNc zF~ShnmpJ${j(nSzO|f2^$$HpmWV}5AqJO=S9RVr=KCf`MF$*+G#@Y4(=C_L}06>o` z439&^r57s#6IumDYdt&!55pKPlOUJaLJK5z!3*X~oHuHLHy8JaWDAcIxbm+dHeoY` ziC@9snR83JOV=5fh^QV~S z8c_ZR6FSrhsAdbwj>+3xa=~3?hfGIVZV{^0nc+sd_x7p<(UNWPh;@d|{pzi!O)W_A~}et00Y21m^@jg51Rj z?)vTU<(tJA3QV(XS4!^rDK7TuTm2xS7v6nI7QDQ|=roOyX)l;nV1mxVq5_u$Lj`8tLJ4gVG=6OlvAct6wyg56Xc5s*1ioZg^6NAxw{r{4m5O@~fRfgk{-@($jp)9lmct zjflyl>30s5&5{cDX^%E9%u2(TdU5mmU({G2Q30^Z@Kq*)d5Va1r3(kD4w^y)s%#wP zcH?!(q9pTWj((~e$efB6@YzK+N^Z}L%y97p->|iH{a(KNLp0xA!jVG6^zG|56ph*HxLW$L* zdR9e;1RSf(DqVq_(EJcv?=(O?;yPE=m9t%(n9z)wuycg?DdjwahWdqU-aS%s6e^sZ zNum<3)l`}$7VM1;kJ*Yn7D8ra-lx7Dy-;hx5HO}Ah6P%#8V6ROsKd4C(({)wE7k9_ zfQm02@e}M-7ksBZ9h6s@el(td{)ZSi&D z4v#sY)K<0)D@ubwiU9TQwOJ#eln&nCWo^8#gmQ$fnLJwPW7%N_jyVam7-u`Mv~KZk z@PTZ>Zgjs+M-Rp>oBUAfN&*a-#SM~eu|~-z#Z94Xwk5SoWwlh=*(GgI zY!R?l#0`qsL2OC1fv}qaY@oJQ)E30rs@kfys2c*(T zZ3fhZw1Z_*+NzC#O{qrGCe*8CHiX#|YK@3Cl-i`(RkcfGNwBt{*-L3FWQnqp+9|TO zgti9MR>@7OEtN@Z*-L1tu$8eE)fUPvh;2o(Ce)*3Eu`9$Xf}bcMYT(0meQ@NnVlQkznn2{xqID`i`40kWfM7S&q?u}zfO8%mo>wxl*nqiCXRk+9e( z+NEriwkkHDY*cL$wyK*cEvj2m*w}2U*+H}`X>BF6t*F?AvZmD*(k+o&NLxxxq?;uh z3O1l^D`=ZVR@AmCt*WW9QrRP7HlVg6XsX(V!nQ4vHUY2}#j>i|4S?DxHVJJdY^tr5 zR>HPbY)fReEuf8%wl*SdR1KA#Y(cbE#Iq4?RLn-vR?}fiX-Tp{wN%=vu&t^aO`~X|WUY#AR?;TP zLtxs5%55pKQf85`O^T+}hRI86N;Xw3h&GL?Y>{k|Y=dP@s20g9YNpDj#G6f(O|6M+ zRc#Am*=&hy1+*=JTNSVt%Bi(dY>?QKX>CcdTPkfUWkYJIv?AIhY^d6mY>~BFRJNr} zlBU&DX>COWl6N8+E&m_ zir9l_Ccz_Un^Id9R?URAky}x5o8&$LeWi6^xWo#&I z654G6uvWoaK&_(KlVNQUu@h+xg*K9HC9<0a#ce6IM%37YYNKi<#5RetrpmTSHcD+1 zWhyo)Q)I1@wyLeFt7#3XErDAqn@Vj`TPmvB7R5HITS}(MCc!pJn^HD|WUY#uDK-Yn zX$smUw1&ba)mub1l}g$xY9`2)v9gre6J*$%RBcfJR?6CJgJ?rwHmS0uHma?W z8%t!3sHVwQz?Ri3H5&ppr8bHiCe=-pZAi8wYEs&jHd1Vs#TLm?vIVpYV%ntIq}f)= zro@{>wkFCOQ7xshRkTfnrqxE$t7?|SY^a+gZA95iYExx4rq#6~*htt*VwTjV)SCjf zl(r3(ZC2EV)Y?ThC9qc1hR~+arfRklHX_+l+JxFbwn=QMEvi!4DA=L3D%gW*Ce)_Y zOJZ$BY!z%lvL?c)*b8YU)V7p1f^7wCErlk@O{&@mZ6?Zvu&s$(RU1*Vs@kb+DB7Uf zM7D)ZsT)N$pxQ>zY&NKyD%!1zRkbBHC~Sz?6|#uhsM<}S*p+Qkmcq6Ywi4Q^*&9){ z4U|o)Z6j(MQZ|ENnYoCe4y%#in*}ze z$X3)=!i|L+61EYr4T##QwN0gMTPU_vn^h}nl-erVt)iO|wAia8%ZsZ zO@%h3wpBKTY)z0=v=Z3^XtqeUs@k?C8zQ!;OKRAQV6BR6N;XEyQ)o@9me89@EsdZy zD`J~LY(r>lfZA1CRGSHGm29hI+NuqTY*S*DY)INF|KgG=kNuTZ3Z1|2zt8*qamO9M z-<|sF{&VO1`>%Wc|M$Os`tkohZu|ZkMRS3FxB_zumbZ(c18N+@T8;<>&5&y&tz+Im zS{;j0hQ3)(c{5qx40XcI@r3|LkeIza4O*RcOX6mo98n)uf{enamW}bJSQ?XbgKgQ1$OZZFkUX=p#hnP zOlUc%>L5uh9h;ET;fMH)xiR)r1}rP`oW2%ycD>1X;3fD+8! z%iuu(mZ$4qH%?vc(4t5VwqLfL@z^hT$xVoHSU{C0R%+#kBDm3_oM|xWxTGGD##9YJ z`4p6g?aPYtRjziq(duT`H!q3LrbRflLb}sO(t|2@lgZH#8e$E8iq@ti#|@0EV-`EJ%WrMQ^reY`$9(gkHgzd`{$D^TWm* z<72O0c;3SBM>N0#6^9UImcm>gZ(Z7!=u z5R`Jr@q>n%R#Znq%$YPaAR`17xfa9=!u)J47Mxdj!GI)|eK%f&lrkL_4Ro#1n`p9t zRi4sb%o)jqUL?f}8zOd|#1XrnB@1|&XC?KH3)sPrFYbo|5IP2MoPO&7X)|u~3Vi|r z#gd-_%8)4if+7sW3_lOFjubNemnea1fOLC99qVsaM(@iDLIsFhwg^0Bl(^m)Z5!^xsd_89TfJ}*wO5F~y~V4uN`y+7en96cqwI5Fe6kqP z;4S4^rIK`}bF{;w!V6DG8hp-O1%NNb*kGH9@M_X4&L3Q$m7~eNif+5E~v8$2%9!`v?FV5dH3qKpKN1)`Q@HvH|1h<*dS=oz`MiM5zf0YqO z1V>CViO%zg-(OX4sA?;oG{Gq$JP;$nlWS&{X{6#})L?`xgdidz;RRob2|F8^KSH7_ zG18><>(XKm+z%zWPwP)F^Dx< z0+N=>)O;7sqUKqxv68$h9phXRh2A8Y%NrkMBPucx>sea9N3YB0xX^{AL3<0cIE|2o z>CUOCekov)Mf$`H$f+#^My~Yb9Hx$b$T`<(z1vXQ>JJnV6_^iRKV`HxgBXmDb3Pem z0fkMfZBsD^)U^OaZ(n$(`(^5qjJB;izVC=ToD4<+1H=v*ClzPP8;6s%iPbqSnSJC= z0!VCRa8+j?7MB}=U=tS{0>fD`=aNn>h%s^XKoy4g9h&jZ z?u-{M+_5xN2T^p1LasjNVAFh9ho!%knP<|wR0(Blhs`_)L1J0ApL`S`;4H$0Y6=|& z14N&Eq5wE}u}+*e+{)m*nUM03J#xJzn^n~F z8o~z^V#XR0N%MEokvh0Vwv-Hr_JuSZbkY)|l7$J}VPo1Wtq*!2oI<6&T-Muh#h}Dj z>Js?JpkJBBM?}Q+p3yY~EF4rwL#{YP3Cj1@_i3DFzF3saModi<6BGq~x(M{TLGW}- zh6*|zA!U9G@@a0k{v3YE1?Euln8DAPGg_-RYt-Ez6n24t?uV<~3GoW)G!EvQ3yN01 zF~PHRYj%XNWH|3j1Q5@G)%iJw zE<}|vl+dQKbFqaBy?Kld%$`ZnvFX5qC{lWKuX;sHML`FJd^*`JTD{=YFS@M$XLAi2 z0R{y4hqU-`${)++FHpDCAQ;_!W1Ht98YnQf(tViu*W|_21R(>j$vFk&3iGLhyf`eYESYCmEECOLt6?=h+`?O*Hc z9stDLWK*gUETA&!pm;LRt4)_RH(^sZTRtgj^;4YP$v%fm_*qOH$=6I07zEDKvS?gF z7!-i8$BYCyuzUlxer(706Qv=^+{V;^i1v}hDTk2Xt<|OC%N38{1rR`_uG}3To#&(p zLkOTCqDV2x0fsBk9b*~e78?jX2J9)K1(xFPwm>1}btf6qX#>>lvyf3+V|;QoM#OV_ zEMy3$6-4X}*yA5Mv3&l04aG(#o{BN07v`kUeK1k zl#hB3q0xJB0R0IKhL$E|q(9FLLF-0MKu#Hm5$QklJ2Uu8N zkgv#?;H|hSiVEP;N+ETRl|sY}L+_BJ!VI|pmhTvwP*84*oMY&z!4G^H?zL270|J0| zUHY2b7DpzS-hCl?PzT4gKR0@Zqg)|rJtfu|r56WR(0dix3r%t+0)3eUMt(7DR;U4Q zB()KcwIEG6rc-p9;D|LLxEHK_6jihGO!U_csrrN=fVfR(?d|9I9!3S+x{g~*Wt$Q9 zU`!Ci@Uw(H$G%8RS>jYQuq_99YsNAriS*0NRvtp_cuNn(M_`82nN2#C3DcqLYY}waa?wSJD(vW zK1)zHHNc^(QmXq9V*QumRjEHMhAx&=;^c>5CHVBMJS$IgX3-_NeH38yw$(OUq=Y&O zApB?IK@SNC0QJM1am%i4`Nh_AZNIk~@wnHH+-&j3ZhFrA1t&8yGjEO67$F&?1!E%! z`3z6<69Cr4oo=?*T3@flpag6OB-jCd5VA@ja*(L!$%6y;IUDholmO7oH?i#BLb%d} zlb{S9PB4df;M><6HoR+%ZMNKQzVU49*BpA`chmDY*IBk}ec@2vN^alCZlzwi?0eVFzj^zxKoQHo@4f&44}b>`>k8{nNXl#pS962J<$Kbk!*2!Dr@xk{=BE zpe+6}3+$PRXm-q*!cl;JtB52fhhP(kZaYDr6sYhK`!%(wM?DRFkEU15X37qUNuHK$ zZ4DoDz;L}oQ7)cxX<@qh?L#gB4#fvqhH*evtt3to?ffxwJOT{RudWz$VPxxbJ6z*H z6)$7kiDh(sUW#y#k3b(5KUVEy%j3H;2L;e%eX90S!-aGw>+R2#U*dR#8-&cpspcT| z8GJD7frBw27`#K>B|^mN&l<&eG(RB78Fek^mNUvh=942Rn-=8`XMo*QmN5@O_I9g2 z+aW~?8CScS5$GaE9`{sOLh`E-5>-LpFF0eg%^iw z^AsK@akX+%C{=l_!es-yEPCxs=bZ>jE%qrDRx%XfZ)r=s31|=CKV}*+sXk&nq+GKP zD(EA>r2;jb|I)NaNnl)bGJ((nD-$Rc&p;L*X-Wn9y zTcAHycnjI^76*>opC6~g5t0IkVI4~9{dh+`eU`XBdzx*O51{6O8XDpXqNonu@a$g# z^}Bqa#x8Q@UV=QAv78ADUIH~6LQ_+JH$sAdoQV6A?j|Wa#xGK)NEPPzdwNK2vSlk+ zGdAS3!^_}0UE;lYszO@FW65G*^F>|;_N(Cue8(zaB11}nTaES7(}nvchC~o0uO5D* z){qkIni5bqGKK6O-IfQnZEhs)>0deb!ngyWAs zkTve)(b$B1{vgf~IBr~~QLgCHlpry~32E^RsPOnNkHpoSlbE0( z1>O`06%wPBOWCObn8!JwP7@VrFE|81uP_H}!q9A1GtuwGeQn00gUrhw;=VG(18=w5A56N%o?I5*_2N&>r!N)We1k{>m#^vE?e zivSd7YsuhT(~ct3%e<}7Z@WagCh-FubP$VeOE3yh8%S%`uKNDz5#$kd;_1wjJiX5~ z2nmE^B(jR2=75dnsd@oB+Qx8UnZs#MMjn)GRBUD99t_|+nb7Fk$-KnO2sQE*3T(v9 zOne{#!W~4OZ&`~>_WH9xqNobB3)3!Mq(>i5?@9M8Yme|AZqGCX(Ti%nb7s?Z2Am@EXMcK^uqo~C6Sj?%I(U=5@Jj@TzC^yh&pGoM2y8pJ zigK#IIB8_Fb7LQYn1Mxjs#M%YmI`S&EbrNv?cmMW3)q@bmJ9eFAZoPMNQg?!ro)C_ zv~qVBd1*JIoZTv)3|{#x46uG7NTkAN_#St?mu0oopa?D9+!q(&VP4rR@acNtKqd?C z3FsHdAmbS5t5q$?po&03S!qtk?6fJW0*cKO5J6Q>l~E^Sipw(=nty}A!{W&UQ`u#u zM0NNmefE#cAP=ifXGneM%w)3%svaZa_BeY)V+Tf8=Gv<{C^K-?f;P5Xp#qXf7pyY*n{5Vpr9k5wDis3~Gxrn?pMM^vaGR%VI6WtRgNeX37BZD%8? zk6MW*KFM6)fe@G??Q0yx^>pEpUbdP(E`vh?k`nFvj*_YhE3g6hrQw6F``p9Ur{7bI zAmy36?H9U&>M3+RwS!1ZiV`pqK2d_#y zmjpoEcB_E|FkVYM7b+@w+$Du)lw0G@i=w1wz1}}$I$5uZZVrnZ1L<8VsYd}2uKLmQ zbRjn4=P51}NxM6D7|++WLdT*6(_+m{g!Cn(FtY%-6`)o*5-LDh1t^5x!ff|yDwt)( z%f@ze?06sH`9Xk8hw4|`!z3;+AQlPycq#8Y5T zLh_uD7laTjqYp3)qzZP(kyKwst^I({yg}sQ%mw;Wv?)RLO;tAlK^>0->}U(8LL*tZ zzQ}CzPELqv5ZRa1f>D>`1`1O-m@6c>;Y6P)&4dLy`)~&*F@;>6J@s8>HyU|#<`@TX zT7gL}u;WXm!-Q1vkbdDiwsAkMV8Mdav_%<co!y@&RwV&)0b8Pj&e=NdKFnrNr;BuM-h>sC(xm{?=E^`Uri5VAVKD0p zj$sQaEl}$i5Zi4PQYP+sVHM94U`7f*QjJV0!L-z`G_UV)at&~0rh}5RoKzo865W0% zVeW$Z^M=!M#;omhY|I+?Q-!wMF;K2cb^SwX9$}#c%ujStkUEsZMK~++o7TC2WOp7$ z2vpDBHaX};#npFc1=-1sOCZA5`>@HopIL%>lLnC0*VHd8A=%mDoUgLD!>qh(XdXgk zZdi)ay%=TGC|cgTTAO+R6cnSEcN$PLgJA)QHUt{OUQ&|F6wDmu!xwQMdP2pCg6Z2t z{nYfArOi*H<@SmKP~Olo%Ff8$B@M7_suFTrT(63WekdlgoLhh(;mDn^SdxNZB*CuW z&UHf;(4_>^aG`x9?jXgn!^HHo2ZH+OLW6m8q)VNXG}d(dJLRk}Owy+BH_s+hMK*xd zFA^<%C9}c@5Hr$D$x|Y3KBrEOT{-foln~SHBR&0MP>;;V!8ekdR;k#G%LME)u5KfJZia99vD3*5hGn1RUKl=OIHDT;8E` zBS^}7TH8p2l=gc-l~f?|sLXoyp;*wD!bz9r1rYAR_e*Dzl4ML!S!n_HTBOf3Afdd_ z25m#hLT(oYH=sO3WkB`^p(OJUX(-JPYV4PVaalg>~kjvkJ41W&5b&n*^bX%b&z6@ zDZ62PG>1#*c_C3OFz9gH1dohJ?C6k$2+9ouF=B{I0U#V7)W(%hb|4?Dyf~_BWN^oF zI|&OC>`a8|A+FpNdI?5D!Cep%0`IPmkq#D+$7s=g-wDUWNTV%i@{elRfC}7rJ z9T*U)DK5c-MG*&<$vnZ{nr;ZK8hya@@}ckW|WKXEY+Q9!c@%+?YWQAZdsmRk?L47052dNF81@BCj;^sNo!{ z0%C)jZPClIPiDy&<>#VikV?=NMq-k*kS`XtK4q$nuvAq{P^nJ{dm$B9U}^!%s)vXc zKZrmG%k3YLDE_hd1b$iWwVWWRu7&xg~ zd;#!$A8grv{rZxXpYpwM_{GdWZ55d#Vt!k(E_PXQh0AGbC#}Y`OZ+&%q}bU&ew9k> z=E|ozU|(VUJIG{!vR*7sN=~6$;AA8>iJz6hOh^LgSd}LCtI!8>x6;TS;&=hHo&_&L=s`$7Y+Un~?QH z1Z4a|2iSg|U}(sS8-l8SC!n7w9My9YjQcGssX+W)))(b_In5AS< z^EV{Kz^KDe$zOv`jHlEU8i>_~=fu2yea>1riS*!bq~sXe(0*|No7T!&)Hyy~Be zT!Dd9921kzy--6;<(*IRx<(U?ZEsFle05mH*(kLr?FT!pK|AV2vnoQThPlfcBcF8N zsqhKNZq}dJ7YUdS^=w0*5P%fHFJF%c&jl^G5C@+`HaNotG_sgT!`nRLlE>CjJPqMl zGzZHO69RLcRxnO~P3ipMh*jDfkBTc!6WYC59j+R%%AJ#u^uSFUp%1iC*sT1%DnvBP zB8noW>2h$_-lGA0pwqPo5?wH}_=O&{EG30>fbuQyFFnN$lFZ?VNfgQhOAW;!{2unV zKot!W2#9Hs2ZEsiYG(0<`OuHbmV|0c0*Gf#1f6n&3CQ-#aO(|PqrTmX=NN2iTZ=T9 zLV9&9!M%w8C;<$-7}Es&Z;hPnxRM1hB3xrDRkI>2 z`LYJM^dTvMGOH(i#>1l!jp-Jd7UM-_L##OKeDGytLi&xa7$%Ejq2^nbVPa$}Vc~)l zHF>xZ_(ceQ`iJH~&DTCMb3CH^idyA@0j?^qoT=oJFVn(}rp!Rg$kxeCC=M$z5FZ@|hf)mq-GuhZ$0QpW ztUU^M86lt}R0*PnAg2}OsjLLS;8vO~4_hD}UNLeCh;>M9FVh4g9aY+5kvcRqJ~zC| z8&o~O@48_<1Wdq>1uO;v!WVeBo8o2W7I{YZo1KCNE`5aSMz{~UKAY~Kh@1u-3VVs7 zD1Syj24i4?fqA1xwxyM3+*}phiH#)2L_^PJat*K`*hPEnOK!mz!3^im7+PZ9JY)Jc zBWZx63*LkH94gg|eH5tTJcj-EXCb(2U{m9mYz!g&L6 z(dqbAQTi^-9LLj+?-7&Im~IQ>ToSB%9oHfib(G`=kw$LH%LJko=OXLb*8`1&&XX8; z8h$lj7X%c60M(ePNp1|V0amc^$lH)#xck_!Gqp=8;tU6aD(FE_EaLZ9kz+~vP(hzo zO6@+uk+7Y}wzk3uT&|`NJpuqwP;YJEfgyIJi6?aF#@YK`UTAlO7OXfzJ{+{^8(bfB zpiWA4r4XyIM2gMBIdg02SOKfZzCD4g8rO%=d_Y}m=B%wZxI}I&oH}bvO=o*gh z3O|JQ?YBaHzV__F$|s?m^m(^OzEf^?u^0iTG?TN!hp=1;G%aYSGl;R z9>|lJ2^!j8T-`C_r=E(m+sboQx_%2#n;8$pFCaj+zQG9+mDiPubzdoQj$b8A?BVQr=qCEGLs9prQoVl5Qeb;P#XCva|Rb=-~*L#oej0xkGS zcyh4VNN{yvup&a{?@C3EF`S4B#a>SM{gu~&SzHt31o1^-0g5tUOM}SU_f9C%glv4_VzVH~27hk!9t50JH^65Nc=G3k6gP<52%sh z1b0kFS567(+42<&l*@$IE9EpBypVAo!7R6=-8=CrNJ=~GXvjLPF-6hR1(9dQB_GYt zy-2Ill#6_KD~rWgLg8&?N~B9tqawW*r=4-Ur@VNlfNF9BS8CH`fhwA_rQ&Vg7mke? zi9@vUW)&`!#p3i9DCdXL&X*dATLXZ_vFf4LwWMQUAi@!+5MtFd3?EvX!JaD9%NGi0 zm`g8@ud$^)Y0;dQ&E`?R2qZk&w}Y3SmKb0f92%IHT{Qu)^VezuxDYgGbp&CcIHRde zg)?OS*R*0Fa%7{3W)udq`=*JMN-(U!xog{Tb&;hNjF`{wOb1pq@frp6gl2VV6Bz0dFiGJ>0sK~TP@x?7V&Mks<_SuZRBw6=?V|p zB>5lbfK>fk1f|H^H$E59gfJ%oRE7%Tsds?P5(`n-T`0_~Hbcl=Fc;UDQ5*xL4ky_>~~^4qKc2|Drk?DZBM1=;nMi{6P zRophMG4Y2d+2Jfr2yn2bl|}^GHd^jbCLDle+ay|cc+vB*lbV^lM2_Elsc7 z63NM#rvwRLB^enX6vZAT6ekw!S4=nFyro^H3wu|pT6kWuRraa^%h)t}!1bRr6;jg0 zsRM2-j;;G~g6*^zbfzzoI*zQ64A^vNf}%=?P5B(J)VftSIQSN8SG8o%LYgQM z#0RYI<_ELM73NQYA);H#1$v66918-hYIr(@3_=aZ$RVeUIgtfwT-U4mme8Ib~oH<8riEHAcs6Ng@0nYhw zoER%<$kT-824S{Xc1pMmn!G%Vg?8ddfSD9md*ocmG@A;c*?RFEySclSK_c}z1`ft7 z);d@ah=8Y6Yhz2YH3}PVF$A&>HtCDha8ZOdov`$r=E~p>cJbUy4m2x4Nv|ZgvUhR} zAd1D+5jz6R&?_Jyu)Lkj@jGE)XLx(m1NHD&{>caQ2#hhb z0S=N{JPr-_BwVz;*s1O{D5*mUX#CsOJ4!ohCLwPzk9}5f)>%q&Cfig?j737IysFZDAH7`PiVL?98vN0OspGM)q|*hznX|RzeEWbv*NYX-#VL1ST%- z%@Y~My!&s1*c+y!1QCs|9!-o7 z`UiDw=!HrKvFRLO5T82n(TXN)I+1+S3kW_K$I@TA#5j)oJnJpankDzeg{$U~BB8{5 z5Gxb|hC0FPqWMMn5`4DTELdP(UZ^BTN5KjK9Fr3m#eV_m(zwhAG9t)4&g1V}x#(wk zC?IMmYMYg|S-Xv`Rd*A*i|e|2#QB6P&?ZI#6$ibTFLZ=KY%==64!+UT&rz_hH`7bW zVtKyA*#*>xQucPP^H@UDsS>mJne~9pRHl*%s)5cP=-W*itar!%!j#EezKv zpi(wo+s|N0>K*S2Rvm(r&u1;LV+pXzYZq&c25g4#OBD_Smr|9dz$>{KB}Q-o zUpo9(HMSNmS+2b+J$sKw(;KFAuzEOns5L|4@?H4=FJ+o1ei`~tl!YtNgoJcpo)Y`k z;#1ZQ+7v7At)m#?mjef`uB=7mtuj%cPKH8n&==%akyKej!OIlpEUmDK@}W1mdQC=W zHbUNIl$s9D*x5Z#S{7^Ku^^{Ix>Q!+YLu-*-Y~m~X`u#n%7|3UR~1UVViMyT?PN-H z#~*J|?G0)4w7M#hbXv^bh;#0UMtI_bxK~HQ#!@?mc?Tx(7&>n|wW$+~sCq#FILgsv zB}kN@N!m;Y7Z%*6>yp4Yv3iJ}JquEquRdd+atyQQ7$}*CH6uA{mx?t~L6q49@7*EX zpv&v>O&`OmydM|`m(uLQ)bI~{+4@DZwRit$t z5*FacH%|b{nu5dXz&U{f!EEjt(&Wp*X`%D?1qYz`PDGolSh;Ds4H7H`Jb`-+2p+j1 zJIx-dK?xJJR7;k13v(|jPc-QtAq@q1S%IYv${7ZGlmm#DF;JlcTH?Yz z61;7h(==JCMMCyd%2(m(S*REg(ShfbezUei>IHI2(k>QL06YRA!}y#qgVwqNY6Ljd zK#XO8S{7STlJh8W>PRIaDe3xD6)j{T@n$Us^T*+a>7fvs7cm^BE^b*Dx-ez3C_+vK zG1Y`RkQSyr-MS&1qT~)u4S@yPl~sUOIMW6g+s7gwH5rimnR`XkV73U7o&y%gC)ua$5ya`mP@|-G zN5Soch)K7+lE}-_hYNOTXXz#O7&Mw?2?R9*?^Iu?hXVYdRU5eKu|G*j()Pwu6;AW4 zObFR~nw}!$Qduac6d5iQvZ8?<1vd1vyb!_C!*eHIPzxb`6Pi}kg&Q72;pGado))&H zXF7D%q+{zs2&RZ#z91eAsgO&sYX{=t@;fv)WW&Ob8&o(2GYp{SeI33U>4$Nct8J-G zoo7-qUVF((m??fuvDFy#fV}n4&oNT4ZF==9Z;B6;R|p;ir>`B^T137}!s8qdYEmK6 zgc$>*VLf(%kdDlPi9=JagPJL@?dskG@esFQNm3=LSS$(Mn9@9ZbD}V+6SN_Ti_Hl9 z7J$AMIoYE{<=I8p$n$rBkK4}xY;Yj&!InV%8l$T7Gox2Bh|Uv^%n00$uf^mO23-ql z=~-c@xXRvRh$D=@P^X38pw=-Gj0wW_U#n(IjH7J%4mmUz=L1)$Ft;};IUXIfX5nXpDk?APl5zeCnCMX16I^fB*xEA!72F{7S}GNpiE;Bp|#c&XOhLfuNBP z^#MN;3uSXL*}iL1_b$;xC_cV=-va-x5UyeUu_px68x$U z_GJ-41cl^{>E;3#bKdamevG^*sj~F+u*=Rfa4|=M=IPv(2e%L(L}Ubtrfp%LY@{8I z_!I?{As)9== z3F+$cjNIAhi{*6n@Fg5jfdaF%>IWrK>t^dsjiJ`hZ}wg$8i z8F;-x(H`)wHCE=|Fa#%2((B@e=%@>tv%w+Pxe8V$RvRHNmh-LmCmEDKoXuxLkxNkx zh;KUI5!O)sMYq#ili{#UFb84?Kp2F-(%EY&u{ zqV7bLhfdOU%RJY+uvI`o_V@ehh z>1Ixcx@wjJTI7!N9grOE7A(x=zKC{&4pb+YvsHqeas z6Sn|iLu}RA$m$471C57Yu&Mp$@F{Bhm#s1dh8?j!36aO+wgb0Pt)781q$|rL1hVQw z;BN7t^D&5J6`O|Vjkn92DJ$tCZOFXjp#d!MEJP&(!y2bGq&6x{rc0kmFJY;K*6Ee$ zzJzPl+fQ}{CW6W-6qcyPwI>P)z&saHowV5;<(aw^B!Y<8pldQlc{p>u${r>WbPfyk z_KukuNYY!DQ^Q`k0D#bqZbAo;7agh6nQVP~oUDK~%M6zRJZ>0aXnq56acoBeyCo|+BWpo|`S4;Mx5}pE2jev|VE$a-56&{9Vk}@1Y@bb=5VKGF(#u$}~ z-83duG~3ZA6d^aKj0KEGoi>s;ai|s-BRu1vhJ_l6t_kUj%KZNRj07 z(tJUu>F3l-t<~L)ol$>bxV}WHpPEyzo4m}CCBUN&x5PfP0yNxwVpVl^m>|e9IUewH z+u5^{Uq<9^U?Jmb30)j^4?%;C;E1k`$RTP;#W6UFNMUn5wDS1JT1JmVA{5@#!P1VE z>hhO8X6}uLJdBZLm?Z5Vy0Pg2NYqe8ZiF?Cx`Q!&!>4mt-(vX_H(apRa}1%G7PDa* zwx$=!#vsZ#w+cU7MI&-+%51(`HpNX ztZkp9n8=9=I6mwNzR89^9Tu{pW$W23-@?V6;at!f#R9v>1j>fttS}gOxr^=)O_P z4a-UxJ^{+Bq{%B&V`ajP;>8E5aS{=*i?e}evD3q41;%v1+La?hB`0DECpLD22BD## zaz{{AF6UUem6PCaZcKIxhIzd7%AM?Rz=7>z_h9M!DUp>Lgv}1Bo&4?o!s-@iL^y?gxp-+w)J`~FHrU7zpKr&h0v zbNdZ zZW(NQ$`NH8DP&Z~GamA`Ui!$yAvC)o^aLW1Qstq7Vfv7`oV7T(ms=m+(q^*it`LO0 zEQ#uDI|-a{n*nA&qu%SrU#J6Vqm)kPY`UT%I|o5=1@-1@W7BCM zQi;W{S>L4>6{`y}o@q;xAzIMLQZyF5mAue%W402@EDNN%z=SwcgB%4s@5@M`YCXQ| z^r?_aEDX3omz|spb%2i!$$})r$yWa`II$G9nD@^)#ol;*1_kvg)*KZ=Gcu2ag!+ZN zH&g|tT*Ps70)U9~JgBbk^*_8X%n`VHHf4ZLh(c{~Y~5`j#-Cj?9i#0)#k3|T6DF3E zMqWRwNh)K2AU3IH>oYWzsb(;pjvj!nIK>!D0+ zz3;Vu8`ko5wo;0H&n{Heh!SXK0p<-LKebAxEVgpjY#Ievd<+xTTB6&~Xn5!;?TBa) zN^I#O+efL3$O$eQ8Vu_JnFqwvj#io#TOsg68^^h7A?pUm6f@tDvR221L+LoVRb@8Y zO?gX17~!cYvD`d?XSi^lDvHB9?|Nfh!`2W-i3NQp5Hym{ULFB=&tZWi5`@LT+sJm7 z_SIa;R#-F&v>bz#Cnaxsya|NNxL{yoBBsOy84gN_8O@>tU}OTmt(DpjRh?nr?qZn3 z6T};k(b3|O=P2`&U1`886%b%#7cMqLdEJ{Gx6{+oJ2B>;QZze_5!!1p3ebMx5);s3 zIEFg$V=fh=G%$ZaF?@tnBMQ0%LxULKS{H{=16#G(L{jR;>@x#gj+zue$)s|fgywt| z_R2s3Ty~sr-+J`8hl3e?dKOq#JTn&#vBT0AOqPk*N)SsF9|(k9K53jVQn`Q(%#}h0 z?-o$aCslPaJp|@5swT+20h6?ZZXtt2_OT32OHa&4kXcq8t1 zrkAin%-heiVOvHxfw+cdQjr^t7^8>M&kv9SUu`TPX;>A1G&}2)q%e<4Q8DC1l;bS-137dQz0xHQ1qX z#AUbaZBpq+aQVGXdET4_xIkDCWUZvKbb1e*R!TMqFLSAk8(`+J4R#tge)h#!HA6|J zrgR>CZ+pT*JO>_g36-wKHyK!AS|MNqn=~!F1Ra}&F3Tm+Z6p(bi^pP(@ z$i}4n?`6U^7CRj5{T%V27N^U_X@x~#u8-&1(poW4x2A$GB-{&H?+$x%o$&|-_<$aR zz+q$Y3uz#jpncB&dKJa50nY)7h-V&RE=IE`%wqhOWvs)p8O&kc4OF+y?^#jNh9r-I z^xBT{1*|*E%$bSfX;jxiNA|%dIaS)3C|tm!Vh0sSb+?{egka~X2aic<+!SJtjvC~J zh-iYf8`#8;VB2;Cj3cczN^YoO*O{+(WrIMc#9|ddrd1&{y$9vrIj%d1Q0Zick1sn3 z*eeAin+Rr|5<=@Z>sr!Y;pf)gBgqU}rnReeaPGGn-3in^BKw$9*X%9i;~;Fa(5+_{ z$BN7v;b_{n!DxZ>n4FOxb# zGXc!9n;kyj(HDv)omNi(T0;XBO7V_>uF#**Heg6pQiv_E`0sv^o$N`>1AYnNVwhMC z$zka~k8BtQ+d&zmz`1M=w*0{zVjh5QnRou!@n;L&^5&*hJE&EM(kIt54M^t%n zG))Bq3(v~c!+=9aarht>nuy&Gy-&)f%{@zca@si6ZKV;K`0CgUJ7y8N zi&d7)(_nW4ZmRMYPPpeD$`Qjl0TG7wre~*DPK0YuK%YMO1ST-LRuv3}R!tr_MrF(% z(mB@~X6#(WOL)d}Xvncm!AXA(0l+N9NPC;On4~z3R_V_M*t;pvH`k5pbGYuT+|lN_ z+sEURHy)i?r5Za1Ay@8e@R=6C_1Jc5(z%za)$;U?=rfgF*###nG;7h;F|=pTw~-8r zyRNZWHlpsWx3o)dj7@)T+aQkdONOCnOCyhQ`MMPlOj`O9-o$r2o2VQj44LBO zI)kfa*!Km~30*24z-Q(klZ#so#q@%&(V@jG-!${gzLJIJ_1?cCQB6FV4vA)=F z*P#(JGXgfa<6wV=&D4bT>yGp+>ki%;64Vq2;p8D4Sx_Q0GS@&D7-UYi#T~##_0$%> z4Xzkd(dg2GfT=1B105w7=ui=i$2%F_K9N6}sRh-XD<%4o14jTt3PMehU10vPAn7Nk zHj_(HKB{hlg-$EWUbmybd;#**OSP1rHc@)w&;o+XjiQW_@i=dOAmCFCC-eR2TT_Sy zV(yZ%`sA^7fc0)3@Vak&K7=vk;%$`1n0Cj=-f-Zk&?u;@Su6*N?|EDjNT^M{CUe?M zW8Gy@4bx{^CIFk}tIaGKH&lnvG+vj_2ifQ}qy;IRuo@VcdfKdWE!UA%w$>hf;mG`R!u%4g|RiGE|(^ zVT=)BIciEnPR(uS0eJj)Wz9si#nrKDX2!?UD)JO@Felkr7&=oHHZe%3%Mso|ZC$5K zAF!fjY`+Udmw0lOu*;5`JMN6Pa@~QurQO(;yekscth1CMsH$M+^hkNIS`?14LXtBF zhE6IG@tr0k6vnAEY`6$3LPjnKI>NKL{63}v#f!XZeuxls2n@0>0^8AzyA^rqLB5_q zHk_6T#!Fw51Yq#7@!~<$2S7s)7#nvw3pPYz_lL@KHBBuC{0u)cu7jAYl zd?e~d{6^T`oe;)t{YSALF+Of~gk|fS#8@Dj<21tsLtkpd_E1!D`Pna|pjDNkHxUPQ zfmbbJ44VjeB)*_fAQm{A9I~_qsz)veN=eNrGbaI=rUbW(F0UFLUFVQhxV=m(47fy! zkb8EmqR=^|Do8CrPn%mf6AL`ygFb(R!1a)LL?D;%q7G_8`4pY~rVC|H(ps<=pol5i zA?>y@E*?(`8URIabU{>=wPi$zM|o^l=zzRi-sm50T&wQcRq*4qsxQpWQPEu-{DoC4 zBN_<3Rd<+y_@(SZK2VYH^3m}HEwei&a8Mv$fd?bt_zJ51A_^+_qnhh{Q=<3{Iunjt6-VQB$5$@fhIQAjeDlV$XK$Q5 z_1|^YZY#BxSq#wz3x zqEWW!HS*z(%vbl=kIY3Z3&9WI6ASW#P%W#fEg7+UnM*YuY4L7gJ2bG2lH zu5IyzB7ZKWi7g;WfV(!J88Ni<7o~g686^Cd@ZfupxqPv8XStXmZzcrz(mrm+3gCIU z95Chd!06AjHx<;%WYp2SV?I1Q6v??h2b5;OW$Lt}w;ZpAD*@fAb z(7ps+l?zGoX|1VQ7ja*)tz4hafMAtRBzC|iD|&rIN{e#%dz_E8H|(AFs$gFah6;jk zz#x69ZPfvgg^}SZkxYvk%pkuerFr&GlSw4YXsn(*5Y??~A}9)H83)Q&7Znvz(3v!9 zFvJ9Eh1>4KQx)C#SsxSbgj#3DqRUTkWym=O?)q|2oUm-uz^53Ln4>)cNDElvo?~!* z5)_`Pp*T+>ESoCq5T1^LWou+&zWGCLNH_*nln0k|@g>*DkJU!e&6|dw}Gz#Rp`Q#U>~Kgv9Y9?X@s8roD=h3-WJ;en z@qUD5PSK9#I;mDzRCHTgx8w`4eP1EQF?(?PG>=ox7aeI%r-F3O7CupR)*3U9a$AMX z7_v3Y_5<7oAVhn52bQ<5O|TG{y4lI(Nu_?jhHgUzY0o7k_cajA7;Dd*a7?_rH;t1t z)=pi%Ua%z^%H@?^KGq=XU`?GxE%J=`P`ip#zOmddtp$oCN@DHf8!RP3Y42UAS{<=5 z_Zz1$FP(4@Lk94^1EvB#4hxA;$j8;hw|*9HD<$Jgw%+^%;#Rw=0J7NZ_Wvq~uslq% zLs6nod5&vq7`Z}#+|9iIWg{|zwhwfXfVBgKZ7dSdL$LE`P7`>1H$0Sq+%I{H=jG(C z4f6SIL3Iz5>D%8lt9Y34DSFKzUh94-58FRtdf>enc+EjA@Y&ZPLHk!^(lgKK2GgT} z;!JER3wy|+G22TK#MqNp^}GTfsY<~OXyTPjXhHbk3Bozwd4||gxNWVVwd6@%<;@=y zTFXLPRVBj^J|39XZUhE&@a#ZR88fCyIEDOiZBu49}cCI7{#wSAEs# z+eDKe*gpqw2@az50|6e^rT3o;UunlEj2uRNk6`MIlP9gtMdrr@7HB#O6DykHgG}kCE&SrdaTu=W=uVd2^!L2<8D_Xue6)7xOGvygU}39vkHSC`$GO!{dH?PoxPqW#^? zZ)M{t??p0)euBMKUjli%>_LE3gf|!pDPj&FmO-NThzUt+DWv&$3p{&>ev}MU3=FEF zi>Trx-sYOx!9q+K;L=N-qdf6t8;wyGJ^3V5*Inio9!i{SfqXzR+TDhdaG^yuS&XlR zPj*cRy%QF?H53MjRpCN%NXs&uQmUT7irBYO>$pfVo-}U|=H5oxm(Re6FM{|Wz_^nM zm2r=>9@YiL07t_^Hm2~C&@fS7Er@zHhA^d95F}AZ?!iwt0?=ik8N<=b4XpV#{%*=d zs&2M9;taw&O-3avLdV9Gq^%l^9mq$QO{^qjpfkeYYWa8Ty43IK)m(uF3 z_6~IzloKS?m5UqOD28lY8%&%+(F+$*Ek0TvNw6}JpOTW+ASrf2j;W>j4xAyC`WzpY zdCz-xZ(7>aj8ZMs>4zJqJbq(K5)5ruiYN|uLfRWy)&bJUHx6xUfeQ3|+IO#yYW*>D z5tfKpRvAjYRz%B`xr|y2n}$(Yxc{pfX zI^NAg*U#%T1Pf4%Iz$K;WCQftB|RuJFgVU+yI0O$2@&{Lj5B1fIcJ9FSPcSk@iN8T z0EZz8of0F*Ya!PgTIUVxq;`d_y})#~dLme^#P~Hupfh=KeLP+`zV}RIIL%uwFSjPz zz-U|%l{6FtVJ~}gHtLWl@_Ng%#Z)j9g^r_pN0gkexK`TMMoS^+H#6KX&WnJL%@+af zH$fa*F<{(L>H&w^M&Z~qo3>(I+4ei`9&dreZH)oQPPH^^ZuA%zi^@svm(J&rxu0?k z6R4lMlG2jB5R+}*I}CoG2cy}~6bU^7{~xnLjK0A>;BOO_W?48Se-BhA7pK$C9hl?{ z)(M&-k_BI$mRm!4{VOn1lm8~u> z+&e~Abw?w$$w$3(*~845+4k3`EC{W^#SSzjSkejknU@biTZLO@dlZF^7uJwq_|(KD z1{uYaPD<5^1TIH%c`kCe9xF-_yhAyXBNWm{T50SmI9=V*=p)ML`dGTFL!}H@>1LGB zL1bd9Fhi)qUdb3y2)Gj+BXpHbQ_bWVpjz?(Hh`Jd!waf{p$u~ohP(@)h)4Q0t><`chBl_`$l^*?5eUB{OuFLK1J# z5HO8LP^mqo%b}R4jSkZ8Q%Y)M3smgaQFtYlC`qp;b*EtQ5Ym}@yfJGM1Pajuii`;D ztwIe@fTxuy%RYcxr+0g16J%>A5oZ(@6vHryQGBWpJcQ$A6?eGMd8hAL*j{RcDC?Ho zx%l8ma*}77xlD&yh}3a&eF6xHGHcAj6KB?`$_j)SSY+(J(3C0-&8D4__A!KJU5gn3 z)roauM4>~|ZS27L*Wm|oT(!R0tU9gE23B{(@1u3Iq4ly{Pgkk~>R4hX3f&t9IM{VJ z#R3IlCtzh(Zu~m-=R`w679sByKwa&agGam*!ryp5ryUTK6LC0T!l4+ZJ8Lu~`%T4@_c(5; zYDq$(yv+kNmA+ujGs=YxyyE0Wkado7S;?dnip{Wz3y*9V`rN{tc!=>3z(kpgacmHB zJQ%Lc!)YnCwi>4g##e^gakk=I3g@_FmN`CBHGcS64@5dLR!L44l~z?iSo!*Mljd|Kj|L?j%W_~)P4&)Ys8zs6(qM|F>8}@(QjXqm*rN`s zc<(vn2t?6w91yKwf*Xo)keS`3s#QZJdmuR3gal6*Thy{yDm8_2^T~-SjEl+!yVzdI zc@RTu9VaW?76y=Z9SR5Ka1iMngGF+nQXWh@;a?-mtLGUrKcUe?#ccp^XSEc!opEyz z!Gto%OIn3^Zb)5YC&J5N;}_(vVr%l^Fv|Bx7s!O}YmLzQg1hKwh_oKw@YDgN6?UNu zHFX&ofw=5gcf~{Rke}KunFq92v=Nvgf%LD6oRR-Xxo|Qw&6shKELU!5%M2foW@hq5iBFvK6j;x= zpHlYPL3q~WGs2Kbs^d&WogCL&a-got7N@u|O2x*}IF_L5PSwJ*Lx5a3>8Wc7A@%sT zCOX4vvq6q0*N@uz+`;W_xb;|ST3CYdQXZ(NJqf?yf#O03>u?xWt1q999O4qd3RHG(o^-q=OyE1gxe# zBlGXf0!(sdZ;7}QYWY2@RNr{@BhKHa2X~Yk(RZglRlh zY7?^)N^&9cgJ2gQqdR=D9(N<4m&(h=^5U&_QS}OxLFMiUgQ1USG>ftam5yW-p zU5#_vFsNf+wL_S)6K@k{x1ZW86jB7Y%&e5k@SwI=eI$#ej~G|&dmo7%fdaV)F&O#A+b2x*xM_j@v9>w&s^{L?6w^2v#U#XQ%_1l# zkf6g^*{zhv_UtqWAvoO_PZA$j&oR<~``I%G)IO;^m{5^vz+?bzy5pD#!jQTvA74|j zyQCHC>Oi{}&?*8I#BEA7R>$tsk;B}(s_#tGt~+^_K_<3&omUoRy^et3i3(Au`1rQ>VpjwWi45Qh25}~e7>G0G4smv+N2QH3*+pq zjuh~-FxZNX zlp!C@MX~L-(z?LJ-csi>p-~4`uSmp)HezBq!gj=@5R?FENem5;6PSf^rM2O3f!|uY zBhqyJ0Z)UFY%gHG19xE`)iE!C_&(Cyi)~ z(EPI+arXfR+pR<~ySH^;3Z)X&K9LJdAjAOXJT2OwQ1!08^)OcIXCQ1*>Sj3o-fp3^^gj!q~Sa@=; z1nCqeavMlW(GGJ#11xZ`WP8itq*K;0TnV29%4-uYT^!_?2x>`&D0Uw5GKBsXY9{Z=eTzJrB1!QqcavZIkn^kVjDx&Y= z5}-+8POEVwm0l!pfMrq}3mm$#4H_IsihENavGAQxks_E0t#t9(rJ2pbY>Hk-18h2Z zAg+#!A?8v^DE)j?;I^PC0V9KxWLs!TOmycy9;`_cY!pWXIUB_5CKf%Sx>YOSEmpGC zVQXUzG;6G8z!Oy=sjwFUi7JC$72WMkCIrB*tP0XI89g6>sXG77?9y?|(2>860KBjIUj7@i< zOau>8t2qqtp^U{~KXFd4k48BFWSr>G9ejGdb3$hGquN5ITQ!bE=jv#e2mY~I;b3)SP(AGXFob+4!lx|w> z-65Uwu}{e2>2i{|x=N+AYfUoZbernk&XgG_cqbvKE8II6L$7a{euBwC9FSR<*8$X( z1%Z^F2l~gIqWowmj~rojehGP`MtyB6CdTRL4@5a0sa;#AraF?CLu3iqX%Nrl@z*%@ zmi4;cH)YywbPlFq&jEA>q%AQyQ0V_RdO~zo*OFfgB^7 zMyG_6Qz(kVCpls;Hp!AwLpJazU8`g)%}t z49$+<@(`F?u%#7?San3#%D@?5aZ^aOW88Bw4dKwkAlx!kFuZg!?XrC+szFfv>{84e z7%BjzostFZ)wUya@Ht@Qn)p^^C2Gv?7n{)ndU%l#7U4j#3tC5fQ!#X5KZG>hYu-Oa(#e4?5 zoe^4A8W~v;&y)o*RhBv!frRXl^tk;bUoHWC0t7HGA!gJ@KVMqfptYqqJ7Z#$6E!kf zVvcR&&Tg`r{c@d0AZHQm)9p8@0x(TGo3yXYQN>|@wE zPD9UdtIm!Zy8*~*rziutRFDs8R_413vEjIujCl3499g#|+X$>;fRQn!f?l78=fZ$C_N^&#N#?yhKi3C=%1MH16AcMLQ zD8u1idN|*&q8Ep=#MMfcCqSOXTQ4H#HUdemLKP*WdYh2wlILxOtzUXZGoSjk z`e$DdSCVzwPfCJ$B;QM;tBf8)_NAXaf1uP-=ASaD7>>~zfbw4Z(3jJX9 zEz0dgLNl;y4!T(;`gSBd{_$i>8Jm^n-CTpX zq%MWAs*{y@tQtrGLK+gVUlrL78`9hh4l<00JOP_LU|98cN_|;a>pf?F(A(Ncx91o> z6vY^?0UcTD+y^O}$+k`-s_jg{!SRnw;di%F$L zK-&7@YTbj8zyLwuSw>b!Fho&d%@-lK8j_S~n#}Da7m8fGaW)3>Te*XEt>hf`R&KF; z#Kl4vI89Za%x*2sRy9!ulnvTop$iWYKgHr0;~`;I$IRj)x&f3IR9X&f_A?F5g>#Od z6gam=2XEEuTQbq?nN#4faN2_j7je-%DMDp+gQgL4_@@Tzn*D~CH;dAUfIu=kCQ_hZ zau^76FzDXqyzPJwET=*5qq~b$ihh(eA|XLxLm9rtEi1uFz51|W>qQW^ zgQa1;=v5jWu*Q`U^k`H0jw zt+mI{j3|6Vm+CRd`67%h3Wrw-dg0JOYh72ELb+5IWW$eMdKri+SYx#s+FtRY@2nMR z8sz=#9$&4wVzeVrf_)gyI8oFRe7`^+c1ClOns9^(7O9~}PUZC>OxW_Wu;;Ag-@bq! zzNzg8U#0RzM+mJ(DeF2yD{H5Jqj6Q9~FWV*8+T(z`Uu%<CutoqX$y^5I9(ykr;#+5y*@M zYssCQfV$AsXI)XeZhl%Qd^a?LY>IE&ZUu%IAhDz-)2QX=ycaU%UrMl8!ZY8Q9__%B z>Ja%thWt`-5({SM*Vvd3eiFly&eylIHOJJx4`afM06au*^hF^ji+ zvpAhq=(t-K9g0oVzI&dh$1NJe&B6u9;3}emol=d&ef30_(srn{$;_DHNlc%|TpT)^ zi<*JC1cKd^Hy{AQ6{#xa(jbYbnV6&IrB90aXr3|0B9Yu48>=34(xfm!JUrWapRg5p zDmYQqOCa>Aa7w1fj+T0H{Js_-j+==mMBc4Z3L!lJFNpe{Ynrq@=;!+eF4?j4!c7*bge$QC)XkLEgxo0 z5REIYx)G1+uW8^M0fnv#ws#}}H<%W<$+NySZImu5GSk{u#%^SICIi}mKWj+9 zF@FbCGA3J%&6CO&^M6!zwU;gBLzlu^)~d;LC)jwM(D`wSpmy7NdaNMyTe2U%B&^u< zda~D&jSQMC4Oh`q0vYfw52Knsrf_UfsD@ZS1&N)&f=XB-1p&SP!m zR)a0D1-MQco@IfIykcH9UR|Ik)9rac!o!n!Yr2s!9vASIPRU-aygpWExFg+q^IY?Z zS^!X~me-BjBnLs$934=`&jsyit{p&2)-D>f$X&dAEaB=Ouu%SD0u>Yx5zrE;woa6X z03&U_T}3mZ*lf)tRoMa%y?Ne+y5#uY&ob)g;3fcEphfwGB$A_j&i&E41*%1xA`J_u zwR2r8w%Vq}M7x-Tr+P39Tt*3)?q_8g#%D2QB8Z>ly+uB(}x@{xmDzfhCTVR2DzGz9E`7;%dr2!{N3 zn8Jv7V6K6E!ANK8JQI(T^EXhELXxXHLz0K+(9W%L`MG*w%1HcwL-HM=9=?WFe>0qN57^5&%W<@+B1$Vxj z1Gxz?tBEy_DO>`^7*;|Z89j#}th-`Uw>+C+jw9*W5@R8eBV7n6sT*|WOXNqQEJE37 zcy8|S;E^l`>|l$`QaE# z5P=?A7<6dD2U!ga{9+Sl;E>m)bD4{q7#0w-B+%m?iy23dfWzD;vH=AkGXz>1NBi0* zW~nSzrRF4GJezhd1Z6$2g<->i1TA|;vTv|$V%J(WPR**GAbE#?gdoj*1tAia4O9x^ zyu(nB4e3e{HWZ7dqE}}6P6c92oqM9hdG=vKXdE40JdpX*N&u5>C_ZnS=?UglmQ}}? zh9I(635NSJ-uus82BIyIujsZrpq@6q>iKH2C5&EoiBTatmjS{Y3W}V8(j77BU4tfy zO>$qD8L#L$CP@{fOv>!^-Yqc|utX1SRaZ4lo;(CR`jaw%wnI`xCOS2q{bQuO)Uc#Y z7M21Vqta*&Ynm{2YKZ<>$T1AkNc2t&g@p=OjzCaAU<((mEklfB9A!W)vA{Cyd{br{ zPbUki@F9u8wkOV4?MhBbnQnEQ14n;OU zHGG$(mbSP4<=3deb0vqMHavRE;jmJ`TW1bJ5pr%31D9mQE~o zjmYJJHif!&{F!92a)K2{JEVCyEt1zU%d(p#;!IPox*NX@E7-BHX+}* zIIXuklo4=41U=XaMUriAR~*7*GI3lF7wH7tOA1%g-W|$3wl|*kLYPvpl&II6U#(Sw z{O8%$0pQIljj@II7z2oSPsJcY=5lWjBO%Wfk_&r2PF+!FoQHV z!upfSU>v#OLVRa?Qs$9D+*uvzz9?^Ne6;Fp>xdJI?=Gwc<6gaELU2>K4<(+pduU1{ zb)_*AHNvVJf<>0&06@vsRrzb)qvfe5m4s;163a7$E1NNKEZ*~F_ui~9Um<~p6%Vx4 zn1$8Tt>iJA)p8FPrbzWQtFncJ6d?jbK$c$bv^gwR+kk1KD%bh)4+*jSlTbeh-HLR!x@iNP?? zQM{*0mwi%UyWu52git(j$hMqMm^wQF|sY@+ zml|R;*QOyky0H|k>^jZT`6|*W_c%iWzDXuj5X|SkSv!T+wuMo!X(_koLaLaG9d=V| zHXVd{l0Zgfe%8ZuOv@xDf}{+6Nj@}RImp`z9RiSjV03E*Jl<6@_gA8Z1-qr0p4Xj7 zIR3Do2vn{nI1DuU*qri|I;r@|9NQKk7si7ZxoIw@eM0_f>VxpXh9VeH7eOA7;M7S~ zzt-KJnkwL_qLAl)B4GtQ!wY(eg}PG-7c50IHxwUo*bIYeF=tnYXKf1RJ0l}Aq+VoZ zA>Dw?9Y_)vNFzpDIv}jl zLO#G%Xtw5`Q@|6`1`41Y#D4?AUQoPz{N8r4PJ;(SPhlU&_MuROrRqNtF&!gUCZ*SH3_R-qR-!*Ls@aNiF=5)s%X~2ae26ZRFQvlv zSRtcB6NWkkKp;{NiiM|#qbjQ|n-`}^RxN&k3LGpCXJ?=zTWC(=HrYOlsBxV#-+kf_ zU+78P;p_64Z-*UTy+PG})r1LbXUq4JFha|^3g&?V6LS2&XEH%C+^_~&rf#Jn$-N1} zM1s*n6snX;kT`KzrHO=Xy_la6(;>Z7BrwkyMJ=aHGGrT%ISQiz0t6&+-cvC#?%O)o z!XOdsEK2w!KYecWp*X2!M+T)rKz*isx8rmomD|aqyY0Wy&r6t&?e9 zf>h>aj}Ry+m2@%P;}@tii;imPz0sMUM*1k=$^&+y4YFM)bg6B?6*WiP$_rriDD6n~ zTR8kXx2&KU>j5?fNQuFs9Fx@KlHmwlfvM#!ycQB2rpu zXhJ#Lqpl?k=awmytdcBbF$l}rSU(Wr+4r3JgB8xa`U1_e2%#xHAaG&tpGwr1k{S*% zdKBtwSS~?>6rv-QuvtOF<3K0GQuz-(q7cKKo2?O^{lODdE zo_bW^Ei=|iJv+d=vk7PhZr2XxI<=k;U3rFrObduuWy++fY>zvs4v38@#SDPk7Y=L9BFEcPDW+k-Y-!` zE@Tiqj#YD7IvsFAvh;{u5gW#??7-ll{c8pq!=7)RYoqIQ`-V<-eH5fXh53h%u}GCO zlVZ3HR5}@esXK|W>JDvHi@gWkmse1AJy#4raWOtC;RN=?sss2}X)@|kn?nmoOUps! zBRlp)gS}wt81~ck_Z*wkn7`dJqIj!a>M>F!RM0z~;QiW)*r?j8>wvx`Ilp zV>D zN+p1_*)A~n5|hCCgd24Q6|-=?1JblmS9JWfRF!h~EU+o!LKcrrvUpdWfCHPE0oc9? z4D=c^2$2Iw^t$%>$%3%EmQ+?cWCuqzgG^_2lcOIzxXwc$qgzKx+^y!1yL(Aw6lP{- z(mvop%VDB-BH0#FlM6a75#Z42J@Nqrhh`Tfw9+6rqJuutbH6G5@ysqYUkg|+E08`h z(Sc$EsFx9?K0Qlaf@q#@5v(B#@hz9aLb47jTR}8hVafx~f)!r6ww+-;8?LM*yVVX} ze({H^qf`}7gOtm1cdjj`#k__MXT2`rE>vYaG1J)CB=M%M<`aOkt zlQ`CKs4Oo?OE_V73tN7>hcCh48uGk8^}Qby9McnveY|Amvym$^q-EbO-QeOmu#C8h z7A25chnrjFNbe>g?jI}8__n_KuNN%B)88T6quYOed8hv10YHWXa^mi4jV5BLk7wSI0Zu)tl^vHZLilFrwL?W zg)*^1l-#?cB$T*T+RFhH!Nc-sS5+k7AWhFJ5SWGtIiMja3^Myxc zWQ_3)gkvKlfvA>)58rYZ(Y9fZhP21$FS6+V5deXaD?<<*2?&u9x0LkIG|FzN1_z6~ zL$;G1(!%l>uUo~5)uGvJo{yz{-ecFNE;YOib`K0#V{aN}`UQIzTPRpqu+@;lay32J z5}#?gQvnh4HDsNPf{+!u!;EbX^-tGHN>!30{DRYxmj*mo?5gCa3RzO7HZn2 zw|HOg0P^hg=eWY4IGgdJk}kg0rJ*Nr0xlLiR#Mw37u@IF{MFxV>v+r7?dSKMyY0Jo z`T6s&E`Im@ckSDn_XQDw`u<=C>Pu$^F=S{!gD8SE8(EuM%S}{%A|{{PkiIRT((LP)eaKNjC%*u4Bcdx73LE?`{DZh8{`u z$gq@^nv2LN;0U|EEG!y$MNj2kPatR4Gkj}E>C&+Ra zZj-W0qIZ1vYE(h=KcEyVZ9E4YL1Uav@;|Sp(uCtz3Bet#!%Vb}{qh!j!X+>@mPt9t}qA)P@@-AUZ2J+*Qg$!Tddl54XitHW}48n%=lzRM-QE$s^exu$DW!gYl*+8qz z;e`(vQMys2FBfg+ss$8vhp&FyJnc&|IzwP~-SPn#fKmZag&?tjCQSP^m0fM`#0Hr} z4Ji+Ihh##Lov5~Ez5~RFl*8RKvH;PcTzmw<`Fz!K(ZvCDv9}U_O%j(RjyY!CcbMU>Y>(Byd+ZV8ma*GCI%pv3=7Vm;`0KiA@r$=pe zzKL=cmz4vg7#VOmK_6P7=g}z0Q+Oi0NFW1>W;WmeML@d0Ko=wuJBD!@7=gkf3t%K7 z3{re?2SoU+Wofn+DvsGNVkB>xC6=(R#9%T%GFDXstpbnMnSV*&JhUNLldS`DCV7(R zxVxl^f)nd)yCvx34FmB&Jfw&ShX{B;!5pwCCqm(a5dI?(`|+Q=+MXXn-&nfLy*Y2h?V)1mnanLBW=2{Q#YVqSWY9 zE6lzeT=6Az36@#WJt;LF(D8*}5O{tJ5gk&VJkih)?vsmuan}jpU=~`lMXo};5nCdu zjrTEARcPp9*mmhXv|zqP=Kx&;rQqe2_f)P(L=21RWuA%xfG>8bTOAR>7GHx^dXE+{ z_EYYFP(V%Vst_%62?X)CiraCqrR+Mo0z+`fN+M%n$5~Jcfl86=)x`q+9F%?}qrQY& zOBm_NQ+#XR)#~R!>XK*;6*^8IVrU~7$T0cwU+W`SCV_BkgKv^ zEc_egKFSKo3lV8OL_yC?mGv~o)@mDdR5L+l{oFKy#o)ibMjwOR(as>*A9;cmvJs}C zXM4o+%(}Hq&};Ni7^5tZq0qOrVB#0nZ4KHUd~CWag!3<&vS(B#(31^(yMDqt{VIVK zmDK4`vrqGJ$RgC0MdT*QR#myBS=8Ou5MLHS0dy%G>ebI4WS={RLL8M?wl6+MFC87| zmUS7+NshNS1OaCee9$RD5EJc@(;-p{MudEP$X2d77RVdG= za6IwKwuY=D<7{>V*H}A|sDZ-KrUI3EaI7W?VRcbX8((VULN378mr zBV^G(yG=liTaS^F#L>iY#j)_4F`tL#-^46zowwGN)kKlz%EGFvzpgo$y$KI zPq@7ZE_Ua~cO%mMT-I*ohn8U(b@o||`G9!n!%FGB&MIaSJRrzlNf0j{`9s11eC80A z3zR-Z)ln8yI`b%eFV2BbL>$MWX&Vayq@mVIKp+M8VYO!~5}1G#4=M!*Uv}e(BbC_- zAubhzcVg;c%|@cd4X(5|g!1(;l<#!=$p*^CZMP=bt94+40$~iv)xJ6w_7jk$ zc8umA6FCDvuLE39qz;W!H{0qISMd9?<&_wGs7kL;x*i z6>#*?+<_)Vyt_f{nbMR#Kz1ZvOc%xm1AurTa3ZM4Mwzw{u&{uXQYC^jDk?JsYO@Gk z{$Q{|?#nVsFc}+Rk?e=E3N~i&G%lO*+I?yv=sUh|h(i>B zsDhSIkQik45x#~FfQJE@1fj*<&i8P^n_b+Q+tYG=np|pit;e5TeciXO-rT+Ktnt48 zl!-6zaFT>$B3w}_NGKtSzo7qgyvQ22L;h16dXIYYRy*@oOc7!nMg=)Chqe2(iG+;I zJwyP}1|7ZH9T2eE)3%HV2_vwlkd%s~H-{McQecYPGitdZT!e8UEz*(UZf{>3&$oO* zxG#su*Tcu}z9<12kTLS=2{O>ON(X>%x7zpir|w)PSx*wi1LGdQ(@w?vRXwG z`o1OfajeZy;Mrh`kR+$4w@63D9|(Y+nYc6O&aV5h_MFd9qbuR& zPZkddrk!cBfvLdF9jJJhw>@bAgY_qYof9g=X4ZbTf^`(R03}fGI${Juf?9<%JudLq zX`Z{)QlAKHq+GY2l`e7ESzKscP1)4_Dl z)vipE3mKX`gnawKg~Uy8gqlU75Muq?RdLC8>)}1i-S~Z^Y3Hug#R9yii#R8c{VF1g zZ!l+=6%ppE^QN=dxIk`f(lYt+)>+FHeTsF&xFwYhO@jWF1F;L(a(wm*Y)PI8)7M-s z!I)oZ&9#RH`^od6x?uc5!>d?_1!Qg#x)?!>F&ZH6YGpi=w-V-hfuaHk!0RbI+Rp3f8oXg4W+1}vwA#w@^r zAwn0k9j|IhOi5ySfu1G9JB|`kOP^>sX3kBc_nMVX;=}z;I7cW*ItbII8;gkFyLCo2 z03x7K=XppR(WLD}=RlQ0fvIiJlF&_hgzmfYPnin1h>Kl13G$trX1aJU1#cFCQfUtP zVV9m9vDBj**G>(jY~oaJ?W-Z8+E%Nh;8{TMHnvPWDk;2BKU@0qL&`8iu0`e);EPBB z>fw|FHZ>C#nK^ZdGWI!OlPi|WIga;BU?XAi--Jj? z37(w9Z78KI@VIDDJfeScLR$_Zy*L&L=ga0UQE3D7BFeU&%(679mbeusu}4(A)VM2h z*cF@{gcVM5Ae_Qlf?)7S!sC~VRIfGd8u!0j;z(h!Fhj zpL6;Qa4ojLOB<+C0|S|Xa|m0oAEb=DK;=*64lSdO%%;Wlm$b=kR0+A>S_|wHC1OF@ zgb>9EhT+Cx`T&L}V<%z2mvHLZO)>0M`_VUarWy!j(m?^7oo1mCa_vhFv+1&qejifuUO$WNbHz8aOtcUQ@ zC#Dp3Qw3TmGu#IJ;o)029pX|dlf z@#3T7M3~2mg5Izl*xI$(OoAahY&vaoU^%Df(H2yooFf?(oZB3V9rA}}8{{8^k*I>5 zkRYgXhkQITg_X$c73^;%U|NO_y~*v(G3v)|jO9<>H&~K1Y?g5*A`gWVoPNP-e3BgW zD0mZL;rLIyd{I)D8VoTBqO<6x#jYkH~?tf|pHEzF|3Bu7)Ov(qz56&2z^^z?;K z3xywFTNhBd0YlG+_KN%@JBR|_1vT8(33_KUs*Gb{_&zuw0%G$<9IX%1G9khiAmg&> zGo0XF%w~zZM^rK3$OwY&BWb7$;A4RRLRnx?vGYLfssM(`2^^h2Iw5MF7-1%^h^sii z4znZSU`B<}+-VQout4M8;Wp+Z*0%S@cu0Qy4T7XdEf-AS#Bs76H_$!G=8B!9M28rzoTgmM zS_!}$U?dzkrHny?ox~t>QBoD<*O79Cd9jQVWbk>Ov`f~TV&F67B<{;@ZS4|K)oy|8 zVAsi;OCD;$h){_s2lJt3!iLjQQ0sH07V--cn8xNghMSw|#l*!O_F!Ntn>$K6W)-eH zKaP4PRa93`?p5PW9b_^V02rvW0O1A`AA#od^eM)xmPcNJ?yz#Y0DFG4YEIbX#G6p7 zS;-3oqvJt}AQ}%a72wX|a|sm5vLmjIgkKipO_nTRLwpokY_ndlv#}Pw3Q3jw!VeO0 zu$oZR8Gehi9u)%QdBrHs&smZamBSt)GtmRba?tFqjPhXZ3b(K@8Lx#nGQ)-ds`Ny= zR|#>t`(ASJ#P!fmE_IH1q>ShYLj~8tX0$59@|x8`nJ=COczQsLI2HAD4FfXsPWrJ11GMHSMPik#Wz0kG~thH~R1J;X9pK}SGceWU}#Q8>bT!|y2Q zlrs@pT49Bb4Go_zJ$p!lT5Sfrf#7oaEl}y94WUu4Sw>DT zZJg;H0JACx?yVh_Q4~64KDnyoU&IoYE>Lqc7m=w9-0(hBWFGNajWfHor4XbmGbMp#ej7e-FK^4uRA)6q^#gQWtv5nK|YaoA`_ zf@m=m!Q0C^^`^8SJ;;qAjn9sv&!kI)9dcQB^D3g^o^0`PWazrj z+fW!E><1qUqvDQuQJZWqCf`bN8_*&O5=q-{>3t9gEaS6de8Vy@P++Fn0SSh~`->6? z(g;e}9}kfboX85=bF36q5wYjDne-hjA#7Cn1Qj?21fkQc+y_0hH!&+~kt;h4XuV&h z>Qd5G6ptsO@)&2Fe4t8CjZR1l(e<}DQAP23ywPs)j*0V%*#QLwsz{JTRVGq(V)OUF zKY2ofaxju>^pSMN_Gf*4Cn?ogiL_JNU2%LB1c6nr0MFEk?sC6wc6eSQBLxlofKxqleLC<|PZ{1oorGOR8$N4B9XkuM<)-f_v}+yWsNV5>zDhN+iyhZq7U5s>kQBwqo*>yT)=BW8DWm}OxQ z7e!ckk?=thIOz!y(tIy)7$jUt*x4m+TbpyZ@t`;eDw(+sEbkO8NL9<5-yYrS=!Ff? z5~ofR=-3$^n!ZS82NOYQaQvHCb)6>S6vW*NFw2Da*hn)iLlzHvogq{t)|~kGc8BXB z^)mbim5OrW;w~YggU&(Edci7<5*t_ut$EgCGnlY#2C)ttmNHJ&k6UMGp3hkY2Rr$t zZ+Z3*!ND01X-M?-`ucTM03b|5R0@^uBI$CVJpSyX8qoC5AxD}eMFWo|1B(tp(Rs+) zr=JM&QA{>ws8<5(*27p*7Lj+N{rN0)Y{IO$tc}IT(pLO%g;)ejJ#7u-zZqOR09=qw z&rZ^V548{EkwUXWP*mB!T}h;;O!OaF0?>2lXZT7lvZo18brmHK?Y~a#*r3X0BR9-m zys(0|C0vsNH&_UPC-@5E?iLcOZqd#TkPlfy?c6mvqy+llZbEr}6(lbqhYtktFGenM zAg27HEA555>yv`ju?vlJ92%V{QHauvQyQhC-DP6!Nwfs)IhJ+5u9-Zl$O8%pI&=`P7>UqLQ0iG~gY-oWy^3cS}P1#@CNx!_g; z?gq&>8&&ifY)-dVtIxYTTKX)a#5dDyJC>b`>c0VqXzEO%7o|vZg%nm06+NA^5LQqi zY1rvYj8dm1gM>R095$lq=Y^S_i)bzVf$JWkD92ard z;N#M023^VIAS_i$=0su{h+uq=Q2~y|=c)qVZQZnYbX&_D(J!E*q_!Y{6K7tx39d~> zr$AzoI<92~N>y>mFBtifHWEXxNo|IiK;jo%G#gWs3DFO$p2C_a`r_`aC%G+E8nE$! znkx(3wbsktj3oh-!gxgJtRRN&!J4og0&rO%DPWx1fhPLyYl@Bz&siRD%SVo+!1COT z_H9>xZB;McE^BpsxUB)6lc&oB3{!qA8IiWI2n2`13^t06#w-P5A2S@0sM6mCF!p&h z%DVUC`>qZB-<|i4et!IW^WOK@p8WOqI_=~9l1jhjB&4eO_ut+*@$1jW9z1`$Z|}R; zjr;5KulM}**In^KoZw3Af!I5)ZB1$ze=S$`-@}m8!rx#^mfft@D1#wf?umrm8YCSEl_&2Qy}GiZgnPh5}l#j?vMRMXd@SDx+ zh_kjs#bl1|M_?_mJd-iAeTPOpfk|IP1NIFyIuyPRnPmBGep`1CQOZTr03-+5iG=45 z5}Xwkk&3V@Kz zMn=R6Y{H+$hn-iigW)25`7p{}d&%k$oGiKP&i5WZYN!>Fz?Ka zrc?|UUq?-MhSvaj^xlIfwHwc78JKd+cfjB`48jDV_m}AeCX$!n8<@gTqkNSH$yCje z328!#65LP##k`i64y>Nez^P<0!9zK>=hK$*3maoZvPPiNf}pxHi6;PLT}<7r;Yre9 zUD1$9dP5;B(gL6ieR?vqUE+O8u$wli&M21CiKm-)R=wP`$KHIh;j5w@R2|$#GOFl@i7e6(usJ>&VkQGa8_C`AqUchyXxpl#*IFldrVkOFTLzY=suHCxsPShE;yMBYx*bs2_#dyQQ1QH-dMYMRt7tX2PHPZOU!tq9+=X9l{ zIZXo0mheVIz`PTGQS_&hIb$MH3(wHf$;>lo(5gk=A)ShH9-Imm#nMZzMBg3_ZL}PN zTA1K@;^j%xK%E#{eu?+Hu}nPAwrC~qDy7M~728Z;t-P}{AYW2-G5K=>Oc&8gAk7F} zymx-U#a!J+Bqpw$^1!qR6B}w4=%VyKQxJF_dl$1ne4WZ7Vv*{QCTWz3SB%EF<16kX z=oZZbfe(F=TPwjn+qYNCBxzA6L^;}5>Ik%J_84kmQs$%hV%xynUu)-$JDnJ$QJr5Y zDN4$n1V@{pj9l)BDj$vjMIbsJd!oBBJBn9Jf~k>8GHj3;GGlEv`?G?QNG{eaCLb99 zYD==A#?YKm23SF?TH5NNb-kkT-X3bWJI5 zLgz*p;jo8?ed3kTVVY%08XRm?14HXymS+6S8YBts!7)i9>X8EywZPrsL&s@DGnxsr zWcXprNdreH5Y4bN|KtK`qLZ>NJ zrI}COnEQ^|IvCM49dozTeJXPlv% zO57q1DFj~_7=;Mcg5zG@o_f#|T9Tex*9VSX8ATY!T)#!MB#x8C+p+x^5budlJaBcE zqlF#p!j5`Srej6nhT}XF>X?Z$ZCbmUKP^tyhBB&wjEd>zC`+tgMI%Up2Y6M}=#YRf zAv&MPLKXscEvqCo))5BAbiCJid@uvcwrn??Ds(uOJ{2r7IU2)POa_}b)ywe?R3B$$ zXdYc*TJx{C<9*8^Bs2=}QdpeLBG}4^GpBoEG=crJ5xL%Kfwmq4#ZDkJRWPSi=oup2 zu@5VYyWT*(5Qcb!WP-;u!hYkKHsNH}laiU4j6k11B4E{kWq)mhw!cOQtHVh2tY|pA zIBG>tVZ7<}cVC`O1kyyb!-*y5t`wf7=2ee%>x6?P`!8gDa{W$!zbC2jTITj(+TETGbS@SBci@~j0XcqfS;4uL1a1$ zmW9%ul|`U95T?LYg^p(0G5Wj&oDwGNBp9Q7m zECRhZ(b5zvSiW&0q$0_;HS|k`V%NC19T4eURr}cnbQ(U1#%d$IB_CTAH(}tmun-=V zNg*XndLUXbmjcak)?(Z%ZpJ)5m&8b|CfTREZl#w2;YacXpcruNGt6F@2$_iuw8P?v z-))D76bU$EIN}(`v%Jl|U<-2L7`!3KLTX%gIy92YsprD1jRNS^LB1gypfqDwt2^<% z&M(-Gkbwu0;MEGxm^@G|lFU(I5CE-$1h;}L4aSS;q_&~#Z7a+M9Z1x1iJE%kBK2W{ z&rdveW=4#cgoQDYaib*rvzuU}F}cEUOBjYwAVjbz%T#VnaJ=BXuFxR|j%!mg^YJ@@ zfR7l}GoKSZ72;5VXST1#_Q?G@wgqjQ>@?Xs;j08o;6+7|K(GjYW6OL}s#9($H3HxSHjT3=<4E0Z8o zZwo72JS{QZbKx8X7BW%qJfN|Yi~?z2H$+Y%LsKnSmE*V)do={6&$E|{-tV!oqvgb4 zIkM@3w?q^Osykz4!d)=S@DS-}*=36>E@NOr5g=2mpmu41aAe9DFob!Kgj&oG#n}d+ zEv{=ieYpIZ{`#QprM7_zZ+?eOhVWG4jD;)RRSSr~7(|UoGam7L;WfS3oh8<0Ayli^ ztdmnO493WcG-+oSux=_`_(6!^$~;n((vUROYNR!@%o2@7H9aA8GgjV5^O7(rcV zYC;a>wz8agVwnIVL#Be*Mv!QwmAiemTk-)2*VwU;4K^MHnk`|jNj>OfOX$&@q>x0# zH)4y%&@2^+P3^{IcMwb?(YIy=p)wT zEofr%TTi>s?c#(f6*A%mY-J5M+{Ge@6=06xHmbrG>~Ym0P%PB&rvgMEV2i=t1m4&~ zVlEh(OU6r-v_Dwt*_mEf!rr#q%}WVkSsr6c8`1Cyac~32wO%{ZD#N|u)B=?wFM-k6 zd0n`ApoC7XzkR3|T;|-6Yh#7gS3+(CYMwe0$vh_XiQBFxR6*934BP72fV$6ljwozspxhIOUfu>oZW00}3`@urtZ z6DEl0iF2J3;zPzz%dQ;yq@v0QK}U7F%vnMSC!5^NB$TT(s)7Vq$(|t^!TZKUEP8z} zb88<3DweyknapjXA#m>Hyb(62(ln{N5J&d$-{R}GITnS;t{h$v@Cwjmo$FqBV!agBc775$kdXjR z&S1d^-jwPwlHwsfm-Y}%kAXW0m_yrGh#+Ga*+!x-wUX>vM~7RtFX;mNjDY?Zvw^h) zG>!*tA;qP1-ltE<2(jnB$8-?5bpR3}o6q6JMSlPS!2}3hNs4lyXI)VB`!^j;3`oq} z*mBbLNwYOP8t-|gZ&C^2P`pGH0;tb&dk8Vh73%=}nD{=%C4gJPKxM5XAvBTE20((O zHzEg%VZVWq%i>G(obg4T=sfWcwfZ`=g+^6TLK;AzI+t}z#N&jbLJH=VOBfIz zO@=;No3x)?GbJBojr=qMrgiXxgpgto1q&%55KzBS(+KxwF~ATUfqert^*kWGtwsqU zJhrCW)}rLJpfzdngeQ#nxL&>{6%-Ggo`QLR2GS6@EmLs%YCgTeN^Pn_Y*LzCit?V+ z(n9^Ao(h8-shM4O1}$TMA@%kXxn5ys0%6Cpq(M;@a|aRa`Pgw`$OtGn+uW}EpeO76R_b|S0{tux`{`(WK0tV5va^MqiEt*oniK`Lmzo5ot;9saX1(t zl66m_UuMPIZ@tM!o80*6k|$xP zP`@9e;AJK^REwQXubdIrTU+XZ48#KGn|9_S#T^#X+%C;X)^{yV*VXhGKy^iyKp@n$ zvxK-4q40ov8kt^et9M>DV1vbp*?`hcnc1DEWK@K{mnKGW`d%S;=!!JupiuH)HOke5xMbz`0RWsPk~L>!5M2kbU9*H-G_Xfws%9!Ue~^!|_)A3)o+E?h2zAp#_oyY!(s zA4Q|H6k6k$Lc#wv%)y(90pKC>(vO#D*tuO95A%gA^@Ym71y75Xx( zu>8d%bRLP19|3#_BhgCU-SY0`#6;~|3M8ah+chdRj;`o1pk!9(DgL@*Dg@&-%LZ;| z1R{H1V1njo?970w?Qw=n?_vWoYzwp0@^LU zX85Xiup&u>yCav8zK@Mxp5oOGK^YEMTTexi2Wt@bwz>DOPKJ?0=7#JxHiN< z#~Ff-jD}Z=fzL_uAfLS@tj_);?rTxv2ca*sNH;7hsF+>OhS%u;0MHXS?79~#FMJ1h zr3hkcl5NuA3Cxj+*JK<^p)zQ^_;PHy)Kw3F?w3vJ%K|(C(3VQ?qB1mSMVUNEDp>A) zyc|wBU@xo0B%|wtF}lK{!({TU2p2OPKS0G>a8r{y5?mmwBS;b;n*0Zn}zi#rq@>(B%mB6@m$3*9>gsPV*is6*>i8(yY`$y=EW zpaoUf!&{bVbaS1eIpqFPyWV z+9YObNkt^aS8-m55%$`A4k;YUR7AFWB!4E*)#aTC1W#EDSUVl0c!BVLn$LAF&D$h_I+o1;Yz1 z5NN@k5=qM#q;&080rr_(;B06(LGT(?ZA7Gz0Y1$kPAQksDenZBIfSaJN`QjT0mfBv z^jrPJU7pMZOQ7w!Wt8q%&JK}os5XhfRS=pPX^1^Iq$>gfMg@BDnl?VQ`6YhfWFS3Z zacx$}1&FL^z22wHd8G-iKxF1NMWt}>5CP4A4ngWta73ePZ z{Y`JSR0X{&E6sSix#D7W(u&_z5#6HO#R8@$=m&xMJy2H2Skv?ZQ^1!3ExkyLqo&p2 zCTd-TGm|wLOK5D^L0;WKDFPZLu6Aa~QZf|7^078kN|FuJG$wi=I6$wF!h(3XOaJx?u z&h#*Xp%E?vEmH&47gxkbdXBMDH|T^i&|WkVeZLXd(+|`|eR%k+VgQNb$%e6E#8YE? zmHH;k81rxi3ai-w!GZ~iCCdT>{OBalkPgw`nF&y4^_k`rKg1!SDIuNF( z2OM%oCn_<>25*ztS1VV2Vnv~=H$XVRK@lD#0U}Q#qH3R*qS@|KsY8*5g>mjnsvCsD z0j{~72P4aRF&MPCFLBk)1T z)o0Qo$I2uKA-Ma_&H_3v0&pO5K$76scL}i~EFsq*s4{?h_+hO*7y^2p(K~jm`)rAnPU+_v7Uy9x$$u9UWe8ls>a6Y@sD_}+ zNryr`jmm3d>@{5bDgLk*2Rww6SP_xNK}kU_yxXI(aoKG^Z&#J0rdg z#4qiTCAL&y9+iE@oe$h-sChf2kmmVUIHcmBZjj(-*S=L+nnObujJ}~IHDQ$y;tUYj z9`lDhhdUe0`uQnDaWPfc_}vWgLSM!OYe5awR3{!Lk4Er3v4qpV8?Onetw@B~MqQMH zvQ-({7i-gmV@a{(z!a}Zz16R&)y>F90*MnXG3B=Jf3DyEkJ;@Y~jXoinT zesy6g{3D~fpDQ>M3nLUl+z8W|>OG6oLXQLnmuMHv%Xks#(I#^8n<`g|YzanBcZF9tCZ-`Msj7BXd@CtT{GO>I?;>`tY;Z0txJ^ zAYE{70#okBVDVdY_{AKZ2HacOR)Lh9f2j|&=}{1`s$NKyF8+NrQz1RjDX4*nHoKQD zG1I`Ck-eZW3nI`MDu*3>Zl5Uz2ty+}m4OA68_&HQqlsxyymWF?m3&*y1egO7K+eRk zgR%t5x$PfyCi(JC(N7=4%dIfVa(LLMsH`J#Lr|IF!eHvmdmta86PYkN9n;~l#T`7Ha>|3#*+6PM$!pbg1-H{9+{sG=nm0O#q#B2X z2g9jb2-a04FbPXTl`AF_mX%Li^_I@2;|74z3~Q0CZXGZ_YOiVHdQslxIAjNz#_j(5 zERJm}IRkP{f%gmtm9oX>k6sv;SrwmaPp3S>eZle%Pq)>WXX^KCC{ zk16UZz}G;G<;{waA8aZ06)6iI#&2+2t^tu_$AF{T zR19WwGw)4>3kFf$91Gr7&Gr~Dh|U(qSLbDAtcsaJ zsA1M60*Als@>38M1hnN^akBO%t=@&RV5H4oX4 z;o+0vdnIyPCD2{(wX(UHw(^;-#vG0WvToE_sN)^^xCc+Kd&nxLE;FV~74DUUHVL3L zI(Z+k#@3Gl$!oJwBJb@>PK(z0Kwt_pK zOEsw)#|O~1X0%>-78i*Nj$^@VqFo^2(e~$;Wt>=z{?(igTZ#_eeEZvvee~^{ao_Xr zPu91s#rK{kZn^UJo%bJ{ZMIrqmy^IXNLQCI%7Bg}jpnw51f5+JvtN)j?8s<9^;U6J zrjfqf$J=P86S~+WU0Av?b^*I`Aq^O~rn7;X2^ovLmxDaGLT`Zt6ha;m2Z_cVjEeB7 zTq_ZK;=D`0EA0p#^civiPrF4VE&)u~u){DBkWQ(a#YDKmn<8Cg0Yi(`oF>`lRQNmv z(8dBGQ9x^+-cJjLrIJ;LlYCO+8C{5_L~{1`>+{~Be&YO#C`+OQ7y&BIRIEXS1B{JH z$KbSDN=R14Wt1J&6^P;Qcp-X+JDhWo@j7MSeop|0PPJx=cX%ttr-wu$06}n!qV)D1 zhlhIy4R!@E1}aY>iaWlOI?@vSxh{!ut7Jiex)W-MU=%W=BJq(N1@N+9NNvOqopF6- zAZKe7SFWkblH^wmZVyT7D9~yvh%W~2H&kIP)+U^Q&Y+lOkQ)qQ2;Ddf-;q*waTrwE zw6RSrP(_F!SO!EE$i-;8K90Wu+(;Tdql$`#Fcb!(9S1&2lt9XeAaa6zypwD9%BinX zmR_zUAy8R)(h#f+Q!nEx%gpubfLLAmeb>sEJ);yeH}|I^%`(aA>V2vdc__|QJXB^q zJoIE?X`*aU7cKz@ z@k&Y67tcaK08@G1p&rV$vSz3d-w-%ayNUc|v{~UTKTtq`b&xlpb3x6`XG0qlbpDQk zYLUyv3y9(j37NOU=IPK;9%9lcJ|$jttXWeE0RW76OaTL}q!B>}5a59VxiZ>f6$UG* zxQ(*Aszs2)BIfbdvM#ty<&-v#H$u^uIl%0oAG3`WHB8(aHG&C7xLP7aBtwT3NbWm| zplMe7ixygMq(ddgev%@Q19eC9?g+vxCpin$st?AH`Vb6(c|Du=nXiVZ0J69%{ADW@h)pn|JTkEWGHO{ANfd%s-ibEXnh z;a0CY6h3>cwmVM?EPXNaTU?RuUt8E?*|7o{ifH))U~uGd#AN=iE1=J+h&~Y!JRJl0 zEnlGTL>%74{`kwzp8|_EQOc?%4o;bY;Ib?N3|lh0*Qa^JMApM;8_+v|&;?57si3Zd z(c{a(>K#fdBqLFq_?uS&p@LawqZmxq+nIi;BcZ66X269dTz7_fK=4L)rXDVRor|#L zz-SyW?aWE3m_#5?d}%i*J`3?Eh2ean??;~q7`WF~);|NA454_gk;~k9JL)QhL@ULq zD=L@Uge%B|)Hz(D{&*r#ewEt5*98d(K~$4Jx}C?e{1?fbn9HoQ%-7y~LA2>3XA}jq z_NLNXM+QZCV&$d)>ZA!$m+!YN>STqW*5%P8Uy9dN)q=8adn3yIHOB14xO#FWoo`|$ zWk#45NoRsHrSaXTah~Q(5eYA*qF3M$6I_iaZ${e4Fg83GP_N3dPztqRpwN7g#^AXo z6>BCXH(-6mqIl0{G8LVVhwm z1s+azH@rU8HRujtE~uDO#=A~DDIbGZ5AS%8h$LIbiP#t>_CAK@>k&V7wgBar2`bZ` zWeo7CxZ#Kp_VDPv_)P``-|GqxG*RIPvS+h=X|igwwL>$YRp!vC{+R4H4xljxM zf@qbLiTofk!5MBU4ZH{98d#Mj|52u2l8XV|Q&?YAKu+XGyY zOuYM~{zB;&R(cR4P(lgIA69X;Y^;L-gK>$lSWLk@svIPauVq>kKL-vW5U_&S`%0W& zFW5v%7;k{#r8@{C?(Sg4wc|BovFUQa+fD6g`=I2XsUu^1Rgl028G<=#5RVkg9-L zeZ80g5WV}#wn@a6_Cw#bsCeMmOnip#l@P+C7qJP+nfjrS2d$JBiO3smPRokNBaL!@ z5Tqhmo)2}L6PYRfz?6PvL-#}aALf|}3{SbNEv`QZ_MQ!HCwV8XX!OWsGOJ-{Wl--g zdJ^j~gy64w^H>~QUE?k~nivnwOVc>>oDxKB!l>{~6@E9q>%iY1HtB{+goXfg{OF|oEX4eo$2@G#0uEeOucsy5Wkue>))3i4f^cupCiE zR39Ruz(i8+Z1XxkW|}FP0Dz6HnR1_n$3z20O6sCR>Qtpfb^FGmO^CgREQMXy=@Hd= z;_@V+t=G-(KX;t#-xJNhH-9dS<;L&lzhB+IIqRA2_n>Me`RIzz6i}E7oMXy}b&bgr z)aLM9tI=L5Xz~zSMC~ox)b`s}?sru&K*2(gCIu>SLf>&OEBPJ>9WlvXb4F6vd2Ua5=S7!qMq`^f<1N*F5P7F~n!QdscXg|$UmRO0)O zfxfjVBZ@ZqTn%%;mJxPFcKQ9>?24v~4*PAKG`Lq;kVR@57$+kMUELHaHHjc5@twZ6 za_Y3zwn@5+P_ck|CmSwz8EL>)IK7>*>np{;#`uDG*nVQ~S@GkW(z|GLt~&!w3-KV} zg9g^IfXx&%&WNl_SA|D`FE}KJA?-mi?Upi2ncLSFs+o*@if>7C+GZ3Yg7v4c83~X< z;F4hreU#o(ZJ0}UhP&fc0Yms{LI~~7g{7<5w0kMT3*jAJVWKU0qi_}ubiIMFp&y(; zaJ0#UIj)o95)=ExXid>IYysPXA(@_^Ja$)-_N6l8d!RW{2{?Kp6r_CpgQ}`7b~wVg zx$hr<;di1qwj0jE}xh;%C`p7WTc5l%H9IxAK3PzFS;@*yZ8 zDGW)CljxCorN%^#Yx>LJ;SB+23oJ;ob}xYrM_KA(ik&`z0r+t^Z9e<5!e!gq1fsKx zt-3k`;7FomP}-xY6*KOi0hNz7V9bP=n`YG!-sqBF%mv)<`i|mQBNsu zmGla7!`n@LG&;MBC0Ow2Hak9XCj7@-^n4n1i1}0-1k1AyW5$!9Tw>Rox*J9;WDLZa zEAzL{Kh7@3RGXlRGLwc*o}M0+TPK(aM~BA&gF|9=1K|N2ZidHHBMZjYpt=SautXTY zcBCGeO&OQkAcKP{Kp7PD9j4&Cv!RXP6wSaZEo7;+J2;0;R8_v`#)SmVf%D+n{mMab zARGFj}6L&9sgzOM2BI7Wq)uM_I+$Su}a?<9UA;2f44otE#DdBYzu;Ctq zdt-K9=(F5Eu&O%zgk1vPWCY5rTtAUON(k_A*jw82g>!aK%NPUCumL8bwwke)&VcN& zWfnbqJe!&}1Iosvi6Lm^vzIOUxrTs>;=#&|?*+9-L!YiWMZi1a$_LhLc5S5U|WEwp9MjXM=a z3!;#luo62(`21kSV0`u}i?N5OffNa|hcd8kTgk_#4IC{JP2qVy!vaQX^=ME&IkyzU zK(A9xNy-6>H&O>Nr>@cBiUY7vA4h={{GBt$sBd^FyL-QduLG1wt z8y-~N%tF%xNK)uQM79cnJMtyGzPnI_Qc~V1^%Eb?0wDFWH?TssC@2@8q^Lp_ke|4r z6&Mc>V<%3L+Sc`DG_H?ek@7-Fii3P}!#Lhs_~X;RrR!X0XZIUk|9IoJzc;PA z{pSA^0Dbr0Cqa96phNNJci}ZC#LvL6)mMgE+Hl`9QCYBz-T8*=8`KTon(>B+u6McE zA&24zRA4eJ>$qzeWPu@}aS@>hmXa_;A@TGr5OBtw4$M~ENY7KsUk`S_LKm`P7p&{^ zcPUh}iKyYZ?>W&H8=(Ao)Ic+Qls*^>xL`83c8DkCT_`eAm=#3CWV6X~78O0{s%w1% zeL&VtBnBMt0c8x70&N?b(E9#bmzz7bz??sL^r`_ej&KIHWLP}&^#`Mq^t_lt&$E?I zVb{(f?HJzAX)0w7gCh4_C@|9@>Zg*KPLLaptZVj+F@(*D1go^N4}=s~1_f-mSVTq! z%!aacs{sz4(H*+{z%Eh@p!1=M;((|}W&6xkG{h7-Cz3kY34hYG4&#f zg_q1x)(LDRS2ScxEDx%xQJitu_;|nw8_z58=A-j3fa$%Eq3b!t#xBNQFpB846)CB7 zp;`&^C>)iDYg0?#i$H|{tV4N$Tpw8M7=*=kLf%K|~( z;nIXI7ZI>>3kcR^8B-!d5OrN~wm)pufi+~B0I?QJGHpS5NG=>Y$!s*^fmF2u1Q~$M zVmt$@5*AM5MyXbG|?-zZahK^<_sZ6&`K0xcD;~hF;az6j5u&?^iFhcH*vDC zG`*6aWBRS4BUu%XSFsT%-O1HdJ0Eh#GI45DG*RcJv-bB(R@&o=x= zo~xirAVpe`4Zw*JhQ#tm>3s-8il(V zirRb5%H0;_823cjNWXweQaI~#DfFyGT3cySskT@Lf!OH&0Wgg44gz3M6Fg|*wZI7w z#3*^VWIW{@phGcH3XSZ$kQULT&RB0IixcF-qjzBTv6VnN8XIx@4ks^~KrI>sZ} zh3{*k#(Gg`%=v}#VSF$qBO}Ecm(g2yu)KgJt&k+rMrhBn$1wp9rekgJ6O5KSVUl1Z zBrXnzf-QrMyOamC^myTfwRskx!a*hQm4>7kbrj+PJERzIwY2VQVq~Z?ta2wb0)+bo zSnzjp_B8Je(Zs@y3q#;yqQ$P*$0^n^Kb zGi*TLy8PAUbyaGLZ5z8_)Hi08S;bKTLwz!W335>a6!V5HZjdFc{NOE}*vMf6vx5W) z=f%l+VqJSCPMYsaM7nl>+R#cbCXmY0I`}7L9O`e}0m5L!La|KnHg=K{8fknpTb)|? z`mk?g3Isb{IV64QT_gDXmgI)7HNeYozd=9^qLVFfA+$p#!?jIu`g}J<%z>PVe9t|j zHeiV?8(3Ub#<9;_+qxs;`vEtZHCRRG$t_unV^%lq_HE2INr_CLJ2e0YF%A^kTN;*e z=Qo$d%&!m2*EOIm=TEyisA^YgEF*A%HoV5`r{7chObGCzNFh!i3Rzc&AyYj!EAfV- zmx_E&%TE(wsi1VlO^-<~o0J)f5#Uavpo(GIaMplvP>_iU4bDyFeYvzfKTTre0*@^m z;?})U*61cW#o{XQ3zZ#i(<1H1l^v{O1H<%$ovluKI8&OFpxSxM*-8px41%WoUrM^P zK_ju$=ZMyr+p!bPZVpQfkf20$O|5B_$_$51_iV`b zs8gqSp9X(S-`r^T0yBzlI2x{Czhp@v;FAQl5(Po&a*#<}^@{FOrxxhz^!tyE%{v;_ z0HV2#9HF>t6wn8i!{l(Jw^zch6t9R2$MlGAm9{dw*cl}06MeT7FHKtOVzD$P;7O*u@|^?_Qqg$NZSv6*{B#k zwYXN%645;%FNQ{pvsyqf!XFZZeQ$l7x_a4kb=t`x_5nHk7`bl;))gdV%0^s_JYgbn zmHn}!tEFkOoF75aF>9fn zXHj~&BkFX~Kd^T*2Eq4`u+YI)&QOO}XH{-v)o+oP#u_?FvN{akwI?lV+&~%}klk1y z^7Q{gL^Ew-67tl%_l1nO3~<*lk8CR(3`B~EB{LsQykgU$l*{be?>f*Sl8uiiUe`)# z8o5yj52}P6;`WC<6=`4*QuRz7>7eZR!?e`>gR442TLgQ6YnJFO*yCsgW+Xn~HpTO4 zgBg7tE-=_A5J9w&5ID1I6ZK$Bo-ePb#52<>lvXGiS7RlL-K{}n zHOuH6Cx(?n~-M` zvt<#K#2-bm1!Y6Ew)XLy$<=;h5uma2hJy#Wjy@xOM#bXnCnKJ-d`Ha*H1Z^5_t4%-8svkp=||sxs~OgVwxKWUp^k1rY9&} zo;_Vdse%3C4r+ilZF>+Q0MJz`CGlDzyOT-@LGZ_!US#59gx7q zLAeQLC8L;80tRHGL-?jZ)P<%uPI4-YX!%>b?Qh}G-*+I~gOK#v8A0r$ zgjp~hZ=;d%QIeX5F7UfN#10%*wDWW|`p_3+E$)g9oixI5AScE`tr7xJ07@mBT2Br#mR%OZ}nGQJnNXW^!i8V+SEnnAvs) z6Dkib^@dumWoOKn(7dJ1qCjBpO42%3fjPww!6d8>Azunq+2E}_1MVXPhz+c%lud(~ za2W$)@73xdvdDB06w7{Og1Md+w{u{E8qh!b(GSyz;LNAxqNAKfW|td%jk z_)gsiPU4ijbsM-5?#$I##O%L8NAg2N?kpbZPC&Q{zOM?V9vD9o1$!}S7c+;0JmN=j zj1SBa&KNTH2WrW3aga?`7a}0Lz;y5?LIO(yk4Nh=a2#9^p4PF`z*4h2G-)agqpDG|Q&(qY2)+ z7Kb^JgywY@1l$kzyMQ}@lHi6Fs}`n2ymS*d4qtb#D=3bRC;lSdVn)lP(+G3jWP}&gJXE132jP=KK_wJ$vhz`c??^zj0EM>} za?-Gqv9Zr!qi>(vok#Rz6FKs1;=;y!sY;;eSxQX?Rh4?SrQv(x8->5W-vLzEg?tqfy&b_%77F zA|V5WR8NMZf>7Zi?=W{bNvH#AsX^tF1_w(Az$XppGYq;#(3BDfDl--Mf>Xn(_KrCN zj0-m1uH;~Z7hB_qCpf9X;UEnW8hODz%Yb&m4zeMgV3?$0IhHDmL_5anij~gIgxO3% zpLh(X=N~VcxotM~z@aKZ!<(vmy6*LEL_4#EZ(FWPdz86zmg<*)w2+ZTGV*BJR$0)F zJAlavNWVvCC8Ft zRR#K3Y-=CQcA$k1+8{j{ewR3jMA{aE)o2;{3XA3yT%IWfX9l=gdW(K51G7eDEQGrp z@d(?JJOQGpZ%a2*a;Uun`RPIWv#B+UanfW6J!8}%92mE75hUwIQQf%DtnD0%+fa>E zwOr+P{^|`}CNzY5GF%inCM;|zz~`p=#FUjz<{}pyp4)kP3d|5mP{m|dQWoW~gSs)d zEy=-oq6^#l$@5^0WR;$ZS5%eo(M0mf{A?gq>UIS-r3v;dMN}IHl$xS&CH4-L%N6{! z*nHJ<=Y*d3B?col;Z^kb)90+K~?I<&8E%`L*6BS zRFUXOsD}rl$r<<=S_;>&zTs&QHA^qN&v?uLSA3BQZPwbR)e%-!)fX$6&UNvX_KUS` zAXc~D;pPCJI}~KYqY@*U*RJcpEXN4?!AqQsRGE4ctoPc5M{jE)q*r_;Jya>O*TgsFk+azxWP&p z08%9o+cIv7a4Xbnx7(3;8dH-OCagq@pzh(Wmog0ketJq=2oNw6Rj9J-3{D)WqRp!o zUn!RSWc_hqH^7-l4zsv3(}FOB7{szTHE?gGZcHK|(7z|wE4vPyVPdqbA&`#(5H=o^ zX!F6=FIk0mQGpOggGj*LNYE9$L!6|m?_(QRb43W`?{Y6#f}R262?U6QxjLiq)R=?` zbOg60>4W;vA!Stxs;jF&94lmo>g=v81yBTx^~TFpVv9JNa&2txMWllY62z(*1yG{t zL*%17rqej+*6hiOGx#3|!S~+)d;nf+v+HV2ve_Zo;*m!E^Ds!jAkAXh_>z>B>fG@4 zHOq#q+IEM3Q_JBGDPXUxfu6v7uPkT}m2T&kb;d{U1DI<<2_Trk!!g&;7q?9nhN&)w zQgPHPMTjH@Z-4~Gzccu9SeU`5Qk3MNy`@BkPsC?JGmt^UpTjlXHRvp^sn-|*hRQAE z&4}(k9;S%T34|6R#Xhx3Yrva+G5X4Swv6TNg0Ts5GtX1*Ec=)n7L2h(DSa9 z8X6eND4_6r)@lzWW(jJLt>dL`8sXb@3=l}lTk;B%b z8f4@20k-=Zg{sBrX(ZC>~EdJ(+CJ2;_{G) zjcG#6@F{*)GMXUmtT6OZjK)C6{7YW~T zXD&o86D8O*P(s2Ba#%d1VDHQGu_Q5dFQk~8Uzb~l?oz{UVji~v#pMB78h1qymeDhS zL(Zyu($Ex9OvG`55c+;ecp<21JOD|^_7f2hEsc1Y>Z3I+VKi(PdGtuzI$YIVKd(i@6|yVtr;WVaA&Ly=_?C=%Jd zyo>vr0$J)pOJG6?P!+8qZ1X5P^J;FVW7JS7Yc@LP-Z3v>fFb}Yx~&KctHbo+W$FD` z>rP)y_HjgkTZp}_m zk>t^A!%Qg2{tQ6@Gm(1&teiQaYG!80+@6;M9`L&e0wH}u#zO={LPFmK&E1lQJ_l3> zhM`h+AX<@)hPsRau;_yaY!kCk3$28pVjY8MW(6eRBmf^sJCZp)=P& zXTKm&kuL0lxzRmKyz#Y?w{bv&jtjZRmG!*BEYLO~+!BND9rj_vR9t*=kjY1=6k&3Q zHD~Y*c=bpEl$>NG1~5D|Oc$I)N|$+d`@7+$ zOI1?|4*(GBwk{HRAB)cZRdS&!AQj=L#i7uIOAx^fa%K|ga&4Dfgv0wP?tWBhz)XR5 zG-^?YWTBJu1fB)C58Ete>PCrT2$~>+Lh6RL!D7=4v3EwwZ&j5h3NYpDz657@`jNe@ zhCt)m739MN)bQOfLl0Ka3woVNIDr+hI75VD0(!ZRa)uBJfG~7-uy`73DbOVWXpg*|(+yVlNc- z9_B%>k>Zl&iW1(=l*sT@?hTT|he76NUKEl*s?XvYkc z#Khn#tg5Sua_Ib*Qx2o%kcO#qBVN5Goc&eZLKFec*TCx&9^c%^GczqU@=fnTZ7Ryb zb}CH;In-a1+k@zX1_1^e#>dtM0fe1NEJ`L}^!Y%t!m?qB)NvBM*>TargFHm)LE#F% z2G?U7WV0MC@FKDB1hW00Yt{;#jIXZ6e1t@CDs&rGUQW-Jc|tg%jqj}Dh0eo7pLAMO z1=va;R9e_>w+YbQbQfbOtc{}u>f;9vYPi>l?4xn<22|zA+&Bz(0kldAnXi=+6+R58 z(z``C*u1u*_b*kU06)tG>jSN5BuZX0+&)t!4iCW)ccY?2qDs)ArlSHB98pB^Qe-S9 z(u%qjF3HEHh52o@c^Xtgom%z?UBm*5sHiHUD5K^qL3~*LSsfn5QGk#!`!zkcFgRL! z7jV2U1)>1OZqUdrA}HTYCAcsOKC>|V3wSkZa|gpCFh&D(a+p9lh7YhBftFTjRJ3$9 zX#(#cX|=WSW?wa^wMo*#DhP+W#*3iP?EDx$!O?p2penr89Jo+u;P;nv*1R!r2TWv9Q5*-*cYo?NyyO%4+H79HFgXOnOMfqyX^$Rqx zbeZOdu6}~dvXueh#vlP1%S77Tn)d5kVHJ{KAnOd=s#qvrT7_dlbr9SiS{Wtn_1B%& zwgbJZA}YFqNDy9$;$@dob}{U!I=rNrJdiW-^LNs8TW=P~=#n zq43)22uTDB_`Wg75=^T3#(ihDYLn#y!2%%7M}cPfzI8;y%hEZ+D0n6%MD1axXpm`< zLNZ!#VNl^ah|MvE#zQS-eN0eqYRloM?gM)RoM$m}Odpz^eb{c}N4bQ!(4*mUQJvQB zJ(V1Q{d!OLu_$()+g!r;+>CH2bTCUQPi$$=B{K(FT6J{T2*3h7&|QS^d9AsAVZ_c? z(u*&EH9XYTWG6KbDdO1UrQkKs5iFB*&?3M>&xs3ADZm|;<=bMNTfs*k6l6TITZU5F z>?{|Su|acf$?TsI;j~?Xa)PVykmPJ+=1pymi_AnVx?b)0GwFM`0<(&Y^(^5E9_Q<6zniKZ*c^1?s6?m2o9(pm93o=gF}Ap9tk(HIU? zdU3rO#LZTrz`y_-l(fRA{3_Y@B#lT(HM7v(+Lg?n<%JYUCkQtX0tO);926%8WTzPq z&Jyiz)-58Aa$hD~1E4kNJ(QbnEjWi;gKZOdVjpZk+=x`93sVD%g0)SBB7%BZya~CC z=PR6>YcVAd3Zc@D-3_YAzO;BMxVIO)xRj+=4~QJXJvn8}l0Js%wN*7}mi#&9eS_9t zj`w62K|@l}Dl5`QVyS{Q%>f}sd7amSBPm68b1NO#r+3eWnhCJ2n-(umIqJk~1C8Z= z_p>CF=Lx>Wq>}Ej!1DXL&}Z$ZG*Juz^Cpvv1q?9~LD9@>$>_e8)tPC?9@c6gf2%`Jh8tQdw)@E(< znoeF_IEisBWf~C3hh~wjlGx!`v))3B@ql?I6Q79@-t*1+g8pV6Gw2WbF$0)FYBXJB@g^34#;Hm)?2;f;orb0MXzN4(^XO z(aJLunu)$TQl+vrRYDsvFg1s|_4tr%tZ^(=`t%bAL)n6Yx9f)lE5O53=0-p%?|`k! zkPR_Ye0coR)@=HI+PA3!b0Q3juwRr0MZujdt_NPCfFMP1ITZ--gp&+6+HKjO>xKe_ z?7wmDab{Xn1xqKX-AY3rq)>MOBi!$ctDs66Oo8V(NwL-ThGL(1hX`Y=7aEzgE|OP3 zJY1bhJWXIaD>sVIjH}wQ_k8JNt3&N_fl;~fS9?3AkRfQ7AXpgCA%mRhRJc&kDE@l| z0!a~q`N@dKrP25Z#IgA7oCS%NB;qN>Ypl6U&hA zS$>0TLpH(6BT|KQ#PMg^0*!FPM-CZy_8{YK9v9-F<483NN{E13Xt2grQVIkm6g52L zMWZ$i?6fOTJV9U@?dPAD-g0`Qq1Hf!Nk<6KqZj8NX-~yb9XO$=9>`r&qE<*A@iGT| zluE^S&?%8bw4gV%T){j^uY$502&PlX5scj^uvY=#J3KH-f;AE#11&2(!yLfm9@ZuS zCq*0gW<1ldM9611C5u^mRS`_!qI{LqechuBM2Dxm-nK?plkA@|YMQ(*Cl-~^Ngc(p z$28R`OGv_NLXL$EK=&LBEYw_V8RIy10*l6a) zA&Q4ML%_qndgmixa$~$8H!suRbM8NSd{cetLB9Y~?2k)&i& zCHpX{SD+r*WPRRBydeXotRyf^I7$d1ey!^1p*uefSw$)eKTpXIk*uWWvKD*kByuRm zv9|0yIv~UH8oz>^?#jh_6?aBCtEqgqDF?Vq2i5{rs++yKMw}=siML2pN)~(GoyLuw z5W`5oF41jfQpjDZ`&(@tlUPcGTW)m>;pSGJ+>!x8vLbCLOcork zUQXEn=M|wls3vU+P=bJZ!PXWHK{)(z$e1U1Ha(>}p(wa1)TbDhf!I>G4hRzkiD-O5 zVbH|I!ASDiaQ&{m^4VpKT!b)?+2>% z<#8l@R2(P3aIf_0#-I@BQ98gytfrlcZXXk%Aje@C*hzuaV0lzyn8;LxtmQ+UgKh42 zKBmBz^kn@5iEu}Tagf+prSA$CnCmSoC=8Q3SR`m%NgVH%8+zs75QdH%#%RG@?Ejngv{C z;m&ZT728I2X`qj47o`1wM*G`up`k(=svRHFWr7C7JP<-`+Z3=g9R&`EBnn&{OdnL) z#mkE{vo$YJ?-cH>AK}*1y;s3L6HkTT?GZY=D>5sT;BN8+tw2JO)bphd3>K8Mjbb8T zx!p3tT%J`7t`iiHpw-r~1m`?D>7=fF-;ix1kcQRF3$(+hPlZf@Dq3^f1i_vKBmRsn^oVd7d+LmVA_%@nLtbn(c3AQJBeMrbn5P#X58pRi*($VsPe1q7A1s zRBXt=fvOEvl?M2p0I*$N1T;~u*LIjjsK|V3wWvpBr+b%2rG2deSbz_-H9X`|NTyMW zBkBX^FJxRxiCRL%NwCAx5-s^p#RpwTWzWN>$IXl9@in^& z5wX(>LU22tSG(<&S|;kG-Oo3@$uhpU?2x;%5?!raz^rPi9vb{(a0 z5)imBS(gE?G^#Tzh--s`k_-GIaQx~R{-%m>ngbeFv_0$!To@4>f>ii@tFevG4K5*X zlX`*?NIq!3A@xBP$Z?|*pr`NhYL{v=f#_kX+h{`>p$uYwqGe2`qO z_le(+OI#y$$WH{T0{;}pw-97C%VPMIQo8_HLUGf8QH@-kH%*^{OjLp}Gr&7pFOuK;fXNy>tdj@sJ?~$-|IEA$+Ble6k`LnX2&8nCrmv@TGNxZ77In zKKSslh8*l@ht8MFol2+(TTN(*ZQ~u%%;k12h$ujRAi{{ULD(KL@z$fR9ta~OKZDxV z*d{96c-$#GGts+pp~OL+=oJqC5)VP<(XO|pNQq1Pse8QVeAr1$E>;o*1|5M$Om&U4 zs1;$j6F9nAwGE;$Qzu{pl9n>7*8%a_P4 zS8qZCy_Y9aK_PWDK`>M?x=Thb3@t#});s6Gk}4UrXQW2qgY{BN7R7E11e?bCxbaI) zaUCqfuEb}Jp@_k_KPnia*RlYGEj`trVo;*Fgc1)=f z);^OH0Ga|ZvZ-8mR^ft2md4sK%(`u{qMLm)BWxuHK=u)B*G3xCwH44PWz+_a8yYH53V_#zgUVfRLzw3B3}Q~1DLh$K zbyn3#8!q2%&=plSucT2}g>NovNM;llwS3TwWA|j+0~w9ML+lF5uQ=;3nm4*S%qjAy zmfP<@8b%snOzP6-am%~1`XJqgKC`m9azYGojAzmFwT*{@rA$Sdgu&-0Lu#(m6tufl zp{^D&=m~6KZM0l^LXql48%-ApDiVw3;k)X=C4_*JC{h6pG6BlQ#;OMvI_&DzzpR~D zK0Xun$vbeoBgwCTFgIX%VeD7Y+Tnb3DuyGjR2@xPOpHyK5a`BGfj4MPUNVX+1y8HO zEMcRGih+x^C-BAT82IMx?sQxOPoyHtiuoP5+ zpkz?u(d9{3k_#!%w}nnDTp5yWsa-TWv>y?5Ep|{9%?uavzJ`znD~f+cF&G8XeLC*NEsX~+n31{ zO1kURS7uJPVUiNHU8*YxxKL_lwvOES!ub9+?77XUVGY8I3TIk2 zGw`w5I7xc=zPOR{-4MAN0XJ+tEj@r_2F_^XWPSy&i0!U}_PoeOy- z0z0D85tRzYC{F`*4+R8-KW$l-aX?)fnX!CW>&&(?N5$No5WR>$OTvH^!!0aG`4Ni5 znwnm)bkJ*>EhMj|R9>`)R@A)pP9C*-@HEFLP48u^u+dyEn6bngSeHr@WPsIH<*p3d z!=8=WlkJZbRaW=Dr99LC}6>$=0Yot#$BSX(|VEhu*#B+;c(Q$RkWaIK)zJ=@7cq&5K8fiFgEPXLWIz^ zg~8#&LB|ny0~{4=P)3DR!6#}Z49oRVk)7Qhju$}ZZi4aAGh1b3UeGBUO~8eF<(hbK z^C%R=Kf5=un2H^Wl?`<>p^HaTUk9nh5ZsZGY%`VK)^JK>6f2Ca0g!>s=#kSY-V|

!gs45JGV@a1yR!ASaze5^$V( zx`L6$x6{YEmML4xB`4y#AjC^5a~`5Fp^)<=^lr3SOeWa?pRv(m(8vXN$CiVmYLz~& zq8`WUfX5eHaq$`!i|Hm6TPejiD6U@cJ(H@;fR#aIkJr9r;{`?)Ei*>;b)|#$O@gd( zvUnP;MorHQkU(g`utjT9=LjmG`LnRt1elz>GPNK!rW=lbUHn0&Rmyi|QMrJzl&FQ= z#Hxf8aG8dk3dW0EH|Dqb#7KY%fa zrc=ww5bQE0fV?3yStSRDVHZI-N+R^YLF`*(buA`bm%7M>jSVc{U}W8KyU|2q+B(M3 zmfc{dhdBgt(Bx|6^<&6pL??g>vPig8wmU(lscLuHg^vA++`C}L#xyRVOYTVtZy_Md1#@im#MZJXS@#1Jq#0Xc zKNt?nY{1(ulVwjh`^{2eDxhK6cD`vjNLs9p zT{pJ;JZ(X#t1BioTX+w2v?poxdcXV>~0ByvN2Uu)kc&N&ja78iWWtta&{@;tU=9HJ=cj^3UOBQ z3E|UDP$v7rqHALPn-;7>9y2_5lVL6`M`fPMwJ0THHW?Y$*g{w8E2~>ya> zOVJ^+k*h^MjWx45o)}t{XOMQrbZ@0H`fg!1=kmT!zK8r$aVp57r zO~ZWpU!l|1Cw*ilOtavkKG}qv$Od%-2@3fWZ9`9thp-=l7gyX6yn#X9EZ6WlGzJl^ zc+#(iX3s^smj@z<_mQES-td~sqFMx45MI2e+-mOwsh!BM* zQQ&mLG2>gnR)lK?L2IhHL!iM)le(ouRD65^5D2p#DOKo}Lp)UGD7H7Urwb^2T@}?8 z*~}H)C3Og5v3MoCm!x+gQ9>v?X$c1#BsE@;eC3uT)=Vidh*LrLvo`jgT|iI=AHqzEu3$8PgJZBjeDn|_A{Ws(?Nl!|>8r_FR_M?H zkC)Y;;(9m|y}=Q%as)${8uHtFk={3j7Fnh?2-61TpKN=qT7!$JkCGdNTID8PW0k0< zC*|-k)Mck7GIcJjr+YR&uLzLVfxwrPAz^~Egj}yVS5LbbjHnwM%r=&!8RXKN z6dLTpVD^!c$gv;@i9pPPZKd>P9Z;k0JFV*cbg&dvAkA;-Na6a-m}uxZNh84mZnb=d zuOU@~Y``IfKBy4TG@Q67=Hwq@=L|d@6b@uCAXsH0zD8u{tVN+q#8MzwtjZVY#WEg{ zSyG>=&0In%Y+f2`G3`bGfLJ5LZs1|!!N>xfIywvxhu*eDB-is1J<+w!z^UjMZ49GM zA5~>eIhVRO%oQ_4PAMcUzQjye6DB)Xc!1Ljs4y5rlBCB9j|rT1)lP)#FS~PFIFXs6 z$mU_sx?D}?LwZ9zag$RqHorzDL3p;Q!htE_HS?6JMq6PT2jx957a<;68>R1Of|nFr zbwN`kTb^EJ(Jv%8QjMRSxkAuq&VHf?ccxJliFY*ma}PS2R75a9`5c)Uat{NpU3;)O zRA(-F0e~pM1P&<*XWkONQoKAdjJ@tX3x{6v<&7eaPk;zxc;oZ$RtwySByrjb^+UH> z?+&EvEA49p?!7}dx3aa$dN8*lvC$O>>9$zbX2RU@)V8JboRSd>^_uAc()9N(hPNQJ!PVtPthT4h$)l5h!!_WlbBlzjUwd@k`bx45DvwD{zZr6SAp088vik1(J4xL6?6<`$t&6ostxFL( zaycb!fpFw28Wa#wB`Fee$-ZOheYa|VnMsaCTZjo@Uf(f$E=0l7)z+SoWT~Mu7TFA# z4M0@W;D+G^p`gZiTChH~iEg(SIBJxJu$qo;ir?j(1nwu44Ko(<9ERp_G8L$1qjYCX zE)6z4Qit4D94#RaH&-_A+kUYN8Vah#dU zX$w}2I)3VSZhWxBMx677^c|FyRRiK=ORKK?sAb2NO1SbDh<&$;TUwl+S`cvy><)M6 zSGpn8M#ZP-)IoMgDN>s5$vvB3tFuQO=3&XcDmyyvlrsq<2cW7}u8O9Ifj+>y=tyUS z@o$>nooHXQa|!x5kY=!q9Mv=vfpD6$M+o72agY)s;o&PN1t2w)e{gpE!@&P{(d7;x zt2ejcu4+s9aFgU5o3!xL0F|e?`*All%IMC@;3Ufmcrd{Nrh$UB-RLEZA>S~zN#k(a znuGxy6mIj-V0j>YU2{HW@Gxa0b&3eEF_Fdz$hPF}L!&_nf`~9&_ueoLrcB^k_jQPb z`x4`Y=2A8Xxe&axws33hWe3+<3qU66X_F#`4RVXY#KPho z)@xtfj`U3u)$xZ3p(v!A_F^;K42+dOgaSHL+YWM`vhMVUA&ce-#pAZUB%WOw_=q1f%pI|yPZ;p3^rxz^S zW~YJ|cGr?j1NCZE324}ijOW(o6RildNZuR@DaO^lVXFK=dc1UtI&xWV1IeE$ac>-V z1+9mstOXsJ@U6%h4ab8ojSF9NBilN*2OQS-NQW`*AGwka-(q1iu4)r*=TfQ_KRD=RgDP(Ir9B2$4jZ5q;ni8W5`p4G`mytZTvg+4xynI@`rjDEG_Q zIX6`m9a@IQJO5Owm?&uF^MWL`A7}>1(OR9za%8>nI!u* zqU=&IlkvL^1?en%PF!L`QV%B>vu>EP(7}jh*SsXcpQdO~RE$<@0QUf^e$mTqLm} zEkQ|g%!`yk1tk0^)^dh%2Tv-&2f+x$FBqw9N#JW0kc7}04DxQ=@VzbU{fypOn|xoH zQ$eb3p$_AKTFR<(@XR|b-sJWJ*BlTJ>NdbS}vI8G4A{DVObpBHXQ%!Nvmswh66 zUjBGx1UN`|Le|fw&RwAg^tuKi!O2gG4K)baIROHot(H#&(l5+_AdWjBxjaz$E|2p% zj*;$|Uz{7lh!qOGY<3|{(=(kTif1ZqMn_ubsp?Xm#*ga((B!4w&~luUGCoS_k&d_ zReJ6<9knaluJ*N-W3b-C+Z^|DxT1FZZ>QgD(Y>oY_7`r|t6qBtd(~H`mwVm3-o0tr zdOh3P!uQ*Y-sgR5T&~jAt!A;aI>}A$be-+F+{brtH0z!2=cm2Al8!SvuJgT6YsJjo z_G7(tXCtS3Y11mb?AJ=!o7<`?Tdl8eI@?zFy7um)k@bv&hK)b){HG7<^ zyK*CA5Ve!DP?xg@z0T0}Ztk|TZ+2a?+f^FcWwX=X%7?6L)86&gw(Zrnu5Cq3ZF_Fb z9?eg0cJ?w;+KfHdb=xU+x31mKUE9;UrNK%I%FSNb9&VGp#@2gzy?g)w00000000lN z?$Y`0``g(lQ&8=00)hE&iyZoS*C>$~aKw|l+r*H69W&28@W_nYmKy>-5B zt+L+lz3*>pw?*6So_WZ09=ES^2j0(5cW-uj`y1}H?DJc9ce}mi?)IMQre^ay?a#ND z-)<*9+vlyhH@V*C_j}bn?{{*qxlKB&ySjVb-l@Cio4n-cb$6caXMNv&Z@%u^tv%tR zZhPHcx7JnLyT12Jvf1un>p_b5y_Y=Y-MhZK+uvt;^=|97>U-WU-t6w3b#ea7t-P?CQ`R@BU^||a<&eG}a z-5K-U?TfqXsoT}=_mW+ytx@ggw{4B>RP!CpuYBivd}g_-p5Ae8@~piw_p`gW>)D>&E8Ew(*KGr4_hYWFd?KCz zm@1JO1XI8Ol&AoTpMjDxnyMoK6!D-VB`RnjihCeTf~brF6%Yi)DrlKM@M(o)KwbLx z|G&%4ajp0J!Rzg|cP%|$9#+P6vGPuhPYZfaF7_50?Cb-5it(+5s zNW~RV7BF>&1yx{D9g!~9X&~ib(3Kz+KadUOyauFKX)I_*0$qZk%>n@0(vRvpG@W0P z8B2$!W@hVto@X$cd_BtkkR z_7(~JA{}T6i%)i718eVC*CQBrVz{0xm0m9bXG8hi-;^-rYuvoNj7Za?MeS8n1vrOkkBzE6WE|n6zP>Mg=)6&bg=d z*(W(h*Kj@xxD%a?)8s9Mr_P9ttyBd08Brf(;8IX3DyN{^1$AbJyhai+NqUel?M1>$ zHO%%Ss;w=NG>y?CmlJz;IPTqO>Q6;hYOqr0e7{mr*C!sLdPGJ69jLx5== z%NPo2l}5qdl*=H@y&(=jkHO7m0bn>F0x0O?C`T%GK;d%8g=rV_EUtz8yWr`_I9Xv+ zrq(9rxRo+vuw1hP2{t((!hLXeo4V9jZKyU_z2MBwJwWP(*JxZjSX>a2IEb~2aiR;$ zzJ-l0=eaj1v3)e0xs+2<)EGQQJyCe)MX6NCPE*s3BUK4ClIdO9mK4*JpBYF8uf?3$ zX_WY;2t>=pyN)0tBTx@=Xf#@QR~?r15aC0*{k6 zm5)C3qRGAE5z9R-I7MKWEUGva(aXY?%>3NDG4vuK5?1r0Z${!g9#hi`p z7$%wVl-xHurUQCy`cSN&ElsMqRCx0u%jFz)xq%3L1fK0*v;zW&@7F`Zay0c!9zZY@ zaqJh$K_EDhkZu5cerEQ%W8#Ht_?qb85997sIKV;GyR5aS2wok3K)$@3(E+O&h-igG z*I|z@K~)=^KB}1X-Ah^=vd9%aZr55O-t-ZrtgDbdzg9xjbAPEN| zK01i_EX8PuN0V=?fopIafF-(vp8{!;B|iCYz#X`pn+DsyZa(W3^9zo36zN+TKFB+O zZ!+SC5S?H$!C`@qPAMtWqRbGZ6UB$FvZxRE zlz&;_op$XetX!z?DT14KHt7qS%&z*tK)`inJ|aM4Dssg(7eIqiN*~)Ra6=3D=F9 zg=&5h39(V^UxD^!Le^__LV+MjQS!N3_Kbp%s^FQT3osOkjwPsv+d(9RC9#HnM9gPl zx*ic-i?xAuyMqTW9n6LuXeqyq=N_RT$B?Q^?!YVV2B>=;=S)$XqO8YbLjn=26(m#{k{Sdfx^)49e=O;Ae1+G7)C>a5y|D`1WkLmEogwAUR)dovBz zl(;pm=^I?;RB2%f;9ms^3Ub}2D7RdjIjX0)wMNsR6+wdiL z4TwHe+Z~N9g!ZjaKmsYVq`lf(`pONhVY3Pa)#ql-sqPX}R*u&zpIV%v5@|GU3=*s1 z=?M2&DwEzU0D#Rwh^H8N8IX;GSpBrFaL+*3*o==MD0zj_qvo7Ne<0X4Wo;KrL3$pa z-5cqRcRRbY>k?2+9#rg#Rg1U{cNU=`Q;>{a+r)Q`Pae(_-n8WKNJv4Tf}7`3Fe46w zY?d7b#3j=V!35ekhGp7$AOt~pf;D5!K}e|M$G73OhzK;$*&?MeX>FNPyh*EX39VV{ zOoqi-K>X~dF z-FC+^&$Y_Tof~pmuBMEXI%`AYezO3@Ndyv#5g%!8nbkW6n9!uks(yUD8dxEzQ1QwE z1A9Buqn;;7rRxm`VNvNcBS@2h-)T$i3rOo8! z=7L#GuRVtw;BAoe3gjDc&5m>0cePItEF<*ZZAsdWPMI`9D)Rj@^6z4SGh))TRf{mK z83i0c62@eHgTn6aT9ra*=CEpUnT!Nd346E-%k?X{Ajucwn{Yv5F@ao27!uUy3N+4g7)OAn$Fb+$9gdU0*b~5RHb) zI8$@eP;EW3gE5t&2!jYX{S#_AYqQI^Fs_DRSI+c16RC&cd9Him!aU>3dL4_0$VO>M ze#U9J5U3|@rdPhA!HdYAP)+ZL7@3VP0m>mulcAy4+{`I_w@OQm;l)>GaGlynnqa={ z)s*O^lOh)u2_yDHEy*%+UNW$|A`3FuX|n^l;)PvT8pSn&u?+$+QKkhj8_{QOB@DN$ ziMhc@d)5+7$4`mApkniXw70k;z3bSYbwwP3X~aRePoXW>@K!z$de6@J9M5vX{%Lz zu3U;uE$Kn-WOz~QuocVD&7?THKq7E~zUx;>Fi^{R!JMU&Q%s2nNY+cn!P%@evqSNZ zovd-)qw(GsV zKBsaZtS#X&tF=C#6w}BZSaAW<`4CkBFidsb%P7sLcyLs9ru`lBqdFO27Cb2`dOSe@T zeVv}`Ce#4H!4?-*Po!=l4Y)=!R#?8n$op4V^61Q~F;!JdxuO-D-FHL`Sy=||gArn5 z zauER!QXt{?$s)RstwTV)duD62<1Ph*kk^Bdy^USnULPuxwR|MezG9W`%J4uWmFab_ zbT$NNpMfC4Bj-iYWf2cb_PUdkz9Bsxpg0?X<59bJ7|>T6Xr>=4Ci|$w{HX>!4eQ2U zS3{LB-c0HVmqLl5#~V%OP}HEDOj~e5Gl2A%7|7g7l7)F6=@(A!60qTKRkv7f6|GNo zWVBQTu4!=2dJ}nuNtSMrx@fww%KX{8QIr2qf`@nk)Rm9cr`1> z5y%M<>?3`xeLK`9gBDBKRvL}D$4Q~rbFvxGCMK0-kX|e(#ZV`rsD{@Sre()~*Ojs~ z>S3uca)_|*!gW#=tb-#@LU~*;bQ-&T!>|vxz3S7Aqe~7sAa|GIl3#8I9of+Al&K`% zVI-5{vHI#t?)=lyYE&Wbdf8yiv)dsbfYWB8uj}x0^o1$88ZBWX*%RK1-FfOQ{&e%Q ztDOfoLw4tsiuuF4(^?dWJ|*r}!nD>(DhDpa7s)yvmHXhe$44^WR-$1RbXM;(2ZWLF zF(5KsD?ul08OIKv7u(J*(&)-4*2B^=&Tp!CIm{50KDBxWl|Pl++-}q%AT0tnoc25f zs=-4;_pS!HK>UfQDWKVHd<9z*Ej5i-JG97Xrt&O`z}eH7_Bd634VJvTO@XMLWJ{1Q zT1lhisimmc`SRV_@09NLy^eL&H&Yx%wyBGqEH?yIMSFL%ILf)(j*hA5WOCh5rd!0F z!_B&DaGV0HP$Pa1?o8R^byL}ZcAJvweyj9VrgOz3ERQ(NM27sZF98Xq!86-dQGy3fs>$1vd-xP zzj?1VRQ2`Lo-Ql8+)W$`XJDx76nqQ2nzr?ssy0q`dwf5<49< zo$nvM0A_>u_H^<7Co9An&|h&wkp_g)lO}66yes`fcz0N)KF|t)T&@%Kz3-o!g`F^W zH_2bG0%LnONr*I?JRBa3-D)o5S5P34)I1{-aiN_BkubWktt1{^nMPttpD}P0Ffc*- zgf#3A)4=Tp4yTQ2D^E?r=k(;p7!%{xO0w_`ZKL^KMMEb>G<>9@kQidCx$80%HV%%B zy-A0&ZoN~`KFLIPsJh4&DiG{;wxHMDt0WzA6L&E-=xqZl*g9<$3)|5z^Yx4PBI}FT zsQ69D4ptn^dHBFujHqP@Fl1DI$~gv~7raR!1{S|48g#H2F*g!UDg{S4!>(R6TT}Mr zv-hE=twrZO;rL-Ra11%unkmtYJpwg(axFlpkc6hB0v!UTLIIT+4kQ}RHY=W2b0kD* z=+YkP?G|%ba9iFEns25}F?eKajcWD*_$0?`%YO7-7>?5s(e`R}L{~=nvWHFWuJIq} z>fg+PDo!z>`PUj&5_(X-x;xRx;W&K>j1u>fzbQgbBJ;@kidcLy!zVwCzQ<4oYdJ8m zfHGmJqi3(%3hg7}w|X%ORi^c&M?}PpCJ2@~wlyU07keY-GJmE@o;D`t4ff-Y1tInt zpXAXg_E|-vN`-}C{#1EvV$Q%^-R$kIARb>MGP4}3PZ3n?t-uzDBF!0ZyG zuIpu>G9H(-U_dbyN6em^wuV92l=D*I^ysqW;&suj`8v(W72wt0Np16ac)SV`OGG^b z!0m}^M7x_}`i0#?MNf>$P?6>9d<2ESP|2*5=nNw)3Z&0MrEbrTIacXs6!;HgCAy8U zVLj68AxCWT>n~E0xb&U?5is?nT>LcvM=%cfIk8ppMf`PuNr2%{(qzwXS#nZmz$XFw z>jEp1TOm6XzT9eJyv)kzp@(uAjwCdPg#acZ(rL>Z)O7Ib9|o2Y*KI~3K2%Grbbym+ zG~|P4$NTA$+=8`GsETAk!OE2L?Tq)-yDP`;JHmK(ZAON!8<2|P{hl0yX3M9c7Ukop zpg{DArwPotI2%VUxYctCsMn7m%NBIpRQrn$Xsp)&QiC#0J6L1M{mD%hTwJ zWosPj@xbXpCv{uh50pWPt4Ld*!J!b{xla#mW(9#mpDn7aQZY4Da+hLQqImT8FDjT| zYp0EM0q>}E#vH~x zmJKP*njbOB zd>dp8s+&YLEOD0XrJOBMDG^I(;_%fIb?LG7O%e}!*viB*j{GZd1fbj5s_$BWP4)1+ z;i|uuopH*GF%Lfo9wVD|O-j3&n#}ogXG9MIL@wQcV0JBM zCm)a*3#OI}WZex|#nF`RGRLUO1`pycy`ghy2c)>hbF%+QmgN#D3{-o)2BRi>P> zQsRcYS(l(#lZ}ore0+AERYp+{M)hfC(D#Xrf*y&JuuxySNu39ga)e?F5l_zGr|O`_@R6(NO6HIf|SwH$N2`l0@_mwRyQVB{AQ+5$LSs=ufJF)Nw`O;U>$;K9P>rJpi{QA z?5offr;=jASse#QqOyh0L~?-u?h+dSosPPa~( zixwb*Z)*5>R7#A>U0QFCt;Bd`V=_@!u1cJfe6;bdSAAHrsUfeE5ba#0MHW$PV>;}!E5>>03lYS9 zUu7OFSIQ$G*nlT0w^;n2+Y;B`$9InrEkU63}RBdtgMu6~Ve8 z6-q%ai|_|@{&z^1T&rbPE83oXt{wJ%B~)F2d`M*-EF7{K1*a)TWH}v^G7mV*8Q3f<}f@6c`Pd=quAzfw~8QUag>5%4&=e z=5tsm;gb`>1j_+i!k8RCEnlMM+Pj`ESrYM5W)^6i#-M`SwZQ|Wt~SXyEjA_;k3z3j zQXqp^Subh*FK&mEUwBtGN9tdb8{O`9KFEKRNvQ}iNm4y(!||}b?AEa*%Ati5o;$@4IdJ) zFA^NNi7l>(R!pz8?7sUhDKs0PB?@_+ zupFXc-i#to2X1nxY=)_SU$}xIgR8WQ=+@jJoZ6=5%Vf@dxAn9;KMzG288aYqJJg14 ziiM?hjgA{xI8IGKoaCtHo;SKs5iC-Gg?FvB8h#;581TL=sTqK|slV3Q(i{zG0m1KW zPT_7I5zhqtVRbFKU(KQ?emKG3lk)54w!`A`gLUzUJ% zTqh+}y@YaWO+r#Q(V4S_UJ4JixSLS_bYsb+*L5o zrCjn4Iiz}R+@jXdDk6cJ2~;V|($TT56pY1G2LXDmjm5-PzbN;qJ0&6RN)r6E44#P2 z&2~pKb{sm8T!7cgjI)~(9`$m#-nBxFO_PM52dT%4VDHSxD4Hd3cUqS=E47PV=SWzgQ zK$77?g$`e{5o9RqB(g;NYh2aL58c5da<2Jl#lu?O1RrGK$G**0ZW(!mWdoW8?JLuJ zzi6!EUE!CJ^E6jILO1Q`BB>XxNJW+SJ|F=PgbJ2Ts!DDTBAnSL6fra820wcbPhNVD zv#)`#c|G5ge>mc=)9$(G-p9YUH97D&H}2!Poy*M>>a+ISTj_Hb-O#Kp<#^|HLt%w6 zyxFExn@zEyptx5&N|zVZYR}7)F=Or%=mH_$wKJvZZu(OYq@a;rbJ)0if`YtSjUB?p zVyT5@fL~03!njhxHf2H&J9VuM8krTqr^Dwr6d4D?$0HXwcQ<4ba>YxM+z%i)|MiksZ3BXX!m5UGl93bX{n<4(^jg{=GOPZLC0$TDB7*) zkqsULK~E@z^{;4in^T0i*vD6=0UX8;C#$unY|Kf4 z2|%n~)y4O!UFHLxo$n5x+6@O!sEd^*B?Gq(gZOlL2;3gTS70M)RtE&uRiqqifrOW- z_)?OWWGU*zEfV=GcPL1yLR8j8j>a>>R7E9l(zS08>w+t5meYG9csHjhx`Bdis+>h> z6w+C)E16vtQ+q=s==pHeWEwSt9*yivBem7q=Uxn+na8OB>v5chZ=_fv2+rnR=VEF}*p8VCGV_&9 zfS)>?E0*bYdIvaVpeh4t1H22vV~le^ebD35ku<~hHnKiMdXbj^v?6>Pqimj?;@RT&E_eV$!a0;F@=NAv7_Y0 ziXlO)_*W0W^HRhh%Yt*$eLKVE40V2AQuNjgqoC`M6?VeP13m!Hch6`YK4)72}> zuQ8t4)@oA9+_JrK!Vf5F1t@M)sIk2c3RZ*05Eu_2%}q;2hBn058oF3E5?3Q3P)jhB zvn5>y!(2_a*xMc-rEm*#)s8tTQa$f0^UCMay6(k`*a(z0G|d-P*7`=_`lQ>Ain$(G zg5k>?yH1FBvtN77Bukc5R?}DJ^OHAi!BznoFde94%S~!iLbE-m)#4S^Z4j~r*&*+M zMnx*|bY~_u!8ArnLZOG^zj>*GimvueuIG=pADoBrm93Slq?7C(ix?=wxK)C-4os9c z8Jfx4ifFHCjNUMU9eVuI^0(Z_?G^1uy^~I?yYL-VIC()Tj5p2Rp&^WDTw}3_0JBhp zz~&(MbsQxRYH1{?4KzR)SdF6W#o7_nQ!xV6dqv?W@wtZ7dVwy4Q|}PVvou%zbz@^k~}qAgD`YThg-IHRZT1Q~72cI5{;FpfD;fu=BY)<8EJfM@5< z4v=6vr4+w+f>rr!*04Z|a@|#dDoOP=j@>13wejwtMEQkg`VMo`-qOyj)(=t&uzK9( zfI!(Nl0n*JF*RO%GO4Zi(;yCAYt2w&JvwFPDhFoGRJin+J)G&7bG{Z*c`#GW!A|hw;PB!QzTMX z!b5K`kJ4dR8Wr?BmaM608t0~RbeocKpk>i-Wylkx0kAyP!mr4)fRQE9ORVlDK zi-R2bLaqwu+HKID%L;^PUZjA=omX_pq`-U(Hp?m~n?Dtt1-_o(KwRy8^0 zN~a8Wc2FY{?43=Cf*}-=G_ur**7seWGJa&)fSWIPAJTm4K7b_}BY@{s(Urt^ptfkNx0KMv#mJ3As{e=?$8ei6ivgRs9`k5or^@|Qzy?UZxA@|c~$1r zcSli`6*C=FnGg|fLcG`uL%$d4d_a@*by0i zD7Ms2Mz}5ZK&WieRX~bUeezqR9xtrkib`ovV+d?Ua?#U;mvTzVE?>$LAsGswtgtXH z7e95=9Yva0k4O$7W(jkwK#!E}9QoB>z63$V3quVFGJ%q~+!e+lvTwo_lVeb4ou#n` zs}*+_<7LlGLx59Y&H)1su#VhFa8I+FDG!~sYvRRa)4PUzB7!BuIw|Q(g>Dskm%dzu z!sH5<`@nsaK<-`gG#}f6OV^*e97UemWRQx`NDsYR6ZGcOI!{jkSkEZZCs|%}?TJjfpDWPaFABk2b+n zuQEJ0zs18X3A)*2*8&S?8&3MI3pL|jf`!tc5aso1gF$H6ltT&uJ) zsVC#S-krj7U?3BTa9U7mD&sV~IT#ruMkQyRlSf9ir-`nC=+2?n>)Qx?QPLDyQf-)8 zZ5~eGAhITBe|M!;+tkKug0Tb}J51h{tY?)%m$d`c%M{vM+=Z^PgkpfE{%Sejd7~Wi z^S;5D_SYH5h7GmN=U01Y_Z)Gjhov5!n3aWf+BC)E*(!r}cA{5=$d@=FCq#S#>MMr< zu22PUDY?aPvjfr?A#@PQwRj8c&xGI?1375I{oFNd1S9qCcm_I=GB^gqFOs=Af@!c4 zfq`M>>Dw7KC_PS*@reREUC*h-#0@yldnhKHn zqO6NQBjh&zb^GLmAr4~{US;jwB;f0uh(9Kq>0(!;QKsF29?MP`9b5O!tyJXD@5u-b z^e@)Vk(@x(g5}+Qz(d=zAv>s`t&v!?43nZ!WNP!+w;C%|Ogvph>txS%@<*;2Abna# zH<@=;bci*?!R|Fy;X}fGeyu}FoVnC#+q!P`p{XZ^p*^{tBt!U0}NeNE!o zR`FhWp%RGNY7m6ELx@0e#p|P$y=ijrD)NeFV)JnOuDWxSdPf^BcvKuZ5Ef0~scXCB ztiwy;^+-KmSU65z=(=V+6Ji76wFdJYinlnS`XugQq}T)qjVm&o9=wM(=4*+D?Uw@P zDy1S5H&K2%x`ZPSacyqXs4ys+0uz)3$DaDx-Z=(Yrf*{S_ZMU@ZM=wFE^8ucU}_2sMNfw0%R)gqzs!|C@(YDnQVGd*&;AkmXrhPGqG{E;RX!!!M937 zM&YP>T%$zg3~w1>S)VzStqS&~k_={*CVE1ao4VD6Vy|T?j|DSu>}6)c{7^r15G{cR zsW5B2$y8iEBMFPLq4{UM4k0wEvNDtJCV_K4y^~;CuF6p1N-NS{WsOYCFf+N+tr*Zc zjaM8%yZWysy%H(+6vJ%`F(-IP4h+2S@u7>}J9v|tr3PlA6YW-!e6$1EMAt?hdg6*U z$0!2ckdv(*RfXfb5Qod05>bB1R_o{49cpWO%u5%eq_dh-AI(F zeG!4G&Qehxu|HlBScBACdHqu)Kyd#WDBu>dx*ucB~wZJcP#tsV|GJ1OzW3 zSEXm9=!HwI;v|KJ+F(?!MuaIKZz(O=3e5H`B2(FOXsZ5!DZdBv%z{o+N)YU#p)z=$ zA-p#qJ@_Fi%3;=sr@=~!GaP}yKMIGWQ)6R2=N4tA$bhE|>u|gvM2in|cX&OWj*8nc z;OFWppoI?y4pU%R51-2qkU-w%&y}AJhjl9~Md^^95QCofA6p4sMHk*ldX2%pYodD8 zg648tW)mL}ZQbI_-i?nTQU#kq{TS}y)GG{Rs(y~Z_4Fj-h?i09xv;A5Vkxph2!+Fx zS_L?0G&%Jq8Z6vdIkr~&Ru&}hiHkEylj*)BJ6iL{YFX2Zi9p>1m6r;tzjSyTV%%x8 zZ5}@*(^+|_Ibwy45!4CgDm|4WGSzOPU9(Htw{@dr#C8ZMFN!{9atnP1R5ok+q!|ND zh6lt7LQGYBw`%H|yv-4*4y_|l369~kuFK<_>3VxlbuyEHfpZYH>}`f}ce^W#b85-G zFe1ExhtucfyIk77A~2M=FjQ_|J~SlD%b+MKgCTjfCGKWrk~Jzr@JS9S9|MFpL?Ehc zwTK&A6*lCCuWG(neU`i>mJl{{rd81a7d{ey7**U2z$t3|JgIa+1g+ZWmvD(MC1>cF zB@xE$K#&m2xr!`5Zx^Er3`WV#G&sW#jg^o?6yH)GRM=kPS(fW{^Y4P?8RSb%J&LDZ zd-Q_dbLeq!Wx8{5;bEtPjUb3JY+LQh!WIq1os&8elYN)ldZ`z=uE@TW=!=|sec# z2UQgBy93fNvK+V@XlOv*2o6$=)1e_>D1q@m)N^$kwKtHLa`S<=gu6?s)b>}+c~tq# zBPx8lj|EUXHPh%L1r{H2T((K+;G&EoqWM@DaOAV2lBFu#F@@Q}#i4lFiQj$MHr~sK z8d!pc8f?i*#OTYH%J!SeGf5vexP)?J2UYQiQp^;>5QT~qN~d0~@F114;>VTx(mm_K zdvOf1yDr%Ek82L^(ner2)ka63tlAvN!~{<$u& zFKz>4!Aa|5p)!$9t9-7xd98Q4);{Y|u{snbge(+_;272=n-$6enIVz) zLYav!aA@f0N%(1yd02jpM;wd}``dvLuAwRGnS;>IV>P&iM0Rrq+0DzjdVEsry~A5% z*24q6p$U%mrGO>Q3EEgoYtn{cH8Wf} zOF=@eW$2CdEF=cjU2vYlmNrwcEzZt{g-OMfzE(lEkCm#RW=TqbxNv>}_&6KbIHbfH zcM0%RO+4ey1{g5nv2E)pqq&8ld^SNar+7#mt9v^Ih;4vD!?avKC6uva!LosT7Ij?s zlFCw!zN>|=Tjc5XgxIVdlU~ItB(x{q#MWfD#I({-((bi!E_g2FDvpyP6au*K8q&Rp zfIHjP-4@~K25p`aMToZFLBe1x`V(+`2 z7|-2Ce%bVnUc^l`)tq5qjBQOI%BZi2j(~zjfXWBL9hE_pZQ*Gt6%cXqG5yzMei67b z$7^14QEvoV+SbucUYX@)TYTfaQi@G-xOn13fN!SMUOBk8T852WDCL+S+D)N(jzVtH zi5}XFW$(EYE6tEDrFlL~AeY%V7L$r)k@G*A0PeWna2*jc8D?)gs-bqW^F8#94mhu0 z45_KM9(y*O<&}RC)L0J54&Hzf`uafS(4uYm1R5Ess_?mkpg7IuW|7K8uSGi`eLa@o zMRlj6ML(3+tN@$VXtdShlBhRCEWL@fhqX)9e&o_d8I%o*>ubHhG)HATzmP<1MnQLu z-MVKs_`yI>fFy=BMhA5Q3Xf!mP_-5ccDZ3D}9QPjzAXe=)MP^zu(D3-_ zuq+_`Ajc*kAzQs@x!j>nsOm{sRK$_0hE`u>PiheADEfo-pd(!Rx2R4(ytuYtA_Oa! zCX#8Ti_XLI68WqV@LhU%&J4vWLx&z|fT;@brCT(z>XwBm8jlBM-SO%+TiZ+f8V9Qu zchg`({1w!4_^qpnO94=#^b~N8QA3Dx;|Y^LGa!~?<`QKV9_<4aj%(7sL$gx(cadp#7uP>Yv0E$h%$s4ARh!^x0O3WM&oYbc{! zV*-V%fL^?T8n-we@}RQs2e>>$9uVR32;qYt#2Qp#9KfA${fphMCw>eO79l@BWCPXW zE@uIe^0u}Qnqk!$I452@CZertsOd{UpdcBA!_}u=5Hpc6hpNM@DmhgTW-xWMU^&V;@|Qldal&x;joqSnY?Z6!iF>E(QIaJqL@ZihFet9j zz;X1`iP#V$H`Qz7;%b!^?Um;hpv%+M^%&IAor5nH@456D1K7@+N7W#(8~ENRZ9;O4~i3UZ_Dw`9P?0D`*>Quz)PF*inR zTLVstD;4g<&1~(*ojfNDsKN_NS3b@1FDFE5|z^qO(&_H}s>h|DzWst&6K4JJG$SgzCRJoVF zsGgvxhzp}`v$?W<@z_3Q8R$x-6t956-vbPZ<)4rF7d6fNx_zDnzyQB;FU)P%8pt*@xa?@;fXgs=FvT1#`&6jYvZ%_JLRqK}I5Y2C(Roy0Pr06u*#;hcnsG8wPkA}Ue@`|WV7I13|5Ke=08HR>F$sF}o z+Ac36EaSk8GWc8*lBQ+&FVdv4XT!-DS6Lz;6LEXD zl_cI#o0!>bRgQSA?M>yPsxOqIMNWq!!)u0>S9y`Ar3cKlJ?XBT5tiI}eo&Mr)Lnc# z4mW9-JRKevL5Z^SGtFWN=odu#A|tSrL2GNM=V2FF&VvmIki@MI4&}=B`&&Fj1 zWpGkdm@)c2>E+GPwfOk7!jLSmaBOu=wuN0>UaVj=5fr)i3Zebemb_F#M@|kVcdyq# zNwNw>BV&Bi=xl?7ya5X<4nDU%7va3&IJpBOL_r2IzV~$ACN^QRBt})?ZZTJk8KmQQ zQjVQ=Jbeoa4hbw5J&@5S^D?v82-@oRs~viYGsBxuVRYI{!3UNQQca4OM>8X@$bQA>K3gQr3gjkyzg5!B*i1zdxkscj)6tY{3q3vuQiv?nBi($}(hLvhms z+1c=$<~6O47(&Uy%)a-fut*PQn$cQ+VSOuhvv+G8grw;~s8!M-yg9XUB0Ag3h0PAC z%~vs`M$9;f+S;#xL(5Q#aF8%2Q>}e)Um4(?7cLiLBTRYB`H;?E>wJ~tYib{KbZ1u1 zh(hGL`>3luA_67Vo)TQe<=l{rrmngcM`#~(8ee&qJ?*u8juIK>mdPG;%3ix0UrJV} z52Qmb|<4WWvN8(kG=?*;9om$Tlc zB02M=WrAKf<+n5f%IlJK9U9eTDRSGUax)q;0c|6JBj$7g=0x+vO|%X!(&-npQv=b{ z+<$yS17W04Wo;VkqFH6M{%#ca5H_?O1>S7zU5!*SQk?*rM+&HmD7-tNcl4zCj^WJ} z>k-nIi!=dlQu?1}jJdwUx>-VK`{kvjNw@YyH`-H+Xb|b)`awbKB`-gCsYd)U+Zo?1NlzK-XoJw&Sy1*AP4~KEV z2Me#22CXVX58J|fh?IEqSZY;Sh)ewbjJGG{3yj00ar1j_fr~tg}!(ncRES7;A91;NW=-pifCt`DrL_ z2>mfYA1k*f7Y6n&OT~3*?Mdw9J;6}FUyoINQo=1kd(H^BBg7d*eG`}<>&cPlcR`Yq zB=0V)UfBi(EIajfut^XK38%Aak3EiT5nc&iu5uN20<8sZZ6UQz3*KM??? zqj_gg!)kgS_Wr)5pdA{8Q!B^j;2n}E({*yxC5i~j@$329*tnBzFpSe9WGdybGU;O~ zw<66Tc<&^*-W8B^=*LL_c0h^0DKV#v0K$(4a633V?Bd7~mu4}t3j*m-`{M_p18y|+XsR;suVY04 zV1TuSO(Gv$7*1EUtbS6H?IwMI6R(9b@Dy4~)R{#teXIfjKp!OglHZqEU!R(WcaILA z6}td-7FaWA^BKC_?D0IV&6xF8@2?_K6R&7s+9qTzZ-|FYfItC6A!$MMz=tKMVL+3e z_XZGD1yp z?^w21MW1+?CZ!By7%Y@d_SwV>Ogg%?oDPm5_(i}1y+__ zt_^Q;r2Ed3O!ZJAsXVvNB}!0ckE&hD&Y){StSAIs%zfSHF75?*yA$R6V*(A8uz1vX zt`elJVXSqE7cCasAos8na~O@M;iEqtBRKb{hb_Ifs(PO63f5p60Io2R23~X6+$^DK z#(k4qqsPQ7C^i*jFIx;rH3~8EQgL}`uj<-o?hRh)9}fapv)JIIVe1FsNTKm19`Y)= z3c(7On(vuAS)s7Z!6zFj@;H;L!o%$91Q7Al{4tT9E|`qzRP~Doy}@IFAbaxreO!^%+%oEUrnVJ%pv1K|Z6KtDr>Y<$TylRX#Stt+TBK zdI9l`x82%0Fm#5LHPd0F{DK2w~4eJig zhg+CJe5AX%q6~g}R^7s=&U^J|3iJ2cJo~M@S2d=zX3DC)Q%JSB? z@JlJ>B~lr+iOT4%sOO5G%w>pMPAwQN49$g=IFn&*#HRHW2&w{dZ#j9M25yLf-NVfB z{UKbeicUvPkKHOfHSW4;VDJ@pTgllp70;^dV|>iE=1b(X>=1)#zHqbZfU!9OT&Dz_ zIrbRM+f`tbM;2o{M2JwxvXiOe=`Uz=QN+4z^KMIqDTerhI#1nz2xEpa*>5nRS+mNQ zZBUz;4RySrs6Mhpv5;#$Lfi}j34$YW&E$b9$IWd^H^y_aQ{ko*6ASIP4xWdBCob=i zJ!lhf@j<7Y-mMF?VVa>r=Zz$w^j7dbV8x7aNDop&JJBSlv*<+^3YKK6%$kMEc?tOz z$igH+Q;8Q0Xc@q$3%E7vnf9}k4)GI^Cd5c~gAe8K7qJ@|yHR4mhNccHV<{5SrCUz& zkKS(1uDK6($bms6HbFRWOdM3IXdMeQ9`YO3TufP4V$RuBr7nfTP6CiY4`WywRvMWw zxcs^4z3Y^+px%vUXr-u(qqC+5l6ty|@RvQG8@GPL1$0K*DlFQntG#365G7#?!|&NSdeau+IchrQuGY9+cOKZfTyetGGuFG2gU47y z*NjW`DO_w~UVD^hQbUYuMg(LE)^cc5(Wvo(!ZjhDSJAK(p(WOj7@uN_vJ1cHZTHj6 zB4(60NCDa1b^ER8iNB@X#{pa0_@yQAhw7U1ys+f0J?6GMt2ujBmUJ2;I&B8-Icex) zfuU70!b+nS^F$jO4hs!s;E=uMXpd7s;x}l=kH31+h{*9lfsV{8&3h%#U9tut!L#ZV zGVQur3Ww4W0!KpPO;vnyZ_o6*;+5`-yb=PRr@2ISD9{|io_u7e z)~7-f-Qe>TQHb?!jr(cyb&v<%@I5aY{X|X^)4p&yTFA&9!t~8tXDG<^l9codsavp! znEQq7Nv(5_-6D+c1pVo2HK5014PJK{naAOQvpelW&EyA?!NW8k0=ryvjVd%7j>|h^ zWZvn~RTY14wc$-FMj6rfjj-Hx_3GKD&W(Ys2<}O3CAV(gveBCN9Mo9uc054Fn1x_v zT;DgZAxH<{hGwa6F0#8`KJI= zCqxh24igxQH+@y(HNw?Dn@?$PhN{Y*^A-{`j7&2aMi4EaRBg|st%wAKHibhhx*i&n z?V>d)6D`o4wG~ZVtc9WOT3n9bbYa*e9j^nNAzia;A}DD0?^+c|K#L-sdoJ%36IJB~ z`UZyUOEekdNvFaSqeR6#Hvb(yEop=}(D)T#syN4m>cy=DI>#M}c}1?F(wHtAYKxeb zF_^KT`mpYzcg1`hI1e$$?!bHjYY@K71!CbfOZ#fOxs(TcJZ=Rm@U@qt zV2zmTCD;cAO|X#Rfh>`D;;gEIjrNi35Ws?b!-K86pHzCu1a~2NjHd z)GsU12Cp`i;Lj$rQHwZH@WE<^Yih#`YNMhHpgjR7&@WxMf}=Bd_+QHPhg&R4)3L(L z;OiQ+vu`M!ILEtSGAi*L*JalU3mdJHgG9VMo+T#>0P!xLn8fKQ=L$Spx#zdkiQ-F( zZ49Keb8qA-$;8LZ9_<>zQQqof6;2oW_qVRelrBU^5)6$xL+Yaks_4yEDRF5q1kLqn z7gm*Xb_2u$KEg^@yUq+;Ga0uI-R(NZx{(d$Mq`&b6E{PEAkAQHa5oMj;RGw78P>tR zG;vvgW!qroUIhj%mOooFEfQsl3J%Xw7-Sf!ERHE{1ZFjRoGPaE^m;%L>_*?ZhSzG( zvSyhPyr9zeRD`a6#RTra<|@=S-EBrMQLKXPwYbDFgVSn)fUO2AR7Fg5`DVSmy_39D zDJ%oqL(pr>ebeKKN6oTP6_+G_HgQPV%jic1^VkZ|oaSZRO^<@&ZwD|aim?Z(q-WtA zu<4qhFl;Fz4D#2^YBIdW2 zkA3ECAsXivveApAZKX{LF%&MBVasPX5(+lKOU9st$H*5OgW?8m#a{KcnyHakt!_Kn z6tGX4kbr}c+mwVUt<^}Eq5NA>;YouA##(B~iXB!GKvJGdKkf@r_Ry`C$z_LrWveAM}_};5& zCQ&5%LFpgW*SV)Bw`+r=y>QVI$238v5ZbaUkr_Ni-XR+dk$A8e(Qhf-8-bB(w?{ee z2Z@mzb)N6nrz)d@LWApH0mPM_G3we)G)?)qsHi=N!=@=S+H(tr4f=DfJfpo`jO|#| zrUn|Om$w#EPfd8_k5R2OT{ZQP2%yx|85>8G&&*BT6c2#Uo0J!Z`(qepzOI^@D$x@= z2v_L?JPIu`nD4B|yq@s0%S&`mK0-J8%{#7~35|XebPJ{)m4)bfs7$iLK|_uN;Z03bgI$BJXxI$)_?3u4^sZygIOjuXrbk_M!Wa*HpzAr7E|)hAm3 zuU~grRIv`Q)rD(-fPoNfry01&b2Z(21#BlUmA4Wy0D!S6`&KiWjA3AtQ!u8SsD^kB z^+jcn_|i4YLNVcIZA3jnr5?}BB`c0W%0Xd% z;9cPHLFb!0*`Rbao9P?{lrB8XADiI@vj!HPA8wQg`AR*V))C29VK7-L)S=2v3Y;HT z!BM^B%x!UU5P>Ebdu~w zU~XB&CX}7ORKMD|$Qo4{0Sq=f<*d?eQPBxfk6>#tP`|MaTaa=S7DdDh3~K5EtH~zP zpVPqaWgvAdpfq#!j#f@oR|{!-ag@EQau=_gdqB%A<5~eR5#G2lC$Jr96-6X~N%v(# zyOyagr>cvX9Yb$~8?y-j+$Rd$h`us3Zhk+Qe&$mP78h#JNfA;G^5W&&Wk^IShM_1u zoZjA7e6=;qLRB=*(Cm|Uri;p%Ut*atbS4zsP=i?Z7lP#|#qdGnnSgpAhlDrW3>*== zNRWXrl-4y$J$?yqWEP+yA=RP;;xMo-4};EKW$&FC+0u?H7L2N10kmgKRBK4` z7o4WLIUhMcNeI5S5nuKIEVsau$3*x z?@A4BosUrgaUlh=v2A+%@Kxyqb-bsGf;wmEhNX0iFjvCy)WV*UilrKitMj51;=jtg zDA3U>rnrKkG2+{Br`NqQQCV-U4q&`D?|8*)fgI06A-xtwsh(x#7!9E!vbB5FtOIQ6 zy9{|xV^D!Z?r?;-M@Ar=jV;@u546hn87S{voPB~+Ilx7-<|a^}ebO`ZSc!;RAxOrK zt;2{I*BS49Wr10mU?6rehmBa3h|DOa!qBEdE~%hbs;TpxEz8+@zPJ0JGEhUEU$TUX z?7rFJKVQmDuZg0`htXr1j6Gnt6^wvVHKA-3!Jflcqn<@k=nRpm)B58#TFlga?hchbq!1iJ z^72Fm57DR`JE z>a!CT(u6j6C(xs{vVd!H*ySsHR;71{TIF17Z%=M+m2<2y*={vLIrG}5ao$-k+U96$ z4He96`C{86g(2)92|)ADRbO(f+3+AWOVG=?Y3hX$#?90SxK(d*jby8mD_b-f3A+u# zp&ml)^(oK8I-txbKa9zYgQc5oz535Ij12PB$hedRC+bbO3rG>^EeQ8M^He(3nXf7* z7FQ5@GV$EOTc+aCZYNnYJ;Ro-7;Q3GPZBk~msQnYt!`bEo0*?V3#bd^OR7Eadj_lv z83AabhwI%EPNb-q(3N?5wi;YCf_oKbk=##v+6r&9(UwGcMifN3Iu*;2aDW8)lG5ZM z3yhwi$X3$_(5n;F)>_Rmncj{9grP?AhOK9~ctRJW(8RMdhSh2+;l)-H(UsIuz7 z+P!Bs+KbWkG6WA-(p#wk8OoOss1B{>@gW7gjT=p=468$!prO?*5%l3vRI_SgK#c+x zrtH!*KI>mB*Q1wQLSVl0Hb|mYfhy23bn?~@)G!hq_>~h4K{Tl+N$IT!OOXR%j4o+} z>_Hfxs=QRrcN(#_*~AR*YFZ0$Qz;)@?#rAJ@}?MF2zRYBvdP$R*vgZ$mF2kAX?1%e zi}+$&fEyJdiVz|s_ZFUnpbJ;!jO;UL)uW2_T>{q!s=U`JMVX?wrI@?`Xpi?GvorDi z?Krx3vB}RB3z@Zvwf%K`%Hy;wNZTQ7g)F`8iv9fFX834an8flAXJmzb;EmPyQwe)@65>F6134UhF-|l zi<1=wh&x09sdiDTZCA(It*(4uo$+OHG}>Itk6>^O^OuT<(Wd*C>}YzHXOje_Xh}$h zK|-taoykfHmyOiBrfu*KYsW$+ocae_>DM|`P%b78vJbwDm4;cNm^vd1i5rpTlhM=P zYV~2~iVF}0WSz~)VY{w*l=$#`{h%bFidl*qmZkPr|k-CovRl$$yj z=t4CgMgAD!h>HYTzX z1PdmNJ1Mtx6X)+P&g}h01Hi(4mN8?rSsX$4RESw5-g&>t8`5PXt``;O0YNhe6 z4FVfZln`%S_Kl+MnsST+&Qkx7j;LD0#S2OD<^}~rcyL6 z1O|eedB%0uUpIJp2tJj1gzm_j?+LMbX|B4#CJaJptq(c!KHYRL2P>zsqISu*dNoW8 zYeaz%iwuNrxNfObJK9)Xnpl?k#`KgI__prJ{C>+hyJB??unMDJFaBp7WM>3|s=;USFflYIe7#xQ_aVSGQej9?l>{ERIfQhy%4E z3gJqQfR8?EGW3``As}su^y+d!wydcm0OtY>EwOl7Utz>W?t-6~N$uwMmV=}yxWy3$ zml@Zr^|WrA1hpU-3{I~~+;1a&HDz#b1fnW9pxiT&upH%`zH|{_iFc&Yi2>JITLsu0 zmjdQC*PFdE5?AJaY_Ne9Wxj|w1~{rVbk7zjDlr(vfxKUruTg9{+XgWfp zZ2asUjA*<#TY~nLJ_SD1qe+cYhct8<=vEtE7#oYI8z%7JNUX}&n8%m9(kSgD9u6Sq zs+q?=;v>IIPB0mYrR~FM zW<;UMh}qsfgVmcCWQaZF_mhAt+z`O4HO^t+a=o0t8A*!|aLfc(EaMJ%Obd2}2+=L= zjUR0iY1{FEc{kc_AwcY~YLvaSBse2APXXa5P_c@AOoHoykX0q>3V!lf_CETG^VkHg zTj}>B436wdmvqOG@Pe#|s&rNWxNVe#DH}IpHlpxg%f)3xUM}-+b7WA+p=37+cbwo& zjzt$O#(6K0Nl{b~sy>X`Q$U_CXnOHc`gJFIge;Xo;F-9k=e<%M=P2QVo-TJx@N^{m zvVd{Ma1c+bNJZ z-Rww(sMG;j_13M{!V*TMoIr(w%x$rd=Ypq06pOjN0=PCciA<@;YNQ||TAhDkQ@HS) z*cOf}CIXb_CSYl(=K{0U9Ol}078VgZ3sa06r2?%3Rv^xEn?q`?a@azajNy84t~6X6 zvcPrLRjyrvhQ&`g*mUYE@2D0{DHA}_4Si9&jZdBf6j_w^f_XZzR)evSnRs*712Aaf zRY9u2)B%HYMD_a;ZJZ3}1JNy{TxURc!G~Lu#i?C6Wi(oc1__)h6~*Z?V**4bH-ANp zsnU>6nq{03bAb%UvkZ?Fokrj@a&2}lE;X%lEoiA3xtx-NBndA|4S;lZqhj``k^``9 zxyMn=1_Ju=kM5((tvvE^A1S}6LA>tTVjQv+LcNZP#v{?M*ymDXCX-cZ4Ce(`h5S0Q z5R$vDJ=6wLEd$~_8rD`Aw`Q1eful$!6|gGL2u;}td9=uHBt<-ytM5J@p z3qw}rN85ua`D=EidEu#uVK*Tfof7LfQ(BhluY;+i(- zt7P*6M1?d|i0J~>f1nVNIILL{V8!e3CX8JL10~LRYP^@d(F$@|uHH%$Fl5kc)YBae;ocoy9fZ5nGrCvZ zzB>lLJZHN?4EdoBZUIR`GTTS^(4T? zgp=U`eQw*jEjh0h#hpW@KaRYJ z=9X_G$6X6S0uy+oTGlEnC>pj4CSOu^D)439uGXtL4w}e^t+2vynCLQ;gcx8!M%0A7 z%eeDnLT(|o;bdtte$gI0U4JL4MXl9Ewr;k#D^bHDfj$xF(LJ-)bmzcT;Rs$A5FTN) zAkQTzWjdwzVGqYs*%hgBhw9aKUYg;(@p+~?2pMCOqE$FuTQL%>Rk>D%{v&t3Q7)bu zMPDP5p~rZ^#HCYIBC(f35`Iv%;Sm60u=SkcU%7js$~4J?dyQ!eoC=`5Pt)kOgjnpU z2Uyauw@Szza3p9;x!Ro9{jFBNid$OW8M_8kR4EYDzNhZTQoy^+XAA+@M$FmCMO6dU zgEA}0#rSL?e4f^*=U1G>7~%Olviv6*SWo`*)u>8M!t!PQ--m_@nMy z3;S=uTzds#v5|KoSGm<3UsO*LIW%trPcY!XTZV9cjxomK`NxZN0lkzI)89=2M!+=Y7lx@F2Xkd|GzO z3xrNEwJan-dl2&so;-R^J~jv6ZmRb*4i&KUyQ^TRQ)#lj8zYdPW9w2XHdE#|2FEQ6 zQ!rAMQIr7vG9z{iqj+ipV{Pp#z<_6>@QM``RP{aR$s$zBU%ka}#n2G`Dv7Ft10ZHz zQWgf(d@G44)_^ao?{V-ofR4g^UdxWX1J)c-Y=n4tbm_IknKEwz?f6x_ctmO;)5meO zm$M#z%>fy%aXt7#1$LIDx!4u{iWbc)XalgBVAb}*hlWneg+P!?o8U`qF_VS?kU0#6 z8d{XBNX5PZ4_(U*AS4b0qQ2-Y)5O9-Vx>)STI-T3zv;{O%{fGAqkjaf| z2buJcrW4R>0->(h=#e^dkZI=2Fc9sEnv_mndV`<>8B5fF8lZ3#FzJ)-l~)4$%T8|O zUvw@PMt*lhCwRH(1WQ8GR7=IAkx5psCO*eRMm$*bd6b0~GPQPT?93ZTx-APT2LaTj zdivsLh;2iYkoAV{!pSJ&A(FZz=q{PFR|j2(joDJU)j-`F-v_mJJCiX^J@WOX(*+oV z(lI85Ha|kGzOjANvT3xhVo-7xWy3=|`=b^fFU?>R3UpoY(ylJGCvAb+5*M*Z(^wxQ zV!d7~+i#MT`d8PF2(VvD+xL*lT6>r&x|nuf2?TluSH?CyGqtz$^U6Q!SKx6e$7hriVji1GPw$)IubE2dwZHQJ+Su){%A_5)(0ZUj= zM;F^ma~QE!e3P?hhg#oAr0<0V&iIWI*%19QaODpE(roXjHCi8oc?X4^^?_8d5KT;(O* z*9Ni%rEchnC_OfcfjPVivaCLb1-9YRR9Vplf$@Nv9<79eS95>nB zD3-HqXB|11LJ>ok>QU_y9YgjRLZYrY(5*GmZR~w15p%2y1qXW}T9{(PGxW4j;&u_2 zUwClnq+6V1SU&RCG{v9Ljj~X29g&$7;ABD(Es{Y!=E&<|fmt538Iq zzXuk>%!OePAx=8nNc6Y5QfIt7q2v3=mU&{FniW%uNgrO+_rO7rNft~~E{N8|I>o<2 zolUu=Stk&{aLfzcc4Av~cYF$zLKlZ!n&bs-32XMR)YLPu9Kd->1*4@TGHM4xcI)n* zG02FJn6#g%j!kEC?xoUXY@1nsgSfE>NQ6qXB3p&$Whh|mry-@;oYGAMtc!fd`7C{! zQx-%;jBM+rAB{nNOv`|T!F^m`J1M9MfX70wa+(pnfUMRBJ&ZS#&^y?&k3KSX35B@k)@Vr_A1md7nIMv9-UogYOc42!ULmm@{-b zZN(*VQYH0XDexzy)Li4{wn^CPqY&XjS1lpKIs$df8`5?$86>zuISPzTv<`_iLtC+V zq@X4lSMgg_V9LXl93f3r+kB%0Rgft=O~KP5nc8Pi)mG^aLZBj{LE*@Q4!BI#z{YE3 zmOY#X3-SqZfRA9jfnl8orQQp$XqbGP98|rS^We2?g^<@E;-a| zOWbIolmvsYWi`R`tW&5+^RSN&n$h4m4G=%EZX?dDdeo2w!|LOv9u-CO)?W{x(y~6< zXGX?vc}lp!^Q`zGnLMWzZ(#c2Y3Zy0 zgi@9WE;;2Ll?P*RCh1;(J_6 z-Dd1=sX?GzKw*?>n)B09tAaOpb0~cobqK>x)Bpi^#FwH%(nRrsQe}J`eAx zUNmrJIOlXL8i)j8^Vd(-Aep_@JieP7;un#`Yko&CbH(a~OkI$v`v&}Eo3y+VRWS?a zB?5pZcN`jGR$H23Fl9^h&c!*5g$^Tcje0pj;x(R8xWl}H6b_@M@DPOf;}UXa-mArE zoow4Y{g;KDqd<=K`lY(kwueTz1yCOH`~cGz?y{_k8}<8hX82p_TQc z?9D|3fl8k1DhpK!CU_z|4|MmR|HSM~Gj1sbFKDlzC+QeuF zATG=B4q9zZBHv|J``rp*r&-lwXfl0bv;&H1GH|r?OrorVHe?15;5Ml!9-%QpL6ahE zyg^MR|!txw%f6bMmU z6YsW=RubZN7bPCgECR$e%0#DfEa@i?o+zMn4;hP0nrt67rd@r>nM0vPV-)5KI&dsg zh*Am&M@Lf;NPK>VJ7!;u!t!bZxpkl2ubSxyQMptIg3xD1xa5pk;7s+WP>+X~0mD%D zp>NvqoK|XCvnjDB``W;~K@r!lYaA<*1t;qqS5QdYUAMn#Zk+;o=5J7x$$8 z3>r=L5R)EF;WaoqLeA5sxl0zO4}}X9ric__zK$Xi905JIp*w9~q?a<}bl_C{tjTvu-&xhs*6-cK`qu~Z zk_=!`2UM7gHz?StKx6`l&o+a{%{qdD6M{*xQ$wKuc9G;a#A`R}v5Ds7-6q&qB#okv z9Cdl`19V{`FzlwT`7gXE2F!~ZB9H~eqNK@}7#q0Zb|nCyKATsh;mPuJ>v7Eb0ao-? zq~O5@g`G?}EX$f!m_3Bv86^bgVO_?71@S`VHZCu!eOY66W0cArXGzjX?}6(CZDj-$$s5@+t#4_|U!g(yud(E;Z}ffxsN7@X$lNT;vt-1W*|E; z%1ngk%g!e6kkHx=2e380-ZT3=TItEd-u12pgf`~B*GPW)TSXskY22P@Xp31L>?)O> z0yjm*hTpmJ$7}1->K*Vz5E%T+(`q!X@M;afb@hW4F$Me2qLY0qOpxYnT|N%36lXd1 z)?9JjR|WceCdDZ=Sv$_WX4OM#q#l%@F1!UN;5+F#whDR{p$s8=2UfDbbBJNxMoUE_ zDh~{(Z)H}{Q>Hc)4cewlQiydJZGqO=!`xf@v~pl89fF*Xk{*u}erZ;6E2YnR+I5)GUM^v;2W18<~aaMCZj~@F~ zjq5?QTSt$Y6-VI^b}cd~yF?8 z96+I3)nkRDu3Fhs<~ni*-ltrj-sH0zuGHFWa@YNAiwnw#4A7+7R;* zs2g^aR3UK66|}n!wfd*R+g@>^BLxp^fqE7KhTqejRYoXko-0epEmLTS*u4?)%Za5P zCwrfQE&F{@k&jB#{tiSnMtrr|dQ^bBG=KqxPBTm)be z7Q_rP%t}W)ib8PM%#x(qPkwn0A#rIA+qDk>qUy~lRKtcKr`Gb~E!hT>kCe(XEQ&8L zL+OZ?HmI1k{p&pB$V_h+&Fm%0T8NfbS07_IRE)&I*z@m86PJ~}Z6rxj>1RBCaL`Dl zW+SYeDj<)b=9aJ`Vf{xEwPv&!`azkvR&`~*_(_D#w-JI>S6?F8We)mN-BJqX$Kfl} zMY@7)w8&pL6n&GR$9O+)#y@BIK>u=u3XLjKq7KqEp> zDii=vDpDv?14soRh)^m}f)oV;3rdPm&@CV+Pz@@IQVBqUP>2*F0YE5}f{iGpBBD}& z6o3r^B?yHgg#r`;Ql$zs0sv5oP$<$0(kN1cKmZhgAxNq;4Js8vr3$nn1prcl6oEiA z3dE~GgGwtx2}%f3DwF_?AguriKqUYJN(o2>2#Z233ZY636rn;OPz5RtD2*zkKr{l2(1F4K#CL%C@msTAxKh_qD1``6h_p1V0*wHT z2q4m}AR|JJA|pzaBA`-aX5$TDkPtF~E#pV?C4=Hz>n-^o-n zI8Zm`Go*^2BRWlsiSzkMjSYl`FK!U}<0uGrzZ1VZ6Q2y=r`h&+%sn;V? zLt8f15$CQx!R4jwB&EdyZGCVGFQlOoTGiaUjU3{x&6e*Uq`y*faJ6 z3HtB$*~YlQ>AzVIXL(FWX(9zj*3zr2u8>_i{1+_&pHmS=e0v@T;B<4)!Q@-eG$K7aqr$1=%I>%Sg8k_Z=xSzAPc8BeF$+r7zlp8WIr*p9e zgtQJxh`zvb&R{5#3QqD2s4<4Q2$zO)PvZ>|2u!|W*el*Yc9rvSSIa*EtQA4#F!SE# zO*X<9;{p0^jXH2nvvgrN05_2C(-`&(>@V8=$eTdE-F2_J99#2^{?rX8*-dxF9?Ql$ z??cbr{_NX+E_Bc%#h(tHHU`@mK7`=< zKIh!y>>KsInjTMb0#Ix$A_&LwnFgB}URDf;bJ#QMp2tj6o(6xak~YD4j`4%VO-j;3;5Q4G45*d5mv-{mA0nfL=Sv9Roc`~>dwky;>())7+d#m3 zhaO-KZMCtr*&fIpSk4c09ND;?xxw$aLbF_L_H_G9ckk{0Z^3xX_tD4jH3~_^s2ryg5n8JA{426i0XIg5f@gg97V9DwqXGSRA z(br^C@e|EX`0rl(-+j@gC^(Ek+%)R}qy}XA6JM}85u9Obc*S3L$IG2z-SR`JPdfKFxw(Ub{)ix{-}krfU)PVBfxcqT$soX> zcbwh$`t}CA3zX1?5r8{I&A{SJD^||=CKY(HtO8nm%NvlveG>JyIe|Snc-CLY$*`p;cBl;jvti!Q^xvWD)%6y(;+X139JVn9nL*DY# z)B-__j`bp1#eK~*D=?#VMmVBwEQx zDfWkn1oS(i$n=hi?=PUba|lP{MNPH)p!^GZ7qLK2ER>Z3sN)}~ z)Yi<|B>ES`BELT2e6G_N8te&eaKmq$Z2ct&bz|eaH7Lhms9DDHKDwS6icGhAgr~}x z>Ni}td#fuD)%G(1>cR2GFRRRXUNT);xtCs)$$CbwB%&+6%f+(=-X&zvHg8`*@}^+? z!*Dkkkxv8ky~68yf)mNEhG&&C`pE(PhFH!>WQT@l?AzSCmdw`)dDD_V9cm+nBf14s z3@ZAO9PP#~)sMPS7$I4Hg9I7`S^x3bCZQr&X^jsj669fA58Bx?Xd>eb*|pGjzf29N7= z7Gw=EPYw|w1H|P~IpP{`A1hushcMCNkP8k4L@)p%7sz!126lZkRz1+8!oHL>AbMKl z>H?u=lN&Fd8r~(1k4n%S@UHARc(m z=5n)_5FA?!si2kdf=?b{&kFX}99`I}>X6dn+}~g^S%L^aEML*4G@}udSP>NfwNzZO zmTxQ?1cF)LLbB%_hjU6YjIxF)2)?e=5*8O>)$Lhp0NkIaH%+vik;o@J7P%3TQ8V_c ztfc%Mi?52%KBuL?8iDKN)Mo-aOVg_qx{kV1v>}jH5okqhm;n`Z6;RM=6-R2T>Plsg zTe&|e=B-KI>N`oX(KM&2W@LB+fPDRX+&M~@maZ1@gaR(od$7VSz@}MlS}`QVUg>~G zs;P18kO3ctfUJlJ$xWzSv>#?n4ksB@vhl(MA60eDO;(c%Vj;jf_uIYo)cdx3bi>to z$<4&flAi(9nL+UN`{0@nfboa`J;dVqajNI@nW;w{9=txFj)1G<1Sn(lVGV^oq_Nd{ z7E~3Z4w;LRYqPe#s^%wDjM?Ctm$v}VtXIwv`?u3-^MnN5U;<3lZ$A)y9Wc&Gxo;sB zLs`ip5cJ_MrlXQnG$u1r-B%jg1b)IRG!x?9@$p}r1Yd|38`h24t#CSlA3I_YE)dxX zv-aEWI47vP>wn){a&_J|$@|#zzcO<_ZG_s+rk9!rdSlDE)-*V!bONbRtBL2Srt`ul zb_?1ci-%uzi#G^-lxS;YSU99b^IJ+>AgO z6$ccNMh^ILH`a53nMXm1*4(uxa>iz%)mCdrp*>YWbx8PNBHjQ~K&-#L={0FMuVIpb zf#%eG8)W>6v5>FUd^zuHlQkaere5Q}I@Hmu6i`~pwmkjUTO8Zm^Me{-(*f@K5o;eu zF~@bre%~5roc=FOtdy0nh(Rm`zo9l$MpbKzt>(J~*J{?x7HZWq2aS7DqtK^<9ispK>WS$S7XezjSmS-Zczz?UvM+~giTTkWdRyms+M zzlm!(c|?8>M)lkaz__IF1`C|9YJrcq!>!U9O}1@)%4JteMQ(e15zt?5iwtGEo?0qh&0L+LWAjlWoNF&LKXrUc-gVvB1NmV=?XK*AaBWcOb49*Dl#{lX=W>c-9xY|M$z~*6 z6(||HvqrcXac<{S(-6+#3`XXJXXjhFyig^#Vzj7D0E4uguw0-iws8z zp3~(Nh3jb2=#yJj;W8Y{QV$J7G_QnymU#kTSetG~s7aDfIhcpk&4+vqP&*$ITJX6d zhRW>+wVRKbN84k5@Q&@x=EyJy8JKtm3urtN6R^6*AXJzq@Px&cq1H(Ok|@Z=W{%5% z5iv)Bl-4q)wi4rC6{(f=hdU?ocw>Vi2LoCkQB~)($;|cQbY%IX&O8K51_Z()l7?5` z1Smq;Iv*=al_%8Tw{vb#eP*C)uugiV!G}++gB~HVe-J#T(*43N36Y7Pce(ek$DZ4F z&pg;PFlOR(sOXb>jDFWnbI*!W%Y~SP9pU2z5H#z&(^1n9E1$ChIAo>6u?7jvOKEE< zG7BiCp|~6mODrWa)fr{1iAww!&LuT2?@nKxED5s6>IJKdQev#f>tb=3I*9xyDPUti z8y25N1;4r%_H`Y~U?>mv3PL4xP*!el7tcG#*PB8Beoojb{OYf_^YJ57Fog(1MRRa)M|moPvOTFbX;w`?6d-~(3u=1ld3);*?p-v^lACdHa8L*8-Cj&fscXl8kI<2q}` zPP)!M;Fm~KoSYHya`0&Mrbv%=^()9)l2{XC8VL6D+h(4eZ6I9mLm?_6%~R==B9gpE z_#u{jMdyYYVaUtMR%x+fP^xzizLRBw7sAx~VP2koWXMpv0Iiaik+HGi^vwqS%wIr( zS#XD{Hong}!Hq5X_iuB*?TlZ(!@0J3n4^vIWW}%Vz0QyuM*HC3Y#LbSjF;~0&hef8 zGl;a)IsR2dQ})K%ZSlUn*fWD0UHQI!eA|CM>wR?VUkZOb69a@dCkZCWZF`9nr68IBSUCc+b$BYlU#)~^W;@qjk$Yf5fF7(mE1VM?_C zbkpoc$IPo;snG`x)t1R86haP>#w`CIMr7m&*lQ6d1q&`pL>EGwHu&mc+(370JCfE~ z>TD!LYWwRTS0-K8`W=#NZD8ZzIrSL$-N6nWPlq}l6bgn>mm&(Xsk8L_`+Pjzp1h_! z^bZ^f0Lfwq4Dy263N4BwWrqVmz9i_nuy|CkFd+DRywlrmBH~m;ft$|yE;iYG6Kju3 z7U7fXKUyHD45Ks7cV30ZC|$0$kqvGht!h4Coz72E z&hZVJ;Ddpdd1wegaL6PGFBl4e1;Dxi;gO&c${9kbDIxe`6eCc?Z(r;TtR89i6LBLw zK5a4+n`v^0ZRo^!57w_Q#OZBjsxE;Rqa2{3i6aMn)Uyre)sig(zZv*d`gzRd_XYB& z5!&-i)p12)cZ#^Ed1l+DWMcQCvS7nkgrMEU_zQfvL*xrbYu|M>=pR7<-yPXo07fOp z8w`tJIC#`onGLtC>O@>bXX%=4-mTYZr)^10wPx+%f3$f`n}e7TD_R%FhhFg=h`Y~0 zkCwTn^JAwm*|m&fuolKv6I{U1Fh0^dBE_C%*dTMWMXp z0|LPCS0uezRblBSDs=&LddaxN_L0{T1xmKKEXcP!LsQ_rjE;d#3uh`kcGMFwYJ_|~ z(1{>$@IGPO=f#e5h@doNlaaLGDwtbAZzi7N==BG zf{D%ba+cdMKAKrUA7D}Sgi%W14FNR`kHd-ML{0H2JCPbn4erZyOtu5`>Y${p#O_@t z;B}K2PkJ7KdGBpp{AWOcK)PZ8dM9K)OB51Ls)9>_7f*7L$?urvABO=*b{MUEGQ+s0 z3jOt30|%xL=1;JXDTfoX?F{xy-BpE2ZUgNIXSN<0;xzc(NT`pdk8QGp# zAa9P8$3nXW!2+er7R^V%VIw2+2q=Y;nk{h-9v#kE;<{^q!&LEt;6fM_ZY1023J|Wu zfn>!8civ)&3>};wF^Crwn<1ldBa5jpjL{o_gl19312;+^w(pUrM_*H@K;NAWJwU?I z%_MO<>b{S)jtgWvy@BeDLEn$5Ap_GR%#`jW$L8Fyo@@$;tNZ3j`KYZ@)u4VXHs~4e zNG?5KvH>Q2#lLTulya_c<%yI9Lntocdn|$BLeLjtT#ZA?f*I2s)`Rt^6t1>2s2U7O zXi}DFy(4t0F!@4|eY+c$H+nDiZ(j{qmVnQ^6WzHz$Yt+Ea=R6Wc^O1yeM&G@JV2(K ztpQEbDxHr69lDY1drCbWf{)x)-i=dCM$b`#u2mAkqszfm)Lcd$$6u1+WAQ_Pm{fQRVQkxzr_&xuP-h)m&v zS@;P%MJTo;Z7#}31GG<@QGW4)+9N6GI{+uHo0gI1gGj0Co^p&RBGW~gtm*Nq?1QY( z421gvsmEB3AG$d)ZAFWEa5EA(qT#JE**BysAe57>FiQhz3e&WDJA7dkueyraDUBSy z!2=Zp5XPcFTG_XucBhz!n+%jXF4mbrLLa^n`O8cYR6~VP&*x-1;tj6nii?aKTX&Cn z#_(0ux4Y-weeVZXnA;j)7a+_7i6w$-j$SVib&V-^-#}ad^#`Ag+&&wgl(=%W*`Y@R zA{NA{!bUuu4Y`AlK9R_ATq7d-3OcJj8CG{tw}8}X<#ZxK{cH_kpn;1BikFN6L=`k2 zj9??Q&>(o~I?uBNHswdON_qpfSFa~JUY|^_OtOe8IOzfwQcb*>K3{6c#Td;>k=c|D zN-@d@Od)mkuYp(R(54A_@WLX=bvCfvk=R@u|dh) zJsff+5hpEP1!2(xzPK$6I2GX!Oj$settm}??GSX)byO1T!x0H1TUdTRo(w0m>DL2&iV64-mgnZi&&1G>R(li1pk< zbobGIlEn%;CDktrCNaKcK=HhMypX>3<%R_7cjNTOH0BI|<##@Q0l=&m5QLtMWvn+_ zt;`2%6pJ-5yh0&BJECo?%c|R~*eI{M0(b;g5^8s$l?Au1v~b*ys6t}7oOH6ojA>I9 z68g#2jp442N1Mz-rq`;34;WLzV`IA9g@>_;GdAMhU^2Y$lZsk&haAqW+q$p`nfcd& zay8Oss*OY?(}u}IpH-{eWn!9^I|@jTUJ-z3QI}R(6*0oAE114jK-RpsPj@n>vYSwP zKo8nCRkH#n#Wjtd!zn1Tw&a$O+rfipAZf06OWLCtidK?HT~60qf*N+?LUF_l7K6qR z^Rcn0K4Uj9rMMu~fVH9PtY(pH(EXQ}WqI9!S(%-K!XoU;UiZ@?jGqdMNm zK!tB?hDM{yx<)`J?MX>#p2kD!SZ#L(@i-276?M7W>RE{&n61eJbF`Q4U@O?f$*)<{ zAGz$`tnM1xappg5=GZo{agIsb>6euSfKXxWDCLkWZbQ^&N_y*b)5^H46$bF?)3hWa zZI9BDOxfBj*Q(j?3ndd!D%h(N>jON+cqMPDniMJ%-FD6oz_!|(e_0SA(DROZ4PM*W z9+EaBpB%c1rYobBn@6pqj>hS1ZD(%-{F&;dT^w=%5-*5ALxXZrXf<1hRbs}wf&19h zGiG2M9tEm_RC#I9@E=fc>)3*3(Kd(`Sa^d|&+Z{fh7B97NpCY)QqrlK;Matnx+d6A zFo4)daU5t+^V5ml9XWkTK{~o2WM(57b1cUc%h!g?N0B}>jxsVTYZ?KXMKsS9GT@w3 z5=)rNKu!)2ZxH2~E+`V6z>V30uPEgZ`WB}p)Y@l*0+^zrrwqZSDE?K|vD$H(jf5e7lQmznLZGGNUR$d8GQ2Z zN{&=ucUEHmHFF4CKl3j}F|r64MU( z>szD5fT>Z+7$_6-Nr@;V#R7D|wOo)e5K3p2)#dBrL}Q>cnv_`B#`;wiiUh_-l^MlJ zmc>AI;?7BOqw;6dn?VyGa9SY@vTE$>DiWyMZ(9l7%a#nNO`H{!f{U{!D5FB?>hE)7 zAR1qILOB@Q4Smk-cPZ1wAgK{O8~~`TT-7-A1}M-#K*vICB_mCRYmf=U%F?&R5nONO z(PU;`gr~S^NAXnH!-r?9_SrO0Xe{+as%dFQkrMz!Q-f&=ipKcipJ|a()k*U}EwxDL zJ+^G)J$Sg4YZ{^Mg=FfwV@I+WD0C2}&1qGXE__lFA~{-SkXB6Nrv;cA(Am`wAFK|* zxaJp3e0FGgI$Td~A15S*SR=Y)K==f9IZmc>C7pvW9muN*#g7Qu+A6GotD-*mmGlu^ zd7ie{jSOv%c+#YE* zi&e3Y|HL^1{h($?}Y5m#v4OO)U$I?kVNs2178NdYcreEVZ?9J#IaK6*tzT zjBkB()e;JewLxw5cqlw-+X~H_?!x#D#QNBC1A9n$oj!>MLuA&agHC6GhC9@dxP;pn zd*Ic?tMUp!u9v_Tx>4^6wUgBJsh#1v)?S@sE9+9IFcI%BGBZn{5vlo^fKa>A#Jv5S zpAQiyowC)P&PQ-AS3_`P4THoDOr?Swe=cSq38g%FkCdVaAZRss)W~sxaud+x4{|cb z3JpiX1|%24f}2D^tm8D{+Bp-n)Ks7%1m#qFXS!jhmPftgHY322XR$$Tw+LK`bX&y`g}!UR(1eZ~)BSep#(fL{})DOhsY&^(HCr zF@V7DFWT?LXi45=IE;bXWm+43mtq}Istu*ed_0$s(HfQ~if9Ehl`pkk2j$TMd?XNW zOp9Cc6BSWoAci&QRHC$6oC>~k@2GGRP`>Y~c@`E`sF|m@NwtE^rGuc#VC@`zgYVjL zaSl|QicZdnb5f!V7v}+T&EqNYu)?vWb&4s>mK?IF+{d+tB_(`B5I+$YgjQ`pAwnk; zy1JZ_O2!7aOSPmTnFX(v)m6~YC|A(X#5fF$kDY}>hfFEg?y%y8xL&l-ghwhl+C@i{ zAu=J}?mHIW?zF-Q-a&ezJsFj5GOa{b5Eu4v~xPXiP2$!4_O*1ZD5p$tC5NB`bdxsV1vAZ(>=qT#1E>-*nh$#>j@;BAPQKbFid*tpErdTz#qHCesrU_pd+LYE5o zn9V|YwVjHa$?J}y00KtB%^nNB*}SwH+Pdwvy~Brqiu^$UAVpgeaeY4ceXMDEyB6qnM zhk7`}tPJI-87hdU%D9HoCIV1stxp|=5r9Ji1_Ra5{T<+$Ft-<=OCh~W>&MYaa{4ED z=z~6#gSx!FaveQkUB_ML?YI1CeFy07a|V$0rZ(HWai)FWZ^yUYTO*({zg`Y=UL+f5 zz{ucF1(X&^bjpp;ogmPNPA9_8_Y zp{QfM130YxOh++MHQ|t_S4xK-5?;Es1~^gHDu9}mK0y?-9PYrRu)t|Hy{l^M zP1agJBqQrm-hB~OrA39@JkC$gW5gNX6l6d%h9oGF(3ip?)b}Vco|GFt_BQjEWl?4% zFbIqCkxJ3ZfZa}?3}>L(OV8GT(ko)8d%b@m!H3q8g6~CM^QL#-j(E+TPjjvBeQnnN zX!CD%`~oN!)biODql)5^+wvh#>sW(E691#wl<9w+8RTE^r;;TzfoLOxyvs7aNb@~q zhf>oDjLP3);6CZefsS&be4-me!ZSqPF9X^l>3yL~i-ncOL$v%xrvyyEG$|1~)$CUTn5)$;>@--GG72pr)#BiIloDrXsY5*yIVc6pD(aS;8D0J`UoXuP$Hzz_azlN ziG2G+%NTjzHWFh^V7b7Y0=KpCFMgr7x3RKc5gS-vL)2UulPY81Rf0zW`2)ImBv|I! zCPnNW&o+Uqt*S40ji4&fb2LR`GWg*fB0?(HA;Nhilub+K!!nW_Lt8T|HU}B((qo?c zmkhglBz~O8$17V9eb9(u$Vzs~OR{S!>34fh^Reqnp3Ait7`FMr!-zfOdIFU>ooRI-N#EMNpLFq#^)9TH|?QGXa?D3v~>75j5=$K7}yIKFUe zH>16Ur~w{S5l*+eu55m9Y+??cpV-8Mak3P0S$#@0A5TxwO+SG{{D z)@y@4GZ*6AwL#;N=qtCbhv43Ptc90{A-QZwwX~N(p&ip?*4S&xcUnB&FeNG|?TZS{ z7ZBsZ>USaJ>5s2qC%cm(0-rOtdPC9#q?{9HlAOy{yc#?6s81a(JmRlP>PF&%e#d|@ zW#hL$H6hJ;PVob*Uz}v3@W!qn{N)NysY~oE(mSNAECMEYWR#lZVqFLo1kkT%IP3`g zWmWAK0eAw|P91Ep+NJ&PN3n8`5B{SCn{mX<{^SJI|KG!^ek? zXhTiGvbt{4mEg2dY2s|A_E9Zq#E~A}$6TvP7!kV*0+4b7`*ymA zA4bHq8{u$lMO+-6su2d#aS2>@NXn#ywA^W8er?L|3hj4H1r;H8ycbe$sLX>*gcV`v z1Xf#Og`=l2r0*}H*P&JH*89{`5A6(dt6-l9IWjK-hf+2Z#E;Y-M^D zt%oCgUiNy1l#L^)8QK3+qxW2m}T{CsLIO*#Un zG;u3KOD9fCnt-U01*!F-zYeh(zKOLlk%N=(yS=17#M6A|L)8Wlg`hqnzTaX#g4N@V zD6>IAIrI&Bx5qxl`oEnHrx;TZJS=_~&5rodey9$)u;YT zjdv&Xy8)BW%BMpgyfl&uq~eWnnba~;kZum$Rpvs3s-lH%#sssGrJ9gYnIWME8W0LF zZzsZK!h?t7wF~kiT^g-)jH@csgr=Da*-~{{{h);OCZ`avQ2S%PcYv80HoZ=9(Mvsw ziM8;L)d0YqNAOYS{rVNcKTyf^Ix``CDHw&>X8wm`9xqd<5^(&0;2?B5RDj{cLIU>6DS?RK(oQ2J zqi#5;{15f2Sx-b7PKY8|C(1bIko=PS?r8vtL?Y!MhlU4rYE=}zi|tEmH4}$Qt;;L~ zAsj3V!EuC$Ky%*2P-~^&Zwugp2?>n|DC(q&vVqvLrQ8)%(cuZ2(PZHB6XbPl_Oy8M zHj4<;dU2DlQo2;GhEX}N%9g3@7XbRJsj)W5-2lyR%OKKv%7E>l1S&Qw=N55NF`FUD zyoDKhh7HYDnF(F%QM0h2%MDP-X$0Wq*PR?lfOUb@L`2lazV8Y%Z!9PD!0ZWQ4rL1sWMXRIeR@ zmk!9nWkE(vTef**^=zcbUb;D=*=zu2dWwA!C<(&sNGE~U_i7STml%pxlu$}oT>yIb z>b2aBdiz|=DN&GQj`4#~y(k9yTKYZ6FOIj=YleF)7~IxVp(wP_}vTUl4HIX#584 z;5$(f5l1j)d@diTb`YhJ?j{=6PsR;NS3T^)!VQn2fUQ@6SdXNGWP>Q(s}~{%&Qu`% z(R3KK zEa_^6b%Pbx6tRlYw$E~^g+n6mz|7_ue4LH)3rPTxnGa#$@H4*0+<1<{DC4HqI%On! zG8uJ|7l|Mm!d!4DNYX{QfDW(-{qoAaFf$G05wcH~MiXI*&wD)N{Y&MwOj1eLd3KIB zqka?%wBo7T+a|?a4a1i-p|2Arv0p>#rO8qh1Kla-Ha>tEg@j4h6j&P}f-Ds+2!Ov~ zWc8qU#G3%x+=SDF?`7Gbe9LdXFRB&HqWsHAwKzoBe1RtSVQE<6S*c$KQXnUW=q`-M zQr+Onl*NPa;0UVfs=deT9(xC((IeD)()@J0&Ud`)opNdVe^2Q&pWNe^H{yPXVD>mx`fs5izfYpK?p+anmrxLVIW(5dLeSEZ~Q)pfx>yi~vDxpRjLJ$L81jB4CGEuE7CTY zMeaJ)P^LkwO6YJkuO;&P})d@r|pSOEz%e z+y&DOvL2`~9ckycpGJ>Wkq(^58dUe4Ptt53P{8#PX#w(L2M8|hS?WfY(1OLs+DslN zS}Hc?``tzW!BT%|m%yPuKQtai6L(Gg1LeNhbv<#XU?62M>+V4Lyg*~~t^7h5qs&9} zpvljTSC@$qnz@8fe)W&WI3Y1592;OryzLeBv#RGPBtEW6v6_5&Rcdg#;I`>ZWFn9XbG+^(ofzhvYg9&oo)~=fZfppG+rSso-u5jOjcGNkn z$P}Fu?||Y7eLOqjcFvWIHxsEa2i@t0c`I=gI;D1?pl@W$jVgxIl68@qLF#{k51TmY zMn*}5V0i4Uda%G9q@oLD2rMdtZtM-&zhMlLrO}wOs2rMvYN`r_az|B`ClA;@asz5% zA@?n+T!A1O5Exk6+S6L0nsWQv0upj}vZxEhr#~g#rZr)TdNqR&H5ML(aR7uB&43`N z`T{suM@pc?AD6rA5he#E;mT}Nc=pr2c>4***!`2;13G)Ir#Nk~sUFGB`e)wx*McGm zNtJ#+p%6S|-8=*1l`O9YHYWCZ8rVY-AI)l^hBF)S5b8Shy--!;BR-h~vs$M{45-U8 zykN+(>M7kB!5fBL5UQ&93P2f8ndyP@JBF-!gM}A5@hytOb4-!26)!Z5ln&=!{jl;Othl?doON*e);Sa%Nky-F!>H2dC=R^Q*S{N={?OK zzW3dS=#OqQde7@*0*S&Je^H7apA?B|CPLlMP9T z$Ft-MhZ9`#)MF!eVZe(Bb$h6hE?fZyFuoQbv`q!344^L$E(wC7{m>D;jTmt(PXl7# z(HuDr&*)E`QI!$(7Hh%f91Qfurdd2zQ-%|@I$J>z4%rJw4J05iZBD@1`Q_ueS%&XZ ztal@j9_U;p4QKpwj&a!*oMEG=Ex&WWyEO0 zdEt51`Wk1wy#2S|oTbvhNU4<0e2%%Nj8*6{qZ3IR!6HTJ(SB8yd18NJuX4OQ%iN+P z5(FU))icKQ&n)6Y4Ob$H2`*T{Eo1~NUTV0i?Lzl@HB!R7wdIM)CEpZ5^=?RnaS}|# z!_{w{dHV)OpU&=&B^}y$ffV=gIE2y3c2M!2c+8-q?d#GU7s0{+SZtz0{Dekf>#zuHs$1bPSUg| zNwYAef=p`?W3>3~iPeo68%w9RI1vD#D9{Blx51mL8#0M@8{cjy=JhP7S}EH2gHKB$ z8{-=@Fa@`^odS_(F{DG#q41objqRN1=Gi*{cVi|9W*LBU(fvqGSz{EQ6#H!KbGJ?* zl2o4y+ERl;4y9$&JY$j!*pG*}3YDZV9YOB)+qi`OU$FiKz?2;=!7wC6`AA_y{7a33 zk;&mlfNA+zQG1LSHgt~cCQ8UzP(nm@12Uv3ZI@bdt`$-gt;7+}7_bplDJv4PB_u&z z?bS+v5d!&5Fg1wRlL;^KaekHLYsrrK_Q$=6#AX7WGv1ni3Mi?^U$^ILWW)+$-_^1u zD{eoT|107ch>DT^n^B!HgVjuS{Iunua;k~a@V+?!w(^uEDfB7}jf9*|8Ec=P=@C_q ze&PkO`o!I8+f8Xgu_F&)d(yrA63 z&L+|`6E&o8HaWJIBk-*ejoZkE%+%8cukk79Y@gLGzE*`uWJS(Bx?3);S#wOZ!H+9y zkT=9fM@XQ|Z3C-bk5vt+$%X6ilgLaDwZtV8u-0gGK2#;+IaI>{KG|NR&|S0{QzzQD z=%t1b?4sz=!tKx{h*y1}ELzGaYv31Rmi4ElV72%nmAEWJ7R1=iyjV`=9Hk}@c)4t( zw_=l@P?a*;?;Bm)Hx`U4I1>h10mXbL#*+iHW2h%rpj4Me-N%qkvjkDiIpaF4tOiJ2 z!@H@=DnUj!&m21sfI(3Ict>a(JzC-pDV3grc=SA zR76us(@Gpv6ujlU6KMqD?h=@i8)r!#gX4@qOjEiLBq>^bn6L*@Rvi00cs)oUGYwBO ztMOWzp{Oc_(~}ciodLKV`IYcH?EQ&Oc^Oa>jXNC7327P@?-v;uA)Nz?jO_v9W68`EJNb&$OcK=#Ol@Q>`%dJ+up?KLnF@sWA~*%;?w>Rj@H(fQq34 zgFhUerYt1UR9=IFjrh7)?wO{^Dj12=xi0>EOv6p6F7ZFQKwLE4(`{CfaZ;(qja`65 z>XiL(VlRM}J%fT2Izw5Jo<PP=<2zKll;>UhbjVB2-QgS1a@|pBqhcd$0EnM2yup78*LLGnqQW)p zi{abU`QwTz(mG@$g!%9nCXj)*8hI5mBv2@CCTYqSDab{kq*b6~o!*N&rtWW3Qob@X z1JU=JU4n6-EyMt25IMR9{h7&r3h6*K?cUvZ6w_CwGGc|Ru(3;OCQoV)oFl=5Y1a5d zTo^bY1U*p+@i_P|mS$fZu+&j=DR>!*75(0!0?aMiRGRlu->Z+#%ME;RJAqhMtX{fA zOYn(vJ;aPjl?(R2G4sD*krBonC^VAP*1FVS#S2 z2Hbgo)q?bHsM!Fls6a}@lcoCz`(=07gS6p-m70?EGAPguxK9Hv$%>Hn968u zagbrw*VG^uDXe#dz{q%ns;a*UC<{2OBlb|LtDkk<-9RwZFzbqtbQ_v=g#*p_UujO^ zLaKOz0md}24M;&$UkpKaF~WMG-tF_mUaIu=l;N&ZVsQs)PUz<*Z**B55mh?5IxGbM zn?^X|)Oxgy?Xq^uqbe;|&Dkr-XkXA5t>tWp8d(G9D4Y_pryG})_z=+Q4L)inWe_-1 z1++#{fqhwtxiYV-r_$#g+WD9WM^IG)PqtR^lMG{ zuPR{Ecu|Y%>aQS&g$nNcFuA8vCOIAfk#2q$+Fkp?z_%I8Tamo;X|$z>O3W$Qj9L}l zwsIyC`yDB~`AW*4aJ2`(jXBRxC|>G@}#@y8CxZY}r%p1?&jYB;6b5K&5zkh2F5GOk{+6T3%+7R%s=ABtGQ z6qX$+5d<%YH+cT}R%^4*;rro12meSa3;rM?s0;jk*Y|wm{QaiC=kysruiyoK z{T*_P{&hc%Cv27T5JqHQx!O^)pu8QW2ry;fYw{b8Zi9@)eRt)-N2UmyL0*00pL+~s zF$#~LaS18QHwAOHP%k~kddsX$6O90WFv}ka}y2ui>XQEjvA;h&;E0CG8(}Dyk z0Q;QXq=<|5P!1zKK+}y5wgJ*Hj|I9db>pEsmmpH`mJM~fEjdao0MKg}F>NTb)u2kc zSxm_UVsPFbJMxJ-gLvWg=1_QcNI%DPRSv|2)JM(I)3RF9ly#P61vEvj6OU=`t%UDk z9i{Qkby>y$C|qv?j+uBmz^bVc)*INY)>rKz+?(>y%(rW&>k>1(yV<>Z0E9+Wbu3=5 zYdqr@*$ua#MysQzrpm_vR!%62q(&fo_%kxg!c9P_v(~?G_Ds;5*Wn6~RF_6xBQ21o zPL7OdQ)&t-1R+E@E9n@uaXcOho5v|NjWDa6tUU?9dVvKoK??*`Q+@ORm~H_)inq_! z&uR_t6L@U>s7xFTXG^#33KQ#oFyHRK#XvlU>x*X0Bz3ZXJ-%TeKDt!~Hku0d_ zC67`V1KJb|Pq8s=y0_8I-TLEdLHE8=C47h672CoMilB8GWV27K@ zuLBloW_;ofu-5g$pN3x?Epx(%Gx7Zu4_!qm9a=g$j=P|}^J$sPAr*NFA{QYa4t!%v z*!mt?6$W4zAVG9$P_kfy#A-#!hgPlh1KBDS58Fpwz?-*b(bTtlYk(yS0^Cwd8MNL3 z@Li4BdV)23&u_&mbeZ=L$RrHO8^uU?GiJPNjp4vzVCU0qoRZc;u zDypjmC6QuFM)S_~rTItTu0$UWHz~AlU64;9UR$LwiVU8>Z=VJV*!e!@5Wl*#M3v zG^q(g^mYKABzl;7k`%hlRC8U`;za`x7=*jyr@p`TBZn5(WB$ypKKaETT@)#LkQ|D2t{@Om)B{am@&Ekk6WwD3Xm6 zxf6Mw5|wtLn;}*y5032MMO+VwNp92L+C_#5k2>3{nDcCL_6~Hd&%_#D;=^Mf=@k`C z6~S>ZG*bejv+=mKneH+_Hp;N>QtGSs?B5dP@Jl$#w;te(=r5-9V@zGx5UwX80;5sM zL|wXvuolIsK!gy91zJGIKnmX#q{I404QTP5^S0V& z>b*U|!4`GMefR8_?S9w4dcnusoV{S;aE_-eXpoPH;(0c}YMmDYZqZ^~#*`q4r^h!? zbPdn(%FH97Va(+>UzK2jj)nlx{u6}2UUU=t(nIVlRkyo!`VWqQt!Lv<9!q8MZc>Kh zS<4Pv2kJXx>lQ{RIIl3R!_@pmXd3 znLnclmQ++hLzq2Wns5l=*wzUn6?_6Lh)ag15$*#p5eu1ECPt`5Nf0Rz_RKV?NMJP> zRo_dcL_bi-yNCl~%Sb{maPjisL&kl*EI3_UD*k3Y4F!AhmYbXD-=56Z=2Sq!#1g`a4FG%nwy4Hg#di40RU*O z+&Zt-!CauZfjnfNu`nz3EoXGv0erepc40s)$>mH%=4}0@K@VnJz0S-MN>+Rze8&hu zgD#&9$B`B*6~*&A8g_K|%Tv{5IOh^{&@eu@j>eNGX5@Iv-{rTtq7 zB{f?5-%1Q2BgfCQ!HTCQrXb19^f*twWJscw_XX_*;XAmy<688kj15_kd zrg%=xS5$N*qlr9;=xjpLJ1Nv5=x&9yQ#S}7!ZVB!mr6QHFCdlLzzEHK44zm%y(-q2 zIihU~ty#xvj7xqIoO+X1h*BPHc$Y9Qo`em+fMl+a>fI7wq~od7ai(D@n=O`&+>cUY zW*k6Jymi>4p@i?*GBe)>>88PkZr|3psOs-=q&EpV4!SQN3Xit78j(V6J)VlM2|#O- zeen6igc5^%e|lt5 zL!=jDXaH}HU?86IzKuS=jjTvK1Y#Q`@*+)#Af>@!pne+x6tF_(WxB=hdyOeFDWTuD z>-Fu!C@!949f2->n!S_(<=c6pu7v^8gxHWcxgFnFpE_uuR4`Tx-#FMnuNY?W z#8JusY;QKXC@E=p%x%Ksq4NBYsCO*!UgXk%E8r6OCL$9)+EfD?S%V%@E-*&ZVKO8G zggFO*+-ue8C1*M8gs1d2K$z2w1+@4otSLb=Emw37g$`t*Co^$K%Kd>6$s~YM_lRlzF4lQ zbX`>wnjwEDG5^SV<#%epB_fqn~lN;EppB4Z?hN)g`AEXN3RJnVuK zgLXO1@ZyALa%<^X-V}cVMEPXS0-kJZh%53ctOLS!s8t`!N{`fZi)TpKR<*~9iYU)T z?W+)4h8`nE9v(Ul53b*>_f-!^1WQV^=~r|QMHdLqZ>3CL z614ej8(Iukb1AAXdq$%-ONV1tq9K@#`c8nR)q1gHc*d}1Mr<*0g_u_g1DPpRAu*Pq z8N**N`)W~mH@&(-uL)@=!OzO7sg4K|Am}d=-U4UJ0t?S$#d~<^7_5G52I&?&{qmv+ z8hLxqcHC>NsgK?M?>t`Vp|-yDgFfr~zBl|N(NI$rNFZV|;U>MGt>1D-&xreuCXg&Y zV4)YlQC$~7hyddsK;w8w2dUl5abg!)5S%w7R~_jtC$jy^7fa z%ez{)oZv)IDAM~^C|UhBB1b2Ro+&0goCnRP_}I=z>}6|EZ-RsZ*>uDlCIF}so@$2x zc*1@32x!PzE;5#LHV}0ILY#~upNvoDJ*zj#inf6~s2qKq5!X)ljb0=x2rnDr*@o3u zoOJY*74}DJFqs(v2;~h2c(i(9bPlPTg4(PgUYmk`tb>j4_kl)7;h%;F({Z>=H7{15 zBrm}%Qi7oX6Vh66)u=F=OmywqpzNItRglok4wD?3jqKZ&v@I4A(ai_Ue&AHl2?R-# zikIAT%RmrW_@E&ol3}6YsDNnv0D@qW zJ;3+rwaqR%&$LZ55IR9f5jx3QnoNO}&1eUwo8&O0ij~6Z7)_q{s747Dh#Z=sJnnoK z$gf4QiI>MSMsHG1GX5Bu z5~rU|mu$Fiy<^mqr}IJIfOGJ_bwc zuOPQ?edINPmQjp}wns-Udp zLJCZe;xqc;EW#l>i^AzK<{j;)Nc~fsp(@)up+6As96T0>1$6T71I8@mVt&xV86#+&mt& zS(XF@MV@$u5Ns)xA+r8_6&<@b&vQZ&^O|R0p zw4__leC`bWI|=n}G!n!DVI=U!>!l8h74h19^wKlX{bpdG)WE&%^$53UiXC=vAqOcy zcVIKahb?c8ht2oB!bV6CYj^e)7*N=mpbI_?UOwkUVokgdTWI(|E5v~00k%|bpzbwN z+u~lX)|5J#FJ-x9r4e|_wm}_@9 zMhIn*0f+#)Ca4F6M*2ilNL1911XO|az>iM*R2v}#FYkT2B~d(!1VS-3JZ3El2UMVe z6|)qo%kN)Op1Z`);-~Q+jBk|XC)ZMF%Bik z))PqfO>m)saRSH7&i*@ZXOxPKB|cB8=g4jimb@v++N1?)y+g$C13>*|$VZCV!`a&k ziH?}VBO|&%5QKNqJ_7D_G1tFaAtx0g4z^tC4_#@Gckk2PHuE^^tx^p)=Q`l~?lX(X zxuabB;n$4?#I2vf?kxc=Bg4r^X%+#F_T+QfyqYPn(S}HQ`>yGir{ELgdI(A?%#aFo zA*62K0`N*GQaSDk@Ut zT=~sYL61d>RpEI9rY7Djz=rL60mQk8@d>l1JENhcLMZ~H)do#-%V?sBUxoOiC2XW( zvJ)?i=oEcSRE0`kkn~W2znhBF6%wX41?O? z)%!C^V>EQ-*&60bcLmZXc0LLfJgDj%by5j-S4IYlpNjoascu&W=(O60g!C$v)+qg{ zonu}4^GzNvEb>}n=gb>xaw z^v~<3p$`E~<+FxU(?2;sG1IQx@UPi-fD@n&GunzYK&UT!GpkHnh-C zn2b5CO(Cud-C>jHAL6w2=NT!f9@8ffiYb77n7-E3rKE5rT)!1KD=qU{%o4d62DkvA ze;A3;@(C9`9>CHgI$W1T45@?yTpGy(FpZ#Q#F=N^A_L(mcuMMUv{J@U;3rxE#LmaY zBC?qco+r=XlPcUdsoZwJ5Tt3vLJ6DCO6;s^G~?UGzHaMnst0IfF=jwC|;q9Mms)tbrAb4>aG7TwGr z3mT120fr6^V!(40^7>VXP!xRWo+>bbli*2ul|h=T5T0gh>QwhVA~buC(C141CGQHO%IxEjlfe+vTq+Ci*x4zNj{<3Q&|>Ul%3_@=|K))sEJjG1R2$L5*Ti zF;Pq|Ek!|U!lJG6q^=X<3M7E{X0S#NM*xcPY6qRihC~Ey&p?I6W+*4)4q_raY{*J4 zjdI=DB*4Oay+o!rb{MowkP^(oWCOt635A%ueGaG0psB)*jNPDL0y$BxZJ=?Roj?e4 zFfu9#3BHyO%pI+?%mQOB2WC&_ikgPn0wch-#L)w@Fz^bHaYk}5mwcpuIHG9GkSvmO z-6k!3%Vk3BW{NN*AvGin)_@=?EaP>xc_q-L<(C=1kLVq)1cK5~TNgHz%77df<6A1N zu-S}W2bAFEOvFO|Rd36TNEBk{@EUs#$7LTCjsq8~jLt}8bOb@+VaXAGbv%^k>43mt!mnZEokk&;Pm#{)0Gz7^ z(O7BrrQZjkG^1WMU8KVu&iM3F^|+`h-#NIxH(_6V$TgB(8ZG;J)Vii%MdUSy6RYF{ z;;@FAUf;ga0Qq$({WIA^P7|8Qs|t#umx_C{jNxfMl_8F0e1N6ucLmgNnCPA5mvFt# z3&hQ#Y-}UYVdNt>oc3lzgLsdoRQOH=Oe2NCOuIU}Q zI=n4$)=wfHDmtKN3-bJX;pt=U&r+^8yFUO+H(LFLhuO(NY6Uo0@#8jQh@Ko*LQ=Xo zF{$({JocUuN7Xio<1-*-wJPMK(KH^=v1G|j4>c-0T?>Mc;j_hWcZWr=;OMOG0k26( zPPJM{Rup!S_PMB;g^9WFIraaf6eyo9< zHYS$Q*y}pLi`L3AcgX{dg3a!(!A_*g(P>^iNNO6aF5wM9P(}JF)Cq(O>1IyVn8M&3 zg)y?Au30jUzOl$gvYQco9#-ZKWjZPA%3oEN55Q9?fR4>8XhngNiaoVZiPD1aMG{?l zJmM8Jxve2w@!Jo)v=s~h8=Rb~VFB}*cUGvV^iF8C&y7xW_AGA(<1D){>bf5mMOB$c zcKTHU>Q1n{ScK%0xL)$Zp1!=QKwn-Ei`rTa13QC)_`;eMyNIvi)7ZuJKg{x* zc;80mk5(070c3PjYiZZYC#vBLR+$uyY1cgnp4*uoQGg>@y=K#b<5i zQI+9AcP+4$hgNpvuV%ftl6T5G|68BpQ#X@R+FvQ{*6ECv^cEi3XQjt`Yi# z_W@!+%~v6!CqWNX67;RlwWiFHwR;xXUcH=OY|iU5wL?1vyRoG^=UGbeK=lCo1DQ@s zRr#2dSH-~7%`@&wxkIs)EKB*+&Va`+K~>Y$$ABgW8MA#lytV;BOErh+xPa*@rM+|gP^Q7%hWC!HlUgRn8>sRy_s z`51loO^Ht%Dam2Cx|+zMQ2<_K~dBv z)df=D6adb2!U7@Q4~6}Np{EWRmMrpaA>xgDzJ_v)>$uJ{<&$h}jkxokG|wIBiLcg^ zn8%oP<=|Fp1L>Ntn1d9$9(?i{zL=-{kJ>uo@3pjTCd8W?yww(`R3$d zI3|Fvlr&+}WT8Qb>^AEw#S?Uu2eRB*qbX!5_>IF11YDTb^zwXnez;3Ox7u#{h=^S|3F;CudEt)r_1uGUA0%0UN;lp9byF zgpq!&cT$8>T5_liXqJmWm^I_d_ARp9&!}B8$s`DDR5O<1E0BC-WI5RqDfZk3O$!8s zhCq1H%yo1`(D5gI6qM^m?q87Fk#?h$^C2a$O~4)wuo9-SwW*baNn5k>F$s#wSJQ<& z+nEn=XQP9tQ$@@~X@SZKq6U1>klN~u{N@vJNiWBcp&c9!DSgJ;lt-3rE3J5%IgF7l z1E{*H0{huX^Sem|No|4qL7v-ttqul>#Xu{G2Q;nL&@{*q9ogxCK>j3?4WV7kzQ)SWnY38tCsC|p^25dO&Y%Y z(r*ATxUR3!F=K{REK#BHT2yb1J8z>TfKhmfQ}Q}hx~!$pxE!0D(kmvNUk)kx*W$v? zh+9e?wkUOZar7!;qsZau#8onq&~P+yV~IYCy^LdZwE#*zV>R9oUO_xWZgXIMCaGcz zAe1mEiRTY44@ApfQ~6v&WWyu~7$_Mjp;>3fUn_~!8s21FaF#7zTK7rvDLpbM8mXJL zP;2~vGAkS?fGD~Z1XH!qQV0kVK*(MR28*D@=T6fF>_ZMT0um98o&}yM-vT8A>2T>S zT^KmV;cecT@>9I*XNl%G z&gN?48c&}6!OlfuNlXvt8n7G&%uX#hN4RFjLl9IP} z@LgWy)!*GR1|lv*$P)%YY--iQ zOj7wVRQv-Gzb#AB%;f&c3hBP4Z32#?Nfrtasdlrl7+OxBHDKI1*$VWDq<5~`+K_^v z>*I~kI5@<4?z(fnIi2X^Ok(#Ne$y^+QQuvib;kbDSAsAi+H5N|)Cet76s91VY@&pf z(H_;)=~*KtEZ1$|K^e*;>CRW6DO#e(ykENrfJNKs>x za8FT&1Vg|hLkbN+iYbkERY~!n+C;s8a8mn%k!gqnZ6OFMChY8=r^SP^q4(G-2@gz_ zvr_Az=2R9U1k`Rt<#G!Vtrf&q)B>yxQ#Km)91O~X!;+6kM|e@z%*O%n*x~J2McRWT zg{pPTzDZ6x+@uF^owL)*>*YL{Npq5b>|QI8EI)F%=)Ti2t1xKbWk@+$y!&Z#gj8W?F&mzJ(A7 z8YjzUk_B!a7c1xa!d`Oyp00x81I$cBc9$I^-7TKMm=DXwK><*z(1v)2wa7TVR)k_z zJ|n)SxwYHp3-NbT^j`wHjci$mO4I<4!-era8b&O(%;>*X4{A&D0p5mci1RNI8DQ)t z906%Ik)rf1G@c7I`%`HYT7;})@3|IwS&$)A<`Rbjb#RGPD7dMyy)q!QCeOhvt*$Ve z%YJqt{nv+-O&2f(rgET@v_~`FTf&prr11RE->$M2OyP?0Ks#I;7kA5VOyJ|91Qch8 zB*XMsJT&=bJ5T3Q=mVjP4lCA1Xq#K3M|K4h0I@mcfk#XVSy#KH$%{r|<>LHqU1HFO ztfaWE3%oRQCbL4bC#R9Dg^7u!CJsgdiOutgj2%G7!pKAIWg(xOI;a5UL+MN-1s)JC z(+KeaP~m`85NLpmoVe4Hm~87WG;aY^~{pG z(s{8OqB}R0sHj7swt>be&heaD*l=VGQApaWPn0e|DNqxTfi$jH*~B$>?qS7RRFR<$ z4e|Wi=7-qhL1Rfw~T`dLm#q7<3A%P@qMW$u#S^P zw@z9f>VZ)~%=sq(lsE-MbH>62!yH0OXk#h>(V8oA9!LtZf!9_P0m2J~PMB>zf_XP3 zO>)yq8H}lp#aM+vg%xQ{nfIIZtE$UCH5E z?>~LXpS|PG_qO}3xaS*do_)+g2XmRJYCb~}yP9?2^PYwQrJD*RNGuSE8&+`9Q2tIU zEP2IGNXJPdzuMj*kz=iKZ$jc2;V;MDQ!7#}yit|M12CM{YUg~U&YxJl6yVF2`cvl` z1A+n%a8qa&GO}wyP3x-|Kqg({rMiaU*}17o)eRwcV~&A}TdOxTZl%__Vdjz#F;Vo@ zz%e46{DKZVVIz2*5Ldi#=auH++u`B+#2q6l06^)+DbyZ-;}c)Km zo9)3s%cY>q5Hc8~4zyV|k6WHoI;vk#L9xMyW+1zFK&v1MRL=oLT4baOTH%F6PMDmN zh|wtOh06(|AU()TiTu@Yp5VMM~5osPSA#k&@BE*@q!4Z@E^aCNZ8Ppzmk};sM zQ603dMM2>1qv<(MQ~Mc7%6No*IYrMHo1m?GhH7+a1~rG6={v=V`@1XA`M{4 zRR%K{H@Pd-Z$vkIwi+BlZf{BrU!qk}K^{jIhS1cNw$|m@ID&S8bcB$u!M^gl4U|&a zk3&~#1de0XEfRzvHKQ}3g+$ZNl&(xooCijI z<8fi6nN5*-&Ypn7K(q>)u!DgH8Jtn&Y6c=HmPloplz&lH5uV@~zl6F-EXUSN}&)<<1Qv;nW5QF2xthTE$wljH^ zHu;A{vZvO@8u_e3Rjk(CWPnba#5Cw3gOmpNfha;!JiuZJ;8Y{vTpgg=+N8kw*;Yq% z6MlQvn2(>wa*tF?p)AO!=TfLO#(kv5l|6)el?Ylsr^U_@T=e5;J53X}FvFdC-br{@ zLOnIg4%j`FCRuRDHq%KUuSQy;8*p}dXPez3Uii&O<=LcFg-FayEmeLKL&JR?g;>yl z=cV_#Hz+B!^b;Cae*41if>uhfZF(cz1w{heL{d`3=@1};+LK);HYJT{B1l%2Gma!+ zl?J9Y(B6XsC1#}v46e0k!NC~YWXgr+>H2&diB>qxNy#8AvV2oGmY5fkZ9;-0ZUZ_f z>V@Ufj3y0@;)Lk{wR#x&dnwJ+%&;7hv3%p8LZRy$mKRYvu!F4Pu<@Ev z@y^$Tcf|rX&9l`@v`GwjogJRKj)dDKgW-2ntZGd1s}p$~ZrBb5*Ouvift)9QA8oQ% zn%8{Y6f7@z9mNb}3S27eiHZo}a%SDmyt8i=P4!D}%ba0@Cs}Q@qLew0qq|O~WzWE} zknsh$uMZ}I323WhD@eMbo!J)P@7m^WYak7#VILf!%zp4LOy6R-mguI5+N+*7DpT1y3*{d2B1%D##4_2PF^2Sr!BoTvh72E zR*8HPAONOZDE!BVDnldV(RQOp0>MlnaENws%ATuiJ-2@@0S_ZBaHHCQ(&9AKdDOep ze=3g+SB=Ny6X<3j$0u-LwSqmmWmy*qMt$gJKB8uiQFz;7--w3Emv<4mAJp=&kF?;W zNQqE!+}GF%)G3V@pMSungxRQ3K*Qg!$M;LHjz-> zOgh7O{(d3jj+zjkNgL}#3n*IV+`40y+kowY6sDC`ScH!wlbo$Vo%zUe0(k**!<|$K zKw}icWRG%8($uQ|5ntakv`5W(Ss`H7#_Cc?g5SQK;JF<~W-)?4pp|^EDD-ahZOzB{N`y zC1W0|04TwM0vSErZAUOr6yd_Ax_fvwt1D;+3!x?HzTc6`$TlsX35|78XMlh!_AZm# zQ5?NWaP2S*PbDrqoc5#A(^s_kXj%JPh>*s4ciFYngz+kiy=fMo6qmIL?8xFUDe^+` zbZC;96Rkumt5~?Xjvz?l=?oYttZmG2tSz~s`nz;bx^@S*gQdBFS8%CE*ht{mL_V;T z$q;vM`y zJ##3WL3FZg!F!|`P@*UaN3q>R*GI>Q0ydG#aqCo(=_5b3weU<2aX3B9*a9ZS%n~6m zLF#UviS|Iml?bH;sRx*&=fJ*A9x73;p=R{gT~B~`8JlK1QOp@4!9 z9Hi!3Ht78t5Au~yZF%BbwkR)<9TdRlg!{Y_t5xr1n-9j#+a_iLnH!a&6E4+A^XvM& zTG3P*H)?$zDGy`aS1}YWBtIN4RRzD2JeIQ6;?^kxG2#s(_RC7etHu+0gcNvw4l&vU z`9vCw`=I_Yvo0RqAjjTjK&!g&fsh*LsJcQVZK?%QkEbuQoWSpy2f`8@GA+{OL_~LNqU(x>Wo{xms}$$qk(+8R zLY^><0t{lt)A;~nOjDF3aDc~SO|WB2F;4A`4UUkrEYDtPY;aX7RQ}x2P}6V}5P5Z< z6||ecb=DYzjFF864?17F2+IIQ;%~jCstE%KlqhfU&`4uVKh5q7YWZ1xby(G;{59{_ xBHCE1TNVb+zE+x_tZTND_AqVEz5B27iXtceKt)g&{-^(oxgwk>NLc*h3V^#Ix3d5M literal 410953 zcmZUZV|N@1!**laZfx5pw#|ucyC=5oHlEm=Od8vna5Awmand&JdEWc`d;N#K*4}Gf z3Kp&s0`lgphUTmlUB6LJUTw1V2;>s}Jww1iK>5EKfF25hebzrwF$a6OO&mm=Mz>Lx zIv^15b?-yy?K);q-$&5GZZ*xWqK?XG#L&S@9;{UVZk;a}n$&J|ONu3f-nz9`vHW2d zF@No0Xb&o7+^Fyg;MxH3t_1I_Ub$K0hHv6^>f<{%-MUr4x9;iy_wyb#D%v}NtDH9=w`TD z3NW-?H8>@FGq|&^UPGohC^frAG|z$k)v2>NB7At&*qZ-7r_Xp7*0kU`nctfWl&NsN zm%bc0n6at`Mf87{y!E;V4cr{SMA4v%-RiUw5<6Q<=3)T&$54TSPN#wwn^jLk+x?Yz zL+}YyyA(*ZwR4MNDKm}oE&|vatSsk@S1K?9Zv~c_U%+`+{BG++U8}b4An~~iGH`2f zM^b-HL%Z?b6v|tJuJ@I~ee(98^K~1vzR7|r1JncVVXNVMvxC*DG)K7ORX`(#Kschp z22Iy(yYSmC2IWmR!u|#&s16j$j;bwS)TY82A>v`!LSj&cEmFKjb-mHbSeq1a5Z=fY z0vcPC*$ww`e(Kd*P;lR=P6&U4`S=7Fr~w)4f=MnkydNGk2v_y$i%~%P2u@_@xMw_Nx(ze2(po;!p zw~cMRrJyABKG>%G8dtM-cT2ZBi>j#@3TZP>gG;bxSpeu@n=iO&h-3O=h>vt5!#VM- zvn#33O$sUae!c-ZbsN6PJL&}+%&(K`w^}S9`6Z$YxbO&H@ltq)2hHc~EcN15?1tEt zjl4HMcr6Y_`Df?cSMIi}Tm~}`dBZb%YE1bp8A`|F0*u%S)tX48Jv22g1xY}whP4DSvGyZfN4c<}S_zw3L23v*R?w{@hb$up>*7{p1_aP=0r@h}4Be zd%eM8hrFjO?Kj%`e#=}oU~HZ3c7q0`Bo^uJ2XC{2K_o+iFsVVys0V<>$llMMRL#{r zYM+n@Shum^!Twv4cD;Tu;*r_ z7+Y?ABm(5$Sk-UZ>4xnx43vI8;QXBoVY;VaH+t%kC21kA)URc zeC^h^){9i+LKb(h{)pfYW2udBxVz5c!G!jL<^Cp%l#>XbcRPS}4mJR)Kx={Zb?w61 zBO0wu^7hLNgC3F_qKUB@Mg9kjvNQswzR>{?Sq>pJlu|yoDH;vK5yfyiMtIIx4jDxd ziQ_IhP6MbC9UhHfV2mF}?gczGs-b4bR8&+7MoHa`ykrIGmW z;rRIf41#nu_}>3JNicn4`Y8V@L=E$Kx(~@N+RG?8i!2VQgA$3uP!u!XI+q^(QbDh! z8QNM-tW_#PpKaX%!0#Ig*#5R8s}XybNf|j_MA#wg4dD^$wD_SaOXwS1JW`QgYdh)$ zj(`?GvDIBPx^)GsdNXa74YMpuCn-mmpB9D5FM*ta;T~oFuQgwJ%sXs}JBghyg*;6v zv4gTjPwg57lE5t2B|T41s>85r!;L7h4>a+3d<7(IykF0=6mOs3h|W)uz6W~#avUtl zCBgWFh`RGtpuO}@D~9`?U(xeH59FeJ{NJ;$XX=^o%oR!xHVetR*3uUlY^jpd)7v+_w!4?s(ad{JFO&z*0nV&g1%keb5aZI=uAqlf(-oAtN_RhMgOh&wy-sR< zQQpl<`Pdi$zpZ&D738L(jkPIA%5VI#)4z6VFjr5*+cm(!9VvoK>e}mR!X|GRI`iC; z)O-I%jc|mob3K~2mI0>uI#YQpYw;!}&VLo13g-__T?%^x;qfs#ky9AaiXIX?Aa8q5 z$o&_GrbR`OWB>{?P$ciQ@h-+vqP4G7xN63E415bbIlC~2MT-I{j&t$Omml*A9CB3I;99vXkc zVe;d=DRXf954mCz_lNC!Hv&XL&l7{TG=$g5MjKx_Z*}vt+cLW9tO4p68{X?eTJl3Q zrb6#@spoAbVbl6_5azOlj>oy*7GnLT2>+xKHt39g&4Kj)%XSOO=BNyeGNyOM^m zTv_J-dMn%cP`sq_uBX7~n~1fgJDg-C)?DQ=#n32kY7fn*NvWi31H-a zVc!)oRT4f=Tih75$nm1J=jh_4P;r;aC`6Bc6a!vpyGH1l?tfmzN@+TNYx=9N7=fiM z`R54FXJhm;veF~soPu+=LDY+$eW+fZA{(OI;C(G<@)99w?+O$;fYu_7!wXZM57LN+ zzL@QbLa;2x)!`sTE3~oap9I$}nu6V~=hG89c=6@mW@!GEN`Xn+nfn6cf1x(=bL{!7 z|CQ8uVOqtwLY^lx@>NY^wH6oV;PJtt7s(awx;O`LL^F3DYU@mYv;h{&rgh_{cX!%{0|Ur04=x z0A)wahtZCZ{SscVb>_S?gVW|J6Cc(F7tOLYgkh(VLvb%ju~TT+u9yMnt&_2lRoss> zC@?%7GdhvlT(34v_JsAddq}7dK|W4Ono|Nb`J8ns`h2QaQv1f&nyyWKSQ z*Vea}jTu}p3w_mF&QX^$ zTC@#EU3($C*=tz9HKH&yS`w;(cAv@`H>r3|cxH4(w4sju4dpQw*`fGf*m%S|C(1Ha zS54G3rj7poI)@eMN*tZ`@P#F4ulStvQy@mGV5~A!ar%D7HvEM^&14T&Y^If2RLXwR zuDBtiHfDAfLi$n^*(~K!E9S|_iOA*8CYm<hM`QLy?t9CPmZpKpKUA$H;3yDDx}8mK6vtf#G=PS?S(h29>WJJ|W&8Nk%tS06c*Q(t2~VyZQ*O@4?Rvg zN*1F|f^Xt-PDH9zS;b)?S4+h_@FUBkDEaFvdmQ|zNw51O_F}4jt(Z-fvYyvn_=C{= zpYdb{)-29(L7trSM{hxQ3DM!u82O(KyQ%Mdi?dU)E72T&--+u#{?i}^Bplp z!0f0CZ6m_cam2{1WbZ@ghY3M6Hzabb6O?rOWWHbwn%GHne9qsql_rBiI|?OqFp#ajIAB?gYXh zaiF7N(CYcok+YFCC*Z+THBwH`olKmxu}grvwJU9;1^Q&F30(w&*a+=ElRG4P8ptF! z5dzB32I4|_bkOm_eJuI(u3OoSdXMG_w--!@04jnx;Hv1{-H8=b2e!6Sjg_MKmUy{* z3!^lbO8z9ASpa!C9o#5~@`#X#@xX3REg84hA~@hoL%(ylFWq((gzE=xe#HKIiSar= zn>Y7X`p4Zieb{!~2Tz85kD=K4dx9axNpc_4Osy|K!R{h(k4VwsC{vQKPmD9X`z4ex67w@krh*v^IgWnmCDuY!CZi)#~ zxfVwWc?bV7AMJY8#aU^pAwVa-!`RM5;WvCh`(1v<`b~?v-c5S0#9sr>4qYN{cT3lM zluDRW4bTjLW(V53B*;?omvQy;apLUc{^NWdffD0gn1RAh7G3Qtoy(kJEyN@?Z$-M4 z9X|}UxIBK>ke2r>KcV#kNt_&U!icAoDI-IdXP2wt>{>w!8Y*GlW=sH)JA*#~^OqL< z(@LWQy~Tx?U!&L{9f$nQbWD~Jg#~@*DiE(6xUe5uWLUlAe+Pa{^~3xz>Y1o2V!d6j zl|h49)vJIDtnuH;lX12x{>pocJy9m&+79;ALcJ>IKdgsn>(o-kXAK>_)yrf=HNHmB?G=PX z3%yXy+wHzStzLSY4K1eKD*yw7pil93>G#H_GbP73z&AmnaY0Oy-!wn;OfvYUw4%}+ z+}`12+%iIHV)hq_#=vVYq$h>jpG3-B$j!J^M`>ria{v7G2Ne(3$y=wh?P%fK=*=B{ zUIZ2Q%;-79NXjKEN)PjD)_ImmoNh=Ie9&`9jJt%J)!gP@HfyUrPxedEH0`de8}Kvv z3zULTyhxO>DSA*ds94|_;~d<;5I&j_@O?REu{}hq0`aBqIY<&~2{P65* zuWu878-B#@A-Vu%+kxGhfpQUuZEJ4=Pca3+6jcLgGCM}HEoMZ6UeG=PdqGJtEU_Zf za~pc{eUoSqvx}0AD7G6bc4ReCUD9iK-(r%#fLozQY}tII4OvNJ(uy@FShI=@o}7*Wxx42NET2et zuk6GUk9qlj_-iLhjX84X0#N16+d?J?+2nsu^I13cfpLM_)!bP3izgRuXK6D0kGkgiBIbCKm*Y zmh`I@{p~c{FT!kNcAC|$F2%L$i(>v`0q*%9i&zqczE#eW!#6~QDLaO>)LNlF2nH;e zIA_1VhE}`|aZ$5l6%_f==*pQ28_WZKkk6i!AsdVn0c$f9S3?Nq4C=s=_v>d ztEjY)V-;22qT4Uj`Q6hOI|(ccoO{5l*K1LtA5-H#ToyqL%`%BhojodMopmT0zTsja zh)tWyk9Np6tE82tHLg1UNAr&`pBS_V*no(%eG8+$;dE-o!2A~86n3WFc##EMI^O)g z>Z(xRtnz7x3tt-!#02D=rWzI1&};js9dqj1nnr(QTHNoUtA3khVSb?yps4EyvZYIGObPJFDq%KK>vxLmYI?8~IdhY!g!zW@~g) zK4#F`biMIi=J|bY{#ZkYDW&Xk#t6$A;(Ic+(AV#xE*qW4dRy9@hH_HM7g5_@|01w{ z>IR^3i?skIB(q;c8G*=6ZfWWKz3Io0$>X_o9&<}I7}jBU>@Qx;Ps3Q#RQANd8(5D9 zlJ4J?xMdC!t*NG5@r|^xt%qr1yw(;^8mUNku1BzY|#}H2Tl4)^<0!%4kD(+hz_k3-)`@HSdrt>{#)N34BEax@4 zG|PmRW;zr`LLzottR7BoJ_J?*VA~Ed`0q=uu8{-W(xq4d0pYFm!a5p3oUObN$lwQ) z4q_bufYfB(siNVEnQ~{pzDPM*!c$9>rk9?Y!b z8Dbtk%y!LaIohJf?w;({77up&H%xKu=WeV^`%0pT^kQ6|YDkkvi<(HA+iDAck(ntT zRm}Y))qOx+YoLd%6cri1wi>9mRw159Yyg1ia&go+w_Ge z3gxgBd#@4FLyt}9jPqNQ!`5Mx0=0cy#p(AZ#q<@d=0bF9FP17LqX1a>s%*;kHw}Du z=ldnCge4Swv+gRoe|v@um!_Jl9?fT*R(~=MvcccfxKisP`N7)u*k3)yp6yM#q)bte zfDivB6W%7sGuRX7r}hxZ`O1V`=NOgud&aMpm!xZ=q^`k*YsRYJo?6m&e>LW)}IyMvMn> zjYD4$8}TB#Nqb^DL-xBSmDt@NZL=9FVd6!al22!KhSfi$X;@!d<>h=~ZWT!>&v7zL z_9TyS+0;XK<0zW5j0m`Ca*pD1k@)zYY_Ax=@89ME5UO{BIW$6-Ddl6A7>s^?6>grW z+1JGltf{RtM<6mN;%X%)&KMb!b5NXCB>?Bs{=O*_s#j~Qi?qwum@F>}&hK+I`XjmT zc8gnoy|bI`o#4;$(NoH6{^Yd!D-hp7{r!kf1(%jbb%Io`m6Zbf2I3z-XoyINlzHZ3 zC_2>{HZAwakKe0mPttkiD_%z)ftjgvXg&VOyg`T<;T&0GjqZ$X1O+z2i)j-1J40QS z@x7cZqwr3*hLzEcpQf(QldeN2IZ38e$F18-nuG)k^Xprin!DfV}gZQtDcRXAZ% z@paG$3LybD6?4c4&KzUFSl$%JTs7(%1*t)1mJ9YFN}%0*yp>atPYL_PK&(c#&m2JK zg<*^UWQ7pTda|#g=khDgZi07Kr<;ZStw?z zNZ4^%<$dQ~oY~oG^-<1>er`>%XyRpmb{5+p4hXmSbsvu*=$^QT%ZmSyJzHVzX8!87 z!V$v^o-2t{bHlieQ~i8RJzRS))3tbUX#iXL4@6ReH&FR;ZoPUt9m%pl*Nm@Yi8b0{NX-^U|QZE`la13Q?xl0bZN)}<8-;9eGFylR*v`*}j@H3P<#M@05! zo*XlmEm1s!Ta32}X=>-ntj#K9C&rvf#OD?F|0wQGWspZ>i3?b(LS8hQl{qn>K<E9AMHEAy4+V1hBMkwL_OT64yUx6rAN&4e zauza9P8Z&{HP#?n3>*;b+5nOjGTcNwEQdpe+;RgPITPZhf7|^xc9{EDzvYTG1vkXq zlp9ER%#YvcgO_%yXfnW_S%vOAwTmcV4zZ&Ba9Io2Q23&NY!x!`McpSrxV?~fb0Vp& z$@g|H7V`qRj6g!7NdI3=r+qNqcPmy_AtH)2Ix=C9^TN~$`Ipm|lh9JQ0%qpG=^CBJ zqa73nm``tFAy?M{;(q38ZoRKBn+(+|?aar+C+q6>Nv1QA)YWSeoy(h7bjU5sCqNyF zV8~+-6lPElD9V0iyaS}&;_SC5;n|x#c~ATlcf~sr+#TPZw`?F4a~S{1O-yWHO`gyn zia9Z!88$>2UG^2Jom&9`T^nyJvPMJ(Vqs8=Vd~r!qd>Dck%y;YNfLX~!QSED_lsei z!zd6kZdm{@oSGDcn5gCVh%T|&SKg&!QwyI}c~GTawa|GwcZM5`>Xh3Aol)B$h7K%Ij)(M~z@sdTx|PB9;1W0{TQ?7NpN)cC zf9zw6fmOzqBGU<$BN4a^7v3x0Jaf6x_AyBe>(EfE{NY|?46FmNfUgpaZ6aJS7N*SI zDT6_yyaTbw$L8d)3NtsI$UvWe+A>#DHqOe711?X16yPB#j2c36d}{d`DlVgfVe;zHXR zEn(z+$Ih@m)T`Z`Q`fL`44YJMuywet&XLtzdh4~s&4 zzCQf?NbAv8{bR;@A2ovF5fLsNUlk80O3;-267M>2w2AW&@(lFKL~ z3qJV5OryAt^ppFcnvI*9R&9^S+_cD+8{W6>E4r-AAW%mn&@P|O->C>Ktjw7zIz$a1 zl2{J??7Ew;tYPZ~`H4O->J9r*xhGu#*1V+zE#G5*4%CG9CbR&b@0cYf zi_nK=vZkB@@tCuW9=NVOUK7#QUyF8gdF~DDRZ{F(dN$8Cgsz2{jAL(ZWi=!jJVKk# zk`kNzMLWpMn)e>al_cKb1G%Fvq1qk8A}0Iu5=dwwC_5V`Asxn)AJr3Lf~!-c$sF8E zi^va!c6X(Z_i^D3ih6uj# zGq}M3w+3~$n-x7y&(hV?FvNdLhhSokEr{D_&NWJ(gE$!w8R{LKTvoF;lS<3qE2vTL z{d{09f{2SV6xwrARb{zMtbXnSl9yA?k=5E0iY-g2C>0eww6HvU*p(PLRHTeRfcKT2DGfH%d> zu_ZhSMA}xXnC7?dhzB_uUpXLVr<*q}2xUH&Mwvul*{NsLB7?hGf)c!zx%0j^b)bA9 z5^!@vy7U`hJ=bV@o#Emc!|-t}I+kpEJ8ua6SYFcz^M`2wzL9d9s|UimVJ3&6yD6gZ z_DtpU3O89gN_4N?thneSQi?22^-MF8i_AUQ*IxP&P4V~!GfYZqx@tvjSC0b9%}=Y< z!cT>Nhq6Hv0;%&{1?!=f2h|Q6HmbL zqSt!t+nJ)HyQpmgtO#zFjSpK3j z9;``RI(?G%K`DQ|bFYrp{K**`eY`CR2G^(OBkb&;8XBMW)dJ!X0w1-lhIG9z6j7TY z)4o;Plri&$oP={-C?sZ#BE@6SX0;FgkS*OD!D%H1@X`A7Ia34HISkLTLXw_%-ZEP; z=Gy^jfQQ98doW6it)4@!)*4Put@J;Bd;F?B~)=)qUeBKo*=p45IaX^Kk97xz2oJ8jJmzw z3q!5n| zs#er|3?I9SHQ1|xSvNPXGMLb3tQw2C66AIm5q=@*RtmB}$8WR-R%BA-3l<1v1M{|L zVr9zB+k(g^s!$bW?i;bAqKzmmPRIub9uE^&KsKPF|0 z$mwick$?Jk-O-4Jxjoo(iBn}B2NCg?C42^uA{}w7@#*#oq^nW6#Owz`Nq~=Dt4iK3OOf(I=iJ(7MKB`dERv(qnJ*Ih~Od z-cHHTG9uBI$be2v6Y(G6xVq&(lq;1q~uxp4oq;vcQc^*S{h%MUfH$>j}_o8iUI(`99 zJ#5W?YAu%xb1_rtZ()pZROZY8xw$DQsCRl?SQQSHckhB>b?F~^PNYKcZXn1EDcQ2w z_LqTJq>p-N=)##^T0?=> zX8O=YQ(PP+`h2s5pA^gQNti*F6B+OU%i;NE*LW5^C4ACv8g>lTWuHFT4)&4elVRH% z-$kA?ll390g?A51f6=2k&>M&ZRz>To+7E0^WJY_dL0D;#fw3;H+F0+W+8bA*ia+() zgt66cqUo9gZ@ZnmcqGg9ezN_T@(gGa2`FHyCf zfW#;mdBler7l&SPF6`Bf{CCs5-WV}74&eD0F{Lm~Yc4Aq35Q$loLK;KPGj_LgUR!uMy6DxI; zD-OQq%36Od*NMrZKECL%pX8tSNok8a7iE)fTl?oW;a^)bNgc;V59&gR4ILN`_I3-p zzW=!BYoci5U_gfQ;&58ci4K`7i#kv;KG|GqG*-C-?hlOxi7gW-v0993BN&k;MO)>j zv>&njhXJmYpQT@la%vMkUDw;|M%xHb(`QbQGWj6|q9=$P zfRf6`JDXiFO5-bTakhIju7*4%Be|V%=~X7aAXV;@*GyjhqMwSpJke;Q{B3A{Ump{* zm9GcSJdbC7q(q1<-TzXV&6peIiFRxMi>d{i(THEBf8<|d+=B8x9g?t6XI(S9e;Sz6Wg)o`HU4%De$BpZ0l<*}FI^%lj6 z<|7s#-v+iBW$R|=obr491%DHg1sit|XIf$dEhmcsUcaO0u~HS#s^Sn}#Cj(Wg7_wp zIN62PBX7;zF&SZa&5&D)=&JiHtFGVv#uwJ7aw?8XYfYCfw?`{PP4pKT=oIWhn9NJ4 z+WiXdZn@eRhivgFq>rEW(whD=jiUhj2lHpOHpxWtcJnj2g?&9jedVzAUv!#rSfME@ z)ZSS0049rwlzKA`H6FQ2xB!etHNwV{70m~SIM(TgOpS2!UrI_3{}~h?+|NaF_lg_Y zNoxj|o;Pmi_Uu{Z0tv&O7R6+unz=xv{i9sd^s|zhtj$eAzqqdLhFbl&!i)m9@wT|5 z{H}>2v~lk!l8!&Vl$hxG@>MnNIv!<hU2;r%THGpJchO@2wE4>D(-jFGFc6GxCo` z$d20W&a-F?V%%aR%DftR(!8VZYova-QNDJDb2O8o&ke$OxvI7I4aK4x z%Wl`t5?DpP7Qb`I{=Re^2Q3fqxv`|VJ^I*n{?9_!9FUR$?F?t!;^mahU`zENQuhOL zL!g#hkPzp|RUt3Sy8MhDv92FxYI=J?$dm_u#qOvOrFkRa(1?gm_BRw#4U4Qnn3m!? zNXR>^A0<73V!Y&TOGs@nE)S;Yp4D#;`QF$FkBWrSH0m%Br#mUh$haJIJ;u6 zK`NI2l8tlD@v2F`4ASijDAG21rs3nu?J0f3)Xlk4QOP%~phv{Kw69@F!-9<6I(&bb zbmnvzL>1z}rz-ufT8rkyc+vEj^2*7!uvk(`;()%05&F(oBd5!VBYMskEX{Q&`IYSLx{$t{f6FPcqqB)ats*IMZqN| z_WQ8pL|uv3iuVnt7ws>mRW!X2B1yoU&_f$1d0D)!@hP=D!I&~8NZzo%py*3XM2h*1 zCwA*e0?ehQGxuvspOLb6AW|E z5_S8QfGv$SR&Gh@z~F$|6rn@_=iO8@jMF9IV#j54y~$W79S(S^GKp={{#iO}MOD5v z-jywHVe%|6x{b5?s*O-8fOe-spoRW0+^C$ zRd^p<1#mM49G6e&#S2O3_Ew?4Oc|8YWmI3N)hHpvUD1Dzi&`DX*lzDO(hW6qMKpZ? zf)e=L*cUYVdsWmvn@8wFra|Z=tFL2grVbFHFrUV(_Q~J2cAX9$@+%WyQncNI3(!Kq zSTS81MUEdK`hojGV;CB$>|Sz>ibCNEpV;ziF=}jA!WB7ZONy|(4%Q9zmG-YyVfUsi znR&+;9YS%+5)8Y^P(b7r zUU4HOn)-OmzM8u~zjNxg2iki-Ul)Dkyq!m_J>iG&oFd5^3Kt^ayQQ{heVW7tvM=@S zN|;OkT&lm1tBawjH6p3WufHPF&w@x4?T`;V;shL!Z`C=3K7xy&DoXyB2^kUsea&M4 z5az%kfk7GA*!shZPCa4ht;#73Ky`>_*++dPV^6avcM$gM+(5pnL=V?{?NzubN7W0z zz11GN!|Teu7o6FA$*U(cj-yq!KyNy86k?k3b40S*aPSi$!;^h_G1}P-wc!}c`MvH@ z=zV`Xlw_kAFjbsThE12e0Lr)zK5;X#^Ti9%*fw+KD97;K18Tx~^DQoe!=FZfMP$x< zQ_u(r&54Gulc&m!=7kIekCKT82jbP0r#A&FwYWpi`t%5W4Wdg z{Oz_O1C%rAbpChb%gZ-dkEX0Yx0!3stR87C!J)jGVSQCZBEQ?pNg26u@H7a<8<2S% zHi1}oFU9VgYTlQ-z9IzAPP#&DD2Cc+mw}Br^s7qGMe2uTu|DDQ7H!lV#0(LO)Al$d zsdeNQ+P8rinXYTwv*em*C?^yUyAm}~Cs%F11A^z~#4@^!5gkcEhYF&+g=Fjv4{oVF0OJNlGUtrl^u+p2i*@>IWS^w|IO!!HUS)dtPv{Yl zK;F2@JUSoc7(Ke_spHB7V7w-yihXrdlNq#QtU}Da40@JvHLgYPSCAp4s+q44Ls#{x zII9mc5Fda~e^IRo8AEcWLA7;SxKK>4m0_uE6WCeqsO+4qCrbKPCRb-UfI}8D7s{AW zT%L!*)FAv9Eu@NvX!q&{;!(S$IELC!Ew}*r;rXybeoJypHu(^s8l7i36@Ma2uM=>- zdaV3(id6ITv-oqEvJ)bfNT6PcW&*gO@vsf)zNuT>SEj0S41&WK&zi+JV~?ta9nBqW zc;YSB0<&fFe$B(UcB8)dHUe>EnpAFz zO)YtK+hZU)&1)!&ZhfPydbr*V)DZ6SmY;BBF{LK>w4*$Z?HsNfe{2^>&&568|@=Inp1>2tG_fPz# zEdqWTXJOOfzmw{%n7C~!3wV%9uhyOuT1(M1p(NJzKkhzw$l^;X1ue@-GI0|8VfqmKDQ5TwtTkt z4?b`ISmbtzlvvt&b>2tm$na|UyZV%7ly0HdC2u6(gmz*x&tY}MpL}|hjzc2^^sL8h zCItrk=bLpAbJFmJCkM?G#^d;`_R?xgd1ltp?~8o~sOwusO zqN#~KoS)92L@&Vq0nghL-ZWEM@SCI1_xP(LjQP}<1$DR3BtmcNL-`%hz-Ot_JeQ}W zVdiNlr}mhXQhfqB1RHtrM@W~oOp7y!qv*s-v(^`cqzJzXxf|jgw@(!=eJGPSH))!t zIYqgy+XYPo-Nry<5`!d7>gx&g4a)Lg#3>o{6-ur8TdSn2V7(z+o;VEy>kl0ct*9{} z2u`4xufY{>3RLNC4$Ltp$uRZ`X8wO@Ngkr+jClC3r$URyT!1ShNH1IXQq&TV43uVv z7W~0x+I(-7f)8ogdF}Ptie%Uy-!X_vjF_a+mwyJnTZpLPk^2=Lddb&C|9MgBzApXC z;a+GuFmsNTUqQhnm<0Br%%GE34$gC(M0C7oq7{gFNdTFhCmFhX{ao*Ao8O&bYMSo4 z1vue_9#%f*$RKs|M6Lsw#c#gitz|svf{P?kH?Oa z%HJl_KwAE){A4~gTZK?5+FNbO5=&Fwy*3PDg9-}%)^S!Tic;HM*GMO)v1iLxwvf+< z#$Y!g{_iC>O%*`m^2OM%@z0f7Ipq%TmM{hxoPEL-z_v~nrG05ITnzVCURyn+L6vKsE<4h z4%j;(SN&N#QE58WvQY{AJz0?()+Dng7djc=mLxEZZRc1d&rrak*Do(t&H!HOr{a<1 zB~(q53!+p!%Sor=arEm~!-yN@6TNWitEYD%X=lbZw`6T^8hV?BMF4PyuT;eIFon5- zv!L?Ll{l{Ft~{gM#}go&4&frTlO<~jXuGVrGV>Uc(0KBaPSrQ3oam3;6#%BvURJ`8BMg&P3`NX6t6p! zqlOMRD}$8khtuRCeH0q1czgVh*R>7V@5L8nMqV+B|CUl7KlAH+ZeUU#%3U^8*&`i0 z!U|~9>;c?ZtY3R}|D@9K$zx`=(Mq*gv%-mn0$;B~)LeO0UADDmmQ%XM`s^lbq^LQV zfVI?kK5;vgKtUEPGT-%S5$u9sTMJ*kd}V#r=fVM(z&0YsYWZ@@7H;18S>BUeQAFn4 zxFGfoZN4IsKeYXev}y;UaQQeQ%(lS8$O- z_tCrR!JWX^%|qBs)b1Ln!!`>6Xqe(_e2>A(-22GcNCfSGJGH^Xk zW(Kk}INKp|!Erq`;#Z;G$n${BTpnIvQZO(#97f!&$EVBLSCvKaIejcS)wfi5CA)rR z5$)j0RPv-~y@?ePPdiyID6aSQAWwA?Q)_!3$-`UB25}!bFSNlBy6x;Pc66RexXw6! zhP+|D=9~gW!KGR7I#KzsmwxM@BWbkL&LiPH(GI`8rV|8(U+RS-4TdcL;4(Y;hTFeG z>)VI0Te3t$W@3>iw>BLgpVsY!Gp{-#c8f|87buH!3@A#pa57K3>@FXlpf2v?ce!b9 z7kF5(B-=cO_dE)c(Ejp_@;mwP7&)eKHX!9d!`cWwSUP$Tpvf)< zh{QoiOGdO{{_RCx7XI@P%UUmmQ(5=d#LjXGQ-$KIV?e1Me`2BN4XMkR0}|({aPO%p z4rxVm5n+Ljg&ZNCTL00lg*I8{7)CSjo#8Vjk{Cyxl2V!SDw?q1wDGk2MAAGL)wTh3 ziKyWou$Yn-!S}Z(4ya-NOO~x&LwxH(TdjHH4Z8JKhAnrcS?8`1gnRkCrf4 z&up>LbUrt(5s6ZK5q>+Tkn4?)c6Pv z+flU!!PVsvOvT|*CW{Dl{JiZ7gQEQC=j`U~M-uFmm63A<;8Ql;{&haN8fnR#HP)t(KxzyWeK)nJJJ3Zpj%zATai!FI%GS@jH zh-fv64#vhr1yp|SmQ{^iyx7%PF9JWQB3iHg+*>~CC#i0uHKF>eJL=-To^IlaCl52> z;&BLV^}RCGEn=5*qtIDmNFz~_v0$^NiO1v(hCzEULr7-y7?{Fyi2rkBGyxDav?mDl zzCH>!Eu~OEx>mj9qwbAWrOX+KY>R5`_Z)?uC+Ss+B>A`3+)3W$V-7&mcPHrwA=k?B zmTBeomWMm+Wb0aWHbF`RH@ek)i&jhY@EqN592mYiZw(545HYJ(D14RUTVC>+_Zm|% zIV;d%MC66kGFET@x!B3NH;HHjiG=u5G1D%qRoAs^B|59>?`T)BR{spf`xh6#}HGL0HyQ+;ekA7Uqn`TjB!o{Q7*3B2E1Nb^EaSr8)f2Q1k?FuVyeN( z=t8xVp?UTvwQfBvKWxwLHFQ}x-CbL7jP2j>$vv=tG-d+wKagD)MtgGVM4eWNAA^-r z)U3HCai_=Yib#?L*M|&Pe!6`BU-qHy5Y7i*oX2D#7{e1YDpQKwG1}HUFlml{Av8(* zr@GTR%$DFa&+@XbIJd>jeXX;~z|kW7Muv~rYNMbqQZ$T=FK|ivcJwiq(Wb*tRLaA6 zfb1aL_Qh4s8A_iw?}>ISynjTmy-*%_yhJDS>~-_=iuP_LVth+&D*}aW#b+^NBQ`Au zdDFCc>INBJ4apvg5irJrN-53K*IzjnNbt~L4Pyp+UZ`b~kkc`+$%z*7;RXs3=FJK? zWhHwD|6w4hN+4Fg3`v>%@ZswsmQh?)ewx9R4!ltV#f1*VaK`v*mW4&ynHdwxeHMe_ z|C7bWCJ$YYw|0majvwj!qo1q>d4%X+tn$Q4SZc=t#{VAxML@d0)Me?QM2=?p*He6< z&W`nh_(3O}+qvACkKR8r#JLsvK=6&|_67V6sH(D40cbqx6q>fAVD9&>ossF85#yUG z+}o^n4%GN!9CYF?O{!=)Yc9(zKd>Gg%@x=Re@iN}RXViVEvs+Ru(vCqzv#)LxB>`o-0#CE( z`WO@5WAvwG0e2v|*^%2$K-eUR*iQqL+WYR|uriesj{|oDF5FPfl9HzzfjFGVs`lg= zb;~zUPK$ZuB6$nSOJ=zts_YSggymL?B-q+Ii4Wn(Tdp1)+kDWLtpan>1_bfe8tual zE;BHrM!Su^^JH#${qqODVdpunp>4p z`k1`jd?BTAvB^DCgxjRYL=NgXM!2orRU7IQ#KgYQ8X?ixGM#WS1`LY40 z6qzLM^PasE9%mFlh3%)Kj8dag&hZpTWyNXt07fD>iH2#xb;#&Q6R=K8-!D73zP&Rp zQ7*Z@#FJy@IfO zBZ4O-^TN%qTMX1q+a*r-H<8y7 z*2tV(>`S~i1kk9CAzHVun|zKT=vC|AXk57Hp@O;y7;@nR7hWM~o(5XFPhiooS=KHQs1=w(NjBFn z6JEr)GXRmxCh`=JIxK9rVv0!gbwUh;rs`PZOZFAm{4_u^OHZ)U-XbKXidx`&Q25laE6Y`98rR-b+9d>_VLcMzdQl~1O3>m%$3t0!XP&=cb3;Y@ zNS{%0!n)#Sea-0g6qeZ*KL)9u9L5~4)HZ(mi9ntxS!7?uRUpoI7KM#_p~TZTOw1;i zB>O{SGfm0I_gKxQ+Ewd>%rbg0WQ#4suz>U2?{+?AQV_ufkbMTzmD_kp%e-3>RiMex z(x)S>w%GJ-2BQ5;ma>${Zj8vLLS;>t7y?E^!a4*3>_+vH!(!qB8_&DsO6LruM4MkF z!@Zm(y;Yh*DN0X>=PbiR@3BZMD`i-Lp4FO+n)AId;T!wQ9cFBBJW99RxA z+Qjq2q>AJprc7w~t|2JWHRTG?gj>xDx>WMs;5HcOT2?HMYDx`j2Wx?xU3XiZL{nUV zD|a+wnT^!be6kfEQwmj7LrDghdYxF}#xYNT6STtM*r8hPYKfd8v76e$l7NP_8jQNz zHPAr>BRprD$-^vP4w8+w2Eu*?Sp_nce3%Se3im*U1@W4Xl8nZ46Ffk}rEO*7NNF%t z)^?4_%of6`1&54a=~tf7nr6K}C2LZy`&M+9+^0soGPAI$a7k$V2C`HmLx)(c`_A40 z`?#Pr2s4SZaqAR8OuE3tJE-_nxl2B(nizM6MB9$lf{NmSXV4MG%@hw<7@$+^p3OD-{mT4!lO)@m9x> zBdTTOovTxx{pS+A7E2C3*HrWadi~2*(+Q8m=$NsJZy$EfYb$pzOHls!oeoXM z*X(k{f0b7`j^0W9FLT`@q9I;@9$#f;MGvoua|UMyxg)2}4c2R|?{*^*WV$Hrg%%A` z?{(X1$ptt&UVY{}n^3Ug2;|HzD(doYaKdXZ6_dM#X?w+3+Z$Y0G%(pD+wG6l??EW>gz5Hd^n`)V2?}6 ziark%r1pnEaunCT#VJ?!lnE>>?+O~_?g|XVn)|nGJXannUB!VmgRuh&uul}$97g39 z);J)MDT|V&^ks(Z#PjB+ts&nUsWw4r^DYViVsNK})-k6$ZYs`_ZXiLYDcU*>xOX1*yUrLz8b@w!j0G+z&_Gn%AaRROd=cY211;z*n=1QsqdeZT zzUk(C#KA+mn~aXL1EEy?|0?;{)vER2mr?H6GyC zA9Ot5lMYTkubbJ~J-7YO3N~BXU!uPpI~FqfAv&1@`jrva6gsoL(8p z@RV!wZh}!f3RFR?{DNca$_gk@>Pzl8If^&o4`q5O6%B?KrA&I=Ww9Psq4zu;Fm!`5 z6hKwku=B-zUQL4R=EJ!SMqLnCT0-?hQF@O~+4%@?ri6yvx-jU{GoAu~?kfit-R9%B z5m~m3vUd4p8Zt^-MvSk7ijLjuo|X08V3iGR*TJXAct#$k0F9dcirRK#41Y$MGJ;?jVP!q^`O=pGlFW9iE^v%CF=S%tR?YWk{luwJBe5q zn^IQV)+`;Ia@dSFy)YCY*1f=N%fLbA;sLwZ%hD*Ztl&;!np)+Eu93bo(suK#g^!0H z1Tg&M;Tn9TZXOk(Zj#O57P3>?QJA(OGBjnQK!x)&GE$is$DM}h&>j40c1l0cZIz*b9ZN_F%HoH|&o~K`D-4Di=$&N|}a}qc`5HFj9uF;OsHNNufch6f33lO~7qNv&d#FLKP74Hgh|0s0xL)Q_7%8=l#Z$ia#??K^ElX`}bH#9A|L<*W*4P%AgbL{P~$ zivkMF0)sqo8X=EIR9>A;S@vmJdfgWaI@Y=&M6_wq@R&uHXdsfoWXM2K&Mr+wW$qPN z7zMAFM8>x|JWhIdDwS{rWim?{i=mNRww;D@Q-kD9%eLjGCoNi)l4mn24lCrUhlsS?^WVVzJ5vgsAmhlyG1Q>(v2Xt_X-xG_iawG`@q3 zE!XK1EV60_=Zg(>z+BE@<|~F8nTEiQ`t&!1kpj#*i&1M-$4KSW7KlVj{^{aG)r*l1 zI3Y6Cy5pZ3Wmk|n`r8%HpyzDygV7!{RxYJ(iZMc{lF2(bWDeJyR>418 zxC0F^i49TEg9fDqQAoG$W5Wv3SW@9aF9k~cgz6Y^pAi9(m0fm|!doAynN+0~%xFwlA;@Rr-JBIVSK2_tBiRORoGcQ_+GRzRjxL51oZ zi-5U$8H1y?=SEie*&7ebchU!KNpVOgWVZu=$QvYa!b^$^;+|C%%PzymyX<;%7P2Um z4>#0`OKeg_t0C2=JC`7$iw$68_p8Giw8Pmd3|A7oxu0cIQ;WABOB1wy_~iEJZZ$N- zJ?N%vb*2>|I~WRL;^9L?ni807LgNpX;qXpT^+}%C%U(Q&3$0eQTtab)Hh8+2{j$PZ zgQEN%>OO8$dBF%FtV)HppFldryCn!=+-d*e5; z-JdEuPBVreKL(k!Xr*b-7Xa9mnKmhU1sz@kaDnwNqqk!!3U(`DO&64c{o#;49dj~c(4+@YzAL$ zMD(dRyNS~R8E`HsH`zjF1#NG3^^xXds1)H0WwRJ_E=gsKVZ_L*si1d38u3;A?J3Jy zD+zKx7(?K7CLSJw#l09d@CZW-onS5tzcGqp)F1D-E{J_KSCsY~?hurS79s*YF?YwxI2;&8AwJ&(U0Fl|xX_*tH;xdj+`$4WI+NWxJ=g9} zx4n=A6$DA_K`sfizLOY9gftAL(whwEn*Gad-OnNhpCH25(TVRJa_!afwQ)#Qq0ME+ zuS9wv5`vH%pU@QZVm63H%L&+L<&{CcN{Wjw0I)Tmy;Jv< z@dnsL+1j}}cT{Xc4Xzdf+eIY!aC@B2oDU7lVrUlFj4E$wFvfyC8YW>4VwNFWO9(KO z?{llPVUNkcg3E6(^<$vL(@j%~1?T#VgDzvsp)7dh3sxP9)}>4;9@~puq-fT45E*+I zPkT8AX|d{qWgFXfWW{LaE1a^I99+@j+0_KAgc{$sC={z=&UUd1J&1%OdvM3iaJ&r> zqxQ`;IJTmsHN6=N5&GrCvoYO1`ikW+11HQ+2lU!<#L32EzB zy%n^%GvFl96LiW@vgg^(%i0(cdOhTekdEL%QO$-05UNi?Q!_RvW{6th=&A9Q@b)uztiC?w3rMl28`necLMOWBloex=4m3r@@m9Z{yKu*ftPScRfrFCNM!AQ?GpzZy9p zI|vA!@l~ydSr-XxXR~wJ6C?{rOTj~V?VXL7#K}!wI)`~KV&gmV*RgIsA8yNK?At(N zkU@0$u1|s>9xnm904UV-~1z|^Ht88zDoWo{ar?jlc1l1+<9d|lnh#V$i0|#oXHlbCL2oV-o1&xs@8?AvR zi}n-{mJeU8TNQ`i!s8sM#O-7$G{y~Lq_?#(lkzm-Shzt=aIc1JJ*2B1uskxd8@W6} zl1CptGnI~*+;(X)vI(o)m!XZg*lQLuz=16(5=*TZH_jni1g3>KtoL zlzzKIm||qyeka5w`4ob@j{7x6v&uXY-kdtr+Q_r9I&OUn_aUgbAlWJ$zJRi@T|l+o z7r1Y=L$_NhFR1*-p+RZEtTS?`6NBc}qZs?fYMhjpdLgy80$warX$YXUgLO4%Ss5~` zXblNhO&S+oE4LxG`qzUXmW_DjfaTEr^x1im32He*C&`R>uANfUbz^=>GqR?Afh0AD z@HEh5ZggbZ6+oJ#l=V+JJP)|_RTExUNnWgT?kZ$`;}L zSrm^_tAkjb;mwZcj0@&JY6G%Mo|w746DJsXs$zG*J_gQaPUhK~8;A|G_>WwUx^0X%hb z7}==M$>W*cM>fL}1Ces$U8zvbCuQyJPM?twT7=g@X500QTUf_V8QZ%OWrn>D2 z3Tbr;Zpsm)Hg>Wdte~p!5N5_r=M2>wPsS#V|!G4(nB~u9@o|$_)wOQmd&A zeQ34u+USG0Aw^o0z5QtUXl2?|xWzf6^wpG6k+|?-M1ux-6DwU;OQM^^58|WYilsL` z)73GTAZ{>Lo(9N^3A$2QDr#nIBclP!Q_*o}yKkx&WCATTgC0w$3vj^(mm1li|*bo(?OtACFIHmxgHFeY1W?xohaqyx1-4-!SwH}Kp}F*MHGOIK&dY4i>P)s@uiAmW9G$sm8+i&MO!RS2og?ncGr75 z#_x9HPb|SegYh^%TiPhAS5&kf{6#=*=TqI$QznpD9%TDL9B!WnZL~r?>MmV8Un|4B zvIarTw{(ILTjyqd&pCJE@)`r`AuxzbI&C4>Ep*5!J?;t|W*+7cem+kL0vX+^k;w#e zJ}49KFA-%oS+3~_iaNe)*$xS~=f9Fk(h3lj5r zXVO&~Cay_EIjm^EHN{;QEd&7Wd|w%# zePld(_oxAY#F#zfl8(#_uQ9EI3t-hqJzj?m$#!DB?)6s|hF7g=gclcmlpHHZu$Hqo z0P384Aw%A1@$JQ$xHQJw0kC83wjSsgd1)6tdZVFw10_S+UeC4e7-O$hke;osxEy`P z?#p4qhSXw?&IQ@gU8y3f32OQX$~CUkDQRh`MiVo`-nB4F*;+{>uZ1qaL&ifBWZCGn zG!%59C&-FQOvVnnzBm#0QFQqU)=k~2t4yZ@p#!Wt!NT6xV=}_(7a{p&R1rs15}~FIf`JK z>`fNDlvCHC!GLiNC$7#zqs&}!QJGTEFmCOP7V(%(Qr6@m_~*+!b?S8 z_C`T4Ck+M;q>li#%9Q|l?>8x)kx9?%eN4ZawO~$q=axUjbw6+Y_YRiS!H%q2$VC3J4T76tmUf^^!I$Ydtpn1j{FiHMrT@WRmf?J?7*w~Ob7W3cE1h-d2N(h+f~M%1`vT8 zh}=q7pQU}f6nmAxgk&X$*;cQEBG0}J>A2MD#uvEWLDAjxkBf5Q&o`ePB;(RTs4F$z zXM5SCPV5<@LGYTi+C9Dp4ZH4W*p~u$3L9LDEL5Ayw;2M-A~9xYAXDxN3K5qDhJz)p zCgaNC#VLsO?0JwU)}jX?p`qc%mghv~T%`mSO;ovvRkPzgq9AD#sA1nqRuRdg^sz#W z_%amO2}g>9EwvP>Q0TF7j@&*5TWCj{jJY^3DkEvBN_kzww$V2v0IYTp^(}oP<@o9S zwHY@!UE|<1^8bh=m@!J6qV@Ug`KT z+^N`m-wu1VJ3c#J%Y z+9bWTyznlldqcV7lv}a@|*r!jfhYRK$TM(+hK_O*dB9 zjYM_4P_$vRZVa)SQ_lTJDiH>}+(n(QrwoHhD;Db#Y%`c>HdB;ojtEm*7JZsZ+YI@p zD~;gC@f&gm7~+P}dpw2365&mdBg&3c?G>6<9a)|$Mka+T1+O31fd_fgrO&;ZDrn8J zok;rsWurf_(W$PIUm;P=DgpLD`>JUwSSXT>@LcOAfo?xDbf%l%cwd?VP^^Ak@wF9sMFs>4#!KY`w9(y#bo?I5iDtTWA5LHFX8x z^l}bphM$SM2PPWh0D{MbRY3s@R#;(!4qoD@lxm1gQrpKN?g_`*tKe6cA8QXaBHPfX zG!eKKMb;WCV2bp)fYjD%-oZfnkG;+29|7Y#TB5~_&&V7UOQ!TkQH20?gHbtV0*_a;Jt2c!*jKI9|qG(>$<8bQRo`=Nr zJp>a3+!o{4s7x{42gjMM;#5WqUyiK2qXsL$kgnYO%8KC-N6uC78l4HIrqu`HQDd(~wt}VKKdZxD%#t zyxQujmby0Pxr{{bc6XZZA|)l27>S8dtFD7(mt)PJ0+q?JGxX@eX4OIf#nHV}GmJye zsoKPFNRDFqv?!9Up@%QX5=y3g?xjGftek~3whkV6u9lbainvh-2*)=Uf3bWQW4udthh!G zH6V5_vG#1@7J*oiZj5<;%)QcWEPOmRsYSWAGDNMPvA}Z1NmnhtEP00Qu;$@+_NdC( z6O#!(KSSC-z+)5Kov#@Uw<_Kfr%q|aiwkk+Y+-oxT+whE;Dwh_lKbkr77nrVA+hYP`kbJpj zd3wkDT-%FKw6?D`vj&2|#uWiLIp-VmR7TZOIK*Fa}Bm`mmGZ6*rO+woY?iKMMt z;9XsxR4m)9miP}OE=9ZMEtgQ@yaEjg9p{DHtXo}|Y|@S*Ea469H;m%0$TzU@l`ZmB zNCn4^1Y2^X6bbFJ#O57cu7(>+?pEHHc30lb4%NEb|A z7<-A*f=K79HIrmBhoQ=|3|GGwt({XIp9H7Dx7wP(siZ*sJ!uSWMIv-XMsKiz)L~}A z-j5HtyJs7a<{fMW%C!S|Jav3FCLkygfTPu^7AEb_;=A4RSDV}SH9W<;3XbDz7R7wx z5;~+QYtLKA)k~=mGcsflls3e>Vu3++_{={dzBolnF&zP5EF^bq{C; z7cI%NB17Z9@i(O7Uz&>+-vS*~xi&#lhjcACg{;Hol8 zM?kj>Gkz5@i{^Y}>els`?}jN9lJD9LDNQ_)TVfU{NF& z6;`h5`BffABS8wIV|IZu*H1CtXR@10Z8O?Ulhid?R^YsIx@184pDd9)taBnOy$IOf zuy&t9Wb0P;SQpE(viNRwN};LautY852VgM~nTQEv(tIZYe}9yd~8mbU1<#F-FOIAuVi^K9He?@*hq0tcDU$) zvEeSv4M2VZk=cxM6msB7<5-4TsW=f~kQ+B{3Ja$c7NzsaT`kOw`x@;BtrXDFNSy~9 zLo+%cpIH(Jd%*l)q3p%+;=B4SWY{~eG?}tpGbl>oJw(cRws%~t z?~FT}FsVBm{vj3c7AaUIvhwbTfK!xKtY!R)k%M8kr+UUs87rvcF zQO1;4y^6EXG-!fgXj3FD^+o$vJs!%Ng>%NW#&_qW&i+QNzHa=+LL|`&<*f!;rPcGp z3eb%%r^F}_KFC1?p~G~WwT`mT)fW<&B9x4RGu3Kmnmk#01udv}t7(ZC_4h!X?;0Lm zD-(=L^Z@q0MiN;lhp2}N8bu{Y%Hbi9tpIaUJ@aN4vsw6naAmnyW*D7n*-m8*V|>)~ z8U;~o0MAA!;n*?RRYtWeu{KCjo0W>HXL}AgL$2L9l)o)kAi2D7Cc2f(^^bs5&Yk8L zFUDs4)CM&#a@cgn1C|`M>gJwJ))QlrInI=fUo85wF6<|jxe&*}9$4Y$ewTdY>vQic zwu>aQQ2gMdfZ0jiTn4$#e+?GlzBB z_QP*o?--#ZT6T1=8Z8zS#$7Hngt;P2@ToA`EwXrA0^N!jEDwymIdEB%R@m=MY=Dd@ zqG13+K?klg<5X``$~=>hG1_%b>dSTD0ZS2t!k5cBnkpMj0kQ_qLCQmAmoBQWrqnn{ z*n~yGRdmRH@rc*bW)7NYfE7(3E4JLRT56qc;PBAZbZrzDljyV7RZd*ONc2Z9GNoBn zg$eYF;Z!ALa20FM2X#bGh}n4*&VaDrE=OwEYT04^*l>5Xh{$J>!BC8kFA*rN$%-yK zmxLBsB1w|*NHo=B>+N=t6gJ&R^5(+=}n2}<*DiXgkk^P;K}R8t%ELOYIj*H=0pdrs8%>RGyd2<-U`^)F(9$3ES&jBZFumG=%hv>*!1JRrX`p;(`(}8s^J5 z^(jqBch(UTu*km%^o5u(!4OFZJfV+VWsvV;48AntjiA#rWc$}MbzgcUEd7*{%$5U& zXG&R6nnPM4l-a7wOo%*%@|=l1=;eo#$XOdm?YP!oMHP$718iD`)6y-kFUBE;FGH-< zmQ$GSHvaLeF%^I`T|Yy{?aL5#KQYQA)E|T>TP2`fgH4UZG_TaK`U7KYZBMi$S(#9* z#YNP-YCGcJq@tBhcu7@aXmAH3YHZ@>^al*e*)<9w=X}QEV*Et)nGm3fWLZbnF%nXP zH;_3aGVdyid2f4aAfc!`FEOHRN9wD8TUyx>!9jrnI;Rz^URA4*Hp|$uS06z%F+HFx zPp#*xV{s6aXu{@6qY%uVuCQteG4ak<5m>Dg$^==W(n!1FDwz?U0$pFULDSU2P=!Jx z^`J9(c}yG@tqH!KLG7=k*{o-?E*sw~6c`gB{gzFT3ah0?Q$U)CSOrf=QF?Mos(`3W zU}NBjCS_2Wqqx%ZM|&FZ@7i&f z9@>A2wmXNMxyt=k!ySUqlOp3ND5$$Gp+L_0kxUET`ZtcG8`HMFVSbS%8TPsjsn-Zp zU9m}}!PkJ0S_$+H%><-5o@Y0^g$jVej0!{(i$G`w+E*$%kEpP1{XC_}nx#e$5DJ*% zE`2LcW;3E$cIoL2GTf*hMQPfgym$*H$u9{>4m6@&FD}eHlbrgk*93;I*nw1@Np(Ci z2%sP}2C|u;n6Wi(yWo06_VQ?yQOq2s*fFW#@aYGk*c!+O)6I0=3R=sdd@5@fC|yHq zd=y(-PVA70Zk*Kz##fWA&k0wd}`}PBPbsAurUkb5($1z;0b-gjX32O<~9_SW($*>i$z8kYk&Ui)LGGEoG}Z^_mU zrS6&DCQ`Arrm(nBD+ypd?>MgzL>#ZF44Jcc(ULo7-wbM3IPhxe-kuPBGG5pf95|kV zktaUxI3pHlgXbLL5BI0`WrpgNOx|yjpk& zFc&n|P4LSd+$;(V7{!d}+}NzbjJ-^el409j#>IHR_M5fpcade^)u;tt>v+AfxK;&r z(v)g~?BZe#pH=rhlU=lVIc3ts%X=9yQ(Z;CsZ1^-5xXQFf zJ%GG*OFaChwRGhgGN9{w>s)nGg>&=5V~)9t@tOZrs9a9Ep`_IRn@H_kwjr0dK=qi7wRDzY*|ira5`jg&FP?_5P?uTn|xaN=b;GU2xU!yg;tqQJzjZ?esKXE1}HdDz2 z6y!u7O?E`K35(e~g0tb^k9w0e9>;?8qP&@~psz58j{O8zYtAi}7DI9O z53kh^*Gh8{bt}+5zG$*cFzPqWvi7Ls!A_y`mC1*JbRcc?8G&1rkUll7Qy~r~6VrN> z3>UMxCOSt1qB+~j^mB=x3T#g{u-O@27WEt@ffSRJzmqgptGUg`8+^7y<}jympdyh} zbB?rsHm)fTN|-c9F>g5y52Uj2*JnCKgpYJW8*U)U`DSW(9WK{Mn3l?bsWVywNQ8xS zDZADz)NE|+v7mG-d|hUCIb>+my&h+7z$7W)j|LAynkYP-a=hHCxeH(@nZ{W5aHtv7 za9h#fldBqd6xIhyAhv#LrDP6XI$Z(Ba2MLP>Y;+e;Nn~!;nLeY_C#r_kxNz0P(`aw z@0G4fg#k-&$LiPM9%K?(&*R) zbxNY!2yQHNF1ABqx06y--1*N$T3Bq9AyzOpwUV?EgLCtqylqdx$Y zO6}#N^m*YC`@d<{!W4_~ef=o&pu1w;BEW3dAdRTSS*5;bNzjV2{(kZ8@K&}`%uV@a zkp$kDIkW;1om38TkaZpgQ#UVsw%0I-5Vh>>yrnYBC5tss(W}L#7o_S#B^D2dS?TW8 z5W3X{&aROj(=qlhUVAY|q#N2~BQ@PMrI9M0ZH#t4MI;B+XRL-}5p~dlq&C1we$**9 zD2}ymJ<7$OOKiLZ(2|Q?50056#=?xgDGdQKtRo#^t7Q@iDzvkvEztv>)NpSA$%QDn zqs6ZCKh`}&BA}rhB;{+E4Mu%2q7>Z*xOgXYxXF!`tLHVE=N6RJXd7Okoxy3yGzv3| zODMlY%bXJccnxN6M#>8D(PSJCEpY747+ftndSVfaZc6Dv%`}CKyN0cfmpIV2%EuL8 z-F_b?S6k3J#}VzAfjEZi=0nwKsPOZ){DwAi?o7b<*bk;w^?FeAwkRx%xzD#O92Izq zxy8369u@)9+z&zoibZN)5mQ1@eS2PO#2E=LUu1fs97!fd?_ew8vC`xaJ;SAdX;%q2 zaN+bkTTm@`sUY)uS}YxKz|?Hx$ep)^Bd79&%4K1C)^78~=i+qIUo}u)mqhVwfpZBUc6dUBNncW-C6{OeixYG78C1?KQscsOX3!g3JP}68I5>4&Vplu zNRCg4uheNFN-(+#XzlAIAebdVMTR~(HvwSljM$f+rJU3T(IA~Sh3My5?{zvc$*48R zWwKd8^R>?SRnq{=AYmITmb?*7J&cQ@gdFpC%#MCV_4z7!h;3K~?1e0y9H{;f8--I+ z1*yp@P>_^AJ@m7Px_5Cz!^LEg=n;phwB7rT^13OBux2)=pWK<`kp>LUNi~BAoDvZ@ zQDLmT3(Gs?`;6DbY0wrdPYAh|TF!3pu3c__T-K=5)!yagzAni2)t z6I^$l1CLssN`{H!5Q^!RN62!zJCsTS+jpv3Q3}t1_Buwnk4Y=%Ubg~qh!n`V)@GWZ zV`PB_dPh?X&{IJaH4$CjaTy9MxQCE?N{gQ_*j*_W)?qR_!Lj29E|6t;u?4~vRD&Tb za8OxX>v*nSn!xus(oBj3RkIZqU3Y%QF?9-8ZJ^x&N`U2#TYyU>A z*4|baDvzF2P0>v^H6SM%$Onp{;*)F~?1r~=8P9d=JeL>>lWXpyOc9Hquoip}r<;lm z2`KazTh+)nb#2N%B=8Kkw=YqYAt(s#F%QVNONG&M+oys#MdsUClbxPm5`c_0C&`&r zhGO9O?hvB3j3_H$s2{%xg0+>ThoPZ0Kr5f7f$pbU zv=q=m?|8;i4?_kvUxnjcoioA)X>pS)q z6^rh>bk;+k7B z0y>zS^=jA&pOlPi3)ibjYCC1+%b!FDuPp#I2&V$vz!{W9HYm}FQ152fr+8Q>=!}Z0)=cwP zO)5-uD22WtXHb{(=v`rGr#qv1EjIMCd3&VlJfnU#Mz=S#Oa)UWU6P#}XR+e&y_btR z4`J=H!)GmT* z-9}6+&B#Pl(AcMBQ9P51hKdr$cQ=R-&#*Y-4ND8HL6VkGcjpu^9x3U zM$y&47=sqWZQXFW0?DJrSrZ2%rTE@SZ9-~^YHhCv(*rKoTvCczLNL^fb4e?MSzPcI z_qzQDZqlk|*2(_Q+L3G>4eN{{BGR-( zwsvTnpCtOMYE3G9y7VUkmU30^bv;VJ8;?sN$Yml%^&fKh5G*G#-uKftk1)BM=z>dexeK;eQ{yC@my5PZS`<>2{ zLxN`9S+#5J1=5ajkmZ`Gsm{6x^b2}WvwgBuNgA=S~&b;{0} zPf9~9`28AAZ)4PC>1o+va=_^socz4iJ=0V>OqE-k>eSs%1CSBLWxOaf4N&)D`e{Oe z#@+fD*T(2ISBR;N0MpHhJyc~$DahiaD^Gn+I4OkLZsSpca2Boi6>6!tjCNomOCS-K zP9d^xbox4A3q(~d0A!)!OijN64V22kI zeH$>NLIa#6R1wlZ;N~2615egKi&oyd6D;NDthZvRAX;q{y~}>|qhY{_9^!*F=U{XT zcl1SbYIiBc5?nG{4UY}=I|-+g0jBFahe`_g2+dh*DE+(g;sv*jEGuDe(Igp|290x2j`J!RCp+@xnlXThZE$$0ISd}EfyBPQ zJxtQxs|;BnAezwyX+I9MS&LKvs<;g3I-+*V1)49J`Ij;3xZ`pCD1iK4tJo+ss~Sl6 zFbdzUDa6qv2C1#q0DV_}S}4a|)UQ1Pu@A3;*W0LBX^waWVm3x2PU|;Aoa5YLMZC9c z>eLZ>cA~SrOz7vAQ67&brEz)<Pq`?U4#d7` zV&_u%ULGmTJ=eCuJyx>Dh%#~&*I|h(jQFJEe64o?!1u>ISD5@sDEYcc>fim z%Mxa$ZRQJ`>3;zP78k~Z4HbUiB2XDO5Y)s-Z*=v(bDhs!g92%!>goZt&YxsPCWyvD z?4*aR9Ez!AwzH%!CK`E;jFovIdwK}>E0b?r3=OzR$fOTQChwOn;Ah%KbY^S~S`&w^ zmQw!MuHi=%M+E@o_>@tiI0d=`4^%6Imko`qj9pqEikIU}y|kBzB3AGXD;I#K#Z!?f znSBtI(d9h#GVxCLU?Qkxqm!%w_=pn6U<^p|C@GJ06#S?6q8@KT@5Jx2wio4y38M5h zkEZD4&BBA$D#}r!XUDzeo)K7TAV3 zlgm$C$0aGBxFw&H>I>$X1M*XtknpuQ>AJiwjp3^zEi9?HrJ(@2ix^$& zsES=RgOtZitIW9#RuT)_2L5qaI}|XB*eg6hWY3R|?->jjjt0!#ne%CC%G`Xe7Qigg zgL+D66VNo4>U__)9T9#9(nwP#eIXSE5q5%#%iZl}z{&LSxT=9}xwl%`+CJn)D_(=D zkW#@FHtD|<8tA5*V_nZ$x*^ZBU zI9!0khrIsF4yW9NP0A+6it8_H^a8JpbGr?S+UAIF&ts5;PDa>VD)e>+H2#rUh5{%u zFvd9-!>P|;5H|EY`Tg2ger|r{oB6Wbix7c|oY?G`4@f)SyL`}@2ak8lkS3NtM`a$L zfJR%BFGhD>-l!3iEvd0Vc5GOyNn7IaOOlJVy$6{~W47dwCDytb5j#|M8%8lCD(Vzf ztd0<0Q9;ofa?#2eMqr-;mBcf%eA^`wLV-FDT`1&4KC{J8vRHqJN7>Vf4qI|&ASVN< zTgfHs4{wl5wv@J!XY99&<59tyA=`tY_rfo}r*~G2Tf58N73YhfC2*0Xx_JZM-;V@` zTo6uO-B^P=j8B{_tI0NA3rF-Hw9c%gNnTFSFPzPdp1;90S}dnjee2#TOn8B3o$}ee zAkq!d&Kg5~uQRWOB}%7O8lxXRrf;BUs6|a9*w2YJ&DEFZ2(V;}WW6t6d$iQ+8Gv_O zOHtEp<)vxuriW!dl5ut-K?Q8pa^=Jp8!3<`;R?q@d=!KyS0*kObqckHvNXJhS~ChS z0;tFs2pO2jP`C7~41cN1=H*Dlf-1;$?r z_|(|KD8&a(^Im=DL#ytMwcbie=_Kx7X16Iw2~7{$wJh?N8-^K!RyDU~7K8%D=T5Ux zs|&ax@)?prP@EZ_awWPWrExMpmk+bAR_smKX5Ke}#13S9cbK+EL^I6S<6TD_KJJR2 zZnkzbb&xcSfZ9XXbGapge&EFfBsuWE3b16&rI9ZvfN-O4Kp76qhZuw_<=9>M_D6Dp zkV5dn0E7rAm0|g@97aAe&2)GpOq&$UPo3{`NOKpSK$N$XMj6Z%@0+oZmYQIf;U~gX z!*#@Z-o4F)!(F+-lwy!$qB{#y$9-DR?UM{Kua(Zp18=m}q6YH8Sp=H|;<2g}P2zkq z1UTD+WGAIZI0zfs3pU}9iO0TqJ>mvjZ$(54=7VX}0X&0SXzZTA_n9d@0K4HR80 z8DOT~OD3E~JN46I1&?o!xc8K7gr6Dib%oS8-iH+-ww*vbS!~$Fa}Hc+;@T|91?8@ zyfMo0*bCqeS&s7{k{Oe$u!O)3%86I$S$1WG}Sreve5Gu#DV3!2`AQ6b?<0tcik zSf-?K-zr-P)8s{Mu6H&EBT zpR-%un>Hi35fi|R?&KhAJy`Go_lw65SffixtBs4NuXUm*dJ^-dC(yi+9<-83Fpn2= zm8r7f-7S@IcsmeV*9Yxkv2DtW8=E2T7u9~++M8N0V=6rF8~FPKv88Pw2Py)gM_zY4 z&%buXpzRe_nvJ&ZF~BvJ_AO@GEK-3jY3qxKzC2dS8Ic$?VwWqtHr~~M7M@QVk6?(P z5C${BJqnvSM5QV9Ikd>c#Ow)E)Y%eRiFXjO^)nZD}}74W&;z*GldPT?NNw zDe}bU8-OiVXO~v6ND_}@{liQ{Meza83S9+Aob!0l1){@E%6(CUy7A)p`y<)nlx4RX-^w?!|6)4dp+er|u8Ow8{z1@d|MQvD(1B&z4rXqVm z)qtiUZ~_OwU5;KmVxOp4N(nh^*n5{8ns!>)Vq${`#eGhKCe*Hl2`|T3LBc4##I176 zMKf2_H1Z67wN>bhD-)k(2pu|)SWsWAEKWqqR=K9edui9+wq6;1VW6|cyR9f64`_x! z$`%&xTF`oJav2vlJ7b^0f+1`vry1oY5F?Tku$ec5~P#I8zpeXR4qY$eN0|Qr5b{?g+zRh z12hZxuBeStxzgcksfai5E$8HpO+-!8_AQP9Nzvw;0vBYiW*b=Ssn?x%%u@kmj1u0- zVOm!hr0#{Z@iqs8(x5TST9Qqp9eQ4tbiV3!H6tU@E@(9~o4%Lp!VH*AEs1iZfH<|aCui9YaHx!bH`kOu{iiRlncUowAM#TQW@$)N?xzd~=4riP!^Ugv$CdJn3ybQq#C;*m+V3fz? zuJxryMnq5^V$;ACOe;2~t1XBS;KAlUGC5fe;o9gzRY6krkiulr30WJCw)-g{&|NEE zq!L1u;EaK-^Tm4%fYVeONZe}VIHYThH;4H5od*sqAD$Khq9KNVwbw$AQrI-dF zU3E!XtY++eYrPBa78>$BfUc%#>Xaz&mif~A#WG4v?9_LT38-FDB?CFe&jOp)*9ymA zzL^N+HrsPbZn8nDDHE5pDBhNq3Um)T%sj65xk2>wMlu~NiZBJX6@$tW#Zs(2ce z+y{*+`4IqPa>}h1ULY)t`^?%aa-d+SrwB|;n8|*)j;d<~T)U7=BsH}AroH7}GH0g@ zpv@?pE$=is_CGLVA9zP>5VA|qz$boYFlhs-2Y?%-yjb;zg7~<-?rGt>BWbrwiM|Gx zF^h~gakbSQlygTczr#Zk4+eHC09E5>pURtF-k;gaV{-3BRLu`dU_^D*a&(|X6_W>> z6OGhpg=CRQ&M(_!BcoOtjH)r%1ms660#y|wPeFzPMW{5Mbi3rGgrW+-INgHAX6%Vr z20CRt5F$?*)S+|*%J-gMe8mS-?FSA{i9AJ7Rr2qOZHRHl-6BKbln#Bnikq83hs~z( z_+}7~BC(SKrDoO5qWVI4IFPGtM5h&I<*3YTPs`n?);3b0R&nc92I^WVnRqqOPY6&> zg~gdu1*xYthH)s+#S;o)l$*h@Gpxt@w*(@g8)mFX8Qzk7`g$1%X z!9 z-c|GVHQ@npWwV7*hFR>RH$-dGbjTeGtOsmjUoxk?Eyk4lnsU4-Ig~J-ofPM&Y)>VRlPY1WY7&t`t_U!acDUfw^=h)8pkD zvzj3cSnEzRrRgoF-*%4VnB?}gwT2w2ND8>CoQE|u0-*||7G0j=z35Hj@I7NOBx~QK z6&BfG5|Z$gkR7yX6vf?@@N(MYba_VF&%@BSqPdq%WVTz-W;490$k0KAwc<}?5YquN z@xMK^rCa^lbSG#}xzsKs2aoIRg-y}aEQ3HSiGW!3iU@;L^&@+Q?mOqeR)C1`5Nj7^ zVpxjjEl{;hgJPm$ylb$7ER>Q0&r>dilq2Q&uG={RT^l<&1}-#3!gQxddvPnpZLfvs zPS%mvR0&Xqen(~BUjjg5C`5DHR=bb8six1UbtQ87QsMU_gsi9C9Jfe1#N)Rz!@x$F zYjYkXKBE>yW)#PcFH)(mB&NY>Y!qQmI^RR!?kYKEJ5>dEfZBp>M@*Z{h@f430t6&_ zLKjwWS41$r;_Dwb%rgdFN4`S52sFv!l2#_@nigEv%wH9PhOwbj#7g9Rc~q{rN_l|e zIBt4g%JkTz=RMT4pbC`40;B-j(po|U2=j;CEv*y@RVCmS=d^UU-7Z39EeJqZeY4#L zmQ+i1vySB%d-tWaT~r(h4^BD_@)Of67dd@)rNfkKw>E{!@VsuE|7WKGY_HjsuxIWI?=O8iW>z}JXS|Wo?;FpHR%FF_ z*KL8LIRw5w5^n`DFqwjP?{jQO?F3}H=88lZLvY;PgyE7&e3eAbBf<%rd6iY82_>jf zEolP6Z2+uob{rRr3xY)$rCz@s&n?u!?7W|GK>)j>(=ZM1gXTD z3p?Q#b~FLhpoLDT=p^Nb>MXfHz>UUOgv}dr%L87|UPhLJbZ{R_wOCSbu;l9|v|)** zMx4~BpPJE^pC=ZtJ+h<(A7Zvx#k zeUU}xLKyDk)gJ0Mr+U+cFBl53NQEO3D;@n?=z$q}3MRHFC#;RX9KKM4s|1)@=T{qx zXy7d1gzPgoF@e0``x}ErL1~fPA1&^0BBV8u6K(Z%>Z+(PBNr`OM1dKur9E4UWS7=f z1JRLwq@cJr>@kPT*a2|E*EiE@nTf_t4p5~y7N`0tzS+{V9GAU8LFwYkIU9%r#uDp; zJ}TW7FmQuK&16K|KwdJH=6X{H;=s=@PlisT1Vrr*EvHmoGt(F@L0~WnYY$?Lg4Dro z8_hy5VPc-fq+cWpEvOYb^wsI>*Aa#UkZ9XFjPrPThUg#eUt)4y?;ZhazoJJupz?ek z(0QSUA#Xbvp^L^{VWdgGcWljV!`dLq=GGOZMGf2(_D~31C_q5EdSNA~lm%vk5{M-& zfv=Y%L~Xw^@({@*x5Mu+lybgz*ao-Wj)kH6ZcD z8g)v*HuR(8s0`iW51zVsuSA^zFsU@gL?-Xs^rXb#T4js1T8{89dn!|npi&ZzJ~06q zCbbX{n|f;wKw5HiLb66V7AXj{-mt%2nD&-Vd!B4h%P%s%n!95ZOz_wlJ$Y-P4X@9N~pXD)G7>& z1{{c7nmfj}8ezv(L#)OTS?FCInfsbb@llIaFkOqEvSA+ZhqqiHQ`<-(Nq7KDp?%}d z>vd~!?VP`Y7vyeAqW5*gesPkQYuYJncb>>N4@JC5T!oVjc4A`pb4p8bmv5jOB(?yi94KEa}un6gVMm~A0$~jh>%TYVRW7-Ft0l~p;Pe3veE{n{X_<&1$oAz z^0yxd)3xu=uOi;BL?igZvC>ZAZ-SvmLr3B+@Sixs_ZLEe!!UW}q_%MIln4&C{+6!2>I98~vGq&N zT1@ z##wLeOo;%Isbzbhwsv_B8KJ}p9NpGic+C;Ft^5MgWYlwB0Rwm46KfIRzXr8--ey+O zfx?hmg7RCfXHT%8%mitu(A$=4B~X(@fGdoH7W)x8djcg*Cky_Bgh?U37FZ$dsEgHP}t>1h0OGMHZN;jH3-w27|?;T=!l%d&9$Lh!6j)!sT3KDxl*+v z;eb^N1I8ZlINgOb#6I_)$h)RoEYcTrRx4$%29wZDsfT*#NEKx7IN-`GvA2|OnH1Er z&9lAJcXbwy2%?i17adZ*mKl#-h~AP&ix(06mcKYq;Y3%;4rFj5DUC}TS{K6rtoEcb z366e8Et$)J1P8IFtHym-ekCf$W@6pfZC>f~n~M*W8YCk(cJ9=7#Kg_a2!tUbRud73 zBePg8aS7-0j@TvN5!MmR-=+{q^RVp+<4%mfOww|PsW>bQ?R0yFPWrOFQiWiV*fplo zuAhmqq}V%|$#$nBildGaRgR0ODHLqsTGGRE>h?5yR$X~Adon{4F+7*iUKbBs)vyap zTq(iWUIwMOux1&QQTzq)ShsEucGH(|$^W{xobG=&0NkP~~o zP!*;QMom7Pm#&NF4-Tl-l`T>TXTS%I?C9R~p;pj=hP)y$ZAI7WSa#f;xJQRL+}cY~ zb0Sjug=FgXlr|;q#tfV~6BSJbDyT6ceYS7b>M!RacFNLn0oa?GsqeRTG(r*kQu1!pu%^<`lH-&&wmg1uA zM-WYokeZyhUEfEbJo4(PiRWch<=`66RrBs?VQO4ODA9_sMCPJ;i99^l*pyf_9AsgD zME-kmFj4b*l{nxwaeHk!x%?Hgh%3r-k8u!NNmPQc`BNS+62RY{=n2Z}7bj$HZn1i@ zEVxuE^rO$VP%9}q%SP?wXR<;^q)P%I=U9o}2AsprEHWwXQ`MBjMq zhbI`EDAZh`a?2~-HQH;RdICXhOy#kJWc48fD`AMPt&CZ{xrQCzR?_02Do~dAWVj$a zB)%ss3ZY;yqe0g&W@<;=wN$%%>%l{Ic2u~E(R*9AStIl%sK(UpH`Cp&Sc>F9VC4dKv*ua*cGr533r{rNBGu3}j6%FdIHXu8CCy9Lx_6d{Yoyt} zvD`B}KJ5-ruDb}|790bD#qSsxrySII=X_~PuMr9%RK_Tflos;-85Sw#D(_4P@sQrf zjXWA0O)^gw@1<@PL>wU_iq}w984O+%fVF>S$=$?MBK?GiS_ZF!SKPT$>>=A1G;8FL zdx(yU^Them^-j5uTZJ-5!17DOtp-_9RGmA!&V6G6-oRovFv5_9~4*+0i5#7;dePng94*qV}1nAbVVG|y} zTw2Uz&Gbe^(LU`q`S4$jgW2vZSsoXr>y#QZ5G)F5z9NDN?4V)r?MZ;SkF{<>Bil)9 zfqf&S(0FdPN{eqFWE{Nw>N8VPQ@GOWH z={JwZz#VX{Ixtu;ysJ+TILge*Yyvf8VhT$ov2ncHRd*_CAYxN@g4XLGTbwD5b0ts% zA4og9^>Teq3kcf5ab^#KzUdku&E0JQ2OVUW!JJbZ#DNIMX+v?yRwP*6;IvuGvNw9k z0(o^&Y$OyF#6WpsGVw=h8ASFg#cbWTFl!Q*pPuYIi^6j2(kba+T?(AnacT-igt4cz zG82MZdh`myqF|WV(Fm5oo8Jw2{lRW z3m4Q(q9E;`u0-09Z2IGe#12(At`=<(robWwr#E)a(8_D46Iv7sQQT)JIQ|6*$2o99 z2bIEff}6pB5a8anMC0oyp(kt*DP0z&K)A^stqEme}ulYIIIxuLGaU{YL}4~NfQ~$3^*$T6?;T)v4t{j*bM`0HzqHF9+_0^ zivtVO}cE(}@+H#XRG%V7*D%2aXKYmvdT2$R+3kmk!f3!?>RM-zAu!jZB| z1zyt;_#}@E#Y}cy?u|2E2s%o=Lq6kiO59yKI*L?ILk@~Yiy-Q2c~U5!6j=iDvAuO| zjKRl>^oA6@MlB368V(~oj0|IyC)hzsc%lkX6^R*qySh=T^5fQu6Jm)E#}8KcBR0w2V{zS8&g^^vqRTm7jJ2^tFCRf z-xi1|2by9QG5#$)#A41;&Gbr&h2p}B#WY2}*vt#nSZ5XKAPbGiDDk=2{Qj-!Z3D`k zV2LpHEX_(fN;2ze1L|2|AYC?1%|NjOtb{<+tLj+oa8MX44v=J`RwQND+IhrLr+PtW zrU^B8q2U1Tn)H??_Z{YX%Uqw}ae!D>^+0geyh+I2FQ#(q0XmOVJVd3bV}a%@c5G7D zgS+pt1D_ZLYG0^Q=0vQ~7>BG0cfss-Q4oo_1YEP=gA&_6b@ZrA1LtYyAhfTj3OS#( z5tGGnqJ7VIdP;JXW!1)K_kwmtnVmva@3k@n1B+Gm`;+^Q5i{l(vacM#@8<#k) zgpnmsQ^lKpH&Wxp2?r1mYr*v)AUeyyokUX*Dl%rHwpbCsO?s3HL9FRKm7q4-P-dWZ z(|)qZkzMF+T_K`kWJzrFEJF~pSu;zj@+kN%j7y_Ewd!AaNeCq>n-bF`=NK4V9o+M~ z#5dJ$nm#cgpOH+>%2tOrZBF@QJruw)1px_ZXLSIC^-_6?RrkvS)#&a>vhtMeX))~! zRZ`2klGKrC_- z50fOXv3YG0AcrHdCL}2EwHRfbSXfX?ygpsKBX3(M5kB+0tMms?XMIF}Sl&6pp9$37 zxs1ZcnW@(-{A(tMyoV_W%zMQkPlk3j$!QUv!BE@H@mmQ8+h!oG;HCXm%bU;(JxZ~7&dP1SPG-qqZ zVtW~jR$9dQN6CabHTwI?9_Hz$Bqv7u3U5hzn1WQ(Q_k~3Zq&X;J196ORtYPGLkFeMhm~S|I|dSc+`mct+@ZUi+Qv)U zHP^k$)q5;y)_&SRTwePXB8~0Oe5y>iZB%;+&2&goe)5|c;>3@h084HSB{r6vhghy8 zW^t&^j^-hFK*wYR*0laL&s|4wil$(t0u?&!SICe`EUh^#sIM84rVTd_YVK8*0?7v>Y2pxvw~ad#(T_!-ys!z~jUBgrRRp48yRg zy~p7`RC0T^G6_@Kd6R6H9F0n>cuq3a>&cLL3$Wd$?GsHb!U&xWEQb>9$17vDX1EF_ zCyRW4EIGie24%K7NH(CLc5qJO6`h8zsVews`M$fq1g@2FKsAB)<2DTC&fhK%uaS}& zA9!3j4upZI?6@=w2G*#Vyv=1*<_WT^PH^k7PY^nj)mo?=EX860U~GRIMRueR5^&{sTgk&+It31}t!{7B5;HEhWa9S2V%mv)b5}kY=`)u6 zILYn$u2}ifnAmJ}a;kp8c^oUj6B*%SOk8$4yCv0*g$9(B!sUJ-lmq##^O-Vpu>F0?@&42ecl> zJiBh!kijt8yxF{mA5MnV^=utZ*A}T>6GyvaI^Fo#fI!(m?@&Prt3`fxJX|~Ef$rSN z7JKEm@q`AcOufFKBN=2PvjQcoJWkb_d59>Znaru}1K*jI1Xg~SiorL93a7%ge( zu?|IzDT-00NJ!&;w5n=}Eh`Q5KM7C)sbRwsM9tq`<|4X)%V#+7065& zI{C3I;Wu_forjwoIiV)NOVDak*}Sa`1BJQmcM)2}Ve94Bwkcea2D*te%iE!-o@XZ4 zwCj{H4ig8C_v;9aZ+;6AHc{&_j1gSdN~2iNL1+hI7RLYGh!B`wQ8tA-nDe* zPyx6ycu^$Y388HAtErvwiv?S|Qp7O}3Be^F5uWY|DV|x$9U|gE&Oug_iC9Q2*BWU1Ru5kp z_>GHtlHFbfX2$M#EMO%1>&Je{G*lzqs3N_`kZ&ya$#2sJFX=K$trhOqXqnT|D8T`j zkT7k-=MnOXT2Eb?ah%hGYO)W=I{JY`){_R?Cd~6HH8ZtchKaC`eHg;laDYyjgjp>@ zE6|%MmJSy>gNQr~q{1eJ$7hc`UU|Y7OA`)@DLoCelT|#rj?(wA#WU7j%-L2}573gl zYwOJ{wxJv+4|kxYL&UCe(acs*p?nQ#QiUCW+Q~Y}kU4u-2R5hI!o;ahw&4~CX3Zq&!t^Qy*+3Uc`F z(cD@IyM#QbWoS6^v6F*~&X1P5h$eatNo9E313P0NW^hzFU)tBbMdkM`p74pAZ&-(< zfJM97ViBc!Cih zz>WxZc^V)+(|=hSUQ0AzKGQN#A+4FiZ4z%I(aQev^Tl9;d^7~L-ys)a2$eBy&QKFi^R>0^m_OP2h`}* zJkLX>fFtbc1w9Hw9-<81aAFJ9)KkE2OoiuOoC8D6?bG!(Q3az_E$=diEtIubi^qoA zFwdtqZ9^tQk+{hhd@eaiuUe^YTgzpjkyS4xZ;x= zkquzFYUvNE=KKhLL|Wo4B$6zX`PUR1K!wEa^{FNV;|N`PpHkzD5~QF&h6s#k5)N-Z zmYqmoTG@fH;S8o54rm768_A}cg3g2nhW8en5#IsrYjrHyF4MUidkrr#^y2{FowK8l zH<6U|pz!Y|DHC|>3@3XTDbq~tDc~&tfb)7tN$EXM+GLAC?oN{M5W}7ZW znNkggH!#3Qpl(30YW zv}r2jLbq4v4`u2tSw{k@tCIoq={a|~l0mR0syavrt_wrQ!@_WCK4o`hbQvcpu-87X zT)f2!OH(Z_y~{ozwrONKt*m&8hvGGh>Ip33V!Avpa<5pGh%^XLf`VcwYQQM4hOMGG zJSDW%zB6toG&nvJQZWnj0SbuQW}?uz>9|`xOh3Nl>gqxdWTy5-&G#7cMH%{iI5w?W z3lQEUc6c1j*>pwdwpZ?F!}yqk49_5>^6SC^jJZV5maQ$=j912{JSuV}r%=wZiCATw z-d#b@p#WjCT=ZLwDtS~rm>y2)4>w(PY`f~OB!!*pcEl#5vIiScCQUUWv4xu1)b%M; zcJgvi9iUiJE$_oTJeG0L46t}Tix)m=+YY&qK;)%-?ID&u-kkOiD5uCin=`|xQf?jkOF{C3i#T(iyOAJt>q7PbNKxh=1*R}c}$RI zCxQids(^NT$y2$6BfOnJx3%Ks7RGlv;sdxqjyEUSs`FY%dh|&fgGWZs%PQ1yICi2A zMrj-m#uySu_qv%Scz_~z!$r;ZVPqs)AS+4@3e}=62*}%rS3Nj^BszMPc#v;* z3E2&$7KH^TUk>fG2H$3D!V6yBD5xObR=8G?y-8>%Y?YV6zKth}a#rMU+)_1ehe!>& zkUe~Tl(`Tv@x;l*4Ui|arL_sx@s;5nS@1h=sf;soZHQ`!5NTY_dd0U9ygXiXk_(75 zAi0jNWF(5GN56e}8OxU<*H(zKgcGv04KxBbsgQ&%MzD#@wN9?LUboi9T>(1r#xQYF zW6gPryle%_d1MfAa_%=Xx9d?k^!U$7B>=?q_p7}nt*^uu#B!pMZ;f~dMQ5=aK8;C< zwIgXNYFDwdQwtoxA2n4ph#fnE zT~3j(Qd3`fjkXcy4;j#AK$~0T<@Ss$o+4Vl+BYV^q7B^;c@F!HfbJUY=agnxzn>^*b zt7auZeZE-~Ut}q-GU$XFL1AN;Kxy{xC0(sR&?aXg#9xDZ3}z|M-Dw^WsOty^I)=Q1EhLC|qAKUP4Q5XV+U&9uVY6X;reU?`#n1q`RYyP!d%R>3pB zf-9iMMvGmP`_}Lyr-4$TlDlk7c%B31m)5YS$ zj!N^)#lTVam+EWC37poBf)35!i@ zi>0~WI=ndWgLQ{cLQX@d2nsUVwtq5^(y64D7p1S5r#|CMQnm5dbVUVgCU$=}3 z`_dB*l^s!OwY7gt;_GTuJB6L84$ceT;mi%Sg&$sMHL+J{b)l@`%3`(^EfHgBQ~W zfaI(1L@s2VrFv>xC#Gb4jHmln!Fz?*xtFUKq?&=N?HF$wV3Fy>0)Y4;m z2CqPItMvk*=23gD_WJnP)q~c4ln1e}W%V06a|}4R7RMhiZE!tuIqBVVU9Wc=UWU>L znq-c+f*A-4A9|M*of?cD&7~iRj$?(!>W?Uip}_%knho0_uvrZ{bRUM-Y8a31I;Ls5 zcoO&6b(Iq6d;|*;H?!qbceI+LX3-sifo`P-i!XRoJ?YmArvc2wy9l!g8>EnsZnF`H zbG_-rMwE|ZcG2K9uf^{YIVgF3JjY-ZBNcs~xYsC0F~sb!8IHoWE;)4uVeoT20L~M1 zIN*j`GmKsDrOlZni@5AOZqV8XYZ}3x=SCpUVtOb-F0@;0j^1zFA+K9qPc|7PGQ|kseEF? zTiyw9sNOVM1-gW~u}Iq5w8Qpdkf8~ugFfhI>ww%pKNbSBC0Y=OqE~$!3?r169(Te4 z>W6yd+_49G%^T(g@+x5=kQmtUdF)8H#HRc&mjWY;mBO`_oFx$AWT2TWtam=zd-HDO zG+dDwl?WmTAso7mihbl2_gwg1eYlGf@6`ec8Q8E2yQSBK^wJ25xJ%?&dgOXL35&=+ z1qFuL;S6>mIk0ukoph@$*dr#CgUk(#>LUh5Q@$4u?*Kh~Y1o0^c(XtxhOO78c(t6- z;D)Hm)A&{O3@YqnE!-bcL1=~jD>aG&P?Nz4op{8n0TQEErR}FntA)sKp|QrmL%Q1A z;jIN47Gw~atZB^J^97|{w!duTMy@aH;ZXEm%lQ$d;Jd@lD2@L2v3RcpDD;W!RKvWN~#Uu}qBpAX1 zjB?)2Y^ybPSV=rVXus)A@lpp=`4^%pfk$ z&{cl8Qi{yiqD<_fv-hnPUo36ZNuUT0elaoCi z?78y+cpA3ll9-vqUAAZ0a2TZ)1DqXiepwlr9!2j->PcjX<9ROav!u^PZ;(5knizw@ z#?F5o)-cF&aa{1dA`!LlZqX{mY6oQ++>Vr>-oabF#9otI+Hut1H|@MORsz9YeQ9h^^M@M2gDmiqIw^ae!yu|S6#U4+dd4Em>F*5) zYdfal-m#Wmky)mM{4EO{rcqb7EUyGLr)pjQb!;>O~t3tmUqtpZ>RglP5 z_o;CKdCpxIJwjk2rsnb!k82D(b267lA(O$qbjbM&VUeK)LWDCKF!S_SWv$}I0ck}w zxv{i+lMr3kufa>wCGPn?`TQl|OYclA6I^x>auSwseJeqAkj5JtGcZN9G^Oqdi)Af$ahA3 z*4mUNQ`+qUB$5+1VzZg-xHKLUJHW1p;Xo1QFS7=n2XSq#rh6(8E?vaX8_c(ZWy{2 zr3gwOI}2tUquz{C=9_RIV7EswS-8#0F~Af<+TXoPaQ*_Py)usI<-HPeRX&&p5J1H| z_rBY~QP&t5)lDNG2ElH)l-?*FPbJs6vmP|y%etAf}uz) z7>+SNdKY;xsQ0rFh1O@F9)g@Zqz{`d7Va2_T=I%0N{V~lNUVAp0lJ8YCwu0ypgRXE zAABJecfd_%_=kPUrc5+1AUc~(Aw``E!DrkOMhoq_T65r~31sN;TGAOCmTYJq9!+C8 zGX;SA60!+fGdKws6?q;5FQj;Ui1E%WX>@mL%(3GPvS_i-YP1pIDi}|%-*Yu~7~8ka ze7+SyYtf3oGArl12)m|8pQg{SSW7RMPh;Nw#A;hG_6L*R5m`1!@(>I z=MGtANJ|ER8hirK^_J&V`Ed(Vm}04pfQwZOro~2x2ah!K*6^MRcj&&9J1Gl$$SEQ- zgy`&HSonE}RC{F3bg_c7vym%gt=f<{=7F50r&B_kOXPN&u|L-25*im#|rDvc)Ya;QVx+bvB(rLxY7^J}d?X^RT9scSJ41DWz6 zYt}kZq{ZDwQ$Xg4AGnt>9cKILAYG)_4`vdzlJaaC8gF-%lof3IAJ1SjdP<{=EYi*7 zfqrLrNm$ZpqS%o|EoLI!zR#(P*cWA%@pUM74W~w}RHm{!nMUcS29|znfi!bLjX~T* z-q2# zpL76K7-y6T$jDmM-B=MkZP_ihW)BQ?Z9>MNfO!3u3eH0|YeP0Z;)4l6=qrK> z?%qJ#MV{0?m`m8hdc_e2D2s%(P*>#4FcY?HJ~D-8_F(EUpan9aQ_zu%Lyxr>8G^<6 z$H=d{vN?5Wf=Q|keTXJYZSgdSlQRj3hFW&w%W(?s;%25o>MD$xLKdnYn>GcIyQ0>) zYdi`DMN8Qaq@R@4=!w>&$;&5m|^|c#QA6*Rlh$1}y$Sv<44lEGenLBXtsY)C_BZH=F36}8+Vu}S%=ZRQxIK1`N~8di&tIw58Y8H6k)g#wxCv5q>FiX&8Nnf=m)sM>bDaU9l}tvD)m}NfqIT$Gq#V2ZTY%< zC$NKuz{B7&U}2uhP6A*azYvFngF)s*fNW!lhQLB5=%suIN{KJ7=d;pp?qjp{;D`bU9_&g|V>Z+N5@N1loeF7#;x7sY01sXj`M zObBe)UOC(_@yr7sg>x*#bSC%}z?ys$C`fqX5Q2s(RkGp^#q>uaNMQDek;AC;aM27V zmxhEFBdALPl$wkcglsd0jAfQWgNm{%k~C{fre&=_w^OdK4jlN)B>-JOqQ7Z_iAmND zTE>m7`N*G3OQvRD$Sq~-r$eh|A~AvWJM{;#hmOd^Vu{u*>`M`y$w&(NELvp3rJ8Bb z+?zbPVBah)14-ePXyZ-B!DycCDk>5mtBQH{S>-p4ykd0lHFpjL5YPhWWggE_^0`{p zH7Od7BaHP&te(e!sFm|OLdea{Pf$imG`>g#?jjZCp3EuWrK6;fVcWrS`h*rRhhx5n zkl}aQMv#xInI?#>6`y?%bE)lNp4d{RqKh3d zZW2kzlTyL)p+;9)37ilQ&PQD)2K_mhTGT*ez{Ry=y^fJa%jIqaM02Y~p^ z4;uP|i^q_$L`^BJQAIBlFmc{HvEiNDjAkWT4%^=nF>PGllHuB=CDRU|d319pMWh&V zr4O}$=WUeF9T{{DXtl=jl{`4^NPG~TE)Lrmd1=nzci%H&wTN5RHjjIwQ9CV~LCN)7 zgS>)7Bah9wTCyH>vk9;k;cL{+4LilFz=0)Wu2r147Dz~zrzUGy%6Mvx+i~8WQFztW z$N`utwqmzVK5X+uC{1SfMy)ZHw8NElDlvJiYp_h7`_vC34X@9#ic1+zt_o0^CtW=) zWJy$HhBkA!Yn(T)$GFNbjK@n}X7^z@G!a1@$c)D6!MHrki;WR=O- z(2k}7Li^4;ZEnZM;J9#z-ZnlpM0@k=bx@F)a)HOvv?eAAoWjzp z9lhxA+b!-clKXrYPtjKVz%o((y`Nv7ZTHCXz>+p%-t-f!Aa>PB46{w}N9a5SM_V+s zyzOAc;JfyiqP5#`2aPC6#T4VJt}vCwWbkXV$KD|KVEDm$$%r-I*lm-^BMjleO6Ou& zzSZ(a9b1Q*Xsu*5!%-+{GKw;2{ns+vb6o;MB)mi85E!=L6&fVjWiLep-gP&u*=@07 zgpVb?D8P(eZ_c5klNN(}FtJN;R$FBxP?Gg=uRn!;)oSl$a0CGhp_vs6yiA1f(248hH|Zh3fpxtIurc;YtH(gcz= zp^>Y7;TJw~6$44K@PSJDAcA9^H=5Kf?`u{xsSGN-WtCP}rJb~*837{qLqyk6#Z;;! zo+}u|CaY>U5U8q?aR92a0K>a>6p9RL9F{@?H~H(O5wFfw-NV?&;9b(X2d!c|Fr-z} z3fgfQOzG?JGq@@w+k_^JyJ+^Gq||Jd_IZq>F;u+aY}Q?yB17;9i4>DDY}is0(K}-c zQjb+#&iBsM$$|kIWaat;L%x> zAR(x%?b_#$NXHstytT@wt`6{KGg_}2-jvFcV)+nY?p9~FoRfkx$lh`Ww6(i++7AV; z%pWnwmV~J3H8z1YF$JGKt7uY`hAJ4qNV(;8!dGC+i^c4w!tNUTI;%9wF#v!D1p&CB z#|LM<$Zua)iVXXkfLqWVqeZCLb;f{-nap+VpbXykUSK&wfstB+%0RUrMP7{7jNvjR z1Hg$}%oJSkU9K68cY;Y6RKnA#UPUY_Op3=GFT@qy*A>4SO&SP15vZ{rSqHV9 zAQq#wyv)G};-X9-!(eZd`BvR!CJC zdp%J6Ls#m{hD|h-TO%7!N(7Gr-8RdR&`S>e%2M3>$LMl$l~_ExAon|xA$XK}z%79u z9f0Atz{9FSoK1*4sy;VtJL^y(NQESLbm~cvnBgl7zjAH>(&fv=kAwwp%4!sy>~X1j zI(4ud*_*O5n=O;~jIGb37B?7fP0)E`oZmBC1Z1y>_q^?RP>(05Y8JMH5HQrU_on4l zUJaJnWk3egBD$IpCwvsXX=N2GZ z0UsX1iJ=FMl3wB#<2_Bw-!hz3AS3NaT$qDZrsB}bVRkQvj_KEpeGDj_gf3r0Ny*Jy zfat2MS*kX@Q&=V^6y3IGR-bM8OzvuB#u8XB9@Vor;n74R!bDN6ubP-td@yXjH90G( z6jE(OB+-wChw#)c45FKx^*zCu?ogFT^L*J|MlCiv0!FwkIUt0FNN;e3JSG5*YoUT+ zxnW@9idv<$9ExN=O>Cj|(s0@O( zf=%kn1q6CX;9j*!jwp`fdHy6YbBvHDzh#PegS0h6l)}QTiyKKv_D_$rii2Ts0H{XC zDt(;L{fBrrhM6v^m?d8G&|Cd2B{VB~wO7KYZ`)8>*wl+oshNu8$-EawfKh?otLkKq z10bB}h4+)q!Z_ar_pS7uXDlA&0?wmmNWDXAkvjLpXUIi6Z7g1BD2KX+!H8RLKe{$$ z%&;qJbSqlW)8 zcskAC@jT?g4{E)W*^ZZvFrbv(>8yx?8x#`R6b9A|*Be6`IcUQ+z6zDpgS+uD2`aT? zk&`k^2$jz!9$CC+ShBZa;TNm3Y1&GubEB`Y4|ygmz}Oav0A}AZV)}l|e**hQbP$`p z8tGTAK$`=LlGz^s%2_n zW-|xp_K}$eLPYk63pdKk13g1_tz~#ueN5WnTm>&|ZG#G0A^4N-zt)st6t<~HN8TbQ zP&%>WIC5)uN1t>G#FoXmUaA*WRc^0!AbX%owXbz&n59i99>v*=qRY41mZrPuxgASF znikK5-^ue)VEvuJi25WI+~Lq>WwX9XaL_t3BR=0$N>P^A0Z|}^b6^!X;_q0l%U0fY z$6148H!%8ejBi^NZnP#J7FTAwy#)#fUBuO3aF4>eNPDh0HK|wIFGw=Vs2Al<`T8aH zuPNZ+ObGN>Tr0z=LBWn`=Ws`*4R>hv>BOjpo2rHnx_iOOtd<#)YT~GrsbN@ zv<>j47)w=I}#f6jSCyK_Y}a%?~5?DyNGOjHtr`TfLWP+ z$`@h8=%|VyR>@oC!+vx4aFp*_CDiAP!vd4H86o>lP{ddl?OhP)*(#sMA#Z8cb2$>eugHX?S@5lwZNdR%o*|}rKs@yw3h`!V>B_LUiWeH zIVd)5+U0RETPQe;jVo&kZx_gMGFGU0WdJ z5wYJI%{B+3eSA~2mRIrNEd71)$nADRdc+=;f$cJio%>?=G43Jg{qQFh30X365A%> z`{AU{xo7iBeX&=I3R*ByPhtUz#1Y$4l{#om*CVZ%luA-gA_UYAAw%Ib=0q8$1${&d89)~3I z`u%-vM53YuA%^@d_A1CfG8fJ?4k%c|AnqP=#_$O6gmgF#ZOVru_%eN6YOAccVA4jW z`cT_DT#mxj+7_~tbUwnrG=+9rB%D=bO6HwxTaSz7R`{cAX?R_3L5fplheZvYTKxJalXh_=~$qzE)SltiGUa{vs%xtG85_RS9iG&dQy1IQf zJO<(BUo5F03pAOBFkFeCs#F}D(q306T&1Rhqo<_c?(aKWnccP|dqe`;NyzL<_^yF4xp)~0dE6!%BYCg@0Eq|ka!K$ko|t1(pVo_*Q&C{hzPh| zL?!GIuw#=9;1!Az}73B>|HUSspu=h^f`h0vtXHE zw_FkS>}BNMu@wY3+nIs*=sYiQ4QK^WQfnH_Aj-+mwZC|B3A(jdE!Lo5oNKVzx%v(J znD<4|%=fi4c(XBwDwdfwZx#CYsQb8;Lqk)z*zPpgEoCID9U0*h5Y6`~; z?@HYEK}p%rg6P&G^98&v+jbsLeN}0-wE+vs>4R5?b{1G+uIhR}sYe*d=fqPayzOoy zCuEUiSX(spj3B`cAS=oey=j4vYvXHszcG%hHAu*?t59PaOuWklj<$v|<+;5B+TntA zX_tpkhnvQxUTNS>ob)kD@s+m534(f5@k6YyMV5xD&V!{bKoE78JP@XJ;zo33;n>rQ zFthZT=Tjb;Pq>jzS%_Gf~8No*$=y zgffUI8H7RtG$it_ZmRWd^EySL!EB*?@is#DffXiri=~|P&J1}ZXx18-*k%ktKG^#? z#pK&8o~-*m;5N`ftrj^caSI^Di3snD(%D`?eHjK~`1P=17_H6$+h-3;NzFW@L$ak- z`7_dI-ipNU8}!+G+`#UZQBO2xr@^vUE_-0QJ6we#!0o`#?{#oxw0RrW?p0Kt5KhUG zOwEThzOD83Hx9>jh2cu(-7d2y%s1_1kp63mcS~AabIO@KO`I+PjJVpg<+zYZ1!rFaWiPcIBB-&t|qs|+H`n5!yUD_Ed(z`s*S zIKyi>jFC1Kd_o0BQF?9$SuW{c1q#gHFhvpfWF`X>Mox+b; z!pl}76dwuYBsztN;_}u5@p%sbEfd>IJ1l0bT(DC5e-LzDELfc6W&!xQV=JGnj|R1i zs7ZiF>~*cCR}L`ry8!F=ve}r&-EqEc@Rss4aKI`LxyxB$aj27&Q(JVN>RX}IuP$85 zzqq{!>#77U(9+Ay9YA}EpO%QQd5p*>pyTNa$KxW=e<1q29N zz~UHiySRRa=5U(k=Q|ZH@EQRI`;KfFA&i9HD11ng7Qq}FebuAlf`XM<^*vasA*Zzb zJS(T7PLRobUUs_-@trpLBE1E+o)OMz%ey)=ju4hM7MZRSKtH{(QPixrIHJb zArUgJ2<4_5QeE1z$|PsPm5qW*_poM-m2TCU*oB%!sgB%6h*Wnp96exFg=ueP8%fjS zZzf$qU$K~U)o-in3qkkIrC6L}lLO9acKz=>%9O+wW+nAVWHzT?ZnYRm#QZj29^VPJ;v*trv{;7u=52`{+>Tu0!B23T^?g zE->t82DIJUGduN9o!GT8-kxU2tJw2l0f8lcG!=X)DtTLA5S0co1Yq8wD}}B%;P9KW z`=Kd`NP;C%bJ-I-1>M1*WT34x=PT%bT{{Nx&@q=a@1L-(`&rU$W&WY8TZc`|zrA4d zmKDt>$Mir6+HAL4Oj0^);48+zs$_uRshA7S788uGH&-b1kUF+7#qx}wtBM1kKn!LR z(W(<$UUC-vT^OBri)fxjToS@U9Jgy}Uo~6i%NER29t>}|BPMfEfbC5L?-KXj^(60% zC26>;(DFd;eYB)lVsuENe0?ai7=h&ME*nfZ9z{2xdkhRnA5b7~9LJ{mA<^Di60DPe zmon6^n&vpfE3=am#fdteM+0gIM$*RFbN50^y5taHf?JwHL;#00)z7-6yVoTP#jw2apr#~Yi8cj?Eq&7KT zwfl_ecBs}eyH;;xsN2>w2rZTHB5Pj;ia8&SNb??BPYkT+;v+|yS|;^q)Psi0*L3xVv{TZ8**mc_9N(7Y83NX)oRuwhSy?NnD7s=4qN_aZ0!L)(Glfyv+uF~5cwDT6mYK=*Oric~! zZ6ep>mC#tEvJ*Cwwy%t&185m0Fx3c`k;eX^8khyc6k0v7~1hPL_73fuRR2 zXL2i7G+k>h0xj^6CjvbS#hJVvaEY5UxeG0WC8y3Ky3}{oR0`cG%ydzK7Uy>?t{!^I z#o+Dk1;z6~wo{{m9^reSd77eE4D&cWNnths;3}1roPt8>TM(rdeD`=_7E*U8Y9UoQ z4+<94wJunCQAy#U&6*uu>|7?Bl(_dJJ)Qu<%i&j_N=1z$YbWOjJu z-0JAfO~4~4p3(S=RM0PX*zQaemjLm4s>R(cO)|Q2TviI?hZDURov648ixp%=rL^ZGUq(1 z9_spr$V1O77>F<>WULP=LFIJ#hox%Bk8Zfbeti8I4;bX$z}NOAb}OvboX)9u2L+c}RfKlVw4L--K5X znXP9_MU)6ESgzhJA15)pJcrkB?3uBv&k?^|Y2yZ{d%ZP$3wniegkjn`2bF5>E3Lsl zTj+YK{k$|N_YpzkS{fjSGn9Nkhh{af@xfd@(AMJ?&_l9wov^2rrB7q>9K#qa9sy$ZeWO-OVQl z)a$TT1lytPp$JTbwy7cBfOArxOnTV-+JkPxHJTRg1k~^_ntEzwZ<(gv?t!JQq!iN* zphW{Ge(F%vy_;Fef;^pGEni?}OVoKuJVsno&dZ}-0tUGjMicr7$49tHEAMLh@9B%F z3IRW~IbmdkiBO^#TX3yB4d6W=uA_p1bAs9dtWscXrx3?t?N!J*AVt&UZc9-?f!Ol( zG8(h218L$Xr#zKBhWap%F)_P*x6-;gXlGEM!M&{yMh(;5C7Z6Vl5T*m-*Iv}OU%Cj zn)c!~)BxZJ=^sRT8kU$%9SSuRdBWDF7;&f#w-|XJyKhr*tQE;3Pm_Cka!~ILvSpFc z-@59=p|jzXv(6cyAsE|8GCFG5dWbC%88=KC=v4foFJ~CTnRIyjDp!S3y4{7-ta!xQ z&<2yDWI&`FmhRrza$&U1D8)%i9Y@jd0PgY8fqMG(fJiEtBthgyNu8 z_E9LoAa~LYdouC)ZJZqH9*2?4=p6z(H7+(QmfB}*H(1;xx9cN3&Zm&6hL4!5=!3;F zp%B!fWlIKgK(qnDbU-^3C9ag)++-oiYvltS;#X6%@mSnh(1wymF{i~3ChjF58sY2# z)Nw{ocakBjdv7|r1B@kdpf*x2K-V?q4uWk}#&ezE!JTHeDGsGf8&0{fBxG!gHZ>`U zn4rzox>s7NzHMdLn67wPZ~{V#pc2{nETWu{v(6y#mcw&=xJ zHO_1~C90!Kq2B~t*(=`juusR|O!lBTNnEQOmN!m>3Ub@?3A2!G!LKWOj(1`Fj89959K;yy6>2Ay# z)Tf7S5vCa;(4N!^D2!d)VBjFkAR!95=t;Fk2*(~m6g&ss#ft7AMh{n_oX$X1 z1JWY4c37SmuOz|KWCo)Mj6<_U}PM*iP?Q; z)Fdi`6=Eh3uXb`|ps`Rwjf@RBP6RGPc~&N96U*Wrw<8g~TuF8k$X$&V8X&=`9$mcT zoXgQQ@f%{}z-hsO7!?q*9J4@-ZDQ`>=xHf?jNnf$5gG>gQR`5 zAga`ooDJ``F#C?4T;tOxRRhT@*FH%J1BGRbYE~y$xfCNKd!Ar?oR?}R0e4nS&0t(a zeGbgI!Kz*ofNLYtT=uK>_LwGiyK)SI329yJ4kQ9=u0;-<)~m*^T|tw(=K_Z-cq<0X zV&!gTWq#lhP<%E)vlHMxgSqmJPzfc>Mm2eSIm68pp@OABsmNA2F;_Y*%xdOhu&5!z zsl`kQTc>)rcO?8N)&!QA@eJ&RaMt~jOYHZh+KTFfu6avj*F6aL9-<;3=ImT`gjj4@7AN}4LlsT^IHECecrOMW zZGBTpnn&@7po9?dIfSZEa8up2T)^{;!i8yK!A%XIQW8a@qfPT`_n!5K#(9Bxe4Nyvbu2&$fDoQu2IEUDA4&XQ;LX;k~g>q93T<$Pc z_1lzH=m^kDjU~wSS)KM#RM82=!6bZ4?L7WXqiKO__8ksbE;nKmHm>f?D`46Q$e%=W zfXO~jJ3v)Gvsh*Gk1?#Y;08*0uR7M)k7sNqdsD-$5a7YrrFh7MmB)#|pn?!;@3>sj zJYRTik%mB}$^tQKX~*r=`X1FB^pt@~zdK}8!wfUL5KqchRNTT?P}e^NBfQN%w`YA! zm?knoZ6gC9^OaDBkBsMx0 z)b)cAjJy!e>Xk_I%2TNJfmIO}5omzYc0j3=tIC0+LY6KfTMqhMx2C&xb^Xk0^EN(&94 zYz0kw%EI;zb1iT>lZKasWpa3M%7g@*UW1oA%FE0-y%9l%GzE1STxC}ZppM7nPT z_qXg4o9ib}Si+GeJ*7q)v=NHtd0J|Pzc)KsoVK}~6n=)V!_ejzZCN&%pB7SWp150b zu)F}{%b;LO>L-I}L!ndF+sNGd(*Yrwvr||vDEwNw;TrhKaikA^92ACkF6cdWoHv%6 z7e-39MzB#8*;JB~`=-y@jO+u$UKxTynZ@=R0rWRqv8@S=24Ttggm&^5ZA~)Fkpry{ zDNr4WpzRUfB2+LWy=-w~!SvI}wcV3J0+^-EzdOAR3qVf6)s;{|G8VYrMJ{O%B2}JM z%`$fZvd1B~Y&b9(tgHA`*;XTh)~>+ANUzyBegd^Ld83Bwe2ckgu@-Sni5rY5n5+fMTU!-auT%=^EBn<4n`kFG z2Uyx`5Y0^VZQWa#J2gS{MCrMYq-eRlPMSheOw8?o(SVZhgskO>0#2*&qY%2iED&qc z-l*#WR!&q-krY%25rxDRmTB2NHU^;-$;d!9KU>hi1*KQPi+r~1@dD>VP;&z~I{^8x z(bn-$+*H794)j-y`JiF&`oNXC)nBL#b}zPj-vQ?3w&XCEY6RWO)@?N_K_59Z*GD zLd83_g`W)7E@jy4Vk>EiP)!+3C~IM~usgj+^vJ7VtwGzu*}kfmM7LHZxFhGg5!oD+ zvfiWDo6N>R=wF0N0d_#<)gK6sPzkkxLao|4&@?@PWf7~h9^q|)yel9D5`%h3sJ52# z`_z8~(Mg4zI@`X~wZ(YExzMfF5Q5$oDfhZJ-m(>>exV{(`FuBsc)`}QH&S~Du7Sfy ze0P%fjh7KIlz_d|9&7V`C-l8Qkc|W_zLDYIcZIbo&b!6v!H9;Jfqs_Jqi+OY;J5AZxT`mY3EeMW~iXh zoq~1-5`+m$IK7wAk}NLN(U{i@lPg}zPinJ?+G}&cD~=2SqEzOeAkmhXb?%L*nV`-J29@GTvRxu%ey%4>zlK1czG7p z-;6kV{khYY_+^=EwbO!(q%`Ne$UPUH#6Z^o?yfr%)^~xL5nis0$IZ3rZP3Bk2~lfF z@OM>$s=p9mz{~;DVfH#ciY@F+ozY45g*)S|tMEs&L!7|=z)2q9d>d|SIZd|oy9ZFZ zQ3+@XTB$||Fz^I2s3n@g1UFpZZKDUE*eno@S72c6eVDL#3PKN3ia>jFYhk)YXA%&_ z#P6{n=a1 z)N&;%q_}`ucsJj$wD>ljEUfo%Qzw>10|set7|q>(&Ax1q9j zdz-$duA9ou;^wiwjt%JYi_qSzzCCd2MY|+Vy>(F{ zvd(MdIZ6c<1}R0C#(x&_$$nn6AqoiWTCVKE6BQx?C2okSnunP1>f0Y!&hz+` zdD&RwM5&}As&vqBr-kUJ410BTpH?Wy?s<$@LfKh34TvqUgwt|WRQObcwf?JPhsD@W zKx(tlP_hR(ok*ls);2TH+{|h!;2f`&;`if<%2t%}>)0-1s%*yE+6IsyyqNP#qZ!#= z%7ji950elM^<`6sTq$6Z86VX`3Z0?yd9##$D?t!xP}2c1un|dOZ7fBlXj>X+-9{IHp(*P^5I>*Byynk*kvRu- zN$K2n6zOW1?sr)3$8#n)0lR7V_c;zi!?tj(ai3(qbzSFX1c2_V(+cT3^IZd2oQIWz zx3|`9Wq5OyD_*HPqb{sFd5!S{(n(aI=ND&L%4jp%&`D0a>n}| z`#SG?F5-kOs33aGeRGh!_6)t_#V5Su1;|FOe!S>3?(ic!3)=}~UyGui-dW!*j7W>k z(X`Jx5>ON2&4B7E$0vLYm-4kB)c`K*X>JwLIyqnzT?Rm>Xe(7cQ& zeJCJPfzr6J2PA2}^=_q$h1`jW+B!(L48haDZwrQ$nh^B&TuhCi(XfL~TN>H^<9#Kfc6kXe$l6g!B_Qd}ycLEy z+4paum878KKJbJbASBL@>?~3fV1p#Dcy{gX4}h;&&e%d_fcc8nhXBorpg15*ea#i6 z*)waq7Q;r)D4p1MBA3$ep=Ji)@sNl_k!%kMy3^%k!BE0Jm|g92pG8cakt^RHV%mW) zU7*uLJgWMTg?r_Dul4fs#VV9yp-p&s+es0jZ1KYY_=!?P+^p%_6c*6ju;Zqs8%Lt2 z5K04Kl&p8Wj_Ol#?Jp6-aCf>|W@tU+wFy8oqbli>s5~QB$*zHZjOQhXt;oH&ttci; z_S?d57LtvO%0$X2gGfr|J1!uK21~x8qtVY5bGp>V@GKMvdK-5K&&WxxVw=}` z0V%eJegifLOo=U9>+>6)A{!l83WZqde)E>NS}~X7hiu$EGvK;$M6O5yd4P>qTUjhq;zUyX%iSvp_aHh?71JUlNdfdUHzp1UN6lJf{!264nO zq^$;-6^A#p8CJ_s39Z=7r^;$DC8%Oic=QZ=*OE(emKnKt z)k`Su{UODlXj9jwM@HHYw_n=06$C_hCkSD7_4GIxm~1H!FbGKZdW^R*{lQ)6?Rb^+ zyRCZ^a}TAgFODh&iO1nrx)6r%VP`WXdfLOtJF_F^Ezcf_>XM(CI0`$l;_SgTnXAcT zZOk2W(^z_43jBTdZmZxX8smc#g&~UylMw}?_1N3i0wtDvO#wkjm1^!5-sMLw5lj#d z(Me{!wkGDvnD2Pond&;0*+TGPl?h6U;FmenUSa*|zBn=BVXI~KQKiMR z$}!}nC7)WEQ6-A_T7c#?H=)ux^Tw2EO&>{Gbn30rd!8A1ts$G%^>`*~sf8%j1s#mI6Cm@Mj;>f3~>= z^hyL@J=K|Z9s*^6oaXOhD+(D3ABMX39P=51CX@(^4|e#N!`z_^2*Ko$x)XI<2Y%ZR zG`F)|YG4SYChhb%*8P}|>EdYD--mn@wS1QK&8wB}GZF|CK1G*IlosBeUwD-(bf_(y zT|kUM5q&$D@jP`!rJ^S>2`XtIw?>agI3%)M*QxJ!p`eEbCeYiVx`m)Xmcyt)XQrpm zQ^~k_PQ^(t!Wk0IK&CcEtxr_`8SIB}Ui&iUjhu_ms+>(=&sox*C9AM+EkS z!30dWaLQ1VKz(z0lPYKNY+bc*_9Q8Dp|b`2i|Kz0jzUe1X?L3VGW9bb**EWy!?|cp zvm})YSvcDf^#p*dFKeHDTuZ%F81_ys?pZ4*#fPU#q^GjlemRDb(+ZsH7^kvPsl&ws zQNAnrb}Agycaj3^R!*pm&AohH0G+DZxE=9^9qdxNgS@pl)9Qqs!(L5=Zz~8rKrgX% z8(k|4yh8K%IC}u1@^0TE$OncFZ`fEYE^2_i%^`B?YEo41CJ3u`gw2a4flw*x!XU+? ztpsjZkyhNYTrD?ISILw&;!JVT0Y(#Q5l~ohehcxUyKRr@zsao zb@yc^->S~$^2(+_yFsOh;P)nM_~!?~%_C~`mxI4zzh)a1h{^~J(D#|2reO=!bdLWl}MtrHqjwCuoCA^eD&+?!ldR z_^A!TVp_?p}JaG1wyuLIQKM_Jph} zR-di3%8gctb9}7|@^B#W7^=rY-h35klQ=C0Atub|1`XvXAl16G+PYDb8fwO!f|it_ zg6*6_tuf|q5ofH9h0}ZhtHk!Y&N-bs2GQU)IDu5!YYsLvhGqba4Z1tfj;N||J`mu| z6mk&(*IBdHM7Nl#ZB-_U!w=fDAd^B3^dGq@<56X>ImFo6)o*Agg+<60NF0N5l{AKC zT3^h7Ef#M}44#|2J+i7??&Fl>xZ$@7Egx?aqX?K2Qd)2v+C;=uD8GRC0?zojK zC}wwg#V;_5fZKh*t)hx)n@Z51RhQD3U17Beb#iVqUJ5AZx+K1diUT5Z)y27Hdf>Hi z?7)=qPcrV6x4d#HmrEwf1>7lbZI z1cy&&XkeB#aE9O*f+653CzuMvJe4S1w7#wav^B`x+vpxTQP;~JRdntO?S(}J1EVVAxUAHR*9q0^2lesD!vMilZ-we(YscE09!eB(gd_g0< zv_g_IK$vtoHd8A`RJ!me^r(x)_CdRKF9wV4o5#0n0Vb?thl2oOLe8L!RT;yK%?YST zP}GeSm>!NcG4YUBP^k9t+QmyFeMCA#TK65J$ya0ysm((<=qKJ(A3e-Kn#|sH#!ySb zvfl&;F@+G7JLdV?5UUZ!^gG60Wwlk;OPGqjj zf_!n#FrpMlT!f`_(`=5z?^_uBY0{#|tbXU>3cbF+2`j8!%caP>6K1=J5}K6#2v;D% zv4Ar%AVmVWtzJ<*6>POPrzRd(J$Uo!Cvse14VK1QiNpC#wniQ`z{`%ekFt3B#4l|t zan92Au}IurO?bi@c>M2_P9n>&o4LByj67h{i;@*hLAVwRKR>TZK|8c`J$Tj#3!2@{ zG33}~KqPAxjJVonhQgNODZ9(GM%U!aK+XqK34LJ{kr40ss^VcahOt53>e4A z=B(uKExv@l-daHN%qc95Rm1zfY^-ML{c zA9gbU4){4MO|*L>R`euv?Um3<2f5?Ms&o@uInMzSBB35T|(y+HfaEg(bXq*;N!Q4an!-@U}a8W+imMh8wno&t>OtQuI5% zfNh^iQ+_@QhBzHM*U?eqacT|`#`mMB%8;2kbfGU3AVx2qTz1NZrw}A-S|WaFI>!%=ieGN0p1E zE=Bm&yU=J8V}sBTE3Vz`nr*lw^GlAKI$udDwyOp@j0%pY%T=>l_7NEZPj0#jr=ZDf z?KsF)FrwX_h-##^LP{vZK2#>PeO@3w5iF{6!2{WdyfM0;S#${1zh~Xt&~4ES!Npe< z17v8>*IAU<6s*{w!$h;3Pbwg}lob8LijcA1mc!JqJSAfHXq6XT+x6$E+6z*KO=yTV zI>fhjGmW&jcjK-0hi=1oC#Q(v zK;Tu#3Z9Y$-5A?O5bSX+ zG?`bUk4A4T-h?h_vd;d!h0DX}3$BikJBeHh(>9iz5(7y!>Pl;{rV3D@Sh+%1McY*` zI!c8Q%WTuwO*j5;nI*Rd$8pNC@onvY{{f}wTu=@>UZYp<1>&h9EZl8HAs)?>T7 z*EY8Dq&aUz<{bivbRa8;sJ<@H9Xz`USZpJr#A*6WwrW|0RXWCYqW0~!K_Pq@+Dk>x zmJaqLYhnrnuBx_cku}45mr#DNOP6l9H0HeA?{0LV=yYu5`Ya5m4*Waz3)(AdHEb=T z5Dg5>3%S^6h=MH#Nyk$_%`1njR_CVypcA(IC{MHo+^PalV~XQXSnz~!-p6nvMGIQW zNwvL6Zxmil;?p~u2O5M(dNfZAWv&J)(#_tZW@?z6;ftCYsR`Q>liw`F*cc;FDI5+p zj4=@6B&i!7I8lK`2G*#vW%$9j*!gCIJ#a06qc#@2eWn9jPQItY7$e7CzS|^D1TG)a z!0`fuk2_Jz&hq+oeIbV|;kyjx8DDaoL4fl8FUlYI6!UL zE9{uOH#><{C2I9m6hH~biwtC;$XSx(Egi##W2o<7`x!-6+I?xducJKh1GVTHsh6y|$MKBAaeG!N?nn zdY`m$Rupk$;7GB0GeD`!G#VW}^h6PQ4$`oZm=SW%Zn<;DIX*Ye?3TNgH~+^`mkJb@@G#X%=~zd?zrGGKG#WmL)|tG7?gj zbyXiiL5`t>CZj4^YFT$w0)T^I(SEXs>kRh^095Jc*>X*SngUD*ERu3>spSC}wG4n0Jn@H|~dG9Kwi` zj#_NbF80d=5u$Bukrl;yx_?9W%oyZ{K7Dulv1S0za>b8WN}!Z za|^W|e^u7PRCm`@cOo=Ukv#ir2>!F5HZsA&HX|VY+mpDZ!fO6%x3~My^nNDm%B9SX z%nGFQmB)dJeP-8KXdk1Y$NIE^siK*4BBW&T2lh?;!xfcASLw_YTo_{rg9rliQJE%% zX=@N2VA^_X0W@vJ$gdb<0@p{u4dGxY2vZETgo-tqAYtT%aOf+-l-q!13asI=aI;K1=a(iR|^D6mNZw6JLcDg;EB zWVSZv6lnRHl<-fW8UeyU1dx9L(SxePSWjCopoqKpCrWy;JB7Lc`sfzFbad$o3M}FY z?y~4=MdoBcX#E)GwuCVVpgBy$2LghYzzkvfs6sfyT9FS5m5-+Hr1j(R&|k_#*ZZJq zdxd(n{7C(*C*X8wf|ch1N=zC%n%3ajXomIH%+j^eITSLd7>E!Pwa zC*p$vp^<1K9|#c;3@ia~0~=xB9ue2B(s^(X1l-fuljR1jg&BG3^q@+ht#vJK=$u9% zJroHQ>PhTq+A!SW#nhlt1;7+c4aVXb^c6q?2a9F4HiA1^>)l%XP;gDSWsBRh-cni{ z(2377Hm#6n<`jz6L0kNg=oDxI6bWqfYV5My0>KS22&nd!mYbCishdtu(+aADU?ryx znAuGNd7cpl27qyW;Cv=69yzsL_&pFDU{Wj6q#4}Hzd^S}-OC4q5NtsC*8!F&b2y(^ zK~K}FSQ1mSdy2BAx%o4C>~D{R|q$N z1jVochUr6dN_ukYpn@1AssvcK1%hd#@q6_#EgsezT%9I`IjK-wHxWNLL>q?P0p*`V zYkA`L8e+Jhq@KDfW;3SDAPfjvv!&Ao5l&kH;<_;`l)xA^)X_Q5Ad>|^{7wiry557l z5dwy0!oV;H7=RDa=>v7yt-GvZV=anm0j!J6ha*0DKT( zlS~xNv)@@l$(zBQ)>8msh>O1`~z}1GW50H^>|LH^4oGC|Vb|5P8kBlQsjNJN2IZ z3T|kE)Llo{VT6G-s8nbj91m^@(0R5L!8%YMx}^<8Ziyji{fdFBW1D^DXoM30qU zCPo{|^&Gvm(ArKQ!yLr2$gdG<08Q+$=n+fygBXfsLO}m-KVS*8pqF<`cLS=Afm@=1 zbr`YYrVVhXPFImHkYKIF*Q=p5ZDtCEM4QxLN?=Xzs-#*1`5K6d!H>RRtU77LT5b-0>Hrt!So>-FtOAg05P!M z&jMP~OWjq70Y1lc0l*NI*|eeM0}%$MKAQs#Ay@#=U>adU5PC23)&i3jR|t8QCwV&1 z0{E@@z@;RCHv?6%?nlX*Qd^EVLbQPEh%n-{*fzWAd^{xR+&IjDRnS5HnYL>~>IH>pD19cI~@DTah|LmPWVJWx#wrEA4oT_yNriAiw4cY+U9ji@Xg+&44YbG?0PZwFSg%>HH=|+tm}Ix-#Zz~|xyhdu zE!N^kp339;ERZ3D3kZT50+G)~U28=$tT(L-bu>-ExO&Z(b-JDd8io%w#{j4yycjr0 zEHiB-ZG+(Xy`1JPlLiQw)DQW*Wt0GE&^gU;d=n(5-VY43a(j-HW{3g929U4U%B{6& z9YWQ#VbE3YY0!mi%?E(2H}JbsA#^CZ){BPi?OWvc;{)SEHw~ga7|i zyW#WpVB`ZAE?p>cg=n|9inwf9LKA?L9BYCN$cxjv*}Iy-OTCne_kA&V7Fk zLI1fH*^7In%|j|*p>@lkj1M<2{NGS&`toE|8ncmXtbW-|Kxa;U*d7_Y@l;jw6U3sr zj*^PL%0hFp50N}9d5zGB5S8!D)Sb?Lih3&|YkC%J&MDy_<0{|iaz1B)EwVR)H5ok0 zmt4AnAlM@PM62!@#+~kE-Z+Fi{ltEx&FNhz36h#72ZZ|Q@Py?b6Xmu9gr-Fh)=r}} zXzglYf*>uirDOXR1N#kzg+%&-GK*|k!9ZscnY3agFB*il!Xcr-CYb;Heew#)#KvDh|zPh<`*0ghVPZ&{k{|QE7 z?l_|Hob{hdg|@hM7U|$p5E1-DBpXupCg_h&4Y|t`TUfOMzsQ5K`B#4JuuMsh<1kYD z2yfN~_o(UWe7<0JqRSZb5RyhEP5^H=C>yFyr3t1F*eDbf*0Fw6q_Y2)cMbmyFVorf zqowjE&<70JvYbW8b`wH>Dh%W73Ly;Xb3Gj52V)_iP;xcMCc~)S^@|S`ec1Q-oogLF5+H}g*5&g4y%&~d!0?EzrgG@Y$Jca} zzG0>cT3#^a*sp=Drk>LS;k|rU`}eatdr^m>@S&8fAIf(`PdoA?ma9&~dmN6nPxn{? z$(nj&FD&ht6L*+r`FhlXVrmqVvm`S`x?D;7RslL!pEO!Hb63qSr9G$p0-kx)wK9kh zWuhI3E3oE|ebLW#Mia~-05MlSS&fbp9VN`_t`3nDI`KW*v7CZ5Hs$PYnLkkkRo6{H zqj6E1%!QTL8w}%Kt1Pvsmm|YM&x-g2+{-L++#rhS#`!BY%#n!!bzE$B9lELYHHE|U zj@ec)IHOn{EE_>=fp=DZiASwvflZV}H_sb(K5wwCH|JRmJO4wm1 z;v|`ZRQ)CFkGZ z-3m@^*Z5+;Z|yP@a$@=BtQ2&`mFCsKrL<}L1J#d{i}J4973RZyvaIoCXOtSvd2_Eg zGZ8Zlp8mUb?kZj1vR*a|Hye)g0P%2+T-7tdA5J@BxF>!=a}4bzkGeFjCjM|~WB&eq zAdP`8{V`r#5oa^t>eTB30XXJh3mN`iy)ARhCqQKKFz-+A0=lI;iKyHqELIUFU}uP1 znVjn{E9Wo$=@sg#fr7@lbk1vYCpPN=ybSE0ExTgYeyCPr$%RujdaYcN_IvmoypN*( zkT}tDWftr;#(TA(|CQ@XFXCFW=?}$Us3PuR>6zf{M8TtZaA~XE%|0`UkMKMRA$h2O zMLbT^AN{L!eDBgsYW}1#7d6}sRH+%s<;S-ivTn=1d~Eq?O7`sGI>y_UXYPl=ffXaQ zjH{w^(8JUU{9(2F!;bl$Rrx`Btn(adx6I9=9R|Zo>D!%^|H+HfGFRikC>A19 z_p8>FuRH#pZ1$8oPV~@SmLrd;OmYpVJ7vJ}Uf5Kezw|lr>56_TbNU(+%@j21e0^kK39_@~tx%HXr2zfdiO+sqYl)#Let@Jf-keUdodc|=<&FYwZ|GdDp zrP!f`_-k_nNiMTQwrw4uaioz<=w@r6yGvvfI=5J^49ybWY?5wBPz%Q z86&u1FQIxN{f5LnkK3GCb5%z{_p>s!6$+!r^kFfrfqP`Wh$ss0zDxbb`aLdevQ-jN z4{bG7z-l$(Wa)6OC=q3fn-(^uZ7E?M4+dzX${*T#c^qYuP*e4y&=<+VfA&fRHVr4s zUP`2r(h4Rq&HsqX@khj*N30L@5M{mu{z_I|^B7ud-cBOww5mtlKz9H;gcm3A7i=$Ei+XUjyoF$ z6ePCY5o@XSMNj$u>0}~f4ot$KlJvU|HzGmBqR_-TZi*=EQzxK4ElXwDj7EB$0wab9 zf7zNHn|0q*f4+;4-o4LpwQK5t$qx}wPgn^RiG~%9guLI6 z?oEEt*Prq7#}ipZcJ|=b7n9ky$Mat5BOLr;1i?<;b%Je+G3qufs-o8z0qvb8ZlWcA zb;%_Z%h}uXc$v*jyh$@=;j5+AkW~fQal?AVujHSwazc(059Rq#jZMoba+XoIKco6n z$Dgcn{5*9zLaM(GOB*v>3f!QBA2qOJR_M?U3Off(+=2%i?Nixg8`C4|lqoF01L2t0 zpZNl0R{y-(vle%q$=!pT1}i*<p zTkI!ewSBkDGZbb^1Q3i^oHOlYoHV{(cnDJ4rVFg^n>{Ea#I@J+yN!V#evh^NV=nWW zAhwvwK?OyWsF>dqD@hJtb4?NolEDEG~;XutHp43X}~N5ics-uYOjpu#|BsJwbF1jX%cI$C2j$TK`Y4 zLuZflF=&+Q2^1vJ%%ntKyy5kAfTfofu2J2@p1h!o4W@tDl3@jLYFx>e8$+gU(AzgE zeWc3k&#DTeZK!-%K*GR5`iu{$6>a=wwscg3goRW6Y6(+QdvB%esX6q7dS9X-I#NR!q zBVyK!R@s&0QMaCJbh#muk7V?Aip^W`12839y1XNO{hkv$Yx%V(k?V2Ae0xWBqnRrv zK}HZYi}u!>ymp)l(~a^BU0+o(SNKu-Q#^}gcr{sSXI5QRd+7))WU*}y+pm~Eyw+rx z*Vqs-3AkMGl;6F>;y{j5l?&D|1JX;#TJCB>e{v&Da5ql9ehDVt zfbO!0H;E9NG>clSB|AP!6@MoD?>Vnde#Efa)EfiAgB1&W+4Fi|{BhD}!k+ycVM+QPsq zg7AvF=$;v|vXU9O1CNghCbf7o_TQuZJN$Hgr>rKMHatx{$k$qF(&f{%it z?GlQ(P6tmzU)})om@3B)G?!1Ep!MRdO7OX9Wu#t?78K5U^w^($`~AdwQ09 zvce)~BY5A6S;=!Ec$NsP-6mz)G$njftgA;dsugehi>0fYWix!R)_{Kx^2j-|!2Co1~pB0W4^pCwW-DSiU_b!Sl0 zX<8Y^CYczQ>Ed@J_By65WMSF|rKBPD(M+aG=MQ~pQhVJ~%-;}AAZf~ZV>wnlO z^z;E*2L)mvdJMesP3xp_iN?_pnLpKgcHX)0$#MlQ{&Zr`>~AFBRT^j?2UVkS3FFra z@M|Lwj3=*o0Wq^luZ;G?6=YOV^@!cBbcS-zv=kVsIG4(bz^!KX)5%+inj{xtSWhA` z6+5c>D`#L~d^IqQy=%9Nu%&OE=flr@M`<=MShE~o_j@{y{CF5t%kqQvQweaUaw;_E zGKC=}@}(%iyZVb`SMItyGC;D@4tCv!vFb5XIf))jyQI^bsQaAYl7<{@#nQ(oG%#&Z z0Pen|)i$GSLngeVyVXC?cl4nNoH-}WEm|QT*nU^*2GCBRk^pEcXCzFeZC|@+Jl*Eu zs(i7bKB7(|P?LAcoN?%_p&=2*4Svki#(&8;=t^oH<8blVQ8lBE*ze-t8jhXWQC?>_)}T-GFYl_WD2837!`lD z?|?J5!@k&eYZ49*04rcA4o-PCA)Gnw_NUFi`(ab!e@|D|;r@2G_GS%w{GiG0rJT1) zLk!empRC_0wcMUje0Um20g0)4vEVl2=#`+}g0gyinJI+i9XJ|sRf7#hC%=3+X+#bx(ggWo|uW4J#TI9KzFCTIUubTV+Ve;D=~`QW&a zOZBglzbUV%3CV2m_V;WYr3OcEi1c@WlH?K35YeUxI{4CzIT5m%s;T?F=zwsmWS^Nz zjoJ6_18XAB1Q`yMHomjmT7NmF;=6mS54Xd0OwT8h=38!?(no+kJkJwI-lb;$9mf`w zA#x&aPtG~+-H^sv#M)8yvU+4aKCO}%lA7WLF0RM|C|i91?1m->OWmEt^u)3ESTYqd zHZI58Y(5``HmP4T{v5Z}V<}g(JaHBzdxF&*rpDE@z&!loiYCt{g+KVb!8#HZS?IJ$ z3i*DlLv6rWmrjo74*3FWg`SPJR$R7eKY-5ox|?4iR%4s4eO<%Pqsht6|NAD-g{qIW zPl+6~)&a`g0Q_Bsm&2S9#;*oN^`?v82okwh*@QlVe4M3x!v>6>*& zsV93f5YOcrx-25_@E^jG@8l3A-LBFkOKomdoD5G{cnT+%>zqa?D;MRpJt|Tzmg-&M z>IXc=bfMe9o^X-p`DlKvYhR{R4o>9Tix_fA;9+=g9zf+^7G%3>~6$UKXrxq?s5_UIe3Vl1hH z(lWRzfcyu2m-E0Myb-|qzX;H$z`YzC-lr5tYPvE!JOT2$gjVk~^;YuMCX*y1zTrO1 zM~YtqjW21xspkVra3To#Yr-qOa^bwTEHNEco|6d5^AK?Cxh*q1mH=7-K+JKZd%s&9 z?QC+%t)=$t!YroLa&=1aM_=Jqj=ckRFP-SQNEdJ)F3e0*ecPmy)BIXgknm|nkbPzN zDbPl$uTk_Ur&QS@7Kp6ON)p!DbNG=S9Ia$*UmbMADdz*bm;0S zxZlMPuW&v z;bsULH=LsHrK%BUZ1GA084=?Vxd>sri2-Yp=8L|#!zM|BYe<;a>9GAkgR%5KfFR)S z`=||6B(IG^m7m@G&(CbC+i~;|RpQtSQx{EZDKWl`5FA&b0K#v=`!Bq6u&C6hu+~NL z&kL9PCKLqlAId23*hnuqnl+AV2hN^xKBK0nMm47RPNf@VZLVt-s|t6&9P*bIr8lQD ze%A-9H@9>#B{k^lN`&brkp0yldO6=xuEl!3=C!BRsJDQ}s&bg!%tMUDidZox5GY1; z;~TY^pC00E3Q0HDs#xADlB@;Rn`4*$4QDZk!IyBFv5EjLrZCv#iDEa0)8pqD4kI(jT*NCl)D|UU%8hN%7CwhC%( zr@{lBWJY_Y={qz`y|rQ9rGg2t3mw$Q{QhWnWMs{EuRYj9ndnb467kw$t>m0X2lrie z-IA*nZWu1EabtP4PAAn32046ELvMz2mEY_h7iBbMK06qcB(t{MvkY0+6$60&V%U;b zuBSE6#Yq-oPhV(4taO9#SBgFusr{O@)eeE<3Vu^6YdYB45e(5vAHBKxQ9IjX{YST4 zL>M{dlaNPIBDKARBw3z=7(@o0+xZk`TL}m6fnYoXCq1+$pByXe!`faIsoVmDBj*|? zQ_|Kx8XtbEVrNs_uy4AkReW3{!FOFgnh6z>`s((W6r09bus(86>^X4-2WjgaOYG$5te;fa78Z-SOC?e| zN-2#sw;hESY4#?$ZPkpJPk%|soJJ}M*_KoI#toe{^yPOuaE-dX@re}h7r7Us6TE*6 zWTB`bojzP+SIIn! zaHN~)Z^TahoAVg^NkDeVFBKbLk*<6P%yB-=AD9!GIaqB#O0TMve4j#B;Py ztMcrvY;u&6#T|m|Tn=iLhm<+#$@F$7rL zvf~@^3hzH!K2jBIL5ayK0vLu$bh2$qo%lamd|AX;^GP+#|AI)reH5!+p`$vy;NRNg z0$Xg&r)5WZE)mk$;$sa3Fgq$yQBsSRCJ7h_3i&R+K^ln4R^fXJflh4YwbFSF%Bf~| zA~@ge1;?>&ma6|GpdfID{$0?<9_g@!IsA zuiMD$D~>X<*D!kIpCb2Jh0C;c$*A(T8dHAEOh^?sCR`3DToK*m+C|Z@m!0ZHhNa(R zX^==E{FWTG(k_WUvw*6!C^m?E3eqVQ#rGJ@M9&fWtSgWHf@~J^RY^Ryq6s`+5rJm8 zTww=fYkyI{b(_P7#qN8_vkKIrHybh=CCa8v8YVX*wq|FQN~b$T*<0oI2_OxApB(zc z+1|L~y%nhPxBBc5>@*!hBjBY^w-g_h7*{*=*$8Ra!b`~E%$6kwS9+J#HBUwM3*Ho& zuOjW@u}3qEOe+2Lo~n++qQ6dPjsNJNutG1|ZG*bofnZo`OY^5U{TF9>P50-f$2!6h z#CKvkoNm?xdfF64gdW6Qj1lrEaB3XyhTjv;0E{N(U1P&Sw+i@;0OAp$N0N z*oIDz^3FP7-4E^WcM#_lu(_GoZlNic%jl>3E_9Q3f0=x@@&C0u#y!Jga!%umox~Mw zbchuK<7B+9OJG@n+P(xB4bRT)_iD^j@ar-sHaGc=YMbjZ5Q;OKUsn*-3ptr)(h=%J z+)TLqkeFkftFJhyi7aAMjph0Y%&lOU4-ACUa;0J;wnmitTpx!e5o&q<0w6G|DG`0ugWyZ6pU+egkN1j1-2Zp z?7o^FWcfP#HF8B;i6QwEoW%2$97Tmvi9&=B`S`gGZ%x*A z+cg@0T0GkGf=Hd!Q*XL8R!5Q%DjSU7MK;%}r~>}nED_L*C?z~L>$8e8@5Hh}rH@W` z-}fw5pWk0*`naF=m%t|#4<60;+6h4N}Ka$4F)tng@sWP;BjigPmTT}kyX~R zeDDh@msk0(Fk$FFxgWWC)~x5sD76v^)u*{gc?RK!O7fqJh4~1I4)BSdKSnd&xse+E zgu4q71*2)AsaQRZkYE{R#ioMk$;yY~Y3RCnn5O1B#Tjg6>c5{;gTB^$+L`IP72h`B z**HY$`>HR8lWG5l*9lF&PPN*s`%*DYwsEl@mNg7+0ceo9L3@jwTWJe67`&B*iQ@G1 zu5^$g^C6SyW)Z01ZWhg5#c`pn1n=r0z1c_r9wFNohGG-O`eWn2AMo z6G>>Lq#gJPOvJha@+e!Lok~HyC6LC==sXH|Ir7GddE3*5=EQ}zo2>tp=?c@U6mo&y z%ccdQz5qSmx%wbJp>!oOiiapOX51VGfwayn{{+{jd05Hc{q3Dy`t-MY7BnZEe{Emt z5pP-5EHJDLJb6iO8D6+3fEfjSk-$zJ9I+`?&^Q&YVSj&%$^?kYZ20xbLo7dP;bDtF{)>+!oq+C#!9k)0R+aH&@nOakVth9msfPaq z=s?ckm0wEuTsS|Q`YVD{;ED$zHIBwd>N1Am|7tqN4mFqW48GXc^eY+iUz%AHlDe^^ zxEa|`whPf_m`$!LHkXt13&VSG1lM#!YEUBDf1%cj?(s({;>@D6nR{NNy+7#evAs=? zY)#uzhVeXVzgtNs0`9Y}_rZKBir7c&2Fh;n$|TD%NlnumxxB?7o$_X0iP6;rSEnTJ zg@wzF<5pWhxB$Hc?*8_^KPx^)1SfG>Y|aU_vk;O0^U{7S&RNw+2-x+5_zyR(Hr85T z)A!;AqL;1$w-dFAQPbrCKXgF(&rGMBwqtrhn0GqD9Kjrp;D1*+WxdIwC@^7+5L30d z=`21*obim`oRK1)ny)h(`z|)zYxIIYsVvD+yb*WfQUpC|T)l`?dHFW5C~i?|r$F(Y)FFigjKRzN(Oba!Z+wWbTa)?(tAxcFzSF_m zK#G_zaO_9Aq%>u-{dk!DgUYJOM5@A-)2yLZ8`|4fLQiUFB6)YQj_<~<^c&t9hLzG- zfwO;?KKW~A5=ooies~_!o z(7`q+iT=@_=!cl(c+7>JdXDChzYW=gyd+Y!?mwA~mE5ZmbLmgg)0ARlZ4UK6M=AWY zf}=mH*vhoeHwm=K6N?-Wfk{xSPD~bLBFkhJUN3R%0<1En_s3k^JMvFNCgHf>?K?@o zVWVwb&}n0FHxp3-Y}7s{Z-l;+E1RSPu0?;W^2u-?& zY6jAHBtxO2PU&P$>^23r5UA?3j#cbou!6IY1R1|dBcsZ(XuO+NBKtejS&dsoBGQ%td@jd+9E}4O@98A^C+w}m?u1U zu@ZYGz|48`$K@`oQAXBQp;`*ifELO(l1LAhW9~uIk{SuQ@O`_3YGQe9kiJS=`A28H z7MJG*q<}I%1^H*mFtDvjLv}$L%bFrD7;T%>-FG~R$ycRc_j3brru{kHR@%Voxp*n3 z6>IgLv&b5vacq*BK6LxXIvU+TDuR1$9`Q@#r?DZ-xUo+L^4qdgE7;)sbejXMLqDq7 z#Dm)_3%3PRw($ig@A{`{4tgHHfJu%=g-2nM zXnl{*ArA&j`Pc6;QF%~?s{PlYeh(F_(FPQ^fgWLGQEh+5h2zY{tjpuJBnwQj;{SC> zC-!e_xl>Z&oyM3HJz24m0WK6LGJ7THMLo|{^;~lrQg%{>3OvslGf`mvWkwN@QG#jy zn;oi9(c(}<7%_ML;<^cXTQY6Ujm?kF9UIq7Sk?#|<4cdFF%hZD z*Pkn6EG~5*oGUi^u&J@3+5E4%c^$CaF1DIw)t0wAs&H+Onw%<$5*WeGuwb8m76k~B~ z;{x=#=QNdNI_kYE)-w(RX%$g95%F0`xYG^T2_^|`e3QZrDG@uDF~KQw-xUzVTwX$j z+WXld4BRA!IIYXHMIH+}U>i^G4-difv8i(OHs?%95Uv^OJ^%(7!s)kOHrzh) zg$H~s2L%^t>F}9hj|TSRX=hwTH+HTJQn}VX&atKFtB#i@IDN=oT*Z;&Ok{sg_t>pI zy{b|Bu&i+9sM0{p#kNMB(_YcYx;$PXAy&FSqr#RI=CX0lpg-ZZV$yDqP9ADf~$_m0h?h6tX zA2mx&S$@fUm7ku8;D6fzi+8&s6VmMP_^(Xh!khOR^O0`t^&}rgxRSft%v}uo+d#+d823u3^4;Qo2UJu?MobASk{QP)g z==ck)F`u>fh=qk4dy_@>&W!9m(+j%cGgo#6_}?=;Dd&&x36eN&HoD|DUPR^u3haJJ zBJEzdj0D^iDwKzLnIF9epN_CIUN;QzEA?7wV9AOhGq}Q6Y@Ab6iVL;x*8g~12gLgM z{r-nDYVLzLtUEn9!_q8jH{^%gQELVnIi*}l`zMMrlimC~ipAHqA>R`&2cmwwBh@I{ zIeu#iifY>h)0h*kk4=#M`}0u(-cT82_$h7ptwURFvE5Y(&sWdBmJPod|Ge>qxSO$B zi~gF-XTBL2FB*gA@(pje#PXp% z+ez;u-wS$^Ye~sQl}1=uMF`<@DLt16c z>WY8@xv79UYcJk#fcI3ERtmnyWZ!-V%u?C;>L9#Nb;NLYwfigtPKih^-bcOu4MFtr zJB0`D{^XroBuGg2Akx<8jkjHrx+dVm$Gwt_uSpdg&S?z8-el?JTgr{h)tx1P4e-Ag zjcq#3vLBhhU$(TEG&8pBEZCQzk)Fuk`OqpBfwI4=X%V?i_P22oEo8HK-(H8G%jZE@5iMkeHcX)g2d#VR( zQ>2&zYldHel6QZO+uzlbaZWRh)h1!7uD48+p^<3h82f!u~^btyIr)>4JfhL zBe?(e5cc6O+o)Ra)Ea-E*O4D#s>fTJq%)eO8&lLgQ5JB)v&2!`Sj89rAl zQm+XbN}E%^8-L8wj+GR9ZEWAAdvnl{irdCQ!rx`bbVa1a`0-kmad5|L2oq6sU~kJA zr)#BwXx1KEfgoO2Qpwl>R7@v+HNKpF+QFkHZ7g{#^Uy>BjBZF-*T`Meb=54%3>xY6 z3|_`@SI}%6e>tucWTnz8xa+Zf+j52Ets7EH@9&r#*rtxH$< z7BVT{MLkrYJ0&S1DV^vEVWDed5PR%IcT~Z0REzWVtIYIRxjX_k0}Buy=X5 zyhoRdY$`Z{7I|*?^>u>t^i4E#eoxt;R+HUy9?KewZEm)vhGUFiD%d^;<60fRB;lx* zThPu0sJCSkAxT02lg-MyRw=}+I=$FB5e;Q!(f=6Z?WEM*dg=99#EW5RtI364?Tw&R zYppf}KJb(S7y2#U(DlTC9)x-+8f>I3navr$4%rD)Eq`)yC#8#d{EQ}7!b$jDTS)eO zMgrdnPSv~WbDlyHrmV#O9q%S6!_GcUyxik$l)r9_epiHDq=I~@-lu(PuXa<`q}$*> z1qq$@-zDor`blv8ru!zn4l|8$-$3_p&y4stA#X#BqDCaO{S+gfKn&rlmU4c5;VKnG zOo2-+^3y_7dowfCIJHX#l3&A~kYY#naH!bPGqGY_Vd)=yCfps`Z?j1;lUlTCmV&^W zg>AVVWOWMF9^lE6Gv0LGA|$!mEOSWFunr{;I_hyxZAT5x9e?AIvK)P|C|F^mA2F!3 zFcmqMEenMVeQo}fY72Q&BVoy-x>K%@7BO~!aG!jcV|icC!3m$EjZ&lg*Q{pdbK1$S z7ZnM30f6vk9}5{V&sOn0kGe?Jh_5FDqs$avR95m->8?~SWnZ5cy%BSeWWnRo!$l_d z?nZw?A+9?oNK<0j%|~n}L+S%h|LKZ*(9XF?w+SUGm|h>OSp;frpG$R9Fn*}#dZoT} zOpi<)hN}Oa6Ra~lRQ{rYnm{m+Z;7#WgEX`x$?K#-+p7e*8 zkdyvvTOERB}5N^aqdk$0W}1w&tIOGe14m@K6lj>GsvX5ZA)HV z`>s8*G*WZBO1bL7u*>kr*OLh>-tByEZlGDhi(m6f3I_Q7>h%hCYM`hYb@RBPZ!PqS z6BjS@iG0IWx&M4&SlH3!wCL@!ll{@t_NPctGPp z>82~0@LGMi!ofI>7!S=<2q!Alw5-+e$MG-qpClKtoOz8`!8?L!9|(wU#1ECvR&T?( z6IR3N(-klFBJSki6>D>>SzE1Iq}(5SgZL+~MtL5bDHQe05Xw^NHXBq!SXPhTt;C-6 zJaGFw@f$Jdy{}GLS4Cs0-)r9E_hf2(*~>3*r*C6;J`Xd}JwV2dJ}#N;a!B!I#3&E+ zY^OUl+KKRBJ70a|x>sM7;X5N=4G+x<(z6wdV>`|M5HV-rPPMPJ+g}+Re{bmFul_4R z*b`YW_tOd?z!{Ewr#4l^xs!c4pzO0MHO`$ETe{mCfJdw`n{=#&9X1!uVv_~x5p3g- zP-&L^QmG<;ov^r&KV+{FLYlQwznXi!(YO?P56UK!llwHeZK52LILGYg7Tr4&6LhM5 zTnq|&Bv3K^j}5FXb7@Xf+LHW_iA(AYr5>l9qxWQ}P_xyrbv1m#+iY|oR?7S?Q5Iy1 zQMstNbuBc{^A_5cqvjf`iHWzpe0y}mZ4|bripaQy_uQy$d1=vgy^AX6eA%^UG{Xh} z@z>l)Xsl6+zj;3H@EbcRexNO)|9#{fz~x}rx1hFn)^q$PjCn?^Q<(Qkff#_l{8m>; zuesS*X~|Whl&dlTPBb4&`Ia;^|8IdSSE>U2+3BfU-*!(RCl5(t!5w(J;Ol zPD0>%NJymCsV*L{@v2sq^M0a;bECeSI3b0fb~yX^R;W6<{3GSqGMB5eJohYGIIxp# zkZhkpy}nJOd}c@L%#L8w^)i1h&Xo6NeDihxS~Jh2(vJMOYQX81J!c(-Ul4!yZPGr$ z*8maJp~yQ$ICrAt5z{FxX_qI`II;dstp5U(;a_*1nHMyPv|Abc;h7wvvO_Y0<;5<% zYfLr+JmCQ5IxVIpd-_WR?nk$s+aj6Giper!2LnNO;8LBJ95T1uMi#1<4)BtDBZ3!rmlWh6BgVZY`b?i zH(K+^cf}nehh>U0HSU?isrAe;tm<95&*)>lAQnoW(T-V@iHD`*I>ic(ix564WoLHJteA^y0QO9@)TE#U6(#$uIM@@}v!Nt$@O&?lT z)lU*6-d@i35)V)?s|!+7?-MI=39}RXGD@+yXfTWoE(-WHn(;p=m`|;oGaM8ba&yqF zzBzMbzp{R_Bj^3A{lg;^b%h0J7!0xFxz%Oq4|;GU`cleZNs#U$6(wXek3H0)a~g2S zb3xU?#2b967Ax??0G~rxAX)3P^0gy=@C1|(!}_V`tJr09i0fQ{oF@XF1nc z);lVKxs;6EO30aU#gyG(OJQKE#Rhq<3_5n%XftMGh_~VFX6Vc~W>Qe}=8Ipg&-p=r zrcWvuoA^jRRQCG2g-*)BcBE`J_TTu8x+I$nq`JV`36ntH$TLSpI5r<%@v)rM%+*=t z%ZD}vKC^|X?we%dRT7yLYDHx8Px=nBmbMFZyH1rcAXqKR#M3X$45ySl-9-s^p8tOU zb@Xcf-?|9vH;jXIXscAbo~%6v7%5gUfdTy zI*jt$e#%Meuc2@WBSD`(A1EtC57_XMtBtFpeg+&2VZd{RQiM0X&T$9y>V1UP^c;*j z5z<(F9BmQtoD4@kadkD}R>BRlYQiQ#dQ|;^?-kLHm9Da@8+l6PwzjS<;a1u zzX_i-rI0qju1%~hR^pc}|ITO@yAHm$a;+bes0q2BWdJ5xdTNZC)lX!df9z%a9JzfZ z(%z7b#~{*|6)Et-kjN3;b$cZM*Na69&f3CjYa@gjaFVPHORgGS2VeBl~vLe@x=}&Z;pf5(}WCJ))m93+`zj2If3@Z}HEoTfOVS8)j zPZAEF6m9&Ogx={?t~v6nA;~3x%rq%;8!|>bYDaN@ItknbsvcR8x zxi}lK!tVwRu$jJ@jFXlh*TGPb7xuxTP*7kgCaVX#QoKrnksI%--4vwyV>PSKwXRkn1SL6`3y~KvSs;~^4(78QoHag7 zDDKryX9Pus){e6>n1m_kTGRotuOUE1_oNe2 z$u=&{8<|L{nDqQo+urdMW3LKD)9np7TH9!@p&)g`>?SG%2*nGfH#gn(!amo@zzWIc zLC>o8yN{J|0})OJ;m;m{(LjPGxztR--$5HJJ~A|OFI#XHt9 zmMG2*6*S%I_ft5`H43O(+;m12ca?>Zg>W#&LU!U55*{6{ozWGaweWzl@a1eSCnu5G z{4bOij3X9CPsBJBXuxI>BWaBLr(9yj=+1|{6WMBu9vv!BA%~Hw5tftPc5b(_FBgz3 zJb`H>cS;<(jvXuo>;>{PkoIik!6?bJn@<+f=Qu-Q!C9#|cubziNsK1V@-7BH8AV zVjy?zuwb<6YL`b!`~tz=@T3t!=JO?08!4wLrval#2vdi9lrFB2+Y^Jr$(PPX7Bn&= z%tleAYIpNmw3b*Wv98G&uA-sYdRxam01+yKU_&LL6z0a|TRploPrEWv<-b7eT?<-P zR4!VZH1;`WBmzHT)|K2nKt|AW@U3n3u&xgKt!!Lj2^<_$C!!a)WlU&UbH@g*Mkvn$ zq6)xNcgl^7qhW`waG7-|mqW)qzQN5>baj^8o4T7DY*zA&-~)&egF)-Ti0H)uG1!L) z#)HQaCCQMkci1Pm8M5Jnq3zSVLPzWU1X?2 z$rmMoZUO$6b3Qh@j?$^#=XSsb*;)j~D#m%_7`oKHUOx+A@|!ku4np(JmLiHbG1?Ux z?kUr@ICV2y1rw0_c&LVhIn0Hn<2eUe+QG1cGJ^vNSFQG+YF%5#F%P5$Md?e#dF_JSB0i!!=wV|H^Ap%AzstfR9)5Ry zMFqL41O|0?P-$Ide&s8s0x2?E8_E*%D1%Q{1(O@gpdZ;L@THM!F83)9g+XjC)DaY< zIv6k*PS2302CORrtVlkUaO^muVepET_nG>WEPdwGo`o@kq!0!ZTNGrbR&rbH7))T= zzPE$hM`jQ{?VKv-5yV;tyLGs38ghe zU=Bu14~8tD8_=ROfO%w7cB0@aPK32+s~#Q_MD@Ad+319+08^XWqeI~4bO60^hN7Uf z(LM0F*Hvi~$ze5Np_q+XfUprhOWh<)@vjSVx(j)@q1Z#62c4`-F0j0&wISRp3!oB` ztMG-Xta7R#dmr5oOlAi7bkojPEF_#vSJK`KS#x=cR{IOm?1aQ9AS1y1#4h5~+V{N_ z%EG0K1F4x^V;K^P|5$o&x1Bhl9_xVtN)A$2;)h`M6_dPv7ot2- zbIG(>tIN}Hb3uHJQxXU+?UeIFcPX`{l%33hgLoaK*8Wiyl6~a!a8!Dhx?pu^#hoH5 z)o-GZnKIgETihX67r2gS>3xDbNi%lHi7{S zV|RKvoO=_IHAHHPoV}&?(wuQIgw4!kIB{CAtuPpnSic>mBp5-H8Mdd3Q`Jk|lNZf$ zTd#Po*4X9PG-r$ldJ>8d4i+2|8l;!U<3N@pZ1p3VEI|q1C{!VmoMKauK+S^fBaUUM z*Dz6_L}^w5aoff}(p-rKXy|vd)fgh{0vQo*(!po*i6jgzBZ`I|1e51WVBC0(qoLm)z+NuU@z1e?C?0BDM ztIIOe>*NiULXI$}S*C+ZXPi0|yXSr;9X!~VA}ZbAY*ZYZiVT2)L$_lI)uOFvpi5=} zFm<`zXtVhw1@U+!u8S(nr;p}*Fsl=E0__!d5%M0daaimGj8xE}v>vHQ+2CYQpGqIB zDQy*z=F!t>4q1H|!+z^9MgWM*eRn=zRMX**Q5%##cBxKCd+Vdvb#oUZfjyVaCyike z7*bu|9$sk;E%=am8A%AtDihOc#j#Gf9->cZAir-qW&|U?U7$2%)bv@|>jNpdqelFZ z@esI6?6tzz9Wi(agb#w{fyyEqlp)IjvvR3vxQ(DkVtZJL^25R1L(@ zQg!4V=AqltQF>~=1XfYjZfeHaqoDh#scg*eo^EQ@`~I=+k)R=>!y*wYu3+#)h~E4C0|={|26g}913sl}Gd zrd(C*3kGQABC(OfPoN53`EH(B5dt68iX3if937^jSEr!HmIk65e&hfVg3uv?wue`w zqZ~Soum%rk~B%nhDh$S@QEjWB0uY5@fUoe6v6 z<-NOGz~(AELFeOd2bJ;2BpU6z<*EfiOTSFtU55SS2QIh;!oW(98`5$5>%J4yzy!d< zX;3NzFrOI!wB-hRg6b~?LX2;LGrc0l8(E-0i_UVP5)ed?%_3sjk( zI8;oGMNzg63ZbtW@t@Fvo}*B#eOdy-aCML^!bKso#zjVsI&g z%M20VKx8d0e1bWe87^R-Y?iyuZ8nYp2eVHy@QljK4PZr6qji7m6D|UBk9ore`Bo%rc zU|bXoGuY+`)9|eeWL}7*bkjB+7c`iutezMv39Kdg0cNwnMams#h84^Yw}=-41&vdM z3KdlAlG(*KbCf7J3_f8kMx#;$@wP3!fhKWg+lP$1;wuCB)S$^Igmh}W{GB$HosJ&Z zg|5EaDu7njcH$67LVgmVQY1pt{E=ItGn&1bRL&qnrY$0uH-@k2QU#VVGkScWx=~7U zkSvfva5B2AEKQSKF@a-H3$&{-V}x9yP?h2tQ>+253s6;eXJNE6bz^zDl`Y(( z1FwQkI~DL zR!sn!IDTbG7fcnGijv;_cYdDUdh+GZ*RLG-ZyoqJ?w=$VT$Be*06zQR`9wYd_y7a% zx}G~+_Mh?3hw-c^6;GNPzaAUvocsZjAcE78&q%4Dk;u0si>XHnvPEqxX=^P^J@1}h z0dZN#;C+F#LxILBgJ+e|gm*c-SO^UFtaiC&x>;;*A$~A4&F)0U=L!!SF}2^eq@<{h z`!F5AK}4Kez4muX?)+LUXp50*7D}v@I)<{cB;I9!U4)aNR=zXMCcbDRrFe} z%{#uT(Zd{&GmMoW0Qx)Gh`~(>(J#rMgS2FAFLBu;l6U3+4z&nzsw}ZTzp@FCT$UdyqlZtJL=c z%bv+)L@r;oFj^)nrzx4-H4)q_5x1589i%DTO)lNU!rLlp+!-zCP3lUYC^jBA;UEA!l2XVmB!GwJc*~y4hh;Y ziK{KfPqi3>sJzoOQzYYJAhbrrQg%$&iY9n$?@vWf05c`4T>|5q@e(rCYMD#=3VZ?@z_92VY{mu2bB~&gg4N0rCmKN9d`R44bz3E zGFb3r3owSLPZ6^P2PvxIX*A~Ty4Nkc>4z9%SXj3CLVZxi%MCjncJ(;iu72$@Vn`dJ zR!?bLFJpB>pnHn#WLnm&76VZoO^nBwvKH0A#Fv-a)B^;}8PCLP;n9}dr7v*CA!x9hV~L&FHl^d73b>vo#~a8$ zL1G#R4&*a8pfz&)uy}^)GlSH1)OU0fLyArXC{?wn@CQ2NzPW<9Bs-Ka0}bJNQ0O6h zV%@(xClkTm%OFp z?Cs8Ij#@TeBW>VPm64L`omdD>N3q3jKAlS=jYoaDPP%L(K7vcDG|(y1kwIpP`=ELA zxD>oG+*tsI7)ip&MVLZ?qh4r#3c96hA7cvC_z`jX+TM(&Z-w2GnUAn@K&gknwB8qgU=e=dVNXnz3`#$yDKzRiGIGa+n>$4?bHqL9%yKodz*k z0CxBhR?wzfEOsR2?nyaJ&%#@V=Xp=++wDE7ccP~FEQx9w znJWC^LPW-M)K)&=h_7rU;udn>lpF|)iIVMAv<JtGX zYM!TVsX(Eu1DJTPLMT8$HWe!0 zV*{ZB>rZQ00#hmj4wDA1rZPuf1C?sq1N_PD%TW-llR>j7>`WK3n=fz|2?+C&i+M0Q z53U7z$EVHq_FB5PVplJO6ud=8mcc^EOVe)K$O^94+%=V)m**p{e*z{3#w@9Zd%d2D z*4+VG=yz>kcW_Oz5Ym?t>o6Ryv)uTsN+?t3z2wI}b79e_G1)$Nd?xVxkx}K*2A^A~ zAH;cPg+|lTOa7jmGWOXZjqdJ z@vIJ90H{#CWU1oR^zSdTS7)%n9L{cl?Ow!1i#}xn!=aIhE@ku_ z$xSouvK9;n5#I-b+B9go0Erin#xy!shM-R3T#|>;Zt^~zE@stN5EssD z;HBX+RKtp?C&?nyt1yBjn!DomFDiv5A0nkuHpyo!?HezpltQkjcYMs;jS_{K&<>$* zi!m$iSnX4_$Pr#j>KVI#z1}QZ%OD>lh5(zul2hP1Q#CglbLo2gOERi_gub#aRzEOb zhq<$cSbZ?^tj-8& zS=ljo!W^`P9;;;OQ};EB1?&4Ss-^_xgvd>{tzv{)#tLgupfCi5!i?<6R2Y5QZMkTW zBts)`JJ(RLt_0j*%Py^SH_sKI>hjATpT{_7Ebq=K=-cM8;t#s_J~f>K15) zBS6R-OBf3rRq@#JbetxLB`QUp4hszWtuzwrrX6~diP>$FS?6D+ zE<Zl^SPb^{4-x$#|e$^?_5_WizN zo`fzEORye73MX|P1SDgrOreJoR)NYWYnp~gTEvxb_XZT>Ln2S-mx!~SWKDj-k>s{r zA1G6*TEI~Yv9Q&~`N2G%kn5s5J3LTeD|h&bGPonjVlUSe9yiiOo7)`DGLkWHr}rj6 z9w!jHw~?KA)|Nhd!0$GDxj=0m&iomxeV}9p9wHHu`=yQD*Hw;=h9aCHopB#2PT!NK zF||d5xkFGP4+WkXirMR-_7AF_cEV@C)|A3$A`N##=>cL37FJ{qLvg7F2o%1PFivF} zTb{b>snTI^H9qn$hdRv2$q~c5JxipTnBP;tn+Qpo;~&l+6J+bRzaDDv)Drq*doDSd#&ruX(KqkA2X` z1T%P64=T6@w&09$Kq6vL9LFjNnEHamJ$Vgd z!BQ10PupOU1U|-gi8u*}6Pvj5{PgWni90;`C*Z*8mhK)tQhPlRN7dD{KUx}!)Ho#2 zvOwsO62L%%rN(V^^t3vK)#oT(9r9gVpP$;c64q()HD3-J^Q#SJS76)YG? z!HxYGz}jIr6%LncRA4-l>D^dM4Fc!7yT&W6wTayz$g@x}M##8xHzEl<-H4=r6V-{1 zMd!!@1qE6Xmy8?KFcN|&DZ^^&A=%|qaI+{Q3>x9@n;%N~TRPU;ip3Qy1ym}=i!_IC zl}H~dDq@A~6o`_r5f}jdKT5i>lNp>FeTEH8rjw->r0QvDRSOX-WO^>*np7SP0sgaK zno6^}v@C*?YSK|=*KYiM@CV-mmMW{~C@wiF1LMDrop%?{J>WgclTIUuZ(T5@o9U{r zej22_emxxDq#ZMmZI0vI6g-x@z@@ zBU#**IT4Ep4uZ%|RmMNI$BGJb0}UHYTj8H>EeyA{k_M9X?9HzI&zM6kBi8vS1RUot;`Zt5}B=gBU~eu zN#O$0!-O&AuXkPt8bqvnj(Qd}vKPvUFR;>tjKr z9+2fj@mrk`^17B_4q27IOWHNOcfmY^6i-%FMjE4NABC4YiR+jT5Q3}}{Rg|K28BcC zakF1rF}S|k=o*xqPUFOhYUi+mDvH8QqhZ)>L?U!eqv zkDR6;_-~l}rPvJx)_uc-{<9~T<*Ry}mVH&XIA%M^ZXJIQD|6y|k848xbDUnqCf=BU z?75kqu5fy`16Zlq9b{WYDh!0StxBlgOusl);R;11&4uy- z1WyHm+XyJ?S1lX*T{2t^4U^C#=8GwEE9;E4*aCnMVYjFEY4z#4#`9u2uLN@EhZy5c zFLzKA*CyZ|+S7%|4)GMGQRz|*Qi-^OI+tGBS;jwY@=f0ot%nC+UWxeyy8QoVeRArY zBEzVz0HQ+|$2i2K`Di&I1iF7K$AyJNxhB)o22wpD)T67iZbu1J=gn417VTy`f@Fp77GX*TcC8&tOjfX<#t+xc-j$Q95^{BO% zb@OL6>r;8zF1Qy2@ZE4l$v}HT*{|I^=`7%E^IirHDFjSXA$Js-_XdqxieUovn|aj( zZo#N+(sCpYDZESWn+CEt3FKpY4Ct~W$ZS&*yl0kk=Jf-RC}Q%4TpClRnCD)=@kVnd zk5jpk!1w1%f3}vbY_8R5e9Gca%apT@y^RlZqd94G!Aiw6fo;({?wp4E+EySYW*%8w z^}Qt`TH2;NGGrJJVo}f&#%1710yYG)5{9I}98%!|4{o|hg5p7=#h)$Tq#wDEDvV7nXHgE35M!Ey6#9@%P&8g zRF3>l1UGCQ?$`nZFttk1Hp00^3$?l(64+bEYEjxMG&avAj+mQjTk!Le2yyKed0eU@q~+Z@1QD_Jc2f{SsxaU$Ap(l=V4pr zkU?4Wr!PvW@L`Sw8zJ7_Ybz6womVt?)N7#PT{qiIxl5Oka(66TP-O6u{ck%UPbv23 zObFy}Htsif%$$#BEurf|LgT+183akmSqoK2b zo$KX$xsv`cXmN*sWqoRTgJ|gp?7GAv?rW45!RzCW)lo?+!wZC9o>bXbUr(%DWuc)9 zRBY55Nsx-WBKYyPwgO;#tmc%{*ShHlV8stJa@A@?VF6yEbw(dS7#pJq8!vV@!$SO) z3G5RZfSUGTsC#?unB&ShP%z|U)UB_8Xg?Q%WG38GS1?S@q}5L2NlyBsMM&su5z%je zvX)uy2_dvP;LgZIJ(JgYstpUO+Z{G_Pd+%@G(aMDh}pPMF`g{)d%Rfh6bRoh1rRIA z=`U6gasqk8NucGR+)2rXc)HLbD&qt&*LR^dFOb`Y#k|61nB7t4AgjPs_(4I-ECja3 z+gA@b6nrm*1bI^q60il`Fda`fKJZ5)YWhBq#5%GTKoca~r2+xNpdf5c5Yl48HgS>i z#!OXM!PWD%eO{IEv4Fb*_vS4CIYa&OZ8y~N!&095ztV5$WVoIrTKm^pz`a&Z9{PB8dY z$pNROs3aRG{HxuBp>)I@-Mrbw+i1^(av)0OZFgI8Uc-nF7G@3%j!eyZkRG)>XNQY! z;S>?@-5e0MRV#+3AkL;qUXHrWPERl1zk1y<1WuiU(DmlvOMy2_O=TlCp1wi`0+Qmr z?>>0fK+{$PgHqu(GGG%ygRj@58^qyuanhZhe|sNnS*wyq(wDqm!S<$FKo)Fw4qIHE zIQNQWV`m1A)}$gfjfq3H*al+`{`CHA=%U7+=APfpxqmGjiKYDo?d!5=Y=q~ne8b*DX9DSMX#05D_9`Q(!j5QV2@c-{b9>^CqucK*!*!zlDGgI0 z1y<7(?yNGZbEpNboOmr9j`v&>wAAg9rAC;QL_UI5XT;8SU66-KX=|j)!PY(|J+_5Z z6j5RE6_;VZw3tvi=OWxP+b(^NM{#88Yyy+k0vH1~tfnj@H;zW)-Zw0-BRk()<#I(m zo*1|UK1J+X*sQmDwois$RwN+KMSQw8DJ(vl2ghP~9AX0Ly#Tr>*@RKGAk%~Tz(Mj6 zVKpd#qJ2H4{Wm^NR$V;P-dTYt%6==QlfrpOxZK_a&5t3E9=E7$Xsq>#s#i-o8&^^q z27NU}JKk~)Zm7oVHHeo7u!EstWOvmXIPoyq6X+FSi;bXk@#e`8QxMariu5Z&>?{^j z0T&boWwTpew;1vJ<(?#8zO+Ifq0@1$rZVUS98iSFK$+uq4#Uu(tX4&(UL;Zw3V9Wy zJh`AKKDxag>U+h<_q!o-$A{N@q7Brp#>3>`&<3cFungTOsSnEPLZV+(UykuU+X4sR zSDge)_tZ1xj)u1(kq7BfLyC14I$Kq7d@NapCOQGQV0JPhf$MuR<&cutw9hL@IQZ8B zpK>;@s5<5p(%gKj4}B)H0=)xna^b7JK5UlazUJOWxUN{bkmXb9YxT?5aTTPRUs*!4 zg+WC0SYf@Z0k8>cB6?WT(GG|2iq-sG$iY%t9t+!Kl8@{qA9(j|X!vAOl!h^ad3VoB z5>+WsdkRXR)A`?XqB-eBI4z@XLYoeAF$62_O?M#5=W4b4XyY6P|~LTzj{Z>69nKFZ0l zVDfPqkfziDoNJ{PB~@yPoYpcc*o246u%VUc?ojDgUJ^r0ryz2lAmqXBrh|2D%;wX< zvW5#*?Q3e9U!a75K3G|{E2Whxwm$jrZS#!-`r3L=QyO z3>`V>@Fy*fNfp{RPCPvJ@7t$GE}VV$$cc5*CC4Bp)hTiK;Zi{F-S6|$*G?ZE9ep`C zcH!HPZ*Lyl{lmlfACYriHtm*)MK>EKT(C0RVJ z&RwXRLBtqf*QVr_meJ?L0XEuxj!x=FS8mROP=O?cDw-T=fIy(q^#ZNfK1fLn_X+ir zB|Jko-#4i(7zKhfol9hZ*Wub6t`emv&2}XiH=-Qpn3A?sKPqB`?lcQ5qj-9+Ok!RV zk(Ou3-f)D#);Wn(gCg(83MeJTsAZ9DIQc1 zd+;nlE0J(YJbzAF^^Kq*T#iGiA2uZbxRAD}CRVLULcKOy2m|4O&A251RuFe+Wxv-1 zsuhgF)YYgNcdS8%Q#3GIsbYu-6o9~_BB)8d>;O0Fey9v`u(UuyhjxK&D%#5dZ#LQl z#9|>&x-dbbiMX`sP4aDSOBlvFolwhb#Ce^P4Avy!Z+!dXkiaFx2tcPae_1k@J;#Ud zxjcxpH1P{BjTl(?a{-Ng;vzHp^N)aZeh!C_SBFJ-Fj)X|9E1o{T$qAK1+5GsyPFPn zV9;=7b!ebI0cEZ?n@Cx`gtsOV^PIEXMDq;1>%eGkQ^F&vp__b)MLf34igocr1}~-= zvG(soFWS9RQ>`~#CIE#QP>%t&%5_u=dZL2fmAmZ@GnkN_f3$o_-#1g|#zG(zU7ps< z@IV3+7KpG10u+%`?Ib69xs=S~5LPcv%_;V?zF~7^jMKC=`Oxt6RrGE}AplSwBko4c zX~yjkbor^qRzQNOT>4}Lo!g4&0j)p@07A$-Ai@HCs0CGi0@gvSlEfbEVqbW3h!oT@ zM1bD6el(LjPJv5L2N~q%1$~_0I+{yb+pS%&5N+_dPC1P3)Vqc=tr4nA$UeYZP7O-J zeqtSlI5L7#%jk91Kv5wWBi;4SRGEeLPa;yc9)%AGFB?5D6Os!Fc#dhUQz+wO0?Vh{ z$ZdT6sY$Tqj9Fi9$FUjgUR2~#n-66lQbH3@D-bPY^Dn@20TRmMvmVw`=J(Z=+YRDQ zL)|r)Tq=<0PiSrv_hH;kMxQrIJqE+Ri`ba3_QctTv_lS_>Osi^g}@|Mv87ZrSnblB zlte)XxH^&$-f!at-%C=E!AFAO z0Kk785#HO!W}8PzX}ltelxkwOy`a0Ox9rWS^vAkNNFl}Xk(xoea6DKiE)pqB|(J< z2tjV)8biMN17np@94}C|_*i4@owZl`UvMRxukBc_YbkNST|^22A)$eB;)k`t#~>(ETQ6E!;uV>k zvEV#YNxC?DDE@a0R%CtTTjqUxsM-OoL8}X~)kKduusF77o^MQ+*hO76BZqLkCQkXB z!9wi9qtIUWLRA7UCEwL86I(@69Pq=iQ^rhqP%hK8<8{eaXsa0zFA4#R zMN6oRUrE7y!$J}(CG;4U6=}4=RcV=#j{rYjdh&Sk`r&~Mz=2o{;R|rJ+)hCtX3`MX zFz~?E{d$#ZtZg!=_@*AAE<--1_rf76BV~~uw_Tbl-b8s2iK8eFMXxEd363!Xd141I zBrS?-f*`G1MkK1D>ao|OK_cHL)V!2MRmFwJspiN7TvQYh1=DcSK~Hi>i_zVw+`0~w zj|fi>M4@Hq=rXwDxV$s;TN(%g;5_B(`XH8h?Y>Se3_CuWh2bj7O8BOgZ`@rD)I4MySm&1cyA3RhI=hvJbu6GYG$qS?{*EY=D|d1l3aJmW=oYM%iP30YSACn7 zz>_J`uT3F;QO_Dl=Ie~YA!*pZy2!xE&q265xd6Dd$p%}8Labwjv1mQ9kb=Yk!Npn@%HpK)hu@8?F_Kh={W?ylxkS55G zs3s-Om=Org4aCHCas`2*H+EPEWg@h4qXT zdgCUXIF1(0@~EmshukjvF$8PG=D=CXOAmY4@>(rbN%%*H)@4Re2+WZ?=nSaG5T;H8 zls=--#Vnx1DPjc*L&&PWHp^VI?ebbP-DcE`sQWwd$U)npxi>$jlLiQ_(t?Dc#2I7k zBAmp~o*?-)$h~U`Xkqx8TF)w-Az_X%mNl1x6C?$zOgqlV(HtJkMz^@IqQebA_XaO9 zD#AxKo_U0U^Gt0$xN0K3XElK6KE;`)X(%A$gHakRQsg*DC48|4e8lhwyctRLXf9qz zYW-#l9>5-3c~lU-dR7rZ zxW`N`T=Ha5*DPY1;>s$wo-U5K)W;Ju1H4n~4TbEkOStL6JzmRvSTn=Lai2nynB=yrkyKvP8i+ql&vb^0L9a^OEMILX7G=GX6K+x! zgjcCZ)2Pb;X7LFF5C~|L*7L_CGh4BF0E}=4afVT!GbljouNoC0DfIY2XGUQZI@PJ` zu0Y|~hIYT-Iwj^PRYDAS6AjrZYNawfbwZ<|e(1a5K5!s&-7*v>W)dS|_cK$O9Ass+ zJ@j}CRVcZpWmbqbY(97kW?&fBF`*CUBO~WLQzMm@>+88jkX0QNnELhvdafz=z4bE` z*wD`1@@$V+2-z#>GQqmIWIaA{y9>)8VV>9W;UV|6XatYN+txy^0s<8~-3>%j>+)>6 zPf8rgjNH*eD>zQ_8?vmFA$7iGS1vX(&`6uuL|EQFM!AvgKQ)*%aYB%IK^kl@p|azu zARtqVn5p6h)BsIQW7%pjK*Ql_qZJVW&QldM;+%go;DT~Ttnju(ODUli?QsA|T<^+& ztXL|EuObu^)R+Z)lm-HVjK+ zLv+NboYG87+UX+INgF6U{Sc)%gkT1^ag;V9hCb5aVF-CXABRBRgdC9=j!bB48kxqlD0_InJ{BqKbPB z8iWaFvCyx&imU0PX{HKM4}wo>#*!W~W*>P;Lekt4J5t9BP}XBm`GFLLuzj7?(c(Q1 zP_*3jm@p9h_%tgf^0vXpIzmwFcGE?i)!x>B8toO(fFTW68MN-ubfTrTxh(SK)VGmY z%S^q3TpuuT=I^~^SCC+};t?OHhM&>T$~T6KKQI~snV+`KIGV9(k!VartwW{+U)CgV zdBniCtx<7~h2lzKY(YzhKx>mO8V83r4Q8|4Lcu}{3((##M;X%!=>fx^+fJt0UiSmh z2v(z{)1|mK1IV;ozl{~g9yskskL>DnZedNxa**n!4!Nhs91{%q5hGNmGA2X2YhqUo zY^_L6Bzy_9MlRG$W7gg#2iR<30&a$(L>3=>J0;E)%W=(j7fx|QcWPiI6FZv-$H<=C zL(?(($g!Cy3yXr+)C<-xKr9|fp0X^~NRgX2b@woKEpInU)U7nWjcKohNOjh4wVoPK zEqs)zLt-AG4N)L6vrUDked?231S==t>x^(m9LYAk-t9&K2`+u`q*I;@PBbrgT1Y8w z*8B(y1p`+s4loE!;>FZQ_=>uku$7tTV^cy^YR#l^-e9ErYjXpfsFw>jS3&yR`y79C&fwd z2A-c~l)2)USp}q8yTZSb$SXx1x?rCc^)`?A(Y3bHpo=(i4jdZMO|J z6N}|^UE3rpB(8vUHnhPNM{QQ&Eckvr`kzD%T*V8G4)AAv4n;Jtg?iw^a#RT}#VN1= zXna3TX6U$NwRw6{ucYRz!BG1SW8SkMiqh5M?S=;eNicQI7+Ct_Iig1yFVRdypOG;|p~k#Fve1k>~Adfy%< zZdnMfs3MS|>^~Ko-Z!)Ef*+%vkh(eE7$~UC9kp#KUTsOSMQoBurj~a7QiY(;6>KU^ zfMBNcc@gJUl|Nz%T-!71fuxh9TfWgn_78)@G$k>4ij4W}NhbulI#9L_Gzw~k!xHt( zJYPuM$}^K%e6*4#%bGFupM~V;CVVF8t$M%-MOEb(O~ci|#|7G7zS$x>mwTu+Ffmh+#KkVdckFL5EW3re|i}=R|*Bk`&t6 zeb?z48Yd22TRE#CQ@%q&v<5a6-v<~!Wa&?q!_(BW0=Wbej+&)qR+=Rpn@StTwTNq5 zVGHfYwhwOihlV|53N6G16WS0=c#Nw3lp$8Yi>)n-)?8QP=3+w27)}>PbWt}V>mvv$ z7HC#ibZ6v3dY00mYCcy6K!rky$G%K1FipmnjaXReqfG;mTYbdL3jCTV9Ke2#<@W-1 z&&HxXhkn{-q8kaq7Av3Rix|A&MBbljkQfDwh~|!y9psM=;$0DbzaOItO!v^!fzR)OQmIU_)=dI2ZU`X z;!N0}^CE_xpNqNPVhXzYh)FOlbXRUSdp1^RZfIo$5P@SbB3-~nWq58*c92@$po*Ob zTLzz0X3u%TfnW%Lo^f|5eqcUQ!!C+8CB$av7zs*Ed^QuKt-P_L(kAEIowU2)i0M~qMl#y|V;dm1buv)n#eW!SwJMwagrUMV) zOKDqMRJU~K59EMH=UyDmmy&JmtR3vBUXgcMe*uPwg6bt*m?>94O8Z>GI_NnUr0ITc zXwOqW3wz4Jg*7i-XDh;Z4yOwtj+e`g9h@;VmK9Ql{al#?E`$0_re}hKpsQDguNOfj z5$m-banB@-ze|;JctG(HAf`4yz4q7On^bke*G}s6J&+3wQW`z$j&+|>C$~I5YX`O$ z6}~X~080n~dJKFOTm*$y#9C|0%*}{?aJG|AH2bls#@UeSQ>>Xnr456-k|8Icw+d8C zqJZ_Xo)b~R&JA1X_KSGl43U-jz)+ZW*`_u@dm1Szp7=W`h^JCz)wQ9ZSa7sxVE( zd?iEkc#P!z0fKe}E9hUmNFFok`#GjmTJ7tC_pX(tk-8f4WdODD=_B}=C{^^&xvBlc zPzi!3CzF*@ksi-X=A>lk3rb>mP$GF%ezj#Pl}4o3p2nS1;~>*x;4o@F0ep+M1_J@Y zjTwEFu+ps-AE}HfX01;G864lVqtLdGQrC*W;2{$Vcp(9F)i|D@8}Pt3BVIXy&U6}! zQf-f9JeWGAuxnpgrjM>fpz+Y(FJzE@YA3;5C~z$+c=1sXCy`s4?m@CR!O(lWh)zZW zy9D;m-Poz@+Ce_31b1WT%Ot(i0fZyCAcZj7E;i)#AWONWO+22+Za0{4O9|RbrP2P; zhc$(dln!R161V3lSScI%JU8h&5WR*M?~-^BEI>dF6~$os3A$tbshv!YH*hAmNQaL1H|2uO(0IT(2Y*1d|%3J`sCbfP*eqv*M!KSgp? ztU;oCu%!yZgzO6259P>vMH>$*{aNlWsQLaDwA+xFJLOC9q!fYJ8%Hmw6}4+A<>|7o z$#dcdx}qx?q8PBKHD@AtghpDNnQ^hPVDU4R1#^bhZCl$-3!}5y13EJ=xGBp8N7j|6 z>OrUXs)@*5;YvwG&r)RDKqm{yO99rwqGA!pK2bem942E?MPH)R%w%!s>mq!>?#(7{U@EViCMqaLRY?#wQDP(T zTzvmcNIlg`&wA`5*c`L~K|sF0kw&KKS$ygQ@$V{>#bJHhG)$$2ikR@l*&>x#JGtP5 z3kyOeE&=YuiTSU1jqC*nx~HE7yHeBL3%+r5kvCx9Tj^Z&qbil}EeXTsx^&NKAn@?`e|`2VAh)z3aqD^M+diq^n8dWG@FIDkbZ!_3MMiK)}5`ub&rf-}Y zR?6L2L6@GwYCA9`Qrno&_cs_zN^a_~4xDyY%Nx%8#_QE}7Ky$Ft7_>#9(P-udn=t) z3^aZPakIfJ$G(Sw4ZSuV7yF(v#F#WMeYqf@nb!v+SQYnLo8%CF z&}MZz(USA}Mykfirs{-pFcNSgExGV{i-wm3=7W*Kg&DhJ%A_SwsRDPKI)_(5N#x%; zS5ABsx>zJYO`g4XIp^x|g&vSquSobp0BOKdp{*Efk#NdmU*$P=We^;w#^~JBhUw7| zi}Iyb-mL=)V)Q=dHZVhgo>g~pLJN3IYu?XkG&pq>i3x}aEfJ)7RDct33d5rOA6?BV zhr9GYD&d`$S=&6*5Ts8V@FMO24a7=n3d2I8G|3>n*4BfFj6j5cVVRPdIJ-pI;1*euBgb4t z32JFo5CZoR>-GliEGTcMgH$_sUXY520BM_EJZEBLmLC4p_>}brM?&>Sbv2Ix218xr zL%J$hWF76(n#BkkKojx#xhqDEk>tRY6a^q13pT(pVIZl%*klS8B+n@cRl4)fk0ppr zQtBDjEQAFSD^d3))mr3o1jR%!$ON`yk<6_j1@86XKET1FWV;&tA%{yKJPH@6k` z2ToDkt)Y?HkSY+cr2x$N0EaWgsJ>nwLnTE5C5Y4(bsw{RCbB~`F>}GyR)V#mOLjE{ zF4b;G*ReO~McU|vI!&PKpe@%Y=6FOt5C-`mqrq!3tV9<>MC*7&7IRAg*5L36ZWPxV z-N3lJ${P)!A`S)=smX?!()N5fe6N+eG2vZB8ODBXhBMDOp> zsZ2V3WG5pcF#$w5oGf9%;ui=Q(wMxY=rBKoO$z)v33_I?`~#ej_GWwqi+`!ndb% zMcQsDi7|j+Hu*_GaG9_VLyi<5LCLyd$R(7!7TwpT64Ys9e-id-NT5cJ3?f&J6NwS(d4rK&bW^(mN>jR3JV zk{54^C38w~v|GelV!p+c(cf#g^0NCBM@)Pa0|p`>n9)Q6?_?c&*4F#wKcHB|%oMWWs#{&l9)JxMJPW-ees4a;n&?>}q_O5<%4|hy zWMD85+Z*(vG|I=Q_)i7{F>aDZYS^CdQ^u>6PpqImc3L}Pq_PZLR4kKo1RbIRG)Jle1M5rL~n{r#RLTh z>Jl95-k?x-_^Q&Dl@Ho*nlvgRyC@*O=_V6d%Y9cFTIX9jf!ot+m+|2gDMbtGQ`k`0 z3HCaR9jvtMg!q%7{xB9F3;iB_ZV^qcI1`t^BJ^Pn50gH#K`aPiyA7c$3@t@JAaaj4 zlHkn_!7RyrxLJWlFjagW_aOQ7A3M7YSKtd$N(a8&9 z2$HAe_^>b$$Y4vN7u-akkZ90smR;KhJ#@6cM2zx?HITr#+jQ1UwJeqr`iLTTf`K_a z6g}k`Frl*SIjMW(A@<6!kUoAQ{1MSv6+Tu%LE}ohBC5=!GEtVCX}k}dA*w(COMnR$ z)XmtQ1!~ziJs9BcdW(;-p?m~5`Pc3cngxR|+wCK$<^eIb0x=EH0BkbnIGXY6q|X;s z@?xSTs(j(EjdIQ&{4Sb3bKrX1x5oJ4=a0-GOSgV<`bDRi3WdS=`Oi3x>FL!$P?9+> zM1FT)4z=|Rl1-IW#QNbGZA5^F8LOPuVLCC8-5gWJBMh7++}JA)A=Q=yHyu-vDd}15 z@VPWww_u~`rp5aQ;Tu;xAF_a?Lf1Jxo{s90%K2rP5(f!o_wJfl$Q~^&k|T)1WwCkU zfrBP3A$*7!ggDc`4OGOA`^~AuDqFaMz*D%z(~lB>SaHwLYYtWN)ZbzSx3QJTDv6PS+Y(b)BAY{izSwc8X zWbV|pv5AU0l&WpLL^CFrHv!{l6EJ+z+*Ym8NArVHOM1L%W-1!tcQK_o8G$$K>kv_j zAs1n}61*nw063s$V-r`L`QLZbm*XyuiV+wtG6yqT^>8zBhn7vR)<0Tp>A{aF+wd<^ zVl6>-8gkA*m(d+QFj+t`9H!O!0xcQ=t3@(dGS|ZV@#_1cIqcA~`qH7wy#*2!8yhB% z1L8ZLv|Q*t)r*VcbVYqhl=?;|yL8dqUp$jE?Ed4bZB&}DSGh?Znz9Bv5YhRW29P(M zE@ZtmrA0b?J9Cyiz9C!dN4lXu%E?++iFq$aU8&%rXOwR^Zn( zIc1{5A_$q5ti{*khR#lY5(Pt?x)64`tKSz~R4@DSNq5VHi=ER;;EY0ejwsRRBtIWo%@_|WA9Wjw z!&$-j>)jT;fh`lhyowwOSAIC@S}S2LB<52PP!S68VDmoKNq{7(WtGoK863U33W2|Q zHP+^H5ZEl@)%J0&sV2M&YlE;VtXG`PqD8p!tQ%TYa##XW&QQdXL@1X7)=bXAC~+|D z_QsrvFon5|3h@2mcAyjFN9G`55KyH;+SMCf)>eIJL5Xh8+a3ltjsO7sxnT+?XqXvHzuHm#;^snlCBkN9R%#oNsDa@?m`xz2=V-n zBrmoJmSZMn=e>*W9I81#A3I4$9#;*D{76|VV9_f1i$L@1@-fdGX2>zfo56m3-3qF~q6Xq17_RQ;poh_`4{&%a|Zz47&54O)<4_frmY&(`E;hI9A$NKQV3++b^)l z&sv(|MBI5XyDPo0SX@gl6plv5eT~3~Q-@0ckOs7UM3x~36+X7Nb!cJOgy_*rD#VQ^ zI5cv#=yMZbGCca}1PR77+^jnyfqL0hmYOpcpC`4cRIQ{sXB^SJajd4TO1~_69UJZzNVhK1)^aN0~Uxx z^PuSr5vN(s*;>6w`>v90oQz0>KqU=v(87V4{8=8#s2Z}LrpvC6O9pB^(zCIyCD&2LkuMYj28Vs z$s=7@fkk}9$=Rp!(Yb;u7>1aQi>RK6LPi*c*MRy=rta?6IF85@R$yxAIl<#_U{NINvB?mNO`%jq-Ly z5#yEMagJ5%veI$s!<2}=3ktFpZ$d$pAsmWGd!s?(=*sr$l}PYOwMBrL0{K{Kx6L$R zqMOorBVCG-piy`NsZEN?Z$Okr4UO(}zGwFEMX#C)^)&3#AT6zg73fF0b9%C%RY%ZA z@i2n|f*WkY)C3Bo_Ms@g_o5Pvvqjn{1!PorZ2un4nk4d+SiMXpHhBmmLAApLbH2r$ zD5C77X}NByvGZWiZXRT!o`we{(xNlH=Y>MmqZyw?BDF|<$UJ#0l5t=`b&8`VAP^#< zV37J*JOkHfTd`Fj3U2dN9i* zfPk>!lcz!`Ayy1YZM%GgG@UNpAbB|Ox-NYkFkyEbXC}jpBkFC4XXcz?DP|s7JOp`# zS|gHBl8CZnW?E%N;Cm_P^&Htmkn@At$ci{t>zXXM;+#!n;GPZ)e+)j-}`gj5FS z)-~PM=bUUw0>F%dheHR(R^^JIfia$T>_LZB%=*RrL^nw2s}$nDQ>8-W& zt5VFA@amAhWadS)9U}$w7P19m zpka3015`LL#gR9wJE3BmY#pzG+QsS;!W)obW#0~n?|I%5094XI8>^|XvhtMq9{A?> z$kQT3PPKagT@m0?)vFFHPNlaaik!%j4{VZ^^Lf5Y`WO8G>%+`1(e+gg~J-p+m zI2}!2ybYuT+rWh^a)%Ec-MA#|*$Lo749<>Jy*?uOb;0{qsnw?$Va0J&I ztpkdbmPT~grgLbPq04KuG`m-+`PCuiYEcL<^+IZ9)nx!8D}-8s@RHV#uH%=Za3W;n2!TJC!?TPsnySqGwn|x8+{C zdKFR+Pdcm~9A|2^afyY4`S!Ymdx{jFJP%mLO;Q4pJg`BOl&{S8?BtJ(I+=`lPuqsJ zsOqnCx80>QmopvSpH(i0hFgJ@N0mBJu^uw#YWm04;p72)D64B9kfrU@5%(||@Y)iX z#Lx|+Ij)eiP#`Bfq~`uLtZq#d{R*zS80NiRw;(&`MgRl0#!X? zbY|fM_=j?Sy`XExOZFndQ8m6mdBKE}E&++t6{PBFK|R%PL;(SNDiI;24MvL&)M!a6 zftj31-MU;uanbBLn9~xH-shw4;I*7;DT_a6|-%wqzJjeeyvn`@b}z zlB>gqiR7T@HCb*FQ?~K`aeb|$s*BRqq^fK6l&CkTo#A;<~+AR#k>)RRhOGQ9wdONHc=tRPbKgXK!vri5D&1=CfSDS zoFF2ok8wtD#T}1_BRYDAp}p2ZH0FFMkZr{1K~S>Qw@|5NBHsa}PQLJO0r52MaXnn& zv4wYMw57@oCzN9aMMZxLPA?;9uG}}=@Q5OksBGH}!(fLAGCb@ZNRVu*u4JrF5hReiK!QT* zVDlxg{%)o**CBF!soM9s`G`KJf1}uaspzj@!P#d=q zf%wM2l{ST!DQM^uk=Fr_;u9Uo;buadK94mTB;}*Cat)s{#p6kFYN`)oltBD{E*p1~}#g;PPZ z8F5YMHa|7=*DA!(2Emc)TCZAdNik}6=COEdjw+Zo*+KJKB;a>VUX1rpWLPVQfwRX6 z5tm9*ZW`09%nyM^3D)%xxmr=~r&2H@_kiSeQDUaOt?d4qJIaWw$zDK@oR*vKOhXx)@DK>_e5UVBu zrE)Zp>yac>=%VR_ItL>8%d$4Q;Z;78h zdhfRrtz>6!#Mgz^C&m}HR#D=ShHAV?l~(U$LzDxBq$0Ed!7{ZPS)n|A9SOsh(CfZ; zJF0vxxj~e$GfLR@bcHJJiknqzQm|gvaK@^`rxm@GZOR9G9S*&CE`sgY3XwjKT^i}q z0Jw}oieYAORx2}(>aTpwL&$zO#vyoskvM?R@hytY>Ai|=$3lv9dVhjM9b(uKHy+QF_0`v5?3oS?fQ zj@RR{Nx^wYEL$)oJef_1kP$w)wHk>_7|mBz&}*)rDq{$RO&Z?M8bon48jS z8)6S2YCt)1-OjlwkZ(Hp>)W3xo?v%3B^uyeBc#BvGlCG7GAsP``#c$FOTh3Sd;9fQyDEE1dFt3dy5W;Z6^p=@C+}Y zL#(n^0W45*I(4XEn3cRfQ@n3mw9Bcsf>J*EQhnLn_6yH_9VkU)Z-)rYBRqVh`v+tb zOiPHUQ3SrdSk}Px=9bJ?V-@Yj(9&xGbEbsWq)`D|J7-&C%E+szMs|{ko!0zZ7)-3> z)43s8vexL8Og_Z5VId7$6ycJX&|Jj@aG>ODMU!zVO9Z|*hNcEz*UBStLK)*DkrWO* z7aw%n+o#_;W(<^*=v6o@7drU?`om%XZ zrBgZJN5Eo)_QmtKyu%(LfGtv$HHGSQyMKWD4H0$6%|Sr6egqU0mse>?jrc$-t2>?N z(5QB{^+t3Nnzskr;~ObGc`AjJPrP<80(BTSY^ng8A}i&1Ek=%p0Gq2Z!BBQ9Cg;k> zJ7K{XOYcqECC4iz)|E?Ih@YYK)WDO$%=&L8F*zHW4nqfZ<@K>w1YA>lCR|zqr_>2m zU@EdK{E>GZD6N6%zS}BGxIJ`S0j9L!c?SX{j9M{fRw1Fgb`@S>*1Yp<$c~+o%@7Wo z*#aAu%$x@=ZTWPg!UavYHIBW{?FMx^)yMLHE;FJ4p{2JDu&h;hoS8w~cw;A-Q1@CP z%w+S5JGLGLa~TXr8xta+EsdN#S!~m(!mMz%Ve9LODu$wuY=FciT{zNpj>cb#kvhQ4 zC4^`&aay1VAVE%WVADlwF>LN`uIj)DDzPCrS%=hXMtE~6)SQlx2D#{?GYdH5+|~~V z2O)NNIe5HR)YpO46~Y$AomY12W!#v}c>}?DN2#<+b~E0Yy@sWV2!aVBjZhpB0X;pX zq|z{=)nIy|NFd)oUTY_4Uo=2m5n4t10oltRDMwy-Jk8fb=*vUhQ0Bl;Uu_+kFWdt$ zgx^DOVh1nU{=6n0dX$qmxHi1ij-PO|7h{)bEXd#vp=a3X3k@-KZw=xk5Fz<_P~1Y3 zx{PQgt5XN#-MM(FGxWMAgoF%;9Apm!DGj0DVBMUy*{S>xB2*EGW@lwUWmjRt=B&F5 z5EuF8@*9tcP7KU%spAFQd{i)VdqoY#(Gfw@@&!~IP!zpdDNz*@>yzx?qPlmGbVn-}f@JV<~Tlt~3w~Uggy!XzgV~P^K()~b%p{c8?33opo7we>qI?i7*{NqCyMV*VK55oXWNjIjw17n zU`8C)bVKvML~blLqUlSqjOPue(ZooT>M5{F`y?H%%7GN12k&7tm(#A0psgKNHIHT5 z-%FujZm>FxM1599CLBS*WDT*`ch4CBtL~NxfiM**pJBA)-sIhqVexRYxu~aG30M)T z$ScTPUV7>}t;UB>yb5CNL1%Nwyl0{mqk#2+FQgEH5%;nY0IL16vWbk&f@@;*<1Yc2 zaE-EgBin3|cW~jlQWv&e24=Td3(e9-WgHz1mmsj9x=U_e&lL?Yn1 z?_EWGaD6g@@P(xb!0D_JTe_epz*sEZw9&Fnnc@dPvocACpkh7Ym zt(t`lA<)GofQVw1F4dS)TY-U zXt5@06ai|>kw)2dqaJK^9t<@^xh-QfOd!z^G6ey0&UKMu+L=-taB&A>?D-gE$R9S9 zEWIt0hRjt)O48yxWB}Mu&IALiR5L_e7Bgst9)|Ec%@FG99hUe8-WLT=4mE*cDCAUz zvyi7%2cr+sD~8cb1v5iK+X|lYknuwFP%0sj_@IEbK;E>ITqL>`bJ!s`QH|WMEK>@B z=36;qSd`u#)NINaz>C3#IFah4s*S^5>$MnVFyAT_VJkeU*aEmm=yipbq}`leI}kdxjkD-n}~-hV)HRf z3-Cx(=Ruz(l^P8gI4K^6t?#KQQ(Zfx9*y_7EGT}ph=&H*qwr7?v4=q@!!fdjVo?1H zhY3(cqgK>$TEugFdYf8yZ4o;!#Qw#KY8qXKFTjWv9uaiOGu!j1f|_U2nC5THafi-v z$dcfl61jk-Ec9y(lo0MpUCM;jtw@_|@zb}L(v^ULbu`?&Ixx&Bi-f8X@imeS_H$Ix0YE@_5bB#k&K?kF=~g3S_!JnU z4mhl!<;iCuT_BDIomJ1ptH*Wult~~+UPXA|_{H8D78$P^oRy3!$Rur4T!usie5MuV zv3e`myCCRJg#>G$rHcY}L((XdQ*AcBx;ajasq?P@L@2oJ&{~xe z8cUNn$+<#GqVRlnvt_G}h^0ud zR==Joe4!~^XGpw<2emB4EOMl(T1{Lqj-D9x;HqGDnT;=)8k>_cc{Z{i9O)$an2C5P zg`3Vc?{nbrgNe@IBp1hF&JzyF2~7@5-lBpnyP8uko#UjIRiUmRF>gOYi6PFgAUsXT zo~p5>@QFH(G!tXQ^75uvGBX4j7=#YmHa7Y`k%?2jj%F=h^7yHGNIaD`4|&sd#fISd zD$InzvXWSQ;N}{hkTxLZ`26mF1fo+YSQgE@T`?E{rN?eA`13=S>Ah7sj4@&sohDf{ zs!a5vGrEchdMJd>@7oM>OI~84$WN;S5>3TsPDl`u9>p*bZM{#sv~R{4z$~yy=x?F_%llv3r{@nV1 zS*{rv=@3-iB;qIO7KUNqdu~hfi%wuiM!Zc`>sXjP>O5KyEOm!^giI*fi6NDsj1q^n zt%E(UeF1c10mHXmM-Xveh=N~&lEgl75G{NG9ZGxd zTmxXd$$b|GRWX4>tTrWeieq@XG4mM~7rIN=q}1vI^-i4K%#32Mp**LKa=B?sG$-q; z6X;qhb=p$~CI~FBK&E@O_av4c?G4E8^YZV6ThTIWD4BW|iS)$SW> zqX_4!R67oq%PF8FP(J2!p~AZ80d4A3z4p-Q66Wc<#iRwZ3~b+2%lRZ>c^Bw9cmvVybhclmrA`p zco3|IO2H8om8wyQlC;aLR)=LtxFcYF+{!38b7M=)T3Kh8n`T5tLj~oWI`Bj$ z7$}sGb`^uK6?C1F$P!8@l(4KvfydRfP9P#RwyO5NIDzrw$i&8mVgenYG;El|fy_BM z%gU6g2$i3kW>x0C=Mj6z?_6t>@_m$b3zaF0H+$B`!#CAJ_~z5q8697H)(5`}4ayeA z^!z6^f#Knnu_f_)MFLq5f<}sDcp>jd9m{a}5^It%UAQXh0c(80I@_vOAg4yc6yjIU zV#I~OP{1Vyp|~;6+xjNOEGq5R0<|doJEwXS@Xa7FaO_ z*Rt@=u|n}0jGYY45w6RHo{(|t+Scn(YSgN+M!vdcD1go&Uf_cta6ne`j^jh2#KI;B zff{n1UIjT&fe2S-F)gMRAV7sE^&$yHnGeyuq6h{80S#WU;68t+YkJ(m;L`k0S(`MH zXPIi%pawI1j^zkFgu8el}M_*{&OG~&QRm=G_v zw{S@n60p2#59P?0OTRiT3D+cH==U;3GQoaAJ@qLre1j$G`68pnGHEr}kj^D2N0DY# z&3tG-4l9L00U;ez42Q1|YJpxd{!G!LDjx_rX@&IB4aZ`TNT&10y6-@4BXQj{<92V; z=ZjYsEC#$ooCXL&2FI~p@$`7eNK=i{JKeLwz6OCMW8#Vr2&35%(-u1lQcO$i$@%?i!g`Rj36Z?43}6H>Kw?Lh-`CYYwmYEvPQPgcXe>%>$O?oh zmPwdm4o-hgF-|KSh%pBOd&Ktux1ioz4c8bQpzXe*q_iZl`Tj5>8xsKJ?wg=}8usB8 z@xp@FPHAtz<1 z=W{~O%iC;$su15KwH{3ivywzbrp#Mk%Ea7NN@hOvyhEI$Lr+w0`PXQJjfpEB#%px% z2yBiNHuhTS{kMhmw<@xOaATD7H>{NWkgCSPqOMn$dp&@hr)=e1FcYFet1K4T^0K(} zpD!c=;rlc)>lmeELhSf!HT%a?b1@We6WC9&8WXy6X11$ae> zd8yJNrv~K+Ld!?MAT4IWPztN61;kMugcRa=I>A^@hkc_f3l0Kg5yG>>FG?8LEd$CH zcP@|WLd?S?aAg-UV_^4#yP8q=O+uB;g7AdKR*|x-dUbqrAqPkdwtUQqokiK_9f5b6 zqBY;@>>!rAvH~1p z_laQ)H;f&xdKxn|=o%CbMw+Q;dQu=AV$dMtpwOpl2-2~Z$Yh*ILadh8&H$EY2Q|T7 zA4Fm1WKZZc~9MdoO!7%@<1cCUE0tSw7lV&9L>ePT5jh z^jBosKSJED!sjN9gn`Ru!yegaNfHHpR!Hq=%10$_1#}jVAqtZfkK#SG`#ow4wx2LH z1ZYPFyz}cwas!HJrF_3TZUo*Wn_!a+)!8N%o(@@kEv?3)z7Xlr1Hai z6qqxVT^EE@z(^HS;zXm{qpAl2)*wP#r<4)_PTi^*ppZl=R5Jpjs6~BuxtKuU(2MYi zD7ZSRVOPf@`o98#&%slSGi9B4mh7IMlwYTC&|vA}o8OvFvxY$Ogf)`Qtr^~6wyV&@ z1;g+(TbsI?N2dh9$8!&OBtN~;QL%pxjDIOT6TKbU1L}tQQy&R2A1a4-S(Ml~ zJrL9nA&R46W)Z}URpipjh?qhKg_dv|RF;JZD+M$agk$)!twiyqZ3AhnA8@3U;2B&^ zSl8SP@>(U&s8t?ou8oy06JnPTM7%nf65;oXhr>#m=d6RiZW*iPOAG1F=3)nhNNTKt ziR>Zl6GWlkYuac|j#StSdSv*n(7r)=B4BnxP@`oCA{s%85m=Z$UbF*I^a5vFb4li3 zlP3kuy!!4tbAD9OHq_h?e=Rht3OXX@czL<6+qwhyw&Pzsj%?6^Y2QeQJ15{it%bI8F=Rtj^41hMVvBcvo`moPYY z0M0fzUK0z%W|D1*x>HQ$hLfF10SkmF&qA*{v?NV=FF8izL^?q!;ahwLHN)diH1#^) zliu{(tR+aK9Np8DU4~_}4Jom!YwVY%{cZ@pKNCw(L=tQ%rj4^^ClnvG8;JX1i-9P3 z%~A{?7hzZ@E}DdyGZjy_ScOLL>{ZYa!L7i5bCCU(N#IPG@?Gg? zDauYgaGrQ>6<3O(j5sEhL?x09vCE+Kh17Y8y27W)3w-x)q7KSufyP`OBKYmZN18ra zKJ|j){DGwO*NQ^7OESj>o+bsmKwF&mxk4*LZ%cTm^_os5q(B5*9P%61E}eUs?g2PId0l5gR25GU4Np zve4kS4n>qufC006M@n83op$D-tEr>0yNjrAojL)zjS!&AKq@WpP4r2|csqzbP&|9~H+CuD$}+de++N<1PqM4dN>yG=NBpNW@^k zEf&h3+LP7E^~QU#xMjlUXi!*~atQ5xvil2?<2a;*+Vd5PMjB+rt7Gv~u%Y_0I-{z? z>D9|<{LykuhdcovbncYI$7qa7IRj42{iagp%jNen$)dxSSR3quy0Z*q8yopk2}_K? zWRNF)4Q3MhO-TUn<>ckJv%#XmQm+E3%9)EBw%UNqNYi!ZN%wmyx=t4${pc>p zXVxXt1(i{`f=PU2kPlfFU$zp=; zvYt?2STMFVwqaWm1NdvseC&KlrFk!BNNgOeW+!N| z~5YDj!KsRTq(1fw!UP~EC&dzy7SzZL#u)3c-=fX4-RQs z-4F-`hS=4q(l9WH;wm62_>WMD<_0dj zx{GZvrz7LqEWd152otWp-!O+Mlm=27Vp(~e)15K)lf>BhIRO$wEy-r)MRsvfmD4j^ z4*RWB7rgNUOKD;*3Ijz30b5kOETAI91jU4x%QQmxagGE7TO-_<*|GuYQ%QKI=whl) zU$B||Y-T3Opz$D_rFQ|&xr8neq3@5IfQwwj)~2rwO8(GMdC5}AQIp15@n!E+1yV?x)Fmphdy#^>PBcz_64w3er zLQ1tcXheBlB6}(9-pJ5~l?Vo4>PjrAUO9s^s~^{j5o!mS@wLtRj>&;9T}KvP(ET=hQC_~? zW3>)uQSOxq*MXY#j$t>8Q4z0TB5ua2enpUJ>tYlNRuiOx!AE3KJT)|G7r0kp(w@&SwJ zMK(HvDPfm-tGkU$4Q?{4#yFjvD3Qbf5H~-h{=@s{%r^~9@qj((mw9svlD{(bQ9!#W z+F@{XPGHo_VLJ~1tWN+vb+;u^-n1xpaY92)h#S&DRRc(U_f|m z&lQINk@ltSgrh@vP^c%djf`t#XA~hWixd*(ypKA&$5U~KT8c@mQ|FPn=2(-NFxm}f zoxHIz*_Blz9`oDr@kLmlVyt(HHXf`n!bibe(eEI8opr3az)kUuBN9UtON~t0te-+N z$@tgf*?4Ml2{q-9jqR~lsm7}i0yvr`h_-LA>6C`O^-n9X6wsMiyq!RC2w8o2yghe# zc!Q4J6q!L=Qt~TrLU-+zTJZDBiYsb&tA#^e#1jPeIbpT_ES6^X2D}Nj7hu*lfL4O4 zoqRG3mydzW1yUvRh(1|^kJ2fj2(Tg3o7B8WKm!mwMQ*|`HjXs<>E3w7sa4sAV^27W z4s|EaC$NN+A~?uX+i=1oz3k)=Dlxl=Rmsqxi%}MLfaG2jub`G=G|OnnonMO*e4*?n zH$G$uXXM`k=^Tj~^4a^oS$mkGx9iP`iG5nqnv$a1z4=8g5iN=>y1Z3&XtA> zgkaWlQ+U?#Dz2m$g<=8el|r&~0{ex;{f$VzUhA~pcW5&aqui8xdU=4ST-NHZ(D>}z zme|FtV3<&u0sTp|xGCj@MZp}@LOKN5rbG(%DX>vzV&E$m0YId+vjFNiQ zpi6`wV5(xrQM9n%6_Y`Qa51H)dStQ%0t`W2vq@&pVGQUinO)l;;?-q@Z`x9Yn%*KB zCFo@rQ!D_ub*EiZCJ~biV9rrNReU7AFRap|+aqkRiv~@tos8(sp_NzErq~ZW3S5QZ zhatI}sG7B$e22hrCeTcryatk^E6f@Pg7j&m6EB3RoDRe9IONEMbyQs-!O_J>bRK}o zz!g!@4iL(ZR;E@R6p&^a*#dkfn8ikMCLUTO)kbvJB|Bv{uAEype3qVF8dBcl%&`aa z9n_F2y_q;N2FmTi>{Y=m2X-+-=V>w?L#lzX1s)G!nCRI30u?6c0We4$KmsmsZU=4b zm?jd}Iks@6vrKq!b;oWl);g3x63CJGtGk$Wig(Jr@Ed@Jkk%6T4q|i5Eq$meyZX$ueyUmleZ4-35*zspJR}`v9lT@sT?h?Sp+pi zNmuN}RLYtH0&KnwfP$gIfX0TKuAfr?d|6JTM^&=RyGs zUH3)cAn^ra)Q*#U9i7SDG{R445;TnMae)IUyqmbiv$s>rhFm%{LjhIKaRJhnA89bb z%Nklsj}hNJzQJjuzRU8wj*yclzfGt9Jf@ z%(P#<(XL_?iUUf5P~Mq(+#k-%dU#ya$jOc7YnZriO9>StP{=M4nWkp&>9U zB`nm+6Uo;-LSM1eS@g0kCGLW@UV0)agPn8ztleQLL6pzRRGYYmsznCkNpAQ?X`T6Yi92T)z7xT+yl!@F*w zJhRaH6jolQ5qB97qjyHGCwb?7bnGA?6A;Ywk#1Xs33a3=zRKuI<}wT_HE!5&JXNm+ zmLy_YUs{hZ+6xUQXqXEt&q&&*^@ncUdAmwxJkZfZtAb@I3wrYBH!+^1TRZNlIs(bm zeYW71-t?r4vqQ|KUIspf@w#y3m)HWvw<~!%Q%||ME<(o99LQ@}E5cTiccMsiOk`82 z*X_LFQTdQwN)2ZV+gd`xG>cYIEY|Vlh>`NB6UTIAdUZT}^yCue(XpQ+(AGgJ-9m?S zDMId{veSTM27h(Xh`qGYyMtMVo2B!Nmqv{AS z17YxhTU4M)+Qt_I8eFMITrj71u$LDUv~Le z&&v8@F_K&uhxHTiq8WlUAKmWdWAkbwVJj5~(VJdc1Ir!^4;`PwafN~bWw~^WHr`OR zn~303`{W{^BAspE^!W$D>r`8Yx5wc=6_4vjr8g@M&rv*eP5KwBDwR(NAo z#$QFeoQNLhvtpr_sm)*Q!exjy`%s%aayM zH7Fn&v-pG*LBZ$^(DupGaAIYfnQ(;^Dx4W~iqaVJ3H(|X&_M1Th%Oc&vk?rz)ZV<0 zEXYdr9bbi-i_^Z2qFKS5f!%RLA=FY$%X=unv6tG{;es`J%8Wi46Gn3d)Zm}&9SE9= zMh6%PP(2x;=T!zDjHn8-?ModbQk1z(#IC|MJA&6r2<3k9@vNF4QoHu0;Y_xf8v5Bo zhGsW`VEGH)5La(RohNq<>%KOz>3?*h*lGraVDost1B{tv*=oMK^Lv1=8Vm$nY$rx> zQTW%6lUYhsg3l#ExTXVlUer7uH4->ez`wXhPU)c~Ch4`0+LHfiQayq6%!bhEV@7$P z2kjUFyIMJXd`%V5zlt$0TN)DpR>0 z8yggGDV{~_TomlBt@8NvENMD|;hh=EF;YUhr&u?T8d&4Ec)p{hSEQF}mX~2su>*8N zRU#|lHVemQ1uR$Ikk1pp#HNbu&cVPlTrr}}ECV6=&v(Jgb&G)w z$r}O&H=JqGbVG_<@Fx^xq1I!=yz`GKux={toM3m0m4R%TCx$JsI{~4fkqG*F^=V!a z$aw0*O}!{8aBwN?HrkD*Ldslwb`x~3HkBJL<&RHs+gCXy)eyODj!+6iy2d|>8!$>? z&=r@ELGb9`Etk&^WMLp4E=0t!MqU~c(gqzqNd~2mx)AJ@-OErYPX~f})T!QhS8|?A z9Knf8g%1see5$n3y9Cu6m5gV)2AuGBoVFFoq+l$wEYRgnz(K3K7@SSeWC?p-OBV?% zRuLSV^D$bzSe{}Y%HY9#yb7>?lZ)&lDN%Xy+xOezZ;{#>1kiRysB{%L^!!0`!7hRE zQJFSBm1HS%q$gBto1#bRu08{armJsIfIybI>zp zkYsMx&dX#3WlwwTbJqNtzE`q5;dp6dWvXmQK+jyEi$%xe z-E?<}9|~UZ?`glCuFN_MKduszY0FIUtMB+fyUd7Elug4oCGF#clTF?|AKbsE(`_ z?eXgn>VQ*Gca~k0%g9cfBNBmw zJc=>X;KNCahbdUs6#DC~8s`q? zduYiW`GYPHQty|;ue9XzTgrKBAs$}l9e9|(6nno5I#Y|nwbgD;=_86dq9TtGZQk1;|g+_0M{W-s~`nZu7Tbj2qV5M z88DIE#3?L#sW4pe@+#jT;*S9OF;Z;7PNFRAo@hPsNtq~}3*40Kc^o2`>jub#W`&H^OcXE-~)SD`yv zQlctUo7d@jH{vx=utw7Z;&dRfR3P>rWZ1X3)AhW>TdrNz**(+KL6JI;7G46@FBd5L zhBzYpIBrwjSxZeaxa&m!!PJ~$zqWy?M+`q}b zRnM0pTBZP1PcHWzMU4c6!b_jLh<_ac_YwkK9hK0$sQ9KwwYAb7fyBP{cTI4M?cS#_0 z6#2#Xh+nMwn#5n;J#FWKN174D96gQK)E=FtO8l4U8Xi?Kw~W$OFh;y31(90v-HH}` z<{G=t(GDr?Q?1sV+in}KD5ithz{-a-lz3ew_@l$zHH|xF3%IG+gH$}beZz#B!vucA z)GKD&NYp?Upml|}-<9r?*d*YaY9EpJ%Rs8lItAc5ChyIz6qQaFCpT?Vcad2A1T-HF2b&{F6o_g%^CCBDwJAxcNs+jh z3ru>4;ZoT4<|2gGSobk9@hTwS8IGqWB@h>-DrY52rM}hi zC#)XG2rLTxd{VLPq%2zwBz3J}l5igLq(0v(MFv^!w%?g?=>9;{#i!e%mEx7_DD_13 zIc&(+L_s-_h}|F6N)gH0&VV67rB&m{G9#2HKxL$iC8j!PD3!D*6a@_2%Wo{DlV)_Y z!eEQ>?+zsdv2xK~E={E20eHS{n|=oH%MeRiz*O?Mruk7;+MvorGp1F%s~i=e5~dY? zwh1W=VMF+1O6%IUP@`DBPtKr_t42fq7H~_onx3`bI!;4Q{v^2FL_47@qVL-K1 zQ)!VVHGuiA5+*q}EY*qO%sV74nvh`JfXFt|#L*Ht8Kr@w4Hw5R^xgwx|Z5>vCCb%vjAk*32u9?i3=DvGl^Y_X+++cHJ z8DBc3uZHwA&2kkY)nQ5_u#+mJWM@ahDJ~IftnhZoX(Ry2#@bm|Qo*+45|)Y=!-#1a zieccU3=>Yw*@s}-136}?yv!(7(zVUjZqAyP2|sVl`phRTdo+=|etfqTKMmUSiVfqi zR*_GU0cTlx4Y9oS25|M!hcz!K%Et`Od33BeUr>fQYM}B6VSMmFCkB6A`kH{AL&s=D zL1Dq%*an=1i)I=lxMRB>L%unwC?y_j$B73-{#CiUsbGe$P)xOM3%6Dhs5epOrfs(N zHsVggiCQVSzco{;EeStLsPb6QG1p!@YO;iFRsNYo(wvoRde={YTo>GFj7tgbz7fts z8Fj7_>Xbylt0$MIW!-0Zn}n|(%bY7*z%U<5w!vRN1EkrK~cD@0X&*e@K$XnNAG zZR@j*(G}?xR3)&Y7v(OZI;|0$Z>LZoNb%{jf=MNxIVy|ZE5(#u@=z&cl11&`G@<5{ z$SVO9u%W$80m14nM2kdm^tTJr6;Bh*aI?o8#NC)WfK@uN+1t8o8~UN4NKZgWNJC5$ zKMVs`5b)`{R|stT8F<$07l#Z$Tjw}VV7f&*roASiAZrJ~!SlJ9;HVWThKx1JDKZTz zw(T581SBo$BH*u+rU3Jm$f$etuM;W*m9#9^}9f-Dmzn% z$J=t9v0Omn7r#`KTUHqZ?If)vSZaDf?keik<^bi1?9)5xuh1DG3PPtV0U$i|Po!PV z2AhSK6oA6Sh6EnkFHh=~2)Dr{sYiynnB3MXV=ITW%WqLKjoy=R{0=xEsIm^IvXw@r z5W98Syo;18S>-GkRkT>zenR^Z^3+qO%g?!f>aQf?Unpf9l;P^zEYXvU0U6nmjb9N3 z)QBgu5FJK{R6r`_hC(*d73%~rM9hw~=#bI)i#emry2&Ano>p61thEClzS$1cDlIIc z_^D4gaL`(y`SGAFA6{yI2NM)e7NeCKkTi{T4Ph9p#!DT7Ndd{*6;Gy#)?HSDY`{LF zxN9(!Q}UmwObBlXF8orKQQ^@%hZDmhPYQaR(z6$cGrj`n3I z9J&v@J)MR1aGCF)TuF9N7SYU9HzMED!Q!W+cKB_8_mg0O0EdAs)uoo1r+Y#7#9*TU zvC0OnzCAj@4#D+B7+Li&rLV4ceA8OcN%cg&4ym#;L*D5ymm5pw)g>UEu-Vj-TX zE(OMjO1ZnAgOAsH2JgOE_fDhioFDn49KBK7=Qc1>a z0w(iVu-Ik@nBZNy8KO8!M=(MQq09Liq$_EgR%oye-VcSQ;wTz1LGn+Y7At_*cAhz2 zbHcAgCbo>HvQ({;L)>G2H4rzG#4U_rdnp@ZoEoCwYG24!|vA@3m|rb07D4vwfdsLfI)yc{e(XXsnvI4T^q zCh}KMfR;f~^iPb`AY}#s?qO2HY~enL7iP#qyqSeMDlj`jZ-jeUW!KI!GDE}Lw?P7O6tQ@9gkd8~g?s9G`PD8&+p$u7ayx=&i&08; z?a2Y1uHhN47+`5ViH4{(KMQ-wiUi^*YHm2J6tp#5D{}^CDg!zuC=Ll=F?R>y<+D21 zN@0OCJ!T-7PUgPSM}b97sv@Z1=K+zZS}&3rtWv|*O0%G7J+1g3eeiq%_rL(hH6d}X z`$5fS0z6}ODpknuM^REfPL$8YI7q5bdk1A|?OvH~0SsaM>yMzOfxZ?yq!jg230nma z86dAM8Um*ys0Te;WiJiW+s|h#&k~DG%)#I{nZ4wFMaf6$lO_?`kYg&-3+q|SfB|!} zYywn!Qm&slK^7#Fx~xb!TIBZGGppZsudt2b*>I)M@#Om9 ziRNE{&pF`JG7(4MY8tI2L);VFRUQ^qF=84U>G^xqmaZnKHx_DlQPvYKlH#YemSdcf zL5RkbI>3hx0svz|^(hGJsRB6;G{UHgJt`8yfMHGpMqKq)>5!#nKyoyfSL#%`0mZpn z%jEIXq4pIyLa-#Y=}s5bB!ZCoz%R+1@(x+XY1#y*c{xL>739#>vu!sfh=|+bI+Ue_ zOIEKj33`mh%miFphExlkp9`vu(C)c(e1!K1#eL~kuoy@4_|@Dr1vz-~45SQEY#}lZ zFgLUYuF2K-bdIY)^MFBa=ga4qp)4q+O`9LJJiQz%Jk;#kDqOayy|N?JH?vwqiuMnTdt>JU~swVWAmK z%%Bl_f$0@LF-$D^i0mmNU@l%bhsY`{c!AEKMFmC~Tg;X`d(jx_p2SnbEhunJ8KRc7 z8)RYOEecW~s5B@UFj*PcnvD#jVt1$S5ex)bTcE<5XVu$Q*!J{oWSz&sdlMMx1*^Gm z?IIkq@u2vO7O>+I=#r153B?H#(9=bc%0=opZtDM@qA}L%8W3SfhLF3yJjKF?e>t_1zC$EP4|o8MvZ|X7Nv! zUnvk&XSUf;d@Y49h$ncshG$9gP$fOa|gZ#0kem`+KPkGS_Fz#^B&#FPaLTRJ)kzGdYVOA+iT|7 zeh#ji#kb;zH^vgE><|M1c<##Ns{9os8Ikj+gRGBJGz!~86n7u+ft}uK zYW{aY6>=reeMnU&r<22BAT?$o0>~A;d}bPsv-G@O+QMHwo2?RwjA`(WGT6y6`obtAJ0@Ri4O+8g_azSf zQ)o~-1sKcdi?n@%WXtMjHM1ENt{sOKOENkrPB*8km6&xY@zOm)stxT#^;T4F(`lDk zbdX}Uf<;O7D6t!5a;8IGvcpO19g4Mc3_f@?dM%R^s!6L1`qRF&l$@@^{Nop+o70FC zD(%DHv~AQc9zwzA?vm{?tAAXgxeAvw{9UNGbyBGwf%*+Clylj%9Fl}7)cOf~p7|G< zE%UD!ws_w(-*JTwP7R@pklz;FE7Q(P?28*A7b`p>u=Zm+5h47Uw1c;v3Q#M(2KdQW z)Tk&1w9uXutRr)!sFKwv0}!KwmW6ZMAd7lH7FMFL@xE|YQ9OjQJ!Q8nIcdH!4eiM? z1jf%ghhIK{1q7+`=_DD*0jYI3Ayg~_4hHxXaLA}G!i8mML5~qDr`L~YZ>{k7HH4D}ba;k@wq=DW zx>N{6Ro+aZqRyGS*&CXYehp5Aqm3*Xv_QLnJ|l;C>0)fZlFMln?b51vv*0;HeI;Si zfjL)@*rcc)TIpgok4lr`=?yzs@iDAfeUw}7k+u{GINZ-eZBravV^rL%A;J#4$lLoT zxkISFdRgwhF;|#W=AdXGVKN*jKLsVGXpSl=}hD~=YX)XLSXN?g0 zI*KtN=?5HVwW3Nk7{&D|gHch>LN*xHWF)_M2~1`b@g9;!z&vp0L-UrWUkcvXoCI0A zipPrfunts5CphwL)=tP6guC5#kcW{;9!$#dT**lmYsat)ox`h1md3$6e$bL3!n`}j z9hTYwIMmnyUJpY71#tek^g(fN2pYdclLoUTBx%DA7@RLCkuO>%p~29*>XcOWH8iYy zg%L_{wm$Y@i>lOw(RBAA^_^~?rm5r`eC9&|haD)I&W;A!8>gJHb3!ZwRusa7WNKC* zq6wBjq);HdQY%0*0tIS~I3AtG(ZUE+>dPpl?!6+m%91+ZNSOsoE*N&$R&DeaI3nE8 z`V__7N`yw!)=+8+|1JF>ZA)BB=6G0NAUKQnZxWjlgz?f@`a^vctEM@9VUp>Q*raMm z)IF>Kse4$ybOa%FdeecqB6=!GZhqecoQFn3O-c}J)3PcgAdkJQoz-cNnV6(QG8@fgHmf_U@b!SwYIU+M<`-E5|ga%q2if?uJaRpf$nCLQi{n%RRvZGu;i((z-KYrN6&4lwY@24bqt=Xc$BX)VP@q%q}3{$krj>wsiS+E*|3^l3T@<51fxUz?Y4ex4t6_x)b}5*bfiIrv=9n?a zS}7cr+<|q0!U<;!J+gxJfw*^^w*|TD!aAxGnV+9F26@ow>7Q2c|lq^>n zdaohQ3UYJgYEzUmqtd&A4` zx#^;1dm&idOCSjsvQ14QyVWKl)-qv(c9Y_E`|nmQ!k#PH>e!N8&|+9XfV2S^ZZb~^ z1_;F$fYcKBTfq2Mi?7~Xu?ob=p_lp4@77gER{^4V8KWWIEeV?>^7lF;*HuYq*jvqZ;La@19VL0m!DR0u~x4KRl8lWNR4Fr}B#fzMvY9#np- z+gpEEq4ZE~Umpb;<#MOxXzNfO;->XuTshuF88{!dJudod&-Y#Gz$ZwX6 zXo+gC>DpH+Ih6+wg@T&X>Z!_V0SOU+G)FU6xU_LzIJ*s1wWrk}hK?vGe_M5SxLOq@ zRJmtW6Dt6>N~-u+{^)Gy<>4F)Vsa|EGkzdMe+Hd{y#V$Jd0cXo5zJKHc4Gzn1!8xM zqKb}IIykuP44zbJ0WpF)Hmt8%MI=UWJ_Kx9kf(X5+7J+udPWjogc;Dri3PwyR0#7u zlX!C3T+^)T17bC<&LZgvQ+|Eds5#=WviSx^M4#+)AYM=?eqw%Eu5KL8K|n$&y|Z0l z5rP>i5b%LVZ?j#p_EZBk_SE7|GXjy~Nb`wj4zi>N3P_P7k|Sok|2I|E0L+INSDM@F zV_`660ILYQ3CU}3eO=@sx1cotQhGXBa7w)kOAIO~5btZ?+Ca-h@o|7N|N@gisjaN$4(N*#5HRM1xVoA3T>vrQUj{6|4h{Tr1h>R zS;te{Y?ORqXj3fLKs|+0ZQzfw2xRKE18`U_kFfaoaHtOqGbAM#*1-NT(RDLASv}=t zmUULGh)SAz=;hO^rtz5i+R%tEs}a!U=-x3#2jY*XESfLAF>^bye>o@u48)&PmL`Za zwxu*QXmWMauEuEEwN zGC^VLJ+|i{EW8Lf%G&wqiHpd-cPzvS-KjKl!=0+`>6T!ci0bXYR1ix}^g+=v1$dL;%87*( zvx%LneO>Bc7Y04ry4&lfQ!pz-`3pel5G=8phe4EV(-f9oQR1uWqO7y^=s?= z4d#!N!UhRLJjSdbCuY$5fDXN7_~dfE+ILY-HqEcQ#It19T6aFsyeVb!(`UY^=#G9c z*B>dD2}dc)GRc8US!vb^t3N58G9KA5$qJC^wSKGP$;VD$r-k)4)gwaVu2~8wh#f}x zA|OBwt%g_T<1<#BM4!3E)YlBkoR)-{~J42%q>aB7u3VI;VikOA3h8OEt8(~^H5V$?HE|PJv z`5&r>m{|=6Bt_8!NG!v$z^pr?E_#Yz-j7yebF{0q@`r&hw6h9?G&2kk6kZcM5<`UR zxq`a3@Xk!c8>G$+%J3z@ue_`17|L?c!+5zHUID{`@pMilLW=lfYG}ZTelbop;$y!- zh*CwMDrJ}g`6AlqFOnjDu@es~wXZ{qyPgo6fSz7i+}4v_Y=v(+3qhXV;suV*8usER za~aI?pfI-$>b7GF3@i=EDu@;#-VUPP>yq+Jt-H@7$sntIB_ROtx%pK1+FdN@nZspQ zvs{!M39iMB)u9+Ip~q2Awoj;g1sq16@{9H;C5)LM`fGdvOG=0$J(dhLs@L#>`wLr} zv}d77GAkUVHq=BW7%tjK+aOFkP?lbUWq@bHTVnpdTMC3&VC?8Fv4b^}?7Df1trO2P zH7liD!?CP!eBg(*nHiGO;{6Wo>*B`50r9}AdiDYrV=Y2Og%3CoHwTrBGBz(DD7ft# zi*b=yiMkz6)Z1R^CHYGk3*7fCRYmWGX&8#s#`U!+z^;%U*4Gc*iP}@L`1Bj>Qctc<~4Y_dh>yIv7Jn{4AkAIO>75_4bk6&(Hc=h@5&)XSUJPhK{-TH+F zurh?=e!Km4L9f^k-D(_Tym+Qw-y2Lekm{P1r8%RMw1gmdch)J?lu)+ErtYiTd>F>W zqTjc3-p_Tz35A~nkp!C<#GVXpAtH^vmcdc8S7lWxP*};89~VaRb%mYU%DPA*Q2|)% z>J3$7WDG{Mz-shyQIoDp@x9rfE+O5WCuA9)I1UvQZ+u1Ms9c~B67K39#RWmozfy+H zuHQmk_&gfAB1u+q91Rkj{HegLnlrFh2M??^R0CO|OUY6(LnaboZh@onG`xylP5Skcu-J=dJ_cz6DCGcEe_Yfkn0M)9za^Ha5y4S^@FU{IL^U+Q<1DrB~OSg z@i=fN68fJrA9_1}T?dmAzLa$^;Q)CIilL^C%d#wvrY_ zl_0te7lTV769{zf%HCe-C^@jD8ywAUC`2(m!NDmU5_he7zZ@SEI~0HFE+OD&`TJ{ph_kdl|-W6 z(hI9`bGMMGF_rkfaQANkZ$e@S5X^=7>y~q?%MB>e5DiEi)+8ZWxOj13< zqskr40>%~bTAF+ujc$tX7rc(w0_`(7<~7bQS>wkg4p?E(QV8^)wnn~$TcX}K3UVsp zF~e4susDw#2zf=~E+iOIUQ41?4AXIdLg52NK9YdYYi4x{$=St>QCt>zzoiB|ZLta9 zrjuTML-gQF=Ln7WoTo$;D$((QNlq#%z^b@6g{&d=0=feH!Equ7;`1yZEhDnJX5x-> z6g{fduTbEx*d&S$5`j~@G7QBT2VtD4=sp1~O`caR(V)gKxAwYe{ltccVcYcCBXM>v&1j0S4NK?&7OD8 zRs9Xg0qnU?psrAknno^ASXZGcN#*0gZUp=}D*BdMo07%N zYov_uf+BDjX$doE6P#TW8Q75~(tYFgFNbD!l0neSl*r^mrCtdHC_@;5mDo9kH2a3QK(iB+9`3S%uiA0(6Ji0W5N-R_%Z`SgcVh|c#;dgp&}s&gpTl#2Zc6wH&#rCYun^_BmB!c=_l1-jk`?OXi`m}H zTzwO#lLb~F9USLAHW32YPe{?s(M;m}9syzbYv%*ySUVq}SVFvH2YoUVwNw>ANM&e= zJXSuRKgpw`LA7X-3}hlBq87Etp^S89`Vz)1I?;K)y0)7MYvJsXM2P0sEU++5ikuJ z99q&>Qmx6D=>qVGS7QKCLn5sw!v2tVjA6ksHMD~Znt;J++9R8H_Z0LWgiaiyTWZ0% zM~O+!sSeTfl*eS-0f)m=pvY@p7rnH6${O~zJwoOlR& zdNlC?haS=hx+oK&G9wj9ZGo@HVwTW41Jz01u1wEVF(#e!Pc-GZpj)YI+FB5eLk$^T z0Wlqp>^BuKszJRWSzt$irU6tWj#=j^Loh;sIen*Q7N>@fgq7JET z(nm1QVxyz;s+f!S5^*1&qb12R!Y&KQW2JhQT%l2@0v80nc}{aA<4_SeZGbYG0|vV8 zlV$_hThrxfou@lxx)f1(ce(eodU#p3VL3c5c;uY|xmz#ZSg{guYlB?oS}^e&yH^nA za32V1Q;nFBQ#xox?e#X5Mcxg=7Z6Zxn#ylH=V=qErrQ?~K?f2bOLQKvF%KfyE|aBD zb>O*C5c@I$lqRm39-b|aiR2)etlY943f=KhZCgvx!$EPd`SLe*C^I`+gN^X-FKhyC zfxYK_*DU~|K0+M3?J%VZX=H>N;1zNs;vugZT_SAB&elj8iB&;iz3ThTL&(BQ>FY+7$n8k!R6wpc7EZHX{!}@T_WAlPX`di41JwY zR^hx>Q#oZ_I%iOHc*e2;j@)DQMbxCjxh7Pt3sg8GC!3i8@kY{Wnk7{j3f#a(vPFZS zrY$g$bPxh(Ggpu!~SveO#4iMuHs-ok4us=X%82 zv88#`lJk)1jMb|e_qYma(XJUOEP=u&slX%7t5U}iMV+^RW z3mq9$#}WKyU0DNpSN2yISdb`p#E^Quy;M%Ha`9`ig4%%SRPL%>10C0`u5KN}59m> zg-1mz%%{1m=+L4WQyN+Ms^J^a6QrA{2Q+YkDVkPYYBv!Wyr*K@MdV`CuK1q;@TgRH z4FQYGc*e7)m=z=9bG^0nCCfd=s%VFnKGudq5+65(SJVYtR~BW&q>pfJ*HnAz z`avhh@C8CdwE5Pk&=M|AJ_5SJfVCOH6UZXChs4)7=^xqHt{5e>X_Af-8!k>3V%!( zaI~^bsX?6KK8aNenAEO5Q9w{oEV&xYEUs@Pdz3&xoMOYJf*>2r0-?$U#tE;Ul9Sz3 zh#klvV7CGcM~kw|yAq6{Ff!ZA(%%{`=dwt$f)lqT4q`R5_+Z19#%Y{H1^M*N)rIZw z6$Yh{d!F1d28Sq#hC^1xb3BhV$V`6@G0fv(N*T1OpcSueo5G!g?og<(J{;MbkhrVr z2$06BXzKlvaX*C!?&xHKG8U%w3XIu3 z#eJ2RN_1fvqm9eSTmaFE)qq6es|Q-wExkeql(+1-z*PIzTflJs6ru6WQ*3i%^DbL4 z-He=zrZ$K#%|!$9xP=IITny#|SUh0mP;T2rlhI%fgg{y^8^-g2P*)ILfRTN8mN8^I zz_x`SOn9aTTS~P=mPR~F(2yldOy_Sd z0!q4C3Rj*U<9%*?@*jY+?t)F*-Ftvwa`=L$uE5DeT$T0P8MRAM4J3`92El-Q9v3zo zg04ExSJiOOn{4T|LOLfp9%MHg^ZjRlvsSOf8_OZyM;TZa_)e~5$0Ts5COzOVs4k@? z6X_?~&DCKglUBJ$!}bSdMwLYf#nFm@6~&AN7G-Wp!t$-ub$PH3!8th!?-HTH$&zyE zB~Qx0tl8sTVUF2;3`Sge;Bj`&(@94mioZnSVZCB81qOyYEvRCsIaq@r$1?9AHQy5i z#a!EQ8#Im_?X3k3EyQCiOGhRAPGHYnd9IpN-9iGU7Uv1WQ!BTQT96Z^lE^z+KtvtM zr>Vjvo387$L=6h{7IfpCZkQ!s35E?7?O@QZ*(X$=1S5y-^_M$>46FKWe4)1qCi9Y0 zMV&N^Wm@HjZJYs=*eI=1Nd@8SH3%iK7+SS&6b2Yh5)4c2tuvR6tqTn8MB{yN9D2ET z_5GK|;ZQ^%*85Rp1I5jeVHe`LcM#XZ5Fe%}yp+mlBr)C)JvYrhMS~99a!awhjr%HN zmBd_^$=xEiFQgGSvt?REs%bc8*@ZQf(Z2+OFfNZ3=7qNk<>aXQFeu~@*@$GOlu%$A z*IN6q3ci>2vIbwguOTu?_Vl;9|S40aa?4<2J1Yx zOm6(wia9VeIIOAEF1vNtYz?s;e+{M18X zK9(X>*)>RDU`sCHkz2Cxf={;68bQ_Vx-9dHV|aq9Tg#dbg4E)W%hw(YSd!F06_ zC0@Mf?5G#fYy%sxC>?M|%}200U`OG%?xUItcN76Pgu*^ga*;23Jz+2mRkSv}#vs-d z?tq>el#6)s=x1PXI{1JaAj(r&nfgL@{BlD=6=H>QItUNkEZUjOy_5lCcV( zH%k`#vW041&zz4i{X-52gRl?{?1Qp+ae-c_%Vc5|iZRNBK%T%YYvjQvBC3~n>Zes>b!=*CyTv8E~=x(mg0H9PC{c|Uq%VL@ppPKfx4Lj zf`lbo2M^-zYx6iWwk2F5j}=QhwvzA~FBH^qqgl1{0vWDNrY>#*RNbP+n}9<7dEV|yfiTPP8K<{X zI8+2gQ5#h1-ixDE+rUC5 zvQR2Q-b^ym8!6ees+$>`V9`C>sYjidNi*eagk?o_3O&5jhK;VS{ckiq6&_IjIxb!Y z%}3Q^^Pho~&I&2dd>L-4JQYPkn%#n$`gU{PWVYfjs637;%UHfvRW@ioV;cc9fb<6Y z*4A43#nH#t^NNp-D4uu`8tsP{MWIz!h)mGpo|`s)Q7jxZ&7{Q(cm`-gVo@2@Rg1S* zN=UFgdg-m0o^orIM*?ie7MunP##GH1H(RN=vMP$6^8|cdOqVI|O_Z$IG&KgcnIA^B zP==s92?#c#M6P%$ie`K2Ora=zu_(vh?_J2pn}alr44g4xt|G35*o?#nl8oJ~hR!%| zX{zcL_&8k`+AORZ?xZV)pab6C*~Fo-xHLp8WatYOs6tvxg^ z2`LlpvJOi`ZB%grZiV<%+)>^(UL`tsuBR>%kZt<9)e#@W5txQtzBtOypH$L(aWF4R*NnW3j~UU0d^t;rntm0i+$!z>8(vbq=L<5KJB`E(FIi=g@el}kkl z0Z~(_KB2kQPFl)m(~cQ4&7x(ybwIC@lv>P4%F24I9P7+j!&5dQz?|+%L|Ha81Mj4{ ztRPv3$j{@xC$K7hHbBG6qXK~7J6JTNYYh+~xGrsz-s|C;lO{I0xoT+VIRINgq`%=@ z5jv_>NDPgUPRRy#R}Agsf*}IXb_uMen|aeYR$0!ybpgCYVFhE9G3plNtt67ONuoI1 z#wX^aW6jf3@!c%L=g_7xR+pi`4F?BWd~VBHN@`tH3U?`sWP!WXi zY$fn$1}z@&N9GV3BjphUEj1fY4qedU#j-q0;CX(oAf0$l0RhG%UkImmw}C=k2w1+? ziR0&i<^;rd>q_#?V z#kd0Ys?%bv%(I?nfcJuP=r zwo`{yx4m;h_SO>jwu+$TdS|n3sV<$%w`^?8%dYLDy7y`9T=#YAPkR;bYbU#U)QSOn zJ=%A>v@}(%An9$&&2@#=MBeAGOJLQlR;<-QyE>KYRIO^d^yzlgU9vXHs8L?EQf%F~ zBTeq@ZSHW9?DpMTWmOky%a)}y#Zv3tm9duD+SsZtuFg+;b=}K%6K&SDg1v7;N2W#V ztvY)2)o!f2YTevQ2U_0X+-08K0vBHGdNkRWZQi-sZqG%#sZrB%XiqS2b-EtzZmdd5 z9J$%k+}@Z5-t0B1+X-`Rnnm2_6;eb%fJ?)0iZt6;jv)mrL-0Qc!&Mj`+);+AFD)!Fy-KjbN_qVETiA!yx zJ25WC(|fwNXcgAx?#*XdLMp4<-Jl+Zh3%ain0vb961A$K5$&Gwo}TRK*4mqER@>XT zX$re@wNkHMc7Z*cY)Wl!b5#j#&h+iwyGLyRD)pzWt=N0DUhUmcnQqv*b=7-m*{YqL z>s!4~ZL6SawkR62d3txN*Frlc4tI9DYT1<|+j`P>z2NW~#7DQUdrxKo3VUkzyQ&k` zmw9(RJ=@z9ik0or)$4j;z0Y_mc5iP-qur+3yqGFd9=wC8dhUCf*KX+@U>5D~ZoN52 zO=`Uz1fAJVi__EBUcED>y_v1+RK2#XL%r&*j`r_+jeFTe*`@5(vDTH(ZcRJ2v%8(w zX`a~DOD$f5JqEY9b9Q^p?AvOtnbSKqjJLNm)veY^rR&_I%7lxfQU%c7wd~8Svvs2| z_E#NL4OXc3YAS-KW)!3i?DW&4-JLpVtF77VsV8lFvEAM900000000000000000000 z0009s*V}#WU?C)LNz!mN&+1Gb>IlXdQ zYO{3p=Gl90_UpB1x>k*(+G-Qhjdf;wXFYYw&AQG8T6=rl=JTxXy&G>&Rc~j#dvUfN z-tFzG+Q)j@c<#@8yK~*%t=qeuyL&gW)bV>eYrW0w<}~c|xz`fCx_a`p*R#i3-kO!u zRqfrb-P(EYJvW&L9kDI%dwaX4ySuj;y{*07&3MzU_Q-jb?srk!b8S~yQ@P%~Et{e1 zsqMYDHaD2NY%^`h?YqrNqdRKs?$Y+_+_$Z@R9>^J_V&51-MT7|Z0V}jCEITAPZd<2 z)EjNrxZRhNcV%wg?)SR%#^mF!XPnoLd)}q$$GW?%TBO&yY`R+`wGBPp&F<^lWsPX) z-j91*UOhWww+~Nx^mliAXOX+NQl`ZBHf`Fy+$HPRcH7u+JZdUoBG*)?<3;<`=U)pKb$cHN6wY2No2z3BDkjlFiaY^}BLH?@y? z+Xr`dJv`$&_r2ZT&h_VZdY!!h9Io|uvvu0;?Du0@?6XO4Fg@cfv$omjyKi={Uav)a zr@K<6x2GOniECd^cG`M;-QK-tn|Aj)yE+}zque{0s?lg`PkY^Wy6xL`%5P?uIyvlh z-tE11?^~-*z3Ec3Yr5Xvt*O0Tx_jJrUY&c~)TrmNdscFm72^(E%_x2kmatGV8u!`GwTy7K2S z?RMO!z2mpJu9X~DihCd+ z(y5_>lByn)A*R$cPrf2AzWeI^{{NFdtyG8NNH+Zp|L}49`X*f9@AWVHvU<>;jfm3J zpWIb_q=a5HNd-oYICTc+H#4_DL*b|2QzF!d8{R_dEPCQ@_lVsB$6A%dSyap#;x{L- zU8+iE0G{*=hQ#9!Xy4Ynw?LdT;EEBj*gZb=Hs&2L@ar@od{3okl>zLjgC26U8?_@d$A&o+b&kHw~~8 z!Xea8v1=TavYm<{q&@eFoCzOgD{0syBwtUr!1?KmqdexZ4*@%T%E0XYuM1kQkF*Zv@D0$NB8lT(Gg0Ydg(#d%@>I;Cp84h>!s|g$ZRia3 zm}fΝZUvqS53P_662gTuz8Rv!4#DO+wm% zCs!XIcriwM5IFCNq3#(L21et--tAX!Xg^r;Y2PHLlGxY5uL03fxyC^EewIn%(T17z z@#YO|$V-}`#B(hYw1WzX&ADZo#GJL!;XFVv_BXZ}nnR`-rh<2Kv(T?<=dv|1(E+mQ zVYn!mJr~M9NxALh#!HH%=*ZK+78YSjz^aDGb3%N}1g-GLzLS8eLo)!L#)O$4E=*#z zO+r5l`rswgk!IOJwxv!t7T|zKi`b)VqF~`v2~vnyn07u;>Nu#10!sXzMYSr4OY`j` z6Sjn9O0Z;D2q^;wjyVGXbJB>ed?O?HKtw=9L_}x!{yz-vIXwU-l+axDqS z(%(z^8xR%`cNw3FoC)juIt6`)^-mG*auvi$q4RcF6%Z68j3_XO@R)^Gkl;J~k%AY9 zut#_kq&3+=9SPB+-xeI0v|5w-Oo5j|DiRqRnI}UGFJ@dWuXrr!@}=hc z+?np1DPpTZ7iBd2K#V~zr3X*02ceXic|yb#%hqIcdnvGEm&rpnY6Rj5=mCPYJkhim z_{E;hK>%2qKz5@?Ob8HiP$BZ7mi!Y!_Pw7&?08`2iGz6^0d<)`U9&Rcg6knIO^&Cl5 zpDqJTh^wW=s|fq4KQ-(k!lM^QP&76MfN&j8ruXOPRJ$X`ZJzyy0@Dw&QF1W@hf0IR z6B-ffOs>3PO!8h@81N`y*xlrL-^rw@PQ4bM3hf38a#V#P<>*2r6O?jQkXI{Tw=no- zX0aNd6i|u=fXUZW!<%yDV*c=5=ZAPQTbg=|D(EA!ML;Pyj!9@QuI{ZVv^{<`q1GpQ zIp=`tQA#mPymam_DL{pb@qMT>S%nFD5$X>NiU>EsP^@zmj30NImPW4#e;!q-3hH$8+9~k8oK@jB&IK6p?T+rG?UAFPbhO{ zvUgKArr5%@N#g52lz8paC=vuN<^WX=T5t>r5C zK%+v)UZIkHEt+H8PRYNsKVpgeYs%?za>ydad6-78;~Aj4SYTfC0NEBcl(S2t{Y}h# zrRNls0{aZUUf6+k87A}#15;W&STSUu5Fqu&;9|l+4~y4?)}Py@-8qKc}qN6Jg~6b>gZ`< zWxS1CUkckS0bNe?)z0(??yp5;>(bmf4MQ6)>xKYeg1ZN>l!=ra7^X;FgJZ9AHsE4b zt{Hgx*UF$H@YMGb)8I)R>XyVlZY zG!H2#UoZ%}r7urYUK&j>GI*F)Tsmq^V^j>sCUwfyb1T*rE$*vm2f!4xytgi9OfI1( z==O^gNVQ<}12IGTJ5D!?0D0>|qTDpTEi@HaHc>=5vcF_tVJ+qTAX*l61f-{F3rjmd zpxdMDePZDOB4*ag3Uaw3WZfKw*N?jk;20ttd}ajzu?^zwgX6;p~26zF-!WRb`{o>yN;`K!GMk)6MkCrs{a9;qoL%i)36YO`p-06Fm7bKW# zh2_bgOy82Hv4VNqCV`F!JM-Y_ZhNqS#g?9R4NcMz>UVlq#Diijc(ygFz=Y-osc`#^ zMr1N@u2;ONQNNvc)BsB-^pyey{HtLiqDFBLts|8gEVol62?4KKGji}ic1`YWB?Y8#+GuLGZ{i* zbtN#+No6Y}Ab#1gde0PEVngpjgXxyySrt=nPsfF8nI07VmfaJ8u?6sLQd5EDrnr(( zmSZT&Odq{L)#{U5Rx|kCWoYcK83;JEX$Hov;|5H?@S1z6nZD({RBGWeoEY!8PEI`` z&T>MH6-j)(27uQ+b;K@N1->(&`1puyT1X)<$~DPdX4#h0I@0v!Mr1n7pfL*JePI32 z3|aSzVjTB8fv9i{2sw2Xm35_Yf`0}!>Dyvq@0ea5gau0_$sYH5{foqzjRNExwxhp# zqAf?;q*F}d?wz^RHoEk$mdUi*hyd1!NcYj3L~H0vEV9FHqX72n$GcI|&hFV1?w9Hp zDPka&V6vJADeo$aO!6mJz zSbirf3xOt!kP49LPoDN;xcwbtR}e`6r5%hS;$3P%$|ium>&z9Z3!)!(+t#8GOIqkr zMLHs9w5z5k82O99Dj-U;$oAfS1txhg=pz9Ofknf1*3P30%N*kJH=hV=h83{J_j;RA zu>-0q4T93Uf{-!YO8k>0QE|GZ>{J?_|KbdhK|YSm+R7T02@RvMW;YPo)I!9O09EWv53OL z)*Dgb8QXCDWTSL;K~sCMoa-!ckGLP$5u0aC-9BFh1fv(wgonJ76TCo7o>z$HFS z>9r6->&k;t4ymB=AxQ3=9>D2X-v)+@*G+6&{D#{!(5p^?!PBL}T1YG2v$fDc;i%f_ zb#dby9WE7ZPW|TwYBf!&cQPD$pA`#y0+nr3`=LxR_GQS?EvZ?CfOpuAp_G|P^De!P z`3n=fvXw>rA!)<5+;{1u$#Z$*8FcL}fs^ z+B>F%EslF@M%rk2ZWhXnk_Di`Aty@ZM6p`>Y64|@-L3-}e4wz11?OlJDGOLOQ^u3H z#VH1z5vnF?Lbirac*kc@HVhYfOAzI{yY3xlMYAPfLu)m<}T6_;; z;BsW9)|7+7d;u;n>uPURXUTbI9%|nLB(NZj5C#t5OG9u> z&`p8pSF*uM(J`hy3{|@h#j^6@kX)`Li)-h|ER5Y;-e8qtuwpM(q1nEL)> zUP>fngL=#u5zON;T{{~Kd2w>8Ku*uHGms3~-uUS}*>&jgt|ko1ApM7d=WG z6KC5fjEFGdnM^9v38nX#6JKqyH0oKheEf?E5koO{OCjkAD3Q69g?>Av>+6#zIhYC2 zPAT^Y9n~XVc~>ZVBnFPtKIwdpI@rR3I#gr88+{$x=a){#$?+RlMSO-LW#;lxt(=vh zS(RP)Mf=W5?hsw9%BK>8M_-7_uMKgsMz2KV&TQSVi>>s!cC|A2$VT2CNYZa(5M7rc z_&}=WK&#G{Jay8dlEN7ub~|ajBtEe4+hD4Qf1&U<;pe;}@i+NC{>L*8*#YUjA!ben6Qw zI}ryVGNo$h_V^@n(A$}3Ao+=QPFV0Rt`CHBd3p(IPd49539AB@bZ^QcBTAtFsx$JW zMGW^GWJ&v*2d`$*U6sSOx`kF%<9gLK;3yDD(u63<7Z_{nV`!mYFeqG|q!0*4K`;)$ zJ(Uk9Od%YY>2ZK&;~SBUQ{-Oy=1c(tJ5;nmgp=OTcvPa%*Ri5hI5c7F!OKwaTv^;0 z=XV+G!A?5PdxiA`2vNMeQjR<|<(j8g%O7OROg)mPPy)u&u)1*>plxoJizNcfNqe2$ zl87_8-rnG^#eJBz@>tBs*ar5-84O^bK`Vi~LUBBX#CsxH{^?`2*Ry_h^IrY#6hsIr zLq*P}4$09EdxxMseul_F`>(bq96e)}Eb9wmI3lt!(n(JRV5-3ssJT0Z*vp&-{31enc83;7Ru0Py(+5$|LqYTF9;7R99%#W|ZLpA4hpZBF+ONj?X95er$ZPZc)nyQ6+2*K= z?@r)U6fllYOFc&Qsg@8h6^B=MY_}GLE>~eedhAYpm#$S;;|rMzEU~mA z@!7)WH(2v9uCuvw8j_5Se+X{5BDv((Ql@;+HrbK5@uhem;>F8McZky{=-85_eu$+U zP3dGU;`ZKXj7yK+T_}E~X){)-PNb`MBM^&uR4c7QW4JydRaXbCtw8aWgs(uXQ+qA|DaVoL$ zW_Iq@R!a{&ljPLtBnbC(d3OPWe1IK1H_qNoKZLC}4)^G{GYR7-RN3-mIJTBwARZT4 z{bvC-O)czma+Y8+Dk~h}qnjo$BMxMw0umWIAsHp0M251Spqy(vwJPINGqJ#ZSN>}Zh}PhGT)78yO1@YySsZ)Uqa=6Gw#KL(Bp6_^u(I+vtfn~~LbCM{Ab$6Bt8#LYK^igsSAa?@i6 zoI?3&;7p7qn4c(-jm8nSAat=h@V$#t#+(cdpn43zJV~CkcBOz&!2!2I-p+#uBHlro zA;q|8?qs^EFA)rd2blN47h}Q|(dB1d4|D2lWx%!h?P1E#k=Agilh~l}kvD|7&B7UI zWo=mL2YI_mDf9|b(4pOE@FN$Ax{;jnwOmPK;wNY)TfA#ELJq_wAsX0MxzWWchkmXk z^7*2U2zNrVqfN$R#!%;M5Sn4S;DelurR$Qk9wfb~0E&AXc~Q5E5Rxe+0PW{m6d~t; zclXk!o&Dz5@M@d$wFae=3gHqtmCN!x)yuZ5Etn$6Drb(CUPL)S0JXn9iif-pX~K~6 z_37`*$)QL>s9|9+4MCe{SCB%}Ox7;*dSsx*T36as1kDad09YGa9Dyemd)KMKPev$L zxzQU$4Oc`IPKGwRf+7Nf8Z1H5OPnG6TR^6PwZ{sUA~Q_f>H^z~Z37b%I}IlB09b%-!af z$h1Bn&ElvuG1CvPofpSNV9d{ApAwRU|r z9-PtDMDh-i(+Kr|bSjmpb|{$Ijb0o;P6E-Mor|;0#~YIZmpBlmJ_)gMp4xFYN-AkN z;Nf(oJP5_dwqz=c;k8CsP6IC(nNJ7bM(zWzC1`Z`mb+#}s6%BuwHv6#T$BDrsX)nhTJmq1i79Z5x^luWTt9zc@TX&KEpT}pkyc0 zYnGF|U0xUCYzz-Ygwkq3Ohnz_D(N0_bxW^yAxYvRrTLOsBW3EcEY7Iajs_Je8|u}% z?^=m`1fc2ljwJY!UQYuI4NjMO!HeD% zwpl()Ik?5GOI{%#YD*2_dS5Da^9AStnA+id? zpyJQIge%6>HrdIw$x;G#WX97X$S-yTy142?1dpV8OxY<#3G%+-r`1ga_KrzxLDE49lR2|QXHtasMQeS|(>+8sjlRKs z5yQN?-VA?YP-i&`kjr3(pPPh0STQbfW?)1|Ftg3Um+;!Rrs7!PXFLeC_Yb$Kwd51s z%`zqfcrkSsnbB6voOXbzv5ZF611c@WX;zRRAZ@HTBVxh=2YjR)0E5VL*i!CDj`;oq z=D_*ihkM-{s};pr2>sa`L$qI5$g0<}?vi~0FaS#nV8WEW#K#9N7(=(!K?ksoec~HX zrw6qL-Z%{#slTi@c}2IN=7C6MD1;AuXB00~RLx@dZvJvhvyTetuFywsQ6}*b(!yH6 zLREpnloIs6PDRpI`jM+Jg-ZNG-dmjllU#&$2NMlSySL{=!~zn?oCw7^2o@d#fV^%4 z8(i--9|$*+NKtmp5jNpE@KdqMij?h%uBHokE4GU8y>K<#gQ%g;Lig)@+F|(DqFJNp z!w{BIr@2N}0m8*TGJDu2Vm2=XG25mik{1npK~G-@~ATF`DktQx(g)+XKF(Li%4N+_!Se>? z3cfgBPbW6`+fkW{JqX!*?k7AVk0yf5H%1^E;Y(Ub!}CgMatP>$r=q(hn+hX}hN$PN zccJlQdDn+5vn<82e4W^8;Ts2J&o0ZIgnN$$y%IsOmgdpnl<0c0BtIFd z)B+Gti5B&^*{?5->pKd@OCy!$pG_NdBZp97kcPW z%&?{icVkZmHSKd30u>zs(Nb?t`_mrE-;WhZx-bWdr)ftCm_okt+`GtDqMOz;o^*j0 z-O?=%&~zC|<|o<5mzXuh zbEgzCX4yEmxlg=g)v<+-l}sm8r2gnQVH)WfdVFYEuqHu#4h7M8gQZhZdOHnKe|qp$ zdPWG{TR4evdr;kgivDkV_gv)4_u)HkXyR?x${Xn$!MmLm^QFwZ(4#+q;wtwi1{(8% z7BybuAH?DvWqaAq?c_R~1C$;I zvNfdR3ivH`Ddrg)XP0u&ACayq4$Q8WB)1q;kU``pTP1ukE|f)n;no* zTysubRwY|!mSOCG(h?Lg8`?3wu+o5DfLz^73wVv%V|z}I3`DU^Xsa-jvhjk9hMUvC zU&`!oG!**)r+L9jYuX*9$8y5^ECL}C_IP_N9?C%Bl=xH>N=32);RaR>-p%pNL$SvK z)I{erO03jWfK}GFR8}C!)Lm_~1)s$N6IRI6v55VsxP#1b;9qsbQ4MV@O$ZIzTF*`S zGXc&yQOn{$QGLEw3MuAA997SD+cBu%Jx?M?&hC1v54He0x1Qf3OKFW-H9WWGPxKGs?IZrzMjeY zDH%aCg11kSIxrWi*5_oP$Lh@s3>WG%5a6#pv$b5sFnBu^7k0-T?s0Q`=R*dM%sS>o zJyJeKavQQ9hYF$FcV=`B(}1g!xtA|{iD>TU#7@SVjh6Pdz&1ZfJhbfMl-EG#9hMRxD?*u@dQ}L^ippxflSw|O!FI`{a*MRW*`~Vd z6^UolRmHIZckiOeH8Y(?_S5!|L0i=^f{{+Yc}EsWd3{iJ!7y-+@Wh3KLxYhz2ZjnF zRFpD@8sbJT_cwOjX(F9HhoE0=qM0wW@(gdp(9)@_5)9i#Ka35Ql z(Ypbb7BS=zrlM{>oaLIN$x@G}F6p|H8u(E~CxjK$MC})^GCi4tDCk`%=O_#fUVw<~ zg54tUu+@&cO@&zmOME;GJu9gK4o*2|>_S#ZX>(jV-$1a+UL1S2%N<(O<~c>7;0y}k z-pHw(X6P2nQ%!317omiL*1W)t-i0|*F_D%bK)YoDN>#%tsN$W7nr1Iecw z3he6vb1Pon1tf-oV|=uj!pIH@^KInC!k>dkWgY0seB5DeM{U+rVdRdWPHTzup*1To zDDwj`H;rHwuq%^$hWRSBJ42W<;2qN!5t+M=SFQ%)3_P_p@+MLH$-K$SD0O| zr)LR6&aN!v$VXjufRE;C%bOfR9%mt4g$j(Kpt^p>`Xwh2%W1J;oq6`aQ zWW)rOkLggMbqYR7p*w%DBWqHo5%eQ++fMM*8brj}17?StsO@|T%LWLQ_;6Ou@ME%N z?ksC|I>an#7M-$N;a0n50r0~r$d#cw@e;|}@`8}`zB2pbVr6(*J_8326FVbuCmRJ2 zaO>_3#RUV_X4-saZMjiLE@}dHIY!Lar0rKnyGG6a?*3Sg*9rl zy1?oISdp8Mz_O(E3Ryt5-b7DZoe$K4I_V$ zmCJ);(=55sBFLK`AzTdOUtciMo0Ah21Iod7n4CamxlIA$J30$oO)b}OOqd7Ax8l20 zM#8|m9-4%D1P7;}%W6cGqt-3=wQg!q62qg6g9vmtS#oSlCLhEC`^lBcoc>jWx9BVJ zBi=1DoyO|pu%D~0ET7y#*AcW=mMrXh;LLpSnsHM>0Cw&zGkXvi#V3^lghDOKaaM{< zEy;rG#st#ig-ebXwPlY`4y=}wAsU>j5t2DhToWP666QJVuQm)Jn^n6$;x+l{}q@xIhjIR$cyBz%qdjWT_zEu;0(VkqsG1+&dDN%P+z~qz`E#h^|O%qez zr-uuocvsI%FZ4d4h;}rmOch7hkW4J&Y0=#M8jliv{amgB){eH{|D?A*n9s zs?l60#_c$iVI&rkR5}AZG{Q-mUgT@e_7fd%i*D>oJlZ9Ce2? zh$VF_u7a2947AjQj#&rF)bfUr^G>bPJtXuKAv_=@K2#ja|rde+(0M+>4K*2!~(ldac5VhlDk;tzdOB~QG= z>AsHNN4a5BoZ?j?>{4c8x;M}6Cr8PoWXsLb2a>QSnp(4=RfFKzSPKPxHy%U27M7j4 zT2`MT2D1=%TCjnM$Q7@E(`@$AixV${NDoQYg3UoG$yvn$%45_!&Fg-s(R-w`5F#Vn z6W?J&LrvSJT*F;%0F~?!jKxFTG?uq ztr@{?bRScKcUtGiY^eGI&1Uvwu9krZxiTR7XtS8&bxMBB z7t%ih%aTUxB z+b6_1&8p;=(%8VQS92=vU@>m0)mFqvdJGxM#X~uWey>8E-yftOu27^s96&(i%il?6 z0mA&0#dBvIRW8mX#Pjp#ASTQl9Az!oo1qR;%=8eUscVl-t`0z|iJ>S~VuTPL$?H^~ zTQOmoP6Y`Q$!2K}I+mf&D)x8XxKa~FTuna1?$~)F+0eY{h0rjTcFMuj?e=ToNECQd zNsh&DU+;JiQfpRPxgIGJ^6rifL}?DXy$pB-KJ!Nb2a$@f_?;~npMkpbuF)uiJ+_D9 z-jn3RVd!&l5wH|Rba{ogsaQ{J%=0cHUfA6^=|c3p76W%-pkn$J3e|O`jhx@rTB+!4 z-ckw)1~HhByX{p;QRYh?R%SG$WjyxGr_ai~=}CVDmPZ;Yg476Ky!QGYTq?sb60r|OiOBKYDDZ1F zDJf@p{YN930y8V)3^#rE$bgz$HKxcA-!` z=m*GajPrCzAP8}3u=V)qmar%~nB}CmDF?5FRgHaE6zxA&Jd;)|;GWGB?wB;26H}*t zO)qraM={8NhoiT5!$_^2hP^;3HOV$>Boa&O=LPG7(DNul_~(L^s2B`BN)Tmm?M)`J z40TX=e_hyk*A=$`f!)J3P{%Co)S*bQxpo{-F}vP7f;7zvwoGZ$wJ_x@*IZ$dBi5`? zpdGC6_ywa4K8NE5pE;*yG9p(@vW&!9gRpABAYYlVhONvbM+S)~6O>9nf~ZUDJk&0# zye)g)n#jSb-CEsSEVfK|Cz}G~B2Rlf{t>G>1MWjggc)Ro0?Ngrzi2&bBB2Z9MHt3;TFr&52rHyjWEC>EOPqq+9m)CG&d+AFZ)+0~OBEV&JDC zhom^lz*l$(D7TbrE{z1{R@7zcb!(_sn+O-vkQM+`CO{|#7i|g#&6VPFRZe3F5Ih_t zgUYV%_0o#J*X}AoOF)77ZabW-G@tKn1s9Z>si5p0CF67<=3kU@-5eS@9rM}BAa$uN zjW9_5Fuy$N;_SK{&hAm@C|S-oBW2_}-i}$&Pemplybi6UnrFw|X(@o%*mXx7vv`}a za1nA)(G3NsPm?2@P>W;|5W=Y~r9x4vHg@+E)`+8*0zVmu;+*KD+BJq)@34lei*S&< zAO;E4nn8>XIEL8i(0*$+z!Wk`y1EwgodWHZUB*-jCeznC+~NRR!*Jc60(eGTRF)ZR zP&pjdPq-0m9l^Wfe2g$#)UMQ7i0_RG)Lg}m^TaOZhI;maWGWX-fs0t-Z00gsb>(5R zdV^&{H@PsNk4YpH9}M~o82BFWo@njC(bRJ;QA|fY1~xxKV(d%E{AJBaDxrM}harcg zKx%5xipWhH-Em=0SacarN^fqWLIQgAX|lnovS~Kxpx(tD2&757hKLLvC0Qkz+X*Z@MNM4DO;x24Q&DEZp{$8>51i(Fq5Sb8ba;E3Ock z)m@T{txhRWAoD1Xfl|IuyE2rWMg~;CZ)N6}YARrz-M=a9N+5N8UCcxmy<2h&@dZKQ zH>LI2Xe}kJ0pv_J0}r&VUZ^L01`N1xYZh^y87YrXSyx30i+tUKtF;)x)lO_0%%1j7Di0S`$PgnQc+%kc z5ozAjS{WlxW%3IfB%|8p)PqHQc|RUza}QKZ5D^X~qH!)@9R=xOSU9N_rJKPYPsE_< zCd$oa3oQFlUDf6F9jWrDE4&6U_@q=iIZK-n&@zff;<#MpXb;XeY`Ro>gxy{{I9qZe zW#Y2G3vY*(82lpdd~c^Xv#^tT_UK3NWTw}5Dh^t)I`7UCki&&AaporDwCo39=@_Z& zQV(25M8>>i&z>-PH??ib^@2oBH>C05`&-l**Y2Zgcjfgbtc>xe?K91FHFrTwL(Hb= zwJ!Cezbq}+j>Tl`qWZnNQswHiDF^qS$;*KZl!};IxSKmBx7NU~oNEhuLyPdmEOzUx z_R7Jzv)s$C2e{-GNTtRBGG6DzH+H8N)>0#hSqpiy)VZswHRB8H-;nCuUOEB9enr9wz063&MO3l~t4IZa=T%PE z2VfU7EcIRjuc90c^N{QzG*CliD5g}OPPv=gR~Q}OGULG^V-Sx~>_0K1r6CseUY4ih znH8Gpk-n2psWvwudB$z6sJ67YtWJ|~$-9*uc%`{*#^maxH#=t79%@gx0*|i7-t?)R z&>YVWx24CHmq0}aXEu<2$3uD3I}Dsk%^uDW2IldVryA+gygExKQ?C;M>O#7=Kt;yo zmA)zjjy;D`tUW*|?zLM^$_R>AC-)|;FH3vI0|+(EqZ-bi53iGEFR#5*t|#lm@F`-8 zSt6yBt~R#%L9dP^G8DoU7TnI2&UN~vrEvFVB=OsTIP+}Sw!Qd6jJRaoXTvzqH=YY9{cf(7Rvv*_N%bvWTHnR)*lwlSt zkx+5BS)@u_=1X^c0Hvtx2Tr>{HD27-k<-wpijE#O!4D|NBtcbksF*@=;)YIp)2+4| zg&eKqJR}_1 z4GQ*FX;N18H9`4FC>okbQ?1E?(TdSzgmT&$$@2SDa|hkfhd7T7?6oix>OoXfc8o4Q z^+t&t=eyrrnskg1DV3jsrtDW%a^2QeStI6&Kt2mM%)^vMS-h1^aPEp{dx%m`Pg!CI z5?`L&_(J;J+}lUiS9jsVhCvD4<2K8~;)O4hycP2L(E$2?4nRThW@|GoXVZiQ5!I7H zcA9=$kFYJz%&Q?Vwhf_u71c)pN-CVfp@;>P8V^ZZK;1`1lXYc9X3<&V!d-Bcl)9i= z_N1oAg-#ZR#JMhR%F;Xrp6c;5A4N#nbSRx&wL{rLoWm0%FgJl1JkxQrC48r|nuM%< z6&s9vJeK#B--@KdHM(X~sZwh2G0wtzGaHA7Zv0tXlkD5@cMyHhP8dE2p#d#-Xn+%c zX!*f8Xth0?!623yYgbiy!<)H-B`0APayW1tK64Pvv6u!n_D17$DPUlb<+rgXmSA1ABSn2gpB&06p zk&42mW3N{z_Cs(@MEy+z+&M8Urj|jJzDa9yXRxM=EW#f#^RWgb-g$Reg1&qK#Leqo zDkP;O>V=FXDxAFFx+>&LiuFgRJg-~Ysx^xC!GccCEfZ-_!v1{%TyV9Pdd%s%CZ%6r zSq#+~p#@>Wn(=zblb5|hC%2q61WF;DHQKoo)eJ}!4z0bh=JTaNaDpa1sq- zx_w0s87?3?y;hK97p7A=407(bwp`YjiQU%Mz8 zO%HiG)i9R?tQX0nuDKl~69L<73!!6G4X+1BL*#njrl#Y0?M8z>qTIN{Ti>-%6`IVZ zIQ(;jj80~7v&_m{Pi+*46d_&#O!3ETB?W|I+q8oS3%8CNf(vRfXFc7RaUhenaz>fV zh-P6%J~?<+`MBFHHiScBQ5Kkq%bYsSq*|vpWstLF%!!J^gU^C2J|0ks%CBn8ZB5x7 zVp};CUD&4~yN+f&X?-kleLh6f=}w#UrEupMLG6o$3%-fku_E-I2Ay2abh zWv{v@gsKlzR$~!%vnowszGD}RH3~BjFGqJ^IFa8?@*rdp)YRllMmf?eddh;2c(%ah z(H^e_GM@bLl{A=#m$SFK&coK-9i8ZocSu1B=i~P<*!s(FYuzT3Clm?qTh85uNQ@S2 z?kq9_U8K1ZV4oS94@*$lD{n6=q5H>%cA5)^v+WWlcXWVJd>=Au^C~%#53`+OuQpbf zVE`)rhrvz(Co3+O%kD9XkdiQvi4sOamtT=Zc~ zgF~^5X2xbSAs)Bbn-edXAwY5yuWT@cYXsV$&2Y#8zTEOx52C`J z9S}s4GG||}qo|G{PIs-$rE~F6T-X zK2vf%U_T3wh+HwqLc~E{G){K%d!1SO@dq(*_3NUjgTBH z%|2)W=2C0axOtUE^rv@nd_d7c7_K8y<;~vf6uxwkmw;V>d7Pb*pmGKW7*8xPc1L0l z6IV~pOovE0ap;R@UAc5_iBw*XqP{#O81|gc4n)mpE#VE$m)6jkvuZ&@QbwrWHOa}3L%#yc|@aWDDVO{ zK6!33sg^axSyeD>$FZxCGa>sIF`ODp#@ZxF5VJnrD+uyx=+|C6`8%#}IuqCwoP^o2 zYOU!=el8`xa(7501sSz9Az~Wm1aWOm5IaSftlq4^;k%N0)_3@Wy_@E>?i@PMyipws zyw0RjWGp^@x(1ml4`KJoQ*z%?BOY4JY6d+nJky4w@dN;|?sTsn8_RKxWBvy6}3uwtBBh;ZLA=>m}zlrc<2O&%*ad zf!7TnkOIc>OyIaebTp*i4g)0ZnszNN4Y7o8btaL@4@&wl7Kb|*5&L2Oh>3T2J`_*1 zyxiA7I`S39siy#=)+;WRiX8H~UawC90Kl<{*H4gBAG8=C$m)b-9mU>smgEJPQ{|)B zixQrxevs76*w*ZtwvH>@k7m~%)t_MKoLf`hvn~P#SE~ww8^r+^9iwZWvml%9bF&#M z-?)nUA70nRy?s>fk4r^jeBCauTOs<_bZe#dyq!LevsD66&GQCzh5 zhklOLpq@ZR@6?Ts)C)#{N7iV)B_~~0x}C~6K5eYQUQtDniaGmKYI4Aq68qX+Zr&T+ zBzs`NVsU;yY|?W)aqa>LX=%MpMT?CSkQb3E`L8G85HOeHi9`0ws$k%I*+uZ9b+EHh zRoxw~m%1guAaG;E6+#8=d)g{MCd%J;PZ1Qn>udzzkAB8|B2y(tw52W5oN5VtIYSO) zTyh}HwR^K!mXgrIfhM43O4*l&ndoT}xW-_<4I7)MXY6$|gWW-Av|vJOXEs?);qi-S zjN>p*y=*a7Ez+|QohrG=8nVFcT2+CrSM+v!(wPf0d#OYn3sl-U+l4X!w zp|I+Sm>_nZ0wUdA)YTL=?sgZLsN8FEyRdDkHOkh#JTC>2qKO|{d8FTiN?4PIK-)Rp zkRAA+Jz>pP8!p|wJ3^#QLfn#5lCt=YBa-G&F>x;m8nI zIi?~Bicf#jK^A67K$5< zr+uvDnvr380^T!_AczF(Th`rVcoDEUfhzF0=dL$U`ueGc{lRt7rDeHhH;AyZ0s;z4 zAcf&;DyIc^))E zxa{jsteOP}cd#=rABwFjgfz(L0?#>qMX*0-y|7PZ%fgbkONVI`rF(4uQVTv_)R$Y} z+xq6;8ERB~Tv4c$6`VFMfjPB?HR?4w?nCEv&mGgtgSt@!Z4Wv*lFfd-`O&e$ed*0C z(a#R|T;jx$7{f1&2Jk_=Uet@S0=o-tRE@o|+i1jPe0k8$)kQ|oBXAV10!kX(NPaZG9V}tRF(lleZyawOZg%W|z#MeO3?wqj{lf2a z#OvF;49l!>I2OPw*RNcLH7q2cWD>C1}?m@Kp#8@A@fQaj^u<|Kwcq+Y5 z&T(M;EGBa_7+QGinf1(rm?|Z8D6k!fWR)0$f~X$|q>6!A9=_SBch z;8snqu=a>nFnuH_^?ayymIUpKtMK$SUDbJn>NOBowIgO~4OkbIb2e$<8n&h(+QrJR zl{R?%d$!@!jzV7YE;k4p;$}L78bVDSbeyq0xRzWIG@Kpp2t*V!khuo)bHaEwJPwI_DxU{RB#bJy`C9k%%{BNx)GG?Yj}oZmPTdB!+R zH$)n#L!9&=8r41>ajC~(83iX%wOpLVP_W1_C%=phH{$^* zKX=qpoU4GWruD*y-AUw(D&>|n#Ur^A=;1C>bzy1)GNC|JxT&MUg(`sIvF%ab&^*Wm zb}b@7^c2Z@0+vd--VU8CN*L#Q@0Bnn}C;F84S@QgQb{i#->@NEv#4CnQo zPdpCI=ga7yB@k{_qP5;)?pQ^o7>V4*VF1%-ZfW}GJ4;~tJrF+VZNhPUGW`J~iop)I z)*iRe^RdXQYB<#>0L*Pbcs3?1AX}8+CQv_G5Vl`KEsCd>X<97LKm`u`(MWEd}MYZR0gf9 zsK$W-TXCGkEq-YQ8V+9{R%7iemT!iSbT+aPPrB@c85mCJ8k>8rP&Av}glr{^qZu1h z!($8BxyuP%kI&RegD_du5x57TXw%K;Xka7~K-k`!3_IXK4W7@ip7Yv?q5>HKt%|@n zCr>OAmuZKwfHE;Qv=!cb_K+OUcXDM=fnp6Wo+hfPUgNa9UV4h}7TAs^?}LdsbvmBU z%$D6`F(RgEA~C|EvB<-TyltO(QFl9qo}l!IEWP!GxQ`f9^z?aE)E%?ltY5O1&;^_1 z7)5^IK`={-h4Kj8raW4J-MBvifssR}fbI8Yy2z!?wGUXeIC8=8)vqK@3RZ%l0{9g7 zDuZ5OFHf?d`a8VhoM)%GXy+%G;{etXWI-OF*fW%>(c|IS6^qc-OgeH6yuR7=Iq{?G zF&6~7-Z0zMZ9jx8mt89JBJ%@y$ZmKKcXJMBA9-I39PzYw?&%cawO~pM;~vL0qOQwZ z?teH9+efWZx@hcuNh7{93EV+gRWdjXYnp!q1!jE_%;%bCe4REySZZSlkKv6w^rvDkyH@MaqP8e6hM@K-u5D2+M#eg z7}ep*;u8gA##hs;+&IU*fl3x}9u-`?Y>;01FqA>zjC$CMscuy*sUVmbB>hhm*PD0sO``@A)vbHnxpLwIqja66dGV0x`Xpw zyuIm=Q%5r-9vu(P0x({!S}r?RVYgtZNm21{a5WgN=U9nm)Lg15Cr_kS*;r=0`YoKn z)D;<;UA*(i`*Sq#HFLNm>qjTBubadje z-wy)e)BDNGC?ILhduNe`dT{$q?1jJ( zfJ9`4b~7Oqk%M{@l*iWY#?xsDb15w|c6+U5u$abUcerIr$ziST+7<}-;$Z>Ige-f| zA$;k4Ck1d?iro^lx|e!MVX-8GFPWB?#40vI(t|*|a1ermP*Ew!vQtCqnUKxG9Z(&g z*dwHbG&_Ck#ddz`LO>QCo2lT^^@#&1aE4ZRa3|CYYY!Vk@<_l1av1KEZNER9NSh!xo(?+Bv}o;fp9`A!yR*+GfKmushh5x`xaf<7K~~>TgxHl zl!0{g8Wx;oXbv5&+L>Px)wK(UjKoGB{eU%W4FOys=u&M^m_GC;T)fMD-IOj*agW#3 z#z?G<+O$dN6so9W_1_0I7e1z^di)|4L4ib0JjY>QXw%B-yLT-IrZi|NA$vn!(iV^} zDtpBPyU8)$@}!+u9&m^xkK4+Pb-Z@bJq_LC0)_81#8eD5d6he4S)+yajhs6Mk}hR8 za@uJw5=Fqa1d0c8QGq!}D<4XHdfe!aQ3GNm+ca09yMs+M%pwJbXI4opK+PQ5zZqW1%wTdt#B>o7b7zGl6S0K_^lm zrnik4ggY19JICQ}gl(IeJ|R!bI@8QjJw@hdEpug+DKKLT zCz0exMI`<$Cq&l14b_9cXP{XCF6B>^gi43j5B2dmEgRHPFp9e) z<;|$Z43-N;V2#%%Ad6UqCCJjb95c17r39r#o{AW0F;9;R-h$$`o?@&f^RaKT^}s^R zAIh&cY7nI^{v7VWo?apPylohk@?P?;S|O2;m)DPs;buuUy(|tD8k4x)W_dLN*vDwV zf}~cN2@aHBmC|e(h_pS2I8^4nJ6&lM40&+%YW6R-sK?wROV(B9BEF6DaRe;H0-zTQ zD3_$sEZjr1Pr8!yAQN|-t$t@&mvl9hg;soNLGPNv6!LwQkK0D+2D*y+2fmRA8Fe%x zw$C*}qf;%`1DJE+5j9S^vjWLWLV=Fi`h1x)(})+q@6vxq%QL5>}gq;M({_-uQoNF zt3WTN8#GnKTnWmsq=C*P&>AzB9tkr12D6+mDlR(#o_tmx#XdN+V->BrhJo7nGSB|HpDqpDKU0@@BtpfEY+$K z9;^|9WGGXcUO9E|e57JG*D_Qo3kwJ`AIJ)Wh#>eR*saG0!Ac4sOg^$>d2W}+Yw?32 zzBE5kG1kZyd)|dLrso0<%b86*i;&xIBo8f#Mad|Ei0LL0tt+ljuaSN4c23jyG?lz!4<*ZH?lNti2%%Q&HExKF+O4tMZ_q4Vzh+U0#~?>neb?F zA;qyIbwJ(@DoRr?XC=#?s=Y{R*PTHy+@Ahk)KN-RKDDsvjY)+Adq&0i?ADPO6J3T! zVDdMB0Spc*g`FMM7LV)If+ zN!-8;vIML5cS z$pd$jZc<7-o>r?UWQ9JmSWD$ixq>M2K0WVCgb}tDX1BKp8@)%&k+Pv)e2b2f#JLgy zH!RsvmZ^e#FDq7^hd2@S%!xsHnzU$oCh*X72FYvJB%EM&Q@2!Tt7!=o+aP`xuCoz4 ze6KQlmss##7DU&iEA3iTM$8F?kmOIZ&$XO92bK}dAOCyX1 zGuVMP#lV|Eut5SVvV2$eAD63u_bhz@PfBMi70K(q3yAGD2K9}e0NE|2A;6XOzR9WO znp#D4jW#bcTwRrw$F6EZkc7(1l;c73YrR}(YGxi@Pkl}#V8dszu`fe#rx22PuLdoa zD-e_M`<@I#XWd?LA&W=}bf26W;UiA~yzR!uTmauwjOe|&*2akAytLFNvXE6cjuzZ; zhlYg6K0ivE;}Y`dq)&h zC>nQCEfOQ{mg;FUFFTO%&JU((g>b3loIqsGEoK*reu_t zmrhF%H6Hn?NPIrmvw3z)-6TC~9vT&{E1xx+_oT+exzCq1ZQBBV5&)H|PKl8Y7gIoTj!Z(6I!ItGDg#7tk>Eh>lA zyJ8sO#8^%*yWD(_!dhM{T5%R!EeYQxh!2kVZC=NA{B##NTWQ2c@>#>c+|M5x1AFT1K*d~2>%0WI_ctP z*o6!wn=Aw%H<;NRi{1?}E)z3q;HjBze5_@D>OzO5pKX!BnlF1J7HvY%Ol-C}W2+0+ zLByQua8(x9E2l{T%g~fo9%bEI!6m|*Su#oYiFhvg`<9_L-EO3e4#~8kSBO;78hl2I z5^l7l0bEiD6tEd+pdC?Q)IoD;MAj+b&CKkDhsgxX$5N&^@dz|YDA01LY2G`ct-r%s zUJLDb!>KeWV&d0h5=aEnuR7E}-IeMKe+EOZ7MUi|GeLW^?Pwdkx?^sNH@24_NJV;? zG>EW=#}@)X_#r3={4@pDR0c00kB!xM_rr~;R|49(fXGShh+28B2u^PzP&XLcuIWI5 zwg(|v;l?(xkW)Z6I_3<@_SDF84(`G0vi-+lcby8=EXm9TAYw=ET;O9U^_w^#mQ|D0 zo=@B^I}phN0#L__ja&#>Czb%T5oS3#hb+9CdA>60=(NuDff0|uX!rG;8y6PQaXXa+;mq5828n%zCLB@T#h38 zZ>@m52kCv$r*ayNKt!h|o2W5xyI$4AXXxWHj5^+M!k3xV!DOtaZtLmoBR4F{czC08 zC^pu?A86a6j?okPXJ=X^$-{)j99S*<`M-Mv$4O>!7cT8L?-#Ecbvdte39YV}4QelXzB9rEyL-w>x^N8>` z%3o)FAth$h11g-{^=i`YE+Uc6jG6Z1WWD!p7OQJWK6#?eJz9{C)GBDw5jnEoUux?1 zYQWES4d-qiTPn1t5lO8kF$V<6La)7s12}!f6is{AR(moCtH5Cg+huZA*11Ha?^jTi zA2uqvd^ycl%sZjk*cJw!)6F;!Toq2y+flMDA%)cm*}shr45VyPMO>AF>r5?%Q+S(% z`oMcSm&dx5cG{poF426bH0tc-@zRSs&~X_E7@6A10+5P}TS#I+9;+z0dh0DpfP%Nz z0Go0Z>M*asRk=si1TrV8TK=z!a$~7TCu6|!v-Sm8@7v@ z^a*L_!>_#^UOz}8k0=?W3Kq058e+~CPT|c5NKvvcd}5*0(G9vh4D~fM4s$J+yk%g% z0Z%wU@?(x;oPgR}k-GRA*HbPcrvELcl1u zH%u|LvB_k)U1a#D4!UZ1p^hYZ8B!fBJ0sCbQ(T%>uD;IZEOoGJg<0nz z1pNY?m2ILgLY7AWFhZjCELQ+MG=z9EqAf0wKYPX|WhJ6^eGwbnyqBcC%GYvz=ie!9 zhAI@;yeSW5BA+ZzA3F^9~hZyp3IjyBjp8iRSh&Qbc`cC%v3sa^qs9BG4&Sv zttw&+!cj&?g5J{V(Ht7K;?(djtv1?UDfFP#xwT=JH6JC-sp)xf0>E`T%fbSn=29Cp zLzo-sSaG1K>3Dc2E6|f%YX(LLy~0k5Wh#QvhKfx$Ro3i*I_7Tm z$pUze@L#208#w@>9%N+7QW0QP*VID;u*^HH)h44dk71u8+7;isJx<9kCZP+W7~yBz z9@e;(gsMSEdU_7T>E9_pKX7+Wxm;gkXLNW@tWeA~cwcG&iiXDEJSf)!bn8fM=#3lR z%Ah!W6~V5B_LvDaeEG~%!NoqIMf5QdHLtD3tD7B+3I)1C}vn>8)6@=0D|P@xQKWRSE-X6o@^<+MI7x-I{hnJN0>E1 z!8xufmx&^?n@|Ghc(E#yE5c0^J3<4l96pXPIqcr&T~P^6A~^8zyr>r31U?X=82dMJ zKAfUUD_QMsTQqO6F=l-27~cVyoaPJwom62k%^b(opq>#GD&c7oz~0v&aXewR>pMt7 z-dNDZzCsZJ+j2*G_fy3&U2Moi(u5mZX~B4LJsrW|HePY`WsBKsG$^RT7I8>k3`(V; z`JROt1iF0o8Gt1|seoAQxing=qgNpq!+HyYIkoQD6KbZhp}9w$VqBv4t)Et_oc9E; z9>Yrf^Hbd9JylFtUd|~Dhw5*_BF#zZ14c7W-QcBQh8R6BoR-xXQLF7|Rp~HSc<1YC zX{2nVuS~o3c6I55r-Z17)-s9D#T+BuoyWfRXNsFq6H(8fm2sydePZ!wHt=1x2t0e> z?(=vjg%UmKP@sG7o>c7^6P2n|aw9W^3SQkL75C@Tg9W>#qeiNS^?RweOfuMee9bUw z8k~6lbKPK!l0|L<`n!5{oQ~}lYs2w`%rz&V9%lH55AGQTRiQ9D5X*$t^Oa*_A*?aY z%dkM-y1QY}Zg?cd<(I`MF=1EDyJ3o%>hWYL5h=%Nr;{ECw@n&ZQ;fE_4(#zH6+vY0 zMN#-PrZREX!HIgi6KFSQ8GS!Gl;b{h7?Q?fxKYzr9=V zstR^ZxOdk|)9e8r+H zoVhly2D=M~fUjA`q+yRTU%S)Ch{wW)`Lf!?3%#U!BG#=OU7AUVg@Sx2#>+~7MqFko zJSwiS8e%sc5uaRO4{+0aAwX!L20Z3-qW=hV-iN)FMDlzvk(TN0b(jm+fn(SZb+xTU zA{wxPbrI?fd3CtFFi!O}(>=gYz`hq@Yi}QXMdNxV2{yUEJw}Q@$J7`4N6mIHDkNx0C5%e7jQ163h3y$z>9DWq$5hD-}XkQ85G8)hyE!=nf{_?CJ;mQpE4cC91Jz!5Y!xL`-71K5!DR!yMk^xf zen>Ars}O@8{a70^85MC@A$DVaxCxug29@mBn+6!=f{kfX6A=p9i%Sid;9=8aV^DSK zZfBwjY(?x#8Ks!{4D8RT*F0lra27|uL%&$zH1bUW0x+F;(pWGMINK)^l>>!s&Zeo< z%KAX9`yZW0vYfzZPON4^57vAK8l{C1hx0Vb+FmP!IxCzKS4tQq;3B zQWUnT@p@|>6 zfcAQ;YO(m6+kU!-D|=;0<9&hWV?8wyo@1uEVWW>*?=lgHM-*M&ATgP6J?AaJ6J5Y> z28ut=7@O!Zsd_c6n(ub>`7i`JbfUm{_gEvQ#jO>5QcOv=u~Eb|>8XI-BM%8kBJ8gS z>h&J>MV2)HsYK!#QT0`*xO3R3GY6Acn+FOF6ph&W)JEIN!N^S9JgzayHFO)`d&9PF zn1{h$^10FhOi`x`HnDgFgB5M7o27bAhKCeEF0?5u7@hf}HQ@%vSptG~bxf>yfiKco zH+Fq4BlQED;y4maJG^;Uw(;s+3^u|9Jgr#bZ`d-Hq~c9FIt!Op_88UnO8WJSlB>hR zVFBRA-3cI63gqND9cQw=sUf$EaexO9?~q~Z(st76hwH&7tX1S;wakUi^0;1-mt;g7 z2FBtuJL=bA>3fHTZMmCkC!}$V#atRhLyWf9o}smKz2vX2fP>ZM)=QHia9X%v zn8WSuW6{_ru+r~3bfkIuPzZ_DMzM-}>(&relWc`DMW?ki-O7PhjE>YWY06w^Z^N`*=+Ikd93z}6%|M1&b9%6MWn`9S3ftHm!htHx5cI87 zt;USwoR#H?Rc~QjTqG21t5oLJWcp||COR7G+zc>uv3sF7^#OdO1dtvEQ{C4*7j3Uv z*{em(_=H?AB)N-QUCYKF3o2CG#k@^KguKtAfTB%(A}d>NOtm;9dICzS%iS4HXp4=qKHrJd%p3x*NbwK zM8e{gep5mYr^|@5okpxwD@nnTkqa3edyCfbUV`A@t#UQPXS2+#Njstn8H8SY9kp=S zL*oG!$6uB_&fIvd!G|gdmQ(8Wm_3eQr#vWla9k=W9uk*|^OVnB=>kL+lv913K!0eMJ8++xFn);)`` zHroo&O}P(?14vc94MJuVI;1E&b$FIKNsDZGY0ozBI833`>y0*|QQ$gC#5_i6Nwku^ zyl@tx%@<%aRerEwSfy~pCpXI<8DI#DInB-L%7Xbks|{OsuZYP7EpYgw-4faSpaSvC#sPHw*0cvmyX&f8t>Sl3OSWyA@Fj##`^7FP~m0Ph!# zApHFOw1-u5#7iw~?})0_FyOcBt~QD#oQ77>Q^R4`Diu1*rR_(}bOk6^*MDw|WHzK` zPkYV5*cGG&3Z*eOdJz{IO1!~Rk6Ukw{~4*vF@V-8r-H$hD^)9|-4ox9^QGr3Q}SsUy<5B-UhxfrRuXXqXs50Wxabu~5uQwmUFsSXnVzM-5z9&W477K1uB+ zC2CzGBk{g9j)RqH^f)ElIqZAwXIOv)VC{e%VdGddErIxtCLw!KJNhKAH<`YHTlnA65jal=x{h%1BZt*brxvhBFr|*J_?G!yy2KvF=hXiSve|8&aAP{l$Ckj&iO;Zw`K_U` z-uTBGm}WZgN6D9@9-ed#N6d50SCq@wuS z+oTGNs&tI`d48@K^1EbTa1CFnZd#6H7h5!T`E1QMoRa=&2O1oB3}kX;~h9hPl{A@4V>MCuJ|<@ zmpy|znB^|Y4+JeWM8ep8qrugY9YABF5$OhSON~`;aZ=%2w9IL|Cl`Kdj@rII-A{>A zt!$Yak6(f!!u~b|mObO3;L(8eO`UcnI1LCcYcPYBTY;ogw*(g!J3#a%e{5o=n(M-KFl8!=WY8}+bF0^nB0qQvuhVpFz z1MBcvCAhvnJ@V_pm&xw9Wj4tQEK&(!A0n}ZahQ!Osb1a=hn4`3bbrF+z~s!%H9srTOjfLlu# z>9*ArU{gl*O2shWP@;IH?nHvOI0&{fM&YwF6Q%d*$Fdl?Jjyemad9Ed69RbiWszM) z2_*3I+iOY%0!cnKPK;?eYRbgR&mK1M*n)G(?i7uuS~WRrOVi19BN=gRv)FPStsjoN zlCiW|F1esr$7$C7TL>Snq|aYU0mVS#1I@G@DV0p31i(*LW0lH_#y~5XQ+SfAK`-SP zbz!W;KLx@3@?=+cmBL0e;gTLh*6{V&tyze+3lsCwK?vUB7Kz^d74Da#GDc@rpE-lD zcjf6CR~o>vdzqFfdS~d9e9R!SWWKWl%BR!Ka=n#q##t*hdWt_s)HLGv z?Kss4sUjFL#0G*^bDmwx5dtZWnwP!iad}+f8xSZjj=IB6Eluu4$&aClr<~@MEO3Oz zD;)T71P(|!b=D5|GpZ-TqZ<{OK8Z1<-z8)(U6WtLKX~5>${^WG;50(1Ssh_S;rkSw zV8hLBAVq0&Oya@2#{-UI^#W8pwhtMZo2a1 z&iJMMv%w0hmFE<)7R_f{#pRV??bIy^hhK1*^8^RB%zf5Kr2Wmsy|qj^#Ry1#)PQ4b zvtOhMn;(JZ!qtk0eraK2K3$~oauATk?_;Q)xZqSfQRI`{%ibL1OiR%w)@wMcyw9jk zb?Pi*=2Hp6p5s7CIK7j3H6sx-n^G_$H9M6GJ6P#v&41z|QIX)nxddj0ndpQ_`W%rMNb z#?e%$*xz_g(m3PSuvuAGgoVY^Hn~14AWTScsHcHo5l1dw#xZdSu#2c>!6>*PcQZ8Q zS>=P>@L6pd>f%tC3_`?CZSAFqWoHIh8wZ0`)?GXZ@ZG1Gev4Go(n_EQ6|!?tXIypH zuyyI-0)epPmbRcHt#h6e^}_7)PUD%l>Vo18^a}^F5*(sLjfcoiAvFQgqASHDyt`B= zoQ-gSUd^3feaT3~Gi&c`M%kWm&H6guj07^sL}-!6K6e!5HY?HE?!C=jHqd2Z082ret6o6#A55k5N9z3wjx z-66zTrU6fy=T`%usdRavew1@eSiHqPxl+thmRH4X@^fmD3i{4#FrZ%*;i}a{qIG13 z;qFdy0~Tzsi_{w8VvfR?gj(tzm9aE;g@??|h}%xxGT>LtRANS@#io8iLc?fm)!Xq(ird*wA$v7iqA|!N4)L(-8 z-ibp56$cIWmQxia*0h<0P=Nc>C1bpIq@o82Z3ZGzOu7^6TuVOlHbQ*-g*-vnalAq9 z_3Yp;raHQ@l7RiJF(*^iqMDxgz|x;!w-RNj!H=pqM~GTTg6Eu2vN8Rz~_q>TWW@8S7B=;w9T+_gT5ls^K6KxaTO@7Maz4+k;&k} z8B7&``rF#0<7$ztHBzcV=iH<%@|e&&hF}pUF^sQ-%J#c!P^Y- z_^chRJO{IS6DMc9U~b9A7{vHCWs{EB2vw>QsKm92#|ke}QevVN;>{`ToTeC3rsHau zR}PFTC@YE}sBa2(7TB$CEirY%#I>aEIzWcEM*6O!e>zJs_1c;*O_Hpaa?CFk;kZwz zDmP;_0!#@gd!fU%2s@NLE^IAjgXW?lB`6;(o^7i1Q!z0l^_3IpvfSA`F3Q4|5F`5b z`^(iPFc~N%8$ywORYePkRQR*3re5{gFqLocM=0oOUM^{Y$>*UF8;Sw2wGHPg>sw>j>AU ze0xbDCUfS5gY|Y$G>CM?-m;@Mc%JP0XM4M0eGvn3HWU-(pq%v(N&x~Rm~{dksc_%` z@6fz7)mZv1<)&D5lSn^3?5u1BKCzpc+GTP(dARFd$d57Ns0`c5w`LQ{(lv7a%c!o+v@*;S?9iaTLM-x473GfQn=b7EDARmJ zxIj>PgDu?ZECViXkRf>yS2~snAYsOj%?R}-O;SXKx~{z*mA7M93Nl35h`S--mN$8 zQ5TsjmU6;`bwiQtX@%k_aDEJTk#o9|XoNaTW=8G4KT$_;Z&u^G79&)Q&73GOg@_cn zqZNXa8u2}7`NLuQovCoU>aoJ5r6q92$nK|m|zhoL+&3uEua>DQRE@|E8{yi>5i0HEy#3(=#e7P_3dV*ISNaF+JSTF z+sIo;I27uNwuD9C;m%>4WMf&>1?$&v70y|pwjqH@NkFmlrmwEg#V#{&5|ioc(6k^@ zFLfruG&L#+%ayEPCBj!}^JMkVRQYiz#;arb$f+cSZ5yiS^?f_qLGU>d(n3)N+2+<+ zT%v@9K(fIK?N;IWN=Br7@9vj$a?p*N$sMQx7Ov~Oe78bCpu-sde50EGnc8b|dJR4f| zs2v0N(%uZI53DTfeO4e)avmN=R-}IvjbO#XN;MVJ2#Rvm0g-j!ljpdXk=d1b2|D_4 zK}RTEvqU3KR{>cRs0?DEbG+b~!d^rhm5xv&1``|P2^r!LlISTFB%#G#N5PgoAZAw9 z&uZToOgw)pQ9JPR+JCdgERxJ)Gp7O6$;rk=Wp%fSTIsX2Q|6K0W7IHy z6A0X`M;NdcU4j|l6db|71V&n$Fx1C-bWriV8Lh}nZug)QXX}E;VmwN4TRjX5%sNEM zHC_wN+(05b2wpSY(3ddq*_UTxkg&k_BHcK>MY6%fIRt>rn7Qj8V(GSuoqaeg#vcFdq`R$|0Z`^h7K9Ltl@exl=1S_alxIh!rF=xDh z${V)05-r&*;GR5IJK*5SFB4w?ix$xgNdSlw-)5eQi7-7=)Nb5dgjNINXTV!lw8}#q~s{2a3$Z+ z3s%xDeqmj3exgK`evKVRDm+sADvtr~y#z^mXiD9MGLNa#af-698R=yadAz02U%rW_|gepI0HK(hD-31lvO+}zDaUU67~YEYaLrXnN(&dE87`%y~fGg z_P&UDR&tes!wU4EG_JA{5vt9;MQc+YUQy`kI^DFW-wXj%0YK2rJzO6mc$h(Q8uDck zJW;SA)u&;&>$YTBj>YZthc-C@rt0>aU!CW$)|QI>r|a&tI)*`(%8xVYp-tpkdtNZY zJoJG&gd9I$(s2XY%#C zUFn8wH4P6#^3fMRuPt;!I3yket93VM$i=;P+u*72`C;sG+<@q&Ctpl%rdC$Q+C;Qtfqk|igg`^Db-}b5-9V;Gp63X1 z!8xlFR6|JijhQmyA45eb{ z7LyFu6y9{m5@tN$b*afSiW6x>EMUoFkji8sMCJwJqM1!GAUxK!C6V6#^tz(W56#26~}-+!$01)3**$iVoy#A)fcwr=r>jD-HT_J2ZgPb_vE&8-=Ge4J?cx+$$KLJw27G(EjuY`P=r9i?c0>D zo|<~AOjB^eQ?RBDWgW|#@EUr!4DiT!-)rs1WLZ;jVPKwOJ+O~7uz_uxCzLn4&8Lr^ zqEY@hG^FeFFRwO<-+JZl{ZSU&%qdp|k<4)P{k+A%H@MN(z&p*If;HC^hOD3>C`4_y z*vC~3x$Q(Fx}=)~_AdK1+U$v-5X@z~hE)DOoTBcb+pBLU9^A$N>}dez=t+)nfRRjd z-(j3Rqa`MV$7z{~^6*~76#JOCdDA2nJ=AEb=%GM8{XBeC1L#gR)>5wO4GDtw9YDSc z^MbtBmlT-;K2Nb};(6qRKscdYUKVTT96?2m=gg|_I#htc^y1s;{W)iM&d^?yLZR&M zUceu&^TL26^_~G{VLu=&JC5=7hC76%rc!1<8wJXAi1c-xOtfJ-9%rqZX;B_}F_QRd zA!s{v*K}}!D_Z#j+>vL+4zzK=TW?r2FhqG!D$WX4;o?KPlw9FKi?him7bk_87bKzj zD2f2uItbCW{ zvWqCXp%=b28As}TYD}n=CXuZ}bSU)2`x%uwue_n%eG+TMm(_T2rh*`cfYPR*L>e?( z5lzb{B`T{mCAe08VcM0<9IfntXl_9Ph?q|E8XX#~@LPv&Kq;WqA|*_`8577vj&X9U zWL;FT9C{H8Lx(X+_=+4Z>!r&I{m9~JAA89T$&~Qu9hAw61$-~;opV)Q`T7l^j9|U1 zd0J)+7e$-C((sn8t3#n@*Qbgf`cA4W#+G*tnV8j&Oj@ZK!E zwBF>XuGo91Bu+M!uyI+1dfym7aJNkI1I>!1>@9G&GrIM7S-R0HoM8=aN7O_!VVn?R z$ZKf~v%*8L!BA0q#$)$;$u2Z{Q-P4H5JnT%-Ey}08ZQ_~?xyCiq;v(Of(ZO24j4<+xiG#)QIe> z1{((IUpLa-Ssj88UL^`!oFBC4#%}VMcJiN!VrRL9#ufs|9^%au$Gq-|2u`soTW$?G zVP(adjN!AH?v)dzr~$Qg5~`QhnJ%{MhfC`V0gzP!^KAN@hN?~(1<%kPi&#l~YSw@( z+g0kE^AAZ4;LnnvK!~#V=1!@NOw(Ii8>(@)?X5u+<1>eF`E`ZaRk`u%M$bjohN6T6 zq^Yo}3sVwZLj1bYwmC^u!QZI$;@N_p*p++(7*2hRz`)tuH z{pUTw_a(mXtCI%vlUh4GonEZ?RWRKahtcjLD6p_+9t;f}if3cUNx!|}<6f;GY^q;E zq=se>DS>P|b6ugp=hRK*PBk~QEr)Q>OG;M(sivu42Rz9Jj>{*O1yQG|$W$9inFiCW zxb8Ay$X7_^_1HFI4|9#uocBC_W35|B+3H}+(#WNcRDwlVKW99q7h*Y^WdfbuEUGrK z;=+l(V6P7pQQRo)T0o$Y)y_FWpPx96^8xxWp;y}2z38*UyN@PBy#h0@EIkd8=v`7? z9{XkfUfD#BpGn&goFOc}B$HSzpLMM-OO{+6^Xuz_aKjZWkU=T%+pBtcFg&+#HWL|b z>MO1jw)V|%@p`ByRIW(YqfnK_d|8Klps|+mXg*#SQ0fQE47VU<0Lm^od35Wp(&YMPTP48y%Emq6hSbP_#4#RYFzXtV%#*f@SoEYmMV0{~ zvYuO#@$jxIXI09ku}WYps##W^lO1u^24X>CnK>-k8$6#0YeP#p3~H@m!fM&@W2R22 zl5}j|h)LHoXInk)%(r)qqP#XQmSZho&ecs8!i$2g(}46LXGs<$c1qqu8*W##@z<(U5Fw9>Y_OHH2uG&U<<2-oAI?Fq zypRMl-8RHUP=L*f!(iGGVzy?Zh&qIW;;FajirA7LQ9H2o44_Ulyl*0FAr;x%&gwlB zjU*cTy~*Z?Ub;X{MpRv&IL+QohWmKLjdOdRJqZl*&Bfl4WsR09iM~axfD>D*PmO$V zjvKBQb(EFpv-ixtqT2HVUR)X;J1s#*3!IEW14rE8iAhrNWK^#@EP;0IUF-5e_q+A^A!eROe7Xo+JS8lCWOptLG z2B1p8Ye${MoqQmtFkwq!>_+XI25uDYFf(hU$TE!uz?sFfCZ~=}Gg>4duQu)?J?A!f zfmo~o+(^EM6c~JpjbWiYN>#WZ5dgjoEvvB0XmwY{6~#BeP?72mQ0rS|AB07x2~gKk zjyP8oZGggn!I3+|=H)o;6ymX^J<3OUp*#bd2lFnAM$X(ZRy|I7%zUM)8)b5??*}O* zFmmg`8Kgq8dy3T=>_;oDEN2ppQu5qoaETU5x;?I}f}wzC6O3UK-&kbu8MLDMmt?bZ z3HV(+-0o0jnettvn0HKa`4Sm;*h>b z=C@USYuvZAE!eYNjn8nwFv)>p3PyU8IfxzQxsLYab7lL$X*R91wiS#OERyNv8#od$ z5V8!gV{P-5&VcE}=DOw@KthC(jT0b@fLMY9dy;WSe;Wy%lEDAonK6tqy+rENi_IhqW$5BIda0htx%eRv&kuwsVUHZXnPlP~3 zwxq)1OJCqNov5!7c(3+_*PvP>CSMaSiWj4Zu+oAqmCc9myKTxKy(bf3KSIbve<= zWgA7-;y1-JeVLT4mmx>Oo94`hI4`s&6`Fde2)U*2M&kN8GPHv07ul~+)Ps__m<7+9 z>wtpippNpAt3lAS*##*An@^87xo%CiOA|Y6ODwaJUQ&GvX7HkoZNrV*wYET({dhu)6l`5OxQSuY1h15y3K{s4Fd5 zYs!40Vg{YZ7`&QF>ii%p4gv{Ki{^7N=HR=vxyd(DV<2t_C!HAA25{n@HR3NrkF0_3 zGIJd(5=JXFQVJQGY*;)ts6lWC({T*SCO;c>d(f8(*#IaJ7s*_^8%5GIqqzVjV}Ly2 z>$TkaM({@<_>W>*b>5%1HS;~J9jsH*dUM{rr#Y+Em217|b9#&JiC+m`EP~_)mD`a>#Hl_J z2ng%FrczX=b2$5sZF0$8VpmSlt5|0edPzEJd51lHAYWdt1M*1dCRyXM1o==r%CqK% zEPo|$u$CO=FLqnGUvRDq&ji0xI+>d(a zQ`vhAA&K0q2%&*qF=hzsyC07RHV(?i+XYWuakFOE{m0|i)nqAzPC`6y@-zP(dBZ>k~G-f zKRkz-z_80MS9*q;*f0fERK?1w;`_8;lA-|C?|^}9oJHckmUkSHbTB~C#-@8xevTNdmH_xqia!7+!ffizSc!1^(@vs3KQsq=Cn|ys#1q9i%*thA|jyl zD*ek|PQJthA#*I3Lz_mxn{ip&-B6%4A~=)F?q@+MqV$Ah2aH%K&0ZKCgJpr3pNXmS2V{2_Hey(yf za1M}>(~+_Gi$LSiV>6no%P!{}{7FTho>~?~?WJ(it@czsugc1X$`?(G z<5TL@p#t-W@qxjP6HJH=xVZF#Y*hu{7=tn=4}qzAy$!jLFUyjVDp^W1bMjGik-`pK zh-B_rkujYzhiG-USSzN4jcE21_od-zLh+u;BYn3PKsoBVAk;}6 zMP4QxMoobt+hu6)iNxB?pz-plX?s9cr5cmeO?Y-*W!eX>(&{-4JPusPvUS)+Tm37<5t`Nrb>xXH4 zcDi_xxOzI;sLh3340!cEFxd??bhXESTkoJNoVD5>MsCHe0~1SlEn~bX9n)d9yb}XN z5uCX9A9UNffS%r+B7!t{1_bV<23;*m3DV0d@#a;5g@OWh$)V?N&12;S2Z}B2XOYIE z*i|~4LR$uwUefKbPf!us@*vyFbNO`u+IKmyNuWeY6<}=JV%36Du`jEIz{5Kz-9bbs zT}X}HMaaA%xcQ>*0{V3p`bPV&>Ox9OWVNbb%xIzNC6eHCCVCaELI%G{OSn4VS;6AuhZ0WWmE*P6V zDwA_5qLG9T35dZ_aP3MU`tub9U7vE0IONBsIS$F1M$VpRCuL@inmABY1h~?*hVU2+ zW14(B?iZ>$0B-9>p#ryW1~#+X7VjUCgv%c{t&*~9ai+<_hp{N9-pqS~Zr=iXomsPH z%w5yJ4wg1K+RWHJOCG2|kpQ`E@VH~0I8#1E0zgT8i%JBFxW_maoHZxtGHgy6@(^ul=IIRu)2dh( zJ`x;3dC!anW-P4laDJ32gO6@j!c6%&Fs^${NuZY4gIbWC!e+$4XqvlwqG)D0Ju>z1YYKQbErS_HFS5^SBYAzK&l~JUfH-NP#0XJ@n_JcbRD7!HJ%| zTDo397jhfgzU;Fa$Sjau?xg@R?<=fUqfBH8hT>;>?SQh z_4j>qBXAjY=T2_rX>jfOM+P3KmONw8zKqXDdE}Sl;DzK7m9G63b;qE)vOplx6hSW; z<^rW06`hTXKqz?^gs{NaEDWkTlms%J(`qxMQR1$$ID#= z;8bq>(}?}5?hM6>UOInnh=Cdrr>Pe1E`uH?L~+>d+3$j-Cl@Spzzx1`w*{zjAV5z8 zYM{COZf6N70=_EJrLiD?PjOuWMN|pxGZjq>fQ4R%rn;EX7ex4&wHdw-bgxZLgUr=3 zGm@@a2B>KztGcgF`Y;zlkmNqVUrGf-NLsfE&TEp*VMH8!`qN)X#wfSI@~oYw&1Db? zU?@3f7Z@a(u5lw0!Y2>~jqxQ|R^t{w0#x9n9pR;@Hd7w@mk+v$K>6nz~{mK{o8%0I;c{($AL5A|FvkTLvXSlJm`NpT{6G7 z2nsT(_^%%Wh{t!yG}!J3DCqjAW^H3{VWe!OmwD(`Al3>VloM-vDr9@AzNNx{{IMd`(?`5oZe64m|IEwCoydb67 zOx6cIZ2I$KA_~`&f;*R53x6LTN-YDhMw4-IY28%lx-0Jlk!cPZ1=Do`h%!#V4 zBSD4=sclQO)(b_p8p%tt!6MT-IZEc?>te>Fi>f6VC(_J0ZKv^t*MSK!J{j9DKexx; zu^Ah_zB-|wmJpvZn;1^7x7C%z!YE2JLcgCo3+6+sS+_z534S5Whs@h7l8alXGqrWy z4;CWp_H)Wg$lr2b%S1*6wv>ILY=R>lAbv`?M=SH8J7u~MX!`cZJm(=Wbp4LK0Vp=I z<$*vUFlb8RNFAqY`7v45hG|lXk9k%R(KPJTM0+Go_#m|E=Ng0vmMUb{=8(3M79{d2 zF`{hU*b04|99#bSv zvSmadcK9pXA+A)SB*u0*4rg$Ru#8S{6tx*qP&%`SLkL$Hw5R zt<1QJQNCYqCFnx5__eQu)O{sNr9L%GBY@n0Sae($#8>nMqR;AY+$*iK5O^z!>!T}9 z(T84wi_H|$KEfDngnKJSY!%@LyY`tn(8x#;G}&3wR77wzhZYONcT3TWu^>(eKw%C!&@wrR&k*URY2BqH&BwkWQ3!~D6_6_t>t-bpUiGafpDcS3$39C!bY2f!dAzYwg&jadQwJPZX;nbl3g=RO$t%t1ZS>T#8%}g^u^u?0hS3%YFgrjut zWR@YL$zPNcHsac;j<(4|A2>V?ipcXG*YVt`M<1Jx>=sxp^oU+5yJ8gJ4~1V9yiiQj z?hCQP16Z#YNWc-c(|V9KYi zOfS{0ljRcafQ&pu1f&U=oRYe&KN>T__$047o0NVtCSnoDvQM|a~o_;+sq5&}J zZ!F@no77jrFym!)ksR2aP3CkISEkv*cLwE~&d!caC8I4v9omGN)n`vMz65KE;wTCc z*)s~m+P=6dArvOUKHBaf4d^Xvg=O|9hq)kYSUm4+unS?f)T1-v=wS1`pQQ^~7kU<{ z?v8FSDewm2iVC#V1=|H(q+=PlI#RgHprn@AO+&Aw5#bCO#vcrPJgq*t1hJ{?8*FHu z%X=6fp8-02h(3o=8X!cvGCo(z1MU{t`kSy-Z)rM7nq%QX@yIhQo4y=)u)G&t<{mlZ zAutex#12HCxae)`*ijKuxZ0hkfLUujEP;MVLmM*9@frLTvCDo4RsoZAyllz_?#US~ zuD-g7W~cE;4OgvclVMD#5_Y~mR8)rww+e8xkJT(j?F#bDc{9W#p6583>s|8k{^DGe zrmqFe3TqiRO<*W>dxZi=t*9q1FyEP{(-xMP{_I_p32Ker&eah2zIKemNcgqfpaRWM7jD}_z46n<3+ciX1L8_6C zQ7js`?Bdpsb&=SYpp7a=q=wlt-nlsR0fflY46qZ}AN%4U}aoUCa z^e*r&vw%qIBkkxU-dMwADM%{RWxD25h71BlysY$uIn=DUgZq`5HI^5Af^_O=2+wUX zISN^}h|S|y&Ujvsj*6aOXJ-g4D3uy5>q!dQ$Q%MnQnp!8KIrMdx+0s>uoD+h!8(^j zYqTL2#!m?Vh=J53v8StCja?C>ZsEhr9G9vTviIrRkkRKjJ+MTD$A$CAl9*sKcNlLX zWxKg5$=h&}sQ^A!^terzI^=wuVzAJiZ3Qa(r{X<{RnWL1JsVWWzPic6m}>zyBks>3 zsiero1HmyKD}ll z#m-nX0|8*cExrN+N3q_XB?oDvXw8Q1nM6o{gX9zYO{)&%!|`MxlVd3)AO?X! z8H8YU(SKCcoQbtXdjT9Xy-8FmVqpsfU|oKHIryJM3CZ|bHCpj$!jHCP5ad}U`G&sh zz4`@XOTMxx#D%{7SbbQAp>3n@Tp zAS^J1FPF3uAT&?9>tk(a46({v-#u1)#j|A$XCZBMODhCpe_Tpw8Y=rR2FJsp=XW_r z)b-8FyuiI!gsG&FI_5n=AYS*fyI}kj8$(Ni!P*-T!F&m3BYF%A)e?-<7^b-eIj!7- zBch=;=vLyc9@V{?Y&z&Cm5%Kd*k5Ckm?=0nfLKzp-*FOUCT`~4(1U`)6GUrVz+b6- z_K?1ci|h7SG6Il*Q6sSuoQ($3gZZjhx)pBoaeG-GeNV{IaWgM9Zjv`Q$%3f|3-JuL zf(fh^lhk)j1neeri*!lcj~+b7xlH@bGdHipHU!h=PW!vUEaEfT;JEDFL9slCz{C|S zezeW(ar2Ifjo}&FaowmexpS|U?8&@0%PIp`^P(LmOVB%`nS%Jr?P>Hoo0X0m7k2m* zUYx4fv9T9#E7%lvDByV5PdXZN2(Fl3mVYg^-$@kr6s{x495z-CRw|sB%WLpQ69-)* zD4?1pPqqNLLdwr=1Zq}eB!JCw;MAzAcSlD&-0|8cm|7ag$v&GaE6{^f#i%FZUbgt- zb@Xt(N*zNo$16Lq8{za5*I%m^Jq>$y3oXumgd^R#I=Xv!_ek7;2A*{-lbX4Ecw%y8 z@++G!wDBvHwzM?H=Ujc{n^298_C9pGKNMS7-#T=l9XZ>?u1x6K5Dc5K$#=e2Ve)hD zwFi#Ivf_;)qfHIDB$DlKFV19}p@SO=kKnh3sVX$6mr zt&9c)5`o_W8;f3fjqWrf%2+)@s~q1iiQhDL&Sru|CV0b!c%G$8TQscXU9=9%;1ZK2 zhAA2pZ##6Os?~IYvoIB`XmF0$;YqW-;QVKs;i%U}?hUlEVm1*0ru3>1$n3Zah}&Fs zgakE9Z$;dOFv(szSjeU!XDfx7q+RZL0bNgC&?D%~7iBNHjy3$fr+1K4PDu8!rLqDR z7<(UtZ(8>fDz#TV@g}FaYDiqI#4x3cdkza!mOjbl^w#YFzgoY>1Op70P2>dVI>Ndh z8|FD`@<6N#6Ru)xZ=QIxhhl)!is-erxGB%vr5PM25180y{{xjV-#2U8ZVVGG|?{mrE8Qr&)&>0Ah)CcCnnq@m4c zVQ8wfu@$`Odp`2v$f??cIj6{?V!`c{Q7)ZbuLCy3r=m%C%4J{>!5g2vBZTZ`0p~I3leOa-j?F)USBy9W=@s${yfo;THEyiOB&?o5)&cU^ zNXKSb<}Gf0)Sx;c`cz79Y6t;GA_2xK#a?>t%;x2J zwQsiV)t%EyVvQ?Q&L=T-xK)jIj?=Q;@LrOU=j-7i`+k!6m_td7OE4x#x>cTL$KIIA zrQXqc$oQV-@EJwd8B`I9N){M0Ij3~Dm2PLnQ0yeJa3FJ&E|`t>pkXq;!f_#zEMa*j zs=-c=g5|HyM!=o*w_jQd;0GjN!$TK*GD__#&ht9E0N9}LQj?KN8XmVtcat{76xUek zF05JCs=5IXRz|npk*MlHFcly&W<_4^?*^Ss;yJ&J3a=*g)G<_?27**8uf*~Wg!|!H z<5@v-!vQ@Zu%h~(ASImMu)q<(ga)a0Y`0~#+k2!D=s~h4bJ%>K4k_01vX{m|Czdai zd#4L*3>Q|2Oc|PeD17L?QC8e*R4Ns`jE9{S(8&{f(ZG)blL*`}N0kzONJbQY8tM*6 zS@F?XkQ22jk|jWgbYQ{@>`QB6_^Oz-69qt(AP8}g)|BCt8#~&62tpP5odGBVu5JLE z#{s`quuWs5o%*4fu-_{z$?OCT~Ap)09LizzWDd28cO8Q0ZT^UO0y?05IDX8H@f zVNg;=4&p{63PK=&ptU5a=~a@tvf!(s?g5K;GkXwR>;t^5QuURA`!D0$IR;~U&X}Ho zGMN-pq^99=-o3aPfH(BPW4ECiV|i`ZBoU#fV*E=F+r`Toy?%a9u)Mo!j>uC5s0cl? z4?UEFGma|czTc=BrklYlczH`*1Mg>4s1(|ZwzW;6+2ICS^vAi+9NJ*3(A0EV9vx`@ z;VPE%Z^}o72H2=D42t)uBw(0es`kUVs1Tno?GRh8EWe{DYFDe5#M_!_r~}rT#yVz6 zJ?_128GWVp)q=TKrDlqgN#w%L=H_urf{gbG*`(_**V#aF^?8mK=Yq}57u^GvqF6yA zm^`KDTwDU%G@*UG*7yi!KyH}I7MWPzS%G|_8w`Wgs`6^w6@i1d1)(>uKEM+9S=5kL zhKeVTW(i-6UwY~I$9CKs%>aT=)1--fd)>}@4O!9Y+q^sW50cS~cy0i`*Rh+T9;poXnCf?$?jdt0i#wNjnr87tGn+ z$W0A7=@kbtJ%X=Jbtu#G*99ihb`-vgUvL%o&Xx5S7YvuU4i<%Au0Wa$0rs>l^^QxMsIC-D7a1;_;^fBU$T(58GhJ{z(-h^{C_L;vi-}a+(Eyd_ z%St+Mi>stF6kKfL4(Jm@PWMu9i!E9RIHa5NcU$V95;z2DXf8TdQc*+ATSMOY!6MN8 zT!wZK^C2ib!EK2lAZhHQGUwy?L^Ta}JG`uV71hIa2sDSUp>}TKuDanI*9+);IS?)8 zA!Sp<@UPmWGcwq%#FJNbI)NItAT3tYR*QpWh5BToq>qj=0zjV@iHh#d3W|>q;yFcR(Q{*sCiKSFw$Xe=IlBx#h_6E$BB179U;BJ4>smhDdi(Zs;0Wz@qn(#TXjLW=0&X@J<-xMXc(d zQR<%9VZCnEwR2@%7!6vW6MuRpHKedpk^r`;1x+TCG5@2pd$hx;2EP|n;;t`_=xgg_h<#yy6ukk zr>O2&7<|?uB!)#iko%+Z2nc3KW5RNX104s7m44)77HI1PvJkF!qa^~6d}!k4H6_qS z4MAK+q6qgY&MTKnmIe7#9zep%iIFo9{Y#1qIt8EXUlubfT&079zL{-$k9T&bLT@Rydrw?kl}(- zL}RE$X<}9Zh3&t=S-o;|*kCh+#*~8=5>eTAb}_!w-6tkXhzpP%_{zLKc0Qb) zp}Rf0+DQu_^MW{sAm2YO`-uxcHlg0;fke!>Zh$0WrN zl)qgC@>n@`yp&c;cHv=)dDc7W8ojrtS<(p`ZUgNPIomOmSPukyui@YcjOhyzVmeGYi2!MT8A4f43hCX+fxtZIFPbFS_LqYG>+SM z8g>gU_H;Ap!k|0(_03m<-sX5b5YB80ySm3U%p$#RR<9U_o>L}H1mdU+zds7nb&-%z zr53XZk<=tUYYSZ>KOfP+cJrv{qF z#nx$wC-yE-K0fmqrFlCPPr&lU;HGZjlo~#zKXJb_A+?vHPDTli!(!?f@4Yfx*c%5t zy`IQrx#f%$x37k5U8Hm<2vG%;g?s{u7%$RyslxQ%vS5`F;)dmD`iQ}HUCGYtM%tj2 z$HqX*H~}W^>;Z`xdk`_Xb1h4cLQ)7r?%qE$-TY#1tVsx5_2g$hmY~#i${U{BEf1en zeBq^X6RVZKklJ`R9>lAdXlcz`P_Fm|q=*~20rQ70Z6O>3(eUJ2M^_T9n2U%?zFil@ zo9WXo_dIfN8Mr>F!hu;);Wa3IWZqJ2EcA+pr?|G`qjF_P2%oSeig>MIyz4|pG`)VC zt?1IEC?M=&*zVrUCm8E@8Q8e?_c5c_w(v+=d(H6X2_W;cmSR}3AxoL0GTop($hz4m zwmXVJj8YIiFczj(Os@PQG)<~eYwCXkI^{s*fyY^*vd6f^%z^E@l+#rp*bYQ&K5lz5 znh=Am2X~wTGYpuGRTPlTp5r#lX5L)dhi0NyVY+lXGfvy_J5gdTEXs1yR;Q@le(^pG z7P-bqrPp_9><33YrEI1SYJ_+~gG(8UkEf0ar_t{^MxO+v_W4Av68EHFwNB}gJprRP zeX_pJDl&-~)%fAHBC^k9⪙NC@9R6x7VBzAaoGg%Iyg!r86{V-t>7&?>B0S)eO+H zZ&uO8FC?Zly4h|Z50bNO#@NqHa0NUeZ#)!ayK;QTaM!{qOVe%5>s&fBWU^irT=IFj z!h)bil$v_AL=+fvg4)3QmszUwfOiN~gA}U*8)C%estWq9Xs6y0#PHKx*jQg;DRm_n z=QglBVvv%{)^H>dtr|B(-xz9*pokeDh4K%^TOpB^y8z<%nU>3^1iQLAabHow9eS0D#EH#suQpJsYT>1pBqY z9oe2Y69=NKyccsmMulQEn;X` znEfuu5qg)L`~y5f$(Qe13Q-X`s_6L=8Pn25)_kYcYD%OY!bMl~gL6Wbny82`S89l3 z^>!)AVCZ_IpRXwcSm@2{xopp|`L6{mfn*T`E0l^n;WNrY3SnZVejSbL=yMm*^BPgQ zW>)*GrM1V+#OrNb!UCrWf6R-E=HrO+Od;O{gq}VGZ61pKKQde;+ChRLF(86hGA_LP;}x{p16^-Q`E~5 zo8{3uHq%EVxvPg>EU0>EenSvpQc8-&YUhwWq4GYwT25hlV)Kn8%dEc90g|xw*$JcB zhoDPjAbN&MlEnnnMmWO5iUoIXwN8}{(xuePuJ=xPccMH_5_7Qv?VnQqg2&kb+c6I;AOS$`z=fbVm*WrwYFF{>Iz+) zIt;VRa790Z2kbEtY*ll5jBD4F^)G-uG_%zD&ub1LC7qmf+5`g0I?~Yrs-I>`3hL{C zk?tuOJJg`FRH5iYd0MyUQ|6&Nxva&>ODUdIV+t58#z;UD6NM@PrtoVgCvDNY!vdTr z*5(P+LK5K~GlogQ_>G$(A!$|i+&N-OSUgZJbpg;N z4yVpMF+#H0LrfwpDKvLicqbmQqrNrlo z5zeOQuQW{!yI3wamgUv;xsXh2*x8!*xUe3%IJ44MRxqP=ZbFCcls+qOCttZpJU;Jo z8%DcD--N+DBspVsaqOfX`F^qno_<=UM44zLIODG8R!}^C%QfijE(K4o0oUQv9#riM zig#`DU!uA8{aa*?*KzVAXeXf$vJ)03S^3=Ic^fDa%v-?eHTq6v}hM-iL!hwr?;zkFH$zC0rbcxBLcc|*jq$7%%;c->7H`lvrS$c znV6+57diMrx^iw!#AP6XNHEa`P18?PZ*m(qb&Nctbjm%NdwPn^&ddjsig;sajZECR zK?(<}A$d>(r+YG0i@RoW6V%^@+pV&u=!j%?zo!n}+7Z?yZG@yK=P>@hcvR}N6VUPcqb!L~|!kk(lO*;eZ_8w`7wgMy-jF8dJc3My$6WV5Nh8Ss+ z-P)d%3%f2)l%bZ;0c=$6w^<{poZUnQ=XA3daOAFO*?e{63x2mXKzV6AorhE_GWD*yR(l z&T+bM#Of*447O++UpdqUGD}KtS%G}DWGf9gC1CcB1 z1LhySIdwS{ExCNSii>4Y{ zVVlC)v`v_O2=hA$-ibxDZK2tVB1K1*&%5|5@wT249p!q;n|`}%ZbPc}y3Zjc&X*(} zlXRJAdPF=t6ai-EYWyX59{y*7gC z_3R#E{JYCZGPKZ^J9~H;-mMzz^l7dQWo@LK1D@AhTbG|Y#**{~9*^UM^0cR?`6qgj$nZ;%``R5svc_9*US&3T;#ffR9)xU=0?C zP(8LyfPq(a@)zAkBxB z+&EVjCLxB=LNAXT9eO>)u7hzddyV2?Ul(mnV*yK8SB-n9D@6UhZ+%tiAiahfmEyl% zwiQIAd0=v}#Ci`$R&3mXU)O>=CbR@&?t3+RwpiJj2mu^>4z)*gTSMwLX&3D8i6?yl zn~hFwy8Qz(tAuS9OKjYlP$ENQ0}tJ#FE1(W5S!0Z8>u#6$6s?XB6DkI*+C42X~;J_U#psVC+ow4+#%@A*J%5E}))V$Dx4pbXVj5-vHc{KN# z?#>tsLB}>Wd4q5%`9`w!X>e2CT$QD__Z@w-oXwC~adNb&uSnfIy1MB3=rAAA%cr<^ zDxTmmX^B=m=crjzS0*DUcV3-FQbyih31IpI9&tYKG zLZx{l0lJ0EZk9T|2D4UDP}qSmyAiq2kqtWZuT{KB$GbXoni`DiKu)rSiad}@4Y!1w z0y#cMNYv{ws*YEA)kD*or4XjW$V?fAq`B2ao|4TGy1Iq$Pm7-z?FI7e8Il5j>xRwQi0eOXIs?kdU`UX%+Cv~rm> z0+x#&jY?rC>@i|)Frnk+5!*PZrWFWztnWBsd{@9k;%;IZ5X&7D5l!Z)*v``hz@6+P z6zcYBJ6uRxa!OksYg!m7&WVk`d4|!RM{l+xJ+>xWn#HB^g(`yMw0T#z-KsMIWMpc^ z*m?EbjHBL%dlgvR>wU?|1j}>+Pwha2NFugVp$nW_Um1>+Kpk8nSwp)fJks`fDNo*d zc}a-qZaMma4zHYddXiJHjFF*9FqX(v`HM7ODj6ZcE6H?7vM4Ozty@Ef3L>s>sN!&; z755omu{4W;m@+3==}6@;6rd z(XtBKp#^8xNY#RuF5_>}gpMT!15iwQBm-`IB$oCbKV5GqGA||5MD^u_Iy8CjvphY@ zb!lFG&2EnL(in(D4%?Pdh&Mq8OyfN+20r}nM?mb?fq+3eC65>jj=W;BP`ncp~0u$#EtB)`?` zW-8PpxkMs|EuMA2)zpqtpdJ8k2ffoWei=Ulh#qnbyha;#$64nl;{x5R6}cs&(KYw5 zgzj$`LW+(|uwWgiOYKYcHOaY;Fo0Vyc#_KO%!+&*4iz>QB!{;u@VAanJa1Yk3Dcrc z*;%$VXL;Ihea1{Ub<1FM2^N8S??+nJ?`1x=E|wL94G30&xYB0i$O_!F-O*z`d3n*3 zvvmX_+m*)C8Q3l%=A0vF+NZc(4HVb9}PY z$=D*jRm+MMIkEUhRb&r4SIuy?OtkFV6>My=?x=E*;uPCcbDP;VUb*d4Vq+Mz7a19q zM(G${!qkFYIOyExwZ!tq-`-))kz;o?6n^tz7-yjYFndS1iDzRu6(Fz)c|8lpRq1fz zeKneOvG%y*8)@S*R^bG;P>7ojOw(Z)a$c0V!|d~jJ88|zHidMwmA`1U>;d)UlS1vJ zJa{xa&U%m-L*TJYB?ht6Zl1WolpzNW*oD4) zS+!~rgHdoaQYf}jn7TRpFB|lym!pAAjJ~U6Q}P;H5h6i zvxd}e^fMrieg*-{%p63fAutPAC zI%Sn0MZb7OB&xAl1?HliD9d$fLpz4MWYB$b4F{7OXGWY`_nQ|j_YtaJZ!(vC9qWDa zC%UfgOSxPSgr8YK81=E4=Cl?g0{d&Y)`dlC$h{e90IdgdHrpPF(3Wsyb7y3kwW29) zxb-!dOp#|=*k#}_XGe({1bWR{%_v`(0?@O~zQA{b5@5ptjic6sO)26_3IZAOTTK+U zR|S-aso|bx$Odb)6&o@3l z6_QKuhS7dy)nVi+Cd*@9_-J6P%Ybu`DhLMJ(E#r_MvRvU$-v-c0AxzEsKpg_D&3nO zl1-8-<%t6v3&b6LQ{a-O55nQ^Z_Wux$($Wt2VyU2M1cXXrOEz&r?stxMd_=C{vk*{c zlT|t5-VU-ikq#&DD=J>|stq4~N(T)%=*AsSgCz|7elc-^ln5nvg_l7$*TK&BMaq+T zGuNSGp-Mr5in;6nm7~H#szdmKBjL)U`h*ixD)-Vmjx$?|LH$+cnZ_d1-6r26y9X-= z{>rnhQsRDKj&4?j-Zb zc_+JDe{ixp0G}#&$ds+zGmg}C#~Zft37hOYb^K&Xv3M{jb!BCV3@tz<9K?aYNu$2K z2@Xrn4v!=AT(Mm&A-lP-2t(P9O~01b%=)$lCNrnO0IFC*y-1>9Da)mAVd}7>MHkUu zJ*x0+As2X?a!YqA`_TA#S9B{U=E8JoK+R6hP@syquNsLbp#!j1gvgktlhU2n)~8U-*KK`Om1UOLgHwT^H#{X)WD07Ev=XEp_dVA_L(6te7 zFQL~EyS!UGGU=ZsfF2LFOTIKAo^8O?+q#ApbG4@;k0Pr63oUd>R^_JGk6&PzqpRH$ zULUt}Y-)nB$3uu4jj5&%@;4?tN0FhnV*Y$&ag{U!JgbKHUIc(HJ^|?wb;Z}i>K?47 zIJ8*X675k49Xmz~?s9J&@ugyLSgn|_F6uUja(m}oRLgM8GZwpv2lo>@u{BR?L&OAQ zGz)~ozX(W3Zr%quAia2wM?oK#)WqBcHJ-Mn=t66y3&@m^iJI$Fk`cA06^f5BYU)Dw za>VQ|$fv5LgmbL+=GZ$4omsX5$s?ZaO;vikU4@Bx2}#zMRV_4PXW^>oHh#Cn`|%9XbS6v5kE(#U2Sp)8=Z8TO-TH_f{(JDS~SOv3X|` zchX%wgh;hlw}Q56-ab>&aS%Di!Ge;J3n&ByQxqCvp{*X|A7zagO91HIYD7q~bEK}p z+S^*LG~Kw@x-^eCNj-3!UXW4AA5HYus`nL8d21P%LeWmd&w@$4_&UzwJ(5<39m`2@ z*d<=-K+wMpNU^YR?%0$LKcdK5SRav{3feon$w3PnpoeTe^j1VxqF6|}yJ3~I+~29+ zU<)#ytqaO-sa2bylT-wSIJ?=(;Hh`-3Js4!dLNh zItLJg-4ALCzC+)bi@?=QHeTVw3p4i&Fkii+ z>#PNf(*VSnIKnr*S=CO_bcC5+m=Xk@z0puQX1{1Ifo@`Z*HCAL&6z>AmHNZjz z!b6&{mbCG`6DM!lJ#J^9YSbLpAi~3WcX*GspiAWfKXuCxto*Wq8r&^a$;mz%v4YPQ zJD(=%S23vrnS$rMqG2MPie&z0aV8R#DvZ6#l55x_n`NQwMG26s67rX)p{iJp5@Fcc#PR3ia@iou82vV-+ z02Ut1F|ap6WXa3q_!phQChG+QMrP~85jnEersbg=xJ@x zTm6b9?d~WkO{Y;I*^3 zbW+8NV1TI@c+9UeHZ_HThDD@>n@Oousa-PZfg{G1bFoh61`}lYh@2j2jL^ zG)r(3HojnEbr6rNs0&PbwhoJp95Zf2vrzHnGlna4p1-YviFPjHi-aZ!CyHM>c>Bhk z1`Q^fnjpKAQXSiClQcRV(|foYQW6*8a!OeXRU3Sh@dVV?oZM(_?M-sKgNwjA6+uUp zSZu-jC1T3+You`OJy<=jVr7hIlOn>gn`j2ThZiO2fV;H~XNYXsTN?u3Z@O zhf&XT{p-t9rsp$ffhM+QP#?BS)6pDYu6DLv-asS^TUHM)9*{!~cbBFz6h_N46>2?% ziXq&WY;?y!BQWL-atB4DIcIvc;3+Y(SO)SN>+(XdaXR#qqmK!2aMkZ_4De}WT(G_% zp?cgC1;Qe61YAM5)~>cU$;@~@j3wl;y&4ST-i~)RCq#HUQ>Q%6MeZ*qA0z=8*_?IX zh!1XHz11QDcA6qZZ$f7n?0lqUk;0c~CB_*PNt2?`H(;VPu{k8BYLqJNQ1QakhK4vj z%JWDhDYEp}gq{sH3XIa1tA5i)EWZ-*hGv6$+zihHiOq7vBzhyNO}cx=BZVDy{!4Th zi{nN~vXCfv-Q_OX-vo!bLcFH7Mhk=tGO@Ql7Y2pknkcu!3AOZd+))6OUCFX#^%Y;LqMmZ(5>_72S(Co@ZuD^3^HBhum%^++vbrFW9>Q4WG)7(ERw6Xspum5mB$pj6N{)W91BtpMfu`B=gq^9A)qEri z^C_i>a~t(DJMy@waPs}b$0LE(zNOOBil^ciz7Gb?`No4Heii$q;U-C(`zLDI0?$u+ zJRxQ&mrnek<%uk9DyEqc)Oe3&6Y59HB~{9PWjpc=V4$aKF$;6bf-jLH7Z)!e*xsA1 z>{x-@-le0&G>@H$pNDdZOvR&%eLmnvh~r~FMrl%3?HHKDEah;Jb+)g1m4bDJ1XlLV z0xZ}O-Om)t%o)=bZN`p%_bB>lCY~^i&7xbr2yN}c36gf=?}5!fwb1UHrP(C;bDae>H=etf&07e@@#uwhX!=a$-v z&tGD4g|ST5vRl#G@!PeN;-7>S7Q+*QsQ2f6u^c+&*8mV)>K`-$T&Eq!Q#XekC1d*Z z5X&6`^cP`zg-rUqTN&)s5#ydy1U$TxEt)NMkj=dI3p8nTu8SToK5cfO&Xb^@(gV09 zfl+b1%ynq@DCweiYmBjG!8tce$a-CuJ)MEd3a+q!fe5bETuz~l@CqH z+KCy1#1=;35Vl%bb8;uoH!D7|US^su)J1KPH&c;c0XC$#QLfo6(m7(avrJlNoD52> z`styD#){w%6Lj`gRVud{P(YLipr}$d4#>jyk`_`S!bNKG{oA6>-2o{u52aEjn;fyK zm?yXr9vI?!%G8X++wl$p@HrP&Q`PBPdVq#kr+9almy_Cf`c0@sWp^*F47pWg5m)nHr zfX0GTvg`y|qdQYON9jA{(M=f}u{PuNXGp%%6kZdZ3(^|LXmyKxAYkua^uS=;E*xt_ zknneftS1SwGkLZM!vKZU1qvS?lDJ(oIu?;ng5A_xE=x2;NcAltY$a_JVnIv#XEz z^;9FC&>o+3KMiVHch2~tk&)h$QbAe3hs9xE*Um}0_P!TiP`0R~F?+Qx8GMYA4wGrz z{yy<>1EBm|cINGHFqM#r0mv)9=p*i*C~(IuXeSlsd$GHe274(L7&KIdnxM2$eHLw( zur8OrS)CHtd0BCU8zu>UURMWT@v_miy9RM=j*(;mM%j0c*cM>afJ6oegc|CrP|$ID zD%#DMua zX7!BnuobAhG`ibKw=HmSZqw10S(6Y}KR*T1%>Y z<6uqs6w+T6i1*z)Z>sX)(batFS-PQ8B4NVsXk*Rj*_X|^W$Gv0^%i9`6ijLjhXh|k zJoR}lwxncEw-B;j1B`uYIc#j(6`5z?BNI0XOZGm-C^XF!`91K&72s|y>WE*0ZL-iY z!+BNJ&xY_KE$Vhw$Hew8I7#WrO}I9!hC{lrdY9myAY^MC2YwGcDP^h>UP5oNm)MwO zWRRX6p|&XsQ*`Ub@&oG;4zwj`?c(+C5EX&uAgV;zYG(j}+AibG;4{#*+JLOhi=YUc zFkb|5$s2_eIy#^Res}wkD3c;90knkcZybwg+~Ys<{5Aki?%~`(n-jT zmu{I?z&q~>{Je#$E?kyfx57Mgb1~(PqXL3@=|TEgpRvKJ6syVy+jpBnKtks}UU0Xr z>-SnF2RT+E+U1yLP)(p440JmLbd!3`&RvFy@jU-zx}qqS$Osg=?X}2t*$cix9bxEZ z*e?V*5)s;P=~v?@yP+_;9+vE&xaJ z#uF|>n%Tadqc2LV040e8nlvzJjE6NjGCloXr~yMhD3g6k-v zoSw<-yx9#(Gw>1LzIFmu>&m_QQ!RrxP{L;}#c9EP5dw&k8kSDGrF^%k*)||qJ4GgD zW{8G@DWal6bu!`m6!TF!=#9H+c72KTcn(WimQ18q2riOC16V{vETkH3>HE&mRnLW3 z4&zT!OZGdgb4mzN?SW(s8R~+Xh=!xsA46%I)7}<@n@>3+nuq$6( z^3th22Lr@m28v^mObhz$LQ;L&b*uxF+0RQ{et_$CAUKxw!fAZ`)O<~Tj@p~Nozm}rIpAnb1+5+#CP5 zi?g@eWzLSJth_t|Jq+=|)C0yDXn?I^2p^1=;o})?0idFT(p+k+Hsl3ip@J#Vot19P z!X+&vyAm%?(xeJBw$xOG3P{W_V;$yt3C<4?Ym;mHcG-XwhElIrhotlJ%wr(I9xUGm zaIqXK+{UPLM2bav~8LuWF1iKAQ9+Op)R0+-D+^TC+8-Oy2h{G^UVijM$6 zx=Oii1deZcK88pgre05*H!8~md`~YSswue(5bfJ9sdTIr1PFVMY1}vPz{dhzWm?ig zRLfQkJIv~$G)3WBoQ7%6is2zj<@OJqJaD6oi_A(i#qDky1JQ164lvq{iVkkV4@bOF z^$IMzRj}VvUGsT5)Zb%U?bvHe<-UuSM9&G4409haz>pwf$zfLQ1(%~MsyiLQd;&`n zIzkqevex&#?0zh=r#%?=P7)JXLaEGy*76?buLzIpEW>^oPNz_sIO@8G9QccL)ie#20O7aL z9taT4KR6~dIW&);`&b!n&e^(ApMY6wO+4lhL6$eEPnaEE&^w{!YmS+sJ8RP$&hv3r`c(bH_mSIZLViiMtjK#SEu?ipi{p}uHe(sW)c{C(((%1$r2DuD90G;2wa_NTZ&r6v01+uFTBb|lPX#q^m< zMIKNq)d{1c=FpfJ?5H&u4@ZcUlzghesYSm_L#p(YnJ%FcTFrvzf@rgND=B7y$RzBG z+Ae`}oZzX%e)mx)Mr8yp2CbG3IS^Peg{Iz1EG1joZce@9Vr0>uEsQZPIlUqD8`$Q> z9hR{i;gfre(+&q+ey)e-=7r=UtIBXRi&#-ps;YUS^`lu=lA7oYW)O>s2PFmc;d`9x z$WGzzPC~#z*##l%1|x4<69ikZHg|_d0x!#$gGD1{O$?)jV-IGsK>Fv?1&N|tm7^?) zBB`cA{hlk3N{+Br%ULsJfVeLhJFSvOdkpBd85rwc1zBGdrI(?1$`xj035=S~R_+#Vu;L){xf(`Q za&4;!Gv}YE8JPm@&z8MA(%WiCbn;|@a0ZIiq=?t1`p!K8;Q_wTS3(2xM;8x!htY?# zvg&S*7*3HRVAG=5p(7;>qn0{}o50!#xd(5k**;e0xmDsj4kfNu7nOSS9=s^|Ip!{+ zs%YNMh{Nf3nwF&O1JREfP__xNC#kGVNwOAdfzG`0N=YesKIWjbX{))BVD9u@YjXs7 zV9DwMPdCK;N#-W*&YP5%2Dx;5cDD14hBGUK&@Y%6>vY;3nF69TVLU>Nbh4H`Udse!r@h!b6y6|lv+$$zsa;(UhOtk2cx--8D{qBa(7rjE z`WfEwtzE)sD+qVxrkTG6v(TDI(uZ6R2d@3&foT zP!cx}P2=4AST(CwA(9|+PbACUX1Kn!|NUHk)D4>oeDi0@9`?v`T$?_V!Aci$5m!ok7$2&nGgO9Du z-1cvFC4n1f7pZhcIqWloGw@32VPY34IXr{9KFJ!h7%q|6v}qPtIqg+5w?c9S#`Qml zc~FExRDjVGZ%SDOTe3tLeB!-*K0Z;aq8pgyX&@1#M-Ch6B#!~$lzB)~BXv%Cm?7{7 zBa>^uHn@Y`N|Zi8Q-`Uy+$4ptu^#eZmr_Mxxl3g6>y25gdn$A*`S&0y-$_b?y!9_U~C_bP&|5w82WFpsD)d> z79ACPomXlREMf|pn||@&Xp(IMRA^|io;r)ODoiyx z;-Ps`qmCH0yyxK&(+$SAFf41^9}L|O8-jIEHq5gGLdVQFG|^^B9Jfy3k5wJyIi zxM#>>OiTv&9JC(Z0j1klT>v;F`mY=q>;>mXM2=BTSdl&4Ag#`|(dRd9cPKRQBl(y#ukb~ou+jBu-B+9EQ;XyLSu4K}=6s;&VvX6}sZ- zc%oM6U!>vSn7#b>tLH28;UMypfXW>2+0aQFPk2*wx0%w|n%471kl_)W;#%_kCfGUK z;x={8YX>mhdhW%_atkIS=wCFbK)&I$V03iPUNC_&q)VMkTnTSaW)O9{z{aL|`NaXP zRPlsbrH~QLUadPGk#yEC9_8PYN@PfPuqUeTC~YF=vh|idHaB%19!ft#5lkpLtmq_@ zPr}JH!D1Cfto|bdO!5HTyYUw=v7>H+Uk#+Zw%FBu7MmEFgty_bR!5%z3ZI?~V)Lu& z%o)xbDb*Bv?{q;y??9Y2op`dqlMHNP4IGhoi_^{u3X-1sk#0<~ss!@p4&614T(r2e z=Y^h!l&>YHGJUSyMdJ*y1D(&*eSRO>xaI!`4NUOXFSh??# zFU<-;+BB|00-W+a%ZR6Fq!c<5Xg!bH`@>H%~UuuvMcHEJeGt+kiR)i_#NYI{) zJTljF_p5zw-052jeNZ{bY+!}G#-Lf3$KsY;JiKW{x47{18L&PG$grUS`qNcJ$dKd8 zUULR)A#TH3<@=HFSETCIF3@mSty0k-8$~Q!xjh7Jw!U_1D5U3z={l^AOIhJ{a`ps) zUKPhW32tdh_GO#{S_wFTI2bBcB?uAQ+~0?)7IF$riwMQhZ5t6p39u?)Y`KWG%vSS~ z4el!B#56o2gUL+HiKB|imlfz35xsjFP#gF_XO8x72uc!CwQg38TZVNGKnHbtT1-QD z6*qpyd)ho}L|-^?@5W=vz&C+X&!9h5;Wc}_#VXIoni2zzL!rbmP^C_pa*IAk%Q<`D za-Q~69;m21om$`v6A(ihTqGZRUko@kYdp)YyP>9+SZhbO4cLbr!L;11Yb$1@Vj~G7 zoUX%NE+%05y|o^MbUdh>(KBx_#6tf$O_hwk}(X zOc5SEk6*7_F0{|Ygiw&5L@ngbb-LKv1Tdw^%^c({=%Hg-^&DsvQFOFk9>ZsLIteTh zdy6lTmopOl(7nfq&y;O4%6LO#JduOG24lK8i_4*z0!y%fHb^WL$<8qXOz1(7T#+Gh z+8C_CL8K!r7`Ji`mAS&g&D?wP0IR@IGcbWYS>iawO=_;S+_ zGeXwDrvz67QNc%&8<|-mJx9qCe)Q4uhi{&~TZ82Wx(1v+6Kys6&51*$wffOVyKyWE zv*WNUWO-ml2e-5_E+K*Ix*WUM!n7DN`v~yYZ2;Ibq@6e3ITQ^FspEap)ws~UU>)xv zb>&N$@1_zLZ8%D&k2?B>SRvxsg+h6@_#KaW`QV{Gz2?x~FW|8n1v=utMF1wHA~QwP z7X5RoMa1^yg&^p<-Bn&cy5Lr4W(%PT$dq0oIG*P{YtBol<;3#}9A8cA>dUzZ>BXm* zn@~9#NxEZVmgQlG7>o{xo(Cc4+3@TIKGWpw;!*;c)UeB#J(&)HWXyx!YtHfk&Q+Wzhny>PE6k8A^)n(ak9@D*TT*WADKzzzNXHt{wV;AM`U&Zv z6c|M6W1gM^Ez+WnLB!X%B=L@4;U?NXa(Bk9$r3;|`1k4P^qql>CgHeFb-=~Msn7~~ zbw}K&im&Ed>U_ku?@K`4ABGv?y ze7~R`@DzKO z4jxM_T3)63qv<4G0fjzICndjIduZJ}COLt9C;}AfgL1P_F(F`wbQK6*Uv|OGoek)3 zyWHVx5K42G{o=wuHpwY1+&XA|h2qYMX!%L*baO<>_TDccsabCRjw;|2Mp9HwD z)}|rg4d=$uWUgw?j+7oNP3@itV9UxG-o%T@_hNVkYeHly)N- zh8VEpl7a!U^N!<3LsNoBaV{$)%HDX} zEWSOgQ3>gPM8cTQ2za1bTqM?=Ak(7CzB&!6)vK;A-ihmM1!Q#8IX39hJJkdhdVoo8 z2PNK7+!lbEGE+yXgmAg&3u-fmnb(NU%J4FQE*l2|GY6+!Rz4Q7p|z_>qbBlgl~1`2 z-d)fQw76?fh`VDpfoph$fP^zMhp~B_3U^p6I>*`_(@owOp0&IY!f1>0=rtjLr($Lc ztIwQYiFSS>T{H#!CpHSFrSz$~)5_7Y9 zycuURNt`W1)+E>wp%Zj!iYbtNZ$Oz;F@5IZ&B2xdZGi-M$Ru`RV@@ifTZIjGVj!Ly zYUvGQ67$LIw)-r_JKUF;Y4@b`J+bsoJ1`Y5MGWJcptHO*+ zta!)qCp_I2qel*wlfDr(@4VaOK9ThVPz`gNaoU+C)dT~G6z>Qk^zXG-Kzz3K3TF3c zq>*Bk3bJMwq++%Pq()WyEi;9I45$G%2@SvXlzaZx!`o*itqTz-t7B2R80 zRZuYBw4NRp6-<=K1qhrMeQwS^;Yr#){Dg8XAurTZgX^O?r}fU?u%)v?jVCq0q)Ztu z(eA3*YP`J&D+0ObuGK{sh4%}2uG`H+A(y>;hUnnLa`FuX5o?TiXCAp~$dHx1%0sa- zJmygq?)8i~8E`aHz5zEaO=74VDCdVS&>ApdSgO#OcW!iAui9v9dx}KwM3{wGEz(4f zAsWI?>A@9XhRJlWn-Z&j3ir6v?qk?Dm*RQD3GSKKzd>GbRD0dGd1z-|R4&P9@kH)Q z89H0_aJMN9PoP#59CT~mHIn4omzu)_prr36s!Z5=+)YZo&iVB2>&$mOO{Qlo(hry| zR=c`rWe#)Fr+L&0ZNqOzra(fLDmx3;eljjOHk>_PxMmBud{ADA%XtoA%AI5v*aQvd z@FE<^!dBf+cm!ZP*4&Z|GZqGNtnGkjtTRC!G6Hib62{CtXW%? zGvKFi9+}xu!r!VQUyBLK!Os*1#qjVfdL@A}zCF5g#Taql< zFL*KQxJ+GpqHKJ7F*YroZOZz%k@JuyGcyBU9w=ol1m(lMtz=^F>`Ow`Oy45~+ZTuk z!UyK!R-ZRU&De`UAWO9|LZV6yzz~^to;qR*+jk!g6-vDZCFHsZj?Q=|ek&#i+hS0okI z3Z-}m<~Ahtl*v90npHK1dWV)`_Kx0OJ`z#&v?}XZJ1QD%rBKH4>uGY);l&VA**wJu zC^IPP3$PM5_YKHuTiEAx_c)%$U26N*i~`<$^?;k(w^&2yL#DdA?3YkuhhsR(y11m!`)Smc);_Doaxw?6 z9>XkfsBmk)aa5nHy{R^g-tdI2ts!f^gHy4+6LrBDwWQh%iw>Wy4R}$}hX`F>LDshd zNCj7{{CmK$?DmEPhsAMEL)(-@qF$h|_)ja{-WfbFIxSq^WM1B$cgV@UfW@&NK{Hmy z;S|p(g?UxW(bF8Fh`e1sp!c*c`L!h9H#>RX=aUf#6c|orw%@PBEUO}SIJz}T#LM8A zdfO~~8GAz2ZALpRig|WD^$5_y_KF~dEZGlumlK+$rKUvf;|IB^s2<+xp4kf)HfH^pI1L!6r()o(*#oZeZ-`w16Mt=GL(W>xNkU1|?A_|U@< zUGQ>QQuU&2X9nhSq!vU9{GCP`>zYp%!vH5pRcA4Y%SmP5m#R+?NqUY!?aK0KTf${1 ztRmM*P3|K?$7-3$5PR0qm+kYHC6stP+&zLoVNEhl+#H*JI<^P4i^F;{lH$F|6BRhs z5u5tq#+nR5O^KHlJS%F48KZeh_J)=QF-Oif#^ymY?c0|T;YD%ia7`dbkyaVO3|h-K zl3z~UH!_TT!?6dbz^d7$hes(K^T(Ll95v2{gf{Lt_Xy0Ok$B(d@($z>;g8QhQ zc$;c-KDI=e)lq;VgOb^hRXj}=Kjzy2H=-+t;@U&}s z#`%B{`*(>(`=SO#U7s9Q_df0LA&c!-ygt`dC=MtI+xDdM%ua;l&`%6JU8s;=J}H>=_t4yCFqS9gn)+n-*dFSAH!nx7|-W^NjAUwcmCS_Ng% zi#%BqSQUroocT8LJS9O(K474C|k<`#9+@p27Ml1cajpxmr#SY zk5EQF^@#>-$&bvRJ-v-gc3Jog>*RyWL|3;JWA^LR#Hd^RTZI0$U#fH(U%AV#tJ#|jfWl+9VLC_ z9$@}2Wj@;rbK&iong@17XT(6zVKd8uUB!fbSF=me=-HaO1?Z)d;`gl;Q@&-^Dl$>Z z+uul?`ENsmP8|2qi{-^zbqCR~k(EEG6bRs=L)5|Ca$(DT5-I}uVH_}q@jq%4EZ1kABs=U~y4QUPYalx0l3xdN+nvA&D8f%9VISL zl>V@{BZ0UuzH0HB2%s^M_FB8Sq2X~#216eOP=bYKj3|V($W7m#*}Qp%+zOLi8S7Fg z)@RF`H$AH=rpV3*#{nYYMzM|+xe^Lc60o_X9<7Hc2ZTv8xao+BCvo)Ql!HzdP4l+^ z?FEgYl?&qm*{*7|EMH$@)t*sPGAX*+tVFK~@q7~gRFx745QL5_u~T4+Dmvm5!4--X z-Z$SaIfXh$p0@yYYs@o|4V?BemExkzdB*K^6EMLhpO?*{YI$KYiWnTd3MZZ-D5E5} znkMqiNN`%rFMGI<$fz|T@Qf7v({d%`8Q_uNyKj4GD#*x87O`E@VQ*R_I8Kf3jG) z-Sx3-iR-MHPpzt9@%AWpH2j_1cr(FQ(N2b!+!`nsFm;So?-#k>!-+JPSiv0(T&ruP zGTe+aou}I9JETlcd*dEz)3!qT*F-?Tlvg`K4V1Ns?_}-N86y}>eO^BeFK<}e$#K~A z(Gol2&8m@QAy0{TqOSO0Da!0`*yP5(QdyV34&zOb(Cm??8Q-}%_IYNr5VKz13c%j@ z;7-8D?JTw;DK=j7?&-G!)zh3)h+95hjocx_7E60&Q+dhw-q>dCN+jv;f$;I%zZDRW(IMdyoSvCcOMAH=N0 zB@(OLo;F?^BsMsWki=wOxVd0cAPItk0-6pR&kY@E;>#wvo(Vvu|VYVx+h_MJd%DeB7@q(3#Zl< z7{m@6jcCNN1j3UGrO9w_Y?ZlX#8wFs=+I!&q~TV|Bo2{u)kY@t$vAt~&Yl1<78L4} zceS>pUNm>YoNaTF>=&>VF-0l|vf4MHUahT69%5-JH(DwT#H2ENIfe1*vgxHNvs^@& zc{AF4bqQL=J!F){YNoR^yWnD$ndZYW6d+95ZoxZLd@S+Dc#4FY$KJH=Nj?Tm2=iA= z1umWs1-v@WAJw++ywyQzZL4@=HLYR7$WA0T)KgRDn3##BDyu*uS09utY!N-PJ(hLd zx)i+Ami+{NMOi&5G*Mad3oY$LBO!nl_v;(4a?^;jEw1{Aqcj~p5@++s7S*weW7eIb z?Vm6~Y(xsvveE|93!#zh5mB9n$9YH0&U+@_?e=ps3*9F+YFOLRMt6qvo5yIygyF^; z;fGe0W_HVLNYcJNZ3vc%s}ib@NL4%)?m$&0HEpV5VD+B&fwe}K^pl#Fa4B}Sb`f!f zDE!thGwNnmccH>)t2}8mC7*T*zaY^Ea`KoNnU1T$!WKAtfj~q}>@7pFL)8`(Z20;= zRS^XAEhBeqEK>_#WNjfO57rh0P`mE>sjD4niX_jZ$=osEu=$#7WqQOCh^Sq09NXL7 zmAj#dq6(&WzQlH_=yM6(RiJsxzh$+$m^w(?b56aT*h*TE_Tw@)(=PD|QOB%xtjQi< zPEFES(lj>f`WKjxGnv^QwKY`bfN0FYY#wI@^+%uHlmlSNl)2?io*QCszOKR23vFmr7d3ACf-G+UFgd4uE4)mu>y7h^3af=`|ELsz=7 z_q6Y#RKaqoLZ_g~A!s{ZJ?}%Pb9xzBUpQXHZxn(t!$Xngj9#;XHd*+Ak#Z8N0_I8S zdkVK%BKMG>LKQ;7XR_T0Gmn7#So)_k%81-Bj@@!Idn5?SK;;^i8hjbg4Z%z0Go=}c z31KK_>&YtRiYOY(y0%l=QwfV5S*4uR!Haz0n4vKak}R1l8^ARQq(cj;OOJ98v0C=f z>XJDKixP72UX}-D(_KfTdJ?!%29xJiaFT>VSMvjc&xg%1Q@ zpOe;jhl(9QiiYXx73*u3G4}@e`&G?KD*XA4r9OF?LnQ(@Ucpq97H6B45-P+XqKw>Z zmSzvRnpkJ8xng$vomyTFjtjBh4~(^_O&o%w%gI9BJa3LJ=(J=TcX`pcn1-fXScxN; z+<21*wMfo<>UO?hG^&L}Dh`W-0)Ix@?arbml96ANI{3BxR(p>Tg6*bPue0XtB7sUO z#$dV~UXYXMuzJsLSXx$(ite*{fMy63U1eBXU9-hqf;+`MNDHB8aS0B=rMM=zwm@-r zrvZvf@BqOnTHGxJDN<gNnUJee=(}>%rRc& zVkwLO;~3x}fJT{N!iiX(1y6SCL~O`q_3PL;H}y$N+fZKx7l-GqEYG3;I5)&MO@%n3 zQI4lNg`#v*Z@S)BDnx$une{LhwGM=`zoz(HzARf9dNIwY{f@ns;ygQ-Nw_P8Jt(HM zI}CX4Z^cyjEU#xlEoarLmnL;xGVD)_+p%p%){Ofl@b`O-w70e#T{GCJSl zKTls)!Qtg^D~oZVThgsryRoa=4HTANB5K8Q(+L52>WJ(JNmJMinO`0J&3D zF3G!m6uKUWM4c+0P3lc3GbU^q_K|wa$LXiNOtX5Y+{!8^|JI*!4uKJX>UfwUxzOoZ zXpC}~AhOC4*RvUso(!!4@jUr#jl;mGJgA8aT^j;}cp8TX;sI58SQBBm(4buB^jJ@8 zy@oDY7%`HwEgOljR&_92H)^M!O3$_5(pcAVL{Ve%_2AaVpnPEV)883&sapdRR`@1_ z9rUU}h=gTpJeVnq*-F6_0{!&-*_;fj$Iy6IqdD945SUm`$d<5!i3hM6MKED4>;M8| z1TcD8=;SUi8rM_UvjzsNZ%!syW8p-qx`T2s2$TRw01b-uV9|z|BGW5T#u#vO&XZCb zfr7w_Y$a z3I$BTpemu{2o|;GYHKw(u4AL45GK{J$`huCKvhGjx`6S(2ACFH5M;PT>Wl(oDxlQp z2ABXe)g5g%!R^jBm5!)#Z>}Ln01@QTpnOm{%yO$NG2i9*i{8S}1NEEkcbeAcp8cx4%)F$j`jbkmSmyguK)O3PkV0_~& z#%RG#RS&%+l&*3QZV>8Ff};o>Ef@os@kJ17mYC5SgaTq#es%L(i**f^CNCj(5HB{S!-b`koavZ zf?dF*EeNOqm<5MCr2+v5H*^xLL!cV#K*ST;5SVfd{j77s`V*YeQ&2HaOX6&Q+JEgj zu)^Bd8Ug}f=&W&_8y&&uTn`BPsWP7`4~(j5v!>IS(nB=ovv65MtWBJsAk@%qO$P!c zw}a<$-BaDo)}ZLTr@(Bfch-RNtWEV0TOb777z$Uj#slTE7&oV&QnpN=t}IwFn|X?g z3XH?msOxE=NDwcKWDCC&F0|G5G{6r4%Cd}tLqOI%D5Wjsb_j-G#B3!!#TxPiKJsLw zj)yfZoCJelA+Kbih6%ydVW#a67{N3hviT|27#IYk7X?M2cu)XL4K%qezMaJ|J!XqM zPOtuH5ilbt1eF3aMr(Igc@TFRV>mn0(;Gmkj_RG(#MXl3h=e6H!TJ;9@obTLsv}K2 zOuMu&s*WHosH%OVr(j$36D#sSQ4lmV{)sy}bD+8I^%%YSCvrmab~beB#h{X5B-T{z zq>fL(UDn9WNQJqaluUO@(V6>qjRcz!K`OBjK)1-1%)!3NzVgup>#Y1 zwv4xc`7GE-9miD9oGs2SzI7q=T)_;gb_Fg3YS4>$qGJe5)tS7?g#=AMlj{gKhM8)t zSaZSj8erh&Bxri7i;iPFGS)$39a)bt?Xorn)<02HbL|@36xg8ST)oBFZsG_Mf@xu@ zw=`A~CZ6(8k7~3}uY6+cnl{9i+LpCIbB@_42K2pGq_ooL=JzO{yS zvuW~Zn2s~OCvRKAiZz&igvN!1^C^33&=j};iUZ1mZ#L^_84n}%tm#tQ(%3dNo7Ri( zOo28!5Ugk{6-TZx>7jgKE%A*>)tDLghp0Ksuh}$z6ib zgb5u7kd7ma2xGRcY7c63hiaiw+16CA42daUY8!>W4maKvd;9ad!# z(#gMnIHXrF-HYydVBA*APZ6)x85F0(MK3{sl^^-rd)YasASSQ|e#NR_ceP_U8Ga(# z1K&LDzaxTiiAIhJif+YIqu*p)i)U3{Qd7`U1DpD!b_?+ybcCfxjB(~ESPOAH@C!nB z*m%P!^R@tbQ=%p>qH##2Fi!Sj|8qpB9jqskN>o|rQ-F-wB~hNyb)M92&b-uU61$k~ zn~3T8)88coS5taI@%sHtrb=HCwzpSi%Q>4Tki&9mAE7-@*52NR;(wnx9egXi(VL!R zfjAED7-32N?~|Smh5xD51&bzYylk&ZPF!qBQbUbLBpwo+Z*s5QXaTua2~EoP%8>uu zg(9jM%J+XuXGb`L&#K~L%akWsC++>7vxJcEvb$Zk$Xi;;g3KgB*f!D`Nz@3t8`kym zKSWqGL0EZl>IGSpIBXe|Uesdnzut2tqmn{~So~;u4e5R{WA|Mcs@41o)z8cPx@UAN zBlEJ&!bvc}SoWM5Mk7kiz>(x|Mq8r=6}PJMnyj~&&l`mp{+x&mEoyV!w4i8wR=&SB z9qw3HT6-wb_toW>^KVodoI*uR=D3EmL$;c6h67KDNJ=J$>TWfufZbF(rHlO#J4gcD zw3JO)lsME0e3iEINl;11j#BCIzjyWP>OjHZ4u1yU2IdZV_SeOW#e6uSc+KsmkNH&X zzXwJ?$YHfWbbd6(?9Q*MN{;n!|IvK@0*zgFri=|3p<+MVoq;y$TJ{G#sQ&%2Q(ua4 zFiyqn5AH=RA*iv9L<-(xx{T2+;a@tuT(j7d)^r)gLl<%$%Vqae>Tvqkh9WDoy^!4# zu>N*!(GTuY+Z37cN-IE0I3bio@;?ju6rv^bXxws9OjdjH!qeeXfmHS2;}M~?Z`Iok zKgwGMwSu}F^F+xdnuV(EstY8gT|FghEg6aY0d>X%V3%Qv^o~ao$f!m+@-42?-@jG| z84qp@poUU;4>~TRlh01N!f1dG^0ct|jW`EmRE(GuyPDDPf01f9{-6U1Y7Tk;o}eTx z)1wRPbo`Sv>MUi>@FN|GQcpwAoVS)(S2=;b#rK>u;Z7k{0HI7cGd(t14ECjG#(}8c zRCH?{Ej-F2dU5E`}6vT*7~|y%^<@44;sh2FvG5= zkuE^G!RwX325xd>;3#}eI-lNgfh~v~)Ko^gYluvOrrjEE* zt|M~jABmr>5&!N;fZ@TSuW=Iv##e#n z#+*wX(J2`k*wtYM`K0?|7Dck7d9A)%cyU?2h3%V%sj7;mO56l_F6YJMCbuS3UDgDr z?aU-*+imoWlFjE?!F5P?d=xMr_4@g}Di$PYw8Csmr$hq7e`^mI*gFz+&r62@=QHe`K=I&B-MOhtQUJ?FK$*B7B}t}} z%xFdiQOZ_$f9U10aCyD5635#%y7Je^R;0JgPqc>H%790x{5rJOrZ^WAA1(b+uDmPctf9y{ASoD;HS&=OxHI_e@ni= zR&hrYp(MbQ7>+|#F8*gaxxnm7d{*_IJlW4O+n)EQ^o~40(XP7vUN5fs=d({Pjn36> zDYBBW9&d7WKSZ+LslB3}noj%AP`bKi|NTS3ZwiVm@2-vg8uNI(yWRVejiwS6111q= zylkm>f+>5x=5*G2Q@s*}mNuz34bJ`$`}mCMyewV-`1+u~PMP&TQ=0oe&0=2bl)Uhf zE3=}F^WDb0k4{||^0@PH_bF+wM@%MvL~NegjV=iIkj^u3>o%?2Iu{yG^qxJ(yFU_I z!%F2562x4Hrrk*wP`L~}W??nz60q46%->t33BF~CKAUR7&5E$#eTp-%9XX=k**@lg8@Qk?Z@97ZCiqCd)9u3AbY z%k4}Y;`+5Vbk`StQ3ayR15{eedZ575P*gRXIOro5PG!r{uFJeun1iHdTa&6YlYuVD z?|2(~UDYvfWjRfhA?I16>i+x#E|y{J<$eklM^e#`#gdZdQ7luEvZUA5c$sC2L^Gdh zZOBm*w_t{MadtzGtJ!a~d(kzqXDz0|fj#KPkb9(0{w`X^xR?!Vow zNt_kay-&r*G$g(t^qvu-4p-G-h*s6Yza$1PN^SoPMD7dRGVA*Cr?m8&wvaJU(i@v< zu|6~f({4rk8l1$SuD{hi>)fpxWY}p4nulF!u}Q3c?ergkFZtW|aSY#lk&$by)=En3l{v;^QadB{J?g(W5+7+JIhYHk5u!>T6TQjemP}r## zNlVfk8%Yc*5n>ZXNW|lQ%oRzYx@w}D(&D8Flp7G1Z}Dl*^zqdV;bw97kR`oIdYPc# zL&CvO7b2r5Vz1`AZvru{V&I2l2iTh z+kC7`7~MaWlo=oaW$N@$O$?>4?cW(k>R$;{^`$CpI^nZV()ImZf!(7F86CZ}%#t_v zhQdFSurx<6^sdvWDJ;piWGCfJ@@kV_#u81{_cAlgNZ%zn-fVe!%isOEax zObVW3cr79}2+R4;X=uf}af=d{>mF*`Rreu}CG$|$A-*S;Mz8g?jb%AUcxl;{UGOc+ zfV!H*%ED8E;8nl}VOfajW6h)&<0Yer7-?FuiRR8c?m)Xu4x_5vyNNiaF>6bXU|_C-}Aipe=mCTgGO+&-m?zhr43YbDImqX zP8%%|f&7ZM>|CMtEHsjWx~9hHRTwKk{6(}Som1L`2bT=RO2O62sl&-I?RU;*DY0|g zN}Tni0RuM-cI@rc%^$YYf&eLn_LH+@(Rr-E3QZSkW&HO%Dx*!+OgTUHqhmB`-_vBp zi;!Rku$eIdbxB)Fg#yJ$Xd#Eq1eU=|i7t+ke`HWd5TY2J1pq{K} zJV+`bdk%FiA}S~40c0-mARqR8Qi$ih(Kx=R!dUp`u^H*3jDSfEDe$El zZOGbvF^y;6vxv0G8IR0~?AYg4Ii{YA45xm$`(#N|YnfCeexe|kq%?3Zrsmm_hW^{H zC9g`2%~O)FFn`Y{itv+Okv!uD%imPcti=AH`9AcBg{4?2pZYK0w|Kx%+IZ?Y zPyVY1A{O1QKZM;cUX?LFb0pHKNTW%f_H6{leejUQe}zuY?N(!ltw8L$DgY#tICLOla782rDpt=*V&G^|w)`Df(+hFxMT^ zAa)gb8okLUto9>XjFE*n)PL@h>=E*|*q212Sm7cA*jlbi=`TQCUP*US9KP@+v@TM-}za;F|(jto_*|%V?z9rFe-#h?993m|q&A|62?5n~r42MDARb zarCx-Q9ME@Ub=gNG_~g@rWE2%><h zjNI%*Af{PKtvcoCerX_#YXlB7I}3gVDZo#phaZsFc6@8;?*G)Tj71H2;5ocR$2)k$ zuW8-LW$UxTxQ`cbz0)snXVFy~hR$4jqJI*8@GT{tw$fIHbGLHMs69mDO1mP7%5sp&t3fZ3A3#0$;F9?A|Id>TIOR-4XeQ^|9x)~ zp)U0uiQM7`5Kw!|jF1NY0?I`oOTm(O;ol`7$dF2b;Cn{#bLdK8rLmUtD^T^dajJs< z)Xp1O)i~l=u)F0Pq&qy?94elx5)${n=M2M9(s5ceUFnv*yPjJT|KnKB{`0oMHt};n zHWAV&kjlemSW0))L^BCfDy|;hEt2|9;;0*K891VmhUY87TCjh&nh+hhU{NU(xL`Ix zGO6Ro?m@IeWb1fNXtwE1zK>%^lpk5WxfpxAr7-Ue;ay)$j()tP2N%TMHHnq8K zcp!ATr8d|%`s#O{z&GV%K9ATym49>KkzP6Iu#vsR2VmSdIv!?Jmw^3u=0S5lX+}M6 z=Iuo-1RFcBW6v_X?7;dCKb;XCk5k`ARo#ZGGv^_tYohmEFQi6{d)x0m75S&&%V?GW z2~B)f+8w-vjiQw+C^*B5EQ^tsdw#(w-iHvSslTy~@P6ur)8wc03q<&je4$X@ik)!y zfpT`m#vfrsch+yIndAvTPvgib9zj=H1f0z}dC`*9*{*Bs94Z$t-;&11D{5>CNew;S zvW#Fp&O<(p{PC1oQ3SSguoBHRl*+Jh9gymYO)@W_xP1 z9EL~oH`us7)Mv43V#BtI3-JJ_7DaXRE9%to>3z5&{vA1Xc;>}LuBSibB%5T)GaUpn zT3JPDtS9us-OjpO5~H`(#a!-9(Ti7*82Dlf%s2FLE@e!PbBt zB8z@!&e_--p^y|Mg~ZF=7l2{P)aaQn%q29v^d;v39k1ASy~SL3d_+=?%w4K!lcU|# zlocxmm>5EMJXU0sGwXXr{b1H0Tl&lB=`xkqk*|qfrEb)N!rLt$EJ82XeaQd-wcOk- z$)Gj`u;nHor77bhT`H5A3HXCYqyd#@y2MQKsNx3lI%Q$G|*uj3|Y zD0FGUM4DM`d$BZJu6^?&JcPr6N9e^C{Xhj9%NOBY?BxrQ*9j^W86?ifU2l=YhWksF zdcFOdv>!M4`fr;uxevSx?of&Kg_VnQMs?2$5(A*m zVl%{xqG}>y7pcCZy4;muCX%{+Jc;BLh|U$Sz*2oAJ$IA6*T?VtR&6`}Lc2wa#dJWPfp}84}~R)MXWezhqABjcP4% z;E;-oHr&tM_WyXTMD{rhth;chB>R%D+L{iB{V&xqDrd@;m>eP{wbSdT7!E7qU zc#U+e;R87_uHSEQV4OrBy?(2gkJ+c_!ZfFOEM*2CQF|xaNzw6hSJjz31ck=taRnAJ z!-o8IP<~(@sTa6>Wm-uuX~_I6Ac$8+?i}}bNsD?Fo`|l`Q>!MnwzFatI7KFNBX!ND znH8VE>VfaaeljPdqIi%#g6|u{FqA0)jAr&N>ls;7fL71XkY+|Q z&%FF3H8%hig@;SAqMeZaFfbI#5}WtY{sbKH(sk@U3F~_)gwgC}{D9Zhs1de@;DeC9 zNVES1&FQnrVST{<^Kk>yj)?I_0v<$~G?~e-r3|_a6U)a`oNF$prRZcg3-N8NHFnNM zQfLm+i~WFz9x4Ig6Vtmr>`*UGBwA}Imj+Bae0VF*G32<;#(rN?O83Vp)-e#>j?L>) zSnz^%UaF&BrE!54WHml%aeqQD6h+i9QR6LPOv{Tj$*h(hjxeU%B8J5vag>@@5#pv~ zI%lT!dF3F&UXg48TFkiOK}$@X93%-JEb2U|uiI6^Of4PhH>;5;PG{GkFk zS>_iQ3llu}t=?qjY}1yI5l1%^QT42-7^!Cpc{>UZ*foD;e;G~k&o)*a&#+}W-iK=^ zmV_VWH7X>&Ei_nb!|EQSh`r`JcrI;U#?Q*;*NjJYDmk4`!mNKc>)BR_C{k%u7@Vx; zZB~m-P0`f(!offSQ4pFmFX4fGS7A>!kYe_4Vc;m^%U4``L%=m^jLh&cter~O?GKo5#A7Ca^AH&brS9?jM(l-yD>E_y71G-Ur@X1vjyCailm41~(;kHT2OXkCY6!N%*T-U2r8f%v; zlg{i+r7w_p+vb9FG$P#;rS?^{6vL3=J|FrOKoqXviq6aB3g@^i8RiaKIX6bByoIkM zn2tp@Mgn4QIOX_NajEQC)ZXiA-PBcAHt#FXnN|>jMO_G!^i5U1^d!=sOEy$cFqLMg zV2x#;g%;rBicDa=={pm5_mxX)j{3f%lS-#Y(FW0ZF#TNHQ!4{9kfOga2VeR#Dks7c z0;X-3PM$<=wlDi-sQ1%&`iO~+?uefRp%c+FG3~^iw0#aVN6zD=^B2D@Cdd-L;wU@e z#}J1Dqq8vsYd@Ww3yd0OIt0?{P!1|J{^&;jo$Lo;%S&_8M(Mn6jfPfHJ-gRl$F&!@ zb~_I^c;xX+tvo9w@Ot-!MHeM~rJgL6^-m6X3~Eeb(cBP5*|Uj}r56=XE$@hOnRG^V?!qww|zni5x?T-O$3Aq zsM@T#J~60ANS#d1C(;h(Wcepd2bqqn-y4@goEP9NvA%Y}eP)Ed;hf#{XqC!K&24NW zI`eXO65CjmQNrbw>SS2ELcY~!rrE{_T~-@>-BdmB!|YpKhxx(bxDN{RLO+AZUV&YX z?4+26GkzM+_!^1;xW4=v*h_G}70pA5zX2R1f2Y9I&4ShOjWqE>L2~<}Uk{E4;0w|d zNa7*sJ{^^EXW^0?uR98__aS-jMF3F-0@ur}Sp)?F+GKe?CC>|OI#qJCO(gI%oC6PA znaaeq8)Ah%;eV-$Rl-Nk2|?d13hu3Ci!cH|?5(+6dZIY03#jfL9+~O)<>wey+9Ve; z+p+9Sb9cX|AN~&AOUVlOI(nA8r((#YJ~yloR%)g9_SgH0HP3A)BlF6;g zXW}q?t=yN038Fbo;U&da5EEJroi-^;e%)t)N#jkZ<12lwrUlay*-Bsu1=9R(QUf8E z=I7FBA;#|a6(P}%wE1PULjVu-U+ePg{*EKWpbLL?K?H{VEx!4WtTV3yH!)YOl^ zTogZ&D0it{lT)b7J?b__toewMcZjtfJbD_gLx8MzXkbEIgX#|bl`8lfH;c8|%?mqf znmDnNZb9NnkmPF-HmtlI4-xOw@1dlC+dLA|Q_-L}&T9S@zB*ujwC0abQm+7g{vG&9 zo-T+I8;+M{;Occosp9JQPJ1@ql|=<=$(Py1;=g3gC5AVTn5Y$j!j|}o z;(4Uw?KeSeK5ZsW{lCLs4n|K{W)uuh&?zzYTl&HooYY3rhs6F zCw(3>*Rp!DpB8+5FwUeMO-SJ@LB$vISh#%F$EGpq$Y>Wx3OkwMv@cS+x3?Ju%iwM? zC2QFf#K%9*i>yd&ovC$5XLr~I`g%eK4w069mP$5*02X+gg1z{U1c~jomX|{8_z(QV z^jv37I*4e_0rl#3!>P0XzMw?~J&I{yqi!5yZJKU_simb&Z7sv!KbY~n*whNJ=#TEr zYltnsUQER)kMOX0cj3<+< z59IXd5tXU*6uPK*R_PNVCmpJVoQ~o}H{FABWSUZ4t}Nf+^NhgqkRUVVd{A==WQu2yaJ8-i?yuJv!b?DWYg+U2WHp%o&`N+%A9cZEl?1YXT&4eKSx4=$~@oJ9)wMea&WL z{pcm|*V#AVz#u4IRC%5LT8>kUQ4fu=Va&y3I|lQ4B`7Yug3DfK`Zx?{;M<7x2SZ|= z(6nanyMcAHxgO^|&S)+sZ>-0aF6>HCVg@FoX05PZQvISiJ_&`smK-XXw=kqCbYc1OrC8 zd_>zF)rtk@s_aX9%y!TN+!)RlQqRKdK0FPMN*Nsmck`@wtobX4auO20{!_m75-*jQ zEv(!<;*3c=bip^h)paM!UJ$a%6J5Vb=FaCd;X9H8^bR&9~;GZ8KJ3cW_FKP$PO8 zd;{MzH&0wGJ&%=?!}81-4}{hVEEYd2RhhK`1gh-rvUu*S|LkL@7o_?sJ>cCwqa%0H z%*i;mdtyxzw8hMHzk<3uiqDd(R*cJY``{KY|H5i`^1QL(s^VJ=jW7`R;-x`y-AA?B*4_t~m$eUiiLoz#=_YfZ7aFoG-+O$+m1H`zLHZjY~p=5J!*r*Gyz- zz0sSThU1Q2+USKO2t{V*Rii@Z^Mr-bfUQ7-I0xFPf|V)x#-B%@=ok=u3O0o$sMZ=H ze>#0GXIe*HtZ|_~L0w*~(Uvo^(wK$tFA-zsGjun1W~w)pOj*35xD{)3OaYLu(8TWQ%P3B75F zJ*q;MJA?B#e!)^EZh$IEt=>k~A=$5DU*7-X6GzRw1K;+j4m)R`ivQ6TpipL2{cojc ztrsLnghMH7*X>3X82P6;G}`w~);Uix=f7TIw|g2hG4gLTc&|RV;*C->H94?7E9W5o zd+ReclFds!91LE~00L0_*hiJ7 zsEhZ4<4Tf{p90N@U!S3wgR~qBX?tTh@x9lY<(so_-p;^cGgfi+Xu3V5R9tt^L4N0t zP(w3vrTtT)Ye7d?Li8f-YPGv#Ue1nrFoef1N6_nO{zAkH%EO)3kfW3uXE^E)PDPn~ z{ta~erGQS&cWkl*#!^P9urUnx;;}wxVdLr8QqW7SA4IT4tU9-B9j7B%4q{-&1E(@u zyykB4`W`nYaRyE|FwxITrOTMT#&Y0p7uM3>qfZ48&YieQrovBJus$?n%M)2K>f^P? zAvU)t%HK{k4z_&8@?_TQ9BDjp!#Kdg`;Utz9wK}1pu}3^WZIG}EF7K|spqn4PZJ+6 zoIj@HIs2=?mJPB{QMO%8{K!-srKWzGkMpVq&Lj3B=ASr>a&K>^sQKa*EbBRcFXIdR z&Mmy5AF=HF8yvhd$uE;SPVn*Sv;Hb4v6Y)0&SY!D{OW|KiS#O zeq644U%8sAaD-iu#-(w!+h*JFvLWkp7!v2-&xA7_LAE!l0iMf$3HruHsNKn526L;FiqWCz|S^s-OF+4_hwF3Qn@Z7Jgm(O)cygc zJwHp(9#a;qmih521UM24)QkNf>3l{zgsYS_GG*giOR3N%lVums`0Ir1^>zTZwoV(} zgyQbg#KH&%Q=+<)HsUJlRK_f80yBWc8BHP5igZS?Z{lj_&0F6;Z|I$W=`(75UG3nk z%)0HXuGa0os@U$NvSg^``1zyG-F>>3TzomF&Jvfn0Z30d5!nyW#0P{60}jgLd9C)o z5Wf9Ac_?-9f;+zXAtZIAaH|hYuI~MhvDZ$ERo_eHH_K;-i^Ra3Y4BHS9bn8rXTAyP z&y;yAcOC_N(%qS=pHU-9J)GD^FQYujH*k3fm694s?|=>EDHSuT_*hiCf8$M#UvvFU z4#yr#<;6*LC_0rc{{Ti@hRBOl;qogC3Pk?-9{YWWGD@)zkWVVRdiThfE}z&|y+5h1 zMoC7bn@m9nnACoB;ug1BSonTVyLsWOj$0Aq%`Rb0E^(j1Drv9# z&S`z%%9C9f|9rtchuEgWw)ZsYPX-ebZnbXH_1_XnKRld(u~>MP+TV@jt`hcWEEe9% zXGl9u>$vO)7e{IH{`8A;eXFYZY0?$udmL4Mi~CT)iP|+D9^26up3L{odEh80X0rXd zLeul|3UNqK7agX~_P*!?3)wevb;^~#g;?3P=qQr~F z%DDt_>ZX@pcwwn;K<#+X#mu$TXl)cC={DZ)hH_I={OZvD7B3i|^@ds5Lj<#D^0+*Y zY~%%82VrL)0^i4ko$1uQp7~=D?)xI!r|GlEl>-yVXJxunZ1m{8Zb!y1OHN-=t0@(M zjz#+4z1+)p#M&0%)Cn=V7avTC(hb{x?3~?`^@{riRcV^MKzkWoLGj#Tm!kh_oZUFB zAE((0U3Tez;wv0zuh}BGB5lm`d>xZFnp=|0s_Wbcd0R2SPLq~>Tth+$h`tG>R0(wl zHBxU^k2~q)HI_y5<2Nr?IIVK&BB=98PC>3pARjtn=BJTae@+WBjOu!6#z+O5LLzu$ z2(E}&*gE{Km9?|^1n~6YuqwxN1|nxBX_mu|~E{VI^D(JZn7v$gX+@9Gd{jZq;ESt4|~ z8PPNOX%Euugba29q%DnBULaGyh=!h4*2@n$h;!HMuSM$idQJuZp@GMA5vOD;(iKNv zB5^8-r9^RP*YV%szX9V`-g)mdCr;Oh4iU4O0_6TG~0?~X@!;J6|1+?OT-G@YN| zNBX1@VIBFj&LL5O*O_O@mP5S2-`~_HLG^-HK+)82mV>S=C0KXwdII+Slc`mJORCTT zNwXcvPTO9P&I|Mgsh`%Ru)1IMW_I_MLf>mFc*RIy!7HYOQ;|a=i%<0|so@f>!jonJ z@v6yohBk$dYC)Z29YmYa7oK0;Fs-DbkL;qWMEEU()}7va#ypGM3zA<#*f;RhS@w-+ zGI8507f$7thoGwkNk9#@<2%WZWj8;@NjuH94SOWSPt`$JSeAMkV7xRb=X_kl zsdLvOm*rf8Hitp^Uhn&MPm2udag%~UVge~21|5l7<`2>UtHOdceL}C{3HLxh=GC5W zw37PpL73VJV8okrhlL6MZRoO8*OrkGg99M*BX80{LXnjjdDDSA`$2GS@3qF&Boc`e zn5Y9DIf%z*Od|t)5P?{kzuh3!;&eG0KVb^~j7!f8h|89#x!if?m`VKU8PjXDgqo;P zdJ$6^g^LwC!y57T)#(hKweD~NH#R-zro`FZ(1Y7jm8dJ`Wm>ZQ1F?;28N7Ey8M(Z9 z2Rjp!I5Ya4^C+o->YHMdiYaj?D&$&}2Kc>JOkE*B>=Z8g>AGBoDVf8bfoI*^ohVi-KjGaE>V z=DZEieQBFNl_C|OqpFefCmy}^OVW|&vX{EZQtnmr@#!DURr}|&eXF!d?N}5+VTIow z@+&L=(eB_XhlJGmjlZ@b{7riGd zzW-#?U9oWy@n%JtnbM_L=yEn(N)PmBDWZut2}7V8ca)-;_TuVw%D)1_WXU-50das2 z??W;i;(vdNNP+6Ck!Tv!PO@SBh$4wiokBHDIXIjKN)IXYQQT~V%1B&!5q^UxYY9`= z51N@D#b`3h$BNor`4BavH62%q&nc2?TRo0i0^skj{5W9vJk8LCf3TZF1uznAZ%C`WU#+N$=Snbj*cBG~_Y3l{#K|CT7Z8n!$oh8q2I z)tmWGyTG6_YcupeDy&4GG(}BLI(_=GYyj{5%C{emiyt6ICtNpwu&BuIX;Ze`6FU** zm%s0CTbe^hIzBKo5|@v9pNz+cS4qw8S%3CX%#pv2nbCAgVSZy8qmoC=QVI?~k#|)n zX2lQK7tXe5RdiPM2_z^d=PL#6z{;bBJpLKgdb2hU+4g2Z^K|G*>By75ehjkxd&7d? zYsoL`_$Na4$K2fJxmb%<^Wag3xP`v@!6&WerRD-I1Zlw=?4@bi-e;C!@-?hOrEv)X<-^N<9s`lVK`C!xQ%tw!b@XhU&Tcp7miXM zWAsz_1Ky5cQ+Dtic(db@Q=!k(%JNt~$+6|t-V!DX3DF?i4kxN;TG=&VTQhBX(4ZdQ z?sJDNJREzSQPmeB>2PLT3C7*8q#M#QDf`jBEFg(OfbM8y6_mfdgPFoW&irN9Rgt=H zIrug9#h_XIa3w1-e9oJRZj`@Z9;(!}!(ky0dvggv2D^4Pec-Ks{M~k|tTL~~AJ-5W z@oPTsC5o8uG1}5IgHSFZgp&Mk1Ea7q)?vnpI6+54&*woYnX9eHmywOiKxT@*wJ6XR z2#(YUcDM}(pZ+U;RxFMm^p*sz+c`sqc7*acKPLQ=Vw6KNhFkR{c-|+stYuXG$z`oN z6PICc>4Xsp$N0R>z*Q9)gvvtv^sg~}*m4GvX?9}5TEbZxOW7deD_>{Fi|c5AOFc(A z3oQF_<*<9^ow!J&M@pqwhW7_5;#r0UgQVKwio}p@&~@>|`!_mbq?5oT4l$#l2!9VE z4o|-{rx{&4$M)pSR9(fuZ#SI(LbT_JNzCODOmusW3C48Phlf8|i>UM|tX@U`sa58z zr7Y-)YlF6ZQ*->h=8MZRAUATQxB0`vJHSNIW_Sl@>6BvOrqL$N-EPy|hngiFZ^}6? z)Ah7ObZFfmR)S&bg!o-tC7+5Jo4%J(Ph z#lR1QyT|Chs_j$~;5aKs$uMGTb0SZGj7Tefl(C*drB>12HFV@QK~K{dZ?t1x4!xCv zUn@VPIk52RnIK_KR!!5)tcjeJLhc8#Wjc9MqF%~Qe1rP`bD-Y&ImBr zufdFDVSiq@&F~8LtPL9U_Dq_6%4PD~O_JH`)To9b925@vGuBaq?yX*GUh-zeof^5% z^4?FvB(mIP-HlGUNKr=X95O}w-Ji`>-t;eke5~Ww2T_@ZP~Kj9W-`I)*Qsu1V@#6; z<3guN+i2G7XiQ#;x}PU2iZ7ItOuJu=ER>aP>7E%6t%254^4O@LnxC;24&M2-!p#%B6ca5gQNDU zxQS@(RC>Xxr;8*f#T-5 zvUjYof%*Eh|Jf5G}St^Q}LP=cq%N(bghtgu{--eAQ7YA z>C1(koP`jLyh3Zz62g0De}t1V6SjWbnOCREyeWKXlPB*|plf=-sJ%^Ai4&IF1|Ze1 zGArcy$QwQf`DODo9?k#wXc-=ql!+w$)e3OeQE7#mf0_!L}{#RE}>|;awP$b}4 zP5$ZAhqfVX-8AapT@EP8){TCM->Oq$; z{5(5bCWz|9nS?iyaHWQo<1x|pW7MyHG%TH6VwT88X`Cb2pr_15`5^smlV;@>8TB8x*ofkAKG%4nX$nei*AD6i5Cpua`xmMn!B4aF zXUm*3w&7)QTmB2_o^8EU0-%aB@@K8BIyOg#T3+g~KXKeI!v=MG3m?P8a}E3jxr96} zam$7`hwr-;=jPP9t3xRkaUARvdIW0D`}^Q*|MF&jn+A=(u4}}~?fqu4g~!W5KddN? zCHN7nyb7hh!;nWSb8&`R!|FasNrNO`G?Qp73fhrD{9WJ^DDJkFyk}(3$|P~(_%O$W zjnV$m3ElCYCqNfJAM`?wZP%X|exE(}#BCX!hmlEmc|CXCPPFcdDR*Rt)fldWL4Y^2 zu#NM6o`l^thnRcW>y$w*0n=YJ!Pw8ZXQPSc4JH9i#&a&U)+SVw^q!T5-R#4Vhat6m zk|aoh=0);vRIvnQx6v4{0rO-3pZ!1kSbv`6n44v*P2HarEHfhc#AXi{{ta9>NTUb6 zu;N3k0+eAHxwk2kcznw0J&7d*jS$@BK|(QAH^nl#J5KEVIn{w1UX*|3v$w=is-hGL z4Xo@KBF-qJCV`;VHSgCp64DE+yo0xtwnFQylo6xYT$Svkl+1>6h{9@_V>J#g;`zF) z-w)GQx8}z7(-#E4bEg(X_`QP01^IBl#fM1P(Cw6%qK6~X>s#qn3oa~!AVFu_%PoRJ zToPqi?qk_c_nFP3#9xEB6m1=u);6+VTrV2}i$R9r9`0j-!J9Q5X!Y;j-$;8}l0Q)R z*n1is%$%(+kfkRJy`{`~a!7OJI-LF0(TgS$vvwTSWc!dk-LjN~%QP$JHHMw&lo9{D znG(YNVdVtt^5}QmqB;jC0*w=~t(3z%N$k&BaZ82WA$2R(Q=>`!W1(?pLB((73Zk%4 zqdYP#|JwVWDV_UFe>C_)8Tiu^)qvn2ClSutV59Z0KfJWxZjY z3}3!D0VB{JpXQR#iBahTbu1$Wah(*7w8`i>45R7)=<&JGLxU##X6&2p=EKZInp$e+ zR&oU}3ZjipZCTBRPKGUsYHa3Uv2WO6`&6(U6c0d=?;&Yq|7=pbv-?@riP_s!0C$@?a_9`{O>Uv~K(=hry7Zc+_ z_Ipz!Usq8K&rW97xm?{Od*c_DM^-C)1$4pOiW+0Ofs%ux+t2{hLkmI2`>)*LDbq?EcGpn46ay1HuCM$)BZJLfs(On*v;=HR>eD1`` zZfo*9O;e>}d8s#BljC~fgGE?|sRm1Kp8OPlqMEF`2wtFj!ki!Cnrh7G>!RqE{L3;w zu`2QZ17tv(zlz$r*YoZ&J%Ve$H+|c9HG9b zq~N;jtp;>kSW>ce)HBFa&~p6Zj&(~y7=WOxOM?*Qhw$arYVz$1*i9mLQj2BGKgFpL zEz20H{&J3uSXzeR2GcjT1%$z36FV7})2p)jn$>P@!2G5NNj`&UO6X!?-6|7=B7BU- zb+FP`4g)V7KMC$pz_(T2oC1eUlD?CxUz}3P9*js}O zM~KQgR-me01v~-g>EPyOIy+Mm`jqj3;7^osr_do`V}~oJ$#A6{m+2J~es|$|L!y)0 zvj!j^tZdUni#4td#NLn-$Yu~WFq6&E?x|^x^*&PxXzJoyt8VVQQRwG{k*+Tag+SX| zgA}ZUZ{we0XAls3v1r<2ytHF*4tk8urbs~p8QOdxy&Tp(=E1eOTC@oD@*S~RQq4&$ z`{Oxc_7g!%iv)`fWyiU0mf$-AW~uOA#dQaa38E5r8|INKlgU!IG}V@Uk&6+Yoa1`I z#C^7F^tu|}=qIiGoz2+g3MZ}F$sFd~f^bHmuW#ty!}=LzO>@_*GW|Oza%*rEgyXGe z6^@|O^jCXU!3YanYO+Z_F%bmvU?Rh57s=QHObC)kukJ86ZMokEqV;Yx=er-qdcB0~ z1->#aw1fk3FG%Y;2Tgfp_pi@{v>XLPg(LHqG29h95_iM`sik2u58;N8>dZPAA z4YjHueqxhgW5wRDK#XI8jH`lvmws~{yIfLdG~C|CCEuZ#yfkqLc3c{H0o9DFz;D#Y zh)WxSLR)3lw}c4(YUD6IUsC~3ywLoW+h*{0=L*9wCqSBgBO)EMGhjgAN!~SKvQ`U$ zDM4HI@}eLd*`DTrB3_}$gbZkBn8Y@ubQMBD#?0=-J@K=obYXk9ho1c*V^*xi03}bb zk&zQKkFXNi$O|$;b~t@9E_4d>GqqMKNH4!t*6iJ8N7TWAm6~AHzD!EuY;7S%AF~oN z0Ke zNZ7HKmep`rZQ4M1645}U>i~p0R16eZU@TWloGjKQ zMRC|AXf+X7c8>?z1MaCE+@`k9?|q=Si}1A0%n`9&z}I4LEx_-bfyK*`;UAV#&9HcM zfR9VB(*T)V*5M)&92TkE-&YcNvKiQnP#6`U&CA;)n3{A!VQ)tHsN;*SteBrzCxNce z8fzX8fvB*;pi@qX7WYt$CL1NO;{gyT&cb*u1cwH#Wb7AI#Trku5Lx(T;maZB9UrMy#$a;IiAw^y$|_!+!o&~-J*KEq+Qcgz zmL|4032|Bh6&Pz@GX1uUsi#Jjk?T+zgiX|UwxvEey<@vVYcLUVX+@^s!115F&Uxf$f-?^PV^op8x%)A6NJs*L^6kP1qQ)RW0<1)Iux2X^Vxmo z(j!EKu~=g4Sl}XhYqm+o4D3gRqB<0BAvJj6_i6a(Zft#vZ!vPoEXH>7ljJzeQS#o{ zRbkOjM%A>L4!Gh%;2(&bjjg@}m`-w^64-AXr63P}OL&8o8rw@WEpy2Oc!X7`%2Cq+ zFuUxzE~!HrnuwH@;BsI_p<)5XYT`idoW;Mzsag~6)Oxv-enZZNRqr)aSd(E0-znHe zlsRz%bcQolwM=^1R^83`^$V5CC)H{6kS)cu#MD}1_3U8#&EBxv)Gm&Mz;{uPg|f`& zm>(-5Zx^M!#s&mtL9d&@ehVZXOm+6r0dw1}B}iTM0W4fl6J{@(Fmg@=7V|N(!{=?5 z?~r9dfR@Lpx7#Q66f%uMlHsy3-j<4rx!OzNz%cAvId+SgyPXM6nYoQTZ!#FTT0gd} zVfDeO#2NXM+O2B6qqq&y|GWADsVe{#GT{CpM0BnQ-RlTR=q&4q6M2=z-&-{ zyJ{6+O45WBAnoL~pnA8dxl8KsE|>!4mwpMnCXG$uO|f1e+^LX(Q?mxhR|RM_44|67d6R9H%N(3e zMWo9lTD`0y>r_H3S(Gp5(H4<}M#FKgSXoe(x!Rcft%PWTd=MOZJ0`JmsG0(vHM0f~ z7ceXE@cQv2IwA@Aj6HL0w|?(Yge%IYH^x~{+UpbmRT`fH9dhwol!`56XdAY$scJfL z9o?7Jr2t!F`j;i;0$37^s#UZJiRXnjE#4<<^wWewX%Z~@V}NAax^-n8GFLko$P7uY zx|;56qY!Fx(Kn8RCDtZD!nc~hSQ8M&4W!_xuMQ|tFfIbV3&pggt`-c{?%@G}^C0^U zKcRFfykQkBjkaaVG)Y0i%f%Idl?t^%e7xy?Rt^IvQS)!QS}?p{y=XpRcq9Q4AFl|+ zaos%)Y^GBoN^A0-_8Nc=(H@VbATp9ZfJ(1`PjD32`=m+)P{b-WiU?@$q;{2fDC4|k z_y7ao```fj=06r8oA{Px`}SFfDqI4kev!(({JIy6+Jm`mZ?Dr1>emGaq$b zaNC$+-akWe)-Pn_tJR^03^ka2mmA}Y;Wq{cAF)@W-TSj|L&vCO+q{(-)g~=uMmM3< zTmZ^U1x3F>j!Row@|V{jD=xlU%V@%ba)rg31>hy%HR!lS+_X#-xm90Y6?v{fM<9iL z-qk?8xj@PgS515dQ8auGMWA;i2TP&!l!!)Yju#d<Y0JgVmB-h0Ul=K1Mlpb_t<9O`Ew@j!^4 zOLqD&xkHyt(>H!-;3nfao#tTxZNQ9&IWqEu*sT>44%0i(Y}LrH5=zeLBGYJLjNl_V zh^uY7$K49FIGIKD`p)4Ujz`;asAVlF!t7`xq1L79t zlRim>g~qjhk*RJZD7l75K`&JP;EDb4(tWi zATc^>nZ(W}b_Pds-wJS$2FrUxCx&M5#8FBSnhMTF`XGcma$c@y$KXNp?Rn5$pdDU_|L(*f^?I+x$?@Ey+}9TXNN59 zf$t|hTsI*X3At<{2n(`hv%Sd6COQ$$`J9l`hqELFI|+f=FnVw4uDU-}IlrKmmCqa^ zX@@IQOgcg|KPfO|VepQC%OkoeZGQ2{D1Bk9jw#rjO(;I`<%+yqS0zikGbNR&ff~|X z%aTy-yMgdKfWi@St@A8@%Hbj$U-}G_@Nt`$rb=RR`8i;_vbZB4T8bV4b;Z zlG2U5syd1Uh5?gJ4WhPCV=xSpWPvw@60@*8TZc|PBowP_Nv=heMvbN;H5wb*D*3^b zSN3q$w|61ZOiEucz>Bm?`V0)`q%{`sDt6RbnN^j7ZadG9fmQEP4e>D@K+ja^%9c0~ z1`g{WaJztrJ&g(05%{g72NkqDI4U8XrbxSuZe?i4F78YO;M=)MmfUo*aVI={pymmH z!d^IJT=v(<1gp2u;PUmO1i)^4-0uSW10$i}u$;_@)|!m!zKH=@-*G5VQMFVlu^kb! zB~dR8`xjMZ%GNP>!@8BQ))eAEn$6Y_!Io3im6C@6WPxSQxJ2o(fSkS?lo2$n`OfLy z@cBY&PRCxOl3MW~{hvMEoZ571!g(n?qZRH{gUS05UkHc}JZw~rkYx=2F1W7A+6 z)>4WBm0|Uu&C)$$;8M{O$qB^jr?A_pTSMjFC(=~ZK{5pQD`v^QyQ-N_v}9+Vi81N` z;Go6W-;0zZcg#n&2){0D()sa}NU4=^pzBWT?~A-6HF-9hal>er1<2VPtPtc?oD;r2 z5aPmt*SDouK504FWv+bYT7jjtn7xSSji<=JV39rYhmcfz%)xti>@qZ z<{|XXqO;eott1Pd^*x>rbB*!g(?(gOcs+7#g&?+_zjM>tiRN zpod^JP{LT)e=oHJjyP5hGwolk;L~buI!l%DK78~M=SEz zQmy8iCcsa8pe7l_9q2lm8)x#9p<9H?1Aw}RBN!mc1&m=RgnEG3K>bIdVY(`|zUrFO zieYR{2*}8+nE+X4a*iw>k@VJ-y3-c>v-Z%|zL29f9dDED2qKHZ!`-xKrOl_IZ_w5x zBci!%fwJ)&WO?(axU|LvD0Z*jYa4Sx8i6BFzUYb6N|Drnj*!iYaB;}A6`XSAv*tr4 zuzALmRs*>};gqd3e(-ph(>|^shTTG!vm0w(;>B|@WZ`mJ2?i0~@LBIda3nkhFENlp zo?Es7-1l3NkE%N2VS4QX#dM?<$~rwiX>H}rzNA25j|3&#gMw9ix?>m|K!EK%IJ}K0 zL0;w?0qVBIGX)AUoO<=kcC{Ic<{%&{^t;(*1u=THiIwbd(4g^Ot{8y#kBSEDMl$Tk zUL0FdIb|CM7wAT7x_0#SPXcITgghGU&8QUFS;6FT0*K@IE}CN7ce%!L$b4+tmbt?W zh8&t#C1=4?6K{iQbxB}LTkfc#0G zN9BIT2r&&6zMgB6=S2$dE0E74(2@nn7$GkXEtS$k6)ZXddqIN>h|91Z)}h044%(iC?AugO4AyG1wk4{o?!gT!b6-_eT0B{v!|D-m2ScUJT_s_ zDjM{M9l$3D)a-yRsX^=U-`BE27$a-9iy)QH%0+kgb6I5N!10dt)1hrr*pPV*iVQS) zysiRDrbJeQspoTAs~pHMVUE;sCvGdh4#N=6q#>-nRqRN?EUx36J*a@Ttu)hhh8Ps$Kg~#BM8R|>7hbh=@ z#0G-aiIG0Z&ShD&gow{baULPiAsG9({jBNl0vKQ^A9WxH9)^R-XRT+ai1Kl39&=-L z(D%LS!$JCi8c?pv*fzz0jp##VqcX2AqkgG^SiViF#!Q5(9wr{!a-W)=_Xr+dh$IABjcC?-`i{PH(RJ1GfjnJl6|ZQm6@z zlUpuuu=VMNuPP`Du3^O}yGOXT9y|x$`>6X^!<%1ZUl&ipvi`R;32GEP=S*(kJ0Br# z?h4W@8uZ%oOF?!`%c3iEsSv^2@~S>aw4JX>f)~L>zQ|p+*Gd(~;o;A5ZHp3XZ zVDU*SlTnPLMX zQ1UO=n#NM#q;8lzrzV%h-mA1;bG@lM}-cYzKpbs&AW)somUQrD$|nLAr^o8_QkXPfHuW(h-~Y4Ht= zLuhh%^AyN>Eae+^j8Te8Si~~aYV|uKXcX?OmY3vfr&|RmMqRVKgKSr;I#@16bFVsq zy%A}{uR3&;fn+C(HE(>cjPg99ywF767=?O9eUz&vJH6Ygf5n-!+oMPcheX z7l=~L$1DgE$Tx!vMLy~`hak#$i31-}g7A9CB+=m?N1;s(w5>pJxLRqSb7C}LMuW_K z=N|KoB$+m{$F7?p?=JP+DvstF4f#)G*q~DOWf#s&1`?r8Sm8E}vkkV_T?84gz5S^wVJhdK={LB&eOy8c{rApC4#?<>!(6Hi83J6XBGJgZR;1Y?Il;&Cpu$@ zOK9q=i&D0ZGYSdEy&;o)1?VbrhYXogIYEk}_5#$Lw`n6&J&`nKH-$hdRa&u|mst&( zawj6zR1kiGxvQdFaK5Ng=0|a1CHV}2h1aYZ#f_^mZiGs`+%5Z+y;J$BT#M#yEH$9~ zq7BZfBA9&y+K!xZ1)5q4^bD({SloB+$<|CTEF{nmc6kI?RGrOI#1@G&8CSUk z5dF_*zPpyfz*RG$AXZ@(S}4wuDfZ#rM7iitY6d8wVDIFkE zJ1A%&DlBXy7GDx)rt!2+TCEqB!?~{tJl}a?2BEzuVg)Y!c=Kq%v|0*qUqBK?CMp2n zhq^MzJ;j4_ffwE-Yl>6}0zgZOG6q&RqneXu#YBysdZZq+JcTMZWbi@?mf|}LF0Ou9 zQQs8YAhi+BqYY;nHG*#xBJ6aaRryl~2wj?H@}^@NU$DB&1ny6^yN|87x%5uFrBE?R zFu!=zzkEbreen@}^C<6RSWMJ-LHKy^M&ApHXFh66AaVKJMH&^a9Yy!@Bw9ep?6wch z=~;&GrTGOA!Iac*8?(4Os~Ts6RO4$3PYrm|><>ep8bM6FQ!yx$Q@Ir{j+fRc?VP7Y zRGNg<(9u`KiX(BTHJIBudr^Zu44%<=?L%A8rFp07=Ka%$w};TzMnD~Gy#pbZKVwK#fk?q zh>+uv=?JLnGA00>y~K+tQrpPtg{4@pcI5Tm+NVkVMI2inLSEkaIn$c*rBaI0$3F&I z_5%QMCx||$^hHYtHD(r zeV*b=wxCU>cpNUwSzwhTZEpj>J`h*JLJ5B1G`68Tsnja9vUj#!mXcA`EF`Fn5VX%W z%v7(GJ423-F||{(D;mWN zJ6r_U70OyUqinDHXbO7!!PZrGWy0fq={xEV&SG^SwBzJm*i!QfKxrJ@e2TB{c32T( zq03#dgRVF_%V(NLOl-m_viev+aU{J)z_Tkn%da$!kLxB3in-Xx zx(E#bh@dsMVzI=PArL&CioHr&h3rDNDWy-)V+OlCQf={ZP2h1T>QpX5G>N$2GVIL( zw8I4Ns=I2!za~fk!;DI#_ z!RbqvZ*_&Tp(thCW<6uVFCIr;_HM|;Y#l+qGU4ky+>FN!k6Op1D+ zS{XBq`89Py+Mm6!F@l9-d1n%=r; z)a$ZWH|1O!wzbxLd#gC}L-CP%%@Z2AQwKAqiq-@ew}N!UpupXJSX%}PJ*5~OeRu@- z+6kb>OKNZ>l-A4*lrCBFjH!HZmT%oB>Iu7FYguT`Sj3h41`4Z?g5jU0V`Vy1dn)hU z6bdKRlKu9qAog$=} zUStCzoq`FT@wtVE72#(vK`WkK?CN##+5I^r;?>Tq8#-l@g9ZwTP@Nb)eWj7^Pg4@X zy#bze)PBcz&w+(;u9h+v5pJv5tRX!Bs1EQ`@YIcN_< zg%E{KtL9Oe0xPZM@oE^FH5m8V;kR&#bKzKGjPg4VA{x+Bn!5XzoqBG9fS z==A5)DJ^QVik^cqCadSl^_Zo3Chmn+uSaGQ&ld@0Kd81w=249a5JgU4f5VIb83FsW0Eq}UeJzPD6cNh%ot<6GTRpN#(dNbxR)CSD3 z5R8_#L9BQoASzKb%Dhmh$f(n=aVhs~!oW==@bp_qXom3Z2$F>sWLxVHwx0&k^ltK) zs40MAq+b_YB8k>zK!(j4xiG$hxiK)?z88YcC!eDCI%*p}Vmqq$ zxdu39w_1GULcw z+3C-SAH@4{b=_L<|+o|eCY_m4u0JwTuFx6)@dz>{xdZ>Fb7*{A|LTsd(t{DC`5zuucnW;-u$c5cFay;LzkYbl3>A$Tu7PkD>ymwt7raEWr$HN7JPNlw4 zY5GnEIR*DTU~4FKjKU=NCKKIZ19o#J9A>=Fc@)+;_r^n$vCXTcn{sc#2As!)XO_V3 zgnMueugxlI7`%ejDY)TTD(tF+$CNc(2jF~rq~^ssyk zLt(WT%n>0b#jrw;W6;}X<{T4wH#Xfj=mr;9NKHuf5OAK^7?*`EvjDZJ#qcK0)gAVr zRXNZHhBVkAnGQXT>CMBOQW;=4j)b^jw2fDEkUn`eKV=x><`5EB8U2=mfXS9r?OB$Y z(P|XY;4V=JY9=N{FJGSr07#EWIKyoh`PNi8nciKXChWaA#9u2aX`jT>XJ(j1 z+A_V$pt17mRoEf;aYTfw*VU#t9y$dxeI*VIhU4buYs`wVN}A-O`F41y53QALOFKT+ zN=k9UNMNYK0SIfCAWD=cgeVAoXH?HzPnmGF}+hqP!Xo6|%(I*`wb4SX+w&QE2)hgJi!;HLtwZHM^1Zh|dIg zLD74poW2>giH!6x4^;TZpI)6Uf=qyr#nc7T=fXgIB~pXWQ@Lk&Zbu!0y@L)I3Oe&n zedq>>{ZfH0Yu+GNZf~OWC$3*BzV#H`wK-u6G6(@mOTMevAv%ZmaDc^+cZr?2bi(=v z$?0XiBmrUNCjsveHqwqEzV(c`G#`@#OYNt&v`eCN1!?yC<^u4b0U7K1Pqf^D)PCNGL2gSPBa;! zp_mQNmy56&15|!e8B^6)2apQIFJXMH`ezi* z;dD4!$zBqjF3wg^>(nl5AFFQ6ZMZg07fCG8!I~NwnMs31gI9EncU=S$v4I&1qRB}l z(}dOU*W1lmo}jx*z1XIsJ+#-dr@|nd8GzxM5#v|&l%1=mCVI6wbS95yq?4%n^OIVb zWW~<1HfF3#B=kH+g0wT(sD1#Qc$1Ow4xe8%qs`r++=FhtuehaLDB6`&R4c$xBB51P z;11-wl=8t;?&?I~$wxT6CmWe>#ZN2BvPHE;c=8~bG5t=4+FCTK2kkrIQ>SIcWiUHo zLB_CjB7HjTMHQMMg~R|?o@-Z4VdKkhEQbqMt(8uJk*}hyN_gwg(7--Bt8QNw>bBT3 zVx9{D50Yqt(VYR-`0(cwQwatLjFOaqfifqYR8nrd1x@JsxKdNOyzEnNh?io_DrkYe zZ&ct6VhAg>AV66HkH9-Ca=nlkJ`5-mCp7~FM*tN{uqxj#lSM$f;lZJFNq99adu;m* zuj9*yVE~0zj(du4TmUFva2HNKMb)D6zVv&5yX5Rt8%`TY$XVS=_b@iSw4_;qYe=uO zad&0LFfd1))H$8Q%e7Uz$bQ`Qj_8LLZd^Jjk%)+M5CK+gOa-gLHmstvJo0sWtI|zH z3=NILH$D}6lh$!JU%F=%Jo5q30oH=&!<2+&463{m;d>&Baw8?lbsfRJV57^~KzXjN zKDY-!@+NzSx>@VoE0qv;a*KTzer|4pB2{s#8WpKRq!ulJtX9}XwlYkq6?_5#M}12~ zr}sai&B-2zf{y{AFoUz0Kw@LGY}v74{zY{Q70YQ5f;~~B#Uq4RXY6DsHaqM&O2>=_ z*qp{|Ue|8p^vo}4oEJ9Vx2Yha{+abkEjvMn7ebD#|SUTEYfLpG`|I3ys84=2&%q= z)(Q7jcpOD#swWhv_`29 z5?ZJv(p>hCIjCAz=g{EED3qF3{edRQw%suilm zX8HtGMUf!jK?FvYeFO|9<#9%f_fn3P4PZDMjnasq_$gvjyOz=}o6iblL4GT==Q41A%ZWSfcI3xzA30h6FI^Su3YTHrFFc9<2;@&3(hx56Ok&)4LytA$+&YipfVdwdDKI7Q~A- zXFi&-fnFiOVIO!-)r|s7FNat$O9odN?*6e&>MK7so8#4-^GAk-gQugs@R;YWm3JM=OkkO3`)5&5D&W)n6KyV*MMZ!xM(Duwj=$Om3zHljF zWN4pK>j^84%LhrvPEf(l2(w}Lwk}#dwsbn3F6Iu%g$$43ZifRVDoX2#o5Gtmo9Pdj zdk6>E?sFA0&LBWE&v;-|;YWv>whenw9YhKyKN)z!A9-Ewm~y$x;hiSV!+OO(Zt>7C z@VG>LxHe`+cp$?igQ%cRV1hJ3VtPRQByEPUQ!|pQWtLS2A(A;bWhc)kWdWFx;c!tH z!e12y_&2W&8h%^TEjSbB<0p-?Me)^DZA#j~2dysXhHuTu3DP)88WC-1_QWI59S6xn zzH&PkT5Sv6lRz zITO<*2C)JIL-~)E3gDDDV^vByFklB4;gb!e3vj3M*pd|!nYh03eRsCgog4}eK`NdU z>wZbbC=%Mny}(woMIPEMv4;s-c5XNy_+d7nDdCrA3l{TsLM;J^UO>@ zt2^w2GVz|)6KzS@n{T;ZmV;^_hP&X6w%SQl&pacIkq^=)byK-4q7cf)B7E+bp@m{; zC>inA6L+2rz6jN#eY#FOTqqt;_=S}lXDYJ93Jvm}oVPwF9D)OzIQa&jw~yi*w+*1d z?F@>q2D{}=wU#f zWu4b-?!~5J{V3ZSRB-F`NPCPaYgiy!1s`~51&q#zB#S|eZ2{~yZ<0~E5R^9|SIOQO z*KklILHo<2EEKLF{mbDHPrIw#?BX2*Q`+H6R7??35vv0PU?6eNT%z~T=rl;334vB8 zCD9q+m5xe@{IMAdH7PAnuYU0RifFfmYPqC%JJm4WN& zRlrb|2(fbluofDsWY!=%S0~4}&N|hs&_>&l#Ho!Ip@)ma<2a?o!ON2lpL6eLMJQZL zxdY2^TS}ioTN9cHdMa1B>;{v zFF0l1@_T7o`wyg`AT6;1)sD%`3J3`K4DC|<=wQJG5kcL5Egr4aZ{YM1N)@F{U6^c@ zcX~*Z#zS%2y^kg)9LWj95HweT*`^mSuo^gkuRL9mN+@2E=81H;^)>=N7}B{@#eX-Qh3xxj1Q={>woQWtOT_r} zcp^Rh?V&* z3@hgDYqCZ=C$m4TSG(QbhfpFAPXel30N5YZ#vxZx%CWid5UDJz&d@%pq1*-|t)SpF z>)rzhA`rFu*&2noG%g=?G?v)*Fe`~G3=^u%!EEb;S^WGnQ(8JBK2oSh0NEx*G>z}D z+@ojX>Z)`-S{NF`q|;)9I^$d6g)jp;gK?Hav7__rvR(dIC%}71F4g zArB3Zj9pv~>Yd;bu6eHPoU4Lu7E-kfHO+^4Ztlb@X|aHEvfKE!9Rl3EH&t3IC+LSd z@>~wkb{y0$uR**K@RJZ2_)7?oc2E_<;}?fBX%6c=3uY2%bvxcl=dNK~4f%V^?3jX) z)R;M0MDjKLtRNgyMIfp}8nF!yj?a#-y*OHPhlPl9L@t;JsoAVX85ZF+HCPlj2lJE0!6Yr@l|vQjsZi5 zx40ADM6u-QW(Q8xhn-a?5EO*d!@n~S#6p{;pcM;`YaZVz)4aUtnAE%GIzzBHEAK*y zcxTw(e3x*de7`X>WywqV6ZV|>w_*+l`mcK@0i>I$B6@F&)}cCizKXM5`Ee? zaQbi11ojia>m7B0<&P>b^=R$N1`hAdS?VL~xO26RWGy;eURw4re9BS?C>d!BnrP%j zzcHTDUWr-xiADNYPRc#2(}=7LlZ!F-$adQ)Swoy}Zh7xh**#w$kFNuRHX7K(5wn<7 z#3K1}-2q>TSXKfR*awJYx)?R861h_Kz;MZ~*M5LxLdXcJ*=^QDUOJ;yyRRd_0)=F0 zg-AgFY#x|wm{*olUIaz47h&+bl z^|=m{!42rP`_7me22WKUS&k|oUQQa{2rI=r)!75XF6j38-J-F#W&$&o2(LtT)#>^Q ziA+=lBY3f?`*kM#+MqcuJ&D`vJknclpzFQ9Zp^XRNK8QKowzJacjWCQG*vZ&><(N7 z4Du<|W0g?gK3wllJ& z;n8-GdCai#wn(QePtv}T?N}JG;IO@-@RxkmIY&^uc#ZKve=O3|qlu_D%}yUYTH!AH zo*oRGF<8kCjMk;x?}hsH0vohZkslDiD@eJ7JL{0?J{`s*)Cjyp<^9C6UMd#5s?}+u zaldx@4+ z8EM?y;-)^zc?jB@;e!SE!`3used7VGeso*LGzTlOAlm9vpoSyG{xN3$HRTpmFi(mU zo>?moQ0Z3o3e79wm=MV8s@f1A(y3;>rm&a67keIJ?sokuxblJ*y2dwUrE1!KRvtCG zVe#+^QN{-rz507~aQjJ;pFJ_rK9xi@>h8jm zd&4t3Q5yA9MXqFE!hs(Mddg!xE1@LaSY>+Z=&ex!6uKyXR-1O4vzi#tHJ%Mp624!a zT3?w8xDjU8=*DYG31#OB*1(qHz9k#aIDRW)yg#|9!(rTkUO_U>n*&fWae&_BJ^|ik zKqNBKRW#aq}vOsQlKpW|KarHXG4 zIqDJG9g*=Bhzc(%`yri_1+M}NWrEmIC@B!plgbk7Ri+_|qSJ+3C>a#_INggxCyh=Q z))9oYxQ$5gLcj!$bX$@Wnn0;bh<5Ip!|)(Cos&>Q$I));3EH;uwDQ4rXFVo^V#P3x zki}%{GJrxIQEA85vkP`LUOE>?Uf`)*Kzt$~9a#avK{VEOZ=OWOQ82pQfz%~eB6Ze< zlnE!c*BE+bpdQH)f=M;lS;8a4wpF^nUiXrXj^%h|;I1irOLTGNBdry}@T;8NjmLJa zE-GQ_^QTy3uQD~3y*Ib!TQ6Dwk4Vy}rgjVTF6dYRErR7RY_9EY!ksKghX^&;6C)LF zBizCOfe=#Swjg34h!+rXvA$j$!vN4h*nyD*VAkPGMPT&fp!=T`!V74!zLVTXtEwq; z3Yr>QCt0+eJH@XE9uUJ+Uq`7kIeQ}-XJ!%fAjC9j4>6E-_z>yxC7M7*`DI4J+RTAB zsV*MHDI_P)KR_VH4H&hgf(FH)Gf`Lwq$@ge4foW2q*(VN8OWtT07%G##)N5_+g-Az z%+vsi-HTjEwp{e^lVnMC-=?l(E*Z-7%}&U>&m2rQdLlc}EE;8IA$&Y8Y{Otmt&5vK%fY7 zKpK5cR;nR%kdY2y294!+ZLm{msR`&wc#gasx+qt$Rc}4R|sghE&8!qa899NVgbm-$#nXSri{onsiY1!Z9WxNn?8PJU+*!v zOxt_K4>U`!xVgOvGbjK*a7V6|O^>;0^W>mn1R{S%e$v zmbb}$_NC`)W{X6N=MfTgeiMR(4xz&^`Gr7baO#cx*blHfc@jF)j|-RigasN;1}Q|6jX-Pjyy#%rpie(ij3`!rZW7Kb`__)5T)HNcv4dv zqMP44RfNJA^sW@Qz{C}nN)yq2E}*y$)8sMjS83Lycq!YI93{FV!*IfnORom-XN@)? zpWfD(?^nJ{X!FKtMcNZ-3f2TV;uvOe7}ew@vh>99c=fwhHW!+KHm3!G?mm4BA~iPL zy9M+zjHbCn0S;R}C{h7>@7Hdep~hxH@ab}eN0m-tmTOcuEN$j|kCDM2wIC|+X)@ut zI{e9)hGm^V!oI87r@Ud7rfRf%1(|HNLQOY@!@|Iz)J7FRIHF#LHXc?<3((LKInb>- z7nO0%h@(>ij}QsnhpSkr3h4Q91X-jPXpf@-F`1HCy~Q32I)GT1@$boIrP0+VDgeu= z@Y!y$AskOxnb~9m#z}L*=DW6V)kz1iFCo|x8###8CsM@cH@4~!y+=3E? zF>ZvT%1V-orE(v-?LjWSf-9s4@nqq@!{!KwR))0QOo>L5Cp7Ac*eb_v+H`b;rP^en zNKuMa9f~TSlZo!?ne*5WQ2d`Y#I@O|)o_rM(u zn(>Ylq#X@0V)&>BUSj%s#LlDkurO)V6J5icMji5j;7{XLLF!GOizSnbAkt41K1VB8 zB3TM{`0YCvFBvOVXM|ud>U83~I^km|v#`T;z6PF)UIecr2&y&Mf-*iKybTR8bUim} zuMi{Q?PZ}@O?Wo@&}S)Yf=EFS;>nTQhuA@UUpTRhn;p8~Or6~rh@E?Vw^ch!@OIh0 z5J-SJk{JNz%7-9<3&e*MVX#8O=5@h}n8-on(h-z|AwO?BN?3>pJ?dhKS{rne9N2NQ z+W6GVt0nl>yX*>3E~9C43AY(&n;$}y0S5M`imHt%2L^V~4sHcwl<#hLJctgM*2~sP zuLxKTAZxkcLIf>?n1~btZEzvQZmGp0eUkg-yIZZXHB{)=8e))aoH7wQYb`Def-sMe zY#JW&QUNhwK?Ve;6!4jVgfKxsehUym#Z^c$0O14Hbgk^*Tq^GsGJ7jx%){8OCKs|1 zE{P23YZO=sezr*&5W}!y!KV_uV+6wY6E?i?Lj%*$oLpN(YEP?g#_{kV3wazMlIPtnW*^iXIU z-pO+!Kp$==uOn>@_pZnUYy>`$5#d`R7ZAae45O78F61a?>O)AMov238_}s=rEM7@) zLhdb|8qugUdyv4#!XF@j)EWo^kH^WPhU97?hAFac4?5P?Sw(7nL@aGtd_lk<1+ifw-5b=}oMphfxD=L1UD%fgQ!c^b2@k-tX2G6*4p2uO&7 zBVl?NEG&N6R9eU$z0$ur>odt0UCT$+g_H?-2}{pro`h{y;Y}#_fyrVju*j`D`Fmi` z9A*|IWOnddK*f*?heV|fM)!2t<(Qt#y@Pl+!RtB7S1KGWZ*(wW7ZK$8fqsj{wM8DL zf|(tJ1q7q|jgAk)e?=%2%`q7JB{#~oZ79WLoPm%~r9#RXWnRVo=%g$*eDW{C_kqct6iUoOE@g_~J2RfmONl`?B zbw*oyFXj&DMOhS;6d)n^Kwz*zq_bqf07NLvx~va9V-HBEJ;wrXMm{V8@Q4^P4!Exe zH?d}z2K&`r69B&Z1S-U@wP&T+V}+=@hz%MqrR~;dtK+X!&9~mrOKj*7qCTTkEBic= z5N880QzuI;E?!w1zN&hFV0HgexH`io3H8aIW|xViDU#r#Y(2%8(|U@G*W6 zwIRYuKZ_?K&*oz+|4>>tY=$sMv*&IeSf{w`pWIvMe~86L*%AKwyPqxCQMstV4{4#6a(0fg(2$ zko@8mkxDDC-v#*m&O5&+2G16wS39K0C+My_m1o(-u8{4}=Pts8G}H56rIhiLi@IxZ z)?!k+arZb<*@%yQ9AglJbnAT&JXO`SQU!*g1uo@&DKwM^#Yx%XDzpNLrB_DLMU=l- zM@PqPam_^W%E@iJqd-mRMS#nuZV~A11_6vUkQ-blYez*-1Eh3uGikTRDKntm!*yYU(L zed_EWbiZ3Mk+XA-)~(W-utlBtuDSfnsgqzdJn8JqaA?!JAb|r|<4PRoHoMsE-M&`9 zmkjo~yM#FdVz6jU;TkN3+}F$9Z2cfD(M_w!yFuwg=0T$UF)FN`@`kH6jh_&Nh~~YL z{UQb+YbLJeZi459VX(>Dps$$0P<6B1HThWj;{iCH$mVLD9iGgfPOIm1?8|SRAWOzQ54-= z15E_OuR&wSlbvER%9SN%gUa!El<+SFgor2B#ED_W5j57Casg0)K1UvKV?lypO{kPq+{`d)BWrk_{+A$6>H4w{B1vQv-q}!sbXzF$PWd zRlciu2y{9Tn7wf{fY(a6>FUAYpjwKw&vFW~`sAVcSXF@X7SiF#-y6Nx3*AfL3k?c3vn-L^y+B;&lv8za5XW+q@I{-o2-68m z>2;acoz_CbvXFr_?33J-kZ%r8&qN<8fV3Z3lyFH=#DVLLPh7buqDQ*-4qI0dpw~Lm zU&uMsswa11C2ocry74&9m&LV1Alu|U`dMR)jG50zlPcyE*f^ocvX3L#Du^ZxVW&6_Ja3rg(?atc(t zlx$L}qx;pMcVUwE#HUM&Qv&N~w%o`S#&B7d+4dsB=~ejFWzS#2pwu5g8f9iiWJP3U zc68FeB0?jZ8d_9t(ec6;u0GPBce+<78`p;ucd(4ySbw~QL`^Azu)#(JqjMmT11XPe zzXOUJVDu9Q3xlT&gURHlYZT$MV(wx-=TB|xE73cx!&pHWLOdrBGhyHxY*NW$GzYO* zm;@_=I@Vn7sofrYfPJaKTn^M8O9z=vbwbi(62C@PiPlOjW!oUx>wP!OL*<)$)xO$3 zYSu1hGvv{aOqX2*2mt&zKPZNe&17Z-DeoEsFpwbKYA==a)kO{({eJ6=W=FMJ3JaL5 z3DtE9_Tqd(U@D1pI~u)IH9w{MJhHBh2hSGesz)YorFaFF3tElR#DXJXp`F$-ykn?n zeOz(q+1uNhH)4fhn0|uTv$?=3UrQM9C=!!FO%8(eW+AG&f?CX*WE%6^3Mcy4&Gc zz}8f9io(ed7`+A2#d!En8&}0E%5^Pgfb#=A6yOW+xke`0LWPKx4>E29FjA(+!uDgK zWCR1H8C42xesw~qwZ;trJWBVYj>uJ3$ICzvDN6BK!-!;M08;W}UW+c)^_+UVB#A*j z?ivUw%D`-i8r>jd*^IS?C6hwX<|=Zl<3czf&zS=0={M50?#!i9GkPrq6~#{|;;61C zm6F190RcF#m{Hs$4wK|MAY0&uzF_$3ttQkDP!Rrct1B;8Ss{V$4O1pP`AcxHmvT+U}$1w4*ID9X2yrS-SS7x0CsyJ zTsP=5G>6D_xv!**gw=kbMe^dzA)v{pZC)(%SWr~j1p36~cee)mtz%SvQ5gWmO#|f# zLwubqD1exvSjRB0%(T63tD|h0QBF*cnO+cj5HbW!?4JB+esH8PxTp6FD}1mJ9xK$l z`l!R4zsPP7m>uQpd6}X z0c8+`M+ouKFS2vi{F!KBmDXU_ZfxcpwrHJ;qNQXhFus68z?Iyfdsyltmvk=0=<>3+ zmH$+T+0sg%Nn#~ZY3y1mE2cc?z5?vcTemfLDT-E|LpWFp$Xd74&yQ2fit?7qc(z-B z4&2QN++|Ct9|T3!ghISfjJXVA3@l;8LWA~@uB?>BNE%O@EJ6k>zNkTk ze;pu05FId=qLi^%U&Xv#tIdlYEI{lgg+PMFp=)O88b57qC5U?fw{dc4Z!5cm)ZQbb zFPX`XM^V_D-h4t$qND;IqG5TkbtqJCIcfCJSsUGn|yvd z^~|6x6yD?D)taqXyLx2}-vnvGjUj78hE8LMS<8Kg1p_r2N+6z%+$bTyc;_+d=c+r? zir^sRj%c2CYeZzT*UtJe{H*%+FO2=q0ysF+T+lw+Ny;gYns#1d@w0~sMZ*W;Q~O)y z@3>l`WMB%#M?p7$eF$+~remGz_h%EE3-U#Rx3mzRbJiIFWom)>vYVb4N!%twYSqrZ zP~4eYT<{$N#qNgBF@X!Y{5~MzrE&${48x7e1(7ijyOaW)yt?6`kV=0yA22|8dfj65 zB~Mg_uiaCiLs|=d?T-m&-zZeb7&P{p;z=5fO8rf>X|6CHc`mFh6I8NYQ?!q z&T$G88lP;`4++MLp>T89qGX>I`*c(yoZ^-9$d`L+TvP_zTA+aYr<^h?VlS?gq#=BV z;2XP%7K31M^Tfwk5|djntbo!Qk?+3tIo0QJ-BdY9r$%+|>J1W5l6g2G&nq@vo*CtPJ z&>kP&AD}=$Y`!GNflr{gaXvJ24~$TA8=yMHs9D}Pj?5blG31v6G5G9_^_7E_&czY} zO|&dL|4mFy9hC_0S^6e9I(B`r@ZtEthV0kV?biLb?%*vZFm|&B`HWdMP zdfs9q7$)9w2O|+sO=K>+%NeTny`WlGMME4A2pF$iM{^QHy%QRmG8~GnZc5%E+xhFf zGhgzo^+kMx&RsMxV>N5?axI&kgm%mN?e6+N6{P$N9p__l%wh`)%PhZ9#P?QB*$?b zxig7L4KQY!tmj0&-PbS#o;SBXe;u&NA5JKR29?_S3Z(jzZS$@uxjE}P=H5<2kJEwU z8J^;;yDgk73K$dzwUNIK>*PUVOH#bT2r#mXzrFI*aB3+P8|>1g-S)THImOQkn+H#z z+tL{2J6^zc6WBKoNMP@@{&mZ$%cAk-uNY zpV8>WIT9DtXR6V4ZN3vZ%v(D7Yf+q0Bp2i4v&}sM#%jP;AU9~B^Hm2Yt@KsAa$@{v ztCNPmQ4KdzkfktmSQ)pPno2q&#luC!c$)g?844FE(cPCS!jc$gKAapa1T=skU8Q@= zcNT2*W!d59PhRW=C7Rn|8T3weo&`^m1)lv>>jap{Zew}L)k<^uaYtw~;A4!tFNNBVhk-o%j}1|mMG_dV&sK`cV$O0A%X4tv zS0>f4&?8hyLSQ~0J{eYN65lEc^twD_$PlQDr??(bsj{?*TAL7t!LNK3#!OP9dUU=% z=y_}VwgPcOy6A%ZOT?$FU8rb}Mtojco z6W#X^AH_pzfbV+YcvyX;vU4Rp>(@{xHJGDx!a0*Cs)CsmBF9VtWW#7&^%4?knOeeB zk6TD?-4qXZ_ka_(+CuGX<|wX4oduJ9zZ{0_${60M`T;$`F@o86j2b{gl3^lmC=0X7 zFHAfGJ%H1^Nvkmsr-LLqN9TvON0M`;k2>T7@uW1I2spwC`hj2|8b`D*O#02Ta9l4d z)i1EyJmW+LuNRn8mkx%#v*6&sMFfc?QNGbcaHCM-op+0Q^2nGk8ghl7&SAn47+@;l zv#mCAcBs1H%vj@98cpF#@W2ZZi~!YvYkB7aGb}88PvQ#~mUNc`zTntg9BV^V4lgkh z3jJmbiYPV4jWMUOHjfkbjhda0tK3O8gWqQB{2%t_Rgt*8x0Hu%EYVP+V^xZ_+o_9a zqH%{c*j_4H6xSUGf%%iC+}(+#`5lETsx-j_AyjoLs;kvVki=|riCgsBWix>B*8$H> zUMvx7W2_JqFlO%`5J>Ne1Q=hmkBc=q0y;u{0#=;p1Vj$TsC}p&=?YJATV;u7kM1qh zh@%LSFtQc;Cu8Y40pWEn-WrTkvKiP4m(mU&qcajECtG_GMZL@_`fKtbBfJk^gUy3J=C;gBB! ztruJNSF>9uAhJlb_@QM@2?Y#YnYr;4z)uW3Btu&hme+pV%tzQ|f`T)y4c{H1O`(A4 zXn}|#x}jNM{c|jk9`T-B22}<7g#DX$H0rFw_4c~t4izxP+jtN(2nI9OJh-)TkE4N0 zf-%n&1>Pj@B|i9#Xn_Z~TP!YjGb!xkpd-~M?Xf53H+8eBX5C2RRUAiM%W`EgwKjnx zrogg=V*z=PKq9|&HO^Yyjzcjls)H@j(4)$5`~>`#Mq)sOdUy&RX-*N=+njNEcR*PI zVY#&q(t`3D=(^A>#M&{LIE+=!J?~t?+wC+7?J}ga@MBvqS#5dN@+ajSL{J-}?j>Xa z0~YQpKqW%(W;SLa`5RicyOrdCrnTldD$$En>o-A2H-6#OKkm=cBTz#|;YT;vIG zAkwE`<5Zk=-cb}&lg#UopA7ND0f>0a;$2Lurz9VH7a2GeOvx4Oa2BT=HVP5i4nAF=62Zwlb!^3nQmlM#uOvuO{ef?{~K==$_fa;@nMyOdC&vc7@P%*3W# z?3H_GAq|+fG6l7#TiCqnRrJTcqzoa^>BphWM^vN=HRTRTnf$7d*XGNsT;8SCG*NZy zonX3~Av+mWF`39c@z4;h>tynIMx3FFw2ZmJ&?20e7WCmeFNX-8i%8iJb)PXVFF}sH zLjC!b^jFZf?wf*kCiCLA1(&m$7#Ivd3d@3*%nTjm!>(Fn4_tlFaa5l?+ChYYRN2kaWPd=df5%F zYKe>ApUvSSHPbaAR-qb4ho3~B8jX1eTy1NfT$Yfo`!a--J-MK)8wuK}+$ zpMhqVByY3>Z2&B^_4hjKoS8mz_~>~CJ`>K8?)Lf(qCtXzx&i6EZXTf@ z#TxZ5A^+%#hpY}xgR}w<9M$i>(4p}ot8u9dZ5lK`tA1U8FH)0p?Et4Z!|Bw zU^pKMuMihl)oJvb|% zG}Mo$>|5J>R;=l&FOXZGwJ`zMV~ z{X5QY^B))=EANlagJ>bZ>EtA*YZNpYYHME)!2;WaL0JyJN*Ydl;3HQIcrly315gXk z%!Q|FfJiVA^r&C2kT~ITRhHfsC?Wy$dc0IU-N8HrLvaMWKvKcQjOAPAKE%s6zAeO* zQM$~OVUYSe>Rs~6!|Q8T9pm5-WGfkVuKQb7B^atEeWJxv71c-AF;5TZEOuwI6?8{# z0-|yxU|DFn4`SDNlCo0bs1{#Tm(@gjB{wp=v<1*o5ZdK2PguL=w1G$oKIMqrQPo;V z)cm=q@x-B25UZB>49Jwj_oTjXZ7V5byk@EVk+C*LqCn#CeviRWHrJ1@% zQv~sWPJ$7sTUgpuLrFJzqR}sgDrR|XEYH5^l&^X8VMv^Y?!+jQ6bml`36({KG8>^S zN;;jX=zHp?JlxfTVQ0}=eKC_wuG8RBsd6~xnx3&$U4~U1?9Hc332pZg={mf4TYoev zx1)1GEoymm19T+$Vb}(wjkM%8*~EEBK+6i)3B{KQH9Q+}rHP?TJ1lpjL4uN^6%jnE zY7cdqf03qKm4(hlWtyFCn<94QK*FbdkV(W7oI~S4Uyy>^F#I|=nX@U zvX^{SgX1!J9H^X29sbvbF z_y{ozwv1vX3A8q>8Cjq-deGW;5lxA!j1@SBd9^TR$U2X|R%{ycE)_2@=jZa!n=}?mQZWgzzuCW9Cd%#$4Ak^XipHU*5A$hjL<2 zRm!Ae)IylTHz?HizWkCX;dLYK?(D2C>|$#q&`}|-qh?~A2f&#YeGN;J3&D)_n$t;W zQ2}%btl^MY3x-BrHh8*8oV7D_Abzm9%D#BFTtXJGq|u2Md7LHaFg>`122oCMrMMVV zNltBTpjS^HN49(3MGvx&!8U1wBs~(csHuTW4j)zb)csnsyU2wi_lM%5D#>w!BE~l@ z5XVEiTPx7dA%?rPFq}>kt^9d<9H8)pd%&whEgN7f!gms0^Ih=AOQaA4DNis#se|10 zD}-lVA*a`?UbkY_y75-976n93l!W*qiB8Dg;X0XP;V|Z!u(Az#%XR6(Nb=t*rv7o+#{ze-V}sthP%7Y_*q6EL_?IVW=u;ft$#-Q9Bcv>sME&R5AL zZZ{5QNIbDhWUa^-vgLvAiSn!HK>dDSoP=e5`k^^6F=sHQxCB$C)vSo5K(;${Fjsne z(7Z!L12<@^UzlQhxk-lZ>F+)fvP>sSls(G2%z&;0s0(!pB{}v)8>#IZr`aPB$}lEl z^)qJmLR`w*#|NSX+YQ*=JOf!~;ugz!am-_c#kxeV3i^AITT6p0DLko748n??2GprQ zi|4rsT7dkiDov;<+QvG{0R{DV41s|~0E~#`kyFBrSgn<0x5hUzQlzaNvLxCXz={bnyb5E2$~~S9C|=A*7iLTA~1v0i_5p zD!2#%jR*@Cj~Jwr$Q)H2GH2NVRu(4KsIn1IcAvO;fx>S6B5&8NxkrNwM~>MhAY78j z>OsxyDxR&~oH*b>9#55`GaDOCbWiQA86SD z@>Llx$XSfh~#fdZ65nio{IAdC1T`)~KS}+rnH#se#+e^2VkEw;JA<^p< z$39IxUu)%$X`5#4UzJ-gB?27Tcb{4>x{RXw_j>Nb-roapg8-+ZEp@TR1Tworg2{ks z96&H3=FkdnF=xV#j+_&?i{erClIv9}qQhsU`NZawnfY(D2w_(_s~*?wU57_F)~v>n zlYN4Gsad37YR;~Oq!io>bIc0NWQ&6#@ehinpu|%XQ&8I}Yh&uiQfJ1iMDz=uv1{dI zHkMo!;`I!NQIvLo<-#6y66l*=Q#C|=!|_xB;YfWPwb7+4X}30K*I9)i*LXrw4%JfR zv$%{=H)|0(GgPd&s`cB`yS`wT9*!ZR%m}kCI1{%Q5zzg?%J1t*0lFaW(4LINhn1tZ z-rPYquCBQ*Cfyem035i3Jzo`#7`oKm%hOb$>h)xWz=0DY5ay{OA+?BJOsEJb&N-JO z2M-)5mi59|znFfzx~wRChCNmI$QmM2@vX;%%dsB@-C3DK0P4ayY`R|^4-a&e%n@K7 zXln3-1wszHrK(itKMTUmg4}X|kUhu{n*}Fi16MnTe&jl2TX?aw-+#6JzbIkM46 zK@%BKZ059ol1xH+of!dSChz*Mel0~n+&t<@{(|H_mM4>#NPWr z#p9NJn-y!Wv`MxRH|8!Le{GSCYPG2EaF>`XES~B0lT9o_koUFrCyZ9J%9MNPow5p&DG>W@z7f7X9BOMv%ZXCOj>{J7^CVNpuf-mP zAn`RWh%j(lNknMHLYTK3>3Cc(Dyqaa4xVeenOd7W&osGx>u*xrBk`Z!9p?BkOPxDz zj?zw{f3#cBK&;HyQwN=xGr4?u6Q&2(9ieuj zOKdIYH0$((^QbBYk2W?JjzgVL{ckYP2aCg*nN&j#+>DCmdVT;-1A4YhB+um_DyO+j+-#$xOl!#AO%XwrXjs`+F zKwd@RnNAHSYo>I}rQl*34t&(2pbqQCx42fBhtSBB-d$d!F8a71#IA+KuCp;C%+-Ck zs;s2m^dQCcZqZgWrDf8wZ+t1RSyG{c3dz$4(q-osS%ntz3|?hrm9Hd5oT9ChsdfMu zMLZFZPbsA`_|@o0s48giUanE)EcBku3jk20g$eYa>4!Kz7E;hz-L5olhwlL#Kw*teH9oi$KF;ARl% z2jQrs8CF*FS!A&Mp5SN>T*?VRaTXxk8*dP4Gt)+3PiuhLQ1Yfa-T zw)Tg7C4N^4h(u5gwV{_n5uF}17vfMG3xn4N7~v6KNE~KCVbLcLDx8+9Piv^u z7OvzPdrhIF(Q10={t-?fm9+@aQV2%^9I0s5$tDy^4uKN^T6Q=QXXlJuD8KSepw;9o+%r-=-t1q!kGYeO%0WD9S*t0_bdwl!ja0 zZ^*i)WXOkFAmJlCM zV4F2?=x*dFC|pQKlI^aJOV&ykSdK`42nkzYt0A%&hzJ9K=&bTOmXL1{zj9lbJ*0beEmCaE zSGBk(d)7u#watH$bj1=aC8@g9LhpGgT9mR$45+Oo1d4L`1S>$;3b>$gAo}KxzPu$1 z$B&gq>;aQy;TO~S=f|Dk=O-5UtaGm|+g{SQP7aooG)L{S7nS-9sG7^itQp&o#LVOn z93#Gwp-g8RNtF#eHKO>&cYQ>+A}Kv((wNO!)zjNX3`81aizaHzj<^m-U1H%*7RPcp zL?L9+!gRb7%gjCG(c&Orj1+i2bIJ=6LWNlqUrx!S_@w4~2Ov0hCDMhLSqK*LQTfqA z>v*}x+)Zp?CFhX?Sb`1iw|Dt4F_O!#ZNSLK4Sq!_2yvMdzjd|gDL3CwXhxKIT)gi) zk}81~Dg!0C1`$-hgf3x$h+kOMa6sc00z+A#T6(gT2Jr73{z-AE8@pLQ74sJ zP&+nwgKd|#(Y!I1`p;R1jj`=DpwWf?I`0ZCdOoib$dV}vtN+H{&Z*Eb(sZV zocFH7rtz5T)><@`ewjcZF(8S$08tBBv^@@(i@0@CV@<&u;u!hc>iwvVe?MwmgbSP+mR8Y&7S|Q)waxJ4&g*j9V@DA={2%7c~ z*ptg|UgX+AB2IOx0)YC%4$neqoQ-O8xk^;R9-|zSJuLX_3ncE>hp9`>#uscfGq#A! zteo|t`DDe?d=LtSc9wiHBW0t0Ay6CerVP!l5SI#@327dY1CdX!je@tLwUu911VZqa z2NDR(B$ZMb*cC*g`cvb*JQf=rA4V5+W3v#W@5t-OtLpICVU6g7GvSK{ev1okr(rh& z_-bE1zqW%iLXH=N2~y-*@T+6dYgDL4!h{@l`326OL4=2K_i0&49`+u|H#v zr^gxKP)CZP@u?FqWUq-H_lS79_+Q6@rVe95cTi!cDppjUOE&lkLiZh`c_Zto7Xhn@ z%;VjBZ$Naq%GnOM>7Zc>5D9aeXJ9yIBHlC#SOH<46hD(5xqe{L?onR{m)(Ir8SsEJ z2@MtTX5 z()NggsGPxQ-CuzNbB70oM3i#Zgw>e{1Y;EmF;bgj#eROirLhvc1Ws_JRTRxyaMtER zGD}Vjk<{G-+5n$18tbtb3u)!mw6pGzL0%>TprttsISy5`4-Kp+fXeYWcmv+y(-2T^ z5m?Hlf`ZpVmUAmm8HSMzg)+(bn};leiZ^jtz+T81C_!${JV%}`(R3DZuZ~AM&Bb!* z*VjFSz6VrP8d?1q=F=8dk^En#~$X?qakeQH(Cs6P$z zRqOctT7lRI)JU4$FqTKW=Oby#ow>k)4gw(@*R^sMsKMS4ze1z{hYN_(f^q}+oEsC8 z{JqlM=t|*4s^VNuBR2;|rQYFN?WeOGzYlq<&ibeeol^MBPw=={34;2zHbgQmGklFZ z0Dxv{G(SyZoei2%GSdy;S=cY7kaDDp&T)F);(mIEwb2#XOhGPFL!Pv>6jr+73AgWN zLcFrgCQ4T9d9UYP5sP!WFyRPYq$3KDpz~pZ8p)e!5&+}V&?u>cN>;kC(HWtLg8R^s zDmBF$vpsU-Q?f#J6r8fDxD2LStd;RJ@AqTfZO@(^4u%HdY+~W89`+|9Rk~|<^NH_m zu}}i2b|B}7yu``E-$;U0AKKh-H|8|y_W8l9Pp^XxB+$z`&D0lS1~`Mg&0{w_VQ?!( zCt&6{>}qN!1@Pmlz{AHf3GLY{W99H~6alXDTYJ3CDCb?3Us8oKoDm4bSPS-)9~7$% z(v-@z=^s39`n!pWSuL_Ybe_d2$;4= z(cECVUschkuWIxp8gPc?+G~xb=<^TRL1=lr4MChXVmH??r*P{SDH7Hq^rR3vu;avE zbu3kugv)$Y>Wffr{z2zl!B49!RA@(H-l#TQGw2pk9U2_)3yg~#l0u`!qstI2Jr9&%~Q$baMW-_Y2 z;+?+}Ko`0eY?5tA&Vt2M1=mC#M^1uhcM2lW0vJk+3Pcw$%7CGYMX7CzdJcoZUv=?= zjE^k)!s8>x5fq+j|Ea5d-M7tHaQ3>Snr&`ciQ1oPF>B^VJjC; zaMw@IE47o{+>06Q3xx-3 z4o7~Xfwe;{oFy$~0r=fI0E)sDU8S*Kfv3hN%samI)!d5IA!%Xfz-dyU)qxd4^Hw2m1fl=pxsTzE0*c1(t% zX75+`)h>eqT~Se~mU>-8&m`SPC71Ln!ZA-B4sEVR3T!+ESWQI}NV{BJY&^ytzI|+- z-BlXEdmI|#vZ+OhYsNmmvPCOTzAV7@><8*^K*&mCr@yro!|orta)@Uz@7jP|oaaWs%{VC0raU?HM@_?^I%cIvc@GWzluGQDS3 z+&-9cChVThS`bDLN*UiYgEDT3!P>LYBZl=WYaUfSOGb@ph2>VBSd?d=gk5jb4Z-gQ zpok9jdCnZoqVvhlA_AAw*~+h%jWDkBD=^ZHaQosZDx`G^ft4&EppN9ZT*UJsIt76K zo>cZgw!|_#*j|N_R|dEN+uLUAlw7iv|lz@ zysEfA5NO-(+JcDq{I>m+>e`ES`ANE?Sg<(c9k<>*J3L!b@uGmr9Ac^Ej)CtR?LbK{ zgbv;Iy<@yoc2HRf&>B&^3zpwel%2YWOK=%m-GDqT8A+1^IOFky^1um0@aC>6%HadT z&G|0s;;=k7VJ0K8n<7~d-Sc)%2lS$bsD-Y!PGT$LhEI=B==!0gNIHcoMqTFAM`5dmh-=E5*Ou1p2zehFs5({NO$U1qDhm<#kIB$T zqyp?lYhXuW&fcbK6rRaY^^p~0CiI$RlvakFXF0-KPtCpR_kG`&?3096H%kDbb`b4nDWe& z^-)>g+W5LK?(xzT4BC>E${8LLVFxSphwK%}Gx6vw1@R@{hV~q}?d_i`0UD(4pF;Uu z6rSa}>wwuNq*cW6^PURZ!OSe6=#agY4zWlu8z9*lczzkmsBVzGZ$hce0*-uE`aKZf zf+8g4sf*Sv=-74EB+ID=iC< z-8AwD$X$6AZt`B)CO8fAnDCecO-yx60X#c-GPdw$@J1UNBWNnN-Jw1cITxf}C|?&8 zTs0s!Ea-)sdNBEz((-7D92;U0!hw)%jYY+7(00txYf!jvz-b{Q9YOhEa62?WL%1hO?HJC5Y!`nK&#bb^1A#N+$`TZmFDE| zhXEI0!q%P|=dy!uq*V4g6&3?th*n#Tph=oDgbr_7i`e-cPA4MGNFp1mqpB&!bq$N0 z8{_15{J&~l!s~%l-H%j3)lRkCgkJ>;UC1dlA3V`(@=&oXzXuUY*4j%c6|VSAHffYR zW-!|O380M1!?>d*;^$D-O~Zz4=5d2@?aCAId>;UO9|yn-)>`@|dYja9G7Tl37=|By zFWP36NG&4j>(-r3R1lPCp#pJCC7k)M2Sk~I*t#i9?oe_`0$A4gN}Cir0*aXTR)jY{`} z$6-p6s&JsW=hX^XV{2|;?&yN{>1j_(&JGy)>{4tx7D;n`+^5$2h%aP1G_@)^{F{}I zW14wXi$N=PBE+Ob`MlCgw|eOS5-F$yArSo#ONi= z0ZtAKZufA6Nv*_&Qi*aOjjyWrj37Kp1Iy2bt~kF=>%r;)-x4@7Ea+uYp=iRH2pseS05Y&dIiP05p}2n`1v2qQ817A*YUnFS*z+Kv*`YUSM> zeJk_{o=m$?TZZJ(O#4@ujpEG=6&g#Eg8A~a(a$h7kX9k5-47T%28=uuPq;!5#-|?c zOtm(!^4w*%=DiGvTQWSbK9ggc;?r8yj%l7%u zKr3o>m19?qkkrN=;(R|DSRF%g7r=_Ixr(Z1T@T5#(6wg=GtKaHo)b*5tkET8jBmxr zftEa5Z5$zwqG9RbWVj-JggA~2rK=Hjcu^i8lJ>Qgq_qt+)q?_3QBb>rZq?^;NS_>T z`HL!?LI}J$QKEEb;bGzesUw5}03jcx1(a)g0>U2cNXdfca9Ni--fj?4uB!YK{ z#tAbVlKnw=6PWC%!=l`*9uQZ#7Xu(YX4yQVAi{i$OCjXokb8{fBpLZoqPow$@lFND zF@_ka0MO?`m~GhDM zCX47hLIX7 zF~h>Q;doo2gKIPanm!UAUYadpMu&1LRjdSWXkOh&YWuZx4=^NVwOTix#iMcppkJM& zNNhzsrI9>|OGEdWmVC^uXxj&%c4OROxA0S>;(8DrA8&3DS$$n^2vn-M1+~DRyf@ya;86p7of&X*3s@oXA&`@|x?df~g?R9B z7sfbkWlWstghN9n=^hJ;I7}~q-!MZ>R6#GA?VH1bVj(0<1-52RfHceEsUc3d8BJ6H zzZqPOfh=E$EL=gy&4L+xU}K18_Etd_bs=(`sJ$pF7f9H}f_P@Fn!xH~bIwn=DO500 zg1jtgH!9Lw_pLhBbQxo019i0{6bO_G^bKz>)`na?hYK)<;kZH;kjxp9H-jYMl9i_c zm5XWulrzXSz2=KwZ30oS`HtG}ZwP?SMfOdMO7A5F&;{k?Eqv9wCS*%V@|4ciWX#?AgFL}PQB0&M=V;ccY;_G zquJFhDCe0}v>9p;Awx4+=P6rwVbucq;DObj&D{^9@OQ>|YQpT%;T5Wk4zuv$*jzOb zsD;DG@{n?JBv*#Z2pDn>?3KJp!x28V7J$V=78r2SJ=ijnmwJP%1D5jJ_nGi`CGpKf zbuP65aLz}V5$T~F!GNspFc*;S2%H!(IA*}SM{->^cP7UIrmTZrVwObTb*IXsR;0Dk z*8~Kq9OIK8C^S;n-$>2|!kg$mn`D$!t)NY<@(iR9&k~$kBD&hm*h4W5RF;E}h+1w` zq;UHQHCXB337AU^viIJ{Dc!(b@=waCDx)y#N-T<7HwNSuzr{Ofi&?A0F86hxlIWbaE!Bn zmdLV6&=eLR>Yy(I8=~5=+tT9W@FFX~PH1kp>Xipm2a=mY(Y;wtc;nw25N9(8^e8ZS(^*_YMS2hCy~57Bu430d57=c? z)hQ_Y|0H#gt_G#z!$)|JG)A_;?o%6z5E-q7;NYrM6s}21J8qS`8&v~d0%tB@J2)Bv z5CE%S>?)?DD#wz9Bsv54O{QJxwkj_HPA0ZeIuA+tz>4?9gHrqu`Z~bxlRA+c zTc8hDO*PtDo4*UI!kq!vU{Gc+m~eunlyg}0N1}iq#qbjs4W&DqCf4Iwl^HiUXNFch zOTLv!%O~A~)dmoP({WUw`4gMdeHP^emF9vt)w5Cv3&k&K^>@xIYRU2-eIYFPPx23p zlOWAiQ;b*#$L}hE7K-60*np5K?ae~4_w9=V$P|qol^4pd=WtOJv+SVv!(PRs9&Q)>{C5zcL~yz zfd(Wi^n6t&Wqj3P9*|cvZSzp-qhjV#>@ulmHRVNd-v>8n;{FmIu@!CUH^wFvQecXk z^au{Ar+Db)u! z#F|}Rv7dHmp^{{SHq5ye(_iLo$}4`uMuP;a!fq%#;`Qa|kOq@_&pA*_Yts8iz{~^e zSKKjjr-@T2VPJ(GRzdJ@nR;*rngP)F2CCOSG?1ae`qs2LtSpf1j|`f1(=B&SB(1BT zT8Hk8VWqzDfDkpCu8?dNQ@RR~sQm{h%u3emWL6c3>=m7W+~gObj>%3Tv_OR@z2aen zg3PMQvMZ9DqyVFX4M$YP%hT6Bk`e2J$Y%M96xcIsI{g18L4?+QET03nZnC622M}YLp*r zvWR6Rq$cO78tcC3a|S3^w7V0SHcJBLd0}!TboenwY$1<@)+O$p{yiyNFg+$;k6elo z;JOJ|f|FCxoE1SI8_gCGwcE%0XJUUS{H#bXnvDiC$JdilHlj@O{ZT88k_+~=kuTWA zR)J12zFHRIE}bmiIjbqnbnNG?LZzhL*(0*d_L(vWca+*}bZxkqS;NFVM61jJ#P)6+ z4n7=VWE(i4^Z_|4swQDvv8eh&ae+%Y5%P6WF=1b!6_iGEE|Npzqe^8Wi8Oi&LmaI` z7&Vqn=N~=NQczyt(x#m%KxCD*U3d9QrkI6D_~}u5_hB2dZ8#=S_(NiGg(*_-qEK=n zdz1!cW zkYi&m?@3>$FnDMpdJ;K! z(aEjEOW1fOI9G_uflrH6cae|h`^HkqM*$3}9KouiC`AE&R1~(u>SEX(A3!f46-rJM z#|Ts?&k9sTF2Se?B^^FNJH_&=5p$dk)3Rd$l!dK#V&UPG&ag0%N>oZ6NYE-1*eP2-1JB%ySC?W$bcnveI zkx`IY_{zM}PIw^$E-$BfTBnW6ySe>@w#MLD)jtV7DLFG^=V5}O+IYtK;j2!b=~-0a z?>Z2>+))Nsg0dJdXB-qoE+{!Xj5ZK4YLF9S2D03<5)$}9M%vb(QJ7R^ts-Z00Iu5e z*S`5RX1jC2HtefCR9FWJQ^wW>6w4m)qs9obk z8e)$JIEMH`aF7>D^=HUiAhm&*_?XRL&J@&l1s@QEnqVS?MAEML` z2r^~DMX$`*if>`odW?tx^ADa%dt0F(v1wF>l^=Z|;?Yi}x5LI9Aj!^@P)5QU<)H6{P$V}OK8jbYDwq*5_J&L|FVZ)@ zI2^;)S2@}A;{E2d*9=(hCrcBk6sT?C5?f%%GLz({oa4nrF>We#G1D;UV|+gPR9rH9 zPC>&lpGX?Em*U9Fs%(`@0_Hl?7AzXRJ-uIJ0vDvzi3;UPn(@un^8Rh*;4x-#eQ?-{ zuVwpJBjh(hF$C!Ck0GgHI#*9aFAP30c5g2VECR5_shp-NLj0zjHODp~R%h(Wi=daP z;-k6tY$)6CuVtDQ`Yl;GlR;UP>kLqkl2=ZfzhlEPWABbD2~6yAETM9b)UC;$&%yn+ zeEOXaAZF;M^7rVAIeyt>=8f1WsI1<11(z?2o|Fe{NA6H_cdyb3SlNbuCDZvktE|~_|>4jN$3pp=*#5gp_9ss zxh@tRl5J-is*;zUGYcx>F<%rY=zI*|Kv9g=7T2ak95F|uW*h^%G($gQb9q3!l>7yL zWL6Y!t6GM~lHSk`iCzRc=_cpXVZo~aTtqkw9laiW4HfkhQZRcI!lYoA5gZyoluWD> zT(yATH5wOHHJcPpFpMRXY;jgNP5{vjthp^x97Z!1?~Zd_`9_=%z&8SjJCn2`Eb|IV zKILz*Zm4$wETIo7t{XjQNR=OSkCcYygc$R>p0q9uN#*_|3lOU(VuK*9$nuY1a2HrM zsfx? z6}HI9pkwD1Y*-k^3Kkk0G!M$Ca?jdywa#i4EzW{q2kdkJbopsWVPwMYdRT8t0fb7T z!V#PAHyoRYa18Bk1G^bgSaW!iVg(0?I|R)I6~vdBV#~pU9a_j^K@lF1jErhK7BMYe zIuo7uT}|jvP_yM3f<$2tuui+lnONAncSpfc;Q?W@ad69Mbv8&TUWj&>+??t6jdy+WNfIu;GYd&)`Jd6N97E;O%jmx zHgr5GTTaprv0uAYl4KIBpnll5#MTC|dIcf?m#Ibt5LQDISv?|PqcPSkZOVagXr;PG z!R3&9Dz%ogE737lh%8oEP1aZSw-FjMfp@$CgDBaVUC>npEx z(rAkBi|BZCV9r2RE5Av{%&uS`!&2q(c~7AytcCDPC^kJMWMVB=63KZX=-|#>iZEGd z&7r|jB9?+GLb3ECQ`5sHvfmZ-uNRhsqVDzGx3hX$DTKqkf>dcpZEAJ$Ixp}4XU0NwAIt$$rs!=Md5hsVw5GXcY9v@9cYpGeJ9(p?g zj_A>dM_gXq@|jVSXhb4Pxp?#=Tq`kfMI-TK%hYr2KUdYZwl42MAymS z1HQ;9F|&##9EKz|=WkZ7*nxBf6RbR@Nt$7V4lE9;!Vqi1>GF+0K#lPTU0VoL+{(9v zqt!A^b#OqxQZ_2H(Zf`jhNgQCts6NkOa(zeTzfMs2(+y0)j^EhfVNUAk6_SFa2kcS zT!G}t_^^5|EV?pol%yrD#q^1o`T9r5l8&J^44(9e#>MJ)!f?wKOrap|W<;l_J9abT z8!X!6=!{1u4a?&7Cb5f8F{CvhV>$WEf|IXhb@PGpuNUUf+P{Yv?~h~@?4(0Fq%cQl z^O#Dg;8z6QLE#``2tIiQO%U$Rp`20`t10D|C9W z35t7Ka{jL4w7-^w$`A&H;Rw1&h$2G8t+0fA-j@qujpQ{c9%S(ctX*sJOx$YX`JSc+z>5~ao z%Z}rP*M^R&7q6X1)_GK|(TRC^?E+&{PDjA5GvxswsrRKqNd$MJ8(A|fxKtv>og4%w z2!Uy=&_cM#aUs{4K|>Oin1|+Dkh)WFsU09;fI0+3F`CME?=TtA4!14IMl}fp(Bv38 zgw<+&pq;~g3gEK!9>D+)-Np1?^&wkBuCWZu9{VV)bEqr^k^Mh<`AJ(%F!_lHhgKAs zQU~QL<4fK{j&Z&6;;Msy#{r=?yYx&nd`*4bt$TwB3|%wVeugRZ8Rt!OSXFV!L2e4Q z>fl#Og9htw*=cD5?YE#zBFMZM8$O+bLxD=5L52{~4J8gZovF)XYfleUFwEr+jiL3- zCyFIxx&X@NR5rWjZJ(pyzXCp02Gwg#QN`A~;=<+7k>5m zZ>{Wy73G+|b=O&~X54}y>M-G?{>=9%j^-YL>Sh9}c<3G(8>k(CSu>FK^FqSi_+r$m z6CXVzi{1zloi6ss{abJ>N^LVu^vJ@EtL?!)(~j}b5w!dQj}W={LVU<XL(Da}Vzs?qW>ZxAk(2q?gT8Ojxdb1fh&mv5|7y#0YN^r6CinsddSY62X!oV7z2srfi{KdjqNTv?%M# z@KmwP92>>BdAps7SQzGjgH|h!R2$uu$^&{O92MMHn282uyHq7F8Qxny%W_pzzP%Bw zQSz|E*E>!f9wi_zRHX)aRYOJ|?kx6NjsU#u&~^mju~)2$Sekefq6abtxYR5K0LHkG zF<|?MnUmM(&|P`pD3hxze;6lK1PBj_EE?F^j{$^jt%KJu%c+c~0fSpZz^FDreC<%; zKWe@%wh0u&^^rYn0odP?5N^nvBvcJZ_zUb>3o8(<=MIIwT99xQjTuERp60>jIp@8r-QC=jewFRG_jI0gV#Ssf5-iRB>M^ z`GqF;y@+68HXJ%O;>3YHHlhy6V50;MO9TiDMDqP-t+mw9869O9ccl@cvQ4GDw_7gW-N4t-SHw+O37jud%!0pPJalcZz7im!V7%nPAb$SW!Z z_3=#69#TO*Jkm;@N--D7>&8z0UOFqJTbFgU3$Ot`0Zwy)inAQeuc1%}W$rm)&@@&D zr_eSX1u#`6AKlS37MIS&pFCDTMt(T!=P`Ra__L4tvxw3fhaKh>QgvniQkk<9t2pqIL!V#7qbC#{bconTyw6@KLLwkb)SGG*^8Q+t!EbQ=9 z@sxxV0Xv3C;8voXtqn`vlxa#r=!1>*p>!W$+?JG&t0yci>wrCHW0hJEC$^9+UfxRr z3y_r#@XqOO-KF`AD0tLFs5iV}yjuK#2TE#GY2qPacIIuyYMb7soxQZg^8>LWQpP24 z<6B>YN!TxJ4}MYQQzUi4hwPO!RC(WEJamULwpro^M~%pnpKrXg!CZS#1n||V2 zB^%E?2;j(2a7pNEUNL~I9S^RKe4;`5jM@EWLc5`w_@aO^n}-~sJhBa<93jwzw@RH( zcYuQmP)4SlTo%-4h9=&3U*XCypkYOtR1=R){Y) z>I4ek=`ybw4u=xx<|CxOm|g|FXbSbb1+qmSDhX}6D`^0$!O0i5ioij(6>wfY1ZkAY zC6!Gbw3^MzG9hV{lV3x9V&i~=nYSn?EPjIN4h4^miUh2sm4a1>2TQp`q}R7!JPxZO z{F1*1y|OyKbYHxMEUcxS!W5aLP_Bm*vG3mzefPvfUlAsLZ8wQGbLoY9j>IpY)Qc<@ zucXD`M@IZ%m25G<$Tkm4fh|yt?hnc`eb%P9;FL3WJ!h~oHR?_>s8T>vz@@1adBuUxkTpVcPB2)|BX;W+;oLAR?@A!cv1UhP{Sx7SO<-H#ZYff{ls& zQARNi9?1#3$+G9qW{mQ+h=L!X1h110@IAVk?IscvZ$8o2qBHI5g5U^0ks_*QQ97~sMnCdP)K7C#fJeRQ{ky|-o}7f94p3jS+fpB8mM-0;)jSKPqK3B0(9PEObbD~1E$b8F$7Np#tz=*f-NY4qJ?k+teMN^x zExyCCL}bD9&Nmsh=G7%xc_`Qu<=T@YU>f23*X+Z{;+UP&I%3*eG%Kd6t4!gNzCroM zTn-W&Ar~g2sQ{3z*2AfXuSmCB5DZU}s&K*9%Hs zvS%HhA=s>Vn7FuqG^SF$pm*~o?!P)U$xITV)%`PLVZ!{hpJi}vE2*|azt)oWq3)tE zHRx&ux%inv+e6$Q2C9WDmZ@y7O(Xx?u8*j+6yU;?BFCdX`sL)MZC?O zmtf6^TC5?8sGzieC56<(`Pkbep-I|-HGoQ?wQSW%lf0mBm}=yh6rM5xIv|l4*hE`x z%)mQWh1p*DBRckpV}_lbi;bF83t}p-=S&84IKw5^#SIC^XWtnJM6Q-djqM+3N?Zbp z^tw}bEsB+*U_2}xau&_18ia19=4>-#qHN(tb!xoNBJzxdFRb(&x>c8p9_rKacC3=i zGx6jgDlB$VZX==}D0>JbL#Rj>)+d5gRzC#bF@i#kizzf}WXXX;44GtS99~o@^#$Ss z)2Jm&dRR~d&6(T;eE5KAitN{aBItnGAcABVc|k(8gs8ypBow$vB~6phMK%;6Fk6RB)Q4jKHUyq?$Ors#=65JS9dw9dz(>PUh&626x()&ttH0QGhyBk$E7G*(JM^-^Z=B+@0{Wfxa?@Fi+-?mGOzj= z2m>OGk;G}VI;;Wq3%s&~xc%%PgozJhV*KJu+vBITSPa76sE3GXTXW*D=T)^l$vmyy zs%wQ}6C{#xdM`g3kKD0AlVn)o=|{`n+p?dmV0hH%n}0#Am$6oEilGD6LG8A%DHg03}aNFw-HGMHdHX|e`9p`Wy;34%nZlu zhOStJa8RJ?us47^Ac4aoz*R>du}vt(Pc?G!Jz^9D7(7<&CGQ#Wa@xl+vsy1VSgLr< zOE}1$@eM8l&%S=HeK3iPV469?HsozeLkLubwejNN^H6YA`6dItL^wIQylrNbzj9EkG;j0drYM*xOKW$ zC{B~Mn1S43#B}UaHiFX~K+hBk2W6WcIp?Rvwy`&}JtuonWIVN*X>=SAKtby(38zt- zR<>mVTPlrjJ z_mp&@0mNS=;PIt}_`rq?%Wl3`Rry|&qpEsQdTP-7O&>+2g$dsE_fUgtC3GaAY3#MB zTN ztzyCZluLJtlqh)z1jI5IWqzb*KpN`wH)8Aoh)*L+FZ9P$4ye-WsOZkV_-Xx+3*M!vd#orb2R#hED2V&AvD>@g> z9f6)H?SBIn7l6>ushR2t!m%ro2&gHU_G5~u2Ean9PVOaG7$x`)Eh*%F2f7QQ*-j$z z`uHb|xDs2`5`kUGo(Do5t`bw|HrO&dRP9y$)rBkuoV<9fqQ2WKZ+;<71mdMMX2|~N z-?CgUvvler@C>}fRO%!hNophNP39rbd2;;uRPaq$=EGH?@0_vN_qlW&qc+i*L}v*S zHvyphNn?Nh( zQ=*%hheqM$Ka|eHZV-Z#3ot#JI!GQHU?H$X&SXN;;oI+XG!2G@o6D?*&reQWG;t|L zFgR2qNR=-q*1oE&TnP|9Bdav#q+}6m^5v@1m|qVG?*PkE1Q^(ONGzYoYg8ymg}4A9 zwD>#cViE-c3z_97R)&_f=kr~*h|DYZMT+@7@J^8di5`fAF+n0C>h&n9-x)g{pRJO5 z;!?5^{U+y}*GJDZsz5)2UF>qqw=i-Y0SM6F86LWCH;Q+y1IAU$HOAc#9UnpJf)kJ+ z5j8dfLbZZ;P~k)z?v-rLyJ`GXvSDjtMrLYAC4A3~=!Zz)yxazWc@!|UT@9ZvgET3) z6pa=-Iq+Cc@~M=|A^UcSF?x&43viN|tt&jFu4>SVFUYid&OWh==%;`lzLAfOCW!B?j1m|6o3 zXGKBC_U#>a>wILe-e3w&l=Qj5H3*Q|jEu7AOp5^V-8>^ji$C}YYfOQ8H4%Os4BC-^D8)A5mE*KOArwJBtxVWKq8o?v|`bXVOmHb>FbloI+9xt zLfU}V1zx{68O+*IfV4A(ie~zAwP=AykH$i31oc~t*+NrmX_!>kQo3HgCO~N-8Zu{& zRb;lYIu;FeR<$t(^O{FzqEU(H3hT6*Mcpth*V#>>&y+@xmwjf{k6s*8iCYn&euYCL zCDV{)atL(u2Hm(W83iLKWrbEpXI8+zCHO=}TKPfy4v7&a;gAZK;^;jRxG=jW6|%s4 zch#gk)SO{K_{S?5n*^YjZ7s&!CZUnQL_}P=%N#h#$r@1UPXh28S3L1XZ0PUY)1Kj} zupf*-6l^$xMRn1>Ss;O07Bt*{1ZH@SBym9u`;WpHxNbYliO|p)oifLHWE{dmgt)6l?IM_-J(UOm>xJd`fI5K39-eyAE25lxnByhWs=-J!{le zMk&R@<7PteIpCD%SJN9Yf{Jj$pge%Ppeoo>b8KM@W;^25AQ+zra9|^C{1jT*>phk8 z^n>ozy)G%B8PG|~u-T_)jRu>OnI75?l=`y?08`L-Nk(?azg5a3GG0Y-+41nAiU(dp zAuAf!craFkEyLNZjw;_Ovwoyn#z_jHp0u7`VaGfmJm;futLNV2p@{XFjkIJ#V58zF-3_-V(?rj;!gX1RrFp9gVTvZpCQq7 z2%QQk6LtIpdf*#v#iYM1SGS9&fWjWOYbrSoE&L@HU?0Csag*yQH(JLh9f}}`rr8A& zC+3Y8nR5k-df!A*cbf@~sV_1e! z8(Qa$?%0rmGFsm)Bd=_qONB$aXd~%W)612XrB(_=p^40yg{cXJ4ZL+XB9$+kkew(aBI*!gYL%(>2v@A^p!ohOrhFDJ9L<(~o6dXaT_`gD$^FCAxUo4t`QVo8Js)_Urb zyHu2*=HOSAnnT^Mwr+utj;i&?`BTl2QNXw06s(^v(t=w^@mB()#X|a4sl09VO7=#p z0o~?1asWcTpz@;e*_CO29#+L_^ZF4>rc0gCgOv!Gl992n-NO$T3Of zB3}d{;;dMRE7lJf;fg%5vKZYEAE9>qJ1CVfXLLKyMCBQxPlLjp2&5--sW>k(AmgZm6xF+BNI&IKLn}-Qqm*u1&AgyidyV@#%sr1mu<9RX5P7qBV3e$Oe%&3TNz8T zfOvu%TWgk6yIeIng%O~G-$HeBM;Q&m?ZQpMN}2)Wasq3(;aRxOB5>zsqY>G>)ue7$ zDe;ozUzXH@sQnz;)COw4uD4>mpoh~A(hgVGk9NvXDSgA|>)?$>_~JoMZCACfK4{Bq zbgRJf%hw7Domlt^hq4E?ba4ubC@MEH+lOKdEg*oQj)RiGrybTgH43vjRiocZBNvh&$%bNTO}y9(P8}(sXSBM} zl4qlk_I^ar5TSc~a0}%aywd8O?e0c$X^ZB13Q@ojfnGJ6V1+0Ep(|`OW-0-h7ght9 zJWaH06I8OI~DDDib?$m7$}<(*!!UIq=+NyVUvRM3pz|Lj>%a4F=PMNQM@Yfe(fINgjf0i^vbud$}0!IWH1p!Jj=aYu|!FJWF9fzVY3?C*n%8BHw2A805 z#70a2WV2`p9W5LYGAvx>A%_iOj-cTP2fpNGA$aV!lH5?tu25!jLroZXXST#j{A)sb z_CjDt8VP>1G){GFj=Hq=xSb<-%lnkZ-U7kX5-;Jb%lJnSJ#-Ho#ww&+&{R0DJv+wB zwRIW8x%7=0<@IEw=!R`0b6gIa@b2Mev()|5 zEz?CN4V|ricpKBu@=*KQcy@ZyKslTbCE1!Iu7*U}F&-dA!9lh{-!{)jYwv>@0cvpE zvqop*00g1H|k77IG3(f@_9*Z?3!s7?2=CTU)Qqjc;H_uD_ zB_5%P)^(9kP)HXf#RW0s_2Em2Z0^yOI1+WgGG7e?o^A8__h$iFX1UKd;}Ny3iI z=-?=9?d2#!EOuGUZz%#lh}@J~-c!Q2R;>c~g-E+Z0dz!T2%m1IahPYXECx(X#f6kL z$&ARng9b|VcBOU%fl@+-Yb3GiNFAEZ>x>}l5nSjW2Y}&J8A6*P85sifO0*98LPqzu zc&Ft_P8E8rSv7cgI9|uK2Eny^3CXXD>H=jHrQsXN0t}1`j2^(ID?$qqan-0m@B?f8j6Q2qdk+G zb5yLM1lLrGNzrs2bSUP|4OPs{jo_3G_2;Ih=8%Vw$P|+006`U@0(q7GKoujfpA+H& zqv*ZMB8*c>RO1t;3*AsN0WSnGV_KGGyrVN@O;;>P+o)Pgnc@su+Xgl6K;>1cw_-D` z3{^MHz|{1ed_g2??Vn$ZH`zF>d4^GzUzemMpcE=l5Fx88Li1wYLsa935U@ZWLBakL~bv3JVZTj0a#ud2Ao;nCY?v z+N2Z(mt9{kNZat?F~KMZD6|lQ-LtJYpSxbHHS|@fmJ!x6d7f8vMkkLleO^ zlB5mrH9i@on}vuS1K2&Mi{x#hrXt1b;8$5@8=Zv`A^5@?!4A#a#R+14??5QJF>?~s zi@zIXN1SDtQWocAb5t7~Az=FhQj^2*;`@tKtVN2;&O_Yk1VK}WWI<%umgD25I}|w( znt(8bojiPtW20!sMpaicby42|HQCHzupc(3Slrm!!gq|ZIsIC$dP89we{5XhznhCE zH+R-n5olq9@D~~PDyDrm*j@0)HmJ*CTFkvbE@2ZiF#zKmzAj*xke4nSS1tu>EhbQ2 zOxL$a{si52(qlBkL|f2k9pKVm3Ra2mRnOqDLsrRkj8QQgEw`A zR3l{S!+}S&Tc;vN0Eni&!e#*6fMj1g`UhB+@rZ|&$p}2+VFObMBTf~J@6C7ys zvvGZSjQhY<)j_yNIjhMuqKcuQf~2D6y-4vlNW2EbE-KV&G{J1=>3hkLG01dTEVdPC zYa8n&w`uEb43?di>RRyh%!)h1Jk93;+~jfb8HSMejht|?U{mVFIb}FFOBNWCAg#eb zSt}7`=84GJ`-O+w_Vp}uR8iXE{Uy8iye8SWCQVqK_M!Z!h!?fHQhOnqOgyG##pvsj z3Z#QZz%>}RI-zkmviH6qjJ=u)8?~v`H|_&_H}4|i8I2POOHKi2GMnihG!&>tUhmAi zm|X-V^V=e0z=W#^L2SI)mn}J4TT-HWCRBw@qjgXr?VoReOYdFPjT%^m;7-VsxdHfC z!hr0|Owi^TT|(qw-JWbfMGTVRPo&@IA&iv{%B@7~>!aBcWMj!zC`9dia2wn*OmrRI z_`2JM@pKZ;$V{b749AlL92yBbmO37>xJd#%n*Ez0&ab+JAuy&OV?Js3kEV>hDO`nM zt#+mtK8+LmF^roH`c8palnwKfpQxHWUN!To4C^Oikun?eO1Obu?-7n$=r4<8LOkPN%3Gh7OUFN|xZY%cX% zJ)&^*76?!F?4BmvJ-uadV2Sx!lgd44PJ$&9FLLH%RbqUeoe;jJenC{Vkk3cPnEZTR zD`9bD_2@YPdSeEhc3Y}4$WVRE_ zh%!3CPy+h%ySG>GZjj!RO(pUMsFg8rP^Mt=I`m;}ISoPEwgrZi^KD zgy<+S8%!B>uHH<$03s0phX@EaGF(}cP){eTC^#qATPG_Oqe>aMF$ENZJ>%DHvUZnY zm7)dsTB;Uuze=lsun=Lp!LmN@k?|u^5$R%5QQ28Qf|Fbl{@kjrfJtTZ8I%YZSg=AC zWo##VUZ~T6g}glbWg%p5QgxJ0?o3iaD_Se0)V#d8$-i3wj{Pqb4MUTBhV#C&W(UHs zo@>tpiP*N%DBzGs&YNI73Tt{3Y+FYg)vCFO>(d-x-r+|5g*L+t?CXL?f%7hFV6?6I-oA_Oidq;6X{&)nWlPkacN(e{oin$Q%#JIiZtbG4f> zdnycm@VF0*!;8?Wm3uk}MR5pseNgyoNCFj*osj35j4D$cskCm6Q*w{6N=Tva9bLhQ z2xG?#T=C{CVRdn`koAKC;*W;|gnHb=#f%Of3EshH7HB>!bGO7mU|#@Rma9&PUOjuR zX(9MpL6_JG?b!#Xp}IQr->f8fQI{3FmVklMxV~k+;F<;k_HIOIaKq0*Os&lpY)c_)Lachw{HAT5#DnCiPfTimLe%S|>V z(cPSO%TbR82>gOXDZ>C}GqY41jnw}nwaSvAu)#Cxapz<(@KTmz!xYC~RD172s%mrB zEtect0=u%DZ!GEM2M8%4EjaNkdmhRs-FF_t#3-;B05@qNLFYf*WId%o8YX%Z#WPho%LW#&SY?M?EtH%$7Dgz3N6kKuLy<~B<%KsP$~r_d0$ZN z>n6ak{9#748^Ot*Yxb&6bDfYJsjmV~(t@~NeWYRC=ne`bw?{+h?rY8wYV3_LsPz4JqQLi=cUiiYlVjCMnk2VjRxerwBy`V32`EB1R@1v zg4ui(>029I6u?o&FA|OcUJ&h8s*KGa#x1B_7R`Q@a>!fJRRBuLvlGNC(dTXHeNoY3 zJDS=Au+*q>*{fz61F-LtnmTg>X)ZV8sB_R%UUl#%B||im@tGNfM`6>0_XLsRuqxvX zPLx4q%L^l|(bmMYI<8reKY2Tsn~0^Z}nca$D5B%vxK)K=_8 z4y&D^ug(a-zJPI(ZW&i+i*o*>J$Uir1A+mSuw+Uoh4#H64a2^Dc(p;>pwE?)p0I%c zCVbjO9c&?6WR?oF9Yhd4pMahqY7w|SpEV2S_H6GKeccRN?g8stdN%uajiMiW66jJC z&#N~|q1?p1w^nqAicLYpmZKJ~Na(i!Zu=(>5*BE<)vyR|Pi)W4kRvf`f}l+I)1NR( z*fT_0j{VBMiwzFWdHB9bBL@+o?RtzezQMRY2Pi}gxvx@4cV)I?#fBrn44A>O0mZuS z7-xc85Z02RHIq*^9{VaXOb%QZs7CDrHMPmCMAYggvcwo*{Sw4MRfw=k32>Jf||rnzOQ8RtwY`>d4f;I}j;U%nm=B;v zelg|S$r0jTQ_`he*%}P_!21i@;J90rDInF{)C1EMDs)d<&P%_p?>Kd`o^X5{w`n03ViPZ-tRmJID z3ye1f6%^ni8&a&ebDY~-)dmj~k{7LI=YZUNQFiw{O;M zg(TKA;$fH#bfvXH8NvFq*==V~Ix;(I6iGF&uB^eW;<}a;YYedC?4mi^Y)Jzi5vhQH zQD#H2z_*kRQsGwT^|{|D5KEPg$vY#ESy%=FrV)Y|fRHMvE(P&>8Gu{V0QF`cE$}0h z!G|<;4RIwt;FOvuF5A&Z-$5GinENkR<|SSP7>Rl(OCcY6TP7{bVs#PQ9XPcNoZxf_ zz7BO|15NLhHBcX8$3pnlttnfcSBsK%PQ7#SSn^VY)pdhaVK0+l>L8 z-zLJA@!NX4lu+5k)OuD|ii3S2bU{sxkx)_4OxtCQ!K0fHcp#|-LKf;qNtdKO5P+yr z8f?ZwkECRVD}}9B%iEoK7M4US0ie2EhI0=|71gW+eTA3k^eUzSakzx6l!$~kZxB7~ ztfR^MuX3|V9pucOhe zLyx=(s)DMF%NCMdshp=>sK`2iX2-~NMJZ0pJ@YH+j;Ute3J|dfQuAqnKhWwBz|2L( z@rIQ$$g_*xT0I!>{q_&VjJQ76p(~h35P^Z3gEt5>*4Anp>1zt$?WFRV(ja(Y+0q}M zpX}wf0|Lj4RFHU`y+}Wc=%B@lvvJK;fJlF&i5J7%9StERO!kXX8|do!FxcQlKtcAO zG`aGvFvE#l1Dj8%Uyy8EEf-dEB1lwCx(IZc24WrQ%Rrilwv;T6no0?Z{m68ol^Vaj28^gPLMF-F2~kTEGXTU zUAl2jv|(N)JZG$7C?PwZ0`E#7;bxiY5(1z#2X@>mzc zt$NP)IBk$npoYy*Y~1N=k8e6;YWA%P(^TEvRi(DIbk%EnozSmZ?9STb9opTv%XY*Q zTB>c7Kr7wdM(%Ld)@XM!g|BRvuWQ`fYDqiY#Zn#J#oIlMy=noscRMckakh5rf>XDy z+dFC_)r)Jkt$BO9y_tx1TSAf1-Iud=Vw2s@I`!4t+}+!i>}0B=w(o8-YWKHQcFAo$ z-LG@5dw0F-)TK=uSSmeq?yXv`+q~UUwr<5~QBJpPotXA_ZQaCE%XX)@YPWD4N4s^s z?)2=-*7mB-v|7~$+9zjj=WLB`%$99UvZ^}md%Evs6jg1CCig3KdJgpU>ho4+tY(`U z?X76;?hf_~d)}LOS?ivSJ=>MYGTYZf)t$ZbH-wXALIJy)aK8Eo~{M`LVvZ#ML_w|lyXr$)VdtxZ{OP2TOzF1>emIqmJa zLVGeRW+F=4-rkZ@5Gt10SC!q#Q%y?tuJ3odZKRT~O!j&@^eZZRY3ok2(-kW9gc};7 zb==bEZrZ9VI}JV7cXyt>I_-=t9_ebUu%DM8 zZp2H*So1Dgb=P&$omK6i zlIwQ2r&qD>Z!J$<-EU*NJ>9$9XNxBp(B06@i?vB)%5Qsn(yLpqdwa7w&qEiz-2o)o zmu7n2%|fDj&_qs~*wU5Q?{9VP4Hj80vwL|J-oo|e?=1Ib)b2X#rMFDj<;L}JZ+B}W z!l~?PDiV74ygRpdT(0cvWZFkeJ$t=&%>^%JLX=lUvI={#()RTD000000000000000 z000000007e>7Q=xK2G0r-nHg#>#g0Hc;Mf7&%9-AYVFkJ*4?{rYtH(8@4fH4fPCsf z?SZ$|Yq#5#v)#KGdQ02Adb+jFJ?q-_z4i{f-0u5!z0~h+UM`<=d4oM`yKC#$y`J6Y zn`XVe*q5=bn)Ir7UfZgzK<2k~EG?Prb#mLi?&)+itnl zNglU!8}0Y!N$RCtG(t?5ekBw=LfHR<$0oU1Z(5hcJ7F^KO>AgSy>5`+Lr^JAK{Tl{;>u zmfCl2SG!HEUhU@9>$&aeoxI-J!?Cs36WL~^9qTx?-L`dY-K^JlbJ)?j?%uJsyT0E0 z+cva)T*-Uc&sDDNy^mE=>^yU5_Zikln^V2o-rim9B&(-)bl7tp+I4pws}8Hao$n>1 zv>mS7cAsAFW}~iij`z20dzsg}cE)WMytL)r*zH}6_3wITyS(RbW~9Apsv@QC?dx~9 zddBkmz8`&n;j0&x4{ZPTY+7<002)pNRc zR=0L;-n-YLq}MB~>hrxl?Csfo?)usG74ke+Zo91J+uY`|(96B99lNPFP2AmXIkoS1 zv(ejGz3p~Io3{4eS8lgEdyQW1=VY}jHD|jXdB)rqOWx0UJ*$%UyVmw@_(ePbOjM}B z6z~8gDXK9QKLaT<6;v2SJZK2XN=-2o_CO$21_dQT0ECnXMe!HkefIxwzq&xdQJVi3 zOwjl66Q-~IRCb#8KV-{)Km5wm;dX~$eoA~9tFlQu7)ymydG5%MWZO?z)pFk@M@+Fj zoUvfVXmPNuHi(sXaXC4V!&o#f1X{g{2&xmubDpQ>tuYpYrddEPhPgPN;D+Z+-06o{ zVWAr%r0_w2%|u6^K)#m1op1v(&}W$$+>AVq2!Xk5}EhTc`u@%Pbhui6aa z($Xv-a=_@+o**i`#XY=7!q6WTvT?4H>xjR!4e&CV!mh&H;qWhcVuNz0HX6-f4v1%8 zas{2lCL?@4oZy?w*j)`><5;#TfP-|b)u3BtI&5xfLfxL%rpazvy?xfs5oGpm1m`Q9 zcn;mud#X9NA^4Ir(*?Dbp_Y-fV?1}ax3#}4ZPbBi;tYGL>1RR1+vo|0&Nv;(9u1`( zC>B@+MZ9K3!|Cx>M)bVjM{z)w4%j^}i@c}F zrTG<8GPhT5lv|4I`(c^7K&|Hm5Gt?U0d}I7id9iqAX1@hQ3{dnbr}&|2A+80`OjA_ zoN?!Ivu9sFI^%VmcH-^2XMgT_*sDy+s;Hzoablt7%3Vt9A(I}!_wD;3xEitzjn2_; z*r$*W2A7E~r-Ct7HeW39p!sGi=opv>4G59=dVs(|%>FT9zBL@w-Ir-FEoOWsYs?BI z0aU83C1x!HNeF-*sSl_?8i`dH1d19(R^-ALzH#j+c+(BQy+G3-O{iBOHJBM8{Dyvk z)+oxGI8gmOGGW(n?ng*j=eB-w>utMn&Rn(Mac1wsgYNOWx37HlcUMrS0kDpK2p5pf zte+~P{8W6(3Z^NA3J09*^gsUKJ?HU*LhmZGwoGLI-fv?b`eZvufeP0s}# zNR)!enYfpNpi>_3G51$F{8ulCNc$Fn(@VBJN*0;w0di+`JI0M&`dJT(v*kvjcXnnt zG19krgy2tza1Y+DnL&{Qk_!PRHRgb>)|{9DO3k4$iH%8zLD)l}N2p`+{)3GFt!sPhQfg+v*raA>NNWi(J~7J%sFupW5N zbLyfpc|IN$b*HL=x529*lOj|$+(&tLG~&e$pxCM@ zI)D(+zc`g9R$5a^*IgW6kp3q!GgBxK9k;o)?c93X&%WN>y5n)DJMTHy-_JkZHt)7; zH+{vojy?#go?ulKep4#RxB5txL|>`rJr~zk?xPtF58y7+L_Vh*CC$2`*gr9>pL*7Q zg0njhaEoQVxJ;8DCu)@gebopBnxs*_ilPF*9{ULm1qaZhmTPbd=p3qWfjudpdu3i8$hjsw09B8&jb z-4BH!5?Z|wrER1t`CKSHMb60sugREIPc5*B3Z8J>gk?nXpj1i@`mF`M-yYFUF6+fE ztlNiSbGr&)3A^%uj8eDAfb0rD4>8iTmLkvH%w%po319?Ng+X>psr0==n%&j!+ucn_ zq#<*LW)zCAQxvwat3FM7@6x7P)-{laMl_0gcB(*))2)OxaZ)iat$j|1s1DLs2Z2WD zDA0bJ+^q6d=T0m|Er&xE>Fz3)gaG)a67t@Jq%nx&x2Y))TrvYp`hxt*+KaWm7n&Xe`vf$Y z{)#O#FdJF7M_+-vtjUVHqA9H`w^w@F3yIDwwP;^l=Gl#CT;^YKQ4JArWE^z*c5Tr- zr-=`j?HXPjQ}G)q<5$Q-lIXUhy;5=xRm3=dwq8n>L(|1@IqlOCOKOl-b=l+R1ft|N zu!ll~1nCf{*(I7`7(&v=V-$?fQ9_iJ-4dCS$hs>)Z}(5)Ei>z^sL*N%OlUEO6bo^l z1j}O(ZBGa)T;-qG0>a#WWYPeVNOi}Oo4wV{+Tm#1!(>XLk(}AZz}7PfD)`|%jyO#H zc$;NZ@mKLsuQzNimX1mpNdUg(gZXEz^pkuIK%BoUzVulLk>12W; z^~vH_jwR_3l1FfqC~~FQsqwb*Bf+q4HW2LNMj$EdG)5pMK?HH3U?&&ts#y%bTGfuK z_q8kp#S31zdJ#-8;mdanQmE(~Y9&l`pp9BV>oJi9B3S90L!_q?&^HEH1>+SBY)TUd z^hp+pW&ugrg7|o8v$|wfcq1t5bJ%gLYu?=R&aH|R5oq*=Gmm9jQjKe0sg*S;yXo1{ z&Y8Zxzj7iXL6KR*h=B{{n{{6ec4^csaV`f^Fcz_Wd|8E<#5BmbM8k}H8R{s)A_V(l5xL4o;|7X=_X}h^JjZ%QMoe6>0jUhJa@a1wn9&0|3J@dZ!e6dJdA0 zjv8P}jkJQN*O@nN&;V9Kur8}I{42~4f|BENaDq0OBEf!@m~HhNbQ;JvNjG%)JJyHj z=VVol7~Ur^d8QQ7oj_-<@OFHCv$} ze<=6O7nk=n8WWpP3Q+dmU~fIJbVm3O4>&!p1a50I&c;9?0i~!*Pu9L3j4ibbk5_<_ zVe`czYVmF!fa3Y(7>^xV(feyPXq zjw?{nbOjXz@oE%G%~ZyFHaO(!38nZ5!H2w9Z;7?BN~R~NR+!~tD#yjoO7I?l@`JcL62iD zgPu~rG~?vKZ#2D5H8&<~JIHOsgwScA8s^!8tikIDMz)H{s+}ryd7#jNmdiA%GSc|B zG_yBg?hYme6z_*wE^Ay>zFdH~@&O%0xxJ50h3ys)hTbv&oY`!>5R6VwLSi%)3X7>A zw_cHEjRcYqc5^5%KZL*#e#7aY&@Gta(ckXuN$ZIu3KzV!M_pTd>CxK#4$p^7E<7yOGSc%M`yE4X2X>)!n!)bLm-a zDOy4Z`bJ7NYg`id&wbUgs2C|Xj2*-dM_1BNlvf1&Se+^AD$s5VDh_Ogrq7-`u|qfk zVCWrVy<%(hG|k^V(McSuU3H3)0*avIDuLr9^^{po1Uc;8c;a8MkkXwgiu$O20+Q>p zzk4;}i!~dQNQ0`dti2teRf5KTf^}aP9&Z++NpCLc29GQ1Ue#J@!f=>`csT5Q4JNna zk-gF5NwfyK0%V4=#yKKd#H_j_;(f1Ft})%?%_*Z6CVSJrTU%rq92M1B*TlWCk20sQ zB?6OU&U;2Cpdm_fgrPMN#h9XeWbWIj1I!&u1dhHnVA+c5Tp<@(gE7*?nnxn&!3dkc ztY&&yc-xa15_ZTJtAsEkaVLe~@Ds=kpQ`Rx-c{OC&!VUr_96%oidQuu3yd4m)%Cq~ zCs+{-qba=iha_kx(XRe5aC8Z_WEGBS`&-9=dqv0QwLS)y`bWAs=x{tY?31D`WQtY5z`!5P}B{np2g2&M87?|VZ`Q2#6 z_Ct9K0J@O9P0Q%%Vx_|#0fA|v3$2e7hQ2%1T2{$wvXG_umx6TL0xNTlh9h`TEJqBw z>Ey3g=y~BU85Wco?h_=^=BZ#l1q->;P~Ix1jKs%RF|`v@<q9uX(SZg&NAbpzba^T0Xk(SF0fJe5-MowhslU zxT_nhX(^Gyi?)Kq61}q8>NM%8gH8}c(#Ox2@fJrfPq$a+CHn$_MR-%s;POyYuWYY8 zBFAD{vAjmA-l>fURAf75`>FOO-IPj*6%r*GPao2ltj#FBp3{72R6kJU<_KOn{HuZK zRpo@3T0H6J4ZCsqqFZiU>%j;WET}k6VIL=hj2|4#a0`fwn}V{j4rz>& zgIn`I2HEB`5mzJzK;cgkJv;S9L~DVC71y}-92Nzz>pXru>-db)yjj4zkekjfY9d6m z5*Q51{BTzsMf8^|`W>`YwfvkjWU_)4ZHH+xrLrnvV_Dt~=hYGOMA5`v8Y(Fd$TWCo zaO7ybc%diOP7!&pic2y)MYy>hOF;G%J2q7&tTn*Y~ zDiXq1QWVUTrWh3h@RziTQHj?DZ*jFEphknXq;t0&+KS632Y%TldL}Gk_O2Q&hFU?G zeXFTiR&9Dt9_i>z?A^Dc-1!CSCs zE&FR&yi`V~^-HZhKL*2bTaq+xcRS_$BiB$%iP2)JFsKW|=zu2dgh&ZvphrBlxPPsYbY}SiKJdvmsd#AvlaK|M z*>rk2lJ%S^i?3?w=Rgt1a{=>^K_4+cyVH?f=5TJ%MjCmIWCh;K&3#hwu?>3Oqn<&= zeFdYs3ttT?DEmdDtOkbVr!eWgf)f|8H{(Aq9!72?AC|Yd`nAA|j2Q6bPmNkael#mG z?v27_?8@+iG9JIRE+?C5m^t?&A~VR_NSW^FF|n*_zr77~o+-^uO5{Va=5F&|HRp3) zPVd}~CgJSxbSOQhk=Q68!m=g{ijSyw*L=XGJlgp+w%XG2?lSbMp%~rcU18`TMkFd7 zegI+un`jnM@`+wwP9|m7q^HhgBb}q*)%iL~D}&Ws_N?$=s4mLg&JnqZt3n%K_9}3? zJ=wzs54K>^e7t#?lY2O|mRN3Wikog}Gl8w?1$Q+T-puioS&f@C#8>ds)+qx0!Znb2J}9Ze%fqRWKRcqEAaE zm$D3(M+I%jkaUbdN3xE==~gX`-+Y!9a7QAGq;XQdQuCpOeB~e94Z5ZUO0HYs+|;}9 z=+8RS3rZtY4ok2sIRA!R(lne3l23VoF=NG&?i zb%lGsY1dW{YuS}Xbzhh*X>k{)u+fOn9#&bn%PQb}K`ntH9ON3C@KZCX3Ni|fn9T*) z4P6ADK*a^Lo{lZ94@Gsm8FHzf_{u&Me;S93N9FtGhZIG0tRPB#YqfRN8cs4zJ`|nI zW*Bc5Q=9~*kyJ@Ev-F?*xaF8%87mjUEB$7~Q zwre5XyR;Lo--S)6XMogI?4B2-xNCrbTbFCzuKNXtUWIoD1gKPgAr8Zm^cO;%bxw2+ zywK<^D-FA2P7v539)|QK%*osO)dx4t7V!@XD{VT|x3>@K_eUsfqd1~1;WymQo{^_^ zQ*_2^2VJbNf`F-5$9Y1=j#QzqGMxMF?cU-}9JJLG_6s6i78$MO3`9;#cV&_IS&UnKz8EXyz+zHc*1yV@F!+Y;LCqm`&NsKWL#}q3NQmDizL;2c?GjkE} zDq7%b9vF928m8rfrbH$r@fFCs0T~Y?0b@MKVJ{AcreZtg?d~%gBU;s4sKq8&#sXHG z+S_4T5XZX#W|vf335&W`5vsy7Z$?6<)|OWdl;m@~-1FNss#qUWC&vS1XH z_JC|t3EAWl_U2Aswwc|iY7VN+kxgc)!&KK}L*=eBEGw@uVWGvRWc>_7#UQ~m*<&&` zOI%k%5Z_$!(S3Ox)EQ0F@scSs06+@%N|n~lSeDTzGoG`5FfOEV2vM?VPMWQvJ~##7 zqw@ADQ5e|-dUWgLQPhz4GUT{`q^p@Jd@ zEU*^gdrnR4xJrlbYnxpfbnA|owsxUKD8Ub#u>$8=?OL9@)~>7yRC+8Z z8D-3za*E#4Yz!C_#PQe&=HIj#IrU%|nq6R3t8nxx91Pa6ZyC^%NRQdinyZYG0M`fx z7hIbtCS#~DxbtJ3?TrayPS>XH|1jR+G5zw~i(5YS1|44UOoz(L%VNWAN5z?go*LC; z1q^O3*Nz)e>9V`V15s{x)9#{H620_36IViw3G2nM#N`6A2yV~=QFo+ASmELGkwgad z74kxe;qnG1xDE{6^8=)t9vZ_tpr}*^&}S-&zbb{SYgElK(U3fK^$?@*N3z#VYAYzc z(cJVCisA(PCij93`wJ%0(PYbyqckL>~^WShQ-C2nkCA6s8ho)!a~7$jdBxTRD~A>A{44 z$w0+)tURLFH0NL-w21C`5(N)mKrp|-tCxIq0Om}zkS3z)_(9_X4h{rlFuRTPTsYaf(R8yakqO3a^@w1G9d&{YOE6@z zw&Vj>wyQnZIu=VR4&KOu&vHBqHo_~j2X0wg+U2DQLKfYLD7g(6!@EO-^^%JM#-cjD zS&H#v-g*O#O2-UpX~Q`UBiNXBavbf4>WF0&rOE|Sc;z5*ymYjrOQ~NY{F`nD)|;gGOUs2f#Ak75(Hw-(&!7DvS9&I z<6J%+K3^4%_@grcScFVXmO5J_5DG*S%?JyoNEvgjp~RS0#8OOg4dE6X>aZ0z7XzH$ zTon&|L##Vq&lYIG&G)eQ+O(W>2!Y0Hqe=|<0jOt2-_tkd`ZwK@yrrS#nU2)%3N^xt zis_Rem6gYueaW z^O&fYwU2^Sp$&~%9ep~4tb|hr26|OuA`Kr?0iU{_xoH~l zRd?#+JsZCC;nlXtCNk?Gkq>FdO)@` z!vZ2L9mM0v`tL?>YI7GXG;*fxbM?8g#!eBWz~mNU5X?Be+na%IxT!0~K~ z;E>^%cD5JoEJxdZEOH}Sf_~+>UcW=D2Dp=B!iNbBJ~9@Ihr)xx-L` zi9(q`SN3|2{al@+Fork5lb>4i_-`R1nw_49G=v;*dRGFcZ+jqM99$zdmk;T&%!&4( z+1IgapNtpSXUMNAOdUKKj6K#|cIG13eE^#BcDjdO7itle-^XQ%pp?~dXyPxJ#1~I( z;3|>_Q_S}X#)yLEH*M%cU@a9_#5GnbA(j^g0s^hhmLjPpI>YD*xGA9Tt88tPDD{L~-inv(6{>?EOhT@= zTk{TGW<(cqH#;(9%T!4e+?zj|X=3i^%O4hm5*sKXsTWM_azbMU%1Bje0d|hF#wS&a zoc`3%PaMtB(PTq2V%%YGp6#ET;&4H)OZ0lf5WpM+Q9wZ_PHr=J=O*mRTaYCELuAKb zAU-uQ8xzvQMbSrrNe*J~o!jmuv@!tIxS^Av$B*iaTIOUkE7E3s!6bnvtT>n12T2hD zH>>6IhrT`Mn!Pv#m10g!$}mOp-WZUPU8Setjo^jugpi{ix9sQ=Q#Ho(ni;8iuAqzc zWqI^yT7x6A&Jk7E?j8)1ndeV|lo|mO9;C;^i`Veo8+w#E^`x~SqFiXHLWeC?^>{_{ zkww|3Dm7!2R@4ebW73R)=Xi}cF>ZWUv(uii_&L0aZp+YFW6r+o195_(Uuk}&(31FX z#aj@fX!~SQGuW9Lvwo0KQ~^m%dri9h7+W!V4YI%!Sh&0+7oHBIlt&7S$D`LX!`(k319wpe{qm zTQH`}`u)a^)a{kv{F}1th%zb070&gdh)k76_H!|w?a~Xw6l%y%GeC9=;+M`}L{>ca zp#2%60g^OHhqmm4cvt|XJBN2fXxLTZc01|^FF+|jcN&g_F;s|^b|2o07lWA|gJFP~ z99XBEstOQ`Mwe)2HDg$V0wudk^K#@pojiE)){~|-H#2YzIRupRd9}-NEp?}NE4;iMP3k4P+#5%kq`nZ|d7 z?3{x`U(64kkyRKI?<9{khH#KyN4DMi-Ic*jP0hU0V0D&t*)0P#X4D4aEQgHzGkW!% zldp4EL`v|3(1C-u(Wa^+quC!ex5y?T&X!I_>J9N?uROhy;3Tc8d0b`ZcAk~T=71&b zBtfv3e0EalFw#)3O>#i*qY~ua+H-*xuap~EBxvPT*VWQ{W(|mTiYg3R`?0jF^`VYY zG&jaG;hgdIDb_fJKreyfCOI_0cWS~7vqj}Cxv*$>QfaMzvJ)0Mz-c2z&l#xRN>wD~ z(a$kzaA6=4S>K~7tOlu1GwW%G_b+p}RSy`NM)ARB}b`pQ+t9YQpe6AeE!SsU)NZ$K2aw-Wao7U&j0u{vP_px~Z3 zWx!&=jHg09O}smTgs||u)jZw}MRkG5hd^$hz7IKOGs=ji8Z%-Gm>Y2)fRIeNb!Sk= zr(${ia%-h*i#z` z;VRwVBL<0V7)+w+Z|)UG2o5zktJD%TeV%&~E`boVl6Pc2v6L+D7n!N)Omq|q3$P;) z8tyKog(N&x%_?1`=hr&tFzp>l#!!m^?jVF#ssb`Fm6LjYJ8NFb^~-tGvXVhc+HZW$ zYsKA}+%>*Dc{bRD9I<|cq8|q4mkm25%k7s~4Nrrc-WMn&R?KX=9j4lb2K0Aeh6}ef zG%DVKqo(ZXgZaL}`{tU=Dho}K89<(aqN0*wf!jf+-78K8o01cp63oGJ{!aUK+4|fkC!rfaaov#nV6eT}*Uhg*v`|HZ zp%5B!WK@*B2J#XTWinC)`jF-B74 z4P8|eoa#ff37!aA4*@%Oqq>6weULwvJ6xWv(mi%njw)LKT=^8l((AtCFe>~2imjRD zosQYk9dxc#QydhmBz|&9Jf|Gj@-L+DfS*MRrc~HQN8E$?X!Hgn%*y+-nlF zwi@oU+;KIKlU34iXG;Vb%oBw}aSTBBWrecTrOb(FQdpFxPo&m`ggB(KaZ%zPascCz zO9q}*{W2lF?tP+i%y!fi%I4=yk$tLUN$0YBMl&gm;RQw3A9)TTue~{u=p`>-&8UEj zTGkFC9l%4XZZbF64g%uNT`v=PR62Tvrp-)eF(#wgP?@Chdv*(M((fhS&q(^hDy5B{ z#v4~zU5#o{$b@Y7X7W1|t7R0Yx{q=1c1}hy)@Gfwxp(7B7E`HGm?nj9e2fk%Ypch^}6{2+AaE>oHU zx1uhUyWXLqW8ZDfrMX;iTvFzma5)TP@edf_oxJ+3R9-yi+9fQ8Ci}4cM#tcZaYv-N zJ)(?5?*T<*c=^K#DXB;>|~hJ?o`C;pyCpD2{GA7X`<+Ury^o6t#A9jzEvI3dlFLV97NN z5&_PS3($Vy#xnVdN#Z(*1fMSyrv_#KK-wraWV#@Dd&+yh&0d`=1!)2mnlkKP1H5~A zr&*~OPGkx{rBn#CC8)!?l5Vi<8_0RSGkP|$L?pBiWvW9_66lRaLQn^qT-{$B|tANBob?k4h6J&Td zy-B=E?T#;?b$&_teFdk5!^JKr2of2JcE=G2Eg2Yg0 z+{O>6V+%&Gq%inT!^RYnmL9U-9HqDuC)f#wt*mm`ZrkeW#fBHJKx&O8$8V1we&{yWT<0qZe=ig+Rp;68Ce^C_`GI;Fv6uKR(Kxm z+_;=89X2jF!DbN~(JQ=4Zq0qx0z0UWupgmR_Gy6H&QEwxz=^bYkysB$vr~{#r4y6O z@-v|+>%R{+S0OJ`ZghaphWkE+)D4Wr!3pUBY|pm=f)8Cs~FTiR+(v-x{CbUIpuX6v>_{%wjMJ&+2uV*1^lF{GxsTqEH~ z5+NzWM>$uFu~rGz*n@(=I6K*rX_DE8&`W`)96A9zg28J?LCg{gDrBmJ(_`V!a2w)g zGkdQ^AM0@s6)kVvH_M^IUcDa+g>kJF3+M2E`pa7A?VcF zjLIh5ar(;M*EA{vg~S)Oyf_x*yR(`a_mQaGKCw51QRM5$nR`!Cg+O>fE?o<>dtiq! zA0T}bA$G6Fk8DDsfaQtqVFh8RrjsY*ew(T-EUJOtz z96rdd@^@#nwGYC;}rhm~!^4MV50_>+t| z?XEl9kCx03Lkg<+>zsD5d&V@SeFV3Yz`Y4gL@kb5z2OIur8a5;N|sq zv}NgEWiwS(KeZxAw<0$o*c4kf&!Xv+Ue3t-zKvEenjk~mvN+Qs#Tg{)iotWE^_<~r z<4KdxRMdfr8qR2Vop!D&blcp@G2L`7c$9qAx)nD;3d0)e$gf61;tE(Q3xXWUe=?=n zFcfY?KMW^Ut*;F5-*PH5Wx5X`nWtx zK!swqo{KI~lP$=lr;yggPbi6<9w!NwwaM~>T3FVWu5lniu!<|A0aR1wH6n}yL?n!ORZ5Q~bG)j7={mWw1vBs>$#Vp$Ns^lE7mscl% zz=nzO##nX!dM}JXz*QpI-Izc^Yoe|(j90@+*GJ1kFq!HtyKKIS1}DP^nbo>CyE3Pt z#zHCtYGt9z!RoDyoxDuZyj>&-u9BV=L#4b-qJm{u0oFB0!q#_LUWZdqnw{ zY;Ktwv4HwZLL-`zEcjIQ(o7`|lPx69@JQ=58!4Srh`5y${@-dPATEn%4|lt0Ff^za z;!Ba+z>liA7R`AJj|-0%9!adc+%Um}Azj|&oA^iu5#UQ^&mqT;WZN#9=s09_PDQS4 zK>!81rh}?u_8R0Fx$<{&yE3TP_bISiEv&W~1ST`l_YC_gXJm;oz)BEP7$4k@cC2BJ z0sH_(cyfyB*E43PA#)=xpVR`sNZwT^%tb8bd+p6`-x=~f_ zLVY43LmHzQ++Ho@uXt?SLKUEnaS%zrzDzwDsmZ31;W8toK>qS zf1Kx1go0^f&%MnahD8GG7_DQoFHG(SbL z2u`m}X291NVF6rXZ*oI8jhH3D*>*+3`U9M0Z2*A50=RUGINhFQ_tY?eqe`Gx2%~SP znvBaQdTllXMZ{Th4UMi7SpFHpV@lyK9W^Y=rDIRk2ZHorp)w*}!q&Xw!KP?MIDww{ zLiOdVMsQJUP&zIo^kmub+DMp$JzEnHv32L2 zIpaL4yaVu{+a%KaRH~K=hT~5-ldz1=?xT={IV88HA86YWO>BHkJ~v?5Tp?%!&+P=6V4cuLDjc=B}j_7 zv~>~zc=qUOOY?H9@kQe7C=H(rnj%hN-H;cq8Izwy(MIC>9f1kDXPm&!ge^9w+jkMh zrPRR6ISYY~$=q~LVv*(L&N>zXh3~TNu&4?j+DWESCO1+BJIp4Igfc5&R#yqR3Cc*O z!975<-bqyaD*lS;h^(J2NC{4&S3>#ZG-?K<5)?!!2!t3`9R9d3s%u|3`W`$CW$rDw z8MlGUb>#@p6zKg22busx{c}SsEcL`7fC($c%6$T%;EX%Uz`elKnUwD;O?hFcb-`bk z7jXb|n;XAx4;;ks@> z%W3^N;z{nCJ2c0!t^q>~LzoiLE4+>cH(YO!hUd1nDv4TpaT;a93O2bLTX<`bF_V;K zv+q4nI%|`j)Z`Y*4cL0B-lU59TwfEyIJk2lDEm6ed%Tq_rmc_Go50T}M8!Vk;O`Ox zlo(8>lPHF(v|M>DADN~=s}%5`Miaq~;W`xPFzcb<^HNq(CvI~#-L9UdqPQw0k0>ud zf}N``W-mvC=#IRas9I}kVIl>&q~T$lDv>jr>6L-kUkmcRyBCIrNl9H3*kibqYa#ug zT<|`e|z`@*!1FY^>xrLhbMC*$*I9AM`%!^?T z=Mf$}vEWWXC_Xpbq|zRoo|Vi;5`tu@5J!6oa@wlZiFes31JlXTZw8%FcA3y`X{&ksZf|(3I1j?LAp1J(7LNB$m=oKs z$~($4__Ei!s>u9 zK{)q{VLdp$*ahO8kr0`O29sdVMr1Hmb*FOS(u5zO)U$^Dd3JX`VVuJevj`z%x7@}h z1PM9o*D6nE4L0V`^_#f}Y21ph5WeEEsU1!40fn{F@%RO(Fxci8XAc{P0l|Q69Mus{ zP;A-2&Fg~F8A1Jb(k1pFpJGo#Y26gO2`oGa<;^$D<|P!!sFwE`*2%d%P@DanLP6A# z*>Eu5GhtyVVasPAF08BzP|)=C&q9!S*<6AY#Q{n!w)n+7=EJ;gqnXUhq?5^D!YUX7 zQbuivQfI~xi*ShH!*Q$TG?O6|*;mo0!B8m^@^I=goAS}Tj7VfqM z?WjPA#&@Sd_uQMI&(LzBSE%>8wGZYIhYem#GglJ|!V3s8xZgi+4=T#x=@E~pL&h9e z>?hQMLm}<61qUhyu`ud8S)dpaq2rXPDZsT^rFh!nJ#-KrG?mMSl>yTGUU^+ey>ttd zz1iUh7M?t?$plz{@ z^7m%nlqh+!LZ}l2&chg-)F*Ijbtr4rWVo>p!Z`GTm}D5RBfAV>hBMvXSpnRutbL7u zcK;HS*pjFo0D|!!i=>)ok;fSw=M9HcKw`7e&rLw57h|zO!YpOhSKc*7zAw&b3Q3AZ z_t%>!!MrXkTeX%QdxuOqI(%2A5lSDuNl5Z|2cAb3d)Y~;>!*6c*UAXO$M32eo?lcK z2VW*!sbl5%N+qgs*_r@HK)AnJ+T!n)!d{&3XB4N<%SE%k+h)^kN~H)S*%fxl0!(;> zn_Gr?)@~B{X&`}dl>i`zA2-JIHOg+&9(;U=ZFt9i&S;e(WcV-{yuk0BNRAF3W@ikx z;V(4d@C(}r#R%1^u4j$7I~|EWO!cpuT37DY&7-iAosOXE`%f-6sw*0Wy!z7CE(h+y zT$EuSRNJiFoxswPnY%}>&w~Z45JQJP9&P9cgqe{&R;26nuxbm}A-5i!39Rx=Y4?rf z*PRCP_igGaI3q8ZBMGHKn=sP`R;ARwXPqE2LJ@vTUi4Igp)Xfmw?eX27_$$Jid2Tg z_{o>dtrS_2IL`=JHGV$ogroZE*J1*hWENs2T}hcW0}L=fNXU350V9x;)#t~b7193*wseU zHJ}r(Yi<;neN@mcr?vz^)eIg21}y80Cl5q=fQ3CtDaSaIMf+E!k1RKQNyImEMd+}NWU zX6YE&nzAocicjDInf#tqV{PLSVcuj!P?ekX%8Q? z*(P6b5Vi~~mvgZy6CJW@z@P~4n%A|HnUVDdp5)%Av=)T6$ov|KEWLYBN)iKuc9=V; zf@2N|;2Sg+8mfq0S+SzqsX1xU>zz9l;_=kIWMQOmrOv=;ewR`?)Pe&VmM6MO!L&f9 zw@pNGS+qT`0@`AkG~A`uwoQ!@`21Hj}+v$+zu?=7a^kRiasU%4RiYyX`bF>W8 z7e(1~T-#4_4%iG?n}~tmQhX;Mv?%lJ3f|n=cavIyAuGLt69;@&`XnvaC0Pc%gNJU7 z^g(H!zWL^Kj^=4IZ-IHTifKt%GbP-H}HIHE$!I~JYL`X?b7$QCx zu3A;6R&Ppj5u#9U#nFgW&vg$aEttdC&}@;Itu;J%HStFX!ILdi^fxEOMlE@e+O+~~`-o>h9lBlC+hQmy54>FfLc> z0fWz10ykE0f~dlQCe0w+<>R7zO&&puGOX~gq5C1+8P+nTUK^x6je_^;SVIx{2=bQq zxPZpQG&-@EO*lm?j^sy0#p2xC(h)~z;%f&e2p!p{h!(pB-Wo1RX7u&H(732@dIl#= zRDkQ76EA5AoKIJKyHtR5HjWqVv+jfnmITTv>A+{|B!^06*gscg%hqc?adlbl-rLVJ zN!{PUypyQwP`@aHfv9qrVlFq!)^z40V!>FBF_0?C z^HV6`ZZ%`kEZ-)%YplpoLRJMwh!58rU9_1erIi=dZJLTr(11Bc7^exjVJNt)8U^+_ zt*5T+`0+H0F-xyO7Qv$VMK^&s7`x6s5QnJmx!6uMVD^}1=3QY zX4ZnV;yta>zS8m7cf4!$l>JM4rKI)1uqq75mg&jNhZgggFmU|R+^7NdfWa~8=`G6p z&M;M3HV({7(A*5;UW2_>I#vs|IVgc+z9isdBM#`q^*lJMNV0adP43)z%w)EMx3zQ} zMlS^TSjhT#yq&U)U9>lW=hqz0XB?O*)I9|)rK3g?j0HBeY2Ki2`5}qfb2-(zB9R?+XtC0YGteq9G_3HpOd{K)4~GeddRxCEe z$R^B2L#18|wDsgh)0GvWRc%b1TmmS=sX(0KW# zP!b;4Pl8X!FWYJq>ESD|Ia7KM2XV%x1se86DDJT$EBHr!bEXsM)h#_95`3^4LXBi2 z!7nP?8$E+rpekYL;$y5IY``Hk=mskF%L*`8*1}L1)~bouJHkQWkE?A%oBc_G(q^a0 zs%ye?)a~VGpzhpd21}nTZf`vf~Y%=-ahHF zgXsgMCqvb8GouJiU8YZT$&BT!KOr36H`RFh6kiGHx4UT^)S^w?{t$s10c`OJx3sOw zdY-y)R@HEuc=#1MvOsnLs#^24l6@QpDWn2mSOwvW5nlIKT)U5n?~v;Kgxa!L!Gv(x zE}$r2YTDy~#uUe{dK;TCs8TUv>+4jiC&dby#HUo?HpT!na-mw}UWM>;f;HRjv~5lyQ;=UZY1oq1XN^wmuged%-r9d#XHe1xq>K5*O4+$gLTkNG~uX{WP zl?mNWd1T_bsQ)4pV{bC?_%Nwmz(dltPQ7vB;x2=>xp{o<4pKQ&(jHNEHM3>DF1lSY zpDctZ#Voc|2xO|s#T)vGhED2eT9C^EXQ#glk zty)frvEd=_R`gsz&;%JRb)CPh_PKEgP^Bbw8Jc!DVJGd29R|Rz7U|pcMIfEKW3^#5 zc|vZ=T?Mm{Oz2(63h9jRj511THp!Lfl3kasXO!JGJXtC6kc=-LZqKQoA>Smf>d!RM z;d@meN%M&-5{o)ZF(F9-6cg`fBe5L-hfUWJL`p>$z%9+t%dxL57Cvgc7EFS4aTMD; zfDY2toqMe6B2!H9XkZSEmkgrV;{9t*<&bSW&CX4JzJWZSDphg;sNsoUldc>CGXTeL zNK9E7167XWMFa%TDS-xEu|at*Eyaln8wZ#P1hIOM>V1LKd2O10KNk~9UK{;$26`C|a7ADH~;)SVqmeaEPWFdk) zwP~6*cCu{)aLcz(%p-kt^Rk#r(jb%?N2@;4aM2mntSlbcA=@?&tVYsr(2@*6xk>x<1wjIIj0){vq zw8~IVBc1BZyE`DlY|~^_NrOu;?;MkFN%L{+Fmyx3;*VCTxb>z&YBZmj+dgvL+Smm0 zrJ!U6=IH>2j-$?fC37PVsrRh=CVMFmEb}+CLzPaGDIap16z;L!52?l~u)Lap$+wF1 z`PjpWTb(%&BR7u3D}iXLI*JQg#Pob!0~kJ=I_0*M=%i$3RfzWk6Hyq2O*2}kRdxnf z%_Lie+4s9`vCpV#zP}iVGA`+48IpvhPn~NA3TQ;x_ON3DsWp3ARfWU=eAEg>iqR{f z%-brnU0=a_tU-AHoUzWh?Nd|ddL=GE%N$(0O(nw_#nY1*VUr}9nf-1 z0x_&~GNh59JUQ5j!2ZuF{VoQ*?MJHhu=BNJ6~g8OkwYuV#8lu=F}LHW9D#EmUpc8M z6$p#28<-Q4GDx)B4}gzP^OM0qTQ0^uS3`SNN|7~!)LU%%-^v^jb&gsOJs!UVn%i0J5*2TQJB1;SIZkANj!(K!LBpCbEc(2Pa6vIqKDOx zn0kElj}3!13Fn0)5g}pGW6dLxMJ1xBQaX0k5hR_n$KYEEO1rlQ!un7&ASw5O1xT18 z+EV9MXuS+26N0SDe7La8TN#q|z;$1fIKj7l z-t76i4F&fl;9uJU)(=#36^9N*X@k~o)|0X*skHAV9&F~_Bfk+<12RsQ@Jz0e3DOh0 zRvU!k10h=7*=WKBZE6@;=cwJ48uM|yM;|{vUsf6!RF|J8XN}bM$~OqmX*Fy<(!B6d z z_rT~k)`Ue1X^!u8M1*v}nPIu^49RA*fW_8v7lw=wg!nWyu-sYE-hv58*jGw~_Bd&R z3R_}0(CtSTm_9=-%~><4pD(ehBdfPO&59fegO<%n66k|1I8Z^A5#&KRvp_-Cz)Bbn zl%*bGoy~DD%n8eyc^vB{P1Vr|`VBVwn2eB>u#HRX@awG*u-}i>RBnl|bIP{Gu_xMP zsQ~Ss>vPgt$F_e(xtK`nrdMA1P5CY*09&_uy;6!y>wb3QtHqchc1ngo?;op6#3>+* zF>THj<~86cY-RSb&0LH-7sKwNZ1MRrSyyfY z@@1*UvLFiE#nK!-Hw0*!K9vw(8z-U8aBFizFUswMfnXX*!NcTM437^|$HXgHJ|as|Sw|Pb+ftOn1yp6|tgYr$%_L^2bv_|{+atnayFLNh zYsd>j+aOtsQpqBH3Gj*o)<;0Ci_mVg3T0`mx}ale@aIF^C{VtTd!i6R?)O`3@w z$V?@RRqA9Sw$bGZ=q8aK1F{8k+d!&T?7@0Z$b1#uurOhuKVXRlLyPuWNY;f+$QKpE zG-+dJuLBhbFNKOvl7z0bFDbKH$aBy=wvvb#<#r6@2SPVrWWx4}lil$yZQ^AP-=j_= z!k9ot&|66f5wW=<^pSBYCrGzRXYpTmzjBW&>{1wl5=v;ip6mSz6m<{@S3P%FIBlE4 zI0*?G&aub{3gDJ$jcmvRnnyl3BIZ+z)t>}pOv`jlJoO@R9w2w3vtv92kzZ97_E=M>Ia%l3*jhAjwL zfHM`|&TLygmX3<>kWDj8T~sWFN=k(HM@pGTp|GKLn(PVcQ)eF5sH*m1Rcd{T_98Y) zfb7FU{8_xPx+=Cxl4a{T9T^`vRT9m0-WuncUr%=pqcOmeAJiJmYz6Uyv-BQtz#Xks ze)p}5=P??u>Jn}=YMYk1n7Y4jRzIw81W*P{Zz|njm24fB@LYcD-!|ZgTO?P|&!Se- z53!b9^&))cPG?>GEuhS$YqzbAmU7KzN(cc%uMFx z1*kR3iJ(B->S57v2&hWzu6o8M>dF;orrvZO_Zue`*H9!Mx|Ck@s2s4hB3yk7hV*oi}joS`asFQou#e z*WkLNiz&}|U4TX5;j^C}#}ykF8qCdoLar`%SZaPWssmUoTwBZ8psxX4MjJ6%KikX; zw)&W6y_U-bmOTTJV;bGnSYBSil2Y0RUBD_vxUobvW5lvD&vfScoD(zt^2EYRh|G3#UNBuvo{ zWm@a-M#lZGN#T_mNvtPQocS0byPPge);Z6H0`zsj#t?2{Y2k zU<|?Zz@9+WCc%TmlNzr~49*`c9u}MSCt@UxM#JTn&jvxbh}N(fNO}8va&h!Jg-RF? z-lf{t3kfMS2c=MvC7!8@0*|AZuo*sK0g`hR$zi%0H=ZxF-4v{F>~yV@H#Vbbwhctp z5H#nX5krc*CALskbd-Bc-TBp5V6D_ObV7m=LG}3=*=p{~#UdOG+;}Gqz3d)tX+JcG45RZ+ zdIr5Hgjz{ic;^(or<18*i4+_&8bbGqIutn}Ge5o|O-H*fr^9HyOs^n-CVI;?e71SD zy<(-pm&@dZY?V6$<;f-V(Q-BGN4zB=R?iWEl3f(=DAuV0;)08Z^v;H(z69I5Ms(;V zf*5Mz+`nB|)#MXsBXTk~drGiENR(=A%2d83NK{d+e6~j*X!13WdhRT&{Hc&QuX)Xb zoSEgEP4H>1h2kCCM)mB|I_Pi@Ety^&1R>Hjc^3<cJUm%1vtMh80)~+;VvN9%Y!E!<>+dUbI;5nm$v3h3D(A!4`dGHfoKODBC0X0 z{Rnh7-AaN&a=5sT+dom3Z``2Tudf0?JGI6F27v+Nk)gX|$@s98Qw_r&vOd8(EKfKT zW!%mPEtE%54ZA0&KmEgGtirX^qV3?OLOalX2_MpIcyq=yM(#>a?F|HEZGXWD2 zljDf?gc%cnMYb=QS*f&_(c9S-PVI&>L3Q~X*51N-&7f#vG3r1W9d>pcm$j^wS4Y`u zFU8@Ia7g4Gobi}*-ZKMakC$bXC5d(2bQkUNtYyes(t0#<1lGO+9AO2t46*TLp~2rv zKuX?Y2P6#P)2gGYM#8gg{Tdzh91O9XU5ZP|L zQ6(@eORW>!HrRAjN|n34?AfQFTR3r|LLYakQj1c-m4flZg!hi4I?oVxthFk!2s~j4 z#x0mnfjFWRV+4Ts$p>lNK#>KEFj~c}#Y-p~vgHFILK0<|zPQ5Ui-VxCyf8r=t|nhk zhr_*Z%f5l{vO|zu8^{#|`-YG{qs|s^zIo1`ary;7h-;14B%c+YI4@6cvVe`q#wb$T zSc&xb)gx-d4LD6KrVV_qfl#nqNp&@0o^<##_Hfw)O+7? zrR3H$vtf;`oUsKCOt;h#AA@S7$)I{lR}c}my%HTA@gf%xrdy+WE?4r6dqzz zuOtIVFu9W#k@B`EFpW7rhTWJd=$pg$DQ8Uij}mAVkj&m<>SNnv z0vCZ@Hhk)#-d{40acB*Yi|62r8_H_xux?+6lv;Y2HlsDv6uK5_Mia^Dg7@o$V1 z2yO?Eofo3#*D_rSN|rWYJ$vlQgGWF}@cU|$YqO-U%_Z@APewZqdI4Jc6YrM&`6 z6UwUF<&9$!NGG#q`6B^qtc?A~m@R30O{se_ow*xcA5E^41@`9R`&b8wCh_q;T@lTf z@LK$M8fY9?BTyU%YD4b`xZ9RJnYgGS+muY*fqg*sNHRZh?>dsGYA_YoizL`R!#c^m z?Y>(vT^HBx4|WQI(TV}r8Bp^mD+~@>wbRAQ()qcyMp_9&WY^v^>=S^ZPacbD`ci&a zETUzWerW8{(8GzL#fqvRb^|-35KQ+nwRqbRXl#KyFpYp1atWCLp-Q$h3aYngiNc&x z0!Ot-0>uGpvCc03=jld>Op#cF+lr&`loFf@bL^DSYs9HYITa=22K`Q@FKYGAou=1q zA!9wLfsI%#daBF?<*APKkZCpq2L=!?*&2#8%frvAr#9n)Mujdfz_RfAvwM}2iO7|4CZGdz*N~}clWQPJ7_1##q=_b5)I}hR|7ABSv$CBZ| zblVJHYgwDk?6y_6ITOKQ-hT=Sl;IGn7J&4w1?1&^C4xt?3;i+ z>Y^e?cGbgA4@8Y!K#v0oucT}0#RHe{okd?I+5u>~`;tJ&x2OvP0Z$`#iJI3-uIp|# zacdM|V|P~&1qevmZL@SOIqMzV{8^T}7z@^IGP`GQD4!)U!#ULp41_z7@MdzqD>8QI z44DXpE-$XOV7MuW?zU3e;_2gKzc1SGBJBYKTz${zbVef|#G_IRb0EuTC z%a0T-7~x%8>9rvp@sFS{pu&l$8{~Pev1CY`U&wqBu4OjVF! z{RmwY*7@<))_7D&L#tC$iNx>75PAs7!;x6P6Km zyA2Avg@bKEb)~+G*{tK(9!ko)35ydd%?77@sZ_XJuoi(Dxv& zkf|#PBy-xXP5E3lK5T@TNlkor%Ek)CFbt~^?82~Q41jsBjk@jN39Um1LK4rYuMt9L zw7O+do$07PgztvR5#71aNF+>G>+u;ujF_!c=(f2e$h_u4G$Q6YR6HRAFnCOrg(0%= zQ`8Ul4F$MXD-cD-T(BEHK+s2G`)KJw+jvtxxXjO^f z0zJfqiGz=n@ZtoaVvqwq3s!>S`Q#E#gB?wcgJ2MHm12tngFkTJmsT|Fyd9h?)woc2 zJ#pUk4SKy`^i8qQmup3HNMI1v>4WzCY2Yxhu|>S4pB8z8C`v{5ebG=(v(X(#}h4>)x zX!iHhMh1AM<%2Qr6b*!S;yfElDGpWfA>5K%eD_&x!O7YX`9X69?=E2rM$QKZg_i{( zrX)>F;K3!|2Z@AznOua({aDNtDRKho;J%Vet`~j8d6gw21iOD^3~;4(j_Q20M}{4A zW#?)Uz502!?veKCLA41J8wi#?8-_(2qRW3kFP%=W;HOX;1-{)KD0;9y_N&CmSa6KQ z_S-nzB|FFx0G}POTCWL2Whm+poSO?C93WJP{Zh8*%?WyhwQ8~3W!$J1j#fPZf~uS- zM+!~d`NvN<3WW7*m$Wy;9qh?M%Lu!Z06e~FZb!W-;A?Au#i~kW+(^{y&zdL1hD95Oo;fHy z=ARWThoH)D*fe$46grEeThNAZLgMzYRWs=u14Pqav))DOP7XoQ0*_=Gg&-SYHpuM0mcXr{qP~e`dzsPKzHvctg=a4>gAOCf1zRA(_xb z0q*f+T4O8^q!@v1@~a3dD$TuQ=G$f%DPAGdKex9gZtpgk$oydKLNo$dO<;Dp9jIyc znP`Aq*+cF>I8+Q4dt0J>s6Pi=xNx;gmgDT>vwlmkAMS< zo&ux{4tzPl-Vsa~#5l+HPJ&NMjcg=sc`Sz(_trpALYFQ<`t!>h+J%KiDHCFxlBa$U zt>73{fNa3DxEK^QLSqQz(z0Sa3Km}2(ZGG%HS&5Hv(vb0A=cxSp%%H8fYKl*VR=!R zAw3aH1Qt{z41I|W9#y$gWu$XTt*$h{Q*G%|R~QV|$DI0vh08*b!eACMR|n|8=92nDpp*I-Ik zuOXM`4GkTg5hxXl1Sz~=y&o*v>sdJCKUyt)!7Nt?XNUliu{xzDPY8$fjPOrNRf^ck z0CR(JChI4edTaStiiL)@<1FGji8*ptda3Q_pyQ;=1~Q8;3%w|A(~`yqhD6>TAh|tpXQ=&9`tlGre0DaDL3l{QHftOFNJMlB1suh1&}y|f*`JO*;T#Tf$an1{Be{bK$%upExedmzYu_Ab%{ApNo&V^uH~Pw+HGw5d`@W#$22 zg#audW4YF9-+HzT!`|!`j-tp9cKAIlOb#sL+r3c)bm%I3R*IXIcQGoBcoE%j8NluC z!ftM&J$yGb20-qxh3*2|D&dF1^-;H3N>J5;V~H|vZjM)EEN7Iy4g-@s$rc-=9Ea?I zIAH<1GH;$5y994dwX|KD4MENa64UP~GL$I&vUXYXt z%sfwQIaj7>>7TnXQv;FS#hW$I4f1{w`s;gLK^bqom<6ULG#3OtfgMVMsU}ei!wjN( zp_vY|7(4sAqj`}Da!%sAOHN4Me_fr(o}RzD!nHcdGq;uZp(f4N&d!n$B!`h67QsNS z#_q(`*nB$00XZh4B3~PTi$O!sfME0&caZ`<%{2xXYVBxU?^sS6?pa~ArmW4HtA-N>osY85xHPP2jtUfBfH3gmzEnqm~7@97KBI zX>I84QMum-A6EfFSmZMxyRpf!-u7JJ!mw`{MAAo?5u6**M(!sM1U0__X;j;dop37iuh?prP}fNOB>5z z!`mA<&uC{S;->Dfd`Z*?XFk2 zfwD8hI~GQ0tR1*!j|#zSu5M@==vEaWk7PnK@!MJWcF(B{X$D72rVuI}(>Ojx^I7x8pgTAVUS}WgnD`cDphnxYmsN; z=-j;rR!Zf}30)5&g&{f$MR9iMMNrpRc(<9`!1>sdWR@dd38wS%-q$sD(Yvhg$YJaZ z;ZtHLfIfQgZ0C9<0jUefdC1NJ#->%=WmrIsm`XG7A%0s2HS~{^s zT8foVlG!stPu3kmhKzl^B5e6clK#fv9T1{{jvCrF_RPi1E1CISEVpQ+Nm8=8`b#wr zWdkBO^abWGV^jc&wwcZqsp?X0NmwIIWL=cgnJ59cmhc+i+%!eZ-pPqL$W6%?m^I46 ze05O3i-O_N#x94ZN5f-T8oe^E#kKKM<-KRy}RzV(}&UHV#ydKItMX|#ggX0l0R!TXA3 zM&fW7Vy$8`7OK8SVnwX=%=>-V`rbD)IIwu+s&2f9;Xygc=kfrn?hI+Cy@N1JfXE)O zVF7492k`$mL(uR}%K-u&qLpVg>3c9*IDK4!m(u5aPAl2eoK<;}LKx62rJF%0*vl1?4RiGG9gL@lE27@RT^r8~&s|<{0 zohE#S>$eI!lv#LhhTkQ-je9_MqT;(BfMmSB9F{B9W-#(Kx^|Ha8L%^z+>fL{>-vaF=ongKNY)iIF%QU-Qh*PX9W7}w5rEJtB)29Yj(uRPnZ zw8}cwGX#dZ5O`RN+Aet~M(azLkpMb&`$rB3WE3_KO0wp#tvh&VAuSb4Ug>WYtA%!y zTw*}qxmnVad^Kw)2KxB5@!7Z+-ixynO3Ain$C-giO`g1^lj^S_VvV7z=;XcD(1m1r zn$EJYE-%{x@7{>R)@o+!&{V7)IvtY&6N!xrCuI4cL`95&)c}f04T}y((7ehHzUvD~ z-r<_lnVc$eD>6l4Cy+hWScqFT?BtBmLY++XiDSJEdh6&OA|bbkxru>9B38BxoAPgO z>KZW+Z)Y|^5h1goeIk|70u7R-BSuJTp59uERSQ>agz;d?h-kZSBC6YVT@Q7)z03E# zi%A(pUpQmesuw+{v1Ps3Na7KtTghAlA|heNB*aR7hg_!0vL9h&=^)TZGY#N*SE{rQ zs-J$qw;>GGlcg>NB1D}Za7x$J0yBcfoFWei6Vi)J+}t*Y>QZvHONz+>z?Cw$0iJDA zhG+{94zyUclie|@aNXy5#4^W>a>C#>0TuGdXlrZ+3slIECE}DU9-N4WECpO0#sHeg zY`WO^B&_lrk=&MVDULCw?GgdR9drd}7~-_jg~y%jT&&g@N7R5M3Y8+h?q8ameHju1 zcEM<|=|RC&NzC4#VD%bQ?@T&IlP^enYs+yMLfYQ631^g?)F6uAqWhLG#K3 zPoox6!yeq=;_!P+H2Tzf$aX~E1`X#9Z5ZGoQP-LfOk1NZFAcXG?I4WiUCiLqjV`_) zmGm5b6sF{}ir`*=cks&cn>sP5oQkB7DTSDDkGOTHWjioa?8o-^iSy2cbd_}FaI|Ls z8m8_YS3AhAr4{Ic)Go-&oQV*k2uAm;T0pOxUw~C1?&?% z3KCV7jY5YTvd#&^9M=rIqT*+sxfI-hlc5fl3uXD~Yq>&xL}C=$v? zM@fN`5ex2l$PkfRwb)GI$5v^C>gbQcYo2nM)K(r&ezzvYnpc;a+&+xB(|Dm48;Niq z$#|PIo7=->=#81uQSX$yzC;;px)-QL79vakg8?8dG+e_uG<3^-V3#kx6Fl{V=#gqaf7ZMp{YQ2E~ukVe; z!*Id}xTxBv$Ln(n==o+iZyh_{cvMT#!s>i|EwZ*csnHys1V%cb_!RS=Y+g~+aA3+Y z6`}V7XvDmSsaE*2;1;$@g+7*4Srwa81MfE&q2Xp9HB3{_lXsNG13il310jLYR4t(4 z1t7%xek;O!#ia|;NJm>`-7yDA6MBp<7NY8b1sBkl@{im_R9>D<>!gD)UE@RIc6GZ3 zXdoo_!GQw^@CdfY0;AlJM6{w{b$bU%cjS$>luW-H4$`E@z*Ct0k58PIk~bR6`!OQ0 zOwXbO{dmB}ZnT7o1T?Y+>r{b*_j>LXq4{A5slwexK}6h7ksDeXNYK$N8p|HTRyujg zS}P?^F1)dV%1>e9=e_L0`-RWMM!mC#Ia=>_J-FVu7L8u(*R}X!xUxex-IFmZ^cEUb zcLg7(J&ogj1>rWb>~0PC$_MDqpulxr3XG@)wsu0IyW2uXgITLw9-Jg2wdf1uJ7?`F&L5!=6 z8^&7ACfD{JzH$p7f;I?ZQA`)fDcd5<`f6*osq zr7L%V-2@HCph8EEJ`JfaTbeRb7l!+Z{HA_;a)*w)I@W%o^Cnff4mLvwpmgm)7vi6Ci3YF;if6s$KJT1KU4}0VX;5FO!#u@26RD2oPR; z33Oe0Z|Z@JDGHQEcNn5=PeKk!jRO5E=StFAa5iy@TRatoyosrF>HX=lB?k(Wh7ZUH3xg!dS8-HvxKq?mBrrnwrBQ3F3XE)baVuSrbjn`W zr?4fx;A{0DNPYP-;&cXs&H5ap$W&I4f;Tx{jhVKKEW zdA{2NRBB6yb_!-uNYVqfQfKjyncd5Ux!qWzZ{oFh)>DoGGSa=;NgzI(T^X zw0oq;Q4W2!h(crR39&9RX@N{Mt!Eu9DPf-qrsj5NUQq-ta63{8uNgvIU#EQEZLv|| znO*dAOyz0#4COX)2A|4fqZ605DA6EIz36N98^~Km2uSZeB77iPfl{SuTDG3z2OQ?@ z>$-NehV?CCm&7J`O6BWf0W5RnrEM%yR=rya-c(OI50}8$bxj%Iq#+BUhO4*jHU<`w zArlgVB5AVAmeck2`gE|i@El1f*g-K*g$I{8)`<*Qgt$PWXtn_ucG81jNCa?XNXT{2 z_9^F|0wvJ(GDh`pI6)l#qC+tmpo}Jio~jQji&W}1v%2gzHDwBfR8|DvHfr@>HkTXa z&K^%79pik8_GE;DfG3B-f+*Cq z9a~j%NHP~vgrIqguRX_yPG(d(%XZdC4-kdhsON>2@Qbs4hIB}mOl!1jKnZ#!DAkVS zaxq~9F|g4}{XA*us;AkG-KBjVw=&w{EL zlfa`nl8OZj?84M3{kqyXhu5L5j$VwToD~Zx?`=mGBUU_Iw$WrB%O3|S>ytRiB{qC( zTr8tH4*4sd42qE;H%r1SU#Q3m>Aj;|tj%@CyDXr!(uS1W6L~P7jfFTB!kXLS@oDa_ zy3vQ0?F+{*!8w}L33d}r*EUs!5PoFLkD3`f&_ZnLrMyp~O99L6x=9j7tf$+Pw*vWL0}l z+Cz7+xr~}@r4EL5jUQO~bo^#p&6bxd`j?I{21QHpI2{`8qay8ZKjQa$knz*QK%`r%u z-mcYESA_Q6Eo*ttctu?_IH8Hy2vL6dbHP1};5}{yJC)}k6=4_wnn_f$S;PZVw;i{! zHK^=h+|F7vc(^I(!!vA$Dg=^o>aOA&1-pjz>J&w5=|0(qdzcPpS&Bz1l#Sq`9f>oe zE=3c*qfKLm4Z*q)U~ORmW8)CP!7xqE z&AAhD)1@@}#m~$jF3$K1gSh>Qr3J?rK1vbrx=c06h%gZL23;*G{S0k18YerWQ#jm5 zwsBFBXTx#`NCHBJdU^Mn{k(cG4b|hfC9?HRwpCc&j|=mnAXxnT*zz1h0`YpK))f(T zL8Jtpl1zSmV{>Kp$UCX2I!N#t`YchOLPJtn(lb#L{oX-Fh*`B4-Q#-Te?g5hZ7UXKmpu-;n8{GOSONNyP+e~z7>AdAmgE5_CmOp3@hT)cKOWmq zm!!LFOI))~l|E|$Bni^Owf1|W1RJoVdh#mLpCdd)Apqli!x>@5QSu#F*t$C8cLs}Ci0nUdUmHtF@YxF?#cx9 z_!}e~S`ra0kb@JC)t4qf@G?h3gW*dIqu$ke1=={SedSqI+W7{dYSKGN>bII6-@hy% z@V$m|N=rNQumf;D%2GT-M3o|!MzOhHqzUcEaqz5NB%u3%qlg_}pgYi`Wlr-uwdoFq z>KnlFV851AA1sWWDVxzFF?{%#@bd71Qm}E-hMJOC+GttK`#% zJCJwgL&es_G@_YFNF9nrd=%iy<#ml};rj~YM)Gobx=VFV5I3a6>Q115UT)2qYX?@W!=OQ7ILrnAU(A+fB&n2a0-p$#%Y5X>#D=%mR6Q1T<|Lr=)ue# zmf#S8b*W;x^k^q(-d{NltnO(j8X9N5KN~?cxxURPXkQZ#MwjCcBPVRPZfr{ zWy5@}N`^;F%q%Z8EtO~=%&_d4TkU7We{`v^Spj$Q$hu>)$;wF5g45MJlB+IXYrUi}-{_(qCm zD{rVzToJyW7F}M5$%=K??wuN5yX1V~@PP87Y!PmXqhf6=tn>TDD`{OP8q%(V+P;3N z$fe3sxVR$|$Q3K~*f2P)@@p=>Y$wJux>*NrMd$I!{OR9g9lT+UQ)}8lw3lr?I6@8! zURWN@aH9(Zj|J>l>9}d@>{_Vg6~io6g3c@2K_SwdNNAn{p#u2BJ#&i{eYXWt%pANA z$WZ?D3+n5)jUi4W1WlxcvPYuk^+P`3`Nh>%Sy4|Lx+^f6TpzG=fjL2Y9a8fqgfepTsl#ZbT_ZTit=z;B z`Cwx3aO2&!M9eYh!%j@HdA9pcAb0G!^#jJ1i-%1gY_|zS z$7Bysw+xB4nXWBF&s}CbZ-)!-%un)!Hh5P=fN3FBL=qG^)t8kqsO>ol3{rhE_cZ z*vy@l_1${3O~+ro#Saq!(8{V5KzgXYVM{9@=KIS6LT$ZVb=a1jrQE;TY0$uaj0!20 z6cm``{bENmF2P*Eh*mY%<$>&>KKF1q(1LG47Qc$=rhtPaT;i1c&#I;>6vmBQw}}(x zyC5RP@>%8?zoroOF>AY|CKbM?@0AO3AVqizIS04sXA)S)y*X8_i*#(!MGWk!lwL(v z;h-{y1lK-k8g*NGnEl7fScl{tiw!6{CqWX9M^?9*#o=So!xOOx2Dp$`wIkGHW7EHr&Y+*1eN!^(7!lnp)e&l95SuW%F)rgXM~%ZF{YW2M&0#a#&r`CSu= zS!;^PO8vM*dt!j@F} zx#JPHCkMr#vdA;lX|c=o(Ze1H&jf@H7u*x8hXJJ>9m%?dmQInBJIQOFYX5c=*45rRl*5Fcr zk7(>fJwIj1yEUWn^K>vdlk@mO;ZkQ+x5FE%^qql|ha& z=*9ww6MZ8I_a@wOo0a81yp%R^GX&Vl3Ux>^nZn@32P0-9n1s`-cjPWekY2-GO-!5J<6)xQ7EMwL4Fe^9d5MP@Hgxv!*ER8^qWy|$ zX

(X}8!?spK67qipdjniQPEP$K;YI>ft&FW1nGamC@GToy!52@QGo5K~KO>9v$; zbIheX zV+3jBa~)KXp-8se5R%_2HZaNh*dr-~iuI}hRAX>8y4bn++}0`(#;c06)sWtp6`M!L zK&*p6wvIMtbS^NrYM*DPd=@e-!veVPgDs_;T2~Q-?w6f5fcnjxdqbFk3; z?t5v>8O^0)gDXv;M!S~#>2t&crUZOZ=2Z=$l)4ptmSs5CTAT?Tn=3lytE=OaKJAcT zm3fSFN#6=IuI8)8+96TeT;y;s@ljogpjej?J~O!NpI}p~Alx$KzdIbnZlFwctRi1z zVaihWT^dV>ZlQ=~M2VzX$PB7DmQ7xq2+QGXuJHC08f3mPGDKy_>-x|jV)g5jS=9`f zH+LHoDLD5pmu@-O?Fya5y1EoRbWDDV-tX{c>f|4_nKHWMhY6qS#m{66;jVmRT#^u^X1! zXzJy)zT#T;Y_n6I!P1wfF;MZ&F=SQ3gwP1YdO4Pxwb7W>an{!I^u#PDY%$g81u4a( zV??n_ypEcBhB7d?QRO4n;)dsIV_%$7RhZoMUBC`N{&#=m_T>IX)=Q zLaVpD8^@f}7@xs8)Ff!wh%MvDefL6LEuiiLuS7UE;B_kO9iaDc=o(cMhB)DN;m5PR zv#^5_V_pUc6++)lf?yYLOf?>$sf_gK-W?M{Z^%2CWtb8Ft7}#0v1)B zgcU$!O4&drOl3~zw4khVY-o0Bl4+@+jiZb*1i*XA1_dfZmE*Q%i@-71aj95AtVE{{ z8Bv2JzCQ`LFW?w=g_kwy&kl2kE*1h+fae2I?(?Ltw&h5{oN*(<71yE`%93ucd8KhB z(B-rWh~k2!2xXn)8TLaEg6W?Jt^QyT+~N>2;2l~e#(L_$;tGu-5nOm_gkpg2QH0Me zx3W5xlh`kzlo@i_39-tO{`Iw6>6Tkxpww9?b8C~x6BZq}mCtcY#h`y_RXRt*f;}`R zQcL?Bd7+qUqeSF%t)*-EKI^!1tLJ9PZuf>88Q z8|h`nZXj|VG5t%T{lne*YnjzG&~U;OU$!y3a~5g5(S4=LAm_M(8OEO5S|rH7K0XXo zY$$hA!LJ2`tW_M;3@OM*o$d4#vyqLb?w!a?e15ROAavazTKROa;?$eMd!+E0-D;J_vw-&gwS1 zA5>;CJN6v~lbCpH?ATJrPL+e4YII0&yO_5mJYBX+Q2TfLqc1WWr}L$Ho@DkmQOn%~hgCz)woP}E%D~b|1v(16;GusSScE>|>&g<-h@*(n zac)?}5H-!nlat=2PX~1b;xu$3VNJ8#zQU~)2CY zn437e@0~jhrrbY#{{WfRT>zgT#U+FV%M1Y@w21hhS)ToSQJgilK-gjdB4ujM(F?8E zql^S$;Z>U?s&>(uVO2KXZcrbJb?-w*Lam5qEbXF|pFPeYm!uGyeibV5Y*lXX z(Xxs9ZJ*xY&YD8$XIHxyO>*meetTM3Nb#`rM!9&aJ+U0pm<57wHM+dmnx?b6TGN{E ztV80K5D5{|QOHfwVc%{{6ti{XA%e&bXVC4IfMUI&eQ!M=qallR9ta>G1g^e7+7;Cq zEr@7AFI8L)bRbe1gJpCEW1v zbMkk&<4B%C=NgA_9s%@v$8(q9Ta9xu-&OlK3$GNd0DOkRDj_S_(<|lbADNfbN#R9o z&@Z(DWyfew4!sZo{iE09tS={*N%|(jLAY0Zv?i+ER87O!2a5IMCr%R_K_6`IUis0U zsEed(nB#Mgk-Q=6fI7FsSi^pgb;(`~V39a;XL=>GC4Lm2m`F(ui7M1j6wkJlQ;*@( z62O8Cl-;Dk5xFOXBBV@>i*z>+<#gA8JfC|fM9totw_u{xdAjTEe$o{PE+2c5W^(ye z0Fw)_09f#Irnyd{scWuI~|fR}RE4eQaiuS2sm~ zlwL$=OzH?rjiNQ3;CZ%drJ0}Bbj9o8uGp>y9I+J=S`kQBnxvqfq!KrI^DLBjqIY+) zE4tUgYB_G%5OZZRNtjMGWY7c%dIrbq7iXzH$J%dAiW-|qP^uB5i4upd})9fuRRTI@8SXfs;N4;YDyl zb#>eWI4?d$foSO<)k}I_s7IDgYBF8(rOdL#+7*BchrpvO+0EAwvs_`Ysky*8by;47Zt1QiWZKRQx9pv znx|nYsEBSa3c3vWOKj;!w9~^(5z+_HI{vBauv247c8g`l4L$>)82mOFgegDtPPFu!aGdm-5B?}-28J~d4;?x zdk&p)w0oAvj`GGKQ_3<@W;Rox&_Io1Ld7MJL3t0ouqFcU!q86&A;w zKRY=}zzl$F4H2*-G&fX_pbTXMiPK_s7s6@Xbjm&gmLzwhvGiUWs8gz#R3TRtdz>&y z%==)uEJfU%b8t#HJ5R;PptwRTeTGIo?W~dzQsstz?TUXnX|;;X3?5@L_L*f#7N$gE zvGDl9s!PbANt`dlL1A+Gq!@dPA<96TrZZC|)@99UUlnmg!!E~6Hnn;P1s&d?J-wdQ zYAHf-0|nIFE5Pdpx5^(OWyaM3nzy6aUd$j7q_Oy{1;a^C%~>`>#Njy-e->M}F0gQ% zIS^$S2oi-L#<(7pfdLqhfa(|lh*(~}Q0a1Fu*RVA=9RnsAhJ9LdbD_`CLI`MFX2d& zJc4x_eePDBQnEGm8#1MWbBlnZy;WhrSD3Sx+j-eE81f*il*S(Sd8B2)>NZ@(xmKRI z_|gnVq5E_;Uq0|I&UpUyovvTR&9(tiMeAm>JKafE^bVVdj&mc?iHN*q#LI?5Ce?Uo z1Yp!jlqAS97$w83r79S73Zr3fE<1 zltRsqnYNBbhm8lCNV(;x7JUwx z%cMoSh1fur`by26*U)V>%jKhE2JW}*9;3Q>IlIP!oFa_we+WAh@6G83UGNIa#_?Df zpw~rDo(|GF0^ps&u@kw5Q$^L;o&A${a*Gz0uCd&B1x|gpjaaw|O z-VEIdMEKBMR0XuHb6?ClJPC--`WqD*FAN#&>Eo_H%SyVVy0As5lhwF@zFHFbTNTo!!F$+bB%qSx8jbY$AJF z4+?jfmDjhOpM-^IkI4iHr37Ai65T|=DwQCcS?fDGpyk!uDD`T-7i%<2BQ_2z4a~ZD zn(=T*A(tVi&k!5T?`X?BOOO}BLIK3(<+hdyGTc;z_&mxHo>lAdvtaQVX5WZ%9 zUuA15*A0ygEtG)f%;j}wki^&UEMMc^6$&oQ~^MHQPR z8aR*9h(1@|Jh#pQ^x1~?PfGaDbq03#ZIElG#{+zgZZ_7_jV+E!K_-}3P2`3X@LcG2 zI*A^RK??E6NYj_q+GKE){-L!zn@r{mQGgzD9r2fwxfKJ;&V)TzW`y_=y)L7+ig3`B zIJ%F^f~B{1HY3a4(c@G-9M%*m8!6}`U8e%T@ZRUTwoV?eD&tbYUk7rIkbG~9`N`EY zVR<}71il`@en-N2TM$e1LV+&J%KC7PKf zdd=}-i2AxJM!kqr?S#u2$hA|`pn>%iAfWZLR@_3!feAg=cW)z#W!lUK0x-LpqTnY4 z-n@t|C%sdPw@Jcw#05r5HCi`ym^xVlayM+~Bs5uKIyu!qhLohMmX4;Lfk`P-(KGb&`Y8!Y#%gGvj<*M8 z^+D5^b)!Ma>8#GJqX zH0qHhJLQxI35w2|>PqhaY7*?d?$GAN#1YPB_YNOckN15Z^+@e&2aTFyP{AFIpl(>v z3l~U?Q}mB{OM!0a%K63xtkrIi*>63jyian#!;Gov!w#LQZ3iXjT*C@qeGY|6mzd&q z7G#LL{>*A9p#ySAmqzFHsCw#M$*vI@BOJ;EnN;t{H}ns#yS3~_>W((kdycglHj_M( zsrJ6nR}B`Frzx#-*`?v}X3%WviKZ#L)fxq&qc%Yh#*i-nYDryghhc}MoFHnt7K5Qg zn|F3wp`H;Q)WaMuVr>IkF~0GkBYNAlvTqI0@Z8>mmQON56M)#}Xqe0@i7|VwdtM~oRVH&6=7G8f^rf7d7&o9*LjbOWp%Sp*etn)yFEf?wrzaf z)CO^>x{ZX^o)DnmJ zf;>BYH$BCsEErr((YQUFvAbf2z|D@(k)v0M;7YZj(0V@dC1c`NO*$Yx}AbdENotg9@y%WO=r^^v|Brr&}M8h2J{yI z+Sp3acu7r*vdT0(x>BkgT09#AJ<;Qi+4t7U!8Vl$YEwk?T7%A|zFWhon(q?v@QueM zM-wX3xn=PaU}wx07eQJ+`oNlJRMO89IjFC_2uy`ere8x~$n$7e!r=lPYdrJ?#(64m zz@kye-K67Q@_I@MYW&tJKDmchFkCJWuo59;4!B5nsL)Z2!# zSiy#BCOgvT9Ye-OIJ}HF!BkM71TWhKmqj(`7^$S&z~^SJz@{ZNu@QGaiJag-s;w0ycYw0b#OMg5F^)@ z+Fj&0&IkSZE`AFKSrUxs&=OPVJw_UGh9&l=&7S6#(H`m+V;_^fS{&a~bn*2O1Q@H)+Q2Lz1#X;% zY}nZ~L>(en5_#%V^zi`Wt(ZnfTzY!gtmI-Z$`1^zw&`s;J#U~d9yV;IDrlNf7$iWr zBq%w}GBG9~fQcCPFjBfjyUU)EXNWH}@sxXSojAHNG6J3`f`=8}-5}61f(-js1(d8p z1VpeDKtAUssUWu@G1;D3Nt-H_HAf)fy0Ts@KG1el*KAqP<=MP%RKrNOBm+k)w}>@} z+PkT^Z`fZCqYdQT5U4>?3|_UC+oZ%<;oC?z0;bosMyR)}r)W*CTxUG&8b`-3&?8N9 zAo#XMaJDW&&e=f$ML=`ip&p}?q@ZmIu!d-Hh?OU|Gu6jLLMYKO%2dKmmX|)PP)fxI z_a8C57&r#}mMZcwF_?9z4w>mjSg#;HR)OXga5R@SQ@P&x=z!^#N^ag~oz(3i(Hv`W z943&s2ZPJP5(DkAHvthCf-o1NZOfD?Fyr)IotCacgPSGmUW7hn)|!@TB`DjZ+6dX~ zz~2-OXyv(x2tOkl1sVI)E(<&}t1eGwye&CoQab=!=i$Ory_2!c>$bShUvc|L40xYC zY#vAo2~)YL0j3T!+^$Br6*ucftT{WJ9511cEh-^=DuS9!2$Fn99Yq!vg~9HXd2X0{ z?`d4NHE+GswaWxXF(^3;rw}F%<>O$pL)oOe2rCtwBsl4kUEWib1K3^&yN25EExM?C ze2fqbGxYKq)CG#0gmOfduC0t_?2_!42J51k;M5Htr7-I8&iIO?Dm`Zl(y|4vUaD}W zqn$F{nl7N@lG06Q}1D_mG~O8Pln@lWrM-$=tWZC<=$=R zENyyyi0TFm72#uIS~#}MdtHaexUnh;M+-sKvJ)&jQAHLMt`P9#^>$lAOanm_P?y~O z6MfMY;8g(y*`r$MvfD|%STQq&L5ApdF;QcVo9@K>mVNGsqXBu8a#E<2po}ll-4-A| zXW{J%$nLmieiYR3=LD1!S3*ANnkETVa|T-B6yB#+$SszP@L#kN1uIT^#~>B_h05ka z?=(h=le^2YsvXMA*6QFn<*XTi64qodMA;E?Iu%bOSKa{#Kb)TX7FIE1r}-_cceW!Wc#N_kr@~~ zMB}vJNr8U#S}U>dNX-;16MJ)W804tUL7uxTxoJ03$M1^Z#8{0XnTue?^E!0Xk)4@u zRYh1gPz}{tJ}%zo`E3|=dh)X#hgweU$``3x6+Wv68=LxB@A;i>1ZFt}?0**(jx{)G|UTb_&IyU2VqTd@>-Ti3`YKIk*)_iqh2W9+mS{PeT~_ z8y+u0Erb}uJY;;$DS&x%VD=iw3lgkW4|90u4OPZLJun`PA>Ulb&eWootCL})Sl+3`WZo~|jRQzg_(IJB0XU%0kaE3#$~5!(8LhfRl+VQV%# zi%JKbl6Mobw>@!0?e4~0piz%OGuc}Kg)F>ON#P3@S`CCr+@-6pKuYm?9{9S$z_w@J z;O1ndrC-wO63vVW74{wYV8ZtF5FI7s&f#=Yh(rW(0a47Bq5!5d2wFs#7^r}lxpE(6 zyeQ65fneO<{6D4#;u=L=vn4y$!!bi5UJb&&F4vLWNucmuJ_Z788# z%5Y{P8I+e6ZpMKNR~sGZ2=<}+PVF5{$J9vMs>wD+2Ic4KhR3>1KI^B%I0UjHvpw~l z??5@ySTwi7hD%p95>1VJ3Ao? z_bK@j850#EzynjYJyW49i^Y3q9wt-}^!##xfFbi;a;*t{u?8*1^}283K~@T$A!zW4 zH@Gi}JMUa!N!jc&~W2dQ2|Pb%6ByBULMS; zI>6C}2e!h24+`udrC1#{V*`R!RlwVftSlR{b_Dhh(raUM+j(uRz!*Wp7Hf{qJ=fkN z%tGJ@Ep8%sC=afccKcYVje2EvNLF8jOhrP`^3~zF0mVkV)3@u4!NTPk5g9oZ%x>sq zd~?j0PBEDBTqWV=Ez}L@)#tNCbIFnK7%Y)ESdRtg2XY*aa>`DW3p=}G8z^=;9XN?W z194l8gZr-3l)h>_qg$J&=?Np&;=rLgdys4axyBt%Cpykv9~ZOu(Cps1%zJ1)Nk3=Q|b_|yUA~L7^R=4!Zyj7^qktV*kY{_ zAKdtp66BQrRH0M}`!*u*3rY&Xv(Ta53WO&WtIBq*A_~iS$H{7%j8nL3uWK%#Bo|hw z$iGr=Zv50(Y2#Lx`U1V7LZ*fyVi0Bxg|;OVPrg4@;NsXnVvl)W6z4P3p^v>Klu=um z9uI17POxtxcD+O>Hxc{o1)J#=Nbwm2Tg5kteDChhiQS;=L5B@Lly(-ETZWdTsD+l@|t~yI#4pa zqN+0{%B7-)HcT{u>l95err0~+(a7q#*#nGfzDEGgXgh=SvoLMT)uDAfwIDuU95bt_ zQ}RV0UI__qdzQVVxTrb?K&-aIy!FT)?0yqkT=lB<2)Gg8YC|ialELHg#<1xIT#(z6 zh)SZbT@BXjM5aYr3T2c3okTzEmTGMt_E%5<1;|clV0x?7z+~OMLeByrx~IK7R-?XjqO=^wiuTcr zXo0HjFiFplOb;To@pv|#+;LMnm&pB(UKj;y-VTI9=a5g4a|R_1+`znC5#~pJJAVUa z6Jw}_Rm=H6NDIQN*-57omQVHd9gcWwf!>sQfEY3#j|7;+Aq!MU;1GgLFJ9P$)hHJg zg2y>^*2cq5M~~L-eF-n7Ro+Sw3i8~#i#{h}tRJRuoEB+JCNrplyam5VCfBxwdGIgB zbll87zVL`R*?bV!FAyc8*co9V&XS23h}-F^*jWYT6};h`$3BA@9^CV?5$_T=ECcel zzE!Y=ZF8ZzS07j-=!Gty1gaF?;@!AHtJ}#u`7oaM{mUPL zMGT(X1d-&hyI45&DTW1Xj!w1%eDJ=H@{LK>i24n=>a3^uJ2+4}7 zQ{drgK|^^Sxh@-gz*Lq-Ci4U;@PQ$HDhHFdkRxN8SlN@*vyO5Ic`_hO8}Z=6cfE)f z&u!)FlTNeY!9jJKj6EwKmUPxt<4uA)M0iygFs;EdByU|+S_uUw>*u35F6Qq$q*{hY zS7VSkc&e`neqBOJu-ux3NmXBh8Qat$jT(Zfl9do3;J|?!3aHMfZ%eSo;=EI5bU0}F zH?c`}EX2!qS2{A}_-4Jqf=xrxgAVvXCG3N&CK5YY=I!?$3*x43sv>w)8kz1e$;*07 zi#(-)he%MY7QpEXuZ;l`&b8GNgnc`pdhf%cU4zt2f3W z1H>i{@wK8_NgB_nMLrH2B_5-m`_4jcJ;iW+PI`G=NVwj+t_Z7{LMS%h45wvCik}S# z_l2}I=U3A;92k~r9i9kwNH!UyI>!l|Z5Iq9`yt7VG!x+p8;+1$eh3iUxib^#r6EW? z2hD&jk;yMvor_cyV2LUf%Lff?Megw5SDaSc#tJx%*vI!Ua;2d^0*`BcM=SDuqS3v$ zHYQrvQUd3B>x*99UNaJ*ZEaBX8TQXUFS>3E-VK!F?PFQ8oiTkot zc3|PXJX{hs(d#x0?=E_LrcW$WH`kQfw5dmO;! zdVtd$5V*^WuRRPH$RVDq@x-ubZxAVHP{npnI4YMSZXD=`w$Bu<>5)aBZ|_R5uqtZG0=SqH%{gYjQo6fcS+zCg zUxRF?;t0!--yYj6Q*%Cp+=Js?!NW}9r8KfD0ZlHHMV#(rXH>&Ll}fvz=&I8Wsjn1$ z??L-rzh;a`af_S0>7}JOc4P&x91J#rNOAL6lCMC=tPN4gwDpB^hRsiacPM&Q7Bc5g z9fKPzQMnz(kBlr?kc_eAV?xo}7?d*=W&>^7eKDT#B0X8_J>mJUR)H5flB<3Z;~S7< z^$%OsxPYzgKXCm6wXOt4qa47P8xXa4%aLm``ec7ZvB`|mE8c1>1qVr69jb}qL-8i% z;PG?0X{HEs9Mxg=(vK+^5xr0oz^R+~ zU~kM|*KX-#-9lY!0G#d5cF&Ki4z+hRX>U|4;m}ssteljAmhE&BJER%(7@FLRI>ifU zv9&OVLTHC#-f_!4Ww+c~#V;bLxR_?X&Vth8n-;I&%4Gr0={*3u}Xd&8Sd>Y+1$g0bj+K%_YZ3bv3yEfTDV zc@N1{YCi{>>5h_Ft6r6(!xJVmJej`{7*o+51U=QW;WMA9Y$V@GTM`6r-fX~X41w|J zX~=8&1jr#JsjkRoNDL@V86DeqZd=g!o~2~3M9hX|QQKxU?PFB+C9RKg=ku4NvG#sCG9`Tf*qPOwX1 z9eg{9sKvpY@8||ABcLAJnj`f~BBch@avrB4Rf~<(h`qX`&LwU&t1w#br?IeB9J3y& zZ^3>MEzq8u2#7LLb05A3--beowC~uoJ}eH?q1q#`a^#V~f@I9#9AOY^)9JGjF{{+Q zZ(wLMExC3&hOO>~5}x&O?cH`VB;CNZLiDjGx{Eku#*dMjx>IuK5I=y)iJekt<;geG zB5cpM%VM7M-NU_(VrY>Y=OS0TiB@49>(1s>DbLWh)+m8dRR6l2cG_24mk8)udSVB%^&wcNK=i z^V^&0_m>g3Xfx;IxYEV9T#V33__P_yGdO$~&vH;IN#Q#pyWWuwX>dY+M7ItH53?dt zpj1T$g@u|kBE)j{j3T;4tn%;{Sp#f*fun9QU}_x;twFd1n8A8 z&J&UI5cxuf3B|ZAT91h)Vwxz%hK{YY=sP{s+f7ypVs~&ba_ubSi6y#&2n7{)G)nE} zFI(0crH5t#2|n?;h>IFX=)xrpFuZw}U)7Kp*g?V9lB6PnL{b@2>=?xMYKBW8lo?@S zw#c_@^N2sfSeLpqOiLskd~b%9Z2$`n?s0coI{~24v%-r(aSKPu!AiA+% z-NP=y;l-fHaC2Q8xD2_hnN3`_;6>O9gN_1y)ne#!@9u<)2uNZmO(l4tXtop3%!roj zDp5J#3l7AKUF0UZcNZbx3i$D-Zv>qc%o_cvrgJXTq{raFVkR9PJexd#B);&r_IL$F z3Uf595mb6%uY9lLcgoi+3wiQB_+r(N1VJ%@iS7?{?;Ggrq!8_Zdb&XoSOM;nzb{u( z_2J})scy&C9S|HZmTtW>_FL$!dpubR$Ios9Nz6RmCyzMPrgyo%6LWI3TQ5 zj}E&@$$rv$F-I&X&0s1GxTH%sXCZ~zjKO(Dc48yx zakZrf07*c$zez^O8wl{5DUPTY2`#jQ!Gy3x^{s=i#}sNRRiQh<+#pB5cx>g5q27Dv zaVoqvx>+n-Mb;kM%A@vmbgj#C_ToK_7kdbNPKb`tOl#>^=`qOcyuTeXEyLYJnXwMb zI_V0|+DX9^4XUs}GA=TyXoF4ovH{8~TqTggX1Bw$C6jv|=Ea;1!O=ByhCfW6l5@nN zAG6)K>T2YFN~5ce@?w<|`7TSW)!qP3u)W)*>*tibl~dJ3GEKuSAyl~P<0Ej8iz44e zk;=kbgNyFjq~@uwic=%+y!6uC+togG0FK43d)@kids?_&aPi zquW+b-Fro61lczic|D`|y$*ZB2!7WA5&^DZgy_WxvV+3-JW)<|1&pG1Eaep0^n+!LmO(+E z(m?sn-Uq`N4BdK}ycbn@-re&pSE%=F7Wa~CtcHBe(Wx5|qEu?8A=z4wfzDpAMl{jx zo7~Asse3LXdkh(YR)`sQqANa$=6=%&FrYyM7qjwB`Jh9px{_dqT>}O>@@Pob%Z7DT z341<%hMR~VB*gOETA1IxD^AKDyr|;AbiWhXrz@lGb;y=CJl^;WMn`Re$5WO|@h+vM z@>%$fGit6=!Ft!W6(TMSE}(qeL&tFiB|8#=S15Oh=#Hk*P$4IXbYn&HnxTiK^cyZz zLDyQE*t%$*OeHGir)`BwSiIu6qBi_zqIl~;#)XaPXp7A!R z4ECAHuIT7UbL)QS*x`*Mz#Po0skMn=T|WX8_|M8Fc_2;#vPloxE)KX8Y|)PIEVEb` z`%CIC9x|OLjtYi@HjuUFg-2aO4?GS!^TC!czgwi@7s{dMl@_G^v#nK#<$h!N^uI zHLkjVp#xJv7Ow+U&Dcn<&0-P)Bljz_Ox!Yp@Uu}B9L02A)B*x{K+`Yd8ppPw=|G|_C1W1ca$JG8Bn)?mJF3Ka@qwgo z0RzO8f5UIyQA&MuYI=}^cTwF=) z@k5bA11DCdEfx%u2DHOS{-d+6PkXTTBC6{MubNRILp0X#Ioo^9;v$`S#ZRdo7%1QU1Dwa|Z`N{{1c{^_s$YV23BNfm~J6CgXU6by@Snd|jeel=dU;L#|}e~dde(!AcFsC|dL zt4QM8N^3txd#;HQa+fjNaG~d*w1pTqj#r8znpTiNo)?y!}GH_EU-IjaBdLXhq1P9(q&Ad`^wJ#2h5O0+w>yjD4zx6*IdTm9VNzXl-c@r%0R2u z>;!;Zn9)+muX^6nQBsJZ3&Zg~LQ=}sGSUZ&0gnP?iF+ff?xmqF`L+Y(XC3uvL#Y)h z?tq3`r0s!u({fXMeY(7tH@?ITL<`u?SBz`Q^GC_=rO-gq6735<;3U-E*G`Jo4T7zh zbIxu__z1nqfX3kZ@x2J*+f(qX%>>pMmu7c=}HV;Zl#De>Pg8>Y&td z&%#+0Gal&5J*4UU>e(h6-JPD$uSA!XL8$ty_g28d;OXKe4dFcUAS3F9qHcFEP8W%D zlJR>C*lpN~#h0p!6;%^mU)XZJH3A$m1C&7lRQ8wC1EXq|9GtJ___GHnJSwrw78*o$ zx=E7{qe}OJS(Wy6QcNQb;w@IcSQeTQiP**iu8Zij<_1YBjj1QQsbqY3NVl#R?wdQY z0mBqH3LzJM&}1}Ow<#>Ch`~;2*(rI>-B|Qogu_W#B8&}zf^|0Vs$!DJ$P!LlAe=i7 z$uXBrRC zaNkJQpfLbg-7jVJip=}DTe-aPW)q5M%YGT9MjfoBmzRc(%wn*6FqC2ss3q5w7riYl znCLOUc;s%ZUVL?y=$|PYg$Nytlz}AKeE#;hfe7%ITB)M-%0yOTI1+IkJu?x3!NtW8 zbDd(f1|=gG2zzYvnC{aH;E@&g(?#Qd1ZTKTMni^7ZhBP;`UYOdgxS|do=dnh zDm_Nm#4FpCpf1(Ra2oI8uVrKQyo-R6?)=%pDZNAN@XabRis}89M+k!l)n*t51-5OM zqOX&8dFX24Of5bO)6uFOW-|g!-nO5;n0i4}TV=y(14T6lxD>NBgyQqEA&o??A?Sl* zM<$ibNorT)&3P(Qg|o@aRjO5%FHeewQM<1gR)GFSPV5yUAFyWHXf8U|C%E5Y9FrM2 zDLD8I8P>d1LT_R+{07TWjD1M%^MnIRt5VfVeFka_ zYZVdcL>@M>GtW>)_PV{4_PC{1KDv~3$Y_OCN&-q!Wy#C=OBZ_Yn8hiTXR*NF8RDDA zL!$yj06`?dNee@@q-8f@>}w1hhgJarVP@QuPC5bB98v<1^LHgTV=m`e! z{Qib8LeB$;{J8X@A<*o>RrH(cjy2uLVN$ZnpU$MV;fTucWunb?kTap@7sSv&I+KVr z-#Av{dQ>mv-K^HVJkjM6CDG}f)oXUu`!T(`mWL&V2;SRms#v54W$6wod9bm!LVdT? zl4i9*c)sz#mR#rw_5@JpRE?F>o%)EwOjH#|X8r9Ca~aRm%7POD8DLx8>Y*O7&0f0MI{hlD-C0egt*l2m4*r?$}(5D zTenLTX-FFdH*7=~NPu6gwrul7kho_v>N&r7)x<%iW|qS)p>`;MgN(@=-5*BE@3vIq zAp}1LuDIRRdGoQ|4128R<2I)#g%`l$2swkB3uWFLW(3nFWj6TEv>i}8X5H!R_5C6} zWl_Yb5YIHDWz5*pl1bo`xugz#sMzsF5V_}Z43EmDZd=3-Xt9diTe(i2RcL1_WsWoX z8Ul(snBk^OJ*4}RYm|v>R6_T!L?kh~jgX5ZW}7EC&rvjRkIWZim%?P2KPMh$l{%J^ z%azGW^4&NY)=yTH!?h(J!#C(@;sXaEbQnpX;p}!~v?6!6jwZ6$G-najStJ$%O&g*j z^pq7hYb$4ro;|&J?A0 zT`OU*cM>GW4)D1^v(x##E1|)y-%cnBwI22}_Pir-NWnNzzmMv9u``n`<2~$fu+sO& zkLO~S?`Ww*QI%QDY=da4qA3%yA{?Tk?01=0=@V>eTU-+5KrK6dR1`0LG!*}1W67V8 zsA9uYSdJkQ`!4DM-i$;zSuN7(1tbj&YI#T}m2O=ac@DCXWm{>Ms+lxOF13t~=nzVU z+)(ZaV@aRiPz3R%)5D#_TT-nXj$k5)y>^dF2PuKl~$Hq`W z33g;0)>4B#te54Y&H=^+=Sq2QI9_Wf?G8R62D%>MP-l-o94S`jmeVBx;rR=@m1vj- z_QOir4I*)CCo03#MO52?#75DMx?!m-GAXAKPE#N!4uk1a_*8;!`?6smtT25bw#rj^ zXguM?3a8mIVVE90%2f{l(Bo*;uB-X&DxnZ7uE;!tf?MpO=eb@zcj z?ogV4Oz0zs(hH{B1J&cWy+4)Uv(qzHUCBtdfv86SeK{Z_+-^zfFI+Ex_5PV`SFs_k z(LhbUV_d=p8Yz{!JCJcwL`8$w?UTbKg2}hT8maOgWmXFe%>+il=2IqfZj9;Mh@=bk z(w3ZFpPaUCl|gTfMypYIS1Vu}5=Oo_+r;-1``>lPoHLhuifMPKpa#Ub&BEjfRa68V zz|cm#JGAs0D+M#{R1Ms0wXCPab`n}q5K=hVT(?u?niQ+I0(8~H2Q}daXgUcwQ|DmZfP5yEfkmmIT={-5M!%du z!qOQ7oDud)j_5s^Ku1J4U?|a;(88U->SZBM2QpA`iEvd{I$8s4_pR_2^5x2NMz|Nu z_@rY?KCqlLDt2q>efpio4nA&q;%Os9%mi3MsC+sHtVOBRv6KrL-#UX95e!_{i8=>*3~aU9df_h% z*B{iKjbUNywH8Nq9^DnHcIs!y@dfFUy(*y;Pp4XGP&P#@hwLkf4`?{sp;H?7XK$o@ zn7OUsiw2;PDefKhI3DL(xpO%t!)a6>aSrCWjk_0RiUB7Ifuq@!Z_Gc0NpK9h}MD!99h;FdMCNZ>z2yOgGg~T#>w?I8=%>p`B_?7`{;#BZ{=Vi)T zGrM+>i@U^jCGj+kWWKbO_LsBv!VywczJ^E>d3FI@re~^5+|Qc+#!FZEJ&I$6YdnoY zb1Zt*Mu_>Bh#l2}NvKvc6+@?jF4ABGdw3f@H(l1A@4nJRc`+EgFv&i*6kcR?nbTFl zhLt67ZgWl*>guxZyV0*t9x)7>y5YvPt&dVwRp&2dqQ-95aZyz{DlHR*G1chm!~( z8S2>T$bjK_3}QsOXbz%WAd>`xa<&0M=>2Nf~Lov55cSEt5Iw9UoFC9`TqN;FD{4Y+aW z?5uSTurPDRqu?X8!Dh!8X8ktLG|oNXOf}J6MK4xOqsv)frAV{Fvt|Y^xSS8PrIm2a zd4r{wAR*d5oq}V|Q|a{^T5lS!P+Nz+jhYIs!}CdP9QK(udyl|K$sIEa2@al%oQ`js z4J2bWgbO{*Ef$X=65EfE{_iwvi>^nlyJ#qtfRMROvVpA+-jS}x;tHplxFcM|DisJI z#G$cM@lT`X*0du{tP1pJxVia5FFubCvvEmLn~t{C=bA5etH(r+O8~J^q(WDQ-tQ`b zTsi!lqL%HZ6Q4?8l#mz>1KgbaxQ;L|#I9f}FGB#O>8|pEouFyx1)U_LCq!s|cLlBu zP9j}5qtH&1$fi2mN}?@=iZFiS0}uugG6LDVnHkV8WrvT;oXPBcIfHh^79MP5a?+{v zddJ*xXnV0t&AFBk)iW!Dinb){8v$-~7e_%No4z~oJRNR#S8||umNk8ZKqrZYRw$WS zj-;`CIl&%c$&Cnd71tu_tM)BdgQr!!?TYPMLOR*mPgr|Y*ch8Cnl_X1l4Gm2r{PRR zNdOIpYFcdSm&vxDFC*^KEJAZ~GZ^b{PNGmm6NghU@tN`a*kX0;2=#uQ~Zb__4F;|?uMR*)|rNm1JaJU0;C z-PhB_3ZcD)(-)mVL5Mm^gn-Kfg#pk6OiBEk-fa6e8`7n0NJr_c;)a-0h#(%BK}1r^ z@P-eM*xh&)!$}uZ#TJ-V)X+}m49Ezl+;%e|oMysMBp@NuKv-#A6H^?g^HUnag7gyQ zJ`D3N?HR|9>OASDhz-TrEZSEK3IwIJoJR_bsPDxF#a)l*nP5zi!3YheprPT@u?y85LUHIGH%4gJLPYkA7-28AJ^Pe3^Ws8JrMzY?lY{Jr?SmG_DG#H8n zfo8(zy2Hes-)V)2LlBJ(DcUV6pf8e=CY1Ur6uR^V2$B2uzt9iP~gqQf5%8N?a{C_H3n{5f?|o`6d@M zBIV((a%r0*2@i#Gj`@6`vn`{rwq#X%lD#a)xmWZY7%{Lm+HVr_0g2#%>$50N1C(Kc zS#6q2bOBoLIZvE|{WacIa3`f4>THYQnpEf_L$rC@>(F|jj&5H2DNVD(TQyoup+2M^ zhk#^sp=Qf#AzyKH5nnr^3lTtYB5Y44xS@BhSf-D~CP+)xV|vhnHb%&Iq>c@US7G95=+e()LtfVWJOhh68!_tN>z#>c>%`AI^ zPfGyi6Ju@NsFguy6H8_N&OerER1LKTWZudjucOH$REmwEX;pZ)G?mXj=0R2F%lC?G z?DxYPDJ;%Bp~PBL^C5zGSV8#mDVzv#O;;3zd9a>RFpA^vB0!O(j?)K*JT&>)?#vv? zUuBvQh=9EqB!#pr&JJ7mv&NfA!l&87^%AMbaefiqZR_-ry#_;`&B?Q{d}0wPr4x+5 zV7ndw`SGUsR$TcB20RMbr;M%Hi)NK)hltYJV^#%WOt~kF4A6~F=nPBr;0_H>Rr2G# ziYHf&mkG)ZWg|6+Y@Z_dn!wsEeBhH~mt0BUw9FVDWAIaOI8IE;m}=8j4r?Qer6usB zaQ#d7piU&g#k9%!o$ zO&N^C2|;4cns;(PT=BjyyWMt?JfRL>Ya7jz=!WIoweOv1%Vc-S*0?hY1~)jGAd)Lq z`1EmjJZ2QOkeYesMx7v6XjjUgG~;L@>0|2p{b78@7`?J1q$N&;CxVPU7?PIav0dsu zIO15#kY)*PA3^JuX|E#5Ub1&eB1n!k=6ABXvf^jSk|3rGOC&{*08WSkj zoTaT(AyZ~VMw)}eCTz_CSEqi$P*4}4XcKVv9d?hC%%$iQn~Pq}YuN|k2p!Bsdvlbr z*Y;wsv0UZR^t;1YvhoB{0|jE~)jEa?kd-fXu{b8#~Z1dY4GR8f~A^LGJCB*#} z?k-s5)us%*7x4|4h#?4-XSw3+{UHWeKOB*3)~TpFYo zlnLsPZKf2goh6*3&0*HBX=mq%Qa%!Ix5 z*zVxI%6-yPdCh1ic!iYU)F+uiTF8>FrX3jRUHZ9r(f4xR>Z(n0E>wb0FO|F4FR0v5 zB%aL98(W`K0frL+p8RZUdEo1$ZX?1g!42fUIU84z-kJ8*JcLTfQghdriXupezGa=` zeq|6MROEgon{w&{gtHa({Mf0rKWQvQM;W=d6{Ac;$BLXKn=gpK@$YyLK?lhgFAUG( zQ@|6k2I%W+auWPCOmepJ4fV5*Vsi+s-rXlVeZ#F)bu8nH~o${QVa znzM2RE=CujpqoXLW+-`*wC)&i;^alZQ;BXILZ#pmn70FUttfn$Am&)$-o?Vz@{c`} zdRqO9CK97Exo$J%I1Ab}1}Df#hzZ0zk@ZG$%`zq?kvbI$u9=(BuHvhCad3;LFT@0j z-M2Jt4z+ZY*!LH_8O$^0bi7I2SMa+shKY})B`8APxE^gBGW=)FC3x#(>_txD&e>0{ zf(y`kLLOpV%UL!^+w#f&MY;+-L2Ux>#Z{mr!vOT~dSHlZJ z=}$usPH1(-v1I0REt6_bQV9$3Frfg(bzO#ok%^0O9izCKaAx;FF64UlqT%e{yT3;u z9BdVsjSEVLd<1$&b?$7?w03h8NPzMo7^DmYKtY z$-wK-!*bo)>2UDSq+{sa*Rjy{_0o=Z1my5#YrM-MEMqM?8yX}FJ3?jQ;GHlND6I|=UAdsI0_T9wM{B(!wKp;i&9rkDr4{BZ5 z6Bu*V)(Nykcd^^u*TZ%az7v!u&qj96+RXNm?+&coQ7EwB{N{GLC~dV(%3Am*qhJG8E_`$yPpDm#JZlm*{G+ zk>ktW)8J%EJI9`c6ky#AfLtb8PmE4X?@np5e7$h9!KodBt~FAO+EVMDxxL3n33q!P zy2l8YDWu&HMt4JwJHTMG?m)fT+{iQZ?6?cZ+lF%4M1qzU+%iYN`Dp||Mpo3DoDhrY zPlREO#I7Wmx1TUiMHQ-%>=;U!$3##niv&1y#2;(CuV@Can^>D&r8zIwkP@Sym6{Rl z2#ke)2AM|?Ia~;5j>dAmz}M7#%cU%2X>{1Ynp8@InG*{^#E9y8w)j9(sdajGk3s1! zl)Uds16M6{{0a-RkAxR|7b=zOOfZX!3X?gOg)rgpIP|>DC8CFg2c- z-qLCA+e!{PRGCDdo4$os!GoWd90s(DZo!3CCjkt{S0F-(E`5vygbA4X#w)(vdj!}_ zrY}CqV_Fe`4f%HX9OY}4KxY`mh;TxWRroxVW~+B;c5RoW;B=!5yg1_y=BYTChrOLb zYQp1v(NpSZz^)kL7xGZ$-QC$lT4?w74Sj{JT;E!R6p**;_KWKz!G~rV#-)b;m z5eSf)!=3xrEfqX3%1WeXf`K6zb)zmE(U`*(*0Q*Blqrk2(*7{L(0FklJ_?|T#9p?+ zYxydmqGs7fDT_U|vr~$tndEx$;x!c!O@pi2+YXC$o)AT;FH1aU42*QdrEq6AD0In! z4|7HWXk)3)@*Y|$=r|1m$A*|EHWY2$X9G%)(Z-L(*o5wFreSaK$m0QK#Mll{Y?MJU z&RAwmx{l$9yQr3;ipD9O0N=et;UVMGQexXyQwF>=f#mPvh0<)dlwEe-`9N%$3;~p z6BmRxbTn?XjP^^`j{?G(d^@B?*T7F%I)NKF;#qna!srf~E9%LrfhgU%VOTT8#d|_f z=60P)x_2SXz#R-zR-CD;8#d??X2>;UXK*MlHwuN}hU|B8GlED6KOGn_3`(e0l-L%! zV~AMUx>QQY4@-9G4npq6c%;>NEaA4yejkZCj5I}`FrU23@_`-FFg4^N?7@V!l?#eA z#3)k@x~$o^o$U}5)=;5NZkd)s7WVHYRB7L$$gokq?Nz$+lwQXqrxl1}rnOjL*MbMd zH`+mL?v|h=v}TW(p^t(;b{dq0HmG1@kEm-3`f%=?)q5tJ)5;hF?RAW;rGV4OAwf7` zg^)_gd0Hjl1RKnkUwFH+3GB-S2gpLCqi{e@ELbl|fLTR=@@rI*@KW;)?_vm2WE&5V zOFGwg!7jimssj2Yq-&8)6d_5b+1FUu@c8N}k{1}1{%%lK5aNKV*6L%y3`zJv--EdV ziPTlNsKf*%*geJ4dRlI4k>|T*^gy9>$igUAXubn{Tko=H*u+CvmEqNr-NCz6)h%nb ze?cQk3`yz`8lpYQ66F3hhc1?%#uhmnS6_=HOBPy3*~$nC;+{z5Am_X~w661TpF@D) zsH&v(0R-y;s%V$i-aee3HU#eXh%D`D0lGy(Wfb;unS@-6-g3hR5n!z7VBfQRE89WW zb21+t#4J`rWdakindgl%>ToC2V`@Of7mk40){k$JqkLZZ0~#K~O2EX`W(xAqPOV}CRVGlU2*Fmssw@MN-G-SSg+gtS^jTgiZ`ftQa| zRI#2-aoL&#sA4)e=)4(HlHd=S-T6fb5`s|5B6+BQ?-!gt?WbnD12?Cj9~Sd)T>4GM zJW>XJXbNR+%HnuIkSF!m9eOQts;;&Qp`*k*TjcJj2#_(5%M>4FZw1lPF*wayq7B9R z+}M~;H2v8Tg>w;KO^XD04~n4`9k%x2Jf+<| zR=F&p?+!0!>y8t6+3vSj0zMA4*5dc;S)X^sTG4`TCvgElBX4T%j`8fyYuYNYGnVQ? zGAzhxWVmSv`9&3WxW1||=Z`*k7lAO+W|k#xgO?(i0x2?p<(VUxJ9GzAJB|csZ5(UHc+!Q1fkG1ZE1C^~Z{lGxmdI-ulmqDm5Ua(GsllKU zQkJkE2RE(OdudYUSsc5vMkI#oX5n&Tn24VJ2>qV2U3)jS9p%gJSyb>PF=YZtY{@gT zjB(FuQHT$BT3J0Ba{%_uz4b1aNo~(|JIK=;BheCVm0%mGUQ-qo#ctk+YE5T&{);Z& zL@VEm;bq zhMzi*=%zKYS z8HohMOCFY=4V+!AB2dVt5D&vAL`m}T{p2vly?FKA(LINrf|V95T1h>z3*CfVJj>W^3*26tI_ zA@&`c&ok2S>M1dSGKdpmfrE>==*$ZgYJ6?R?_m9+S+-Lo6NwqmoxinYv!z~C@uLv|${YL`0xn7t^I;si|9Ccr6X^26>(!Lr3h!xe-87;am9!H$= zQ9+ZnEF5i3jcsAHU<|XZ(gfckD zDDIUy3pJza!B2f)fog`&t*p8OC?wPfkyurM#);oLs}m2vLzH>w!!omrx8F;qYTM~b zo*1Yci>3r}GK_&B`;e30gAwYX?5OLlEa1@0(zG4`4=PhC3bd-#63Av>k(;4+m4;+X z2Iw6JpU|^-{-zN0Rg43?F@W>Gy0-S%4Goa0`ZgS_aCi$RSXSfQBYC{Njh5Kg=W^o% zy6fDAxdo>M5W)q~_{KTp#}8>ax#bPC`p7^In=Bn+;Dcy_Yk+)d1-AiF8(c2-xDU!; z=pf~~e|etoC3fE4;7&m6NHn1NmL$UX zkfd7Zcd3okbgSy|knDhu6?rXEkBM&`-@As!$`nIOND-us7%BWV9ZA#MvhmJG3;VPp&8)gjE0?_s zSVF!<3Z`UaMs8+c)b%K)prmP5^@&%9uNmc@b!44SEeKwdP!{&$AX|8#!$aOTvGyR) zGVTO|51FTsGm;}-n~`E#`U^*}uD9c+nM5%?k?1X;w!-vI<2WPv$k__QQBYJ?|TsjaYDg|n_5 zWq~~JcjR7#mJ!6BTo^qrsckeU*7UO}KO#L3ieIUT597S_aMigK9_r`-iHLgCCA(c^ zLR@iHT79KZhg3M$8+=;43=iggvLOKDG2&rZUEalRdf^%n&~^d~%^VQMSZv4ULOI{Bmlp8F0e7p+kDwy7r2snr?x>c6j zc3}?z)s*=sjI)<9iOxeJZ+tvV%2ExOcW9yuY}xq~d*+{mxg&D!dSHP)=<2YEK5v1( z)|87pW^XgGw_`y82H%Y?qK@2X1q`Evmwiig%KTtnLn(Jyp4N%?@y`(Q{ z?!c?U$%=g?Ze!}zN5*XHu@kPBcpVD$x&Xd_U;+593G@k!Ca4c7=w&($y$@l9R!Kp` zp@1@~=s96Sak11&1~B+xqUoS*Sd~6Han4PGWudNzpd`8Iw`x_>5my|714^aU2-w%6 zsO{+*Pk0?54I0IqFHRfg&?y}^#d8-@gaqS?kC(8H#V*!zC)gjj*8RR9@|2PWRRU7l zrQ=#BrLHWhGctLi!0MCk(rr9`nP|&m%Vun+R!zT-vef1td)<;0ncM8fwS4d%PN7xi zyG*kuA`1s_j`H2J*{;xs3Q5`o0yI^_6-U?0<>^L#+qb8JK;ws`D07I$$}{y{dQd#P zd~y(8<&J(x1qCkNGwZ`oH1ng!aMb9#3QhHaU_N(+-9Aj)FxQkJTRc8d>gQSSm}98$ z+?01^$j72@+frW692ayMO{YhpsJ4C|6YOqei1rkbAoI#pqjqBGyze-^&}-}}?ssbA zuLfZ-``JhdMU`Yd@iBbkCFb_J%rfoA7jE(Gv|Qj)wa=v87zipz;6j?wtJ{MQ#$&k; z>pIN$m4gYd$1&s1#wV7!B>5y~PGB>E!FSb(9S)hI#pzMmwt|enuO*p@mg0!<%H%OA zteMBF3=nO(EjHv7y@%`@eykZ7-w3P7>1p+Zj#Gg^VvxiuN%N@>T<4US(O~|u# zxQ}JW9~)@c7QO-bbVD0^)?|lpE=MyFiFI{6x;*tEV_0Vw9&iUf8GCHHLtgWwT}5HH z6arfUs>S-~#8#`eRt`nzZ>t!VHFw9ZV;>|SYO6#HzD#A%TjpQ3ae5q5&Oz4gl5Ykv zbxzz2-uag@d)&qBCqI4h83`*1iWGu0k>O@1X_6H}rScBF*h6b9PN9oP-m&Fm?xx*! zby}W+kRDUv-54;svP-Wa0l{4oZg)xr8i%-01|Fh3XC%$8L-dQQ9?4MSPWNNck`X}2 z8i2}K!f%GP;z6T9ZtU6_+s7+|Y%5Ht8NS3Jxd3&s5tw zo`O(B_dR;B#dJ3qopG#XU0vX7N%4ExIU+~Toy6xvwo%VzcXaw%t_ob`nA4L6Y#q6r zVdit!4Pq@a=GDs825G7ln#lrOCjn=A-1T`6i^v-C7JdXktWBPjd$N{+t}j!={bF(muIJ)5+w-wEmIjU-Vj#Mo6XIwYxSvcW(PFs=>UjU zM{{lQF}QZmPq9!jB=OJ}iM=uIELH+}SIO5!J1Qhx&G`+B<%7!X!y^3TR3#W{#t_fiu+hC2jRN>~x-37?e@2x0|x@>D! z`+HQenmDw`EU8+ygw4}#{dcR?NPuhX9&4~vz|Jv z;DH@)nzCr3)(Ix{620?k1NzvvueY9(U0ASM0@i0>4;{oceOc_uUJ6BIF0o=G8crPC zB(Br5h~tt{L7PgBEiYmfJ{Mbk$1)a-(+NKsWS_N{dzq_-Oun8e5QQl|Bo22r1jiC6 z8=M>-9YQ6c2bHS8vwL%}A_V&UNOhiihy$!L8xpLT5D=-?7#0ZhV}Z2@S#0ZkyGZwr zx>7{}*4}Z}ixy1pU}a9K!d^-WGQeER^^K0V6=>WlEq+iUXw7V7nqDuq#zlC4(Nk&#WkZ z*k{jj0#V5b*ru619#S}vYP9#+aQ`0d=@WBtL)*+Em^e)m_XP?&cTT{&ffRb@#|Q-$ z{Z0pz!j?WNj}gWk>jv4`P=&H2%kYYhF-gj-4sM=7eL_gl3>gY(*(Am5G3|XumzFuS z)K{?BIR?evhj3bXjlo_nH)~H9#cC#{OCsVHp=ad7%C}O3%8_+Sv`Ydm9DqC)a!xRZ zI5$33PhE=3j>tGQ)!cF)cHtFPi#^mQx)e;&E}28Po2S-FD;=bmg-m7WcQ{L2;Ejz` zy!IYmuv*Jw(+1=F3{$NvTs#8viDGz4g+OSxU8lsViB@<+I6h%P2Hd)w(0+iRYa265 z;1I$`HCn~d!2#hRkw!&@n4}yox|^2EgB|1zP(wmT_cgNaV_fpdiq52MWF!ktZ!r<3 z%l5?65N6r=#gc_37Zr|!jZfUt!6$r++@YRD>#)|_k7tl5Ww_W?yA@44v9^bv)H2LL z1v?~Lx3+}|aVi%?aSIH50tDZ2gBX+nIm{skX7+PN8+fK~?ldqyqH@kzW|CIg7D+2L zWDhv#I3_SNc(`H=8qk7OQ{9--;jBmOF_3R3zB%IWa^3YUtiWbsn65HVhJ=)WHum8gMZCo(A?z%-FYKc2e&nQ!?rTwXM0f1iBYzlW=aA z6&D_^0pFDeNuk10;xPQNRN?357+_!_jzc{@Mu5MJv`rMcf*I!yuw*-4fP=Y{agkH# zhHkJoRRK!eIqm)Dxl#q|(glz`g3+SmyzG}CnoAN=>}?^t^}KpZZbqeValso^yt=W) zXQK+K*n|X&4T{SnyxHCZORCQ#1~&27h(Tti0i^=Ps|s7hATt#{sSWX=l9=0~@z9mW zvTmy-?#X(ky1sZ(i|oHvkgXv}U2G{(Om{hu@hS!o`dx2N&>4ZD&y{4j21J$?tiFtx z7NI>0C_=lof3~Nug zYeoc7i_G@qFto76N@;#>f)m^XE}e!U2P&lRW?@LbFo2DFDTQ#Bwe3E$RWrqTy>mdE z(_}L&otlVt2~sl~8bp14?;leh<$|f3pjWw^4Xc&}fIx3t*eS{1U z$b4l%hM%qwmeG^DJl*HRJZIQ8Qq0<5DR&c%+ASOQf;?bT1b$0X+Uf%OZyrzMrZazOG3@BvVhyaWhFy7v7Fn0%EsWZbu*rJt}s1PG! zGaJ29|8JK1N$)#fYgAg0eT(AJE)jXzl%sgl)Z@VbrZr*37?)_lC2w86>?U2MmXIU| za<#mvw29-h1NR!R;le4QW4N>3F%W63S#8-x6NC!TTwln8&M%PM_lfGusEAAphnem9 zMQKXtPO)yXz+soFiVGBovpv2j;w|)si@$((40F+$jgA9e)6=J_un{c`h+Woq2Yq9R z8G9Y`tn(V~Y^qRg;Zo9&phK_<506z$kV8xk*^}m>fek|LqvPU=$15{&LG?=~g(dfg zD0s>Hjd4}FH*ZhchgOTaORdR<#t`KLjoC7n-9Qzq5x3=p*z9-4U456?D+?g?^J8$L zXoruson_Ti96ZGY!I{CEl}yRZ8f1eI);z0P4Rt+$yb`F-y%E|_%Sgp&0_!^)*LlI& zF{TNiaXw|sT*6!zd*iTjS@dN>^m?93cuSOE;#bE+;Nq(N&eWpRUfY1BP<3dXq)ypt z8rd)!(b%^fR%uiE$#S&PgFbcX&dzc678lNO4T~k1b{9=UuiB<;V594%Dw%52Bdo9y zixRBG(eoCWX5TEpFgTgPLm^^x_hW3%p=Fb@BwHFoX(_fcJNHbBwq4_CL2C!NX{?IA z7Ae?fW7w)Ul_2b=?=!j{H?MC91^Ocb?QyG;H@AGtsO_;XM$$J-QC1P@^X!7og(C2j z<30-3Hd~PaV>`%ToM8j$@CY=Wh!L1CG|_}#T5GuIF(}xu&iUV!5J+f-uCyl$TTG)- z8MgrWouq-s4NXc19TR%njWxFujFDmRw+gA09!Aokn1ahOM92*k+^?5dp}e;Ht4jHo z1oqaexm^VofmmoJZzxH|mqm{xj&i(5a)7VSXYP!uYn&Hlp@db|z_Ca&4M;VMT@GH| zt8{wZR_J*0gx*ZMC`-CRCUR3mRa^${^vWNRHVJZA&~UO-Es6|+K5>Hs!8)zjo-UqE zIJ!L8B}+#gVPMK!IjnH_QeV4`OuS%O6ue23NHY+BDR_CYDdwMDY&Od7MZ*Myv`JbF zFV}O?$ZC7KsuFz}sVqB&*6dJZn7Nj$&uL(Tx`J&nYQ`JOtRVQOVMFl*8PsJEgyEpj zs}%_<mFk@^!JbC)WLK1sv{=m`UX;=}kgTdfs1Ed4mX1d<17+pt+AxSJ)d{K4k zwsL%kCe}k5h2`ztNfNtI%7_xazi@J$X#=v{_H48l@nj1t$lc~wpW-KIsFWAAHuT2S zkoQ!OP^H}1@mg(VuqeVFS5;`JR99COo6w*vT0_CUS5E^DZ>mQj?F{BBl*9JRTF~J; z?{=mpXpT^nss?FHO0pfCB)3K!CseKYRipAARuZmc=cq zn>GsCQ8ub=32i3TTS#ms!VQaM7SKl2ro@{h+HGu=Hj!IZwMtt9V4FzVD{4)Wt(8rn zn?+kv8&S0-wXj2BBVv}wCd#%^Z6Iw@wxt_NHYseN*+AM`2Fj+zOJKHzY^2y*MYK|E zLA6bkHb&H&MGc7AD%vc_menh23urd2s*R9LujVREtE@T zO_8xA*(+k36q`y`%A;V|LuyNEp|qv38wFcQY)Ne@*i&R|sMuD;7RehDW|6f`inf$Z zs#_&i%AvHX+M;Yyn6xyn6QLNZCkjB(@ZrP_~gap|q-PRT~9On*p*mQf(&2(QHC(QZ0&0X-jIAwJnHjjjE}# zTR~RRQL;9M(3>D^6xk^@s+&bt+eJ1LWg}>t18Ef66J=FwNwAi}wwolagqte1jioJ# zwMsUiY!S4jvJI*Yl&Wl`wuv@|!COOQO{$|}OJP>ULux465w$C5Nwh( zuxzB-2Ef`?u$vGzB{o*o7Sy4z8!2riuxvtY4Xu%~gK9|GO^P4Tw#omd3&jg&P7JL~KE|Lt@$# zwh(Q#Euz^AYLT@{+LGEzY@*navMq>~#TLX?!bZY2l-jD=sb)sf4TW1)%VLt+p<7!- zt&nX&vZm6O$yUO)idzKP4XTZzwjj1t+E&{N*;dHeN^C0F+KIH0ZB$z&TM}(l+LaqE zq}mqJQ)yJ$N^DYWM#_}hiZ+djTNcr6L~J%vwk5Gf)gxpk)lk|aWwecHT$fdPN*dt`vk+dsh zmeNCLmdPf;Hk51z)PrJH$fndSl~ipCZBEr!&qXq!@ORkWhmY>8|OVQf~`)J=sA zibl|*VNHa#qMKE0EtMNet*Y878%VY&woz5I6KPSj1+a$7Z7SInvR1(+)iWb%rpTKS zuq$CLwko!&rpiUMt7zFaq}eT!lVVMZjfGQUBVvuIHWOuR*ofLvn+dY3Vy4hGlC6@q z2GZDw*r{z*8!1M~EumXf8&oT1)SD?bM#x6h6||t(Y^!LEqD_dcs#`6UTLrL7VOs!g zDJ`~w+ETVsn@SrAHkDS&Y*g6@Y^t`EHmOams7&dO_JJ@wx-f;RHn_6t7uzWQjMZ)EtG7E*w~WVp=~JGL9#YYsy0oMwy3sKWR%%W zsGCr25wTXv*nrr!rEC=1g4mN}2GmitO_ZBaY=pLjvQ@Gu*r>Kf)LSUnO^R%kwneoC zvZUCXDvgybjf-lnp|MkB3uEwRZ3?y`*b8M= z(JiQKsMXts&4q}o$xHj%Q}t7xIILtz^t*$UWJ(pJ*8 ziA|$uNwSLAs%(v@qiI!b6J)jvWY|%%OJoMfY^vE*+P12j6*j3$Xq!PB1x=||)TOYY zvJHu}lW2y*TO?Z>Hll4JYy{a`Dw|QXOK937YEiIlCACb#Y=pL@u|sJiX$^@@l19>- z5NuUVq$_Hs%57B}QcZ}qLTwGIR>W4#khV(MNwOBfv{G9_n_EiSNNkC;qS~t1hRCMU zEvUAtp|Ys9lx(QBRj{q9mdZ_%Y@1PS6KHKjwn4JlN^D6sRBa)(O4&-bsBEROuh0BR zDJ$=urtC`3_wVqh^tArS_w@~b9{*4h3U@4wucsTSAS*3Ikj2IYYoBoJ5FjjT2~;|$R7qzpD(m1kLWM&0Dn-|dvRQkA0~-DBI>H9k%S3em z#L58q+#zES=?VfxN3h5$Uo5LOj1k57rP1U$(}m173+NzbTfRlWGfBaw*GT$Aw1nIw zg2C0K6teO8;ib`#*xnHIUI&a^m(Y;|oxN93+V1|!j?@Ki`LcgP)(SZ$4O z>cI@M2;)wze#Pm*a>s0n*;x3#0zic&XfxcMf@|QGny`T}`FS(X?&Y)z!vu2#+@o%# zPk}|uGT6_p=8SKbJ!V7iLlqL~JrRJmGl}s?G&o6=R#YX}-S`U<*sdYvLbjCMTvTX1 z1xZlBUF@t;4`4iQvXSQNrhe)E-R7^Ip{H&UF`Be(xEXaDHI*KLx zNV1_Bl`p!X2o*!S7(%7`iw>LRL~Kb=fRFFKgC*||%~lNy0uhp_V+$WSm(C2OemzgLL8oCFdR2Z4 zP!&@cW`;{seAE`eMfz1SG*10zzWh9A=Kk%RJofv_e{JK34DNU1mO0vn zszRgg{H^pOSxlk!Dli5t1{#{j(jXR2U?*bDSpNiDTqq!bQ#?Vb9$@YmBb|zqLF=5a zV-3)NU%h#I_qishHX?2p=dATtx)K5dt<%8*soga^tN{f@;X*>5J-MI6*D6dnrFtQd znMoChNyE9|aO7M)CT132fMRazI4a#+yxnFMz6HT~EAJ{cIgzzEz_xB2_oLYRitvV> z;+QCz1%1t;KRieXcthe4aryv5V;m3|@C;ieyF3UzWGqt(h1>up@EYP!FzUbnV9Aj~ zqEnhA{e2yDjA)ezhpH8mOAL4I7;@8?+|94h9&=El#-}|hVUxkgxv~-zk|slO#Zuav zg*doii(vppj6$Lc#i6VqJ(>3QLmDqYJub{k@rM+889fqPallfVw%2H)P@-BYIZ)_o zv?>EC`KX{tQSvIFRaY3N3$i~ONXrJzkt&WtD@RC2L>O30z$79>JlN>@@IXe=fVK;r zlZX)1??7dce+m%VcE)LW@%IzB4BiI+!8C}vfY(BMh>RIkBZ zzno5V_QQ{y4|v#m^~WAu`snYTetiD->Urme&l`+pwsWg*8}Hsa!?%7LIo!D8togZl z;OFmOXB)o#)%~B@>s@1KuHL-=Yi=CeoA|{k-usF8M_9{<>tfC&03eBsATs(08VCv{}PZ5lp`q!r&X0OV5dg z&&zKdr+hus>A{f(Cg7+@<2`D^+wD4Cdc7gVWN{3%U4&{)aD{nH7WRpw=KD^X3Cjb8 zSMP|O(+3Ofs-3^I3SXmCan+d}rJODyRasq>?^5S!dI7jK@C#AdFE(WH+P64jerJdz z1ovA+UMl;Zd*PQ<0n(zHJ!nR6s8?rwVZBU29HJ>Oj~1iz3u{R2t2pWda8o6z*|KWd z7zxDv<;Q7Gjvm$A^~Y^lRZ*g4rmt84Iul6Ky%nmU6K^KcBjfTfugt2tU!H<2@sXO5 zrqsF4MxW~_>Wl~?%&69hudGh0Jt55S@PLwYe&I>D-!(3g!cYg%((C9_Au=ai*0eB?%TC$h=>lAzYqdqs@bu z+wOFx>@;jT3;~QEXuhUQPlRi5OR7RK?gwWq`_oaiyjmAh@r@mH?gY_GQie^7sfA52 zP1jvn5wIkW)63|FU@&)`>O&_m;1WPFOhL1Q#;65h!RSn6E84g+QWMv@S39@CB-2$? zUbNoqnn}GV=R^7g`>R7-j9P&8LKxHMRARNY=Uque8WF?-8>NAb7(m6Nr~?a9nk8%p z7fpDzmJG`ustwQ|#HVKXgbjAY&6;8ob@zDS-8s`?0w&cj5&4gs2%DNRua{Q;I{lwe zh5Eud&JE9j!OvtE9)u200~O1)H2J_;2+>Y_~sP^;lgJnG1lsGN&K z)Q&<0jHw=Bh7ZO9KL{*K^8IR1KRgV8>%jTGiT1eC=4q*5pp@uk5ip%2I%Y=7i~!a6 zj$UG3giICEiAcDFEojX)a8eMEIvloCx;1gm+)#W-Y-bQXl=OvPWje;EGStZ2@x;17 zpcJrn##nk68J+^I0|iB%xM^4Ed`mLJM+`%T!Im?F+dD{@W!F&{0#&FVMXx>4dxRN$uaX+KNpkJKPlfd2*tz% zavblwP^&v#Zpz6mi8Lsk-ipSj?a36d>%AJ%kdl5eQW9rIwH{G%iKi9cfkG&ppu$Qz zQTkvwX9nBF1M=Tv-n_cZon%fjC~QodCl(2pFFwyz+VHuP)Bk*5j7k zyX%Kraodi&Vr_yZkwUM*lFp=gWV^p3bd&9uOV*0?Bo@u=m3M>yUyk7(VB{lu9^glL z6s*ZU#I+Vxjs_0z?PviB!U@1Ecs{zC7{6dY+8{tIGJ&2D*U!^TDSD|(Fpgm_&o%gH zL+_vjos!@$nBRzwfD1jsNH%ktji`&*p`BTRjUe54_rJaM_0z5AJm-CU@5SGi;d7p| zmhZ=0@w;-A0Wdyh7{sG2Nck08L+N|AD77x}7s_=&m4kPXbO8a@>uVrz2fQH168e6^ z_>M6E*qzlB88PEo9fo2SQFJTl;dx505X|)YeKTI3zWVjoj{Iif^L*#$M_%}D9ABS# z@9V$McE;0b<3F^y-08;J<2A<=@^^6WuD7imG21qG=bg{yzUOWEymQCh-}{cU-(Nm< z?b}>u=Nqi~oKJg<=a24v^Tyt|_q*=8;m5z6GcUckoNB#j>qm|_<5xKG`@^-KGV!@} zZJlm6TjuYU+dq2Y`t!rS+Wpt|pSb?dzkG7q&DrJGU4HS~p1I-qzg&NK@!O7eZ26aL zuFZ9~93Qyl*5jVOZhHOg^RHig`2Xc17w@jUf4%eL$+n=`5TN%|gNNb#GvjN*Y2Zoy zwgtHpOhU_A33)yja@cIcWWaIzDh!9m7#=Ij=3qR520GM`_Ujc22o6huF3iAnJydKg z!ShYNYM5hJJG?f12c8jIQ~ITjy^}yC4nBLXUg=CBQ-sl=e=41DD_VNH4#Y-_P%U0V zb1lP9<`t4*(Uxu$@dM`S{e!J1$RR1gt40$&8}nK5)3l>7tv!;khT&iw!?EmL*r1AQ zK~0rmY1^noo{5EQ%7(Fp*xQiQzVX1+#9nrsA9=gkP*G%BV8C!Lq_* z6{vBpcMll2y%N-}&wF`MS zpo#>O-fg5WU_l_;VK6)Cos(u{&(gZ+jB>dvkPsxHnt^GIGLT^Ue;=7%+GJ?`EC?Ng z7XG^qYjt7Al3yQJ?z~Z#eo(=12C2q5L9gwxfOnlVfK}oFK)TqBVbB9 z=N^%qpw_rcxef|AX!Fgghq~x}_4lOHmr+M=*PT{z!2KFg!P=3@aaZB%(zQdvu+OwI zj)%A&l?#dGDx6zi(!8~l%Rf;XNzuQCUn88g*x*9>&*uRL_c8~#q(sO*)phmMAa)9s zImLvo$d2O`LZA(0X>H!^);9v=n4U0g9B~ovwQ!iseXTd7*S4${-rCD_fajgs>w!rnSZ zmrHI3WCdPwBrO|fN#j>;$v3@Y1oQ;vhtWgoT2!yKu*VBfwCWE~3uR3Qa6w|q*sud7 zWTW(T$ccz2g`f$q#$*tJdH_A^41vA4+6`zN$lG2nrBrgRAZvpm&aUP|zqlb>hyk$e zhYFB6ZF35}dc416(dB9mpBIfZxO~s0#zSUB&M)CwMHs+-GY?w^xnmdI^^ko3Y2vRz ztGqH)l`hy?g&|8)v|9z}s4jA+WDZL`a5cuGaakj-vgq_Hy}M&9_4etCf~b1~i|iyj zqFjeOEGzLPl6X%u-7J2uBA?94*aK@HAwc~3g)#^(ZEQ|sW*!hxvG16Hg~nw_d$*S% zfceKPx#80q2c`X+_g?*h?%YmY>D;pmwceofAVp|ocj>72J23Q>2{U;{1{ULp1^c-j z1lYo|r9N=0V>#p@{Z_@%Ba3gVGe-EDCX&<3xeT!(+B4P$C)$Gav zHph$*XTXsHxZ%i$6ORq~&wy)?m)0Of!IH9TNNHAd8B&3@l(6!s`XEvma*)K)a21XR z6~}C~RH(?0+{y6M)e8m+5SWEr1r7)>(BjkZ6g-6C;+>QSwIvela)c_Pr^@);gygck z0{zl=hdD$VNPSB7t3;^Q>_Go=@G!)K4?5swI7<7j?RRM4!?2wBDj~=Y1hPI*YF#Zd zvA&0iNu{3I8b55R@lsaQDk4h#W=bWOFF-m;YzPG>6&W|QDdd#*g(K%NG?BLC!M-4X z5V)}Y3iZI+`a(sWZ-N;9k`oO3@J==`!4Cg#~o6Eu%HEHg>t%n!kgmdm@^S1E4ei$9-A>c z9DDbS#0N92`%NcE_IJVq*b99v$_8Q%ixuRJ;kt{tk6|bk@ebc!C1+RD@=NsY3ce==w(Sju%T&7gc1TOaG^rp);yDyQu1!2^yRE4 zlqIiuTrzCKqiho9vJ~7z%DFLH+HB!v0suCXr|3*YMqQ|0f|D5hE=uu{I&J7wGmx4P zweJI;+d)VmuNPhqh(S#&y*e|%i?K|`=8fRT1?n83w-Rrt76~rO@L_@tE1@TYCzT2P3mC@ByzrG;y3)p@vy$@eZTcVy zN*B&$G{t~VP_#KH#IoYYZZx1wn~4-DSV%A0kD)&HdNjs|+fgJ|iv<_U$??ZA=YyR? zCZ93gGf&F+=|wO1kzX9rg;8IfoFOH<&Saj$%GAls8Wm5Qy>u$SWtJfo0+be219T|v zkVgijkPZy|OBjBN#JS?j<}nDa!D_UAYH|opVD2oUDiDJ}kiH(Uj;OY?rt2#3@2j$m zH$J|fCHO)_k&`KEVFTq!($tSsIM6(HY1U||blLR5&$X%BBH~yGw*Hn52_vYHZt<+G z<3A)zo0|wt!|=in&Ob%i{9s8R0r*4&3*fxbs#-EW%w-`%~vV)?Gk=ePGizdQ2+Bq(5c>4){c36Ea7l20-wQSMcOEszWw zEfpr{|A2I_?Ce~LspF9}1TeHu1Y)`as$e7X z9c6LTinB`Sn_!QRMYIxT8Cg)*?>c+T0OTrCGSVcAD!jDNknBgFOMO|Gj={xiOCn|v zOSdNNG^nYHlg*K!n6p5u#0g1l69>wAJLzJ`W#PHh~mSv0#! zgKK+15fDJaIRS-%eh%fci{+E;Xokx^Swd92C(N%hmV`F_2oT2D&_XlU*+9-&Dq*+? z6DA82rpH_ah_EHYRDAeE_&;pAA7ePE+rx4%BT*=6#X%Ll6B)97%~1YI;&rONzG0ah zGho3hR=!PWXGN_E0WxQyCbJS*OQ@bzoWcx6f-d3#M1D#Ghk0|k=uwPt)E?@lhV;%i z#6JK=maeuyt+N3fbB{y-NQnq~KKjh`grOG@!zVGxn3hpIA^J+E9nm&%YHw(2H+o>4 zO;AudjNnSIkt&LCRzfN-AQFp%AAQ{cc365K5t>2>U|K1*vWp0^O4?VfWh(vjs!2eC z5Pb~n&+8P*0YyYHIIPzc&m|oeSGM!>=oS}$1x{VPrsIlJ0~A239*L7Nt2O6DoVJNb z_zV0cB*Qw8DgrKv)vWk{ZETNcz|vX*BoD?J%c_ojG7fSy3mPiJM(bQA9(S)@x|*rInFLP*aRb;B?NWl+G8sHcCztN^umNdsIESB zP%Rf`3L#Xv4gyiru+JGM0mqjrwOx6ntvg^~R~35YS~Ry+?^( zH%JRdL6u&tt~yYvH-O3SWdM?HWfpSdL;BRz(ci?N%gNVBHZCF_99Cc`9q7Eh(ZqT#9Udfz)5AKWH(lkzM577BSCb?|W+u;0 zoFmtAat?#vSYG*>=Dl%+7A$ zWi?wvYs)Q`g_y)s_zjASJtFJYqM~Ng`>bcgBwiI0d)^ViABXHI z7%1jNAfnJP_D-c~NYATtvtg=}CQ0k++FW~rHYRQexW4VT3EmSgSUe!2gl3je2~wKC zP!H>T(<)w5y6(jD#c+X%r8!=4MYFmwEgeknYO;7og8ri|M#80mBV5sY(C%vBcahy@^I_8zn`yea zYPXA_>)yypiop&nPE{~W*6>mjd1P+KgW^=}B#~B;@#BjYi)~8ewqqp)1aZ2ay-k;n z{EK^d>hNS)3tD%bhpdx+e2+|Dge7v=DQOZydoMQ!|;VD-OuPQ@e`ksleGPb>`Lcxrx3`VS* zvP{Yi=SWYd4(T-xQ5^Lbg#@bT$R75ek~FbEivYkw)aZ2@ASAOGQkp)+aj$swxHzZ> z2%daknL#K@zKmePKr4m$MZ_q8xj>9F>sNfuv?jhyMQrX%>Kahg@{YpXVpXe!zN2na zd+1&Tv~z6`-)l*W!8I&7?dsgc(8#AFGpSQTrqd%H%SUS=gjR2CkFp&x2+WF0hSk~v zMzDw$q%RcdebAh(V>05b!!iZ4PkNfyS`-bQFJOn!xEbP`woQjRmI7ar(({Kit2$0y zOvbM4qx27-mgLIOKw(@{wK+sm#Ys_op=zdz1Zvkh^|-*XGU$~aNuXSez~FFxt2qY1 zp-3rd0tWVzrdu?HmwXu{qni;)4Ff1ls=gb(%@$fzwfWbUDN*j;5+&uA%eY!6gucp# z4y1qx$&g@%h*M5^LTSz%f_rbQoT>W+Iec^@*Umz07prDNepFHMCsm~@GY;sS@^<@j zM#U5`Lz(EQKczFA zlFQby8%mBMyYI5s5Zi+;vubFXjG2krEImRgRe({!yRsshm@C(<2DtdWO&4sZfU)U} zAka5*yCop5ly;kN6Uz4ogm_&qtqq==sb8u1h|CATi%jRagASd+I#ybXxH-;IMJ89E#idsNRD-Oy&7MGw9RRjuQTLgi{<<81TWup%!INF}v z2C4zUc@^zC&xO)}z~J#8rkGY*Rn?{~0~y1O=ylc962)tNoAk)}OedNj!rKF?PMwd^ zQShR;J+`u7E5^Wp%n!bFs?`eh&5I4;pThiwk%JC3j>8UMrs6stvDQhJh#Q+sQ6Qyd zK*mQ6xCawppF&Y~pgKRguV!3pkbE*K6JE*|Yklog-&yst=$xaaq8)s4dz-s;g~?bu~Vt@CgLIH z?#TK(?Bs{q*9yAlp|pz*6*W;Cd5CGp zc#>YzE)gx-<~LEnWsXuz4`x4*jTZEo7~Jip#lXP&)fqjPddMXlja-{KtMZ1#WP{4O zo^p#VR?zv1$juKg-)8th2#VU9ib{hOt&(5ZgstKGt0mK%6dYHaed0Z8Q|xrw4t91v z>1VX|T`;LqgO=3Bh=73)B^=79U3^j%2VkovL4ZBC|0U+L|ye3Px1)&=f z8O`pnQ`N_g)a`j>qM>K-O@mCv(KjrUv>c6HT^O#z;0P4>)EW0sCM`s(^X&U7x&j&- zP5_ZJ1&-Y26C4M<1o3 zn-!q1q@#NfRM&}VTNv;ixX6ZO(WN=;0U#&62XAgDJY*xQY^L>s?Ewr}KtQGI?CS@S z7LsG6b!M1O*%`pxdh>C+%vi!lSn->Z85|=LuqAcU_W+F|!38LSjo&X+(&iK}%ZQ-h z3`IX=IIbz4PloDt&y!aQ$+6AOMD0v*@Gy*;FV2jv?`t$RBXSE|EnE|`V8D@9%-!f-j+R)pUULA%j0Zhf`3@LwK&Tk(+kMZpjyDt1PllZ z-6!mme7Y4-ECV(ppnAD2}#8{6Oge3-i2H4?07PPiU zSq|F}Wb}y@_8@B?o*;@Gou1jWKEVtjED%BX^t?N>>uQBN#75Bi*3~V2Og1Icncxdz z@VtdeT`u9vAge8gP!#LS4(@mj%fR)s-P2TrjE{t;v4dL>l$n?`R6*=UPSVY|VJx<; zEguRb(z*Rm_T`pJ4L(f}MM+fadP=lT@ufLIVy{`RP^^%o#)`W@KD9TF38xDpp}9&*{OVumc3vzq6nfKpyC4oLHFdWH`KE$9TB0A1W3 z^Ar~;v24AhAT};as(0yr1tU11Jn}w8dytriS&Hk3l<~l%=wkO1s}b=S{cN{hS+w{^ zuHo*n^=L{mx!N5DV=gTWQP)A~fU3>-?6`|-Q8|wZUZ2s-cLX$f7o}jntT!<3-j2|WQ5xDK@C4< zI|GKrC2)%7UaO+~Tqch*6r-0yd~$_3Ekhk`ERV31<)QZm1x)z#+ zSxH-b^MirRsbCxGs z#SfrOTeMNQ4vSNzp?eDa2DbvSMISkH@A5((1>p5QcbMTw!{t&4Pb~`N=a=hX^kDBYUu%6(aDwOkRlMoqlaH`xFlQX z=dYH00ujOw_FIo1Sc<7Wf{X{zm$stC|{sMDmXDmsK!R^7MEMJiHY4JjHvdQFb& z1*!%4DO8WMPIa9>LjA=}keAnsFJ>L(${(wk(Qt@&vr6j`$AGC^&!Ib%- z92U-u2%9neb%7M%hIk2YaKyZQVx25SXxks*A6B}earfG`Q zt_9K6-poDDcX&mwS!9W``i`&Xe8dF~8+C-q+DnP3kg^{V&#<5{fEvJ)Im{tsB408! z@zDIW@bXCEHsdd``77*eWzEfR3h`7Bx=i3QhU5^N8_gk615iqZSZ)M58T1_|E5mgq z`X)gv(y`yQQk8Lv)?d$38V$wHjXN~s1hfn7sU`ShNqusf*`4=p4+q6!p{a* z2xFINP3@2$Zd>&&%CARWV8sfCBBkQBE>My7_n!&W2p(Mb?6f%4r$5~dU-I1(mlWVJYThI~>E-Dn8Gixi2 z<<|gg_3?vL--otu#pD%JKyZ{tXS&KCFkWSdd)}HD7AC|MDRo_U35;UQ5t-tW;vH9? zCaZ85lz5lX*<{J!Hr~y>`cUgq%UdZKyQNH@tMjz%-AZDX@hA zkD$!h$|{ASQv6t3x!Nv~BpMz%D6}I_yo$q)A}E<%L}N zZWSxYpnwLYqWTN695IFXL*3)D)Tys|4kajNX9YUy_i@%!QW zsHVqa*_7GO(?yR~g&8c#!mEeqE|@S(C4w0OnJcoK1d~2)`rTD-qA2hqOl`5?8GL;x zfE(r$O(CLQvt2OIuQ(r%C%Re>yEq&>k*qFU`j3_~?jV?CEOjOzd2SYXVpq8*>tA#- zSV<#)EA_p@H+gglB$9S&>IMS%r7%D$(vsoc+X4{@wNIm|)OZFm&w>idvB(XQGR!$` zswzXf!3&vWe3(a_Lxf2@%nF5^hj0!kqJ*!0aSLyzS|PQ(ED6?^PLJJ_veSV4{!%IV zA(Gn9&t7&wE(LxjHS&btw}70gE(I>=$Z)&r@0r2Z^+8h)j*A?aRZ7H$TCBlk}3(lrqF^EG+ zd9eA&Imj;~6I`rbG|Z6L*#|DY@(T+HV1P*VIRXND>V-a00K+{%vEE}$e>Mjg2zt$1 zc-S$N>OK`GnV8Ub>U3Dc3h1(C=lFYw-$z=t)5EYZ!V?*d6v{sb$wAu96 zj#az1iZREd8kjN8t;|vZx3-LaT~!}s%JgxpMkrJ=sU`?YJi(r^EsVcNPk4=&UiOGv zTMYY2gNTP;9BWE96$^6E5<2=#tr~|+HlW}Dwq%jI7wW0n3oN1Pz_;*^tyxg)2G1F6kIATcmMEsJJrs#S1Ut>2+o%<-8SHdkv!q8NiPg|wURmEi79k^9yvY;QV1 z&Y3KY^_s86*ig5thdCuFoG;1Aql*;G9xhJVuy(5tBlW83PGOZE? zu7bzBrwL~pMmz+nH3gC|Wx=Iy+7HKSmY8Y*w3J<|5Z4;D+{hh0sT7-ijl_k(Ym9=~>x@b<}WuC2c z4SGr9r|&G?qQV}AH>~d#amfC+37teS+rkwnIrb^U4-(2aJRByKs~4ZW4&~j;xo{&# zFXP__WDpd`NuYR=d#GDj1`a3fx4c)Fjk(bA!N%fYj_Mm&`0NCE(7?bSo#|wBX<71; zg}%_M(+H1o2QrP$7z}>+80Q#g2oC{q{w+8>XH#gbD(-3aO`D!! z>aL;!sL*i}2n`8bT9TNAAUKG>Gq~Vs)Ho7eL&MHZ*-vTT)oQ7tI2gND5XZ%%XGLD( zs=i)w2F8&CxJ8c0*<&^|7zGizt1$}KoT(()OFG*ZxL%nyGwxk$>~v&1+-A4HUv)Z) zq@m3U`3uu&fl{9(t%L-@V1SV&mkg~LA!GC>!Y(8Or>%)ku%BLrbWI1jaa2z^1OyO4 z$W#?FmEcr?&z3X`Lg!QLk9qA-olByWYoxlm0^qJ#BM@Sn<)RZ+1U2_ZO6-Q34Ruy_ z;l59i@&E~4E(zijWrsAGHgZE{miR!1i?qE|CLka)%^0E%m_D^#J;)wfNwFUo9HAo| zVa%cksFxA%X0l-R0ID3rGjnD5Q`6*OyWNVzxKLG%;w3&Te)cG-MZX%ydbUBNfl9>N zy=ZJ+l4DptoIdP~zN|mmhRU&R|5fw%IG+ z8troJL3cu;s`z@A{i2apR4D!B0562Tb!dy28Wb!ru+<)_R| z7S0r7aJ6~Ab5Xv~rX}0n>(5&5_B=AN!jgQ=Jv7Zfcj5q|A^amN;ABFOjrrhE#oH7R z4ZCGNu4x%$1b;DwL1Gnhos}>kxDwAzcLhNrM35vWw<80#*-75SstEEVg)6*MyyN-G z!@pR=S^m9=08MQ*;K1j!t(XcExlDiFAjI*9#<7+^lnEjv3F z$~Dt7>jB%YgXi*E}pdQ0Vf?%c*G?qSY9uzKOkXKWREH^vxePQlw=uF5`Y&;OU z(D?l$faF4?abla4G&NkQzV#@Nw08Wa8f3k0&A~FrD^UgrAz_0AhKLs8@#@ni9>UtP z6Y)7FfXaj^VUSqL`%YoCJELDiz8UbXlqx9Omo!4Tds;c-qRU=elu&3rH$#PS$G zM%9QGAQB+rL-UJQloa>8BT9Cbxt0}`&jF>Mm0%Uzek!o&NSaMjd42{7@IyYMy;&lXk}GE%Eh7B_u+m<5>^pFaz1S*mpb`}LEd6KL&i;X(AcPoLW1b>QbhFe=Qro0H@O^# zIO`CQtA%L*QNT@o?I}FEIKpKRQEh-D1%baHm7}GPAc}I(GW6K9Yz6;yvI0&#+t-cX>CP8JvQ|7to5RcVS}+W5V>;( zmXJrQSgMd`LM?chDB;LbJgF-7wojTLIXa}>IJB&>x7iA&1v3inkxGELN*^mA0D8TJBxf_{x+W#= z4CeD6JWPmp(Z*Kn5Fyi!suOZyhexGIYJurX1Tu<5Rdm7$%gmyFP#G|?`5N4X&vVxwl9-#f@+=d zi_b<8;b@-k6-3S;$+BQE#ybzf0tgfQNGxAbg zI)o9u7)|w7H#=0U*pz9|*O~%{%mq4hjz+d%Py?)(*|*(-ft_K&s1m31B#ywsb3`HI zHhWQCFR!X?N@s~8$Y{kgvVg0+CSbI5xusgX3XZpUU?D5g)G7rco_M`O2{Dx)0Z@hJ zrgWrR?n>2u3I-wI;J#dQK=GmUQ}WM(DP~*;bUB z(9yIDxg>s4V4atgT_0)bzgGtEc*h~{gtkEg5-Lbwt3iO7af6pxC?@3?se^Z-9oGyy z=LQH`LK7|!>auJ=&6kb+N8&87q$W>|>H}h9FB(^D8=zf1X2x-MAu%lRA-bwZTOiY; zUCW{P^H=aCqXeldmxmJZCTPrRdiAU=gqEI@oQ+yU6F((5E+bvwHbEY3P!|%tbSc)6 zbz3#|h_FASmir~0USb=N()){%0;Nr8{3b)oD{te#+XV>_8jNxzUOme;;w>{zEai32 zWf23JX{&|{CS>G^UTwbAma0T#RG}Y*EFgZRXXDC}0|b$+xuKx6ezG@YcTnzB78V4+ zpBUG%oVJz*K~&#dFN=A|7L8qLBP6szY;2$@4%tCdNy;f-X*@HV>pm*WIpofDtz)a$ zIlWXs{ZX9GSqf<|EPDyioe>v93jv#zMAmaZywh@8Zu$_kQ`V#kDh3;{B8xeD_X}-x zhzN1apzCc~gA5@TK;{4{K-Iq(Eb%$i6RA2jotSN42vuV1TDT?@y;+X(cSAl_o0q_REHqJfpTZt_8L)!B}9-J$lVdri9#Qy?Q7$}ftNz3jGBXO$N< zlOV2=VK>2VVFnOeM!KAbs2;M4BdCXtgC%kbIsAY+Cqm3w^Z|X^L4rOIA%u@KJ+D!a zkGb`i60JE_hgi@jzZwVJd#pxHY^1J}x>BSdSU~|50Epa6?ex;%>?3Nymxd{HBKx_Z zjuEn|5+}MM! zz&>F!Kv!kl_S<1$FBw4Fs5C49Bz0c#CRm-3$4;RzsJ}bCud#$ruChIqIBc*w!KT3( zI4jU9%fj5DoYAkbRBP^8;U6^iT+&US$2w!=1L6u6QZ6NqZdX^vZzepxs9yNwxg0sn zh7bzW1iRSbeq9LAV*vWn;!_R{5oa+*d?Lci2q29sv7p4X{lg}~-nDSb!ykq~km$oE z0+BIvj7NI;P~Jv|LDTA5?Wqt?aS)K4=@CFF&Z3SiL#7OT_WfEelV3eaEvHWB46_AP zb_Hu=vBVnF6jL(epu6|hW}6a1R!*-rN514YmB%(8^tx{=b0vfp< zbW=svz*VBh?(44JZXS)W(k|uLs(H`fGP||}oH>Y6u!mYhupJT2V2aBe;>uF=qgJY> zpQvj2jtcmg7Pe3tCkbO0(U?VW(rYlwvPDk_c2cO%$kFF& za*jy7mq*<%A%_?hS*6tyGO*lfxIters19*J^**85Kts7n%k)k`?sn{C0$;tgWWg*i z#&xd=(f5S?z>xRQ#1_S!;3EtcS{}fGzH6KYhstz^1)E2IUM+eGkznD|Hsi823eq{; zS(}m=e5#EubDUpY#@bc5mBI{ntEOwI>y^DB;}F;Bi14Z9H?&{2R#Ob9!U@N7+}?sj zz=rLf#~KED0X>s^pEDyG9pEonI>pQcPnmt$;6q@$%Z^MYOh9LS3LQ0>4yJ)|)c7oB zv-}9#$=Y-6Z(?wo3FeX6_)r*Og?=zG!<)Vut+3+-I*<($*1LuyND|05!>Qm++L!Zy z^le%v368%I`LwfAxxX=uoU@80HG2b_)_R;`1HfC3mQQgbfhH4YwQ5yYv_kAUy9^DE zQNluKp=cGybx2iRFoYxTYR!6yFJtwZoM?IrFrti4CFXpNeC*QZ{#IHXYbI zGk4V}#~7=U6=+|;JmoO0jW9m?F@8=w!i>mBeojA!7TJu5qRAwr+0JD89K#Ay^LP9`y$ zY%=;uh;?;jG7-w-W$w)x{M@L*y|=RN=4JEU2pBUM9mgp58rf$@cb{}B-a$z7y5ZGS zq%_oLgYjX}VK@UYRapkawsb1bfdeeaAs3-o=QR__V*g8FxiCnxkFIzZ8lr81$!W5; zo?%jxMjYjFp(icGoJFWSn5BXYr0k8RbB8@Rcu5FhwIJh%?^K}Nj@4hC=P9wwuEk3; z$I7s3&4~>OPAnbIP#9GF-VIz>6DJY`i_})ez(El57LM1I%Zp7hW5(<0k&b1YW5j5! zEI^YiVOI0__a?B|`$uM0r8WB?=IJiz)Rb zvK1-SAGF13;W-dtuObB9*r0=PnVRZ`2yVfx+faC&Q<$_7WiBQ<^790Mi<=N;s)D3^ zDHcYpF)z!BHb@CT#UtECtvKp1lBAiitM0A&3Mb)U&_ae)-`iDfDo#2Q>+juVO^(WIvTL5i0G)w5K0~@LjA*U zZi^;)s+bpK4XHI~kB6MU2j6@E0q_US-=3m@t>^P8qoT~>H24){(P%HE?Jek~U%yng z#i#6^)E;0e8;Cg_r(%wNk~0W@Ft}zzGKZj2NkzNH$dD#FUCWH9n_NxUJ`x)C`)i0R zj7Qd{7z_^nfD^#Y&k3B-*n;&6H1uHL2_w?ar0qns+)VrvXuzj6j^N#4LjW1Qgy4o8 zUA4ruJeFV}B;K`2;}%qN#E#3Io~)fT>nic0MI>$CaPXjGaEt863BsloQ(_m$1sW(Y zP+71#3wp!BCD^tdsK-_sCF;6SZ%3;`9FjWFWQz~5Z+?cj-n*T6hXQ~e%?jT#aBfr5 z@TyIHOpBPii@Tv>&PS&B_~^DoAsYkf4xAU{4o@snqfM|pXdKN~3BSIlkxS9Qo#qkf z)IyUJTjzoU3g62h0i`cX1V$j#oY#@jIjiWq@+?4+USZPQbDDsqgPab z3yJg|U=jrhDy3S5vmCrbAdfG~q1exR#}n*J=+50K)8O1FoCz{9U%gMTr=*%+6nGMq z^yyj_?>l!z$0QggcYIIOJ9P`kcekF7^4tLkR|_IJFObEliJ;!c=kwY@I4#8F=>bJL zu%)F227@5M*jsK2Nm}{NNME5PkZVr~?Lpui%ELHEp;H_S9K;1`(-lI9Kx$8vEs95S z)EPOe38{h>TVg8qVNj$@ptpHa&C^sj#C{dt<(PDV;rdCUsx&OULofE5CU(XRj^LF` z&NnSJBPbfW6;=1GOdQZqd|LN%^a=kQ|`QEvbntpcOP=XE9U8 z3X{|$N(&ubd#mkoP7n~@avs@>>VO^q(P`IN31Yt7>B5gMpTSPh^ zQpyHTmk@-s&8F(IPgAiWIXqMa!IiL=b~1y9<3U zx~YhFQvwu-VC55veu|$4I8fEzb$0Txg9juyZf!Rro=2-fEgF%o-MI+uJ5ZcpffpY` z3Z&r7+IhClinDl^OPvQT{aags19%J%B>4DVRWUG0F%ZXVQtaZmOfXm@GBP=7M-fdLkN8u>&)ClaW+djM)vzzyxeeXm@b08 z797&y1YqS`7NRCBGVmF&=U6qNgLt9iXKGC=xhvXw8;1Ed2KgQFdLnqkCVT;hChzMD zCA>ptq!obUm1FX0KJvP*O;al5Mdn$mvHqk(Jo~?C27A&+$WgS3D?gI_b*&Z6+o|`RWN?h+cOm}4dEk;&8F@a=)pmZ2}j|%owBBi8(qw{(8`qnpX z%eGtI<6Nk7%P7Km67@@+>zt3ZJf_9$%d9#+8XkkC92|NV5>ALUS+-6!z!E$Pg%{`^ z^$FZl+X1&-pUy6fayhE&xF;{iTC2XNfwdB#qCo*|)}}7+NDlocHEp7^9qT6`_IJiA zP~q?iZueUXAU4o2-CRMSDv&4BiJGGxsJ5)xedJh-a&VnPy!CJPRWAVkF;?X%)m$ZR;1R%m?f zykVs+uel~_t=@=(B@WoB^X~GZ#>hz7*3XgSK`a=?`KD+f;3q7eK${v9;09fo0}ZUr zng!E_d12Zcsvp9)S#Zk1B=ZO@fR&9T}=cz z@45V3$sq@pU^h%?_;UOqt{LG++^OBGdsOML70w$|aIAqby1kalE>W9SE zK#d)M&57GX~em?S$}3KuJ5G9*?XC24+R znhb;jkja-36p;5Z)IJcZ(1Z+=fKavJxGP)EvWaGmOda^z<;l8vAQAY;c!0cQ*S-X! zKiFA5#uE&>W<4a=-&p34Lo!f+{#dc5iE59w-!1H)|RjzBHLS8CM{HB@1d6s;DQ z=U1f&h1ol!C5r zFN5L9a^sl$=-xSDECB~+?R5|^L79qpM6+WNOPUE_^7wdE7H$Ody+^rsMB)&e-b|Zv z0wvqLvfbDQh%tzV@SbxQEa7LW5TGf%+3N5(Bk!OBvu*F$>CH#r&~2H3d9s&tv1qkG zKqc~aUQvh;RcwHF}-7}h;r?|^AMq|Ourphys0c$CKNP`D= zwsgr1ZD1btUzpLnc>=8i*d?T(Jf+Ea@QwqiD`oBMZmH}gX!$iNB=9R@rohYjl=AM7;$Pq;U&QvTtOH|zYWwi^#X2IXM#qZynaY^LTNX5y2*fJEV6v*fDN+W~AMhxiy}6_H03~ftt8h1mh)#u$d=WDeGsM zOwD_Ad%6~qeMrEwW*b0NNja8mzKR>LeW^%ZTQ-vm~JhM29Y0e(f}9Cg)`D9F9W|H6S4=cdHSqNSAD&s%F7Sb&K1w zS`hs_Dm_33kQd^=17k@})Uv>)znqw{+3Ql#hc!JX!msLBi-iuFgv=>-l!~%{WMz=5 z4)%2&!oDL(Stb!Wf!HT~p)uy1oO{7ue(;e7n26cH$b&Nh%CmFc%qR)OVtoh+MjS<| z|36XdlXVfZQX{a2Yzx8&t*V6!GG(aX6U5z_62OF>!rd7xyS^yJjrN98axxK8#b|my zeTFG6HR+7Yzb%5b@RSS0&nV0vf0Y?$!zX#Vk+tesD-fthEnNc3#>;fayYfs3JX82aMTgHwj!3YymeYvUFe@y%QXq`F9N9|; zo1q7}h#9 z;;pjkMM`;24PcQ7lK_q&QdZNzdeFo{0jh%Sg7B^ukcG3%fy2f^0@zO>t@V(r(yhOU zBKVA>(=sw4o{u>4Xe>2if(jXJvpSP(TW3RjL>!0|!1Qq!%qvs2X>}M?GpM#&=TnT} zI5Q);G3X`2#0DcJ2-Z?Upu#318LJ)Y^e8>WW&!mwojbA~3`t*- z8YkR($_h%Z(_g&BSWiJ(^@JKIJ*7Y}!_AliUDV+qczR^Eb}aUyFJh7iTgnc=r{X?E zW!&WHAa@Y-c@Pj-SO@M?I})KwzA`k<^J=$}%dG=jC_N-GVD+KdO0iYv2JmeI$Dg?X zeoTU$RajYjhE4t9#rjsHAywFyw54w-bO0YrWfW~WD1(-hE1v;peLh} zo}g=y6&<~_hLB&a*XtNrQ+DseL@-rL_`+`p<>lGo-xD; z*+nJC#?n`?7KEa}QRK{Dh)5`(XLGtj?(LlIgGE`Lp$A-L94ZW>3t@)v5I9@kV3cuR zcEI?cZl*-uzwaq&?M~>qMn(ldOUMjkoe}1so&@4GRFzMy;AYe$LT5}gj>3b?PRd|H z0z?=*Ar!C8jX|fD7oy|PfK*H)I&fo32QyP_$<^tOf^w_7X!eZ^VV?J6RyhVLxdmYv zg9()`j^a0{-`OpKq$Hc9IS>$T$#doD+R};;fO<{pwr23qF#T!3rjZ`lU>f+$w97nK zrCz|B;RBS_&H9-ZnOUO&fJu6#< z6a?w^Fx)OU=&%os;jaM_Fl;VJq8^#YbdEI0ekI{x5FrSy7GDUU;DiJs>AnI3LMv~X zUnekuML4^z@K59lXiAj!Od&ofc>xJ!Zn83<@*P8)KW_YYJ$+SNOdklE*Qr&o5@r@i zL??O%=J^~7(&sJbMa+-{OnQv-?%ZHC$4!9sN3OPJIC~C|M8_71H3)~?7F^4GEcRVP ztoN)&2K$#k6-_UfQ7^2oStY%#_lZud!ex~x$F5(yi?l&S7P%mIEsNHFVlpabPkRYx zeuW@JI1hlE!UPf+;$9Tsod6dOv7tC6~>TgRc^7zA*A#79oa zrB(%=!_06SrpAuvTT2&`aQxI`?D|=V@Kz&QC!gsV2xjyb#snb-*pZp9o6(w>#m@gV z*_RZj5{A&75K-l1P+uaDWjryO3-}$@K$|a<%s(o>jjGUfOy=M-WF2TPLaJ#U$4WP0 zPF+OW8)%af_K3r9kYcSmA|_9PMprXqa0+Z7M+hq&CGB zb}&<{C_7j@8VC$!w6j)Gvpxop$apF`{NRy!HpkdfBw}_*c{j%sfu4f$`KmXL2-(S% z;pazdW}IM!kAkTQ&hDhtKZNbF+7`L=n=!b=eri*&gB1ckC@nha$_y53Ud`0G+SRE@ zfiss5L>u!-z=jNJY(*pi`CR9FJU1%QMX|ZKTg-DHjlr@f3`B9Zce6aHE!guDw&EF_ zSE1~z8baI$ zoZ$GOC&Xb(k7yCLIWia+gW}7VLfSKB&fGGG$xuQ|bqD4J@fAcV=ohD0Cpah+ME19i ztBA%eM19RiCOBIiFPT)Jj6Mk=m6icbMaTxJ zCbFatbkpRm3m)J$0{De+dg-7H{fw*`8tp!xvTg?KhE@QP!K{u&6{U$7JGWFZY|to2 z;iBmKRvWX@>EzBRAF_**lnFP)cMG#PK)Xz(h-^tfG`sYp=_~pjVcgI}w zw%qd-q#w$&>>Tj`8cvmY50?MxH=M|f`abz(FBuzW-Cdq(8^%v#v0cMJA+SF z(b<tAah~)1)k>81^TnWHKV?_QeN6Xfa24>_KRDlS4b#Q7iSi*ZV-f! z0jUT+!%;0rl=%sl_c!P3r)0tPSV;Sl_Wv0Qwkkm{PxZRBNX%quCK8d+Uc(s1DdI7fr z>$;6ZwHC5*4pu)#Du|W5l=l#(9ynV}qmO$`-vl)Hi$HI;NktZI8Vp5+Vo>NG5cVCo ziJr{8m-3Jn5J80uNqyM)+GFi@SkYC9Myvuc>I3mQtO<7lPSUgI7-LAMsAoyMp!{-e z;mz>!CACr07afv9FpZRxbze}i$(pgs>nJe%9@E7$%hQBfR;9fR2@QltL~xfC*VJoS z*saJTcc4CD!N8V(3E0^i1Az0icY1M*D2#krA(3%5a8Wxsg~)ahbW-_3dAUHtfgOX!DK*kg=1u2* ziR9eOG_x5QkczNN#8WstVHKkZ0xEh-tcDlA9Wp{tXEQlBa}yYk1F4k_#!RhyM7%0S z`iVZ-W3v;{0QAfo_+$6;I$%`K4cb|b>F?p>68Bz1CbK4MY%huxSXE~Ho^hf*AQ zk39XnT}1dYz9ToyYzQyL93k%@9r*PNF^>cw$>L;|aMj+im7<2bm@euQA(%Ey1P|WY z#A0qs=y<{?boA@>q)=x#Ma|(L&6@g1s?UDqodk3~$50`4Eg3scIHfHGxzC_$y;6D{ ze+u$(%QN{27^PT#b&S<8x$&KOw=E*2`1b?NH!f%!xPU599b8~GN|YCb!0Do!;u5Hi zQ9qr&4M& z;#Tb-+ei0^usp|!q-7JUhanjQ1QvkVOG#}w73iqXO?i43RlzV!%Mo#P9Qa%w#cJ5) zJgPv77YxBeHbB{BI1Klw*^^Og>;er53T=b^>0dZcjB zyxpDIJJw;ZAR`%2;Q&8HhzJpjg;NFkvCyy!=1n-_4A^+ewCB_H+>aX4;xZ5@xK>Ib zO8qxzry*EDk(CL@yU1vuo{16Lh2M5+8iOipz!C7%#jFX_(GxdMBQ7lcDn{K>j4*iv z+DfxzX0Ekw0%NZ3nOT=DGp5ou(=@XQ2w4I7{tCVj?6f6hF-(ZQ^B?SVO|^ia^HCGE z2u#w`JPH7VlCcwl^67=&sjNsua+zNkBI2HC16chCp_NjVrW+O$TV*t22&`n;Y^Q2S zNJesdWNp`gk$v*;OyGc^(N6naJ}eRh<1)rp@%1?7JnSQlZ;)yizQ7EatBRvn9u%Ut z?GunC7H)c(S4b;g4ou}XHc@%}fuAu~tzn7LleG%6;i(Dmfsdp&oEtk z+&C~9tb9BoH5tq*VIl6gc_{ZNUC+>6QQpp5od__}Gml#O4b|nI zi!m`I>7Mn@Xb5A_!VweBl3M3!ql^L|L~JvU4I$`0kwLdY1skm%L(ypx=JM?<+QibN z93ORl4bO@sx0Z4K&aINaVzRN$22GgVJM7su#$g=7Md;ag9l%5HtHBIHfmE?apFBV= z&h6MXW>+~QS8(bf18rx17t-VeTq04lvObB^vH_6L$U?3=ds05G1_XKDx_(p;RI&%tus%}MLy32JZDE-as*%Op^*DsiwZl|y% zFU+C1O89-A9=N8n*%_k^azL(7@p0-*4{Pgx~Yhg0E7>Wr4ijIOP4O+9c@x;6*n znNeAYY0M{rJtKS7W~Lk$`{4WF4}d-otUr1GR%WAq41s+lSr#AE+@Ne*1(Ge_+8V*3 z3tOsI=MeP@d%_AwK>(ifgrNR}H8CDQ<2q8}E+QHb%(_`z5H?;z-jS~e4&z4pk^l_q z!xroaK&DfHPsC~TP)=Y+fJ&Sd(@p+~0Nl+xAub^!Vg=-0LQiEmD9rDUuC9!;=7P0A z6zk1nq=sNVn}Z7)kt?5zyg_?`wyT0ImY_c2j zsgA2BdRjDf8MBn~xLOcclb5QZfI(f0Kx9;YedO$%T1He%je?E>G=X7KgvuWT03>mnM9E=o$>48{P41_mD!Cd-MX@#QU1k=oueHQ6{xk?sNN_aE zF%nrJS`bw!dlhp!XSCt8kJ=xN(^}BF`Q^3Dy0hC+(m{yH`;oH zgSEKJ=i>GX$S4l;fs3-4KpZ8Q_mFA>nd3 ze5*<_R3_DxmWcQ)DrQ7-pS~@UG2oK1V4@J)#0dcM9=0ZGATbEtpJnvFH9&5uf{(8h z$jk=WsBlFJZK@A$0bm};oEYGKglz;Fsq{PG57DFyXd8K`FF+z6C#QbJlk}xh0kBnR z7-}*5wZDb7fqYQZ>pSaO_6E)~3t+CKAvtLz;StP&4tktK9B}?ZPaw^x`u9CXRO{13 zl>t!u;-DMmo6WawZeoasKtq zBKFWs2V0K}#hTt2vCP5BSI4U%)FLfZ3CH$&Sw|&LJ6ZLZ(#Ecf2eH@WZ%In4REq0K ztvYBcl~e!~Z3b{QrSECzgm&Nt54@VY9d34*HBQLGaEoZqvr|8R1p#Q zMyC%Xspx6lB+l>$KE%XTs1Du9OTC?@g>&6Bu7n9#7`j>xiY@%odJAPSMj(7Q_pQzA zix5~qcz1(a5bC>1w9BeHRBd<*9P*RZlyq!PE5SE}y6sY>Dtal=*Ann7IGnV0z-20% zBx6TCE{Wx>K0ei27P1^vph2YTD}8Ch5{#o|1%@>;a?HsWP{fcuP8tCo9Zwqz4X{2Z zh6X~sUBpw<8_1c6j8=(B6zcyM8DtDaEh%6{u6U%~) zZ5UxJAe(07K{B0%zH(gfGu{|%4!1zF5qwg?2qDV8r&IFd)Ky(F22;k4TkL}|2{=Gt z?$T_6D#RgGy7ge#Jb2)SOo`Tj%rf_s4PXH22!*nd)U%80YOxm&qhD~@9GLs&S%HyfgoGQKW>OiEM*D|1Z+*|}c~H(+q}T~4C% zE*&`LL=x*=qMB4)r2441^Aoi)s8lRK9jVcDODY}L21T!EcvYYqTU@r}@2;!}`9_&` z71o(NMDUf4$z@=x3JNv5B}n^|9DqPNS(hl^dwj^i7XJu@frBVy(UAONieDl+=_!K+ ztdfA#%*n68i$?LiPXrYVj`3Yg#pAf_8|;D&4YTmZEv2D%SQ^3RYe?;GBWZnJqVy@G zXUH58@K~W1X94^jvjca4rAVYSoS}kgML>4vfJ<59GbJvV`-Zhf)#+53KI}}9 zI1vT3^SL>_;}xYLGALhE_l0{aqktinJBNizKE;k z659P}=&buG<>9|B;cSInk#=9ZMF;89A^4E7c0G)^yIqwMk&7ETMc&Ib(4Cjq4fNvl zpz(!^FC|0UzMjnF zM8!8!x5))2`T<)$nyv>GRfKs+Wa4(m#``RZHTD@R@Ch~;z;9b#0_=zl%=b_sR~hYC zR6^FG(;O2<7SB*KqS4fhEv}VOOe@;w$9O_r3ydg@aLl^gX5~1$DnbZk?3~-g#zzps z3zq|@l){2*-7v%G=8y&XouIt1$+Yjr5vqF?QK}7-8PBJw%3D?}qc-4E^inNdLV97k zU~X0nf%F48fLQ(Ym#x$|c8lm3h~Q6wH${P2A57ff9ZTZknb9c2To{q1V9uxEoF4$F z66ttAfZhK01`mNRZ3Oy@w7|e_0(!+PBkH{aCZ$Y8NF(j4aZ09^%a~YKkVs40nQTqB zk?XlL+}>HBrIVMF;Q+@8=}@V%q>#JLPz}>fTaGlDU#(XxjVZ^HWOIw}0S;wDDX?0C z2*x-fp4vk(k1{+z!RCC$Z6V<3L6Me%OKJ4PvGt_24g@0%uo9*?9m7|o1m`Wik-&3E z{xF^I9Jpp?RxGCDbC zT8Q}9&Iuw8ZCI;B3wOs<(Q5PEd}O9PEbC<>tL%{Gs`?efq;W)V$!WAMs@a;BOq1SZ zscd!LjLi_->5f_$Ywci@F*D(Y!*f`@mBx%my5xshe ze;1*8W}~Au$s{Im?_LNrY5h0^7jFXsMwjFbc2{$FcZVnXxCVK8mR8Z>D6-)>FLw21 zfC!uFW+Cs1$dG10!&>z>X3mwIAlO?sESBFk6$apBJG>qU+pVHFHF$oGh^Ze5N7A%$ zd2T8~QA914o=AW}nkLRnJeG+bXOk~wzDFOrG5kp!py{i*>4;=Up6Dry4?P5EOf9&V z1{YalNm665xC_ma#rAoA-sYvjgRG^06Qw{UXe(NKUEL=SD`6kSkbNK&oOr*X9;uU! zrsPwmDkKOux)l1oUTUeUkw-R&Bv9tTRANJg<^^VSNPAC$DeW05#BXZ@7DCK>2zhkj z%pTjyr|9l+(`rsijX-0?#}dJl&oGS6e#I;K0UT2Cy zLCDN1?vAV^SfKC~CMIumlhj-){=Kwm?6lJES;r1pDD;U%2c{35vF!*R!c-wv^kITE zLHvqT5{H4_15O3M47*lLxd5a-BfXi%z-fU{l^JsNFz8x z&qmqMDnb@oL+H_y*_W~GDWHk!7Zsik_}a%zu}!*0+bdJMKwn`f`#&=A^Ij4undwb} zQ}Ksm#j6kB0*r_VCsw~t9m_8!5I0)ZJ;aN@ZQe-_T>wylgVEi*QHTn*7)CNKA2Dps zyd`WuQh9$d8<}vioL3WDyQVhFqT=`_8VC#-l=5L7ydAa*t~*65A$D0;w~aMc zglf$cnh^*naI)9tK=7B2l&b_%BLx-lFSdUlKg@iZzG7JL`zB}-$P7EfBs3s&C<@qs zOKb=rCsRue>q(7K3l4?={-o@)*(BMm z?1T&#c;}Eb;i~W_M=yc^J6Aj@%|+OX$v0sE-HC#TKB%h~V*+ZDr&(cuh+blem@g>I zty~3&P|UH2DW=&P$Dd#Y#o4t=$T?ln>nf*AJd*JB1sFgG zpooGBfY=pTR`aUFok$RK4O*IDx$>}<9QXJ1Y6qAAr7D-@RMA+5iQZ9aG}-X4KNusgNxHypyPP}S@2wP3OpI< zpd)czs#5ggq#k`Wuh%AxI|1m@FRe(J$EHSvf)S3i7=;SHUOQC;!AImK;Q?E>CHdIR zS96VXK~Xk=wYFIdOj=`%A>_&yZ8tw2B(b_$CFx9&eU?FC?QvBn$RmP zhd_B);o|S1aZ}If0$f;1bz=n!RY95Lbq-Qaxpa{TMi-G;YwTtZf$++6g&9-MfpJ|T z$_nfr&`i)O-3L(2U<_mwa@@F}tY{ULzojK7UPkh>?3I;vFPwg{Rdq3?DC&!R0Hmq) zmhVfp1yxI5mzO@4j_&>>Sw++aM^;QL5mCvFFtwGwQH0e6j0gtnSP>MI;^Bec9OO`Q z2ra4oionhHu@Q7rTq&4?U#&umJG;>W`M9{sD1oElMOWnvJCL6>s;md8<0eW8QKeB3 zF>-xQuk}&trKrU)v1*Y-b|~pe&$d}Ca1F9Dqhbkm#aVL!)G|F|{8w{F)H9kislMo@ z^fQxow{$IiyQg2~(QMd*p6yTr_oT0f9hu7^#!OjV5~JZt%8ii&RwtOSh_*iB5K>*}leU5^nxVZ70M1AQ zgD?V8_f)wQ=qO4YF(wbgmGE@N1OpFj_oe29Yb6;#f=;dN5U3cLaqCaLvjAUrqGrxd zVp&)0?_m-pcX&F3z{4>*i5`dnACcmLdRBG7>8s$CJJCN3t;q7F1{^x;JJS{SV4hgG z4>=M-5ou+)h^4=sx{5Ie6C?*)sk6kx<7sP1PY^MIZQDAsrRgW`5Qqm(pAT&URVYp0 z$w1{n@}wo*Tc=^l5Wu>-q?D}3cr!+QoN}xK7W(Wtz3Pd@x{po*3fxbm2ZNBuC*csR zc5dKp7jlowg%(}5Dt#Ml@-&BEn8K8eRgAP(V=UhJiL?e|V?tU&4o0|R6DU;>3RUeD z3x#Ts?otuGLb7U3&AA@-t;jThEl;k}x-JUP9r9Xb%EWeyEMNvoON6&WE~CT&i>3#= ziS{$nFt`Z9ve}6kDW?y@7LX{{fdh9C=kQK{X`K$o@kRP;}kOADIMVqK>~V&C!VfY!pxAl zK9Jov8IUuM5WfSomNa|gaa7XGo~RhKsjG_NA}A+ZB_L!}?p5N8p=S@o2dsddSUIFD z1EQ@vwkmWCbK|dN_9-vY!qkwMsXcIpfHI*ZCF)CRAx_|!3Uqd?jESp1bBVsAX!yd9DVfh3kLNn67 zfJgE1g=mmE;^ zyF}Xj}4~0wULJ)8eebKrNqSMi(Q0gPHQbOQq>WhI;Ysb zD}_=jhjMM$9EMZDDcjtF8{%P44X|6MOGr3S%W$QpbDfVPM)M+UwAz6BLNe*cPQwQ^ zfWxg-1oJ5-&>+D8G9UyH zvQqi#1JcGICV>~IG1nHUlsty%z$z-fb|e(YJ060@by17(3SHni7->}ip>CbZwV1D@ z-<4o>uvia_0)#gz^a{v^#0*RYL;!>gi)(Mg)dumbKu2bMSu3q~_VA`YIFq326)mZp z*p%t^R)li9n_dmGT3tEwbCOgD>H2rIhqJ6fDq_J@fZ)h}b@cgbNH@mB1pFJTSb?@s z-WdHzC7&0RTVTjEQ!|zs8QG$u8!QR^!T_Kk#mr5{oRi-BZ`Pc*oIdugh1_tZMMz8X zCV&mX7vyADBa68}g!g`4YW)6%Ni-uD3fZj0xa+(I zfvKU>&wFbPMeee)Nzfw2^Z*qMfupT6F)QOC@CUyR`F85y@q>cS{ipE}AsUdB`HD<3 zs#wu`@lIE-ExFp%zhE!RX;8}J1o{}RwV)lrsYW38IVT2ZW2>YBw~H0!>#s?@C)Hk zqGON*c^ncn$GKKyR92ad^kE_c8NpJFnFE2ZJ0v}pp#&Ni%Z+kERCiG7VrsZ7KO09u z3Fo4^O=>K2y`nMRXuuaO5zXL;KV8k`=m3Eh2?fRlD-0@hD2Oz{vdf?>zOXO~+_lg= z4@ZukY@qWAEE*H2Z!{nYTtff1kXMO}OYe<(UpS`!XoYp)tjjS22RU}=te+~T8K7V& zLm*&O68FLak?!fOP*@2e5l$4fRZM`3g4NsG&4Z-(OF%*65c11#=a%MdxeOYiA>?O_ zlxLiM96IV&vs3R*mdN&%3q0v!3-jO-*tFRrs{-t|D!xx|fdT-A4#gTddKttd5$!q> zfc<0>8aj30Sw*KNSj-$w4zOrj;y7c~F-Id{deO+C&N#PW=vli#Z3DELeqmDXI-8{N zGQxa_6&-DZxiS!+p;={7*FIFplDi3=r`XczP=Sk1c4)lG8m@Klk&VgrE(ea92hla-{$b0w~4ydJG&(8suP=l-u1q;P9rM7%J>MpZ5 z2f7i`5I!l>js^)IfRgCLg*+xdGad2KSI)jy(!41n=U6a>6;;&ByvoGUbpQ{M!|gY1 zLk_9p)ivX>a!W!EGpO}dZh|C=q5FH!WM;L`7BCvtLVd+Y{ zZh(bCH^P$0-jI~4*@{a(d3ULS}{ z9p(CMvG!F$jjmflgU+_-JTJk@q*JKlLIeWFaLZ=w4dLr;vy@3(pLVAf!yj4s zA%2_*zI+PHQR?~9JhF@qeD=RxnqDIzxhX40+(@~y(3PMDn-24%d-Xk?cc?*TyU=y` zBC2j9CKX#2<(?V3lpUoS)r5z`w;SkTB9URmg$PbGN(1P=tL4&TnTQ-zAZIh)=YgkB zy2vs{yHqR?oUok;Tijw9rgR2pWz~lf){G7|m{$V~zd&rjB5i{XZ#JyKXqwRzdAndY zg71xLovW~~OU^o^+A&UQDqyCDAOUtrE!Zda3Uz#cXJ22MmBpBX0qI75-$`iMkaj;u z$p=B63CU-zM(|`(6e5<-FN@5*&#*X6Z&+|-%3@Qu$OdxdOLVHDsegz~_g#c_^6i|B zglJHJRJ2ln;MuoovEF2vRbFP-OsOchNaxB;>vr?VUBU3433!2ydEO}%hLpNrw^^kE zj}&I;7kiaxoc16$sQM^8FXAeZuQ(avihcI{rS}$4V(Q*+nuH@UD?Llp+nI8haoWtl zj~v%9m9Vr=O5Q=RG}DgA5}Ru*d5=V1fWRH%ER&Kv;}s=S>htvs`q1OcrOwqCmk$3) zVB_bradSLKe8aX_!hT)+5w}GfHg}%JfeBT}tHyjQ#h*YvQi} zl&uK~0j^c<&|~Q%Qa)4{z;JyUb!RQV0thiJVZg<=3yotT6No6ai*4^(;-*zj1CVcE z&Ws((nv(&Z%*NhTIp~&&W5Sv4n-SsD1ulb*#jxTyCc8k`D3DBnV+64R>65CYAYh^l z)5DKmPAaG!w24=X7Nwjb*9F*c8Ao*_i$|?YJHULRaLplLVk$P0XyDQFR16O;|DQW@ z>owocXD)9%IQhlb-T+uYr@!6$u#0WkR@W}j3#qq?ESo6M3@>F(+G9q|D z!pR5g*#a;jTniZx5OD<%(QTLpQ2H=Bab62lyaN^XQh4TtYpLRk^n~lR5gk+z$U|#y`Oteg2^=g1rWI5E%-v#ZxIV z&>1(K_-SLz(zIVgMEGoWoKHbOtLo86RC-e5ODh>$Wz<@?+ijSfCRJo?HxQSYA4s81 zbXIJFB>?Jqth?`FttOw{j5YdKr%9<4%+{33SMOlN*R4HyRDmDlLa2yxVzY7|1J;;R zhZ#o^{5~K7-bIb*_{ZT=2N;_We9#mQei|^to(UL_dSF`UA{6!aCSJ^HVD&}wv}eUu zI3r8Vg&^UhyQU~X?gWu#JDj8~+=9p?DzKQ3Q)e|nIQ;NMi@1OfVxQ4lfmegn0yE2| z3Oskhkds{NWIa&XrzLy5ih*Qs7|Iu3&k0+@ge|^^A&js{ko`ugg=8CsW{J5+4Cxm};hz8{iFop9Ps16OL zh z3CiD}o1c`EuJRYo53{ah0_J?HYe!XBy59n{vqfdd{6`Lk(K!cwh^oq9spfq`@Ecvz zu6fCT_T_c!#zYqd!730yGC*=5!eWauHSY%D>vW!&<|54l;A3L#V}ec3ddv{MO6z-L zw*>8>w=;YK%VUs!82agS)*M}&gG}CrU{GTj6(=kQ(4Jis= zWu5g2Q0O&K$jb5%FDijX3z6qdE}29vphhCP(CE}4GQ@Wsq6tfB>=U_`AUFA>&P5KH zYFtf>Ya^35khQCHJ>=s))U%qxwQDKbAuXTWqvHu%yW1h9{qjws-=wW9tq9jcUiAt_ zMq3!xZ-7u_3asac7!+!7W`c{!K(NCi3o(917&WO-s~86cEm2EAYURBvAuW|b5FmjL z;0u%T=SSD@wQ78r113(EI;w171|VeWCW^smd{ys-l{B)k4X1TjRa`<;(Oq`I z)@E?R`Y;J`06sAffE;0OQ|lTK5YnLvOdwero*rosuYMkm^vV4fY!++w=c@|fL;fv!^CoB zP_by+R`tpO07oLCQ6PX6*+9{Y)n&#lqF~Ps>2=ZPg`#kYxRvdVF2|C{24E@}5=ye1 zA*@U(IR$dcy!g(q+Lp2G$iZgJHfu7YOwcR2e(wYLTqXl$mI~R$_T2k_) zF&gNkkhWd&l)eGUMA52`1Lvs|Y@8ZC1z@Q7_!O+Q>_d)RmG~V4G{J=H38GmCL^_QZL`$a)Mowz)-B>fcLbl{4g5Mu7yh@+73GUQ?x;_C$#rg5c

ap?^dW#gV_%1cRpXyBC7ue9 zY=CkP>J4QIq^&@~Q***|!Mzd8_X$f)!vkNTE#Pl%kh~nJ@V;_=#V>KUq#!VanQDB* zs4z^kZ)3>b?#1mS>Yx!t;4pK9lf@9BmT@|GyPeIoiA0&De$A{zBcTklr|IMZLIGSm zjWCud;#Xcpyq)!Xh%U0a=%$3i48D<30~sHLe#}M|H8=+OSyDrOMa;PC<+5bm(}Mk# zll86K8oL08Gh^}-7EWlOvgNFU#1?zT7&f)in1oUT!;IYd)S5y>p!86TeXST`qV#*4 zrNVyZP7%4BNHjHcna4aD8iI~5m zVqeTqD{YOH#jb`_Gouujx*Rzv6imGQqv`_8*EQ^62wK5peVJ_cPiLg3twV#QMoXsj zQG*W|Jb?y}X8?hk&jFjUTy|ho<4Ye=$!RnYgrqZrRdA5VG{9i#z(jKtxpg$&y$(Fj0m+OkgJwhcYoP zB?^`b?G}<;1cvtQ3f_fv^Kq+(4y5^3YCK^>wIM@G!b};bunn=nweSg;Xqiql_IDfJ zNL$2cZCZ;C^b*#5!0n_<>RmQP?6uf$cX&F=Ec3b$RCQ&_E6iIUTtL`0zN8 znzVGStCWcc$^qiGS;g#w1vrtQ%QP-LkSA=*7*+?deZNbTS;=mM+3Y&;fk_>sE;)}~ z^MZnahk9_5U4-uDJ0)qsBU#ss0eXx7Nc8y4IKMXsV@{F?J5jz($qTIyY-c zFb~|eVC9mei1CPJnYueZjyA30#rUKs{~Mqf*(fmh2v&P!iy5X$FrT- z!xBfgAS!4jCp7TQJuU3G#j)|%#$|0Lk>XHQBkW7IxMMY?a94&D4IIC>EGkqOuzm6` zRHwEC=c8G$hMvg?i>V8(Cfy+bV1tBO@8tDxSb28`l?~;yPo2mhgG3BgiEoi>8&4s| zn9?=+NF;v1;5bQHEqX6J`5d1oGnhl8C!86F=B!Q9YiFb(<_fSXf*v)1HfzFr5xuij z`!^t-#M?Bxx)WTUNHEYiJS&w;5hPWdx=-!~S(7+2mQwhIa1vu$w19j;LxLJ1o_HIk znh>-d2l9=G-I#eH%K7`unED9jvm{ujJ$ofb9vzFHY(h9*74YGPbhz0}gKTjmd(eh`N z9h_)a$CqwurqL4PAt-|36DF?gx!axX+hT2`K~;n|&vBiM^BHTWv!jR=a>2pYh_p69 zMV9@K^Th(gv0<>T>(Mg z1Qg{Ql~CHwY6^D+B{luIx*f^2I=2#gTS)Unrd#E}%BNuGrs$6-C34!UTj62H|s`Ois-+3_>Uv z)wmlyAs4%7b|iz^hGXu7&1rK;Y5!5KoxI1#GyQE;unWTq&qF39TBWk4~kQfjk&k&D;nx$1vW1pp=tMFooF zIXnznE-+Yx#+!LF(XtK>;$Y#!pp{t~3+fO=(r%ztW*G|D;<~ao@+X^X+FWGLvXtP?8*r((l4HItL|3TNJnv2d>#9K<2EY)NHG+c~bi&qL1o?Tl${`EI;uP@L zQ%ZDJ9r9R_?|Q>$jgeMT#p3&`Z!q?9--En{=rB67+T@14!V5-c!+AwL$Xb3s$9(j> zHXdsAAJ*LJtWA~_T*!57lNwj?r0HmCv}nW8`>Q#aXx@YE{SXW@5+*N6(2 z>%ANck;v}nkFh$O^(abY+@lF-F}{llCzOO2!Qsg7C4}z^U(k`Ii4PKSkwQH^j!eX+ zL{KUwFUXj&X+~#jEEZH=wa3{Ek2D}-BhS8HY~-?fCSdh-xVbC@= z6ZPY}{BIIOXHMXjR5CvcZ(Evs_|_F^%t($8lBD1*j?FQZkZ>?Rdos1((|Qx1x|| zI8|p5MUj-=KLzpkBouiL+}!~A-#k2&4EW3k8HY=EpHar`VMZ3@4~S z=94OSeT>$)AVDnU2eIu^j+i!q!qviH*hi*J8E9(U>p5_d!l@Gib@l{BXE3oS7UT>V z7~q#Y0hoBt0+?o@aM~TDQ(u+mMQ~DUut0#tU!W{hm%w|r6!{9f@OKCEZ*(** zs8*EOu#EwJjS;+DhI@Z;fb@ZmF~ShShqeUNBuTPXfO$eKY7rz$f^ai*rJguO8BZ#Q zz#Smej%mr%=&DA*p#>E(N^O}Xk%(nc zpmQ0m$GV&Mtf^f^%QaD=tzgu!%8v@(&93JOC7dhd`hHh!r73&Gs8^?HDfhL9nfY;N zEt3$#kOiF9E(9WF7FOMl>8cDecy$p3pdQr*Y9z+1}8{dbSj>>^>^zAo}&6&NV!19*yYO^h5 zrhH8{$}A1MA>KbO1iy15vgf>?7ry$3f~4Nb1%+jbCy5pA;q9p1__``C5SYlb@J}M3 zWmO5#5Ez3R=D^>K201r)fxd&5eoS)-H)fpJ@ldKDUZZohzzf1(0py+iRwd4CW{>fE zCIfUl3>N(WqQ%81v(BSuUV)gQk`SQ^XJdalgNdsLNSAUwzW91<$UtD7lSMr*LSm{k z20jOiSFsc(het5Di4V}#5HaJ#t_M7(nys|c`*- z@e-729x3UyjV(DV_{c3?HM}xg%FZOV{H=$xg0v3~fc|MmJgHsKj`dash-r~UjI8Jk zAz&U44Hr&{RMp(K7Pqn6M{ac16@`Q_3V}n5a4xA2%#|f6QS{S3cvMX*qJ&h@#amFG z+caVLreP_7OtBYi-bgHA3C9j-G0v2>aXkkpCGC+0&67^M<*cJiJ~0ACWEKDcT|2YP zMJ*g-Hm1h$jHBx)NWMJ7PnRL zSyO(PIsiw&Awmon7ZI}4y#(m$%se)I2nT6nbI2Y;xI`ZM$clVz; z=hqx{<=@}0-aUVF;{Q=qEB;9-D!lgIJ$~%_+n#%WcmIFRx930Jy*lIl{o8-HpB{3@ zKh6b}0CGhD@|d>k{_&ad<&5WGFv-~i?v5B3Zb%<#R9~BRf%kBRZmB3k13vexHLT2< z!c}NwYo*5`>KqTUJx9tBd1zXxmP<0*eeBgL+?g^JhNQX6cGsZDbEiuAPOOieoQLU_&yCB!UeMv(kT*I5ILZ()LV@!-3!SZTNCnXM7?6aYvtI4 z*Fyg^)2CZ>x*_(yt3FEXbJt;R>sx+cy&!fV9lv*@O5q&>I^00EY2EH3Il`}>UFIZiU0>lg7 z`*ft=PrImR7Lr0yV3G+Pjp#Y)$yOvGbl&GA;el$wgtn%!yQID_y)YVpP9d;=J_DXZlMsY*Sdn_qrSqJov2VZp1i=ey)r#Ow7 zjTlf^uZ^qF^$aURxT>$RG{WH`r%uFN*!N)2#ON+&x9)0Z3feUl6&l19a7K`CbO^-W z4$HKHzLl6Ma`ujEy@GT!cyGKEa@)?ED%(581)i`MH-pK0+$w32?9l_uK2(oniVuY3 z2HJ!~VxfgGZN!9Ht$Yu#i?qPt_5zH-xwj3K0dF$vbdT3)5+*OCqk^Qhpk2;y8!s(|;K}I8#+g{nQSraw6@S-gd_Ts@z)tJXlo zst71PS(=3z9tH(#an;m=x1J2Jb+CX5E&iin^1_vXJI7__D&MIeG9+gyc^Wxz9 z01-$I0Sq_2E~CsBfJ1y$56D%M;?{oE=`+s~YW9db)j+7EGiXPCqUzsV@Z)U` z@X}E5;Mvt0pKn;0<9gR=XG72+-6LL`7O6W0oWtZWndmM&2Ql%JpTYdBcJ4(5G*)if z_`A5LcEBNDSPkYh5UhvB*r~MMu;c_YNvozR8fNf zqW<$9cS;f>cwKrGEog6Cgcow1=M3y%WD0&)yiFqA63zn)Dq9F5klZ?|(%m^c;U5gt zAJfc@^_PU|Ey7F`o z2bX}m>WJVz8g~U;VBiv-HWg7V3Is}qyg?9EA=Jk$pL+_z0~f}$dQ4tBV_HgaCHLJE zwDkqhd$K5M_*vX%tuLpWpJFx!R%OTpG$(UaL^krYbed9R*4c{Zr#S{S){B^-o^%WX zqB@>jTY!Xsh$zYukKYRGIk#+l`#wTheXjO#V}Ln^=yy+hgwG3hDYW3;4rZz_*eA`c zMdpwzRwLfoX|yh?nTt~?%cSiar`#Ixc~35RoT=Ko+ah=x6%wF}y`uI`4ViPh7e|ec z#6CFj zm=$XWseG%gP+|4y6c2vo0y z5iXjkTG%-y=wC2LgOd;o*2H=8&1vmFCXjKWk2@-T|mPueM3 zaw<-3a*lW27Id(z9bKSmm2)xbDhY~O2(!iGxLc$U&bm8Ybo(sWhpa%_=`LK2oV#hb zRB0|l3Zlz3brC)h<{D|k_JkKlb=wL#0TPDAg1e@VGLYJVXAY@8vNA1Qlto??)fFja z9vUVL0~s6}-oDc?(T6>rZS(qz3ucGFIY@h9l8*K}yDxsC>mPCXI@oxVPk3B_t=eo2 z6b0|BDceb(t_}T>Mr5n`sN({TXtDP%F35~n8&@`)xdH&Qb9MavZHoO>4lyQ4)M zU~yoh<^Cq_Zo?Du+S_RNkZFr^c{^JlLk!uzsk=z+abP(0$j9D{?#~=|l=wA0qAWrbmee zX*`}FRgYF|=?udR-1hR5FuRHJb%euG!A4D*ZWzgDbrqF~NVE({nEB(tQnz|w-P%CP zHZWa5Hv2tjjf8dCZl5}x-)~R77|wdXdg z1_Mn7#l=8)Vai3w++Fm7CK=6);RrvALGfgAVdFv}JzX3MxZX@_F(VJTms^DDWcw-= zn$os7CQ3m;_Xh8)=B{B}AceN~#nsTSgh^4|ZOj{mTaC2`dJroF6D9D|=!G}PwtOb! z1*Pr8!oV0sxf~4mO2`e8Z(>cr4wC}Ui=a~8R!=j^F0k?DzA0i79m`4=s`5+7xCJZ(Gj$UbJx!F@o?&H$F zrX3J*Dr`V#C#$2f!tRAMJPSbAd(>d(H8q}o3K+5&@Zl8Bj1lmZ9}@Go0BT}Ah!|Jh-ghynW6!ArWb96~ z3Fbo!9C9TCU=W3cG=s5Rw&WyPu(bs%M~Y^tUggNo#jej&3BxWA*bQo7$W}#pCpZ$P zS$l)*C)vABk2~Elxp0SuL>X=%0=5HsyM#4yy#qu<3kW9uqx$|^4vz`UeQKJod!>R{ zP4O_5*G!L0I8xrs(uL*HWjaAkx$f|#d)QhyJ|N#VhfYY7F#?7z&wD0d+Y?TCB|Yj_`qCh?9Sq%9Fm4* zX;@>`A{@<$hzh*jV%hDAuoi@z7O;xd-6Yrqr8XHuyqku5&{0a7W2X4tLrrRwZ`{iB z+7w<3WZwC0FVz;bp=5y8MM6Rq)94d8-Hk_7DVnn8dwF>nPeqj}Cmku;0Q(QfRTY!V zdpbbkdUP*^#F9W<5Zt5UzWXlkhhW}$>0hfRy%{XMsNaORMl^8fpwLU4#C-PaaL>}* zTGoAt-gC=;WWJK!O)2%JlPJK5q(r3Qde@R{WyaZA4K;kNkB6>|o;wdsp1-yuMWuc3 zsa^sM4kic_(9l&-I`Gtxm0EsKEnEu_^)Oj$aoK7#BpB*RhK6KOWhZTgnV+ST`Ntx`X>PlC(-CiO@D_A?ozN70^SY; z3wb3Trz{r93NyizV~McDO}3S2Jin;Y_LY!ERG(=u?^CZ$KthVyR!qXOG*w8M_U@gA z<~VY3lff`O5QVNd%@D6 zN2jm*1}JQ%@|dHP$|Ez-a6r{PMjq=U1J0%kdsb%))H}1{uyC_yRf;;u6AEZB;aQO9 zx@8wb5@8E~Rf$4D_ZVKO*>xtbkYTDaRpWOV;L&7&RqHok3^3r;D`m$p1^Pi!Q{@+i ztz2nK1xF*qhGrUa(fLQ@TqE5(Zzwq6DaViJUeCDeUwGnq-aUKfzW#gX`^)qDuDkW? z{wk=xFS+^2|8^^9p!~$m3XpUU_Jt23S>BuX6e<`I0V1E4 zu@0UCN7@5-!I7=pXku5k#Ftc89*1{zuZ3#fAHcghb5?^L9H7w4wrxSoLKIM6iHgvu z*Di6<(G-kpJCdE!DRi`&wR-d_zLxAD_R$!ZISbwgxs=Uso(G1`ZpYRH6d{uek;W{P z8@ofuz+w)pZmcI0eJUBUWS$B=Q?Tw%Csu7LEFhX`_RJ&t3k;XH}>ea#EQ;e=}J_XS_ z_mKo4Wnv6^;PP7#Je{0y5;*>zHpJF-Vf5kZ1F`0qFWO#P`;BtU70*b(91`oO*=Zya zi+6{Km#OYZ&5t9>_Z7abMucYlmE*rMACu)pRCko}HDIOQ8e@=eBs82DT!4bO7Ofy; zux`FT+L-VzK*!3_P36{LhF}*l@(q+kth)~wF|BUp>~Lm#VXVhZ*i0`MkGy&{YviMZ z#-<->6x(}}S-7b#<0d^73xS;#P}h?fC9IW1B~_u}T{AdERL3EKudk=nujyy$B~iQ7 z-q%*aEUeGB_0R&%ZY^|Gw-dDn6;XbG2=>RTfJ=umsQcEpFsNSyITF{Pr=Tnjeb$vu z@jWR!(GBJY$&Bz%93rO5;$w4M=ZwjrL*{w- z?*k)js=DiEJ8|J0Z+4yrY7C7D9{Hx#oChW;RfofhgDCdCREJU2Mpc&>svAqDJ94-u z7NP2U%GD|}gXeIaZ<<4DncX$95vEbL6tCG9qtcr*;k7Zzo0g$glL9fXBB0}Ijxv@$ zV@_QZ1eu>(va(c46-D{eg%2!ZrA(4Snc1WsXC7#7y{)8qJVNu`4Xv7n3ZNHm8w$*y z(x9!-j0ejTYRZ58wLn?Oj`^aI4gn>LVG%8j6{g)=aP5LwGT3w34SsP zv{eQ}!N8#7jm9T&zHd!5}yt@;-vy?+RfS z0F7ao9)e(jRJj_{^SF3)FDJ%I8t6-TxmTEekR8}OQMF?YYh9Mwm)BMt1 z<4Fcj2fS*5g@a^5cSa3RBv7pf&7PaPd`UIiefvdDZbQk8FwPrf+bnWdSosDw*@IY{ zexrni;`mTt1pvMqFPi05A)6PxDhrbTFbS$Zka&uRJ%J)P)h9ELWRN}Fmdpn$QTNIj zAe;A2Is}y>@*fsc6;)EPR`ch{caGx5CD00wU<}Xhfxb#WV?AtuAjKRP^-^X z!vK!a1?=?mCQ=5QxGd3UBen=3gn-6G!c4z0{CU}uxp1;1V!&a4PZDR1 z#KxXY4e?kMJmw6CDCgWal6j!y=|q6p1LB8mmw7ALQMnrV6s^ANc@Y^=DHdrcZ1qBe zV((g_!b8$VyAVAd4E3H0TWFD*?Mq|WNA52m9hjXM8~1R#iShbfYtR8eE;J_V+cK?1 zXV#n5#XK2B@}WY-*z5-a5!bR^h z>T7zl3YJQd#$*asJU*X1TEr!}DCSPxtvq(F{oE$=ng%WTFrT$l0dPvujx+MhFB*>4 zPo;Nbspx9$s@0c>S6Zc&7uWm08+a1=BCI zmUrePB*Z`}-Q}t%!@IURJ>#&!Hk|^HA)STksSBE{Zjox)xQ-c?1Y;DzE#j;9mb5&K z5nYeSpiTe?=Ne_D2^xh$7gfkXDb?R}`^yd!x0lz756v{Z;KWm~Q)4*$fmFgReRHZj z-WIhALD*WC++%SRan-Y68&&kMpSLv(IBH4+qY;uYB3KdsX?7x6%|o5Zv{Rq(8}N)6m4NV zdwlXs$v`NomR(|isbG}BRgUrlNc|^KXaOI`^0yU^47ku(u!ys5i7mJfv$;c#XA;L z#8u0wY;gOJ5YbLb$zD`;yFTMGwm%HXg~94LJr_!-Ppl`^LFXvpE6-`OY;6N6_YtcC zCH8}*E*d?H))7;4xe4RSUqnRu=P(Ew<5a4YoaO+KSj04Xda2Ajra+f-dyL)_Wy-d6 z5N9sJ;)xw4Rd)KDai?YrKIv7_3W`z&5SDOwV^RVX72_2pOGaqSU1S3ZYR8ATnC~0& z?qpG1YCsHwmg70a;Wg^2_zn9|S7W?M22X^8i!RO191~KD$KeO>WqL;O;tE6ZX6bW8iuXv}sX+~&y>=;ajXoEzm0C`q7il8{S*Fg+M>CLVZjX+hQkKL-G z5%i+W=SWpLNQe<~+K7tfaYHdmyeQ5*; z>FJd#+3Q z$ufNo3pW8zvlL-#!DpIrxDSvZ@QuYCN|Yna9_)e%5QyK5I^v})D+1C63kJNjJxVvR z9BpV`j{B%9^(v#4Gp=m?ibO`chp@pyyHFGbAcd6b+DY&&LBi6VcCu3_j1ZC!mz^E& zI^-SFgoSIGm7&^QJtQJHd&Mr#e=^S;;UpQWq=Rp%<@V}9hitU^lkmrw}z1jMT4 z;FE?NRN@B8+LaI0x~cJDLWD9$hzvvr2~vdRrf_H3c9sn^;E&I?DyB4e*d{sdHrNSh z%g$kL^w637sJ8Y$_^|| zPnhtOb40=#h}UY6KqJAWY~fK=tPuf+_ZOOGV4C^~4ljIxEaIPEqmx43yw1k`gSjv! zv|Q&H@!k+7uO*(n9Akk_Dh{{_wE|uJ6jLQ4RIXAkBGa~SZ_RA%SnfC%P!r~wXoGzt zU{8=hdxfdxjCV9?>IWqOz;y$8f^5M=N*u-)xQ#+6-?1V|uiqT0t2sir$S7i_)?;53 zN?if<+dtWvD$$GBH*%!*5G=KKD{!n#$ehB1v?Y1ntGO#TK)Pl|5>w?s3+%coX)_Ka znB5k9uIm!0GBP>XsF^qV!<6anIAF06BY$B;7!;Rib@C(j}L_E z+#j=05VegX3cOIKf#qeYN)gB%1kGqwl%h=B!fcrjR4=7i&Fo14>_bKp+4!2LIZYHb zf<4`X#NGkbF*qGgtIURx&@E^qtBh#-6%w9s30^8sC@Pc}bgXYkONq}_>Ep+%Iyo%+ z)nXe|R1>oyXA(ROiW(ErUSyy=9osY>l~^P94oG?TP4NsP!!tx-&OorR*4plB;hbR@ z^?}yH^@5#LR&hPyRcINdzgi($HC zbwqW2Q7yYbn^a$6RZlo~IhQD98QNL{E74&IeQiNQ(E+W(`5Lo9FJzJq@H;LS5Sl%6 zsLROtuXY_rm{F++l>41MDI6EhjKVs z*I*APrZ(_+lJI#9$HPStEx~olv0PE1AuP{A7z4@IIV+ZUXgdX@P9KTGCa0}BI4o;R1!5xtx`V)5 zk*Y=ZZfK`W#WbnJGK0kO8&?%cCuHcb8`NaZWzD)I)706U_DCt67<5Uyfi1#d`39^x zN!O+fJtq;ajk52#82!pHO(kRvWpCz`HEzYx>6Ud09g1}dRcF5yS)eoac`anxGD<34 zzn`J;jFO}*+^jgM-Gu>ggdWoug7^(Dsgm$K%45p-f-XSF;&v$w*Hwnx)cwk~5*DMA>X=!O%3`8l}eshe+fwVz`5 z7=xJ7(gZ^e_6_}@OF9a8uEStNE2iX5dm(`!|WYk!WU@VlYCm4awoBM&JRt&g-h4T`#GrgJ20RjZ{La^p{jfDaSGK z!>78Vbjhk^$B1B+r>9mLP*7(X8CnO&1FT2RKd+j@KfEuc%7Xkv*%ZLs1=6l#Gu<<%|>2A-aY#+LMTOJi*JxY2$Ho5bW0 z-x!Pn!g5t4S0iu1r75gmUM_%_@ktUPYJ=!j3rR$U6K6wgy9hbh3=<3GnS)IZtHsmx z)gD^Z!e9?(vi^1{5Qa2Fi)JV7QQ1`XM-2q1b` zE}6X5%ZNQfz&~udr^+Hh`}DYRCw+OZSXKzVUh$RMlxiJV>~2~<=oXXGllvDol+w(2Erbt;y$19OK3 z5dAGSbOx(Fslz4G`h`crSs*toT?)#J0oFn#V(38YOdZM?pT0l8y8GkTf1f%&{_*Sc z-(Gw>{&D`2Ab;AG3*kO{)CC}$0#fz&(Zs|ErIcfiMD2;Z}dU*qSOe} zpT%N%aD+>gaVDsQJ)c~$JDktgO6Rdkfa?j;&V zE_-njAdwgvak(dr@}}DZPp|y~PgYliR1BbTY`WV6j36M)V->R_zkw@*#bK z1T)593^BX9{PN;KUW>`$4{>k|iBX2hzq84Mjh7W8NbA1We^qW{b|%Gb^D+TR7zhu> zEMf)-1nmIG>wP0{JKZ4-mQIPGUo$bk<5wT8gJ8y-2fJRv9wFNAc8z(Jov5mZZ_O8BbEmFKL+?KauANz*K>1wryEqXeWOf#^X;lEgsAAW2cw+9bfmAJA;Umri4r|(t&fm& zMQKroM&oF0G7IX@wM8$}i&Da10Ql$Op?sh-Li_D=yjvWS3>SqPd3=K`;4)*rH&_Qs zGds=%Q8r){oL=&1Qd2_rrTx=n%9vOJ{{Y?Wdft6m= ziOWAfYjb7jWUGXT#JUGalVc@1VI?WY7Ra2a5UPh>6%QyrN?9??XV zR53|cZAQls?ZjKL$-v4Q_gE;{RT!K@TY8W-i1=*K$tw2uMymq5e359e?|O7? z&?|_FpnKK@QsZ+DuQ7<6awZ~BK|-`I=Dd)Kio86?M}VP|ek!E5!}Sj+1PEx2Qc`uc zL*xKq1#-r#@di$h%aRo4hz58rCpB&(LSG1>d=$GD;-sb)vN{qX011frzXSr1Sm68? zQn$SeKFSs~aF*dB)}!nT zycf_h$46V^Fk=>U$H+Sd#atp%{JnWJ(z*F*LxQ-Xm6@pkjS20LY=e8S*q2uTY!cx8 z)t5pe^pb<=mqbup&0TZ(H2fxH8{_^qm9k9pqWRTuyl2vcTuv1&9T za8}@LrAkCCBf>gyo=A;8$WC!6>%pX23S0qt)+in{bIoH|l0_=l*}ijilTUC=jY2g? zAX-DlsL_!QduN^#UBsqxoD@VY&WZO4m&pDcupGE@)5*+@>MAr;Liq5>T$l_>OwV#M zrRY|8p?*MJxUfSD2BOhlX&oCu~SpS)h!)*bFA!s-l*a-@t>`r!&Zk zpDS(pI247;tX_hl82vz^%jn^Ikb-P=4%DvCIfo%;k&qW>B9qSwCaK$V?)$pua^wdJ z(-zQ_IM0Pw@0_PrJWx&KMsPO(%&nM+BQx`i49v`w@m)y_hN&-^!2~-;I|qNcrI5&y zQWM6sd3^e*zitL~c*A^Wl5oVd5XukpUC$XvlDIRpOJL6p@fDZTgb>4L$;0v18Q|1> z0U{tjAcKXFbSpC1xq*BOqAP3ylj=h#==9O?UcjF{s1RWWTY0kB{#7jP@+%N;r$XOG z0)-#Fh`ne!PLlZO{%ql%qY~w_(1(D)32S50Eqkpm(7!^(3tmXp>$$b&Ra2a#MlMhR zcQfesL**AT1`r3pkzG`B3GmG1)j5u7FYTW*A}XTn-#x9FAk;wHYvsxfdr_)YEYup8 zQbj;iE7rL+01*oi3F;|M!n!76^&0tjL!ctYh5^I4ZHjyipJaMw0r)}*qn@I$ujhcz=+P3Ev0a)DS^O1ULOElqP5VCKL zA!w_yBWTe?Uxs1yXzDXMQ2LeFoBQ|(hQrgP%07l z+C7K>^r4k00v3{aMFmS|WXb}<^gY!9D==hnA)b?Zxf71cE)K~|oWh|DKw!y~8qxUS z8?83|>|*)Q_s3Eq)7M>ph)4(r!yr{U7y9K4LNx&Q0K++&)VyCn)m$pfG+l~2g)}Kp z11f}i)kI556=k$iR&chre&52-S@}C7i@m!c@qdw92gge*Tb;C!g!N6mxQ9Ua9hw#N z-CmZ9s*RF$H+`<;e_%;V{PwAnxPc)$Me*K6(;y)-n~A1)eKzny79e_Y3$CG#>$|wV z$|~BbrYn1e)ItO>x&*LRa8Risyn6hvL~?6&%(P;LgWwNKyD&a!@k4G;pLnBv=xHEgMUllDoMUAV~w*`aO z?6n*-pzaCU8m!&jlKlO@lbZ)GytTZS5C+(fkfGwAjKL8n?RO1CI>fxf!7#f}J@rg` z%dQ^tv2S5w&Zlvt(b zjq3&wmPM!M`uQHHteg8;oEQ8?YLp=Xbrn-cK(CG1mU&lMd@+enJAl;B*|G+I}J zSpI)WV#6kUwjT|5P#fGr0!;FjvJ(;UcnHtoW|kBcDG#qAEWyeA{!R@R-&{@tvBWt; zI#ucepwTZ@YPSvYX&@pR)x!HnrsA#b*%4XfA_PUX?}|gjAX!DVP@p(&0wkg7_qf*% zYi-2q_&U8|e?4>K#y?mV;{f@U6Ou}ILr$SKmGj*eI{Chnqfux$oPvY3gyPPhESfjo6X~otno37 zI)gb~DpUhm@miG9HXg>X zkJ4a1$=9Ic`O1)im*x1xu`r?4ONp71>njD=th7Ctgc2o{CJOoxC*G9S!243~#A8T{ z;oMaDEfo{sRV7v~AL?x#mAJ4ND0Y=lVJ#GxGkiaZ3Lmi$88@?!5v$JYad@0|DnDy3 zbMu66uxG=o7@m4fbGOkxgz(d+U{fAYP9>pnaM1PfguLA>3TYq==nmj2u2KV~Pii3W zy@(+})u10K9>qFUAVr8kOXn;IB3!vwVw1ahC#lw#grO-C)N^eEg;7*^65%GDDqi+F zu=sM>_V^B)Aa&MQv_%;%h3FTKQ{+2cm)g0P%xq{^9>|wTT4WP4^$JAkOAQ}qmRSZW ztHhzwQoH76-Fxkz>@)+VBFAbvrNqmHdvu2~W79j5R5Fo74jSMZ9%3aEZG@R_FvopS`#GjQ*r_U9|SbH(?~d*3_%bH?w-ZhCLs z_4DU${dV=&JNvh7{)JcHee?ePbNLYy_wBg+CO@3?KR7E^BcT2@&TPe9P&wX4>|j~^ zM|+v3TkopXdexkU%FV(ABn`?dbA%4vz zL?isaZMedp;+qPn6HbM3cGFvzE4fC8nWMW?8)A?h^?4x25!z7u*^pA&*~w&@(G;XH|&!7=FA&WVc_+6l{y zgl*N658$QGBdLlAs*>ANKL)9DfdzyO?ZJ&z<4vn!L!^KvM~4#LM@isXn7G#!Sf@Z# z+VR;;Z_dc?dA5J4dbtj@vW|gdt02;@1>F~*{bYXowg;Iv54%pm!O&p~NOvmX8Winm z0f6a9jjhxSz-cH?Y!bgV_Q@ z6)xB)MT4_ie65i7uQK?=J7Dx^Tm&!Jn&HsMJH0OX&d+Q(##IQgpGakM+%;cAF=Ha~ zueJdZ!@92DgglN*szAaJhsYilBH?tQKODHCvT3+=6|5Lt@j5dQ2*REjE+~jq)XO6$ z91*3Y1n!Xah`Xt-amA}YY7pkg-CMNaTCuH|(|n5pJRw$Jd5Ef3W9yCJW1|Ps zX_=fLV5r4pFw&?H=uGZkbpKL3_%+X12J{+oa7@PM#&g@u?D0roM^7G{Y!z_BpbJtT zoMj87%5Sd( zji%nQ5V@nrC5a}vJ_8UYgO=4f$8vOTl2jD#$UK^#t+7IzS)z|C9ttpV>{_4|m!l}k zFv~`q;JmFspxqAfV0SQFb9mP52|0;ZJz_e6z{*n!4(vLbnpkyb zc1#b;FUZf%DO~YNNQy>XZt}=TTykG(7uqx4gq`g3S?AWPHa|QNNUZG`_;WKzGfEB# z(}Z%Dt28Ma!PzD;i~|2kNO6W&HPHcIwI_G6TGg86#=)>s2anthG-^63ai-u8C0?iL zKS)rH1nrx{77f}z4J0eQEDn^Rl$*uz22ZGi82DykMp@%S&)Hy z>~x2hXz~GpA#_%Rh$+3f%cBg;Ge43<)eac}8iTh{-xDPP^n)8fun-IiNJI@V56P!< zlMtUf^qL}*q$1UYJz@~9g|Pf!LuR9iT8a4PMNkQT$povpl^&^_?LTXNq1T7dzAH9Z@UTfd_$E%fV3(sPZEs8s0to7 zM=j^o+<9{h&`BPW%MsnIC2c2NCn3wY_B<}mb{vZzF7=5q^FNfvNl>QL7S*IG?C;G8 zvJuDE*7CEKp2x=FwCg{6JUwf%$oODn5LH3N6`r6M_Mx0ABr1lVQxdi`IG08$oes@zUt$2J%`sj=9xtloW^<2&~D-Fd=q@HVp;^C{`1D4lp2J7F6t_ z&W@c+W}=AeQ>Zv*AWp^}B`@kss|O?j2FO?J&02mAuLDUr2yw;HgE5JQ8mR+MM~Lvz z*FA}K&{liTX;t>@N|R%iE(32dASOZ9w2b~#hggt-P(Ir8n9UDgYIa-DQB_fxJvm20 z?)polKBR&zQz4*;V(J#xQ2AW~F9rl1=xVRm*nm8Q@WCY1!9l8Z&<3Ac5Gs%hhdqvM zY@C#gt|TikjTpP8O@tu0Ds1u8J{n8RdllNxuMs_<&terMeBgaUCrV3+`t9d!kc zw*(NzArYqoAvG7Aw`bB=Rlpbny&e!A7#De}CdL3!1J*meN)JR1%$gW^gDiywfZZRC zT!)A&Wmmx?4C93z%&&M#bbOX;L9!jOQL^90p$B;?&I^w>2VN_P0WEzF4Mne5b5;0y z?<_iVqk?d!?J^aBy`GJfk=eQiCxHE(oN9f-fuVZ6C%)C`w}~VN^)tIKpeq=3vpz62 znx!0pz~D{pF;7GY>Z6I(X5INT-Fm~c^^Y&>LfFKyx9C%^Aem0!MK>)Ssx7R#fiQ@i zV){hI#O9YC4RdDtGy`S`Wne{iV8|Wr@?mIz#+phK*}!pP1Rp#aZ)1yEw0f0X%6`jS zRH!iFU4m7#U>u3dNf*_*@YTumgvp3}1bDY|&k_PCj-c%1G71gql z0ufLZ`*@APoCXdtZg1m+mVJD7vV~xJ^nuR(aj!BOL{i62G<(|_JH~`b#hmVxw3EiK zdM%qs=KTV6ju8t%XN9_C<%}|imN9wSUxYQA?oAI8uFtapEz&NrZ6%rYl(V|vn;jIL z83$GL8~`UF=(cye?_p(K%Ea6~6`9`XY&baYY2quQ3$AKdkO*kWP@l=g1KSV`wvm$^ zl#&?^A!~siX)(HKhZFcDLeFR|q}&XK4Fd-hVzqcu1IeAbYV{fiXDScSS|B~Vhr&yU zKSsLn(&NThRiht}k(u==d1<)X6Rpa7O+d5`ECCQaV|xcn?IBUzL#eH+<5Q+maM9l@ zR)S|1KM!}tXzG4t5CNo=rkRZntc|jHAr^Q>J_`~9x^jH?zR1DU+2D^IF{gvP^zF*o zha~Js23mOq3?g<7VnRx{eAF~lT4V=+|2CrMg;E1y{glldh)3C99VSNu0!VfyuNpGe zhJ+Cj-f04&Sv8_ziE=f?!k1gMK1QqB>f1qHI0a`<2Th=v*8hrxbg=MBuGmp>K57t7G&^y1YGJ<{^U zZ>MxsGm$WRR7($5w8#)hVjsl>0AeBH?tZvzjHU1?Di`FHn6`*Xjbl1E`AKwAtcnSE zt8n0BBn~1VVb(Dc%w*!WW*i7TK>?>8V%%m>Cs4?A#tPj5^)41r8<3&4x12Gz+qZuW zD~H!7d84p;jKdWfJk+ksNdK7wx+1%{Un&(tVI5je!_uAAUHk3Tx{SvHE^B9SMnoEm zetZyaP9`Y=UMt6x2oM<1!y0ztB)iF>T=&QnU?l)0>vlyPVrk45rqTJtD1LoN>X{u0 zFl0&6Ln+p!G19KmOe?D# z6b48U^N~25&QjDQx{X2M{#BLpG2tDPw_-Th+H#occ-u@+fwfjhMJU5yXhK-IC7caI z1JXO*YG`!z=XpDb<49F*oHL!>DA!gVJduWlLmZqWE+H2=Th@CST<(LQ78lsPZzRF@ z2TV%91$*Up^-It=6Q+Zn0YMjJ8#2i95^)Ga?GjyEL|(ZGMs_m<5!5dyWO*{+-a z-E?@?ABAVaVnLmR}qZJRou0kSo1x^E^hIR2?Kl z*N51i%shEC;;`X)k;wA_uNj-RVpBN1NITa;T(FPGi$?6~!la&0JduZpH8GZCsI7-A)6y47@Gp#m8mRJ>F` z%VZ`ct7t&dt*!H*S|P4mdeU2ACoFuORGmqcMjQnS@4&x~&o~rOwUj|nboZ>~x#X@v-K06ZxZV*$9q?66a)?FF5XiLyX~!kem6}_dWV4!J z&WjDapv=-)Ws94_zj*Wkj>tbo3+oj7jHTYL}ot8E)5uhO}TwyXM=zjM^pA*eq<6!q5NMZRRW!OU? zTy~V@Us0ge-FoPSDP%^BSJu{`?<7aH1bNNj`oU`vRTK`$DQ!x|wu1CC&dvkEd~>25 z#QrY%p~jzAKHv7mMee0OjN$i|~X@wgFV-(!ln!gYr`qaD!{2uDHcK<6$2 ziTNrK4NlvV6q1!`88_SZpr&0rENQT5A}4Wx(z-hQtP0^;TZf{Cp_?DUp()BJ9g*aZRhiS6LQQ^0lI2r8zMPFE=!z=^$f^(245UE6LIwM=V7yi5Xm+GK43FJ1{c9bhRG+a1FtO zJ;;|Vevr1amyl77R|L9hGftu!Or!O~H@+D+EY=DlUL6Yd<;BgHQ9%GEk?aJuq z24Mi>r36LFPoF+0jMUL=3k&6TB>H`sF8?VXMX@}z+C#gPK{xi5)K*#fJN}& zyunI*V;joTPoBr+IVZQ11bSa$rWi{iwXBs2sSTBs0=L4#|9a`t@b+ZZ4H3kMfV6uS z$L0jji0@rMDZ{8#n=poiE%dh?iNw8lsUhp+5ME%!i9%xQ>Z=Eq_^pqH!#WxTlp3)s zAi~B~xkqkdZhD1S=KQ3DTAq*?s&!KNOk>h!rpt_XCs!SIQ3fW92rNz`7DK$r(-9Cw zmL!RHU!phd`X4y@k!LD2+8BZ~8k>_8fdF0vP6ZXgrwg`*=71r@$clNC2O;v1DMxH` zS_DbpFfg-BxJpofaEG9JkV3(+RdT#aFf=~_kem$TV4`O-jP4*{bqTB3og)w}^LWIy zBZa+EL~V;Y<56j|tqL3k2n)%&B=gB#0Z>0aB5lxB0oVi!kVzjQx5EC{%BOor)>K5S zx1YKQ)ABlb85D`;<}yYwVgdVn0c8pdzeQ~Li^K6L$G_pD&Wrbj;>)3Y0N>D|oHBMia2ADTY+*k#!uLm0Zvjg;+ms29@$QnKCr~_yroQtR+APjZ` zh>Zvo<0JjyMi`vBnjyiZZb>V`OJazQleu>5(04g70P4v<*pcvUegbJ@ZYD!E1vf^!xJc!u}M> zD1M2nP=>tRCUKz@ETLm@A$BE%cZCi+BmUP%JXKFiD+UF6|`RKl;QcdJC$aB9^f=w zQTd8JJkp~GF=5pl`lXd&_;+EI8TL_2G}8T}w#YywT#^GQF%2+@Jr=2 zFhv9@3j!D}AkxC4LNR_irE^OW9&-4!GSu?v!Z6-RrWqDQEqXATSJzw;6=CHiSvReq zL*EA|vLQ7nwVEK7Qeg97ju2fbxRJNlZDO21F%j(%fMA0e0DAXnL~WzPlwQQ($`_30 zKSjWo;)bP8sKz7qu~jBI*q7b6999k+ci@?~NMkTC3=81{3m-W(msyl-QY+xE9u)cI zGP&49hvM{%qpH?LxpgyaADc;W&B<+j?wZc+v*feZaI+`}$ivDK6exjuE3B}g8|sSz z%f(n05s(KN*y&79OZRjIV}Y>{9Ui%M88R7qkbSmfuXw(i*SsXfbv6bRtWlKohM;J* ztwsS09j(qYsWJ!wSVTE?t1l*XwhP@3u0Ym#;;}8KDhF(-VnnDo z<7aTKOvu6c_MKl=05x?#4q&Mc1z8VK2MzUR1V_dX@Q@Ys6sO9`OsI;Kf{E7=dVsl? zER!%do`lg`Brl8UtoV|tgRB6ifg}k4MpPQ{phimg3bM)uM1^sQgCMmNwpwM^HePm7 z(IvUzy}Y0!ZptD=vOb1$T!u2*h5+maPzhkuw+ye#IY>8gr zk9N4BpS3$Qk#LKKP@Gw-xfdeOEsM$x`dlp5#_v_2iUGlDQe2PLN+>?!=xhk!p?gp> zBSB{&?2GL04dpHZRMgm}t0tAZ$?$-=14fsTXEK)sVy(4q+dh&9Uh+x5pFoQpDUQi9 z^McTH)I~Kj8E92*6yvH7WdSxratBI_M-AO%fpHMEv8;t3w%D>UjY3Gi-%v}(#+#4? zD6iX$9vWcpwrdhRXIW5k+z1bdscRCN;6jWU^qS}1jbKOE8e&IvA`%2WAFM+J3&Rkr zeenf|aw)w_TtQXt$yCz1cX-~|5F-12aGdy0FhO)_WI80**9bZ~9DW0HHFb;J&S|o) z+OYLghfRnG7hjn7W9QRSQ!ksg!j1Aig_4he95rH4+s@1XeEAx5g420?P&#?6Hq^lMT9xdI<(A9!QwefhC}Ik za{mq(=4hOSyn_Off-0&2j9)DHn$rX!iw+=6*RpX0Y>3X*Onip}w&8ntEC!O0LYxwb zF42u9N^(b&-vMPul;g&0wFELa|0b&FrM`Sg5FlT9ekQTY@;rAN-5C@T6n)1QkW~ZOsK)dTeWcmPyH}vN)qiwnIk|9VX<s zlKH9BKzVlZ7D~gXEdU5gofgX6G*yXf*`%u@IRmOe%P&aq_u?RS?6Mz`AmVgC(z{pK z-$g#Q-yS5tl%1_6;m8SA@>GnyV6nrX!(3UIBwI<9DZt|1_Z#991G>x)$(<&r>m%V% z^4j+hv}R@v>2(g*fbEb-ccdsL3g~3m^W$h0=37i7tdKhz`PbXqf{z^1{$|BNo4MyI zDfG?b00<8_G~p41iixp%Fw(FXa;@b_tHx7o4%QF1()(ap@D`XKt~O zCJ};1Q(qTRr4T?wY(oeS??y**W8IW++5ty`)zI9KEm(_No(e{c~LYxTJx@GnIiyqMCV{mC#WHU zL`c|7SfbtY8w%+VwaKg3INo*f7?B0QUpu#zIVqL`)DhBHx#E(4DK@a`GAzFkWnn8=%-iA4^^hRhdIsbf5x$=e|f1c~P>mP7bnJh#|!xK=*EuIrD$VNBKl zIk=314CqV((zzRzJ1fM-Hy5kOQ!%oWBkh`ww(%uMe)ScR#l%=dv*eDK)q<&lS6hyV z-*JLEisbEwFF62qka$2WY(cOjO~gK-J}o~hjfdYgk|LIQzQ|hX+iq+riVl@lU2!^t zy9>n;rUzgs1vCTsL#Kjgas*B5K9bUXv=0 zP77LhE>lonoGIEyTyH((?x1>8{Kgl^o>Z~)#spLpJvFRK!8Z)RXe^3o}dkP)*d!ZnzTuqeHL$d~TxZZc>%cflKCV6vub zu&mQ)u5TA3^`e&;jCHtoUrC2fNR0XbDI7^k?q@BAl6aL=gNT$^XLF*%KqBw4}8A z>zl6=z6G_jwKB`KRy72IQvO)K5#on3DUmhCx$y;}>;s;MM{+85vr1|cW*Nsdy##8? ze0&2C6reC5Blw0;KKOgcj4aX*fpN@{Vm1yPY-l!AA(=IU1KG>(Hu~TbDnWI?_O**v zfbkIhAzpuK@YV+JG@Ja$zVXaHQWkNUQ8Wt)#EdvhaQ#kuhaiANUXm~eAgvK)6PSi` zUvYrScKUunBBqv#77*~+yf{n<63kyRV3LjqSr27!#gzlIXm~QPSG1ckLWzO$c z)(n^~;F;gOJcdlXYwxis5UvoHv5WMo1!6VOJrEBQhNmXxUg?Q7;aRzqh@4Y*-RLhM z`#~z0Sx-RY@!KLFECGqG--j~+s8RG}IZG@{wm=)Cj8^1@E^-kpK;#(1A{H0)pu zKySQc{W>*QijxC&BsWBB)_+Eo*65mZJWq_l8aqXYY*T~=ATnPeAgEPy*87KsDYe6_goyT=i}S}#sS-l{Qi;h} zYh-;b8`F+kO)xeb@0kJ(C`JrJtj$AFeiEWAl|^$%ymRgEa}J?yu1A;T2D?bW! zRi{nEx7ykS{@xQY=uZ^4HkSp2pTe#^eS#>uIIy)ySrD2`PRfWwA{!V22Mjg>m1rJZ zUzw^oJbu(-j^TCVs!kgiLA_+c8<5*6vdtouxa)`&07zKpltsD0^D!j$Sd>TAx|S$T=>8xs7{5KJQs!?!R|l|NB6hygpo%h%b!5VZgAg~v+pCe5Yx{l_p*#;9q${+D+rf=-l2R>f zr$ec8ac{DbZ3-(O?K!HZ3IZ6QO}E><+bV^-Tdf71<+({@abjvi)shPUx$>BzR%kPi zZe4mw`H0ck=O^X##^>pwR-NOMjpxUu%S{}cbciK;4i6cl928uy~WiUwX!o!Mb^IExeI&LcB2xrC1GutMB$gUi&gjf|dRycY3oik7Lz zAQ2G^!utC=pzsaMOv#U{8x(z8Gm9>`KWU>xkARIt2Jw92{5JMZ9POjS1wzSv+8*Us z(h%CGTx1R(Ll!kNv*r_{0lq-ZyJC|ZU8G{2L6GAWicv%O(^hu{t2p#+K^Dma1f7b8 zh2$haMc5~YY#9`;W44C+w3N%`J711HBE4&eTuL-}B2r2Yp}YYIC?48~({Vg}#|hCm zCZdyNp@Elhb42@zqSqX}>#WSJ5L&VzW8iuc%U^Tmjlx#x5#2nh zqiR=RPZLaHj_7R&{AXfXuS&mbf|xfBOvbmFY)$jR5-rm)gz2%c5Q9;URT3%zd7{HC z)Ig+_K!tv&f+Mv$(eb-~voIxW+`V1M+=4RW3tXRUYJ(abt00OfZkKJ!M_rhyKnby6 zks%8(`!m&p+x39|J<*7Tukd{)`z({Zi>R>-&uTeR$NLx~4nF(n2h@VB@@X zDs~opG$Ugp%eNqq4IAnmw%nj_k3jT$7WT-8#dt6r)qGQ&T@D^g7T&9?E)H1`B{&4E zKYQ1F=)BsZq`zZI_mWsi6#!s?TyFPYfedpGgi>jE57nKN>7@Cn{l{2p6!+GBc*&j6 zgD%e#=t^r~&7;$iPb+oGD(SoXhl3PLc)dQ$YG5fT=G9)FI~1ioI99Ba$y2PSM<@&k z(jvu@k|#HD{G-F8hK?{HXE?8w;V$fjPQL$UKL{t*3NBHy3@Tc^~0vcc{0ZCKn zMY*63RGuETi$p_h3Si7+Z(QGO@=8~^WkEwuq{m$((f~0wBkkeViW1NTJX7`%3>9Z$ z0<8q{T78i6kdogpUIe0pxgGi?cSezC;;vRw*+#o+B!?>}FN0;VIZY^;#m8oOXGL96 zPM4CB3G%Q`Xid#w8aRg!rT7S7!cT^66a>VWvkTJ_3yg;ixEurOAabs``Jk8$(ql30 zTS=<=^2u?`nw=NIOQ4+H8Bw-aivt9blc96&2&yw5y3)@32k3$($Rv|aQ(-QVPDtS^ zS0U#w2b(%DH-j;5k8pgdA*O_S3g?(5G6)Y($iFhNAH;PY^$O5s(76m3sDg&dyQ3M# zuyA6MRN@F8#n;&CS*lSI@Fz_@La^qlIEU8y6OWJ0h1QXk!b~u zR;PwYZ;4nhE4Q?gXTIho_QMrON&yv zYl);3lvk|xFjOM|lphLT%K41Y&ovJm{znD~`ozR+$jaWhvKJ&RWTfo*+}6i6n!YrX zcb-z}7cXoVui+Uno*Y2vd65BB*fS!Ck$cdC0x4z??6q=v(~ej>35*zlS+2!xpeJ~XdFjVuTa~OqGm_zu`6Hj|maRf5=cfgs-(y`x-zq-_^>Rc?~JS*0=Py&*k5nClGCk`vh z2q7I}M;t!)sHkPzLx@LOmldKL4ZKnx2q1(5Ow$Y!21gKQIUNPKZ9A%)WupujYYZ;m zV+;*sTVP4x(W?znNZ>Hkw@6PVMEia{`j+<9fge{yOwzJikQa)K^{JlBdw8g1=HdBd z_?9R~d>6J2wGLvVi($jbW`~F79G(UON3qHvjZmI^>_M~!8xesU z@cdB7#j5f%&%|mGuaxg9`o+n@L*1Cls)iCGJ_~TCu-qA4Jwd}3pc5kR9a7cqZ+r5$Rn3$Q5>g9LhxZ^{a0!b7%G?tl_@n)Ue-p(1@~>H(mD z(Z^J%%q8aAn_^2J8_JofQc=;2F5-m`QBRnt+Dj;1Gf}wOSHkMS}>|)vt zOCN|g-E!ZJL+2|4m6?gJ67j7vK9JXB5}Ga-p+`CvNgqfh?22Zy_IotSbsxIdXKwmj zk6I#KSW{>0!DFF!sX$`GV+$(a#>|*HT?;`ST2q7pn8em2;KTbwJrV~}WZj#{lN0UJ z@hy~yl{t4KQ|EP!>4ag$Q|rxm`RHLnqYQ_uP0^7*IO>ykrs(R=X9>I7-WrRVd`IXe z=!EM7ZoTMXu&+<>VNtfR?2H@J#oq(8(Kf3(a{HrCaiN5olo`meSCJRPs?!xSFr9(- z+J3Ai9CGA@^x<}phtSx60SE)5_XJ(63D_D4P87QzOWnQC6PDGa!*v6g`S23gD$&QE z>w*H6py+XgaT`wpak#l{mlT?pTKEskF90pcZMh~9#C!+-lgXZg52!YQi&sSiBqYLlWkvRVE5&D#J;J{sU2JnjWTU6 zt zZuNxmMm7fFdMSX=3M3*N6keZ)N{8;^VIsb@1A@VSR6>(KFYmC-vTOnCF9`h$E;NBI zJ<&ec^5$cz)54}M)(ZyTKuCcU#ddmAcF3(I3O`qgE)4F*EhVbwl5wjNKnJn$ti)FH z3qfX79Oc`Pr?n4JqS$!k?{e9>e_BVGamAh*mc}F_V7*u|&XiOXdaMEZLETRqnSSd_ zXmtXe0p84dgYL2M4qMcnkm1IZ^7FB?S{R#jD4%fHD)jDV%tA_HE4GihV&wV!4q+Z! z^x!!PqKGk)gL3CS#RGAJbhFQFTPN7EZ7+emB|#yKtH&y9#4_)^`J&@qpd%T>gx4NMO<6f>k0;pL>}}2pomE9gx_Om9~U}Q z;=^{;@2o4ap`}GNrD%ck{dl1Y17CS9&ZA0obh0N~v8n1#aqK#K+cqIngecW#R5xUM zFcHJElrk{FMoP$S>~+EL>7_mDpoX3K-PE4ynN$T;M(>IOLQZ*PHG_N$I2N?y#B@%; zS(N&k0q<4)D()S*Q@KPlq9?glo(W3_IW?yDW{-!Q^y$;MFXQY<2HeRdd_)k$APXOo zOf+=5L2n4f?nuPQ8w~PmQzR;ttA-<1#N*Dui_je>S<4EK;+VsN`Nh({VF;pvWaF0b@drc3l`Jxd6dOgkppX zIw=DRqh39w&)4b6cr{07JNA4DCl&-S8RqIH3K_6&meI`Tm9+MVYOu@_G6)#kmJ$e1 z#Zz^^Xl>uyKKCM!(qiC(pw6lIw8_HxLRcG2)#OdY;-fb_fV#eQtxk8LplrtoM-TG) z>ei8B+P8AmSv^V!%!z=XXBHvx5`Lx?*9}HVC1`~B^N$BODlTfEbW>=iq`$jeOup)}O~(q3W0 z6AS?YloL2eY2bNMypb(=ispSxXm385EZh*$S-EP+1Uin#LM%bwyFeVrS-8pupnTd`=k*zH`~edQdhbkF69mW8ij^|$M1nM$+8Q|HUv*S z1-x`=1nlS?73$~eUcfEd>$eO)bTj-Ez|2}doU>>AVKC+U;b{4Y`xIGmo(_OlRG*Wi>fSkbp~8X3EKv zCwXF$Nw&_;>%pe$CT>)rN(dGWPCIX@fZd}y{9Qdb*~SVv;yy6HED&LNI`6>9N#$V( zLK^KA&Sxul??PL@X-5KVrWECros1zOVFDXpRs1~JXzlKo0=@LfLdpdLYm%!s^P-s;o+1H^xkLHyKv%7p z^E*ABd`33ACsAh`DpElJ;RXTpDDmPo%U8+`(t%^;Jse^4YsomXA4%sGBZb%`!=n2b z`JHo!0fc=;A#F%81oABdp2RaPuNZCXQpbMuI+b=3mu$V*g3Au_j^Rl31oy01m5dMu z_RFKhQ}@g67BnUYj?yuuBjeS?5)GpV1LvT3$RnrcTF|Yjs$xQ+s+R|DD>k74wI#?j zH7k;2fkzR%j~pm@7sv7PdFJP3%eoK(eZW}D(q+k{A@H5bqI?Sc%v z3cd}&@+9%GDj&K*ULF$1Er^_)!q8Z)BA;{L1l{NL0;1yP*e`COfG%jrdM*erOiS-yOXlc?Q<>l1(pKpV{Bhr)Y<^)H?GE97{+{hRM3kY=!-mg)A4s`QV#iOSU)i3FvBi8k4rSIqRgg)I6 zJR@Dwt0_~CZB9+Fos8br2HzjuYXq`kdO>;!PC6j4J`43IuoYaA$#P{%x^neR#ja|Z z(Q;G?CDHozRIz0#ye^Iiv~7#`j*&!?sC->~0Frv!k%+(>SLoJXu@9O%w|J zB7{N;A3MFUs4gyQ0^Zm^#wlD%h#+GV+|&9?aY%vX5RaCrLBet}K2zfSM{+$EcTYK^ z?d@$>2JEvDly$k4i+0E1P{(8^19CL!a+Dx0$fW{P+10yv)+b<8XFpY#y*Xvlpna!` zb|Q;*&;nDiEs&@Z$f&#Xl&O(zwvkLBm_8gLVzCK;%k+zh%#YB-dU5g&AecXypL&r4 z!o%gu9?9-4w57#Yfq)bsptr?@#o!<*G?WPjClJw!Ftc|K0rdo~K$MaIU@xf-oP9u@ z7J||2fg)B9b*Ri`9!PcbqjR65Gl0a}b5Y8E4IhErZEe;80*n$|M?#`j4*w?A zcry1*Gm`WPJqb14M-Xd*dJ85y3yQ&UfXyL%voj(S9NY&S>DBI6cL4`qQD$ltmsXNd zRoyt`!A%chU=tQeb9;*PzWfrZl^yRVh`X*lIx?3-vkr>urC21Z3RO-gSt(V<1tPdH zR>YO?Uu(~~gfMO7A0MVp@*i1~OPfoaj4iev2dWIuMuxW7k>?BOU zTL=)SQ@OT@V1R{vHhFUbnv1Oll)y%a$F6EL--NKhbN{Ow|1~P3ow#`R1T}q}=!Kqbe6X!9epd&S54_$stB_CAg>T{<= zNOiur(~P;HNkT`K!JJ6kfT@Y*o%ZwEJvnRkfKF|^&Uie|s+7{DT{wap=f~a{A*j}< zD*`Qs^`qNorOBK!5-|jDfT9=-K)f-+aAvot65iLWD^{5;l!UCy z3VKdgn#mtZbC79YGsSgkG^!J%bqICCJRI?*IFC9An$$!?!ULuwz?w7(OG41YY(-(G zhSKSVrNtX_W8B#0|HbiXZ8s{6~Yd#C1j3T&^4KFSGk2z9H zdV>{N)J4@2!%$AHF}6c$PB&lKds#iUolcRG9`Tb_PA=e;LJ}0`5Hh)WeX-9Wo=kCG z9G7o!UGIqq?B*y4ofL|eh=UbDQhwlcF+`d*`OJC;g6BO~t(g#hDWZa9#qT$-3Fw0O zwuecz8xzN`y=Q9=@K_)r2o~9qm0RL?UuCb{x*>==$z?>?e;}8R4~;2U{EO3N7~JfDE{PSU>@y2xAz4a8X0D>z(oZNGc~T>bmvk(Q-CFfm=N9ALV@qD96g@Ci-j`z zOd}b1AsH2u?>oZkDmodcNR{HE715~^;uEy}1$`2lVI_;Y34}3|YiSUQw4DU3_@nzb zGBjG3bi%^Y3@s&hw6j+=TFUIW6vi(W6$oOTn+z0`_d!4?@o@~~c}NgjSF%GVW=HG1 z8Ia7-&9^af_t5!{aVTrqQIK4ZVn!fCr=m#g0Ha}*RX6~F#YDxlR7genQ(}mX2@nD> zdQX}xB-vKTBxrJqkSCl~M{_M#pHen10)0~W=?gb*R+ivsy$!hRcE(y`rpwH7FG93aYuL}uq3YPe6K%~-|@Cp;0E zEki+tP>yCn1U`}?kC|i?ESaBse){9L=O4Y@dgGqFbJu@;djCMFPyR_MD!=^21!S2` zW{U^QSOFW4ZUX>70uld!0Fe9u0RR9YfkFLteZ2VkChgumc1h75>z1=4-p_XC+2y>K zdwY)!X3uln$Gv;A+O2iv>20)kI_&qnRP}>^k<`O^vHg+IOyV zEn87Fwkpou$4_>9X}aT0atUv3&9&||*VF(2HmvUT?+L13eV5-p==4#^yIUG&u}>{` zXQ8ibYO6yb?T1Y@+cxTG=9;Y(uHNre)9brp_Aw-O-)G*>8{F!;cWZ&6F6#PJdQlJ3}P)#u!RP{;dpZPNcHBn#se}CWh>%ZFTr2oJ7{MR+kG~eEz{pvM1 z3PC@@{HwDkU(6-;UEmIWkV2;>SH|?8wg!%~bahstu`~o9EN$^yK9U2neAkb&ijBLL zJ1}~^VWp>=>uIl-UG3Ga!xiO~p;VdoZ#mz4e_5r^dOG)>p!F3^t~h?@+@%bNB+yx}9NrJ^Rb&VprAd_0`WTw_r6ZnV;gW#DK}+ru3NMYtd_@cP4g*~Z zy-a-$UL)iDTvDjlbi@P!tvyo05oC9Iyir^<+J>wHLq9za#1A$74g0oC%HQMKpVsOC znxp{kaea893gYTwkyfZ*ye?)GbKX3B$OEdQwWl}K!$w$rC9vsJi?Q-qVX`C(jR+k( z3-__S$b{ubMonj%tZ6o-Y!h%yN$LvvEl$amJ%izgAAbw!A2t_6~Pmsf2w>el0tup2ev|^fT`sv34n!zjVzMwv|X)V7I_O0DYWJf)baTVX7O0zKx&N)exXox;O}Drkb;`%lQo zKV>fs1Rm%^$v!7Svp{>#tKRlyEW)~N*^Y)Blsr1`d8ZMh7GePkqr-mZ_oVJ_UAhB- zZK#NJ>Gi?Ew0fO^DvTpXT@Rk1eA={?h*^L+%}53}(4drlECEz-@UE=_m5RRskdES} zjflzNvDH^j74Wj=PK&nQghz{`thgkgi(%;pJyu7v5@#e?DHkXo zx~j%jHguOPqupc=IJoB|5uYwalEappDm1{XYtQopN>NNby~$c9%~%h2*E-MO`Y~wY3&1r+bldgH^K$3JuwPh{ix*n-ZN? zJYC)l=>`RBXe zXqV~pq)_66>K3vXb2U>Oc+4bf5k4u_!j$>1sNAnulv(bFVYmn%pgq^Q)URTk)@@Ik)4Xx#T=LC9erAzCKe75cx((UN5GFyaNz;86i%D(55Ll z8!|mbS<@-mpy6m2<$=mcSPu4CI@FcRBb>lME_)}dTD0$L%dEVy88#Yu0tTh-$zV|` zlh;|b9c< z8Z@7kg{&4b`DvDVk!aiAoM?KlRQ8RE5+2qyq+x5Uso8gAj);rkZPXO^*tbU$0~hLE z*K?%ER%5Czdb4mjqIFQ&@{-P79M;WUxl3_Y0Zi_hQ+n~4YPD?1BpF1VXHkkhNq7-X zq$k8y`7pgMTqGUV@(7S6OgSY`>NU*l5eig`#W&~k2CW=4Kqg|DbH;OzxT%FC12HM* zS`Ht*Ow_`iy@ZOgaJIYl__}AmDtt~CwYlS12%3BW(d46sOVZR~!RU`YM9-77=U&SK zzA<7lr$c`L%)lqH7Ru%I)7;s0=}TMSU4I%}v=Nh8!F3}T^gY6qUDu;LWyL8E7rH>s z@7=&Q{3-|^F&BbOgAF|;p^Q9+&Kv8ov9**7c_hZ$sQaZwOF8+ZUW`2JMxRK&NshGIvlr6~R#wCl!B@&_W4%^c zHwYG2B(SaXJS=^kJzP#n7)w1ifL*9-9@0Kf6&kvZ=Qz&mX|hw$ljn+(nMbj!EH>^D z+=DX96He8csK*ePof?P+ti5}INwcBD%E?OH?xk%YOXTEZa(Z3J#QVg8yiZR(2CkG! zQcF^sN(GHwoZhJDq+URS%%v`U;z&J!@D~Be%yw~I>^tGO+NX7V(OQ(yeMm)F%8H19 z&KZSo3+M4_V@SGLKy31+iP#h7-P7El!zX}riU;%1Zk-d|rL!I?2G&qpt%yTB*YYj}Mq2W%L& zzP}ekcXiWRYVxi$8R-b{u%D+J?U&I4$vE2~F%wn9oMVrz25yCBNX zeK9sdDlk`U!Z;maq}sT{szHti6OkKCrl_1pfPG2nao1Fh*G#~Z5LlGtL4T~YuIW}) zt^5V4FF!REZ55G0BzWLjk(9KdA2LzL4gTGFUki*M1<>XPDe^<&(Xx zRP(6#6%Ag&L!@dBVzQ)ys(oX0d@sr?der$y;KOF{7N2*834t$<15>IfQ*~r%QlhKU z3JbrlmlzAqFp6^S{H`dKx6JlR15f z8g+-GrNGBApRm}407tr~iOF7{wO?g6z})ERj~#v6cg7=kBySoT*jC)O; z1Jgq?r~|D?8K~KP6^h&CmM+mYeI58vEK%2OUPG{W!ypc3fF`fp7~YL+Ex;j$DLA$a z^AOV3k_n;ElXoMN8f?a3nF8~7qHe~^;TZy|5{7#An@i?py*SH~4R#<9lJ%*0uR8fc@4h{q6BcbP>P&_@PNuCT@64G!1$%jRSp^cQ)&R5n;gs%)EK zp!@8*q%P6<`0#8@NEaP#j`HVA>=w#nCFWnPyQW3ex+Ur$#VWe%17lqDLct}nl5%J( zDYtER?7ia1Cq8~%)+}%t01Vp9S$u645Onre{Vh3spH#%-P6<&wVRX>4qlhV0LOjC3 zL1N}^RSvDv20A#lckEhuE+OiB%hhv*>xPx(Yc7 z;+sSg-O`zZ^(}y`O4P`1`QOh&)a2ny$){ngyVskDVcFWXOI=%VOP+uy4ClKXR;AtIl;wLGr$x z;n9V!X54{Y=*2--$!|%oE}T|ua|RL9FJUX$gANY~pgrvKn4eFAbUgntgz?R6)E3)> zNbaob*HVet#E$o|6nx21Y%+G&x7zXGO)8!g&JP81NSDJ?iy=*#_GiF@eGlfNi%v=- zBSnPKQ>D=rbPC>k+qvo92+^IV6yEbV!Plc`shxmRI+@9-UA+`eM&5;Umx}+}6dPg9^aoB!BIMZ{W*K)B%80S+g&qZFc)@m$zm7p$< z$aI9*qz601LUIc*o*-G4Lh7U(U{dIW>9L|E4vA`c;j$!!?Sm)?a2h~T1jMY&0vv$z zL~!r3rl;LKh0ULJ)mgprgqOn=T#G5tO^Jq`byS(u2~jRtI#Q9x*ALC}p5gUt!(Rd_P8 zm<*mGQ2b3HOgwyJF-Cu0jiIqp>WcZjw6%ogu+A55y=GT?4 zwb>o0-+Wm77#cRfQ;Tz|W?ZSG1xz78#A2xuN0o8hAeKeMmDJtx+;diKZxzC)lZVCh zi`{b%PeT)k=d6~(h>;*h87j7@*VNICU4|{<4EtA*I_+Ko`c^VsB%r0Of-uwUd{ z+gm$q^;?);i5jm&Kchn4_H|9=J*M2g@rH?u@RtvqCHqRF90sbMRvO&M*9?2E7{v*z zDDBiaQ=d(JDtmRpeTk;(QntTu*ooPk&RROv#iHz|OQ1v|ds&9qCn%QGPm1|0&mk>G z2+m8@EUvgXA3aJ6UP@9?KBmU1fx@7X>5Yu5Zg>c$LP;m!VKnTJW+6uN1l3EV#<_#? zWZty}dqV|#M!kyxCt{|_4l-iSMg^8sy{WG*NVjOSsT|o$a_LD87Z?zT4-^Pf0*pw(?4OkG>L0VryAa;w_GqwY^NSmi>C zg2rB}w(unf^F)Oj5o^;_({J;{wViOYtrzQJC#Y%#T|rIHA1aJ-Bz{^ljXzu2)#bgM z@$g%nI^9*5O{!`wogQh_fJ!FY_Ij8fUi&UcG>=gD@yVK>0p>L1(3=EWQLiyoTq2Ob zT~IcvSKTZ;Sy5_K_ky<50Mz!{U zB?ll)LRMJ_1|^W{{*9-5Wy!OIRmZB#AUKnUt-ksf&a^hYkx8)mB8PZjfRtHwzZ042 z&QrWm!A~m@dP3t# z%sD!zP4VB^uCcCO;;@D(ug}LD49cjxq>~P0I0_^%;A+6eCz6#wqql{?LocQ=UUbtG z=#c{*rArt1BAdQ%J6<1A$Eq%ebgiRo)8y0W#$7hOD7oyismTs6xNs|6mIRO+QfTs-v97$|h=^eEjqR~G@(6;UxJ7Q5SeQ>d^y!sL zVs1*sR5XPGxtBSLPZ9ak+M)g>8(S_JU3lQ@IB4o=(B>8jgf(o~HIhydEt4Aa!=3hG zTJgY9?gYdw7a$a>E?#lK2Mt|LjBGDQpR6<*xuJyyiGA6>5`^<{qMj^mrIsHUEnCZr zfDtiWR6FULqZCL}qc(drD!$>8O3_U5*FmE`2eQC~EY;u8;O~?es$SBkg|(n;ap)y6 z-nbB*)FPJTkP8vPM1rBC(un3~Dq7%l!|*uWm}K_p`v$k3)D+u6E$h%+yI`Wzo~mG? zK!xa4o?#aYOk))B?X`wgkF%)4_9)*+1VJ%W>M^QOVzY!qezpsU-%*jNM>Xz2d9~)t zQMwCji9Iu+krhyy^2vh*FqWGnA(ykQz}O3U)d5M!O{Lh;#3;S8Y9T|@)b6SZ)pGo6 zoz7CTcs1bMENsN4@Ghy0`W^%fZ>;U(3F_|7vrU3X?Irw6kW375r? z#)90Y0y)O!n^H2bMy3e)}9mTSjjvzqDZwpW3D1AfB8R;yN#D@H`p~GW_mX_0+^qfX7gbLCTC_ zca8;103gNU=fb31h~ZhB`F}NrDusO{~fOItzy}b()b@rSSVu>fMP)Qt~j>9)a@I?GE-AY zxGa<#RVLU7W)?_k3gfK80%X#}Ry#(} z*#z$fI80AOQdBg^rXddpG4>rqvHHCDHYFtGIQGoV9wg70o5B<`V$yfCbLPHQn}^cXdXJnt*OS4tc}%S zAp|5bMXj3eJ4`FNs^4o;vDYev=pYbzFoF2y2+Fu&p6jqWWcOAz*%t>Vi9` z>lm^V+mjU$8c5>-^UMqr5Uf!yC%fJW4CEs*nt&R^iP-iK)viYDTQx#9eS8|p;@F3m z=VZ+$bGUTmj+oriLkx&FgVE{4*KVeOw#aK;hIMOs11$kkek;jGgRNJ6M!uHz$SUT+%Yq2o72(gKN|~dxQQ&3mvGv%TaNxcG%PzJ~1yThmz$^J(a@b) zsHw?{NU^H|Ehr#t>@aWxwuI<0aDcJ)XKIXY4UbPk1C^BZE6@kd>*;I?5n z?53-vs=#=MCP%JA3jI9ZeY6adxisH~sl`mNA@mQenV?;68>ZY6giX;089G@+7`m2@ zdIX}+0sZAwOt(-KWekfSbAs)C&o;!6>&K7OW-PD45u|cp&{~)kMC%BA*JEQ>aCHLu z%VssTal|KJ#ZXfu=#{1dP8qR)X)M|TnRxL{nlp_>2pYH8Swz_3)z7%Qo~dYH#XY{r zLM2eJTC^Gxy*{?7;)ZHA1&Aslkt{;+2$mdm)fK!j_y|i^ zh&FH3eqJQPx=&dwMhz=W8zA{qFCfFwU~VCI3LLzH7bwgg>^TN7dJ`Fi0}!pWc*{mj z;t3f;bfwAlU+@$-99+Kou=OZGy53u`*vE9%1k?Rq0+lP>Mx(2oQU;$uy43b1axI z3W`at93Rr*7xK@@7U)5wl2hYavD|5BptMI4>FE|1%g7ovE{rT)mZh z{rBH|9{_(zKmkaIQmRm(0ZJegC z2}wv&D$*#>iqHv2(g08@G%G?QLI6+*&=e{oN~uDjN(j&(w4zd_A^|`vNChAb3JF0# z2$X_>KnT!-KmkBB1p=)Apwg%`f;1^Yp-DiZl|UsZtqRg9O38 zBS5VpB><%&(uxHlp-L%8fY6Fi4FZJ<&?v-!(zGc5@CYgg`wEJvpFHpPKi~6j>&&&^ zkMoSpH@*GP{@mF2)`s12rTzW^eh^RwL<@mM zM7*_X&ej&@$aoHXqT!%oTdDVJpP8oBzdn8WksEqA@4rU(+k24W+vf4yL|$4MiIJZ9 zyz_m%`ZV+Im!nPgH+c`2!(OfFLpVZTmp0tilW?7hVX&Pm&nWSS@SY^Jtcbj^*@aCo?*4Swj(LohN z(@k~Yde;ESIk%U)vEYhJEOqu`p&FWv9mnXrr>tg2xGND+T? z97>p({xNktfK;qX!eASt+ep5Z@2o(ojxE5b8%23vGu}CHvsNA+_sS(GHFLHSPgVW5GVk-6~P`Q7>)NgnjmiE~$w8Vywb?5bKJAA%OIrAgAgN9zdT! z#_oXdFL^B(;H_7o=K6=;Fs@%VS+3+m%~~koR(?$~#Mhh4TnH-?u*=^hl$TmVhZ|!` zfU-xqEx`vi-af)Fr6IOs%j>3-q>DZPxE$?}NS!&M?w~Vz;u|Jz8O&uY6}-dW{ac`=JLXb_iKC>y3en z?0nP8Fyv=)j?^>4L7V#=9{N_%a;7vM2L@PL4i(JgjF7!i&Nsw1MjKXnah?gu52 zUKFNilr0v_uQ@4NO#{f5&DYVw)yom-2eJoX9I`%8$OrX3F#!(P<+l9-1`Gj~Ko~{5 zXU%*J`KskNt}K~AHh0Jq_EwV6fDed;1<(Y-;7@eE%5T3g)2{w|?DIT5;kMLX$6ucN zvY_vk;yuo<Dr8*BG&Jxw3C`&l%J9ClBi;CU?j#|pEj3{uCw8>yog zTAy$hs+A>+L5=)wLx4(ex!CtcQ(dTb^W%XPO1lwR8}ZIo1LrP18o3k!$(FL&2$A2ZN?BJ(w{CC}glCpc2&QYP5l@IQ z8MId{m67e)K5?=7%`_wug}RjD!&AIFqz;>TJ+=ycjbrK4A;?1VB%lxy?skU0FO6n# z-hMyb*Vb&ME6Z&q6C}i$oh3B-=2*Nyoha;nK?N{C5HZ76`Yr&#K-I-sbh1ZeEbPIV z9PsEoTbCY54G294Uhfv7v-}POqA~bB0Qdv%ywtJmKRcsiDwpp!>n&eLxg1hU0KS7Ba>NkATchfUT(kd0ea&|uOYqu8JB3_s3!(^_I0UtRIemG>njVIT zPK=#lxqold-lo?szVN~EOv#{lsA2X5*AZS?F+kH{l5}YCp8a!N@!c_~KI=E_55I{C6 z1L)lW<5_zL5G-mim?*Lf6h(;OrVS5#b;WVkn8xIP z45G{fq0kXhSP1Z73?^uA@vb8ep@>94#Az1?dF5c z5O9QH4S1*wUt1nzcef7l#(g@yc>&^=Yz08b&oTSv`{w)R`aB$gZYKTi4w1?cx%_kW zE!se~6Tl>x@q$$$H`;V=>`((aF}Hv#yZSg0z%TM$N4$)JuDhrL=3Uu6DZ|a zTJGBPz&rZ-k43KzNKxL^7&ZPy=os}qmI&Zo=%wW57PFApXI@i9+Mizg?@4?D4@T@- zH4SgpMnJt=L1I3nu>wrx=bl9t4`5jr(q0r=0-=lAYpm+3sw%ezafVPuVYg z=|FmZoD#4f0D*iVlZY^@6vb24uuS)mvY#Rl5cmr1$E;?`D7Lx6rZJ7e4DF~aS%9P8 z-!KQ?d>>+jX6dG2n={Dhjs7L3>GBIVgq(S(br}O}UF;whS)rjmVu=v3FTlM4Sa{xa zknwgR2|d(aR)OpK3yqd#J7nbuuwN0xI!f^I?&lS9sze)6FjY(5I1RW?p`ttqx@T*8 zkiq%@Kpar;vzdyB0bPnk`%UbAk&+NWT`bdWwE<7dH4pS71*v(CRf(&sH+bo*s2*fcNa=iDm_(H#XT*b8;{v!#m(YqpPWJAm zwNvFm0E}6+U8O^!ePALrY&P@u0O6D?KshjNVJx>Zbu^^&-*2dgg` z@%VP>=h@RZkY}rG|4?b?Q%?Aq2^eJ*zSEQgPhDlia{Gq;-7G04Iw-|ZNTr1V2*fXr z%3NilCz9C>am$2azDd zn_AmQqRT;Db?d{CgSnv!b!r6$u76_gxJ;he>IJc3S_hA$P@_Pp7_xv*I%LVq;<4OS30B8AjC(W zlOIZ-aUp#C$F{Jh)iGlQDe-B1B*P+aj0L+{2H4P{JWNG05-3O{PEkKng@rf3W+jfG zMzdp$ej&P75HQneVb|GRZWD$}Xi`U#!cUBylCYjI5;LD99fPy56b7awKZmWpS}heG z=<;+51P4Vp4-0RVa>On#(tH=}KA$_7mrC}(y(lqJJ z4nKToL+msH7&};?D*JH>wLrw=C6#PYNTYfvk4nH$x2BHXx&37ps;=&ohd}%5y5o=i9+-d{1h&&XB0f+ zM}yl+9Z-=F4Nxz>UNE3cLfrG#Tt4s}v>|b85uxAK3130OIV`j)905nT#bTwq&2&XZCVyl75SbN%sRPRfM}lhXwK4dKQqmC!c*>a| zgV7Rtz_`m*)9{D`dGGH1->mujy{9|p?Y_TyI^(8b{3iF0rOpOCAyt4k`oR2EhCC`E zSnUne<=+T3pgt7^5DNH?pnUDl_o@O$95>foa}&8lz(SW=24K+>mOmh=wL1zgxG&qp zI-6D!YypPAzUS0#(;UimJDLv3&q$mI3=9Z*i3_frM@}rXMYKZ@vjA1pAQGVk`i^|8 zjo^JcSY9*ljjY@CY*@yIXStK}C~^Soxc7SxHz6^s+4U zZ<~<58W<54Bmob(WU&H{?x9TIyTzNy1`R=lOBn8A&v7z}V)zE*G$>w{RPK5%l%1FH zhCr^Rz2?I<`(&G-VHm<6FotS^5%+2Z!OF#ks6l zs&(`cEnN5#m<&=DY#Q*|gbaAyP4amsSXcr^Y-p5E9y8-HvihhBviBC>qW%p2D!yk^l2BT|&vM`!g$Wq21BnVBEGNY$^`b(oH~+2ej& zRXSWBhs>mb?K==EwBrG*CQPBPbQ-CBqisbrg2n|a@Z+@)a#ZMUjM+1p>EDG=U-$S? zMHT+r-g`dj$A4XZe}A9ddHY^D{u6A{gSSTlzrt76iC+=Ua?D$ZBt8;0V$ZvV>&eJ^ z8!EgyvNS0{WY%1EZpiCpvjdo9dLkM%1$MegLJ_40(q1~Lbd=)0v}7!zP|UGj^Ag2s zv<2FO#-g{-7A_z+SOy_)N&^*?cWMu?al_)0OkYe`uX?m24Z5%H5X#$rf)5IFo0B|t znT)_oY>4ADC)9&m#2yB!((|wKmsB1&A9YAY64xXaC_8&m>cqtJx)AF zFKI6{Ei#I1P6~VAGgY#l?&vB^0y8RH7C|NUy~tH-)pJ^8h@$@J3F|4njR8b^B| zb_6OEyYugWJ`aQB1?q*+3!Z*46%GgMYR#|9vwZ5+Gnog%izyXzp`oL-#s#7Hb&SdO z&V2R&D3m1i0uC3cVF*W%Ar|4v<~5iaQXt@;Xvl`4ugPU2VvO=TBV8%IGe@1{bvP)V zGkO#uk5#3hp8^sO#W$CvGxlvDGhZDdy*w?7P)%R1F&jEuAf#Em9}Z#8)Zg2*^dKJb z`0K7TK=n;iM;wwnhX92#eSH$1y1NMVPOA|N zG{8Zf3AmF?ffYQ;6!c0t5@c}LJIhPLivKTg2d-52;P)d@I21ASqVoL~nan0~G@8Umt!XgneB@#M3B6>?3?Y1Oue+BO zOWeo=fNhYw5wXq+VRL~Wh~o-y`n<9jB&6s#&B9MQrNisO91JQGZa%E45=GeuM$GTs zq-KVPA2Z)C+g&v4Ji5Oi0*1Wz^R9a1a8TIW{PQ2Zf3?-N^YH}mg2OrpVhGq82ZI3c z5C|(P(4}#hzCARp6+OL2`-C@P2bM}NArllfDblzk@%cLtLA|A*`ph&gLXA9dE zeXW>nLc^)AgGk;Nc*}}*AfZd8q&|ZSAX*@stU?$O5MVJzGW9O8WJ{F#n6U-`V?c0h zCE+FH{jgjB;(OD8=R_?zTwara3mz&n(g|(AummT=$Qc$nwl@bgr3e>fX=VV94<(DR zLV-{-bUXu`owt&41n(O?}<1*W26}&&v!MzOG4AL6=KPZvz_~80&jH=n# ztFtW6i29jY9a;6a)-dZ8EPKl6NxS(nYky?b)o}(I`*immsAR>}*9`b$q2a_M;UM~y zsMWyQt#i`^#l>w-fGA@Vn@?HqO*G)}@N;a_Oru+CTV1N>sGYdiy6v+|7j32)=Z!&X zvcu>up+~sZ(~pmD=E2r2-g;rsUa*m>ulcF@;eeq?^bJ}tu33DkG> z%>uW$Vm*>2xm@kkFLl#*FV4g8R+)bjC~K9{eNuni3E&dM%jA&pncpz4W)hgF{8`}+ zw4Gd0du=K!VK2sJ>F@DNVErrBg*AmhnGw^ds;Ud?KP;iT$@uK*C~6Gyq(#J?;hrBZ z?MzExU`WGtMAC(L6Cq30mqm1!%}tAEJ+{DPF~FX(6BC}7#NuCNjzyzQUOeheC_EOn z;_j9SLWpJvfrP#@x0!Oyav$g96Hh#=Rn~trRa}tII zv)i?X?AR(OB%uTPRx*2cd^OFD?LC)*P?D9enD`-4`3T3{UnM0saC{C16{KW)C*rqL zOh*;d0*HpBEcDaRi4oanE^7@Jql1_P z%SRvEmKwEZyW#^bU(BL3y6>xA#xE?pzS-~^ZNVnf-s0jiLK)2&Wh#M8L>_a91Qq!eTPWm`m2y%QNY7pI7P&Qf zbT0S>5X+noa=vX3Lq7c?2bh|I;-|~8stNJam_{@f{n2Wegr?+a&2VX@W`+!u&rLk% z3wGUm5ubzaz5ow^1#L7B!96S$VV_S_p;~$MshmM))X80Jun;jsgGsW$j4&W9Flx6P zxvbd&$TYG0S6CqZMa_2vSw95J0|%H_dnJYg)CqrAtYS)tK{!G^>BLk)pL)fdvfYX> zoWAOxA2+bbRAs%9)$d4ApsAfDHm*i$<1Ai9U=UT33tDMQaB{ z6yIRvGG@bq5`BbIB3cRK1Hcsn!VS9M2Va-nIQBsV6(SO^FBvzak#>nUv|9dYHfTNx zlkv{&$8f!lJhzT*v`+Vp7$;_jHg%dKkaGWn7m zApR@k+Lm-*A_1Zq#+2-i1mhiO(4hTWWEC1KLHt@phCFQ0V|i(AZLf_dEp-660aTZn zE}k4m0CXut&x-DA6f+InFIa=j9YiotQ#lO=488}jQTOdo&yri{7PFd>XY zrU>8HG3ktuh|9>>W5}j8LI_2xlH{YDGK4BJ)g!2OGB+|p;7yQynvFFMNRQMnV4OMjIOFzyvraklr%_6@_&IK>FZHVs zg@FuMfULE}Yts{%@1BflgUsnvf!&OZuc~J8^;vtDf=nV~CXlJpj!PA;jskRMMu|`9 zATJW8sWar-(Z@o{4Cb^K3(o}cI-PL&`Zeqwl-w) zCxen7mfDwt+Ad+oZvS-V3jzi4nuPNp!y~ECZ32+`K3LN|%FAS~#MIh1o?G1ak$#}BMW;cqKoY9T5h>ZiB{Rde*UOoFCUvvS zacU7bRiUmRUIAo~T?T4o&RLCvybR5YI0DY=ATnhj=Ef>#6DNkTDtqo_5xC<4lcbmR zn3s9`W;v+tO9hi-f}F+OKf>DRUM6Aa~+y z`+4gKA|aIl2UG=bK}0G}>k!JuMyR_Oq8ylIiD>qU#R-=1;2esb&ndbyObDk1t{0#& zdUnLTb!hAm1Ue0kGad>KZ7~tyjWp=vlM2$z*FA22PytW;D58r0#Q*VkBvXY63OWRC E@bQNTNdN!< diff --git a/vcpkg.json b/vcpkg.json index eccafa372cac..c8b9831f013e 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -80,5 +80,12 @@ "libwebp" ] } - } + }, + "overrides": [ + { + "$note": "Remove this when the custom port for libmagic gets removed", + "name": "dirent", + "version": "1.23.2#3" + } + ] } From 71ef95350f19c0e3da675821288fa20d112b1837 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 13 Mar 2024 11:58:10 -0400 Subject: [PATCH 245/456] Add a test for concurrent array metadata writes. (#4806) Add a test for concurrent array metadata writes. With the fragment name generation, this test is tagged as `[!mayfail]`, because there's no means of sorting fragments which are written within the same millisecond. Once sub millisecond temporal disambiguation is implemented, this test should serve as a regression case. --- TYPE: NO_HISTORY DESC: Add a test for concurrent array metadata writes. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/unit-capi-metadata.cc | 56 +++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/test/src/unit-capi-metadata.cc b/test/src/unit-capi-metadata.cc index a69e4656616f..f3d178a8b3da 100644 --- a/test/src/unit-capi-metadata.cc +++ b/test/src/unit-capi-metadata.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -332,6 +332,60 @@ TEST_CASE_METHOD( tiledb_array_free(&array); } +TEST_CASE_METHOD( + CMetadataFx, + "C API: Metadata, sub-millisecond writes", + "[!mayfail][capi][metadata][sub-millisecond]") { + int32_t one = 1; + int32_t two = 2; + const void* v_r = nullptr; + tiledb_datatype_t v_type; + uint32_t v_num; + + // Run the test body 100 times + for (int i = 0; i < 100; i++) { + // Create and open array in write mode + create_default_array_1d(); + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx_, array_name_.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); + + // Write to disk twice + rc = tiledb_array_put_metadata(ctx_, array, "aaa", TILEDB_INT32, 1, &one); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_close(ctx_, array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_put_metadata(ctx_, array, "aaa", TILEDB_INT32, 1, &two); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_close(ctx_, array); + REQUIRE(rc == TILEDB_OK); + tiledb_array_free(&array); + + // Open the array in read mode + rc = tiledb_array_alloc(ctx_, array_name_.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + REQUIRE(rc == TILEDB_OK); + + // Read + rc = tiledb_array_get_metadata(ctx_, array, "aaa", &v_type, &v_num, &v_r); + CHECK(rc == TILEDB_OK); + CHECK(v_type == TILEDB_INT32); + CHECK(v_num == 1); + CHECK(*((const int32_t*)v_r) == 2); + + // Cleanup + rc = tiledb_array_close(ctx_, array); + REQUIRE(rc == TILEDB_OK); + tiledb_array_free(&array); + remove_dir(array_name_, ctx_, vfs_); + } +} + TEST_CASE_METHOD( CMetadataFx, "C API: Metadata, UTF-8", "[capi][metadata][utf-8]") { // Create default array From e6e18080c1edc4a26761be98d97db484a10a4a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Wed, 13 Mar 2024 17:58:40 +0100 Subject: [PATCH 246/456] CPack artifacts file name format compliance (#4811) This PR fixes these concerns with release artifact filenames: 1. Windows having `amd64` in name instead of `x86_64` 2. Package name missing full tag as name 3. Package name missing commit hash Number 1. is fixed by checking CMake variabe CMAKE_SYSTEM_PROCESSOR and if its set to `AMD64` the packaging logic uses `x86_64` instead. Number 2. and 3. are fixed by packaging process requiring `TILEDB_PACKAGE_VERSION` environment variable to be present when calling packaging commands (`make package` or `cpack ...` or similar) An example pipeline in my fork: https://github.com/dudoslav/TileDB/actions/runs/8266657611 An example release `t14` in my fork: https://github.com/dudoslav/TileDB/releases/tag/t14 (Please wait for them to finish) --- TYPE: NO_HISTORY DESC: Fix CPack packaging issues with file name format --- .github/workflows/append-release-cmake.yml | 2 +- .github/workflows/release.yml | 14 ++++++++++++-- cmake/inputs/CustomCPackOptions.cmake.in | 8 ++++++++ cmake/package.cmake | 11 +++++++++-- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/workflows/append-release-cmake.yml b/.github/workflows/append-release-cmake.yml index 91a4e26e1c29..3eb179b1239f 100644 --- a/.github/workflows/append-release-cmake.yml +++ b/.github/workflows/append-release-cmake.yml @@ -38,7 +38,7 @@ jobs: - name: Render template run: | - PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-([^-]+).(tar.gz|zip)$" + PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-([^-]+)-([^-]+).(tar.gz|zip)$" RELLIST="output/releases.csv" MODULE="output/DownloadPrebuiltTileDB.cmake" cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8584e0d18709..e41069f419fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,9 +17,18 @@ jobs: steps: - name: Checkout TileDB uses: actions/checkout@v3 + - name: Set variables + id: get-values + run: | + release_hash=$( echo ${{ github.sha }} | cut -c-7 - ) + ref=${{ github.head_ref || github.ref_name }} + echo "release_version=${ref##*/}-$release_hash" >> $GITHUB_OUTPUT + shell: bash - name: CMake Configure run: cmake -S . -B build -DTILEDB_CMAKE_IDE=ON - name: CPack Package Source + env: + TILEDB_PACKAGE_VERSION: ${{ steps.get-values.outputs.release_version }} run: cd build && cpack --config CPackSourceConfig.cmake - name: Upload Release Artifacts uses: actions/upload-artifact@v3 @@ -79,8 +88,7 @@ jobs: run: | release_hash=$( echo ${{ github.sha }} | cut -c-7 - ) ref=${{ github.head_ref || github.ref_name }} - echo "release_hash=$release_hash" >> $GITHUB_OUTPUT - echo "archive_name=tiledb-${{ matrix.platform }}-${ref##*/}-$release_hash" >> $GITHUB_OUTPUT + echo "release_version=${ref##*/}-$release_hash" >> $GITHUB_OUTPUT shell: bash - name: Install manylinux prerequisites if: matrix.manylinux @@ -106,6 +114,8 @@ jobs: ${{ matrix.cmake_args }} shell: bash - name: Build TileDB + env: + TILEDB_PACKAGE_VERSION: ${{ steps.get-values.outputs.release_version }} run: cmake --build build --config Release --target package - name: Upload release artifacts uses: actions/upload-artifact@v3 diff --git a/cmake/inputs/CustomCPackOptions.cmake.in b/cmake/inputs/CustomCPackOptions.cmake.in index 71d8fbea0686..65187e155323 100644 --- a/cmake/inputs/CustomCPackOptions.cmake.in +++ b/cmake/inputs/CustomCPackOptions.cmake.in @@ -5,3 +5,11 @@ set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY OFF) # Enable HASH set(CPACK_PACKAGE_CHECKSUM "SHA256") + +set(PACKAGE_VERSION $ENV{TILEDB_PACKAGE_VERSION}) +if(NOT PACKAGE_VERSION) + message(FATAL_ERROR "In order to package TileDB specify environment variable TILEDB_PACKAGE_VERSION") +endif() + +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CPACK_SOURCE_PACKAGE_FILE_NAME}-${PACKAGE_VERSION}") +set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}-${PACKAGE_VERSION}") diff --git a/cmake/package.cmake b/cmake/package.cmake index 08bfae9049c3..19ed3f47ef40 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -29,6 +29,13 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") else() set(CPACK_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) endif() +elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CPACK_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") + set(CPACK_SYSTEM_PROCESSOR "x86_64") + else() + set(CPACK_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) + endif() else() set(CPACK_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) set(CPACK_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}) @@ -36,10 +43,10 @@ endif() set(CPACK_SOURCE_IGNORE_FILES ".*\.git;.*build.*/.*") -set(CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-source-${TILEDB_VERSION}") +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-source") string(TOLOWER ${CPACK_SOURCE_PACKAGE_FILE_NAME} CPACK_SOURCE_PACKAGE_FILE_NAME) -set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${CPACK_SYSTEM_NAME}-${CPACK_SYSTEM_PROCESSOR}${CPACK_NOAVX2}-${TILEDB_VERSION}") +set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${CPACK_SYSTEM_NAME}-${CPACK_SYSTEM_PROCESSOR}${CPACK_NOAVX2}") string(TOLOWER ${CPACK_PACKAGE_FILE_NAME} CPACK_PACKAGE_FILE_NAME) include(CPack) From 9aafbb4581de455241370bf29d4651c2fc433360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Fri, 15 Mar 2024 15:48:14 +0100 Subject: [PATCH 247/456] Improve pattern recognizing release artifacts (#4817) After changing release artifact names the `releases.csv` file generator logic broke since the REGEX matcher expected different input. This PR changes the matcher REGEX to match everything after all required fields: ![1710495096_grim](https://github.com/TileDB-Inc/TileDB/assets/5787996/397d7947-e4a0-4bb0-9748-e36cbb0f3d5a) example release: https://github.com/dudoslav/TileDB/releases/tag/t15.0.1-rc1 --- TYPE: NO_HISTORY DESC: Change regex matching release artifacts --- .github/workflows/append-release-cmake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/append-release-cmake.yml b/.github/workflows/append-release-cmake.yml index 3eb179b1239f..93da2eb51d67 100644 --- a/.github/workflows/append-release-cmake.yml +++ b/.github/workflows/append-release-cmake.yml @@ -38,7 +38,7 @@ jobs: - name: Render template run: | - PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-([^-]+)-([^-]+).(tar.gz|zip)$" + PATTERN="tiledb-([^-]+)-([^-]+)(-noavx2)?-(.+)\.(tar\.gz|zip)$" RELLIST="output/releases.csv" MODULE="output/DownloadPrebuiltTileDB.cmake" cp cmake/inputs/DownloadPrebuiltTileDB.cmake $MODULE From 33773d3fba7691628bf0b33cfa3c832630e784c5 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:14:43 +0200 Subject: [PATCH 248/456] Throw error when an upload fails due to bad state. (#4815) This PR will throw an error if the state when uploading files to s3 is not ok. More details can be found in the shortcut story. The gist is that we are returning an OK status even if the upload is unsuccessful. Below you can see my reproduction. Thanks to @gspowley. 1. I created a valid VCF dataset in s3: ```dist/bin/tiledbvcf create --uri s3://tiledb-dstara/my_dataset``` 2. After ingesting some vcf files into a valid dataset we can see 21 files ``` dimitrisstaratzis@iMac TileDB-VCF % aws s3 ls s3://tiledb-dstara/my_dataset/data/__fragments/__1710345309063_1710345309063_59cf338875644cd28114f95923a018e3_21/ 2024-03-13 17:55:22 12164 __fragment_metadata.tdb 2024-03-13 17:55:14 177 a0.tdb 2024-03-13 17:55:20 177 a1.tdb 2024-03-13 17:55:13 143 a2.tdb 2024-03-13 17:55:13 179 a3.tdb 2024-03-13 17:55:14 161 a3_var.tdb 2024-03-13 17:55:14 179 a4.tdb 2024-03-13 17:55:15 141 a4_var.tdb 2024-03-13 17:55:21 179 a5.tdb 2024-03-13 17:55:13 166 a5_var.tdb 2024-03-13 17:55:14 182 a6.tdb 2024-03-13 17:55:15 196 a6_var.tdb 2024-03-13 17:55:13 179 a7.tdb 2024-03-13 17:55:14 191 a7_var.tdb 2024-03-13 17:55:14 179 a8.tdb 2024-03-13 17:55:15 147 a8_var.tdb 2024-03-13 17:55:13 8 d0.tdb 2024-03-13 17:55:14 45 d0_var.tdb 2024-03-13 17:55:14 191 d1.tdb 2024-03-13 17:55:13 8 d2.tdb 2024-03-13 17:55:13 102 d2_var.tdb ``` 3. I am now using an env variable to inject a random error in the state. (See in shortcut story) ``` dimitrisstaratzis@iMac TileDB-VCF % export TILEDB_S3_INJECT_ERROR=1 dimitrisstaratzis@iMac TileDB-VCF % dist/bin/tiledbvcf store --uri s3://tiledb-dstara/my_dataset small.vcf.gz [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 ``` and I get 16 files instead of the correct 21. So the user does not know something is wrong: ``` dimitrisstaratzis@iMac TileDB-VCF % aws s3 ls s3://tiledb-dstara/my_dataset/data/__fragments/__1710347463937_1710347463937_97d481ba63664dcca766d91b20e029b2_21/ 2024-03-13 18:31:24 12164 __fragment_metadata.tdb 2024-03-13 18:31:15 143 a2.tdb 2024-03-13 18:31:15 179 a3.tdb 2024-03-13 18:31:16 161 a3_var.tdb 2024-03-13 18:31:15 179 a4.tdb 2024-03-13 18:31:16 141 a4_var.tdb 2024-03-13 18:31:15 179 a5.tdb 2024-03-13 18:31:21 182 a6.tdb 2024-03-13 18:31:15 179 a7.tdb 2024-03-13 18:31:15 191 a7_var.tdb 2024-03-13 18:31:14 179 a8.tdb 2024-03-13 18:31:15 8 d0.tdb 2024-03-13 18:31:16 45 d0_var.tdb 2024-03-13 18:31:15 191 d1.tdb 2024-03-13 18:31:15 8 d2.tdb 2024-03-13 18:31:15 102 d2_var.tdb ``` 4. With my fix an error is thrown so the user is aware and can retry ``` dimitrisstaratzis@iMac TileDB-VCF % dist/bin/tiledbvcf store --uri s3://tiledb-dstara/my_dataset small.vcf.gz [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 0 [S3::flush_object] error = 1 [S3::flush_object] Aborting multipart upload [S3::flush_object] error = 0 [2024-03-13 19:10:54.265] [tiledb-vcf] [Process: 96652] [Thread: 76198470] [critical] Exception: TileDB internal: [GlobalOrderWriter::dowork] ([TileDB::S3] Error: Failed to flush S3 object s3://tiledb-dstara/my_dataset_error_fix/variant_stats/__fragments/__1710349852046_1710349852046_0ccf455080ac435a9da6e10bd5b48f20_21/d1.tdbSuccess) ``` [sc-42315] --- TYPE: BUG DESC: Throw error when an upload fails due to bad state. --- test/src/unit-s3.cc | 2 +- tiledb/sm/filesystem/s3.cc | 29 ++++++++++++++++++----------- tiledb/sm/filesystem/s3.h | 4 +++- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index aa2c68a593cd..dec8e72dbe52 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -124,7 +124,7 @@ TEST_CASE_METHOD(S3Fx, "Test S3 multiupload abort path", "[s3]") { CHECK(!exists); // Flush the file - CHECK(s3_.flush_object(URI(largefile)).ok()); + CHECK(!s3_.flush_object(URI(largefile)).ok()); // After flushing, the file does not exist CHECK(s3_.is_object(URI(largefile), &exists).ok()); diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index d1d13b5e72d8..708716f8891f 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -634,12 +634,12 @@ Status S3::disconnect() { make_multipart_abort_request(*state); auto outcome = client_->AbortMultipartUpload(abort_request); if (!outcome.IsSuccess()) { - const Status st = LOG_STATUS(Status_S3Error( + ret_st = LOG_STATUS(Status_S3Error( std::string("Failed to disconnect and flush S3 objects. ") + outcome_error_message(outcome))); - if (!st.ok()) { - ret_st = st; - } + } else { + ret_st = LOG_STATUS(Status_S3Error( + std::string("Failed to disconnect and flush S3 objects. "))); } } return Status::Ok(); @@ -714,7 +714,7 @@ Status S3::flush_object(const URI& uri) { throw_if_not_ok(wait_for_object_to_propagate(move(bucket), move(key))); - return finish_flush_object(std::move(outcome), uri, buff); + return finish_flush_object(std::move(outcome), uri, buff, false); } else { Aws::S3::Model::AbortMultipartUploadRequest abort_request = make_multipart_abort_request(*state); @@ -723,7 +723,7 @@ Status S3::flush_object(const URI& uri) { state_lck.unlock(); - return finish_flush_object(std::move(outcome), uri, buff); + return finish_flush_object(std::move(outcome), uri, buff, true); } } @@ -803,6 +803,9 @@ void S3::finalize_and_flush_object(const URI& uri) { throw S3Exception{ std::string("Failed to flush S3 object ") + uri.c_str() + outcome_error_message(outcome)}; + } else { + throw S3Exception{ + std::string("Failed to flush S3 object ") + uri.c_str()}; } } @@ -1689,7 +1692,8 @@ template Status S3::finish_flush_object( const Aws::Utils::Outcome& outcome, const URI& uri, - Buffer* const buff) { + Buffer* const buff, + bool is_abort) { Aws::Http::URI aws_uri = uri.c_str(); UniqueWriteLock unique_wl(&multipart_upload_rwlock_); @@ -1701,10 +1705,13 @@ Status S3::finish_flush_object( file_buffers_lck.unlock(); tdb_delete(buff); - if (!outcome.IsSuccess()) { - return LOG_STATUS(Status_S3Error( - std::string("Failed to flush S3 object ") + uri.c_str() + - outcome_error_message(outcome))); + if (!outcome.IsSuccess() || is_abort) { + std::string error_message = + std::string("Failed to flush S3 object ") + uri.c_str(); + if (!is_abort) { + error_message += outcome_error_message(outcome); + } + return LOG_STATUS(Status_S3Error(error_message)); } return Status::Ok(); diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index ebbe827a59a4..8c074479aee2 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -1509,13 +1509,15 @@ class S3 : FilesystemBase { * @param outcome The returned outcome from the complete or abort request. * @param uri The URI of the S3 file to be written to. * @param buff The file buffer associated with 'uri'. + * @param is_abort Should be true only when this is an abort request. * @return Status */ template Status finish_flush_object( const Aws::Utils::Outcome& outcome, const URI& uri, - Buffer* const buff); + Buffer* const buff, + bool is_abort); /** * Writes the input buffer to a file by issuing one PutObject From f063a690edbe05e66495223c2ca93e79dad2f514 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Wed, 20 Mar 2024 05:04:51 -0400 Subject: [PATCH 249/456] Instrument WriterBase with tdb::pmr containers for memory tracking. (#4805) This adds memory tracking to containers used in the writers. The tracking added in `IndexedList` updates `WriterTileTupleVector` to use memory tracking in all writers. I also did some digging to look for other containers that were used heavily in the writers and added tracking to some members in `GlobalWriteState` --- TYPE: NO_HISTORY DESC: Instrument WriterBase with tdb::pmr containers for memory tracking. --- test/src/unit-result-tile.cc | 4 +-- tiledb/common/indexed_list.h | 27 +++++++++++++---- tiledb/common/memory_tracker.cc | 12 +++++--- tiledb/common/memory_tracker.h | 8 +++-- .../common/test/unit_memory_tracker_types.cc | 2 +- tiledb/sm/query/legacy/reader.cc | 8 ++--- tiledb/sm/query/readers/reader_base.cc | 8 +++++ tiledb/sm/query/readers/result_tile.h | 6 ++-- tiledb/sm/query/test/unit_query_condition.cc | 10 +++---- .../sm/query/writers/global_order_writer.cc | 30 ++++++++++++------- tiledb/sm/query/writers/global_order_writer.h | 20 +++++++++---- tiledb/sm/query/writers/ordered_writer.cc | 12 ++++---- tiledb/sm/query/writers/unordered_writer.cc | 10 ++++--- tiledb/sm/query/writers/unordered_writer.h | 6 ++-- tiledb/sm/query/writers/writer_base.cc | 29 +++++++++++++----- tiledb/sm/query/writers/writer_base.h | 10 +++---- tiledb/sm/tile/tile.cc | 2 +- tiledb/type/apply_with_type.h | 13 -------- 18 files changed, 134 insertions(+), 83 deletions(-) diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index 046cb1c8e7af..dd7d01cae3d3 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -283,7 +283,7 @@ TEST_CASE_METHOD( std::iota(range_indexes.begin(), range_indexes.end(), 0); auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_count(num_cells, 1, resource); ResultTile::compute_results_count_sparse_string( &rt, @@ -426,7 +426,7 @@ TEST_CASE_METHOD( std::iota(range_indexes.begin(), range_indexes.end(), 0); auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_count(num_cells, 1, resource); ResultTile::compute_results_count_sparse_string( &rt, diff --git a/tiledb/common/indexed_list.h b/tiledb/common/indexed_list.h index e3e87302264f..43be546d9768 100644 --- a/tiledb/common/indexed_list.h +++ b/tiledb/common/indexed_list.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,9 +33,15 @@ #ifndef TILEDB_INDEXED_LIST_H #define TILEDB_INDEXED_LIST_H +#include "pmr.h" + #include #include +namespace tiledb::sm { +class MemoryTracker; +} + namespace tiledb::common { /** @@ -51,9 +57,15 @@ class IndexedList { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Default constructor. */ - IndexedList() { - } + /** Deleted Default constructor. */ + IndexedList() = delete; + + /** + * Constructor. + * + * @param memory_tracker The memory tracker for the underlying containers. + */ + explicit IndexedList(shared_ptr memory_tracker); DISABLE_COPY_AND_COPY_ASSIGN(IndexedList); DISABLE_MOVE_AND_MOVE_ASSIGN(IndexedList); @@ -124,7 +136,7 @@ class IndexedList { vec_.reserve(num); for (uint64_t n = 0; n < num; n++) { - emplace_back(); + emplace_back(memory_tracker_); } } @@ -177,8 +189,11 @@ class IndexedList { } private: + /** The memory tracker for the underlying list. */ + shared_ptr memory_tracker_; + /** List that contains all the elements. */ - std::list list_; + tdb::pmr::list list_; /** Vector that contains a pointer to the elements allowing indexed access. */ std::vector vec_; diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index c7d9071bdc20..aee787332e1f 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -64,8 +64,6 @@ std::string memory_type_to_str(MemoryType type) { return "GenericTileIO"; case MemoryType::RTREE: return "RTree"; - case MemoryType::TILE_BITMAP: - return "TileBitmap"; case MemoryType::TILE_DATA: return "TileData"; case MemoryType::TILE_HILBERT_VALUES: @@ -90,10 +88,16 @@ std::string memory_type_to_str(MemoryType type) { return "Domains"; case MemoryType::TILE_SUMS: return "TileSums"; - case MemoryType::TILE_WRITER_DATA: - return "TileWriterData"; + case MemoryType::WRITER_TILE_DATA: + return "WriterTileData"; case MemoryType::METADATA: return "Metadata"; + case MemoryType::WRITER_DATA: + return "WriterData"; + case MemoryType::RESULT_TILE_BITMAP: + return "ResultTileBitmap"; + case MemoryType::RESULT_TILE: + return "ResultTile"; } auto val = std::to_string(static_cast(type)); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 4bda450af2c1..a7271bab86c5 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -99,7 +99,7 @@ namespace tiledb::sm { -//** The type of memory to track. */ +/** The type of memory to track. */ enum class MemoryType { ENUMERATION, ENUMERATION_PATHS, @@ -108,7 +108,6 @@ enum class MemoryType { FILTERED_DATA_BLOCK, GENERIC_TILE_IO, RTREE, - TILE_BITMAP, TILE_DATA, TILE_HILBERT_VALUES, TILE_OFFSETS, @@ -122,7 +121,10 @@ enum class MemoryType { METADATA, DOMAINS, TILE_SUMS, - TILE_WRITER_DATA + WRITER_TILE_DATA, + WRITER_DATA, + RESULT_TILE, + RESULT_TILE_BITMAP }; /** diff --git a/tiledb/common/test/unit_memory_tracker_types.cc b/tiledb/common/test/unit_memory_tracker_types.cc index e43a85d1952a..b3e9106a0914 100644 --- a/tiledb/common/test/unit_memory_tracker_types.cc +++ b/tiledb/common/test/unit_memory_tracker_types.cc @@ -38,7 +38,7 @@ using namespace tiledb::common; TEST_CASE("memory_type_to_str") { - auto max = static_cast(tiledb::sm::MemoryType::TILE_WRITER_DATA); + auto max = static_cast(tiledb::sm::MemoryType::RESULT_TILE_BITMAP); size_t failures = 0; for (int8_t i = 0; i < 127; i++) { auto val = static_cast(i); diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 0cf5015261a3..1a655cb51ef3 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -454,7 +454,7 @@ Status Reader::compute_range_result_coords( } } else { // Sparse auto resource = - query_memory_tracker_->get_resource(MemoryType::TILE_BITMAP); + query_memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(coords_num, 1, resource); // Compute result and overwritten bitmap per dimension @@ -1726,7 +1726,7 @@ Status Reader::dense_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - IndexedList sparse_result_tiles; + IndexedList sparse_result_tiles(query_memory_tracker_); RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); // Compute result cell slabs. @@ -1788,7 +1788,7 @@ Status Reader::get_all_result_coords( if (fragment_metadata_[tile->frag_idx()]->has_timestamps() && partial_overlap) { auto resource = - query_memory_tracker_->get_resource(MemoryType::TILE_BITMAP); + query_memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(coords_num, 1, resource); RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( *(frag_meta->array_schema().get()), *tile, result_bitmap)); @@ -1984,7 +1984,7 @@ Status Reader::sparse_read() { // `sparse_result_tiles` will hold all the relevant result tiles of // sparse fragments std::vector result_coords; - IndexedList sparse_result_tiles; + IndexedList sparse_result_tiles(query_memory_tracker_); RETURN_NOT_OK(compute_result_coords(sparse_result_tiles, result_coords)); std::vector result_tiles; diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index d5948d81170d..e380de81c308 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -31,6 +31,7 @@ */ #include "tiledb/sm/query/readers/reader_base.h" +#include "tiledb/common/indexed_list.h" #include "tiledb/common/logger.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" @@ -1369,3 +1370,10 @@ template void ReaderBase::validate_attribute_order( std::vector&); } // namespace tiledb::sm + +template <> +IndexedList::IndexedList( + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , list_(memory_tracker->get_resource(sm::MemoryType::RESULT_TILE)) { +} diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index 189631211192..1f4e004ef408 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -828,7 +828,7 @@ class ResultTileWithBitmap : public ResultTile { */ ResultTileWithBitmap(shared_ptr memory_tracker) : ResultTile(memory_tracker) - , bitmap_(memory_tracker_->get_resource(MemoryType::TILE_BITMAP)) { + , bitmap_(memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP)) { } ResultTileWithBitmap( @@ -837,7 +837,7 @@ class ResultTileWithBitmap : public ResultTile { const FragmentMetadata& frag_md, shared_ptr memory_tracker) : ResultTile(frag_idx, tile_idx, frag_md, memory_tracker) - , bitmap_(memory_tracker_->get_resource(MemoryType::TILE_BITMAP)) + , bitmap_(memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP)) , result_num_(cell_num_) { } @@ -989,7 +989,7 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { , used_(false) { if (!dups || include_delete_meta) { post_dedup_bitmap_.emplace( - this->memory_tracker_->get_resource(MemoryType::TILE_BITMAP)); + this->memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP)); } } diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 60882000f871..51d98666f542 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -2725,7 +2725,7 @@ void test_apply_cells_sparse( // Apply the query condition. auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, *result_tile, result_bitmap) @@ -2752,7 +2752,7 @@ void test_apply_cells_sparse( // Apply the query condition. auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap_eq_null(cells, 1, resource); REQUIRE(query_condition_eq_null .apply_sparse( @@ -2824,7 +2824,7 @@ void test_apply_cells_sparse( // Apply the query condition. auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, *result_tile, result_bitmap) @@ -3298,7 +3298,7 @@ void validate_qc_apply_sparse( ResultTile& result_tile, bool negated = false) { auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector sparse_result_bitmap(cells, 1, resource); REQUIRE(tp.qc_ .apply_sparse( @@ -5082,7 +5082,7 @@ TEST_CASE( // Apply the query condition. auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::TILE_BITMAP); + MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); REQUIRE(query_condition .apply_sparse(*array_schema, result_tile, result_bitmap) diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 20d20785ad98..86bb48ba5031 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -112,6 +112,13 @@ GlobalOrderWriter::GlobalOrderWriter( GlobalOrderWriter::~GlobalOrderWriter() { } +GlobalOrderWriter::GlobalWriteState::GlobalWriteState( + shared_ptr memory_tracker) + : last_tiles_(memory_tracker->get_resource(MemoryType::WRITER_TILE_DATA)) + , last_var_offsets_(memory_tracker->get_resource(MemoryType::WRITER_DATA)) + , cells_written_(memory_tracker->get_resource(MemoryType::WRITER_DATA)) { +} + /* ****************************** */ /* API */ /* ****************************** */ @@ -174,7 +181,7 @@ Status GlobalOrderWriter::alloc_global_write_state() { return logger_->status( Status_WriterError("Cannot initialize global write state; State not " "properly finalized")); - global_write_state_.reset(new GlobalWriteState); + global_write_state_.reset(tdb_new(GlobalWriteState, query_memory_tracker_)); // Alloc FragmentMetadata object global_write_state_->frag_meta_ = this->create_fragment_metadata(); @@ -201,7 +208,7 @@ Status GlobalOrderWriter::init_global_write_state() { auto last_tiles_it = global_write_state_->last_tiles_.emplace( std::piecewise_construct, std::forward_as_tuple(name), - std::forward_as_tuple()); + std::forward_as_tuple(query_memory_tracker_)); last_tiles_it.first->second.emplace_back( array_schema_, cell_num_per_tile, @@ -749,7 +756,8 @@ Status GlobalOrderWriter::global_write() { RETURN_CANCEL_OR_ERROR(compute_coord_dups(&coord_dups)); } - std::unordered_map tiles; + tdb::pmr::unordered_map tiles( + query_memory_tracker_->get_resource(MemoryType::WRITER_TILE_DATA)); RETURN_CANCEL_OR_ERROR(prepare_full_tiles(coord_dups, &tiles)); // Find number of tiles and gather stats @@ -859,15 +867,15 @@ void GlobalOrderWriter::nuke_global_write_state() { Status GlobalOrderWriter::prepare_full_tiles( const std::set& coord_dups, - std::unordered_map* tiles) const { + tdb::pmr::unordered_map* tiles) const { auto timer_se = stats_->start_timer("prepare_tiles"); // Initialize attribute and coordinate tiles for (const auto& it : buffers_) { - (*tiles).emplace( + tiles->emplace( std::piecewise_construct, std::forward_as_tuple(it.first), - std::forward_as_tuple()); + std::forward_as_tuple(query_memory_tracker_)); } auto num = buffers_.size(); @@ -877,7 +885,7 @@ Status GlobalOrderWriter::prepare_full_tiles( std::advance(buff_it, i); const auto& name = buff_it->first; RETURN_CANCEL_OR_ERROR( - prepare_full_tiles(name, coord_dups, &(*tiles)[name])); + prepare_full_tiles(name, coord_dups, &tiles->at(name))); return Status::Ok(); }); @@ -919,7 +927,7 @@ Status GlobalOrderWriter::prepare_full_tiles_fixed( } // First fill the last tile - auto& last_tile = global_write_state_->last_tiles_[name][0]; + auto& last_tile = global_write_state_->last_tiles_.at(name)[0]; uint64_t cell_idx = 0; uint64_t last_tile_cell_idx = global_write_state_->cells_written_[name] % cell_num_per_tile; @@ -1100,7 +1108,7 @@ Status GlobalOrderWriter::prepare_full_tiles_var( return Status::Ok(); // First fill the last tile - auto& last_tile = global_write_state_->last_tiles_[name][0]; + auto& last_tile = global_write_state_->last_tiles_.at(name)[0]; auto& last_var_offset = global_write_state_->last_var_offsets_[name]; uint64_t cell_idx = 0; uint64_t last_tile_cell_idx = @@ -1378,7 +1386,7 @@ Status GlobalOrderWriter::prepare_full_tiles_var( uint64_t GlobalOrderWriter::num_tiles_to_write( uint64_t start, uint64_t tile_num, - std::unordered_map& tiles) { + tdb::pmr::unordered_map& tiles) { // Cache variables to prevent map lookups. const auto buf_names = buffer_names(); std::vector var_size; @@ -1390,7 +1398,7 @@ uint64_t GlobalOrderWriter::num_tiles_to_write( for (auto& name : buf_names) { var_size.emplace_back(array_schema_.var_size(name)); nullable.emplace_back(array_schema_.is_nullable(name)); - writer_tile_vectors.emplace_back(&tiles[name]); + writer_tile_vectors.emplace_back(&tiles.at(name)); } // Make sure we don't write more than the desired fragment size. diff --git a/tiledb/sm/query/writers/global_order_writer.h b/tiledb/sm/query/writers/global_order_writer.h index bf48eafb2320..c15b81f67c8d 100644 --- a/tiledb/sm/query/writers/global_order_writer.h +++ b/tiledb/sm/query/writers/global_order_writer.h @@ -58,6 +58,16 @@ class GlobalOrderWriter : public WriterBase { * by successive query submissions until the query is finalized. */ struct GlobalWriteState { + /** Deleted Default Constructor. */ + GlobalWriteState() = delete; + + /** + * Constructor. + * + * @param memory_tracker The memory tracker for the underlying containers. + */ + explicit GlobalWriteState(shared_ptr memory_tracker); + /** * Stores the last tile of each attribute/dimension for each write * operation. The key is the attribute/dimension name. For fixed-sized @@ -66,7 +76,7 @@ class GlobalOrderWriter : public WriterBase { * second tile is the values tile. In both cases, the third tile stores a * validity tile for nullable attributes. */ - std::unordered_map last_tiles_; + tdb::pmr::unordered_map last_tiles_; /** * Stores the last offset into the var size tile buffer for var size @@ -75,13 +85,13 @@ class GlobalOrderWriter : public WriterBase { * Note: Once tiles are created with the correct size from the beginning, * this variable can go awaty. */ - std::unordered_map last_var_offsets_; + tdb::pmr::unordered_map last_var_offsets_; /** * Stores the number of cells written for each attribute/dimension across * the write operations. */ - std::unordered_map cells_written_; + tdb::pmr::unordered_map cells_written_; /** The fragment metadata that the writer will focus on. */ shared_ptr frag_meta_; @@ -301,7 +311,7 @@ class GlobalOrderWriter : public WriterBase { */ Status prepare_full_tiles( const std::set& coord_dups, - std::unordered_map* tiles) const; + tdb::pmr::unordered_map* tiles) const; /** * Applicable only to write in global order. It prepares only full @@ -373,7 +383,7 @@ class GlobalOrderWriter : public WriterBase { uint64_t num_tiles_to_write( uint64_t start, uint64_t tile_num, - std::unordered_map& tiles); + tdb::pmr::unordered_map& tiles); /** * Close the current fragment and start a new one. The closed fragment will diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index cc951acc7ee3..1cce7f7c9592 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -203,7 +203,7 @@ Status OrderedWriter::ordered_write() { tiles.emplace( std::piecewise_construct, std::forward_as_tuple(buff.first), - std::forward_as_tuple()); + std::forward_as_tuple(query_memory_tracker_)); } if (attr_num > tile_num) { // Parallelize over attributes @@ -211,14 +211,14 @@ Status OrderedWriter::ordered_write() { auto buff_it = buffers_.begin(); std::advance(buff_it, i); const auto& attr = buff_it->first; - auto& attr_tile_batches = tiles[attr]; + auto& attr_tile_batches = tiles.at(attr); return prepare_filter_and_write_tiles( attr, attr_tile_batches, frag_meta, &dense_tiler, 1); })); } else { // Parallelize over tiles for (const auto& buff : buffers_) { const auto& attr = buff.first; - auto& attr_tile_batches = tiles[attr]; + auto& attr_tile_batches = tiles.at(attr); RETURN_NOT_OK(prepare_filter_and_write_tiles( attr, attr_tile_batches, frag_meta, &dense_tiler, thread_num)); } @@ -233,7 +233,7 @@ Status OrderedWriter::ordered_write() { const auto var_size = array_schema_.var_size(attr); if (has_min_max_metadata(attr, var_size) && array_schema_.var_size(attr)) { - auto& attr_tile_batches = tiles[attr]; + auto& attr_tile_batches = tiles.at(attr); frag_meta->convert_tile_min_max_var_sizes_to_offsets(attr); for (auto& batch : attr_tile_batches) { uint64_t idx = 0; @@ -249,7 +249,7 @@ Status OrderedWriter::ordered_write() { } else { // Parallelize over tiles for (const auto& buff : buffers_) { const auto& attr = buff.first; - auto& attr_tile_batches = tiles[attr]; + auto& attr_tile_batches = tiles.at(attr); const auto var_size = array_schema_.var_size(attr); if (has_min_max_metadata(attr, var_size) && array_schema_.var_size(attr)) { @@ -257,7 +257,7 @@ Status OrderedWriter::ordered_write() { RETURN_NOT_OK(parallel_for( compute_tp, 0, attr_tile_batches.size(), [&](uint64_t b) { const auto& attr = buff.first; - auto& batch = tiles[attr][b]; + auto& batch = tiles.at(attr)[b]; auto idx = b * thread_num; for (auto& tile : batch) { frag_meta->set_tile_min_var(attr, idx, tile.min()); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 147f6c89349c..ee8ecb0d1e71 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -89,6 +89,7 @@ UnorderedWriter::UnorderedWriter( remote_query, fragment_name) , frag_uri_(std::nullopt) + , cell_pos_(query_memory_tracker_->get_resource(MemoryType::WRITER_DATA)) , written_buffers_(written_buffers) , is_coords_pass_(true) { // Check the layout is unordered. @@ -372,7 +373,7 @@ Status UnorderedWriter::compute_coord_dups() { } Status UnorderedWriter::prepare_tiles( - std::unordered_map* tiles) const { + tdb::pmr::unordered_map* tiles) const { auto timer_se = stats_->start_timer("prepare_tiles"); // Initialize attribute tiles @@ -380,10 +381,10 @@ Status UnorderedWriter::prepare_tiles( for (const auto& it : buffers_) { const auto& name = it.first; if (written_buffers_.count(name) == 0) { - (*tiles).emplace( + tiles->emplace( std::piecewise_construct, std::forward_as_tuple(name), - std::forward_as_tuple()); + std::forward_as_tuple(query_memory_tracker_)); } } @@ -675,7 +676,8 @@ Status UnorderedWriter::unordered_write() { frag_uri_ = frag_meta_->fragment_uri(); // Prepare tiles - std::unordered_map tiles; + tdb::pmr::unordered_map tiles( + query_memory_tracker_->get_resource(MemoryType::WRITER_TILE_DATA)); RETURN_CANCEL_OR_ERROR(prepare_tiles(&tiles)); // No tiles diff --git a/tiledb/sm/query/writers/unordered_writer.h b/tiledb/sm/query/writers/unordered_writer.h index 2136ba509e74..5a9871c44b00 100644 --- a/tiledb/sm/query/writers/unordered_writer.h +++ b/tiledb/sm/query/writers/unordered_writer.h @@ -88,7 +88,7 @@ class UnorderedWriter : public WriterBase { Status alloc_frag_meta(); /** Returns the cell position vector. */ - std::vector& cell_pos() { + tdb::pmr::vector& cell_pos() { return cell_pos_; } @@ -119,7 +119,7 @@ class UnorderedWriter : public WriterBase { * The positions that resulted from sorting and according to which the cells * must be re-arranged. */ - std::vector cell_pos_; + tdb::pmr::vector cell_pos_; /** The set with the positions of duplicate coordinates/cells. */ std::set coord_dups_; @@ -173,7 +173,7 @@ class UnorderedWriter : public WriterBase { * @return Status */ Status prepare_tiles( - std::unordered_map* tiles) const; + tdb::pmr::unordered_map* tiles) const; /** * It prepares the tiles for the input attribute or dimension, re-organizing diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 262a71331938..8f659f9c7cf8 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -311,7 +311,7 @@ void WriterBase::refresh_config() { shared_ptr WriterBase::create_fragment_metadata() { return make_shared( - HERE(), &storage_manager_->resources(), array_memory_tracker_); + HERE(), &storage_manager_->resources(), query_memory_tracker_); } Status WriterBase::add_written_fragment_info(const URI& uri) { @@ -623,7 +623,8 @@ Status WriterBase::close_files(shared_ptr meta) const { } std::vector WriterBase::compute_mbrs( - const std::unordered_map& tiles) const { + const tdb::pmr::unordered_map& tiles) + const { auto timer_se = stats_->start_timer("compute_coord_meta"); // Applicable only if there are coordinates @@ -669,7 +670,7 @@ std::vector WriterBase::compute_mbrs( void WriterBase::set_coords_metadata( const uint64_t start_tile_idx, const uint64_t end_tile_idx, - const std::unordered_map& tiles, + const tdb::pmr::unordered_map& tiles, const std::vector& mbrs, shared_ptr meta) const { // Applicable only if there are coordinates @@ -701,7 +702,7 @@ void WriterBase::set_coords_metadata( Status WriterBase::compute_tiles_metadata( uint64_t tile_num, - std::unordered_map& tiles) const { + tdb::pmr::unordered_map& tiles) const { auto compute_tp = storage_manager_->compute_tp(); // Parallelize over attributes? @@ -795,7 +796,7 @@ Status WriterBase::create_fragment( array_->array_schema_latest_ptr(), fragment_uri_, timestamp_range, - array_memory_tracker_, + query_memory_tracker_, dense, has_timestamps, has_delete_metadata); @@ -805,7 +806,7 @@ Status WriterBase::create_fragment( } Status WriterBase::filter_tiles( - std::unordered_map* tiles) { + tdb::pmr::unordered_map* tiles) { auto timer_se = stats_->start_timer("filter_tiles"); auto status = parallel_for( storage_manager_->compute_tp(), 0, tiles->size(), [&](uint64_t i) { @@ -1048,7 +1049,7 @@ Status WriterBase::write_tiles( const uint64_t start_tile_idx, const uint64_t end_tile_idx, shared_ptr frag_meta, - std::unordered_map* const tiles) { + tdb::pmr::unordered_map* const tiles) { auto timer_se = stats_->start_timer("write_num_tiles"); assert(!tiles->empty()); @@ -1202,3 +1203,17 @@ bool WriterBase::remote_query() const { } } // namespace tiledb::sm + +template <> +IndexedList::IndexedList( + shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , list_(memory_tracker->get_resource(sm::MemoryType::WRITER_TILE_DATA)) { +} + +template <> +IndexedList>:: + IndexedList(shared_ptr memory_tracker) + : memory_tracker_(memory_tracker) + , list_(memory_tracker->get_resource(sm::MemoryType::WRITER_TILE_DATA)) { +} diff --git a/tiledb/sm/query/writers/writer_base.h b/tiledb/sm/query/writers/writer_base.h index 99be55ce2ca8..10a4464f5c9f 100644 --- a/tiledb/sm/query/writers/writer_base.h +++ b/tiledb/sm/query/writers/writer_base.h @@ -249,7 +249,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { * @return MBRs. */ std::vector compute_mbrs( - const std::unordered_map& tiles) + const tdb::pmr::unordered_map& tiles) const; /** @@ -265,7 +265,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { void set_coords_metadata( const uint64_t start_tile_idx, const uint64_t end_tile_idx, - const std::unordered_map& tiles, + const tdb::pmr::unordered_map& tiles, const std::vector& mbrs, shared_ptr meta) const; @@ -279,7 +279,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { */ Status compute_tiles_metadata( uint64_t tile_num, - std::unordered_map& tiles) const; + tdb::pmr::unordered_map& tiles) const; /** * Returns the i-th coordinates in the coordinate buffers in string @@ -305,7 +305,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { * of the pipeline. */ Status filter_tiles( - std::unordered_map* tiles); + tdb::pmr::unordered_map* tiles); /** * Runs the input tiles for the input attribute through the filter pipeline. @@ -441,7 +441,7 @@ class WriterBase : public StrategyBase, public IQueryStrategy { const uint64_t start_tile_idx, const uint64_t end_tile_idx, shared_ptr frag_meta, - std::unordered_map* tiles); + tdb::pmr::unordered_map* tiles); /** * Writes the input tiles for the input attribute/dimension to storage. diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index b678cdfdbbbb..fd0c1ec1c935 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -174,7 +174,7 @@ WriterTile::WriterTile( type, cell_size, size, - memory_tracker->get_resource(MemoryType::TILE_WRITER_DATA)) + memory_tracker->get_resource(MemoryType::WRITER_TILE_DATA)) , filtered_buffer_(0) { } diff --git a/tiledb/type/apply_with_type.h b/tiledb/type/apply_with_type.h index 1ed32996fdf0..f59fb9f3c613 100644 --- a/tiledb/type/apply_with_type.h +++ b/tiledb/type/apply_with_type.h @@ -59,43 +59,33 @@ inline auto apply_with_type(Fn&& f, Datatype type, Args&&... args) { switch (type) { case Datatype::INT32: { return f(int32_t{}, std::forward(args)...); - break; } case Datatype::INT64: { return f(int64_t{}, std::forward(args)...); - break; } case Datatype::INT8: { return f(int8_t{}, std::forward(args)...); - break; } case Datatype::UINT8: { return f(uint8_t{}, std::forward(args)...); - break; } case Datatype::INT16: { return f(int16_t{}, std::forward(args)...); - break; } case Datatype::UINT16: { return f(uint16_t{}, std::forward(args)...); - break; } case Datatype::UINT32: { return f(uint32_t{}, std::forward(args)...); - break; } case Datatype::UINT64: { return f(uint64_t{}, std::forward(args)...); - break; } case Datatype::FLOAT32: { return f(float{}, std::forward(args)...); - break; } case Datatype::FLOAT64: { return f(double{}, std::forward(args)...); - break; } case Datatype::DATETIME_YEAR: case Datatype::DATETIME_MONTH: @@ -120,16 +110,13 @@ inline auto apply_with_type(Fn&& f, Datatype type, Args&&... args) { case Datatype::TIME_FS: case Datatype::TIME_AS: { return f(int64_t{}, std::forward(args)...); - break; } case Datatype::STRING_ASCII: { return f(char{}, std::forward(args)...); - break; } default: { throw std::logic_error( "Datatype::" + datatype_str(type) + " is not a valid Datatype"); - break; } } } From f8bbb749a95cb9fdbfb33c7edff16dbc4a7716ce Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 20 Mar 2024 08:59:47 -0400 Subject: [PATCH 250/456] Tracking vector allocations inside of query condition. (#4816) I added a QueryCondition::Params class, which contains the memory tracker used by the query and the array schema. This allowed to easily add memory tracking to vectors allocated inside of query condition. --- TYPE: NO_HISTORY DESC: Tracking vector allocations inside of query condition. --------- Co-authored-by: Luc Rancourt --- tiledb/common/memory_tracker.cc | 4 +- tiledb/common/memory_tracker.h | 2 +- tiledb/sm/query/legacy/reader.cc | 15 +- tiledb/sm/query/query_condition.cc | 88 +++-- tiledb/sm/query/query_condition.h | 76 ++++- tiledb/sm/query/readers/dense_reader.cc | 7 +- tiledb/sm/query/readers/result_tile.h | 5 +- .../query/readers/sparse_index_reader_base.cc | 37 ++- tiledb/sm/query/test/unit_query_condition.cc | 312 ++++++++---------- 9 files changed, 282 insertions(+), 264 deletions(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index aee787332e1f..6cf0ede70c38 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -76,8 +76,6 @@ std::string memory_type_to_str(MemoryType type) { return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; - case MemoryType::TILE_QUERY_CONDITIONS: - return "TileQueryConditions"; case MemoryType::ATTRIBUTES: return "Attributes"; case MemoryType::DIMENSION_LABELS: @@ -98,6 +96,8 @@ std::string memory_type_to_str(MemoryType type) { return "ResultTileBitmap"; case MemoryType::RESULT_TILE: return "ResultTile"; + case MemoryType::QUERY_CONDITION: + return "QueryCondition"; } auto val = std::to_string(static_cast(type)); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index a7271bab86c5..9a66994e6ccb 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -114,7 +114,6 @@ enum class MemoryType { TILE_MAX_VALS, TILE_MIN_VALS, TILE_NULL_COUNTS, - TILE_QUERY_CONDITIONS, ATTRIBUTES, DIMENSION_LABELS, DIMENSIONS, @@ -124,6 +123,7 @@ enum class MemoryType { WRITER_TILE_DATA, WRITER_DATA, RESULT_TILE, + QUERY_CONDITION, RESULT_TILE_BITMAP }; diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 1a655cb51ef3..f171a3c159a5 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -353,17 +353,19 @@ Status Reader::apply_query_condition( if (stride == UINT64_MAX) stride = 1; + QueryCondition::Params params(query_memory_tracker_, array_schema_); if (condition_.has_value()) { RETURN_NOT_OK(condition_->apply( - array_schema_, fragment_metadata_, result_cell_slabs, stride)); + params, fragment_metadata_, result_cell_slabs, stride)); } // Apply delete conditions. if (!delete_and_update_conditions_.empty()) { for (uint64_t i = 0; i < delete_and_update_conditions_.size(); i++) { // For legacy, always run the timestamped condition. + RETURN_NOT_OK(timestamped_delete_and_update_conditions_[i].apply( - array_schema_, fragment_metadata_, result_cell_slabs, stride)); + params, fragment_metadata_, result_cell_slabs, stride)); } } @@ -371,7 +373,7 @@ Status Reader::apply_query_condition( if (!delete_timestamps_condition_.empty()) { // Remove cells with partial overlap from the bitmap. RETURN_NOT_OK(delete_timestamps_condition_.apply( - array_schema_, fragment_metadata_, result_cell_slabs, stride)); + params, fragment_metadata_, result_cell_slabs, stride)); } return Status::Ok(); @@ -476,8 +478,10 @@ Status Reader::compute_range_result_coords( // Apply partial overlap condition, if required. const auto frag_meta = fragment_metadata_[tile->frag_idx()]; if (process_partial_timestamps(*frag_meta)) { + QueryCondition::Params params( + query_memory_tracker_, *(frag_meta->array_schema().get())); RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( - *(frag_meta->array_schema().get()), *tile, result_bitmap)); + params, *tile, result_bitmap)); } // Gather results @@ -1790,8 +1794,9 @@ Status Reader::get_all_result_coords( auto resource = query_memory_tracker_->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(coords_num, 1, resource); + QueryCondition::Params params(query_memory_tracker_, array_schema_); RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( - *(frag_meta->array_schema().get()), *tile, result_bitmap)); + params, *tile, result_bitmap)); for (uint64_t i = 0; i < coords_num; ++i) { if (result_bitmap[i]) { diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index 5c2c8854a869..fb4a52482b41 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -32,6 +32,7 @@ #include "tiledb/sm/query/query_condition.h" #include "tiledb/common/logger.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_condition_combination_op.h" #include "tiledb/sm/enums/query_condition_op.h" @@ -616,7 +617,7 @@ void QueryCondition::apply_ast_node( const ByteVecValue& fill_value, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const { + tdb::pmr::vector& result_cell_bitmap) const { const std::string& field_name = node->get_field_name(); const void* condition_value_content = node->get_value_ptr(); const size_t condition_value_size = node->get_value_size(); @@ -768,7 +769,7 @@ void QueryCondition::apply_ast_node( const ByteVecValue& fill_value, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const { + tdb::pmr::vector& result_cell_bitmap) const { switch (node->get_op()) { case QueryConditionOp::ALWAYS_TRUE: apply_ast_node( @@ -907,7 +908,7 @@ void QueryCondition::apply_ast_node( const uint64_t stride, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const { + tdb::pmr::vector& result_cell_bitmap) const { std::string node_field_name = node->get_field_name(); const auto nullable = array_schema.is_nullable(node_field_name); @@ -1151,12 +1152,13 @@ void QueryCondition::apply_ast_node( template void QueryCondition::apply_tree( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, const std::vector>& fragment_metadata, uint64_t stride, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const { + tdb::pmr::vector& result_cell_bitmap) const { + const auto& array_schema = params.GetSchema(); if (!node->is_expr()) { apply_ast_node( node, @@ -1191,7 +1193,7 @@ void QueryCondition::apply_tree( for (const auto& child : node->get_children()) { apply_tree( child, - array_schema, + params, fragment_metadata, stride, result_cell_slabs, @@ -1203,12 +1205,15 @@ void QueryCondition::apply_tree( } else if constexpr (std::is_same_v< CombinationOp, std::logical_or>) { - std::vector combination_op_bitmap(result_bitmap_size, 1); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); + tdb::pmr::vector combination_op_bitmap( + result_bitmap_size, 1, resource); for (const auto& child : node->get_children()) { apply_tree( child, - array_schema, + params, fragment_metadata, stride, result_cell_slabs, @@ -1227,12 +1232,15 @@ void QueryCondition::apply_tree( * = a /\ (cl1'(q; cl2'(q; 0))) */ case QueryConditionCombinationOp::OR: { - std::vector combination_op_bitmap(result_bitmap_size, 0); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); + tdb::pmr::vector combination_op_bitmap( + result_bitmap_size, 0, resource); for (const auto& child : node->get_children()) { apply_tree( child, - array_schema, + params, fragment_metadata, stride, result_cell_slabs, @@ -1257,7 +1265,7 @@ void QueryCondition::apply_tree( } Status QueryCondition::apply( - const ArraySchema& array_schema, + const QueryCondition::Params& params, const std::vector>& fragment_metadata, std::vector& result_cell_slabs, const uint64_t stride) const { @@ -1270,10 +1278,13 @@ Status QueryCondition::apply( total_lengths += elem.length_; } - std::vector result_cell_bitmap(total_lengths, 1); + auto resource = + params.GetMemoryTracker()->get_resource(MemoryType::QUERY_CONDITION); + tdb::pmr::vector result_cell_bitmap(total_lengths, 1, resource); + apply_tree( tree_, - array_schema, + params, fragment_metadata, stride, result_cell_slabs, @@ -1934,7 +1945,7 @@ void QueryCondition::apply_ast_node_dense( template void QueryCondition::apply_tree_dense( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile* result_tile, const uint64_t start, const uint64_t src_cell, @@ -1945,7 +1956,7 @@ void QueryCondition::apply_tree_dense( if (!node->is_expr()) { apply_ast_node_dense( node, - array_schema, + params.GetSchema(), result_tile, start, src_cell, @@ -1978,7 +1989,7 @@ void QueryCondition::apply_tree_dense( for (const auto& child : node->get_children()) { apply_tree_dense( child, - array_schema, + params, result_tile, start, src_cell, @@ -1992,14 +2003,17 @@ void QueryCondition::apply_tree_dense( } else if constexpr (std::is_same_v< CombinationOp, std::logical_or>) { - std::vector combination_op_bitmap(result_buffer_size, 1); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); + tdb::pmr::vector combination_op_bitmap( + result_buffer_size, 1, resource); span combination_op_span( combination_op_bitmap.data(), result_buffer_size); for (const auto& child : node->get_children()) { apply_tree_dense( child, - array_schema, + params, result_tile, start, src_cell, @@ -2019,13 +2033,16 @@ void QueryCondition::apply_tree_dense( * = a /\ (cl1'(q; cl2'(q; 0))) */ case QueryConditionCombinationOp::OR: { - std::vector combination_op_bitmap(result_buffer_size, 0); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); + tdb::pmr::vector combination_op_bitmap( + result_buffer_size, 0, resource); span combination_op_span( combination_op_bitmap.data(), result_buffer_size); for (const auto& child : node->get_children()) { apply_tree_dense( child, - array_schema, + params, result_tile, start, src_cell, @@ -2052,7 +2069,7 @@ void QueryCondition::apply_tree_dense( } Status QueryCondition::apply_dense( - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile* result_tile, const uint64_t start, const uint64_t length, @@ -2068,7 +2085,7 @@ Status QueryCondition::apply_dense( span result_span(result_buffer + start, length); apply_tree_dense( tree_, - array_schema, + params, result_tile, start, src_cell, @@ -2741,10 +2758,11 @@ void QueryCondition::apply_ast_node_sparse( template void QueryCondition::apply_tree_sparse( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile& result_tile, CombinationOp combination_op, tdb::pmr::vector& result_bitmap) const { + const auto& array_schema = params.GetSchema(); if (!node->is_expr()) { apply_ast_node_sparse( node, array_schema, result_tile, combination_op, result_bitmap); @@ -2774,7 +2792,7 @@ void QueryCondition::apply_tree_sparse( for (const auto& child : node->get_children()) { apply_tree_sparse( child, - array_schema, + params, result_tile, std::multiplies(), result_bitmap); @@ -2783,14 +2801,15 @@ void QueryCondition::apply_tree_sparse( // Handle the cl'(q, a) case. // This cases on whether the combination op = OR. } else if constexpr (std::is_same_v>) { - auto resource = result_bitmap.get_allocator().resource(); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); tdb::pmr::vector combination_op_bitmap( result_bitmap_size, 1, resource); for (const auto& child : node->get_children()) { apply_tree_sparse( child, - array_schema, + params, result_tile, std::multiplies(), combination_op_bitmap); @@ -2807,14 +2826,15 @@ void QueryCondition::apply_tree_sparse( * = a /\ (cl1'(q; cl2'(q; 0))) */ case QueryConditionCombinationOp::OR: { - auto resource = result_bitmap.get_allocator().resource(); + auto resource = params.GetMemoryTracker()->get_resource( + MemoryType::QUERY_CONDITION); tdb::pmr::vector combination_op_bitmap( result_bitmap_size, 0, resource); for (const auto& child : node->get_children()) { apply_tree_sparse( child, - array_schema, + params, result_tile, QCMax(), combination_op_bitmap); @@ -2838,15 +2858,11 @@ void QueryCondition::apply_tree_sparse( template Status QueryCondition::apply_sparse( - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile& result_tile, tdb::pmr::vector& result_bitmap) { apply_tree_sparse( - tree_, - array_schema, - result_tile, - std::multiplies(), - result_bitmap); + tree_, params, result_tile, std::multiplies(), result_bitmap); return Status::Ok(); } @@ -2869,7 +2885,7 @@ uint64_t QueryCondition::condition_index() const { // Explicit template instantiations. template Status QueryCondition::apply_sparse( - const ArraySchema& array_schema, ResultTile&, tdb::pmr::vector&); + const QueryCondition::Params&, ResultTile&, tdb::pmr::vector&); template Status QueryCondition::apply_sparse( - const ArraySchema& array_schema, ResultTile&, tdb::pmr::vector&); + const QueryCondition::Params&, ResultTile&, tdb::pmr::vector&); } // namespace tiledb::sm diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 33115e16f320..8ef3df48b018 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -48,6 +48,7 @@ namespace tiledb { namespace sm { class FragmentMetadata; +class MemoryTracker; struct ResultCellSlab; class ResultTile; @@ -55,6 +56,49 @@ enum class QueryConditionCombinationOp : uint8_t; class QueryCondition { public: + /** Class to pass query parameters to query condition processing. */ + class Params { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + Params() = delete; + + /** Constructor setting all parameters. */ + Params( + shared_ptr memory_tracker, + const ArraySchema& array_schema) + : memory_tracker_(memory_tracker) + , schema_(array_schema) { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Returns the memory tracker. */ + shared_ptr GetMemoryTracker() const { + return memory_tracker_; + } + + /** Returns the array schema. */ + const ArraySchema& GetSchema() const { + return schema_; + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Memory tracker. */ + shared_ptr memory_tracker_; + + /** Array schema. */ + const ArraySchema& schema_; + }; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -192,7 +236,7 @@ class QueryCondition { /** * Applies this query condition to `result_cell_slabs`. * - * @param array_schema The array schema associated with `result_cell_slabs`. + * @param params Query condition parameters. * @param fragment_metadata The fragment metadata. * @param result_cell_slabs The cell slabs to filter. Mutated to remove cell * slabs that do not meet the criteria in this query condition. @@ -200,7 +244,7 @@ class QueryCondition { * @return Status */ Status apply( - const ArraySchema& array_schema, + const QueryCondition::Params& params, const std::vector>& fragment_metadata, std::vector& result_cell_slabs, uint64_t stride) const; @@ -208,7 +252,7 @@ class QueryCondition { /** * Applies this query condition to a set of cells. * - * @param array_schema The array schema. + * @param params Query condition parameters. * @param result_tile The result tile to get the cells from. * @param start The start cell. * @param length The number of cells to process. @@ -219,7 +263,7 @@ class QueryCondition { * @return Status */ Status apply_dense( - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile* result_tile, const uint64_t start, const uint64_t length, @@ -231,14 +275,14 @@ class QueryCondition { /** * Applies this query condition to a set of cells. * - * @param array_schema The array schema. + * @param params Query condition parameters. * @param result_tile The result tile to get the cells from. * @param result_bitmap The bitmap to use for results. * @return Status */ template Status apply_sparse( - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile& result_tile, tdb::pmr::vector& result_bitmap); @@ -354,7 +398,7 @@ class QueryCondition { const ByteVecValue& fill_value, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const; + tdb::pmr::vector& result_cell_bitmap) const; /** * Applies a value node on primitive-typed result cell slabs. @@ -379,7 +423,7 @@ class QueryCondition { const ByteVecValue& fill_value, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const; + tdb::pmr::vector& result_cell_bitmap) const; /** * Applies a value node to filter result cells from the input @@ -401,14 +445,14 @@ class QueryCondition { uint64_t stride, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const; + tdb::pmr::vector& result_cell_bitmap) const; /** * Applies the query condition represented with the AST to * `result_cell_slabs`. * * @param node The node to apply. - * @param array_schema The array schema associated with `result_cell_slabs`. + * @param params Query condition parameters. * @param fragment_metadata The fragment metadata. * @param stride The stride between cells. * @param combination_op The combination op. @@ -420,12 +464,12 @@ class QueryCondition { template > void apply_tree( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, const std::vector>& fragment_metadata, uint64_t stride, const std::vector& result_cell_slabs, CombinationOp combination_op, - std::vector& result_cell_bitmap) const; + tdb::pmr::vector& result_cell_bitmap) const; /** * Applies a value node on a dense result tile, @@ -518,7 +562,7 @@ class QueryCondition { * Applies the query condition represented with the AST to a set of cells. * * @param node The node to apply. - * @param array_schema The array schema. + * @param params Query condition parameters. * @param result_tile The result tile to get the cells from. * @param start The start cell. * @param src_cell The cell offset in the source tile. @@ -531,7 +575,7 @@ class QueryCondition { template > void apply_tree_dense( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile* result_tile, const uint64_t start, const uint64_t src_cell, @@ -628,7 +672,7 @@ class QueryCondition { * Applies the query condition represented with the AST to a set of cells. * * @param node The node to apply. - * @param array_schema The array schema. + * @param params Query condition parameters. * @param result_tile The result tile to get the cells from. * @param combination_op The combination op. * @param result_bitmap The bitmap to use for results. @@ -639,7 +683,7 @@ class QueryCondition { typename CombinationOp = std::logical_and> void apply_tree_sparse( const tdb_unique_ptr& node, - const ArraySchema& array_schema, + const QueryCondition::Params& params, ResultTile& result_tile, CombinationOp combination_op, tdb::pmr::vector& result_bitmap) const; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 60a3d5a18ba4..60dd2871bdaf 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -1005,10 +1005,13 @@ Status DenseReader::apply_query_condition( } } - RETURN_NOT_OK(condition_->apply_dense( + QueryCondition::Params params( + query_memory_tracker_, *(fragment_metadata_[frag_domains[i].fid()] ->array_schema() - .get()), + .get())); + RETURN_NOT_OK(condition_->apply_dense( + params, result_space_tile.result_tile(frag_domains[i].fid()), start, end - start + 1, diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index 1f4e004ef408..e303562969ed 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -40,6 +40,7 @@ #include #include "tiledb/common/common.h" +#include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/fragment/fragment_metadata.h" @@ -984,8 +985,8 @@ class GlobalOrderResultTile : public ResultTileWithBitmap { , hilbert_values_(this->memory_tracker_->get_resource( MemoryType::TILE_HILBERT_VALUES)) , post_dedup_bitmap_(nullopt) - , per_cell_delete_condition_(this->memory_tracker_->get_resource( - MemoryType::TILE_QUERY_CONDITIONS)) + , per_cell_delete_condition_( + this->memory_tracker_->get_resource(MemoryType::QUERY_CONDITION)) , used_(false) { if (!dups || include_delete_meta) { post_dedup_bitmap_.emplace( diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 49d0e7bcb369..fed07985afd1 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -728,8 +728,10 @@ void SparseIndexReaderBase::apply_query_condition( } // Remove cells with partial overlap from the bitmap. + QueryCondition::Params params( + query_memory_tracker_, *(frag_meta->array_schema().get())); RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( - *(frag_meta->array_schema().get()), *rt, rt->bitmap())); + params, *rt, rt->bitmap())); rt->count_cells(); } @@ -743,10 +745,10 @@ void SparseIndexReaderBase::apply_query_condition( if (frag_meta->has_delete_meta() && !deletes_consolidation_no_purge_) { // Remove cells deleted cells using the open timestamp. + QueryCondition::Params params( + query_memory_tracker_, *(frag_meta->array_schema().get())); RETURN_NOT_OK(delete_timestamps_condition_.apply_sparse( - *(frag_meta->array_schema().get()), - *rt, - rt->post_dedup_bitmap())); + params, *rt, rt->post_dedup_bitmap())); if (array_schema_.allows_dups()) { rt->count_cells(); } @@ -754,10 +756,10 @@ void SparseIndexReaderBase::apply_query_condition( // Compute the result of the query condition for this tile. if (condition_.has_value()) { + QueryCondition::Params params( + query_memory_tracker_, *(frag_meta->array_schema().get())); RETURN_NOT_OK(condition_->apply_sparse( - *(frag_meta->array_schema().get()), - *rt, - rt->post_dedup_bitmap())); + params, *rt, rt->post_dedup_bitmap())); if (array_schema_.allows_dups()) { rt->count_cells(); } @@ -785,19 +787,20 @@ void SparseIndexReaderBase::apply_query_condition( // start. if (delete_timestamp >= frag_meta->timestamp_range().first) { // Apply timestamped condition or regular condition. + QueryCondition::Params params( + query_memory_tracker_, + *(frag_meta->array_schema().get())); if (!frag_meta->has_timestamps() || delete_timestamp > frag_meta->timestamp_range().second) { - RETURN_NOT_OK(delete_and_update_conditions_[i] - .apply_sparse( - *(frag_meta->array_schema().get()), - *rt, - rt->post_dedup_bitmap())); + RETURN_NOT_OK( + delete_and_update_conditions_[i] + .apply_sparse( + params, *rt, rt->post_dedup_bitmap())); } else { - RETURN_NOT_OK(timestamped_delete_and_update_conditions_[i] - .apply_sparse( - *(frag_meta->array_schema().get()), - *rt, - rt->post_dedup_bitmap())); + RETURN_NOT_OK( + timestamped_delete_and_update_conditions_[i] + .apply_sparse( + params, *rt, rt->post_dedup_bitmap())); } if (deletes_consolidation_no_purge_) { diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 51d98666f542..7289dc4c2e0c 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -58,16 +58,17 @@ void check_ast_str(QueryCondition qc, std::string expect) { TEST_CASE( "QueryCondition: Test default constructor", "[QueryCondition][default_constructor]") { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); QueryCondition query_condition; REQUIRE(query_condition.empty()); REQUIRE(query_condition.field_names().empty()); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); std::vector result_cell_slabs; std::vector> frag_md; - REQUIRE( - query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE(query_condition.apply(params, frag_md, result_cell_slabs, 1).ok()); } TEST_CASE("QueryCondition: Test init", "[QueryCondition][value_constructor]") { @@ -190,6 +191,7 @@ TEST_CASE( } TEST_CASE("QueryCondition: Test blob type", "[QueryCondition][blob]") { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); std::string field_name = "blob_attr"; std::byte value{5}; @@ -202,16 +204,17 @@ TEST_CASE("QueryCondition: Test blob type", "[QueryCondition][blob]") { QueryConditionOp::LT) .ok()); - shared_ptr array_schema = make_shared( - HERE(), ArrayType::DENSE, tiledb::test::create_test_memory_tracker()); + shared_ptr array_schema = + make_shared(HERE(), ArrayType::DENSE, memory_tracker); shared_ptr attr = make_shared(HERE(), "blob_attr", Datatype::BLOB); REQUIRE(array_schema->add_attribute(attr).ok()); std::vector result_cell_slabs; std::vector> frag_md; + QueryCondition::Params params(memory_tracker, *array_schema); REQUIRE_THROWS_WITH( - query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1), + query_condition.apply(params, frag_md, result_cell_slabs, 1), Catch::Matchers::ContainsSubstring( "Cannot perform query comparison; Unsupported datatype " + datatype_str(Datatype::BLOB))); @@ -1065,6 +1068,7 @@ void test_apply_cells( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const char* const cmp_value = "ae"; QueryCondition query_condition; REQUIRE(query_condition @@ -1129,10 +1133,10 @@ void test_apply_cells( array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); - REQUIRE( - query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE(query_condition.apply(params, frag_md, result_cell_slabs, 1).ok()); // Verify the result cell slabs contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -1166,10 +1170,11 @@ void test_apply_cells( array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); + QueryCondition::Params params(memory_tracker, *array_schema); REQUIRE(query_condition_eq_null - .apply(*array_schema, frag_md, result_cell_slabs_eq_null, 1) + .apply(params, frag_md, result_cell_slabs_eq_null, 1) .ok()); REQUIRE(result_cell_slabs_eq_null.size() == (cells / 2)); @@ -1235,8 +1240,7 @@ void test_apply_cells( std::vector fill_result_cell_slabs; fill_result_cell_slabs.emplace_back(std::move(fill_result_cell_slab)); REQUIRE( - query_condition.apply(*array_schema, frag_md, fill_result_cell_slabs, 1) - .ok()); + query_condition.apply(params, frag_md, fill_result_cell_slabs, 1).ok()); // Verify the fill result cell slabs contain the expected cells. auto fill_expected_iter = fill_expected_cell_idx_vec.begin(); @@ -1262,6 +1266,7 @@ void test_apply_cells( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const T cmp_value = 5; QueryCondition query_condition; REQUIRE( @@ -1315,10 +1320,10 @@ void test_apply_cells( array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); - REQUIRE( - query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE(query_condition.apply(params, frag_md, result_cell_slabs, 1).ok()); // Verify the result cell slabs contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -1377,8 +1382,7 @@ void test_apply_cells( std::vector fill_result_cell_slabs; fill_result_cell_slabs.emplace_back(std::move(fill_result_cell_slab)); REQUIRE( - query_condition.apply(*array_schema, frag_md, fill_result_cell_slabs, 1) - .ok()); + query_condition.apply(params, frag_md, fill_result_cell_slabs, 1).ok()); // Verify the fill result cell slabs contain the expected cells. auto fill_expected_iter = fill_expected_cell_idx_vec.begin(); @@ -1564,6 +1568,7 @@ void test_apply( */ template <> void test_apply(const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); REQUIRE((type == Datatype::STRING_ASCII || type == Datatype::STRING_UTF8)); const std::string field_name = "foo"; @@ -1571,7 +1576,6 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { const char* fill_value = "ac"; // Initialize the array schema. - auto memory_tracker = tiledb::test::get_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -1585,11 +1589,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; - auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + auto dim{ + make_shared(HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -1607,8 +1608,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { true); // Initialize the result tile. - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileSizes tile_sizes( var_size ? (cells + 1) * constants::cell_var_offset_size : 2 * cells * sizeof(char), @@ -1634,12 +1634,12 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { */ template void test_apply(const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const T fill_value = 3; // Initialize the array schema. - auto memory_tracker = tiledb::test::get_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -1648,11 +1648,8 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; - auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + auto dim{ + make_shared(HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -1677,8 +1674,7 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -1735,6 +1731,7 @@ TEST_CASE("QueryCondition: Test apply", "[QueryCondition][apply]") { TEST_CASE( "QueryCondition: Test empty/null strings", "[QueryCondition][empty_string][null_string]") { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const char* fill_value = "ac"; @@ -1748,7 +1745,6 @@ TEST_CASE( return; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -1763,10 +1759,7 @@ TEST_CASE( .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -1782,7 +1775,7 @@ TEST_CASE( array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); // Initialize the result tile. @@ -1795,8 +1788,7 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, *frag_md[0], tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, *frag_md[0], memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -1892,8 +1884,8 @@ TEST_CASE( ResultCellSlab result_cell_slab(&result_tile, 0, cells); std::vector result_cell_slabs; result_cell_slabs.emplace_back(std::move(result_cell_slab)); - REQUIRE( - query_condition.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE(query_condition.apply(params, frag_md, result_cell_slabs, 1).ok()); // Verify the result cell slabs contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -1936,6 +1928,7 @@ void test_apply_cells_dense( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const char* const cmp_value = "ae"; QueryCondition query_condition; REQUIRE(query_condition @@ -1991,17 +1984,12 @@ void test_apply_cells_dense( // Apply the query condition. std::vector result_bitmap(cells, 1); - REQUIRE(query_condition - .apply_dense( - *array_schema, - result_tile, - 0, - 10, - 0, - 1, - nullptr, - result_bitmap.data()) - .ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE( + query_condition + .apply_dense( + params, result_tile, 0, 10, 0, 1, nullptr, result_bitmap.data()) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -2024,9 +2012,10 @@ void test_apply_cells_dense( // Apply the query condition. std::vector result_bitmap_eq_null(cells, 1); + QueryCondition::Params params(memory_tracker, *array_schema); REQUIRE(query_condition_eq_null .apply_dense( - *array_schema, + params, result_tile, 0, 10, @@ -2057,6 +2046,7 @@ void test_apply_cells_dense( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const T cmp_value = 5; QueryCondition query_condition; REQUIRE( @@ -2101,17 +2091,12 @@ void test_apply_cells_dense( // Apply the query condition. std::vector result_bitmap(cells, 1); - REQUIRE(query_condition - .apply_dense( - *array_schema, - result_tile, - 0, - 10, - 0, - 1, - nullptr, - result_bitmap.data()) - .ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE( + query_condition + .apply_dense( + params, result_tile, 0, 10, 0, 1, nullptr, result_bitmap.data()) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -2294,6 +2279,7 @@ void test_apply_dense( template <> void test_apply_dense( const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); REQUIRE((type == Datatype::STRING_ASCII || type == Datatype::STRING_UTF8)); const std::string field_name = "foo"; @@ -2301,7 +2287,6 @@ void test_apply_dense( const char* fill_value = "ac"; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -2316,10 +2301,7 @@ void test_apply_dense( .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2346,8 +2328,7 @@ void test_apply_dense( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2364,12 +2345,12 @@ void test_apply_dense( */ template void test_apply_dense(const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const T fill_value = 3; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -2379,10 +2360,7 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2407,8 +2385,7 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2466,6 +2443,7 @@ TEST_CASE( TEST_CASE( "QueryCondition: Test empty/null strings dense", "[QueryCondition][empty_string][null_string][dense]") { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const char* fill_value = "ac"; @@ -2479,7 +2457,6 @@ TEST_CASE( return; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -2494,10 +2471,7 @@ TEST_CASE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -2524,8 +2498,7 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -2618,17 +2591,12 @@ TEST_CASE( // Apply the query condition. std::vector result_bitmap(cells, 1); - REQUIRE(query_condition - .apply_dense( - *array_schema, - &result_tile, - 0, - 10, - 0, - 1, - nullptr, - result_bitmap.data()) - .ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE( + query_condition + .apply_dense( + params, &result_tile, 0, 10, 0, 1, nullptr, result_bitmap.data()) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -2670,6 +2638,7 @@ void test_apply_cells_sparse( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const char* const cmp_value = "ae"; QueryCondition query_condition; REQUIRE(query_condition @@ -2724,12 +2693,12 @@ void test_apply_cells_sparse( } // Apply the query condition. - auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::RESULT_TILE_BITMAP); + QueryCondition::Params params(memory_tracker, *array_schema); + auto resource = memory_tracker->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); - REQUIRE(query_condition - .apply_sparse(*array_schema, *result_tile, result_bitmap) - .ok()); + REQUIRE( + query_condition.apply_sparse(params, *result_tile, result_bitmap) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -2751,12 +2720,13 @@ void test_apply_cells_sparse( REQUIRE(query_condition_eq_null.check(*array_schema).ok()); // Apply the query condition. - auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::RESULT_TILE_BITMAP); + QueryCondition::Params params(memory_tracker, *array_schema); + auto resource = + memory_tracker->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap_eq_null(cells, 1, resource); REQUIRE(query_condition_eq_null .apply_sparse( - *array_schema, *result_tile, result_bitmap_eq_null) + params, *result_tile, result_bitmap_eq_null) .ok()); // Verify the result bitmap contain the expected cells. @@ -2780,6 +2750,7 @@ void test_apply_cells_sparse( shared_ptr array_schema, ResultTile* const result_tile, void* values) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const T cmp_value = 5; QueryCondition query_condition; REQUIRE( @@ -2823,12 +2794,12 @@ void test_apply_cells_sparse( } // Apply the query condition. - auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::RESULT_TILE_BITMAP); + QueryCondition::Params params(memory_tracker, *array_schema); + auto resource = memory_tracker->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); - REQUIRE(query_condition - .apply_sparse(*array_schema, *result_tile, result_bitmap) - .ok()); + REQUIRE( + query_condition.apply_sparse(params, *result_tile, result_bitmap) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); @@ -3011,6 +2982,7 @@ void test_apply_sparse( template <> void test_apply_sparse( const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); REQUIRE((type == Datatype::STRING_ASCII || type == Datatype::STRING_UTF8)); const std::string field_name = "foo"; @@ -3018,7 +2990,6 @@ void test_apply_sparse( const char* fill_value = "ac"; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -3033,10 +3004,7 @@ void test_apply_sparse( .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3063,8 +3031,7 @@ void test_apply_sparse( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -3081,12 +3048,12 @@ void test_apply_sparse( */ template void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const T fill_value = 3; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -3096,10 +3063,7 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3124,8 +3088,7 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { var_size ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -3255,6 +3218,7 @@ void validate_qc_apply( shared_ptr array_schema, ResultTile& result_tile, bool negated = false) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); ResultCellSlab result_cell_slab(&result_tile, 0, cells); std::vector result_cell_slabs; result_cell_slabs.emplace_back(std::move(result_cell_slab)); @@ -3265,9 +3229,10 @@ void validate_qc_apply( array_schema, URI(), std::make_pair(0, 0), - tiledb::test::create_test_memory_tracker(), + memory_tracker, true); - REQUIRE(tp.qc_.apply(*array_schema, frag_md, result_cell_slabs, 1).ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE(tp.qc_.apply(params, frag_md, result_cell_slabs, 1).ok()); REQUIRE(result_cell_slabs.size() == tp.expected_slabs_.size()); uint64_t result_cell_slabs_size = result_cell_slabs.size(); for (uint64_t i = 0; i < result_cell_slabs_size; ++i) { @@ -3297,22 +3262,21 @@ void validate_qc_apply_sparse( shared_ptr array_schema, ResultTile& result_tile, bool negated = false) { - auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::RESULT_TILE_BITMAP); + auto memory_tracker = tiledb::test::get_test_memory_tracker(); + QueryCondition::Params params(memory_tracker, *array_schema); + auto resource = memory_tracker->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector sparse_result_bitmap(cells, 1, resource); - REQUIRE(tp.qc_ - .apply_sparse( - *array_schema, result_tile, sparse_result_bitmap) - .ok()); + REQUIRE( + tp.qc_.apply_sparse(params, result_tile, sparse_result_bitmap) + .ok()); for (uint64_t i = 0; i < cells; ++i) { CHECK(sparse_result_bitmap[i] == tp.expected_bitmap_[i]); } tdb::pmr::vector sparse_result_bitmap1(cells, 2, resource); - REQUIRE(tp.qc_ - .apply_sparse( - *array_schema, result_tile, sparse_result_bitmap1) - .ok()); + REQUIRE( + tp.qc_.apply_sparse(params, result_tile, sparse_result_bitmap1) + .ok()); for (uint64_t i = 0; i < cells; ++i) { CHECK(sparse_result_bitmap1[i] == tp.expected_bitmap_[i] * 2); } @@ -3339,10 +3303,12 @@ void validate_qc_apply_dense( shared_ptr array_schema, ResultTile& result_tile, bool negated = false) { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); std::vector dense_result_bitmap(cells, 1); + QueryCondition::Params params(memory_tracker, *array_schema); REQUIRE(tp.qc_ .apply_dense( - *array_schema, + params, &result_tile, 0, 10, @@ -3859,12 +3825,12 @@ void populate_test_params_vector( TEST_CASE( "QueryCondition: Test combinations", "[QueryCondition][combinations]") { // Setup. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const Datatype type = Datatype::UINT64; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -3872,10 +3838,7 @@ TEST_CASE( .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -3900,8 +3863,7 @@ TEST_CASE( std::nullopt, std::nullopt, std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4148,12 +4110,12 @@ TEST_CASE( "QueryCondition: Test combinations, string", "[QueryCondition][combinations][string]") { // Setup. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const Datatype type = GENERATE(Datatype::STRING_ASCII, Datatype::STRING_UTF8); // Initialize the array schema. - auto memory_tracker = tiledb::test::get_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -4164,11 +4126,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; - auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + auto dim{ + make_shared(HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4194,8 +4153,7 @@ TEST_CASE( 0, std::nullopt, std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4509,12 +4467,12 @@ TEST_CASE( "QueryCondition: Test combinations, string with UTF-8 data", "[QueryCondition][combinations][string][utf-8]") { // Setup. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const Datatype type = Datatype::STRING_UTF8; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -4525,11 +4483,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; - auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + auto dim{ + make_shared(HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4613,8 +4568,7 @@ TEST_CASE( 0, std::nullopt, std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4838,12 +4792,12 @@ TEST_CASE( "QueryCondition: Test combinations, nullable attributes", "[QueryCondition][combinations][nullable]") { // Setup. + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const Datatype type = Datatype::FLOAT32; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -4852,10 +4806,7 @@ TEST_CASE( .ok()); auto domain{make_shared(HERE(), memory_tracker)}; auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4880,8 +4831,7 @@ TEST_CASE( std::nullopt, cells * constants::cell_validity_size, 0); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -4930,6 +4880,7 @@ TEST_CASE( TEST_CASE( "QueryCondition: Test empty/null strings sparse", "[QueryCondition][empty_string][null_string][sparse]") { + auto memory_tracker = tiledb::test::get_test_memory_tracker(); const std::string field_name = "foo"; const uint64_t cells = 10; const char* fill_value = "ac"; @@ -4943,7 +4894,6 @@ TEST_CASE( return; // Initialize the array schema. - auto memory_tracker = tiledb::test::create_test_memory_tracker(); shared_ptr array_schema = make_shared(HERE(), ArrayType::DENSE, memory_tracker); Attribute attr(field_name, type); @@ -4957,11 +4907,8 @@ TEST_CASE( REQUIRE( array_schema->add_attribute(make_shared(HERE(), attr)).ok()); auto domain{make_shared(HERE(), memory_tracker)}; - auto dim{make_shared( - HERE(), - "dim1", - Datatype::UINT32, - tiledb::test::get_test_memory_tracker())}; + auto dim{ + make_shared(HERE(), "dim1", Datatype::UINT32, memory_tracker)}; uint32_t bounds[2] = {1, cells}; Range range(bounds, 2 * sizeof(uint32_t)); REQUIRE(dim->set_domain(range).ok()); @@ -4988,8 +4935,7 @@ TEST_CASE( nullable ? std::optional(cells * constants::cell_validity_size) : std::nullopt, nullable ? std::optional(0) : std::nullopt); - ResultTile result_tile( - 0, 0, frag_md, tiledb::test::get_test_memory_tracker()); + ResultTile result_tile(0, 0, frag_md, memory_tracker); ResultTile::TileData tile_data{nullptr, nullptr, nullptr}; result_tile.init_attr_tile( constants::format_version, @@ -5081,12 +5027,12 @@ TEST_CASE( } // Apply the query condition. - auto resource = tiledb::test::get_test_memory_tracker()->get_resource( - MemoryType::RESULT_TILE_BITMAP); + auto resource = memory_tracker->get_resource(MemoryType::RESULT_TILE_BITMAP); tdb::pmr::vector result_bitmap(cells, 1, resource); - REQUIRE(query_condition - .apply_sparse(*array_schema, result_tile, result_bitmap) - .ok()); + QueryCondition::Params params(memory_tracker, *array_schema); + REQUIRE( + query_condition.apply_sparse(params, result_tile, result_bitmap) + .ok()); // Verify the result bitmap contain the expected cells. auto expected_iter = expected_cell_idx_vec.begin(); From 8ea85dcfc441d1096ad6491cf36936046b19a06a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:29:23 -0400 Subject: [PATCH 251/456] Single-process sub-millisecond temporal disambiguation of random labels. (#4800) Add a submillisecond counter on `random_label` generation to disambiguate fragments which are written within the same millisecond on a single-process. --- TYPE: BUG DESC: Single-process sub-millisecond temporal disambiguation of random labels. --- tiledb/common/random/CMakeLists.txt | 3 +- tiledb/common/random/random_label.cc | 74 +++++++--- tiledb/common/random/random_label.h | 72 ++++++++- tiledb/common/random/test/CMakeLists.txt | 4 +- .../test/unit_random_label_generator.cc | 138 ++++++++++++++++++ 5 files changed, 263 insertions(+), 28 deletions(-) create mode 100644 tiledb/common/random/test/unit_random_label_generator.cc diff --git a/tiledb/common/random/CMakeLists.txt b/tiledb/common/random/CMakeLists.txt index edff498f6394..4717ba38f59f 100644 --- a/tiledb/common/random/CMakeLists.txt +++ b/tiledb/common/random/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,7 @@ gather_sources(${SOURCES}) commence(object_library seedable_global_PRNG) this_target_sources(${SOURCES}) this_target_link_libraries(export) + this_target_object_libraries(baseline time) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/common/random/random_label.cc b/tiledb/common/random/random_label.cc index cc962b94d0ab..ba04ae92bfca 100644 --- a/tiledb/common/random/random_label.cc +++ b/tiledb/common/random/random_label.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,33 +31,65 @@ */ #include "tiledb/common/random/random_label.h" -#include "tiledb/common/random/prng.h" - -#include -#include namespace tiledb::common { -/** - * Legacy code provides randomness using UUIDs, which are always 128 bits, - * represented as a 32-digit hexadecimal value. - * - * To ensure backward compatibility, this function formats the PRNG-generated - * values to be precisely a 32-digit hexadecimal value. Each value is padded - * with 0s such that it makes up one 16-digit half of the full 32-digit number. - */ -std::string random_label() { +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ +RandomLabelGenerator::RandomLabelGenerator() + : prev_time_(tiledb::sm::utils::time::timestamp_now_ms()) { +} + +/* ********************************* */ +/* API */ +/* ********************************* */ +std::string RandomLabelGenerator::generate() { PRNG& prng = PRNG::get(); - std::stringstream ss; + std::lock_guard lock(mtx_); + auto now = tiledb::sm::utils::time::timestamp_now_ms(); - // Generate and format a 128-bit, 32-digit hexadecimal random number - auto rand1 = prng(); - ss << std::hex << std::setw(16) << std::setfill('0') << rand1; - auto rand2 = prng(); - ss << std::hex << std::setw(16) << std::setfill('0') << rand2; + // If no label has been generated this millisecond, generate a new one. + if (now != prev_time_) { + prev_time_ = now; + counter_ = static_cast(prng()); + // Clear the top bit of the counter such that a full 2 billion values + // could be generated within a single millisecond. + counter_ &= 0x7FFFFFFF; + } else { + counter_ += 1; + if (counter_ == 0) { + throw RandomLabelException("Maximum generation frequency exceeded."); + } + } - // Return label string + // Generate and format a 128-bit, 32-digit hexadecimal random number + std::stringstream ss; + ss << std::hex << std::setw(8) << std::setfill('0') << counter_; + ss << std::hex << std::setw(8) << std::setfill('0') + << static_cast(prng()); + ss << std::hex << std::setw(16) << std::setfill('0') << prng(); return ss.str(); } +std::string RandomLabelGenerator::generate_random_label() { + static RandomLabelGenerator generator; + return generator.generate(); +} + +/** + * Wrapper function for `generate_random_label`, which returns a PRNG-generated + * label as a 32-digit hexadecimal random number. + * (Ex. f258d22d4db9139204eef2b4b5d860cc). + * + * @pre If multiple labels are generated within the same millisecond, they will + * be sorted using a counter on the most significant 4 bytes. + * @note Labels may be 0-padded to ensure exactly a 128-bit, 32-digit length. + * + * @return A random label. + */ +std::string random_label() { + return RandomLabelGenerator::generate_random_label(); +} + } // namespace tiledb::common diff --git a/tiledb/common/random/random_label.h b/tiledb/common/random/random_label.h index 600a5831039b..6675605f4350 100644 --- a/tiledb/common/random/random_label.h +++ b/tiledb/common/random/random_label.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,16 +33,80 @@ #ifndef TILEDB_HELPERS_H #define TILEDB_HELPERS_H +#include +#include +#include #include +#include "tiledb/common/exception/exception.h" +#include "tiledb/common/random/prng.h" +#include "tiledb/sm/misc/tdb_time.h" + namespace tiledb::common { +class RandomLabelException : public StatusException { + public: + explicit RandomLabelException(const std::string& message) + : StatusException("RandomLabel", message) { + } +}; + +/** + * Generates a pseudeo-random label, formatted as a 32-digit hexadecimal number. + * (Ex. f258d22d4db9139204eef2b4b5d860cc). + * + * @pre If multiple labels are generated within the same millisecond, they will + * be sorted using a counter on the most significant 4 bytes. + * @note Use of wrapper `random_label()` is encouraged in production code. + */ +class RandomLabelGenerator { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + DISABLE_COPY_AND_COPY_ASSIGN(RandomLabelGenerator); + DISABLE_MOVE_AND_MOVE_ASSIGN(RandomLabelGenerator); + + /** Default destructor. */ + ~RandomLabelGenerator() = default; + + protected: + /** Protected constructor, abstracted by public-facing accessor. */ + RandomLabelGenerator(); + + /* ********************************* */ + /* API */ + /* ********************************* */ + /** Generate a random label. */ + std::string generate(); + + public: + /** Generate a random label. */ + static std::string generate_random_label(); + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Mutex which protects against simultaneous random label generation. */ + std::mutex mtx_; + + /** The time (in milliseconds) of the last label creation. */ + uint64_t prev_time_; + + /** The submillsecond counter portion of the random label. */ + uint32_t counter_; +}; + /** - * Returns a PRNG-generated label as a 32-digit hexadecimal random number. + * Wrapper function for `generate_random_label`, which returns a PRNG-generated + * label as a 32-digit hexadecimal random number. * (Ex. f258d22d4db9139204eef2b4b5d860cc). * - * Note: the random number is actually the combination of two 16-digit numbers. - * The values are 0-padded to ensure exactly a 128-bit, 32-digit length. + * @pre If multiple labels are generated within the same millisecond, they will + * be sorted using a counter on the most significant 4 bytes. + * @note Labels may be 0-padded to ensure exactly a 128-bit, 32-digit length. * * @return A random label. */ diff --git a/tiledb/common/random/test/CMakeLists.txt b/tiledb/common/random/test/CMakeLists.txt index 2a8754ac7dcd..508aab5f121b 100644 --- a/tiledb/common/random/test/CMakeLists.txt +++ b/tiledb/common/random/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,6 @@ include(unit_test) commence(unit_test seedable_global_PRNG) - this_target_sources(unit_seedable_global_PRNG.cc) + this_target_sources(unit_random_label_generator.cc unit_seedable_global_PRNG.cc) this_target_object_libraries(seedable_global_PRNG) conclude(unit_test) diff --git a/tiledb/common/random/test/unit_random_label_generator.cc b/tiledb/common/random/test/unit_random_label_generator.cc new file mode 100644 index 000000000000..529608a42ed0 --- /dev/null +++ b/tiledb/common/random/test/unit_random_label_generator.cc @@ -0,0 +1,138 @@ +/** + * @file tiledb/common/random/test/unit_random_label_generator.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * Tests for the random label generator. + */ + +#include +#include "../random_label.h" + +using namespace tiledb::common; +using namespace tiledb::sm; + +size_t generate_labels(std::vector& labels) { + size_t labels_size = labels.size(); + auto now = utils::time::timestamp_now_ms(); + size_t idx = 0; + while ((utils::time::timestamp_now_ms()) < now + 100 && idx < labels_size) { + labels[idx++] = random_label(); + } + + return idx; +} + +void validate_labels(std::vector& labels, size_t num_labels) { + // Given the label randomness and the fact that we're racing the processor, + // the best we can do here (for now) is assert that there's 10 ordered groups. + // In this manner, groups are defined as sharing the first 4 bytes. + uint64_t num_groups = 0; + uint64_t this_group = 0; + for (size_t i = 1; i < num_labels; i++) { + bool match = true; + for (size_t j = 0; j < 4; j++) { + if (labels[i - 1][j] != labels[i][j]) { + match = false; + break; + } + } + if (!match) { + if (this_group > 10) { + num_groups += 1; + } + this_group = 0; + continue; + } + + // We share a prefix so assert that they're ordered. + REQUIRE(labels[i] > labels[i - 1]); + this_group += 1; + } + + REQUIRE(num_groups > 10); +} + +TEST_CASE( + "RandomLabelGenerator: serial generation", + "[RandomLabelGenerator][serial]") { + // Generate a random label to validate initialization. + auto label = random_label(); + REQUIRE(label.size() == 32); + + // Test one million strings. Let's assume the buffer overflow check works. + std::vector labels{1000000}; + auto num_labels = generate_labels(labels); + validate_labels(labels, num_labels); +} + +TEST_CASE( + "RandomLabelGenerator: parallel generation", + "[RandomLabelGenerator][parallel]") { + const unsigned nthreads = 20; + std::vector threads; + std::vector> labels{nthreads}; + size_t num_labels[nthreads]; + + // Pre-allocate our buffers so we're getting as much contention as possible + for (size_t i = 0; i < nthreads; i++) { + labels[i].resize(1000000); + } + + // Generate labels simultaneously in multiple threads. + for (size_t i = 0; i < nthreads; i++) { + auto num_ptr = &num_labels[i]; + auto vec_ptr = &labels[i]; + threads.emplace_back([num_ptr, vec_ptr]() { + auto num = generate_labels(*vec_ptr); + *num_ptr = num; + }); + } + + // Wait for all of our threads to finish. + for (auto& t : threads) { + t.join(); + } + + // Check that we've generated the correct number of random labels. + std::unordered_set label_set; + size_t total_labels = 0; + for (size_t i = 0; i < nthreads; i++) { + total_labels += num_labels[i]; + for (size_t j = 0; j < num_labels[i]; j++) { + label_set.insert(labels[i][j]); + } + } + REQUIRE(label_set.size() == total_labels); + + // Sort and validate the parallel threads as if they were serially generated. + std::vector all_labels{total_labels}; + size_t idx = 0; + for (auto label : label_set) { + all_labels[idx++] = label; + } + std::sort(all_labels.begin(), all_labels.end()); + validate_labels(all_labels, total_labels); +} From f03d9b4a33a4d32d5214309295c6b7af1e34517a Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Thu, 21 Mar 2024 09:01:32 -0400 Subject: [PATCH 252/456] Expose VFSExperimental and relocate CallbackWrapperCPP. (#4820) @kounelisagis reported issues working on [SC-43316](https://app.shortcut.com/tiledb-inc/story/43316/add-wrapping-for-ls-recursive) and found that #4625 did not expose VFSExperimental. This exposes the VFS experimental C++ APIs and relocates CallbackWrapperCPP to fix build errors from using `tiledb::sm` in an external header. The C APIs were exposed in #4615. --- TYPE: BUG DESC: Expose VFSExperimental and relocate CallbackWrapperCPP. --------- Co-authored-by: Agis Kounelis --- test/src/unit-cppapi-vfs.cc | 7 +++-- tiledb/CMakeLists.txt | 1 + tiledb/sm/cpp_api/tiledb_experimental | 1 + tiledb/sm/cpp_api/vfs_experimental.h | 34 ++++++++++++++++++++--- tiledb/sm/filesystem/ls_scanner.h | 39 --------------------------- 5 files changed, 36 insertions(+), 46 deletions(-) diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index 0a0c78bced4e..28a803ce628d 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -651,14 +651,13 @@ TEST_CASE("CPP API: Throwing filter", "[cppapi][vfs][ls-recursive]") { TEST_CASE( "CPP API: CallbackWrapperCPP construction validation", "[ls-recursive][callback][wrapper]") { - using tiledb::sm::CallbackWrapperCPP; tiledb::VFSExperimental::LsObjects data; auto cb = [&](std::string_view, uint64_t) -> bool { return true; }; SECTION("Null callback") { - CHECK_THROWS(CallbackWrapperCPP(nullptr)); + CHECK_THROWS(tiledb::VFSExperimental::CallbackWrapperCPP(nullptr)); } SECTION("Valid callback") { - CHECK_NOTHROW(CallbackWrapperCPP(cb)); + CHECK_NOTHROW(tiledb::VFSExperimental::CallbackWrapperCPP(cb)); } } @@ -677,7 +676,7 @@ TEST_CASE( data.emplace_back(path, object_size); return true; }; - tiledb::sm::CallbackWrapperCPP wrapper(cb); + tiledb::VFSExperimental::CallbackWrapperCPP wrapper(cb); SECTION("Callback return true accepts object") { CHECK(wrapper("file.txt", 10) == true); diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index e245c55109fc..490b4023e71e 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -123,6 +123,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/utils.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/version.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/vfs.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/vfs_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/arrowio ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/arrow_io_impl.h ) diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index f3954ac06a9b..4af4bbe940f9 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -45,5 +45,6 @@ #include "query_condition_experimental.h" #include "query_experimental.h" #include "subarray_experimental.h" +#include "vfs_experimental.h" #endif // TILEDB_EXPERIMENTAL_CPP_H diff --git a/tiledb/sm/cpp_api/vfs_experimental.h b/tiledb/sm/cpp_api/vfs_experimental.h index 149b27afca84..7f0506cd0aa6 100644 --- a/tiledb/sm/cpp_api/vfs_experimental.h +++ b/tiledb/sm/cpp_api/vfs_experimental.h @@ -82,6 +82,35 @@ class VFSExperimental { */ using LsObjects = std::vector>; + /** Class to wrap C++ FilePredicate for passing to the C API. */ + class CallbackWrapperCPP { + public: + /** Default constructor is deleted */ + CallbackWrapperCPP() = delete; + + /** Constructor */ + CallbackWrapperCPP(LsCallback cb) + : cb_(cb) { + if (cb_ == nullptr) { + throw std::logic_error("ls_recursive callback function cannot be null"); + } + } + + /** + * Operator to wrap the FilePredicate used in the C++ API. + * + * @param path The path of the object. + * @param size The size of the object in bytes. + * @return True if the object should be included, False otherwise. + */ + bool operator()(std::string_view path, uint64_t size) { + return cb_(path, size); + } + + private: + LsCallback cb_; + }; + /* ********************************* */ /* PUBLIC STATIC METHODS */ /* ********************************* */ @@ -117,7 +146,7 @@ class VFSExperimental { const VFS& vfs, const std::string& uri, LsCallback cb) { - tiledb::sm::CallbackWrapperCPP wrapper(cb); + CallbackWrapperCPP wrapper(cb); ctx.handle_error(tiledb_vfs_ls_recursive( ctx.ptr().get(), vfs.ptr().get(), @@ -195,8 +224,7 @@ class VFSExperimental { */ static int32_t ls_callback_wrapper( const char* path, size_t path_len, uint64_t object_size, void* data) { - tiledb::sm::CallbackWrapperCPP* cb = - static_cast(data); + CallbackWrapperCPP* cb = static_cast(data); return (*cb)({path, path_len}, object_size); } }; diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h index 92bc435bd9e7..029338165806 100644 --- a/tiledb/sm/filesystem/ls_scanner.h +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -316,45 +316,6 @@ class CallbackWrapperCAPI { void* data_; }; -/** Class to wrap C++ FilePredicate for passing to the C API. */ -class CallbackWrapperCPP { - public: - /** - * Typedef for ls callback function used to check if a single - * result should be included in the final results returned from ls_recursive. - * - * @param path The path of a visited object for the relative filesystem. - * @param size The size of the current object in bytes. - * @return True if the result should be included, else false. - */ - using LsCallback = std::function; - - /** Default constructor is deleted */ - CallbackWrapperCPP() = delete; - - /** Constructor */ - CallbackWrapperCPP(LsCallback cb) - : cb_(cb) { - if (cb_ == nullptr) { - throw LsScanException("ls_recursive callback function cannot be null"); - } - } - - /** - * Operator to wrap the FilePredicate used in the C++ API. - * - * @param path The path of the object. - * @param size The size of the object in bytes. - * @return True if the object should be included, False otherwise. - */ - bool operator()(std::string_view path, uint64_t size) { - return cb_(path, size); - } - - private: - LsCallback cb_; -}; - /** * Implements the `ls_filtered` function for `std::filesystem` which can be used * for Posix and Win32 From b774c0c6a840a17b856d6c5c326e9da7c4f19884 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:10:26 -0400 Subject: [PATCH 253/456] Remove uuid in favor of random_label. (#4819) Remove `uuid` in favor of `random_label`. --- TYPE: NO_HISTORY DESC: Remove uuid in favor of random_label. --- tiledb/CMakeLists.txt | 1 - tiledb/sm/array/CMakeLists.txt | 1 - tiledb/sm/array_schema/CMakeLists.txt | 5 +- tiledb/sm/array_schema/enumeration.cc | 10 +- tiledb/sm/misc/CMakeLists.txt | 16 +- tiledb/sm/misc/test/CMakeLists.txt | 5 +- tiledb/sm/misc/test/compile_uuid_main.cc | 34 ----- tiledb/sm/misc/test/unit_uuid.cc | 87 ----------- tiledb/sm/misc/uuid.cc | 174 ---------------------- tiledb/sm/misc/uuid.h | 59 -------- tiledb/storage_format/uri/CMakeLists.txt | 2 +- tiledb/storage_format/uri/generate_uri.cc | 8 +- 12 files changed, 13 insertions(+), 389 deletions(-) delete mode 100644 tiledb/sm/misc/test/compile_uuid_main.cc delete mode 100644 tiledb/sm/misc/test/unit_uuid.cc delete mode 100644 tiledb/sm/misc/uuid.cc delete mode 100644 tiledb/sm/misc/uuid.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 490b4023e71e..b7bc62f6f7b0 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -236,7 +236,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/tdb_time.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/types.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/uuid.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc diff --git a/tiledb/sm/array/CMakeLists.txt b/tiledb/sm/array/CMakeLists.txt index 5aaebf284da9..163d84dd5bc6 100644 --- a/tiledb/sm/array/CMakeLists.txt +++ b/tiledb/sm/array/CMakeLists.txt @@ -38,7 +38,6 @@ commence(object_library array) context_resources fragment generic_tile_io - uuid vfs ) if(TILEDB_STATS) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 171331ad3166..602d9b9af264 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -38,8 +38,7 @@ commence(object_library attribute) constants filter_pipeline range - stringx - uuid) + stringx) conclude(object_library) # @@ -63,7 +62,7 @@ conclude(object_library) # commence(object_library enumeration) this_target_sources(enumeration.cc) - this_target_object_libraries(buffer constants uuid) + this_target_object_libraries(buffer constants seedable_global_PRNG) conclude(object_library) # diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc index dabe1818a3a0..658a7219ee37 100644 --- a/tiledb/sm/array_schema/enumeration.cc +++ b/tiledb/sm/array_schema/enumeration.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,7 +34,7 @@ #include #include "tiledb/common/memory_tracker.h" -#include "tiledb/sm/misc/uuid.h" +#include "tiledb/common/random/random_label.h" #include "enumeration.h" @@ -75,10 +75,8 @@ Enumeration::Enumeration( } if (path_name_.empty()) { - std::string tmp_uuid; - throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); - path_name_ = - "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); + path_name_ = "__" + tiledb::common::random_label() + "_" + + std::to_string(constants::enumerations_version); } if (path_name.find("/") != std::string::npos) { diff --git a/tiledb/sm/misc/CMakeLists.txt b/tiledb/sm/misc/CMakeLists.txt index f6b033b82741..db755e822aa5 100644 --- a/tiledb/sm/misc/CMakeLists.txt +++ b/tiledb/sm/misc/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2023 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -71,20 +71,6 @@ commence(object_library time) this_target_sources(tdb_time.cc) conclude(object_library) -# -# `uuid` object library -# -commence(object_library uuid) - this_target_sources(uuid.cc) - this_target_object_libraries(baseline) - if(WIN32) - this_target_link_libraries(rpcrt4) - else() - find_package(OpenSSL_EP REQUIRED) - this_target_link_libraries(OpenSSL::Crypto) - endif() -conclude(object_library) - add_test_subdirectory() # diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index c61fe26da6fc..a0dac49341cb 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022-2023 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ include(unit_test) commence(unit_test misc) - this_target_object_libraries(math uuid) + this_target_object_libraries(math) this_target_link_libraries(tiledb_test_support_lib) # change to `this_target_include_directories` when available target_include_directories(unit_misc PRIVATE "${CMAKE_SOURCE_DIR}") @@ -36,6 +36,5 @@ commence(unit_test misc) unit_hilbert.cc unit_integral_type_casts.cc unit_math.cc - unit_uuid.cc ) conclude(unit_test) diff --git a/tiledb/sm/misc/test/compile_uuid_main.cc b/tiledb/sm/misc/test/compile_uuid_main.cc deleted file mode 100644 index 2db30c1d6e52..000000000000 --- a/tiledb/sm/misc/test/compile_uuid_main.cc +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @file compile_uuid_main.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "../uuid.h" - -int main() { - (void)tiledb::sm::uuid::generate_uuid(nullptr, false); - return 0; -} diff --git a/tiledb/sm/misc/test/unit_uuid.cc b/tiledb/sm/misc/test/unit_uuid.cc deleted file mode 100644 index 8dbf830b29cd..000000000000 --- a/tiledb/sm/misc/test/unit_uuid.cc +++ /dev/null @@ -1,87 +0,0 @@ -/** - * @file unit_uuid.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests the UUID utility functions. - */ - -#include -#include -#include -#include - -#include "tiledb/sm/global_state/global_state.h" -#include "tiledb/sm/misc/uuid.h" - -using namespace tiledb::sm; - -std::mutex catch2_macro_mutex; - -// A thread-safe variant of the REQUIRE macro. -#define REQUIRE_SAFE(a) \ - { \ - std::lock_guard lock(catch2_macro_mutex); \ - REQUIRE(a); \ - } - -void cancel_all_tasks(StorageManager*) { -} - -TEST_CASE("UUID: Test generate", "[uuid]") { - SECTION("- Serial") { - std::string uuid0, uuid1, uuid2; - REQUIRE(uuid::generate_uuid(&uuid0).ok()); - REQUIRE(uuid0.length() == 36); - REQUIRE(uuid::generate_uuid(&uuid1).ok()); - REQUIRE(uuid1.length() == 36); - REQUIRE(uuid0 != uuid1); - - REQUIRE(uuid::generate_uuid(&uuid2, false).ok()); - REQUIRE(uuid2.length() == 32); - } - - SECTION("- Threaded") { - const unsigned nthreads = 20; - std::vector uuids(nthreads); - std::vector threads; - for (unsigned i = 0; i < nthreads; i++) { - threads.emplace_back([&uuids, i]() { - std::string& uuid = uuids[i]; - REQUIRE_SAFE(uuid::generate_uuid(&uuid).ok()); - REQUIRE_SAFE(uuid.length() == 36); - }); - } - for (auto& t : threads) { - t.join(); - } - // Check uniqueness - std::set uuid_set; - uuid_set.insert(uuids.begin(), uuids.end()); - REQUIRE(uuid_set.size() == uuids.size()); - } -} diff --git a/tiledb/sm/misc/uuid.cc b/tiledb/sm/misc/uuid.cc deleted file mode 100644 index 052cce47bdcd..000000000000 --- a/tiledb/sm/misc/uuid.cc +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @file uuid.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines a platform-independent UUID generator. - */ - -#include -#include - -#include "tiledb/sm/misc/uuid.h" - -#ifdef _WIN32 -#include -#else -#include -#include -#include -#endif - -using namespace tiledb::common; - -namespace tiledb::sm::uuid { - -/** Mutex to guard UUID generation. */ -static std::mutex uuid_mtx; - -#ifdef _WIN32 - -/** - * Generate a UUID using Win32 RPC API. - */ -Status generate_uuid_win32(std::string* uuid_str) { - if (uuid_str == nullptr) - return Status_UtilsError("Null UUID string argument"); - - UUID uuid; - RPC_STATUS rc = UuidCreate(&uuid); - if (rc != RPC_S_OK) - return Status_UtilsError("Unable to generate Win32 UUID: creation error"); - - char* buf = nullptr; - rc = UuidToStringA(&uuid, reinterpret_cast(&buf)); - if (rc != RPC_S_OK) - return Status_UtilsError( - "Unable to generate Win32 UUID: string conversion error"); - - *uuid_str = std::string(buf); - - rc = RpcStringFreeA(reinterpret_cast(&buf)); - if (rc != RPC_S_OK) - return Status_UtilsError("Unable to generate Win32 UUID: free error"); - - return Status::Ok(); -} - -#else - -/** - * Generate a UUID using OpenSSL. - * - * Initially from: https://gist.github.com/kvelakur/9069c9896577c3040030 - * "Generating a Version 4 UUID using OpenSSL" - */ -Status generate_uuid_openssl(std::string* uuid_str) { - if (uuid_str == nullptr) - return Status_UtilsError("Null UUID string argument"); - - union { - struct { - uint32_t time_low; - uint16_t time_mid; - uint16_t time_hi_and_version; - uint8_t clk_seq_hi_res; - uint8_t clk_seq_low; - uint8_t node[6]; - }; - uint8_t __rnd[16]; - } uuid; - - int rc = RAND_bytes(uuid.__rnd, sizeof(uuid)); - if (rc < 1) { - char err_msg[256]; - ERR_error_string_n(ERR_get_error(), err_msg, sizeof(err_msg)); - return Status_UtilsError( - "Cannot generate random bytes with OpenSSL: " + std::string(err_msg)); - } - - // Refer Section 4.2 of RFC-4122 - // https://tools.ietf.org/html/rfc4122#section-4.2 - uuid.clk_seq_hi_res = (uint8_t)((uuid.clk_seq_hi_res & 0x3F) | 0x80); - uuid.time_hi_and_version = - (uint16_t)((uuid.time_hi_and_version & 0x0FFF) | 0x4000); - - // Format the UUID as a string. - char buf[128]; - rc = snprintf( - buf, - sizeof(buf), - "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", - uuid.time_low, - uuid.time_mid, - uuid.time_hi_and_version, - uuid.clk_seq_hi_res, - uuid.clk_seq_low, - uuid.node[0], - uuid.node[1], - uuid.node[2], - uuid.node[3], - uuid.node[4], - uuid.node[5]); - - if (rc < 0) - return Status_UtilsError("Error formatting UUID string"); - - *uuid_str = std::string(buf); - - return Status::Ok(); -} - -#endif - -Status generate_uuid(std::string* uuid, bool hyphenate) { - if (uuid == nullptr) - return Status_UtilsError("Null UUID string argument"); - - std::string uuid_str; - { - // OpenSSL is not threadsafe, so grab a lock here. We are locking in the - // Windows case as well just to be careful. - std::unique_lock lck(uuid_mtx); -#ifdef _WIN32 - RETURN_NOT_OK(generate_uuid_win32(&uuid_str)); -#else - RETURN_NOT_OK(generate_uuid_openssl(&uuid_str)); -#endif - } - - uuid->clear(); - for (unsigned i = 0; i < uuid_str.length(); i++) { - if (uuid_str[i] == '-' && !hyphenate) - continue; - uuid->push_back(uuid_str[i]); - } - - return Status::Ok(); -} - -} // namespace tiledb::sm::uuid diff --git a/tiledb/sm/misc/uuid.h b/tiledb/sm/misc/uuid.h deleted file mode 100644 index 5999007f8b59..000000000000 --- a/tiledb/sm/misc/uuid.h +++ /dev/null @@ -1,59 +0,0 @@ -/** - * @file uuid.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declares a platform-independent UUID generator. - */ - -#ifndef TILEDB_UUID_H -#define TILEDB_UUID_H - -#include "tiledb/common/status.h" - -using namespace tiledb::common; - -namespace tiledb { -namespace sm { -namespace uuid { - -/** - * Generates a 128-bit UUID. The string is formatted with hyphens like: - * 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx' where 'x' is a hexadecimal digit. - * Note: this function internally acquires a lock. - * - * @param uuid Output parameter which will store the UUID in string format. - * @param hyphenate If false, the UUID string will not be hyphenated. - * @return Status - */ -Status generate_uuid(std::string* uuid, bool hyphenate = true); - -} // namespace uuid -} // namespace sm -} // namespace tiledb - -#endif diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index e92ebae65092..b0b7949accf7 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -31,7 +31,7 @@ include(object_library) # commence(object_library uri_format) this_target_sources(parse_uri.cc generate_uri.cc) - this_target_object_libraries(baseline fragment time uuid vfs) + this_target_object_libraries(baseline fragment time vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index fe0e0a9738aa..d823936c13d2 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -27,9 +27,9 @@ */ #include "tiledb/storage_format/uri/generate_uri.h" +#include "tiledb/common/random/random_label.h" #include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/misc/uuid.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -42,9 +42,6 @@ std::string generate_timestamped_name( uint64_t timestamp_start, uint64_t timestamp_end, std::optional version) { - std::string uuid; - throw_if_not_ok(sm::uuid::generate_uuid(&uuid, false)); - if (timestamp_start > timestamp_end) { throw std::logic_error( "Error generating timestamped name; " @@ -52,7 +49,8 @@ std::string generate_timestamped_name( } std::stringstream ss; - ss << "/__" << timestamp_start << "_" << timestamp_end << "_" << uuid; + ss << "/__" << timestamp_start << "_" << timestamp_end << "_" + << random_label(); if (version.has_value()) { ss << "_" << version.value(); From 07b26fb1e49eff99b514d661c9435ca67a414824 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 21 Mar 2024 22:59:30 +0100 Subject: [PATCH 254/456] Do not load group metadata when getting it from REST. (#4821) This was introduced in https://github.com/TileDB-Inc/TileDB/pull/4760 where the client tries to load the group metadata when serializing the group metadata request to REST, which causes an infinite loop. The fix reverts to the correct behavior, where we don't try to load the metadata when serializing it from the client. --- TYPE: BUG DESC: Do not load group metadata when getting it from REST. --- tiledb/sm/serialization/group.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index de8611677273..9a233f8bc088 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -70,7 +70,7 @@ Status group_metadata_to_capnp( auto config_builder = group_metadata_builder->initConfig(); RETURN_NOT_OK(config_to_capnp(group->config(), &config_builder)); - auto metadata = group->metadata(); + Metadata* metadata = load ? group->metadata() : group->unsafe_metadata(); if (metadata->num()) { auto metadata_builder = group_metadata_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &metadata_builder)); @@ -157,7 +157,9 @@ Status group_details_to_capnp( } } - auto metadata = group->metadata(); + Metadata* metadata = group->group_uri().is_tiledb() ? + group->unsafe_metadata() : + group->metadata(); if (metadata->num()) { auto group_metadata_builder = group_details_builder->initMetadata(); RETURN_NOT_OK(metadata_to_capnp(metadata, &group_metadata_builder)); From af35543fa8dcca9ba72e11b4acad17dd15147948 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Fri, 22 Mar 2024 02:28:24 -0400 Subject: [PATCH 255/456] Support triggering REST CI workflow on a specific ref (#4823) Support triggering REST CI workflow on a specific ref (commit/tag). This adds an input option on the manual trigger button for this workflow; the existing dropdown _only_ supports branches and tags. image --- TYPE: NO_HISTORY DESC: Support triggering REST CI workflow on a specific ref --- .github/workflows/ci-rest.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml index 0f5caab07213..327788a1208a 100644 --- a/.github/workflows/ci-rest.yml +++ b/.github/workflows/ci-rest.yml @@ -3,6 +3,10 @@ name: REST CI on: workflow_call: workflow_dispatch: + inputs: + ref: + type: string + required: false push: branches: @@ -35,5 +39,5 @@ jobs: token: ${{ secrets.TILEDB_REST_CI_PAT }} # Pass TileDB core ref to test against REST. # github.head_ref will only be set for PRs, so fallback to github.ref_name if triggered via push event. - inputs: '{ "tiledb_ref": "${{ github.head_ref || github.ref_name }}"}' + inputs: '{ "tiledb_ref": "${{ github.event.inputs.ref || github.head_ref || github.ref_name }}"}' wait-for-completion-timeout: 90m From ceee95ed305db8ac440ed288c2a16e7b988b5b5b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 22 Mar 2024 15:51:41 +0200 Subject: [PATCH 256/456] Support Microsoft Entra ID authentication to Azure. (#4126) [SC-25944](https://app.shortcut.com/tiledb-inc/story/25944/support-active-directory-authentication-to-azure) To authenticate to Azure, we currently support shared keys and SAS tokens. The former have the disadvantage of [being discouraged by Microsoft](https://learn.microsoft.com/en-us/azure/storage/common/authorize-data-access#:~:text=Microsoft%20recommends%20that%20you%20disallow%20Shared%20Key%20authorization%20for%20your%20storage%20account.) and the latter have the disadvantage of having a usually short expiry time (_edit:_ and being impossible to individually revoke). This PR adds support for Microsoft Entra ID authentication (formerly known as Azure Active Directory), which is what Microsoft recommends. It will be used for HTTPS endpoints if neither a shared key or SAS token are specified in the config. We use the `ChainedTokenCredential` class from the [`azure-identity-cpp`](https://azuresdkdocs.blob.core.windows.net/$web/cpp/azure-identity/1.5.0/index.html) package, and try to get credentials from the following sources in order: * [Environment variables](https://azuresdkdocs.blob.core.windows.net/$web/cpp/azure-identity/1.5.0/index.html#environment-variables:~:text=Azure%20Virtual%20Machines-,Environment%20Variables,-DefaultAzureCredential) * [The Azure CLI](https://learn.microsoft.com/cli/azure/authenticate-azure-cli) (for local development) * [Managed identities](https://learn.microsoft.com/en-gb/azure/active-directory/managed-identities-azure-resources/overview) (apps hosted on Azure can get automatically authenticated) * [Workload identities](https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview) (for apps hosted in Kubernetes) At initialization time we try to fetch an Active Directory token. If all three credential sources fail, we fall back to anonymous authentication. ### TODO - [x] Verify that it works locally. - [ ] Verify that accessing a publicly available blob with Entra ID credentials that do not grant access to it works. * If it doesn't, we have to remove the anonymous access fallback and add a config option (whether it will be opt-in or out is TBD). --- TYPE: FEATURE DESC: Support Microsoft Entra ID authentication to Azure. --------- Co-authored-by: Luc Rancourt --- cmake/Modules/FindAzureCore_EP.cmake | 148 ------------------ cmake/Modules/FindAzureStorageBlobs_EP.cmake | 136 ---------------- cmake/Modules/FindAzureStorageCommon_EP.cmake | 142 ----------------- cmake/Modules/FindWIL_EP.cmake | 95 ----------- cmake/TileDB-Superbuild.cmake | 9 -- cmake/inputs/Config.cmake.in | 1 + tiledb/CMakeLists.txt | 32 +--- tiledb/sm/filesystem/azure.cc | 101 +++++++++--- tiledb/sm/filesystem/azure.h | 3 + vcpkg.json | 1 + 10 files changed, 91 insertions(+), 577 deletions(-) delete mode 100644 cmake/Modules/FindAzureCore_EP.cmake delete mode 100644 cmake/Modules/FindAzureStorageBlobs_EP.cmake delete mode 100644 cmake/Modules/FindAzureStorageCommon_EP.cmake delete mode 100644 cmake/Modules/FindWIL_EP.cmake diff --git a/cmake/Modules/FindAzureCore_EP.cmake b/cmake/Modules/FindAzureCore_EP.cmake deleted file mode 100644 index f58739fdac1e..000000000000 --- a/cmake/Modules/FindAzureCore_EP.cmake +++ /dev/null @@ -1,148 +0,0 @@ -# -# FindAzureCore_EP.cmake -# -# -# The MIT License -# -# Copyright (c) 2018-2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# This module finds the Azure core SDK, installing it with an ExternalProject if -# necessary. It then defines the imported by target Azure::azure-core. - -# Include some common helper functions. -include(TileDBCommon) - -# Azure dependencies -if (TILEDB_VCPKG) - return() -endif() - -############################################################################### -# Start superbuild/unmanaged/legacy version -############################################################################### - -# First check for a static version in the EP prefix. -find_library(AZURECORE_LIBRARIES - NAMES - libazure-core${CMAKE_STATIC_LIBRARY_SUFFIX} - azure-core${CMAKE_STATIC_LIBRARY_SUFFIX} - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES lib lib64 - NO_DEFAULT_PATH - ) - -#on win32... '.lib' also used for lib ref'ing .dll!!! -#So, if perchance, in some environments, a .lib existed along with its corresponding .dll, -#then we could be incorrectly assuming/attempting a static build and probably will fail to -#appropriately include/package the .dll, since the code is (incorrectly) assuming .lib is indicative of a -#static library being used. - -if (AZURECORE_LIBRARIES) - set(AZURECORE_STATIC_EP_FOUND TRUE) - find_path(AZURECORE_INCLUDE_DIR - NAMES azure/core.hpp - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES include - NO_DEFAULT_PATH - ) -elseif(NOT TILEDB_FORCE_ALL_DEPS) - set(AZURECORE_STATIC_EP_FOUND FALSE) - # Static EP not found, search in system paths. - find_library(AZURECORE_LIBRARIES - NAMES - libazure-core #*nix name - azure-core #windows name - PATH_SUFFIXES lib64 lib bin - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) - find_path(AZURECORE_INCLUDE_DIR - NAMES azure/core.hpp - PATH_SUFFIXES include - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Azurecore - REQUIRED_VARS AZURECORE_LIBRARIES AZURECORE_INCLUDE_DIR - ) - -if (NOT AZURECORE_FOUND) - if (TILEDB_SUPERBUILD) - message(STATUS "Could NOT find azure-core") - message(STATUS "Adding azure-core as an external project") - - set(DEPENDS) - if (TARGET ep_openssl) - list(APPEND DEPENDS ep_openssl) - endif() - if (TARGET ep_curl) - list(APPEND DEPENDS ep_curl) - endif() - # Define windows dependencies - if (WIN32) - if(TARGET ep_wil) - list(APPEND DEPENDS ep_wil) - endif() - endif() - - ExternalProject_Add(ep_azure_core - PREFIX "externals" - URL "https://github.com/Azure/azure-sdk-for-cpp/archive/azure-core_1.10.0.zip" - URL_HASH SHA1=95fd8d56b439145228570c16ade5c0a73bbf6904 - DOWNLOAD_NAME azure-core_1.10.0.zip - SOURCE_SUBDIR sdk/core/azure-core - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=$ - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_PREFIX_PATH=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_INSTALL_PREFIX=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - -DWARNINGS_AS_ERRORS=OFF - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON - LOG_DOWNLOAD TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE ${TILEDB_LOG_OUTPUT_ON_FAILURE} - DEPENDS ${DEPENDS} - ) - - list(APPEND TILEDB_EXTERNAL_PROJECTS ep_azure_core) - list(APPEND FORWARD_EP_CMAKE_ARGS - -DTILEDB_AZURE_CORE_EP_BUILT=TRUE - ) - else () - message(FATAL_ERROR "Could not find ep_azure_core (required).") - endif () -endif () - -if (AZURECORE_FOUND AND NOT TARGET Azure::azure-core) - add_library(Azure::azure-core UNKNOWN IMPORTED) - set_target_properties(Azure::azure-core PROPERTIES - IMPORTED_LOCATION "${AZURECORE_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${AZURECORE_INCLUDE_DIR}" - ) -endif() diff --git a/cmake/Modules/FindAzureStorageBlobs_EP.cmake b/cmake/Modules/FindAzureStorageBlobs_EP.cmake deleted file mode 100644 index fa1fb8677c8d..000000000000 --- a/cmake/Modules/FindAzureStorageBlobs_EP.cmake +++ /dev/null @@ -1,136 +0,0 @@ -# -# FindAzureStorageBlobs_EP.cmake -# -# -# The MIT License -# -# Copyright (c) 2018-2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# This module finds the Azure Storage Blobs SDK, installing it with an ExternalProject if -# necessary. It then defines the imported by target Azure_Storage_Blobs::Azure_Storage_Blobs. - -# Include some common helper functions. -include(TileDBCommon) - -# Azure dependencies -if (TILEDB_VCPKG) - return() -endif() - -############################################################################### -# Start superbuild/unmanaged/legacy version -############################################################################### - -find_library(AZURE_STORAGE_BLOBS_LIBRARIES - NAMES - libazure-storage-blobs${CMAKE_STATIC_LIBRARY_SUFFIX} - azure-storage-blobs${CMAKE_STATIC_LIBRARY_SUFFIX} - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES lib lib64 - NO_DEFAULT_PATH - ) - -#on win32... '.lib' also used for lib ref'ing .dll!!! -#So, if perchance, in some environments, a .lib existed along with its corresponding .dll, -#then we could be incorrectly assuming/attempting a static build and probably will fail to -#appropriately include/package the .dll, since the code is (incorrectly) assuming .lib is indicative of a -#static library being used. - -if (AZURE_STORAGE_BLOBS_LIBRARIES) - set(AZURE_STORAGE_BLOBS_STATIC_EP_FOUND TRUE) - find_path(AZURE_STORAGE_BLOBS_INCLUDE_DIR - NAMES azure/storage/blobs.hpp - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES include - NO_DEFAULT_PATH - ) -elseif(NOT TILEDB_FORCE_ALL_DEPS) - set(AZURE_STORAGE_BLOBS_STATIC_EP_FOUND FALSE) - # Static EP not found, search in system paths. - find_library(AZURE_STORAGE_BLOBS_LIBRARIES - NAMES - libazure-storage-blobs #*nix name - azure-storage-blobs #windows name - PATH_SUFFIXES lib64 lib bin - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) - find_path(AZURE_STORAGE_BLOBS_INCLUDE_DIR - NAMES azure/storage/blobs.hpp - PATH_SUFFIXES include - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Azure_Storage_Blobs - REQUIRED_VARS AZURE_STORAGE_BLOBS_LIBRARIES AZURE_STORAGE_BLOBS_INCLUDE_DIR - ) - - -if (NOT AZURE_STORAGE_BLOBS_FOUND) - if (TILEDB_SUPERBUILD) - message(STATUS "Could NOT find azure-storage-blobs") - message(STATUS "Adding azure-storage-blobs as an external project") - - set(DEPENDS ep_azure_storage_common) - - ExternalProject_Add(ep_azure_storage_blobs - PREFIX "externals" - URL "https://github.com/Azure/azure-sdk-for-cpp/archive/azure-storage-blobs_12.6.0.zip" - URL_HASH SHA1=4d033a68b74f486284542528e558509c8624f21d - DOWNLOAD_NAME azure-storage-blobs_12.6.0.zip - SOURCE_SUBDIR sdk/storage/azure-storage-blobs - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=$ - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_PREFIX_PATH=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_INSTALL_PREFIX=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - -DWARNINGS_AS_ERRORS=OFF - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON - LOG_DOWNLOAD TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE ${TILEDB_LOG_OUTPUT_ON_FAILURE} - DEPENDS ${DEPENDS} - ) - - list(APPEND TILEDB_EXTERNAL_PROJECTS ep_azure_storage_blobs) - list(APPEND FORWARD_EP_CMAKE_ARGS - -DTILEDB_AZURE_STORAGE_BLOBS_EP_BUILT=TRUE - ) - else () - message(FATAL_ERROR "Could not find ep_azure_storage_blobs (required).") - endif () -endif () - -if (AZURE_STORAGE_BLOBS_FOUND AND NOT TARGET Azure::azure-storage-blobs) - add_library(Azure::azure-storage-blobs UNKNOWN IMPORTED) - set_target_properties(Azure::azure-storage-blobs PROPERTIES - IMPORTED_LOCATION "${AZURE_STORAGE_BLOBS_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${AZURE_STORAGE_BLOBS_INCLUDE_DIR}" - ) -endif() diff --git a/cmake/Modules/FindAzureStorageCommon_EP.cmake b/cmake/Modules/FindAzureStorageCommon_EP.cmake deleted file mode 100644 index d750c623e8a3..000000000000 --- a/cmake/Modules/FindAzureStorageCommon_EP.cmake +++ /dev/null @@ -1,142 +0,0 @@ -# -# FindAzureStorageCommon_EP.cmake -# -# -# The MIT License -# -# Copyright (c) 2018-2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# This module finds the Azure Storage Common SDK, installing it with an ExternalProject if -# necessary. It then defines the imported by target Azure_Storage_Common::Azure_Storage_Common. - -# Include some common helper functions. -include(TileDBCommon) - -# Azure dependencies -if (TILEDB_VCPKG) - return() -endif() - -############################################################################### -# Start superbuild/unmanaged/legacy version -############################################################################### - -find_library(AZURE_STORAGE_COMMON_LIBRARIES - NAMES - libazure-storage-common${CMAKE_STATIC_LIBRARY_SUFFIX} - azure-storage-common${CMAKE_STATIC_LIBRARY_SUFFIX} - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES lib lib64 - NO_DEFAULT_PATH - ) - -#on win32... '.lib' also used for lib ref'ing .dll!!! -#So, if perchance, in some environments, a .lib existed along with its corresponding .dll, -#then we could be incorrectly assuming/attempting a static build and probably will fail to -#appropriately include/package the .dll, since the code is (incorrectly) assuming .lib is indicative of a -#static library being used. - -if (AZURE_STORAGE_COMMON_LIBRARIES) - set(AZURE_STORAGE_COMMON_STATIC_EP_FOUND TRUE) - find_path(AZURE_STORAGE_COMMON_INCLUDE_DIR - NAMES azure/storage/common/storage_common.hpp - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - PATH_SUFFIXES include - NO_DEFAULT_PATH - ) -elseif(NOT TILEDB_FORCE_ALL_DEPS) - set(AZURE_STORAGE_COMMON_STATIC_EP_FOUND FALSE) - # Static EP not found, search in system paths. - find_library(AZURE_STORAGE_COMMON_LIBRARIES - NAMES - libazure-storage-common #*nix name - azure-storage-common #windows name - PATH_SUFFIXES lib64 lib bin - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) - find_path(AZURE_STORAGE_COMMON_INCLUDE_DIR - NAMES azure/storage/common/storage_common.hpp - PATH_SUFFIXES include - ${TILEDB_DEPS_NO_DEFAULT_PATH} - ) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Azure_Storage_Common - REQUIRED_VARS AZURE_STORAGE_COMMON_LIBRARIES AZURE_STORAGE_COMMON_INCLUDE_DIR - ) - - -if (NOT AZURE_STORAGE_COMMON_FOUND) - if (TILEDB_SUPERBUILD) - message(STATUS "Could NOT find azure-storage-common") - message(STATUS "Adding azure-storage-common as an external project") - - set(DEPENDS ep_azure_core) - if (TARGET ep_openssl) - list(APPEND DEPENDS ep_openssl) - endif() - - ExternalProject_Add(ep_azure_storage_common - PREFIX "externals" - URL "https://github.com/Azure/azure-sdk-for-cpp/archive/azure-storage-common_12.3.2.zip" - URL_HASH SHA1=09fc97b3f4c4f8e8976704dfc1ecefd14b2ed1bc - DOWNLOAD_NAME azure-storage-common_12.3.2.zip - SOURCE_SUBDIR sdk/storage/azure-storage-common - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=$ - -DBUILD_SHARED_LIBS=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_PREFIX_PATH=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_INSTALL_PREFIX=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - -DWARNINGS_AS_ERRORS=OFF - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON - LOG_DOWNLOAD TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE ${TILEDB_LOG_OUTPUT_ON_FAILURE} - DEPENDS ${DEPENDS} - ) - - list(APPEND TILEDB_EXTERNAL_PROJECTS ep_azure_storage_common) - list(APPEND FORWARD_EP_CMAKE_ARGS - -DTILEDB_AZURE_STORAGE_COMMON_EP_BUILT=TRUE - ) - else () - message(FATAL_ERROR "Could not find ep_azure_storage_common (required).") - endif () -endif () - -if (AZURE_STORAGE_COMMON_FOUND AND NOT TARGET Azure::azure-storage-common) - add_library(Azure::azure-storage-common UNKNOWN IMPORTED) - set_target_properties(Azure::azure-storage-common PROPERTIES - IMPORTED_LOCATION "${AZURE_STORAGE_COMMON_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${AZURE_STORAGE_COMMON_INCLUDE_DIR}" - ) - if (WIN32) - target_link_libraries(Azure::azure-storage-common INTERFACE WebServices) - endif() -endif() diff --git a/cmake/Modules/FindWIL_EP.cmake b/cmake/Modules/FindWIL_EP.cmake deleted file mode 100644 index 7f2f301a0ee2..000000000000 --- a/cmake/Modules/FindWIL_EP.cmake +++ /dev/null @@ -1,95 +0,0 @@ -# -# FindWIL_EP.cmake -# -# -# The MIT License -# -# Copyright (c) 2018-2021 TileDB, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -# This module finds the Windows Implementation Libraries, installing it with an ExternalProject if -# necessary. It then defines the imported by target wil::wil. - -# Include some common helper functions. -include(TileDBCommon) - -# Azure dependencies -if (TILEDB_VCPKG) - return() -endif() - -############################################################################### -# Start superbuild/unmanaged/legacy version -############################################################################### - -# First check for a static version in the EP prefix. -find_package(wil - PATHS "${TILEDB_EP_INSTALL_PREFIX}" - ${TILEDB_DEPS_NO_DEFAULT_PATH} -) - -if (NOT wil_FOUND) - if (TILEDB_SUPERBUILD) - message(STATUS "Could NOT find wil") - message(STATUS "Adding wil as an external project") - - ExternalProject_Add(ep_wil - PREFIX "externals" - URL "https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip" - URL_HASH SHA1=e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5 - DOWNLOAD_NAME wil-v1.0.230629.1.zip - CMAKE_ARGS - -DCMAKE_BUILD_TYPE=$ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_PREFIX_PATH=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_INSTALL_PREFIX=${TILEDB_EP_INSTALL_PREFIX} - -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - "-DCMAKE_C_FLAGS=${CFLAGS_DEF}" - "-DCMAKE_CXX_FLAGS=${CXXFLAGS_DEF}" - -DWIL_BUILD_PACKAGING=OFF - -DWIL_BUILD_TESTS=OFF - LOG_DOWNLOAD TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE ${TILEDB_LOG_OUTPUT_ON_FAILURE} - DEPENDS ${DEPENDS} - ) - - list(APPEND TILEDB_EXTERNAL_PROJECTS ep_wil) - list(APPEND FORWARD_EP_CMAKE_ARGS - -DTILEDB_wil_EP_BUILT=TRUE - ) - else () - message(FATAL_ERROR "Could not find ep_wil (required).") - endif () -endif () - - -if (wil_FOUND AND NOT TARGET wil::wil) - add_library(wil::wil UNKNOWN IMPORTED) - set_target_properties(wil::wil PROPERTIES - IMPORTED_LOCATION "${wil_LIBRARIES}" - INTERFACE_INCLUDE_DIRECTORIES "${wil_INCLUDE_DIR}" - ) -endif() diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index cd1b81f9307d..f963294790c8 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -129,15 +129,6 @@ if (TILEDB_S3 AND NOT TILEDB_VCPKG) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindAWSSDK_EP.cmake) endif() -if (TILEDB_AZURE AND NOT TILEDB_VCPKG) - if (WIN32) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindWIL_EP.cmake) - endif() - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindAzureCore_EP.cmake) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindAzureStorageCommon_EP.cmake) - include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindAzureStorageBlobs_EP.cmake) -endif() - if (TILEDB_TESTS) include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindCatch_EP.cmake) endif() diff --git a/cmake/inputs/Config.cmake.in b/cmake/inputs/Config.cmake.in index 6723ff3b036a..df94a0f00a1f 100644 --- a/cmake/inputs/Config.cmake.in +++ b/cmake/inputs/Config.cmake.in @@ -27,6 +27,7 @@ if(NOT @BUILD_SHARED_LIBS@) # NOT BUILD_SHARED_LIBS find_dependency(AWSSDK COMPONENTS @AWS_SERVICES@) endif() if(@TILEDB_AZURE@) # TILEDB_AZURE + find_dependency(azure-identity-cpp) find_dependency(azure-storage-blobs-cpp) endif() if(@TILEDB_GCS@) # TILEDB_GCS diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index b7bc62f6f7b0..21807023f7a8 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -538,31 +538,13 @@ endif() if (TILEDB_AZURE) message(STATUS "The TileDB library is compiled with Azure support.") - if (TILEDB_VCPKG) - find_package(azure-storage-blobs-cpp CONFIG REQUIRED) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - Azure::azure-storage-blobs - ) - else() - find_package(AzureCore_EP REQUIRED) - find_package(AzureStorageCommon_EP REQUIRED) - find_package(AzureStorageBlobs_EP REQUIRED) - if (NOT WIN32) - if (NOT TARGET LibXml2::LibXml2) - find_package(LibXml2 REQUIRED) - endif() - endif() - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB - INTERFACE - Azure::azure-storage-blobs - Azure::azure-storage-common - Azure::azure-core) - # Link xml after Azure to allow symbol resolution on all platforms - if (NOT WIN32) - target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE LibXml2::LibXml2) - endif() - endif() + find_package(azure-identity-cpp CONFIG REQUIRED) + find_package(azure-storage-blobs-cpp CONFIG REQUIRED) + target_link_libraries(TILEDB_CORE_OBJECTS_ILIB + INTERFACE + Azure::azure-identity + Azure::azure-storage-blobs + ) endif() if (TILEDB_GCS) diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 4da5bfc33d66..8b8f2d882452 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -35,6 +35,7 @@ #include #include +#include #include #include "tiledb/common/common.h" @@ -112,6 +113,17 @@ std::string get_blob_endpoint( return result; } +/** + * Check if config has a SAS token set + * @param config Configuration parameters. + * @return whether there is a SAS token in the config + */ +static bool has_sas_token(const Config& config) { + std::string sas_token = get_config_with_env_fallback( + config, "vfs.azure.storage_sas_token", "AZURE_STORAGE_SAS_TOKEN"); + return !sas_token.empty(); +} + AzureParameters::AzureParameters(const Config& config) : max_parallel_ops_( config.get("vfs.azure.max_parallel_ops", Config::must_find)) @@ -131,7 +143,8 @@ AzureParameters::AzureParameters(const Config& config) , account_key_(get_config_with_env_fallback( config, "vfs.azure.storage_account_key", "AZURE_STORAGE_KEY")) , blob_endpoint_(get_blob_endpoint(config, account_name_)) - , ssl_cfg_(config) { + , ssl_cfg_(config) + , has_sas_token_(has_sas_token(config)) { } Status Azure::init(const Config& config, ThreadPool* const thread_pool) { @@ -174,33 +187,77 @@ Azure::AzureClientSingleton::get(const AzureParameters& params) { std::lock_guard lck(client_init_mtx_); - if (!client_) { - ::Azure::Storage::Blobs::BlobClientOptions options; - options.Retry.MaxRetries = params.max_retries_; - options.Retry.RetryDelay = params.retry_delay_; - options.Retry.MaxRetryDelay = params.max_retry_delay_; - - options.Transport.Transport = create_transport(params.ssl_cfg_); - - // Construct the Azure SDK blob service client. - // We pass a shared key if it was specified. - if (!params.account_key_.empty()) { - client_ = - tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( - ::Azure::Storage::Blobs::BlobServiceClient, - params.blob_endpoint_, - make_shared<::Azure::Storage::StorageSharedKeyCredential>( - HERE(), params.account_name_, params.account_key_), - options)); - } else { + if (client_) { + return *client_; + } + + ::Azure::Storage::Blobs::BlobClientOptions options; + options.Retry.MaxRetries = params.max_retries_; + options.Retry.RetryDelay = params.retry_delay_; + options.Retry.MaxRetryDelay = params.max_retry_delay_; + options.Transport.Transport = create_transport(params.ssl_cfg_); + + // Construct the Azure SDK blob service client. + // We pass a shared key if it was specified. + if (!params.account_key_.empty()) { + client_ = + tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( + ::Azure::Storage::Blobs::BlobServiceClient, + params.blob_endpoint_, + make_shared<::Azure::Storage::StorageSharedKeyCredential>( + HERE(), params.account_name_, params.account_key_), + options)); + return *client_; + } + + // Otherwise, if we did not specify an SAS token + // and we are connecting to an HTTPS endpoint, + // use ChainedTokenCredential to authenticate using Microsoft Entra ID. + if (!params.has_sas_token_ && + utils::parse::starts_with(params.blob_endpoint_, "https://")) { + try { + ::Azure::Core::Credentials::TokenCredentialOptions cred_options; + cred_options.Retry = options.Retry; + cred_options.Transport = options.Transport; + auto credential = make_shared<::Azure::Identity::ChainedTokenCredential>( + HERE(), + std::vector< + std::shared_ptr<::Azure::Core::Credentials::TokenCredential>>{ + make_shared<::Azure::Identity::EnvironmentCredential>( + HERE(), cred_options), + make_shared<::Azure::Identity::AzureCliCredential>( + HERE(), cred_options), + make_shared<::Azure::Identity::ManagedIdentityCredential>( + HERE(), cred_options), + make_shared<::Azure::Identity::WorkloadIdentityCredential>( + HERE(), cred_options)}); + // If a token is not available we wouldn't know it until we make a + // request and it would be too late. Try getting a token, and if it + // fails fall back to anonymous authentication. + ::Azure::Core::Credentials::TokenRequestContext tokenContext; + + // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-storage-blobs_12.7.0/sdk/storage/azure-storage-blobs/src/blob_service_client.cpp#L84 + tokenContext.Scopes.emplace_back("https://storage.azure.com/.default"); + std::ignore = credential->GetToken(tokenContext, {}); client_ = tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( ::Azure::Storage::Blobs::BlobServiceClient, params.blob_endpoint_, + credential, options)); + return *client_; + } catch (...) { + LOG_INFO( + "Failed to get Microsoft Entra ID token, falling back to anonymous " + "authentication"); } } + client_ = tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( + ::Azure::Storage::Blobs::BlobServiceClient, + params.blob_endpoint_, + options)); + return *client_; } @@ -964,8 +1021,8 @@ Status Azure::write_blocks( } state = &state_iter->second; - // We're done reading and writing from 'block_list_upload_states_'. Mutating - // the 'state' element does not affect the thread-safety of + // We're done reading and writing from 'block_list_upload_states_'. + // Mutating the 'state' element does not affect the thread-safety of // 'block_list_upload_states_'. } diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index 3d5ca37e6a7d..fdd059947b95 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -104,6 +104,9 @@ struct AzureParameters { /** SSL configuration. */ SSLConfig ssl_cfg_; + + /** Whether the config specifies a SAS token. */ + bool has_sas_token_; }; class Azure { diff --git a/vcpkg.json b/vcpkg.json index c8b9831f013e..19ed762ab250 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -17,6 +17,7 @@ "description": "Support Azure Blob Storage", "dependencies": [ "azure-storage-blobs-cpp", + "azure-identity-cpp", { "name": "libxml2", "default-features": false, From b75488186771fdbe8cc01c89577664f281b3cbe3 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:04:15 -0400 Subject: [PATCH 257/456] Remove mayfail from temporal regression test. (#4825) Remove `[!mayfail]` from the temporal regression test. --- TYPE: NO_HISTORY DESC: Remove mayfail from sub-millisecond temporal regression test. --- test/src/unit-capi-metadata.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit-capi-metadata.cc b/test/src/unit-capi-metadata.cc index f3d178a8b3da..1503864034a1 100644 --- a/test/src/unit-capi-metadata.cc +++ b/test/src/unit-capi-metadata.cc @@ -335,7 +335,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( CMetadataFx, "C API: Metadata, sub-millisecond writes", - "[!mayfail][capi][metadata][sub-millisecond]") { + "[capi][metadata][sub-millisecond]") { int32_t one = 1; int32_t two = 2; const void* v_r = nullptr; From 202930837f8b1ff57fcee954ca0f08009a30858c Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:54:30 -0400 Subject: [PATCH 258/456] Add CPP byte-type datatype checks. (#4826) Fix `datatype_is_byte` to appropriately validate `std::byte` Datatypes. Also add a C++ byte-type check and add the byte case to `typeError`. ---- TYPE: IMPROVEMENT DESC: Add CPP byte-type checks. --- test/src/unit-cppapi-type.cc | 11 +++++++++-- test/src/unit-enum-helpers.cc | 8 ++++++++ tiledb/sm/cpp_api/exception.h | 8 +++++++- tiledb/sm/cpp_api/type.h | 13 ++++++++++++- tiledb/sm/enums/datatype.h | 2 +- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/test/src/unit-cppapi-type.cc b/test/src/unit-cppapi-type.cc index cd2d388ba4be..880a8c9cd71f 100644 --- a/test/src/unit-cppapi-type.cc +++ b/test/src/unit-cppapi-type.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,8 @@ #include #include "tiledb/sm/cpp_api/tiledb" +using namespace tiledb; + struct MyData { int a; float b; @@ -40,7 +42,6 @@ struct MyData { }; TEST_CASE("C++ API: Types", "[cppapi][types]") { - using namespace tiledb; Context ctx; CHECK( @@ -123,6 +124,12 @@ TEST_CASE("C++ API: Types", "[cppapi][types]") { CHECK_NOTHROW(impl::type_check>( TILEDB_STRING_ASCII, TILEDB_VAR_NUM)); + // Validate byte datatypes + CHECK_NOTHROW(impl::type_check(TILEDB_BLOB)); + CHECK_NOTHROW(impl::type_check(TILEDB_GEOM_WKB)); + CHECK_NOTHROW(impl::type_check(TILEDB_GEOM_WKT)); + CHECK_THROWS(impl::type_check(TILEDB_BOOL)); + auto a1 = Attribute::create(ctx, "a1"); auto a2 = Attribute::create(ctx, "a2"); auto a3 = Attribute::create(ctx, "a3"); diff --git a/test/src/unit-enum-helpers.cc b/test/src/unit-enum-helpers.cc index 1ff811d35ab7..31402c46b7e7 100644 --- a/test/src/unit-enum-helpers.cc +++ b/test/src/unit-enum-helpers.cc @@ -63,3 +63,11 @@ TEST_CASE( REQUIRE_THROWS(datatype_max_integral_value(Datatype::FLOAT64)); REQUIRE_THROWS(datatype_max_integral_value(Datatype::STRING_ASCII)); } + +TEST_CASE("Test datatype_is_byte", "[enums][datatype][datatype_is_byte]") { + auto datatype = + GENERATE(Datatype::BLOB, Datatype::GEOM_WKB, Datatype::GEOM_WKT); + + CHECK(datatype_is_byte(datatype)); + CHECK(!datatype_is_byte(Datatype::BOOL)); +} diff --git a/tiledb/sm/cpp_api/exception.h b/tiledb/sm/cpp_api/exception.h index 7edd776ceccb..7d10125365fd 100644 --- a/tiledb/sm/cpp_api/exception.h +++ b/tiledb/sm/cpp_api/exception.h @@ -7,7 +7,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -90,6 +90,12 @@ inline void type_check(tiledb_datatype_t type, unsigned num = 0) { "type (" + impl::type_to_str(type) + ")"); } + } else if (tiledb_byte_type(type)) { + if (!std::is_same::value) { + throw TypeError( + "Static type does not match expected container type std::byte for " + "tiledb byte type"); + } } else if (tiledb_datetime_type(type)) { if (!std::is_same::value) { throw TypeError( diff --git a/tiledb/sm/cpp_api/type.h b/tiledb/sm/cpp_api/type.h index db7331bf5c3f..f8efb63cef75 100644 --- a/tiledb/sm/cpp_api/type.h +++ b/tiledb/sm/cpp_api/type.h @@ -7,7 +7,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -294,6 +294,17 @@ inline bool tiledb_string_type(tiledb_datatype_t type) { } } +inline bool tiledb_byte_type(tiledb_datatype_t type) { + switch (type) { + case TILEDB_BLOB: + case TILEDB_GEOM_WKB: + case TILEDB_GEOM_WKT: + return true; + default: + return false; + } +} + inline bool tiledb_datetime_type(tiledb_datatype_t type) { switch (type) { case TILEDB_DATETIME_YEAR: diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index 7b84f00f18e9..897766241da6 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -410,7 +410,7 @@ inline bool datatype_is_time(Datatype type) { /** Returns true if the input datatype is a std::byte type. */ inline bool datatype_is_byte(Datatype type) { return ( - type == Datatype::BOOL || type == Datatype::GEOM_WKB || + type == Datatype::BLOB || type == Datatype::GEOM_WKB || type == Datatype::GEOM_WKT); } From 93a1d62891249740c4af18c9c64bec7bd4be82c6 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:24:11 -0400 Subject: [PATCH 259/456] Add v2_21_1 arrays to the backward compatibility matrix. (#4828) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index d7bfa7a884eb..51c0670bb4b1 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From d48fea6049a84daffd4e3413588564ca590ef735 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 25 Mar 2024 15:56:12 -0700 Subject: [PATCH 260/456] Add new and delete to DataBlock so it can be used with std::allocator (#4075) This PR changes the notion of `DataBlock` slightly. We now consider the `DataBlock` to be a single thing that is allocated. According, we no longer consider `PoolAllocator` as an actual allocator. Rather, we overload `new` and `delete` in `DataBlockImpl` so that `DataBlock`s can be used directly with `std::allocator`. --- TYPE: NO_HISTORY DESC: Add new and delete to DataBlock so it can be used with std::allocator --------- Co-authored-by: Luc Rancourt --- .../tiledb/common/dag/data_block/data_block.h | 62 ++++------ .../common/dag/data_block/pool_allocator.h | 13 +- .../dag/data_block/test/unit_data_block.cc | 113 ++++++++++++++---- .../data_block/test/unit_pool_allocator.cc | 13 -- 4 files changed, 117 insertions(+), 84 deletions(-) diff --git a/experimental/tiledb/common/dag/data_block/data_block.h b/experimental/tiledb/common/dag/data_block/data_block.h index d76b9e87dee0..4ad3d4039fe2 100644 --- a/experimental/tiledb/common/dag/data_block/data_block.h +++ b/experimental/tiledb/common/dag/data_block/data_block.h @@ -50,9 +50,6 @@ namespace tiledb::common { /** * The fixed size (in bytes) of each `DataBlock`. */ -namespace { -constexpr static const size_t chunk_size_ = 4'194'304; // 4M -} /** * A fixed size block, an untyped carrier (of `std::byte`) for data to @@ -76,15 +73,16 @@ constexpr static const size_t chunk_size_ = 4'194'304; // 4M * The `DataBlockImpl` class provides the interface for `DataBlock`, but is also * parameterized by`Allocator`. */ -template +template class DataBlockImpl { - static Allocator allocator_; + constexpr static const size_t chunk_size_ = chunk_size; /** * Type aliases for `DataBlock` storage. The underlying storage for the * `DataBlock` is a chunk of `std::byte`. The `DataBlock` maintains this * storage as a `shared_ptr`. */ + using allocator_t = PoolAllocator; using storage_t = std::shared_ptr; using span_t = tcb::span; using pointer_t = std::byte*; @@ -101,23 +99,21 @@ class DataBlockImpl { * The actual stored chunk. The `storage_` is the `shared_ptr` to the chunk, * while `data_` is a pointer to the actual bytes. */ + storage_t storage_; pointer_t data_{nullptr}; public: - /** - * Constructor used if the `Allocator` is `std::allocator` of `std::byte`. - */ - template - explicit DataBlockImpl( - size_t init_size = 0UL, - typename std::enable_if< - std::is_same_v>>::type* = 0) - : capacity_{chunk_size_} - , size_{init_size} - , storage_{allocator_.allocate(chunk_size_), - [&](auto px) { allocator_.deallocate(px, chunk_size_); }} - , data_{storage_.get()} { + // @todo: Do some initialization here. + // @todo: Default initialize? + // Although there is a "size" argument, we don't actually do anything with it. + // We always return a single datablock of size chunk_size_. + void* operator new(size_t) { + return reinterpret_cast(allocator_t{}.allocate()); + } + + void operator delete(void* ptr) { + allocator_t{}.deallocate(reinterpret_cast(ptr)); } /** @@ -126,15 +122,11 @@ class DataBlockImpl { * `shared_ptr` constructor. It is expected that his deleter will simply * return the chunk to the pool. See pool_allocator.h for details. */ - template - explicit DataBlockImpl( - size_t init_size = 0UL, - typename std::enable_if< - std::is_same_v>>::type* = nullptr) - : capacity_{chunk_size_} + DataBlockImpl(size_t init_size = 0UL) + : capacity_{chunk_size_} , size_{init_size} - , storage_{allocator_.allocate(), - [&](auto px) { allocator_.deallocate(px); }} + , storage_{allocator_t{}.allocate(), + [&](auto px) { allocator_t{}.deallocate(px); }} , data_{storage_.get()} { } @@ -161,7 +153,7 @@ class DataBlockImpl { using DataBlockConstIterator = span_t::iterator; using value_type = span_t::value_type; - using allocator_type = SingletonPoolAllocator; + using allocator_type = allocator_t; using size_type = std::size_t; using difference_type = std::ptrdiff_t; using reference = value_type&; @@ -279,21 +271,15 @@ class DataBlockImpl { } }; -template -Allocator DataBlockImpl::allocator_; - /** * The `DataBlock` class is the `DataBlockImpl` above, specialized to the * PoolAllocator with chunk size specified by our defined `chunk_size_`. */ -using DataBlock = DataBlockImpl>; - -/** - * Function for getting new `DataBlock`s - */ -[[maybe_unused]] static DataBlock make_data_block(size_t init_size) { - return DataBlock{init_size}; -} +template +class DataBlock : public DataBlockImpl { + using Base = DataBlockImpl; + using Base::Base; +}; } // namespace tiledb::common diff --git a/experimental/tiledb/common/dag/data_block/pool_allocator.h b/experimental/tiledb/common/dag/data_block/pool_allocator.h index d068cd3e6e97..07462d850380 100644 --- a/experimental/tiledb/common/dag/data_block/pool_allocator.h +++ b/experimental/tiledb/common/dag/data_block/pool_allocator.h @@ -209,7 +209,7 @@ class PoolAllocatorImpl { */ auto aligned_start{reinterpret_cast( reinterpret_cast(new_bytes + sizeof(pointer) + (align - 1)) & - reinterpret_cast(page_mask))}; + static_cast(page_mask))}; for (size_t i = 0; i < chunks_per_array; ++i) { push_chunk(aligned_start + i * chunk_size); @@ -403,17 +403,12 @@ template class PoolAllocator { public: using value_type = typename SingletonPoolAllocator::value_type; + using pointer_type = value_type*; PoolAllocator() = default; - value_type* allocate() { + pointer_type allocate() { return _allocator.allocate(); } - value_type* allocate(size_t) { - return _allocator.allocate(); - } - void deallocate(value_type* a) { - return _allocator.deallocate(a); - } - void deallocate(value_type* a, size_t) { + void deallocate(pointer_type a) { return _allocator.deallocate(a); } size_t num_instances() { diff --git a/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc b/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc index 5ddda5214729..35f70a2f51bf 100644 --- a/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc +++ b/experimental/tiledb/common/dag/data_block/test/unit_data_block.cc @@ -40,6 +40,8 @@ #include "unit_data_block.h" #include +#include +#include /** * @todo Use proper checks for preprocessor directives to selectively include @@ -56,11 +58,49 @@ #include "experimental/tiledb/common/dag/utility/range_join.h" using namespace tiledb::common; +TEST_CASE( + "DataBlock: Test create DataBlock with constructors", "[data_block]") { + [[maybe_unused]] auto da = DataBlock<4'096>{}; + CHECK(da.size() == 0); + CHECK(da.capacity() == 4096); +} + +TEST_CASE("DataBlock: Test create DataBlock new / delete", "[data_block]") { + auto da = new DataBlock<4'096>; + CHECK(da->size() == 0); + CHECK(da->capacity() == 4096); + delete da; +} + +TEST_CASE("DataBlock: Test create DataBlock with make_unique", "[data_block]") { + auto da = std::make_unique>(); + CHECK(da->size() == 0); + CHECK(da->capacity() == 4096); +} + +TEST_CASE("DataBlock: Test create DataBlock with make_shared", "[data_block]") { + auto da = std::make_shared>(); + CHECK(da->size() == 0); + CHECK(da->capacity() == 4096); +} + +TEST_CASE("DataBlock: Test with forward_list", "[data_block]") { + std::forward_list> slist; + slist.emplace_front(); + CHECK(slist.front().size() == 0); + CHECK(slist.front().capacity() == 4096); + + // This is okay since `DataBlock`s are shared ptrs to underlying memory. + // We don't need an array new. + std::vector> vec{4}; +} + /** * Test constructor */ TEST_CASE("DataBlock: Test create DataBlock", "[data_block]") { SECTION("Constructor compilation tests") { + using DataBlock = DataBlock<4'096>; DataBlock da; CHECK(da.size() == 0); CHECK(da.capacity() == DataBlock::max_size()); @@ -168,6 +208,9 @@ void db_test_1(const DB& db) { * Invoke simple API tests with a basic `DataBlock` */ TEST_CASE("DataBlock: Test API of variously sized DataBlock", "[data_block]") { + using DataBlock = DataBlock<4'096>; + size_t chunk_size_ = 4'096; + SECTION("Specific size constructed") { auto db = DataBlock{1}; db_test_0(db); @@ -205,27 +248,6 @@ TEST_CASE("DataBlock: Test API of variously sized DataBlock", "[data_block]") { } } -/** - * Invoke the simple tests with a `DataBlock` created with an `std::allocator`. - */ -TEST_CASE( - "DataBlock: Test create DataBlock with std::allocator", - "[data_block]") { - auto db = DataBlockImpl>{chunk_size_}; - CHECK(db.size() == chunk_size_); - CHECK(db.capacity() == DataBlock::max_size()); - - db_test_0(db); - db_test_1(db); - - auto dc = DataBlock{chunk_size_}; - CHECK(dc.size() == chunk_size_); - CHECK(dc.capacity() == DataBlock::max_size()); - - db_test_0(dc); - db_test_1(dc); -} - /** * Test iterating through a `DataBlock` */ @@ -251,7 +273,9 @@ void db_test_2(DB& db) { * `PoolAllocator`. */ TEST_CASE("DataBlock: Iterate through data_block", "[data_block]") { - auto db = DataBlockImpl>{}; + static constexpr size_t chunk_size_ = 4'096; + using DataBlock = DataBlock; + auto db = DataBlock{}; db_test_2(db); auto dc = DataBlock{chunk_size_}; db_test_2(dc); @@ -262,12 +286,13 @@ TEST_CASE("DataBlock: Iterate through data_block", "[data_block]") { * `std::allocator` and with our `PoolAllocator`. */ TEST_CASE("DataBlock: Iterate through 8 data_blocks", "[data_block]") { + using DataBlock = DataBlock<4'096>; for (size_t i = 0; i < 8; ++i) { - auto db = DataBlockImpl>{}; + auto db = DataBlock{}; db_test_2(db); } for (size_t i = 0; i < 8; ++i) { - auto db = DataBlock{chunk_size_}; + auto db = DataBlock{4'096}; db_test_2(db); } } @@ -276,6 +301,8 @@ TEST_CASE("DataBlock: Iterate through 8 data_blocks", "[data_block]") { * Verify some properties of `DataBlock`s */ TEST_CASE("DataBlock: Get span", "[data_block]") { + using DataBlock = DataBlock<4'194'304>; + auto a = DataBlock{}; auto b = DataBlock{}; auto c = DataBlock{}; @@ -324,6 +351,9 @@ TEST_CASE("DataBlock: Get span", "[data_block]") { * Test resizing. */ TEST_CASE("DataBlock: Test resize", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock<4'194'304>; + auto a = DataBlock{chunk_size_}; auto b = DataBlock{chunk_size_}; auto c = DataBlock{chunk_size_}; @@ -346,6 +376,8 @@ TEST_CASE("DataBlock: Test resize", "[data_block]") { } TEST_CASE("DataBlock: Test (shallow) copying", "[data_block]") { + using DataBlock = DataBlock<4'194'304>; + auto a = DataBlock{DataBlock::max_size()}; auto b = DataBlock{DataBlock::max_size()}; auto c = DataBlock{DataBlock::max_size()}; @@ -519,6 +551,8 @@ TEST_CASE( "[data_block]") { std::byte* x; std::byte* y; + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; { auto a = DataBlock{chunk_size_}; @@ -545,6 +579,9 @@ TEST_CASE( TEST_CASE( "DataBlock: Test DataBlock allocation/deallocation from pool, etc", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{chunk_size_}; auto b = DataBlock{chunk_size_}; auto c = DataBlock{chunk_size_}; @@ -584,6 +621,9 @@ TEST_CASE( TEST_CASE( "DataBlock: Test DataBlock allocation/deallocation on copying, etc", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{chunk_size_}; auto b = DataBlock{chunk_size_}; auto c = DataBlock{chunk_size_}; @@ -677,6 +717,9 @@ TEST_CASE( * `std::fill` algorithm. */ void test_std_fill(size_t test_size) { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{test_size}; auto b = DataBlock{test_size}; auto c = DataBlock{test_size}; @@ -749,6 +792,8 @@ void test_std_fill(size_t test_size) { * `std::fill` algorithm. */ TEST_CASE("DataBlock: Fill with std::fill", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + SECTION("chunk_size") { test_std_fill(chunk_size_); } @@ -764,6 +809,9 @@ TEST_CASE("DataBlock: Fill with std::fill", "[data_block]") { * Verify some properties of joined `DataBlock`s. */ void test_join(size_t test_size) { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{test_size}; auto b = DataBlock{test_size}; auto c = DataBlock{test_size}; @@ -824,6 +872,8 @@ void test_join(size_t test_size) { } TEST_CASE("DataBlock: Join data_blocks (join view)", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + SECTION("chunk_size") { test_join(chunk_size_); } @@ -840,6 +890,9 @@ TEST_CASE("DataBlock: Join data_blocks (join view)", "[data_block]") { * library algorithms. */ void test_join_std_fill(size_t test_size) { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{test_size}; auto b = DataBlock{test_size}; auto c = DataBlock{test_size}; @@ -876,6 +929,8 @@ void test_join_std_fill(size_t test_size) { } TEST_CASE("DataBlock: Join data_blocks std::fill", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + SECTION("chunk_size") { test_join_std_fill(chunk_size_); } @@ -891,6 +946,9 @@ TEST_CASE("DataBlock: Join data_blocks std::fill", "[data_block]") { * Test `operator[]` of joined `DataBlock`s. */ void test_join_operator_bracket(size_t test_size) { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{test_size}; auto b = DataBlock{test_size}; auto c = DataBlock{test_size}; @@ -933,6 +991,8 @@ void test_join_operator_bracket(size_t test_size) { } TEST_CASE("DataBlock: Join data_blocks operator[]", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + SECTION("chunk_size") { test_join_operator_bracket(chunk_size_); } @@ -948,6 +1008,9 @@ TEST_CASE("DataBlock: Join data_blocks operator[]", "[data_block]") { * Additional standard library uses of joined `DataBlock`s. */ void test_operator_bracket_loops(size_t test_size) { + constexpr size_t chunk_size_ = 4'194'304; + using DataBlock = DataBlock; + auto a = DataBlock{test_size}; auto b = DataBlock{test_size}; auto c = DataBlock{test_size}; @@ -1024,6 +1087,8 @@ void test_operator_bracket_loops(size_t test_size) { } TEST_CASE("DataBlock: Join data_blocks loops operator[]", "[data_block]") { + constexpr size_t chunk_size_ = 4'194'304; + SECTION("chunk_size") { test_operator_bracket_loops(chunk_size_); } diff --git a/experimental/tiledb/common/dag/data_block/test/unit_pool_allocator.cc b/experimental/tiledb/common/dag/data_block/test/unit_pool_allocator.cc index 16aabcd39b18..3d4980f624da 100644 --- a/experimental/tiledb/common/dag/data_block/test/unit_pool_allocator.cc +++ b/experimental/tiledb/common/dag/data_block/test/unit_pool_allocator.cc @@ -367,19 +367,6 @@ TEST_CASE( } } -/** - * Test allocator interface for `PoolAllocator`. It should be conformant, but - * g++ seems to be kind of persnickety. - */ -#ifndef __GNUG__ -TEST_CASE( - "Pool Allocator: Allocate vector with PoolAllocator - compile only", - "[PoolAllocator]") { - std::vector> v(10); - CHECK(v.size() == 10); -} -#endif - /** * Allocate a large number of chunks. */ From f1629d1426cd40f93397cc1047fd247d1723d8a3 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:16:24 +0100 Subject: [PATCH 261/456] Fix nightlies after datablock merge. (#4831) --- TYPE: NO_HISTORY DESC: Fix nightlies after datablock merge. --- .../data_block/test/unit_blocks_and_ports.cc | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc b/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc index 86681845b622..aa4c96c98f08 100644 --- a/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc +++ b/experimental/tiledb/common/dag/data_block/test/unit_blocks_and_ports.cc @@ -58,8 +58,8 @@ using NullMover2 = ItemMover; TEST_CASE( "BlocksAndPorts: Create Source and Sink with DataBlock", "[blocks_and_ports]") { - auto pn = Source{}; - auto cn = Sink{}; + auto pn = Source>{}; + auto cn = Sink>{}; attach(pn, cn); } @@ -69,11 +69,11 @@ TEST_CASE( */ TEST_CASE( "BlocksAndPorts: Manual set source port values", "[blocks_and_ports]") { - auto source = Source{}; - auto sink = Sink{}; + auto source = Source>{}; + auto sink = Sink>{}; - DataBlock x{DataBlock::max_size()}; - DataBlock y{DataBlock::max_size()}; + DataBlock<> x{DataBlock<>::max_size()}; + DataBlock<> y{DataBlock<>::max_size()}; SECTION("set source in bound pair") { attach(source, sink); @@ -90,7 +90,7 @@ TEST_CASE( } } -bool zeroize(DataBlock& x) { +bool zeroize(DataBlock<>& x) { for (auto& j : x) { j = std::byte{0}; } @@ -100,15 +100,15 @@ bool zeroize(DataBlock& x) { return e == x.end(); } -void iotize(DataBlock& x) { +void iotize(DataBlock<>& x) { std::iota( reinterpret_cast(x.begin()), reinterpret_cast(x.end()), uint8_t{0}); } -bool check_iotized(DataBlock& x) { - bool ret = [&](DataBlock& z) { +bool check_iotized(DataBlock<>& x) { + bool ret = [&](DataBlock<>& z) { uint8_t b{0}; for (auto&& j : z) { if (static_cast(j) != b) { @@ -124,8 +124,8 @@ bool check_iotized(DataBlock& x) { return ret; } -bool check_zeroized(DataBlock& x) { - bool ret = [&](DataBlock& z) { +bool check_zeroized(DataBlock<>& x) { + bool ret = [&](DataBlock<>& z) { uint8_t b{0}; for (auto&& j : z) { if (static_cast(j) != b) { @@ -144,18 +144,18 @@ bool check_zeroized(DataBlock& x) { * Test operation of inject and extract. */ TEST_CASE("BlocksAndPorts: Manual extract sink values", "[blocks_and_ports]") { - auto source = Source{}; - auto sink = Sink{}; + auto source = Source>{}; + auto sink = Sink>{}; - DataBlock x{DataBlock::max_size()}; - DataBlock y{DataBlock::max_size()}; + DataBlock<> x{DataBlock<>::max_size()}; + DataBlock<> y{DataBlock<>::max_size()}; SECTION("set source in unbound pair") { CHECK_THROWS(sink.extract().has_value() == false); } SECTION("set source in bound pair") { - CHECK(x.size() == DataBlock::max_size()); - CHECK(y.size() == DataBlock::max_size()); + CHECK(x.size() == DataBlock<>::max_size()); + CHECK(y.size() == DataBlock<>::max_size()); iotize(x); zeroize(y); check_iotized(x); @@ -180,11 +180,11 @@ TEST_CASE("BlocksAndPorts: Manual extract sink values", "[blocks_and_ports]") { TEST_CASE( "BlocksAndPorts: Manual transfer from Source to Sink", "[blocks_and_ports]") { - Source source; - Sink sink; + Source> source; + Sink> sink; - DataBlock x{DataBlock::max_size()}; - DataBlock y{DataBlock::max_size()}; + DataBlock<> x{DataBlock<>::max_size()}; + DataBlock<> y{DataBlock<>::max_size()}; [[maybe_unused]] auto dx = x.data(); [[maybe_unused]] auto dy = y.data(); @@ -268,11 +268,11 @@ TEST_CASE( TEST_CASE( "BlocksAndPorts: Manual transfer from Source to Sink, async policy", "[blocks_and_ports]") { - Source source; - Sink sink; + Source> source; + Sink> sink; - DataBlock x{DataBlock::max_size()}; - DataBlock y{DataBlock::max_size()}; + DataBlock<> x{DataBlock<>::max_size()}; + DataBlock<> y{DataBlock<>::max_size()}; [[maybe_unused]] auto dx = x.data(); [[maybe_unused]] auto dy = y.data(); @@ -312,11 +312,11 @@ TEST_CASE( TEST_CASE( "BlocksAndPorts: Async transfer from Source to Sink", "[blocks_and_ports]") { - Source source; - Sink sink; + Source> source; + Sink> sink; - DataBlock x{DataBlock::max_size()}; - DataBlock y{DataBlock::max_size()}; + DataBlock<> x{DataBlock<>::max_size()}; + DataBlock<> y{DataBlock<>::max_size()}; [[maybe_unused]] auto dx = x.data(); [[maybe_unused]] auto dy = y.data(); @@ -333,7 +333,7 @@ TEST_CASE( auto state_machine = sink.get_mover(); CHECK(str(state_machine->state()) == "st_00"); - std::optional b; + std::optional> b; auto source_node = [&]() { CHECK(source.inject(x) == true); @@ -389,8 +389,8 @@ TEST_CASE( TEST_CASE("BlocksAndPorts: Async pass n blocks", "[blocks_and_ports]") { [[maybe_unused]] constexpr bool debug = false; - Source source; - Sink sink; + Source> source; + Sink> sink; attach(source, sink); @@ -411,7 +411,7 @@ TEST_CASE("BlocksAndPorts: Async pass n blocks", "[blocks_and_ports]") { CHECK(std::equal(input.begin(), input.end(), output.begin()) == false); - auto p = PoolAllocator{}; + auto p = PoolAllocator::max_size()>{}; size_t init_allocations = p.num_allocations(); size_t max_allocated = 0; @@ -432,7 +432,7 @@ TEST_CASE("BlocksAndPorts: Async pass n blocks", "[blocks_and_ports]") { std::this_thread::sleep_for(std::chrono::microseconds(random_us(500))); - source.inject(make_data_block(*i++)); + source.inject(DataBlock<>{*i++}); max_allocated = std::max(max_allocated, p.num_allocated()); std::this_thread::sleep_for(std::chrono::microseconds(random_us(500))); From 04ba137ad9f33587c85ccf7530a266425776009d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 26 Mar 2024 18:06:04 +0100 Subject: [PATCH 262/456] Fix timestamp usage with random label. (#4830) Random label uses the current time to know if a new label needs to use an incrementing counter so that labels generated in the same millisecond sort properly when reading back. Since the code at the higher level used a timestamp and the random label used another, it was possible to end up with timestamped names having the same millisecond, but not sorting properly. The fix is to have the random label code return the timestamp it used to the caller so that the caller can use it to generate timestamped names. --- TYPE: NO_HISTORY DESC: Fix timestamp usage with random label. --- tiledb/common/random/random_label.cc | 21 ++++-------- tiledb/common/random/random_label.h | 39 +++++++++++++++++++++-- tiledb/sm/array/array.cc | 8 ++--- tiledb/sm/array/array.h | 8 ----- tiledb/sm/metadata/metadata.cc | 34 +++++++++----------- tiledb/sm/metadata/metadata.h | 12 ++----- tiledb/storage_format/uri/generate_uri.cc | 9 ++++-- 7 files changed, 69 insertions(+), 62 deletions(-) diff --git a/tiledb/common/random/random_label.cc b/tiledb/common/random/random_label.cc index ba04ae92bfca..d985d4fd6211 100644 --- a/tiledb/common/random/random_label.cc +++ b/tiledb/common/random/random_label.cc @@ -44,7 +44,7 @@ RandomLabelGenerator::RandomLabelGenerator() /* ********************************* */ /* API */ /* ********************************* */ -std::string RandomLabelGenerator::generate() { +RandomLabelWithTimestamp RandomLabelGenerator::generate() { PRNG& prng = PRNG::get(); std::lock_guard lock(mtx_); auto now = tiledb::sm::utils::time::timestamp_now_ms(); @@ -69,26 +69,19 @@ std::string RandomLabelGenerator::generate() { ss << std::hex << std::setw(8) << std::setfill('0') << static_cast(prng()); ss << std::hex << std::setw(16) << std::setfill('0') << prng(); - return ss.str(); + return {ss.str(), now}; } -std::string RandomLabelGenerator::generate_random_label() { +RandomLabelWithTimestamp RandomLabelGenerator::generate_random_label() { static RandomLabelGenerator generator; return generator.generate(); } -/** - * Wrapper function for `generate_random_label`, which returns a PRNG-generated - * label as a 32-digit hexadecimal random number. - * (Ex. f258d22d4db9139204eef2b4b5d860cc). - * - * @pre If multiple labels are generated within the same millisecond, they will - * be sorted using a counter on the most significant 4 bytes. - * @note Labels may be 0-padded to ensure exactly a 128-bit, 32-digit length. - * - * @return A random label. - */ std::string random_label() { + return RandomLabelGenerator::generate_random_label().random_label_; +} + +RandomLabelWithTimestamp random_label_with_timestamp() { return RandomLabelGenerator::generate_random_label(); } diff --git a/tiledb/common/random/random_label.h b/tiledb/common/random/random_label.h index 6675605f4350..3e74518f71c8 100644 --- a/tiledb/common/random/random_label.h +++ b/tiledb/common/random/random_label.h @@ -51,6 +51,26 @@ class RandomLabelException : public StatusException { } }; +/** Simple struct to return a label with timestamp. */ +struct RandomLabelWithTimestamp { + /** + * Construct a new random label with timestamp object + * + * @param random_label Random label. + * @param timestamp Timestamp the random label was created. + */ + RandomLabelWithTimestamp(std::string random_label, uint64_t timestamp) + : random_label_(random_label) + , timestamp_(timestamp) { + } + + /** Random label. */ + std::string random_label_; + + /** Timestamp the label was created. */ + uint64_t timestamp_; +}; + /** * Generates a pseudeo-random label, formatted as a 32-digit hexadecimal number. * (Ex. f258d22d4db9139204eef2b4b5d860cc). @@ -77,12 +97,12 @@ class RandomLabelGenerator { /* ********************************* */ /* API */ /* ********************************* */ - /** Generate a random label. */ - std::string generate(); + /** Generate a random label with a timestamp. */ + RandomLabelWithTimestamp generate(); public: /** Generate a random label. */ - static std::string generate_random_label(); + static RandomLabelWithTimestamp generate_random_label(); private: /* ********************************* */ @@ -112,6 +132,19 @@ class RandomLabelGenerator { */ std::string random_label(); +/** + * Wrapper function for `generate_random_label`, which returns a PRNG-generated + * label as a 32-digit hexadecimal random number. + * (Ex. f258d22d4db9139204eef2b4b5d860cc). + * + * @pre If multiple labels are generated within the same millisecond, they will + * be sorted using a counter on the most significant 4 bytes. + * @note Labels may be 0-padded to ensure exactly a 128-bit, 32-digit length. + * + * @return A random label with timestamp. + */ +RandomLabelWithTimestamp random_label_with_timestamp(); + } // namespace tiledb::common #endif // TILEDB_HELPERS_H diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index cf8e9bea8622..395be3468b8b 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -380,7 +380,7 @@ Status Array::open( set_array_schemas_all(std::move(array_schemas.value())); // Set the timestamp - opened_array_->metadata().reset(timestamp_for_new_component()); + opened_array_->metadata().reset(timestamp_end_opened_at()); } else if ( query_type == QueryType::DELETE || query_type == QueryType::UPDATE) { { @@ -421,7 +421,7 @@ Status Array::open( } // Updates the timestamp to use for metadata. - opened_array_->metadata().reset(timestamp_for_new_component()); + opened_array_->metadata().reset(timestamp_end_opened_at()); } else { throw ArrayException("Cannot open array; Unsupported query type."); } @@ -1137,10 +1137,6 @@ bool Array::use_refactored_array_open() const { return refactored_array_open || use_refactored_query_submit(); } -uint64_t Array::timestamp_for_new_component() const { - return new_component_timestamp_.value_or(utils::time::timestamp_now_ms()); -} - bool Array::use_refactored_query_submit() const { auto found = false; auto refactored_query_submit = false; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 380deef7e9c7..3b7286d709cc 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -585,14 +585,6 @@ class Array { new_component_timestamp_.value_or(0); } - /** - * Returns the timestamp to use when writing components (fragment, - * metadata, etc.) - * - * If set to use the lastest time, this will get the time when called. - */ - uint64_t timestamp_for_new_component() const; - /** Directly set the timestamp start value. */ inline void set_timestamp_start(uint64_t timestamp_start) { array_dir_timestamp_start_ = timestamp_start; diff --git a/tiledb/sm/metadata/metadata.cc b/tiledb/sm/metadata/metadata.cc index d5b585147e6a..b93c3a50ca81 100644 --- a/tiledb/sm/metadata/metadata.cc +++ b/tiledb/sm/metadata/metadata.cc @@ -60,12 +60,12 @@ Metadata::Metadata(shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , metadata_map_(memory_tracker_->get_resource(MemoryType::METADATA)) , metadata_index_(memory_tracker_->get_resource(MemoryType::METADATA)) - , timestamp_range_([]() -> std::pair { + , loaded_metadata_uris_(memory_tracker_->get_resource(MemoryType::METADATA)) + , timestamped_name_([]() -> std::string { auto t = utils::time::timestamp_now_ms(); - return std::make_pair(t, t); - }()) - , loaded_metadata_uris_( - memory_tracker_->get_resource(MemoryType::METADATA)) { + return tiledb::storage_format::generate_timestamped_name( + t, t, std::nullopt); + }()) { build_metadata_index(); } @@ -79,7 +79,7 @@ Metadata& Metadata::operator=(Metadata& other) { metadata_map_.emplace(k, v); } - timestamp_range_ = other.timestamp_range_; + timestamped_name_ = other.timestamped_name_; for (auto& uri : other.loaded_metadata_uris_) { loaded_metadata_uris_.emplace_back(uri); @@ -106,7 +106,6 @@ void Metadata::clear() { metadata_map_.clear(); metadata_index_.clear(); loaded_metadata_uris_.clear(); - timestamp_range_ = std::make_pair(0, 0); uri_ = URI(); } @@ -118,10 +117,8 @@ URI Metadata::get_uri(const URI& array_uri) { } void Metadata::generate_uri(const URI& array_uri) { - auto ts_name = tiledb::storage_format::generate_timestamped_name( - timestamp_range_.first, timestamp_range_.second, std::nullopt); uri_ = array_uri.join_path(constants::array_metadata_dir_name) - .join_path(ts_name); + .join_path(timestamped_name_); } std::map Metadata::deserialize( @@ -189,10 +186,6 @@ void Metadata::serialize(Serializer& serializer) const { } } -const std::pair& Metadata::timestamp_range() const { - return timestamp_range_; -} - void Metadata::del(const char* key) { assert(key != nullptr); @@ -317,8 +310,10 @@ void Metadata::set_loaded_metadata_uris( loaded_metadata_uris_.push_back(uri.uri_); } - timestamp_range_.first = loaded_metadata_uris.front().timestamp_range_.first; - timestamp_range_.second = loaded_metadata_uris.back().timestamp_range_.second; + timestamped_name_ = tiledb::storage_format::generate_timestamped_name( + loaded_metadata_uris.front().timestamp_range_.first, + loaded_metadata_uris.back().timestamp_range_.second, + std::nullopt); } const tdb::pmr::vector& Metadata::loaded_metadata_uris() const { @@ -327,14 +322,15 @@ const tdb::pmr::vector& Metadata::loaded_metadata_uris() const { void Metadata::reset(uint64_t timestamp) { clear(); - timestamp = (timestamp != 0) ? timestamp : utils::time::timestamp_now_ms(); - timestamp_range_ = std::make_pair(timestamp, timestamp); + timestamped_name_ = tiledb::storage_format::generate_timestamped_name( + timestamp, timestamp, std::nullopt); } void Metadata::reset( const uint64_t timestamp_start, const uint64_t timestamp_end) { clear(); - timestamp_range_ = std::make_pair(timestamp_start, timestamp_end); + timestamped_name_ = tiledb::storage_format::generate_timestamped_name( + timestamp_start, timestamp_end, std::nullopt); } Metadata::iterator Metadata::begin() const { diff --git a/tiledb/sm/metadata/metadata.h b/tiledb/sm/metadata/metadata.h index 2e45614eaea8..ec883dfdf514 100644 --- a/tiledb/sm/metadata/metadata.h +++ b/tiledb/sm/metadata/metadata.h @@ -136,9 +136,6 @@ class Metadata { /** Serializes all key-value metadata items into the input buffer. */ void serialize(Serializer& serializer) const; - /** Returns the timestamp range. */ - const std::pair& timestamp_range() const; - /** * Deletes a metadata item. * @@ -257,12 +254,6 @@ class Metadata { /** Mutex for thread-safety. */ mutable std::mutex mtx_; - /** - * The timestamp range covered by the metadata that was read or written. - * This is used to determine the metadata file name. - */ - std::pair timestamp_range_; - /** * The URIs of the metadata files that have been loaded to this object. * This is needed to know which files to delete upon consolidation. @@ -272,6 +263,9 @@ class Metadata { /** The URI of the array metadata file. */ URI uri_; + /** Timestamped name. */ + std::string timestamped_name_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/storage_format/uri/generate_uri.cc b/tiledb/storage_format/uri/generate_uri.cc index d823936c13d2..03c8ad2a7daa 100644 --- a/tiledb/storage_format/uri/generate_uri.cc +++ b/tiledb/storage_format/uri/generate_uri.cc @@ -48,9 +48,14 @@ std::string generate_timestamped_name( "start timestamp cannot be after end timestamp."); } + auto lbl = random_label_with_timestamp(); + if (timestamp_start == 0 && timestamp_end == 0) { + timestamp_start = timestamp_end = lbl.timestamp_; + } + std::stringstream ss; ss << "/__" << timestamp_start << "_" << timestamp_end << "_" - << random_label(); + << lbl.random_label_; if (version.has_value()) { ss << "_" << version.value(); @@ -61,8 +66,6 @@ std::string generate_timestamped_name( std::string generate_timestamped_name( uint64_t timestamp, format_version_t format_version) { - timestamp = - (timestamp != 0) ? timestamp : sm::utils::time::timestamp_now_ms(); return generate_timestamped_name(timestamp, timestamp, format_version); } From ea6a0e7112737c029a68c37c6f4a9efba6197c13 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Wed, 27 Mar 2024 17:23:35 +0200 Subject: [PATCH 263/456] Make porting tiledb_unit integration tests to REST CI easier (#4678) This PR groups some improvements on testing infra for REST tests: - Introduce class `VFSTestSetup` as an easy way to set up an integration test in `tiledb_unit` for all VFSs and REST. - Adapts existing REST-CI tests and adds some new tests to use that class. - Removes `--enable-rest-tests` bootstrap and corresponding `TILEDB_TESTS_ENABLE_REST` options. - Adds tags [rest] for `tiledb_unit` tests to run on REST CI, [non-rest] for tests that don't apply to REST, and [rest-fails] to tests that should be run on REST but fail because of issues logged here: https://app.shortcut.com/tiledb-inc/story/40489/issues-found-while-running-tiledb-unit-core-tests-against-rest-cloud-server Note: REST-CI runner needs to be adapted after this is merged, but it's in a separate repo if I understand well. Also an extra runner will be added to test Query v3 that will be setting `use_refactored_array_open_and_query_submit` config to `true` by default and run all [rest] tests. --- TYPE: NO_HISTORY DESC: Make porting tiledb_unit integration tests to REST CI easier --------- Co-authored-by: Luc Rancourt --- .github/workflows/full-ci.yml | 1 + bootstrap | 5 - bootstrap.ps1 | 11 +- cmake/Options/BuildOptions.cmake | 1 - cmake/TileDB-Superbuild.cmake | 2 - examples/cpp_api/CMakeLists.txt | 3 - test/CMakeLists.txt | 13 +- test/performance/CMakeLists.txt | 2 - test/src/unit-capi-any.cc | 93 +-- test/src/unit-capi-rest-dense_array.cc | 433 ++++------- test/src/unit-capi-vfs.cc | 8 +- test/src/unit-cppapi-array.cc | 683 +++++++----------- .../unit-cppapi-global-order-writes-remote.cc | 39 +- test/src/unit-rest-enumerations.cc | 67 +- test/src/unit.cc | 2 +- test/support/src/helpers.cc | 45 +- test/support/src/helpers.h | 3 +- test/support/src/vfs_helpers.cc | 21 +- test/support/src/vfs_helpers.h | 190 ++++- tiledb/sm/c_api/tiledb.cc | 22 +- tiledb/sm/cpp_api/config.h | 2 + tiledb/sm/storage_manager/storage_manager.cc | 20 +- .../storage_manager_canonical.h | 6 +- 23 files changed, 681 insertions(+), 991 deletions(-) diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 7ef0d811ce2d..e071dda3c65b 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -60,6 +60,7 @@ jobs: matrix_image: ubuntu-22.04 matrix_compiler_cc: 'gcc-13' matrix_compiler_cxx: 'g++-13' + timeout: 120 bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' ci5: diff --git a/bootstrap b/bootstrap index 627ca30b593e..6060194bf987 100755 --- a/bootstrap +++ b/bootstrap @@ -68,7 +68,6 @@ Configuration: --enable-ccache enables use of ccache (if present) --enable-arrow-tests enables the compilation of the arrow adapter unit tests --enable-experimental-features enables experimental TileDB features - --enable-rest-tests enables REST tests --enable-aws-s3-config enables AWS S3 configuration for tests --enable=arg1,arg2... same as "--enable-arg1 --enable-arg2 ..." @@ -115,7 +114,6 @@ tiledb_ccache="OFF" tiledb_arrow_tests="OFF" tiledb_experimental_features="OFF" tiledb_build_webp="ON" -tiledb_tests_enable_rest="OFF" tiledb_tests_aws_s3_config="OFF" enable_multiple="" while test $# != 0; do @@ -156,7 +154,6 @@ while test $# != 0; do --enable-ccache) tiledb_ccache="ON";; --enable-arrow-tests) tiledb_arrow_tests="ON";; --enable-experimental-features) tiledb_experimental_features="ON";; - --enable-rest-tests) tiledb_tests_enable_rest="ON";; --enable-aws-s3-config) tiledb_tests_aws_s3_config="ON";; --enable=*) s=`arg "$1"` enable_multiple+="${enable_multiple:+,}${s}";; @@ -188,7 +185,6 @@ for en in "${enables[@]}"; do static-tiledb) echo "Argument '--enable-static-tiledb' is obsolete and will be removed in a future version. Use --linkage=static instead." enable_static_tiledb="ON";; experimental-features) tiledb_experimental_features="ON";; - rest-tests) tiledb_tests_enable_rest="ON";; aws-s3-config) tiledb_tests_aws_s3_config="ON";; *) die "Unknown option: --enable-$en" ;; esac @@ -263,7 +259,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \ -DTILEDB_SANITIZER="${sanitizer}" \ -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \ - -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ ${tiledb_disable_avx2} \ ${vcpkg_base_triplet} \ diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 66fa18b6ba30..7b3a98f63ca5 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -86,9 +86,6 @@ Enables building TileDB Experimental features .PARAMETER EnableArrowTests Enables the compilation of the arrow adapter unit tests -.PARAMETER EnableRestTests -Enables REST unit tests - .PARAMETER EnableAwsS3Config Enables AWS S3 configuration for unit tests @@ -149,7 +146,6 @@ Param( [switch]$EnableExperimentalFeatures, [switch]$EnableBuildDeps, [switch]$EnableArrowTests, - [switch]$EnableRestTests, [switch]$EnableAwsS3Config, [switch]$DisableWebP, [switch]$DisableWerror, @@ -309,11 +305,6 @@ if ($EnableArrowTests.IsPresent) { $ArrowTests="ON" } -$RestTests="OFF" -if ($EnableRestTests.IsPresent) { - $RestTests="ON" -} - $ConfigureS3="OFF" if ($EnableAwsS3Config.IsPresent) { $ConfigureS3="ON" @@ -357,7 +348,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake $ArchFlag -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake $ArchFlag -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index c88c3f10ac0a..7baebff4d23c 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -34,7 +34,6 @@ option(TILEDB_LOG_OUTPUT_ON_FAILURE "If true, print error logs if dependency sub option(TILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK "If true, skip check needed path length for awssdk (TILEDB_S3) dependent builds" OFF) option(TILEDB_EXPERIMENTAL_FEATURES "If true, build and include experimental features" OFF) option(TILEDB_TESTS_AWS_S3_CONFIG "Use an S3 config appropriate for AWS in tests" OFF) -option(TILEDB_TESTS_ENABLE_REST "Enables REST tests (requires running REST server)" OFF) option(CMAKE_EXPORT_COMPILE_COMMANDS "cmake compile commands" ON) diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index f963294790c8..9f0d894536d9 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -58,7 +58,6 @@ set(INHERITED_CMAKE_ARGS -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DTILEDB_EXPERIMENTAL_FEATURES=${TILEDB_EXPERIMENTAL_FEATURES} -DTILEDB_TESTS_AWS_S3_CONFIG=${TILEDB_TESTS_AWS_S3_CONFIG} - -DTILEDB_TESTS_ENABLE_REST=${TILEDB_TESTS_ENABLE_REST} ) if (libxml2_DIR) @@ -70,7 +69,6 @@ endif() if (TILEDB_TESTS) list(APPEND INHERITED_CMAKE_ARGS -DTILEDB_TESTS_AWS_S3_CONFIG=${TILEDB_TESTS_AWS_S3_CONFIG} - -DTILEDB_TESTS_ENABLE_REST=${TILEDB_TESTS_ENABLE_REST} ) endif() diff --git a/examples/cpp_api/CMakeLists.txt b/examples/cpp_api/CMakeLists.txt index 0226f9f2b6e5..68a4e05c869e 100644 --- a/examples/cpp_api/CMakeLists.txt +++ b/examples/cpp_api/CMakeLists.txt @@ -37,9 +37,6 @@ function(build_TileDB_example_cppapi TARGET) # to libzstd or libcurl to work. target_link_libraries(${TARGET}_cpp PUBLIC pthread dl) endif() - if (TILEDB_TESTS_ENABLE_REST) - target_compile_definitions(${TARGET}_cpp PRIVATE -DTILEDB_TESTS_ENABLE_REST) - endif() add_dependencies(examples_cpp ${TARGET}_cpp) endfunction() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c725e1f444e9..17aafc177acf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -150,6 +150,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-capi-partial-attribute-write.cc src/unit-capi-query.cc src/unit-capi-query_2.cc + src/unit-capi-rest-dense_array.cc src/unit-capi-smoke-test.cc src/unit-capi-sparse_array.cc src/unit-capi-sparse_heter.cc @@ -186,6 +187,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-Reader.cc src/unit-request-handlers.cc src/unit-resource-pool.cc + src/unit-rest-enumerations.cc src/unit-result-coords.cc src/unit-result-tile.cc src/unit-s3.cc @@ -257,13 +259,6 @@ if (TILEDB_SERIALIZATION) ) endif() -if (TILEDB_TESTS_ENABLE_REST) - list(APPEND TILEDB_UNIT_TEST_SOURCES - src/unit-capi-rest-dense_array.cc - src/unit-rest-enumerations.cc - ) -endif() - if (TILEDB_ARROW_TESTS) list(APPEND TILEDB_UNIT_TEST_SOURCES src/unit-arrow.cc @@ -329,10 +324,6 @@ if (TILEDB_TESTS_AWS_S3_CONFIG) target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_TESTS_AWS_S3_CONFIG) endif() -if (TILEDB_TESTS_ENABLE_REST) - target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_TESTS_ENABLE_REST) -endif() - if (TILEDB_SERIALIZATION) target_compile_definitions(tiledb_unit PRIVATE -DTILEDB_SERIALIZATION) endif() diff --git a/test/performance/CMakeLists.txt b/test/performance/CMakeLists.txt index 6352041104c4..cb8ecd773ce1 100644 --- a/test/performance/CMakeLists.txt +++ b/test/performance/CMakeLists.txt @@ -30,8 +30,6 @@ find_package(Catch_EP REQUIRED) # These options not exposed in bootstrap script. option(TILEDB_TESTS_AWS_S3_CONFIG "Use an S3 config appropriate for AWS in tests" OFF) -option(TILEDB_TESTS_ENABLE_REST "Enables REST tests (requires running REST server)" OFF) - # Include TileDB core header directories set(TILEDB_CORE_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") diff --git a/test/src/unit-capi-any.cc b/test/src/unit-capi-any.cc index db6684f50f0a..d740e9d33150 100644 --- a/test/src/unit-capi-any.cc +++ b/test/src/unit-capi-any.cc @@ -32,6 +32,7 @@ */ #include +#include #include #include "tiledb/sm/c_api/tiledb.h" @@ -44,11 +45,7 @@ struct AnyFx { const float C3 = 1.2f; const double C4 = 2.3; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; + tiledb::test::VFSTestSetup vfs_test_setup_; void create_array(const std::string& array_name); void delete_array(const std::string& array_name); @@ -58,16 +55,13 @@ struct AnyFx { // Create a simple dense 1D array void AnyFx::create_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(NULL, &ctx); - REQUIRE(rc == TILEDB_OK); + tiledb_ctx_t* ctx = vfs_test_setup_.ctx_c; // Create dimensions uint64_t dim_domain[] = {1, 4}; uint64_t tile_extent = 2; tiledb_dimension_t* d1; - rc = tiledb_dimension_alloc( + int rc = tiledb_dimension_alloc( ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extent, &d1); REQUIRE(rc == TILEDB_OK); @@ -113,18 +107,14 @@ void AnyFx::create_array(const std::string& array_name) { tiledb_dimension_free(&d1); tiledb_domain_free(&domain); tiledb_array_schema_free(&array_schema); - tiledb_ctx_free(&ctx); } void AnyFx::write_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(NULL, &ctx); - REQUIRE(rc == TILEDB_OK); + tiledb_ctx_t* ctx = vfs_test_setup_.ctx_c; // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -170,18 +160,15 @@ void AnyFx::write_array(const std::string& array_name) { ctx, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); - // Second finalize must create no problem - rc = - tiledb::test::finalize_query_wrapper(ctx, array_name, &query, serialize_); + rc = tiledb_query_submit_and_finalize(ctx, query); REQUIRE(rc == TILEDB_OK); + // Calling finalize on a remote Global order write query is not allowed + rc = tiledb_query_finalize(ctx, query); + if (vfs_test_setup_.is_rest()) { + REQUIRE(rc == TILEDB_ERR); + } else { + REQUIRE(rc == TILEDB_OK); + } // Close array rc = tiledb_array_close(ctx, array); @@ -190,18 +177,14 @@ void AnyFx::write_array(const std::string& array_name) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); } void AnyFx::read_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(NULL, &ctx); - REQUIRE(rc == TILEDB_OK); + tiledb_ctx_t* ctx = vfs_test_setup_.ctx_c; // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -230,13 +213,9 @@ void AnyFx::read_array(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = tiledb::test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx, query); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_finalize(ctx, query); REQUIRE(rc == TILEDB_OK); // Check results @@ -264,42 +243,20 @@ void AnyFx::read_array(const std::string& array_name) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); delete[] buffer_a1_off; delete[] buffer_a1_val; } void AnyFx::delete_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(NULL, &ctx); - REQUIRE(rc == TILEDB_OK); - - // Remove array - tiledb_object_t type; - rc = tiledb_object_type(ctx, array_name.c_str(), &type); - REQUIRE(rc == TILEDB_OK); - if (type == TILEDB_ARRAY) { - rc = tiledb_object_remove(ctx, array_name.c_str()); - REQUIRE(rc == TILEDB_OK); + auto obj = tiledb::Object::object(vfs_test_setup_.ctx(), array_name); + if (obj.type() == tiledb::Object::Type::Array) { + tiledb_array_delete(vfs_test_setup_.ctx_c, array_name.c_str()); } - - // Clean up - tiledb_ctx_free(&ctx); } -TEST_CASE_METHOD(AnyFx, "C API: Test `ANY` datatype", "[capi][any]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - std::string array_name = "foo"; +TEST_CASE_METHOD( + AnyFx, "C API: Test `ANY` datatype", "[capi][any][rest-fails][sc-40489]") { + auto array_name = vfs_test_setup_.array_uri("foo"); delete_array(array_name); create_array(array_name); write_array(array_name); diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index 11f336b7734b..553cec876881 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -70,30 +70,19 @@ struct DenseArrayRESTFx { const tiledb_datatype_t DIM_TYPE = TILEDB_INT64; const int ITER_NUM = 10; - tiledb_encryption_type_t encryption_type = TILEDB_NO_ENCRYPTION; - const char* encryption_key = nullptr; + VFSTestSetup vfs_test_setup_; + std::string array_uri_; // TileDB context and VFS tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; - // Vector of supported filsystems - const std::vector> fs_vec_; - - const std::string rest_server_username_ = "unit"; - const std::string TILEDB_URI_PREFIX = - "tiledb://" + rest_server_username_ + "/"; - std::set to_deregister_; - // Functions DenseArrayRESTFx(); - ~DenseArrayRESTFx(); - void create_temp_dir(const std::string& path); - void remove_temp_dir(const std::string& path); - void check_sorted_reads(const std::string& path); - void check_incomplete_reads(const std::string& path); - void check_sorted_writes(const std::string& path); - void check_simultaneous_writes(const std::string& path); + void check_sorted_reads(); + void check_incomplete_reads(); + void check_sorted_writes(); + void check_simultaneous_writes(); void create_dense_array(const std::string& array_name); void create_dense_array_1_attribute(const std::string& array_name); void write_dense_array(const std::string& array_name); @@ -201,40 +190,11 @@ struct DenseArrayRESTFx { }; DenseArrayRESTFx::DenseArrayRESTFx() - : fs_vec_(vfs_test_get_fs_vec()) { - // Initialize vfs test - REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); + : ctx_(vfs_test_setup_.ctx_c) + , vfs_(vfs_test_setup_.vfs_c) { std::srand(0); } -DenseArrayRESTFx::~DenseArrayRESTFx() { - for (const auto& uri : to_deregister_) { - // Deregister array and delete data. - REQUIRE_NOTHROW(tiledb_array_delete(ctx_, uri.c_str())); - } - - // Close vfs test. Removes S3 bucket, Azure container, etc. - REQUIRE(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); - - tiledb_vfs_free(&vfs_); - CHECK(vfs_ == nullptr); - tiledb_ctx_free(&ctx_); - CHECK(ctx_ == nullptr); -} - -void DenseArrayRESTFx::create_temp_dir(const std::string& path) { - remove_temp_dir(path); - REQUIRE(tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); -} - -void DenseArrayRESTFx::remove_temp_dir(const std::string& path) { - int is_dir = 0; - REQUIRE(tiledb_vfs_is_dir(ctx_, vfs_, path.c_str(), &is_dir) == TILEDB_OK); - if (is_dir) { - REQUIRE(tiledb_vfs_remove_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); - } -} - void DenseArrayRESTFx::create_dense_array_2D( const std::string& array_name, const int64_t tile_extent_0, @@ -287,28 +247,8 @@ void DenseArrayRESTFx::create_dense_array_2D( REQUIRE(rc == TILEDB_OK); // Create the array - if (encryption_type != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); - tiledb_config_t* config; - tiledb_error_t* error = nullptr; - rc = tiledb_config_alloc(&config, &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); - std::string encryption_type_string = - encryption_type_str((tiledb::sm::EncryptionType)encryption_type); - rc = tiledb_config_set( - config, "sm.encryption_type", encryption_type_string.c_str(), &error); - REQUIRE(error == nullptr); - rc = tiledb_config_set(config, "sm.encryption_key", encryption_key, &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); - REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok()); - tiledb_config_free(&config); - } rc = tiledb_array_create(ctx_, array_name.c_str(), array_schema); REQUIRE(rc == TILEDB_OK); - to_deregister_.insert(array_name); // Clean up tiledb_attribute_free(&a); @@ -366,25 +306,6 @@ int* DenseArrayRESTFx::read_dense_array_2D( tiledb_array_t* array; rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - if (encryption_type != TILEDB_NO_ENCRYPTION) { - tiledb_config_t* cfg; - tiledb_error_t* err = nullptr; - rc = tiledb_config_alloc(&cfg, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - std::string encryption_type_string = - encryption_type_str((tiledb::sm::EncryptionType)encryption_type); - rc = tiledb_config_set( - cfg, "sm.encryption_type", encryption_type_string.c_str(), &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_array_set_config(ctx_, array, cfg); - REQUIRE(rc == TILEDB_OK); - tiledb_config_free(&cfg); - } rc = tiledb_array_open(ctx_, array, query_type); CHECK(rc == TILEDB_OK); @@ -462,25 +383,6 @@ void DenseArrayRESTFx::write_dense_array_by_tiles( tiledb_array_t* array; rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - if (encryption_type != TILEDB_NO_ENCRYPTION) { - tiledb_config_t* cfg; - tiledb_error_t* err = nullptr; - rc = tiledb_config_alloc(&cfg, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - std::string encryption_type_string = - encryption_type_str((tiledb::sm::EncryptionType)encryption_type); - rc = tiledb_config_set( - cfg, "sm.encryption_type", encryption_type_string.c_str(), &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_array_set_config(ctx_, array, cfg); - REQUIRE(rc == TILEDB_OK); - tiledb_config_free(&cfg); - } rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -582,7 +484,7 @@ void DenseArrayRESTFx::write_dense_subarray_2D( tiledb_query_free(&query); } -void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { +void DenseArrayRESTFx::check_sorted_reads() { // Parameters used in this test int64_t domain_size_0 = 5000; int64_t domain_size_1 = 10000; @@ -595,11 +497,11 @@ void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { uint64_t capacity = 1000000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = TILEDB_URI_PREFIX + path + "sorted_reads_array"; + array_uri_ = vfs_test_setup_.array_uri("sorted_reads_array"); // Create a dense integer array create_dense_array_2D( - array_name, + array_uri_, tile_extent_0, tile_extent_1, domain_0_lo, @@ -613,7 +515,7 @@ void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { // Write array cells with value = row id * COLUMNS + col id // to disk tile by tile write_dense_array_by_tiles( - array_name, domain_size_0, domain_size_1, tile_extent_0, tile_extent_1); + array_uri_, domain_size_0, domain_size_1, tile_extent_0, tile_extent_1); // Test random subarrays and check with corresponding value set by // row_id*dim1+col_id. Top left corner is always 4,4. @@ -632,7 +534,7 @@ void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { // Read subarray int* buffer = read_dense_array_2D( - array_name, d0_lo, d0_hi, d1_lo, d1_hi, TILEDB_READ, TILEDB_ROW_MAJOR); + array_uri_, d0_lo, d0_hi, d1_lo, d1_hi, TILEDB_READ, TILEDB_ROW_MAJOR); REQUIRE(buffer != nullptr); bool allok = true; @@ -659,26 +561,10 @@ void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; - if (encryption_type != TILEDB_NO_ENCRYPTION) { - rc = tiledb_config_alloc(&cfg, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - std::string encryption_type_string = - encryption_type_str((tiledb::sm::EncryptionType)encryption_type); - rc = tiledb_config_set( - cfg, "sm.encryption_type", encryption_type_string.c_str(), &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key, &err); - REQUIRE(rc == TILEDB_OK); - REQUIRE(err == nullptr); - rc = tiledb_array_set_config(ctx_, array, cfg); - REQUIRE(rc == TILEDB_OK); - } rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -726,7 +612,7 @@ void DenseArrayRESTFx::check_sorted_reads(const std::string& path) { tiledb_query_free(&query); } -void DenseArrayRESTFx::check_incomplete_reads(const std::string& path) { +void DenseArrayRESTFx::check_incomplete_reads() { // Parameters used in this test int64_t domain_size_0 = 5000; int64_t domain_size_1 = 10000; @@ -739,11 +625,11 @@ void DenseArrayRESTFx::check_incomplete_reads(const std::string& path) { uint64_t capacity = 1000000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = TILEDB_URI_PREFIX + path + "incomplete_reads_array"; + array_uri_ = vfs_test_setup_.array_uri("incomplete_reads_array"); // Create a dense integer array create_dense_array_2D( - array_name, + array_uri_, tile_extent_0, tile_extent_1, domain_0_lo, @@ -757,11 +643,11 @@ void DenseArrayRESTFx::check_incomplete_reads(const std::string& path) { // Write array cells with value = row id * COLUMNS + col id // to disk tile by tile write_dense_array_by_tiles( - array_name, domain_size_0, domain_size_1, tile_extent_0, tile_extent_1); + array_uri_, domain_size_0, domain_size_1, tile_extent_0, tile_extent_1); // Open array tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); // Create query @@ -828,7 +714,7 @@ void DenseArrayRESTFx::check_incomplete_reads(const std::string& path) { tiledb_query_free(&query); } -void DenseArrayRESTFx::check_sorted_writes(const std::string& path) { +void DenseArrayRESTFx::check_sorted_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -841,11 +727,11 @@ void DenseArrayRESTFx::check_sorted_writes(const std::string& path) { uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = TILEDB_URI_PREFIX + path + "sorted_writes_array"; + array_uri_ = vfs_test_setup_.array_uri("sorted_writes_array"); // Create a dense integer array create_dense_array_2D( - array_name, + array_uri_, tile_extent_0, tile_extent_1, domain_0_lo, @@ -879,7 +765,7 @@ void DenseArrayRESTFx::check_sorted_writes(const std::string& path) { // Write 2D subarray write_dense_subarray_2D( - array_name, + array_uri_, subarray, TILEDB_WRITE, TILEDB_ROW_MAJOR, @@ -888,7 +774,7 @@ void DenseArrayRESTFx::check_sorted_writes(const std::string& path) { // Read back the same subarray int* read_buffer = read_dense_array_2D( - array_name, + array_uri_, subarray[0], subarray[1], subarray[2], @@ -913,7 +799,7 @@ void DenseArrayRESTFx::check_sorted_writes(const std::string& path) { } } -void DenseArrayRESTFx::check_simultaneous_writes(const std::string& path) { +void DenseArrayRESTFx::check_simultaneous_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -926,12 +812,11 @@ void DenseArrayRESTFx::check_simultaneous_writes(const std::string& path) { uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = - TILEDB_URI_PREFIX + path + "simultaneous_writes_array"; + array_uri_ = vfs_test_setup_.array_uri("simultaneous_writes_array"); // Create a dense integer array create_dense_array_2D( - array_name, + array_uri_, tile_extent_0, tile_extent_1, domain_0_lo, @@ -965,7 +850,7 @@ void DenseArrayRESTFx::check_simultaneous_writes(const std::string& path) { const int writes_per_thread = 5; for (int j = 0; j < writes_per_thread; j++) { write_dense_subarray_2D( - array_name, + array_uri_, subarrays[i].data(), TILEDB_WRITE, TILEDB_ROW_MAJOR, @@ -1055,7 +940,6 @@ void DenseArrayRESTFx::create_dense_array(const std::string& array_name) { // Create array rc = tiledb_array_create(ctx_, array_name.c_str(), array_schema); CHECK(rc == TILEDB_OK); - to_deregister_.insert(array_name); // Clean up tiledb_attribute_free(&a1); @@ -1115,7 +999,8 @@ void DenseArrayRESTFx::create_dense_array_1_attribute( // Create array rc = tiledb_array_create(ctx_, array_name.c_str(), array_schema); CHECK(rc == TILEDB_OK); - to_deregister_.insert(array_name); + // this is needed for test cleanup to know the array needs to be deleted + array_uri_ = array_name; // Clean up tiledb_attribute_free(&a1); @@ -1280,45 +1165,34 @@ void DenseArrayRESTFx::write_dense_array_missing_attributes( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, sorted reads", - "[capi][dense][rest]") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - check_sorted_reads(temp_dir); + "[capi][rest][dense]") { + check_sorted_reads(); } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, sorted writes", - "[capi][dense][rest]") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - check_sorted_writes(temp_dir); + "[capi][rest][dense][interference]") { + check_sorted_writes(); } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, simultaneous writes", - "[capi][dense][rest][dense-simultaneous-writes]") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - check_simultaneous_writes(temp_dir); + "[capi][rest][dense][dense-simultaneous-writes]") { + check_simultaneous_writes(); } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, global order reads", - "[capi][dense][rest]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "global_order_reads/"; - create_dense_array(array_name); - write_dense_array(array_name); + "[capi][rest][dense]") { + array_uri_ = vfs_test_setup_.array_uri("global_order_reads"); + create_dense_array(array_uri_); + write_dense_array(array_uri_); tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); uint64_t subarray[] = {1, 4, 1, 4}; @@ -1375,33 +1249,25 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, missing attributes in writes", - "[capi][dense][rest][dense-write-missing-attributes]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "dense_write_missing_attributes/"; - create_temp_dir(temp_dir); - create_dense_array(array_name); - write_dense_array_missing_attributes(array_name); + "[capi][rest][dense][dense-write-missing-attributes]") { + array_uri_ = vfs_test_setup_.array_uri("dense_write_missing_attributes"); + create_dense_array(array_uri_); + write_dense_array_missing_attributes(array_uri_); } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, read subarrays with empty cells", - "[capi][dense][rest][dense-read-empty]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "dense_read_empty/"; - create_temp_dir(temp_dir); - - create_dense_array_1_attribute(array_name); + "[capi][rest][dense][dense-read-empty]") { + array_uri_ = vfs_test_setup_.array_uri("dense_read_empty"); + create_dense_array_1_attribute(array_uri_); // Write a slice const char* attributes[] = {"a1"}; int write_a1[] = {1, 2, 3, 4}; uint64_t write_a1_size = sizeof(write_a1); tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -1444,7 +1310,7 @@ TEST_CASE_METHOD( INT_MIN}; int read_a1[16]; uint64_t read_a1_size = sizeof(read_a1); - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1471,21 +1337,16 @@ TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, read subarrays with empty areas, merging " "adjacent cell ranges", - "[capi][dense][rest][dense-read-empty][dense-read-empty-merge]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "dense_read_empty_merge/"; - create_temp_dir(temp_dir); - - create_dense_array_1_attribute(array_name); + "[capi][rest][dense][dense-read-empty][dense-read-empty-merge]") { + array_uri_ = vfs_test_setup_.array_uri("dense_read_empty_merge"); + create_dense_array_1_attribute(array_uri_); // Write a slice const char* attributes[] = {"a1"}; int write_a1[] = {1, 2, 3, 4}; uint64_t write_a1_size = sizeof(write_a1); tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -1528,7 +1389,7 @@ TEST_CASE_METHOD( INT_MIN}; int read_a1[16]; uint64_t read_a1_size = sizeof(read_a1); - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1554,20 +1415,15 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, multi-fragment reads", - "[capi][dense][rest][dense-multi-fragment]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "dense_multi_fragment/"; - create_temp_dir(temp_dir); - - create_dense_array_1_attribute(array_name); + "[capi][rest][dense][dense-multi-fragment]") { + array_uri_ = vfs_test_setup_.array_uri("dense_multi_fragment"); + create_dense_array_1_attribute(array_uri_); // Write slice [1,2], [3,4] int write_a1[] = {1, 2, 3, 4, 5, 6, 7, 8}; uint64_t write_a1_size = sizeof(write_a1); tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -1592,7 +1448,7 @@ TEST_CASE_METHOD( // Write slice [2,3], [2,3] int write_a2[] = {101, 102, 103, 104}; uint64_t write_a2_size = sizeof(write_a2); - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -1634,7 +1490,7 @@ TEST_CASE_METHOD( INT_MIN}; int read_a[16]; uint64_t read_a_size = sizeof(read_a); - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1659,14 +1515,12 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, check if open", - "[capi][dense][rest][dense-is-open]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "dense_is_open/"; - create_dense_array(array_name); + "[capi][rest][dense][dense-is-open]") { + array_uri_ = vfs_test_setup_.array_uri("dense_is_open"); + create_dense_array(array_uri_); tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); int is_open; @@ -1694,15 +1548,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, get schema from opened array", - "[capi][dense][rest][dense-get-schema]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "dense_get_schema/"; - create_dense_array(array_name); + "[capi][rest][dense][dense-get-schema]") { + array_uri_ = vfs_test_setup_.array_uri("dense_get_schema"); + create_dense_array(array_uri_); // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1724,17 +1576,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, set subarray in sparse writes should error", - "[capi][dense][rest][dense-set-subarray-sparse]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "dense_set_subarray_sparse"; - create_temp_dir(temp_dir); - create_dense_array(array_name); + "[capi][rest][dense][dense-set-subarray-sparse]") { + array_uri_ = vfs_test_setup_.array_uri("dense_set_subarray_sparse"); + create_dense_array(array_uri_); // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -1765,18 +1613,15 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, incomplete reads", - "[capi][dense][rest][incomplete]") { + "[capi][rest][dense][incomplete]") { // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - check_incomplete_reads(temp_dir); - remove_temp_dir(temp_dir); + check_incomplete_reads(); } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, get nonempty domain", - "[capi][dense][rest]") { + "[capi][rest][dense]") { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -1789,14 +1634,11 @@ TEST_CASE_METHOD( uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "nonempty_domain_array"; + array_uri_ = vfs_test_setup_.array_uri("nonempty_domain_array"); // Create a dense integer array create_dense_array_2D( - array_name, + array_uri_, tile_extent_0, tile_extent_1, domain_0_lo, @@ -1809,7 +1651,7 @@ TEST_CASE_METHOD( // Check nonempty domain before writing tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); int64_t nonempty_domain[4]; int32_t is_empty; @@ -1835,7 +1677,7 @@ TEST_CASE_METHOD( // Write 2D subarray write_dense_subarray_2D( - array_name, + array_uri_, subarray, TILEDB_WRITE, TILEDB_ROW_MAJOR, @@ -1846,7 +1688,7 @@ TEST_CASE_METHOD( delete[] buffer; // Open array - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1867,16 +1709,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, get max buffer sizes", - "[capi][dense][rest]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "max_buffer_sizes_array"; - create_dense_array(array_name); + "[capi][rest][dense]") { + array_uri_ = vfs_test_setup_.array_uri("max_buffer_sizes_array"); + create_dense_array(array_uri_); // Check max buffer sizes with empty array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1884,10 +1723,10 @@ TEST_CASE_METHOD( tiledb_array_free(&array); // Write array - write_dense_array(array_name); + write_dense_array(array_uri_); // Check max buffer sizes for whole domain - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1900,50 +1739,47 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, error without rest server configured", - "[capi][dense][rest][rest-no-config]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = - TILEDB_URI_PREFIX + temp_dir + "dense_set_subarray_sparse"; - create_dense_array(array_name); - - // Set config to use a non-default environment prefix. - // Prevents test from picking up on REST CI environment configuration. - tiledb_config_t* cfg; - tiledb_error_t* err; - int rc = tiledb_config_alloc(&cfg, &err); - CHECK(rc == TILEDB_OK); - CHECK(err == nullptr); - rc = tiledb_config_set(cfg, "config.env_var_prefix", "UNIT_", &err); - CHECK(rc == TILEDB_OK); - CHECK(err == nullptr); - - // Create context without a REST config - tiledb_ctx_t* ctx; - REQUIRE(tiledb_ctx_alloc(cfg, &ctx) == TILEDB_OK); - - tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); - CHECK(rc == TILEDB_ERR); + "[capi][rest][dense][rest-no-config]") { + // test only applies to REST + array_uri_ = vfs_test_setup_.array_uri("dense_set_subarray_sparse"); + create_dense_array(array_uri_); + + if (vfs_test_setup_.is_rest()) { + // Set config to use a non-default environment prefix. + // Prevents test from picking up on REST CI environment configuration. + tiledb_config_t* cfg; + tiledb_error_t* err; + int rc = tiledb_config_alloc(&cfg, &err); + CHECK(rc == TILEDB_OK); + CHECK(err == nullptr); + rc = tiledb_config_set(cfg, "config.env_var_prefix", "UNIT_", &err); + CHECK(rc == TILEDB_OK); + CHECK(err == nullptr); + + // Create context without a REST config + tiledb_ctx_t* ctx2; + REQUIRE(tiledb_ctx_alloc(cfg, &ctx2) == TILEDB_OK); + + tiledb_array_t* array; + rc = tiledb_array_alloc(ctx2, array_uri_.c_str(), &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx2, array, TILEDB_WRITE); + REQUIRE(rc == TILEDB_ERR); - // Clean up - CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); - tiledb_config_free(&cfg); - tiledb_error_free(&err); - tiledb_array_free(&array); - tiledb_ctx_free(&ctx); + // Clean up + CHECK(tiledb_array_close(ctx2, array) == TILEDB_OK); + tiledb_config_free(&cfg); + tiledb_error_free(&err); + tiledb_array_free(&array); + tiledb_ctx_free(&ctx2); + } } TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, datetimes", - "[capi][dense][rest][datetime]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "datetime_array"; - + "[capi][rest][dense][datetime]") { + array_uri_ = vfs_test_setup_.array_uri("datetime_array"); int64_t dim_domain[] = {1, 10}; int64_t tile_extents[] = {2}; tiledb_dimension_t* d1; @@ -1976,8 +1812,7 @@ TEST_CASE_METHOD( // Create array REQUIRE( - tiledb_array_create(ctx_, array_name.c_str(), array_schema) == TILEDB_OK); - to_deregister_.insert(array_name); + tiledb_array_create(ctx_, array_uri_.c_str(), array_schema) == TILEDB_OK); // Clean up tiledb_attribute_free(&a1); @@ -1991,7 +1826,7 @@ TEST_CASE_METHOD( void* buffers[] = {buffer_a1}; uint64_t buffer_sizes[] = {sizeof(buffer_a1)}; tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_WRITE) == TILEDB_OK); tiledb_query_t* query; REQUIRE(tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query) == TILEDB_OK); @@ -2010,7 +1845,7 @@ TEST_CASE_METHOD( buffer_a1[i] = 0; int64_t subarray[] = {2, 5}; buffer_sizes[0] = sizeof(buffer_a1); - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); REQUIRE(tiledb_query_alloc(ctx_, array, TILEDB_READ, &query) == TILEDB_OK); REQUIRE(tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR) == TILEDB_OK); @@ -2034,11 +1869,8 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, array metadata", - "[capi][dense][rest][metadata]") { - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); - std::string array_name = TILEDB_URI_PREFIX + temp_dir + "metadata_array"; - + "[capi][rest][dense][metadata]") { + array_uri_ = vfs_test_setup_.array_uri("metadata_array"); int64_t dim_domain[] = {1, 10}; int64_t tile_extents[] = {2}; tiledb_dimension_t* d1; @@ -2066,8 +1898,7 @@ TEST_CASE_METHOD( // Create array REQUIRE( - tiledb_array_create(ctx_, array_name.c_str(), array_schema) == TILEDB_OK); - to_deregister_.insert(array_name); + tiledb_array_create(ctx_, array_uri_.c_str(), array_schema) == TILEDB_OK); // Clean up tiledb_attribute_free(&a1); @@ -2077,7 +1908,7 @@ TEST_CASE_METHOD( // Write some metadata values tiledb_array_t* array; - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_WRITE) == TILEDB_OK); int32_t v = 5; float f[] = {1.1f, 1.2f}; @@ -2091,7 +1922,7 @@ TEST_CASE_METHOD( tiledb_array_free(&array); // Read metadata and check values. - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); uint64_t num_metadata = 0; REQUIRE( @@ -2120,14 +1951,14 @@ TEST_CASE_METHOD( std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Open for writing and delete a key. - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_WRITE) == TILEDB_OK); REQUIRE(tiledb_array_delete_metadata(ctx_, array, "aaa") == TILEDB_OK); REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); tiledb_array_free(&array); // Read metadata and check values again. - REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_alloc(ctx_, array_uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); REQUIRE( tiledb_array_get_metadata_num(ctx_, array, &num_metadata) == TILEDB_OK); diff --git a/test/src/unit-capi-vfs.cc b/test/src/unit-capi-vfs.cc index e50339e01ece..eeb87cd8d7d1 100644 --- a/test/src/unit-capi-vfs.cc +++ b/test/src/unit-capi-vfs.cc @@ -131,7 +131,13 @@ TEST_CASE("C API: Test virtual filesystem", "[capi][vfs]") { bool hdfs_enabled; bool azure_enabled; bool gcs_enabled; - get_supported_fs(&s3_enabled, &hdfs_enabled, &azure_enabled, &gcs_enabled); + bool rest_s3_enabled; + get_supported_fs( + &s3_enabled, + &hdfs_enabled, + &azure_enabled, + &gcs_enabled, + &rest_s3_enabled); // Sections to test each filesystem, if enabled ordinary_vfs x(config); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 12ec1d2f1dcb..1c6b8a4c646a 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -30,6 +30,7 @@ * Tests the C++ API for array related functions. */ +#include #include #include "test/support/src/helpers.h" #include "test/support/src/serialization_wrappers.h" @@ -51,19 +52,9 @@ struct CPPArrayFx { static const unsigned d1_tile = 10; static const unsigned d2_tile = 5; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - test::ServerQueryBuffers server_buffers_; - CPPArrayFx() - : vfs(ctx) { - using namespace tiledb; - - if (vfs.is_dir("cpp_unit_array")) - vfs.remove_dir("cpp_unit_array"); - + : ctx(vfs_test_setup_.ctx()) + , array_uri_{vfs_test_setup_.array_uri("cpp_unit_array")} { Domain domain(ctx); auto d1 = Dimension::create(ctx, "d1", {{-100, 100}}, d1_tile); auto d2 = Dimension::create(ctx, "d2", {{0, 100}}, d2_tile); @@ -84,19 +75,16 @@ struct CPPArrayFx { schema.set_domain(domain); schema.add_attributes(a1, a2, a3, a4, a5); - Array::create("cpp_unit_array", schema); - } - - ~CPPArrayFx() { - if (vfs.is_dir("cpp_unit_array")) - vfs.remove_dir("cpp_unit_array"); + // set the array_uri so that it's deleted on cleanup + Array::create(array_uri_, schema); } + test::VFSTestSetup vfs_test_setup_; Context ctx; - VFS vfs; + std::string array_uri_; }; -TEST_CASE("Config", "[cppapi][config]") { +TEST_CASE("Config", "[cppapi][config][non-rest]") { // Primarily to instansiate operator[]/= template tiledb::Config cfg; cfg["vfs.s3.region"] = "us-east-1a"; @@ -105,9 +93,9 @@ TEST_CASE("Config", "[cppapi][config]") { CHECK((std::string)cfg["vfs.s3.use_virtual_addressing"] == "true"); } -TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { +TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { SECTION("Dimensions") { - ArraySchema schema(ctx, "cpp_unit_array"); + ArraySchema schema(ctx, array_uri_); CHECK(schema.domain().ndim() == 2); auto a = schema.domain().dimensions()[0].domain(); auto b = schema.domain().dimensions()[1].domain(); @@ -123,7 +111,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { } SECTION("Make Buffer") { - Array array(ctx, "cpp_unit_array", TILEDB_WRITE); + Array array(ctx, array_uri_, TILEDB_WRITE); Subarray subarray(ctx, array); CHECK_THROWS(subarray.set_subarray({1, 2})); // Wrong type CHECK_THROWS(subarray.set_subarray({1, 2})); // Wrong num @@ -135,10 +123,21 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { tiledb::Config cfg; cfg["a"] = "1"; cfg["b"] = "10"; - Context ctx1(cfg); + auto vfs_test_setup1 = tiledb::test::VFSTestSetup(cfg.ptr().get()); + auto ctx1 = vfs_test_setup1.ctx(); + auto array_uri1 = vfs_test_setup1.array_uri("cpp_unit_array1"); // Create an array with ctx - Array array(ctx1, "cpp_unit_array", TILEDB_READ); + Domain domain(ctx); + auto d1 = Dimension::create(ctx, "d1", {{-100, 100}}, d1_tile); + domain.add_dimension(d1); + auto a1 = Attribute::create(ctx, "a1"); // (int, 1) + ArraySchema schema(ctx, TILEDB_DENSE); + schema.set_domain(domain); + schema.add_attribute(a1); + + Array::create(array_uri1, schema); + Array array(ctx1, array_uri1, TILEDB_READ); // Check that the config values are correct CHECK((std::string)array.config()["a"] == "1"); @@ -177,61 +176,62 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { uint64_t* offsets_back; uint64_t offsets_back_nelem = 0; - REQUIRE( - Array::encryption_type(ctx, "cpp_unit_array") == TILEDB_NO_ENCRYPTION); - - try { - Array array(ctx, "cpp_unit_array", TILEDB_READ); - CHECK(array.query_type() == TILEDB_READ); - CHECK(array.is_open()); + if (vfs_test_setup_.is_rest()) { + // this API is not supported on remote arrays + REQUIRE_THROWS( + Array::encryption_type(ctx, array_uri_) == TILEDB_NO_ENCRYPTION); + } else { + REQUIRE(Array::encryption_type(ctx, array_uri_) == TILEDB_NO_ENCRYPTION); + } - // Close and reopen - array.close(); - CHECK(!array.is_open()); - array.open(TILEDB_WRITE); - CHECK(array.is_open()); - CHECK(array.query_type() == TILEDB_WRITE); + Array array(ctx, array_uri_, TILEDB_READ); + CHECK(array.query_type() == TILEDB_READ); + CHECK(array.is_open()); - Query query(ctx, array, TILEDB_WRITE); - CHECK(query.query_type() == TILEDB_WRITE); - query.set_subarray(subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2buf.second); - query.set_offsets_buffer("a2", a2buf.first); - query.set_data_buffer("a3", a3); - query.set_data_buffer("a4", a4buf.second); - query.set_offsets_buffer("a4", a4buf.first); - query.set_data_buffer("a5", a5); - query.set_layout(TILEDB_ROW_MAJOR); - REQUIRE(query.submit() == Query::Status::COMPLETE); + // Close and reopen + array.close(); + CHECK(!array.is_open()); + array.open(TILEDB_WRITE); + CHECK(array.is_open()); + CHECK(array.query_type() == TILEDB_WRITE); - // check a1 buffers - query.get_data_buffer( - "a1", &buf_back, &buf_back_nelem, &buf_back_elem_size); - REQUIRE(buf_back == a1.data()); - REQUIRE(buf_back_nelem == 2); - REQUIRE(buf_back_elem_size == sizeof(int)); - - // check a2 buffers - query.get_data_buffer( - "a2", &buf_back, &buf_back_nelem, &buf_back_elem_size); - query.get_offsets_buffer("a2", &offsets_back, &offsets_back_nelem); - REQUIRE(buf_back == a2buf.second.data()); - REQUIRE(buf_back_nelem == 7); - REQUIRE(buf_back_elem_size == sizeof(char)); - REQUIRE(offsets_back == a2buf.first.data()); - REQUIRE(offsets_back_nelem == 2); - CHECK(!query.has_results()); + Query query(ctx, array, TILEDB_WRITE); + CHECK(query.query_type() == TILEDB_WRITE); + query.set_subarray(subarray); + query.set_data_buffer("a1", a1); + query.set_data_buffer("a2", a2buf.second); + query.set_offsets_buffer("a2", a2buf.first); + query.set_data_buffer("a3", a3); + query.set_data_buffer("a4", a4buf.second); + query.set_offsets_buffer("a4", a4buf.first); + query.set_data_buffer("a5", a5); + query.set_layout(TILEDB_ROW_MAJOR); + REQUIRE(query.submit() == Query::Status::COMPLETE); + + // check a1 buffers + query.get_data_buffer( + "a1", &buf_back, &buf_back_nelem, &buf_back_elem_size); + REQUIRE(buf_back == a1.data()); + REQUIRE(buf_back_nelem == 2); + REQUIRE(buf_back_elem_size == sizeof(int)); + + // check a2 buffers + query.get_data_buffer( + "a2", &buf_back, &buf_back_nelem, &buf_back_elem_size); + query.get_offsets_buffer("a2", &offsets_back, &offsets_back_nelem); + REQUIRE(buf_back == a2buf.second.data()); + REQUIRE(buf_back_nelem == 7); + REQUIRE(buf_back_elem_size == sizeof(char)); + REQUIRE(offsets_back == a2buf.first.data()); + REQUIRE(offsets_back_nelem == 2); + CHECK(!query.has_results()); - query.finalize(); - array.close(); + query.finalize(); + array.close(); - tiledb::Array::consolidate(ctx, "cpp_unit_array"); - } catch (std::exception& e) { - std::cout << e.what() << std::endl; - } + if (!vfs_test_setup_.is_rest()) { + tiledb::Array::consolidate(ctx, array_uri_); - { std::fill(std::begin(a1), std::end(a1), 0); std::fill(std::begin(a2buf.first), std::end(a2buf.first), 0); std::fill(std::begin(a2buf.second), std::end(a2buf.second), 0); @@ -243,7 +243,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { Point{{0, 0, 0}, 0}); std::fill(std::begin(a5), std::end(a5), Point{{0, 0, 0}, 0}); - Array array(ctx, "cpp_unit_array", TILEDB_READ); + Array array(ctx, array_uri_, TILEDB_READ); a1.resize(2); a2buf.first.resize(2); @@ -344,7 +344,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { auto a2buf = ungroup_var_buffer(a2); auto a4buf = ungroup_var_buffer(a4); - Array array(ctx, "cpp_unit_array", TILEDB_WRITE); + Array array(ctx, array_uri_, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); query.set_subarray(subarray); query.set_data_buffer("a1", a1); @@ -357,25 +357,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { query.set_layout(TILEDB_GLOBAL_ORDER); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - // Submit query - auto rc = submit_query_wrapper( - ctx, - "cpp_unit_array", - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + REQUIRE_NOTHROW(query.submit_and_finalize()); // Check non-empty domain while array open in write mode CHECK_THROWS(array.non_empty_domain(1)); @@ -421,7 +403,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { SECTION("Global order write - no dummy values") { std::vector a1 = {1, 2}; std::vector subarray = {0, 1, 0, 0}; - Array array(ctx, "cpp_unit_array", TILEDB_WRITE); + Array array(ctx, array_uri_, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); query.set_subarray(subarray); query.set_data_buffer("a1", a1); @@ -433,11 +415,12 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic]") { } } -TEST_CASE("C++ API: Zero length buffer", "[cppapi][zero-length]") { - bool serialize = false, refactored_query_v2 = false; - const std::string array_name_1d = "cpp_unit_array_1d"; - Context ctx; - VFS vfs(ctx); +TEST_CASE( + "C++ API: Zero length buffer", + "[cppapi][zero-length][rest-fails][sc-40479]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array_1d")}; tiledb_layout_t write_layout = TILEDB_GLOBAL_ORDER; tiledb_array_type_t array_type = TILEDB_DENSE; @@ -455,15 +438,6 @@ TEST_CASE("C++ API: Zero length buffer", "[cppapi][zero-length]") { SECTION("NON_NULL_PTR") { null_pointer = false; } - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif } SECTION("UNORDERED") { @@ -486,19 +460,16 @@ TEST_CASE("C++ API: Zero length buffer", "[cppapi][zero-length]") { } } - if (vfs.is_dir(array_name_1d)) - vfs.remove_dir(array_name_1d); - ArraySchema schema(ctx, array_type); Domain domain(ctx); domain.add_dimension(Dimension::create(ctx, "d", {{0, 2}}, 3)); schema.set_domain(domain); schema.add_attribute(Attribute::create>(ctx, "a")); schema.add_attribute(Attribute::create(ctx, "b")); - Array::create(array_name_1d, schema); + Array::create(array_uri, schema); { - Array array(ctx, array_name_1d, TILEDB_WRITE); + Array array(ctx, array_uri, TILEDB_WRITE); std::vector a, coord = {0, 1, 2}; std::vector a_offset = {0, 0, 0}; @@ -517,22 +488,18 @@ TEST_CASE("C++ API: Zero length buffer", "[cppapi][zero-length]") { q.set_offsets_buffer("a", a_offset); q.set_data_buffer("b", b); - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = submit_query_wrapper( - ctx, - array_name_1d, - &q, - server_buffers_, - serialize, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (write_layout != TILEDB_GLOBAL_ORDER) { + q.submit(); + q.finalize(); + } else { + q.submit_and_finalize(); + } array.close(); } { - Array array(ctx, array_name_1d, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); std::vector a(3); std::vector a_offset = {1, 1, 1}; @@ -559,26 +526,21 @@ TEST_CASE("C++ API: Zero length buffer", "[cppapi][zero-length]") { CHECK(b[i] == i + 1); } } - - if (vfs.is_dir(array_name_1d)) - vfs.remove_dir(array_name_1d); + Array::delete_array(ctx, array_uri); } -TEST_CASE("C++ API: Incorrect offsets", "[cppapi][invalid-offsets]") { - const std::string array_name_1d = "cpp_unit_array_1d"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name_1d)) - vfs.remove_dir(array_name_1d); +TEST_CASE("C++ API: Incorrect offsets", "[cppapi][invalid-offsets][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array_1d")}; ArraySchema schema(ctx, TILEDB_SPARSE); Domain domain(ctx); domain.add_dimension(Dimension::create(ctx, "d", {{0, 1000}}, 1001)); schema.set_domain(domain); schema.add_attribute(Attribute::create>(ctx, "a")); - Array::create(array_name_1d, schema); - Array array(ctx, array_name_1d, TILEDB_WRITE); + Array::create(array_uri, schema); + Array array(ctx, array_uri, TILEDB_WRITE); std::vector a, coord = {10, 20, 30}; std::vector a_offset = {0, 0, 0}; @@ -589,19 +551,18 @@ TEST_CASE("C++ API: Incorrect offsets", "[cppapi][invalid-offsets]") { a_offset = {0, 2, 1}; Query q(ctx, array, TILEDB_WRITE); q.set_layout(TILEDB_GLOBAL_ORDER); - q.set_coordinates(coord); + q.set_data_buffer("d", coord); q.set_data_buffer("a", a); q.set_offsets_buffer("a", a_offset); REQUIRE_THROWS(q.submit()); } array.close(); - - if (vfs.is_dir(array_name_1d)) - vfs.remove_dir(array_name_1d); } -TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi][dense]") { +TEST_CASE( + "C++ API: Read subarray with expanded domain", + "[cppapi][dense][rest][check]") { const std::vector tile_layouts = {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}, cell_layouts = {TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR}; @@ -610,12 +571,9 @@ TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi][dense]") { for (auto tile_layout : tile_layouts) { for (auto cell_layout : cell_layouts) { for (int tile_extent : tile_extents) { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -659,15 +617,14 @@ TEST_CASE("C++ API: Read subarray with expanded domain", "[cppapi][dense]") { for (int i = 0; i < 16; i++) { REQUIRE(data[i] == i + 1); } - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } } } } -TEST_CASE("C++ API: Consolidation of empty arrays", "[cppapi][consolidation]") { +TEST_CASE( + "C++ API: Consolidation of empty arrays", + "[cppapi][consolidation][non-rest]") { Context ctx; VFS vfs(ctx); const std::string array_name = "cpp_unit_array"; @@ -692,7 +649,7 @@ TEST_CASE("C++ API: Consolidation of empty arrays", "[cppapi][consolidation]") { TEST_CASE( "C++ API: Consolidation of sequential fragment writes", - "[cppapi][consolidation][sequential]") { + "[cppapi][consolidation][sequential][non-rest]") { tiledb::Config cfg; cfg["sm.consolidation.total_buffer_size"] = "1048576"; Context ctx(cfg); @@ -742,7 +699,7 @@ TEST_CASE( vfs.remove_dir(array_name); } -TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption]") { +TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption][non-rest]") { const char key[] = "0123456789abcdeF0123456789abcdeF"; tiledb::Config cfg; cfg["sm.consolidation.total_buffer_size"] = "1048576"; @@ -825,7 +782,9 @@ TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption]") { vfs.remove_dir(array_name); } -TEST_CASE("C++ API: Encrypted array, std::string key", "[cppapi][encryption]") { +TEST_CASE( + "C++ API: Encrypted array, std::string key", + "[cppapi][encryption][non-rest]") { const std::string key = "0123456789abcdeF0123456789abcdeF"; tiledb::Config cfg; cfg["sm.encryption_type"] = "AES_256_GCM"; @@ -899,12 +858,10 @@ TEST_CASE("C++ API: Encrypted array, std::string key", "[cppapi][encryption]") { TEST_CASE( "C++ API: Open array with anonymous attribute", - "[cppapi][open-array-anon-attr]") { - Context ctx; - VFS vfs(ctx); - const std::string array_name = "cppapi_open_array_anon_attr"; - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][open-array-anon-attr][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cppapi_open_array_anon_attr")}; // Create array Domain domain(ctx); @@ -912,24 +869,20 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain); schema.add_attribute(Attribute::create(ctx, "")); - Array::create(array_name, schema); + Array::create(array_uri, schema); - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); auto reloaded_schema = array.schema(); REQUIRE(reloaded_schema.attribute_num() == 1); array.close(); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } -TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { - Context ctx; - VFS vfs(ctx); - const std::string array_name = "cppapi_open_array_at"; - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); +TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cppapi_open_array_at")}; // Create array Domain domain(ctx); @@ -937,10 +890,10 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write array - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_layout(TILEDB_ROW_MAJOR); std::vector a_w = {1, 2, 3, 4}; @@ -952,7 +905,7 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { auto first_timestamp = TILEDB_TIMESTAMP_NOW_MS; // Normal read - Array array_r(ctx, array_name, TILEDB_READ); + Array array_r(ctx, array_uri, TILEDB_READ); std::vector subarray = {1, 4}; std::vector a_r(4); Query query_r(ctx, array_r); @@ -964,20 +917,14 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { CHECK(std::equal(a_r.begin(), a_r.end(), a_w.begin())); // Read from 0 timestamp - Array array_r_at_0(ctx, array_name, TILEDB_READ); + Array array_r_at_0(ctx, array_uri, TILEDB_READ); array_r_at_0.close(); array_r_at_0.set_open_timestamp_end(0); array_r_at_0.open(TILEDB_READ); CHECK(array_r_at_0.open_timestamp_end() == 0); - SECTION("Testing Array::Array") { - // Nothing to do - just for clarity - } - - SECTION("Testing Array::open") { - array_r_at_0.close(); - array_r_at_0.open(TILEDB_READ); - } + array_r_at_0.close(); + array_r_at_0.open(TILEDB_READ); std::vector a_r_at_0(4); Query query_r_at_0(ctx, array_r_at_0); @@ -992,20 +939,14 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { // Read from later timestamp auto timestamp = TILEDB_TIMESTAMP_NOW_MS; - Array array_r_at(ctx, array_name, TILEDB_READ); + Array array_r_at(ctx, array_uri, TILEDB_READ); array_r_at.close(); array_r_at.set_open_timestamp_end(timestamp); array_r_at.open(TILEDB_READ); CHECK(array_r_at.open_timestamp_end() == timestamp); - SECTION("Testing Array::Array") { - // Nothing to do - just for clarity - } - - SECTION("Testing Array::open") { - array_r_at.close(); - array_r_at.open(TILEDB_READ); - } + array_r_at.close(); + array_r_at.open(TILEDB_READ); std::vector a_r_at(4); Query query_r_at(ctx, array_r_at); @@ -1017,7 +958,7 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { array_r_at.close(); // Reopen at first timestamp. - Array array_reopen_at(ctx, array_name, TILEDB_READ); + Array array_reopen_at(ctx, array_uri, TILEDB_READ); array_reopen_at.set_open_timestamp_end(first_timestamp); array_reopen_at.reopen(); CHECK(array_reopen_at.open_timestamp_end() == first_timestamp); @@ -1029,13 +970,11 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at]") { query_r_reopen_at.submit(); CHECK(std::equal(a_r_reopen_at.begin(), a_r_reopen_at.end(), a_w.begin())); array_reopen_at.close(); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( - "C++ API: Open encrypted array at", "[cppapi][open-encrypted-array-at]") { + "C++ API: Open encrypted array at", + "[cppapi][open-encrypted-array-at][non-rest]") { const char key[] = "0123456789abcdeF0123456789abcdeF"; tiledb::Config cfg; cfg["sm.encryption_type"] = "AES_256_GCM"; @@ -1132,14 +1071,10 @@ TEST_CASE( TEST_CASE( "C++ API: Writing single cell with global order", - "[cppapi][sparse][global]") { - bool serialize = false, refactored_query_v2 = false; - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][sparse][global][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -1148,41 +1083,25 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_SPARSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = {1}; - std::vector coords_w = {0, 0}; - Array array_w(ctx, array_name, TILEDB_WRITE); + std::vector rows = {0}; + std::vector cols = {0}; + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); - query_w.set_coordinates(coords_w) + query_w.set_data_buffer("rows", rows) + .set_data_buffer("cols", cols) .set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("a", data_w); - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialize, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); - + query_w.submit_and_finalize(); array_w.close(); // Read - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector data(1); @@ -1193,22 +1112,14 @@ TEST_CASE( array.close(); REQUIRE(data[0] == 1); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Writing single byte cell with global order", - "[cppapi][std::byte]") { - bool serialize = false, refactored_query_v2 = false; - const std::string array_name = "cpp_unit_array"; - - Context ctx; - VFS vfs(ctx); - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + "[cppapi][std::byte][rest-fails][sc-40489]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; auto datatype = GENERATE( tiledb_datatype_t::TILEDB_BLOB, @@ -1221,39 +1132,21 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a", datatype)); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::byte data_w{1}; - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("a", (void*)(&data_w), 1); - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialize, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_w.submit_and_finalize(); array_w.close(); // Read - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); Subarray subarray(ctx, array); subarray.add_range(0, 0, 0); @@ -1265,18 +1158,13 @@ TEST_CASE( array.close(); REQUIRE(data == data_w); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } -TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); +TEST_CASE( + "C++ API: Write cell with large cell val num", "[cppapi][sparse][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create array with a large fixed-length attribute const size_t cell_val_num = 70000; @@ -1285,7 +1173,7 @@ TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { .set_filter_list(FilterList(ctx).add_filter( Filter(ctx, TILEDB_FILTER_BZIP2))); Array::create( - array_name, + array_uri, ArraySchema(ctx, TILEDB_SPARSE) .set_domain(Domain(ctx).add_dimension( Dimension::create(ctx, "cols", {{0, 9}}, 5))) @@ -1299,11 +1187,11 @@ TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { for (auto i = 0u; i < cell_val_num; i++) data_w[i] = 2 * i; - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_layout(TILEDB_UNORDERED) .set_data_buffer("a", data_w) - .set_coordinates(coords_w) + .set_data_buffer("cols", coords_w) .submit(); query_w.finalize(); array_w.close(); @@ -1311,11 +1199,11 @@ TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { // Read and check results std::vector data_r(cell_val_num, -1); std::vector coords_r = {4}; - Array array_r(ctx, array_name, TILEDB_READ); + Array array_r(ctx, array_uri, TILEDB_READ); Query query_r(ctx, array_r); query_r.set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data_r) - .set_coordinates(coords_r); + .set_data_buffer("cols", coords_r); REQUIRE(query_r.submit() == Query::Status::COMPLETE); auto result_num = query_r.result_buffer_elements()["a"].second; @@ -1324,20 +1212,15 @@ TEST_CASE("C++ API: Write cell with large cell val num", "[cppapi][sparse]") { REQUIRE(data_r[i] == 2 * i); array_r.close(); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } using namespace tiledb::test; -TEST_CASE("C++ API: Test heterogeneous dimensions", "[cppapi][sparse][heter]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); +TEST_CASE( + "C++ API: Test heterogeneous dimensions", "[cppapi][sparse][heter][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create array auto d1 = Dimension::create(ctx, "d1", {1.0f, 20.0f}, 5.0f); @@ -1348,10 +1231,10 @@ TEST_CASE("C++ API: Test heterogeneous dimensions", "[cppapi][sparse][heter]") { ArraySchema schema(ctx, TILEDB_SPARSE); schema.add_attribute(a); schema.set_domain(dom); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write - Array array(ctx, array_name, TILEDB_WRITE); + Array array(ctx, array_uri, TILEDB_WRITE); std::vector buff_d1 = {1.1f, 1.2f, 1.3f, 1.4f}; std::vector buff_d2 = {1, 2, 3, 4}; std::vector buff_a = {1, 2, 3, 4}; @@ -1364,7 +1247,7 @@ TEST_CASE("C++ API: Test heterogeneous dimensions", "[cppapi][sparse][heter]") { array.close(); // Read - Array array_r(ctx, array_name, TILEDB_READ); + Array array_r(ctx, array_uri, TILEDB_READ); std::vector buff_d1_r(4); std::vector buff_d2_r(4); std::vector buff_a_r(4); @@ -1391,20 +1274,14 @@ TEST_CASE("C++ API: Test heterogeneous dimensions", "[cppapi][sparse][heter]") { CHECK(buff_d1 == buff_d1_r); CHECK(buff_d2 == buff_d2_r); CHECK(buff_a == buff_a_r); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Test string dimensions, 1d", - "[cppapi][sparse][string-dims][1d]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][sparse][string-dims][1d][rest-fails][sc-40489]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create array auto d = Dimension::create(ctx, "d", TILEDB_STRING_ASCII, nullptr, nullptr); @@ -1414,10 +1291,10 @@ TEST_CASE( auto a = Attribute::create(ctx, "a"); schema.add_attribute(a); schema.set_domain(dom); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write - Array array(ctx, array_name, TILEDB_WRITE); + Array array(ctx, array_uri, TILEDB_WRITE); std::vector buff_a = {3, 2, 1, 4}; std::string d_data("ccbbddddaa"); uint64_t d_off[] = {0, 2, 4, 8}; @@ -1432,7 +1309,7 @@ TEST_CASE( array.close(); // Non-empty domain - Array array_r(ctx, array_name, TILEDB_READ); + Array array_r(ctx, array_uri, TILEDB_READ); auto non_empty_domain = array_r.non_empty_domain_var("d"); CHECK(non_empty_domain.first == std::string("aa")); CHECK(non_empty_domain.second == std::string("dddd")); @@ -1521,20 +1398,15 @@ TEST_CASE( // Close array array_r.close(); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + Array::delete_array(ctx, array_uri); } TEST_CASE( "C++ API: Test string dimensions, 1d, col-major", - "[cppapi][sparse][string-dims][1d][col-major]") { - Context ctx; - VFS vfs(ctx); - - std::string array_name = "tes_string_dims"; - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][sparse][string-dims][1d][col-major][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("test_string_dims")}; // Create array auto d = Dimension::create(ctx, "d", TILEDB_STRING_ASCII, nullptr, nullptr); @@ -1546,10 +1418,10 @@ TEST_CASE( schema.set_tile_order(TILEDB_COL_MAJOR); schema.set_cell_order(TILEDB_COL_MAJOR); schema.set_domain(dom); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write - Array array(ctx, array_name, TILEDB_WRITE); + Array array(ctx, array_uri, TILEDB_WRITE); std::vector buff_a = {3, 2, 1, 4}; std::string d_data("ccbbddddaa"); uint64_t d_off[] = {0, 2, 4, 8}; @@ -1564,7 +1436,7 @@ TEST_CASE( array.close(); // Non-empty domain - Array array_r(ctx, array_name, TILEDB_READ); + Array array_r(ctx, array_uri, TILEDB_READ); auto non_empty_domain = array_r.non_empty_domain_var("d"); CHECK(non_empty_domain.first == std::string("aa")); CHECK(non_empty_domain.second == std::string("dddd")); @@ -1589,21 +1461,14 @@ TEST_CASE( // Close array array_r.close(); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Sparse global order, dimension only read", - "[cppapi][sparse][global][read][dimension-only]") { - bool serialize = false, refactored_query_v2 = false; - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][sparse][global][read][dimension-only][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -1612,42 +1477,24 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_SPARSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = {1}; - std::vector coords_w = {0, 0}; - Array array_w(ctx, array_name, TILEDB_WRITE); + std::vector rows_w = {0}; + std::vector cols_w = {0}; + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); - query_w.set_coordinates(coords_w) + query_w.set_data_buffer("rows", rows_w) + .set_data_buffer("cols", cols_w) .set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("a", data_w); - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialize, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); - + query_w.submit_and_finalize(); array_w.close(); // Read - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector rows(1); @@ -1658,21 +1505,14 @@ TEST_CASE( array.close(); REQUIRE(rows[0] == 0); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Unordered with dups, dimension only read", - "[cppapi][sparse][unordered][dups][read][dimension-only]") { - bool serialize = false, refactored_query_v2 = false; - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][sparse][unordered][dups][read][dimension-only][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -1682,43 +1522,24 @@ TEST_CASE( schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); schema.set_allows_dups(true); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = {1}; - std::vector coords_w = {0, 0}; - Array array_w(ctx, array_name, TILEDB_WRITE); + std::vector rows_w = {0}; + std::vector cols_w = {0}; + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); - query_w.set_coordinates(coords_w) + query_w.set_data_buffer("rows", rows_w) + .set_data_buffer("cols", cols_w) .set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("a", data_w); - - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialize, - refactored_query_v2); - - REQUIRE(rc == TILEDB_OK); + query_w.submit_and_finalize(); array_w.close(); // Read - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector rows(1); @@ -1729,20 +1550,14 @@ TEST_CASE( array.close(); REQUIRE(rows[0] == 0); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Read subarray with multiple ranges", - "[cppapi][dense][multi-range]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][dense][multi-range][rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -1751,12 +1566,12 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_subarray(Subarray(ctx, array_w).set_subarray({0, 3, 0, 3})) .set_layout(TILEDB_ROW_MAJOR) @@ -1765,7 +1580,7 @@ TEST_CASE( array_w.close(); // Read - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); std::vector data(12); query.add_range(0, 0, 1) @@ -1783,19 +1598,14 @@ TEST_CASE( for (int i = 8; i < 12; i++) { REQUIRE(data[i] == i + 5); } - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( - "C++ API: Array open VFS calls, dense", "[cppapi][dense][vfs-calls]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "C++ API: Array open VFS calls, dense", + "[cppapi][dense][vfs-calls][non-rest]") { + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -1804,12 +1614,12 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_subarray(Subarray(ctx, array_w).set_subarray({0, 3, 0, 3})) .set_layout(TILEDB_ROW_MAJOR) @@ -1820,7 +1630,7 @@ TEST_CASE( // Open for read. Stats::reset(); Stats::enable(); - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); array.close(); Stats::disable(); @@ -1845,7 +1655,8 @@ TEST_CASE( } TEST_CASE( - "C++ API: Array open VFS calls, sparse", "[cppapi][sparse][vfs-calls]") { + "C++ API: Array open VFS calls, sparse", + "[cppapi][sparse][vfs-calls][non-rest]") { const std::string array_name = "cpp_unit_array"; Context ctx; VFS vfs(ctx); @@ -1864,10 +1675,12 @@ TEST_CASE( // Write std::vector data_w = {1}; - std::vector coords_w = {0, 0}; + std::vector rows = {0}; + std::vector cols = {0}; Array array_w(ctx, array_name, TILEDB_WRITE); Query query_w(ctx, array_w); - query_w.set_coordinates(coords_w) + query_w.set_data_buffer("rows", rows) + .set_data_buffer("cols", cols) .set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("a", data_w); query_w.submit(); @@ -1901,7 +1714,8 @@ TEST_CASE( std::string::npos); } -TEST_CASE("C++ API: Array write and read from MemFS", "[cppapi][memfs]") { +TEST_CASE( + "C++ API: Array write and read from MemFS", "[cppapi][memfs][non-rest]") { const std::string array_name = "mem://cpp_unit_array"; Context ctx; @@ -1954,7 +1768,7 @@ TEST_CASE("C++ API: Array write and read from MemFS", "[cppapi][memfs]") { TEST_CASE( "C++ API: Array on s3 with empty subfolders", - "[cppapi][s3][empty_subfolders]") { + "[cppapi][s3][empty_subfolders][non-rest]") { const std::string array_bucket = "s3://tiledb-" + random_label() + "/"; const std::string array_name = array_bucket + "cpp_unit_array/"; @@ -2040,7 +1854,7 @@ TEST_CASE( TEST_CASE( "C++ API: Write and read to an array with experimental build enabled", - "[cppapi][array][experimental]") { + "[cppapi][array][experimental][non-rest]") { if constexpr (!is_experimental_build) { return; } @@ -2162,13 +1976,12 @@ TEST_CASE( } TEST_CASE( - "C++ API: Close array with running query", "[cppapi][close-before-read]") { - const std::string array_name = "cpp_unit_array"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "C++ API: Close array with running query", + "[cppapi][close-before-read][non-rest]") { + // async queries not supported on remote arrays (REST) + tiledb::test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; // Create Domain domain(ctx); @@ -2177,12 +1990,12 @@ TEST_CASE( ArraySchema schema(ctx, TILEDB_DENSE); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write std::vector data_w = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_subarray(Subarray(ctx, array_w).set_subarray({0, 3, 0, 3})) .set_layout(TILEDB_ROW_MAJOR) @@ -2191,7 +2004,7 @@ TEST_CASE( array_w.close(); // Open for read. - Array array(ctx, array_name, TILEDB_READ); + Array array(ctx, array_uri, TILEDB_READ); std::vector subarray_read = {0, 3, 0, 3}; std::vector a_read(16); diff --git a/test/src/unit-cppapi-global-order-writes-remote.cc b/test/src/unit-cppapi-global-order-writes-remote.cc index 5feee0c12109..771fcdc3f717 100644 --- a/test/src/unit-cppapi-global-order-writes-remote.cc +++ b/test/src/unit-cppapi-global-order-writes-remote.cc @@ -39,12 +39,7 @@ #include using namespace tiledb; - -#ifndef TILEDB_TESTS_ENABLE_REST -constexpr bool rest_tests = false; -#else -constexpr bool rest_tests = true; -#endif +using namespace tiledb::test; template struct RemoteGlobalOrderWriteFx { @@ -59,21 +54,9 @@ struct RemoteGlobalOrderWriteFx { , submit_cell_count_(submit_cell_count) , total_cell_count_(total_cells) , extent_(extent) - , fs_vec_(test::vfs_test_get_fs_vec()) { - REQUIRE(test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_).ok()); - ctx_ = Context(ctx_c_); - std::string temp_dir = fs_vec_[0]->temp_dir(); - if constexpr (rest_tests) { - array_uri_ = "tiledb://unit/"; - } - array_uri_ += temp_dir + array_name_; - test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir); - } - - ~RemoteGlobalOrderWriteFx() { - Array::delete_array(ctx_, array_uri_); - REQUIRE(test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok()); - } + , array_name_{"global-array-" + std::to_string(total_cell_count_)} + , array_uri_(vfs_test_setup_.array_uri(array_name_)) + , ctx_{vfs_test_setup_.ctx()} {}; // Create a simple dense array void create_array() { @@ -183,7 +166,7 @@ struct RemoteGlobalOrderWriteFx { if (i + submit_cell_count_ < total_cell_count_) { query.submit(); } else { - if constexpr (rest_tests) { + if (vfs_test_setup_.is_rest()) { if (check_finalize_fails) { CHECK_THROWS_WITH( query.finalize(), @@ -379,17 +362,16 @@ struct RemoteGlobalOrderWriteFx { read_array(extent_); } - Context ctx_; bool is_var_; bool is_nullable_; const unsigned submit_cell_count_; const uint64_t total_cell_count_; const uint64_t extent_; - const std::string array_name_ = - "global-array-" + std::to_string(total_cell_count_); - // Full URI initialized using fs_vec_ random temp directory. + const std::string array_name_; + test::VFSTestSetup vfs_test_setup_; std::string array_uri_; + Context ctx_; // Vectors to store all the data wrote to the array. // + We will use these vectors to validate subsequent read. @@ -399,11 +381,6 @@ struct RemoteGlobalOrderWriteFx { std::string var_data_wrote_; std::vector var_offsets_wrote_; std::vector var_validity_wrote_; - - // Vector of supported filsystems - tiledb_ctx_handle_t* ctx_c_{nullptr}; - tiledb_vfs_handle_t* vfs_c_{nullptr}; - const std::vector> fs_vec_; }; typedef std::tuple TestTypes; diff --git a/test/src/unit-rest-enumerations.cc b/test/src/unit-rest-enumerations.cc index c7f13c01b194..e93b42e4caff 100644 --- a/test/src/unit-rest-enumerations.cc +++ b/test/src/unit-rest-enumerations.cc @@ -27,7 +27,7 @@ * * @section DESCRIPTION * - * Tests serialization of Enumerations via a REST server. + * Tests end to end enumerations. */ #include "test/support/src/vfs_helpers.h" @@ -42,30 +42,28 @@ using namespace tiledb; struct RESTEnumerationFx { RESTEnumerationFx(); - ~RESTEnumerationFx(); void create_array(const std::string& array_name); - void rm_array(); - std::string bucket_; + tiledb::test::VFSTestSetup vfs_test_setup_; std::string uri_; - Config cfg_; Context ctx_; - VFS vfs_; }; TEST_CASE_METHOD( RESTEnumerationFx, "Create array test", "[rest][enumeration][create-array]") { - create_array("simple-array-create"); + uri_ = vfs_test_setup_.array_uri("simple-array-create"); + create_array(uri_); } TEST_CASE_METHOD( RESTEnumerationFx, "Simple enumeration query", "[rest][enumeration][simple-query]") { - create_array("simple-query"); + uri_ = vfs_test_setup_.array_uri("simple-query"); + create_array(uri_); Array array(ctx_, uri_, TILEDB_READ); Subarray subarray(ctx_, array); @@ -86,26 +84,30 @@ TEST_CASE_METHOD( REQUIRE(query.submit() == Query::Status::COMPLETE); REQUIRE(attr1_read[1] == 1); REQUIRE(attr1_read[3] == 1); + array.close(); } TEST_CASE_METHOD( RESTEnumerationFx, "Get enumeration", "[rest][enumeration][get-enumeration]") { - create_array("get-enumeration"); + uri_ = vfs_test_setup_.array_uri("get-enumeration"); + create_array(uri_); Array array(ctx_, uri_, TILEDB_READ); auto enmr = ArrayExperimental::get_enumeration(ctx_, array, "my_enum"); std::vector expected = {"fred", "wilma", "barney", "pebbles"}; REQUIRE(enmr.as_vector() == expected); + array.close(); } TEST_CASE_METHOD( RESTEnumerationFx, "Get previously loaded enumeration", "[rest][enumeration][get-enumeration]") { - create_array("get-enumeration"); + uri_ = vfs_test_setup_.array_uri("get-enumeration"); + create_array(uri_); Array array(ctx_, uri_, TILEDB_READ); auto enmr1 = ArrayExperimental::get_enumeration(ctx_, array, "my_enum"); @@ -115,16 +117,19 @@ TEST_CASE_METHOD( std::vector expected = {"fred", "wilma", "barney", "pebbles"}; REQUIRE(enmr2.as_vector() == expected); + array.close(); } TEST_CASE_METHOD( RESTEnumerationFx, "Enumeration Extension", "[rest][enumeration][extension]") { - create_array("get-enumeration"); + uri_ = vfs_test_setup_.array_uri("extension"); + create_array(uri_); Array old_array(ctx_, uri_, TILEDB_READ); auto old_enmr = ArrayExperimental::get_enumeration(ctx_, old_array, "fruit"); + old_array.close(); std::vector fruit = { "apple", "blueberry", "cherry", "durian", "elderberry"}; @@ -137,38 +142,13 @@ TEST_CASE_METHOD( Array new_array(ctx_, uri_, TILEDB_READ); auto enmr = ArrayExperimental::get_enumeration(ctx_, new_array, "fruit"); REQUIRE(enmr.as_vector() == fruit); -} - -Config& setup_config(Config& cfg) { - cfg["vfs.s3.endpoint_override"] = "localhost:9999"; - cfg["vfs.s3.scheme"] = "https"; - cfg["vfs.s3.use_virtual_addressing"] = "false"; - cfg["ssl.verify"] = "false"; - return cfg; + new_array.close(); } RESTEnumerationFx::RESTEnumerationFx() - : bucket_("s3://enumeration-tests") - , ctx_(setup_config(cfg_)) - , vfs_(ctx_) { - if (!vfs_.is_bucket(bucket_)) { - vfs_.create_bucket(bucket_); - } -} - -RESTEnumerationFx::~RESTEnumerationFx() { - rm_array(); - if (vfs_.is_bucket(bucket_)) { - vfs_.remove_bucket(bucket_); - } -} + : ctx_(vfs_test_setup_.ctx()){}; void RESTEnumerationFx::create_array(const std::string& array_name) { - uri_ = "tiledb://unit/" + bucket_ + "/" + array_name; - - // Ensure that no array exists at uri_ - rm_array(); - // Create a simple array for testing. This ends up with just five elements in // the array. dim is an int32_t dimension, attr1 is an enumeration with string // values and int32_t attribute values. attr2 is a float attribute. @@ -206,14 +186,14 @@ void RESTEnumerationFx::create_array(const std::string& array_name) { AttributeExperimental::set_enumeration_name(ctx_, attr3, "fruit"); schema.add_attribute(attr3); - Array::create(uri_, schema); + Array::create(array_name, schema); // Attribute data std::vector attr1_values = {0, 1, 2, 1, 0}; std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; std::vector attr3_values = {0, 1, 2, 3, 4}; - Array array(ctx_, uri_, TILEDB_WRITE); + Array array(ctx_, array_name, TILEDB_WRITE); Subarray subarray(ctx_, array); subarray.set_subarray({1, 5}); Query query(ctx_, array); @@ -226,10 +206,3 @@ void RESTEnumerationFx::create_array(const std::string& array_name) { query.finalize(); array.close(); } - -void RESTEnumerationFx::rm_array() { - auto obj = Object::object(ctx_, uri_); - if (obj.type() == Object::Type::Array) { - Array::delete_array(ctx_, uri_); - } -} diff --git a/test/src/unit.cc b/test/src/unit.cc index 8ae943a718fd..a419d5a9a978 100644 --- a/test/src/unit.cc +++ b/test/src/unit.cc @@ -21,7 +21,7 @@ int main(const int argc, char** const argv) { // Define acceptable VFS values. const std::vector vfs_fs = { - "native", "s3", "hdfs", "azure", "gcs"}; + "native", "s3", "hdfs", "azure", "gcs", "rest-s3"}; // Build a pipe-separated string of acceptable VFS values. std::ostringstream vfs_fs_oss; diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index f7192213ad9e..bef37f282a45 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -796,54 +796,27 @@ void get_supported_fs( bool* s3_supported, bool* hdfs_supported, bool* azure_supported, - bool* gcs_supported) { + bool* gcs_supported, + bool* rest_s3_supported) { // Override VFS support if the user used the '--vfs' command line argument. if (g_vfs.empty()) { *s3_supported = tiledb::sm::filesystem::s3_enabled; *hdfs_supported = tiledb::sm::filesystem::hdfs_enabled; *azure_supported = tiledb::sm::filesystem::azure_enabled; *gcs_supported = tiledb::sm::filesystem::gcs_enabled; + *rest_s3_supported = false; } else { if (!(g_vfs == "native" || g_vfs == "s3" || g_vfs == "hdfs" || - g_vfs == "azure" || g_vfs == "gcs")) { + g_vfs == "azure" || g_vfs == "gcs" || g_vfs == "rest-s3")) { throw std::runtime_error( "Failed to get supported fs. Invalid --vfs command line argument."); } - if (g_vfs == "native") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "s3") { - *s3_supported = true; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "hdfs") { - *s3_supported = false; - *hdfs_supported = true; - *azure_supported = false; - *gcs_supported = false; - } - - if (g_vfs == "azure") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = true; - *gcs_supported = false; - } - - if (g_vfs == "gcs") { - *s3_supported = false; - *hdfs_supported = false; - *azure_supported = false; - *gcs_supported = true; - } + *s3_supported = g_vfs == "s3"; + *hdfs_supported = g_vfs == "hdfs"; + *azure_supported = g_vfs == "azure"; + *gcs_supported = g_vfs == "gcs"; + *rest_s3_supported = g_vfs == "rest-s3"; } } diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index ddf3bc581a7c..81ab41f80f28 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -504,7 +504,8 @@ void get_supported_fs( bool* s3_supported, bool* hdfs_supported, bool* azure_supported, - bool* gcs_supported); + bool* gcs_supported, + bool* rest_s3_supported); /** * Opens an array. diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index 98d831136cd5..ed48e83a1df5 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -58,8 +58,13 @@ std::vector> vfs_test_get_fs_vec() { bool supports_hdfs = false; bool supports_azure = false; bool supports_gcs = false; + bool supports_rest_s3 = false; get_supported_fs( - &supports_s3, &supports_hdfs, &supports_azure, &supports_gcs); + &supports_s3, + &supports_hdfs, + &supports_azure, + &supports_gcs, + &supports_rest_s3); if (supports_s3) { fs_vec.emplace_back(std::make_unique()); @@ -78,6 +83,10 @@ std::vector> vfs_test_get_fs_vec() { fs_vec.emplace_back(std::make_unique("gs://")); } + if (supports_rest_s3) { + fs_vec.emplace_back(std::make_unique(true)); + } + fs_vec.emplace_back(std::make_unique()); fs_vec.emplace_back(std::make_unique()); @@ -128,16 +137,16 @@ Status vfs_test_close( void vfs_test_remove_temp_dir( tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const std::string& path) { int is_dir = 0; - REQUIRE(tiledb_vfs_is_dir(ctx, vfs, path.c_str(), &is_dir) == TILEDB_OK); + CHECK(tiledb_vfs_is_dir(ctx, vfs, path.c_str(), &is_dir) == TILEDB_OK); if (is_dir) { - REQUIRE(tiledb_vfs_remove_dir(ctx, vfs, path.c_str()) == TILEDB_OK); + CHECK(tiledb_vfs_remove_dir(ctx, vfs, path.c_str()) == TILEDB_OK); } } void vfs_test_create_temp_dir( tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const std::string& path) { vfs_test_remove_temp_dir(ctx, vfs, path); - REQUIRE(tiledb_vfs_create_dir(ctx, vfs, path.c_str()) == TILEDB_OK); + CHECK(tiledb_vfs_create_dir(ctx, vfs, path.c_str()) == TILEDB_OK); } Status SupportedFsS3::prepare_config( @@ -207,6 +216,10 @@ std::string SupportedFsS3::temp_dir() { return temp_dir_; } +bool SupportedFsS3::is_rest() { + return rest_; +} + Status SupportedFsHDFS::prepare_config( tiledb_config_t* config, tiledb_error_t* error) { (void)config; diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index bce5c43d70ca..aeb3c7042923 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -95,7 +95,6 @@ void vfs_test_remove_temp_dir( void vfs_test_create_temp_dir( tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const std::string& path); - /** * This class defines and manipulates * a list of supported filesystems. @@ -149,6 +148,20 @@ class SupportedFs { * @return string directory name */ virtual std::string temp_dir() = 0; + + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + virtual bool is_rest() = 0; + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + virtual bool is_local() = 0; }; /** @@ -157,10 +170,11 @@ class SupportedFs { */ class SupportedFsS3 : public SupportedFs { public: - SupportedFsS3() + SupportedFsS3(bool rest = false) : s3_prefix_("s3://") , s3_bucket_(s3_prefix_ + "tiledb-" + random_label() + "/") - , temp_dir_(s3_bucket_ + "tiledb_test/") { + , temp_dir_(s3_bucket_ + "tiledb_test/") + , rest_(rest) { } ~SupportedFsS3() = default; @@ -204,6 +218,22 @@ class SupportedFsS3 : public SupportedFs { */ virtual std::string temp_dir(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + virtual bool is_rest(); + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return false; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -217,6 +247,9 @@ class SupportedFsS3 : public SupportedFs { /** The directory name of the S3 filesystem. */ std::string temp_dir_; + + /** If the filesystem is accessed via REST. */ + bool rest_; }; /** @@ -269,6 +302,24 @@ class SupportedFsHDFS : public SupportedFs { */ virtual std::string temp_dir(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + inline bool is_rest() { + return false; + } + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return false; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -331,6 +382,24 @@ class SupportedFsAzure : public SupportedFs { */ virtual std::string temp_dir(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + inline bool is_rest() { + return false; + } + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return false; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -398,6 +467,24 @@ class SupportedFsGCS : public SupportedFs { */ virtual std::string temp_dir(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + inline bool is_rest() { + return false; + } + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return false; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -478,6 +565,24 @@ class SupportedFsLocal : public SupportedFs { */ std::string file_prefix(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + inline bool is_rest() { + return false; + } + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return true; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -550,6 +655,24 @@ class SupportedFsMem : public SupportedFs { */ virtual std::string temp_dir(); + /** + * Checks if the filesystem is accessed via REST + * + * @return true if REST, false if not + */ + inline bool is_rest() { + return false; + } + + /** + * Checks if the filesystem is local or remote + * + * @return true if local, false if not + */ + inline bool is_local() { + return true; + } + private: /* ********************************* */ /* ATTRIBUTES */ @@ -749,6 +872,67 @@ struct TemporaryDirectoryFixture { const std::vector> supported_filesystems_; }; +/* + * Class to instantiate when setting up a test case to run for all VFSs and + * REST + * + * Example usage: In the beginning of the test define a VFSTestSetup variable + * and then just use the Context and the rest of the resources it encapsulates + * in the test. The resources will be automatically get freed when the variable + * will get out of scope. + * + * { + * tiledb::test::VFSTestSetup vfs_test_setup{"foo_array"}; + * auto ctx = vfs_test_setup.ctx(); + * Array array(ctx, array_uri, TILEDB_WRITE); + * ... + * } // ctx context is destroyed and VFS directories removed + * + */ +struct VFSTestSetup { + VFSTestSetup(tiledb_config_t* config = nullptr) + : fs_vec(vfs_test_get_fs_vec()) + , ctx_c{nullptr} + , vfs_c{nullptr} + , cfg_c{config} { + vfs_test_init(fs_vec, &ctx_c, &vfs_c, cfg_c).ok(); + temp_dir = fs_vec[0]->temp_dir(); + vfs_test_create_temp_dir(ctx_c, vfs_c, temp_dir); + }; + + bool is_rest() { + return fs_vec[0]->is_rest(); + } + + bool is_local() { + return fs_vec[0]->is_local(); + } + + std::string array_uri(const std::string& array_name) { + return ( + fs_vec[0]->is_rest() ? "tiledb://unit/" + temp_dir + array_name : + temp_dir + array_name); + } + + Context ctx() { + return Context(ctx_c, false); + } + + ~VFSTestSetup() { + vfs_test_remove_temp_dir(ctx_c, vfs_c, temp_dir); + CHECK(vfs_test_close(fs_vec, ctx_c, vfs_c).ok()); + + tiledb_ctx_free(&ctx_c); + tiledb_vfs_free(&vfs_c); + }; + + std::vector> fs_vec; + tiledb_ctx_handle_t* ctx_c; + tiledb_vfs_handle_t* vfs_c; + tiledb_config_handle_t* cfg_c; + std::string temp_dir; +}; + /** * Denies write access to a local filesystem path. * diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index c8adf5ebd270..36bd99269d22 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2937,30 +2937,10 @@ int32_t tiledb_array_encryption_type( if (array_uri == nullptr || encryption_type == nullptr) return TILEDB_ERR; - // For easy reference - auto storage_manager = ctx->storage_manager(); auto uri = tiledb::sm::URI(array_uri); - - // Load URIs from the array directory - optional array_dir; - try { - array_dir.emplace( - storage_manager->resources(), - uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); - } catch (const std::logic_error& le) { - auto st = Status_ArrayDirectoryError(le.what()); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - // Get encryption type tiledb::sm::EncryptionType enc; - throw_if_not_ok( - ctx->storage_manager()->array_get_encryption(array_dir.value(), &enc)); + throw_if_not_ok(ctx->storage_manager()->array_get_encryption(uri, &enc)); *encryption_type = static_cast(enc); diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index dc4046238450..f145892056f4 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -817,10 +817,12 @@ class Config { * together with the open array
* **Default**: true * - `rest.use_refactored_array_open`
+ * **Experimental**
* If true, the new, experimental REST routes and APIs for opening an array * will be used
* **Default**: false * - `rest.use_refactored_array_open_and_query_submit`
+ * **Experimental**
* If true, the new, experimental REST routes and APIs for opening an array * and submitting a query will be used
* **Default**: false diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 1a0cc587c89a..b8aa69e34678 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -775,15 +775,27 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_from_name( } Status StorageManagerCanonical::array_get_encryption( - const ArrayDirectory& array_dir, EncryptionType* encryption_type) { - const URI& uri = array_dir.uri(); - + const URI& uri, EncryptionType* encryption_type) { if (uri.is_invalid()) { return logger_->status(Status_StorageManagerError( "Cannot get array encryption; Invalid array URI")); } - const URI& schema_uri = array_dir.latest_array_schema_uri(); + if (uri.is_tiledb()) { + throw std::invalid_argument( + "Getting the encryption type of remote arrays is not supported."); + } + + // Load URIs from the array directory + optional array_dir; + array_dir.emplace( + resources_, + uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); + + const URI& schema_uri = array_dir->latest_array_schema_uri(); // Read tile header auto&& header = diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 4309f946e320..9f5017d7a2d1 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -372,13 +372,11 @@ class StorageManagerCanonical { /** * Retrieves the encryption type from an array. * - * @param array_dir The ArrayDirectory object used to retrieve the - * various URIs in the array directory. + * @param uri The URI of the array. * @param encryption_type Set to the encryption type of the array. * @return Status */ - Status array_get_encryption( - const ArrayDirectory& array_dir, EncryptionType* encryption_type); + Status array_get_encryption(const URI& uri, EncryptionType* encryption_type); /** * Pushes an async query to the queue. From d9ad8b486572a5f5fe89886adcd583f95b1e4796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Thu, 28 Mar 2024 15:21:59 +0100 Subject: [PATCH 264/456] Move HDFS into nightly CI. (#4835) This PR moves HDFS CI into `nightly-test.yml`. The HDFS CI is starting to take too long and since we announce deprecation, moving into nightly CI is ok. Example run: https://github.com/dudoslav/TileDB/actions/runs/8465016892 --- TYPE: NO_HISTORY DESC: Move HDFS CI job into nightly CI. --- .github/workflows/full-ci.yml | 21 +++++---------------- .github/workflows/nightly-test.yml | 14 +++++++++++++- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index e071dda3c65b..e09438bd48a4 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -54,16 +54,6 @@ jobs: bootstrap_args: '--enable-serialization --enable-release-symbols' ci4: - uses: ./.github/workflows/ci-linux_mac.yml - with: - ci_backend: HDFS - matrix_image: ubuntu-22.04 - matrix_compiler_cc: 'gcc-13' - matrix_compiler_cxx: 'g++-13' - timeout: 120 - bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' - - ci5: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: S3 @@ -71,7 +61,7 @@ jobs: timeout: 120 bootstrap_args: '--enable=s3,serialization,tools --enable-release-symbols' - ci6: + ci5: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: GCS @@ -79,14 +69,14 @@ jobs: timeout: 120 bootstrap_args: '--enable-gcs --enable-release-symbols' - ci7: + ci6: uses: ./.github/workflows/ci-linux_mac.yml with: ci_option: Experimental matrix_image: ubuntu-22.04 bootstrap_args: '--enable=experimental-features,serialization --enable-release-symbols' - ci8: + ci7: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: S3 @@ -94,7 +84,7 @@ jobs: timeout: 120 bootstrap_args: '--enable-s3 --enable-release-symbols' - ci9: + ci8: uses: ./.github/workflows/ci-linux_mac.yml with: ci_option: ASAN @@ -105,7 +95,7 @@ jobs: bootstrap_args: '--enable-serialization' asan: true - ci10: + ci9: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: AZURE @@ -152,7 +142,6 @@ jobs: ci7, ci8, ci9, - ci10, ci_manylinux, ci_msvc, backward_compatibility, diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index db03f78205ed..323985d64508 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -65,11 +65,23 @@ jobs: run: | cmake --build build --target check --config ${{ matrix.config || 'Release' }} + test_hdfs: + uses: ./.github/workflows/ci-linux_mac.yml + with: + ci_backend: HDFS + matrix_image: ubuntu-22.04 + matrix_compiler_cc: 'gcc-13' + matrix_compiler_cxx: 'g++-13' + timeout: 180 + bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' + create_issue_on_fail: permissions: issues: write runs-on: ubuntu-latest - needs: test + needs: + - test + - test_hdfs if: failure() || cancelled() steps: - name: Checkout TileDB `dev` From 723bb80a1d5b59ea42acb040dd8f113fcaba4f35 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 29 Mar 2024 05:52:31 -0400 Subject: [PATCH 265/456] Add REST CI test to read/write group metadata. (#4833) Add REST CI test to read/write group metadata. --- TYPE: NO_HISTORY --- test/src/unit-capi-group.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 802faa304970..4c1dee4ecda0 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB Inc. + * @copyright Copyright (c) 2022-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -411,12 +411,14 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( GroupFx, - "C API: Group Metadata, write/read", - "[capi][group][metadata][read]") { + "C API: Group Metadata, write/read, rest", + "[capi][group][metadata][rest][regression][sc-4821]") { // Create and open group in write mode // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(temp_dir); + std::string fs_temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(fs_temp_dir); + std::string temp_dir = + fs_vec_[0]->is_rest() ? "tiledb://unit/" + fs_temp_dir : fs_temp_dir; std::string group1_uri = temp_dir + "group1"; REQUIRE(tiledb_group_create(ctx_, group1_uri.c_str()) == TILEDB_OK); @@ -510,7 +512,7 @@ TEST_CASE_METHOD( rc = tiledb_group_close(ctx_, group); REQUIRE(rc == TILEDB_OK); tiledb_group_free(&group); - remove_temp_dir(temp_dir); + remove_temp_dir(fs_temp_dir); } TEST_CASE_METHOD( From b29d6098ff3f893107928bd072e9d968e1e484a0 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Fri, 29 Mar 2024 07:40:47 -0600 Subject: [PATCH 266/456] Remove `Status` from `class Subarray` (#4837) This PR removes `Status` from all member functions of `class Subarray`. It also removes all occurrence of `Status` from `subarray/subarray.cc` except for the ones necessary to use `parallel_for`. There are a handful of changes that tagged along, although these were minimized to alleviate review. These other changes include (1) adding blocks to `if` statements without them, (2) removal of some inconsequential `const` declarations, (3) removal of some dead code. In almost all cases, follow-on changes were not pursued. One exception is `Query::non_overlapping_ranges`, which is proxy for a subarray function; this PR changes its signature. Another was in the test helper `subarray_equiv`, which requires more rewriting that the rest of the call sites. --- TYPE: NO_HISTORY DESC: Remove `Status` from `class Subarray` --- test/src/unit-CellSlabIter.cc | 4 +- test/src/unit-DenseTiler.cc | 3 +- test/src/unit-ReadCellSlabIter.cc | 16 +- test/src/unit-Subarray.cc | 10 +- test/src/unit-SubarrayPartitioner-sparse.cc | 26 +- test/src/unit-cppapi-subarray.cc | 12 +- test/support/src/helpers.cc | 112 +-- test/support/src/helpers.h | 4 - tiledb/sm/c_api/tiledb.cc | 71 +- tiledb/sm/c_api/tiledb_filestore.cc | 11 +- .../dimension_label/dimension_label_query.cc | 10 +- tiledb/sm/query/legacy/cell_slab_iter.cc | 4 +- tiledb/sm/query/legacy/reader.cc | 4 +- tiledb/sm/query/query.cc | 32 +- tiledb/sm/query/query.h | 2 +- tiledb/sm/query/readers/dense_reader.cc | 2 +- .../query/readers/sparse_index_reader_base.cc | 7 +- tiledb/sm/query_plan/test/unit_query_plan.cc | 2 +- tiledb/sm/serialization/query.cc | 10 +- tiledb/sm/subarray/subarray.cc | 923 ++++++++---------- tiledb/sm/subarray/subarray.h | 129 +-- tiledb/sm/subarray/subarray_partitioner.cc | 67 +- .../sm/subarray/test/unit_add_ranges_list.cc | 8 +- tiledb/sm/subarray/tile_cell_slab_iter.cc | 4 +- 24 files changed, 627 insertions(+), 846 deletions(-) diff --git a/test/src/unit-CellSlabIter.cc b/test/src/unit-CellSlabIter.cc index 775bdea910ed..1f0a32cfba89 100644 --- a/test/src/unit-CellSlabIter.cc +++ b/test/src/unit-CellSlabIter.cc @@ -314,7 +314,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{5, 15, 3, 5, 11, 14}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); uint64_t tile_coords_0[] = {0}; uint64_t tile_coords_1[] = {1}; @@ -434,7 +434,7 @@ TEST_CASE_METHOD( {1, 2, 5, 8}, }; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); check_iter(subarray, c_cell_slabs); diff --git a/test/src/unit-DenseTiler.cc b/test/src/unit-DenseTiler.cc index f73f2834fed7..6177bf4685ac 100644 --- a/test/src/unit-DenseTiler.cc +++ b/test/src/unit-DenseTiler.cc @@ -150,7 +150,8 @@ void DenseTilerFx::add_ranges( uint64_t range_size, tiledb::sm::Subarray* subarray) { for (size_t i = 0; i < ranges.size(); ++i) - CHECK(subarray->add_range((uint32_t)i, Range(ranges[i], range_size)).ok()); + CHECK_NOTHROW( + subarray->add_range((uint32_t)i, Range(ranges[i], range_size))); } void DenseTilerFx::open_array( diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index b84c0a58bec0..4e736400aafd 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -247,7 +247,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{5, 15}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {1, 100}; @@ -321,7 +321,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{5, 15}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {20, 30}; @@ -395,7 +395,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{5, 15, 3, 5, 11, 14}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice_1 = {5, 12}; @@ -480,7 +480,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{3, 15, 18, 20}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {3, 12}; @@ -704,7 +704,7 @@ TEST_CASE_METHOD( Subarray subarray; SubarrayRanges ranges = {{2, 3}, {2, 6}}; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {1, 6, 1, 6}; @@ -890,7 +890,7 @@ TEST_CASE_METHOD( Subarray subarray; SubarrayRanges ranges = {{2, 3}, {2, 6}}; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {6, 6, 6, 6}; @@ -1089,7 +1089,7 @@ TEST_CASE_METHOD( Subarray subarray; SubarrayRanges ranges = {{2, 3}, {2, 6}}; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice = {3, 6, 5, 6}; @@ -1332,7 +1332,7 @@ TEST_CASE_METHOD( Subarray subarray; SubarrayRanges ranges = {{3, 5}, {2, 4, 5, 6}}; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Create result space tiles std::vector slice_1 = {3, 5, 2, 4}; diff --git a/test/src/unit-Subarray.cc b/test/src/unit-Subarray.cc index 0263a685de37..e24cbdd243b2 100644 --- a/test/src/unit-Subarray.cc +++ b/test/src/unit-Subarray.cc @@ -125,7 +125,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{5, 7, 6, 15, 33, 43}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); // Prepare correct tile coordinates std::vector> c_tile_coords; @@ -248,7 +248,7 @@ TEST_CASE_METHOD( SubarrayRanges ranges = {{2, 2, 6, 10}, {2, 6, 5, 10}}; Layout subarray_layout = Layout::ROW_MAJOR; create_subarray(array_->array_, ranges, subarray_layout, &subarray); - CHECK(subarray.compute_tile_coords().ok()); + CHECK_NOTHROW(subarray.compute_tile_coords()); auto tile_coords = subarray.tile_coords(); CHECK(tile_coords == c_tile_coords); @@ -310,11 +310,11 @@ TEST_CASE_METHOD( subarray.crop_to_tile(&tile_coords[0], Layout::ROW_MAJOR); const Range* range = nullptr; CHECK(cropped_subarray.range_num() == 2); - CHECK(cropped_subarray.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(cropped_subarray.get_range(0, 0, &range)); CHECK(!memcmp(range->data(), &c_range_0_0[0], 2 * sizeof(uint64_t))); - CHECK(cropped_subarray.get_range(1, 0, &range).ok()); + CHECK_NOTHROW(cropped_subarray.get_range(1, 0, &range)); CHECK(!memcmp(range->data(), &c_range_1_0[0], 2 * sizeof(uint64_t))); - CHECK(cropped_subarray.get_range(1, 1, &range).ok()); + CHECK_NOTHROW(cropped_subarray.get_range(1, 1, &range)); CHECK(!memcmp(range->data(), &c_range_1_1[0], 2 * sizeof(uint64_t))); close_array(ctx_, array_); diff --git a/test/src/unit-SubarrayPartitioner-sparse.cc b/test/src/unit-SubarrayPartitioner-sparse.cc index c13cd854cbbd..8001bc6bc1ea 100644 --- a/test/src/unit-SubarrayPartitioner-sparse.cc +++ b/test/src/unit-SubarrayPartitioner-sparse.cc @@ -2287,7 +2287,7 @@ TEST_CASE_METHOD( // Check unsplittable tiledb::sm::Subarray subarray( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray.add_range(0, Range("bb", "bb"), true).ok()); + CHECK_NOTHROW(subarray.add_range(0, Range("bb", "bb"), true)); ThreadPool tp(4); Config config; SubarrayPartitioner partitioner( @@ -2315,7 +2315,7 @@ TEST_CASE_METHOD( auto partition = partitioner.current(); CHECK(partition.range_num() == 1); const Range* range = nullptr; - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("bb", 2)); CHECK(range->end_str() == std::string("bb", 2)); @@ -2323,7 +2323,7 @@ TEST_CASE_METHOD( // Check full tiledb::sm::Subarray subarray_full( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray_full.add_range(0, Range("a", "bb"), true).ok()); + CHECK_NOTHROW(subarray_full.add_range(0, Range("a", "bb"), true)); SubarrayPartitioner partitioner_full( &config, subarray_full, @@ -2342,7 +2342,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_full.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("a", 1)); CHECK(range->end_str() == std::string("bb", 2)); @@ -2350,7 +2350,7 @@ TEST_CASE_METHOD( // Check split tiledb::sm::Subarray subarray_split( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray_split.add_range(0, Range("a", "bb"), true).ok()); + CHECK_NOTHROW(subarray_split.add_range(0, Range("a", "bb"), true)); SubarrayPartitioner partitioner_split( &config, subarray_split, @@ -2370,7 +2370,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_split.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("a", 1)); CHECK(range->end_str() == std::string("a\x7F", 2)); @@ -2378,7 +2378,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_split.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("b", 1)); CHECK(range->end_str() == std::string("bb", 2)); @@ -2387,7 +2387,7 @@ TEST_CASE_METHOD( // Check no split 2 MBRs tiledb::sm::Subarray subarray_no_split( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray_no_split.add_range(0, Range("bb", "cc"), true).ok()); + CHECK_NOTHROW(subarray_no_split.add_range(0, Range("bb", "cc"), true)); SubarrayPartitioner partitioner_no_split( &config, subarray_no_split, @@ -2408,7 +2408,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_no_split.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("bb", 2)); CHECK(range->end_str() == std::string("cc", 2)); @@ -2416,7 +2416,7 @@ TEST_CASE_METHOD( // Check split 2 MBRs tiledb::sm::Subarray subarray_split_2( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray_split_2.add_range(0, Range("bb", "cc"), true).ok()); + CHECK_NOTHROW(subarray_split_2.add_range(0, Range("bb", "cc"), true)); SubarrayPartitioner partitioner_split_2( &config, subarray_split_2, @@ -2437,7 +2437,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_split_2.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("bb", 2)); CHECK(range->end_str() == std::string("b\x7F", 2)); @@ -2445,7 +2445,7 @@ TEST_CASE_METHOD( CHECK(!unsplittable); partition = partitioner_split_2.current(); CHECK(partition.range_num() == 1); - CHECK(partition.get_range(0, 0, &range).ok()); + CHECK_NOTHROW(partition.get_range(0, 0, &range)); CHECK(range != nullptr); CHECK(range->start_str() == std::string("c", 1)); CHECK(range->end_str() == std::string("cc", 2)); @@ -2554,7 +2554,7 @@ TEST_CASE_METHOD( tiledb::sm::Subarray subarray( array->array_.get(), layout, &g_helper_stats, g_helper_logger()); - CHECK(subarray.add_range(0, Range("cc", "ccd"), true).ok()); + CHECK_NOTHROW(subarray.add_range(0, Range("cc", "ccd"), true)); ThreadPool tp(4); Config config; SubarrayPartitioner partitioner( diff --git a/test/src/unit-cppapi-subarray.cc b/test/src/unit-cppapi-subarray.cc index 96455c04ff7c..cceacda7adbc 100644 --- a/test/src/unit-cppapi-subarray.cc +++ b/test/src/unit-cppapi-subarray.cc @@ -467,29 +467,29 @@ TEST_CASE( SECTION("- Upper bound OOB") { int range[] = {0, 100}; auto r = Range(&range[0], &range[1], sizeof(int)); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(0, r).ok()); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(0, r)); } SECTION("- Lower bound OOB") { int range[] = {-1, 2}; auto r = Range(&range[0], &range[1], sizeof(int)); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(0, r).ok()); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(0, r)); } SECTION("- Second range OOB") { int range[] = {1, 4}; auto r = Range(&range[0], &range[1], sizeof(int)); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(0, r).ok()); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(0, r)); int range2[] = {10, 20}; auto r2 = Range(&range2[0], &range2[1], sizeof(int)); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(1, r2).ok()); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(1, r2)); } SECTION("- Valid ranges") { int range[] = {0, 1}; auto r = Range(&range[0], &range[1], sizeof(int)); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(0, r).ok()); - CHECK(subarray.ptr().get()->subarray_->add_range_unsafe(1, r).ok()); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(0, r)); + CHECK_NOTHROW(subarray.ptr().get()->subarray_->add_range_unsafe(1, r)); expected = TILEDB_OK; } diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index bef37f282a45..b70891aa8cf2 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -308,10 +308,10 @@ void check_subarray( uint64_t dim_range_num = 0; const type::Range* range; for (unsigned i = 0; i < dim_num; ++i) { - CHECK(subarray.get_range_num(i, &dim_range_num).ok()); + CHECK_NOTHROW(subarray.get_range_num(i, &dim_range_num)); CHECK(dim_range_num == ranges[i].size() / 2); for (uint64_t j = 0; j < dim_range_num; ++j) { - CHECK(subarray.get_range(i, j, &range).ok()); + CHECK_NOTHROW(subarray.get_range(i, j, &range)); auto r = (const T*)range->data(); CHECK(r[0] == ranges[i][2 * j]); @@ -359,71 +359,32 @@ void check_subarray( } } -template -void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2) { - CHECK(subarray1.range_num() == subarray2.range_num()); - // Check dim num - auto dim_num1 = subarray1.dim_num(); - auto dim_num2 = subarray2.dim_num(); - CHECK(dim_num1 == dim_num2); - - tiledb::sm::ByteVec sa1bytes, sa2bytes; - //.to_byte_vect() only valid when range_num() == 1, but should be same for - // both and resulting bytes, empty or otherwise, should be the same as well. - CHECK( - subarray1.to_byte_vec(&sa1bytes).ok() == - subarray2.to_byte_vec(&sa2bytes).ok()); - CHECK(sa1bytes == sa2bytes); - - const std::vector> sa1tilecoords = - subarray1.tile_coords(); - const std::vector> sa2tilecoords = - subarray2.tile_coords(); - CHECK(sa1tilecoords == sa2tilecoords); - - // Check ranges - uint64_t dim_range_num1 = 0; - const type::Range* range1; - uint64_t dim_range_num2 = 0; - const type::Range* range2; - if (dim_num1 == dim_num2) { - for (unsigned i = 0; i < dim_num1; ++i) { - CHECK(subarray1.get_range_num(i, &dim_range_num1).ok()); - CHECK(subarray2.get_range_num(i, &dim_range_num2).ok()); - CHECK(dim_range_num1 == dim_range_num2); - if (dim_range_num1 == dim_range_num2) { - for (uint64_t j = 0; j < dim_range_num1; ++j) { - CHECK(subarray1.get_range(i, j, &range1).ok()); - CHECK(subarray2.get_range(i, j, &range2).ok()); - auto r1 = (const T*)range1->data(); - auto r2 = (const T*)range2->data(); - CHECK(r1[0] == r2[0]); - CHECK(r1[1] == r2[1]); - } - } - } - } -} - template bool subarray_equiv( tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2) { bool equiv_state = 1; // assume true - - equiv_state &= (subarray1.range_num() == subarray2.range_num()); + auto rn1 = subarray1.range_num(); + auto rn2 = subarray2.range_num(); + if (rn1 != rn2) { + return false; + } // Check dim num auto dim_num1 = subarray1.dim_num(); auto dim_num2 = subarray2.dim_num(); - equiv_state &= (dim_num1 == dim_num2); + if (dim_num1 != dim_num2) { + return false; + } + /* + * `to_byte_vect()` is only valid when `range_num() == 1`, so only + * convert in that case, since `to_byte_vect()` will throw otherwise + */ tiledb::sm::ByteVec sa1bytes, sa2bytes; - //.to_byte_vect() only valid when range_num() == 1, but should be same for - // both and resulting bytes, empty or otherwise, should be the same as well. - equiv_state &= - (subarray1.to_byte_vec(&sa1bytes).ok() == - subarray2.to_byte_vec(&sa2bytes).ok()); - equiv_state &= (sa1bytes == sa2bytes); + if (rn1 == 1) { + subarray1.to_byte_vec(&sa1bytes); + subarray2.to_byte_vec(&sa2bytes); + equiv_state &= (sa1bytes == sa2bytes); + } const std::vector> sa1tilecoords = subarray1.tile_coords(); @@ -438,13 +399,17 @@ bool subarray_equiv( const type::Range* range2; if (dim_num1 == dim_num2) { for (unsigned i = 0; i < dim_num1; ++i) { - equiv_state &= (subarray1.get_range_num(i, &dim_range_num1).ok()); - equiv_state &= (subarray2.get_range_num(i, &dim_range_num2).ok()); + try { + subarray1.get_range_num(i, &dim_range_num1); + subarray2.get_range_num(i, &dim_range_num2); + } catch (...) { + return false; + } equiv_state &= (dim_range_num1 == dim_range_num2); if (dim_range_num1 == dim_range_num2) { for (uint64_t j = 0; j < dim_range_num1; ++j) { - CHECK(subarray1.get_range(i, j, &range1).ok()); - CHECK(subarray2.get_range(i, j, &range2).ok()); + CHECK_NOTHROW(subarray1.get_range(i, j, &range1)); + CHECK_NOTHROW(subarray2.get_range(i, j, &range2)); auto r1 = (const T*)range1->data(); auto r2 = (const T*)range2->data(); equiv_state &= (r1[0] == r2[0]); @@ -840,7 +805,7 @@ void create_subarray( auto dim_range_num = ranges[d].size() / 2; for (size_t j = 0; j < dim_range_num; ++j) { type::Range range(&ranges[d][2 * j], 2 * sizeof(T)); - CHECK(ret.add_range(d, std::move(range), true).ok()); + CHECK_NOTHROW(ret.add_range(d, std::move(range), true)); } } @@ -1938,27 +1903,6 @@ template void check_subarray( template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); -template void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); - template bool subarray_equiv( tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); template bool subarray_equiv( diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 81ab41f80f28..3aecbc6f817e 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -251,10 +251,6 @@ template void check_subarray( tiledb::Subarray& subarray, const SubarrayRanges& ranges); -template -void check_subarray_equiv( - tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); - template bool subarray_equiv( tiledb::sm::Subarray& subarray1, tiledb::sm::Subarray& subarray2); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 36bd99269d22..944349b769b0 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1753,10 +1753,7 @@ int32_t tiledb_subarray_set_coalesce_ranges( // Sanity check if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok( - subarray->subarray_->set_coalesce_ranges(coalesce_ranges != 0)); - + subarray->subarray_->set_coalesce_ranges(coalesce_ranges != 0); return TILEDB_OK; } @@ -1767,7 +1764,7 @@ int32_t tiledb_subarray_set_subarray( if (sanity_check(ctx, subarray_obj) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok(subarray_obj->subarray_->set_subarray(subarray_vals)); + subarray_obj->subarray_->set_subarray(subarray_vals); return TILEDB_OK; } @@ -1782,7 +1779,7 @@ int32_t tiledb_subarray_add_range( if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok(subarray->subarray_->add_range(dim_idx, start, end, stride)); + subarray->subarray_->add_range(dim_idx, start, end, stride); return TILEDB_OK; } @@ -1795,9 +1792,7 @@ int32_t tiledb_subarray_add_point_ranges( uint64_t count) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->add_point_ranges(dim_idx, start, count)); - + subarray->subarray_->add_point_ranges(dim_idx, start, count); return TILEDB_OK; } @@ -1810,10 +1805,7 @@ int32_t tiledb_subarray_add_range_by_name( const void* stride) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok( - subarray->subarray_->add_range_by_name(dim_name, start, end, stride)); - + subarray->subarray_->add_range_by_name(dim_name, start, end, stride); return TILEDB_OK; } @@ -1827,10 +1819,7 @@ int32_t tiledb_subarray_add_range_var( uint64_t end_size) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->add_range_var( - dim_idx, start, start_size, end, end_size)); - + subarray->subarray_->add_range_var(dim_idx, start, start_size, end, end_size); return TILEDB_OK; } @@ -1844,10 +1833,8 @@ int32_t tiledb_subarray_add_range_var_by_name( uint64_t end_size) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->add_range_var_by_name( - dim_name, start, start_size, end, end_size)); - + subarray->subarray_->add_range_var_by_name( + dim_name, start, start_size, end, end_size); return TILEDB_OK; } @@ -1858,9 +1845,7 @@ int32_t tiledb_subarray_get_range_num( uint64_t* range_num) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->get_range_num(dim_idx, range_num)); - + subarray->subarray_->get_range_num(dim_idx, range_num); return TILEDB_OK; } @@ -1871,10 +1856,7 @@ int32_t tiledb_subarray_get_range_num_from_name( uint64_t* range_num) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok( - subarray->subarray_->get_range_num_from_name(dim_name, range_num)); - + subarray->subarray_->get_range_num_from_name(dim_name, range_num); return TILEDB_OK; } @@ -1888,10 +1870,7 @@ int32_t tiledb_subarray_get_range( const void** stride) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok( - subarray->subarray_->get_range(dim_idx, range_idx, start, end, stride)); - + subarray->subarray_->get_range(dim_idx, range_idx, start, end, stride); return TILEDB_OK; } @@ -1904,10 +1883,8 @@ int32_t tiledb_subarray_get_range_var_size( uint64_t* end_size) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->get_range_var_size( - dim_idx, range_idx, start_size, end_size)); - + subarray->subarray_->get_range_var_size( + dim_idx, range_idx, start_size, end_size); return TILEDB_OK; } @@ -1921,10 +1898,8 @@ int32_t tiledb_subarray_get_range_from_name( const void** stride) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->get_range_from_name( - dim_name, range_idx, start, end, stride)); - + subarray->subarray_->get_range_from_name( + dim_name, range_idx, start, end, stride); return TILEDB_OK; } @@ -1937,10 +1912,8 @@ int32_t tiledb_subarray_get_range_var_size_from_name( uint64_t* end_size) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->get_range_var_size_from_name( - dim_name, range_idx, start_size, end_size)); - + subarray->subarray_->get_range_var_size_from_name( + dim_name, range_idx, start_size, end_size); return TILEDB_OK; } @@ -1953,10 +1926,7 @@ int32_t tiledb_subarray_get_range_var( void* end) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok( - subarray->subarray_->get_range_var(dim_idx, range_idx, start, end)); - + subarray->subarray_->get_range_var(dim_idx, range_idx, start, end); return TILEDB_OK; } @@ -1969,10 +1939,7 @@ int32_t tiledb_subarray_get_range_var_from_name( void* end) { if (sanity_check(ctx, subarray) == TILEDB_ERR) return TILEDB_ERR; - - throw_if_not_ok(subarray->subarray_->get_range_var_from_name( - dim_name, range_idx, start, end)); - + subarray->subarray_->get_range_var_from_name(dim_name, range_idx, start, end); return TILEDB_OK; } diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index f8c933659e01..d68d103b8155 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -296,7 +296,7 @@ int32_t tiledb_filestore_uri_import( uint64_t range_arr[] = {static_cast(0), last_space_tile_boundary}; tiledb::type::Range subarray_range( static_cast(range_arr), sizeof(uint64_t) * 2); - throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + subarray.add_range(0, std::move(subarray_range)); query.set_subarray(subarray); auto tiledb_cloud_fix = [&](uint64_t start, uint64_t end) { @@ -312,8 +312,7 @@ int32_t tiledb_filestore_uri_import( uint64_t cloud_fix_range_arr[] = {start, end}; tiledb::type::Range subarray_range_cloud_fix( static_cast(cloud_fix_range_arr), sizeof(uint64_t) * 2); - throw_if_not_ok( - subarray_cloud_fix.add_range(0, std::move(subarray_range_cloud_fix))); + subarray_cloud_fix.add_range(0, std::move(subarray_range_cloud_fix)); query.set_subarray(subarray_cloud_fix); uint64_t data_buff_len = end - start + 1; throw_if_not_ok(query.set_data_buffer( @@ -440,7 +439,7 @@ int32_t tiledb_filestore_uri_export( uint64_t subarray_range_arr[] = {start_range, end_range}; tiledb::type::Range subarray_range( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); - throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + subarray.add_range(0, std::move(subarray_range)); tiledb::sm::Query query(context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); @@ -558,7 +557,7 @@ int32_t tiledb_filestore_buffer_import( static_cast(0), static_cast(size - 1)}; tiledb::type::Range subarray_range( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); - throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + subarray.add_range(0, std::move(subarray_range)); query.set_subarray(subarray); uint64_t size_tmp = size; @@ -620,7 +619,7 @@ int32_t tiledb_filestore_buffer_export( static_cast(offset), static_cast(offset + size - 1)}; tiledb::type::Range subarray_range( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); - throw_if_not_ok(subarray.add_range(0, std::move(subarray_range))); + subarray.add_range(0, std::move(subarray_range)); tiledb::sm::Query query(context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); diff --git a/tiledb/sm/query/dimension_label/dimension_label_query.cc b/tiledb/sm/query/dimension_label/dimension_label_query.cc index dff7f6577c0b..e6563af7f140 100644 --- a/tiledb/sm/query/dimension_label/dimension_label_query.cc +++ b/tiledb/sm/query/dimension_label/dimension_label_query.cc @@ -167,8 +167,7 @@ void DimensionLabelQuery::initialize_read_labels_query( if (!parent_subarray.is_default(dim_idx) && !parent_subarray.has_label_ranges(dim_idx)) { Subarray subarray{*this->subarray()}; - throw_if_not_ok(subarray.set_ranges_for_dim( - 0, parent_subarray.ranges_for_dim(dim_idx))); + subarray.set_ranges_for_dim(0, parent_subarray.ranges_for_dim(dim_idx)); set_subarray(subarray); } @@ -193,8 +192,7 @@ void DimensionLabelQuery::initialize_ordered_write_query( // Set the subarray if it has index ranges added to it. if (!parent_subarray.is_default(dim_idx)) { Subarray subarray{*this->subarray()}; - throw_if_not_ok(subarray.set_ranges_for_dim( - 0, parent_subarray.ranges_for_dim(dim_idx))); + subarray.set_ranges_for_dim(0, parent_subarray.ranges_for_dim(dim_idx)); if (subarray.range_num() > 1) { throw DimensionLabelQueryStatusException( "Dimension label writes can only be set for a single range."); @@ -209,8 +207,8 @@ void DimensionLabelQuery::initialize_ordered_write_query( uint64_t count = *index_buffer.buffer_size_ / datatype_size(array_schema().dimension_ptr(0)->type()); Subarray subarray{*this->subarray()}; - throw_if_not_ok(subarray.set_coalesce_ranges(true)); - throw_if_not_ok(subarray.add_point_ranges(0, index_buffer.buffer_, count)); + subarray.set_coalesce_ranges(true); + subarray.add_point_ranges(0, index_buffer.buffer_, count); if (subarray.range_num() > 1) { throw DimensionLabelQueryStatusException( "The dimension data must contain consecutive points when writing to " diff --git a/tiledb/sm/query/legacy/cell_slab_iter.cc b/tiledb/sm/query/legacy/cell_slab_iter.cc index da941cc68a9c..f4c1b723e519 100644 --- a/tiledb/sm/query/legacy/cell_slab_iter.cc +++ b/tiledb/sm/query/legacy/cell_slab_iter.cc @@ -252,12 +252,12 @@ Status CellSlabIter::init_ranges() { ranges_.resize(dim_num); for (unsigned d = 0; d < dim_num; ++d) { auto dim_dom = (const T*)array_domain[d].data(); - RETURN_NOT_OK(subarray_->get_range_num(d, &range_num)); + subarray_->get_range_num(d, &range_num); ranges_[d].reserve(range_num); tile_extent = *(const T*)array_schema.domain().tile_extent(d).data(); dim_domain_start = dim_dom[0]; for (uint64_t j = 0; j < range_num; ++j) { - RETURN_NOT_OK(subarray_->get_range(d, j, &r)); + subarray_->get_range(d, j, &r); create_ranges( (const T*)(*r).data(), tile_extent, dim_domain_start, &ranges_[d]); } diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index f171a3c159a5..b789aa6392e7 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -1575,7 +1575,7 @@ Status Reader::compute_result_cell_slabs_global( tile_subarrays.emplace_back( subarray.crop_to_tile((const T*)&tc[0], cell_order)); auto& tile_subarray = tile_subarrays.back(); - throw_if_not_ok(tile_subarray.template compute_tile_coords()); + tile_subarray.template compute_tile_coords(); RETURN_NOT_OK(compute_result_cell_slabs_row_col( tile_subarray, @@ -1742,7 +1742,7 @@ Status Reader::dense_read() { std::vector result_tiles; auto& subarray = read_state_.partitioner_.current(); - RETURN_NOT_OK(subarray.compute_tile_coords()); + subarray.compute_tile_coords(); RETURN_NOT_OK(compute_result_cell_slabs( subarray, result_space_tiles, diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 94c7e15575e5..57fa870545fc 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -195,8 +195,9 @@ Status Query::get_est_result_size(const char* name, uint64_t* size) { rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } - return subarray_.get_est_result_size_internal( + subarray_.get_est_result_size_internal( name, size, &config_, storage_manager_->compute_tp()); + return Status::Ok(); } Status Query::get_est_result_size( @@ -226,8 +227,9 @@ Status Query::get_est_result_size( rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } - return subarray_.get_est_result_size( + subarray_.get_est_result_size( name, size_off, size_val, &config_, storage_manager_->compute_tp()); + return Status::Ok(); } Status Query::get_est_result_size_nullable( @@ -267,8 +269,9 @@ Status Query::get_est_result_size_nullable( rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } - return subarray_.get_est_result_size_nullable( + subarray_.get_est_result_size_nullable( name, size_val, size_validity, &config_, storage_manager_->compute_tp()); + return Status::Ok(); } Status Query::get_est_result_size_nullable( @@ -306,13 +309,14 @@ Status Query::get_est_result_size_nullable( rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } - return subarray_.get_est_result_size_nullable( + subarray_.get_est_result_size_nullable( name, size_off, size_val, size_validity, &config_, storage_manager_->compute_tp()); + return Status::Ok(); } std::unordered_map @@ -1372,8 +1376,8 @@ Status Query::set_est_result_size( return LOG_STATUS(Status_SerializationError( "Cannot set estimated result size; Unsupported query type.")); } - - return subarray_.set_est_result_size(est_result_size, max_mem_size); + subarray_.set_est_result_size(est_result_size, max_mem_size); + return Status::Ok(); } Status Query::set_layout(Layout layout) { @@ -1514,7 +1518,7 @@ void Query::set_subarray(const void* subarray) { } // Set the subarray. - throw_if_not_ok(subarray_.set_subarray(subarray)); + subarray_.set_subarray(subarray); } const Subarray* Query::subarray() const { @@ -1567,7 +1571,7 @@ Status Query::set_subarray_unsafe(const NDRange& subarray) { if (!subarray.empty()) { auto dim_num = array_schema_->dim_num(); for (unsigned d = 0; d < dim_num; ++d) - RETURN_NOT_OK(sub.add_range_unsafe(d, subarray[d])); + sub.add_range_unsafe(d, subarray[d]); } assert(layout_ == sub.layout()); @@ -1764,7 +1768,7 @@ bool Query::use_refactored_sparse_unordered_with_dups_reader( layout == Layout::UNORDERED && array_schema.allows_dups(); } -tuple> Query::non_overlapping_ranges() { +bool Query::non_overlapping_ranges() { return subarray_.non_overlapping_ranges(storage_manager_->compute_tp()); } @@ -1875,10 +1879,7 @@ Status Query::create_strategy(bool skip_checks_serialization) { dimension_label_increasing_)); } else if (use_refactored_sparse_unordered_with_dups_reader( layout_, *array_schema_)) { - auto&& [st, non_overlapping_ranges]{Query::non_overlapping_ranges()}; - RETURN_NOT_OK(st); - - if (*non_overlapping_ranges || !subarray_.is_set() || + if (Query::non_overlapping_ranges() || !subarray_.is_set() || subarray_.range_num() == 1) { strategy_ = tdb_unique_ptr(tdb_new( SparseUnorderedWithDupsReader, @@ -1897,10 +1898,7 @@ Status Query::create_strategy(bool skip_checks_serialization) { !array_schema_->dense() && (layout_ == Layout::GLOBAL_ORDER || layout_ == Layout::UNORDERED)) { // Using the reader for unordered queries to do deduplication. - auto&& [st, non_overlapping_ranges]{Query::non_overlapping_ranges()}; - RETURN_NOT_OK(st); - - if (*non_overlapping_ranges || !subarray_.is_set() || + if (Query::non_overlapping_ranges() || !subarray_.is_set() || subarray_.range_num() == 1) { strategy_ = tdb_unique_ptr(tdb_new( SparseGlobalOrderReader, diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index c02cea963581..b8d093310253 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -688,7 +688,7 @@ class Query { Layout layout, const ArraySchema& array_schema); /** Returns if all ranges for this query are non overlapping. */ - tuple> non_overlapping_ranges(); + bool non_overlapping_ranges(); /** Returns true if this is a dense query */ bool is_dense() const; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 60dd2871bdaf..eae579ad9530 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -230,7 +230,7 @@ Status DenseReader::dense_read() { // For easy reference. const auto dim_num = array_schema_.dim_num(); auto& subarray = read_state_.partitioner_.current(); - RETURN_NOT_OK(subarray.compute_tile_coords()); + subarray.compute_tile_coords(); const auto& domain{array_schema_.domain()}; // Cache tile extents. diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index fed07985afd1..7470f3e2ad1a 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -379,10 +379,10 @@ Status SparseIndexReaderBase::load_initial_data() { // Tile ranges computation will not stop if it exceeds memory budget. // This is ok as it is a soft limit and will be taken into consideration // below. - RETURN_NOT_OK(subarray_.precompute_all_ranges_tile_overlap( + subarray_.precompute_all_ranges_tile_overlap( storage_manager_->compute_tp(), read_state_.frag_idx(), - &tmp_read_state_)); + &tmp_read_state_); if (tmp_read_state_.memory_used_tile_ranges() > memory_budget_.ratio_tile_ranges() * memory_budget_.total_budget()) @@ -391,8 +391,7 @@ Status SparseIndexReaderBase::load_initial_data() { } else { for (const auto& [name, _] : aggregates_) { if (array_schema_.is_dim(name)) { - throw_if_not_ok(subarray_.load_relevant_fragment_rtrees( - storage_manager_->compute_tp())); + subarray_.load_relevant_fragment_rtrees(storage_manager_->compute_tp()); break; } } diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index fedbc453a7f0..02776c226f17 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -134,7 +134,7 @@ TEST_CASE_METHOD(QueryPlanFx, "Query plan dump_json", "[query_plan][dump]") { stats::Stats stats("foo"); Subarray subarray(array_shared.get(), &stats, logger_); uint64_t r[2]{0, 1}; - REQUIRE(subarray.add_range(0, r, r + 1, nullptr).ok()); + subarray.add_range(0, r, r + 1, nullptr); query.set_subarray(subarray); std::vector data(2); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 7d98392eddaf..2f76659e335f 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -263,7 +263,7 @@ Status subarray_to_capnp( Status subarray_from_capnp( const capnp::Subarray::Reader& reader, Subarray* subarray) { - RETURN_NOT_OK(subarray->set_coalesce_ranges(reader.getCoalesceRanges())); + subarray->set_coalesce_ranges(reader.getCoalesceRanges()); auto ranges_reader = reader.getRanges(); uint32_t dim_num = ranges_reader.size(); for (uint32_t i = 0; i < dim_num; i++) { @@ -273,14 +273,14 @@ Status subarray_from_capnp( auto data_ptr = data.asBytes(); if (range_reader.hasBufferSizes()) { auto ranges = range_buffers_from_capnp(range_reader); - RETURN_NOT_OK(subarray->set_ranges_for_dim(i, ranges)); + subarray->set_ranges_for_dim(i, ranges); // Set default indicator subarray->set_is_default(i, range_reader.getHasDefaultRange()); } else { // Handle 1.7 style ranges where there is a single range with no sizes Range range(data_ptr.begin(), data.size()); - RETURN_NOT_OK(subarray->set_ranges_for_dim(i, {range})); + subarray->set_ranges_for_dim(i, {range}); subarray->set_is_default(i, range_reader.getHasDefaultRange()); } } @@ -514,12 +514,12 @@ Status subarray_partitioner_from_capnp( partition_info_reader.getSubarray(), &partition_info->partition_)); if (compute_current_tile_overlap) { - throw_if_not_ok(partition_info->partition_.precompute_tile_overlap( + partition_info->partition_.precompute_tile_overlap( partition_info->start_, partition_info->end_, &config, compute_tp, - true)); + true); } } diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index d535b75e4900..b155e12ecb02 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -67,13 +67,12 @@ using namespace tiledb::common; using namespace tiledb::sm::stats; using namespace tiledb::type; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Class for query status exceptions. */ -class SubarrayStatusException : public StatusException { +class SubarrayException : public StatusException { public: - explicit SubarrayStatusException(const std::string& msg) + explicit SubarrayException(const std::string& msg) : StatusException("Subarray", msg) { } }; @@ -189,9 +188,8 @@ void Subarray::add_index_ranges_from_label( const bool is_point_ranges, const void* start, const uint64_t count) { - is_point_ranges ? - throw_if_not_ok(add_point_ranges(dim_idx, start, count, false)) : - throw_if_not_ok(add_ranges_list(dim_idx, start, count, false)); + is_point_ranges ? add_point_ranges(dim_idx, start, count, false) : + add_ranges_list(dim_idx, start, count, false); } void Subarray::add_label_range( @@ -203,7 +201,7 @@ void Subarray::add_label_range( // A label range has already been set on this dimension. Do the following: // * Check this label is the same label that rangers were already set. if (dim_label_ref.name() != label_range_subset_[dim_idx].value().name_) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range] Dimension is already to set to use " "dimension label '" + label_range_subset_[dim_idx].value().name_ + "'"); @@ -217,7 +215,7 @@ void Subarray::add_label_range( // value of the entire domain). if (range_subset_[dim_label_ref.dimension_index()] .is_explicitly_initialized()) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range] Dimension '" + std::to_string(dim_idx) + "' already has ranges set to it."); } @@ -236,7 +234,7 @@ void Subarray::add_label_range( label_range_subset_[dim_idx].value().ranges_.add_range( range, read_range_oob_error); if (!error_status.ok()) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range] Cannot add label range for dimension label '" + dim_label_ref.name() + "'; " + error_status.message()); } @@ -254,17 +252,17 @@ void Subarray::add_label_range( const void* stride) { // Check input range data is valid data. if (start == nullptr || end == nullptr) { - throw SubarrayStatusException("[add_label_range] Invalid range"); + throw SubarrayException("[add_label_range] Invalid range"); } if (stride != nullptr) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range] Setting range stride is currently unsupported"); } // Get dimension label range and check the label is in fact fixed-sized. const auto& dim_label_ref = array_->array_schema_latest().dimension_label(label_name); if (dim_label_ref.label_cell_val_num() == constants::var_num) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range] Cannot add a fixed-sized range to a variable sized " "dimension label"); } @@ -284,14 +282,14 @@ void Subarray::add_label_range_var( // Check the input range data is valid. if ((start == nullptr && start_size != 0) || (end == nullptr && end_size != 0)) { - throw SubarrayStatusException("[add_label_range_var] Invalid range"); + throw SubarrayException("[add_label_range_var] Invalid range"); } // Get the dimension label range and check the label is in fact // variable-sized. const auto& dim_label_ref = array_->array_schema_latest().dimension_label(label_name); if (dim_label_ref.label_cell_val_num() != constants::var_num) { - throw SubarrayStatusException( + throw SubarrayException( "[add_label_range_var] Cannot add a variable-sized range to a " "fixed-sized dimension label"); } @@ -302,12 +300,12 @@ void Subarray::add_label_range_var( err_on_range_oob_); } -Status Subarray::add_range( +void Subarray::add_range( uint32_t dim_idx, Range&& range, const bool read_range_oob_error) { auto dim_num = array_->array_schema_latest().dim_num(); if (dim_idx >= dim_num) { - return logger_->status(Status_SubarrayError( - "Cannot add range to dimension; Invalid dimension index")); + throw SubarrayException( + "Cannot add range to dimension; Invalid dimension index"); } // Must reset the result size and tile overlap @@ -318,20 +316,21 @@ Status Subarray::add_range( auto dim_name = array_->array_schema_latest().dimension_ptr(dim_idx)->name(); auto&& [error_status, oob_warning] = range_subset_[dim_idx].add_range(range, read_range_oob_error); - if (!error_status.ok()) - return logger_->status(Status_SubarrayError( + if (!error_status.ok()) { + throw SubarrayException( "Cannot add range to dimension '" + dim_name + "'; " + - error_status.message())); - if (oob_warning.has_value()) + error_status.message()); + } + if (oob_warning.has_value()) { LOG_WARN(oob_warning.value() + " on dimension '" + dim_name + "'"); + } // Update is default and stats counter. is_default_[dim_idx] = range_subset_[dim_idx].is_implicitly_initialized(); stats_->add_counter("add_range_dim_" + std::to_string(dim_idx), 1); - return Status::Ok(); } -Status Subarray::add_range_unsafe(uint32_t dim_idx, const Range& range) { +void Subarray::add_range_unsafe(uint32_t dim_idx, const Range& range) { // Must reset the result size and tile overlap est_result_size_computed_ = false; tile_overlap_.clear(); @@ -339,19 +338,18 @@ Status Subarray::add_range_unsafe(uint32_t dim_idx, const Range& range) { // Add the range throw_if_not_ok(range_subset_[dim_idx].add_range_unrestricted(range)); is_default_[dim_idx] = range_subset_[dim_idx].is_implicitly_initialized(); - return Status::Ok(); } -Status Subarray::set_subarray(const void* subarray) { +void Subarray::set_subarray(const void* subarray) { if (!array_->array_schema_latest().domain().all_dims_same_type()) - return LOG_STATUS( - Status_SubarrayError("Cannot set subarray; Function not applicable to " - "heterogeneous domains")); + throw SubarrayException( + "Cannot set subarray; Function not applicable to " + "heterogeneous domains"); if (!array_->array_schema_latest().domain().all_dims_fixed()) - return LOG_STATUS( - Status_SubarrayError("Cannot set subarray; Function not applicable to " - "domains with variable-sized dimensions")); + throw SubarrayException( + "Cannot set subarray; Function not applicable to " + "domains with variable-sized dimensions"); add_default_ranges(); if (subarray != nullptr) { @@ -362,12 +360,10 @@ Status Subarray::set_subarray(const void* subarray) { auto r_size = 2 * array_->array_schema_latest().dimension_ptr(d)->coord_size(); Range range(&s_ptr[offset], r_size); - RETURN_NOT_OK(this->add_range(d, std::move(range), err_on_range_oob_)); + this->add_range(d, std::move(range), err_on_range_oob_); offset += r_size; } } - - return Status::Ok(); } void Subarray::set_subarray_unsafe(const void* subarray) { @@ -380,37 +376,38 @@ void Subarray::set_subarray_unsafe(const void* subarray) { auto r_size = 2 * array_->array_schema_latest().dimension_ptr(d)->coord_size(); Range range(&s_ptr[offset], r_size); - throw_if_not_ok(this->add_range_unsafe(d, std::move(range))); + this->add_range_unsafe(d, std::move(range)); offset += r_size; } } } -Status Subarray::add_range( +void Subarray::add_range( unsigned dim_idx, const void* start, const void* end, const void* stride) { if (dim_idx >= this->array_->array_schema_latest().dim_num()) - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Invalid dimension index")); + throw SubarrayException("Cannot add range; Invalid dimension index"); if (has_label_ranges(dim_idx)) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( "Cannot add range to to dimension; A range is already set on a " - "dimension label for this dimension")); + "dimension label for this dimension"); } - if (start == nullptr || end == nullptr) - return LOG_STATUS(Status_SubarrayError("Cannot add range; Invalid range")); + if (start == nullptr || end == nullptr) { + throw SubarrayException("Cannot add range; Invalid range"); + } - if (stride != nullptr) - return LOG_STATUS(Status_SubarrayError( - "Cannot add range; Setting range stride is currently unsupported")); + if (stride != nullptr) { + throw SubarrayException( + "Cannot add range; Setting range stride is currently unsupported"); + } if (this->array_->array_schema_latest() .domain() .dimension_ptr(dim_idx) - ->var_size()) - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Range must be fixed-sized")); + ->var_size()) { + throw SubarrayException("Cannot add range; Range must be fixed-sized"); + } // Prepare a temp range std::vector range; @@ -421,34 +418,30 @@ Status Subarray::add_range( std::memcpy(&range[coord_size], end, coord_size); // Add range - return this->add_range( - dim_idx, Range(&range[0], 2 * coord_size), err_on_range_oob_); + this->add_range(dim_idx, Range(&range[0], 2 * coord_size), err_on_range_oob_); } -Status Subarray::add_point_ranges( +void Subarray::add_point_ranges( unsigned dim_idx, const void* start, uint64_t count, bool check_for_label) { if (dim_idx >= this->array_->array_schema_latest().dim_num()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Invalid dimension index")); + throw SubarrayException("Cannot add range; Invalid dimension index"); } if (check_for_label && label_range_subset_[dim_idx].has_value()) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( "Cannot add range to to dimension; A range is already set on a " - "dimension label for this dimension")); + "dimension label for this dimension"); } if (start == nullptr) { - return LOG_STATUS( - Status_SubarrayError("Cannot add ranges; Invalid start pointer")); + throw SubarrayException("Cannot add ranges; Invalid start pointer"); } if (this->array_->array_schema_latest() .domain() .dimension_ptr(dim_idx) ->var_size()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Range must be fixed-sized")); + throw SubarrayException("Cannot add range; Range must be fixed-sized"); } // Prepare a temp range @@ -464,45 +457,38 @@ Status Subarray::add_point_ranges( std::memcpy(&range[coord_size], ptr, coord_size); // Add range - auto st = this->add_range( + this->add_range( dim_idx, Range(&range[0], 2 * coord_size), err_on_range_oob_); - if (!st.ok()) { - return LOG_STATUS(std::move(st)); - } } - return Status::Ok(); } -Status Subarray::add_ranges_list( +void Subarray::add_ranges_list( unsigned dim_idx, const void* start, uint64_t count, bool check_for_label) { if (dim_idx >= this->array_->array_schema_latest().dim_num()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Invalid dimension index")); + throw SubarrayException("Cannot add range; Invalid dimension index"); } if (check_for_label && label_range_subset_[dim_idx].has_value()) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( "Cannot add range to to dimension; A range is already set on a " - "dimension label for this dimension")); + "dimension label for this dimension"); } if (count % 2) { - return LOG_STATUS(Status_SubarrayError( + throw SubarrayException( "add_ranges_list: Invalid count " + std::to_string(count) + - ", count must be a multiple of 2 ")); + ", count must be a multiple of 2 "); } if (start == nullptr) { - return LOG_STATUS( - Status_SubarrayError("Cannot add ranges; Invalid start pointer")); + throw SubarrayException("Cannot add ranges; Invalid start pointer"); } if (this->array_->array_schema_latest() .domain() .dimension_ptr(dim_idx) ->var_size()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Range must be fixed-sized")); + throw SubarrayException("Cannot add range; Range must be fixed-sized"); } // Prepare a temp range @@ -513,53 +499,46 @@ Status Subarray::add_ranges_list( uint8_t* ptr = (uint8_t*)start + 2 * coord_size * i; // Add range - auto st = - this->add_range(dim_idx, Range(ptr, 2 * coord_size), err_on_range_oob_); - if (!st.ok()) { - return LOG_STATUS(std::move(st)); - } + this->add_range(dim_idx, Range(ptr, 2 * coord_size), err_on_range_oob_); } - return Status::Ok(); } -Status Subarray::add_range_by_name( +void Subarray::add_range_by_name( const std::string& dim_name, const void* start, const void* end, const void* stride) { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - return add_range(dim_idx, start, end, stride); + add_range(dim_idx, start, end, stride); } -Status Subarray::add_range_var( +void Subarray::add_range_var( unsigned dim_idx, const void* start, uint64_t start_size, const void* end, uint64_t end_size) { if (dim_idx >= array_->array_schema_latest().dim_num()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Invalid dimension index")); + throw SubarrayException("Cannot add range; Invalid dimension index"); } if (has_label_ranges(dim_idx)) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( "Cannot add range to to dimension; A range is already set on a " - "dimension label for this dimension")); + "dimension label for this dimension"); } if ((start == nullptr && start_size != 0) || (end == nullptr && end_size != 0)) { - return LOG_STATUS(Status_SubarrayError("Cannot add range; Invalid range")); + throw SubarrayException("Cannot add range; Invalid range"); } if (!array_->array_schema_latest() .domain() .dimension_ptr(dim_idx) ->var_size()) { - return LOG_STATUS( - Status_SubarrayError("Cannot add range; Range must be variable-sized")); + throw SubarrayException("Cannot add range; Range must be variable-sized"); } // Get read_range_oob config setting @@ -568,17 +547,17 @@ Status Subarray::add_range_var( assert(found); if (read_range_oob != "error" && read_range_oob != "warn") { - return LOG_STATUS(Status_SubarrayError( + throw SubarrayException( "Invalid value " + read_range_oob + - " for sm.read_range_obb. Acceptable values are 'error' or 'warn'.")); + " for sm.read_range_obb. Acceptable values are 'error' or 'warn'."); } // Add range - return this->add_range( + this->add_range( dim_idx, Range(start, start_size, end, end_size), err_on_range_oob_); } -Status Subarray::add_range_var_by_name( +void Subarray::add_range_var_by_name( const std::string& dim_name, const void* start, uint64_t start_size, @@ -586,7 +565,7 @@ Status Subarray::add_range_var_by_name( uint64_t end_size) { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - return add_range_var(dim_idx, start, start_size, end, end_size); + add_range_var(dim_idx, start, start_size, end, end_size); } const std::vector& Subarray::get_attribute_ranges( @@ -597,7 +576,7 @@ const std::vector& Subarray::get_attribute_ranges( const std::string& Subarray::get_label_name(const uint32_t dim_index) const { if (!has_label_ranges(dim_index)) { - throw SubarrayStatusException("[get_label_name] No label ranges set"); + throw SubarrayException("[get_label_name] No label ranges set"); } return label_range_subset_[dim_index]->name_; } @@ -613,7 +592,7 @@ void Subarray::get_label_range( .dimension_index(); if (!label_range_subset_[dim_idx].has_value() || label_range_subset_[dim_idx].value().name_ != label_name) { - throw SubarrayStatusException( + throw SubarrayException( "[get_label_range] No ranges set on dimension label '" + label_name + "'"); } @@ -644,7 +623,7 @@ void Subarray::get_label_range_var( .dimension_index(); if (!label_range_subset_[dim_idx].has_value() || label_range_subset_[dim_idx].value().name_ != label_name) { - throw SubarrayStatusException( + throw SubarrayException( "[get_label_range_var] No ranges set on dimension label '" + label_name + "'"); } @@ -663,7 +642,7 @@ void Subarray::get_label_range_var_size( .dimension_index(); if (!label_range_subset_[dim_idx].has_value() || label_range_subset_[dim_idx].value().name_ != label_name) { - throw SubarrayStatusException( + throw SubarrayException( "[get_label_range_var_size] No ranges set on dimension label '" + label_name + "'"); } @@ -672,44 +651,39 @@ void Subarray::get_label_range_var_size( *end_size = range.end_size(); } -Status Subarray::get_range_var( +void Subarray::get_range_var( unsigned dim_idx, uint64_t range_idx, void* start, void* end) const { uint64_t start_size = 0; uint64_t end_size = 0; - throw_if_not_ok( - this->get_range_var_size(dim_idx, range_idx, &start_size, &end_size)); + this->get_range_var_size(dim_idx, range_idx, &start_size, &end_size); const void* range_start; const void* range_end; const void* stride; - RETURN_NOT_OK( - get_range(dim_idx, range_idx, &range_start, &range_end, &stride)); + get_range(dim_idx, range_idx, &range_start, &range_end, &stride); std::memcpy(start, range_start, start_size); std::memcpy(end, range_end, end_size); - - return Status::Ok(); } -Status Subarray::get_range_num_from_name( +void Subarray::get_range_num_from_name( const std::string& dim_name, uint64_t* range_num) const { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - - return get_range_num(dim_idx, range_num); + get_range_num(dim_idx, range_num); } -Status Subarray::get_range( +void Subarray::get_range( unsigned dim_idx, uint64_t range_idx, const void** start, const void** end, const void** stride) const { *stride = nullptr; - return this->get_range(dim_idx, range_idx, start, end); + this->get_range(dim_idx, range_idx, start, end); } -Status Subarray::get_range_from_name( +void Subarray::get_range_from_name( const std::string& dim_name, uint64_t range_idx, const void** start, @@ -717,28 +691,27 @@ Status Subarray::get_range_from_name( const void** stride) const { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - - return get_range(dim_idx, range_idx, start, end, stride); + get_range(dim_idx, range_idx, start, end, stride); } -Status Subarray::get_range_var_size_from_name( +void Subarray::get_range_var_size_from_name( const std::string& dim_name, uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) const { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - return get_range_var_size(dim_idx, range_idx, start_size, end_size); + get_range_var_size(dim_idx, range_idx, start_size, end_size); } -Status Subarray::get_range_var_from_name( +void Subarray::get_range_var_from_name( const std::string& dim_name, uint64_t range_idx, void* start, void* end) const { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - return get_range_var(dim_idx, range_idx, start, end); + get_range_var(dim_idx, range_idx, start, end); } const shared_ptr Subarray::array() const { return array_; @@ -910,37 +883,35 @@ bool Subarray::empty(uint32_t dim_idx) const { return range_subset_[dim_idx].is_empty(); } -Status Subarray::get_range( +void Subarray::get_range( uint32_t dim_idx, uint64_t range_idx, const Range** range) const { auto dim_num = array_->array_schema_latest().dim_num(); - if (dim_idx >= dim_num) - return logger_->status( - Status_SubarrayError("Cannot get range; Invalid dimension index")); + if (dim_idx >= dim_num) { + throw SubarrayException("Cannot get range; Invalid dimension index"); + } auto range_num = range_subset_[dim_idx].num_ranges(); - if (range_idx >= range_num) - return logger_->status( - Status_SubarrayError("Cannot get range; Invalid range index")); + if (range_idx >= range_num) { + throw SubarrayException("Cannot get range; Invalid range index"); + } *range = &range_subset_[dim_idx][range_idx]; - - return Status::Ok(); } -Status Subarray::get_range( +void Subarray::get_range( uint32_t dim_idx, uint64_t range_idx, const void** start, const void** end) const { auto dim_num = array_->array_schema_latest().dim_num(); - if (dim_idx >= dim_num) - return logger_->status( - Status_SubarrayError("Cannot get range; Invalid dimension index")); + if (dim_idx >= dim_num) { + throw SubarrayException("Cannot get range; Invalid dimension index"); + } auto range_num = range_subset_[dim_idx].num_ranges(); - if (range_idx >= range_num) - return logger_->status( - Status_SubarrayError("Cannot get range; Invalid range index")); + if (range_idx >= range_num) { + throw SubarrayException("Cannot get range; Invalid range index"); + } auto& range = range_subset_[dim_idx][range_idx]; auto is_var = range.var_size(); @@ -951,51 +922,46 @@ Status Subarray::get_range( *start = range_subset_[dim_idx][range_idx].start_fixed(); *end = range_subset_[dim_idx][range_idx].end_fixed(); } - - return Status::Ok(); } -Status Subarray::get_range_var_size( +void Subarray::get_range_var_size( uint32_t dim_idx, uint64_t range_idx, - uint64_t* start, - uint64_t* end) const { + uint64_t* start_size, + uint64_t* end_size) const { const auto& schema = array_->array_schema_latest(); auto dim_num = schema.dim_num(); - if (dim_idx >= dim_num) - return logger_->status(Status_SubarrayError( - "Cannot get var range size; Invalid dimension index")); + if (dim_idx >= dim_num) { + throw SubarrayException( + "Cannot get var range size; Invalid dimension index"); + } auto dim{schema.domain().dimension_ptr(dim_idx)}; - if (!dim->var_size()) - return logger_->status(Status_SubarrayError( + if (!dim->var_size()) { + throw SubarrayException( "Cannot get var range size; Dimension " + dim->name() + - " is not var sized")); + " is not var sized"); + } auto range_num = range_subset_[dim_idx].num_ranges(); - if (range_idx >= range_num) - return logger_->status( - Status_SubarrayError("Cannot get var range size; Invalid range index")); - - *start = range_subset_[dim_idx][range_idx].start_size(); - *end = range_subset_[dim_idx][range_idx].end_size(); + if (range_idx >= range_num) { + throw SubarrayException("Cannot get var range size; Invalid range index"); + } - return Status::Ok(); + *start_size = range_subset_[dim_idx][range_idx].start_size(); + *end_size = range_subset_[dim_idx][range_idx].end_size(); } -Status Subarray::get_range_num(uint32_t dim_idx, uint64_t* range_num) const { +void Subarray::get_range_num(uint32_t dim_idx, uint64_t* range_num) const { auto dim_num = array_->array_schema_latest().dim_num(); if (dim_idx >= dim_num) { - return LOG_STATUS(Status_SubarrayError( + throw SubarrayException( "Cannot get number of ranges for a dimension; Invalid dimension " "index " + std::to_string(dim_idx) + " requested, " + std::to_string(dim_num - 1) + - " max avail.")); + " max avail."); } - *range_num = range_subset_[dim_idx].num_ranges(); - - return Status::Ok(); } Subarray Subarray::get_subarray(uint64_t start, uint64_t end) const { @@ -1009,7 +975,7 @@ Subarray Subarray::get_subarray(uint64_t start, uint64_t end) const { for (unsigned d = 0; d < dim_num; ++d) { if (!range_subset_[d].is_implicitly_initialized()) { for (uint64_t r = start_coords[d]; r <= end_coords[d]; ++r) { - throw_if_not_ok(ret.add_range_unsafe(d, range_subset_[d][r])); + ret.add_range_unsafe(d, range_subset_[d][r]); } } } @@ -1078,18 +1044,6 @@ bool Subarray::is_unary() const { return true; } -bool Subarray::is_unary(uint64_t range_idx) const { - auto coords = get_range_coords(range_idx); - auto dim_num = this->dim_num(); - - for (unsigned d = 0; d < dim_num; ++d) { - if (!range_subset_[d][coords[d]].unary()) - return false; - } - - return true; -} - void Subarray::set_is_default(uint32_t dim_index, bool is_default) { if (is_default) { auto dim{array_->array_schema_latest().dimension_ptr(dim_index)}; @@ -1110,7 +1064,7 @@ void Subarray::set_config(const QueryType query_type, const Config& config) { std::string read_range_oob_str = config.get("sm.read_range_oob", &found); assert(found); if (read_range_oob_str != "error" && read_range_oob_str != "warn") - throw SubarrayStatusException( + throw SubarrayException( "[set_config] Invalid value " + read_range_oob_str + " for sm.read_range_obb. Acceptable values are 'error' or 'warn'."); err_on_range_oob_ = read_range_oob_str == "error"; @@ -1121,54 +1075,50 @@ const Config* Subarray::config() const { return &config_; } -Status Subarray::set_coalesce_ranges(bool coalesce_ranges) { - if (count_set_ranges()) - return LOG_STATUS( - Status_SubarrayError("non-default ranges have been set, cannot change " - "coalesce_ranges setting!")); +void Subarray::set_coalesce_ranges(bool coalesce_ranges) { + if (count_set_ranges()) { + throw SubarrayException( + "non-default ranges have been set, cannot change " + "coalesce_ranges setting!"); + } // trying to mimic conditions at ctor() coalesce_ranges_ = coalesce_ranges; add_default_ranges(); - return Status::Ok(); } -Status Subarray::to_byte_vec(std::vector* byte_vec) const { +void Subarray::to_byte_vec(std::vector* byte_vec) const { if (range_num() != 1) { - return logger_->status(Status_SubarrayError( - "Cannot export to byte vector; The subarray must be unary")); + throw SubarrayException( + "Cannot export to byte vector; The subarray must be unary"); } - byte_vec->clear(); - for (const auto& subset : range_subset_) { auto offset = byte_vec->size(); byte_vec->resize(offset + subset[0].size()); std::memcpy(&(*byte_vec)[offset], subset[0].data(), subset[0].size()); } - - return Status::Ok(); } Layout Subarray::layout() const { return layout_; } -Status Subarray::get_est_result_size_internal( +void Subarray::get_est_result_size_internal( const char* name, uint64_t* size, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension name cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension name cannot be null"); } // Check size pointer if (size == nullptr) { - return logger_->status(Status_SubarrayError( - "Cannot get estimated result size; Input size cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; Input size cannot be null"); } // Check if name is attribute or dimension @@ -1178,27 +1128,27 @@ Status Subarray::get_est_result_size_internal( // Check if attribute/dimension exists if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( std::string("Cannot get estimated result size; Attribute/Dimension '") + - name + "' does not exist")); + name + "' does not exist"); } // Check if the attribute/dimension is fixed-sized if (array_schema.var_size(name)) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must be fixed-sized")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must be fixed-sized"); } // Check if attribute/dimension is nullable if (array_schema.is_nullable(name)) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must not be nullable")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must not be nullable"); } // Compute tile overlap for each fragment - RETURN_NOT_OK(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size = static_cast(std::ceil(est_result_size_[name].size_fixed_)); // If the size is non-zero, ensure it is large enough to @@ -1206,27 +1156,25 @@ Status Subarray::get_est_result_size_internal( const auto cell_size = array_schema.cell_size(name); if (*size > 0 && *size < cell_size) *size = cell_size; - - return Status::Ok(); } -Status Subarray::get_est_result_size( +void Subarray::get_est_result_size( const char* name, uint64_t* size_off, uint64_t* size_val, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension name cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension name cannot be null"); } // Check size pointer if (size_off == nullptr || size_val == nullptr) { - return logger_->status(Status_SubarrayError( - "Cannot get estimated result size; Input sizes cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; Input sizes cannot be null"); } // Check if name is attribute or dimension @@ -1236,27 +1184,27 @@ Status Subarray::get_est_result_size( // Check if attribute/dimension exists if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( std::string("Cannot get estimated result size; Attribute/Dimension '") + - name + "' does not exist")); + name + "' does not exist"); } // Check if the attribute/dimension is var-sized if (!array_schema.var_size(name)) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must be var-sized")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must be var-sized"); } // Check if attribute/dimension is nullable if (array_schema.is_nullable(name)) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must not be nullable")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must not be nullable"); } // Compute tile overlap for each fragment - RETURN_NOT_OK(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size_off = static_cast(std::ceil(est_result_size_[name].size_fixed_)); *size_val = @@ -1276,26 +1224,24 @@ Status Subarray::get_est_result_size( } else { *size_off = 0; } - - return Status::Ok(); } -Status Subarray::get_est_result_size_nullable( +void Subarray::get_est_result_size_nullable( const char* name, uint64_t* size, uint64_t* size_validity, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute name cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute name cannot be null"); // Check size pointer if (size == nullptr || size_validity == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get estimated result size; Input sizes cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; Input sizes cannot be null"); // Check if name is attribute const auto& array_schema = array_->array_schema_latest(); @@ -1303,24 +1249,24 @@ Status Subarray::get_est_result_size_nullable( // Check if attribute exists if (!is_attr) - return logger_->status(Status_SubarrayError( + throw SubarrayException( std::string("Cannot get estimated result size; Attribute '") + name + - "' does not exist")); + "' does not exist"); // Check if the attribute is fixed-sized if (array_schema.var_size(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be fixed-sized")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be fixed-sized"); // Check if attribute is nullable if (!array_schema.is_nullable(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be nullable")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be nullable"); // Compute tile overlap for each fragment - RETURN_NOT_OK(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size = static_cast(std::ceil(est_result_size_[name].size_fixed_)); *size_validity = static_cast(std::ceil(est_result_size_[name].size_validity_)); @@ -1332,27 +1278,25 @@ Status Subarray::get_est_result_size_nullable( *size = cell_size; *size_validity = 1; } - - return Status::Ok(); } -Status Subarray::get_est_result_size_nullable( +void Subarray::get_est_result_size_nullable( const char* name, uint64_t* size_off, uint64_t* size_val, uint64_t* size_validity, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute name cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute name cannot be null"); // Check size pointer if (size_off == nullptr || size_val == nullptr || size_validity == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get estimated result size; Input sizes cannot be null")); + throw SubarrayException( + "Cannot get estimated result size; Input sizes cannot be null"); // Check if name is attribute const auto& array_schema = array_->array_schema_latest(); @@ -1360,24 +1304,24 @@ Status Subarray::get_est_result_size_nullable( // Check if attribute exists if (!is_attr) - return logger_->status(Status_SubarrayError( + throw SubarrayException( std::string("Cannot get estimated result size; Attribute '") + name + - "' does not exist")); + "' does not exist"); // Check if the attribute is var-sized if (!array_schema.var_size(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be var-sized")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be var-sized"); // Check if attribute is nullable if (!array_schema.is_nullable(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be nullable")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be nullable"); // Compute tile overlap for each fragment - RETURN_NOT_OK(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size_off = static_cast(std::ceil(est_result_size_[name].size_fixed_)); *size_val = @@ -1404,24 +1348,24 @@ Status Subarray::get_est_result_size_nullable( *size_off = 0; *size_validity = 0; } - - return Status::Ok(); } -Status Subarray::get_max_memory_size( +void Subarray::get_max_memory_size( const char* name, uint64_t* size, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name - if (name == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension cannot be null")); + if (name == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension cannot be null"); + } // Check size pointer - if (size == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Input size cannot be null")); + if (size == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Input size cannot be null"); + } // Check if name is attribute or dimension const auto& array_schema = array_->array_schema_latest(); @@ -1430,46 +1374,46 @@ Status Subarray::get_max_memory_size( // Check if attribute/dimension exists if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { - return logger_->status(Status_SubarrayError( + throw SubarrayException( std::string("Cannot get max memory size; Attribute/Dimension '") + - name + "' does not exist")); + name + "' does not exist"); } // Check if the attribute/dimension is fixed-sized if (name != constants::coords && array_schema.var_size(name)) { - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension must be fixed-sized")); + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension must be fixed-sized"); } // Check if attribute/dimension is nullable if (array_schema.is_nullable(name)) { - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must not be nullable")); + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must not be nullable"); } // Compute tile overlap for each fragment - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size = max_mem_size_[name].size_fixed_; - - return Status::Ok(); } -Status Subarray::get_max_memory_size( +void Subarray::get_max_memory_size( const char* name, uint64_t* size_off, uint64_t* size_val, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name - if (name == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension cannot be null")); + if (name == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension cannot be null"); + } // Check size pointer - if (size_off == nullptr || size_val == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Input sizes cannot be null")); + if (size_off == nullptr || size_val == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Input sizes cannot be null"); + } // Check if name is attribute or dimension const auto& array_schema = array_->array_schema_latest(); @@ -1477,120 +1421,124 @@ Status Subarray::get_max_memory_size( bool is_attr = array_schema.is_attr(name); // Check if attribute/dimension exists - if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) - return logger_->status(Status_SubarrayError( + if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { + throw SubarrayException( std::string("Cannot get max memory size; Attribute/Dimension '") + - name + "' does not exist")); + name + "' does not exist"); + } // Check if the attribute/dimension is var-sized - if (!array_schema.var_size(name)) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension must be var-sized")); + if (!array_schema.var_size(name)) { + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension must be var-sized"); + } // Check if attribute/dimension is nullable - if (array_schema.is_nullable(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute/Dimension must not be nullable")); + if (array_schema.is_nullable(name)) { + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute/Dimension must not be nullable"); + } // Compute tile overlap for each fragment - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size_off = max_mem_size_[name].size_fixed_; *size_val = max_mem_size_[name].size_var_; - - return Status::Ok(); } -Status Subarray::get_max_memory_size_nullable( +void Subarray::get_max_memory_size_nullable( const char* name, uint64_t* size, uint64_t* size_validity, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute name - if (name == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute cannot be null")); - + if (name == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Attribute cannot be null"); + } // Check size pointer - if (size == nullptr || size_validity == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Input sizes cannot be null")); - + if (size == nullptr || size_validity == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Input sizes cannot be null"); + } // Check if name is attribute const auto& array_schema = array_->array_schema_latest(); bool is_attr = array_schema.is_attr(name); // Check if attribute exists - if (!is_attr) - return logger_->status(Status_SubarrayError( + if (!is_attr) { + throw SubarrayException( std::string("Cannot get max memory size; Attribute '") + name + - "' does not exist")); + "' does not exist"); + } // Check if the attribute is fixed-sized - if (array_schema.var_size(name)) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute must be fixed-sized")); + if (array_schema.var_size(name)) { + throw SubarrayException( + "Cannot get max memory size; Attribute must be fixed-sized"); + } // Check if attribute is nullable - if (!array_schema.is_nullable(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be nullable")); + if (!array_schema.is_nullable(name)) { + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be nullable"); + } // Compute tile overlap for each fragment - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size = max_mem_size_[name].size_fixed_; *size_validity = max_mem_size_[name].size_validity_; - - return Status::Ok(); } -Status Subarray::get_max_memory_size_nullable( +void Subarray::get_max_memory_size_nullable( const char* name, uint64_t* size_off, uint64_t* size_val, uint64_t* size_validity, - const Config* const config, - ThreadPool* const compute_tp) { + const Config* config, + ThreadPool* compute_tp) { // Check attribute/dimension name - if (name == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension cannot be null")); + if (name == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension cannot be null"); + } // Check size pointer - if (size_off == nullptr || size_val == nullptr || size_validity == nullptr) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Input sizes cannot be null")); - + if (size_off == nullptr || size_val == nullptr || size_validity == nullptr) { + throw SubarrayException( + "Cannot get max memory size; Input sizes cannot be null"); + } // Check if name is attribute or dimension const auto& array_schema = array_->array_schema_latest(); bool is_attr = array_schema.is_attr(name); // Check if attribute exists - if (!is_attr) - return logger_->status(Status_SubarrayError( + if (!is_attr) { + throw SubarrayException( std::string("Cannot get max memory size; Attribute '") + name + - "' does not exist")); + "' does not exist"); + } // Check if the attribute is var-sized - if (!array_schema.var_size(name)) - return logger_->status(Status_SubarrayError( - "Cannot get max memory size; Attribute/Dimension must be var-sized")); + if (!array_schema.var_size(name)) { + throw SubarrayException( + "Cannot get max memory size; Attribute/Dimension must be var-sized"); + } // Check if attribute is nullable - if (!array_schema.is_nullable(name)) - return logger_->status( - Status_SubarrayError("Cannot get estimated result size; " - "Attribute must be nullable")); + if (!array_schema.is_nullable(name)) { + throw SubarrayException( + "Cannot get estimated result size; " + "Attribute must be nullable"); + } // Compute tile overlap for each fragment - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); *size_off = max_mem_size_[name].size_fixed_; *size_val = max_mem_size_[name].size_var_; *size_validity = max_mem_size_[name].size_validity_; - - return Status::Ok(); } std::vector Subarray::get_range_coords(uint64_t range_idx) const { @@ -1731,12 +1679,12 @@ NDRange Subarray::ndrange(const std::vector& range_coords) const { void Subarray::set_attribute_ranges( const std::string& attr_name, const std::vector& ranges) { if (!array_->array_schema_latest().is_attr(attr_name)) { - throw SubarrayStatusException( + throw SubarrayException( "[set_attribute_ranges] No attribute named " + attr_name + "'."); } auto search = attr_range_subset_.find(attr_name); if (search != attr_range_subset_.end()) { - throw SubarrayStatusException( + throw SubarrayException( "[set_attribute_ranges] Ranges are already set for attribute '" + attr_name + "'."); } @@ -1750,14 +1698,14 @@ const std::vector& Subarray::ranges_for_label( .dimension_index(); if (!label_range_subset_[dim_idx].has_value() || label_range_subset_[dim_idx].value().name_ != label_name) { - throw SubarrayStatusException( + throw SubarrayException( "[ranges_for_label] No ranges set on dimension label '" + label_name + "'"); } return label_range_subset_[dim_idx]->get_ranges(); } -Status Subarray::set_ranges_for_dim( +void Subarray::set_ranges_for_dim( uint32_t dim_idx, const std::vector& ranges) { auto dim{array_->array_schema_latest().dimension_ptr(dim_idx)}; range_subset_[dim_idx] = @@ -1765,10 +1713,10 @@ Status Subarray::set_ranges_for_dim( is_default_[dim_idx] = false; // Add each range individually so that contiguous // ranges may be coalesced. - for (const auto& range : ranges) + for (const auto& range : ranges) { throw_if_not_ok(range_subset_[dim_idx].add_range_unrestricted(range)); + } is_default_[dim_idx] = range_subset_[dim_idx].is_implicitly_initialized(); - return Status::Ok(); } void Subarray::set_label_ranges_for_dim( @@ -1785,7 +1733,7 @@ void Subarray::set_label_ranges_for_dim( } } -Status Subarray::split( +void Subarray::split( unsigned splitting_dim, const ByteVecValue& splitting_value, Subarray* r1, @@ -1803,20 +1751,18 @@ Status Subarray::split( if (d == splitting_dim) { auto dim{array_->array_schema_latest().dimension_ptr(d)}; dim->split_range(r, splitting_value, &sr1, &sr2); - RETURN_NOT_OK(r1->add_range_unsafe(d, sr1)); - RETURN_NOT_OK(r2->add_range_unsafe(d, sr2)); + r1->add_range_unsafe(d, sr1); + r2->add_range_unsafe(d, sr2); } else { if (!range_subset_[d].is_implicitly_initialized()) { - RETURN_NOT_OK(r1->add_range_unsafe(d, r)); - RETURN_NOT_OK(r2->add_range_unsafe(d, r)); + r1->add_range_unsafe(d, r); + r2->add_range_unsafe(d, r); } } } - - return Status::Ok(); } -Status Subarray::split( +void Subarray::split( uint64_t splitting_range, unsigned splitting_dim, const ByteVecValue& splitting_value, @@ -1834,36 +1780,34 @@ Status Subarray::split( Range sr1, sr2; for (unsigned d = 0; d < dim_num; ++d) { - RETURN_NOT_OK(this->get_range_num(d, &range_num)); + this->get_range_num(d, &range_num); if (d != splitting_dim) { if (!range_subset_[d].is_implicitly_initialized()) { for (uint64_t j = 0; j < range_num; ++j) { const auto& r = range_subset_[d][j]; - RETURN_NOT_OK(r1->add_range_unsafe(d, r)); - RETURN_NOT_OK(r2->add_range_unsafe(d, r)); + r1->add_range_unsafe(d, r); + r2->add_range_unsafe(d, r); } } } else { // d == splitting_dim if (splitting_range != UINT64_MAX) { // Need to split multiple ranges for (uint64_t j = 0; j <= splitting_range; ++j) { const auto& r = range_subset_[d][j]; - RETURN_NOT_OK(r1->add_range_unsafe(d, r)); + r1->add_range_unsafe(d, r); } for (uint64_t j = splitting_range + 1; j < range_num; ++j) { const auto& r = range_subset_[d][j]; - RETURN_NOT_OK(r2->add_range_unsafe(d, r)); + r2->add_range_unsafe(d, r); } } else { // Need to split a single range const auto& r = range_subset_[d][0]; auto dim{array_schema.dimension_ptr(d)}; dim->split_range(r, splitting_value, &sr1, &sr2); - RETURN_NOT_OK(r1->add_range_unsafe(d, sr1)); - RETURN_NOT_OK(r2->add_range_unsafe(d, sr2)); + r1->add_range_unsafe(d, sr1); + r2->add_range_unsafe(d, sr2); } } } - - return Status::Ok(); } const std::vector>& Subarray::tile_coords() const { @@ -1871,12 +1815,13 @@ const std::vector>& Subarray::tile_coords() const { } template -Status Subarray::compute_tile_coords() { +void Subarray::compute_tile_coords() { auto timer_se = stats_->start_timer("read_compute_tile_coords"); - - if (array_->array_schema_latest().tile_order() == Layout::ROW_MAJOR) - return compute_tile_coords_row(); - return compute_tile_coords_col(); + if (array_->array_schema_latest().tile_order() == Layout::ROW_MAJOR) { + compute_tile_coords_row(); + } else { + compute_tile_coords_col(); + } } template @@ -1896,13 +1841,13 @@ const SubarrayTileOverlap* Subarray::subarray_tile_overlap() const { return &tile_overlap_; } -Status Subarray::compute_relevant_fragment_est_result_sizes( +void Subarray::compute_relevant_fragment_est_result_sizes( const std::vector& names, uint64_t range_start, uint64_t range_end, std::vector>* result_sizes, std::vector>* mem_sizes, - ThreadPool* const compute_tp) { + ThreadPool* compute_tp) { // For easy reference const auto& array_schema = array_->array_schema_latest(); auto fragment_metadata = array_->fragment_metadata(); @@ -1912,7 +1857,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( ((cell_order_ == Layout::HILBERT) ? Layout::ROW_MAJOR : cell_order_) : layout_; - RETURN_NOT_OK(load_relevant_fragment_tile_var_sizes(names, compute_tp)); + load_relevant_fragment_tile_var_sizes(names, compute_tp); // Prepare result sizes vectors auto range_num = range_end - range_start + 1; @@ -1941,7 +1886,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( std::min(range_start + (t + 1) * ranges_per_thread - 1, range_end); auto r_coords = get_range_coords(r_start); for (uint64_t r = r_start; r <= r_end; ++r) { - RETURN_NOT_OK(compute_relevant_fragment_est_result_sizes( + compute_relevant_fragment_est_result_sizes( array_schema, all_dims_same_type, all_dims_fixed, @@ -1952,7 +1897,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( r, r_coords, &(*result_sizes)[r - range_start], - &frag_tiles[r - range_start])); + &frag_tiles[r - range_start]); // Get next range coordinates if (layout == Layout::ROW_MAJOR) { @@ -1976,10 +1921,9 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( // Global order - noop } } - return Status::Ok(); }); - RETURN_NOT_OK(status); + throw_if_not_ok(status); // Compute the mem sizes vector mem_sizes->resize(range_num); @@ -2020,8 +1964,6 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( } } } - - return Status::Ok(); } std::unordered_map @@ -2029,7 +1971,7 @@ Subarray::get_est_result_size_map( const Config* const config, ThreadPool* const compute_tp) { // If the result sizes have not been computed, compute them first if (!est_result_size_computed_) - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); return est_result_size_; } @@ -2039,19 +1981,17 @@ Subarray::get_max_mem_size_map( const Config* const config, ThreadPool* const compute_tp) { // If the result sizes have not been computed, compute them first if (!est_result_size_computed_) - throw_if_not_ok(compute_est_result_size(config, compute_tp)); + compute_est_result_size(config, compute_tp); return max_mem_size_; } -Status Subarray::set_est_result_size( +void Subarray::set_est_result_size( std::unordered_map& est_result_size, std::unordered_map& max_mem_size) { est_result_size_ = est_result_size; max_mem_size_ = max_mem_size; est_result_size_computed_ = true; - - return Status::Ok(); } void Subarray::sort_and_merge_ranges(ThreadPool* const compute_tp) { @@ -2075,8 +2015,7 @@ void Subarray::sort_and_merge_ranges(ThreadPool* const compute_tp) { ranges_sorted_ = true; } -tuple> Subarray::non_overlapping_ranges( - ThreadPool* const compute_tp) { +bool Subarray::non_overlapping_ranges(ThreadPool* compute_tp) { sort_and_merge_ranges(compute_tp); std::atomic non_overlapping_ranges = true; @@ -2085,17 +2024,14 @@ tuple> Subarray::non_overlapping_ranges( 0, array_->array_schema_latest().dim_num(), [&](uint64_t dim_idx) { - auto&& [status, nor]{non_overlapping_ranges_for_dim(dim_idx)}; - - if (!*nor) { + bool nor = non_overlapping_ranges_for_dim(dim_idx); + if (!nor) { non_overlapping_ranges = false; } - - return status; + return Status::Ok(); }); - RETURN_NOT_OK_TUPLE(st, nullopt); - - return {Status::Ok(), non_overlapping_ranges}; + throw_if_not_ok(st); + return non_overlapping_ranges; } /* ****************************** */ @@ -2183,17 +2119,17 @@ void Subarray::compute_range_offsets() { } } -Status Subarray::compute_est_result_size( - const Config* const config, ThreadPool* const compute_tp) { +void Subarray::compute_est_result_size( + const Config* config, ThreadPool* compute_tp) { auto timer_se = stats_->start_timer("read_compute_est_result_size"); - if (est_result_size_computed_) - return Status::Ok(); + if (est_result_size_computed_) { + return; + } // TODO: This routine is used in the path for the C APIs that estimate // result sizes. We need to refactor this routine to handle the scenario // where `tile_overlap_` may be truncated to fit the memory budget. - RETURN_NOT_OK( - precompute_tile_overlap(0, range_num() - 1, config, compute_tp, true)); + precompute_tile_overlap(0, range_num() - 1, config, compute_tp, true); // Prepare estimated result size vector for all // attributes/dimension and zipped coords @@ -2221,8 +2157,8 @@ Status Subarray::compute_est_result_size( // Compute the estimated result and max memory sizes std::vector> result_sizes; std::vector> mem_sizes; - RETURN_NOT_OK(compute_relevant_fragment_est_result_sizes( - names, 0, range_num - 1, &result_sizes, &mem_sizes, compute_tp)); + compute_relevant_fragment_est_result_sizes( + names, 0, range_num - 1, &result_sizes, &mem_sizes, compute_tp); // Accummulate the individual estimated result sizes std::vector est_vec(num, ResultSize{0.0, 0.0, 0.0}); @@ -2284,27 +2220,25 @@ Status Subarray::compute_est_result_size( max_mem_size_[names[i]] = mem_vec[i]; } est_result_size_computed_ = true; - - return Status::Ok(); } bool Subarray::est_result_size_computed() { return est_result_size_computed_; } -Status Subarray::compute_relevant_fragment_est_result_sizes( +void Subarray::compute_relevant_fragment_est_result_sizes( const ArraySchema& array_schema, bool all_dims_same_type, bool all_dims_fixed, const std::vector>& fragment_meta, - const std::vector& names, + const std::vector& name, const std::vector& var_sizes, const std::vector& nullable, uint64_t range_idx, const std::vector& range_coords, std::vector* result_sizes, std::set>* frag_tiles) { - result_sizes->resize(names.size(), {0.0, 0.0, 0.0}); + result_sizes->resize(name.size(), {0.0, 0.0, 0.0}); const uint64_t translated_range_idx = range_idx - tile_overlap_.range_idx_start(); @@ -2320,20 +2254,20 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( // Parse tile ranges for (const auto& tr : overlap->tile_ranges_) { for (uint64_t tid = tr.first; tid <= tr.second; ++tid) { - for (size_t n = 0; n < names.size(); ++n) { + for (size_t n = 0; n < name.size(); ++n) { // Zipped coords applicable only in homogeneous domains - if (names[n] == constants::coords && !all_dims_same_type) + if (name[n] == constants::coords && !all_dims_same_type) continue; // If this attribute does not exist, skip it as this is likely a new // attribute added as a result of schema evolution - if (!meta->array_schema()->is_field(names[n])) { + if (!meta->array_schema()->is_field(name[n])) { continue; } frag_tiles->insert(std::pair(f, tid)); - auto tile_size = meta->tile_size(names[n], tid); - auto attr_datatype_size = datatype_size(array_schema.type(names[n])); + auto tile_size = meta->tile_size(name[n], tid); + auto attr_datatype_size = datatype_size(array_schema.type(name[n])); if (!var_sizes[n]) { (*result_sizes)[n].size_fixed_ += tile_size; if (nullable[n]) @@ -2343,7 +2277,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( } else { tile_size -= constants::cell_var_offset_size; (*result_sizes)[n].size_fixed_ += tile_size; - auto tile_var_size = meta->tile_var_size(names[n], tid); + auto tile_var_size = meta->tile_var_size(name[n], tid); (*result_sizes)[n].size_var_ += tile_var_size; if (nullable[n]) (*result_sizes)[n].size_validity_ += @@ -2358,20 +2292,20 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( for (const auto& t : overlap->tiles_) { auto tid = t.first; auto ratio = t.second; - for (size_t n = 0; n < names.size(); ++n) { + for (size_t n = 0; n < name.size(); ++n) { // Zipped coords applicable only in homogeneous domains - if (names[n] == constants::coords && !all_dims_same_type) + if (name[n] == constants::coords && !all_dims_same_type) continue; // If this attribute does not exist, skip it as this is likely a new // attribute added as a result of schema evolution - if (!meta->array_schema()->is_field(names[n])) { + if (!meta->array_schema()->is_field(name[n])) { continue; } frag_tiles->insert(std::pair(f, tid)); - auto tile_size = meta->tile_size(names[n], tid); - auto attr_datatype_size = datatype_size(array_schema.type(names[n])); + auto tile_size = meta->tile_size(name[n], tid); + auto attr_datatype_size = datatype_size(array_schema.type(name[n])); if (!var_sizes[n]) { (*result_sizes)[n].size_fixed_ += tile_size * ratio; if (nullable[n]) @@ -2383,7 +2317,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( } else { tile_size -= constants::cell_var_offset_size; (*result_sizes)[n].size_fixed_ += tile_size * ratio; - auto tile_var_size = meta->tile_var_size(names[n], tid); + auto tile_var_size = meta->tile_var_size(name[n], tid); (*result_sizes)[n].size_var_ += tile_var_size * ratio; if (nullable[n]) (*result_sizes)[n].size_validity_ += @@ -2410,9 +2344,9 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( uint64_t max_size_fixed = UINT64_MAX; uint64_t max_size_var = UINT64_MAX; uint64_t max_size_validity = UINT64_MAX; - for (size_t n = 0; n < names.size(); ++n) { + for (size_t n = 0; n < name.size(); ++n) { // Zipped coords applicable only in homogeneous domains - if (names[n] == constants::coords && !all_dims_same_type) + if (name[n] == constants::coords && !all_dims_same_type) continue; if (var_sizes[n]) { @@ -2420,7 +2354,7 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( utils::math::safe_mul(cell_num, constants::cell_var_offset_size); } else { max_size_fixed = - utils::math::safe_mul(cell_num, array_schema.cell_size(names[n])); + utils::math::safe_mul(cell_num, array_schema.cell_size(name[n])); } if (nullable[n]) max_size_validity = @@ -2434,12 +2368,10 @@ Status Subarray::compute_relevant_fragment_est_result_sizes( (*result_sizes)[n].size_validity_, max_size_validity); } } - - return Status::Ok(); } template -Status Subarray::compute_tile_coords_col() { +void Subarray::compute_tile_coords_col() { std::vector> coords_set; const auto& array_schema = array_->array_schema_latest(); auto domain = array_schema.domain().domain(); @@ -2494,12 +2426,10 @@ Status Subarray::compute_tile_coords_col() { // Compute `tile_coords_map_` for (size_t i = 0; i < tile_coords_.size(); ++i, ++tile_coords_pos) tile_coords_map_[tile_coords_[i]] = i; - - return Status::Ok(); } template -Status Subarray::compute_tile_coords_row() { +void Subarray::compute_tile_coords_row() { std::vector> coords_set; const auto& array_schema = array_->array_schema_latest(); auto domain = array_schema.domain().domain(); @@ -2554,16 +2484,14 @@ Status Subarray::compute_tile_coords_row() { // Compute `tile_coords_map_` for (size_t i = 0; i < tile_coords_.size(); ++i, ++tile_coords_pos) tile_coords_map_[tile_coords_[i]] = i; - - return Status::Ok(); } -Status Subarray::precompute_tile_overlap( - const uint64_t start_range_idx, - const uint64_t end_range_idx, +void Subarray::precompute_tile_overlap( + uint64_t start_range_idx, + uint64_t end_range_idx, const Config* config, - ThreadPool* const compute_tp, - const bool override_memory_constraint) { + ThreadPool* compute_tp, + bool override_memory_constraint) { auto timer_se = stats_->start_timer("read_compute_tile_overlap"); // If the `tile_overlap_` has already been precomputed and contains @@ -2573,7 +2501,7 @@ Status Subarray::precompute_tile_overlap( if (tile_overlap_computed) { stats_->add_counter("precompute_tile_overlap.tile_overlap_cache_hit", 1); tile_overlap_.update_range(start_range_idx, end_range_idx); - return Status::Ok(); + return; } stats_->add_counter( @@ -2588,7 +2516,7 @@ Status Subarray::precompute_tile_overlap( // Lookup the target maximum tile overlap size. bool found = false; uint64_t max_tile_overlap_size = 0; - RETURN_NOT_OK(config->get( + throw_if_not_ok(config->get( "sm.max_tile_overlap_size", &max_tile_overlap_size, &found)); assert(found); @@ -2621,9 +2549,9 @@ Status Subarray::precompute_tile_overlap( relevant_fragments_ = relevant_fragment_generator.compute_relevant_fragments(compute_tp); } - RETURN_NOT_OK(load_relevant_fragment_rtrees(compute_tp)); - RETURN_NOT_OK(compute_relevant_fragment_tile_overlap( - compute_tp, &tile_overlap, &tile_overlap_ctx)); + load_relevant_fragment_rtrees(compute_tp); + compute_relevant_fragment_tile_overlap( + compute_tp, &tile_overlap, &tile_overlap_ctx); if (tmp_tile_overlap_end == tile_overlap_end || tile_overlap.byte_size() >= max_tile_overlap_size) { @@ -2647,12 +2575,10 @@ Status Subarray::precompute_tile_overlap( stats_->add_counter( "precompute_tile_overlap.ranges_computed", tile_overlap_.range_idx_end() - tile_overlap_.range_idx_start() + 1); - - return Status::Ok(); } -Status Subarray::precompute_all_ranges_tile_overlap( - ThreadPool* const compute_tp, +void Subarray::precompute_all_ranges_tile_overlap( + ThreadPool* compute_tp, const std::vector& frag_tile_idx, ITileRange* tile_ranges) { auto timer_se = stats_->start_timer("read_compute_simple_tile_overlap"); @@ -2672,7 +2598,7 @@ Status Subarray::precompute_all_ranges_tile_overlap( relevant_fragment_generator.update_range_coords(nullptr); relevant_fragments_ = relevant_fragment_generator.compute_relevant_fragments(compute_tp); - RETURN_NOT_OK(load_relevant_fragment_rtrees(compute_tp)); + load_relevant_fragment_rtrees(compute_tp); // Each thread will use one bitmap per dimensions. const auto num_threads = compute_tp->concurrency_level(); @@ -2723,7 +2649,6 @@ Status Subarray::precompute_all_ranges_tile_overlap( meta[f]->compute_tile_bitmap( range_subset_[d][r], d, &tile_bitmaps[d]); } - return Status::Ok(); }); RETURN_NOT_OK(status_ranges); @@ -2759,11 +2684,8 @@ Status Subarray::precompute_all_ranges_tile_overlap( return Status::Ok(); }); - RETURN_NOT_OK(status); - + throw_if_not_ok(status); tile_ranges->done_adding_tile_ranges(); - - return Status::Ok(); } Subarray Subarray::clone() const { @@ -2970,8 +2892,7 @@ void Subarray::get_expanded_coordinates( } } -Status Subarray::load_relevant_fragment_rtrees( - ThreadPool* const compute_tp) const { +void Subarray::load_relevant_fragment_rtrees(ThreadPool* compute_tp) const { auto timer_se = stats_->start_timer("read_load_relevant_rtrees"); auto meta = array_->fragment_metadata(); @@ -2982,15 +2903,13 @@ Status Subarray::load_relevant_fragment_rtrees( meta[relevant_fragments_[f]]->load_rtree(*encryption_key); return Status::Ok(); }); - RETURN_NOT_OK(status); - - return Status::Ok(); + throw_if_not_ok(status); } -Status Subarray::compute_relevant_fragment_tile_overlap( - ThreadPool* const compute_tp, - SubarrayTileOverlap* const tile_overlap, - ComputeRelevantTileOverlapCtx* const fn_ctx) { +void Subarray::compute_relevant_fragment_tile_overlap( + ThreadPool* compute_tp, + SubarrayTileOverlap* tile_overlap, + ComputeRelevantTileOverlapCtx* fn_ctx) { auto timer_se = stats_->start_timer("read_compute_relevant_tile_overlap"); const auto range_num = tile_overlap->range_num(); @@ -3003,21 +2922,20 @@ Status Subarray::compute_relevant_fragment_tile_overlap( parallel_for(compute_tp, 0, relevant_fragments_.size(), [&](uint64_t i) { const auto f = relevant_fragments_[i]; const auto dense = meta[f]->dense(); - return compute_relevant_fragment_tile_overlap( + compute_relevant_fragment_tile_overlap( meta[f], f, dense, compute_tp, tile_overlap, fn_ctx); + return Status::Ok(); }); - RETURN_NOT_OK(status); - - return Status::Ok(); + throw_if_not_ok(status); } -Status Subarray::compute_relevant_fragment_tile_overlap( +void Subarray::compute_relevant_fragment_tile_overlap( shared_ptr meta, unsigned frag_idx, bool dense, - ThreadPool* const compute_tp, - SubarrayTileOverlap* const tile_overlap, - ComputeRelevantTileOverlapCtx* const fn_ctx) { + ThreadPool* compute_tp, + SubarrayTileOverlap* tile_overlap, + ComputeRelevantTileOverlapCtx* fn_ctx) { const auto num_threads = compute_tp->concurrency_level(); const auto range_num = fn_ctx->range_len_; @@ -3037,16 +2955,13 @@ Status Subarray::compute_relevant_fragment_tile_overlap( range, is_default_, tile_overlap->at(frag_idx, r)); } } - return Status::Ok(); }); - RETURN_NOT_OK(status); - - return Status::Ok(); + throw_if_not_ok(status); } -Status Subarray::load_relevant_fragment_tile_var_sizes( - const std::vector& names, ThreadPool* const compute_tp) const { +void Subarray::load_relevant_fragment_tile_var_sizes( + const std::vector& names, ThreadPool* compute_tp) const { const auto& array_schema = array_->array_schema_latest(); auto encryption_key = array_->encryption_key(); auto meta = array_->fragment_metadata(); @@ -3060,8 +2975,9 @@ Status Subarray::load_relevant_fragment_tile_var_sizes( } // No var-sized attributes/dimensions in `names` - if (var_names.empty()) - return Status::Ok(); + if (var_names.empty()) { + return; + } // Load all metadata for tile var sizes among fragments. for (const auto& var_name : var_names) { @@ -3074,15 +2990,11 @@ Status Subarray::load_relevant_fragment_tile_var_sizes( if (!schema->is_field(var_name)) { return Status::Ok(); } - meta[f]->load_tile_var_sizes(*encryption_key, var_name); return Status::Ok(); }); - - RETURN_NOT_OK(status); + throw_if_not_ok(status); } - - return Status::Ok(); } const RelevantFragments& Subarray::relevant_fragments() const { @@ -3101,19 +3013,17 @@ void Subarray::set_stats(const stats::StatsData& data) { stats_->populate_with_data(data); } -tuple> Subarray::non_overlapping_ranges_for_dim( - const uint64_t dim_idx) { +bool Subarray::non_overlapping_ranges_for_dim(const uint64_t dim_idx) { const auto& ranges = range_subset_[dim_idx].ranges(); auto dim{array_->array_schema_latest().dimension_ptr(dim_idx)}; if (ranges.size() > 1) { for (uint64_t r = 0; r < ranges.size() - 1; r++) { if (dim->overlap(ranges[r], ranges[r + 1])) - return {Status::Ok(), false}; + return false; } } - - return {Status::Ok(), true}; + return true; } template @@ -3140,7 +3050,7 @@ void Subarray::crop_to_tile_impl(const T* tile_coords, Subarray& ret) const { &overlaps); if (overlaps) { - throw_if_not_ok(ret.add_range_unsafe(d, Range(new_range, r_size))); + ret.add_range_unsafe(d, Range(new_range, r_size)); ret.original_range_idx_.resize(dim_num()); ret.original_range_idx_[d].resize(i + 1); ret.original_range_idx_[d][i++] = r; @@ -3150,16 +3060,16 @@ void Subarray::crop_to_tile_impl(const T* tile_coords, Subarray& ret) const { } // Explicit instantiations -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); -template Status Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); +template void Subarray::compute_tile_coords(); template const int8_t* Subarray::tile_coords_ptr( const std::vector& tile_coords, @@ -3251,5 +3161,4 @@ Subarray::LabelRangeSubset::LabelRangeSubset( , ranges_{RangeSetAndSuperset(type, Range(), false, coalesce_ranges)} { } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index bd36e8647700..30251f07469f 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -64,8 +64,7 @@ namespace tiledb::type { class Range; } -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class ArraySchema; @@ -363,9 +362,8 @@ class Subarray { * This is only valid for arrays with homogenous dimension data types. * * @param subarray A pointer to the range data to use. - * @returns Status error */ - Status set_subarray(const void* subarray); + void set_subarray(const void* subarray); /** * Sets the subarray using a pointer to raw range data that stores one range @@ -441,7 +439,7 @@ class Subarray { uint64_t end_size); /** Adds a range along the dimension with the given index. */ - Status add_range( + void add_range( uint32_t dim_idx, Range&& range, const bool read_range_oob_error = true); /** @@ -450,7 +448,7 @@ class Subarray { * The range components must be of the same type as the domain type of the * underlying array. */ - Status add_range( + void add_range( unsigned dim_idx, const void* start, const void* end, const void* stride); /** @@ -462,9 +460,8 @@ class Subarray { * @param check_for_label If ``true``, verify no label ranges set on this * dimension. This should check for labels unless being called by * ``add_index_ranges_from_label`` to update label ranges with index values. - * @return Status */ - Status add_point_ranges( + void add_point_ranges( unsigned dim_idx, const void* start, uint64_t count, @@ -480,12 +477,11 @@ class Subarray { * @param check_for_label If ``true``, verify no label ranges set on this * dimension. This should check for labels unless being called by * ``add_index_ranges_from_label`` to update label ranges with index values. - * @return Status * @note The pairs list is logically { {begin1,end1}, {begin2,end2}, ...} but * because of typing considerations from the C api is simply presented as * a linear list of individual items, though they should be multiple of 2 */ - Status add_ranges_list( + void add_ranges_list( unsigned dim_idx, const void* start, uint64_t count, @@ -495,7 +491,7 @@ class Subarray { * Adds a variable-sized range to the (read/write) query on the input * dimension by index, in the form of (start, end). */ - Status add_range_var( + void add_range_var( unsigned dim_idx, const void* start, uint64_t start_size, @@ -506,7 +502,7 @@ class Subarray { * Adds a range along the dimension with the given index, without * performing any error checks. */ - Status add_range_unsafe(uint32_t dim_idx, const Range& range); + void add_range_unsafe(uint32_t dim_idx, const Range& range); /** * Adds a range to the (read/write) query on the input dimension by name, @@ -514,7 +510,7 @@ class Subarray { * The range components must be of the same type as the domain type of the * underlying array. */ - Status add_range_by_name( + void add_range_by_name( const std::string& dim_name, const void* start, const void* end, @@ -524,7 +520,7 @@ class Subarray { * Adds a variable-sized range to the (read/write) query on the input * dimension by name, in the form of (start, end). */ - Status add_range_var_by_name( + void add_range_var_by_name( const std::string& dim_name, const void* start, uint64_t start_size, @@ -617,7 +613,7 @@ class Subarray { * Retrieves the number of ranges of the subarray for the given dimension * name. */ - Status get_range_num_from_name( + void get_range_num_from_name( const std::string& dim_name, uint64_t* range_num) const; /** @@ -628,9 +624,8 @@ class Subarray { * @param start The range start to retrieve. * @param end The range end to retrieve. * @param stride The range stride to retrieve. - * @return Status */ - Status get_range_from_name( + void get_range_from_name( const std::string& dim_name, uint64_t range_idx, const void** start, @@ -644,9 +639,8 @@ class Subarray { * @param range_idx The id of the range to retrieve. * @param start_size range start size in bytes * @param end_size range end size in bytes - * @return Status */ - Status get_range_var_size_from_name( + void get_range_var_size_from_name( const std::string& dim_name, uint64_t range_idx, uint64_t* start_size, @@ -660,9 +654,8 @@ class Subarray { * @param range_idx The id of the range to retrieve. * @param start The range start to retrieve. * @param end The range end to retrieve. - * @return Status */ - Status get_range_var_from_name( + void get_range_var_from_name( const std::string& dim_name, uint64_t range_idx, void* start, @@ -745,7 +738,7 @@ class Subarray { * @param override_memory_constraint When true, this forces the * routine to compute tile overlap for all ranges. */ - Status precompute_tile_overlap( + void precompute_tile_overlap( uint64_t start_range_idx, uint64_t end_range_idx, const Config* config, @@ -759,8 +752,8 @@ class Subarray { * @param frag_tile_idx The current tile index, per fragment. * @param tile_ranges The resulting tile ranges. */ - Status precompute_all_ranges_tile_overlap( - ThreadPool* const compute_tp, + void precompute_all_ranges_tile_overlap( + ThreadPool* compute_tp, const std::vector& frag_tile_idx, ITileRange* tile_ranges); @@ -785,9 +778,8 @@ class Subarray { * @param result_sizes The result sizes to be retrieved for all given names. * @param frag_tiles The set of unique (fragment id, tile id) pairs across * all ranges, which is update by this function in a thread-safe manner. - * @return Status */ - Status compute_relevant_fragment_est_result_sizes( + void compute_relevant_fragment_est_result_sizes( const ArraySchema& array_schema, bool all_dims_same_type, bool all_dims_fixed, @@ -839,7 +831,7 @@ class Subarray { * Subarray::add_range() is called after this function. In that case, * make sure to make a copy in the caller function. */ - Status get_range( + void get_range( uint32_t dim_idx, uint64_t range_idx, const Range** range) const; /** @@ -850,7 +842,7 @@ class Subarray { * Subarray::add_range() is called after this function. In that case, * make sure to make a copy in the caller function. */ - Status get_range( + void get_range( uint32_t dim_idx, uint64_t range_idx, const void** start, @@ -865,9 +857,8 @@ class Subarray { * @param range_idx The id of the range to retrieve. * @param start_size range start size in bytes * @param end_size range end size in bytes - * @return Status */ - Status get_range_var_size( + void get_range_var_size( uint32_t dim_idx, uint64_t range_idx, uint64_t* start_size, @@ -881,13 +872,12 @@ class Subarray { * @param range_idx The id of the range to retrieve. * @param start The range start to retrieve. * @param end The range end to retrieve. - * @return Status */ - Status get_range_var( + void get_range_var( unsigned dim_idx, uint64_t range_idx, void* start, void* end) const; /** Retrieves the number of ranges on the given dimension index. */ - Status get_range_num(uint32_t dim_idx, uint64_t* range_num) const; + void get_range_num(uint32_t dim_idx, uint64_t* range_num) const; /** * Retrieves a range from a dimension index in the form (start, end, stride). @@ -897,9 +887,8 @@ class Subarray { * @param start The range start to retrieve. * @param end The range end to retrieve. * @param stride The range stride to retrieve. - * @return Status */ - Status get_range( + void get_range( unsigned dim_idx, uint64_t range_idx, const void** start, @@ -930,17 +919,11 @@ class Subarray { */ bool is_unary() const; - /** - * Returns ``true`` if the subarray range with the input id is unary - * (i.e., consisting of a single point in the 1D domain). - */ - bool is_unary(uint64_t range_idx) const; - /** * Gets the estimated result size (in bytes) for the input fixed-sized * attribute/dimension. */ - Status get_est_result_size_internal( + void get_est_result_size_internal( const char* name, uint64_t* size, const Config* config, @@ -950,7 +933,7 @@ class Subarray { * Gets the estimated result size (in bytes) for the input var-sized * attribute/dimension. */ - Status get_est_result_size( + void get_est_result_size( const char* name, uint64_t* size_off, uint64_t* size_val, @@ -961,7 +944,7 @@ class Subarray { * Gets the estimated result size (in bytes) for the input fixed-sized, * nullable attribute. */ - Status get_est_result_size_nullable( + void get_est_result_size_nullable( const char* name, uint64_t* size, uint64_t* size_validity, @@ -972,7 +955,7 @@ class Subarray { * Gets the estimated result size (in bytes) for the input var-sized, * nullable attribute. */ - Status get_est_result_size_nullable( + void get_est_result_size_nullable( const char* name, uint64_t* size_off, uint64_t* size_val, @@ -987,7 +970,7 @@ class Subarray { * Gets the maximum memory required to produce the result (in bytes) * for the input fixed-sized attribute/dimension. */ - Status get_max_memory_size( + void get_max_memory_size( const char* name, uint64_t* size, const Config* config, @@ -997,7 +980,7 @@ class Subarray { * Gets the maximum memory required to produce the result (in bytes) * for the input var-sized attribute/dimension. */ - Status get_max_memory_size( + void get_max_memory_size( const char* name, uint64_t* size_off, uint64_t* size_val, @@ -1008,7 +991,7 @@ class Subarray { * Gets the maximum memory required to produce the result (in bytes) * for the input fixed-sized, nullable attribute. */ - Status get_max_memory_size_nullable( + void get_max_memory_size_nullable( const char* name, uint64_t* size, uint64_t* size_validity, @@ -1019,7 +1002,7 @@ class Subarray { * Gets the maximum memory required to produce the result (in bytes) * for the input var-sized, nullable attribute. */ - Status get_max_memory_size_nullable( + void get_max_memory_size_nullable( const char* name, uint64_t* size_off, uint64_t* size_val, @@ -1085,13 +1068,15 @@ class Subarray { * default coalesc-ranges=true semantics of internal class constructor, but * giving capi clients ability to turn off if desired. */ - Status set_coalesce_ranges(bool coalesce_ranges = true); + void set_coalesce_ranges(bool coalesce_ranges = true); /** * Flattens the subarray ranges in a byte vector. Errors out * if the subarray is not unary. + * + * @note Only used in test helper `subarray_equiv`; keep it that way. */ - Status to_byte_vec(std::vector* byte_vec) const; + void to_byte_vec(std::vector* byte_vec) const; /** Returns the subarray layout. */ Layout layout() const; @@ -1162,11 +1147,10 @@ class Subarray { * * @param dim_idx Index of dimension to set * @param ranges `Range` vector that will be copied and set - * @return Status * * @note Intended for serialization only */ - Status set_ranges_for_dim(uint32_t dim_idx, const std::vector& ranges); + void set_ranges_for_dim(uint32_t dim_idx, const std::vector& ranges); /** * Directly sets the dimension label ranges for the given dimension index, @@ -1175,7 +1159,6 @@ class Subarray { * @param dim_idx Index of dimension to set * @param name Name of the dimension label to set * @param ranges `Range` vector that will be copied and set - * @return Status * * @note Intended for serialization only */ @@ -1188,7 +1171,7 @@ class Subarray { * Splits the subarray along the splitting dimension and value into * two new subarrays `r1` and `r2`. */ - Status split( + void split( unsigned splitting_dim, const ByteVecValue& splitting_value, Subarray* r1, @@ -1198,7 +1181,7 @@ class Subarray { * Splits the subarray along the splitting range, dimension and value * into two new subarrays `r1` and `r2`. */ - Status split( + void split( uint64_t splitting_range, unsigned splitting_dim, const ByteVecValue& splitting_value, @@ -1252,10 +1235,9 @@ class Subarray { * be sorted on the array tile order. * * @tparam T The subarray datatype. - * @return Status */ template - Status compute_tile_coords(); + void compute_tile_coords(); /** * Computes the estimated result sizes for the input attribute/dimension @@ -1270,7 +1252,7 @@ class Subarray { * the maximum memory size for all ranges (i.e., based on whether * it overlaps a unique tile versus all previous ranges in the vector). */ - Status compute_relevant_fragment_est_result_sizes( + void compute_relevant_fragment_est_result_sizes( const std::vector& names, uint64_t range_start, uint64_t range_end, @@ -1283,9 +1265,8 @@ class Subarray { * * @param est_result_size map to set * @param max_mem_size map to set - * @return Status */ - Status set_est_result_size( + void set_est_result_size( std::unordered_map& est_result_size, std::unordered_map& max_mem_size); @@ -1352,8 +1333,7 @@ class Subarray { void sort_and_merge_ranges(ThreadPool* const compute_tp); /** Returns if all ranges for this subarray are non overlapping. */ - tuple> non_overlapping_ranges( - ThreadPool* const compute_tp); + bool non_overlapping_ranges(ThreadPool* const compute_tp); /** Returns if ranges will be coalesced as they are added. */ inline bool coalesce_ranges() const { @@ -1380,7 +1360,7 @@ class Subarray { void reset_default_ranges(); /** Loads the R-Trees of all relevant fragments in parallel. */ - Status load_relevant_fragment_rtrees(ThreadPool* compute_tp) const; + void load_relevant_fragment_rtrees(ThreadPool* compute_tp) const; private: /* ********************************* */ @@ -1560,27 +1540,25 @@ class Subarray { void add_default_ranges(); /** Computes the estimated result size for all attributes/dimensions. */ - Status compute_est_result_size(const Config* config, ThreadPool* compute_tp); + void compute_est_result_size(const Config* config, ThreadPool* compute_tp); /** * Compute `tile_coords_` and `tile_coords_map_`. The coordinates will * be sorted on col-major tile order. * * @tparam T The subarray datatype. - * @return Status */ template - Status compute_tile_coords_col(); + void compute_tile_coords_col(); /** * Compute `tile_coords_` and `tile_coords_map_`. The coordinates will * be sorted on row-major tile order. * * @tparam T The subarray datatype. - * @return Status */ template - Status compute_tile_coords_row(); + void compute_tile_coords_row(); /** Returns a deep copy of this Subarray. */ Subarray clone() const; @@ -1621,7 +1599,7 @@ class Subarray { * @param fn_ctx An opaque context object to be used between successive * invocations. */ - Status compute_relevant_fragment_tile_overlap( + void compute_relevant_fragment_tile_overlap( ThreadPool* compute_tp, SubarrayTileOverlap* tile_overlap, ComputeRelevantTileOverlapCtx* fn_ctx); @@ -1636,9 +1614,8 @@ class Subarray { * @param tile_overlap Mutated to store the computed tile overlap. * @param fn_ctx An opaque context object to be used between successive * invocations. - * @return Status */ - Status compute_relevant_fragment_tile_overlap( + void compute_relevant_fragment_tile_overlap( shared_ptr meta, unsigned frag_idx, bool dense, @@ -1650,7 +1627,7 @@ class Subarray { * Load the var-sized tile sizes for the input names and from the * relevant fragments. */ - Status load_relevant_fragment_tile_var_sizes( + void load_relevant_fragment_tile_var_sizes( const std::vector& names, ThreadPool* compute_tp) const; /** @@ -1659,8 +1636,7 @@ class Subarray { * @param dim_idx dimension index. * @return true if the ranges are non overlapping, false otherwise. */ - tuple> non_overlapping_ranges_for_dim( - const uint64_t dim_idx); + bool non_overlapping_ranges_for_dim(uint64_t dim_idx); /** * Returns a cropped version of the subarray, constrained in the @@ -1671,7 +1647,6 @@ class Subarray { void crop_to_tile_impl(const T* tile_coords, Subarray& ret) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SUBARRAY_H diff --git a/tiledb/sm/subarray/subarray_partitioner.cc b/tiledb/sm/subarray/subarray_partitioner.cc index e22fbf28ac51..e44e9b541346 100644 --- a/tiledb/sm/subarray/subarray_partitioner.cc +++ b/tiledb/sm/subarray/subarray_partitioner.cc @@ -695,7 +695,7 @@ Status SubarrayPartitioner::calibrate_current_start_end(bool* must_split_slab) { auto dim_num = subarray_.dim_num(); uint64_t num; for (unsigned i = 0; i < dim_num; ++i) { - RETURN_NOT_OK(subarray_.get_range_num(i, &num)); + subarray_.get_range_num(i, &num); range_num.push_back(num); } @@ -794,8 +794,8 @@ SubarrayPartitioner SubarrayPartitioner::clone() const { Status SubarrayPartitioner::compute_current_start_end(bool* found) { // Compute the tile overlap. Note that the ranges in `tile_overlap` may have // been truncated the ending bound due to memory constraints. - RETURN_NOT_OK(subarray_.precompute_tile_overlap( - state_.start_, state_.end_, config_, compute_tp_)); + subarray_.precompute_tile_overlap( + state_.start_, state_.end_, config_, compute_tp_); const SubarrayTileOverlap* const tile_overlap = subarray_.subarray_tile_overlap(); assert(tile_overlap->range_idx_start() == state_.start_); @@ -820,13 +820,13 @@ Status SubarrayPartitioner::compute_current_start_end(bool* found) { // Compute the estimated result sizes std::vector> result_sizes; std::vector> memory_sizes; - RETURN_NOT_OK(subarray_.compute_relevant_fragment_est_result_sizes( + subarray_.compute_relevant_fragment_est_result_sizes( names, tile_overlap->range_idx_start(), tile_overlap->range_idx_end(), &result_sizes, &memory_sizes, - compute_tp_)); + compute_tp_); bool done = false; current_.start_ = tile_overlap->range_idx_start(); @@ -932,7 +932,7 @@ void SubarrayPartitioner::compute_splitting_value_on_tiles( const Range* r; for (auto d : dims) { auto dim{array_schema.domain().dimension_ptr(d)}; - throw_if_not_ok(range.get_range(d, 0, &r)); + range.get_range(d, 0, &r); auto tiles_apart = dim->tile_num(*r) - 1; if (tiles_apart != 0) { *splitting_dim = d; @@ -1005,7 +1005,7 @@ void SubarrayPartitioner::compute_splitting_value_single_range( const Range* r; for (auto d : dims) { auto dim{array_schema.dimension_ptr(d)}; - throw_if_not_ok(range.get_range(d, 0, &r)); + range.get_range(d, 0, &r); if (!r->unary()) { *splitting_dim = d; dim->splitting_value(*r, splitting_value, unsplittable); @@ -1050,7 +1050,7 @@ void SubarrayPartitioner::compute_splitting_value_single_range_hilbert( // Check for unsplittable again auto dim{array_schema.dimension_ptr(*splitting_dim)}; const Range* r; - throw_if_not_ok(range.get_range(*splitting_dim, 0, &r)); + range.get_range(*splitting_dim, 0, &r); if (dim->smaller_than(*splitting_value, *r)) { *unsplittable = true; return; @@ -1111,7 +1111,7 @@ Status SubarrayPartitioner::compute_splitting_value_multi_range( const Range* r; for (auto d : dims) { // Check if we need to split the multiple ranges - RETURN_NOT_OK(partition.get_range_num(d, &range_num)); + partition.get_range_num(d, &range_num); if (range_num > 1) { assert(d == dims.back()); *splitting_dim = d; @@ -1121,7 +1121,7 @@ Status SubarrayPartitioner::compute_splitting_value_multi_range( } // Check if we need to split single range - throw_if_not_ok(partition.get_range(d, 0, &r)); + partition.get_range(d, 0, &r); auto dim{array_schema.dimension_ptr(d)}; if (!r->unary()) { *splitting_dim = d; @@ -1160,38 +1160,34 @@ bool SubarrayPartitioner::must_split(Subarray* partition) { // Compute max memory sizes if (var_size) { if (!nullable) { - throw_if_not_ok(partition->get_max_memory_size( + partition->get_max_memory_size( b.first.c_str(), &mem_size_fixed, &mem_size_var, config_, - compute_tp_)); + compute_tp_); } else { - throw_if_not_ok(partition->get_max_memory_size_nullable( + partition->get_max_memory_size_nullable( b.first.c_str(), &mem_size_fixed, &mem_size_var, &mem_size_validity, config_, - compute_tp_)); + compute_tp_); } } else { if (!nullable) { - throw_if_not_ok(partition->get_max_memory_size( - b.first.c_str(), &mem_size_fixed, config_, compute_tp_)); + partition->get_max_memory_size( + b.first.c_str(), &mem_size_fixed, config_, compute_tp_); } else { - throw_if_not_ok(partition->get_est_result_size_nullable( - b.first.c_str(), - &size_fixed, - &size_validity, - config_, - compute_tp_)); - throw_if_not_ok(partition->get_max_memory_size_nullable( + partition->get_est_result_size_nullable( + b.first.c_str(), &size_fixed, &size_validity, config_, compute_tp_); + partition->get_max_memory_size_nullable( b.first.c_str(), &mem_size_fixed, &mem_size_validity, config_, - compute_tp_)); + compute_tp_); } } @@ -1199,28 +1195,28 @@ bool SubarrayPartitioner::must_split(Subarray* partition) { if (!skip_split_on_est_size_) { if (var_size) { if (!nullable) { - throw_if_not_ok(partition->get_est_result_size( - b.first.c_str(), &size_fixed, &size_var, config_, compute_tp_)); + partition->get_est_result_size( + b.first.c_str(), &size_fixed, &size_var, config_, compute_tp_); } else { - throw_if_not_ok(partition->get_est_result_size_nullable( + partition->get_est_result_size_nullable( b.first.c_str(), &size_fixed, &size_var, &size_validity, config_, - compute_tp_)); + compute_tp_); } } else { if (!nullable) { - throw_if_not_ok(partition->get_est_result_size_internal( - b.first.c_str(), &size_fixed, config_, compute_tp_)); + partition->get_est_result_size_internal( + b.first.c_str(), &size_fixed, config_, compute_tp_); } else { - throw_if_not_ok(partition->get_est_result_size_nullable( + partition->get_est_result_size_nullable( b.first.c_str(), &size_fixed, &size_validity, config_, - compute_tp_)); + compute_tp_); } } } @@ -1337,7 +1333,7 @@ Status SubarrayPartitioner::split_top_single_range(bool* unsplittable) { // Split remaining range into two ranges Subarray r1, r2; - RETURN_NOT_OK(range.split(splitting_dim, splitting_value, &r1, &r2)); + range.split(splitting_dim, splitting_value, &r1, &r2); // Update list state_.single_range_.pop_front(); @@ -1380,8 +1376,7 @@ Status SubarrayPartitioner::split_top_multi_range(bool* unsplittable) { // Split partition into two partitions Subarray p1; Subarray p2; - RETURN_NOT_OK(partition.split( - splitting_range, splitting_dim, splitting_value, &p1, &p2)); + partition.split(splitting_range, splitting_dim, splitting_value, &p1, &p2); // Update list state_.multi_range_.pop_front(); @@ -1436,7 +1431,7 @@ void SubarrayPartitioner::compute_range_uint64( for (uint32_t d = 0; d < dim_num; ++d) { auto dim{array_schema.dimension_ptr(d)}; auto var = dim->var_size(); - throw_if_not_ok(range.get_range(d, 0, &r)); + range.get_range(d, 0, &r); empty_start = var ? (r->start_size() == 0) : r->empty(); empty_end = var ? (r->end_size() == 0) : r->empty(); auto max_default = diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 040c7a129d06..61ea4bda2c97 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -113,17 +113,17 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { // underneath add_ranges_list to determine the size of the values being // iterated over. uint64_t ranges[] = {1, 2, 4, 5, 7, 8, 10, 11}; - CHECK(sa.add_ranges_list(0, ranges, 8).ok()); - CHECK(sa.add_ranges_list(1, ranges, 8).ok()); + CHECK_NOTHROW(sa.add_ranges_list(0, ranges, 8)); + CHECK_NOTHROW(sa.add_ranges_list(1, ranges, 8)); uint64_t range_num; - CHECK(sa.get_range_num(0, &range_num).ok()); + CHECK_NOTHROW(sa.get_range_num(0, &range_num)); CHECK(range_num == 4); // Check ranges for (uint32_t dim_idx = 0; dim_idx < 1; dim_idx++) { for (uint32_t idx = 0; idx < range_num; idx++) { const void *start, *end; - CHECK(sa.get_range(dim_idx, idx, &start, &end).ok()); + CHECK_NOTHROW(sa.get_range(dim_idx, idx, &start, &end)); CHECK(*(uint64_t*)start == ranges[idx * 2]); CHECK(*(uint64_t*)end == ranges[idx * 2 + 1]); } diff --git a/tiledb/sm/subarray/tile_cell_slab_iter.cc b/tiledb/sm/subarray/tile_cell_slab_iter.cc index 09d9bdbdc7b3..34db084a8807 100644 --- a/tiledb/sm/subarray/tile_cell_slab_iter.cc +++ b/tiledb/sm/subarray/tile_cell_slab_iter.cc @@ -296,10 +296,10 @@ Status TileCellSlabIter::init_ranges(const Subarray& subarray) { const tiledb::type::Range* r; ranges_.resize(dim_num_); for (int d = 0; d < dim_num_; ++d) { - RETURN_NOT_OK(subarray.get_range_num(d, &range_num)); + subarray.get_range_num(d, &range_num); ranges_[d].reserve(range_num); for (uint64_t j = 0; j < range_num; ++j) { - RETURN_NOT_OK(subarray.get_range(d, j, &r)); + subarray.get_range(d, j, &r); auto range = (const T*)(*r).data(); ranges_[d].emplace_back(range[0], range[1]); } From 849052ec2a251acb7fa924844ffbbadab742580a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:00:25 -0400 Subject: [PATCH 267/456] Instrument FragmentConsolidator for memory measurement. (#4840) Instrumentation of FragmentConsolidator for memory measurement. --- TYPE: NO_HISTORY DESC: Instrument FragmentConsolidator for memory measurement. --- tiledb/common/memory_tracker.cc | 58 ++++++++++--------- tiledb/common/memory_tracker.h | 29 +++++----- .../common/test/unit_memory_tracker_types.cc | 2 +- .../sm/consolidator/fragment_consolidator.cc | 16 +++-- .../sm/consolidator/fragment_consolidator.h | 23 ++++---- .../test/unit_fragment_consolidator.cc | 8 +-- 6 files changed, 72 insertions(+), 64 deletions(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 6cf0ede70c38..5e34e76c300b 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -50,54 +50,56 @@ class MemoryTrackerException : public common::StatusException { std::string memory_type_to_str(MemoryType type) { switch (type) { + case MemoryType::ATTRIBUTES: + return "Attributes"; + case MemoryType::CONSOLIDATION_BUFFERS: + return "ConsolidationBuffers"; + case MemoryType::DIMENSION_LABELS: + return "DimensionLabels"; + case MemoryType::DIMENSIONS: + return "Dimensions"; + case MemoryType::DOMAINS: + return "Domains"; case MemoryType::ENUMERATION: return "Enumeration"; case MemoryType::ENUMERATION_PATHS: return "EnumerationPaths"; - case MemoryType::FOOTER: - return "Footer"; case MemoryType::FILTERED_DATA: return "FilteredData"; case MemoryType::FILTERED_DATA_BLOCK: return "FilteredDataBlock"; + case MemoryType::FOOTER: + return "Footer"; case MemoryType::GENERIC_TILE_IO: return "GenericTileIO"; + case MemoryType::METADATA: + return "Metadata"; + case MemoryType::QUERY_CONDITION: + return "QueryCondition"; + case MemoryType::RESULT_TILE: + return "ResultTile"; + case MemoryType::RESULT_TILE_BITMAP: + return "ResultTileBitmap"; case MemoryType::RTREE: return "RTree"; case MemoryType::TILE_DATA: return "TileData"; case MemoryType::TILE_HILBERT_VALUES: return "TileHilbertValues"; - case MemoryType::TILE_OFFSETS: - return "TileOffsets"; case MemoryType::TILE_MAX_VALS: return "TileMaxVals"; case MemoryType::TILE_MIN_VALS: return "TileMinVals"; case MemoryType::TILE_NULL_COUNTS: return "TileNullCounts"; - case MemoryType::ATTRIBUTES: - return "Attributes"; - case MemoryType::DIMENSION_LABELS: - return "DimensionLabels"; - case MemoryType::DIMENSIONS: - return "Dimensions"; - case MemoryType::DOMAINS: - return "Domains"; + case MemoryType::TILE_OFFSETS: + return "TileOffsets"; case MemoryType::TILE_SUMS: return "TileSums"; - case MemoryType::WRITER_TILE_DATA: - return "WriterTileData"; - case MemoryType::METADATA: - return "Metadata"; case MemoryType::WRITER_DATA: return "WriterData"; - case MemoryType::RESULT_TILE_BITMAP: - return "ResultTileBitmap"; - case MemoryType::RESULT_TILE: - return "ResultTile"; - case MemoryType::QUERY_CONDITION: - return "QueryCondition"; + case MemoryType::WRITER_TILE_DATA: + return "WriterTileData"; } auto val = std::to_string(static_cast(type)); @@ -116,24 +118,24 @@ std::string memory_tracker_type_to_str(MemoryTrackerType type) { return "ArrayRead"; case MemoryTrackerType::ARRAY_WRITE: return "ArrayWrite"; + case MemoryTrackerType::CONSOLIDATOR: + return "Consolidator"; case MemoryTrackerType::ENUMERATION_CREATE: return "EnumerationCreate"; + case MemoryTrackerType::EPHEMERAL: + return "Ephemeral"; case MemoryTrackerType::FRAGMENT_INFO_LOAD: return "FragmentInfoLoad"; + case MemoryTrackerType::GROUP: + return "Group"; case MemoryTrackerType::QUERY_READ: return "QueryRead"; case MemoryTrackerType::QUERY_WRITE: return "QueryWrite"; - case MemoryTrackerType::CONSOLIDATOR: - return "Consolidator"; case MemoryTrackerType::REST_CLIENT: return "RestClient"; - case MemoryTrackerType::EPHEMERAL: - return "Ephemeral"; case MemoryTrackerType::SCHEMA_EVOLUTION: return "SchemaEvolution"; - case MemoryTrackerType::GROUP: - return "Group"; } auto val = std::to_string(static_cast(type)); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 9a66994e6ccb..1c1502e1e083 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -101,30 +101,31 @@ namespace tiledb::sm { /** The type of memory to track. */ enum class MemoryType { + ATTRIBUTES, + CONSOLIDATION_BUFFERS, + DIMENSION_LABELS, + DIMENSIONS, + DOMAINS, ENUMERATION, ENUMERATION_PATHS, - FOOTER, FILTERED_DATA, FILTERED_DATA_BLOCK, + FOOTER, GENERIC_TILE_IO, + METADATA, + QUERY_CONDITION, + RESULT_TILE, + RESULT_TILE_BITMAP, RTREE, TILE_DATA, TILE_HILBERT_VALUES, - TILE_OFFSETS, TILE_MAX_VALS, TILE_MIN_VALS, TILE_NULL_COUNTS, - ATTRIBUTES, - DIMENSION_LABELS, - DIMENSIONS, - METADATA, - DOMAINS, + TILE_OFFSETS, TILE_SUMS, - WRITER_TILE_DATA, WRITER_DATA, - RESULT_TILE, - QUERY_CONDITION, - RESULT_TILE_BITMAP + WRITER_TILE_DATA }; /** @@ -142,14 +143,14 @@ enum class MemoryTrackerType { ARRAY_LOAD, ARRAY_READ, ARRAY_WRITE, + CONSOLIDATOR, ENUMERATION_CREATE, + EPHEMERAL, FRAGMENT_INFO_LOAD, + GROUP, QUERY_READ, QUERY_WRITE, - CONSOLIDATOR, REST_CLIENT, - GROUP, - EPHEMERAL, SCHEMA_EVOLUTION }; diff --git a/tiledb/common/test/unit_memory_tracker_types.cc b/tiledb/common/test/unit_memory_tracker_types.cc index b3e9106a0914..e8386e31a823 100644 --- a/tiledb/common/test/unit_memory_tracker_types.cc +++ b/tiledb/common/test/unit_memory_tracker_types.cc @@ -38,7 +38,7 @@ using namespace tiledb::common; TEST_CASE("memory_type_to_str") { - auto max = static_cast(tiledb::sm::MemoryType::RESULT_TILE_BITMAP); + auto max = static_cast(tiledb::sm::MemoryType::WRITER_TILE_DATA); size_t failures = 0; for (int8_t i = 0; i < 127; i++) { auto val = static_cast(i); diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 6a37c8ee2856..d0d2a0e3c00c 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -58,6 +58,14 @@ class FragmentConsolidatorException : public StatusException { } }; +FragmentConsolidationWorkspace::FragmentConsolidationWorkspace( + shared_ptr memory_tracker) + : backing_buffer_( + memory_tracker->get_resource(MemoryType::CONSOLIDATION_BUFFERS)) + , buffers_(memory_tracker->get_resource(MemoryType::CONSOLIDATION_BUFFERS)) + , sizes_(memory_tracker->get_resource(MemoryType::CONSOLIDATION_BUFFERS)) { +} + void FragmentConsolidationWorkspace::resize_buffers( stats::Stats* stats, const FragmentConsolidationConfig& config, @@ -239,7 +247,7 @@ Status FragmentConsolidator::consolidate( return st; } - FragmentConsolidationWorkspace cw; + FragmentConsolidationWorkspace cw(consolidator_memory_tracker_); uint32_t step = 0; std::vector to_consolidate; @@ -380,7 +388,7 @@ Status FragmentConsolidator::consolidate_fragments( "Cannot consolidate; Not all fragments could be found")); } - FragmentConsolidationWorkspace cw; + FragmentConsolidationWorkspace cw(consolidator_memory_tracker_); // Consolidate the selected fragments URI new_fragment_uri; @@ -835,8 +843,8 @@ Status FragmentConsolidator::compute_next_to_consolidate( void FragmentConsolidator::set_query_buffers( Query* query, FragmentConsolidationWorkspace& cw) const { - std::vector>* buffers{&cw.buffers()}; - std::vector* buffer_sizes{&cw.sizes()}; + auto buffers = &cw.buffers(); + auto buffer_sizes = &cw.sizes(); const auto& array_schema = query->array_schema(); auto dim_num = array_schema.dim_num(); diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 7375e644cee6..4c3afb03e548 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,6 +36,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger_public.h" +#include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" @@ -108,7 +109,7 @@ struct FragmentConsolidationConfig : Consolidator::ConsolidationConfigBase { */ class FragmentConsolidationWorkspace { public: - FragmentConsolidationWorkspace() = default; + FragmentConsolidationWorkspace(shared_ptr memory_tracker); // Disable copy and move construction/assignment so we don't have // to think about it. @@ -132,29 +133,25 @@ class FragmentConsolidationWorkspace { const ArraySchema& array_schema, std::unordered_map& avg_cell_sizes); - /** - * Accessor for buffers - */ - std::vector>& buffers() { + /** Accessor for buffers. */ + tdb::pmr::vector>& buffers() { return buffers_; } - /** - * Access for sizes - */ - std::vector& sizes() { + /** Accessor for sizes. */ + tdb::pmr::vector& sizes() { return sizes_; }; private: /*** The backing buffer used for all buffers. */ - std::vector backing_buffer_; + tdb::pmr::vector backing_buffer_; /*** Spans that point to non-overlapping sections of the buffer. */ - std::vector> buffers_; + tdb::pmr::vector> buffers_; /*** The size of each span. */ - std::vector sizes_; + tdb::pmr::vector sizes_; }; /** Handles fragment consolidation. */ diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index b5e728fff267..aa18fc920026 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -222,10 +222,10 @@ TEST_CASE( cfg.with_delete_meta_ = with_delete_meta; cfg.buffer_size_ = 1000; - FragmentConsolidationWorkspace cw; + FragmentConsolidationWorkspace cw(tiledb::test::get_test_memory_tracker()); cw.resize_buffers(&statistics, cfg, *schema, avg_cell_sizes); - std::vector>& buffers{cw.buffers()}; - std::vector& buffer_sizes{cw.sizes()}; + auto& buffers = cw.buffers(); + auto& buffer_sizes = cw.sizes(); // Validate. CHECK(buffers.size() == expected_sizes.size()); From 0c50eeb227fae51ad2d2697bfe3fbc3628e9fb9b Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 1 Apr 2024 15:13:41 +0300 Subject: [PATCH 268/456] Fix crash getting file size on non existent blob on Azure. (#4836) This fixes getting a file size on azure when the blob is non existent. We would end up not getting an error and trying to read an empty array. --- TYPE: BUG DESC: Fix crash getting file size on non existent blob on Azure. --- test/src/unit-vfs.cc | 14 +++++++++++++- tiledb/sm/filesystem/azure.cc | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index b1c9b8b1f54c..939bfd58e6d6 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -442,6 +442,19 @@ TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) { Config config = set_config_params(disable_multipart, max_parallel_ops); VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; + // Getting file_size on a nonexistent blob shouldn't crash on Azure + uint64_t nbytes = 0; + URI non_existent = URI(path.to_string() + "non_existent"); + if (path.is_file()) { +#ifdef _WIN32 + CHECK(!vfs.file_size(non_existent, &nbytes).ok()); +#else + CHECK_THROWS(vfs.file_size(non_existent, &nbytes)); +#endif + } else { + CHECK(!vfs.file_size(non_existent, &nbytes).ok()); + } + // Set up bool exists = false; if (path.is_gcs() || path.is_s3() || path.is_azure()) { @@ -494,7 +507,6 @@ TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) { CHECK(exists); // Get file sizes - uint64_t nbytes = 0; require_tiledb_ok(vfs.file_size(largefile, &nbytes)); CHECK(nbytes == (buffer_size)); require_tiledb_ok(vfs.file_size(smallfile, &nbytes)); diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 8b8f2d882452..ea461f24b408 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -700,9 +700,9 @@ Status Azure::blob_size(const URI& uri, uint64_t* const nbytes) const { if (response.Blobs.empty()) { error_message = "Blob does not exist."; + } else { + *nbytes = static_cast(response.Blobs[0].BlobSize); } - - *nbytes = static_cast(response.Blobs[0].BlobSize); } catch (const ::Azure::Storage::StorageException& e) { error_message = e.Message; } From cab84bc230a29dee5699c1c96f9dd84559bcaea4 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 2 Apr 2024 17:39:18 +0200 Subject: [PATCH 269/456] Unstatus compressors directory. (#4842) --- TYPE: NO_HISTORY DESC: Unstatus compressors directory. --- test/src/unit-compression-dd.cc | 74 ++++----- test/src/unit-compression-rle.cc | 43 +++-- tiledb/common/exception/status.h | 8 - tiledb/sm/compressors/bzip_compressor.cc | 48 +++--- tiledb/sm/compressors/bzip_compressor.h | 11 +- tiledb/sm/compressors/dd_compressor.cc | 152 +++++++++--------- tiledb/sm/compressors/dd_compressor.h | 22 +-- tiledb/sm/compressors/delta_compressor.h | 2 +- tiledb/sm/compressors/gzip_compressor.cc | 38 ++--- tiledb/sm/compressors/gzip_compressor.h | 11 +- tiledb/sm/compressors/lz4_compressor.cc | 27 ++-- tiledb/sm/compressors/lz4_compressor.h | 11 +- tiledb/sm/compressors/rle_compressor.cc | 63 ++++---- tiledb/sm/compressors/rle_compressor.h | 11 +- tiledb/sm/compressors/util/gzip_wrappers.cc | 32 ++-- tiledb/sm/compressors/util/gzip_wrappers.h | 6 +- .../util/tdb_gzip_embedded_data.cc | 11 +- tiledb/sm/compressors/zstd_compressor.cc | 36 ++--- tiledb/sm/compressors/zstd_compressor.h | 10 +- tiledb/sm/filter/compression_filter.cc | 35 ++-- tiledb/sm/misc/mgc_dict.cc | 9 +- 21 files changed, 290 insertions(+), 370 deletions(-) diff --git a/test/src/unit-compression-dd.cc b/test/src/unit-compression-dd.cc index 14bc60d51c3d..1cb0763cdac1 100644 --- a/test/src/unit-compression-dd.cc +++ b/test/src/unit-compression-dd.cc @@ -46,21 +46,19 @@ TEST_CASE( int data[] = {1}; auto comp_in_buff = new tiledb::sm::ConstBuffer(data, sizeof(data)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer(comp_out_buff->data(), comp_out_buff->size()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(data)); + auto st = decomp_out_buff->realloc(sizeof(data)); REQUIRE(st.ok()); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int*)decomp_out_buff->data(); @@ -80,21 +78,18 @@ TEST_CASE( int data[] = {1, 2}; auto comp_in_buff = new tiledb::sm::ConstBuffer(data, sizeof(data)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer(comp_out_buff->data(), comp_out_buff->size()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(data)); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(decomp_out_buff->realloc(sizeof(data))); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int*)decomp_out_buff->data(); @@ -115,21 +110,19 @@ TEST_CASE( int data[] = {100, 300, 200}; auto comp_in_buff = new tiledb::sm::ConstBuffer(data, sizeof(data)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer(comp_out_buff->data(), comp_out_buff->size()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(data)); + auto st = decomp_out_buff->realloc(sizeof(data)); REQUIRE(st.ok()); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int*)decomp_out_buff->data(); @@ -151,21 +144,19 @@ TEST_CASE( int data[] = {100, 300, 200, 600}; auto comp_in_buff = new tiledb::sm::ConstBuffer(data, sizeof(data)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer( comp_out_buff->data(), comp_out_buff->offset()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(data)); + auto st = decomp_out_buff->realloc(sizeof(data)); REQUIRE(st.ok()); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int*)decomp_out_buff->data(); @@ -195,21 +186,18 @@ TEST_CASE( // Compress auto comp_in_buff = new tiledb::sm::ConstBuffer(data, n * sizeof(int)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT32, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer( comp_out_buff->data(), comp_out_buff->offset()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(int) * n); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(decomp_out_buff->realloc(sizeof(int) * n)); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT32, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int*)decomp_out_buff->data(); @@ -230,21 +218,19 @@ TEST_CASE( int8_t data[] = {-100, -101, 100, 101}; auto comp_in_buff = new tiledb::sm::ConstBuffer(data, sizeof(data)); auto comp_out_buff = new tiledb::sm::Buffer(); - auto st = tiledb::sm::DoubleDelta::compress( - tiledb::sm::Datatype::INT8, comp_in_buff, comp_out_buff); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::compress( + tiledb::sm::Datatype::INT8, comp_in_buff, comp_out_buff)); // Decompress auto decomp_in_buff = new tiledb::sm::ConstBuffer( comp_out_buff->data(), comp_out_buff->offset()); auto decomp_out_buff = new tiledb::sm::Buffer(); - st = decomp_out_buff->realloc(sizeof(data)); + auto st = decomp_out_buff->realloc(sizeof(data)); REQUIRE(st.ok()); tiledb::sm::PreallocatedBuffer prealloc_buf( decomp_out_buff->data(), decomp_out_buff->alloced_size()); - st = tiledb::sm::DoubleDelta::decompress( - tiledb::sm::Datatype::INT8, decomp_in_buff, &prealloc_buf); - REQUIRE(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::DoubleDelta::decompress( + tiledb::sm::Datatype::INT8, decomp_in_buff, &prealloc_buf)); // Check data auto decomp_data = (int8_t*)decomp_out_buff->data(); diff --git a/test/src/unit-compression-rle.cc b/test/src/unit-compression-rle.cc index 282c36cab4d9..1f418e0aeac9 100644 --- a/test/src/unit-compression-rle.cc +++ b/test/src/unit-compression-rle.cc @@ -50,14 +50,13 @@ TEST_CASE("Compression-RLE: Test invalid format", "[compression][rle]") { auto compressed = new Buffer(); // Test empty buffer - auto st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(!st.ok()); + REQUIRE_THROWS(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); delete input; // Test input buffer invalid format auto buff = new Buffer(); int int_v = 0; - st = buff->write(&int_v, sizeof(int)); + auto st = buff->write(&int_v, sizeof(int)); REQUIRE(st.ok()); char char_v = 'a'; st = buff->write(&char_v, sizeof(char)); @@ -66,8 +65,7 @@ TEST_CASE("Compression-RLE: Test invalid format", "[compression][rle]") { st = compressed->realloc(1000000); REQUIRE(st.ok()); input = new ConstBuffer(buff->data(), buff->size()); - st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(!st.ok()); + REQUIRE_THROWS(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); delete input; delete compressed; @@ -89,8 +87,7 @@ TEST_CASE("Compression-RLE: Test all values unique", "[compression][rle]") { // Create an input buffer and compress auto input = new ConstBuffer(data, sizeof(data)); - st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); delete input; // Decompress @@ -100,8 +97,8 @@ TEST_CASE("Compression-RLE: Test all values unique", "[compression][rle]") { PreallocatedBuffer prealloc_buf( decompressed->data(), decompressed->alloced_size()); REQUIRE(st.ok()); - st = tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf); - CHECK(st.ok()); + REQUIRE_NOTHROW( + tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf)); CHECK_FALSE(memcmp(data, decompressed->data(), sizeof(data))); delete input; @@ -129,8 +126,7 @@ TEST_CASE("Compression-RLE: Test all values the same", "[compression][rle]") { // Compress data auto input = new ConstBuffer(data, sizeof(data)); - st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); CHECK(compressed->size() == run_size); delete input; @@ -140,8 +136,8 @@ TEST_CASE("Compression-RLE: Test all values the same", "[compression][rle]") { PreallocatedBuffer prealloc_buf( decompressed->data(), decompressed->alloced_size()); input = new ConstBuffer(compressed->data(), compressed->size()); - st = tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf); - CHECK(st.ok()); + REQUIRE_NOTHROW( + tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf)); CHECK_FALSE(memcmp(data, decompressed->data(), sizeof(data))); delete input; @@ -172,8 +168,7 @@ TEST_CASE( st = compressed->realloc(compressed_size); REQUIRE(st.ok()); auto input = new ConstBuffer(data, sizeof(data)); - st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); CHECK(compressed->size() == 21 * run_size); delete input; @@ -184,8 +179,8 @@ TEST_CASE( PreallocatedBuffer prealloc_buf( decompressed->data(), decompressed->alloced_size()); input = new ConstBuffer(compressed->data(), compressed->size()); - st = tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf); - CHECK(st.ok()); + REQUIRE_NOTHROW( + tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf)); CHECK_FALSE(memcmp(data, decompressed->data(), sizeof(int))); delete input; @@ -217,8 +212,7 @@ TEST_CASE( st = compressed->realloc(compressed_size); REQUIRE(st.ok()); auto input = new ConstBuffer(data, sizeof(data)); - st = tiledb::sm::RLE::compress(sizeof(int), input, compressed); - CHECK(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::RLE::compress(sizeof(int), input, compressed)); CHECK(compressed->size() == 32 * run_size); delete input; @@ -228,8 +222,8 @@ TEST_CASE( PreallocatedBuffer prealloc_buf( decompressed->data(), decompressed->alloced_size()); input = new ConstBuffer(compressed->data(), compressed->size()); - st = tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf); - CHECK(st.ok()); + REQUIRE_NOTHROW( + tiledb::sm::RLE::decompress(sizeof(int), input, &prealloc_buf)); CHECK_FALSE(memcmp(data, decompressed->data(), sizeof(data))); delete input; @@ -274,8 +268,7 @@ TEST_CASE( st = compressed->realloc(compressed_size); REQUIRE(st.ok()); auto input = new ConstBuffer(data, sizeof(data)); - st = tiledb::sm::RLE::compress(value_size, input, compressed); - CHECK(st.ok()); + REQUIRE_NOTHROW(tiledb::sm::RLE::compress(value_size, input, compressed)); CHECK(compressed->size() == 21 * run_size); delete input; @@ -286,8 +279,8 @@ TEST_CASE( PreallocatedBuffer prealloc_buf( decompressed->data(), decompressed->alloced_size()); input = new ConstBuffer(compressed->data(), compressed->size()); - st = tiledb::sm::RLE::decompress(value_size, input, &prealloc_buf); - CHECK(st.ok()); + REQUIRE_NOTHROW( + tiledb::sm::RLE::decompress(value_size, input, &prealloc_buf)); CHECK_FALSE(memcmp(data, decompressed->data(), sizeof(data))); delete input; diff --git a/tiledb/common/exception/status.h b/tiledb/common/exception/status.h index f718906360b9..bc7eaae277e7 100644 --- a/tiledb/common/exception/status.h +++ b/tiledb/common/exception/status.h @@ -271,18 +271,10 @@ inline Status Status_ArraySchemaEvolutionError(const std::string& msg) { inline Status Status_IOError(const std::string& msg) { return {"[TileDB::IO] Error", msg}; } -/** Return a GZip error class Status with a given message **/ -inline Status Status_GZipError(const std::string& msg) { - return {"[TileDB::GZip] Error", msg}; -} /** Return a ChecksumError error class Status with a given message **/ inline Status Status_ChecksumError(const std::string& msg) { return {"[TileDB::ChecksumError] Error", msg}; } -/** Return a Compression error class Status with a given message **/ -inline Status Status_CompressionError(const std::string& msg) { - return {"[TileDB::Compression] Error", msg}; -} /** Return a Tile error class Status with a given message **/ inline Status Status_TileError(const std::string& msg) { return {"[TileDB::Tile] Error", msg}; diff --git a/tiledb/sm/compressors/bzip_compressor.cc b/tiledb/sm/compressors/bzip_compressor.cc index 239c2b0384a3..1dc9542b6504 100644 --- a/tiledb/sm/compressors/bzip_compressor.cc +++ b/tiledb/sm/compressors/bzip_compressor.cc @@ -42,12 +42,18 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status BZip::compress( +class BZipException : public StatusException { + public: + explicit BZipException(const std::string& message) + : StatusException("BZipException", message) { + } +}; + +void BZip::compress( int level, ConstBuffer* input_buffer, Buffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with BZip; invalid buffer format")); + throw BZipException("Failed compressing with BZip; invalid buffer format"); // Compress auto in_size = (unsigned int)input_buffer->size(); @@ -65,42 +71,38 @@ Status BZip::compress( if (rc != BZ_OK) { switch (rc) { case BZ_CONFIG_ERROR: - return Status_CompressionError( + throw BZipException( "BZip compression error: library has been miscompiled"); case BZ_PARAM_ERROR: - return Status_CompressionError( + throw BZipException( "BZip compression error: 'output_buffer' or 'output_buffer_size' " "is NULL"); case BZ_MEM_ERROR: - return Status_CompressionError( - "BZip compression error: insufficient memory"); + throw BZipException("BZip compression error: insufficient memory"); case BZ_OUTBUFF_FULL: - return Status_CompressionError( + throw BZipException( "BZip compression error: compressed size exceeds limits for " "'output_buffer_size'"); default: - return Status_CompressionError( - "BZip compression error: unknown error code"); + throw BZipException("BZip compression error: unknown error code"); } } // Set size of compressed data output_buffer->advance_size(out_size); output_buffer->advance_offset(out_size); - - return Status::Ok(); } -Status BZip::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { +void BZip::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { return BZip::compress(BZip::default_level(), input_buffer, output_buffer); } -Status BZip::decompress( +void BZip::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with BZip; invalid buffer format")); + throw BZipException( + "Failed decompressing with BZip; invalid buffer format"); // Decompress auto out_size = (unsigned int)output_buffer->free_space(); @@ -116,30 +118,26 @@ Status BZip::decompress( if (rc != BZ_OK) { switch (rc) { case BZ_CONFIG_ERROR: - return Status_CompressionError( + throw BZipException( "BZip decompression error: library has been miscompiled"); case BZ_PARAM_ERROR: - return Status_CompressionError( + throw BZipException( "BZip decompression error: 'output_buffer' or 'output_buffer_size' " "is NULL"); case BZ_MEM_ERROR: - return Status_CompressionError( - "BZip decompression error: insufficient memory"); + throw BZipException("BZip decompression error: insufficient memory"); case BZ_DATA_ERROR: case BZ_DATA_ERROR_MAGIC: case BZ_UNEXPECTED_EOF: - return Status_CompressionError( + throw BZipException( "BZip decompression error: compressed data is corrupted"); default: - return Status_CompressionError( - "BZip decompression error: unknown error code "); + throw BZipException("BZip decompression error: unknown error code "); } } // Set size of compressed data output_buffer->advance_offset(out_size); - - return Status::Ok(); } uint64_t BZip::overhead(uint64_t nbytes) { diff --git a/tiledb/sm/compressors/bzip_compressor.h b/tiledb/sm/compressors/bzip_compressor.h index 030cd745a15f..7e9af315471b 100644 --- a/tiledb/sm/compressors/bzip_compressor.h +++ b/tiledb/sm/compressors/bzip_compressor.h @@ -33,7 +33,7 @@ #ifndef TILEDB_BZIP_COMPRESSOR_H #define TILEDB_BZIP_COMPRESSOR_H -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" using namespace tiledb::common; @@ -53,9 +53,8 @@ class BZip { * @param level Compression level. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress( + static void compress( int level, ConstBuffer* input_buffer, Buffer* output_buffer); /** @@ -63,18 +62,16 @@ class BZip { * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress(ConstBuffer* input_buffer, Buffer* output_buffer); + static void compress(ConstBuffer* input_buffer, Buffer* output_buffer); /** * Decompression function. * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ - static Status decompress( + static void decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); /** Returns the default compression level. */ diff --git a/tiledb/sm/compressors/dd_compressor.cc b/tiledb/sm/compressors/dd_compressor.cc index 0010e2b05805..e46b6c4657a6 100644 --- a/tiledb/sm/compressors/dd_compressor.cc +++ b/tiledb/sm/compressors/dd_compressor.cc @@ -35,6 +35,13 @@ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" +class DoubleDeltaException : public StatusException { + public: + explicit DoubleDeltaException(const std::string& message) + : StatusException("DoubleDeltaException", message) { + } +}; + /* ****************************** */ /* MACROS */ /* ****************************** */ @@ -51,7 +58,7 @@ const uint64_t DoubleDelta::OVERHEAD = 17; /* API */ /* ****************************** */ -Status DoubleDelta::compress( +void DoubleDelta::compress( Datatype type, ConstBuffer* input_buffer, Buffer* output_buffer) { switch (type) { case Datatype::BLOB: @@ -110,17 +117,16 @@ Status DoubleDelta::compress( return DoubleDelta::compress(input_buffer, output_buffer); case Datatype::FLOAT32: case Datatype::FLOAT64: - return LOG_STATUS( - Status_CompressionError("DoubleDelta tile compression is not yet " - "supported for float types.")); + throw DoubleDeltaException( + "DoubleDelta tile compression is not yet supported for float types."); } assert(false); - return LOG_STATUS(Status_CompressionError( - "Cannot compress tile with DoubleDelta; Unsupported datatype")); + throw DoubleDeltaException( + "Cannot compress tile with DoubleDelta; Unsupported datatype"); } -Status DoubleDelta::decompress( +void DoubleDelta::decompress( Datatype type, ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { @@ -181,14 +187,14 @@ Status DoubleDelta::decompress( return DoubleDelta::decompress(input_buffer, output_buffer); case Datatype::FLOAT32: case Datatype::FLOAT64: - return LOG_STATUS( - Status_CompressionError("DoubleDelta tile decompression is not yet " - "supported for float types.")); + throw DoubleDeltaException( + "DoubleDelta tile decompression is not yet supported for float " + "types."); } assert(false); - return LOG_STATUS(Status_CompressionError( - "Cannot decompress tile with DoubleDelta; Unupported datatype")); + throw DoubleDeltaException( + "Cannot decompress tile with DoubleDelta; Unupported datatype"); } uint64_t DoubleDelta::overhead(uint64_t) { @@ -201,7 +207,7 @@ uint64_t DoubleDelta::overhead(uint64_t) { /* ****************************** */ template -Status DoubleDelta::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { +void DoubleDelta::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { // Calculate number of values and handle trivial case uint64_t value_size = sizeof(T); uint64_t num = input_buffer->size() / value_size; @@ -210,29 +216,29 @@ Status DoubleDelta::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { // Calculate bitsize (ignoring the sign bit) auto in = (T*)input_buffer->data(); unsigned int bitsize; - RETURN_NOT_OK(compute_bitsize(in, num, &bitsize)); + compute_bitsize(in, num, &bitsize); assert(bitsize <= std::numeric_limits::max()); auto bitsize_c = static_cast(bitsize); // Write bitsize and number of values - RETURN_NOT_OK(output_buffer->write(&bitsize_c, sizeof(uint8_t))); - RETURN_NOT_OK(output_buffer->write(&num, sizeof(uint64_t))); + throw_if_not_ok(output_buffer->write(&bitsize_c, sizeof(uint8_t))); + throw_if_not_ok(output_buffer->write(&num, sizeof(uint64_t))); // Trivial case - no compression if (bitsize >= sizeof(T) * 8 - 1) { - RETURN_NOT_OK(output_buffer->write(in, input_buffer->size())); - return Status::Ok(); + throw_if_not_ok(output_buffer->write(in, input_buffer->size())); + return; } // Write first value - RETURN_NOT_OK(output_buffer->write(&in[0], value_size)); + throw_if_not_ok(output_buffer->write(&in[0], value_size)); if (num == 1) - return Status::Ok(); + return; // Write second value - RETURN_NOT_OK(output_buffer->write(&in[1], value_size)); + throw_if_not_ok(output_buffer->write(&in[1], value_size)); if (num == 2) - return Status::Ok(); + return; // Write double deltas int64_t prev_delta = int64_t(in[1]) - int64_t(in[0]); @@ -241,27 +247,23 @@ Status DoubleDelta::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { for (uint64_t i = 2; i < num; ++i) { int64_t cur_delta = int64_t(in[i]) - int64_t(in[i - 1]); int64_t dd = cur_delta - prev_delta; - RETURN_NOT_OK( - write_double_delta(output_buffer, dd, bitsize, &chunk, &bit_in_chunk)); + write_double_delta(output_buffer, dd, bitsize, &chunk, &bit_in_chunk); prev_delta = cur_delta; } // Write whatever is left in the chunk if (bit_in_chunk < 63) - RETURN_NOT_OK(output_buffer->write(&chunk, sizeof(uint64_t))); - - return Status::Ok(); + throw_if_not_ok(output_buffer->write(&chunk, sizeof(uint64_t))); } template -Status DoubleDelta::compute_bitsize( - T* in, uint64_t num, unsigned int* bitsize) { +void DoubleDelta::compute_bitsize(T* in, uint64_t num, unsigned int* bitsize) { // Find maximum absolute double delta *bitsize = 0; int64_t max = 0; if (num <= 2) { *bitsize = 0; - return Status::Ok(); + return; } int64_t prev_delta = int64_t(in[1]) - int64_t(in[0]); char delta_out_of_bounds = 0; @@ -275,68 +277,64 @@ Status DoubleDelta::compute_bitsize( } // Handle error if (delta_out_of_bounds) { - return LOG_STATUS( - Status_CompressionError("Cannot compress with DoubleDelta; Some " - "negative double delta is out of bounds")); + throw DoubleDeltaException( + "Cannot compress with DoubleDelta; Some negative double delta is out " + "of bounds"); } // Calculate bitsize of the maximum absolute double delta do { ++(*bitsize); max >>= 1; } while (max); - return Status::Ok(); } template -Status DoubleDelta::decompress( +void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Read bitsize and number of values uint8_t bitsize_c = 0; uint64_t num = 0; uint64_t value_size = sizeof(T); - RETURN_NOT_OK(input_buffer->read(&bitsize_c, sizeof(uint8_t))); - RETURN_NOT_OK(input_buffer->read(&num, sizeof(uint64_t))); + throw_if_not_ok(input_buffer->read(&bitsize_c, sizeof(uint8_t))); + throw_if_not_ok(input_buffer->read(&num, sizeof(uint64_t))); auto bitsize = static_cast(bitsize_c); auto out = (T*)output_buffer->cur_data(); // Trivial case - no compression if (bitsize >= sizeof(T) * 8 - 1) { - RETURN_NOT_OK(output_buffer->write( + throw_if_not_ok(output_buffer->write( input_buffer->cur_data(), input_buffer->nbytes_left_to_read())); - return Status::Ok(); + return; } // Read first value T value; - RETURN_NOT_OK(input_buffer->read(&value, value_size)); - RETURN_NOT_OK(output_buffer->write(&value, value_size)); + throw_if_not_ok(input_buffer->read(&value, value_size)); + throw_if_not_ok(output_buffer->write(&value, value_size)); if (num == 1) - return Status::Ok(); + return; // Read second value - RETURN_NOT_OK(input_buffer->read(&value, value_size)); - RETURN_NOT_OK(output_buffer->write(&value, value_size)); + throw_if_not_ok(input_buffer->read(&value, value_size)); + throw_if_not_ok(output_buffer->write(&value, value_size)); if (num == 2) - return Status::Ok(); + return; // Read first chunk uint64_t chunk; - RETURN_NOT_OK(input_buffer->read(&chunk, sizeof(uint64_t))); + throw_if_not_ok(input_buffer->read(&chunk, sizeof(uint64_t))); int bit_in_chunk = 63; // Leftmost bit (MSB) // Decompress rest of the values int64_t dd; for (uint64_t i = 2; i < num; ++i) { - RETURN_NOT_OK( - read_double_delta(input_buffer, &dd, bitsize, &chunk, &bit_in_chunk)); + read_double_delta(input_buffer, &dd, bitsize, &chunk, &bit_in_chunk); value = (T)(dd + 2 * (int64_t)out[i - 1] - (int64_t)out[i - 2]); - RETURN_NOT_OK(output_buffer->write(&value, value_size)); + throw_if_not_ok(output_buffer->write(&value, value_size)); } - - return Status::Ok(); } -Status DoubleDelta::read_double_delta( +void DoubleDelta::read_double_delta( ConstBuffer* buff, int64_t* double_delta, int bitsize, @@ -350,7 +348,7 @@ Status DoubleDelta::read_double_delta( // Read chunk and reset if (*bit_in_chunk < 0) { - RETURN_NOT_OK(buff->read(chunk, sizeof(uint64_t))); + throw_if_not_ok(buff->read(chunk, sizeof(uint64_t))); *bit_in_chunk = 63; } @@ -372,7 +370,7 @@ Status DoubleDelta::read_double_delta( // Read chunk and reset if (*bit_in_chunk < 0 && buff->offset() != buff->size()) { - RETURN_NOT_OK(buff->read(chunk, sizeof(uint64_t))); + throw_if_not_ok(buff->read(chunk, sizeof(uint64_t))); *bit_in_chunk = 63; bits_to_read_from_chunk = std::min(*bit_in_chunk + 1, bits_left_to_read); } @@ -380,11 +378,9 @@ Status DoubleDelta::read_double_delta( // Set sign (*double_delta) *= sign; - - return Status::Ok(); } -Status DoubleDelta::write_double_delta( +void DoubleDelta::write_double_delta( Buffer* buff, int64_t double_delta, int bitsize, @@ -398,7 +394,7 @@ Status DoubleDelta::write_double_delta( // Write chunk and reset if (*bit_in_chunk < 0) { - RETURN_NOT_OK(buff->write(chunk, sizeof(uint64_t))); + throw_if_not_ok(buff->write(chunk, sizeof(uint64_t))); *bit_in_chunk = 63; *chunk = 0; } @@ -422,54 +418,52 @@ Status DoubleDelta::write_double_delta( // Write chunk and reset if (*bit_in_chunk < 0) { - RETURN_NOT_OK(buff->write(chunk, sizeof(uint64_t))); + throw_if_not_ok(buff->write(chunk, sizeof(uint64_t))); *bit_in_chunk = 63; *chunk = 0; bits_to_fill_in_chunk = std::min((*bit_in_chunk) + 1, bits_left_to_write); } } - - return Status::Ok(); } // Explicit template instantiations -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::compress( +template void DoubleDelta::compress( ConstBuffer* input_buffer, Buffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); -template Status DoubleDelta::decompress( +template void DoubleDelta::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); } // namespace tiledb::sm diff --git a/tiledb/sm/compressors/dd_compressor.h b/tiledb/sm/compressors/dd_compressor.h index 517e3c136c19..ca8ade9b9b7b 100644 --- a/tiledb/sm/compressors/dd_compressor.h +++ b/tiledb/sm/compressors/dd_compressor.h @@ -33,7 +33,7 @@ #ifndef TILEDB_DOUBLE_DELTA_H #define TILEDB_DOUBLE_DELTA_H -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" using namespace tiledb::common; @@ -90,7 +90,6 @@ class DoubleDelta { * @param type The type of the input values. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status * * @note The function will fail with an error in two cases: (i) the output * buffer incurs a memory allocation error, and (ii) some double delta @@ -101,7 +100,7 @@ class DoubleDelta { * resulting in a positive number. However, both these two cases are * extreme. */ - static Status compress( + static void compress( Datatype type, ConstBuffer* input_buffer, Buffer* output_buffer); /** @@ -110,9 +109,8 @@ class DoubleDelta { * @param type The type of the original decompressed values. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ - static Status decompress( + static void decompress( Datatype type, ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); @@ -127,7 +125,7 @@ class DoubleDelta { /** Templated version of *compress* on the type of buffer values. */ template - static Status compress(ConstBuffer* input_buffer, Buffer* output_buffer); + static void compress(ConstBuffer* input_buffer, Buffer* output_buffer); /** * Calculates the bitsize all the double deltas will have. Note that @@ -137,10 +135,9 @@ class DoubleDelta { * @param in The input buffer. * @param num The number of values in the buffer. * @param bitsize The bitsize of the double deltas to be retrieved. - * @return Status */ template - static Status compute_bitsize(T* in, uint64_t num, unsigned int* bitsize); + static void compute_bitsize(T* in, uint64_t num, unsigned int* bitsize); /** * Decompression function. @@ -148,10 +145,9 @@ class DoubleDelta { * @tparam The datatype of the values. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ template - static Status decompress( + static void decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); /** @@ -164,9 +160,8 @@ class DoubleDelta { * This parameters represents some state about the current chunk. * @param bit_in_chunk Part of the chunk state. Stores the current bit * in the chunk the next read will start from. - * @return Status */ - static Status read_double_delta( + static void read_double_delta( ConstBuffer* buff, int64_t* double_delta, int bitsize, @@ -185,9 +180,8 @@ class DoubleDelta { * This parameters represents some state about the current chunk. * @param bit_in_chunk Part of the chunk state. Stores the current bit * in the chunk the next write will start from. - * @return Status */ - static Status write_double_delta( + static void write_double_delta( Buffer* buff, int64_t double_delta, int bitsize, diff --git a/tiledb/sm/compressors/delta_compressor.h b/tiledb/sm/compressors/delta_compressor.h index be07f3151960..47f2a7e887d2 100644 --- a/tiledb/sm/compressors/delta_compressor.h +++ b/tiledb/sm/compressors/delta_compressor.h @@ -33,7 +33,7 @@ #ifndef TILEDB_DELTA_H #define TILEDB_DELTA_H -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" using namespace tiledb::common; diff --git a/tiledb/sm/compressors/gzip_compressor.cc b/tiledb/sm/compressors/gzip_compressor.cc index e80d2c39ca9f..d3fc3a70f547 100644 --- a/tiledb/sm/compressors/gzip_compressor.cc +++ b/tiledb/sm/compressors/gzip_compressor.cc @@ -43,16 +43,22 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status GZip::compress( +class GZipException : public StatusException { + public: + explicit GZipException(const std::string& message) + : StatusException("GZipException", message) { + } +}; + +void GZip::compress( int level, ConstBuffer* input_buffer, Buffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with GZip; invalid buffer format")); + throw GZipException("Failed compressing with GZip; invalid buffer format"); if (level > GZip::maximum_level()) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with GZip; invalid compression level.")); + throw GZipException( + "Failed compressing with GZip; invalid compression level."); int ret; z_stream strm; @@ -68,7 +74,7 @@ Status GZip::compress( if ((ret != Z_MEM_ERROR && ret != Z_STREAM_ERROR)) { (void)deflateEnd(&strm); } - return LOG_STATUS(Status_GZipError("Cannot compress with GZIP")); + throw GZipException("Cannot compress with GZIP"); } // Compress @@ -83,26 +89,24 @@ Status GZip::compress( // Return if (ret == Z_STREAM_ERROR || strm.avail_in != 0) - return LOG_STATUS(Status_GZipError("Cannot compress with GZIP")); + throw GZipException("Cannot compress with GZIP"); // Set size of compressed data uint64_t compressed_size = output_buffer->free_space() - strm.avail_out; output_buffer->advance_size(compressed_size); output_buffer->advance_offset(compressed_size); - - return Status::Ok(); } -Status GZip::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { +void GZip::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { return GZip::compress(GZip::default_level(), input_buffer, output_buffer); } -Status GZip::decompress( +void GZip::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with GZip; invalid buffer format")); + throw GZipException( + "Failed decompressing with GZip; invalid buffer format"); int ret; z_stream strm; @@ -116,7 +120,7 @@ Status GZip::decompress( ret = inflateInit(&strm); if (ret != Z_OK) { - return LOG_STATUS(Status_GZipError("Cannot decompress with GZIP")); + throw GZipException("Cannot decompress with GZIP"); } // Decompress @@ -127,8 +131,7 @@ Status GZip::decompress( ret = inflate(&strm, Z_FINISH); if (ret != Z_STREAM_END) { - return LOG_STATUS( - Status_GZipError("Cannot decompress with GZIP, Stream Error")); + throw GZipException("Cannot decompress with GZIP, Stream Error"); } // Set size of decompressed data @@ -137,9 +140,6 @@ Status GZip::decompress( // Clean up (void)inflateEnd(&strm); - - // Success - return Status::Ok(); } uint64_t GZip::overhead(uint64_t buffer_size) { diff --git a/tiledb/sm/compressors/gzip_compressor.h b/tiledb/sm/compressors/gzip_compressor.h index 43c42ac435ab..3ffae5e410c6 100644 --- a/tiledb/sm/compressors/gzip_compressor.h +++ b/tiledb/sm/compressors/gzip_compressor.h @@ -33,7 +33,7 @@ #ifndef TILEDB_GZIP_H #define TILEDB_GZIP_H -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" #include @@ -55,9 +55,8 @@ class GZip { * @param level Compression level. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress( + static void compress( int level, ConstBuffer* input_buffer, Buffer* output_buffer); /** @@ -65,18 +64,16 @@ class GZip { * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress(ConstBuffer* input_buffer, Buffer* output_buffer); + static void compress(ConstBuffer* input_buffer, Buffer* output_buffer); /** * Decompression function. * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ - static Status decompress( + static void decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); /** Returns the compression overhead for the given input. */ diff --git a/tiledb/sm/compressors/lz4_compressor.cc b/tiledb/sm/compressors/lz4_compressor.cc index 361d812c172d..f31191c38796 100644 --- a/tiledb/sm/compressors/lz4_compressor.cc +++ b/tiledb/sm/compressors/lz4_compressor.cc @@ -43,11 +43,17 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status LZ4::compress(int, ConstBuffer* input_buffer, Buffer* output_buffer) { +class LZ4Exception : public StatusException { + public: + explicit LZ4Exception(const std::string& message) + : StatusException("LZ4Exception", message) { + } +}; + +void LZ4::compress(int, ConstBuffer* input_buffer, Buffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with LZ4; invalid buffer format")); + throw LZ4Exception("Failed compressing with LZ4; invalid buffer format"); // Compress #if LZ4_VERSION_NUMBER >= 10705 @@ -66,25 +72,22 @@ Status LZ4::compress(int, ConstBuffer* input_buffer, Buffer* output_buffer) { // Check error if (ret < 0) - return Status_CompressionError("LZ4 compression failed"); + throw LZ4Exception("LZ4 compression failed"); // Set size of compressed data output_buffer->advance_size(static_cast(ret)); output_buffer->advance_offset(static_cast(ret)); - - return Status::Ok(); } -Status LZ4::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { +void LZ4::compress(ConstBuffer* input_buffer, Buffer* output_buffer) { return LZ4::compress(LZ4::default_level(), input_buffer, output_buffer); } -Status LZ4::decompress( +void LZ4::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with LZ4; invalid buffer format")); + throw LZ4Exception("Failed decompressing with LZ4; invalid buffer format"); // Decompress int ret = LZ4_decompress_safe( @@ -95,12 +98,10 @@ Status LZ4::decompress( // Check error if (ret < 0) - return Status_CompressionError("LZ4 decompression failed"); + throw LZ4Exception("LZ4 decompression failed"); // Set size of decompressed data output_buffer->advance_offset(static_cast(ret)); - - return Status::Ok(); } uint64_t LZ4::overhead(uint64_t nbytes) { diff --git a/tiledb/sm/compressors/lz4_compressor.h b/tiledb/sm/compressors/lz4_compressor.h index 311ce6b2820c..66df65f7e065 100644 --- a/tiledb/sm/compressors/lz4_compressor.h +++ b/tiledb/sm/compressors/lz4_compressor.h @@ -33,7 +33,7 @@ #ifndef TILEDB_LZ4_COMPRESSOR_H #define TILEDB_LZ4_COMPRESSOR_H -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" using namespace tiledb::common; @@ -53,9 +53,8 @@ class LZ4 { * @param level Compression level. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress( + static void compress( int level, ConstBuffer* input_buffer, Buffer* output_buffer); /** @@ -63,18 +62,16 @@ class LZ4 { * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress(ConstBuffer* input_buffer, Buffer* output_buffer); + static void compress(ConstBuffer* input_buffer, Buffer* output_buffer); /** * Decompression function. * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ - static Status decompress( + static void decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); /** Returns the default compression level. */ diff --git a/tiledb/sm/compressors/rle_compressor.cc b/tiledb/sm/compressors/rle_compressor.cc index 549a4f0c82f1..e2ee50fcd8ce 100644 --- a/tiledb/sm/compressors/rle_compressor.cc +++ b/tiledb/sm/compressors/rle_compressor.cc @@ -41,12 +41,18 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status RLE::compress( +class RLEException : public StatusException { + public: + explicit RLEException(const std::string& message) + : StatusException("RLEException", message) { + } +}; + +void RLE::compress( uint64_t value_size, ConstBuffer* input_buffer, Buffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with RLE; null input buffer")); + throw RLEException("Failed compressing with RLE; null input buffer"); unsigned int cur_run_len = 1; unsigned int max_run_len = 65535; auto input_cur = (const unsigned char*)input_buffer->data() + value_size; @@ -56,12 +62,12 @@ Status RLE::compress( // Trivial case if (value_num == 0) - return Status::Ok(); + return; // Sanity check on input buffer if (input_buffer->size() % value_size != 0) { - return LOG_STATUS(Status_CompressionError( - "Failed compressing with RLE; invalid input buffer format")); + throw RLEException( + "Failed compressing with RLE; invalid input buffer format"); } // Make runs @@ -71,11 +77,11 @@ Status RLE::compress( ++cur_run_len; } else { // Save the run // Copy to output buffer - RETURN_NOT_OK(output_buffer->write(input_prev, value_size)); + throw_if_not_ok(output_buffer->write(input_prev, value_size)); byte = (unsigned char)(cur_run_len >> 8); - RETURN_NOT_OK(output_buffer->write(&byte, sizeof(char))); + throw_if_not_ok(output_buffer->write(&byte, sizeof(char))); byte = (unsigned char)(cur_run_len % 256); - RETURN_NOT_OK(output_buffer->write(&byte, sizeof(char))); + throw_if_not_ok(output_buffer->write(&byte, sizeof(char))); // Reset current run length cur_run_len = 1; @@ -87,23 +93,20 @@ Status RLE::compress( } // Save final run - RETURN_NOT_OK(output_buffer->write(input_prev, value_size)); + throw_if_not_ok(output_buffer->write(input_prev, value_size)); byte = (unsigned char)(cur_run_len >> 8); - RETURN_NOT_OK(output_buffer->write(&byte, sizeof(char))); + throw_if_not_ok(output_buffer->write(&byte, sizeof(char))); byte = (unsigned char)(cur_run_len % 256); - RETURN_NOT_OK(output_buffer->write(&byte, sizeof(char))); - - return Status::Ok(); + throw_if_not_ok(output_buffer->write(&byte, sizeof(char))); } -Status RLE::decompress( +void RLE::decompress( uint64_t value_size, ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with RLE; null input buffer")); + throw RLEException("Failed decompressing with RLE; null input buffer"); auto input_cur = static_cast(input_buffer->data()); uint64_t run_size = value_size + 2 * sizeof(char); @@ -112,12 +115,12 @@ Status RLE::decompress( // Trivial case if (run_num == 0) - return Status::Ok(); + return; // Sanity check on input buffer format if (input_buffer->size() % run_size != 0) { - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with RLE; invalid input buffer format")); + throw RLEException( + "Failed decompressing with RLE; invalid input buffer format"); } // Decompress runs @@ -130,13 +133,11 @@ Status RLE::decompress( // Copy to output buffer for (uint64_t j = 0; j < run_len; ++j) - RETURN_NOT_OK(output_buffer->write(input_cur, value_size)); + throw_if_not_ok(output_buffer->write(input_cur, value_size)); // Update input/output tracking info input_cur += run_size; } - - return Status::Ok(); } uint64_t RLE::overhead(uint64_t nbytes, uint64_t value_size) { @@ -193,15 +194,15 @@ tuple RLE::calculate_compression_params( output_strings_size}; } -Status RLE::compress( +void RLE::compress( const span input, uint64_t rle_len_size, uint64_t string_len_size, span output) { if (input.empty() || output.empty() || rle_len_size == 0 || string_len_size == 0) { - return LOG_STATUS(Status_CompressionError( - "Failed compressing strings with RLE; empty input arguments")); + throw RLEException( + "Failed compressing strings with RLE; empty input arguments"); } if (rle_len_size <= 1) { @@ -245,19 +246,17 @@ Status RLE::compress( compress(input, output); } } - - return Status::Ok(); } -Status RLE::decompress( +void RLE::decompress( const span input, const uint8_t rle_len_size, const uint8_t string_len_size, span output, span output_offsets) { if (input.empty() || rle_len_size == 0 || string_len_size == 0) { - return LOG_STATUS(Status_CompressionError( - "Failed decompressing strings with RLE; empty input arguments")); + throw RLEException( + "Failed decompressing strings with RLE; empty input arguments"); } if (rle_len_size <= 1) { @@ -301,8 +300,6 @@ Status RLE::decompress( decompress(input, output, output_offsets); } } - - return Status::Ok(); } } // namespace sm diff --git a/tiledb/sm/compressors/rle_compressor.h b/tiledb/sm/compressors/rle_compressor.h index 56f043fe3b73..0cfbcada00a8 100644 --- a/tiledb/sm/compressors/rle_compressor.h +++ b/tiledb/sm/compressors/rle_compressor.h @@ -34,7 +34,6 @@ #define TILEDB_RLE_COMPRESSOR_H #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/endian.h" @@ -59,9 +58,8 @@ class RLE { * @param value_size The size of a single value. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the compressed data to. - * @return Status */ - static Status compress( + static void compress( uint64_t value_size, ConstBuffer* input_buffer, Buffer* output_buffer); /** @@ -70,9 +68,8 @@ class RLE { * @param value_size The size of a single. * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the decompressed data. - * @return Status */ - static Status decompress( + static void decompress( uint64_t value_size, ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); @@ -205,7 +202,7 @@ class RLE { * length|string_size|string] items. Memory is allocated and owned by the * caller */ - static Status compress( + static void compress( const span input, uint64_t rle_len_size, uint64_t string_len_size, @@ -222,7 +219,7 @@ class RLE { * @param output_offsets Output offsets reconstructed from input * Memory is allocated and owned by the caller */ - static Status decompress( + static void decompress( const span input, const uint8_t rle_len_size, const uint8_t string_len_size, diff --git a/tiledb/sm/compressors/util/gzip_wrappers.cc b/tiledb/sm/compressors/util/gzip_wrappers.cc index a895bce12524..cb8912e34078 100644 --- a/tiledb/sm/compressors/util/gzip_wrappers.cc +++ b/tiledb/sm/compressors/util/gzip_wrappers.cc @@ -35,7 +35,7 @@ #include -Status gzip_compress( +void gzip_compress( shared_ptr& out_gzipped_buf, const void* in_bytes, uint64_t nbytes) { @@ -50,20 +50,17 @@ Status gzip_compress( tiledb::sm::ConstBuffer const_in_buf(in_bytes, nbytes); // Ensure space in output buffer for worst acceptable case. if (!out_gzipped_buf->realloc(nbytes + overhead_size).ok()) { - return LOG_STATUS( - Status_CompressionError("gzip output buffer allocation error")); + throw std::logic_error("gzip output buffer allocation error"); } out_gzipped_buf->reset_size(); out_gzipped_buf->advance_size(overhead_size); out_gzipped_buf->advance_offset(overhead_size); - if (auto stat = - tiledb::sm::GZip::compress(9, &const_in_buf, out_gzipped_buf.get()); - !stat.ok()) { - // TODO: Handle possibility that 'error' is just 'not enuf buffer', i.e. - // unable to compress into <= space of in_buf (currently that is not - // returned as a distinct error from Gzip::compress()) - return stat; - } + tiledb::sm::GZip::compress(9, &const_in_buf, out_gzipped_buf.get()); + + // TODO: Handle possibility that 'error' is just 'not enuf buffer', i.e. + // unable to compress into <= space of in_buf (currently that is not + // returned as a distinct error from Gzip::compress()) + // adjust to size actually used out_gzipped_buf->set_size(out_gzipped_buf->offset()); uint64_t compressed_size = out_gzipped_buf->offset() - overhead_size; @@ -71,15 +68,13 @@ Status gzip_compress( // return next 'write()' position to beginning of buffer out_gzipped_buf->reset_offset(); // write sizes to beginning of buffer - RETURN_NOT_OK( + throw_if_not_ok( out_gzipped_buf->write(&uncompressed_size, sizeof(uncompressed_size))); - RETURN_NOT_OK( + throw_if_not_ok( out_gzipped_buf->write(&compressed_size, sizeof(compressed_size))); - - return Status::Ok(); } -Status gzip_decompress( +void gzip_decompress( shared_ptr& out_buf, const uint8_t* comp_buf) { /** * Expected compressed buffer format: @@ -101,8 +96,5 @@ Status gzip_decompress( tiledb::sm::ConstBuffer gzipped_input_buffer(comp_data, compressed_size); - RETURN_NOT_OK( - tiledb::sm::GZip::decompress(&gzipped_input_buffer, &pa_gunzip_out_buf)); - - return Status::Ok(); + tiledb::sm::GZip::decompress(&gzipped_input_buffer, &pa_gunzip_out_buf); } diff --git a/tiledb/sm/compressors/util/gzip_wrappers.h b/tiledb/sm/compressors/util/gzip_wrappers.h index 9790ce062378..bafacfc7cf30 100644 --- a/tiledb/sm/compressors/util/gzip_wrappers.h +++ b/tiledb/sm/compressors/util/gzip_wrappers.h @@ -48,9 +48,8 @@ * * @param out_gzipped_buf The returned buffer with the compressed data. * @param in_buf The input buffer containing data to be compressed - * @return Status */ -Status gzip_compress( +void gzip_compress( shared_ptr& out_gzipped_buf, const void* in_bytes, uint64_t nbytes); @@ -61,9 +60,8 @@ Status gzip_compress( * * @param out_buf The input buffer containing data to be compressed * @param comp_buf The buffer with the compressed data to be decompressed - * @return Status */ -Status gzip_decompress( +void gzip_decompress( shared_ptr& out_buf, const uint8_t* comp_buf); #endif // TILEDB_GZIP_WRAPPER_H diff --git a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc index 6c15ad3d66a4..a3384443fb24 100644 --- a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc +++ b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc @@ -110,10 +110,7 @@ int main(int argc, char* argv[]) { tiledb::sm::Buffer* out_buffer_ptr = zipped_buf->buffer_ptr(0); assert(out_buffer_ptr != nullptr); out_buffer_ptr->reset_offset(); - if (!tiledb::sm::GZip::compress(9, &const_inbuf, out_buffer_ptr).ok()) { - printf("Error compressing data!\n"); - exit(-4); - } + tiledb::sm::GZip::compress(9, &const_inbuf, out_buffer_ptr); std::cerr << "sizes input " << inbuf->size() << ", compressed " << out_buffer_ptr->size() << std::endl; if (0) // leave available for future possible diagnostic need @@ -181,8 +178,7 @@ int main(int argc, char* argv[]) { // brief sanity check that wrapper compression matches unwrapped compression shared_ptr out_gzipped_buf = make_shared(HERE()); - throw_if_not_ok( - gzip_compress(out_gzipped_buf, inbuf->data(0), inbuf->size())); + gzip_compress(out_gzipped_buf, inbuf->data(0), inbuf->size()); if (out_gzipped_buf->size() != tdb_gzip_buf->size()) { printf( "Error, compressed data sizes mismatch! %" PRIu64 ", %" PRIu64 "u\n", @@ -204,8 +200,7 @@ int main(int argc, char* argv[]) { // brief sanity check the decompressed()d data matches original tdb_gzip_buf->set_offset(0); - throw_if_not_ok(gzip_decompress( - expanded_buffer, static_cast(tdb_gzip_buf->data()))); + gzip_decompress(expanded_buffer, static_cast(tdb_gzip_buf->data())); if (expanded_buffer->size() != inbuf->size()) { fprintf(stderr, "re-expanded size different from original size!\n"); exit(-29); diff --git a/tiledb/sm/compressors/zstd_compressor.cc b/tiledb/sm/compressors/zstd_compressor.cc index a799277b9df3..203d1c0bac31 100644 --- a/tiledb/sm/compressors/zstd_compressor.cc +++ b/tiledb/sm/compressors/zstd_compressor.cc @@ -42,19 +42,25 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -Status ZStd::compress( +class ZStdException : public StatusException { + public: + explicit ZStdException(const std::string& message) + : StatusException("ZStdException", message) { + } +}; + +void ZStd::compress( int level, shared_ptr> compress_ctx_pool, ConstBuffer* input_buffer, Buffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed compressing with ZStd; invalid buffer format")); + throw ZStdException("Failed compressing with ZStd; invalid buffer format"); if (compress_ctx_pool == nullptr) { - return LOG_STATUS(Status_CompressionError( - "Failed compressing with ZStd; Resource pool not initialized")); + throw ZStdException( + "Failed compressing with ZStd; Resource pool not initialized"); } ResourceGuard context_guard(*compress_ctx_pool); @@ -72,30 +78,27 @@ Status ZStd::compress( // Handle error if (ZSTD_isError(zstd_ret) != 0) { const char* msg = ZSTD_getErrorName(zstd_ret); - return LOG_STATUS(Status_CompressionError( - std::string("ZStd compression failed: ") + msg)); + throw ZStdException(std::string("ZStd compression failed: ") + msg); } // Set size of compressed data output_buffer->advance_size(zstd_ret); output_buffer->advance_offset(zstd_ret); - - return Status::Ok(); } -Status ZStd::decompress( +void ZStd::decompress( shared_ptr> decompress_ctx_pool, ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with ZStd; invalid buffer format")); + throw ZStdException( + "Failed decompressing with ZStd; invalid buffer format"); if (decompress_ctx_pool == nullptr) { - return LOG_STATUS(Status_CompressionError( - "Failed decompressing with ZStd; Resource pool not initialized")); + throw ZStdException( + "Failed decompressing with ZStd; Resource pool not initialized"); } ResourceGuard context_guard(*decompress_ctx_pool); @@ -112,14 +115,11 @@ Status ZStd::decompress( // Check error if (ZSTD_isError(zstd_ret) != 0) { const char* msg = ZSTD_getErrorName(zstd_ret); - return LOG_STATUS(Status_CompressionError( - std::string("ZStd decompression failed: ") + msg)); + throw ZStdException(std::string("ZStd decompression failed: ") + msg); } // Set size decompressed data output_buffer->advance_offset(zstd_ret); - - return Status::Ok(); } uint64_t ZStd::overhead(uint64_t nbytes) { diff --git a/tiledb/sm/compressors/zstd_compressor.h b/tiledb/sm/compressors/zstd_compressor.h index 31da26114de3..9db1c63cf74c 100644 --- a/tiledb/sm/compressors/zstd_compressor.h +++ b/tiledb/sm/compressors/zstd_compressor.h @@ -34,7 +34,6 @@ #define TILEDB_ZSTD_H #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/misc/resource_pool.h" @@ -91,9 +90,8 @@ class ZStd { * @param compress_ctx_pool Resource pool to manage compression context reuse * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress( + static void compress( int level, shared_ptr> compress_ctx_pool, ConstBuffer* input_buffer, @@ -104,9 +102,8 @@ class ZStd { * * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write to the compressed data. - * @return Status */ - static Status compress(ConstBuffer* input_buffer, Buffer* output_buffer); + static void compress(ConstBuffer* input_buffer, Buffer* output_buffer); /** * Decompression function. @@ -115,9 +112,8 @@ class ZStd { * reuse * @param input_buffer Input buffer to read from. * @param output_buffer Output buffer to write the decompressed data to. - * @return Status */ - static Status decompress( + static void decompress( shared_ptr> decompress_ctx_pool, ConstBuffer* input_buffer, diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index ab14da94507a..107e0c13bf18 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -363,27 +363,26 @@ Status CompressionFilter::compress_part( uint32_t orig_size = (uint32_t)output->size(); switch (compressor_) { case Compressor::GZIP: - RETURN_NOT_OK(GZip::compress(level_, &input_buffer, output)); + GZip::compress(level_, &input_buffer, output); break; case Compressor::ZSTD: - RETURN_NOT_OK(ZStd::compress( - level_, zstd_compress_ctx_pool_, &input_buffer, output)); + ZStd::compress(level_, zstd_compress_ctx_pool_, &input_buffer, output); break; case Compressor::LZ4: - RETURN_NOT_OK(LZ4::compress(level_, &input_buffer, output)); + LZ4::compress(level_, &input_buffer, output); break; case Compressor::RLE: - RETURN_NOT_OK(RLE::compress(cell_size, &input_buffer, output)); + RLE::compress(cell_size, &input_buffer, output); break; case Compressor::BZIP2: - RETURN_NOT_OK(BZip::compress(level_, &input_buffer, output)); + BZip::compress(level_, &input_buffer, output); break; case Compressor::DOUBLE_DELTA: - RETURN_NOT_OK(DoubleDelta::compress( + DoubleDelta::compress( reinterpret_datatype_ == Datatype::ANY ? filter_data_type_ : reinterpret_datatype_, &input_buffer, - output)); + output); break; case Compressor::DICTIONARY_ENCODING: return LOG_STATUS( @@ -446,20 +445,20 @@ Status CompressionFilter::decompress_part( assert(0); break; case Compressor::GZIP: - st = GZip::decompress(&input_buffer, &output_buffer); + GZip::decompress(&input_buffer, &output_buffer); break; case Compressor::ZSTD: - st = ZStd::decompress( + ZStd::decompress( zstd_decompress_ctx_pool_, &input_buffer, &output_buffer); break; case Compressor::LZ4: - st = LZ4::decompress(&input_buffer, &output_buffer); + LZ4::decompress(&input_buffer, &output_buffer); break; case Compressor::RLE: - st = RLE::decompress(cell_size, &input_buffer, &output_buffer); + RLE::decompress(cell_size, &input_buffer, &output_buffer); break; case Compressor::BZIP2: - st = BZip::decompress(&input_buffer, &output_buffer); + BZip::decompress(&input_buffer, &output_buffer); break; case Compressor::DELTA: Delta::decompress( @@ -469,7 +468,7 @@ Status CompressionFilter::decompress_part( &output_buffer); break; case Compressor::DOUBLE_DELTA: - st = DoubleDelta::decompress( + DoubleDelta::decompress( reinterpret_datatype_ == Datatype::ANY ? filter_data_type_ : reinterpret_datatype_, &input_buffer, @@ -594,8 +593,8 @@ Status CompressionFilter::compress_var_string_coords( reinterpret_cast(data_buffer->data()), output_size_ub); if (compressor_ == Compressor::RLE) { - RETURN_NOT_OK(RLE::compress( - input_view, rle_len_bytesize, string_len_bytesize, output_view)); + RLE::compress( + input_view, rle_len_bytesize, string_len_bytesize, output_view); RETURN_NOT_OK(output_metadata.write(&rle_len_bytesize, sizeof(uint8_t))); RETURN_NOT_OK(output_metadata.write(&string_len_bytesize, sizeof(uint8_t))); } else if (compressor_ == Compressor::DICTIONARY_ENCODING) { @@ -645,12 +644,12 @@ Status CompressionFilter::decompress_var_string_coords( uint8_t rle_len_bytesize, string_len_bytesize; RETURN_NOT_OK(input_metadata.read(&rle_len_bytesize, sizeof(uint8_t))); RETURN_NOT_OK(input_metadata.read(&string_len_bytesize, sizeof(uint8_t))); - throw_if_not_ok(RLE::decompress( + RLE::decompress( input_view, rle_len_bytesize, string_len_bytesize, output_view, - offsets_view)); + offsets_view); } else if (compressor_ == Compressor::DICTIONARY_ENCODING) { uint8_t ids_bytesize = 0, string_len_bytesize = 0; uint32_t dict_size = 0; diff --git a/tiledb/sm/misc/mgc_dict.cc b/tiledb/sm/misc/mgc_dict.cc index ef834a556f96..5a42f253e851 100644 --- a/tiledb/sm/misc/mgc_dict.cc +++ b/tiledb/sm/misc/mgc_dict.cc @@ -55,12 +55,9 @@ void magic_dict::prepare_data() { return; expanded_buffer_ = make_shared(HERE()); - if (!gzip_decompress( - expanded_buffer_, - reinterpret_cast(&magic_mgc_compressed_bytes[0])) - .ok()) { - throw std::runtime_error("gzip_decompress failure!"); - } + gzip_decompress( + expanded_buffer_, + reinterpret_cast(&magic_mgc_compressed_bytes[0])); uncompressed_magic_dict_ = expanded_buffer_.get()->data(); } From 5745303cd950b2c759ca93867af0a0c07c88a99c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 2 Apr 2024 23:50:32 +0300 Subject: [PATCH 270/456] Document how to build a subset of dependencies with vcpkg. (#4844) Addresses https://github.com/TileDB-Inc/TileDB/pull/4570#pullrequestreview-1973581001 --- TYPE: NO_HISTORY --- doc/dev/BUILD.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/dev/BUILD.md b/doc/dev/BUILD.md index 1717cb0fa908..8d1380fbcb04 100644 --- a/doc/dev/BUILD.md +++ b/doc/dev/BUILD.md @@ -135,6 +135,10 @@ Vcpkg will not be automatically downloaded if: In these cases CMake will find the dependencies based on the rules of the [`find_package`](https://cmake.org/cmake/help/latest/command/find_package.html) command. The user is responsible for providing the dependencies. +#### Building a subset of dependencies with vcpkg + +To build only a subset of the dependencies with vcpkg, you will need to create empty port overlays for them. A guide with more details is available in the [Conda feedstock for TileDB](https://github.com/conda-forge/tiledb-feedstock/blob/main/recipe/tiledb-patches/system-ports/README.md). See also [the vcpkg team's guidance on system dependencies](https://devblogs.microsoft.com/cppblog/using-system-package-manager-dependencies-with-vcpkg/). + ### Building with sanitizers TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as the [vcpkg base triplet](#configuration-options) corresponding to your platform: From 8373822ff4d08b73b44ecdfa79a86b98a350a47d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:54:51 +0200 Subject: [PATCH 271/456] Update dev history with 2.21.0 and 2.21.1 content. (#4847) --- TYPE: NO_HISTORY DESC: Update dev history with 2.21.0 and 2.21.1 content. --- HISTORY.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index bd326298fc93..74e9d26f957e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,70 @@ +# TileDB v2.21.1 Release Notes + +## Defects removed + +* Do not load group metadata when getting it from REST. [#4821](https://github.com/TileDB-Inc/TileDB/pull/4821) + +# TileDB v2.21.0 Release Notes + +## Deprecation announcements + +* Building with Visual Studio 2019 (MSVC toolchains v14.29 and earlier) will become unsuported in version 2.28, to be released in Q3 2024. +* HDFS support will be removed in version 2.28, to be released in Q3 2024. +* Support for GLIBC 2.17 (manylinux2014, based on CentOS 7) will be removed in TileDB version 2.31, to be released in Q4 2024. It will be replaced by support for GLIBC 2.28 (manylinux_2_28; RHEL8; Rocky Linux 8). + +## New features + +* Add datatypes GEOM_WKB and GEOM_WKT. [#4640](https://github.com/TileDB-Inc/TileDB/pull/4640) + +## TileDB backend API support: + +* Query Plan REST support. [#4519](https://github.com/TileDB-Inc/TileDB/pull/4519) +* Consolidation Plan REST support. [#4537](https://github.com/TileDB-Inc/TileDB/pull/4537) + +## Improvements + +* Optimize sparse reads when full non empty domain is requested. [#4710](https://github.com/TileDB-Inc/TileDB/pull/4710) +* Lazily initialize the Azure SDK client. [#4703](https://github.com/TileDB-Inc/TileDB/pull/4703) +* Remove QueryCondition requirement for Enumeration values to be valid. [#4707](https://github.com/TileDB-Inc/TileDB/pull/4707) +* Convert dimension function pointer members to subclass methods. [#4774](https://github.com/TileDB-Inc/TileDB/pull/4774) +* Implement dense dimension aggregates. [#4801](https://github.com/TileDB-Inc/TileDB/pull/4801) + +## Defects removed + +* Fix documentation of `Subarray::set_config` to match `tiledb_subarray_set_config`. [#4672](https://github.com/TileDB-Inc/TileDB/pull/4672) +* Fix stats bug in StorageManager. [#4708](https://github.com/TileDB-Inc/TileDB/pull/4708) +* Add explicit casts to fix compile errors. [#4712](https://github.com/TileDB-Inc/TileDB/pull/4712) +* Fix bug with new minio behavior. [#4725](https://github.com/TileDB-Inc/TileDB/pull/4725) +* Support reading V1 group details with explicit version in the name. [#4744](https://github.com/TileDB-Inc/TileDB/pull/4744) +* Update crop_range to clamp to domain range. [#4781](https://github.com/TileDB-Inc/TileDB/pull/4781) +* Fix segfault in ArraySchema::check. [#4787](https://github.com/TileDB-Inc/TileDB/pull/4787) +* Fix crash in aggregates on dimensions. [#4794](https://github.com/TileDB-Inc/TileDB/pull/4794) +* Fix defects with query channels. [#4786](https://github.com/TileDB-Inc/TileDB/pull/4786) +* Remove trailing slash from `rest.server_address` configuration option. [#4790](https://github.com/TileDB-Inc/TileDB/pull/4790) +* Provide a clear exception message when performing remote incomplete queries with aggregates. [#4798](https://github.com/TileDB-Inc/TileDB/pull/4798) + +## API changes + +### C API + +* Support VFS `ls_recursive` API for posix filesystem. [#4778](https://github.com/TileDB-Inc/TileDB/pull/4778) + +## Test only changes + +* Add unit_vfs to tests for windows/osx/linux. [#4748](https://github.com/TileDB-Inc/TileDB/pull/4748) + +## Build System Changes + +* Update vcpkg version baseline to https://github.com/microsoft/vcpkg/commit/72010900b7cee36cea77aebb97695095c9358eaf. [#4553](https://github.com/TileDB-Inc/TileDB/pull/4553) +* Add vcpkg triplets for RelWithDebInfo. [#4669](https://github.com/TileDB-Inc/TileDB/pull/4669) +* Reintroduce the `TILEDB_STATIC` option under a deprecation warning. [#4732](https://github.com/TileDB-Inc/TileDB/pull/4732) +* Fix linker errors when building with MSVC. [#4759](https://github.com/TileDB-Inc/TileDB/pull/4759) +* CPack in Release Workflow. [#4694](https://github.com/TileDB-Inc/TileDB/pull/4694) +* Single release CMake file that contains all required links/hashes. [#4631](https://github.com/TileDB-Inc/TileDB/pull/4631) +* Improve build performance when GCS is enabled. [#4777](https://github.com/TileDB-Inc/TileDB/pull/4777) +* Fix compiling unit tests when HDFS is enabled. [#4795](https://github.com/TileDB-Inc/TileDB/pull/4795) +* Add CPack support to CMakeLists.txt. [#4645](https://github.com/TileDB-Inc/TileDB/pull/4645) + # TileDB v2.20.1 Release Notes ## Defects removed From 9588acfef0ffdb36f29d42c5ba64883298c3ce04 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:38:49 +0200 Subject: [PATCH 272/456] Bump dev version to 2.23.0. (#4849) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.23.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index 1535c996190d..aeda2704a5ed 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.22' +version = '2.23' # The full version, including alpha/beta/rc tags. -release = '2.22.0' +release = '2.23.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index 4b32ddb980b3..2d839be4e815 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 22 +#define TILEDB_VERSION_MINOR 23 #define TILEDB_VERSION_PATCH 0 From eefea1f2af7afb817b21c28d5c366069766307c5 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 3 Apr 2024 21:48:46 +0300 Subject: [PATCH 273/456] Improve diagnostics when an Azure endpoint is not configured. (#4845) [SC-43870](https://app.shortcut.com/tiledb-inc/story/43870/remove-azure-warning-when-not-using-azure-credentials) This PR changes the Azure VFS to fail with an explicit error if an operation on the Azure VFS was attempted, but an endpoint to Azure Blob Storage could not be determined based on the `vfs.azure.storage_account_name` and `vfs.azure.blob_endpoint` configuration options. The previous behavior was to emit a warning, which happened at the initialization of the VFS class if Azure is enabled, regardless of whether Azure is being actually used. That warning is no longer emitted. --- TYPE: IMPROVEMENT DESC: Improve diagnostics when an Azure endpoint is not configured. --- tiledb/sm/filesystem/azure.cc | 28 +++++++++++++++++++--------- tiledb/sm/filesystem/azure.h | 30 ++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index ea461f24b408..e2402b3e8ee3 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -79,7 +79,7 @@ std::string get_config_with_env_fallback( return result; } -std::string get_blob_endpoint( +std::optional get_blob_endpoint( const Config& config, const std::string& account_name) { std::string sas_token = get_config_with_env_fallback( config, "vfs.azure.storage_sas_token", "AZURE_STORAGE_SAS_TOKEN"); @@ -90,9 +90,7 @@ std::string get_blob_endpoint( if (!account_name.empty()) { result = "https://" + account_name + ".blob.core.windows.net"; } else { - LOG_WARN( - "Neither the 'vfs.azure.storage_account_name' nor the " - "'vfs.azure.blob_endpoint' options are specified."); + return std::nullopt; } } else if (!(utils::parse::starts_with(result, "http://") || utils::parse::starts_with(result, "https://"))) { @@ -124,7 +122,20 @@ static bool has_sas_token(const Config& config) { return !sas_token.empty(); } -AzureParameters::AzureParameters(const Config& config) +std::optional AzureParameters::create(const Config& config) { + auto account_name = get_config_with_env_fallback( + config, "vfs.azure.storage_account_name", "AZURE_STORAGE_ACCOUNT"); + auto blob_endpoint = get_blob_endpoint(config, account_name); + if (!blob_endpoint) { + return std::nullopt; + } + return AzureParameters{config, account_name, *blob_endpoint}; +} + +AzureParameters::AzureParameters( + const Config& config, + const std::string& account_name, + const std::string& blob_endpoint) : max_parallel_ops_( config.get("vfs.azure.max_parallel_ops", Config::must_find)) , block_list_block_size_(config.get( @@ -138,11 +149,10 @@ AzureParameters::AzureParameters(const Config& config) "vfs.azure.max_retry_delay_ms", Config::must_find))) , use_block_list_upload_(config.get( "vfs.azure.use_block_list_upload", Config::must_find)) - , account_name_(get_config_with_env_fallback( - config, "vfs.azure.storage_account_name", "AZURE_STORAGE_ACCOUNT")) + , account_name_(account_name) , account_key_(get_config_with_env_fallback( config, "vfs.azure.storage_account_key", "AZURE_STORAGE_KEY")) - , blob_endpoint_(get_blob_endpoint(config, account_name_)) + , blob_endpoint_(blob_endpoint) , ssl_cfg_(config) , has_sas_token_(has_sas_token(config)) { } @@ -153,7 +163,7 @@ Status Azure::init(const Config& config, ThreadPool* const thread_pool) { Status_AzureError("Can't initialize with null thread pool.")); } thread_pool_ = thread_pool; - azure_params_ = config; + azure_params_ = AzureParameters::create(config); return Status::Ok(); } diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index fdd059947b95..489ac581e4a2 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -70,7 +70,13 @@ namespace sm { struct AzureParameters { AzureParameters() = delete; - AzureParameters(const Config& config); + /** + * Creates an AzureParameters object. + * + * @return AzureParameters or nullopt if config does not have + * a storage account or blob endpoint set. + */ + static std::optional create(const Config& config); /** The maximum number of parallel requests. */ uint64_t max_parallel_ops_; @@ -107,6 +113,12 @@ struct AzureParameters { /** Whether the config specifies a SAS token. */ bool has_sas_token_; + + private: + AzureParameters( + const Config& config, + const std::string& account_name, + const std::string& blob_endpoint); }; class Azure { @@ -362,7 +374,21 @@ class Azure { * use of the BlobServiceClient. */ const ::Azure::Storage::Blobs::BlobServiceClient& client() const { - assert(azure_params_); + // This branch can be entered in two circumstances: + // 1. The init method has not been called yet. + // 2. The init method has been called, but the config (or environment + // variables) do not contain enough information to get the Azure + // endpoint. + // We don't distinguish between the two, because only the latter can + // happen under normal circumstances, and the former is a C.41 issue + // that will go away once the class is C.41 compliant. + if (!azure_params_) { + throw StatusException(Status_AzureError( + "Azure VFS is not configured. Please set the " + "'vfs.azure.storage_account_name' and/or " + "'vfs.azure.blob_endpoint' configuration options.")); + } + return client_singleton_.get(*azure_params_); } From 5997aff8fc1c6480ddc91341916aea0001944a27 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 3 Apr 2024 21:49:22 +0300 Subject: [PATCH 274/456] Always use vcpkg. (#4570) [SC-36911](https://app.shortcut.com/tiledb-inc/story/36911/remove-enable-vcpkg-options-from-cmake-command-line-and-bootstrap-scripts) This PR changes the severity of the message when we configure with `-DTILEDB_VCPKG=OFF` from a `DEPRECATION` to a `FATAL_ERROR`. This means that using the legacy `ExternalProject`-based mechanism for acquiring dependencies is no longer supported. The `ExternalProject` modules have become dead code and will be removed in a subsequent PR. --- TYPE: BUILD DESC: Vcpkg is always enabled; turning the `TILEDB_VCPKG` option off is no longer supported and fails. --- .github/workflows/nightly-test.yml | 4 ++-- bootstrap | 9 --------- bootstrap.ps1 | 20 +------------------- cmake/Options/BuildOptions.cmake | 2 +- 4 files changed, 4 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 323985d64508..d53b0d50fafe 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -47,13 +47,13 @@ jobs: EXPERIMENTAL: ${{ matrix.experimental || 'OFF' }} working-directory: ${{ matrix.working_directory || github.workspace }} run: | - cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }} + cmake -B build -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }} - name: Configure TileDB CMake (Windows) if: contains(matrix.os, 'windows') working-directory: ${{ matrix.working_directory || github.workspace }} run: | - cmake -B build -S $env:GITHUB_WORKSPACE ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} + cmake -B build -S $env:GITHUB_WORKSPACE -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} - name: Build TileDB working-directory: ${{ matrix.working_directory || github.workspace }} diff --git a/bootstrap b/bootstrap index 6060194bf987..a149b12b7e13 100755 --- a/bootstrap +++ b/bootstrap @@ -35,7 +35,6 @@ Configuration: --prefix=PREFIX install files in tree rooted at PREFIX ['"${default_prefix}"'] --vcpkg-base-triplet=TRIPLET vcpkg base triplet to use when building with sanitizers - --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] --linkage specify the linkage of tiledb. Defaults to shared. @@ -49,7 +48,6 @@ Configuration: --disable-stats disables internal TileDB statistics --disable-avx2 disables use of AVX2 instructions --disable-webp disables building of webp library - --disable-vcpkg disables use of vcpkg for downloading and building dependencies --enable-static-tiledb enables building TileDB as a static library (deprecated, use --linkage=static) --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) @@ -90,7 +88,6 @@ Example: # Parse arguments prefix_dirs="${default_prefix}" vcpkg_base_triplet="" -tiledb_vcpkg="ON" dependency_dir="${default_dependency}" sanitizer="" build_type="Release" @@ -122,8 +119,6 @@ while test $# != 0; do prefix_dirs="$dir";; --vcpkg-base-triplet=*) vcpkg_base_triplet=`arg "$1"` vcpkg_base_triplet="-DTILEDB_VCPKG_BASE_TRIPLET=${vcpkg_base_triplet}";; - --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." - tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` dependency_dir="$dir";; --linkage=*) linkage=`arg "$1"`;; @@ -135,7 +130,6 @@ while test $# != 0; do --disable-stats) tiledb_stats="OFF";; --disable-avx2) tiledb_disable_avx2="-DCOMPILER_SUPPORTS_AVX2=FALSE";; --disable-webp) tiledb_build_webp="OFF";; - --disable-vcpkg) tiledb_vcpkg="OFF";; --enable-assertions) tiledb_assertions="ON";; --enable-debug) build_type="Debug";; --enable-release-symbols) build_type="RelWithDebInfo";; @@ -167,8 +161,6 @@ done IFS=',' read -ra enables <<< "$enable_multiple" for en in "${enables[@]}"; do case "$en" in - vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." - tiledb_vcpkg="ON";; assertions) tiledb_assertions="ON";; debug) build_type="Debug";; release-symbols) build_type="RelWithDebInfo";; @@ -239,7 +231,6 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DCMAKE_CXX_COMPILER="${cxx_compiler}" \ -DCMAKE_PREFIX_PATH="${dependency_dir}" \ -DBUILD_SHARED_LIBS=${build_shared_libs} \ - -DTILEDB_VCPKG=${tiledb_vcpkg} \ -DTILEDB_ASSERTIONS=${tiledb_assertions} \ -DTILEDB_VERBOSE=${tiledb_verbose} \ -DTILEDB_HDFS=${tiledb_hdfs} \ diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 7b3a98f63ca5..03384495ec9d 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -54,10 +54,6 @@ Enable build with code coverage support. .PARAMETER EnableVerbose Enable verbose status messages. -.PARAMETER EnableVcpkg -Enables building dependencies with vcpkg. -Deprecated, this is now the default behavior. - .PARAMETER EnableAzure Enables building with the Azure storage backend. @@ -104,9 +100,6 @@ Disable building the TileDB tests. .Parameter DisableStats Disables internal TileDB statistics. -.PARAMETER DisableVcpkg -Disables building dependencies with vcpkg. - .PARAMETER BuildProcesses Number of parallel compile jobs. @@ -135,8 +128,6 @@ Param( [switch]$EnableReleaseSymbols, [switch]$EnableCoverage, [switch]$EnableVerbose, - [switch]$EnableVcpkg, - [switch]$DisableVcpkg, [switch]$EnableAzure, [switch]$EnableS3, [switch]$EnableGcs, @@ -210,15 +201,6 @@ else { $_RemoveDeprecations = "OFF" } -# Set vcpkg flag -$UseVcpkg = "ON" -if ($EnableVcpkg.IsPresent) { - Write-Warning "-EnableVcpkg is deprecated and will be removed in a future version. Vcpkg is now enabled by default. Use -DisableVcpkg to disable it." -} -elseif ($DisableVcpkg.IsPresent) { - $UseVcpkg = "OFF" -} - # Set TileDB Azure flag $UseAzure = "OFF" if ($EnableAzure.IsPresent) { @@ -348,7 +330,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake $ArchFlag -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake $ArchFlag -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$_RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 7baebff4d23c..4bc86d5f7330 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -49,7 +49,7 @@ if (DEFINED TILEDB_STATIC) endif() if (NOT TILEDB_VCPKG) - message(DEPRECATION "Disabling TILEDB_VCPKG is deprecated and will be removed in a future version.") + message(FATAL_ERROR "Disabling TILEDB_VCPKG is not supported.") endif() # enable assertions by default for debug builds From adafa1ec8f2754a77892d5f9ee7a4658edc95d44 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 8 Apr 2024 14:42:04 +0300 Subject: [PATCH 275/456] Do not attempt Azure shared key authentication if no account name is specified. (#4856) Partly addresses SC-44603. --- TYPE: IMPROVEMENT DESC: Do not attempt Azure shared key authentication if no account name is specified. --- tiledb/sm/filesystem/azure.cc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index e2402b3e8ee3..78280251897f 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -210,14 +210,22 @@ Azure::AzureClientSingleton::get(const AzureParameters& params) { // Construct the Azure SDK blob service client. // We pass a shared key if it was specified. if (!params.account_key_.empty()) { - client_ = - tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( - ::Azure::Storage::Blobs::BlobServiceClient, - params.blob_endpoint_, - make_shared<::Azure::Storage::StorageSharedKeyCredential>( - HERE(), params.account_name_, params.account_key_), - options)); - return *client_; + // If we don't have an account name, warn and try other authentication + // methods. + if (params.account_name_.empty()) { + LOG_WARN( + "Azure storage account name must be set when specifying account key. " + "Account key will be ignored."); + } else { + client_ = + tdb_unique_ptr<::Azure::Storage::Blobs::BlobServiceClient>(tdb_new( + ::Azure::Storage::Blobs::BlobServiceClient, + params.blob_endpoint_, + make_shared<::Azure::Storage::StorageSharedKeyCredential>( + HERE(), params.account_name_, params.account_key_), + options)); + return *client_; + } } // Otherwise, if we did not specify an SAS token From e0bc0dd38bf4907704d7c5a6066e0a59cec0d744 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 9 Apr 2024 10:54:25 +0300 Subject: [PATCH 276/456] Port more tests to REST CI (#4838) As per title, more tests in tiledb_unit are adapted to run on the REST CI runner. After this PR is merged we'll have 160 tests ported to REST-CI runner, of which 143 will be running and passing and 17 will be disabled until the issues found and logged here:https://app.shortcut.com/tiledb-inc/story/40489/issues-found-while-running-tiledb-unit-core-tests-against-rest-cloud-server are fixed. --- TYPE: NO_HISTORY DESC: Port more tests to REST CI --- test/src/cpp-integration-query-condition.cc | 81 +- test/src/test-capi-consolidation-plan.cc | 71 +- test/src/unit-backwards_compat.cc | 47 +- test/src/unit-capi-array.cc | 296 ++--- test/src/unit-capi-attributes.cc | 152 +-- test/src/unit-capi-consolidation.cc | 1076 ++++++++--------- test/src/unit-capi-dense_array.cc | 954 ++++----------- test/src/unit-capi-dense_neg.cc | 92 +- test/src/unit-capi-group.cc | 10 +- test/src/unit-capi-incomplete-2.cc | 269 +---- test/src/unit-capi-incomplete.cc | 301 +---- test/src/unit-capi-nullable.cc | 44 +- test/src/unit-capi-rest-dense_array.cc | 6 +- test/src/unit-capi-serialized_queries.cc | 341 +----- ...-capi-serialized_queries_using_subarray.cc | 241 +--- test/src/unit-capi-sparse_array.cc | 981 ++++++--------- test/src/unit-capi-sparse_heter.cc | 67 +- test/src/unit-capi-sparse_neg.cc | 47 +- test/src/unit-capi-sparse_neg_2.cc | 46 +- test/src/unit-capi-string.cc | 170 +-- test/src/unit-capi-string_dims.cc | 517 ++------ ...it-cppapi-consolidation-with-timestamps.cc | 265 +--- test/src/unit-cppapi-string-dims.cc | 123 +- test/src/unit-query-plan.cc | 55 +- test/src/unit-sparse-global-order-reader.cc | 80 +- test/support/src/vfs_helpers.cc | 9 + test/support/src/vfs_helpers.h | 34 +- 27 files changed, 1850 insertions(+), 4525 deletions(-) diff --git a/test/src/cpp-integration-query-condition.cc b/test/src/cpp-integration-query-condition.cc index d18e2e428727..4c794769f9bd 100644 --- a/test/src/cpp-integration-query-condition.cc +++ b/test/src/cpp-integration-query-condition.cc @@ -36,6 +36,7 @@ #include #include +#include #include #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/misc/utils.h" @@ -91,6 +92,7 @@ inline int index_from_row_col(int r, int c) { * @param set_dups Whether the array allows coordinate duplicates. * @param a_data_read Data buffer to store cell values on attribute a. * @param b_data_read Data buffer to store cell values on attribute b. + * @param array_uri URI of array to create. */ void create_array( Context& ctx, @@ -98,7 +100,8 @@ void create_array( bool set_dups, bool add_utf8_attr, std::vector& a_data_read, - std::vector& b_data_read) { + std::vector& b_data_read, + const std::string& array_uri = array_name) { Domain domain(ctx); domain.add_dimension(Dimension::create(ctx, "rows", {{1, num_rows}}, 4)) .add_dimension(Dimension::create(ctx, "cols", {{1, num_rows}}, 4)); @@ -123,7 +126,7 @@ void create_array( attr_c.set_nullable(true); schema.add_attribute(attr_c); } - Array::create(array_name, schema); + Array::create(array_uri, schema); // Write some initial data and close the array. std::vector row_dims; @@ -179,7 +182,7 @@ void create_array( } if (array_type == TILEDB_SPARSE) { - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_layout(TILEDB_UNORDERED) .set_data_buffer("rows", row_dims) @@ -197,7 +200,7 @@ void create_array( query_w.finalize(); array_w.close(); } else if (array_type == TILEDB_DENSE) { - Array array_w(ctx, array_name, TILEDB_WRITE); + Array array_w(ctx, array_uri, TILEDB_WRITE); Query query_w(ctx, array_w); query_w.set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_data) @@ -215,7 +218,7 @@ void create_array( } // Open and read the entire array to save data for future comparisons. - Array array1(ctx, array_name, TILEDB_READ); + Array array1(ctx, array_uri, TILEDB_READ); Query query1(ctx, array1); query1.set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_data_read) @@ -316,15 +319,12 @@ struct TestParams { TEST_CASE( "Testing read query with empty QC, with no range.", - "[query][query-condition][empty]") { + "[query][query-condition][empty][rest]") { // Initial setup. std::srand(static_cast(time(0))); - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + test::VFSTestSetup vfs_test_setup; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri(array_name)}; // Create an empty query condition QueryCondition qc(ctx); @@ -351,7 +351,8 @@ TEST_CASE( params.set_dups_, params.add_utf8_attr_, a_data_read, - b_data_read); + b_data_read, + array_uri); // Create the query, which reads over the entire array with query condition Config config; @@ -359,8 +360,10 @@ TEST_CASE( config.set("sm.query.sparse_global_order.reader", "legacy"); config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); } - Context ctx2 = Context(config); - Array array(ctx2, array_name, TILEDB_READ); + + auto vfs_test_setup2 = tiledb::test::VFSTestSetup(config.ptr().get(), false); + auto ctx2 = vfs_test_setup2.ctx(); + Array array(ctx2, array_uri, TILEDB_READ); Query query(ctx2, array); // Set a subarray for dense. @@ -378,15 +381,12 @@ TEST_CASE( TEST_CASE( "Testing read query with basic QC, with no range.", - "[query][query-condition]") { + "[query][query-condition][rest]") { // Initial setup. std::srand(static_cast(time(0))); - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + test::VFSTestSetup vfs_test_setup; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri(array_name)}; // Define query condition (b < 4.0). QueryCondition qc(ctx); @@ -415,7 +415,8 @@ TEST_CASE( params.set_dups_, params.add_utf8_attr_, a_data_read, - b_data_read); + b_data_read, + array_uri); // Create the query, which reads over the entire array with query condition // (b < 4.0). @@ -424,8 +425,9 @@ TEST_CASE( config.set("sm.query.sparse_global_order.reader", "legacy"); config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); } - Context ctx2 = Context(config); - Array array(ctx2, array_name, TILEDB_READ); + auto vfs_test_setup2 = tiledb::test::VFSTestSetup(config.ptr().get(), false); + auto ctx2 = vfs_test_setup2.ctx(); + Array array(ctx2, array_uri, TILEDB_READ); Query query(ctx2, array); // Set a subarray for dense. @@ -500,23 +502,16 @@ TEST_CASE( query.finalize(); array.close(); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } } TEST_CASE( "Testing read query with basic negated QC, with no range.", - "[query][query-condition][negation]") { + "[query][query-condition][negation][rest]") { // Initial setup. std::srand(static_cast(time(0))); - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + test::VFSTestSetup vfs_test_setup; + Context ctx{vfs_test_setup.ctx()}; + auto array_uri{vfs_test_setup.array_uri(array_name)}; // Define query condition (b < 4.0). QueryCondition qc(ctx); @@ -547,7 +542,8 @@ TEST_CASE( params.set_dups_, params.add_utf8_attr_, a_data_read, - b_data_read); + b_data_read, + array_uri); // Create the query, which reads over the entire array with query condition // (b < 4.0). @@ -556,8 +552,11 @@ TEST_CASE( config.set("sm.query.sparse_global_order.reader", "legacy"); config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); } - Context ctx2 = Context(config); - Array array(ctx2, array_name, TILEDB_READ); + + vfs_test_setup.update_config(config.ptr().get()); + Context ctx2 = vfs_test_setup.ctx(); + + Array array(ctx2, array_uri, TILEDB_READ); Query query(ctx2, array); // Set a subarray for dense. @@ -632,10 +631,6 @@ TEST_CASE( query.finalize(); array.close(); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } } TEST_CASE( diff --git a/test/src/test-capi-consolidation-plan.cc b/test/src/test-capi-consolidation-plan.cc index 11e6ea64e6bd..39c5316becbf 100644 --- a/test/src/test-capi-consolidation-plan.cc +++ b/test/src/test-capi-consolidation-plan.cc @@ -43,16 +43,9 @@ using namespace tiledb; using namespace tiledb::test; -#ifndef TILEDB_TESTS_ENABLE_REST -constexpr bool rest_tests = false; -#else -constexpr bool rest_tests = true; -#endif - struct ConsolidationPlanFx { // Constructors/destructors. ConsolidationPlanFx(); - ~ConsolidationPlanFx(); // Functions. void create_sparse_array(bool allows_dups = false, bool encrypt = false); @@ -67,38 +60,24 @@ struct ConsolidationPlanFx { bool is_array(const std::string& array_name); void check_last_error(std::string expected); - // TileDB context. - Context ctx_; - // Full URI initialized using fs_vec_ random temp directory. - std::string array_name_; + VFSTestSetup vfs_test_setup_; - // Vector of supported filsystems - tiledb_vfs_handle_t* vfs_c_{nullptr}; - tiledb_ctx_handle_t* ctx_c_{nullptr}; - const std::vector> fs_vec_; + // TileDB context + tiledb_ctx_t* ctx_c_; + std::string array_name_; + Context ctx_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; }; -ConsolidationPlanFx::ConsolidationPlanFx() - : fs_vec_(test::vfs_test_get_fs_vec()) { +ConsolidationPlanFx::ConsolidationPlanFx() { Config config; config.set("sm.consolidation.buffer_size", "1000"); - REQUIRE( - test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_, config.ptr().get()).ok()); - ctx_ = Context(ctx_c_); - std::string temp_dir = fs_vec_[0]->temp_dir(); - if constexpr (rest_tests) { - array_name_ = "tiledb://unit/"; - } - array_name_ += temp_dir + "test_consolidation_plan_array"; - test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir); -} - -ConsolidationPlanFx::~ConsolidationPlanFx() { - Array::delete_array(ctx_, array_name_); - REQUIRE(test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok()); + vfs_test_setup_.update_config(config.ptr().get()); + ctx_c_ = vfs_test_setup_.ctx_c; + ctx_ = vfs_test_setup_.ctx(); + array_name_ = vfs_test_setup_.array_uri("test_consolidation_plan_array"); } void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) { @@ -174,7 +153,7 @@ void ConsolidationPlanFx::write_sparse( void ConsolidationPlanFx::check_last_error(std::string expected) { const char* msg = "unset"; tiledb_error_t* err{nullptr}; - tiledb_ctx_get_last_error(ctx_.ptr().get(), &err); + tiledb_ctx_get_last_error(ctx_c_, &err); if (err != nullptr) { tiledb_error_message(err, &msg); } @@ -193,23 +172,20 @@ TEST_CASE_METHOD( tiledb_consolidation_plan_t* consolidation_plan{}; CHECK( - TILEDB_OK == tiledb_consolidation_plan_create_with_mbr( - ctx_.ptr().get(), - array.ptr().get(), - 1024 * 1024, - &consolidation_plan)); + TILEDB_OK == + tiledb_consolidation_plan_create_with_mbr( + ctx_c_, array.ptr().get(), 1024 * 1024, &consolidation_plan)); uint64_t num_nodes = 11; CHECK( TILEDB_OK == tiledb_consolidation_plan_get_num_nodes( - ctx_.ptr().get(), consolidation_plan, &num_nodes)); + ctx_c_, consolidation_plan, &num_nodes)); CHECK(num_nodes == 0); uint64_t num_fragments = 11; CHECK( - TILEDB_ERR == - tiledb_consolidation_plan_get_num_fragments( - ctx_.ptr().get(), consolidation_plan, 0, &num_fragments)); + TILEDB_ERR == tiledb_consolidation_plan_get_num_fragments( + ctx_c_, consolidation_plan, 0, &num_fragments)); CHECK(num_fragments == 11); check_last_error( "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); @@ -217,7 +193,7 @@ TEST_CASE_METHOD( const char* frag_uri = nullptr; CHECK( TILEDB_ERR == tiledb_consolidation_plan_get_fragment_uri( - ctx_.ptr().get(), consolidation_plan, 0, 0, &frag_uri)); + ctx_c_, consolidation_plan, 0, 0, &frag_uri)); CHECK(frag_uri == nullptr); check_last_error( "Error: ConsolidationPlan: Trying to access a node that doesn't exist."); @@ -236,16 +212,13 @@ TEST_CASE_METHOD( tiledb_consolidation_plan_t* consolidation_plan{}; CHECK( - TILEDB_OK == tiledb_consolidation_plan_create_with_mbr( - ctx_.ptr().get(), - array.ptr().get(), - 1024 * 1024, - &consolidation_plan)); + TILEDB_OK == + tiledb_consolidation_plan_create_with_mbr( + ctx_c_, array.ptr().get(), 1024 * 1024, &consolidation_plan)); // Check dump char* str = nullptr; - tiledb_consolidation_plan_dump_json_str( - ctx_.ptr().get(), consolidation_plan, &str); + tiledb_consolidation_plan_dump_json_str(ctx_c_, consolidation_plan, &str); std::string plan(str); CHECK(plan == "{\n \"nodes\": [\n ]\n}\n"); diff --git a/test/src/unit-backwards_compat.cc b/test/src/unit-backwards_compat.cc index 425e7afdf7f4..09778fd6f62a 100644 --- a/test/src/unit-backwards_compat.cc +++ b/test/src/unit-backwards_compat.cc @@ -1247,17 +1247,6 @@ TEST_CASE( "Backwards compatibility: Upgrades an array of older version and " "write/read it", "[backwards-compat][upgrade-version][write-read-new-version]") { - bool serialize = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialize = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - std::string array_name(arrays_dir + "/non_split_coords_v1_4_0"); Context ctx; std::string schema_folder; @@ -1281,13 +1270,7 @@ TEST_CASE( .set_data_buffer("d2", d2_read1); ServerQueryBuffers server_buffers; - submit_query_wrapper( - ctx, - array_name, - &query_read1, - server_buffers, - serialize, - refactored_query_v2); + query_read1.submit(); array_read1.close(); for (int i = 0; i < 4; i++) { @@ -1308,30 +1291,12 @@ TEST_CASE( query_write.set_data_buffer("d1", d1_write); query_write.set_data_buffer("d2", d2_write); - submit_query_wrapper( - ctx, - array_name, - &query_write, - server_buffers, - serialize, - refactored_query_v2); + query_write.submit_and_finalize(); array_write.close(); FragmentInfo fragment_info(ctx, array_name); fragment_info.load(); - - if (serialize) { - FragmentInfo deserialized_fragment_info(ctx, array_name); - tiledb_fragment_info_serialize( - ctx.ptr().get(), - array_name.c_str(), - fragment_info.ptr().get(), - deserialized_fragment_info.ptr().get(), - tiledb_serialization_type_t(0)); - fragment_info = deserialized_fragment_info; - } - fragment_uri = fragment_info.fragment_uri(1); // old version fragment @@ -1353,13 +1318,7 @@ TEST_CASE( .set_data_buffer("d1", d1_read2) .set_data_buffer("d2", d2_read2); - submit_query_wrapper( - ctx, - array_name, - &query_read2, - server_buffers, - serialize, - refactored_query_v2); + query_read2.submit(); array_read2.close(); for (int i = 0; i < 2; i++) { diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 123212deb8d6..3f8633119bbd 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -93,12 +93,6 @@ struct ArrayFx { tiledb_encryption_type_t encryption_type_ = TILEDB_NO_ENCRYPTION; const char* encryption_key_ = nullptr; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; - // Functions ArrayFx(); ~ArrayFx(); @@ -391,9 +385,7 @@ void ArrayFx::write_fragment(tiledb_array_t* array, uint64_t timestamp) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", buffer, &buffer_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx_, query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx_, query); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Clean up @@ -470,7 +462,9 @@ TEST_CASE_METHOD( } TEST_CASE_METHOD( - ArrayFx, "C API: Test array with encryption", "[capi][array][encryption]") { + ArrayFx, + "C API: Test array with encryption", + "[capi][array][encryption][non-rest]") { // Create array schema tiledb_array_schema_t* array_schema; int rc = tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &array_schema); @@ -883,28 +877,22 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ArrayFx, "C API: Test opening array at timestamp, reads", - "[capi][array][open-at][reads]") { + "[capi][array][open-at][reads][rest]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); + std::string array_path = temp_dir + "array-open-at-reads"; + std::string array_name = vfs_array_uri(fs_vec_[0], array_path); - std::string array_name = temp_dir + "array-open-at-reads"; SECTION("- without encryption") { encryption_type_ = TILEDB_NO_ENCRYPTION; encryption_key_ = nullptr; - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif } - SECTION("- with encryption") { - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + if (!fs_vec_[0]->is_rest()) { + SECTION("- with encryption") { + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + } } create_temp_dir(temp_dir); @@ -951,13 +939,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1006,13 +988,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_upd, &buffer_upd_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1025,7 +1001,7 @@ TEST_CASE_METHOD( rc = tiledb_vfs_ls( ctx_, vfs_, - get_commit_dir(array_name).c_str(), + get_commit_dir(array_path).c_str(), &get_fragment_timestamps, &fragment_timestamps); CHECK(rc == TILEDB_OK); @@ -1070,13 +1046,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1134,13 +1104,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1191,13 +1155,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1254,13 +1212,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Clean up but don't close the array yet (we will reopen it). @@ -1291,13 +1243,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Clean up but don't close the array yet (we will reopen it). @@ -1331,13 +1277,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1401,13 +1341,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1471,13 +1405,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1511,28 +1439,22 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ArrayFx, "C API: Test opening array at timestamp, writes", - "[capi][array][open-at][writes]") { + "[capi][array][open-at][writes][rest-fails][sc-42722]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); + std::string array_name = + vfs_array_uri(fs_vec_[0], temp_dir + "array-open-at-writes"); - std::string array_name = temp_dir + "array-open-at-writes"; SECTION("- without encryption") { encryption_type_ = TILEDB_NO_ENCRYPTION; encryption_key_ = nullptr; - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif } - SECTION("- with encryption") { - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + if (!fs_vec_[0]->is_rest()) { + SECTION("- with encryption") { + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + } } create_temp_dir(temp_dir); @@ -1584,13 +1506,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Get written timestamp @@ -1655,13 +1571,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1712,13 +1622,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -1739,10 +1643,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ArrayFx, "C API: Check writing coordinates out of bounds", - "[capi][array][array-write-coords-oob]") { - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "array-write-coords-oob"; + "[capi][array][array-write-coords-oob][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + std::string array_name = + vfs_array_uri(fs_vec_[0], temp_dir + "array-write-coords-oob"); create_temp_dir(temp_dir); int dimension = 0; @@ -1756,7 +1660,6 @@ TEST_CASE_METHOD( // Create TileDB context tiledb_config_t* cfg = nullptr; tiledb_error_t* err = nullptr; - tiledb_ctx_t* ctx = nullptr; SECTION("- Check out-of-bounds coordinates") { check_coords_oob = true; @@ -1842,59 +1745,58 @@ TEST_CASE_METHOD( } } - REQUIRE(tiledb_ctx_alloc(cfg, &ctx) == TILEDB_OK); - REQUIRE(err == nullptr); + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, cfg).ok(); tiledb_config_free(&cfg); // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Submit query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = - tiledb_query_set_data_buffer(ctx, query, "a", buffer_a1, &buffer_a1_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "a", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, "d1", buffer_coords_dim1, &buffer_coords_size); + ctx_, query, "d1", buffer_coords_dim1, &buffer_coords_size); CHECK(rc == TILEDB_OK); if (dimension == 2) { rc = tiledb_query_set_data_buffer( - ctx, query, "d2", buffer_coords_dim2, &buffer_coords_size); + ctx_, query, "d2", buffer_coords_dim2, &buffer_coords_size); CHECK(rc == TILEDB_OK); } - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit_and_finalize(ctx_, query); if (check_coords_oob) { CHECK(rc == TILEDB_ERR); } else { CHECK(rc == TILEDB_OK); } - rc = tiledb_query_finalize(ctx, query); - CHECK(rc == TILEDB_OK); // Close array and clean up - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( - ArrayFx, "C API: Test empty array", "[capi][array][array-empty]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "array_empty"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + ArrayFx, "C API: Test empty array", "[capi][array][array-empty][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + std::string array_name = vfs_array_uri(fs_vec_[0], temp_dir + "array_empty"); + + create_temp_dir(temp_dir); create_sparse_vector(array_name); @@ -1937,15 +1839,17 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( - ArrayFx, "C API: Test deletion of array", "[capi][array][delete]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "array_delete"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + ArrayFx, "C API: Test deletion of array", "[capi][array][delete][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + std::string array_path = temp_dir + "array_delete"; + std::string array_name = vfs_array_uri(fs_vec_[0], array_path); + + create_temp_dir(temp_dir); + create_dense_vector(array_name); // Conditionally consolidate @@ -1961,11 +1865,17 @@ TEST_CASE_METHOD( write_fragment(array, 5); // Check write - CHECK(tiledb::test::num_commits(array_name) == 3); - CHECK(tiledb::test::num_fragments(array_name) == 3); + // Get the number of write files in the commit directory + tiledb::Context ctx(ctx_, false); + tiledb::VFS vfs(ctx); + CommitsDirectory commits_dir(vfs, array_path); + CHECK(commits_dir.file_count(constants::write_file_suffix) == 3); + auto uris = vfs.ls( + array_path + "/" + tiledb::sm::constants::array_fragments_dir_name); + CHECK(static_cast(uris.size()) == 3); // Conditionally consolidate commits - if (consolidate) { + if (consolidate && !fs_vec_[0]->is_rest()) { tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -1979,16 +1889,22 @@ TEST_CASE_METHOD( tiledb_config_free(&cfg); // Validate working directory - CHECK(tiledb::test::num_commits(array_name) == 3); - CHECK(tiledb::test::num_fragments(array_name) == 3); + CommitsDirectory commits_dir2(vfs, array_path); + CHECK(commits_dir2.file_count(constants::write_file_suffix) == 3); + auto uris2 = vfs.ls( + array_path + "/" + tiledb::sm::constants::array_fragments_dir_name); + CHECK(static_cast(uris2.size()) == 3); } // Delete array data rc = tiledb_array_delete(ctx_, array_name.c_str()); // Validate working directory after delete - CHECK(tiledb::test::num_commits(array_name) == 0); - CHECK(tiledb::test::num_fragments(array_name) == 0); + CommitsDirectory commits_dir3(vfs, array_path); + CHECK(commits_dir3.file_count(constants::write_file_suffix) == 0); + auto uris3 = vfs.ls( + array_path + "/" + tiledb::sm::constants::array_fragments_dir_name); + CHECK(static_cast(uris3.size()) == 0); // Try to open array rc = tiledb_array_open(ctx_, array, TILEDB_READ); @@ -1996,18 +1912,18 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( ArrayFx, "C API: Test query errors, getting subarray info from write queries in " "sparse arrays", - "[capi][query][error][sparse]") { - SupportedFsLocal local_fs; + "[capi][query][error][sparse][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "query_error_sparse"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + vfs_array_uri(fs_vec_[0], temp_dir + "query_error_sparse"); + create_temp_dir(temp_dir); create_sparse_vector(array_name); @@ -2053,17 +1969,17 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( ArrayFx, "C API: Test query errors, dense writes", - "[capi][query][error][dense]") { - SupportedFsLocal local_fs; + "[capi][query][error][dense][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "query_error_dense"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + vfs_array_uri(fs_vec_[0], temp_dir + "query_error_dense"); + create_temp_dir(temp_dir); create_dense_array(array_name); @@ -2127,17 +2043,18 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( ArrayFx, "C API: Test query errors, dense unordered writes", - "[capi][query][error][dense]") { - SupportedFsLocal local_fs; + "[capi][query][error][dense][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "query_error_dense"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + vfs_array_uri(fs_vec_[0], temp_dir + "query_error_dense"); + + create_temp_dir(temp_dir); create_dense_array(array_name); @@ -2164,17 +2081,18 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( ArrayFx, "C API: Test query errors, dense reads in global order", - "[capi][query][error][dense]") { - SupportedFsLocal local_fs; + "[capi][query][error][dense][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "query_error_dense"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + vfs_array_uri(fs_vec_[0], temp_dir + "query_error_dense"); + + create_temp_dir(temp_dir); create_dense_array(array_name); @@ -2216,7 +2134,7 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_query_free(&query); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( diff --git a/test/src/unit-capi-attributes.cc b/test/src/unit-capi-attributes.cc index 17e61fc3ec2a..2c350bc03858 100644 --- a/test/src/unit-capi-attributes.cc +++ b/test/src/unit-capi-attributes.cc @@ -62,12 +62,6 @@ struct Attributesfx { // Vector of supported filesystems const std::vector> fs_vec_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; - // Functions Attributesfx(); ~Attributesfx(); @@ -156,7 +150,7 @@ void Attributesfx::create_dense_vector( TEST_CASE_METHOD( Attributesfx, "C API: Test attributes with illegal filesystem characters in the name", - "[capi][attributes][illegal_name]") { + "[capi][attributes][illegal_name][rest]") { const std::vector attr_names = { "miles!hour", "miles#hour", "miles$hour", "miles%hour", "miles&hour", "miles'hour", "miles(hour", "miles)hour", "miles*hour", "miles+hour", @@ -164,25 +158,12 @@ TEST_CASE_METHOD( "miles?hour", "miles@hour", "miles[hour", "miles]hour", "miles[hour", "miles\"hour", "mileshour", "miles\\hour", "miles|hour"}; - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + size_t num = 0; for (const auto& attr_name : attr_names) { for (const auto& fs : fs_vec_) { std::string temp_dir = fs->temp_dir(); - std::string array_name = temp_dir + "array-illegal-char"; - // serialization is not supported for memfs arrays - if (serialize_ && - tiledb::sm::utils::parse::starts_with(array_name, "mem://")) { - continue; - } + std::string array_name = vfs_array_uri( + fs, temp_dir + "array-illegal-char" + std::to_string(num++)); // Create new TileDB context with file lock config disabled, rest the // same. @@ -225,13 +206,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); - rc = tiledb::test::submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -259,13 +234,8 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); + CHECK(rc == TILEDB_OK); // Close array and clean up @@ -280,6 +250,7 @@ TEST_CASE_METHOD( CHECK(buffer_read_size == sizeof(buffer_read_c)); remove_temp_dir(temp_dir); + CHECK(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); } } } @@ -287,27 +258,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( Attributesfx, "C API: Test attributes with std::byte", - "[capi][attributes][byte]") { + "[capi][attributes][byte][rest-fails][sc-40489]") { auto datatype = GENERATE(TILEDB_BLOB, TILEDB_GEOM_WKB, TILEDB_GEOM_WKT); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif for (const auto& fs : fs_vec_) { std::string temp_dir = fs->temp_dir(); - std::string array_name = temp_dir; + std::string array_name = vfs_array_uri(fs, temp_dir + "byte-attribute"); std::string attr_name = "a"; - // serialization is not supported for memfs arrays - if (serialize_ && - tiledb::sm::utils::parse::starts_with(array_name, "mem://")) { - continue; - } // Create new TileDB context with file lock config disabled, rest the // same. @@ -346,13 +303,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_write, &buffer_write_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -407,25 +358,25 @@ TEST_CASE_METHOD( ts_open = ts_open + 2; rc = tiledb_array_schema_evolution_set_timestamp_range( ctx_, schema_evolution2, ts_open, ts_open); - if (serialize_) { - // Serialize the array schema evolution - tiledb_buffer_t* buffer; - rc = tiledb_serialize_array_schema_evolution( - ctx_, - schema_evolution2, - (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, - 0, - &buffer); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_deserialize_array_schema_evolution( - ctx_, - buffer, - (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, - 1, - &schema_evolution2); - REQUIRE(rc == TILEDB_OK); - tiledb_buffer_free(&buffer); - } +#ifdef TILEDB_SERIALIZATION + // Serialize the array schema evolution + tiledb_buffer_t* buffer; + rc = tiledb_serialize_array_schema_evolution( + ctx_, + schema_evolution2, + (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, + 0, + &buffer); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_deserialize_array_schema_evolution( + ctx_, + buffer, + (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, + 1, + &schema_evolution2); + REQUIRE(rc == TILEDB_OK); + tiledb_buffer_free(&buffer); +#endif rc = tiledb_array_evolve(ctx_, array_name.c_str(), schema_evolution2); REQUIRE(rc == TILEDB_OK); @@ -460,14 +411,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -492,24 +436,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( Attributesfx, "C API: Test attributes with tiledb_bool datatype", - "[capi][attributes][tiledb_bool]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif + "[capi][attributes][tiledb_bool][rest]") { for (const auto& fs : fs_vec_) { std::string temp_dir = fs->temp_dir(); - std::string array_name = temp_dir; - // serialization is not supported for memfs arrays - if (serialize_ && - tiledb::sm::utils::parse::starts_with(array_name, "mem://")) { - continue; - } + std::string array_name = vfs_array_uri(fs, temp_dir + "bool-attribute"); std::string attr_name = "attr"; @@ -554,13 +484,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_write, &buffer_write_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up @@ -588,13 +512,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array and clean up diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index 19bbe334bc87..1f7527120ec4 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/encryption_type.h" @@ -46,46 +47,32 @@ using namespace tiledb::test; /** Tests for C API consolidation. */ struct ConsolidationFx { - // Constants - const char* DENSE_VECTOR_NAME = "test_consolidate_dense_vector"; - const char* DENSE_VECTOR_FRAG_DIR = - "test_consolidate_dense_vector/__fragments"; - const char* DENSE_VECTOR_FRAG_META_DIR = - "test_consolidate_dense_vector/__fragment_meta"; - const char* DENSE_ARRAY_NAME = "test_consolidate_dense_array"; - const char* DENSE_ARRAY_COMMITS_DIR = - "test_consolidate_dense_array/__commits"; - const char* DENSE_ARRAY_FRAG_DIR = "test_consolidate_dense_array/__fragments"; - const char* SPARSE_ARRAY_NAME = "test_consolidate_sparse_array"; - const char* SPARSE_ARRAY_COMMITS_DIR = - "test_consolidate_sparse_array/__commits"; - const char* SPARSE_ARRAY_FRAG_DIR = - "test_consolidate_sparse_array/__fragments"; - const char* SPARSE_HETEROGENEOUS_ARRAY_NAME = - "test_consolidate_sparse_heterogeneous_array"; - const char* SPARSE_STRING_ARRAY_NAME = "test_consolidate_sparse_string_array"; - const char* SPARSE_STRING_ARRAY_FRAG_DIR = - "test_consolidate_sparse_string_array/__fragments"; - const char* SPARSE_STRING_ARRAY_FRAG_META_DIR = - "test_consolidate_sparse_string_array/__fragment_meta"; + VFSTestSetup vfs_test_setup_; // TileDB context tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; + std::string dense_vector_uri_; + std::string dense_vector_frag_dir_; + std::string dense_vector_frag_meta_dir_; + std::string dense_array_uri_; + std::string dense_array_commits_dir_; + std::string dense_array_frag_dir_; + std::string sparse_array_uri_; + std::string sparse_array_commits_dir_; + std::string sparse_array_frag_dir_; + std::string sparse_heterogeneous_array_uri_; + std::string sparse_string_array_uri_; + std::string sparse_string_array_fragment_dir_; + std::string sparse_string_array_frag_meta_dir_; + // Encryption parameters tiledb_encryption_type_t encryption_type_ = TILEDB_NO_ENCRYPTION; const char* encryption_key_ = nullptr; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; - // Constructors/destructors ConsolidationFx(); - ~ConsolidationFx(); // Functions void create_dense_vector(); @@ -182,16 +169,35 @@ struct ConsolidationFx { static int get_vac_files_callback(const char* path, void* data); }; -ConsolidationFx::ConsolidationFx() { - ctx_ = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx_) == TILEDB_OK); - vfs_ = nullptr; - REQUIRE(tiledb_vfs_alloc(ctx_, nullptr, &vfs_) == TILEDB_OK); -} - -ConsolidationFx::~ConsolidationFx() { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); +ConsolidationFx::ConsolidationFx() + : ctx_(vfs_test_setup_.ctx_c) + , vfs_(vfs_test_setup_.vfs_c) + , dense_vector_uri_( + vfs_test_setup_.array_uri("test_consolidate_dense_vector")) + , dense_vector_frag_dir_(vfs_test_setup_.array_uri( + "test_consolidate_dense_vector/__fragments", true)) + , dense_vector_frag_meta_dir_(vfs_test_setup_.array_uri( + "test_consolidate_dense_vector/__fragment_meta", true)) + , dense_array_uri_( + vfs_test_setup_.array_uri("test_consolidate_dense_array")) + , dense_array_commits_dir_(vfs_test_setup_.array_uri( + "test_consolidate_dense_array/__commits", true)) + , dense_array_frag_dir_(vfs_test_setup_.array_uri( + "test_consolidate_dense_array/__fragments", true)) + , sparse_array_uri_( + vfs_test_setup_.array_uri("test_consolidate_sparse_array")) + , sparse_array_commits_dir_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_array/__commits", true)) + , sparse_array_frag_dir_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_array/__fragments", true)) + , sparse_heterogeneous_array_uri_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_heterogeneous_array")) + , sparse_string_array_uri_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_string_array", true)) + , sparse_string_array_fragment_dir_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_string_array/__fragments", true)) + , sparse_string_array_frag_meta_dir_(vfs_test_setup_.array_uri( + "test_consolidate_sparse_string_array/__fragment_meta", true)) { } void ConsolidationFx::create_dense_vector() { @@ -234,8 +240,6 @@ void ConsolidationFx::create_dense_vector() { // Create array if (encryption_type_ != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -249,11 +253,13 @@ void ConsolidationFx::create_dense_vector() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - REQUIRE(tiledb_ctx_alloc(cfg, &ctx_) == TILEDB_OK); - REQUIRE(tiledb_vfs_alloc(ctx_, cfg, &vfs_) == TILEDB_OK); + // Create new context + vfs_test_setup_.update_config(cfg); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; tiledb_config_free(&cfg); } - rc = tiledb_array_create(ctx_, DENSE_VECTOR_NAME, array_schema); + rc = tiledb_array_create(ctx_, dense_vector_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -335,8 +341,6 @@ void ConsolidationFx::create_dense_array() { // Create array if (encryption_type_ != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -350,11 +354,13 @@ void ConsolidationFx::create_dense_array() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - REQUIRE(tiledb_ctx_alloc(cfg, &ctx_) == TILEDB_OK); - REQUIRE(tiledb_vfs_alloc(ctx_, cfg, &vfs_) == TILEDB_OK); + // Create new context + vfs_test_setup_.update_config(cfg); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; tiledb_config_free(&cfg); } - rc = tiledb_array_create(ctx_, DENSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, dense_array_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -437,8 +443,6 @@ void ConsolidationFx::create_sparse_array() { // Create array if (encryption_type_ != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -452,11 +456,13 @@ void ConsolidationFx::create_sparse_array() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - REQUIRE(tiledb_ctx_alloc(cfg, &ctx_) == TILEDB_OK); - REQUIRE(tiledb_vfs_alloc(ctx_, cfg, &vfs_) == TILEDB_OK); + // Create new context + vfs_test_setup_.update_config(cfg); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; tiledb_config_free(&cfg); } - rc = tiledb_array_create(ctx_, SPARSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, sparse_array_uri_.c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Clean up @@ -541,8 +547,6 @@ void ConsolidationFx::create_sparse_heterogeneous_array() { // Create array if (encryption_type_ != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -556,11 +560,14 @@ void ConsolidationFx::create_sparse_heterogeneous_array() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - REQUIRE(tiledb_ctx_alloc(cfg, &ctx_) == TILEDB_OK); - REQUIRE(tiledb_vfs_alloc(ctx_, cfg, &vfs_) == TILEDB_OK); + // Create new context + vfs_test_setup_.update_config(cfg); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; tiledb_config_free(&cfg); } - rc = tiledb_array_create(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, array_schema); + rc = tiledb_array_create( + ctx_, sparse_heterogeneous_array_uri_.c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Clean up @@ -643,8 +650,6 @@ void ConsolidationFx::create_sparse_string_array() { // Create array if (encryption_type_ != TILEDB_NO_ENCRYPTION) { - tiledb_ctx_free(&ctx_); - tiledb_vfs_free(&vfs_); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; rc = tiledb_config_alloc(&cfg, &err); @@ -658,11 +663,14 @@ void ConsolidationFx::create_sparse_string_array() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - REQUIRE(tiledb_ctx_alloc(cfg, &ctx_) == TILEDB_OK); - REQUIRE(tiledb_vfs_alloc(ctx_, cfg, &vfs_) == TILEDB_OK); + // Do not remove the array when recreating context to set the new config + vfs_test_setup_.update_config(cfg); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; tiledb_config_free(&cfg); } - rc = tiledb_array_create(ctx_, SPARSE_STRING_ARRAY_NAME, array_schema); + rc = + tiledb_array_create(ctx_, sparse_string_array_uri_.c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Clean up @@ -695,7 +703,7 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { uint64_t a_4_size = sizeof(a_4); tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; @@ -892,7 +900,7 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); @@ -990,7 +998,7 @@ void ConsolidationFx:: // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); @@ -1079,7 +1087,7 @@ void ConsolidationFx::write_dense_vector_consolidatable_1() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1169,7 +1177,7 @@ void ConsolidationFx::write_dense_vector_consolidatable_2() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1247,7 +1255,7 @@ void ConsolidationFx::write_dense_vector_del_1() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1355,7 +1363,7 @@ void ConsolidationFx::write_dense_vector_del_2() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1463,7 +1471,7 @@ void ConsolidationFx::write_dense_vector_del_3() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1557,7 +1565,7 @@ void ConsolidationFx::write_dense_vector_del_3() { void ConsolidationFx::write_dense_array_metadata() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); REQUIRE(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1633,7 +1641,7 @@ void ConsolidationFx::write_dense_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1680,13 +1688,7 @@ void ConsolidationFx::write_dense_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1718,7 +1720,7 @@ void ConsolidationFx::write_dense_subarray( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1769,13 +1771,7 @@ void ConsolidationFx::write_dense_subarray( CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1828,7 +1824,7 @@ void ConsolidationFx::write_sparse_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1879,13 +1875,7 @@ void ConsolidationFx::write_sparse_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1923,7 +1913,7 @@ void ConsolidationFx::write_sparse_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -1974,13 +1964,7 @@ void ConsolidationFx::write_sparse_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2022,7 +2006,7 @@ void ConsolidationFx::write_sparse_row(uint64_t row_idx) { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2131,7 +2115,8 @@ void ConsolidationFx::write_sparse_heterogeneous_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, &array); + int rc = + tiledb_array_alloc(ctx_, sparse_heterogeneous_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2182,13 +2167,7 @@ void ConsolidationFx::write_sparse_heterogeneous_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_HETEROGENEOUS_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2227,7 +2206,8 @@ void ConsolidationFx::write_sparse_heterogeneous_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, &array); + int rc = + tiledb_array_alloc(ctx_, sparse_heterogeneous_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2278,13 +2258,7 @@ void ConsolidationFx::write_sparse_heterogeneous_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_HETEROGENEOUS_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2342,7 +2316,7 @@ void ConsolidationFx::write_sparse_string_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_STRING_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_string_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2396,13 +2370,7 @@ void ConsolidationFx::write_sparse_string_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_STRING_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2444,7 +2412,7 @@ void ConsolidationFx::write_sparse_string_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_STRING_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_string_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2498,13 +2466,7 @@ void ConsolidationFx::write_sparse_string_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_STRING_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Close array rc = tiledb_array_close(ctx_, array); @@ -2518,7 +2480,7 @@ void ConsolidationFx::write_sparse_string_unordered() { void ConsolidationFx::read_dense_array_metadata() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); REQUIRE(rc == TILEDB_OK); tiledb_config_t* cfg; tiledb_error_t* err = nullptr; @@ -2589,7 +2551,7 @@ void ConsolidationFx::read_dense_vector(uint64_t timestamp) { // Set array configuration tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_set_open_timestamp_end(ctx_, array, timestamp); CHECK(rc == TILEDB_OK); @@ -2665,7 +2627,7 @@ void ConsolidationFx::read_dense_vector_with_gaps() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2734,7 +2696,7 @@ void ConsolidationFx::read_dense_vector_consolidatable_1() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2806,7 +2768,7 @@ void ConsolidationFx::read_dense_vector_consolidatable_2() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2874,7 +2836,7 @@ void ConsolidationFx::read_dense_vector_del_1() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -2942,7 +2904,7 @@ void ConsolidationFx::read_dense_vector_del_2() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3015,7 +2977,7 @@ void ConsolidationFx::read_dense_vector_del_3() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3097,7 +3059,7 @@ void ConsolidationFx::read_dense_full_subarray() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3161,14 +3123,7 @@ void ConsolidationFx::read_dense_full_subarray() { CHECK(rc == TILEDB_OK); // Submit query - // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; @@ -3227,7 +3182,7 @@ void ConsolidationFx::read_dense_subarray_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3291,13 +3246,7 @@ void ConsolidationFx::read_dense_subarray_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Finalize query @@ -3363,7 +3312,7 @@ void ConsolidationFx::read_dense_four_tiles() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3480,7 +3429,7 @@ void ConsolidationFx::read_sparse_full_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3546,13 +3495,7 @@ void ConsolidationFx::read_sparse_full_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -3593,7 +3536,7 @@ void ConsolidationFx::read_sparse_unordered_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3659,13 +3602,7 @@ void ConsolidationFx::read_sparse_unordered_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -3712,7 +3649,7 @@ void ConsolidationFx::read_sparse_rows() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3823,7 +3760,8 @@ void ConsolidationFx::read_sparse_heterogeneous_full_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, &array); + int rc = + tiledb_array_alloc(ctx_, sparse_heterogeneous_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -3889,13 +3827,7 @@ void ConsolidationFx::read_sparse_heterogeneous_full_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_HETEROGENEOUS_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -3936,7 +3868,8 @@ void ConsolidationFx::read_sparse_heterogeneous_unordered_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, &array); + int rc = + tiledb_array_alloc(ctx_, sparse_heterogeneous_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -4002,13 +3935,7 @@ void ConsolidationFx::read_sparse_heterogeneous_unordered_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_HETEROGENEOUS_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -4051,7 +3978,7 @@ void ConsolidationFx::read_sparse_string_full_unordered() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_STRING_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_string_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -4122,13 +4049,7 @@ void ConsolidationFx::read_sparse_string_full_unordered() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_STRING_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -4176,7 +4097,7 @@ void ConsolidationFx::read_sparse_string_unordered_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_STRING_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_string_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); if (encryption_type_ != TILEDB_NO_ENCRYPTION) { tiledb_config_t* cfg; @@ -4247,13 +4168,7 @@ void ConsolidationFx::read_sparse_string_unordered_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_STRING_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -4291,7 +4206,8 @@ uint32_t ConsolidationFx::get_num_fragments_to_vacuum_dense() { // Create fragment info object tiledb_fragment_info_t* fragment_info = nullptr; - int rc = tiledb_fragment_info_alloc(ctx_, DENSE_ARRAY_NAME, &fragment_info); + int rc = tiledb_fragment_info_alloc( + ctx_, dense_array_uri_.c_str(), &fragment_info); CHECK(rc == TILEDB_OK); // Load fragment info @@ -4344,7 +4260,7 @@ void ConsolidationFx::consolidate_dense( REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); } - rc = tiledb_array_consolidate(ctx_, DENSE_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, dense_array_uri_.c_str(), cfg); REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } @@ -4385,7 +4301,7 @@ void ConsolidationFx::consolidate_sparse( REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); } - rc = tiledb_array_consolidate(ctx_, SPARSE_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, sparse_array_uri_.c_str(), cfg); REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } @@ -4412,10 +4328,12 @@ void ConsolidationFx::consolidate_sparse_heterogeneous() { rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_array_consolidate(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate( + ctx_, sparse_heterogeneous_array_uri_.c_str(), cfg); tiledb_config_free(&cfg); } else { - rc = tiledb_array_consolidate(ctx_, SPARSE_HETEROGENEOUS_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate( + ctx_, sparse_heterogeneous_array_uri_.c_str(), cfg); } tiledb_config_free(&cfg); @@ -4448,10 +4366,10 @@ void ConsolidationFx::consolidate_sparse_string( rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_array_consolidate(ctx_, SPARSE_STRING_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, sparse_string_array_uri_.c_str(), cfg); tiledb_config_free(&cfg); } else { - rc = tiledb_array_consolidate(ctx_, SPARSE_STRING_ARRAY_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, sparse_string_array_uri_.c_str(), cfg); } tiledb_config_free(&cfg); @@ -4467,7 +4385,7 @@ void ConsolidationFx::vacuum_dense(const std::string& mode) { rc = tiledb_config_set(cfg, "sm.vacuum.mode", mode.c_str(), &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_array_vacuum(ctx_, DENSE_ARRAY_NAME, cfg); + rc = tiledb_array_vacuum(ctx_, dense_array_uri_.c_str(), cfg); REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); @@ -4492,37 +4410,39 @@ void ConsolidationFx::vacuum_sparse( REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_array_vacuum(ctx_, SPARSE_ARRAY_NAME, cfg); + rc = tiledb_array_vacuum(ctx_, sparse_array_uri_.c_str(), cfg); REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } void ConsolidationFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; + if (!vfs_test_setup_.is_rest()) { + if (!is_array(array_name)) + return; - CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); + CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); + } } void ConsolidationFx::remove_dense_vector() { - remove_array(DENSE_VECTOR_NAME); + remove_array(dense_vector_uri_.c_str()); } void ConsolidationFx::remove_dense_array() { - remove_array(DENSE_ARRAY_NAME); + remove_array(dense_array_uri_.c_str()); } void ConsolidationFx::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); + remove_array(sparse_array_uri_.c_str()); } void ConsolidationFx::remove_sparse_heterogeneous_array() { - remove_array(SPARSE_HETEROGENEOUS_ARRAY_NAME); + remove_array(sparse_heterogeneous_array_uri_.c_str()); } void ConsolidationFx::remove_sparse_string_array() { - remove_array(SPARSE_STRING_ARRAY_NAME); + remove_array(sparse_string_array_uri_.c_str()); } bool ConsolidationFx::is_array(const std::string& array_name) { @@ -4539,20 +4459,21 @@ void ConsolidationFx::check_commits_dir_dense( // Check number of consolidated commits files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_ARRAY_COMMITS_DIR, &get_commits_num, &data); + ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_commits_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_commits); // Check number of wrt files data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_ARRAY_COMMITS_DIR, &get_wrt_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_wrt_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_wrt); // Check number of ignore files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_ARRAY_COMMITS_DIR, &get_ignore_num, &data); + ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_ignore_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_ignore); } @@ -4565,20 +4486,21 @@ void ConsolidationFx::check_commits_dir_sparse( // Check number of consolidated commits files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, SPARSE_ARRAY_COMMITS_DIR, &get_commits_num, &data); + ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_commits_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_commits); // Check number of wrt files data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, SPARSE_ARRAY_COMMITS_DIR, &get_wrt_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_wrt_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_wrt); // Check number of ignore files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, SPARSE_ARRAY_COMMITS_DIR, &get_ignore_num, &data); + ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_ignore_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_ignore); } @@ -4589,7 +4511,7 @@ void ConsolidationFx::check_ok_num(int num_ok) { // Check number of ok files data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, SPARSE_ARRAY_NAME, &get_ok_num, &data); + rc = tiledb_vfs_ls(ctx_, vfs_, sparse_array_uri_.c_str(), &get_ok_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_ok); } @@ -4597,14 +4519,7 @@ void ConsolidationFx::check_ok_num(int num_ok) { TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, dense", - "[capi][consolidation][dense]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][dense][non-rest]") { remove_dense_array(); create_dense_array(); @@ -4622,18 +4537,15 @@ TEST_CASE_METHOD( read_dense_subarray_full(); } - // Encrypted remote arrays are not supported - if (!serialize_) { - SECTION("- write (encrypted) subarray, full") { - remove_dense_array(); - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; - create_dense_array(); - write_dense_subarray(); - write_dense_full(); - consolidate_dense(); - read_dense_subarray_full(); - } + SECTION("- write (encrypted) subarray, full") { + remove_dense_array(); + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + create_dense_array(); + write_dense_subarray(); + write_dense_full(); + consolidate_dense(); + read_dense_subarray_full(); } remove_dense_array(); @@ -4642,14 +4554,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse", - "[capi][consolidation][sparse]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][sparse][non-rest]") { remove_sparse_array(); create_sparse_array(); @@ -4667,18 +4572,15 @@ TEST_CASE_METHOD( read_sparse_unordered_full(); } - // Encrypted remote arrays are not supported - if (!serialize_) { - SECTION("- write (encrypted) unordered, full") { - remove_sparse_array(); - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; - create_sparse_array(); - write_sparse_unordered(); - write_sparse_full(); - consolidate_sparse(); - read_sparse_unordered_full(); - } + SECTION("- write (encrypted) unordered, full") { + remove_sparse_array(); + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + create_sparse_array(); + write_sparse_unordered(); + write_sparse_full(); + consolidate_sparse(); + read_sparse_unordered_full(); } remove_sparse_array(); @@ -4773,7 +4675,7 @@ int ConsolidationFx::get_vac_files_callback(const char* path, void* data) { void ConsolidationFx::get_array_meta_files_dense( std::vector& files) { files.clear(); - tiledb::sm::URI dense_array_uri(DENSE_ARRAY_NAME); + tiledb::sm::URI dense_array_uri(dense_array_uri_.c_str()); int rc = tiledb_vfs_ls( ctx_, vfs_, @@ -4788,7 +4690,7 @@ void ConsolidationFx::get_array_meta_files_dense( void ConsolidationFx::get_array_meta_vac_files_dense( std::vector& files) { files.clear(); - tiledb::sm::URI dense_array_uri(DENSE_ARRAY_NAME); + tiledb::sm::URI dense_array_uri(dense_array_uri_.c_str()); int rc = tiledb_vfs_ls( ctx_, vfs_, @@ -4802,7 +4704,7 @@ void ConsolidationFx::get_array_meta_vac_files_dense( void ConsolidationFx::get_vac_files_dense(std::vector& files) { files.clear(); - tiledb::sm::URI dense_array_uri(DENSE_ARRAY_NAME); + tiledb::sm::URI dense_array_uri(dense_array_uri_.c_str()); int rc = tiledb_vfs_ls( ctx_, vfs_, dense_array_uri.c_str(), &get_vac_files_callback, &files); CHECK(rc == TILEDB_OK); @@ -4812,7 +4714,7 @@ void ConsolidationFx::get_vac_files_dense(std::vector& files) { TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, wrong configs", - "[capi][consolidation][adv][config]") { + "[capi][consolidation][adv][config][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -4894,7 +4796,7 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Test min frags (currently set to 5) > max frags (currently set to 2) - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); rc = tiledb_config_set( @@ -4907,13 +4809,13 @@ TEST_CASE_METHOD( config, "sm.consolidation.step_size_ratio", "-1", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); rc = tiledb_config_set( config, "sm.consolidation.step_size_ratio", "1.5", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); rc = tiledb_config_set( config, "sm.consolidation.step_size_ratio", "0.5", &error); @@ -4925,16 +4827,17 @@ TEST_CASE_METHOD( tiledb_config_set(config, "sm.consolidation.amplification", "-1", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check that there are 4 fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 4); @@ -4946,7 +4849,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #1", - "[capi][consolidation][adv][adv-1]") { + "[capi][consolidation][adv][adv-1][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -4980,11 +4883,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -4992,7 +4895,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -5004,7 +4908,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #2", - "[capi][consolidation][adv][adv-2]") { + "[capi][consolidation][adv][adv-2][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5038,11 +4942,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5050,7 +4954,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5062,7 +4967,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #3", - "[capi][consolidation][adv][adv-3]") { + "[capi][consolidation][adv][adv-3][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5096,11 +5001,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5108,7 +5013,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 4); @@ -5120,7 +5026,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #4", - "[capi][consolidation][adv][adv-4]") { + "[capi][consolidation][adv][adv-4][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5154,11 +5060,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5166,7 +5072,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -5178,7 +5085,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #5", - "[capi][consolidation][adv][adv-5]") { + "[capi][consolidation][adv][adv-5][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5212,11 +5119,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5224,7 +5131,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5236,7 +5144,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #6", - "[capi][consolidation][adv][adv-6]") { + "[capi][consolidation][adv][adv-6][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5270,11 +5178,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5282,7 +5190,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -5294,7 +5203,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #7", - "[capi][consolidation][adv][adv-7]") { + "[capi][consolidation][adv][adv-7][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5328,11 +5237,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5340,7 +5249,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5352,7 +5262,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation #8", - "[capi][consolidation][adv][adv-8]") { + "[capi][consolidation][adv][adv-8][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5386,11 +5296,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5398,7 +5308,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -5413,7 +5324,7 @@ TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, fragments that don't coincide with space tiles " "#1", - "[capi][consolidation][not-coinciding-1]") { + "[capi][consolidation][not-coinciding-1][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments_not_coinciding(); @@ -5447,11 +5358,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5459,7 +5370,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -5475,7 +5387,7 @@ TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, fragments that don't coincide with space tiles " "#2", - "[capi][consolidation][not-coinciding-2]") { + "[capi][consolidation][not-coinciding-2][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments_not_coinciding_with_gaps(); @@ -5513,11 +5425,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5525,7 +5437,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -5536,7 +5449,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, small buffer size", - "[capi][consolidation][adv][buffer-size]") { + "[capi][consolidation][adv][buffer-size][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5568,11 +5481,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5580,7 +5493,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5591,7 +5505,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, encrypted array", - "[capi][consolidation][adv][encryption]") { + "[capi][consolidation][adv][encryption][non-rest]") { remove_dense_vector(); encryption_type_ = TILEDB_AES_256_GCM; encryption_key_ = "0123456789abcdeF0123456789abcdeF"; @@ -5631,14 +5545,14 @@ TEST_CASE_METHOD( rc = tiledb_config_set(cfg, "sm.encryption_key", encryption_key_, &err); REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), cfg); CHECK(rc == TILEDB_OK); // Check correctness read_dense_vector(); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5646,7 +5560,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5658,7 +5573,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, overwritten fragments, no deletion", - "[capi][consolidation][adv][overwritten-no-del]") { + "[capi][consolidation][adv][overwritten-no-del][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -5692,11 +5607,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5704,7 +5619,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 4); @@ -5715,7 +5631,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, overwritten fragments, deletion #1", - "[capi][adv][consolidation][overwritten-del-1]") { + "[capi][adv][consolidation][overwritten-del-1][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_del_1(); @@ -5749,14 +5665,14 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check correctness read_dense_vector_del_1(); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5764,7 +5680,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5775,7 +5692,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, overwritten fragments, deletion #2", - "[capi][consolidation][adv][overwritten-del-2]") { + "[capi][consolidation][adv][overwritten-del-2][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_del_2(); @@ -5813,11 +5730,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5825,7 +5742,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -5836,7 +5754,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, overwritten fragments, deletion #3", - "[capi][consolidation][adv][overwritten-del-3]") { + "[capi][consolidation][adv][overwritten-del-3][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_del_3(); @@ -5870,11 +5788,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5882,7 +5800,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); @@ -5896,7 +5815,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, non-consolidatable", - "[capi][consolidation][adv][non-consolidatable]") { + "[capi][consolidation][adv][non-consolidatable][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_consolidatable_1(); @@ -5930,11 +5849,11 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -5942,7 +5861,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -5954,7 +5874,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test advanced consolidation, consolidatable", - "[capi][consolidation][adv][consolidatable]") { + "[capi][consolidation][adv][consolidatable][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_consolidatable_2(); @@ -6006,7 +5926,7 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check correctness @@ -6015,12 +5935,13 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == ((should_consolidate) ? 3 : 2)); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check correctness @@ -6028,7 +5949,8 @@ TEST_CASE_METHOD( // Check number of fragments data.num = 0; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == ((should_consolidate) ? 1 : 2)); @@ -6039,7 +5961,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation and time traveling", - "[capi][consolidation][time-traveling]") { + "[capi][consolidation][time-traveling][non-rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -6056,20 +5978,21 @@ TEST_CASE_METHOD( REQUIRE(err == nullptr); // Consolidate - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, cfg); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), cfg); CHECK(rc == TILEDB_OK); tiledb_config_free(&cfg); // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 5); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_VECTOR_FRAG_META_DIR, &get_meta_num, &data); + ctx_, vfs_, dense_vector_frag_meta_dir_.c_str(), &get_meta_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 0); @@ -6080,13 +6003,13 @@ TEST_CASE_METHOD( read_dense_vector(3); read_dense_vector(); - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, NULL); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), NULL); CHECK(rc == TILEDB_OK); read_dense_vector(); // Open array - before timestamp 4, the array will appear empty tiledb_array_t* array; - rc = tiledb_array_alloc(ctx_, DENSE_VECTOR_NAME, &array); + rc = tiledb_array_alloc(ctx_, dense_vector_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_set_open_timestamp_end(ctx_, array, 1); CHECK(rc == TILEDB_OK); @@ -6134,7 +6057,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidating fragment metadata", - "[capi][consolidation][fragment-meta]") { + "[capi][consolidation][fragment-meta][rest]") { remove_dense_vector(); create_dense_vector(); write_dense_vector_4_fragments(); @@ -6150,19 +6073,20 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - this will consolidate only the fragment metadata - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 4); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_VECTOR_FRAG_META_DIR, &get_meta_num, &data); + ctx_, vfs_, dense_vector_frag_meta_dir_.c_str(), &get_meta_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -6176,7 +6100,8 @@ TEST_CASE_METHOD( // Check data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 8); @@ -6190,53 +6115,56 @@ TEST_CASE_METHOD( read_dense_vector(8); // Consolidate - this will consolidate only the fragment metadata - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_VECTOR_FRAG_META_DIR, &get_meta_num, &data); + ctx_, vfs_, dense_vector_frag_meta_dir_.c_str(), &get_meta_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); - // This will vacuum fragments - no effect on consolidated fragment metadata - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, NULL); - CHECK(rc == TILEDB_OK); - CHECK(data.num == 2); - - // Check - data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_VECTOR_FRAG_DIR, &get_dir_num, &data); - CHECK(rc == TILEDB_OK); - CHECK(data.num == 8); - read_dense_vector(1); - read_dense_vector(2); - read_dense_vector(3); - read_dense_vector(4); - read_dense_vector(5); - read_dense_vector(6); - read_dense_vector(7); - read_dense_vector(8); + if (!vfs_test_setup_.is_rest()) { + // This will vacuum fragments - no effect on consolidated fragment metadata + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), NULL); + CHECK(rc == TILEDB_OK); + CHECK(data.num == 2); + + // Check + data = {ctx_, vfs_, 0}; + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_vector_frag_dir_.c_str(), &get_dir_num, &data); + CHECK(rc == TILEDB_OK); + CHECK(data.num == 8); + read_dense_vector(1); + read_dense_vector(2); + read_dense_vector(3); + read_dense_vector(4); + read_dense_vector(5); + read_dense_vector(6); + read_dense_vector(7); + read_dense_vector(8); + } // Test wrong vacuum mode rc = tiledb_config_set(config, "sm.vacuum.mode", "foo", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); // Vacuum consolidated fragment metadata rc = tiledb_config_set(config, "sm.vacuum.mode", "fragment_meta", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_vacuum(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_vacuum(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, DENSE_VECTOR_FRAG_META_DIR, &get_meta_num, &data); + ctx_, vfs_, dense_vector_frag_meta_dir_.c_str(), &get_meta_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -6254,7 +6182,7 @@ TEST_CASE_METHOD( rc = tiledb_config_set(config, "sm.consolidation.mode", "foo", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_consolidate(ctx_, DENSE_VECTOR_NAME, config); + rc = tiledb_array_consolidate(ctx_, dense_vector_uri_.c_str(), config); CHECK(rc == TILEDB_ERR); tiledb_config_free(&config); @@ -6264,14 +6192,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse heterogeneous", - "[capi][consolidation][sparse][heter]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][sparse][heter][non-rest]") { remove_sparse_heterogeneous_array(); create_sparse_heterogeneous_array(); @@ -6289,18 +6210,15 @@ TEST_CASE_METHOD( read_sparse_heterogeneous_unordered_full(); } - // Encrypted remote arrays are not supported - if (!serialize_) { - SECTION("- write (encrypted) unordered, full") { - remove_sparse_heterogeneous_array(); - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; - create_sparse_heterogeneous_array(); - write_sparse_heterogeneous_unordered(); - write_sparse_heterogeneous_full(); - consolidate_sparse_heterogeneous(); - read_sparse_heterogeneous_unordered_full(); - } + SECTION("- write (encrypted) unordered, full") { + remove_sparse_heterogeneous_array(); + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + create_sparse_heterogeneous_array(); + write_sparse_heterogeneous_unordered(); + write_sparse_heterogeneous_full(); + consolidate_sparse_heterogeneous(); + read_sparse_heterogeneous_unordered_full(); } remove_sparse_heterogeneous_array(); @@ -6309,14 +6227,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse string", - "[capi][consolidation][sparse][string]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][sparse][string][non-rest]") { remove_sparse_string_array(); create_sparse_string_array(); @@ -6334,18 +6245,15 @@ TEST_CASE_METHOD( read_sparse_string_unordered_full(); } - // Encrypted remote arrays are not supported - if (!serialize_) { - SECTION("- write (encrypted) unordered, full") { - remove_sparse_string_array(); - encryption_type_ = TILEDB_AES_256_GCM; - encryption_key_ = "0123456789abcdeF0123456789abcdeF"; - create_sparse_string_array(); - write_sparse_string_unordered(); - write_sparse_string_full(); - consolidate_sparse_string(); - read_sparse_string_unordered_full(); - } + SECTION("- write (encrypted) unordered, full") { + remove_sparse_string_array(); + encryption_type_ = TILEDB_AES_256_GCM; + encryption_key_ = "0123456789abcdeF0123456789abcdeF"; + create_sparse_string_array(); + write_sparse_string_unordered(); + write_sparse_string_full(); + consolidate_sparse_string(); + read_sparse_string_unordered_full(); } remove_sparse_string_array(); @@ -6354,14 +6262,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidating fragment metadata, sparse string", - "[capi][consolidation][fragment-meta][sparse][string]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][fragment-meta][sparse][string][rest]") { remove_sparse_string_array(); create_sparse_string_array(); write_sparse_string_full(); @@ -6381,20 +6282,28 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); // Consolidate - this will consolidate only the fragment metadata - rc = tiledb_array_consolidate(ctx_, SPARSE_STRING_ARRAY_NAME, config); + rc = tiledb_array_consolidate(ctx_, sparse_string_array_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, SPARSE_STRING_ARRAY_FRAG_DIR, &get_dir_num, &data); + ctx_, + vfs_, + sparse_string_array_fragment_dir_.c_str(), + &get_dir_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, SPARSE_STRING_ARRAY_FRAG_META_DIR, &get_meta_num, &data); + ctx_, + vfs_, + sparse_string_array_frag_meta_dir_.c_str(), + &get_meta_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -6405,13 +6314,17 @@ TEST_CASE_METHOD( rc = tiledb_config_set(config, "sm.vacuum.mode", "fragment_meta", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - rc = tiledb_array_vacuum(ctx_, SPARSE_STRING_ARRAY_NAME, config); + rc = tiledb_array_vacuum(ctx_, sparse_string_array_uri_.c_str(), config); CHECK(rc == TILEDB_OK); // Check number of consolidated metadata files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, SPARSE_STRING_ARRAY_FRAG_META_DIR, &get_meta_num, &data); + ctx_, + vfs_, + sparse_string_array_frag_meta_dir_.c_str(), + &get_meta_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -6426,14 +6339,7 @@ TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidating fragment metadata, sparse string, pass only " "context", - "[capi][consolidation][fragment-meta][sparse][string]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][fragment-meta][sparse][string][non-rest]") { remove_sparse_string_array(); create_sparse_string_array(); write_sparse_string_full(); @@ -6452,24 +6358,34 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); + vfs_test_setup_.update_config(config); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; // Consolidate - this will consolidate only the fragment metadata - rc = tiledb_array_consolidate(ctx, SPARSE_STRING_ARRAY_NAME, nullptr); + rc = + tiledb_array_consolidate(ctx_, sparse_string_array_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check number of fragments - get_num_struct data = {ctx, vfs_, 0}; + get_num_struct data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx, vfs_, SPARSE_STRING_ARRAY_FRAG_DIR, &get_dir_num, &data); + ctx_, + vfs_, + sparse_string_array_fragment_dir_.c_str(), + &get_dir_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 2); // Check number of consolidated metadata files - data = {ctx, vfs_, 0}; + data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx, vfs_, SPARSE_STRING_ARRAY_FRAG_META_DIR, &get_meta_num, &data); + ctx_, + vfs_, + sparse_string_array_frag_meta_dir_.c_str(), + &get_meta_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); @@ -6481,24 +6397,27 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - // Create new context - tiledb_ctx_free(&ctx); - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); + vfs_test_setup_.update_config(config); + ctx_ = vfs_test_setup_.ctx_c; + vfs_ = vfs_test_setup_.vfs_c; - rc = tiledb_array_vacuum(ctx, SPARSE_STRING_ARRAY_NAME, nullptr); + rc = tiledb_array_vacuum(ctx_, sparse_string_array_uri_.c_str(), nullptr); CHECK(rc == TILEDB_OK); // Check number of consolidated metadata files - data = {ctx, vfs_, 0}; + data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx, vfs_, SPARSE_STRING_ARRAY_FRAG_META_DIR, &get_meta_num, &data); + ctx_, + vfs_, + sparse_string_array_frag_meta_dir_.c_str(), + &get_meta_num, + &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 1); // Validate read read_sparse_string_full_unordered(); - tiledb_ctx_free(&ctx); tiledb_config_free(&config); remove_sparse_string_array(); } @@ -6506,14 +6425,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation and timestamps", - "[capi][consolidation][timestamps]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][timestamps][non-rest]") { remove_dense_array(); create_dense_array(); @@ -6585,14 +6497,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test vacuuming and timestamps", - "[capi][vacuuming][timestamps]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][vacuuming][timestamps][non-rest]") { remove_dense_array(); create_dense_array(); @@ -6656,14 +6561,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, dense, commits", - "[capi][consolidation][dense][commits]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][dense][commits][rest]") { remove_dense_array(); create_dense_array(); @@ -6691,20 +6589,23 @@ TEST_CASE_METHOD( check_commits_dir_dense(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. - consolidate_dense(); - vacuum_dense(); - read_dense_full_subarray(); - check_commits_dir_dense(1, 1, 1); + // Fragment consolidation not yet supported on remote arrays + if (!vfs_test_setup_.is_rest()) { + consolidate_dense(); + vacuum_dense(); + read_dense_full_subarray(); + check_commits_dir_dense(1, 1, 1); - // Consolidation to get rid of ignore file. - consolidate_dense("commits"); - read_dense_full_subarray(); - check_commits_dir_dense(2, 1, 1); + // Consolidation to get rid of ignore file. + consolidate_dense("commits"); + read_dense_full_subarray(); + check_commits_dir_dense(2, 1, 1); - // Second vacuum works. - vacuum_dense("commits"); - read_dense_full_subarray(); - check_commits_dir_dense(1, 0, 0); + // Second vacuum works. + vacuum_dense("commits"); + read_dense_full_subarray(); + check_commits_dir_dense(1, 0, 0); + } } SECTION("- write subarray, full") { @@ -6730,24 +6631,26 @@ TEST_CASE_METHOD( check_commits_dir_dense(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. - consolidate_dense(); - vacuum_dense(); - read_dense_subarray_full(); - check_commits_dir_dense(1, 1, 1); + // Fragment consolidation not yet supported on remote arrays + if (!vfs_test_setup_.is_rest()) { + consolidate_dense(); + vacuum_dense(); + read_dense_subarray_full(); + check_commits_dir_dense(1, 1, 1); - // Consolidation to get rid of ignore file. - consolidate_dense("commits"); - read_dense_subarray_full(); - check_commits_dir_dense(2, 1, 1); + // Consolidation to get rid of ignore file. + consolidate_dense("commits"); + read_dense_subarray_full(); + check_commits_dir_dense(2, 1, 1); - // Second vacuum works. - vacuum_dense("commits"); - read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + // Second vacuum works. + vacuum_dense("commits"); + read_dense_subarray_full(); + check_commits_dir_dense(1, 0, 0); + } } - // Encrypted remote arrays are not supported - if (!serialize_) { + if (!vfs_test_setup_.is_rest()) { SECTION("- write (encrypted) subarray, full") { remove_dense_array(); encryption_type_ = TILEDB_AES_256_GCM; @@ -6799,14 +6702,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse, commits", - "[capi][consolidation][sparse][commits]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][sparse][commits][rest]") { remove_sparse_array(); create_sparse_array(); @@ -6834,20 +6730,23 @@ TEST_CASE_METHOD( check_commits_dir_sparse(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. - consolidate_sparse(); - vacuum_sparse(); - read_sparse_full_unordered(); - check_commits_dir_sparse(1, 1, 1); + // Fragment consolidation not yet supported on remote arrays + if (!vfs_test_setup_.is_rest()) { + consolidate_sparse(); + vacuum_sparse(); + read_sparse_full_unordered(); + check_commits_dir_sparse(1, 1, 1); - // Consolidation to get rid of ignore file. - consolidate_sparse("commits"); - read_sparse_full_unordered(); - check_commits_dir_sparse(2, 1, 1); + // Consolidation to get rid of ignore file. + consolidate_sparse("commits"); + read_sparse_full_unordered(); + check_commits_dir_sparse(2, 1, 1); - // Second vacuum works. - vacuum_sparse("commits"); - read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + // Second vacuum works. + vacuum_sparse("commits"); + read_sparse_full_unordered(); + check_commits_dir_sparse(1, 0, 0); + } } SECTION("- write unordered, full") { @@ -6874,24 +6773,26 @@ TEST_CASE_METHOD( check_commits_dir_sparse(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. - consolidate_sparse(); - vacuum_sparse(); - read_sparse_unordered_full(); - check_commits_dir_sparse(1, 1, 1); + // Fragment consolidation not yet supported on remote arrays + if (!vfs_test_setup_.is_rest()) { + consolidate_sparse(); + vacuum_sparse(); + read_sparse_unordered_full(); + check_commits_dir_sparse(1, 1, 1); - // Consolidation to get rid of ignore file. - consolidate_sparse("commits"); - read_sparse_unordered_full(); - check_commits_dir_sparse(2, 1, 1); + // Consolidation to get rid of ignore file. + consolidate_sparse("commits"); + read_sparse_unordered_full(); + check_commits_dir_sparse(2, 1, 1); - // Second vacuum works. - vacuum_sparse("commits"); - read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + // Second vacuum works. + vacuum_sparse("commits"); + read_sparse_unordered_full(); + check_commits_dir_sparse(1, 0, 0); + } } - // Encrypted remote arrays are not supported - if (!serialize_) { + if (!vfs_test_setup_.is_rest()) { SECTION("- write (encrypted) unordered, full") { remove_sparse_array(); encryption_type_ = TILEDB_AES_256_GCM; @@ -6946,17 +6847,14 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse, commits, mixed versions", - "[capi][consolidation][commits][mixed-versions]") { + "[capi][consolidation][commits][mixed-versions][non-rest]") { if constexpr (is_experimental_build) { return; } -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -6965,14 +6863,15 @@ TEST_CASE_METHOD( std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; REQUIRE( tiledb_vfs_copy_dir( - ctx_, vfs_, v11_arrays_dir.c_str(), SPARSE_ARRAY_NAME) == TILEDB_OK); + ctx_, vfs_, v11_arrays_dir.c_str(), sparse_array_uri_.c_str()) == + TILEDB_OK); // Write v11 fragment. write_sparse_full(); // Upgrade to latest version. REQUIRE( - tiledb_array_upgrade_version(ctx_, SPARSE_ARRAY_NAME, nullptr) == + tiledb_array_upgrade_version(ctx_, sparse_array_uri_.c_str(), nullptr) == TILEDB_OK); // Consolidation works. @@ -7021,14 +6920,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, dense split fragments", - "[capi][consolidation][dense][split-fragments]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][dense][split-fragments][non-rest]") { remove_dense_array(); create_dense_array(); write_dense_subarray(1, 2, 1, 2); @@ -7038,7 +6930,8 @@ TEST_CASE_METHOD( // Create fragment info object tiledb_fragment_info_t* fragment_info = nullptr; - int rc = tiledb_fragment_info_alloc(ctx_, DENSE_ARRAY_NAME, &fragment_info); + int rc = tiledb_fragment_info_alloc( + ctx_, dense_array_uri_.c_str(), &fragment_info); CHECK(rc == TILEDB_OK); // Load fragment info @@ -7067,7 +6960,8 @@ TEST_CASE_METHOD( // Consolidate const char* uris[2] = {strrchr(uri1, '/') + 1, strrchr(uri2, '/') + 1}; - rc = tiledb_array_consolidate_fragments(ctx_, DENSE_ARRAY_NAME, uris, 2, cfg); + rc = tiledb_array_consolidate_fragments( + ctx_, dense_array_uri_.c_str(), uris, 2, cfg); CHECK(rc == TILEDB_OK); tiledb_config_free(&cfg); @@ -7075,7 +6969,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_ARRAY_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_array_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 5); @@ -7083,13 +6978,14 @@ TEST_CASE_METHOD( read_dense_four_tiles(); // Vacuum - rc = tiledb_array_vacuum(ctx_, DENSE_ARRAY_NAME, NULL); + rc = tiledb_array_vacuum(ctx_, dense_array_uri_.c_str(), NULL); CHECK(rc == TILEDB_OK); read_dense_four_tiles(); // Check number of fragments data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, DENSE_ARRAY_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, dense_array_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -7100,7 +6996,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse split fragments", - "[capi][consolidation][sparse][split-fragments]") { + "[capi][consolidation][sparse][split-fragments][non-rest]") { remove_sparse_array(); create_sparse_array(); write_sparse_row(0); @@ -7110,7 +7006,8 @@ TEST_CASE_METHOD( // Create fragment info object tiledb_fragment_info_t* fragment_info = nullptr; - int rc = tiledb_fragment_info_alloc(ctx_, SPARSE_ARRAY_NAME, &fragment_info); + int rc = tiledb_fragment_info_alloc( + ctx_, sparse_array_uri_.c_str(), &fragment_info); CHECK(rc == TILEDB_OK); // Load fragment info @@ -7139,8 +7036,8 @@ TEST_CASE_METHOD( // Consolidate const char* uris[2] = {strrchr(uri1, '/') + 1, strrchr(uri2, '/') + 1}; - rc = - tiledb_array_consolidate_fragments(ctx_, SPARSE_ARRAY_NAME, uris, 2, cfg); + rc = tiledb_array_consolidate_fragments( + ctx_, sparse_array_uri_.c_str(), uris, 2, cfg); CHECK(rc == TILEDB_OK); tiledb_config_free(&cfg); @@ -7148,7 +7045,8 @@ TEST_CASE_METHOD( // Check number of fragments get_num_struct data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, SPARSE_ARRAY_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, sparse_array_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 5); @@ -7156,13 +7054,14 @@ TEST_CASE_METHOD( read_sparse_rows(); // Vacuum - rc = tiledb_array_vacuum(ctx_, SPARSE_ARRAY_NAME, NULL); + rc = tiledb_array_vacuum(ctx_, sparse_array_uri_.c_str(), NULL); CHECK(rc == TILEDB_OK); read_sparse_rows(); // Check number of fragments data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls(ctx_, vfs_, SPARSE_ARRAY_FRAG_DIR, &get_dir_num, &data); + rc = tiledb_vfs_ls( + ctx_, vfs_, sparse_array_frag_dir_.c_str(), &get_dir_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == 3); @@ -7173,7 +7072,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, empty array", - "[capi][consolidation][empty]") { + "[capi][consolidation][empty][non-rest]") { auto sparse = GENERATE(true, false); auto mode = GENERATE( std::string("commits"), @@ -7199,7 +7098,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, sparse string, no progress", - "[capi][consolidation][sparse][string][no-progress]") { + "[capi][consolidation][sparse][string][no-progress][non-rest]") { remove_sparse_string_array(); create_sparse_string_array(); @@ -7222,14 +7121,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationFx, "C API: Test consolidation, fragments/commits out of order", - "[capi][consolidation][fragments-commits][out-of-order]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][consolidation][fragments-commits][out-of-order][non-rest]") { remove_sparse_array(); create_sparse_array(); diff --git a/test/src/unit-capi-dense_array.cc b/test/src/unit-capi-dense_array.cc index ce94c7f7c351..6c0249be287c 100644 --- a/test/src/unit-capi-dense_array.cc +++ b/test/src/unit-capi-dense_array.cc @@ -37,7 +37,6 @@ #include #include "test/support/src/helpers.h" -#include "test/support/src/serialization_wrappers.h" #include "test/support/src/vfs_helpers.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/win.h" @@ -104,24 +103,21 @@ struct DenseArrayFx { // Vector of supported filsystems const std::vector> fs_vec_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Functions DenseArrayFx(); ~DenseArrayFx(); void create_temp_dir(const std::string& path); void remove_temp_dir(const std::string& path); - void check_sorted_reads(const std::string& path); - void check_sorted_writes(const std::string& path); - void check_invalid_cell_num_in_dense_writes(const std::string& path); - void check_simultaneous_writes(const std::string& path); - void check_cancel_and_retry_writes(const std::string& path); - void check_return_coords(const std::string& path, bool split_coords); - void check_non_empty_domain(const std::string& path); + void check_sorted_reads(); + void check_sorted_writes(); + void check_invalid_cell_num_in_dense_writes(); + void check_simultaneous_writes(); + void check_cancel_and_retry_writes(); + void check_return_coords(bool split_coords); + void check_non_empty_domain(); void create_dense_vector(const std::string& path); void create_dense_array(const std::string& array_name); void create_dense_array_1_attribute(const std::string& array_name); @@ -265,7 +261,8 @@ struct DenseArrayFx { }; DenseArrayFx::DenseArrayFx() - : fs_vec_(vfs_test_get_fs_vec()) { + : fs_vec_(vfs_test_get_fs_vec()) + , prefix_(vfs_array_uri(fs_vec_[0], fs_vec_[0]->temp_dir())) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); std::srand(0); @@ -510,8 +507,7 @@ int* DenseArrayFx::read_dense_array_2D( REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } - rc = array_open_wrapper( - ctx_, query_type, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, query_type); CHECK(rc == TILEDB_OK); // Create query @@ -533,13 +529,7 @@ int* DenseArrayFx::read_dense_array_2D( CHECK(!has_results); // Read from array - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Now the query must have results @@ -547,6 +537,12 @@ int* DenseArrayFx::read_dense_array_2D( CHECK(rc == TILEDB_OK); CHECK(has_results); + rc = tiledb_query_finalize(ctx_, query); + REQUIRE(rc == TILEDB_OK); + rc = + tiledb_query_finalize(ctx_, query); // Second time must create no problem + REQUIRE(rc == TILEDB_OK); + // Close array rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -604,9 +600,8 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); - CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); + REQUIRE(rc == TILEDB_OK); // Submit query #1 - Dense tiledb_query_t* query_1; @@ -619,13 +614,7 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_1, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_1); CHECK(rc == TILEDB_OK); tiledb_query_free(&query_1); @@ -640,13 +629,7 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { rc = tiledb_query_set_data_buffer( ctx_, query_2, "d", coords_2, &coords_2_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_2, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_2); CHECK(rc == TILEDB_OK); tiledb_query_free(&query_2); @@ -661,13 +644,7 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { rc = tiledb_query_set_data_buffer( ctx_, query_3, "d", coords_3, &coords_3_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_3, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_3); CHECK(rc == TILEDB_OK); tiledb_query_free(&query_3); @@ -683,13 +660,7 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_4, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_4); CHECK(rc == TILEDB_OK); tiledb_query_free(&query_4); @@ -743,8 +714,7 @@ void DenseArrayFx::write_dense_array_by_tiles( REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query @@ -776,7 +746,12 @@ void DenseArrayFx::write_dense_array_by_tiles( buffer_size = k * l * sizeof(int); buffer_sizes[0] = buffer_size; - auto rc{tiledb_query_submit(ctx_, query)}; + if ((i == domain_size_0 - tile_extent_0) && + (j == domain_size_1 - tile_extent_1)) { + rc = tiledb_query_submit_and_finalize(ctx_, query); + } else { + rc = tiledb_query_submit(ctx_, query); + } const char* msg = "unset"; tiledb_error_t* err{nullptr}; @@ -790,10 +765,6 @@ void DenseArrayFx::write_dense_array_by_tiles( } } - // Finalize query - rc = tiledb_query_finalize(ctx_, query); - REQUIRE(rc == TILEDB_OK); - // Close array rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -822,9 +793,8 @@ void DenseArrayFx::write_dense_subarray_2D( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK_SAFE(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); - CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, query_type); + CHECK_SAFE(rc == TILEDB_OK); // Create query tiledb_query_t* query; @@ -839,13 +809,13 @@ void DenseArrayFx::write_dense_subarray_2D( REQUIRE_SAFE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + if (query_layout != TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit(ctx_, query); + REQUIRE_SAFE(rc == TILEDB_OK); + rc = tiledb_query_finalize(ctx_, query); + } else { + tiledb_query_submit_and_finalize(ctx_, query); + } REQUIRE_SAFE(rc == TILEDB_OK); // Close array @@ -873,8 +843,7 @@ void DenseArrayFx::write_dense_subarray_2D_with_cancel( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, query_type); CHECK(rc == TILEDB_OK); // Create query @@ -957,7 +926,7 @@ void DenseArrayFx::write_dense_subarray_2D_with_cancel( tiledb_query_free(&query); } -void DenseArrayFx::check_sorted_reads(const std::string& path) { +void DenseArrayFx::check_sorted_reads() { // Parameters used in this test int64_t domain_size_0 = 2500; int64_t domain_size_1 = 5000; @@ -970,7 +939,7 @@ void DenseArrayFx::check_sorted_reads(const std::string& path) { uint64_t capacity = 250000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = path + "sorted_reads_array"; + std::string array_name = prefix_ + "sorted_reads_array"; // Create a dense integer array create_dense_array_2D( @@ -1056,8 +1025,7 @@ void DenseArrayFx::check_sorted_reads(const std::string& path) { REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Check out of bounds subarray @@ -1105,7 +1073,7 @@ void DenseArrayFx::check_sorted_reads(const std::string& path) { tiledb_query_free(&query); } -void DenseArrayFx::check_sorted_writes(const std::string& path) { +void DenseArrayFx::check_sorted_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -1118,7 +1086,7 @@ void DenseArrayFx::check_sorted_writes(const std::string& path) { uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = path + "sorted_writes_array"; + std::string array_name = prefix_ + "sorted_writes_array"; // Create a dense integer array create_dense_array_2D( @@ -1190,8 +1158,7 @@ void DenseArrayFx::check_sorted_writes(const std::string& path) { } } -void DenseArrayFx::check_invalid_cell_num_in_dense_writes( - const std::string& path) { +void DenseArrayFx::check_invalid_cell_num_in_dense_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -1204,7 +1171,7 @@ void DenseArrayFx::check_invalid_cell_num_in_dense_writes( uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = path + "invalid_cell_num_dense_writes_array"; + std::string array_name = prefix_ + "invalid_cell_num_dense_writes_array"; // Create a dense integer array create_dense_array_2D( @@ -1228,8 +1195,7 @@ void DenseArrayFx::check_invalid_cell_num_in_dense_writes( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Global order @@ -1241,9 +1207,7 @@ void DenseArrayFx::check_invalid_cell_num_in_dense_writes( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx_, query); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx_, query); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_ERR); tiledb_query_free(&query); @@ -1255,13 +1219,7 @@ void DenseArrayFx::check_invalid_cell_num_in_dense_writes( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_ERR); tiledb_query_free(&query); @@ -1273,7 +1231,7 @@ void DenseArrayFx::check_invalid_cell_num_in_dense_writes( tiledb_array_free(&array); } -void DenseArrayFx::check_simultaneous_writes(const std::string& path) { +void DenseArrayFx::check_simultaneous_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -1286,7 +1244,7 @@ void DenseArrayFx::check_simultaneous_writes(const std::string& path) { uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = path + "simultaneous_writes_array"; + std::string array_name = prefix_ + "simultaneous_writes_array"; // Create a dense integer array create_dense_array_2D( @@ -1343,7 +1301,7 @@ void DenseArrayFx::check_simultaneous_writes(const std::string& path) { } } -void DenseArrayFx::check_cancel_and_retry_writes(const std::string& path) { +void DenseArrayFx::check_cancel_and_retry_writes() { // Parameters used in this test int64_t domain_size_0 = 100; int64_t domain_size_1 = 100; @@ -1356,7 +1314,7 @@ void DenseArrayFx::check_cancel_and_retry_writes(const std::string& path) { uint64_t capacity = 1000; tiledb_layout_t cell_order = TILEDB_ROW_MAJOR; tiledb_layout_t tile_order = TILEDB_ROW_MAJOR; - std::string array_name = path + "cancel_and_retry_writes_array"; + std::string array_name = prefix_ + "cancel_and_retry_writes_array"; // Create a dense integer array create_dense_array_2D( @@ -1682,9 +1640,8 @@ void DenseArrayFx::create_large_dense_array_1_attribute( tiledb_array_schema_free(&array_schema); } -void DenseArrayFx::check_return_coords( - const std::string& path, bool split_coords) { - std::string array_name = path + "return_coords"; +void DenseArrayFx::check_return_coords(bool split_coords) { + std::string array_name = prefix_ + "return_coords"; create_dense_array(array_name); write_dense_array(array_name); read_dense_array_with_coords_full_global(array_name, split_coords); @@ -1734,8 +1691,7 @@ void DenseArrayFx::write_dense_array(const std::string& array_name) { tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query @@ -1756,14 +1712,7 @@ void DenseArrayFx::write_dense_array(const std::string& array_name) { rc = tiledb_query_set_data_buffer( ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); - - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1815,8 +1764,7 @@ void DenseArrayFx::write_dense_array_missing_attributes( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query @@ -1838,13 +1786,7 @@ void DenseArrayFx::write_dense_array_missing_attributes( // Observe we omit setting buffer for one of the attributes (a3) // Submit query - this should fail - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_ERR); // Close array @@ -1880,8 +1822,7 @@ void DenseArrayFx::write_partial_dense_array(const std::string& array_name) { tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_WRITE, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query @@ -1906,13 +1847,7 @@ void DenseArrayFx::write_partial_dense_array(const std::string& array_name) { rc = tiledb_query_set_subarray(ctx_, query, subarray); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1970,13 +1905,10 @@ void DenseArrayFx::write_large_dense_array(const std::string& array_name) { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); + CHECK(rc == TILEDB_OK); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2024,9 +1956,8 @@ void DenseArrayFx::read_dense_vector_mixed(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); tiledb_config_free(&cfg); } - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); - CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + REQUIRE(rc == TILEDB_OK); // Preparation uint64_t subarray[] = {1, 410}; @@ -2043,19 +1974,17 @@ void DenseArrayFx::read_dense_vector_mixed(const std::string& array_name) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_a) == a_size); for (int i = 0; i < 410; ++i) @@ -2098,8 +2027,7 @@ void DenseArrayFx::read_dense_array_with_coords_full_global( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2160,19 +2088,17 @@ void DenseArrayFx::read_dense_array_with_coords_full_global( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) == buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) == buffer_a2_off_size); @@ -2247,8 +2173,7 @@ void DenseArrayFx::read_dense_array_with_coords_full_row( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2309,19 +2234,17 @@ void DenseArrayFx::read_dense_array_with_coords_full_row( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) == buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) == buffer_a2_off_size); @@ -2394,8 +2317,7 @@ void DenseArrayFx::read_dense_array_with_coords_full_col( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2456,19 +2378,17 @@ void DenseArrayFx::read_dense_array_with_coords_full_col( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) == buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) == buffer_a2_off_size); @@ -2543,8 +2463,7 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_global( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2605,19 +2524,17 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_global( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) <= buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) <= buffer_a2_off_size); @@ -2691,8 +2608,7 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_row( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2753,19 +2669,17 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_row( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) == buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) == buffer_a2_off_size); @@ -2839,8 +2753,7 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_col( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -2901,19 +2814,17 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_col( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Check buffers CHECK(sizeof(c_buffer_a1) == buffer_a1_size); CHECK(sizeof(c_buffer_a2_off) == buffer_a2_off_size); @@ -2968,8 +2879,7 @@ void DenseArrayFx::read_large_dense_array( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create query @@ -2991,19 +2901,17 @@ void DenseArrayFx::read_large_dense_array( CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); CHECK(status == TILEDB_COMPLETED); + // Finalize query + rc = tiledb_query_finalize(ctx_, query); + CHECK(rc == TILEDB_OK); + // Close array rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3013,16 +2921,15 @@ void DenseArrayFx::read_large_dense_array( tiledb_query_free(&query); } -void DenseArrayFx::check_non_empty_domain(const std::string& path) { - std::string array_name = path + "dense_non_empty_domain"; +void DenseArrayFx::check_non_empty_domain() { + std::string array_name = prefix_ + "dense_non_empty_domain"; create_dense_array(array_name); // Check empty domain tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); int is_empty; uint64_t domain[4]; @@ -3044,8 +2951,7 @@ void DenseArrayFx::check_non_empty_domain(const std::string& path) { write_partial_dense_array(array_name); // Open array - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Check non-empty domain @@ -3096,202 +3002,105 @@ void DenseArrayFx::check_non_empty_domain(const std::string& path) { TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, sorted reads", - "[capi][dense][sorted_reads][longtest]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][dense][sorted_reads][longtest][rest]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_sorted_reads(temp_dir); + check_sorted_reads(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, invalid number of cells in dense writes", - "[capi][dense][invalid_cell_num_writes]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif + "[capi][dense][invalid_cell_num_writes][rest]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_invalid_cell_num_in_dense_writes(temp_dir); + check_invalid_cell_num_in_dense_writes(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, sorted writes", - "[capi][dense][sorted_writes]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][dense][sorted_writes][rest]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_sorted_writes(temp_dir); + check_sorted_writes(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, simultaneous writes", - "[capi][dense][dense-simultaneous-writes]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[capi][dense][dense-simultaneous-writes][rest]") { // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_simultaneous_writes(temp_dir); + check_simultaneous_writes(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, cancel and retry writes", - "[capi][dense][async][cancel]") { - SECTION("- No serialization nor outside subarray") { - serialize_ = false; + "[capi][dense][async][cancel][non-rest]") { + SECTION("- No outside subarray") { use_external_subarray_ = false; } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization, no outside subarray") { - serialize_ = true; - use_external_subarray_ = false; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SECTION("- no serialization, with outside subarray") { - serialize_ = false; + SECTION("- Outside subarray") { use_external_subarray_ = true; } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization, with outside subarray") { - serialize_ = true; - use_external_subarray_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_cancel_and_retry_writes(temp_dir); + check_cancel_and_retry_writes(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, return coordinates", - "[capi][dense][return-coords]") { + "[capi][dense][return-coords][rest]") { bool split_coords = false; - SECTION("- No serialization") { - serialize_ = false; - - SECTION("-- zipped coordinates") { - split_coords = false; - } - - SECTION("-- split coordinates") { - split_coords = true; - } + SECTION("-- zipped coordinates") { + split_coords = false; } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - SECTION("-- zipped coordinates") { - split_coords = false; - } - - SECTION("-- split coordinates") { - split_coords = true; - } + SECTION("-- split coordinates") { + split_coords = true; } -#endif // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_return_coords(temp_dir, split_coords); + check_return_coords(split_coords); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, non-empty domain", - "[capi][dense][dense-non-empty]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); + "[capi][dense][dense-non-empty][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_non_empty_domain(temp_dir); + check_non_empty_domain(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, invalid set query buffer", - "[capi][dense][invalid_set_query_buffer]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); + "[capi][dense][invalid_set_query_buffer][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_invalid_set_query_buffer"; // Create and write dense array - std::string array_name = temp_dir + "dense_invalid_set_query_buffer"; create_dense_array(array_name); write_dense_array(array_name); @@ -3299,8 +3108,7 @@ TEST_CASE_METHOD( tiledb_array_t* array; int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Allocate query @@ -3332,27 +3140,14 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_ERR); // Check no buffers set - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_ERR); // Issue an incomplete query rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); // This is an incomplete query, do not finalize - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check that setting a new attribute for an incomplete query fails @@ -3366,10 +3161,8 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Close array - if (serialize_ == false) { - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - } + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); // Clean up tiledb_query_free(&query); @@ -3381,13 +3174,12 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, open array checks", - "[capi][dense][open-array-checks]") { - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); + "[capi][dense][open-array-checks][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_open_array"; // Create and write dense array - std::string array_name = temp_dir + "dense_open_array"; create_dense_array(array_name); // Allocate array @@ -3442,13 +3234,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_ERR); // Close array @@ -3456,13 +3242,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Submitting the query after closing an array should fail - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_ERR); // Clean up @@ -3475,13 +3255,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, reopen array checks", - "[capi][dense][dense-reopen-array-checks]") { - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); + "[capi][dense][dense-reopen-array-checks][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_open_array"; - // Create and write dense array - std::string array_name = temp_dir + "dense_reopen_array"; create_dense_array(array_name); write_dense_array(array_name); @@ -3522,13 +3300,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_subarray(ctx_, query_1, &subarray[0]); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_1, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_1); CHECK(rc == TILEDB_OK); // The read query should not see the second fragment @@ -3550,13 +3322,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_subarray(ctx_, query_2, &subarray[0]); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query_2, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query_2); CHECK(rc == TILEDB_OK); // The new query see the updated array @@ -3594,21 +3360,12 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, URI ending in a slash", - "[capi][dense][uri-ending-slash]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "with_ending_slash/"; + "[capi][dense][uri-ending-slash][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "with_ending_slash"; + + // Create and write dense array create_dense_array(array_name); write_dense_array(array_name); read_dense_array_with_coords_full_global(array_name, true); @@ -3618,21 +3375,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, missing attributes in writes", - "[capi][dense][write-missing-attributes]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_write_missing_attributes/"; + "[capi][dense][write-missing-attributes][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_write_missing_attributes"; + create_dense_array(array_name); write_dense_array_missing_attributes(array_name); remove_temp_dir(temp_dir); @@ -3641,21 +3388,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, read subarrays with empty cells", - "[capi][dense][read_empty]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_empty/"; + "[capi][dense][read_empty][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_empty"; create_dense_array_1_attribute(array_name); @@ -3679,13 +3415,7 @@ TEST_CASE_METHOD( uint64_t subarray[] = {2, 3, 1, 2}; rc = tiledb_query_set_subarray(ctx_, query, subarray); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3726,13 +3456,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3748,21 +3472,10 @@ TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, read subarrays with empty areas, merging " "adjacent cell ranges", - "[capi][dense][dense-read-empty][dense-read-empty-merge]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_empty_merge/"; + "[capi][dense][dense-read-empty][dense-read-empty-merge][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_empty_merge"; create_dense_array_1_attribute(array_name); @@ -3786,13 +3499,7 @@ TEST_CASE_METHOD( uint64_t subarray[] = {2, 3, 2, 3}; rc = tiledb_query_set_subarray(ctx_, query, subarray); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3833,13 +3540,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3854,21 +3555,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, multi-fragment reads", - "[capi][dense][dense-multi-fragment]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_multi_fragment/"; + "[capi][dense][dense-multi-fragment][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_multi_fragment/"; create_dense_array_1_attribute(array_name); @@ -3891,13 +3581,7 @@ TEST_CASE_METHOD( uint64_t subarray_1[] = {1, 2, 1, 4}; rc = tiledb_query_set_subarray(ctx_, query, subarray_1); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3921,13 +3605,7 @@ TEST_CASE_METHOD( uint64_t subarray_2[] = {2, 3, 2, 3}; rc = tiledb_query_set_subarray(ctx_, query, subarray_2); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3967,13 +3645,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", read_a, &read_a_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -3988,21 +3660,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, check if open", - "[capi][dense][dense-is-open]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_is_open/"; + "[capi][dense][dense-is-open][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_is_open/"; + create_dense_array(array_name); tiledb_array_t* array; @@ -4036,21 +3698,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, get schema from opened array", - "[capi][dense][dense-get-schema]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_get_schema/"; + "[capi][dense][dense-get-schema][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_get_schema/"; + create_dense_array(array_name); // Open array @@ -4079,22 +3731,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, read in col-major after updates", - "[capi][dense][dense-col-updates]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "dense-col-updates"; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); + "[capi][dense][dense-col-updates][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense-col-updates"; + create_dense_array_1_attribute(array_name); // ------ WRITE QUERIES ------ // @@ -4117,13 +3758,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &wq1, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, wq1); REQUIRE(rc == TILEDB_OK); // Clean up @@ -4150,13 +3785,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, wq2, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &wq2, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, wq2); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, wq2); REQUIRE(rc == TILEDB_OK); @@ -4185,13 +3814,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, wq3, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &wq3, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, wq3); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, wq3); REQUIRE(rc == TILEDB_OK); @@ -4221,13 +3844,8 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_subarray(ctx_, query, subarray_read); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -4255,28 +3873,17 @@ TEST_CASE_METHOD( // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - check_sorted_reads(temp_dir); + check_sorted_reads(); remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, splitting to unary ranges", - "[capi][dense][splitting][unary-range]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "unary-range"; + "[capi][dense][splitting][unary-range][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "unary-range"; // Create and write dense array create_dense_array(array_name); @@ -4304,8 +3911,7 @@ TEST_CASE_METHOD( rc = tiledb_array_set_config(ctx_, array, config); CHECK(rc == TILEDB_OK); - rc = array_open_wrapper( - ctx_, TILEDB_READ, (serialize_ && refactored_query_v2_), &array); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); tiledb_query_t* query; @@ -4333,14 +3939,7 @@ TEST_CASE_METHOD( tiledb_query_status_t status; int c_a1[] = {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15}; for (uint64_t i = 0; i < 16; i++) { - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_status(ctx_, query, &status); @@ -4362,21 +3961,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, subarray default dim", - "[capi][dense][default-dim]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "default-dim"; + "[capi][dense][default-dim][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "default-dim"; // Create and write dense array create_dense_array(array_name); @@ -4403,13 +3991,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; @@ -4432,21 +4014,11 @@ TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, overlapping fragments, same tile, cell order " "change", - "[capi][dense][overlapping-fragments][same-tile][cell-order-change]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_same_tile"; + "[capi][dense][overlapping-fragments][same-tile][cell-order-change][" + "rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_same_tile"; create_dense_array_same_tile(array_name); @@ -4480,13 +4052,7 @@ TEST_CASE_METHOD( uint64_t subarray[] = {1, 3, 3, 4}; rc = tiledb_query_set_subarray(ctx_, query, subarray); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_free(&query); @@ -4513,13 +4079,7 @@ TEST_CASE_METHOD( uint64_t subarray_2[] = {4, 5, 1, 4}; rc = tiledb_query_set_subarray(ctx_, query, subarray_2); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -4555,13 +4115,7 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], read_a2, &read_a2_size); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); @@ -4577,18 +4131,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, simple multi-index", - "[capi][dense][multi-index][simple]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_multi_index_simple"; + "[capi][dense][multi-index][simple][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_multi_index_simple"; create_large_dense_array_1_attribute(array_name); @@ -4634,18 +4180,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, complex multi-index", - "[capi][dense][multi-index][complex]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_multi_index_complex"; + "[capi][dense][multi-index][complex][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_multi_index_complex"; create_large_dense_array_1_attribute(array_name); @@ -4708,18 +4246,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, multi-index, cross tile boundary", - "[capi][dense][multi-index][cross-tile-boundary]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_multi_index_cross_tile"; + "[capi][dense][multi-index][cross-tile-boundary][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_multi_index_cross_tile"; create_large_dense_array_1_attribute(array_name); @@ -4765,18 +4295,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, multi-index, out of order ranges", - "[capi][dense][multi-index][out-of-order-ranges]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_multi_out_of_order"; + "[capi][dense][multi-index][out-of-order-ranges][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_multi_out_of_order"; create_large_dense_array_1_attribute(array_name); @@ -4822,18 +4344,10 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayFx, "C API: Test dense array, multi-index, out of order ranges, coalesce", - "[capi][dense][multi-index][out-of-order-ranges-2]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - std::string array_name = temp_dir + "dense_read_multi_index_coalesce"; + "[capi][dense][multi-index][out-of-order-ranges-2][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); + std::string array_name = prefix_ + "dense_read_multi_index_coalesce"; create_large_dense_array_1_attribute(array_name); diff --git a/test/src/unit-capi-dense_neg.cc b/test/src/unit-capi-dense_neg.cc index 734ab104a2b3..100d8e821cd4 100644 --- a/test/src/unit-capi-dense_neg.cc +++ b/test/src/unit-capi-dense_neg.cc @@ -55,12 +55,6 @@ struct DenseNegFx { // Vector of supported filsystems const std::vector> fs_vec_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - // Functions DenseNegFx(); ~DenseNegFx(); @@ -235,8 +229,7 @@ void DenseNegFx::write_dense_vector(const std::string& path) { rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -270,8 +263,7 @@ void DenseNegFx::write_dense_array_global(const std::string& path) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -304,8 +296,7 @@ void DenseNegFx::write_dense_array_row(const std::string& path) { rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -338,8 +329,7 @@ void DenseNegFx::write_dense_array_col(const std::string& path) { rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -372,8 +362,7 @@ void DenseNegFx::read_dense_vector(const std::string& path) { rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); int a_c[] = {0, 1, 2, 3}; @@ -410,8 +399,7 @@ void DenseNegFx::read_dense_array_global(const std::string& path) { rc = tiledb_query_set_subarray(ctx_, query, subarray); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); int a_c[] = {1, 20, 3, 40, 50, 6, 70, 8, 9, 100, 11, 120, 130, 140, 150, 160}; @@ -448,8 +436,8 @@ void DenseNegFx::read_dense_array_row(const std::string& path) { rc = tiledb_query_set_subarray(ctx_, query, subarray); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); int a_c[] = {1, 20, 50, 6, 3, 40, 70, 8, 9, 100, 130, 140, 11, 120, 150, 160}; @@ -486,8 +474,8 @@ void DenseNegFx::read_dense_array_col(const std::string& path) { rc = tiledb_query_set_subarray(ctx_, query, subarray); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); int a_c[] = {1, 3, 9, 11, 20, 40, 100, 120, 50, 70, 130, 150, 6, 8, 140, 160}; @@ -506,55 +494,35 @@ void DenseNegFx::read_dense_array_col(const std::string& path) { TEST_CASE_METHOD( DenseNegFx, "C API: Test 1d dense vector with negative domain", - "[capi][dense-neg][dense-neg-vector]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; + "[capi][dense-neg][dense-neg-vector][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "dense_neg_vector"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + vfs_array_uri(fs_vec_[0], temp_dir + "dense_neg_vector"); create_dense_vector(vector_name); write_dense_vector(vector_name); read_dense_vector(vector_name); - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( DenseNegFx, "C API: Test 2d dense array with negative domain", - "[capi][dense-neg][dense-neg-array]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "dense_neg_array"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); - - create_dense_array(vector_name); - write_dense_array_global(vector_name); - write_dense_array_row(vector_name); - write_dense_array_col(vector_name); - read_dense_array_global(vector_name); - read_dense_array_row(vector_name); - read_dense_array_col(vector_name); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][dense-neg][dense-neg-array][rest]") { + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + std::string array_name = + vfs_array_uri(fs_vec_[0], temp_dir + "dense_neg_array"); + + create_dense_array(array_name); + write_dense_array_global(array_name); + write_dense_array_row(array_name); + write_dense_array_col(array_name); + read_dense_array_global(array_name); + read_dense_array_row(array_name); + read_dense_array_col(array_name); + + remove_temp_dir(temp_dir); } diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 4c1dee4ecda0..87a4468e7782 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -415,12 +415,10 @@ TEST_CASE_METHOD( "[capi][group][metadata][rest][regression][sc-4821]") { // Create and open group in write mode // TODO: refactor for each supported FS. - std::string fs_temp_dir = fs_vec_[0]->temp_dir(); - create_temp_dir(fs_temp_dir); - std::string temp_dir = - fs_vec_[0]->is_rest() ? "tiledb://unit/" + fs_temp_dir : fs_temp_dir; + std::string temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + std::string group1_uri = vfs_array_uri(fs_vec_[0], temp_dir + "group1"); - std::string group1_uri = temp_dir + "group1"; REQUIRE(tiledb_group_create(ctx_, group1_uri.c_str()) == TILEDB_OK); tiledb_group_t* group; @@ -512,7 +510,7 @@ TEST_CASE_METHOD( rc = tiledb_group_close(ctx_, group); REQUIRE(rc == TILEDB_OK); tiledb_group_free(&group); - remove_temp_dir(fs_temp_dir); + remove_temp_dir(temp_dir); } TEST_CASE_METHOD( diff --git a/test/src/unit-capi-incomplete-2.cc b/test/src/unit-capi-incomplete-2.cc index fbe58ef12f4b..641ab8f25537 100644 --- a/test/src/unit-capi-incomplete-2.cc +++ b/test/src/unit-capi-incomplete-2.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb.h" #include @@ -49,18 +50,16 @@ struct IncompleteFx2 { const char* DENSE_ARRAY_NAME = "test_async_dense"; const char* SPARSE_ARRAY_NAME = "test_async_sparse"; + // VFS setup + VFSTestSetup vfs_test_setup_; + // TileDB context tiledb_ctx_t* ctx_; - - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + std::string sparse_array_uri_; + std::string dense_array_uri_; // Constructors/destructors IncompleteFx2(); - ~IncompleteFx2(); // Functions void create_dense_array(); @@ -77,19 +76,13 @@ struct IncompleteFx2 { void check_sparse_until_complete(); void check_sparse_unsplittable_overflow(); void check_sparse_unsplittable_complete(); - void remove_dense_array(); - void remove_sparse_array(); - void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); }; -IncompleteFx2::IncompleteFx2() { - ctx_ = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx_) == TILEDB_OK); -} - -IncompleteFx2::~IncompleteFx2() { - tiledb_ctx_free(&ctx_); +IncompleteFx2::IncompleteFx2() + : ctx_(vfs_test_setup_.ctx_c) + , sparse_array_uri_(vfs_test_setup_.array_uri(SPARSE_ARRAY_NAME)) + , dense_array_uri_(vfs_test_setup_.array_uri(DENSE_ARRAY_NAME)) { } void IncompleteFx2::create_dense_array() { @@ -161,7 +154,7 @@ void IncompleteFx2::create_dense_array() { CHECK(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx_, DENSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, dense_array_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -243,7 +236,7 @@ void IncompleteFx2::create_sparse_array() { CHECK(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx_, SPARSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, sparse_array_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -293,7 +286,7 @@ void IncompleteFx2::write_dense_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -318,13 +311,7 @@ void IncompleteFx2::write_dense_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -377,7 +364,7 @@ void IncompleteFx2::write_sparse_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -409,13 +396,7 @@ void IncompleteFx2::write_sparse_full() { CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -427,21 +408,6 @@ void IncompleteFx2::write_sparse_full() { tiledb_query_free(&query); } -void IncompleteFx2::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; - - CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); -} - -void IncompleteFx2::remove_dense_array() { - remove_array(DENSE_ARRAY_NAME); -} - -void IncompleteFx2::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); -} - bool IncompleteFx2::is_array(const std::string& array_name) { tiledb_object_t type = TILEDB_INVALID; REQUIRE(tiledb_object_type(ctx_, array_name.c_str(), &type) == TILEDB_OK); @@ -462,7 +428,7 @@ void IncompleteFx2::check_dense_incomplete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -479,14 +445,7 @@ void IncompleteFx2::check_dense_incomplete() { rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -496,7 +455,7 @@ void IncompleteFx2::check_dense_incomplete() { CHECK(status == TILEDB_INCOMPLETE); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -527,7 +486,7 @@ void IncompleteFx2::check_dense_until_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -545,14 +504,7 @@ void IncompleteFx2::check_dense_until_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check buffer @@ -567,14 +519,7 @@ void IncompleteFx2::check_dense_until_complete() { CHECK(status == TILEDB_INCOMPLETE); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -588,7 +533,7 @@ void IncompleteFx2::check_dense_until_complete() { CHECK(buffer_sizes[0] == 2 * sizeof(int)); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -614,7 +559,7 @@ void IncompleteFx2::check_dense_shrink_buffer_size() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -632,14 +577,7 @@ void IncompleteFx2::check_dense_shrink_buffer_size() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check buffer @@ -660,14 +598,7 @@ void IncompleteFx2::check_dense_shrink_buffer_size() { CHECK(rc == TILEDB_OK); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -680,7 +611,7 @@ void IncompleteFx2::check_dense_shrink_buffer_size() { CHECK(buffer_a1[0] == 2); // Free/finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -707,7 +638,7 @@ void IncompleteFx2::check_dense_unsplittable_overflow() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -728,14 +659,7 @@ void IncompleteFx2::check_dense_unsplittable_overflow() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Get status @@ -748,7 +672,7 @@ void IncompleteFx2::check_dense_unsplittable_overflow() { CHECK(buffer_sizes[1] == 0); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -775,7 +699,7 @@ void IncompleteFx2::check_dense_unsplittable_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -796,14 +720,8 @@ void IncompleteFx2::check_dense_unsplittable_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); // Check buffers @@ -811,7 +729,7 @@ void IncompleteFx2::check_dense_unsplittable_complete() { CHECK(!memcmp(buffer_a2_var, c_buffer_a2_var, sizeof(c_buffer_a2_var))); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -837,7 +755,7 @@ void IncompleteFx2::check_dense_reset_buffers() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -855,14 +773,8 @@ void IncompleteFx2::check_dense_reset_buffers() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); // Check buffer @@ -877,14 +789,8 @@ void IncompleteFx2::check_dense_reset_buffers() { CHECK(status == TILEDB_INCOMPLETE); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); // Check status @@ -898,7 +804,7 @@ void IncompleteFx2::check_dense_reset_buffers() { CHECK(buffer_sizes[0] == 2 * sizeof(int)); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -921,7 +827,7 @@ void IncompleteFx2::check_sparse_incomplete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -945,14 +851,7 @@ void IncompleteFx2::check_sparse_incomplete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -986,7 +885,7 @@ void IncompleteFx2::check_sparse_until_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1010,14 +909,7 @@ void IncompleteFx2::check_sparse_until_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -1032,14 +924,7 @@ void IncompleteFx2::check_sparse_until_complete() { CHECK(buffer_sizes[0] == sizeof(int)); // Resubmit the query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -1053,14 +938,7 @@ void IncompleteFx2::check_sparse_until_complete() { CHECK(buffer_sizes[0] == sizeof(int)); // Resubmit the query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check status @@ -1092,7 +970,7 @@ void IncompleteFx2::check_sparse_unsplittable_overflow() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1119,14 +997,8 @@ void IncompleteFx2::check_sparse_unsplittable_overflow() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -1135,7 +1007,7 @@ void IncompleteFx2::check_sparse_unsplittable_overflow() { CHECK(buffer_sizes[0] == 0); // Finalize query - rc = finalize_query_wrapper(ctx_, SPARSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -1159,7 +1031,7 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1186,14 +1058,7 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check buffers @@ -1201,7 +1066,7 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { CHECK(!memcmp(buffer_a2_var, c_buffer_a2_var, sizeof(c_buffer_a2_var))); // Finalize query - rc = finalize_query_wrapper(ctx_, SPARSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -1216,18 +1081,7 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { TEST_CASE_METHOD( IncompleteFx2, "C API: Test incomplete read queries 2, dense", - "[capi][incomplete-2][dense-incomplete-2]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_dense_array(); + "[capi][incomplete-2][dense-incomplete-2][rest]") { create_dense_array(); write_dense_full(); check_dense_incomplete(); @@ -1236,29 +1090,16 @@ TEST_CASE_METHOD( check_dense_unsplittable_overflow(); check_dense_unsplittable_complete(); check_dense_reset_buffers(); - remove_dense_array(); } TEST_CASE_METHOD( IncompleteFx2, "C API: Test incomplete read queries 2, sparse", - "[capi][incomplete-2][sparse-incomplete-2]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_sparse_array(); + "[capi][incomplete-2][sparse-incomplete-2][rest-fails][sc-42873]") { create_sparse_array(); write_sparse_full(); check_sparse_incomplete(); check_sparse_until_complete(); check_sparse_unsplittable_overflow(); check_sparse_unsplittable_complete(); - remove_sparse_array(); } diff --git a/test/src/unit-capi-incomplete.cc b/test/src/unit-capi-incomplete.cc index ae7807360038..2d4d6dc8d151 100644 --- a/test/src/unit-capi-incomplete.cc +++ b/test/src/unit-capi-incomplete.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/serialization/query.h" @@ -51,18 +52,16 @@ struct IncompleteFx { const char* DENSE_ARRAY_NAME = "test_async_dense"; const char* SPARSE_ARRAY_NAME = "test_async_sparse"; + // VFS setup + VFSTestSetup vfs_test_setup_; + // TileDB context tiledb_ctx_t* ctx_; - - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + std::string sparse_array_uri_; + std::string dense_array_uri_; // Constructors/destructors IncompleteFx(); - ~IncompleteFx(); // Functions void create_dense_array(); @@ -79,19 +78,13 @@ struct IncompleteFx { void check_sparse_until_complete(); void check_sparse_unsplittable_overflow(); void check_sparse_unsplittable_complete(); - void remove_dense_array(); - void remove_sparse_array(); - void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); }; -IncompleteFx::IncompleteFx() { - ctx_ = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx_) == TILEDB_OK); -} - -IncompleteFx::~IncompleteFx() { - tiledb_ctx_free(&ctx_); +IncompleteFx::IncompleteFx() + : ctx_(vfs_test_setup_.ctx_c) + , sparse_array_uri_(vfs_test_setup_.array_uri(SPARSE_ARRAY_NAME)) + , dense_array_uri_(vfs_test_setup_.array_uri(DENSE_ARRAY_NAME)) { } void IncompleteFx::create_dense_array() { @@ -163,7 +156,7 @@ void IncompleteFx::create_dense_array() { CHECK(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx_, DENSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, dense_array_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -245,7 +238,7 @@ void IncompleteFx::create_sparse_array() { CHECK(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx_, SPARSE_ARRAY_NAME, array_schema); + rc = tiledb_array_create(ctx_, sparse_array_uri_.c_str(), array_schema); CHECK(rc == TILEDB_OK); // Clean up @@ -295,7 +288,7 @@ void IncompleteFx::write_dense_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -319,13 +312,7 @@ void IncompleteFx::write_dense_full() { ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -378,7 +365,7 @@ void IncompleteFx::write_sparse_full() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -409,14 +396,8 @@ void IncompleteFx::write_sparse_full() { ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - CHECK(rc == TILEDB_OK); + rc = tiledb_query_submit_and_finalize(ctx_, query); + REQUIRE(rc == TILEDB_OK); // Close array rc = tiledb_array_close(ctx_, array); @@ -427,21 +408,6 @@ void IncompleteFx::write_sparse_full() { tiledb_query_free(&query); } -void IncompleteFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; - - CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); -} - -void IncompleteFx::remove_dense_array() { - remove_array(DENSE_ARRAY_NAME); -} - -void IncompleteFx::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); -} - bool IncompleteFx::is_array(const std::string& array_name) { tiledb_object_t type = TILEDB_INVALID; REQUIRE(tiledb_object_type(ctx_, array_name.c_str(), &type) == TILEDB_OK); @@ -462,7 +428,7 @@ void IncompleteFx::check_dense_incomplete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -480,14 +446,7 @@ void IncompleteFx::check_dense_incomplete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -497,7 +456,7 @@ void IncompleteFx::check_dense_incomplete() { CHECK(status == TILEDB_INCOMPLETE); // Free/finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -528,7 +487,7 @@ void IncompleteFx::check_dense_until_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -546,14 +505,7 @@ void IncompleteFx::check_dense_until_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffer @@ -568,14 +520,7 @@ void IncompleteFx::check_dense_until_complete() { CHECK(status == TILEDB_INCOMPLETE); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -589,7 +534,7 @@ void IncompleteFx::check_dense_until_complete() { CHECK(buffer_sizes[0] == 2 * sizeof(int)); // Free/finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -615,7 +560,7 @@ void IncompleteFx::check_dense_shrink_buffer_size() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -633,14 +578,7 @@ void IncompleteFx::check_dense_shrink_buffer_size() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffer @@ -661,14 +599,7 @@ void IncompleteFx::check_dense_shrink_buffer_size() { CHECK(rc == TILEDB_OK); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -681,7 +612,7 @@ void IncompleteFx::check_dense_shrink_buffer_size() { CHECK(buffer_a1[0] == 2); // Free/finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -708,7 +639,7 @@ void IncompleteFx::check_dense_unsplittable_overflow() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -729,14 +660,7 @@ void IncompleteFx::check_dense_unsplittable_overflow() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Get status @@ -749,7 +673,7 @@ void IncompleteFx::check_dense_unsplittable_overflow() { CHECK(buffer_sizes[1] == 0); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -776,7 +700,7 @@ void IncompleteFx::check_dense_unsplittable_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -797,14 +721,7 @@ void IncompleteFx::check_dense_unsplittable_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -812,7 +729,7 @@ void IncompleteFx::check_dense_unsplittable_complete() { CHECK(!memcmp(buffer_a2_var, c_buffer_a2_var, sizeof(c_buffer_a2_var))); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -838,7 +755,7 @@ void IncompleteFx::check_dense_reset_buffers() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, dense_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -856,14 +773,7 @@ void IncompleteFx::check_dense_reset_buffers() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffer @@ -878,14 +788,7 @@ void IncompleteFx::check_dense_reset_buffers() { CHECK(status == TILEDB_INCOMPLETE); // Resubmit query - rc = submit_query_wrapper( - ctx_, - DENSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -899,7 +802,7 @@ void IncompleteFx::check_dense_reset_buffers() { CHECK(buffer_sizes[0] == 2 * sizeof(int)); // Finalize query - rc = finalize_query_wrapper(ctx_, DENSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -925,7 +828,7 @@ void IncompleteFx::check_sparse_incomplete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -943,14 +846,7 @@ void IncompleteFx::check_sparse_incomplete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -960,7 +856,7 @@ void IncompleteFx::check_sparse_incomplete() { CHECK(status == TILEDB_INCOMPLETE); // Finalize query - rc = finalize_query_wrapper(ctx_, SPARSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -991,7 +887,7 @@ void IncompleteFx::check_sparse_until_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1009,14 +905,7 @@ void IncompleteFx::check_sparse_until_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -1031,14 +920,7 @@ void IncompleteFx::check_sparse_until_complete() { CHECK(buffer_sizes[0] == sizeof(int)); // Resubmit the query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -1060,14 +942,7 @@ void IncompleteFx::check_sparse_until_complete() { */ if (!use_refactored_sparse_global_order_reader()) { // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check status @@ -1103,7 +978,7 @@ void IncompleteFx::check_sparse_unsplittable_overflow() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1125,14 +1000,7 @@ void IncompleteFx::check_sparse_unsplittable_overflow() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -1140,19 +1008,13 @@ void IncompleteFx::check_sparse_unsplittable_overflow() { CHECK(status == TILEDB_INCOMPLETE); CHECK(buffer_sizes[0] == 0); - // For remote arrays the reason is always TILEDB_REASON_USER_BUFFER_SIZE, but - // we can't test it here since we simulate "remote" arrays by using a local - // URI so the array->is_remote() check will fail, and we won't get the - // correct result. - if (!serialize_) { - tiledb_query_status_details_t details; - rc = tiledb_query_get_status_details(ctx_, query, &details); - CHECK(rc == TILEDB_OK); - CHECK(details.incomplete_reason == TILEDB_REASON_USER_BUFFER_SIZE); - } + tiledb_query_status_details_t details; + rc = tiledb_query_get_status_details(ctx_, query, &details); + CHECK(rc == TILEDB_OK); + CHECK(details.incomplete_reason == TILEDB_REASON_USER_BUFFER_SIZE); // Finalize query - rc = finalize_query_wrapper(ctx_, SPARSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -1179,7 +1041,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() { // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); + int rc = tiledb_array_alloc(ctx_, sparse_array_uri_.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -1200,14 +1062,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Check buffers @@ -1215,7 +1070,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() { CHECK(!memcmp(buffer_a2_var, c_buffer_a2_var, sizeof(c_buffer_a2_var))); // Finalize query - rc = finalize_query_wrapper(ctx_, SPARSE_ARRAY_NAME, &query, serialize_); + rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -1230,18 +1085,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() { TEST_CASE_METHOD( IncompleteFx, "C API: Test incomplete read queries, dense", - "[capi][incomplete][dense-incomplete][serialization]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_dense_array(); + "[capi][incomplete][dense-incomplete][serialization][rest]") { create_dense_array(); write_dense_full(); check_dense_incomplete(); @@ -1250,54 +1094,25 @@ TEST_CASE_METHOD( check_dense_unsplittable_overflow(); check_dense_unsplittable_complete(); check_dense_reset_buffers(); - remove_dense_array(); } TEST_CASE_METHOD( IncompleteFx, "C API: Test incomplete read queries, sparse", - "[capi][incomplete][sparse][serialization]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_sparse_array(); + "[capi][incomplete][sparse][serialization][rest]") { create_sparse_array(); write_sparse_full(); check_sparse_incomplete(); check_sparse_until_complete(); check_sparse_unsplittable_overflow(); check_sparse_unsplittable_complete(); - remove_sparse_array(); } -#ifdef TILEDB_SERIALIZATION - TEST_CASE_METHOD( IncompleteFx, "C API: Test incomplete read queries, dense, serialized", - "[capi][incomplete][dense][serialization]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_dense_array(); + "[capi][incomplete][dense][serialization][rest]") { create_dense_array(); write_dense_full(); check_dense_incomplete(); - remove_dense_array(); } - -#endif diff --git a/test/src/unit-capi-nullable.cc b/test/src/unit-capi-nullable.cc index bd93abc576ac..0be84fe640eb 100644 --- a/test/src/unit-capi-nullable.cc +++ b/test/src/unit-capi-nullable.cc @@ -33,6 +33,7 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/temporary_local_directory.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/vfs/vfs_api_internal.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/array_type.h" @@ -48,12 +49,6 @@ using namespace tiledb::test; class NullableArrayFx { public: - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - struct test_dim_t { test_dim_t( const string& name, @@ -200,8 +195,7 @@ class NullableArrayFx { const void* subarray); }; -NullableArrayFx::NullableArrayFx() - : serialize_(false) { +NullableArrayFx::NullableArrayFx() { // Create a config. tiledb_config_t* config = nullptr; tiledb_error_t* error = nullptr; @@ -224,7 +218,7 @@ NullableArrayFx::~NullableArrayFx() { } const string NullableArrayFx::array_path(const string& array_name) { - return temp_dir_.path() + array_name; + return vfs_array_uri(vfs_test_get_fs_vec()[0], temp_dir_.path() + array_name); } void NullableArrayFx::create_array( @@ -410,13 +404,11 @@ void NullableArrayFx::write( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_path(array_name), - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + if (layout != TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit(ctx_, query); + } else { + rc = tiledb_query_submit_and_finalize(ctx_, query); + } REQUIRE(rc == TILEDB_OK); // Clean up @@ -517,13 +509,7 @@ void NullableArrayFx::read( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_path(array_name), - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Clean up @@ -848,20 +834,12 @@ void NullableArrayFx::do_2d_nullable_test( } } +// TODO: Add [rest] tag and fix test issues with cleanup +// because of the use of dynamic section TEST_CASE_METHOD( NullableArrayFx, "C API: Test 2D array with nullable attributes", "[capi][2d][nullable]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Define the attributes. vector attrs; attrs.emplace_back("a1", TILEDB_INT32, 1, true); diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index 553cec876881..f985aeaa4954 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -1179,7 +1179,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, simultaneous writes", - "[capi][rest][dense][dense-simultaneous-writes]") { + "[capi][non-rest][dense][dense-simultaneous-writes]") { + // Parallel array open requests on the same array fail on + // remote arrays, as locking cannot work on Cloud REST + // server to synchronize them, so we exclude this test + // from REST-CI check_simultaneous_writes(); } diff --git a/test/src/unit-capi-serialized_queries.cc b/test/src/unit-capi-serialized_queries.cc index be67e8187035..a91a5ef68a80 100644 --- a/test/src/unit-capi-serialized_queries.cc +++ b/test/src/unit-capi-serialized_queries.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_serialization.h" @@ -54,23 +55,8 @@ using namespace tiledb; using ResultSetType = std::map; -using tiledb::test::ServerQueryBuffers; -using tiledb::test::submit_query_wrapper; - namespace { -#ifdef _WIN32 -static const char PATH_SEPARATOR = '\\'; -static std::string current_dir() { - return sm::Win::current_dir(); -} -#else -static const char PATH_SEPARATOR = '/'; -static std::string current_dir() { - return sm::Posix::current_dir(); -} -#endif - template bool check_result(const T a, const T b, size_t start, size_t end) { auto a_exp = T(a.begin() + start, a.begin() + end); @@ -99,34 +85,15 @@ bool check_result( } struct SerializationFx { - const std::string tmpdir = "serialization_test_dir"; - const std::string array_name = "testarray"; - const std::string array_uri = - current_dir() + PATH_SEPARATOR + tmpdir + "/" + array_name; - + test::VFSTestSetup vfs_test_setup_; + tiledb_ctx_t* ctx_; Context ctx; - VFS vfs; - - // Serialization parameters - bool serialize_ = true; - bool refactored_query_v2_ = false; - bool finalize_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; + const std::string array_uri; SerializationFx() - : vfs(ctx) { - if (vfs.is_dir(tmpdir)) - vfs.remove_dir(tmpdir); - vfs.create_dir(tmpdir); - if (!vfs.is_dir(tmpdir)) - std::cerr << "'created' but not finding dir '" << tmpdir << "'" - << std::endl; - } - - ~SerializationFx() { - if (vfs.is_dir(tmpdir)) - vfs.remove_dir(tmpdir); + : ctx_{vfs_test_setup_.ctx_c} + , ctx{vfs_test_setup_.ctx()} + , array_uri{vfs_test_setup_.array_uri("testarray")} { } static void check_read_stats(const Query& query) { @@ -218,15 +185,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the write stats check_write_stats(query); @@ -268,23 +227,15 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the write stats check_write_stats(query); } void write_sparse_array() { - std::vector coords = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, - 6, 6, 7, 7, 8, 8, 9, 9, 10, 10}; + std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::vector d2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; std::vector a1; std::vector a2; std::vector a2_nullable; @@ -308,7 +259,8 @@ struct SerializationFx { Array array(ctx, array_uri, TILEDB_WRITE); Query query(ctx, array); query.set_layout(TILEDB_UNORDERED); - query.set_coordinates(coords); + query.set_data_buffer("d1", d1); + query.set_data_buffer("d2", d2); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -316,15 +268,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the write stats check_write_stats(query); @@ -341,15 +285,7 @@ struct SerializationFx { query.set_condition(qc); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the delete stats check_delete_stats(query); @@ -392,15 +328,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the write stats check_write_stats(query); @@ -412,8 +340,7 @@ struct SerializationFx { TEST_CASE_METHOD( SerializationFx, "Query serialization, dense", - "[query][dense][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][dense][serialization][rest]") { create_array(TILEDB_DENSE); auto expected_results = write_dense_array(); @@ -435,15 +362,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -485,15 +404,7 @@ TEST_CASE_METHOD( query.set_condition(condition); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -551,15 +462,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -607,15 +510,7 @@ TEST_CASE_METHOD( // Submit initial query. set_buffers(query); - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); @@ -629,15 +524,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -653,15 +540,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -680,15 +559,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "Query serialization, sparse", - "[query][sparse][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][sparse][serialization][rest]") { create_array(TILEDB_SPARSE); write_sparse_array(); SECTION("- Read all") { Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); - std::vector coords(1000); std::vector a1(1000); std::vector a2(1000); std::vector a2_nullable(1000); @@ -697,7 +574,6 @@ TEST_CASE_METHOD( std::vector subarray = {1, 10, 1, 10}; query.set_subarray(subarray); - query.set_coordinates(coords); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -705,15 +581,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -732,20 +600,21 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "Query serialization, sparse, old client", - "[query][sparse][serialization][old-client]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][sparse][serialization][old-client][rest]") { create_array(TILEDB_SPARSE); write_sparse_array(); Config config; config.set("sm.query.sparse_global_order.reader", "legacy"); config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); - auto ctx_client = Context(config); + + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx_c; + auto ctx_client = vfs_test_setup_.ctx(); SECTION("- Read all") { Array array(ctx_client, array_uri, TILEDB_READ); Query query(ctx_client, array); - std::vector coords(1000); std::vector a1(1000); std::vector a2(1000); std::vector a2_nullable(1000); @@ -755,7 +624,6 @@ TEST_CASE_METHOD( query.set_layout(TILEDB_GLOBAL_ORDER); query.set_subarray(subarray); - query.set_coordinates(coords); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -763,15 +631,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -791,7 +651,6 @@ TEST_CASE_METHOD( SerializationFx, "Query serialization, split coords, sparse", "[query][sparse][serialization][split-coords]") { - refactored_query_v2_ = GENERATE(true, false); create_array(TILEDB_SPARSE); write_sparse_array_split_coords(); @@ -815,15 +674,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -843,8 +694,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "Query serialization, dense ranges", - "[query][dense][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][dense][serialization][rest]") { create_array(TILEDB_DENSE); write_dense_array_ranges(); @@ -867,15 +717,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -909,15 +751,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -955,15 +789,7 @@ TEST_CASE_METHOD( // Submit initial query. set_buffers(query); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); @@ -978,15 +804,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); // Submit query - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -1001,15 +819,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); // Submit query - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -1028,8 +838,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "Query serialization, sparse delete", - "[query][sparse][delete][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][sparse][delete][serialization][rest]") { create_array(TILEDB_SPARSE); write_sparse_array(); write_sparse_delete(); @@ -1037,7 +846,6 @@ TEST_CASE_METHOD( SECTION("- Read all") { Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); - std::vector coords(1000); std::vector a1(1000); std::vector a2(1000); std::vector a2_nullable(1000); @@ -1046,7 +854,6 @@ TEST_CASE_METHOD( std::vector subarray = {1, 10, 1, 10}; query.set_subarray(subarray); - query.set_coordinates(coords); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -1054,15 +861,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -1081,17 +880,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "Global order writes serialization", - "[global-order-write][serialization][dense]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[global-order-write][serialization][dense][rest]") { uint64_t tile_extent = 2; ArraySchema schema(ctx, TILEDB_DENSE); Domain domain(ctx); @@ -1150,29 +939,13 @@ TEST_CASE_METHOD( begin += chunk_size; end = std::min(last_space_tile, end + chunk_size); - // Simulate REST submit() if (begin < end) { - submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + query.submit(); } } // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); - + query.submit_and_finalize(); REQUIRE(query.query_status() == Query::Status::COMPLETE); // Read and validate results @@ -1195,14 +968,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer( "a3", a3_result_offsets.data(), a3_result_offsets.size()); - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); for (uint64_t i = 0; i < ncells; ++i) { @@ -1224,13 +990,7 @@ TEST_CASE_METHOD( SerializationFx, "Derialization of a var size read query correctly resets original buffer " "sizes", - "[capi][query][serialization][reset-buffers]") { -#ifndef TILEDB_SERIALIZATION - // Only makes sense in serialization - return; -#endif - refactored_query_v2_ = GENERATE(true, false); - + "[capi][query][serialization][reset-buffers][rest]") { create_array(TILEDB_SPARSE); write_sparse_array(); @@ -1251,16 +1011,7 @@ TEST_CASE_METHOD( set_buffers(); Query::Status status; do { - int rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - true, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); - + query.submit(); status = query.query_status(); } while (status == Query::Status::INCOMPLETE); diff --git a/test/src/unit-capi-serialized_queries_using_subarray.cc b/test/src/unit-capi-serialized_queries_using_subarray.cc index a8d6499212bc..cda100807327 100644 --- a/test/src/unit-capi-serialized_queries_using_subarray.cc +++ b/test/src/unit-capi-serialized_queries_using_subarray.cc @@ -35,6 +35,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" @@ -55,23 +56,8 @@ using namespace tiledb; using ResultSetType = std::map; -using tiledb::test::ServerQueryBuffers; -using tiledb::test::submit_query_wrapper; - namespace { -#ifdef _WIN32 -static const char PATH_SEPARATOR = '\\'; -static std::string current_dir() { - return sm::Win::current_dir(); -} -#else -static const char PATH_SEPARATOR = '/'; -static std::string current_dir() { - return sm::Posix::current_dir(); -} -#endif - template bool check_result(const T a, const T b, size_t start, size_t end) { auto a_exp = T(a.begin() + start, a.begin() + end); @@ -100,34 +86,15 @@ bool check_result( } struct SerializationFx { - const std::string tmpdir = "serialization_test_dir"; - const std::string array_name = "testarray"; - const std::string array_uri = - current_dir() + PATH_SEPARATOR + tmpdir + "/" + array_name; - + test::VFSTestSetup vfs_test_setup_; + tiledb_ctx_t* ctx_; Context ctx; - VFS vfs; - - // Serialization parameters - bool serialize_ = true; - bool refactored_query_v2_ = false; - bool finalize_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; + const std::string array_uri; SerializationFx() - : vfs(ctx) { - if (vfs.is_dir(tmpdir)) - vfs.remove_dir(tmpdir); - vfs.create_dir(tmpdir); - if (!vfs.is_dir(tmpdir)) - std::cerr << "'created' but not finding dir '" << tmpdir << "'" - << std::endl; - } - - ~SerializationFx() { - if (vfs.is_dir(tmpdir)) - vfs.remove_dir(tmpdir); + : ctx_{vfs_test_setup_.ctx_c} + , ctx{vfs_test_setup_.ctx()} + , array_uri{vfs_test_setup_.array_uri("testarray")} { } static void check_read_stats(const Query& query) { @@ -226,15 +193,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); // The deserialized query should also include the write stats check_write_stats(query); @@ -278,20 +237,12 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); } void write_sparse_array() { - std::vector coords = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, - 6, 6, 7, 7, 8, 8, 9, 9, 10, 10}; + std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::vector d2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; std::vector a1; std::vector a2; std::vector a2_nullable; @@ -315,7 +266,8 @@ struct SerializationFx { Array array(ctx, array_uri, TILEDB_WRITE); Query query(ctx, array); query.set_layout(TILEDB_UNORDERED); - query.set_coordinates(coords); + query.set_data_buffer("d1", d1); + query.set_data_buffer("d2", d2); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -323,15 +275,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); } void write_sparse_array_split_coords() { @@ -371,15 +315,7 @@ struct SerializationFx { query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); } }; @@ -388,8 +324,12 @@ struct SerializationFx { TEST_CASE_METHOD( SerializationFx, "subarray - Query serialization, dense", - "[query][dense][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][dense][serialization][rest][sc-40489]") { + if (!vfs_test_setup_.is_rest()) { + SUCCEED("sc-40489: this test passes on remote arrays only"); + return; + } + create_array(TILEDB_DENSE); auto expected_results = write_dense_array(); check_subarray_stats(2, 2); @@ -417,15 +357,7 @@ TEST_CASE_METHOD( check_subarray_stats(3, 3); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); // Check stats after serialization @@ -465,15 +397,7 @@ TEST_CASE_METHOD( query.set_condition(condition); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the write stats @@ -535,15 +459,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); check_subarray_stats(5, 5); @@ -578,15 +494,7 @@ TEST_CASE_METHOD( // Submit initial query. set_buffers(query); - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); check_subarray_stats(5, 5); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); @@ -599,15 +507,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); check_subarray_stats(7, 7); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); @@ -620,15 +520,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); check_subarray_stats(9, 9); REQUIRE(query.query_status() == Query::Status::COMPLETE); @@ -644,15 +536,13 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "subarray - Query serialization, sparse", - "[query][sparse][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][sparse][serialization][rest]") { create_array(TILEDB_SPARSE); write_sparse_array(); SECTION("- Read all") { Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); - std::vector coords(1000); std::vector a1(1000); std::vector a2(1000); std::vector a2_nullable(1000); @@ -664,7 +554,6 @@ TEST_CASE_METHOD( cppapi_subarray.set_subarray(subarray); query.set_subarray(cppapi_subarray); - query.set_coordinates(coords); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -672,15 +561,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); auto result_el = query.result_buffer_elements_nullable(); @@ -696,7 +577,6 @@ TEST_CASE_METHOD( SerializationFx, "subarray - Query serialization, split coords, sparse", "[query][sparse][serialization][split-coords]") { - refactored_query_v2_ = GENERATE(true, false); create_array(TILEDB_SPARSE); write_sparse_array_split_coords(); @@ -722,15 +602,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); auto result_el = query.result_buffer_elements_nullable(); @@ -746,8 +618,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SerializationFx, "subarray - Query serialization, dense ranges", - "[query][dense][serialization]") { - refactored_query_v2_ = GENERATE(true, false); + "[query][dense][serialization][rest]") { create_array(TILEDB_DENSE); write_dense_array_ranges(); @@ -772,15 +643,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); auto result_el = query.result_buffer_elements_nullable(); @@ -812,15 +675,7 @@ TEST_CASE_METHOD( query.set_offsets_buffer("a3", a3_offsets); // Submit query - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); auto result_el = query.result_buffer_elements_nullable(); @@ -855,15 +710,7 @@ TEST_CASE_METHOD( // Submit initial query. set_buffers(query); - auto rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); auto result_el = query.result_buffer_elements_nullable(); @@ -876,15 +723,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); // Submit query - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::INCOMPLETE); result_el = query.result_buffer_elements_nullable(); @@ -897,15 +736,7 @@ TEST_CASE_METHOD( // Reset buffers, serialize and resubmit set_buffers(query); // Submit query - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - finalize_); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); result_el = query.result_buffer_elements_nullable(); diff --git a/test/src/unit-capi-sparse_array.cc b/test/src/unit-capi-sparse_array.cc index 2766031cb88b..4bfc283727d6 100644 --- a/test/src/unit-capi-sparse_array.cc +++ b/test/src/unit-capi-sparse_array.cc @@ -86,12 +86,8 @@ struct SparseArrayFx { // Vector of supported filsystems const std::vector> fs_vec_; - - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Functions SparseArrayFx(); @@ -118,7 +114,7 @@ struct SparseArrayFx { void check_sparse_array_global_with_all_duplicates_dedup( const std::string& array_name); void check_sparse_array_no_results(const std::string& array_name); - void check_non_empty_domain(const std::string& path); + void check_non_empty_domain(const std::string& array_name); void check_invalid_offsets(const std::string& array_name); void write_partial_sparse_array(const std::string& array_name); void write_sparse_array_missing_attributes(const std::string& array_name); @@ -138,7 +134,6 @@ struct SparseArrayFx { tiledb_filter_type_t compressor, tiledb_layout_t tile_order, tiledb_layout_t cell_order); - void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); /** @@ -223,6 +218,8 @@ SparseArrayFx::SparseArrayFx() for (const auto& fs : fs_vec_) create_temp_dir(fs->temp_dir()); + + prefix_ = vfs_array_uri(fs_vec_[0], fs_vec_[0]->temp_dir()); } SparseArrayFx::~SparseArrayFx() { @@ -235,19 +232,6 @@ SparseArrayFx::~SparseArrayFx() { tiledb_ctx_free(&ctx_); } -bool SparseArrayFx::is_array(const std::string& array_name) { - tiledb_object_t type = TILEDB_INVALID; - REQUIRE(tiledb_object_type(ctx_, array_name.c_str(), &type) == TILEDB_OK); - return type == TILEDB_ARRAY; -} - -void SparseArrayFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; - - CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); -} - void SparseArrayFx::create_temp_dir(const std::string& path) { remove_temp_dir(path); REQUIRE(tiledb_vfs_create_dir(ctx_, vfs_, path.c_str()) == TILEDB_OK); @@ -982,9 +966,11 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_no_check( tiledb_config_set(config, "sm.check_coord_dups", "false", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers @@ -1026,51 +1012,50 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_no_check( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], buffers[2], &buffer_sizes[2]); + ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit query - this is unsafe but it should be passing - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); } void SparseArrayFx::check_sparse_array_unordered_with_duplicates_dedup( @@ -1084,9 +1069,11 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_dedup( tiledb_config_set(config, "sm.dedup_coords", "true", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers for WRITE @@ -1128,45 +1115,45 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_dedup( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], buffers[2], &buffer_sizes[2]); + ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit WRITE query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1194,37 +1181,37 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_dedup( sizeof(r_buffer_coords_dim1)}; // Open array - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create READ query - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); + ctx_, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); + ctx_, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); + ctx_, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); + ctx_, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); + ctx_, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit READ query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Check correctness @@ -1262,13 +1249,12 @@ void SparseArrayFx::check_sparse_array_unordered_with_duplicates_dedup( sizeof(c_buffer_coords_dim2))); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); } void SparseArrayFx::check_sparse_array_unordered_with_all_duplicates_dedup( @@ -1282,9 +1268,11 @@ void SparseArrayFx::check_sparse_array_unordered_with_all_duplicates_dedup( tiledb_config_set(config, "sm.dedup_coords", "true", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers for WRITE @@ -1326,45 +1314,45 @@ void SparseArrayFx::check_sparse_array_unordered_with_all_duplicates_dedup( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], buffers[2], &buffer_sizes[2]); + ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit WRITE query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1392,41 +1380,41 @@ void SparseArrayFx::check_sparse_array_unordered_with_all_duplicates_dedup( sizeof(r_buffer_coords_dim1)}; // Open array - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create READ query - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); + ctx_, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); + ctx_, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); + ctx_, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); + ctx_, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); + ctx_, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); // Submit READ query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1452,8 +1440,6 @@ void SparseArrayFx::check_sparse_array_unordered_with_all_duplicates_dedup( r_buffer_coords_dim2, c_buffer_coords_dim2, sizeof(c_buffer_coords_dim2))); - - tiledb_ctx_free(&ctx); } void SparseArrayFx::check_sparse_array_global_with_duplicates_error( @@ -1529,10 +1515,8 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_error( CHECK(rc == TILEDB_OK); // Submit query - rc = tiledb_query_submit(ctx_, query); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); // Close array rc = tiledb_array_close(ctx_, array); @@ -1554,9 +1538,11 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_no_check( tiledb_config_set(config, "sm.check_coord_dups", "false", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers @@ -1598,20 +1584,20 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_no_check( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); @@ -1620,21 +1606,21 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_no_check( ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit query - this is unsafe but it should be passing - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Finalize query - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -1644,7 +1630,6 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_no_check( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); } void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( @@ -1658,9 +1643,11 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( tiledb_config_set(config, "sm.dedup_coords", "true", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers for WRITE @@ -1702,20 +1689,20 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); @@ -1724,23 +1711,23 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit WRITE query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1768,14 +1755,14 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( sizeof(r_buffer_coords_dim1)}; // Open array - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create READ query - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); + ctx_, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); @@ -1784,27 +1771,27 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( ctx_, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); + ctx_, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); + ctx_, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); + ctx_, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit READ query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Finalize query - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1845,8 +1832,6 @@ void SparseArrayFx::check_sparse_array_global_with_duplicates_dedup( r_buffer_coords_dim2, c_buffer_coords_dim2, sizeof(c_buffer_coords_dim2))); - - tiledb_ctx_free(&ctx); } void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( @@ -1860,9 +1845,11 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( tiledb_config_set(config, "sm.dedup_coords", "true", &error) == TILEDB_OK); REQUIRE(error == nullptr); - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); - REQUIRE(error == nullptr); + + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); tiledb_config_free(&config); // Prepare cell buffers for WRITE @@ -1904,20 +1891,20 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); @@ -1926,23 +1913,23 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], buffers[5], &buffer_sizes[4]); + ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); CHECK(rc == TILEDB_OK); // Submit WRITE query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -1970,14 +1957,14 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( sizeof(r_buffer_coords_dim1)}; // Open array - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create READ query - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); + ctx_, query, attributes[0], r_buffers[0], &r_buffer_sizes[0]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[1], r_buffers[2], &r_buffer_sizes[2]); @@ -1986,25 +1973,25 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( ctx_, query, attributes[1], (uint64_t*)r_buffers[1], &r_buffer_sizes[1]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); + ctx_, query, attributes[2], r_buffers[3], &r_buffer_sizes[3]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); + ctx_, query, attributes[3], r_buffers[4], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); + ctx_, query, attributes[4], r_buffers[5], &r_buffer_sizes[4]); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); // Submit READ query - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx, query); + rc = tiledb_query_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up @@ -2031,12 +2018,9 @@ void SparseArrayFx::check_sparse_array_global_with_all_duplicates_dedup( r_buffer_coords_dim2, c_buffer_coords_dim2, sizeof(c_buffer_coords_dim2))); - - tiledb_ctx_free(&ctx); } -void SparseArrayFx::check_non_empty_domain(const std::string& path) { - std::string array_name = path + "sparse_non_empty"; +void SparseArrayFx::check_non_empty_domain(const std::string& array_name) { create_sparse_array(array_name); // Check empty domain @@ -2193,7 +2177,7 @@ void SparseArrayFx::check_sparse_array_no_results( CHECK(rc == TILEDB_OK); // Prepare the buffers that will store the result - uint64_t buffer_size = 0; + uint64_t buffer_size = 1; auto buffer = new int[buffer_size / sizeof(int)]; REQUIRE(buffer != nullptr); @@ -2385,93 +2369,61 @@ void SparseArrayFx::write_sparse_array_missing_attributes( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, sorted reads", - "[capi][sparse][sorted-reads][longtest]") { - std::string array_name; + "[capi][sparse][sorted-reads][longtest][rest]") { + // TODO: refactor for each supported FS. + std::string array_name = prefix_ + ARRAY; SECTION("- no compression, row/row-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_NONE, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR); } SECTION("- no compression, col/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_NONE, TILEDB_COL_MAJOR, TILEDB_COL_MAJOR); } SECTION("- no compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_NONE, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- gzip compression, row/row-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_GZIP, TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR); } SECTION("- gzip compression, col/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_GZIP, TILEDB_COL_MAJOR, TILEDB_COL_MAJOR); } SECTION("- gzip compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_GZIP, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- bzip compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_BZIP2, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- lz4 compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_LZ4, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- rle compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_RLE, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- zstd compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_ZSTD, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } SECTION("- double-delta compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_DOUBLE_DELTA, @@ -2480,21 +2432,18 @@ TEST_CASE_METHOD( } SECTION("- delta compression, row/col-major") { - // TODO: refactor for each supported FS. - std::string temp_dir = fs_vec_[0]->temp_dir(); - array_name = temp_dir + ARRAY; check_sorted_reads( array_name, TILEDB_FILTER_DELTA, TILEDB_ROW_MAJOR, TILEDB_COL_MAJOR); } + + remove_temp_dir(fs_vec_[0]->temp_dir()); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, duplicates", - "[capi][sparse][dups]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "dups"; + "[capi][sparse][dups][rest-fails][sc-42987]") { + std::string array_name = prefix_ + "dups"; create_sparse_array(array_name); SECTION("- unordered, error check") { @@ -2533,21 +2482,16 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, non-empty domain", - "[capi][sparse][non-empty]") { - SupportedFsLocal local_fs; - std::string temp_dir = local_fs.file_prefix() + local_fs.temp_dir(); - create_temp_dir(temp_dir); - check_non_empty_domain(temp_dir); - remove_temp_dir(temp_dir); + "[capi][sparse][non-empty][rest]") { + std::string array_name = prefix_ + "sparse_non_empty"; + check_non_empty_domain(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, invalid offsets on write", - "[capi][sparse][invalid-offsets]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "invalid_offs"; + "[capi][sparse][invalid-offsets][rest]") { + std::string array_name = prefix_ + "invalid_offs"; create_sparse_array(array_name); check_invalid_offsets(array_name); } @@ -2555,10 +2499,8 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, no results", - "[capi][sparse][no-results]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "no_results"; + "[capi][sparse][no-results][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "no_results"; create_sparse_array(array_name); write_partial_sparse_array(array_name); check_sparse_array_no_results(array_name); @@ -2567,10 +2509,8 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, missing attributes in writes", - "[capi][sparse][write-missing-attributes]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_write_missing_attributes"; + "[capi][sparse][write-missing-attributes][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "sparse_write_missing_attributes"; create_sparse_array(array_name); write_sparse_array_missing_attributes(array_name); check_sparse_array_no_results(array_name); @@ -2579,26 +2519,20 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, error setting subarray on sparse write", - "[capi][sparse][set-subarray]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_set_subarray"; + "[capi][sparse][set-subarray][rest]") { + std::string array_name = prefix_ + "sparse_set_subarray"; create_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); // Set subarray @@ -2609,181 +2543,160 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_ERR); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, check if coords exist", - "[capi][sparse][coords-exist]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_coords_exist"; + "[capi][sparse][coords-exist][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "sparse_coords_exist"; create_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); // Set attribute buffers int a1[] = {1, 2}; uint64_t a1_size = sizeof(a1); - rc = tiledb_query_set_data_buffer(ctx, query, "a1", a1, &a1_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); char a2[] = {'a', 'b'}; uint64_t a2_size = sizeof(a2); uint64_t a2_off[] = {0, 1}; uint64_t a2_off_size = sizeof(a2_off); - rc = tiledb_query_set_data_buffer(ctx, query, "a2", a2, &a2_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a2", a2, &a2_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer(ctx, query, "a2", a2_off, &a2_off_size); + rc = tiledb_query_set_offsets_buffer(ctx_, query, "a2", a2_off, &a2_off_size); CHECK(rc == TILEDB_OK); float a3[] = {1.1f, 1.2f, 2.1f, 2.2f}; uint64_t a3_size = sizeof(a3); - rc = tiledb_query_set_data_buffer(ctx, query, "a3", a3, &a3_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a3", a3, &a3_size); CHECK(rc == TILEDB_OK); // Submit query - should error - CHECK(tiledb_query_submit(ctx, query) == TILEDB_ERR); + CHECK(tiledb_query_submit(ctx_, query) == TILEDB_ERR); // Set coordinates uint64_t coords_dim1[] = {1, 1}; uint64_t coords_dim2[] = {1, 2}; uint64_t coords_size = sizeof(coords_dim1); - rc = - tiledb_query_set_data_buffer(ctx, query, "d1", coords_dim1, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d1", coords_dim1, &coords_size); CHECK(rc == TILEDB_OK); - rc = - tiledb_query_set_data_buffer(ctx, query, "d2", coords_dim2, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d2", coords_dim2, &coords_size); CHECK(rc == TILEDB_OK); // Submit query - ok - CHECK(tiledb_query_submit(ctx, query) == TILEDB_OK); + CHECK(tiledb_query_submit(ctx_, query) == TILEDB_OK); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, global order check on write", - "[capi][sparse][write-global-check]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_write_global_check"; + "[capi][sparse][write-global-check][rest]") { + std::string array_name = prefix_ + "sparse_write_global_check"; create_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); // Set attribute buffers int a1[] = {1, 2}; uint64_t a1_size = sizeof(a1); - rc = tiledb_query_set_data_buffer(ctx, query, "a1", a1, &a1_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); char a2[] = {'a', 'b'}; uint64_t a2_size = sizeof(a2); uint64_t a2_off[] = {0, 1}; uint64_t a2_off_size = sizeof(a2_off); - rc = tiledb_query_set_data_buffer(ctx, query, "a2", a2, &a2_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a2", a2, &a2_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer(ctx, query, "a2", a2_off, &a2_off_size); + rc = tiledb_query_set_offsets_buffer(ctx_, query, "a2", a2_off, &a2_off_size); CHECK(rc == TILEDB_OK); float a3[] = {1.1f, 1.2f, 2.1f, 2.2f}; uint64_t a3_size = sizeof(a3); - rc = tiledb_query_set_data_buffer(ctx, query, "a3", a3, &a3_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a3", a3, &a3_size); CHECK(rc == TILEDB_OK); // Set coordinates uint64_t coords_dim1[] = {1, 1}; uint64_t coords_dim2[] = {2, 1}; uint64_t coords_size = sizeof(coords_dim1); - rc = - tiledb_query_set_data_buffer(ctx, query, "d1", coords_dim1, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d1", coords_dim1, &coords_size); CHECK(rc == TILEDB_OK); - rc = - tiledb_query_set_data_buffer(ctx, query, "d2", coords_dim2, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d2", coords_dim2, &coords_size); CHECK(rc == TILEDB_OK); // Submit query - ok - CHECK(tiledb_query_submit(ctx, query) == TILEDB_ERR); + CHECK(tiledb_query_submit(ctx_, query) == TILEDB_ERR); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, invalidate cached max buffer sizes", - "[capi][sparse][invalidate-max-sizes]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_invalidate_max_sizes"; + "[capi][sparse][invalidate-max-sizes][rest]") { + std::string array_name = prefix_ + "sparse_invalidate_max_sizes"; create_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // ---- First READ query (empty) tiledb_query_t* empty_query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &empty_query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &empty_query); CHECK(rc == TILEDB_OK); // Get max buffer sizes for empty query @@ -2795,25 +2708,25 @@ TEST_CASE_METHOD( // Set attribute buffers auto a1 = (int*)malloc(a1_size); - rc = tiledb_query_set_data_buffer(ctx, empty_query, "a1", a1, &a1_size); + rc = tiledb_query_set_data_buffer(ctx_, empty_query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); auto a2_off = (uint64_t*)malloc(a2_off_size); auto a2 = (char*)malloc(a2_size); - rc = tiledb_query_set_data_buffer(ctx, empty_query, "a2", a2, &a2_size); + rc = tiledb_query_set_data_buffer(ctx_, empty_query, "a2", a2, &a2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, empty_query, "a2", a2_off, &a2_off_size); + ctx_, empty_query, "a2", a2_off, &a2_off_size); CHECK(rc == TILEDB_OK); auto a3 = (float*)malloc(a3_size); - rc = tiledb_query_set_data_buffer(ctx, empty_query, "a3", a3, &a3_size); + rc = tiledb_query_set_data_buffer(ctx_, empty_query, "a3", a3, &a3_size); CHECK(rc == TILEDB_OK); auto coords_dim1 = (uint64_t*)malloc(coords_size); auto coords_dim2 = (uint64_t*)malloc(coords_size); rc = tiledb_query_set_data_buffer( - ctx, empty_query, "d1", coords_dim1, &coords_size); + ctx_, empty_query, "d1", coords_dim1, &coords_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, empty_query, "d2", coords_dim2, &coords_size); + ctx_, empty_query, "d2", coords_dim2, &coords_size); CHECK(rc == TILEDB_OK); // Set subarray @@ -2827,7 +2740,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - CHECK(tiledb_query_submit(ctx, empty_query) == TILEDB_OK); + CHECK(tiledb_query_submit(ctx_, empty_query) == TILEDB_OK); // Check that there are no results CHECK(a1_size == 0); @@ -2846,7 +2759,7 @@ TEST_CASE_METHOD( // ---- Second READ query (non-empty) tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set max buffer sizes for non-empty query @@ -2858,24 +2771,24 @@ TEST_CASE_METHOD( // Set attribute buffers a1 = (int*)malloc(a1_size); - rc = tiledb_query_set_data_buffer(ctx, query, "a1", a1, &a1_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); a2_off = (uint64_t*)malloc(a2_off_size); a2 = (char*)malloc(a2_size); - rc = tiledb_query_set_data_buffer(ctx, query, "a2", a2, &a2_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a2", a2, &a2_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer(ctx, query, "a2", a2_off, &a2_off_size); + rc = tiledb_query_set_offsets_buffer(ctx_, query, "a2", a2_off, &a2_off_size); CHECK(rc == TILEDB_OK); a3 = (float*)malloc(a3_size); - rc = tiledb_query_set_data_buffer(ctx, query, "a3", a3, &a3_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a3", a3, &a3_size); CHECK(rc == TILEDB_OK); coords_dim1 = (uint64_t*)malloc(coords_size); coords_dim2 = (uint64_t*)malloc(coords_size); - rc = - tiledb_query_set_data_buffer(ctx, query, "d1", coords_dim1, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d1", coords_dim1, &coords_size); CHECK(rc == TILEDB_OK); - rc = - tiledb_query_set_data_buffer(ctx, query, "d2", coords_dim2, &coords_size); + rc = tiledb_query_set_data_buffer( + ctx_, query, "d2", coords_dim2, &coords_size); CHECK(rc == TILEDB_OK); // Set subarray @@ -2891,7 +2804,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - CHECK(tiledb_query_submit(ctx, query) == TILEDB_OK); + CHECK(tiledb_query_submit(ctx_, query) == TILEDB_OK); // Check that there are no results REQUIRE(a1_size == 2 * sizeof(int)); @@ -2920,15 +2833,14 @@ TEST_CASE_METHOD( tiledb_query_free(&query); // Clean up - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, encrypted", - "[capi][sparse][encryption]") { + "[capi][sparse][encryption][non-rest]") { std::string array_name; encryption_type = TILEDB_AES_256_GCM; encryption_key = "0123456789abcdeF0123456789abcdeF"; @@ -2944,31 +2856,25 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, calibrate est size", - "[capi][sparse][calibrate-est-size]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_calibrate_est_size"; - remove_array(array_name); + "[capi][sparse][calibrate-est-size][rest]") { + std::string array_name = prefix_ + "sparse_calibrate_est_size"; + create_sparse_array(array_name); // Write twice (2 fragments) write_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[2]; uint64_t a1_size = sizeof(a1); @@ -2993,7 +2899,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3012,44 +2918,35 @@ TEST_CASE_METHOD( CHECK(a2[2] == 'b'); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, calibrate est size, unary", - "[capi][sparse][calibrate-est-size-unary]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_calibrate_est_size_unary"; - remove_array(array_name); + "[capi][sparse][calibrate-est-size-unary][rest]") { + std::string array_name = prefix_ + "sparse_calibrate_est_size_unary"; + create_sparse_array(array_name); // Write twice (2 fragments) write_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[1]; uint64_t a1_size = sizeof(a1); @@ -3074,7 +2971,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3089,45 +2986,36 @@ TEST_CASE_METHOD( CHECK(a2_size == sizeof(char)); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, calibrate est size, huge range", - "[capi][sparse][calibrate-est-size-huge-range]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_calibrate_est_size_huge_range"; + "[capi][sparse][calibrate-est-size-huge-range][rest]") { + std::string array_name = prefix_ + "sparse_calibrate_est_size_huge_range"; const uint64_t dim_domain[4] = {1, UINT64_MAX - 1, 1, UINT64_MAX - 1}; - remove_array(array_name); + create_sparse_array(array_name, TILEDB_ROW_MAJOR, dim_domain); // Write twice (2 fragments) write_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[2] = {-1, -1}; uint64_t a1_size = sizeof(a1); @@ -3152,7 +3040,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3171,41 +3059,32 @@ TEST_CASE_METHOD( CHECK(a2_size == 3 * sizeof(char)); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-subarray, 2D, complete", - "[capi][sparse][MR][2D][complete]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_subarray_2d_complete"; - remove_array(array_name); + "[capi][sparse][MR][2D][complete][rest]") { + std::string array_name = prefix_ + "sparse_multi_subarray_2d_complete"; + create_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[20]; uint64_t a1_size = sizeof(a1); @@ -3237,7 +3116,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3263,24 +3142,19 @@ TEST_CASE_METHOD( CHECK(coords_dim2[4] == 4); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-subarray, 2D, multiplicities", - "[capi][sparse][multi-subarray-2d-multiplicities]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_subarray_2d_multiplicities"; - remove_array(array_name); + "[capi][sparse][multi-subarray-2d-multiplicities][rest]") { + std::string array_name = prefix_ + "sparse_multi_subarray_2d_multiplicities"; + create_sparse_array(array_name); write_sparse_array(array_name); @@ -3298,20 +3172,22 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(config, &ctx) == TILEDB_OK); + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); + tiledb_config_free(&config); // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[20]; uint64_t a1_size = sizeof(a1); @@ -3345,7 +3221,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3359,41 +3235,32 @@ TEST_CASE_METHOD( CHECK(coords_size == 7 * sizeof(uint64_t)); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-subarray, 2D, incomplete", - "[capi][sparse][multi-subarray-2d-incomplete]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_subarray_2d_incomplete"; - remove_array(array_name); + "[capi][sparse][multi-subarray-2d-incomplete][rest]") { + std::string array_name = prefix_ + "sparse_multi_subarray_2d_incomplete"; + create_sparse_array(array_name); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[3]; uint64_t a1_size = sizeof(a1); @@ -3425,7 +3292,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3449,7 +3316,7 @@ TEST_CASE_METHOD( CHECK(coords_dim2[1] == 4); } - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_status(ctx_, query, &status); REQUIRE(rc == TILEDB_OK); @@ -3476,41 +3343,32 @@ TEST_CASE_METHOD( } // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-subarray, 2D, complete, col", - "[capi][sparse][multi-subarray-2d-complete-col]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_subarray_2d_complete_col"; - remove_array(array_name); + "[capi][sparse][multi-subarray-2d-complete-col][rest]") { + std::string array_name = prefix_ + "sparse_multi_subarray_2d_complete_col"; + create_sparse_array(array_name, TILEDB_COL_MAJOR); write_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); int a1[20]; uint64_t a1_size = sizeof(a1); @@ -3542,7 +3400,7 @@ TEST_CASE_METHOD( rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx, query); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_status_t status; rc = tiledb_query_get_status(ctx_, query, &status); @@ -3556,24 +3414,18 @@ TEST_CASE_METHOD( check_counts(span(coords_dim2, 5), {0, 0, 2, 1, 2}); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array 2, multi-range subarray, row-major", - "[capi][sparse][multi-range-row]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_multi_range_row"; - remove_array(array_name); + "[capi][sparse][multi-range-row][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_row"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -3733,18 +3585,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, col-major", - "[capi][sparse][multi-range-col]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_multi_range_col"; - remove_array(array_name); + "[capi][sparse][multi-range-col][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_col"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -3904,18 +3751,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, row-major, incomplete 1", - "[capi][sparse][multi-range-row-incomplete-1]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_row_incomplete_1"; - remove_array(array_name); + "[capi][sparse][multi-range-row-incomplete-1][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_row_incomplete_1"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -4095,18 +3937,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, col-major, incomplete 1", - "[capi][sparse][multi-range-col-incomplete-1]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_col_incomplete_1"; - remove_array(array_name); + "[capi][sparse][multi-range-col-incomplete-1][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_col_incomplete_1"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -4286,18 +4123,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, row-major, incomplete 2", - "[capi][sparse][multi-range-row-incomplete-2]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_row_incomplete_2"; - remove_array(array_name); + "[capi][sparse][multi-range-row-incomplete-2][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_row_incomplete_2"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -4517,18 +4349,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, col-major, incomplete 2", - "[capi][sparse][multi-range-col-incomplete-2]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_col_incomplete_2"; - remove_array(array_name); + "[capi][sparse][multi-range-col-incomplete-2][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_col_incomplete_2"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -4748,18 +4575,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, row-major, incomplete 3", - "[capi][sparse][multi-range-row-incomplete-3]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_row_incomplete_3"; - remove_array(array_name); + "[capi][sparse][multi-range-row-incomplete-3][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "sparse_multi_range_row_incomplete_3"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -5019,18 +4841,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, row-major, incomplete 4", - "[capi][sparse][multi-range-row-incomplete-4]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_row_incomplete_4"; - remove_array(array_name); + "[capi][sparse][multi-range-row-incomplete-4][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_row_incomplete_4"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -5330,18 +5147,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, col-major, incomplete 4", - "[capi][sparse][multi-range-col-incomplete-4]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_col_incomplete_4"; - remove_array(array_name); + "[capi][sparse][multi-range-col-incomplete-4][rest]") { + std::string array_name = prefix_ + "sparse_multi_range_col_incomplete_4"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -5641,19 +5453,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, row-major, incomplete 5", - "[capi][sparse][multi-range-row-incomplete-5]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_row_incomplete_5"; - remove_array(array_name); - + "[capi][sparse][multi-range-row-incomplete-5][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "sparse_multi_range_row_incomplete_5"; // Create array uint64_t domain[] = {1, 10, 1, 10}; create_sparse_array(array_name, TILEDB_ROW_MAJOR, domain); @@ -5771,18 +5577,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, multi-range subarray, col-major, incomplete 5", - "[capi][sparse][multi-range-col-incomplete-5]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_multi_range_col_incomplete_5"; - remove_array(array_name); + "[capi][sparse][multi-range-col-incomplete-5][rest-fails][sc-43108]") { + std::string array_name = prefix_ + "sparse_multi_range_col_incomplete_5"; // Create array uint64_t domain[] = {1, 10, 1, 10}; @@ -5901,45 +5702,27 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, global order with 0-sized buffers", - "[capi][sparse][global-check][zero-buffers]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_write_global_check"; + "[capi][sparse][global-check][zero-buffers][rest]") { + std::string array_name = prefix_ + "sparse_write_global_check"; create_sparse_array(array_name); - // Create TileDB context - tiledb_ctx_t* ctx = nullptr; - REQUIRE(tiledb_ctx_alloc(nullptr, &ctx) == TILEDB_OK); - // Open array tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create WRITE query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); // Prepare attribute buffers @@ -5952,46 +5735,36 @@ TEST_CASE_METHOD( uint64_t zero_size = 0; // Set buffers with zero size - rc = tiledb_query_set_data_buffer(ctx, query, "a1", a1, &zero_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &zero_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer(ctx, query, "a2", a2, &zero_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a2", a2, &zero_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer(ctx, query, "a2", a2_off, &zero_size); + rc = tiledb_query_set_offsets_buffer(ctx_, query, "a2", a2_off, &zero_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer(ctx, query, "a3", a3, &zero_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "a3", a3, &zero_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer(ctx, query, "d1", coords_dim1, &zero_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "d1", coords_dim1, &zero_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer(ctx, query, "d2", coords_dim2, &zero_size); + rc = tiledb_query_set_data_buffer(ctx_, query, "d2", coords_dim2, &zero_size); CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array - CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); + CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); // Clean up tiledb_query_free(&query); tiledb_array_free(&array); - tiledb_ctx_free(&ctx); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, split coordinate buffers", "[capi][sparse][split-coords]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_split_coords"; + std::string array_name = prefix_ + "sparse_split_coords"; create_sparse_array(array_name); // ---- WRITE ---- @@ -6178,27 +5951,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, split coordinate buffers, global write", - "[capi][sparse][split-coords][global]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_split_coords_global"; + "[capi][sparse][split-coords][global][rest]") { + std::string array_name = prefix_ + "sparse_split_coords_global"; create_sparse_array(array_name); // ---- WRITE ---- @@ -6267,13 +6026,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -6335,13 +6088,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); tiledb_query_status_t status; @@ -6393,17 +6140,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, split coordinate buffers, errors", - "[capi][sparse][split-coords][errors]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_split_coords_errors"; + "[capi][sparse][split-coords][errors][rest]") { + std::string array_name = prefix_ + "sparse_split_coords_errors"; create_sparse_array(array_name); // Prepare cell buffers @@ -6578,17 +6321,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, split coordinate buffers for reads", - "[capi][sparse][split-coords][read]") { - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_split_coords_read"; + "[capi][sparse][split-coords][read][rest]") { + std::string array_name = prefix_ + "sparse_split_coords_read"; create_sparse_array(array_name); // ---- WRITE ---- @@ -6775,18 +6514,14 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "C API: Test sparse array, split coordinate buffers for reads, subset of " "dimensions", - "[capi][sparse][split-coords][read][subset]") { - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_split_coords_read_subset"; + "[capi][sparse][split-coords][read][subset][rest]") { + std::string array_name = prefix_ + "sparse_split_coords_read_subset"; create_sparse_array(array_name); // ---- WRITE ---- @@ -6966,28 +6701,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_array(array_name); } TEST_CASE_METHOD( SparseArrayFx, "Sparse array: 2D, multi write global order", - "[capi][sparse][2D][multi-write]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - // Create and write array - SupportedFsLocal local_fs; - std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + - "sparse_split_coords_read_subset"; + "[capi][sparse][2D][multi-write][rest]") { + std::string array_name = prefix_ + "sparse_split_coords_read_subset"; create_sparse_array(array_name); std::vector d1 = {1, 1, 2, 2}; @@ -7055,14 +6775,7 @@ TEST_CASE_METHOD( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Create new buffers of smaller size to test being able to write multiple @@ -7118,13 +6831,7 @@ TEST_CASE_METHOD( } // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -7196,7 +6903,7 @@ TEST_CASE_METHOD( SupportedFsLocal local_fs; std::string array_name = local_fs.file_prefix() + local_fs.temp_dir() + "serialize_array_directory"; - remove_array(array_name); + create_sparse_array(array_name); // Write twice (2 fragments) diff --git a/test/src/unit-capi-sparse_heter.cc b/test/src/unit-capi-sparse_heter.cc index efe4de2f1956..3d9ce6229a41 100644 --- a/test/src/unit-capi-sparse_heter.cc +++ b/test/src/unit-capi-sparse_heter.cc @@ -61,9 +61,9 @@ struct SparseHeterFx { // Serialization parameters bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Functions SparseHeterFx(); @@ -147,9 +147,13 @@ SparseHeterFx::SparseHeterFx() : fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); + auto temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + prefix_ = vfs_array_uri(fs_vec_[0], temp_dir); } SparseHeterFx::~SparseHeterFx() { + remove_temp_dir(fs_vec_[0]->temp_dir()); // Close vfs test REQUIRE(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); tiledb_vfs_free(&vfs_); @@ -529,13 +533,11 @@ void SparseHeterFx::write_sparse_array_float_int64( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + if (layout == TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit_and_finalize(ctx_, query); + } else { + rc = tiledb_query_submit(ctx_, query); + } REQUIRE(rc == TILEDB_OK); // Close array @@ -578,9 +580,12 @@ void SparseHeterFx::write_sparse_array_int64_float( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, layout); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_submit(ctx_, query); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_finalize(ctx_, query); + + if (layout == TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit_and_finalize(ctx_, query); + } else { + rc = tiledb_query_submit(ctx_, query); + } REQUIRE(rc == TILEDB_OK); // Close array @@ -723,21 +728,8 @@ void SparseHeterFx::check_read_sparse_array_int64_float( TEST_CASE_METHOD( SparseHeterFx, "C API: Test sparse array with heterogeneous domains (float, int64)", - "[capi][sparse][heter][float-int64]") { - SECTION("- No serialization") { - serialize_ = false; - } - SECTION("- Serialization") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - } - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_array_heter"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][heter][float-int64][non-rest]") { + std::string array_name = prefix_ + "sparse_array_heter"; // Create array float dom_f[] = {1.0f, 20.0f}; @@ -1085,26 +1077,13 @@ TEST_CASE_METHOD( buff_d1_r, buff_d2_r, buff_a_r); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( SparseHeterFx, "C API: Test sparse array with heterogeneous domains (int64, float)", - "[capi][sparse][heter][int64-float]") { - SECTION("- No serialization") { - serialize_ = false; - } - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_array_heter"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][heter][int64-float][non-rest]") { + std::string array_name = prefix_ + "sparse_array_heter"; // Create array float dom_f[] = {1.0f, 20.0f}; @@ -1452,6 +1431,4 @@ TEST_CASE_METHOD( buff_d1_r, buff_d2_r, buff_a_r); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-capi-sparse_neg.cc b/test/src/unit-capi-sparse_neg.cc index c2e1bdff3e1b..de1389a702a2 100644 --- a/test/src/unit-capi-sparse_neg.cc +++ b/test/src/unit-capi-sparse_neg.cc @@ -52,14 +52,10 @@ struct SparseNegFx { tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; // Vector of supported filsystems const std::vector> fs_vec_; - - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Functions SparseNegFx(); @@ -80,9 +76,13 @@ SparseNegFx::SparseNegFx() : fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); + auto temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + prefix_ = vfs_array_uri(fs_vec_[0], temp_dir); } SparseNegFx::~SparseNegFx() { + remove_temp_dir(fs_vec_[0]->temp_dir()); // Close vfs test REQUIRE(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); tiledb_vfs_free(&vfs_); @@ -238,8 +238,7 @@ void SparseNegFx::write_sparse_vector(const std::string& path) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -317,8 +316,7 @@ void SparseNegFx::read_sparse_vector(const std::string& path) { rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); int a_c[] = {0, 1}; @@ -487,43 +485,22 @@ void SparseNegFx::read_sparse_array_col(const std::string& path) { TEST_CASE_METHOD( SparseNegFx, "C API: Test 1d sparse vector with negative domain", - "[capi][sparse-neg][sparse-neg-vector]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_neg_vector"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); - + "[capi][sparse-neg][sparse-neg-vector][rest]") { + std::string vector_name = prefix_ + "sparse_neg_vector"; create_sparse_vector(vector_name); write_sparse_vector(vector_name); read_sparse_vector(vector_name); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( SparseNegFx, "C API: Test 2d sparse array with negative domain", - "[capi][sparse-neg][sparse-neg-array]") { + "[capi][sparse-neg][sparse-neg-array][rest]") { SupportedFsLocal local_fs; - std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_neg_array"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); - + std::string vector_name = prefix_ + +"sparse_neg_array"; create_sparse_array(vector_name); write_sparse_array(vector_name); read_sparse_array_global(vector_name); read_sparse_array_row(vector_name); read_sparse_array_col(vector_name); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-capi-sparse_neg_2.cc b/test/src/unit-capi-sparse_neg_2.cc index f94897dd05e3..f11917023a6b 100644 --- a/test/src/unit-capi-sparse_neg_2.cc +++ b/test/src/unit-capi-sparse_neg_2.cc @@ -54,12 +54,8 @@ struct SparseNegFx2 { // Vector of supported filsystems const std::vector> fs_vec_; - - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Functions SparseNegFx2(); @@ -79,9 +75,13 @@ SparseNegFx2::SparseNegFx2() : fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); + auto temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + prefix_ = vfs_array_uri(fs_vec_[0], temp_dir); } SparseNegFx2::~SparseNegFx2() { + remove_temp_dir(fs_vec_[0]->temp_dir()); // Close vfs test REQUIRE(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); tiledb_vfs_free(&vfs_); @@ -237,8 +237,7 @@ void SparseNegFx2::write_sparse_vector(const std::string& path) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -319,8 +318,7 @@ void SparseNegFx2::read_sparse_vector(const std::string& path) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, path, &query, server_buffers_, serialize_, refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); int a_c[] = {0, 1}; @@ -458,42 +456,22 @@ void SparseNegFx2::read_sparse_array_col(const std::string& path) { TEST_CASE_METHOD( SparseNegFx2, "C API: Test 1d sparse vector with negative domain 2", - "[capi][sparse-neg-2][sparse-neg-vector-2]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_neg_vector"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse-neg-2][sparse-neg-vector-2][rest]") { + std::string vector_name = prefix_ + "sparse_neg_vector"; create_sparse_vector(vector_name); write_sparse_vector(vector_name); read_sparse_vector(vector_name); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( SparseNegFx2, "C API: Test 2d sparse array with negative domain 2", - "[capi][sparse-neg-2][sparse-neg-array-2]") { - SupportedFsLocal local_fs; - std::string vector_name = - local_fs.file_prefix() + local_fs.temp_dir() + "sparse_neg_array"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse-neg-2][sparse-neg-array-2][rest]") { + std::string vector_name = prefix_ + +"sparse_neg_array"; create_sparse_array(vector_name); write_sparse_array(vector_name); read_sparse_array_row(vector_name); read_sparse_array_col(vector_name); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-capi-string.cc b/test/src/unit-capi-string.cc index 3f17621257a8..6d56c4da91cb 100644 --- a/test/src/unit-capi-string.cc +++ b/test/src/unit-capi-string.cc @@ -33,6 +33,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb.h" #include @@ -62,88 +63,84 @@ uint64_t UTF16_OFFSET_2 = sizeof(u"aαbβ") - UTF16_NULL_SIZE; uint64_t UTF16_OFFSET_3 = sizeof(u"aαbβcγ") - UTF16_NULL_SIZE; struct StringFx { - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - + StringFx(); void create_array(const std::string& array_name); - void delete_array(const std::string& array_name); void read_array(const std::string& array_name); void write_array(const std::string& array_name); + + VFSTestSetup vfs_test_setup_; + tiledb_ctx_t* ctx_; }; +StringFx::StringFx() + : ctx_(vfs_test_setup_.ctx_c) { +} + // Create a simple dense 1D array with three string attributes void StringFx::create_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Create dimensions uint64_t dim_domain[] = {1, 4}; uint64_t tile_extent = 2; tiledb_dimension_t* d1; - rc = tiledb_dimension_alloc( - ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extent, &d1); + int rc = tiledb_dimension_alloc( + ctx_, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extent, &d1); REQUIRE(rc == TILEDB_OK); // Create domain tiledb_domain_t* domain; - rc = tiledb_domain_alloc(ctx, &domain); + rc = tiledb_domain_alloc(ctx_, &domain); REQUIRE(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx, domain, d1); + rc = tiledb_domain_add_dimension(ctx_, domain, d1); REQUIRE(rc == TILEDB_OK); // Create fixed-sized UTF-8 attribute tiledb_attribute_t* a1; - rc = tiledb_attribute_alloc(ctx, "a1", TILEDB_STRING_ASCII, &a1); + rc = tiledb_attribute_alloc(ctx_, "a1", TILEDB_STRING_ASCII, &a1); REQUIRE(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx, a1, 2); + rc = tiledb_attribute_set_cell_val_num(ctx_, a1, 2); REQUIRE(rc == TILEDB_OK); // Create variable-sized UTF-8 attribute tiledb_attribute_t* a2; - rc = tiledb_attribute_alloc(ctx, "a2", TILEDB_STRING_UTF8, &a2); + rc = tiledb_attribute_alloc(ctx_, "a2", TILEDB_STRING_UTF8, &a2); REQUIRE(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx, a2, TILEDB_VAR_NUM); + rc = tiledb_attribute_set_cell_val_num(ctx_, a2, TILEDB_VAR_NUM); REQUIRE(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx, a2, TILEDB_FILTER_GZIP, -1); + rc = set_attribute_compression_filter(ctx_, a2, TILEDB_FILTER_GZIP, -1); REQUIRE(rc == TILEDB_OK); // Create variable-sized UTF-16 attribute tiledb_attribute_t* a3; - rc = tiledb_attribute_alloc(ctx, "a3", TILEDB_STRING_UTF16, &a3); + rc = tiledb_attribute_alloc(ctx_, "a3", TILEDB_STRING_UTF16, &a3); REQUIRE(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx, a3, TILEDB_VAR_NUM); + rc = tiledb_attribute_set_cell_val_num(ctx_, a3, TILEDB_VAR_NUM); REQUIRE(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx, a3, TILEDB_FILTER_ZSTD, -1); + rc = set_attribute_compression_filter(ctx_, a3, TILEDB_FILTER_ZSTD, -1); REQUIRE(rc == TILEDB_OK); // Create array schema tiledb_array_schema_t* array_schema; - rc = tiledb_array_schema_alloc(ctx, TILEDB_DENSE, &array_schema); + rc = tiledb_array_schema_alloc(ctx_, TILEDB_DENSE, &array_schema); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); + rc = tiledb_array_schema_set_cell_order(ctx_, array_schema, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); + rc = tiledb_array_schema_set_tile_order(ctx_, array_schema, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_set_domain(ctx, array_schema, domain); + rc = tiledb_array_schema_set_domain(ctx_, array_schema, domain); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx, array_schema, a1); + rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a1); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx, array_schema, a2); + rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a2); REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx, array_schema, a3); + rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a3); REQUIRE(rc == TILEDB_OK); // Check array schema - rc = tiledb_array_schema_check(ctx, array_schema); + rc = tiledb_array_schema_check(ctx_, array_schema); REQUIRE(rc == TILEDB_OK); // Create array - rc = tiledb_array_create(ctx, array_name.c_str(), array_schema); + rc = tiledb_array_create(ctx_, array_name.c_str(), array_schema); REQUIRE(rc == TILEDB_OK); // Clean up @@ -153,15 +150,9 @@ void StringFx::create_array(const std::string& array_name) { tiledb_dimension_free(&d1); tiledb_domain_free(&domain); tiledb_array_schema_free(&array_schema); - tiledb_ctx_free(&ctx); } void StringFx::write_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Prepare buffers void* buffer_a1 = std::malloc(sizeof(UTF8_STRINGS) - UTF8_NULL_SIZE); uint64_t buffer_a2_offsets[] = { @@ -188,68 +179,56 @@ void StringFx::write_array(const std::string& array_name) { // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_WRITE); + rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); // Create query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3"}; - rc = tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[0], buffers[0], &buffer_sizes[0]); + ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[1], buffers[2], &buffer_sizes[2]); + ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); + ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, attributes[2], buffers[4], &buffer_sizes[4]); + ctx_, query, attributes[2], buffers[4], &buffer_sizes[4]); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, attributes[2], (uint64_t*)buffers[3], &buffer_sizes[3]); + ctx_, query, attributes[2], (uint64_t*)buffers[3], &buffer_sizes[3]); REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); std::free(buffer_a1); std::free(buffer_a2); std::free(buffer_a3); } void StringFx::read_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Compute max buffer sizes @@ -269,36 +248,30 @@ void StringFx::read_array(const std::string& array_name) { // Create query tiledb_query_t* query; - rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); + rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, "a1", buffer_a1, &buffer_a1_size); + ctx_, query, "a1", buffer_a1, &buffer_a1_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, "a2", buffer_a2_val, &buffer_a2_val_size); + ctx_, query, "a2", buffer_a2_val, &buffer_a2_val_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, "a2", buffer_a2_off, &buffer_a2_off_size); + ctx_, query, "a2", buffer_a2_off, &buffer_a2_off_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( - ctx, query, "a3", buffer_a3_val, &buffer_a3_val_size); + ctx_, query, "a3", buffer_a3_val, &buffer_a3_val_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( - ctx, query, "a3", buffer_a3_off, &buffer_a3_off_size); + ctx_, query, "a3", buffer_a3_off, &buffer_a3_off_size); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); + rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx, query, subarray); + rc = tiledb_query_set_subarray(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Check results @@ -320,13 +293,12 @@ void StringFx::read_array(const std::string& array_name) { CHECK(buffer_a3_off[3] == UTF16_OFFSET_3); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); std::free(buffer_a1); std::free(buffer_a2_off); std::free(buffer_a2_val); @@ -334,40 +306,10 @@ void StringFx::read_array(const std::string& array_name) { std::free(buffer_a3_val); } -void StringFx::delete_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - - // Remove array - tiledb_object_t type; - rc = tiledb_object_type(ctx, array_name.c_str(), &type); - REQUIRE(rc == TILEDB_OK); - if (type == TILEDB_ARRAY) { - rc = tiledb_object_remove(ctx, array_name.c_str()); - REQUIRE(rc == TILEDB_OK); - } - - // Clean up - tiledb_ctx_free(&ctx); -} - -TEST_CASE_METHOD(StringFx, "C API: Test string support", "[capi][string]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - std::string array_name = "foo"; - delete_array(array_name); +TEST_CASE_METHOD( + StringFx, "C API: Test string support", "[capi][string][rest]") { + std::string array_name = vfs_test_setup_.array_uri("foo"); create_array(array_name); write_array(array_name); read_array(array_name); - delete_array(array_name); } diff --git a/test/src/unit-capi-string_dims.cc b/test/src/unit-capi-string_dims.cc index e2d3135ebd96..0b9e7c6fad2e 100644 --- a/test/src/unit-capi-string_dims.cc +++ b/test/src/unit-capi-string_dims.cc @@ -58,14 +58,13 @@ struct StringDimsFx { tiledb_ctx_t* ctx_; tiledb_vfs_t* vfs_; - // Vector of supported filsystems - const std::vector> fs_vec_; - // Serialization parameters bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - tiledb::test::ServerQueryBuffers server_buffers_; + + // Vector of supported filsystems + const std::vector> fs_vec_; + // Path to prepend to array name according to filesystem/mode + std::string prefix_; // Used to get the number of directories or files of another directory struct get_num_struct { @@ -179,9 +178,13 @@ StringDimsFx::StringDimsFx() : fs_vec_(vfs_test_get_fs_vec()) { // Initialize vfs test REQUIRE(vfs_test_init(fs_vec_, &ctx_, &vfs_).ok()); + auto temp_dir = fs_vec_[0]->temp_dir(); + create_temp_dir(temp_dir); + prefix_ = vfs_array_uri(fs_vec_[0], temp_dir); } StringDimsFx::~StringDimsFx() { + remove_temp_dir(fs_vec_[0]->temp_dir()); // Close vfs test REQUIRE(vfs_test_close(fs_vec_, ctx_, vfs_).ok()); tiledb_vfs_free(&vfs_); @@ -368,13 +371,11 @@ void StringDimsFx::write_array_1d( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + if (layout == TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit_and_finalize(ctx, query); + } else { + rc = tiledb_query_submit(ctx, query); + } CHECK(rc == TILEDB_OK); // Close array @@ -425,13 +426,11 @@ void StringDimsFx::write_array_2d( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + if (layout == TILEDB_GLOBAL_ORDER) { + rc = tiledb_query_submit_and_finalize(ctx, query); + } else { + rc = tiledb_query_submit(ctx, query); + } CHECK(rc == TILEDB_OK); // Close array @@ -716,14 +715,7 @@ void StringDimsFx::read_array_1d( rc = tiledb_array_get_uri(ctx, array, &array_uri); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx, query); CHECK(rc == TILEDB_OK); // Get status @@ -837,14 +829,7 @@ void StringDimsFx::read_array_2d( const char* array_uri; rc = tiledb_array_get_uri(ctx, array, &array_uri); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx, query); CHECK(rc == TILEDB_OK); // Get status @@ -904,14 +889,7 @@ void StringDimsFx::read_array_2d_default_string_range( const char* array_uri; rc = tiledb_array_get_uri(ctx, array, &array_uri); CHECK(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx, - array_uri, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx, query); CHECK(rc == TILEDB_OK); // Get status @@ -931,21 +909,8 @@ void StringDimsFx::read_array_2d_default_string_range( TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, array schema", - "[capi][sparse][string-dims][array-schema]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][array-schema][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create dimension tiledb_domain_t* domain; @@ -1035,29 +1000,14 @@ TEST_CASE_METHOD( tiledb_array_schema_free(&array_schema); tiledb_domain_free(&domain); tiledb_dimension_free(&d); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, check duplicates, global " "order", - "[capi][sparse][string-dims][duplicates][global]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][duplicates][global][rest]") { + std::string array_name = prefix_ + "string_dims"; create_array( ctx_, @@ -1102,14 +1052,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_ERR); // Close array @@ -1119,29 +1062,14 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, check duplicates, " "unordered", - "[capi][sparse][string-dims][duplicates][unordered]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][duplicates][unordered][rest]") { + std::string array_name = prefix_ + +"string_dims"; create_array( ctx_, @@ -1187,14 +1115,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_ERR); // Close array @@ -1204,29 +1125,14 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, check global order " "violation", - "[capi][sparse][string-dims][global-order][violation]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][global-order][violation][rest]") { + std::string array_name = prefix_ + "string_dims"; create_array( ctx_, @@ -1272,14 +1178,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_ERR); // Close array @@ -1289,28 +1188,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, errors", - "[capi][sparse][string-dims][errors]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][errors][rest-fails][sc-43167]") { + std::string array_name = prefix_ + "string_dims"; // Create array create_array( @@ -1359,14 +1243,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -1402,7 +1279,7 @@ TEST_CASE_METHOD( ctx_, query, TILEDB_COORDS, buff, &buff_size); REQUIRE(rc == TILEDB_ERR); int data[1]; - uint64_t data_size; + uint64_t data_size = 4; rc = tiledb_query_set_data_buffer(ctx_, query, "d", data, &data_size); REQUIRE(rc == TILEDB_OK); @@ -1419,28 +1296,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 1d", - "[capi][sparse][string-dims][1d][basic]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][1d][basic][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array create_array( @@ -1662,28 +1524,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 1d, consolidation", - "[capi][sparse][string-dims][1d][consolidation]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][1d][consolidation][non-rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array create_array( @@ -1853,28 +1700,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 1d, allow duplicates", - "[capi][sparse][string-dims][1d][allow-dups]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][1d][allow-dups][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array create_array( @@ -1968,28 +1800,13 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 1d, dedup", - "[capi][sparse][string-dims][1d][dedup]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][1d][dedup][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array create_array( @@ -2019,16 +1836,17 @@ TEST_CASE_METHOD( rc = tiledb_config_set(config, "sm.dedup_coords", "true", &error); CHECK(rc == TILEDB_OK); - // Create context - tiledb_ctx_t* ctx; - rc = tiledb_ctx_alloc(config, &ctx); - CHECK(rc == TILEDB_OK); + tiledb_ctx_free(&ctx_); + tiledb_vfs_free(&vfs_); + // reallocate with input config + vfs_test_init(fs_vec_, &ctx_, &vfs_, config).ok(); + tiledb_config_free(&config); // Write std::vector d_off = {0, 2, 4, 8}; std::string d_val("ccccddddaa"); std::vector a = {2, 3, 4, 1}; - write_array_1d(ctx, array_name, TILEDB_UNORDERED, d_off, d_val, a); + write_array_1d(ctx_, array_name, TILEDB_UNORDERED, d_off, d_val, a); // Clean up tiledb_config_free(&config); @@ -2037,9 +1855,9 @@ TEST_CASE_METHOD( // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx, array, TILEDB_READ); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); // Check non-empty domain @@ -2073,7 +1891,7 @@ TEST_CASE_METHOD( tiledb_query_status_t status; tiledb_query_status_details_t details; read_array_1d( - ctx, + ctx_, array, layout, "a", @@ -2094,34 +1912,18 @@ TEST_CASE_METHOD( CHECK(c_a_matches); // Close array - rc = tiledb_array_close(ctx, array); + rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); // Clean up tiledb_array_free(&array); - tiledb_ctx_free(&ctx); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 2d", - "[capi][sparse][string-dims][2d]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][2d][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array int32_t dom[] = {1, 10}; @@ -2323,77 +2125,66 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); - rc = tiledb_config_set(config, "sm.consolidation.mode", "fragments", &error); - CHECK(rc == TILEDB_OK); - - // Consolidate - rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_vacuum(ctx_, array_name.c_str(), nullptr); - CHECK(rc == TILEDB_OK); - - // Open array - rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_READ); - CHECK(rc == TILEDB_OK); - - // Read [a, ff], [1, 10] - r_d1_off.resize(20); - r_d1_val.resize(20); - r_d2.resize(20); - r_a.resize(20); - read_array_2d( - ctx_, - array, - TILEDB_GLOBAL_ORDER, - "a", - "ff", - 1, - 10, - &r_d1_off, - &r_d1_val, - &r_d2, - &r_a, - &status); - CHECK(status == TILEDB_COMPLETED); - CHECK(r_d1_val == "aaabbbccddddff"); - c_d1_off = {0, 1, 3, 4, 6, 8, 12}; - CHECK(r_d1_off == c_d1_off); - c_d2 = {2, 1, 2, 2, 3, 4, 3}; - CHECK(r_d2 == c_d2); - c_a = {15, 11, 16, 12, 13, 14, 17}; - CHECK(r_a == c_a); - - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_config_free(&config); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + if (!fs_vec_[0]->is_rest()) { + rc = + tiledb_config_set(config, "sm.consolidation.mode", "fragments", &error); + CHECK(rc == TILEDB_OK); + + // Consolidate + rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_vacuum(ctx_, array_name.c_str(), nullptr); + CHECK(rc == TILEDB_OK); + + // Open array + rc = tiledb_array_alloc(ctx_, array_name.c_str(), &array); + CHECK(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + CHECK(rc == TILEDB_OK); + + // Read [a, ff], [1, 10] + r_d1_off.resize(20); + r_d1_val.resize(20); + r_d2.resize(20); + r_a.resize(20); + read_array_2d( + ctx_, + array, + TILEDB_GLOBAL_ORDER, + "a", + "ff", + 1, + 10, + &r_d1_off, + &r_d1_val, + &r_d2, + &r_a, + &status); + CHECK(status == TILEDB_COMPLETED); + CHECK(r_d1_val == "aaabbbccddddff"); + c_d1_off = {0, 1, 3, 4, 6, 8, 12}; + CHECK(r_d1_off == c_d1_off); + c_d2 = {2, 1, 2, 2, 3, 4, 3}; + CHECK(r_d2 == c_d2); + c_a = {15, 11, 16, 12, 13, 14, 17}; + CHECK(r_a == c_a); + + // Close array + rc = tiledb_array_close(ctx_, array); + CHECK(rc == TILEDB_OK); + + // Clean up + tiledb_array_free(&array); + tiledb_config_free(&config); + } } TEST_CASE_METHOD( StringDimsFx, "C API: Test multiple var size global writes 1", - "[capi][sparse][var-size][multiple-global-writes-1]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][var-size][multiple-global-writes-1][rest-fails][sc-" + "43168]") { + std::string array_name = prefix_ + "string_dims"; create_array( ctx_, @@ -2441,14 +2232,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Write "b, 2" @@ -2456,13 +2240,7 @@ TEST_CASE_METHOD( a_data[0] = 2; // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2508,28 +2286,14 @@ TEST_CASE_METHOD( rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); tiledb_array_free(&array); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test multiple var size global writes 2", - "[capi][sparse][var-size][multiple-global-writes-2]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][var-size][multiple-global-writes-2][rest-fails][sc-" + "43168]") { + std::string array_name = prefix_ + "string_dims"; create_array( ctx_, @@ -2577,14 +2341,7 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Write "b, 2" @@ -2592,14 +2349,7 @@ TEST_CASE_METHOD( a_data[0] = 2; // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); // Write c, 3; d, 4 and e, 5. @@ -2614,13 +2364,7 @@ TEST_CASE_METHOD( a_size = 3 * sizeof(int32_t); // Submit query - rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); // Close array @@ -2666,28 +2410,13 @@ TEST_CASE_METHOD( rc = tiledb_array_close(ctx_, array); CHECK(rc == TILEDB_OK); tiledb_array_free(&array); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } TEST_CASE_METHOD( StringDimsFx, "C API: Test sparse array with string dimensions, 2d, default ranges", - "[capi][sparse][string-dims][2d][default-ranges]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - SupportedFsLocal local_fs; - std::string array_name = - local_fs.file_prefix() + local_fs.temp_dir() + "string_dims"; - create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + "[capi][sparse][string-dims][2d][default-ranges][rest]") { + std::string array_name = prefix_ + "string_dims"; // Create array int32_t dom[] = {1, 10}; @@ -2848,6 +2577,4 @@ TEST_CASE_METHOD( // Clean up tiledb_config_free(&config); - - remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 4f6d614775ba..cfb8e0de257f 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -53,12 +53,6 @@ struct ConsolidationWithTimestampsFx { VFS vfs_; sm::StorageManager* sm_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - // Constructors/destructors. ConsolidationWithTimestampsFx(); ~ConsolidationWithTimestampsFx(); @@ -183,14 +177,7 @@ void ConsolidationWithTimestampsFx::write_sparse( query.set_data_buffer("d2", dim2); // Submit query - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + query.submit_and_finalize(); // Close array. array.close(); @@ -220,14 +207,7 @@ void ConsolidationWithTimestampsFx::write_sparse_v11(uint64_t timestamp) { query.set_data_buffer("d2", buffer_coords_dim2); // Submit query - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + query.submit_and_finalize(); // Close array. array.close(); @@ -323,15 +303,7 @@ void ConsolidationWithTimestampsFx::read_sparse( } // Submit the query. - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(query.query_status() == Query::Status::COMPLETE); // Get the query stats. @@ -367,15 +339,7 @@ void ConsolidationWithTimestampsFx::reopen_sparse( } // Submit the query. - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(query.query_status() == Query::Status::COMPLETE); // Get the query stats. @@ -404,17 +368,7 @@ bool ConsolidationWithTimestampsFx::is_array(const std::string& array_name) { TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps", - "[cppapi][consolidation-with-timestamps][write-check]") { - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[cppapi][consolidation-with-timestamps][write-check][non-rest]") { remove_sparse_array(); create_sparse_array(); @@ -441,19 +395,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, check directory contents", - "[cppapi][consolidation-with-timestamps][sparse-unordered-with-dups]") { + "[cppapi][consolidation-with-timestamps][sparse-unordered-with-dups][non-" + "rest]") { remove_sparse_array(); create_sparse_array(true); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); @@ -505,14 +451,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, global read", - "[cppapi][consolidation-with-timestamps][global-read]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[cppapi][consolidation-with-timestamps][global-read][non-rest]") { remove_sparse_array(); create_sparse_array(); @@ -559,21 +498,12 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, check directory contents of " "old array", - "[cppapi][consolidation-with-timestamps][sparse-unordered-with-dups]") { + "[cppapi][consolidation-with-timestamps][sparse-unordered-with-dups][non-" + "rest]") { if constexpr (is_experimental_build) { return; } - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - remove_sparse_array(); create_sparse_array_v11(); // Write first fragment. @@ -629,14 +559,8 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, global read, all cells same " "coords", - "[cppapi][consolidation-with-timestamps][global-read][same-coords]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[cppapi][consolidation-with-timestamps][global-read][same-coords][non-" + "rest]") { remove_sparse_array(); create_sparse_array(); @@ -675,14 +599,8 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, global read, same coords " "across tiles", - "[cppapi][consolidation-with-timestamps][global-read][across-tiles]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[cppapi][consolidation-with-timestamps][global-read][across-tiles][non-" + "rest]") { remove_sparse_array(); bool allow_dups = GENERATE(true, false); create_sparse_array(allow_dups); @@ -738,19 +656,10 @@ TEST_CASE_METHOD( "CPP API: Test consolidation with timestamps, global read, all cells same " "coords, with memory budget", "[cppapi][consolidation-with-timestamps][global-read][same-coords][mem-" - "budget]") { + "budget][non-rest]") { remove_sparse_array(); create_sparse_array(); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write fragments. for (uint64_t i = 0; i < 50; i++) { std::vector a(1); @@ -791,19 +700,10 @@ TEST_CASE_METHOD( "CPP API: Test consolidation with timestamps, global read, same cells " "across tiles, with memory budget", "[cppapi][consolidation-with-timestamps][global-read][across-tiles][mem-" - "budget]") { + "budget][non-rest]") { remove_sparse_array(); create_sparse_array(); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write fragments. // We write 8 cells per fragment for 6 fragments. Then it gets consolidated // into one. So we'll get in order 6xcell1, 6xcell2... total 48 cells. Tile @@ -851,18 +751,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, partial read with dups", - "[cppapi][consolidation-with-timestamps][partial-read][dups]") { + "[cppapi][consolidation-with-timestamps][partial-read][dups][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); @@ -945,18 +838,12 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, partial read without dups", - "[cppapi][consolidation-with-timestamps][partial-read][no-dups]") { + "[cppapi][consolidation-with-timestamps][partial-read][no-dups][non-" + "rest]") { remove_sparse_array(); // No duplicates. create_sparse_array(); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); @@ -1024,18 +911,11 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, full and partial read with " "dups", - "[cppapi][consolidation-with-timestamps][partial-read][dups]") { + "[cppapi][consolidation-with-timestamps][partial-read][dups][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1177,18 +1057,11 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, reopen", - "[cppapi][consolidation-with-timestamps][partial-read][dups]") { + "[cppapi][consolidation-with-timestamps][partial-read][dups][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1292,18 +1165,11 @@ TEST_CASE_METHOD( "CPP API: Test read timestamps of unconsolidated fragments, unordered " "reader", "[cppapi][consolidation-with-timestamps][read-timestamps][unordered-" - "reader]") { + "reader][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1392,7 +1258,7 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test read timestamps, unordered reader, overlapping ranges", "[cppapi][consolidation-with-timestamps][read-timestamps][unordered-" - "reader][overlapping-ranges][SC-18541]") { + "reader][overlapping-ranges][SC-18541][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); @@ -1409,15 +1275,6 @@ TEST_CASE_METHOD( sm_ = ctx_.ptr().get()->storage_manager(); vfs_ = VFS(ctx_); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1456,15 +1313,7 @@ TEST_CASE_METHOD( query.add_range(1, 2, 3); // Submit/finalize the query - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(query.query_status() == Query::Status::COMPLETE); // Expect to read what the first 2 writes wrote: each element will @@ -1522,18 +1371,11 @@ TEST_CASE_METHOD( "CPP API: Test read timestamps of unconsolidated fragments, global order " "reader", "[cppapi][consolidation-with-timestamps][read-timestamps][global-order-" - "reader]") { + "reader][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(true, false); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1632,7 +1474,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, check number of tiles read", - "[cppapi][consolidation-with-timestamps][partial-read][stats]") { + "[cppapi][consolidation-with-timestamps][partial-read][stats][non-rest]") { remove_sparse_array(); // Enable duplicates. create_sparse_array(true); @@ -1718,15 +1560,6 @@ TEST_CASE_METHOD( remove_sparse_array(); create_sparse_array(false); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1746,15 +1579,6 @@ TEST_CASE_METHOD( remove_sparse_array(); create_sparse_array(false); - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif // Write first fragment. write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); // Write second fragment. @@ -1785,14 +1609,8 @@ TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, consolidate consolidated" " fragment with timestamps", - "[cppapi][consolidation-with-timestamps][frag-w-timestamps]") { + "[cppapi][consolidation-with-timestamps][frag-w-timestamps][non-rest]") { remove_sparse_array(); -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif bool dups = GENERATE(true, false); @@ -1869,14 +1687,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ConsolidationWithTimestampsFx, "CPP API: Test consolidation with timestamps, ts tiles kept in memory", - "[cppapi][consolidation-with-timestamps][ts-tile-in-memory]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - + "[cppapi][consolidation-with-timestamps][ts-tile-in-memory][non-rest]") { remove_sparse_array(); // Enable duplicates. @@ -1908,15 +1719,7 @@ TEST_CASE_METHOD( query.set_data_buffer(tiledb_timestamps(), timestamps); // Submit the query. - auto rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(query.query_status() == Query::Status::INCOMPLETE); // Validate. @@ -1931,15 +1734,7 @@ TEST_CASE_METHOD( !memcmp(c_ts.data(), timestamps.data(), c_ts.size() * sizeof(uint64_t))); // Submit again. - rc = test::submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(query.query_status() == Query::Status::COMPLETE); // Validate. diff --git a/test/src/unit-cppapi-string-dims.cc b/test/src/unit-cppapi-string-dims.cc index abc7e961a17d..d865dc8dbca3 100644 --- a/test/src/unit-cppapi-string-dims.cc +++ b/test/src/unit-cppapi-string-dims.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/common/logger_public.h" @@ -1451,25 +1452,17 @@ void write_sparse_array_string_dim( const std::string& array_name, std::string& data, std::vector& data_offsets, - tiledb_layout_t layout, - const bool serialized, - const bool refactored_query_v2) { + tiledb_layout_t layout) { Array array(ctx, array_name, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); query.set_layout(layout); query.set_data_buffer("dim1", (char*)data.data(), data.size()); query.set_offsets_buffer("dim1", data_offsets.data(), data_offsets.size()); - - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout != TILEDB_GLOBAL_ORDER) { + query.submit(); + } else { + query.submit_and_finalize(); + } array.close(); } @@ -1479,9 +1472,7 @@ void read_and_check_sparse_array_string_dim( const std::string& array_name, std::string& expected_data, std::vector& expected_offsets, - tiledb_layout_t layout, - const bool serialized, - const bool refactored_query_v2) { + tiledb_layout_t layout) { Array array(ctx, array_name, TILEDB_READ); std::vector offsets_back(expected_offsets.size()); @@ -1494,16 +1485,7 @@ void read_and_check_sparse_array_string_dim( query.set_offsets_buffer("dim1", offsets_back.data(), offsets_back.size()); query.set_layout(layout); - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element data and offsets are properly returned CHECK(data_back == expected_data); @@ -1514,16 +1496,7 @@ void read_and_check_sparse_array_string_dim( TEST_CASE( "C++ API: Test filtering of string dimensions on sparse arrays", - "[cppapi][string-dims][rle-strings][dict-strings][sparse]") { - std::string array_name = "test_rle_string_dim"; - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - + "[cppapi][string-dims][rle-strings][dict-strings][sparse][rest]") { // Create data buffer to use std::stringstream repetitions; size_t repetition_num = 100; @@ -1538,12 +1511,9 @@ TEST_CASE( return start += 10; }); - Context ctx; - VFS vfs(ctx); - - // Create the array - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + test::VFSTestSetup vfs_test_setup; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("test_rle_string_dim")}; Domain domain(ctx); auto dim = @@ -1567,87 +1537,36 @@ TEST_CASE( SECTION("Unordered write") { write_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_UNORDERED); SECTION("Row major read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_UNORDERED); } } SECTION("Global order write") { write_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_GLOBAL_ORDER); SECTION("Row major read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_dim( - ctx, - array_name, - data, - data_elem_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, data, data_elem_offsets, TILEDB_UNORDERED); } } - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( diff --git a/test/src/unit-query-plan.cc b/test/src/unit-query-plan.cc index f3fe5ec7fe02..8df96f60fda8 100644 --- a/test/src/unit-query-plan.cc +++ b/test/src/unit-query-plan.cc @@ -37,32 +37,25 @@ #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" -using namespace tiledb; - -#ifndef TILEDB_TESTS_ENABLE_REST -constexpr bool rest_tests = false; -#else -constexpr bool rest_tests = true; -#endif +using namespace tiledb::test; struct QueryPlanFx { QueryPlanFx(); - ~QueryPlanFx(); void create_dense_array(const std::string& array_name); void create_sparse_array(const std::string& array_name); - // Vector of supported filsystems - tiledb_ctx_handle_t* ctx_c_{nullptr}; - tiledb_vfs_handle_t* vfs_c_{nullptr}; - const std::vector> fs_vec_; + VFSTestSetup vfs_test_setup_; - std::string temp_dir_; - std::string abs_uri_; + // TileDB context + tiledb_ctx_t* ctx_c_; std::string uri_; - Context ctx_; }; +QueryPlanFx::QueryPlanFx() + : ctx_c_(vfs_test_setup_.ctx_c) { +} + TEST_CASE_METHOD( QueryPlanFx, "C API: tiledb_query_get_plan API lifecycle checks", @@ -253,31 +246,8 @@ TEST_CASE_METHOD( tiledb_array_free(&array); } -QueryPlanFx::QueryPlanFx() - : fs_vec_(test::vfs_test_get_fs_vec()) { - auto rc = test::vfs_test_init(fs_vec_, &ctx_c_, &vfs_c_); - if (!rc.ok()) { - throw std::runtime_error("Error initializing vfs in test set up."); - } - - ctx_ = Context(ctx_c_); - temp_dir_ = fs_vec_[0]->temp_dir(); - test::vfs_test_create_temp_dir(ctx_c_, vfs_c_, temp_dir_); -} - -QueryPlanFx::~QueryPlanFx() { - test::vfs_test_remove_temp_dir(ctx_c_, vfs_c_, temp_dir_); - test::vfs_test_close(fs_vec_, ctx_c_, vfs_c_).ok(); - tiledb_vfs_free(&vfs_c_); -} - void QueryPlanFx::create_dense_array(const std::string& array_name) { - if constexpr (rest_tests) { - uri_ = "tiledb://unit/"; - } - - abs_uri_ = temp_dir_ + "/" + array_name; - uri_ += abs_uri_; + uri_ = vfs_test_setup_.array_uri(array_name); // Create array schema tiledb_array_schema_t* array_schema; @@ -347,12 +317,7 @@ void QueryPlanFx::create_dense_array(const std::string& array_name) { } void QueryPlanFx::create_sparse_array(const std::string& array_name) { - if constexpr (rest_tests) { - uri_ = "tiledb://unit/"; - } - - abs_uri_ = temp_dir_ + "/" + array_name; - uri_ += abs_uri_; + uri_ = vfs_test_setup_.array_uri(array_name); // Create dimensions uint64_t tile_extents[] = {2, 2}; diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 77580430b1e6..d2ddfcbd6521 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -31,6 +31,7 @@ */ #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" @@ -1122,22 +1123,10 @@ TEST_CASE_METHOD( TEST_CASE( "Sparse global order reader: user buffer cannot fit single cell", - "[sparse-global-order][user-buffer][too-small]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - - std::string array_name = "test_sparse_global_order"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + "[sparse-global-order][user-buffer][too-small][rest]") { + VFSTestSetup vfs_test_setup; + std::string array_name = vfs_test_setup.array_uri("test_sparse_global_order"); + auto ctx = vfs_test_setup.ctx(); // Create array with var-sized attribute. Domain dom(ctx); @@ -1171,15 +1160,7 @@ TEST_CASE( query.set_offsets_buffer("a", a1_offsets); // Submit query - ServerQueryBuffers server_buffers_; - auto rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query.submit_and_finalize(); // Read using a buffer that can't fit a single result Array array2(ctx, array_name, TILEDB_READ); @@ -1198,48 +1179,26 @@ TEST_CASE( // The user buffer cannot fit a single result so it should return Incomplete // with the right reason - rc = submit_query_wrapper( - ctx, - array_name, - &query2, - server_buffers_, - serialized, - refactored_query_v2, - false); - REQUIRE(rc == TILEDB_OK); + query2.submit(); REQUIRE(query2.query_status() == Query::Status::INCOMPLETE); - // For remote arrays the reason is always TILEDB_REASON_USER_BUFFER_SIZE, - // but we can't test it here since we simulate "remote" arrays by using a - // local URI so the array->is_remote() check will fail, and we won't get the - // correct result. - if (!serialized) { - tiledb_query_status_details_t details; - rc = tiledb_query_get_status_details( - ctx.ptr().get(), query2.ptr().get(), &details); - CHECK(rc == TILEDB_OK); - CHECK(details.incomplete_reason == TILEDB_REASON_USER_BUFFER_SIZE); - } + tiledb_query_status_details_t details; + auto rc = tiledb_query_get_status_details( + ctx.ptr().get(), query2.ptr().get(), &details); + CHECK(rc == TILEDB_OK); + CHECK(details.incomplete_reason == TILEDB_REASON_USER_BUFFER_SIZE); array2.close(); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } } TEST_CASE( "Sparse global order reader: attribute copy memory limit", - "[sparse-global-order][attribute-copy][memory-limit]") { - std::string array_name = "test_sparse_global_order"; + "[sparse-global-order][attribute-copy][memory-limit][rest]") { Config config; config["sm.mem.total_budget"] = "10000"; - Context ctx(config); - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } + VFSTestSetup vfs_test_setup(config.ptr().get()); + std::string array_name = vfs_test_setup.array_uri("test_sparse_global_order"); + auto ctx = vfs_test_setup.ctx(); // Create array with var-sized attribute. Domain dom(ctx); @@ -1274,8 +1233,7 @@ TEST_CASE( query.set_data_buffer("d1", d1); query.set_data_buffer("a", a1_data); query.set_offsets_buffer("a", a1_offsets); - CHECK_NOTHROW(query.submit()); - CHECK_NOTHROW(query.finalize()); + CHECK_NOTHROW(query.submit_and_finalize()); // Read using a budget that can only fit one of the var size tiles. Array array2(ctx, array_name, TILEDB_READ); @@ -1304,10 +1262,6 @@ TEST_CASE( CHECK(result_num == 4); array2.close(); - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } } TEST_CASE_METHOD( diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index ed48e83a1df5..802a0342a4db 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -149,6 +149,15 @@ void vfs_test_create_temp_dir( CHECK(tiledb_vfs_create_dir(ctx, vfs, path.c_str()) == TILEDB_OK); } +std::string vfs_array_uri( + const std::unique_ptr& fs, const std::string& array_name) { + if (fs->is_rest()) { + return ("tiledb://unit/" + array_name); + } else { + return array_name; + } +} + Status SupportedFsS3::prepare_config( [[maybe_unused]] tiledb_config_t* config, [[maybe_unused]] tiledb_error_t* error) { diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index aeb3c7042923..16e0c18cdee7 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -95,6 +95,10 @@ void vfs_test_remove_temp_dir( void vfs_test_create_temp_dir( tiledb_ctx_t* ctx, tiledb_vfs_t* vfs, const std::string& path); + +std::string vfs_array_uri( + const std::unique_ptr& fs, const std::string& array_name); + /** * This class defines and manipulates * a list of supported filesystems. @@ -884,22 +888,36 @@ struct TemporaryDirectoryFixture { * { * tiledb::test::VFSTestSetup vfs_test_setup{"foo_array"}; * auto ctx = vfs_test_setup.ctx(); + * auto array_uri = vfs_test_setup.array_uri("quickstart_sparse"); * Array array(ctx, array_uri, TILEDB_WRITE); * ... * } // ctx context is destroyed and VFS directories removed * */ struct VFSTestSetup { - VFSTestSetup(tiledb_config_t* config = nullptr) + VFSTestSetup(tiledb_config_t* config = nullptr, bool remove_tmpdir = true) : fs_vec(vfs_test_get_fs_vec()) , ctx_c{nullptr} , vfs_c{nullptr} , cfg_c{config} { vfs_test_init(fs_vec, &ctx_c, &vfs_c, cfg_c).ok(); temp_dir = fs_vec[0]->temp_dir(); - vfs_test_create_temp_dir(ctx_c, vfs_c, temp_dir); + if (remove_tmpdir) { + vfs_test_remove_temp_dir(ctx_c, vfs_c, temp_dir); + } + tiledb_vfs_create_dir(ctx_c, vfs_c, temp_dir.c_str()); }; + void update_config(tiledb_config_t* config) { + // free resources + tiledb_ctx_free(&ctx_c); + tiledb_vfs_free(&vfs_c); + cfg_c = config; + + // reallocate with input config + vfs_test_init(fs_vec, &ctx_c, &vfs_c, cfg_c).ok(); + } + bool is_rest() { return fs_vec[0]->is_rest(); } @@ -908,10 +926,12 @@ struct VFSTestSetup { return fs_vec[0]->is_local(); } - std::string array_uri(const std::string& array_name) { - return ( - fs_vec[0]->is_rest() ? "tiledb://unit/" + temp_dir + array_name : - temp_dir + array_name); + std::string array_uri( + const std::string& array_name, bool strip_tiledb_prefix = false) { + auto uri = (fs_vec[0]->is_rest() && !strip_tiledb_prefix) ? + ("tiledb://unit/" + temp_dir + array_name) : + (temp_dir + array_name); + return uri; } Context ctx() { @@ -920,7 +940,7 @@ struct VFSTestSetup { ~VFSTestSetup() { vfs_test_remove_temp_dir(ctx_c, vfs_c, temp_dir); - CHECK(vfs_test_close(fs_vec, ctx_c, vfs_c).ok()); + vfs_test_close(fs_vec, ctx_c, vfs_c).ok(); tiledb_ctx_free(&ctx_c); tiledb_vfs_free(&vfs_c); From 690479ff47862ea89d70ad34fde62513323fea8f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 9 Apr 2024 22:58:20 +0300 Subject: [PATCH 277/456] Fix build errors with CMake 3.29.1. (#4859) CMake 3.29.1 had a change that broke vcpkg's `vcpkg_cmake_config_fixup` command. This PR updates the port containing the command to adapt to the new behavior. The change will be upstreamed with microsoft/vcpkg#38017. Validated locally. For ease of review, the first commit copies the existing port as-is, and the second commit updates it. Fixes #4857 Fixes TileDB-Inc/TileDB-CSharp#405 Fixes the root cause of TileDB-Inc/conda-forge-nightly-controller#83 --- TYPE: NO_HISTORY --- ports/README.md | 1 + ports/vcpkg-cmake-config/copyright | 23 ++ ports/vcpkg-cmake-config/portfile.cmake | 12 + .../vcpkg-port-config.cmake | 1 + ports/vcpkg-cmake-config/vcpkg.json | 6 + .../vcpkg_cmake_config_fixup.cmake | 266 ++++++++++++++++++ 6 files changed, 309 insertions(+) create mode 100644 ports/vcpkg-cmake-config/copyright create mode 100644 ports/vcpkg-cmake-config/portfile.cmake create mode 100644 ports/vcpkg-cmake-config/vcpkg-port-config.cmake create mode 100644 ports/vcpkg-cmake-config/vcpkg.json create mode 100644 ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake diff --git a/ports/README.md b/ports/README.md index 7faf08053d33..a03b41d7d668 100644 --- a/ports/README.md +++ b/ports/README.md @@ -34,3 +34,4 @@ After copying the port, add an entry to the table below. You should also contrib |`openssl`|Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024.| |`pcre2`|To be removed alongside libmagic.| |`azure-storage-common-cpp`|Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221).| +|`vcpkg-cmake-config`|Patching to fix build issues with CMake 3.29.1. (https://github.com/microsoft/vcpkg/pull/38017)| diff --git a/ports/vcpkg-cmake-config/copyright b/ports/vcpkg-cmake-config/copyright new file mode 100644 index 000000000000..2e4eac8264fa --- /dev/null +++ b/ports/vcpkg-cmake-config/copyright @@ -0,0 +1,23 @@ +Copyright (c) Microsoft Corporation + +All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ports/vcpkg-cmake-config/portfile.cmake b/ports/vcpkg-cmake-config/portfile.cmake new file mode 100644 index 000000000000..fc3dbafd5184 --- /dev/null +++ b/ports/vcpkg-cmake-config/portfile.cmake @@ -0,0 +1,12 @@ +if(NOT TARGET_TRIPLET STREQUAL _HOST_TRIPLET) + # make FATAL_ERROR in CI when issue #16773 fixed + message(WARNING "vcpkg-cmake-config is a host-only port; please mark it as a host port in your dependencies.") +endif() + +file(INSTALL + "${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake" + "${CMAKE_CURRENT_LIST_DIR}/vcpkg-port-config.cmake" + "${CMAKE_CURRENT_LIST_DIR}/copyright" + DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") + +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/ports/vcpkg-cmake-config/vcpkg-port-config.cmake b/ports/vcpkg-cmake-config/vcpkg-port-config.cmake new file mode 100644 index 000000000000..980d411315c7 --- /dev/null +++ b/ports/vcpkg-cmake-config/vcpkg-port-config.cmake @@ -0,0 +1 @@ +include("${CMAKE_CURRENT_LIST_DIR}/vcpkg_cmake_config_fixup.cmake") diff --git a/ports/vcpkg-cmake-config/vcpkg.json b/ports/vcpkg-cmake-config/vcpkg.json new file mode 100644 index 000000000000..df1da76c8f70 --- /dev/null +++ b/ports/vcpkg-cmake-config/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "vcpkg-cmake-config", + "version-date": "2024-04-07", + "documentation": "https://vcpkg.io/en/docs/README.html", + "license": "MIT" +} diff --git a/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake b/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake new file mode 100644 index 000000000000..246588129a91 --- /dev/null +++ b/ports/vcpkg-cmake-config/vcpkg_cmake_config_fixup.cmake @@ -0,0 +1,266 @@ +include_guard(GLOBAL) + +function(vcpkg_cmake_config_fixup) + cmake_parse_arguments(PARSE_ARGV 0 "arg" "DO_NOT_DELETE_PARENT_CONFIG_PATH;NO_PREFIX_CORRECTION" "PACKAGE_NAME;CONFIG_PATH;TOOLS_PATH" "") + + if(DEFINED arg_UNPARSED_ARGUMENTS) + message(FATAL_ERROR "vcpkg_cmake_config_fixup was passed extra arguments: ${arg_UNPARSED_ARGUMENTS}") + endif() + if(NOT arg_PACKAGE_NAME) + set(arg_PACKAGE_NAME "${PORT}") + endif() + if(NOT arg_CONFIG_PATH) + set(arg_CONFIG_PATH "share/${arg_PACKAGE_NAME}") + endif() + if(NOT arg_TOOLS_PATH) + set(arg_TOOLS_PATH "tools/${PORT}") + endif() + set(target_path "share/${arg_PACKAGE_NAME}") + + string(REPLACE "." "\\." EXECUTABLE_SUFFIX "${VCPKG_TARGET_EXECUTABLE_SUFFIX}") + + set(debug_share "${CURRENT_PACKAGES_DIR}/debug/${target_path}") + set(release_share "${CURRENT_PACKAGES_DIR}/${target_path}") + + if(NOT arg_CONFIG_PATH STREQUAL "share/${arg_PACKAGE_NAME}") + if(arg_CONFIG_PATH STREQUAL "share") + set(arg_CONFIG_PATH z_vcpkg_share) + file(RENAME "${CURRENT_PACKAGES_DIR}/debug/share" "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") + file(RENAME "${CURRENT_PACKAGES_DIR}/share" "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") + endif() + + set(debug_config "${CURRENT_PACKAGES_DIR}/debug/${arg_CONFIG_PATH}") + set(release_config "${CURRENT_PACKAGES_DIR}/${arg_CONFIG_PATH}") + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + if(NOT EXISTS "${debug_config}") + message(FATAL_ERROR "'${debug_config}' does not exist.") + endif() + + # This roundabout handling enables CONFIG_PATH = share + file(MAKE_DIRECTORY "${debug_share}") + file(GLOB files "${debug_config}/*") + file(COPY ${files} DESTINATION "${debug_share}") + file(REMOVE_RECURSE "${debug_config}") + endif() + + file(GLOB files "${release_config}/*") + file(COPY ${files} DESTINATION "${release_share}") + file(REMOVE_RECURSE "${release_config}") + + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + get_filename_component(debug_config_dir_name "${debug_config}" NAME) + string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) + if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) + file(REMOVE_RECURSE "${debug_config}") + else() + get_filename_component(debug_config_parent_dir "${debug_config}" DIRECTORY) + get_filename_component(debug_config_dir_name "${debug_config_parent_dir}" NAME) + string(TOLOWER "${debug_config_dir_name}" debug_config_dir_name) + if(debug_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) + file(REMOVE_RECURSE "${debug_config_parent_dir}") + endif() + endif() + endif() + + get_filename_component(release_config_dir_name "${release_config}" NAME) + string(TOLOWER "${release_config_dir_name}" release_config_dir_name) + if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) + file(REMOVE_RECURSE "${release_config}") + else() + get_filename_component(release_config_parent_dir "${release_config}" DIRECTORY) + get_filename_component(release_config_dir_name "${release_config_parent_dir}" NAME) + string(TOLOWER "${release_config_dir_name}" release_config_dir_name) + if(release_config_dir_name STREQUAL "cmake" AND NOT arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) + file(REMOVE_RECURSE "${release_config_parent_dir}") + endif() + endif() + endif() + + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + if(NOT EXISTS "${debug_share}") + message(FATAL_ERROR "'${debug_share}' does not exist.") + endif() + endif() + + file(GLOB_RECURSE release_targets + "${release_share}/*-release.cmake" + ) + foreach(release_target IN LISTS release_targets) + file(READ "${release_target}" contents) + string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") + string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \"]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") + file(WRITE "${release_target}" "${contents}") + endforeach() + + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + file(GLOB_RECURSE debug_targets + "${debug_share}/*-debug.cmake" + ) + foreach(debug_target IN LISTS debug_targets) + file(RELATIVE_PATH debug_target_rel "${debug_share}" "${debug_target}") + + file(READ "${debug_target}" contents) + string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") + string(REGEX REPLACE "\\\${_IMPORT_PREFIX}/bin/([^ \";]+${EXECUTABLE_SUFFIX})" "\${_IMPORT_PREFIX}/${arg_TOOLS_PATH}/\\1" contents "${contents}") + string(REPLACE "\${_IMPORT_PREFIX}/lib" "\${_IMPORT_PREFIX}/debug/lib" contents "${contents}") + string(REPLACE "\${_IMPORT_PREFIX}/bin" "\${_IMPORT_PREFIX}/debug/bin" contents "${contents}") + file(WRITE "${release_share}/${debug_target_rel}" "${contents}") + + file(REMOVE "${debug_target}") + endforeach() + endif() + + #Fix ${_IMPORT_PREFIX} and absolute paths in cmake generated targets and configs; + #Since those can be renamed we have to check in every *.cmake, but only once. + file(GLOB_RECURSE main_cmakes "${release_share}/*.cmake") + if(NOT DEFINED Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) + vcpkg_list(SET Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) + endif() + foreach(already_fixed_up IN LISTS Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP) + vcpkg_list(REMOVE_ITEM main_cmakes "${already_fixed_up}") + endforeach() + vcpkg_list(APPEND Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP ${main_cmakes}) + set(Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP "${Z_VCPKG_CMAKE_CONFIG_ALREADY_FIXED_UP}" CACHE INTERNAL "") + + foreach(main_cmake IN LISTS main_cmakes) + file(READ "${main_cmake}" contents) + # Note: I think the following comment is no longer true, since we now require the path to be `share/blah` + # however, I don't know it for sure. + # - nimazzuc + + #This correction is not correct for all cases. To make it correct for all cases it needs to consider + #original folder deepness to CURRENT_PACKAGES_DIR in comparison to the moved to folder deepness which + #is always at least (>=) 2, e.g. share/${PORT}. Currently the code assumes it is always 2 although + #this requirement is only true for the *Config.cmake. The targets are not required to be in the same + #folder as the *Config.cmake! + if(NOT arg_NO_PREFIX_CORRECTION) + string(REGEX REPLACE +[[get_filename_component\(_IMPORT_PREFIX "\${CMAKE_CURRENT_LIST_FILE}" PATH\)( +get_filename_component\(_IMPORT_PREFIX "\${_IMPORT_PREFIX}" PATH\))*]] +[[get_filename_component(_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH) +get_filename_component(_IMPORT_PREFIX "${_IMPORT_PREFIX}" PATH)]] + contents "${contents}") # see #1044 for details why this replacement is necessary. See #4782 why it must be a regex. + string(REGEX REPLACE +[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\./(\.\./)*" ABSOLUTE\)]] +[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] + contents "${contents}") + string(REGEX REPLACE +[[(get_filename_component\(PACKAGE_\${CMAKE_FIND_PACKAGE_NAME}_COUNTER_[1-9][0-9]*) "\${CMAKE_CURRENT_LIST_DIR}/\.\./(\.\./)*" ABSOLUTE\)]] +"\\1 \"\${CMAKE_CURRENT_LIST_DIR}/../../\" ABSOLUTE)" # adapt to https://gitlab.kitware.com/cmake/cmake/-/merge_requests/9390 + contents "${contents}") + string(REGEX REPLACE +[[get_filename_component\(PACKAGE_PREFIX_DIR "\${CMAKE_CURRENT_LIST_DIR}/\.\.((\\|/)\.\.)*" ABSOLUTE\)]] +[[get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../" ABSOLUTE)]] + contents "${contents}") # This is a meson-related workaround, see https://github.com/mesonbuild/meson/issues/6955 + string(REGEX REPLACE +[[(get_filename_component\(PACKAGE_\${CMAKE_FIND_PACKAGE_NAME}_COUNTER_[1-9][0-9]*) "\${CMAKE_CURRENT_LIST_DIR}/\.\.((\\|/)\.\.)*" ABSOLUTE\)]] +"\\1 \"\${CMAKE_CURRENT_LIST_DIR}/../../\" ABSOLUTE)" + contents "${contents}") + endif() + + # Merge release and debug configurations of target property INTERFACE_LINK_LIBRARIES. + string(REPLACE "${release_share}/" "${debug_share}/" debug_cmake "${main_cmake}") + if(DEFINED VCPKG_BUILD_TYPE) + # Skip. Warning: A release-only port in a dual-config installation + # may pull release dependencies into the debug configuration. + elseif(NOT contents MATCHES "INTERFACE_LINK_LIBRARIES") + # Skip. No relevant properties. + elseif(NOT contents MATCHES "# Generated CMake target import file\\.") + # Skip. No safe assumptions about a matching debug import file. + elseif(NOT EXISTS "${debug_cmake}") + message(SEND_ERROR "Did not find a debug import file matching '${main_cmake}'") + else() + file(READ "${debug_cmake}" debug_contents) + while(contents MATCHES "set_target_properties\\(([^ \$]*) PROPERTIES[^)]*\\)") + set(matched_command "${CMAKE_MATCH_0}") + string(REPLACE "+" "\\+" target "${CMAKE_MATCH_1}") + if(NOT debug_contents MATCHES "set_target_properties\\(${target} PROPERTIES[^)]*\\)") + message(SEND_ERROR "Did not find a debug configuration for target '${target}'.") + endif() + set(debug_command "${CMAKE_MATCH_0}") + string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" release_line "${matched_command}") + set(release_libs "${CMAKE_MATCH_1}") + string(REGEX MATCH " INTERFACE_LINK_LIBRARIES \"([^\"]*)\"" debug_line "${debug_command}") + set(debug_libs "${CMAKE_MATCH_1}") + z_vcpkg_cmake_config_fixup_merge(merged_libs release_libs debug_libs) + string(REPLACE "${release_line}" " INTERFACE_LINK_LIBRARIES \"${merged_libs}\"" updated_command "${matched_command}") + string(REPLACE "set_target_properties" "set_target_properties::done" updated_command "${updated_command}") # Prevend 2nd match + string(REPLACE "${matched_command}" "${updated_command}" contents "${contents}") + endwhile() + string(REPLACE "set_target_properties::done" "set_target_properties" contents "${contents}") # Restore original command + endif() + + #Fix absolute paths to installed dir with ones relative to ${CMAKE_CURRENT_LIST_DIR} + #This happens if vcpkg built libraries are directly linked to a target instead of using + #an imported target. + string(REPLACE "${CURRENT_INSTALLED_DIR}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") + file(TO_CMAKE_PATH "${CURRENT_PACKAGES_DIR}" cmake_current_packages_dir) + string(REPLACE "${cmake_current_packages_dir}" [[${VCPKG_IMPORT_PREFIX}]] contents "${contents}") + # If ${VCPKG_IMPORT_PREFIX} was actually used, inject a definition of it: + string(FIND "${contents}" [[${VCPKG_IMPORT_PREFIX}]] index) + if (NOT index STREQUAL "-1") + get_filename_component(main_cmake_dir "${main_cmake}" DIRECTORY) + # Calculate relative to be a sequence of "../" + file(RELATIVE_PATH relative "${main_cmake_dir}" "${cmake_current_packages_dir}") + string(PREPEND contents "get_filename_component(VCPKG_IMPORT_PREFIX \"\${CMAKE_CURRENT_LIST_DIR}\/${relative}\" ABSOLUTE)\n") + endif() + + file(WRITE "${main_cmake}" "${contents}") + endforeach() + + file(GLOB_RECURSE unused_files + "${debug_share}/*[Tt]argets.cmake" + "${debug_share}/*[Cc]onfig.cmake" + "${debug_share}/*[Cc]onfigVersion.cmake" + "${debug_share}/*[Cc]onfig-version.cmake" + ) + foreach(unused_file IN LISTS unused_files) + file(REMOVE "${unused_file}") + endforeach() + + # Remove /debug// if it's empty. + file(GLOB_RECURSE remaining_files "${debug_share}/*") + if(remaining_files STREQUAL "") + file(REMOVE_RECURSE "${debug_share}") + endif() + + # Remove /debug/share/ if it's empty. + file(GLOB_RECURSE remaining_files "${CURRENT_PACKAGES_DIR}/debug/share/*") + if(remaining_files STREQUAL "") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + endif() +endfunction() + +# Merges link interface library lists for release and debug +# into a single expression which use generator expression as necessary. +function(z_vcpkg_cmake_config_fixup_merge out_var release_var debug_var) + set(release_libs "VCPKG;${${release_var}}") + string(REGEX REPLACE ";optimized;([^;]*)" ";\\1" release_libs "${release_libs}") + string(REGEX REPLACE ";debug;([^;]*)" ";" release_libs "${release_libs}") + list(REMOVE_AT release_libs 0) + list(FILTER release_libs EXCLUDE REGEX [[^\\[$]<\\[$]:]]) + list(TRANSFORM release_libs REPLACE [[^\\[$]<\\[$]>:(.*)>$]] "\\1") + + set(debug_libs "VCPKG;${${debug_var}}") + string(REGEX REPLACE ";optimized;([^;]*)" ";" debug_libs "${debug_libs}") + string(REGEX REPLACE ";debug;([^;]*)" ";\\1" debug_libs "${debug_libs}") + list(REMOVE_AT debug_libs 0) + list(FILTER debug_libs EXCLUDE REGEX [[^\\[$]<\\[$]>:]]) + list(TRANSFORM debug_libs REPLACE [[^\\[$]<\\[$]:(.*)>$]] "\\1") + + set(merged_libs "") + foreach(release_lib debug_lib IN ZIP_LISTS release_libs debug_libs) + if(release_lib STREQUAL debug_lib) + list(APPEND merged_libs "${release_lib}") + else() + if(release_lib) + list(APPEND merged_libs "\\\$<\\\$>:${release_lib}>") + endif() + if(debug_lib) + list(APPEND merged_libs "\\\$<\\\$:${debug_lib}>") + endif() + endif() + endforeach() + set("${out_var}" "${merged_libs}" PARENT_SCOPE) +endfunction() From bc6d9dca5c6fcfbbb6782607c8bac5f002f5a796 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:49:59 +0200 Subject: [PATCH 278/456] Increase HDFS timeout in nightlies. (#4861) We added more tests to REST, which ends up increasing the number of tests running against HDFS. --- TYPE: NO_HISTORY DESC: Increase HDFS timeout in nightlies. --- .github/workflows/nightly-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index d53b0d50fafe..169963ad3bcc 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -72,7 +72,7 @@ jobs: matrix_image: ubuntu-22.04 matrix_compiler_cc: 'gcc-13' matrix_compiler_cxx: 'g++-13' - timeout: 180 + timeout: 300 bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' create_issue_on_fail: From 8869d179e944d62c593d8af4136e022a0a2fc16f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 10 Apr 2024 17:13:26 +0300 Subject: [PATCH 279/456] Support GCS service account impersonation et. al. (#4799) [SC-42590](https://app.shortcut.com/tiledb-inc/story/42590) [SC-42830](https://app.shortcut.com/tiledb-inc/story/42830) This PR adds support for [Google Cloud service account impersonation](https://cloud.google.com/docs/authentication/use-service-account-impersonation) to the GCS VFS. This is enabled with a new config option `vfs.gcs.impersonate_service_account` that also supports chained impersonation if given a comma-separated list of service accounts. Because we were using some older APIs to configure the GCS client and these don't support impersonation, I also switched to the newer APIs. The migration process was straightforward. For ease of review, you are suggested to look at each commit individually. --- TYPE: CONFIG DESC: Add `vfs.gcs.impersonate_service_account` option that specifies a service account to impersonate, or a comma-separated list for chained impersonation. --- TYPE: IMPROVEMENT DESC: Stop using deprecated Google Cloud SDK APIs. --------- Co-authored-by: Isaiah Norton --- test/src/unit-capi-config.cc | 3 + test/src/unit-cppapi-config.cc | 2 +- test/src/unit-vfs.cc | 53 +++++++++ tiledb/api/c_api/config/config_api_external.h | 5 + tiledb/sm/config/config.cc | 5 + tiledb/sm/config/config.h | 3 + tiledb/sm/cpp_api/config.h | 5 + tiledb/sm/filesystem/gcs.cc | 112 +++++++++++++----- tiledb/sm/filesystem/gcs.h | 24 +++- 9 files changed, 178 insertions(+), 34 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 6418fdcc7b77..2b48573bfc2e 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -690,6 +690,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["vfs.read_logging_mode"] = ""; all_param_values["vfs.gcs.endpoint"] = ""; all_param_values["vfs.gcs.project_id"] = ""; + all_param_values["vfs.gcs.impersonate_service_account"] = ""; all_param_values["vfs.gcs.max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); all_param_values["vfs.gcs.multi_part_size"] = "5242880"; @@ -760,6 +761,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { vfs_param_values["read_logging_mode"] = ""; vfs_param_values["gcs.endpoint"] = ""; vfs_param_values["gcs.project_id"] = ""; + vfs_param_values["gcs.impersonate_service_account"] = ""; vfs_param_values["gcs.max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); vfs_param_values["gcs.multi_part_size"] = "5242880"; @@ -823,6 +825,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { std::map gcs_param_values; gcs_param_values["endpoint"] = ""; gcs_param_values["project_id"] = ""; + gcs_param_values["impersonate_service_account"] = ""; gcs_param_values["max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); gcs_param_values["multi_part_size"] = "5242880"; diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 7be58bb59d32..4334db9e90ec 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -60,7 +60,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") { names.push_back(it->first); } // Check number of VFS params in default config object. - CHECK(names.size() == 65); + CHECK(names.size() == 66); } TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") { diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 939bfd58e6d6..e0fcf2160f22 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -37,6 +37,11 @@ #include #include "tiledb/sm/filesystem/azure.h" #endif +#ifdef HAVE_GCS +#include +#include +#include "tiledb/sm/filesystem/gcs.h" +#endif #include "test/support/src/vfs_helpers.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/global_state/unit_test_config.h" @@ -750,3 +755,51 @@ TEST_CASE("Validate vfs.s3.custom_headers.*", "[s3][custom-headers]") { REQUIRE_THROWS_WITH(s3.flush_object(uri), matcher); } #endif + +#ifdef HAVE_GCS +TEST_CASE( + "Validate GCS service account impersonation", "[gcs][impersonation]") { + ThreadPool thread_pool(2); + Config cfg = set_config_params(true); + GCS gcs; + std::string impersonate_service_account, target_service_account; + std::vector delegates; + + SECTION("Simple") { + impersonate_service_account = "account1"; + target_service_account = "account1"; + delegates = {}; + } + + SECTION("Delegated") { + impersonate_service_account = "account1,account2,account3"; + target_service_account = "account3"; + delegates = {"account1", "account2"}; + } + + // Test parsing an edge case. + SECTION("Invalid") { + impersonate_service_account = ","; + target_service_account = ""; + delegates = {""}; + } + + require_tiledb_ok(cfg.set( + "vfs.gcs.impersonate_service_account", impersonate_service_account)); + + require_tiledb_ok(gcs.init(cfg, &thread_pool)); + + auto credentials = gcs.make_credentials({}); + + // We are using an internal class only for inspection purposes. + auto impersonate_credentials = + dynamic_cast( + credentials.get()); + + REQUIRE(impersonate_credentials != nullptr); + REQUIRE( + impersonate_credentials->target_service_account() == + target_service_account); + REQUIRE(impersonate_credentials->delegates() == delegates); +} +#endif diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 3401f8ebbc6b..2e70e937fc5c 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -391,6 +391,11 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `vfs.gcs.project_id`
* Set the GCS project id.
* **Default**: "" + * - `vfs.gcs.impersonate_service_account`
+ * Set the GCS service account to impersonate. A chain of impersonated + * accounts can be formed by specifying many service accounts, separated by a + * comma.
+ * **Default**: "" * - `vfs.gcs.multi_part_size`
* The part size (in bytes) used in GCS multi part writes. * Any `uint64_t` value is acceptable. Note: diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 6afe3ad5c4ce..c443af7666e3 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -185,6 +185,7 @@ const std::string Config::VFS_AZURE_RETRY_DELAY_MS = "800"; const std::string Config::VFS_AZURE_MAX_RETRY_DELAY_MS = "60000"; const std::string Config::VFS_GCS_ENDPOINT = ""; const std::string Config::VFS_GCS_PROJECT_ID = ""; +const std::string Config::VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT = ""; const std::string Config::VFS_GCS_MAX_PARALLEL_OPS = Config::SM_IO_CONCURRENCY_LEVEL; const std::string Config::VFS_GCS_MULTI_PART_SIZE = "5242880"; @@ -420,6 +421,9 @@ const std::map default_config_values = { "vfs.azure.max_retry_delay_ms", Config::VFS_AZURE_MAX_RETRY_DELAY_MS), std::make_pair("vfs.gcs.endpoint", Config::VFS_GCS_ENDPOINT), std::make_pair("vfs.gcs.project_id", Config::VFS_GCS_PROJECT_ID), + std::make_pair( + "vfs.gcs.impersonate_service_account", + Config::VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT), std::make_pair( "vfs.gcs.max_parallel_ops", Config::VFS_GCS_MAX_PARALLEL_OPS), std::make_pair("vfs.gcs.multi_part_size", Config::VFS_GCS_MULTI_PART_SIZE), @@ -509,6 +513,7 @@ const std::set Config::unserialized_params_ = { "vfs.s3.aws_external_id", "vfs.s3.aws_load_frequency", "vfs.s3.aws_session_name", + "vfs.gcs.impersonate_service_account", "rest.username", "rest.password", "rest.token", diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 5754effc26cf..4667bbd91008 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -457,6 +457,9 @@ class Config { /** GCS project id. */ static const std::string VFS_GCS_PROJECT_ID; + /** GCS service account(s) to impersonate. */ + static const std::string VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT; + /** GCS max parallel ops. */ static const std::string VFS_GCS_MAX_PARALLEL_OPS; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index f145892056f4..c4b124ae74d9 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -569,6 +569,11 @@ class Config { * - `vfs.gcs.project_id`
* Set the GCS project id.
* **Default**: "" + * - `vfs.gcs.impersonate_service_account`
+ * Set the GCS service account to impersonate. A chain of impersonated + * accounts can be formed by specifying many service accounts, separated by + * a comma.
+ * **Default**: "" * - `vfs.gcs.multi_part_size`
* The part size (in bytes) used in GCS multi part writes. * Any `uint64_t` value is acceptable. Note: diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index b8b145f7c934..54383460fdad 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -102,6 +102,9 @@ Status GCS::init(const Config& config, ThreadPool* const thread_pool) { } project_id_ = config.get("vfs.gcs.project_id", &found); assert(found); + impersonate_service_account_ = + config.get("vfs.gcs.impersonate_service_account", &found); + assert(found); RETURN_NOT_OK(config.get( "vfs.gcs.max_parallel_ops", &max_parallel_ops_, &found)); assert(found); @@ -127,20 +130,83 @@ Status GCS::init(const Config& config, ThreadPool* const thread_pool) { return Status::Ok(); } +/** + * Builds a chain of service account impersonation credentials. + * + * @param credentials The set of credentials to start the chain. + * @param service_accounts A comma-separated list of service accounts, where + * each account will be used to impersonate the next. + * @options Options to set to the credentials. + * @return The new set of credentials. + */ +static shared_ptr apply_impersonation( + shared_ptr credentials, + std::string service_accounts, + google::cloud::Options options) { + if (service_accounts.empty()) { + return credentials; + } + auto last_comma_pos = service_accounts.rfind(','); + // If service_accounts is a comma-separated list, we have to extract the first + // items to a vector and pass them via DelegatesOption, and pass only the last + // account to MakeImpersonateServiceAccountCredentials. + if (last_comma_pos != std::string_view::npos) { + // Create a view over all service accounts except the last one. + auto delegates_str = + std::string_view(service_accounts).substr(0, last_comma_pos); + std::vector delegates; + while (true) { + auto comma_pos = delegates_str.find(','); + // Get the characters before the comma. We don't have to check for npos + // yet; substr will trim the size if it is too big. + delegates.push_back(std::string(delegates_str.substr(0, comma_pos))); + if (comma_pos != std::string_view::npos) { + // If there is another comma, discard it and the characters before it. + delegates_str = delegates_str.substr(comma_pos + 1); + } else { + // Otherwise exit the loop; we have processed all intermediate service + // accounts. + break; + } + } + options.set(std::move(delegates)); + // Trim service_accounts to its last member. + service_accounts = service_accounts.substr(last_comma_pos + 1); + } + // If service_accounts had any comas, by now it should be left to just the + // last part. + if (service_accounts.find(',') != std::string::npos) { + throw std::logic_error( + "Internal error: service_accounts string was not decomposed."); + } + // Create the credential. + return google::cloud::MakeImpersonateServiceAccountCredentials( + std::move(credentials), std::move(service_accounts), std::move(options)); +} + +std::shared_ptr GCS::make_credentials( + const google::cloud::Options& options) const { + shared_ptr creds = nullptr; + if (!endpoint_.empty() || getenv("CLOUD_STORAGE_EMULATOR_ENDPOINT")) { + creds = google::cloud::MakeInsecureCredentials(); + } else { + creds = google::cloud::MakeGoogleDefaultCredentials(options); + } + return apply_impersonation(creds, impersonate_service_account_, options); +} + Status GCS::init_client() const { assert(state_ == State::INITIALIZED); std::lock_guard lck(client_init_mtx_); - // Client is a google::cloud::storage::StatusOr which compares (in)valid as - // bool if (client_) { return Status::Ok(); } - google::cloud::storage::ChannelOptions channel_options; + google::cloud::Options ca_options; if (!ssl_cfg_.ca_file().empty()) { - channel_options.set_ssl_root_path(ssl_cfg_.ca_file()); + ca_options.set(ssl_cfg_.ca_file()); } if (!ssl_cfg_.ca_path().empty()) { @@ -150,43 +216,27 @@ Status GCS::init_client() const { } // Note that the order here is *extremely important* - // We must call ::GoogleDefaultCredentials *with* a channel_options + // We must call make_credentials *with* a ca_options // argument, or else the Curl handle pool will be default-initialized // with no root dir (CURLOPT_CAINFO), defaulting to build host path. - // Later initializations of ClientOptions/Client with the channel_options + // Later initializations of ClientOptions/Client with the ca_options // do not appear to sufficiently reset the internal option, leading to // CA verification failures when using lib from systemA on systemB. - // Ideally we could use CreateDefaultClientOptions(channel_options) - // signature, but that function is header-only/unimplemented - // (as of GCS 1.15). // Creates the client using the credentials file pointed to by the // env variable GOOGLE_APPLICATION_CREDENTIALS try { - shared_ptr creds = nullptr; - if (!endpoint_.empty() || getenv("CLOUD_STORAGE_EMULATOR_ENDPOINT")) { - creds = google::cloud::storage::oauth2::CreateAnonymousCredentials(); - } else { - auto status_or_creds = - google::cloud::storage::oauth2::GoogleDefaultCredentials( - channel_options); - if (!status_or_creds) { - return LOG_STATUS(Status_GCSError( - "Failed to initialize GCS credentials: " + - status_or_creds.status().message())); - } - creds = *status_or_creds; - } - google::cloud::storage::ClientOptions client_options( - creds, channel_options); + auto client_options = ca_options; + client_options.set( + make_credentials(ca_options)); if (!endpoint_.empty()) { - client_options.set_endpoint(endpoint_); + client_options.set(endpoint_); } - client_ = tdb_unique_ptr(tdb_new( - google::cloud::storage::Client, - client_options, - google::cloud::storage::LimitedTimeRetryPolicy( - std::chrono::milliseconds(request_timeout_ms_)))); + client_options.set( + make_shared( + HERE(), std::chrono::milliseconds(request_timeout_ms_))); + client_ = tdb_unique_ptr( + tdb_new(google::cloud::storage::Client, client_options)); } catch (const std::exception& e) { return LOG_STATUS( Status_GCSError("Failed to initialize GCS: " + std::string(e.what()))); diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index 30cdc4268a44..9a2b83bbaf4f 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -49,11 +49,17 @@ using namespace tiledb::common; -namespace google::cloud::storage { +namespace google::cloud { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +class Credentials; +class Options; +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +namespace storage { GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN class Client; GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END -} // namespace google::cloud::storage +} // namespace storage +} // namespace google::cloud namespace tiledb { @@ -309,6 +315,17 @@ class GCS { */ Status flush_object(const URI& uri); + /** + * Creates a GCS credentials object. + * + * This method is intended to be used by testing code only. + * + * @param options Options to configure the credentials. + * @return shared pointer to credentials + */ + std::shared_ptr make_credentials( + const google::cloud::Options& options) const; + private: /* ********************************* */ /* PRIVATE DATATYPES */ @@ -426,6 +443,9 @@ class GCS { // The GCS project id. std::string project_id_; + // A comma-separated list with the GCS service accounts to impersonate. + std::string impersonate_service_account_; + // The GCS REST client. mutable tdb_unique_ptr client_; From 0ae11e2014d5bf059f6cb8adf5e59ae08887f3b7 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 11 Apr 2024 15:26:22 +0300 Subject: [PATCH 280/456] Support specifying GCP account credentials as a config option. (#4855) [SC-44515](https://app.shortcut.com/tiledb-inc/story/44515/support-specifying-gcp-account-credentials-as-a-config-option) --- TYPE: CONFIG DESC: Add `vfs.gcs.service_account_credential` config option that specifies a Google Cloud service account credential JSON string. --- TYPE: CONFIG DESC: Add `vfs.gcs.external_account_credential` config option that specifies a Google Cloud Workload Identity Federation credential JSON string. --- test/src/unit-capi-config.cc | 6 ++ test/src/unit-cppapi-config.cc | 2 +- test/src/unit-vfs.cc | 94 ++++++++++++++++++- tiledb/api/c_api/config/config_api_external.h | 11 +++ tiledb/sm/config/config.cc | 9 ++ tiledb/sm/config/config.h | 6 ++ tiledb/sm/cpp_api/config.h | 11 +++ tiledb/sm/filesystem/gcs.cc | 18 +++- tiledb/sm/filesystem/gcs.h | 6 ++ 9 files changed, 160 insertions(+), 3 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 2b48573bfc2e..b0d3c63c5827 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -690,6 +690,8 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["vfs.read_logging_mode"] = ""; all_param_values["vfs.gcs.endpoint"] = ""; all_param_values["vfs.gcs.project_id"] = ""; + all_param_values["vfs.gcs.service_account_key"] = ""; + all_param_values["vfs.gcs.workload_identity_configuration"] = ""; all_param_values["vfs.gcs.impersonate_service_account"] = ""; all_param_values["vfs.gcs.max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); @@ -761,6 +763,8 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { vfs_param_values["read_logging_mode"] = ""; vfs_param_values["gcs.endpoint"] = ""; vfs_param_values["gcs.project_id"] = ""; + vfs_param_values["gcs.service_account_key"] = ""; + vfs_param_values["gcs.workload_identity_configuration"] = ""; vfs_param_values["gcs.impersonate_service_account"] = ""; vfs_param_values["gcs.max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); @@ -825,6 +829,8 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { std::map gcs_param_values; gcs_param_values["endpoint"] = ""; gcs_param_values["project_id"] = ""; + gcs_param_values["service_account_key"] = ""; + gcs_param_values["workload_identity_configuration"] = ""; gcs_param_values["impersonate_service_account"] = ""; gcs_param_values["max_parallel_ops"] = std::to_string(std::thread::hardware_concurrency()); diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 4334db9e90ec..18504404d556 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -60,7 +60,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") { names.push_back(it->first); } // Check number of VFS params in default config object. - CHECK(names.size() == 66); + CHECK(names.size() == 68); } TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") { diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index e0fcf2160f22..95008baf51a7 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -758,7 +758,8 @@ TEST_CASE("Validate vfs.s3.custom_headers.*", "[s3][custom-headers]") { #ifdef HAVE_GCS TEST_CASE( - "Validate GCS service account impersonation", "[gcs][impersonation]") { + "Validate GCS service account impersonation", + "[gcs][credentials][impersonation]") { ThreadPool thread_pool(2); Config cfg = set_config_params(true); GCS gcs; @@ -802,4 +803,95 @@ TEST_CASE( target_service_account); REQUIRE(impersonate_credentials->delegates() == delegates); } + +TEST_CASE( + "Validate GCS service account credentials", + "[gcs][credentials][service-account]") { + ThreadPool thread_pool(2); + Config cfg = set_config_params(true); + GCS gcs; + // The content of the credentials does not matter; it does not get parsed + // until it is used in an API request, which we are not doing. + std::string service_account_key = "{\"foo\": \"bar\"}"; + + require_tiledb_ok( + cfg.set("vfs.gcs.service_account_key", service_account_key)); + + require_tiledb_ok(gcs.init(cfg, &thread_pool)); + + auto credentials = gcs.make_credentials({}); + + // We are using an internal class only for inspection purposes. + auto service_account = + dynamic_cast( + credentials.get()); + + REQUIRE(service_account != nullptr); + REQUIRE(service_account->json_object() == service_account_key); +} + +TEST_CASE( + "Validate GCS service account credentials with impersonation", + "[gcs][credentials][service-account-and-impersonation]") { + ThreadPool thread_pool(2); + Config cfg = set_config_params(true); + GCS gcs; + // The content of the credentials does not matter; it does not get parsed + // until it is used in an API request, which we are not doing. + std::string service_account_key = "{\"foo\": \"bar\"}"; + std::string impersonate_service_account = "account1,account2,account3"; + + require_tiledb_ok( + cfg.set("vfs.gcs.service_account_key", service_account_key)); + require_tiledb_ok(cfg.set( + "vfs.gcs.impersonate_service_account", impersonate_service_account)); + + require_tiledb_ok(gcs.init(cfg, &thread_pool)); + + auto credentials = gcs.make_credentials({}); + + // We are using an internal class only for inspection purposes. + auto impersonate_credentials = + dynamic_cast( + credentials.get()); + REQUIRE(impersonate_credentials != nullptr); + REQUIRE(impersonate_credentials->target_service_account() == "account3"); + REQUIRE( + impersonate_credentials->delegates() == + std::vector{"account1", "account2"}); + + auto inner_service_account = + dynamic_cast( + impersonate_credentials->base_credentials().get()); + + REQUIRE(inner_service_account != nullptr); + REQUIRE(inner_service_account->json_object() == service_account_key); +} + +TEST_CASE( + "Validate GCS external account credentials", + "[gcs][credentials][external-account]") { + ThreadPool thread_pool(2); + Config cfg = set_config_params(true); + GCS gcs; + // The content of the credentials does not matter; it does not get parsed + // until it is used in an API request, which we are not doing. + std::string workload_identity_configuration = "{\"foo\": \"bar\"}"; + + require_tiledb_ok(cfg.set( + "vfs.gcs.workload_identity_configuration", + workload_identity_configuration)); + + require_tiledb_ok(gcs.init(cfg, &thread_pool)); + + auto credentials = gcs.make_credentials({}); + + // We are using an internal class only for inspection purposes. + auto external_account = + dynamic_cast( + credentials.get()); + + REQUIRE(external_account != nullptr); + REQUIRE(external_account->json_object() == workload_identity_configuration); +} #endif diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 2e70e937fc5c..655089bcc3d7 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -391,6 +391,17 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `vfs.gcs.project_id`
* Set the GCS project id.
* **Default**: "" + * - `vfs.gcs.service_account_key`
+ * Set the JSON string with GCS service account key. Takes precedence + * over `vfs.gcs.workload_identity_configuration` if both are specified. If + * neither is specified, Application Default Credentials will be used.
+ * **Default**: "" + * - `vfs.gcs.workload_identity_configuration`
+ * Set the JSON string with Workload Identity Federation configuration. + * `vfs.gcs.service_account_key` takes precedence over this if both are + * specified. If neither is specified, Application Default Credentials will + * be used.
+ * **Default**: "" * - `vfs.gcs.impersonate_service_account`
* Set the GCS service account to impersonate. A chain of impersonated * accounts can be formed by specifying many service accounts, separated by a diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index c443af7666e3..e669cd02c606 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -185,6 +185,8 @@ const std::string Config::VFS_AZURE_RETRY_DELAY_MS = "800"; const std::string Config::VFS_AZURE_MAX_RETRY_DELAY_MS = "60000"; const std::string Config::VFS_GCS_ENDPOINT = ""; const std::string Config::VFS_GCS_PROJECT_ID = ""; +const std::string Config::VFS_GCS_SERVICE_ACCOUNT_KEY = ""; +const std::string Config::VFS_GCS_WORKLOAD_IDENTITY_CONFIGURATION = ""; const std::string Config::VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT = ""; const std::string Config::VFS_GCS_MAX_PARALLEL_OPS = Config::SM_IO_CONCURRENCY_LEVEL; @@ -421,6 +423,11 @@ const std::map default_config_values = { "vfs.azure.max_retry_delay_ms", Config::VFS_AZURE_MAX_RETRY_DELAY_MS), std::make_pair("vfs.gcs.endpoint", Config::VFS_GCS_ENDPOINT), std::make_pair("vfs.gcs.project_id", Config::VFS_GCS_PROJECT_ID), + std::make_pair( + "vfs.gcs.service_account_key", Config::VFS_GCS_SERVICE_ACCOUNT_KEY), + std::make_pair( + "vfs.gcs.workload_identity_configuration", + Config::VFS_GCS_WORKLOAD_IDENTITY_CONFIGURATION), std::make_pair( "vfs.gcs.impersonate_service_account", Config::VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT), @@ -513,6 +520,8 @@ const std::set Config::unserialized_params_ = { "vfs.s3.aws_external_id", "vfs.s3.aws_load_frequency", "vfs.s3.aws_session_name", + "vfs.gcs.service_account_key", + "vfs.gcs.workload_identity_configuration", "vfs.gcs.impersonate_service_account", "rest.username", "rest.password", diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 4667bbd91008..bd93bce3e8fd 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -460,6 +460,12 @@ class Config { /** GCS service account(s) to impersonate. */ static const std::string VFS_GCS_IMPERSONATE_SERVICE_ACCOUNT; + /** GCS service account key JSON string. */ + static const std::string VFS_GCS_SERVICE_ACCOUNT_KEY; + + /** GCS external account credentials JSON string. */ + static const std::string VFS_GCS_WORKLOAD_IDENTITY_CONFIGURATION; + /** GCS max parallel ops. */ static const std::string VFS_GCS_MAX_PARALLEL_OPS; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index c4b124ae74d9..927118e10d29 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -569,6 +569,17 @@ class Config { * - `vfs.gcs.project_id`
* Set the GCS project id.
* **Default**: "" + * - `vfs.gcs.service_account_key`
+ * Set the JSON string with GCS service account key. Takes precedence + * over `vfs.gcs.workload_identity_configuration` if both are specified. If + * neither is specified, Application Default Credentials will be used.
+ * **Default**: "" + * - `vfs.gcs.workload_identity_configuration`
+ * Set the JSON string with Workload Identity Federation configuration. + * `vfs.gcs.service_account_key` takes precedence over this if both are + * specified. If neither is specified, Application Default Credentials will + * be used.
+ * **Default**: "" * - `vfs.gcs.impersonate_service_account`
* Set the GCS service account to impersonate. A chain of impersonated * accounts can be formed by specifying many service accounts, separated by diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 54383460fdad..ccf3839394ed 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -102,6 +102,11 @@ Status GCS::init(const Config& config, ThreadPool* const thread_pool) { } project_id_ = config.get("vfs.gcs.project_id", &found); assert(found); + service_account_key_ = config.get("vfs.gcs.service_account_key", &found); + assert(found); + workload_identity_configuration_ = + config.get("vfs.gcs.workload_identity_configuration", &found); + assert(found); impersonate_service_account_ = config.get("vfs.gcs.impersonate_service_account", &found); assert(found); @@ -187,7 +192,18 @@ static shared_ptr apply_impersonation( std::shared_ptr GCS::make_credentials( const google::cloud::Options& options) const { shared_ptr creds = nullptr; - if (!endpoint_.empty() || getenv("CLOUD_STORAGE_EMULATOR_ENDPOINT")) { + if (!service_account_key_.empty()) { + if (!workload_identity_configuration_.empty()) { + LOG_WARN( + "Both GCS service account key and workload identity configuration " + "were specified; picking the former"); + } + creds = google::cloud::MakeServiceAccountCredentials( + service_account_key_, options); + } else if (!workload_identity_configuration_.empty()) { + creds = google::cloud::MakeExternalAccountCredentials( + workload_identity_configuration_, options); + } else if (!endpoint_.empty() || getenv("CLOUD_STORAGE_EMULATOR_ENDPOINT")) { creds = google::cloud::MakeInsecureCredentials(); } else { creds = google::cloud::MakeGoogleDefaultCredentials(options); diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index 9a2b83bbaf4f..e2d5c83441b9 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -443,6 +443,12 @@ class GCS { // The GCS project id. std::string project_id_; + // The GCS service account credentials JSON string. + std::string service_account_key_; + + // The GCS external account credentials JSON string. + std::string workload_identity_configuration_; + // A comma-separated list with the GCS service accounts to impersonate. std::string impersonate_service_account_; From d7b798bf12b166081177ffb0c07f020f609681cb Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 11 Apr 2024 18:22:34 +0300 Subject: [PATCH 281/456] Mark the newly introduced GCS configuration options as experimental. (#4872) [SC-44515](https://app.shortcut.com/tiledb-inc/story/44515/support-specifying-gcp-account-credentials-as-a-config-option) [SC-42590](https://app.shortcut.com/tiledb-inc/story/42590/support-service-account-impersonation-in-gcs) --- TYPE: NO_HISTORY --- tiledb/api/c_api/config/config_api_external.h | 3 +++ tiledb/sm/cpp_api/config.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 655089bcc3d7..0b62b147ef00 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -392,17 +392,20 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * Set the GCS project id.
* **Default**: "" * - `vfs.gcs.service_account_key`
+ * **Experimental**
* Set the JSON string with GCS service account key. Takes precedence * over `vfs.gcs.workload_identity_configuration` if both are specified. If * neither is specified, Application Default Credentials will be used.
* **Default**: "" * - `vfs.gcs.workload_identity_configuration`
+ * **Experimental**
* Set the JSON string with Workload Identity Federation configuration. * `vfs.gcs.service_account_key` takes precedence over this if both are * specified. If neither is specified, Application Default Credentials will * be used.
* **Default**: "" * - `vfs.gcs.impersonate_service_account`
+ * **Experimental**
* Set the GCS service account to impersonate. A chain of impersonated * accounts can be formed by specifying many service accounts, separated by a * comma.
diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 927118e10d29..74aee2a827bb 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -570,17 +570,20 @@ class Config { * Set the GCS project id.
* **Default**: "" * - `vfs.gcs.service_account_key`
+ * **Experimental**
* Set the JSON string with GCS service account key. Takes precedence * over `vfs.gcs.workload_identity_configuration` if both are specified. If * neither is specified, Application Default Credentials will be used.
* **Default**: "" * - `vfs.gcs.workload_identity_configuration`
+ * **Experimental**
* Set the JSON string with Workload Identity Federation configuration. * `vfs.gcs.service_account_key` takes precedence over this if both are * specified. If neither is specified, Application Default Credentials will * be used.
* **Default**: "" * - `vfs.gcs.impersonate_service_account`
+ * **Experimental**
* Set the GCS service account to impersonate. A chain of impersonated * accounts can be formed by specifying many service accounts, separated by * a comma.
From 8046ae2c8a0ee3d1edf888aefea60adb2dbdc23c Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Fri, 12 Apr 2024 14:31:23 +0300 Subject: [PATCH 282/456] Config serialization should take into account environment variables (#4865) sc-44928 We were getting failures while testing Query v3 , and specifically when setting up the REST-CI environment for it, because the `rest.use_refactored_array_open_and_query_submit` config we were setting was not getting propagated to the REST server. After debugging it seems that Config serialization code was calling `param_values()` method to get the config variables to set in Cap'n'proto, which is not taking into account environment variables. This PR also moves `param_values()` to private to prevent other classes from similar mistakes. `ConfiIter` is the only class that still uses it and I only saw that class being used in `S3Parameters::load_headers`. It's worth investigating further if that's a problem or not. --- TYPE: BUG DESC: Config serialization should take into account environment variables --- test/src/unit-cppapi-config.cc | 22 +++++++++++++++++++++- tiledb/sm/config/config.cc | 11 +++++++++++ tiledb/sm/config/config.h | 18 +++++++++++++++--- tiledb/sm/serialization/config.cc | 5 +++-- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index 18504404d556..d84e9996b1b6 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -34,9 +34,24 @@ #include #include "test/support/src/helpers.h" +#include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/sm/c_api/tiledb_serialization.h" #include "tiledb/sm/cpp_api/tiledb" +using namespace tiledb::sm; + +class tiledb::sm::WhiteboxConfig { + public: + WhiteboxConfig(tiledb::sm::Config config) + : config_(config){}; + + const std::map& get_all_params() const { + return config_.param_values(); + } + + tiledb::sm::Config config_; +}; + TEST_CASE("C++ API: Config", "[cppapi][config]") { tiledb::Config config; config["foo"] = "bar"; @@ -159,7 +174,12 @@ TEST_CASE("C++ API: Config Serialization", "[cppapi][config][serialization]") { CHECK(rc == TILEDB_OK); tiledb::Config config2(&config2_ptr); - bool config_equal = config1 == config2; + auto cfg1 = config1.ptr().get()->config(); + auto cfg2 = config2.ptr().get()->config(); + // Check that the deserialized config already contains the values set in + // environment variables + bool config_equal = cfg1.get_all_params_from_config_or_env() == + WhiteboxConfig(cfg2).get_all_params(); CHECK(config_equal); // Check for inequality diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index e669cd02c606..a3434eddceb0 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -932,6 +932,17 @@ const char* Config::get_from_config_or_env( return *found ? value_config : ""; } +const std::map +Config::get_all_params_from_config_or_env() const { + std::map values; + bool found = false; + for (const auto& [key, value] : param_values_) { + std::string val = get_from_config_or_env(key, &found); + values.emplace(key, val); + } + return values; +} + template optional Config::get_internal(const std::string& key) const { auto value = get_internal_string(key); diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index bd93bce3e8fd..6b9f47a16898 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -61,12 +61,20 @@ using namespace tiledb::common; namespace tiledb::sm { +class WhiteboxConfig; + /** * This class manages the TileDB configuration options. * It is implemented as a simple map from string to string. * Parsing to appropriate types happens on demand. */ class Config { + friend class ConfigIter; + /** + * WhiteboxConfig makes available internals of Config for testing. + */ + friend class WhiteboxConfig; + public: /* ****************************** */ /* CONFIG DEFAULTS */ @@ -698,9 +706,6 @@ class Config { Status get_vector( const std::string& param, std::vector* value, bool* found) const; - /** Returns the param -> value map. */ - const std::map& param_values() const; - /** Gets the set parameters. */ const std::set& set_params() const; @@ -715,6 +720,10 @@ class Config { /** Compares configs for equality. */ bool operator==(const Config& rhs) const; + /** Get all config params taking into account environment variables */ + const std::map get_all_params_from_config_or_env() + const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -791,6 +800,9 @@ class Config { template optional get_internal_string(const std::string& key) const; + + /** Returns the param -> value map. */ + const std::map& param_values() const; }; /** diff --git a/tiledb/sm/serialization/config.cc b/tiledb/sm/serialization/config.cc index 9b1ff7102875..c911a082b095 100644 --- a/tiledb/sm/serialization/config.cc +++ b/tiledb/sm/serialization/config.cc @@ -61,9 +61,10 @@ namespace serialization { Status config_to_capnp( const Config& config, capnp::Config::Builder* config_builder) { - auto entries = config_builder->initEntries(config.param_values().size()); + auto config_params = config.get_all_params_from_config_or_env(); + auto entries = config_builder->initEntries(config_params.size()); uint64_t i = 0; - for (const auto& kv : config.param_values()) { + for (const auto& kv : config_params) { entries[i].setKey(kv.first); entries[i].setValue(kv.second); ++i; From 88aed9bdcc18b1a57c86932f17a2ee0f8bd9bc94 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:54:24 -0400 Subject: [PATCH 283/456] Add internal Config option 'sm.memory.tracker.reporter.wait_time_ms'. (#4867) Add _internal_ `Config` option, `sm.memory.tracker.reporter.wait_time_ms`, to toggle thread block time in `MemoryTrackerManager::run`. --- TYPE: NO_HISTORY DESC: Add internal Config option to toggle thread wait time in the MemoryTracker. --- tiledb/common/memory_tracker.cc | 4 +++- tiledb/common/memory_tracker.h | 12 +++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 5e34e76c300b..70b992a4e5c8 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -368,7 +368,9 @@ void MemoryTrackerReporter::run() { while (true) { std::unique_lock lk(mutex_); - cv_.wait_for(lk, std::chrono::milliseconds(1000), [&] { return stop_; }); + int wait_time = wait_time_ms_.has_value() ? wait_time_ms_.value() : 1000; + cv_.wait_for( + lk, std::chrono::milliseconds(wait_time), [&] { return stop_; }); if (stop_) { return; diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 1c1502e1e083..9c64157a8191 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -85,6 +85,12 @@ * objects and arrays encoded one per line). At runtime the reporter appends * a JSON blob once a second to this logfile that can then be analyzed using * whatever scripts or software as appropriate. + * + * Users may also set configuration key + * 'sm.memory.tracker.reporter.wait_time_ms' to toggle the duration, in + * milliseconds, that the calling thread is blocked in + * 'MemoryTrackerReporter::run' before the condition variable is notified. By + * default, the thread will wait for 1000 ms. */ #ifndef TILEDB_MEMORY_TRACKER_H @@ -419,6 +425,7 @@ class MemoryTrackerReporter { const Config& cfg, shared_ptr manager) : manager_(manager) , filename_(cfg.get("sm.memory.tracker.reporter.filename")) + , wait_time_ms_(cfg.get("sm.memory.tracker.reporter.wait_time_ms")) , stop_(false) { } @@ -441,9 +448,12 @@ class MemoryTrackerReporter { /** The MemoryTrackerManager instance on the parent ContextResources. */ shared_ptr manager_; - /** An filename set in the config. */ + /** A filename set in the config. */ std::optional filename_; + /** A wait time (in milliseconds) set in the config. */ + std::optional wait_time_ms_; + /** The background reporter thread. */ std::thread thread_; From 86ef4c5dc13f1cb74792ec8f8c16e4a1125eaa9b Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 16 Apr 2024 18:17:48 +0300 Subject: [PATCH 284/456] Stabilize REST CI (#4876) Moves generators in the beginning of tests. Flow of execution and clean up can get unclear/messy when GENERATORS and SECTIONs are interleaved. Testing: I re-ran the REST-CI 3 times and it passed all of them. Not sure the problem is fixed, but for sure it didn't make it worse. sc-42190 --- TYPE: NO_HISTORY DESC: Stabilize REST CI --- test/src/cpp-integration-query-condition.cc | 11 ++-- .../unit-cppapi-global-order-writes-remote.cc | 55 ++++++++++++------- test/src/unit-cppapi-string-dims.cc | 2 +- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/test/src/cpp-integration-query-condition.cc b/test/src/cpp-integration-query-condition.cc index 4c794769f9bd..6f22ca1df321 100644 --- a/test/src/cpp-integration-query-condition.cc +++ b/test/src/cpp-integration-query-condition.cc @@ -320,6 +320,11 @@ struct TestParams { TEST_CASE( "Testing read query with empty QC, with no range.", "[query][query-condition][empty][rest]") { + // Generate test parameters. + TestParams params = GENERATE( + TestParams(TILEDB_SPARSE, TILEDB_GLOBAL_ORDER, false, true), + TestParams(TILEDB_SPARSE, TILEDB_UNORDERED, true, false), + TestParams(TILEDB_DENSE, TILEDB_ROW_MAJOR, false, false)); // Initial setup. std::srand(static_cast(time(0))); test::VFSTestSetup vfs_test_setup; @@ -338,12 +343,6 @@ TEST_CASE( std::vector a_data_read_2(num_rows * num_rows); std::vector b_data_read_2(num_rows * num_rows); - // Generate test parameters. - TestParams params = GENERATE( - TestParams(TILEDB_SPARSE, TILEDB_GLOBAL_ORDER, false, true), - TestParams(TILEDB_SPARSE, TILEDB_UNORDERED, true, false), - TestParams(TILEDB_DENSE, TILEDB_ROW_MAJOR, false, false)); - // Setup by creating buffers to store all elements of the original array. create_array( ctx, diff --git a/test/src/unit-cppapi-global-order-writes-remote.cc b/test/src/unit-cppapi-global-order-writes-remote.cc index 771fcdc3f717..ef0976491a1d 100644 --- a/test/src/unit-cppapi-global-order-writes-remote.cc +++ b/test/src/unit-cppapi-global-order-writes-remote.cc @@ -47,6 +47,7 @@ struct RemoteGlobalOrderWriteFx { uint64_t total_cells, uint64_t extent, uint64_t submit_cell_count, + tiledb_array_type_t array_type, bool is_var = true, bool is_nullable = true) : is_var_(is_var) @@ -56,7 +57,8 @@ struct RemoteGlobalOrderWriteFx { , extent_(extent) , array_name_{"global-array-" + std::to_string(total_cell_count_)} , array_uri_(vfs_test_setup_.array_uri(array_name_)) - , ctx_{vfs_test_setup_.ctx()} {}; + , ctx_(vfs_test_setup_.ctx()) + , array_type_(array_type){}; // Create a simple dense array void create_array() { @@ -64,12 +66,10 @@ struct RemoteGlobalOrderWriteFx { domain.add_dimension(Dimension::create( ctx_, "cols", {{1, total_cell_count_}}, extent_)); - auto array_type = GENERATE(TILEDB_DENSE, TILEDB_SPARSE); - - ArraySchema schema(ctx_, array_type); + ArraySchema schema(ctx_, array_type_); schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); - if (array_type == TILEDB_SPARSE) { + if (array_type_ == TILEDB_SPARSE) { schema.set_capacity(extent_); } @@ -87,14 +87,14 @@ struct RemoteGlobalOrderWriteFx { Array::create(array_uri_, schema); Array array(ctx_, array_uri_, TILEDB_READ); - CHECK(array.schema().array_type() == array_type); + CHECK(array.schema().array_type() == array_type_); CHECK( array.schema() .domain() .dimension(0) .template domain() .second == total_cell_count_); - if (array_type == TILEDB_SPARSE) { + if (array_type_ == TILEDB_SPARSE) { CHECK(array.schema().capacity() == extent_); } array.close(); @@ -372,6 +372,7 @@ struct RemoteGlobalOrderWriteFx { test::VFSTestSetup vfs_test_setup_; std::string array_uri_; Context ctx_; + tiledb_array_type_t array_type_; // Vectors to store all the data wrote to the array. // + We will use these vectors to validate subsequent read. @@ -384,7 +385,11 @@ struct RemoteGlobalOrderWriteFx { }; typedef std::tuple TestTypes; -TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { +TEMPLATE_LIST_TEST_CASE( + "Global order remote writes", + "[rest][global][global-order][write]", + TestTypes) { + auto array_type = GENERATE(TILEDB_DENSE, TILEDB_SPARSE); typedef TestType T; uint64_t cells; uint64_t extent; @@ -396,7 +401,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 10; chunk_size = 3; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -404,7 +410,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 10; chunk_size = 19; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -412,7 +419,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 10; chunk_size = 20; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -420,7 +428,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 10; chunk_size = 10; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -429,7 +438,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 10; chunk_size = 5; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -437,7 +447,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 5; chunk_size = 6; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -445,7 +456,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 20; extent = 5; chunk_size = 3; // Should not divide evenly into `cells` for this test. - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -453,7 +465,8 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 50; extent = 5; chunk_size = 12; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } @@ -461,13 +474,17 @@ TEMPLATE_LIST_TEST_CASE("Global order remote writes", "[rest]", TestTypes) { cells = 50; extent = 10; chunk_size = 18; - RemoteGlobalOrderWriteFx fx(cells, extent, chunk_size, var, nullable); + RemoteGlobalOrderWriteFx fx( + cells, extent, chunk_size, array_type, var, nullable); fx.run_test(); } } -TEST_CASE("Remote global order writes finalize errors", "[rest][finalize]") { - RemoteGlobalOrderWriteFx fx(20, 10, 3, true, true); +TEST_CASE( + "Remote global order writes finalize errors", + "[rest][global][global-order][write][finalize]") { + auto array_type = GENERATE(TILEDB_DENSE, TILEDB_SPARSE); + RemoteGlobalOrderWriteFx fx(20, 10, 3, array_type, true, true); fx.create_array(); fx.write_array(true); } diff --git a/test/src/unit-cppapi-string-dims.cc b/test/src/unit-cppapi-string-dims.cc index d865dc8dbca3..f489d7e6b01e 100644 --- a/test/src/unit-cppapi-string-dims.cc +++ b/test/src/unit-cppapi-string-dims.cc @@ -1497,6 +1497,7 @@ void read_and_check_sparse_array_string_dim( TEST_CASE( "C++ API: Test filtering of string dimensions on sparse arrays", "[cppapi][string-dims][rle-strings][dict-strings][sparse][rest]") { + auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); // Create data buffer to use std::stringstream repetitions; size_t repetition_num = 100; @@ -1519,7 +1520,6 @@ TEST_CASE( auto dim = Dimension::create(ctx, "dim1", TILEDB_STRING_ASCII, nullptr, nullptr); - auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); // Create compressor as a filter Filter filter(ctx, f); // Create filter list From 6dff6fae0b09c16177fe85ec9b62c128bd721ad7 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Tue, 16 Apr 2024 20:37:43 +0300 Subject: [PATCH 285/456] Add end-to-end cloud test workflow with Microsoft Entra ID. (#4841) This PR adds the foundations for testing connectivity of TileDB to the real clouds by adding a new workflow named `test-cloud-e2e`. Currently only Azure with Microsoft Entra ID authentication is provided. Authentication happens [with OpenID Connect](https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-azure). A new environment was created that has read-only access to the `core` container of the `tiledbci` storage account, and we test that the file `test.txt` in that container exists. The test is configured with environment variables, making it reusable for subsequent tests with other cloud providers. --- TYPE: NO_HISTORY --------- Co-authored-by: Theodore Tsirpanis --- .github/workflows/test-cloud-e2e.yml | 107 +++++++++++++++++++++++++++ test/src/unit-vfs.cc | 20 +++++ 2 files changed, 127 insertions(+) create mode 100644 .github/workflows/test-cloud-e2e.yml diff --git a/.github/workflows/test-cloud-e2e.yml b/.github/workflows/test-cloud-e2e.yml new file mode 100644 index 000000000000..845389c52b6a --- /dev/null +++ b/.github/workflows/test-cloud-e2e.yml @@ -0,0 +1,107 @@ +name: End-to-End cloud service tests +on: + workflow_dispatch: + inputs: + run_azure: + description: 'Run Azure tests' + required: true + default: true + type: boolean + push: + branches: + - dev + - release-* + +env: + VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' + SCCACHE_GHA_ENABLED: "true" + +jobs: + azure: + runs-on: ubuntu-latest + if: inputs.run_azure != 'false' + environment: azure-e2e-test + env: + bootstrap_args: --enable-azure --enable-ccache + permissions: + id-token: write # Get OIDC token for authentication to Azure + name: Azure + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Configure required environment variables for vcpkg to use + # GitHub's Action Cache + - uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - name: Prevent vpckg from building debug variants + run: python ./scripts/ci/patch_vcpkg_triplets.py + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: 'Configure libtiledb' + id: configure + shell: bash + run: | + set -e pipefail + + # Show CMake Version + cmake --version + + source $GITHUB_WORKSPACE/scripts/ci/bootstrap_libtiledb.sh + + - name: 'Build libtiledb' + id: build + shell: bash + run: | + set -e pipefail + + ##################################################### + # Build libtiledb using previous bootstrap + + source $GITHUB_WORKSPACE/scripts/ci/build_libtiledb.sh + + - name: 'Az CLI login' + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: 'Test libtiledb' + id: test + shell: bash + env: + # Allow forks to specify different values. + AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT || 'tiledbci' }} + TILEDB_VFS_E2E_TEST_FILE: ${{ vars.AZURE_E2E_TEST_FILE || 'azure://tiledb/test.txt' }} + run: | + set -e pipefail + + cd $GITHUB_WORKSPACE/build + + ################################################### + # Run tests + + # Bypass Catch2 Framework stdout interception with awk on test output + ./tiledb/test/tiledb_unit -d yes "[vfs-e2e]" | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + + - name: 'Test status check' + run: | + # tiledb_unit is configured to set a variable TILEDB_CI_SUCCESS=1 + # following the test run. If this variable is not set, the build should fail. + # see https://github.com/TileDB-Inc/TileDB/pull/1400 (5f0623f4d3) + if [[ "${{ steps.test.outputs.TILEDB_CI_SUCCESS }}" -ne 1 ]]; then + exit 1; + fi + + - name: "Print log files (failed build only)" + run: | + source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh + if: ${{ failure() }} # only run this job if the build step failed diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 95008baf51a7..7ba6cc422070 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -552,6 +552,26 @@ TEMPLATE_LIST_TEST_CASE("VFS: File I/O", "[vfs][uri][file_io]", AllBackends) { } } +TEST_CASE("VFS: Test end-to-end", "[.vfs-e2e]") { + auto test_file_ptr = getenv("TILEDB_VFS_E2E_TEST_FILE"); + if (test_file_ptr == nullptr) { + FAIL("TILEDB_VFS_E2E_TEST_FILE variable is not specified"); + } + URI test_file{test_file_ptr}; + + ThreadPool compute_tp(1); + ThreadPool io_tp(1); + // Will be configured from environment variables. + Config config; + + VFS vfs{&g_helper_stats, &compute_tp, &io_tp, config}; + REQUIRE(vfs.supports_uri_scheme(test_file)); + + uint64_t nbytes = 0; + require_tiledb_ok(vfs.file_size(test_file, &nbytes)); + CHECK(nbytes > 0); +} + TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { ThreadPool compute_tp(4); ThreadPool io_tp(4); From f5824102cc8d51162e3b6e0156e42677a9b2f15e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:56:46 -0400 Subject: [PATCH 286/456] Add v2_22_0 arrays to backward compatibility matrix. (#4882) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 51c0670bb4b1..b5159501e1c3 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 4feebfab24e83129eeaffc62e857a9cab9fe2da3 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:54:09 +0200 Subject: [PATCH 287/456] Clarify the documentation for the non empty domain CAPI. (#4885) This clarifies the parameters to the original non empty domain CAPI. --- TYPE: IMPROVEMENT DESC: Clarify the documentation for the non empty domain CAPI. --- tiledb/sm/c_api/tiledb.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index 39e5048a41bb..a18fbcb03df3 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -3367,7 +3367,12 @@ TILEDB_EXPORT int32_t tiledb_array_vacuum( /** * Retrieves the non-empty domain from an array. This is the union of the - * non-empty domains of the array fragments. + * non-empty domains of the array fragments. This API only works for arrays that + * have numeric dimensions and all dimensions of the same type. + * + * The domain passed in is memory that can contain the number of dimensions * 2 + * * sizeof(type of dimension). The previous two is because we return the non + * empty domain as min/max. Dimensions are returned in order. * * **Example:** * From 7c4c7c706d602e5824525b3cba1a451fdd90a128 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Mon, 22 Apr 2024 10:07:16 +0300 Subject: [PATCH 288/456] Vac files should only be removed if paths removal was fully successful (#4889) - Vacuum files are only removed on a fully successful vacuum. - The presence of the vacuum files will allow to open any array where a vacuum failed. - The presence of the vacuum files will allow to resume and complete any vacuum that could have previously failed. - Test coverage for the scenarios above: dense/sparse, failure points at frag dir removal and commits file removal. Followups for extending test coverage further: https://app.shortcut.com/tiledb-inc/story/45564 https://app.shortcut.com/tiledb-inc/story/45565 --- TYPE: BUG DESC: Vac files should only be removed if paths removal was fully successful --- test/src/unit-capi-consolidation.cc | 556 +++++++++++++++--- .../sm/consolidator/fragment_consolidator.cc | 15 +- 2 files changed, 468 insertions(+), 103 deletions(-) diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index 1f7527120ec4..1699d8fea73c 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -34,6 +34,7 @@ #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" #include "tiledb/common/stdx_string.h" +#include "tiledb/platform/platform.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/misc/tdb_time.h" @@ -132,11 +133,13 @@ struct ConsolidationFx { void consolidate_sparse_heterogeneous(); void consolidate_sparse_string( uint64_t buffer_size = 10000, bool error_expected = false); - void vacuum_dense(const std::string& mode = "fragments"); + void vacuum_dense( + const std::string& mode = "fragments", bool expect_fail = false); void vacuum_sparse( const std::string& mode = "fragments", uint64_t start = 0, - uint64_t end = UINT64_MAX); + uint64_t end = UINT64_MAX, + bool expect_fail = false); void remove_dense_vector(); void remove_dense_array(); void remove_sparse_array(); @@ -144,12 +147,12 @@ struct ConsolidationFx { void remove_sparse_string_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); - void check_commits_dir_dense(int num_commits, int num_wrt, int num_ignore); - void check_commits_dir_sparse(int num_commits, int num_wrt, int num_ignore); + void check_commits_dir( + int num_commits, int num_wrt, int num_ignore, bool dense = true); void check_ok_num(int num_ok); void get_array_meta_files_dense(std::vector& files); void get_array_meta_vac_files_dense(std::vector& files); - void get_vac_files_dense(std::vector& files); + void get_vac_files(std::vector& files, bool dense = true); // Used to get the number of directories or files of another directory struct get_num_struct { @@ -4376,7 +4379,7 @@ void ConsolidationFx::consolidate_sparse_string( REQUIRE(rc == (error_expected ? TILEDB_ERR : TILEDB_OK)); } -void ConsolidationFx::vacuum_dense(const std::string& mode) { +void ConsolidationFx::vacuum_dense(const std::string& mode, bool expect_fail) { int rc; tiledb_config_t* cfg; tiledb_error_t* err = nullptr; @@ -4386,13 +4389,14 @@ void ConsolidationFx::vacuum_dense(const std::string& mode) { REQUIRE(rc == TILEDB_OK); REQUIRE(err == nullptr); rc = tiledb_array_vacuum(ctx_, dense_array_uri_.c_str(), cfg); - REQUIRE(rc == TILEDB_OK); + + REQUIRE(rc == (expect_fail ? TILEDB_ERR : TILEDB_OK)); tiledb_config_free(&cfg); } void ConsolidationFx::vacuum_sparse( - const std::string& mode, uint64_t start, uint64_t end) { + const std::string& mode, uint64_t start, uint64_t end, bool expect_fail) { int rc; tiledb_config_t* cfg; tiledb_error_t* err = nullptr; @@ -4411,7 +4415,7 @@ void ConsolidationFx::vacuum_sparse( REQUIRE(err == nullptr); rc = tiledb_array_vacuum(ctx_, sparse_array_uri_.c_str(), cfg); - REQUIRE(rc == TILEDB_OK); + REQUIRE(rc == (expect_fail ? TILEDB_ERR : TILEDB_OK)); tiledb_config_free(&cfg); } @@ -4451,56 +4455,34 @@ bool ConsolidationFx::is_array(const std::string& array_name) { return type == TILEDB_ARRAY; } -void ConsolidationFx::check_commits_dir_dense( - int num_commits, int num_wrt, int num_ignore) { +void ConsolidationFx::check_commits_dir( + int num_commits, int num_wrt, int num_ignore, bool dense) { int32_t rc; get_num_struct data; - // Check number of consolidated commits files - data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls( - ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_commits_num, &data); - CHECK(rc == TILEDB_OK); - CHECK(data.num == num_commits); - - // Check number of wrt files - data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls( - ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_wrt_num, &data); - CHECK(rc == TILEDB_OK); - CHECK(data.num == num_wrt); - - // Check number of ignore files - data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls( - ctx_, vfs_, dense_array_commits_dir_.c_str(), &get_ignore_num, &data); - CHECK(rc == TILEDB_OK); - CHECK(data.num == num_ignore); -} - -void ConsolidationFx::check_commits_dir_sparse( - int num_commits, int num_wrt, int num_ignore) { - int32_t rc; - get_num_struct data; + std::string array_commits_uri = dense_array_commits_dir_; + if (!dense) { + array_commits_uri = sparse_array_commits_dir_; + } // Check number of consolidated commits files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_commits_num, &data); + ctx_, vfs_, array_commits_uri.c_str(), &get_commits_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_commits); // Check number of wrt files data = {ctx_, vfs_, 0}; - rc = tiledb_vfs_ls( - ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_wrt_num, &data); + rc = + tiledb_vfs_ls(ctx_, vfs_, array_commits_uri.c_str(), &get_wrt_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_wrt); // Check number of ignore files data = {ctx_, vfs_, 0}; rc = tiledb_vfs_ls( - ctx_, vfs_, sparse_array_commits_dir_.c_str(), &get_ignore_num, &data); + ctx_, vfs_, array_commits_uri.c_str(), &get_ignore_num, &data); CHECK(rc == TILEDB_OK); CHECK(data.num == num_ignore); } @@ -4702,11 +4684,22 @@ void ConsolidationFx::get_array_meta_vac_files_dense( CHECK(rc == TILEDB_OK); } -void ConsolidationFx::get_vac_files_dense(std::vector& files) { +void ConsolidationFx::get_vac_files( + std::vector& files, bool dense) { files.clear(); - tiledb::sm::URI dense_array_uri(dense_array_uri_.c_str()); + std::string array_uri = dense_array_uri_; + if (!dense) { + array_uri = sparse_array_uri_; + } + tiledb::sm::URI dense_array_uri(array_uri.c_str()); int rc = tiledb_vfs_ls( - ctx_, vfs_, dense_array_uri.c_str(), &get_vac_files_callback, &files); + ctx_, + vfs_, + dense_array_uri.add_trailing_slash() + .join_path(tiledb::sm::constants::array_commits_dir_name) + .c_str(), + &get_vac_files_callback, + &files); CHECK(rc == TILEDB_OK); } @@ -6571,22 +6564,22 @@ TEST_CASE_METHOD( write_dense_subarray(); consolidate_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(1, 2, 0); + check_commits_dir(1, 2, 0); // Vacuum works. vacuum_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // Second consolidation works. consolidate_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(2, 0, 0); + check_commits_dir(2, 0, 0); // Second vacuum works. vacuum_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. // Fragment consolidation not yet supported on remote arrays @@ -6594,17 +6587,17 @@ TEST_CASE_METHOD( consolidate_dense(); vacuum_dense(); read_dense_full_subarray(); - check_commits_dir_dense(1, 1, 1); + check_commits_dir(1, 1, 1); // Consolidation to get rid of ignore file. consolidate_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(2, 1, 1); + check_commits_dir(2, 1, 1); // Second vacuum works. vacuum_dense("commits"); read_dense_full_subarray(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); } } @@ -6618,17 +6611,17 @@ TEST_CASE_METHOD( // Vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // Second consolidation works. consolidate_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(2, 0, 0); + check_commits_dir(2, 0, 0); // Second vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. // Fragment consolidation not yet supported on remote arrays @@ -6636,17 +6629,17 @@ TEST_CASE_METHOD( consolidate_dense(); vacuum_dense(); read_dense_subarray_full(); - check_commits_dir_dense(1, 1, 1); + check_commits_dir(1, 1, 1); // Consolidation to get rid of ignore file. consolidate_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(2, 1, 1); + check_commits_dir(2, 1, 1); // Second vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); } } @@ -6666,33 +6659,33 @@ TEST_CASE_METHOD( // Vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // Second consolidation works. consolidate_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(2, 0, 0); + check_commits_dir(2, 0, 0); // Second vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); // After fragment consolidation and vacuuming, array is still valid. consolidate_dense(); vacuum_dense(); read_dense_subarray_full(); - check_commits_dir_dense(1, 1, 1); + check_commits_dir(1, 1, 1); // Consolidation to get rid of ignore file. consolidate_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(2, 1, 1); + check_commits_dir(2, 1, 1); // Second vacuum works. vacuum_dense("commits"); read_dense_subarray_full(); - check_commits_dir_dense(1, 0, 0); + check_commits_dir(1, 0, 0); } } @@ -6712,22 +6705,22 @@ TEST_CASE_METHOD( write_sparse_unordered(); consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 2, 0); + check_commits_dir(1, 2, 0, false); // Vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // Second consolidation works. consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(2, 0, 0); + check_commits_dir(2, 0, 0, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // After fragment consolidation and vacuuming, array is still valid. // Fragment consolidation not yet supported on remote arrays @@ -6735,17 +6728,17 @@ TEST_CASE_METHOD( consolidate_sparse(); vacuum_sparse(); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 1, 1); + check_commits_dir(1, 1, 1, false); // Consolidation to get rid of ignore file. consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(2, 1, 1); + check_commits_dir(2, 1, 1, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); } } @@ -6755,22 +6748,22 @@ TEST_CASE_METHOD( write_sparse_full(); consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 2, 0); + check_commits_dir(1, 2, 0, false); // Vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // Second consolidation works. consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(2, 0, 0); + check_commits_dir(2, 0, 0, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // After fragment consolidation and vacuuming, array is still valid. // Fragment consolidation not yet supported on remote arrays @@ -6778,17 +6771,17 @@ TEST_CASE_METHOD( consolidate_sparse(); vacuum_sparse(); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 1, 1); + check_commits_dir(1, 1, 1, false); // Consolidation to get rid of ignore file. consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(2, 1, 1); + check_commits_dir(2, 1, 1, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); } } @@ -6804,38 +6797,38 @@ TEST_CASE_METHOD( write_sparse_full(); consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 2, 0); + check_commits_dir(1, 2, 0, false); // Vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // Second consolidation works. consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(2, 0, 0); + check_commits_dir(2, 0, 0, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // After fragment consolidation and vacuuming, array is still valid. consolidate_sparse(); vacuum_sparse(); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 1, 1); + check_commits_dir(1, 1, 1, false); // Consolidation to get rid of ignore file. consolidate_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(2, 1, 1); + check_commits_dir(2, 1, 1, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_unordered_full(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); } } @@ -6878,40 +6871,40 @@ TEST_CASE_METHOD( write_sparse_unordered(); consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 1, 0); + check_commits_dir(1, 1, 0, false); check_ok_num(1); // Vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); check_ok_num(0); // Second consolidation works. consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(2, 0, 0); + check_commits_dir(2, 0, 0, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); // After fragment consolidation and vacuuming, array is still valid. consolidate_sparse(); vacuum_sparse(); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 1, 1); + check_commits_dir(1, 1, 1, false); // Consolidation to get rid of ignore file. consolidate_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(2, 1, 1); + check_commits_dir(2, 1, 1, false); // Second vacuum works. vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 0); + check_commits_dir(1, 0, 0, false); remove_sparse_array(); } @@ -6939,8 +6932,8 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Get fragment URIs - const char* uri1; - rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 1, &uri1); + const char* uri; + rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 1, &uri); CHECK(rc == TILEDB_OK); const char* uri2; rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 3, &uri2); @@ -6959,7 +6952,7 @@ TEST_CASE_METHOD( REQUIRE(err == nullptr); // Consolidate - const char* uris[2] = {strrchr(uri1, '/') + 1, strrchr(uri2, '/') + 1}; + const char* uris[2] = {strrchr(uri, '/') + 1, strrchr(uri2, '/') + 1}; rc = tiledb_array_consolidate_fragments( ctx_, dense_array_uri_.c_str(), uris, 2, cfg); CHECK(rc == TILEDB_OK); @@ -7015,8 +7008,8 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Get fragment URIs - const char* uri1; - rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 1, &uri1); + const char* uri; + rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 1, &uri); CHECK(rc == TILEDB_OK); const char* uri2; rc = tiledb_fragment_info_get_fragment_uri(ctx_, fragment_info, 3, &uri2); @@ -7035,7 +7028,7 @@ TEST_CASE_METHOD( REQUIRE(err == nullptr); // Consolidate - const char* uris[2] = {strrchr(uri1, '/') + 1, strrchr(uri2, '/') + 1}; + const char* uris[2] = {strrchr(uri, '/') + 1, strrchr(uri2, '/') + 1}; rc = tiledb_array_consolidate_fragments( ctx_, sparse_array_uri_.c_str(), uris, 2, cfg); CHECK(rc == TILEDB_OK); @@ -7132,7 +7125,374 @@ TEST_CASE_METHOD( vacuum_sparse("fragments"); vacuum_sparse("commits"); read_sparse_full_unordered(); - check_commits_dir_sparse(1, 0, 1); + check_commits_dir(1, 0, 1, false); remove_sparse_array(); } + +TEST_CASE_METHOD( + ConsolidationFx, + "C API: Test vacuuming leaves array dir in a consistent state", + "[capi][vacuuming][fail][frag]") { + // This test uses a permissions trick to make vacuuming fail in the middle of + // the process, this trick does not work on windows or remote stores + if constexpr (tiledb::platform::is_os_windows) { + return; + } + if (!vfs_test_setup_.is_local()) { + return; + } + char* manylinux_var = getenv("TILEDB_MANYLINUX"); + if (manylinux_var && strlen(manylinux_var) > 0) { + return; + } + + bool dense_test = true; + std::string array_uri; + std::string commits_uri; + + SECTION("- sparse array") { + dense_test = false; + array_uri = sparse_array_uri_; + commits_uri = sparse_array_commits_dir_; + } + SECTION("- dense array") { + dense_test = true; + array_uri = dense_array_uri_; + commits_uri = dense_array_commits_dir_; + } + + if (dense_test) { + remove_dense_array(); + create_dense_array(); + } else { + remove_sparse_array(); + create_sparse_array(); + } + + auto start1 = tiledb::sm::utils::time::timestamp_now_ms(); + if (dense_test) { + write_dense_subarray(1, 2, 3, 4); + write_dense_subarray(1, 2, 3, 4); + } else { + write_sparse_full(); + write_sparse_full(); + } + auto end1 = tiledb::sm::utils::time::timestamp_now_ms(); + + if (dense_test) { + consolidate_dense("fragments", start1, end1); + } else { + consolidate_sparse("fragments", start1, end1); + } + + // Get the uri of a fragment which is supposed to be vacuumed + tiledb_fragment_info_t* fragment_info = nullptr; + int rc = tiledb_fragment_info_alloc(ctx_, array_uri.c_str(), &fragment_info); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_fragment_info_load(ctx_, fragment_info); + REQUIRE(rc == TILEDB_OK); + + const char* uri; + rc = tiledb_fragment_info_get_to_vacuum_uri(ctx_, fragment_info, 1, &uri); + REQUIRE(rc == TILEDB_OK); + + { + // Remove the write permission bit for the fragment and start vacuuming + DenyWriteAccess dwa{tiledb::sm::URI(std::string(uri)).to_path()}; + + // This fails when trying to remove the fragment directory + if (dense_test) { + vacuum_dense("fragments", true); + } else { + vacuum_sparse("fragments", start1, end1, true); + } + } + + tiledb_fragment_info_free(&fragment_info); + + // The .vac files should always remain when vacuuming fails. + std::vector vac_files; + get_vac_files(vac_files, dense_test); + CHECK(!vac_files.empty()); + + // Array open should work fine if vacuuming failed midway + tiledb_array_t* array; + rc = tiledb_array_alloc(ctx_, array_uri.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_close(ctx_, array); + REQUIRE(rc == TILEDB_OK); + tiledb_array_free(&array); + + // Clean up + if (dense_test) { + remove_dense_array(); + } else { + remove_sparse_array(); + } +} + +TEST_CASE_METHOD( + ConsolidationFx, + "C API: Test vacuuming leaves array dir in a consistent state", + "[capi][vacuuming][fail][wrt]") { + // This test uses a permissions trick to make vacuuming fail in the middle of + // the process, this trick does not work on windows or remote stores + if constexpr (tiledb::platform::is_os_windows) { + return; + } + if (!vfs_test_setup_.is_local()) { + return; + } + char* manylinux_var = getenv("TILEDB_MANYLINUX"); + if (manylinux_var && strlen(manylinux_var) > 0) { + return; + } + + bool dense_test = true; + std::string array_uri; + std::string commits_uri; + + SECTION("- sparse array") { + dense_test = false; + array_uri = sparse_array_uri_; + commits_uri = sparse_array_commits_dir_; + } + SECTION("- dense array") { + dense_test = true; + array_uri = dense_array_uri_; + commits_uri = dense_array_commits_dir_; + } + + if (dense_test) { + remove_dense_array(); + create_dense_array(); + } else { + remove_sparse_array(); + create_sparse_array(); + } + + auto start1 = tiledb::sm::utils::time::timestamp_now_ms(); + if (dense_test) { + write_dense_subarray(1, 2, 3, 4); + write_dense_subarray(1, 2, 3, 4); + } else { + write_sparse_full(); + write_sparse_full(); + } + auto end1 = tiledb::sm::utils::time::timestamp_now_ms(); + + if (dense_test) { + consolidate_dense("fragments", start1, end1); + } else { + consolidate_sparse("fragments", start1, end1); + } + + { + // We need to remove the write permission bit for the parent directory + // (commits dir) so that consolidator won't be able to delete the wrt file + DenyWriteAccess dwa{tiledb::sm::URI(commits_uri).to_path()}; + + // This fails when trying to remove the commit file for the fragment + if (dense_test) { + vacuum_dense("fragments", true); + } else { + vacuum_sparse("fragments", start1, end1, true); + } + } + + // The .vac files should always remain when vacuuming fails. + std::vector vac_files; + get_vac_files(vac_files, dense_test); + CHECK(!vac_files.empty()); + + // Array open should work fine if vacuuming failed midway + tiledb_array_t* array; + int rc = tiledb_array_alloc(ctx_, array_uri.c_str(), &array); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_open(ctx_, array, TILEDB_READ); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_array_close(ctx_, array); + REQUIRE(rc == TILEDB_OK); + tiledb_array_free(&array); + + // Clean up + if (dense_test) { + remove_dense_array(); + } else { + remove_sparse_array(); + } +} + +TEST_CASE_METHOD( + ConsolidationFx, + "C API: Test vacuuming resumes fine in case of error deleting fragment", + "[capi][vacuuming][resume][frag]") { + bool dense_test = true; + std::string array_uri; + std::string commits_uri; + + SECTION("- sparse array") { + dense_test = false; + array_uri = sparse_array_uri_; + commits_uri = sparse_array_commits_dir_; + } + SECTION("- dense array") { + dense_test = true; + array_uri = dense_array_uri_; + commits_uri = dense_array_commits_dir_; + } + + if (dense_test) { + remove_dense_array(); + create_dense_array(); + } else { + remove_sparse_array(); + create_sparse_array(); + } + + auto start1 = tiledb::sm::utils::time::timestamp_now_ms(); + if (dense_test) { + write_dense_subarray(1, 2, 3, 4); + write_dense_subarray(1, 2, 3, 4); + } else { + write_sparse_full(); + write_sparse_full(); + } + auto end1 = tiledb::sm::utils::time::timestamp_now_ms(); + + if (dense_test) { + consolidate_dense("fragments", start1, end1); + } else { + consolidate_sparse("fragments", start1, end1); + } + + // Get the uri of a fragment which is supposed to be vacuumed + tiledb_fragment_info_t* fragment_info = nullptr; + int rc = tiledb_fragment_info_alloc(ctx_, array_uri.c_str(), &fragment_info); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_fragment_info_load(ctx_, fragment_info); + REQUIRE(rc == TILEDB_OK); + const char* uri; + rc = tiledb_fragment_info_get_to_vacuum_uri(ctx_, fragment_info, 1, &uri); + REQUIRE(rc == TILEDB_OK); + + // Simulate partial vacuuming, only 1 fragment was deleted, + // vacuum file still exists + rc = tiledb_vfs_remove_dir(ctx_, vfs_, uri); + REQUIRE(rc == TILEDB_OK); + + tiledb_fragment_info_free(&fragment_info); + + // Vacuuming resumes fine if vac file still exists. + if (dense_test) { + vacuum_dense("fragments"); + } else { + vacuum_sparse("fragments"); + } + + // No vacuum files left + std::vector vac_files; + get_vac_files(vac_files, dense_test); + + CHECK(vac_files.empty()); + + check_commits_dir(0, 1, 0, dense_test); + + // Clean up + if (dense_test) { + remove_dense_array(); + } else { + remove_sparse_array(); + } +} + +TEST_CASE_METHOD( + ConsolidationFx, + "C API: Test vacuuming resumes fine in case of error deleting commit file", + "[capi][vacuuming][resume][wrt]") { + bool dense_test = true; + std::string array_uri; + std::string commits_uri; + + SECTION("- sparse array") { + dense_test = false; + array_uri = sparse_array_uri_; + commits_uri = sparse_array_commits_dir_; + } + SECTION("- dense array") { + dense_test = true; + array_uri = dense_array_uri_; + commits_uri = dense_array_commits_dir_; + } + + if (dense_test) { + remove_dense_array(); + create_dense_array(); + } else { + remove_sparse_array(); + create_sparse_array(); + } + + auto start1 = tiledb::sm::utils::time::timestamp_now_ms(); + if (dense_test) { + write_dense_subarray(1, 2, 3, 4); + write_dense_subarray(1, 2, 3, 4); + } else { + write_sparse_full(); + write_sparse_full(); + } + auto end1 = tiledb::sm::utils::time::timestamp_now_ms(); + + if (dense_test) { + consolidate_dense("fragments", start1, end1); + } else { + consolidate_sparse("fragments", start1, end1); + } + + // Get the uri of a fragment which is supposed to be vacuumed + tiledb_fragment_info_t* fragment_info = nullptr; + int rc = tiledb_fragment_info_alloc(ctx_, array_uri.c_str(), &fragment_info); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_fragment_info_load(ctx_, fragment_info); + REQUIRE(rc == TILEDB_OK); + const char* uri; + rc = tiledb_fragment_info_get_to_vacuum_uri(ctx_, fragment_info, 1, &uri); + REQUIRE(rc == TILEDB_OK); + auto frag_name = tiledb::sm::URI(uri).last_path_part(); + + auto commit_uri = tiledb::sm::URI(commits_uri) + .add_trailing_slash() + .join_path(frag_name + ".wrt"); + + // Simulate partial vacuuming, delete only the commit file for only one + // fragment, vacuum file still exists + rc = tiledb_vfs_remove_file(ctx_, vfs_, commit_uri.c_str()); + REQUIRE(rc == TILEDB_OK); + + tiledb_fragment_info_free(&fragment_info); + + // Vacuuming resumes fine if vac file still exists. + if (dense_test) { + vacuum_dense("fragments"); + } else { + vacuum_sparse("fragments"); + } + + // No vacuum files left + std::vector vac_files; + get_vac_files(vac_files, dense_test); + + CHECK(vac_files.empty()); + + check_commits_dir(0, 1, 0, dense_test); + + // Clean up + if (dense_test) { + remove_dense_array(); + } else { + remove_sparse_array(); + } +} diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index d0d2a0e3c00c..d0bb71e355c2 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -446,17 +446,12 @@ void FragmentConsolidator::vacuum(const char* array_name) { array_dir.write_commit_ignore_file(commit_uris_to_ignore); } - // Delete the vacuum files. auto vfs = storage_manager_->vfs(); auto compute_tp = storage_manager_->compute_tp(); - vfs->remove_files( - compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); // Delete fragment directories throw_if_not_ok(parallel_for( compute_tp, 0, fragment_uris_to_vacuum.size(), [&](size_t i) { - RETURN_NOT_OK(vfs->remove_dir(fragment_uris_to_vacuum[i])); - // Remove the commit file, if present. auto commit_uri = array_dir.get_commit_uri(fragment_uris_to_vacuum[i]); bool is_file = false; @@ -465,8 +460,18 @@ void FragmentConsolidator::vacuum(const char* array_name) { RETURN_NOT_OK(vfs->remove_file(commit_uri)); } + bool is_dir = false; + RETURN_NOT_OK(vfs->is_dir(fragment_uris_to_vacuum[i], &is_dir)); + if (is_dir) { + RETURN_NOT_OK(vfs->remove_dir(fragment_uris_to_vacuum[i])); + } + return Status::Ok(); })); + + // Delete the vacuum files. + vfs->remove_files( + compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); } /* ****************************** */ From f81be54e28a6d39a7bb6b221a095052b63c5c807 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 22 Apr 2024 14:07:55 +0300 Subject: [PATCH 289/456] Deprecate C++ APIs that rely on deprecated APIs. (#4879) As part of removing the deprecated APIs, we discovered that there are some non-deprecated C++ APIs that are calling deprecated C APIs and would break once the latter get removed. This PR marks these C++ APIs as deprecated, to be removed alongside the C APIs. --- TYPE: CPP_API DESC: `Query::submit_async` is deprecated. Call `Query::submit()` on another thread instead. --- TYPE: CPP_API DESC: Overloads of methods and constructors in `Array` and `ArraySchema` that accept encryption keys are deprecated. Specify the encryption key with the `sm.encryption_type` and `sm.encryption_key` config options instead. --- tiledb/sm/cpp_api/array.h | 3 +++ tiledb/sm/cpp_api/array_schema.h | 1 + tiledb/sm/cpp_api/query.h | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tiledb/sm/cpp_api/array.h b/tiledb/sm/cpp_api/array.h index b11d0b9af4d4..c5155b637260 100644 --- a/tiledb/sm/cpp_api/array.h +++ b/tiledb/sm/cpp_api/array.h @@ -661,6 +661,7 @@ class Array { * @param config Configuration parameters for the consolidation. */ // clang-format on + TILEDB_DEPRECATED static void consolidate( const Context& ctx, const std::string& uri, @@ -819,6 +820,7 @@ class Array { * @param encryption_key The encryption key to use. */ // clang-format on + TILEDB_DEPRECATED static void create( const std::string& uri, const ArraySchema& schema, @@ -1176,6 +1178,7 @@ class Array { * @param config Configuration parameters for the consolidation. */ // clang-format on + TILEDB_DEPRECATED static void consolidate_metadata( const Context& ctx, const std::string& uri, diff --git a/tiledb/sm/cpp_api/array_schema.h b/tiledb/sm/cpp_api/array_schema.h index 1e1681e90f2d..f7d7c3e44a19 100644 --- a/tiledb/sm/cpp_api/array_schema.h +++ b/tiledb/sm/cpp_api/array_schema.h @@ -179,6 +179,7 @@ class ArraySchema : public Schema { * @param encryption_type The encryption type to use. * @param encryption_key The encryption key to use. */ + TILEDB_DEPRECATED ArraySchema( const Context& ctx, const std::string& uri, diff --git a/tiledb/sm/cpp_api/query.h b/tiledb/sm/cpp_api/query.h index a7603be2d90e..22246ab706da 100644 --- a/tiledb/sm/cpp_api/query.h +++ b/tiledb/sm/cpp_api/query.h @@ -303,6 +303,8 @@ class Query { /** * Submit an async query, with callback. Call returns immediately. * + * Deprecated, call `submit()` on another thread instead. + * * @note Same notes apply as `Query::submit()`. * * **Example:** @@ -316,7 +318,7 @@ class Query { * @param callback Callback function. */ template - void submit_async(const Fn& callback) { + TILEDB_DEPRECATED void submit_async(const Fn& callback) { std::function wrapper = [&](void*) { callback(); }; auto& ctx = ctx_.get(); ctx.handle_error(tiledb::impl::tiledb_query_submit_async_func( @@ -326,6 +328,8 @@ class Query { /** * Submit an async query, with no callback. Call returns immediately. * + * Deprecated, call `submit()` on another thread instead. + * * @note Same notes apply as `Query::submit()`. * * **Example:** @@ -336,7 +340,7 @@ class Query { * query.submit_async(); * @endcode */ - void submit_async() { + TILEDB_DEPRECATED void submit_async() { submit_async([]() {}); } From 8031e3076df88190163bb91197f99407575a2e15 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 22 Apr 2024 08:53:24 -0400 Subject: [PATCH 290/456] Test concurrent sub-millisecond group metadata writes. (#4893) Add a test for concurrent `Group Metadata` writes within the same sub-millisecond. --- [sc-45351] --- TYPE: NO_HISTORY DESC: Test concurrent sub-millisecond Group Metadata writes. --- test/src/unit-capi-metadata.cc | 52 ++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/test/src/unit-capi-metadata.cc b/test/src/unit-capi-metadata.cc index 1503864034a1..84b7d05c8e95 100644 --- a/test/src/unit-capi-metadata.cc +++ b/test/src/unit-capi-metadata.cc @@ -334,8 +334,8 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( CMetadataFx, - "C API: Metadata, sub-millisecond writes", - "[capi][metadata][sub-millisecond]") { + "C API: Array Metadata, sub-millisecond writes", + "[capi][array][metadata][sub-millisecond]") { int32_t one = 1; int32_t two = 2; const void* v_r = nullptr; @@ -386,6 +386,54 @@ TEST_CASE_METHOD( } } +TEST_CASE_METHOD( + CMetadataFx, + "C API: Group Metadata, sub-millisecond writes", + "[capi][group][metadata][sub-millisecond]") { + std::string group_name = "test_group_meta_sub_millisecond_writes"; + int32_t one = 1; + int32_t two = 2; + const void* v_r = nullptr; + tiledb_datatype_t v_type; + uint32_t v_num; + + // Run the test body 100 times + for (int i = 0; i < 100; i++) { + // Create and open group in write mode + create_dir(group_name, ctx_, vfs_); + REQUIRE(tiledb_group_create(ctx_, group_name.c_str()) == TILEDB_OK); + tiledb_group_t* group; + REQUIRE(tiledb_group_alloc(ctx_, group_name.c_str(), &group) == TILEDB_OK); + REQUIRE(tiledb_group_open(ctx_, group, TILEDB_WRITE) == TILEDB_OK); + + // Write to disk twice + int rc = + tiledb_group_put_metadata(ctx_, group, "aaa", TILEDB_INT32, 1, &one); + REQUIRE(tiledb_group_close(ctx_, group) == TILEDB_OK); + REQUIRE(tiledb_group_open(ctx_, group, TILEDB_WRITE) == TILEDB_OK); + rc = tiledb_group_put_metadata(ctx_, group, "aaa", TILEDB_INT32, 1, &two); + CHECK(rc == TILEDB_OK); + REQUIRE(tiledb_group_close(ctx_, group) == TILEDB_OK); + tiledb_group_free(&group); + + // Open the group in read mode + REQUIRE(tiledb_group_alloc(ctx_, group_name.c_str(), &group) == TILEDB_OK); + REQUIRE(tiledb_group_open(ctx_, group, TILEDB_READ) == TILEDB_OK); + + // Read + rc = tiledb_group_get_metadata(ctx_, group, "aaa", &v_type, &v_num, &v_r); + CHECK(rc == TILEDB_OK); + CHECK(v_type == TILEDB_INT32); + CHECK(v_num == 1); + CHECK(*((const int32_t*)v_r) == 2); + + // Cleanup + REQUIRE(tiledb_group_close(ctx_, group) == TILEDB_OK); + tiledb_group_free(&group); + remove_dir(group_name, ctx_, vfs_); + } +} + TEST_CASE_METHOD( CMetadataFx, "C API: Metadata, UTF-8", "[capi][metadata][utf-8]") { // Create default array From 4fe78f8cebb678ef5b5d5e49fcde40ab93e8a61f Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 22 Apr 2024 15:54:05 +0300 Subject: [PATCH 291/456] Move the spdlog header files to `logger.cc`. (#4881) [SC-44926](https://app.shortcut.com/tiledb-inc/story/44926/move-spdlog-header-files-to-logger-cc) Spdlog includes `Windows.h` on Windows which includes lots of code, including some problematic defines like `DELETE`. This PR refactors the `Logger` class to no longer need including spdlog. We accomplish this by forward-declaring the `spdlog::logger` class and updating the formatted logging template methods to format the string with `fmt` themselves, and pass the string to the overload that logs strings. Formatting the strings is avoided if the respective log level is enabled. I added a method to check that. --- TYPE: NO_HISTORY --- test/support/src/helpers.h | 1 - tiledb/common/logger.cc | 24 ++++ tiledb/common/logger.h | 116 ++++++++++-------- tiledb/common/logger_public.h | 5 +- tiledb/sm/array/consistency.h | 1 - tiledb/sm/array/test/unit_consistency.cc | 1 + tiledb/sm/array_schema/dimension.h | 1 - tiledb/sm/compressors/dict_compressor.h | 1 - .../sm/consolidator/array_meta_consolidator.h | 1 - tiledb/sm/consolidator/commits_consolidator.h | 1 - tiledb/sm/consolidator/consolidator.h | 1 - .../sm/consolidator/fragment_consolidator.h | 1 - .../consolidator/fragment_meta_consolidator.h | 1 - tiledb/sm/filesystem/posix.h | 1 - tiledb/sm/misc/utils.h | 1 - tiledb/sm/query/legacy/reader.h | 1 - tiledb/sm/query/query.h | 1 - tiledb/sm/query/readers/dense_reader.h | 1 - .../query/readers/ordered_dim_label_reader.h | 1 - .../readers/sparse_global_order_reader.h | 1 - .../sparse_unordered_with_dups_reader.h | 1 - tiledb/sm/query/strategy_base.h | 1 - tiledb/sm/query/writers/dense_tiler.h | 1 - tiledb/sm/query_plan/test/unit_query_plan.cc | 1 + tiledb/sm/rest/curl.h | 6 +- tiledb/sm/rest/rest_client.h | 1 - tiledb/sm/storage_manager/context.h | 5 +- tiledb/sm/storage_manager/context_resources.h | 1 - .../storage_manager_canonical.h | 1 - tiledb/sm/subarray/subarray.h | 1 - tiledb/sm/subarray/subarray_partitioner.h | 1 - 31 files changed, 103 insertions(+), 79 deletions(-) diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index 3aecbc6f817e..bf179ff434a4 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -33,7 +33,6 @@ #ifndef TILEDB_TEST_HELPERS_H #define TILEDB_TEST_HELPERS_H -#include #include "test/support/src/coords_workaround.h" #include "test/support/src/mem_helpers.h" #include "tiledb.h" diff --git a/tiledb/common/logger.cc b/tiledb/common/logger.cc index ce2e8d769f5a..2c7e5235662a 100644 --- a/tiledb/common/logger.cc +++ b/tiledb/common/logger.cc @@ -34,6 +34,8 @@ #include #include #include +#include + #ifndef _WIN32 #include #endif @@ -42,6 +44,24 @@ namespace tiledb::common { +spdlog::level::level_enum to_spdlog_level(Logger::Level lvl) { + switch (lvl) { + case Logger::Level::FATAL: + return spdlog::level::critical; + case Logger::Level::ERR: + return spdlog::level::err; + case Logger::Level::WARN: + return spdlog::level::warn; + case Logger::Level::INFO: + return spdlog::level::info; + case Logger::Level::DBG: + return spdlog::level::debug; + default: + assert(lvl == Logger::Level::TRACE); + return spdlog::level::trace; + } +} + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -189,6 +209,10 @@ void Logger::fatal(const std::stringstream& msg) { exit(1); } +bool Logger::should_log(Logger::Level lvl) { + return logger_->should_log(to_spdlog_level(lvl)); +} + void Logger::set_level(Logger::Level lvl) { switch (lvl) { case Logger::Level::FATAL: diff --git a/tiledb/common/logger.h b/tiledb/common/logger.h index da7a3d9733c1..f0de1df2a6c4 100644 --- a/tiledb/common/logger.h +++ b/tiledb/common/logger.h @@ -29,29 +29,14 @@ * * This file defines class Logger, which is implemented as a wrapper around * `spdlog`. By policy `spdlog` must remain encapsulated as an implementation - * and not be exposed as a dependency of the TileDB library. Accordingly, this - * header should not be included as a header in any other header file. For - * inclusion in a header (notably for use within the definition of - * template-dependent functions), include the header `logger_public.h`. - * - * The reason for this restriction is a technical limitation in template - * instantiation. Part of the interface to `spdlog` consists of template - * functions with variadic template arguments. Instantiation of such function - * does not instantiate a variadic function (for exmaple `printf`) but rather a - * function with a fixed number of arguments that depend upon the argument list. - * Such variadic template argument lists cannot be forwarded across the - * boundaries of compilation units, so exposing variadic template arguments - * necessarily exposes the dependency upon `spdlog`. Thus this file `logger.h`, - * which does have such arguments, must remain entirely within the library, but - * `logger_public.h`, which does not have such arguments, may be exposed without - * creating an additional external dependency. + * and not be exposed as a dependency of the TileDB library. */ #pragma once #ifndef TILEDB_LOGGER_H #define TILEDB_LOGGER_H -#include +#include #include #include @@ -59,6 +44,10 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" +namespace spdlog { +class logger; +} + namespace tiledb { namespace common { @@ -123,12 +112,15 @@ class Logger { * * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for * details. - * @param arg positional argument to format. - * @param args optional additional positional arguments to format. + * @param args positional arguments to format. */ - template - void trace(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->trace(fmt::runtime(fmt), arg1, args...); + template + void trace(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::TRACE)) + return; + trace(fmt::format(fmt, std::forward(args)...)); } /** @@ -157,12 +149,15 @@ class Logger { * * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for * details. - * @param arg positional argument to format. - * @param args optional additional positional arguments to format. + * @param args positional arguments to format. */ - template - void debug(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->debug(fmt::runtime(fmt), arg1, args...); + template + void debug(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::DBG)) + return; + debug(fmt::format(fmt, std::forward(args)...)); } /** @@ -191,12 +186,15 @@ class Logger { * * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for * details. - * @param arg positional argument to format. - * @param args optional additional positional arguments to format. + * @param args positional arguments to format. */ - template - void info(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->info(fmt::runtime(fmt), arg1, args...); + template + void info(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::INFO)) + return; + info(fmt::format(fmt, std::forward(args)...)); } /** @@ -225,12 +223,15 @@ class Logger { * * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for * details. - * @param arg positional argument to format. - * @param args optional additional positional arguments to format. + * @param args positional arguments to format. */ - template - void warn(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->warn(fmt::runtime(fmt), arg1, args...); + template + void warn(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::WARN)) + return; + warn(fmt::format(fmt, std::forward(args)...)); } /** @@ -258,12 +259,15 @@ class Logger { * * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for * details. - * @param arg1 positional argument to format. - * @param args optional additional positional arguments to format. + * @param args positional arguments to format. */ - template - void error(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->error(fmt::runtime(fmt), arg1, args...); + template + void error(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::ERR)) + return; + error(fmt::format(fmt, std::forward(args)...)); } /** @@ -287,6 +291,22 @@ class Logger { */ void critical(const std::stringstream& msg); + /** + * A formatted critical statment. + * + * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for + * details. + * @param args positional arguments to format. + */ + template + void critical(fmt::format_string fmt, Args&&... args) { + // Check that this level is enabled to avoid needlessly formatting the + // string. + if (!should_log(Level::FATAL)) + return; + critical(fmt::format(fmt, std::forward(args)...)); + } + /** * Log a message from a Status object and return * the same Status object @@ -324,17 +344,11 @@ class Logger { void fatal(const std::stringstream& msg); /** - * A formatted critical statment. + * Check whether events of a given level are logged. * - * @param fmt A fmtlib format string, see http://fmtlib.net/latest/ for - * details. - * @param arg positional argument to format. - * @param args optional additional positional arguments to format. + * @param lvl The level of events to check. */ - template - void critical(std::string_view fmt, const Arg1& arg1, const Args&... args) { - logger_->critical(fmt::runtime(fmt), arg1, args...); - } + bool should_log(Logger::Level lvl); /** * Set the logger level. diff --git a/tiledb/common/logger_public.h b/tiledb/common/logger_public.h index 58e489feb87b..fa64ca3828f4 100644 --- a/tiledb/common/logger_public.h +++ b/tiledb/common/logger_public.h @@ -28,8 +28,9 @@ * @section DESCRIPTION * * This file defines simple logging functions that can be exposed (by expedient) - * to the public API. Their implementations are in `logger.cc`. See the - * documentation in `logger.h` for the full story. + * to the public API. Their implementations are in `logger.cc`. They are + * declared in a separate header from `logger.h` for historical reasons, when + * `logger.h` was leaking spdlog's headers. */ #pragma once diff --git a/tiledb/sm/array/consistency.h b/tiledb/sm/array/consistency.h index 67a265e8dadf..53eb34138d08 100644 --- a/tiledb/sm/array/consistency.h +++ b/tiledb/sm/array/consistency.h @@ -37,7 +37,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/filesystem/uri.h" diff --git a/tiledb/sm/array/test/unit_consistency.cc b/tiledb/sm/array/test/unit_consistency.cc index af24db27845b..c2e2b7cf2286 100644 --- a/tiledb/sm/array/test/unit_consistency.cc +++ b/tiledb/sm/array/test/unit_consistency.cc @@ -33,6 +33,7 @@ #include #include +#include "tiledb/common/logger.h" #include "unit_consistency.h" using namespace tiledb; diff --git a/tiledb/sm/array_schema/dimension.h b/tiledb/sm/array_schema/dimension.h index 45665f6e3ea7..a62f78a8ae77 100644 --- a/tiledb/sm/array_schema/dimension.h +++ b/tiledb/sm/array_schema/dimension.h @@ -43,7 +43,6 @@ #include "tiledb/common/blank.h" #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/macros.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" diff --git a/tiledb/sm/compressors/dict_compressor.h b/tiledb/sm/compressors/dict_compressor.h index 1aff103ffb5c..2dc3c7e6a0d7 100644 --- a/tiledb/sm/compressors/dict_compressor.h +++ b/tiledb/sm/compressors/dict_compressor.h @@ -34,7 +34,6 @@ #define TILEDB_DICT_COMPRESSOR_H #include "tiledb/common/common.h" -#include "tiledb/common/logger.h" #include "tiledb/sm/misc/endian.h" #include diff --git a/tiledb/sm/consolidator/array_meta_consolidator.h b/tiledb/sm/consolidator/array_meta_consolidator.h index 6da4cc997485..0393db98e709 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.h +++ b/tiledb/sm/consolidator/array_meta_consolidator.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" diff --git a/tiledb/sm/consolidator/commits_consolidator.h b/tiledb/sm/consolidator/commits_consolidator.h index 27ec1c4375b4..b5c1100b4ef9 100644 --- a/tiledb/sm/consolidator/commits_consolidator.h +++ b/tiledb/sm/consolidator/commits_consolidator.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index ddc1b8aea649..8f324995a660 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 4c3afb03e548..f1a5141e45e8 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/pmr.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.h b/tiledb/sm/consolidator/fragment_meta_consolidator.h index 230d3490f5b2..eb5027b0c17a 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.h +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" diff --git a/tiledb/sm/filesystem/posix.h b/tiledb/sm/filesystem/posix.h index 1c397e8e21ce..a97e1a0f678f 100644 --- a/tiledb/sm/filesystem/posix.h +++ b/tiledb/sm/filesystem/posix.h @@ -45,7 +45,6 @@ #include #include -#include "tiledb/common/logger.h" #include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/filesystem_base.h" diff --git a/tiledb/sm/misc/utils.h b/tiledb/sm/misc/utils.h index 2dbae1d5f365..804610a968ec 100644 --- a/tiledb/sm/misc/utils.h +++ b/tiledb/sm/misc/utils.h @@ -41,7 +41,6 @@ #include #include "constants.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" namespace tiledb { diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index d544edee0332..6cec8f488a7a 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -37,7 +37,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/indexed_list.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/query/iquery_strategy.h" diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index b8d093310253..a2dff571217b 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -39,7 +39,6 @@ #include #include -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/dimension.h" diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index dcbf60f3fa63..e44c7903fdc2 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -36,7 +36,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dynamic_array.h" diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.h b/tiledb/sm/query/readers/ordered_dim_label_reader.h index 3c5dd9ff8ef1..afb8ad1fc3df 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.h +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.h @@ -36,7 +36,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/misc/types.h" diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.h b/tiledb/sm/query/readers/sparse_global_order_reader.h index d213af6a253e..c113dc6c6128 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.h +++ b/tiledb/sm/query/readers/sparse_global_order_reader.h @@ -36,7 +36,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/query/iquery_strategy.h" diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h index 7f271682c5f9..1ec82e2c70c4 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.h @@ -36,7 +36,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/query/iquery_strategy.h" diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index d1d157c10c5d..8aa91b6301a4 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -34,7 +34,6 @@ #define TILEDB_STRATEGY_BASE_H #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/misc/types.h" diff --git a/tiledb/sm/query/writers/dense_tiler.h b/tiledb/sm/query/writers/dense_tiler.h index fdcfa6d5789c..9404e0c5de77 100644 --- a/tiledb/sm/query/writers/dense_tiler.h +++ b/tiledb/sm/query/writers/dense_tiler.h @@ -38,7 +38,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/query/query_buffer.h" diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 02776c226f17..ba33ddb15df7 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -36,6 +36,7 @@ #include "../query_plan.h" #include "external/include/nlohmann/json.hpp" #include "test/support/src/mem_helpers.h" +#include "tiledb/common/logger.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/enums/array_type.h" diff --git a/tiledb/sm/rest/curl.h b/tiledb/sm/rest/curl.h index dc7bf103646c..3270f3ce5b65 100644 --- a/tiledb/sm/rest/curl.h +++ b/tiledb/sm/rest/curl.h @@ -48,7 +48,6 @@ #include #include "tiledb/common/dynamic_memory/dynamic_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/buffer/buffer_list.h" #include "tiledb/sm/config/config.h" @@ -59,6 +58,11 @@ using namespace tiledb::common; namespace tiledb { + +namespace common { +class Logger; +} + namespace sm { /** diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 25eb29507ddd..3c03328127fc 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -36,7 +36,6 @@ #include #include -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/group/group.h" diff --git a/tiledb/sm/storage_manager/context.h b/tiledb/sm/storage_manager/context.h index cfed00dc1bb7..a4811694fb08 100644 --- a/tiledb/sm/storage_manager/context.h +++ b/tiledb/sm/storage_manager/context.h @@ -34,7 +34,6 @@ #define TILEDB_CONTEXT_H #include "tiledb/common/exception/exception.h" -#include "tiledb/common/logger.h" #include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/stats/global_stats.h" @@ -43,6 +42,10 @@ #include +namespace tiledb::common { +class Logger; +} + using namespace tiledb::common; namespace tiledb::sm { diff --git a/tiledb/sm/storage_manager/context_resources.h b/tiledb/sm/storage_manager/context_resources.h index dea51cfde416..515dcdd450b3 100644 --- a/tiledb/sm/storage_manager/context_resources.h +++ b/tiledb/sm/storage_manager/context_resources.h @@ -34,7 +34,6 @@ #define TILEDB_CONTEXT_RESOURCES_H #include "tiledb/common/exception/exception.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/vfs.h" diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 9f5017d7a2d1..2a255ebd2394 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -47,7 +47,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/array/array_directory.h" diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 30251f07469f..c07854bfef25 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -36,7 +36,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" diff --git a/tiledb/sm/subarray/subarray_partitioner.h b/tiledb/sm/subarray/subarray_partitioner.h index 2915fd7aefe7..e3df1540eb1c 100644 --- a/tiledb/sm/subarray/subarray_partitioner.h +++ b/tiledb/sm/subarray/subarray_partitioner.h @@ -37,7 +37,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/logger_public.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/stats/stats.h" From 531d05f55db112a1a2af9c1c90b1a5fcd381e432 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:57:46 +0200 Subject: [PATCH 292/456] Fix documentation for tiledb_array_consolidate_fragments. (#4888) Clarified the fragment_uris parameter documentation. --- TYPE: NO_HISTORY DESC: Fix documentation for tiledb_array_consolidate_fragments. --- tiledb/sm/c_api/tiledb_experimental.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index bebb1c9ae9c9..cedfe674a869 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -746,7 +746,8 @@ TILEDB_EXPORT capi_return_t tiledb_ctx_alloc_with_error( * @param[in] ctx The TileDB context. * @param[in] array_uri The name of the TileDB array whose metadata will * be consolidated. - * @param[in] fragment_uris URIs of the fragments to consolidate. + * @param[in] fragment_uris Fragment names of the fragments to consolidate. The + * names can be recovered using tiledb_fragment_info_get_fragment_name. * @param[in] num_fragments Number of URIs to consolidate. * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config from \p ctx). From 775b7de19f2b6164e46e8b0d04d30ff95237e137 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 22 Apr 2024 21:44:29 +0300 Subject: [PATCH 293/456] Always use Ninja on Windows CI. (#4886) [SC-44745](https://app.shortcut.com/tiledb-inc/story/44745/remove-ninja-workaround-for-windows-builds) In #4759, I switched the VS2022 builds to use Ninja due to errors caused by MSVC toolset version mismatch but could not make the VS2019 builds use Ninja as well and resorted to some conditional logic to support both Ninja and MSBuild. I subsequently found out what the problem with VS2019 was (the action to install Ninja was running before the checkout action, and the latter deleted the downloaded Ninja), and this PR enables Ninja on all Visual Studio versions and deletes the conditional logic, simplifying the workflow. Ninja is preferred to MSBuild because of the superior parallelization capabilities. For coverage, MSBuild is still being used on nightly builds. --- TYPE: NO_HISTORY --- .github/workflows/build-windows.yml | 43 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 0fb5879d53ac..9b62cff059db 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -60,22 +60,8 @@ jobs: TILEDB_ARROW_TESTS: ${{ matrix.TILEDB_ARROW_TESTS }} TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' - # On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory - CONFIG_PATH_FIXUP: ${{ matrix.os == 'windows-2019' && 'Release' || '' }} VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' steps: - # By default Visual Studio chooses the earliest installed toolset version - # for the main build and vcpkg chooses the latest. Force it to use the - # latest (14.39 currently). - - name: Setup MSVC toolset (VS 2022) - uses: TheMrMilchmann/setup-msvc-dev@v3 - if: matrix.os == 'windows-2022' - with: - arch: x64 - toolset: 14.39 - - name: Install Ninja (VS 2022) - uses: seanmiddleditch/gha-setup-ninja@v4 - if: matrix.os == 'windows-2022' - name: 'tiledb env prep' run: | $env:BUILD_BUILDDIRECTORY = $env:GITHUB_WORKSPACE.replace("TileDB\TileDB","tdbbd") # 't'ile'db' 'b'uild 'd'ir @@ -141,6 +127,17 @@ jobs: submodules: true fetch-depth: 0 + - name: Setup MSVC toolset + uses: TheMrMilchmann/setup-msvc-dev@v3 + with: + arch: x64 + # By default Visual Studio 2022 chooses the earliest installed toolset + # version for the main build and vcpkg chooses the latest. Force it to + # use the latest (14.39 currently). + toolset: ${{ matrix.os == 'windows-2022' && '14.39' || '' }} + # This must happen after checkout, because checkout would remove the directory. + - name: Install Ninja + uses: seanmiddleditch/gha-setup-ninja@v4 - name: Prevent vcpkg from building debug variants run: python $env:GITHUB_WORKSPACE/scripts/ci/patch_vcpkg_triplets.py @@ -155,7 +152,7 @@ jobs: # allow double-checking path cmd /c "echo $PATH" - $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + " -CMakeGenerator ${{ matrix.os == 'windows-2022' && 'Ninja' || '`"Visual Studio 16 2019`"' }}" + $bootstrapOptions = $env:TILEDB_BASE_BOOTSTRAP_OPTIONS + " -CMakeGenerator Ninja" if ($env:TILEDB_S3 -eq "ON") { $bootstrapOptions = "-EnableS3 " + $bootstrapOptions } @@ -284,7 +281,7 @@ jobs: # Actually run tests - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\$env:CONFIG_PATH_FIXUP\tiledb_unit.exe -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\tiledb_unit.exe -d=yes" Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { @@ -292,7 +289,7 @@ jobs: $host.SetShouldExit($LastExitCode) } - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\$env:CONFIG_PATH_FIXUP\unit_vfs -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\tiledb\sm\filesystem\test\unit_vfs -d=yes" Write-Host "cmds: '$cmds'" Invoke-Expression $cmds if ($LastExitCode -ne 0) { @@ -300,7 +297,7 @@ jobs: $host.SetShouldExit($LastExitCode) } - $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\$env:CONFIG_PATH_FIXUP\test_assert.exe -d=yes" + $cmds = "$env:BUILD_BUILDDIRECTORY\tiledb\test\ci\test_assert.exe -d=yes" Invoke-Expression $cmds if ($LastExitCode -ne 0) { Write-Host "Tests failed. test_assert exit status: " $LastExitCocde @@ -322,7 +319,7 @@ jobs: $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api") $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\test_app_data") - Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe | + Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\c_api\") -Filter *.exe | Foreach-Object { try { Set-Location -path $TestAppDir @@ -355,7 +352,7 @@ jobs: $TestAppDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api") $TestAppDataDir = (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\test_app_data") - Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\$env:CONFIG_PATH_FIXUP\") -Filter *.exe | + Get-ChildItem (Join-Path $env:BUILD_BUILDDIRECTORY "tiledb\examples\cpp_api\") -Filter *.exe | Foreach-Object { try { Set-Location -path $TestAppDir @@ -400,12 +397,12 @@ jobs: cd build # Build zip artifact - cmake ${{ matrix.os != 'windows-2019' && '-G Ninja' || '' }} -DCMAKE_BUILD_TYPE="$CMakeBuildType" -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. + cmake -G Ninja -DCMAKE_BUILD_TYPE="$CMakeBuildType" -DCMAKE_PREFIX_PATH="$env:BUILD_BUILDDIRECTORY\dist;$env:BUILD_BUILDDIRECTORY\vcpkg_installed\x64-windows" .. cmake --build . --config $CMakeBuildType -v - #.\$env:CONFIG_PATH_FIXUP\ExampleExe.exe - $cmd = ".\$env:CONFIG_PATH_FIXUP\ExampleExe.exe" + #.\ExampleExe.exe + $cmd = ".\ExampleExe.exe" Write-Host "cmd: '$cmd'" Invoke-Expression $cmd From 3925eeba21c1ed6fc81335038f1c365dad7841bf Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:53:33 +0300 Subject: [PATCH 294/456] Add C++ API for tiledb_array_consolidate_fragments. (#4884) Adding a missing C++ API. To pass the test I created I needed to use the last part of the full fragment URIs which I got from the FragmentInfo. I believe I should have been able to use the URIs as is, without further modifications. This is happening because of: https://github.com/TileDB-Inc/TileDB/blob/6dff6fae0b09c16177fe85ec9b62c128bd721ad7/tiledb/sm/consolidator/fragment_consolidator.cc#L377 I decided to keep this PR just a C++ addition. I can follow-up with an investigation if the behavior I describe above is not the expected. [sc-44902] --- TYPE: CPP_API DESC: Add C++ API for tiledb_array_consolidate_fragments. --- test/src/unit-cppapi-consolidation.cc | 37 +++++++++++++++++++++++ tiledb/sm/cpp_api/array.h | 43 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/test/src/unit-cppapi-consolidation.cc b/test/src/unit-cppapi-consolidation.cc index 845d2b611025..4febf411d467 100644 --- a/test/src/unit-cppapi-consolidation.cc +++ b/test/src/unit-cppapi-consolidation.cc @@ -205,6 +205,43 @@ TEST_CASE( remove_array(array_name); } +TEST_CASE( + "C++ API: Test consolidation with fragment list", + "[cppapi][consolidation]") { + std::string array_name = "cppapi_consolidation"; + remove_array(array_name); + + create_array(array_name); + write_array(array_name, {1, 2}, {1, 2}); + write_array(array_name, {3, 3}, {3}); + CHECK(tiledb::test::num_fragments(array_name) == 2); + + read_array(array_name, {1, 3}, {1, 2, 3}); + + Context ctx; + Config config; + + FragmentInfo fragment_info(ctx, array_name); + fragment_info.load(); + std::string fragment_name1 = fragment_info.fragment_uri(0); + std::string fragment_name2 = fragment_info.fragment_uri(1); + std::string short_fragment_name1 = + fragment_name1.substr(fragment_name1.find_last_of('/') + 1); + std::string short_fragment_name2 = + fragment_name2.substr(fragment_name2.find_last_of('/') + 1); + + const char* fragment_uris[2] = { + short_fragment_name1.c_str(), short_fragment_name2.c_str()}; + + REQUIRE_NOTHROW( + Array::consolidate(ctx, array_name, fragment_uris, 2, &config)); + CHECK(tiledb::test::num_fragments(array_name) == 3); + + read_array(array_name, {1, 3}, {1, 2, 3}); + + remove_array(array_name); +} + TEST_CASE( "C++ API: Test consolidation with timestamp and max domain", "[cppapi][consolidation][timestamp][maxdomain]") { diff --git a/tiledb/sm/cpp_api/array.h b/tiledb/sm/cpp_api/array.h index c5155b637260..5151f7ed8c86 100644 --- a/tiledb/sm/cpp_api/array.h +++ b/tiledb/sm/cpp_api/array.h @@ -643,6 +643,49 @@ class Array { config ? config->ptr().get() : nullptr)); } + /** + * @brief Consolidates the fragments with the input uris into a single + * fragment. + * + * You must first finalize all queries to the array before consolidation can + * begin (as consolidation temporarily acquires an exclusive lock on the + * array). + * + * **Example:** + * @code{.cpp} + * const char* fragment_uris[2] = { + * "__1712657401931_1712657401931_285cf8a0eff4df875a04cfbea96d5c00_21", + * "__1712657401948_1712657401948_285cf8a0efdsafas6a5a04cfbesajads_21"}; + * + * tiledb::Array::consolidate( + * ctx, + * "s3://bucket-name/array-name", + * fragment_uris, + * 2, + * config); + * @endcode + * + * @param ctx TileDB context + * @param array_uri The URI of the TileDB array to be consolidated. + * @param fragment_uris Fragment names of the fragments to consolidate. The + * names can be recovered using tiledb_fragment_info_get_fragment_name. + * @param num_fragments The number of fragments to consolidate. + * @param config Configuration parameters for the consolidation. + */ + static void consolidate( + const Context& ctx, + const std::string& array_uri, + const char* fragment_uris[], + const size_t num_fragments, + Config* const config = nullptr) { + ctx.handle_error(tiledb_array_consolidate_fragments( + ctx.ptr().get(), + array_uri.c_str(), + fragment_uris, + num_fragments, + config ? config->ptr().get() : nullptr)); + } + // clang-format off /** * @copybrief Array::consolidate(const Context&,const std::string&,tiledb_encryption_type_t,const void*,uint32_t,const Config&) From 5298c2252a7ebb34021439553be84b7e5a1bf3d3 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 24 Apr 2024 13:01:08 -0500 Subject: [PATCH 295/456] Fix C query condition examples (#4912) The examples were attempting to use query conditions that were already destroyed in the last example which led to erroneous output. Shoutout to @rroelke who was the one that noticed the disparity. [sc-46017] --- TYPE: BUG DESC: Fix C query condition examples --- examples/c_api/query_condition_dense.c | 4 ++-- examples/c_api/query_condition_sparse.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/c_api/query_condition_dense.c b/examples/c_api/query_condition_dense.c index 35468e9c3d19..29bc5f81a56b 100644 --- a/examples/c_api/query_condition_dense.c +++ b/examples/c_api/query_condition_dense.c @@ -308,7 +308,6 @@ int main() { const char* eve = "eve"; tiledb_query_condition_init(ctx, qc1, "b", eve, strlen(eve), TILEDB_LT); read_array_with_qc(ctx, qc1); - tiledb_query_condition_free(&qc1); printf("\n"); // Execute a read query with query condition `c >= 1`. @@ -338,7 +337,6 @@ int main() { read_array_with_qc(ctx, qc5); tiledb_query_condition_free(&qc3); tiledb_query_condition_free(&qc4); - tiledb_query_condition_free(&qc5); printf("\n"); // Execute a read query with query condition `3.0f <= d AND d <= 4.0f AND a != @@ -356,6 +354,8 @@ int main() { tiledb_query_condition_alloc(ctx, &qc8); tiledb_query_condition_combine(ctx, qc7, qc1, TILEDB_AND, &qc8); read_array_with_qc(ctx, qc8); + tiledb_query_condition_free(&qc1); + tiledb_query_condition_free(&qc5); tiledb_query_condition_free(&qc6); tiledb_query_condition_free(&qc7); tiledb_query_condition_free(&qc8); diff --git a/examples/c_api/query_condition_sparse.c b/examples/c_api/query_condition_sparse.c index 3478e7bc8f2f..fe59630ff3f8 100644 --- a/examples/c_api/query_condition_sparse.c +++ b/examples/c_api/query_condition_sparse.c @@ -298,7 +298,6 @@ int main() { const char* eve = "eve"; tiledb_query_condition_init(ctx, qc1, "b", eve, strlen(eve), TILEDB_LT); read_array_with_qc(ctx, qc1); - tiledb_query_condition_free(&qc1); printf("\n"); // Execute a read query with query condition `c >= 1`. @@ -328,7 +327,6 @@ int main() { read_array_with_qc(ctx, qc5); tiledb_query_condition_free(&qc3); tiledb_query_condition_free(&qc4); - tiledb_query_condition_free(&qc5); printf("\n"); // Execute a read query with query condition `3.0f <= d AND d <= 4.0f AND a != @@ -346,6 +344,8 @@ int main() { tiledb_query_condition_alloc(ctx, &qc8); tiledb_query_condition_combine(ctx, qc7, qc1, TILEDB_AND, &qc8); read_array_with_qc(ctx, qc8); + tiledb_query_condition_free(&qc1); + tiledb_query_condition_free(&qc5); tiledb_query_condition_free(&qc6); tiledb_query_condition_free(&qc7); tiledb_query_condition_free(&qc8); From d1a5f225cb7f329e2186ba057547624cc15a3f08 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Fri, 26 Apr 2024 01:40:09 -0600 Subject: [PATCH 296/456] Subarray Rework B: FieldDataSize (#4891) Add `struct FieldDataSize`. There are already two incorrect versions of this within `class Subarray`, one which uses `double` (??!) and one which uses `uint64_t`. The correct type for sizes of objects held within program memory is `size_t`, per the C++ standard. Change return type of `get_max_memory_size` to `FieldDataSize` and remove output arguments. Combine all four `get_max_memory_size*` functions into a single one, thus removing both copypasta and extraneous checks. Change return type of `Subarray::get_est_result_size` to `FieldDataView` and remove output arguments. Combine all four `get_est_result_size*` functions into a single one, thus again removing both copypasta and extraneous checks. Rework the `Query::get_est_result_size_*` functions that directly serve the C API. Rationalize their names. Move validation for `Query::get_est_result_size_*` to separate functions, which yet again removes a large amount of copypasta. Add null pointer checks to C API functions `tiledb_query_est_result_size*`. Previous null checks were not exhaustive and did not happen in the C API implementation function. Fix defect in `ArraySchema::is_field`, which did not consider a dimension label as a field. Removed unused function `get_next_range_coords`. Added another item to `.gitignore` to cope with an increase requirement from AWS for short lengths in the build directory prefix. --- TYPE: NO_HISTORY DESC: Subarray Rework B: FieldDataSize --- .gitignore | 1 + tiledb/sm/array_schema/array_schema.cc | 3 +- tiledb/sm/c_api/tiledb.cc | 71 ++- tiledb/sm/query/query.cc | 215 ++++----- tiledb/sm/query/query.h | 82 +++- tiledb/sm/subarray/subarray.cc | 492 +++------------------ tiledb/sm/subarray/subarray.h | 119 ++--- tiledb/sm/subarray/subarray_partitioner.cc | 116 +---- 8 files changed, 327 insertions(+), 772 deletions(-) diff --git a/.gitignore b/.gitignore index 5c529f44c40e..fea5a2fdd6f2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .vscode* *.sw? build/* +build-*/* dist/* cmake-build-*/* core/bin/* diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index e5bee621663b..76a0bc22424b 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -775,7 +775,8 @@ bool ArraySchema::is_dim_label(const std::string& name) const { } bool ArraySchema::is_field(const std::string& name) const { - return is_attr(name) || is_dim(name) || is_special_attribute(name); + return is_attr(name) || is_dim(name) || is_dim_label(name) || + is_special_attribute(name); } bool ArraySchema::is_nullable(const std::string& name) const { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 944349b769b0..0b67cda4795b 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1541,11 +1541,17 @@ int32_t tiledb_query_get_est_result_size( const tiledb_query_t* query, const char* name, uint64_t* size) { - if (sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; - - throw_if_not_ok(query->query_->get_est_result_size(name, size)); - + } + if (name == nullptr) { + throw CAPIStatusException("Pointer to field name may not be NULL"); + } + if (size == nullptr) { + throw CAPIStatusException("Pointer to size may not be NULL"); + } + auto est_size{query->query_->get_est_result_size_fixed_nonnull(name)}; + *size = est_size.fixed_; return TILEDB_OK; } @@ -1555,10 +1561,22 @@ int32_t tiledb_query_get_est_result_size_var( const char* name, uint64_t* size_off, uint64_t* size_val) { - if (sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; + } + if (name == nullptr) { + throw CAPIStatusException("Pointer to field name may not be NULL"); + } + if (size_off == nullptr) { + throw CAPIStatusException("Pointer to offset size may not be NULL"); + } + if (size_val == nullptr) { + throw CAPIStatusException("Pointer to value size may not be NULL"); + } - throw_if_not_ok(query->query_->get_est_result_size(name, size_off, size_val)); + auto est_size{query->query_->get_est_result_size_variable_nonnull(name)}; + *size_off = est_size.fixed_; + *size_val = est_size.variable_; return TILEDB_OK; } @@ -1569,12 +1587,22 @@ int32_t tiledb_query_get_est_result_size_nullable( const char* name, uint64_t* size_val, uint64_t* size_validity) { - if (sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; + } + if (name == nullptr) { + throw CAPIStatusException("Pointer to field name may not be NULL"); + } + if (size_val == nullptr) { + throw CAPIStatusException("Pointer to value size may not be NULL"); + } + if (size_validity == nullptr) { + throw CAPIStatusException("Pointer to validity size may not be NULL"); + } - throw_if_not_ok(query->query_->get_est_result_size_nullable( - name, size_val, size_validity)); - + auto est_size{query->query_->get_est_result_size_fixed_nullable(name)}; + *size_val = est_size.fixed_; + *size_validity = est_size.validity_; return TILEDB_OK; } @@ -1585,12 +1613,25 @@ int32_t tiledb_query_get_est_result_size_var_nullable( uint64_t* size_off, uint64_t* size_val, uint64_t* size_validity) { - if (sanity_check(ctx, query) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; - - throw_if_not_ok(query->query_->get_est_result_size_nullable( - name, size_off, size_val, size_validity)); - + } + if (name == nullptr) { + throw CAPIStatusException("Pointer to field name may not be NULL"); + } + if (size_off == nullptr) { + throw CAPIStatusException("Pointer to offset size may not be NULL"); + } + if (size_val == nullptr) { + throw CAPIStatusException("Pointer to value size may not be NULL"); + } + if (size_validity == nullptr) { + throw CAPIStatusException("Pointer to validity size may not be NULL"); + } + auto est_size{query->query_->get_est_result_size_variable_nullable(name)}; + *size_off = est_size.fixed_; + *size_val = est_size.variable_; + *size_validity = est_size.validity_; return TILEDB_OK; } diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 57fa870545fc..4d88f1a39905 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -151,172 +151,111 @@ Query::~Query() { /* API */ /* ****************************** */ -Status Query::get_est_result_size(const char* name, uint64_t* size) { - if (type_ != QueryType::READ) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Operation currently " - "only unsupported for read queries")); +void Query::field_require_array_fixed( + const std::string_view origin, const char* name) { + if (!array_schema_->is_field(name)) { + throw QueryException( + std::string{origin} + ": '" + name + "' is not an array field"); } - - if (name == nullptr) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Name cannot be null")); + if (array_schema_->var_size(name)) { + throw QueryException( + std::string{origin} + ": '" + name + "' is not fixed-sized"); } +} - if (name == constants::coords && - !array_schema_->domain().all_dims_same_type()) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Not applicable to zipped " - "coordinates in arrays with heterogeneous domain")); +void Query::field_require_array_variable( + const std::string_view origin, const char* name) { + if (!array_schema_->is_field(name)) { + throw QueryException( + std::string{origin} + ": '" + name + "' is not an array field"); } - - if (name == constants::coords && !array_schema_->domain().all_dims_fixed()) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Not applicable to zipped " - "coordinates in arrays with domains with variable-sized dimensions")); + if (!array_schema_->var_size(name)) { + throw QueryException( + std::string{origin} + ": '" + name + "' is not variable-sized"); } +} - if (array_schema_->is_nullable(name)) { - return logger_->status(Status_QueryError( - std::string( - "Cannot get estimated result size; Input attribute/dimension '") + - name + "' is nullable")); +void Query::field_require_array_nullable( + const std::string_view origin, const char* name) { + if (!array_schema_->attribute(name)) { + throw QueryException( + std::string{origin} + ": '" + name + + "' is not the name of an attribute"); } - - if (array_->is_remote() && !subarray_.est_result_size_computed()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) { - return logger_->status( - Status_QueryError("Error in query estimate result size; remote " - "array with no rest client.")); - } - - RETURN_NOT_OK( - rest_client->get_query_est_result_sizes(array_->array_uri(), this)); + if (!array_schema_->is_nullable(name)) { + throw QueryException( + std::string{origin} + ": attribute '" + name + "' is not nullable"); } - - subarray_.get_est_result_size_internal( - name, size, &config_, storage_manager_->compute_tp()); - return Status::Ok(); } -Status Query::get_est_result_size( - const char* name, uint64_t* size_off, uint64_t* size_val) { - if (type_ != QueryType::READ) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Operation currently " - "only supported for read queries")); - } - +void Query::field_require_array_nonnull( + const std::string_view origin, const char* name) { if (array_schema_->is_nullable(name)) { - return logger_->status(Status_QueryError( - std::string( - "Cannot get estimated result size; Input attribute/dimension '") + - name + "' is nullable")); - } - - if (array_->is_remote() && !subarray_.est_result_size_computed()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) { - return logger_->status( - Status_QueryError("Error in query estimate result size; remote " - "array with no rest client.")); - } - - RETURN_NOT_OK( - rest_client->get_query_est_result_sizes(array_->array_uri(), this)); + throw QueryException( + std::string(origin) + ": field '" + name + + "' is not a nonnull array field"); } - - subarray_.get_est_result_size( - name, size_off, size_val, &config_, storage_manager_->compute_tp()); - return Status::Ok(); } -Status Query::get_est_result_size_nullable( - const char* name, uint64_t* size_val, uint64_t* size_validity) { - if (type_ != QueryType::READ) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Operation currently " - "only supported for read queries")); - } - - if (name == nullptr) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Name cannot be null")); - } +constexpr std::string_view origin_est_result_size{ + "query estimated result size"}; - if (!array_schema_->attribute(name)) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Nullable API is only" - "applicable to attributes")); - } - - if (!array_schema_->is_nullable(name)) { - return logger_->status(Status_QueryError( - std::string("Cannot get estimated result size; Input attribute '") + - name + "' is not nullable")); +FieldDataSize Query::internal_est_result_size(const char* name) { + if (type_ != QueryType::READ) { + throw QueryException( + std::string{origin_est_result_size} + + ": operation currently supported only for read queries"); } - if (array_->is_remote() && !subarray_.est_result_size_computed()) { auto rest_client = storage_manager_->rest_client(); if (rest_client == nullptr) { - return logger_->status( - Status_QueryError("Error in query estimate result size; remote " - "array with no rest client.")); + throw QueryStatusException( + "Error in query estimate result size; " + "remote array with no rest client."); } - - RETURN_NOT_OK( + throw_if_not_ok( rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } - - subarray_.get_est_result_size_nullable( - name, size_val, size_validity, &config_, storage_manager_->compute_tp()); - return Status::Ok(); + return subarray_.get_est_result_size( + name, &config_, storage_manager_->compute_tp()); } -Status Query::get_est_result_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity) { - if (type_ != QueryType::READ) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Operation currently " - "only supported for read queries")); - } - - if (!array_schema_->attribute(name)) { - return logger_->status(Status_QueryError( - "Cannot get estimated result size; Nullable API is only" - "applicable to attributes")); - } - - if (!array_schema_->is_nullable(name)) { - return logger_->status(Status_QueryError( - std::string("Cannot get estimated result size; Input attribute '") + - name + "' is not nullable")); +FieldDataSize Query::get_est_result_size_fixed_nonnull(const char* name) { + field_require_array_fixed(origin_est_result_size, name); + if (name == constants::coords) { + if (!array_schema_->domain().all_dims_same_type()) { + throw QueryException( + std::string{origin_est_result_size} + + ": not applicable to zipped coordinates " + "in arrays with heterogeneous domain"); + } + if (!array_schema_->domain().all_dims_fixed()) { + throw QueryException( + std::string{origin_est_result_size} + + ": not applicable to zipped coordinates " + "in arrays with domains with variable-sized dimensions"); + } } + field_require_array_nonnull(origin_est_result_size, name); + return internal_est_result_size(name); +} - if (array_->is_remote() && !subarray_.est_result_size_computed()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) { - return logger_->status( - Status_QueryError("Error in query estimate result size; remote " - "array with no rest client.")); - } +FieldDataSize Query::get_est_result_size_variable_nonnull(const char* name) { + field_require_array_variable(origin_est_result_size, name); + field_require_array_nonnull(origin_est_result_size, name); + return internal_est_result_size(name); +} - RETURN_NOT_OK( - rest_client->get_query_est_result_sizes(array_->array_uri(), this)); - } +FieldDataSize Query::get_est_result_size_fixed_nullable(const char* name) { + field_require_array_fixed(origin_est_result_size, name); + field_require_array_nullable(origin_est_result_size, name); + return internal_est_result_size(name); +} - subarray_.get_est_result_size_nullable( - name, - size_off, - size_val, - size_validity, - &config_, - storage_manager_->compute_tp()); - return Status::Ok(); +FieldDataSize Query::get_est_result_size_variable_nullable(const char* name) { + field_require_array_variable(origin_est_result_size, name); + field_require_array_nullable(origin_est_result_size, name); + return internal_est_result_size(name); } std::unordered_map diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index a2dff571217b..637aa019106e 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -63,12 +63,13 @@ namespace tiledb { namespace sm { /** Class for query status exceptions. */ -class QueryStatusException : public StatusException { +class QueryException : public StatusException { public: - explicit QueryStatusException(const std::string& msg) + explicit QueryException(const std::string& msg) : StatusException("Query", msg) { } }; +using QueryStatusException = QueryException; class Array; class ArrayDimensionLabelQueries; @@ -170,36 +171,93 @@ class Query { /* API */ /* ********************************* */ + /** + * Require that a name be that of a fixed-size field from the source array. + * + * Validate these conditions and throw if any are not met. + * - The name is that of a field in the array + * - The field is fixed-sized + * + * @param origin The name of the operation that this validation is a part of + * @param name The name of a field + */ + void field_require_array_fixed( + const std::string_view origin, const char* name); + + /** + * Require that a name be that of a variable-size field from the source array. + * + * Validate these conditions and throw if any are not met. + * - The name is that of a field in the array + * - The field is variable-sized + * + * @param origin The name of the operation that this validation is a part of + * @param name The name of a field + */ + void field_require_array_variable( + const std::string_view origin, const char* name); + + /** + * Require that a field be a nullable field from the source array. + * + * Validate these conditions and throw if any are not met. + * - The name is that of an attribute of the source array + * - The attribute is nullable + * + * @param origin The name of the operation that this validation is a part of + * @param name The name of a field + */ + void field_require_array_nullable( + const std::string_view origin, const char* name); + + /** + * Require that a field be a nonnull field from the source array. + * + * Validate these conditions and throw if any are not met. + * - The name is that of a field in the array + * - The field is not nullable + * + * @param origin The name of the operation that this validation is a part of. + * @param name The name of a field + */ + void field_require_array_nonnull( + const std::string_view origin, const char* name); + /** * Gets the estimated result size (in bytes) for the input fixed-sized * attribute/dimension. */ - Status get_est_result_size(const char* name, uint64_t* size); + FieldDataSize get_est_result_size_fixed_nonnull(const char* name); /** * Gets the estimated result size (in bytes) for the input var-sized * attribute/dimension. */ - Status get_est_result_size( - const char* name, uint64_t* size_off, uint64_t* size_val); + FieldDataSize get_est_result_size_variable_nonnull(const char* name); /** * Gets the estimated result size (in bytes) for the input fixed-sized, * nullable attribute. */ - Status get_est_result_size_nullable( - const char* name, uint64_t* size_val, uint64_t* size_validity); + FieldDataSize get_est_result_size_fixed_nullable(const char* name); /** * Gets the estimated result size (in bytes) for the input var-sized, * nullable attribute. */ - Status get_est_result_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity); + FieldDataSize get_est_result_size_variable_nullable(const char* name); + private: + /** + * Common part of all `est_result_size_*` functions, called after argument + * validation. + * + * @param name The name of a field + * @return estimated result size + */ + FieldDataSize internal_est_result_size(const char* name); + + public: /** Retrieves the number of written fragments. */ Status get_written_fragment_num(uint32_t* num) const; diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index b155e12ecb02..71386e063cc0 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -1103,22 +1103,12 @@ Layout Subarray::layout() const { return layout_; } -void Subarray::get_est_result_size_internal( - const char* name, - uint64_t* size, - const Config* config, - ThreadPool* compute_tp) { +FieldDataSize Subarray::get_est_result_size( + const char* name, const Config* config, ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) { throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension name cannot be null"); - } - - // Check size pointer - if (size == nullptr) { - throw SubarrayException( - "Cannot get estimated result size; Input size cannot be null"); + "Cannot get estimated result size; field name cannot be null"); } // Check if name is attribute or dimension @@ -1129,290 +1119,74 @@ void Subarray::get_est_result_size_internal( // Check if attribute/dimension exists if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { throw SubarrayException( - std::string("Cannot get estimated result size; Attribute/Dimension '") + - name + "' does not exist"); + std::string("Cannot get estimated result size; ") + + "there is no field named '" + name + "'"); } - // Check if the attribute/dimension is fixed-sized - if (array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must be fixed-sized"); - } - - // Check if attribute/dimension is nullable - if (array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must not be nullable"); - } - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size = static_cast(std::ceil(est_result_size_[name].size_fixed_)); - - // If the size is non-zero, ensure it is large enough to - // contain at least one cell. - const auto cell_size = array_schema.cell_size(name); - if (*size > 0 && *size < cell_size) - *size = cell_size; -} - -void Subarray::get_est_result_size( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension name cannot be null"); - } - - // Check size pointer - if (size_off == nullptr || size_val == nullptr) { - throw SubarrayException( - "Cannot get estimated result size; Input sizes cannot be null"); - } - - // Check if name is attribute or dimension - const auto& array_schema = array_->array_schema_latest(); - const bool is_dim = array_schema.is_dim(name); - const bool is_attr = array_schema.is_attr(name); - - // Check if attribute/dimension exists - if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { - throw SubarrayException( - std::string("Cannot get estimated result size; Attribute/Dimension '") + - name + "' does not exist"); - } - - // Check if the attribute/dimension is var-sized - if (!array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must be var-sized"); - } - - // Check if attribute/dimension is nullable - if (array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must not be nullable"); - } - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size_off = - static_cast(std::ceil(est_result_size_[name].size_fixed_)); - *size_val = - static_cast(std::ceil(est_result_size_[name].size_var_)); - - // If the value size is non-zero, ensure both it and the offset size - // are large enough to contain at least one cell. Otherwise, ensure - // the offset size is also zero. - if (*size_val > 0) { - const auto off_cell_size = constants::cell_var_offset_size; - if (*size_off < off_cell_size) - *size_off = off_cell_size; - - const uint64_t val_cell_size = datatype_size(array_schema.type(name)); - if (*size_val < val_cell_size) - *size_val = val_cell_size; - } else { - *size_off = 0; - } -} - -void Subarray::get_est_result_size_nullable( - const char* name, - uint64_t* size, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute name cannot be null"); - - // Check size pointer - if (size == nullptr || size_validity == nullptr) - throw SubarrayException( - "Cannot get estimated result size; Input sizes cannot be null"); - - // Check if name is attribute - const auto& array_schema = array_->array_schema_latest(); - const bool is_attr = array_schema.is_attr(name); - - // Check if attribute exists - if (!is_attr) - throw SubarrayException( - std::string("Cannot get estimated result size; Attribute '") + name + - "' does not exist"); - - // Check if the attribute is fixed-sized - if (array_schema.var_size(name)) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be fixed-sized"); - - // Check if attribute is nullable - if (!array_schema.is_nullable(name)) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be nullable"); + bool is_variable_sized{ + name != constants::coords && array_schema.var_size(name)}; + bool is_nullable{array_schema.is_nullable(name)}; // Compute tile overlap for each fragment compute_est_result_size(config, compute_tp); - *size = static_cast(std::ceil(est_result_size_[name].size_fixed_)); - *size_validity = - static_cast(std::ceil(est_result_size_[name].size_validity_)); - - // If the size is non-zero, ensure it is large enough to - // contain at least one cell. - const auto cell_size = array_schema.cell_size(name); - if (*size > 0 && *size < cell_size) { - *size = cell_size; - *size_validity = 1; - } -} - -void Subarray::get_est_result_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute name cannot be null"); - // Check size pointer - if (size_off == nullptr || size_val == nullptr || size_validity == nullptr) - throw SubarrayException( - "Cannot get estimated result size; Input sizes cannot be null"); - - // Check if name is attribute - const auto& array_schema = array_->array_schema_latest(); - const bool is_attr = array_schema.is_attr(name); - - // Check if attribute exists - if (!is_attr) - throw SubarrayException( - std::string("Cannot get estimated result size; Attribute '") + name + - "' does not exist"); - - // Check if the attribute is var-sized - if (!array_schema.var_size(name)) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be var-sized"); - - // Check if attribute is nullable - if (!array_schema.is_nullable(name)) - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be nullable"); - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size_off = - static_cast(std::ceil(est_result_size_[name].size_fixed_)); - *size_val = - static_cast(std::ceil(est_result_size_[name].size_var_)); - *size_validity = - static_cast(std::ceil(est_result_size_[name].size_validity_)); - - // If the value size is non-zero, ensure both it and the offset and - // validity sizes are large enough to contain at least one cell. Otherwise, - // ensure the offset and validity sizes are also zero. - if (*size_val > 0) { - const uint64_t off_cell_size = constants::cell_var_offset_size; - if (*size_off < off_cell_size) - *size_off = off_cell_size; - - const uint64_t val_cell_size = datatype_size(array_schema.type(name)); - if (*size_val < val_cell_size) - *size_val = val_cell_size; - - const uint64_t validity_cell_size = constants::cell_validity_size; - if (*size_validity < validity_cell_size) - *size_validity = validity_cell_size; + FieldDataSize r{ + static_cast(std::ceil(est_result_size_[name].size_fixed_)), + is_variable_sized ? + static_cast(std::ceil(est_result_size_[name].size_var_)) : + 0, + is_nullable ? static_cast( + std::ceil(est_result_size_[name].size_validity_)) : + 0}; + /* + * Special fix-ups may be necessary if data is empty or very short. + */ + if (is_variable_sized) { + if (r.variable_ == 0) { + // Assert: no variable data for a variable-sized field + // Ensure that there are no offsets. + r.fixed_ = 0; + r.validity_ = 0; + } else { + // Ensure that there's space for at least one offset value. + const auto off_cell_size = constants::cell_var_offset_size; + if (r.fixed_ < off_cell_size) { + r.fixed_ = off_cell_size; + } + // Ensure that there's space for at least one data value. + const auto val_cell_size = datatype_size(array_schema.type(name)); + if (r.variable_ < val_cell_size) { + r.variable_ = val_cell_size; + } + if (is_nullable) { + const auto validity_cell_size = constants::cell_validity_size; + if (r.validity_ < validity_cell_size) { + r.validity_ = validity_cell_size; + } + } + } } else { - *size_off = 0; - *size_validity = 0; - } -} - -void Subarray::get_max_memory_size( - const char* name, - uint64_t* size, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension cannot be null"); - } - - // Check size pointer - if (size == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Input size cannot be null"); - } - - // Check if name is attribute or dimension - const auto& array_schema = array_->array_schema_latest(); - bool is_dim = array_schema.is_dim(name); - bool is_attr = array_schema.is_attr(name); - - // Check if attribute/dimension exists - if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { - throw SubarrayException( - std::string("Cannot get max memory size; Attribute/Dimension '") + - name + "' does not exist"); - } - - // Check if the attribute/dimension is fixed-sized - if (name != constants::coords && array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension must be fixed-sized"); - } - - // Check if attribute/dimension is nullable - if (array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must not be nullable"); + /* + * If the fixed data is not empty, ensure it is large enough to contain at + * least one cell. + */ + const auto cell_size = array_schema.cell_size(name); + if (0 < r.fixed_ && r.fixed_ < cell_size) { + r.fixed_ = cell_size; + if (is_nullable) { + r.validity_ = 1; + } + } } - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size = max_mem_size_[name].size_fixed_; + return r; } -void Subarray::get_max_memory_size( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - const Config* config, - ThreadPool* compute_tp) { +FieldDataSize Subarray::get_max_memory_size( + const char* name, const Config* config, ThreadPool* compute_tp) { // Check attribute/dimension name if (name == nullptr) { throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension cannot be null"); - } - - // Check size pointer - if (size_off == nullptr || size_val == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Input sizes cannot be null"); + "Cannot get max memory size; field name cannot be null"); } // Check if name is attribute or dimension @@ -1423,122 +1197,20 @@ void Subarray::get_max_memory_size( // Check if attribute/dimension exists if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { throw SubarrayException( - std::string("Cannot get max memory size; Attribute/Dimension '") + - name + "' does not exist"); - } - - // Check if the attribute/dimension is var-sized - if (!array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension must be var-sized"); + std::string("Cannot get max memory size; ") + + "there is no field named '" + name + "'"); } - // Check if attribute/dimension is nullable - if (array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute/Dimension must not be nullable"); - } + bool is_variable_sized{ + name != constants::coords && array_schema.var_size(name)}; + bool is_nullable{array_schema.is_nullable(name)}; // Compute tile overlap for each fragment compute_est_result_size(config, compute_tp); - *size_off = max_mem_size_[name].size_fixed_; - *size_val = max_mem_size_[name].size_var_; -} - -void Subarray::get_max_memory_size_nullable( - const char* name, - uint64_t* size, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute name - if (name == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Attribute cannot be null"); - } - // Check size pointer - if (size == nullptr || size_validity == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Input sizes cannot be null"); - } - // Check if name is attribute - const auto& array_schema = array_->array_schema_latest(); - bool is_attr = array_schema.is_attr(name); - - // Check if attribute exists - if (!is_attr) { - throw SubarrayException( - std::string("Cannot get max memory size; Attribute '") + name + - "' does not exist"); - } - - // Check if the attribute is fixed-sized - if (array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get max memory size; Attribute must be fixed-sized"); - } - - // Check if attribute is nullable - if (!array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be nullable"); - } - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size = max_mem_size_[name].size_fixed_; - *size_validity = max_mem_size_[name].size_validity_; -} - -void Subarray::get_max_memory_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension cannot be null"); - } - - // Check size pointer - if (size_off == nullptr || size_val == nullptr || size_validity == nullptr) { - throw SubarrayException( - "Cannot get max memory size; Input sizes cannot be null"); - } - // Check if name is attribute or dimension - const auto& array_schema = array_->array_schema_latest(); - bool is_attr = array_schema.is_attr(name); - - // Check if attribute exists - if (!is_attr) { - throw SubarrayException( - std::string("Cannot get max memory size; Attribute '") + name + - "' does not exist"); - } - - // Check if the attribute is var-sized - if (!array_schema.var_size(name)) { - throw SubarrayException( - "Cannot get max memory size; Attribute/Dimension must be var-sized"); - } - - // Check if attribute is nullable - if (!array_schema.is_nullable(name)) { - throw SubarrayException( - "Cannot get estimated result size; " - "Attribute must be nullable"); - } - - // Compute tile overlap for each fragment - compute_est_result_size(config, compute_tp); - *size_off = max_mem_size_[name].size_fixed_; - *size_val = max_mem_size_[name].size_var_; - *size_validity = max_mem_size_[name].size_validity_; + return { + max_mem_size_[name].size_fixed_, + is_variable_sized ? max_mem_size_[name].size_var_ : 0, + is_nullable ? max_mem_size_[name].size_validity_ : 0}; } std::vector Subarray::get_range_coords(uint64_t range_idx) const { @@ -1575,36 +1247,6 @@ std::vector Subarray::get_range_coords(uint64_t range_idx) const { return ret; } -void Subarray::get_next_range_coords( - std::vector* range_coords) const { - auto dim_num = array_->array_schema_latest().dim_num(); - auto layout = - (layout_ == Layout::UNORDERED) ? - ((cell_order_ == Layout::HILBERT) ? Layout::ROW_MAJOR : cell_order_) : - layout_; - - if (layout == Layout::ROW_MAJOR) { - auto d = dim_num - 1; - ++(*range_coords)[d]; - while ((*range_coords)[d] >= range_subset_[d].num_ranges() && d != 0) { - (*range_coords)[d] = 0; - --d; - ++(*range_coords)[d]; - } - } else if (layout == Layout::COL_MAJOR) { - auto d = (unsigned)0; - ++(*range_coords)[d]; - while ((*range_coords)[d] >= range_subset_[d].num_ranges() && - d != dim_num - 1) { - (*range_coords)[d] = 0; - ++d; - ++(*range_coords)[d]; - } - } else { - // Global order - noop - } -} - uint64_t Subarray::range_idx(const std::vector& range_coords) const { uint64_t ret = 0; auto dim_num = this->dim_num(); diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index c07854bfef25..bc11c1f50041 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -65,6 +65,23 @@ class Range; namespace tiledb::sm { +struct FieldDataSize { + /** + * The size of fixed-length field data in bytes. + */ + size_t fixed_; + + /** + * The size of variable-length field data in bytes. + */ + size_t variable_; + + /** + * The size of validity data in bytes. + */ + size_t validity_; +}; + class Array; class ArraySchema; class OpenedArray; @@ -918,96 +935,28 @@ class Subarray { */ bool is_unary() const; - /** - * Gets the estimated result size (in bytes) for the input fixed-sized - * attribute/dimension. - */ - void get_est_result_size_internal( - const char* name, - uint64_t* size, - const Config* config, - ThreadPool* compute_tp); - - /** - * Gets the estimated result size (in bytes) for the input var-sized - * attribute/dimension. - */ - void get_est_result_size( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - const Config* config, - ThreadPool* compute_tp); - - /** - * Gets the estimated result size (in bytes) for the input fixed-sized, - * nullable attribute. - */ - void get_est_result_size_nullable( - const char* name, - uint64_t* size, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp); - - /** - * Gets the estimated result size (in bytes) for the input var-sized, - * nullable attribute. - */ - void get_est_result_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp); - /** Returns whether the estimated result size has been computed or not. */ bool est_result_size_computed(); - /* - * Gets the maximum memory required to produce the result (in bytes) - * for the input fixed-sized attribute/dimension. - */ - void get_max_memory_size( - const char* name, - uint64_t* size, - const Config* config, - ThreadPool* compute_tp); - /** - * Gets the maximum memory required to produce the result (in bytes) - * for the input var-sized attribute/dimension. - */ - void get_max_memory_size( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - const Config* config, - ThreadPool* compute_tp); - - /* - * Gets the maximum memory required to produce the result (in bytes) - * for the input fixed-sized, nullable attribute. + * The estimated result size in bytes for a field. + * + * This function handles all fields. The field may be a dimension or + * attribute, fixed-length or variable, nullable or not. If a particular + * size is not relevant to the field type, then it's returned as zero. */ - void get_max_memory_size_nullable( - const char* name, - uint64_t* size, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp); + FieldDataSize get_est_result_size( + const char* name, const Config* config, ThreadPool* compute_tp); /** - * Gets the maximum memory required to produce the result (in bytes) - * for the input var-sized, nullable attribute. + * The maximum memory in bytes required for a field. + * + * This function handles all fields. The field may be a dimension or + * attribute, fixed-length or variable, nullable or not. If a particular + * size is not relevant to the field type, then it's returned as zero. */ - void get_max_memory_size_nullable( - const char* name, - uint64_t* size_off, - uint64_t* size_val, - uint64_t* size_validity, - const Config* config, - ThreadPool* compute_tp); + FieldDataSize get_max_memory_size( + const char* name, const Config* config, ThreadPool* compute_tp); /** * Returns the range coordinates (for all dimensions) given a flattened @@ -1017,12 +966,6 @@ class Subarray { */ std::vector get_range_coords(uint64_t range_idx) const; - /** - * Advances the input range coords to the next coords along the - * subarray range layout. - */ - void get_next_range_coords(std::vector* range_coords) const; - /** * Returns a subarray consisting of the dimension ranges specified by * the input. diff --git a/tiledb/sm/subarray/subarray_partitioner.cc b/tiledb/sm/subarray/subarray_partitioner.cc index e44e9b541346..50ba727681c2 100644 --- a/tiledb/sm/subarray/subarray_partitioner.cc +++ b/tiledb/sm/subarray/subarray_partitioner.cc @@ -1135,99 +1135,29 @@ Status SubarrayPartitioner::compute_splitting_value_multi_range( } bool SubarrayPartitioner::must_split(Subarray* partition) { - const auto& array_schema = subarray_.array()->array_schema_latest(); - bool must_split = false; - - uint64_t size_fixed; - uint64_t size_var; - uint64_t size_validity; - uint64_t mem_size_fixed; - uint64_t mem_size_var; - uint64_t mem_size_validity; for (const auto& b : budget_) { - // Compute max sizes + /* + * Compute max memory size and, if needed, estimated result size + */ auto name = b.first; - auto var_size = array_schema.var_size(name); - auto nullable = array_schema.is_nullable(name); - - // Compute est sizes - size_fixed = 0; - size_var = 0; - size_validity = 0; - mem_size_fixed = 0; - mem_size_var = 0; - mem_size_validity = 0; - // Compute max memory sizes - if (var_size) { - if (!nullable) { - partition->get_max_memory_size( - b.first.c_str(), - &mem_size_fixed, - &mem_size_var, - config_, - compute_tp_); - } else { - partition->get_max_memory_size_nullable( - b.first.c_str(), - &mem_size_fixed, - &mem_size_var, - &mem_size_validity, - config_, - compute_tp_); - } - } else { - if (!nullable) { - partition->get_max_memory_size( - b.first.c_str(), &mem_size_fixed, config_, compute_tp_); - } else { - partition->get_est_result_size_nullable( - b.first.c_str(), &size_fixed, &size_validity, config_, compute_tp_); - partition->get_max_memory_size_nullable( - b.first.c_str(), - &mem_size_fixed, - &mem_size_validity, - config_, - compute_tp_); - } - } - - // Compute estimated result sizes - if (!skip_split_on_est_size_) { - if (var_size) { - if (!nullable) { - partition->get_est_result_size( - b.first.c_str(), &size_fixed, &size_var, config_, compute_tp_); - } else { - partition->get_est_result_size_nullable( - b.first.c_str(), - &size_fixed, - &size_var, - &size_validity, - config_, - compute_tp_); - } - } else { - if (!nullable) { - partition->get_est_result_size_internal( - b.first.c_str(), &size_fixed, config_, compute_tp_); - } else { - partition->get_est_result_size_nullable( - b.first.c_str(), - &size_fixed, - &size_validity, - config_, - compute_tp_); - } - } - } + auto mem_size{ + partition->get_max_memory_size(b.first.c_str(), config_, compute_tp_)}; + auto est_size{ + skip_split_on_est_size_ ? + // Skip the estimate and use a default object that's all zeros. + FieldDataSize{} : + // Perform the estimate + partition->get_est_result_size( + b.first.c_str(), config_, compute_tp_)}; // If we try to split a unary range because of memory budgets, throw an // error. This can happen when the memory budget cannot fit even one tile. // It will cause the reader to process the query cell by cell, which will // make it very slow. if (!skip_unary_partitioning_budget_check_ && - (mem_size_fixed > memory_budget_ || mem_size_var > memory_budget_var_ || - mem_size_validity > memory_budget_validity_)) { + (mem_size.fixed_ > memory_budget_ || + mem_size.variable_ > memory_budget_var_ || + mem_size.validity_ > memory_budget_validity_)) { if (partition->is_unary()) { throw SubarrayPartitionerStatusException( "Trying to partition a unary range because of memory budget, this " @@ -1241,16 +1171,16 @@ bool SubarrayPartitioner::must_split(Subarray* partition) { // Check for budget overflow if ((!skip_split_on_est_size_ && - (size_fixed > b.second.size_fixed_ || size_var > b.second.size_var_ || - size_validity > b.second.size_validity_)) || - mem_size_fixed > memory_budget_ || mem_size_var > memory_budget_var_ || - mem_size_validity > memory_budget_validity_) { - must_split = true; - break; + (est_size.fixed_ > b.second.size_fixed_ || + est_size.variable_ > b.second.size_var_ || + est_size.validity_ > b.second.size_validity_)) || + mem_size.fixed_ > memory_budget_ || + mem_size.variable_ > memory_budget_var_ || + mem_size.validity_ > memory_budget_validity_) { + return true; } } - - return must_split; + return false; } Status SubarrayPartitioner::next_from_multi_range(bool* unsplittable) { From 6302ab02e806459c1966c011e17037caadb3edc1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 26 Apr 2024 15:16:38 +0300 Subject: [PATCH 297/456] Update nightly workflow to account for `macos-latest` meaning macOS in Apple Silicon. (#4913) [SC-46173](https://app.shortcut.com/tiledb-inc/story/46173) Since yesterday, the `macos-latest` images mean macOS in Apple SIlicon. This PR updates the nightly workflow to account for that. Should fix macOS failures with ASAN. --- TYPE: NO_HISTORY --- .github/workflows/nightly-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 169963ad3bcc..82694da57c66 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -16,7 +16,7 @@ jobs: - os: macos-latest - os: macos-latest # ASAN build sanitizer: "address" - base_triplet: "x64-osx" + base_triplet: "arm64-osx" - os: macos-latest experimental: ON - os: windows-latest From c46af50b941baf393e6bc9e63c7cd01cfac22ecb Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Fri, 26 Apr 2024 11:24:43 -0700 Subject: [PATCH 298/456] Implement iterator facade for external sort. (#4914) The `iterator_facade` class (in tiledb/common/iterator_facade.h) implements, well, an iterator facade. An iterator must provide a fairly rich set of functions to be fully conformant to the C++20 ranges iterator concepts (and to conform to the `LegacyIterator` named requirements). Providing that functionality involves an enormous amount of boilerplate and can have subtle issues (e.g., with `const`, with the required type aliases, etc). The `iterator_facade` class generates all of the boilerplate, based on just a handful of functions that the user has to provide. Use of the `iterator_facade` class is based on CRTP. A simple iterator only needs to provide three functions in the simplest case to be able to meet all of the requirements of an iterator. For example, we can define an iterator that wraps up a raw pointer and strides through its data by 10 elements at a time. ``` template a class pointer_wrapper : public iterator_facade> { Value *current_ = nullptr; public: pointer_wrapper() = default; explicit pointer_wrapper(Value* i) : current_(i) {} Value &dereference() const noexcept { return *current_; } void advance(int off) noexcept { current_ += 10 * off; } auto distance_to(pointer_wrapper o) const noexcept { return o.current_ - current_; } }; ``` To do the port, I removed all dependencies on other parts of the `neo-fun` library, primarily by substituting standard library functions, type traits, and concepts for those in the `neo-fun` library. I also added some functionality to support assignment to an iterator. I also ported the unit testing from the `neo-fun` repository. I added a significant number of new tests, primarily to verify that an iterator based on `iterator_facade` would conform to the expected range and iterator concepts. Note that there are a few tests that return slightly unexpected results (in that they differ from , e.g., what should be an equivalent iterator from `std::vector`). Those should have no impact on our immediate intended usage of `iterator_facade`, but at some point, we should address those issues * The `iterator_category` of some const variants does not appear to be correct * `operator[]` seems to return nonsense in one usage mode in unit_iterator_facade.cc The implementation of `iterator_facade` is ported from the implementation at https://github.com/vector-of-bool/neo-fun/commit/b6c38c8, which I cloned on Mar 21, 2024 to use as the beginning of the port. The downloaded code is licensed under the Boost Software License 1.0 (which essentially the same as the MIT license). This `iterator_facade` is a modernized approach to Boost.Iterator and is greatly simplified via the use of concepts and other modern C++ features (Boost.Iterator dates from the early 2000s). (One of the authors of Boost.Graph is Jeremy Siek, a former PhD student in my group at Indiana University.) --- TYPE: IMPROVEMENT DESC: Implement iterator facade for external sort. --- .../tiledb/common/dag/utility/range_join.h | 2 +- .../dag/utility/test/compile_utils_main.cc | 2 +- .../utility => tiledb/common}/arrow_proxy.hpp | 5 +- tiledb/common/iterator_facade.h | 495 ++++++++ tiledb/common/test/CMakeLists.txt | 5 + tiledb/common/test/unit_iterator_facade.cc | 1026 +++++++++++++++++ 6 files changed, 1532 insertions(+), 3 deletions(-) rename {experimental/tiledb/common/dag/utility => tiledb/common}/arrow_proxy.hpp (94%) create mode 100644 tiledb/common/iterator_facade.h create mode 100644 tiledb/common/test/unit_iterator_facade.cc diff --git a/experimental/tiledb/common/dag/utility/range_join.h b/experimental/tiledb/common/dag/utility/range_join.h index 89f39f36ec54..db9a660f2e83 100644 --- a/experimental/tiledb/common/dag/utility/range_join.h +++ b/experimental/tiledb/common/dag/utility/range_join.h @@ -73,7 +73,7 @@ #include #include #include -#include "arrow_proxy.hpp" +#include "tiledb/common/arrow_proxy.hpp" #include "external/include/span/span.hpp" diff --git a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc index c9f29280f626..b158e690c650 100644 --- a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc +++ b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc @@ -26,12 +26,12 @@ * THE SOFTWARE. */ -#include "../arrow_proxy.hpp" #include "../bounded_buffer.h" #include "../print_types.h" #include "../range_join.h" #include "../spinlock.h" #include "../traits.h" +#include "tiledb/common/arrow_proxy.hpp" int main() { } diff --git a/experimental/tiledb/common/dag/utility/arrow_proxy.hpp b/tiledb/common/arrow_proxy.hpp similarity index 94% rename from experimental/tiledb/common/dag/utility/arrow_proxy.hpp rename to tiledb/common/arrow_proxy.hpp index 92203459ca1b..5a8cc0c23e19 100644 --- a/experimental/tiledb/common/dag/utility/arrow_proxy.hpp +++ b/tiledb/common/arrow_proxy.hpp @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -52,4 +52,7 @@ struct arrow_proxy { } }; +template +arrow_proxy(T&&) -> arrow_proxy; + #endif // TILEDB_ARROW_PROXY_HPP diff --git a/tiledb/common/iterator_facade.h b/tiledb/common/iterator_facade.h new file mode 100644 index 000000000000..be97d52e4417 --- /dev/null +++ b/tiledb/common/iterator_facade.h @@ -0,0 +1,495 @@ +/** + * @file iterator_facade.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements an `iterator_facade` that will be used as part of TileDB + * external sort. The implementation is a modified version of + * `iterator_facade` from the `vectorofbool/neo-fun` github repository, + * (https://github.com/vector-of-bool/neo-fun/commit/b6c38c8 Mar 21, 2024). It + * is used here under the terms of the Boost Software License 1.0 and is + * Copyright (c) the author(s). + */ + +#ifndef TILEDB_ITERATOR_FACADE_H +#define TILEDB_ITERATOR_FACADE_H + +#include +#include +#include + +#include "arrow_proxy.hpp" + +namespace detail { + +// clang-format off +template +concept sized_sentinel_of = + requires(const Iter& it, const Sentinel& sentinel) { + it.distance_to(sentinel); + }; + +template +struct infer_difference_type { + using type = std::ptrdiff_t; +}; + +template + requires sized_sentinel_of +struct infer_difference_type { + static const T& _it; + using type = decltype(_it.distance_to(_it)); +}; + +template +using infer_difference_type_t = typename infer_difference_type::type; + +template +struct infer_value_type { + static const T& _it; + using type = std::remove_cvref_t; +}; + +template + requires requires { typename T::value_type; } +struct infer_value_type { + using type = typename T::value_type; +}; + +template +using infer_value_type_t = typename infer_value_type::type; + +template +concept can_increment = + requires(T& t) { + t.increment(); + }; + +template +concept can_decrement = + requires(T& t) { + t.decrement(); + }; + +template +concept can_advance = + requires(T& t, const infer_difference_type_t d) { + t.advance(d); + }; + +template +concept can_to_address = + requires (const T& t) { + { t.to_address() } -> std::contiguous_iterator; + }; + +template +concept iter_is_random_access = + sized_sentinel_of && + can_advance; + +template +concept iter_is_contiguous = iter_is_random_access and can_to_address; + +template +concept iter_is_bidirectional = + iter_is_random_access || + can_decrement; + +template +concept iter_is_single_pass = requires { + requires bool(T::single_pass_iterator); +}; + +template +concept iter_is_forward = !iter_is_single_pass && requires(const T& item) { + { item == item }; +}; + +// clang-format on + +// This causes internalASTError in cppcheck 2.13.0_1 (Homebrew) on macOS 14.3.1 +template +concept noexcept_incrementable = requires(T& iter) { + { iter.increment() } noexcept; +} || requires(T& item) { + { item += 1 } noexcept; +}; + +template +concept iter_diff = std::convertible_to>; + +struct iterator_facade_base; + +template +concept iter_facade_type = + std::is_base_of_v>; + +template +concept iter_self = iter_facade_type; + +template +concept random_access_iter_self = iter_self && iter_is_random_access; + +template +concept bidirectional_iter_self = iter_self && iter_is_bidirectional; + +struct iterator_facade_base { + /** + * If this is a random_access_iterator, returns the distance from the right + * to the left, i.e. how many times to apply ++right to reach `left`. + */ + template Sent> + [[nodiscard]] constexpr friend auto operator-( + const Sent& sent, const Self& self) noexcept { + return self.distance_to(sent); + } + + template Diff> + [[nodiscard]] constexpr friend Self operator-( + const Self& self, Diff off) noexcept { + using diff_type = infer_difference_type_t; + using signed_diff_type = std::make_signed_t; + return self + -static_cast(off); + } + + template Diff> + [[nodiscard]] constexpr friend Self operator+( + const Self& self, Diff off) noexcept { + auto cp = self; + return cp += off; + } + + template Diff> + [[nodiscard]] constexpr friend Self operator+( + Diff off, const Self& self) noexcept { + return self + off; + } + + template Diff> + constexpr friend Self& operator+=(Self& self, Diff off) noexcept { + self.advance(static_cast>(off)); + return self; + } + + template Diff> + constexpr friend Self& operator-=(Self& self, Diff off) noexcept { + return self = self - off; + } + + /** + * Advance the iterator one position forward. Implemented as a call to + * `.increment()`, if present, otherwise `*this += 1` + * + * @todo If `increment()` is not present, but advance() is, we should use + * advance(1) + */ + template + constexpr friend Self& operator++(Self& s) noexcept( + noexcept_incrementable) { + if constexpr (can_increment) { + // If there is an increment(), assume it is the most efficient way to + // advance, even if we have an advance() + s.increment(); + } else if constexpr (iter_is_random_access) { + // Just offset by one + s += 1; + } else { + static_assert( + iter_is_random_access || can_increment, + "Iterator subclass must provide an `increment` or `advance(n)` " + "method"); + } + return s; + } + + // clang-format off + template + constexpr friend + std::conditional_t< + detail::iter_is_single_pass, + void, + std::remove_cvref_t> + operator++(Self& self, int) noexcept(noexcept_incrementable) { + // clang-format on + if constexpr (detail::iter_is_single_pass) { + // The iterator is a single-pass iterator. It isn't safe to make and + // return an old copy. + ++self; + } else { + auto cp = self; + ++self; + return cp; + } + } + + template + constexpr friend Self& operator--(Self& self) noexcept { + if constexpr (can_decrement) { + self.decrement(); + } else { + self -= 1; + } + return self; + } + + template + constexpr friend std::remove_cvref_t operator--( + Self& self, int) noexcept { + auto cp = self; + --self; + return cp; + } + + /** + * With three-way comparison, we can get away with much simpler + * comparison/equality operators, since we can also rely on synthesized + * rewrites + */ + template S> + [[nodiscard]] constexpr friend std::strong_ordering operator<=>( + const Self& self, const S& right) noexcept { + auto dist = self - right; + auto rel = dist <=> 0; + return rel; + } + + /** + * With three-way comparison, we can get away with much simpler + * comparison/equality operators, since we can also rely on synthesized + * rewrites + */ + template S> + [[nodiscard]] constexpr friend bool operator<( + const Self& self, const S& right) noexcept { + auto dist = self - right; + auto rel = dist < 0; + return rel; + } +}; + +} // namespace detail + +/** + * An iterator_facade fills-out the interface of an iterator based on just a few + * methods be present on the derived class (provided as the CRTP parameter). + * + * The following methods MUST be provided: + * + * - Derived::dereference() - Return value of the operator*(). Need not be an + * actual reference. + * - Derived::increment() OR Derived::advance(ptrdiff_t) - (Or both) Used to + * implement operator++() and operator++(int). If both methods are provided, + * `increment()` will be prefered for single increment operations. + * + * With these two methods defined, the iterator is a forward_iterator. If + * `advance(ptrdiff_t)` is provided, the iterator is also bidirectional. + * + * + * ====== Iterator Equality + * + * The generated iterator type is equality-comparible with any object of type + * S if the derived class implements `distance_to(S)`. This includes sentinel + * types and other instances of the iterator. + * + * + * ====== Single-pass Input + * + * If the class provides a static member `single_pass_iterator` that is `true`, + * then the iterator will be an input_iterator, and the operator++(int) + * (postfix decrement) will return `void`. + * + * + * ====== Bidirectional + * + * If the following are provided, the iterator is bidirectional: + * + * - Derived::decrement() OR Derived::advance(ptrdiff_t) - Used to implement + * operator--() and operator--(int). + * + * Note that unless the requirements for Random Access are met, `advance()` + * will only be called with `1` or `-1`. + * + * + * ====== Random Access + * + * If the following are provided, the iterator is random_access_iterator: + * + * - Derived::advance(ptrdiff_t p) - Move the iterator by `p` (may be negative!) + * - Derived::distance_to(Derived other) - Return the "distance" to `other`, + * that is: The number of types *this must be incremented to be equal to + * `other`, or the numner of times *this must be decremented to reach `other` + * (which should yield a negative number). + * + * These two methods are used to implement the remainder of the iterator + * functionality. + * + * + * NOTE: A specialization of std::iterator_traits is provided that + * provides the required iterator typedefs. You cannot access e.g. + * `iterator_facade<...>::value_type` directly, and must instead go through + * `iterator_traits::value_type`. + */ +template +class iterator_facade : public detail::iterator_facade_base { + public: + using self_type = Derived; + + private: + constexpr self_type& _self() noexcept { + return static_cast(*this); + } + constexpr const self_type& _self() const noexcept { + return static_cast(*this); + } + + public: + /** + * Implement operator* in terms of `.dereference()` + */ + [[nodiscard]] constexpr decltype(auto) operator*() const + noexcept(noexcept(_self().dereference())) { + return _self().dereference(); + } + + /** + * Implement arrow in terms of `operator*`, but: + * + * If the return type of operator* is a non-reference type, returns an + * arrow_proxy that wraps the returned value. + * + * If the return type is a reference type, returns a pointer to the returned + * object. + */ + constexpr decltype(auto) operator->() const + noexcept(noexcept(_self().dereference())) { + if constexpr (detail::can_to_address) { + return _self().to_address(); + } else if constexpr (std::is_reference_v< + std::iter_reference_t>) { + // If operator*() returns a reference, just return that address + return std::addressof(**this); + } else { + // It returned a value, so we need to wrap it in an arrow_proxy for the + // caller + return arrow_proxy{**this}; + } + } + + template D> + [[nodiscard]] constexpr decltype(auto) operator[](D pos) const noexcept { + return *(_self() + pos); + } +}; + +template +class iterator_wrapper_facade : public iterator_facade { + using base = typename iterator_wrapper_facade::iterator_facade; + using self_type = Derived; + + protected: + InnerIterator wrapped_iterator; + + public: + iterator_wrapper_facade() = default; + explicit iterator_wrapper_facade(InnerIterator it) + : wrapped_iterator(it) { + } + + template D> + constexpr void advance(D off) noexcept { + std::advance(wrapped_iterator, off); + } + + // clang-format off + constexpr auto distance_to(const self_type& other) const noexcept + requires std::derived_from< + std::random_access_iterator_tag, + typename std::iterator_traits::iterator_category + > { + return other.wrapped_iterator - wrapped_iterator; + } + // clang-format on + + constexpr friend bool operator==( + const self_type& left, const self_type& other) noexcept { + return left.wrapped_iterator == other.wrapped_iterator; + } +}; + +// Putting things in std:: namespace is a bad idea, but it is necessary for +// `iterator_traits` to work correctly with `iterator_facade`. +namespace std { + +template + requires std::is_base_of_v +struct iterator_traits { + static const Derived& _const_it; + using value_type = detail::infer_value_type_t; + using reference = decltype(*_const_it); + using pointer = decltype(_const_it.operator->()); + using difference_type = detail::infer_difference_type_t; + + // Pick the iterator category based on the interfaces that it provides + // @todo Add support for output iterators and for contiguous iterators + using iterator_category = std::conditional_t< + // Contiguous? + detail::iter_is_contiguous, + std::contiguous_iterator_tag, + // Not + std::conditional_t< + // Random access? + detail::iter_is_random_access, + std::random_access_iterator_tag, + // Nope + std::conditional_t< + // Bidirectional? + detail::iter_is_bidirectional, + std::bidirectional_iterator_tag, + // Noh + std::conditional_t< + // Is it single-pass? + detail::iter_is_forward, + // Otherwise it is a forward iterator + std::forward_iterator_tag, + // Than means it is an input iterator + std::input_iterator_tag>>>>; + + using iterator_concept = iterator_category; +}; + +template + requires detail::can_to_address and + std::derived_from> +struct pointer_traits { + using pointer = decltype(std::addressof(*std::declval())); + using element_type = std::remove_pointer_t; + using difference_type = std::ptrdiff_t; +}; + +} // namespace std +#endif diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 552212e48a3d..24935f815c56 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -41,3 +41,8 @@ commence(unit_test memory_tracker_types) this_target_object_libraries(baseline) conclude(unit_test) +commence(unit_test iterator_facade) + this_target_sources(main.cc unit_iterator_facade.cc) + this_target_object_libraries(baseline) +conclude(unit_test) + diff --git a/tiledb/common/test/unit_iterator_facade.cc b/tiledb/common/test/unit_iterator_facade.cc new file mode 100644 index 000000000000..8eaaf72ca3f0 --- /dev/null +++ b/tiledb/common/test/unit_iterator_facade.cc @@ -0,0 +1,1026 @@ + +/** + * @file unit_iterator_facade.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for `iterator_facade` that will be used as + * part of TileDB external sort. The tests include those from + * `iterator_facade.test.cpp` from the `vectorofbool/neo-fun` github repository, + * (https://github.com/vector-of-bool/neo-fun/commit/b6c38c8 Mar 21, 2024). It + * is used here under the terms of the Boost Software License 1.0 and is + * Copyright (c) the author(s). + * + */ + +#include +#include +#include +#include "../iterator_facade.h" + +TEST_CASE("iterator_facade: Null test", "[iterator_facade][null_test]") { + REQUIRE(true); +} + +struct null_iterator : public iterator_facade {}; + +TEST_CASE("iterator_facade: null_iterator", "[iterator_facade]") { + null_iterator it; + (void)it; +} + +struct minimal_iterator : public iterator_facade { + int value = 0; + + int dereference() const { + return value; + } + int distance_to(minimal_iterator o) const { + return *o - value; + } + void advance(int off) { + value += off; + } + + // This should be inferred, but it seems to be necessary to specify it here. + // @todo Implement inference for this in iterator_facade. + bool operator==(minimal_iterator o) const noexcept { + return *o == **this; + } +}; + +TEST_CASE("iterator_facade: minimal_iterator", "[iterator_facade]") { + CHECK(std::input_iterator); + CHECK(!std::output_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); +} + +/* + * The following are from vector-of-bool neo-fun/iterator_facade.test.cc + */ +namespace { + +template +class as_string_iterator + : public iterator_wrapper_facade, Iter> { + using as_string_iterator::iterator_wrapper_facade::iterator_wrapper_facade; + + public: + std::string dereference() const noexcept { + return std::to_string(*this->wrapped_iterator); + } +}; + +template +as_string_iterator(It) -> as_string_iterator; + +class iota_iterator : public iterator_facade { + int _value = 0; + + public: + iota_iterator() = default; + explicit iota_iterator(int i) + : _value(i) { + } + + // Since this is a fake iterator, and is maintaining its "value" itself, + // we can't return a reference to the value. + int dereference() const noexcept { + return _value; + } + + // Trying to do this breaks iterator_traits for iterator_facade. + // Since this is just a contrived situation, it's not worth trying to fix. + // Note that it will work -- we can set values, but the iterator_traits + // will not be correct -- and wil cause compiler error if we try to use. + // int& dereference() noexcept { return _value; } + + void advance(int off) noexcept { + _value += off; + } + int distance_to(iota_iterator o) const noexcept { + return *o - **this; + } + bool operator==(iota_iterator o) const noexcept { + return *o == **this; + } +}; + +} // namespace + +TEST_CASE("Create an iota_iterator") { + iota_iterator it; + iota_iterator stop{44}; + + CHECK(std::input_iterator); + CHECK(!std::output_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + + CHECK((stop - it) == 44); + CHECK(*(stop - 4) == 40); + CHECK(it < stop); + CHECK(it <= stop); + CHECK_FALSE(it > stop); + CHECK_FALSE(it >= stop); + CHECK(std::distance(it, stop) == 44); + + CHECK(it[33] == 33); + CHECK(it[-9] == -9); + CHECK(stop[2] == 46); + CHECK(stop[-44] == 0); + + CHECK((stop - it) == 44); + CHECK((it - stop) == -44); + + CHECK(it != stop); + CHECK((it + 44) == stop); + CHECK(it == (stop - 44)); +} + +namespace { + +class mutable_iota_iterator : public iterator_facade { + mutable int _value{0}; + + public: + mutable_iota_iterator() = default; + explicit mutable_iota_iterator(int i) + : _value(i) { + } + + // This is the same kind of contrived iterator as before, but here the + // iterator is referring to an external value, which it can return by + // reference, even if the iterator is const. + int& dereference() const noexcept { + return _value; + } + + void advance(int off) noexcept { + _value += off; + } + int distance_to(mutable_iota_iterator o) const noexcept { + return *o - **this; + } + bool operator==(mutable_iota_iterator o) const noexcept { + return *o == **this; + } +}; +} // namespace + +TEST_CASE("mutable_iota_iterator") { + CHECK(std::input_iterator); + CHECK(std::output_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + + mutable_iota_iterator it; + CHECK(*it == 0); + *it = 42; + CHECK(*it == 42); +} + +TEST_CASE("arrow_proxy") { + arrow_proxy s{""}; + s->append("Hello, "); + s->append("world!"); + CHECK(*s.operator->() == "Hello, world!"); +} + +TEST_CASE("Trivial iterator") { + struct deref_iter : iterator_facade { + int* value = nullptr; + auto& dereference() /*const*/ noexcept { + // cppcheck-suppress nullPointer + return *value; + } + auto dereference() const noexcept { + return *value; + } + + deref_iter(int& i) + : value(&i) { + } + }; + + // Just some very simple tests. The trivial iterator does not even have + // increment or decrement. + + int i = 12; + deref_iter it{i}; + + CHECK(*it == 12); + i = 7; + CHECK(*it == 7); +} + +TEST_CASE("Single-pass iterator") { + struct in_iter : iterator_facade { + int value = 0; + enum { single_pass_iterator = true }; + + const int& dereference() const noexcept { + return value; + } + void increment() noexcept { + ++value; + } + }; + + in_iter it; + CHECK(*it == 0); + static_assert(std::is_same_v); + ++it; + CHECK(*it == 1); + static_assert(std::is_void_v); +} + +TEST_CASE("Transforming iterator") { + std::vector values = {1, 2, 3, 4}; + as_string_iterator it{values.begin()}; + + CHECK(*it == "1"); + ++it; + CHECK(*it == "2"); + CHECK_FALSE(it == as_string_iterator(values.begin())); + // Post-increment returns a copy of the iterator + auto copy = it++; + CHECK(*copy == "2"); + + static_assert( + std::is_same_v()), arrow_proxy>); + // Even though we are acting on a temporary, the append() will return a new + // string + auto thirty_four = it->append("4"); + CHECK(thirty_four == "34"); + + copy = copy - 1; + CHECK(*copy == "1"); + CHECK(*(copy + 3) == "4"); + CHECK(*(3 + copy) == "4"); + + ++copy; + auto copy2 = copy--; + CHECK(*copy == "1"); + CHECK(*copy2 == "2"); + + // Advance by a negative number created from an unsigned + CHECK(*copy == "1"); + ++copy; + copy -= 1u; + CHECK(*copy == "1"); +} + +TEST_CASE("Sentinel support") { + struct until_7_iter : iterator_facade { + int value = 0; + struct sentinel_type {}; + + auto dereference() const noexcept { + return value; + } + auto increment() noexcept { + ++value; + } + + auto distance_to(sentinel_type) const noexcept { + return 7 - value; + } + bool operator==(sentinel_type s) const noexcept { + return distance_to(s) == 0; + } + }; + + struct seven_range { + auto begin() { + return until_7_iter(); + } + auto end() { + return until_7_iter::sentinel_type(); + } + }; + + int sum = 0; + for (auto i : seven_range()) { + sum += i; + CHECK(i < 7); + } + CHECK(sum == (1 + 2 + 3 + 4 + 5 + 6)); + + auto it = seven_range().begin(); + auto stop = seven_range().end(); + CHECK(it != stop); + CHECK(stop != it); + CHECK_FALSE(it == stop); + CHECK_FALSE(stop == it); + +#if !_MSC_VER + /// XXX: Last checked, MSVC has an issue finding the correct operator-() via + /// ADL. If you're using MSVC and reading this comment in the future, revive + /// this snippet and try again. + CHECK((until_7_iter::sentinel_type() - it) == 7); +#endif +} + +namespace { +enum class month : int { + january, + february, + march, + april, + may, + june, + july, + august, + september, + october, + november, + december, +}; + +/* + * Example of a simple iterator that iterates over the months of the year, + * copied from + * `https://vector-of-bool.github.io/2020/06/13/cpp20-iter-facade.html` + */ +class month_iterator : public iterator_facade { + month _cur = month::january; + + public: + month_iterator() = default; + explicit month_iterator(month m) + : _cur(m) { + } + + auto begin() const { + return *this; + } + auto end() const { + return month_iterator(month(int(month::december) + 1)); + } + + void increment() { + _cur = month(int(_cur) + 1); + } + void decrement() { + _cur = month(int(_cur) - 1); + } + void advance(int off) { + _cur = month(int(_cur) + off); + } + + const month& dereference() const { + return _cur; + } + + int distance_to(month_iterator other) const { + return int(other._cur) - int(_cur); + } + // This shouldn't be necessary, but seems to be required to determine + // the iterator category. + bool operator==(month_iterator other) const { + return _cur == other._cur; + } +}; + +/* + * A second month_iterator to test writing to the iterator. + */ + +class month_iterator_2 : public iterator_facade { + // Since we are faking an iterator over data, we can make the value mutable. + mutable month _cur = month::january; + + public: + month_iterator_2() = default; + explicit month_iterator_2(month m) + : _cur(m) { + } + + month& dereference() const { + return _cur; + } + + void advance(int n) { + _cur = month(int(_cur) + n); + } + int distance_to(month_iterator_2 o) const { + return int(o._cur) - int(_cur); + } + + bool operator==(month_iterator_2 o) const { + return _cur == o._cur; + } +}; + +} // namespace + +TEMPLATE_TEST_CASE( + "iterator_facade: month_iterator", + "[iterator_facade]", + month_iterator, + month_iterator_2) { + CHECK(std::input_iterator); + if (std::is_same_v) { + CHECK(std::output_iterator); + } else { + CHECK(!std::output_iterator); + } + + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + + TestType it; + CHECK(*it == month::january); + ++it; + CHECK(*it == month::february); + ++it; + CHECK(*it == month::march); + ++it; + CHECK(*it == month::april); + ++it; + CHECK(*it == month::may); + ++it; + CHECK(*it == month::june); + ++it; + CHECK(*it == month::july); + ++it; + CHECK(*it == month::august); + ++it; + CHECK(*it == month::september); + ++it; + CHECK(*it == month::october); + ++it; + CHECK(*it == month::november); + ++it; + CHECK(*it == month::december); + // We should be able to increment once more and get the sentinel value, + // but this is not working for some reason. + // @todo Fix this. + // ++it; + // CHECK(it == month_iterator::sentinel_type()); +} + +TEST_CASE("iterator_facade: month_iterator_2 write") { + month_iterator_2 it; + *it = month::august; + CHECK(*it == month::august); + ++it; + CHECK(*it == month::september); + + *it = month::may; + CHECK(*it == month::may); + it++; + CHECK(*it == month::june); + CHECK(*it++ == month::june); + CHECK(*it == month::july); + + it.advance(3); + CHECK(*it == month::october); + *it = month::july; + it += 3; + CHECK(*it == month::october); + + auto a = *(it + 0); + CHECK(a == month::october); + auto b = *(it + 1); + CHECK(b == month::november); + + // These fail for some reason. + // @todo Fix this. + // auto c = it[0]; + // CHECK(c == month::october); + // auto d = it[1]; + // CHECK(d == month::november); + + CHECK(*(it + 0) == month::october); + CHECK(*(it + 1) == month::november); + + // These fail for some reason. + // @todo Fix this. + // CHECK(it[0] == month::october); + // CHECK(it[1] == month::november); +} + +/* + * Test arrow and arrow_proxy + */ +struct bar { + int val = -1; + bar() + : val(199) { + } + explicit bar(int x) + : val(x) { + } + int set(int x) { + auto tmp = val; + val = x; + return tmp; + } + int get() { + return val; + } +}; + +struct foo { + std::vector values{ + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}, + bar{0}}; + + class iterator : public iterator_facade { + size_t index_ = 0; + std::vector::iterator values_iter; + + public: + iterator() = delete; + iterator(std::vector::iterator values_iter, size_t index = 0) + : index_(index) + , values_iter(values_iter) { + } + + auto& dereference() const { + return values_iter[index_]; + } + // auto &dereference() { return values_iter[index_]; } + + bool equal_to(iterator o) const { + return index_ == o.index_; + } + void advance(int n) { + index_ += n; + } + int distance_to(iterator o) const { + return o.index_ - index_; + } + }; + + auto begin() { + return iterator{values.begin()}; + } + auto end() { + return iterator{values.begin(), size(values)}; + } +}; + +TEST_CASE("iterator_facade: arrow") { + std::vector x(10); + foo f; + auto it = f.begin(); + CHECK((*it).get() == 0); + CHECK(it[0].get() == 0); + CHECK(it->get() == 0); + + CHECK((*(it + 1)).get() == 0); + CHECK(it[1].get() == 0); + CHECK((*(it + 1)).set(42) == 0); + CHECK((*(it + 1)).get() == 42); + CHECK(it[1].get() == 42); + ++it; + CHECK((*(it + 0)).get() == 42); + CHECK(it[0].get() == 42); + --it; + CHECK((*(it + 1)).get() == 42); + CHECK(it[1].get() == 42); + + CHECK(it[0].get() == 0); + CHECK(it->set(43) == 0); + CHECK(it->get() == 43); + CHECK(it->set(41) == 43); + CHECK(it->get() == 41); + CHECK(it[0].get() == 41); + CHECK(it[1].get() == 42); + + CHECK(f.values[0].get() == 41); + CHECK(f.values[1].get() == 42); + CHECK(f.values[2].get() == 0); +} + +namespace { + +/** + * This structure wraps an std::vector and provides a simple iterator over it. + * The iterator that is wrapped is always the plain iterator, not a + * const_iterator; we define a const_iterator also using iterator_facade. Note + * that if the simple_mutable_struct is const, the internally stored vector will + * also be const, and so the begin() used below will return a const_iterator to + * the vector. We can either conditionally set the wrapped iterator type to be + * const or non-const, or we can mark the internal vector as mutable. We could + * also make the simple_mutable_iterator prameterized on iterator type. Below we + * use the second option. + */ +template +struct simple_mutable_struct { + static_assert(!std::is_const_v); + + // One option: Wrap just iterator -- need to make this mutable so that begin() + // returns just an iterator even if the simple_mutable_struct is const + /* mutable */ std::vector value; + + simple_mutable_struct() { + value.resize(10); + } + + template + class simple_mutable_iterator + : public iterator_facade> { + // Second option, wrap the iterator using the appropriate const or non-const + using Iter = std::conditional_t< + std::is_const_v, + typename std::vector>::const_iterator, + typename std::vector>::iterator>; + Iter current_; + + public: + simple_mutable_iterator() = default; + explicit simple_mutable_iterator(Iter i) + : current_(i) { + } + + template + requires(!std::is_const_v) + explicit simple_mutable_iterator(simple_mutable_iterator i) + : current_(i.current_) { + } + + V& dereference() const noexcept { + return *current_; + } + + void advance(int off) noexcept { + current_ += off; + } + + auto distance_to(simple_mutable_iterator o) const noexcept { + return o.current_ - current_; + } + + bool operator==(simple_mutable_iterator o) const noexcept { + return o.current_ == current_; + } + }; + + using iterator = simple_mutable_iterator; + using const_iterator = simple_mutable_iterator; + + using value_type = std::iterator_traits::value_type; + using reference = std::iterator_traits::reference; + using const_reference = std::iterator_traits::reference; + + auto begin() { + return iterator{value.begin()}; + } + auto end() { + return iterator{value.end()}; + } + auto begin() const { + return const_iterator{value.begin()}; + } + auto end() const { + return const_iterator{value.end()}; + } + auto cbegin() const { + return const_iterator{value.begin()}; + } + auto cend() const { + return const_iterator{value.end()}; + } + auto size() const { + return value.size(); + } +}; + +template +class pointer_wrapper : public iterator_facade> { + Value* current_ = nullptr; + + public: + pointer_wrapper() = default; + explicit pointer_wrapper(Value* i) + : current_(i) { + } + + Value& dereference() const noexcept { + return *current_; + } + + constexpr void advance(int off) noexcept { + current_ += off; + } + + constexpr auto distance_to(pointer_wrapper o) const noexcept { + return o.current_ - current_; + } + + // This is supposed to be inferred.... Either from equal_to or from + // distance_to -- but it seems to be necessary to specify it here. + constexpr bool operator==(pointer_wrapper o) const noexcept { + return o.current_ == current_; + } +}; +} // namespace + +template +void iterator_test(Iterator begin, [[maybe_unused]] Iterator end) { + Iterator it = begin; + ++it; + CHECK(*it == 14); + it++; + CHECK(*it == 15); + it += 3; + CHECK(*it == 18); + CHECK(*it++ == 18); + CHECK(*it == 19); + CHECK(*it-- == 19); + CHECK(*it == 18); + CHECK(*++it == 19); + CHECK(*--it == 18); + it -= 2; + CHECK(*it == 16); + it--; + CHECK(*it == 15); + --it; + CHECK(*it == 14); + + CHECK(it[0] == 14); + + it = begin; + for (int i = 0; i < 10; ++i) { + CHECK(it[i] == 13 + i); + } + + it = begin; + CHECK(it == it); + CHECK(it <= it); + CHECK(it >= it); + + auto it2 = it; + CHECK(it == it2); + CHECK(it2 == it); + CHECK(it2 <= it); + CHECK(it2 >= it); + ++it; + CHECK(it != it2); + CHECK(it2 != it); + CHECK(it2 < it); + CHECK(it > it2); + CHECK(!(it <= it2)); + CHECK(!(it2 >= it)); +} + +template +void struct_test(S& s) { + auto it = s.begin(); + + std::iota(s.begin(), s.end(), 13); + CHECK(std::equal(s.begin(), s.end(), s.value.begin())); + CHECK(std::equal(s.cbegin(), s.cend(), s.value.cbegin())); + CHECK(std::equal(s.begin(), s.end(), s.cbegin())); + CHECK(std::equal(s.cbegin(), s.cend(), s.begin())); + CHECK(std::equal( + s.begin(), + s.end(), + std::vector{13, 14, 15, 16, 17, 18, 19, 20, 21, 22}.begin())); + + iterator_test(s.begin(), s.end()); + iterator_test(s.cbegin(), s.cend()); + + int counter = 0; + for (auto i : s) { + CHECK(i == 13 + counter); + ++counter; + } + it = s.begin(); + *it = 17; + CHECK(*it == 17); + + std::iota(s.begin(), s.end(), 13); + [[maybe_unused]] auto cit = s.cbegin(); + counter = 0; + for (auto c = s.cbegin(); c != s.cend(); ++c) { + CHECK(*c == 13 + counter); + ++counter; + } + + // Error: trying to assign to const_iterator + // *cit = 0; +}; + +template +void const_struct_test(const S& s) { + [[maybe_unused]] auto it = s.begin(); + + CHECK(std::equal(s.begin(), s.end(), s.value.begin())); + CHECK(std::equal(s.cbegin(), s.cend(), s.value.cbegin())); + CHECK(std::equal(s.begin(), s.end(), s.cbegin())); + CHECK(std::equal(s.cbegin(), s.cend(), s.begin())); + CHECK(std::equal( + s.begin(), + s.end(), + std::vector{13, 14, 15, 16, 17, 18, 19, 20, 21, 22}.begin())); + + iterator_test(s.begin(), s.end()); + iterator_test(s.cbegin(), s.cend()); +}; + +TEST_CASE("simple_mutable_struct", "[iterator_facade]") { + simple_mutable_struct s; + + struct_test(s); + const_struct_test(s); + + std::iota(s.begin(), s.end(), 13); + + iterator_test(s.begin(), s.end()); + iterator_test(s.cbegin(), s.cend()); +} + +template +void range_test() { + CHECK(std::ranges::range); + CHECK(std::ranges::input_range); + + if (is_output) { + CHECK(std::ranges::output_range); + } else { + CHECK(!std::ranges::output_range); + } + + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + + if (std::same_as< + std::remove_cvref_t, + std::vector>) { + CHECK(std::ranges::contiguous_range); + } else { + CHECK(!std::ranges::contiguous_range); + } + + CHECK(std::ranges::sized_range); +} + +TEST_CASE("simple_mutable_struct range concepts", "[iterator_facade]") { + range_test>(); + range_test>(); + range_test const, false>(); + + range_test, false>(); + range_test, false>(); + range_test, false>(); +} + +template +void iterator_test() { + CHECK(std::input_iterator); + + if (is_output) { + CHECK( + std::output_iterator::value_type>); + } else { + CHECK( + !std::output_iterator::value_type>); + } + + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + if (std::is_same_v< + std::remove_cvref_t, + typename std::vector< + typename std::iterator_traits::value_type>::iterator> || + std::is_pointer_v) { + CHECK(std::contiguous_iterator); + } else { + CHECK(!std::contiguous_iterator); + } +} + +TEST_CASE("simple_mutable_struct iterator concepts", "[iterator_facade]") { + iterator_test::iterator>(); + iterator_test::iterator>(); + iterator_test::const_iterator, false>(); + + iterator_test>(); + iterator_test, false>(); + iterator_test, false>(); + + iterator_test(); + iterator_test(); + iterator_test(); + + // These will all fail (evidently) since the container is const. + // The range tests pass, however, even though the range concepts are + // defined in terms of the iterator concepts.... + // @todo Fix so that iterator and range checks are consistent. +#if 0 + iterator_test::iterator, false>(); + iterator_test::iterator, false>(); + iterator_test::const_iterator, false>(); + iterator_test>(); + iterator_test, false>(); + iterator_test, false>(); +#endif +} + +template < + class I, + class Value, + class Category = std::random_access_iterator_tag, + class Reference = Value&, + class ConstReference = const Reference, + class RValueReference = Value&&, + class Difference = std::ptrdiff_t> +void iterator_types_test() { + CHECK(std::is_same_v, std::remove_cvref_t>); + CHECK(std::is_same_v, Reference>); + // This is a C++23 concept -- enable once we move to C++23 + // CHECK(std::is_same_v, Reference>); + CHECK(std::is_same_v, Difference>); + CHECK(std::is_same_v, RValueReference>); + CHECK(std::is_same_v< + typename std::iterator_traits::iterator_category, + Category>); +} + +TEST_CASE("simple_mutable_struct iterator types", "[iterator_facade]") { + iterator_types_test::iterator, int>(); + iterator_types_test::const_iterator, const int>(); + iterator_types_test::iterator, int>(); + iterator_types_test::const_iterator, const int>(); +} + +TEST_CASE("pointer wrapper iterator types", "[iterator_facade]") { + iterator_types_test, int>(); + iterator_types_test(); + + iterator_types_test, const int>(); + iterator_types_test(); + iterator_types_test, int const>(); + iterator_types_test(); + + iterator_types_test::iterator, int>(); + iterator_types_test::const_iterator, const int>(); + // @todo: Why are these only forward iterators? + iterator_types_test< + const simple_mutable_struct::iterator, + int, + std::forward_iterator_tag>(); + iterator_types_test< + const simple_mutable_struct::const_iterator, + const int, + std::forward_iterator_tag>(); + + // @todo: Why are thest only input iterators? + iterator_types_test< + const pointer_wrapper, + int, + std::forward_iterator_tag>(); + iterator_types_test< + const pointer_wrapper, + const int, + std::forward_iterator_tag>(); + + iterator_types_test< + const pointer_wrapper, + int const, + std::forward_iterator_tag>(); +} From 2d33a5c8c637c4ec6c5ade642046c6067954badb Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Mon, 29 Apr 2024 15:13:43 +0300 Subject: [PATCH 299/456] Make closing a group that is not open a no-op (#4917) Issue reported from Cloud team: When attempting to close a TileDB group that is not open an error is raised polluting the server log. On the other hand, attempting to close a TileDB Array does not result to any error log (as per https://github.com/TileDB-Inc/TileDB/pull/2889). This PR unifies the behavior between closing and Array and a Group. Another solution would have been to check if the group is open on the server side but this would mean special casing for Groups, when I don't see the benefit of it. [sc-46186] --- TYPE: IMPROVEMENT DESC: Make closing a group that is not open a no-op --- tiledb/sm/group/group.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 4aea136fb6ac..41c04d23a8d0 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -217,7 +217,7 @@ Status Group::open(QueryType query_type) { Status Group::close() { // Check if group is open if (!is_open_) - return Status_GroupError("Cannot close group; Group not open."); + return Status::Ok(); if (remote_) { // Update group metadata for write queries if metadata was written by the From fd05e143265bc4ba222afced2b50f0d2e27b1d62 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 29 Apr 2024 16:27:16 +0300 Subject: [PATCH 300/456] Expand `TILEDB_REMOVE_DEPRECATIONS` to exclude all deprecated C and C++ APIs. (#4887) [SC-23937](https://app.shortcut.com/tiledb-inc/story/23937/build-artifact-disabling-deprecated-functions) > [!NOTE] > Depends on #4879. As part of preparing to remove the deprecated APIs, we need a mechanism to exclude them from being compiled. There exists already the `TILEDB_REMOVE_DEPRECATIONS` option, but it covers only a subset of the deprecated APIs in the C++ `Array` class. This PR expands `TILEDB_REMOVE_DEPRECATIONS` to cover every deprecated C and C++ API. When this option is set, a define with the same name is defined and the deprecated APIs are not compiled into the library. This flag also gets publicly exported from the `TileDB::tiledb` target. --- TYPE: BUILD DESC: Update the `TILEDB_REMOVE_DEPRECATIONS` option to exclude all deprecated C and C++ APIs from the library binary and the headers. --- CMakeLists.txt | 1 + cmake/Modules/Doxygen.cmake | 1 + examples/c_api/CMakeLists.txt | 4 + examples/cpp_api/CMakeLists.txt | 4 + tiledb/CMakeLists.txt | 7 +- .../group/group_api_external_experimental.h | 4 + tiledb/doxygen/source/c-api.rst | 43 +- tiledb/doxygen/source/conf.py | 3 +- tiledb/sm/c_api/tiledb.cc | 2 + tiledb/sm/c_api/tiledb.h | 589 +------- tiledb/sm/c_api/tiledb_deprecated.h | 631 ++++++++ tiledb/sm/c_api/tiledb_experimental.h | 4 + tiledb/sm/cpp_api/array.h | 284 ---- tiledb/sm/cpp_api/array_deprecated.h | 284 ++++ tiledb/sm/cpp_api/array_schema.h | 67 +- tiledb/sm/cpp_api/array_schema_deprecated.h | 93 ++ tiledb/sm/cpp_api/core_interface.h | 4 +- tiledb/sm/cpp_api/query.h | 1318 +---------------- tiledb/sm/cpp_api/query_deprecated.h | 1298 ++++++++++++++++ 19 files changed, 2386 insertions(+), 2255 deletions(-) create mode 100644 tiledb/sm/c_api/tiledb_deprecated.h create mode 100644 tiledb/sm/cpp_api/array_schema_deprecated.h create mode 100644 tiledb/sm/cpp_api/query_deprecated.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d643c3ac7b9..9354a378e993 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,6 +318,7 @@ endif() list(APPEND TILEDB_C_API_FILENAME_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/sm/c_api/tiledb.h" + "${CMAKE_SOURCE_DIR}/tiledb/sm/c_api/tiledb_deprecated.h" "${CMAKE_SOURCE_DIR}/tiledb/sm/c_api/tiledb_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/sm/c_api/tiledb_version.h" "${CMAKE_SOURCE_DIR}/tiledb/sm/c_api/tiledb_experimental.h" diff --git a/cmake/Modules/Doxygen.cmake b/cmake/Modules/Doxygen.cmake index a8a2467dc481..89a585371e65 100644 --- a/cmake/Modules/Doxygen.cmake +++ b/cmake/Modules/Doxygen.cmake @@ -34,6 +34,7 @@ if(DOXYGEN_FOUND) list(APPEND TILEDB_C_API_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h" "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/c_api/tiledb.h" + "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/c_api/tiledb_deprecated.h" ) file(GLOB TILEDB_CPP_API_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/tiledb/sm/cpp_api/*.h" diff --git a/examples/c_api/CMakeLists.txt b/examples/c_api/CMakeLists.txt index e52506ebb09e..9decfd8f48ef 100644 --- a/examples/c_api/CMakeLists.txt +++ b/examples/c_api/CMakeLists.txt @@ -48,6 +48,10 @@ endfunction() # Get the example sources file(GLOB TILEDB_EXAMPLE_SOURCES_CAPI "*.c") +if(TILEDB_REMOVE_DEPRECATIONS) + list(FILTER TILEDB_EXAMPLE_SOURCES_CAPI EXCLUDE REGEX "async") +endif() + # Iterate over all example sources and call the build function foreach(EXAMPLE_SOURCE ${TILEDB_EXAMPLE_SOURCES_CAPI}) # Get the binary name diff --git a/examples/cpp_api/CMakeLists.txt b/examples/cpp_api/CMakeLists.txt index 68a4e05c869e..1b1a6964296d 100644 --- a/examples/cpp_api/CMakeLists.txt +++ b/examples/cpp_api/CMakeLists.txt @@ -43,6 +43,10 @@ endfunction() # Get the example sources file(GLOB TILEDB_EXAMPLE_SOURCES_CPPAPI "*.cc") +if(TILEDB_REMOVE_DEPRECATIONS) + list(FILTER TILEDB_EXAMPLE_SOURCES_CPPAPI EXCLUDE REGEX "async") +endif() + # Iterate over all example sources and call the build function foreach(EXAMPLE_SOURCE ${TILEDB_EXAMPLE_SOURCES_CPPAPI}) # Get the binary name diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 21807023f7a8..01db9d78dda8 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -83,6 +83,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_deprecated.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_evolution.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/as_built_experimental.h @@ -113,6 +114,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_channel.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_condition.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_deprecated.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/query_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/schema_base.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/stats.h @@ -480,7 +482,6 @@ endif() if (TILEDB_REMOVE_DEPRECATIONS) add_definitions(-DTILEDB_REMOVE_DEPRECATIONS) -else() message(STATUS "The TileDB library is compiled without deprecated APIs.") endif() @@ -826,6 +827,10 @@ else() set_target_properties(tiledb PROPERTIES EXPORT_NAME tiledb_static) endif() +if(TILEDB_REMOVE_DEPRECATIONS) + target_compile_definitions(tiledb PUBLIC TILEDB_REMOVE_DEPRECATIONS) +endif() + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/sm/c_api/tiledb_version.h" ver) string(REGEX MATCH "TILEDB_VERSION_MAJOR ([0-9]*)" _ ${ver}) diff --git a/tiledb/api/c_api/group/group_api_external_experimental.h b/tiledb/api/c_api/group/group_api_external_experimental.h index 8ddcbf89ac6d..469931ae5fef 100644 --- a/tiledb/api/c_api/group/group_api_external_experimental.h +++ b/tiledb/api/c_api/group/group_api_external_experimental.h @@ -403,6 +403,7 @@ TILEDB_EXPORT capi_return_t tiledb_group_remove_member( TILEDB_EXPORT capi_return_t tiledb_group_get_member_count( tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* count) TILEDB_NOEXCEPT; +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Get a member of a group by index and details of group. * Deprecated, use \p tiledb_group_get_member_by_index_v2 instead. @@ -441,6 +442,7 @@ TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_index( char** uri, tiledb_object_t* type, char** name) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS /** * Get a member of a group by index and details of group @@ -481,6 +483,7 @@ TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index_v2( tiledb_object_t* type, tiledb_string_t** name) TILEDB_NOEXCEPT; +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Get a member of a group by name and details of group. * Deprecated, use \p tiledb_group_get_member_by_name_v2. @@ -518,6 +521,7 @@ TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_name( const char* name, char** uri, tiledb_object_t* type) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS /** * Get a member of a group by name and details of group. diff --git a/tiledb/doxygen/source/c-api.rst b/tiledb/doxygen/source/c-api.rst index e58b63dd11bb..97599da53693 100644 --- a/tiledb/doxygen/source/c-api.rst +++ b/tiledb/doxygen/source/c-api.rst @@ -221,8 +221,6 @@ Array :project: TileDB-C .. doxygenfunction:: tiledb_array_get_open_timestamp_end :project: TileDB-C -.. doxygenfunction:: tiledb_array_delete_fragments - :project: TileDB-C .. doxygenfunction:: tiledb_array_delete_fragments_list :project: TileDB-C .. doxygenfunction:: tiledb_array_open @@ -241,12 +239,8 @@ Array :project: TileDB-C .. doxygenfunction:: tiledb_array_create :project: TileDB-C -.. doxygenfunction:: tiledb_array_create_with_key - :project: TileDB-C .. doxygenfunction:: tiledb_array_consolidate :project: TileDB-C -.. doxygenfunction:: tiledb_array_consolidate_with_key - :project: TileDB-C .. doxygenfunction:: tiledb_array_vacuum :project: TileDB-C .. doxygenfunction:: tiledb_array_get_schema @@ -316,8 +310,6 @@ Array Schema :project: TileDB-C .. doxygenfunction:: tiledb_array_schema_load :project: TileDB-C -.. doxygenfunction:: tiledb_array_schema_load_with_key - :project: TileDB-C .. doxygenfunction:: tiledb_array_schema_get_array_type :project: TileDB-C .. doxygenfunction:: tiledb_array_schema_get_capacity @@ -436,8 +428,6 @@ Query :project: TileDB-C .. doxygenfunction:: tiledb_query_get_config :project: TileDB-C -.. doxygenfunction:: tiledb_query_set_subarray - :project: TileDB-C .. doxygenfunction:: tiledb_query_set_data_buffer :project: TileDB-C .. doxygenfunction:: tiledb_query_set_offsets_buffer @@ -462,8 +452,6 @@ Query :project: TileDB-C .. doxygenfunction:: tiledb_query_submit :project: TileDB-C -.. doxygenfunction:: tiledb_query_submit_async - :project: TileDB-C .. doxygenfunction:: tiledb_query_get_status :project: TileDB-C .. doxygenfunction:: tiledb_query_get_type @@ -474,30 +462,6 @@ Query :project: TileDB-C .. doxygenfunction:: tiledb_query_has_results :project: TileDB-C -.. doxygenfunction:: tiledb_query_add_range - :project: TileDB-C -.. doxygenfunction:: tiledb_query_add_range_by_name - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_from_name - :project: TileDB-C -.. doxygenfunction:: tiledb_query_add_range_var - :project: TileDB-C -.. doxygenfunction:: tiledb_query_add_range_var_by_name - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_var - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_var_from_name - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_var_size - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_var_size_from_name - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_num - :project: TileDB-C -.. doxygenfunction:: tiledb_query_get_range_num_from_name - :project: TileDB-C .. doxygenfunction:: tiledb_query_get_est_result_size :project: TileDB-C .. doxygenfunction:: tiledb_query_get_est_result_size_var @@ -748,7 +712,7 @@ Fragment Info :project: TileDB-C .. doxygenfunction:: tiledb_fragment_info_load :project: TileDB-C -.. doxygenfunction:: tiledb_fragment_info_get_fragment_name +.. doxygenfunction:: tiledb_fragment_info_get_fragment_name_v2 :project: TileDB-C .. doxygenfunction:: tiledb_fragment_info_get_fragment_num :project: TileDB-C @@ -812,6 +776,11 @@ Experimental .. autodoxygenfile:: tiledb_experimental.h :project: TileDB-C +Deprecated +------------- +.. autodoxygenfile:: tiledb_deprecated.h + :project: TileDB-C + Serialization ------------- .. autodoxygenfile:: tiledb_serialization.h diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index aeda2704a5ed..c6f583939874 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -87,11 +87,12 @@ breathe_projects = {'TileDB-C': doxygen_xml_dir, 'TileDB-C++': doxygen_xml_dir} breathe_default_project = 'TileDB-C' breathe_projects_source = { - 'TileDB-C': (c_api_src_path, ['tiledb.h', 'tiledb_experimental.h', 'tiledb_serialization.h']), + 'TileDB-C': (c_api_src_path, ['tiledb.h', 'tiledb_deprecated.h', 'tiledb_experimental.h', 'tiledb_serialization.h']), 'TileDB-C++': (cpp_api_src_path, ['tiledb', 'tiledb_experimental']) } breathe_domain_by_file_pattern = { '*/c_api/tiledb.h': 'c', + '*/c_api/tiledb_deprecated.h': 'c', '*/c_api/tiledb_experimental.h': 'c', '*/c_api/tiledb_serialization.h': 'c', '*/cpp_api/tiledb': 'cpp', diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 0b67cda4795b..b975c5069660 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -7307,6 +7307,7 @@ CAPI_INTERFACE( ctx, array, serialization_type, request, response); } +#ifndef TILEDB_REMOVE_DEPRECATIONS /* ****************************** */ /* C++ API */ /* ****************************** */ @@ -7318,6 +7319,7 @@ int32_t tiledb::impl::tiledb_query_submit_async_func( return api_entry( ctx, query, callback_func, callback_data); } +#endif /* ****************************** */ /* FRAGMENT INFO */ diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index a18fbcb03df3..0d9781a0276b 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -624,38 +624,6 @@ TILEDB_EXPORT int32_t tiledb_array_schema_load( const char* array_uri, tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; -/** - * Retrieves the schema of an encrypted array from the disk, creating an array - * schema struct. - * - * **Example:** - * - * @code{.c} - * // Load AES-256 key from disk, environment variable, etc. - * uint8_t key[32] = ...; - * tiledb_array_schema_t* array_schema; - * tiledb_array_schema_load_with_key( - * ctx, "s3://tiledb_bucket/my_array", TILEDB_AES_256_GCM, - * key, sizeof(key), &array_schema); - * // Make sure to free the array schema in the end - * @endcode - * - * @param ctx The TileDB context. - * @param array_uri The array whose schema will be retrieved. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @param array_schema The array schema to be retrieved, or `NULL` upon error. - * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_schema_load_with_key( - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length, - tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; - /** * Retrieves the array type. * @@ -1041,36 +1009,6 @@ TILEDB_EXPORT int32_t tiledb_query_get_config( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_config_t** config) TILEDB_NOEXCEPT; -/** - * Indicates that the query will write or read a subarray, and provides - * the appropriate information. - * - * **Example:** - * - * The following sets a 2D subarray [0,10], [20, 30] to the query. - * - * @code{.c} - * uint64_t subarray[] = { 0, 10, 20, 30}; - * tiledb_query_set_subarray(ctx, query, subarray); - * @endcode - * - * @param ctx The TileDB context. - * @param query The TileDB query. - * @param subarray The subarray in which the array read/write will be - * constrained on. It should be a sequence of [low, high] pairs (one - * pair per dimension). For the case of writes, this is meaningful only - * for dense arrays. Note that `subarray` must have the same type as the - * domain. - * @return `TILEDB_OK` for success or `TILEDB_ERR` for error. - * - * @note This will error if the query is already initialized. - * - * @note This function will error for writes to sparse arrays. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_set_subarray( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - const void* subarray) TILEDB_NOEXCEPT; /** * Indicates that the query will write or read a subarray, and provides @@ -1458,57 +1396,6 @@ TILEDB_EXPORT void tiledb_query_free(tiledb_query_t** query) TILEDB_NOEXCEPT; TILEDB_EXPORT int32_t tiledb_query_submit(tiledb_ctx_t* ctx, tiledb_query_t* query) TILEDB_NOEXCEPT; -/** - * Submits a TileDB query in asynchronous mode. - * - * **Examples:** - * - * Submit without a callback. - * - * @code{.c} - * tiledb_query_submit_async(ctx, query, NULL, NULL); - * @endcode - * - * Submit with a callback function `print` that takes as input message - * `msg` and prints it upon completion of the query. - * - * @code{.c} - * const char* msg = "Query completed"; - * tiledb_query_submit_async(ctx, &query, foo, msg); - * @endcode - * - * @param ctx The TileDB context. - * @param query The query to be submitted. - * @param callback The function to be called when the query completes. - * @param callback_data The data to be passed to the \p callback function. - * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. - * - * @note `tiledb_query_finalize` must be invoked after finish writing in - * global layout (via repeated invocations of `tiledb_query_submit`), - * in order to flush any internal state. - * - * @note For the case of reads, if the returned status is `TILEDB_INCOMPLETE`, - * TileDB could not fit the entire result in the user's buffers. In this - * case, the user should consume the read results (if any), optionally - * reset the buffers with `tiledb_query_set_buffer`, and then resubmit the - * query until the status becomes `TILEDB_COMPLETED`. If all buffer sizes - * after the termination of this function become 0, then this means that - * **no** useful data was read into the buffers, implying that larger - * buffers are needed for the query to proceed. In this case, the users - * must reallocate their buffers (increasing their size), reset the buffers - * with `tiledb_query_set_buffer`, and resubmit the query. - * - * @note \p callback will be executed in a thread managed by TileDB's internal - * thread pool. To allow TileDB to reuse the thread and avoid starving the - * thread pool, long-running callbacks should be dispatched to another - * thread. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_submit_async( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - void (*callback)(void*), - void* callback_data) TILEDB_NOEXCEPT; - /** * Checks if the query has returned any results. Applicable only to * read queries; it sets `has_results` to `0 in the case of writes. @@ -1610,360 +1497,6 @@ TILEDB_EXPORT int32_t tiledb_query_get_array( tiledb_ctx_t* ctx, tiledb_query_t* query, tiledb_array_t** array) TILEDB_NOEXCEPT; -/** - * Adds a 1D range along a subarray dimension index, which is in the form - * (start, end, stride). The datatype of the range components - * must be the same as the type of the domain of the array in the query. - * - * **Example:** - * - * @code{.c} - * uint32_t dim_idx = 2; - * int64_t start = 10; - * int64_t end = 20; - * tiledb_query_add_range(ctx, query, dim_idx, &start, &end, nullptr); - * @endcode - * - * @param ctx The TileDB context. - * @param query The query to add the range to. - * @param dim_idx The index of the dimension to add the range to. - * @param start The range start. - * @param end The range end. - * @param stride The range stride. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note The stride is currently unsupported. Use `nullptr` as the - * stride argument. - */ - -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - uint32_t dim_idx, - const void* start, - const void* end, - const void* stride) TILEDB_NOEXCEPT; - -/** - * Adds a 1D range along a subarray dimension name, which is in the form - * (start, end, stride). The datatype of the range components - * must be the same as the type of the domain of the array in the query. - * - * **Example:** - * - * @code{.c} - * char* dim_name = "rows"; - * int64_t start = 10; - * int64_t end = 20; - * tiledb_query_add_range_by_name(ctx, query, dim_name, &start, &end, nullptr); - * @endcode - * - * @param ctx The TileDB context. - * @param query The query to add the range to. - * @param dim_name The name of the dimension to add the range to. - * @param start The range start. - * @param end The range end. - * @param stride The range stride. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note The stride is currently unsupported. Use `nullptr` as the - * stride argument. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_by_name( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - const char* dim_name, - const void* start, - const void* end, - const void* stride) TILEDB_NOEXCEPT; - -/** - * Adds a 1D variable-sized range along a subarray dimension index, which is in - * the form (start, end). Applicable only to variable-sized dimensions. - * - * **Example:** - * - * @code{.c} - * uint32_t dim_idx = 2; - * char start[] = "a"; - * char end[] = "bb"; - * tiledb_query_add_range_var(ctx, query, dim_idx, start, 1, end, 2); - * @endcode - * - * @param ctx The TileDB context. - * @param query The query to add the range to. - * @param dim_idx The index of the dimension to add the range to. - * @param start The range start. - * @param start_size The size of the range start in bytes. - * @param end The range end. - * @param end_size The size of the range end in bytes. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_var( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - uint32_t dim_idx, - const void* start, - uint64_t start_size, - const void* end, - uint64_t end_size) TILEDB_NOEXCEPT; - -/** - * Adds a 1D variable-sized range along a subarray dimension name, which is in - * the form (start, end). Applicable only to variable-sized dimensions. - * - * **Example:** - * - * @code{.c} - * char* dim_name = "rows"; - * char start[] = "a"; - * char end[] = "bb"; - * tiledb_query_add_range_var_by_name(ctx, query, dim_name, start, 1, end, 2); - * @endcode - * - * @param ctx The TileDB context. - * @param query The query to add the range to. - * @param dim_name The name of the dimension to add the range to. - * @param start The range start. - * @param start_size The size of the range start in bytes. - * @param end The range end. - * @param end_size The size of the range end in bytes. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_var_by_name( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - const char* dim_name, - const void* start, - uint64_t start_size, - const void* end, - uint64_t end_size) TILEDB_NOEXCEPT; - -/** - * Retrieves the number of ranges of the query subarray along a given dimension - * index. - * - * **Example:** - * - * @code{.c} - * uint64_t range_num; - * tiledb_query_get_range_num(ctx, query, dim_idx, &range_num); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_idx The index of the dimension whose range number to retrieve. - * @param range_num The number of ranges to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_num( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - uint32_t dim_idx, - uint64_t* range_num) TILEDB_NOEXCEPT; - -/** - * Retrieves the number of ranges of the query subarray along a given dimension - * name. - * - * **Example:** - * - * @code{.c} - * uint64_t range_num; - * tiledb_query_get_range_num_from_name(ctx, query, dim_name, &range_num); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_name The name of the dimension whose range number to retrieve. - * @param range_num The number of ranges to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_num_from_name( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - const char* dim_name, - uint64_t* range_num) TILEDB_NOEXCEPT; - -/** - * Retrieves a specific range of the query subarray along a given dimension - * index. - * - * **Example:** - * - * @code{.c} - * const void* start; - * const void* end; - * const void* stride; - * tiledb_query_get_range( - * ctx, query, dim_idx, range_idx, &start, &end, &stride); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_idx The index of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start The range start to retrieve. - * @param end The range end to retrieve. - * @param stride The range stride to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - uint32_t dim_idx, - uint64_t range_idx, - const void** start, - const void** end, - const void** stride) TILEDB_NOEXCEPT; - -/** - * Retrieves a specific range of the query subarray along a given dimension - * name. - * - * **Example:** - * - * @code{.c} - * const void* start; - * const void* end; - * const void* stride; - * tiledb_query_get_range_from_name( - * ctx, query, dim_name, range_idx, &start, &end, &stride); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_name The name of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start The range start to retrieve. - * @param end The range end to retrieve. - * @param stride The range stride to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_from_name( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - const char* dim_name, - uint64_t range_idx, - const void** start, - const void** end, - const void** stride) TILEDB_NOEXCEPT; - -/** - * Retrieves a range's start and end size for a given variable-length - * dimension index at a given range index. - * - * **Example:** - * - * @code{.c} - * uint64_t start_size; - * uint64_t end_size; - * tiledb_query_get_range_var_size( - * ctx, query, dim_idx, range_idx, &start_size, &end_size); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_idx The index of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start_size range start size in bytes - * @param end_size range end size in bytes - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_size( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - uint32_t dim_idx, - uint64_t range_idx, - uint64_t* start_size, - uint64_t* end_size) TILEDB_NOEXCEPT; - -/** - * Retrieves a range's start and end size for a given variable-length - * dimension name at a given range index. - * - * **Example:** - * - * @code{.c} - * uint64_t start_size; - * uint64_t end_size; - * tiledb_query_get_range_var_size_from_name( - * ctx, query, dim_name, range_idx, &start_size, &end_size); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_name The name of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start_size range start size in bytes - * @param end_size range end size in bytes - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_size_from_name( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - const char* dim_name, - uint64_t range_idx, - uint64_t* start_size, - uint64_t* end_size) TILEDB_NOEXCEPT; - -/** - * Retrieves a specific range of the query subarray along a given - * variable-length dimension index. - * - * **Example:** - * - * @code{.c} - * const void* start; - * const void* end; - * tiledb_query_get_range_var( - * ctx, query, dim_idx, range_idx, &start, &end); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_idx The index of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start The range start to retrieve. - * @param end The range end to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - uint32_t dim_idx, - uint64_t range_idx, - void* start, - void* end) TILEDB_NOEXCEPT; - -/** - * Retrieves a specific range of the query subarray along a given - * variable-length dimension name. - * - * **Example:** - * - * @code{.c} - * const void* start; - * const void* end; - * tiledb_query_get_range_var_from_name( - * ctx, query, dim_name, range_idx, &start, &end); - * @endcode - * - * @param ctx The TileDB context - * @param query The query. - * @param dim_name The name of the dimension to retrieve the range from. - * @param range_idx The index of the range to retrieve. - * @param start The range start to retrieve. - * @param end The range end to retrieve. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_from_name( - tiledb_ctx_t* ctx, - const tiledb_query_t* query, - const char* dim_name, - uint64_t range_idx, - void* start, - void* end) TILEDB_NOEXCEPT; /** * Retrieves the estimated result size for a fixed-sized attribute/dimension. @@ -2949,34 +2482,6 @@ TILEDB_EXPORT int32_t tiledb_array_get_open_timestamp_end( tiledb_array_t* array, uint64_t* timestamp_end) TILEDB_NOEXCEPT; -/** - * Deletes array fragments written between the input timestamps. - * - * **Example:** - * - * @code{.c} - * tiledb_array_delete_fragments( - * ctx, array, "hdfs:///temp/my_array", 0, UINT64_MAX); - * @endcode - * - * @param ctx The TileDB context. - * @param array The array to delete the fragments from. - * @param uri The URI of the fragments' parent Array. - * @param timestamp_start The epoch timestamp in milliseconds. - * @param timestamp_end The epoch timestamp in milliseconds. Use UINT64_MAX for - * the current timestamp. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note This function was deprecated in release 2.18 in favor of - * tiledb_array_delete_fragments_v2. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_delete_fragments( - tiledb_ctx_t* ctx, - tiledb_array_t* array, - const char* uri, - uint64_t timestamp_start, - uint64_t timestamp_end) TILEDB_NOEXCEPT; - /** * Deletes array fragments written between the input timestamps. * @@ -3252,36 +2757,6 @@ TILEDB_EXPORT int32_t tiledb_array_create( const char* array_uri, const tiledb_array_schema_t* array_schema) TILEDB_NOEXCEPT; -/** - * Creates a new encrypted TileDB array given an input schema. - * - * Encrypted arrays can only be created through this function. - * - * **Example:** - * - * @code{.c} - * uint8_t key[32] = ...; - * tiledb_array_create_with_key( - * ctx, "hdfs:///tiledb_arrays/my_array", array_schema, - * TILEDB_AES_256_GCM, key, sizeof(key)); - * @endcode - * - * @param ctx The TileDB context. - * @param array_uri The array name. - * @param array_schema The array schema. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_create_with_key( - tiledb_ctx_t* ctx, - const char* array_uri, - const tiledb_array_schema_t* array_schema, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length) TILEDB_NOEXCEPT; - /** * Depending on the consoliation mode in the config, consolidates either the * fragment files, fragment metadata files, or array metadata files into a @@ -3309,40 +2784,6 @@ TILEDB_EXPORT int32_t tiledb_array_consolidate( const char* array_uri, tiledb_config_t* config) TILEDB_NOEXCEPT; -/** - * Depending on the consoliation mode in the config, consolidates either the - * fragment files, fragment metadata files, or array metadata files into a - * single file. - * - * **Example:** - * - * @code{.c} - * uint8_t key[32] = ...; - * tiledb_array_consolidate_with_key( - * ctx, "hdfs:///tiledb_arrays/my_array", - * TILEDB_AES_256_GCM, key, sizeof(key), nullptr); - * @endcode - * - * @param ctx The TileDB context. - * @param array_uri The name of the TileDB array to be consolidated. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config from `ctx`). - * The `sm.consolidation.mode` parameter determines which type of - * consolidation to perform. - * - * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_consolidate_with_key( - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length, - tiledb_config_t* config) TILEDB_NOEXCEPT; - /** * Cleans up the array, such as consolidated fragments and array metadata. * Note that this will coarsen the granularity of time traveling (see docs @@ -4139,29 +3580,6 @@ TILEDB_EXPORT int32_t tiledb_fragment_info_get_config( TILEDB_EXPORT int32_t tiledb_fragment_info_load( tiledb_ctx_t* ctx, tiledb_fragment_info_t* fragment_info) TILEDB_NOEXCEPT; -/** - * Gets the name of a fragment. Deprecated, use - * \p tiledb_fragment_info_get_fragment_name_v2 instead. - * - * **Example:** - * - * @code{.c} - * const char* name; - * tiledb_fragment_info_get_fragment_name(ctx, fragment_info, 1, &name); - * @endcode - * - * @param ctx The TileDB context. - * @param fragment_info The fragment info object. - * @param fid The index of the fragment of interest. - * @param name The fragment name to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_fragment_info_get_fragment_name( - tiledb_ctx_t* ctx, - tiledb_fragment_info_t* fragment_info, - uint32_t fid, - const char** name) TILEDB_NOEXCEPT; - /** * Gets the name of a fragment. * @@ -4937,4 +4355,11 @@ TILEDB_EXPORT int32_t tiledb_fragment_info_dump( } #endif +/* ********************************* */ +/* DEPRECATED API */ +/* ********************************* */ +#ifndef TILEDB_REMOVE_DEPRECATIONS +#include "tiledb_deprecated.h" +#endif // TILEDB_REMOVE_DEPRECATIONS + #endif // TILEDB_H diff --git a/tiledb/sm/c_api/tiledb_deprecated.h b/tiledb/sm/c_api/tiledb_deprecated.h new file mode 100644 index 000000000000..f7585137819a --- /dev/null +++ b/tiledb/sm/c_api/tiledb_deprecated.h @@ -0,0 +1,631 @@ +/* + * @file tiledb_deprecated.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the deprecated C API for TileDB. + */ + +#ifndef TILEDB_DEPRECATED_H +#define TILEDB_DEPRECATED_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Retrieves the schema of an encrypted array from the disk, creating an array + * schema struct. + * + * **Example:** + * + * @code{.c} + * // Load AES-256 key from disk, environment variable, etc. + * uint8_t key[32] = ...; + * tiledb_array_schema_t* array_schema; + * tiledb_array_schema_load_with_key( + * ctx, "s3://tiledb_bucket/my_array", TILEDB_AES_256_GCM, + * key, sizeof(key), &array_schema); + * // Make sure to free the array schema in the end + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The array whose schema will be retrieved. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @param array_schema The array schema to be retrieved, or `NULL` upon error. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_schema_load_with_key( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length, + tiledb_array_schema_t** array_schema) TILEDB_NOEXCEPT; + +/** + * Indicates that the query will write or read a subarray, and provides + * the appropriate information. + * + * **Example:** + * + * The following sets a 2D subarray [0,10], [20, 30] to the query. + * + * @code{.c} + * uint64_t subarray[] = { 0, 10, 20, 30}; + * tiledb_query_set_subarray(ctx, query, subarray); + * @endcode + * + * @param ctx The TileDB context. + * @param query The TileDB query. + * @param subarray The subarray in which the array read/write will be + * constrained on. It should be a sequence of [low, high] pairs (one + * pair per dimension). For the case of writes, this is meaningful only + * for dense arrays. Note that `subarray` must have the same type as the + * domain. + * @return `TILEDB_OK` for success or `TILEDB_ERR` for error. + * + * @note This will error if the query is already initialized. + * + * @note This function will error for writes to sparse arrays. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_set_subarray( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + const void* subarray) TILEDB_NOEXCEPT; + +/** + * Submits a TileDB query in asynchronous mode. + * + * **Examples:** + * + * Submit without a callback. + * + * @code{.c} + * tiledb_query_submit_async(ctx, query, NULL, NULL); + * @endcode + * + * Submit with a callback function `print` that takes as input message + * `msg` and prints it upon completion of the query. + * + * @code{.c} + * const char* msg = "Query completed"; + * tiledb_query_submit_async(ctx, &query, foo, msg); + * @endcode + * + * @param ctx The TileDB context. + * @param query The query to be submitted. + * @param callback The function to be called when the query completes. + * @param callback_data The data to be passed to the \p callback function. + * @return `TILEDB_OK` for success and `TILEDB_OOM` or `TILEDB_ERR` for error. + * + * @note `tiledb_query_finalize` must be invoked after finish writing in + * global layout (via repeated invocations of `tiledb_query_submit`), + * in order to flush any internal state. + * + * @note For the case of reads, if the returned status is `TILEDB_INCOMPLETE`, + * TileDB could not fit the entire result in the user's buffers. In this + * case, the user should consume the read results (if any), optionally + * reset the buffers with `tiledb_query_set_buffer`, and then resubmit the + * query until the status becomes `TILEDB_COMPLETED`. If all buffer sizes + * after the termination of this function become 0, then this means that + * **no** useful data was read into the buffers, implying that larger + * buffers are needed for the query to proceed. In this case, the users + * must reallocate their buffers (increasing their size), reset the buffers + * with `tiledb_query_set_buffer`, and resubmit the query. + * + * @note \p callback will be executed in a thread managed by TileDB's internal + * thread pool. To allow TileDB to reuse the thread and avoid starving the + * thread pool, long-running callbacks should be dispatched to another + * thread. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_submit_async( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + void (*callback)(void*), + void* callback_data) TILEDB_NOEXCEPT; + +/** + * Adds a 1D range along a subarray dimension index, which is in the form + * (start, end, stride). The datatype of the range components + * must be the same as the type of the domain of the array in the query. + * + * **Example:** + * + * @code{.c} + * uint32_t dim_idx = 2; + * int64_t start = 10; + * int64_t end = 20; + * tiledb_query_add_range(ctx, query, dim_idx, &start, &end, nullptr); + * @endcode + * + * @param ctx The TileDB context. + * @param query The query to add the range to. + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start. + * @param end The range end. + * @param stride The range stride. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note The stride is currently unsupported. Use `nullptr` as the + * stride argument. + */ + +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + uint32_t dim_idx, + const void* start, + const void* end, + const void* stride) TILEDB_NOEXCEPT; + +/** + * Adds a 1D range along a subarray dimension name, which is in the form + * (start, end, stride). The datatype of the range components + * must be the same as the type of the domain of the array in the query. + * + * **Example:** + * + * @code{.c} + * char* dim_name = "rows"; + * int64_t start = 10; + * int64_t end = 20; + * tiledb_query_add_range_by_name(ctx, query, dim_name, &start, &end, nullptr); + * @endcode + * + * @param ctx The TileDB context. + * @param query The query to add the range to. + * @param dim_name The name of the dimension to add the range to. + * @param start The range start. + * @param end The range end. + * @param stride The range stride. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note The stride is currently unsupported. Use `nullptr` as the + * stride argument. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_by_name( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + const char* dim_name, + const void* start, + const void* end, + const void* stride) TILEDB_NOEXCEPT; + +/** + * Adds a 1D variable-sized range along a subarray dimension index, which is in + * the form (start, end). Applicable only to variable-sized dimensions. + * + * **Example:** + * + * @code{.c} + * uint32_t dim_idx = 2; + * char start[] = "a"; + * char end[] = "bb"; + * tiledb_query_add_range_var(ctx, query, dim_idx, start, 1, end, 2); + * @endcode + * + * @param ctx The TileDB context. + * @param query The query to add the range to. + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start. + * @param start_size The size of the range start in bytes. + * @param end The range end. + * @param end_size The size of the range end in bytes. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_var( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + uint32_t dim_idx, + const void* start, + uint64_t start_size, + const void* end, + uint64_t end_size) TILEDB_NOEXCEPT; + +/** + * Adds a 1D variable-sized range along a subarray dimension name, which is in + * the form (start, end). Applicable only to variable-sized dimensions. + * + * **Example:** + * + * @code{.c} + * char* dim_name = "rows"; + * char start[] = "a"; + * char end[] = "bb"; + * tiledb_query_add_range_var_by_name(ctx, query, dim_name, start, 1, end, 2); + * @endcode + * + * @param ctx The TileDB context. + * @param query The query to add the range to. + * @param dim_name The name of the dimension to add the range to. + * @param start The range start. + * @param start_size The size of the range start in bytes. + * @param end The range end. + * @param end_size The size of the range end in bytes. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_range_var_by_name( + tiledb_ctx_t* ctx, + tiledb_query_t* query, + const char* dim_name, + const void* start, + uint64_t start_size, + const void* end, + uint64_t end_size) TILEDB_NOEXCEPT; + +/** + * Retrieves the number of ranges of the query subarray along a given dimension + * index. + * + * **Example:** + * + * @code{.c} + * uint64_t range_num; + * tiledb_query_get_range_num(ctx, query, dim_idx, &range_num); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_idx The index of the dimension whose range number to retrieve. + * @param range_num The number of ranges to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_num( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + uint32_t dim_idx, + uint64_t* range_num) TILEDB_NOEXCEPT; + +/** + * Retrieves the number of ranges of the query subarray along a given dimension + * name. + * + * **Example:** + * + * @code{.c} + * uint64_t range_num; + * tiledb_query_get_range_num_from_name(ctx, query, dim_name, &range_num); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_name The name of the dimension whose range number to retrieve. + * @param range_num The number of ranges to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_num_from_name( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + const char* dim_name, + uint64_t* range_num) TILEDB_NOEXCEPT; + +/** + * Retrieves a specific range of the query subarray along a given dimension + * index. + * + * **Example:** + * + * @code{.c} + * const void* start; + * const void* end; + * const void* stride; + * tiledb_query_get_range( + * ctx, query, dim_idx, range_idx, &start, &end, &stride); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_idx The index of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start The range start to retrieve. + * @param end The range end to retrieve. + * @param stride The range stride to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + uint32_t dim_idx, + uint64_t range_idx, + const void** start, + const void** end, + const void** stride) TILEDB_NOEXCEPT; + +/** + * Retrieves a specific range of the query subarray along a given dimension + * name. + * + * **Example:** + * + * @code{.c} + * const void* start; + * const void* end; + * const void* stride; + * tiledb_query_get_range_from_name( + * ctx, query, dim_name, range_idx, &start, &end, &stride); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_name The name of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start The range start to retrieve. + * @param end The range end to retrieve. + * @param stride The range stride to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_from_name( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + const char* dim_name, + uint64_t range_idx, + const void** start, + const void** end, + const void** stride) TILEDB_NOEXCEPT; + +/** + * Retrieves a range's start and end size for a given variable-length + * dimension index at a given range index. + * + * **Example:** + * + * @code{.c} + * uint64_t start_size; + * uint64_t end_size; + * tiledb_query_get_range_var_size( + * ctx, query, dim_idx, range_idx, &start_size, &end_size); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_idx The index of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start_size range start size in bytes + * @param end_size range end size in bytes + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_size( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + uint32_t dim_idx, + uint64_t range_idx, + uint64_t* start_size, + uint64_t* end_size) TILEDB_NOEXCEPT; + +/** + * Retrieves a range's start and end size for a given variable-length + * dimension name at a given range index. + * + * **Example:** + * + * @code{.c} + * uint64_t start_size; + * uint64_t end_size; + * tiledb_query_get_range_var_size_from_name( + * ctx, query, dim_name, range_idx, &start_size, &end_size); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_name The name of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start_size range start size in bytes + * @param end_size range end size in bytes + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_size_from_name( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + const char* dim_name, + uint64_t range_idx, + uint64_t* start_size, + uint64_t* end_size) TILEDB_NOEXCEPT; + +/** + * Retrieves a specific range of the query subarray along a given + * variable-length dimension index. + * + * **Example:** + * + * @code{.c} + * const void* start; + * const void* end; + * tiledb_query_get_range_var( + * ctx, query, dim_idx, range_idx, &start, &end); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_idx The index of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start The range start to retrieve. + * @param end The range end to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + uint32_t dim_idx, + uint64_t range_idx, + void* start, + void* end) TILEDB_NOEXCEPT; + +/** + * Retrieves a specific range of the query subarray along a given + * variable-length dimension name. + * + * **Example:** + * + * @code{.c} + * const void* start; + * const void* end; + * tiledb_query_get_range_var_from_name( + * ctx, query, dim_name, range_idx, &start, &end); + * @endcode + * + * @param ctx The TileDB context + * @param query The query. + * @param dim_name The name of the dimension to retrieve the range from. + * @param range_idx The index of the range to retrieve. + * @param start The range start to retrieve. + * @param end The range end to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_get_range_var_from_name( + tiledb_ctx_t* ctx, + const tiledb_query_t* query, + const char* dim_name, + uint64_t range_idx, + void* start, + void* end) TILEDB_NOEXCEPT; + +/** + * Deletes array fragments written between the input timestamps. + * + * **Example:** + * + * @code{.c} + * tiledb_array_delete_fragments( + * ctx, array, "hdfs:///temp/my_array", 0, UINT64_MAX); + * @endcode + * + * @param ctx The TileDB context. + * @param array The array to delete the fragments from. + * @param uri The URI of the fragments' parent Array. + * @param timestamp_start The epoch timestamp in milliseconds. + * @param timestamp_end The epoch timestamp in milliseconds. Use UINT64_MAX for + * the current timestamp. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note This function was deprecated in release 2.18 in favor of + * tiledb_array_delete_fragments_v2. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_delete_fragments( + tiledb_ctx_t* ctx, + tiledb_array_t* array, + const char* uri, + uint64_t timestamp_start, + uint64_t timestamp_end) TILEDB_NOEXCEPT; + +/** + * Creates a new encrypted TileDB array given an input schema. + * + * Encrypted arrays can only be created through this function. + * + * **Example:** + * + * @code{.c} + * uint8_t key[32] = ...; + * tiledb_array_create_with_key( + * ctx, "hdfs:///tiledb_arrays/my_array", array_schema, + * TILEDB_AES_256_GCM, key, sizeof(key)); + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The array name. + * @param array_schema The array schema. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_create_with_key( + tiledb_ctx_t* ctx, + const char* array_uri, + const tiledb_array_schema_t* array_schema, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length) TILEDB_NOEXCEPT; + +/** + * Depending on the consoliation mode in the config, consolidates either the + * fragment files, fragment metadata files, or array metadata files into a + * single file. + * + * **Example:** + * + * @code{.c} + * uint8_t key[32] = ...; + * tiledb_array_consolidate_with_key( + * ctx, "hdfs:///tiledb_arrays/my_array", + * TILEDB_AES_256_GCM, key, sizeof(key), nullptr); + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The name of the TileDB array to be consolidated. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @param config Configuration parameters for the consolidation + * (`nullptr` means default, which will use the config from `ctx`). + * The `sm.consolidation.mode` parameter determines which type of + * consolidation to perform. + * + * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_consolidate_with_key( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length, + tiledb_config_t* config) TILEDB_NOEXCEPT; + +/** + * Gets the name of a fragment. Deprecated, use + * \p tiledb_fragment_info_get_fragment_name_v2 instead. + * + * **Example:** + * + * @code{.c} + * const char* name; + * tiledb_fragment_info_get_fragment_name(ctx, fragment_info, 1, &name); + * @endcode + * + * @param ctx The TileDB context. + * @param fragment_info The fragment info object. + * @param fid The index of the fragment of interest. + * @param name The fragment name to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_fragment_info_get_fragment_name( + tiledb_ctx_t* ctx, + tiledb_fragment_info_t* fragment_info, + uint32_t fid, + const char** name) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_DEPRECATED_H diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index cedfe674a869..399eb9dcfe0b 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -375,6 +375,7 @@ TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( TILEDB_EXPORT int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) TILEDB_NOEXCEPT; +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Note: This API is deprecated and replaced with tiledb_array_delete (above). * @@ -393,6 +394,7 @@ TILEDB_EXPORT int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) */ TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_delete_array( tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS /** * Evolve array schema of an array. @@ -519,6 +521,7 @@ TILEDB_EXPORT int32_t tiledb_subarray_add_point_ranges( const void* start, uint64_t count) TILEDB_NOEXCEPT; +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Adds a set of point ranges along subarray dimension index. Each value * in the target array is added as `add_range(x,x)` for count elements. @@ -549,6 +552,7 @@ TILEDB_DEPRECATED_EXPORT int32_t tiledb_query_add_point_ranges( uint32_t dim_idx, const void* start, uint64_t count) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS /** * Get the number of relevant fragments from the subarray. Should only be diff --git a/tiledb/sm/cpp_api/array.h b/tiledb/sm/cpp_api/array.h index 5151f7ed8c86..32ba99fb5336 100644 --- a/tiledb/sm/cpp_api/array.h +++ b/tiledb/sm/cpp_api/array.h @@ -345,28 +345,6 @@ class Array { ctx.handle_error(tiledb_array_delete(ctx.ptr().get(), uri.c_str())); } - /** - * Deletes the fragments written between the input timestamps of an array - * with the input uri. - * - * @param uri The URI of the fragments' parent Array. - * @param timestamp_start The epoch start timestamp in milliseconds. - * @param timestamp_end The epoch end timestamp in milliseconds. Use - * UINT64_MAX for the current timestamp. - */ - TILEDB_DEPRECATED - void delete_fragments( - const std::string& uri, - uint64_t timestamp_start, - uint64_t timestamp_end) const { - throw std::logic_error( - "This method is deprecated. Please use " - "Array::delete_fragments(ctx, uri, timestamp_start, timestamp_end)"); - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_array_delete_fragments_v2( - ctx.ptr().get(), uri.c_str(), timestamp_start, timestamp_end)); - } - /** * Deletes the fragments written between the input timestamps of an array * with the input uri. @@ -599,50 +577,6 @@ class Array { ctx.ptr().get(), uri.c_str(), config ? config->ptr().get() : nullptr)); } - /** - * @brief Consolidates the fragments of an encrypted array into a single - * fragment. - * - * You must first finalize all queries to the array before consolidation can - * begin (as consolidation temporarily acquires an exclusive lock on the - * array). - * - * **Example:** - * @code{.cpp} - * // Load AES-256 key from disk, environment variable, etc. - * uint8_t key[32] = ...; - * tiledb::Array::consolidate( - * ctx, - * "s3://bucket-name/array-name", - * TILEDB_AES_256_GCM, - * key, - * sizeof(key)); - * @endcode - * - * @param ctx TileDB context - * @param array_uri The URI of the TileDB array to be consolidated. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @param config Configuration parameters for the consolidation. - */ - TILEDB_DEPRECATED - static void consolidate( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length, - Config* const config = nullptr) { - ctx.handle_error(tiledb_array_consolidate_with_key( - ctx.ptr().get(), - uri.c_str(), - encryption_type, - encryption_key, - key_length, - config ? config->ptr().get() : nullptr)); - } - /** * @brief Consolidates the fragments with the input uris into a single * fragment. @@ -686,40 +620,6 @@ class Array { config ? config->ptr().get() : nullptr)); } - // clang-format off - /** - * @copybrief Array::consolidate(const Context&,const std::string&,tiledb_encryption_type_t,const void*,uint32_t,const Config&) - * - * See @ref Array::consolidate( - * const Context&, - * const std::string&, - * tiledb_encryption_type_t, - * const void*, - * uint32_t,const Config&) "Array::consolidate" - * - * @param ctx TileDB context - * @param array_uri The URI of the TileDB array to be consolidated. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param config Configuration parameters for the consolidation. - */ - // clang-format on - TILEDB_DEPRECATED - static void consolidate( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const std::string& encryption_key, - Config* const config = nullptr) { - return consolidate( - ctx, - uri, - encryption_type, - encryption_key.data(), - (uint32_t)encryption_key.size(), - config); - } - /** * Cleans up the array, such as consolidated fragments and array metadata. * Note that this will coarsen the granularity of time traveling (see docs @@ -781,102 +681,6 @@ class Array { return ArraySchema(ctx, schema); } - /** - * Loads the array schema from an encrypted array. - * - * **Example:** - * @code{.cpp} - * auto schema = tiledb::Array::load_schema(ctx, - * "s3://bucket-name/array-name", key_type, key, key_len); - * @endcode - * - * @param ctx The TileDB context. - * @param uri The array URI. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @return The loaded ArraySchema object. - */ - TILEDB_DEPRECATED - static ArraySchema load_schema( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length) { - tiledb_array_schema_t* schema; - ctx.handle_error(tiledb_array_schema_load_with_key( - ctx.ptr().get(), - uri.c_str(), - encryption_type, - encryption_key, - key_length, - &schema)); - return ArraySchema(ctx, schema); - } - - /** - * @brief Creates a new encrypted TileDB array given an input schema. - * - * **Example:** - * @code{.cpp} - * // Load AES-256 key from disk, environment variable, etc. - * uint8_t key[32] = ...; - * tiledb::Array::create("s3://bucket-name/array-name", schema, - * TILEDB_AES_256_GCM, key, sizeof(key)); - * @endcode - * - * @param uri URI where array will be created. - * @param schema The array schema. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - */ - TILEDB_DEPRECATED - static void create( - const std::string& uri, - const ArraySchema& schema, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length) { - auto& ctx = schema.context(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_array_schema_check(c_ctx, schema.ptr().get())); - ctx.handle_error(tiledb_array_create_with_key( - c_ctx, - uri.c_str(), - schema.ptr().get(), - encryption_type, - encryption_key, - key_length)); - } - - // clang-format off - /** - * @copybrief Array::create(const std::string&,const ArraySchema&,tiledb_encryption_type_t,const void*,uint32_t) - * - * See @ref Array::create(const std::string&,const ArraySchema&,tiledb_encryption_type_t,const void*,uint32_t) "Array::create" - * - * @param uri URI where array will be created. - * @param schema The array schema. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - */ - // clang-format on - TILEDB_DEPRECATED - static void create( - const std::string& uri, - const ArraySchema& schema, - tiledb_encryption_type_t encryption_type, - const std::string& encryption_key) { - return create( - uri, - schema, - encryption_type, - encryption_key.data(), - (uint32_t)encryption_key.size()); - } - /** * Gets the encryption type the given array was created with. * @@ -1156,94 +960,6 @@ class Array { consolidate(ctx, uri, config_aux); } - /** - * @brief Consolidates the metadata of an encrypted array. - * - * You must first finalize all queries to the array before consolidation can - * begin (as consolidation temporarily acquires an exclusive lock on the - * array). - * - * **Example:** - * @code{.cpp} - * // Load AES-256 key from disk, environment variable, etc. - * uint8_t key[32] = ...; - * tiledb::Array::consolidate_metadata( - * ctx, - * "s3://bucket-name/array-name", - * TILEDB_AES_256_GCM, - * key, - * sizeof(key)); - * @endcode - * - * @param ctx TileDB context - * @param array_uri The URI of the TileDB array whose - * metadata will be consolidated. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - * @param config Configuration parameters for the consolidation. - */ - TILEDB_DEPRECATED - static void consolidate_metadata( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length, - Config* const config = nullptr) { - Config local_cfg; - Config* config_aux = config; - if (!config_aux) { - config_aux = &local_cfg; - } - - (*config_aux)["sm.consolidation.mode"] = "array_meta"; - consolidate( - ctx, uri, encryption_type, encryption_key, key_length, config_aux); - } - - // clang-format off - /** - * @copybrief Array::consolidate_metadata(const Context&, const std::string&, tiledb_encryption_type_t, const void*,uint32_t, const Config&) - * - * See @ref Array::consolidate_metadata( - * const Context&, - * const std::string&, - * tiledb_encryption_type_t, - * const void*, - * uint32_t,const Config&) "Array::consolidate_metadata" - * - * @param ctx TileDB context - * @param array_uri The URI of the TileDB array whose - * metadata will be consolidated. - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param config Configuration parameters for the consolidation. - */ - // clang-format on - TILEDB_DEPRECATED - static void consolidate_metadata( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const std::string& encryption_key, - Config* const config = nullptr) { - Config local_cfg; - Config* config_aux = config; - if (!config_aux) { - config_aux = &local_cfg; - } - - (*config_aux)["sm.consolidation.mode"] = "array_meta"; - consolidate( - ctx, - uri, - encryption_type, - encryption_key.data(), - (uint32_t)encryption_key.size(), - config_aux); - } - /** * @brief Upgrades an array to the latest format version. * diff --git a/tiledb/sm/cpp_api/array_deprecated.h b/tiledb/sm/cpp_api/array_deprecated.h index 7c8ef78bcda8..7f8617962184 100644 --- a/tiledb/sm/cpp_api/array_deprecated.h +++ b/tiledb/sm/cpp_api/array_deprecated.h @@ -218,3 +218,287 @@ void open( std::string enc_key_str((const char*)encryption_key, key_length); open(query_type, encryption_type, enc_key_str); } + +/** + * Deletes the fragments written between the input timestamps of an array + * with the input uri. + * + * @param uri The URI of the fragments' parent Array. + * @param timestamp_start The epoch start timestamp in milliseconds. + * @param timestamp_end The epoch end timestamp in milliseconds. Use + * UINT64_MAX for the current timestamp. + */ +TILEDB_DEPRECATED +void delete_fragments( + const std::string& uri, + uint64_t timestamp_start, + uint64_t timestamp_end) const { + throw std::logic_error( + "This method is deprecated. Please use " + "Array::delete_fragments(ctx, uri, timestamp_start, timestamp_end)"); + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_array_delete_fragments_v2( + ctx.ptr().get(), uri.c_str(), timestamp_start, timestamp_end)); +} + +/** + * @brief Consolidates the fragments of an encrypted array into a single + * fragment. + * + * You must first finalize all queries to the array before consolidation can + * begin (as consolidation temporarily acquires an exclusive lock on the + * array). + * + * **Example:** + * @code{.cpp} + * // Load AES-256 key from disk, environment variable, etc. + * uint8_t key[32] = ...; + * tiledb::Array::consolidate( + * ctx, + * "s3://bucket-name/array-name", + * TILEDB_AES_256_GCM, + * key, + * sizeof(key)); + * @endcode + * + * @param ctx TileDB context + * @param array_uri The URI of the TileDB array to be consolidated. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @param config Configuration parameters for the consolidation. + */ +TILEDB_DEPRECATED +static void consolidate( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length, + Config* const config = nullptr) { + ctx.handle_error(tiledb_array_consolidate_with_key( + ctx.ptr().get(), + uri.c_str(), + encryption_type, + encryption_key, + key_length, + config ? config->ptr().get() : nullptr)); +} + +// clang-format off + /** + * @copybrief Array::consolidate(const Context&,const std::string&,tiledb_encryption_type_t,const void*,uint32_t,const Config&) + * + * See @ref Array::consolidate( + * const Context&, + * const std::string&, + * tiledb_encryption_type_t, + * const void*, + * uint32_t,const Config&) "Array::consolidate" + * + * @param ctx TileDB context + * @param array_uri The URI of the TileDB array to be consolidated. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param config Configuration parameters for the consolidation. + */ +// clang-format on +TILEDB_DEPRECATED +static void consolidate( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const std::string& encryption_key, + Config* const config = nullptr) { + return consolidate( + ctx, + uri, + encryption_type, + encryption_key.data(), + (uint32_t)encryption_key.size(), + config); +} + +/** + * Loads the array schema from an encrypted array. + * + * **Example:** + * @code{.cpp} + * auto schema = tiledb::Array::load_schema(ctx, + * "s3://bucket-name/array-name", key_type, key, key_len); + * @endcode + * + * @param ctx The TileDB context. + * @param uri The array URI. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @return The loaded ArraySchema object. + */ +TILEDB_DEPRECATED +static ArraySchema load_schema( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length) { + tiledb_array_schema_t* schema; + ctx.handle_error(tiledb_array_schema_load_with_key( + ctx.ptr().get(), + uri.c_str(), + encryption_type, + encryption_key, + key_length, + &schema)); + return ArraySchema(ctx, schema); +} + +/** + * @brief Creates a new encrypted TileDB array given an input schema. + * + * **Example:** + * @code{.cpp} + * // Load AES-256 key from disk, environment variable, etc. + * uint8_t key[32] = ...; + * tiledb::Array::create("s3://bucket-name/array-name", schema, + * TILEDB_AES_256_GCM, key, sizeof(key)); + * @endcode + * + * @param uri URI where array will be created. + * @param schema The array schema. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + */ +TILEDB_DEPRECATED +static void create( + const std::string& uri, + const ArraySchema& schema, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length) { + auto& ctx = schema.context(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_array_schema_check(c_ctx, schema.ptr().get())); + ctx.handle_error(tiledb_array_create_with_key( + c_ctx, + uri.c_str(), + schema.ptr().get(), + encryption_type, + encryption_key, + key_length)); +} + +// clang-format off + /** + * @copybrief Array::create(const std::string&,const ArraySchema&,tiledb_encryption_type_t,const void*,uint32_t) + * + * See @ref Array::create(const std::string&,const ArraySchema&,tiledb_encryption_type_t,const void*,uint32_t) "Array::create" + * + * @param uri URI where array will be created. + * @param schema The array schema. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + */ +// clang-format on +TILEDB_DEPRECATED +static void create( + const std::string& uri, + const ArraySchema& schema, + tiledb_encryption_type_t encryption_type, + const std::string& encryption_key) { + return create( + uri, + schema, + encryption_type, + encryption_key.data(), + (uint32_t)encryption_key.size()); +} + +/** + * @brief Consolidates the metadata of an encrypted array. + * + * You must first finalize all queries to the array before consolidation can + * begin (as consolidation temporarily acquires an exclusive lock on the + * array). + * + * **Example:** + * @code{.cpp} + * // Load AES-256 key from disk, environment variable, etc. + * uint8_t key[32] = ...; + * tiledb::Array::consolidate_metadata( + * ctx, + * "s3://bucket-name/array-name", + * TILEDB_AES_256_GCM, + * key, + * sizeof(key)); + * @endcode + * + * @param ctx TileDB context + * @param array_uri The URI of the TileDB array whose + * metadata will be consolidated. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + * @param config Configuration parameters for the consolidation. + */ +TILEDB_DEPRECATED +static void consolidate_metadata( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length, + Config* const config = nullptr) { + Config local_cfg; + Config* config_aux = config; + if (!config_aux) { + config_aux = &local_cfg; + } + + (*config_aux)["sm.consolidation.mode"] = "array_meta"; + consolidate( + ctx, uri, encryption_type, encryption_key, key_length, config_aux); +} + +// clang-format off + /** + * @copybrief Array::consolidate_metadata(const Context&, const std::string&, tiledb_encryption_type_t, const void*,uint32_t, const Config&) + * + * See @ref Array::consolidate_metadata( + * const Context&, + * const std::string&, + * tiledb_encryption_type_t, + * const void*, + * uint32_t,const Config&) "Array::consolidate_metadata" + * + * @param ctx TileDB context + * @param array_uri The URI of the TileDB array whose + * metadata will be consolidated. + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param config Configuration parameters for the consolidation. + */ +// clang-format on +TILEDB_DEPRECATED +static void consolidate_metadata( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const std::string& encryption_key, + Config* const config = nullptr) { + Config local_cfg; + Config* config_aux = config; + if (!config_aux) { + config_aux = &local_cfg; + } + + (*config_aux)["sm.consolidation.mode"] = "array_meta"; + consolidate( + ctx, + uri, + encryption_type, + encryption_key.data(), + (uint32_t)encryption_key.size(), + config_aux); +} diff --git a/tiledb/sm/cpp_api/array_schema.h b/tiledb/sm/cpp_api/array_schema.h index f7d7c3e44a19..16d472db254a 100644 --- a/tiledb/sm/cpp_api/array_schema.h +++ b/tiledb/sm/cpp_api/array_schema.h @@ -133,66 +133,6 @@ class ArraySchema : public Schema { schema_ = std::shared_ptr(schema, deleter_); } - /** - * Loads the schema of an existing encrypted array. - * - * **Example:** - * @code{.cpp} - * // Load AES-256 key from disk, environment variable, etc. - * uint8_t key[32] = ...; - * tiledb::Context ctx; - * tiledb::ArraySchema schema(ctx, "s3://bucket-name/array-name", - * TILEDB_AES_256_GCM, key, sizeof(key)); - * @endcode - * - * @param ctx TileDB context - * @param uri URI of array - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - * @param key_length Length in bytes of the encryption key. - */ - TILEDB_DEPRECATED - ArraySchema( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const void* encryption_key, - uint32_t key_length) - : Schema(ctx) { - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - tiledb_array_schema_t* schema; - ctx.handle_error(tiledb_array_schema_load_with_key( - c_ctx, - uri.c_str(), - encryption_type, - encryption_key, - key_length, - &schema)); - schema_ = std::shared_ptr(schema, deleter_); - } - - /** - * Loads the schema of an existing encrypted array. - * - * @param ctx TileDB context - * @param uri URI of array - * @param encryption_type The encryption type to use. - * @param encryption_key The encryption key to use. - */ - TILEDB_DEPRECATED - ArraySchema( - const Context& ctx, - const std::string& uri, - tiledb_encryption_type_t encryption_type, - const std::string& encryption_key) - : ArraySchema( - ctx, - uri, - encryption_type, - encryption_key.data(), - (uint32_t)encryption_key.size()) { - } - /** * Loads the schema of an existing array with the input C array * schema object. @@ -672,6 +612,13 @@ class ArraySchema : public Schema { return ""; } +/* ********************************* */ +/* DEPRECATED API */ +/* ********************************* */ +#ifndef TILEDB_REMOVE_DEPRECATIONS +#include "array_schema_deprecated.h" +#endif // TILEDB_REMOVE_DEPRECATIONS + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/cpp_api/array_schema_deprecated.h b/tiledb/sm/cpp_api/array_schema_deprecated.h new file mode 100644 index 000000000000..6348cd9a1a34 --- /dev/null +++ b/tiledb/sm/cpp_api/array_schema_deprecated.h @@ -0,0 +1,93 @@ +/** + * @file array_schema_deprecated.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * C++ Deprecated API for TileDB ArraySchema. + */ + +#include "tiledb.h" + +/** + * Loads the schema of an existing encrypted array. + * + * **Example:** + * @code{.cpp} + * // Load AES-256 key from disk, environment variable, etc. + * uint8_t key[32] = ...; + * tiledb::Context ctx; + * tiledb::ArraySchema schema(ctx, "s3://bucket-name/array-name", + * TILEDB_AES_256_GCM, key, sizeof(key)); + * @endcode + * + * @param ctx TileDB context + * @param uri URI of array + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + * @param key_length Length in bytes of the encryption key. + */ +TILEDB_DEPRECATED +ArraySchema( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const void* encryption_key, + uint32_t key_length) + : Schema(ctx) { + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_array_schema_t* schema; + ctx.handle_error(tiledb_array_schema_load_with_key( + c_ctx, + uri.c_str(), + encryption_type, + encryption_key, + key_length, + &schema)); + schema_ = std::shared_ptr(schema, deleter_); +} + +/** + * Loads the schema of an existing encrypted array. + * + * @param ctx TileDB context + * @param uri URI of array + * @param encryption_type The encryption type to use. + * @param encryption_key The encryption key to use. + */ +TILEDB_DEPRECATED +ArraySchema( + const Context& ctx, + const std::string& uri, + tiledb_encryption_type_t encryption_type, + const std::string& encryption_key) + : ArraySchema( + ctx, + uri, + encryption_type, + encryption_key.data(), + (uint32_t)encryption_key.size()) { +} diff --git a/tiledb/sm/cpp_api/core_interface.h b/tiledb/sm/cpp_api/core_interface.h index c74a1f56217e..8011510ce36b 100644 --- a/tiledb/sm/cpp_api/core_interface.h +++ b/tiledb/sm/cpp_api/core_interface.h @@ -42,6 +42,7 @@ namespace tiledb { namespace impl { +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * @cond * Doxygen is disabled for this function, as it conflicts with the C @@ -56,12 +57,13 @@ namespace impl { * @param callback_data Data to pass callback * @return TILEDB_OK for success and TILEDB_ERR for error. */ -TILEDB_EXPORT int tiledb_query_submit_async_func( +TILEDB_DEPRECATED_EXPORT int tiledb_query_submit_async_func( tiledb_ctx_t* ctx, tiledb_query_t* query, void* callback_func, void* callback_data = nullptr) noexcept; /** @endcond */ +#endif // TILEDB_REMOVE_DEPRECATIONS inline size_t type_size(tiledb_datatype_t type) { return tiledb_datatype_size(type); diff --git a/tiledb/sm/cpp_api/query.h b/tiledb/sm/cpp_api/query.h index 22246ab706da..4507c47ff0fd 100644 --- a/tiledb/sm/cpp_api/query.h +++ b/tiledb/sm/cpp_api/query.h @@ -300,50 +300,6 @@ class Query { return query_status(); } - /** - * Submit an async query, with callback. Call returns immediately. - * - * Deprecated, call `submit()` on another thread instead. - * - * @note Same notes apply as `Query::submit()`. - * - * **Example:** - * @code{.cpp} - * // Create query - * tiledb::Query query(...); - * // Submit with callback - * query.submit_async([]() { std::cout << "Callback: query completed.\n"; }); - * @endcode - * - * @param callback Callback function. - */ - template - TILEDB_DEPRECATED void submit_async(const Fn& callback) { - std::function wrapper = [&](void*) { callback(); }; - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb::impl::tiledb_query_submit_async_func( - ctx.ptr().get(), query_.get(), &wrapper, nullptr)); - } - - /** - * Submit an async query, with no callback. Call returns immediately. - * - * Deprecated, call `submit()` on another thread instead. - * - * @note Same notes apply as `Query::submit()`. - * - * **Example:** - * @code{.cpp} - * // Create query - * tiledb::Query query(...); - * // Submit with no callback - * query.submit_async(); - * @endcode - */ - TILEDB_DEPRECATED void submit_async() { - submit_async([]() {}); - } - /** * Flushes all internal state of a query object and finalizes the query. * This is applicable only to global layout writes. It has no effect for @@ -505,362 +461,6 @@ class Query { return elements; } - /** - * Adds a 1D range along a subarray dimension index, in the form - * (start, end, stride). The datatype of the range - * must be the same as the dimension datatype. - * - * **Example:** - * - * @code{.cpp} - * // Set a 1D range on dimension 0, assuming the domain type is int64. - * int64_t start = 10; - * int64_t end = 20; - * // Stride is optional - * subarray.add_range(0, start, end); - * @endcode - * - * @tparam T The dimension datatype - * @param dim_idx The index of the dimension to add the range to. - * @param start The range start to add. - * @param end The range end to add. - * @param stride The range stride to add. - * @return Reference to this Query - */ - template - TILEDB_DEPRECATED Query& add_range( - uint32_t dim_idx, T start, T end, T stride = 0) { - impl::type_check(schema_.domain().dimension(dim_idx).type()); - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_query_add_range( - ctx.ptr().get(), - query_.get(), - dim_idx, - &start, - &end, - (stride == 0) ? nullptr : &stride)); - return *this; - } - - /** - * Adds a 1D range along a subarray dimension name, specified by its name, in - * the form (start, end, stride). The datatype of the range must be the same - * as the dimension datatype. - * - * **Example:** - * - * @code{.cpp} - * // Set a 1D range on dimension "rows", assuming the domain type is int64. - * int64_t start = 10; - * int64_t end = 20; - * const std::string dim_name = "rows"; - * // Stride is optional - * subarray.add_range(dim_name, start, end); - * @endcode - * - * @tparam T The dimension datatype - * @param dim_name The name of the dimension to add the range to. - * @param start The range start to add. - * @param end The range end to add. - * @param stride The range stride to add. - * @return Reference to this Query - */ - template - TILEDB_DEPRECATED Query& add_range( - const std::string& dim_name, T start, T end, T stride = 0) { - impl::type_check(schema_.domain().dimension(dim_name).type()); - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_query_add_range_by_name( - ctx.ptr().get(), - query_.get(), - dim_name.c_str(), - &start, - &end, - (stride == 0) ? nullptr : &stride)); - return *this; - } - - /** - * Adds a 1D string range along a subarray dimension index, in the form - * (start, end). Applicable only to variable-sized dimensions - * - * **Example:** - * - * @code{.cpp} - * // Set a 1D range on variable-sized string dimension "rows" - * std::string start = "ab""; - * std::string end = "d"; - * // Stride is optional - * subarray.add_range(0, start, end); - * @endcode - * - * @tparam T The dimension datatype - * @param dim_idx The index of the dimension to add the range to. - * @param start The range start to add. - * @param end The range end to add. - * @return Reference to this Query - */ - TILEDB_DEPRECATED - Query& add_range( - uint32_t dim_idx, const std::string& start, const std::string& end) { - impl::type_check(schema_.domain().dimension(dim_idx).type()); - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_query_add_range_var( - ctx.ptr().get(), - query_.get(), - dim_idx, - start.c_str(), - start.size(), - end.c_str(), - end.size())); - return *this; - } - - /** - * Adds a 1D string range along a subarray dimension name, in the form (start, - * end). Applicable only to variable-sized dimensions - * - * **Example:** - * - * @code{.cpp} - * // Set a 1D range on variable-sized string dimension "rows" - * std::string start = "ab""; - * std::string end = "d"; - * const std::string dim_name = "rows"; - * // Stride is optional - * subarray.add_range(dim_name, start, end); - * @endcode - * - * @tparam T The dimension datatype - * @param dim_name The name of the dimension to add the range to. - * @param start The range start to add. - * @param end The range end to add. - * @return Reference to this Query - */ - TILEDB_DEPRECATED - Query& add_range( - const std::string& dim_name, - const std::string& start, - const std::string& end) { - impl::type_check(schema_.domain().dimension(dim_name).type()); - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_query_add_range_var_by_name( - ctx.ptr().get(), - query_.get(), - dim_name.c_str(), - start.c_str(), - start.size(), - end.c_str(), - end.size())); - return *this; - } - - /** - * Retrieves the number of ranges for a given dimension index. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_idx = 0; - * uint64_t range_num = query.range_num(dim_idx); - * @endcode - * - * @param dim_idx The dimension index. - * @return The number of ranges. - */ - TILEDB_DEPRECATED - uint64_t range_num(unsigned dim_idx) const { - auto& ctx = ctx_.get(); - uint64_t range_num; - ctx.handle_error(tiledb_query_get_range_num( - ctx.ptr().get(), query_.get(), dim_idx, &range_num)); - return range_num; - } - - /** - * Retrieves the number of ranges for a given dimension name. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_name = "rows"; - * uint64_t range_num = query.range_num(dim_name); - * @endcode - * - * @param dim_name The dimension name. - * @return The number of ranges. - */ - TILEDB_DEPRECATED - uint64_t range_num(const std::string& dim_name) const { - auto& ctx = ctx_.get(); - uint64_t range_num; - ctx.handle_error(tiledb_query_get_range_num_from_name( - ctx.ptr().get(), query_.get(), dim_name.c_str(), &range_num)); - return range_num; - } - - /** - * Retrieves a range for a given dimension index and range id. - * The template datatype must be the same as that of the - * underlying array. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_idx = 0; - * unsigned range_idx = 0; - * auto range = query.range(dim_idx, range_idx); - * @endcode - * - * @tparam T The dimension datatype. - * @param dim_idx The dimension index. - * @param range_idx The range index. - * @return A triplet of the form (start, end, stride). - */ - template - TILEDB_DEPRECATED std::array range( - unsigned dim_idx, uint64_t range_idx) { - impl::type_check(schema_.domain().dimension(dim_idx).type()); - auto& ctx = ctx_.get(); - const void *start, *end, *stride; - ctx.handle_error(tiledb_query_get_range( - ctx.ptr().get(), - query_.get(), - dim_idx, - range_idx, - &start, - &end, - &stride)); - std::array ret = { - {*(const T*)start, - *(const T*)end, - (stride == nullptr) ? 0 : *(const T*)stride}}; - return ret; - } - - /** - * Retrieves a range for a given dimension name and range id. - * The template datatype must be the same as that of the - * underlying array. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_name = "rows"; - * unsigned range_idx = 0; - * auto range = query.range(dim_name, range_idx); - * @endcode - * - * @tparam T The dimension datatype. - * @param dim_name The dimension name. - * @param range_idx The range index. - * @return A triplet of the form (start, end, stride). - */ - template - TILEDB_DEPRECATED std::array range( - const std::string& dim_name, uint64_t range_idx) { - impl::type_check(schema_.domain().dimension(dim_name).type()); - auto& ctx = ctx_.get(); - const void *start, *end, *stride; - ctx.handle_error(tiledb_query_get_range_from_name( - ctx.ptr().get(), - query_.get(), - dim_name.c_str(), - range_idx, - &start, - &end, - &stride)); - std::array ret = { - {*(const T*)start, - *(const T*)end, - (stride == nullptr) ? 0 : *(const T*)stride}}; - return ret; - } - - /** - * Retrieves a range for a given variable length string dimension index and - * range id. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_idx = 0; - * unsigned range_idx = 0; - * std::array range = query.range(dim_idx, range_idx); - * @endcode - * - * @param dim_idx The dimension index. - * @param range_idx The range index. - * @return A pair of the form (start, end). - */ - TILEDB_DEPRECATED - std::array range(unsigned dim_idx, uint64_t range_idx) { - impl::type_check(schema_.domain().dimension(dim_idx).type()); - auto& ctx = ctx_.get(); - uint64_t start_size, end_size; - ctx.handle_error(tiledb_query_get_range_var_size( - ctx.ptr().get(), - query_.get(), - dim_idx, - range_idx, - &start_size, - &end_size)); - - std::string start; - start.resize(start_size); - std::string end; - end.resize(end_size); - ctx.handle_error(tiledb_query_get_range_var( - ctx.ptr().get(), query_.get(), dim_idx, range_idx, &start[0], &end[0])); - std::array ret = {{std::move(start), std::move(end)}}; - return ret; - } - - /** - * Retrieves a range for a given variable length string dimension name and - * range id. - * - * **Example:** - * - * @code{.cpp} - * unsigned dim_name = "rows"; - * unsigned range_idx = 0; - * std::array range = query.range(dim_name, range_idx); - * @endcode - * - * @param dim_name The dimension name. - * @param range_idx The range index. - * @return A pair of the form (start, end). - */ - TILEDB_DEPRECATED - std::array range( - const std::string& dim_name, uint64_t range_idx) { - impl::type_check(schema_.domain().dimension(dim_name).type()); - auto& ctx = ctx_.get(); - uint64_t start_size, end_size; - ctx.handle_error(tiledb_query_get_range_var_size_from_name( - ctx.ptr().get(), - query_.get(), - dim_name.c_str(), - range_idx, - &start_size, - &end_size)); - - std::string start; - start.resize(start_size); - std::string end; - end.resize(end_size); - ctx.handle_error(tiledb_query_get_range_var_from_name( - ctx.ptr().get(), - query_.get(), - dim_name.c_str(), - range_idx, - &start[0], - &end[0])); - std::array ret = {{std::move(start), std::move(end)}}; - return ret; - } - /** * Retrieves the estimated result size for a fixed-size attribute. * This is an estimate and may not be sufficient to read all results for the @@ -1018,112 +618,6 @@ class Query { return std::make_pair(t1, t2); } - /** - * The query set_subarray function has been deprecated. - * See the documentation for Subarray::set_subarray(), or use other - * Subarray provided APIs. - * Consult the current documentation for more information. - * - * Sets a subarray, defined in the order dimensions were added. - * Coordinates are inclusive. For the case of writes, this is meaningful only - * for dense arrays, and specifically dense writes. - * - * @note `set_subarray(std::vector)` is preferred as it is safer. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_READ); - * int subarray[] = {0, 3, 0, 3}; - * Query query(ctx, array); - * query.set_subarray(subarray, 4); - * @endcode - * - * @tparam T Type of array domain. - * @param pairs Subarray pointer defined as an array of [start, stop] values - * per dimension. - * @param size The number of subarray elements. - */ - template - TILEDB_DEPRECATED Query& set_subarray(const T* pairs, uint64_t size) { - impl::type_check(schema_.domain().type()); - auto& ctx = ctx_.get(); - if (size != schema_.domain().ndim() * 2) { - throw SchemaMismatch( - "Subarray should have num_dims * 2 values: (low, high) for each " - "dimension."); - } - ctx.handle_error( - tiledb_query_set_subarray(ctx.ptr().get(), query_.get(), pairs)); - return *this; - } - - /** - * Sets a subarray, defined in the order dimensions were added. - * Coordinates are inclusive. For the case of writes, this is meaningful only - * for dense arrays, and specifically dense writes. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_READ); - * std::vector subarray = {0, 3, 0, 3}; - * Query query(ctx, array); - * query.set_subarray(subarray); - * @endcode - * - * @tparam Vec Vector datatype. Should always be a vector of the domain type. - * @param pairs The subarray defined as a vector of [start, stop] coordinates - * per dimension. - */ - template - TILEDB_DEPRECATED Query& set_subarray(const Vec& pairs) { - return set_subarray(pairs.data(), pairs.size()); - } - - /** - * Sets a subarray, defined in the order dimensions were added. - * Coordinates are inclusive. For the case of writes, this is meaningful only - * for dense arrays, and specifically dense writes. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_READ); - * Query query(ctx, array); - * query.set_subarray({0, 3, 0, 3}); - * @endcode - * - * @tparam T Type of array domain. - * @param pairs List of [start, stop] coordinates per dimension. - */ - template - TILEDB_DEPRECATED Query& set_subarray(const std::initializer_list& l) { - return set_subarray(std::vector(l)); - } - - /** - * Sets a subarray, defined in the order dimensions were added. - * Coordinates are inclusive. - * - * @note set_subarray(std::vector) is preferred and avoids an extra copy. - * - * @tparam T Type of array domain. - * @param pairs The subarray defined as pairs of [start, stop] per dimension. - */ - template - TILEDB_DEPRECATED Query& set_subarray( - const std::vector>& pairs) { - std::vector buf; - buf.reserve(pairs.size() * 2); - std::for_each( - pairs.begin(), pairs.end(), [&buf](const std::array& p) { - buf.push_back(p[0]); - buf.push_back(p[1]); - }); - return set_subarray(buf); - } - /** * Prepare a query with the contents of a subarray. * @@ -1178,61 +672,14 @@ class Query { } /** - * Set the coordinate buffer. - * - * The coordinate buffer has been deprecated. Set the coordinates for - * each individual dimension with the `set_buffer` API. Consult the current - * documentation for more information. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * // Write to points (0,1) and (2,3) in a 2D array with int domain. - * int coords[] = {0, 1, 2, 3}; - * Query query(ctx, array); - * query.set_layout(TILEDB_UNORDERED).set_coordinates(coords, 4); - * @endcode - * - * @note set_coordinates(std::vector) is preferred as it is safer. - * - * @tparam T Type of array domain. - * @param buf Coordinate array buffer pointer - * @param size The number of elements in the coordinate array buffer - * **/ - template - TILEDB_DEPRECATED Query& set_coordinates(T* buf, uint64_t size) { - impl::type_check(schema_.domain().type()); - return set_data_buffer("__coords", buf, size); - } - - /** - * Set the coordinate buffer for unordered queries. - * - * The coordinate buffer has been deprecated. Set the coordinates for - * each individual dimension with the `set_buffer` API. Consult the current - * documentation for more information. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * // Write to points (0,1) and (2,3) in a 2D array with int domain. - * std::vector coords = {0, 1, 2, 3}; - * Query query(ctx, array); - * query.set_layout(TILEDB_UNORDERED).set_coordinates(coords); - * @endcode + * Sets the data for a fixed/var-sized attribute/dimension. * - * @tparam Vec Vector datatype. Should always be a vector of the domain type. - * @param buf Coordinate vector - * **/ - template - TILEDB_DEPRECATED Query& set_coordinates(Vec& buf) { - return set_coordinates(buf.data(), buf.size()); - } - - /** - * Sets a buffer for a fixed-sized attribute/dimension. + * The caller owns the buffer provided and is responsible for freeing the + * memory associated with it. For writes, the buffer holds values to be + * written which can be freed at any time after query completion. For reads, + * the buffer is allocated by the caller and will contain data read by the + * query after completion. The freeing of this memory is up to the caller once + * they are done referencing the read data. * * **Example:** * @code{.cpp} @@ -1240,10 +687,11 @@ class Query { * tiledb::Array array(ctx, array_name, TILEDB_WRITE); * int data_a1[] = {0, 1, 2, 3}; * Query query(ctx, array); - * query.set_buffer("a1", data_a1, 4); + * query.set_data_buffer("a1", data_a1, 4); * @endcode * - * @note set_buffer(std::string, std::vector) is preferred as it is safer. + * @note set_data_buffer(std::string, std::vector) is preferred as it is + * safer. * * @tparam T Attribute/Dimension value type * @param name Attribute/Dimension name @@ -1252,27 +700,15 @@ class Query { * @param nelements Number of array elements **/ template - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, T* buff, uint64_t nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (name != "__coords" && !is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); - else if (name == "__coords") - impl::type_check(schema_.domain().type()); + Query& set_data_buffer(const std::string& name, T* buff, uint64_t nelements) { + // Check we have the correct type. + impl::type_check(field_type(name)); return set_data_buffer(name, buff, nelements, sizeof(T)); } /** - * Sets a buffer for a fixed-sized attribute/dimension. + * Sets the data for a fixed/var-sized attribute/dimension. * * **Example:** * @code{.cpp} @@ -1280,7 +716,7 @@ class Query { * tiledb::Array array(ctx, array_name, TILEDB_WRITE); * std::vector data_a1 = {0, 1, 2, 3}; * Query query(ctx, array); - * query.set_buffer("a1", data_a1); + * query.set_data_buffer("a1", data_a1); * @endcode * * @tparam T Attribute/Dimension value type @@ -1288,291 +724,12 @@ class Query { * @param buf Buffer vector with elements of the attribute/dimension type. **/ template - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, std::vector& buf) { - return set_data_buffer(name, buf.data(), buf.size(), sizeof(T)); + Query& set_data_buffer(const std::string& name, std::vector& buf) { + return set_data_buffer(name, buf.data(), buf.size()); } /** - * Sets a buffer for a fixed-sized attribute/dimension. - * - * @note This unsafe version does not perform type checking; the given buffer - * is assumed to be the correct type, and the size of an element in the given - * buffer is assumed to be the size of the datatype of the attribute. - * - * @param name Attribute/Dimension name - * @param buff Buffer array pointer with elements of the attribute type. - * @param nelements Number of array elements in buffer - **/ - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, void* buff, uint64_t nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (name != "__coords" && !is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - - // Compute element size (in bytes). - size_t element_size = 0; - if (name == "__coords") - element_size = tiledb_datatype_size(schema_.domain().type()); - else if (is_attr) - element_size = tiledb_datatype_size(schema_.attribute(name).type()); - else if (is_dim) - element_size = - tiledb_datatype_size(schema_.domain().dimension(name).type()); - - return set_data_buffer(name, buff, nelements, element_size); - } - - /** - * Sets a buffer for a variable-sized attribute/dimension. - * - * **Example:** - * - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * int data_a1[] = {0, 1, 2, 3}; - * uint64_t offsets_a1[] = {0, 8}; - * Query query(ctx, array); - * query.set_buffer("a1", offsets_a1, 2, data_a1, 4); - * @endcode - * - * @note set_buffer(std::string, std::vector, std::vector) is preferred as it - * is safer. - * - * @tparam T Attribute/Dimension value type - * @param name Attribute/Dimension name - * @param offsets Offsets array pointer where a new element begins in the data - * buffer. - * @param offsets_nelements Number of elements in offsets buffer. - * @param data Buffer array pointer with elements of the attribute type. - * For variable sized attributes, the buffer should be flattened. - * @param data_nelements Number of array elements in data buffer. - **/ - template - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, - uint64_t* offsets, - uint64_t offset_nelements, - T* data, - uint64_t data_nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); - - set_data_buffer(name, data, data_nelements, sizeof(T)); - set_offsets_buffer(name, offsets, offset_nelements); - return *this; - } - - /** - * Sets a buffer for a variable-sized attribute/dimension. - * - * @note This unsafe version does not perform type checking; the given buffer - * is assumed to be the correct type, and the size of an element in the given - * buffer is assumed to be the size of the datatype of the attribute. - * - * @param name Attribute/Dimension name - * @param offsets Offsets array pointer where a new element begins in the data - * buffer. - * @param offsets_nelements Number of elements in offsets buffer. - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements in data buffer. - **/ - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, - uint64_t* offsets, - uint64_t offset_nelements, - void* data, - uint64_t data_nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - - // Compute element size (in bytes). - auto type = is_attr ? schema_.attribute(name).type() : - schema_.domain().dimension(name).type(); - size_t element_size = tiledb_datatype_size(type); - - set_data_buffer(name, data, data_nelements, element_size); - set_offsets_buffer(name, offsets, offset_nelements); - return *this; - } - - /** - * Sets a buffer for a variable-sized attribute/dimension. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * std::vector data_a1 = {0, 1, 2, 3}; - * std::vector offsets_a1 = {0, 8}; - * Query query(ctx, array); - * query.set_buffer("a1", offsets_a1, data_a1); - * @endcode - * - * @tparam T Attribute/Dimension value type - * @param name Attribute/Dimension name - * @param offsets Offsets where a new element begins in the data buffer. - * @param data Buffer vector with elements of the attribute type. - * For variable sized attributes, the buffer should be flattened. E.x. - * an attribute of type std::string should have a buffer Vec type of - * std::string, where the values of each cell are concatenated. - **/ - template - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, - std::vector& offsets, - std::vector& data) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); - - set_data_buffer(name, data.data(), data.size(), sizeof(T)); - set_offsets_buffer(name, offsets.data(), offsets.size()); - return *this; - } - - /** - * Sets a buffer for a variable-sized attribute/dimension. - * - * @tparam T Attribute/Dimension value type - * @param attr Attribute/Dimension name - * @param buf Pair of offset, data buffers - **/ - template - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, - std::pair, std::vector>& buf) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); - - set_data_buffer(name, buf.second); - set_offsets_buffer(name, buf.first); - return *this; - } - - /** - * Sets a buffer for a string-typed variable-sized attribute/dimension. - * - * @param name Attribute/Dimension name - * @param offsets Offsets where a new element begins in the data buffer. - * @param data Pre-allocated string buffer. - **/ - TILEDB_DEPRECATED Query& set_buffer( - const std::string& name, - std::vector& offsets, - std::string& data) { - // Checks - auto is_attr = schema_.has_attribute(name); - auto is_dim = schema_.domain().has_dimension(name); - if (!is_attr && !is_dim) - throw TileDBError( - std::string("Cannot set buffer; Attribute/Dimension '") + name + - "' does not exist"); - else if (is_attr) - impl::type_check(schema_.attribute(name).type()); - else if (is_dim) - impl::type_check(schema_.domain().dimension(name).type()); - - set_data_buffer(name, &data[0], data.size(), sizeof(char)); - set_offsets_buffer(name, offsets.data(), offsets.size()); - return *this; - } - - /** - * Sets the data for a fixed/var-sized attribute/dimension. - * - * The caller owns the buffer provided and is responsible for freeing the - * memory associated with it. For writes, the buffer holds values to be - * written which can be freed at any time after query completion. For reads, - * the buffer is allocated by the caller and will contain data read by the - * query after completion. The freeing of this memory is up to the caller once - * they are done referencing the read data. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * int data_a1[] = {0, 1, 2, 3}; - * Query query(ctx, array); - * query.set_data_buffer("a1", data_a1, 4); - * @endcode - * - * @note set_data_buffer(std::string, std::vector) is preferred as it is - * safer. - * - * @tparam T Attribute/Dimension value type - * @param name Attribute/Dimension name - * @param buff Buffer array pointer with elements of the - * attribute/dimension type. - * @param nelements Number of array elements - **/ - template - Query& set_data_buffer(const std::string& name, T* buff, uint64_t nelements) { - // Check we have the correct type. - impl::type_check(field_type(name)); - - return set_data_buffer(name, buff, nelements, sizeof(T)); - } - - /** - * Sets the data for a fixed/var-sized attribute/dimension. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * std::vector data_a1 = {0, 1, 2, 3}; - * Query query(ctx, array); - * query.set_data_buffer("a1", data_a1); - * @endcode - * - * @tparam T Attribute/Dimension value type - * @param name Attribute/Dimension name - * @param buf Buffer vector with elements of the attribute/dimension type. - **/ - template - Query& set_data_buffer(const std::string& name, std::vector& buf) { - return set_data_buffer(name, buf.data(), buf.size()); - } - - /** - * Sets the data for a fixed/var-sized attribute/dimension. + * Sets the data for a fixed/var-sized attribute/dimension. * * The caller owns the buffer provided and is responsible for freeing the * memory associated with it. For writes, the buffer holds values to be @@ -1744,333 +901,6 @@ class Query { name, validity_bytemap.data(), validity_bytemap.size()); } - /** - * Sets a buffer for a fixed-sized, nullable attribute. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * int data_a1[] = {0, 1, 2, 3}; - * uint8_t validity_bytemap[] = {1, 1, 0, 1}; - * Query query(ctx, array); - * query.set_buffer("a1", data_a1, 4, validity_bytemap, 4); - * @endcode - * - * @note set_buffer_nullable(std::string, std::vector, std::vector) - * is preferred as it is safer. - * - * @tparam T Attribute value type - * @param name Attribute name - * @param data Buffer array pointer with elements of the - * attribute type. - * @param data_nelements Number of array elements in `data` - * @param validity_bytemap The validity bytemap buffer. - * @param validity_bytemap_nelements The number of values within - * `validity_bytemap_nelements` - **/ - template - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - T* data, - uint64_t data_nelements, - uint8_t* validity_bytemap, - uint64_t validity_bytemap_nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - else - impl::type_check(schema_.attribute(name).type()); - - set_data_buffer(name, data, data_nelements); - set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); - return *this; - } - - /** - * Sets a buffer for a fixed-sized, nullable attribute. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * std::vector data_a1 = {0, 1, 2, 3}; - * std::vector validity_bytemap = {1, 1, 0, 1}; - * Query query(ctx, array); - * query.set_buffer("a1", data_a1, validity_bytemap); - * @endcode - * - * @tparam T Attribute value type - * @param name Attribute name - * @param buf Buffer vector with elements of the attribute/dimension type. - * @param validity_bytemap Buffer vector with elements of the attribute - * validity values. - **/ - template - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - std::vector& buf, - std::vector& validity_bytemap) { - set_data_buffer(name, buf.data(), buf.size(), sizeof(T)); - set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); - return *this; - } - - /** - * Sets a buffer for a fixed-sized, nullable attribute. - * - * @note This unsafe version does not perform type checking; the given buffer - * is assumed to be the correct type, and the size of an element in the given - * buffer is assumed to be the size of the datatype of the attribute. - * - * @param name Attribute name - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements in buffer - * @param validity_bytemap The validity bytemap buffer. - * @param validity_bytemap_nelements The number of values within - * `validity_bytemap_nelements` - **/ - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - void* data, - uint64_t data_nelements, - uint8_t* validity_bytemap, - uint64_t validity_bytemap_nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - - // Compute element size (in bytes). - size_t element_size = tiledb_datatype_size(schema_.attribute(name).type()); - - set_data_buffer(name, data, data_nelements, element_size); - set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); - return *this; - } - - /** - * Sets a buffer for a variable-sized, nullable attribute. - * - * @note This unsafe version does not perform type checking; the given buffer - * is assumed to be the correct type, and the size of an element in the given - * buffer is assumed to be the size of the datatype of the attribute. - * - * @param name Attribute name - * @param offsets Offsets array pointer where a new element begins in the data - * buffer. - * @param offsets_nelements Number of elements in offsets buffer. - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements in data buffer. - * @param validity_bytemap The validity bytemap buffer. - * @param validity_bytemap_nelements The number of values within - * `validity_bytemap_nelements` - **/ - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - uint64_t* offsets, - uint64_t offset_nelements, - void* data, - uint64_t data_nelements, - uint8_t* validity_bytemap, - uint64_t validity_bytemap_nelements) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - - // Compute element size (in bytes). - auto type = schema_.attribute(name).type(); - size_t element_size = tiledb_datatype_size(type); - - set_data_buffer(name, data, data_nelements, element_size); - set_offsets_buffer(name, offsets, offset_nelements); - set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); - return *this; - } - - /** - * Sets a buffer for a variable-sized, nullable attribute. - * - * **Example:** - * @code{.cpp} - * tiledb::Context ctx; - * tiledb::Array array(ctx, array_name, TILEDB_WRITE); - * std::vector data_a1 = {0, 1, 2, 3}; - * std::vector offsets_a1 = {0, 8}; - * std::vector validity_bytemap = {1, 1, 0, 1}; - * Query query(ctx, array); - * query.set_buffer("a1", offsets_a1, data_a1, validity_bytemap); - * @endcode - * - * @tparam T Attribute value type - * @param name Attribute name - * @param offsets Offsets where a new element begins in the data buffer. - * @param data Buffer vector with elements of the attribute type. - * For variable sized attributes, the buffer should be flattened. E.x. - * an attribute of type std::string should have a buffer Vec type of - * std::string, where the values of each cell are concatenated. - * @param validity_bytemap Buffer vector with elements of the attribute - * validity values. - **/ - template - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - std::vector& offsets, - std::vector& data, - std::vector& validity_bytemap) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - else - impl::type_check(schema_.attribute(name).type()); - - set_data_buffer(name, data.data(), data.size(), sizeof(T)); - set_offsets_buffer(name, offsets.data(), offsets.size()); - set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); - return *this; - } - - /** - * Sets a buffer for a variable-sized, nullable attribute. - * - * @tparam T Attribute value type - * @param attr Attribute name - * @param buf Pair of offset, data, validity bytemap buffers - **/ - template - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - std::tuple, std::vector, std::vector>& - buf) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - impl::type_check(schema_.attribute(name).type()); - - set_data_buffer(name, std::get<1>(buf)); - set_offsets_buffer(name, std::get<0>(buf)); - set_validity_buffer(name, std::get<2>(buf)); - return *this; - } - - /** - * Sets a buffer for a string-typed variable-sized, nullable attribute. - * - * @param name Attribute name - * @param offsets Offsets where a new element begins in the data buffer. - * @param data Pre-allocated string buffer. - * @param validity_bytemap Buffer vector with elements of the attribute - * validity values. - **/ - TILEDB_DEPRECATED Query& set_buffer_nullable( - const std::string& name, - std::vector& offsets, - std::string& data, - std::vector& validity_bytemap) { - // Checks - auto is_attr = schema_.has_attribute(name); - if (!is_attr) - throw TileDBError( - std::string("Cannot set buffer; Attribute '") + name + - "' does not exist"); - else - impl::type_check(schema_.attribute(name).type()); - - set_data_buffer(name, &data[0], data.size(), sizeof(char)); - set_offsets_buffer(name, offsets.data(), offsets.size()); - set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); - return *this; - } - - /** - * Gets a buffer for a fixed-sized attribute/dimension. - * - * @param name Attribute/dimension name - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements. - * @param element_size Size of array elements (in bytes). - **/ - TILEDB_DEPRECATED Query& get_buffer( - const std::string& name, - void** data, - uint64_t* data_nelements, - uint64_t* element_size) { - auto ctx = ctx_.get(); - uint64_t* data_nbytes = nullptr; - auto elem_size_iter = element_sizes_.find(name); - if (elem_size_iter == element_sizes_.end()) { - throw TileDBError( - "[TileDB::C++API] Error: No buffer set for attribute '" + name + - "'!"); - } - - ctx.handle_error(tiledb_query_get_data_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); - - assert(*data_nbytes % elem_size_iter->second == 0); - - *data_nelements = (*data_nbytes) / elem_size_iter->second; - *element_size = elem_size_iter->second; - - return *this; - } - - /** - * Gets a buffer for a var-sized attribute/dimension. - * - * @param name Attribute/dimension name - * @param offsets Offsets array pointer with elements of uint64_t type. - * @param offsets_nelements Number of array elements. - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements. - * @param element_size Size of array elements (in bytes). - **/ - TILEDB_DEPRECATED Query& get_buffer( - const std::string& name, - uint64_t** offsets, - uint64_t* offsets_nelements, - void** data, - uint64_t* data_nelements, - uint64_t* element_size) { - auto ctx = ctx_.get(); - uint64_t* offsets_nbytes = nullptr; - uint64_t* data_nbytes = nullptr; - auto elem_size_iter = element_sizes_.find(name); - if (elem_size_iter == element_sizes_.end()) { - throw TileDBError( - "[TileDB::C++API] Error: No buffer set for attribute '" + name + - "'!"); - } - - ctx.handle_error(tiledb_query_get_data_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); - ctx.handle_error(tiledb_query_get_offsets_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), offsets, &offsets_nbytes)); - - assert(*data_nbytes % elem_size_iter->second == 0); - assert(*offsets_nbytes % sizeof(uint64_t) == 0); - - *data_nelements = (*data_nbytes) / elem_size_iter->second; - *offsets_nelements = (*offsets_nbytes) / sizeof(uint64_t); - *element_size = elem_size_iter->second; - - return *this; - } - /** * Retrieves the data buffer of a fixed/var-sized attribute/dimension. * @@ -2152,107 +982,6 @@ class Query { return *this; } - /** - * Gets a buffer for a fixed-sized, nullable attribute. - * - * @param name Attribute name - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements. - * @param data_element_size Size of array elements (in bytes). - * @param validity_bytemap Buffer array pointer with elements of the - * attribute validity values. - * @param validity_bytemap_nelements Number of validity bytemap elements. - **/ - TILEDB_DEPRECATED Query& get_buffer_nullable( - const std::string& name, - void** data, - uint64_t* data_nelements, - uint64_t* data_element_size, - uint8_t** validity_bytemap, - uint64_t* validity_bytemap_nelements) { - auto ctx = ctx_.get(); - uint64_t* data_nbytes = nullptr; - uint64_t* validity_bytemap_nbytes = nullptr; - auto elem_size_iter = element_sizes_.find(name); - if (elem_size_iter == element_sizes_.end()) { - throw TileDBError( - "[TileDB::C++API] Error: No buffer set for attribute '" + name + - "'!"); - } - - ctx.handle_error(tiledb_query_get_data_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); - ctx.handle_error(tiledb_query_get_validity_buffer( - ctx.ptr().get(), - query_.get(), - name.c_str(), - validity_bytemap, - &validity_bytemap_nbytes)); - - assert(*data_nbytes % elem_size_iter->second == 0); - - *data_nelements = *data_nbytes / elem_size_iter->second; - *data_element_size = elem_size_iter->second; - *validity_bytemap_nelements = *validity_bytemap_nbytes / sizeof(uint8_t); - - return *this; - } - - /** - * Gets a buffer for a var-sized, nullable attribute. - * - * @param name Attribute name - * @param offsets Offsets array pointer with elements of uint64_t type. - * @param offsets_nelements Number of array elements. - * @param data Buffer array pointer with elements of the attribute type. - * @param data_nelements Number of array elements. - * @param element_size Size of array elements (in bytes). - * @param validity_bytemap Buffer array pointer with elements of the - * attribute validity values. - * @param validity_bytemap_nelements Number of validity bytemap elements. - **/ - TILEDB_DEPRECATED Query& get_buffer_nullable( - const std::string& name, - uint64_t** offsets, - uint64_t* offsets_nelements, - void** data, - uint64_t* data_nelements, - uint64_t* element_size, - uint8_t** validity_bytemap, - uint64_t* validity_bytemap_nelements) { - auto ctx = ctx_.get(); - uint64_t* offsets_nbytes = nullptr; - uint64_t* data_nbytes = nullptr; - uint64_t* validity_bytemap_nbytes = nullptr; - auto elem_size_iter = element_sizes_.find(name); - if (elem_size_iter == element_sizes_.end()) { - throw TileDBError( - "[TileDB::C++API] Error: No buffer set for attribute '" + name + - "'!"); - } - - ctx.handle_error(tiledb_query_get_data_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); - ctx.handle_error(tiledb_query_get_offsets_buffer( - ctx.ptr().get(), query_.get(), name.c_str(), offsets, &offsets_nbytes)); - ctx.handle_error(tiledb_query_get_validity_buffer( - ctx.ptr().get(), - query_.get(), - name.c_str(), - validity_bytemap, - &validity_bytemap_nbytes)); - - assert(*data_nbytes % elem_size_iter->second == 0); - assert(*offsets_nbytes % sizeof(uint64_t) == 0); - - *data_nelements = (*data_nbytes) / elem_size_iter->second; - *offsets_nelements = (*offsets_nbytes) / sizeof(uint64_t); - *element_size = elem_size_iter->second; - *validity_bytemap_nelements = *validity_bytemap_nbytes / sizeof(uint8_t); - - return *this; - } - /** Returns a JSON-formatted string of the stats. */ std::string stats() { auto ctx = ctx_.get(); @@ -2330,6 +1059,13 @@ class Query { return array_.get(); } +/* ********************************* */ +/* DEPRECATED API */ +/* ********************************* */ +#ifndef TILEDB_REMOVE_DEPRECATIONS +#include "query_deprecated.h" +#endif // TILEDB_REMOVE_DEPRECATIONS + private: friend class QueryExperimental; @@ -2423,6 +1159,7 @@ class Query { return cell_val_num == TILEDB_VAR_NUM; } +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Sets a buffer for a fixed-sized attribute. * @@ -2489,6 +1226,7 @@ class Query { &std::get<0>(buff_sizes_[attr]))); return *this; } +#endif // TILEDB_REMOVE_DEPRECATIONS /** * Sets the data for a fixed/var-sized attribute/dimension. @@ -2525,6 +1263,7 @@ class Query { return *this; } +#ifndef TILEDB_REMOVE_DEPRECATIONS /** * Sets a buffer for a nullable, fixed-sized attribute. * @@ -2614,6 +1353,7 @@ class Query { &std::get<2>(buff_sizes_[attr]))); return *this; } +#endif // TILEDB_REMOVE_DEPRECATIONS }; /* ********************************* */ diff --git a/tiledb/sm/cpp_api/query_deprecated.h b/tiledb/sm/cpp_api/query_deprecated.h new file mode 100644 index 000000000000..87a16ce13dc4 --- /dev/null +++ b/tiledb/sm/cpp_api/query_deprecated.h @@ -0,0 +1,1298 @@ +/** + * @file query_deprecated.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * C++ Deprecated API for TileDB Query. + */ + +#include "tiledb.h" + +/** + * Submit an async query, with callback. Call returns immediately. + * + * Deprecated, call `submit()` on another thread instead. + * + * @note Same notes apply as `Query::submit()`. + * + * **Example:** + * @code{.cpp} + * // Create query + * tiledb::Query query(...); + * // Submit with callback + * query.submit_async([]() { std::cout << "Callback: query completed.\n"; }); + * @endcode + * + * @param callback Callback function. + */ +template +TILEDB_DEPRECATED void submit_async(const Fn& callback) { + std::function wrapper = [&](void*) { callback(); }; + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb::impl::tiledb_query_submit_async_func( + ctx.ptr().get(), query_.get(), &wrapper, nullptr)); +} + +/** + * Submit an async query, with no callback. Call returns immediately. + * + * Deprecated, call `submit()` on another thread instead. + * + * @note Same notes apply as `Query::submit()`. + * + * **Example:** + * @code{.cpp} + * // Create query + * tiledb::Query query(...); + * // Submit with no callback + * query.submit_async(); + * @endcode + */ +TILEDB_DEPRECATED void submit_async() { + submit_async([]() {}); +} + +/** + * Adds a 1D range along a subarray dimension index, in the form + * (start, end, stride). The datatype of the range + * must be the same as the dimension datatype. + * + * **Example:** + * + * @code{.cpp} + * // Set a 1D range on dimension 0, assuming the domain type is int64. + * int64_t start = 10; + * int64_t end = 20; + * // Stride is optional + * subarray.add_range(0, start, end); + * @endcode + * + * @tparam T The dimension datatype + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @param stride The range stride to add. + * @return Reference to this Query + */ +template +TILEDB_DEPRECATED Query& add_range( + uint32_t dim_idx, T start, T end, T stride = 0) { + impl::type_check(schema_.domain().dimension(dim_idx).type()); + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_query_add_range( + ctx.ptr().get(), + query_.get(), + dim_idx, + &start, + &end, + (stride == 0) ? nullptr : &stride)); + return *this; +} + +/** + * Adds a 1D range along a subarray dimension name, specified by its name, in + * the form (start, end, stride). The datatype of the range must be the same + * as the dimension datatype. + * + * **Example:** + * + * @code{.cpp} + * // Set a 1D range on dimension "rows", assuming the domain type is int64. + * int64_t start = 10; + * int64_t end = 20; + * const std::string dim_name = "rows"; + * // Stride is optional + * subarray.add_range(dim_name, start, end); + * @endcode + * + * @tparam T The dimension datatype + * @param dim_name The name of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @param stride The range stride to add. + * @return Reference to this Query + */ +template +TILEDB_DEPRECATED Query& add_range( + const std::string& dim_name, T start, T end, T stride = 0) { + impl::type_check(schema_.domain().dimension(dim_name).type()); + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_query_add_range_by_name( + ctx.ptr().get(), + query_.get(), + dim_name.c_str(), + &start, + &end, + (stride == 0) ? nullptr : &stride)); + return *this; +} + +/** + * Adds a 1D string range along a subarray dimension index, in the form + * (start, end). Applicable only to variable-sized dimensions + * + * **Example:** + * + * @code{.cpp} + * // Set a 1D range on variable-sized string dimension "rows" + * std::string start = "ab""; + * std::string end = "d"; + * // Stride is optional + * subarray.add_range(0, start, end); + * @endcode + * + * @tparam T The dimension datatype + * @param dim_idx The index of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this Query + */ +TILEDB_DEPRECATED +Query& add_range( + uint32_t dim_idx, const std::string& start, const std::string& end) { + impl::type_check(schema_.domain().dimension(dim_idx).type()); + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_query_add_range_var( + ctx.ptr().get(), + query_.get(), + dim_idx, + start.c_str(), + start.size(), + end.c_str(), + end.size())); + return *this; +} + +/** + * Adds a 1D string range along a subarray dimension name, in the form (start, + * end). Applicable only to variable-sized dimensions + * + * **Example:** + * + * @code{.cpp} + * // Set a 1D range on variable-sized string dimension "rows" + * std::string start = "ab""; + * std::string end = "d"; + * const std::string dim_name = "rows"; + * // Stride is optional + * subarray.add_range(dim_name, start, end); + * @endcode + * + * @tparam T The dimension datatype + * @param dim_name The name of the dimension to add the range to. + * @param start The range start to add. + * @param end The range end to add. + * @return Reference to this Query + */ +TILEDB_DEPRECATED +Query& add_range( + const std::string& dim_name, + const std::string& start, + const std::string& end) { + impl::type_check(schema_.domain().dimension(dim_name).type()); + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_query_add_range_var_by_name( + ctx.ptr().get(), + query_.get(), + dim_name.c_str(), + start.c_str(), + start.size(), + end.c_str(), + end.size())); + return *this; +} + +/** + * Retrieves the number of ranges for a given dimension index. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_idx = 0; + * uint64_t range_num = query.range_num(dim_idx); + * @endcode + * + * @param dim_idx The dimension index. + * @return The number of ranges. + */ +TILEDB_DEPRECATED +uint64_t range_num(unsigned dim_idx) const { + auto& ctx = ctx_.get(); + uint64_t range_num; + ctx.handle_error(tiledb_query_get_range_num( + ctx.ptr().get(), query_.get(), dim_idx, &range_num)); + return range_num; +} + +/** + * Retrieves the number of ranges for a given dimension name. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_name = "rows"; + * uint64_t range_num = query.range_num(dim_name); + * @endcode + * + * @param dim_name The dimension name. + * @return The number of ranges. + */ +TILEDB_DEPRECATED +uint64_t range_num(const std::string& dim_name) const { + auto& ctx = ctx_.get(); + uint64_t range_num; + ctx.handle_error(tiledb_query_get_range_num_from_name( + ctx.ptr().get(), query_.get(), dim_name.c_str(), &range_num)); + return range_num; +} + +/** + * Retrieves a range for a given dimension index and range id. + * The template datatype must be the same as that of the + * underlying array. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_idx = 0; + * unsigned range_idx = 0; + * auto range = query.range(dim_idx, range_idx); + * @endcode + * + * @tparam T The dimension datatype. + * @param dim_idx The dimension index. + * @param range_idx The range index. + * @return A triplet of the form (start, end, stride). + */ +template +TILEDB_DEPRECATED std::array range(unsigned dim_idx, uint64_t range_idx) { + impl::type_check(schema_.domain().dimension(dim_idx).type()); + auto& ctx = ctx_.get(); + const void *start, *end, *stride; + ctx.handle_error(tiledb_query_get_range( + ctx.ptr().get(), + query_.get(), + dim_idx, + range_idx, + &start, + &end, + &stride)); + std::array ret = { + {*(const T*)start, + *(const T*)end, + (stride == nullptr) ? 0 : *(const T*)stride}}; + return ret; +} + +/** + * Retrieves a range for a given dimension name and range id. + * The template datatype must be the same as that of the + * underlying array. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_name = "rows"; + * unsigned range_idx = 0; + * auto range = query.range(dim_name, range_idx); + * @endcode + * + * @tparam T The dimension datatype. + * @param dim_name The dimension name. + * @param range_idx The range index. + * @return A triplet of the form (start, end, stride). + */ +template +TILEDB_DEPRECATED std::array range( + const std::string& dim_name, uint64_t range_idx) { + impl::type_check(schema_.domain().dimension(dim_name).type()); + auto& ctx = ctx_.get(); + const void *start, *end, *stride; + ctx.handle_error(tiledb_query_get_range_from_name( + ctx.ptr().get(), + query_.get(), + dim_name.c_str(), + range_idx, + &start, + &end, + &stride)); + std::array ret = { + {*(const T*)start, + *(const T*)end, + (stride == nullptr) ? 0 : *(const T*)stride}}; + return ret; +} + +/** + * Retrieves a range for a given variable length string dimension index and + * range id. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_idx = 0; + * unsigned range_idx = 0; + * std::array range = query.range(dim_idx, range_idx); + * @endcode + * + * @param dim_idx The dimension index. + * @param range_idx The range index. + * @return A pair of the form (start, end). + */ +TILEDB_DEPRECATED +std::array range(unsigned dim_idx, uint64_t range_idx) { + impl::type_check(schema_.domain().dimension(dim_idx).type()); + auto& ctx = ctx_.get(); + uint64_t start_size, end_size; + ctx.handle_error(tiledb_query_get_range_var_size( + ctx.ptr().get(), + query_.get(), + dim_idx, + range_idx, + &start_size, + &end_size)); + + std::string start; + start.resize(start_size); + std::string end; + end.resize(end_size); + ctx.handle_error(tiledb_query_get_range_var( + ctx.ptr().get(), query_.get(), dim_idx, range_idx, &start[0], &end[0])); + std::array ret = {{std::move(start), std::move(end)}}; + return ret; +} + +/** + * Retrieves a range for a given variable length string dimension name and + * range id. + * + * **Example:** + * + * @code{.cpp} + * unsigned dim_name = "rows"; + * unsigned range_idx = 0; + * std::array range = query.range(dim_name, range_idx); + * @endcode + * + * @param dim_name The dimension name. + * @param range_idx The range index. + * @return A pair of the form (start, end). + */ +TILEDB_DEPRECATED +std::array range( + const std::string& dim_name, uint64_t range_idx) { + impl::type_check(schema_.domain().dimension(dim_name).type()); + auto& ctx = ctx_.get(); + uint64_t start_size, end_size; + ctx.handle_error(tiledb_query_get_range_var_size_from_name( + ctx.ptr().get(), + query_.get(), + dim_name.c_str(), + range_idx, + &start_size, + &end_size)); + + std::string start; + start.resize(start_size); + std::string end; + end.resize(end_size); + ctx.handle_error(tiledb_query_get_range_var_from_name( + ctx.ptr().get(), + query_.get(), + dim_name.c_str(), + range_idx, + &start[0], + &end[0])); + std::array ret = {{std::move(start), std::move(end)}}; + return ret; +} + +/** + * The query set_subarray function has been deprecated. + * See the documentation for Subarray::set_subarray(), or use other + * Subarray provided APIs. + * Consult the current documentation for more information. + * + * Sets a subarray, defined in the order dimensions were added. + * Coordinates are inclusive. For the case of writes, this is meaningful only + * for dense arrays, and specifically dense writes. + * + * @note `set_subarray(std::vector)` is preferred as it is safer. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_READ); + * int subarray[] = {0, 3, 0, 3}; + * Query query(ctx, array); + * query.set_subarray(subarray, 4); + * @endcode + * + * @tparam T Type of array domain. + * @param pairs Subarray pointer defined as an array of [start, stop] values + * per dimension. + * @param size The number of subarray elements. + */ +template +TILEDB_DEPRECATED Query& set_subarray(const T* pairs, uint64_t size) { + impl::type_check(schema_.domain().type()); + auto& ctx = ctx_.get(); + if (size != schema_.domain().ndim() * 2) { + throw SchemaMismatch( + "Subarray should have num_dims * 2 values: (low, high) for each " + "dimension."); + } + ctx.handle_error( + tiledb_query_set_subarray(ctx.ptr().get(), query_.get(), pairs)); + return *this; +} + +/** + * Sets a subarray, defined in the order dimensions were added. + * Coordinates are inclusive. For the case of writes, this is meaningful only + * for dense arrays, and specifically dense writes. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_READ); + * std::vector subarray = {0, 3, 0, 3}; + * Query query(ctx, array); + * query.set_subarray(subarray); + * @endcode + * + * @tparam Vec Vector datatype. Should always be a vector of the domain type. + * @param pairs The subarray defined as a vector of [start, stop] coordinates + * per dimension. + */ +template +TILEDB_DEPRECATED Query& set_subarray(const Vec& pairs) { + return set_subarray(pairs.data(), pairs.size()); +} + +/** + * Sets a subarray, defined in the order dimensions were added. + * Coordinates are inclusive. For the case of writes, this is meaningful only + * for dense arrays, and specifically dense writes. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_READ); + * Query query(ctx, array); + * query.set_subarray({0, 3, 0, 3}); + * @endcode + * + * @tparam T Type of array domain. + * @param pairs List of [start, stop] coordinates per dimension. + */ +template +TILEDB_DEPRECATED Query& set_subarray(const std::initializer_list& l) { + return set_subarray(std::vector(l)); +} + +/** + * Sets a subarray, defined in the order dimensions were added. + * Coordinates are inclusive. + * + * @note set_subarray(std::vector) is preferred and avoids an extra copy. + * + * @tparam T Type of array domain. + * @param pairs The subarray defined as pairs of [start, stop] per dimension. + */ +template +TILEDB_DEPRECATED Query& set_subarray( + const std::vector>& pairs) { + std::vector buf; + buf.reserve(pairs.size() * 2); + std::for_each(pairs.begin(), pairs.end(), [&buf](const std::array& p) { + buf.push_back(p[0]); + buf.push_back(p[1]); + }); + return set_subarray(buf); +} + +/** + * Set the coordinate buffer. + * + * The coordinate buffer has been deprecated. Set the coordinates for + * each individual dimension with the `set_buffer` API. Consult the current + * documentation for more information. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * // Write to points (0,1) and (2,3) in a 2D array with int domain. + * int coords[] = {0, 1, 2, 3}; + * Query query(ctx, array); + * query.set_layout(TILEDB_UNORDERED).set_coordinates(coords, 4); + * @endcode + * + * @note set_coordinates(std::vector) is preferred as it is safer. + * + * @tparam T Type of array domain. + * @param buf Coordinate array buffer pointer + * @param size The number of elements in the coordinate array buffer + * **/ +template +TILEDB_DEPRECATED Query& set_coordinates(T* buf, uint64_t size) { + impl::type_check(schema_.domain().type()); + return set_data_buffer("__coords", buf, size); +} + +/** + * Set the coordinate buffer for unordered queries. + * + * The coordinate buffer has been deprecated. Set the coordinates for + * each individual dimension with the `set_buffer` API. Consult the current + * documentation for more information. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * // Write to points (0,1) and (2,3) in a 2D array with int domain. + * std::vector coords = {0, 1, 2, 3}; + * Query query(ctx, array); + * query.set_layout(TILEDB_UNORDERED).set_coordinates(coords); + * @endcode + * + * @tparam Vec Vector datatype. Should always be a vector of the domain type. + * @param buf Coordinate vector + * **/ +template +TILEDB_DEPRECATED Query& set_coordinates(Vec& buf) { + return set_coordinates(buf.data(), buf.size()); +} + +/** + * Sets a buffer for a fixed-sized attribute/dimension. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * int data_a1[] = {0, 1, 2, 3}; + * Query query(ctx, array); + * query.set_buffer("a1", data_a1, 4); + * @endcode + * + * @note set_buffer(std::string, std::vector) is preferred as it is safer. + * + * @tparam T Attribute/Dimension value type + * @param name Attribute/Dimension name + * @param buff Buffer array pointer with elements of the + * attribute/dimension type. + * @param nelements Number of array elements + **/ +template +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, T* buff, uint64_t nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (name != "__coords" && !is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + else if (is_attr) + impl::type_check(schema_.attribute(name).type()); + else if (is_dim) + impl::type_check(schema_.domain().dimension(name).type()); + else if (name == "__coords") + impl::type_check(schema_.domain().type()); + + return set_data_buffer(name, buff, nelements, sizeof(T)); +} + +/** + * Sets a buffer for a fixed-sized attribute/dimension. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * std::vector data_a1 = {0, 1, 2, 3}; + * Query query(ctx, array); + * query.set_buffer("a1", data_a1); + * @endcode + * + * @tparam T Attribute/Dimension value type + * @param name Attribute/Dimension name + * @param buf Buffer vector with elements of the attribute/dimension type. + **/ +template +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, std::vector& buf) { + return set_data_buffer(name, buf.data(), buf.size(), sizeof(T)); +} + +/** + * Sets a buffer for a fixed-sized attribute/dimension. + * + * @note This unsafe version does not perform type checking; the given buffer + * is assumed to be the correct type, and the size of an element in the given + * buffer is assumed to be the size of the datatype of the attribute. + * + * @param name Attribute/Dimension name + * @param buff Buffer array pointer with elements of the attribute type. + * @param nelements Number of array elements in buffer + **/ +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, void* buff, uint64_t nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (name != "__coords" && !is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + + // Compute element size (in bytes). + size_t element_size = 0; + if (name == "__coords") + element_size = tiledb_datatype_size(schema_.domain().type()); + else if (is_attr) + element_size = tiledb_datatype_size(schema_.attribute(name).type()); + else if (is_dim) + element_size = + tiledb_datatype_size(schema_.domain().dimension(name).type()); + + return set_data_buffer(name, buff, nelements, element_size); +} + +/** + * Sets a buffer for a variable-sized attribute/dimension. + * + * **Example:** + * + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * int data_a1[] = {0, 1, 2, 3}; + * uint64_t offsets_a1[] = {0, 8}; + * Query query(ctx, array); + * query.set_buffer("a1", offsets_a1, 2, data_a1, 4); + * @endcode + * + * @note set_buffer(std::string, std::vector, std::vector) is preferred as it + * is safer. + * + * @tparam T Attribute/Dimension value type + * @param name Attribute/Dimension name + * @param offsets Offsets array pointer where a new element begins in the data + * buffer. + * @param offsets_nelements Number of elements in offsets buffer. + * @param data Buffer array pointer with elements of the attribute type. + * For variable sized attributes, the buffer should be flattened. + * @param data_nelements Number of array elements in data buffer. + **/ +template +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, + uint64_t* offsets, + uint64_t offset_nelements, + T* data, + uint64_t data_nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (!is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + else if (is_attr) + impl::type_check(schema_.attribute(name).type()); + else if (is_dim) + impl::type_check(schema_.domain().dimension(name).type()); + + set_data_buffer(name, data, data_nelements, sizeof(T)); + set_offsets_buffer(name, offsets, offset_nelements); + return *this; +} + +/** + * Sets a buffer for a variable-sized attribute/dimension. + * + * @note This unsafe version does not perform type checking; the given buffer + * is assumed to be the correct type, and the size of an element in the given + * buffer is assumed to be the size of the datatype of the attribute. + * + * @param name Attribute/Dimension name + * @param offsets Offsets array pointer where a new element begins in the data + * buffer. + * @param offsets_nelements Number of elements in offsets buffer. + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements in data buffer. + **/ +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, + uint64_t* offsets, + uint64_t offset_nelements, + void* data, + uint64_t data_nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (!is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + + // Compute element size (in bytes). + auto type = is_attr ? schema_.attribute(name).type() : + schema_.domain().dimension(name).type(); + size_t element_size = tiledb_datatype_size(type); + + set_data_buffer(name, data, data_nelements, element_size); + set_offsets_buffer(name, offsets, offset_nelements); + return *this; +} + +/** + * Sets a buffer for a variable-sized attribute/dimension. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * std::vector data_a1 = {0, 1, 2, 3}; + * std::vector offsets_a1 = {0, 8}; + * Query query(ctx, array); + * query.set_buffer("a1", offsets_a1, data_a1); + * @endcode + * + * @tparam T Attribute/Dimension value type + * @param name Attribute/Dimension name + * @param offsets Offsets where a new element begins in the data buffer. + * @param data Buffer vector with elements of the attribute type. + * For variable sized attributes, the buffer should be flattened. E.x. + * an attribute of type std::string should have a buffer Vec type of + * std::string, where the values of each cell are concatenated. + **/ +template +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, + std::vector& offsets, + std::vector& data) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (!is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + else if (is_attr) + impl::type_check(schema_.attribute(name).type()); + else if (is_dim) + impl::type_check(schema_.domain().dimension(name).type()); + + set_data_buffer(name, data.data(), data.size(), sizeof(T)); + set_offsets_buffer(name, offsets.data(), offsets.size()); + return *this; +} + +/** + * Sets a buffer for a variable-sized attribute/dimension. + * + * @tparam T Attribute/Dimension value type + * @param attr Attribute/Dimension name + * @param buf Pair of offset, data buffers + **/ +template +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, + std::pair, std::vector>& buf) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (!is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + else if (is_attr) + impl::type_check(schema_.attribute(name).type()); + else if (is_dim) + impl::type_check(schema_.domain().dimension(name).type()); + + set_data_buffer(name, buf.second); + set_offsets_buffer(name, buf.first); + return *this; +} + +/** + * Sets a buffer for a string-typed variable-sized attribute/dimension. + * + * @param name Attribute/Dimension name + * @param offsets Offsets where a new element begins in the data buffer. + * @param data Pre-allocated string buffer. + **/ +TILEDB_DEPRECATED Query& set_buffer( + const std::string& name, + std::vector& offsets, + std::string& data) { + // Checks + auto is_attr = schema_.has_attribute(name); + auto is_dim = schema_.domain().has_dimension(name); + if (!is_attr && !is_dim) + throw TileDBError( + std::string("Cannot set buffer; Attribute/Dimension '") + name + + "' does not exist"); + else if (is_attr) + impl::type_check(schema_.attribute(name).type()); + else if (is_dim) + impl::type_check(schema_.domain().dimension(name).type()); + + set_data_buffer(name, &data[0], data.size(), sizeof(char)); + set_offsets_buffer(name, offsets.data(), offsets.size()); + return *this; +} + +/** + * Sets a buffer for a fixed-sized, nullable attribute. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * int data_a1[] = {0, 1, 2, 3}; + * uint8_t validity_bytemap[] = {1, 1, 0, 1}; + * Query query(ctx, array); + * query.set_buffer("a1", data_a1, 4, validity_bytemap, 4); + * @endcode + * + * @note set_buffer_nullable(std::string, std::vector, std::vector) + * is preferred as it is safer. + * + * @tparam T Attribute value type + * @param name Attribute name + * @param data Buffer array pointer with elements of the + * attribute type. + * @param data_nelements Number of array elements in `data` + * @param validity_bytemap The validity bytemap buffer. + * @param validity_bytemap_nelements The number of values within + * `validity_bytemap_nelements` + **/ +template +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + T* data, + uint64_t data_nelements, + uint8_t* validity_bytemap, + uint64_t validity_bytemap_nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + else + impl::type_check(schema_.attribute(name).type()); + + set_data_buffer(name, data, data_nelements); + set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); + return *this; +} + +/** + * Sets a buffer for a fixed-sized, nullable attribute. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * std::vector data_a1 = {0, 1, 2, 3}; + * std::vector validity_bytemap = {1, 1, 0, 1}; + * Query query(ctx, array); + * query.set_buffer("a1", data_a1, validity_bytemap); + * @endcode + * + * @tparam T Attribute value type + * @param name Attribute name + * @param buf Buffer vector with elements of the attribute/dimension type. + * @param validity_bytemap Buffer vector with elements of the attribute + * validity values. + **/ +template +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + std::vector& buf, + std::vector& validity_bytemap) { + set_data_buffer(name, buf.data(), buf.size(), sizeof(T)); + set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); + return *this; +} + +/** + * Sets a buffer for a fixed-sized, nullable attribute. + * + * @note This unsafe version does not perform type checking; the given buffer + * is assumed to be the correct type, and the size of an element in the given + * buffer is assumed to be the size of the datatype of the attribute. + * + * @param name Attribute name + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements in buffer + * @param validity_bytemap The validity bytemap buffer. + * @param validity_bytemap_nelements The number of values within + * `validity_bytemap_nelements` + **/ +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + void* data, + uint64_t data_nelements, + uint8_t* validity_bytemap, + uint64_t validity_bytemap_nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + + // Compute element size (in bytes). + size_t element_size = tiledb_datatype_size(schema_.attribute(name).type()); + + set_data_buffer(name, data, data_nelements, element_size); + set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); + return *this; +} + +/** + * Sets a buffer for a variable-sized, nullable attribute. + * + * @note This unsafe version does not perform type checking; the given buffer + * is assumed to be the correct type, and the size of an element in the given + * buffer is assumed to be the size of the datatype of the attribute. + * + * @param name Attribute name + * @param offsets Offsets array pointer where a new element begins in the data + * buffer. + * @param offsets_nelements Number of elements in offsets buffer. + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements in data buffer. + * @param validity_bytemap The validity bytemap buffer. + * @param validity_bytemap_nelements The number of values within + * `validity_bytemap_nelements` + **/ +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + uint64_t* offsets, + uint64_t offset_nelements, + void* data, + uint64_t data_nelements, + uint8_t* validity_bytemap, + uint64_t validity_bytemap_nelements) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + + // Compute element size (in bytes). + auto type = schema_.attribute(name).type(); + size_t element_size = tiledb_datatype_size(type); + + set_data_buffer(name, data, data_nelements, element_size); + set_offsets_buffer(name, offsets, offset_nelements); + set_validity_buffer(name, validity_bytemap, validity_bytemap_nelements); + return *this; +} + +/** + * Sets a buffer for a variable-sized, nullable attribute. + * + * **Example:** + * @code{.cpp} + * tiledb::Context ctx; + * tiledb::Array array(ctx, array_name, TILEDB_WRITE); + * std::vector data_a1 = {0, 1, 2, 3}; + * std::vector offsets_a1 = {0, 8}; + * std::vector validity_bytemap = {1, 1, 0, 1}; + * Query query(ctx, array); + * query.set_buffer("a1", offsets_a1, data_a1, validity_bytemap); + * @endcode + * + * @tparam T Attribute value type + * @param name Attribute name + * @param offsets Offsets where a new element begins in the data buffer. + * @param data Buffer vector with elements of the attribute type. + * For variable sized attributes, the buffer should be flattened. E.x. + * an attribute of type std::string should have a buffer Vec type of + * std::string, where the values of each cell are concatenated. + * @param validity_bytemap Buffer vector with elements of the attribute + * validity values. + **/ +template +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + std::vector& offsets, + std::vector& data, + std::vector& validity_bytemap) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + else + impl::type_check(schema_.attribute(name).type()); + + set_data_buffer(name, data.data(), data.size(), sizeof(T)); + set_offsets_buffer(name, offsets.data(), offsets.size()); + set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); + return *this; +} + +/** + * Sets a buffer for a variable-sized, nullable attribute. + * + * @tparam T Attribute value type + * @param attr Attribute name + * @param buf Pair of offset, data, validity bytemap buffers + **/ +template +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + std::tuple, std::vector, std::vector>& + buf) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + impl::type_check(schema_.attribute(name).type()); + + set_data_buffer(name, std::get<1>(buf)); + set_offsets_buffer(name, std::get<0>(buf)); + set_validity_buffer(name, std::get<2>(buf)); + return *this; +} + +/** + * Sets a buffer for a string-typed variable-sized, nullable attribute. + * + * @param name Attribute name + * @param offsets Offsets where a new element begins in the data buffer. + * @param data Pre-allocated string buffer. + * @param validity_bytemap Buffer vector with elements of the attribute + * validity values. + **/ +TILEDB_DEPRECATED Query& set_buffer_nullable( + const std::string& name, + std::vector& offsets, + std::string& data, + std::vector& validity_bytemap) { + // Checks + auto is_attr = schema_.has_attribute(name); + if (!is_attr) + throw TileDBError( + std::string("Cannot set buffer; Attribute '") + name + + "' does not exist"); + else + impl::type_check(schema_.attribute(name).type()); + + set_data_buffer(name, &data[0], data.size(), sizeof(char)); + set_offsets_buffer(name, offsets.data(), offsets.size()); + set_validity_buffer(name, validity_bytemap.data(), validity_bytemap.size()); + return *this; +} + +/** + * Gets a buffer for a fixed-sized attribute/dimension. + * + * @param name Attribute/dimension name + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements. + * @param element_size Size of array elements (in bytes). + **/ +TILEDB_DEPRECATED Query& get_buffer( + const std::string& name, + void** data, + uint64_t* data_nelements, + uint64_t* element_size) { + auto ctx = ctx_.get(); + uint64_t* data_nbytes = nullptr; + auto elem_size_iter = element_sizes_.find(name); + if (elem_size_iter == element_sizes_.end()) { + throw TileDBError( + "[TileDB::C++API] Error: No buffer set for attribute '" + name + "'!"); + } + + ctx.handle_error(tiledb_query_get_data_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); + + assert(*data_nbytes % elem_size_iter->second == 0); + + *data_nelements = (*data_nbytes) / elem_size_iter->second; + *element_size = elem_size_iter->second; + + return *this; +} + +/** + * Gets a buffer for a var-sized attribute/dimension. + * + * @param name Attribute/dimension name + * @param offsets Offsets array pointer with elements of uint64_t type. + * @param offsets_nelements Number of array elements. + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements. + * @param element_size Size of array elements (in bytes). + **/ +TILEDB_DEPRECATED Query& get_buffer( + const std::string& name, + uint64_t** offsets, + uint64_t* offsets_nelements, + void** data, + uint64_t* data_nelements, + uint64_t* element_size) { + auto ctx = ctx_.get(); + uint64_t* offsets_nbytes = nullptr; + uint64_t* data_nbytes = nullptr; + auto elem_size_iter = element_sizes_.find(name); + if (elem_size_iter == element_sizes_.end()) { + throw TileDBError( + "[TileDB::C++API] Error: No buffer set for attribute '" + name + "'!"); + } + + ctx.handle_error(tiledb_query_get_data_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); + ctx.handle_error(tiledb_query_get_offsets_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), offsets, &offsets_nbytes)); + + assert(*data_nbytes % elem_size_iter->second == 0); + assert(*offsets_nbytes % sizeof(uint64_t) == 0); + + *data_nelements = (*data_nbytes) / elem_size_iter->second; + *offsets_nelements = (*offsets_nbytes) / sizeof(uint64_t); + *element_size = elem_size_iter->second; + + return *this; +} + +/** + * Gets a buffer for a fixed-sized, nullable attribute. + * + * @param name Attribute name + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements. + * @param data_element_size Size of array elements (in bytes). + * @param validity_bytemap Buffer array pointer with elements of the + * attribute validity values. + * @param validity_bytemap_nelements Number of validity bytemap elements. + **/ +TILEDB_DEPRECATED Query& get_buffer_nullable( + const std::string& name, + void** data, + uint64_t* data_nelements, + uint64_t* data_element_size, + uint8_t** validity_bytemap, + uint64_t* validity_bytemap_nelements) { + auto ctx = ctx_.get(); + uint64_t* data_nbytes = nullptr; + uint64_t* validity_bytemap_nbytes = nullptr; + auto elem_size_iter = element_sizes_.find(name); + if (elem_size_iter == element_sizes_.end()) { + throw TileDBError( + "[TileDB::C++API] Error: No buffer set for attribute '" + name + "'!"); + } + + ctx.handle_error(tiledb_query_get_data_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); + ctx.handle_error(tiledb_query_get_validity_buffer( + ctx.ptr().get(), + query_.get(), + name.c_str(), + validity_bytemap, + &validity_bytemap_nbytes)); + + assert(*data_nbytes % elem_size_iter->second == 0); + + *data_nelements = *data_nbytes / elem_size_iter->second; + *data_element_size = elem_size_iter->second; + *validity_bytemap_nelements = *validity_bytemap_nbytes / sizeof(uint8_t); + + return *this; +} + +/** + * Gets a buffer for a var-sized, nullable attribute. + * + * @param name Attribute name + * @param offsets Offsets array pointer with elements of uint64_t type. + * @param offsets_nelements Number of array elements. + * @param data Buffer array pointer with elements of the attribute type. + * @param data_nelements Number of array elements. + * @param element_size Size of array elements (in bytes). + * @param validity_bytemap Buffer array pointer with elements of the + * attribute validity values. + * @param validity_bytemap_nelements Number of validity bytemap elements. + **/ +TILEDB_DEPRECATED Query& get_buffer_nullable( + const std::string& name, + uint64_t** offsets, + uint64_t* offsets_nelements, + void** data, + uint64_t* data_nelements, + uint64_t* element_size, + uint8_t** validity_bytemap, + uint64_t* validity_bytemap_nelements) { + auto ctx = ctx_.get(); + uint64_t* offsets_nbytes = nullptr; + uint64_t* data_nbytes = nullptr; + uint64_t* validity_bytemap_nbytes = nullptr; + auto elem_size_iter = element_sizes_.find(name); + if (elem_size_iter == element_sizes_.end()) { + throw TileDBError( + "[TileDB::C++API] Error: No buffer set for attribute '" + name + "'!"); + } + + ctx.handle_error(tiledb_query_get_data_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), data, &data_nbytes)); + ctx.handle_error(tiledb_query_get_offsets_buffer( + ctx.ptr().get(), query_.get(), name.c_str(), offsets, &offsets_nbytes)); + ctx.handle_error(tiledb_query_get_validity_buffer( + ctx.ptr().get(), + query_.get(), + name.c_str(), + validity_bytemap, + &validity_bytemap_nbytes)); + + assert(*data_nbytes % elem_size_iter->second == 0); + assert(*offsets_nbytes % sizeof(uint64_t) == 0); + + *data_nelements = (*data_nbytes) / elem_size_iter->second; + *offsets_nelements = (*offsets_nbytes) / sizeof(uint64_t); + *element_size = elem_size_iter->second; + *validity_bytemap_nelements = *validity_bytemap_nbytes / sizeof(uint8_t); + + return *this; +} From 4dc7d0987d62a5c358c3feddaf81e62aa8f26a6c Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:05:32 -0400 Subject: [PATCH 301/456] Migrate Array APIs out of StorageManager: get_non_empty_domain. (#4892) Migrate `Array` APIs out of `StorageManager`: `array_get_non_empty_domain`. --- [sc-44988] --- TYPE: NO_HISTORY DESC: Migrate Array APIs out of StorageManager: array_get_non_empty_domain. --- tiledb/sm/array/array.cc | 59 ++++++++++++++- tiledb/sm/array/array.h | 22 +++++- tiledb/sm/c_api/tiledb.cc | 5 +- tiledb/sm/c_api/tiledb.h | 2 +- tiledb/sm/storage_manager/storage_manager.cc | 72 ++----------------- .../storage_manager_canonical.h | 33 +-------- 6 files changed, 90 insertions(+), 103 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 395be3468b8b..a7a389b4cafb 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1061,13 +1061,68 @@ Metadata& Array::metadata() { const NDRange Array::non_empty_domain() { if (!non_empty_domain_computed()) { - // Compute non-empty domain throw_if_not_ok(compute_non_empty_domain()); } - return loaded_non_empty_domain(); } +void Array::non_empty_domain(NDRange* domain, bool* is_empty) { + if (domain == nullptr) { + throw std::invalid_argument("[non_empty_domain] Domain object is null"); + } + if (is_empty == nullptr) { + throw std::invalid_argument("[non_empty_domain] is_empty* is null"); + } + + if (!is_open()) { + throw ArrayException("[non_empty_domain] Array is not open"); + } + + QueryType query_type{get_query_type()}; + if (query_type != QueryType::READ) { + throw ArrayException("[non_empty_domain] Array not opened for reads"); + } + + *domain = non_empty_domain(); + *is_empty = domain->empty(); +} + +void Array::non_empty_domain(void* domain, bool* is_empty) { + if (domain == nullptr) { + throw std::invalid_argument("[non_empty_domain] Domain object is null"); + } + if (is_empty == nullptr) { + throw std::invalid_argument("[non_empty_domain] is_empty* is null"); + } + + if (!array_schema_latest().domain().all_dims_same_type()) { + throw ArrayException( + "[non_empty_domain] Function non-applicable to arrays with " + "heterogenous dimensions"); + } + + if (!array_schema_latest().domain().all_dims_fixed()) { + throw ArrayException( + "[non_empty_domain] Function non-applicable to arrays with " + "variable-sized dimensions"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + if (*is_empty) { + return; + } + + const auto& array_schema = array_schema_latest(); + auto dim_num = array_schema.dim_num(); + auto domain_c = (unsigned char*)domain; + uint64_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&domain_c[offset], dom[d].data(), dom[d].size()); + offset += dom[d].size(); + } +} + bool Array::serialize_non_empty_domain() const { auto found = false; auto serialize_ned_array_open = false; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 3b7286d709cc..e4598fd1e2e6 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -754,10 +754,30 @@ class Array { /** * Returns the non-empty domain of the opened array. If the non_empty_domain - * has not been computed or loaded it will be loaded first + * has not been computed or loaded it will be loaded first. */ const NDRange non_empty_domain(); + /** + * Retrieves the non-empty domain from the opened array. + * This is the union of the non-empty domains of the array fragments. + * + * @param domain The domain to be retrieved. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain(NDRange* domain, bool* is_empty); + + /** + * Retrieves the non-empty domain from the opened array. + * This is the union of the non-empty domains of the array fragments. + * + * @pre The array dimensions are numeric and of the same type. + * + * @param domain The domain to be retrieved. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain(void* domain, bool* is_empty); + /** * Retrieves the array metadata object that is already loaded. If it's not yet * loaded it will be empty. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index b975c5069660..e626d1d8871b 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2795,10 +2795,7 @@ int32_t tiledb_array_get_non_empty_domain( return TILEDB_ERR; bool is_empty_b; - - throw_if_not_ok(ctx->storage_manager()->array_get_non_empty_domain( - array->array_.get(), domain, &is_empty_b)); - + array->array_->non_empty_domain(domain, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index 0d9781a0276b..b7f9e382fffa 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index b8aa69e34678..2cd868d688a8 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -497,64 +497,6 @@ Status StorageManagerCanonical::array_upgrade_version( return Status::Ok(); } -Status StorageManagerCanonical::array_get_non_empty_domain( - Array* array, NDRange* domain, bool* is_empty) { - if (domain == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Domain object is null")); - - if (array == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array object is null")); - - if (!array->is_open()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array is not open")); - - QueryType query_type{array->get_query_type()}; - if (query_type != QueryType::READ) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array not opened for reads")); - - *domain = array->non_empty_domain(); - *is_empty = domain->empty(); - - return Status::Ok(); -} - -Status StorageManagerCanonical::array_get_non_empty_domain( - Array* array, void* domain, bool* is_empty) { - if (array == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array object is null")); - - if (!array->array_schema_latest().domain().all_dims_same_type()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Function non-applicable to arrays with " - "heterogenous dimensions")); - - if (!array->array_schema_latest().domain().all_dims_fixed()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Function non-applicable to arrays with " - "variable-sized dimensions")); - - NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); - if (*is_empty) - return Status::Ok(); - - const auto& array_schema = array->array_schema_latest(); - auto dim_num = array_schema.dim_num(); - auto domain_c = (unsigned char*)domain; - uint64_t offset = 0; - for (unsigned d = 0; d < dim_num; ++d) { - std::memcpy(&domain_c[offset], dom[d].data(), dom[d].size()); - offset += dom[d].size(); - } - - return Status::Ok(); -} - Status StorageManagerCanonical::array_get_non_empty_domain_from_index( Array* array, unsigned idx, void* domain, bool* is_empty) { // Check if array is open - must be open for reads @@ -578,7 +520,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_from_index( } NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); if (*is_empty) return Status::Ok(); @@ -599,7 +541,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_from_name( "Cannot get non-empty domain; Array is not open")); NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); const auto& array_schema = array->array_schema_latest(); auto& array_domain{array_schema.domain()}; @@ -647,7 +589,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_index( } NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); if (*is_empty) { *start_size = 0; *end_size = 0; @@ -672,7 +614,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_name( "Cannot get non-empty domain; Invalid dimension name")); NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); const auto& array_schema = array->array_schema_latest(); auto& array_domain{array_schema.domain()}; @@ -722,7 +664,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_from_index( } NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); if (*is_empty) return Status::Ok(); @@ -743,7 +685,7 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_from_name( "Cannot get non-empty domain; Invalid dimension name")); NDRange dom; - RETURN_NOT_OK(array_get_non_empty_domain(array, &dom, is_empty)); + array->non_empty_domain(&dom, is_empty); const auto& array_schema = array->array_schema_latest(); auto& array_domain{array_schema.domain()}; diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 2a255ebd2394..06c34262f314 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -61,8 +61,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class OpenedArray; @@ -243,31 +242,6 @@ class StorageManagerCanonical { */ Status array_upgrade_version(const URI& uri, const Config& config); - /** - * Retrieves the non-empty domain from an array. This is the union of the - * non-empty domains of the array fragments. - * - * @param array An open array object (must be already open). - * @param domain The domain to be retrieved. - * @param is_empty `true` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain( - Array* array, NDRange* domain, bool* is_empty); - - /** - * Retrieves the non-empty domain from an array. This is the union of the - * non-empty domains of the array fragments. - * - * @param array An open array object (must be already open). - * @param domain The domain to be retrieved. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain(Array* array, void* domain, bool* is_empty); - /** * Retrieves the non-empty domain from an array on the given dimension. * This is the union of the non-empty domains of the array fragments. @@ -719,7 +693,6 @@ class StorageManagerCanonical { Status set_default_tags(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_STORAGE_MANAGER_CANONICAL_H From 2335eb4b0195f5c5a5df2069ce344f0a3558650b Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Mon, 29 Apr 2024 20:06:07 +0300 Subject: [PATCH 302/456] Port last batch of tests to REST-CI (#4911) This PR finally replaces all calls to `submit_query_wrapper` with `query.submit` calls by adapting the last few tests. It also removes the wrapper code as it's not useful anymore. This doesn't mean that we are fully covered and don't need to port any more tests. It's just completes the first milestone which was removing the serialization wrappers from our tests. All issues found are tracked here: https://app.shortcut.com/tiledb-inc/story/40489/issues-found-while-running-tiledb-unit-core-tests-against-rest-cloud-server --- TYPE: NO_HISTORY DESC: Port last batch of tests to REST-CI --- test/src/unit-cppapi-deletes.cc | 412 +++++------ test/src/unit-cppapi-filter.cc | 336 ++------- test/src/unit-cppapi-hilbert.cc | 639 ++++++------------ test/src/unit-cppapi-nullable.cc | 70 +- .../unit-cppapi-partial-attribute-write.cc | 212 +----- test/src/unit-cppapi-subarray.cc | 36 +- test/src/unit-cppapi-updates.cc | 108 +-- test/src/unit-cppapi-var-offsets.cc | 639 +++++++----------- test/src/unit-empty-var-length.cc | 167 +---- test/src/unit-tile-metadata.cc | 78 +-- test/support/src/helpers.cc | 461 +------------ test/support/src/helpers.h | 105 +-- test/support/src/vfs_helpers.cc | 5 +- .../test/unit_capi_query_aggregate.cc | 152 +---- 14 files changed, 869 insertions(+), 2551 deletions(-) diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 409a349fcd1b..69354f3ce025 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -33,6 +33,7 @@ #include #include "test/support/src/ast_helpers.h" #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/api/c_api/context/context_api_internal.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array_directory.h" @@ -50,27 +51,26 @@ using namespace tiledb; using namespace tiledb::test; struct DeletesFx { - // Constants. const char* SPARSE_ARRAY_NAME = "test_deletes_array"; const std::string GROUP_NAME = "test_deletes_group/"; // TileDB context. + VFSTestSetup vfs_test_setup_; Context ctx_; VFS vfs_; sm::StorageManager* sm_; - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; + const std::string array_name_; + // to be used for direct VFS operations in [rest] testcases where array_name_ + // is prepended with tiledb://{namespace} so not suitable + const std::string vfs_array_name_; + const std::string group_name_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; // Constructors/destructors. DeletesFx(); - ~DeletesFx(); // Functions. void set_legacy(); @@ -115,25 +115,24 @@ struct DeletesFx { }; DeletesFx::DeletesFx() - : vfs_(ctx_) { + : vfs_(ctx_) + , array_name_(vfs_test_setup_.array_uri(SPARSE_ARRAY_NAME)) + , vfs_array_name_(vfs_test_setup_.array_uri(SPARSE_ARRAY_NAME, true)) + , group_name_(vfs_test_setup_.array_uri(GROUP_NAME)) { Config config; config.set("sm.consolidation.buffer_size", "1000"); - ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx(); vfs_ = VFS(ctx_); -} - -DeletesFx::~DeletesFx() { + sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::set_purge_deleted_cells() { Config config; config.set("sm.consolidation.buffer_size", "1000"); config.set("sm.consolidation.purge_deleted_cells", "true"); - - ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); - vfs_ = VFS(ctx_); + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx(); } void DeletesFx::set_legacy() { @@ -141,10 +140,8 @@ void DeletesFx::set_legacy() { config.set("sm.consolidation.buffer_size", "1000"); config.set("sm.query.sparse_global_order.reader", "legacy"); config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); - - ctx_ = Context(config); - sm_ = ctx_.ptr().get()->storage_manager(); - vfs_ = VFS(ctx_); + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx(); } void DeletesFx::create_dir(const std::string& path) { @@ -200,24 +197,12 @@ void DeletesFx::create_sparse_array(bool allows_dups, bool encrypt) { schema.set_coords_filter_list(filter_list); if (encrypt) { - Array::create(SPARSE_ARRAY_NAME, schema, enc_type_, key_); + Array::create(array_name_.c_str(), schema, enc_type_, key_); } else { - Array::create(SPARSE_ARRAY_NAME, schema); + Array::create(array_name_.c_str(), schema); } } -void DeletesFx::create_sparse_array_v11() { - // Get the v11 sparse array. - std::string v11_arrays_dir = - std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; - REQUIRE( - tiledb_vfs_copy_dir( - ctx_.ptr().get(), - vfs_.ptr().get(), - v11_arrays_dir.c_str(), - SPARSE_ARRAY_NAME) == TILEDB_OK); -} - void DeletesFx::write_sparse( std::vector a1, std::vector dim1, @@ -229,14 +214,14 @@ void DeletesFx::write_sparse( if (encrypt) { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp), EncryptionAlgorithm(AESGCM, key_.c_str())); } else { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp)); } @@ -249,18 +234,24 @@ void DeletesFx::write_sparse( query.set_data_buffer("d2", dim2); // Submit/finalize the query. - submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + query.submit_and_finalize(); // Close array. array->close(); } +void DeletesFx::create_sparse_array_v11() { + // Get the v11 sparse array. + std::string v11_arrays_dir = + std::string(TILEDB_TEST_INPUTS_DIR) + "/arrays/sparse_array_v11"; + REQUIRE( + tiledb_vfs_copy_dir( + ctx_.ptr().get(), + vfs_.ptr().get(), + v11_arrays_dir.c_str(), + array_name_.c_str()) == TILEDB_OK); +} + void DeletesFx::write_sparse_v11(uint64_t timestamp) { // Prepare cell buffers. std::vector buffer_a1{0, 1, 2, 3}; @@ -271,7 +262,7 @@ void DeletesFx::write_sparse_v11(uint64_t timestamp) { std::vector buffer_coords_dim2{1, 2, 4, 3}; // Open array. - Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_WRITE, timestamp); + Array array(ctx_, array_name_.c_str(), TILEDB_WRITE, timestamp); // Create query. Query query(ctx_, array, TILEDB_WRITE); @@ -285,13 +276,7 @@ void DeletesFx::write_sparse_v11(uint64_t timestamp) { query.set_data_buffer("d2", buffer_coords_dim2); // Submit/finalize the query. - submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + query.submit_and_finalize(); // Close array. array.close(); @@ -322,14 +307,14 @@ void DeletesFx::read_sparse( if (encrypt) { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_READ, TemporalPolicy(TimeTravel, timestamp), EncryptionAlgorithm(AESGCM, key_.c_str())); } else { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_READ, TemporalPolicy(TimeTravel, timestamp)); } @@ -342,14 +327,8 @@ void DeletesFx::read_sparse( query.set_data_buffer("d2", dim2); // Submit the query. - submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); + query.submit(); + CHECK(query.query_status() == Query::Status::COMPLETE); // Get the query stats. @@ -361,21 +340,21 @@ void DeletesFx::read_sparse( void DeletesFx::consolidate_sparse(bool vacuum) { auto config = ctx_.config(); - Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); + Array::consolidate(ctx_, array_name_, &config); if (vacuum) { - REQUIRE_NOTHROW(Array::vacuum(ctx_, SPARSE_ARRAY_NAME, &config)); + REQUIRE_NOTHROW(Array::vacuum(ctx_, array_name_, &config)); } } void DeletesFx::consolidate_commits_sparse(bool vacuum) { auto config = ctx_.config(); config["sm.consolidation.mode"] = "commits"; - Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); + Array::consolidate(ctx_, array_name_.c_str(), &config); if (vacuum) { config["sm.vacuum.mode"] = "commits"; - REQUIRE_NOTHROW(Array::vacuum(ctx_, SPARSE_ARRAY_NAME, &config)); + REQUIRE_NOTHROW(Array::vacuum(ctx_, array_name_.c_str(), &config)); } } @@ -386,10 +365,10 @@ void DeletesFx::consolidate_sparse_with_timestamps( std::to_string(timestamp_start).c_str(); config["sm.consolidation.timestamp_end"] = std::to_string(timestamp_end).c_str(); - Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); + Array::consolidate(ctx_, array_name_.c_str(), &config); if (vacuum) { - REQUIRE_NOTHROW(Array::vacuum(ctx_, SPARSE_ARRAY_NAME, &config)); + REQUIRE_NOTHROW(Array::vacuum(ctx_, array_name_.c_str(), &config)); } } @@ -400,14 +379,14 @@ void DeletesFx::write_delete_condition( if (encrypt) { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_DELETE, TemporalPolicy(TimeTravel, timestamp), EncryptionAlgorithm(AESGCM, key_.c_str())); } else { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_DELETE, TemporalPolicy(TimeTravel, timestamp)); } @@ -416,20 +395,7 @@ void DeletesFx::write_delete_condition( Query query(ctx_, *array, TILEDB_DELETE); query.set_condition(qc); - // Submit the query. In certain tests we want to check if this call throws, so - // we call directly query.submit() if serialization is not enabled. - if (!serialize_) { - query.submit(); - } else { - submit_query_wrapper( - ctx_, - SPARSE_ARRAY_NAME, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - } + query.submit(); CHECK(query.query_status() == Query::Status::COMPLETE); // Close array. @@ -443,14 +409,14 @@ void DeletesFx::check_delete_conditions( if (encrypt) { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_READ, TemporalPolicy(TimeTravel, timestamp), EncryptionAlgorithm(AESGCM, key_.c_str())); } else { array = std::make_unique( ctx_, - SPARSE_ARRAY_NAME, + array_name_.c_str(), TILEDB_READ, TemporalPolicy(TimeTravel, timestamp)); } @@ -484,7 +450,7 @@ void DeletesFx::remove_array(const std::string& array_name) { } void DeletesFx::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); + remove_array(array_name_.c_str()); } bool DeletesFx::is_array(const std::string& array_name) { @@ -538,17 +504,6 @@ TEST_CASE_METHOD( remove_sparse_array(); bool encrypt = GENERATE(true, false); - -#ifdef TILEDB_SERIALIZATION - // serialization not supported for encrypted arrays - if (!encrypt) { - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } - } -#endif - create_sparse_array(false, encrypt); // Define query condition (a1 < 4). @@ -614,15 +569,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, reading with delete condition", - "[cppapi][deletes][read]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - remove_sparse_array(); - + "[cppapi][deletes][read][rest-fails][sc-45852]") { bool consolidate = GENERATE(true, false); bool purge_deleted_cells = GENERATE(true, false); bool vacuum = GENERATE(true, false); @@ -655,7 +602,7 @@ TEST_CASE_METHOD( } // Consolidate with delete. - if (consolidate) { + if (consolidate && !vfs_test_setup_.is_rest()) { consolidate_sparse(vacuum); } @@ -715,8 +662,6 @@ TEST_CASE_METHOD( c_dim1_3.data(), dim1_3.data(), c_dim1_3.size() * sizeof(uint64_t))); CHECK(!memcmp( c_dim2_3.data(), dim2_3.data(), c_dim2_3.size() * sizeof(uint64_t))); - - remove_sparse_array(); } TEST_CASE_METHOD( @@ -724,12 +669,10 @@ TEST_CASE_METHOD( "CPP API: Test deletes, reading with delete condition, consolidated " "fragment", "[cppapi][deletes][read][consolidated]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -911,16 +854,7 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, reading with delete condition, delete duplicates " "from later fragments", - "[cppapi][deletes][duplicates]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_sparse_array(); - + "[cppapi][deletes][duplicates][rest-fails][sc-45852]") { bool purge_deleted_cells = GENERATE(true, false); bool consolidate = GENERATE(true, false); bool vacuum = GENERATE(true, false); @@ -954,7 +888,7 @@ TEST_CASE_METHOD( } // Consolidate with delete. - if (consolidate) { + if (consolidate && !vfs_test_setup_.is_rest()) { consolidate_sparse(vacuum); } @@ -982,21 +916,12 @@ TEST_CASE_METHOD( CHECK(!memcmp(c_a1.data(), a1.data(), c_a1.size() * sizeof(int))); CHECK(!memcmp(c_dim1.data(), dim1.data(), c_dim1.size() * sizeof(uint64_t))); CHECK(!memcmp(c_dim2.data(), dim2.data(), c_dim2.size() * sizeof(uint64_t))); - - remove_sparse_array(); } TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, commits consolidation", "[cppapi][deletes][commits][consolidation]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - bool vacuum = GENERATE(false, true); remove_sparse_array(); @@ -1048,12 +973,10 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, consolidation, delete same cell earlier", "[cppapi][deletes][consolidation][same-cell]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -1175,12 +1098,10 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, multiple consolidation with deletes", "[cppapi][deletes][consolidation][multiple]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -1296,12 +1217,10 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Test deletes, multiple cells with same coords in same fragment", "[cppapi][deletes][consolidation][multiple-cells-same-coords]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -1483,12 +1402,10 @@ TEST_CASE_METHOD( "across tiles", "[cppapi][deletes][consolidation][multiple-cells-same-coords][across-" "tiles]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -1669,12 +1586,10 @@ TEST_CASE_METHOD( "CPP API: Test consolidating fragment with delete timestamp, with purge " "option", "[cppapi][deletes][consolidation][with-delete-meta][purge]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + // SC-46282 Test fails on remote filesystems (vacuum) + if (!vfs_test_setup_.is_local()) { + return; } -#endif remove_sparse_array(); @@ -1723,6 +1638,11 @@ TEST_CASE_METHOD( "CPP API: Test writing delete in middle of fragment consolidated without " "timestamps", "[cppapi][deletes][write][old-consolidated-fragment]") { + // Copying across filesystems is not supported currently + if (!vfs_test_setup_.is_local()) { + return; + } + if constexpr (is_experimental_build) { return; } @@ -1739,7 +1659,7 @@ TEST_CASE_METHOD( consolidate_sparse(); // Upgrade to latest version. - Array::upgrade_version(ctx_, SPARSE_ARRAY_NAME); + Array::upgrade_version(ctx_, array_name_); // Define query condition (d2 == 3). QueryCondition qc2(ctx_); @@ -1764,18 +1684,17 @@ TEST_CASE_METHOD( create_sparse_array(); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 1); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 3); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 2); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == 2); // Try to delete a fragment uri that doesn't exist std::string extraneous_fragment = - std::string(SPARSE_ARRAY_NAME) + "/" + + std::string(array_name_) + "/" + tiledb::sm::constants::array_fragments_dir_name + "/extraneous"; const char* extraneous_fragments[1] = {extraneous_fragment.c_str()}; REQUIRE_THROWS_WITH( - Array::delete_fragments_list( - ctx_, SPARSE_ARRAY_NAME, extraneous_fragments, 1), + Array::delete_fragments_list(ctx_, array_name_, extraneous_fragments, 1), Catch::Matchers::ContainsSubstring("Failed to delete fragments_list")); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 2); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == 2); remove_sparse_array(); } @@ -1783,16 +1702,7 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of fragment writes by timestamp and uri", - "[cppapi][deletes][fragments]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - remove_sparse_array(); - + "[cppapi][deletes][fragments][rest-fails][sc-46338]") { // Conditionally consolidate and vacuum bool consolidate = GENERATE(true, false); bool vacuum = GENERATE(true, false); @@ -1807,12 +1717,13 @@ TEST_CASE_METHOD( write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 3); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 5); write_sparse({0, 1, 2, 3}, {1, 1, 1, 2}, {1, 2, 4, 3}, 7); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); + CHECK(tiledb::test::num_fragments(ctx_, vfs_array_name_) == 4); if (consolidate) { + auto vfs = VFS(ctx_); consolidate_commits_sparse(vacuum); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); - CommitsDirectory commits_dir(vfs_, SPARSE_ARRAY_NAME); + CHECK(tiledb::test::num_fragments(ctx_, vfs_array_name_) == 4); + CommitsDirectory commits_dir(vfs, vfs_array_name_); if (vacuum) { CHECK(commits_dir.dir_size() == 1); } else { @@ -1825,24 +1736,25 @@ TEST_CASE_METHOD( // Delete fragments SECTION("delete fragments by timestamps") { - Array::delete_fragments(ctx_, SPARSE_ARRAY_NAME, 2, 6); + Array::delete_fragments(ctx_, array_name_, 2, 6); } SECTION("delete fragments by uris") { - FragmentInfo fragment_info(ctx_, std::string(SPARSE_ARRAY_NAME)); + FragmentInfo fragment_info(ctx_, std::string(array_name_)); fragment_info.load(); auto fragment_name1 = fragment_info.fragment_uri(1); auto fragment_name2 = fragment_info.fragment_uri(2); const char* fragment_uris[2] = { fragment_name1.c_str(), fragment_name2.c_str()}; - Array::delete_fragments_list(ctx_, SPARSE_ARRAY_NAME, fragment_uris, 2); + Array::delete_fragments_list(ctx_, array_name_, fragment_uris, 2); } // Check commits directory after deletion if (consolidate) { + auto vfs = VFS(ctx_); /* Note: An ignore file is written by delete_fragments if there are * consolidated commits to be ignored by the delete. */ - CommitsDirectory commits_dir(vfs_, SPARSE_ARRAY_NAME); + CommitsDirectory commits_dir(vfs, vfs_array_name_); CHECK( commits_dir.file_count( tiledb::sm::constants::con_commits_file_suffix) == 1); @@ -1854,7 +1766,7 @@ TEST_CASE_METHOD( CHECK(commits_dir.dir_size() == 4); } } - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 2); + CHECK(tiledb::test::num_fragments(ctx_, vfs_array_name_) == 2); // Read array uint64_t buffer_size = 4; @@ -1869,21 +1781,12 @@ TEST_CASE_METHOD( CHECK(c_a1 == a1); CHECK(c_dim1 == dim1); CHECK(c_dim2 == dim2); - - remove_sparse_array(); } TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of fragment writes consolidated with timestamps", "[cppapi][deletes][fragments][consolidation_with_timestamps]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - remove_sparse_array(); bool vacuum = GENERATE(true, false); @@ -1901,29 +1804,29 @@ TEST_CASE_METHOD( num_commits++; num_fragments++; if (!vacuum) { - CommitsDirectory commits_dir(vfs_, SPARSE_ARRAY_NAME); + CommitsDirectory commits_dir(vfs_, array_name_); CHECK( commits_dir.file_count(tiledb::sm::constants::vacuum_file_suffix) == 1); } else { num_commits -= 2; num_fragments -= 2; } - CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == num_commits); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == num_fragments); + CHECK(tiledb::test::num_commits(ctx_, array_name_) == num_commits); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == num_fragments); // Delete fragments at timestamps 2 - 4 - Array::delete_fragments(ctx_, SPARSE_ARRAY_NAME, 2, 4); + Array::delete_fragments(ctx_, array_name_, 2, 4); if (!vacuum) { // Vacuum after deletion auto config = ctx_.config(); - Array::vacuum(ctx_, SPARSE_ARRAY_NAME, &config); + Array::vacuum(ctx_, array_name_, &config); num_commits -= 2; num_fragments -= 2; } // Validate working directory - CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == num_commits); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == num_fragments); + CHECK(tiledb::test::num_commits(ctx_, array_name_) == num_commits); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == num_fragments); // Read array uint64_t buffer_size = 4; @@ -1944,15 +1847,8 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of array data", "[cppapi][deletes][array]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); - } -#endif - remove_sparse_array(); - auto array_name = std::string(SPARSE_ARRAY_NAME) + "/"; + auto array_name = array_name_ + "/"; // Conditionally consolidate // Note: there's no need to vacuum; delete_array will delete all fragments @@ -1967,14 +1863,14 @@ TEST_CASE_METHOD( std::string extraneous_file_path = array_name + "extraneous_file"; vfs_.touch(extraneous_file_path); std::unique_ptr array = - std::make_unique(ctx_, SPARSE_ARRAY_NAME, TILEDB_WRITE); + std::make_unique(ctx_, array_name_, TILEDB_WRITE); int v = 100; array->put_metadata("aaa", TILEDB_INT32, 1, &v); array->close(); // Check write - CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == 4); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); + CHECK(tiledb::test::num_commits(ctx_, array_name_) == 4); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == 4); auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto meta = @@ -1985,19 +1881,19 @@ TEST_CASE_METHOD( // Consolidate commits auto config = ctx_.config(); config["sm.consolidation.mode"] = "commits"; - Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); + Array::consolidate(ctx_, array_name_, &config); // Consolidate fragment metadata config["sm.consolidation.mode"] = "fragment_meta"; - Array::consolidate(ctx_, SPARSE_ARRAY_NAME, &config); + Array::consolidate(ctx_, array_name_, &config); // Validate working directory - CommitsDirectory commits_dir(vfs_, SPARSE_ARRAY_NAME); + CommitsDirectory commits_dir(vfs_, array_name_); CHECK(commits_dir.dir_size() == 5); CHECK( commits_dir.file_count( tiledb::sm::constants::con_commits_file_suffix) == 1); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == 4); auto frag_meta = vfs_.ls( array_name + tiledb::sm::constants::array_fragment_meta_dir_name); CHECK(frag_meta.size() == 1); @@ -2008,12 +1904,12 @@ TEST_CASE_METHOD( // Check working directory after delete REQUIRE(vfs_.is_file(extraneous_file_path)); - CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 0); + CHECK(tiledb::test::num_fragments(ctx_, array_name_) == 0); validate_array_dir_after_delete(array_name); // Try to open array REQUIRE_THROWS_WITH( - std::make_unique(ctx_, SPARSE_ARRAY_NAME, TILEDB_READ), + std::make_unique(ctx_, array_name_, TILEDB_READ), Catch::Matchers::ContainsSubstring("Array does not exist")); remove_sparse_array(); @@ -2025,19 +1921,17 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of older-versioned array data", "[cppapi][deletes][array][older_version]") { - if constexpr (is_experimental_build) { + // Copying across filesystems is not supported currently + if (!vfs_test_setup_.is_local()) { return; } -#ifdef TILEDB_SERIALIZATION - serialize_ = GENERATE(false, true); - if (serialize_) { - refactored_query_v2_ = GENERATE(true, false); + if constexpr (is_experimental_build) { + return; } -#endif remove_sparse_array(); - auto array_name = std::string(SPARSE_ARRAY_NAME) + "/"; + auto array_name = array_name_ + "/"; // Write to v11 array create_sparse_array_v11(); @@ -2085,32 +1979,32 @@ TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of invalid group writes", "[cppapi][deletes][group][invalid]") { - create_dir(GROUP_NAME); + create_dir(group_name_); // Create and open group in write mode - tiledb::Group::create(ctx_, GROUP_NAME); - tiledb::Group group(ctx_, std::string(GROUP_NAME), TILEDB_WRITE); + tiledb::Group::create(ctx_, group_name_); + tiledb::Group group(ctx_, group_name_, TILEDB_WRITE); // Try to delete group REQUIRE_THROWS_WITH( - group.delete_group(GROUP_NAME), + group.delete_group(group_name_), Catch::Matchers::ContainsSubstring( "Query type must be MODIFY_EXCLUSIVE")); group.close(); // Try to delete group after close REQUIRE_THROWS_WITH( - group.delete_group(GROUP_NAME), + group.delete_group(group_name_), Catch::Matchers::ContainsSubstring("Group is not open")); - remove_dir(GROUP_NAME); + remove_dir(group_name_); } TEST_CASE_METHOD( DeletesFx, "CPP API: Deletion of group writes", "[cppapi][deletes][group]") { - create_dir(GROUP_NAME); + create_dir(group_name_); // Conditionally consolidate and vacuum bool consolidate = GENERATE(true, false); @@ -2120,8 +2014,8 @@ TEST_CASE_METHOD( } // Create group - tiledb::Group::create(ctx_, GROUP_NAME); - std::string array_path = tiledb::sm::URI(GROUP_NAME + "array/").to_string(); + tiledb::Group::create(ctx_, group_name_); + std::string array_path = tiledb::sm::URI(group_name_ + "array/").to_string(); create_simple_array(array_path); // Set expected @@ -2130,7 +2024,7 @@ TEST_CASE_METHOD( }; // Write to group - tiledb::Group group(ctx_, GROUP_NAME, TILEDB_WRITE); + tiledb::Group group(ctx_, group_name_, TILEDB_WRITE); group.add_member(array_path, false); int32_t v = 123; group.put_metadata("test_deletes_meta", TILEDB_INT32, 1, &v); @@ -2143,7 +2037,7 @@ TEST_CASE_METHOD( group.close(); // Add extraneous file - std::string extraneous_file_path = GROUP_NAME + "extraneous_file"; + std::string extraneous_file_path = group_name_ + "extraneous_file"; vfs_.touch(extraneous_file_path); // Validate group structure @@ -2155,28 +2049,28 @@ TEST_CASE_METHOD( // Validate group data REQUIRE(vfs_.is_file(extraneous_file_path)); - REQUIRE(vfs_.is_file(GROUP_NAME + tiledb::sm::constants::group_filename)); + REQUIRE(vfs_.is_file(group_name_ + tiledb::sm::constants::group_filename)); auto group_detail_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_detail_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_detail_dir_name); CHECK(group_detail_dir.size() == 1); auto group_meta_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 3); // Conditionally consolidate and vacuum group and validate data if (consolidate) { auto config = ctx_.config(); config["sm.consolidation.mode"] = "group_meta"; - Group::consolidate_metadata(ctx_, GROUP_NAME, &config); + Group::consolidate_metadata(ctx_, group_name_, &config); group_meta_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 5); if (vacuum) { config["sm.vacuum.mode"] = "group_meta"; - Group::vacuum_metadata(ctx_, GROUP_NAME, &config); + Group::vacuum_metadata(ctx_, group_name_, &config); group_meta_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 1); } } @@ -2184,11 +2078,11 @@ TEST_CASE_METHOD( // Delete group in modify exclusive mode // Note: delete_group will close the group, no need to do so here. group.open(TILEDB_MODIFY_EXCLUSIVE); - group.delete_group(GROUP_NAME.c_str()); + group.delete_group(group_name_.c_str()); // Validate group data REQUIRE(vfs_.is_file(extraneous_file_path)); - validate_group_dir_after_delete(GROUP_NAME); + validate_group_dir_after_delete(group_name_); // Try to open group REQUIRE_THROWS_WITH( @@ -2201,22 +2095,24 @@ TEST_CASE_METHOD( array->close(); // Clean up - remove_dir(GROUP_NAME); + remove_dir(group_name_); } TEST_CASE_METHOD( DeletesFx, "CPP API: Recursive deletion of group writes", "[cppapi][deletes][group][recursive]") { - create_dir(GROUP_NAME); + create_dir(group_name_); // Create groups - tiledb::Group::create(ctx_, GROUP_NAME); - std::string array_path = tiledb::sm::URI(GROUP_NAME + "array/").to_string(); + tiledb::Group::create(ctx_, group_name_); + std::string array_path = tiledb::sm::URI(group_name_ + "array/").to_string(); create_simple_array(array_path); - std::string group2_path = tiledb::sm::URI(GROUP_NAME + "group2/").to_string(); + std::string group2_path = + tiledb::sm::URI(group_name_ + "group2/").to_string(); tiledb::Group::create(ctx_, group2_path); - std::string array2_path = tiledb::sm::URI(GROUP_NAME + "array2/").to_string(); + std::string array2_path = + tiledb::sm::URI(group_name_ + "array2/").to_string(); create_simple_array(array2_path); // Set expected @@ -2229,7 +2125,7 @@ TEST_CASE_METHOD( }; // Write to group - tiledb::Group group(ctx_, GROUP_NAME, TILEDB_WRITE); + tiledb::Group group(ctx_, group_name_, TILEDB_WRITE); tiledb::Group group2(ctx_, group2_path, TILEDB_WRITE); group.add_member(array_path, false); group.add_member(group2_path, false); @@ -2240,7 +2136,7 @@ TEST_CASE_METHOD( group2.close(); // Add extraneous file - std::string extraneous_file_path = GROUP_NAME + "extraneous_file"; + std::string extraneous_file_path = group_name_ + "extraneous_file"; vfs_.touch(extraneous_file_path); // Validate group structure @@ -2257,12 +2153,12 @@ TEST_CASE_METHOD( // Validate group data REQUIRE(vfs_.is_file(extraneous_file_path)); - REQUIRE(vfs_.is_file(GROUP_NAME + tiledb::sm::constants::group_filename)); + REQUIRE(vfs_.is_file(group_name_ + tiledb::sm::constants::group_filename)); auto group_detail_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_detail_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_detail_dir_name); CHECK(group_detail_dir.size() == 1); auto group_meta_dir = - vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); + vfs_.ls(group_name_ + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 1); auto array_schema = list_schemas(array_path); CHECK(array_schema.size() == 1); @@ -2271,11 +2167,11 @@ TEST_CASE_METHOD( // Recursively delete group in modify exclusive mode group.open(TILEDB_MODIFY_EXCLUSIVE); - group.delete_group(GROUP_NAME.c_str(), true); + group.delete_group(group_name_.c_str(), true); // Validate group data REQUIRE(vfs_.is_file(extraneous_file_path)); - validate_group_dir_after_delete(GROUP_NAME); + validate_group_dir_after_delete(group_name_); validate_group_dir_after_delete(group2_path); validate_array_dir_after_delete(array_path); validate_array_dir_after_delete(array2_path); @@ -2289,5 +2185,5 @@ TEST_CASE_METHOD( Catch::Matchers::ContainsSubstring("Array does not exist")); // Clean up - remove_dir(GROUP_NAME); + remove_dir(group_name_); } diff --git a/test/src/unit-cppapi-filter.cc b/test/src/unit-cppapi-filter.cc index 0866ca14c049..976525bfc5d8 100644 --- a/test/src/unit-cppapi-filter.cc +++ b/test/src/unit-cppapi-filter.cc @@ -32,8 +32,11 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" +using namespace tiledb; + static void check_filters( const tiledb::FilterList& answer, const tiledb::FilterList& check) { REQUIRE(check.nfilters() == answer.nfilters()); @@ -45,7 +48,6 @@ static void check_filters( } TEST_CASE("C++ API: Filter options", "[cppapi][filter]") { - using namespace tiledb; Context ctx; // Test filter creation and option setting/getting @@ -105,7 +107,6 @@ TEST_CASE("C++ API: Filter options", "[cppapi][filter]") { } TEST_CASE("C++ API: Filter lists", "[cppapi][filter]") { - using namespace tiledb; Context ctx; Filter f1(ctx, TILEDB_FILTER_BIT_WIDTH_REDUCTION), @@ -138,14 +139,12 @@ TEST_CASE("C++ API: Filter lists", "[cppapi][filter]") { REQUIRE(list.nfilters() == 3); } -TEST_CASE("C++ API: Filter lists on array", "[cppapi][filter]") { - using namespace tiledb; - Context ctx; - VFS vfs(ctx); - std::string array_name = "cpp_unit_array"; - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); +TEST_CASE( + "C++ API: Filter lists on array", + "[cppapi][filter][rest-fails][sc-45554]") { + test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; // Create schema with filter lists FilterList a1_filters(ctx); @@ -236,10 +235,6 @@ TEST_CASE("C++ API: Filter lists on array", "[cppapi][filter]") { check_filters(a1_filters, schema_r.attribute("a1").filter_list()); check_filters(a2_filters, schema_r.attribute("a2").filter_list()); array.close(); - - // Clean up - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } void write_sparse_array_string_attr( @@ -247,10 +242,7 @@ void write_sparse_array_string_attr( const std::string& array_name, std::string& data, std::vector& data_offsets, - tiledb_layout_t layout, - const bool serialized, - const bool refactored_query_v2) { - using namespace tiledb; + tiledb_layout_t layout) { // Write to array std::vector d1 = {0, 10, 20, 20, 30, 30, 40}; std::vector d2 = {0, 10, 20, 30, 30, 40, 40}; @@ -263,15 +255,12 @@ void write_sparse_array_string_attr( query.set_data_buffer("a1", data).set_offsets_buffer("a1", data_offsets); // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } + array.close(); } @@ -280,10 +269,7 @@ void read_and_check_sparse_array_string_attr( const std::string& array_name, std::string& expected_data, std::vector& expected_offsets, - tiledb_layout_t layout, - const bool serialized, - const bool refactored_query_v2) { - using namespace tiledb; + tiledb_layout_t layout) { Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); @@ -296,15 +282,7 @@ void read_and_check_sparse_array_string_attr( query.set_offsets_buffer("a1", attr_off); // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element offsets are properly returned CHECK(attr_val == expected_data); @@ -315,24 +293,13 @@ void read_and_check_sparse_array_string_attr( TEST_CASE( "C++ API: Filter strings with RLE or Dictionary encoding, sparse array", - "[cppapi][filter][rle-strings][dict-strings][sparse]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - using namespace tiledb; - Context ctx; - VFS vfs(ctx); - std::string array_name = "cpp_unit_array"; - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); - + "[cppapi][filter][rle-strings][dict-strings][sparse][rest]") { auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); + test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; + // Create schema with filter lists FilterList a1_filters(ctx); a1_filters.add_filter({ctx, f}); @@ -366,88 +333,36 @@ TEST_CASE( SECTION("Unordered write") { write_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); SECTION("Row major read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); } } SECTION("Global order write") { write_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); SECTION("Row major read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); } } - - // Clean up - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } void write_dense_array_string_attr( @@ -455,10 +370,7 @@ void write_dense_array_string_attr( const std::string& array_name, std::string& data, std::vector& data_offsets, - tiledb_layout_t layout, - const bool serialized, - const bool refactored_query_v2) { - using namespace tiledb; + tiledb_layout_t layout) { Array array(ctx, array_name, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); @@ -467,15 +379,11 @@ void write_dense_array_string_attr( query.set_layout(layout); query.set_subarray(Subarray(ctx, array).set_subarray({0, 1, 0, 2})); - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -484,10 +392,7 @@ void read_and_check_dense_array_string_attr( tiledb::Context ctx, const std::string& array_name, std::string& expected_data, - std::vector& expected_offsets, - const bool serialized, - const bool refactored_query_v2) { - using namespace tiledb; + std::vector& expected_offsets) { Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); @@ -500,15 +405,7 @@ void read_and_check_dense_array_string_attr( query.set_offsets_buffer("a1", attr_off); // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element offsets are properly returned CHECK(attr_val == expected_data); @@ -519,27 +416,13 @@ void read_and_check_dense_array_string_attr( TEST_CASE( "C++ API: Filter strings with RLE or Dictionary encoding, dense array", - "[cppapi][filter][rle-strings][dict-strings][dense]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - using namespace tiledb; - Context ctx; - VFS vfs(ctx); - std::string array_name = "cpp_unit_array"; - - // Create the array - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[cppapi][filter][rle-strings][dict-strings][dense][rest]") { + auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; - auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); // Create schema with filter lists FilterList a1_filters(ctx); a1_filters.add_filter({ctx, f}); @@ -573,55 +456,28 @@ TEST_CASE( SECTION("Ordered write") { write_dense_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_ROW_MAJOR); read_and_check_dense_array_string_attr( - ctx, array_name, a1_data, a1_offsets, serialized, refactored_query_v2); + ctx, array_name, a1_data, a1_offsets); } SECTION("Global order write") { write_dense_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); read_and_check_dense_array_string_attr( - ctx, array_name, a1_data, a1_offsets, serialized, refactored_query_v2); + ctx, array_name, a1_data, a1_offsets); } - - // Clean up - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Filter UTF-8 strings with RLE or Dictionary encoding, sparse " "array", - "[cppapi][filter][rle-strings][dict-strings][sparse][utf-8]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - using namespace tiledb; - Context ctx; - VFS vfs(ctx); - std::string array_name = "cpp_unit_array"; - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); - + "[cppapi][filter][rle-strings][dict-strings][sparse][utf-8][rest]") { auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); + test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; + // Create schema with filter lists FilterList a1_filters(ctx); a1_filters.add_filter({ctx, f}); @@ -659,104 +515,48 @@ TEST_CASE( SECTION("Unordered write") { write_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); SECTION("Row major read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); } } SECTION("Global order write") { write_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); SECTION("Row major read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_ROW_MAJOR, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_ROW_MAJOR); } SECTION("Global order read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_GLOBAL_ORDER, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_GLOBAL_ORDER); } SECTION("Unordered read") { read_and_check_sparse_array_string_attr( - ctx, - array_name, - a1_data, - a1_offsets, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, a1_data, a1_offsets, TILEDB_UNORDERED); } } - - // Clean up - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( "C++ API: Filter buffer with some empty strings with RLE or Dictionary " "encoding", - "[cppapi][filter][rle-strings][dict-strings][empty-strings]") { - using namespace tiledb; - Context ctx; - VFS vfs(ctx); - std::string array_name = "cpp_unit_array"; - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); - + "[cppapi][filter][rle-strings][dict-strings][empty-strings][rest]") { auto f = GENERATE(TILEDB_FILTER_RLE, TILEDB_FILTER_DICTIONARY); + test::VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("cpp_unit_array")}; + // Create array with string dimension and one attribute ArraySchema schema(ctx, TILEDB_SPARSE); @@ -838,8 +638,4 @@ TEST_CASE( } array_r.close(); - - // Clean up - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } diff --git a/test/src/unit-cppapi-hilbert.cc b/test/src/unit-cppapi-hilbert.cc index 4cec2a37de1e..2a36bc1ccf17 100644 --- a/test/src/unit-cppapi-hilbert.cc +++ b/test/src/unit-cppapi-hilbert.cc @@ -32,13 +32,13 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" using namespace tiledb; using namespace tiledb::test; -void create_int32_array(const std::string& array_name) { - Context ctx; +void create_int32_array(Context ctx, const std::string& array_name) { Domain domain(ctx); auto d1 = Dimension::create(ctx, "d1", {{0, 100}}); auto d2 = Dimension::create(ctx, "d2", {{0, 200}}); @@ -53,8 +53,8 @@ void create_int32_array(const std::string& array_name) { Array::create(array_name, schema); } -void create_int32_array_negative_domain(const std::string& array_name) { - Context ctx; +void create_int32_array_negative_domain( + Context ctx, const std::string& array_name) { Domain domain(ctx); auto d1 = Dimension::create(ctx, "d1", {{-50, 50}}); auto d2 = Dimension::create(ctx, "d2", {{-100, 100}}); @@ -69,8 +69,7 @@ void create_int32_array_negative_domain(const std::string& array_name) { Array::create(array_name, schema); } -void create_float32_array(const std::string& array_name) { - Context ctx; +void create_float32_array(Context ctx, const std::string& array_name) { Domain domain(ctx); auto d1 = Dimension::create(ctx, "d1", {{0.0, 1.0}}); auto d2 = Dimension::create(ctx, "d2", {{0.0, 2.0}}); @@ -85,8 +84,7 @@ void create_float32_array(const std::string& array_name) { Array::create(array_name, schema); } -void create_string_array(const std::string& array_name) { - Context ctx; +void create_string_array(Context ctx, const std::string& array_name) { Domain domain(ctx); auto d1 = Dimension::create(ctx, "d1", TILEDB_STRING_ASCII, nullptr, nullptr); auto d2 = Dimension::create(ctx, "d2", TILEDB_STRING_ASCII, nullptr, nullptr); @@ -103,35 +101,30 @@ void create_string_array(const std::string& array_name) { template void write_2d_array( + Context ctx, const std::string& array_name, std::vector& buff_d1, std::vector& buff_d2, std::vector& buff_a, - tiledb_layout_t layout, - const bool serialized = false, - const bool refactored_query_v2 = false) { - Context ctx; + tiledb_layout_t layout) { Array array_w(ctx, array_name, TILEDB_WRITE); Query query_w(ctx, array_w, TILEDB_WRITE); query_w.set_data_buffer("a", buff_a); query_w.set_data_buffer("d1", buff_d1); query_w.set_data_buffer("d2", buff_d2); query_w.set_layout(layout); - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + + if (layout == TILEDB_GLOBAL_ORDER) { + query_w.submit_and_finalize(); + } else { + query_w.submit(); + } array_w.close(); } void write_2d_array( + Context ctx, const std::string& array_name, std::vector& off_d1, std::string& buff_d1, @@ -139,7 +132,6 @@ void write_2d_array( std::string& buff_d2, std::vector& buff_a, tiledb_layout_t layout) { - Context ctx; Array array_w(ctx, array_name, TILEDB_WRITE); Query query_w(ctx, array_w, TILEDB_WRITE); query_w.set_data_buffer("a", buff_a); @@ -224,7 +216,7 @@ TEST_CASE("C++ API: Test Hilbert, errors", "[cppapi][hilbert][error]") { CHECK_NOTHROW(vfs.remove_dir(array_name)); // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Hilbert order not applicable to write queries Array array_w(ctx, array_name, TILEDB_WRITE); @@ -257,43 +249,20 @@ TEST_CASE("C++ API: Test Hilbert, errors", "[cppapi][hilbert][error]") { TEST_CASE( "C++ API: Test Hilbert, test 2D, int32, write unordered, read global", - "[cppapi][hilbert][2d][int32]") { - bool serialized = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialized = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialized = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - test::ServerQueryBuffers server_buffers_; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][int32][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write array std::vector buff_a = {3, 2, 1, 4}; std::vector buff_d1 = {1, 1, 4, 5}; std::vector buff_d2 = {1, 3, 2, 4}; write_2d_array( - array_name, - buff_d1, - buff_d2, - buff_a, - TILEDB_UNORDERED, - serialized, - refactored_query_v2); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Read SECTION("- Global order") { @@ -307,14 +276,7 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_layout(TILEDB_GLOBAL_ORDER); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_r, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_r.submit(); array_r.close(); // Check results @@ -337,14 +299,7 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_layout(TILEDB_ROW_MAJOR); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_r, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_r.submit(); array_r.close(); // Check results @@ -367,14 +322,7 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_layout(TILEDB_COL_MAJOR); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_r, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_r.submit(); array_r.close(); // Check results @@ -398,14 +346,7 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_layout(TILEDB_UNORDERED); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_r, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_r.submit(); array_r.close(); // Check results @@ -426,7 +367,8 @@ TEST_CASE( // fully removed for overlapping ranges, this section can be deleted. Config cfg; cfg["sm.merge_overlapping_ranges_experimental"] = "false"; - ctx = Context(cfg); + vfs_test_setup.update_config(cfg.ptr().get()); + ctx = vfs_test_setup.ctx(); // regression test for sc-11244 Array array_r(ctx, array_name, TILEDB_READ); @@ -447,15 +389,7 @@ TEST_CASE( query_r.add_range("d2", (int32_t)2, (int32_t)4); query_r.set_layout(TILEDB_UNORDERED); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_r, - server_buffers_, - serialized, - refactored_query_v2, - false); - REQUIRE(rc == TILEDB_OK); + query_r.submit(); CHECK(query_r.query_status() == tiledb::Query::Status::COMPLETE); // check number of results uint64_t num = query_r.result_buffer_elements()["a"].second; @@ -467,32 +401,24 @@ TEST_CASE( check_counts(span(r_buff_d1.data(), num), {0, 3, 0, 0, 2, 1}); check_counts(span(r_buff_d2.data(), num), {0, 1, 2, 2, 1}); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, int32, 2D, partitioner", - "[cppapi][hilbert][2d][int32][partitioner]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][int32][partitioner][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write array std::vector buff_d1 = {1, 1, 4, 5}; std::vector buff_d2 = {1, 3, 2, 4}; std::vector buff_a = {3, 2, 1, 4}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- entire domain") { // Read array @@ -587,36 +513,17 @@ TEST_CASE( array_r.close(); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, test writing in global order", - "[cppapi][hilbert][write][global-order]") { - bool serialized = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialized = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialized = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][write][global-order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write array std::vector buff_a = {3, 2, 1, 4}; @@ -635,40 +542,26 @@ TEST_CASE( buff_d1 = {1, 1, 5, 4}; buff_d2 = {3, 1, 4, 2}; - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + query_w.submit_and_finalize(); + array_w.close(); } TEST_CASE( - "C++ API: Test Hilbert, slicing", "[cppapi][hilbert][read][slicing]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "C++ API: Test Hilbert, slicing", + "[cppapi][hilbert][read][slicing][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write array std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {1, 1, 5, 4}; std::vector buff_d2 = {1, 3, 4, 2}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- Row-major") { Array array_r(ctx, array_name, TILEDB_READ); @@ -715,39 +608,31 @@ TEST_CASE( CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, multiple fragments, read in global order", - "[cppapi][hilbert][read][multiple-fragments][global-order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][read][multiple-fragments][global-order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {1, 1, 5, 4}; std::vector buff_d2 = {1, 3, 4, 2}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {2, 2, 3, 7}; buff_d2 = {1, 2, 7, 7}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -776,32 +661,24 @@ TEST_CASE( CHECK(r_buff_a == c_buff_a); CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, int32, unsplittable", - "[cppapi][hilbert][read][2d][int32][unsplittable]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][read][2d][int32][unsplittable][rest-fails][sc-43108]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {1, 1, 5, 4}; std::vector buff_d2 = {1, 3, 4, 2}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -816,15 +693,11 @@ TEST_CASE( CHECK(query_r.query_status() == Query::Status::INCOMPLETE); CHECK(query_r.result_buffer_elements()["a"].second == 0); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, consolidation", - "[cppapi][hilbert][consolidation]") { + "[cppapi][hilbert][consolidation][non-rest]") { Config cfg; cfg["sm.consolidation.buffer_size"] = "10000"; @@ -837,21 +710,21 @@ TEST_CASE( CHECK_NOTHROW(vfs.remove_dir(array_name)); // Create array - create_int32_array(array_name); + create_int32_array(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {1, 1, 5, 4}; std::vector buff_d2 = {1, 3, 4, 2}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {2, 2, 3, 7}; buff_d2 = {1, 2, 7, 7}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Consolidate and vacuum Config config; @@ -897,17 +770,13 @@ TEST_CASE( TEST_CASE( "C++ API: Test Hilbert, 2D, int32, negative, read/write in global order", - "[cppapi][hilbert][int32][negative][write][read][global-order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][int32][negative][write][read][global-order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write array std::vector buff_a = {3, 2, 1, 4}; @@ -925,8 +794,7 @@ TEST_CASE( buff_a = {2, 3, 4, 1}; buff_d1 = {-49, -49, -45, -46}; buff_d2 = {-97, -99, -96, -98}; - CHECK_NOTHROW(query_w.submit()); - query_w.finalize(); + CHECK_NOTHROW(query_w.submit_and_finalize()); array_w.close(); // Read @@ -951,32 +819,24 @@ TEST_CASE( CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, int32, negative, 2D, partitioner", - "[cppapi][hilbert][2d][int32][negative][partitioner]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][int32][negative][partitioner][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write array std::vector buff_d1 = {-49, -49, -46, -45}; std::vector buff_d2 = {-99, -97, -98, -96}; std::vector buff_a = {3, 2, 1, 4}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- entire domain") { // Read array @@ -1072,32 +932,24 @@ TEST_CASE( array_r.close(); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, int32, negative, slicing", - "[cppapi][hilbert][2d][int32][negative][read][slicing]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][int32][negative][read][slicing][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write array std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {-49, -49, -45, -46}; std::vector buff_d2 = {-99, -97, -96, -98}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- Row-major") { Array array_r(ctx, array_name, TILEDB_READ); @@ -1146,41 +998,33 @@ TEST_CASE( CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, int32, negative, multiple fragments, read in " "global order", "[cppapi][hilbert][2d][int32][negative][read][multiple-fragments][global-" - "order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {-49, -49, -45, -46}; std::vector buff_d2 = {-99, -97, -96, -98}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {-48, -48, -47, -43}; buff_d2 = {-99, -98, -93, -93}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -1201,10 +1045,6 @@ TEST_CASE( CHECK(r_buff_a == c_buff_a); CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( @@ -1222,21 +1062,21 @@ TEST_CASE( CHECK_NOTHROW(vfs.remove_dir(array_name)); // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {-49, -49, -45, -46}; std::vector buff_d2 = {-99, -97, -96, -98}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {-48, -48, -47, -43}; buff_d2 = {-99, -98, -93, -93}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Consolidate and vacuum Config config; @@ -1274,24 +1114,21 @@ TEST_CASE( TEST_CASE( "C++ API: Test Hilbert, 2d, int32, negative, unsplittable", - "[cppapi][hilbert][read][2d][int32][negative][unsplittable]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][read][2d][int32][negative][unsplittable][rest-fails][sc-" + "43108]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_int32_array_negative_domain(array_name); + create_int32_array_negative_domain(ctx, array_name); // Write first fragment std::vector buff_a = {3, 2, 4, 1}; std::vector buff_d1 = {-49, -49, -45, -46}; std::vector buff_d2 = {-99, -97, -96, -98}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -1306,25 +1143,17 @@ TEST_CASE( CHECK(query_r.query_status() == Query::Status::INCOMPLETE); CHECK(query_r.result_buffer_elements()["a"].second == 0); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2D, float32, read/write in global order", - "[cppapi][hilbert][float32][write][read][global-order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][float32][write][read][global-order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write array std::vector buff_a = {2, 3, 1, 4}; @@ -1342,8 +1171,7 @@ TEST_CASE( buff_a = {3, 2, 1, 4}; buff_d1 = {0.1f, 0.1f, 0.4f, 0.5f}; buff_d2 = {0.1f, 0.3f, 0.2f, 0.4f}; - CHECK_NOTHROW(query_w.submit()); - query_w.finalize(); + CHECK_NOTHROW(query_w.submit_and_finalize()); array_w.close(); // Read @@ -1373,32 +1201,24 @@ TEST_CASE( CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, float32, 2D, partitioner", "[cppapi][hilbert][2d][float32][partitioner]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write array std::vector buff_a = {2, 3, 1, 4}; std::vector buff_d1 = {0.1f, 0.1f, 0.41f, 0.4f}; std::vector buff_d2 = {0.3f, 0.1f, 0.41f, 0.4f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- entire domain") { // Read array @@ -1509,32 +1329,24 @@ TEST_CASE( array_r.close(); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, float32, slicing", - "[cppapi][hilbert][2d][float32][read][slicing]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][float32][read][slicing][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write array std::vector buff_a = {2, 3, 1, 4}; std::vector buff_d1 = {0.1f, 0.1f, 0.4f, 0.5f}; std::vector buff_d2 = {0.3f, 0.1f, 0.2f, 0.4f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); SECTION("- Col-major") { Array array_r(ctx, array_name, TILEDB_READ); @@ -1583,41 +1395,33 @@ TEST_CASE( CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, float32, multiple fragments, read in " "global order", "[cppapi][hilbert][2d][float32][read][multiple-fragments][global-" - "order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write first fragment std::vector buff_a = {2, 3, 1, 4}; std::vector buff_d1 = {0.1f, 0.1f, 0.4f, 0.5f}; std::vector buff_d2 = {0.3f, 0.1f, 0.2f, 0.4f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {0.2f, 0.2f, 0.3f, 0.7f}; buff_d2 = {0.2f, 0.1f, 0.7f, 0.7f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -1648,15 +1452,11 @@ TEST_CASE( CHECK(r_buff_a == c_buff_a); CHECK(r_buff_d1 == c_buff_d1); CHECK(r_buff_d2 == c_buff_d2); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, float32, consolidation", - "[cppapi][hilbert][2d][float32][consolidation]") { + "[cppapi][hilbert][2d][float32][consolidation][non-rest]") { Config cfg; cfg["sm.consolidation.buffer_size"] = "10000"; @@ -1669,21 +1469,21 @@ TEST_CASE( CHECK_NOTHROW(vfs.remove_dir(array_name)); // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write first fragment std::vector buff_a = {2, 3, 1, 4}; std::vector buff_d1 = {0.1f, 0.1f, 0.4f, 0.5f}; std::vector buff_d2 = {0.3f, 0.1f, 0.2f, 0.4f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; buff_d1 = {0.2f, 0.2f, 0.3f, 0.7f}; buff_d2 = {0.2f, 0.1f, 0.7f, 0.7f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); // Consolidate and vacuum Config config; @@ -1731,24 +1531,21 @@ TEST_CASE( TEST_CASE( "C++ API: Test Hilbert, 2d, float32, unsplittable", - "[cppapi][hilbert][read][2d][float32][unsplittable]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][read][2d][float32][unsplittable][rest-fails][sc-" + "43108]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_float32_array(array_name); + create_float32_array(ctx, array_name); // Write first fragment std::vector buff_a = {2, 3, 1, 4}; std::vector buff_d1 = {0.1f, 0.1f, 0.4f, 0.5f}; std::vector buff_d2 = {0.3f, 0.1f, 0.2f, 0.4f}; write_2d_array( - array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, array_name, buff_d1, buff_d2, buff_a, TILEDB_UNORDERED); Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r, TILEDB_READ); @@ -1763,25 +1560,17 @@ TEST_CASE( CHECK(query_r.query_status() == Query::Status::INCOMPLETE); CHECK(query_r.result_buffer_elements()["a"].second == 0); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2D, string, read/write in global order", - "[cppapi][hilbert][string][write][read][global-order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][string][write][read][global-order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); // Write array std::vector buff_a = {2, 3, 1, 4}; @@ -1808,8 +1597,7 @@ TEST_CASE( query_w.set_offsets_buffer("d1", off_d1); query_w.set_data_buffer("d2", buff_d2); query_w.set_offsets_buffer("d2", off_d2); - CHECK_NOTHROW(query_w.submit()); - query_w.finalize(); + CHECK_NOTHROW(query_w.submit_and_finalize()); array_w.close(); // Read @@ -1850,27 +1638,19 @@ TEST_CASE( CHECK(r_buff_d2 == c_buff_d2); CHECK(r_off_d2 == c_off_d2); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, string, multiple fragments, read in " "global order", "[cppapi][hilbert][2d][string][read][multiple-fragments][global-" - "order]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "order][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); // Write first fragment std::vector buff_a = {2, 3, 1, 4}; @@ -1879,7 +1659,14 @@ TEST_CASE( auto buff_d2 = std::string("stockstopt1cat"); std::vector off_d2 = {0, 5, 9, 11}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; @@ -1888,7 +1675,14 @@ TEST_CASE( buff_d2 = std::string("aceyellowredgrey"); off_d2 = {0, 3, 9, 12}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); // Read Array array_r(ctx, array_name, TILEDB_READ); @@ -1932,15 +1726,11 @@ TEST_CASE( CHECK(r_off_d1 == c_off_d1); CHECK(r_buff_d2 == c_buff_d2); CHECK(r_off_d2 == c_off_d2); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, string, consolidation", - "[cppapi][hilbert][2d][string][consolidation]") { + "[cppapi][hilbert][2d][string][consolidation][non-rest]") { Context ctx; VFS vfs(ctx); std::string array_name = "hilbert_array"; @@ -1950,7 +1740,7 @@ TEST_CASE( CHECK_NOTHROW(vfs.remove_dir(array_name)); // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); // Write first fragment std::vector buff_a = {2, 3, 1, 4}; @@ -1959,7 +1749,14 @@ TEST_CASE( auto buff_d2 = std::string("stockstopt1cat"); std::vector off_d2 = {0, 5, 9, 11}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); // Write second fragment buff_a = {5, 6, 7, 8}; @@ -1968,7 +1765,14 @@ TEST_CASE( buff_d2 = std::string("aceyellowredgrey"); off_d2 = {0, 3, 9, 12}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); // Consolidate and vacuum Config config; @@ -2030,17 +1834,13 @@ TEST_CASE( TEST_CASE( "C++ API: Test Hilbert, 2d, string, slicing", - "[cppapi][hilbert][2d][string][read][slicing]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][string][read][slicing][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); std::vector buff_a = {2, 3, 1, 4}; auto buff_d1 = std::string("cameldog331a"); @@ -2048,7 +1848,14 @@ TEST_CASE( auto buff_d2 = std::string("stockstopt1cat"); std::vector off_d2 = {0, 5, 9, 11}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); SECTION("- Row-major") { // Read @@ -2134,25 +1941,17 @@ TEST_CASE( CHECK(r_buff_d2 == c_buff_d2); CHECK(r_off_d2 == c_off_d2); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, string, 2D, partitioner", - "[cppapi][hilbert][2d][string][partitioner]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][2d][string][partitioner][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); // Write std::vector buff_a = {2, 3, 1, 4}; @@ -2161,7 +1960,14 @@ TEST_CASE( auto buff_d2 = std::string("stockstopt1cat"); std::vector off_d2 = {0, 5, 9, 11}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); SECTION("- entire domain") { // Read array @@ -2368,25 +2174,17 @@ TEST_CASE( array_r.close(); } - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } TEST_CASE( "C++ API: Test Hilbert, 2d, string, unsplittable", - "[cppapi][hilbert][read][2d][string][unsplittable]") { - Context ctx; - VFS vfs(ctx); - std::string array_name = "hilbert_array"; - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); + "[cppapi][hilbert][read][2d][string][unsplittable][rest]") { + VFSTestSetup vfs_test_setup{}; + Context ctx{vfs_test_setup.ctx()}; + auto array_name{vfs_test_setup.array_uri("hilbert_array")}; // Create array - create_string_array(array_name); + create_string_array(ctx, array_name); // Write std::vector buff_a = {2, 3, 1, 4}; @@ -2395,7 +2193,14 @@ TEST_CASE( auto buff_d2 = std::string("stockstopt1cat"); std::vector off_d2 = {0, 5, 9, 11}; write_2d_array( - array_name, off_d1, buff_d1, off_d2, buff_d2, buff_a, TILEDB_UNORDERED); + ctx, + array_name, + off_d1, + buff_d1, + off_d2, + buff_d2, + buff_a, + TILEDB_UNORDERED); // Read Array array_r(ctx, array_name, TILEDB_READ); @@ -2417,8 +2222,4 @@ TEST_CASE( CHECK(query_r.query_status() == Query::Status::INCOMPLETE); CHECK(query_r.result_buffer_elements()["a"].second == 0); array_r.close(); - - // Remove array - if (vfs.is_dir(array_name)) - CHECK_NOTHROW(vfs.remove_dir(array_name)); } diff --git a/test/src/unit-cppapi-nullable.cc b/test/src/unit-cppapi-nullable.cc index 4aec4688d273..b2ae676464e4 100644 --- a/test/src/unit-cppapi-nullable.cc +++ b/test/src/unit-cppapi-nullable.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/misc/utils.h" @@ -44,10 +45,6 @@ using namespace tiledb::test; class NullableArrayCppFx { public: - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - template struct test_dim_t { test_dim_t( @@ -113,12 +110,6 @@ class NullableArrayCppFx { vector* validity_bytemap_; }; - /** Constructor. */ - NullableArrayCppFx(); - - /** Destructor. */ - ~NullableArrayCppFx(); - /** * Creates, writes, and reads nullable attributes. * @@ -140,19 +131,6 @@ class NullableArrayCppFx { /** The C++ API context object. */ Context ctx_; - /** The C++ API VFS object. */ - VFS vfs_; - - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - - /** - * Removes a directory using `vfs_`. - * - * @param path The directory path. - */ - void remove_dir(const string& path); - /** * Creates a TileDB array. * @@ -199,18 +177,6 @@ class NullableArrayCppFx { const vector& subarray); }; -NullableArrayCppFx::NullableArrayCppFx() - : vfs_(ctx_) { -} - -NullableArrayCppFx::~NullableArrayCppFx() { -} - -void NullableArrayCppFx::remove_dir(const string& path) { - if (vfs_.is_dir(path)) - vfs_.remove_dir(path); -} - template void NullableArrayCppFx::create_array( const string& array_name, @@ -219,8 +185,6 @@ void NullableArrayCppFx::create_array( const vector>& test_attrs, const tiledb_layout_t cell_order, const tiledb_layout_t tile_order) { - remove_dir(array_name); - // Create the domain. Domain domain(ctx_); @@ -298,14 +262,11 @@ void NullableArrayCppFx::write( } // Submit query - auto rc = submit_query_wrapper( - ctx_, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } // Clean up array.close(); @@ -372,7 +333,9 @@ void NullableArrayCppFx::do_2d_nullable_test( const tiledb_layout_t cell_order, const tiledb_layout_t tile_order, const tiledb_layout_t write_order) { - const string array_name = "cpp_2d_nullable_array"; + VFSTestSetup vfs_test_setup{}; + ctx_ = vfs_test_setup.ctx(); + auto array_name{vfs_test_setup.array_uri("cpp_2d_nullable_array")}; // Skip row-major and col-major writes for sparse arrays. if (array_type == TILEDB_SPARSE && @@ -552,29 +515,16 @@ void NullableArrayCppFx::do_2d_nullable_test( expected_a3_read_buffer_validity.data(), a3_read_buffer_validity.size())); } - - remove_dir(array_name); } TEST_CASE_METHOD( NullableArrayCppFx, "C++ API: Test 2D array with nullable attributes", - "[cppapi][2d][nullable]") { + "[cppapi][2d][nullable][rest]") { vector> attrs; attrs.emplace_back("a1", false /* var */, true /* nullable */); attrs.emplace_back("a2", false /* var */, true /* nullable */); attrs.emplace_back("a3", true /* var */, true /* nullable */); - - SECTION("no serialization") { - serialize_ = false; - } - SECTION("serialization enabled global order write") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - } - for (auto attr_iter = attrs.begin(); attr_iter != attrs.end(); ++attr_iter) { vector> test_attrs(attrs.begin(), attr_iter + 1); for (const tiledb_array_type_t array_type : {TILEDB_DENSE, TILEDB_SPARSE}) { diff --git a/test/src/unit-cppapi-partial-attribute-write.cc b/test/src/unit-cppapi-partial-attribute-write.cc index c62ebd8c8d0c..4b81ee8a357a 100644 --- a/test/src/unit-cppapi-partial-attribute-write.cc +++ b/test/src/unit-cppapi-partial-attribute-write.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" @@ -42,19 +43,14 @@ using namespace tiledb::test; /** Tests for CPP API partial attribute write. */ struct CppPartialAttrWriteFx { - // Constants. - const std::string ARRAY_NAME = "test_partial_attr_write_array"; + VFSTestSetup vfs_test_setup_; // TileDB context. Context ctx_; - VFS vfs_; - - // Buffers to allocate on server side for serialized queries - test::ServerQueryBuffers server_buffers_; + const std::string array_name_; // Constructors/destructors. CppPartialAttrWriteFx(); - ~CppPartialAttrWriteFx(); // Functions. void create_sparse_array(bool allows_dups = false); @@ -63,8 +59,6 @@ struct CppPartialAttrWriteFx { std::vector dim1, std::vector dim2, uint64_t timestamp, - const bool serialized, - const bool refactored_query_v2, std::unique_ptr& array, std::unique_ptr& query); void write_sparse_dims_and_a1( @@ -73,40 +67,23 @@ struct CppPartialAttrWriteFx { std::vector dim2, std::vector a1, uint64_t timestamp, - const bool serialized, - const bool refactored_query_v2, std::unique_ptr& array, std::unique_ptr& query); - void write_sparse_a1( - Query& query, - std::vector a1, - const bool serialized, - const bool refactored_query_v2); - void write_sparse_a2( - Query& query, - std::vector a2, - const bool serialized, - const bool refactored_query_v2); + void write_sparse_a1(Query& query, std::vector a1); + void write_sparse_a2(Query& query, std::vector a2); void read_sparse( std::vector& a1, std::vector& a2, std::vector& dim1, std::vector& dim2); - - void remove_array(); - void remove_array(const std::string& array_name); - bool is_array(const std::string& array_name); }; CppPartialAttrWriteFx::CppPartialAttrWriteFx() - : vfs_(ctx_) { + : array_name_(vfs_test_setup_.array_uri("test_partial_attr_write_array")) { Config config; config["sm.allow_separate_attribute_writes"] = "true"; - ctx_ = Context(config); - vfs_ = VFS(ctx_); -} - -CppPartialAttrWriteFx::~CppPartialAttrWriteFx() { + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx(); } void CppPartialAttrWriteFx::create_sparse_array(bool allows_dups) { @@ -140,7 +117,7 @@ void CppPartialAttrWriteFx::create_sparse_array(bool allows_dups) { filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - Array::create(ARRAY_NAME, schema); + Array::create(array_name_, schema); } void CppPartialAttrWriteFx::write_sparse_dims( @@ -148,12 +125,10 @@ void CppPartialAttrWriteFx::write_sparse_dims( std::vector dim1, std::vector dim2, uint64_t timestamp, - const bool serialized, - const bool refactored_query_v2, std::unique_ptr& array, std::unique_ptr& query) { // Open array. - array = std::make_unique(ctx_, ARRAY_NAME, TILEDB_WRITE, timestamp); + array = std::make_unique(ctx_, array_name_, TILEDB_WRITE, timestamp); // Create query. query = std::make_unique(ctx_, *array, TILEDB_WRITE); @@ -161,15 +136,7 @@ void CppPartialAttrWriteFx::write_sparse_dims( query->set_data_buffer("d1", dim1); query->set_data_buffer("d2", dim2); - CHECK( - TILEDB_OK == test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - query.get(), - server_buffers_, - serialized, - refactored_query_v2, - false)); + query->submit(); } void CppPartialAttrWriteFx::write_sparse_dims_and_a1( @@ -178,12 +145,10 @@ void CppPartialAttrWriteFx::write_sparse_dims_and_a1( std::vector dim2, std::vector a1, uint64_t timestamp, - const bool serialized, - const bool refactored_query_v2, std::unique_ptr& array, std::unique_ptr& query) { // Open array. - array = std::make_unique(ctx_, ARRAY_NAME, TILEDB_WRITE, timestamp); + array = std::make_unique(ctx_, array_name_, TILEDB_WRITE, timestamp); // Create query. query = std::make_unique(ctx_, *array, TILEDB_WRITE); @@ -192,51 +157,19 @@ void CppPartialAttrWriteFx::write_sparse_dims_and_a1( query->set_data_buffer("d2", dim2); query->set_data_buffer("a1", a1); - CHECK( - TILEDB_OK == test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - query.get(), - server_buffers_, - serialized, - refactored_query_v2, - false)); + query->submit(); } -void CppPartialAttrWriteFx::write_sparse_a1( - Query& query, - std::vector a1, - const bool serialized, - const bool refactored_query_v2) { +void CppPartialAttrWriteFx::write_sparse_a1(Query& query, std::vector a1) { query.set_data_buffer("a1", a1); - CHECK( - TILEDB_OK == test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - &query, - server_buffers_, - serialized, - refactored_query_v2, - false)); + query.submit(); } void CppPartialAttrWriteFx::write_sparse_a2( - Query& query, - std::vector a2, - const bool serialized, - const bool refactored_query_v2) { + Query& query, std::vector a2) { query.set_data_buffer("a2", a2); - - CHECK( - TILEDB_OK == test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - &query, - server_buffers_, - serialized, - refactored_query_v2, - false)); + query.submit(); } void CppPartialAttrWriteFx::read_sparse( @@ -245,7 +178,7 @@ void CppPartialAttrWriteFx::read_sparse( std::vector& dim1, std::vector& dim2) { // Open array. - Array array(ctx_, ARRAY_NAME, TILEDB_READ); + Array array(ctx_, array_name_, TILEDB_READ); // Create query. Query query(ctx_, array, TILEDB_READ); @@ -263,32 +196,15 @@ void CppPartialAttrWriteFx::read_sparse( array.close(); } -void CppPartialAttrWriteFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) { - return; - } - - vfs_.remove_dir(array_name); -} - -void CppPartialAttrWriteFx::remove_array() { - remove_array(ARRAY_NAME); -} - -bool CppPartialAttrWriteFx::is_array(const std::string& array_name) { - return vfs_.is_dir(array_name); -} - TEST_CASE_METHOD( CppPartialAttrWriteFx, "CPP API: Test partial attribute write, not all dimensions set on first " "write", "[cppapi][partial-attribute-write][not-all-dims-set]") { - remove_array(); create_sparse_array(); // Open array. - Array array(ctx_, ARRAY_NAME, TILEDB_WRITE); + Array array(ctx_, array_name_, TILEDB_WRITE); // Create query. std::vector dim1(10); @@ -300,22 +216,12 @@ TEST_CASE_METHOD( "Query: [check_buffer_names] Dimension buffer d2 is not set"); array.close(); - - remove_array(); } TEST_CASE_METHOD( CppPartialAttrWriteFx, "CPP API: Test partial attribute write, basic test", - "[cppapi][partial-attribute-write][basic]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - remove_array(); + "[cppapi][partial-attribute-write][basic][rest]") { create_sparse_array(); // Write fragment, seperating dimensions and attributes. @@ -326,14 +232,10 @@ TEST_CASE_METHOD( {1, 1, 1, 2, 3, 4, 3, 3}, {1, 2, 4, 3, 1, 2, 3, 4}, 1, - serialized, - refactored_query_v2, array, query); - write_sparse_a1( - *query, {0, 1, 2, 3, 4, 5, 6, 7}, serialized, refactored_query_v2); - write_sparse_a2( - *query, {8, 9, 10, 11, 12, 13, 14, 15}, serialized, refactored_query_v2); + write_sparse_a1(*query, {0, 1, 2, 3, 4, 5, 6, 7}); + write_sparse_a2(*query, {8, 9, 10, 11, 12, 13, 14, 15}); query->finalize(); array->close(); @@ -348,22 +250,12 @@ TEST_CASE_METHOD( CHECK(a2 == std::vector({8, 9, 10, 11, 12, 13, 14, 15})); CHECK(dim1 == std::vector({1, 1, 1, 2, 3, 4, 3, 3})); CHECK(dim2 == std::vector({1, 2, 4, 3, 1, 2, 3, 4})); - - remove_array(); } TEST_CASE_METHOD( CppPartialAttrWriteFx, "CPP API: Test partial attribute write, basic test 2", - "[cppapi][partial-attribute-write][basic2]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - remove_array(); + "[cppapi][partial-attribute-write][basic2][rest]") { create_sparse_array(); // Write fragment, seperating dimensions and attributes. @@ -375,12 +267,9 @@ TEST_CASE_METHOD( {1, 2, 4, 3, 1, 2, 3, 4}, {0, 1, 2, 3, 4, 5, 6, 7}, 1, - serialized, - refactored_query_v2, array, query); - write_sparse_a2( - *query, {8, 9, 10, 11, 12, 13, 14, 15}, serialized, refactored_query_v2); + write_sparse_a2(*query, {8, 9, 10, 11, 12, 13, 14, 15}); query->finalize(); array->close(); @@ -395,22 +284,12 @@ TEST_CASE_METHOD( CHECK(a2 == std::vector({8, 9, 10, 11, 12, 13, 14, 15})); CHECK(dim1 == std::vector({1, 1, 1, 2, 3, 4, 3, 3})); CHECK(dim2 == std::vector({1, 2, 4, 3, 1, 2, 3, 4})); - - remove_array(); } TEST_CASE_METHOD( CppPartialAttrWriteFx, "CPP API: Test partial attribute write, rewrite", - "[cppapi][partial-attribute-write][rewrite]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - remove_array(); + "[cppapi][partial-attribute-write][rewrite][rest]") { create_sparse_array(); // Write fragment. @@ -421,24 +300,16 @@ TEST_CASE_METHOD( {1, 1, 1, 2, 3, 4, 3, 3}, {1, 2, 4, 3, 1, 2, 3, 4}, 1, - serialized, - refactored_query_v2, array, query); - write_sparse_a1( - *query, {0, 1, 2, 3, 4, 5, 6, 7}, serialized, refactored_query_v2); + write_sparse_a1(*query, {0, 1, 2, 3, 4, 5, 6, 7}); // Try to rewrite an attribute, will throw an exception. CHECK_THROWS_WITH( - write_sparse_a1( - *query, - {8, 9, 10, 11, 12, 13, 14, 15}, - serialized, - refactored_query_v2), + write_sparse_a1(*query, {8, 9, 10, 11, 12, 13, 14, 15}), "[TileDB::Query] Error: Buffer a1 was already written"); - write_sparse_a2( - *query, {8, 9, 10, 11, 12, 13, 14, 15}, serialized, refactored_query_v2); + write_sparse_a2(*query, {8, 9, 10, 11, 12, 13, 14, 15}); query->finalize(); array->close(); @@ -454,22 +325,12 @@ TEST_CASE_METHOD( CHECK(a2 == std::vector({8, 9, 10, 11, 12, 13, 14, 15})); CHECK(dim1 == std::vector({1, 1, 1, 2, 3, 4, 3, 3})); CHECK(dim2 == std::vector({1, 2, 4, 3, 1, 2, 3, 4})); - - remove_array(); } TEST_CASE_METHOD( CppPartialAttrWriteFx, "CPP API: Test partial attribute write, missing attributes", - "[cppapi][partial-attribute-write][missing-attributes]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(true, false); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - remove_array(); + "[cppapi][partial-attribute-write][missing-attributes][rest]") { create_sparse_array(); // Write fragment, seperating dimensions and attributes. @@ -480,14 +341,15 @@ TEST_CASE_METHOD( {1, 1, 1, 2, 3, 4, 3, 3}, {1, 2, 4, 3, 1, 2, 3, 4}, 1, - serialized, - refactored_query_v2, array, query); - write_sparse_a1( - *query, {0, 1, 2, 3, 4, 5, 6, 7}, serialized, refactored_query_v2); - CHECK_THROWS_WITH( - query->finalize(), "UnorderWriter: Not all buffers already written"); + write_sparse_a1(*query, {0, 1, 2, 3, 4, 5, 6, 7}); + if (vfs_test_setup_.is_rest()) { + CHECK_THROWS(query->finalize()); + } else { + CHECK_THROWS_WITH( + query->finalize(), "UnorderWriter: Not all buffers already written"); + } array->close(); size_t buffer_size = 8; @@ -501,6 +363,4 @@ TEST_CASE_METHOD( CHECK(a2 == std::vector({0, 0, 0, 0, 0, 0, 0, 0})); CHECK(dim1 == std::vector({0, 0, 0, 0, 0, 0, 0, 0})); CHECK(dim2 == std::vector({0, 0, 0, 0, 0, 0, 0, 0})); - - remove_array(); } diff --git a/test/src/unit-cppapi-subarray.cc b/test/src/unit-cppapi-subarray.cc index cceacda7adbc..ffe46e7ed5af 100644 --- a/test/src/unit-cppapi-subarray.cc +++ b/test/src/unit-cppapi-subarray.cc @@ -33,6 +33,7 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/serialization_wrappers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/misc/utils.h" @@ -423,23 +424,23 @@ TEST_CASE("C++ API: Test subarray (dense)", "[cppapi][dense][subarray]") { } } -#ifdef TILEDB_SERIALIZATION TEST_CASE( "C++ API: Test serialized OOB subarray error (dense)", - "[cppapi][dense][subarray][oob][serialization]") { - bool query_v2 = GENERATE(true, false); - - std::string array_name = "cpp_unit_array"; + "[cppapi][dense][subarray][oob][serialization][rest]") { + auto reader_type = GENERATE("legacy", "refactored"); tiledb::Config cfg; - cfg.set("config.logging_level", "3"); - cfg["sm.query.dense.reader"] = GENERATE("legacy", "refactored"); - tiledb::Context ctx(cfg); + cfg["sm.query.dense.reader"] = reader_type; + tiledb::test::VFSTestSetup vfs_test_setup(cfg.ptr().get()); - VFS vfs(ctx); - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); + // SC-45976 : Different behaviour between local and remote arrays when + // submitting query with ranges oob of domain + if (!vfs_test_setup.is_rest()) { + return; } + tiledb::Context ctx(vfs_test_setup.ctx()); + std::string array_name = vfs_test_setup.array_uri("cpp_unit_array"); + // Create tiledb::Domain domain(ctx); domain.add_dimension(tiledb::Dimension::create(ctx, "rows", {{0, 3}}, 4)) @@ -499,21 +500,14 @@ TEST_CASE( std::vector a(4); query.set_data_buffer("a", a); - tiledb::test::ServerQueryBuffers buffers; - CHECK( - submit_query_wrapper( - ctx, array_name, &query, buffers, true, query_v2, false) == expected); - if (expected == TILEDB_OK) { + CHECK_NOTHROW(query.submit()); CHECK(query.query_status() == tiledb::Query::Status::COMPLETE); CHECK(a == std::vector{1, 2, 3, 4}); - } - - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); + } else { + CHECK_THROWS(query.submit()); } } -#endif TEST_CASE( "C++ API: Test subarray (incomplete) - Subarray-query", diff --git a/test/src/unit-cppapi-updates.cc b/test/src/unit-cppapi-updates.cc index bf0fcb5d8182..81d92fefe650 100644 --- a/test/src/unit-cppapi-updates.cc +++ b/test/src/unit-cppapi-updates.cc @@ -32,29 +32,18 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" using namespace tiledb; TEST_CASE( "C++ API updates: test writing two identical fragments", - "[updates][updates-identical-fragments]") { - bool serialized = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialized = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialized = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - const std::string array_name = "updates_identical_fragments"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "[updates][updates-identical-fragments][rest]") { + test::VFSTestSetup vfs_test_setup; + const std::string array_name = + vfs_test_setup.array_uri("updates_identical_fragments"); + Context ctx{vfs_test_setup.ctx()}; // Create const int rowmin = 0, rowmax = 9, row_ext = rowmax - rowmin + 1; @@ -90,15 +79,7 @@ TEST_CASE( .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a1", data_a1) .set_offsets_buffer("a1", offsets_a1); - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w1, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_w1.submit(); array_w1.close(); // Second write @@ -110,14 +91,7 @@ TEST_CASE( .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a1", data_a1) .set_offsets_buffer("a1", offsets_a1); - rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w2, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + query_w2.submit(); array_w2.close(); // Read @@ -134,41 +108,21 @@ TEST_CASE( .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a1", r_data_a1) .set_offsets_buffer("a1", r_offsets_a1); - rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialized, - refactored_query_v2, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); REQUIRE(query.query_status() == Query::Status::COMPLETE); array.close(); for (int i = 0; i < (int)data_a1.size(); i++) REQUIRE(r_data_a1[i] == i); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE( - "C++ API updates: empty second write", "[updates][updates-empty-write]") { - bool serialized = false, refactored_query_v2 = false; -#ifdef TILEDB_SERIALIZATION - serialized = GENERATE(false, true); - if (serialized) { - refactored_query_v2 = GENERATE(true, false); - } -#endif - - const std::string array_name = "updates_empty_write"; - Context ctx; - VFS vfs(ctx); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + "C++ API updates: empty second write", + "[updates][updates-empty-write][rest-fails][sc-45709]") { + test::VFSTestSetup vfs_test_setup; + const std::string array_name = + vfs_test_setup.array_uri("updates_empty_write"); + Context ctx{vfs_test_setup.ctx()}; // Create Domain domain(ctx); @@ -188,16 +142,11 @@ TEST_CASE( query_w1.set_layout(layout).set_data_buffer("d", data).set_offsets_buffer( "d", offsets); - // Submit query - test::ServerQueryBuffers server_buffers_; - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w1, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query_w1.submit_and_finalize(); + } else { + query_w1.submit(); + } array_w1.close(); @@ -208,18 +157,11 @@ TEST_CASE( query_w2.set_layout(layout).set_data_buffer("d", data).set_offsets_buffer( "d", offsets); - // Submit query - rc = test::submit_query_wrapper( - ctx, - array_name, - &query_w2, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query_w2.submit_and_finalize(); + } else { + query_w2.submit(); + } array_w2.close(); - - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } diff --git a/test/src/unit-cppapi-var-offsets.cc b/test/src/unit-cppapi-var-offsets.cc index 2d0c3fbaac13..8d7832b625b1 100644 --- a/test/src/unit-cppapi-var-offsets.cc +++ b/test/src/unit-cppapi-var-offsets.cc @@ -33,25 +33,15 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/cpp_api/tiledb" using namespace tiledb; struct VariableOffsetsFx { - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - test::ServerQueryBuffers server_buffers_; - - void create_sparse_array(const std::string& array_name) { - Context ctx; - VFS vfs(ctx); - - // Create the array - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); + test::VFSTestSetup vfs_test_setup_; + void create_sparse_array(Context ctx, const std::string& array_name) { Domain dom(ctx); dom.add_dimension(Dimension::create(ctx, "d1", {{1, 4}}, 2)) .add_dimension(Dimension::create(ctx, "d2", {{1, 4}}, 2)); @@ -86,14 +76,11 @@ struct VariableOffsetsFx { query.set_offsets_buffer("attr", data_offsets); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -118,14 +105,11 @@ struct VariableOffsetsFx { reinterpret_cast(data_offsets.data()), data_offsets.size()); - /* TODO: enable this when sc21681 is fixed - // Submit query - auto rc = test::submit_query_wrapper( - ctx, array_name, &query, server_buffers_, serialize_, - refactored_query_v2_); REQUIRE(rc == TILEDB_OK); - */ - CHECK_NOTHROW(query.submit()); - query.finalize(); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -147,15 +131,7 @@ struct VariableOffsetsFx { query.set_offsets_buffer("attr", attr_off); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element offsets are properly returned CHECK(attr_val == expected_data); @@ -182,12 +158,6 @@ struct VariableOffsetsFx { query.set_offsets_buffer( "attr", reinterpret_cast(attr_off.data()), attr_off.size()); - /* TODO: enable this when sc21681 is fixed - // Submit query - auto rc = test::submit_query_wrapper( - ctx, array_name, &query, server_buffers_, serialize_, - refactored_query_v2_, false); REQUIRE(rc == TILEDB_OK); - */ CHECK_NOTHROW(query.submit()); // Check the element offsets are properly returned @@ -223,30 +193,16 @@ struct VariableOffsetsFx { query.set_offsets_buffer("attr", attr_off); // Check that first partial read returns expected results - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); + Query::Status status = query.query_status(); CHECK(status == Query::Status::INCOMPLETE); CHECK(attr_val == exp_data_part1); CHECK(attr_off == exp_off_part1); // Check that second partial read returns expected results - rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); + status = query.query_status(); CHECK(status == Query::Status::COMPLETE); CHECK(attr_val == exp_data_part2); @@ -274,15 +230,7 @@ struct VariableOffsetsFx { query.add_range("d2", d2_start, d2_end); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element offsets are properly returned uint64_t offset_elem_num = 0, data_vals_num = 0, validity_elem_num = 0; @@ -295,14 +243,7 @@ struct VariableOffsetsFx { array.close(); } - void create_dense_array(const std::string& array_name) { - Context ctx; - VFS vfs(ctx); - - // Create the array - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); - + void create_dense_array(Context ctx, const std::string& array_name) { Domain dom(ctx); dom.add_dimension(Dimension::create(ctx, "d1", {{1, 4}}, 2)) .add_dimension(Dimension::create(ctx, "d2", {{1, 4}}, 2)); @@ -353,14 +294,11 @@ struct VariableOffsetsFx { } // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -404,13 +342,11 @@ struct VariableOffsetsFx { Subarray(ctx, array).set_subarray({1, 2, 1, 2})); } - /* TODO: enable this when sc21681 is fixed - auto rc = test::submit_query_wrapper( - ctx, array_name, &query, server_buffers_, serialize_, - refactored_query_v2_); REQUIRE(rc == TILEDB_OK); - */ - CHECK_NOTHROW(query.submit()); - query.finalize(); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -441,15 +377,7 @@ struct VariableOffsetsFx { query.set_offsets_buffer("attr", attr_off); // Submit query - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); // Check the element offsets are properly returned CHECK(attr_val == expected_data); @@ -486,12 +414,6 @@ struct VariableOffsetsFx { query.set_offsets_buffer( "attr", reinterpret_cast(attr_off.data()), attr_off.size()); - /* TODO: enable this when sc21681 is fixed - // Submit query - auto rc = test::submit_query_wrapper( - ctx, array_name, &query, server_buffers_, serialize_, - refactored_query_v2_, false); REQUIRE(rc == TILEDB_OK); - */ CHECK_NOTHROW(query.submit()); query.finalize(); @@ -522,28 +444,12 @@ struct VariableOffsetsFx { query.set_offsets_buffer("attr", attr_off); // Check that first partial read returns expected results - auto rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(attr_val == exp_data_part1); CHECK(attr_off == exp_off_part1); // Check that second partial read returns expected results - rc = test::submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_, - false); - REQUIRE(rc == TILEDB_OK); + query.submit(); CHECK(attr_val == exp_data_part2); CHECK(attr_off == exp_off_part2); @@ -554,16 +460,12 @@ struct VariableOffsetsFx { TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test element offsets : sparse array", - "[var-offsets][element-offset][sparse]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_element_offset"; - create_sparse_array(array_name); + "[var-offsets][element-offset][sparse][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_element_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_sparse_array(ctx, array_name); std::vector data = {1, 2, 3, 4, 5, 6}; - Context ctx; SECTION("Byte offsets (default case)") { Config config = ctx.config(); @@ -608,7 +510,8 @@ TEST_CASE_METHOD( Config config; // Change config of offsets format from bytes to elements config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); std::vector element_offsets = {0, 1, 3, 5}; @@ -645,26 +548,17 @@ TEST_CASE_METHOD( } } } - - // Clean up - VFS vfs(ctx); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test element offsets : dense array", - "[var-offsets][element-offset][dense]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_element_offset"; - create_dense_array(array_name); + "[var-offsets][element-offset][dense][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_element_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_dense_array(ctx, array_name); std::vector data = {1, 2, 3, 4, 5, 6}; - Context ctx; SECTION("Byte offsets (default case)") { Config config = ctx.config(); @@ -687,7 +581,8 @@ TEST_CASE_METHOD( Config config; // Change config of offsets format from bytes to elements config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); std::vector element_offsets = {0, 1, 3, 5}; @@ -702,25 +597,16 @@ TEST_CASE_METHOD( read_and_check_dense_array(ctx, array_name, data, element_offsets); } } - - // Clean up - VFS vfs(ctx); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test offsets extra element: sparse array", - "[var-offsets][extra-offset][sparse]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_extra_offset"; - create_sparse_array(array_name); - - Context ctx; + "[var-offsets][extra-offset][sparse][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_extra_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_sparse_array(ctx, array_name); + std::vector data = {1, 2, 3, 4, 5, 6}; std::vector data_offsets = {0, 4, 12, 20}; std::vector element_offsets = {0, 1, 3, 5}; @@ -752,7 +638,8 @@ TEST_CASE_METHOD( SECTION("Byte offsets (default config)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of bytes data_offsets.push_back(sizeof(data[0]) * data.size()); @@ -793,7 +680,8 @@ TEST_CASE_METHOD( SECTION("Element offsets") { config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating the total number of elements element_offsets.push_back(data.size()); @@ -834,7 +722,8 @@ TEST_CASE_METHOD( SECTION("Query unwritten coordinates") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of bytes data_offsets.push_back(sizeof(data[0]) * data.size()); @@ -874,7 +763,8 @@ TEST_CASE_METHOD( } SECTION("User offsets buffer too small") { - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); Array array_w(ctx, array_name, TILEDB_WRITE); std::vector d1 = {1, 2, 3, 4}; @@ -1000,8 +890,8 @@ TEST_CASE_METHOD( SECTION("Byte offsets (default config)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); - Context ctx(config); - + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of bytes data_offsets.push_back(sizeof(data[0]) * data.size()); @@ -1081,7 +971,8 @@ TEST_CASE_METHOD( SECTION("Element offsets") { config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of elements element_offsets.push_back(data.size()); @@ -1103,25 +994,28 @@ TEST_CASE_METHOD( data_elem_off_part2, TILEDB_ROW_MAJOR); } - SECTION("Global order read") { - partial_read_and_check_sparse_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2, - TILEDB_GLOBAL_ORDER); - } - SECTION("Unordered read") { - partial_read_and_check_sparse_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2, - TILEDB_UNORDERED); + // SC-45586 + if (!vfs_test_setup_.is_rest()) { + SECTION("Global order read") { + partial_read_and_check_sparse_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2, + TILEDB_GLOBAL_ORDER); + } + SECTION("Unordered read") { + partial_read_and_check_sparse_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2, + TILEDB_UNORDERED); + } } } SECTION("Global order write") { @@ -1137,25 +1031,28 @@ TEST_CASE_METHOD( data_elem_off_part2, TILEDB_ROW_MAJOR); } - SECTION("Global order read") { - partial_read_and_check_sparse_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2, - TILEDB_GLOBAL_ORDER); - } - SECTION("Unordered read") { - partial_read_and_check_sparse_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2, - TILEDB_UNORDERED); + // SC-45586 + if (!vfs_test_setup_.is_rest()) { + SECTION("Global order read") { + partial_read_and_check_sparse_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2, + TILEDB_GLOBAL_ORDER); + } + SECTION("Unordered read") { + partial_read_and_check_sparse_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2, + TILEDB_UNORDERED); + } } } } @@ -1167,7 +1064,8 @@ TEST_CASE_METHOD( ctx, array_name, data, data_offsets, TILEDB_UNORDERED); // Submit read query - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); @@ -1240,25 +1138,16 @@ TEST_CASE_METHOD( } } } - - // Clean up - VFS vfs(ctx); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test offsets extra element: dense array", - "[var-offsets][extra-offset][dense]") { -#ifdef TILEDB_SERIALIZATION - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_extra_offset"; - create_dense_array(array_name); - - Context ctx; + "[var-offsets][extra-offset][dense][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_extra_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_dense_array(ctx, array_name); + std::vector data = {1, 2, 3, 4, 5, 6}; std::vector data_offsets = {0, 4, 12, 20}; std::vector element_offsets = {0, 1, 3, 5}; @@ -1276,10 +1165,13 @@ TEST_CASE_METHOD( SECTION("Extra element") { config["sm.var_offsets.extra_element"] = "true"; + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); SECTION("Byte offsets (default config)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of bytes data_offsets.push_back(sizeof(data[0]) * data.size()); @@ -1298,8 +1190,8 @@ TEST_CASE_METHOD( SECTION("Element offsets") { config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); - + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating the total number of // elements element_offsets.push_back(data.size()); @@ -1316,71 +1208,75 @@ TEST_CASE_METHOD( } } - SECTION("User offsets buffer too small") { - // Use element offsets to cover this code path as well - config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); - - Array array_w(ctx, array_name, TILEDB_WRITE); - Query query_w(ctx, array_w, TILEDB_WRITE); - query_w.set_layout(TILEDB_ROW_MAJOR) - .set_subarray( - Subarray(ctx, array_w).set_subarray({1, 2, 1, 2})); - - // Try to write without allocating memory for the extra element - query_w.set_data_buffer("attr", data); - query_w.set_offsets_buffer("attr", element_offsets); - CHECK_THROWS(query_w.submit()); - - // Write data with extra element - element_offsets.push_back(data.size()); - query_w.set_data_buffer("attr", data); - query_w.set_offsets_buffer("attr", element_offsets); - CHECK_NOTHROW(query_w.submit()); - array_w.close(); - - // Submit read query - Array array_r(ctx, array_name, TILEDB_READ); - Query query_r(ctx, array_r, TILEDB_READ); - - // Assume no size for the extra element - std::vector attr_val(data.size()); - std::vector attr_off(element_offsets.size() - 1); - query_r.set_data_buffer("attr", attr_val); - query_r.set_offsets_buffer("attr", attr_off); - query_r.set_subarray( - Subarray(ctx, array_r).set_subarray({1, 2, 1, 2})); - - // First partial read because offsets don't fit - CHECK_NOTHROW(query_r.submit()); - CHECK(query_r.query_status() == Query::Status::INCOMPLETE); - std::vector data_exp1 = {1, 2, 3, 0, 0, 0}; - std::vector data_off_exp1 = {0, 1, 3, 0}; - // check returned data - auto data_num = query_r.result_buffer_elements()["attr"].second; - CHECK(data_num == 3); - CHECK(attr_val == data_exp1); - // check returned offsets - auto offset_num = query_r.result_buffer_elements()["attr"].first; - CHECK(offset_num == 3); - CHECK(attr_off == data_off_exp1); - - // Second partial read - reset_read_buffers(attr_val, attr_off); - CHECK_NOTHROW(query_r.submit()); - CHECK(query_r.query_status() == Query::Status::COMPLETE); - std::vector data_exp2 = {4, 5, 6, 0, 0, 0}; - std::vector data_off_exp2 = {0, 2, 3, 0}; - // check returned data - data_num = query_r.result_buffer_elements()["attr"].second; - CHECK(data_num == 3); - CHECK(attr_val == data_exp2); - // check returned offsets - offset_num = query_r.result_buffer_elements()["attr"].first; - CHECK(offset_num == 3); - CHECK(attr_off == data_off_exp2); - - array_r.close(); + // SC-45586 + if (!vfs_test_setup_.is_rest()) { + SECTION("User offsets buffer too small") { + // Use element offsets to cover this code path as well + config["sm.var_offsets.mode"] = "elements"; + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); + + Array array_w(ctx, array_name, TILEDB_WRITE); + Query query_w(ctx, array_w, TILEDB_WRITE); + query_w.set_layout(TILEDB_ROW_MAJOR) + .set_subarray( + Subarray(ctx, array_w).set_subarray({1, 2, 1, 2})); + + // Try to write without allocating memory for the extra element + query_w.set_data_buffer("attr", data); + query_w.set_offsets_buffer("attr", element_offsets); + CHECK_THROWS(query_w.submit()); + + // Write data with extra element + element_offsets.push_back(data.size()); + query_w.set_data_buffer("attr", data); + query_w.set_offsets_buffer("attr", element_offsets); + CHECK_NOTHROW(query_w.submit()); + array_w.close(); + + // Submit read query + Array array_r(ctx, array_name, TILEDB_READ); + Query query_r(ctx, array_r, TILEDB_READ); + + // Assume no size for the extra element + std::vector attr_val(data.size()); + std::vector attr_off(element_offsets.size() - 1); + query_r.set_data_buffer("attr", attr_val); + query_r.set_offsets_buffer("attr", attr_off); + query_r.set_subarray( + Subarray(ctx, array_r).set_subarray({1, 2, 1, 2})); + + // First partial read because offsets don't fit + CHECK_NOTHROW(query_r.submit()); + CHECK(query_r.query_status() == Query::Status::INCOMPLETE); + std::vector data_exp1 = {1, 2, 3, 0, 0, 0}; + std::vector data_off_exp1 = {0, 1, 3, 0}; + // check returned data + auto data_num = query_r.result_buffer_elements()["attr"].second; + CHECK(data_num == 3); + CHECK(attr_val == data_exp1); + // check returned offsets + auto offset_num = query_r.result_buffer_elements()["attr"].first; + CHECK(offset_num == 3); + CHECK(attr_off == data_off_exp1); + + // Second partial read + reset_read_buffers(attr_val, attr_off); + CHECK_NOTHROW(query_r.submit()); + CHECK(query_r.query_status() == Query::Status::COMPLETE); + std::vector data_exp2 = {4, 5, 6, 0, 0, 0}; + std::vector data_off_exp2 = {0, 2, 3, 0}; + // check returned data + data_num = query_r.result_buffer_elements()["attr"].second; + CHECK(data_num == 3); + CHECK(attr_val == data_exp2); + // check returned offsets + offset_num = query_r.result_buffer_elements()["attr"].first; + CHECK(offset_num == 3); + CHECK(attr_off == data_off_exp2); + + array_r.close(); + } } } } @@ -1416,7 +1312,8 @@ TEST_CASE_METHOD( SECTION("Byte offsets (default config)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element indicating total number of bytes data_offsets.push_back(sizeof(data[0]) * data.size()); @@ -1424,7 +1321,6 @@ TEST_CASE_METHOD( // Expect an extra element offset on each read data_off_part1.push_back(sizeof(data_part1[0]) * data_part1.size()); data_off_part2.push_back(sizeof(data_part2[0]) * data_part2.size()); - SECTION("Ordered write") { write_dense_array( ctx, array_name, data, data_offsets, TILEDB_ROW_MAJOR); @@ -1448,44 +1344,48 @@ TEST_CASE_METHOD( data_off_part2); } } - - SECTION("Element offsets") { - config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); - - // Write data with extra element indicating total number of elements - element_offsets.push_back(data.size()); - - // Expect an extra element offset on each read - data_elem_off_part1.push_back(data_part1.size()); - data_elem_off_part2.push_back(data_part2.size()); - - SECTION("Ordered write") { - write_dense_array( - ctx, array_name, data, element_offsets, TILEDB_ROW_MAJOR); - partial_read_and_check_dense_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2); - } - SECTION("Global order write") { - write_dense_array( - ctx, array_name, data, element_offsets, TILEDB_GLOBAL_ORDER); - partial_read_and_check_dense_array( - ctx, - array_name, - data_part1, - data_elem_off_part1, - data_part2, - data_elem_off_part2); + // SC-45586 + if (!vfs_test_setup_.is_rest()) { + SECTION("Element offsets") { + config["sm.var_offsets.mode"] = "elements"; + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); + + // Write data with extra element indicating total number of elements + element_offsets.push_back(data.size()); + + // Expect an extra element offset on each read + data_elem_off_part1.push_back(data_part1.size()); + data_elem_off_part2.push_back(data_part2.size()); + + SECTION("Ordered write") { + write_dense_array( + ctx, array_name, data, element_offsets, TILEDB_ROW_MAJOR); + partial_read_and_check_dense_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2); + } + SECTION("Global order write") { + write_dense_array( + ctx, array_name, data, element_offsets, TILEDB_GLOBAL_ORDER); + partial_read_and_check_dense_array( + ctx, + array_name, + data_part1, + data_elem_off_part1, + data_part2, + data_elem_off_part2); + } } } SECTION("User offsets buffer too small") { - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Write data with extra element data_offsets.push_back(sizeof(data[0]) * data.size()); write_dense_array( @@ -1566,24 +1466,15 @@ TEST_CASE_METHOD( } } } - - // Clean up - VFS vfs(ctx); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test 32-bit offsets: sparse array", - "[var-offsets][32bit-offset][sparse]") { -#ifdef TILEDB_SERIALIZATION - /* TODO: set this to true this when sc21681 is fixed */ - serialize_ = false; - // refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_32bit_offset"; - create_sparse_array(array_name); + "[var-offsets][32bit-offset][sparse][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_32bit_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_sparse_array(ctx, array_name); std::vector data = {1, 2, 3, 4, 5, 6}; // Create 32 bit byte offsets buffer to use @@ -1592,7 +1483,8 @@ TEST_CASE_METHOD( Config config; // Change config of offsets bitsize from 64 to 32 config["sm.var_offsets.bitsize"] = 32; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); SECTION("Byte offsets (default case)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); @@ -1634,7 +1526,8 @@ TEST_CASE_METHOD( SECTION("Element offsets") { // Change config of offsets format from bytes to elements config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Create 32 bit element offsets buffer to use std::vector data_element_offsets = {0, 1, 3, 5}; @@ -1675,7 +1568,8 @@ TEST_CASE_METHOD( SECTION("Extra element") { config["sm.var_offsets.extra_element"] = "true"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Check the extra element is included in the offsets uint32_t data_size = static_cast(sizeof(data[0]) * data.size()); @@ -1714,28 +1608,15 @@ TEST_CASE_METHOD( } } } - - // Clean up - config["sm.var_offsets.extra_element"] = "false"; - config["sm.var_offsets.mode"] = "bytes"; - config["sm.var_offsets.bitsize"] = 64; - Context ctx2(config); - VFS vfs(ctx2); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test 32-bit offsets: dense array", - "[var-offsets][32bit-offset][dense]") { -#ifdef TILEDB_SERIALIZATION - /* TODO: set this to true this when sc21681 is fixed */ - serialize_ = false; - refactored_query_v2_ = GENERATE(true, false); -#endif - std::string array_name = "test_32bit_offset"; - create_dense_array(array_name); + "[var-offsets][32bit-offset][dense][rest]") { + std::string array_name = vfs_test_setup_.array_uri("test_32bit_offset"); + auto ctx = vfs_test_setup_.ctx(); + create_dense_array(ctx, array_name); std::vector data = {1, 2, 3, 4, 5, 6}; // Create 32 bit offsets byte buffer to use @@ -1744,7 +1625,8 @@ TEST_CASE_METHOD( Config config; // Change config of offsets bitsize from 64 to 32 config["sm.var_offsets.bitsize"] = 32; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); SECTION("Byte offsets (default case)") { CHECK((std::string)config["sm.var_offsets.mode"] == "bytes"); @@ -1764,7 +1646,8 @@ TEST_CASE_METHOD( SECTION("Element offsets") { // Change config of offsets format from bytes to elements config["sm.var_offsets.mode"] = "elements"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Create 32 bit element offsets buffer to use std::vector data_element_offsets = {0, 1, 3, 5}; @@ -1783,7 +1666,8 @@ TEST_CASE_METHOD( SECTION("Extra element") { config["sm.var_offsets.extra_element"] = "true"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); // Check the extra element is included in the offsets uint32_t data_size = static_cast(sizeof(data[0]) * data.size()); @@ -1800,22 +1684,15 @@ TEST_CASE_METHOD( read_and_check_dense_array(ctx, array_name, data, data_byte_offsets); } } - - // Clean up - config["sm.var_offsets.extra_element"] = "false"; - config["sm.var_offsets.mode"] = "bytes"; - config["sm.var_offsets.bitsize"] = 64; - Context ctx2(config); - VFS vfs(ctx2); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } TEST_CASE_METHOD( VariableOffsetsFx, "C++ API: Test 32-bit offsets: sparse array with string dimension", - "[var-offsets-dim][32bit-offset][sparse]") { - std::string array_name = "test_32bit_offset_string_dim"; + "[var-offsets-dim][32bit-offset][sparse][rest]") { + std::string array_name = + vfs_test_setup_.array_uri("test_32bit_offset_string_dim"); + auto ctx = vfs_test_setup_.ctx(); /* Write an array with string dimension and make sure we get back @@ -1829,7 +1706,6 @@ TEST_CASE_METHOD( // Create and write array { - Context ctx; Domain domain(ctx); domain.add_dimension( Dimension::create(ctx, "dim1", TILEDB_STRING_ASCII, nullptr, nullptr)); @@ -1857,7 +1733,8 @@ TEST_CASE_METHOD( config["sm.var_offsets.bitsize"] = 32; // Add extra element config["sm.var_offsets.extra_element"] = "true"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); std::vector offsets_back(5); std::string data_back; @@ -1888,7 +1765,8 @@ TEST_CASE_METHOD( config["sm.var_offsets.bitsize"] = 32; // Add extra element config["sm.var_offsets.extra_element"] = "true"; - Context ctx(config); + vfs_test_setup_.update_config(config.ptr().get()); + ctx = vfs_test_setup_.ctx(); std::vector offsets_back(14); @@ -1922,9 +1800,4 @@ TEST_CASE_METHOD( CHECK(offsets_back[idx] == guard_val); } } - - Context ctx; - VFS vfs(ctx); - if (vfs.is_dir(array_name)) - vfs.remove_dir(array_name); } diff --git a/test/src/unit-empty-var-length.cc b/test/src/unit-empty-var-length.cc index e1eb329a4c8b..0feba4cd43dc 100644 --- a/test/src/unit-empty-var-length.cc +++ b/test/src/unit-empty-var-length.cc @@ -32,6 +32,7 @@ #include #include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/cpp_api/tiledb" @@ -39,6 +40,7 @@ #include using namespace tiledb::test; +using namespace tiledb; float buffer_a1[4] = {0.0f, 0.1f, 0.2f, 0.3f}; int32_t buffer_a4[4] = {1, 2, 3, 4}; @@ -59,30 +61,25 @@ uint64_t UTF8_OFFSET_4_FOR_EMPTY = sizeof(UTF8_STRINGS_VAR_FOR_EMPTY) - UTF8_NULL_SIZE_FOR_EMPTY; struct StringEmptyFx { - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - // Buffers to allocate on server side for serialized queries - ServerQueryBuffers server_buffers_; - + VFSTestSetup vfs_test_setup_; + tiledb_ctx_t* ctx; + StringEmptyFx(); void create_array(const std::string& array_name); - void delete_array(const std::string& array_name); void read_array(const std::string& array_name); void write_array(const std::string& array_name); }; +StringEmptyFx::StringEmptyFx() + : ctx(vfs_test_setup_.ctx_c) { +} + // Create a simple dense 1D array with three string attributes void StringEmptyFx::create_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Create dimensions uint64_t dim_domain[] = {1, 8}; uint64_t tile_extent = 2; tiledb_dimension_t* d1; - rc = tiledb_dimension_alloc( + int rc = tiledb_dimension_alloc( ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extent, &d1); REQUIRE(rc == TILEDB_OK); @@ -162,15 +159,9 @@ void StringEmptyFx::create_array(const std::string& array_name) { tiledb_dimension_free(&d1); tiledb_domain_free(&domain); tiledb_array_schema_free(&array_schema); - tiledb_ctx_free(&ctx); } void StringEmptyFx::write_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Prepare buffers uint64_t buffer_a1_size = sizeof(buffer_a1); uint64_t buffer_a1_offsets[] = { @@ -227,7 +218,7 @@ void StringEmptyFx::write_array(const std::string& array_name) { // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); @@ -267,13 +258,7 @@ void StringEmptyFx::write_array(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit_and_finalize(ctx, query); REQUIRE(rc == TILEDB_OK); // Close array @@ -283,20 +268,14 @@ void StringEmptyFx::write_array(const std::string& array_name) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); std::free(buffer_a2); std::free(buffer_a3); } void StringEmptyFx::read_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - // Open array tiledb_array_t* array; - rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); + int rc = tiledb_array_alloc(ctx, array_name.c_str(), &array); CHECK(rc == TILEDB_OK); rc = tiledb_array_open(ctx, array, TILEDB_READ); CHECK(rc == TILEDB_OK); @@ -395,13 +374,7 @@ void StringEmptyFx::read_array(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); // Submit query - rc = submit_query_wrapper( - ctx, - array_name, - &query, - server_buffers_, - serialize_, - refactored_query_v2_); + rc = tiledb_query_submit(ctx, query); REQUIRE(rc == TILEDB_OK); // Check results @@ -446,7 +419,6 @@ void StringEmptyFx::read_array(const std::string& array_name) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); - tiledb_ctx_free(&ctx); std::free(buffer_d1); std::free(buffer_a1_off); std::free(buffer_a1_val); @@ -458,51 +430,23 @@ void StringEmptyFx::read_array(const std::string& array_name) { std::free(buffer_a4_val); } -void StringEmptyFx::delete_array(const std::string& array_name) { - // Create TileDB context - tiledb_ctx_t* ctx; - int rc = tiledb_ctx_alloc(nullptr, &ctx); - REQUIRE(rc == TILEDB_OK); - - // Remove array - tiledb_object_t type; - rc = tiledb_object_type(ctx, array_name.c_str(), &type); - REQUIRE(rc == TILEDB_OK); - if (type == TILEDB_ARRAY) { - rc = tiledb_object_remove(ctx, array_name.c_str()); - REQUIRE(rc == TILEDB_OK); - } - - // Clean up - tiledb_ctx_free(&ctx); -} - TEST_CASE_METHOD( - StringEmptyFx, "C API: Test empty support", "[capi][empty-var-length]") { - std::string array_name = "empty_string"; - - SECTION("no serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialize_ = true; - refactored_query_v2_ = GENERATE(true, false); - } -#endif - - delete_array(array_name); + StringEmptyFx, + "C API: Test empty support", + "[capi][empty-var-length][rest]") { + std::string array_name = vfs_test_setup_.array_uri("empty_string"); create_array(array_name); write_array(array_name); read_array(array_name); - delete_array(array_name); } struct StringEmptyFx2 { + VFSTestSetup vfs_test_setup_; + Context ctx; + StringEmptyFx2(); void create_array(const std::string& array_name); void write_array(const std::string& array_name); void read_array(const std::string& array_name); - void delete_array(const std::string& array_name); std::vector offsets = { 0, 4, 8, 11, 13, 14, 14, 17, 21, 24, 24, 24, 27, 32, 35, 38}; @@ -513,10 +457,10 @@ struct StringEmptyFx2 { '$', 'e', '1', '\x17', ' ', '1', '\x14', '('}; }; -void StringEmptyFx2::create_array(const std::string& array_name) { - using namespace tiledb; - Context ctx; +StringEmptyFx2::StringEmptyFx2() + : ctx(vfs_test_setup_.ctx()){}; +void StringEmptyFx2::create_array(const std::string& array_name) { Domain domain(ctx); domain.add_dimension(Dimension::create(ctx, "__dim_0", {0, 3}, 1)); domain.add_dimension(Dimension::create(ctx, "__dim_1", {0, 3}, 1)); @@ -528,19 +472,10 @@ void StringEmptyFx2::create_array(const std::string& array_name) { attr.set_cell_val_num(TILEDB_VAR_NUM); schema.add_attribute(attr); - VFS vfs(ctx); - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } Array::create(array_name, schema); } void StringEmptyFx2::write_array(const std::string& array_name) { - using namespace tiledb; - - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - auto array = tiledb::Array(ctx, array_name, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); @@ -555,11 +490,6 @@ void StringEmptyFx2::write_array(const std::string& array_name) { } void StringEmptyFx2::read_array(const std::string& array_name) { - using namespace tiledb; - - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - std::vector r_offsets(16); std::vector r_data(38); @@ -577,47 +507,36 @@ void StringEmptyFx2::read_array(const std::string& array_name) { REQUIRE(r_data == StringEmptyFx2::data); } -void StringEmptyFx2::delete_array(const std::string& array_name) { - using namespace tiledb; - - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - - if (Object::object(ctx, array_name).type() == Object::Type::Array) { - Object::remove(ctx, array_name); - } -} - TEST_CASE_METHOD( StringEmptyFx2, "C++ API: Test empty support", - "[cppapi][empty-var-length]") { - std::string array_name = "empty_string2"; - delete_array(array_name); + "[cppapi][empty-var-length][rest]") { + std::string array_name = vfs_test_setup_.array_uri("empty_string2"); create_array(array_name); write_array(array_name); read_array(array_name); - delete_array(array_name); } struct StringEmptyFx3 { + VFSTestSetup vfs_test_setup_; + Context ctx; + StringEmptyFx3(); std::vector offsets = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; std::vector data = {'\0'}; }; +StringEmptyFx3::StringEmptyFx3() + : ctx(vfs_test_setup_.ctx()){}; + TEST_CASE_METHOD( StringEmptyFx3, "C++ API: Test var-length string only empty", - "[cppapi][empty-var-length]") { - using namespace tiledb; - - std::string array_name = "empty_string3"; + "[cppapi][empty-var-length][rest]") { + std::string array_name = vfs_test_setup_.array_uri("empty_string3"); // create { - Context ctx; - Domain domain(ctx); domain.add_dimension( Dimension::create(ctx, "__dim_0", {0, 3}, 1)); @@ -631,18 +550,11 @@ TEST_CASE_METHOD( attr.set_cell_val_num(TILEDB_VAR_NUM); schema.add_attribute(attr); - VFS vfs(ctx); - if (vfs.is_dir(array_name)) { - vfs.remove_dir(array_name); - } Array::create(array_name, schema); } // write { - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - auto array = tiledb::Array(ctx, array_name, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); @@ -656,9 +568,6 @@ TEST_CASE_METHOD( // read whole array { - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - std::vector r_offsets(16); std::vector r_data(16); @@ -684,9 +593,6 @@ TEST_CASE_METHOD( // 0s and the data buffer is empty in this case because all of // the queried cells are empty. { - auto cfg = tiledb::Config(); - auto ctx = tiledb::Context(cfg); - std::vector r_offsets(4); std::vector r_data(4); @@ -709,9 +615,4 @@ TEST_CASE_METHOD( REQUIRE(r_offsets == q2_result_offsets); REQUIRE(r_data[0] == StringEmptyFx3::data[0]); } - - Context ctx; - if (Object::object(ctx, array_name).type() == Object::Type::Array) { - Object::remove(ctx, array_name); - } } diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index 8b92b13f9b0e..c25d4bfa058d 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -44,9 +44,6 @@ using namespace tiledb; template struct CPPFixedTileMetadataFx { - // Buffers to allocate on server side for serialized queries - test::ServerQueryBuffers server_buffers_; - CPPFixedTileMetadataFx() : vfs_(ctx_) { if (vfs_.is_dir(ARRAY_NAME)) @@ -118,9 +115,7 @@ struct CPPFixedTileMetadataFx { tiledb_layout_t layout, bool nullable, bool all_null, - uint64_t cell_val_num, - const bool serialized, - const bool refactored_query_v2) { + uint64_t cell_val_num) { std::default_random_engine random_engine; // Generate random, sorted strings for the string ascii type. @@ -263,14 +258,11 @@ struct CPPFixedTileMetadataFx { } // Submit query - auto rc = test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } array.close(); } @@ -658,17 +650,6 @@ TEMPLATE_LIST_TEST_CASE_METHOD( FixedTypesUnderTest) { typedef TestType T; std::string test = GENERATE("nullable", "all null", "non nullable"); - - bool serialized = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialized = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialized = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif tiledb_layout_t layout = GENERATE(TILEDB_UNORDERED, TILEDB_GLOBAL_ORDER, TILEDB_ROW_MAJOR); @@ -685,13 +666,7 @@ TEMPLATE_LIST_TEST_CASE_METHOD( for (uint64_t f = 0; f < num_frag; f++) { // Write a fragment. CPPFixedTileMetadataFx::write_fragment( - f, - layout, - nullable, - all_null, - cell_val_num, - serialized, - refactored_query_v2); + f, layout, nullable, all_null, cell_val_num); } for (uint64_t f = 0; f < num_frag; f++) { @@ -702,9 +677,6 @@ TEMPLATE_LIST_TEST_CASE_METHOD( } struct CPPVarTileMetadataFx { - // Buffers to allocate on server side for serialized queries - test::ServerQueryBuffers server_buffers_; - CPPVarTileMetadataFx() : vfs_(ctx_) { if (vfs_.is_dir(ARRAY_NAME)) @@ -739,12 +711,7 @@ struct CPPVarTileMetadataFx { } void write_fragment( - uint64_t f, - tiledb_layout_t layout, - bool nullable, - bool all_null, - const bool serialized, - const bool refactored_query_v2) { + uint64_t f, tiledb_layout_t layout, bool nullable, bool all_null) { std::default_random_engine random_engine; uint64_t max_string_size = 100; @@ -840,14 +807,12 @@ struct CPPVarTileMetadataFx { } // Submit query - auto rc = test::submit_query_wrapper( - ctx_, - ARRAY_NAME, - &query, - server_buffers_, - serialized, - refactored_query_v2); - REQUIRE(rc == TILEDB_OK); + if (layout == TILEDB_GLOBAL_ORDER) { + query.submit_and_finalize(); + } else { + query.submit(); + } + array.close(); } @@ -1063,18 +1028,6 @@ TEST_CASE_METHOD( "TileMetadata: var data type tile", "[tile-metadata][var-data]") { std::string test = GENERATE("nullable", "all null", "non nullable"); - - bool serialized = false, refactored_query_v2 = false; - SECTION("no serialization") { - serialized = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("serialization enabled global order write") { - serialized = true; - refactored_query_v2 = GENERATE(true, false); - } -#endif - tiledb_layout_t layout = GENERATE(TILEDB_UNORDERED, TILEDB_GLOBAL_ORDER, TILEDB_ROW_MAJOR); @@ -1088,8 +1041,7 @@ TEST_CASE_METHOD( for (uint64_t f = 0; f < num_frag; f++) { // Write a fragment. - CPPVarTileMetadataFx::write_fragment( - f, layout, nullable, all_null, serialized, refactored_query_v2); + CPPVarTileMetadataFx::write_fragment(f, layout, nullable, all_null); } for (uint64_t f = 0; f < num_frag; f++) { diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index b70891aa8cf2..428828babded 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1232,8 +1232,7 @@ void read_array( tiledb_query_free(&query); } -int32_t num_commits(const std::string& array_name) { - Context ctx; +int32_t num_commits(Context ctx, const std::string& array_name) { VFS vfs(ctx); // Get the number of write files in the commit directory @@ -1241,8 +1240,13 @@ int32_t num_commits(const std::string& array_name) { return commits_dir.file_count(sm::constants::write_file_suffix); } -int32_t num_fragments(const std::string& array_name) { +int32_t num_commits(const std::string& array_name) { Context ctx; + + return num_commits(ctx, array_name); +} + +int32_t num_fragments(Context ctx, const std::string& array_name) { VFS vfs(ctx); // Get all URIs in the array directory @@ -1251,6 +1255,12 @@ int32_t num_fragments(const std::string& array_name) { return static_cast(uris.size()); } +int32_t num_fragments(const std::string& array_name) { + Context ctx; + + return num_fragments(ctx, array_name); +} + std::string random_string(const uint64_t l) { static const char char_set[] = "0123456789" @@ -1437,451 +1447,6 @@ int deserialize_array_and_query( ctx, c_buff, TILEDB_CAPNP, clientside ? 1 : 0, array_uri, query, &array); } -int array_open_wrapper( - tiledb_ctx_t* client_ctx, - tiledb_query_type_t query_type, - bool serialize, - tiledb_array_t** open_array) { - // Open array -#ifndef TILEDB_SERIALIZATION - return tiledb_array_open(client_ctx, *open_array, query_type); -#endif - - if (!serialize) { - return tiledb_array_open(client_ctx, *open_array, query_type); - } - - // Get the config to use it for creating fresh contexts - tiledb_config_t* config; - int rc = tiledb_ctx_get_config(client_ctx, &config); - REQUIRE(rc == TILEDB_OK); - - // this helper only applies to refactored array open, so set it in the config - // Note: we actually set use_refactored_array_open_and_query_submit instead of - // simple use_refactored_array_open here, because we want array_open_wrapper - // to be usable in query_v3 tests that require that flag to be set right - // from the beginning for full Array objects to be retrieved on array open. - // There are dedicated tests in unit-capi-array.cc that are testing array v2 - // feature with just setting use_refactored_array_open config variable. - tiledb_error_t* error = nullptr; - rc = tiledb_config_set( - config, - "rest.use_refactored_array_open_and_query_submit", - "true", - &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); - REQUIRE( - tiledb_array_set_config(client_ctx, *open_array, config) == TILEDB_OK); - - auto serialization_type = - (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP; - - // 1. Client: Serialize array_open request to Server. First set the query_type - // that will be serialized - auto type = static_cast(query_type); - (*open_array)->array_->set_query_type(type); - - tiledb_buffer_t* buffer; - rc = tiledb_serialize_array_open( - client_ctx, *open_array, serialization_type, 1, &buffer); - REQUIRE(rc == TILEDB_OK); - - // 2. Server : Receive and deserialize array_open_request - // Allocate new context for server - tiledb_ctx_t* server_ctx; - tiledb_ctx_alloc(config, &server_ctx); - tiledb_array_t* deserialized_array_server = nullptr; - rc = tiledb_deserialize_array_open( - server_ctx, buffer, serialization_type, 0, &deserialized_array_server); - REQUIRE(rc == TILEDB_OK); - tiledb_buffer_free(&buffer); - - // Check that the original and de-serialized array have the same query type - REQUIRE(deserialized_array_server->array_->get_query_type() == type); - - // 3. Server: Open the array the request was received for in the requested - // mode - // This is needed in test, as the deserialized array has a dummy array_uri of - // "deserialized_array" set instead of the original one. The Cloud side - // already knows the array URI so it's not a problem in the real life scenario - deserialized_array_server->array_->set_array_uri( - (*open_array)->array_->array_uri()); - - // if we need to add start and end timestamps uncomment this code - // rc = tiledb_array_set_open_timestamp_start(server_ctx, - // deserialized_array_server, array->array_->timestamp_start()); - // REQUIRE_SAFE(rc == TILEDB_OK); - // rc = tiledb_array_set_open_timestamp_end(server_ctx, - // deserialized_array_server, array->array_->timestamp_end()); REQUIRE_SAFE(rc - // == TILEDB_OK); - rc = tiledb_array_open(server_ctx, deserialized_array_server, query_type); - REQUIRE(rc == TILEDB_OK); - - // 4. Server -> Client: Send opened Array (serialize) - tiledb_buffer_t* buff; - rc = tiledb_serialize_array( - server_ctx, - deserialized_array_server, - (tiledb_serialization_type_t)tiledb::sm::SerializationType::CAPNP, - 1, - &buff); - REQUIRE(rc == TILEDB_OK); - rc = tiledb_array_close(server_ctx, deserialized_array_server); - CHECK(rc == TILEDB_OK); - tiledb_array_free(&deserialized_array_server); - tiledb_ctx_free(&server_ctx); - CHECK(server_ctx == nullptr); - - // 5. Client: Receive and deserialize Array (into deserialized_array_client), - // in the same way that rest_client does. - auto st = tiledb::sm::serialization::array_deserialize( - (*open_array)->array_.get(), - tiledb::sm::SerializationType::CAPNP, - buff->buffer(), - client_ctx->storage_manager(), - tiledb::test::create_test_memory_tracker()); - REQUIRE(st.ok()); - - // 6. Server: Close array and clean up - tiledb_buffer_free(&buff); - - return rc; -} - -int submit_query_wrapper( - const Context& client_ctx, - const std::string& array_uri, - Query* query, - ServerQueryBuffers& buffers, - bool serialize_query, - bool refactored_query_v2, - bool finalize) { - auto query_c = query->ptr().get(); - return submit_query_wrapper( - client_ctx.ptr().get(), - array_uri, - &query_c, - buffers, - serialize_query, - refactored_query_v2, - finalize); -} - -int submit_query_wrapper( - tiledb_ctx_t* client_ctx, - const std::string& array_uri, - tiledb_query_t** query, - ServerQueryBuffers& buffers, - bool serialize, - bool refactored_query_v2, - bool finalize) { - int rc = 0; -#ifndef TILEDB_SERIALIZATION - rc = tiledb_query_submit(client_ctx, *query); - if (rc != TILEDB_OK) { - return rc; - } - - if (finalize) { - // Finalize query - rc = tiledb_query_finalize(client_ctx, *query); - } - - return rc; -#endif - - if (!serialize) { - rc = tiledb_query_submit(client_ctx, *query); - if (rc != TILEDB_OK) { - return rc; - } - - if (finalize) { - // Finalize query - rc = tiledb_query_finalize(client_ctx, *query); - } - - return rc; - } - - // Get the config to use it for creating fresh contexts - tiledb_config_t* config; - rc = tiledb_query_get_config(client_ctx, *query, &config); - CHECK(rc == TILEDB_OK); - - // Get the array of that query - tiledb_array_t* array; - REQUIRE(tiledb_query_get_array(client_ctx, *query, &array) == TILEDB_OK); - - // if refactored query v2 set it in the config - if (refactored_query_v2) { - tiledb_error_t* error = nullptr; - rc = tiledb_config_set( - config, - "rest.use_refactored_array_open_and_query_submit", - "true", - &error); - REQUIRE(rc == TILEDB_OK); - REQUIRE(error == nullptr); - - REQUIRE(tiledb_array_close(client_ctx, array) == TILEDB_OK); - REQUIRE(tiledb_array_set_config(client_ctx, array, config) == TILEDB_OK); - tiledb_query_type_t query_type; - REQUIRE_SAFE( - tiledb_query_get_type(client_ctx, *query, &query_type) == TILEDB_OK); - REQUIRE(tiledb_array_open(client_ctx, array, query_type) == TILEDB_OK); - } - - // Get the query type - tiledb_query_type_t query_type; - REQUIRE_SAFE( - tiledb_query_get_type(client_ctx, *query, &query_type) == TILEDB_OK); - - // 1. Client -> Server : Serialize and send query request - tiledb_query_t* server_deser_query; - std::vector serialized; - - rc = serialize_query(client_ctx, *query, &serialized, 1); - REQUIRE(rc == TILEDB_OK); - - // 2. Server: Deserialize query request - // Allocate new context for server - tiledb_ctx_t* server_ctx; - tiledb_ctx_alloc(config, &server_ctx); - tiledb_array_t* array_server = nullptr; - if (refactored_query_v2) { - rc = deserialize_array_and_query( - server_ctx, serialized, &server_deser_query, array_uri.c_str(), 0); - REQUIRE_SAFE(rc == TILEDB_OK); - } else { - // Open array - int rc = tiledb_array_alloc(server_ctx, array_uri.c_str(), &array_server); - REQUIRE_SAFE(rc == TILEDB_OK); - - rc = tiledb_array_set_open_timestamp_start( - server_ctx, array_server, array->array_->timestamp_start()); - REQUIRE_SAFE(rc == TILEDB_OK); - rc = tiledb_array_set_open_timestamp_end( - server_ctx, array_server, array->array_->timestamp_end_opened_at()); - REQUIRE_SAFE(rc == TILEDB_OK); - - rc = tiledb_array_open(server_ctx, array_server, query_type); - REQUIRE_SAFE(rc == TILEDB_OK); - - rc = tiledb_query_alloc( - server_ctx, array_server, query_type, &server_deser_query); - REQUIRE_SAFE(rc == TILEDB_OK); - rc = tiledb::test::deserialize_query( - server_ctx, serialized, server_deser_query, 0); - REQUIRE_SAFE(rc == TILEDB_OK); - } - - // This is a feature of the server, not a bug, quoting from query_from_capnp: - // "On reads, just set null pointers with accurate size so that the - // server can introspect and allocate properly sized buffers separately." - // Empty buffers will naturally break query_submit so to go on in test we - // need to allocate here as if we were the server. - if (query_type == TILEDB_READ) { - allocate_query_buffers_server_side(server_ctx, server_deser_query, buffers); - } - - // 3. Server: Submit query WITHOUT re-opening the array, using under the hood - // the array found in the deserialized query - rc = tiledb_query_submit(server_ctx, server_deser_query); - if (rc != TILEDB_OK) { - return rc; - } - - if (finalize) { - tiledb_query_status_t status; - rc = tiledb_query_get_status(server_ctx, server_deser_query, &status); - CHECK(status == TILEDB_COMPLETED); - - rc = tiledb_query_finalize(server_ctx, server_deser_query); - if (rc != TILEDB_OK) { - return rc; - } - } - - // Serialize the new query and "send it over the network" (server-side) - // 4. Server -> Client : Send query response - std::vector serialized2; - rc = serialize_query(server_ctx, server_deser_query, &serialized2, 0); - if (rc != TILEDB_OK) { - return rc; - } - - if (!refactored_query_v2) { - // Close array and clean up - rc = tiledb_array_close(server_ctx, array_server); - CHECK(rc == TILEDB_OK); - tiledb_array_free(&array_server); - } - - tiledb_ctx_free(&server_ctx); - CHECK(server_ctx == nullptr); - - // 5. Client: Deserialize query - rc = deserialize_query(client_ctx, serialized2, *query, 1); - REQUIRE(rc == TILEDB_OK); - - // Clean up. - tiledb_query_free(&server_deser_query); - - return rc; -} - -int finalize_query_wrapper( - const Context& client_ctx, - const std::string& array_uri, - Query* query, - bool serialize_query) { - auto query_c = query->ptr().get(); - return finalize_query_wrapper( - client_ctx.ptr().get(), array_uri, &query_c, serialize_query); -} - -int finalize_query_wrapper( - tiledb_ctx_t* client_ctx, - const std::string& array_uri, - tiledb_query_t** query, - bool serialize) { -#ifndef TILEDB_SERIALIZATION - return tiledb_query_finalize(client_ctx, *query); -#endif - - if (!serialize) { - return tiledb_query_finalize(client_ctx, *query); - } - - // Get the config to use it for creating fresh contexts - tiledb_config_t* config; - auto rc = tiledb_query_get_config(client_ctx, *query, &config); - CHECK(rc == TILEDB_OK); - - // Get the query type - tiledb_query_type_t query_type; - REQUIRE_SAFE( - tiledb_query_get_type(client_ctx, *query, &query_type) == TILEDB_OK); - - // 1. Client -> Server : Send query request - std::vector serialized; - - rc = serialize_query(client_ctx, *query, &serialized, 1); - REQUIRE(rc == TILEDB_OK); - - // 2. Server: Deserialize query request - // Allocate new context for server - tiledb_ctx_t* server_ctx; - tiledb_ctx_alloc(config, &server_ctx); - tiledb_query_t* server_deser_query; - rc = deserialize_array_and_query( - server_ctx, serialized, &server_deser_query, array_uri.c_str(), 0); - - // 3. Server: Finalize query - rc = tiledb_query_finalize(server_ctx, server_deser_query); - if (rc != TILEDB_OK) { - return rc; - } - - /* - * TODO: This is disabled because it fails today as the previous serialization - * step before finalize, sets the query buffers to NULL for READ queries, so - * the following serialization step will hit an assert because of empty - * buffers, as query_finalize is a NOOP for READ queries. - * - * // Serialize the new query and "send it over the network" (server-side) - * 3. Server -> Client : Send query response - * std::vector serialized2; - * rc = tiledb_query_v2_serialize( - * ctx, array_uri.c_str(), serialized, false, server_deser_query, query); - * REQUIRE_SAFE(rc == TILEDB_OK); - */ - - // Clean up. - tiledb_query_free(&server_deser_query); - - return rc; -} - -void allocate_query_buffers_server_side( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - ServerQueryBuffers& query_buffers) { - int rc = 0; - auto buffer_names = query->query_->buffer_names(); - const auto aggregate_names = query->query_->aggregate_buffer_names(); - buffer_names.insert( - buffer_names.end(), aggregate_names.begin(), aggregate_names.end()); - - for (uint64_t i = 0; i < buffer_names.size(); i++) { - const auto& name = buffer_names[i]; - const auto& buff = query->query_->buffer(name); - const auto& schema = query->query_->array_schema(); - - // TODO: This is yet another instance where there needs to be a common - // mechanism for reporting the common properties of a field. - // Refactor to use query_field_t. - bool var_size = false; - bool nullable = false; - if (query->query_->is_aggregate(name)) { - var_size = - query->query_->get_aggregate(name).value()->aggregation_var_sized(); - nullable = - query->query_->get_aggregate(name).value()->aggregation_nullable(); - } else { - var_size = schema.var_size(name); - nullable = schema.is_nullable(name); - } - - if (var_size && buff.buffer_var_ == nullptr) { - // Variable-sized buffer - query_buffers.attr_or_dim_data.emplace_back(*buff.buffer_var_size_); - query_buffers.attr_or_dim_off.emplace_back( - (*buff.buffer_size_) / sizeof(sm::constants::cell_var_offset_size)); - rc = tiledb_query_set_data_buffer( - ctx, - query, - name.c_str(), - query_buffers.attr_or_dim_data.back().data(), - buff.buffer_var_size_); - REQUIRE_SAFE(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx, - query, - name.c_str(), - query_buffers.attr_or_dim_off.back().data(), - buff.buffer_size_); - REQUIRE_SAFE(rc == TILEDB_OK); - } - - if (name == TILEDB_COORDS || (!var_size && buff.buffer_ == nullptr)) { - // Fixed-length buffer or Coords - query_buffers.attr_or_dim.emplace_back(*buff.buffer_size_); - rc = tiledb_query_set_data_buffer( - ctx, - query, - name.c_str(), - query_buffers.attr_or_dim.back().data(), - buff.buffer_size_); - REQUIRE_SAFE(rc == TILEDB_OK); - } - - if (nullable) { - // nullable - query_buffers.attr_or_dim_nullable.emplace_back( - *buff.validity_vector_.buffer_size()); - rc = tiledb_query_set_validity_buffer( - ctx, - query, - name.c_str(), - query_buffers.attr_or_dim_nullable.back().data(), - buff.validity_vector_.buffer_size()); - } - } -} - template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); template void check_subarray( diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index bf179ff434a4..ab393af9e023 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -761,12 +761,24 @@ void read_array( tiledb_layout_t layout, const QueryBuffers& buffers); +/** + * Returns the number of commits in the input array, + * given a context. + */ +int32_t num_commits(Context ctx, const std::string& array_name); + /** * Returns the number of commits in the input array, * appropriately excluding special files and subdirectories. */ int32_t num_commits(const std::string& array_name); +/** + * Returns the number of fragments in the input array, + * given a context. + */ +int32_t num_fragments(Context ctx, const std::string& array_name); + /** * Returns the number of fragments in the input array, * appropriately excluding special files and subdirectories. @@ -905,99 +917,6 @@ int deserialize_array_and_query( const char* array_uri, bool clientside); -/** - * Helper method that wraps tiledb_array_open() and inserts a serialization - * step, if serialization is enabled. This wrapper models exclusively the - * refactored array open (array open v2) serialization path. The added - * serialization steps are designed to closely mimic the behavior of the REST - * server. - * - * @param ctx Context. - * @param query_type Type of query to open the array for. - * @param serialize True if this is a remote array open, false if not. - * @param open_array Output open array. - */ -int array_open_wrapper( - tiledb_ctx_t* ctx, - tiledb_query_type_t query_type, - bool serialize, - tiledb_array_t** open_array); - -/** - * C API - * Helper method that wraps tiledb_query_submit() and inserts serialization - * steps, if serialization is enabled. The added serialization steps are - * designed to closely mimic the behavior of the REST server. - * - * @param ctx Context. - * @param array_uri URI of the array the query submit is for. - * @param query Input/Output tiledb query object to submit/get back. - * @param buffers Stack allocated buffers to be used by the step simulating - * "server side buffer allocation" for READ queries. They can only be allocated - * e.g. once in the test Fixture definition or just before the call to - * submit_query_wrapper. - * @param serialize_query True if this is a remote array open, false if not. - * @param refactored_query_v2 If - * "rest.use_refactored_array_open_and_query_submit" should be true. - * @param finalize Finalize or not the query after submitting it. - */ -int submit_query_wrapper( - tiledb_ctx_t* ctx, - const std::string& array_uri, - tiledb_query_t** query, - ServerQueryBuffers& buffers, - bool serialize_query, - bool refactored_query_v2 = false, - bool finalize = true); - -/** C++ wrapper of submit_query_wrapper */ -int submit_query_wrapper( - const Context& ctx, - const std::string& array_uri, - Query* query, - ServerQueryBuffers& buffers, - bool serialize_query, - bool refactored_query_v2 = false, - bool finalize = true); - -/** - * C API - * Helper method that wraps tiledb_query_finalize() and inserts serialization - * steps, if serialization is enabled. The added serialization steps are - * designed to closely mimic the behavior of the REST server. - * - * @param ctx Context. - * @param array_uri URI of the array the query finalize is for. - * @param query Input/Output tiledb query object to submit/get back. - * @param serialize_query True if this is a remote array open, false if not. - */ -int finalize_query_wrapper( - const Context& ctx, - const std::string& array_uri, - Query* query, - bool serialize_query); - -/** C++ wrapper of finalize_query_wrapper */ -int finalize_query_wrapper( - tiledb_ctx_t* ctx, - const std::string& array_uri, - tiledb_query_t** query, - bool serialize_query); - -/** - * Helper function that allocates buffers on a query object that has been - * deserialized on the "server" side. This is a necassary step for serialized - * READ queries. - * - * @param ctx Context. - * @param query Query for which we are allocating the buffers. - * @param query_buffers Input/Output allocated buffers. - */ -void allocate_query_buffers_server_side( - tiledb_ctx_t* ctx, - tiledb_query_t* query, - ServerQueryBuffers& query_buffers); - } // namespace tiledb::test #endif diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index 802a0342a4db..326cc583b5dc 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -173,8 +173,7 @@ Status SupportedFsS3::prepare_config( config, "vfs.s3.use_virtual_addressing", "false", &error) == TILEDB_OK); REQUIRE( - tiledb_config_set(config, "vfs.s3.verify_ssl", "false", &error) == - TILEDB_OK); + tiledb_config_set(config, "ssl.verify", "false", &error) == TILEDB_OK); REQUIRE(error == nullptr); #endif @@ -468,7 +467,7 @@ tiledb::sm::Config VFSTestBase::create_test_config() { cfg.set("vfs.s3.endpoint_override", "localhost:9999").ok(); cfg.set("vfs.s3.scheme", "https").ok(); cfg.set("vfs.s3.use_virtual_addressing", "false").ok(); - cfg.set("vfs.s3.verify_ssl", "false").ok(); + cfg.set("ssl.verify", "false").ok(); } cfg.set("vfs.azure.storage_account_name", "devstoreaccount1").ok(); cfg.set( diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index 3945c4637576..72a8c41e6efe 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -41,11 +41,6 @@ using namespace tiledb::test; struct QueryAggregateFx : TemporaryDirectoryFixture { - // Serialization parameters - bool serialize_ = false; - bool refactored_query_v2_ = false; - ServerQueryBuffers server_buffers_; - QueryAggregateFx() : array_name_(temp_dir_ + "queryaggregate_array") { rm_array(); @@ -335,15 +330,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates COUNT test", "[capi][query_aggregate][count]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -371,16 +357,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Count", &count, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); - + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(count == 9); // Clean up @@ -394,15 +371,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates SUM test", "[capi][query_aggregate][sum]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -434,16 +402,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Sum", &sum, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); - + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(sum == 55); // Clean up @@ -458,15 +417,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MEAN test", "[capi][query_aggregate][mean]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -499,15 +449,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Mean", &mean, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(mean == 5.5); @@ -523,15 +465,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MIN test", "[capi][query_aggregate][min]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -563,15 +496,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Min", &min, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(min == 1); @@ -587,15 +512,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates MAX test", "[capi][query_aggregate][max]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -627,15 +543,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Max", &max, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(max == 10); @@ -651,15 +559,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates NULL_COUNT test", "[capi][query_aggregate][null_count]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -692,15 +591,7 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx, query, "Null", &nullcount, &size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - REQUIRE(rc == 0); + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); CHECK(nullcount == 8); @@ -812,15 +703,6 @@ TEST_CASE_METHOD( QueryAggregateFx, "C API: Query aggregates serialization test", "[capi][query_aggregate][serialization][incompletes]") { - SECTION("- No serialization") { - serialize_ = false; - } -#ifdef TILEDB_SERIALIZATION - SECTION("- Serialization") { - serialize_ = true; - } -#endif - tiledb_array_t* array; REQUIRE(tiledb_array_alloc(ctx, array_name_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); @@ -857,23 +739,11 @@ TEST_CASE_METHOD( REQUIRE( tiledb_query_set_data_buffer(ctx, query, "a", &a, &a_size) == TILEDB_OK); - int rc = submit_query_wrapper( - ctx, - array_name_.c_str(), - &query, - server_buffers_, - serialize_, - false, - false); - - if (serialize_) { - CHECK(rc != 0); - } else { - tiledb_query_status_t status; - REQUIRE(tiledb_query_get_status(ctx, query, &status) == TILEDB_OK); - REQUIRE(status == TILEDB_INCOMPLETE); - CHECK(rc == 0); - } + REQUIRE(tiledb_query_submit(ctx, query) == TILEDB_OK); + + tiledb_query_status_t status; + REQUIRE(tiledb_query_get_status(ctx, query, &status) == TILEDB_OK); + REQUIRE(status == TILEDB_INCOMPLETE); // Clean up CHECK(tiledb_aggregate_free(ctx, &sum_op) == TILEDB_OK); From fe9f6285627a1143e57b6589c67d3ebdb58ee412 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 29 Apr 2024 12:45:05 -0700 Subject: [PATCH 303/456] Implement var_length_view and unit tests for external sort. (#4918) This PR implements the `var_length_view` class, which provides a single view of var length data. The `var_length_view` class takes a data range and an index range and provides a view of a range of subranges, where the ith subrange is the ith var length element. Example: ``` std::vector x {1, 2, 3, 4, 5, 6, 7, 8, 9}; // Data range std::vector indices {0, 4, 7, 9}; // Index range var_length_view v(x, indices); CHECK(std::ranges::equal(v[0], std::vector{1, 2, 3, 4})); // v[0] is the subrange 1, 2, 3, 4 CHECK(std::ranges::equal(v[1], std::vector{5, 6, 7})); // v[1] is the subrange 5, 6, 7 CHECK(std::ranges::equal(v[2], std::vector{8, 9})); // v[2] is the subrange 8, 9 ``` Here, we are given var length data, specified in the vectors `x` and `indices`. The `var_length_view` `v` is a single range (a single container) whose elements are `{1, 2, 3, 4}`, `{5, 6, 7}`, and `{8, 9}`. The `var_length_view` uses `iterator_facade` to specify its iterators, so it is fully compliant with the concepts for C++20 ranges and iterators. The unit tests verify range and iterator concepts, and exercise expected behavior of the `var_length_view`. **Note:** In this implementation, dereferencing an iterator returns a proxy -- a `std::subrange` object and so will be subject to some of the concomitant limitations. (Eric Niebler has a discussion about proxy iterators at http://ericniebler.com/2015/01/28/to-be-or-not-to-be-an-iterator/ -- it is somewhat dated -- and predates his work on ranges v3 -- but is still relevant. He references N3351 (https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3351.pdf) and the who's who of C++ luminaries who wrote it :-) -- and identifies some subtle issues). --- TYPE: IMPROVEMENT DESC: Implement var_length_view and unit tests for external sort. --- tiledb/common/test/CMakeLists.txt | 4 +- tiledb/common/test/unit_var_length_view.cc | 319 +++++++++++++++++++++ tiledb/common/var_length_view.h | 191 ++++++++++++ 3 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 tiledb/common/test/unit_var_length_view.cc create mode 100644 tiledb/common/var_length_view.h diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 24935f815c56..92fe7a1e25aa 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -41,8 +41,8 @@ commence(unit_test memory_tracker_types) this_target_object_libraries(baseline) conclude(unit_test) -commence(unit_test iterator_facade) - this_target_sources(main.cc unit_iterator_facade.cc) +commence(unit_test common_utils) + this_target_sources(main.cc unit_iterator_facade.cc unit_var_length_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_var_length_view.cc b/tiledb/common/test/unit_var_length_view.cc new file mode 100644 index 000000000000..45f7988f302b --- /dev/null +++ b/tiledb/common/test/unit_var_length_view.cc @@ -0,0 +1,319 @@ +/** + * @file unit_var_length_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the var_length_view class. + */ + +#include +#include +#include +#include "../var_length_view.h" + +TEST_CASE("var_length_view: Null test", "[var_length_view][null_test]") { + REQUIRE(true); +} + +// Test that the var_length_view satisfies the expected concepts +TEST_CASE("var_length_view: Range concepts", "[var_length_view][concepts]") { + using test_type = var_length_view, std::vector>; + + CHECK(std::ranges::range); + CHECK(!std::ranges::borrowed_range); + CHECK(std::ranges::sized_range); + CHECK(std::ranges::view); + CHECK(std::ranges::input_range); + CHECK(!std::ranges:: + output_range>); + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + CHECK(!std::ranges::contiguous_range); + CHECK(std::ranges::common_range); + CHECK(std::ranges::viewable_range); + + CHECK(std::ranges::view); +} + +// Test that the var_length_view iterators satisfy the expected concepts +TEST_CASE("var_length_view: Iterator concepts", "[var_length_view][concepts]") { + using test_type = var_length_view, std::vector>; + using test_type_iterator = std::ranges::iterator_t; + using test_type_const_iterator = std::ranges::iterator_t; + + CHECK(std::input_or_output_iterator); + CHECK(std::input_or_output_iterator); + CHECK(std::input_iterator); + CHECK(std::input_iterator); + CHECK(!std::output_iterator< + test_type_iterator, + std::ranges::range_value_t>); + CHECK(!std::output_iterator< + test_type_const_iterator, + std::ranges::range_value_t>); + CHECK(std::forward_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + CHECK(std::random_access_iterator); +} + +// Test that the var_length_view value_type satisfies the expected concepts +TEST_CASE( + "var_length_view: value_type concepts", "[var_length_view][concepts]") { + using test_type = var_length_view, std::vector>; + CHECK(std::ranges::range); + + using test_iterator_type = std::ranges::iterator_t; + using test_iterator_value_type = std::iter_value_t; + using test_iterator_reference_type = + std::iter_reference_t; + + using range_value_type = std::ranges::range_value_t; + using range_reference_type = std::ranges::range_reference_t; + + CHECK(std::is_same_v); + CHECK(std::is_same_v); +} + +// Simple test that the var_length_view can be constructed +TEST_CASE("var_length_view: Basic constructor", "[var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + + auto u = var_length_view(r, o); + auto v = var_length_view{r, o}; + var_length_view w(r, o); + var_length_view x{r, o}; + + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); +} + +// Check that the sizes of the var_length_view are correct +TEST_CASE("var_length_view: size()", "[var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + + auto v = var_length_view{r, o}; + CHECK(size(v) == 3); + CHECK(v.begin()->size() == 3); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 4); + + size_t count = 0; + for (auto i : v) { + (void)i; + count++; + } + CHECK(count == 3); + count = 0; + for (auto i = v.begin(); i != v.end(); ++i) { + count++; + } + CHECK(count == 3); +} + +// Check that various properties of iterators hold +TEST_CASE("var_length_view: Basic iterators", "[var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + + auto v = var_length_view(r, o); + + auto a = v.begin(); + auto b = v.end(); + auto c = begin(v); + auto d = end(v); + + SECTION("Check begin and end") { + REQUIRE(a == a); + CHECK(a == c); + CHECK(b == d); + } + SECTION("Check dereference") { + CHECK(std::equal(a->begin(), a->end(), r.begin())); + CHECK(std::equal(a->begin(), a->end(), c->begin())); + // CHECK(std::ranges::equal(*a, r)); // size() not the same + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("Check pre-increment + dereference") { + ++a; + ++c; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 3)); + CHECK(std::ranges::equal(*a, *c)); + ++a; + ++c; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 6)); + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("Check post-increment + dereference") { + a++; + c++; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 3)); + CHECK(std::ranges::equal(*a, *c)); + a++; + c++; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 6)); + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("operator[]") { + CHECK(std::equal(a[0].begin(), a[0].end(), r.begin())); + CHECK(std::equal(a[1].begin(), a[1].end(), r.begin() + 3)); + CHECK(std::equal(a[2].begin(), a[2].end(), r.begin() + 6)); + CHECK(std::equal(a[0].begin(), a[0].end(), c[0].begin())); + CHECK(std::equal(a[1].begin(), a[1].end(), c[1].begin())); + CHECK(std::equal(a[2].begin(), a[2].end(), c[2].begin())); + CHECK(!std::equal(a[0].begin(), a[0].end(), c[1].begin())); + CHECK(!std::equal(a[0].begin(), a[0].end(), c[2].begin())); + + CHECK(a[0][0] == 1.0); + CHECK(a[0][1] == 2.0); + CHECK(a[0][2] == 3.0); + CHECK(a[1][0] == 4.0); + CHECK(a[1][1] == 5.0); + CHECK(a[1][2] == 6.0); + CHECK(a[2][0] == 7.0); + CHECK(a[2][1] == 8.0); + CHECK(a[2][2] == 9.0); + CHECK(a[2][3] == 10.0); + } + SECTION("More operator[]") { + size_t count = 0; + for (auto i : v) { + for (auto j : i) { + ++count; + CHECK(j == count); + } + } + } + SECTION("Equality, etc") { + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + + CHECK(a != b); + CHECK(!(a == b)); + CHECK(a < b); + CHECK(!(a > b)); + CHECK(a <= b); + CHECK(!(a >= b)); + + ++a; + CHECK(a != c); + CHECK(!(a == c)); + CHECK(!(a < c)); + CHECK(a > c); + CHECK(!(a <= c)); + CHECK(a >= c); + --a; + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + c++; + CHECK(a != c); + CHECK(!(a == c)); + CHECK(!(a > c)); + CHECK(a < c); + CHECK(!(a >= c)); + CHECK(a <= c); + c--; + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + } +} + +// Test that we can read and write to the elements of the var_length_view +TEST_CASE("var_length_view: Viewness", "[var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector s = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + std::vector m = {0, 3, 6, 10}; + std::vector q = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + std::vector p = {0, 2, 7, 10}; + std::vector n = {0, 3, 6, 10}; + + auto v = var_length_view(r, o); + auto w = var_length_view(q, p); + auto u = var_length_view(q, n); + auto x = var_length_view(r, m); + auto y = var_length_view(s, m); + auto z = var_length_view(s, n); + + REQUIRE(v.begin() == v.begin()); + CHECK(v.begin() != w.begin()); + CHECK(v.begin() != u.begin()); + CHECK(w.begin() != u.begin()); + + REQUIRE(v.end() == v.end()); + CHECK(v.end() != w.end()); + CHECK(v.end() != u.end()); + CHECK(w.end() != u.end()); + + for (size_t i = 0; i < 3; ++i) { + CHECK(v.size() == x.size()); + CHECK(std::equal( + v.begin()[i].begin(), v.begin()[i].end(), x.begin()[i].begin())); + for (long j = 0; j < x.size(); ++j) { + CHECK(v.begin()[i][j] == x.begin()[i][j]); + } + } + for (auto&& i : y) { + for (auto& j : i) { + j += 13.0; + } + } + for (size_t i = 0; i < 3; ++i) { + CHECK(v.size() == x.size()); + for (long j = 0; j < x.size(); ++j) { + CHECK(y.begin()[i][j] == v.begin()[i][j] + 13.0); + CHECK(z.begin()[i][j] == y.begin()[i][j]); + CHECK(z.cbegin()[i][j] == y.begin()[i][j]); + CHECK(z.cbegin()[i][j] == y.cbegin()[i][j]); + CHECK(z.cbegin()[i][j] == z.begin()[i][j]); + } + } +} diff --git a/tiledb/common/var_length_view.h b/tiledb/common/var_length_view.h new file mode 100644 index 000000000000..09feaaed52f6 --- /dev/null +++ b/tiledb/common/var_length_view.h @@ -0,0 +1,191 @@ +/** + * @file var_length_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains the definition of the var_length_view class, which splits + * a given view into subranges of variable length, as delimited by adjacent + * pairs of values in an index range. + * + * Usage example: + * ``` + * std::vector x {1, 2, 3, 4, 5, 6, 7, 8, 9}; // Data range + * std::vector indices {0, 4, 7, 9}; // Index range + * var_length_view v(x, indices); + * CHECK(std::ranges::equal(v[0], std::vector{1, 2, 3, 4})); + * CHECK(std::ranges::equal(v[1], std::vector{5, 6, 7})); + * CHECK(std::ranges::equal(v[2], std::vector{8, 9})); + * ``` + */ + +#ifndef TILEDB_VAR_LENGTH_VIEW_H +#define TILEDB_VAR_LENGTH_VIEW_H + +#include + +#include "iterator_facade.h" + +/** + * A view that splits a view into subranges of variable length, as delimited by + * a range of indices. The resulting view is a range of subranges, each of which + * is a view into the original data range. The iterator over the var_length_view + * is a random_access_iterator that dereferences to a subrange of the data + * range. + * + * @tparam R Type of the data range, assumed to be a random access range. + * @tparam I Type of the index range, assumed to be a random access range. + * + * @todo R could be a view rather than a range. + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +class var_length_view : public std::ranges::view_base { + // Forward reference of the iterator over the range of variable length data + template + struct private_iterator; + + // The type of the iterator over the var length data range, and the type that + // can index into it + using data_iterator_type = std::ranges::iterator_t; + using data_index_type = std::iter_difference_t; + + // The type of the iterator over the index range -- It should derference to + // something that can index into the data range (e.g., the data_index_type) + using index_iterator_type = std::ranges::iterator_t; + + // They type dereferenced by the iterator is a subrange + using var_length_type = std::ranges::subrange; + using var_length_iterator = private_iterator; + using var_length_const_iterator = private_iterator; + + std::ranges::iterator_t data_begin_; + std::ranges::iterator_t data_end_; + + // const_iterator is c++23. For now we just use an iterator to const + std::ranges::iterator_t index_begin_; + std::ranges::iterator_t index_end_; + + public: + var_length_view(R& data, const I& index) + : data_begin_(std::ranges::begin(data)) + , data_end_(std::ranges::end(data)) + , index_begin_(std::ranges::cbegin(index)) + , index_end_(std::ranges::cend(index) - 1) { + } + + auto begin() { + return var_length_iterator(data_begin_, index_begin_, 0); + } + + auto end() { + return var_length_iterator( + data_begin_, index_begin_, index_end_ - index_begin_); + } + + auto begin() const { + return var_length_const_iterator(data_begin_, index_begin_, 0); + } + + auto end() const { + return var_length_const_iterator( + data_begin_, index_begin_, index_end_ - index_begin_); + } + + auto cbegin() const { + return var_length_const_iterator(data_begin_, index_begin_, 0); + } + + auto cend() const { + return var_length_const_iterator( + data_begin_, index_begin_, index_end_ - index_begin_); + } + + auto size() const { + return index_end_ - index_begin_; + } + + private: + template + struct private_iterator : public iterator_facade> { + using value_type_ = Value; + + data_index_type index_; + data_iterator_type data_begin_; + index_iterator_type offsets_begin_; + + private_iterator() = default; + private_iterator( + data_iterator_type data_begin, + index_iterator_type offsets_begin, + data_index_type index = 0) + : index_(index) + , data_begin_(data_begin) + , offsets_begin_(offsets_begin) { + // ... + } + + /************************************************************************* + * Functions needed for iterator_facade + * Here we just supply the minimum needed to make the iterator work + *************************************************************************/ + + /* + * The critical function for defining the iterator as it determines the + * value type, etc and also does the dereference. In the case of the + * var_length_view, the value type is a subrange of the data range, and + * the subrage return value must be a proxy object, which we return by + * value. This is fine, since we can't assign a subrange to another. + * We do, however, want to be able to modify the contents of the subrange. + */ + value_type_ dereference() const { + return { + data_begin_ + offsets_begin_[index_], + data_begin_ + offsets_begin_[index_ + 1]}; + } + + auto advance(data_index_type n) { + index_ += n; + return *this; + } + + auto distance_to(const private_iterator& other) const { + return other.index_ - index_; + } + + bool operator==(const private_iterator& other) const { + return data_begin_ == other.data_begin_ && + offsets_begin_ == other.offsets_begin_ && index_ == other.index_; + } + + static const bool single_pass_iterator = false; + }; + + // using iterator = private_iterator; +}; + +#endif // TILEDB_VAR_LENGTH_VIEW_H From 910b844dcaccc4b4692578d882201cb95271e49e Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 29 Apr 2024 15:11:17 -0700 Subject: [PATCH 304/456] Implement permutation view for external sort. (#4920) This pull request implements the `permutation_view` class. A `permutation_view` provides a view over an underlying range that is permuted by indirecting with a given index range. It is required that the permutation range is the same size as the data range and that it contain the values 0, 1, ..., N-1, in arbitrary order, where N is the size of the data range. Example: ```c++ std::vector v{1, 2, 3, 4, 5}; std::vector p{4, 3, 2, 1, 0}; permutation_view view(v, p); CHECK(std::ranges::equal(view, std::vector{5, 4, 3, 2, 1})); ``` The implementation uses `iterator_facade` to realize a C++20 standards conformant iterator and so that `permutation_view` satisfies the appropriate C++20 ranges concepts. The `permutation_view` class has been tested with `var_length_view`. Various range and iterator concepts are also tested. (This PR also cleans up the documentation and the structure of the `var_length_view` class.) --- TYPE: IMPROVEMENT DESC: Implement permutation view for external sort. --- tiledb/common/permutation_view.h | 226 ++++++++++++++++++++ tiledb/common/test/CMakeLists.txt | 2 +- tiledb/common/test/unit_permutation_view.cc | 202 +++++++++++++++++ tiledb/common/var_length_view.h | 66 ++++-- 4 files changed, 476 insertions(+), 20 deletions(-) create mode 100644 tiledb/common/permutation_view.h create mode 100644 tiledb/common/test/unit_permutation_view.cc diff --git a/tiledb/common/permutation_view.h b/tiledb/common/permutation_view.h new file mode 100644 index 000000000000..4cb7fa9cd0a2 --- /dev/null +++ b/tiledb/common/permutation_view.h @@ -0,0 +1,226 @@ +/** + * @file permutation_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains the definition of the permutation_view class, which + * applies index indirection to a random access range, as specified by the + * indices in another random access range (the permutation). It is required + * that the permutation range is the same size as the data range and that it + * contain the values 0, 1, ..., N-1, in arbitrary order, where N is the size of + * the data range. For a data range `r` and a permutation `p`, and a + * permutation_view `v`, based on `r` and `p`, the expression `v[i]` is + * equivalent to `r[p[i]]`. + */ + +#ifndef TILEDB_PERMUTATION_VIEW_H +#define TILEDB_PERMUTATION_VIEW_H + +#include +#include +#include "iterator_facade.h" + +/** + * A view that creates a permutation of the underlying view, as determined by + * a permutation range. + * + * @tparam R Type of the data range, assumed to be a random access range. + * @tparam P Type of the index range, assumed to be a random access range. + * + * @todo R could be a view rather than a range. + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +class permutation_view : public std::ranges::view_base { + /** Forward reference of the iterator over the range of permuted data. */ + template + struct private_iterator; + + /** The data type of the permutation view. */ + using data_iterator_type = std::ranges::iterator_t; + + /** The index type of the permutation view indexing into underlying range. */ + using data_index_type = std::iter_difference_t; + + /** The type of the iterator over the index range -- It should derference to + * something that can index into the data range (e.g., the data_index_type)*/ + using index_iterator_type = std::ranges::iterator_t; + + /** The value_type dereferenced by the iterator is the value_type R */ + using permuted_value_type = std::ranges::range_value_t; + + /** The value_type dereferenced by the iterator is the value_type R */ + using permuted_reference = std::ranges::range_reference_t; + + /** The type of the iterator over the permuted data range */ + using permuted_iterator = private_iterator; + + /** The type of the const iterator over the permuted data range */ + using permuted_const_iterator = private_iterator; + + public: + /** Primary constructor */ + permutation_view(R& data, const P& permutation) + : data_begin_(std::ranges::begin(data)) + , data_end_(std::ranges::end(data)) + , index_begin_(std::ranges::begin(permutation)) + , index_end_(std::ranges::end(permutation)) { + // @todo Should this throw an exception instead? + assert( + static_cast(std::ranges::size(data)) == + std::ranges::size(permutation)); + } + + /** Return iterator to the beginning of the permutation view */ + auto begin() { + return permuted_iterator{data_begin_, index_begin_, 0}; + } + + /** Return iterator to the end of the permutation view */ + auto end() { + return permuted_iterator{ + data_begin_, index_begin_, data_end_ - data_begin_}; + } + + /** Return const iterator to the beginning of the permutation view */ + auto begin() const { + return permuted_const_iterator{data_begin_, index_begin_, 0}; + } + + /** Return const iterator to the end of the permutation view */ + auto end() const { + return permuted_const_iterator{ + data_begin_, index_begin_, data_end_ - data_begin_}; + } + + /** Return const iterator to the beginning of the permutation view */ + auto cbegin() const { + return permuted_const_iterator{data_begin_, index_begin_, 0}; + } + + /** Return const iterator to the end of the permutation view */ + auto cend() const { + return permuted_const_iterator{ + data_begin_, index_begin_, data_end_ - data_begin_}; + } + + /** Size of the permutation view */ + auto size() const { + return data_end_ - data_begin_; + } + + /** Accessor */ + auto& operator[](size_t i) const { + // More general? return *(data_begin_ + *(index_begin_ + i)); + return data_begin_[index_begin_[i]]; + } + + private: + /** + * Private iterator class for permuted view + * @tparam Value The type of the value that the iterator dereferences to + */ + template + struct private_iterator + : public iterator_facade> { + // using value_type_ = Value; + + public: + /** Default constructor */ + private_iterator() = default; + + /** + * Primary constructor + * @param data_begin The beginning of the data range + * @param index_begin The beginning of the index range + * @param index Offset into the data range + */ + private_iterator( + data_iterator_type data_begin, + index_iterator_type index_begin, + data_index_type index = 0) + : index_(index) + , data_begin_(data_begin) + , index_begin_(index_begin) { + } + + /** Dereference the iterator */ + // value_type_& + // std::ranges::range_reference_t + Reference dereference() const { + // More general? + // return *(data_begin_ + *(index_begin_ + index_)); + return data_begin_[index_begin_[index_]]; + } + + /** Advance the iterator by n */ + auto advance(data_index_type n) { + index_ += n; + return *this; + } + + /** Return the distance to another iterator */ + auto distance_to(const private_iterator& other) const { + return other.index_ - index_; + } + + /** Equality comparison of two iterators */ + bool operator==(const private_iterator& other) const { + return data_begin_ == other.data_begin_ && + index_begin_ == other.index_begin_ && index_ == other.index_; + } + + private: + /** Index to current location of iterator */ + data_index_type index_; + + /** Iterator to the beginning of the data range */ + data_iterator_type data_begin_; + + /** Iterator to the beginning of the index range */ + index_iterator_type index_begin_; + + /** Flag to indicate that the iterator is not a single pass iterator */ + static const bool single_pass_iterator = false; + }; + + /** Iterator to the beginning of the data range */ + std::ranges::iterator_t data_begin_; + + /** Iterator to the end of the data range */ + std::ranges::iterator_t data_end_; + + /** const_iterator is c++23. For now we just use an iterator to const */ + /** Iterator to the beginning of the index range */ + std::ranges::iterator_t index_begin_; + + /** Iterator to the end of the index range */ + std::ranges::iterator_t index_end_; +}; + +#endif // TILEDB_PERMUTATION_VIEW_H diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 92fe7a1e25aa..7aee2298f311 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_iterator_facade.cc unit_var_length_view.cc) + this_target_sources(main.cc unit_iterator_facade.cc unit_permutation_view.cc unit_var_length_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_permutation_view.cc b/tiledb/common/test/unit_permutation_view.cc new file mode 100644 index 000000000000..bead37c87051 --- /dev/null +++ b/tiledb/common/test/unit_permutation_view.cc @@ -0,0 +1,202 @@ +/** + * @file unit_permutation_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the permutation_view class. + */ + +#include +#include +#include +#include "../permutation_view.h" +#include "../var_length_view.h" + +TEST_CASE("permutation_view: Null test", "[permutation_view][null_test]") { + REQUIRE(true); +} + +// Test that the permutation_view satisfies the expected concepts +TEST_CASE("permutation_view: Range concepts", "[permutation_view][concepts]") { + using test_type = permutation_view, std::vector>; + + CHECK(std::ranges::range); + CHECK(!std::ranges::borrowed_range); + CHECK(std::ranges::sized_range); + CHECK(std::ranges::view); + CHECK(std::ranges::input_range); + CHECK(std::ranges:: + output_range>); + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + CHECK(!std::ranges::contiguous_range); + CHECK(std::ranges::common_range); + CHECK(std::ranges::viewable_range); + + CHECK(std::ranges::view); +} + +// Test that the permutation_view iterators satisfy the expected concepts +TEST_CASE( + "permutation_view: Iterator concepts", "[permutation_view][concepts]") { + using test_type = permutation_view, std::vector>; + using test_type_iterator = std::ranges::iterator_t; + using test_type_const_iterator = std::ranges::iterator_t; + + CHECK(std::input_or_output_iterator); + CHECK(std::input_or_output_iterator); + CHECK(std::input_iterator); + CHECK(std::input_iterator); + CHECK(std::output_iterator< + test_type_iterator, + std::ranges::range_value_t>); + CHECK(!std::output_iterator< + test_type_const_iterator, + std::ranges::range_value_t>); + CHECK(std::forward_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + CHECK(std::random_access_iterator); +} + +// Test that the permutation_view value_type satisfies the expected concepts +TEST_CASE( + "permutation_view: value_type concepts", "[permutation_view][concepts]") { + using test_type = permutation_view, std::vector>; + CHECK(std::ranges::range); + + using test_iterator_type = std::ranges::iterator_t; + using test_iterator_value_type = std::iter_value_t; + using test_iterator_reference_type = + std::iter_reference_t; + + using range_value_type = std::ranges::range_value_t; + using range_reference_type = std::ranges::range_reference_t; + + CHECK(std::is_same_v); + CHECK(std::is_same_v); +} + +TEST_CASE("permutation_view: simple constructor", "permutation_view") { + std::vector v{1, 2, 3, 4, 5}; + std::vector p{4, 3, 2, 1, 0}; + permutation_view view(v, p); + CHECK(view.size() == 5); + CHECK(view[0] == 5); + CHECK(view[1] == 4); + CHECK(view[2] == 3); + CHECK(view[3] == 2); + CHECK(view[4] == 1); + + int off = 1; + for (auto&& i : view) { + CHECK(i == 6 - off++); + } + + CHECK(std::ranges::equal(view, std::vector{5, 4, 3, 2, 1})); +} + +TEST_CASE( + "permutation_view: check various iterator properties hold", + "[permutation_view]") { + std::vector v{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + std::vector p{9, 1, 3, 2, 8, 6, 5, 7, 4, 0}; + std::vector expected{10, 2, 4, 3, 9, 7, 6, 8, 5, 1}; + + permutation_view view(v, p); + CHECK(std::equal(view.begin(), view.end(), expected.begin())); + CHECK(std::equal(view.cbegin(), view.cend(), expected.begin())); + CHECK(std::ranges::equal(view, expected)); + + auto it = view.begin(); + CHECK(*it == 10); + CHECK(*(it + 1) == 2); + CHECK(it[2] == 4); + it[3] = 100; + CHECK(it[3] == 100); + CHECK(view[3] == 100); + *it++ = 200; + CHECK(view[0] == 200); + CHECK(*it == 2); + + CHECK(it == view.begin() + 1); + CHECK(it > view.begin()); + CHECK(it >= view.begin()); + CHECK(view.begin() < it); + CHECK(view.begin() <= it); + CHECK(it < view.end()); + CHECK(it <= view.end()); + CHECK(view.end() > it); + CHECK(view.end() >= it); + + --it; + CHECK(it == view.begin()); + view[9] = 1000; + CHECK(it[9] == 1000); + CHECK(view.end() - view.begin() == 10); + CHECK(view.end() - it == 10); + ++it; + CHECK(it == view.begin() + 1); + CHECK(view.end() - it == 9); + + auto it2 = it + 5; + CHECK(it2 - it == 5); + CHECK(it2 - 5 == it); + CHECK(it2 - 6 == view.begin()); + CHECK(it2 - view.begin() == 6); + + CHECK(*it2 == 6); + + auto cit = view.cbegin(); + (void)cit; + // Error: Not assignable + // *cit = 1; +} + +TEST_CASE( + "permutation_view: permute var length view", + "[permutation_view][var_length_view]") { + std::vector q = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + std::vector p = {0, 2, 7, 10}; + std::vector o = {2, 0, 1}; + + std::vector> expected{ + {21.0, 20.0}, {19.0, 18.0, 17.0, 16.0, 15.0}, {14.0, 13.0, 12.0}}; + auto w = var_length_view(q, p); + CHECK(std::ranges::equal(*(w.begin()), expected[0])); + CHECK(std::ranges::equal(*(w.begin() + 1), expected[1])); + CHECK(std::ranges::equal(*(w.begin() + 2), expected[2])); + + auto x = permutation_view(w, o); + CHECK(std::ranges::equal(*(x.begin()), expected[2])); + CHECK(std::ranges::equal(*(x.begin() + 1), expected[0])); + CHECK(std::ranges::equal(*(x.begin() + 2), expected[1])); +} diff --git a/tiledb/common/var_length_view.h b/tiledb/common/var_length_view.h index 09feaaed52f6..adec0fd87309 100644 --- a/tiledb/common/var_length_view.h +++ b/tiledb/common/var_length_view.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2024 TileDB, Inc. + * @copyright Copyright (c) 2018-2021 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -69,28 +69,27 @@ class var_length_view : public std::ranges::view_base { template struct private_iterator; - // The type of the iterator over the var length data range, and the type that - // can index into it + /** The type of the iterator over the var length data range */ using data_iterator_type = std::ranges::iterator_t; + + /** The type that can index into the var length data range */ using data_index_type = std::iter_difference_t; - // The type of the iterator over the index range -- It should derference to - // something that can index into the data range (e.g., the data_index_type) + /** The type of the iterator over the index range -- It should dereference to + * something that can index into the data range (e.g., the data_index_type) */ using index_iterator_type = std::ranges::iterator_t; - // They type dereferenced by the iterator is a subrange + /** The type dereferenced by the iterator is a subrange */ using var_length_type = std::ranges::subrange; - using var_length_iterator = private_iterator; - using var_length_const_iterator = private_iterator; - std::ranges::iterator_t data_begin_; - std::ranges::iterator_t data_end_; + /** The type of the iterator over the var length view */ + using var_length_iterator = private_iterator; - // const_iterator is c++23. For now we just use an iterator to const - std::ranges::iterator_t index_begin_; - std::ranges::iterator_t index_end_; + /** The type of the const iterator over the var length view */ + using var_length_const_iterator = private_iterator; public: + /** Primary constructor */ var_length_view(R& data, const I& index) : data_begin_(std::ranges::begin(data)) , data_end_(std::ranges::end(data)) @@ -98,33 +97,40 @@ class var_length_view : public std::ranges::view_base { , index_end_(std::ranges::cend(index) - 1) { } + /** Return iterator to the beginning of the var length view */ auto begin() { return var_length_iterator(data_begin_, index_begin_, 0); } + /** Return iterator to the end of the var length view */ auto end() { return var_length_iterator( data_begin_, index_begin_, index_end_ - index_begin_); } + /** Return const iterator to the beginning of the var length view */ auto begin() const { return var_length_const_iterator(data_begin_, index_begin_, 0); } + /** Return const iterator to the end of the var length view */ auto end() const { return var_length_const_iterator( data_begin_, index_begin_, index_end_ - index_begin_); } + /** Return const iterator to the beginning of the var length view */ auto cbegin() const { return var_length_const_iterator(data_begin_, index_begin_, 0); } + /** Return const iterator to the end of the var length view */ auto cend() const { return var_length_const_iterator( data_begin_, index_begin_, index_end_ - index_begin_); } + /** Return the number of subranges in the var length view */ auto size() const { return index_end_ - index_begin_; } @@ -134,11 +140,10 @@ class var_length_view : public std::ranges::view_base { struct private_iterator : public iterator_facade> { using value_type_ = Value; - data_index_type index_; - data_iterator_type data_begin_; - index_iterator_type offsets_begin_; - + /** Default constructor */ private_iterator() = default; + + /** Primary constructor */ private_iterator( data_iterator_type data_begin, index_iterator_type offsets_begin, @@ -146,7 +151,6 @@ class var_length_view : public std::ranges::view_base { : index_(index) , data_begin_(data_begin) , offsets_begin_(offsets_begin) { - // ... } /************************************************************************* @@ -168,24 +172,48 @@ class var_length_view : public std::ranges::view_base { data_begin_ + offsets_begin_[index_ + 1]}; } + /** Advance the iterator by n */ auto advance(data_index_type n) { index_ += n; return *this; } + /** Return the distance to another iterator */ auto distance_to(const private_iterator& other) const { return other.index_ - index_; } + /** Compare two iterators for equality */ bool operator==(const private_iterator& other) const { return data_begin_ == other.data_begin_ && offsets_begin_ == other.offsets_begin_ && index_ == other.index_; } + /** Flag to indicate that the iterator is not a single pass iterator */ static const bool single_pass_iterator = false; + + /** The index to the current location of the iterator */ + data_index_type index_; + + /** Iterator to the beginning of the data range */ + data_iterator_type data_begin_; + + /** Iterator to the beginning of the index range */ + index_iterator_type offsets_begin_; }; - // using iterator = private_iterator; + /** The beginning of the data range */ + std::ranges::iterator_t data_begin_; + + /** The end of the data range */ + std::ranges::iterator_t data_end_; + + // const_iterator is c++23. For now we just use an iterator to const + /** The beginning of the index range */ + std::ranges::iterator_t index_begin_; + + /** The end of the index range */ + std::ranges::iterator_t index_end_; }; #endif // TILEDB_VAR_LENGTH_VIEW_H From 2e93e43112ebe232f37ea463d5700fd9c38bb34c Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Tue, 30 Apr 2024 06:46:49 -0700 Subject: [PATCH 305/456] Implement proxy sort for external sort. (#4922) This PR provides four overloads of a "proxy sort" algorithm and four overloads of a stable "proxy sort". Instead of directly sorting a range `x` such that `x[I] <= x[I+1]`, a proxy sort sorts a permutation range `p` so that `x[p[I]] <= x[p[I+1`]]`. Four variations of proxy_sort and stable_proxy_sort are provided * `proxy_sort_no_init`: Sorts the permutation, given an initial permutation. By default, < is used to compare, but an overload is provided that takes a comparison function. * `proxy_sort`: Sorts the permutation, initializing it to the identity. By default, < is used to compare, but an overload is provided that takes a comparison function. * `stable_proxy_sort_no_init`: Performs stable sort of the permutation, given an initial permutation. By default, < is used to compare, but an overload is provided that takes a comparison function. * `stable_proxy_sort`: Performs stable sort of the permutation, initializing it to the identity. By default, < is used to compare, but an overload is provided that takes a comparison function. Example: ```c++ std::vector x{3, 1, 4, 1, 5, 9, 2, 6, 5, 3}; std::vector perm(x.size()); std::vector expected{1, 1, 2, 3, 3, 4, 5, 5, 6, 9}; proxy_sort(x, perm); for (size_t i = 0; i < x.size(); ++i) { CHECK(x[perm[i]] == expected[i]); } ``` --- TYPE: IMPROVEMENT DESC: Implement proxy sort for external sort. --- tiledb/common/proxy_sort.h | 235 ++++++++++++++++++++++++++ tiledb/common/test/CMakeLists.txt | 2 +- tiledb/common/test/unit_proxy_sort.cc | 191 +++++++++++++++++++++ 3 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 tiledb/common/proxy_sort.h create mode 100644 tiledb/common/test/unit_proxy_sort.cc diff --git a/tiledb/common/proxy_sort.h b/tiledb/common/proxy_sort.h new file mode 100644 index 000000000000..f3996c941a46 --- /dev/null +++ b/tiledb/common/proxy_sort.h @@ -0,0 +1,235 @@ +/** + * @file proxy_sort.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements a proxy sort algorithm that will be used as part of + * TileDB external sort. + * + * Four variations of proxy_sort and stable_proxy_sort are provided + * * proxy_sort_no_init: Sorts the permutation, given an initial permutation. + * By default, < is used to compare, but an overload is provided that takes a + * comparison function. + * + * * proxy_sort: Sorts the permutation, initializing it to the identity. By + * default, < is used to compare, but an overload is provided that takes a + * comparison function. + * + * * stable_proxy_sort_no_init: Performs stable sort of the permutation, given + * an initial permutation. By default, < is used to compare, but an overload is + * provided that takes a comparison function. + * + * * stable_proxy_sort: Performs stable sort of the permutation, initializing it + * to the identity. By default, < is used to compare, but an overload is + * provided that takes a comparison function. + * + * std::vector x{3, 1, 4, 1, 5, 9, 2, 6, 5, 3}; + * std::vector perm(x.size()); + * std::vector expected{1, 1, 2, 3, 3, 4, 5, 5, 6, 9}; + * proxy_sort(x, perm); + * for (size_t i = 0; i < x.size(); ++i) { + * CHECK(x[perm[i]] == expected[i]); + * } + * + * @todo Parallelize + */ + +#ifndef TILEDB_PROXY_SORT_H +#define TILEDB_PROXY_SORT_H + +#include +#include +#include +#include + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Does not initialize + * perm. It is assumed that perm is a valid permuation of [0, x.size()). + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +void proxy_sort_no_init(const R& x, I& perm) { + assert(perm.size() == x.size()); + std::sort( + perm.begin(), perm.end(), [&](auto a, auto b) { return x[a] < x[b]; }); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Initializes perm to + * the identity permutation. + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +void proxy_sort(const R& x, I& perm) { + assert(perm.size() == x.size()); + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Does not initialize + * perm. It is assumed that perm is a valid permuation of [0, x.size()). + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @tparam Compare Type of the comparison function + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I, + class Compare> +void proxy_sort_no_init(const R& x, I& perm, Compare comp) { + assert(perm.size() == x.size()); + std::sort(perm.begin(), perm.end(), [&](auto a, auto b) { + return comp(x[a], x[b]); + }); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Initializes perm to + * the identity permutation. + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @tparam Compare Type of the comparison function + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I, + class Compare> +void proxy_sort(const R& x, I& perm, Compare comp) { + assert(perm.size() == x.size()); + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm, comp); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Does not initialize + * perm. It is assumed that perm is a valid permuation of [0, x.size()). + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +void stable_proxy_sort_no_init(const R& x, I& perm) { + assert(perm.size() == x.size()); + std::stable_sort( + perm.begin(), perm.end(), [&](auto a, auto b) { return x[a] < x[b]; }); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Initializes perm to + * the identity permutation. + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +void stable_proxy_sort(const R& x, I& perm) { + assert(perm.size() == x.size()); + std::iota(perm.begin(), perm.end(), 0); + stable_proxy_sort_no_init(x, perm); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Does not initialize + * perm. It is assumed that perm is a valid permuation of [0, x.size()). + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @tparam Compare Type of the comparison function + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I, + class Compare> +void stable_proxy_sort_no_init(const R& x, I& perm, Compare comp) { + assert(perm.size() == x.size()); + std::stable_sort(perm.begin(), perm.end(), [&](auto a, auto b) { + return comp(x[a], x[b]); + }); +} + +/** + * Sorts the elements of the input range `perm` such that the elements of the + * input range x are sorted when indirected through `perm`. Initializes perm to + * the identity permutation. + * @tparam R Type of the random access range x + * @tparam I Type of the random access range perm. + * @tparam Compare Type of the comparison function + * @param x Random access range to "sort" + * @param perm Random access range of indices to be permuted such that + * x[perm[i]] is sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I, + class Compare> +void stable_proxy_sort(const R& x, I& perm, Compare comp) { + assert(perm.size() == x.size()); + std::iota(perm.begin(), perm.end(), 0); + stable_proxy_sort_no_init(x, perm, comp); +} + +#endif // TILEDB_PROXY_SORT_H diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 7aee2298f311..801bf975df2d 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_iterator_facade.cc unit_permutation_view.cc unit_var_length_view.cc) + this_target_sources(main.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_proxy_sort.cc b/tiledb/common/test/unit_proxy_sort.cc new file mode 100644 index 000000000000..1704d7d0e9bf --- /dev/null +++ b/tiledb/common/test/unit_proxy_sort.cc @@ -0,0 +1,191 @@ +/** + * @file unit_proxy_sort.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for `proxy_sort` that will be used as + * part of TileDB external sort. + */ + +#include +#include +#include +#include +#include "../proxy_sort.h" + +TEST_CASE("proxy_sort: Null test", "[proxy_sort][null_test]") { + REQUIRE(true); +} + +TEST_CASE("proxy_sort: four ways, less than", "[proxy_sort]") { + std::vector x{3, 1, 4, 1, 5, 9, 2, 6, 5, 3}; + std::vector y(x); + std::vector expected{1, 1, 2, 3, 3, 4, 5, 5, 6, 9}; + std::vector perm(x.size()); + + SECTION("proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm); + } + SECTION("proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm); + } + SECTION("proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm, std::less<>()); + } + SECTION("proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm, std::less<>()); + } + SECTION("stable_proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + stable_proxy_sort_no_init(x, perm); + } + SECTION("stable_proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + stable_proxy_sort(x, perm); + } + SECTION("stable_proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + stable_proxy_sort_no_init(x, perm, std::less<>()); + } + SECTION("stable_proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + stable_proxy_sort(x, perm, std::less<>()); + } + + CHECK(std::equal(x.begin(), x.end(), y.begin())); + + for (size_t i = 1; i < x.size(); ++i) { + CHECK(x[perm[i - 1]] <= x[perm[i]]); + } + for (size_t i = 0; i < x.size(); ++i) { + CHECK(x[perm[i]] == expected[i]); + } +} + +TEST_CASE("proxy_sort: strings two ways, greater than", "[proxy_sort]") { + std::vector x{ + "three", + "point", + "one", + "four", + "one", + "five", + "nine", + "two", + "six", + "five", + "three"}; + std::vector y(x); + std::vector expected{ + "five", + "five", + "four", + "nine", + "one", + "one", + "point", + "six", + "three", + "three", + "two"}; + std::vector reverse_expected(expected.rbegin(), expected.rend()); + std::vector perm(x.size()); + + SECTION("less than") { + SECTION("proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm); + } + SECTION("proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm); + } + SECTION("proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm, std::less<>()); + } + SECTION("proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm, std::less<>()); + } + SECTION("stable_proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm); + } + SECTION("stable_proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm); + } + SECTION("stable_proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm, std::less<>()); + } + SECTION("stable_proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm, std::less<>()); + } + + CHECK(std::equal(x.begin(), x.end(), y.begin())); + + for (size_t i = 1; i < x.size(); ++i) { + CHECK(x[perm[i - 1]] <= x[perm[i]]); + } + for (size_t i = 0; i < x.size(); ++i) { + CHECK(x[perm[i]] == expected[i]); + } + } + SECTION("greater than") { + SECTION("proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + proxy_sort_no_init(x, perm, std::greater<>()); + } + SECTION("proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + proxy_sort(x, perm, std::greater<>()); + } + SECTION("stable_proxy_sort_no_init") { + std::iota(perm.begin(), perm.end(), 0); + stable_proxy_sort_no_init(x, perm, std::greater<>()); + } + SECTION("stable_proxy_sort") { + std::fill(perm.begin(), perm.end(), 0); + stable_proxy_sort(x, perm, std::greater<>()); + } + CHECK(std::equal(x.begin(), x.end(), y.begin())); + + for (size_t i = 1; i < x.size(); ++i) { + CHECK(x[perm[i - 1]] >= x[perm[i]]); + } + for (size_t i = 0; i < x.size(); ++i) { + CHECK(x[perm[i]] == reverse_expected[i]); + } + } +} From 7f511fdac2aa00349deaac2af7285027e01b57e9 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Wed, 1 May 2024 00:41:26 -0700 Subject: [PATCH 306/456] Implement alt var length view for external sort. (#4925) This PR implements an alternate view for var length data. Instead of maintaining an offsets range, as in the first `var_length_view`, this class materializes all of the individual subranges over the var length data. The associated unit tests for this PR include tests for sorting. Advantages: * The `alt_var_length_view` has the advantage of not returning a proxy when accessed and, as a result, can be directly sorted. Also, not having to create the proxy on access can result in more efficiency. * The `alt_var_length_view` class can be constructed with either the "Arrow" format for offsets or the "TileDB" format. In the latter case, the constructor takes the offsets ranges, as well as the final value. Disadvanatages: * The `alt_var_length_view` requires more storage. An individual `std::ranges::subrange` consumes 16 bytes, whereas a `size_t` consumes 8 bytes. On the other hand, if sorting of the original `var_length_view` is of interest, the offsets range plus a permutation range will also result in 16 bytes storage per var length element. Notes: * The `alt_var_length_view` class ends up being a fairly thin wrapper around `std::vector>`. However, it is considered not to be good practice to inherit from `std::vector`, so it was kept as member data. * Because this view does maintain internal data that is not O(1), the copy constructor and copy assignment operator are disabled. --- TYPE: IMPROVEMENT DESC: Implement alt var length view for external sort. --- TYPE: NO_HISTORY | FEATURE | BUG | IMPROVEMENT | DEPRECATION | C_API | CPP_API | BREAKING_BEHAVIOR | BREAKING_API | FORMAT DESC: --- tiledb/common/alt_var_length_view.h | 183 +++++++ tiledb/common/test/CMakeLists.txt | 2 +- .../common/test/unit_alt_var_length_view.cc | 459 ++++++++++++++++++ tiledb/common/test/unit_iterator_facade.cc | 87 +--- tiledb/common/var_length_view.h | 2 +- 5 files changed, 648 insertions(+), 85 deletions(-) create mode 100644 tiledb/common/alt_var_length_view.h create mode 100644 tiledb/common/test/unit_alt_var_length_view.cc diff --git a/tiledb/common/alt_var_length_view.h b/tiledb/common/alt_var_length_view.h new file mode 100644 index 000000000000..1c4bab8a9660 --- /dev/null +++ b/tiledb/common/alt_var_length_view.h @@ -0,0 +1,183 @@ +/** + * @file alt_alt_var_length_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains the definition of the alt_var_length_view class, which + * splits a given view into subranges of variable length, as delimited by + * adjacent pairs of values in an index range. + * + * The difference between `alt_var_length_view` and `var_length_view` is that + * `alt_var_length_view` maintains a materialized range of subranges, whereas + * `var_length_view` creates subrange views on the fly as proxy objects. As a + * result + * * An `alt_var_length_view` does not need to refer to the offsets array after + * it is constructed + * * An `alt_var_length_view` can be sorted + * + * + * Usage example: + * ``` + * std::vector x {1, 2, 3, 4, 5, 6, 7, 8, 9}; // Data range + * std::vector indices {0, 4, 7, 9}; // Index range + * alt_var_length_view v(x, indices); + * CHECK(std::ranges::equal(v[0], std::vector{1, 2, 3, 4})); + * CHECK(std::ranges::equal(v[1], std::vector{5, 6, 7})); + * CHECK(std::ranges::equal(v[2], std::vector{8, 9})); + * ``` + */ + +#ifndef TILEDB_ALT_VAR_LENGTH_VIEW_H +#define TILEDB_ALT_VAR_LENGTH_VIEW_H + +#include + +#include "iterator_facade.h" + +/** + * A view that splits a view into subranges of variable length, as delimited by + * a range of indices. The resulting view is a range of subranges, each of which + * is a view into the original data range. The iterator over the + * alt_var_length_view is a random_access_iterator that dereferences to a + * subrange of the data range. + * + * @tparam R Type of the data range, assumed to be a random access range. + * @tparam I Type of the index range, assumed to be a random access range. + * + * @todo R could be a view rather than a range. + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range I> +class alt_var_length_view : public std::ranges::view_base { + // Forward reference of the iterator over the range of variable length data + template + struct private_iterator; + + /** The type of the iterator over the var length data range */ + using data_iterator_type = std::ranges::iterator_t; + + /** The type that can index into the var length data range */ + using data_index_type = std::iter_difference_t; + + /** The type dereferenced by the iterator is a subrange */ + using var_length_type = std::ranges::subrange; + + /** The type of the iterator over the var length view */ + using var_length_iterator = private_iterator; + + /** The type of the const iterator over the var length view */ + using var_length_const_iterator = private_iterator; + + public: + /***************************************************************************** + * Constructors + * @note: Since this view has ownership of the subranges, we don't allow + * copying, but do allow moves. + ****************************************************************************/ + + /** Default constructor */ + alt_var_length_view() = default; + + /** Copy constructor is deleted */ + alt_var_length_view(const alt_var_length_view&) = delete; + + /** Move constructor */ + alt_var_length_view(alt_var_length_view&&) = default; + + /** Copy assignment is deleted */ + alt_var_length_view& operator=(const alt_var_length_view&) = delete; + + /** Move assignment */ + alt_var_length_view& operator=(alt_var_length_view&&) = default; + + /** Primary constructor. All offsets are contained in the input (notably, the + * index to the end of the data range). */ + alt_var_length_view(R& data, const I& index) { + auto data_begin(std::ranges::begin(data)); + + subranges_.reserve(std::ranges::size(index) - 1); + for (size_t i = 0; i < std::ranges::size(index) - 1; ++i) { + subranges_.emplace_back(data_begin + index[i], data_begin + index[i + 1]); + } + } + + /** Constructor. The offsets do not contain the final index value (which would + * be the end of the data range), so the final index is passed in as a + * separate argument. + */ + alt_var_length_view(R& data, const I& index, data_index_type end_index) { + auto data_begin(std::ranges::begin(data)); + + subranges_.reserve(std::ranges::size(index) - 1); + + for (size_t i = 0; i < std::ranges::size(index) - 1; ++i) { + subranges_.emplace_back(data_begin + index[i], data_begin + index[i + 1]); + } + subranges_.emplace_back(data_begin + index.back(), data_begin + end_index); + } + + /** Return iterator to the beginning of the var length view */ + auto begin() { + return subranges_.begin(); + } + + /** Return iterator to the end of the var length view */ + auto end() { + return subranges_.end(); + } + + /** Return const iterator to the beginning of the var length view */ + auto begin() const { + return subranges_.cbegin(); + } + + /** Return const iterator to the end of the var length view */ + auto end() const { + return subranges_.cend(); + } + + /** Return const iterator to the beginning of the var length view */ + auto cbegin() const { + return subranges_.cbegin(); + } + + /** Return const iterator to the end of the var length view */ + auto cend() const { + return subranges_.cend(); + } + + /** Return the number of subranges in the var length view */ + auto size() const { + return std::ranges::size(subranges_); + } + + private: + std::vector subranges_; +}; + +#endif // TILEDB_ALT_VAR_LENGTH_VIEW_H diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 801bf975df2d..c5af2d47d80b 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc) + this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_alt_var_length_view.cc b/tiledb/common/test/unit_alt_var_length_view.cc new file mode 100644 index 000000000000..fd56d10aec09 --- /dev/null +++ b/tiledb/common/test/unit_alt_var_length_view.cc @@ -0,0 +1,459 @@ +/** + * @file unit_alt_var_length_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the alt_var_length_view class. + */ + +#include +#include +#include +#include "../alt_var_length_view.h" + +TEST_CASE( + "alt_var_length_view: Null test", "[alt_var_length_view][null_test]") { + REQUIRE(true); +} + +// Test that the alt_var_length_view satisfies the expected concepts +TEST_CASE( + "alt_var_length_view: Range concepts", "[alt_var_length_view][concepts]") { + using test_type = alt_var_length_view, std::vector>; + + CHECK(std::ranges::range); + CHECK(!std::ranges::borrowed_range); + CHECK(std::ranges::sized_range); + CHECK(std::ranges::view); + CHECK(std::ranges::input_range); + + // @todo: Do we actually want alt_var_length_view to be an output_range? + CHECK(std::ranges:: + output_range>); + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + + // @todo: Do we actually want alt_var_length_view to be a contiguous_range? + CHECK(std::ranges::contiguous_range); + CHECK(std::ranges::common_range); + CHECK(std::ranges::viewable_range); + + CHECK(std::ranges::view); +} + +// Test that the alt_var_length_view iterators satisfy the expected concepts +TEST_CASE( + "alt_var_length_view: Iterator concepts", + "[alt_var_length_view][concepts]") { + using test_type = alt_var_length_view, std::vector>; + using test_type_iterator = std::ranges::iterator_t; + using test_type_const_iterator = std::ranges::iterator_t; + + CHECK(std::input_or_output_iterator); + CHECK(std::input_or_output_iterator); + CHECK(std::input_iterator); + CHECK(std::input_iterator); + + // @todo: Do we actually want alt_var_length_view to be an output_range? + CHECK(std::output_iterator< + test_type_iterator, + std::ranges::range_value_t>); + CHECK(!std::output_iterator< + test_type_const_iterator, + std::ranges::range_value_t>); + CHECK(std::forward_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + CHECK(std::random_access_iterator); +} + +// Test that the alt_var_length_view value_type satisfies the expected concepts +TEST_CASE( + "alt_var_length_view: value_type concepts", + "[alt_var_length_view][concepts]") { + using test_type = alt_var_length_view, std::vector>; + CHECK(std::ranges::range); + + using test_iterator_type = std::ranges::iterator_t; + using test_iterator_value_type = std::iter_value_t; + using test_iterator_reference_type = + std::iter_reference_t; + + using range_value_type = std::ranges::range_value_t; + using range_reference_type = std::ranges::range_reference_t; + + CHECK(std::is_same_v); + CHECK(std::is_same_v); +} + +// Simple test that the alt_var_length_view can be constructed +TEST_CASE("alt_var_length_view: Basic constructor", "[alt_var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + + SECTION("Arrow format") { + std::vector o = {0, 3, 6, 10}; + auto u = alt_var_length_view(r, o); + auto v = alt_var_length_view{r, o}; + alt_var_length_view w(r, o); + alt_var_length_view x{r, o}; + + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + } + SECTION("TileDB format") { + std::vector o = {0, 3, 6}; + auto u = alt_var_length_view(r, o, 10); + auto v = alt_var_length_view{r, o, 10}; + alt_var_length_view w(r, o, 10); + alt_var_length_view x{r, o, 10}; + + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + } +} + +// Check that the sizes of the alt_var_length_view are correct +TEST_CASE("alt_var_length_view: size()", "[alt_var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + + alt_var_length_view, std::vector> v; + SECTION("arrow format") { + std::vector o = {0, 3, 6, 10}; + v = alt_var_length_view{r, o}; + } + CHECK(size(v) == 3); + std::vector o = {0, 3, 6}; + v = alt_var_length_view{r, o, 10}; + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 4); + + size_t count = 0; + for (auto i : v) { + (void)i; + count++; + } + CHECK(count == 3); + count = 0; + for (auto i = v.begin(); i != v.end(); ++i) { + count++; + } + CHECK(count == 3); +} + +// Check that various properties of iterators hold +TEST_CASE("alt_var_length_view: Basic iterators", "[alt_var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + + alt_var_length_view, std::vector> v; + auto end_v = GENERATE(0, 10); + + if (end_v == 0) { + std::vector o = {0, 3, 6, 10}; + v = alt_var_length_view(r, o); + } else { + std::vector o = {0, 3, 6}; + v = alt_var_length_view(r, o, end_v); + } + + auto a = v.begin(); + auto b = v.end(); + auto c = begin(v); + auto d = end(v); + + SECTION("Check begin and end") { + REQUIRE(a == a); + CHECK(a == c); + CHECK(b == d); + } + SECTION("Check dereference") { + CHECK(std::equal(a->begin(), a->end(), r.begin())); + CHECK(std::equal(a->begin(), a->end(), c->begin())); + // CHECK(std::ranges::equal(*a, r)); // size() not the same + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("Check pre-increment + dereference") { + ++a; + ++c; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 3)); + CHECK(std::ranges::equal(*a, *c)); + ++a; + ++c; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 6)); + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("Check post-increment + dereference") { + a++; + c++; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 3)); + CHECK(std::ranges::equal(*a, *c)); + a++; + c++; + CHECK(std::equal(a->begin(), a->end(), c->begin())); + CHECK(std::equal(a->begin(), a->end(), r.begin() + 6)); + CHECK(std::ranges::equal(*a, *c)); + } + SECTION("operator[]") { + CHECK(std::equal(a[0].begin(), a[0].end(), r.begin())); + CHECK(std::equal(a[1].begin(), a[1].end(), r.begin() + 3)); + CHECK(std::equal(a[2].begin(), a[2].end(), r.begin() + 6)); + CHECK(std::equal(a[0].begin(), a[0].end(), c[0].begin())); + CHECK(std::equal(a[1].begin(), a[1].end(), c[1].begin())); + CHECK(std::equal(a[2].begin(), a[2].end(), c[2].begin())); + CHECK(!std::equal(a[0].begin(), a[0].end(), c[1].begin())); + CHECK(!std::equal(a[0].begin(), a[0].end(), c[2].begin())); + + CHECK(a[0][0] == 1.0); + CHECK(a[0][1] == 2.0); + CHECK(a[0][2] == 3.0); + CHECK(a[1][0] == 4.0); + CHECK(a[1][1] == 5.0); + CHECK(a[1][2] == 6.0); + CHECK(a[2][0] == 7.0); + CHECK(a[2][1] == 8.0); + CHECK(a[2][2] == 9.0); + CHECK(a[2][3] == 10.0); + } + SECTION("More operator[]") { + size_t count = 0; + for (auto i : v) { + for (auto j : i) { + ++count; + CHECK(j == count); + } + } + } + SECTION("Equality, etc") { + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + + CHECK(a != b); + CHECK(!(a == b)); + CHECK(a < b); + CHECK(!(a > b)); + CHECK(a <= b); + CHECK(!(a >= b)); + + ++a; + CHECK(a != c); + CHECK(!(a == c)); + CHECK(!(a < c)); + CHECK(a > c); + CHECK(!(a <= c)); + CHECK(a >= c); + --a; + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + c++; + CHECK(a != c); + CHECK(!(a == c)); + CHECK(!(a > c)); + CHECK(a < c); + CHECK(!(a >= c)); + CHECK(a <= c); + c--; + CHECK(a == c); + CHECK(!(a != c)); + CHECK(!(a < c)); + CHECK(!(a > c)); + CHECK(a <= c); + CHECK(a >= c); + } +} + +// Test that we can read and write to the elements of the alt_var_length_view +TEST_CASE("alt_var_length_view: Viewness", "[alt_var_length_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector s = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + std::vector m = {0, 3, 6, 10}; + std::vector q = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + std::vector p = {0, 2, 7, 10}; + std::vector n = {0, 3, 6, 10}; + + alt_var_length_view, std::vector> u, v, w, x, y, + z; + SECTION("Arrow format") { + u = alt_var_length_view(q, n); + v = alt_var_length_view(r, o); + w = alt_var_length_view(q, p); + x = alt_var_length_view(r, m); + y = alt_var_length_view(s, m); + z = alt_var_length_view(s, n); + } + SECTION("TileDB format") { + std::vector o = {0, 3, 6}; + std::vector p = {0, 2, 7}; + u = alt_var_length_view(q, o, 10); + v = alt_var_length_view(r, o, 10); + w = alt_var_length_view(q, p, 10); + x = alt_var_length_view(r, o, 10); + y = alt_var_length_view(s, o, 10); + z = alt_var_length_view(s, o, 10); + } + + REQUIRE(v.begin() == v.begin()); + CHECK(v.begin() != w.begin()); + CHECK(v.begin() != u.begin()); + CHECK(w.begin() != u.begin()); + + REQUIRE(v.end() == v.end()); + CHECK(v.end() != w.end()); + CHECK(v.end() != u.end()); + CHECK(w.end() != u.end()); + + for (size_t i = 0; i < 3; ++i) { + CHECK(v.size() == x.size()); + CHECK(std::equal( + v.begin()[i].begin(), v.begin()[i].end(), x.begin()[i].begin())); + for (size_t j = 0; j < x.size(); ++j) { + CHECK(v.begin()[i][j] == x.begin()[i][j]); + } + } + for (auto&& i : y) { + for (auto& j : i) { + j += 13.0; + } + } + for (size_t i = 0; i < 3; ++i) { + CHECK(v.size() == x.size()); + for (size_t j = 0; j < x.size(); ++j) { + CHECK(y.begin()[i][j] == v.begin()[i][j] + 13.0); + CHECK(z.begin()[i][j] == y.begin()[i][j]); + CHECK(z.cbegin()[i][j] == y.begin()[i][j]); + CHECK(z.cbegin()[i][j] == y.cbegin()[i][j]); + CHECK(z.cbegin()[i][j] == z.begin()[i][j]); + } + } +} + +// Test that the alt_var_length_view can be sorted +TEST_CASE("alt_var_length_view: Sort", "[alt_var_length_view]") { + std::vector r = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}; + std::vector o = {0, 3, 6, 10, 12}; + alt_var_length_view, std::vector> v(r, o); + + SECTION("Sort by size, ascending") { + std::vector> expected = { + {11.0, 12.0}, + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {7.0, 8.0, 9.0, 10.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { + return a.size() < b.size(); + }); + + CHECK(v.begin()->size() == 2); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 3); + CHECK((v.begin() + 3)->size() == 4); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + SECTION("Sort by size, descending") { + std::vector> expected = { + {7.0, 8.0, 9.0, 10.0}, + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {11.0, 12.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { + return a.size() > b.size(); + }); + + CHECK(v.begin()->size() == 4); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 3); + CHECK((v.begin() + 3)->size() == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + SECTION("Sort by first element, ascending") { + std::vector> expected = { + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {7.0, 8.0, 9.0, 10.0}, + {11.0, 12.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a[0] < b[0]; }); + + CHECK(v.begin()->size() == 3); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 4); + CHECK((v.begin() + 3)->size() == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + SECTION("Sort by first element, descending") { + std::vector> expected = { + {11.0, 12.0}, + {7.0, 8.0, 9.0, 10.0}, + {4.0, 5.0, 6.0}, + {1.0, 2.0, 3.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a[0] > b[0]; }); + + CHECK(v.begin()->size() == 2); + CHECK((v.begin() + 1)->size() == 4); + CHECK((v.begin() + 2)->size() == 3); + CHECK((v.begin() + 3)->size() == 3); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } +} diff --git a/tiledb/common/test/unit_iterator_facade.cc b/tiledb/common/test/unit_iterator_facade.cc index 8eaaf72ca3f0..603d1304bb69 100644 --- a/tiledb/common/test/unit_iterator_facade.cc +++ b/tiledb/common/test/unit_iterator_facade.cc @@ -1,4 +1,3 @@ - /** * @file unit_iterator_facade.cc * @@ -364,6 +363,7 @@ enum class month : int { october, november, december, + end, }; /* @@ -410,50 +410,11 @@ class month_iterator : public iterator_facade { return _cur == other._cur; } }; - -/* - * A second month_iterator to test writing to the iterator. - */ - -class month_iterator_2 : public iterator_facade { - // Since we are faking an iterator over data, we can make the value mutable. - mutable month _cur = month::january; - - public: - month_iterator_2() = default; - explicit month_iterator_2(month m) - : _cur(m) { - } - - month& dereference() const { - return _cur; - } - - void advance(int n) { - _cur = month(int(_cur) + n); - } - int distance_to(month_iterator_2 o) const { - return int(o._cur) - int(_cur); - } - - bool operator==(month_iterator_2 o) const { - return _cur == o._cur; - } -}; - } // namespace TEMPLATE_TEST_CASE( - "iterator_facade: month_iterator", - "[iterator_facade]", - month_iterator, - month_iterator_2) { - CHECK(std::input_iterator); - if (std::is_same_v) { - CHECK(std::output_iterator); - } else { - CHECK(!std::output_iterator); - } + "iterator_facade: month_iterator", "[iterator_facade]", month_iterator) { + CHECK(!std::output_iterator); CHECK(std::forward_iterator); CHECK(std::bidirectional_iterator); @@ -483,6 +444,7 @@ TEMPLATE_TEST_CASE( CHECK(*it == month::november); ++it; CHECK(*it == month::december); + // We should be able to increment once more and get the sentinel value, // but this is not working for some reason. // @todo Fix this. @@ -490,47 +452,6 @@ TEMPLATE_TEST_CASE( // CHECK(it == month_iterator::sentinel_type()); } -TEST_CASE("iterator_facade: month_iterator_2 write") { - month_iterator_2 it; - *it = month::august; - CHECK(*it == month::august); - ++it; - CHECK(*it == month::september); - - *it = month::may; - CHECK(*it == month::may); - it++; - CHECK(*it == month::june); - CHECK(*it++ == month::june); - CHECK(*it == month::july); - - it.advance(3); - CHECK(*it == month::october); - *it = month::july; - it += 3; - CHECK(*it == month::october); - - auto a = *(it + 0); - CHECK(a == month::october); - auto b = *(it + 1); - CHECK(b == month::november); - - // These fail for some reason. - // @todo Fix this. - // auto c = it[0]; - // CHECK(c == month::october); - // auto d = it[1]; - // CHECK(d == month::november); - - CHECK(*(it + 0) == month::october); - CHECK(*(it + 1) == month::november); - - // These fail for some reason. - // @todo Fix this. - // CHECK(it[0] == month::october); - // CHECK(it[1] == month::november); -} - /* * Test arrow and arrow_proxy */ diff --git a/tiledb/common/var_length_view.h b/tiledb/common/var_length_view.h index adec0fd87309..089ee78abd7d 100644 --- a/tiledb/common/var_length_view.h +++ b/tiledb/common/var_length_view.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From a273d7a00bfa7ffddcacd2c5ddb3de423cde0f03 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 1 May 2024 11:51:37 +0200 Subject: [PATCH 307/456] Ensure correct cell order in stride and remove dead code in domain. (#4926) get_end_of_cell_slab is dead code and can be removed. stride should not be called for Hilbert layout. [sc-46462] --- TYPE: NO_HISTORY DESC: Ensure correct cell order in stride and remove dead code in domain. --- tiledb/sm/array_schema/domain.cc | 55 ++++---------------------------- tiledb/sm/array_schema/domain.h | 17 ---------- 2 files changed, 6 insertions(+), 66 deletions(-) diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 0aca71521d9c..18a16f7471f8 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -420,37 +420,6 @@ void Domain::get_tile_coords(const T* coords, T* tile_coords) const { } } -template -void Domain::get_end_of_cell_slab( - T* subarray, T* start, Layout layout, T* end) const { - if (layout == Layout::GLOBAL_ORDER || layout == cell_order_) { - auto dim_dom = (const T*)domain(dim_num_ - 1).data(); - auto tile_extent = *(const T*)this->tile_extent(dim_num_ - 1).data(); - - if (cell_order_ == Layout::ROW_MAJOR) { - for (unsigned d = 0; d < dim_num_; ++d) - end[d] = start[d]; - auto tile_idx = - Dimension::tile_idx(start[dim_num_ - 1], dim_dom[0], tile_extent); - end[dim_num_ - 1] = - Dimension::tile_coord_low(tile_idx + 1, dim_dom[0], tile_extent) - 1; - end[dim_num_ - 1] = - std::min(end[dim_num_ - 1], subarray[2 * (dim_num_ - 1) + 1]); - } else { - auto dim_dom = (const T*)domain(0).data(); - auto tile_extent = *(const T*)this->tile_extent(0).data(); - for (unsigned d = 0; d < dim_num_; ++d) - end[d] = start[d]; - auto tile_idx = Dimension::tile_idx(start[0], dim_dom[0], tile_extent); - end[0] = Dimension::tile_coord_high(tile_idx, dim_dom[0], tile_extent); - end[0] = std::min(end[0], subarray[1]); - } - } else { - for (unsigned d = 0; d < dim_num_; ++d) - end[d] = start[d]; - } -} - template void Domain::get_next_tile_coords(const T* domain, T* tile_coords) const { // Invoke the proper function based on the tile order @@ -574,8 +543,13 @@ Status Domain::set_null_tile_extents_to_range() { template uint64_t Domain::stride(Layout subarray_layout) const { if (dim_num_ == 1 || subarray_layout == Layout::GLOBAL_ORDER || - subarray_layout == cell_order_) + subarray_layout == cell_order_) { return UINT64_MAX; + } + + if (cell_order_ == Layout::HILBERT) { + throw std::logic_error("Stride cannot be computed for Hilbert cell order"); + } uint64_t ret = 1; if (cell_order_ == Layout::ROW_MAJOR) { @@ -1159,23 +1133,6 @@ template void Domain::get_tile_subarray( const double* tile_coords, double* tile_subarray) const; -template void Domain::get_end_of_cell_slab( - int8_t* subarray, int8_t* start, Layout layout, int8_t* end) const; -template void Domain::get_end_of_cell_slab( - uint8_t* subarray, uint8_t* start, Layout layout, uint8_t* end) const; -template void Domain::get_end_of_cell_slab( - int16_t* subarray, int16_t* start, Layout layout, int16_t* end) const; -template void Domain::get_end_of_cell_slab( - uint16_t* subarray, uint16_t* start, Layout layout, uint16_t* end) const; -template void Domain::get_end_of_cell_slab( - int* subarray, int* start, Layout layout, int* end) const; -template void Domain::get_end_of_cell_slab( - unsigned* subarray, unsigned* start, Layout layout, unsigned* end) const; -template void Domain::get_end_of_cell_slab( - int64_t* subarray, int64_t* start, Layout layout, int64_t* end) const; -template void Domain::get_end_of_cell_slab( - uint64_t* subarray, uint64_t* start, Layout layout, uint64_t* end) const; - template void Domain::get_tile_coords( const int8_t* coords, int8_t* tile_coords) const; template void Domain::get_tile_coords( diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 856a97918727..40ecc0e31aa9 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -294,23 +294,6 @@ class Domain { template void get_tile_coords(const T* coords, T* tile_coords) const; - /** - * Retrieves the end of a cell slab starting at the `start` input - * coordinates. The cell slab is determined based on the domain - * tile/cell order and the input query `layout`. Essentially a - * cell slab is a contiguous (in the global cell order) range of - * cells that follow the query layout. - * - * @tparam T The domain type. - * @param subarray The subarray in which the end of the cell slab must - * be contained. - * @param start The start coordinates. - * @param layout The query layout. - * @param end The cell slab end coordinates to be retrieved. - */ - template - void get_end_of_cell_slab(T* subarray, T* start, Layout layout, T* end) const; - /** * Retrieves the next tile coordinates along the array tile order within a * given tile domain. Applicable only to **dense** arrays. From 7f8da54c45655ecc2cc8521c4157987e6865686e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 1 May 2024 12:00:26 +0200 Subject: [PATCH 308/456] Fix nightlies after external sort merges. (#4927) [sc-46464] --- TYPE: NO_HISTORY DESC: Fix nightlies after external sort merges. --- tiledb/common/test/unit_iterator_facade.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tiledb/common/test/unit_iterator_facade.cc b/tiledb/common/test/unit_iterator_facade.cc index 603d1304bb69..c45a7e27e783 100644 --- a/tiledb/common/test/unit_iterator_facade.cc +++ b/tiledb/common/test/unit_iterator_facade.cc @@ -38,6 +38,7 @@ #include #include +#include #include #include "../iterator_facade.h" @@ -508,7 +509,7 @@ struct foo { void advance(int n) { index_ += n; } - int distance_to(iterator o) const { + size_t distance_to(iterator o) const { return o.index_ - index_; } }; @@ -844,7 +845,8 @@ void iterator_test() { CHECK(std::forward_iterator); CHECK(std::bidirectional_iterator); CHECK(std::random_access_iterator); - if (std::is_same_v< + if constexpr ( + std::is_same_v< std::remove_cvref_t, typename std::vector< typename std::iterator_traits::value_type>::iterator> || From 30b8ad0e6bfe59aaf1696b6a6bfc057cf636b8ae Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 1 May 2024 12:50:00 +0200 Subject: [PATCH 309/456] Fix wrong fallback cell order for Hilbert. (#4924) This fixes the fallback cell order for the Hilbert order on writes. When the domain is too large and that cells cannot be differentiated by the Hilbert number because we don't have enough bits per dimensions, we should fallback to row major cell ordering. A missing condition made us fallback to column order ordering. [sc-44758] --- TYPE: BUG DESC: Fix wrong fallback cell order for Hilbert. --- test/regression/CMakeLists.txt | 1 + test/regression/targets/sc-44758.cc | 135 ++++++++++++++++++++++++++++ tiledb/sm/array_schema/domain.cc | 2 +- 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 test/regression/targets/sc-44758.cc diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index a14dd3198b48..87d6377ed5b6 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -35,6 +35,7 @@ set(SOURCES targets/sc-15387.cc targets/sc-18836.cc targets/sc-23712.cc + targets/sc-44758.cc ) if (TILEDB_SERIALIZATION) diff --git a/test/regression/targets/sc-44758.cc b/test/regression/targets/sc-44758.cc new file mode 100644 index 000000000000..e0fb4fda8c25 --- /dev/null +++ b/test/regression/targets/sc-44758.cc @@ -0,0 +1,135 @@ +#include +#include +#include + +#include +#include + +#include + +using namespace tiledb; + +static void create_array(const std::string& array_uri); +static void write_array(const std::string& array_uri); +static void read_array(const std::string& array_uri); + +TEST_CASE( + "Properly sort data in Hilbert order when all cells are in the same bucket", + "[hilbert][bug][sc44758]") { + std::string array_uri = "test_hilbert_order"; + + // Test setup + create_array(array_uri); + write_array(array_uri); + read_array(array_uri); +} + +void create_array(const std::string& array_uri) { + Context ctx; + + auto obj = Object::object(ctx, array_uri); + if (obj.type() != Object::Type::Invalid) { + Object::remove(ctx, array_uri); + } + + auto X = Dimension::create( + ctx, + "x", + {{std::numeric_limits::lowest(), + std::numeric_limits::max()}}); + auto Y = Dimension::create( + ctx, + "y", + {{std::numeric_limits::lowest(), + std::numeric_limits::max()}}); + + Domain dom(ctx); + dom.add_dimension(X).add_dimension(Y); + + auto attr = Attribute::create(ctx, "a"); + + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_cell_order(TILEDB_HILBERT).set_domain(dom).add_attribute(attr); + + Array::create(array_uri, schema); +} + +void write_fragment( + Context& ctx, + Array& array, + std::vector x, + std::vector y, + std::vector a) { + Query query(ctx, array, TILEDB_WRITE); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("x", x) + .set_data_buffer("y", y) + .set_data_buffer("a", a); + REQUIRE(query.submit() == Query::Status::COMPLETE); +} + +void write_array(const std::string& array_uri) { + Context ctx; + Array array(ctx, array_uri, TILEDB_WRITE); + write_fragment( + ctx, + array, + {50913.5209f, 46300.4576f, 53750.3951f, 47779.8514f, 45815.4787f, + 45738.904f, 47445.7143f, 51352.0412f, 49088.727f, 52722.6237f, + 48501.783f, 53915.4312f, 50512.0801f, 45781.8652f, 53743.3637f, + 51288.4185f, 54457.4034f, 52333.0674f, 50988.1421f, 49246.9677f, + 53489.8377f, 49678.9367f, 50262.7812f, 45269.6639f, 54301.9674f}, + {6119.8819f, 2227.1279f, 4709.1357f, -6009.2908f, -3196.8194f, + 3999.3447f, -956.7883f, -9022.1859f, 7735.0127f, 2641.4245f, + -3325.7246f, -4835.4291f, 1449.9719f, -5958.2026f, 7479.1415f, + -4966.7886f, 8656.5012f, -690.8002f, 1651.4824f, -9181.8585f, + -1045.1637f, -8038.3517f, -7083.2645f, -7555.8585f, -3279.0184f}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}); + write_fragment( + ctx, + array, + {48932.8545f, 53999.9728f, 52448.9716f, 53026.5806f, 53609.8738f, + 49870.8329f, 53261.7657f, 54868.0211f, 50919.4791f, 51548.2142f, + 46907.8445f, 45835.1908f, 53411.073f, 52597.0232f, 47379.0257f, + 50703.926f, 47457.7695f, 54561.2923f, 49672.1336f, 48719.4054f, + 51188.1191f, 52083.7624f, 51569.5062f, 52931.5174f, 51622.6334f}, + {-2084.0598f, 780.3959f, -5696.0102f, 7110.3894f, 2958.4756f, + -8536.3301f, -2389.5892f, 5234.3587f, 321.5067f, 7850.7334f, + -265.8565f, 9017.0814f, -737.5592f, 1569.3621f, 4444.4227f, + -4509.9735f, -7676.8195f, -3205.2129f, -370.9372f, 5879.6844f, + 4343.399f, -5246.6839f, 9784.3999f, -7532.3645f, -7613.6955f}, + {25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, + 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49}); + write_fragment( + ctx, + array, + {51088.8635f, 50685.8091f, 54907.3208f, 53226.0392f, 49276.2669f, + 48473.3678f, 46088.6933f, 49581.7425f, 45380.7934f, 47440.2517f, + 48541.5523f, 46043.6958f, 45821.4628f, 54135.571f, 46101.602f, + 46876.8079f, 47082.2505f, 46077.7971f, 48246.9454f, 50715.8986f, + 46061.9485f, 54009.0435f, 46262.2024f, 46478.2223f, 51952.6307f}, + {9111.9753f, -8600.7575f, -9750.4502f, -1009.7165f, -2659.2155f, + 8411.8389f, 1178.1284f, -4547.992f, 2341.4306f, 7600.4032f, + -4077.5538f, 5656.9615f, 35.4158f, -9610.731f, -8035.895f, + 2742.678f, 6426.1031f, 9734.5399f, -3222.952f, -4063.2662f, + -6085.3865f, 2549.7113f, 1882.7361f, 7581.7167f, -5296.0846f}, + {50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74}); + array.close(); +} + +void read_array(const std::string& array_uri) { + Context ctx; + Array array(ctx, array_uri, TILEDB_READ); + + std::vector x(10000000); + std::vector y(10000000); + std::vector a(10000000); + Query query(ctx, array, TILEDB_READ); + query.set_layout(TILEDB_UNORDERED) + .set_data_buffer("x", x) + .set_data_buffer("y", y) + .set_data_buffer("a", a); + REQUIRE(query.submit() == Query::Status::COMPLETE); +} diff --git a/tiledb/sm/array_schema/domain.cc b/tiledb/sm/array_schema/domain.cc index 18a16f7471f8..f37aa8336702 100644 --- a/tiledb/sm/array_schema/domain.cc +++ b/tiledb/sm/array_schema/domain.cc @@ -272,7 +272,7 @@ int Domain::cell_order_cmp_2(const void* coord_a, const void* coord_b) { int Domain::cell_order_cmp( const type::DomainDataRef& left, const type::DomainDataRef& right) const { - if (cell_order_ == Layout::ROW_MAJOR) { + if (cell_order_ == Layout::ROW_MAJOR || cell_order_ == Layout::HILBERT) { for (unsigned d = 0; d < dim_num_; ++d) { auto res = cell_order_cmp_func_[d]( dimension_ptr(d), From 04f1d838a69da3ff290ec80d5bc2830f9424c5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Wed, 1 May 2024 16:45:36 +0200 Subject: [PATCH 310/456] Add faketime tests (#4883) This PR introduces timing tests that run with mocked time. During the test all calls to acquire timestamp, either by using posix `time()` or modern C++ `std::chrono::clock::now()` are mocked and should return the same timestamp. This way we should be able to test sub-millisecond writes/reads. This PR adds: 1. libfaketime port 2. timing tests target 3. modified timing tests command with proper environment variables These tests fail (as expected) on TileDB version 2.20.0: https://github.com/dudoslav/TileDB/actions/runs/8535232007/job/23381209269#step:14:97 --- TYPE: NO_HISTORY DESC: Add faketime tests --- .github/workflows/ci-linux_mac.yml | 2 ++ ports/README.md | 15 ++++++------ ports/libfaketime/CMakeLists.txt | 38 ++++++++++++++++++++++++++++++ ports/libfaketime/portfile.cmake | 22 +++++++++++++++++ ports/libfaketime/vcpkg.json | 14 +++++++++++ test/CMakeLists.txt | 16 +++++++++++++ vcpkg.json | 6 ++++- 7 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 ports/libfaketime/CMakeLists.txt create mode 100644 ports/libfaketime/portfile.cmake create mode 100644 ports/libfaketime/vcpkg.json diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 0b9dd95d6096..80b7aaf06d3c 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -192,6 +192,8 @@ jobs: ./tiledb/test/ci/test_assert -d yes ./tiledb/test/tiledb_unit -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ./tiledb/tiledb/sm/filesystem/test/unit_vfs -d yes | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' + + ctest -R tiledb_timing_unit | awk '/1: ::set-output/{sub(/.*1: /, ""); print; next} 1' ################################################### # Stop helper processes, if applicable diff --git a/ports/README.md b/ports/README.md index a03b41d7d668..dd79a1d7f362 100644 --- a/ports/README.md +++ b/ports/README.md @@ -28,10 +28,11 @@ After copying the port, add an entry to the table below. You should also contrib ## List of port overlays -|Port|Reason| -|----|------| -|`libmagic`|Updating to the upstream port deferred due to failures.| -|`openssl`|Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024.| -|`pcre2`|To be removed alongside libmagic.| -|`azure-storage-common-cpp`|Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221).| -|`vcpkg-cmake-config`|Patching to fix build issues with CMake 3.29.1. (https://github.com/microsoft/vcpkg/pull/38017)| +| Port | Reason | +|----------------------------|---------------------------------------------------------------------------------------------------------| +| `libmagic` | Updating to the upstream port deferred due to failures. | +| `openssl` | Pinning to OpenSSL 1.1 until we can move to 3.0 in January 2024. | +| `pcre2` | To be removed alongside libmagic. | +| `azure-storage-common-cpp` | Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221). | +| `libfaketime` | Port does not yet exist upstream | +| `vcpkg-cmake-config` | Patching to fix build issues with CMake 3.29.1. (https://github.com/microsoft/vcpkg/pull/38017) | diff --git a/ports/libfaketime/CMakeLists.txt b/ports/libfaketime/CMakeLists.txt new file mode 100644 index 000000000000..08e01c4f6ebd --- /dev/null +++ b/ports/libfaketime/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.12) + +project(faketime VERSION 0.9.10) + +add_library( + ${PROJECT_NAME} + src/libfaketime.c +) + +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries( + ${PROJECT_NAME} + rt + ) +endif() + +target_include_directories(${PROJECT_NAME} + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/src +) + +file(GLOB_RECURSE PUBLIC_HEADERS "src/*.h") + +target_sources( + ${PROJECT_NAME} + PUBLIC + FILE_SET HEADERS + BASE_DIRS src + FILES ${PUBLIC_HEADERS} +) + +install(TARGETS ${PROJECT_NAME} + FILE_SET HEADERS + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) \ No newline at end of file diff --git a/ports/libfaketime/portfile.cmake b/ports/libfaketime/portfile.cmake new file mode 100644 index 000000000000..17b545488736 --- /dev/null +++ b/ports/libfaketime/portfile.cmake @@ -0,0 +1,22 @@ +vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO wolfcw/libfaketime + REF v${VERSION} + SHA512 07c431bee21e31343b680d1322dd529ea276e3cc4dbec61646c12bf5d0263163faf6186efeb36b199e24b655578a493c43e3b7a7acf8eba8b9ff84a1e94d618b +) + +file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION "${SOURCE_PATH}") + +vcpkg_cmake_configure( + SOURCE_PATH ${SOURCE_PATH} +) + +vcpkg_cmake_install() +vcpkg_copy_pdbs() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") + +# Handle copyright and usage +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING") diff --git a/ports/libfaketime/vcpkg.json b/ports/libfaketime/vcpkg.json new file mode 100644 index 000000000000..ee5c0e8a24f5 --- /dev/null +++ b/ports/libfaketime/vcpkg.json @@ -0,0 +1,14 @@ +{ + "name": "libfaketime", + "version": "0.9.10", + "license": "GPL-2.0-only", + "description": "libfaketime modifies the system time for a single application", + "supports": "!windows", + "homepage": "https://github.com/wolfcw/libfaketime", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + } + ] +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 17aafc177acf..455255d78b34 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -362,6 +362,22 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU ) endif() +# Only produce timing tests for UNIX based systems (faketime constraint) +find_library( + LIBFAKETIME + NAMES faketime +) + +if (LIBFAKETIME) + add_test( + NAME "tiledb_timing_unit" + COMMAND tiledb_unit --durations=yes [sub-millisecond] + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + + set_property(TEST tiledb_timing_unit PROPERTY ENVIRONMENT_MODIFICATION "FAKETIME=set:2020-12-24 20:30:00;LD_PRELOAD=set:${LIBFAKETIME}") +endif() + add_test( NAME "tiledb_unit" COMMAND tiledb_unit --durations=yes diff --git a/vcpkg.json b/vcpkg.json index 19ed762ab250..21f28a1eacd1 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -66,7 +66,11 @@ "tests": { "description": "Build tiledb tests", "dependencies": [ - "catch2" + "catch2", + { + "name": "libfaketime", + "platform": "!windows" + } ] }, "tools": { From a6bdf20e5065c64fdfe08225beb5a22d00f53ce4 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Wed, 1 May 2024 10:17:37 -0600 Subject: [PATCH 311/456] C API support - to_string_view (#4915) Add function `tiledb::api::to_string_view()`. This function validates a candidate C-style string and converts it to a `std::string_view`. It's specific to the C API, with a template argument that allows an error message to identify an erroneous argument. Converted C API functions `tiledb_query_get_est_result_size*` to use this function. Changed relevant functions in `class Query` and `class Subarray` to use `string_view` and not a raw `char *`. This is a followup to #4891, where this work was out of scope. Had the conversion function already existed before that PR, it would have used there. --- TYPE: NO_HISTORY DESC: C API support - to_string_view --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/api/c_api_support/CMakeLists.txt | 1 + tiledb/api/c_api_support/c_api_support.h | 1 + .../c_api_support/cpp_string/CMakeLists.txt | 40 ++++ .../c_api_support/cpp_string/cpp_string.cc | 34 ++++ .../api/c_api_support/cpp_string/cpp_string.h | 156 ++++++++++++++ .../cpp_string/test/CMakeLists.txt | 32 +++ .../compile_capi_support_cpp_string_main.cc | 41 ++++ .../cpp_string/test/unit_cpp_string.cc | 191 ++++++++++++++++++ tiledb/sm/c_api/tiledb.cc | 26 +-- tiledb/sm/query/query.cc | 83 ++++---- tiledb/sm/query/query.h | 30 +-- tiledb/sm/subarray/subarray.cc | 45 +++-- tiledb/sm/subarray/subarray.h | 4 +- 13 files changed, 597 insertions(+), 87 deletions(-) create mode 100644 tiledb/api/c_api_support/cpp_string/CMakeLists.txt create mode 100644 tiledb/api/c_api_support/cpp_string/cpp_string.cc create mode 100644 tiledb/api/c_api_support/cpp_string/cpp_string.h create mode 100644 tiledb/api/c_api_support/cpp_string/test/CMakeLists.txt create mode 100644 tiledb/api/c_api_support/cpp_string/test/compile_capi_support_cpp_string_main.cc create mode 100644 tiledb/api/c_api_support/cpp_string/test/unit_cpp_string.cc diff --git a/tiledb/api/c_api_support/CMakeLists.txt b/tiledb/api/c_api_support/CMakeLists.txt index 0c3e215767e7..525c821a8379 100644 --- a/tiledb/api/c_api_support/CMakeLists.txt +++ b/tiledb/api/c_api_support/CMakeLists.txt @@ -26,5 +26,6 @@ include(common) +add_subdirectory(cpp_string) add_subdirectory(exception_wrapper) add_subdirectory(handle) diff --git a/tiledb/api/c_api_support/c_api_support.h b/tiledb/api/c_api_support/c_api_support.h index 5113e9ec4e50..a576c32b468f 100644 --- a/tiledb/api/c_api_support/c_api_support.h +++ b/tiledb/api/c_api_support/c_api_support.h @@ -37,6 +37,7 @@ #define TILEDB_CAPI_SUPPORT_H #include "argument_validation.h" +#include "tiledb/api/c_api_support/cpp_string/cpp_string.h" #include "tiledb/api/c_api_support/exception_wrapper/capi_definition.h" #include "tiledb/api/c_api_support/exception_wrapper/exception_wrapper.h" #if __has_include("capi_function_override.h") diff --git a/tiledb/api/c_api_support/cpp_string/CMakeLists.txt b/tiledb/api/c_api_support/cpp_string/CMakeLists.txt new file mode 100644 index 000000000000..ce93060c81d4 --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/CMakeLists.txt @@ -0,0 +1,40 @@ +# +# tiledb/api/c_api_support/cpp_string/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + cpp_string.cc +) +gather_sources(${SOURCES}) + +commence(object_library capi_support_cpp_string) + this_target_sources(${SOURCES}) + this_target_object_libraries(baseline) +conclude(object_library) + +add_test_subdirectory() diff --git a/tiledb/api/c_api_support/cpp_string/cpp_string.cc b/tiledb/api/c_api_support/cpp_string/cpp_string.cc new file mode 100644 index 000000000000..f168efdb7c41 --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/cpp_string.cc @@ -0,0 +1,34 @@ +/* + * @file tiledb/api/c_api_support/cpp_string/cpp_string.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION This is a stub file for future use. + * + */ + +#include "cpp_string.h" + +namespace tiledb::api {} diff --git a/tiledb/api/c_api_support/cpp_string/cpp_string.h b/tiledb/api/c_api_support/cpp_string/cpp_string.h new file mode 100644 index 000000000000..2da0226e26dd --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/cpp_string.h @@ -0,0 +1,156 @@ +/* + * @file tiledb/api/c_api_support/cpp_string/cpp_string.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#ifndef TILEDB_CAPI_SUPPORT_CPP_STRING_H +#define TILEDB_CAPI_SUPPORT_CPP_STRING_H + +#include // std::memchr +#include // std::string_view +#include "../argument_validation.h" // CAPIException + +namespace tiledb::api { + +/** + * The default maximum length of C strings accepted by the C API + * + * This value is used as a template argument, should there ever be a need to + * use something other than the default length. A large class of uses are for + * the names of query fields; at present these do not have a defined maximum + * admissible length, but ought to have. This default value is large enough for + * most purposes. + */ +constexpr size_t default_max_c_string_length = 65534; + +/** + * Validate that a pointer is to a C-style string and convert it to a string + * view. + * + * This function is a defense against malformed inputs coming in through the C + * API. This function validates the following properties of the candidate input: + * - The pointer is not null. (It points to something.) + * - The string is null-terminated. (It has a length.) + * - The string length is admissible. (It's short enough.) + * The admissible string length might vary depending on how it's being used. + * + * This function does not throw. It returns an empty string if there's a + * validation error. + * + * @section Maturity + * + * There's one failure that this function cannot withstand at present. If the + * pointer is to a non-terminated string at the end of a page, there will be an + * access violation fault that necessarily happens during the search for a + * terminating NUL character. It would be possible to recover from such an error + * by catching the resulting signal and causing this function to simply return + * false. This behavior, however, is not nearly as simple as it might appear. + * + * The proper requirement for such behavior is that it be able to deal with + * multiple simultaneous invocations of the function. As this function is + * defensive, it should be able to handle many simultaneous calls correctly, as + * an intentional attacker might generate them specifically with an eye to + * cause the defense to fault. + * + * @param candidate_c_string A pointer to be validated as a null-terminated + * character string + * @return A string view corresponding to the candidate if it is valid. A null + * string view (`.data() == nullptr`) otherwise. + */ +template +std::string_view to_string_view_internal( + const char* candidate_c_string) noexcept { + if (candidate_c_string == nullptr) { + return {}; + } + /* + * Note that we search for N+1 characters. In particular, zero-length C-style + * strings require 1 character of storage for the terminating NUL. + */ + auto p{ + static_cast(std::memchr(candidate_c_string, '\0', N + 1))}; + if (p == nullptr) { + return {}; + } + return {candidate_c_string, static_cast(p - candidate_c_string)}; +} + +/** + * A structural type, holding a string, that can be used as a template argument. + * + * @tparam N length of the string + */ +template +struct StringConstant { + /** + * The string itself and a terminating null character + */ + char data_[N]; + + /** + * Ordinary constructor whose argument is expected to be a string literal. + * + * @param string A null terminated string. + */ + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr StringConstant(const char (&string)[N]) { + std::copy_n(string, N, data_); + } + + // NOLINTNEXTLINE(google-explicit-constructor) + consteval operator std::string_view() const { + return {data_, N - 1}; + } +}; + +/** + * String conversion function that throws errors on validation. + * + * @tparam description The description of the string for error messages + * @param candidate_c_string Possibly a pointer to a C-style string + * @return + */ +template +inline std::string_view to_string_view(const char* candidate_c_string) { + if (candidate_c_string == nullptr) { + std::string_view d{description}; + throw CAPIException("Pointer to " + std::string(d) + " may not be NULL"); + } + auto sv{to_string_view_internal(candidate_c_string)}; + if (sv.data() == nullptr) { + std::string_view d{description}; + throw CAPIException( + "Invalid " + std::string(d) + "; no terminating NUL character"); + } + return sv; +} + +} // namespace tiledb::api + +#endif // TILEDB_CAPI_SUPPORT_CPP_STRING_H diff --git a/tiledb/api/c_api_support/cpp_string/test/CMakeLists.txt b/tiledb/api/c_api_support/cpp_string/test/CMakeLists.txt new file mode 100644 index 000000000000..bcc57584ea6d --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/api/c_api_support/cpp_string/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_support_cpp_string) + this_target_sources(unit_cpp_string.cc) + this_target_object_libraries(capi_support_cpp_string) +conclude(unit_test) diff --git a/tiledb/api/c_api_support/cpp_string/test/compile_capi_support_cpp_string_main.cc b/tiledb/api/c_api_support/cpp_string/test/compile_capi_support_cpp_string_main.cc new file mode 100644 index 000000000000..ab10f39881e6 --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/test/compile_capi_support_cpp_string_main.cc @@ -0,0 +1,41 @@ +/* + * @file + * tiledb/api/c_api_support/cpp_string/test/compile_capi_support_cpp_string_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#include "../cpp_string.h" + +int main() { + auto sv{tiledb::api::to_string_view<"a">("")}; + if (sv.data() == nullptr) { + return 1; + } + return 0; +} diff --git a/tiledb/api/c_api_support/cpp_string/test/unit_cpp_string.cc b/tiledb/api/c_api_support/cpp_string/test/unit_cpp_string.cc new file mode 100644 index 000000000000..caa1daadeea8 --- /dev/null +++ b/tiledb/api/c_api_support/cpp_string/test/unit_cpp_string.cc @@ -0,0 +1,191 @@ +/* + * @file tiledb/api/c_api_support/cpp_string/test/unit_cpp_string.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#define CATCH_CONFIG_MAIN +#include + +#include "../cpp_string.h" + +// ------------------------------------------------------- +// to_string_view_internal +// ------------------------------------------------------- +/* + * Proxy to move the function under test into the global namespace + */ +template +std::string_view to_cpp_string_view_internal(const char* p) { + /* + * We launder the pointer to avoid overzealous optimizers from noticing that + * we're passing short strings to a function with large bounds for what it + * might access. + */ + const char* q{std::launder(p)}; + return tiledb::api::to_string_view_internal(q); +} + +TEST_CASE("C API Support to_string_view_internal - max default, null input") { + auto sv{to_cpp_string_view_internal(nullptr)}; + CHECK(sv.data() == nullptr); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max default, empty string") { + const char empty_string[]{""}; + auto sv{to_cpp_string_view_internal(empty_string)}; + REQUIRE(sv.data() != nullptr); + CHECK(sv.data() == empty_string); + CHECK(sv.length() == 0); +} + +TEST_CASE( + "C API Support to_string_view_internal - max default, length 1 string") { + const char one_string[]{"X"}; + auto sv{to_cpp_string_view_internal(one_string)}; + REQUIRE(sv.data() != nullptr); + CHECK(sv.data() == one_string); + CHECK(sv.length() == 1); +} + +TEST_CASE("C API Support to_string_view_internal - max 0, null input") { + auto sv{to_cpp_string_view_internal<0>(nullptr)}; + CHECK(sv.data() == nullptr); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 0, empty string") { + const char empty_string[]{""}; + static_assert(sizeof(empty_string) == 1); + auto sv{to_cpp_string_view_internal<0>(empty_string)}; + REQUIRE(sv.data() != nullptr); + CHECK(sv.data() == empty_string); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 0, length 1 string") { + /* + * The first character is not NUL, so the string is considered unterminated. + * The only valid input when template argument is 0 are zero-length strings. + */ + const char one_string[]{"X"}; + auto sv{to_cpp_string_view_internal<0>(one_string)}; + CHECK(sv.data() == nullptr); + CHECK(sv.data() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, null input") { + auto sv{to_cpp_string_view_internal<2>(nullptr)}; + CHECK(sv.data() == nullptr); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, empty string") { + const char empty_string[]{""}; + static_assert(sizeof(empty_string) == 1); + auto sv{to_cpp_string_view_internal<2>(empty_string)}; + REQUIRE(sv.data() != nullptr); + CHECK(sv.data() == empty_string); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, length 1 string") { + const char one_string[]{"1"}; + auto sv{to_cpp_string_view_internal<2>(one_string)}; + CHECK(sv.data() == one_string); + CHECK(sv.length() == 1); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, length 2 string") { + // Not terminated within bounds + const char two_string[]{"12"}; + auto sv{to_cpp_string_view_internal<2>(two_string)}; + CHECK(sv.data() == two_string); + CHECK(sv.length() == 2); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, length 3 string") { + // Not terminated within bounds + const char three_string[]{"123"}; + auto sv{to_cpp_string_view_internal<0>(three_string)}; + CHECK(sv.data() == nullptr); + CHECK(sv.data() == 0); +} + +TEST_CASE("C API Support to_string_view_internal - max 2, length 4 string") { + // Not terminated within bounds + const char four_string[]{"1234"}; + auto sv{to_cpp_string_view_internal<0>(four_string)}; + CHECK(sv.data() == nullptr); + CHECK(sv.data() == 0); +} + +// ------------------------------------------------------- +// to_string_view +// ------------------------------------------------------- + +template +inline std::string_view to_string_view(const char* p) { + // See above for rationale for `launder` + const char* q{std::launder(p)}; + return tiledb::api::to_string_view(q); +} + +TEST_CASE("C API Support to_string_view - null input") { + CHECK_THROWS(to_string_view<"xyzzy">(nullptr)); + CHECK_THROWS_WITH( + to_string_view<"xyzzy">(nullptr), + Catch::Matchers::ContainsSubstring("xyzzy")); +} + +TEST_CASE("C API Support to_string_view - empty string") { + const char empty_string[]{""}; + auto sv{to_string_view<"a">(empty_string)}; + CHECK(sv.data() == empty_string); + CHECK(sv.length() == 0); +} + +TEST_CASE("C API Support to_string_view - length 3 string") { + const char three_string[]{"123"}; + auto sv{to_string_view<"a">(three_string)}; + CHECK(sv.data() == three_string); + CHECK(sv.length() == 3); +} + +TEST_CASE("C API Support to_string_view - invalid candidate") { + /* + * A block of memory initialized without NUL characters, large enough to + * trigger an overflow. + */ + std::vector bad_data(tiledb::api::default_max_c_string_length + 1, 'X'); + auto p{bad_data.data()}; + CHECK_THROWS(to_string_view<"xyzzy">(p)); + CHECK_THROWS_WITH( + to_string_view<"xyzzy">(p), Catch::Matchers::ContainsSubstring("xyzzy")); +} diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index e626d1d8871b..ed25afe2370a 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1544,13 +1544,11 @@ int32_t tiledb_query_get_est_result_size( if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; } - if (name == nullptr) { - throw CAPIStatusException("Pointer to field name may not be NULL"); - } + auto field_name{to_string_view<"field name">(name)}; if (size == nullptr) { throw CAPIStatusException("Pointer to size may not be NULL"); } - auto est_size{query->query_->get_est_result_size_fixed_nonnull(name)}; + auto est_size{query->query_->get_est_result_size_fixed_nonnull(field_name)}; *size = est_size.fixed_; return TILEDB_OK; } @@ -1564,9 +1562,7 @@ int32_t tiledb_query_get_est_result_size_var( if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; } - if (name == nullptr) { - throw CAPIStatusException("Pointer to field name may not be NULL"); - } + auto field_name{to_string_view<"field name">(name)}; if (size_off == nullptr) { throw CAPIStatusException("Pointer to offset size may not be NULL"); } @@ -1574,7 +1570,8 @@ int32_t tiledb_query_get_est_result_size_var( throw CAPIStatusException("Pointer to value size may not be NULL"); } - auto est_size{query->query_->get_est_result_size_variable_nonnull(name)}; + auto est_size{ + query->query_->get_est_result_size_variable_nonnull(field_name)}; *size_off = est_size.fixed_; *size_val = est_size.variable_; @@ -1590,9 +1587,7 @@ int32_t tiledb_query_get_est_result_size_nullable( if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; } - if (name == nullptr) { - throw CAPIStatusException("Pointer to field name may not be NULL"); - } + auto field_name{to_string_view<"field name">(name)}; if (size_val == nullptr) { throw CAPIStatusException("Pointer to value size may not be NULL"); } @@ -1600,7 +1595,7 @@ int32_t tiledb_query_get_est_result_size_nullable( throw CAPIStatusException("Pointer to validity size may not be NULL"); } - auto est_size{query->query_->get_est_result_size_fixed_nullable(name)}; + auto est_size{query->query_->get_est_result_size_fixed_nullable(field_name)}; *size_val = est_size.fixed_; *size_validity = est_size.validity_; return TILEDB_OK; @@ -1616,9 +1611,7 @@ int32_t tiledb_query_get_est_result_size_var_nullable( if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; } - if (name == nullptr) { - throw CAPIStatusException("Pointer to field name may not be NULL"); - } + auto field_name{to_string_view<"field name">(name)}; if (size_off == nullptr) { throw CAPIStatusException("Pointer to offset size may not be NULL"); } @@ -1628,7 +1621,8 @@ int32_t tiledb_query_get_est_result_size_var_nullable( if (size_validity == nullptr) { throw CAPIStatusException("Pointer to validity size may not be NULL"); } - auto est_size{query->query_->get_est_result_size_variable_nullable(name)}; + auto est_size{ + query->query_->get_est_result_size_variable_nullable(field_name)}; *size_off = est_size.fixed_; *size_val = est_size.variable_; *size_validity = est_size.validity_; diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 4d88f1a39905..2374a0a2b6b5 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -152,47 +152,52 @@ Query::~Query() { /* ****************************** */ void Query::field_require_array_fixed( - const std::string_view origin, const char* name) { - if (!array_schema_->is_field(name)) { + const std::string_view origin, std::string_view field_name) { + if (!array_schema_->is_field(field_name.data())) { throw QueryException( - std::string{origin} + ": '" + name + "' is not an array field"); + std::string{origin} + ": '" + std::string{field_name} + + "' is not an array field"); } - if (array_schema_->var_size(name)) { + if (array_schema_->var_size(field_name.data())) { throw QueryException( - std::string{origin} + ": '" + name + "' is not fixed-sized"); + std::string{origin} + ": '" + std::string{field_name} + + "' is not fixed-sized"); } } void Query::field_require_array_variable( - const std::string_view origin, const char* name) { - if (!array_schema_->is_field(name)) { + const std::string_view origin, std::string_view field_name) { + if (!array_schema_->is_field(field_name.data())) { throw QueryException( - std::string{origin} + ": '" + name + "' is not an array field"); + std::string{origin} + ": '" + std::string{field_name} + + "' is not an array field"); } - if (!array_schema_->var_size(name)) { + if (!array_schema_->var_size(field_name.data())) { throw QueryException( - std::string{origin} + ": '" + name + "' is not variable-sized"); + std::string{origin} + ": '" + std::string{field_name} + + "' is not variable-sized"); } } void Query::field_require_array_nullable( - const std::string_view origin, const char* name) { - if (!array_schema_->attribute(name)) { + const std::string_view origin, std::string_view field_name) { + if (!array_schema_->attribute(field_name.data())) { throw QueryException( - std::string{origin} + ": '" + name + + std::string{origin} + ": '" + std::string{field_name} + "' is not the name of an attribute"); } - if (!array_schema_->is_nullable(name)) { + if (!array_schema_->is_nullable(field_name.data())) { throw QueryException( - std::string{origin} + ": attribute '" + name + "' is not nullable"); + std::string{origin} + ": attribute '" + std::string{field_name} + + "' is not nullable"); } } void Query::field_require_array_nonnull( - const std::string_view origin, const char* name) { - if (array_schema_->is_nullable(name)) { + const std::string_view origin, std::string_view field_name) { + if (array_schema_->is_nullable(field_name.data())) { throw QueryException( - std::string(origin) + ": field '" + name + + std::string(origin) + ": field '" + std::string(field_name) + "' is not a nonnull array field"); } } @@ -200,7 +205,7 @@ void Query::field_require_array_nonnull( constexpr std::string_view origin_est_result_size{ "query estimated result size"}; -FieldDataSize Query::internal_est_result_size(const char* name) { +FieldDataSize Query::internal_est_result_size(std::string_view field_name) { if (type_ != QueryType::READ) { throw QueryException( std::string{origin_est_result_size} + @@ -217,12 +222,13 @@ FieldDataSize Query::internal_est_result_size(const char* name) { rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } return subarray_.get_est_result_size( - name, &config_, storage_manager_->compute_tp()); + field_name, &config_, storage_manager_->compute_tp()); } -FieldDataSize Query::get_est_result_size_fixed_nonnull(const char* name) { - field_require_array_fixed(origin_est_result_size, name); - if (name == constants::coords) { +FieldDataSize Query::get_est_result_size_fixed_nonnull( + std::string_view field_name) { + field_require_array_fixed(origin_est_result_size, field_name); + if (field_name == constants::coords) { if (!array_schema_->domain().all_dims_same_type()) { throw QueryException( std::string{origin_est_result_size} + @@ -236,26 +242,29 @@ FieldDataSize Query::get_est_result_size_fixed_nonnull(const char* name) { "in arrays with domains with variable-sized dimensions"); } } - field_require_array_nonnull(origin_est_result_size, name); - return internal_est_result_size(name); + field_require_array_nonnull(origin_est_result_size, field_name); + return internal_est_result_size(field_name); } -FieldDataSize Query::get_est_result_size_variable_nonnull(const char* name) { - field_require_array_variable(origin_est_result_size, name); - field_require_array_nonnull(origin_est_result_size, name); - return internal_est_result_size(name); +FieldDataSize Query::get_est_result_size_variable_nonnull( + std::string_view field_name) { + field_require_array_variable(origin_est_result_size, field_name); + field_require_array_nonnull(origin_est_result_size, field_name); + return internal_est_result_size(field_name); } -FieldDataSize Query::get_est_result_size_fixed_nullable(const char* name) { - field_require_array_fixed(origin_est_result_size, name); - field_require_array_nullable(origin_est_result_size, name); - return internal_est_result_size(name); +FieldDataSize Query::get_est_result_size_fixed_nullable( + std::string_view field_name) { + field_require_array_fixed(origin_est_result_size, field_name); + field_require_array_nullable(origin_est_result_size, field_name); + return internal_est_result_size(field_name); } -FieldDataSize Query::get_est_result_size_variable_nullable(const char* name) { - field_require_array_variable(origin_est_result_size, name); - field_require_array_nullable(origin_est_result_size, name); - return internal_est_result_size(name); +FieldDataSize Query::get_est_result_size_variable_nullable( + std::string_view field_name) { + field_require_array_variable(origin_est_result_size, field_name); + field_require_array_nullable(origin_est_result_size, field_name); + return internal_est_result_size(field_name); } std::unordered_map diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 637aa019106e..b8782b0b53ac 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -179,10 +179,10 @@ class Query { * - The field is fixed-sized * * @param origin The name of the operation that this validation is a part of - * @param name The name of a field + * @param field_name The name of a field */ void field_require_array_fixed( - const std::string_view origin, const char* name); + const std::string_view origin, std::string_view field_name); /** * Require that a name be that of a variable-size field from the source array. @@ -192,10 +192,10 @@ class Query { * - The field is variable-sized * * @param origin The name of the operation that this validation is a part of - * @param name The name of a field + * @param field_name The name of a field */ void field_require_array_variable( - const std::string_view origin, const char* name); + const std::string_view origin, std::string_view field_name); /** * Require that a field be a nullable field from the source array. @@ -205,10 +205,10 @@ class Query { * - The attribute is nullable * * @param origin The name of the operation that this validation is a part of - * @param name The name of a field + * @param field_name The name of a field */ void field_require_array_nullable( - const std::string_view origin, const char* name); + const std::string_view origin, std::string_view field_name); /** * Require that a field be a nonnull field from the source array. @@ -218,44 +218,46 @@ class Query { * - The field is not nullable * * @param origin The name of the operation that this validation is a part of. - * @param name The name of a field + * @param field_name The name of a field */ void field_require_array_nonnull( - const std::string_view origin, const char* name); + const std::string_view origin, std::string_view field_name); /** * Gets the estimated result size (in bytes) for the input fixed-sized * attribute/dimension. */ - FieldDataSize get_est_result_size_fixed_nonnull(const char* name); + FieldDataSize get_est_result_size_fixed_nonnull(std::string_view field_name); /** * Gets the estimated result size (in bytes) for the input var-sized * attribute/dimension. */ - FieldDataSize get_est_result_size_variable_nonnull(const char* name); + FieldDataSize get_est_result_size_variable_nonnull( + std::string_view field_name); /** * Gets the estimated result size (in bytes) for the input fixed-sized, * nullable attribute. */ - FieldDataSize get_est_result_size_fixed_nullable(const char* name); + FieldDataSize get_est_result_size_fixed_nullable(std::string_view field_name); /** * Gets the estimated result size (in bytes) for the input var-sized, * nullable attribute. */ - FieldDataSize get_est_result_size_variable_nullable(const char* name); + FieldDataSize get_est_result_size_variable_nullable( + std::string_view field_name); private: /** * Common part of all `est_result_size_*` functions, called after argument * validation. * - * @param name The name of a field + * @param field_name The name of a field * @return estimated result size */ - FieldDataSize internal_est_result_size(const char* name); + FieldDataSize internal_est_result_size(std::string_view field_name); public: /** Retrieves the number of written fragments. */ diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 71386e063cc0..1c4f137c082a 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -1104,39 +1104,45 @@ Layout Subarray::layout() const { } FieldDataSize Subarray::get_est_result_size( - const char* name, const Config* config, ThreadPool* compute_tp) { - // Check attribute/dimension name - if (name == nullptr) { - throw SubarrayException( - "Cannot get estimated result size; field name cannot be null"); + std::string_view field_name, const Config* config, ThreadPool* compute_tp) { + /* + * This check throws a logic error because we expect the field name to have + * already been validated in the C API. + */ + if (field_name.data() == nullptr) { + throw std::logic_error( + "Cannot get estimated result size; field name is null"); } // Check if name is attribute or dimension const auto& array_schema = array_->array_schema_latest(); - const bool is_dim = array_schema.is_dim(name); - const bool is_attr = array_schema.is_attr(name); + const bool is_dim = array_schema.is_dim(field_name.data()); + const bool is_attr = array_schema.is_attr(field_name.data()); // Check if attribute/dimension exists - if (!ArraySchema::is_special_attribute(name) && !is_dim && !is_attr) { + if (!ArraySchema::is_special_attribute(field_name.data()) && !is_dim && + !is_attr) { throw SubarrayException( std::string("Cannot get estimated result size; ") + - "there is no field named '" + name + "'"); + "there is no field named '" + std::string(field_name) + "'"); } bool is_variable_sized{ - name != constants::coords && array_schema.var_size(name)}; - bool is_nullable{array_schema.is_nullable(name)}; + field_name != constants::coords && + array_schema.var_size(field_name.data())}; + bool is_nullable{array_schema.is_nullable(field_name.data())}; // Compute tile overlap for each fragment compute_est_result_size(config, compute_tp); FieldDataSize r{ - static_cast(std::ceil(est_result_size_[name].size_fixed_)), - is_variable_sized ? - static_cast(std::ceil(est_result_size_[name].size_var_)) : - 0, - is_nullable ? static_cast( - std::ceil(est_result_size_[name].size_validity_)) : + static_cast( + std::ceil(est_result_size_[field_name.data()].size_fixed_)), + is_variable_sized ? static_cast(std::ceil( + est_result_size_[field_name.data()].size_var_)) : + 0, + is_nullable ? static_cast(std::ceil( + est_result_size_[field_name.data()].size_validity_)) : 0}; /* * Special fix-ups may be necessary if data is empty or very short. @@ -1154,7 +1160,8 @@ FieldDataSize Subarray::get_est_result_size( r.fixed_ = off_cell_size; } // Ensure that there's space for at least one data value. - const auto val_cell_size = datatype_size(array_schema.type(name)); + const auto val_cell_size = + datatype_size(array_schema.type(field_name.data())); if (r.variable_ < val_cell_size) { r.variable_ = val_cell_size; } @@ -1170,7 +1177,7 @@ FieldDataSize Subarray::get_est_result_size( * If the fixed data is not empty, ensure it is large enough to contain at * least one cell. */ - const auto cell_size = array_schema.cell_size(name); + const auto cell_size = array_schema.cell_size(field_name.data()); if (0 < r.fixed_ && r.fixed_ < cell_size) { r.fixed_ = cell_size; if (is_nullable) { diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index bc11c1f50041..f8d8105f1204 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -946,7 +946,9 @@ class Subarray { * size is not relevant to the field type, then it's returned as zero. */ FieldDataSize get_est_result_size( - const char* name, const Config* config, ThreadPool* compute_tp); + std::string_view field_name, + const Config* config, + ThreadPool* compute_tp); /** * The maximum memory in bytes required for a field. From 98df10da7fbddd275c7fc31ff3b01c23524e5935 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 1 May 2024 19:37:56 +0200 Subject: [PATCH 312/456] Update dev history with content from 2.22, 2.18.5 and 2.21.2. (#4933) --- TYPE: NO_HISTORY DESC: Update dev history with content from 2.22, 2.18.5 and 2.21.2. --------- Co-authored-by: Theodore Tsirpanis --- HISTORY.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 74e9d26f957e..492993062f74 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,36 @@ +# TileDB v2.22.0 Release Notes + +## Deprecation announcements + +* Support for downloading dependencies with CMake `ExternalProject`s by specifying `-DTILEDB_VCPKG=OFF` will be removed in 2.23. Vcpkg will be downloaded and used to manage dependencies by default. See https://github.com/TileDB-Inc/TileDB/blob/dev/doc/dev/BUILD.md for help on how to disable automatically downloading vcpkg and acquire dependencies from the system. + +## Configuration changes + +* Add `vfs.gcs.service_account_key` config option that specifies a Google Cloud service account credential JSON string. [#4855](https://github.com/TileDB-Inc/TileDB/pull/4855) +* Add `vfs.gcs.workload_identity_configuration` config option that specifies a Google Cloud Workload Identity Federation credential JSON string. [#4855](https://github.com/TileDB-Inc/TileDB/pull/4855) + +## New features + +* Support Microsoft Entra ID authentication to Azure. [#4126](https://github.com/TileDB-Inc/TileDB/pull/4126) + +## Improvements + +* Allow to set buffers for geometry types with CPP API more easily. [#4826](https://github.com/TileDB-Inc/TileDB/pull/4826) + +## Defects removed + +* Throw error when an upload fails due to bad state. [#4815](https://github.com/TileDB-Inc/TileDB/pull/4815) +* Single-process sub-millisecond temporal disambiguation of random labels. [#4800](https://github.com/TileDB-Inc/TileDB/pull/4800) +* Expose VFSExperimental and relocate CallbackWrapperCPP. [#4820](https://github.com/TileDB-Inc/TileDB/pull/4820) +* Do not load group metadata when getting it from REST. [#4821](https://github.com/TileDB-Inc/TileDB/pull/4821) +* Fix crash getting file size on non existent blob on Azure. [#4836](https://github.com/TileDB-Inc/TileDB/pull/4836) + +# TileDB v2.21.2 Release Notes + +## Defects removed + +* Fix crash getting file size on non existent blob on Azure. [#4836](https://github.com/TileDB-Inc/TileDB/pull/4836) + # TileDB v2.21.1 Release Notes ## Defects removed @@ -225,6 +258,17 @@ * Remove UnitTestConfig::array_encryption_key_length. [#4482](https://github.com/TileDB-Inc/TileDB/pull/4482) +# TileDB v2.18.5 Release Notes + +## Defects removed + +* Fix out of order consolidation. [#4597](https://github.com/TileDB-Inc/TileDB/pull/4597) +* Vac files should only be removed if paths removal was fully successful. [#4889](https://github.com/TileDB-Inc/TileDB/pull/4889) + +## Build System Changes + +* Fix linker errors when building with MSVC. [#4759](https://github.com/TileDB-Inc/TileDB/pull/4759) + # TileDB v2.18.4 Release Notes ## Defects removed From f7a528ad6a0ff5338c8ad0f0fb079a7a076563ee Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 1 May 2024 19:50:52 +0200 Subject: [PATCH 313/456] Bump dev version to 2.24.0. (#4934) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.24.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index c6f583939874..b39f626ca33d 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.23' +version = '2.24' # The full version, including alpha/beta/rc tags. -release = '2.23.0' +release = '2.24.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index 2d839be4e815..f5b57af5d7bf 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 23 +#define TILEDB_VERSION_MINOR 24 #define TILEDB_VERSION_PATCH 0 From 3a88d3cb2b97922cb1f942e5514df8802e50af3f Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Wed, 1 May 2024 11:00:33 -0700 Subject: [PATCH 314/456] Additional constructors for external sort classes. (#4928) This PR is stacked on #4925. It adds additional constructors for the two view classes for var length data. The previous implementations had constructors that just took input ranges, and constructed a var length view over the data specified by those ranges. However, in general, we may have a range that is not completely filled with data, and so may need to specify the data to be viewed in some other way. The complete set of constructors take * ranges for data and offsets (arrow format) - the size of the resulting view is one less than the size of the offset range * ranges plus sizes for data and offsets (arrow format) - the size of the resulting view is one less than the size given for the offset range * iterator pairs for data and offsets (arrow format) - the size of the resulting view is one less than the difference between end and begin of the offsets pair * iterator pairs for data and offsets, with sizes (arrow format) - the size of the resulting view is one less than the size given for the offset range * ranges for data and offsets (tilledb format) - the size of the resulting view is equal to the size of the offset range (takes an extra argument for last offset value) * ranges plus sizes for data and offsets (tiledb format) - the size of the resulting view is equal to the size given for the offset range (takes an extra argument for last offset value) * iterator pairs for data and offsets (tiledb format) - the size of the resulting view is equal to the difference between end and begin of the offsets pair (takes an extra argument for last offset value) * iterator pairs for data and offsets, with sizes (tiledb format) - the size of the resulting view is equal than the size given for the offset range (takes an extra argument for last offset value) [sc-44576] --- TYPE: NO_HISTORY DESC: Additional constructors for external sort classes. --- tiledb/common/alt_var_length_view.h | 191 ++++++++++++++++-- .../common/test/unit_alt_var_length_view.cc | 127 +++++++++++- tiledb/common/test/unit_var_length_view.cc | 53 ++++- tiledb/common/var_length_view.h | 105 ++++++++-- 4 files changed, 430 insertions(+), 46 deletions(-) diff --git a/tiledb/common/alt_var_length_view.h b/tiledb/common/alt_var_length_view.h index 1c4bab8a9660..44c2194e805d 100644 --- a/tiledb/common/alt_var_length_view.h +++ b/tiledb/common/alt_var_length_view.h @@ -34,10 +34,10 @@ * The difference between `alt_var_length_view` and `var_length_view` is that * `alt_var_length_view` maintains a materialized range of subranges, whereas * `var_length_view` creates subrange views on the fly as proxy objects. As a - * result - * * An `alt_var_length_view` does not need to refer to the offsets array after - * it is constructed - * * An `alt_var_length_view` can be sorted + * result: + * - An `alt_var_length_view` does not need to refer to the offsets array + * after it is constructed + * - An `alt_var_length_view` can be sorted * * * Usage example: @@ -69,12 +69,15 @@ * @tparam I Type of the index range, assumed to be a random access range. * * @todo R could be a view rather than a range. + * @todo Would using `std::ranges::view_interface` be better tha `view_base`? */ template < std::ranges::random_access_range R, std::ranges::random_access_range I> class alt_var_length_view : public std::ranges::view_base { - // Forward reference of the iterator over the range of variable length data + /** + * Forward reference of the iterator over the range of variable length data + */ template struct private_iterator; @@ -115,30 +118,167 @@ class alt_var_length_view : public std::ranges::view_base { /** Move assignment */ alt_var_length_view& operator=(alt_var_length_view&&) = default; - /** Primary constructor. All offsets are contained in the input (notably, the - * index to the end of the data range). */ + /** + * Constructor taking iterator pairs for the data and index ranges, arrow + * format + */ + alt_var_length_view( + std::ranges::iterator_t data_begin, + [[maybe_unused]] std::ranges::iterator_t data_end, + std::ranges::iterator_t begin_index, + std::ranges::iterator_t index_end) { + auto num_subranges = index_end - begin_index - 1; + + subranges_.reserve(num_subranges); + for (long i = 0; i < num_subranges; ++i) { + subranges_.emplace_back( + data_begin + begin_index[i], data_begin + begin_index[i + 1]); + } + } + + /** + * Constructor taking iterator pairs for the data and index ranges, tiledb + * format + */ + alt_var_length_view( + std::ranges::iterator_t data_begin, + [[maybe_unused]] std::ranges::iterator_t data_end, + std::ranges::iterator_t begin_index, + std::ranges::iterator_t index_end, + data_index_type missing_index) { + auto num_subranges = index_end - begin_index; + + subranges_.reserve(num_subranges); + for (long i = 0; i < num_subranges - 1; ++i) { + subranges_.emplace_back( + data_begin + begin_index[i], data_begin + begin_index[i + 1]); + } + subranges_.emplace_back( + data_begin + begin_index[num_subranges - 1], + data_begin + missing_index); + } + + /** + * Constructor taking iterator pairs for the data and index ranges, with + * sizes, arrow format + */ + alt_var_length_view( + std::ranges::iterator_t data_begin, + [[maybe_unused]] std::ranges::iterator_t data_end, + [[maybe_unused]] std::ranges::range_difference_t n_data, + std::ranges::iterator_t begin_index, + [[maybe_unused]] std::ranges::iterator_t index_end, + std::ranges::range_difference_t n_index) { + auto num_subranges = n_index - 1; + + subranges_.reserve(num_subranges); + for (long i = 0; i < num_subranges; ++i) { + subranges_.emplace_back( + data_begin + begin_index[i], data_begin + begin_index[i + 1]); + } + } + + /** + * Constructor taking iterator pairs for the data and index ranges, with + * sizes, tiledb format + */ + alt_var_length_view( + std::ranges::iterator_t data_begin, + [[maybe_unused]] std::ranges::iterator_t data_end, + [[maybe_unused]] std::ranges::range_difference_t n_data, + std::ranges::iterator_t begin_index, + [[maybe_unused]] std::ranges::iterator_t index_end, + std::ranges::range_difference_t n_index, + data_index_type missing_index) { + auto num_subranges = n_index; + + subranges_.reserve(num_subranges); + for (long i = 0; i < num_subranges - 1; ++i) { + subranges_.emplace_back( + data_begin + begin_index[i], data_begin + begin_index[i + 1]); + } + subranges_.emplace_back( + data_begin + begin_index[num_subranges - 1], + data_begin + missing_index); + } + + /** + * Constructor taking ranges for the data and index ranges arrow + * format + */ alt_var_length_view(R& data, const I& index) { + auto num_subranges = std::ranges::size(index) - 1; + auto data_begin(std::ranges::begin(data)); + auto index_begin(std::ranges::begin(index)); + + subranges_.reserve(num_subranges); + + for (size_t i = 0; i < num_subranges; ++i) { + subranges_.emplace_back( + data_begin + index_begin[i], data_begin + index_begin[i + 1]); + } + } + + /** + * Constructor taking ranges for the data and index ranges + * tiledb format + */ + alt_var_length_view(R& data, const I& index, data_index_type missing_index) { + auto num_subranges = std::ranges::size(index); auto data_begin(std::ranges::begin(data)); + [[maybe_unused]] auto index_begin(std::ranges::begin(index)); + + subranges_.reserve(num_subranges); - subranges_.reserve(std::ranges::size(index) - 1); - for (size_t i = 0; i < std::ranges::size(index) - 1; ++i) { + for (size_t i = 0; i < num_subranges - 1; ++i) { subranges_.emplace_back(data_begin + index[i], data_begin + index[i + 1]); } + subranges_.emplace_back( + data_begin + index.back(), data_begin + missing_index); } - /** Constructor. The offsets do not contain the final index value (which would - * be the end of the data range), so the final index is passed in as a - * separate argument. + /** + * Constructor taking ranges for the data and index ranges, with sizes, arrow + * format */ - alt_var_length_view(R& data, const I& index, data_index_type end_index) { + alt_var_length_view( + R& data, + [[maybe_unused]] std::ranges::range_difference_t n_data, + const I& index, + std::ranges::range_difference_t n_index) { + auto num_subranges = n_index - 1; auto data_begin(std::ranges::begin(data)); + auto index_begin(std::ranges::begin(index)); - subranges_.reserve(std::ranges::size(index) - 1); + subranges_.reserve(num_subranges); - for (size_t i = 0; i < std::ranges::size(index) - 1; ++i) { + for (long i = 0; i < num_subranges; ++i) { + subranges_.emplace_back( + data_begin + index_begin[i], data_begin + index_begin[i + 1]); + } + } + + /** + * Constructor taking ranges for the data and index ranges, with sizes, + * tiledb format + */ + alt_var_length_view( + R& data, + [[maybe_unused]] std::ranges::range_difference_t n_data, + const I& index, + std::ranges::range_difference_t n_index, + data_index_type missing_index) { + auto num_subranges = n_index; + auto data_begin(std::ranges::begin(data)); + [[maybe_unused]] auto index_begin(std::ranges::begin(index)); + + subranges_.reserve(num_subranges); + + for (long i = 0; i < num_subranges - 1; ++i) { subranges_.emplace_back(data_begin + index[i], data_begin + index[i + 1]); } - subranges_.emplace_back(data_begin + index.back(), data_begin + end_index); + subranges_.emplace_back( + data_begin + index[num_subranges - 1], data_begin + missing_index); } /** Return iterator to the beginning of the var length view */ @@ -180,4 +320,23 @@ class alt_var_length_view : public std::ranges::view_base { std::vector subranges_; }; +/** Deduction guide for alt_var_length_view */ +template +alt_var_length_view(R, R, I, I) + -> alt_var_length_view, std::ranges::subrange>; + +/** Deduction guide for alt_var_length_view */ +template +alt_var_length_view(R, R, I, I, J) + -> alt_var_length_view, std::ranges::subrange>; + +/** Deduction guide for alt_var_length_view */ +template +alt_var_length_view(R, R, J, I, I, K) + -> alt_var_length_view, std::ranges::subrange>; + +/** Deduction guide for alt_var_length_view */ +template +alt_var_length_view(R, R, J, I, I, K, L) + -> alt_var_length_view, std::ranges::subrange>; #endif // TILEDB_ALT_VAR_LENGTH_VIEW_H diff --git a/tiledb/common/test/unit_alt_var_length_view.cc b/tiledb/common/test/unit_alt_var_length_view.cc index fd56d10aec09..9f0ee32ad0b5 100644 --- a/tiledb/common/test/unit_alt_var_length_view.cc +++ b/tiledb/common/test/unit_alt_var_length_view.cc @@ -116,9 +116,44 @@ TEST_CASE( // Simple test that the alt_var_length_view can be constructed TEST_CASE("alt_var_length_view: Basic constructor", "[alt_var_length_view]") { std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + std::vector> expected = { + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {7.0, 8.0, 9.0, 10.0}, + }; + + SECTION("iterator pair") { + auto u = alt_var_length_view(r.begin(), r.end(), o.begin(), o.end()); + auto v = alt_var_length_view{r.begin(), r.end(), o.begin(), o.end()}; + alt_var_length_view w(r.begin(), r.end(), o.begin(), o.end()); + alt_var_length_view x{r.begin(), r.end(), o.begin(), o.end()}; - SECTION("Arrow format") { - std::vector o = {0, 3, 6, 10}; + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + SECTION("iterator pair with size") { + auto u = alt_var_length_view(r.begin(), r.end(), 6, o.begin(), o.end(), 3); + auto v = alt_var_length_view{r.begin(), r.end(), 6, o.begin(), o.end(), 3}; + alt_var_length_view w(r.begin(), r.end(), 6, o.begin(), o.end(), 3); + alt_var_length_view x{r.begin(), r.end(), 6, o.begin(), o.end(), 3}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + SECTION("range") { auto u = alt_var_length_view(r, o); auto v = alt_var_length_view{r, o}; alt_var_length_view w(r, o); @@ -128,18 +163,94 @@ TEST_CASE("alt_var_length_view: Basic constructor", "[alt_var_length_view]") { CHECK(size(v) == 3); CHECK(size(w) == 3); CHECK(size(x) == 3); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + + SECTION("range with size") { + auto u = alt_var_length_view(r, 6, o, 3); + auto v = alt_var_length_view{r, 6, o, 3}; + alt_var_length_view w(r, 6, o, 3); + alt_var_length_view x{r, 6, o, 3}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } } - SECTION("TileDB format") { - std::vector o = {0, 3, 6}; - auto u = alt_var_length_view(r, o, 10); - auto v = alt_var_length_view{r, o, 10}; - alt_var_length_view w(r, o, 10); - alt_var_length_view x{r, o, 10}; + + SECTION("iterator pair, tiledb format") { + auto u = + alt_var_length_view(r.begin(), r.end(), o.begin(), o.end() - 1, 10); + auto v = + alt_var_length_view{r.begin(), r.end(), o.begin(), o.end() - 1, 10}; + alt_var_length_view w(r.begin(), r.end(), o.begin(), o.end() - 1, 10); + alt_var_length_view x{r.begin(), r.end(), o.begin(), o.end() - 1, 10}; + + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + + SECTION("iterator pair with size, tiledb format") { + auto u = + alt_var_length_view(r.begin(), r.end(), 6, o.begin(), o.end(), 2, 6); + auto v = + alt_var_length_view{r.begin(), r.end(), 6, o.begin(), o.end(), 2, 6}; + alt_var_length_view w(r.begin(), r.end(), 6, o.begin(), o.end(), 2, 6); + alt_var_length_view x{r.begin(), r.end(), 6, o.begin(), o.end(), 2, 6}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + + SECTION("range, tiledb format") { + auto u = alt_var_length_view(r, std::ranges::views::take(o, 3), 10); + auto v = alt_var_length_view{r, std::ranges::views::take(o, 3), 10}; + alt_var_length_view w(r, std::ranges::views::take(o, 3), 10); + alt_var_length_view x{r, std::ranges::views::take(o, 3), 10}; CHECK(size(u) == 3); CHECK(size(v) == 3); CHECK(size(w) == 3); CHECK(size(x) == 3); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } + + SECTION("range with size, tiledb format") { + auto u = alt_var_length_view(r, 6, o, 2, 6); + auto v = alt_var_length_view{r, 6, o, 2, 6}; + alt_var_length_view w(r, 6, o, 2, 6); + alt_var_length_view x{r, 6, o, 2, 6}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } } } diff --git a/tiledb/common/test/unit_var_length_view.cc b/tiledb/common/test/unit_var_length_view.cc index 45f7988f302b..9540a824a8f1 100644 --- a/tiledb/common/test/unit_var_length_view.cc +++ b/tiledb/common/test/unit_var_length_view.cc @@ -103,19 +103,54 @@ TEST_CASE( } // Simple test that the var_length_view can be constructed -TEST_CASE("var_length_view: Basic constructor", "[var_length_view]") { +TEST_CASE("var_length_view: Constructors", "[var_length_view]") { std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; std::vector o = {0, 3, 6, 10}; - auto u = var_length_view(r, o); - auto v = var_length_view{r, o}; - var_length_view w(r, o); - var_length_view x{r, o}; + SECTION("iterator pair") { + auto u = var_length_view(r.begin(), r.end(), o.begin(), o.end()); + auto v = var_length_view{r.begin(), r.end(), o.begin(), o.end()}; + var_length_view w(r.begin(), r.end(), o.begin(), o.end()); + var_length_view x{r.begin(), r.end(), o.begin(), o.end()}; - CHECK(size(u) == 3); - CHECK(size(v) == 3); - CHECK(size(w) == 3); - CHECK(size(x) == 3); + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + } + SECTION("iterator pair with size") { + auto u = var_length_view(r.begin(), r.end(), 6, o.begin(), o.end(), 3); + auto v = var_length_view{r.begin(), r.end(), 6, o.begin(), o.end(), 3}; + var_length_view w(r.begin(), r.end(), 6, o.begin(), o.end(), 3); + var_length_view x{r.begin(), r.end(), 6, o.begin(), o.end(), 3}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + } + SECTION("range") { + auto u = var_length_view(r, o); + auto v = var_length_view{r, o}; + var_length_view w(r, o); + var_length_view x{r, o}; + + CHECK(size(u) == 3); + CHECK(size(v) == 3); + CHECK(size(w) == 3); + CHECK(size(x) == 3); + } + SECTION("range with size") { + auto u = var_length_view(r, 6, o, 3); + auto v = var_length_view{r, 6, o, 3}; + var_length_view w(r, 6, o, 3); + var_length_view x{r, 6, o, 3}; + + CHECK(size(u) == 2); + CHECK(size(v) == 2); + CHECK(size(w) == 2); + CHECK(size(x) == 2); + } } // Check that the sizes of the var_length_view are correct diff --git a/tiledb/common/var_length_view.h b/tiledb/common/var_length_view.h index 089ee78abd7d..f5c020e85906 100644 --- a/tiledb/common/var_length_view.h +++ b/tiledb/common/var_length_view.h @@ -60,6 +60,7 @@ * @tparam I Type of the index range, assumed to be a random access range. * * @todo R could be a view rather than a range. + * @todo: Should we use `view_interface` instead of view_base? */ template < std::ranges::random_access_range R, @@ -73,10 +74,12 @@ class var_length_view : public std::ranges::view_base { using data_iterator_type = std::ranges::iterator_t; /** The type that can index into the var length data range */ - using data_index_type = std::iter_difference_t; + using data_index_type = std::ranges::range_difference_t; - /** The type of the iterator over the index range -- It should dereference to - * something that can index into the data range (e.g., the data_index_type) */ + /** + * The type of the iterator over the index range -- It should dereference to + * something that can index into the data range (e.g., the data_index_type) + */ using index_iterator_type = std::ranges::iterator_t; /** The type dereferenced by the iterator is a subrange */ @@ -89,12 +92,72 @@ class var_length_view : public std::ranges::view_base { using var_length_const_iterator = private_iterator; public: - /** Primary constructor */ + /***************************************************************************** + * Constructors + * The full litany of constructors (for each input range): + * var_l_view(data_begin, data_end, index_begin, index_end); + * var_l_view(data_begin, data_end, n_data, index_begin, index_end, n_index); + * var_l_view(data, index); + * var_l_view(data, n_data, index, n_index); + ****************************************************************************/ + + /** Constructor taking iterator pairs for the data and index ranges */ + var_length_view( + std::ranges::iterator_t data_begin, + std::ranges::iterator_t data_end, + std::ranges::iterator_t index_begin, + std::ranges::iterator_t index_end) + : data_begin_(data_begin) + , data_end_(data_end) + , index_begin_(index_begin) + , index_end_(index_end) + , num_subranges_(index_end - index_begin - 1) { + } + + /** + * Constructor taking iterator pairs for the data and index ranges, along + * with sizes + */ + var_length_view( + std::ranges::iterator_t data_begin, + std::ranges::iterator_t data_end, + std::ranges::range_difference_t n_data, + std::ranges::iterator_t index_begin, + std::ranges::iterator_t index_end, + std::ranges::range_difference_t n_index) + : data_begin_(data_begin) + , data_end_(data_begin + n_data) + , index_begin_(index_begin) + , index_end_(index_begin + n_index) + , num_subranges_(index_end_ - index_begin_ - 1) { + assert(data_end - data_begin >= n_data); + assert(index_end - index_begin >= n_index); + } + + /** Constructor taking ranges for the data and index ranges */ var_length_view(R& data, const I& index) : data_begin_(std::ranges::begin(data)) , data_end_(std::ranges::end(data)) , index_begin_(std::ranges::cbegin(index)) - , index_end_(std::ranges::cend(index) - 1) { + , index_end_(std::ranges::cend(index)) + , num_subranges_(std::ranges::size(index) - 1) { + } + + /** + * Constructor taking ranges for the data and index ranges, along with sizes + */ + var_length_view( + R& data, + std::ranges::range_difference_t n_data, + const I& index, + std::ranges::range_difference_t n_index) + : data_begin_(std::ranges::begin(data)) + , data_end_(std::ranges::begin(data) + n_data) + , index_begin_(std::ranges::cbegin(index)) + , index_end_(std::ranges::cbegin(index) + n_index) + , num_subranges_(n_index - 1) { + assert(data_end_ - data_begin_ >= n_data); + assert(index_end_ - index_begin_ >= n_index); } /** Return iterator to the beginning of the var length view */ @@ -104,8 +167,7 @@ class var_length_view : public std::ranges::view_base { /** Return iterator to the end of the var length view */ auto end() { - return var_length_iterator( - data_begin_, index_begin_, index_end_ - index_begin_); + return var_length_iterator(data_begin_, index_begin_, num_subranges_); } /** Return const iterator to the beginning of the var length view */ @@ -115,8 +177,7 @@ class var_length_view : public std::ranges::view_base { /** Return const iterator to the end of the var length view */ auto end() const { - return var_length_const_iterator( - data_begin_, index_begin_, index_end_ - index_begin_); + return var_length_const_iterator(data_begin_, index_begin_, num_subranges_); } /** Return const iterator to the beginning of the var length view */ @@ -126,13 +187,12 @@ class var_length_view : public std::ranges::view_base { /** Return const iterator to the end of the var length view */ auto cend() const { - return var_length_const_iterator( - data_begin_, index_begin_, index_end_ - index_begin_); + return var_length_const_iterator(data_begin_, index_begin_, num_subranges_); } /** Return the number of subranges in the var length view */ auto size() const { - return index_end_ - index_begin_; + return num_subranges_; } private: @@ -212,8 +272,27 @@ class var_length_view : public std::ranges::view_base { /** The beginning of the index range */ std::ranges::iterator_t index_begin_; - /** The end of the index range */ + /** + * The end of the index range. This the actual end, not the element that + * points to the end of the data. + */ std::ranges::iterator_t index_end_; + + /** + * Length of the active index range (the number of subranges). The number of + * subranges is one less than size of the (arrow format) index range. + */ + std::ranges::range_difference_t num_subranges_; }; +/** Deduction guide for var_length_view */ +template +var_length_view(R, R, I, I) + -> var_length_view, std::ranges::subrange>; + +/** Deduction guide for var_length_view */ +template +var_length_view(R, R, J, I, I, K) + -> var_length_view, std::ranges::subrange>; + #endif // TILEDB_VAR_LENGTH_VIEW_H From 60bd1d08d61d267b8cf0caa0a8e1c57679451f8e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 1 May 2024 14:57:01 -0400 Subject: [PATCH 315/456] Migrate Array APIs out of StorageManager: non_empty_domain_*_from_index. (#4909) Migrate `Array` APIs out of `StorageManager`: `non_empty_domain_*_from_index`. This includes 3 separate APIs: * `non_empty_domain_from_index` * `non_empty_domain_var_size_from_index` * `non_empty_domain_var_from_index` --- [sc-44989] [sc-44991] [sc-44993] --- TYPE: NO_HISTORY DESC: Migrate Array APIs out of StorageManager: non_empty_domain_*_from_index. --- tiledb/sm/array/array.cc | 87 +++++++++++++++++ tiledb/sm/array/array.h | 36 +++++++ tiledb/sm/c_api/tiledb.cc | 34 +++---- tiledb/sm/storage_manager/storage_manager.cc | 97 ------------------- .../storage_manager_canonical.h | 50 ---------- 5 files changed, 140 insertions(+), 164 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index a7a389b4cafb..786405465223 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1123,6 +1123,93 @@ void Array::non_empty_domain(void* domain, bool* is_empty) { } } +void Array::non_empty_domain_from_index( + unsigned idx, void* domain, bool* is_empty) { + if (!is_open_) { + throw ArrayException("[non_empty_domain_from_index] Array is not open"); + } + // For easy reference + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + + // Sanity checks + if (idx >= array_schema.dim_num()) { + throw ArrayException( + "Cannot get non-empty domain; Invalid dimension index"); + } + if (array_domain.dimension_ptr(idx)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + + array_domain.dimension_ptr(idx)->name() + "' is var-sized"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + if (*is_empty) { + return; + } + + std::memcpy(domain, dom[idx].data(), dom[idx].size()); +} + +void Array::non_empty_domain_var_size_from_index( + unsigned idx, uint64_t* start_size, uint64_t* end_size, bool* is_empty) { + // For easy reference + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + + // Sanity checks + if (idx >= array_schema.dim_num()) { + throw ArrayException( + "Cannot get non-empty domain; Invalid dimension index"); + } + if (!array_domain.dimension_ptr(idx)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + + array_domain.dimension_ptr(idx)->name() + "' is fixed-sized"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + if (*is_empty) { + *start_size = 0; + *end_size = 0; + return; + } + + *start_size = dom[idx].start_size(); + *end_size = dom[idx].end_size(); +} + +void Array::non_empty_domain_var_from_index( + unsigned idx, void* start, void* end, bool* is_empty) { + // For easy reference + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + + // Sanity checks + if (idx >= array_schema.dim_num()) { + throw ArrayException( + "Cannot get non-empty domain; Invalid dimension index"); + } + if (!array_domain.dimension_ptr(idx)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + + array_domain.dimension_ptr(idx)->name() + "' is fixed-sized"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + if (*is_empty) { + return; + } + + auto start_str = dom[idx].start_str(); + std::memcpy(start, start_str.data(), start_str.size()); + auto end_str = dom[idx].end_str(); + std::memcpy(end, end_str.data(), end_str.size()); +} + bool Array::serialize_non_empty_domain() const { auto found = false; auto serialize_ned_array_open = false; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index e4598fd1e2e6..04ae9a643047 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -778,6 +778,42 @@ class Array { */ void non_empty_domain(void* domain, bool* is_empty); + /** + * Returns the non-empty domain of the opened array on the given dimension. + * This is the union of the non-empty domains of the array fragments. + * + * @param idx The dimension index. + * @param domain The domain to be retrieved. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_from_index(unsigned idx, void* domain, bool* is_empty); + + /** + * Returns the non-empty domain size of the opened array on the given + * dimension. This is the union of the non-empty domains of the array + * fragments. Applicable only to var-sized dimensions. + * + * @param idx The dimension index. + * @param start_size The size in bytes of the range start. + * @param end_size The size in bytes of the range end. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_var_size_from_index( + unsigned idx, uint64_t* start_size, uint64_t* end_size, bool* is_empty); + + /** + * Returns the non-empty domain of the opened array on the given dimension. + * This is the union of the non-empty domains of the array fragments. + * Applicable only to var-sized dimensions. + * + * @param idx The dimension index. + * @param start The domain range start to set. + * @param end The domain range end to set. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_var_from_index( + unsigned idx, void* start, void* end, bool* is_empty); + /** * Retrieves the array metadata object that is already loaded. If it's not yet * loaded it will be empty. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index ed25afe2370a..09026fa6bf44 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2801,14 +2801,13 @@ int32_t tiledb_array_get_non_empty_domain_from_index( uint32_t idx, void* domain, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + ensure_output_pointer_is_valid(is_empty); bool is_empty_b; - - throw_if_not_ok(ctx->storage_manager()->array_get_non_empty_domain_from_index( - array->array_.get(), idx, domain, &is_empty_b)); - + array->array_->non_empty_domain_from_index(idx, domain, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; @@ -2840,15 +2839,16 @@ int32_t tiledb_array_get_non_empty_domain_var_size_from_index( uint64_t* start_size, uint64_t* end_size, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + ensure_output_pointer_is_valid(start_size); + ensure_output_pointer_is_valid(end_size); + ensure_output_pointer_is_valid(is_empty); bool is_empty_b = true; - - throw_if_not_ok( - ctx->storage_manager()->array_get_non_empty_domain_var_size_from_index( - array->array_.get(), idx, start_size, end_size, &is_empty_b)); - + array->array_->non_empty_domain_var_size_from_index( + idx, start_size, end_size, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; @@ -2882,15 +2882,15 @@ int32_t tiledb_array_get_non_empty_domain_var_from_index( void* start, void* end, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + ensure_output_pointer_is_valid(start); + ensure_output_pointer_is_valid(end); + ensure_output_pointer_is_valid(is_empty); bool is_empty_b = true; - - throw_if_not_ok( - ctx->storage_manager()->array_get_non_empty_domain_var_from_index( - array->array_.get(), idx, start, end, &is_empty_b)); - + array->array_->non_empty_domain_var_from_index(idx, start, end, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 2cd868d688a8..22e441d7654e 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -497,37 +497,6 @@ Status StorageManagerCanonical::array_upgrade_version( return Status::Ok(); } -Status StorageManagerCanonical::array_get_non_empty_domain_from_index( - Array* array, unsigned idx, void* domain, bool* is_empty) { - // Check if array is open - must be open for reads - if (!array->is_open()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array is not open")); - - // For easy reference - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - - // Sanity checks - if (idx >= array_schema.dim_num()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension index")); - if (array_domain.dimension_ptr(idx)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += array_domain.dimension_ptr(idx)->name(); - errmsg += "' is variable-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - if (*is_empty) - return Status::Ok(); - - std::memcpy(domain, dom[idx].data(), dom[idx].size()); - return Status::Ok(); -} - Status StorageManagerCanonical::array_get_non_empty_domain_from_name( Array* array, const char* name, void* domain, bool* is_empty) { // Sanity check @@ -567,41 +536,6 @@ Status StorageManagerCanonical::array_get_non_empty_domain_from_name( "' does not exist")); } -Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_index( - Array* array, - unsigned idx, - uint64_t* start_size, - uint64_t* end_size, - bool* is_empty) { - // For easy reference - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - - // Sanity checks - if (idx >= array_schema.dim_num()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension index")); - if (!array_domain.dimension_ptr(idx)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += array_domain.dimension_ptr(idx)->name(); - errmsg += "' is fixed-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - if (*is_empty) { - *start_size = 0; - *end_size = 0; - return Status::Ok(); - } - - *start_size = dom[idx].start_size(); - *end_size = dom[idx].end_size(); - - return Status::Ok(); -} - Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_name( Array* array, const char* name, @@ -646,37 +580,6 @@ Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_name( "' does not exist")); } -Status StorageManagerCanonical::array_get_non_empty_domain_var_from_index( - Array* array, unsigned idx, void* start, void* end, bool* is_empty) { - // For easy reference - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - - // Sanity checks - if (idx >= array_schema.dim_num()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension index")); - if (!array_domain.dimension_ptr(idx)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += array_domain.dimension_ptr(idx)->name(); - errmsg += "' is fixed-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - - if (*is_empty) - return Status::Ok(); - - auto start_str = dom[idx].start_str(); - std::memcpy(start, start_str.data(), start_str.size()); - auto end_str = dom[idx].end_str(); - std::memcpy(end, end_str.data(), end_str.size()); - - return Status::Ok(); -} - Status StorageManagerCanonical::array_get_non_empty_domain_var_from_name( Array* array, const char* name, void* start, void* end, bool* is_empty) { // Sanity check diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 06c34262f314..a1a541154b9e 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -242,20 +242,6 @@ class StorageManagerCanonical { */ Status array_upgrade_version(const URI& uri, const Config& config); - /** - * Retrieves the non-empty domain from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * - * @param array An open array object (must be already open). - * @param idx The dimension index. - * @param domain The domain to be retrieved. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_from_index( - Array* array, unsigned idx, void* domain, bool* is_empty); - /** * Retrieves the non-empty domain from an array on the given dimension. * This is the union of the non-empty domains of the array fragments. @@ -270,26 +256,6 @@ class StorageManagerCanonical { Status array_get_non_empty_domain_from_name( Array* array, const char* name, void* domain, bool* is_empty); - /** - * Retrieves the non-empty domain size from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * Applicable only to var-sized dimensions. - * - * @param array An open array object (must be already open). - * @param idx The dimension index. - * @param start_size The size in bytes of the range start. - * @param end_size The size in bytes of the range end. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_var_size_from_index( - Array* array, - unsigned idx, - uint64_t* start_size, - uint64_t* end_size, - bool* is_empty); - /** * Retrieves the non-empty domain size from an array on the given dimension. * This is the union of the non-empty domains of the array fragments. @@ -310,22 +276,6 @@ class StorageManagerCanonical { uint64_t* end_size, bool* is_empty); - /** - * Retrieves the non-empty domain from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * Applicable only to var-sized dimensions. - * - * @param array An open array object (must be already open). - * @param idx The dimension index. - * @param start The domain range start to set. - * @param end The domain range end to set. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_var_from_index( - Array* array, unsigned idx, void* start, void* end, bool* is_empty); - /** * Retrieves the non-empty domain from an array on the given dimension. * This is the union of the non-empty domains of the array fragments. From bc57250f1b64d6481fc722d53cd3dd5c0e5bd8bd Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Wed, 1 May 2024 16:36:07 -0500 Subject: [PATCH 316/456] Add missing include required by std::copy_n use. (#4931) Use of `std::copy_n`, added in #4915, requires adding the `algorithms` header under `g++` compilation [sc-46538] --- TYPE: NO_HISTORY DESC: Add `algorithms` header for `std::copy_n` --- tiledb/api/c_api_support/cpp_string/cpp_string.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tiledb/api/c_api_support/cpp_string/cpp_string.h b/tiledb/api/c_api_support/cpp_string/cpp_string.h index 2da0226e26dd..4e679c40dc03 100644 --- a/tiledb/api/c_api_support/cpp_string/cpp_string.h +++ b/tiledb/api/c_api_support/cpp_string/cpp_string.h @@ -32,6 +32,7 @@ #ifndef TILEDB_CAPI_SUPPORT_CPP_STRING_H #define TILEDB_CAPI_SUPPORT_CPP_STRING_H +#include // std::copy_n #include // std::memchr #include // std::string_view #include "../argument_validation.h" // CAPIException From c3b45ab9624f34ee99e5a54f8dd46e24d738ba4f Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Thu, 2 May 2024 00:59:28 +0300 Subject: [PATCH 317/456] Fix broken unit build on macos. (#4932) This fixes ``` unit_iterator_facade.cc:625:22: error: missing 'typename' prior to dependent type name 'std::iterator_tra its::value_type' using value_type = std::iterator_traits::value_type; ``` which I got building units on my macos. [sc-46539] --- TYPE: NO_HISTORY DESC: Fix broken unit build on macos. --- tiledb/common/test/unit_iterator_facade.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tiledb/common/test/unit_iterator_facade.cc b/tiledb/common/test/unit_iterator_facade.cc index c45a7e27e783..4ceafc0e88d8 100644 --- a/tiledb/common/test/unit_iterator_facade.cc +++ b/tiledb/common/test/unit_iterator_facade.cc @@ -622,9 +622,10 @@ struct simple_mutable_struct { using iterator = simple_mutable_iterator; using const_iterator = simple_mutable_iterator; - using value_type = std::iterator_traits::value_type; - using reference = std::iterator_traits::reference; - using const_reference = std::iterator_traits::reference; + using value_type = typename std::iterator_traits::value_type; + using reference = typename std::iterator_traits::reference; + using const_reference = + typename std::iterator_traits::reference; auto begin() { return iterator{value.begin()}; From 9671bad645fdeaaa483182340be14ed506fa60d2 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 2 May 2024 01:51:52 -0400 Subject: [PATCH 318/456] Migrate Array APIs out of StorageManager: non_empty_domain_*_from_name. (#4910) Migrate `Array` APIs out of `StorageManager`: `non_empty_domain_*_from_name`. This includes 3 separate APIs: * `non_empty_domain_from_name` * `non_empty_domain_var_size_from_name` * `non_empty_domain_var_from_name` --- [sc-44990] [sc-44992] [sc-44994] --- TYPE: IMPROVEMENT DESC: Migrate Array APIs out of StorageManager: non_empty_domain_*_from_name. --- tiledb/sm/array/array.cc | 118 +++++++++++++++++ tiledb/sm/array/array.h | 40 ++++++ tiledb/sm/c_api/tiledb.cc | 35 ++--- tiledb/sm/storage_manager/storage_manager.cc | 122 ------------------ .../storage_manager_canonical.h | 50 ------- 5 files changed, 176 insertions(+), 189 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 786405465223..f3822471d47b 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1210,6 +1210,124 @@ void Array::non_empty_domain_var_from_index( std::memcpy(end, end_str.data(), end_str.size()); } +void Array::non_empty_domain_from_name( + std::string_view field_name, void* domain, bool* is_empty) { + // Sanity check + if (field_name.data() == nullptr) { + throw std::invalid_argument( + "[non_empty_domain_from_name] Invalid dimension name"); + } + + // Check if array is open - must be open for reads + if (!is_open_) { + throw ArrayException("[non_empty_domain_from_name] Array is not open"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + auto dim_num = array_schema.dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + const auto& dim_name{array_schema.dimension_ptr(d)->name()}; + if (field_name == dim_name) { + // Sanity check + if (array_domain.dimension_ptr(d)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + dim_name + + "' is variable-sized"); + } + if (!*is_empty) { + std::memcpy(domain, dom[d].data(), dom[d].size()); + } + return; + } + } + + throw ArrayException( + "Cannot get non-empty domain; Dimension name '" + + std::string(field_name) + "' does not exist"); +} + +void Array::non_empty_domain_var_size_from_name( + std::string_view field_name, + uint64_t* start_size, + uint64_t* end_size, + bool* is_empty) { + // Sanity check + if (field_name.data() == nullptr) { + throw std::invalid_argument("[non_empty_domain] Invalid dimension name"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + auto dim_num = array_schema.dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + const auto& dim_name{array_schema.dimension_ptr(d)->name()}; + if (field_name == dim_name) { + // Sanity check + if (!array_domain.dimension_ptr(d)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + dim_name + + "' is fixed-sized"); + } + if (*is_empty) { + *start_size = 0; + *end_size = 0; + } else { + *start_size = dom[d].start_size(); + *end_size = dom[d].end_size(); + } + return; + } + } + + throw ArrayException( + "Cannot get non-empty domain; Dimension name '" + + std::string(field_name) + "' does not exist"); +} + +void Array::non_empty_domain_var_from_name( + std::string_view field_name, void* start, void* end, bool* is_empty) { + // Sanity check + if (field_name.data() == nullptr) { + throw std::invalid_argument("[non_empty_domain] Invalid dimension name"); + } + + NDRange dom; + non_empty_domain(&dom, is_empty); + + const auto& array_schema = array_schema_latest(); + auto& array_domain{array_schema.domain()}; + auto dim_num = array_schema.dim_num(); + for (unsigned d = 0; d < dim_num; ++d) { + const auto& dim_name{array_schema.dimension_ptr(d)->name()}; + if (field_name == dim_name) { + // Sanity check + if (!array_domain.dimension_ptr(d)->var_size()) { + throw ArrayException( + "Cannot get non-empty domain; Dimension '" + dim_name + + "' is fixed-sized"); + } + if (!*is_empty) { + auto start_str = dom[d].start_str(); + std::memcpy(start, start_str.data(), start_str.size()); + auto end_str = dom[d].end_str(); + std::memcpy(end, end_str.data(), end_str.size()); + } + return; + } + } + + throw ArrayException( + "Cannot get non-empty domain; Dimension name '" + + std::string(field_name) + "' does not exist"); +} + bool Array::serialize_non_empty_domain() const { auto found = false; auto serialize_ned_array_open = false; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 04ae9a643047..1af31137ee76 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -814,6 +814,46 @@ class Array { void non_empty_domain_var_from_index( unsigned idx, void* start, void* end, bool* is_empty); + /** + * Returns the non-empty domain of the opened array on the given dimension. + * This is the union of the non-empty domains of the array fragments. + * + * @param field_name The dimension name. + * @param domain The domain to be retrieved. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_from_name( + std::string_view field_name, void* domain, bool* is_empty); + + /** + * Returns the non-empty domain size of the open array on the given dimension. + * This is the union of the non-empty domains of the array fragments. + * Applicable only to var-sized dimensions. + * + * @param field_name The dimension name. + * @param start_size The size in bytes of the range start. + * @param end_size The size in bytes of the range end. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_var_size_from_name( + std::string_view field_name, + uint64_t* start_size, + uint64_t* end_size, + bool* is_empty); + + /** + * Returns the non-empty domain of the opened array on the given dimension. + * This is the union of the non-empty domains of the array fragments. + * Applicable only to var-sized dimensions. + + * @param field_name The dimension name. + * @param start The domain range start to set. + * @param end The domain range end to set. + * @param is_empty `true` if the non-empty domain (and array) is empty. + */ + void non_empty_domain_var_from_name( + std::string_view field_name, void* start, void* end, bool* is_empty); + /** * Retrieves the array metadata object that is already loaded. If it's not yet * loaded it will be empty. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 09026fa6bf44..12e0ffd0ed88 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2819,14 +2819,14 @@ int32_t tiledb_array_get_non_empty_domain_from_name( const char* name, void* domain, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + auto field_name{to_string_view<"field name">(name)}; + ensure_output_pointer_is_valid(is_empty); bool is_empty_b; - - throw_if_not_ok(ctx->storage_manager()->array_get_non_empty_domain_from_name( - array->array_.get(), name, domain, &is_empty_b)); - + array->array_->non_empty_domain_from_name(field_name, domain, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; @@ -2861,15 +2861,16 @@ int32_t tiledb_array_get_non_empty_domain_var_size_from_name( uint64_t* start_size, uint64_t* end_size, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + ensure_output_pointer_is_valid(start_size); + ensure_output_pointer_is_valid(end_size); + ensure_output_pointer_is_valid(is_empty); bool is_empty_b = true; - - throw_if_not_ok( - ctx->storage_manager()->array_get_non_empty_domain_var_size_from_name( - array->array_.get(), name, start_size, end_size, &is_empty_b)); - + array->array_->non_empty_domain_var_size_from_name( + name, start_size, end_size, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; @@ -2903,15 +2904,15 @@ int32_t tiledb_array_get_non_empty_domain_var_from_name( void* start, void* end, int32_t* is_empty) { - if (sanity_check(ctx, array) == TILEDB_ERR) + if (sanity_check(ctx, array) == TILEDB_ERR) { return TILEDB_ERR; + } + ensure_output_pointer_is_valid(start); + ensure_output_pointer_is_valid(end); + ensure_output_pointer_is_valid(is_empty); bool is_empty_b = true; - - throw_if_not_ok( - ctx->storage_manager()->array_get_non_empty_domain_var_from_name( - array->array_.get(), name, start, end, &is_empty_b)); - + array->array_->non_empty_domain_var_from_name(name, start, end, &is_empty_b); *is_empty = (int32_t)is_empty_b; return TILEDB_OK; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 22e441d7654e..a448eddad48d 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -497,128 +497,6 @@ Status StorageManagerCanonical::array_upgrade_version( return Status::Ok(); } -Status StorageManagerCanonical::array_get_non_empty_domain_from_name( - Array* array, const char* name, void* domain, bool* is_empty) { - // Sanity check - if (name == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension name")); - - // Check if array is open - must be open for reads - if (!array->is_open()) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Array is not open")); - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - auto dim_num = array_schema.dim_num(); - for (unsigned d = 0; d < dim_num; ++d) { - const auto& dim_name{array_schema.dimension_ptr(d)->name()}; - if (name == dim_name) { - // Sanity check - if (array_domain.dimension_ptr(d)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += dim_name + "' is variable-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - if (!*is_empty) - std::memcpy(domain, dom[d].data(), dom[d].size()); - return Status::Ok(); - } - } - - return logger_->status(Status_StorageManagerError( - std::string("Cannot get non-empty domain; Dimension name '") + name + - "' does not exist")); -} - -Status StorageManagerCanonical::array_get_non_empty_domain_var_size_from_name( - Array* array, - const char* name, - uint64_t* start_size, - uint64_t* end_size, - bool* is_empty) { - // Sanity check - if (name == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension name")); - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - auto dim_num = array_schema.dim_num(); - for (unsigned d = 0; d < dim_num; ++d) { - const auto& dim_name{array_schema.dimension_ptr(d)->name()}; - if (name == dim_name) { - // Sanity check - if (!array_domain.dimension_ptr(d)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += dim_name + "' is fixed-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - if (*is_empty) { - *start_size = 0; - *end_size = 0; - } else { - *start_size = dom[d].start_size(); - *end_size = dom[d].end_size(); - } - - return Status::Ok(); - } - } - - return logger_->status(Status_StorageManagerError( - std::string("Cannot get non-empty domain; Dimension name '") + name + - "' does not exist")); -} - -Status StorageManagerCanonical::array_get_non_empty_domain_var_from_name( - Array* array, const char* name, void* start, void* end, bool* is_empty) { - // Sanity check - if (name == nullptr) - return logger_->status(Status_StorageManagerError( - "Cannot get non-empty domain; Invalid dimension name")); - - NDRange dom; - array->non_empty_domain(&dom, is_empty); - - const auto& array_schema = array->array_schema_latest(); - auto& array_domain{array_schema.domain()}; - auto dim_num = array_schema.dim_num(); - for (unsigned d = 0; d < dim_num; ++d) { - const auto& dim_name{array_schema.dimension_ptr(d)->name()}; - if (name == dim_name) { - // Sanity check - if (!array_domain.dimension_ptr(d)->var_size()) { - std::string errmsg = "Cannot get non-empty domain; Dimension '"; - errmsg += dim_name + "' is fixed-sized"; - return logger_->status(Status_StorageManagerError(errmsg)); - } - - if (!*is_empty) { - auto start_str = dom[d].start_str(); - std::memcpy(start, start_str.data(), start_str.size()); - auto end_str = dom[d].end_str(); - std::memcpy(end, end_str.data(), end_str.size()); - } - - return Status::Ok(); - } - } - - return logger_->status(Status_StorageManagerError( - std::string("Cannot get non-empty domain; Dimension name '") + name + - "' does not exist")); -} - Status StorageManagerCanonical::array_get_encryption( const URI& uri, EncryptionType* encryption_type) { if (uri.is_invalid()) { diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index a1a541154b9e..7bcd42d6d58b 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -242,56 +242,6 @@ class StorageManagerCanonical { */ Status array_upgrade_version(const URI& uri, const Config& config); - /** - * Retrieves the non-empty domain from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * - * @param array An open array object (must be already open). - * @param name The dimension name. - * @param domain The domain to be retrieved. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_from_name( - Array* array, const char* name, void* domain, bool* is_empty); - - /** - * Retrieves the non-empty domain size from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * Applicable only to var-sized dimensions. - * - * @param array An open array object (must be already open). - * @param name The dimension name. - * @param start_size The size in bytes of the range start. - * @param end_size The size in bytes of the range end. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_var_size_from_name( - Array* array, - const char* name, - uint64_t* start_size, - uint64_t* end_size, - bool* is_empty); - - /** - * Retrieves the non-empty domain from an array on the given dimension. - * This is the union of the non-empty domains of the array fragments. - * Applicable only to var-sized dimensions. - * - * @param array An open array object (must be already open). - * @param name The dimension name. - * @param start The domain range start to set. - * @param end The domain range end to set. - * @param is_empty `ture` if the non-empty domain is empty (the array - * is empty). - * @return Status - */ - Status array_get_non_empty_domain_var_from_name( - Array* array, const char* name, void* start, void* end, bool* is_empty); - /** * Retrieves the encryption type from an array. * From 2d50d985cf595f1ac49287129082df1a24d0f44c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 2 May 2024 10:12:55 +0300 Subject: [PATCH 319/456] Move several APIs out of experimental. (#4919) [SC-46300](https://app.shortcut.com/tiledb-inc/story/46300) [SC-46301](https://app.shortcut.com/tiledb-inc/story/46301) This PR moves the following C APIs (and their C++ equivalents) to stable: * `tiledb_group_*` * `tiledb_array_delete` * `tiledb_array_delete_array` * `tiledb_array_upgrade_version` The changes are simply moving code around. --- TYPE: C_API DESC: Experimental APIs related to groups, deleting arrays and upgrading the format version of arrays were moved to stable. You can use them without including ``. --- TYPE: CPP_API DESC: The experimental `Group` class was moved to stable. You can use it without including ``. --- CMakeLists.txt | 1 - test/src/unit-cppapi-deletes.cc | 2 +- test/src/unit-cppapi-group.cc | 1 - tiledb/CMakeLists.txt | 1 - tiledb/api/c_api/group/group_api_external.h | 667 +++++++++++++++- .../group/group_api_external_experimental.h | 712 ------------------ tiledb/sm/c_api/tiledb.h | 58 ++ tiledb/sm/c_api/tiledb_experimental.h | 59 -- tiledb/sm/cpp_api/group.h | 511 +++++++++++++ tiledb/sm/cpp_api/group_experimental.h | 557 -------------- tiledb/sm/cpp_api/tiledb_experimental | 1 - 11 files changed, 1236 insertions(+), 1334 deletions(-) delete mode 100644 tiledb/api/c_api/group/group_api_external_experimental.h delete mode 100644 tiledb/sm/cpp_api/group_experimental.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9354a378e993..438eb39f39e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -354,7 +354,6 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter/filter_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter_list/filter_list_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/group/group_api_external.h" - "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/group/group_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query/query_api_enum.h" diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 69354f3ce025..1f7c7a0afdfb 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -38,7 +38,7 @@ #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/c_api/tiledb_struct_def.h" -#include "tiledb/sm/cpp_api/group_experimental.h" +#include "tiledb/sm/cpp_api/group.h" #include "tiledb/sm/cpp_api/tiledb" #ifdef _WIN32 diff --git a/test/src/unit-cppapi-group.cc b/test/src/unit-cppapi-group.cc index acb63ce10d26..44b35e654c9f 100644 --- a/test/src/unit-cppapi-group.cc +++ b/test/src/unit-cppapi-group.cc @@ -43,7 +43,6 @@ #include "tiledb/sm/c_api/tiledb.h" #include "tiledb/sm/cpp_api/group.h" -#include "tiledb/sm/cpp_api/group_experimental.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/global_state/unit_test_config.h" #include "tiledb/sm/misc/utils.h" diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 01db9d78dda8..f09dd69c1bf3 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -106,7 +106,6 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/filter_list.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/fragment_info.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/group.h - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/group_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/log.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/object_iter.h diff --git a/tiledb/api/c_api/group/group_api_external.h b/tiledb/api/c_api/group/group_api_external.h index a780f395c143..e0474dfd5e96 100644 --- a/tiledb/api/c_api/group/group_api_external.h +++ b/tiledb/api/c_api/group/group_api_external.h @@ -34,7 +34,10 @@ #define TILEDB_CAPI_GROUP_API_EXTERNAL_H #include "../api_external_common.h" -#include "tiledb/api/c_api/context/context_api_external.h" +#include "../datatype/datatype_api_external.h" +#include "../object/object_api_external.h" +#include "../query/query_api_external.h" +#include "group_api_external.h" #ifdef __cplusplus extern "C" { @@ -59,6 +62,668 @@ typedef struct tiledb_group_handle_t tiledb_group_t; TILEDB_EXPORT capi_return_t tiledb_group_create(tiledb_ctx_t* ctx, const char* group_uri) TILEDB_NOEXCEPT; +/** + * Creates a new TileDB group. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "my_group", &group); + * @endcode + * + * @param ctx The TileDB context. + * @param group_uri The group URI. + * @param group The TileDB group to be allocated + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_alloc( + tiledb_ctx_t* ctx, + const char* group_uri, + tiledb_group_t** group) TILEDB_NOEXCEPT; + +/** + * Destroys a TileDB group, freeing associated memory. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "my_group", &group); + * tiledb_group_free(&group); + * @endcode + * + * @param group The TileDB group to be freed + */ +TILEDB_EXPORT void tiledb_group_free(tiledb_group_t** group) TILEDB_NOEXCEPT; + +/** + * Opens a TileDB group. The group is opened using a query type as input. + * This is to indicate that queries created for this `tiledb_group_t` + * object will inherit the query type. In other words, `tiledb_group_t` + * objects are opened to receive only one type of queries. + * They can always be closed and be re-opened with another query type. + * Also there may be many different `tiledb_group_t` + * objects created and opened with different query types. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "hdfs:///tiledb_groups/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * @endcode + * + * @param ctx The TileDB context. + * @param group The group object to be opened. + * @param query_type The type of queries the group object will be receiving. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note If the same group object is opened again without being closed, + * an error will be set and TILEDB_ERR returned. + * @note The config should be set before opening an group. + * @note If the group is to be opened at a specfic time interval, the + * `timestamp{start, end}` values should be set to a config that's set to + * the group object before opening the group. + */ +TILEDB_EXPORT capi_return_t tiledb_group_open( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + tiledb_query_type_t query_type) TILEDB_NOEXCEPT; + +/** + * Closes a TileDB group. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "hdfs:///tiledb_groups/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tiledb_group_close(ctx, group); + * @endcode + * + * @param ctx The TileDB context. + * @param group The group object to be closed. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note If the group object has already been closed, the function has + * no effect. + */ +TILEDB_EXPORT capi_return_t +tiledb_group_close(tiledb_ctx_t* ctx, tiledb_group_t* group) TILEDB_NOEXCEPT; + +/** + * Sets the group config. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * // Set the config for the given group. + * tiledb_config_t* config; + * tiledb_group_set_config(ctx, group, config); + * tiledb_group_open(ctx, group, TILEDB_READ); + * @endcode + * + * @param ctx The TileDB context. + * @param group The group to set the config for. + * @param config The config to be set. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @pre The config must be set on a closed group. + */ +TILEDB_EXPORT capi_return_t tiledb_group_set_config( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + tiledb_config_t* config) TILEDB_NOEXCEPT; + +/** + * Gets the group config. + * + * **Example:** + * + * @code{.c} + * // Retrieve the config for the given group. + * tiledb_config_t* config; + * tiledb_group_get_config(ctx, group, config); + * @endcode + * + * @param ctx The TileDB context. + * @param group The group to set the config for. + * @param config Set to the retrieved config. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_config( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + tiledb_config_t** config) TILEDB_NOEXCEPT; + +/** + * It puts a metadata key-value item to an open group. The group must + * be opened in WRITE mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in WRITE mode. + * @param key The key of the metadata item to be added. UTF-8 encodings + * are acceptable. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. + * @param value The metadata value in binary form. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note The writes will take effect only upon closing the group. + */ +TILEDB_EXPORT capi_return_t tiledb_group_put_metadata( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* key, + tiledb_datatype_t value_type, + uint32_t value_num, + const void* value) TILEDB_NOEXCEPT; + +/** + * Deletes written data from an open group. The group must + * be opened in MODIFY_EXCLSUIVE mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in MODIFY_EXCLUSIVE mode. + * @param uri The address of the group item to be deleted. + * @param recursive True if all data inside the group is to be deleted. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note if recursive == false, data added to the group will be left as-is. + */ +TILEDB_EXPORT int32_t tiledb_group_delete_group( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* uri, + const uint8_t recursive) TILEDB_NOEXCEPT; + +/** + * Deletes a metadata key-value item from an open group. The group must + * be opened in WRITE mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in WRITE mode. + * @param key The key of the metadata item to be deleted. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note The writes will take effect only upon closing the group. + * + * @note If the key does not exist, this will take no effect + * (i.e., the function will not error out). + */ +TILEDB_EXPORT capi_return_t tiledb_group_delete_metadata( + tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key) TILEDB_NOEXCEPT; + +/** + * It gets a metadata key-value item from an open group. The group must + * be opened in READ mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param key The key of the metadata item to be retrieved. UTF-8 encodings + * are acceptable. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. Keys with empty values are indicated + * by value_num == 1 and value == NULL. + * @param value The metadata value in binary form. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note If the key does not exist, then `value` will be NULL. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_metadata( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* key, + tiledb_datatype_t* value_type, + uint32_t* value_num, + const void** value) TILEDB_NOEXCEPT; + +/** + * It gets then number of metadata items in an open group. The group must + * be opened in READ mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param num The number of metadata items to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_metadata_num( + tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* num) TILEDB_NOEXCEPT; + +/** + * It gets a metadata item from an open group using an index. + * The group must be opened in READ mode, otherwise the function will + * error out. + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param index The index used to get the metadata. + * @param key The metadata key. + * @param key_len The metadata key length. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. + * @param value The metadata value in binary form. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_metadata_from_index( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t index, + const char** key, + uint32_t* key_len, + tiledb_datatype_t* value_type, + uint32_t* value_num, + const void** value) TILEDB_NOEXCEPT; + +/** + * Checks whether a key exists in metadata from an open group. The group must + * be opened in READ mode, otherwise the function will error out. + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param key The key to be checked. UTF-8 encoding are acceptable. + * @param value_type The datatype of the value, if any. + * @param has_key Set to `1` if the metadata with given key exists, else `0`. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + * + * @note If the key does not exist, then `value` will be NULL. + */ +TILEDB_EXPORT capi_return_t tiledb_group_has_metadata_key( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* key, + tiledb_datatype_t* value_type, + int32_t* has_key) TILEDB_NOEXCEPT; + +/** + * Add a member to a group + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in WRITE mode. + * @param uri URI of member to add + * @param relative is the URI relative to the group + * @param name optional name group member can be given to be looked up by. + * Can be set to NULL. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_add_member( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* uri, + const uint8_t relative, + const char* name) TILEDB_NOEXCEPT; + +/** + * Remove a member from a group + * + * * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_remove_member(ctx, group, "s3://tiledb_bucket/my_array"); + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in WRITE mode. + * @param name_or_uri Name or URI of member to remove. If the URI is + * registered multiple times in the group, the name needs to be specified so + * that the correct one can be removed. Note that if a URI is registered as + * both a named and unnamed member, the unnamed member will be removed + * successfully using the URI. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_remove_member( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name_or_uri) TILEDB_NOEXCEPT; + +/** + * Get the count of members in a group + * + * * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * uint64_t count = 0; + * tiledb_group_get_member_count(ctx, group, &count); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param count number of members in group + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_member_count( + tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* count) TILEDB_NOEXCEPT; + +#ifndef TILEDB_REMOVE_DEPRECATIONS +/** + * Get a member of a group by index and details of group. + * Deprecated, use \p tiledb_group_get_member_by_index_v2 instead. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * char *uri; + * tiledb_object_t type; + * tiledb_group_get_member_by_index(ctx, group, 0, &uri, &type); + * + * free(uri); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param index index of member to fetch + * @param uri URI of member. + * @param type type of member + * @param name name of member. NULL if name was not set + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_index( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t index, + char** uri, + tiledb_object_t* type, + char** name) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS + +/** + * Get a member of a group by index and details of group + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tiledb_string_t *uri, *name; + * tiledb_object_t type; + * tiledb_group_get_member_by_index_v2(ctx, group, 0, &uri, &type, &name); + * + * tiledb_string_free(uri); + * tiledb_string_free(name); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param index index of member to fetch + * @param uri Handle to the URI of the member. + * @param type type of member + * @param name Handle to the name of the member. NULL if name was not set + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index_v2( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + uint64_t index, + tiledb_string_t** uri, + tiledb_object_t* type, + tiledb_string_t** name) TILEDB_NOEXCEPT; + +#ifndef TILEDB_REMOVE_DEPRECATIONS +/** + * Get a member of a group by name and details of group. + * Deprecated, use \p tiledb_group_get_member_by_name_v2. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", + * "group2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * char *uri; + * tiledb_object_t type; + * tiledb_group_get_member_by_name(ctx, group, "array1", &uri, &type); + * + * free(uri); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param name name of member to fetch + * @param uri URI of member + * @param type type of member + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_name( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name, + char** uri, + tiledb_object_t* type) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS + +/** + * Get a member of a group by name and details of group. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", + * "group2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tilledb_string_t *uri; + * tiledb_object_t type; + * tiledb_group_get_member_by_name(ctx, group, "array1", &uri, &type); + * + * tiledb_string_free(uri); + * + * @endcode + * + * @param ctx The TileDB context. + * @param group An group opened in READ mode. + * @param name name of member to fetch + * @param uri Handle to the URI of the member. + * @param type type of member + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_name_v2( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name, + tiledb_string_t** uri, + tiledb_object_t* type) TILEDB_NOEXCEPT; + +/* (clang format was butchering the tiledb_group_add_member() calls) */ +/* clang-format off */ +/** + * Get a member of a group by name and relative characteristic of that name + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_WRITE); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", true, + * "array1"); + * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", + * false, "group2"); + * + * tiledb_group_close(ctx, group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * uint8_t is_relative; + * tiledb_group_get_is_relative_uri_by_name(ctx, group, "array1", &is_relative); + * + * @endcode + * + * @param[in] ctx The TileDB context. + * @param[in] group An group opened in READ mode. + * @param[in] name name of member to fetch + * @param[out] is_relative to receive relative characteristic of named member + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +/* clang-format on */ +TILEDB_EXPORT capi_return_t tiledb_group_get_is_relative_uri_by_name( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char* name, + uint8_t* relative) TILEDB_NOEXCEPT; + +/** + * Checks if the group is open. + * + * @param ctx The TileDB context. + * @param group The group to be checked. + * @param is_open `1` if the group is open and `0` otherwise. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_is_open( + tiledb_ctx_t* ctx, tiledb_group_t* group, int32_t* is_open) TILEDB_NOEXCEPT; + +/** + * Retrieves the URI the group was opened with. It outputs an error + * if the group is not open. + * + * @param ctx The TileDB context. + * @param group The input group. + * @param group_uri The group URI to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_uri( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + const char** group_uri) TILEDB_NOEXCEPT; + +/** + * Retrieves the query type with which the group was opened. + * + * **Example:** + * + * @code{.c} + * tiledb_group_t* group; + * tiledb_group_alloc(ctx, "s3://tiledb_groups/my_group", &group); + * tiledb_group_open(ctx, group, TILEDB_READ); + * tiledb_query_type_t query_type; + * tiledb_group_get_type(ctx, group, &query_type); + * @endcode + * + * @param ctx The TileDB context. + * @param group The group. + * @param query_type The query type to be retrieved. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_get_query_type( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + tiledb_query_type_t* query_type) TILEDB_NOEXCEPT; + +/** + * Dump a string representation of a group + * + * @param ctx The TileDB context. + * @param group The group. + * @param dump_ascii The output string. The caller takes ownership + * of the c-string. + * @param recursive should we recurse into sub-groups + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_dump_str( + tiledb_ctx_t* ctx, + tiledb_group_t* group, + char** dump_ascii, + const uint8_t recursive) TILEDB_NOEXCEPT; + +/** + * Consolidates the group metadata into a single group metadata file. + * + * **Example:** + * + * @code{.c} + * tiledb_group_consolidate_metadata( + * ctx, "tiledb:///groups/mygroup", nullptr); + * @endcode + * + * @param ctx The TileDB context. + * @param group_uri The name of the TileDB group whose metadata will + * be consolidated. + * @param config Configuration parameters for the consolidation + * (`nullptr` means default, which will use the config from `ctx`). + * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_consolidate_metadata( + tiledb_ctx_t* ctx, + const char* group_uri, + tiledb_config_t* config) TILEDB_NOEXCEPT; + +/** + * Cleans up the group metadata + * Note that this will coarsen the granularity of time traveling (see docs + * for more information). + * + * **Example:** + * + * @code{.c} + * tiledb_group_vacuum_metadata( + * ctx, "tiledb:///groups/mygroup", nullptr); + * @endcode + * + * @param ctx The TileDB context. + * @param group_uri The name of the TileDB group to vacuum. + * @param config Configuration parameters for the vacuuming + * (`nullptr` means default, which will use the config from `ctx`). + * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. + */ +TILEDB_EXPORT capi_return_t tiledb_group_vacuum_metadata( + tiledb_ctx_t* ctx, + const char* group_uri, + tiledb_config_t* config) TILEDB_NOEXCEPT; + #ifdef __cplusplus } #endif diff --git a/tiledb/api/c_api/group/group_api_external_experimental.h b/tiledb/api/c_api/group/group_api_external_experimental.h deleted file mode 100644 index 469931ae5fef..000000000000 --- a/tiledb/api/c_api/group/group_api_external_experimental.h +++ /dev/null @@ -1,712 +0,0 @@ -/** - * @file tiledb/api/c_api/group/group_api_external.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declares the group C API for TileDB. - */ - -#ifndef TILEDB_CAPI_GROUP_API_EXTERNAL_EXPERIMENTAL_H -#define TILEDB_CAPI_GROUP_API_EXTERNAL_EXPERIMENTAL_H - -#include "../api_external_common.h" -#include "../datatype/datatype_api_external.h" -#include "../object/object_api_external.h" -#include "../query/query_api_external.h" -#include "group_api_external.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Creates a new TileDB group. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "my_group", &group); - * @endcode - * - * @param ctx The TileDB context. - * @param group_uri The group URI. - * @param group The TileDB group to be allocated - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_alloc( - tiledb_ctx_t* ctx, - const char* group_uri, - tiledb_group_t** group) TILEDB_NOEXCEPT; - -/** - * Destroys a TileDB group, freeing associated memory. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "my_group", &group); - * tiledb_group_free(&group); - * @endcode - * - * @param group The TileDB group to be freed - */ -TILEDB_EXPORT void tiledb_group_free(tiledb_group_t** group) TILEDB_NOEXCEPT; - -/** - * Opens a TileDB group. The group is opened using a query type as input. - * This is to indicate that queries created for this `tiledb_group_t` - * object will inherit the query type. In other words, `tiledb_group_t` - * objects are opened to receive only one type of queries. - * They can always be closed and be re-opened with another query type. - * Also there may be many different `tiledb_group_t` - * objects created and opened with different query types. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "hdfs:///tiledb_groups/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * @endcode - * - * @param ctx The TileDB context. - * @param group The group object to be opened. - * @param query_type The type of queries the group object will be receiving. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note If the same group object is opened again without being closed, - * an error will be set and TILEDB_ERR returned. - * @note The config should be set before opening an group. - * @note If the group is to be opened at a specfic time interval, the - * `timestamp{start, end}` values should be set to a config that's set to - * the group object before opening the group. - */ -TILEDB_EXPORT capi_return_t tiledb_group_open( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - tiledb_query_type_t query_type) TILEDB_NOEXCEPT; - -/** - * Closes a TileDB group. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "hdfs:///tiledb_groups/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * tiledb_group_close(ctx, group); - * @endcode - * - * @param ctx The TileDB context. - * @param group The group object to be closed. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note If the group object has already been closed, the function has - * no effect. - */ -TILEDB_EXPORT capi_return_t -tiledb_group_close(tiledb_ctx_t* ctx, tiledb_group_t* group) TILEDB_NOEXCEPT; - -/** - * Sets the group config. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * // Set the config for the given group. - * tiledb_config_t* config; - * tiledb_group_set_config(ctx, group, config); - * tiledb_group_open(ctx, group, TILEDB_READ); - * @endcode - * - * @param ctx The TileDB context. - * @param group The group to set the config for. - * @param config The config to be set. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @pre The config must be set on a closed group. - */ -TILEDB_EXPORT capi_return_t tiledb_group_set_config( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - tiledb_config_t* config) TILEDB_NOEXCEPT; - -/** - * Gets the group config. - * - * **Example:** - * - * @code{.c} - * // Retrieve the config for the given group. - * tiledb_config_t* config; - * tiledb_group_get_config(ctx, group, config); - * @endcode - * - * @param ctx The TileDB context. - * @param group The group to set the config for. - * @param config Set to the retrieved config. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_config( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - tiledb_config_t** config) TILEDB_NOEXCEPT; - -/** - * It puts a metadata key-value item to an open group. The group must - * be opened in WRITE mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in WRITE mode. - * @param key The key of the metadata item to be added. UTF-8 encodings - * are acceptable. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. - * @param value The metadata value in binary form. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note The writes will take effect only upon closing the group. - */ -TILEDB_EXPORT capi_return_t tiledb_group_put_metadata( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* key, - tiledb_datatype_t value_type, - uint32_t value_num, - const void* value) TILEDB_NOEXCEPT; - -/** - * Deletes written data from an open group. The group must - * be opened in MODIFY_EXCLSUIVE mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in MODIFY_EXCLUSIVE mode. - * @param uri The address of the group item to be deleted. - * @param recursive True if all data inside the group is to be deleted. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note if recursive == false, data added to the group will be left as-is. - */ -TILEDB_EXPORT int32_t tiledb_group_delete_group( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* uri, - const uint8_t recursive) TILEDB_NOEXCEPT; - -/** - * Deletes a metadata key-value item from an open group. The group must - * be opened in WRITE mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in WRITE mode. - * @param key The key of the metadata item to be deleted. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note The writes will take effect only upon closing the group. - * - * @note If the key does not exist, this will take no effect - * (i.e., the function will not error out). - */ -TILEDB_EXPORT capi_return_t tiledb_group_delete_metadata( - tiledb_ctx_t* ctx, tiledb_group_t* group, const char* key) TILEDB_NOEXCEPT; - -/** - * It gets a metadata key-value item from an open group. The group must - * be opened in READ mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param key The key of the metadata item to be retrieved. UTF-8 encodings - * are acceptable. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. Keys with empty values are indicated - * by value_num == 1 and value == NULL. - * @param value The metadata value in binary form. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note If the key does not exist, then `value` will be NULL. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_metadata( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* key, - tiledb_datatype_t* value_type, - uint32_t* value_num, - const void** value) TILEDB_NOEXCEPT; - -/** - * It gets then number of metadata items in an open group. The group must - * be opened in READ mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param num The number of metadata items to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_metadata_num( - tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* num) TILEDB_NOEXCEPT; - -/** - * It gets a metadata item from an open group using an index. - * The group must be opened in READ mode, otherwise the function will - * error out. - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param index The index used to get the metadata. - * @param key The metadata key. - * @param key_len The metadata key length. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. - * @param value The metadata value in binary form. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_metadata_from_index( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - uint64_t index, - const char** key, - uint32_t* key_len, - tiledb_datatype_t* value_type, - uint32_t* value_num, - const void** value) TILEDB_NOEXCEPT; - -/** - * Checks whether a key exists in metadata from an open group. The group must - * be opened in READ mode, otherwise the function will error out. - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param key The key to be checked. UTF-8 encoding are acceptable. - * @param value_type The datatype of the value, if any. - * @param has_key Set to `1` if the metadata with given key exists, else `0`. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - * - * @note If the key does not exist, then `value` will be NULL. - */ -TILEDB_EXPORT capi_return_t tiledb_group_has_metadata_key( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* key, - tiledb_datatype_t* value_type, - int32_t* has_key) TILEDB_NOEXCEPT; - -/** - * Add a member to a group - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in WRITE mode. - * @param uri URI of member to add - * @param relative is the URI relative to the group - * @param name optional name group member can be given to be looked up by. - * Can be set to NULL. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_add_member( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* uri, - const uint8_t relative, - const char* name) TILEDB_NOEXCEPT; - -/** - * Remove a member from a group - * - * * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_remove_member(ctx, group, "s3://tiledb_bucket/my_array"); - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in WRITE mode. - * @param name_or_uri Name or URI of member to remove. If the URI is - * registered multiple times in the group, the name needs to be specified so - * that the correct one can be removed. Note that if a URI is registered as - * both a named and unnamed member, the unnamed member will be removed - * successfully using the URI. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_remove_member( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* name_or_uri) TILEDB_NOEXCEPT; - -/** - * Get the count of members in a group - * - * * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * uint64_t count = 0; - * tiledb_group_get_member_count(ctx, group, &count); - * - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param count number of members in group - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_member_count( - tiledb_ctx_t* ctx, tiledb_group_t* group, uint64_t* count) TILEDB_NOEXCEPT; - -#ifndef TILEDB_REMOVE_DEPRECATIONS -/** - * Get a member of a group by index and details of group. - * Deprecated, use \p tiledb_group_get_member_by_index_v2 instead. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * char *uri; - * tiledb_object_t type; - * tiledb_group_get_member_by_index(ctx, group, 0, &uri, &type); - * - * free(uri); - * - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param index index of member to fetch - * @param uri URI of member. - * @param type type of member - * @param name name of member. NULL if name was not set - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_index( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - uint64_t index, - char** uri, - tiledb_object_t* type, - char** name) TILEDB_NOEXCEPT; -#endif // TILEDB_REMOVE_DEPRECATIONS - -/** - * Get a member of a group by index and details of group - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * tiledb_string_t *uri, *name; - * tiledb_object_t type; - * tiledb_group_get_member_by_index_v2(ctx, group, 0, &uri, &type, &name); - * - * tiledb_string_free(uri); - * tiledb_string_free(name); - * - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param index index of member to fetch - * @param uri Handle to the URI of the member. - * @param type type of member - * @param name Handle to the name of the member. NULL if name was not set - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_index_v2( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - uint64_t index, - tiledb_string_t** uri, - tiledb_object_t* type, - tiledb_string_t** name) TILEDB_NOEXCEPT; - -#ifndef TILEDB_REMOVE_DEPRECATIONS -/** - * Get a member of a group by name and details of group. - * Deprecated, use \p tiledb_group_get_member_by_name_v2. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", - * "group2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * char *uri; - * tiledb_object_t type; - * tiledb_group_get_member_by_name(ctx, group, "array1", &uri, &type); - * - * free(uri); - * - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param name name of member to fetch - * @param uri URI of member - * @param type type of member - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT capi_return_t tiledb_group_get_member_by_name( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* name, - char** uri, - tiledb_object_t* type) TILEDB_NOEXCEPT; -#endif // TILEDB_REMOVE_DEPRECATIONS - -/** - * Get a member of a group by name and details of group. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", "array1"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", - * "group2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * tilledb_string_t *uri; - * tiledb_object_t type; - * tiledb_group_get_member_by_name(ctx, group, "array1", &uri, &type); - * - * tiledb_string_free(uri); - * - * @endcode - * - * @param ctx The TileDB context. - * @param group An group opened in READ mode. - * @param name name of member to fetch - * @param uri Handle to the URI of the member. - * @param type type of member - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_member_by_name_v2( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* name, - tiledb_string_t** uri, - tiledb_object_t* type) TILEDB_NOEXCEPT; - -/* (clang format was butchering the tiledb_group_add_member() calls) */ -/* clang-format off */ -/** - * Get a member of a group by name and relative characteristic of that name - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_bucket/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_WRITE); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_array", true, - * "array1"); - * tiledb_group_add_member(ctx, group, "s3://tiledb_bucket/my_group_2", - * false, "group2"); - * - * tiledb_group_close(ctx, group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * uint8_t is_relative; - * tiledb_group_get_is_relative_uri_by_name(ctx, group, "array1", &is_relative); - * - * @endcode - * - * @param[in] ctx The TileDB context. - * @param[in] group An group opened in READ mode. - * @param[in] name name of member to fetch - * @param[out] is_relative to receive relative characteristic of named member - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -/* clang-format on */ -TILEDB_EXPORT capi_return_t tiledb_group_get_is_relative_uri_by_name( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char* name, - uint8_t* relative) TILEDB_NOEXCEPT; - -/** - * Checks if the group is open. - * - * @param ctx The TileDB context. - * @param group The group to be checked. - * @param is_open `1` if the group is open and `0` otherwise. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_is_open( - tiledb_ctx_t* ctx, tiledb_group_t* group, int32_t* is_open) TILEDB_NOEXCEPT; - -/** - * Retrieves the URI the group was opened with. It outputs an error - * if the group is not open. - * - * @param ctx The TileDB context. - * @param group The input group. - * @param group_uri The group URI to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_uri( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - const char** group_uri) TILEDB_NOEXCEPT; - -/** - * Retrieves the query type with which the group was opened. - * - * **Example:** - * - * @code{.c} - * tiledb_group_t* group; - * tiledb_group_alloc(ctx, "s3://tiledb_groups/my_group", &group); - * tiledb_group_open(ctx, group, TILEDB_READ); - * tiledb_query_type_t query_type; - * tiledb_group_get_type(ctx, group, &query_type); - * @endcode - * - * @param ctx The TileDB context. - * @param group The group. - * @param query_type The query type to be retrieved. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_get_query_type( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - tiledb_query_type_t* query_type) TILEDB_NOEXCEPT; - -/** - * Dump a string representation of a group - * - * @param ctx The TileDB context. - * @param group The group. - * @param dump_ascii The output string. The caller takes ownership - * of the c-string. - * @param recursive should we recurse into sub-groups - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_dump_str( - tiledb_ctx_t* ctx, - tiledb_group_t* group, - char** dump_ascii, - const uint8_t recursive) TILEDB_NOEXCEPT; - -/** - * Consolidates the group metadata into a single group metadata file. - * - * **Example:** - * - * @code{.c} - * tiledb_group_consolidate_metadata( - * ctx, "tiledb:///groups/mygroup", nullptr); - * @endcode - * - * @param ctx The TileDB context. - * @param group_uri The name of the TileDB group whose metadata will - * be consolidated. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config from `ctx`). - * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_consolidate_metadata( - tiledb_ctx_t* ctx, - const char* group_uri, - tiledb_config_t* config) TILEDB_NOEXCEPT; - -/** - * Cleans up the group metadata - * Note that this will coarsen the granularity of time traveling (see docs - * for more information). - * - * **Example:** - * - * @code{.c} - * tiledb_group_vacuum_metadata( - * ctx, "tiledb:///groups/mygroup", nullptr); - * @endcode - * - * @param ctx The TileDB context. - * @param group_uri The name of the TileDB group to vacuum. - * @param config Configuration parameters for the vacuuming - * (`nullptr` means default, which will use the config from `ctx`). - * @return `TILEDB_OK` on success, and `TILEDB_ERR` on error. - */ -TILEDB_EXPORT capi_return_t tiledb_group_vacuum_metadata( - tiledb_ctx_t* ctx, - const char* group_uri, - tiledb_config_t* config) TILEDB_NOEXCEPT; - -#ifdef __cplusplus -} -#endif - -#endif // TILEDB_CAPI_DATATYPE_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/sm/c_api/tiledb.h b/tiledb/sm/c_api/tiledb.h index b7f9e382fffa..0b2ca9f28290 100644 --- a/tiledb/sm/c_api/tiledb.h +++ b/tiledb/sm/c_api/tiledb.h @@ -2757,6 +2757,64 @@ TILEDB_EXPORT int32_t tiledb_array_create( const char* array_uri, const tiledb_array_schema_t* array_schema) TILEDB_NOEXCEPT; +/** + * Deletes all written array data. + * + * **Example:** + * + * @code{.c} + * tiledb_array_delete(ctx, "hdfs:///temp/my_array"); + * @endcode + * + * @param ctx The TileDB context. + * @param uri The Array's URI. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) + TILEDB_NOEXCEPT; + +#ifndef TILEDB_REMOVE_DEPRECATIONS +/** + * Note: This API is deprecated and replaced with tiledb_array_delete (above). + * + * Deletes all written array data. + * + * **Example:** + * + * @code{.c} + * tiledb_array_delete_array(ctx, array, "hdfs:///temp/my_array"); + * @endcode + * + * @param ctx The TileDB context. + * @param array The array to delete the data from. + * @param uri The Array's URI. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_delete_array( + tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri) TILEDB_NOEXCEPT; +#endif // TILEDB_REMOVE_DEPRECATIONS + +/** + * Upgrades an array to the latest format version. + * + * **Example:** + * + * @code{.c} + * const char* array_uri="test_array"; + * tiledb_array_upgrade_version(ctx, array_uri); + * @endcode + * + * @param ctx The TileDB context. + * @param array_uri The uri of the array. + * @param config Configuration parameters for the upgrade + * (`nullptr` means default, which will use the config from `ctx`). + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_upgrade_version( + tiledb_ctx_t* ctx, + const char* array_uri, + tiledb_config_t* config) TILEDB_NOEXCEPT; + /** * Depending on the consoliation mode in the config, consolidates either the * fragment files, fragment metadata files, or array metadata files into a diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 399eb9dcfe0b..56bd92901856 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -43,7 +43,6 @@ */ #include "tiledb/api/c_api/attribute/attribute_api_external_experimental.h" #include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" -#include "tiledb/api/c_api/group/group_api_external_experimental.h" #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" @@ -359,43 +358,6 @@ TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( /* ARRAY */ /* ********************************* */ -/** - * Deletes all written array data. - * - * **Example:** - * - * @code{.c} - * tiledb_array_delete(ctx, "hdfs:///temp/my_array"); - * @endcode - * - * @param ctx The TileDB context. - * @param uri The Array's URI. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) - TILEDB_NOEXCEPT; - -#ifndef TILEDB_REMOVE_DEPRECATIONS -/** - * Note: This API is deprecated and replaced with tiledb_array_delete (above). - * - * Deletes all written array data. - * - * **Example:** - * - * @code{.c} - * tiledb_array_delete_array(ctx, array, "hdfs:///temp/my_array"); - * @endcode - * - * @param ctx The TileDB context. - * @param array The array to delete the data from. - * @param uri The Array's URI. - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_DEPRECATED_EXPORT int32_t tiledb_array_delete_array( - tiledb_ctx_t* ctx, tiledb_array_t* array, const char* uri) TILEDB_NOEXCEPT; -#endif // TILEDB_REMOVE_DEPRECATIONS - /** * Evolve array schema of an array. * @@ -460,27 +422,6 @@ TILEDB_EXPORT capi_return_t tiledb_array_get_enumeration( TILEDB_EXPORT capi_return_t tiledb_array_load_all_enumerations( tiledb_ctx_t* ctx, const tiledb_array_t* array) TILEDB_NOEXCEPT; -/** - * Upgrades an array to the latest format version. - * - * **Example:** - * - * @code{.c} - * const char* array_uri="test_array"; - * tiledb_array_upgrade_version(ctx, array_uri); - * @endcode - * - * @param ctx The TileDB context. - * @param array_uri The uri of the array. - * @param config Configuration parameters for the upgrade - * (`nullptr` means default, which will use the config from `ctx`). - * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. - */ -TILEDB_EXPORT int32_t tiledb_array_upgrade_version( - tiledb_ctx_t* ctx, - const char* array_uri, - tiledb_config_t* config) TILEDB_NOEXCEPT; - /* ********************************* */ /* QUERY */ /* ********************************* */ diff --git a/tiledb/sm/cpp_api/group.h b/tiledb/sm/cpp_api/group.h index 448001488f02..0d1e544e402e 100644 --- a/tiledb/sm/cpp_api/group.h +++ b/tiledb/sm/cpp_api/group.h @@ -40,6 +40,517 @@ namespace tiledb { +class Group { + public: + /** + * @brief Constructor. Opens the group for the given query type. The + * destructor calls the `close()` method. + * + * **Example:** + * + * @code{.cpp} + * // Open the group for reading + * tiledb::Context ctx; + * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ); + * @endcode + * + * @param ctx TileDB context. + * @param group_uri The group URI. + * @param query_type Query type to open the group for. + */ + Group( + const Context& ctx, + const std::string& group_uri, + tiledb_query_type_t query_type) + : Group(ctx, group_uri, query_type, nullptr) { + } + + /** + * @brief Constructor. Sets a config to the group and opens it for the given + * query type. The destructor calls the `close()` method. + * + * **Example:** + * + * @code{.cpp} + * // Open the group for reading + * tiledb::Context ctx; + * tiledb::Config cfg; + * cfg["rest.username"] = "user"; + * cfg["rest.password"] = "pass"; + * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ, cfg); + * @endcode + * + * @param ctx TileDB context. + * @param group_uri The group URI. + * @param query_type Query type to open the group for. + * @param config COnfiguration parameters + */ + Group( + const Context& ctx, + const std::string& group_uri, + tiledb_query_type_t query_type, + const Config& config) + : Group(ctx, group_uri, query_type, config.ptr().get()) { + } + + Group(const Group&) = default; + Group(Group&&) = default; + Group& operator=(const Group&) = default; + Group& operator=(Group&&) = default; + + /** Destructor; calls `close()`. */ + ~Group() { + if (owns_c_ptr_ && is_open()) { + close(false); + } + } + + /** + * @brief Opens the group using a query type as input. + * + * This is to indicate that queries created for this `Group` + * object will inherit the query type. In other words, `Group` + * objects are opened to receive only one type of queries. + * They can always be closed and be re-opened with another query type. + * Also there may be many different `Group` + * objects created and opened with different query types. For + * instance, one may create and open an group object `group_read` for + * reads and another one `group_write` for writes, and interleave + * creation and submission of queries for both these group objects. + * + * **Example:** + * @code{.cpp} + * // Open the group for writing + * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_WRITE); + * // Close and open again for reading. + * group.close(); + * group.open(TILEDB_READ); + * @endcode + * + * @param query_type The type of queries the group object will be receiving. + * @throws TileDBError if the group is already open or other error occurred. + */ + void open(tiledb_query_type_t query_type) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_group_open(c_ctx, group_.get(), query_type)); + } + + /** + * Sets the group config. + * + * @pre The group must be closed. + */ + void set_config(const Config& config) const { + auto& ctx = ctx_.get(); + ctx.handle_error(tiledb_group_set_config( + ctx.ptr().get(), group_.get(), config.ptr().get())); + } + + /** Retrieves the group config. */ + Config config() const { + auto& ctx = ctx_.get(); + tiledb_config_t* config = nullptr; + ctx.handle_error( + tiledb_group_get_config(ctx.ptr().get(), group_.get(), &config)); + return Config(&config); + } + + /** + * Closes the group. This must be called directly if you wish to check that + * any changes to the group were committed. This is automatically called + * by the destructor but any errors encountered are logged instead of throwing + * an exception from a destructor. + * + * **Example:** + * @code{.cpp} + * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ); + * group.close(); + * @endcode + */ + void close(bool should_throw = true) { + auto& ctx = ctx_.get(); + auto rc = tiledb_group_close(ctx.ptr().get(), group_.get()); + if (rc != TILEDB_OK && should_throw) { + ctx.handle_error(rc); + } else if (rc != TILEDB_OK) { + auto msg = ctx.get_last_error_message(); + tiledb_log_warn(ctx.ptr().get(), msg.c_str()); + } + } + + /** + * + * Create a TileDB Group + * + * * **Example:** + * @code{.cpp} + * tiledb::Group::create(ctx, "s3://bucket-name/group-name"); + * @endcode + * + * @param ctx tiledb context + * @param uri URI where group will be created. + */ + // clang-format on + static void create(const tiledb::Context& ctx, const std::string& uri) { + ctx.handle_error(tiledb_group_create(ctx.ptr().get(), uri.c_str())); + } + + /** Checks if the group is open. */ + bool is_open() const { + auto& ctx = ctx_.get(); + int open = 0; + ctx.handle_error( + tiledb_group_is_open(ctx.ptr().get(), group_.get(), &open)); + return bool(open); + } + + /** Returns the group URI. */ + std::string uri() const { + auto& ctx = ctx_.get(); + const char* uri = nullptr; + ctx.handle_error(tiledb_group_get_uri(ctx.ptr().get(), group_.get(), &uri)); + return std::string(uri); + } + + /** Returns the query type the group was opened with. */ + tiledb_query_type_t query_type() const { + auto& ctx = ctx_.get(); + tiledb_query_type_t query_type; + ctx.handle_error(tiledb_group_get_query_type( + ctx.ptr().get(), group_.get(), &query_type)); + return query_type; + } + + /** + * Puts a metadata key-value item to an open group. The group must + * be opened in WRITE mode, otherwise the function will error out. + * + * @param key The key of the metadata item to be added. UTF-8 encodings + * are acceptable. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. + * @param value The metadata value in binary form. + * + * @note The writes will take effect only upon closing the group. + */ + void put_metadata( + const std::string& key, + tiledb_datatype_t value_type, + uint32_t value_num, + const void* value) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_group_put_metadata( + c_ctx, group_.get(), key.c_str(), value_type, value_num, value)); + } + + /** + * Deletes all written data from an open group. The group must + * be opened in MODIFY_EXCLUSIVE mode, otherwise the function will error out. + * + * @param uri The address of the group item to be deleted. + * @param recursive True if all data inside the group is to be deleted. + * + * @note if recursive == false, data added to the group will be left as-is. + * @post This is destructive; the group may not be reopened after delete. + */ + void delete_group(const std::string& uri, bool recursive = false) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error( + tiledb_group_delete_group(c_ctx, group_.get(), uri.c_str(), recursive)); + } + + /** + * Deletes a metadata key-value item from an open group. The group must + * be opened in WRITE mode, otherwise the function will error out. + * + * @param key The key of the metadata item to be deleted. + * + * @note The writes will take effect only upon closing the group. + * + * @note If the key does not exist, this will take no effect + * (i.e., the function will not error out). + */ + void delete_metadata(const std::string& key) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error( + tiledb_group_delete_metadata(c_ctx, group_.get(), key.c_str())); + } + + /** + * Gets a metadata key-value item from an open group. The group must + * be opened in READ mode, otherwise the function will error out. + * + * @param key The key of the metadata item to be retrieved. UTF-8 encodings + * are acceptable. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. Keys with empty values are indicated + * by value_num == 1 and value == NULL. + * @param value The metadata value in binary form. + * + * @note If the key does not exist, then `value` will be NULL. + */ + void get_metadata( + const std::string& key, + tiledb_datatype_t* value_type, + uint32_t* value_num, + const void** value) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_group_get_metadata( + c_ctx, group_.get(), key.c_str(), value_type, value_num, value)); + } + + /** + * Checks if key exists in metadata from an open group. The group must + * be opened in READ mode, otherwise the function will error out. + * + * @param key The key of the metadata item to be retrieved. UTF-8 encodings + * are acceptable. + * @param value_type The datatype of the value associated with the key (if + * any). + * @return true if the key exists, else false. + * @note If the key does not exist, then `value_type` will not be modified. + */ + bool has_metadata(const std::string& key, tiledb_datatype_t* value_type) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + int32_t has_key; + ctx.handle_error(tiledb_group_has_metadata_key( + c_ctx, group_.get(), key.c_str(), value_type, &has_key)); + return has_key == 1; + } + + /** + * Returns then number of metadata items in an open group. The group must + * be opened in READ mode, otherwise the function will error out. + */ + uint64_t metadata_num() const { + uint64_t num; + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_group_get_metadata_num(c_ctx, group_.get(), &num)); + return num; + } + + /** + * Gets a metadata item from an open group using an index. + * The group must be opened in READ mode, otherwise the function will + * error out. + * + * @param index The index used to get the metadata. + * @param key The metadata key. + * @param value_type The datatype of the value. + * @param value_num The value may consist of more than one items of the + * same datatype. This argument indicates the number of items in the + * value component of the metadata. Keys with empty values are indicated + * by value_num == 1 and value == NULL. + * @param value The metadata value in binary form. + */ + void get_metadata_from_index( + uint64_t index, + std::string* key, + tiledb_datatype_t* value_type, + uint32_t* value_num, + const void** value) { + const char* key_c; + uint32_t key_len; + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_group_get_metadata_from_index( + c_ctx, + group_.get(), + index, + &key_c, + &key_len, + value_type, + value_num, + value)); + key->resize(key_len); + std::memcpy((void*)key->data(), key_c, key_len); + } + + /** + * + * Add a member to a group + * + * @param uri of member to add + * @param relative is the URI relative to the group location + */ + void add_member( + const std::string& uri, + const bool& relative, + std::optional name = std::nullopt) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + const char* name_cstr = nullptr; + if (name.has_value()) { + name_cstr = name->c_str(); + } + ctx.handle_error(tiledb_group_add_member( + c_ctx, group_.get(), uri.c_str(), relative, name_cstr)); + } + + /** + * Remove a member from a group + * + * @param name_or_uri Name or URI of member to remove. If the URI is + * registered multiple times in the group, the name needs to be specified so + * that the correct one can be removed. Note that if a URI is registered as + * both a named and unnamed member, the unnamed member will be removed + * successfully using the URI. + */ + void remove_member(const std::string& name_or_uri) { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error( + tiledb_group_remove_member(c_ctx, group_.get(), name_or_uri.c_str())); + } + + uint64_t member_count() const { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + uint64_t count = 0; + ctx.handle_error( + tiledb_group_get_member_count(c_ctx, group_.get(), &count)); + return count; + } + + tiledb::Object member(uint64_t index) const { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_string_t* uri; + tiledb_object_t type; + tiledb_string_t* name; + ctx.handle_error(tiledb_group_get_member_by_index_v2( + c_ctx, group_.get(), index, &uri, &type, &name)); + return tiledb::Object( + type, + impl::convert_to_string(&uri).value(), + impl::convert_to_string(&name)); + } + + tiledb::Object member(std::string name) const { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_string_t* uri; + tiledb_object_t type; + ctx.handle_error(tiledb_group_get_member_by_name_v2( + c_ctx, group_.get(), name.c_str(), &uri, &type)); + return tiledb::Object(type, impl::convert_to_string(&uri).value(), name); + } + + /** + * retrieve the relative attribute for a named member + * + * @param name of member to retrieve associated relative indicator. + */ + bool is_relative(std::string name) const { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + uint8_t is_relative; + ctx.handle_error(tiledb_group_get_is_relative_uri_by_name( + c_ctx, group_.get(), name.c_str(), &is_relative)); + return is_relative != 0; + } + + std::string dump(const bool recursive) const { + auto& ctx = ctx_.get(); + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + char* str; + ctx.handle_error( + tiledb_group_dump_str(c_ctx, group_.get(), &str, recursive)); + + std::string ret(str); + free(str); + return ret; + } + + /** + * Consolidates the group metadata into a single group metadata file. + * + * **Example:** + * @code{.cpp} + * tiledb::Group::consolidate_metadata(ctx, "s3://bucket-name/group-name"); + * @endcode + * + * @param ctx TileDB context + * @param uri The URI of the TileDB group to be consolidated. + * @param config Configuration parameters for the consolidation. + */ + static void consolidate_metadata( + const Context& ctx, + const std::string& uri, + Config* const config = nullptr) { + ctx.handle_error(tiledb_group_consolidate_metadata( + ctx.ptr().get(), uri.c_str(), config ? config->ptr().get() : nullptr)); + } + + /** + * Cleans up the group metadata. + * + * **Example:** + * @code{.cpp} + * tiledb::Group::vacuum_metadata(ctx, "s3://bucket-name/group-name"); + * @endcode + * + * @param ctx TileDB context + * @param uri The URI of the TileDB group to vacuum. + * @param config Configuration parameters for the vacuuming. + */ + static void vacuum_metadata( + const Context& ctx, + const std::string& uri, + Config* const config = nullptr) { + ctx.handle_error(tiledb_group_vacuum_metadata( + ctx.ptr().get(), uri.c_str(), config ? config->ptr().get() : nullptr)); + } + + private: + Group( + const Context& ctx, + const std::string& group_uri, + tiledb_query_type_t query_type, + tiledb_config_t* config) + : ctx_(ctx) { + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + tiledb_group_t* group; + ctx.handle_error(tiledb_group_alloc(c_ctx, group_uri.c_str(), &group)); + group_ = std::shared_ptr(group, deleter_); + if (config) { + ctx.handle_error(tiledb_group_set_config(c_ctx, group, config)); + } + ctx.handle_error(tiledb_group_open(c_ctx, group, query_type)); + } + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Flag indicating ownership of the TileDB C group object */ + bool owns_c_ptr_ = true; + + /** Pointer to the TileDB C group object. */ + std::shared_ptr group_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ +}; + /** * Creates a new group. A Group is a logical grouping of Objects on the * storage system (a directory). diff --git a/tiledb/sm/cpp_api/group_experimental.h b/tiledb/sm/cpp_api/group_experimental.h deleted file mode 100644 index f7bb3aa5601e..000000000000 --- a/tiledb/sm/cpp_api/group_experimental.h +++ /dev/null @@ -1,557 +0,0 @@ -/** - * @file group_experimental.h - * - * @author Ravi Gaddipati - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declares the C++ API for the TileDB groups. - */ - -#ifndef TILEDB_CPP_API_GROUP_EXPERIMENTAL_H -#define TILEDB_CPP_API_GROUP_EXPERIMENTAL_H - -#include "capi_string.h" -#include "context.h" -#include "object.h" -#include "tiledb.h" - -namespace tiledb { -class Group { - public: - /** - * @brief Constructor. Opens the group for the given query type. The - * destructor calls the `close()` method. - * - * **Example:** - * - * @code{.cpp} - * // Open the group for reading - * tiledb::Context ctx; - * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ); - * @endcode - * - * @param ctx TileDB context. - * @param group_uri The group URI. - * @param query_type Query type to open the group for. - */ - Group( - const Context& ctx, - const std::string& group_uri, - tiledb_query_type_t query_type) - : Group(ctx, group_uri, query_type, nullptr) { - } - - /** - * @brief Constructor. Sets a config to the group and opens it for the given - * query type. The destructor calls the `close()` method. - * - * **Example:** - * - * @code{.cpp} - * // Open the group for reading - * tiledb::Context ctx; - * tiledb::Config cfg; - * cfg["rest.username"] = "user"; - * cfg["rest.password"] = "pass"; - * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ, cfg); - * @endcode - * - * @param ctx TileDB context. - * @param group_uri The group URI. - * @param query_type Query type to open the group for. - * @param config COnfiguration parameters - */ - Group( - const Context& ctx, - const std::string& group_uri, - tiledb_query_type_t query_type, - const Config& config) - : Group(ctx, group_uri, query_type, config.ptr().get()) { - } - - Group(const Group&) = default; - Group(Group&&) = default; - Group& operator=(const Group&) = default; - Group& operator=(Group&&) = default; - - /** Destructor; calls `close()`. */ - ~Group() { - if (owns_c_ptr_ && is_open()) { - close(false); - } - } - - /** - * @brief Opens the group using a query type as input. - * - * This is to indicate that queries created for this `Group` - * object will inherit the query type. In other words, `Group` - * objects are opened to receive only one type of queries. - * They can always be closed and be re-opened with another query type. - * Also there may be many different `Group` - * objects created and opened with different query types. For - * instance, one may create and open an group object `group_read` for - * reads and another one `group_write` for writes, and interleave - * creation and submission of queries for both these group objects. - * - * **Example:** - * @code{.cpp} - * // Open the group for writing - * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_WRITE); - * // Close and open again for reading. - * group.close(); - * group.open(TILEDB_READ); - * @endcode - * - * @param query_type The type of queries the group object will be receiving. - * @throws TileDBError if the group is already open or other error occurred. - */ - void open(tiledb_query_type_t query_type) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_group_open(c_ctx, group_.get(), query_type)); - } - - /** - * Sets the group config. - * - * @pre The group must be closed. - */ - void set_config(const Config& config) const { - auto& ctx = ctx_.get(); - ctx.handle_error(tiledb_group_set_config( - ctx.ptr().get(), group_.get(), config.ptr().get())); - } - - /** Retrieves the group config. */ - Config config() const { - auto& ctx = ctx_.get(); - tiledb_config_t* config = nullptr; - ctx.handle_error( - tiledb_group_get_config(ctx.ptr().get(), group_.get(), &config)); - return Config(&config); - } - - /** - * Closes the group. This must be called directly if you wish to check that - * any changes to the group were committed. This is automatically called - * by the destructor but any errors encountered are logged instead of throwing - * an exception from a destructor. - * - * **Example:** - * @code{.cpp} - * tiledb::Group group(ctx, "s3://bucket-name/group-name", TILEDB_READ); - * group.close(); - * @endcode - */ - void close(bool should_throw = true) { - auto& ctx = ctx_.get(); - auto rc = tiledb_group_close(ctx.ptr().get(), group_.get()); - if (rc != TILEDB_OK && should_throw) { - ctx.handle_error(rc); - } else if (rc != TILEDB_OK) { - auto msg = ctx.get_last_error_message(); - tiledb_log_warn(ctx.ptr().get(), msg.c_str()); - } - } - - /** - * - * Create a TileDB Group - * - * * **Example:** - * @code{.cpp} - * tiledb::Group::create(ctx, "s3://bucket-name/group-name"); - * @endcode - * - * @param ctx tiledb context - * @param uri URI where group will be created. - */ - // clang-format on - static void create(const tiledb::Context& ctx, const std::string& uri) { - ctx.handle_error(tiledb_group_create(ctx.ptr().get(), uri.c_str())); - } - - /** Checks if the group is open. */ - bool is_open() const { - auto& ctx = ctx_.get(); - int open = 0; - ctx.handle_error( - tiledb_group_is_open(ctx.ptr().get(), group_.get(), &open)); - return bool(open); - } - - /** Returns the group URI. */ - std::string uri() const { - auto& ctx = ctx_.get(); - const char* uri = nullptr; - ctx.handle_error(tiledb_group_get_uri(ctx.ptr().get(), group_.get(), &uri)); - return std::string(uri); - } - - /** Returns the query type the group was opened with. */ - tiledb_query_type_t query_type() const { - auto& ctx = ctx_.get(); - tiledb_query_type_t query_type; - ctx.handle_error(tiledb_group_get_query_type( - ctx.ptr().get(), group_.get(), &query_type)); - return query_type; - } - - /** - * Puts a metadata key-value item to an open group. The group must - * be opened in WRITE mode, otherwise the function will error out. - * - * @param key The key of the metadata item to be added. UTF-8 encodings - * are acceptable. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. - * @param value The metadata value in binary form. - * - * @note The writes will take effect only upon closing the group. - */ - void put_metadata( - const std::string& key, - tiledb_datatype_t value_type, - uint32_t value_num, - const void* value) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_group_put_metadata( - c_ctx, group_.get(), key.c_str(), value_type, value_num, value)); - } - - /** - * Deletes all written data from an open group. The group must - * be opened in MODIFY_EXCLUSIVE mode, otherwise the function will error out. - * - * @param uri The address of the group item to be deleted. - * @param recursive True if all data inside the group is to be deleted. - * - * @note if recursive == false, data added to the group will be left as-is. - * @post This is destructive; the group may not be reopened after delete. - */ - void delete_group(const std::string& uri, bool recursive = false) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error( - tiledb_group_delete_group(c_ctx, group_.get(), uri.c_str(), recursive)); - } - - /** - * Deletes a metadata key-value item from an open group. The group must - * be opened in WRITE mode, otherwise the function will error out. - * - * @param key The key of the metadata item to be deleted. - * - * @note The writes will take effect only upon closing the group. - * - * @note If the key does not exist, this will take no effect - * (i.e., the function will not error out). - */ - void delete_metadata(const std::string& key) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error( - tiledb_group_delete_metadata(c_ctx, group_.get(), key.c_str())); - } - - /** - * Gets a metadata key-value item from an open group. The group must - * be opened in READ mode, otherwise the function will error out. - * - * @param key The key of the metadata item to be retrieved. UTF-8 encodings - * are acceptable. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. Keys with empty values are indicated - * by value_num == 1 and value == NULL. - * @param value The metadata value in binary form. - * - * @note If the key does not exist, then `value` will be NULL. - */ - void get_metadata( - const std::string& key, - tiledb_datatype_t* value_type, - uint32_t* value_num, - const void** value) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_group_get_metadata( - c_ctx, group_.get(), key.c_str(), value_type, value_num, value)); - } - - /** - * Checks if key exists in metadata from an open group. The group must - * be opened in READ mode, otherwise the function will error out. - * - * @param key The key of the metadata item to be retrieved. UTF-8 encodings - * are acceptable. - * @param value_type The datatype of the value associated with the key (if - * any). - * @return true if the key exists, else false. - * @note If the key does not exist, then `value_type` will not be modified. - */ - bool has_metadata(const std::string& key, tiledb_datatype_t* value_type) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - int32_t has_key; - ctx.handle_error(tiledb_group_has_metadata_key( - c_ctx, group_.get(), key.c_str(), value_type, &has_key)); - return has_key == 1; - } - - /** - * Returns then number of metadata items in an open group. The group must - * be opened in READ mode, otherwise the function will error out. - */ - uint64_t metadata_num() const { - uint64_t num; - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_group_get_metadata_num(c_ctx, group_.get(), &num)); - return num; - } - - /** - * Gets a metadata item from an open group using an index. - * The group must be opened in READ mode, otherwise the function will - * error out. - * - * @param index The index used to get the metadata. - * @param key The metadata key. - * @param value_type The datatype of the value. - * @param value_num The value may consist of more than one items of the - * same datatype. This argument indicates the number of items in the - * value component of the metadata. Keys with empty values are indicated - * by value_num == 1 and value == NULL. - * @param value The metadata value in binary form. - */ - void get_metadata_from_index( - uint64_t index, - std::string* key, - tiledb_datatype_t* value_type, - uint32_t* value_num, - const void** value) { - const char* key_c; - uint32_t key_len; - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error(tiledb_group_get_metadata_from_index( - c_ctx, - group_.get(), - index, - &key_c, - &key_len, - value_type, - value_num, - value)); - key->resize(key_len); - std::memcpy((void*)key->data(), key_c, key_len); - } - - /** - * - * Add a member to a group - * - * @param uri of member to add - * @param relative is the URI relative to the group location - */ - void add_member( - const std::string& uri, - const bool& relative, - std::optional name = std::nullopt) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - const char* name_cstr = nullptr; - if (name.has_value()) { - name_cstr = name->c_str(); - } - ctx.handle_error(tiledb_group_add_member( - c_ctx, group_.get(), uri.c_str(), relative, name_cstr)); - } - - /** - * Remove a member from a group - * - * @param name_or_uri Name or URI of member to remove. If the URI is - * registered multiple times in the group, the name needs to be specified so - * that the correct one can be removed. Note that if a URI is registered as - * both a named and unnamed member, the unnamed member will be removed - * successfully using the URI. - */ - void remove_member(const std::string& name_or_uri) { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - ctx.handle_error( - tiledb_group_remove_member(c_ctx, group_.get(), name_or_uri.c_str())); - } - - uint64_t member_count() const { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - uint64_t count = 0; - ctx.handle_error( - tiledb_group_get_member_count(c_ctx, group_.get(), &count)); - return count; - } - - tiledb::Object member(uint64_t index) const { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - tiledb_string_t* uri; - tiledb_object_t type; - tiledb_string_t* name; - ctx.handle_error(tiledb_group_get_member_by_index_v2( - c_ctx, group_.get(), index, &uri, &type, &name)); - return tiledb::Object( - type, - impl::convert_to_string(&uri).value(), - impl::convert_to_string(&name)); - } - - tiledb::Object member(std::string name) const { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - tiledb_string_t* uri; - tiledb_object_t type; - ctx.handle_error(tiledb_group_get_member_by_name_v2( - c_ctx, group_.get(), name.c_str(), &uri, &type)); - return tiledb::Object(type, impl::convert_to_string(&uri).value(), name); - } - - /** - * retrieve the relative attribute for a named member - * - * @param name of member to retrieve associated relative indicator. - */ - bool is_relative(std::string name) const { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - uint8_t is_relative; - ctx.handle_error(tiledb_group_get_is_relative_uri_by_name( - c_ctx, group_.get(), name.c_str(), &is_relative)); - return is_relative != 0; - } - - std::string dump(const bool recursive) const { - auto& ctx = ctx_.get(); - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - char* str; - ctx.handle_error( - tiledb_group_dump_str(c_ctx, group_.get(), &str, recursive)); - - std::string ret(str); - free(str); - return ret; - } - - /** - * Consolidates the group metadata into a single group metadata file. - * - * **Example:** - * @code{.cpp} - * tiledb::Group::consolidate_metadata(ctx, "s3://bucket-name/group-name"); - * @endcode - * - * @param ctx TileDB context - * @param uri The URI of the TileDB group to be consolidated. - * @param config Configuration parameters for the consolidation. - */ - static void consolidate_metadata( - const Context& ctx, - const std::string& uri, - Config* const config = nullptr) { - ctx.handle_error(tiledb_group_consolidate_metadata( - ctx.ptr().get(), uri.c_str(), config ? config->ptr().get() : nullptr)); - } - - /** - * Cleans up the group metadata. - * - * **Example:** - * @code{.cpp} - * tiledb::Group::vacuum_metadata(ctx, "s3://bucket-name/group-name"); - * @endcode - * - * @param ctx TileDB context - * @param uri The URI of the TileDB group to vacuum. - * @param config Configuration parameters for the vacuuming. - */ - static void vacuum_metadata( - const Context& ctx, - const std::string& uri, - Config* const config = nullptr) { - ctx.handle_error(tiledb_group_vacuum_metadata( - ctx.ptr().get(), uri.c_str(), config ? config->ptr().get() : nullptr)); - } - - private: - Group( - const Context& ctx, - const std::string& group_uri, - tiledb_query_type_t query_type, - tiledb_config_t* config) - : ctx_(ctx) { - tiledb_ctx_t* c_ctx = ctx.ptr().get(); - tiledb_group_t* group; - ctx.handle_error(tiledb_group_alloc(c_ctx, group_uri.c_str(), &group)); - group_ = std::shared_ptr(group, deleter_); - if (config) { - ctx.handle_error(tiledb_group_set_config(c_ctx, group, config)); - } - ctx.handle_error(tiledb_group_open(c_ctx, group, query_type)); - } - - /* ********************************* */ - /* PRIVATE ATTRIBUTES */ - /* ********************************* */ - - /** The TileDB context. */ - std::reference_wrapper ctx_; - - /** Deleter wrapper. */ - impl::Deleter deleter_; - - /** Flag indicating ownership of the TileDB C group object */ - bool owns_c_ptr_ = true; - - /** Pointer to the TileDB C group object. */ - std::shared_ptr group_; - - /* ********************************* */ - /* PRIVATE METHODS */ - /* ********************************* */ -}; - -} // namespace tiledb - -#endif // TILEDB_CPP_API_GROUP_EXPERIMENTAL_H diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index 4af4bbe940f9..63f208ff0056 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -41,7 +41,6 @@ #include "dimension_label_experimental.h" #include "enumeration_experimental.h" #include "consolidation_plan_experimental.h" -#include "group_experimental.h" #include "query_condition_experimental.h" #include "query_experimental.h" #include "subarray_experimental.h" From af175e808f77b5b6c3db19b44e628cb5a7e80718 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 2 May 2024 01:31:28 -0600 Subject: [PATCH 320/456] Fix the documentation for the `array_schema` object library (#4935) This change removes any confusion that in its present state the `array_schema` object library is not properly specified as a freestanding library capable of resolving all its symbols without additional external dependencies. [sc-46557] --- TYPE: NO_HISTORY DESC: Fix the documentation for the `array_schema` object library --- tiledb/sm/array_schema/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 602d9b9af264..0389401e9f56 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -74,10 +74,10 @@ commence(object_library array_schema) attribute domain enumeration fragment time uri_format vfs) conclude(object_library) -# This is linked outside the object_library scope because ContextResources -# is recompiled as part of the capi_context_stub. Including context_resources -# here like this prevents many headaches revolving around duplicate symbols -# when linking executables. +# This is linked outside the object_library scope as an active subversion of +# the principle that object libraries must compile as standalone modules. As +# long as this directive is necessary to have to the compile test pass, we +# cannot consider `array_schema` as a properly-constructed object library. target_link_libraries(compile_array_schema PRIVATE context_resources generic_tile_io) add_test_subdirectory() From d861e95970846254671b384856f4ee06b031696a Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 2 May 2024 00:56:12 -0700 Subject: [PATCH 321/456] Implement zip_view for external sort. (#4930) This PR implements a `zip_view` for zipping together a set of ranges. It is intended to implement the `std::ranges::zip_view` as defined for C++23. From https://en.cppreference.com/w/cpp/ranges/zip_view: 1. A zip_view is a range adaptor that takes one or more views, and produces a view whose ith element is a tuple-like value consisting of the ith elements of all views. The size of produced view is the minimum of sizes of all adapted views. 2. `zip()` is a customization point object that constructs a `zip_view` Currently, the `zip_view` only supports zipping together ranges that are `random_access_range`s. In addition, the `size()` and `end()` functions are only provided if all of the ranges are `sized_range`s The iterator from a `zip_view` is essentially a tuple of pointers to the beginning of each of the zipped ranges, plus an index that keeps track of the iterator's location in the zipped ranges. The size of a `zip_view` is the size of the smallest range. The end iterator of a zip view is the begin iterator plus the size of the zip_view. Unit tests have similar coverage to the tests for the var length views. Tests also include zipping a var length view and iterating through with `std::for_each` and `for`. [sc-43639] --- TYPE: IMPROVEMENT DESC: Implement zip_view for external sort. --- tiledb/common/test/CMakeLists.txt | 2 +- tiledb/common/test/unit_zip_view.cc | 292 ++++++++++++++++++++++++++++ tiledb/common/zip_view.h | 261 +++++++++++++++++++++++++ 3 files changed, 554 insertions(+), 1 deletion(-) create mode 100644 tiledb/common/test/unit_zip_view.cc create mode 100644 tiledb/common/zip_view.h diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index c5af2d47d80b..e6f7df42ed5c 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc) + this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc unit_zip_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_zip_view.cc b/tiledb/common/test/unit_zip_view.cc new file mode 100644 index 000000000000..0ca003509b5b --- /dev/null +++ b/tiledb/common/test/unit_zip_view.cc @@ -0,0 +1,292 @@ +/** + * @file unit_zip_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the zip_view class. + */ + +#include +#include +#include +#include "../zip_view.h" + +#include "../alt_var_length_view.h" + +TEST_CASE("zip_view: Null test", "[zip_view][null_test]") { + REQUIRE(true); +} + +TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { + using test_type = zip_view, std::vector>; + + CHECK(std::ranges::range); + CHECK(!std::ranges::borrowed_range); + CHECK(std::ranges::sized_range); + CHECK(std::ranges::input_range); + CHECK(!std::ranges:: + output_range>); + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + CHECK(!std::ranges::contiguous_range); + CHECK(std::ranges::common_range); + + // @todo Fix so that it passes on ubuntu. + // CHECK(std::ranges::viewable_range); + + // @todo: Should this be a view? + CHECK(!std::ranges::view); +} + +TEST_CASE("zip_view: Iterator concepts", "[zip_view][concepts]") { + using test_type = zip_view, std::vector>; + using test_type_iterator = std::ranges::iterator_t; + using test_type_const_iterator = std::ranges::iterator_t; + + CHECK(std::input_or_output_iterator); + CHECK(std::input_or_output_iterator); + CHECK(std::input_iterator); + CHECK(std::input_iterator); + CHECK(!std::output_iterator< + test_type_iterator, + std::ranges::range_value_t>); + CHECK(!std::output_iterator< + test_type_const_iterator, + std::ranges::range_value_t>); + CHECK(std::forward_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + CHECK(std::random_access_iterator); +} + +// Test that the zip_view value_type satisfies the expected concepts +TEST_CASE("zip_view: value_type concepts", "[zip_view][concepts]") { + using test_type = zip_view, std::vector>; + CHECK(std::ranges::range); + + using test_iterator_type = std::ranges::iterator_t; + using test_iterator_value_type = std::iter_value_t; + using test_iterator_reference_type = + std::iter_reference_t; + + using range_value_type = std::ranges::range_value_t; + using range_reference_type = std::ranges::range_reference_t; + + CHECK(std::is_same_v); + CHECK(std::is_same_v); +} + +TEST_CASE("zip_view: constructor", "[zip_view]") { + std::vector a{1, 2, 3}; + std::vector b{4, 5, 6}; + std::vector c{7, 8, 9}; + + SECTION("Zip one range") { + auto z = zip(a); + auto it = z.begin(); + CHECK(*it == std::tuple{1}); + ++it; + CHECK(*it == std::tuple{2}); + ++it; + CHECK(*it == std::tuple{3}); + it = z.begin(); + std::get<0>(*it) = 99; + CHECK(a[0] == 99); + } + + SECTION("Zip three ranges") { + auto z = zip(a, b, c); + auto it = z.begin(); + CHECK(*it == std::tuple{1, 4, 7}); + ++it; + CHECK(*it == std::tuple{2, 5, 8}); + ++it; + CHECK(*it == std::tuple{3, 6, 9}); + it = z.begin(); + std::get<0>(*it) = 41; + std::get<1>(*it) = 42; + std::get<2>(*it) = 43; + CHECK(a[0] == 41); + CHECK(b[0] == 42); + CHECK(c[0] == 43); + } +} + +TEST_CASE("zip_view: size()", "[zip_view]") { + std::vector a{1, 2, 3}; + std::vector b{4, 5, 6, 7, 8, 9}; + std::vector c{10, 11, 12, 13}; + + CHECK(zip(a).size() == 3); + CHECK(zip(b).size() == 6); + CHECK(zip(c).size() == 4); + CHECK(zip(a, b).size() == 3); + CHECK(zip(a, c).size() == 3); + CHECK(zip(b, c).size() == 4); + CHECK(zip(a, b, c).size() == 3); +} + +TEST_CASE("zip_view: end()", "[zip_view]") { + std::vector a{1, 2, 3}; + std::vector b{4, 5, 6, 7, 8, 9}; + std::vector c{10, 11, 12, 13}; + + [[maybe_unused]] auto x = zip(a).begin(); + [[maybe_unused]] auto y = zip(a).end(); + + CHECK(zip(a).end() == zip(a).begin() + 3); + CHECK(zip(b).end() == zip(b).begin() + 6); + CHECK(zip(c).end() == zip(c).begin() + 4); + CHECK(zip(a, b).end() == zip(a, b).begin() + 3); + CHECK(zip(a, c).end() == zip(a, c).begin() + 3); + CHECK(zip(b, c).end() == zip(b, c).begin() + 4); + CHECK(zip(a, b, c).end() == zip(a, b, c).begin() + 3); + + CHECK(zip(a).end() - zip(a).begin() == 3); + CHECK(zip(b).end() - zip(b).begin() == 6); + CHECK(zip(c).end() - zip(c).begin() == 4); + CHECK(zip(a, b).end() - zip(a, b).begin() == 3); + CHECK(zip(a, c).end() - zip(a, c).begin() == 3); + CHECK(zip(b, c).end() - zip(b, c).begin() == 4); + CHECK(zip(a, b, c).end() - zip(a, b, c).begin() == 3); +} + +TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { + std::vector a{1, 2, 3}; + std::vector b{4, 5, 6, 7, 8, 9}; + std::vector c{10, 11, 12, 13}; + + auto z = zip(a, b, c); + auto it = z.begin(); + auto it2 = z.begin(); + CHECK(it == it2); + CHECK(*it == *it2); + it++; + CHECK(it != it2); + it2++; + CHECK(it == it2); + CHECK(*it == *it2); + auto jt = z.end(); + CHECK(it != jt); + CHECK(it < jt); + CHECK(it <= jt); + CHECK(jt > it); + CHECK(jt >= it); + CHECK(jt == jt); + CHECK(jt >= jt); + CHECK(jt <= jt); + + it = z.begin(); + auto x = *it++; + CHECK(x == std::tuple{1, 4, 10}); + CHECK(it == z.begin() + 1); + + it = z.begin(); + auto y = *++it; + CHECK(y == std::tuple{2, 5, 11}); + CHECK(it == z.begin() + 1); + + CHECK(it[0] == *it); + CHECK(it[1] == *(it + 1)); + CHECK(it[2] == *(it + 2)); + CHECK(it[0] == std::tuple{2, 5, 11}); +} + +TEST_CASE("zip_view: alt_var_length_view", "[zip_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + auto v = alt_var_length_view{r, o}; + std::vector a{1, 2, 3}; + std::vector b{4, 5, 6, 7, 8, 9}; + std::vector c{10, 11, 12, 13}; + + auto z = zip(a, b, c, v); + auto it = z.begin(); + CHECK(std::get<0>(*it) == 1); + CHECK(std::get<1>(*it) == 4); + CHECK(std::get<2>(*it) == 10); + CHECK(std::ranges::equal( + std::get<3>(*it++), std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal( + std::get<3>(*it++), std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal( + std::get<3>(*it++), std::vector{7.0, 8.0, 9.0, 10.0})); +} + +TEST_CASE( + "zip_view: for, std::for_each with alt_var_length_view", "[zip_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + auto v = alt_var_length_view{r, o}; + std::vector a{8, 6, 7}; + + auto z = zip(a, v); + + SECTION("for_each") { + size_t count = 0; + std::for_each(z.begin(), z.end(), [&a, &count](auto x) { + auto&& [i, j] = x; + CHECK(i == a[count]); + switch (count) { + case 0: + CHECK(std::ranges::equal(j, std::vector{1.0, 2.0, 3.0})); + break; + case 1: + CHECK(std::ranges::equal(j, std::vector{4.0, 5.0, 6.0})); + break; + case 2: + CHECK( + std::ranges::equal(j, std::vector{7.0, 8.0, 9.0, 10.0})); + break; + } + ++count; + }); + } + SECTION("for") { + size_t count = 0; + for (auto x : z) { + auto&& [i, j] = x; + CHECK(i == a[count]); + switch (count) { + case 0: + CHECK(std::ranges::equal(j, std::vector{1.0, 2.0, 3.0})); + break; + case 1: + CHECK(std::ranges::equal(j, std::vector{4.0, 5.0, 6.0})); + break; + case 2: + CHECK( + std::ranges::equal(j, std::vector{7.0, 8.0, 9.0, 10.0})); + break; + } + ++count; + } + } +} diff --git a/tiledb/common/zip_view.h b/tiledb/common/zip_view.h new file mode 100644 index 000000000000..a502bda0ae71 --- /dev/null +++ b/tiledb/common/zip_view.h @@ -0,0 +1,261 @@ +/** + * @file zip_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements a zip view for zipping together a set of ranges. + * It is intended to implement the zip view as defined for C++23. From + * https://en.cppreference.com/w/cpp/ranges/zip_view: + * 1) A zip_view is a range adaptor that takes one or more views, and produces + * a view whose ith element is a tuple-like value consisting of the ith elements + * of all views. The size of produced view is the minimum of sizes of all + * adapted views. 2) zip is a customization point object that constructs a + * zip_view. + */ + +#ifndef TILEDB_ZIP_VIEW_H +#define TILEDB_ZIP_VIEW_H + +#include +#include "iterator_facade.h" + +// @todo Should this take viewable ranges? +// template +template +class zip_view { + /** + * Forward declaration to private (random access) iterator class + * @todo gneralize to non-random access ranges + */ + template + struct private_iterator; + + using iterator_type = private_iterator; + using const_iterator_type = private_iterator; + + public: + /**************************************************************************** + * Constructors + ****************************************************************************/ + + /** + * Construct a zip view from a set of ranges. The ranges are stored in a + * tuple. + * @param rngs The ranges to zip + * + * @tparam Ranges The types of the ranges to zip + * @param rngs The ranges to zip + */ + template + zip_view(Ranges&&... rngs) + : ranges_{std::forward(rngs)...} { + } + + /**************************************************************************** + * + * Iterator accessors. If all the ranges are random access ranges (which + * we assume is the case for now), an iterator is a tuple of begin iterators + * for each of + * the ranges being zipped, plus an index. + * + * @note The end iterator is only defined if all the ranges are sized ranges, + * in which case the size of the zipped view is the minimum of the sizes of + * the ranges being zipped, and the end iterator is the begin iterator plus + * the size of the zipped view. + * + ****************************************************************************/ + + /** Return an iterator to the beginning of the zipped view. */ + auto begin() { + return std::apply( + [](auto&&... rngs) { + return iterator_type(std::ranges::begin(rngs)...); + }, + ranges_); + } + + /** + * Return an iterator to the end of the zipped view. See above for what we + * mean by "end" + */ + auto end() + requires(std::ranges::sized_range && ...) + { + return std::apply( + [this](auto&&... rngs) { + return iterator_type(std::ranges::begin(rngs)..., this->size()); + }, + ranges_); + } + + /** Return an iterator to the beginning of a const zipped view. */ + auto begin() const { + return std::apply( + [](auto&&... rngs) { + return const_iterator_type(std::ranges::cbegin(rngs)...); + }, + ranges_); + } + + /** + * Return an iterator to the end of a const zipped view. See above for what + * we mean by "end" + */ + auto end() const + requires(std::ranges::sized_range && ...) + { + return std::apply( + [this](auto&&... rngs) { + return const_iterator_type( + std::ranges::cbegin(rngs)..., this->size()); + }, + ranges_); + } + /** Return an iterator to the beginning of a const zipped view. */ + auto cbegin() const { + return std::apply( + [](auto&&... rngs) { + return const_iterator_type(std::ranges::cbegin(rngs)...); + }, + ranges_); + } + + /** + * Return an iterator to the end of a const zipped view. See above for what + * we mean by "end" + */ + auto cend() const { + return std::apply( + [this](auto&&... rngs) { + return const_iterator_type( + std::ranges::cbegin(rngs)..., this->size()); + }, + ranges_); + } + + /** + * @brief The size of the zipped view is the size of the smallest range in the + * view. Requires that all the ranges are sized ranges. + * @return The size of the smallest range in the zip view + */ + auto size() const + requires(std::ranges::sized_range && ...) + { + return std::apply( + [](auto&&... rngs) { return std::min({std::ranges::size(rngs)...}); }, + ranges_); + } + + private: + /** + * This is a very straightforward iterator for a zip view over a set of + * random access ranges. It keeps an iterator for each of the ranges being + * zipped, along with an index into them. + * @tparam Rs + */ + template + struct private_iterator : public iterator_facade> { + using value_type_ = std::tuple...>; + using index_type = + std::common_type_t...>; + + /** Default constructor */ + private_iterator() = default; + + /** Construct an iterator from a set of begin iterators */ + private_iterator( + std::ranges::iterator_t... begins, index_type index = 0) + : index_(index) + , begins_(begins...) { + } + + /************************************************************************* + * Functions needed for iterator_facade + * Here we just supply the minimum needed to make the iterator work + *************************************************************************/ + + /** + * Dereference the iterator -- the critical function for defining the + * iterator sinc the facade bases many type aliases and other functions + * based on it and its signature + */ + value_type_ dereference() const { + return std::apply( + [this](auto&&... iters) { return value_type_(iters[index_]...); }, + begins_); + } + + /** Advance the iterator by n */ + auto advance(index_type n) { + index_ += n; + return *this; + } + + /** Return the distance to another iterator */ + auto distance_to(const private_iterator& other) const { + return other.index_ - index_; + } + + /** Compare two iterators for equality */ + bool operator==(const private_iterator& other) const { + return begins_ == other.begins_ && index_ == other.index_; + } + + /************************************************************************* + * Data members + *************************************************************************/ + + /** Current index of iterator */ + index_type index_; + + /** Begin iterators for each of the ranges being zipped */ + std::tuple...> begins_; + }; + + private: + /** The ranges being zipped */ + std::tuple ranges_; +}; + +/** + * Define "zip()" cpo for creating zip views + */ +namespace _zip { +struct _fn { + // @todo Should this take viewable ranges? + // template + template + auto constexpr operator()(T&&... t) const { + return zip_view{std::forward(t)...}; + } +}; +} // namespace _zip +inline namespace _cpo { +inline constexpr auto zip = _zip::_fn{}; +} // namespace _cpo + +#endif // TILEDB_ZIP_VIEW_H From 29420ea9afd233034bd7780d821117229cdbe658 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 2 May 2024 15:39:54 +0300 Subject: [PATCH 322/456] [Backport-release-2.23] Fix wrong fallback cell order for Hilbert. (#4924) (#4940) Backport 30b8ad0e6bfe59aaf1696b6a6bfc057cf636b8ae from #4924. --- TYPE: BUG DESC: Fix wrong fallback cell order for Hilbert. Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> From 01400c8f367741bea4500e153680542c193edbc5 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 2 May 2024 15:28:18 +0200 Subject: [PATCH 323/456] Fix nightlies after external sort merges. (#4937) [sc-46464] TYPE: NO_HISTORY DESC: Fix nightlies after external sort merges. --- tiledb/common/var_length_view.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tiledb/common/var_length_view.h b/tiledb/common/var_length_view.h index f5c020e85906..2054f1ef68bc 100644 --- a/tiledb/common/var_length_view.h +++ b/tiledb/common/var_length_view.h @@ -120,10 +120,10 @@ class var_length_view : public std::ranges::view_base { */ var_length_view( std::ranges::iterator_t data_begin, - std::ranges::iterator_t data_end, + [[maybe_unused]] std::ranges::iterator_t data_end, std::ranges::range_difference_t n_data, std::ranges::iterator_t index_begin, - std::ranges::iterator_t index_end, + [[maybe_unused]] std::ranges::iterator_t index_end, std::ranges::range_difference_t n_index) : data_begin_(data_begin) , data_end_(data_begin + n_data) From d82572489a8d03ef695f5a441393a32351d22398 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 2 May 2024 12:30:04 -0700 Subject: [PATCH 324/456] Write iter_swap for zip_view for external sort. (#4943) This PR stacks on #4930. The actual code for implementing the PR is fairly small, but with extensive unit tests. The PR contains `swap` function for `zip_view` in order to be able to use `std::sort` on a `zip_view`. The difficulty is that dereferencing a `zip_view` iterator returns a proxy, specifically a tuple of references. In order to enable `swap` to be found for this via ADL, we write ```c++ template requires(std::is_swappable::value && ...) void swap(std::tuple&& x, std::tuple&& y) { using std::get; using std::swap; [&](std::index_sequence) { (swap(get(x), get(y)), ...); }(std::make_index_sequence()); } ``` and then put it into the `std` namespace: ```c++ namespace std { using ::swap; } ``` The provided unit tests exercise `swap`, `iter_swap`, and `sort`, including on `zip_view`s containing `alt_var_length_view`s. (NOTE: I didn't specifically write an `item_swap` as `std::iter_swap` falls back to `swap`.) The PR also changes `alt_var_length_view` and `zip_view` to derive from `std::ranges::view_interface` rather than `std::ranges::view_base` in order to automagically get `operator[]` for the view. NOTES: 1. Not all implementations of `std::sort` use `std::swap` for all problem sizes, but will use insertion sort instead for small sizes. 2. For insertion sort to work properly with a tuple of references, i.e., a proxy, it is important to get all of the details of `value_type` and `reference` correct. There was a bug that took a while to track down of `value_type` not being defined in the `zip_iterator`. The `iterator_facade` will create the wrong default thing. --- TYPE: IMPROVEMENT DESC: Write iter_swap for zip_view for external sort. --- tiledb/common/alt_var_length_view.h | 22 +- tiledb/common/test/CMakeLists.txt | 2 +- .../common/test/unit_alt_var_length_view.cc | 29 ++ tiledb/common/test/unit_sort_zip.cc | 361 ++++++++++++++++++ tiledb/common/test/unit_zip_view.cc | 75 +++- tiledb/common/zip_view.h | 160 ++++++-- 6 files changed, 607 insertions(+), 42 deletions(-) create mode 100644 tiledb/common/test/unit_sort_zip.cc diff --git a/tiledb/common/alt_var_length_view.h b/tiledb/common/alt_var_length_view.h index 44c2194e805d..bb16b7d41609 100644 --- a/tiledb/common/alt_var_length_view.h +++ b/tiledb/common/alt_var_length_view.h @@ -65,16 +65,28 @@ * alt_var_length_view is a random_access_iterator that dereferences to a * subrange of the data range. * - * @tparam R Type of the data range, assumed to be a random access range. - * @tparam I Type of the index range, assumed to be a random access range. + * @tparam R Type of the data range, required to be viewable_range + * @tparam I Type of the index range, required to be viewable_range * - * @todo R could be a view rather than a range. - * @todo Would using `std::ranges::view_interface` be better tha `view_base`? + * @note We use view_interface instead of view_base to get operator[] (among + * other things). */ + +/** @todo This should go in some common place */ +#define GCC_VERSION \ + (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) + +#if GCC_VERSION > 120000 +template + requires std::ranges::random_access_range && + std::ranges::random_access_range +#else template < std::ranges::random_access_range R, std::ranges::random_access_range I> -class alt_var_length_view : public std::ranges::view_base { +#endif +class alt_var_length_view + : public std::ranges::view_interface> { /** * Forward reference of the iterator over the range of variable length data */ diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index e6f7df42ed5c..702aa5ac8639 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_var_length_view.cc unit_zip_view.cc) + this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_view.cc unit_zip_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_alt_var_length_view.cc b/tiledb/common/test/unit_alt_var_length_view.cc index 9f0ee32ad0b5..4dce07830a6a 100644 --- a/tiledb/common/test/unit_alt_var_length_view.cc +++ b/tiledb/common/test/unit_alt_var_length_view.cc @@ -66,6 +66,12 @@ TEST_CASE( CHECK(std::ranges::view); } +struct dummy_compare { + bool operator()(auto&&, auto&&) const { + return true; + } +}; + // Test that the alt_var_length_view iterators satisfy the expected concepts TEST_CASE( "alt_var_length_view: Iterator concepts", @@ -92,6 +98,29 @@ TEST_CASE( CHECK(std::bidirectional_iterator); CHECK(std::random_access_iterator); CHECK(std::random_access_iterator); + + using ZI = test_type_iterator; + CHECK(std::indirectly_writable>); + CHECK(std::indirectly_writable>); + + CHECK(std::indirectly_movable); + + CHECK(std::indirectly_movable_storable); + + CHECK(std::movable>); + CHECK(std::constructible_from< + std::iter_value_t, + std::iter_rvalue_reference_t>); + CHECK(std::assignable_from< + std::iter_value_t&, + std::iter_rvalue_reference_t>); + CHECK(std::forward_iterator); + + CHECK(std::indirectly_swappable); + CHECK(std::permutable); + CHECK(!std::sortable); + + CHECK(std::sortable); } // Test that the alt_var_length_view value_type satisfies the expected concepts diff --git a/tiledb/common/test/unit_sort_zip.cc b/tiledb/common/test/unit_sort_zip.cc new file mode 100644 index 000000000000..0a0b229bddde --- /dev/null +++ b/tiledb/common/test/unit_sort_zip.cc @@ -0,0 +1,361 @@ +/** + * @file unit_sort_zip.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the sort_zip class. + */ + +#include +#include +#include +#include "../alt_var_length_view.h" +#include "../zip_view.h" + +TEST_CASE("sort_zip: Null test", "[zip_view][null_test]") { + REQUIRE(true); +} + +std::vector q = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; +std::vector r = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + +std::vector o = {0, 3, 6, 10}; +std::vector p = {0, 2, 7, 10}; + +using avl_test_type = + alt_var_length_view, std::vector>; + +TEST_CASE("sort_zip: swap alt_var_length_view", "[zip_view]") { + auto a = avl_test_type{q, o}; + std::swap(a[0], a[1]); + CHECK(std::ranges::equal(a[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(a[1], std::vector{1.0, 2.0, 3.0})); + + std::swap(*(a.begin()), *((a.begin()) + 1)); + CHECK(std::ranges::equal(a[0], std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal(a[1], std::vector{4.0, 5.0, 6.0})); + + auto x = a.begin(); + auto y = x + 1; + std::swap(*x, *y); + CHECK(std::ranges::equal(a[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(a[1], std::vector{1.0, 2.0, 3.0})); + + auto b = avl_test_type{r, p}; + std::swap(b[0], b[1]); + CHECK(std::ranges::equal( + b[0], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); + CHECK(std::ranges::equal(b[1], std::vector{21.0, 20.0})); +} + +TEST_CASE("sort_zip: iter_swap alt_var_length_view", "[zip_view]") { + auto a = avl_test_type{q, o}; + auto a0 = a.begin(); + auto a1 = a0 + 1; + std::iter_swap(a0, a1); + CHECK(std::ranges::equal(a0[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(a0[1], std::vector{1.0, 2.0, 3.0})); + + std::iter_swap(a0, a0 + 1); + CHECK(std::ranges::equal(a0[0], std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal(a0[1], std::vector{4.0, 5.0, 6.0})); + + std::swap(*a0, *a1); + CHECK(std::ranges::equal(a0[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(a0[1], std::vector{1.0, 2.0, 3.0})); + + std::swap(*a0, *(a0 + 1)); + CHECK(std::ranges::equal(a0[0], std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal(a0[1], std::vector{4.0, 5.0, 6.0})); + + auto b = avl_test_type{r, p}; + auto b0 = b.begin(); + auto b1 = b0 + 1; + std::iter_swap(b0, b1); + CHECK(std::ranges::equal( + b[0], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); + CHECK(std::ranges::equal(b[1], std::vector{21.0, 20.0})); + + std::iter_swap(b1, b0); + CHECK(std::ranges::equal(b[0], std::vector{21.0, 20.0})); + CHECK(std::ranges::equal( + b[1], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); + + std::iter_swap(b0, b0); + CHECK(std::ranges::equal(b[0], std::vector{21.0, 20.0})); + CHECK(std::ranges::equal( + b[1], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); + + std::iter_swap(b1, b1); + CHECK(std::ranges::equal(b[0], std::vector{21.0, 20.0})); + CHECK(std::ranges::equal( + b[1], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); +} + +TEST_CASE("sort_zip: iter_swap zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + auto v = alt_var_length_view{r, o}; + + SECTION("a") { + auto z = zip(a); + iter_swap(z.begin(), z.begin()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + auto it = z.begin(); + iter_swap(it, it + 1); + CHECK(a == std::vector{2, 1, 3, 4, 5}); + } + SECTION("a, b") { + auto z = zip(a, b); + iter_swap(z.begin(), z.begin()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + CHECK(b == std::vector{5, 4, 3, 2, 1}); + auto it = z.begin(); + iter_swap(it, it + 1); + CHECK(a == std::vector{2, 1, 3, 4, 5}); + CHECK(b == std::vector{4, 5, 3, 2, 1}); + it++; + iter_swap(it, it + 1); + CHECK(a == std::vector{2, 3, 1, 4, 5}); + CHECK(b == std::vector{4, 3, 5, 2, 1}); + } + SECTION("a, b, v") { + auto z = zip(a, b, v); + iter_swap(z.begin(), z.begin()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + CHECK(b == std::vector{5, 4, 3, 2, 1}); + CHECK(std::ranges::equal(v[0], std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal(v[1], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(v[2], std::vector{7.0, 8.0, 9.0, 10.0})); + auto it = z.begin(); + iter_swap(it, it + 1); + CHECK(a == std::vector{2, 1, 3, 4, 5}); + CHECK(b == std::vector{4, 5, 3, 2, 1}); + CHECK(std::ranges::equal(v[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(v[1], std::vector{1.0, 2.0, 3.0})); + CHECK(std::ranges::equal(v[2], std::vector{7.0, 8.0, 9.0, 10.0})); + it++; + iter_swap(it, it + 1); + CHECK(a == std::vector{2, 3, 1, 4, 5}); + CHECK(b == std::vector{4, 3, 5, 2, 1}); + CHECK(std::ranges::equal(v[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(v[1], std::vector{7.0, 8.0, 9.0, 10.0})); + CHECK(std::ranges::equal(v[2], std::vector{1.0, 2.0, 3.0})); + } +} + +template +concept one = requires(Out&& o, T&& t) { *o = std::forward(t); }; +template +concept two = + requires(Out&& o, T&& t) { *std::forward(o) = std::forward(t); }; +template +concept three = requires(Out&& o, T&& t) { + const_cast&&>(*o) = std::forward(t); +}; +template +concept four = requires(Out&& o, T&& t) { + const_cast&&>(*std::forward(o)) = + std::forward(t); +}; + +TEST_CASE("sort_zip: mini sort zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + + SECTION("a") { + auto z = zip(a); + std::sort(z.begin(), z.end()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + } +} + +TEST_CASE("sort_zip: range sort zip view concepts", "[zip_view]") { + using VI = std::ranges::iterator_t>; + using ZI = std::ranges::iterator_t>>; + + CHECK(std::forward_iterator); + CHECK(std::indirectly_movable_storable); + CHECK(std::indirectly_swappable); + CHECK(std::permutable); + CHECK(std::sortable); + + CHECK(std::indirectly_readable); + + // This will work in C++23 but not C++20 + // Hence, we cannot use std::ranges::sort with zip_view in C++20 +#if 0 + { + int i = 0; + using T = std::tuple; + + auto xx = T{i}; + const_cast&&>(xx) = xx; + } +#endif + + CHECK(one>); + CHECK(one>); + CHECK(two>); + CHECK(two>); + + // These need to pass in order for zip_view to work with std::ranges::sort + // (all due to std::tuple issues -- see above) + CHECK(!three>); + CHECK(!three>); + CHECK(!four>); + CHECK(!four>); + + // Once three and four don't pass, the following chain will not pass + CHECK(!std::indirectly_writable>); + CHECK(!std::indirectly_writable>); + CHECK(!std::indirectly_movable); + CHECK(!std::indirectly_movable_storable); + CHECK(!std::permutable); + CHECK(!std::sortable); + + CHECK(std::movable>); + CHECK(std::constructible_from< + std::iter_value_t, + std::iter_rvalue_reference_t>); + CHECK(std::assignable_from< + std::iter_value_t&, + std::iter_rvalue_reference_t>); + CHECK(std::forward_iterator); + CHECK(std::indirectly_swappable); + CHECK(std::is_swappable_v); +} + +#if 0 +TEST_CASE("sort_zip: range sort zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + + SECTION("a") { + auto z = zip(a); + std::ranges::sort(z); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + } +} +#endif + +TEST_CASE("sort_zip: swap zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + auto z0 = zip(a); + + swap(z0[0], z0[1]); + CHECK(a == std::vector{2, 1, 3, 4, 5}); + CHECK(b == std::vector{5, 4, 3, 2, 1}); + + auto z1 = zip(a, b); + swap(z1[2], z1[3]); + CHECK(a == std::vector{2, 1, 4, 3, 5}); + CHECK(b == std::vector{5, 4, 2, 3, 1}); +} + +TEST_CASE("sort_zip: mini iter_swap zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + auto z0 = zip(a); + auto z00 = z0.begin(); + auto z01 = z00 + 1; + + iter_swap(z00, z01); + CHECK(a == std::vector{2, 1, 3, 4, 5}); + CHECK(b == std::vector{5, 4, 3, 2, 1}); + + auto z1 = zip(a, b); + swap(z1[2], z1[3]); + CHECK(a == std::vector{2, 1, 4, 3, 5}); + CHECK(b == std::vector{5, 4, 2, 3, 1}); +} + +TEST_CASE("sort_zip: sort zip view", "[zip_view]") { + std::vector a = {1, 2, 3, 4, 5}; + std::vector b = {5, 4, 3, 2, 1}; + + SECTION("a") { + auto z = zip(a); + std::sort(z.begin(), z.end()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + } + SECTION("b") { + auto z = zip(b); + std::sort(z.begin(), z.end()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + } + SECTION("a, b") { + auto z = zip(a, b); + std::sort(z.begin(), z.end()); + CHECK(a == std::vector{1, 2, 3, 4, 5}); + CHECK(b == std::vector{5, 4, 3, 2, 1}); + } + SECTION("b, a") { + auto z = zip(b, a); + std::sort(z.begin(), z.end()); + CHECK(a == std::vector{5, 4, 3, 2, 1}); + CHECK(b == std::vector{1, 2, 3, 4, 5}); + } +} + +TEST_CASE( + "sort_zip: sort zip view containing alt_var_length_view", "[zip_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + auto v = alt_var_length_view{r, o}; + std::vector a{8, 6, 7}; + + auto z = zip(a, v); + std::sort(z.begin(), z.end(), [](const auto& x, const auto& y) { + return std::get<0>(x) < std::get<0>(y); + }); + CHECK(a == std::vector{6, 7, 8}); + CHECK(std::ranges::equal(v[0], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(v[1], std::vector{7.0, 8.0, 9.0, 10.0})); + CHECK(std::ranges::equal(v[2], std::vector{1.0, 2.0, 3.0})); +} + +TEST_CASE("sort_zip: sort zip view using alt_var_length_view", "[zip_view]") { + std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector o = {0, 3, 6, 10}; + auto v = alt_var_length_view{r, o}; + std::vector a{8, 6, 7}; + + auto z = zip(a, v); + std::sort(z.begin(), z.end(), [](const auto& x, const auto& y) { + return std::get<1>(x)[0] > std::get<1>(y)[0]; + }); + CHECK(a == std::vector{7, 6, 8}); + CHECK(std::ranges::equal(v[0], std::vector{7.0, 8.0, 9.0, 10.0})); + CHECK(std::ranges::equal(v[1], std::vector{4.0, 5.0, 6.0})); + CHECK(std::ranges::equal(v[2], std::vector{1.0, 2.0, 3.0})); +} diff --git a/tiledb/common/test/unit_zip_view.cc b/tiledb/common/test/unit_zip_view.cc index 0ca003509b5b..fe4f71a88c5d 100644 --- a/tiledb/common/test/unit_zip_view.cc +++ b/tiledb/common/test/unit_zip_view.cc @@ -32,15 +32,16 @@ #include #include +#include #include -#include "../zip_view.h" - #include "../alt_var_length_view.h" +#include "../zip_view.h" TEST_CASE("zip_view: Null test", "[zip_view][null_test]") { REQUIRE(true); } +/** Test that the zip_view type satisfies the expected range concepts */ TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { using test_type = zip_view, std::vector>; @@ -55,35 +56,75 @@ TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { CHECK(std::ranges::random_access_range); CHECK(!std::ranges::contiguous_range); CHECK(std::ranges::common_range); + CHECK(std::ranges::viewable_range); - // @todo Fix so that it passes on ubuntu. - // CHECK(std::ranges::viewable_range); - - // @todo: Should this be a view? - CHECK(!std::ranges::view); + CHECK(std::ranges::view); } +/** Test that the zip_view iterator satisfies the expected concepts */ TEST_CASE("zip_view: Iterator concepts", "[zip_view][concepts]") { using test_type = zip_view, std::vector>; using test_type_iterator = std::ranges::iterator_t; using test_type_const_iterator = std::ranges::iterator_t; + // using test_type_const_iterator = decltype((const test_type){}.begin()); + CHECK(std::is_same_v< + test_type_const_iterator, + decltype(std::declval().begin())>); CHECK(std::input_or_output_iterator); - CHECK(std::input_or_output_iterator); CHECK(std::input_iterator); - CHECK(std::input_iterator); + + CHECK(std::input_or_output_iterator); + + /* + * The following tests fail for the const iterator. Whcih seems to be + * correct behavior: + * cf https://quuxplusone.github.io/blog/2023/08/13/non-const-iterable-ranges/ + * + * The tests here are the fine grained constituents of forward iterator, + * which we include to zoom in on exactly where the failure is. It appears + * to be in the common_reference concepts below. The concept checks fail in + * that the code does not even compile. + */ + CHECK(!std::input_iterator); + CHECK(!std::indirectly_readable); + CHECK(!std::common_reference_with< + std::iter_reference_t&&, + std::iter_value_t&>); + + CHECK(std::common_reference_with< + std::iter_reference_t&&, + std::iter_reference_t&>); + CHECK(std::common_reference_with< + std::iter_reference_t&&, + std::iter_rvalue_reference_t&>); + CHECK(!std::output_iterator< - test_type_iterator, + test_type_const_iterator, std::ranges::range_value_t>); + CHECK(!std::forward_iterator); + CHECK(!std::bidirectional_iterator); + CHECK(!std::random_access_iterator); + + /* + * These will not compile + using T = std::iter_reference_t; + using U = std::iter_value_t; + + CHECK(std::same_as< + std::common_reference_t, + std::common_reference_t>); + CHECK(std::convertible_to>); + CHECK(std::convertible_to>); + */ + CHECK(!std::output_iterator< - test_type_const_iterator, + test_type_iterator, std::ranges::range_value_t>); + CHECK(std::forward_iterator); - CHECK(std::forward_iterator); CHECK(std::bidirectional_iterator); - CHECK(std::bidirectional_iterator); CHECK(std::random_access_iterator); - CHECK(std::random_access_iterator); } // Test that the zip_view value_type satisfies the expected concepts @@ -103,6 +144,7 @@ TEST_CASE("zip_view: value_type concepts", "[zip_view][concepts]") { CHECK(std::is_same_v); } +/** Test zip_view constructors */ TEST_CASE("zip_view: constructor", "[zip_view]") { std::vector a{1, 2, 3}; std::vector b{4, 5, 6}; @@ -139,6 +181,7 @@ TEST_CASE("zip_view: constructor", "[zip_view]") { } } +/** Test that size returns the min of the sizes of the input ranges */ TEST_CASE("zip_view: size()", "[zip_view]") { std::vector a{1, 2, 3}; std::vector b{4, 5, 6, 7, 8, 9}; @@ -153,6 +196,7 @@ TEST_CASE("zip_view: size()", "[zip_view]") { CHECK(zip(a, b, c).size() == 3); } +/** The end() of the zipped ranges should be begin() + size() */ TEST_CASE("zip_view: end()", "[zip_view]") { std::vector a{1, 2, 3}; std::vector b{4, 5, 6, 7, 8, 9}; @@ -178,6 +222,7 @@ TEST_CASE("zip_view: end()", "[zip_view]") { CHECK(zip(a, b, c).end() - zip(a, b, c).begin() == 3); } +/** Exercise the zip_view iterator */ TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { std::vector a{1, 2, 3}; std::vector b{4, 5, 6, 7, 8, 9}; @@ -219,6 +264,7 @@ TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { CHECK(it[0] == std::tuple{2, 5, 11}); } +/** Test zip with an alt_var_length_view */ TEST_CASE("zip_view: alt_var_length_view", "[zip_view]") { std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; std::vector o = {0, 3, 6, 10}; @@ -240,6 +286,7 @@ TEST_CASE("zip_view: alt_var_length_view", "[zip_view]") { std::get<3>(*it++), std::vector{7.0, 8.0, 9.0, 10.0})); } +/** Use zip_view with std::for_each and with range-based for */ TEST_CASE( "zip_view: for, std::for_each with alt_var_length_view", "[zip_view]") { std::vector r = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; diff --git a/tiledb/common/zip_view.h b/tiledb/common/zip_view.h index a502bda0ae71..c2f453a7be88 100644 --- a/tiledb/common/zip_view.h +++ b/tiledb/common/zip_view.h @@ -35,6 +35,23 @@ * of all views. The size of produced view is the minimum of sizes of all * adapted views. 2) zip is a customization point object that constructs a * zip_view. + * + * @note This implementation is specialized to random access iterators. In the + * implementation, we keep begin iterators for all of the constituents and the + * iterator keeps and index and indexes into the begin iterators. There are + * two alternatives: + * + * 1) Use a tuple of iterators that we advance, rather than keeping an index. + * This would work for all ranges (down to input ranges), but would be slower + * on advancing. It might be faster on dereferencing, since we wouldn't have + * to index into the tuple. + * + * 2) Maintain a pointer back to the container from which the iterator came. + * In that case the iterator would not need to maintain its own set of + * iterators. However, this would still only work for random access ranges and + * there would be an additional level of indirection. + * + * @todo Implement and evaluate the above alternatives */ #ifndef TILEDB_ZIP_VIEW_H @@ -43,25 +60,35 @@ #include #include "iterator_facade.h" -// @todo Should this take viewable ranges? -// template +/* + * @todo Should this take viewable ranges? E.g., + * template + * + * See discussion of randeom access above + */ template -class zip_view { +class zip_view : public std::ranges::view_interface> { /** * Forward declaration to private (random access) iterator class - * @todo gneralize to non-random access ranges + * @todo Generalize to non-random access ranges */ template - struct private_iterator; + struct zip_iterator; + + template + struct zip_reference; - using iterator_type = private_iterator; - using const_iterator_type = private_iterator; + using iterator_type = zip_iterator; + using const_iterator_type = zip_iterator; public: /**************************************************************************** * Constructors ****************************************************************************/ + /** Default constructor */ + zip_view() = default; + /** * Construct a zip view from a set of ranges. The ranges are stored in a * tuple. @@ -71,7 +98,7 @@ class zip_view { * @param rngs The ranges to zip */ template - zip_view(Ranges&&... rngs) + constexpr explicit zip_view(Ranges&&... rngs) : ranges_{std::forward(rngs)...} { } @@ -87,6 +114,11 @@ class zip_view { * the ranges being zipped, and the end iterator is the begin iterator plus * the size of the zipped view. * + * @note Some of these should be automatically generated via CRTP with + * view_interface, but they don't seem to be. + * + * @todo Investigate defined behavior of CRTP with view_interface + * ****************************************************************************/ /** Return an iterator to the beginning of the zipped view. */ @@ -116,7 +148,7 @@ class zip_view { auto begin() const { return std::apply( [](auto&&... rngs) { - return const_iterator_type(std::ranges::cbegin(rngs)...); + return const_iterator_type(std::ranges::begin(rngs)...); }, ranges_); } @@ -130,11 +162,11 @@ class zip_view { { return std::apply( [this](auto&&... rngs) { - return const_iterator_type( - std::ranges::cbegin(rngs)..., this->size()); + return const_iterator_type(std::ranges::begin(rngs)..., this->size()); }, ranges_); } + /** Return an iterator to the beginning of a const zipped view. */ auto cbegin() const { return std::apply( @@ -178,17 +210,24 @@ class zip_view { * @tparam Rs */ template - struct private_iterator : public iterator_facade> { - using value_type_ = std::tuple...>; - using index_type = - std::common_type_t...>; + struct zip_iterator : public iterator_facade> { + using index_type = std::common_type_t>>...>; + + /** + * The reference and value types for the iterator. Since the iterator + * returns a proxy -- tuple of references -- it is very important to get + * these exactly right, particularly for things like std::swap and + * std::sort. + */ + using reference = std::tuple...>; + using value_type = std::tuple...>; /** Default constructor */ - private_iterator() = default; + zip_iterator() = default; /** Construct an iterator from a set of begin iterators */ - private_iterator( - std::ranges::iterator_t... begins, index_type index = 0) + zip_iterator(std::ranges::iterator_t... begins, index_type index = 0) : index_(index) , begins_(begins...) { } @@ -203,9 +242,11 @@ class zip_view { * iterator sinc the facade bases many type aliases and other functions * based on it and its signature */ - value_type_ dereference() const { + reference dereference() const { return std::apply( - [this](auto&&... iters) { return value_type_(iters[index_]...); }, + [this](Is&&... iters) { + return reference(std::forward(iters)[index_]...); + }, begins_); } @@ -216,12 +257,12 @@ class zip_view { } /** Return the distance to another iterator */ - auto distance_to(const private_iterator& other) const { + auto distance_to(const zip_iterator& other) const { return other.index_ - index_; } /** Compare two iterators for equality */ - bool operator==(const private_iterator& other) const { + bool operator==(const zip_iterator& other) const { return begins_ == other.begins_ && index_ == other.index_; } @@ -234,6 +275,48 @@ class zip_view { /** Begin iterators for each of the ranges being zipped */ std::tuple...> begins_; + + /* + * Should we decide to start using std::ranges for containers and + * algorithms, we will need to implement iter_move and iter_swap for proxy + * iterators. + */ +#if 0 + friend constexpr auto iter_move(const zip_iterator& rhs) { + return std::apply( + [&](_Ts&&... __elts) { + return std::tuple< + std::invoke_result_t...>( + std::ranges::iter_move( + (std::forward<_Ts>(__elts) + rhs.index_))...); + }, + rhs.begins_); + } + + friend constexpr void iter_swap( + const zip_iterator& lhs, const zip_iterator& rhs) { + [&](std::index_sequence<_Is...>) { + (std::ranges::iter_swap( + std::get<_Is>(lhs.begins_) + lhs.index_, + std::get<_Is>(rhs.begins_) + rhs.index_), + ...); + }(std::make_index_sequence{}); + } + +#else + /** Function to swap values pointed to by two iterators */ + friend constexpr void iter_swap( + const zip_iterator& lhs, const zip_iterator& rhs) { + [&](std::index_sequence<_Is...>) { + (std::swap( + std::get<_Is>(lhs.begins_)[lhs.index_], + std::get<_Is>(rhs.begins_)[rhs.index_]), + ...); + }(std::make_index_sequence{}); + } +#endif + + friend class zip_view; }; private: @@ -258,4 +341,37 @@ inline namespace _cpo { inline constexpr auto zip = _zip::_fn{}; } // namespace _cpo +/** + * Define "swap()" for tuples of references + * + * Based on nwgraph implementation + * + * Also, Eric Niebler can "live with this" + * template< class T, class U > + * void swap( pair< T&, U& > && a, pair< T&, U& > && b ) + * { + * swap(a.first, b.first); + * swap(a.second, b.second); + * } + * + * We generalize Eric's example to tuples. + * + * @note Different C++ libraries implement std::sort differently. Some use + * std::swap at all levels of the algorithm. Others use insertion sort for + * small length containers -- which will not invoke std::swap. (In that case + * it is very important to get the reference/value types of the iterator right.) + */ +namespace std { +template + requires(std::is_swappable::value && ...) +void swap(std::tuple&& x, std::tuple&& y) { + using std::get; + using std::swap; + + [&](std::index_sequence) { + (swap(get(x), get(y)), ...); + }(std::make_index_sequence()); +} +} // namespace std + #endif // TILEDB_ZIP_VIEW_H From d00866b64eab47929170fe61f3498319d7b9711d Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Fri, 3 May 2024 06:56:45 -0700 Subject: [PATCH 325/456] Implement proxy sort permutation for external sort. (#4944) This PR implements proxy sorting for `permutation_view`. The goal is to be able to efficiently sort a `zip_view` containing a number of ranges. With the PR, we can sort a zipped `permutation_view` as follows: ```c++ auto z = zip(u, v, w, x); auto p = permutation_view(z); sort(p); ``` resulting in `z` being sorted along `u` (then `v`, etc). Because the iterator for `permutation_view` only returns a value, with no information about the index, we can't sort a `permutation_view` directly with, e.g., `std::sort`. To enable sorting, we instead provide sorting member functions to the `permutation_view` and have free functions that are overloaded on `permutation_view` invoke the member functions. The functions added include * proxy sort member functions in `permutation_view` with `_no_init`, `stable_`, and comparison function variants. * proxy sort free functions overloaded on `permutation_view` with variants as above * Overloads of `std::sort` and `std::stable_sort`. These invoke the `no_init` member function, the assumption being that we just want to stack permutations on top of each other. Since the `proxy_sort` functions mutate the indexes in the `permutation_view`, the interfaces to the existing `proxy_sort` function were updated to take a forwarding reference. A few notes: * Sorting a `permutation_view` will mutate the underlying permutation index range. Right now, the permutation view is just a view over two containers (the one to be permuted and the permutation itself). Sorting the `permutation_view` will change the ordering of the container passed in to the constructor. * When sorting a `permutation_view` with a tuple value type, the sorting will be done with the built-in tuple comparator. We may want to add variants that let us specify which members to sort on (using indexes passed in as variadic template parameters). However, using a specialized comparator is probably good enough. [sc-44172] --- TYPE: IMPROVEMENT DESC: Implement proxy sort permutation for external sort. --------- Co-authored-by: Luc Rancourt --- tiledb/common/permutation_view.h | 340 ++++++++++++++++++- tiledb/common/proxy_sort.h | 23 +- tiledb/common/test/CMakeLists.txt | 2 +- tiledb/common/test/unit_permutation_sort.cc | 341 ++++++++++++++++++++ tiledb/common/test/unit_sort_zip.cc | 21 +- tiledb/common/test/unit_zip_view.cc | 25 ++ 6 files changed, 718 insertions(+), 34 deletions(-) create mode 100644 tiledb/common/test/unit_permutation_sort.cc diff --git a/tiledb/common/permutation_view.h b/tiledb/common/permutation_view.h index 4cb7fa9cd0a2..9fa2274c940e 100644 --- a/tiledb/common/permutation_view.h +++ b/tiledb/common/permutation_view.h @@ -35,6 +35,14 @@ * the data range. For a data range `r` and a permutation `p`, and a * permutation_view `v`, based on `r` and `p`, the expression `v[i]` is * equivalent to `r[p[i]]`. + * + * To create an inverse permutation: + * + * std::vector perm(N); + * proxy_sort(shuffled, perm); + * + * std::vector i_perm(N); + * proxy_sort(perm, i_perm); */ #ifndef TILEDB_PERMUTATION_VIEW_H @@ -43,6 +51,7 @@ #include #include #include "iterator_facade.h" +#include "proxy_sort.h" /** * A view that creates a permutation of the underlying view, as determined by @@ -52,30 +61,32 @@ * @tparam P Type of the index range, assumed to be a random access range. * * @todo R could be a view rather than a range. + * @todo Implement with view_interface. */ template < - std::ranges::random_access_range R, - std::ranges::random_access_range P> -class permutation_view : public std::ranges::view_base { + std::ranges::random_access_range ElementRange, + std::ranges::random_access_range IndexRange> +class permutation_view : public std::ranges::view_interface< + permutation_view> { /** Forward reference of the iterator over the range of permuted data. */ template struct private_iterator; /** The data type of the permutation view. */ - using data_iterator_type = std::ranges::iterator_t; + using data_iterator_type = std::ranges::iterator_t; /** The index type of the permutation view indexing into underlying range. */ using data_index_type = std::iter_difference_t; /** The type of the iterator over the index range -- It should derference to * something that can index into the data range (e.g., the data_index_type)*/ - using index_iterator_type = std::ranges::iterator_t; + using index_iterator_type = std::ranges::iterator_t; - /** The value_type dereferenced by the iterator is the value_type R */ - using permuted_value_type = std::ranges::range_value_t; + /** The permuted view value_type is value_type of ElementRange */ + using permuted_value_type = std::ranges::range_value_t; - /** The value_type dereferenced by the iterator is the value_type R */ - using permuted_reference = std::ranges::range_reference_t; + /** The permuted view reference is the reference type of ElementRange */ + using permuted_reference = std::ranges::range_reference_t; /** The type of the iterator over the permuted data range */ using permuted_iterator = private_iterator; @@ -84,8 +95,11 @@ class permutation_view : public std::ranges::view_base { using permuted_const_iterator = private_iterator; public: - /** Primary constructor */ - permutation_view(R& data, const P& permutation) + /** + * Primary constructor + * @note IndexRange is not const + */ + permutation_view(ElementRange& data, IndexRange& permutation) : data_begin_(std::ranges::begin(data)) , data_end_(std::ranges::end(data)) , index_begin_(std::ranges::begin(permutation)) @@ -130,7 +144,7 @@ class permutation_view : public std::ranges::view_base { } /** Size of the permutation view */ - auto size() const { + size_t size() const { return data_end_ - data_begin_; } @@ -140,6 +154,115 @@ class permutation_view : public std::ranges::view_base { return data_begin_[index_begin_[i]]; } + /***************************************************************************** + * Member functions for sorting the permutation view. These will sort the + * permutation view with the type given of the data range. If this is a + * tuple, it will sort based on the first element, then the second, etc. + * @note These functions are not part of the view interface + * @todo Add variants to sort tuple based on given indexes, rather than by + * default order + ****************************************************************************/ + + /** + * Apply `proxy_sort` to the permutation view. This will permute the index + * range so that the indexed data range is sorted. The index range will be + * initialized with the identity permutation. + */ + void proxy_sort() { + ::proxy_sort( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_)); + } + + /** + * Apply `proxy_sort_no_init` to the permutation view. This will permute the + * index range so that the indexed data range is sorted. The index range is + * assumed to be a valid permutation of [0, data.size()) + */ + void proxy_sort_no_init() { + ::proxy_sort_no_init( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_)); + } + + /** + * Apply `proxy_sort` to the permutation view. This will permute the index + * range so that the indexed data range is sorted based on the comparison + * functor `cmp`. The index range will be initialized with the identity + * permutation. + */ + template + void proxy_sort(Compare comp) { + ::proxy_sort( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_), + comp); + } + + /** + * Apply `proxy_sort` to the permutation view. This will permute the index + * range so that the indexed data range is sorted based on the comparison + * functor `cmp`. The index range is assumed to be a valid permutation of [0, + * data.size()) + */ + template + void proxy_sort_no_init(Compare comp) { + ::proxy_sort_no_init( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_), + comp); + } + + /** + * Apply `stable_proxy_sort` to the permutation view. This will permute the + * index range so that the indexed data range is sorted. The index range will + * be initialized with the identity permutation. + */ + void stable_proxy_sort() { + ::stable_proxy_sort( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_)); + } + + /** + * Apply `stable_proxy_sort_no_init` to the permutation view. This will + * permute the index range so that the indexed data range is sorted. The + * index range is assumed to be a valid permutation of [0, data.size()) + */ + void stable_proxy_sort_no_init() { + ::stable_proxy_sort_no_init( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_)); + } + + /** + * Apply `stable_proxy_sort` to the permutation view. This will permute the + * index range so that the indexed data range is sorted based on the + * comparison functor `cmp`. The index range will be initialized with the + * identity permutation. + */ + template + void stable_proxy_sort(Compare comp) { + ::stable_proxy_sort( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_), + comp); + } + + /** + * Apply `stable_proxy_sort` to the permutation view. This will permute the + * index range so that the indexed data range is sorted based on the + * comparison functor `cmp`. The index range is assumed to be a valid + * permutation of [0, data.size()) + */ + template + void stable_proxy_sort_no_init(Compare comp) { + ::stable_proxy_sort_no_init( + std::ranges::subrange(data_begin_, data_end_), + std::ranges::subrange(index_begin_, index_end_), + comp); + } + private: /** * Private iterator class for permuted view @@ -210,17 +333,202 @@ class permutation_view : public std::ranges::view_base { }; /** Iterator to the beginning of the data range */ - std::ranges::iterator_t data_begin_; + data_iterator_type data_begin_; /** Iterator to the end of the data range */ - std::ranges::iterator_t data_end_; + data_iterator_type data_end_; /** const_iterator is c++23. For now we just use an iterator to const */ /** Iterator to the beginning of the index range */ - std::ranges::iterator_t index_begin_; + index_iterator_type index_begin_; /** Iterator to the end of the index range */ - std::ranges::iterator_t index_end_; + index_iterator_type index_end_; }; +/** + * Overload of `proxy_sort` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void proxy_sort_no_init(permutation_view& x) { + x.proxy_sort_no_init(); +} + +/** + * Overload of `proxy_sort_no_init` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void proxy_sort(permutation_view& x) { + x.proxy_sort(); +} + +/** + * Overload of `proxy_sort` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void proxy_sort_no_init(permutation_view& x, Compare comp) { + x.proxy_sort_no_init(comp); +} + +/** + * Overload of `proxy_sort_no_init` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void proxy_sort(permutation_view& x, Compare comp) { + x.proxy_sort(comp); +} + +/** + * Overload of `stable_proxy_sort` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void stable_proxy_sort_no_init(permutation_view& x) { + x.stable_proxy_sort_no_init(); +} + +/** + * Overload of `stable_proxy_sort_no_init` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void stable_proxy_sort(permutation_view& x) { + x.stable_proxy_sort(); +} + +/** + * Overload of `stable_proxy_sort` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void stable_proxy_sort_no_init(permutation_view& x, Compare comp) { + x.stable_proxy_sort_no_init(comp); +} + +/** + * Overload of `stable_proxy_sort_no_init` for permutation_view. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void stable_proxy_sort(permutation_view& x, Compare comp) { + x.stable_proxy_sort(comp); +} + +/** + * Overloads of `std::sort` for permutation_view. We take the view "as-is", + * i.e., we use `proxy_sort_no_init` to do the sorting. + */ +namespace std { + +/** + * Overload of `std::sort` for permutation_view. We take the view "as-is", + * i.e., we use `proxy_sort_no_init` to do the sorting. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void sort(permutation_view& x) { + x.proxy_sort_no_init(); +} + +/** + * Overload of `std::sort` for permutation_view. We take the view "as-is", + * i.e., we use `proxy_sort_no_init` to do the sorting. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void sort(permutation_view& x, Compare comp) { + x.proxy_sort_no_init(comp); +} + +/** + * Overload of `std::stable_sort` for permutation_view. We take the view + * "as-is", i.e., we use `proxy_sort_no_init` to do the sorting. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @param x permutation_view to be sorted + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P> +void stable_sort(permutation_view& x) { + x.stable_proxy_sort_no_init(); +} + +/** + * Overload of `std::stable_sort` for permutation_view. We take the view + * "as-is", i.e., we use `proxy_sort_no_init` to do the sorting. + * @tparam R Type of data range + * @tparam P Type of permutation range + * @tparam Compare Type of comparison function + * @param x permutation_view to be sorted + * @param comp Comparison function + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range P, + class Compare> +void stable_sort(permutation_view& x, Compare comp) { + x.stable_proxy_sort_no_init(comp); +} + +} // namespace std #endif // TILEDB_PERMUTATION_VIEW_H diff --git a/tiledb/common/proxy_sort.h b/tiledb/common/proxy_sort.h index f3996c941a46..01fa5b240bb9 100644 --- a/tiledb/common/proxy_sort.h +++ b/tiledb/common/proxy_sort.h @@ -79,10 +79,12 @@ template < std::ranges::random_access_range R, std::ranges::random_access_range I> -void proxy_sort_no_init(const R& x, I& perm) { +void proxy_sort_no_init(const R& x, I&& perm) { assert(perm.size() == x.size()); std::sort( - perm.begin(), perm.end(), [&](auto a, auto b) { return x[a] < x[b]; }); + std::forward(perm).begin(), + std::forward(perm).end(), + [&](auto a, auto b) { return x[a] < x[b]; }); } /** @@ -98,10 +100,10 @@ void proxy_sort_no_init(const R& x, I& perm) { template < std::ranges::random_access_range R, std::ranges::random_access_range I> -void proxy_sort(const R& x, I& perm) { +void proxy_sort(R&& x, I&& perm) { assert(perm.size() == x.size()); std::iota(perm.begin(), perm.end(), 0); - proxy_sort_no_init(x, perm); + proxy_sort_no_init(std::forward(x), std::forward(perm)); } /** @@ -120,11 +122,12 @@ template < std::ranges::random_access_range R, std::ranges::random_access_range I, class Compare> -void proxy_sort_no_init(const R& x, I& perm, Compare comp) { +void proxy_sort_no_init(const R& x, I&& perm, Compare comp) { assert(perm.size() == x.size()); - std::sort(perm.begin(), perm.end(), [&](auto a, auto b) { - return comp(x[a], x[b]); - }); + std::sort( + std::forward(perm).begin(), + std::forward(perm).end(), + [&](auto a, auto b) { return comp(x[a], x[b]); }); } /** @@ -143,10 +146,10 @@ template < std::ranges::random_access_range R, std::ranges::random_access_range I, class Compare> -void proxy_sort(const R& x, I& perm, Compare comp) { +void proxy_sort(R&& x, I&& perm, Compare comp) { assert(perm.size() == x.size()); std::iota(perm.begin(), perm.end(), 0); - proxy_sort_no_init(x, perm, comp); + proxy_sort_no_init(std::forward(x), std::forward(perm), comp); } /** diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 702aa5ac8639..a7b3d4bf94e9 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_view.cc unit_zip_view.cc) + this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_sort.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_view.cc unit_zip_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_permutation_sort.cc b/tiledb/common/test/unit_permutation_sort.cc new file mode 100644 index 000000000000..eb5a1cea4693 --- /dev/null +++ b/tiledb/common/test/unit_permutation_sort.cc @@ -0,0 +1,341 @@ +/** + * @file unit_permutation_sort.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for sorting for/with a permutation view. + */ + +#include +#include +#include +#include +#include +#include +#include "../alt_var_length_view.h" +#include "../permutation_view.h" +#include "../proxy_sort.h" +#include "../var_length_view.h" +#include "../zip_view.h" + +TEST_CASE("permutation_sort: Null test", "[permutation_sort][null_test]") { + REQUIRE(true); +} + +TEST_CASE("permutation_sort: integers", "[permutation_sort]") { + size_t seed = Catch::rngSeed(); + size_t N = 100'000; + std::mt19937 g(seed); + + std::vector init(N); + std::iota(begin(init), end(init), 0); + + std::vector perm(N); + std::iota(begin(perm), end(perm), 0); + + auto init_19 = std::vector(N); + std::iota(begin(init_19), end(init_19), 19); + + std::vector unshuffled(init); + std::vector shuffled(unshuffled); + + std::shuffle(shuffled.begin(), shuffled.end(), g); + std::vector shuffled_copy(shuffled); + + /* Sort the shuffled vector */ + proxy_sort_no_init(shuffled, perm); + + /* Generate the inverse permutation */ + std::vector i_perm(init); + proxy_sort_no_init(perm, i_perm); + + SECTION("check setup") { + CHECK(!std::equal(shuffled.begin(), shuffled.end(), unshuffled.begin())); + CHECK(std::equal(shuffled.begin(), shuffled.end(), shuffled_copy.begin())); + + auto b = permutation_view(init_19, perm); + CHECK(std::equal(b.begin(), b.end(), perm.begin(), [](auto a, auto b) { + return a == b + 19; + })); + + auto yy = permutation_view(shuffled, perm); // -> unshuffled + auto zz = permutation_view(yy, i_perm); // -> shuffled + CHECK(std::equal(begin(zz), end(zz), begin(shuffled))); + } + + SECTION("sort, permute") { + auto x = permutation_view(shuffled, perm); + CHECK(std::equal(x.begin(), x.end(), unshuffled.begin())); + CHECK(std::equal(shuffled.begin(), shuffled.end(), shuffled_copy.begin())); + } + + SECTION("sort, permute, zip") { + /* Create permutation view */ + auto x = permutation_view(shuffled, perm); + + /* Zip permutation view with init_19 */ + auto z = zip(x, init_19); + + /* First component of z should appear to be unshuffled */ + CHECK(std::equal(begin(z), z.end(), begin(unshuffled), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + + CHECK(std::equal(begin(z), end(z), begin(init), [](auto a, auto b) { + return std::get<1>(a) == b + 19; + })); + } + + SECTION("sort, zip, permute") { + /* Zip shuffled with init_19 */ + auto z = zip(shuffled, init_19); + + /* Permute the zipped view */ + auto x = permutation_view(z, perm); + + /* First component of x should appear to be unshuffled */ + CHECK(std::equal(begin(x), end(x), begin(unshuffled), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + + CHECK(std::equal(begin(x), end(x), begin(perm), [](auto a, auto b) { + return std::get<1>(a) == b + 19; + })); + } + + SECTION("sort, permute, inv_permute") { + /* Create permutation view */ + auto x = permutation_view(shuffled, perm); + + /* Inverse permute the permutation view */ + auto y = permutation_view(x, i_perm); + + /* y should appear to be shuffled */ + CHECK(std::equal(begin(y), end(y), begin(shuffled))); + } +} + +TEST_CASE("permutation_sort: multiple integers", "[permutation_sort]") { + size_t seed = Catch::rngSeed(); + size_t N = 100'000; + std::mt19937 g(seed); + + std::vector init(N); + std::iota(begin(init), end(init), 0); + + auto x = std::vector>(7, std::vector(init)); + for (auto& v : x) { + std::shuffle(begin(v), end(v), g); + } + + auto perm = std::vector>(7, std::vector(init)); + for (size_t i = 0; i < size(x); ++i) { + proxy_sort(x[i], perm[i]); + } + + auto i_perm = std::vector>(7, std::vector(init)); + for (size_t i = 0; i < size(perm); ++i) { + proxy_sort(perm[i], i_perm[i]); + } + + SECTION("check setup") { + for (size_t i = 0; i < size(x); ++i) { + auto a = permutation_view(x[i], perm[i]); + CHECK(std::equal(begin(a), end(a), begin(init))); + + auto b = permutation_view(a, i_perm[i]); + CHECK(std::equal(begin(b), end(b), begin(x[i]))); + } + } + + SECTION("sort, zip, permute some") { + /* Zip shuffled with init_19 */ + auto z = zip(x[0], x[1], x[2], x[3], x[4], x[5], x[6]); + + /* Permute the zipped view */ + auto x = permutation_view(z, perm[0]); + + /* First component of x should appear to be unshuffled */ + CHECK(std::equal(begin(x), end(x), begin(init), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + } + + SECTION("sort, zip, permute all") { + for (size_t j = 0; j < size(x); ++j) { + auto z = + zip(x[j % 7], + x[(j + 1) % 7], + x[(j + 2) % 7], + x[(j + 3) % 7], + x[(j + 4) % 7], + x[(j + 5) % 7], + x[(j + 6) % 7]); + + auto p0 = permutation_view(z, perm[j % 7]); + CHECK(std::equal(begin(p0), end(p0), begin(init), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + auto p1 = permutation_view(z, perm[(j + 1) % 7]); + CHECK(std::equal(begin(p1), end(p1), begin(init), [](auto a, auto b) { + return std::get<1>(a) == b; + })); + auto p2 = permutation_view(z, perm[(j + 2) % 7]); + CHECK(std::equal(begin(p2), end(p2), begin(init), [](auto a, auto b) { + return std::get<2>(a) == b; + })); + auto p3 = permutation_view(z, perm[(j + 3) % 7]); + CHECK(std::equal(begin(p3), end(p3), begin(init), [](auto a, auto b) { + return std::get<3>(a) == b; + })); + auto p4 = permutation_view(z, perm[(j + 4) % 7]); + CHECK(std::equal(begin(p4), end(p4), begin(init), [](auto a, auto b) { + return std::get<4>(a) == b; + })); + auto p5 = permutation_view(z, perm[(j + 5) % 7]); + CHECK(std::equal(begin(p5), end(p5), begin(init), [](auto a, auto b) { + return std::get<5>(a) == b; + })); + auto p6 = permutation_view(z, perm[(j + 6) % 7]); + CHECK(std::equal(begin(p6), end(p6), begin(init), [](auto a, auto b) { + return std::get<6>(a) == b; + })); + } + } +} + +TEST_CASE( + "permutation_sort: direct proxy_sort multiple integers", + "[permutation_sort]") { + size_t seed = Catch::rngSeed(); + size_t N = 100'000; + std::mt19937 g(seed); + + std::vector init(N); + std::iota(begin(init), end(init), 0); + + auto x = std::vector>(7, std::vector(init)); + for (auto& v : x) { + std::shuffle(begin(v), end(v), g); + } + + CHECK(!std::equal(begin(x[0]), end(x[0]), begin(init))); + + auto perm = std::vector>(7, std::vector(init)); + auto i_perm = std::vector>(7, std::vector(init)); + + SECTION("zip, sort some") { + /* Zip shuffled with init_19 */ + auto z = zip(x[0], x[1], x[2], x[3], x[4], x[5], x[6]); + + /* Create permutation view with zipped view */ + auto x = permutation_view(z, perm[0]); + + SECTION("less") { + SECTION("x.proxy_sort_no_init") { + x.proxy_sort_no_init(); + } + SECTION("x.proxy_sort_") { + x.proxy_sort(); + } + + SECTION("x.proxy_sort_no_init") { + x.proxy_sort_no_init(std::less<>()); + } + SECTION("x.proxy_sort_") { + x.proxy_sort(std::less<>()); + } + SECTION("x.proxy_sort_no_init") { + proxy_sort_no_init(x); + } + SECTION("x.proxy_sort_") { + proxy_sort(x); + } + SECTION("x.proxy_sort_no_init") { + proxy_sort_no_init(x, std::less<>()); + } + SECTION("x.proxy_sort_") { + proxy_sort(x, std::less<>()); + } + SECTION("std::sort") { + std::sort(x); + } + SECTION("std::sort") { + std::sort(x, std::less<>()); + } + + /* First component of x should appear to be unshuffled */ + CHECK(std::equal(begin(x), end(x), begin(init), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + } + SECTION("greater") { + auto reverse_index = std::vector(init); + std::ranges::reverse(reverse_index); + SECTION("x.proxy_sort_no_init") { + x.proxy_sort_no_init(std::greater<>()); + } + SECTION("x.proxy_sort_") { + x.proxy_sort(std::greater<>()); + } + SECTION("x.proxy_sort_no_init") { + proxy_sort_no_init(x, std::greater<>()); + } + SECTION("x.proxy_sort_") { + proxy_sort(x, std::greater<>()); + } + SECTION("std::sort") { + std::sort(x, std::greater<>()); + } + + /* First component of x should appear to be unshuffled */ + CHECK(std::equal( + begin(x), end(x), begin(reverse_index), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + } + } + + SECTION("sort, zip, permute all") { + for (size_t j = 0; j < size(x); ++j) { + auto z = + zip(x[j % 7], + x[(j + 1) % 7], + x[(j + 2) % 7], + x[(j + 3) % 7], + x[(j + 4) % 7], + x[(j + 5) % 7], + x[(j + 6) % 7]); + + auto v = permutation_view(z, perm[0]); + proxy_sort(v); + CHECK(std::equal(begin(v), end(v), begin(init), [](auto a, auto b) { + return std::get<0>(a) == b; + })); + } + } +} diff --git a/tiledb/common/test/unit_sort_zip.cc b/tiledb/common/test/unit_sort_zip.cc index 0a0b229bddde..cc5f67187207 100644 --- a/tiledb/common/test/unit_sort_zip.cc +++ b/tiledb/common/test/unit_sort_zip.cc @@ -40,17 +40,17 @@ TEST_CASE("sort_zip: Null test", "[zip_view][null_test]") { REQUIRE(true); } -std::vector q = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; -std::vector r = { - 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; - -std::vector o = {0, 3, 6, 10}; -std::vector p = {0, 2, 7, 10}; - using avl_test_type = alt_var_length_view, std::vector>; TEST_CASE("sort_zip: swap alt_var_length_view", "[zip_view]") { + std::vector q = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector r = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + + std::vector o = {0, 3, 6, 10}; + std::vector p = {0, 2, 7, 10}; + auto a = avl_test_type{q, o}; std::swap(a[0], a[1]); CHECK(std::ranges::equal(a[0], std::vector{4.0, 5.0, 6.0})); @@ -74,6 +74,13 @@ TEST_CASE("sort_zip: swap alt_var_length_view", "[zip_view]") { } TEST_CASE("sort_zip: iter_swap alt_var_length_view", "[zip_view]") { + std::vector q = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; + std::vector r = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; + + std::vector o = {0, 3, 6, 10}; + std::vector p = {0, 2, 7, 10}; + auto a = avl_test_type{q, o}; auto a0 = a.begin(); auto a1 = a0 + 1; diff --git a/tiledb/common/test/unit_zip_view.cc b/tiledb/common/test/unit_zip_view.cc index fe4f71a88c5d..00c0be0df4d3 100644 --- a/tiledb/common/test/unit_zip_view.cc +++ b/tiledb/common/test/unit_zip_view.cc @@ -41,6 +41,29 @@ TEST_CASE("zip_view: Null test", "[zip_view][null_test]") { REQUIRE(true); } +TEST_CASE("zip_view: Should not copy", "[zip_view]") { + struct foo : public std::vector { + explicit foo(int N) + : std::vector(N) { + } + foo(const foo&) + : std::vector() { + CHECK(false); + CHECK(true); + } + foo(foo&&) = delete; + foo& operator=(foo&) = delete; + foo& operator=(foo&&) = delete; + foo() = delete; + ~foo() = default; + }; + + auto f = foo(10); + auto g = foo(10); + auto h = foo(10); + [[maybe_unused]] auto z = zip(f, g, h); +} + /** Test that the zip_view type satisfies the expected range concepts */ TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { using test_type = zip_view, std::vector>; @@ -230,6 +253,7 @@ TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { auto z = zip(a, b, c); auto it = z.begin(); + CHECK(it == begin(z)); auto it2 = z.begin(); CHECK(it == it2); CHECK(*it == *it2); @@ -239,6 +263,7 @@ TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { CHECK(it == it2); CHECK(*it == *it2); auto jt = z.end(); + CHECK(jt == end(z)); CHECK(it != jt); CHECK(it < jt); CHECK(it <= jt); From ca3b3cd40749c2c84bcaa1f8ceafbcb116600e5f Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Sun, 5 May 2024 14:57:11 +0200 Subject: [PATCH 326/456] Fix bad type casts in unit_permutation_sort.cc. (#4949) This should fix the windows nightlies. --- TYPE: NO_HISTORY DESC: Fix bad type casts in unit_permutation_sort.cc. --- tiledb/common/test/unit_permutation_sort.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/common/test/unit_permutation_sort.cc b/tiledb/common/test/unit_permutation_sort.cc index eb5a1cea4693..41b374562901 100644 --- a/tiledb/common/test/unit_permutation_sort.cc +++ b/tiledb/common/test/unit_permutation_sort.cc @@ -47,7 +47,7 @@ TEST_CASE("permutation_sort: Null test", "[permutation_sort][null_test]") { } TEST_CASE("permutation_sort: integers", "[permutation_sort]") { - size_t seed = Catch::rngSeed(); + unsigned int seed = Catch::rngSeed(); size_t N = 100'000; std::mt19937 g(seed); @@ -140,7 +140,7 @@ TEST_CASE("permutation_sort: integers", "[permutation_sort]") { } TEST_CASE("permutation_sort: multiple integers", "[permutation_sort]") { - size_t seed = Catch::rngSeed(); + unsigned int seed = Catch::rngSeed(); size_t N = 100'000; std::mt19937 g(seed); @@ -231,7 +231,7 @@ TEST_CASE("permutation_sort: multiple integers", "[permutation_sort]") { TEST_CASE( "permutation_sort: direct proxy_sort multiple integers", "[permutation_sort]") { - size_t seed = Catch::rngSeed(); + unsigned int seed = Catch::rngSeed(); size_t N = 100'000; std::mt19937 g(seed); From a1d753feb0a419d1b897c417af34766f3c7ae587 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 7 May 2024 11:25:02 +0200 Subject: [PATCH 327/456] Remove deprecated APIs from tests, part 1. (#4947) This removed deprecated APIs from tests. It will be split into multiple PRs to facilitate code reviewing. [sc-46307] --- TYPE: NO_HISTORY DESC: Remove deprecated APIs from tests, part 1. --- test/regression/targets/sc-12024.cc | 14 +- test/regression/targets/sc-18836.cc | 21 +- test/regression/targets/sc-24079.cc | 4 +- test/src/cpp-integration-filter-pipeline.cc | 6 +- test/src/cpp-integration-query-condition.cc | 80 +++-- .../test-capi-dimension-label-encrypted.cc | 27 -- test/src/unit-arrow.cc | 8 +- test/src/unit-backwards_compat.cc | 80 +++-- test/src/unit-capi-any.cc | 10 +- test/src/unit-capi-array.cc | 165 +++++++-- test/src/unit-capi-array_schema.cc | 12 +- test/src/unit-capi-attributes.cc | 57 ++- test/src/unit-capi-consolidation.cc | 327 +++++++++++++++--- test/src/unit-capi-dense_array.cc | 301 +++++++++++++--- test/src/unit-capi-dense_array_2.cc | 10 +- test/src/unit-capi-dense_neg.cc | 56 ++- test/src/unit-capi-dense_vector.cc | 52 ++- 17 files changed, 989 insertions(+), 241 deletions(-) diff --git a/test/regression/targets/sc-12024.cc b/test/regression/targets/sc-12024.cc index a3a74042c27f..c34c5b0cc32c 100644 --- a/test/regression/targets/sc-12024.cc +++ b/test/regression/targets/sc-12024.cc @@ -31,8 +31,11 @@ TEST_CASE("C++ API: Truncated values (ch12024)", "[cppapi][sparse]") { Query q(ctx, array, TILEDB_WRITE); q.set_layout(TILEDB_GLOBAL_ORDER); q.set_data_buffer("a", a); - coords = {139200.35, -682.75}; - q.set_coordinates(coords); + + std::vector y = {139200.35}; + std::vector z = {-682.75}; + q.set_data_buffer("Y", y); + q.set_data_buffer("Z", z); REQUIRE_THROWS_WITH( q.submit(), "[TileDB::Dimension] Error: Coordinate -682.75 is out of domain " @@ -42,8 +45,11 @@ TEST_CASE("C++ API: Truncated values (ch12024)", "[cppapi][sparse]") { Query q(ctx, array, TILEDB_WRITE); q.set_layout(TILEDB_GLOBAL_ORDER); q.set_data_buffer("a", a); - coords = {139200.34, -682.73}; - q.set_coordinates(coords); + + std::vector y = {139200.34}; + std::vector z = {-682.73}; + q.set_data_buffer("Y", y); + q.set_data_buffer("Z", z); REQUIRE_THROWS_WITH( q.submit(), "[TileDB::Dimension] Error: Coordinate 139200.34 is out of domain " diff --git a/test/regression/targets/sc-18836.cc b/test/regression/targets/sc-18836.cc index 8906f3ae8f6d..67cdd2dddc10 100644 --- a/test/regression/targets/sc-18836.cc +++ b/test/regression/targets/sc-18836.cc @@ -37,17 +37,18 @@ void create_array() { std::vector data2 = { 13.2f, 14.1f, 14.2f, 15.1f, 15.2f, 15.3f, 16.1f, 18.3f, 19.1f}; - // Set the subarray to write into - std::vector subarray = {1, 9}; - // Open array for writing Array array(ctx, array_name, TILEDB_WRITE); + // Set the subarray to write into + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 9); + Query query(ctx, array); query.set_layout(TILEDB_ROW_MAJOR) - .set_buffer("a1", data1) + .set_data_buffer("a1", data1) .set_validity_buffer("a1", a1_validity_buf) - .set_buffer("a2", data2) + .set_data_buffer("a2", data2) .set_subarray(subarray); query.submit(); @@ -69,20 +70,22 @@ TEST_CASE( // Prepare the array for reading Array array(ctx, array_name, TILEDB_READ); - const std::vector subarray = {1, 9}; - // Prepare the vectors that will hold the results std::vector a1_buffer(9); std::vector a2_buffer(9); std::vector a1_validity_buf(9); + // Set the subarray to write into + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 9); + // Prepare the query Query query(ctx, array, TILEDB_READ); query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) - .set_buffer("a1", a1_buffer) + .set_data_buffer("a1", a1_buffer) .set_validity_buffer("a1", a1_validity_buf) - .set_buffer("a2", a2_buffer); + .set_data_buffer("a2", a2_buffer); QueryCondition qc1(ctx); float val = 15.1f; diff --git a/test/regression/targets/sc-24079.cc b/test/regression/targets/sc-24079.cc index 10e92ad6fd37..3df0368e5688 100644 --- a/test/regression/targets/sc-24079.cc +++ b/test/regression/targets/sc-24079.cc @@ -75,7 +75,9 @@ TEST_CASE( std::vector a1_read(10); tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); - query.set_subarray({domain_lo, domain_hi}); + tiledb::Subarray subarray(ctx, array); + subarray.add_range(0, domain_lo, domain_hi); + query.set_subarray(subarray); query.set_data_buffer("a1", a1_read); query.set_data_buffer("d1", d1_read); query.submit(); diff --git a/test/src/cpp-integration-filter-pipeline.cc b/test/src/cpp-integration-filter-pipeline.cc index 61c9730a8896..3974003b2d05 100644 --- a/test/src/cpp-integration-filter-pipeline.cc +++ b/test/src/cpp-integration-filter-pipeline.cc @@ -1,5 +1,5 @@ /** - * @file unit-filter-pipeline.cc + * @file cpp-integration-filter-pipeline.cc * * @section LICENSE * @@ -118,7 +118,9 @@ TEST_CASE( std::vector a1_read(10); tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); - query.set_subarray({domain_lo, domain_hi}); + tiledb::Subarray sub(ctx, array); + sub.set_subarray({domain_lo, domain_hi}); + query.set_subarray(sub); query.set_data_buffer("a1", a1_read); query.set_data_buffer("d1", d1_read); query.submit(); diff --git a/test/src/cpp-integration-query-condition.cc b/test/src/cpp-integration-query-condition.cc index 6f22ca1df321..017a04cdc05d 100644 --- a/test/src/cpp-integration-query-condition.cc +++ b/test/src/cpp-integration-query-condition.cc @@ -224,10 +224,12 @@ void create_array( .set_data_buffer("a", a_data_read) .set_data_buffer("b", b_data_read); + Subarray subarray1(ctx, array1); if (array_type == TILEDB_DENSE) { int range[] = {1, num_rows}; - query1.add_range("rows", range[0], range[1]) + subarray1.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query1.set_subarray(subarray1); } query1.submit(); @@ -366,10 +368,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -430,10 +434,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -559,10 +565,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -686,8 +694,10 @@ TEST_CASE( // Define range. int range[] = {2, 3}; - query.add_range("rows", range[0], range[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query(a_data_read_2, b_data_read_2, qc, params.layout_, query); @@ -816,8 +826,10 @@ TEST_CASE( // Define range. int range[] = {2, 3}; int range1[] = {7, 10}; - query.add_range("rows", range1[0], range1[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range1[0], range1[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query(a_data_read_2, b_data_read_2, qc, params.layout_, query); @@ -955,8 +967,10 @@ TEST_CASE( // Define range. int range[] = {2, 3}; int range1[] = {7, 10}; - query.add_range("rows", range[0], range[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range1[0], range1[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query(a_data_read_2, b_data_read_2, qc, params.layout_, query); @@ -1079,8 +1093,10 @@ TEST_CASE( // Define range. int range[] = {7, 14}; - query.add_range("rows", range[0], range[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query(a_data_read_2, b_data_read_2, qc, params.layout_, query); @@ -1241,8 +1257,10 @@ TEST_CASE( // Define range. int range[] = {7, 14}; - query.add_range("rows", range[0], range[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query(a_data_read_2, b_data_read_2, qc, params.layout_, query); @@ -1413,8 +1431,10 @@ TEST_CASE( // Define range. int range[] = {7, 14}; - query.add_range("rows", range[0], range[1]) + Subarray subarray(ctx2, array); + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); // Perform query and validate. perform_query( @@ -1542,7 +1562,9 @@ TEST_CASE( std::vector vals = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}; Query query_w(ctx, array, TILEDB_WRITE); int range[] = {1, 10}; - query_w.add_range("d", range[0], range[1]); + Subarray subarray_w(ctx, array); + subarray_w.add_range("d", range[0], range[1]); + query_w.set_subarray(subarray_w); query_w.set_layout(TILEDB_ROW_MAJOR); query_w.set_buffer("a", vals); REQUIRE(query_w.submit() == Query::Status::COMPLETE); @@ -1551,7 +1573,9 @@ TEST_CASE( std::vector vals2 = {7, 7, 7, 7}; Query query_w2(ctx, array, TILEDB_WRITE); int range2[] = {3, 6}; - query_w2.add_range("d", range2[0], range2[1]); + Subarray subarray_w2(ctx, array); + subarray_w2.add_range("d", range2[0], range2[1]); + query_w2.set_subarray(subarray_w2); query_w2.set_layout(TILEDB_ROW_MAJOR); query_w2.set_buffer("a", vals2); REQUIRE(query_w2.submit() == Query::Status::COMPLETE); @@ -1568,7 +1592,9 @@ TEST_CASE( std::vector vals_read(10); Query query_r(ctx, array_r, TILEDB_READ); - query_r.add_range("d", range[0], range[1]); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range("d", range[0], range[1]); + query_r.set_subarray(subarray_r); query_r.set_layout(TILEDB_ROW_MAJOR); query_r.set_buffer("a", vals_read); query_r.set_condition(qc); @@ -1657,10 +1683,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -1801,10 +1829,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -1959,10 +1989,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -2101,10 +2133,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -2281,10 +2315,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -2505,10 +2541,12 @@ TEST_CASE( Query query(ctx2, array); // Set a subarray for dense. + Subarray subarray(ctx2, array); if (params.array_type_ == TILEDB_DENSE) { int range[] = {1, num_rows}; - query.add_range("rows", range[0], range[1]) + subarray.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query.set_subarray(subarray); } // Perform query and validate. @@ -2611,7 +2649,9 @@ TEST_CASE( std::vector rlabs(10); Query rquery(ctx, rarray, TILEDB_READ); - rquery.set_subarray(subarray) + Subarray sub(ctx, rarray); + sub.set_subarray(subarray); + rquery.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("rows", rrows) .set_data_buffer("labs", rlabs); diff --git a/test/src/test-capi-dimension-label-encrypted.cc b/test/src/test-capi-dimension-label-encrypted.cc index 22233a9ace9d..b0622cb0b0b2 100644 --- a/test/src/test-capi-dimension-label-encrypted.cc +++ b/test/src/test-capi-dimension-label-encrypted.cc @@ -82,20 +82,6 @@ TEST_CASE_METHOD( tiledb_array_create(ctx_encrypt, array_name.c_str(), array_schema)); tiledb_array_schema_free(&array_schema); } - SECTION("Create array with deprecated API") { - tiledb_encryption_type_t encryption_type{TILEDB_AES_256_GCM}; - std::string encryption_key{"0123456789abcdeF0123456789abcdeF"}; - const uint32_t encryption_key_length{ - static_cast(strlen(encryption_key.c_str()))}; - require_tiledb_ok(tiledb_array_create_with_key( - ctx, - array_name.c_str(), - array_schema, - encryption_type, - encryption_key.c_str(), - encryption_key_length)); - tiledb_array_schema_free(&array_schema); - } // Check the array schema cannot be loaded without the encryption key. tiledb_array_schema_t* loaded_array_schema{nullptr}; @@ -178,19 +164,6 @@ TEST_CASE_METHOD( tiledb_array_create(ctx_encrypt, array_name.c_str(), array_schema)); tiledb_array_schema_free(&array_schema); } - SECTION("Create array with deprecated API") { - tiledb_encryption_type_t encryption_type{TILEDB_AES_256_GCM}; - const uint32_t encryption_key_length{ - static_cast(strlen(encryption_key.c_str()))}; - require_tiledb_ok(tiledb_array_create_with_key( - ctx, - array_name.c_str(), - array_schema, - encryption_type, - encryption_key.c_str(), - encryption_key_length)); - tiledb_array_schema_free(&array_schema); - } // Open array for writing. tiledb_array_t* array; diff --git a/test/src/unit-arrow.cc b/test/src/unit-arrow.cc index 333934abedd7..0b126f894e21 100644 --- a/test/src/unit-arrow.cc +++ b/test/src/unit-arrow.cc @@ -255,7 +255,9 @@ void test_for_column_size(size_t col_size) { Query query(ctx, array); query.set_layout(TILEDB_COL_MAJOR); int32_t range_max = static_cast(col_size > 0 ? col_size - 1 : 0); - query.add_range(0, (int32_t)0, (int32_t)range_max); + Subarray subarray(ctx, array); + subarray.add_range(0, (int32_t)0, (int32_t)range_max); + query.set_subarray(subarray); std::vector vec_array; std::vector vec_schema; @@ -315,7 +317,9 @@ void test_for_column_size(size_t col_size) { Query query(ctx, array); query.set_layout(TILEDB_COL_MAJOR); int32_t range_max = static_cast(col_size > 0 ? col_size - 1 : 0); - query.add_range(0, static_cast(0), range_max); + Subarray subarray(ctx, array); + subarray.add_range(0, static_cast(0), range_max); + query.set_subarray(subarray); allocate_query_buffers(&query); query.submit(); diff --git a/test/src/unit-backwards_compat.cc b/test/src/unit-backwards_compat.cc index 09778fd6f62a..e72bd4a161ed 100644 --- a/test/src/unit-backwards_compat.cc +++ b/test/src/unit-backwards_compat.cc @@ -60,6 +60,8 @@ static const std::string groups_dir = template void set_query_coords( + Context& ctx, + Array* array, const Domain& domain, Query* query, void** coordinates, @@ -78,8 +80,11 @@ void set_query_coords( static_cast(*expected_coordinates)[i] = 1; } + Subarray sub(ctx, *array); + sub.set_subarray(static_cast(subarray), 2 * ndim); + query->set_coordinates(static_cast(*coordinates), ndim); - query->set_subarray(static_cast(subarray), 2 * ndim); + query->set_subarray(sub); std::free(subarray); } @@ -89,6 +94,7 @@ void set_query_dimension_buffer( const Domain& domain, const uint64_t dim_idx, Query* query, + Subarray* subarray, void** buffer, void** expected_buffer) { Dimension dimension = domain.dimension(dim_idx); @@ -101,8 +107,9 @@ void set_query_dimension_buffer( static_cast(*expected_buffer)[0] = 1; auto dom = dimension.domain(); - query->set_buffer(dimension.name(), static_cast(*buffer), buffer_size); - query->add_range(dim_idx, dom.first, dom.second); + query->set_data_buffer( + dimension.name(), static_cast(*buffer), buffer_size); + subarray->add_range(dim_idx, dom.first, dom.second); } template @@ -110,6 +117,7 @@ void set_query_var_dimension_buffer( const Domain& domain, const uint64_t dim_idx, Query* query, + Subarray* subarray, uint64_t** offsets, void** buffer, uint64_t** expected_offsets, @@ -130,7 +138,7 @@ void set_query_var_dimension_buffer( memset(*buffer, 0, buffer_size); query->set_buffer( dimension.name(), *offsets, 1, static_cast(*buffer), buffer_size); - query->add_range(dim_idx, std::string("1"), std::string("1")); + subarray->add_range(dim_idx, std::string("1"), std::string("1")); } } // namespace @@ -195,7 +203,9 @@ TEST_CASE( std::vector coords_read(8); Query query_r(ctx, array); - query_r.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read) .set_coordinates(coords_read); @@ -501,43 +511,43 @@ TEST_CASE( switch (domain.type()) { case TILEDB_INT8: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_UINT8: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_INT16: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_UINT16: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_INT32: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_UINT32: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_INT64: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_UINT64: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_FLOAT32: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_FLOAT64: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; case TILEDB_DATETIME_YEAR: case TILEDB_DATETIME_MONTH: @@ -562,7 +572,7 @@ TEST_CASE( case TILEDB_TIME_FS: case TILEDB_TIME_AS: set_query_coords( - domain, query, &coordinates, &expected_coordinates); + ctx, array, domain, query, &coordinates, &expected_coordinates); break; default: REQUIRE(false); @@ -721,7 +731,9 @@ TEST_CASE( Array array(ctx, old_array_name, TILEDB_READ); Query query_r(ctx, array); - query_r.set_subarray(subarray) + Subarray subarray_r(ctx, array); + subarray_r.set_subarray(subarray); + query_r.set_subarray(subarray_r) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read) .set_coordinates(coords_read); @@ -781,6 +793,8 @@ TEST_CASE( auto query = new Query(encrypted ? ctx_encrypt : ctx, *array); + auto subarray = new Subarray(ctx, *array); + std::unordered_map> buffers; for (auto attr : array->schema().attributes()) { @@ -1020,43 +1034,43 @@ TEST_CASE( switch (dim.type()) { case TILEDB_INT8: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_UINT8: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_INT16: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_UINT16: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_INT32: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_UINT32: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_INT64: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_UINT64: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_FLOAT32: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_FLOAT64: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_DATETIME_YEAR: case TILEDB_DATETIME_MONTH: @@ -1081,13 +1095,14 @@ TEST_CASE( case TILEDB_TIME_FS: case TILEDB_TIME_AS: set_query_dimension_buffer( - domain, i, query, &buffer, &expected_results); + domain, i, query, subarray, &buffer, &expected_results); break; case TILEDB_STRING_ASCII: { set_query_var_dimension_buffer( domain, i, query, + subarray, &offsets, &buffer, &expected_offsets, @@ -1104,6 +1119,7 @@ TEST_CASE( } // Submit query + query->set_subarray(*subarray); query->submit(); delete query; @@ -1263,7 +1279,9 @@ TEST_CASE( std::vector d2_read1(4); Query query_read1(ctx, array_read1); - query_read1.set_subarray(subarray_read1) + Subarray subarray_r(ctx, array_read1); + subarray_r.set_subarray(subarray_read1); + query_read1.set_subarray(subarray_r) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read1) .set_data_buffer("d1", d1_read1) @@ -1312,7 +1330,9 @@ TEST_CASE( std::vector d2_read2(4); Query query_read2(ctx, array_read2); - query_read2.set_subarray(subarray_read2) + Subarray subarray_r2(ctx, array_read2); + subarray_r2.set_subarray(subarray_read2); + query_read2.set_subarray(subarray_r2) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read2) .set_data_buffer("d1", d1_read2) diff --git a/test/src/unit-capi-any.cc b/test/src/unit-capi-any.cc index d740e9d33150..6ae61323a1b2 100644 --- a/test/src/unit-capi-any.cc +++ b/test/src/unit-capi-any.cc @@ -192,7 +192,7 @@ void AnyFx::read_array(const std::string& array_name) { // Get maximum buffer sizes uint64_t size_off = 32; uint64_t size_val = 32; - uint64_t subarray[] = {1, 4}; + uint64_t sub[] = {1, 4}; // Prepare cell buffers auto buffer_a1_off = new uint64_t[size_off / sizeof(uint64_t)]; @@ -202,6 +202,9 @@ void AnyFx::read_array(const std::string& array_name) { tiledb_query_t* query; rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx, query, "a1", buffer_a1_val, &size_val); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_offsets_buffer( @@ -209,7 +212,9 @@ void AnyFx::read_array(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx, query, subarray); + rc = tiledb_subarray_set_subarray(ctx, subarray, sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx, query, subarray); REQUIRE(rc == TILEDB_OK); // Submit query @@ -243,6 +248,7 @@ void AnyFx::read_array(const std::string& array_name) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); delete[] buffer_a1_off; delete[] buffer_a1_val; } diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 3f8633119bbd..0fe991525716 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -978,12 +978,19 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_upd, &buffer_upd_size); @@ -996,6 +1003,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); std::vector fragment_timestamps; rc = tiledb_vfs_ls( @@ -1035,13 +1043,19 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Submit query + // Create subarray int64_t subarray_read[] = {1, 10}; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1054,6 +1068,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); // Check correctness int32_t buffer_read_c[] = {1, 2, 3, 4, 50, 60, 70, 8, 9, 10}; @@ -1094,12 +1109,18 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); CHECK(timestamp_get == 0); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1112,6 +1133,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1145,12 +1167,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1163,6 +1191,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1202,12 +1231,18 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); CHECK(timestamp_get == fragment_timestamps[1]); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1217,6 +1252,7 @@ TEST_CASE_METHOD( // Clean up but don't close the array yet (we will reopen it). tiledb_query_free(&query); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1233,12 +1269,18 @@ TEST_CASE_METHOD( rc = tiledb_array_reopen(ctx_, array); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1248,6 +1290,7 @@ TEST_CASE_METHOD( // Clean up but don't close the array yet (we will reopen it). tiledb_query_free(&query); + tiledb_subarray_free(&sub); // Check correctness int buffer_read_reopen_c[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -1267,12 +1310,18 @@ TEST_CASE_METHOD( rc = tiledb_array_reopen(ctx_, array); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1285,6 +1334,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&sub); // Check correctness int buffer_read_reopen_start_c[] = { @@ -1331,12 +1381,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1349,6 +1405,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1395,12 +1452,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1413,6 +1476,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1558,13 +1622,20 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); CHECK(timestamp_get == 0); + // Create subarray + int64_t subarray_read[] = {1, 10}; + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - int64_t subarray_read[] = {1, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); int buffer_read[10]; uint64_t buffer_read_size = sizeof(buffer_read); @@ -1579,6 +1650,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1612,12 +1684,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, "a", buffer_read, &buffer_read_size); @@ -1630,6 +1708,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); tiledb_config_free(&cfg); // Check correctness @@ -1917,9 +1996,9 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( ArrayFx, - "C API: Test query errors, getting subarray info from write queries in " + "C API: Test subarray errors, getting subarray info from write queries in " "sparse arrays", - "[capi][query][error][sparse][rest]") { + "[capi][subarray][error][sparse][rest]") { std::string temp_dir = fs_vec_[0]->temp_dir(); std::string array_name = vfs_array_uri(fs_vec_[0], temp_dir + "query_error_sparse"); @@ -1937,17 +2016,20 @@ TEST_CASE_METHOD( // Prepare query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); + + // Prepare subarray + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); uint64_t range_num; - // Note: deprecated function. - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Note: deprecated function. const void *start, *end, *stride; - rc = tiledb_query_get_range(ctx_, query, 0, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*static_cast(start) == -1); CHECK(*static_cast(end) == 2); @@ -1955,10 +2037,12 @@ TEST_CASE_METHOD( int64_t s = 10; int64_t e = 20; - rc = tiledb_query_add_range(ctx_, query, 0, &s, &e, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s, &e, nullptr); CHECK(rc == TILEDB_ERR); - int64_t subarray[] = {-1, 2}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + int64_t sub[] = {-1, 2}; + rc = tiledb_subarray_set_subarray(ctx_, subarray, sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_ERR); // Close array @@ -1968,6 +2052,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); remove_temp_dir(temp_dir); } @@ -1993,45 +2078,50 @@ TEST_CASE_METHOD( int32_t a[] = {1, 2, 3, 4}; uint64_t a_size = sizeof(a); - // Prepare query + // Prepare query/subarray tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // The default const void *start, *end, *stride; - rc = tiledb_query_get_range(ctx_, query, 0, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(const uint64_t*)start == 1); CHECK(*(const uint64_t*)end == 10); int64_t s = 1; int64_t e = 2; - rc = tiledb_query_add_range(ctx_, query, 0, &s, &e, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s, &e, nullptr); CHECK(rc == TILEDB_OK); - int64_t subarray[] = {2, 3, 4, 5}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + int64_t sub[] = {2, 3, 4, 5}; + rc = tiledb_subarray_set_subarray(ctx_, subarray, sub); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s, &e, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s, &e, nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 2); - rc = tiledb_query_get_range(ctx_, query, 0, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(const uint64_t*)start == 2); CHECK(*(const uint64_t*)end == 3); - rc = tiledb_query_get_range(ctx_, query, 1, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 1, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(const uint64_t*)start == 4); CHECK(*(const uint64_t*)end == 5); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_ERR); @@ -2042,6 +2132,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); remove_temp_dir(temp_dir); } @@ -2113,16 +2204,21 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); - int64_t subarray[] = {2, 3, 4, 5}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + int64_t sub[] = {2, 3, 4, 5}; + rc = tiledb_subarray_set_subarray(ctx_, subarray, sub); CHECK(rc == TILEDB_OK); int64_t s = 1; int64_t e = 2; - rc = tiledb_query_add_range(ctx_, query, 0, &s, &e, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s, &e, nullptr); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_ERR); @@ -2133,6 +2229,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); remove_temp_dir(temp_dir); } diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 6641e5b2eff1..e04735b5ac41 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -1988,7 +1988,7 @@ TEST_CASE_METHOD( int is_empty = false; rc = tiledb_array_get_non_empty_domain_wrapper(ctx_, array, dom, &is_empty); REQUIRE(rc == TILEDB_ERR); - void* subarray = nullptr; + void* sub = nullptr; // Get non-empty domain per dimension dom = nullptr; @@ -2007,12 +2007,17 @@ TEST_CASE_METHOD( ctx_, array, "d2", dom, &is_empty); REQUIRE(rc == TILEDB_OK); + // Subarray checks + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, subarray, sub); + REQUIRE(rc == TILEDB_ERR); + // Query checks tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); - REQUIRE(rc == TILEDB_ERR); void* buff = nullptr; uint64_t size = 1024; rc = tiledb_query_set_data_buffer(ctx_, query, "buff", buff, &size); @@ -2031,6 +2036,7 @@ TEST_CASE_METHOD( tiledb_domain_free(&read_dom); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); tiledb_array_schema_free(&array_schema); remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-capi-attributes.cc b/test/src/unit-capi-attributes.cc index 2c350bc03858..f2cd264df0af 100644 --- a/test/src/unit-capi-attributes.cc +++ b/test/src/unit-capi-attributes.cc @@ -195,13 +195,20 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_a1, &buffer_a1_size); @@ -214,6 +221,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); int buffer_read[10]; uint64_t buffer_read_size = sizeof(buffer_read); @@ -224,12 +232,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); @@ -243,6 +257,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); // Check correctness int buffer_read_c[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -292,13 +307,20 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_write, &buffer_write_size); @@ -311,6 +333,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); uint64_t ts_open = 0; if (datatype == TILEDB_BLOB) { @@ -401,12 +424,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); @@ -419,6 +448,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); // Check correctness CHECK(!std::memcmp(buffer_read, buffer_write, buffer_write_size)); @@ -473,13 +503,20 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_write, &buffer_write_size); @@ -492,6 +529,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); int buffer_read[10]; uint64_t buffer_read_size = sizeof(buffer_read); @@ -502,12 +540,18 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + // Submit query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attr_name.c_str(), buffer_read, &buffer_read_size); @@ -520,6 +564,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&sub); // Check correctness CHECK(!std::memcmp(buffer_read, buffer_write, buffer_write_size)); diff --git a/test/src/unit-capi-consolidation.cc b/test/src/unit-capi-consolidation.cc index 1699d8fea73c..3cb1459e5cfb 100644 --- a/test/src/unit-capi-consolidation.cc +++ b/test/src/unit-capi-consolidation.cc @@ -733,6 +733,8 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 tiledb_query_t* query_1; uint64_t subarray[] = {1, 200}; @@ -740,8 +742,13 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -781,8 +788,13 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -821,8 +833,13 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -862,8 +879,13 @@ void ConsolidationFx::write_dense_vector_4_fragments(uint64_t timestamp) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -908,6 +930,8 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 tiledb_query_t* query_1; uint64_t subarray[] = {1, 198}; @@ -915,8 +939,13 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -930,8 +959,13 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -945,8 +979,13 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -960,8 +999,13 @@ void ConsolidationFx::write_dense_vector_4_fragments_not_coinciding() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -1006,6 +1050,8 @@ void ConsolidationFx:: rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 tiledb_query_t* query_1; uint64_t subarray[] = {1, 200}; @@ -1013,8 +1059,13 @@ void ConsolidationFx:: CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1028,8 +1079,13 @@ void ConsolidationFx:: CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1043,8 +1099,13 @@ void ConsolidationFx:: CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -1058,8 +1119,13 @@ void ConsolidationFx:: CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -1114,6 +1180,8 @@ void ConsolidationFx::write_dense_vector_consolidatable_1() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 tiledb_query_t* query_1; uint64_t subarray[] = {1, 100}; @@ -1121,8 +1189,13 @@ void ConsolidationFx::write_dense_vector_consolidatable_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1137,8 +1210,13 @@ void ConsolidationFx::write_dense_vector_consolidatable_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1153,8 +1231,13 @@ void ConsolidationFx::write_dense_vector_consolidatable_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -1204,6 +1287,8 @@ void ConsolidationFx::write_dense_vector_consolidatable_2() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 tiledb_query_t* query_1; uint64_t subarray[] = {1, 100}; @@ -1211,8 +1296,13 @@ void ConsolidationFx::write_dense_vector_consolidatable_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1227,8 +1317,13 @@ void ConsolidationFx::write_dense_vector_consolidatable_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1282,6 +1377,8 @@ void ConsolidationFx::write_dense_vector_del_1() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 - Dense tiledb_query_t* query_1; uint64_t subarray[] = {1, 200}; @@ -1289,8 +1386,13 @@ void ConsolidationFx::write_dense_vector_del_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1304,8 +1406,13 @@ void ConsolidationFx::write_dense_vector_del_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1319,8 +1426,13 @@ void ConsolidationFx::write_dense_vector_del_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray3); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray3); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -1335,8 +1447,13 @@ void ConsolidationFx::write_dense_vector_del_1() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -1390,6 +1507,8 @@ void ConsolidationFx::write_dense_vector_del_2() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 - Dense tiledb_query_t* query_1; uint64_t subarray[] = {1, 200}; @@ -1397,8 +1516,13 @@ void ConsolidationFx::write_dense_vector_del_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1412,8 +1536,13 @@ void ConsolidationFx::write_dense_vector_del_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1427,8 +1556,13 @@ void ConsolidationFx::write_dense_vector_del_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray3); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray3); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -1443,8 +1577,13 @@ void ConsolidationFx::write_dense_vector_del_2() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -1498,6 +1637,8 @@ void ConsolidationFx::write_dense_vector_del_3() { rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Submit query #1 - Dense tiledb_query_t* query_1; uint64_t subarray[] = {1, 200}; @@ -1505,8 +1646,13 @@ void ConsolidationFx::write_dense_vector_del_3() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -1520,8 +1666,13 @@ void ConsolidationFx::write_dense_vector_del_3() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_2, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, subarray2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray2); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_2, "a", a_2, &a_2_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_2); @@ -1536,8 +1687,13 @@ void ConsolidationFx::write_dense_vector_del_3() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_3, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_3, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_3, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_3, "a", a_3, &a_3_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_3); @@ -1551,8 +1707,13 @@ void ConsolidationFx::write_dense_vector_del_3() { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray4); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray4); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -1747,6 +1908,8 @@ void ConsolidationFx::write_dense_subarray( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_t* sub; + // Create query tiledb_query_t* query; const char* attributes[] = {"a1", "a2", "a3"}; @@ -1755,8 +1918,13 @@ void ConsolidationFx::write_dense_subarray( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); @@ -2587,14 +2755,21 @@ void ConsolidationFx::read_dense_vector(uint64_t timestamp) { int a[410]; uint64_t a_size = sizeof(a); + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2659,14 +2834,21 @@ void ConsolidationFx::read_dense_vector_with_gaps() { int a[410]; uint64_t a_size = sizeof(a); + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2728,14 +2910,21 @@ void ConsolidationFx::read_dense_vector_consolidatable_1() { uint64_t a_size = sizeof(a); uint64_t subarray[] = {1, 101}; + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2800,14 +2989,21 @@ void ConsolidationFx::read_dense_vector_consolidatable_2() { uint64_t a_size = sizeof(a); uint64_t subarray[] = {1, 300}; + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2868,14 +3064,21 @@ void ConsolidationFx::read_dense_vector_del_1() { uint64_t a_size = sizeof(a); uint64_t subarray[] = {1, 400}; + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2936,14 +3139,21 @@ void ConsolidationFx::read_dense_vector_del_2() { uint64_t a_size = sizeof(a); uint64_t subarray[] = {1, 400}; + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -3009,14 +3219,21 @@ void ConsolidationFx::read_dense_vector_del_3() { uint64_t a_size = sizeof(a); uint64_t subarray[] = {1, 400}; + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -3101,14 +3318,21 @@ void ConsolidationFx::read_dense_full_subarray() { auto buffer_a2_validity = (uint8_t*)malloc(buffer_a2_validity_size); auto buffer_a3 = (float*)malloc(buffer_a3_size); + tiledb_subarray_t* sub; + // Create query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -3224,14 +3448,21 @@ void ConsolidationFx::read_dense_subarray_full() { auto buffer_a2_validity = (uint8_t*)malloc(buffer_a2_validity_size); auto buffer_a3 = (float*)malloc(buffer_a3_size); + tiledb_subarray_t* sub; + // Create query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -3354,14 +3585,21 @@ void ConsolidationFx::read_dense_four_tiles() { auto buffer_a2_validity = (uint8_t*)malloc(buffer_a2_validity_size); auto buffer_a3 = (float*)malloc(buffer_a3_size); + tiledb_subarray_t* sub; + // Create query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -6015,14 +6253,21 @@ TEST_CASE_METHOD( int a[410]; uint64_t a_size = sizeof(a); + tiledb_subarray_t* sub; + // Submit query tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); diff --git a/test/src/unit-capi-dense_array.cc b/test/src/unit-capi-dense_array.cc index 6c0249be287c..caba13b314bc 100644 --- a/test/src/unit-capi-dense_array.cc +++ b/test/src/unit-capi-dense_array.cc @@ -512,13 +512,19 @@ int* DenseArrayFx::read_dense_array_2D( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, query_type, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE(rc == TILEDB_OK); @@ -605,13 +611,19 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { // Submit query #1 - Dense tiledb_query_t* query_1; + tiledb_subarray_t* sub; uint64_t subarray[] = {1, 300}; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query_1); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_1, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_1, "a", a_1, &a_1_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_1); @@ -656,8 +668,13 @@ void DenseArrayFx::write_dense_vector_mixed(const std::string& array_name) { CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query_4, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_4, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_4, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query_4, "a", a_4, &a_4_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query_4); @@ -798,12 +815,19 @@ void DenseArrayFx::write_dense_subarray_2D( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, query_type, &query); REQUIRE_SAFE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE_SAFE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE_SAFE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE_SAFE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE_SAFE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE_SAFE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE_SAFE(rc == TILEDB_OK); @@ -848,13 +872,19 @@ void DenseArrayFx::write_dense_subarray_2D_with_cancel( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, query_type, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE(rc == TILEDB_OK); @@ -1030,6 +1060,7 @@ void DenseArrayFx::check_sorted_reads() { // Check out of bounds subarray tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); @@ -1046,20 +1077,45 @@ void DenseArrayFx::check_sorted_reads() { REQUIRE(rc == TILEDB_OK); int64_t subarray_1[] = {-1, 5, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_1); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_1); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); int64_t subarray_2[] = {0, 5000000, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); int64_t subarray_3[] = {0, 5, -1, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_3); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_3); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); int64_t subarray_4[] = {0, 5, 10, 100000000}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_4); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_4); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); int64_t subarray_5[] = {0, 5, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_5); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_5); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -1827,6 +1883,7 @@ void DenseArrayFx::write_partial_dense_array(const std::string& array_name) { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); @@ -1844,8 +1901,13 @@ void DenseArrayFx::write_partial_dense_array(const std::string& array_name) { ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {3, 4, 3, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit_and_finalize(ctx_, query); CHECK(rc == TILEDB_OK); @@ -1966,12 +2028,18 @@ void DenseArrayFx::read_dense_vector_mixed(const std::string& array_name) { // Submit query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -2053,12 +2121,18 @@ void DenseArrayFx::read_dense_array_with_coords_full_global( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2199,12 +2273,18 @@ void DenseArrayFx::read_dense_array_with_coords_full_row( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2343,12 +2423,18 @@ void DenseArrayFx::read_dense_array_with_coords_full_col( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2489,12 +2575,18 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_global( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2634,12 +2726,18 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_row( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2779,12 +2877,18 @@ void DenseArrayFx::read_dense_array_with_coords_subarray_col( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size); CHECK(rc == TILEDB_OK); @@ -2888,12 +2992,17 @@ void DenseArrayFx::read_large_dense_array( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, layout); CHECK(rc == TILEDB_OK); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); for (auto& range : ranges) { - rc = tiledb_query_add_range( - ctx_, query, range.i_, &range.min_, &range.max_, nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, range.i_, &range.min_, &range.max_, nullptr); REQUIRE(rc == TILEDB_OK); } + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); uint64_t buffer_a1_size = a1.size() * sizeof(uint64_t); rc = tiledb_query_set_data_buffer( @@ -2919,6 +3028,7 @@ void DenseArrayFx::read_large_dense_array( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void DenseArrayFx::check_non_empty_domain() { @@ -3113,11 +3223,17 @@ TEST_CASE_METHOD( // Allocate query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {1, 4, 1, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Aux variables uint64_t off[1]; @@ -3292,14 +3408,20 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Submit a read query + tiledb_subarray_t* sub; tiledb_query_t* query_1; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query_1); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query_1, "a1", &a1_buffer, &a1_buff_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_1, &subarray[0]); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_1, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query_1); CHECK(rc == TILEDB_OK); @@ -3320,8 +3442,13 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer( ctx_, query_2, "a1", &a1_buffer, &a1_buff_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query_2, &subarray[0]); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query_2, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query_2); CHECK(rc == TILEDB_OK); @@ -3405,6 +3532,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -3413,8 +3541,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[0], write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {2, 3, 1, 2}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -3451,8 +3584,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); @@ -3489,6 +3627,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -3497,8 +3636,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[0], write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {2, 3, 2, 3}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -3535,8 +3679,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); @@ -3571,6 +3720,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -3579,8 +3729,13 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx_, query, "a1", write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray_1[] = {1, 2, 1, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_1); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_1); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -3603,8 +3758,13 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx_, query, "a1", write_a2, &write_a2_size); CHECK(rc == TILEDB_OK); uint64_t subarray_2[] = {2, 3, 2, 3}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -3641,8 +3801,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", read_a, &read_a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -3777,10 +3942,16 @@ TEST_CASE_METHOD( uint64_t a2_size = sizeof(a2); int64_t subarray[] = {3, 4, 2, 4}; tiledb_query_t* wq2; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &wq2); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, wq2, subarray); - REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, wq2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, wq2, "a1", a2, &a2_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, wq2, TILEDB_ROW_MAJOR); @@ -3808,8 +3979,13 @@ TEST_CASE_METHOD( tiledb_query_t* wq3; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &wq3); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, wq3, subarray_2); - REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, wq3, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, wq3, "a1", a3, &a3_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, wq3, TILEDB_COL_MAJOR); @@ -3842,8 +4018,13 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); REQUIRE(rc == TILEDB_OK); @@ -3927,13 +4108,20 @@ TEST_CASE_METHOD( rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - int64_t range[] = {0, 4}; - rc = tiledb_query_add_range(ctx_, query, 0, &range[0], &range[1], nullptr); + int64_t range[] = {1, 4}; + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &range[0], &range[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &range[0], &range[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &range[0], &range[1], nullptr); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Read all the data. tiledb_query_status_t status; @@ -3953,6 +4141,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); tiledb_config_free(&config); remove_temp_dir(temp_dir); @@ -3980,17 +4169,24 @@ TEST_CASE_METHOD( rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + int a1[16]; uint64_t a1_size = sizeof(a1); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - int64_t range[] = {0, 4}; - rc = tiledb_query_add_range(ctx_, query, 0, &range[0], &range[1], nullptr); + int64_t range[] = {1, 4}; + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &range[0], &range[1], nullptr); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", a1, &a1_size); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -4006,6 +4202,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); remove_temp_dir(temp_dir); } @@ -4036,6 +4233,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -4050,8 +4248,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[1], write_a2_offs, &write_a2_offs_size); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {1, 3, 3, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); tiledb_query_free(&query); @@ -4077,8 +4280,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[1], write_a2_offs_2, &write_a2_offs_size_2); CHECK(rc == TILEDB_OK); uint64_t subarray_2[] = {4, 5, 1, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -4106,8 +4314,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); rc = tiledb_query_set_offsets_buffer( diff --git a/test/src/unit-capi-dense_array_2.cc b/test/src/unit-capi-dense_array_2.cc index 9b2db72f18ac..4a5f9eb18a70 100644 --- a/test/src/unit-capi-dense_array_2.cc +++ b/test/src/unit-capi-dense_array_2.cc @@ -492,10 +492,14 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); // Set a multi-range subarray + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array_, &subarray); uint64_t start = 1, end = 1; - rc = tiledb_query_add_range(ctx_, query, 0, &start, &end, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start, &end, nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &start, &end, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start, &end, nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Submit @@ -504,4 +508,6 @@ TEST_CASE_METHOD( // Clean up close_array(ctx_, array_); + tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } diff --git a/test/src/unit-capi-dense_neg.cc b/test/src/unit-capi-dense_neg.cc index 100d8e821cd4..469e9346e625 100644 --- a/test/src/unit-capi-dense_neg.cc +++ b/test/src/unit-capi-dense_neg.cc @@ -253,14 +253,20 @@ void DenseNegFx::write_dense_array_global(const std::string& path) { int a[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; uint64_t a_size = sizeof(a); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit_and_finalize(ctx_, query); @@ -287,10 +293,16 @@ void DenseNegFx::write_dense_array_row(const std::string& path) { uint64_t a_size = sizeof(a); int64_t subarray[] = {0, 1, -1, 1}; tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -320,10 +332,16 @@ void DenseNegFx::write_dense_array_col(const std::string& path) { uint64_t a_size = sizeof(a); int64_t subarray[] = {-2, -1, -1, 0}; tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); @@ -353,10 +371,16 @@ void DenseNegFx::read_dense_vector(const std::string& path) { int64_t subarray[] = {-1, 2}; uint64_t a_size = sizeof(a); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -390,14 +414,20 @@ void DenseNegFx::read_dense_array_global(const std::string& path) { int a[16]; uint64_t a_size = sizeof(a); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -427,14 +457,20 @@ void DenseNegFx::read_dense_array_row(const std::string& path) { int a[16]; uint64_t a_size = sizeof(a); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -465,14 +501,20 @@ void DenseNegFx::read_dense_array_col(const std::string& path) { int a[16]; uint64_t a_size = sizeof(a); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); diff --git a/test/src/unit-capi-dense_vector.cc b/test/src/unit-capi-dense_vector.cc index a1a78a006d57..ac65d782165f 100644 --- a/test/src/unit-capi-dense_vector.cc +++ b/test/src/unit-capi-dense_vector.cc @@ -194,6 +194,7 @@ void DenseVectorFx::check_read( void* read_buffers[] = {buffer}; uint64_t read_buffer_sizes[] = {sizeof(buffer)}; tiledb_query_t* read_query = nullptr; + tiledb_subarray_t* sub; const char* attributes[] = {ATTR_NAME.c_str()}; // Open array @@ -210,8 +211,13 @@ void DenseVectorFx::check_read( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, read_query, layout); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, read_query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, read_query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, read_query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, read_query); @@ -246,6 +252,7 @@ void DenseVectorFx::check_update(const std::string& path) { // Update tiledb_query_t* update_query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &update_query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -257,8 +264,13 @@ void DenseVectorFx::check_update(const std::string& path) { REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, update_query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, update_query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, update_query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, update_query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, update_query); @@ -287,8 +299,13 @@ void DenseVectorFx::check_update(const std::string& path) { REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, read_query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, read_query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, read_query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, read_query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, read_query); @@ -434,6 +451,7 @@ TEST_CASE_METHOD( for (int i = 0; i < 20; ++i) write_buffer_0[i] = 0; tiledb_query_t* write_query_0; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &write_query_0); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -445,8 +463,13 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, write_query_0, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, write_query_0, write_subarray_0); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, write_subarray_0); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, write_query_0, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, write_query_0); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, write_query_0); @@ -486,8 +509,13 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, write_query_1, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, write_query_1, write_subarray_1); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, write_subarray_1); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, write_query_1, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, write_query_1); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, write_query_1); @@ -565,8 +593,13 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, write_query_3, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, write_query_3, write_subarray_3); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, write_subarray_3); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, write_query_3, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, write_query_3); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, write_query_3); @@ -601,8 +634,13 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, read_query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, read_query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, read_query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, read_query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_finalize(ctx_, read_query); From 9a4b885178848c7b7c7b6ffd335fb19692cadbb4 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 7 May 2024 16:57:02 +0200 Subject: [PATCH 328/456] Remove deprecated APIs from tests, part 2. (#4951) This removed deprecated APIs from tests. It will be split into multiple PRs to facilitate code reviewing. [sc-46307] --- TYPE: NO_HISTORY DESC: Remove deprecated APIs from tests, part 2. --- test/src/unit-capi-filestore.cc | 13 +- test/src/unit-capi-group.cc | 68 +-- test/src/unit-capi-incomplete-2.cc | 88 +++- test/src/unit-capi-incomplete.cc | 82 +++- test/src/unit-capi-nullable.cc | 8 +- test/src/unit-capi-query.cc | 36 +- test/src/unit-capi-query_2.cc | 543 +++++++++++++++++------ test/src/unit-capi-rest-dense_array.cc | 125 +++++- test/src/unit-capi-serialized_queries.cc | 24 +- test/src/unit-capi-smoke-test.cc | 8 +- test/src/unit-capi-sparse_array.cc | 321 ++++++++++---- test/src/unit-capi-sparse_heter.cc | 56 ++- test/src/unit-capi-sparse_neg.cc | 8 +- test/src/unit-capi-sparse_neg_2.cc | 28 +- test/src/unit-capi-sparse_real.cc | 40 +- test/src/unit-capi-sparse_real_2.cc | 40 +- test/src/unit-capi-string.cc | 10 +- test/src/unit-capi-string_dims.cc | 124 ++++-- tiledb/sm/cpp_api/query_experimental.h | 4 +- 19 files changed, 1214 insertions(+), 412 deletions(-) diff --git a/test/src/unit-capi-filestore.cc b/test/src/unit-capi-filestore.cc index 2526487718b4..ec1434849983 100644 --- a/test/src/unit-capi-filestore.cc +++ b/test/src/unit-capi-filestore.cc @@ -305,9 +305,14 @@ TEST_CASE_METHOD( uint64_t subarray_read[] = {0, file_content.size() - 1}; uint64_t size = buffer.size(); tiledb_query_t* query; + tiledb_subarray_t* sub; tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); - tiledb_query_set_subarray(ctx_, query, subarray_read); + tiledb_subarray_alloc(ctx_, array, &sub); + tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + tiledb_query_set_subarray_t(ctx_, query, sub); + tiledb_subarray_free(&sub); + tiledb_query_set_data_buffer( ctx_, query, @@ -425,9 +430,13 @@ TEST_CASE_METHOD( uint64_t subarray_read[] = {0, file_content.size() - 1}; uint64_t size = buffer.size(); tiledb_query_t* query; + tiledb_subarray_t* sub; tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); - tiledb_query_set_subarray(ctx_, query, subarray_read); + tiledb_subarray_alloc(ctx_, array, &sub); + tiledb_subarray_set_subarray(ctx_, sub, subarray_read); + tiledb_query_set_subarray_t(ctx_, query, sub); + tiledb_subarray_free(&sub); tiledb_query_set_data_buffer( ctx_, query, diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index 87a4468e7782..ef3afe1b70da 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -85,7 +85,7 @@ struct GroupFx { std::string get_golden_walk(const std::string& path); std::string get_golden_ls(const std::string& path); std::vector> read_group( - tiledb_group_t* group, bool use_get_member_by_index_v2 = true) const; + tiledb_group_t* group) const; void set_group_timestamp( tiledb_group_t* group, const uint64_t& timestamp) const; void write_group_metadata(const char* group_uri) const; @@ -229,49 +229,30 @@ void GroupFx::vacuum(const char* group_uri) const { } std::vector> GroupFx::read_group( - tiledb_group_t* group, bool use_get_member_by_index_v2) const { + tiledb_group_t* group) const { std::vector> ret; uint64_t count = 0; int rc = tiledb_group_get_member_count(ctx_, group, &count); REQUIRE(rc == TILEDB_OK); - if (use_get_member_by_index_v2) { - for (uint64_t i = 0; i < count; i++) { - std::string uri; - tiledb_object_t type; - tiledb_string_t *uri_handle, *name_handle; - rc = tiledb_group_get_member_by_index_v2( - ctx_, group, i, &uri_handle, &type, &name_handle); - REQUIRE(rc == TILEDB_OK); - const char* uri_str; - size_t uri_size; - rc = tiledb_string_view(uri_handle, &uri_str, &uri_size); - CHECK(rc == TILEDB_OK); - uri = std::string(uri_str, uri_size); - rc = tiledb_string_free(&uri_handle); + for (uint64_t i = 0; i < count; i++) { + std::string uri; + tiledb_object_t type; + tiledb_string_t *uri_handle, *name_handle; + rc = tiledb_group_get_member_by_index_v2( + ctx_, group, i, &uri_handle, &type, &name_handle); + REQUIRE(rc == TILEDB_OK); + const char* uri_str; + size_t uri_size; + rc = tiledb_string_view(uri_handle, &uri_str, &uri_size); + CHECK(rc == TILEDB_OK); + uri = std::string(uri_str, uri_size); + rc = tiledb_string_free(&uri_handle); + CHECK(rc == TILEDB_OK); + if (name_handle) { + rc = tiledb_string_free(&name_handle); CHECK(rc == TILEDB_OK); - if (name_handle) { - rc = tiledb_string_free(&name_handle); - CHECK(rc == TILEDB_OK); - } - ret.emplace_back(uri, type); - } - } else { - // When tiledb_group_get_member_by_index gets removed, the else part can - // simply be removed. - for (uint64_t i = 0; i < count; i++) { - std::string uri; - tiledb_object_t type; - char *uri_ptr, *name_ptr; - rc = tiledb_group_get_member_by_index( - ctx_, group, i, &uri_ptr, &type, &name_ptr); - REQUIRE(rc == TILEDB_OK); - uri = uri_ptr == nullptr ? "" : std::string(uri_ptr); - std::free(uri_ptr); - if (name_ptr) { - std::free(name_ptr); - } - ret.emplace_back(uri, type); } + ret.emplace_back(uri, type); } return ret; } @@ -570,13 +551,6 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( GroupFx, "C API: Group, write/read", "[capi][group][metadata][read]") { - bool use_get_member_by_index_v2 = true; - SECTION("Using tiledb_group_get_member_by_index_v2") { - use_get_member_by_index_v2 = true; - } - SECTION("Using tiledb_group_get_member_by_index") { - use_get_member_by_index_v2 = false; - } // Create and open group in write mode // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); @@ -646,12 +620,12 @@ TEST_CASE_METHOD( REQUIRE(rc == TILEDB_OK); std::vector> group1_received = - read_group(group1, use_get_member_by_index_v2); + read_group(group1); REQUIRE_THAT( group1_received, Catch::Matchers::UnorderedEquals(group1_expected)); std::vector> group2_received = - read_group(group2, use_get_member_by_index_v2); + read_group(group2); REQUIRE_THAT( group2_received, Catch::Matchers::UnorderedEquals(group2_expected)); diff --git a/test/src/unit-capi-incomplete-2.cc b/test/src/unit-capi-incomplete-2.cc index 641ab8f25537..7800be26dd60 100644 --- a/test/src/unit-capi-incomplete-2.cc +++ b/test/src/unit-capi-incomplete-2.cc @@ -435,13 +435,19 @@ void IncompleteFx2::check_dense_incomplete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -493,13 +499,19 @@ void IncompleteFx2::check_dense_until_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -566,13 +578,19 @@ void IncompleteFx2::check_dense_shrink_buffer_size() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -645,6 +663,7 @@ void IncompleteFx2::check_dense_unsplittable_overflow() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -653,8 +672,13 @@ void IncompleteFx2::check_dense_unsplittable_overflow() { rc = tiledb_query_set_offsets_buffer( ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -706,6 +730,7 @@ void IncompleteFx2::check_dense_unsplittable_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -714,8 +739,13 @@ void IncompleteFx2::check_dense_unsplittable_complete() { rc = tiledb_query_set_offsets_buffer( ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -762,13 +792,19 @@ void IncompleteFx2::check_dense_reset_buffers() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -843,12 +879,17 @@ void IncompleteFx2::check_sparse_incomplete() { // Set subarray uint64_t s0[] = {1, 2}; uint64_t s1[] = {1, 2}; + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -867,6 +908,7 @@ void IncompleteFx2::check_sparse_incomplete() { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); // Check buffer int c_buffer_a1[1] = {0}; @@ -901,12 +943,17 @@ void IncompleteFx2::check_sparse_until_complete() { // Create some subarray uint64_t s0[] = {1, 2}; uint64_t s1[] = {1, 2}; + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -956,6 +1003,7 @@ void IncompleteFx2::check_sparse_until_complete() { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void IncompleteFx2::check_sparse_unsplittable_overflow() { @@ -989,12 +1037,17 @@ void IncompleteFx2::check_sparse_unsplittable_overflow() { // Set some subarray uint64_t s0[] = {1, 1}; uint64_t s1[] = {2, 2}; + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -1017,6 +1070,7 @@ void IncompleteFx2::check_sparse_unsplittable_overflow() { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void IncompleteFx2::check_sparse_unsplittable_complete() { @@ -1050,12 +1104,17 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { // Create some subarray uint64_t s0[] = {1, 1}; uint64_t s1[] = {2, 2}; + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -1076,6 +1135,7 @@ void IncompleteFx2::check_sparse_unsplittable_complete() { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( diff --git a/test/src/unit-capi-incomplete.cc b/test/src/unit-capi-incomplete.cc index 2d4d6dc8d151..fe26c1f8c042 100644 --- a/test/src/unit-capi-incomplete.cc +++ b/test/src/unit-capi-incomplete.cc @@ -435,13 +435,19 @@ void IncompleteFx::check_dense_incomplete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -494,13 +500,19 @@ void IncompleteFx::check_dense_until_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -567,13 +579,19 @@ void IncompleteFx::check_dense_shrink_buffer_size() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -646,6 +664,7 @@ void IncompleteFx::check_dense_unsplittable_overflow() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -654,8 +673,13 @@ void IncompleteFx::check_dense_unsplittable_overflow() { rc = tiledb_query_set_offsets_buffer( ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); - REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -707,6 +731,7 @@ void IncompleteFx::check_dense_unsplittable_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -715,8 +740,13 @@ void IncompleteFx::check_dense_unsplittable_complete() { rc = tiledb_query_set_offsets_buffer( ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -762,13 +792,19 @@ void IncompleteFx::check_dense_reset_buffers() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -835,13 +871,19 @@ void IncompleteFx::check_sparse_incomplete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -894,13 +936,19 @@ void IncompleteFx::check_sparse_until_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -985,6 +1033,7 @@ void IncompleteFx::check_sparse_unsplittable_overflow() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -994,8 +1043,13 @@ void IncompleteFx::check_sparse_unsplittable_overflow() { ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); @@ -1048,6 +1102,7 @@ void IncompleteFx::check_sparse_unsplittable_complete() { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -1056,8 +1111,13 @@ void IncompleteFx::check_sparse_unsplittable_complete() { rc = tiledb_query_set_offsets_buffer( ctx_, query, attributes[0], (uint64_t*)buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); diff --git a/test/src/unit-capi-nullable.cc b/test/src/unit-capi-nullable.cc index 0be84fe640eb..8ec4190ba0ef 100644 --- a/test/src/unit-capi-nullable.cc +++ b/test/src/unit-capi-nullable.cc @@ -431,6 +431,7 @@ void NullableArrayFx::read( // Create the read query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); @@ -505,8 +506,13 @@ void NullableArrayFx::read( } // Set the subarray to read. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); diff --git a/test/src/unit-capi-query.cc b/test/src/unit-capi-query.cc index 6add17a5782e..7c1ce25cf43e 100644 --- a/test/src/unit-capi-query.cc +++ b/test/src/unit-capi-query.cc @@ -185,9 +185,16 @@ void QueryFx::test_get_buffer_write(const std::string& path) { // Prepare query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); CHECK(rc == TILEDB_OK); // Get unset buffers @@ -292,9 +299,16 @@ void QueryFx::test_get_buffer_write_decoupled(const std::string& path) { // Prepare query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); CHECK(rc == TILEDB_OK); // Get unset buffers @@ -391,9 +405,16 @@ void QueryFx::test_get_buffer_read(const std::string& path) { // Prepare query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); CHECK(rc == TILEDB_OK); // Get unset buffers @@ -503,9 +524,16 @@ void QueryFx::test_get_buffer_read_decoupled(const std::string& path) { // Prepare query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); CHECK(rc == TILEDB_OK); // Get unset buffers diff --git a/test/src/unit-capi-query_2.cc b/test/src/unit-capi-query_2.cc index 712cfc943d82..2a3b7dcac0bf 100644 --- a/test/src/unit-capi-query_2.cc +++ b/test/src/unit-capi-query_2.cc @@ -569,10 +569,16 @@ void Query2Fx::write_dense_array( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, &domain[0]); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, domain.data()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", (void*)a.data(), &a_size); @@ -701,88 +707,95 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); CHECK(layout == TILEDB_UNORDERED); + tiledb_subarray_t* subarray; + tiledb_subarray_alloc(ctx_, array, &subarray); + // Check getting range num from an invalid dimension index uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 2, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 2, &range_num); CHECK(rc == TILEDB_ERR); // Check getting range from an invalid dimension index const void *start, *end, *stride; - rc = tiledb_query_get_range(ctx_, query, 2, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 2, 0, &start, &end, &stride); CHECK(rc == TILEDB_ERR); // Check getting range from an invalid range index - rc = tiledb_query_get_range(ctx_, query, 0, 1, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 1, &start, &end, &stride); CHECK(rc == TILEDB_ERR); // Add null range uint64_t v = 0; - rc = tiledb_query_add_range(ctx_, query, 0, &v, nullptr, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &v, nullptr, nullptr); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_add_range(ctx_, query, 0, nullptr, &v, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, nullptr, &v, nullptr); CHECK(rc == TILEDB_ERR); // Add non-null stride - rc = tiledb_query_add_range(ctx_, query, 0, &v, &v, &v); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &v, &v, &v); CHECK(rc == TILEDB_ERR); // Variable-sized range - rc = tiledb_query_add_range_var(ctx_, query, 0, &v, 1, &v, 1); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, &v, 1, &v, 1); REQUIRE(rc == TILEDB_ERR); // Add ranges outside the subarray domain uint64_t inv_r1[] = {0, 0}; - rc = tiledb_query_add_range(ctx_, query, 0, &inv_r1[0], &inv_r1[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &inv_r1[0], &inv_r1[1], nullptr); CHECK(rc == TILEDB_ERR); uint64_t inv_r2[] = {0, 20}; - rc = tiledb_query_add_range(ctx_, query, 1, &inv_r2[0], &inv_r2[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &inv_r2[0], &inv_r2[1], nullptr); CHECK(rc == TILEDB_ERR); uint64_t inv_r3[] = {11, 20}; - rc = tiledb_query_add_range(ctx_, query, 1, &inv_r3[0], &inv_r3[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &inv_r3[0], &inv_r3[1], nullptr); CHECK(rc == TILEDB_ERR); // Add range with invalid end points uint64_t inv_r4[] = {5, 4}; - rc = tiledb_query_add_range(ctx_, query, 0, &inv_r4[0], &inv_r4[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &inv_r4[0], &inv_r4[1], nullptr); CHECK(rc == TILEDB_ERR); // Add valid ranges uint64_t r1[] = {1, 3}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {2, 8}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r3[] = {2, 2}; - rc = tiledb_query_add_range(ctx_, query, 1, &r3[0], &r3[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r3[0], &r3[1], nullptr); CHECK(rc == TILEDB_OK); // Check range num - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 2); - rc = tiledb_query_get_range_num(ctx_, query, 1, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 1, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check getting range from an invalid range index - rc = tiledb_query_get_range(ctx_, query, 0, 2, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 2, &start, &end, &stride); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_get_range(ctx_, query, 1, 1, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 1, 1, &start, &end, &stride); CHECK(rc == TILEDB_ERR); // Check ranges - rc = tiledb_query_get_range(ctx_, query, 0, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == 1); CHECK(*(uint64_t*)end == 3); CHECK(stride == nullptr); - rc = tiledb_query_get_range(ctx_, query, 0, 1, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 1, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == 2); CHECK(*(uint64_t*)end == 8); CHECK(stride == nullptr); - rc = tiledb_query_get_range(ctx_, query, 1, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 1, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == 2); CHECK(*(uint64_t*)end == 2); @@ -797,6 +810,8 @@ TEST_CASE_METHOD( CHECK(config == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -824,30 +839,28 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create query - tiledb_query_t* query = nullptr; - rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); // Check that there is a single range initially per dimension uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); - rc = tiledb_query_get_range_num(ctx_, query, 1, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 1, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check ranges const void *start, *end, *stride; - rc = tiledb_query_get_range(ctx_, query, 0, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 0, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == 1); CHECK(*(uint64_t*)end == 10); CHECK(stride == nullptr); - rc = tiledb_query_get_range(ctx_, query, 1, 0, &start, &end, &stride); + rc = tiledb_subarray_get_range(ctx_, subarray, 1, 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == 1); CHECK(*(uint64_t*)end == 10); @@ -858,8 +871,8 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); CHECK(array == nullptr); - tiledb_query_free(&query); - CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -892,12 +905,23 @@ TEST_CASE_METHOD( rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + // Add range with NaN double range[] = {std::numeric_limits::quiet_NaN(), 10}; - rc = tiledb_query_add_range(ctx_, query, 0, &range[0], &range[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &range[0], &range[1], nullptr); CHECK(rc == TILEDB_ERR); double range2[] = {1.3, 4.2}; - rc = tiledb_query_add_range(ctx_, query, 0, &range2[0], &range2[0], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &range2[0], &range2[0], nullptr); + CHECK(rc == TILEDB_OK); + + // Set the subarray on the query + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Clean-up @@ -907,6 +931,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -994,6 +1020,7 @@ TEST_CASE_METHOD( std::vector b_val = {1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6}; tiledb_array_t* array = nullptr; tiledb_query_t* query = nullptr; + tiledb_subarray_t* subarray = nullptr; uint64_t size, size_off, size_val; SECTION("- row-major") { @@ -1019,9 +1046,15 @@ TEST_CASE_METHOD( rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 20}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1039,7 +1072,9 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {20, 30}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1057,10 +1092,14 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 2 ranges") { uint64_t r1[] = {1, 1}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {20, 30}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1078,7 +1117,9 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 1 range") { uint64_t r[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1099,10 +1140,14 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 2 ranges") { uint64_t r1[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {10, 12}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1145,13 +1190,19 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create subarray + // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 20}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1169,7 +1220,9 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {20, 30}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1187,10 +1240,14 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 2 ranges") { uint64_t r1[] = {1, 1}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {20, 30}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1208,7 +1265,9 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 1 range") { uint64_t r[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1229,10 +1288,14 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 2 ranges") { uint64_t r1[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {10, 12}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1265,6 +1328,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -1294,6 +1359,7 @@ TEST_CASE_METHOD( 1, 2, 2, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9, 9, 10}; tiledb_array_t* array = nullptr; tiledb_query_t* query = nullptr; + tiledb_subarray_t* subarray = nullptr; uint64_t size, size_off, size_val; SECTION("- row-major") { @@ -1317,9 +1383,15 @@ TEST_CASE_METHOD( rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 27}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1337,7 +1409,9 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {30, 40}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1355,10 +1429,14 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 2 ranges") { uint64_t r1[] = {1, 1}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {30, 40}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1376,7 +1454,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tiles, 1 range") { uint64_t r[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1401,10 +1481,14 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tiles, 2 ranges") { uint64_t r1[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {23, 24}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1431,7 +1515,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tile ranges, 1 range") { uint64_t r[] = {2, 18}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1449,10 +1535,14 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tile ranges, 2 ranges") { uint64_t r1[] = {2, 18}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {19, 28}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1470,7 +1560,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap tiles and tile ranges") { uint64_t r[] = {2, 20}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1504,13 +1596,19 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create subarray + // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 27}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1528,7 +1626,9 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {30, 40}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1546,10 +1646,14 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 2 ranges") { uint64_t r1[] = {1, 1}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {30, 40}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1567,7 +1671,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tiles, 1 range") { uint64_t r[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1592,10 +1698,14 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tiles, 2 ranges") { uint64_t r1[] = {3, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {23, 24}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1622,7 +1732,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tile ranges, 1 range") { uint64_t r[] = {2, 18}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1640,10 +1752,14 @@ TEST_CASE_METHOD( SECTION("-- Overlap only tile ranges, 2 ranges") { uint64_t r1[] = {2, 18}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r2[] = {19, 28}; - rc = tiledb_query_add_range(ctx_, query, 0, &r2[0], &r2[1], nullptr); + rc = + tiledb_subarray_add_range(ctx_, subarray, 0, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1661,7 +1777,9 @@ TEST_CASE_METHOD( SECTION("-- Overlap tiles and tile ranges") { uint64_t r[] = {2, 20}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1685,6 +1803,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -1709,6 +1829,7 @@ TEST_CASE_METHOD( std::vector b_val = {1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6}; tiledb_array_t* array = nullptr; tiledb_query_t* query = nullptr; + tiledb_subarray_t* subarray = nullptr; uint64_t size, size_off, size_val; SECTION("- row-major") { @@ -1728,15 +1849,21 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create subarray + // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 10, 1, 10}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1766,9 +1893,11 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {1, 2, 7, 8}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1792,16 +1921,22 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 4 ranges") { uint64_t r11[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r12[] = {5, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r12[0], &r12[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r21[] = {6, 7}; - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r21[0], &r21[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r22[] = {9, 10}; - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r22[0], &r22[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1825,9 +1960,11 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 1 range") { uint64_t r[] = {2, 3, 5, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1879,16 +2016,22 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 4 ranges") { uint64_t r11[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r12[] = {4, 4}; - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r12[0], &r12[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r21[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r21[0], &r21[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r22[] = {7, 8}; - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r22[0], &r22[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1940,15 +2083,21 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create subarray + // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + SECTION("-- Full overlap") { uint64_t r[] = {1, 10, 1, 10}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1972,9 +2121,11 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 1 range") { uint64_t r[] = {1, 2, 7, 8}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -1998,16 +2149,22 @@ TEST_CASE_METHOD( SECTION("-- No overlap, 4 ranges") { uint64_t r11[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r12[] = {5, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r12[0], &r12[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r21[] = {6, 7}; - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r21[0], &r21[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r22[] = {9, 10}; - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r22[0], &r22[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -2031,9 +2188,11 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 1 range") { uint64_t r[] = {2, 3, 5, 6}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[2], &r[3], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[2], &r[3], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -2073,16 +2232,22 @@ TEST_CASE_METHOD( SECTION("-- Partial overlap, 4 ranges") { uint64_t r11[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r12[] = {4, 4}; - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &r12[0], &r12[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r21[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r21[0], &r21[1], nullptr); CHECK(rc == TILEDB_OK); uint64_t r22[] = {7, 8}; - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &r22[0], &r22[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper( ctx_, query, TILEDB_COORDS, &size); @@ -2124,6 +2289,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2229,10 +2396,17 @@ TEST_CASE_METHOD( rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + uint64_t r[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[0], &r[1], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2302,10 +2476,17 @@ TEST_CASE_METHOD( rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + uint64_t r[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r[0], &r[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r[0], &r[1], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2323,6 +2504,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2368,13 +2551,20 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query = nullptr; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); + uint64_t r1[] = {1, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); uint64_t r2[] = {1, 4}; CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r2[0], &r2[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2392,6 +2582,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2437,13 +2629,20 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query = nullptr; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); + uint64_t r1[] = {2, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); uint64_t r2[] = {1, 3}; CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r2[0], &r2[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2461,6 +2660,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2506,19 +2707,26 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query = nullptr; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); + uint64_t r11[] = {1, 1}; uint64_t r12[] = {2, 2}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r12[0], &r12[1], nullptr); uint64_t r21[] = {1, 1}; uint64_t r22[] = {3, 4}; CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r21[0], &r21[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r22[0], &r22[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2536,6 +2744,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2581,19 +2791,26 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query = nullptr; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); + uint64_t r11[] = {2, 2}; uint64_t r12[] = {3, 3}; - rc = tiledb_query_add_range(ctx_, query, 0, &r11[0], &r11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r11[0], &r11[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &r12[0], &r12[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r12[0], &r12[1], nullptr); uint64_t r21[] = {1, 1}; uint64_t r22[] = {3, 4}; CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r21[0], &r21[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r21[0], &r21[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r22[0], &r22[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r22[0], &r22[1], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2611,6 +2828,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2658,13 +2877,20 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query = nullptr; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); + CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); + uint64_t r1[] = {2, 3}; - rc = tiledb_query_add_range(ctx_, query, 0, &r1[0], &r1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &r1[0], &r1[1], nullptr); uint64_t r2[] = {1, 3}; CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &r2[0], &r2[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &r2[0], &r2[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_wrapper(ctx_, query, "a", &size); CHECK(rc == TILEDB_OK); @@ -2682,6 +2908,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -2774,10 +3002,16 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, &domain[0]); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, domain.data()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", (void*)a.data(), &a_size); @@ -2873,6 +3107,11 @@ TEST_CASE_METHOD( rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + // set dimensions char d1_data[] = "abbccdddd"; uint64_t d1_data_size = sizeof(d1_data) - 1; // Ignore '\0' @@ -2887,40 +3126,46 @@ TEST_CASE_METHOD( char s1[] = "a"; char s2[] = "cc"; // Variable-sized range - rc = tiledb_query_add_range_var_by_name(ctx_, query, "d1", s1, 1, s2, 2); + rc = + tiledb_subarray_add_range_var_by_name(ctx_, subarray, "d1", s1, 1, s2, 2); CHECK(rc == TILEDB_OK); // fixed-sized range uint64_t r[] = {1, 2}; - rc = tiledb_query_add_range_by_name(ctx_, query, "d2", &r[0], &r[1], nullptr); + rc = tiledb_subarray_add_range_by_name( + ctx_, subarray, "d2", &r[0], &r[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check number of ranges on each dimension uint64_t range_num; - rc = tiledb_query_get_range_num_from_name(ctx_, query, "d1", &range_num); + rc = + tiledb_subarray_get_range_num_from_name(ctx_, subarray, "d1", &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); - rc = tiledb_query_get_range_num_from_name(ctx_, query, "d2", &range_num); + rc = + tiledb_subarray_get_range_num_from_name(ctx_, subarray, "d2", &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check ranges const void *start, *end, *stride; - rc = tiledb_query_get_range_from_name( - ctx_, query, "d2", 0, &start, &end, &stride); + rc = tiledb_subarray_get_range_from_name( + ctx_, subarray, "d2", 0, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(const uint64_t*)start == 1); CHECK(*(const uint64_t*)end == 2); uint64_t start_size = 0, end_size = 0; - rc = tiledb_query_get_range_var_size_from_name( - ctx_, query, "d1", 0, &start_size, &end_size); + rc = tiledb_subarray_get_range_var_size_from_name( + ctx_, subarray, "d1", 0, &start_size, &end_size); CHECK(rc == TILEDB_OK); CHECK(start_size == 1); CHECK(end_size == 2); std::vector start_data(start_size); std::vector end_data(end_size); - rc = tiledb_query_get_range_var_from_name( - ctx_, query, "d1", 0, start_data.data(), end_data.data()); + rc = tiledb_subarray_get_range_var_from_name( + ctx_, subarray, "d1", 0, start_data.data(), end_data.data()); CHECK(rc == TILEDB_OK); CHECK(std::string(start_data.begin(), start_data.end()) == "a"); CHECK(std::string(end_data.begin(), end_data.end()) == "cc"); @@ -2932,6 +3177,8 @@ TEST_CASE_METHOD( CHECK(array == nullptr); tiledb_query_free(&query); CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } @@ -3049,8 +3296,14 @@ TEST_CASE_METHOD( ctx_, query2, "a", (uint64_t*)offsets2.data(), &offsets_size); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query2, &dom); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, &dom); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query2, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query2); CHECK(rc == TILEDB_OK); @@ -3094,23 +3347,19 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_READ); CHECK(rc == TILEDB_OK); - // Create query - tiledb_query_t* query = nullptr; - rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); - CHECK(rc == TILEDB_OK); - // Add ranges uint64_t ranges[] = {1, 3, 7, 10}; - rc = tiledb_query_add_point_ranges(ctx_, query, 0, ranges, 4); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_point_ranges(ctx_, query, 1, ranges, 4); + rc = tiledb_subarray_add_point_ranges(ctx_, subarray, 0, ranges, 4); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_point_ranges(ctx_, subarray, 1, ranges, 4); CHECK(rc == TILEDB_OK); // Check that there are four ranges uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 4); @@ -3118,8 +3367,8 @@ TEST_CASE_METHOD( for (uint32_t dim_idx = 0; dim_idx < 1; dim_idx++) { for (uint32_t idx = 0; idx < 4; idx++) { const void *start, *end, *stride; - rc = tiledb_query_get_range( - ctx_, query, dim_idx, idx, &start, &end, &stride); + rc = tiledb_subarray_get_range( + ctx_, subarray, dim_idx, idx, &start, &end, &stride); CHECK(rc == TILEDB_OK); CHECK(*(uint64_t*)start == ranges[idx]); CHECK(*(uint64_t*)end == ranges[idx]); @@ -3132,8 +3381,8 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); CHECK(array == nullptr); - tiledb_query_free(&query); - CHECK(query == nullptr); + tiledb_subarray_free(&subarray); + CHECK(subarray == nullptr); remove_array(array_name); } diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index f985aeaa4954..3f9d4a290122 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -311,13 +311,19 @@ int* DenseArrayRESTFx::read_dense_array_2D( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, query_type, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); - REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE(rc == TILEDB_OK); @@ -391,6 +397,7 @@ void DenseArrayRESTFx::write_dense_array_by_tiles( for (int64_t j = 0; j < domain_size_1; j += tile_extent_1) { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -407,7 +414,13 @@ void DenseArrayRESTFx::write_dense_array_by_tiles( (domain_size_1 - j); int64_t subarray[4] = {i, i + tile_rows - 1, j, j + tile_cols - 1}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE(rc == TILEDB_OK); int64_t k = 0, l = 0; @@ -459,12 +472,19 @@ void DenseArrayRESTFx::write_dense_subarray_2D( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, query_type, &query); REQUIRE_SAFE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); REQUIRE_SAFE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE_SAFE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE_SAFE(rc == TILEDB_OK); @@ -578,26 +598,34 @@ void DenseArrayRESTFx::check_sorted_reads() { // Check out of bounds subarray tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_config(ctx_, query, cfg); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_config(ctx_, sub, cfg); + REQUIRE(rc == TILEDB_OK); int64_t subarray_1[] = {-1, 5, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_1); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_1); CHECK(rc == TILEDB_ERR); int64_t subarray_2[] = {0, 5000000, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_2); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); CHECK(rc == TILEDB_ERR); int64_t subarray_3[] = {0, 5, -1, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_3); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_3); CHECK(rc == TILEDB_ERR); int64_t subarray_4[] = {0, 5, 10, 100000000}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_4); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_4); CHECK(rc == TILEDB_ERR); int64_t subarray_5[] = {0, 5, 10, 10}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_5); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_5); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_finalize(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -656,7 +684,12 @@ void DenseArrayRESTFx::check_incomplete_reads() { int64_t subarray[] = {0, 50, 0, 50}; int32_t attr_buffer[100]; uint64_t attr_buffer_size = sizeof(attr_buffer); - REQUIRE(tiledb_query_set_subarray(ctx_, query, subarray) == TILEDB_OK); + + tiledb_subarray_t* sub; + REQUIRE(tiledb_subarray_alloc(ctx_, array, &sub) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx_, sub, subarray) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx_, query, sub) == TILEDB_OK); + tiledb_subarray_free(&sub); unsigned num_incompletes = 0; std::vector all_attr_values; @@ -1213,10 +1246,14 @@ TEST_CASE_METHOD( auto buffer_coords = (uint64_t*)malloc(buffer_coords_size); tiledb_query_t* query; + tiledb_subarray_t* sub; REQUIRE(tiledb_query_alloc(ctx_, array, TILEDB_READ, &query) == TILEDB_OK); REQUIRE( tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER) == TILEDB_OK); - REQUIRE(tiledb_query_set_subarray(ctx_, query, subarray) == TILEDB_OK); + REQUIRE(tiledb_subarray_alloc(ctx_, array, &sub) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx_, sub, subarray) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx_, query, sub) == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE( tiledb_query_set_data_buffer( ctx_, query, "a1", buffer_a1, &buffer_a1_size) == TILEDB_OK); @@ -1276,6 +1313,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -1284,8 +1322,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[0], write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {2, 3, 1, 2}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -1322,8 +1365,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); @@ -1355,6 +1403,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -1363,8 +1412,13 @@ TEST_CASE_METHOD( ctx_, query, attributes[0], write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {2, 3, 2, 3}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -1401,8 +1455,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray_read); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_read); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer( ctx_, query, attributes[0], read_a1, &read_a1_size); CHECK(rc == TILEDB_OK); @@ -1432,6 +1491,7 @@ TEST_CASE_METHOD( rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -1440,8 +1500,13 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx_, query, "a1", write_a1, &write_a1_size); CHECK(rc == TILEDB_OK); uint64_t subarray_1[] = {1, 2, 1, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_1); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_1); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -1464,8 +1529,13 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer(ctx_, query, "a1", write_a2, &write_a2_size); CHECK(rc == TILEDB_OK); uint64_t subarray_2[] = {2, 3, 2, 3}; - rc = tiledb_query_set_subarray(ctx_, query, subarray_2); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray_2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); rc = tiledb_array_close(ctx_, array); @@ -1502,8 +1572,13 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a1", read_a, &read_a_size); CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); @@ -1593,14 +1668,20 @@ TEST_CASE_METHOD( // Create WRITE query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); uint64_t subarray[] = {1, 1, 1, 1}; // Set some subarray BEFORE setting the layout to UNORDERED - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Unordered writes are only possible for sparse arrays. rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -1857,7 +1938,11 @@ TEST_CASE_METHOD( tiledb_query_set_data_buffer( ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]) == TILEDB_OK); - REQUIRE(tiledb_query_set_subarray(ctx_, query, &subarray[0]) == TILEDB_OK); + tiledb_subarray_t* sub; + REQUIRE(tiledb_subarray_alloc(ctx_, array, &sub) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx_, sub, subarray) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx_, query, sub) == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE(tiledb_query_submit(ctx_, query) == TILEDB_OK); REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); diff --git a/test/src/unit-capi-serialized_queries.cc b/test/src/unit-capi-serialized_queries.cc index a91a5ef68a80..82790a7f75bf 100644 --- a/test/src/unit-capi-serialized_queries.cc +++ b/test/src/unit-capi-serialized_queries.cc @@ -218,8 +218,10 @@ struct SerializationFx { Array array(ctx, array_uri, TILEDB_WRITE); Query query(ctx, array); - query.add_range(0, subarray[0], subarray[1]); - query.add_range(1, subarray[2], subarray[3]); + Subarray sub(ctx, array); + sub.add_range(0, subarray[0], subarray[1]); + sub.add_range(1, subarray[2], subarray[3]); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -708,8 +710,10 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.add_range(0, subarray[0], subarray[1]); - query.add_range(1, subarray[2], subarray[3]); + Subarray sub(ctx, array); + sub.add_range(0, subarray[0], subarray[1]); + sub.add_range(1, subarray[2], subarray[3]); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -742,8 +746,10 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {3, 4, 3, 4}; - query.add_range(0, subarray[0], subarray[1]); - query.add_range(1, subarray[2], subarray[3]); + Subarray sub(ctx, array); + sub.add_range(0, subarray[0], subarray[1]); + sub.add_range(1, subarray[2], subarray[3]); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -775,8 +781,10 @@ TEST_CASE_METHOD( std::vector a3_data(60); std::vector a3_offsets(4); std::vector subarray = {3, 4, 3, 4}; - query.add_range(0, subarray[0], subarray[1]); - query.add_range(1, subarray[2], subarray[3]); + Subarray sub(ctx, array); + sub.add_range(0, subarray[0], subarray[1]); + sub.add_range(1, subarray[2], subarray[3]); + query.set_subarray(sub); auto set_buffers = [&](Query& q) { q.set_data_buffer("a1", a1); diff --git a/test/src/unit-capi-smoke-test.cc b/test/src/unit-capi-smoke-test.cc index 2255b9a69550..6c7eb2e4a28c 100644 --- a/test/src/unit-capi-smoke-test.cc +++ b/test/src/unit-capi-smoke-test.cc @@ -677,6 +677,7 @@ void SmokeTestFx::read( // Create the read query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); @@ -755,8 +756,13 @@ void SmokeTestFx::read( } // Set the subarray to read. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Create the attribute condition objects. tiledb_query_condition_t* combined_query_condition = nullptr; diff --git a/test/src/unit-capi-sparse_array.cc b/test/src/unit-capi-sparse_array.cc index 4bfc283727d6..be3b4eb1d1df 100644 --- a/test/src/unit-capi-sparse_array.cc +++ b/test/src/unit-capi-sparse_array.cc @@ -399,10 +399,15 @@ int* SparseArrayFx::read_sparse_array_2D( int64_t s1[] = {domain_1_lo, domain_1_hi}; rc = tiledb_query_set_layout(ctx_, query, query_layout); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -424,6 +429,7 @@ int* SparseArrayFx::read_sparse_array_2D( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); // Success - return the created buffer return buffer; @@ -2193,10 +2199,15 @@ void SparseArrayFx::check_sparse_array_no_results( uint64_t s1[] = {1, 2}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Check that it has no results before submission int has_results; @@ -2229,6 +2240,7 @@ void SparseArrayFx::check_sparse_array_no_results( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); delete[] buffer; } @@ -2539,8 +2551,13 @@ TEST_CASE_METHOD( uint64_t s0[] = {1, 1}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_ERR); // Submit rc = tiledb_query_submit(ctx_, query); @@ -2552,6 +2569,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -2734,10 +2752,15 @@ TEST_CASE_METHOD( uint64_t s1[] = {3, 3}; rc = tiledb_query_set_layout(ctx_, empty_query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, empty_query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, empty_query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, empty_query, subarray); + CHECK(rc == TILEDB_OK); // Submit query CHECK(tiledb_query_submit(ctx_, empty_query) == TILEDB_OK); @@ -2756,6 +2779,7 @@ TEST_CASE_METHOD( free(coords_dim1); free(coords_dim2); tiledb_query_free(&empty_query); + tiledb_subarray_free(&subarray); // ---- Second READ query (non-empty) tiledb_query_t* query; @@ -2798,10 +2822,14 @@ TEST_CASE_METHOD( s1[1] = 2; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query CHECK(tiledb_query_submit(ctx_, query) == TILEDB_OK); @@ -2831,6 +2859,7 @@ TEST_CASE_METHOD( free(coords_dim1); free(coords_dim2); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); // Clean up CHECK(tiledb_array_close(ctx_, array) == TILEDB_OK); @@ -2894,10 +2923,15 @@ TEST_CASE_METHOD( uint64_t s1[] = {1, 2}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -2923,6 +2957,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -2966,10 +3001,15 @@ TEST_CASE_METHOD( uint64_t s1[] = {1, 1}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -2991,6 +3031,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3035,10 +3076,15 @@ TEST_CASE_METHOD( uint64_t s1[] = {1, UINT64_MAX - 1}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -3064,6 +3110,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3107,14 +3154,19 @@ TEST_CASE_METHOD( uint64_t s11[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s00[0], &s00[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s00[0], &s00[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s01[0], &s01[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s01[0], &s01[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s10[0], &s10[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s10[0], &s10[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -3147,6 +3199,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3210,16 +3263,21 @@ TEST_CASE_METHOD( uint64_t s11[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s00[0], &s00[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s00[0], &s00[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s01[0], &s01[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s01[0], &s01[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s00[0], &s00[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s00[0], &s00[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s10[0], &s10[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s10[0], &s10[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -3240,6 +3298,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3283,14 +3342,19 @@ TEST_CASE_METHOD( uint64_t s11[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s00[0], &s00[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s00[0], &s00[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s01[0], &s01[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s01[0], &s01[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s10[0], &s10[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s10[0], &s10[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -3348,6 +3412,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3391,14 +3456,19 @@ TEST_CASE_METHOD( uint64_t s11[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s00[0], &s00[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s00[0], &s00[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s01[0], &s01[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s01[0], &s01[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s10[0], &s10[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s10[0], &s10[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s11[0], &s11[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s11[0], &s11[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); CHECK(rc == TILEDB_OK); @@ -3419,6 +3489,7 @@ TEST_CASE_METHOD( // Clean up tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3480,14 +3551,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -3585,6 +3661,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3646,14 +3723,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -3751,6 +3833,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3812,14 +3895,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -3937,6 +4025,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -3998,14 +4087,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -4123,6 +4217,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -4184,14 +4279,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -4349,6 +4449,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -4410,14 +4511,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -4575,6 +4681,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -4636,14 +4743,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -4841,6 +4953,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -4902,14 +5015,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -5147,6 +5265,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -5208,14 +5327,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -5453,6 +5577,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -5513,14 +5638,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -5577,6 +5707,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -5638,14 +5769,19 @@ TEST_CASE_METHOD( int64_t s1[] = {3, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s0[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -5702,6 +5838,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -5809,6 +5946,7 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -5895,8 +6033,13 @@ TEST_CASE_METHOD( uint64_t subarray[] = {1, 4, 1, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -6002,6 +6145,7 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); @@ -6084,8 +6228,13 @@ TEST_CASE_METHOD( uint64_t subarray[] = {1, 4, 1, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -6372,6 +6521,7 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -6457,8 +6607,13 @@ TEST_CASE_METHOD( uint64_t subarray[] = {1, 4, 1, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); @@ -6566,6 +6721,7 @@ TEST_CASE_METHOD( // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -6647,8 +6803,13 @@ TEST_CASE_METHOD( uint64_t subarray[] = {1, 4, 1, 4}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); diff --git a/test/src/unit-capi-sparse_heter.cc b/test/src/unit-capi-sparse_heter.cc index 3d9ce6229a41..cd077ba0069c 100644 --- a/test/src/unit-capi-sparse_heter.cc +++ b/test/src/unit-capi-sparse_heter.cc @@ -424,11 +424,16 @@ void SparseHeterFx::check_est_result_size_float_int64( tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 0, &subarray_f[0], &subarray_f[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 1, &subarray_i[0], &subarray_i[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &subarray_f[0], &subarray_f[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &subarray_i[0], &subarray_i[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check error for zipped coordinates @@ -452,6 +457,7 @@ void SparseHeterFx::check_est_result_size_float_int64( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseHeterFx::check_est_result_size_int64_float( @@ -470,11 +476,16 @@ void SparseHeterFx::check_est_result_size_int64_float( tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 0, &subarray_i[0], &subarray_i[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &subarray_i[0], &subarray_i[1], nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &subarray_f[0], &subarray_f[1], nullptr); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 1, &subarray_f[0], &subarray_f[1], nullptr); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check error for zipped coordinates @@ -498,6 +509,7 @@ void SparseHeterFx::check_est_result_size_int64_float( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseHeterFx::write_sparse_array_float_int64( @@ -631,12 +643,17 @@ void SparseHeterFx::check_read_sparse_array_float_int64( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, layout); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 0, &subarray_f[0], &subarray_f[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &subarray_f[0], &subarray_f[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 1, &subarray_i[0], &subarray_i[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &subarray_i[0], &subarray_i[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -659,6 +676,7 @@ void SparseHeterFx::check_read_sparse_array_float_int64( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseHeterFx::check_read_sparse_array_int64_float( @@ -695,12 +713,17 @@ void SparseHeterFx::check_read_sparse_array_int64_float( REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, layout); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 0, &subarray_i[0], &subarray_i[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range( + ctx_, subarray, 0, &subarray_i[0], &subarray_i[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range( - ctx_, query, 1, &subarray_f[0], &subarray_f[1], nullptr); + rc = tiledb_subarray_add_range( + ctx_, subarray, 1, &subarray_f[0], &subarray_f[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -723,6 +746,7 @@ void SparseHeterFx::check_read_sparse_array_int64_float( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( diff --git a/test/src/unit-capi-sparse_neg.cc b/test/src/unit-capi-sparse_neg.cc index de1389a702a2..4b070946ee26 100644 --- a/test/src/unit-capi-sparse_neg.cc +++ b/test/src/unit-capi-sparse_neg.cc @@ -304,10 +304,16 @@ void SparseNegFx::read_sparse_vector(const std::string& path) { uint64_t coords_size = sizeof(coords); int64_t subarray[] = {-1, 2}; tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "d0", coords, &coords_size); diff --git a/test/src/unit-capi-sparse_neg_2.cc b/test/src/unit-capi-sparse_neg_2.cc index f11917023a6b..e3aeb2e927b0 100644 --- a/test/src/unit-capi-sparse_neg_2.cc +++ b/test/src/unit-capi-sparse_neg_2.cc @@ -309,8 +309,13 @@ void SparseNegFx2::read_sparse_vector(const std::string& path) { int64_t s0[] = {-1, 2}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); @@ -335,6 +340,7 @@ void SparseNegFx2::read_sparse_vector(const std::string& path) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseNegFx2::read_sparse_array_row(const std::string& path) { @@ -367,10 +373,15 @@ void SparseNegFx2::read_sparse_array_row(const std::string& path) { int64_t s1[] = {-2, 1}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -393,6 +404,7 @@ void SparseNegFx2::read_sparse_array_row(const std::string& path) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseNegFx2::read_sparse_array_col(const std::string& path) { @@ -425,10 +437,15 @@ void SparseNegFx2::read_sparse_array_col(const std::string& path) { int64_t s1[] = {-2, 1}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_COL_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s0[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_submit(ctx_, query); REQUIRE(rc == TILEDB_OK); @@ -451,6 +468,7 @@ void SparseNegFx2::read_sparse_array_col(const std::string& path) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( diff --git a/test/src/unit-capi-sparse_real.cc b/test/src/unit-capi-sparse_real.cc index dead9ad65cb7..24cb7625034b 100644 --- a/test/src/unit-capi-sparse_real.cc +++ b/test/src/unit-capi-sparse_real.cc @@ -256,11 +256,17 @@ void SparseRealFx::read_sparse_array(const std::string& path) { float coords_dim2[16]; uint64_t coords_size = sizeof(coords_dim1); tiledb_query_t* query; + tiledb_subarray_t* sub; float subarray[] = {-180.0f, 180.0f, -90.0f, 90.0f}; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -309,11 +315,17 @@ void SparseRealFx::read_sparse_array_next_partition_bug( float coords_dim2[4]; uint64_t coords_size = sizeof(coords_dim1); tiledb_query_t* query; + tiledb_subarray_t* sub; float subarray[] = {-180.0f, 180.0f, -90.0f, 90.0f}; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -463,6 +475,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); @@ -480,13 +493,23 @@ TEST_CASE_METHOD( // Check Nan float subarray[] = { -180.0f, std::numeric_limits::quiet_NaN(), -90.0f, 90.0f}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Check infinity subarray[1] = std::numeric_limits::infinity(); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Clean up rc = tiledb_array_close(ctx_, array); @@ -564,11 +587,18 @@ TEST_CASE_METHOD( double coords_dim2[1]; uint64_t coords_size = sizeof(coords_dim1); tiledb_query_t* query; + tiledb_subarray_t* sub; double subarray[] = {-180.0, -180.0, 1.0, 1.0}; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); diff --git a/test/src/unit-capi-sparse_real_2.cc b/test/src/unit-capi-sparse_real_2.cc index 97fe65ccd8c2..cfd526e7369b 100644 --- a/test/src/unit-capi-sparse_real_2.cc +++ b/test/src/unit-capi-sparse_real_2.cc @@ -263,10 +263,15 @@ void SparseRealFx2::read_sparse_array(const std::string& path) { float s1[] = {-90.0f, 90.0f}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); @@ -297,6 +302,7 @@ void SparseRealFx2::read_sparse_array(const std::string& path) { // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void SparseRealFx2::read_sparse_array_next_partition_bug( @@ -322,10 +328,15 @@ void SparseRealFx2::read_sparse_array_next_partition_bug( float s1[] = {-90.0f, 90.0f}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); @@ -350,6 +361,7 @@ void SparseRealFx2::read_sparse_array_next_partition_bug( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -422,10 +434,15 @@ TEST_CASE_METHOD( // Create some subarray float s0[] = {-180.0f, std::numeric_limits::quiet_NaN()}; float s1[] = {-90.0f, std::numeric_limits::infinity()}; - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_ERR); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); // Clean up rc = tiledb_array_close(ctx_, array); @@ -433,6 +450,7 @@ TEST_CASE_METHOD( tiledb_array_free(&array); tiledb_config_free(&config); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } @@ -473,10 +491,15 @@ TEST_CASE_METHOD( float s1[] = {-20.0f, -20.0f}; rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &s0[0], &s0[1], nullptr); + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &s0[0], &s0[1], nullptr); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 1, &s1[0], &s1[1], nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 1, &s1[0], &s1[1], nullptr); REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); + CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", a, &a_size); REQUIRE(rc == TILEDB_OK); @@ -507,6 +530,7 @@ TEST_CASE_METHOD( // Clean up tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); } diff --git a/test/src/unit-capi-string.cc b/test/src/unit-capi-string.cc index 6d56c4da91cb..92a3195ed899 100644 --- a/test/src/unit-capi-string.cc +++ b/test/src/unit-capi-string.cc @@ -248,6 +248,7 @@ void StringFx::read_array(const std::string& array_name) { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer( @@ -267,8 +268,13 @@ void StringFx::read_array(const std::string& array_name) { REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query rc = tiledb_query_submit(ctx_, query); diff --git a/test/src/unit-capi-string_dims.cc b/test/src/unit-capi-string_dims.cc index 0b9e7c6fad2e..465da750acc7 100644 --- a/test/src/unit-capi-string_dims.cc +++ b/test/src/unit-capi-string_dims.cc @@ -636,13 +636,25 @@ void StringDimsFx::get_est_result_size_var( tiledb_query_t* query; int rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var( - ctx_, query, dim_idx, start.data(), start.size(), end.data(), end.size()); + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range_var( + ctx_, + subarray, + dim_idx, + start.data(), + start.size(), + end.data(), + end.size()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); rc = tiledb_query_get_est_result_size_var( ctx_, query, dim_name.c_str(), size_off, size_val); CHECK(rc == TILEDB_OK); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void StringDimsFx::read_array_1d( @@ -660,35 +672,40 @@ void StringDimsFx::read_array_1d( tiledb_query_t* query; int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var( - ctx, query, 0, start.data(), start.size(), end.data(), end.size()); + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range_var( + ctx, subarray, 0, start.data(), start.size(), end.data(), end.size()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check range num uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check getting range from an invalid range index uint64_t start_size = 0, end_size = 0; - rc = tiledb_query_get_range_var_size( - ctx_, query, 0, 2, &start_size, &end_size); + rc = tiledb_subarray_get_range_var_size( + ctx_, subarray, 0, 2, &start_size, &end_size); CHECK(rc == TILEDB_ERR); std::vector start_data(start_size); std::vector end_data(end_size); - rc = tiledb_query_get_range_var( - ctx_, query, 0, 2, start_data.data(), end_data.data()); + rc = tiledb_subarray_get_range_var( + ctx_, subarray, 0, 2, start_data.data(), end_data.data()); CHECK(rc == TILEDB_ERR); // Check ranges - rc = tiledb_query_get_range_var_size( - ctx_, query, 0, 0, &start_size, &end_size); + rc = tiledb_subarray_get_range_var_size( + ctx_, subarray, 0, 0, &start_size, &end_size); CHECK(rc == TILEDB_OK); start_data.resize(start_size); end_data.resize(end_size); - rc = tiledb_query_get_range_var( - ctx_, query, 0, 0, start_data.data(), end_data.data()); + rc = tiledb_subarray_get_range_var( + ctx_, subarray, 0, 0, start_data.data(), end_data.data()); CHECK(rc == TILEDB_OK); CHECK(std::string(start_data.data(), start_data.size()) == start); CHECK(std::string(end_data.data(), end_data.size()) == end); @@ -733,6 +750,7 @@ void StringDimsFx::read_array_1d( // Clean up tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void StringDimsFx::read_array_2d( @@ -752,54 +770,59 @@ void StringDimsFx::read_array_2d( tiledb_query_t* query; int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var( + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range_var( ctx, - query, + subarray, 0, d1_start.data(), d1_start.size(), d1_end.data(), d1_end.size()); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx, query, 1, &d2_start, &d2_end, nullptr); + rc = tiledb_subarray_add_range(ctx, subarray, 1, &d2_start, &d2_end, nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check range num d1 uint64_t range_num; - rc = tiledb_query_get_range_num(ctx_, query, 0, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 0, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check range num d2 - rc = tiledb_query_get_range_num(ctx_, query, 1, &range_num); + rc = tiledb_subarray_get_range_num(ctx_, subarray, 1, &range_num); CHECK(rc == TILEDB_OK); CHECK(range_num == 1); // Check getting range from an invalid range index uint64_t d1_start_size = 0, d1_end_size = 0; - rc = tiledb_query_get_range_var_size( - ctx_, query, 0, 2, &d1_start_size, &d1_end_size); + rc = tiledb_subarray_get_range_var_size( + ctx_, subarray, 0, 2, &d1_start_size, &d1_end_size); CHECK(rc == TILEDB_ERR); std::vector d1_start_data(d1_start_size); std::vector d1_end_data(d1_end_size); - rc = tiledb_query_get_range_var( - ctx_, query, 0, 2, d1_start_data.data(), d1_end_data.data()); + rc = tiledb_subarray_get_range_var( + ctx_, subarray, 0, 2, d1_start_data.data(), d1_end_data.data()); CHECK(rc == TILEDB_ERR); // Check ranges - rc = tiledb_query_get_range_var_size( - ctx_, query, 0, 0, &d1_start_size, &d1_end_size); + rc = tiledb_subarray_get_range_var_size( + ctx_, subarray, 0, 0, &d1_start_size, &d1_end_size); CHECK(rc == TILEDB_OK); d1_start_data.resize(d1_start_size); d1_end_data.resize(d1_end_size); - rc = tiledb_query_get_range_var( - ctx_, query, 0, 0, d1_start_data.data(), d1_end_data.data()); + rc = tiledb_subarray_get_range_var( + ctx_, subarray, 0, 0, d1_start_data.data(), d1_end_data.data()); CHECK(rc == TILEDB_OK); CHECK(std::string(d1_start_data.data(), d1_start_data.size()) == d1_start); CHECK(std::string(d1_end_data.data(), d1_end_data.size()) == d1_end); const void *d2_start_data, *d2_end_data, *stride; - rc = tiledb_query_get_range( - ctx_, query, 1, 0, &d2_start_data, &d2_end_data, &stride); + rc = tiledb_subarray_get_range( + ctx_, subarray, 1, 0, &d2_start_data, &d2_end_data, &stride); CHECK(rc == TILEDB_OK); CHECK(*(int32_t*)d2_start_data == d2_start); CHECK(*(int32_t*)d2_end_data == d2_end); @@ -844,6 +867,7 @@ void StringDimsFx::read_array_2d( // Clean up tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } void StringDimsFx::read_array_2d_default_string_range( @@ -861,9 +885,14 @@ void StringDimsFx::read_array_2d_default_string_range( tiledb_query_t* query; int rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); // Add range for int dimension - rc = tiledb_query_add_range(ctx, query, 1, &d2_start, &d2_end, nullptr); + rc = tiledb_subarray_add_range(ctx, subarray, 1, &d2_start, &d2_end, nullptr); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Set query buffers @@ -1271,8 +1300,14 @@ TEST_CASE_METHOD( // Set subarray and buffer rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, dom); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, dom); REQUIRE(rc == TILEDB_ERR); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); int32_t buff[10]; uint64_t buff_size = sizeof(buff); rc = tiledb_query_set_data_buffer( @@ -1365,36 +1400,46 @@ TEST_CASE_METHOD( tiledb_query_t* query; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); char s1[] = "a"; char s2[] = "ee"; // Check we can add empty ranges - rc = tiledb_query_add_range_var(ctx_, query, 0, s1, 0, s2, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, s1, 0, s2, 2); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var(ctx_, query, 0, s1, 1, s2, 0); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, s1, 1, s2, 0); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var(ctx_, query, 0, nullptr, 0, s2, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, nullptr, 0, s2, 2); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range_var(ctx_, query, 0, s1, 1, nullptr, 0); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, s1, 1, nullptr, 0); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Clean query and re-alloc tiledb_query_free(&query); + tiledb_subarray_free(&subarray); rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); // Check errors when adding range - rc = tiledb_query_add_range(ctx_, query, 0, s1, s2, nullptr); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, s1, s2, nullptr); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_add_range_var(ctx_, query, 1, s1, 1, s2, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 1, s1, 1, s2, 2); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_add_range_var(ctx_, query, 0, nullptr, 1, s2, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, nullptr, 1, s2, 2); CHECK(rc == TILEDB_ERR); - rc = tiledb_query_add_range_var(ctx_, query, 0, s1, 1, nullptr, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, s1, 1, nullptr, 2); CHECK(rc == TILEDB_ERR); // Add string range - rc = tiledb_query_add_range_var(ctx_, query, 0, s1, 1, s2, 2); + rc = tiledb_subarray_add_range_var(ctx_, subarray, 0, s1, 1, s2, 2); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Check error on getting estimated result size @@ -1411,6 +1456,7 @@ TEST_CASE_METHOD( // Clean query tiledb_query_free(&query); + tiledb_subarray_free(&subarray); // Set layout tiledb_layout_t layout = TILEDB_ROW_MAJOR; diff --git a/tiledb/sm/cpp_api/query_experimental.h b/tiledb/sm/cpp_api/query_experimental.h index 3f6eb9cffa42..1cf8f052c280 100644 --- a/tiledb/sm/cpp_api/query_experimental.h +++ b/tiledb/sm/cpp_api/query_experimental.h @@ -387,7 +387,9 @@ class QueryExperimental { * tiledb::Context ctx; * auto array = Array(ctx, uri, TILEDB_READ); * Query query(ctx, array); - * query.add_range("dim", 1, 5) + * Subarray subarray(ctx, array); + * subarray.add_range("dim", 1, 5); + * query.set_subarray(subarray); * QueryChannel default_channel = * QueryExperimental::get_default_channel(query); * ChannelOperation operation = From 1bcce0137910ead10f98c47905a551819a9543cc Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 8 May 2024 18:24:48 +0300 Subject: [PATCH 329/456] Use unique pointers to manage the lifetime of queries when consolidating. (#4946) [SC-46580](https://app.shortcut.com/tiledb-inc/story/46580/fragment-consolidation-uses-raw-pointers-for-the-read-and-write-queries) This PR updates the `FragmentConsolidator` class to manage the lifetime of queries using unique pointers, instead of manually allocating and deallocating them. This increases exception safety and allows us to remove several calls to `tdb_delete`. --- TYPE: NO_HISTORY --- .../sm/consolidator/fragment_consolidator.cc | 73 +++++++------------ .../sm/consolidator/fragment_consolidator.h | 4 +- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index d0bb71e355c2..d556f3911abd 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -558,20 +558,15 @@ Status FragmentConsolidator::consolidate_internal( cw.resize_buffers(stats_, config_, array_schema, average_var_cell_sizes); // Create queries - auto query_r = (Query*)nullptr; - auto query_w = (Query*)nullptr; - auto st = create_queries( + tdb_unique_ptr query_r = nullptr; + tdb_unique_ptr query_w = nullptr; + throw_if_not_ok(create_queries( array_for_reads, array_for_writes, union_non_empty_domains, - &query_r, - &query_w, - new_fragment_uri); - if (!st.ok()) { - tdb_delete(query_r); - tdb_delete(query_w); - return st; - } + query_r, + query_w, + new_fragment_uri)); // Get the vacuum URI URI vac_uri; @@ -579,29 +574,19 @@ Status FragmentConsolidator::consolidate_internal( vac_uri = array_for_reads->array_directory().get_vacuum_uri(*new_fragment_uri); } catch (std::exception& e) { - tdb_delete(query_r); - tdb_delete(query_w); std::throw_with_nested( std::logic_error("[FragmentConsolidator::consolidate_internal] ")); } // Read from one array and write to the other - try { - copy_array(query_r, query_w, cw); - } catch (...) { - tdb_delete(query_r); - tdb_delete(query_w); - throw; - }; + copy_array(query_r.get(), query_w.get(), cw); // Finalize write query - st = query_w->finalize(); + auto st = query_w->finalize(); if (!st.ok()) { - tdb_delete(query_r); - tdb_delete(query_w); bool is_dir = false; - auto st2 = storage_manager_->vfs()->is_dir(*new_fragment_uri, &is_dir); - (void)st2; // Perhaps report this once we support an error stack + throw_if_not_ok( + storage_manager_->vfs()->is_dir(*new_fragment_uri, &is_dir)); if (is_dir) throw_if_not_ok(storage_manager_->vfs()->remove_dir(*new_fragment_uri)); return st; @@ -614,8 +599,6 @@ Status FragmentConsolidator::consolidate_internal( vac_uri, to_consolidate); if (!st.ok()) { - tdb_delete(query_r); - tdb_delete(query_w); bool is_dir = false; throw_if_not_ok( storage_manager_->vfs()->is_dir(*new_fragment_uri, &is_dir)); @@ -624,10 +607,6 @@ Status FragmentConsolidator::consolidate_internal( return st; } - // Clean up - tdb_delete(query_r); - tdb_delete(query_w); - return st; } @@ -667,8 +646,8 @@ Status FragmentConsolidator::create_queries( shared_ptr array_for_reads, shared_ptr array_for_writes, const NDRange& subarray, - Query** query_r, - Query** query_w, + tdb_unique_ptr& query_r, + tdb_unique_ptr& query_w, URI* new_fragment_uri) { auto timer_se = stats_->start_timer("consolidate_create_queries"); @@ -679,48 +658,50 @@ Status FragmentConsolidator::create_queries( // is not a user input prone to errors). // Create read query - *query_r = tdb_new(Query, storage_manager_, array_for_reads); - RETURN_NOT_OK((*query_r)->set_layout(Layout::GLOBAL_ORDER)); + query_r = + tdb_unique_ptr(tdb_new(Query, storage_manager_, array_for_reads)); + RETURN_NOT_OK(query_r->set_layout(Layout::GLOBAL_ORDER)); // Dense consolidation will do a tile aligned read. if (dense) { NDRange read_subarray = subarray; auto& domain{array_for_reads->array_schema_latest().domain()}; domain.expand_to_tiles(&read_subarray); - RETURN_NOT_OK((*query_r)->set_subarray_unsafe(read_subarray)); + RETURN_NOT_OK(query_r->set_subarray_unsafe(read_subarray)); } // Enable consolidation with timestamps on the reader, if applicable. if (config_.with_timestamps_ && !dense) { - throw_if_not_ok((*query_r)->set_consolidation_with_timestamps()); + throw_if_not_ok(query_r->set_consolidation_with_timestamps()); } // Get last fragment URI, which will be the URI of the consolidated fragment - auto first = (*query_r)->first_fragment_uri(); - auto last = (*query_r)->last_fragment_uri(); + auto first = query_r->first_fragment_uri(); + auto last = query_r->last_fragment_uri(); auto write_version = array_for_reads->array_schema_latest().write_version(); auto fragment_name = storage_format::generate_consolidated_fragment_name( first, last, write_version); // Create write query - *query_w = tdb_new(Query, storage_manager_, array_for_writes, fragment_name); - RETURN_NOT_OK((*query_w)->set_layout(Layout::GLOBAL_ORDER)); - RETURN_NOT_OK((*query_w)->disable_checks_consolidation()); - (*query_w)->set_fragment_size(config_.max_fragment_size_); + query_w = tdb_unique_ptr( + tdb_new(Query, storage_manager_, array_for_writes, fragment_name)); + RETURN_NOT_OK(query_w->set_layout(Layout::GLOBAL_ORDER)); + RETURN_NOT_OK(query_w->disable_checks_consolidation()); + query_w->set_fragment_size(config_.max_fragment_size_); if (array_for_reads->array_schema_latest().dense()) { - RETURN_NOT_OK((*query_w)->set_subarray_unsafe(subarray)); + RETURN_NOT_OK(query_w->set_subarray_unsafe(subarray)); } // Set the processed conditions on new fragment. const auto& delete_and_update_tiles_location = - (*query_r)->array()->array_directory().delete_and_update_tiles_location(); + query_r->array()->array_directory().delete_and_update_tiles_location(); std::vector processed_conditions; processed_conditions.reserve(delete_and_update_tiles_location.size()); for (auto& location : delete_and_update_tiles_location) { processed_conditions.emplace_back(location.condition_marker()); } - (*query_w)->set_processed_conditions(processed_conditions); + query_w->set_processed_conditions(processed_conditions); // Set the URI for the new fragment. auto frag_uri = diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index f1a5141e45e8..aec52750d769 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -316,8 +316,8 @@ class FragmentConsolidator : public Consolidator { shared_ptr array_for_reads, shared_ptr array_for_writes, const NDRange& subarray, - Query** query_r, - Query** query_w, + tdb_unique_ptr& query_r, + tdb_unique_ptr& query_w, URI* new_fragment_uri); /** From 1220908f2398079a5448cbd8fd14bdbc22f51eb0 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 8 May 2024 19:26:45 +0300 Subject: [PATCH 330/456] Remove deprecated APIs from tests, part 3. (#4954) This removed deprecated APIs from tests. It will be split into multiple PRs to facilitate code reviewing. [sc-46307] --- TYPE: NO_HISTORY DESC: Remove deprecated APIs from tests, part 3. --- test/src/unit-compression-delta.cc | 4 +- test/src/unit-cppapi-array.cc | 138 ++++++++---- test/src/unit-cppapi-checksum.cc | 8 +- test/src/unit-cppapi-consolidation-sparse.cc | 4 +- ...it-cppapi-consolidation-with-timestamps.cc | 6 +- test/src/unit-cppapi-consolidation.cc | 8 +- test/src/unit-cppapi-datetimes.cc | 4 +- test/src/unit-cppapi-dense-qc-coords-mode.cc | 8 +- test/src/unit-cppapi-enumerations.cc | 28 ++- test/src/unit-cppapi-fill_values.cc | 18 +- test/src/unit-cppapi-filter.cc | 4 +- test/src/unit-cppapi-float-scaling-filter.cc | 4 +- test/src/unit-cppapi-hilbert.cc | 26 ++- test/src/unit-cppapi-nullable.cc | 4 +- test/src/unit-cppapi-query.cc | 17 +- test/src/unit-cppapi-schema-evolution.cc | 18 +- test/src/unit-cppapi-schema.cc | 5 +- test/src/unit-cppapi-string-dims.cc | 196 ++++++++++-------- test/src/unit-cppapi-subarray.cc | 85 ++++---- test/src/unit-cppapi-time.cc | 4 +- test/src/unit-cppapi-updates.cc | 4 +- test/src/unit-cppapi-var-offsets.cc | 14 +- test/src/unit-cppapi-xor-filter.cc | 4 +- test/src/unit-dense-reader.cc | 70 ++++++- test/src/unit-duplicates.cc | 57 ++++- test/src/unit-empty-var-length.cc | 30 ++- test/src/unit-query-plan.cc | 17 +- test/src/unit-request-handlers.cc | 10 +- test/src/unit-sparse-global-order-reader.cc | 16 +- .../unit-sparse-unordered-with-dups-reader.cc | 16 +- test/support/src/helpers.cc | 13 +- .../test/unit_capi_query_aggregate.cc | 114 +++++++++- .../query_field/test/unit_capi_query_field.cc | 6 +- 33 files changed, 676 insertions(+), 284 deletions(-) diff --git a/test/src/unit-compression-delta.cc b/test/src/unit-compression-delta.cc index 06a274e9c51e..f120d9439739 100644 --- a/test/src/unit-compression-delta.cc +++ b/test/src/unit-compression-delta.cc @@ -179,9 +179,11 @@ TEST_CASE( // Sanity check reading array.open(TILEDB_READ); std::vector subarray = {0, 10}; + Subarray sub(ctx, array); + sub.set_subarray(subarray); std::vector a1_read(2); Query query_r(ctx, array); - query_r.set_subarray(subarray) + query_r.set_subarray(sub) .set_layout(TILEDB_UNORDERED) .set_data_buffer("a1", a1_read); REQUIRE(query_r.submit() == Query::Status::COMPLETE); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 1c6b8a4c646a..60c82690a217 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -197,7 +197,9 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { Query query(ctx, array, TILEDB_WRITE); CHECK(query.query_type() == TILEDB_WRITE); - query.set_subarray(subarray); + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2buf.second); query.set_offsets_buffer("a2", a2buf.first); @@ -254,6 +256,8 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { a1.resize(48); Query query(ctx, array); + Subarray sub(ctx, array); + sub.set_subarray(subarray); CHECK(!query.has_results()); @@ -265,7 +269,7 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { query.set_offsets_buffer("a4", a4buf.first); query.set_data_buffer("a5", a5); query.set_layout(TILEDB_ROW_MAJOR); - query.set_subarray(subarray); + query.set_subarray(sub); // Make sure no segfault when called before submit query.result_buffer_elements(); @@ -346,7 +350,9 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { Array array(ctx, array_uri_, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); - query.set_subarray(subarray); + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2buf.second); query.set_offsets_buffer("a2", a2buf.first); @@ -405,7 +411,9 @@ TEST_CASE_METHOD(CPPArrayFx, "C++ API: Arrays", "[cppapi][basic][rest]") { std::vector subarray = {0, 1, 0, 0}; Array array(ctx, array_uri_, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); - query.set_subarray(subarray); + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_data_buffer("a1", a1); query.set_layout(TILEDB_GLOBAL_ORDER); // Incorrect subarray for global order @@ -509,8 +517,10 @@ TEST_CASE( a = {}; const std::vector subarray = {0, 2}; Query q(ctx, array, TILEDB_READ); + Subarray s(ctx, array); + s.set_subarray(subarray); q.set_layout(TILEDB_GLOBAL_ORDER); - q.set_subarray(subarray); + q.set_subarray(s); q.set_data_buffer("a", a); q.set_offsets_buffer("a", a_offset); q.set_data_buffer("b", b); @@ -603,7 +613,9 @@ TEST_CASE( Query query(ctx, array); const std::vector subarray = {0, 3, 0, 3}; std::vector data(16); - query.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data); query.submit(); @@ -757,7 +769,9 @@ TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption][non-rest]") { std::vector subarray = {0, 3}; std::vector a_read(4); Query query_r(ctx, array); - query_r.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read); query_r.submit(); @@ -769,7 +783,9 @@ TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption][non-rest]") { Array array_3(ctx, array_name, TILEDB_READ); a_read = std::vector(4, 0); Query query_r2(ctx, array_3); - query_r2.set_subarray(subarray) + Subarray sub2(ctx, array_3); + sub2.set_subarray(subarray); + query_r2.set_subarray(sub2) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read); query_r2.submit(); @@ -843,7 +859,9 @@ TEST_CASE( std::vector subarray = {0, 3}; std::vector a_read(4); Query query_r(ctx, array); - query_r.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read); query_r.submit(); @@ -909,7 +927,9 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at][rest]") { std::vector subarray = {1, 4}; std::vector a_r(4); Query query_r(ctx, array_r); - query_r.set_subarray(subarray) + Subarray sub(ctx, array_r); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r); query_r.submit(); @@ -928,7 +948,9 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at][rest]") { std::vector a_r_at_0(4); Query query_r_at_0(ctx, array_r_at_0); - query_r_at_0.set_subarray(subarray) + Subarray sub2(ctx, array_r_at_0); + sub2.set_subarray(subarray); + query_r_at_0.set_subarray(sub2) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r_at_0); query_r_at_0.submit(); @@ -950,7 +972,9 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at][rest]") { std::vector a_r_at(4); Query query_r_at(ctx, array_r_at); - query_r_at.set_subarray(subarray) + Subarray sub3(ctx, array_r_at); + sub3.set_subarray(subarray); + query_r_at.set_subarray(sub3) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r_at_0); query_r_at.submit(); @@ -964,7 +988,9 @@ TEST_CASE("C++ API: Open array at", "[cppapi][open-array-at][rest]") { CHECK(array_reopen_at.open_timestamp_end() == first_timestamp); std::vector a_r_reopen_at(4); Query query_r_reopen_at(ctx, array_reopen_at); - query_r_reopen_at.set_subarray(subarray) + Subarray sub4(ctx, array_reopen_at); + sub4.set_subarray(subarray); + query_r_reopen_at.set_subarray(sub4) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r_reopen_at); query_r_reopen_at.submit(); @@ -1007,7 +1033,9 @@ TEST_CASE( std::vector subarray = {1, 4}; std::vector a_r(4); Query query_r(ctx, array_r); - query_r.set_subarray(subarray) + Subarray sub(ctx, array_r); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r); query_r.submit(); @@ -1031,7 +1059,9 @@ TEST_CASE( std::vector a_r_at_0(4); Query query_r_at_0(ctx, array_r_at_0); - query_r_at_0.set_subarray(subarray) + Subarray sub2(ctx, array_r_at_0); + sub2.set_subarray(subarray); + query_r_at_0.set_subarray(sub2) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r_at_0); query_r_at_0.submit(); @@ -1058,7 +1088,9 @@ TEST_CASE( std::vector a_r_at(4); Query query_r_at(ctx, array_r_at); - query_r_at.set_subarray(subarray) + Subarray sub3(ctx, array_r_at); + sub3.set_subarray(subarray); + query_r_at.set_subarray(sub3) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_r_at_0); query_r_at.submit(); @@ -1105,7 +1137,9 @@ TEST_CASE( Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector data(1); - query.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data); query.submit(); @@ -1256,8 +1290,10 @@ TEST_CASE( query_r.set_data_buffer("d2", buff_d2_r); query_r.set_data_buffer("a", buff_a_r); query_r.set_layout(TILEDB_UNORDERED); - query_r.add_range(0, 1.0f, 20.0f); - query_r.add_range(1, (int64_t)1, (int64_t)30); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range(0, 1.0f, 20.0f); + subarray_r.add_range(1, (int64_t)1, (int64_t)30); + query_r.set_subarray(subarray_r); query_r.submit(); auto ret = query.result_buffer_elements(); @@ -1324,48 +1360,53 @@ TEST_CASE( std::string s1("a", 1); std::string s2("ee", 2); Query query_r(ctx, array_r, TILEDB_READ); + Subarray subarray_r(ctx, array_r); SECTION("Non empty range") { - query_r.add_range(0, s1, s2); - CHECK_THROWS(query_r.add_range(1, s1, s2)); + subarray_r.add_range(0, s1, s2); + CHECK_THROWS(subarray_r.add_range(1, s1, s2)); + query_r.set_subarray(subarray_r); // Check range - CHECK_THROWS(query_r.range(1, 1)); - std::array range = query_r.range(0, 0); + CHECK_THROWS(subarray_r.range(1, 1)); + std::array range = subarray_r.range(0, 0); CHECK(range[0] == s1); CHECK(range[1] == s2); } SECTION("Empty first range") { - query_r.add_range(0, "", s2); - CHECK_THROWS(query_r.add_range(1, "", s2)); + subarray_r.add_range(0, "", s2); + CHECK_THROWS(subarray_r.add_range(1, "", s2)); + query_r.set_subarray(subarray_r); // Check range - CHECK_THROWS(query_r.range(1, 1)); - std::array range = query_r.range(0, 0); + CHECK_THROWS(subarray_r.range(1, 1)); + std::array range = subarray_r.range(0, 0); CHECK(range[0] == ""); CHECK(range[1] == s2); } SECTION("Empty second range") { - query_r.add_range(0, s1, ""); - CHECK_THROWS(query_r.add_range(1, s1, "")); + subarray_r.add_range(0, s1, ""); + CHECK_THROWS(subarray_r.add_range(1, s1, "")); + query_r.set_subarray(subarray_r); // Check range - CHECK_THROWS(query_r.range(1, 1)); - std::array range = query_r.range(0, 0); + CHECK_THROWS(subarray_r.range(1, 1)); + std::array range = subarray_r.range(0, 0); CHECK(range[0] == s1); CHECK(range[1] == ""); empty_results = true; } SECTION("Empty ranges") { - query_r.add_range(0, std::string(""), std::string("")); - CHECK_THROWS(query_r.add_range(1, std::string(""), std::string(""))); + subarray_r.add_range(0, std::string(""), std::string("")); + CHECK_THROWS(subarray_r.add_range(1, std::string(""), std::string(""))); + query_r.set_subarray(subarray_r); // Check range - CHECK_THROWS(query_r.range(1, 1)); - std::array range = query_r.range(0, 0); + CHECK_THROWS(subarray_r.range(1, 1)); + std::array range = subarray_r.range(0, 0); CHECK(range[0] == ""); CHECK(range[1] == ""); empty_results = true; @@ -1448,7 +1489,9 @@ TEST_CASE( std::string s1("a", 1); std::string s2("ee", 2); Query query_r(ctx, array_r, TILEDB_READ); - query_r.add_range(0, s1, s2); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range(0, s1, s2); + query_r.set_subarray(subarray_r); std::string data; data.resize(10); std::vector offsets(4); @@ -1498,7 +1541,9 @@ TEST_CASE( Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector rows(1); - query.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub) .set_layout(TILEDB_GLOBAL_ORDER) .set_data_buffer("rows", rows); query.submit(); @@ -1543,7 +1588,9 @@ TEST_CASE( Query query(ctx, array); const std::vector subarray = {0, 0, 0, 0}; std::vector rows(1); - query.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub) .set_layout(TILEDB_UNORDERED) .set_data_buffer("rows", rows); query.submit(); @@ -1582,10 +1629,10 @@ TEST_CASE( // Read Array array(ctx, array_uri, TILEDB_READ); Query query(ctx, array); + Subarray subarray(ctx, array); + subarray.add_range(0, 0, 1).add_range(0, 3, 3).add_range(1, 0, 3); std::vector data(12); - query.add_range(0, 0, 1) - .add_range(0, 3, 3) - .add_range(1, 0, 3) + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data); query.submit(); @@ -1747,9 +1794,10 @@ TEST_CASE( // Read Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 2).add_range(1, 2, 4); std::vector data(6); - query.add_range(0, 1, 2) - .add_range(1, 2, 4) + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data); query.submit(); @@ -2009,7 +2057,9 @@ TEST_CASE( std::vector a_read(16); Query query(ctx, array); - query.set_subarray(subarray_read) + Subarray sub(ctx, array); + sub.set_subarray(subarray_read); + query.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", a_read); query.submit_async(); diff --git a/test/src/unit-cppapi-checksum.cc b/test/src/unit-cppapi-checksum.cc index bf793432c124..89a91dae8be1 100644 --- a/test/src/unit-cppapi-checksum.cc +++ b/test/src/unit-cppapi-checksum.cc @@ -111,7 +111,9 @@ static void run_checksum_test(tiledb_filter_type_t filter_type) { std::string a2_read_data; a2_read_data.resize(7); Query query_r(ctx2, array); - query_r.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer(tiledb::test::TILEDB_COORDS, coords_read) .set_data_buffer("a1", a1_read) @@ -182,7 +184,9 @@ static void run_checksum_test(tiledb_filter_type_t filter_type) { std::string a2_read_data2; a2_read_data2.resize(7); Query query_r2(ctx, array); - query_r2.set_subarray(subarray) + Subarray sub2(ctx, array); + sub2.set_subarray(subarray); + query_r2.set_subarray(sub2) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer(tiledb::test::TILEDB_COORDS, coords_read2) .set_data_buffer("a1", a1_read2) diff --git a/test/src/unit-cppapi-consolidation-sparse.cc b/test/src/unit-cppapi-consolidation-sparse.cc index 99f8de5755a3..f1b60494f781 100644 --- a/test/src/unit-cppapi-consolidation-sparse.cc +++ b/test/src/unit-cppapi-consolidation-sparse.cc @@ -78,9 +78,11 @@ void read_array( Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); query.set_layout(TILEDB_ROW_MAJOR); + Subarray subarray(ctx, array); for (auto& r : ranges) { - query.add_range(0, r, r); + subarray.add_range(0, r, r); } + query.set_subarray(subarray); std::vector d(100); std::vector values(100); query.set_data_buffer("d", d); diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index cfb8e0de257f..91f97861af5d 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -1309,8 +1309,10 @@ TEST_CASE_METHOD( query.set_data_buffer(tiledb_timestamps(), timestamps); // Add overlapping ranges - query.add_range(1, 2, 3); - query.add_range(1, 2, 3); + Subarray subarray(ctx_, array); + subarray.add_range(1, 2, 3); + subarray.add_range(1, 2, 3); + query.set_subarray(subarray); // Submit/finalize the query query.submit(); diff --git a/test/src/unit-cppapi-consolidation.cc b/test/src/unit-cppapi-consolidation.cc index 4febf411d467..47f6eb36e1e7 100644 --- a/test/src/unit-cppapi-consolidation.cc +++ b/test/src/unit-cppapi-consolidation.cc @@ -63,7 +63,9 @@ void write_array( Array array(ctx, array_name, TILEDB_WRITE); Query query(ctx, array, TILEDB_WRITE); query.set_layout(TILEDB_ROW_MAJOR); - query.set_subarray(subarray); + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_data_buffer("a", values); query.submit(); array.close(); @@ -77,7 +79,9 @@ void read_array( Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); query.set_layout(TILEDB_ROW_MAJOR); - query.set_subarray(subarray); + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub); std::vector values(10); query.set_data_buffer("a", values); query.submit(); diff --git a/test/src/unit-cppapi-datetimes.cc b/test/src/unit-cppapi-datetimes.cc index 898ffb7dd06a..e5e94ab0f852 100644 --- a/test/src/unit-cppapi-datetimes.cc +++ b/test/src/unit-cppapi-datetimes.cc @@ -128,9 +128,11 @@ TEST_CASE("C++ API: Datetime dimension", "[cppapi][datetime]") { std::vector subarray_r = {0, 9}; Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r); + Subarray sub(ctx, array_r); + sub.set_subarray(subarray_r); query_r.set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data_r) - .set_subarray(subarray_r); + .set_subarray(sub); REQUIRE(query_r.submit() == Query::Status::COMPLETE); auto result_num = query_r.result_buffer_elements()["a"].second; diff --git a/test/src/unit-cppapi-dense-qc-coords-mode.cc b/test/src/unit-cppapi-dense-qc-coords-mode.cc index 07820a1d6cc9..6fdeac95ed44 100644 --- a/test/src/unit-cppapi-dense-qc-coords-mode.cc +++ b/test/src/unit-cppapi-dense-qc-coords-mode.cc @@ -75,7 +75,9 @@ struct CPPDenseQcCoordsModeFx { Array array(ctx_, array_name, TILEDB_WRITE); Query query(ctx_, array, TILEDB_WRITE); - query.set_subarray(subarray); + Subarray sub(ctx_, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_data_buffer("a1", a1_buff); query.set_layout(TILEDB_ROW_MAJOR); REQUIRE(query.submit() == Query::Status::COMPLETE); @@ -167,7 +169,9 @@ TEST_CASE_METHOD( std::vector subarray = {1, 10, 1, 10}; std::vector d1(100); std::vector d2(100); - query.set_subarray(subarray); + Subarray sub(ctx_, array); + sub.set_subarray(subarray); + query.set_subarray(sub); query.set_layout(read_layout); query.set_data_buffer("d1", d1); query.set_data_buffer("d2", d2); diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc index baf04bbea852..73ad75c4d5b0 100644 --- a/test/src/unit-cppapi-enumerations.cc +++ b/test/src/unit-cppapi-enumerations.cc @@ -413,7 +413,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -449,7 +451,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -490,7 +494,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -536,7 +542,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -568,7 +576,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -593,7 +603,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr1", attr1) @@ -639,7 +651,9 @@ TEST_CASE_METHOD( auto array = Array(ctx_, uri_, TILEDB_READ); Query query(ctx_, array); - query.add_range("dim", 1, 5) + Subarray subarray(ctx_, array); + subarray.add_range("dim", 1, 5); + query.set_subarray(subarray) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("dim", dim) .set_data_buffer("attr3", attr3) diff --git a/test/src/unit-cppapi-fill_values.cc b/test/src/unit-cppapi-fill_values.cc index a8efe8f77752..47c1cfe1cb31 100644 --- a/test/src/unit-cppapi-fill_values.cc +++ b/test/src/unit-cppapi-fill_values.cc @@ -587,8 +587,10 @@ TEST_CASE( Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); - query.add_range(0, 2, 3); - query.add_range(0, 9, 10); + Subarray subarray(ctx, array); + subarray.add_range(0, 2, 3); + subarray.add_range(0, 9, 10); + query.set_subarray(subarray); auto est_a1 = query.est_result_size("a1"); auto est_a2 = query.est_result_size_var("a2"); auto est_a3 = query.est_result_size("a3"); @@ -655,8 +657,10 @@ TEST_CASE( Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); - query.add_range(0, 2, 3); - query.add_range(0, 9, 10); + Subarray subarray(ctx, array); + subarray.add_range(0, 2, 3); + subarray.add_range(0, 9, 10); + query.set_subarray(subarray); auto est_a1 = query.est_result_size("a1"); auto est_a2 = query.est_result_size_var("a2"); auto est_a3 = query.est_result_size("a3"); @@ -754,8 +758,10 @@ TEST_CASE( Array array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); - query.add_range(0, 2, 3); - query.add_range(0, 9, 10); + Subarray subarray(ctx, array); + subarray.add_range(0, 2, 3); + subarray.add_range(0, 9, 10); + query.set_subarray(subarray); auto est_a1 = query.est_result_size_nullable("a1"); auto est_a2 = query.est_result_size_var_nullable("a2"); auto est_a3 = query.est_result_size_nullable("a3"); diff --git a/test/src/unit-cppapi-filter.cc b/test/src/unit-cppapi-filter.cc index 976525bfc5d8..f6e8e0ba3abe 100644 --- a/test/src/unit-cppapi-filter.cc +++ b/test/src/unit-cppapi-filter.cc @@ -208,7 +208,9 @@ TEST_CASE( std::string a2_read_data; a2_read_data.resize(7); Query query_r(ctx, array); - query_r.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query_r.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a1", a1_read) .set_data_buffer("a2", a2_read_data) diff --git a/test/src/unit-cppapi-float-scaling-filter.cc b/test/src/unit-cppapi-float-scaling-filter.cc index 8a9adf681b4d..e7451d5d1f35 100644 --- a/test/src/unit-cppapi-float-scaling-filter.cc +++ b/test/src/unit-cppapi-float-scaling-filter.cc @@ -149,10 +149,12 @@ struct FloatScalingFilterTestStruct { Query query_r(ctx, array_r); query_r.set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", a_data_read); + Subarray subarray_r(ctx, array_r); if (array_type == TILEDB_DENSE) { int range[] = {1, dim_hi}; - query_r.add_range("rows", range[0], range[1]) + subarray_r.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query_r.set_subarray(subarray_r); } query_r.submit(); diff --git a/test/src/unit-cppapi-hilbert.cc b/test/src/unit-cppapi-hilbert.cc index 2a36bc1ccf17..318951a81260 100644 --- a/test/src/unit-cppapi-hilbert.cc +++ b/test/src/unit-cppapi-hilbert.cc @@ -384,9 +384,11 @@ TEST_CASE( query_r.set_data_buffer("d1", r_buff_d1); query_r.set_data_buffer("d2", r_buff_d2); - query_r.add_range("d1", (int32_t)1, (int32_t)5); - query_r.add_range("d2", (int32_t)1, (int32_t)3); - query_r.add_range("d2", (int32_t)2, (int32_t)4); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range("d1", (int32_t)1, (int32_t)5); + subarray_r.add_range("d2", (int32_t)1, (int32_t)3); + subarray_r.add_range("d2", (int32_t)2, (int32_t)4); + query_r.set_subarray(subarray_r); query_r.set_layout(TILEDB_UNORDERED); // Submit query query_r.submit(); @@ -1874,8 +1876,10 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_offsets_buffer("d2", r_off_d2); query_r.set_layout(TILEDB_ROW_MAJOR); - query_r.add_range(0, std::string("3"), std::string("z")); - query_r.add_range(1, std::string("a"), std::string("vase")); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range(0, std::string("3"), std::string("z")); + subarray_r.add_range(1, std::string("a"), std::string("vase")); + query_r.set_subarray(subarray_r); CHECK_NOTHROW(query_r.submit()); // Check results @@ -1915,8 +1919,10 @@ TEST_CASE( query_r.set_data_buffer("d2", r_buff_d2); query_r.set_offsets_buffer("d2", r_off_d2); query_r.set_layout(TILEDB_GLOBAL_ORDER); - query_r.add_range(0, std::string("3"), std::string("z")); - query_r.add_range(1, std::string("a"), std::string("vase")); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range(0, std::string("3"), std::string("z")); + subarray_r.add_range(1, std::string("a"), std::string("vase")); + query_r.set_subarray(subarray_r); CHECK_NOTHROW(query_r.submit()); // Check results. Hilbert values: @@ -2087,8 +2093,10 @@ TEST_CASE( query_r.set_offsets_buffer("d1", r_off_d1); query_r.set_data_buffer("d2", r_buff_d2); query_r.set_offsets_buffer("d2", r_off_d2); - query_r.add_range(0, std::string("1a", 2), std::string("w", 1)); - query_r.add_range(1, std::string("ca", 2), std::string("t1", 2)); + Subarray subarray_r(ctx, array_r); + subarray_r.add_range(0, std::string("1a", 2), std::string("w", 1)); + subarray_r.add_range(1, std::string("ca", 2), std::string("t1", 2)); + query_r.set_subarray(subarray_r); query_r.set_layout(TILEDB_GLOBAL_ORDER); CHECK_NOTHROW(query_r.submit()); diff --git a/test/src/unit-cppapi-nullable.cc b/test/src/unit-cppapi-nullable.cc index b2ae676464e4..3a353c81e1ba 100644 --- a/test/src/unit-cppapi-nullable.cc +++ b/test/src/unit-cppapi-nullable.cc @@ -314,7 +314,9 @@ void NullableArrayCppFx::read( } // Set the subarray to read. - query.set_subarray(subarray); + Subarray sub(ctx_, array); + sub.set_subarray(subarray); + query.set_subarray(sub); // Submit the query. REQUIRE(query.submit() == Query::Status::COMPLETE); diff --git a/test/src/unit-cppapi-query.cc b/test/src/unit-cppapi-query.cc index 10f5de0fa39d..f04df5d755ab 100644 --- a/test/src/unit-cppapi-query.cc +++ b/test/src/unit-cppapi-query.cc @@ -206,7 +206,9 @@ TEST_CASE( Array array1(ctx, array_name, TILEDB_READ); Query query1(ctx, array1); int range[] = {1, 2}; - query1.add_range(0, range[0], range[1]).add_range(1, range[0], range[1]); + Subarray subarray1(ctx, array1); + subarray1.add_range(0, range[0], range[1]).add_range(1, range[0], range[1]); + query1.set_subarray(subarray1); auto est_size = query1.est_result_size("a"); std::vector data(est_size); query1.set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", data); @@ -266,21 +268,22 @@ TEST_CASE( // Add 1 range per dimension std::string s1("a", 1); std::string s2("cc", 2); - CHECK_NOTHROW(query.add_range("d1", s1, s2)); + Subarray subarray(ctx, array); + CHECK_NOTHROW(subarray.add_range("d1", s1, s2)); int range[] = {1, 2}; - CHECK_NOTHROW(query.add_range("d2", range[0], range[1])); + CHECK_NOTHROW(subarray.add_range("d2", range[0], range[1])); // Check number of ranges on each dimension - int range_num = query.range_num("d1"); + int range_num = subarray.range_num("d1"); CHECK(range_num == 1); - range_num = query.range_num("d2"); + range_num = subarray.range_num("d2"); CHECK(range_num == 1); // Check ranges - std::array range1 = query.range("d1", 0); + std::array range1 = subarray.range("d1", 0); CHECK(range1[0] == s1); CHECK(range1[1] == s2); - std::array range2 = query.range("d2", 0); + std::array range2 = subarray.range("d2", 0); CHECK(range2[0] == 1); CHECK(range2[1] == 2); CHECK(range2[2] == 0); diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 83a64f6e33e0..a6e2d2370180 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -180,8 +180,9 @@ TEST_CASE( // Prepare the query Query query(ctx, array, TILEDB_READ); - query.add_range(0, 1, 4) - .add_range(1, 1, 4) + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(1, 1, 4); + query.set_subarray(subarray) .set_layout(layout) .set_data_buffer("a", data) .set_data_buffer("d1", d1_data) @@ -301,8 +302,9 @@ TEST_CASE( // Prepare the query Query query(ctx, array, TILEDB_READ); - query.add_range(0, 1, 4) - .add_range(1, 1, 4) + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(1, 1, 4); + query.set_subarray(subarray) .set_layout(layout) .set_data_buffer("a", a_data) .set_data_buffer("b", b_data) @@ -452,10 +454,10 @@ TEST_CASE( // Prepare the query Query query(ctx, array, TILEDB_READ); - query.add_range(0, 1, 4) - .add_range(0, 1, 4) - .add_range(1, 1, 4) - .add_range(1, 1, 4) + Subarray subarray(ctx, array); + subarray.add_range(0, 1, 4).add_range(0, 1, 4).add_range(1, 1, 4).add_range( + 1, 1, 4); + query.set_subarray(subarray) .set_layout(layout) .set_data_buffer("a", a_data) .set_data_buffer("b", b_data) diff --git a/test/src/unit-cppapi-schema.cc b/test/src/unit-cppapi-schema.cc index 6dc878aa9f4f..b0c731c8638b 100644 --- a/test/src/unit-cppapi-schema.cc +++ b/test/src/unit-cppapi-schema.cc @@ -250,9 +250,10 @@ TEST_CASE( CHECK_THROWS(array.non_empty_domain()); std::vector subarray = {1, 2, 1, 3}; - // Query checks + // Query/subarray checks Query query(ctx, array, TILEDB_READ); - CHECK_THROWS(query.set_subarray(subarray)); + Subarray sub(ctx, array); + CHECK_THROWS(sub.set_subarray(subarray)); std::vector buff = {1, 2, 4}; CHECK_THROWS(query.set_data_buffer(tiledb::test::TILEDB_COORDS, buff)); diff --git a/test/src/unit-cppapi-string-dims.cc b/test/src/unit-cppapi-string-dims.cc index f489d7e6b01e..2424df91e31f 100644 --- a/test/src/unit-cppapi-string-dims.cc +++ b/test/src/unit-cppapi-string-dims.cc @@ -167,12 +167,14 @@ TEST_CASE( // Prepare a read query. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); + query_read.set_subarray(subarray_read); // Prepare buffers with small enough sizes to ensure the string dimension // must split. @@ -254,6 +256,7 @@ TEST_CASE( // Prepare a read query. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); auto dim3_non_empty_domain = array_read.non_empty_domain_var(2); @@ -264,7 +267,7 @@ TEST_CASE( // editors. switch (option) { case 1: - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); expected_dim1 = {"bb", "c", "d"}; expected_dim2 = {1, 1, 2}; @@ -272,11 +275,11 @@ TEST_CASE( expected_a1_data = {2, 3, 4}; break; case 2: - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c", "d"}; expected_dim2 = {1, 2}; @@ -284,9 +287,9 @@ TEST_CASE( expected_a1_data = {3, 4}; break; case 3: - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"a", "bb", "c", "d", "ee", "f"}; expected_dim2 = {1, 1, 1, 2, 2, 2}; @@ -294,15 +297,15 @@ TEST_CASE( expected_a1_data = {1, 2, 3, 4, 5, 6}; break; case 4: - query_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range(0, std::string("c"), std::string("d")); expected_dim1 = {"c", "d"}; expected_dim2 = {1, 2}; expected_dim3 = {"i", "j"}; expected_a1_data = {3, 4}; break; case 5: - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"c", "d"}; expected_dim2 = {1, 2}; @@ -310,9 +313,9 @@ TEST_CASE( expected_a1_data = {3, 4}; break; case 6: - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"bb", "c", "d"}; expected_dim2 = {1, 1, 2}; @@ -320,9 +323,9 @@ TEST_CASE( expected_a1_data = {2, 3, 4}; break; case 7: - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c", "d"}; expected_dim2 = {1, 2}; @@ -330,8 +333,8 @@ TEST_CASE( expected_a1_data = {3, 4}; break; case 8: - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c", "d"}; expected_dim2 = {1, 2}; @@ -339,9 +342,9 @@ TEST_CASE( expected_a1_data = {3, 4}; break; case 9: - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c", "d", "ee"}; expected_dim2 = {1, 2, 2}; @@ -349,8 +352,8 @@ TEST_CASE( expected_a1_data = {3, 4, 5}; break; case 10: - query_read.add_range(0, std::string("c"), std::string("ee")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("ee")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kk")); expected_dim1 = {"c", "d", "ee"}; expected_dim2 = {1, 2, 2}; @@ -358,6 +361,7 @@ TEST_CASE( expected_a1_data = {3, 4, 5}; break; } + query_read.set_subarray(subarray_read); std::vector dim1; std::vector dim1_offsets; @@ -481,6 +485,7 @@ TEST_CASE( // Prepare a read query. Do not set a range for the last string dimension. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); @@ -497,7 +502,7 @@ TEST_CASE( case 1: expected_result_num = 4; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -507,11 +512,11 @@ TEST_CASE( case 2: expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -521,9 +526,9 @@ TEST_CASE( case 3: expected_result_num = 6; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"a", "a", "bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2, 1, 2}; @@ -533,7 +538,7 @@ TEST_CASE( case 4: expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range(0, std::string("c"), std::string("d")); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; expected_dim3 = {"i", "l"}; @@ -542,8 +547,8 @@ TEST_CASE( case 5: expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; @@ -553,9 +558,9 @@ TEST_CASE( case 6: expected_result_num = 4; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -565,9 +570,9 @@ TEST_CASE( case 7: expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -578,8 +583,8 @@ TEST_CASE( expected_result_num = 1; initial_result_num = 1; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c"}; expected_dim2 = {1}; @@ -590,9 +595,9 @@ TEST_CASE( expected_result_num = 3; initial_result_num = 3; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"a", "bb", "c"}; expected_dim2 = {2, 2, 1}; @@ -600,6 +605,7 @@ TEST_CASE( expected_a1_data = {4, 5, 3}; break; } + query_read.set_subarray(subarray_read); std::vector dim1; std::vector dim1_offsets; @@ -721,6 +727,7 @@ TEST_CASE( // Prepare a read query. Do not set a range for the last string dimension. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); @@ -738,7 +745,7 @@ TEST_CASE( which_option = 1; expected_result_num = 4; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -749,11 +756,11 @@ TEST_CASE( which_option = 2; expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -764,9 +771,9 @@ TEST_CASE( which_option = 3; expected_result_num = 6; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"a", "a", "bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2, 1, 2}; @@ -777,7 +784,7 @@ TEST_CASE( which_option = 4; expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range(0, std::string("c"), std::string("d")); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; expected_dim3 = {"i", "l"}; @@ -787,8 +794,8 @@ TEST_CASE( which_option = 5; expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; @@ -799,9 +806,9 @@ TEST_CASE( which_option = 6; expected_result_num = 4; initial_expected_read_status = Query::Status::INCOMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -812,9 +819,9 @@ TEST_CASE( which_option = 7; expected_result_num = 2; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -826,8 +833,8 @@ TEST_CASE( expected_result_num = 1; initial_result_num = 1; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c"}; expected_dim2 = {1}; @@ -839,15 +846,16 @@ TEST_CASE( expected_result_num = 3; initial_result_num = 3; initial_expected_read_status = Query::Status::COMPLETE; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"a", "bb", "c"}; expected_dim2 = {2, 2, 1}; expected_dim3 = {"j", "kk", "i"}; expected_a1_data = {4, 5, 3}; } + query_read.set_subarray(subarray_read); std::vector dim1_offsets; std::vector dim2; @@ -995,6 +1003,7 @@ TEST_CASE( // Prepare a read query. Ranges set vary in the different switch cases. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); @@ -1007,7 +1016,7 @@ TEST_CASE( switch (option) { case 1: expected_result_num = 4; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -1017,11 +1026,11 @@ TEST_CASE( break; case 2: expected_result_num = 2; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -1030,9 +1039,9 @@ TEST_CASE( break; case 3: expected_result_num = 6; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"a", "a", "bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2, 1, 2}; @@ -1041,7 +1050,7 @@ TEST_CASE( break; case 4: expected_result_num = 2; - query_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range(0, std::string("c"), std::string("d")); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; expected_dim3 = {"i", "l"}; @@ -1049,8 +1058,8 @@ TEST_CASE( break; case 5: expected_result_num = 2; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; @@ -1059,9 +1068,9 @@ TEST_CASE( break; case 6: expected_result_num = 4; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -1070,9 +1079,9 @@ TEST_CASE( break; case 7: expected_result_num = 2; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -1081,8 +1090,8 @@ TEST_CASE( break; case 8: expected_result_num = 1; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c"}; expected_dim2 = {1}; @@ -1091,9 +1100,9 @@ TEST_CASE( break; case 9: expected_result_num = 3; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"a", "bb", "c"}; expected_dim2 = {2, 2, 1}; @@ -1101,6 +1110,7 @@ TEST_CASE( expected_a1_data = {4, 5, 3}; break; } + query_read.set_subarray(subarray_read); std::vector dim1; std::vector dim1_offsets; @@ -1247,6 +1257,7 @@ TEST_CASE( // Prepare a read query. Ranges set vary in the different switch cases. Array array_read(ctx, array_name, TILEDB_READ); Query query_read(ctx, array_read, TILEDB_READ); + Subarray subarray_read(ctx, array_read); auto dim1_non_empty_domain = array_read.non_empty_domain_var(0); auto dim2_non_empty_domain = array_read.non_empty_domain(1); @@ -1259,7 +1270,7 @@ TEST_CASE( switch (option) { case 1: expected_result_num = 4; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -1269,11 +1280,11 @@ TEST_CASE( break; case 2: expected_result_num = 2; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -1282,9 +1293,9 @@ TEST_CASE( break; case 3: expected_result_num = 6; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"a", "a", "bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2, 1, 2}; @@ -1293,7 +1304,7 @@ TEST_CASE( break; case 4: expected_result_num = 2; - query_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range(0, std::string("c"), std::string("d")); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; expected_dim3 = {"i", "l"}; @@ -1301,8 +1312,8 @@ TEST_CASE( break; case 5: expected_result_num = 2; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"c", "c"}; expected_dim2 = {1, 2}; @@ -1311,9 +1322,9 @@ TEST_CASE( break; case 6: expected_result_num = 4; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( 1, dim2_non_empty_domain.first, dim2_non_empty_domain.second); expected_dim1 = {"bb", "bb", "c", "c"}; expected_dim2 = {1, 2, 1, 2}; @@ -1322,9 +1333,9 @@ TEST_CASE( break; case 7: expected_result_num = 2; - query_read.add_range( + subarray_read.add_range( std::string("dim1"), std::string("az"), std::string("de")); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"bb", "c"}; expected_dim2 = {2, 1}; @@ -1333,8 +1344,8 @@ TEST_CASE( break; case 8: expected_result_num = 1; - query_read.add_range(0, std::string("c"), std::string("d")); - query_read.add_range( + subarray_read.add_range(0, std::string("c"), std::string("d")); + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"c"}; expected_dim2 = {1}; @@ -1343,9 +1354,9 @@ TEST_CASE( break; case 9: expected_result_num = 3; - query_read.add_range( + subarray_read.add_range( 0, dim1_non_empty_domain.first, dim1_non_empty_domain.second); - query_read.add_range( + subarray_read.add_range( std::string("dim3"), std::string("i"), std::string("kl")); expected_dim1 = {"a", "bb", "c"}; expected_dim2 = {2, 2, 1}; @@ -1353,6 +1364,7 @@ TEST_CASE( expected_a1_data = {4, 5, 3}; break; } + query_read.set_subarray(subarray_read); std::vector dim1; std::vector dim1_offsets; @@ -1479,8 +1491,12 @@ void read_and_check_sparse_array_string_dim( std::string data_back; data_back.resize(expected_data.size()); + Subarray subarray(ctx, array); + subarray.add_range( + "dim1", std::string("ATSD987JIO"), std::string("TGSD987JPO")); + Query query(ctx, array, TILEDB_READ); - query.add_range("dim1", std::string("ATSD987JIO"), std::string("TGSD987JPO")); + query.set_subarray(subarray); query.set_data_buffer("dim1", (char*)data_back.data(), data_back.size()); query.set_offsets_buffer("dim1", offsets_back.data(), offsets_back.size()); query.set_layout(layout); diff --git a/test/src/unit-cppapi-subarray.cc b/test/src/unit-cppapi-subarray.cc index ffe46e7ed5af..e781b774a7a6 100644 --- a/test/src/unit-cppapi-subarray.cc +++ b/test/src/unit-cppapi-subarray.cc @@ -80,8 +80,10 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); int range[] = {0, 0}; - query.add_range(0, range[0], range[1]); - query.add_range(1, range[0], range[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, range[0], range[1]); + subarray.add_range(1, range[0], range[1]); + query.set_subarray(subarray); auto est_size = query.est_result_size("a"); REQUIRE(est_size == 4); @@ -118,7 +120,9 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); int range[] = {1, 2}; - query.add_range(0, range[0], range[1]).add_range(1, range[0], range[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, range[0], range[1]).add_range(1, range[0], range[1]); + query.set_subarray(subarray); auto est_size = query.est_result_size("a"); REQUIRE(est_size == 4); @@ -158,27 +162,29 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); int range0[] = {0, 0}, range1[] = {2, 2}; - query.add_range(0, range0[0], range0[1]); - query.add_range(1, range0[0], range0[1]); - query.add_range(0, range1[0], range1[1]); - query.add_range(1, range1[0], range1[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, range0[0], range0[1]); + subarray.add_range(1, range0[0], range0[1]); + subarray.add_range(0, range1[0], range1[1]); + subarray.add_range(1, range1[0], range1[1]); int64_t inv_range[] = {0, 1}; - CHECK_THROWS(query.add_range(1, inv_range[0], inv_range[1])); + CHECK_THROWS(subarray.add_range(1, inv_range[0], inv_range[1])); + query.set_subarray(subarray); // Get range - auto range = query.range(0, 0); + auto range = subarray.range(0, 0); CHECK(range[0] == 0); CHECK(range[1] == 0); CHECK(range[2] == 0); - range = query.range(1, 1); + range = subarray.range(1, 1); CHECK(range[0] == 2); CHECK(range[1] == 2); CHECK(range[2] == 0); - CHECK_THROWS(range = query.range(1, 3)); + CHECK_THROWS(range = subarray.range(1, 3)); std::array range2 = {{0, 0, 0}}; - CHECK_THROWS(range2 = query.range(1, 1)); + CHECK_THROWS(range2 = subarray.range(1, 1)); auto est_size = query.est_result_size("a"); REQUIRE(est_size == 4); @@ -235,10 +241,12 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Query query(ctx, array); int range0[] = {0, 1}, range1[] = {2, 3}; - query.add_range(0, range0[0], range0[1]); - query.add_range(1, range0[0], range0[1]); - query.add_range(0, range1[0], range1[1]); - query.add_range(1, range1[0], range1[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, range0[0], range0[1]); + subarray.add_range(1, range0[0], range0[1]); + subarray.add_range(0, range1[0], range1[1]); + subarray.add_range(1, range1[0], range1[1]); + query.set_subarray(subarray); auto est_size = query.est_result_size("a"); std::vector data(est_size); @@ -282,8 +290,10 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { query.set_config(config); int range[] = {1, 4}; int range2[] = {-1, 3}; - REQUIRE_THROWS(query.add_range(0, range[0], range[1])); - REQUIRE_THROWS(query.add_range(1, range2[0], range2[1])); + Subarray subarray(ctx, array); + REQUIRE_THROWS(subarray.add_range(0, range[0], range[1])); + REQUIRE_THROWS(subarray.add_range(1, range2[0], range2[1])); + query.set_subarray(subarray); } SECTION("- Read ranges oob error - Subarray-cppapi") { @@ -319,8 +329,11 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { query.set_config(config); int range[] = {1, 4}; int range2[] = {-1, 3}; - query.add_range(0, range[0], range[1]); - query.add_range(1, range2[0], range2[1]); + Subarray subarray(ctx, array); + subarray.set_config(config); + subarray.add_range(0, range[0], range[1]); + subarray.add_range(1, range2[0], range2[1]); + query.set_subarray(subarray); auto est_size = query.est_result_size("a"); REQUIRE(est_size == 12); @@ -335,18 +348,6 @@ TEST_CASE("C++ API: Test subarray", "[cppapi][sparse][subarray]") { REQUIRE(data[1] == 3); } - SECTION("- set_subarray Write ranges oob warn - Subarray-query") { - tiledb::Array array(ctx, array_name, TILEDB_WRITE); - tiledb::Query query(ctx, array); - // throws on sparse array (set_subarray not supported) - REQUIRE_THROWS(query.set_subarray({1, 4, -1, 3})); - tiledb::Config config; - config.set("sm.read_range_oob", "warn"); - query.set_config(config); - // throws on sparse array - REQUIRE_THROWS(query.set_subarray({1, 4, -1, 3})); - } - SECTION("- Read ranges oob warn - Subarray-cppapi") { tiledb::Array array(ctx, array_name, TILEDB_READ); tiledb::Subarray subarray(ctx, array); @@ -572,14 +573,16 @@ TEST_CASE( int row_range[] = {0, 1}; int col_range0[] = {12277, 13499}; int col_range1[] = {13500, 17486}; - query.add_range(0, row_range[0], row_range[1]); - query.add_range(1, col_range0[0], col_range0[1]); - query.add_range(1, col_range1[0], col_range1[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, row_range[0], row_range[1]); + subarray.add_range(1, col_range0[0], col_range0[1]); + subarray.add_range(1, col_range1[0], col_range1[1]); + query.set_subarray(subarray); // Test range num - auto range_num = query.range_num(0); + auto range_num = subarray.range_num(0); CHECK(range_num == 1); - range_num = query.range_num(1); + range_num = subarray.range_num(1); // Ranges `col_range0` and `col_range1` are coalesced. CHECK(range_num == 1); @@ -1034,9 +1037,11 @@ TEST_CASE( int row_range[] = {1, 1}; int col_range0[] = {12277, 12277}; int col_range1[] = {12277, 13160}; - query.add_range(0, row_range[0], row_range[1]); - query.add_range(1, col_range0[0], col_range0[1]); - query.add_range(1, col_range1[0], col_range1[1]); + Subarray subarray(ctx, array); + subarray.add_range(0, row_range[0], row_range[1]); + subarray.add_range(1, col_range0[0], col_range0[1]); + subarray.add_range(1, col_range1[0], col_range1[1]); + query.set_subarray(subarray); query.set_layout(TILEDB_UNORDERED); // Allocate buffers large enough to hold 2 cells at a time. diff --git a/test/src/unit-cppapi-time.cc b/test/src/unit-cppapi-time.cc index 592d45c89446..37a92ac23120 100644 --- a/test/src/unit-cppapi-time.cc +++ b/test/src/unit-cppapi-time.cc @@ -148,9 +148,11 @@ TEST_CASE("C++ API: Time dimension", "[cppapi][time]") { std::vector subarray_r = {0, 9}; Array array_r(ctx, array_name, TILEDB_READ); Query query_r(ctx, array_r); + Subarray sub(ctx, array_r); + sub.set_subarray(subarray_r); query_r.set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a", data_r) - .set_subarray(subarray_r); + .set_subarray(sub); REQUIRE(query_r.submit() == Query::Status::COMPLETE); auto result_num = query_r.result_buffer_elements()["a"].second; diff --git a/test/src/unit-cppapi-updates.cc b/test/src/unit-cppapi-updates.cc index 81d92fefe650..03cac576303d 100644 --- a/test/src/unit-cppapi-updates.cc +++ b/test/src/unit-cppapi-updates.cc @@ -104,7 +104,9 @@ TEST_CASE( std::vector r_data_a1; r_data_a1.resize(300); - query.set_subarray(subarray) + Subarray sub(ctx, array); + sub.set_subarray(subarray); + query.set_subarray(sub) .set_layout(TILEDB_ROW_MAJOR) .set_data_buffer("a1", r_data_a1) .set_offsets_buffer("a1", r_offsets_a1); diff --git a/test/src/unit-cppapi-var-offsets.cc b/test/src/unit-cppapi-var-offsets.cc index 8d7832b625b1..6c5020f513df 100644 --- a/test/src/unit-cppapi-var-offsets.cc +++ b/test/src/unit-cppapi-var-offsets.cc @@ -226,8 +226,10 @@ struct VariableOffsetsFx { // Query outside unwritten coordinates of the array int64_t d1_start = 1, d1_end = 2; int64_t d2_start = 3, d2_end = 4; - query.add_range("d1", d1_start, d1_end); - query.add_range("d2", d2_start, d2_end); + Subarray subarray(ctx, array); + subarray.add_range("d1", d1_start, d1_end); + subarray.add_range("d2", d2_start, d2_end); + query.set_subarray(subarray); // Submit query query.submit(); @@ -1742,7 +1744,9 @@ TEST_CASE_METHOD( auto array = tiledb::Array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); - query.add_range(0, std::string("aa"), std::string("dddd")); + Subarray subarray(ctx, array); + subarray.add_range(0, std::string("aa"), std::string("dddd")); + query.set_subarray(subarray); query.set_data_buffer("dim1", (char*)data_back.data(), data_back.size()); query.set_offsets_buffer( "dim1", (uint64_t*)offsets_back.data(), offsets_back.size()); @@ -1782,7 +1786,9 @@ TEST_CASE_METHOD( auto array = tiledb::Array(ctx, array_name, TILEDB_READ); Query query(ctx, array, TILEDB_READ); // this query range should return empty result - query.add_range(0, std::string("xyz"), std::string("xyz")); + Subarray subarray(ctx, array); + subarray.add_range(0, std::string("xyz"), std::string("xyz")); + query.set_subarray(subarray); query.set_data_buffer("dim1", (char*)data_back.data(), data_back.size()); // here we set the buffer at an offset of 2*uint64_t (== 4 * uint32_t) diff --git a/test/src/unit-cppapi-xor-filter.cc b/test/src/unit-cppapi-xor-filter.cc index 2ae04e08fe86..4c639e84cca9 100644 --- a/test/src/unit-cppapi-xor-filter.cc +++ b/test/src/unit-cppapi-xor-filter.cc @@ -103,10 +103,12 @@ void xor_filter_api_test(Context& ctx, tiledb_array_type_t array_type) { Query query_r(ctx, array_r); query_r.set_layout(TILEDB_ROW_MAJOR).set_data_buffer("a", a_data_read); + Subarray subarray_r(ctx, array_r); if (array_type == TILEDB_DENSE) { int range[] = {1, xor_dim_hi}; - query_r.add_range("rows", range[0], range[1]) + subarray_r.add_range("rows", range[0], range[1]) .add_range("cols", range[0], range[1]); + query_r.set_subarray(subarray_r); } query_r.submit(); diff --git a/test/src/unit-dense-reader.cc b/test/src/unit-dense-reader.cc index 51d61cf3ee46..28d3bebb273a 100644 --- a/test/src/unit-dense-reader.cc +++ b/test/src/unit-dense-reader.cc @@ -317,6 +317,7 @@ void CDenseFx::write_1d_fragment( // Create the query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -325,8 +326,13 @@ void CDenseFx::write_1d_fragment( REQUIRE(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query. rc = tiledb_query_submit(ctx_, query); @@ -356,6 +362,7 @@ void CDenseFx::write_1d_fragment_strings( // Create the query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -366,8 +373,13 @@ void CDenseFx::write_1d_fragment_strings( REQUIRE(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query. rc = tiledb_query_submit(ctx_, query); @@ -399,6 +411,7 @@ void CDenseFx::write_1d_fragment_fixed_strings( // Create the query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); @@ -412,8 +425,13 @@ void CDenseFx::write_1d_fragment_fixed_strings( REQUIRE(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Submit query. rc = tiledb_query_submit(ctx_, query); @@ -443,12 +461,18 @@ void CDenseFx::read( // Create query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); @@ -534,12 +558,18 @@ void CDenseFx::read_evolved( // Create query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); @@ -576,12 +606,18 @@ void CDenseFx::read_strings( // Create query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); @@ -655,12 +691,18 @@ void CDenseFx::read_strings_evolved( // Create query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); @@ -706,12 +748,18 @@ void CDenseFx::read_fixed_strings( // Create query. tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-duplicates.cc b/test/src/unit-duplicates.cc index 4ba0745cac5d..cd10644e2a87 100644 --- a/test/src/unit-duplicates.cc +++ b/test/src/unit-duplicates.cc @@ -137,6 +137,7 @@ TEST_CASE_METHOD( // Create the query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -176,8 +177,13 @@ TEST_CASE_METHOD( // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", data_r, &data_r_size); @@ -230,6 +236,7 @@ TEST_CASE_METHOD( // Create the query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); @@ -303,8 +310,13 @@ TEST_CASE_METHOD( // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", data_r, &data_r_size); @@ -365,8 +377,13 @@ TEST_CASE_METHOD( // Create query rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); rc = tiledb_query_set_layout(ctx_, query, TILEDB_ROW_MAJOR); CHECK(rc == TILEDB_OK); rc = tiledb_query_set_data_buffer(ctx_, query, "a", data_r, &data_r_size); @@ -493,12 +510,19 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer(ctx_, query, "d", coords_r, &coords_r_size); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + // Set multi-range subarray to query int start_1 = 1, end_1 = 2; int start_2 = 4, end_2 = 10; - rc = tiledb_query_add_range(ctx_, query, 0, &start_1, &end_1, NULL); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start_1, &end_1, NULL); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &start_2, &end_2, NULL); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start_2, &end_2, NULL); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Submit query @@ -534,6 +558,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); // Validate results CHECK(num_found_coords[1] == 2); @@ -613,9 +638,16 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer(ctx_, query, "d", coords_r, &coords_r_size); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + // Set empty subarray to query int start_1 = 9, end_1 = 10; - rc = tiledb_query_add_range(ctx_, query, 0, &start_1, &end_1, NULL); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start_1, &end_1, NULL); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Submit query @@ -635,6 +667,7 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -702,12 +735,19 @@ TEST_CASE_METHOD( rc = tiledb_query_set_data_buffer(ctx_, query, "d", coords_r, &coords_r_size); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray = nullptr; + rc = tiledb_subarray_alloc(ctx_, array, &subarray); + CHECK(rc == TILEDB_OK); + // Set multi-range subarray to query int start_1 = 9, end_1 = 10; int start_2 = 1, end_2 = 2; - rc = tiledb_query_add_range(ctx_, query, 0, &start_1, &end_1, NULL); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start_1, &end_1, NULL); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_add_range(ctx_, subarray, 0, &start_2, &end_2, NULL); CHECK(rc == TILEDB_OK); - rc = tiledb_query_add_range(ctx_, query, 0, &start_2, &end_2, NULL); + rc = tiledb_query_set_subarray_t(ctx_, query, subarray); CHECK(rc == TILEDB_OK); // Submit query @@ -730,4 +770,5 @@ TEST_CASE_METHOD( CHECK(rc == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } diff --git a/test/src/unit-empty-var-length.cc b/test/src/unit-empty-var-length.cc index 0feba4cd43dc..845b7436eef5 100644 --- a/test/src/unit-empty-var-length.cc +++ b/test/src/unit-empty-var-length.cc @@ -1,5 +1,5 @@ /** - * @file unit-capi-empty-var-length.cc + * @file unit-empty-var-length.cc * * @section LICENSE * @@ -282,6 +282,7 @@ void StringEmptyFx::read_array(const std::string& array_name) { // Create query tiledb_query_t* query; + tiledb_subarray_t* sub; rc = tiledb_query_alloc(ctx, array, TILEDB_READ, &query); REQUIRE(rc == TILEDB_OK); rc = tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); @@ -289,8 +290,13 @@ void StringEmptyFx::read_array(const std::string& array_name) { // Set subarray uint64_t subarray[] = {1, 5}; - rc = tiledb_query_set_subarray(ctx, query, subarray); - CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_alloc(ctx, array, &sub); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx, sub, subarray); + REQUIRE(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx, query, sub); + REQUIRE(rc == TILEDB_OK); + tiledb_subarray_free(&sub); // Set buffer uint64_t buffer_d1_size = 1024; @@ -498,8 +504,10 @@ void StringEmptyFx2::read_array(const std::string& array_name) { query.set_data_buffer("", r_data); query.set_offsets_buffer("", r_offsets); - query.add_range(0, (uint64_t)0, (uint64_t)3); - query.add_range(1, (uint64_t)0, (uint64_t)3); + Subarray subarray(ctx, array); + subarray.add_range(0, (uint64_t)0, (uint64_t)3); + subarray.add_range(1, (uint64_t)0, (uint64_t)3); + query.set_subarray(subarray); query.submit(); @@ -576,8 +584,10 @@ TEST_CASE_METHOD( query.set_data_buffer("", r_data); query.set_offsets_buffer("", r_offsets); - query.add_range(0, (uint64_t)0, (uint64_t)3); - query.add_range(1, (uint64_t)0, (uint64_t)3); + Subarray subarray(ctx, array); + subarray.add_range(0, (uint64_t)0, (uint64_t)3); + subarray.add_range(1, (uint64_t)0, (uint64_t)3); + query.set_subarray(subarray); query.submit(); @@ -601,8 +611,10 @@ TEST_CASE_METHOD( query.set_data_buffer("", r_data); query.set_offsets_buffer("", r_offsets); - query.add_range(0, (uint64_t)0, (uint64_t)1); - query.add_range(1, (uint64_t)1, (uint64_t)2); + Subarray subarray(ctx, array); + subarray.add_range(0, (uint64_t)0, (uint64_t)1); + subarray.add_range(1, (uint64_t)1, (uint64_t)2); + query.set_subarray(subarray); query.submit(); diff --git a/test/src/unit-query-plan.cc b/test/src/unit-query-plan.cc index 8df96f60fda8..fbcb90a5761d 100644 --- a/test/src/unit-query-plan.cc +++ b/test/src/unit-query-plan.cc @@ -72,7 +72,11 @@ TEST_CASE_METHOD( CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_ROW_MAJOR) == TILEDB_OK); int64_t dom[] = {1, 2, 1, 2}; - CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_OK); + tiledb_subarray_t* sub; + CHECK(tiledb_subarray_alloc(ctx_c_, array, &sub) == TILEDB_OK); + CHECK(tiledb_subarray_set_subarray(ctx_c_, sub, &dom) == TILEDB_OK); + CHECK(tiledb_query_set_subarray_t(ctx_c_, query, sub) == TILEDB_OK); + tiledb_subarray_free(&sub); std::vector d(4); uint64_t size = 1; @@ -86,7 +90,10 @@ TEST_CASE_METHOD( // API lifecycle checks // It's not possible to set subarrays, layout, query condition or new buffers // once the query plan got generated. - CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_ERR); + CHECK(tiledb_subarray_alloc(ctx_c_, array, &sub) == TILEDB_OK); + CHECK(tiledb_subarray_set_subarray(ctx_c_, sub, &dom) == TILEDB_OK); + CHECK(tiledb_query_set_subarray_t(ctx_c_, query, sub) == TILEDB_ERR); + tiledb_subarray_free(&sub); CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_COL_MAJOR) == TILEDB_ERR); tiledb_query_condition_t* qc; CHECK(tiledb_query_condition_alloc(ctx_c_, &qc) == TILEDB_OK); @@ -127,7 +134,11 @@ TEST_CASE_METHOD( CHECK(tiledb_query_set_layout(ctx_c_, query, TILEDB_ROW_MAJOR) == TILEDB_OK); int64_t dom[] = {1, 2, 1, 2}; - CHECK(tiledb_query_set_subarray(ctx_c_, query, &dom) == TILEDB_OK); + tiledb_subarray_t* sub; + CHECK(tiledb_subarray_alloc(ctx_c_, array, &sub) == TILEDB_OK); + CHECK(tiledb_subarray_set_subarray(ctx_c_, sub, &dom) == TILEDB_OK); + CHECK(tiledb_query_set_subarray_t(ctx_c_, query, sub) == TILEDB_OK); + tiledb_subarray_free(&sub); std::vector d(4); uint64_t size = 1; diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 8585cf1a9a61..ff0c59655638 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -205,12 +205,17 @@ TEST_CASE_METHOD( REQUIRE(tiledb_array_alloc(ctx, uri_.c_str(), &array) == TILEDB_OK); REQUIRE(tiledb_array_open(ctx, array, TILEDB_READ) == TILEDB_OK); + // Create subarray + int32_t dom[] = {1, 2, 1, 2}; + tiledb_subarray_t* sub; + REQUIRE(tiledb_subarray_alloc(ctx, array, &sub) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx, sub, dom) == TILEDB_OK); + // Create query tiledb_query_t* query = nullptr; REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_ROW_MAJOR) == TILEDB_OK); - int32_t dom[] = {1, 2, 1, 2}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx, query, sub) == TILEDB_OK); uint64_t size = 1; std::vector a1(2); @@ -240,6 +245,7 @@ TEST_CASE_METHOD( REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_query_free(&query); tiledb_array_free(&array); + tiledb_subarray_free(&sub); tiledb_ctx_free(&ctx); } diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index d2ddfcbd6521..beae5bba3bb2 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -365,8 +365,14 @@ int32_t CSparseGlobalOrderFx::read( if (set_subarray) { // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray.data()); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray.data()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); } if (qc_idx != 0) { @@ -448,8 +454,14 @@ int32_t CSparseGlobalOrderFx::read_strings( if (set_subarray) { // Set subarray. - rc = tiledb_query_set_subarray(ctx_, query, subarray.data()); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray.data()); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); } rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 3f3192690679..b93568d34419 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -410,8 +410,14 @@ int32_t CSparseUnorderedWithDupsFx::read( if (set_subarray) { // Set subarray. int subarray[] = {1, 200}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); + CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); } if (qc_idx != 0) { @@ -675,8 +681,14 @@ void CSparseUnorderedWithDupsVarDataFx::read_and_check_data(bool set_subarray) { if (set_subarray) { // Set subarray. int64_t subarray[] = {1, 4, 1, 4}; - rc = tiledb_query_set_subarray(ctx_, query, subarray); + tiledb_subarray_t* sub; + rc = tiledb_subarray_alloc(ctx_, array, &sub); + CHECK(rc == TILEDB_OK); + rc = tiledb_subarray_set_subarray(ctx_, sub, subarray); + CHECK(rc == TILEDB_OK); + rc = tiledb_query_set_subarray_t(ctx_, query, sub); CHECK(rc == TILEDB_OK); + tiledb_subarray_free(&sub); } std::vector data(3); diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 428828babded..2c5561b29b60 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -1180,16 +1180,24 @@ void read_array( rc = tiledb_query_set_layout(ctx, query, layout); CHECK(rc == TILEDB_OK); + // Create subarray + tiledb_subarray_t* subarray; + rc = tiledb_subarray_alloc(ctx, array, &subarray); + CHECK(rc == TILEDB_OK); + auto dim_num = (unsigned)ranges.size(); for (unsigned i = 0; i < dim_num; ++i) { auto dim_range_num = ranges[i].size() / 2; for (size_t j = 0; j < dim_range_num; ++j) { - rc = tiledb_query_add_range( - ctx, query, i, &ranges[i][2 * j], &ranges[i][2 * j + 1], nullptr); + rc = tiledb_subarray_add_range( + ctx, subarray, i, &ranges[i][2 * j], &ranges[i][2 * j + 1], nullptr); CHECK(rc == TILEDB_OK); } } + // Set the subarray + tiledb_query_set_subarray_t(ctx, query, subarray); + // Set buffers for (const auto& b : buffers) { if (b.second.var_ == nullptr) { // Fixed-sized @@ -1230,6 +1238,7 @@ void read_array( // Clean up tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } int32_t num_commits(Context ctx, const std::string& array_name) { diff --git a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc index 72a8c41e6efe..768846beffd6 100644 --- a/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc +++ b/tiledb/api/c_api/query_aggregate/test/unit_capi_query_aggregate.cc @@ -230,7 +230,10 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 9, 1, 2}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx, subarray, &dom) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); // nullptr context tiledb_query_channel_t* default_channel = nullptr; @@ -324,6 +327,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -340,7 +344,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 9, 1, 2}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -365,6 +378,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -381,7 +395,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -411,6 +434,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -427,7 +451,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -459,6 +492,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -475,7 +509,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -506,6 +549,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -522,7 +566,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -553,6 +606,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -569,7 +623,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -601,6 +664,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -617,7 +681,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 9, 1, 2}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -654,6 +727,7 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -668,7 +742,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_alloc(ctx, array, TILEDB_READ, &query) == TILEDB_OK); REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 2, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); std::vector d(4); uint64_t size = 1; @@ -697,6 +780,7 @@ TEST_CASE_METHOD( REQUIRE(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } TEST_CASE_METHOD( @@ -713,7 +797,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 10, 1, 1}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 0, &dom[0], &dom[1], nullptr) == + TILEDB_OK); + REQUIRE( + tiledb_subarray_add_range(ctx, subarray, 1, &dom[2], &dom[3], nullptr) == + TILEDB_OK); + + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_channel_t* default_channel; REQUIRE( @@ -751,4 +844,5 @@ TEST_CASE_METHOD( CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); tiledb_query_free(&query); + tiledb_subarray_free(&subarray); } diff --git a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc index e9105f50fb97..b6819ae756e5 100644 --- a/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc +++ b/tiledb/api/c_api/query_field/test/unit_capi_query_field.cc @@ -292,7 +292,10 @@ TEST_CASE_METHOD(QueryFieldFx, "C API: get_field", "[capi][query_field]") { REQUIRE(tiledb_query_set_layout(ctx, query, TILEDB_UNORDERED) == TILEDB_OK); int64_t dom[] = {1, 9, 1, 2}; - REQUIRE(tiledb_query_set_subarray(ctx, query, &dom) == TILEDB_OK); + tiledb_subarray_t* subarray; + REQUIRE(tiledb_subarray_alloc(ctx, array, &subarray) == TILEDB_OK); + REQUIRE(tiledb_subarray_set_subarray(ctx, subarray, &dom) == TILEDB_OK); + REQUIRE(tiledb_query_set_subarray_t(ctx, query, subarray) == TILEDB_OK); tiledb_query_field_t* field = nullptr; tiledb_datatype_t type; @@ -393,4 +396,5 @@ TEST_CASE_METHOD(QueryFieldFx, "C API: get_field", "[capi][query_field]") { tiledb_query_free(&query); CHECK(tiledb_array_close(ctx, array) == TILEDB_OK); tiledb_array_free(&array); + tiledb_subarray_free(&subarray); } From 6e3dacc5c057406127bf4d8748cd6d9b05c8e63d Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 9 May 2024 06:46:32 -0400 Subject: [PATCH 331/456] Migrate APIs out of StorageManager: group_close_for_reads. (#4953) Migrate APIs out of `StorageManager`: `group_close_for_reads`. --- [sc-46725] --- TYPE: NO_HISTORY DESC: Migrate APIs out of StorageManager: group_close_for_reads. --- tiledb/sm/group/group.cc | 2 +- tiledb/sm/group/group.h | 10 ++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 5 ----- tiledb/sm/storage_manager/storage_manager_canonical.h | 8 -------- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 41c04d23a8d0..dfb44ce1d79c 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -246,7 +246,7 @@ Status Group::close() { // Storage manager does not own the group schema for remote groups. } else { if (query_type_ == QueryType::READ) { - RETURN_NOT_OK(storage_manager_->group_close_for_reads(this)); + RETURN_NOT_OK(close_for_reads()); } else if ( query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 2dfc5aab22fa..3170bfe31d69 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -108,6 +108,16 @@ class Group { Status open( QueryType query_type, uint64_t timestamp_start, uint64_t timestamp_end); + /** + * Closes a group opened for reads. + * + * @return Status + */ + inline Status close_for_reads() { + // Closing a group opened for reads does nothing at present. + return Status::Ok(); + } + /** Closes the group and frees all memory. */ Status close(); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index a448eddad48d..3a9c56a03a18 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -133,11 +133,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::group_close_for_reads(Group*) { - // Closing a group does nothing at present - return Status::Ok(); -} - Status StorageManagerCanonical::group_close_for_writes(Group* group) { // Flush the group metadata RETURN_NOT_OK(store_metadata( diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 7bcd42d6d58b..f96c6f0a1660 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -148,14 +148,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Closes an group opened for reads. - * - * @param group The group to be closed. - * @return Status - */ - Status group_close_for_reads(tiledb::sm::Group* group); - /** * Closes an group opened for writes. * From 474fc1ef554bb954c1fc8b460b0a002bf8d98df0 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 9 May 2024 06:46:56 -0400 Subject: [PATCH 332/456] Migrate APIs out of StorageManager: array_get_encryption. (#4950) Migrate APIs out of `StorageManager`: `array_get_encryption`. --- [sc-44995] --- TYPE: NO_HISTORY DESC: Migrate APIs out of StorageManager: array_get_encryption. --- tiledb/sm/array/array.cc | 30 ++++++++++++++++++ tiledb/sm/array/array.h | 13 ++++++++ tiledb/sm/c_api/tiledb.cc | 8 ++--- tiledb/sm/storage_manager/storage_manager.cc | 31 ------------------- .../storage_manager_canonical.h | 9 ------ 5 files changed, 47 insertions(+), 44 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f3822471d47b..837beac2ef01 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -552,6 +552,36 @@ void Array::delete_fragments_list(const std::vector& fragment_uris) { } } +Status Array::encryption_type( + ContextResources& resources, + const URI& uri, + EncryptionType* encryption_type) { + if (uri.is_invalid()) { + throw ArrayException("[encryption_type] Invalid array URI"); + } + + if (uri.is_tiledb()) { + throw std::invalid_argument( + "Getting the encryption type of remote arrays is not supported."); + } + + // Load URIs from the array directory + optional array_dir; + array_dir.emplace( + resources, + uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); + + // Read tile header + auto&& header = GenericTileIO::read_generic_tile_header( + resources, array_dir->latest_array_schema_uri(), 0); + *encryption_type = static_cast(header.encryption_type); + + return Status::Ok(); +} + shared_ptr Array::get_enumeration( const std::string& enumeration_name) { return get_enumerations({enumeration_name})[0]; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 1af31137ee76..54ce47c60f08 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -468,6 +468,19 @@ class Array { opened_array_->fragment_metadata() = fragment_metadata; } + /** + * Retrieves the encryption type. + * + * @param resources The context resources. + * @param uri The URI of the array. + * @param encryption_type Set to the encryption type. + * @return Status + */ + static Status encryption_type( + ContextResources& resources, + const URI& uri, + EncryptionType* encryption_type); + /** * Get the enumeration for the given name. * diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 12e0ffd0ed88..9110f43136ca 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2934,14 +2934,14 @@ int32_t tiledb_array_encryption_type( const char* array_uri, tiledb_encryption_type_t* encryption_type) { // Sanity checks - if (array_uri == nullptr || encryption_type == nullptr) + if (array_uri == nullptr || encryption_type == nullptr) { return TILEDB_ERR; + } - auto uri = tiledb::sm::URI(array_uri); // Get encryption type tiledb::sm::EncryptionType enc; - throw_if_not_ok(ctx->storage_manager()->array_get_encryption(uri, &enc)); - + throw_if_not_ok(sm::Array::encryption_type( + ctx->resources(), tiledb::sm::URI(array_uri), &enc)); *encryption_type = static_cast(enc); return TILEDB_OK; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 3a9c56a03a18..5ebcfc97450a 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -492,37 +492,6 @@ Status StorageManagerCanonical::array_upgrade_version( return Status::Ok(); } -Status StorageManagerCanonical::array_get_encryption( - const URI& uri, EncryptionType* encryption_type) { - if (uri.is_invalid()) { - return logger_->status(Status_StorageManagerError( - "Cannot get array encryption; Invalid array URI")); - } - - if (uri.is_tiledb()) { - throw std::invalid_argument( - "Getting the encryption type of remote arrays is not supported."); - } - - // Load URIs from the array directory - optional array_dir; - array_dir.emplace( - resources_, - uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY); - - const URI& schema_uri = array_dir->latest_array_schema_uri(); - - // Read tile header - auto&& header = - GenericTileIO::read_generic_tile_header(resources_, schema_uri, 0); - *encryption_type = static_cast(header.encryption_type); - - return Status::Ok(); -} - Status StorageManagerCanonical::async_push_query(Query* query) { cancelable_tasks_.execute( compute_tp(), diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index f96c6f0a1660..d30fb290fe05 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -234,15 +234,6 @@ class StorageManagerCanonical { */ Status array_upgrade_version(const URI& uri, const Config& config); - /** - * Retrieves the encryption type from an array. - * - * @param uri The URI of the array. - * @param encryption_type Set to the encryption type of the array. - * @return Status - */ - Status array_get_encryption(const URI& uri, EncryptionType* encryption_type); - /** * Pushes an async query to the queue. * From 7387605811b82e994a896a051741e6afd5946b0e Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 10 May 2024 15:58:18 -0400 Subject: [PATCH 333/456] Migrate APIs out of StorageManager: delete_group. (#4955) Migrate APIs out of `StorageManager`: `delete_group`. --- [sc-46728] --- TYPE: NO_HISTORY DESC: Migrate APIs out of StorageManager: delete_group --- tiledb/sm/group/group.cc | 22 +++++++++++++- tiledb/sm/group/group.h | 8 +++-- tiledb/sm/storage_manager/storage_manager.cc | 30 ------------------- .../storage_manager_canonical.h | 7 ----- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index dfb44ce1d79c..722e7500fb0f 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -332,7 +332,27 @@ void Group::delete_group(const URI& uri, bool recursive) { } } } - storage_manager_->delete_group(uri.c_str()); + + auto& vfs = resources_.vfs(); + auto& compute_tp = resources_.compute_tp(); + auto group_dir = GroupDirectory( + &vfs, &compute_tp, uri, 0, std::numeric_limits::max()); + + // Delete the group detail, group metadata and group files + vfs.remove_files(&compute_tp, group_dir.group_detail_uris()); + vfs.remove_files(&compute_tp, group_dir.group_meta_uris()); + vfs.remove_files(&compute_tp, group_dir.group_meta_uris_to_vacuum()); + vfs.remove_files(&compute_tp, group_dir.group_meta_vac_uris_to_vacuum()); + vfs.remove_files(&compute_tp, group_dir.group_file_uris()); + + // Delete all tiledb child directories + // Note: using vfs().ls() here could delete user data + std::vector dirs; + auto parent_dir = group_dir.uri().c_str(); + for (auto group_dir_name : constants::group_dir_names) { + dirs.emplace_back(URI(parent_dir + group_dir_name)); + } + vfs.remove_dirs(&compute_tp, dirs); } // Clear metadata and other pending changes to avoid patching a deleted group. metadata_.clear(); diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 3170bfe31d69..de59cde06b5b 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -127,11 +127,13 @@ class Group { void clear(); /** - * Deletes data from and closes a group opened in MODIFY_EXCLUSIVE mode. * - * Note: if recursive == false, data added to the group will be left as-is. + * Handles local and remote deletion of data from a group with the given URI. * - * @param uri The address of the group to be deleted. + * @pre The group must be opened in MODIFY_EXCLUSIVE mode. + * @note If recursive == false, data added to the group will be left as-is. + * + * @param uri The URI of the group to be deleted. * @param recursive True if all data inside the group is to be deleted. */ void delete_group(const URI& uri, bool recursive = false); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 5ebcfc97450a..9d8f3020c945 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -221,36 +221,6 @@ void StorageManagerCanonical::delete_fragments( })); } -void StorageManagerCanonical::delete_group(const char* group_name) { - if (group_name == nullptr) { - throw Status_StorageManagerError( - "[delete_group] Group name cannot be null"); - } - - auto group_dir = GroupDirectory( - vfs(), - compute_tp(), - URI(group_name), - 0, - std::numeric_limits::max()); - - // Delete the group detail, group metadata and group files - vfs()->remove_files(compute_tp(), group_dir.group_detail_uris()); - vfs()->remove_files(compute_tp(), group_dir.group_meta_uris()); - vfs()->remove_files(compute_tp(), group_dir.group_meta_uris_to_vacuum()); - vfs()->remove_files(compute_tp(), group_dir.group_meta_vac_uris_to_vacuum()); - vfs()->remove_files(compute_tp(), group_dir.group_file_uris()); - - // Delete all tiledb child directories - // Note: using vfs()->ls() here could delete user data - std::vector dirs; - auto parent_dir = group_dir.uri().c_str(); - for (auto group_dir_name : constants::group_dir_names) { - dirs.emplace_back(URI(parent_dir + group_dir_name)); - } - vfs()->remove_dirs(compute_tp(), dirs); -} - Status StorageManagerCanonical::array_create( const URI& array_uri, const shared_ptr& array_schema, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index d30fb290fe05..b1ee5bf07c31 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -188,13 +188,6 @@ class StorageManagerCanonical { void delete_fragments( const char* array_name, uint64_t timestamp_start, uint64_t timestamp_end); - /** - * Cleans up the group data. - * - * @param group_name The name of the group whose data is to be deleted. - */ - void delete_group(const char* group_name); - /** * Creates a TileDB array storing its schema. * From 748b80b12fd1463711d610d27afe3dca37d996de Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 10 May 2024 20:54:28 -0400 Subject: [PATCH 334/456] Eliminate storage_manager argument from post_array_from_rest. (#4964) As a larger part of eliminating class `StorageManager`, all usages of member variable `storage_manager_` must also be eliminated from class `Array`. To that effect, this PR removes `storage_manager` as an argument in `post_array_to_rest`, subsequently also removing it from `array_from_capnp` and `array_deserialize`. --- [sc-47009] --- TYPE: NO_HISTORY DESC: Eliminate the storage_manager argument from post_array_from_rest. --- test/src/unit-capi-array.cc | 5 +- test/src/unit-enumerations.cc | 6 +- tiledb/sm/array/array.cc | 12 +-- tiledb/sm/c_api/tiledb.cc | 16 ++-- tiledb/sm/rest/rest_client.cc | 144 ++++++++++++++----------------- tiledb/sm/rest/rest_client.h | 14 ++- tiledb/sm/serialization/array.cc | 78 ++++++++--------- tiledb/sm/serialization/array.h | 19 ++-- tiledb/sm/serialization/query.cc | 22 ++--- 9 files changed, 143 insertions(+), 173 deletions(-) diff --git a/test/src/unit-capi-array.cc b/test/src/unit-capi-array.cc index 0fe991525716..fc643e84bc22 100644 --- a/test/src/unit-capi-array.cc +++ b/test/src/unit-capi-array.cc @@ -2605,13 +2605,12 @@ TEST_CASE_METHOD( 1, &buff); REQUIRE(rc == TILEDB_OK); - auto st = tiledb::sm::serialization::array_deserialize( + tiledb::sm::serialization::array_deserialize( array->array_.get(), tiledb::sm::SerializationType::CAPNP, buff->buffer(), - ctx_->storage_manager(), + ctx_->context().resources(), memory_tracker_); - REQUIRE(st.ok()); // 6. Server: Close array and clean up rc = tiledb_array_close(ctx_, deserialized_array_server); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index c7f5747d0e24..4ed163239caa 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -2950,8 +2950,8 @@ void EnumerationFx::ser_des_array( SerializationType stype) { Buffer buf; throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side)); - throw_if_not_ok(serialization::array_deserialize( - out, stype, buf, ctx.storage_manager(), memory_tracker_)); + serialization::array_deserialize( + out, stype, buf, ctx.resources(), memory_tracker_); } #else // No TILEDB_SERIALIZATION diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 837beac2ef01..4c9e4494ae1a 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -173,11 +173,7 @@ Status Array::open_without_fragments( } set_array_schema_latest(array_schema_latest.value()); } else { - auto st = rest_client->post_array_from_rest( - array_uri_, storage_manager_, this); - if (!st.ok()) { - throw StatusException(st); - } + rest_client->post_array_from_rest(array_uri_, resources_, this); } } else { { @@ -338,11 +334,7 @@ Status Array::open( throw_if_not_ok(st); set_array_schema_latest(array_schema_latest.value()); } else { - auto st = rest_client->post_array_from_rest( - array_uri_, storage_manager_, this); - if (!st.ok()) { - throw StatusException(st); - } + rest_client->post_array_from_rest(array_uri_, resources_, this); } } else if (query_type == QueryType::READ) { { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9110f43136ca..44774639d5a8 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3431,14 +3431,14 @@ int32_t tiledb_deserialize_array( auto memory_tracker = ctx->context().resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); - if (SAVE_ERROR_CATCH( - ctx, - tiledb::sm::serialization::array_deserialize( - (*array)->array_.get(), - (tiledb::sm::SerializationType)serialize_type, - buffer->buffer(), - ctx->storage_manager(), - memory_tracker))) { + try { + tiledb::sm::serialization::array_deserialize( + (*array)->array_.get(), + (tiledb::sm::SerializationType)serialize_type, + buffer->buffer(), + ctx->context().resources(), + memory_tracker); + } catch (StatusException& e) { delete *array; *array = nullptr; return TILEDB_ERR; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 5c1a758104a3..1429b36b3da2 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2023 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -74,8 +74,22 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { + +class RestClientException : public StatusException { + public: + explicit RestClientException(const std::string& message) + : StatusException("RestClient", message) { + } +}; + +class RestClientDisabledException : public RestClientException { + public: + explicit RestClientDisabledException() + : RestClientException( + "Cannot use rest client; serialization not enabled.") { + } +}; #ifdef TILEDB_SERIALIZATION @@ -347,26 +361,25 @@ Status RestClient::post_array_schema_to_rest( return sc; } -Status RestClient::post_array_from_rest( - const URI& uri, StorageManager* storage_manager, Array* array) { +void RestClient::post_array_from_rest( + const URI& uri, ContextResources& resources, Array* array) { if (array == nullptr) { - return LOG_STATUS(Status_SerializationError( - "Error getting remote array; array is null.")); + throw RestClientException("Error getting remote array; array is null."); } Buffer buff; - RETURN_NOT_OK( + throw_if_not_ok( serialization::array_open_serialize(*array, serialization_type_, &buff)); // Wrap in a list BufferList serialized; - RETURN_NOT_OK(serialized.add_buffer(std::move(buff))); + throw_if_not_ok(serialized.add_buffer(std::move(buff))); // Init curl and form the URL Curl curlc(logger_); std::string array_ns, array_uri; - RETURN_NOT_OK(uri.get_rest_components(&array_ns, &array_uri)); + throw_if_not_ok(uri.get_rest_components(&array_ns, &array_uri)); const std::string cache_key = array_ns + ":" + array_uri; - RETURN_NOT_OK( + throw_if_not_ok( curlc.init(config_, extra_headers_, &redirect_meta_, &redirect_mtx_)); std::string url = redirect_uri(cache_key) + "/v2/arrays/" + array_ns + "/" + curlc.url_escape(array_uri) + "/?"; @@ -377,25 +390,22 @@ Status RestClient::post_array_from_rest( // Get the data Buffer returned_data; - RETURN_NOT_OK(curlc.post_data( + throw_if_not_ok(curlc.post_data( stats_, url, serialization_type_, &serialized, &returned_data, cache_key)); - if (returned_data.data() == nullptr || returned_data.size() == 0) - return LOG_STATUS(Status_RestError( - "Error getting array from REST; server returned no data.")); + if (returned_data.data() == nullptr || returned_data.size() == 0) { + throw RestClientException( + "Error getting array from REST; server returned no data."); + } // Ensure data has a null delimiter for cap'n proto if using JSON - RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); - return serialization::array_deserialize( - array, - serialization_type_, - returned_data, - storage_manager, - memory_tracker_); + throw_if_not_ok(ensure_json_null_delimited_string(&returned_data)); + serialization::array_deserialize( + array, serialization_type_, returned_data, resources, memory_tracker_); } void RestClient::delete_array_from_rest(const URI& uri) { @@ -1654,13 +1664,11 @@ Status RestClient::init( ThreadPool*, const std::shared_ptr&, ContextResources&) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::set_header(const std::string&, const std::string&) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } tuple>> @@ -1672,40 +1680,33 @@ RestClient::get_array_schema_from_rest(const URI&) { } Status RestClient::post_array_schema_to_rest(const URI&, const ArraySchema&) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } -Status RestClient::post_array_from_rest(const URI&, StorageManager*, Array*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); +void RestClient::post_array_from_rest(const URI&, ContextResources&, Array*) { + throw RestClientDisabledException(); } void RestClient::delete_array_from_rest(const URI&) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } void RestClient::post_delete_fragments_to_rest( const URI&, Array*, uint64_t, uint64_t) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } void RestClient::post_delete_fragments_list_to_rest( const URI&, Array*, const std::vector&) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::deregister_array_from_rest(const URI&) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::get_array_non_empty_domain(Array*, uint64_t, uint64_t) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::get_array_max_buffer_sizes( @@ -1713,20 +1714,17 @@ Status RestClient::get_array_max_buffer_sizes( const ArraySchema&, const void*, std::unordered_map>*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::get_array_metadata_from_rest( const URI&, uint64_t, uint64_t, Array*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_array_metadata_to_rest( const URI&, uint64_t, uint64_t, Array*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } std::vector> @@ -1737,37 +1735,32 @@ RestClient::post_enumerations_from_rest( Array*, const std::vector&, shared_ptr) { - throw Status_RestError("Cannot use rest client; serialization not enabled."); + throw RestClientDisabledException(); } void RestClient::post_query_plan_from_rest(const URI&, Query&, QueryPlan&) { - throw Status_RestError("Cannot use rest client; serialization not enabled."); + throw RestClientDisabledException(); } Status RestClient::submit_query_to_rest(const URI&, Query*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::finalize_query_to_rest(const URI&, Query*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::submit_and_finalize_query_to_rest(const URI&, Query*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::get_query_est_result_sizes(const URI&, Query*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_array_schema_evolution_to_rest( const URI&, ArraySchemaEvolution*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } tuple> RestClient::check_array_exists_from_rest( @@ -1787,58 +1780,47 @@ tuple> RestClient::check_group_exists_from_rest( } Status RestClient::post_group_metadata_from_rest(const URI&, Group*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_fragment_info_from_rest(const URI&, FragmentInfo*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::put_group_metadata_to_rest(const URI&, Group*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_group_create_to_rest(const URI&, Group*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_group_from_rest(const URI&, Group*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::patch_group_to_rest(const URI&, Group*) { - return LOG_STATUS( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } void RestClient::delete_group_from_rest(const URI&, bool) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_consolidation_to_rest(const URI&, const Config&) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } Status RestClient::post_vacuum_to_rest(const URI&, const Config&) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } std::vector> RestClient::post_consolidation_plan_from_rest( const URI&, const Config&, uint64_t) { - throw StatusException( - Status_RestError("Cannot use rest client; serialization not enabled.")); + throw RestClientDisabledException(); } #endif // TILEDB_SERIALIZATION -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 3c03328127fc..72fb1ad20df6 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2023 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -44,8 +44,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class ArraySchema; class ArraySchemaEvolution; @@ -130,11 +129,11 @@ class RestClient { * Post the array config and get an array from rest server * * @param uri of array being loaded - * @param storage_manager storage manager of array being loaded + * @param resources the context resources * @param array array to load into */ - Status post_array_from_rest( - const URI& uri, StorageManager* storage_manager, Array* array); + void post_array_from_rest( + const URI& uri, ContextResources& resources, Array* array); /** * Post a data array schema to rest server @@ -569,7 +568,6 @@ class RestClient { Status ensure_json_null_delimited_string(Buffer* buffer); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_REST_CLIENT_H diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 72c6a1c142e2..602132df68be 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -64,6 +64,14 @@ class ArraySerializationException : public StatusException { } }; +class ArraySerializationDisabledException : public ArraySerializationException { + public: + explicit ArraySerializationDisabledException() + : ArraySerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION Status metadata_to_capnp( @@ -223,9 +231,9 @@ Status array_to_capnp( return Status::Ok(); } -Status array_from_capnp( +void array_from_capnp( const capnp::Array::Reader& array_reader, - StorageManager* storage_manager, + ContextResources& resources, Array* array, const bool client_side, shared_ptr memory_tracker) { @@ -240,7 +248,7 @@ Status array_from_capnp( if (array_reader.hasQueryType()) { auto query_type_str = array_reader.getQueryType(); QueryType query_type = QueryType::READ; - RETURN_NOT_OK(query_type_enum(query_type_str, &query_type)); + throw_if_not_ok(query_type_enum(query_type_str, &query_type)); array->set_query_type(query_type); array->set_timestamps( @@ -286,13 +294,10 @@ Status array_from_capnp( if (array_reader.hasArrayDirectory()) { const auto& array_directory_reader = array_reader.getArrayDirectory(); auto array_dir = array_directory_from_capnp( - array_directory_reader, - storage_manager->resources(), - array->array_uri()); + array_directory_reader, resources, array->array_uri()); array->set_array_directory(std::move(*array_dir)); } else { - array->set_array_directory( - ArrayDirectory(storage_manager->resources(), array->array_uri())); + array->set_array_directory(ArrayDirectory(resources, array->array_uri())); } if (array_reader.hasFragmentMetadataAll()) { @@ -302,8 +307,8 @@ Status array_from_capnp( fragment_metadata.reserve(fragment_metadata_all_reader.size()); for (auto frag_meta_reader : fragment_metadata_all_reader) { auto meta = make_shared( - HERE(), &storage_manager->resources(), array->memory_tracker()); - RETURN_NOT_OK(fragment_metadata_from_capnp( + HERE(), &resources, array->memory_tracker()); + throw_if_not_ok(fragment_metadata_from_capnp( array->array_schema_latest_ptr(), frag_meta_reader, meta)); if (client_side) { meta->set_rtree_loaded(); @@ -316,19 +321,17 @@ Status array_from_capnp( if (array_reader.hasNonEmptyDomain()) { const auto& nonempty_domain_reader = array_reader.getNonEmptyDomain(); // Deserialize - RETURN_NOT_OK( + throw_if_not_ok( utils::deserialize_non_empty_domain(nonempty_domain_reader, array)); array->set_non_empty_domain_computed(true); } if (array_reader.hasArrayMetadata()) { const auto& array_metadata_reader = array_reader.getArrayMetadata(); - RETURN_NOT_OK( + throw_if_not_ok( metadata_from_capnp(array_metadata_reader, array->unsafe_metadata())); array->set_metadata_loaded(true); } - - return Status::Ok(); } Status array_open_to_capnp( @@ -533,11 +536,11 @@ Status array_serialize( return Status::Ok(); } -Status array_deserialize( +void array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager, + ContextResources& resources, shared_ptr memory_tracker) { try { switch (serialize_type) { @@ -550,13 +553,12 @@ Status array_deserialize( kj::StringPtr(static_cast(serialized_buffer.data())), array_builder); capnp::Array::Reader array_reader = array_builder.asReader(); - RETURN_NOT_OK(array_from_capnp( - array_reader, storage_manager, array, true, memory_tracker)); + array_from_capnp(array_reader, resources, array, true, memory_tracker); break; } case SerializationType::CAPNP: { // Set traversal limit from config - uint64_t limit = storage_manager->config() + uint64_t limit = resources.config() .get("rest.capnp_traversal_limit") .value(); ::capnp::ReaderOptions readerOptions; @@ -571,27 +573,23 @@ Status array_deserialize( serialized_buffer.size() / sizeof(::capnp::word)), readerOptions); capnp::Array::Reader array_reader = reader.getRoot(); - RETURN_NOT_OK(array_from_capnp( - array_reader, storage_manager, array, true, memory_tracker)); + array_from_capnp(array_reader, resources, array, true, memory_tracker); break; } default: { - return LOG_STATUS(Status_SerializationError( - "Error deserializing array; Unknown serialization type " - "passed")); + throw ArraySerializationException( + "Error deserializing array; Unknown serialization type passed"); } } } catch (kj::Exception& e) { - return LOG_STATUS(Status_SerializationError( + throw ArraySerializationException( "Error deserializing array; kj::Exception: " + - std::string(e.getDescription().cStr()))); + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - return LOG_STATUS(Status_SerializationError( - "Error deserializing array; exception " + std::string(e.what()))); + throw ArraySerializationException( + "Error deserializing array; exception " + std::string(e.what())); } - - return Status::Ok(); } Status array_open_serialize( @@ -704,34 +702,30 @@ Status array_serialize(Array*, SerializationType, Buffer*, const bool) { "Cannot serialize; serialization not enabled.")); } -Status array_deserialize( +void array_deserialize( Array*, SerializationType, const Buffer&, - StorageManager*, + ContextResources&, shared_ptr) { - return LOG_STATUS(Status_SerializationError( - "Cannot deserialize; serialization not enabled.")); + throw ArraySerializationDisabledException(); } Status array_open_serialize(const Array&, SerializationType, Buffer*) { - return LOG_STATUS(Status_SerializationError( - "Cannot serialize; serialization not enabled.")); + throw ArraySerializationDisabledException(); } Status array_open_deserialize(Array*, SerializationType, const Buffer&) { - return LOG_STATUS(Status_SerializationError( - "Cannot deserialize; serialization not enabled.")); + throw ArraySerializationDisabledException(); + ; } Status metadata_serialize(Metadata*, SerializationType, Buffer*) { - return LOG_STATUS(Status_SerializationError( - "Cannot serialize; serialization not enabled.")); + throw ArraySerializationDisabledException(); } Status metadata_deserialize(Metadata*, SerializationType, const Buffer&) { - return LOG_STATUS(Status_SerializationError( - "Cannot deserialize; serialization not enabled.")); + throw ArraySerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/array.h b/tiledb/sm/serialization/array.h index e048510c5349..4f7f4fc1111d 100644 --- a/tiledb/sm/serialization/array.h +++ b/tiledb/sm/serialization/array.h @@ -69,7 +69,7 @@ Status array_to_capnp( /** * Deserialize an Array from Cap'n proto * @param array_reader cap'n proto class - * @param storage_manager the storage manager associated with the array + * @param resources the context resources associated with the array * @param array Array to deserialize into * @param client_side Allows to specify different behavior depending on who is * @param memory_tracker Memory tracker to use for memory allocations. @@ -78,9 +78,9 @@ Status array_to_capnp( * @param memory_tracker Memory tracker to use on the deserialized object. * @return Status */ -Status array_from_capnp( +void array_from_capnp( const capnp::Array::Reader& array_reader, - StorageManager* storage_manager, + ContextResources& resources, Array* array, const bool client_side, shared_ptr memory_tracker); @@ -134,11 +134,20 @@ Status array_serialize( Buffer* serialized_buffer, const bool client_side); -Status array_deserialize( +/** + * Deserialize an array via Cap'n Proto. + * + * @param array array object to set the array details into + * @param serialize_type format the data is serialized in: Cap'n Proto of JSON + * @param serialized_buffer buffer to read serialized bytes from + * @param resources the context resources associated with the array + * @param memory_tracker the memory tracker to use on the deserialized object + */ +void array_deserialize( Array* array, SerializationType serialize_type, const Buffer& serialized_buffer, - StorageManager* storage_manager, + ContextResources& resources, shared_ptr memory_tracker); /** diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 2f76659e335f..c95d5d3ef8a9 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -80,9 +80,7 @@ using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { -namespace serialization { +namespace tiledb::sm::serialization { #ifdef TILEDB_SERIALIZATION @@ -2297,12 +2295,12 @@ Status array_from_query_deserialize( query_builder); capnp::Query::Reader query_reader = query_builder.asReader(); // Deserialize array instance. - RETURN_NOT_OK(array_from_capnp( + array_from_capnp( query_reader.getArray(), - storage_manager, + storage_manager->resources(), &array, false, - memory_tracker)); + memory_tracker); break; } case SerializationType::CAPNP: { @@ -2329,12 +2327,12 @@ Status array_from_query_deserialize( capnp::Query::Reader query_reader = reader.getRoot(); // Deserialize array instance. - RETURN_NOT_OK(array_from_capnp( + array_from_capnp( query_reader.getArray(), - storage_manager, + storage_manager->resources(), &array, false, - memory_tracker)); + memory_tracker); break; } default: @@ -3219,6 +3217,4 @@ Status query_est_result_size_deserialize( #endif // TILEDB_SERIALIZATION -} // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::serialization From 9d8fcb127b36bb0a5b5dc0e835dfe02ca14a14b5 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 13 May 2024 15:41:04 +0300 Subject: [PATCH 335/456] Fix AVX2 support detection. (#4969) [SC-47213](https://app.shortcut.com/tiledb-inc/story/47213) A typo in #4449 prevented AVX2 from being detected in GCC. This PR fixes the typo and adds logging to allow validating whether AVX2 was detected. Fixes #4967. --- TYPE: BUILD DESC: Fix AVX2 support detection. --- cmake/Modules/CheckAVX2Support.cmake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/Modules/CheckAVX2Support.cmake b/cmake/Modules/CheckAVX2Support.cmake index b91fcf7acf44..2954b2f66da5 100644 --- a/cmake/Modules/CheckAVX2Support.cmake +++ b/cmake/Modules/CheckAVX2Support.cmake @@ -53,7 +53,7 @@ function (CheckAVX2Support) endif() cmake_push_check_state() - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${COMPILER_AVX2_FLAG}") check_cxx_source_runs(" #include int main() { @@ -64,4 +64,9 @@ function (CheckAVX2Support) COMPILER_SUPPORTS_AVX2 ) cmake_pop_check_state() + if (COMPILER_SUPPORTS_AVX2) + message(STATUS "AVX2 support detected.") + else() + message(STATUS "AVX2 support not detected.") + endif() endfunction() From 8de7d1ca4460f16127ddf64ca65092cff2d4efce Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 13 May 2024 08:43:16 -0400 Subject: [PATCH 336/456] Add v2_23_0 arrays to backward compatibility matrix. (#4965) TYPE: NO_HISTORY --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index b5159501e1c3..484558844bfe 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 62098dc9dc14a017fdfe0dec1610b05e64496e8a Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Mon, 13 May 2024 13:15:25 -0600 Subject: [PATCH 337/456] Simplify handling of unsupported C API `stride` arguments (#4963) The C API contains arguments for `stride` that are unsupported. This PR simplifies the internal handling of them. The main tactic for simplification is deal with these arguments in the C API implementation function and remove the arguments from the functions that support them. Add `ensure_subarray_is_valid` replaces the old `sanity_check` function for subarray. Added validation to all relevant dimension label functions, which heretofore weren't validating subarray arguments in any way. --- TYPE: NO_HISTORY DESC: Simplify handling of unsupported C API `stride` arguments --- .../api/c_api_support/argument_validation.h | 15 +++ tiledb/sm/c_api/api_argument_validator.h | 18 +-- tiledb/sm/c_api/tiledb.cc | 115 ++++++++---------- tiledb/sm/c_api/tiledb_dimension_label.cc | 18 ++- tiledb/sm/query_plan/test/unit_query_plan.cc | 2 +- tiledb/sm/subarray/subarray.cc | 46 ++----- tiledb/sm/subarray/subarray.h | 49 ++------ 7 files changed, 109 insertions(+), 154 deletions(-) diff --git a/tiledb/api/c_api_support/argument_validation.h b/tiledb/api/c_api_support/argument_validation.h index 94fd6f782470..6dc1a0c578bc 100644 --- a/tiledb/api/c_api_support/argument_validation.h +++ b/tiledb/api/c_api_support/argument_validation.h @@ -73,6 +73,21 @@ inline void ensure_output_pointer_is_valid(void* p) { } } +/** + * Ensure that the output pointer for a stride argument is null. + * + * The C API has arguments for the "stride" of a range, but does not support + * such arguments at the present time. This validation ensures that the argument + * is null. + * + * @param p The value of a `stride` argument to a C API function + */ +inline void ensure_unsupported_stride_is_null(const void* p) { + if (p != nullptr) { + throw CAPIException("Stride is currently unsupported"); + } +} + } // namespace tiledb::api #endif // TILEDB_CAPI_SUPPORT_ARGUMENT_VALIDATION_H diff --git a/tiledb/sm/c_api/api_argument_validator.h b/tiledb/sm/c_api/api_argument_validator.h index 665420aa3724..df62b27c37b8 100644 --- a/tiledb/sm/c_api/api_argument_validator.h +++ b/tiledb/sm/c_api/api_argument_validator.h @@ -69,17 +69,17 @@ inline int32_t sanity_check(tiledb_ctx_t* ctx, const tiledb_array_t* array) { return TILEDB_OK; } -inline int32_t sanity_check( - tiledb_ctx_t* ctx, const tiledb_subarray_t* subarray) { - if (subarray == nullptr || subarray->subarray_ == nullptr || - subarray->subarray_->array() == nullptr) { - auto st = Status_Error("Invalid TileDB subarray object"); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; +namespace tiledb::api { +/** + * Returns if a subarray handle (old style) is valid. Throws otherwise. + */ +inline void ensure_subarray_is_valid(const tiledb_subarray_t* p) { + if (p == nullptr || p->subarray_ == nullptr || + p->subarray_->array() == nullptr) { + throw CAPIException("Invalid TileDB subarray object"); } - return TILEDB_OK; } +} // namespace tiledb::api inline int32_t sanity_check( tiledb_ctx_t* ctx, const tiledb_array_schema_t* array_schema) { diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 44774639d5a8..34c28865e0a0 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1072,12 +1072,11 @@ int32_t tiledb_query_set_subarray_t( tiledb_query_t* query, const tiledb_subarray_t* subarray) { // Sanity check - if (sanity_check(ctx, query) == TILEDB_ERR || - sanity_check(ctx, subarray) == TILEDB_ERR) + if (sanity_check(ctx, query) == TILEDB_ERR) { return TILEDB_ERR; - + } + ensure_subarray_is_valid(subarray); query->query_->set_subarray(*subarray->subarray_); - return TILEDB_OK; } @@ -1762,9 +1761,8 @@ capi_return_t tiledb_subarray_alloc( } int32_t tiledb_subarray_set_config( - tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, tiledb_config_t* config) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + tiledb_ctx_t*, tiledb_subarray_t* subarray, tiledb_config_t* config) { + ensure_subarray_is_valid(subarray); api::ensure_config_is_valid(config); subarray->subarray_->set_config( tiledb::sm::QueryType::READ, config->config()); @@ -1784,196 +1782,185 @@ void tiledb_subarray_free(tiledb_subarray_t** subarray) { } int32_t tiledb_subarray_set_coalesce_ranges( - tiledb_ctx_t* ctx, tiledb_subarray_t* subarray, int coalesce_ranges) { - // Sanity check - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + tiledb_ctx_t*, tiledb_subarray_t* subarray, int coalesce_ranges) { + ensure_subarray_is_valid(subarray); subarray->subarray_->set_coalesce_ranges(coalesce_ranges != 0); return TILEDB_OK; } int32_t tiledb_subarray_set_subarray( - tiledb_ctx_t* ctx, - tiledb_subarray_t* subarray_obj, - const void* subarray_vals) { - if (sanity_check(ctx, subarray_obj) == TILEDB_ERR) - return TILEDB_ERR; - + tiledb_ctx_t*, tiledb_subarray_t* subarray_obj, const void* subarray_vals) { + ensure_subarray_is_valid(subarray_obj); subarray_obj->subarray_->set_subarray(subarray_vals); - return TILEDB_OK; } int32_t tiledb_subarray_add_range( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, const void* end, const void* stride) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; - - subarray->subarray_->add_range(dim_idx, start, end, stride); - + ensure_subarray_is_valid(subarray); + ensure_unsupported_stride_is_null(stride); + subarray->subarray_->add_range(dim_idx, start, end); return TILEDB_OK; } int32_t tiledb_subarray_add_point_ranges( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, uint64_t count) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->add_point_ranges(dim_idx, start, count); return TILEDB_OK; } int32_t tiledb_subarray_add_range_by_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_subarray_t* subarray, const char* dim_name, const void* start, const void* end, const void* stride) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; - subarray->subarray_->add_range_by_name(dim_name, start, end, stride); + ensure_subarray_is_valid(subarray); + ensure_unsupported_stride_is_null(stride); + subarray->subarray_->add_range_by_name(dim_name, start, end); return TILEDB_OK; } int32_t tiledb_subarray_add_range_var( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_subarray_t* subarray, uint32_t dim_idx, const void* start, uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->add_range_var(dim_idx, start, start_size, end, end_size); return TILEDB_OK; } int32_t tiledb_subarray_add_range_var_by_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, tiledb_subarray_t* subarray, const char* dim_name, const void* start, uint64_t start_size, const void* end, uint64_t end_size) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->add_range_var_by_name( dim_name, start, start_size, end, end_size); return TILEDB_OK; } int32_t tiledb_subarray_get_range_num( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t* range_num) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_num(dim_idx, range_num); return TILEDB_OK; } int32_t tiledb_subarray_get_range_num_from_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t* range_num) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_num_from_name(dim_name, range_num); return TILEDB_OK; } int32_t tiledb_subarray_get_range( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, const void** start, const void** end, const void** stride) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; - subarray->subarray_->get_range(dim_idx, range_idx, start, end, stride); + ensure_subarray_is_valid(subarray); + ensure_output_pointer_is_valid(start); + ensure_output_pointer_is_valid(end); + if (stride != nullptr) { + *stride = nullptr; + } + subarray->subarray_->get_range(dim_idx, range_idx, start, end); return TILEDB_OK; } int32_t tiledb_subarray_get_range_var_size( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_var_size( dim_idx, range_idx, start_size, end_size); return TILEDB_OK; } int32_t tiledb_subarray_get_range_from_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, const void** start, const void** end, const void** stride) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; - subarray->subarray_->get_range_from_name( - dim_name, range_idx, start, end, stride); + ensure_subarray_is_valid(subarray); + ensure_output_pointer_is_valid(start); + ensure_output_pointer_is_valid(end); + if (stride != nullptr) { + *stride = nullptr; + } + subarray->subarray_->get_range_from_name(dim_name, range_idx, start, end); return TILEDB_OK; } int32_t tiledb_subarray_get_range_var_size_from_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_var_size_from_name( dim_name, range_idx, start_size, end_size); return TILEDB_OK; } int32_t tiledb_subarray_get_range_var( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, uint32_t dim_idx, uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_var(dim_idx, range_idx, start, end); return TILEDB_OK; } int32_t tiledb_subarray_get_range_var_from_name( - tiledb_ctx_t* ctx, + tiledb_ctx_t*, const tiledb_subarray_t* subarray, const char* dim_name, uint64_t range_idx, void* start, void* end) { - if (sanity_check(ctx, subarray) == TILEDB_ERR) - return TILEDB_ERR; + ensure_subarray_is_valid(subarray); subarray->subarray_->get_range_var_from_name(dim_name, range_idx, start, end); return TILEDB_OK; } diff --git a/tiledb/sm/c_api/tiledb_dimension_label.cc b/tiledb/sm/c_api/tiledb_dimension_label.cc index 866365ea61ef..01f472bf72a2 100644 --- a/tiledb/sm/c_api/tiledb_dimension_label.cc +++ b/tiledb/sm/c_api/tiledb_dimension_label.cc @@ -130,7 +130,9 @@ capi_return_t tiledb_subarray_add_label_range( const void* start, const void* end, const void* stride) { - subarray->subarray_->add_label_range(label_name, start, end, stride); + ensure_subarray_is_valid(subarray); + ensure_unsupported_stride_is_null(stride); + subarray->subarray_->add_label_range(label_name, start, end); return TILEDB_OK; } @@ -141,6 +143,7 @@ capi_return_t tiledb_subarray_add_label_range_var( uint64_t start_size, const void* end, uint64_t end_size) { + ensure_subarray_is_valid(subarray); subarray->subarray_->add_label_range_var( label_name, start, start_size, end, end_size); return TILEDB_OK; @@ -148,6 +151,7 @@ capi_return_t tiledb_subarray_add_label_range_var( capi_return_t tiledb_subarray_get_label_name( tiledb_subarray_t* subarray, uint32_t dim_idx, const char** label_name) { + ensure_subarray_is_valid(subarray); const auto& name = subarray->subarray_->get_label_name(dim_idx); *label_name = name.c_str(); return TILEDB_OK; @@ -160,7 +164,13 @@ capi_return_t tiledb_subarray_get_label_range( const void** start, const void** end, const void** stride) { - subarray->subarray_->get_label_range(dim_name, range_idx, start, end, stride); + ensure_subarray_is_valid(subarray); + ensure_output_pointer_is_valid(start); + ensure_output_pointer_is_valid(end); + if (stride != nullptr) { + *stride = nullptr; + } + subarray->subarray_->get_label_range(dim_name, range_idx, start, end); return TILEDB_OK; } @@ -168,6 +178,7 @@ capi_return_t tiledb_subarray_get_label_range_num( const tiledb_subarray_t* subarray, const char* dim_name, uint64_t* range_num) { + ensure_subarray_is_valid(subarray); subarray->subarray_->get_label_range_num(dim_name, range_num); return TILEDB_OK; } @@ -178,6 +189,7 @@ capi_return_t tiledb_subarray_get_label_range_var( uint64_t range_idx, void* start, void* end) { + ensure_subarray_is_valid(subarray); subarray->subarray_->get_label_range_var(dim_name, range_idx, start, end); return TILEDB_OK; } @@ -188,6 +200,7 @@ capi_return_t tiledb_subarray_get_label_range_var_size( uint64_t range_idx, uint64_t* start_size, uint64_t* end_size) { + ensure_subarray_is_valid(subarray); subarray->subarray_->get_label_range_var_size( dim_name, range_idx, start_size, end_size); return TILEDB_OK; @@ -197,6 +210,7 @@ capi_return_t tiledb_subarray_has_label_ranges( const tiledb_subarray_t* subarray, const uint32_t dim_idx, int32_t* has_label_ranges) { + ensure_subarray_is_valid(subarray); bool has_ranges = subarray->subarray_->has_label_ranges(dim_idx); *has_label_ranges = has_ranges ? 1 : 0; return TILEDB_OK; diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index ba33ddb15df7..a0168e896c4b 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -135,7 +135,7 @@ TEST_CASE_METHOD(QueryPlanFx, "Query plan dump_json", "[query_plan][dump]") { stats::Stats stats("foo"); Subarray subarray(array_shared.get(), &stats, logger_); uint64_t r[2]{0, 1}; - subarray.add_range(0, r, r + 1, nullptr); + subarray.add_range(0, r, r + 1); query.set_subarray(subarray); std::vector data(2); diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 1c4f137c082a..1354a462b4de 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -246,18 +246,11 @@ void Subarray::add_label_range( } void Subarray::add_label_range( - const std::string& label_name, - const void* start, - const void* end, - const void* stride) { + const std::string& label_name, const void* start, const void* end) { // Check input range data is valid data. if (start == nullptr || end == nullptr) { throw SubarrayException("[add_label_range] Invalid range"); } - if (stride != nullptr) { - throw SubarrayException( - "[add_label_range] Setting range stride is currently unsupported"); - } // Get dimension label range and check the label is in fact fixed-sized. const auto& dim_label_ref = array_->array_schema_latest().dimension_label(label_name); @@ -382,8 +375,7 @@ void Subarray::set_subarray_unsafe(const void* subarray) { } } -void Subarray::add_range( - unsigned dim_idx, const void* start, const void* end, const void* stride) { +void Subarray::add_range(unsigned dim_idx, const void* start, const void* end) { if (dim_idx >= this->array_->array_schema_latest().dim_num()) throw SubarrayException("Cannot add range; Invalid dimension index"); @@ -397,11 +389,6 @@ void Subarray::add_range( throw SubarrayException("Cannot add range; Invalid range"); } - if (stride != nullptr) { - throw SubarrayException( - "Cannot add range; Setting range stride is currently unsupported"); - } - if (this->array_->array_schema_latest() .domain() .dimension_ptr(dim_idx) @@ -504,13 +491,10 @@ void Subarray::add_ranges_list( } void Subarray::add_range_by_name( - const std::string& dim_name, - const void* start, - const void* end, - const void* stride) { + const std::string& dim_name, const void* start, const void* end) { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - add_range(dim_idx, start, end, stride); + add_range(dim_idx, start, end); } void Subarray::add_range_var( @@ -585,8 +569,7 @@ void Subarray::get_label_range( const std::string& label_name, uint64_t range_idx, const void** start, - const void** end, - const void** stride) const { + const void** end) const { auto dim_idx = array_->array_schema_latest() .dimension_label(label_name) .dimension_index(); @@ -599,7 +582,6 @@ void Subarray::get_label_range( const auto& range = label_range_subset_[dim_idx].value().ranges_[range_idx]; *start = range.start_fixed(); *end = range.end_fixed(); - *stride = nullptr; } void Subarray::get_label_range_num( @@ -659,8 +641,7 @@ void Subarray::get_range_var( const void* range_start; const void* range_end; - const void* stride; - get_range(dim_idx, range_idx, &range_start, &range_end, &stride); + get_range(dim_idx, range_idx, &range_start, &range_end); std::memcpy(start, range_start, start_size); std::memcpy(end, range_end, end_size); @@ -673,25 +654,14 @@ void Subarray::get_range_num_from_name( get_range_num(dim_idx, range_num); } -void Subarray::get_range( - unsigned dim_idx, - uint64_t range_idx, - const void** start, - const void** end, - const void** stride) const { - *stride = nullptr; - this->get_range(dim_idx, range_idx, start, end); -} - void Subarray::get_range_from_name( const std::string& dim_name, uint64_t range_idx, const void** start, - const void** end, - const void** stride) const { + const void** end) const { unsigned dim_idx = array_->array_schema_latest().domain().get_dimension_index(dim_name); - get_range(dim_idx, range_idx, start, end, stride); + get_range(dim_idx, range_idx, start, end); } void Subarray::get_range_var_size_from_name( diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index f8d8105f1204..c9e5d89c7769 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -429,13 +429,9 @@ class Subarray { * @param label_name The name of the dimension label to add the range to. * @param start The range start. * @param end The range end. - * @param stride The range stride. */ void add_label_range( - const std::string& label_name, - const void* start, - const void* end, - const void* stride); + const std::string& label_name, const void* start, const void* end); /** * Adds a variable-sized range along the dimension with the given index for @@ -459,13 +455,11 @@ class Subarray { uint32_t dim_idx, Range&& range, const bool read_range_oob_error = true); /** - * Adds a range to the subarray on the input dimension by index, - * in the form of (start, end, stride). + * Adds a range to the subarray on the input dimension by index. * The range components must be of the same type as the domain type of the * underlying array. */ - void add_range( - unsigned dim_idx, const void* start, const void* end, const void* stride); + void add_range(unsigned dim_idx, const void* start, const void* end); /** * @brief Set point ranges from an array @@ -521,16 +515,12 @@ class Subarray { void add_range_unsafe(uint32_t dim_idx, const Range& range); /** - * Adds a range to the (read/write) query on the input dimension by name, - * in the form of (start, end, stride). + * Adds a range to the (read/write) query on the input dimension by name. * The range components must be of the same type as the domain type of the * underlying array. */ void add_range_by_name( - const std::string& dim_name, - const void* start, - const void* end, - const void* stride); + const std::string& dim_name, const void* start, const void* end); /** * Adds a variable-sized range to the (read/write) query on the input @@ -567,22 +557,19 @@ class Subarray { const std::string& get_label_name(const uint32_t dim_index) const; /** - * Retrieves a range from a dimension label name in the form (start, end, - * stride). + * Retrieves a range from a dimension label name. * * @param label_name The name of the dimension label to retrieve the range * from. * @param range_idx The id of the range to retrieve. * @param start The range start to retrieve. * @param end The range end to retrieve. - * @param stride The range stride to retrieve. */ void get_label_range( const std::string& label_name, uint64_t range_idx, const void** start, - const void** end, - const void** stride) const; + const void** end) const; /** * Retrieves the number of ranges of the subarray for the given dimension @@ -633,20 +620,18 @@ class Subarray { const std::string& dim_name, uint64_t* range_num) const; /** - * Retrieves a range from a dimension name in the form (start, end, stride). + * Retrieves a range from a dimension name. * * @param dim_name The dimension to retrieve the range from. * @param range_idx The id of the range to retrieve. * @param start The range start to retrieve. * @param end The range end to retrieve. - * @param stride The range stride to retrieve. */ void get_range_from_name( const std::string& dim_name, uint64_t range_idx, const void** start, - const void** end, - const void** stride) const; + const void** end) const; /** * Retrieves a range's sizes for a variable-length dimension name @@ -895,22 +880,6 @@ class Subarray { /** Retrieves the number of ranges on the given dimension index. */ void get_range_num(uint32_t dim_idx, uint64_t* range_num) const; - /** - * Retrieves a range from a dimension index in the form (start, end, stride). - * - * @param dim_idx The dimension to retrieve the range from. - * @param range_idx The id of the range to retrieve. - * @param start The range start to retrieve. - * @param end The range end to retrieve. - * @param stride The range stride to retrieve. - */ - void get_range( - unsigned dim_idx, - uint64_t range_idx, - const void** start, - const void** end, - const void** stride) const; - /** * ``True`` if the specified dimension is set by default. * From d0ddec4efea452593b8c5d74caaa261f48831d68 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Tue, 14 May 2024 01:38:52 -0600 Subject: [PATCH 338/456] Update `c_api` to use CMake environments (#4959) Update object library and unit tests within the `c_api` directory tree to use CMake environments. This replaces direct calls to `add_library`, `add_executable`, and `add_test`. The modules affected: * `array`, `buffer`, `buffer_list`, `config`, `context`, `data_order`, `datatype`, `dimension_label`, `filesystem`, `filter`, `filter_list`, `query_aggregate`, `query_plan`, `query_type`, `vfs` A small number of object library declarations were not converted. These were only those that cannot currently be linked standalone. The documentation of these remaining ones was tightened up. [sc-46986] --- TYPE: NO_HISTORY DESC: Update `c_api` to use CMake environments --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/api/c_api/array/test/CMakeLists.txt | 37 +++++++------------ tiledb/api/c_api/buffer/CMakeLists.txt | 29 ++++----------- tiledb/api/c_api/buffer/test/CMakeLists.txt | 18 +++------ ...in.cc => compile_capi_buffer_stub_main.cc} | 0 tiledb/api/c_api/buffer_list/CMakeLists.txt | 33 +++++------------ .../api/c_api/buffer_list/test/CMakeLists.txt | 19 +++------- ... => compile_capi_buffer_list_stub_main.cc} | 0 tiledb/api/c_api/config/CMakeLists.txt | 32 +++++----------- tiledb/api/c_api/config/test/CMakeLists.txt | 23 +++--------- ...in.cc => compile_capi_config_stub_main.cc} | 0 tiledb/api/c_api/context/CMakeLists.txt | 30 +++++---------- tiledb/api/c_api/context/test/CMakeLists.txt | 23 +++--------- ...n.cc => compile_capi_context_stub_main.cc} | 0 .../api/c_api/data_order/test/CMakeLists.txt | 18 +++------ tiledb/api/c_api/datatype/test/CMakeLists.txt | 18 +++------ .../api/c_api/dimension_label/CMakeLists.txt | 25 +++++-------- .../c_api/dimension_label/test/CMakeLists.txt | 23 +++--------- ...compile_capi_dimension_label_stub_main.cc} | 4 +- .../api/c_api/filesystem/test/CMakeLists.txt | 18 +++------ tiledb/api/c_api/filter/CMakeLists.txt | 22 ++++------- tiledb/api/c_api/filter/test/CMakeLists.txt | 24 +++--------- ...in.cc => compile_capi_filter_stub_main.cc} | 2 +- tiledb/api/c_api/filter_list/CMakeLists.txt | 22 ++++------- .../api/c_api/filter_list/test/CMakeLists.txt | 23 +++--------- ... => compile_capi_filter_list_stub_main.cc} | 0 tiledb/api/c_api/group/CMakeLists.txt | 18 ++------- tiledb/api/c_api/object/CMakeLists.txt | 12 +----- tiledb/api/c_api/query/test/CMakeLists.txt | 18 +++------ .../api/c_api/query_aggregate/CMakeLists.txt | 14 +++---- tiledb/api/c_api/query_field/CMakeLists.txt | 7 +++- tiledb/api/c_api/query_plan/CMakeLists.txt | 13 +++---- tiledb/api/c_api/vfs/CMakeLists.txt | 21 ++++------- tiledb/api/c_api/vfs/test/CMakeLists.txt | 23 +++--------- tiledb/sm/query_plan/CMakeLists.txt | 15 +++++--- 34 files changed, 186 insertions(+), 398 deletions(-) rename tiledb/api/c_api/buffer/test/{compile_capi_buffer_main.cc => compile_capi_buffer_stub_main.cc} (100%) rename tiledb/api/c_api/buffer_list/test/{compile_capi_buffer_list_main.cc => compile_capi_buffer_list_stub_main.cc} (100%) rename tiledb/api/c_api/config/test/{compile_capi_config_main.cc => compile_capi_config_stub_main.cc} (100%) rename tiledb/api/c_api/context/test/{compile_capi_context_main.cc => compile_capi_context_stub_main.cc} (100%) rename tiledb/api/c_api/dimension_label/test/{compile_capi_dimension_label_main.cc => compile_capi_dimension_label_stub_main.cc} (92%) rename tiledb/api/c_api/filter/test/{compile_capi_filter_main.cc => compile_capi_filter_stub_main.cc} (97%) rename tiledb/api/c_api/filter_list/test/{compile_capi_filter_list_main.cc => compile_capi_filter_list_stub_main.cc} (100%) diff --git a/tiledb/api/c_api/array/test/CMakeLists.txt b/tiledb/api/c_api/array/test/CMakeLists.txt index ebd617387ae8..c6f5c0a1694d 100644 --- a/tiledb/api/c_api/array/test/CMakeLists.txt +++ b/tiledb/api/c_api/array/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,28 +25,19 @@ # include(common NO_POLICY_SCOPE) -find_package(Catch_EP REQUIRED) - -# #TODO: link only against a new CAPI array handle, not all core objects -add_executable( - unit_capi_array EXCLUDE_FROM_ALL -) +include(unit_test) -target_link_libraries(unit_capi_array - PUBLIC - Catch2::Catch2WithMain - export - TILEDB_CORE_OBJECTS_ILIB - TILEDB_CORE_OBJECTS -) +find_package(Catch_EP REQUIRED) -# Sources for tests -target_sources(unit_capi_array PUBLIC - unit_capi_array.cc -) +# Maturity +# +# This unit links against the whole library out of necessity. There's not +# yet an `array` object library, much less a C API handle one. +# +commence(unit_test capi_array) + this_target_sources(unit_capi_array.cc) + this_target_link_libraries(export + TILEDB_CORE_OBJECTS_ILIB + TILEDB_CORE_OBJECTS) +conclude(unit_test) -add_test( - NAME "unit_capi_array" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) diff --git a/tiledb/api/c_api/buffer/CMakeLists.txt b/tiledb/api/c_api/buffer/CMakeLists.txt index abb891bb8544..0cde6d70656e 100644 --- a/tiledb/api/c_api/buffer/CMakeLists.txt +++ b/tiledb/api/c_api/buffer/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,31 +25,18 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES buffer_api.cc ) gather_sources(${SOURCES}) -# -# Object library for other units to depend upon -# -add_library(capi_buffer_stub OBJECT ${SOURCES}) -target_link_libraries(capi_buffer_stub PUBLIC export) -target_link_libraries(capi_buffer_stub PUBLIC buffer $) -# -# We need to build this as a stub-based library because the API wrappers make -# reference to `context`, even though the functions in the buffer section of the -# API do not use a context for error handling. -# -target_link_libraries(capi_buffer_stub PUBLIC capi_context_stub $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_buffer EXCLUDE_FROM_ALL) -target_sources(compile_capi_buffer PRIVATE test/compile_capi_buffer_main.cc) -target_link_libraries(compile_capi_buffer PRIVATE capi_buffer_stub) -add_dependencies(all_link_complete compile_capi_buffer) +commence(object_library capi_buffer_stub) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(capi_context_stub) + this_target_object_libraries(buffer) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/buffer/test/CMakeLists.txt b/tiledb/api/c_api/buffer/test/CMakeLists.txt index 033119703045..a9907b29a6fe 100644 --- a/tiledb/api/c_api/buffer/test/CMakeLists.txt +++ b/tiledb/api/c_api/buffer/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,15 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_buffer EXCLUDE_FROM_ALL) -target_sources(unit_capi_buffer PUBLIC unit_capi_buffer.cc) -target_link_libraries(unit_capi_buffer PUBLIC capi_buffer_stub) -target_link_libraries(unit_capi_buffer PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_buffer" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_buffer) + this_target_sources(unit_capi_buffer.cc) + this_target_object_libraries(capi_buffer_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/buffer/test/compile_capi_buffer_main.cc b/tiledb/api/c_api/buffer/test/compile_capi_buffer_stub_main.cc similarity index 100% rename from tiledb/api/c_api/buffer/test/compile_capi_buffer_main.cc rename to tiledb/api/c_api/buffer/test/compile_capi_buffer_stub_main.cc diff --git a/tiledb/api/c_api/buffer_list/CMakeLists.txt b/tiledb/api/c_api/buffer_list/CMakeLists.txt index 12957e1e8134..1408f32da566 100644 --- a/tiledb/api/c_api/buffer_list/CMakeLists.txt +++ b/tiledb/api/c_api/buffer_list/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,33 +25,18 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES buffer_list_api.cc ) gather_sources(${SOURCES}) -# -# Object library for other units to depend upon -# -add_library(capi_buffer_list_stub OBJECT ${SOURCES}) -target_link_libraries(capi_buffer_list_stub PUBLIC export) -target_link_libraries(capi_buffer_list_stub PUBLIC buffer $) -target_link_libraries(capi_buffer_list_stub PUBLIC buffer_list $) - -# -# We need to build this as a stub-based library because the API wrappers make -# reference to `context`, even though the functions in the buffer list section -# of the API do not use a context for error handling. -# -target_link_libraries(capi_buffer_list_stub PUBLIC capi_context_stub $) - - # - # Test-compile of object library ensures link-completeness - # - add_executable(compile_capi_buffer_list EXCLUDE_FROM_ALL) - target_sources(compile_capi_buffer_list PRIVATE test/compile_capi_buffer_list_main.cc) - target_link_libraries(compile_capi_buffer_list PRIVATE capi_buffer_list_stub) - add_dependencies(all_link_complete compile_capi_buffer_list) +commence(object_library capi_buffer_list_stub) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(capi_context_stub) + this_target_object_libraries(buffer buffer_list) +conclude(object_library) - add_test_subdirectory() +add_test_subdirectory() diff --git a/tiledb/api/c_api/buffer_list/test/CMakeLists.txt b/tiledb/api/c_api/buffer_list/test/CMakeLists.txt index cc5be67485f3..4cba0e4e2ff1 100644 --- a/tiledb/api/c_api/buffer_list/test/CMakeLists.txt +++ b/tiledb/api/c_api/buffer_list/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,16 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_buffer_list EXCLUDE_FROM_ALL) -target_sources(unit_capi_buffer_list PUBLIC unit_capi_buffer_list.cc) -target_link_libraries(unit_capi_buffer_list PUBLIC capi_buffer_list_stub) -target_link_libraries(unit_capi_buffer_list PUBLIC capi_buffer_stub) -target_link_libraries(unit_capi_buffer_list PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_buffer_list" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_buffer_list) + this_target_sources(unit_capi_buffer_list.cc) + this_target_object_libraries(capi_buffer_stub capi_buffer_list_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/buffer_list/test/compile_capi_buffer_list_main.cc b/tiledb/api/c_api/buffer_list/test/compile_capi_buffer_list_stub_main.cc similarity index 100% rename from tiledb/api/c_api/buffer_list/test/compile_capi_buffer_list_main.cc rename to tiledb/api/c_api/buffer_list/test/compile_capi_buffer_list_stub_main.cc diff --git a/tiledb/api/c_api/config/CMakeLists.txt b/tiledb/api/c_api/config/CMakeLists.txt index c3c5cff1cb8d..2f4e3235b31d 100644 --- a/tiledb/api/c_api/config/CMakeLists.txt +++ b/tiledb/api/c_api/config/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,33 +25,19 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES config_api.cc ) gather_sources(${SOURCES}) -# -# Object library designated as `_stub` version because it depends on the stub -# context library. -# -add_library(capi_config_stub OBJECT ${SOURCES}) -target_link_libraries(capi_config_stub PUBLIC export) -target_link_libraries(capi_config_stub PUBLIC config $) -target_link_libraries(capi_config_stub PUBLIC capi_error $) -# -# We need to build this as a stub-based library because the API wrappers make -# reference to `context`, even though the functions in the config section of the -# API do not use a context for error handling. -# -target_link_libraries(capi_config_stub PUBLIC storage_manager_stub) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_config_stub EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_config_stub PRIVATE capi_config_stub) -target_sources(compile_capi_config_stub PRIVATE test/compile_capi_config_main.cc) -add_dependencies(all_link_complete compile_capi_config_stub) +commence(object_library capi_config_stub) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_link_libraries(storage_manager_stub) + this_target_object_libraries(capi_error) + this_target_object_libraries(config) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/config/test/CMakeLists.txt b/tiledb/api/c_api/config/test/CMakeLists.txt index 37cceab5014e..107ce9da9792 100644 --- a/tiledb/api/c_api/config/test/CMakeLists.txt +++ b/tiledb/api/c_api/config/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,9 @@ # THE SOFTWARE. # -add_executable(unit_capi_config EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_config PUBLIC Catch2::Catch2WithMain) +include(unit_test) -# Sources for code under test -target_link_libraries(unit_capi_config PUBLIC capi_config_stub) - -# Sources for tests -target_sources(unit_capi_config PUBLIC - unit_capi_config.cc - ) - -add_test( - NAME "unit_capi_config" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_config) + this_target_sources(unit_capi_config.cc) + this_target_object_libraries(capi_config_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/config/test/compile_capi_config_main.cc b/tiledb/api/c_api/config/test/compile_capi_config_stub_main.cc similarity index 100% rename from tiledb/api/c_api/config/test/compile_capi_config_main.cc rename to tiledb/api/c_api/config/test/compile_capi_config_stub_main.cc diff --git a/tiledb/api/c_api/context/CMakeLists.txt b/tiledb/api/c_api/context/CMakeLists.txt index a35a9038c97b..17ce807610ba 100644 --- a/tiledb/api/c_api/context/CMakeLists.txt +++ b/tiledb/api/c_api/context/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,7 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES context_api.cc @@ -37,25 +38,12 @@ list(APPEND OTHER_SOURCES ../../../sm/storage_manager/context_resources.cc ) -# -# The stubbed object library is compiled with the `StorageManagerStub`. Anything -# that needs actual storage manager behavior will fail. If all that's needed is -# a reference, it will work fine. -# -add_library(capi_context_stub OBJECT ${SOURCES} ${OTHER_SOURCES}) -target_link_libraries(capi_context_stub PUBLIC export) -target_link_libraries(capi_context_stub PUBLIC storage_manager_stub) -target_link_libraries(capi_context_stub PUBLIC capi_config_stub $) -target_link_libraries(capi_context_stub PUBLIC vfs $) -target_link_libraries(capi_context_stub PUBLIC stats $) -target_link_libraries(capi_context_stub PUBLIC thread_pool $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_context_stub EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_context_stub PRIVATE capi_context_stub) -target_sources(compile_capi_context_stub PRIVATE test/compile_capi_context_main.cc) -add_dependencies(all_link_complete compile_capi_context_stub) +commence(object_library capi_context_stub) + this_target_sources(${SOURCES} ${OTHER_SOURCES}) + this_target_link_libraries(export) + this_target_link_libraries(storage_manager_stub) + this_target_object_libraries(capi_config_stub) + this_target_object_libraries(vfs stats thread_pool) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/context/test/CMakeLists.txt b/tiledb/api/c_api/context/test/CMakeLists.txt index 1fc2abe2208d..8b993e2a3a78 100644 --- a/tiledb/api/c_api/context/test/CMakeLists.txt +++ b/tiledb/api/c_api/context/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,9 @@ # THE SOFTWARE. # -add_executable(unit_capi_context EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_context PUBLIC Catch2::Catch2WithMain) +include(unit_test) -# Sources for code under test -target_link_libraries(unit_capi_context PUBLIC capi_context_stub) - -# Sources for tests -target_sources(unit_capi_context PUBLIC - unit_capi_context.cc - ) - -add_test( - NAME "unit_capi_context" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_context) + this_target_sources(unit_capi_context.cc) + this_target_object_libraries(capi_context_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/context/test/compile_capi_context_main.cc b/tiledb/api/c_api/context/test/compile_capi_context_stub_main.cc similarity index 100% rename from tiledb/api/c_api/context/test/compile_capi_context_main.cc rename to tiledb/api/c_api/context/test/compile_capi_context_stub_main.cc diff --git a/tiledb/api/c_api/data_order/test/CMakeLists.txt b/tiledb/api/c_api/data_order/test/CMakeLists.txt index d9e52accce03..23f13a7668fc 100644 --- a/tiledb/api/c_api/data_order/test/CMakeLists.txt +++ b/tiledb/api/c_api/data_order/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,15 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_data_order EXCLUDE_FROM_ALL) -target_sources(unit_capi_data_order PUBLIC unit_capi_data_order.cc) -target_link_libraries(unit_capi_data_order PUBLIC capi_data_order) -target_link_libraries(unit_capi_data_order PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_data_order" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_data_order) + this_target_sources(unit_capi_data_order.cc) + this_target_object_libraries(capi_data_order) +conclude(unit_test) diff --git a/tiledb/api/c_api/datatype/test/CMakeLists.txt b/tiledb/api/c_api/datatype/test/CMakeLists.txt index 1a36b2a4a574..58caf9cc86f6 100644 --- a/tiledb/api/c_api/datatype/test/CMakeLists.txt +++ b/tiledb/api/c_api/datatype/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,15 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_datatype EXCLUDE_FROM_ALL) -target_sources(unit_capi_datatype PUBLIC unit_capi_datatype.cc) -target_link_libraries(unit_capi_datatype PUBLIC capi_datatype) -target_link_libraries(unit_capi_datatype PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_datatype" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_datatype) + this_target_sources(unit_capi_datatype.cc) + this_target_object_libraries(capi_datatype) +conclude(unit_test) diff --git a/tiledb/api/c_api/dimension_label/CMakeLists.txt b/tiledb/api/c_api/dimension_label/CMakeLists.txt index e8f9d0d214ba..710cd3fe86e3 100644 --- a/tiledb/api/c_api/dimension_label/CMakeLists.txt +++ b/tiledb/api/c_api/dimension_label/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES dimension_label_api.cc @@ -33,21 +34,13 @@ list(APPEND SOURCES gather_sources(${SOURCES}) # -# Object library for other units to depend upon +# Object library for dimension label # -add_library(capi_dimension_label_stub OBJECT ${SOURCES}) -target_link_libraries(capi_dimension_label_stub PUBLIC capi_context_stub $) -target_link_libraries(capi_dimension_label_stub PUBLIC capi_datatype $) -target_link_libraries(capi_dimension_label_stub PUBLIC capi_data_order $) -target_link_libraries(capi_dimension_label_stub PUBLIC array_schema $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_dimension_label_stub EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_dimension_label_stub PRIVATE capi_dimension_label_stub) -target_link_libraries(compile_capi_dimension_label_stub PRIVATE generic_tile_io) -target_sources(compile_capi_dimension_label_stub PRIVATE test/compile_capi_dimension_label_main.cc) -add_dependencies(all_link_complete compile_capi_dimension_label_stub) +commence(object_library capi_dimension_label_stub) + this_target_sources(dimension_label_api.cc) + this_target_link_libraries(export) + this_target_object_libraries(capi_datatype capi_data_order capi_context_stub) + this_target_object_libraries(array_schema) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/dimension_label/test/CMakeLists.txt b/tiledb/api/c_api/dimension_label/test/CMakeLists.txt index b87d945bdc8e..ed16721b3167 100644 --- a/tiledb/api/c_api/dimension_label/test/CMakeLists.txt +++ b/tiledb/api/c_api/dimension_label/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,20 +25,9 @@ # include(common NO_POLICY_SCOPE) +include(unit_test) -add_executable(unit_capi_dimension_label EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_dimension_label PUBLIC Catch2::Catch2WithMain) -target_link_libraries(unit_capi_dimension_label PUBLIC capi_dimension_label_stub) -target_link_libraries(unit_capi_dimension_label PUBLIC generic_tile_io) - -# Sources for tests -target_sources(unit_capi_dimension_label PUBLIC - unit_capi_dimension_label.cc -) - -add_test( - NAME "unit_capi_dimension_label" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_dimension_label) + this_target_sources(unit_capi_dimension_label.cc) + this_target_object_libraries(capi_dimension_label_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_main.cc b/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc similarity index 92% rename from tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_main.cc rename to tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc index 76325a59e333..039db80f1356 100644 --- a/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_main.cc +++ b/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc @@ -22,6 +22,7 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. @@ -30,6 +31,7 @@ #include "../dimension_label_api_internal.h" int main() { - (void)sizeof(tiledb_dimension_label_handle_t); + // any function will drag in the whole library + tiledb::api::ensure_dimension_label_is_valid(nullptr); return 0; } diff --git a/tiledb/api/c_api/filesystem/test/CMakeLists.txt b/tiledb/api/c_api/filesystem/test/CMakeLists.txt index cdccf961290d..f7b0ede9fa63 100644 --- a/tiledb/api/c_api/filesystem/test/CMakeLists.txt +++ b/tiledb/api/c_api/filesystem/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,15 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_filesystem EXCLUDE_FROM_ALL) -target_sources(unit_capi_filesystem PUBLIC unit_capi_filesystem.cc) -target_link_libraries(unit_capi_filesystem PUBLIC capi_filesystem) -target_link_libraries(unit_capi_filesystem PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_filesystem" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_filesystem) + this_target_sources(unit_capi_filesystem.cc) + this_target_object_libraries(capi_filesystem) +conclude(unit_test) diff --git a/tiledb/api/c_api/filter/CMakeLists.txt b/tiledb/api/c_api/filter/CMakeLists.txt index 0a2ba279d4d8..1ee1b9a5c52d 100644 --- a/tiledb/api/c_api/filter/CMakeLists.txt +++ b/tiledb/api/c_api/filter/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,25 +25,17 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES filter_api.cc ) gather_sources(${SOURCES}) -# -# Object library for other units to depend upon -# -add_library(capi_filter_stub OBJECT ${SOURCES}) -target_link_libraries(capi_filter_stub PUBLIC capi_context_stub $) -target_link_libraries(capi_filter_stub PUBLIC all_filters $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_filter_stub EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_filter_stub PRIVATE capi_filter_stub) -target_sources(compile_capi_filter_stub PRIVATE test/compile_capi_filter_main.cc) -add_dependencies(all_link_complete compile_capi_filter_stub) +commence(object_library capi_filter_stub) + this_target_sources(${SOURCES}) + this_target_object_libraries(capi_context_stub) + this_target_object_libraries(all_filters) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/filter/test/CMakeLists.txt b/tiledb/api/c_api/filter/test/CMakeLists.txt index 131cc66e61e2..2df2af61aeea 100644 --- a/tiledb/api/c_api/filter/test/CMakeLists.txt +++ b/tiledb/api/c_api/filter/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,21 +24,9 @@ # THE SOFTWARE. # -include(common NO_POLICY_SCOPE) +include(unit_test) -add_executable(unit_capi_filter EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_filter PUBLIC Catch2::Catch2WithMain) -target_link_libraries(unit_capi_filter PUBLIC capi_filter_stub) - -# Sources for tests -target_sources(unit_capi_filter PUBLIC - # Only contains a single null test at present - unit_capi_filter.cc -) - -add_test( - NAME "unit_capi_filter" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_filter) + this_target_sources(unit_capi_filter.cc) + this_target_object_libraries(capi_filter_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/filter/test/compile_capi_filter_main.cc b/tiledb/api/c_api/filter/test/compile_capi_filter_stub_main.cc similarity index 97% rename from tiledb/api/c_api/filter/test/compile_capi_filter_main.cc rename to tiledb/api/c_api/filter/test/compile_capi_filter_stub_main.cc index bdf53d953ef2..224de3f32de2 100644 --- a/tiledb/api/c_api/filter/test/compile_capi_filter_main.cc +++ b/tiledb/api/c_api/filter/test/compile_capi_filter_stub_main.cc @@ -29,6 +29,6 @@ #include "../filter_api_internal.h" int main() { - (void)sizeof(tiledb_filter_handle_t); + tiledb_filter_free(nullptr); return 0; } diff --git a/tiledb/api/c_api/filter_list/CMakeLists.txt b/tiledb/api/c_api/filter_list/CMakeLists.txt index 1763aac6bc6e..617099bdf0d9 100644 --- a/tiledb/api/c_api/filter_list/CMakeLists.txt +++ b/tiledb/api/c_api/filter_list/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,25 +25,17 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES filter_list_api.cc ) gather_sources(${SOURCES}) -# -# Object library for other units to depend upon -# -add_library(capi_filter_list_stub OBJECT ${SOURCES}) -target_link_libraries(capi_filter_list_stub PUBLIC filter_pipeline $) -target_link_libraries(capi_filter_list_stub PUBLIC capi_filter_stub $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_filter_list_stub EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_filter_list_stub PRIVATE capi_filter_list_stub) -target_sources(compile_capi_filter_list_stub PRIVATE test/compile_capi_filter_list_main.cc) -add_dependencies(all_link_complete compile_capi_filter_list_stub) +commence(object_library capi_filter_list_stub) + this_target_sources(${SOURCES}) + this_target_object_libraries(capi_filter_stub) + this_target_object_libraries(filter_pipeline) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/filter_list/test/CMakeLists.txt b/tiledb/api/c_api/filter_list/test/CMakeLists.txt index 31741a73526b..d7e5ece35b70 100644 --- a/tiledb/api/c_api/filter_list/test/CMakeLists.txt +++ b/tiledb/api/c_api/filter_list/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,9 @@ # THE SOFTWARE. # -include(common NO_POLICY_SCOPE) +include(unit_test) -add_executable(unit_capi_filter_list EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_filter_list PUBLIC Catch2::Catch2WithMain) -target_link_libraries(unit_capi_filter_list PUBLIC capi_filter_list_stub) - -# Sources for tests -target_sources(unit_capi_filter_list PUBLIC - unit_capi_filter_list.cc -) - -add_test( - NAME "unit_capi_filter_list" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_filter_list) + this_target_sources(unit_capi_filter_list.cc) + this_target_object_libraries(capi_filter_list_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/filter_list/test/compile_capi_filter_list_main.cc b/tiledb/api/c_api/filter_list/test/compile_capi_filter_list_stub_main.cc similarity index 100% rename from tiledb/api/c_api/filter_list/test/compile_capi_filter_list_main.cc rename to tiledb/api/c_api/filter_list/test/compile_capi_filter_list_stub_main.cc diff --git a/tiledb/api/c_api/group/CMakeLists.txt b/tiledb/api/c_api/group/CMakeLists.txt index 580b87291ac1..b19ce9521119 100644 --- a/tiledb/api/c_api/group/CMakeLists.txt +++ b/tiledb/api/c_api/group/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -34,19 +34,9 @@ gather_sources(${SOURCES}) # # Object library for other units to depend upon # +# Maturity Note: We can't make a proper group unit library until we've +# removed its dependence on StorageManager. +# add_library(capi_group OBJECT ${SOURCES}) target_link_libraries(capi_group PUBLIC export) target_link_libraries(capi_group PUBLIC baseline $) - -# -# Test-compile of object library ensures link-completeness -# -# Note: We can't make a proper group unit library until we've -# removed its dependence on StorageManager. -# -# add_executable(compile_capi_group EXCLUDE_FROM_ALL) -# target_link_libraries(compile_capi_group PRIVATE capi_group) -# target_sources(compile_capi_group PRIVATE test/compile_capi_group_main.cc) -# add_dependencies(all_link_complete compile_capi_group) - -#add_test_subdirectory() diff --git a/tiledb/api/c_api/object/CMakeLists.txt b/tiledb/api/c_api/object/CMakeLists.txt index 37d2c6758a7b..b3fb5fe44ce3 100644 --- a/tiledb/api/c_api/object/CMakeLists.txt +++ b/tiledb/api/c_api/object/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -39,7 +39,7 @@ gather_sources(${SOURCES}) # # Maturity Warning: `capi_object` can't be a full object library unit the # `StorageManager` dependency is removed. Note that the test-compile is -# commented out. This is the reason that it hasn't been converted to use the +# absent. This is the reason that it hasn't been converted to use the # `object_library` environment for CMake. # add_library(capi_object OBJECT ${SOURCES}) @@ -48,12 +48,4 @@ target_link_libraries(capi_object PUBLIC baseline $) target_link_libraries(capi_object PUBLIC constants $) target_link_libraries(capi_object PUBLIC exception_wrapper $) -# -# Test-compile of object library ensures link-completeness -# -# add_executable(compile_capi_object EXCLUDE_FROM_ALL) -# target_link_libraries(compile_capi_object PRIVATE capi_object) -# target_sources(compile_capi_object PRIVATE test/compile_capi_object_main.cc) -# add_dependencies(all_link_complete compile_capi_object) - add_test_subdirectory() diff --git a/tiledb/api/c_api/query/test/CMakeLists.txt b/tiledb/api/c_api/query/test/CMakeLists.txt index 3083d00ce508..6dd18b072b33 100644 --- a/tiledb/api/c_api/query/test/CMakeLists.txt +++ b/tiledb/api/c_api/query/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,15 +24,9 @@ # THE SOFTWARE. # -find_package(Catch_EP REQUIRED) +include(unit_test) -add_executable(unit_capi_query_type EXCLUDE_FROM_ALL) -target_sources(unit_capi_query_type PUBLIC unit_capi_query_type.cc) -target_link_libraries(unit_capi_query_type PUBLIC capi_query) -target_link_libraries(unit_capi_query_type PUBLIC Catch2::Catch2WithMain) - -add_test( - NAME "unit_capi_query_type" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_query_type) + this_target_sources(unit_capi_query_type.cc) + this_target_object_libraries(capi_query) +conclude(unit_test) diff --git a/tiledb/api/c_api/query_aggregate/CMakeLists.txt b/tiledb/api/c_api/query_aggregate/CMakeLists.txt index d1bd23f1bc78..71bf3dabfa62 100644 --- a/tiledb/api/c_api/query_aggregate/CMakeLists.txt +++ b/tiledb/api/c_api/query_aggregate/CMakeLists.txt @@ -31,17 +31,15 @@ list(APPEND SOURCES gather_sources(${SOURCES}) # -# Object library for other units to depend upon +# Object library for C API query_aggregate +# +# Maturity: +# This object library does not link standalone. There's no corresponding +# `query_aggregate` object library to link to, since at present nothing +# involving query is able to link standalone. # add_library(capi_query_aggregate OBJECT ${SOURCES}) target_link_libraries(capi_query_aggregate PUBLIC export) target_link_libraries(capi_query_aggregate PUBLIC baseline $) -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_query_aggregate EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_query_aggregate PRIVATE capi_query_aggregate) -target_sources(compile_capi_query_aggregate PRIVATE test/compile_capi_query_aggregate_main.cc) - add_test_subdirectory() diff --git a/tiledb/api/c_api/query_field/CMakeLists.txt b/tiledb/api/c_api/query_field/CMakeLists.txt index 9b627780676d..b8fd508cedd9 100644 --- a/tiledb/api/c_api/query_field/CMakeLists.txt +++ b/tiledb/api/c_api/query_field/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2023 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -41,7 +41,10 @@ list(APPEND SOURCES gather_sources(${SOURCES}) # -# Object library for other units to depend upon +# Object library for C API query_field +# +# Maturity Note: object library `capi_query_field` cannot be linked stand-alone +# at present. # add_library(capi_query_field OBJECT ${SOURCES}) target_link_libraries(capi_query_field PUBLIC export) diff --git a/tiledb/api/c_api/query_plan/CMakeLists.txt b/tiledb/api/c_api/query_plan/CMakeLists.txt index 72a693f72496..544e29bde3e7 100644 --- a/tiledb/api/c_api/query_plan/CMakeLists.txt +++ b/tiledb/api/c_api/query_plan/CMakeLists.txt @@ -31,18 +31,15 @@ list(APPEND SOURCES gather_sources(${SOURCES}) # -# Object library for other units to depend upon +# Object library for C API query plan +# +# Maturity: +# This object library cannot link standalone because `query_plan` cannot link +# standalone. # add_library(capi_query_plan OBJECT ${SOURCES}) target_link_libraries(capi_query_plan PUBLIC export) target_link_libraries(capi_query_plan PUBLIC baseline $) target_link_libraries(capi_query_plan PUBLIC query_plan $) -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_query_plan EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_query_plan PRIVATE capi_query_plan) -target_sources(compile_capi_query_plan PRIVATE test/compile_capi_query_plan_main.cc) - add_test_subdirectory() diff --git a/tiledb/api/c_api/vfs/CMakeLists.txt b/tiledb/api/c_api/vfs/CMakeLists.txt index aab90a4a5d0b..6c93365061b3 100644 --- a/tiledb/api/c_api/vfs/CMakeLists.txt +++ b/tiledb/api/c_api/vfs/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,24 +25,17 @@ # include(common NO_POLICY_SCOPE) +include(object_library) list(APPEND SOURCES vfs_api.cc ) gather_sources(${SOURCES}) -# -# Object library for other units to depend upon -# -add_library(capi_vfs OBJECT ${SOURCES}) -target_link_libraries(capi_vfs PUBLIC capi_context_stub $) - -# -# Test-compile of object library ensures link-completeness -# -add_executable(compile_capi_vfs EXCLUDE_FROM_ALL) -target_link_libraries(compile_capi_vfs PRIVATE capi_vfs) -target_sources(compile_capi_vfs PRIVATE test/compile_capi_vfs_main.cc) -add_dependencies(all_link_complete compile_capi_vfs) +commence(object_library capi_vfs) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(capi_context_stub) +conclude(object_library) add_test_subdirectory() diff --git a/tiledb/api/c_api/vfs/test/CMakeLists.txt b/tiledb/api/c_api/vfs/test/CMakeLists.txt index f7aae2be5a47..90a3d1078933 100644 --- a/tiledb/api/c_api/vfs/test/CMakeLists.txt +++ b/tiledb/api/c_api/vfs/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2022 TileDB, Inc. +# Copyright (c) 2022-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,9 @@ # THE SOFTWARE. # -include(common NO_POLICY_SCOPE) +include(unit_test) -add_executable(unit_capi_vfs EXCLUDE_FROM_ALL) -find_package(Catch_EP REQUIRED) -target_link_libraries(unit_capi_vfs PUBLIC Catch2::Catch2WithMain) -target_link_libraries(unit_capi_vfs PUBLIC capi_vfs) - -# Sources for tests -target_sources(unit_capi_vfs PUBLIC - unit_capi_vfs.cc -) - -add_test( - NAME "unit_capi_vfs" - COMMAND $ - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) +commence(unit_test capi_vfs) + this_target_sources(unit_capi_vfs.cc) + this_target_object_libraries(capi_vfs) +conclude(unit_test) diff --git a/tiledb/sm/query_plan/CMakeLists.txt b/tiledb/sm/query_plan/CMakeLists.txt index c9ab9ca7056e..b7be22e5d0bb 100644 --- a/tiledb/sm/query_plan/CMakeLists.txt +++ b/tiledb/sm/query_plan/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2023-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,13 +25,16 @@ # include(common NO_POLICY_SCOPE) +# +# Object library for query plan +# +# Maturity: +# This library cannot link standalone because it depends upon `class Query`. +# This class is at present unable to be part of a standalone object library. +# Note that the unit test for this module links in the whole library. +# add_library(query_plan OBJECT query_plan.cc) target_link_libraries(query_plan PUBLIC baseline $) target_link_libraries(query_plan PUBLIC buffer $) -add_executable(compile_query_plan EXCLUDE_FROM_ALL) -target_link_libraries(compile_query_plan PRIVATE query_plan) -target_sources(compile_query_plan PRIVATE test/compile_query_plan_main.cc $ -) - add_test_subdirectory() From 69028d8d3657a10dc808dbada82cdd310bfb933d Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 14 May 2024 12:27:24 +0300 Subject: [PATCH 339/456] Add array uri to tiledb_array_deserialize (#4961) [sc-47129 ](https://app.shortcut.com/tiledb-inc/story/47129/add-array-uri-to-tiledb-array-deserialize ) Today `tiledb_array_deserialize` is using a hardcoded uri name (`"deserialized_array"`) , whereas `tiledb_deserialize_query_and_array` is requiring the actual `array_uri` as an argument. The former is problematic for wrapping those calls in python, so this ticket is aiming to unify the two to both receive the `array_uri` as input. (history on why this is needed here: https://app.shortcut.com/tiledb-inc/story/47139/investigate-if-core-serialization-apis-could-be-used-externally-to-pass-cache-open-arrays) There's no backward compatibility issue here as this is a deserialization function only called by REST Server, so we don't need to keep both versions of `tiledb_array_deserialize`. That said, the REST-CI tests that run based on latest dev on Core and latest master in Cloud-REST will break after this change. We will handle this by adapting the REST-CI script to patch temporarily Cloud-REST master with the needed change and remove when 2.24 is released https://github.com/TileDB-Inc/TileDB-Internal/pull/27 . Another idea was to maintain both flavors until the release is ready to go out and then remove but C doesn't support overloading functions. Future work to track: https://github.com/TileDB-Inc/TileDB/pull/4961 --- TYPE: IMPROVEMENT DESC: Add array uri to tiledb_array_deserialize --- test/support/src/serialization_wrappers.cc | 8 +++++++- tiledb/sm/c_api/tiledb.cc | 6 ++++-- tiledb/sm/c_api/tiledb_serialization.h | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/support/src/serialization_wrappers.cc b/test/support/src/serialization_wrappers.cc index fa7dec6fc129..1ae4d58ea95c 100644 --- a/test/support/src/serialization_wrappers.cc +++ b/test/support/src/serialization_wrappers.cc @@ -141,7 +141,13 @@ int array_serialize_wrapper( REQUIRE(rc == TILEDB_OK); // Load array from the rest server - rc = tiledb_deserialize_array(ctx, buff, serialize_type, 0, new_array); + rc = tiledb_deserialize_array( + ctx, + buff, + serialize_type, + 0, + array->array_->array_uri().c_str(), + new_array); REQUIRE(rc == TILEDB_OK); // Clean up. diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 34c28865e0a0..487d09d65cec 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3378,6 +3378,7 @@ int32_t tiledb_deserialize_array( const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t, + const char* array_uri, tiledb_array_t** array) { // Sanity check @@ -3393,7 +3394,7 @@ int32_t tiledb_deserialize_array( } // Check array URI - auto uri = tiledb::sm::URI("deserialized_array"); + auto uri = tiledb::sm::URI(array_uri); if (uri.is_invalid()) { auto st = Status_Error("Failed to create TileDB array object; Invalid URI"); delete *array; @@ -6932,9 +6933,10 @@ CAPI_INTERFACE( const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, + const char* array_uri, tiledb_array_t** array) { return api_entry( - ctx, buffer, serialize_type, client_side, array); + ctx, buffer, serialize_type, client_side, array_uri, array); } CAPI_INTERFACE( diff --git a/tiledb/sm/c_api/tiledb_serialization.h b/tiledb/sm/c_api/tiledb_serialization.h index 84bfe305ca0d..821a7ed1d38c 100644 --- a/tiledb/sm/c_api/tiledb_serialization.h +++ b/tiledb/sm/c_api/tiledb_serialization.h @@ -28,6 +28,8 @@ * @section DESCRIPTION * * This file declares the TileDB C API for serialization. + * APIs defined in this header are currently unstable and subject to breaking + * changes between minor releases. */ #ifndef TILEDB_SERIALIZATION_H @@ -106,6 +108,7 @@ TILEDB_EXPORT int32_t tiledb_serialize_array( * @param client_side Allows to specify different behavior depending on who is * serializing, the client (1) or the Cloud server (0). This is sometimes needed * since they are both using the same Core library APIs for serialization. + * @param array_uri uri of the array to deserialize. * @param array Will be set to a newly allocated array. * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. */ @@ -114,6 +117,7 @@ TILEDB_EXPORT int32_t tiledb_deserialize_array( const tiledb_buffer_t* buffer, tiledb_serialization_type_t serialize_type, int32_t client_side, + const char* array_uri, tiledb_array_t** array) TILEDB_NOEXCEPT; /** From 1129008a930ca6c25ceaad7d404fb94f8d026171 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 14 May 2024 12:33:56 +0300 Subject: [PATCH 340/456] Restore rest.resubmit_incomplete config parameter (#4970) For some inexplicable reason I thought `rest.resubmit_incomplete` config variable wasn't used anymore and removed it in https://github.com/TileDB-Inc/TileDB/pull/3879 . This PR is for reverting this change. --- TYPE: IMPROVEMENT DESC: Restore rest.resubmit_incomplete config parameter --- test/src/unit-capi-rest-dense_array.cc | 7 ++++++- tiledb/api/c_api/config/config_api_external.h | 4 ++++ tiledb/sm/cpp_api/config.h | 4 ++++ tiledb/sm/rest/rest_client.cc | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/test/src/unit-capi-rest-dense_array.cc b/test/src/unit-capi-rest-dense_array.cc index 3f9d4a290122..cd4cff6328bb 100644 --- a/test/src/unit-capi-rest-dense_array.cc +++ b/test/src/unit-capi-rest-dense_array.cc @@ -1699,7 +1699,12 @@ TEST_CASE_METHOD( DenseArrayRESTFx, "C API: REST Test dense array, incomplete reads", "[capi][rest][dense][incomplete]") { - // TODO: refactor for each supported FS. + tiledb_error_t* error; + tiledb_config_t* config; + tiledb_config_alloc(&config, &error); + REQUIRE( + tiledb_config_set(config, "rest.resubmit_incomplete", "false", &error) == + TILEDB_OK); check_incomplete_reads(); } diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 0b62b147ef00..8be1968935c8 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -623,6 +623,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * Authentication token for REST server (used instead of * username/password).
* **Default**: "" + * - `rest.resubmit_incomplete`
+ * If true, incomplete queries received from server are automatically + * resubmitted before returning to user control.
+ * **Default**: "true" * - `rest.ignore_ssl_validation`
* Have curl ignore ssl peer and host validation for REST server.
* **Default**: false diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 74aee2a827bb..e2cd3bc727b1 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -801,6 +801,10 @@ class Config { * Authentication token for REST server (used instead of * username/password).
* **Default**: "" + * - `rest.resubmit_incomplete`
+ * If true, incomplete queries received from server are automatically + * resubmitted before returning to user control.
+ * **Default**: "true" * - `rest.ignore_ssl_validation`
* Have curl ignore ssl peer and host validation for REST server.
* **Default**: false diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 1429b36b3da2..c06aa2cfa3b6 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -144,6 +144,10 @@ Status RestClient::init( if (c_str != nullptr) RETURN_NOT_OK(serialization_type_enum(c_str, &serialization_type_)); + bool found = false; + RETURN_NOT_OK(config_->get( + "rest.resubmit_incomplete", &resubmit_incomplete_, &found)); + return Status::Ok(); } From 739bd584140a725098e1c0e801146f007072d94e Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 14 May 2024 15:36:13 +0300 Subject: [PATCH 341/456] Fix traversal limit in array metadata serialization (#4971) [sc-46026] Reading the VCF `allele_count` array with a specific `tiledb://` URI was failing with `TileDBError: [TileDB::Serialization] Error: Error deserializing array metadata; kj::Exception: Exceeded message traversal limit. See capnp::ReaderOptions.` This is a classic issue we have seen before where Capnp is reaching the traversal limit and is solved by explicitly setting the traversal limit to `rest.capnp_traversal_limit` config option. As validation , I have run the failing python script provided in the ticket description and now it passes. --- TYPE: BUG DESC: Fix traversal limit in array metadata serialization --- tiledb/api/c_api/group/group_api.cc | 1 + tiledb/sm/c_api/tiledb.cc | 1 + tiledb/sm/rest/rest_client.cc | 10 +++++-- tiledb/sm/serialization/array.cc | 19 +++++++++--- tiledb/sm/serialization/array.h | 1 + tiledb/sm/serialization/group.cc | 45 +++++++++++++++++++++++------ 6 files changed, 62 insertions(+), 15 deletions(-) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 74bf9f123007..6fbcf4f865b3 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -566,6 +566,7 @@ capi_return_t tiledb_deserialize_group_metadata( throw_if_not_ok(tiledb::sm::serialization::metadata_deserialize( group->group().unsafe_metadata(), + group->group().config(), static_cast(serialize_type), buffer->buffer())); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 487d09d65cec..7e2848823ac7 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -4038,6 +4038,7 @@ int32_t tiledb_deserialize_array_metadata( // Deserialize throw_if_not_ok(tiledb::sm::serialization::metadata_deserialize( array->array_->unsafe_metadata(), + array->array_->config(), (tiledb::sm::SerializationType)serialize_type, buffer->buffer())); diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index c06aa2cfa3b6..c291323f39ea 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -626,7 +626,10 @@ Status RestClient::get_array_metadata_from_rest( // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::metadata_deserialize( - array->unsafe_metadata(), serialization_type_, returned_data); + array->unsafe_metadata(), + array->config(), + serialization_type_, + returned_data); } Status RestClient::post_array_metadata_to_rest( @@ -1404,7 +1407,10 @@ Status RestClient::post_group_metadata_from_rest(const URI& uri, Group* group) { // Ensure data has a null delimiter for cap'n proto if using JSON RETURN_NOT_OK(ensure_json_null_delimited_string(&returned_data)); return serialization::metadata_deserialize( - group->unsafe_metadata(), serialization_type_, returned_data); + group->unsafe_metadata(), + group->config(), + serialization_type_, + returned_data); } Status RestClient::put_group_metadata_to_rest(const URI& uri, Group* group) { diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 602132df68be..1c56887dd36b 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -430,6 +430,7 @@ Status metadata_serialize( Status metadata_deserialize( Metadata* metadata, + const Config& config, SerializationType serialize_type, const Buffer& serialized_buffer) { if (metadata == nullptr) @@ -453,11 +454,20 @@ Status metadata_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + config.get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader msg_reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); auto reader = msg_reader.getRoot(); // Deserialize @@ -724,7 +734,8 @@ Status metadata_serialize(Metadata*, SerializationType, Buffer*) { throw ArraySerializationDisabledException(); } -Status metadata_deserialize(Metadata*, SerializationType, const Buffer&) { +Status metadata_deserialize( + Metadata*, const Config&, SerializationType, const Buffer&) { throw ArraySerializationDisabledException(); } diff --git a/tiledb/sm/serialization/array.h b/tiledb/sm/serialization/array.h index 4f7f4fc1111d..06c1d7b31276 100644 --- a/tiledb/sm/serialization/array.h +++ b/tiledb/sm/serialization/array.h @@ -183,6 +183,7 @@ Status metadata_serialize( Status metadata_deserialize( Metadata* metadata, + const Config& config, SerializationType serialize_type, const Buffer& serialized_buffer); diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index 9a233f8bc088..bbb2e4e4b883 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -433,11 +433,20 @@ Status group_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + group->config().get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); capnp::Group::Reader group_reader = reader.getRoot(); RETURN_NOT_OK(group_from_capnp(group_reader, group)); break; @@ -533,11 +542,20 @@ Status group_details_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + group->config().get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); capnp::Group::GroupDetails::Reader group_details_reader = reader.getRoot(); RETURN_NOT_OK(group_details_from_capnp(group_details_reader, group)); @@ -637,11 +655,20 @@ Status group_update_deserialize( break; } case SerializationType::CAPNP: { + // Set traversal limit from config + uint64_t limit = + group->config().get("rest.capnp_traversal_limit").value(); + ::capnp::ReaderOptions readerOptions; + // capnp uses the limit in words + readerOptions.traversalLimitInWords = limit / sizeof(::capnp::word); + const auto mBytes = reinterpret_cast(serialized_buffer.data()); - ::capnp::FlatArrayMessageReader reader(kj::arrayPtr( - reinterpret_cast(mBytes), - serialized_buffer.size() / sizeof(::capnp::word))); + ::capnp::FlatArrayMessageReader reader( + kj::arrayPtr( + reinterpret_cast(mBytes), + serialized_buffer.size() / sizeof(::capnp::word)), + readerOptions); capnp::Group::Reader group_reader = reader.getRoot(); RETURN_NOT_OK(group_from_capnp(group_reader, group)); break; From 6aca860d2fade1bf069d0870dc949d8e74dcd79a Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Tue, 14 May 2024 16:01:02 +0300 Subject: [PATCH 342/456] Move FragmentMetadata::tile_offsets to new fragment OndemandMetadata. (#4848) Move `FragmentMetadata::tile_offsets_` to new fragment OndemandMetadata. Kept it as simple as possible so we get early feedback and potentially merge individual work pieces: - This PR is moving only tile_offsets, the intent is to move all lazily loaded metadata. - It might be possible we get rid of the `LoadedMetadata` mappings altogether and model optionality with `std::optional` but it's not done in this PR to minimize the changes. - Once the loaded metadata is moved to the queries, `tile_offsets_mtx_` and `mtx_` are not going to be needed anymore and can be removed. --- TYPE: NO_HISTORY DESC: Move FragmentMetadata::tile_offsets to new fragment OndemandMetadata. --------- Co-authored-by: Luc Rancourt --- test/src/unit-ReadCellSlabIter.cc | 16 +- test/src/unit-Reader.cc | 2 +- test/src/unit-result-coords.cc | 2 +- test/src/unit-result-tile.cc | 6 +- .../unit-sparse-unordered-with-dups-reader.cc | 2 +- test/src/unit-tile-metadata.cc | 180 +- test/support/src/helpers.cc | 16 + test/support/src/helpers.h | 12 + tiledb/CMakeLists.txt | 5 +- tiledb/sm/array/array.cc | 6 +- tiledb/sm/fragment/fragment_info.cc | 10 +- tiledb/sm/fragment/fragment_metadata.cc | 1882 +++-------------- tiledb/sm/fragment/fragment_metadata.h | 686 +----- .../sm/fragment/loaded_fragment_metadata.cc | 544 +++++ tiledb/sm/fragment/loaded_fragment_metadata.h | 790 +++++++ .../sm/fragment/ondemand_fragment_metadata.cc | 675 ++++++ .../sm/fragment/ondemand_fragment_metadata.h | 279 +++ .../v1v2preloaded_fragment_metadata.cc | 267 +++ .../v1v2preloaded_fragment_metadata.h | 226 ++ tiledb/sm/query/query_condition.cc | 6 +- tiledb/sm/query/readers/filtered_data.h | 25 +- .../query/readers/ordered_dim_label_reader.cc | 9 +- tiledb/sm/query/readers/reader_base.cc | 24 +- tiledb/sm/query/readers/result_tile.h | 22 +- .../readers/sparse_global_order_reader.cc | 1 + .../query/readers/sparse_index_reader_base.cc | 10 +- .../sparse_unordered_with_dups_reader.cc | 2 +- tiledb/sm/query/test/unit_query_condition.cc | 70 +- tiledb/sm/query/writers/writer_base.cc | 5 +- tiledb/sm/serialization/array.cc | 7 +- tiledb/sm/serialization/fragment_info.cc | 7 +- tiledb/sm/serialization/fragment_metadata.cc | 97 +- tiledb/sm/subarray/subarray.cc | 19 +- tools/src/commands/info_command.cc | 15 +- 34 files changed, 3516 insertions(+), 2409 deletions(-) create mode 100644 tiledb/sm/fragment/loaded_fragment_metadata.cc create mode 100644 tiledb/sm/fragment/loaded_fragment_metadata.h create mode 100644 tiledb/sm/fragment/ondemand_fragment_metadata.cc create mode 100644 tiledb/sm/fragment/ondemand_fragment_metadata.h create mode 100644 tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc create mode 100644 tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h diff --git a/test/src/unit-ReadCellSlabIter.cc b/test/src/unit-ReadCellSlabIter.cc index 4e736400aafd..db7512c5e3ba 100644 --- a/test/src/unit-ReadCellSlabIter.cc +++ b/test/src/unit-ReadCellSlabIter.cc @@ -262,7 +262,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -336,7 +336,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -414,7 +414,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -497,7 +497,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -720,7 +720,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -906,7 +906,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -1105,7 +1105,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); @@ -1351,7 +1351,7 @@ TEST_CASE_METHOD( HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 6b053017984e..8f21d9f0e9a3 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -261,7 +261,7 @@ TEST_CASE_METHOD( HERE(), nullptr, schema, - URI(), + generate_fragment_uri(&array), std::make_pair(0, 0), tracker_, true); diff --git a/test/src/unit-result-coords.cc b/test/src/unit-result-coords.cc index b4cbe94daba7..ca91091a82f1 100644 --- a/test/src/unit-result-coords.cc +++ b/test/src/unit-result-coords.cc @@ -110,7 +110,7 @@ CResultCoordsFx::CResultCoordsFx(uint64_t num_cells) { HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); diff --git a/test/src/unit-result-tile.cc b/test/src/unit-result-tile.cc index dd7d01cae3d3..f6c67a25ee26 100644 --- a/test/src/unit-result-tile.cc +++ b/test/src/unit-result-tile.cc @@ -118,7 +118,7 @@ CResultTileFx::CResultTileFx() HERE(), nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), memory_tracker_, false); @@ -198,7 +198,7 @@ TEST_CASE_METHOD( FragmentMetadata frag_md( nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), memory_tracker_, true); @@ -311,7 +311,7 @@ TEST_CASE_METHOD( FragmentMetadata frag_md( nullptr, array_->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array_->array_.get()), std::make_pair(0, 0), memory_tracker_, true); diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index b93568d34419..95fd8f3fa566 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -781,7 +781,7 @@ CSparseUnorderedWithDupsVarDataFx::open_default_array_1d_with_fragments( HERE(), nullptr, array->array_->array_schema_latest_ptr(), - URI(), + generate_fragment_uri(array->array_.get()), std::make_pair(0, 0), tiledb::test::create_test_memory_tracker(), true); diff --git a/test/src/unit-tile-metadata.cc b/test/src/unit-tile-metadata.cc index c25d4bfa058d..f967bf7efe3e 100644 --- a/test/src/unit-tile-metadata.cc +++ b/test/src/unit-tile-metadata.cc @@ -285,17 +285,19 @@ struct CPPFixedTileMetadataFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); + frag_meta[f]->loaded_metadata()->load_fragment_min_max_sum_null_count( + enc_key); // Load the metadata and validate coords netadata. bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { std::vector names{"d"}; - frag_meta[f]->load_rtree(enc_key); - frag_meta[f]->load_tile_min_values(enc_key, names); - frag_meta[f]->load_tile_max_values(enc_key, names); - frag_meta[f]->load_tile_sum_values(enc_key, names); - frag_meta[f]->load_tile_null_count_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_rtree(enc_key); + frag_meta[f]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); // Validation. // Min/max/sum for all null tile are invalid. @@ -307,18 +309,18 @@ struct CPPFixedTileMetadataFx { // Validate no min. CHECK_THROWS_WITH( - frag_meta[f]->get_min("d"), + frag_meta[f]->loaded_metadata()->get_min("d"), "FragmentMetadata: Trying to access fragment min metadata that's " "not present"); // Validate no max. CHECK_THROWS_WITH( - frag_meta[f]->get_max("d"), + frag_meta[f]->loaded_metadata()->get_max("d"), "FragmentMetadata: Trying to access fragment max metadata that's " "not present"); // Validate sum. - auto sum = frag_meta[f]->get_sum("d"); + auto sum = frag_meta[f]->loaded_metadata()->get_sum("d"); CHECK(*(int64_t*)sum == correct_sum); } @@ -341,7 +343,8 @@ struct CPPFixedTileMetadataFx { "present"); // Validate sum. - auto sum = frag_meta[f]->get_tile_sum("d", tile_idx); + auto sum = + frag_meta[f]->loaded_metadata()->get_tile_sum("d", tile_idx); CHECK(*(int64_t*)sum == correct_sum); // Validate the tile metadata structure. @@ -359,19 +362,19 @@ struct CPPFixedTileMetadataFx { if constexpr (std::is_same::value) { // Validate no min. CHECK_THROWS_WITH( - frag_meta[f]->get_min("a"), + frag_meta[f]->loaded_metadata()->get_min("a"), "FragmentMetadata: Trying to access fragment min metadata that's " "not present"); // Validate no max. CHECK_THROWS_WITH( - frag_meta[f]->get_max("a"), + frag_meta[f]->loaded_metadata()->get_max("a"), "FragmentMetadata: Trying to access fragment max metadata that's " "not present"); // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_sum("a"), + frag_meta[f]->loaded_metadata()->get_sum("a"), "FragmentMetadata: Trying to access fragment sum metadata that's " "not present"); } else { @@ -379,7 +382,7 @@ struct CPPFixedTileMetadataFx { if (!all_null) { if constexpr (std::is_same::value) { // Validate min. - auto& min = frag_meta[f]->get_min("a"); + auto& min = frag_meta[f]->loaded_metadata()->get_min("a"); CHECK(min.size() == cell_val_num); // For strings, the index is stored in a signed value, switch to @@ -393,7 +396,7 @@ struct CPPFixedTileMetadataFx { cell_val_num)); // Validate max. - auto& max = frag_meta[f]->get_max("a"); + auto& max = frag_meta[f]->loaded_metadata()->get_max("a"); CHECK(max.size() == cell_val_num); // For strings, the index is stored in a signed value, switch to @@ -408,23 +411,23 @@ struct CPPFixedTileMetadataFx { // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_sum("a"), + frag_meta[f]->loaded_metadata()->get_sum("a"), "FragmentMetadata: Trying to access fragment sum metadata " "that's not present"); } else { // Validate min. - auto& min = frag_meta[f]->get_min("a"); + auto& min = frag_meta[f]->loaded_metadata()->get_min("a"); CHECK(min.size() == sizeof(TestType)); CHECK(0 == memcmp(min.data(), &correct_mins_[f], min.size())); // Validate max. - auto& max = frag_meta[f]->get_max("a"); + auto& max = frag_meta[f]->loaded_metadata()->get_max("a"); CHECK(max.size() == sizeof(TestType)); CHECK(0 == memcmp(max.data(), &correct_maxs_[f], max.size())); if constexpr (!std::is_same::value) { // Validate sum. - auto sum = frag_meta[f]->get_sum("a"); + auto sum = frag_meta[f]->loaded_metadata()->get_sum("a"); if constexpr (std::is_integral_v) { CHECK(*(int64_t*)sum == correct_sums_int_[f]); } else { @@ -437,11 +440,11 @@ struct CPPFixedTileMetadataFx { // Check null count. if (nullable) { - auto nc = frag_meta[f]->get_null_count("a"); + auto nc = frag_meta[f]->loaded_metadata()->get_null_count("a"); CHECK(nc == correct_null_counts_[f]); } else { CHECK_THROWS_WITH( - frag_meta[f]->get_null_count("a"), + frag_meta[f]->loaded_metadata()->get_null_count("a"), "FragmentMetadata: Trying to access fragment null count metadata " "that's not present"); } @@ -449,10 +452,11 @@ struct CPPFixedTileMetadataFx { // Load attribute metadata. std::vector names{"a"}; - frag_meta[f]->load_tile_min_values(enc_key, names); - frag_meta[f]->load_tile_max_values(enc_key, names); - frag_meta[f]->load_tile_sum_values(enc_key, names); - frag_meta[f]->load_tile_null_count_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); // Validate attribute metadta. // Min/max/sum for all null tile are invalid. @@ -471,7 +475,7 @@ struct CPPFixedTileMetadataFx { // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_tile_sum("a", 0), + frag_meta[f]->loaded_metadata()->get_tile_sum("a", 0), "FragmentMetadata: Trying to access tile sum metadata that's not " "present"); } else { @@ -508,7 +512,7 @@ struct CPPFixedTileMetadataFx { // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_tile_sum("a", tile_idx), + frag_meta[f]->loaded_metadata()->get_tile_sum("a", tile_idx), "FragmentMetadata: Trying to access tile sum metadata that's " "not " "present"); @@ -554,7 +558,8 @@ struct CPPFixedTileMetadataFx { if constexpr (!std::is_same::value) { // Validate sum. - auto sum = frag_meta[f]->get_tile_sum("a", tile_idx); + auto sum = + frag_meta[f]->loaded_metadata()->get_tile_sum("a", tile_idx); if constexpr (std::is_integral_v) { CHECK(*(int64_t*)sum == correct_tile_sums_int_[f][tile_idx]); CHECK( @@ -575,11 +580,12 @@ struct CPPFixedTileMetadataFx { // Check null count. for (uint64_t tile_idx = 0; tile_idx < num_tiles_; tile_idx++) { if (nullable) { - auto nc = frag_meta[f]->get_tile_null_count("a", tile_idx); + auto nc = + frag_meta[f]->loaded_metadata()->get_tile_null_count("a", tile_idx); CHECK(nc == correct_tile_null_counts_[f][tile_idx]); } else { CHECK_THROWS_WITH( - frag_meta[f]->get_tile_null_count("a", tile_idx), + frag_meta[f]->loaded_metadata()->get_tile_null_count("a", tile_idx), "FragmentMetadata: Trying to access tile null count metadata " "that's not present"); } @@ -830,17 +836,19 @@ struct CPPVarTileMetadataFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - frag_meta[f]->load_fragment_min_max_sum_null_count(enc_key); + frag_meta[f]->loaded_metadata()->load_fragment_min_max_sum_null_count( + enc_key); // Load the metadata and validate coords netadata. bool has_coords = layout != TILEDB_ROW_MAJOR; if (has_coords) { std::vector names{"d"}; - frag_meta[f]->load_rtree(enc_key); - frag_meta[f]->load_tile_min_values(enc_key, names); - frag_meta[f]->load_tile_max_values(enc_key, names); - frag_meta[f]->load_tile_sum_values(enc_key, names); - frag_meta[f]->load_tile_null_count_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_rtree(enc_key); + frag_meta[f]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); // Validation. // Min/max/sum for all null tile are invalid. @@ -852,18 +860,18 @@ struct CPPVarTileMetadataFx { // Validate no min. CHECK_THROWS_WITH( - frag_meta[f]->get_min("d"), + frag_meta[f]->loaded_metadata()->get_min("d"), "FragmentMetadata: Trying to access fragment min metadata that's " "not present"); // Validate no max. CHECK_THROWS_WITH( - frag_meta[f]->get_max("d"), + frag_meta[f]->loaded_metadata()->get_max("d"), "FragmentMetadata: Trying to access fragment max metadata that's " "not present"); // Validate sum. - auto sum = frag_meta[f]->get_sum("d"); + auto sum = frag_meta[f]->loaded_metadata()->get_sum("d"); CHECK(*(int64_t*)sum == correct_sum); } @@ -886,7 +894,8 @@ struct CPPVarTileMetadataFx { "present"); // Validate sum. - auto sum = frag_meta[f]->get_tile_sum("d", tile_idx); + auto sum = + frag_meta[f]->loaded_metadata()->get_tile_sum("d", tile_idx); CHECK(*(int64_t*)sum == correct_sum); // Validate the tile metadata structure. @@ -903,7 +912,7 @@ struct CPPVarTileMetadataFx { // Min/max/sum for all null tile are invalid. if (!all_null) { // Validate min. - auto& min = frag_meta[f]->get_min("a"); + auto& min = frag_meta[f]->loaded_metadata()->get_min("a"); CHECK(min.size() == strings_[correct_mins_[f]].size()); CHECK( 0 == strncmp( @@ -912,7 +921,7 @@ struct CPPVarTileMetadataFx { strings_[correct_mins_[f]].size())); // Validate max. - auto& max = frag_meta[f]->get_max("a"); + auto& max = frag_meta[f]->loaded_metadata()->get_max("a"); CHECK(max.size() == strings_[correct_maxs_[f]].size()); CHECK( 0 == strncmp( @@ -922,18 +931,18 @@ struct CPPVarTileMetadataFx { // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_sum("a"), + frag_meta[f]->loaded_metadata()->get_sum("a"), "FragmentMetadata: Trying to access fragment sum metadata that's " "not present"); } // Check null count. if (nullable) { - auto nc = frag_meta[f]->get_null_count("a"); + auto nc = frag_meta[f]->loaded_metadata()->get_null_count("a"); CHECK(nc == correct_null_counts_[f]); } else { CHECK_THROWS_WITH( - frag_meta[f]->get_null_count("a"), + frag_meta[f]->loaded_metadata()->get_null_count("a"), "FragmentMetadata: Trying to access fragment null count metadata " "that's not present"); } @@ -941,10 +950,11 @@ struct CPPVarTileMetadataFx { // Load attribute metadata. std::vector names{"a"}; - frag_meta[f]->load_tile_min_values(enc_key, names); - frag_meta[f]->load_tile_max_values(enc_key, names); - frag_meta[f]->load_tile_sum_values(enc_key, names); - frag_meta[f]->load_tile_null_count_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[f]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); // Validate attribute metadata. // Min/max/sum for all null tile are invalid. @@ -974,7 +984,7 @@ struct CPPVarTileMetadataFx { // Validate no sum. CHECK_THROWS_WITH( - frag_meta[f]->get_tile_sum("a", tile_idx), + frag_meta[f]->loaded_metadata()->get_tile_sum("a", tile_idx), "FragmentMetadata: Trying to access tile sum metadata that's not " "present"); @@ -988,11 +998,12 @@ struct CPPVarTileMetadataFx { // Check null count. for (uint64_t tile_idx = 0; tile_idx < num_tiles_; tile_idx++) { if (nullable) { - auto nc = frag_meta[f]->get_tile_null_count("a", tile_idx); + auto nc = + frag_meta[f]->loaded_metadata()->get_tile_null_count("a", tile_idx); CHECK(nc == correct_tile_null_counts_[f][tile_idx]); } else { CHECK_THROWS_WITH( - frag_meta[f]->get_tile_null_count("a", tile_idx), + frag_meta[f]->loaded_metadata()->get_tile_null_count("a", tile_idx), "FragmentMetadata: Trying to access tile null count metadata " "that's not present"); } @@ -1157,36 +1168,38 @@ struct CPPFixedTileMetadataPartialFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); + frag_meta[0]->loaded_metadata()->load_fragment_min_max_sum_null_count( + enc_key); // Do fragment metadata first for attribute. { // Validate min. - auto& min = frag_meta[0]->get_min("a"); + auto& min = frag_meta[0]->loaded_metadata()->get_min("a"); CHECK(min.size() == sizeof(double)); double correct_min = 1.1; CHECK(0 == memcmp(min.data(), &correct_min, min.size())); // Validate max. - auto& max = frag_meta[0]->get_max("a"); + auto& max = frag_meta[0]->loaded_metadata()->get_max("a"); CHECK(max.size() == sizeof(double)); double correct_max = 4.9; CHECK(0 == memcmp(max.data(), &correct_max, max.size())); // Validate sum. - auto sum = frag_meta[0]->get_sum("a"); + auto sum = frag_meta[0]->loaded_metadata()->get_sum("a"); double correct_sum = 46.7; CHECK(*(double*)sum - correct_sum < 0.0001); } // Load attribute metadata. std::vector names{"a"}; - frag_meta[0]->load_tile_min_values(enc_key, names); - frag_meta[0]->load_tile_max_values(enc_key, names); - frag_meta[0]->load_tile_sum_values(enc_key, names); - frag_meta[0]->load_tile_null_count_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); std::vector correct_tile_mins{1.1, 2.1, 3.2, 4.1}; std::vector correct_tile_maxs{1.7, 2.6, 3.8, 4.9}; @@ -1203,7 +1216,7 @@ struct CPPFixedTileMetadataPartialFx { CHECK(0 == memcmp(&max, &correct_tile_maxs[tile_idx], sizeof(double))); // Validate sum. - auto sum = frag_meta[0]->get_tile_sum("a", tile_idx); + auto sum = frag_meta[0]->loaded_metadata()->get_tile_sum("a", tile_idx); CHECK(*(double*)sum - correct_tile_sums[tile_idx] < 0.0001); // Validate the tile metadata structure. @@ -1335,18 +1348,19 @@ struct CPPVarTileMetadataPartialFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); + frag_meta[0]->loaded_metadata()->load_fragment_min_max_sum_null_count( + enc_key); // Do fragment metadata first for attribute. { // Validate min. - auto& min = frag_meta[0]->get_min("a"); + auto& min = frag_meta[0]->loaded_metadata()->get_min("a"); std::string correct_min = "1.1"; CHECK(min.size() == correct_min.size()); CHECK(0 == memcmp(min.data(), correct_min.data(), min.size())); // Validate max. - auto& max = frag_meta[0]->get_max("a"); + auto& max = frag_meta[0]->loaded_metadata()->get_max("a"); std::string correct_max = "4.9"; CHECK(max.size() == correct_max.size()); CHECK(0 == memcmp(max.data(), correct_max.data(), max.size())); @@ -1354,10 +1368,11 @@ struct CPPVarTileMetadataPartialFx { // Load attribute metadata. std::vector names{"a"}; - frag_meta[0]->load_tile_min_values(enc_key, names); - frag_meta[0]->load_tile_max_values(enc_key, names); - frag_meta[0]->load_tile_sum_values(enc_key, names); - frag_meta[0]->load_tile_null_count_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); std::vector correct_tile_mins{"1.1", "2.1", "3.2", "4.1"}; std::vector correct_tile_maxs{"1.7", "2.6", "3.8", "4.9"}; @@ -1501,48 +1516,50 @@ struct CPPTileMetadataStringDimFx { // Load fragment metadata. auto frag_meta = array->array_->fragment_metadata(); auto& enc_key = array->array_->get_encryption_key(); - frag_meta[0]->load_fragment_min_max_sum_null_count(enc_key); + frag_meta[0]->loaded_metadata()->load_fragment_min_max_sum_null_count( + enc_key); // Do fragment metadata first. { // Validate mins. - auto& min = frag_meta[0]->get_min("a"); + auto& min = frag_meta[0]->loaded_metadata()->get_min("a"); CHECK(min.size() == sizeof(double)); CHECK(*static_cast(static_cast(min.data())) == 4); CHECK_THROWS_WITH( - frag_meta[0]->get_min("d1"), + frag_meta[0]->loaded_metadata()->get_min("d1"), "FragmentMetadata: Trying to access fragment min metadata that's " "not present"); CHECK_THROWS_WITH( - frag_meta[0]->get_min("d2"), + frag_meta[0]->loaded_metadata()->get_min("d2"), "FragmentMetadata: Trying to access fragment min metadata that's " "not present"); // Validate maxs. - auto& max = frag_meta[0]->get_max("a"); + auto& max = frag_meta[0]->loaded_metadata()->get_max("a"); CHECK(max.size() == sizeof(double)); CHECK(*static_cast(static_cast(max.data())) == 7); CHECK_THROWS_WITH( - frag_meta[0]->get_max("d1"), + frag_meta[0]->loaded_metadata()->get_max("d1"), "FragmentMetadata: Trying to access fragment max metadata that's " "not present"); CHECK_THROWS_WITH( - frag_meta[0]->get_max("d2"), + frag_meta[0]->loaded_metadata()->get_max("d2"), "FragmentMetadata: Trying to access fragment max metadata that's " "not present"); } // Load metadata. std::vector names{"a", "d1", "d2"}; - frag_meta[0]->load_rtree(enc_key); - frag_meta[0]->load_tile_min_values(enc_key, names); - frag_meta[0]->load_tile_max_values(enc_key, names); - frag_meta[0]->load_tile_sum_values(enc_key, names); - frag_meta[0]->load_tile_null_count_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_rtree(enc_key); + frag_meta[0]->loaded_metadata()->load_tile_min_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_max_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_sum_values(enc_key, names); + frag_meta[0]->loaded_metadata()->load_tile_null_count_values( + enc_key, names); // Validate min. CHECK(frag_meta[0]->get_tile_min_as("a", 0) == 4); @@ -1567,7 +1584,8 @@ struct CPPTileMetadataStringDimFx { "present"); // Validate sum. - CHECK(*(double*)frag_meta[0]->get_tile_sum("a", 0) == 22); + CHECK( + *(double*)frag_meta[0]->loaded_metadata()->get_tile_sum("a", 0) == 22); // Validate the tile metadata structure. auto full_tile_data_a = frag_meta[0]->get_tile_metadata("a", 0); diff --git a/test/support/src/helpers.cc b/test/support/src/helpers.cc index 2c5561b29b60..fff8e7340d14 100644 --- a/test/support/src/helpers.cc +++ b/test/support/src/helpers.cc @@ -56,6 +56,7 @@ #include "tiledb/sm/misc/tile_overlap.h" #include "tiledb/sm/serialization/array.h" #include "tiledb/sm/serialization/query.h" +#include "tiledb/storage_format/uri/generate_uri.h" int setenv_local(const char* __name, const char* __value) { #ifdef _WIN32 @@ -1456,6 +1457,21 @@ int deserialize_array_and_query( ctx, c_buff, TILEDB_CAPNP, clientside ? 1 : 0, array_uri, query, &array); } +sm::URI generate_fragment_uri(sm::Array* array) { + if (array == nullptr) { + return sm::URI(tiledb::storage_format::generate_timestamped_name( + TILEDB_TIMESTAMP_NOW_MS, constants::format_version)); + } + + uint64_t timestamp = array->timestamp_end_opened_at(); + auto write_version = array->array_schema_latest().write_version(); + + auto new_fragment_str = tiledb::storage_format::generate_timestamped_name( + timestamp, write_version); + auto frag_dir_uri = array->array_directory().get_fragments_dir(write_version); + return frag_dir_uri.join_path(new_fragment_str); +} + template void check_subarray( tiledb::sm::Subarray& subarray, const SubarrayRanges& ranges); template void check_subarray( diff --git a/test/support/src/helpers.h b/test/support/src/helpers.h index ab393af9e023..3f25d72ee70f 100644 --- a/test/support/src/helpers.h +++ b/test/support/src/helpers.h @@ -917,6 +917,18 @@ int deserialize_array_and_query( const char* array_uri, bool clientside); +/** + * Helper function that generates a fragment uri for an array. + * If the `array` arg is specified, the full URI is generated using + * the write version of the array schema and the timestamp + * the array was opened at. If unspecified, the function + * will generate only the fragment name using the current timestamp + * and the latest library format version. + * + * @param array A TileDB array + * @return A test fragment uri + */ +sm::URI generate_fragment_uri(sm::Array* array); } // namespace tiledb::test #endif diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index f09dd69c1bf3..83fa0454a265 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -217,6 +217,9 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_identifier.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_info.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/fragment_metadata.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/loaded_fragment_metadata.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/ondemand_fragment_metadata.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/global_state/global_state.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/global_state/signal_handlers.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/global_state/watchdog.cc @@ -1041,4 +1044,4 @@ configure_file( install(FILES ${CMAKE_CURRENT_BINARY_DIR}/tiledb.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) -include(${CMAKE_SOURCE_DIR}/cmake/package.cmake) \ No newline at end of file +include(${CMAKE_SOURCE_DIR}/cmake/package.cmake) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 4c9e4494ae1a..52822f54537e 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -1478,7 +1478,7 @@ std::unordered_map Array::get_average_var_cell_sizes() return Status::Ok(); } - fragment_metadata[f]->load_tile_var_sizes( + fragment_metadata[f]->loaded_metadata()->load_tile_var_sizes( *encryption_key(), var_name); return Status::Ok(); })); @@ -1502,7 +1502,9 @@ std::unordered_map Array::get_average_var_cell_sizes() // Go through all tiles. for (uint64_t t = 0; t < fragment_metadata[f]->tile_num(); t++) { - total_size += fragment_metadata[f]->tile_var_size(var_name, t); + total_size += + fragment_metadata[f]->loaded_metadata()->tile_var_size( + var_name, t); cell_num += fragment_metadata[f]->cell_num(t); } } diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index e83d42687a20..a886a9ade66e 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -474,7 +474,7 @@ Status FragmentInfo::get_mbr_num(uint32_t fid, uint64_t* mbr_num) { } auto meta = single_fragment_info_vec_[fid].meta(); - meta->load_rtree(enc_key_); + meta->loaded_metadata()->load_rtree(enc_key_); *mbr_num = meta->mbrs().size(); return Status::Ok(); @@ -496,7 +496,7 @@ Status FragmentInfo::get_mbr( Status_FragmentInfoError("Cannot get MBR; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - meta->load_rtree(enc_key_); + meta->loaded_metadata()->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -578,7 +578,7 @@ Status FragmentInfo::get_mbr_var_size( Status_FragmentInfoError("Cannot get MBR; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - meta->load_rtree(enc_key_); + meta->loaded_metadata()->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -658,7 +658,7 @@ Status FragmentInfo::get_mbr_var( Status_FragmentInfoError("Cannot get MBR var; Fragment is not sparse")); auto meta = single_fragment_info_vec_[fid].meta(); - meta->load_rtree(enc_key_); + meta->loaded_metadata()->load_rtree(enc_key_); const auto& mbrs = meta->mbrs(); if (mid >= mbrs.size()) @@ -881,7 +881,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { } if (preload_rtrees & !meta->dense()) { - meta->load_rtree(enc_key_); + meta->loaded_metadata()->load_rtree(enc_key_); } return Status::Ok(); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index fa2f2691911c..4a8a603170e7 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -44,6 +44,8 @@ #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/fragment/fragment_identifier.h" #include "tiledb/sm/fragment/fragment_metadata.h" +#include "tiledb/sm/fragment/ondemand_fragment_metadata.h" +#include "tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/utils.h" @@ -66,36 +68,20 @@ using namespace tiledb::type; namespace tiledb::sm { -class FragmentMetadataStatusException : public StatusException { - public: - explicit FragmentMetadataStatusException(const std::string& message) - : StatusException("FragmentMetadata", message) { - } -}; - /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ FragmentMetadata::FragmentMetadata( - ContextResources* resources, shared_ptr memory_tracker) + ContextResources* resources, + shared_ptr memory_tracker, + format_version_t version) : resources_(resources) , memory_tracker_(memory_tracker) - , rtree_(RTree(nullptr, constants::rtree_fanout, memory_tracker_)) - , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_validity_offsets_( - memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_min_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) - , tile_min_var_buffer_( - memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) - , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) - , tile_max_var_buffer_( - memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) - , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) - , tile_null_counts_( - memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) { + , version_(version) + , loaded_metadata_( + LoadedFragmentMetadata::create(*this, memory_tracker, version_)) + , loaded_metadata_ptr_(loaded_metadata_.get()) { } FragmentMetadata::FragmentMetadata( @@ -120,32 +106,17 @@ FragmentMetadata::FragmentMetadata( , has_delete_meta_(has_deletes_meta) , sparse_tile_num_(0) , meta_file_size_(0) - , rtree_(RTree( - &array_schema_->domain(), constants::rtree_fanout, memory_tracker_)) , tile_index_base_(0) - , tile_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_validity_offsets_( - memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) - , tile_min_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) - , tile_min_var_buffer_( - memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) - , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) - , tile_max_var_buffer_( - memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) - , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) - , tile_null_counts_( - memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) - , version_(array_schema_->write_version()) + , version_(FragmentID{fragment_uri}.array_format_version()) , timestamp_range_(timestamp_range) - , array_uri_(array_schema_->array_uri()) { + , array_uri_(array_schema_->array_uri()) + , loaded_metadata_( + LoadedFragmentMetadata::create(*this, memory_tracker, version_)) + , loaded_metadata_ptr_(loaded_metadata_.get()) { build_idx_map(); array_schema_name_ = array_schema_->name(); } -FragmentMetadata::~FragmentMetadata() = default; - /* ****************************** */ /* API */ /* ****************************** */ @@ -153,7 +124,7 @@ FragmentMetadata::~FragmentMetadata() = default; void FragmentMetadata::set_mbr(uint64_t tile, const NDRange& mbr) { // For easy reference tile += tile_index_base_; - throw_if_not_ok(rtree_.set_leaf(tile, mbr)); + throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf(tile, mbr)); return expand_non_empty_domain(mbr); } @@ -167,8 +138,8 @@ void FragmentMetadata::set_tile_offset( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid < tile_offsets_[idx].size()); - tile_offsets_[idx][tid] = file_sizes_[idx]; + assert(tid < loaded_metadata_ptr_->tile_offsets()[idx].size()); + loaded_metadata_ptr_->tile_offsets()[idx][tid] = file_sizes_[idx]; file_sizes_[idx] += step; } @@ -178,8 +149,8 @@ void FragmentMetadata::set_tile_var_offset( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid < tile_var_offsets_[idx].size()); - tile_var_offsets_[idx][tid] = file_var_sizes_[idx]; + assert(tid < loaded_metadata_ptr_->tile_var_offsets()[idx].size()); + loaded_metadata_ptr_->tile_var_offsets()[idx][tid] = file_var_sizes_[idx]; file_var_sizes_[idx] += step; } @@ -189,8 +160,8 @@ void FragmentMetadata::set_tile_var_size( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid < tile_var_sizes_[idx].size()); - tile_var_sizes_[idx][tid] = size; + assert(tid < loaded_metadata_ptr_->tile_var_sizes()[idx].size()); + loaded_metadata_ptr_->tile_var_sizes()[idx][tid] = size; } void FragmentMetadata::set_tile_validity_offset( @@ -199,8 +170,9 @@ void FragmentMetadata::set_tile_validity_offset( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid < tile_validity_offsets_[idx].size()); - tile_validity_offsets_[idx][tid] = file_validity_sizes_[idx]; + assert(tid < loaded_metadata_ptr_->tile_validity_offsets()[idx].size()); + loaded_metadata_ptr_->tile_validity_offsets()[idx][tid] = + file_validity_sizes_[idx]; file_validity_sizes_[idx] += step; } @@ -212,8 +184,11 @@ void FragmentMetadata::set_tile_min( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * size; - assert(tid < tile_min_buffer_[idx].size() / size); - memcpy(&tile_min_buffer_[idx][buff_offset], min.data(), size); + assert(tid < loaded_metadata_ptr_->tile_min_buffer()[idx].size() / size); + memcpy( + &loaded_metadata_ptr_->tile_min_buffer()[idx][buff_offset], + min.data(), + size); } void FragmentMetadata::set_tile_min_var_size( @@ -223,9 +198,12 @@ void FragmentMetadata::set_tile_min_var_size( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * sizeof(uint64_t); - assert(tid < tile_min_buffer_[idx].size() / sizeof(uint64_t)); + assert( + tid < + loaded_metadata_ptr_->tile_min_buffer()[idx].size() / sizeof(uint64_t)); - auto offset = (uint64_t*)&tile_min_buffer_[idx][buff_offset]; + auto offset = + (uint64_t*)&loaded_metadata_ptr_->tile_min_buffer()[idx][buff_offset]; *offset = size; } @@ -236,16 +214,24 @@ void FragmentMetadata::set_tile_min_var( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * sizeof(uint64_t); - assert(tid < tile_min_buffer_[idx].size() / sizeof(uint64_t)); - - auto offset = (uint64_t*)&tile_min_buffer_[idx][buff_offset]; - auto size = buff_offset != tile_min_buffer_[idx].size() - sizeof(uint64_t) ? - offset[1] - offset[0] : - tile_min_var_buffer_[idx].size() - offset[0]; + assert( + tid < + loaded_metadata_ptr_->tile_min_buffer()[idx].size() / sizeof(uint64_t)); + + auto offset = + (uint64_t*)&loaded_metadata_ptr_->tile_min_buffer()[idx][buff_offset]; + auto size = + buff_offset != loaded_metadata_ptr_->tile_min_buffer()[idx].size() - + sizeof(uint64_t) ? + offset[1] - offset[0] : + loaded_metadata_ptr_->tile_min_var_buffer()[idx].size() - offset[0]; // Copy var data if (size) { // avoid (potentially) illegal index ref's when size is zero - memcpy(&tile_min_var_buffer_[idx][offset[0]], min.data(), size); + memcpy( + &loaded_metadata_ptr_->tile_min_var_buffer()[idx][offset[0]], + min.data(), + size); } } @@ -257,8 +243,11 @@ void FragmentMetadata::set_tile_max( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * size; - assert(tid < tile_max_buffer_[idx].size() / size); - memcpy(&tile_max_buffer_[idx][buff_offset], max.data(), size); + assert(tid < loaded_metadata_ptr_->tile_max_buffer()[idx].size() / size); + memcpy( + &loaded_metadata_ptr_->tile_max_buffer()[idx][buff_offset], + max.data(), + size); } void FragmentMetadata::set_tile_max_var_size( @@ -268,9 +257,12 @@ void FragmentMetadata::set_tile_max_var_size( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * sizeof(uint64_t); - assert(tid < tile_max_buffer_[idx].size() / sizeof(uint64_t)); + assert( + tid < + loaded_metadata_ptr_->tile_max_buffer()[idx].size() / sizeof(uint64_t)); - auto offset = (uint64_t*)&tile_max_buffer_[idx][buff_offset]; + auto offset = + (uint64_t*)&loaded_metadata_ptr_->tile_max_buffer()[idx][buff_offset]; *offset = size; } @@ -281,16 +273,24 @@ void FragmentMetadata::set_tile_max_var( auto idx = it->second; tid += tile_index_base_; auto buff_offset = tid * sizeof(uint64_t); - assert(tid < tile_max_buffer_[idx].size() / sizeof(uint64_t)); - - auto offset = (uint64_t*)&tile_max_buffer_[idx][buff_offset]; - auto size = buff_offset != tile_max_buffer_[idx].size() - sizeof(uint64_t) ? - offset[1] - offset[0] : - tile_max_var_buffer_[idx].size() - offset[0]; + assert( + tid < + loaded_metadata_ptr_->tile_max_buffer()[idx].size() / sizeof(uint64_t)); + + auto offset = + (uint64_t*)&loaded_metadata_ptr_->tile_max_buffer()[idx][buff_offset]; + auto size = + buff_offset != loaded_metadata_ptr_->tile_max_buffer()[idx].size() - + sizeof(uint64_t) ? + offset[1] - offset[0] : + loaded_metadata_ptr_->tile_max_var_buffer()[idx].size() - offset[0]; // Copy var data if (size) { // avoid (potentially) illegal index ref's when size is zero - memcpy(&tile_max_var_buffer_[idx][offset[0]], max.data(), size); + memcpy( + &loaded_metadata_ptr_->tile_max_var_buffer()[idx][offset[0]], + max.data(), + size); } } @@ -301,10 +301,13 @@ void FragmentMetadata::convert_tile_min_max_var_sizes_to_offsets( auto idx = it->second; // Fix the min offsets. - uint64_t offset = tile_min_var_buffer_[idx].size(); - auto offsets = (uint64_t*)tile_min_buffer_[idx].data() + tile_index_base_; + uint64_t offset = loaded_metadata_ptr_->tile_min_var_buffer()[idx].size(); + auto offsets = + (uint64_t*)loaded_metadata_ptr_->tile_min_buffer()[idx].data() + + tile_index_base_; for (uint64_t i = tile_index_base_; - i < tile_min_buffer_[idx].size() / sizeof(uint64_t); + i < + loaded_metadata_ptr_->tile_min_buffer()[idx].size() / sizeof(uint64_t); i++) { auto size = *offsets; *offsets = offset; @@ -313,13 +316,15 @@ void FragmentMetadata::convert_tile_min_max_var_sizes_to_offsets( } // Allocate min var data buffer. - tile_min_var_buffer_[idx].resize(offset); + loaded_metadata_ptr_->tile_min_var_buffer()[idx].resize(offset); // Fix the max offsets. - offset = tile_max_var_buffer_[idx].size(); - offsets = (uint64_t*)tile_max_buffer_[idx].data() + tile_index_base_; + offset = loaded_metadata_ptr_->tile_max_var_buffer()[idx].size(); + offsets = (uint64_t*)loaded_metadata_ptr_->tile_max_buffer()[idx].data() + + tile_index_base_; for (uint64_t i = tile_index_base_; - i < tile_max_buffer_[idx].size() / sizeof(uint64_t); + i < + loaded_metadata_ptr_->tile_max_buffer()[idx].size() / sizeof(uint64_t); i++) { auto size = *offsets; *offsets = offset; @@ -328,7 +333,7 @@ void FragmentMetadata::convert_tile_min_max_var_sizes_to_offsets( } // Allocate min var data buffer. - tile_max_var_buffer_[idx].resize(offset); + loaded_metadata_ptr_->tile_max_var_buffer()[idx].resize(offset); } void FragmentMetadata::set_tile_sum( @@ -337,9 +342,12 @@ void FragmentMetadata::set_tile_sum( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid * sizeof(uint64_t) < tile_sums_[idx].size()); + assert( + tid * sizeof(uint64_t) < loaded_metadata_ptr_->tile_sums()[idx].size()); memcpy( - &tile_sums_[idx][tid * sizeof(uint64_t)], sum.data(), sizeof(uint64_t)); + &loaded_metadata_ptr_->tile_sums()[idx][tid * sizeof(uint64_t)], + sum.data(), + sizeof(uint64_t)); } void FragmentMetadata::set_tile_null_count( @@ -348,8 +356,8 @@ void FragmentMetadata::set_tile_null_count( assert(it != idx_map_.end()); auto idx = it->second; tid += tile_index_base_; - assert(tid < tile_null_counts_[idx].size()); - tile_null_counts_[idx][tid] = null_count; + assert(tid < loaded_metadata_ptr_->tile_null_counts()[idx].size()); + loaded_metadata_ptr_->tile_null_counts()[idx][tid] = null_count; } template <> @@ -373,8 +381,10 @@ void FragmentMetadata::compute_fragment_min_max_sum_null_count() { const auto type = array_schema_->type(name); // Compute null count. - fragment_null_counts_[idx] = std::accumulate( - tile_null_counts_[idx].begin(), tile_null_counts_[idx].end(), 0); + loaded_metadata_ptr_->fragment_null_counts()[idx] = std::accumulate( + loaded_metadata_ptr_->tile_null_counts()[idx].begin(), + loaded_metadata_ptr_->tile_null_counts()[idx].end(), + 0); if (var_size) { min_max_var(name); @@ -600,7 +610,7 @@ void FragmentMetadata::add_max_buffer_sizes_dense( if (array_schema_->var_size(it.first)) { auto cell_num = this->cell_num(tid); it.second.first += cell_num * constants::cell_var_offset_size; - it.second.second += tile_var_size(it.first, tid); + it.second.second += loaded_metadata_ptr_->tile_var_size(it.first, tid); } else { it.second.first += cell_num(tid) * array_schema_->cell_size(it.first); } @@ -613,11 +623,12 @@ void FragmentMetadata::add_max_buffer_sizes_sparse( const NDRange& subarray, std::unordered_map>* buffer_sizes) { - load_rtree(encryption_key); + loaded_metadata_ptr_->load_rtree(encryption_key); // Get tile overlap std::vector is_default(subarray.size(), false); - auto tile_overlap = rtree_.get_tile_overlap(subarray, is_default); + auto tile_overlap = + loaded_metadata_ptr_->rtree().get_tile_overlap(subarray, is_default); // Handle tile ranges for (const auto& tr : tile_overlap.tile_ranges_) { @@ -626,7 +637,8 @@ void FragmentMetadata::add_max_buffer_sizes_sparse( if (array_schema_->var_size(it.first)) { auto cell_num = this->cell_num(tid); it.second.first += cell_num * constants::cell_var_offset_size; - it.second.second += tile_var_size(it.first, tid); + it.second.second += + loaded_metadata_ptr_->tile_var_size(it.first, tid); } else { it.second.first += cell_num(tid) * array_schema_->cell_size(it.first); } @@ -641,7 +653,7 @@ void FragmentMetadata::add_max_buffer_sizes_sparse( if (array_schema_->var_size(it.first)) { auto cell_num = this->cell_num(tid); it.second.first += cell_num * constants::cell_var_offset_size; - it.second.second += tile_var_size(it.first, tid); + it.second.second += loaded_metadata_ptr_->tile_var_size(it.first, tid); } else { it.second.first += cell_num(tid) * array_schema_->cell_size(it.first); } @@ -677,20 +689,6 @@ uint64_t FragmentMetadata::fragment_size() const { return size; } -void FragmentMetadata::get_tile_overlap( - const NDRange& range, - std::vector& is_default, - TileOverlap* tile_overlap) { - assert(version_ <= 2 || loaded_metadata_.rtree_); - *tile_overlap = rtree_.get_tile_overlap(range, is_default); -} - -void FragmentMetadata::compute_tile_bitmap( - const Range& range, unsigned d, std::vector* tile_bitmap) { - assert(version_ <= 2 || loaded_metadata_.rtree_); - rtree_.compute_tile_bitmap(range, d, tile_bitmap); -} - void FragmentMetadata::init_domain(const NDRange& non_empty_domain) { auto& domain{array_schema_->domain()}; @@ -726,41 +724,20 @@ void FragmentMetadata::init(const NDRange& non_empty_domain) { last_tile_cell_num_ = 0; // Initialize tile offsets - tile_offsets_.resize(num); - tile_offsets_mtx_.resize(num); + loaded_metadata_ptr_->resize_offsets(num); + file_sizes_.resize(num); for (unsigned int i = 0; i < num; ++i) file_sizes_[i] = 0; - // Initialize variable tile offsets - tile_var_offsets_.resize(num); - tile_var_offsets_mtx_.resize(num); file_var_sizes_.resize(num); for (unsigned int i = 0; i < num; ++i) file_var_sizes_[i] = 0; - // Initialize variable tile sizes - tile_var_sizes_.resize(num); - // Initialize validity tile offsets - tile_validity_offsets_.resize(num); file_validity_sizes_.resize(num); for (unsigned int i = 0; i < num; ++i) file_validity_sizes_[i] = 0; - - // Initialize tile min/max/sum/null count - tile_min_buffer_.resize(num); - tile_min_var_buffer_.resize(num); - tile_max_buffer_.resize(num); - tile_max_var_buffer_.resize(num); - tile_sums_.resize(num); - tile_null_counts_.resize(num); - - // Initialize fragment min/max/sum/null count - fragment_mins_.resize(num); - fragment_maxs_.resize(num); - fragment_sums_.resize(num); - fragment_null_counts_.resize(num); } std::vector> FragmentMetadata::load( @@ -1210,7 +1187,7 @@ void FragmentMetadata::store_v15_or_higher( void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { for (auto& it : idx_map_) { auto i = it.second; - assert(num_tiles >= tile_offsets_[i].size()); + assert(num_tiles >= loaded_metadata_ptr_->tile_offsets()[i].size()); // Get the fixed cell size const auto is_dim = array_schema_->is_dim(it.first); @@ -1218,10 +1195,10 @@ void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { const auto cell_size = var_size ? constants::cell_var_offset_size : array_schema_->cell_size(it.first); - tile_offsets_[i].resize(num_tiles, 0); - tile_var_offsets_[i].resize(num_tiles, 0); - tile_var_sizes_[i].resize(num_tiles, 0); - tile_validity_offsets_[i].resize(num_tiles, 0); + loaded_metadata_ptr_->tile_offsets()[i].resize(num_tiles, 0); + loaded_metadata_ptr_->tile_var_offsets()[i].resize(num_tiles, 0); + loaded_metadata_ptr_->tile_var_sizes()[i].resize(num_tiles, 0); + loaded_metadata_ptr_->tile_validity_offsets()[i].resize(num_tiles, 0); // No metadata for dense coords if (!array_schema_->dense() || !is_dim) { @@ -1230,23 +1207,26 @@ void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { if (TileMetadataGenerator::has_min_max_metadata( type, is_dim, var_size, cell_val_num)) { - tile_min_buffer_[i].resize(num_tiles * cell_size, 0); - tile_max_buffer_[i].resize(num_tiles * cell_size, 0); + loaded_metadata_ptr_->tile_min_buffer()[i].resize( + num_tiles * cell_size, 0); + loaded_metadata_ptr_->tile_max_buffer()[i].resize( + num_tiles * cell_size, 0); } if (TileMetadataGenerator::has_sum_metadata( type, var_size, cell_val_num)) { if (!var_size) - tile_sums_[i].resize(num_tiles * sizeof(uint64_t), 0); + loaded_metadata_ptr_->tile_sums()[i].resize( + num_tiles * sizeof(uint64_t), 0); } if (array_schema_->is_nullable(it.first)) - tile_null_counts_[i].resize(num_tiles, 0); + loaded_metadata_ptr_->tile_null_counts()[i].resize(num_tiles, 0); } } if (!dense_) { - throw_if_not_ok(rtree_.set_leaf_num(num_tiles)); + throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf_num(num_tiles)); sparse_tile_num_ = num_tiles; } } @@ -1373,274 +1353,12 @@ const std::string& FragmentMetadata::array_schema_name() { return array_schema_name_; } -void FragmentMetadata::load_tile_offsets( - const EncryptionKey& encryption_key, std::vector& names) { - // Sort 'names' in ascending order of their index. The - // motivation is to load the offsets in order of their - // layout for sequential reads to the file. - std::sort( - names.begin(), - names.end(), - [&](const std::string& lhs, const std::string& rhs) { - assert(idx_map_.count(lhs) > 0); - assert(idx_map_.count(rhs) > 0); - return idx_map_[lhs] < idx_map_[rhs]; - }); - - // The fixed offsets are located before the - // var offsets. Load all of the fixed offsets - // first. - for (const auto& name : names) { - load_tile_offsets(encryption_key, idx_map_[name]); - } - - // Load all of the var offsets. - for (const auto& name : names) { - if (array_schema_->var_size(name)) { - load_tile_var_offsets(encryption_key, idx_map_[name]); - } - } - - // Load all of the var offsets. - for (const auto& name : names) { - if (array_schema_->is_nullable(name)) { - load_tile_validity_offsets(encryption_key, idx_map_[name]); - } - } -} - -void FragmentMetadata::load_tile_min_values( - const EncryptionKey& encryption_key, std::vector& names) { - // Sort 'names' in ascending order of their index. The - // motivation is to load the offsets in order of their - // layout for sequential reads to the file. - std::sort( - names.begin(), - names.end(), - [&](const std::string& lhs, const std::string& rhs) { - assert(idx_map_.count(lhs) > 0); - assert(idx_map_.count(rhs) > 0); - return idx_map_[lhs] < idx_map_[rhs]; - }); - - // Load all the min values. - for (const auto& name : names) { - load_tile_min_values(encryption_key, idx_map_[name]); - } -} - -void FragmentMetadata::load_tile_max_values( - const EncryptionKey& encryption_key, std::vector& names) { - // Sort 'names' in ascending order of their index. The - // motivation is to load the offsets in order of their - // layout for sequential reads to the file. - std::sort( - names.begin(), - names.end(), - [&](const std::string& lhs, const std::string& rhs) { - assert(idx_map_.count(lhs) > 0); - assert(idx_map_.count(rhs) > 0); - return idx_map_[lhs] < idx_map_[rhs]; - }); - - // Load all the max values. - for (const auto& name : names) { - load_tile_max_values(encryption_key, idx_map_[name]); - } -} - -void FragmentMetadata::load_tile_sum_values( - const EncryptionKey& encryption_key, std::vector& names) { - // Sort 'names' in ascending order of their index. The - // motivation is to load the offsets in order of their - // layout for sequential reads to the file. - std::sort( - names.begin(), - names.end(), - [&](const std::string& lhs, const std::string& rhs) { - assert(idx_map_.count(lhs) > 0); - assert(idx_map_.count(rhs) > 0); - return idx_map_[lhs] < idx_map_[rhs]; - }); - - // Load all the sum values. - for (const auto& name : names) { - load_tile_sum_values(encryption_key, idx_map_[name]); - } -} - -void FragmentMetadata::load_tile_null_count_values( - const EncryptionKey& encryption_key, std::vector& names) { - // Sort 'names' in ascending order of their index. The - // motivation is to load the offsets in order of their - // layout for sequential reads to the file. - std::sort( - names.begin(), - names.end(), - [&](const std::string& lhs, const std::string& rhs) { - assert(idx_map_.count(lhs) > 0); - assert(idx_map_.count(rhs) > 0); - return idx_map_[lhs] < idx_map_[rhs]; - }); - - // Load all the null count values. - for (const auto& name : names) { - load_tile_null_count_values(encryption_key, idx_map_[name]); - } -} - -void FragmentMetadata::load_fragment_min_max_sum_null_count( - const EncryptionKey& encryption_key) { - if (loaded_metadata_.fragment_min_max_sum_null_count_) { - return; - } - - if (version_ <= 11) { - return; - } - - std::lock_guard lock(mtx_); - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.fragment_min_max_sum_null_count_offset_); - resources_->stats().add_counter( - "read_fragment_min_max_sum_null_count_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_fragment_min_max_sum_null_count(deserializer); - - loaded_metadata_.fragment_min_max_sum_null_count_ = true; -} - -void FragmentMetadata::load_processed_conditions( - const EncryptionKey& encryption_key) { - if (loaded_metadata_.processed_conditions_) { - return; - } - - if (version_ <= 15) { - return; - } - - std::lock_guard lock(mtx_); - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.processed_conditions_offsets_); - resources_->stats().add_counter( - "read_processed_conditions_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_processed_conditions(deserializer); - - loaded_metadata_.processed_conditions_ = true; -} - -uint64_t FragmentMetadata::file_offset( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_offsets_[idx]) { - throw std::logic_error( - "Trying to access tile offsets metadata that's not loaded"); - } - - return tile_offsets_[idx][tile_idx]; -} - -uint64_t FragmentMetadata::file_var_offset( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_var_offsets_[idx]) { - throw std::logic_error( - "Trying to access tile var offsets metadata that's not loaded"); - } - - return tile_var_offsets_[idx][tile_idx]; -} - -uint64_t FragmentMetadata::file_validity_offset( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_validity_offsets_[idx]) { - throw std::logic_error( - "Trying to access tile validity offsets metadata that's not loaded"); - } - - return tile_validity_offsets_[idx][tile_idx]; -} - const NDRange& FragmentMetadata::mbr(uint64_t tile_idx) const { - return rtree_.leaf(tile_idx); + return loaded_metadata_ptr_->rtree().leaf(tile_idx); } const tdb::pmr::vector& FragmentMetadata::mbrs() const { - return rtree_.leaves(); -} - -uint64_t FragmentMetadata::persisted_tile_size( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_offsets_[idx]) { - throw std::logic_error( - "Trying to access persisted tile offsets metadata that's not present"); - } - - auto tile_num = this->tile_num(); - - auto tile_size = - (tile_idx != tile_num - 1) ? - tile_offsets_[idx][tile_idx + 1] - tile_offsets_[idx][tile_idx] : - file_sizes_[idx] - tile_offsets_[idx][tile_idx]; - return tile_size; -} - -uint64_t FragmentMetadata::persisted_tile_var_size( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - - if (!loaded_metadata_.tile_var_offsets_[idx]) { - throw std::logic_error( - "Trying to access persisted tile var offsets metadata that's not " - "present"); - } - - auto tile_num = this->tile_num(); - - auto tile_size = (tile_idx != tile_num - 1) ? - tile_var_offsets_[idx][tile_idx + 1] - - tile_var_offsets_[idx][tile_idx] : - file_var_sizes_[idx] - tile_var_offsets_[idx][tile_idx]; - return tile_size; -} - -uint64_t FragmentMetadata::persisted_tile_validity_size( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_validity_offsets_[idx]) { - throw std::logic_error( - "Trying to access persisted tile validity offsets metadata that's not " - "present"); - } - - auto tile_num = this->tile_num(); - - auto tile_size = - (tile_idx != tile_num - 1) ? - tile_validity_offsets_[idx][tile_idx + 1] - - tile_validity_offsets_[idx][tile_idx] : - file_validity_sizes_[idx] - tile_validity_offsets_[idx][tile_idx]; - return tile_size; + return loaded_metadata_ptr_->rtree().leaves(); } uint64_t FragmentMetadata::tile_size( @@ -1651,19 +1369,6 @@ uint64_t FragmentMetadata::tile_size( cell_num * array_schema_->cell_size(name); } -uint64_t FragmentMetadata::tile_var_size( - const std::string& name, uint64_t tile_idx) { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_var_sizes_[idx]) { - throw FragmentMetadataStatusException( - "Trying to access tile var size metadata that's not loaded"); - } - - return tile_var_sizes_[idx][tile_idx]; -} - template T FragmentMetadata::get_tile_min_as( const std::string& name, uint64_t tile_idx) const { @@ -1676,7 +1381,7 @@ T FragmentMetadata::get_tile_min_as( auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; - if (!loaded_metadata_.tile_min_[idx]) { + if (!loaded_metadata_ptr_->loaded_metadata().tile_min_[idx]) { throw FragmentMetadataStatusException( "Trying to access tile min metadata that's not loaded"); } @@ -1691,7 +1396,8 @@ T FragmentMetadata::get_tile_min_as( } auto size = array_schema_->cell_size(name); - const void* min = &tile_min_buffer_[idx][tile_idx * size]; + const void* min = + &loaded_metadata_ptr_->tile_min_buffer()[idx][tile_idx * size]; if constexpr (std::is_same_v) { return min; } else { @@ -1712,7 +1418,7 @@ std::string_view FragmentMetadata::get_tile_min_as( auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; - if (!loaded_metadata_.tile_min_[idx]) { + if (!loaded_metadata_ptr_->loaded_metadata().tile_min_[idx]) { throw FragmentMetadataStatusException( "Trying to access tile min metadata that's not loaded"); } @@ -1728,22 +1434,26 @@ std::string_view FragmentMetadata::get_tile_min_as( using sv_size_cast = std::string_view::size_type; if (var_size) { auto tile_num = this->tile_num(); - auto offsets = (uint64_t*)tile_min_buffer_[idx].data(); + auto offsets = + (uint64_t*)loaded_metadata_ptr_->tile_min_buffer()[idx].data(); auto min_offset = offsets[tile_idx]; auto size = tile_idx == tile_num - 1 ? static_cast( - tile_min_var_buffer_[idx].size() - min_offset) : + loaded_metadata_ptr_->tile_min_var_buffer()[idx].size() - + min_offset) : static_cast(offsets[tile_idx + 1] - min_offset); if (size == 0) { return {}; } - const char* min = &tile_min_var_buffer_[idx][min_offset]; + const char* min = + &loaded_metadata_ptr_->tile_min_var_buffer()[idx][min_offset]; return {min, size}; } else { auto size = static_cast(array_schema_->cell_size(name)); - const void* min = &tile_min_buffer_[idx][tile_idx * size]; + const void* min = + &loaded_metadata_ptr_->tile_min_buffer()[idx][tile_idx * size]; return {static_cast(min), size}; } } @@ -1760,7 +1470,7 @@ T FragmentMetadata::get_tile_max_as( auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; - if (!loaded_metadata_.tile_max_[idx]) { + if (!loaded_metadata_ptr_->loaded_metadata().tile_max_[idx]) { throw FragmentMetadataStatusException( "Trying to access tile max metadata that's not loaded"); } @@ -1775,7 +1485,8 @@ T FragmentMetadata::get_tile_max_as( } auto size = array_schema_->cell_size(name); - const void* max = &tile_max_buffer_[idx][tile_idx * size]; + const void* max = + &loaded_metadata_ptr_->tile_max_buffer()[idx][tile_idx * size]; if constexpr (std::is_same_v) { return max; } else { @@ -1796,7 +1507,7 @@ std::string_view FragmentMetadata::get_tile_max_as( auto it = idx_map_.find(name); assert(it != idx_map_.end()); auto idx = it->second; - if (!loaded_metadata_.tile_max_[idx]) { + if (!loaded_metadata_ptr_->loaded_metadata().tile_max_[idx]) { throw FragmentMetadataStatusException( "Trying to access tile max metadata that's not loaded"); } @@ -1812,147 +1523,30 @@ std::string_view FragmentMetadata::get_tile_max_as( using sv_size_cast = std::string_view::size_type; if (var_size) { auto tile_num = this->tile_num(); - auto offsets = (uint64_t*)tile_max_buffer_[idx].data(); + auto offsets = + (uint64_t*)loaded_metadata_ptr_->tile_max_buffer()[idx].data(); auto max_offset = offsets[tile_idx]; auto size = tile_idx == tile_num - 1 ? static_cast( - tile_max_var_buffer_[idx].size() - max_offset) : + loaded_metadata_ptr_->tile_max_var_buffer()[idx].size() - + max_offset) : static_cast(offsets[tile_idx + 1] - max_offset); if (size == 0) { return {}; } - const char* max = &tile_max_var_buffer_[idx][max_offset]; + const char* max = + &loaded_metadata_ptr_->tile_max_var_buffer()[idx][max_offset]; return {max, size}; } else { auto size = static_cast(array_schema_->cell_size(name)); - const void* max = &tile_max_buffer_[idx][tile_idx * size]; + const void* max = + &loaded_metadata_ptr_->tile_max_buffer()[idx][tile_idx * size]; return {static_cast(max), size}; } } -const void* FragmentMetadata::get_tile_sum( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_sum_[idx]) { - throw FragmentMetadataStatusException( - "Trying to access tile sum metadata that's not loaded"); - } - - auto type = array_schema_->type(name); - auto var_size = array_schema_->var_size(name); - auto cell_val_num = array_schema_->cell_val_num(name); - if (!TileMetadataGenerator::has_sum_metadata(type, var_size, cell_val_num)) { - throw FragmentMetadataStatusException( - "Trying to access tile sum metadata that's not present"); - } - - const void* sum = &tile_sums_[idx][tile_idx * sizeof(uint64_t)]; - return sum; -} - -uint64_t FragmentMetadata::get_tile_null_count( - const std::string& name, uint64_t tile_idx) const { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.tile_null_count_[idx]) { - throw FragmentMetadataStatusException( - "Trying to access tile null count metadata that's not loaded"); - } - - if (!array_schema_->is_nullable(name)) { - throw FragmentMetadataStatusException( - "Trying to access tile null count metadata that's not present"); - } - - return tile_null_counts_[idx][tile_idx]; -} - -std::vector& FragmentMetadata::get_min(const std::string& name) { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.fragment_min_max_sum_null_count_) { - throw FragmentMetadataStatusException( - "Trying to access fragment min metadata that's not loaded"); - } - - const auto type = array_schema_->type(name); - const auto is_dim = array_schema_->is_dim(name); - const auto var_size = array_schema_->var_size(name); - const auto cell_val_num = array_schema_->cell_val_num(name); - if (!TileMetadataGenerator::has_min_max_metadata( - type, is_dim, var_size, cell_val_num)) { - throw FragmentMetadataStatusException( - "Trying to access fragment min metadata that's not present"); - } - - return fragment_mins_[idx]; -} - -std::vector& FragmentMetadata::get_max(const std::string& name) { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.fragment_min_max_sum_null_count_) { - throw FragmentMetadataStatusException( - "Trying to access fragment max metadata that's not loaded"); - } - - const auto type = array_schema_->type(name); - const auto is_dim = array_schema_->is_dim(name); - const auto var_size = array_schema_->var_size(name); - const auto cell_val_num = array_schema_->cell_val_num(name); - if (!TileMetadataGenerator::has_min_max_metadata( - type, is_dim, var_size, cell_val_num)) { - throw FragmentMetadataStatusException( - "Trying to access fragment max metadata that's not present"); - } - - return fragment_maxs_[idx]; -} - -void* FragmentMetadata::get_sum(const std::string& name) { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.fragment_min_max_sum_null_count_) { - throw FragmentMetadataStatusException( - "Trying to access fragment sum metadata that's not loaded"); - } - - const auto type = array_schema_->type(name); - const auto var_size = array_schema_->var_size(name); - const auto cell_val_num = array_schema_->cell_val_num(name); - if (!TileMetadataGenerator::has_sum_metadata(type, var_size, cell_val_num)) { - throw FragmentMetadataStatusException( - "Trying to access fragment sum metadata that's not present"); - } - - return &fragment_sums_[idx]; -} - -uint64_t FragmentMetadata::get_null_count(const std::string& name) { - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - if (!loaded_metadata_.fragment_min_max_sum_null_count_) { - throw FragmentMetadataStatusException( - "Trying to access fragment null count metadata that's not loaded"); - } - - if (!array_schema_->is_nullable(name)) { - throw FragmentMetadataStatusException( - "Trying to access fragment null count metadata that's not present"); - } - - return fragment_null_counts_[idx]; -} - TileMetadata FragmentMetadata::get_tile_metadata( const std::string& name, const uint64_t tile_idx) const { auto var_size = array_schema_->var_size(name); @@ -1965,14 +1559,14 @@ TileMetadata FragmentMetadata::get_tile_metadata( uint64_t null_count = 0; if (array_schema_->is_nullable(name)) { - null_count = get_tile_null_count(name, tile_idx); + null_count = loaded_metadata_ptr_->get_tile_null_count(name, tile_idx); } unsigned dim_idx = 0; const NDRange* mbr = nullptr; if (is_dim) { dim_idx = array_schema_->domain().get_dimension_index(name); - mbr = &rtree_.leaf(tile_idx); + mbr = &loaded_metadata_ptr_->rtree().leaf(tile_idx); } if (var_size) { @@ -2001,7 +1595,7 @@ TileMetadata FragmentMetadata::get_tile_metadata( const auto cell_val_num = array_schema_->cell_val_num(name); const void* sum = nullptr; if (TileMetadataGenerator::has_sum_metadata(type, false, cell_val_num)) { - sum = get_tile_sum(name, tile_idx); + sum = loaded_metadata_ptr_->get_tile_sum(name, tile_idx); } return {count, null_count, min, cell_size, max, cell_size, sum}; @@ -2010,28 +1604,10 @@ TileMetadata FragmentMetadata::get_tile_metadata( void FragmentMetadata::set_processed_conditions( std::vector& processed_conditions) { - processed_conditions_ = processed_conditions; - processed_conditions_set_ = std::unordered_set( - processed_conditions.begin(), processed_conditions.end()); -} - -std::vector& FragmentMetadata::get_processed_conditions() { - if (!loaded_metadata_.processed_conditions_) { - throw std::logic_error( - "Trying to access processed conditions metadata that's not present"); - } - - return processed_conditions_; -} - -std::unordered_set& -FragmentMetadata::get_processed_conditions_set() { - if (!loaded_metadata_.processed_conditions_) { - throw std::logic_error( - "Trying to access processed condition set metadata that's not present"); - } - - return processed_conditions_set_; + loaded_metadata_ptr_->processed_conditions() = processed_conditions; + loaded_metadata_ptr_->processed_conditions_set() = + std::unordered_set( + processed_conditions.begin(), processed_conditions.end()); } uint64_t FragmentMetadata::first_timestamp() const { @@ -2068,111 +1644,6 @@ void FragmentMetadata::write_footer(Serializer& serializer) const { write_generic_tile_offsets(serializer); } -void FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { - if (version_ <= 2) { - return; - } - - std::lock_guard lock(mtx_); - - if (loaded_metadata_.rtree_) { - return; - } - - auto tile = read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); - resources_->stats().add_counter("read_rtree_size", tile->size()); - - // Use the serialized buffer size to approximate memory usage of the rtree. - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(tile->size(), MemoryType::RTREE)) { - throw FragmentMetadataStatusException( - "Cannot load R-tree; Insufficient memory budget; Needed " + - std::to_string(tile->size()) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + std::to_string(memory_tracker_->get_memory_budget())); - } - - Deserializer deserializer(tile->data(), tile->size()); - rtree_.deserialize(deserializer, &array_schema_->domain(), version_); - - loaded_metadata_.rtree_ = true; -} - -void FragmentMetadata::free_rtree() { - auto freed = rtree_.free_memory(); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory(freed, MemoryType::RTREE); - } - loaded_metadata_.rtree_ = false; -} - -void FragmentMetadata::free_tile_offsets() { - for (uint64_t i = 0; i < tile_offsets_.size(); i++) { - std::lock_guard lock(tile_offsets_mtx_[i]); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory( - tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); - } - tile_offsets_[i].clear(); - loaded_metadata_.tile_offsets_[i] = false; - } - - for (uint64_t i = 0; i < tile_var_offsets_.size(); i++) { - std::lock_guard lock(tile_var_offsets_mtx_[i]); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory( - tile_var_offsets_[i].size() * sizeof(uint64_t), - MemoryType::TILE_OFFSETS); - } - tile_var_offsets_[i].clear(); - loaded_metadata_.tile_var_offsets_[i] = false; - } - - for (uint64_t i = 0; i < tile_offsets_.size(); i++) { - std::lock_guard lock(tile_offsets_mtx_[i]); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory( - tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); - } - tile_offsets_[i].clear(); - loaded_metadata_.tile_offsets_[i] = false; - } - - for (uint64_t i = 0; i < tile_validity_offsets_.size(); i++) { - std::lock_guard lock(mtx_); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory( - tile_validity_offsets_[i].size() * sizeof(uint64_t), - MemoryType::TILE_OFFSETS); - } - tile_validity_offsets_[i].clear(); - loaded_metadata_.tile_validity_offsets_[i] = false; - } - - for (uint64_t i = 0; i < tile_var_sizes_.size(); i++) { - std::lock_guard lock(mtx_); - if (memory_tracker_ != nullptr) { - memory_tracker_->release_memory( - tile_var_sizes_[i].size() * sizeof(uint64_t), - MemoryType::TILE_OFFSETS); - } - tile_var_sizes_[i].clear(); - loaded_metadata_.tile_var_sizes_[i] = false; - } -} - -void FragmentMetadata::load_tile_var_sizes( - const EncryptionKey& encryption_key, const std::string& name) { - if (version_ <= 2) { - return; - } - - auto it = idx_map_.find(name); - assert(it != idx_map_.end()); - auto idx = it->second; - load_tile_var_sizes(encryption_key, idx); -} - /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ @@ -2355,291 +1826,104 @@ std::vector FragmentMetadata::compute_overlapping_tile_ids( } while (utils::geometry::coords_in_rect( tile_coords, subarray_tile_domain, dim_num)); - // Clean up - tdb_delete_array(subarray_tile_domain); - tdb_delete_array(tile_coords); - - return tids; -} - -template -std::vector> -FragmentMetadata::compute_overlapping_tile_ids_cov(const T* subarray) const { - assert(dense_); - std::vector> tids; - auto dim_num = array_schema_->dim_num(); - - // Temporary domain vector - auto coord_size{array_schema_->domain().dimension_ptr(0)->coord_size()}; - auto temp_size = 2 * dim_num * coord_size; - std::vector temp(temp_size); - uint8_t offset = 0; - for (unsigned d = 0; d < dim_num; ++d) { - std::memcpy(&temp[offset], domain_[d].data(), domain_[d].size()); - offset += domain_[d].size(); - } - auto metadata_domain = (const T*)&temp[0]; - - // Check if there is any overlap - if (!utils::geometry::overlap(subarray, metadata_domain, dim_num)) - return tids; - - // Initialize subarray tile domain - auto subarray_tile_domain = tdb_new_array(T, 2 * dim_num); - get_subarray_tile_domain(subarray, subarray_tile_domain); - - auto tile_subarray = tdb_new_array(T, 2 * dim_num); - auto tile_overlap = tdb_new_array(T, 2 * dim_num); - bool overlap; - double cov; - - // Initialize tile coordinates - auto tile_coords = tdb_new_array(T, dim_num); - for (unsigned int i = 0; i < dim_num; ++i) - tile_coords[i] = subarray_tile_domain[2 * i]; - - // Walk through all tiles in subarray tile domain - auto& domain{array_schema_->domain()}; - uint64_t tile_pos; - do { - domain.get_tile_subarray(metadata_domain, tile_coords, tile_subarray); - utils::geometry::overlap( - subarray, tile_subarray, dim_num, tile_overlap, &overlap); - assert(overlap); - cov = utils::geometry::coverage(tile_overlap, tile_subarray, dim_num); - tile_pos = domain.get_tile_pos(metadata_domain, tile_coords); - tids.emplace_back(tile_pos, cov); - domain.get_next_tile_coords(subarray_tile_domain, tile_coords); - } while (utils::geometry::coords_in_rect( - tile_coords, subarray_tile_domain, dim_num)); - - // Clean up - tdb_delete_array(subarray_tile_domain); - tdb_delete_array(tile_coords); - tdb_delete_array(tile_subarray); - tdb_delete_array(tile_overlap); - - return tids; -} - -template -void FragmentMetadata::get_subarray_tile_domain( - const T* subarray, T* subarray_tile_domain) const { - // For easy reference - auto dim_num = array_schema_->dim_num(); - - // Calculate subarray in tile domain - for (unsigned d = 0; d < dim_num; ++d) { - auto domain = (const T*)domain_[d].data(); - auto tile_extent = *(const T*)array_schema_->domain().tile_extent(d).data(); - auto overlap = std::max(subarray[2 * d], domain[0]); - subarray_tile_domain[2 * d] = - Dimension::tile_idx(overlap, domain[0], tile_extent); - - overlap = std::min(subarray[2 * d + 1], domain[1]); - subarray_tile_domain[2 * d + 1] = - Dimension::tile_idx(overlap, domain[0], tile_extent); - } -} - -void FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { - std::lock_guard lock(mtx_); - - // Case the non-empty domain is not initialized yet - if (non_empty_domain_.empty()) { - non_empty_domain_ = mbr; - return; - } - - // Expand existing non-empty domain - array_schema_->domain().expand_ndrange(mbr, &non_empty_domain_); -} - -void FragmentMetadata::load_tile_offsets( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 2) { - return; - } - - // If the tile offset is already loaded, exit early to avoid the lock - if (loaded_metadata_.tile_offsets_[idx]) { - return; - } - - std::lock_guard lock(tile_offsets_mtx_[idx]); - - if (loaded_metadata_.tile_offsets_[idx]) { - return; - } - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_offsets_[idx]); - resources_->stats().add_counter("read_tile_offsets_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_tile_offsets(idx, deserializer); - - loaded_metadata_.tile_offsets_[idx] = true; -} - -void FragmentMetadata::load_tile_var_offsets( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 2) { - return; - } - - // If the tile var offset is already loaded, exit early to avoid the lock - if (loaded_metadata_.tile_var_offsets_[idx]) { - return; - } - - std::lock_guard lock(tile_var_offsets_mtx_[idx]); - - if (loaded_metadata_.tile_var_offsets_[idx]) { - return; - } - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_var_offsets_[idx]); - resources_->stats().add_counter("read_tile_var_offsets_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_tile_var_offsets(idx, deserializer); - - loaded_metadata_.tile_var_offsets_[idx] = true; -} - -void FragmentMetadata::load_tile_var_sizes( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 2) { - return; - } - - std::lock_guard lock(mtx_); - - if (loaded_metadata_.tile_var_sizes_[idx]) { - return; - } - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_var_sizes_[idx]); - resources_->stats().add_counter("read_tile_var_sizes_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_tile_var_sizes(idx, deserializer); - - loaded_metadata_.tile_var_sizes_[idx] = true; -} - -void FragmentMetadata::load_tile_validity_offsets( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ <= 6) { - return; - } - - std::lock_guard lock(mtx_); - - if (loaded_metadata_.tile_validity_offsets_[idx]) { - return; - } - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_validity_offsets_[idx]); - resources_->stats().add_counter( - "read_tile_validity_offsets_size", tile->size()); - - ConstBuffer cbuff(tile->data(), tile->size()); - load_tile_validity_offsets(idx, &cbuff); - - loaded_metadata_.tile_validity_offsets_[idx] = true; -} - -void FragmentMetadata::load_tile_min_values( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ < constants::tile_metadata_min_version) { - return; - } - - std::lock_guard lock(mtx_); - - if (loaded_metadata_.tile_min_[idx]) { - return; - } - - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_min_offsets_[idx]); - resources_->stats().add_counter("read_tile_min_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_tile_min_values(idx, deserializer); + // Clean up + tdb_delete_array(subarray_tile_domain); + tdb_delete_array(tile_coords); - loaded_metadata_.tile_min_[idx] = true; + return tids; } -void FragmentMetadata::load_tile_max_values( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ < constants::tile_metadata_min_version) { - return; - } - - std::lock_guard lock(mtx_); +template +std::vector> +FragmentMetadata::compute_overlapping_tile_ids_cov(const T* subarray) const { + assert(dense_); + std::vector> tids; + auto dim_num = array_schema_->dim_num(); - if (loaded_metadata_.tile_max_[idx]) { - return; + // Temporary domain vector + auto coord_size{array_schema_->domain().dimension_ptr(0)->coord_size()}; + auto temp_size = 2 * dim_num * coord_size; + std::vector temp(temp_size); + uint8_t offset = 0; + for (unsigned d = 0; d < dim_num; ++d) { + std::memcpy(&temp[offset], domain_[d].data(), domain_[d].size()); + offset += domain_[d].size(); } + auto metadata_domain = (const T*)&temp[0]; - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_max_offsets_[idx]); - resources_->stats().add_counter("read_tile_max_size", tile->size()); + // Check if there is any overlap + if (!utils::geometry::overlap(subarray, metadata_domain, dim_num)) + return tids; - Deserializer deserializer(tile->data(), tile->size()); - load_tile_max_values(idx, deserializer); + // Initialize subarray tile domain + auto subarray_tile_domain = tdb_new_array(T, 2 * dim_num); + get_subarray_tile_domain(subarray, subarray_tile_domain); - loaded_metadata_.tile_max_[idx] = true; -} + auto tile_subarray = tdb_new_array(T, 2 * dim_num); + auto tile_overlap = tdb_new_array(T, 2 * dim_num); + bool overlap; + double cov; -void FragmentMetadata::load_tile_sum_values( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ < constants::tile_metadata_min_version) { - return; - } + // Initialize tile coordinates + auto tile_coords = tdb_new_array(T, dim_num); + for (unsigned int i = 0; i < dim_num; ++i) + tile_coords[i] = subarray_tile_domain[2 * i]; - std::lock_guard lock(mtx_); + // Walk through all tiles in subarray tile domain + auto& domain{array_schema_->domain()}; + uint64_t tile_pos; + do { + domain.get_tile_subarray(metadata_domain, tile_coords, tile_subarray); + utils::geometry::overlap( + subarray, tile_subarray, dim_num, tile_overlap, &overlap); + assert(overlap); + cov = utils::geometry::coverage(tile_overlap, tile_subarray, dim_num); + tile_pos = domain.get_tile_pos(metadata_domain, tile_coords); + tids.emplace_back(tile_pos, cov); + domain.get_next_tile_coords(subarray_tile_domain, tile_coords); + } while (utils::geometry::coords_in_rect( + tile_coords, subarray_tile_domain, dim_num)); - if (loaded_metadata_.tile_sum_[idx]) { - return; - } + // Clean up + tdb_delete_array(subarray_tile_domain); + tdb_delete_array(tile_coords); + tdb_delete_array(tile_subarray); + tdb_delete_array(tile_overlap); - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_sum_offsets_[idx]); - resources_->stats().add_counter("read_tile_sum_size", tile->size()); + return tids; +} - Deserializer deserializer(tile->data(), tile->size()); - load_tile_sum_values(idx, deserializer); +template +void FragmentMetadata::get_subarray_tile_domain( + const T* subarray, T* subarray_tile_domain) const { + // For easy reference + auto dim_num = array_schema_->dim_num(); - loaded_metadata_.tile_sum_[idx] = true; -} + // Calculate subarray in tile domain + for (unsigned d = 0; d < dim_num; ++d) { + auto domain = (const T*)domain_[d].data(); + auto tile_extent = *(const T*)array_schema_->domain().tile_extent(d).data(); + auto overlap = std::max(subarray[2 * d], domain[0]); + subarray_tile_domain[2 * d] = + Dimension::tile_idx(overlap, domain[0], tile_extent); -void FragmentMetadata::load_tile_null_count_values( - const EncryptionKey& encryption_key, unsigned idx) { - if (version_ < constants::tile_metadata_min_version) { - return; + overlap = std::min(subarray[2 * d + 1], domain[1]); + subarray_tile_domain[2 * d + 1] = + Dimension::tile_idx(overlap, domain[0], tile_extent); } +} +void FragmentMetadata::expand_non_empty_domain(const NDRange& mbr) { std::lock_guard lock(mtx_); - if (loaded_metadata_.tile_null_count_[idx]) { + // Case the non-empty domain is not initialized yet + if (non_empty_domain_.empty()) { + non_empty_domain_ = mbr; return; } - auto tile = read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_null_count_offsets_[idx]); - resources_->stats().add_counter("read_tile_null_count_size", tile->size()); - - Deserializer deserializer(tile->data(), tile->size()); - load_tile_null_count_values(idx, deserializer); - - loaded_metadata_.tile_null_count_[idx] = true; + // Expand existing non-empty domain + array_schema_->domain().expand_ndrange(mbr, &non_empty_domain_); } // ===== FORMAT ===== @@ -2769,7 +2053,7 @@ void FragmentMetadata::load_mbrs(Deserializer& deserializer) { mbr_num = deserializer.read(); // Set leaf level - throw_if_not_ok(rtree_.set_leaf_num(mbr_num)); + throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf_num(mbr_num)); auto& domain{array_schema_->domain()}; auto dim_num = domain.dim_num(); for (uint64_t m = 0; m < mbr_num; ++m) { @@ -2778,12 +2062,12 @@ void FragmentMetadata::load_mbrs(Deserializer& deserializer) { uint64_t r_size{2 * domain.dimension_ptr(d)->coord_size()}; mbr[d] = Range(deserializer.get_ptr(r_size), r_size); } - throw_if_not_ok(rtree_.set_leaf(m, mbr)); + throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf(m, mbr)); } // Build R-tree bottom-up if (mbr_num > 0) { - rtree_.build_tree(); + loaded_metadata_ptr_->rtree().build_tree(); } sparse_tile_num_ = mbr_num; @@ -2901,465 +2185,6 @@ void FragmentMetadata::load_non_empty_domain_v5_or_higher( } } -// Applicable only to versions 1 and 2 -void FragmentMetadata::load_tile_offsets(Deserializer& deserializer) { - uint64_t tile_offsets_num = 0; - unsigned int attribute_num = array_schema_->attribute_num(); - - // Allocate tile offsets - tile_offsets_.resize(attribute_num + 1); - tile_offsets_mtx_.resize(attribute_num + 1); - - // For all attributes, get the tile offsets - for (unsigned int i = 0; i < attribute_num + 1; ++i) { - // Get number of tile offsets - tile_offsets_num = deserializer.read(); - - if (tile_offsets_num == 0) - continue; - - auto size = tile_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile offsets; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - // Get tile offsets - tile_offsets_[i].resize(tile_offsets_num); - deserializer.read(&tile_offsets_[i][0], size); - } - - loaded_metadata_.tile_offsets_.resize( - array_schema_->attribute_num() + 1, true); -} - -void FragmentMetadata::load_tile_offsets( - unsigned idx, Deserializer& deserializer) { - uint64_t tile_offsets_num = 0; - - // Get number of tile offsets - tile_offsets_num = deserializer.read(); - - // Get tile offsets - if (tile_offsets_num != 0) { - auto size = tile_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile offsets; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_offsets_[idx].resize(tile_offsets_num); - deserializer.read(&tile_offsets_[idx][0], size); - } -} - -// ===== FORMAT ===== -// tile_var_offsets_attr#0_num (uint64_t) -// tile_var_offsets_attr#0_#1 (uint64_t) tile_var_offsets_attr#0_#2 -// (uint64_t) -// ... -// ... -// tile_var_offsets_attr#_num(uint64_t) -// tile_var_offsets_attr#_#1 (uint64_t) -// tile_ver_offsets_attr#_#2 (uint64_t) ... -void FragmentMetadata::load_tile_var_offsets(Deserializer& deserializer) { - unsigned int attribute_num = array_schema_->attribute_num(); - uint64_t tile_var_offsets_num = 0; - - // Allocate tile offsets - tile_var_offsets_.resize(attribute_num); - tile_var_offsets_mtx_.resize(attribute_num); - - // For all attributes, get the variable tile offsets - for (unsigned int i = 0; i < attribute_num; ++i) { - // Get number of tile offsets - tile_var_offsets_num = deserializer.read(); - - if (tile_var_offsets_num == 0) - continue; - - auto size = tile_var_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - // Get variable tile offsets - tile_var_offsets_[i].resize(tile_var_offsets_num); - deserializer.read(&tile_var_offsets_[i][0], size); - } - - loaded_metadata_.tile_var_offsets_.resize( - array_schema_->attribute_num(), true); -} - -void FragmentMetadata::load_tile_var_offsets( - unsigned idx, Deserializer& deserializer) { - uint64_t tile_var_offsets_num = 0; - - // Get number of tile offsets - tile_var_offsets_num = deserializer.read(); - - // Get variable tile offsets - if (tile_var_offsets_num != 0) { - auto size = tile_var_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_var_offsets_[idx].resize(tile_var_offsets_num); - deserializer.read(&tile_var_offsets_[idx][0], size); - } -} - -// ===== FORMAT ===== -// tile_var_sizes_attr#0_num (uint64_t) -// tile_var_sizes_attr#0_#1 (uint64_t) tile_sizes_attr#0_#2 (uint64_t) ... -// ... -// tile_var_sizes_attr#_num(uint64_t) -// tile_var_sizes__attr#_#1 (uint64_t) -// tile_var_sizes_attr#_#2 (uint64_t) ... -void FragmentMetadata::load_tile_var_sizes(Deserializer& deserializer) { - unsigned int attribute_num = array_schema_->attribute_num(); - uint64_t tile_var_sizes_num = 0; - - // Allocate tile sizes - tile_var_sizes_.resize(attribute_num); - - // For all attributes, get the variable tile sizes - for (unsigned int i = 0; i < attribute_num; ++i) { - // Get number of tile sizes - tile_var_sizes_num = deserializer.read(); - - if (tile_var_sizes_num == 0) - continue; - - auto size = tile_var_sizes_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var sizes; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - // Get variable tile sizes - tile_var_sizes_[i].resize(tile_var_sizes_num); - deserializer.read(&tile_var_sizes_[i][0], size); - } - - loaded_metadata_.tile_var_sizes_.resize(array_schema_->attribute_num(), true); -} - -void FragmentMetadata::load_tile_var_sizes( - unsigned idx, Deserializer& deserializer) { - uint64_t tile_var_sizes_num = 0; - - // Get number of tile sizes - tile_var_sizes_num = deserializer.read(); - - // Get variable tile sizes - if (tile_var_sizes_num != 0) { - auto size = tile_var_sizes_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var sizes; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_var_sizes_[idx].resize(tile_var_sizes_num); - deserializer.read(&tile_var_sizes_[idx][0], size); - } -} - -void FragmentMetadata::load_tile_validity_offsets( - unsigned idx, ConstBuffer* buff) { - uint64_t tile_validity_offsets_num = 0; - - // Get number of tile offsets - if (!buff->read(&tile_validity_offsets_num, sizeof(uint64_t)).ok()) { - throw FragmentMetadataStatusException( - "Cannot load fragment metadata; Reading number of validity tile " - "offsets failed"); - } - - // Get tile offsets - if (tile_validity_offsets_num != 0) { - auto size = tile_validity_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile validity offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_validity_offsets_[idx].resize(tile_validity_offsets_num); - if (!buff->read(&tile_validity_offsets_[idx][0], size).ok()) { - throw FragmentMetadataStatusException( - "Cannot load fragment metadata; Reading validity tile offsets " - "failed"); - } - } -} - -// ===== FORMAT ===== -// tile_min_values#0_size_buffer (uint64_t) -// tile_min_values#0_size_buffer_var (uint64_t) -// tile_min_values#0_buffer -// tile_min_values#0_buffer_var -// ... -// tile_min_values#_size_buffer (uint64_t) -// tile_min_values#_size_buffer_var (uint64_t) -// tile_min_values#_buffer -// tile_min_values#_buffer_var -void FragmentMetadata::load_tile_min_values( - unsigned idx, Deserializer& deserializer) { - uint64_t buffer_size = 0; - uint64_t var_buffer_size = 0; - - // Get buffer size - buffer_size = deserializer.read(); - - // Get var buffer size - var_buffer_size = deserializer.read(); - - // Get tile mins - if (buffer_size != 0) { - auto size = buffer_size + var_buffer_size; - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_MIN_VALS)) { - throw FragmentMetadataStatusException( - "Cannot load min values; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_min_buffer_[idx].resize(buffer_size); - deserializer.read(&tile_min_buffer_[idx][0], buffer_size); - - if (var_buffer_size) { - tile_min_var_buffer_[idx].resize(var_buffer_size); - deserializer.read(&tile_min_var_buffer_[idx][0], var_buffer_size); - } - } -} - -// ===== FORMAT ===== -// tile_max_values#0_size_buffer (uint64_t) -// tile_max_values#0_size_buffer_var (uint64_t) -// tile_max_values#0_buffer -// tile_max_values#0_buffer_var -// ... -// tile_max_values#_size_buffer (uint64_t) -// tile_max_values#_size_buffer_var (uint64_t) -// tile_max_values#_buffer -// tile_max_values#_buffer_var -void FragmentMetadata::load_tile_max_values( - unsigned idx, Deserializer& deserializer) { - uint64_t buffer_size = 0; - uint64_t var_buffer_size = 0; - - // Get buffer size - buffer_size = deserializer.read(); - - // Get var buffer size - var_buffer_size = deserializer.read(); - - // Get tile maxs - if (buffer_size != 0) { - auto size = buffer_size + var_buffer_size; - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_MAX_VALS)) { - throw FragmentMetadataStatusException( - "Cannot load max values; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_max_buffer_[idx].resize(buffer_size); - deserializer.read(&tile_max_buffer_[idx][0], buffer_size); - - if (var_buffer_size) { - tile_max_var_buffer_[idx].resize(var_buffer_size); - deserializer.read(&tile_max_var_buffer_[idx][0], var_buffer_size); - } - } -} - -// ===== FORMAT ===== -// tile_sum_values_attr#0_num (uint64_t) -// tile_sum_value_attr#0_#1 (uint64_t) tile_sum_value_attr#0_#2 (uint64_t) -// ... -// ... -// tile_sum_values_attr#_num (uint64_t) -// tile_sum_value_attr#_#1 (uint64_t) -// tile_sum_value_attr#_#2 (uint64_t) ... -void FragmentMetadata::load_tile_sum_values( - unsigned idx, Deserializer& deserializer) { - uint64_t tile_sum_num = 0; - - // Get number of tile sums - tile_sum_num = deserializer.read(); - - // Get tile sums - if (tile_sum_num != 0) { - auto size = tile_sum_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_SUMS)) { - throw FragmentMetadataStatusException( - "Cannot load sum values; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_sums_[idx].resize(size); - deserializer.read(tile_sums_[idx].data(), size); - } -} - -// ===== FORMAT ===== -// tile_nc_values_attr#0_num (uint64_t) -// tile_nc_value_attr#0_#1 (uint64_t) tile_nc_value_attr#0_#2 (uint64_t) ... -// ... -// tile_nc_values_attr#_num (uint64_t) -// tile_nc_value_attr#_#1 (uint64_t) -// tile_nc_value_attr#_#2 (uint64_t) ... -void FragmentMetadata::load_tile_null_count_values( - unsigned idx, Deserializer& deserializer) { - uint64_t tile_null_count_num = 0; - - // Get number of tile null counts - tile_null_count_num = deserializer.read(); - - // Get tile null count - if (tile_null_count_num != 0) { - auto size = tile_null_count_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_NULL_COUNTS)) { - throw FragmentMetadataStatusException( - "Cannot load null count values; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } - - tile_null_counts_[idx].resize(tile_null_count_num); - deserializer.read(&tile_null_counts_[idx][0], size); - } -} - -// ===== FORMAT ===== -// fragment_min_size_attr#0 (uint64_t) -// fragment_min_attr#0 (min_size) -// fragment_max_size_attr#0 (uint64_t) -// fragment_max_attr#0 (max_size) -// fragment_sum_attr#0 (uint64_t) -// fragment_null_count_attr#0 (uint64_t) -// ... -// fragment_min_size_attr# (uint64_t) -// fragment_min_attr# (min_size) -// fragment_max_size_attr# (uint64_t) -// fragment_max_attr# (max_size) -// fragment_sum_attr# (uint64_t) -// fragment_null_count_attr# (uint64_t) -void FragmentMetadata::load_fragment_min_max_sum_null_count( - Deserializer& deserializer) { - auto num = num_dims_and_attrs(); - - for (unsigned int i = 0; i < num; ++i) { - // Get min. - uint64_t min_size; - min_size = deserializer.read(); - - fragment_mins_[i].resize(min_size); - deserializer.read(fragment_mins_[i].data(), min_size); - - // Get max. - uint64_t max_size; - max_size = deserializer.read(); - - fragment_maxs_[i].resize(max_size); - deserializer.read(fragment_maxs_[i].data(), max_size); - - // Get sum. - fragment_sums_[i] = deserializer.read(); - - // Get null count. - fragment_null_counts_[i] = deserializer.read(); - } -} - -// ===== FORMAT ===== -// condition_num (uint64_t) -// processed_condition_size#0 (uint64_t) -// processed_condition#0 -// ... -// processed_condition_size# (uint64_t) -// processed_condition# -void FragmentMetadata::load_processed_conditions(Deserializer& deserializer) { - // Get num conditions. - uint64_t num; - num = deserializer.read(); - - processed_conditions_.reserve(num); - for (uint64_t i = 0; i < num; i++) { - uint64_t size; - size = deserializer.read(); - - std::string condition; - condition.resize(size); - deserializer.read(condition.data(), size); - - processed_conditions_.emplace_back(condition); - } - - processed_conditions_set_ = std::unordered_set( - processed_conditions_.begin(), processed_conditions_.end()); -} - void FragmentMetadata::load_version(Deserializer& deserializer) { version_ = deserializer.read(); } @@ -3628,9 +2453,16 @@ void FragmentMetadata::load_v1_v2( load_non_empty_domain(deserializer); load_mbrs(deserializer); load_bounding_coords(deserializer); - load_tile_offsets(deserializer); - load_tile_var_offsets(deserializer); - load_tile_var_sizes(deserializer); + auto v1v2_meta = + dynamic_cast(loaded_metadata_ptr_); + if (v1v2_meta == nullptr) { + throw std::logic_error( + "FragmentMetadata::load_v1_v2 unable to dynamic_cast " + "loaded_metadata_ptr_"); + } + v1v2_meta->load_tile_offsets(deserializer); + v1v2_meta->load_tile_var_offsets(deserializer); + v1v2_meta->load_tile_var_sizes(deserializer); load_last_tile_cell_num(deserializer); load_file_sizes(deserializer); load_file_var_sizes(deserializer); @@ -3652,10 +2484,6 @@ void FragmentMetadata::load_footer( std::unordered_map> array_schemas) { std::lock_guard lock(mtx_); - if (loaded_metadata_.footer_) { - return; - } - std::shared_ptr tile; if (fragment_metadata_tile == nullptr) { has_consolidated_footer_ = false; @@ -3721,37 +2549,10 @@ void FragmentMetadata::load_footer( has_delete_meta_ * 2; num += (version_ >= 5) ? array_schema_->dim_num() : 0; - tile_offsets_.resize(num); - tile_offsets_mtx_.resize(num); - tile_var_offsets_.resize(num); - tile_var_offsets_mtx_.resize(num); - tile_var_sizes_.resize(num); - tile_validity_offsets_.resize(num); - tile_min_buffer_.resize(num); - tile_min_var_buffer_.resize(num); - tile_max_buffer_.resize(num); - tile_max_var_buffer_.resize(num); - tile_sums_.resize(num); - tile_null_counts_.resize(num); - - fragment_mins_.resize(num); - fragment_maxs_.resize(num); - fragment_sums_.resize(num); - fragment_null_counts_.resize(num); - - loaded_metadata_.tile_offsets_.resize(num, false); - loaded_metadata_.tile_var_offsets_.resize(num, false); - loaded_metadata_.tile_var_sizes_.resize(num, false); - loaded_metadata_.tile_validity_offsets_.resize(num, false); - loaded_metadata_.tile_min_.resize(num, false); - loaded_metadata_.tile_max_.resize(num, false); - loaded_metadata_.tile_sum_.resize(num, false); - loaded_metadata_.tile_null_count_.resize(num, false); + loaded_metadata_ptr_->resize_offsets(num); load_generic_tile_offsets(deserializer); - loaded_metadata_.footer_ = true; - // If the footer_size is not set lets calculate from how much of the // buffer we read if (footer_size_ == 0) { @@ -3885,15 +2686,15 @@ void FragmentMetadata::store_rtree( } shared_ptr FragmentMetadata::write_rtree() { - rtree_.build_tree(); + loaded_metadata_ptr_->rtree().build_tree(); SizeComputationSerializer size_computation_serializer; - rtree_.serialize(size_computation_serializer); + loaded_metadata_ptr_->rtree().serialize(size_computation_serializer); auto tile{WriterTile::from_generic( size_computation_serializer.size(), memory_tracker_)}; Serializer serializer(tile->data(), tile->size()); - rtree_.serialize(serializer); + loaded_metadata_ptr_->rtree().serialize(serializer); return tile; } @@ -4025,13 +2826,14 @@ void FragmentMetadata::store_tile_offsets( void FragmentMetadata::write_tile_offsets( unsigned idx, Serializer& serializer) { // Write number of tile offsets - uint64_t tile_offsets_num = tile_offsets_[idx].size(); + uint64_t tile_offsets_num = loaded_metadata_ptr_->tile_offsets()[idx].size(); serializer.write(tile_offsets_num); // Write tile offsets if (tile_offsets_num != 0) { serializer.write( - &tile_offsets_[idx][0], tile_offsets_num * sizeof(uint64_t)); + &loaded_metadata_ptr_->tile_offsets()[idx][0], + tile_offsets_num * sizeof(uint64_t)); } } @@ -4054,13 +2856,15 @@ void FragmentMetadata::write_tile_var_offsets( unsigned idx, Serializer& serializer) { // Write tile offsets for each attribute // Write number of offsets - uint64_t tile_var_offsets_num = tile_var_offsets_[idx].size(); + uint64_t tile_var_offsets_num = + loaded_metadata_ptr_->tile_var_offsets()[idx].size(); serializer.write(tile_var_offsets_num); // Write tile offsets if (tile_var_offsets_num != 0) { serializer.write( - &tile_var_offsets_[idx][0], tile_var_offsets_num * sizeof(uint64_t)); + &loaded_metadata_ptr_->tile_var_offsets()[idx][0], + tile_var_offsets_num * sizeof(uint64_t)); } } @@ -4082,13 +2886,15 @@ void FragmentMetadata::store_tile_var_sizes( void FragmentMetadata::write_tile_var_sizes( unsigned idx, Serializer& serializer) { // Write number of sizes - uint64_t tile_var_sizes_num = tile_var_sizes_[idx].size(); + uint64_t tile_var_sizes_num = + loaded_metadata_ptr_->tile_var_sizes()[idx].size(); serializer.write(tile_var_sizes_num); // Write tile sizes if (tile_var_sizes_num != 0) { serializer.write( - &tile_var_sizes_[idx][0], tile_var_sizes_num * sizeof(uint64_t)); + &loaded_metadata_ptr_->tile_var_sizes()[idx][0], + tile_var_sizes_num * sizeof(uint64_t)); } } @@ -4110,13 +2916,14 @@ void FragmentMetadata::store_tile_validity_offsets( void FragmentMetadata::write_tile_validity_offsets( unsigned idx, Serializer& serializer) { // Write number of tile offsets - uint64_t tile_validity_offsets_num = tile_validity_offsets_[idx].size(); + uint64_t tile_validity_offsets_num = + loaded_metadata_ptr_->tile_validity_offsets()[idx].size(); serializer.write(tile_validity_offsets_num); // Write tile validity offsets if (tile_validity_offsets_num != 0) { serializer.write( - &tile_validity_offsets_[idx][0], + &loaded_metadata_ptr_->tile_validity_offsets()[idx][0], tile_validity_offsets_num * sizeof(uint64_t)); } } @@ -4138,21 +2945,27 @@ void FragmentMetadata::store_tile_mins( void FragmentMetadata::write_tile_mins(unsigned idx, Serializer& serializer) { // Write size of buffer - uint64_t tile_mins_buffer_size = tile_min_buffer_[idx].size(); + uint64_t tile_mins_buffer_size = + loaded_metadata_ptr_->tile_min_buffer()[idx].size(); serializer.write(tile_mins_buffer_size); // Write size of buffer var - uint64_t tile_mins_var_buffer_size = tile_min_var_buffer_[idx].size(); + uint64_t tile_mins_var_buffer_size = + loaded_metadata_ptr_->tile_min_var_buffer()[idx].size(); serializer.write(tile_mins_var_buffer_size); // Write tile buffer if (tile_mins_buffer_size != 0) { - serializer.write(&tile_min_buffer_[idx][0], tile_mins_buffer_size); + serializer.write( + &loaded_metadata_ptr_->tile_min_buffer()[idx][0], + tile_mins_buffer_size); } // Write tile var buffer if (tile_mins_var_buffer_size != 0) { - serializer.write(&tile_min_var_buffer_[idx][0], tile_mins_var_buffer_size); + serializer.write( + &loaded_metadata_ptr_->tile_min_var_buffer()[idx][0], + tile_mins_var_buffer_size); } } @@ -4173,21 +2986,27 @@ void FragmentMetadata::store_tile_maxs( void FragmentMetadata::write_tile_maxs(unsigned idx, Serializer& serializer) { // Write size of buffer - uint64_t tile_maxs_buffer_size = tile_max_buffer_[idx].size(); + uint64_t tile_maxs_buffer_size = + loaded_metadata_ptr_->tile_max_buffer()[idx].size(); serializer.write(tile_maxs_buffer_size); // Write size of buffer var - uint64_t tile_maxs_var_buffer_size = tile_max_var_buffer_[idx].size(); + uint64_t tile_maxs_var_buffer_size = + loaded_metadata_ptr_->tile_max_var_buffer()[idx].size(); serializer.write(tile_maxs_var_buffer_size); // Write tile buffer if (tile_maxs_buffer_size != 0) { - serializer.write(&tile_max_buffer_[idx][0], tile_maxs_buffer_size); + serializer.write( + &loaded_metadata_ptr_->tile_max_buffer()[idx][0], + tile_maxs_buffer_size); } // Write tile var buffer if (tile_maxs_var_buffer_size != 0) { - serializer.write(&tile_max_var_buffer_[idx][0], tile_maxs_var_buffer_size); + serializer.write( + &loaded_metadata_ptr_->tile_max_var_buffer()[idx][0], + tile_maxs_var_buffer_size); } } @@ -4208,12 +3027,15 @@ void FragmentMetadata::store_tile_sums( void FragmentMetadata::write_tile_sums(unsigned idx, Serializer& serializer) { // Write number of tile sums - uint64_t tile_sums_num = tile_sums_[idx].size() / sizeof(uint64_t); + uint64_t tile_sums_num = + loaded_metadata_ptr_->tile_sums()[idx].size() / sizeof(uint64_t); serializer.write(tile_sums_num); // Write tile sums if (tile_sums_num != 0) { - serializer.write(tile_sums_[idx].data(), tile_sums_num * sizeof(uint64_t)); + serializer.write( + loaded_metadata_ptr_->tile_sums()[idx].data(), + tile_sums_num * sizeof(uint64_t)); } } @@ -4235,13 +3057,15 @@ void FragmentMetadata::store_tile_null_counts( void FragmentMetadata::write_tile_null_counts( unsigned idx, Serializer& serializer) { // Write number of tile null counts - uint64_t tile_null_counts_num = tile_null_counts_[idx].size(); + uint64_t tile_null_counts_num = + loaded_metadata_ptr_->tile_null_counts()[idx].size(); serializer.write(tile_null_counts_num); // Write tile null counts if (tile_null_counts_num != 0) { serializer.write( - &tile_null_counts_[idx][0], tile_null_counts_num * sizeof(uint64_t)); + &loaded_metadata_ptr_->tile_null_counts()[idx][0], + tile_null_counts_num * sizeof(uint64_t)); } } @@ -4253,22 +3077,25 @@ void FragmentMetadata::store_fragment_min_max_sum_null_count( // Store all attributes. for (unsigned int i = 0; i < num; ++i) { // Store min. - uint64_t min_size = fragment_mins_[i].size(); + uint64_t min_size = loaded_metadata_ptr_->fragment_mins()[i].size(); serializer.write(min_size); - serializer.write(fragment_mins_[i].data(), min_size); + serializer.write( + loaded_metadata_ptr_->fragment_mins()[i].data(), min_size); // Store max. - uint64_t max_size = fragment_maxs_[i].size(); + uint64_t max_size = loaded_metadata_ptr_->fragment_maxs()[i].size(); serializer.write(max_size); - serializer.write(fragment_maxs_[i].data(), max_size); + serializer.write( + loaded_metadata_ptr_->fragment_maxs()[i].data(), max_size); // Store sum. - serializer.write(fragment_sums_[i]); + serializer.write(loaded_metadata_ptr_->fragment_sums()[i]); // Store null count. - serializer.write(fragment_null_counts_[i]); + serializer.write( + loaded_metadata_ptr_->fragment_null_counts()[i]); } }; @@ -4289,10 +3116,11 @@ void FragmentMetadata::store_processed_conditions( const EncryptionKey& encryption_key, uint64_t* nbytes) { auto serialize_processed_conditions = [this](Serializer& serializer) { // Store num conditions. - uint64_t num = processed_conditions_.size(); + uint64_t num = loaded_metadata_ptr_->processed_conditions().size(); serializer.write(num); - for (auto& processed_condition : processed_conditions_) { + for (auto& processed_condition : + loaded_metadata_ptr_->processed_conditions()) { uint64_t size = processed_condition.size(); serializer.write(size); @@ -4334,11 +3162,11 @@ void FragmentMetadata::compute_fragment_min_max_sum(const std::string& name) { T max = metadata_generator_type_data::max; // Get data and tile num. - auto min_values = - static_cast(static_cast(tile_min_buffer_[idx].data())); - auto max_values = - static_cast(static_cast(tile_max_buffer_[idx].data())); - auto& null_count_values = tile_null_counts_[idx]; + auto min_values = static_cast(static_cast( + loaded_metadata_ptr_->tile_min_buffer()[idx].data())); + auto max_values = static_cast(static_cast( + loaded_metadata_ptr_->tile_max_buffer()[idx].data())); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile. @@ -4351,10 +3179,12 @@ void FragmentMetadata::compute_fragment_min_max_sum(const std::string& name) { } // Copy min max values. - fragment_mins_[idx].resize(sizeof(T)); - fragment_maxs_[idx].resize(sizeof(T)); - memcpy(fragment_mins_[idx].data(), &min, sizeof(T)); - memcpy(fragment_maxs_[idx].data(), &max, sizeof(T)); + loaded_metadata_ptr_->fragment_mins()[idx].resize(sizeof(T)); + loaded_metadata_ptr_->fragment_maxs()[idx].resize(sizeof(T)); + memcpy( + loaded_metadata_ptr_->fragment_mins()[idx].data(), &min, sizeof(T)); + memcpy( + loaded_metadata_ptr_->fragment_maxs()[idx].data(), &max, sizeof(T)); } if (has_sum) { @@ -4385,9 +3215,9 @@ void FragmentMetadata::compute_fragment_min_max_sum( void* max = nullptr; // Get data and tile num. - auto min_values = tile_min_buffer_[idx].data(); - auto max_values = tile_max_buffer_[idx].data(); - auto& null_count_values = tile_null_counts_[idx]; + auto min_values = loaded_metadata_ptr_->tile_min_buffer()[idx].data(); + auto max_values = loaded_metadata_ptr_->tile_max_buffer()[idx].data(); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile. @@ -4410,13 +3240,15 @@ void FragmentMetadata::compute_fragment_min_max_sum( // Copy values. if (min != nullptr) { - fragment_mins_[idx].resize(cell_val_num); - memcpy(fragment_mins_[idx].data(), min, cell_val_num); + loaded_metadata_ptr_->fragment_mins()[idx].resize(cell_val_num); + memcpy( + loaded_metadata_ptr_->fragment_mins()[idx].data(), min, cell_val_num); } if (max != nullptr) { - fragment_maxs_[idx].resize(cell_val_num); - memcpy(fragment_maxs_[idx].data(), max, cell_val_num); + loaded_metadata_ptr_->fragment_maxs()[idx].resize(cell_val_num); + memcpy( + loaded_metadata_ptr_->fragment_maxs()[idx].data(), max, cell_val_num); } } @@ -4427,9 +3259,9 @@ void FragmentMetadata::compute_fragment_sum( int64_t sum_data = 0; // Get data and tile num. - auto values = - static_cast(static_cast(tile_sums_[idx].data())); - auto& null_count_values = tile_null_counts_[idx]; + auto values = static_cast( + static_cast(loaded_metadata_ptr_->tile_sums()[idx].data())); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile, swallowing overflow exception. @@ -4452,7 +3284,8 @@ void FragmentMetadata::compute_fragment_sum( } // Copy value. - memcpy(&fragment_sums_[idx], &sum_data, sizeof(int64_t)); + memcpy( + &loaded_metadata_ptr_->fragment_sums()[idx], &sum_data, sizeof(int64_t)); } template <> @@ -4462,9 +3295,9 @@ void FragmentMetadata::compute_fragment_sum( uint64_t sum_data = 0; // Get data and tile num. - auto values = - static_cast(static_cast(tile_sums_[idx].data())); - auto& null_count_values = tile_null_counts_[idx]; + auto values = static_cast( + static_cast(loaded_metadata_ptr_->tile_sums()[idx].data())); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile, swallowing overflow exception. @@ -4480,7 +3313,8 @@ void FragmentMetadata::compute_fragment_sum( } // Copy value. - memcpy(&fragment_sums_[idx], &sum_data, sizeof(uint64_t)); + memcpy( + &loaded_metadata_ptr_->fragment_sums()[idx], &sum_data, sizeof(uint64_t)); } template <> @@ -4490,9 +3324,9 @@ void FragmentMetadata::compute_fragment_sum( double sum_data = 0; // Get data and tile num. - auto values = - static_cast(static_cast(tile_sums_[idx].data())); - auto& null_count_values = tile_null_counts_[idx]; + auto values = static_cast( + static_cast(loaded_metadata_ptr_->tile_sums()[idx].data())); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile, swallowing overflow exception. @@ -4511,7 +3345,8 @@ void FragmentMetadata::compute_fragment_sum( } // Copy value. - memcpy(&fragment_sums_[idx], &sum_data, sizeof(double)); + memcpy( + &loaded_metadata_ptr_->fragment_sums()[idx], &sum_data, sizeof(double)); } void FragmentMetadata::min_max_var(const std::string& name) { @@ -4535,13 +3370,13 @@ void FragmentMetadata::min_max_var(const std::string& name) { uint64_t max_size = 0; // Get data and tile num. - auto min_offsets = - static_cast(static_cast(tile_min_buffer_[idx].data())); - auto max_offsets = - static_cast(static_cast(tile_max_buffer_[idx].data())); - auto min_values = tile_min_var_buffer_[idx].data(); - auto max_values = tile_max_var_buffer_[idx].data(); - auto& null_count_values = tile_null_counts_[idx]; + auto min_offsets = static_cast( + static_cast(loaded_metadata_ptr_->tile_min_buffer()[idx].data())); + auto max_offsets = static_cast( + static_cast(loaded_metadata_ptr_->tile_max_buffer()[idx].data())); + auto min_values = loaded_metadata_ptr_->tile_min_var_buffer()[idx].data(); + auto max_values = loaded_metadata_ptr_->tile_max_var_buffer()[idx].data(); + auto& null_count_values = loaded_metadata_ptr_->tile_null_counts()[idx]; auto tile_num = this->tile_num(); // Process tile by tile. @@ -4550,12 +3385,14 @@ void FragmentMetadata::min_max_var(const std::string& name) { auto min_value = min_values + min_offsets[t]; auto min_value_size = t == tile_num - 1 ? - tile_min_var_buffer_[idx].size() - min_offsets[t] : + loaded_metadata_ptr_->tile_min_var_buffer()[idx].size() - + min_offsets[t] : min_offsets[t + 1] - min_offsets[t]; auto max_value = max_values + max_offsets[t]; auto max_value_size = t == tile_num - 1 ? - tile_max_var_buffer_[idx].size() - max_offsets[t] : + loaded_metadata_ptr_->tile_max_var_buffer()[idx].size() - + max_offsets[t] : max_offsets[t + 1] - max_offsets[t]; if (min == nullptr && max == nullptr) { min = min_value; @@ -4599,13 +3436,13 @@ void FragmentMetadata::min_max_var(const std::string& name) { // Copy values. if (min != nullptr) { - fragment_mins_[idx].resize(min_size); - memcpy(fragment_mins_[idx].data(), min, min_size); + loaded_metadata_ptr_->fragment_mins()[idx].resize(min_size); + memcpy(loaded_metadata_ptr_->fragment_mins()[idx].data(), min, min_size); } if (max != nullptr) { - fragment_maxs_[idx].resize(max_size); - memcpy(fragment_maxs_[idx].data(), max, max_size); + loaded_metadata_ptr_->fragment_maxs()[idx].resize(max_size); + memcpy(loaded_metadata_ptr_->fragment_maxs()[idx].data(), max, max_size); } } @@ -4642,23 +3479,6 @@ void FragmentMetadata::store_footer(const EncryptionKey&) { resources_->stats().add_counter("write_frag_meta_footer_size", tile->size()); } -void FragmentMetadata::resize_tile_offsets_vectors(uint64_t size) { - tile_offsets_mtx().resize(size); - tile_offsets().resize(size); -} - -void FragmentMetadata::resize_tile_var_offsets_vectors(uint64_t size) { - tile_var_offsets_mtx().resize(size); - tile_var_offsets().resize(size); -} - -void FragmentMetadata::resize_tile_var_sizes_vectors(uint64_t size) { - tile_var_sizes().resize(size); -} -void FragmentMetadata::resize_tile_validity_offsets_vectors(uint64_t size) { - tile_validity_offsets().resize(size); -} - void FragmentMetadata::clean_up() { auto fragment_metadata_uri = fragment_uri_.join_path(constants::fragment_metadata_filename); diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index f23dd1950623..119e248ae120 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -43,6 +43,7 @@ #include "tiledb/common/pmr.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/fragment/loaded_fragment_metadata.h" #include "tiledb/sm/misc/types.h" #include "tiledb/sm/rtree/rtree.h" #include "tiledb/sm/storage_manager/context_resources.h" @@ -63,6 +64,13 @@ class EncryptionKey; class TileMetadata; class MemoryTracker; +class FragmentMetadataStatusException : public StatusException { + public: + explicit FragmentMetadataStatusException(const std::string& message) + : StatusException("FragmentMetadata", message) { + } +}; + /** Stores the metadata structures of a fragment. */ class FragmentMetadata { public: @@ -78,7 +86,9 @@ class FragmentMetadata { * metadata corresponds to. */ FragmentMetadata( - ContextResources* resources, shared_ptr memory_tracker); + ContextResources* resources, + shared_ptr memory_tracker, + format_version_t version); /** * Constructor. @@ -107,7 +117,7 @@ class FragmentMetadata { bool has_delete_mata = false); /** Destructor. */ - ~FragmentMetadata(); + ~FragmentMetadata() = default; DISABLE_COPY_AND_COPY_ASSIGN(FragmentMetadata); DISABLE_MOVE_AND_MOVE_ASSIGN(FragmentMetadata); @@ -116,22 +126,6 @@ class FragmentMetadata { /* TYPE DEFINITIONS */ /* ********************************* */ - /** Keeps track of which metadata is loaded. */ - struct LoadedMetadata { - bool footer_ = false; - bool rtree_ = false; - std::vector tile_offsets_; - std::vector tile_var_offsets_; - std::vector tile_var_sizes_; - std::vector tile_validity_offsets_; - std::vector tile_min_; - std::vector tile_max_; - std::vector tile_sum_; - std::vector tile_null_count_; - bool fragment_min_max_sum_null_count_ = false; - bool processed_conditions_ = false; - }; - /** * Stores the start offsets of the generic tiles stored in the * metadata file, each separately storing the various metadata @@ -326,85 +320,6 @@ class FragmentMetadata { return tile_index_base_; } - /** Returns the tile offsets. */ - inline const tdb::pmr::vector>& tile_offsets() - const { - return tile_offsets_; - } - - /** Returns the variable tile offsets. */ - inline const tdb::pmr::vector>& tile_var_offsets() - const { - return tile_var_offsets_; - } - - /** Returns the sizes of the uncompressed variable tiles. */ - inline const tdb::pmr::vector>& tile_var_sizes() - const { - return tile_var_sizes_; - } - - /** Returns the validity tile offsets. */ - inline const tdb::pmr::vector>& - tile_validity_offsets() const { - return tile_validity_offsets_; - } - - /** Returns the tile min buffers. */ - inline const tdb::pmr::vector>& tile_min_buffer() - const { - return tile_min_buffer_; - } - - /** Returns the tile min buffers variable length data. */ - inline const tdb::pmr::vector>& tile_min_var_buffer() - const { - return tile_min_var_buffer_; - } - - /** Returns the tile max buffers. */ - inline const tdb::pmr::vector>& tile_max_buffer() - const { - return tile_max_buffer_; - } - - /** Returns the tile max buffers variable length data. */ - inline const tdb::pmr::vector>& tile_max_var_buffer() - const { - return tile_max_var_buffer_; - } - - /** Returns the tile sum values for fixed sized data. */ - inline const tdb::pmr::vector>& tile_sums() const { - return tile_sums_; - } - - /** Returns the tile null count values for attributes/dimensions. */ - inline const tdb::pmr::vector>& tile_null_counts() - const { - return tile_null_counts_; - } - - /** Returns the fragment mins. */ - inline const std::vector>& fragment_mins() const { - return fragment_mins_; - } - - /** Returns the fragment maxs. */ - inline const std::vector>& fragment_maxs() const { - return fragment_maxs_; - } - - /** Returns the fragment sums. */ - inline const std::vector& fragment_sums() const { - return fragment_sums_; - } - - /** Returns the fragment null counts. */ - inline const std::vector& fragment_null_counts() const { - return fragment_null_counts_; - } - /** Returns the fragment timestamp range. */ inline const std::pair& timestamp_range() const { return timestamp_range_; @@ -420,30 +335,11 @@ class FragmentMetadata { return non_empty_domain_; } - /** Returns an RTree for the MBRs. */ - inline const RTree& rtree() const { - return rtree_; - } - /** Returns the generic tile offsets. */ inline const GenericTileOffsets& generic_tile_offsets() const { return gt_offsets_; } - /** - * Retrieves the overlap of all MBRs with the input ND range. - */ - void get_tile_overlap( - const NDRange& range, - std::vector& is_default, - TileOverlap* tile_overlap); - - /** - * Compute tile bitmap for the curent fragment/range/dimension. - */ - void compute_tile_bitmap( - const Range& range, unsigned d, std::vector* tile_bitmap); - /** * Initializes the fragment metadata structures. * @@ -746,38 +642,6 @@ class FragmentMetadata { /** Return the array schema name. */ const std::string& array_schema_name(); - /** - * Retrieves the starting offset of the input tile of the input attribute - * or dimension in the file. If the attribute/dimension is var-sized, it - * returns the starting offset of the offsets tile. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return The file offset to be retrieved. - */ - uint64_t file_offset(const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the starting offset of the input tile of input attribute or - * dimension in the file. The attribute/dimension must be var-sized. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return The file offset to be retrieved. - */ - uint64_t file_var_offset(const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the starting offset of the input validity tile of the - * input attribute in the file. - * - * @param name The input attribute. - * @param tile_idx The index of the tile in the metadata. - * @return The file offset to be retrieved. - */ - uint64_t file_validity_offset( - const std::string& name, uint64_t tile_idx) const; - uint64_t footer_size() const; /** Returns the MBR of the input tile. */ @@ -786,42 +650,6 @@ class FragmentMetadata { /** Returns all the MBRs of all tiles in the fragment. */ const tdb::pmr::vector& mbrs() const; - /** - * Retrieves the size of the tile when it is persisted (e.g. the size of the - * compressed tile on disk) for a given attribute or dimension and tile index. - * If the attribute/dimension is var-sized, this will return the persisted - * size of the offsets tile. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return Size. - */ - uint64_t persisted_tile_size( - const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the size of the tile when it is persisted (e.g. the size of the - * compressed tile on disk) for a given var-sized attribute or dimension - * and tile index. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return Size. - */ - uint64_t persisted_tile_var_size( - const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the size of the validity tile when it is persisted (e.g. the size - * of the compressed tile on disk) for a given attribute. - * - * @param name The input attribute. - * @param tile_idx The index of the tile in the metadata. - * @return Size. - */ - uint64_t persisted_tile_validity_size( - const std::string& name, uint64_t tile_idx) const; - /** * Returns the (uncompressed) tile size for a given attribute or dimension * and tile index. If the attribute/dimension is var-sized, this will return @@ -833,16 +661,6 @@ class FragmentMetadata { */ uint64_t tile_size(const std::string& name, uint64_t tile_idx) const; - /** - * Retrieves the (uncompressed) tile size for a given var-sized attribute or - * dimension and tile index. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return Size. - */ - uint64_t tile_var_size(const std::string& name, uint64_t tile_idx); - /** * Retrieves the tile min value for a given attribute or dimension and tile * index. @@ -866,59 +684,6 @@ class FragmentMetadata { template T get_tile_max_as(const std::string& name, uint64_t tile_idx) const; - /** - * Retrieves the tile sum value for a given attribute or dimension and tile - * index. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return Sum. - */ - const void* get_tile_sum(const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the tile null count value for a given attribute or dimension - * and tile index. - * - * @param name The input attribute/dimension. - * @param tile_idx The index of the tile in the metadata. - * @return Count. - */ - uint64_t get_tile_null_count( - const std::string& name, uint64_t tile_idx) const; - - /** - * Retrieves the min value for a given attribute or dimension. - * - * @param name The input attribute/dimension. - * @return Value. - */ - std::vector& get_min(const std::string& name); - - /** - * Retrieves the max value for a given attribute or dimension. - * - * @param name The input attribute/dimension. - * @return Value. - */ - std::vector& get_max(const std::string& name); - - /** - * Retrieves the sum value for a given attribute or dimension. - * - * @param name The input attribute/dimension. - * @return Sum. - */ - void* get_sum(const std::string& name); - - /** - * Retrieves the null count value for a given attribute or dimension. - * - * @param name The input attribute/dimension. - * @return Count. - */ - uint64_t get_null_count(const std::string& name); - /** * Returns the tile metadata for a tile. * @@ -937,24 +702,6 @@ class FragmentMetadata { */ void set_processed_conditions(std::vector& processed_conditions); - /** - * Retrieves the processed conditions. The processed conditions is the list - * of delete/update conditions that have already been applied for this - * fragment and don't need to be applied again. - * - * @return Processed conditions. - */ - std::vector& get_processed_conditions(); - - /** - * Retrieves the processed conditions set. The processed conditions is the - * list of delete/update conditions that have already been applied for this - * fragment and don't need to be applied again. - * - * @return Processed conditions set. - */ - std::unordered_set& get_processed_conditions_set(); - /** Returns the first timestamp of the fragment timestamp range. */ uint64_t first_timestamp() const; @@ -967,84 +714,6 @@ class FragmentMetadata { /** Serializes the fragment metadata footer into the input buffer. */ void write_footer(Serializer& serializer) const; - /** Loads the R-tree from storage. */ - void load_rtree(const EncryptionKey& encryption_key); - - /** Frees the memory associated with the rtree. */ - void free_rtree(); - - /** Frees the memory associated with tile_offsets. */ - void free_tile_offsets(); - - /** - * Loads the variable tile sizes for the input attribute or dimension idx - * from storage. - * */ - void load_tile_var_sizes( - const EncryptionKey& encryption_key, const std::string& name); - - /** - * Loads tile offsets for the attribute/dimension names. - * - * @param encryption_key The key the array got opened with. - * @param names The attribute/dimension names. - */ - void load_tile_offsets( - const EncryptionKey& encryption_key, std::vector& names); - - /** - * Loads min values for the attribute names. - * - * @param encryption_key The key the array got opened with. - * @param names The attribute names. - */ - void load_tile_min_values( - const EncryptionKey& encryption_key, std::vector& names); - - /** - * Loads max values for the attribute names. - * - * @param encryption_key The key the array got opened with. - * @param names The attribute names. - */ - void load_tile_max_values( - const EncryptionKey& encryption_key, std::vector& names); - - /** - * Loads sum values for the attribute names. - * - * @param encryption_key The key the array got opened with. - * @param names The attribute names. - */ - void load_tile_sum_values( - const EncryptionKey& encryption_key, std::vector& names); - - /** - * Loads null count values for the attribute names. - * - * @param encryption_key The key the array got opened with. - * @param names The attribute names. - */ - void load_tile_null_count_values( - const EncryptionKey& encryption_key, std::vector& names); - - /** - * Loads the min max sum null count values for the fragment. - * - * @param encryption_key The key the array got opened with. - */ - void load_fragment_min_max_sum_null_count( - const EncryptionKey& encryption_key); - - /** - * Loads the processed conditions for the fragment. The processed conditions - * is the list of delete/update conditions that have already been applied for - * this fragment and don't need to be applied again. - * - * @param encryption_key The key the array got opened with. - */ - void load_processed_conditions(const EncryptionKey& encryption_key); - /** * Checks if the fragment overlaps partially (not fully) with a given * array open - end time. Assumes overlapping fragment and array open - close @@ -1120,86 +789,6 @@ class FragmentMetadata { return tile_index_base_; } - /** tile_offsets accessor */ - tdb::pmr::vector>& tile_offsets() { - return tile_offsets_; - } - - /** tile_offsets_mtx accessor */ - std::deque& tile_offsets_mtx() { - return tile_offsets_mtx_; - } - - /** tile_var_offsets accessor */ - tdb::pmr::vector>& tile_var_offsets() { - return tile_var_offsets_; - } - - /** tile_var_offsets_mtx accessor */ - std::deque& tile_var_offsets_mtx() { - return tile_var_offsets_mtx_; - } - - /** tile_var_sizes accessor */ - tdb::pmr::vector>& tile_var_sizes() { - return tile_var_sizes_; - } - - /** tile_validity_offsets accessor */ - tdb::pmr::vector>& tile_validity_offsets() { - return tile_validity_offsets_; - } - - /** tile_min_buffer accessor */ - tdb::pmr::vector>& tile_min_buffer() { - return tile_min_buffer_; - } - - /** tile_min_var_buffer accessor */ - tdb::pmr::vector>& tile_min_var_buffer() { - return tile_min_var_buffer_; - } - - /** tile_max_buffer accessor */ - tdb::pmr::vector>& tile_max_buffer() { - return tile_max_buffer_; - } - - /** tile_max_var_buffer accessor */ - tdb::pmr::vector>& tile_max_var_buffer() { - return tile_max_var_buffer_; - } - - /** tile_sums accessor */ - tdb::pmr::vector>& tile_sums() { - return tile_sums_; - } - - /** tile_null_counts accessor */ - tdb::pmr::vector>& tile_null_counts() { - return tile_null_counts_; - } - - /** fragment_mins accessor */ - std::vector>& fragment_mins() { - return fragment_mins_; - } - - /** fragment_maxs accessor */ - std::vector>& fragment_maxs() { - return fragment_maxs_; - } - - /** fragment_sums accessor */ - std::vector& fragment_sums() { - return fragment_sums_; - } - - /** fragment_null_counts accessor */ - std::vector& fragment_null_counts() { - return fragment_null_counts_; - } - /** version accessor */ uint32_t& version() { return version_; @@ -1220,11 +809,6 @@ class FragmentMetadata { return non_empty_domain_; } - /** rtree accessor */ - RTree& rtree() { - return rtree_; - } - /** gt_offsets_ accessor */ inline GenericTileOffsets& generic_tile_offsets() { return gt_offsets_; @@ -1235,36 +819,14 @@ class FragmentMetadata { resources_ = cr; } - /** loaded_metadata_.rtree_ accessor */ - void set_rtree_loaded() { - loaded_metadata_.rtree_ = true; + inline LoadedFragmentMetadata* loaded_metadata() { + return loaded_metadata_ptr_; } - /** loaded_metadata_ accessor */ - inline void set_loaded_metadata(const LoadedMetadata& loaded_metadata) { - loaded_metadata_ = loaded_metadata; + inline const LoadedFragmentMetadata* loaded_metadata() const { + return loaded_metadata_ptr_; } - /** - * Resize tile offsets related vectors. - */ - void resize_tile_offsets_vectors(uint64_t size); - - /** - * Resize tile var offsets related vectors. - */ - void resize_tile_var_offsets_vectors(uint64_t size); - - /** - * Resize tile var sizes related vectors. - */ - void resize_tile_var_sizes_vectors(uint64_t size); - - /** - * Resize tile validity offsets related vectors. - */ - void resize_tile_validity_offsets_vectors(uint64_t size); - private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -1338,109 +900,21 @@ class FragmentMetadata { /** Number of sparse tiles. */ uint64_t sparse_tile_num_; - /** Keeps track of which metadata has been loaded. */ - LoadedMetadata loaded_metadata_; - /** The size of the fragment metadata file. */ uint64_t meta_file_size_; /** Local mutex for thread-safety. */ std::mutex mtx_; - /** Mutex per tile offset loading. */ - std::deque tile_offsets_mtx_; - - /** Mutex per tile var offset loading. */ - std::deque tile_var_offsets_mtx_; - /** The non-empty domain of the fragment. */ NDRange non_empty_domain_; - /** An RTree for the MBRs. */ - RTree rtree_; - /** * The tile index base which is added to tile indices in setter functions. * Only used in global order writes. */ uint64_t tile_index_base_; - /** - * The tile offsets in their corresponding attribute files. Meaningful only - * when there is compression. - */ - tdb::pmr::vector> tile_offsets_; - - /** - * The variable tile offsets in their corresponding attribute files. - * Meaningful only for variable-sized tiles. - */ - tdb::pmr::vector> tile_var_offsets_; - - /** - * The sizes of the uncompressed variable tiles. - * Meaningful only when there is compression for variable tiles. - */ - tdb::pmr::vector> tile_var_sizes_; - - /** - * The validity tile offsets in their corresponding attribute files. - * Meaningful only when there is compression. - */ - tdb::pmr::vector> tile_validity_offsets_; - - /** - * The tile min buffers, for variable attributes/dimensions, this will store - * offsets. - */ - tdb::pmr::vector> tile_min_buffer_; - - /** - * The tile min buffers variable length data. - */ - tdb::pmr::vector> tile_min_var_buffer_; - - /** - * The tile max buffers, for variable attributes/dimensions, this will store - * offsets. - */ - tdb::pmr::vector> tile_max_buffer_; - - /** - * The tile max buffers variable length data. - */ - tdb::pmr::vector> tile_max_var_buffer_; - - /** - * The tile sum values, ignored for var sized attributes/dimensions. - */ - tdb::pmr::vector> tile_sums_; - - /** - * The tile null count values for attributes/dimensions. - */ - tdb::pmr::vector> tile_null_counts_; - - /** - * Fragment min values. - */ - std::vector> fragment_mins_; - - /** - * Fragment max values. - */ - std::vector> fragment_maxs_; - - /** - * Fragment sum values, ignored for var sized attributes/dimensions. - */ - std::vector fragment_sums_; - - /** - * Null count for fragment for attributes/dimensions. - */ - std::vector fragment_null_counts_; - /** The format version of this metadata. */ uint32_t version_; @@ -1453,14 +927,9 @@ class FragmentMetadata { /** The uri of the array the metadata belongs to. */ URI array_uri_; - /** Set of already processed delete/update conditions for this fragment. */ - std::unordered_set processed_conditions_set_; + shared_ptr loaded_metadata_; - /** - * Ordered list of already processed delete/update conditions for this - * fragment. - */ - std::vector processed_conditions_; + LoadedFragmentMetadata* loaded_metadata_ptr_; /* ********************************* */ /* PRIVATE METHODS */ @@ -1520,51 +989,6 @@ class FragmentMetadata { */ void expand_non_empty_domain(const NDRange& mbr); - /** - * Loads the tile offsets for the input attribute or dimension idx - * from storage. - */ - void load_tile_offsets(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the variable tile offsets for the input attribute or dimension idx - * from storage. - */ - void load_tile_var_offsets(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the variable tile sizes for the input attribute or dimension idx - * from storage. - * */ - void load_tile_var_sizes(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the validity tile offsets for the input attribute idx from storage. - */ - void load_tile_validity_offsets( - const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the min values for the input attribute idx from storage. - */ - void load_tile_min_values(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the max values for the input attribute idx from storage. - */ - void load_tile_max_values(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the sum values for the input attribute idx from storage. - */ - void load_tile_sum_values(const EncryptionKey& encryption_key, unsigned idx); - - /** - * Loads the null count values for the input attribute idx from storage. - */ - void load_tile_null_count_values( - const EncryptionKey& encryption_key, unsigned idx); - /** Loads the generic tile offsets from the buffer. */ void load_generic_tile_offsets(Deserializer& deserializer); @@ -1701,76 +1125,6 @@ class FragmentMetadata { */ void load_non_empty_domain_v5_or_higher(Deserializer& deserializer); - /** - * Loads the tile offsets for the input attribute from the input buffer. - * Applicable to versions 1 and 2 - */ - void load_tile_offsets(Deserializer& deserializer); - - /** - * Loads the tile offsets for the input attribute or dimension from the - * input buffer. - */ - void load_tile_offsets(unsigned idx, Deserializer& deserializer); - - /** - * Loads the variable tile offsets from the input buffer. - * Applicable to versions 1 and 2 - */ - void load_tile_var_offsets(Deserializer& deserializer); - - /** - * Loads the variable tile offsets for the input attribute or dimension from - * the input buffer. - */ - void load_tile_var_offsets(unsigned idx, Deserializer& deserializer); - - /** Loads the variable tile sizes from the input buffer. */ - void load_tile_var_sizes(Deserializer& deserializer); - - /** - * Loads the variable tile sizes for the input attribute or dimension - * from the input buffer. - */ - void load_tile_var_sizes(unsigned idx, Deserializer& deserializer); - - /** - * Loads the validity tile offsets for the input attribute from the - * input buffer. - */ - void load_tile_validity_offsets(unsigned idx, ConstBuffer* buff); - - /** - * Loads the min values for the input attribute from the input buffer. - */ - void load_tile_min_values(unsigned idx, Deserializer& deserializer); - - /** - * Loads the max values for the input attribute from the input buffer. - */ - void load_tile_max_values(unsigned idx, Deserializer& deserializer); - - /** - * Loads the sum values for the input attribute from the input buffer. - */ - void load_tile_sum_values(unsigned idx, Deserializer& deserializer); - - /** - * Loads the null count values for the input attribute from the input - * buffer. - */ - void load_tile_null_count_values(unsigned idx, Deserializer& deserializer); - - /** - * Loads the min max sum null count values for the fragment. - */ - void load_fragment_min_max_sum_null_count(Deserializer& deserializer); - - /** - * Loads the processed conditions for the fragment. - */ - void load_processed_conditions(Deserializer& deserializer); - /** Loads the format version from the buffer. */ void load_version(Deserializer& deserializer); @@ -2090,6 +1444,10 @@ class FragmentMetadata { * This builds the index mapping for attribute/dimension name to id. */ void build_idx_map(); + + friend class LoadedFragmentMetadata; + friend class OndemandFragmentMetadata; + friend class V1V2PreloadedFragmentMetadata; }; } // namespace sm diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.cc b/tiledb/sm/fragment/loaded_fragment_metadata.cc new file mode 100644 index 000000000000..0bc0b7b1955d --- /dev/null +++ b/tiledb/sm/fragment/loaded_fragment_metadata.cc @@ -0,0 +1,544 @@ +/** + * @file loaded_fragment_metadata.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements the LoadedFragmentMetadata class. + */ + +#include "tiledb/common/common.h" + +#include "tiledb/common/memory_tracker.h" +#include "tiledb/sm/fragment/fragment_metadata.h" +#include "tiledb/sm/fragment/ondemand_fragment_metadata.h" +#include "tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h" +#include "tiledb/sm/tile/tile_metadata_generator.h" + +using namespace tiledb::common; +using namespace tiledb::type; + +namespace tiledb::sm { + +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ + +LoadedFragmentMetadata::LoadedFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker) + : parent_fragment_(parent) + , memory_tracker_(memory_tracker) + , rtree_(RTree( + parent.array_schema() ? &parent.array_schema()->domain() : nullptr, + constants::rtree_fanout, + memory_tracker_)) + , tile_offsets_(memory_tracker->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_offsets_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_var_sizes_(memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_validity_offsets_( + memory_tracker_->get_resource(MemoryType::TILE_OFFSETS)) + , tile_min_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_min_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MIN_VALS)) + , tile_max_buffer_(memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_max_var_buffer_( + memory_tracker_->get_resource(MemoryType::TILE_MAX_VALS)) + , tile_sums_(memory_tracker_->get_resource(MemoryType::TILE_SUMS)) + , tile_null_counts_( + memory_tracker_->get_resource(MemoryType::TILE_NULL_COUNTS)) { +} + +/* ********************************* */ +/* API */ +/* ********************************* */ + +LoadedFragmentMetadata* LoadedFragmentMetadata::create( + FragmentMetadata& parent, + shared_ptr memory_tracker, + format_version_t version) { + if (version <= 2) { + return tdb_new(V1V2PreloadedFragmentMetadata, parent, memory_tracker); + } + + return tdb_new(OndemandFragmentMetadata, parent, memory_tracker); +} + +uint64_t LoadedFragmentMetadata::persisted_tile_size( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_offsets_[idx]) { + throw std::logic_error( + "Trying to access persisted tile offsets metadata that's not present"); + } + + auto tile_num = parent_fragment_.tile_num(); + + auto tile_size = + (tile_idx != tile_num - 1) ? + tile_offsets_[idx][tile_idx + 1] - tile_offsets_[idx][tile_idx] : + parent_fragment_.file_sizes_[idx] - tile_offsets_[idx][tile_idx]; + return tile_size; +} + +void LoadedFragmentMetadata::load_tile_offsets( + const EncryptionKey& encryption_key, std::vector& names) { + sort_names_by_index(names); + + // The fixed offsets are located before the + // var offsets. Load all of the fixed offsets + // first. + for (const auto& name : names) { + load_tile_offsets(encryption_key, parent_fragment_.idx_map_[name]); + } + + // Load all of the var offsets. + for (const auto& name : names) { + if (parent_fragment_.array_schema_->var_size(name)) { + load_tile_var_offsets(encryption_key, parent_fragment_.idx_map_[name]); + } + } + + // Load all of the validity offsets. + for (const auto& name : names) { + if (parent_fragment_.array_schema_->is_nullable(name)) { + load_tile_validity_offsets( + encryption_key, parent_fragment_.idx_map_[name]); + } + } +} + +void LoadedFragmentMetadata::free_tile_offsets() { + for (uint64_t i = 0; i < tile_offsets_.size(); i++) { + std::lock_guard lock(tile_offsets_mtx_[i]); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory( + tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); + } + tile_offsets_[i].clear(); + loaded_metadata_.tile_offsets_[i] = false; + } + + for (uint64_t i = 0; i < tile_var_offsets_.size(); i++) { + std::lock_guard lock(tile_var_offsets_mtx_[i]); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory( + tile_var_offsets_[i].size() * sizeof(uint64_t), + MemoryType::TILE_OFFSETS); + } + tile_var_offsets_[i].clear(); + loaded_metadata_.tile_var_offsets_[i] = false; + } + + for (uint64_t i = 0; i < tile_offsets_.size(); i++) { + std::lock_guard lock(tile_offsets_mtx_[i]); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory( + tile_offsets_[i].size() * sizeof(uint64_t), MemoryType::TILE_OFFSETS); + } + tile_offsets_[i].clear(); + loaded_metadata_.tile_offsets_[i] = false; + } + + for (uint64_t i = 0; i < tile_validity_offsets_.size(); i++) { + std::lock_guard lock(parent_fragment_.mtx_); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory( + tile_validity_offsets_[i].size() * sizeof(uint64_t), + MemoryType::TILE_OFFSETS); + } + tile_validity_offsets_[i].clear(); + loaded_metadata_.tile_validity_offsets_[i] = false; + } + + for (uint64_t i = 0; i < tile_var_sizes_.size(); i++) { + std::lock_guard lock(parent_fragment_.mtx_); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory( + tile_var_sizes_[i].size() * sizeof(uint64_t), + MemoryType::TILE_OFFSETS); + } + tile_var_sizes_[i].clear(); + loaded_metadata_.tile_var_sizes_[i] = false; + } +} + +uint64_t LoadedFragmentMetadata::file_offset( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_offsets_[idx]) { + throw std::logic_error( + "Trying to access tile offsets metadata that's not loaded"); + } + + return tile_offsets_[idx][tile_idx]; +} + +void LoadedFragmentMetadata::resize_tile_offsets_vectors(uint64_t size) { + tile_offsets_mtx_.resize(size); + tile_offsets_.resize(size); +} + +void LoadedFragmentMetadata::resize_tile_var_offsets_vectors(uint64_t size) { + tile_var_offsets_mtx_.resize(size); + tile_var_offsets_.resize(size); +} + +void LoadedFragmentMetadata::resize_offsets(uint64_t size) { + resize_tile_offsets_vectors(size); + resize_tile_var_offsets_vectors(size); + resize_tile_var_sizes_vectors(size); + tile_validity_offsets().resize(size); + tile_min_buffer().resize(size); + tile_min_var_buffer().resize(size); + tile_max_buffer().resize(size); + tile_max_var_buffer().resize(size); + tile_sums().resize(size); + tile_null_counts().resize(size); + fragment_mins_.resize(size); + fragment_maxs_.resize(size); + fragment_sums_.resize(size); + fragment_null_counts_.resize(size); + loaded_metadata_.tile_offsets_.resize(size, false); + loaded_metadata_.tile_var_offsets_.resize(size, false); + loaded_metadata_.tile_var_sizes_.resize(size, false); + loaded_metadata_.tile_validity_offsets_.resize(size, false); + loaded_metadata_.tile_min_.resize(size, false); + loaded_metadata_.tile_max_.resize(size, false); + loaded_metadata_.tile_sum_.resize(size, false); + loaded_metadata_.tile_null_count_.resize(size, false); +} + +uint64_t LoadedFragmentMetadata::file_var_offset( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_var_offsets_[idx]) { + throw std::logic_error( + "Trying to access tile var offsets metadata that's not loaded"); + } + + return tile_var_offsets_[idx][tile_idx]; +} + +uint64_t LoadedFragmentMetadata::persisted_tile_var_size( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + + if (!loaded_metadata_.tile_var_offsets_[idx]) { + throw std::logic_error( + "Trying to access persisted tile var offsets metadata that's not " + "present"); + } + + auto tile_num = parent_fragment_.tile_num(); + + auto tile_size = (tile_idx != tile_num - 1) ? + tile_var_offsets_[idx][tile_idx + 1] - + tile_var_offsets_[idx][tile_idx] : + parent_fragment_.file_var_sizes_[idx] - + tile_var_offsets_[idx][tile_idx]; + return tile_size; +} + +uint64_t LoadedFragmentMetadata::tile_var_size( + const std::string& name, uint64_t tile_idx) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_var_sizes_[idx]) { + throw FragmentMetadataStatusException( + "Trying to access tile var size metadata that's not loaded"); + } + + return tile_var_sizes_[idx][tile_idx]; +} + +void LoadedFragmentMetadata::load_tile_var_sizes( + const EncryptionKey& encryption_key, const std::string& name) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + load_tile_var_sizes(encryption_key, idx); +} + +void LoadedFragmentMetadata::resize_tile_var_sizes_vectors(uint64_t size) { + tile_var_sizes_.resize(size); +} + +void LoadedFragmentMetadata::sort_names_by_index( + std::vector& names) { + // Sort 'names' in ascending order of their index. The + // motivation is to load the offsets in order of their + // layout for sequential reads to the file. + std::sort( + names.begin(), + names.end(), + [&](const std::string& lhs, const std::string& rhs) { + assert(parent_fragment_.idx_map_.count(lhs) > 0); + assert(parent_fragment_.idx_map_.count(rhs) > 0); + return parent_fragment_.idx_map_[lhs] < parent_fragment_.idx_map_[rhs]; + }); +} + +void LoadedFragmentMetadata::load_tile_min_values( + const EncryptionKey& encryption_key, std::vector& names) { + // Load all the min values. + for (const auto& name : names) { + load_tile_min_values(encryption_key, parent_fragment_.idx_map_[name]); + } +} + +void LoadedFragmentMetadata::load_tile_max_values( + const EncryptionKey& encryption_key, std::vector& names) { + sort_names_by_index(names); + + // Load all the max values. + for (const auto& name : names) { + load_tile_max_values(encryption_key, parent_fragment_.idx_map_[name]); + } +} + +void LoadedFragmentMetadata::load_tile_sum_values( + const EncryptionKey& encryption_key, std::vector& names) { + sort_names_by_index(names); + + // Load all the sum values. + for (const auto& name : names) { + load_tile_sum_values(encryption_key, parent_fragment_.idx_map_[name]); + } +} + +void LoadedFragmentMetadata::load_tile_null_count_values( + const EncryptionKey& encryption_key, std::vector& names) { + sort_names_by_index(names); + + // Load all the null count values. + for (const auto& name : names) { + load_tile_null_count_values( + encryption_key, parent_fragment_.idx_map_[name]); + } +} + +std::vector& LoadedFragmentMetadata::get_processed_conditions() { + if (!loaded_metadata_.processed_conditions_) { + throw std::logic_error( + "Trying to access processed conditions metadata that's not present"); + } + + return processed_conditions_; +} + +std::unordered_set& +LoadedFragmentMetadata::get_processed_conditions_set() { + if (!loaded_metadata_.processed_conditions_) { + throw std::logic_error( + "Trying to access processed condition set metadata that's not present"); + } + + return processed_conditions_set_; +} + +std::vector& LoadedFragmentMetadata::get_min(const std::string& name) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.fragment_min_max_sum_null_count_) { + throw FragmentMetadataStatusException( + "Trying to access fragment min metadata that's not loaded"); + } + + const auto type = parent_fragment_.array_schema_->type(name); + const auto is_dim = parent_fragment_.array_schema_->is_dim(name); + const auto var_size = parent_fragment_.array_schema_->var_size(name); + const auto cell_val_num = parent_fragment_.array_schema_->cell_val_num(name); + if (!TileMetadataGenerator::has_min_max_metadata( + type, is_dim, var_size, cell_val_num)) { + throw FragmentMetadataStatusException( + "Trying to access fragment min metadata that's not present"); + } + + return fragment_mins_[idx]; +} + +std::vector& LoadedFragmentMetadata::get_max(const std::string& name) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.fragment_min_max_sum_null_count_) { + throw FragmentMetadataStatusException( + "Trying to access fragment max metadata that's not loaded"); + } + + const auto type = parent_fragment_.array_schema_->type(name); + const auto is_dim = parent_fragment_.array_schema_->is_dim(name); + const auto var_size = parent_fragment_.array_schema_->var_size(name); + const auto cell_val_num = parent_fragment_.array_schema_->cell_val_num(name); + if (!TileMetadataGenerator::has_min_max_metadata( + type, is_dim, var_size, cell_val_num)) { + throw FragmentMetadataStatusException( + "Trying to access fragment max metadata that's not present"); + } + + return fragment_maxs_[idx]; +} + +void* LoadedFragmentMetadata::get_sum(const std::string& name) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.fragment_min_max_sum_null_count_) { + throw FragmentMetadataStatusException( + "Trying to access fragment sum metadata that's not loaded"); + } + + const auto type = parent_fragment_.array_schema_->type(name); + const auto var_size = parent_fragment_.array_schema_->var_size(name); + const auto cell_val_num = parent_fragment_.array_schema_->cell_val_num(name); + if (!TileMetadataGenerator::has_sum_metadata(type, var_size, cell_val_num)) { + throw FragmentMetadataStatusException( + "Trying to access fragment sum metadata that's not present"); + } + + return &fragment_sums_[idx]; +} + +uint64_t LoadedFragmentMetadata::get_null_count(const std::string& name) { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.fragment_min_max_sum_null_count_) { + throw FragmentMetadataStatusException( + "Trying to access fragment null count metadata that's not loaded"); + } + + if (!parent_fragment_.array_schema_->is_nullable(name)) { + throw FragmentMetadataStatusException( + "Trying to access fragment null count metadata that's not present"); + } + + return fragment_null_counts_[idx]; +} + +uint64_t LoadedFragmentMetadata::get_tile_null_count( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_null_count_[idx]) { + throw FragmentMetadataStatusException( + "Trying to access tile null count metadata that's not loaded"); + } + + if (!parent_fragment_.array_schema_->is_nullable(name)) { + throw FragmentMetadataStatusException( + "Trying to access tile null count metadata that's not present"); + } + + return tile_null_counts_[idx][tile_idx]; +} + +const void* LoadedFragmentMetadata::get_tile_sum( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_sum_[idx]) { + throw FragmentMetadataStatusException( + "Trying to access tile sum metadata that's not loaded"); + } + + auto type = parent_fragment_.array_schema_->type(name); + auto var_size = parent_fragment_.array_schema_->var_size(name); + auto cell_val_num = parent_fragment_.array_schema_->cell_val_num(name); + if (!TileMetadataGenerator::has_sum_metadata(type, var_size, cell_val_num)) { + throw FragmentMetadataStatusException( + "Trying to access tile sum metadata that's not present"); + } + + const void* sum = &tile_sums_[idx][tile_idx * sizeof(uint64_t)]; + return sum; +} + +void LoadedFragmentMetadata::resize_tile_validity_offsets_vectors( + uint64_t size) { + tile_validity_offsets().resize(size); +} + +uint64_t LoadedFragmentMetadata::file_validity_offset( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_validity_offsets_[idx]) { + throw std::logic_error( + "Trying to access tile validity offsets metadata that's not loaded"); + } + + return tile_validity_offsets_[idx][tile_idx]; +} + +uint64_t LoadedFragmentMetadata::persisted_tile_validity_size( + const std::string& name, uint64_t tile_idx) const { + auto it = parent_fragment_.idx_map_.find(name); + assert(it != parent_fragment_.idx_map_.end()); + auto idx = it->second; + if (!loaded_metadata_.tile_validity_offsets_[idx]) { + throw std::logic_error( + "Trying to access persisted tile validity offsets metadata that's not " + "present"); + } + + auto tile_num = parent_fragment_.tile_num(); + + auto tile_size = (tile_idx != tile_num - 1) ? + tile_validity_offsets_[idx][tile_idx + 1] - + tile_validity_offsets_[idx][tile_idx] : + parent_fragment_.file_validity_sizes_[idx] - + tile_validity_offsets_[idx][tile_idx]; + return tile_size; +} + +// TODO: maybe remove, this is unused at the moment +void LoadedFragmentMetadata::free_rtree() { + auto freed = rtree_.free_memory(); + if (memory_tracker_ != nullptr) { + memory_tracker_->release_memory(freed, MemoryType::RTREE); + } + loaded_metadata_.rtree_ = false; +} + +/* ********************************* */ +/* PRIVATE METHODS */ +/* ********************************* */ + +} // namespace tiledb::sm diff --git a/tiledb/sm/fragment/loaded_fragment_metadata.h b/tiledb/sm/fragment/loaded_fragment_metadata.h new file mode 100644 index 000000000000..b7d1845ebe8f --- /dev/null +++ b/tiledb/sm/fragment/loaded_fragment_metadata.h @@ -0,0 +1,790 @@ +/** + * @file loaded_fragment_metadata.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class LoadedFragmentMetadata. + */ + +#ifndef TILEDB_LOADED_FRAGMENT_METADATA_H +#define TILEDB_LOADED_FRAGMENT_METADATA_H + +#include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/sm/rtree/rtree.h" +#include "tiledb/sm/storage_manager/context_resources.h" + +namespace tiledb { +namespace sm { + +class FragmentMetadata; + +/** Collection of lazily loaded fragment metadata */ +class LoadedFragmentMetadata { + public: + /* ********************************* */ + /* TYPE DEFINITIONS */ + /* ********************************* */ + + /** Keeps track of which metadata is loaded. */ + struct LoadedMetadata { + bool rtree_ = false; + std::vector tile_offsets_; + std::vector tile_var_offsets_; + std::vector tile_var_sizes_; + std::vector tile_validity_offsets_; + std::vector tile_min_; + std::vector tile_max_; + std::vector tile_sum_; + std::vector tile_null_count_; + bool fragment_min_max_sum_null_count_ = false; + bool processed_conditions_ = false; + }; + + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /* Destructor */ + virtual ~LoadedFragmentMetadata() = default; + + /** + * Constructor. + * + * @param resources A context resources instance. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + */ + LoadedFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Create loaded metadata objects given a format version + * + * @param parent The parent fragment + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + * @param version The format version of the fragment + */ + static LoadedFragmentMetadata* create( + FragmentMetadata& parent, + shared_ptr memory_tracker, + format_version_t version); + + /** Returns the tile offsets. */ + inline const tdb::pmr::vector>& tile_offsets() + const { + return tile_offsets_; + } + + /** tile_offsets accessor */ + tdb::pmr::vector>& tile_offsets() { + return tile_offsets_; + } + + /** Returns the variable tile offsets. */ + inline const tdb::pmr::vector>& tile_var_offsets() + const { + return tile_var_offsets_; + } + + /** tile_var_offsets accessor */ + tdb::pmr::vector>& tile_var_offsets() { + return tile_var_offsets_; + } + + /** tile_var_offsets_mtx accessor */ + std::deque& tile_var_offsets_mtx() { + return tile_var_offsets_mtx_; + } + + /** Returns the sizes of the uncompressed variable tiles. */ + inline const tdb::pmr::vector>& tile_var_sizes() + const { + return tile_var_sizes_; + } + + /** tile_var_sizes accessor */ + tdb::pmr::vector>& tile_var_sizes() { + return tile_var_sizes_; + } + + /** Returns the validity tile offsets. */ + inline const tdb::pmr::vector>& + tile_validity_offsets() const { + return tile_validity_offsets_; + } + + /** tile_validity_offsets accessor */ + tdb::pmr::vector>& tile_validity_offsets() { + return tile_validity_offsets_; + } + + /** + * Resize tile validity offsets related vectors. + * + * @param size The new size + */ + void resize_tile_validity_offsets_vectors(uint64_t size); + + /** + * Retrieves the starting offset of the input validity tile of the + * input attribute in the file. + * + * @param name The input attribute. + * @param tile_idx The index of the tile in the metadata. + * @return The file offset to be retrieved. + */ + uint64_t file_validity_offset( + const std::string& name, uint64_t tile_idx) const; + + /** + * Retrieves the size of the validity tile when it is persisted (e.g. the size + * of the compressed tile on disk) for a given attribute. + * + * @param name The input attribute. + * @param tile_idx The index of the tile in the metadata. + * @return Size. + */ + uint64_t persisted_tile_validity_size( + const std::string& name, uint64_t tile_idx) const; + + /** Returns the tile min buffers variable length data. */ + inline const tdb::pmr::vector>& tile_min_var_buffer() + const { + return tile_min_var_buffer_; + } + + /** tile_min_var_buffer accessor */ + tdb::pmr::vector>& tile_min_var_buffer() { + return tile_min_var_buffer_; + } + + /** + * Retrieves the size of the tile when it is persisted (e.g. the size of the + * compressed tile on disk) for a given attribute or dimension and tile index. + * If the attribute/dimension is var-sized, this will return the persisted + * size of the offsets tile. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return Size. + */ + uint64_t persisted_tile_size( + const std::string& name, uint64_t tile_idx) const; + + /** + * Loads tile offsets for the attribute/dimension names. + * + * @param encryption_key The key the array got opened with. + * @param names The attribute/dimension names. + */ + void load_tile_offsets( + const EncryptionKey& encryption_key, std::vector& names); + + /** Frees the memory associated with tile_offsets. */ + void free_tile_offsets(); + + /** + * Retrieves the starting offset of the input tile of the input attribute + * or dimension in the file. If the attribute/dimension is var-sized, it + * returns the starting offset of the offsets tile. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return The file offset to be retrieved. + */ + uint64_t file_offset(const std::string& name, uint64_t tile_idx) const; + + /** + * Resize tile offsets related vectors. + * + * @param size The new size + */ + void resize_tile_offsets_vectors(uint64_t size); + + /** + * Resize tile var offsets related vectors. + * + * @param size The new size + */ + void resize_tile_var_offsets_vectors(uint64_t size); + + /** + * Resize tile var sizes related vectors. + * + * @param size The new size + */ + void resize_tile_var_sizes_vectors(uint64_t size); + + /** + * Retrieves the size of the tile when it is persisted (e.g. the size of the + * compressed tile on disk) for a given var-sized attribute or dimension + * and tile index. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return Size. + */ + uint64_t persisted_tile_var_size( + const std::string& name, uint64_t tile_idx) const; + + /** + * Retrieves the starting offset of the input tile of input attribute or + * dimension in the file. The attribute/dimension must be var-sized. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return The file offset to be retrieved. + */ + uint64_t file_var_offset(const std::string& name, uint64_t tile_idx) const; + + /** + * Retrieves the (uncompressed) tile size for a given var-sized attribute or + * dimension and tile index. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return Size. + */ + uint64_t tile_var_size(const std::string& name, uint64_t tile_idx); + + /** + * Loads the variable tile sizes for the input attribute or dimension idx + * from storage. + * */ + void load_tile_var_sizes( + const EncryptionKey& encryption_key, const std::string& name); + + /** + * Loads min values for the attribute names. + * + * @param encryption_key The key the array got opened with. + * @param names The attribute names. + */ + void load_tile_min_values( + const EncryptionKey& encryption_key, std::vector& names); + + /** + * Loads max values for the attribute names. + * + * @param encryption_key The key the array got opened with. + * @param names The attribute names. + */ + void load_tile_max_values( + const EncryptionKey& encryption_key, std::vector& names); + + /** + * Loads sum values for the attribute names. + * + * @param encryption_key The key the array got opened with. + * @param names The attribute names. + */ + void load_tile_sum_values( + const EncryptionKey& encryption_key, std::vector& names); + + /** + * Loads null count values for the attribute names. + * + * @param encryption_key The key the array got opened with. + * @param names The attribute names. + */ + void load_tile_null_count_values( + const EncryptionKey& encryption_key, std::vector& names); + + /** Loads the R-tree from storage. */ + virtual void load_rtree(const EncryptionKey& encryption_key) = 0; + + /** + * Loads the min max sum null count values for the fragment. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_fragment_min_max_sum_null_count( + const EncryptionKey& encryption_key) = 0; + + /** + * Loads the processed conditions for the fragment. The processed conditions + * is the list of delete/update conditions that have already been applied for + * this fragment and don't need to be applied again. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_processed_conditions( + const EncryptionKey& encryption_key) = 0; + + /** + * Retrieves the processed conditions. The processed conditions is the list + * of delete/update conditions that have already been applied for this + * fragment and don't need to be applied again. + * + * @return Processed conditions. + */ + std::vector& get_processed_conditions(); + + /** + * Retrieves the processed conditions set. The processed conditions is the + * list of delete/update conditions that have already been applied for this + * fragment and don't need to be applied again. + * + * @return Processed conditions set. + */ + std::unordered_set& get_processed_conditions_set(); + + /** + * Retrieves the min value for a given attribute or dimension. + * + * @param name The input attribute/dimension. + * @return Value. + */ + std::vector& get_min(const std::string& name); + + /** + * Retrieves the max value for a given attribute or dimension. + * + * @param name The input attribute/dimension. + * @return Value. + */ + std::vector& get_max(const std::string& name); + + /** + * Retrieves the sum value for a given attribute or dimension. + * + * @param name The input attribute/dimension. + * @return Sum. + */ + void* get_sum(const std::string& name); + + /** + * Retrieves the null count value for a given attribute or dimension. + * + * @param name The input attribute/dimension. + * @return Count. + */ + uint64_t get_null_count(const std::string& name); + + /** + * Retrieves the tile sum value for a given attribute or dimension and tile + * index. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return Sum. + */ + const void* get_tile_sum(const std::string& name, uint64_t tile_idx) const; + + /** + * Retrieves the tile null count value for a given attribute or dimension + * and tile index. + * + * @param name The input attribute/dimension. + * @param tile_idx The index of the tile in the metadata. + * @return Count. + */ + uint64_t get_tile_null_count( + const std::string& name, uint64_t tile_idx) const; + + /** Returns the tile null count values for attributes/dimensions. */ + inline const tdb::pmr::vector>& tile_null_counts() + const { + return tile_null_counts_; + } + + /** tile_null_counts accessor */ + tdb::pmr::vector>& tile_null_counts() { + return tile_null_counts_; + } + + /** Returns the tile sum values for fixed sized data. */ + inline const tdb::pmr::vector>& tile_sums() const { + return tile_sums_; + } + + /** tile_sums accessor */ + tdb::pmr::vector>& tile_sums() { + return tile_sums_; + } + + /** Returns the tile max buffers. */ + inline const tdb::pmr::vector>& tile_max_buffer() + const { + return tile_max_buffer_; + } + + /** tile_max_buffer accessor */ + tdb::pmr::vector>& tile_max_buffer() { + return tile_max_buffer_; + } + + /** Returns the tile max buffers variable length data. */ + inline const tdb::pmr::vector>& tile_max_var_buffer() + const { + return tile_max_var_buffer_; + } + + /** tile_max_var_buffer accessor */ + tdb::pmr::vector>& tile_max_var_buffer() { + return tile_max_var_buffer_; + } + + /** Returns an RTree for the MBRs. */ + inline const RTree& rtree() const { + return rtree_; + } + + /** rtree accessor */ + RTree& rtree() { + return rtree_; + } + + /** loaded_metadata_.rtree_ accessor */ + void set_rtree_loaded() { + loaded_metadata_.rtree_ = true; + } + + /** + * Retrieves the overlap of all MBRs with the input ND range. + * + * @param range The range to use + * @param is_default If default range should be used + * @param tile_overlap The resulted tile overlap + */ + virtual void get_tile_overlap( + const NDRange& range, + std::vector& is_default, + TileOverlap* tile_overlap) = 0; + + /** + * Compute tile bitmap for the curent fragment/range/dimension. + * + * @param range The range to use + * @param d The dimension index + * @param tile_bitmap The resulted tile bitmap + */ + virtual void compute_tile_bitmap( + const Range& range, unsigned d, std::vector* tile_bitmap) = 0; + + /** Frees the memory associated with the rtree. */ + void free_rtree(); + + /** + * Sets loaded metadata, used in serialization + * + * @param loaded_metadata The metadata to set + */ + inline void set_loaded_metadata(const LoadedMetadata& loaded_metadata) { + loaded_metadata_ = loaded_metadata; + } + + /** loaded_metadata accessor . */ + inline const LoadedMetadata& loaded_metadata() const { + return loaded_metadata_; + } + + /** loaded_metadata accessor */ + LoadedMetadata& loaded_metadata() { + return loaded_metadata_; + } + + /** + * Resizes all offsets and reset their loaded flags. + * + * @param The new size + */ + void resize_offsets(uint64_t size); + + /** Returns the fragment mins. */ + inline const std::vector>& fragment_mins() const { + return fragment_mins_; + } + + /** Returns the fragment maxs. */ + inline const std::vector>& fragment_maxs() const { + return fragment_maxs_; + } + + /** Returns the fragment sums. */ + inline const std::vector& fragment_sums() const { + return fragment_sums_; + } + + /** Returns the fragment null counts. */ + inline const std::vector& fragment_null_counts() const { + return fragment_null_counts_; + } + + /** fragment_mins accessor */ + std::vector>& fragment_mins() { + return fragment_mins_; + } + + /** fragment_maxs accessor */ + std::vector>& fragment_maxs() { + return fragment_maxs_; + } + + /** fragment_sums accessor */ + std::vector& fragment_sums() { + return fragment_sums_; + } + + /** fragment_null_counts accessor */ + std::vector& fragment_null_counts() { + return fragment_null_counts_; + } + + /** Returns the tile min buffers. */ + inline const tdb::pmr::vector>& tile_min_buffer() + const { + return tile_min_buffer_; + } + + /** tile_min_buffer accessor */ + tdb::pmr::vector>& tile_min_buffer() { + return tile_min_buffer_; + } + + /** Returns the processed conditions vector. */ + inline const std::vector& processed_conditions() const { + return processed_conditions_; + } + + /** processed_conditions_ accessor */ + std::vector& processed_conditions() { + return processed_conditions_; + } + + /** Returns the processed conditions set */ + std::unordered_set& processed_conditions_set() { + return processed_conditions_set_; + } + + protected: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + FragmentMetadata& parent_fragment_; + + /** + * The memory tracker of the array this fragment metadata corresponds to. + */ + shared_ptr memory_tracker_; + + /** An RTree for the MBRs. */ + RTree rtree_; + + /** + * The tile offsets in their corresponding attribute files. Meaningful only + * when there is compression. + */ + tdb::pmr::vector> tile_offsets_; + + /** Mutex per tile offset loading. */ + std::deque tile_offsets_mtx_; + + /** Mutex per tile var offset loading. */ + std::deque tile_var_offsets_mtx_; + + /** + * The variable tile offsets in their corresponding attribute files. + * Meaningful only for variable-sized tiles. + */ + tdb::pmr::vector> tile_var_offsets_; + + /** + * The sizes of the uncompressed variable tiles. + * Meaningful only when there is compression for variable tiles. + */ + tdb::pmr::vector> tile_var_sizes_; + + /** + * The validity tile offsets in their corresponding attribute files. + * Meaningful only when there is compression. + */ + tdb::pmr::vector> tile_validity_offsets_; + + /** + * The tile min buffers, for variable attributes/dimensions, this will store + * offsets. + */ + tdb::pmr::vector> tile_min_buffer_; + + /** + * The tile min buffers variable length data. + */ + tdb::pmr::vector> tile_min_var_buffer_; + + /** + * The tile max buffers, for variable attributes/dimensions, this will store + * offsets. + */ + tdb::pmr::vector> tile_max_buffer_; + + /** + * The tile max buffers variable length data. + */ + tdb::pmr::vector> tile_max_var_buffer_; + + /** + * The tile sum values, ignored for var sized attributes/dimensions. + */ + tdb::pmr::vector> tile_sums_; + + /** + * The tile null count values for attributes/dimensions. + */ + tdb::pmr::vector> tile_null_counts_; + + /** + * Fragment min values. + */ + std::vector> fragment_mins_; + + /** + * Fragment max values. + */ + std::vector> fragment_maxs_; + + /** + * Fragment sum values, ignored for var sized attributes/dimensions. + */ + std::vector fragment_sums_; + + /** + * Null count for fragment for attributes/dimensions. + */ + std::vector fragment_null_counts_; + + /** + * Ordered list of already processed delete/update conditions for this + * fragment. + */ + std::vector processed_conditions_; + + /** Set of already processed delete/update conditions for this fragment. */ + std::unordered_set processed_conditions_set_; + + /** Keeps track of which metadata has been loaded. */ + LoadedMetadata loaded_metadata_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** + * Sorts a dims vector by index + * + * @param names the vector to sort + */ + void sort_names_by_index(std::vector& names); + + /** + * Loads the tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_offsets( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the variable tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_offsets( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the variable tile sizes for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_sizes( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the validity tile offsets for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_validity_offsets( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the min values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_min_values( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the max values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_max_values( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the sum values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_sum_values( + const EncryptionKey& encryption_key, unsigned idx) = 0; + + /** + * Loads the null count values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_null_count_values( + const EncryptionKey& encryption_key, unsigned idx) = 0; +}; + +} // namespace sm +} // namespace tiledb + +#endif // TILEDB_LOADED_FRAGMENT_METADATA_H diff --git a/tiledb/sm/fragment/ondemand_fragment_metadata.cc b/tiledb/sm/fragment/ondemand_fragment_metadata.cc new file mode 100644 index 000000000000..c0674610fc5d --- /dev/null +++ b/tiledb/sm/fragment/ondemand_fragment_metadata.cc @@ -0,0 +1,675 @@ +/** + * @file ondemand_fragment_metadata.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements the OndemandFragmentMetadata class. + */ + +#include "tiledb/common/common.h" + +#include "tiledb/common/memory_tracker.h" +#include "tiledb/sm/fragment/fragment_metadata.h" +#include "tiledb/sm/fragment/ondemand_fragment_metadata.h" +#include "tiledb/sm/query/readers/aggregators/tile_metadata.h" +#include "tiledb/sm/tile/tile.h" +#include "tiledb/storage_format/serialization/serializers.h" + +using namespace tiledb::common; +using namespace tiledb::type; + +namespace tiledb::sm { + +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ + +OndemandFragmentMetadata::OndemandFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker) + : LoadedFragmentMetadata(parent, memory_tracker) { +} + +/* ********************************* */ +/* API */ +/* ********************************* */ + +void OndemandFragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.rtree_) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.rtree_); + parent_fragment_.resources_->stats().add_counter( + "read_rtree_size", tile->size()); + + // Use the serialized buffer size to approximate memory usage of the rtree. + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(tile->size(), MemoryType::RTREE)) { + throw FragmentMetadataStatusException( + "Cannot load R-tree; Insufficient memory budget; Needed " + + std::to_string(tile->size()) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + std::to_string(memory_tracker_->get_memory_budget())); + } + + Deserializer deserializer(tile->data(), tile->size()); + rtree_.deserialize( + deserializer, + &parent_fragment_.array_schema_->domain(), + parent_fragment_.version_); + + loaded_metadata_.rtree_ = true; +} + +void OndemandFragmentMetadata::load_fragment_min_max_sum_null_count( + const EncryptionKey& encryption_key) { + if (loaded_metadata_.fragment_min_max_sum_null_count_) { + return; + } + + if (parent_fragment_.version_ <= 11) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, + parent_fragment_.gt_offsets_.fragment_min_max_sum_null_count_offset_); + parent_fragment_.resources_->stats().add_counter( + "read_fragment_min_max_sum_null_count_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_fragment_min_max_sum_null_count(deserializer); + + loaded_metadata_.fragment_min_max_sum_null_count_ = true; +} + +void OndemandFragmentMetadata::load_processed_conditions( + const EncryptionKey& encryption_key) { + if (loaded_metadata_.processed_conditions_) { + return; + } + + if (parent_fragment_.version_ <= 15) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, + parent_fragment_.gt_offsets_.processed_conditions_offsets_); + parent_fragment_.resources_->stats().add_counter( + "read_processed_conditions_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_processed_conditions(deserializer); + + loaded_metadata_.processed_conditions_ = true; +} + +/* ********************************* */ +/* PRIVATE METHODS */ +/* ********************************* */ + +void OndemandFragmentMetadata::load_tile_offsets( + const EncryptionKey& encryption_key, unsigned idx) { + // If the tile offset is already loaded, exit early to avoid the lock + if (loaded_metadata_.tile_offsets_[idx]) { + return; + } + + std::lock_guard lock(tile_offsets_mtx_[idx]); + + if (loaded_metadata_.tile_offsets_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_offsets_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_offsets(idx, deserializer); + + loaded_metadata_.tile_offsets_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_offsets( + unsigned idx, Deserializer& deserializer) { + uint64_t tile_offsets_num = 0; + + // Get number of tile offsets + tile_offsets_num = deserializer.read(); + + // Get tile offsets + if (tile_offsets_num != 0) { + auto size = tile_offsets_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile offsets; Insufficient memory budget; Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_offsets_[idx].resize(tile_offsets_num); + deserializer.read(&tile_offsets_[idx][0], size); + } +} + +void OndemandFragmentMetadata::load_tile_var_offsets( + const EncryptionKey& encryption_key, unsigned idx) { + // If the tile var offset is already loaded, exit early to avoid the lock + if (loaded_metadata_.tile_var_offsets_[idx]) { + return; + } + + std::lock_guard lock(tile_var_offsets_mtx_[idx]); + + if (loaded_metadata_.tile_var_offsets_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_var_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_var_offsets_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_var_offsets(idx, deserializer); + + loaded_metadata_.tile_var_offsets_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_var_offsets( + unsigned idx, Deserializer& deserializer) { + uint64_t tile_var_offsets_num = 0; + + // Get number of tile offsets + tile_var_offsets_num = deserializer.read(); + + // Get variable tile offsets + if (tile_var_offsets_num != 0) { + auto size = tile_var_offsets_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile var offsets; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_var_offsets_[idx].resize(tile_var_offsets_num); + deserializer.read(&tile_var_offsets_[idx][0], size); + } +} + +void OndemandFragmentMetadata::load_tile_var_sizes( + const EncryptionKey& encryption_key, unsigned idx) { + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_var_sizes_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_var_sizes_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_var_sizes_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_var_sizes(idx, deserializer); + + loaded_metadata_.tile_var_sizes_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_var_sizes( + unsigned idx, Deserializer& deserializer) { + uint64_t tile_var_sizes_num = 0; + + // Get number of tile sizes + tile_var_sizes_num = deserializer.read(); + + // Get variable tile sizes + if (tile_var_sizes_num != 0) { + auto size = tile_var_sizes_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile var sizes; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_var_sizes_[idx].resize(tile_var_sizes_num); + deserializer.read(&tile_var_sizes_[idx][0], size); + } +} + +void OndemandFragmentMetadata::load_tile_validity_offsets( + const EncryptionKey& encryption_key, unsigned idx) { + if (parent_fragment_.version_ <= 6) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_validity_offsets_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_validity_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_validity_offsets_size", tile->size()); + + ConstBuffer cbuff(tile->data(), tile->size()); + load_tile_validity_offsets(idx, &cbuff); + + loaded_metadata_.tile_validity_offsets_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_validity_offsets( + unsigned idx, ConstBuffer* buff) { + uint64_t tile_validity_offsets_num = 0; + + // Get number of tile offsets + if (!buff->read(&tile_validity_offsets_num, sizeof(uint64_t)).ok()) { + throw FragmentMetadataStatusException( + "Cannot load fragment metadata; Reading number of validity tile " + "offsets failed"); + } + + // Get tile offsets + if (tile_validity_offsets_num != 0) { + auto size = tile_validity_offsets_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile validity offsets; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_validity_offsets_[idx].resize(tile_validity_offsets_num); + if (!buff->read(&tile_validity_offsets_[idx][0], size).ok()) { + throw FragmentMetadataStatusException( + "Cannot load fragment metadata; Reading validity tile offsets " + "failed"); + } + } +} + +// ===== FORMAT ===== +// tile_min_values#0_size_buffer (uint64_t) +// tile_min_values#0_size_buffer_var (uint64_t) +// tile_min_values#0_buffer +// tile_min_values#0_buffer_var +// ... +// tile_min_values#_size_buffer (uint64_t) +// tile_min_values#_size_buffer_var (uint64_t) +// tile_min_values#_buffer +// tile_min_values#_buffer_var +void OndemandFragmentMetadata::load_tile_min_values( + unsigned idx, Deserializer& deserializer) { + uint64_t buffer_size = 0; + uint64_t var_buffer_size = 0; + + // Get buffer size + buffer_size = deserializer.read(); + + // Get var buffer size + var_buffer_size = deserializer.read(); + + // Get tile mins + if (buffer_size != 0) { + auto size = buffer_size + var_buffer_size; + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_MIN_VALS)) { + throw FragmentMetadataStatusException( + "Cannot load min values; Insufficient memory budget; Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_min_buffer_[idx].resize(buffer_size); + deserializer.read(&tile_min_buffer_[idx][0], buffer_size); + + if (var_buffer_size) { + tile_min_var_buffer_[idx].resize(var_buffer_size); + deserializer.read(&tile_min_var_buffer_[idx][0], var_buffer_size); + } + } +} + +// ===== FORMAT ===== +// tile_max_values#0_size_buffer (uint64_t) +// tile_max_values#0_size_buffer_var (uint64_t) +// tile_max_values#0_buffer +// tile_max_values#0_buffer_var +// ... +// tile_max_values#_size_buffer (uint64_t) +// tile_max_values#_size_buffer_var (uint64_t) +// tile_max_values#_buffer +// tile_max_values#_buffer_var +void OndemandFragmentMetadata::load_tile_max_values( + unsigned idx, Deserializer& deserializer) { + uint64_t buffer_size = 0; + uint64_t var_buffer_size = 0; + + // Get buffer size + buffer_size = deserializer.read(); + + // Get var buffer size + var_buffer_size = deserializer.read(); + + // Get tile maxs + if (buffer_size != 0) { + auto size = buffer_size + var_buffer_size; + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_MAX_VALS)) { + throw FragmentMetadataStatusException( + "Cannot load max values; Insufficient memory budget; Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_max_buffer_[idx].resize(buffer_size); + deserializer.read(&tile_max_buffer_[idx][0], buffer_size); + + if (var_buffer_size) { + tile_max_var_buffer_[idx].resize(var_buffer_size); + deserializer.read(&tile_max_var_buffer_[idx][0], var_buffer_size); + } + } +} + +// ===== FORMAT ===== +// tile_sum_values_attr#0_num (uint64_t) +// tile_sum_value_attr#0_#1 (uint64_t) tile_sum_value_attr#0_#2 (uint64_t) +// ... +// ... +// tile_sum_values_attr#_num (uint64_t) +// tile_sum_value_attr#_#1 (uint64_t) +// tile_sum_value_attr#_#2 (uint64_t) ... +void OndemandFragmentMetadata::load_tile_sum_values( + unsigned idx, Deserializer& deserializer) { + uint64_t tile_sum_num = 0; + + // Get number of tile sums + tile_sum_num = deserializer.read(); + + // Get tile sums + if (tile_sum_num != 0) { + auto size = tile_sum_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_SUMS)) { + throw FragmentMetadataStatusException( + "Cannot load sum values; Insufficient memory budget; Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_sums_[idx].resize(size); + deserializer.read(tile_sums_[idx].data(), size); + } +} + +// ===== FORMAT ===== +// tile_nc_values_attr#0_num (uint64_t) +// tile_nc_value_attr#0_#1 (uint64_t) tile_nc_value_attr#0_#2 (uint64_t) ... +// ... +// tile_nc_values_attr#_num (uint64_t) +// tile_nc_value_attr#_#1 (uint64_t) +// tile_nc_value_attr#_#2 (uint64_t) ... +void OndemandFragmentMetadata::load_tile_null_count_values( + unsigned idx, Deserializer& deserializer) { + uint64_t tile_null_count_num = 0; + + // Get number of tile null counts + tile_null_count_num = deserializer.read(); + + // Get tile null count + if (tile_null_count_num != 0) { + auto size = tile_null_count_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_NULL_COUNTS)) { + throw FragmentMetadataStatusException( + "Cannot load null count values; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + tile_null_counts_[idx].resize(tile_null_count_num); + deserializer.read(&tile_null_counts_[idx][0], size); + } +} + +// ===== FORMAT ===== +// condition_num (uint64_t) +// processed_condition_size#0 (uint64_t) +// processed_condition#0 +// ... +// processed_condition_size# (uint64_t) +// processed_condition# +void OndemandFragmentMetadata::load_processed_conditions( + Deserializer& deserializer) { + // Get num conditions. + uint64_t num; + num = deserializer.read(); + + processed_conditions_.reserve(num); + for (uint64_t i = 0; i < num; i++) { + uint64_t size; + size = deserializer.read(); + + std::string condition; + condition.resize(size); + deserializer.read(condition.data(), size); + + processed_conditions_.emplace_back(condition); + } + + processed_conditions_set_ = std::unordered_set( + processed_conditions_.begin(), processed_conditions_.end()); +} + +// ===== FORMAT ===== +// fragment_min_size_attr#0 (uint64_t) +// fragment_min_attr#0 (min_size) +// fragment_max_size_attr#0 (uint64_t) +// fragment_max_attr#0 (max_size) +// fragment_sum_attr#0 (uint64_t) +// fragment_null_count_attr#0 (uint64_t) +// ... +// fragment_min_size_attr# (uint64_t) +// fragment_min_attr# (min_size) +// fragment_max_size_attr# (uint64_t) +// fragment_max_attr# (max_size) +// fragment_sum_attr# (uint64_t) +// fragment_null_count_attr# (uint64_t) +void OndemandFragmentMetadata::load_fragment_min_max_sum_null_count( + Deserializer& deserializer) { + auto num = parent_fragment_.num_dims_and_attrs(); + + for (unsigned int i = 0; i < num; ++i) { + // Get min. + uint64_t min_size; + min_size = deserializer.read(); + + fragment_mins_[i].resize(min_size); + deserializer.read(fragment_mins_[i].data(), min_size); + + // Get max. + uint64_t max_size; + max_size = deserializer.read(); + + fragment_maxs_[i].resize(max_size); + deserializer.read(fragment_maxs_[i].data(), max_size); + + // Get sum. + fragment_sums_[i] = deserializer.read(); + + // Get null count. + fragment_null_counts_[i] = deserializer.read(); + } +} + +void OndemandFragmentMetadata::load_tile_min_values( + const EncryptionKey& encryption_key, unsigned idx) { + if (parent_fragment_.version_ < constants::tile_metadata_min_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_min_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_min_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_min_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_min_values(idx, deserializer); + + loaded_metadata_.tile_min_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_max_values( + const EncryptionKey& encryption_key, unsigned idx) { + if (parent_fragment_.version_ < constants::tile_metadata_min_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_max_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_max_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_max_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_max_values(idx, deserializer); + + loaded_metadata_.tile_max_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_sum_values( + const EncryptionKey& encryption_key, unsigned idx) { + if (parent_fragment_.version_ < constants::tile_metadata_min_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_sum_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, parent_fragment_.gt_offsets_.tile_sum_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_sum_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_sum_values(idx, deserializer); + + loaded_metadata_.tile_sum_[idx] = true; +} + +void OndemandFragmentMetadata::load_tile_null_count_values( + const EncryptionKey& encryption_key, unsigned idx) { + if (parent_fragment_.version_ < constants::tile_metadata_min_version) { + return; + } + + std::lock_guard lock(parent_fragment_.mtx_); + + if (loaded_metadata_.tile_null_count_[idx]) { + return; + } + + auto tile = parent_fragment_.read_generic_tile_from_file( + encryption_key, + parent_fragment_.gt_offsets_.tile_null_count_offsets_[idx]); + parent_fragment_.resources_->stats().add_counter( + "read_tile_null_count_size", tile->size()); + + Deserializer deserializer(tile->data(), tile->size()); + load_tile_null_count_values(idx, deserializer); + + loaded_metadata_.tile_null_count_[idx] = true; +} + +void OndemandFragmentMetadata::get_tile_overlap( + const NDRange& range, + std::vector& is_default, + TileOverlap* tile_overlap) { + assert(loaded_metadata_.rtree_); + *tile_overlap = rtree_.get_tile_overlap(range, is_default); +} + +void OndemandFragmentMetadata::compute_tile_bitmap( + const Range& range, unsigned d, std::vector* tile_bitmap) { + assert(loaded_metadata_.rtree_); + rtree_.compute_tile_bitmap(range, d, tile_bitmap); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/fragment/ondemand_fragment_metadata.h b/tiledb/sm/fragment/ondemand_fragment_metadata.h new file mode 100644 index 000000000000..651741b7be3b --- /dev/null +++ b/tiledb/sm/fragment/ondemand_fragment_metadata.h @@ -0,0 +1,279 @@ +/** + * @file ondemand_fragment_metadata.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class OndemandFragmentMetadata. + */ + +#ifndef TILEDB_ONDEMAND_FRAGMENT_METADATA_H +#define TILEDB_ONDEMAND_FRAGMENT_METADATA_H + +#include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" +#include "tiledb/sm/fragment/loaded_fragment_metadata.h" + +namespace tiledb { +namespace sm { + +/** Collection of lazily loaded fragment metadata */ +class OndemandFragmentMetadata : public LoadedFragmentMetadata { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor. + * + * @param resources A context resources instance. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + */ + OndemandFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker); + + /* Destructor */ + ~OndemandFragmentMetadata() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Loads the R-tree from storage. */ + virtual void load_rtree(const EncryptionKey& encryption_key) override; + + /** + * Loads the min max sum null count values for the fragment. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_fragment_min_max_sum_null_count( + const EncryptionKey& encryption_key) override; + + /** + * Loads the processed conditions for the fragment. The processed conditions + * is the list of delete/update conditions that have already been applied for + * this fragment and don't need to be applied again. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_processed_conditions( + const EncryptionKey& encryption_key) override; + + /** + * Retrieves the overlap of all MBRs with the input ND range. + * + * @param range The range to use + * @param is_default If default range should be used + * @param tile_overlap The resulted tile overlap + */ + virtual void get_tile_overlap( + const NDRange& range, + std::vector& is_default, + TileOverlap* tile_overlap) override; + + /** + * Compute tile bitmap for the curent fragment/range/dimension. + * + * @param range The range to use + * @param d The dimension index + * @param tile_bitmap The resulted tile bitmap + */ + virtual void compute_tile_bitmap( + const Range& range, + unsigned d, + std::vector* tile_bitmap) override; + + private: + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** + * Loads the tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the tile offsets for the input attribute or dimension from the + * input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_offsets(unsigned idx, Deserializer& deserializer); + + /** + * Loads the variable tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the variable tile offsets for the input attribute or dimension from + * the input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_var_offsets(unsigned idx, Deserializer& deserializer); + + /** + * Loads the variable tile sizes for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_sizes( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the variable tile sizes for the input attribute or dimension from + * the input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_var_sizes(unsigned idx, Deserializer& deserializer); + + /** + * Loads the validity tile offsets for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_validity_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the validity tile offsets for the input attribute from the + * input buffer. + * + * @param idx Dimension index + * @param buff Input buffer + */ + void load_tile_validity_offsets(unsigned idx, ConstBuffer* buff); + + /** + * Loads the min values for the input attribute from the input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_min_values(unsigned idx, Deserializer& deserializer); + + /** + * Loads the min values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_min_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the max values for the input attribute from the input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_max_values(unsigned idx, Deserializer& deserializer); + + /** + * Loads the max values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_max_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the sum values for the input attribute from the input buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_sum_values(unsigned idx, Deserializer& deserializer); + + /** + * Loads the sum values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_sum_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the null count values for the input attribute from the input + * buffer. + * + * @param idx Dimension index + * @param deserializer A deserializer to read data from + */ + void load_tile_null_count_values(unsigned idx, Deserializer& deserializer); + + /** + * Loads the null count values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_null_count_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the min max sum null count values for the fragment. + * + * @param deserializer A deserializer to read data from + */ + void load_fragment_min_max_sum_null_count(Deserializer& deserializer); + + /** + * Loads the processed conditions for the fragment. + * + * @param deserializer A deserializer to read data from + */ + void load_processed_conditions(Deserializer& deserializer); +}; + +} // namespace sm +} // namespace tiledb + +#endif // TILEDB_ONDEMAND_FRAGMENT_METADATA_H diff --git a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc new file mode 100644 index 000000000000..ea33b27b4d00 --- /dev/null +++ b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc @@ -0,0 +1,267 @@ +/** + * @file v1v2preloaded_fragment_metadata.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements the V1V2PreloadedFragmentMetadata class. + */ + +#include "tiledb/common/common.h" + +#include "tiledb/common/memory_tracker.h" +#include "tiledb/sm/fragment/fragment_metadata.h" +#include "tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h" + +using namespace tiledb::common; +using namespace tiledb::type; + +namespace tiledb::sm { + +/* ********************************* */ +/* CONSTRUCTORS & DESTRUCTORS */ +/* ********************************* */ + +V1V2PreloadedFragmentMetadata::V1V2PreloadedFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker) + : LoadedFragmentMetadata(parent, memory_tracker) { +} + +/* ********************************* */ +/* API */ +/* ********************************* */ + +// Applicable only to versions 1 and 2 +void V1V2PreloadedFragmentMetadata::load_tile_offsets( + Deserializer& deserializer) { + uint64_t tile_offsets_num = 0; + unsigned int attribute_num = parent_fragment_.array_schema_->attribute_num(); + + // Allocate tile offsets + tile_offsets_.resize(attribute_num + 1); + tile_offsets_mtx_.resize(attribute_num + 1); + + // For all attributes, get the tile offsets + for (unsigned int i = 0; i < attribute_num + 1; ++i) { + // Get number of tile offsets + tile_offsets_num = deserializer.read(); + + if (tile_offsets_num == 0) + continue; + + auto size = tile_offsets_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile offsets; Insufficient memory budget; Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + // Get tile offsets + tile_offsets_[i].resize(tile_offsets_num); + deserializer.read(&tile_offsets_[i][0], size); + } + + loaded_metadata_.tile_offsets_.resize( + parent_fragment_.array_schema_->attribute_num() + 1, true); +} + +// ===== FORMAT ===== +// tile_var_offsets_attr#0_num (uint64_t) +// tile_var_offsets_attr#0_#1 (uint64_t) tile_var_offsets_attr#0_#2 +// (uint64_t) +// ... +// ... +// tile_var_offsets_attr#_num(uint64_t) +// tile_var_offsets_attr#_#1 (uint64_t) +// tile_ver_offsets_attr#_#2 (uint64_t) ... +void V1V2PreloadedFragmentMetadata::load_tile_var_offsets( + Deserializer& deserializer) { + unsigned int attribute_num = parent_fragment_.array_schema_->attribute_num(); + uint64_t tile_var_offsets_num = 0; + + // Allocate tile offsets + tile_var_offsets_.resize(attribute_num); + tile_var_offsets_mtx_.resize(attribute_num); + + // For all attributes, get the variable tile offsets + for (unsigned int i = 0; i < attribute_num; ++i) { + // Get number of tile offsets + tile_var_offsets_num = deserializer.read(); + + if (tile_var_offsets_num == 0) + continue; + + auto size = tile_var_offsets_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile var offsets; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + // Get variable tile offsets + tile_var_offsets_[i].resize(tile_var_offsets_num); + deserializer.read(&tile_var_offsets_[i][0], size); + } + + loaded_metadata_.tile_var_offsets_.resize( + parent_fragment_.array_schema_->attribute_num(), true); +} + +// ===== FORMAT ===== +// tile_var_sizes_attr#0_num (uint64_t) +// tile_var_sizes_attr#0_#1 (uint64_t) tile_sizes_attr#0_#2 (uint64_t) ... +// ... +// tile_var_sizes_attr#_num(uint64_t) +// tile_var_sizes__attr#_#1 (uint64_t) +// tile_var_sizes_attr#_#2 (uint64_t) ... +void V1V2PreloadedFragmentMetadata::load_tile_var_sizes( + Deserializer& deserializer) { + unsigned int attribute_num = parent_fragment_.array_schema_->attribute_num(); + uint64_t tile_var_sizes_num = 0; + + // Allocate tile sizes + tile_var_sizes_.resize(attribute_num); + + // For all attributes, get the variable tile sizes + for (unsigned int i = 0; i < attribute_num; ++i) { + // Get number of tile sizes + tile_var_sizes_num = deserializer.read(); + + if (tile_var_sizes_num == 0) + continue; + + auto size = tile_var_sizes_num * sizeof(uint64_t); + if (memory_tracker_ != nullptr && + !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { + throw FragmentMetadataStatusException( + "Cannot load tile var sizes; Insufficient memory budget; " + "Needed " + + std::to_string(size) + " but only had " + + std::to_string(memory_tracker_->get_memory_available()) + + " from budget " + + std::to_string(memory_tracker_->get_memory_budget())); + } + + // Get variable tile sizes + tile_var_sizes_[i].resize(tile_var_sizes_num); + deserializer.read(&tile_var_sizes_[i][0], size); + } + + loaded_metadata_.tile_var_sizes_.resize( + parent_fragment_.array_schema_->attribute_num(), true); +} + +void V1V2PreloadedFragmentMetadata::load_rtree(const EncryptionKey&) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_fragment_min_max_sum_null_count( + const EncryptionKey&) { + // N/A for v1_v2 preloaded meta + return; +} + +/* ********************************* */ +/* PRIVATE METHODS */ +/* ********************************* */ + +void V1V2PreloadedFragmentMetadata::load_tile_offsets( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_var_offsets( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_var_sizes( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_validity_offsets( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_min_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_max_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_sum_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_tile_null_count_values( + const EncryptionKey&, unsigned) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::load_processed_conditions( + const EncryptionKey&) { + // N/A for v1_v2 preloaded meta + return; +} + +void V1V2PreloadedFragmentMetadata::get_tile_overlap( + const NDRange& range, + std::vector& is_default, + TileOverlap* tile_overlap) { + *tile_overlap = rtree_.get_tile_overlap(range, is_default); +} + +void V1V2PreloadedFragmentMetadata::compute_tile_bitmap( + const Range& range, unsigned d, std::vector* tile_bitmap) { + rtree_.compute_tile_bitmap(range, d, tile_bitmap); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h new file mode 100644 index 000000000000..7c1e8e5e6bfa --- /dev/null +++ b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.h @@ -0,0 +1,226 @@ +/** + * @file v1v2preloaded_fragment_metadata.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class V1V2PreloadedFragmentMetadata. + */ + +#ifndef TILEDB_V1V2PRELOADED_FRAGMENT_METADATA_H +#define TILEDB_V1V2PRELOADED_FRAGMENT_METADATA_H + +#include "tiledb/common/common.h" +#include "tiledb/common/pmr.h" +#include "tiledb/sm/fragment/loaded_fragment_metadata.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/sm/rtree/rtree.h" +#include "tiledb/sm/storage_manager/context_resources.h" + +namespace tiledb { +namespace sm { + +/** Collection of lazily loaded fragment metadata */ +class V1V2PreloadedFragmentMetadata : public LoadedFragmentMetadata { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** + * Constructor. + * + * @param resources A context resources instance. + * @param memory_tracker The memory tracker of the array this fragment + * metadata corresponds to. + */ + V1V2PreloadedFragmentMetadata( + FragmentMetadata& parent, shared_ptr memory_tracker); + + /* Destructor */ + ~V1V2PreloadedFragmentMetadata() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** + * Loads the tile offsets for the input attribute from the input buffer. + * Applicable to versions 1 and 2 + * + * @param deserializer A deserializer to read data from + */ + void load_tile_offsets(Deserializer& deserializer); + + /** + * Loads the variable tile offsets from the input buffer. + * Applicable to versions 1 and 2 + * + * @param deserializer A deserializer to read data from + */ + void load_tile_var_offsets(Deserializer& deserializer); + + /** + * Loads the variable tile sizes from the input buffer. + * + * @param deserializer A deserializer to read data from + */ + void load_tile_var_sizes(Deserializer& deserializer); + + /** + * Loads the R-tree from storage. + * + * @param deserializer A deserializer to read data from + */ + virtual void load_rtree(const EncryptionKey& encryption_key) override; + + /** + * Loads the min max sum null count values for the fragment. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_fragment_min_max_sum_null_count( + const EncryptionKey& encryption_key) override; + + /** + * Loads the processed conditions for the fragment. The processed conditions + * is the list of delete/update conditions that have already been applied for + * this fragment and don't need to be applied again. + * + * @param encryption_key The key the array got opened with. + */ + virtual void load_processed_conditions( + const EncryptionKey& encryption_key) override; + + /** + * Retrieves the overlap of all MBRs with the input ND range. + * + * @param range The range to use + * @param is_default If default range should be used + * @param tile_overlap The resulted tile overlap + */ + virtual void get_tile_overlap( + const NDRange& range, + std::vector& is_default, + TileOverlap* tile_overlap) override; + + /** + * Compute tile bitmap for the curent fragment/range/dimension. + * + * @param range The range to use + * @param d The dimension index + * @param tile_bitmap The resulted tile bitmap + */ + virtual void compute_tile_bitmap( + const Range& range, + unsigned d, + std::vector* tile_bitmap) override; + + private: + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** + * Loads the tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the variable tile offsets for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the variable tile sizes for the input attribute or dimension idx + * from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_var_sizes( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the validity tile offsets for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_validity_offsets( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the min values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_min_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the max values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_max_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the sum values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_sum_values( + const EncryptionKey& encryption_key, unsigned idx) override; + + /** + * Loads the null count values for the input attribute idx from storage. + * + * @param encription_key The encryption key + * @param idx Dimension index + */ + virtual void load_tile_null_count_values( + const EncryptionKey& encryption_key, unsigned idx) override; +}; + +} // namespace sm +} // namespace tiledb + +#endif // TILEDB_V1V2PRELOADED_FRAGMENT_METADATA_H diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index fb4a52482b41..f48c39148a00 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -646,8 +646,10 @@ void QueryCondition::apply_ast_node( // delete condition was already processed, GT condition is always true. if (field_name == constants::delete_timestamps && (!fragment_metadata[f]->has_delete_meta() || - fragment_metadata[f]->get_processed_conditions_set().count( - condition_marker_) != 0)) { + fragment_metadata[f] + ->loaded_metadata() + ->get_processed_conditions_set() + .count(condition_marker_) != 0)) { assert(Op == QueryConditionOp::GT); for (size_t c = starting_index; c < starting_index + length; ++c) { result_cell_bitmap[c] = 1; diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index 4b66a38dd3a2..da4b5c9c9efb 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -327,7 +327,8 @@ class FilteredData { */ inline void* fixed_filtered_data( const FragmentMetadata* fragment, const ResultTile* rt) { - auto offset{fragment->file_offset(name_, rt->tile_idx())}; + auto offset{ + fragment->loaded_metadata()->file_offset(name_, rt->tile_idx())}; ensure_data_block_current(TileType::FIXED, fragment, rt, offset); return current_data_block(TileType::FIXED)->data_at(offset); } @@ -345,7 +346,8 @@ class FilteredData { return nullptr; } - auto offset{fragment->file_var_offset(name_, rt->tile_idx())}; + auto offset{ + fragment->loaded_metadata()->file_var_offset(name_, rt->tile_idx())}; ensure_data_block_current(TileType::VAR, fragment, rt, offset); return current_data_block(TileType::VAR)->data_at(offset); } @@ -363,7 +365,8 @@ class FilteredData { return nullptr; } - auto offset{fragment->file_validity_offset(name_, rt->tile_idx())}; + auto offset{fragment->loaded_metadata()->file_validity_offset( + name_, rt->tile_idx())}; ensure_data_block_current(TileType::NULLABLE, fragment, rt, offset); return current_data_block(TileType::NULLABLE)->data_at(offset); } @@ -443,11 +446,12 @@ class FilteredData { const uint64_t tile_idx) { switch (type) { case TileType::FIXED: - return fragment->file_offset(name_, tile_idx); + return fragment->loaded_metadata()->file_offset(name_, tile_idx); case TileType::VAR: - return fragment->file_var_offset(name_, tile_idx); + return fragment->loaded_metadata()->file_var_offset(name_, tile_idx); case TileType::NULLABLE: - return fragment->file_validity_offset(name_, tile_idx); + return fragment->loaded_metadata()->file_validity_offset( + name_, tile_idx); default: throw std::logic_error("Unexpected"); } @@ -467,11 +471,14 @@ class FilteredData { const uint64_t tile_idx) { switch (type) { case TileType::FIXED: - return fragment->persisted_tile_size(name_, tile_idx); + return fragment->loaded_metadata()->persisted_tile_size( + name_, tile_idx); case TileType::VAR: - return fragment->persisted_tile_var_size(name_, tile_idx); + return fragment->loaded_metadata()->persisted_tile_var_size( + name_, tile_idx); case TileType::NULLABLE: - return fragment->persisted_tile_validity_size(name_, tile_idx); + return fragment->loaded_metadata()->persisted_tile_validity_size( + name_, tile_idx); default: throw std::logic_error("Unexpected"); } diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index 33d80255b61a..b385649d9c6b 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -329,8 +329,10 @@ void OrderedDimLabelReader::load_label_min_max_values() { [&](const uint64_t i) { auto& fragment = fragment_metadata_[i]; std::vector names = {label_name_}; - fragment->load_tile_min_values(*encryption_key, names); - fragment->load_tile_max_values(*encryption_key, names); + fragment->loaded_metadata()->load_tile_min_values( + *encryption_key, names); + fragment->loaded_metadata()->load_tile_max_values( + *encryption_key, names); return Status::Ok(); })); } @@ -450,7 +452,8 @@ OrderedDimLabelReader::get_array_tile_indexes_for_range( uint64_t OrderedDimLabelReader::label_tile_size(unsigned f, uint64_t t) const { uint64_t tile_size = fragment_metadata_[f]->tile_size(label_name_, t); if (label_var_size_) { - tile_size += fragment_metadata_[f]->tile_var_size(label_name_, t); + tile_size += + fragment_metadata_[f]->loaded_metadata()->tile_var_size(label_name_, t); } return tile_size; diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index e380de81c308..db093b5de563 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -450,7 +450,8 @@ void ReaderBase::load_tile_offsets( filtered_names.emplace_back(name); } - fragment->load_tile_offsets(*encryption_key, filtered_names); + fragment->loaded_metadata()->load_tile_offsets( + *encryption_key, filtered_names); return Status::Ok(); })); } @@ -482,7 +483,8 @@ void ReaderBase::load_tile_var_sizes( continue; } - fragment->load_tile_var_sizes(*encryption_key, name); + fragment->loaded_metadata()->load_tile_var_sizes( + *encryption_key, name); } return Status::Ok(); @@ -518,10 +520,14 @@ void ReaderBase::load_tile_metadata( } } - fragment->load_tile_max_values(*encryption_key, to_load); - fragment->load_tile_min_values(*encryption_key, to_load); - fragment->load_tile_sum_values(*encryption_key, to_load); - fragment->load_tile_null_count_values(*encryption_key, to_load); + fragment->loaded_metadata()->load_tile_max_values( + *encryption_key, to_load); + fragment->loaded_metadata()->load_tile_min_values( + *encryption_key, to_load); + fragment->loaded_metadata()->load_tile_sum_values( + *encryption_key, to_load); + fragment->loaded_metadata()->load_tile_null_count_values( + *encryption_key, to_load); return Status::Ok(); })); @@ -540,7 +546,8 @@ void ReaderBase::load_processed_conditions() { auto& fragment = fragment_metadata_[i]; if (fragment->has_delete_meta()) { - fragment->load_processed_conditions(*encryption_key); + fragment->loaded_metadata()->load_processed_conditions( + *encryption_key); } return Status::Ok(); @@ -1058,7 +1065,8 @@ uint64_t ReaderBase::get_attribute_tile_size( tile_size += fragment_metadata_[f]->tile_size(name, t); if (array_schema_.var_size(name)) { - tile_size += fragment_metadata_[f]->tile_var_size(name, t); + tile_size += + fragment_metadata_[f]->loaded_metadata()->tile_var_size(name, t); } if (array_schema_.is_nullable(name)) { diff --git a/tiledb/sm/query/readers/result_tile.h b/tiledb/sm/query/readers/result_tile.h index e303562969ed..72cef169e50b 100644 --- a/tiledb/sm/query/readers/result_tile.h +++ b/tiledb/sm/query/readers/result_tile.h @@ -95,24 +95,30 @@ class ResultTile { const bool validity_only, const uint64_t tile_idx) : tile_size_(validity_only ? 0 : fragment->tile_size(name, tile_idx)) - , tile_persisted_size_(fragment->persisted_tile_size(name, tile_idx)) + , tile_persisted_size_( + fragment->loaded_metadata()->persisted_tile_size(name, tile_idx)) , tile_var_size_( var_size && !validity_only ? - std::optional(fragment->tile_var_size(name, tile_idx)) : + std::optional(fragment->loaded_metadata()->tile_var_size( + name, tile_idx)) : std::nullopt) , tile_var_persisted_size_( - var_size ? std::optional(fragment->persisted_tile_var_size( - name, tile_idx)) : - std::nullopt) + var_size ? + std::optional( + fragment->loaded_metadata()->persisted_tile_var_size( + name, tile_idx)) : + std::nullopt) , tile_validity_size_( nullable ? std::optional( fragment->cell_num(tile_idx) * constants::cell_validity_size) : std::nullopt) , tile_validity_persisted_size_( - nullable ? std::optional(fragment->persisted_tile_validity_size( - name, tile_idx)) : - std::nullopt) { + nullable ? + std::optional( + fragment->loaded_metadata()->persisted_tile_validity_size( + name, tile_idx)) : + std::nullopt) { } TileSizes( diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 58b4308c0c75..6db4435d9049 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -1583,6 +1583,7 @@ void SparseGlobalOrderReader::copy_delete_meta_tiles( if (*src_buff_condition_indexes != std::numeric_limits::max()) { auto& condition_marker = fragment_metadata_[rt->frag_idx()] + ->loaded_metadata() ->get_processed_conditions() [*src_buff_condition_indexes]; converted_index = diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 7470f3e2ad1a..1f62acee4c1f 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -287,7 +287,8 @@ uint64_t SparseIndexReaderBase::get_coord_tiles_size( tiles_size += fragment_metadata_[f]->tile_size(dim_names_[d], t); if (is_dim_var_size_[d]) { - tiles_size += fragment_metadata_[f]->tile_var_size(dim_names_[d], t); + tiles_size += fragment_metadata_[f]->loaded_metadata()->tile_var_size( + dim_names_[d], t); } } } @@ -776,9 +777,10 @@ void SparseIndexReaderBase::apply_query_condition( for (uint64_t i = 0; i < delete_and_update_conditions_.size(); i++) { if (!frag_meta->has_delete_meta() || - frag_meta->get_processed_conditions_set().count( - delete_and_update_conditions_[i].condition_marker()) == - 0) { + frag_meta->loaded_metadata() + ->get_processed_conditions_set() + .count(delete_and_update_conditions_[i] + .condition_marker()) == 0) { auto delete_timestamp = delete_and_update_conditions_[i].condition_timestamp(); diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index ec9dc6d35acf..3dceedf8dba1 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -292,7 +292,7 @@ void SparseUnorderedWithDupsReader::load_tile_offsets_data() { for (unsigned f = tile_offsets_min_frag_idx_; f < tile_offsets_max_frag_idx_; f++) { - fragment_metadata_[f]->free_tile_offsets(); + fragment_metadata_[f]->loaded_metadata()->free_tile_offsets(); } tile_offsets_min_frag_idx_ = tile_offsets_max_frag_idx_; diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index 7289dc4c2e0c..ca66dbe7eacc 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -42,8 +42,10 @@ #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/query_condition_combination_op.h" #include "tiledb/sm/enums/query_condition_op.h" +#include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/query/readers/result_cell_slab.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include #include @@ -1131,7 +1133,9 @@ void test_apply_cells( HERE(), nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -1168,7 +1172,9 @@ void test_apply_cells( HERE(), nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -1318,7 +1324,9 @@ void test_apply_cells( HERE(), nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -1602,7 +1610,9 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -1661,7 +1671,9 @@ void test_apply(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -1773,7 +1785,9 @@ TEST_CASE( HERE(), nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -2313,7 +2327,9 @@ void test_apply_dense( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -2372,7 +2388,9 @@ void test_apply_dense(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -2483,7 +2501,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -3016,7 +3036,9 @@ void test_apply_sparse( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -3075,7 +3097,9 @@ void test_apply_sparse(const Datatype type, bool var_size, bool nullable) { FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -3227,7 +3251,9 @@ void validate_qc_apply( HERE(), nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -3850,7 +3876,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -4139,7 +4167,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -4496,7 +4526,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -4818,7 +4850,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); @@ -4920,7 +4954,9 @@ TEST_CASE( FragmentMetadata frag_md( nullptr, array_schema, - URI(), + URI(tiledb::storage_format::generate_timestamped_name( + tiledb::sm::utils::time::timestamp_now_ms(), + constants::format_version)), std::make_pair(0, 0), memory_tracker, true); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 8f659f9c7cf8..d297410969c5 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -311,7 +311,10 @@ void WriterBase::refresh_config() { shared_ptr WriterBase::create_fragment_metadata() { return make_shared( - HERE(), &storage_manager_->resources(), query_memory_tracker_); + HERE(), + &storage_manager_->resources(), + query_memory_tracker_, + array_->array_schema_latest().write_version()); } Status WriterBase::add_written_fragment_info(const URI& uri) { diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 1c56887dd36b..0791ebb1503e 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -307,11 +307,14 @@ void array_from_capnp( fragment_metadata.reserve(fragment_metadata_all_reader.size()); for (auto frag_meta_reader : fragment_metadata_all_reader) { auto meta = make_shared( - HERE(), &resources, array->memory_tracker()); + HERE(), + &resources, + array->memory_tracker(), + frag_meta_reader.getVersion()); throw_if_not_ok(fragment_metadata_from_capnp( array->array_schema_latest_ptr(), frag_meta_reader, meta)); if (client_side) { - meta->set_rtree_loaded(); + meta->loaded_metadata()->set_rtree_loaded(); } fragment_metadata.emplace_back(meta); } diff --git a/tiledb/sm/serialization/fragment_info.cc b/tiledb/sm/serialization/fragment_info.cc index 3c18e57d139b..a26071d38ecd 100644 --- a/tiledb/sm/serialization/fragment_info.cc +++ b/tiledb/sm/serialization/fragment_info.cc @@ -228,7 +228,10 @@ single_fragment_info_from_capnp( auto memory_tracker = fragment_info->resources()->create_memory_tracker(); meta = make_shared( - HERE(), fragment_info->resources(), memory_tracker); + HERE(), + fragment_info->resources(), + memory_tracker, + frag_meta_reader.getVersion()); auto st = fragment_metadata_from_capnp(schema->second, frag_meta_reader, meta); } else { @@ -251,7 +254,7 @@ single_fragment_info_from_capnp( expanded_non_empty_domain, meta}; // This is needed so that we don't try to load rtee from disk - single_frag_info.meta()->set_rtree_loaded(); + single_frag_info.meta()->loaded_metadata()->set_rtree_loaded(); return {Status::Ok(), single_frag_info}; } diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index 999c860db8fb..10aadaad73de 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -162,7 +162,7 @@ Status fragment_metadata_from_capnp( frag_meta->set_schema_name(array_schema->name()); frag_meta->set_dense(array_schema->dense()); - FragmentMetadata::LoadedMetadata loaded_metadata; + LoadedFragmentMetadata::LoadedMetadata loaded_metadata; // num_dims_and_attrs() requires a set array schema, so it's important // schema is set above on the fragment metadata object. @@ -172,7 +172,7 @@ Status fragment_metadata_from_capnp( // refactored query, but readers on the server side require these vectors to // have the first dimension properly allocated when loading their data on // demand. - frag_meta->resize_tile_offsets_vectors(num_dims_and_attrs); + frag_meta->loaded_metadata()->resize_tile_offsets_vectors(num_dims_and_attrs); loaded_metadata.tile_offsets_.resize(num_dims_and_attrs, false); // There is a difference in the metadata loaded for versions >= 2 @@ -182,7 +182,7 @@ Status fragment_metadata_from_capnp( auto tileoffsets_reader = frag_meta_reader.getTileOffsets(); uint64_t i = 0; for (const auto& t : tileoffsets_reader) { - auto& last = frag_meta->tile_offsets()[i]; + auto& last = frag_meta->loaded_metadata()->tile_offsets()[i]; last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -195,13 +195,14 @@ Status fragment_metadata_from_capnp( // refactored query, but readers on the server side require these vectors to // have the first dimension properly allocated when loading its data on // demand. - frag_meta->resize_tile_var_offsets_vectors(num_dims_and_attrs); + frag_meta->loaded_metadata()->resize_tile_var_offsets_vectors( + num_dims_and_attrs); loaded_metadata.tile_var_offsets_.resize(num_dims_and_attrs, false); if (frag_meta_reader.hasTileVarOffsets()) { auto tilevaroffsets_reader = frag_meta_reader.getTileVarOffsets(); uint64_t i = 0; for (const auto& t : tilevaroffsets_reader) { - auto& last = frag_meta->tile_var_offsets()[i]; + auto& last = frag_meta->loaded_metadata()->tile_var_offsets()[i]; last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -214,13 +215,14 @@ Status fragment_metadata_from_capnp( // refactored query, but readers on the server side require these vectors to // have the first dimension properly allocated when loading its data on // demand. - frag_meta->resize_tile_var_sizes_vectors(num_dims_and_attrs); + frag_meta->loaded_metadata()->resize_tile_var_sizes_vectors( + num_dims_and_attrs); loaded_metadata.tile_var_sizes_.resize(num_dims_and_attrs, false); if (frag_meta_reader.hasTileVarSizes()) { auto tilevarsizes_reader = frag_meta_reader.getTileVarSizes(); uint64_t i = 0; for (const auto& t : tilevarsizes_reader) { - auto& last = frag_meta->tile_var_sizes()[i]; + auto& last = frag_meta->loaded_metadata()->tile_var_sizes()[i]; last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -232,13 +234,14 @@ Status fragment_metadata_from_capnp( // This field may not be present here in some usecases such as refactored // query, but readers on the server side require this vector to have the first // dimension properly allocated when loading its data on demand. - frag_meta->resize_tile_validity_offsets_vectors(num_dims_and_attrs); + frag_meta->loaded_metadata()->resize_tile_validity_offsets_vectors( + num_dims_and_attrs); loaded_metadata.tile_validity_offsets_.resize(num_dims_and_attrs, false); if (frag_meta_reader.hasTileValidityOffsets()) { auto tilevalidityoffsets_reader = frag_meta_reader.getTileValidityOffsets(); uint64_t i = 0; for (const auto& t : tilevalidityoffsets_reader) { - auto& last = frag_meta->tile_validity_offsets()[i]; + auto& last = frag_meta->loaded_metadata()->tile_validity_offsets()[i]; last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -249,7 +252,8 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileMinBuffer()) { auto tileminbuffer_reader = frag_meta_reader.getTileMinBuffer(); for (const auto& t : tileminbuffer_reader) { - auto& last = frag_meta->tile_min_buffer().emplace_back(); + auto& last = + frag_meta->loaded_metadata()->tile_min_buffer().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -260,7 +264,8 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileMinVarBuffer()) { auto tileminvarbuffer_reader = frag_meta_reader.getTileMinVarBuffer(); for (const auto& t : tileminvarbuffer_reader) { - auto& last = frag_meta->tile_min_var_buffer().emplace_back(); + auto& last = + frag_meta->loaded_metadata()->tile_min_var_buffer().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -270,7 +275,8 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileMaxBuffer()) { auto tilemaxbuffer_reader = frag_meta_reader.getTileMaxBuffer(); for (const auto& t : tilemaxbuffer_reader) { - auto& last = frag_meta->tile_max_buffer().emplace_back(); + auto& last = + frag_meta->loaded_metadata()->tile_max_buffer().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -281,7 +287,8 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileMaxVarBuffer()) { auto tilemaxvarbuffer_reader = frag_meta_reader.getTileMaxVarBuffer(); for (const auto& t : tilemaxvarbuffer_reader) { - auto& last = frag_meta->tile_max_var_buffer().emplace_back(); + auto& last = + frag_meta->loaded_metadata()->tile_max_var_buffer().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -291,7 +298,7 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileSums()) { auto tilesums_reader = frag_meta_reader.getTileSums(); for (const auto& t : tilesums_reader) { - auto& last = frag_meta->tile_sums().emplace_back(); + auto& last = frag_meta->loaded_metadata()->tile_sums().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -302,7 +309,8 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasTileNullCounts()) { auto tilenullcounts_reader = frag_meta_reader.getTileNullCounts(); for (const auto& t : tilenullcounts_reader) { - auto& last = frag_meta->tile_null_counts().emplace_back(); + auto& last = + frag_meta->loaded_metadata()->tile_null_counts().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -314,7 +322,7 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasFragmentMins()) { auto fragmentmins_reader = frag_meta_reader.getFragmentMins(); for (const auto& t : fragmentmins_reader) { - auto& last = frag_meta->fragment_mins().emplace_back(); + auto& last = frag_meta->loaded_metadata()->fragment_mins().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -324,7 +332,7 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasFragmentMaxs()) { auto fragmentmaxs_reader = frag_meta_reader.getFragmentMaxs(); for (const auto& t : fragmentmaxs_reader) { - auto& last = frag_meta->fragment_maxs().emplace_back(); + auto& last = frag_meta->loaded_metadata()->fragment_maxs().emplace_back(); last.reserve(t.size()); for (const auto& v : t) { last.emplace_back(v); @@ -333,16 +341,19 @@ Status fragment_metadata_from_capnp( } if (frag_meta_reader.hasFragmentSums()) { auto fragmentsums_reader = frag_meta_reader.getFragmentSums(); - frag_meta->fragment_sums().reserve(fragmentsums_reader.size()); + frag_meta->loaded_metadata()->fragment_sums().reserve( + fragmentsums_reader.size()); for (const auto& fragment_sum : fragmentsums_reader) { - frag_meta->fragment_sums().emplace_back(fragment_sum); + frag_meta->loaded_metadata()->fragment_sums().emplace_back(fragment_sum); } } if (frag_meta_reader.hasFragmentNullCounts()) { auto fragmentnullcounts_reader = frag_meta_reader.getFragmentNullCounts(); - frag_meta->fragment_null_counts().reserve(fragmentnullcounts_reader.size()); + frag_meta->loaded_metadata()->fragment_null_counts().reserve( + fragmentnullcounts_reader.size()); for (const auto& fragment_null_count : fragmentnullcounts_reader) { - frag_meta->fragment_null_counts().emplace_back(fragment_null_count); + frag_meta->loaded_metadata()->fragment_null_counts().emplace_back( + fragment_null_count); } } @@ -358,7 +369,8 @@ Status fragment_metadata_from_capnp( auto data = frag_meta_reader.getRtree(); auto& domain = array_schema->domain(); // If there are no levels, we still need domain_ properly initialized - frag_meta->rtree().reset(&domain, constants::rtree_fanout); + frag_meta->loaded_metadata()->rtree().reset( + &domain, constants::rtree_fanout); Deserializer deserializer(data.begin(), data.size()); // What we actually deserialize is not something written on disk in a // possibly historical format, but what has been serialized in @@ -367,7 +379,7 @@ Status fragment_metadata_from_capnp( // the version of a fragment is on disk, we will be serializing _on wire_ in // fragment_metadata_to_capnp in the "modern" (post v5) way, so we need to // deserialize it as well in that way. - frag_meta->rtree().deserialize( + frag_meta->loaded_metadata()->rtree().deserialize( deserializer, &domain, constants::format_version); } @@ -390,10 +402,9 @@ Status fragment_metadata_from_capnp( if (frag_meta_reader.hasGtOffsets()) { generic_tile_offsets_from_capnp( frag_meta_reader.getGtOffsets(), frag_meta->generic_tile_offsets()); - loaded_metadata.footer_ = true; } - frag_meta->set_loaded_metadata(loaded_metadata); + frag_meta->loaded_metadata()->set_loaded_metadata(loaded_metadata); return Status::Ok(); } @@ -474,7 +485,7 @@ void generic_tile_offsets_to_capnp( void fragment_meta_sizes_offsets_to_capnp( const FragmentMetadata& frag_meta, capnp::FragmentMetadata::Builder* frag_meta_builder) { - auto& tile_offsets = frag_meta.tile_offsets(); + auto& tile_offsets = frag_meta.loaded_metadata()->tile_offsets(); if (!tile_offsets.empty()) { auto builder = frag_meta_builder->initTileOffsets(tile_offsets.size()); for (uint64_t i = 0; i < tile_offsets.size(); ++i) { @@ -484,7 +495,7 @@ void fragment_meta_sizes_offsets_to_capnp( } } } - auto& tile_var_offsets = frag_meta.tile_var_offsets(); + auto& tile_var_offsets = frag_meta.loaded_metadata()->tile_var_offsets(); if (!tile_var_offsets.empty()) { auto builder = frag_meta_builder->initTileVarOffsets(tile_var_offsets.size()); @@ -495,7 +506,7 @@ void fragment_meta_sizes_offsets_to_capnp( } } } - auto& tile_var_sizes = frag_meta.tile_var_sizes(); + auto& tile_var_sizes = frag_meta.loaded_metadata()->tile_var_sizes(); if (!tile_var_sizes.empty()) { auto builder = frag_meta_builder->initTileVarSizes(tile_var_sizes.size()); for (uint64_t i = 0; i < tile_var_sizes.size(); ++i) { @@ -505,7 +516,8 @@ void fragment_meta_sizes_offsets_to_capnp( } } } - auto& tile_validity_offsets = frag_meta.tile_validity_offsets(); + auto& tile_validity_offsets = + frag_meta.loaded_metadata()->tile_validity_offsets(); if (!tile_validity_offsets.empty()) { auto builder = frag_meta_builder->initTileValidityOffsets( tile_validity_offsets.size()); @@ -554,7 +566,7 @@ Status fragment_metadata_to_capnp( } } - auto& tile_min_buffer = frag_meta.tile_min_buffer(); + auto& tile_min_buffer = frag_meta.loaded_metadata()->tile_min_buffer(); if (!tile_min_buffer.empty()) { auto builder = frag_meta_builder->initTileMinBuffer(tile_min_buffer.size()); for (uint64_t i = 0; i < tile_min_buffer.size(); ++i) { @@ -564,7 +576,8 @@ Status fragment_metadata_to_capnp( } } } - auto& tile_min_var_buffer = frag_meta.tile_min_var_buffer(); + auto& tile_min_var_buffer = + frag_meta.loaded_metadata()->tile_min_var_buffer(); if (!tile_min_var_buffer.empty()) { auto builder = frag_meta_builder->initTileMinVarBuffer(tile_min_var_buffer.size()); @@ -575,7 +588,7 @@ Status fragment_metadata_to_capnp( } } } - auto& tile_max_buffer = frag_meta.tile_max_buffer(); + auto& tile_max_buffer = frag_meta.loaded_metadata()->tile_max_buffer(); if (!tile_max_buffer.empty()) { auto builder = frag_meta_builder->initTileMaxBuffer(tile_max_buffer.size()); for (uint64_t i = 0; i < tile_max_buffer.size(); ++i) { @@ -585,7 +598,8 @@ Status fragment_metadata_to_capnp( } } } - auto& tile_max_var_buffer = frag_meta.tile_max_var_buffer(); + auto& tile_max_var_buffer = + frag_meta.loaded_metadata()->tile_max_var_buffer(); if (!tile_max_var_buffer.empty()) { auto builder = frag_meta_builder->initTileMaxVarBuffer(tile_max_var_buffer.size()); @@ -596,7 +610,7 @@ Status fragment_metadata_to_capnp( } } } - auto& tile_sums = frag_meta.tile_sums(); + auto& tile_sums = frag_meta.loaded_metadata()->tile_sums(); if (!tile_sums.empty()) { auto builder = frag_meta_builder->initTileSums(tile_sums.size()); for (uint64_t i = 0; i < tile_sums.size(); ++i) { @@ -606,7 +620,7 @@ Status fragment_metadata_to_capnp( } } } - auto& tile_null_counts = frag_meta.tile_null_counts(); + auto& tile_null_counts = frag_meta.loaded_metadata()->tile_null_counts(); if (!tile_null_counts.empty()) { auto builder = frag_meta_builder->initTileNullCounts(tile_null_counts.size()); @@ -617,7 +631,7 @@ Status fragment_metadata_to_capnp( } } } - auto& fragment_mins = frag_meta.fragment_mins(); + auto& fragment_mins = frag_meta.loaded_metadata()->fragment_mins(); if (!fragment_mins.empty()) { auto builder = frag_meta_builder->initFragmentMins(fragment_mins.size()); for (uint64_t i = 0; i < fragment_mins.size(); ++i) { @@ -627,7 +641,7 @@ Status fragment_metadata_to_capnp( } } } - auto& fragment_maxs = frag_meta.fragment_maxs(); + auto& fragment_maxs = frag_meta.loaded_metadata()->fragment_maxs(); if (!fragment_maxs.empty()) { auto builder = frag_meta_builder->initFragmentMaxs(fragment_maxs.size()); for (uint64_t i = 0; i < fragment_maxs.size(); ++i) { @@ -637,14 +651,15 @@ Status fragment_metadata_to_capnp( } } } - auto& fragment_sums = frag_meta.fragment_sums(); + auto& fragment_sums = frag_meta.loaded_metadata()->fragment_sums(); if (!fragment_sums.empty()) { auto builder = frag_meta_builder->initFragmentSums(fragment_sums.size()); for (uint64_t i = 0; i < fragment_sums.size(); ++i) { builder.set(i, fragment_sums[i]); } } - auto& fragment_null_counts = frag_meta.fragment_null_counts(); + auto& fragment_null_counts = + frag_meta.loaded_metadata()->fragment_null_counts(); if (!fragment_null_counts.empty()) { auto builder = frag_meta_builder->initFragmentNullCounts(fragment_null_counts.size()); @@ -669,11 +684,11 @@ Status fragment_metadata_to_capnp( // TODO: Can this be done better? Does this make a lot of copies? SizeComputationSerializer size_computation_serializer; - frag_meta.rtree().serialize(size_computation_serializer); + frag_meta.loaded_metadata()->rtree().serialize(size_computation_serializer); std::vector buff(size_computation_serializer.size()); Serializer serializer(buff.data(), buff.size()); - frag_meta.rtree().serialize(serializer); + frag_meta.loaded_metadata()->rtree().serialize(serializer); auto vec = kj::Vector(); vec.addAll( diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 1354a462b4de..4336db12ac19 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -1572,7 +1572,8 @@ void Subarray::compute_relevant_fragment_est_result_sizes( tile_size / cell_size * constants::cell_validity_size; } else { tile_size -= constants::cell_var_offset_size; - auto tile_var_size = meta->tile_var_size(names[i], ft.second); + auto tile_var_size = + meta->loaded_metadata()->tile_var_size(names[i], ft.second); mem_vec[i].size_fixed_ += tile_size; mem_vec[i].size_var_ += tile_var_size; if (nullable[i]) @@ -1896,7 +1897,8 @@ void Subarray::compute_relevant_fragment_est_result_sizes( } else { tile_size -= constants::cell_var_offset_size; (*result_sizes)[n].size_fixed_ += tile_size; - auto tile_var_size = meta->tile_var_size(name[n], tid); + auto tile_var_size = + meta->loaded_metadata()->tile_var_size(name[n], tid); (*result_sizes)[n].size_var_ += tile_var_size; if (nullable[n]) (*result_sizes)[n].size_validity_ += @@ -1936,7 +1938,8 @@ void Subarray::compute_relevant_fragment_est_result_sizes( } else { tile_size -= constants::cell_var_offset_size; (*result_sizes)[n].size_fixed_ += tile_size * ratio; - auto tile_var_size = meta->tile_var_size(name[n], tid); + auto tile_var_size = + meta->loaded_metadata()->tile_var_size(name[n], tid); (*result_sizes)[n].size_var_ += tile_var_size * ratio; if (nullable[n]) (*result_sizes)[n].size_validity_ += @@ -2265,7 +2268,7 @@ void Subarray::precompute_all_ranges_tile_overlap( const auto r_end = std::min((t + 1) * ranges_per_thread - 1, range_num - 1); for (uint64_t r = r_start; r <= r_end; ++r) { - meta[f]->compute_tile_bitmap( + meta[f]->loaded_metadata()->compute_tile_bitmap( range_subset_[d][r], d, &tile_bitmaps[d]); } return Status::Ok(); @@ -2519,7 +2522,8 @@ void Subarray::load_relevant_fragment_rtrees(ThreadPool* compute_tp) const { auto status = parallel_for(compute_tp, 0, relevant_fragments_.size(), [&](uint64_t f) { - meta[relevant_fragments_[f]]->load_rtree(*encryption_key); + meta[relevant_fragments_[f]]->loaded_metadata()->load_rtree( + *encryption_key); return Status::Ok(); }); throw_if_not_ok(status); @@ -2570,7 +2574,7 @@ void Subarray::compute_relevant_fragment_tile_overlap( compute_tile_overlap(r + tile_overlap->range_idx_start(), frag_idx); } else { // Sparse fragment const auto& range = this->ndrange(r + tile_overlap->range_idx_start()); - meta->get_tile_overlap( + meta->loaded_metadata()->get_tile_overlap( range, is_default_, tile_overlap->at(frag_idx, r)); } } @@ -2609,7 +2613,8 @@ void Subarray::load_relevant_fragment_tile_var_sizes( if (!schema->is_field(var_name)) { return Status::Ok(); } - meta[f]->load_tile_var_sizes(*encryption_key, var_name); + meta[f]->loaded_metadata()->load_tile_var_sizes( + *encryption_key, var_name); return Status::Ok(); }); throw_if_not_ok(status); diff --git a/tools/src/commands/info_command.cc b/tools/src/commands/info_command.cc index e181dd6a9948..06418ed6b597 100644 --- a/tools/src/commands/info_command.cc +++ b/tools/src/commands/info_command.cc @@ -142,15 +142,18 @@ void InfoCommand::print_tile_sizes() const { uint64_t tile_num = f->tile_num(); std::vector names; names.push_back(name); - f->load_tile_offsets(enc_key, names); - f->load_tile_var_sizes(enc_key, name); + f->loaded_metadata()->load_tile_offsets(enc_key, names); + f->loaded_metadata()->load_tile_var_sizes(enc_key, name); for (uint64_t tile_idx = 0; tile_idx < tile_num; tile_idx++) { - persisted_tile_size += f->persisted_tile_size(name, tile_idx); + persisted_tile_size += + f->loaded_metadata()->persisted_tile_size(name, tile_idx); in_memory_tile_size += f->tile_size(name, tile_idx); num_tiles++; if (var_size) { - persisted_tile_size += f->persisted_tile_var_size(name, tile_idx); - in_memory_tile_size += f->tile_var_size(name, tile_idx); + persisted_tile_size += + f->loaded_metadata()->persisted_tile_var_size(name, tile_idx); + in_memory_tile_size += + f->loaded_metadata()->tile_var_size(name, tile_idx); num_tiles++; } } @@ -298,7 +301,7 @@ void InfoCommand::write_text_mbrs() const { auto fragment_metadata = array.fragment_metadata(); std::stringstream text; for (const auto& f : fragment_metadata) { - f->load_rtree(*encryption_key); + f->loaded_metadata()->load_rtree(*encryption_key); const auto& mbrs = f->mbrs(); for (const auto& mbr : mbrs) { auto str_mbr = mbr_to_string(mbr, schema.domain()); From acfb4e8816ed9cdd858a5cb673a536ee40811aa1 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Tue, 14 May 2024 18:54:58 +0300 Subject: [PATCH 343/456] Enable array open v2 and query v3 by default. (#4974) [sc-47320] [sc-38435] This PR just changes the two configuration variables that toggle "Array open v2" and "Query v3" features to ON by default. --- TYPE: FEATURE DESC: Enable array open v2 and query v3 by default. --- test/src/unit-capi-config.cc | 12 ++++++------ tiledb/api/c_api/config/config_api_external.h | 10 +++++----- tiledb/sm/config/config.cc | 4 ++-- tiledb/sm/cpp_api/config.h | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index b0d3c63c5827..5462ca601e36 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -239,8 +239,8 @@ void check_save_to_file() { ss << "rest.retry_initial_delay_ms 500\n"; ss << "rest.server_address https://api.tiledb.com\n"; ss << "rest.server_serialization_format CAPNP\n"; - ss << "rest.use_refactored_array_open false\n"; - ss << "rest.use_refactored_array_open_and_query_submit false\n"; + ss << "rest.use_refactored_array_open true\n"; + ss << "rest.use_refactored_array_open_and_query_submit true\n"; ss << "sm.allow_separate_attribute_writes false\n"; ss << "sm.allow_updates_experimental false\n"; ss << "sm.check_coord_dups true\n"; @@ -576,13 +576,13 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { CHECK(rc == TILEDB_OK); CHECK(error == nullptr); rc = tiledb_config_set( - config, "rest.use_refactored_array_open", "true", &error); + config, "rest.use_refactored_array_open", "false", &error); CHECK(rc == TILEDB_OK); CHECK(error == nullptr); rc = tiledb_config_set( config, "rest.use_refactored_array_open_and_query_submit", - "true", + "false", &error); CHECK(rc == TILEDB_OK); CHECK(error == nullptr); @@ -610,8 +610,8 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["rest.load_metadata_on_array_open"] = "false"; all_param_values["rest.load_non_empty_domain_on_array_open"] = "false"; all_param_values["rest.load_enumerations_on_array_open"] = "false"; - all_param_values["rest.use_refactored_array_open"] = "true"; - all_param_values["rest.use_refactored_array_open_and_query_submit"] = "true"; + all_param_values["rest.use_refactored_array_open"] = "false"; + all_param_values["rest.use_refactored_array_open_and_query_submit"] = "false"; all_param_values["sm.allow_separate_attribute_writes"] = "false"; all_param_values["sm.allow_updates_experimental"] = "false"; all_param_values["sm.encryption_key"] = ""; diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 8be1968935c8..b4aaafca98f9 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -661,13 +661,13 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * with the open array
* **Default**: true * - `rest.use_refactored_array_open`
- * If true, the new, experimental REST routes and APIs for opening an array + * If true, the new REST routes and APIs for opening an array * will be used
- * **Default**: false + * **Default**: true * - `rest.use_refactored_array_open_and_query_submit`
- * If true, the new, experimental REST routes and APIs for opening an array - * and submitting a query will be used
- * **Default**: false + * If true, the new REST routes and APIs for opening an array and submitting + * a query will be used
+ * **Default**: true * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
* **Default**: 524288 (512KB) diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index a3434eddceb0..01d27c788aaf 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -94,8 +94,8 @@ const std::string Config::REST_CURL_VERBOSE = "false"; const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; -const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false"; -const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "false"; +const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "true"; +const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "true"; const std::string Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES = "false"; const std::string Config::SM_ALLOW_UPDATES_EXPERIMENTAL = "false"; const std::string Config::SM_ENCRYPTION_KEY = ""; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index e2cd3bc727b1..2cb91cf1bdaf 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -841,14 +841,14 @@ class Config { * **Default**: true * - `rest.use_refactored_array_open`
* **Experimental**
- * If true, the new, experimental REST routes and APIs for opening an array + * If true, the new REST routes and APIs for opening an array * will be used
- * **Default**: false + * **Default**: true * - `rest.use_refactored_array_open_and_query_submit`
* **Experimental**
- * If true, the new, experimental REST routes and APIs for opening an array - * and submitting a query will be used
- * **Default**: false + * If true, the new REST routes and APIs for opening an array and + * submitting a query will be used
+ * **Default**: true * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
* **Default**: 524288 (512KB) From 85f838813ea964b14591e51935966bb759cdec97 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 14 May 2024 20:38:28 -0400 Subject: [PATCH 344/456] Migrate delete_array and delete_fragments from StorageManager to Array. (#4880) Migrate `Array` APIs out of `StorageManager`: `delete_array` and `delete_fragments`. Note: this migration required addition of new, `static` versions of these internal APIs on class `Array`, which effectively do _exactly_ what the versions in `StorageManager` were previously doing. This allows for a "hierarchical-like" structure among the inter-dependent APIs `delete_group`, `delete_array`, and `delete_fragments`. --- [sc-23372] [sc-47161] [sc-47162] --- TYPE: IMPROVEMENT DESC: Migrate Array APIs out of StorageManager: delete_array and delete_fragments. --- tiledb/sm/array/array.cc | 100 +++++++++++++++--- tiledb/sm/array/array.h | 62 ++++++++++- tiledb/sm/group/group.cc | 3 +- tiledb/sm/storage_manager/storage_manager.cc | 70 ------------ .../storage_manager_canonical.h | 17 --- 5 files changed, 143 insertions(+), 109 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 52822f54537e..f5c1cf0ca103 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -485,23 +485,48 @@ Status Array::close() { return Status::Ok(); } -void Array::delete_array(const URI& uri) { - // Check that data deletion is allowed - ensure_array_is_valid_for_delete(uri); - - // Delete array data - if (remote_) { - auto rest_client = resources_.rest_client(); - if (rest_client == nullptr) { - throw ArrayException("[delete_array] Remote array with no REST client."); +void Array::delete_fragments( + ContextResources& resources, + const URI& uri, + uint64_t timestamp_start, + uint64_t timestamp_end, + std::optional array_dir) { + // Get the fragment URIs to be deleted + if (array_dir == std::nullopt) { + array_dir = ArrayDirectory(resources, uri, timestamp_start, timestamp_end); + } + auto filtered_fragment_uris = array_dir->filtered_fragment_uris(true); + const auto& fragment_uris = filtered_fragment_uris.fragment_uris(); + + // Retrieve commit uris to delete and ignore + std::vector commit_uris_to_delete; + std::vector commit_uris_to_ignore; + for (auto& fragment : fragment_uris) { + auto commit_uri = array_dir->get_commit_uri(fragment.uri_); + commit_uris_to_delete.emplace_back(commit_uri); + if (array_dir->consolidated_commit_uris_set().count(commit_uri.c_str()) != + 0) { + commit_uris_to_ignore.emplace_back(commit_uri); } - rest_client->delete_array_from_rest(uri); - } else { - storage_manager_->delete_array(uri.c_str()); } - // Close the array - throw_if_not_ok(this->close()); + // Write ignore file + if (commit_uris_to_ignore.size() != 0) { + array_dir->write_commit_ignore_file(commit_uris_to_ignore); + } + + // Delete fragments and commits + auto vfs = &(resources.vfs()); + throw_if_not_ok(parallel_for( + &resources.compute_tp(), 0, fragment_uris.size(), [&](size_t i) { + RETURN_NOT_OK(vfs->remove_dir(fragment_uris[i].uri_)); + bool is_file = false; + RETURN_NOT_OK(vfs->is_file(commit_uris_to_delete[i], &is_file)); + if (is_file) { + RETURN_NOT_OK(vfs->remove_file(commit_uris_to_delete[i])); + } + return Status::Ok(); + })); } void Array::delete_fragments( @@ -519,11 +544,54 @@ void Array::delete_fragments( rest_client->post_delete_fragments_to_rest( uri, this, timestamp_start, timestamp_end); } else { - storage_manager_->delete_fragments( - uri.c_str(), timestamp_start, timestamp_end); + Array::delete_fragments(resources_, uri, timestamp_start, timestamp_end); } } +void Array::delete_array(ContextResources& resources, const URI& uri) { + auto& vfs = resources.vfs(); + auto array_dir = + ArrayDirectory(resources, uri, 0, std::numeric_limits::max()); + + // Delete fragments and commits + Array::delete_fragments( + resources, uri, 0, std::numeric_limits::max(), array_dir); + + // Delete array metadata, fragment metadata and array schema files + // Note: metadata files may not be present, try to delete anyway + vfs.remove_files(&resources.compute_tp(), array_dir.array_meta_uris()); + vfs.remove_files(&resources.compute_tp(), array_dir.fragment_meta_uris()); + vfs.remove_files(&resources.compute_tp(), array_dir.array_schema_uris()); + + // Delete all tiledb child directories + // Note: using vfs.ls() here could delete user data + std::vector dirs; + auto parent_dir = array_dir.uri().c_str(); + for (auto array_dir_name : constants::array_dir_names) { + dirs.emplace_back(URI(parent_dir + array_dir_name)); + } + vfs.remove_dirs(&resources.compute_tp(), dirs); +} + +void Array::delete_array(const URI& uri) { + // Check that data deletion is allowed + ensure_array_is_valid_for_delete(uri); + + // Delete array data + if (uri.is_tiledb()) { + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw ArrayException("[delete_array] Remote array with no REST client."); + } + rest_client->delete_array_from_rest(uri); + } else { + Array::delete_array(resources_, uri); + } + + // Close the array + throw_if_not_ok(this->close()); +} + void Array::delete_fragments_list(const std::vector& fragment_uris) { // Check that data deletion is allowed ensure_array_is_valid_for_delete(array_uri_); diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 54ce47c60f08..64d8e74f5816 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -420,26 +420,78 @@ class Array { Status close(); /** - * Deletes the Array data with given URI. + * Performs deletion of local fragments with the given parent URI, + * between the provided timestamps. * - * @param uri The uri of the Array whose data is to be deleted. + * @param resources The context resources. + * @param uri The uri of the Array whose fragments are to be deleted. + * @param timestamp_start The start timestamp at which to delete fragments. + * @param timestamp_end The end timestamp at which to delete fragments. + * @param array_dir An optional ArrayDirectory from which to delete fragments. * - * @pre The Array must be open for exclusive writes + * @section Maturity Notes + * This is legacy code, ported from StorageManager during its removal process. + * Its existence supports the non-static `delete_fragments` API below, + * performing the actual deletion of fragments. This function is slated for + * removal and should be directly integrated into the function below. */ - void delete_array(const URI& uri); + static void delete_fragments( + ContextResources& resources, + const URI& uri, + uint64_t timestamp_start, + uint64_t timstamp_end, + std::optional array_dir = std::nullopt); /** - * Deletes the fragments from the Array with given URI. + * Handles local and remote deletion of fragments between the provided + * timestamps from an open array with the given URI. * * @param uri The uri of the Array whose fragments are to be deleted. * @param timestamp_start The start timestamp at which to delete fragments. * @param timestamp_end The end timestamp at which to delete fragments. * * @pre The Array must be open for exclusive writes + * + * @section Maturity Notes + * This API is an interim version of its final product, awaiting rewrite. + * As is, it handles the incoming URI and invokes the remote or local function + * call accordingly. The local, static function above is legacy code which + * exists only to support this function. A rewrite should integrate the two + * and remove the need for any static APIs. */ void delete_fragments( const URI& uri, uint64_t timestamp_start, uint64_t timstamp_end); + /** + * Performs deletion of the local array with the given parent URI. + * + * @param resources The context resources. + * @param uri The uri of the Array whose data is to be deleted. + * + * @section Maturity Notes + * This is legacy code, ported from StorageManager during its removal process. + * Its existence supports the non-static `delete_array` API below, + * performing the actual deletion of array data. This function is slated for + * removal and should be directly integrated into the function below. + */ + static void delete_array(ContextResources& resources, const URI& uri); + + /** + * Handles local and remote deletion of an open array with the given URI. + * + * @param uri The uri of the Array whose data is to be deleted. + * + * @pre The Array must be open for exclusive writes + * + * @section Maturity Notes + * This API is an interim version of its final product, awaiting rewrite. + * As is, it handles the incoming URI and invokes the remote or local function + * call accordingly. The local, static function above is legacy code which + * exists only to support this function. A rewrite should integrate the two + * and remove the need for any static APIs. + */ + void delete_array(const URI& uri); + /** * Deletes the fragments with the given URIs from the Array with given URI. * diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 722e7500fb0f..726bfd5e130c 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -35,6 +35,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" +#include "tiledb/sm/array/array.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/query_type.h" @@ -324,7 +325,7 @@ void Group::delete_group(const URI& uri, bool recursive) { } if (member->type() == ObjectType::ARRAY) { - storage_manager_->delete_array(member_uri.to_string().c_str()); + Array::delete_array(resources_, member_uri); } else if (member->type() == ObjectType::GROUP) { Group group_rec(resources_, member_uri, storage_manager_); throw_if_not_ok(group_rec.open(QueryType::MODIFY_EXCLUSIVE)); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 9d8f3020c945..ab8601533461 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -151,76 +151,6 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { return Status::Ok(); } -void StorageManagerCanonical::delete_array(const char* array_name) { - if (array_name == nullptr) { - throw std::invalid_argument("[delete_array] Array name cannot be null"); - } - - // Delete fragments and commits - delete_fragments(array_name, 0, std::numeric_limits::max()); - - auto array_dir = ArrayDirectory( - resources(), URI(array_name), 0, std::numeric_limits::max()); - - // Delete array metadata, fragment metadata and array schema files - // Note: metadata files may not be present, try to delete anyway - vfs()->remove_files(compute_tp(), array_dir.array_meta_uris()); - vfs()->remove_files(compute_tp(), array_dir.fragment_meta_uris()); - vfs()->remove_files(compute_tp(), array_dir.array_schema_uris()); - - // Delete all tiledb child directories - // Note: using vfs()->ls() here could delete user data - std::vector dirs; - auto parent_dir = array_dir.uri().c_str(); - for (auto array_dir_name : constants::array_dir_names) { - dirs.emplace_back(URI(parent_dir + array_dir_name)); - } - vfs()->remove_dirs(compute_tp(), dirs); -} - -void StorageManagerCanonical::delete_fragments( - const char* array_name, uint64_t timestamp_start, uint64_t timestamp_end) { - if (array_name == nullptr) { - throw std::invalid_argument("[delete_fragments] Array name cannot be null"); - } - - auto array_dir = ArrayDirectory( - resources(), URI(array_name), timestamp_start, timestamp_end); - - // Get the fragment URIs to be deleted - auto filtered_fragment_uris = array_dir.filtered_fragment_uris(true); - const auto& fragment_uris = filtered_fragment_uris.fragment_uris(); - - // Retrieve commit uris to delete and ignore - std::vector commit_uris_to_delete; - std::vector commit_uris_to_ignore; - for (auto& fragment : fragment_uris) { - auto commit_uri = array_dir.get_commit_uri(fragment.uri_); - commit_uris_to_delete.emplace_back(commit_uri); - if (array_dir.consolidated_commit_uris_set().count(commit_uri.c_str()) != - 0) { - commit_uris_to_ignore.emplace_back(commit_uri); - } - } - - // Write ignore file - if (commit_uris_to_ignore.size() != 0) { - array_dir.write_commit_ignore_file(commit_uris_to_ignore); - } - - // Delete fragments and commits - throw_if_not_ok( - parallel_for(compute_tp(), 0, fragment_uris.size(), [&](size_t i) { - RETURN_NOT_OK(vfs()->remove_dir(fragment_uris[i].uri_)); - bool is_file = false; - RETURN_NOT_OK(vfs()->is_file(commit_uris_to_delete[i], &is_file)); - if (is_file) { - RETURN_NOT_OK(vfs()->remove_file(commit_uris_to_delete[i])); - } - return Status::Ok(); - })); -} - Status StorageManagerCanonical::array_create( const URI& array_uri, const shared_ptr& array_schema, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index b1ee5bf07c31..948a79e406a9 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -171,23 +171,6 @@ class StorageManagerCanonical { tdb_shared_ptr group, const EncryptionKey& encryption_key); - /** - * Cleans up the array data. - * - * @param array_name The name of the array whose data is to be deleted. - */ - void delete_array(const char* array_name); - - /** - * Cleans up the array fragments. - * - * @param array_name The name of the array whose fragments are to be deleted. - * @param timestamp_start The start timestamp at which to delete. - * @param timestamp_end The end timestamp at which to delete. - */ - void delete_fragments( - const char* array_name, uint64_t timestamp_start, uint64_t timestamp_end); - /** * Creates a TileDB array storing its schema. * From 42dbf36a350931f8a3ee1abf9373be6d9eac1f69 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 15 May 2024 03:40:49 +0300 Subject: [PATCH 345/456] Run standalone unit tests on Windows. (#4921) [SC-46305](https://app.shortcut.com/tiledb-inc/story/46305/add-standalone-windows-tests-to-ci) This PR cleans up the `unit-test-runs.yml` workflow enables it to run on Windows (and cleans it up), and fixes Windows failures in the test programs. --- TYPE: NO_HISTORY --- .github/workflows/unit-test-runs.yml | 61 +++++++++++-------- tiledb/api/c_api/vfs/test/unit_capi_vfs.cc | 21 +++++++ .../c_api_test_support/testsupport_capi_vfs.h | 3 +- .../common/test/unit_alt_var_length_view.cc | 8 ++- tiledb/common/test/unit_var_length_view.cc | 8 ++- tiledb/common/test/unit_zip_view.cc | 5 ++ .../test/unit_vfs_read_log_modes.cc | 3 +- tiledb/sm/filesystem/uri.cc | 2 +- tiledb/sm/filesystem/win.cc | 2 +- tiledb/sm/fragment/fragment_identifier.cc | 2 +- .../aggregators/test/unit_aggregators.cc | 8 +-- 11 files changed, 85 insertions(+), 38 deletions(-) diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index 2c17b2dd6670..dea85a3fea47 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -3,13 +3,6 @@ on: workflow_call: env: - BACKWARDS_COMPATIBILITY_ARRAYS: OFF - TILEDB_ASSERTIONS: ON - TILEDB_S3: OFF - TILEDB_AZURE: OFF - TILEDB_GCS: OFF - TILEDB_SERIALIZATION: ON - TILEDB_TOOLS: ON VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' jobs: @@ -17,15 +10,32 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest] + os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false name: Build - ${{ matrix.os }} + timeout-minutes: 90 + defaults: + run: + shell: bash steps: - uses: actions/checkout@v3 - name: 'Homebrew setup' run: brew install automake pkg-config - if: ${{ startsWith(matrix.os, 'macos-') == true }} + if: ${{ startsWith(matrix.os, 'macos-') }} + + - name: Install Ninja + uses: seanmiddleditch/gha-setup-ninja@v4 + + - name: Setup MSVC toolset + uses: TheMrMilchmann/setup-msvc-dev@v3 + if: ${{ startsWith(matrix.os, 'windows-') }} + with: + arch: x64 + # By default Visual Studio 2022 chooses the earliest installed toolset + # version for the main build and vcpkg chooses the latest. Force it to + # use the latest (14.39 currently). + toolset: '14.39' # Configure required environment variables for vcpkg to use # GitHub's Action Cache @@ -45,28 +55,25 @@ jobs: echo "uname -v: " $(uname -v) printenv shell: bash + if: ${{ !startsWith(matrix.os, 'windows-') }} - - name: 'Build and run standalone unit tests' - id: test + - name: 'Build standalone unit tests' run: | - - mkdir -p $GITHUB_WORKSPACE/build - pushd $GITHUB_WORKSPACE/build - - cmake .. \ - -DTILEDB_AZURE=${TILEDB_AZURE}\ - -DTILEDB_GCS=${TILEDB_GCS} \ - -DTILEDB_S3=${TILEDB_S3} \ - -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} \ - -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS} - make -j4 + cmake -S . \ + -B build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DTILEDB_SERIALIZATION=ON \ + -DTILEDB_ASSERTIONS=ON + cmake --build build -j4 # Build all unit tests - make -C tiledb tests -j4 - # Run all unit tests - make -C tiledb test ARGS="-R '^unit_'" -R "test_assert" - make -C tiledb test ARGS="-R 'test_ci_asserts'" + cmake --build build/tiledb --target tests -j4 + + - name: 'Run standalone unit tests' + run: | + ctest --test-dir build/tiledb -R '(^unit_|test_assert)' --no-tests=error + ctest --test-dir build/tiledb -R 'test_ci_asserts' - popd - name: "Print log files (failed build only)" run: | source $GITHUB_WORKSPACE/scripts/ci/print_logs.sh diff --git a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc index 1a2984ebda5d..a3edeaa0262f 100644 --- a/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc +++ b/tiledb/api/c_api/vfs/test/unit_capi_vfs.cc @@ -32,6 +32,7 @@ #include #include "../vfs_api_internal.h" #include "tiledb/api/c_api_test_support/testsupport_capi_vfs.h" +#include "tiledb/platform/platform.h" using tiledb::api::test_support::ordinary_vfs; using tiledb::api::test_support::ordinary_vfs_fh; @@ -246,6 +247,12 @@ TEST_CASE("C API: tiledb_vfs_is_dir argument validation", "[capi][vfs]") { CHECK(tiledb_status(rc) == TILEDB_ERR); } SECTION("null uri") { + if constexpr (tiledb::platform::is_os_windows) { + // Windows handles empty (which gets converted from null) paths + // differently. Reconsider when the logic gets unified across paltforms + // (SC-28225). + return; + } auto rc{tiledb_vfs_is_dir(x.ctx, x.vfs, nullptr, &is_dir)}; CHECK(tiledb_status(rc) == TILEDB_ERR); } @@ -331,6 +338,10 @@ TEST_CASE("C API: tiledb_vfs_move_dir argument validation", "[capi][vfs]") { } TEST_CASE("C API: tiledb_vfs_copy_dir argument validation", "[capi][vfs]") { + if constexpr (tiledb::platform::is_os_windows) { + // This API is not yet supported on Windows. + return; + } ordinary_vfs x; SECTION("success") { auto rc{tiledb_vfs_copy_dir(x.ctx, x.vfs, TEST_URI, "new_dir")}; @@ -412,6 +423,12 @@ TEST_CASE("C API: tiledb_vfs_is_file argument validation", "[capi][vfs]") { CHECK(tiledb_status(rc) == TILEDB_ERR); } SECTION("null uri") { + if constexpr (tiledb::platform::is_os_windows) { + // Windows handles empty (which gets converted from null) paths + // differently. Reconsider when the logic gets unified across paltforms + // (SC-28225). + return; + } auto rc{tiledb_vfs_is_file(x.ctx, x.vfs, nullptr, &is_file)}; CHECK(tiledb_status(rc) == TILEDB_ERR); } @@ -477,6 +494,10 @@ TEST_CASE("C API: tiledb_vfs_move_file argument validation", "[capi][vfs]") { } TEST_CASE("C API: tiledb_vfs_copy_file argument validation", "[capi][vfs]") { + if constexpr (tiledb::platform::is_os_windows) { + // This API is not yet supported on Windows. + return; + } ordinary_vfs x; SECTION("success") { auto rc{tiledb_vfs_copy_file(x.ctx, x.vfs, TEST_URI, "new_uri")}; diff --git a/tiledb/api/c_api_test_support/testsupport_capi_vfs.h b/tiledb/api/c_api_test_support/testsupport_capi_vfs.h index 11e7ed85696b..aaff4eadeae0 100644 --- a/tiledb/api/c_api_test_support/testsupport_capi_vfs.h +++ b/tiledb/api/c_api_test_support/testsupport_capi_vfs.h @@ -65,7 +65,8 @@ struct ordinary_vfs_fh { tiledb_ctx_handle_t* ctx{vfs.ctx}; // allow access to ordinary_vfs's ctx tiledb_vfs_fh_handle_t* vfs_fh{nullptr}; ordinary_vfs_fh() { - auto rc = tiledb_vfs_open(vfs.ctx, vfs.vfs, " ", TILEDB_VFS_WRITE, &vfs_fh); + auto rc = tiledb_vfs_open( + vfs.ctx, vfs.vfs, "test.txt", TILEDB_VFS_WRITE, &vfs_fh); if (rc != TILEDB_OK) { throw std::runtime_error("error creating test vfs file handle"); } diff --git a/tiledb/common/test/unit_alt_var_length_view.cc b/tiledb/common/test/unit_alt_var_length_view.cc index 4dce07830a6a..dd0d75a37b35 100644 --- a/tiledb/common/test/unit_alt_var_length_view.cc +++ b/tiledb/common/test/unit_alt_var_length_view.cc @@ -475,14 +475,20 @@ TEST_CASE("alt_var_length_view: Viewness", "[alt_var_length_view]") { } REQUIRE(v.begin() == v.begin()); + REQUIRE(v.end() == v.end()); + + // MSVC in debug iterators mode fails with an assert if iterators of different + // collections are compared. + // https://learn.microsoft.com/en-us/cpp/standard-library/debug-iterator-support?view=msvc-170#incompatible-iterators +#if !defined(_ITERATOR_DEBUG_LEVEL) || _ITERATOR_DEBUG_LEVEL != 2 CHECK(v.begin() != w.begin()); CHECK(v.begin() != u.begin()); CHECK(w.begin() != u.begin()); - REQUIRE(v.end() == v.end()); CHECK(v.end() != w.end()); CHECK(v.end() != u.end()); CHECK(w.end() != u.end()); +#endif for (size_t i = 0; i < 3; ++i) { CHECK(v.size() == x.size()); diff --git a/tiledb/common/test/unit_var_length_view.cc b/tiledb/common/test/unit_var_length_view.cc index 9540a824a8f1..e0d0e2026ae4 100644 --- a/tiledb/common/test/unit_var_length_view.cc +++ b/tiledb/common/test/unit_var_length_view.cc @@ -319,14 +319,20 @@ TEST_CASE("var_length_view: Viewness", "[var_length_view]") { auto z = var_length_view(s, n); REQUIRE(v.begin() == v.begin()); + REQUIRE(v.end() == v.end()); + + // MSVC in debug iterators mode fails with an assert if iterators of different + // collections are compared. + // https://learn.microsoft.com/en-us/cpp/standard-library/debug-iterator-support?view=msvc-170#incompatible-iterators +#if !defined(_ITERATOR_DEBUG_LEVEL) || _ITERATOR_DEBUG_LEVEL != 2 CHECK(v.begin() != w.begin()); CHECK(v.begin() != u.begin()); CHECK(w.begin() != u.begin()); - REQUIRE(v.end() == v.end()); CHECK(v.end() != w.end()); CHECK(v.end() != u.end()); CHECK(w.end() != u.end()); +#endif for (size_t i = 0; i < 3; ++i) { CHECK(v.size() == x.size()); diff --git a/tiledb/common/test/unit_zip_view.cc b/tiledb/common/test/unit_zip_view.cc index 00c0be0df4d3..6abc6b8d0992 100644 --- a/tiledb/common/test/unit_zip_view.cc +++ b/tiledb/common/test/unit_zip_view.cc @@ -284,8 +284,13 @@ TEST_CASE("zip_view: basic iterator properties", "[zip_view]") { CHECK(it == z.begin() + 1); CHECK(it[0] == *it); + // MSVC in debug iterators mode fails with an assert if out of range iterators + // are dereferenced. + // https://learn.microsoft.com/en-us/cpp/standard-library/debug-iterator-support +#if !defined(_ITERATOR_DEBUG_LEVEL) || _ITERATOR_DEBUG_LEVEL != 2 CHECK(it[1] == *(it + 1)); CHECK(it[2] == *(it + 2)); +#endif CHECK(it[0] == std::tuple{2, 5, 11}); } diff --git a/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc b/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc index 5b995533aa19..789a3213d18e 100644 --- a/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc +++ b/tiledb/sm/filesystem/test/unit_vfs_read_log_modes.cc @@ -74,7 +74,8 @@ TEST_CASE("VFS Read Log Modes", "[vfs][read-logging-modes]") { char buffer[123]; for (auto& uri : uris_to_read) { // None of these files exist, so we expect every read to fail. - REQUIRE_THROWS(res.vfs().read(URI(uri), 123, buffer, 456)); + REQUIRE_THROWS( + throw_if_not_ok(res.vfs().read(URI(uri), 123, buffer, 456))); } } } diff --git a/tiledb/sm/filesystem/uri.cc b/tiledb/sm/filesystem/uri.cc index a0efc94aff0f..0c9c85d12fdd 100644 --- a/tiledb/sm/filesystem/uri.cc +++ b/tiledb/sm/filesystem/uri.cc @@ -111,7 +111,7 @@ URI URI::add_trailing_slash() const { } URI URI::remove_trailing_slash() const { - if (uri_.back() == '/') { + if (!uri_.empty() && uri_.back() == '/') { std::string uri_str = uri_; uri_str.pop_back(); return URI(uri_str); diff --git a/tiledb/sm/filesystem/win.cc b/tiledb/sm/filesystem/win.cc index b642a804baec..265768f5fd68 100644 --- a/tiledb/sm/filesystem/win.cc +++ b/tiledb/sm/filesystem/win.cc @@ -326,7 +326,7 @@ Status Win::init(const Config& config) { } bool Win::is_dir(const std::string& path) const { - return PathIsDirectory(path.c_str()); + return PathFileExists(path.c_str()) && PathIsDirectory(path.c_str()); } bool Win::is_file(const std::string& path) const { diff --git a/tiledb/sm/fragment/fragment_identifier.cc b/tiledb/sm/fragment/fragment_identifier.cc index 88ff947224f3..fd6cf9294beb 100644 --- a/tiledb/sm/fragment/fragment_identifier.cc +++ b/tiledb/sm/fragment/fragment_identifier.cc @@ -110,7 +110,7 @@ FragmentID::FragmentID(const URI& uri) } FragmentID::FragmentID(const std::string_view& path) - : FragmentID(URI(path.data())) { + : FragmentID(URI(path)) { } } // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 6ec03eca483b..511d7c43b5ef 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -481,17 +481,17 @@ void basic_aggregation_test(std::vector expected_results) { 10, 10, &fixed_data[0], - sizeof(T), + sizeof(fixed_data[0]), &fixed_data[0], - sizeof(T), + sizeof(fixed_data[0]), zero.data()); auto tile_metadata = TileMetadata( 10, 5, &fixed_data[0], - sizeof(T), + sizeof(fixed_data[0]), &fixed_data[0], - sizeof(T), + sizeof(fixed_data[0]), full_tile_sum.data()); if (aggregator.has_value()) { // Regular attribute. From c28e084282f48ea127f0c0d5c3f19168c08c9204 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Thu, 16 May 2024 08:59:02 +0300 Subject: [PATCH 346/456] throw error on dimension drop in schema evolution. (#4958) Adding an error message when trying to drop a non existent attribute or a dimension with the Schema Evolution API. + Test [sc-30774] --- TYPE: BUG DESC: Throw error on dimension drop in schema evolution. --- test/src/unit-cppapi-schema-evolution.cc | 44 +++++++++++++++++++ .../sm/array_schema/array_schema_evolution.cc | 7 ++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index a6e2d2370180..15bf64b0775d 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -107,6 +107,50 @@ TEST_CASE( } } +TEST_CASE( + "C++ API: SchemaEvolution, check error when dropping dimension", + "[cppapi][schema][evolution][drop]") { + using namespace tiledb; + Context ctx; + VFS vfs(ctx); + + std::string array_uri = "test_schema_evolution_array"; + + Domain domain(ctx); + auto id1 = Dimension::create(ctx, "d1", {{-100, 100}}, 10); + auto id2 = Dimension::create(ctx, "d2", {{0, 100}}, 5); + domain.add_dimension(id1).add_dimension(id2); + + auto a1 = Attribute::create(ctx, "a1"); + auto a2 = Attribute::create(ctx, "a2"); + + ArraySchema schema(ctx, TILEDB_DENSE); + schema.set_domain(domain); + schema.add_attribute(a1); + schema.add_attribute(a2); + schema.set_cell_order(TILEDB_ROW_MAJOR); + schema.set_tile_order(TILEDB_COL_MAJOR); + + if (vfs.is_dir(array_uri)) { + vfs.remove_dir(array_uri); + } + + Array::create(array_uri, schema); + + auto evolution = ArraySchemaEvolution(ctx); + + // try to drop dimension d1 + evolution.drop_attribute("d1"); + + // check that an exception is thrown + CHECK_THROWS(evolution.array_evolve(array_uri)); + + // Clean up + if (vfs.is_dir(array_uri)) { + vfs.remove_dir(array_uri); + } +} + TEST_CASE( "C++ API: SchemaEvolution, add attributes and read", "[cppapi][schema][evolution][add]") { diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 117dee9607b4..560db92f149f 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -152,9 +152,12 @@ shared_ptr ArraySchemaEvolution::evolve_schema( for (auto& attr_name : attributes_to_drop_) { bool has_attr = false; throw_if_not_ok(schema->has_attribute(attr_name, &has_attr)); - if (has_attr) { - throw_if_not_ok(schema->drop_attribute(attr_name)); + if (!has_attr) { + throw ArraySchemaEvolutionException( + "Cannot drop attribute; Input attribute name refers to a dimension " + "or does not exist"); } + throw_if_not_ok(schema->drop_attribute(attr_name)); } // Drop enumerations From 342edda24ec56eca5a8fda8f50a5475d42157704 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 16 May 2024 01:59:59 -0400 Subject: [PATCH 347/456] Migrate APIs out of StorageManager: store_group_detail. (#4966) Migrate APIs out of `StorageManager`: `store_group_detail`. [sc-46727] --- TYPE: NO_HISTORY DESC: Migrate APIs out of StorageManager: store_group_detail. --- tiledb/sm/group/group_details.cc | 35 +++++++++++++++++- tiledb/sm/group/group_details.h | 25 ++++++++++--- tiledb/sm/storage_manager/storage_manager.cc | 36 ++----------------- .../storage_manager_canonical.h | 15 -------- 4 files changed, 56 insertions(+), 55 deletions(-) diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index a777732e729d..8853775ca336 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -44,6 +44,7 @@ #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/rest/rest_client.h" +#include "tiledb/sm/tile/generic_tile_io.h" using namespace tiledb::common; @@ -202,6 +203,38 @@ GroupDetails::members() const { return members_; } +Status GroupDetails::store( + ContextResources& resources, + const URI& group_detail_folder_uri, + const URI& group_detail_uri, + const EncryptionKey& encryption_key) { + // Serialize + auto members = members_to_serialize(); + SizeComputationSerializer size_computation_serializer; + serialize(members, size_computation_serializer); + + auto tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources.ephemeral_memory_tracker())}; + + Serializer serializer(tile->data(), tile->size()); + serialize(members, serializer); + resources.stats().add_counter("write_group_size", tile->size()); + + // Check if the array schema directory exists + // If not create it, this is caused by a pre-v10 array + bool group_detail_dir_exists = false; + auto& vfs = resources.vfs(); + throw_if_not_ok( + vfs.is_dir(group_detail_folder_uri, &group_detail_dir_exists)); + if (!group_detail_dir_exists) { + throw_if_not_ok(vfs.create_dir(group_detail_folder_uri)); + } + GenericTileIO::store_data(resources, group_detail_uri, tile, encryption_key); + + return Status::Ok(); +} + std::optional> GroupDetails::deserialize( Deserializer& deserializer, const URI& group_uri) { uint32_t version = 0; diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index 9e28bf8822d3..88fec9599d46 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,12 +41,12 @@ #include "tiledb/sm/group/group_directory.h" #include "tiledb/sm/group/group_member.h" #include "tiledb/sm/metadata/metadata.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class GroupDetails { public: @@ -131,6 +131,21 @@ class GroupDetails { */ void delete_member(const shared_ptr group_member); + /** + * Store the group details + * + * @param resources the context resources + * @param group_detail_folder_uri group details folder + * @param group_detail_uri uri for detail file to write + * @param encryption_key encryption key for at-rest encryption + * @return status + */ + Status store( + ContextResources& resources, + const URI& group_detail_folder_uri, + const URI& group_detail_uri, + const EncryptionKey& encryption_key); + /** * Serializes the object members into a binary buffer. * @@ -264,7 +279,7 @@ class GroupDetails { /** Invalidate the built lookup tables. */ void invalidate_lookups(); }; -} // namespace sm -} // namespace tiledb + +} // namespace tiledb::sm #endif // TILEDB_GROUP_DETAILS_H diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ab8601533461..ec515fab5114 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -142,10 +142,10 @@ Status StorageManagerCanonical::group_close_for_writes(Group* group) { if (group->group_details()->is_modified()) { const URI& group_detail_folder_uri = group->group_detail_uri(); auto group_detail_uri = group->generate_detail_uri(); - RETURN_NOT_OK(store_group_detail( + throw_if_not_ok(group->group_details()->store( + resources_, group_detail_folder_uri, group_detail_uri, - group->group_details(), *group->encryption_key())); } return Status::Ok(); @@ -856,38 +856,6 @@ Status StorageManagerCanonical::set_tag( return Status::Ok(); } -Status StorageManagerCanonical::store_group_detail( - const URI& group_detail_folder_uri, - const URI& group_detail_uri, - tdb_shared_ptr group, - const EncryptionKey& encryption_key) { - // Serialize - auto members = group->members_to_serialize(); - SizeComputationSerializer size_computation_serializer; - group->serialize(members, size_computation_serializer); - - auto tile{WriterTile::from_generic( - size_computation_serializer.size(), - resources_.ephemeral_memory_tracker())}; - - Serializer serializer(tile->data(), tile->size()); - group->serialize(members, serializer); - - stats()->add_counter("write_group_size", tile->size()); - - // Check if the array schema directory exists - // If not create it, this is caused by a pre-v10 array - bool group_detail_dir_exists = false; - RETURN_NOT_OK( - vfs()->is_dir(group_detail_folder_uri, &group_detail_dir_exists)); - if (!group_detail_dir_exists) - RETURN_NOT_OK(vfs()->create_dir(group_detail_folder_uri)); - - GenericTileIO::store_data(resources_, group_detail_uri, tile, encryption_key); - - return Status::Ok(); -} - Status StorageManagerCanonical::store_array_schema( const shared_ptr& array_schema, const EncryptionKey& encryption_key) { diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 948a79e406a9..b012174c14ce 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -156,21 +156,6 @@ class StorageManagerCanonical { */ Status group_close_for_writes(tiledb::sm::Group* group); - /** - * Store the group details - * - * @param group_detail_folder_uri group details folder - * @param group_detail_uri uri for detail file to write - * @param group to serialize and store - * @param encryption_key encryption key for at-rest encryption - * @return status - */ - Status store_group_detail( - const URI& group_detail_folder_uri, - const URI& group_detail_uri, - tdb_shared_ptr group, - const EncryptionKey& encryption_key); - /** * Creates a TileDB array storing its schema. * From c73b5234a60137a0cf030a173b7c19e7cd088297 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 May 2024 09:01:29 +0300 Subject: [PATCH 348/456] Add constructor argument to Query to override the memory budget. (#4968) [SC-45187](https://app.shortcut.com/tiledb-inc/story/45187/enable-to-set-reader-budget-on-consolidation) This PR adds an optional argument to the internal `Query` class, that overrides the total memory budget. This will be useful for better memory management during consolidation. --- TYPE: NO_HISTORY --- test/src/unit-Reader.cc | 1 + tiledb/sm/query/query.cc | 5 ++++- tiledb/sm/query/query.h | 11 ++++++++++- tiledb/sm/query/readers/dense_reader.cc | 12 ++++++++---- tiledb/sm/query/readers/dense_reader.h | 3 +++ .../sm/query/readers/sparse_index_reader_base.cc | 2 +- .../sm/query/readers/sparse_index_reader_base.h | 16 +++++++++++++--- tiledb/sm/query/strategy_base.h | 13 +++++++++++++ 8 files changed, 53 insertions(+), 10 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 8f21d9f0e9a3..2b68f1350b73 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -175,6 +175,7 @@ TEST_CASE_METHOD( context.storage_manager(), array.opened_array(), config, + nullopt, buffers, aggregate_buffers, subarray, diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 2374a0a2b6b5..8e785f49ef32 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -74,7 +74,8 @@ namespace sm { Query::Query( StorageManager* storage_manager, shared_ptr array, - optional fragment_name) + optional fragment_name, + optional memory_budget) : query_memory_tracker_( storage_manager->resources().create_memory_tracker()) , array_shared_(array) @@ -105,6 +106,7 @@ Query::Query( , is_dimension_label_ordered_read_(false) , dimension_label_increasing_(true) , fragment_size_(std::numeric_limits::max()) + , memory_budget_(memory_budget) , query_remote_buffer_storage_(std::nullopt) , default_channel_{make_shared(HERE(), *this, 0)} { assert(array->is_open()); @@ -1756,6 +1758,7 @@ Status Query::create_strategy(bool skip_checks_serialization) { storage_manager_, opened_array_, config_, + memory_budget_, buffers_, aggregate_buffers_, subarray_, diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index b8782b0b53ac..b4e1db07d197 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -155,11 +155,14 @@ class Query { * writes. * @param fragment_base_uri Optional base name for new fragment. Only used for * writes and only if fragment_uri is empty. + * @param memory_budget Total memory budget. If set to nullopt, the value + * will be obtained from the sm.mem.total_budget config option. */ Query( StorageManager* storage_manager, shared_ptr array, - optional fragment_name = nullopt); + optional fragment_name = nullopt, + optional memory_budget = nullopt); /** Destructor. */ virtual ~Query(); @@ -1089,6 +1092,12 @@ class Query { */ uint64_t fragment_size_; + /** + * Memory budget. If set to nullopt, the value will be obtained from the + * sm.mem.total_budget config option. + */ + optional memory_budget_; + /** Already written buffers. */ std::unordered_set written_buffers_; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index eae579ad9530..b8a13d4d2ebc 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -72,7 +72,9 @@ DenseReader::DenseReader( shared_ptr logger, StrategyParams& params, bool remote_query) - : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) { + : ReaderBase(stats, logger->clone("DenseReader", ++logger_id_), params) + , memory_budget_(params.memory_budget().value_or(0)) + , memory_budget_from_query_(params.memory_budget()) { elements_mode_ = false; // Sanity checks. @@ -121,9 +123,11 @@ QueryStatusDetailsReason DenseReader::status_incomplete_reason() const { void DenseReader::refresh_config() { // Get config values. bool found = false; - throw_if_not_ok( - config_.get("sm.mem.total_budget", &memory_budget_, &found)); - assert(found); + if (!memory_budget_from_query_.has_value()) { + throw_if_not_ok( + config_.get("sm.mem.total_budget", &memory_budget_, &found)); + assert(found); + } throw_if_not_ok(config_.get( "sm.mem.tile_upper_memory_limit", &tile_upper_memory_limit_, &found)); assert(found); diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index e44c7903fdc2..5d54af461242 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -158,6 +158,9 @@ class DenseReader : public ReaderBase, public IQueryStrategy { /** Total memory budget. */ uint64_t memory_budget_; + /** Total memory budget if overridden by the query. */ + optional memory_budget_from_query_; + /** Target upper memory limit for tiles. */ uint64_t tile_upper_memory_limit_; diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 1f62acee4c1f..b20a46ce4ff2 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -72,7 +72,7 @@ SparseIndexReaderBase::SparseIndexReaderBase( : ReaderBase(stats, logger, params) , read_state_(array_->fragment_metadata().size()) , tmp_read_state_(array_->fragment_metadata().size()) - , memory_budget_(config_, reader_string) + , memory_budget_(config_, reader_string, params.memory_budget()) , include_coords_(include_coords) , memory_used_for_coords_total_(0) , deletes_consolidation_no_purge_( diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.h b/tiledb/sm/query/readers/sparse_index_reader_base.h index c9303391001f..3ca26a1c18c0 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.h +++ b/tiledb/sm/query/readers/sparse_index_reader_base.h @@ -110,7 +110,12 @@ class MemoryBudget { MemoryBudget() = delete; - MemoryBudget(Config& config, std::string reader_string) { + MemoryBudget( + Config& config, + std::string reader_string, + optional total_budget) + : total_budget_(total_budget.value_or(0)) + , memory_budget_from_query_(total_budget) { refresh_config(config, reader_string); } @@ -128,8 +133,10 @@ class MemoryBudget { * @param reader_string String to identify the reader settings to load. */ void refresh_config(Config& config, std::string reader_string) { - total_budget_ = - config.get("sm.mem.total_budget", Config::must_find); + if (!memory_budget_from_query_.has_value()) { + total_budget_ = + config.get("sm.mem.total_budget", Config::must_find); + } ratio_coords_ = config.get( "sm.mem.reader." + reader_string + ".ratio_coords", Config::must_find); @@ -192,6 +199,9 @@ class MemoryBudget { /** Total memory budget. */ uint64_t total_budget_; + /** Total memory budget if overridden by the query. */ + optional memory_budget_from_query_; + /** How much of the memory budget is reserved for coords. */ double ratio_coords_; diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 8aa91b6301a4..9d31a053a831 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -70,6 +70,7 @@ class StrategyParams { StorageManager* storage_manager, shared_ptr array, Config& config, + optional memory_budget, std::unordered_map& buffers, std::unordered_map& aggregate_buffers, Subarray& subarray, @@ -82,6 +83,7 @@ class StrategyParams { , storage_manager_(storage_manager) , array_(array) , config_(config) + , memory_budget_(memory_budget) , buffers_(buffers) , aggregate_buffers_(aggregate_buffers) , subarray_(subarray) @@ -119,6 +121,11 @@ class StrategyParams { return config_; }; + /** Return the memory budget, if set. */ + inline optional memory_budget() { + return memory_budget_; + } + /** Return the buffers. */ inline std::unordered_map& buffers() { return buffers_; @@ -174,6 +181,12 @@ class StrategyParams { /** Config for query-level parameters only. */ Config& config_; + /** + * Memory budget for the query. If set to nullopt, the value will be obtained + * from the sm.mem.total_budget config option. + */ + optional memory_budget_; + /** Buffers. */ std::unordered_map& buffers_; From 4d9474fa998dcdba0e50ed0e17ae7fff943031f8 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 16 May 2024 02:05:54 -0400 Subject: [PATCH 349/456] Add RestClient accessor in class Context. (#4978) Add `RestClient` accessor in class `Context`. [sc-47374] --- TYPE: NO_HISTORY DESC: Add RestClient accessor in class Context. --- tiledb/sm/c_api/tiledb.cc | 55 +++++------------------------ tiledb/sm/storage_manager/context.h | 12 ++++++- 2 files changed, 19 insertions(+), 48 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 7e2848823ac7..8a581341ea23 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -470,18 +470,9 @@ int32_t tiledb_array_schema_load( } if (uri.is_tiledb()) { - // Check REST client - auto rest_client = ctx->storage_manager()->rest_client(); - if (rest_client == nullptr) { - auto st = Status_Error( - "Failed to load array schema; remote array with no REST client."); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - + auto& rest_client = ctx->context().rest_client(); auto&& [st, array_schema_rest] = - rest_client->get_array_schema_from_rest(uri); + rest_client.get_array_schema_from_rest(uri); if (!st.ok()) { LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); @@ -553,20 +544,9 @@ int32_t tiledb_array_schema_load_with_key( } if (uri.is_tiledb()) { - // Check REST client - auto rest_client = ctx->storage_manager()->rest_client(); - if (rest_client == nullptr) { - delete *array_schema; - *array_schema = nullptr; - auto st = Status_Error( - "Failed to load array schema; remote array with no REST client."); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - + auto& rest_client = ctx->context().rest_client(); auto&& [st, array_schema_rest] = - rest_client->get_array_schema_from_rest(uri); + rest_client.get_array_schema_from_rest(uri); if (!st.ok()) { LOG_STATUS_NO_RETURN_VALUE(st); save_error(ctx, st); @@ -2578,17 +2558,8 @@ int32_t tiledb_array_create( } if (uri.is_tiledb()) { - // Check REST client - auto rest_client = ctx->storage_manager()->rest_client(); - if (rest_client == nullptr) { - auto st = Status_Error( - "Failed to create array; remote array with no REST client."); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - - throw_if_not_ok(rest_client->post_array_schema_to_rest( + auto& rest_client = ctx->context().rest_client(); + throw_if_not_ok(rest_client.post_array_schema_to_rest( uri, *(array_schema->array_schema_.get()))); } else { // Create key @@ -2652,18 +2623,8 @@ int32_t tiledb_array_create_with_key( save_error(ctx, st); return TILEDB_ERR; } - - // Check REST client - auto rest_client = ctx->storage_manager()->rest_client(); - if (rest_client == nullptr) { - auto st = Status_Error( - "Failed to create array; remote array with no REST client."); - LOG_STATUS_NO_RETURN_VALUE(st); - save_error(ctx, st); - return TILEDB_ERR; - } - - throw_if_not_ok(rest_client->post_array_schema_to_rest( + auto& rest_client = ctx->context().rest_client(); + throw_if_not_ok(rest_client.post_array_schema_to_rest( uri, *(array_schema->array_schema_.get()))); } else { // Create key diff --git a/tiledb/sm/storage_manager/context.h b/tiledb/sm/storage_manager/context.h index a4811694fb08..5ef63c33ccf7 100644 --- a/tiledb/sm/storage_manager/context.h +++ b/tiledb/sm/storage_manager/context.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -117,6 +117,16 @@ class Context { return &(resources_.io_tp()); } + [[nodiscard]] inline RestClient& rest_client() const { + auto x = resources_.rest_client(); + if (!x) { + throw std::runtime_error( + "Failed to retrieve RestClient; the underlying instance is null and " + "may not have been configured."); + } + return *(x.get()); + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ From 5c0169983b245241bb80e3d25be7d5e6833e76aa Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 16 May 2024 00:08:01 -0600 Subject: [PATCH 350/456] Bring object library `generic_tile_io` into conformance with standalone link policy (#4975) PR #4051 took object library `generic_tile_io` out of conformance with the policy that each OL should link standalone. This PR corrects this. [sc-47341] --- TYPE: NO_HISTORY DESC: Bring object library `generic_tile_io` into conformance with standalone link policy --- .../test/compile_capi_dimension_label_stub_main.cc | 1 - tiledb/sm/array_schema/CMakeLists.txt | 2 +- tiledb/sm/tile/CMakeLists.txt | 7 +------ 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc b/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc index 039db80f1356..f82c1434e36f 100644 --- a/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc +++ b/tiledb/api/c_api/dimension_label/test/compile_capi_dimension_label_stub_main.cc @@ -22,7 +22,6 @@ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 0389401e9f56..362393d2a625 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -78,6 +78,6 @@ conclude(object_library) # the principle that object libraries must compile as standalone modules. As # long as this directive is necessary to have to the compile test pass, we # cannot consider `array_schema` as a properly-constructed object library. -target_link_libraries(compile_array_schema PRIVATE context_resources generic_tile_io) +target_link_libraries(compile_array_schema PRIVATE generic_tile_io) add_test_subdirectory() diff --git a/tiledb/sm/tile/CMakeLists.txt b/tiledb/sm/tile/CMakeLists.txt index f52ea816f9b0..c23f54bec7ff 100644 --- a/tiledb/sm/tile/CMakeLists.txt +++ b/tiledb/sm/tile/CMakeLists.txt @@ -44,6 +44,7 @@ commence(object_library generic_tile_io) baseline buffer constants + context_resources tiledb_crypto filter_pipeline tile @@ -51,10 +52,4 @@ commence(object_library generic_tile_io) ) conclude(object_library) -# This is linked outside the object_library scope because ContextResources -# is recompiled as part of the capi_context_stub. Including context_resources -# here like this prevents many headaches revolving around duplicate symbols -# when linking executables. -target_link_libraries(compile_generic_tile_io PRIVATE context_resources) - add_test_subdirectory() From 25fd4ce87851b7f9cdb85c115b3d10810d6780b2 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Thu, 16 May 2024 09:59:45 +0300 Subject: [PATCH 351/456] Unordered tile/cell order is now rejected when creating an ArraySchema (#4973) This PR disables the ability to create an Array Schema with UNORDERED tile/cell order. + Tests (C/C++). [sc-47276] --- TYPE: BUG DESC: Rejecting unordered tile/cell order when creating an ArraySchema. --- test/src/unit-capi-array_schema.cc | 20 ++++++++++++++++++++ test/src/unit-cppapi-schema.cc | 4 ++++ tiledb/sm/array_schema/array_schema.cc | 10 ++++++++++ 3 files changed, 34 insertions(+) diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index e04735b5ac41..f4d520671378 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -1081,6 +1081,26 @@ TEST_CASE_METHOD( tiledb_array_schema_free(&array_schema); } +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test array schema with invalid cell/tile order", + "[capi][array-schema]") { + // Create array schema + tiledb_array_schema_t* array_schema; + int rc = tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &array_schema); + REQUIRE(rc == TILEDB_OK); + + // Check that UNORDERED order fails + rc = tiledb_array_schema_set_tile_order(ctx_, array_schema, TILEDB_UNORDERED); + REQUIRE(rc == TILEDB_ERR); + + rc = tiledb_array_schema_set_cell_order(ctx_, array_schema, TILEDB_UNORDERED); + REQUIRE(rc == TILEDB_ERR); + + // Clean up + tiledb_array_schema_free(&array_schema); +} + TEST_CASE_METHOD( ArraySchemaFx, "C API: Test array schema with invalid dimension domain and tile extent", diff --git a/test/src/unit-cppapi-schema.cc b/test/src/unit-cppapi-schema.cc index b0c731c8638b..2a798efd68d7 100644 --- a/test/src/unit-cppapi-schema.cc +++ b/test/src/unit-cppapi-schema.cc @@ -74,6 +74,8 @@ TEST_CASE("C++ API: Schema", "[cppapi][schema]") { schema.add_attribute(a2); schema.add_attribute(a3); schema.add_attribute(a4); + CHECK_THROWS(schema.set_cell_order(TILEDB_UNORDERED)); + CHECK_THROWS(schema.set_tile_order(TILEDB_UNORDERED)); schema.set_cell_order(TILEDB_ROW_MAJOR); schema.set_tile_order(TILEDB_COL_MAJOR); CHECK_THROWS(schema.set_allows_dups(1)); @@ -152,6 +154,8 @@ TEST_CASE("C++ API: Schema", "[cppapi][schema]") { schema.add_attribute(a2); schema.add_attribute(a3); schema.add_attribute(a4); + CHECK_THROWS(schema.set_cell_order(TILEDB_UNORDERED)); + CHECK_THROWS(schema.set_tile_order(TILEDB_UNORDERED)); schema.set_cell_order(TILEDB_ROW_MAJOR); schema.set_tile_order(TILEDB_COL_MAJOR); schema.set_allows_dups(true); diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 76a0bc22424b..b0fbc42f51b9 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -1524,6 +1524,11 @@ Status ArraySchema::set_cell_order(Layout cell_order) { Status_ArraySchemaError("Cannot set cell order; Hilbert order is only " "applicable to sparse arrays")); + if (cell_order == Layout::UNORDERED) + return LOG_STATUS(Status_ArraySchemaError( + "Cannot set cell order; Cannot create ArraySchema " + "with UNORDERED cell order")); + cell_order_ = cell_order; return Status::Ok(); @@ -1618,6 +1623,11 @@ Status ArraySchema::set_tile_order(Layout tile_order) { return LOG_STATUS(Status_ArraySchemaError( "Cannot set tile order; Hilbert order is not applicable to tiles")); + if (tile_order == Layout::UNORDERED) + return LOG_STATUS(Status_ArraySchemaError( + "Cannot set tile order; Cannot create ArraySchema " + "with UNORDERED tile order")); + tile_order_ = tile_order; return Status::Ok(); } From ff9c0653bb07b588d46b471bee12f21de66e3865 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Thu, 16 May 2024 02:48:55 -0600 Subject: [PATCH 352/456] Remove `StorageManager::config()` (#4976) Remove the member function `StorageManager::config()` and replace it (ultimately) with access to the `Config` instance through `ContextResources`. Add `ContextResources` member variables paralleling `StorageManager` ones where necessary. Add a direct accessor for the C API handle for `Context`. Documented some bit-rotted code discovered during the development of this PR. [sc-47339] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::config()` --- ...it-cppapi-consolidation-with-timestamps.cc | 7 ++++++ test/performance/msys_handle_leakage/unit.cc | 5 ++++ tiledb/api/c_api/context/context_api.cc | 3 +-- .../api/c_api/context/context_api_internal.h | 4 ++++ tiledb/api/c_api/group/group_api.cc | 6 ++--- tiledb/api/c_api/vfs/vfs_api.cc | 11 +++++---- .../storage_manager_override.h | 3 --- tiledb/sm/c_api/tiledb.cc | 12 ++++------ tiledb/sm/c_api/tiledb_filestore.cc | 14 +++++------ .../consolidator/array_meta_consolidator.cc | 2 +- tiledb/sm/consolidator/consolidator.cc | 3 ++- tiledb/sm/consolidator/consolidator.h | 5 ++++ .../sm/consolidator/fragment_consolidator.cc | 2 +- .../consolidator/group_meta_consolidator.cc | 2 +- tiledb/sm/group/group.cc | 2 +- .../deletes_and_updates.cc | 5 +--- tiledb/sm/query/query.cc | 7 +++--- tiledb/sm/query/query.h | 5 ++++ tiledb/sm/query/readers/reader_base.cc | 6 ++--- tiledb/sm/query/strategy_base.cc | 3 ++- tiledb/sm/query/strategy_base.h | 23 +++++++++++++++++-- tiledb/sm/serialization/query.cc | 3 ++- tiledb/sm/storage_manager/storage_manager.cc | 6 +---- .../storage_manager_canonical.h | 3 --- 24 files changed, 86 insertions(+), 56 deletions(-) diff --git a/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc b/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc index 77282da14bdd..880a97500382 100644 --- a/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/performance/msys_handle_leakage/unit-cppapi-consolidation-with-timestamps.cc @@ -30,6 +30,13 @@ * Tests the CPP API for consolidation with timestamps. */ +/* + * WARNING: + * [2024/05/13] This file has succumbed to bit rot. It used to be a copy of + * a file by the same name within `tiledb_unit`, but it no longer is. It + * should not be expected to compile. + */ + #include #include "test/support/src/helpers.h" #include "tiledb/api/c_api/context/context_api_internal.h" diff --git a/test/performance/msys_handle_leakage/unit.cc b/test/performance/msys_handle_leakage/unit.cc index e158ff57b560..d69c85768b64 100644 --- a/test/performance/msys_handle_leakage/unit.cc +++ b/test/performance/msys_handle_leakage/unit.cc @@ -35,6 +35,11 @@ * unit.cc along with some previously observed results. */ +/* + * WARNING: + * [2024/05/13] This file has succumbed to bit rot. It no longer compiles. + */ + // clang-format off //.\tiledb\test\performance\RelWithDebInfo\tiledb_explore_msys_handle_leakage.exe --read-sparse-iters=1 --perform-query=1 --consolidate-sparse-iters=1 --wait-for-keypress both //./tiledb/test/performance/tiledb_explore_msys_handle_leakage.exe --read-sparse-iters=1 --perform-query=1 --consolidate-sparse-sparse-iters=1 --wait-for-keypress both diff --git a/tiledb/api/c_api/context/context_api.cc b/tiledb/api/c_api/context/context_api.cc index df5fd2c2f05c..9ff7b8de04f9 100644 --- a/tiledb/api/c_api/context/context_api.cc +++ b/tiledb/api/c_api/context/context_api.cc @@ -85,8 +85,7 @@ capi_return_t tiledb_ctx_get_stats( capi_return_t tiledb_ctx_get_config( tiledb_ctx_handle_t* ctx, tiledb_config_handle_t** config) { api::ensure_output_pointer_is_valid(config); - *config = - tiledb_config_handle_t::make_handle(ctx->storage_manager()->config()); + *config = tiledb_config_handle_t::make_handle(ctx->config()); return TILEDB_OK; } diff --git a/tiledb/api/c_api/context/context_api_internal.h b/tiledb/api/c_api/context/context_api_internal.h index 31ccfef34d6d..b557cd819093 100644 --- a/tiledb/api/c_api/context/context_api_internal.h +++ b/tiledb/api/c_api/context/context_api_internal.h @@ -63,6 +63,10 @@ struct tiledb_ctx_handle_t return ctx_.resources(); } + inline tiledb::sm::Config& config() { + return ctx_.resources().config(); + } + inline tiledb::sm::StorageManager* storage_manager() { return ctx_.storage_manager(); } diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 6fbcf4f865b3..3f2246b4b021 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -577,8 +577,7 @@ capi_return_t tiledb_group_consolidate_metadata( tiledb_ctx_handle_t* ctx, const char* group_uri, tiledb_config_t* config) { ensure_group_uri_argument_is_valid(group_uri); - auto cfg = - (config == nullptr) ? ctx->storage_manager()->config() : config->config(); + auto cfg = (config == nullptr) ? ctx->config() : config->config(); throw_if_not_ok( ctx->storage_manager()->group_metadata_consolidate(group_uri, cfg)); @@ -589,8 +588,7 @@ capi_return_t tiledb_group_vacuum_metadata( tiledb_ctx_handle_t* ctx, const char* group_uri, tiledb_config_t* config) { ensure_group_uri_argument_is_valid(group_uri); - auto cfg = - (config == nullptr) ? ctx->storage_manager()->config() : config->config(); + auto cfg = (config == nullptr) ? ctx->config() : config->config(); ctx->storage_manager()->group_metadata_vacuum(group_uri, cfg); return TILEDB_OK; diff --git a/tiledb/api/c_api/vfs/vfs_api.cc b/tiledb/api/c_api/vfs/vfs_api.cc index 8491f82ea094..6beb9f4de894 100644 --- a/tiledb/api/c_api/vfs/vfs_api.cc +++ b/tiledb/api/c_api/vfs/vfs_api.cc @@ -57,14 +57,15 @@ capi_return_t tiledb_vfs_alloc( ensure_output_pointer_is_valid(vfs); // Create VFS object - auto stats = ctx->storage_manager()->stats(); - auto compute_tp = ctx->storage_manager()->compute_tp(); - auto io_tp = ctx->storage_manager()->io_tp(); - auto ctx_config = ctx->storage_manager()->config(); + auto& resources{ctx->resources()}; + auto& stats{resources.stats()}; + auto& compute_tp{resources.compute_tp()}; + auto& io_tp{resources.io_tp()}; + auto& ctx_config{resources.config()}; if (config) { ctx_config.inherit((config->config())); } - *vfs = tiledb_vfs_t::make_handle(stats, compute_tp, io_tp, ctx_config); + *vfs = tiledb_vfs_t::make_handle(&stats, &compute_tp, &io_tp, ctx_config); return TILEDB_OK; } diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index d4bbe3cdd9b6..b16ae2acc691 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -71,9 +71,6 @@ class StorageManagerStub { inline stats::Stats* stats() { return &resources_.stats(); } - const Config& config() { - return config_; - } inline VFS* vfs() { throw std::logic_error("StorageManagerStub does not instantiate a VFS"); } diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 8a581341ea23..9890faf1837c 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2669,7 +2669,7 @@ int32_t tiledb_array_consolidate( tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0, - (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); return TILEDB_OK; } @@ -2688,7 +2688,7 @@ int32_t tiledb_array_consolidate_with_key( static_cast(encryption_type), encryption_key, key_length, - (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); return TILEDB_OK; @@ -2715,7 +2715,7 @@ int32_t tiledb_array_consolidate_fragments( nullptr, 0, uris, - (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); return TILEDB_OK; @@ -2725,7 +2725,7 @@ int32_t tiledb_array_vacuum( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { tiledb::sm::Consolidator::array_vacuum( array_uri, - (config == nullptr) ? ctx->storage_manager()->config() : config->config(), + (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); return TILEDB_OK; @@ -3068,9 +3068,7 @@ int32_t tiledb_array_upgrade_version( // Upgrade version throw_if_not_ok(ctx->storage_manager()->array_upgrade_version( - uri, - (config == nullptr) ? ctx->storage_manager()->config() : - config->config())); + uri, (config == nullptr) ? ctx->config() : config->config())); return TILEDB_OK; } diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index d68d103b8155..137ac504787e 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -89,7 +89,7 @@ int32_t tiledb_filestore_schema_create( &context.resources().stats(), context.compute_tp(), context.io_tp(), - context.storage_manager()->config()); + context.resources().config()); uint64_t file_size; throw_if_not_ok(vfs.file_size(tiledb::sm::URI(uri), &file_size)); if (file_size) { @@ -202,7 +202,7 @@ int32_t tiledb_filestore_uri_import( &context.resources().stats(), context.compute_tp(), context.io_tp(), - context.storage_manager()->config()); + context.resources().config()); uint64_t file_size; throw_if_not_ok(vfs.file_size(tiledb::sm::URI(file_uri), &file_size)); if (!file_size) { @@ -271,8 +271,8 @@ int32_t tiledb_filestore_uri_import( // timestamped fragments in row-major order. bool is_tiledb_uri = array->is_remote(); uint64_t tile_extent = compute_tile_extent_based_on_file_size(file_size); - auto buffer_size = get_buffer_size_from_config( - context.storage_manager()->config(), tile_extent); + auto buffer_size = + get_buffer_size_from_config(context.resources().config(), tile_extent); tiledb::sm::Query query(context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::GLOBAL_ORDER)); @@ -390,7 +390,7 @@ int32_t tiledb_filestore_uri_export( &context.resources().stats(), context.compute_tp(), context.io_tp(), - context.storage_manager()->config()); + context.resources().config()); if (!vfs.open_file(tiledb::sm::URI(file_uri), tiledb::sm::VFSMode::VFS_WRITE) .ok()) { throw api::CAPIException( @@ -422,8 +422,8 @@ int32_t tiledb_filestore_uri_export( uint64_t file_size = *static_cast(file_size_ptr); uint64_t tile_extent = compute_tile_extent_based_on_file_size(file_size); - auto buffer_size = get_buffer_size_from_config( - context.storage_manager()->config(), tile_extent); + auto buffer_size = + get_buffer_size_from_config(context.resources().config(), tile_extent); std::vector data(buffer_size); uint64_t start_range = 0; diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index cde5d10eb2c7..0efb104da0b7 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -146,7 +146,7 @@ void ArrayMetaConsolidator::vacuum(const char* array_name) { Status ArrayMetaConsolidator::set_config(const Config& config) { // Set the consolidation config for ease of use - Config merged_config = storage_manager_->config(); + Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; RETURN_NOT_OK(merged_config.get( diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 13186653aa09..301b4ee92680 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -105,7 +105,8 @@ ConsolidationMode Consolidator::mode_from_config( /* ****************************** */ Consolidator::Consolidator(StorageManager* storage_manager) - : storage_manager_(storage_manager) + : resources_(storage_manager->resources()) + , storage_manager_(storage_manager) , consolidator_memory_tracker_( storage_manager_->resources().create_memory_tracker()) , stats_(storage_manager_->stats()->create_child("Consolidator")) diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 8f324995a660..5a2340a921b5 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -238,6 +238,11 @@ class Consolidator { /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** + * Resources used to perform the operation + */ + ContextResources& resources_; + /** The storage manager. */ StorageManager* storage_manager_; diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index d556f3911abd..4b4f55cc5a93 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -914,7 +914,7 @@ void FragmentConsolidator::set_query_buffers( Status FragmentConsolidator::set_config(const Config& config) { // Set the consolidation config for ease of use - Config merged_config = storage_manager_->config(); + Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; config_.amplification_ = 0.0f; diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index ba0a74d55882..a8f3c8e99884 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -138,7 +138,7 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { Status GroupMetaConsolidator::set_config(const Config& config) { // Set the consolidation config for ease of use - Config merged_config = storage_manager_->config(); + Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; RETURN_NOT_OK(merged_config.get( diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 726bfd5e130c..c6389d4e1d11 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -70,7 +70,7 @@ Group::Group( : memory_tracker_(resources.create_memory_tracker()) , group_uri_(group_uri) , storage_manager_(storage_manager) - , config_(storage_manager_->config()) + , config_(resources.config()) , remote_(group_uri.is_tiledb()) , metadata_(memory_tracker_) , metadata_loaded_(false) diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index 29e5c4a095ba..df73c3e95ea0 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -165,10 +165,7 @@ Status DeletesAndUpdates::dowork() { auto uri = commit_uri.join_path(new_fragment_str); GenericTileIO::store_data( - storage_manager_->resources(), - uri, - serialized_condition, - *array_->encryption_key()); + resources_, uri, serialized_condition, *array_->encryption_key()); return Status::Ok(); } diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 8e785f49ef32..ebd543df22c7 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -76,12 +76,14 @@ Query::Query( shared_ptr array, optional fragment_name, optional memory_budget) - : query_memory_tracker_( + : resources_(storage_manager->resources()) + , query_memory_tracker_( storage_manager->resources().create_memory_tracker()) , array_shared_(array) , array_(array_shared_.get()) , opened_array_(array->opened_array()) , array_schema_(array->array_schema_latest_ptr()) + , config_(resources_.config()) , type_(array_->get_query_type()) , layout_( (type_ == QueryType::READ || array_schema_->dense()) ? @@ -130,9 +132,6 @@ Query::Query( callback_data_ = nullptr; status_ = QueryStatus::UNINITIALIZED; - if (storage_manager != nullptr) - config_ = storage_manager->config(); - // Set initial subarray configuration subarray_.set_config(type_, config_); diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index b4e1db07d197..f0cc32e2bc59 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -922,6 +922,11 @@ class Query { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * Resource used for operations + */ + ContextResources& resources_; + /** The query memory tracker. */ shared_ptr query_memory_tracker_; diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index db093b5de563..437cadb742e1 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -1001,7 +1001,7 @@ Status ReaderBase::unfilter_tile( t_min, t_max, concurrency_level, - storage_manager_->config())); + resources_.config())); } // Prevent processing past the end of chunks in case there are more @@ -1020,7 +1020,7 @@ Status ReaderBase::unfilter_tile( tvar_min, tvar_max, concurrency_level, - storage_manager_->config())); + resources_.config())); } } @@ -1044,7 +1044,7 @@ Status ReaderBase::unfilter_tile( tval_min, tval_max, concurrency_level, - storage_manager_->config())); + resources_.config())); } return Status::Ok(); diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 91efeee59487..878c62eb8ea1 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -47,7 +47,8 @@ namespace sm { StrategyBase::StrategyBase( stats::Stats* stats, shared_ptr logger, StrategyParams& params) - : array_memory_tracker_(params.array_memory_tracker()) + : resources_(params.resources()) + , array_memory_tracker_(params.array_memory_tracker()) , query_memory_tracker_(params.query_memory_tracker()) , stats_(stats) , logger_(logger) diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 9d31a053a831..122ad3864d68 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -37,7 +37,8 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/misc/types.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" +#include "tiledb/sm/storage_manager/context_resources.h" +#include "tiledb/sm/storage_manager/storage_manager.h" namespace tiledb { namespace sm { @@ -78,7 +79,8 @@ class StrategyParams { std::optional& condition, DefaultChannelAggregates& default_channel_aggregates, bool skip_checks_serialization) - : array_memory_tracker_(array_memory_tracker) + : resources_(storage_manager->resources()) + , array_memory_tracker_(array_memory_tracker) , query_memory_tracker_(query_memory_tracker) , storage_manager_(storage_manager) , array_(array) @@ -97,6 +99,13 @@ class StrategyParams { /* API */ /* ********************************* */ + /** + * Accessor for the resources + */ + inline ContextResources& resources() { + return resources_; + } + /** Return the array memory tracker. */ inline shared_ptr array_memory_tracker() { return array_memory_tracker_; @@ -166,6 +175,11 @@ class StrategyParams { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** + * Resources used to perform operations + */ + ContextResources& resources_; + /** Array Memory tracker. */ shared_ptr array_memory_tracker_; @@ -263,6 +277,11 @@ class StrategyBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ + /** + * Resources used for operations + */ + ContextResources& resources_; + /** The array memory tracker. */ shared_ptr array_memory_tracker_; diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index c95d5d3ef8a9..a6477626cfd1 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -2310,7 +2310,8 @@ Status array_from_query_deserialize( "Could not deserialize query; buffer is not 8-byte aligned.")); // Set traversal limit from config - uint64_t limit = storage_manager->config() + uint64_t limit = storage_manager->resources() + .config() .get("rest.capnp_traversal_limit") .value(); ::capnp::ReaderOptions readerOptions; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index ec515fab5114..24ffd03a5a98 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -122,7 +122,7 @@ StorageManagerCanonical::~StorageManagerCanonical() { bool found{false}; bool use_malloc_trim{false}; const Status& st = - config().get("sm.mem.malloc_trim", &use_malloc_trim, &found); + config_.get("sm.mem.malloc_trim", &use_malloc_trim, &found); if (st.ok() && found && use_malloc_trim) { tdb_malloc_trim(); } @@ -445,10 +445,6 @@ bool StorageManagerCanonical::cancellation_in_progress() { return cancellation_in_progress_; } -const Config& StorageManagerCanonical::config() const { - return config_; -} - void StorageManagerCanonical::decrement_in_progress() { std::unique_lock lck(queries_in_progress_mtx_); queries_in_progress_--; diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index b012174c14ce..67321e8ec12d 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -209,9 +209,6 @@ class StorageManagerCanonical { /** Returns true while all tasks are being cancelled. */ bool cancellation_in_progress(); - /** Returns the configuration parameters. */ - const Config& config() const; - /** Returns the current map of any set tags. */ const std::unordered_map& tags() const; From 390d0a32b32cf9fc9ca26bad0af1f3c7759cf743 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 16 May 2024 12:00:03 +0300 Subject: [PATCH 353/456] Update documentation of `vfs.gcs.project_id` to mention that it is only needed when creating buckets. (#4977) [SC-47367](https://app.shortcut.com/tiledb-inc/story/47367/update-documentation-of-vfs-gcs-project-id-to-mention-that-it-is-only-needed-when-creating-buckets) [This](https://github.com/TileDB-Inc/TileDB/blob/acfb4e8816ed9cdd858a5cb673a536ee40811aa1/tiledb/sm/filesystem/gcs.cc#L276-L277) is the only place this config option is used. [Bucket names are globally unique across projects.](https://cloud.google.com/storage/docs/buckets#considerations) --- TYPE: NO_HISTORY --- tiledb/api/c_api/config/config_api_external.h | 3 ++- tiledb/sm/cpp_api/config.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index b4aaafca98f9..e0d7035a519e 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -389,7 +389,8 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * attempts, in milliseconds. * **Default**: 60000 * - `vfs.gcs.project_id`
- * Set the GCS project id.
+ * Set the GCS project ID to create new buckets to. Not required unless you + * are going to use the VFS to create buckets.
* **Default**: "" * - `vfs.gcs.service_account_key`
* **Experimental**
diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 2cb91cf1bdaf..5f4d9f1b3db5 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -567,7 +567,8 @@ class Config { * attempts, in milliseconds. * **Default**: 60000 * - `vfs.gcs.project_id`
- * Set the GCS project id.
+ * Set the GCS project ID to create new buckets to. Not required unless you + * are going to use the VFS to create buckets.
* **Default**: "" * - `vfs.gcs.service_account_key`
* **Experimental**
From 93dab0ad01d117eed186e4cd8651f7b1d640a3f6 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 16 May 2024 08:42:52 -0400 Subject: [PATCH 354/456] Remove StorageManager::stats(). (#4979) Remove `StorageManager::stats()`. [sc-47389] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::stats()`. --- tiledb/api/c_api/vfs/vfs_api.cc | 2 +- .../storage_manager_override.h | 5 +---- tiledb/sm/consolidator/consolidator.cc | 9 ++++----- tiledb/sm/consolidator/consolidator.h | 6 ++---- tiledb/sm/query/query.cc | 15 ++++++--------- tiledb/sm/query/query.h | 12 ++++-------- tiledb/sm/storage_manager/storage_manager.cc | 13 ++++++------- .../storage_manager/storage_manager_canonical.h | 5 ----- tiledb/sm/subarray/subarray.cc | 9 +++++---- 9 files changed, 29 insertions(+), 47 deletions(-) diff --git a/tiledb/api/c_api/vfs/vfs_api.cc b/tiledb/api/c_api/vfs/vfs_api.cc index 6beb9f4de894..09bf912e7d41 100644 --- a/tiledb/api/c_api/vfs/vfs_api.cc +++ b/tiledb/api/c_api/vfs/vfs_api.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index b16ae2acc691..351b91f5522e 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -68,9 +68,6 @@ class StorageManagerStub { inline common::ThreadPool* io_tp() { return &resources_.io_tp(); } - inline stats::Stats* stats() { - return &resources_.stats(); - } inline VFS* vfs() { throw std::logic_error("StorageManagerStub does not instantiate a VFS"); } diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 301b4ee92680..a29d493263a9 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -107,10 +107,9 @@ ConsolidationMode Consolidator::mode_from_config( Consolidator::Consolidator(StorageManager* storage_manager) : resources_(storage_manager->resources()) , storage_manager_(storage_manager) - , consolidator_memory_tracker_( - storage_manager_->resources().create_memory_tracker()) - , stats_(storage_manager_->stats()->create_child("Consolidator")) - , logger_(storage_manager_->logger()->clone("Consolidator", ++logger_id_)) { + , consolidator_memory_tracker_(resources_.create_memory_tracker()) + , stats_(resources_.stats().create_child("Consolidator")) + , logger_(resources_.logger()->clone("Consolidator", ++logger_id_)) { consolidator_memory_tracker_->set_type(MemoryTrackerType::CONSOLIDATOR); } diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 5a2340a921b5..0b4f0ef44248 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -238,9 +238,7 @@ class Consolidator { /* PROTECTED ATTRIBUTES */ /* ********************************* */ - /** - * Resources used to perform the operation - */ + /** Resources used to perform the operation. */ ContextResources& resources_; /** The storage manager. */ diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index ebd543df22c7..d5821f75db17 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -64,8 +64,7 @@ using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -77,8 +76,7 @@ Query::Query( optional fragment_name, optional memory_budget) : resources_(storage_manager->resources()) - , query_memory_tracker_( - storage_manager->resources().create_memory_tracker()) + , query_memory_tracker_(resources_.create_memory_tracker()) , array_shared_(array) , array_(array_shared_.get()) , opened_array_(array->opened_array()) @@ -90,8 +88,8 @@ Query::Query( Layout::ROW_MAJOR : Layout::UNORDERED) , storage_manager_(storage_manager) - , stats_(storage_manager_->stats()->create_child("Query")) - , logger_(storage_manager->logger()->clone("Query", ++logger_id_)) + , stats_(resources_.stats().create_child("Query")) + , logger_(resources_.logger()->clone("Query", ++logger_id_)) , dim_label_queries_(nullptr) , has_coords_buffer_(false) , has_zipped_coords_buffer_(false) @@ -2099,5 +2097,4 @@ RestClient* Query::rest_client() const { return storage_manager_->rest_client(); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index f0cc32e2bc59..ab983521e441 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -59,8 +59,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** Class for query status exceptions. */ class QueryException : public StatusException { @@ -922,9 +921,7 @@ class Query { /* PRIVATE ATTRIBUTES */ /* ********************************* */ - /** - * Resource used for operations - */ + /** Resource used for operations. */ ContextResources& resources_; /** The query memory tracker. */ @@ -1183,7 +1180,6 @@ class Query { void copy_aggregates_data_to_user_buffer(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_QUERY_H diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 24ffd03a5a98..801b26f3c1d0 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -866,8 +866,7 @@ Status StorageManagerCanonical::store_array_schema( resources_.ephemeral_memory_tracker())}; Serializer serializer(tile->data(), tile->size()); array_schema->serialize(serializer); - - stats()->add_counter("write_array_schema_size", tile->size()); + resources_.stats().add_counter("write_array_schema_size", tile->size()); // Delete file if it exists already bool exists; @@ -926,14 +925,14 @@ Status StorageManagerCanonical::store_array_schema( Status StorageManagerCanonical::store_metadata( const URI& uri, const EncryptionKey& encryption_key, Metadata* metadata) { - auto timer_se = stats()->start_timer("write_meta"); + auto timer_se = resources_.stats().start_timer("write_meta"); // Trivial case - if (metadata == nullptr) + if (metadata == nullptr) { return Status::Ok(); + } // Serialize array metadata - SizeComputationSerializer size_computation_serializer; metadata->serialize(size_computation_serializer); @@ -946,8 +945,8 @@ Status StorageManagerCanonical::store_metadata( resources_.ephemeral_memory_tracker())}; Serializer serializer(tile->data(), tile->size()); metadata->serialize(serializer); - - stats()->add_counter("write_meta_size", size_computation_serializer.size()); + resources_.stats().add_counter( + "write_meta_size", size_computation_serializer.size()); // Create a metadata file name URI metadata_uri = metadata->get_uri(uri); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 67321e8ec12d..5b949d089083 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -412,11 +412,6 @@ class StorageManagerCanonical { return &(resources_.vfs()); } - /** Returns `stats_`. */ - [[nodiscard]] inline stats::Stats* stats() { - return &(resources_.stats()); - } - /** Returns the internal logger object. */ shared_ptr logger() const; diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 4336db12ac19..3f853876b147 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -130,10 +130,11 @@ Subarray::Subarray( const bool coalesce_ranges, StorageManager* storage_manager) : stats_( - parent_stats ? parent_stats->create_child("Subarray") : + parent_stats ? + parent_stats->create_child("Subarray") : storage_manager ? - storage_manager->stats()->create_child("subSubarray") : - nullptr) + storage_manager->resources().stats().create_child("subSubarray") : + nullptr) , logger_(logger->clone("Subarray", ++logger_id_)) , array_(opened_array) , layout_(layout) From 372507c4f6da0053efcdcdcd3768fff98b1b79d2 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 16 May 2024 16:16:46 -0400 Subject: [PATCH 355/456] Remove StorageManager::vfs() (#4982) Remove `StorageManager::vfs()` [sc-47537] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::vfs()`. --- tiledb/api/c_api/context/context_api.cc | 5 +- .../storage_manager_override.h | 3 - tiledb/sm/array/test/unit_consistency.cc | 8 +- .../consolidator/array_meta_consolidator.cc | 16 ++-- .../sm/consolidator/commits_consolidator.cc | 15 ++- tiledb/sm/consolidator/consolidator.cc | 12 +-- tiledb/sm/consolidator/consolidator.h | 4 +- .../sm/consolidator/fragment_consolidator.cc | 27 +++--- .../fragment_meta_consolidator.cc | 13 +-- .../consolidator/group_meta_consolidator.cc | 13 ++- tiledb/sm/group/group.cc | 4 +- .../deletes_and_updates.cc | 2 +- tiledb/sm/query/readers/filtered_data.h | 10 +- tiledb/sm/query/strategy_base.h | 18 ++-- .../sm/query/writers/global_order_writer.cc | 25 ++--- tiledb/sm/query/writers/ordered_writer.cc | 6 +- tiledb/sm/query/writers/unordered_writer.cc | 5 +- tiledb/sm/query/writers/writer_base.cc | 31 +++---- tiledb/sm/storage_manager/storage_manager.cc | 93 ++++++++++--------- .../storage_manager_canonical.h | 5 - 20 files changed, 148 insertions(+), 167 deletions(-) diff --git a/tiledb/api/c_api/context/context_api.cc b/tiledb/api/c_api/context/context_api.cc index 9ff7b8de04f9..4ea72796c6de 100644 --- a/tiledb/api/c_api/context/context_api.cc +++ b/tiledb/api/c_api/context/context_api.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -108,7 +108,8 @@ capi_return_t tiledb_ctx_get_last_error( capi_return_t tiledb_ctx_is_supported_fs( tiledb_ctx_t* ctx, tiledb_filesystem_t fs, int32_t* is_supported) { ensure_output_pointer_is_valid(is_supported); - *is_supported = (int32_t)ctx->storage_manager()->vfs()->supports_fs( + + *is_supported = (int32_t)ctx->context().resources().vfs().supports_fs( static_cast(fs)); return TILEDB_OK; } diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 351b91f5522e..8d123cd21fb0 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -68,9 +68,6 @@ class StorageManagerStub { inline common::ThreadPool* io_tp() { return &resources_.io_tp(); } - inline VFS* vfs() { - throw std::logic_error("StorageManagerStub does not instantiate a VFS"); - } inline Status cancel_all_tasks() { return Status{}; }; diff --git a/tiledb/sm/array/test/unit_consistency.cc b/tiledb/sm/array/test/unit_consistency.cc index c2e2b7cf2286..212a24dfbc40 100644 --- a/tiledb/sm/array/test/unit_consistency.cc +++ b/tiledb/sm/array/test/unit_consistency.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -186,7 +186,7 @@ TEST_CASE( REQUIRE(x.is_open(uri) == false); // Clean up - REQUIRE(sm.vfs()->remove_dir(uri).ok()); + REQUIRE(resources.vfs().remove_dir(uri).ok()); } TEST_CASE( @@ -227,7 +227,7 @@ TEST_CASE( // Clean up for (auto uri : uris) { - REQUIRE(sm.vfs()->remove_dir(uri).ok()); + REQUIRE(resources.vfs().remove_dir(uri).ok()); } } @@ -283,5 +283,5 @@ TEST_CASE( REQUIRE(array.get()->close().ok()); REQUIRE(x.registry_size() == 0); REQUIRE(x.is_open(uri) == false); - REQUIRE(sm.vfs()->remove_dir(uri).ok()); + REQUIRE(resources.vfs().remove_dir(uri).ok()); } diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 0efb104da0b7..11a77d2f16ff 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -112,9 +112,8 @@ Status ArrayMetaConsolidator::consolidate( throw_if_not_ok(array_for_writes.close()); // Write vacuum file - RETURN_NOT_OK( - storage_manager_->vfs()->write(vac_uri, data.c_str(), data.size())); - RETURN_NOT_OK(storage_manager_->vfs()->close_file(vac_uri)); + throw_if_not_ok(resources_.vfs().write(vac_uri, data.c_str(), data.size())); + throw_if_not_ok(resources_.vfs().close_file(vac_uri)); return Status::Ok(); } @@ -126,18 +125,15 @@ void ArrayMetaConsolidator::vacuum(const char* array_name) { } // Get the array metadata URIs and vacuum file URIs to be vacuum - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_.vfs(); auto compute_tp = storage_manager_->compute_tp(); auto array_dir = ArrayDirectory( - storage_manager_->resources(), - URI(array_name), - 0, - std::numeric_limits::max()); + resources_, URI(array_name), 0, std::numeric_limits::max()); // Delete the array metadata and vacuum files - vfs->remove_files(compute_tp, array_dir.array_meta_uris_to_vacuum()); - vfs->remove_files(compute_tp, array_dir.array_meta_vac_uris_to_vacuum()); + vfs.remove_files(compute_tp, array_dir.array_meta_uris_to_vacuum()); + vfs.remove_files(compute_tp, array_dir.array_meta_vac_uris_to_vacuum()); } /* ****************************** */ diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index b01167d93487..8bc3c6b9a175 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -82,7 +82,7 @@ Status CommitsConsolidator::consolidate( // Get the array uri to consolidate from the array directory. auto array_dir = ArrayDirectory( - storage_manager_->resources(), + resources_, URI(array_name), 0, utils::time::timestamp_now_ms(), @@ -96,7 +96,7 @@ Status CommitsConsolidator::consolidate( // Get the file name. auto& to_consolidate = array_dir.commit_uris_to_consolidate(); Consolidator::write_consolidated_commits_file( - write_version, array_dir, to_consolidate, storage_manager_); + write_version, array_dir, to_consolidate, resources_); return Status::Ok(); } @@ -109,18 +109,17 @@ void CommitsConsolidator::vacuum(const char* array_name) { // Get the array metadata URIs and vacuum file URIs to be vacuum ArrayDirectory array_dir( - storage_manager_->resources(), + resources_, URI(array_name), 0, utils::time::timestamp_now_ms(), ArrayDirectoryMode::COMMITS); // Delete the commits and vacuum files - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_.vfs(); auto compute_tp = storage_manager_->compute_tp(); - vfs->remove_files(compute_tp, array_dir.commit_uris_to_vacuum()); - vfs->remove_files( - compute_tp, array_dir.consolidated_commits_uris_to_vacuum()); + vfs.remove_files(compute_tp, array_dir.commit_uris_to_vacuum()); + vfs.remove_files(compute_tp, array_dir.consolidated_commits_uris_to_vacuum()); } } // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index a29d493263a9..1a6fe6050161 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -256,7 +256,7 @@ void Consolidator::write_consolidated_commits_file( format_version_t write_version, ArrayDirectory array_dir, const std::vector& commit_uris, - StorageManager* storage_manager) { + ContextResources& resources) { // Compute the file name. auto name = storage_format::generate_consolidated_fragment_name( commit_uris.front(), commit_uris.back(), write_version); @@ -274,7 +274,7 @@ void Consolidator::write_consolidated_commits_file( // the size variable. if (stdx::string::ends_with( uri.to_string(), constants::delete_file_suffix)) { - throw_if_not_ok(storage_manager->vfs()->file_size(uri, &file_sizes[i])); + throw_if_not_ok(resources.vfs().file_size(uri, &file_sizes[i])); total_size += file_sizes[i]; total_size += sizeof(storage_size_t); } @@ -295,8 +295,8 @@ void Consolidator::write_consolidated_commits_file( uri.to_string(), constants::delete_file_suffix)) { memcpy(&data[file_index], &file_sizes[i], sizeof(storage_size_t)); file_index += sizeof(storage_size_t); - throw_if_not_ok(storage_manager->vfs()->read( - uri, 0, &data[file_index], file_sizes[i])); + throw_if_not_ok( + resources.vfs().read(uri, 0, &data[file_index], file_sizes[i])); file_index += file_sizes[i]; } } @@ -305,9 +305,9 @@ void Consolidator::write_consolidated_commits_file( URI consolidated_commits_uri = array_dir.get_commits_dir(write_version) .join_path(name + constants::con_commits_file_suffix); - throw_if_not_ok(storage_manager->vfs()->write( + throw_if_not_ok(resources.vfs().write( consolidated_commits_uri, data.data(), data.size())); - throw_if_not_ok(storage_manager->vfs()->close_file(consolidated_commits_uri)); + throw_if_not_ok(resources.vfs().close_file(consolidated_commits_uri)); } void Consolidator::array_vacuum( diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 0b4f0ef44248..1c7d10ae2388 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -181,13 +181,13 @@ class Consolidator { * @param write_version Write version. * @param array_dir ArrayDirectory where the data is stored. * @param commit_uris Commit files to include. - * @param storage_manager The storage manager. + * @param resources The context resources. */ static void write_consolidated_commits_file( format_version_t write_version, ArrayDirectory array_dir, const std::vector& commit_uris, - StorageManager* storage_manager); + ContextResources& resources); /** * Cleans up the array, such as its consolidated fragments and array diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 4b4f55cc5a93..374597efdcb9 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -446,7 +446,7 @@ void FragmentConsolidator::vacuum(const char* array_name) { array_dir.write_commit_ignore_file(commit_uris_to_ignore); } - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_.vfs(); auto compute_tp = storage_manager_->compute_tp(); // Delete fragment directories @@ -455,22 +455,22 @@ void FragmentConsolidator::vacuum(const char* array_name) { // Remove the commit file, if present. auto commit_uri = array_dir.get_commit_uri(fragment_uris_to_vacuum[i]); bool is_file = false; - RETURN_NOT_OK(vfs->is_file(commit_uri, &is_file)); + throw_if_not_ok(vfs.is_file(commit_uri, &is_file)); if (is_file) { - RETURN_NOT_OK(vfs->remove_file(commit_uri)); + throw_if_not_ok(vfs.remove_file(commit_uri)); } bool is_dir = false; - RETURN_NOT_OK(vfs->is_dir(fragment_uris_to_vacuum[i], &is_dir)); + throw_if_not_ok(vfs.is_dir(fragment_uris_to_vacuum[i], &is_dir)); if (is_dir) { - RETURN_NOT_OK(vfs->remove_dir(fragment_uris_to_vacuum[i])); + throw_if_not_ok(vfs.remove_dir(fragment_uris_to_vacuum[i])); } return Status::Ok(); })); // Delete the vacuum files. - vfs->remove_files( + vfs.remove_files( compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); } @@ -585,10 +585,9 @@ Status FragmentConsolidator::consolidate_internal( auto st = query_w->finalize(); if (!st.ok()) { bool is_dir = false; - throw_if_not_ok( - storage_manager_->vfs()->is_dir(*new_fragment_uri, &is_dir)); + throw_if_not_ok(resources_.vfs().is_dir(*new_fragment_uri, &is_dir)); if (is_dir) - throw_if_not_ok(storage_manager_->vfs()->remove_dir(*new_fragment_uri)); + throw_if_not_ok(resources_.vfs().remove_dir(*new_fragment_uri)); return st; } @@ -600,10 +599,9 @@ Status FragmentConsolidator::consolidate_internal( to_consolidate); if (!st.ok()) { bool is_dir = false; - throw_if_not_ok( - storage_manager_->vfs()->is_dir(*new_fragment_uri, &is_dir)); + throw_if_not_ok(resources_.vfs().is_dir(*new_fragment_uri, &is_dir)); if (is_dir) - throw_if_not_ok(storage_manager_->vfs()->remove_dir(*new_fragment_uri)); + throw_if_not_ok(resources_.vfs().remove_dir(*new_fragment_uri)); return st; } @@ -1016,9 +1014,8 @@ Status FragmentConsolidator::write_vacuum_file( } auto data = ss.str(); - RETURN_NOT_OK( - storage_manager_->vfs()->write(vac_uri, data.c_str(), data.size())); - RETURN_NOT_OK(storage_manager_->vfs()->close_file(vac_uri)); + throw_if_not_ok(resources_.vfs().write(vac_uri, data.c_str(), data.size())); + throw_if_not_ok(resources_.vfs().close_file(vac_uri)); return Status::Ok(); } diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 4afa82423b23..47cf67ed2ca2 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -97,7 +97,7 @@ Status FragmentMetaConsolidator::consolidate( first, last, write_version); auto frag_md_uri = array_dir.get_fragment_metadata_dir(write_version); - RETURN_NOT_OK(storage_manager_->vfs()->create_dir(frag_md_uri)); + throw_if_not_ok(resources_.vfs().create_dir(frag_md_uri)); uri = URI(frag_md_uri.to_string() + name + constants::meta_file_suffix); // Get the consolidated fragment metadata version @@ -171,10 +171,10 @@ Status FragmentMetaConsolidator::consolidate( EncryptionKey enc_key; RETURN_NOT_OK(enc_key.set_key(encryption_type, encryption_key, key_length)); - GenericTileIO tile_io(storage_manager_->resources(), uri); + GenericTileIO tile_io(resources_, uri); [[maybe_unused]] uint64_t nbytes = 0; tile_io.write_generic(tile, enc_key, &nbytes); - RETURN_NOT_OK(storage_manager_->vfs()->close_file(uri)); + throw_if_not_ok(resources_.vfs().close_file(uri)); return Status::Ok(); } @@ -202,7 +202,7 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { } } - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_.vfs(); auto compute_tp = storage_manager_->compute_tp(); // Vacuum @@ -211,8 +211,9 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { auto& uri = fragment_meta_uris[i]; FragmentID fragment_id{uri}; auto timestamp_range{fragment_id.timestamp_range()}; - if (timestamp_range.second != t_latest) - RETURN_NOT_OK(vfs->remove_file(uri)); + if (timestamp_range.second != t_latest) { + throw_if_not_ok(vfs.remove_file(uri)); + } return Status::Ok(); })); } diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index a8f3c8e99884..2849110112f6 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -99,9 +99,8 @@ Status GroupMetaConsolidator::consolidate( throw_if_not_ok(group_for_writes.close()); // Write vacuum file - RETURN_NOT_OK( - storage_manager_->vfs()->write(vac_uri, data.c_str(), data.size())); - RETURN_NOT_OK(storage_manager_->vfs()->close_file(vac_uri)); + throw_if_not_ok(resources_.vfs().write(vac_uri, data.c_str(), data.size())); + throw_if_not_ok(resources_.vfs().close_file(vac_uri)); return Status::Ok(); } @@ -113,12 +112,12 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { } // Get the group metadata URIs and vacuum file URIs to be vacuumed - auto vfs = storage_manager_->vfs(); + auto& vfs = resources_.vfs(); auto compute_tp = storage_manager_->compute_tp(); GroupDirectory group_dir; try { group_dir = GroupDirectory( - vfs, + &vfs, compute_tp, URI(group_name), 0, @@ -128,8 +127,8 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { } // Delete the group metadata and vacuum files - vfs->remove_files(compute_tp, group_dir.group_meta_uris_to_vacuum()); - vfs->remove_files(compute_tp, group_dir.group_meta_vac_uris_to_vacuum()); + vfs.remove_files(compute_tp, group_dir.group_meta_uris_to_vacuum()); + vfs.remove_files(compute_tp, group_dir.group_meta_vac_uris_to_vacuum()); } /* ****************************** */ diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index c6389d4e1d11..81b592b3e210 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -163,7 +163,7 @@ Status Group::open( try { group_dir_ = make_shared( HERE(), - storage_manager_->vfs(), + &resources_.vfs(), storage_manager_->compute_tp(), group_uri_, timestamp_start, @@ -177,7 +177,7 @@ Status Group::open( try { group_dir_ = make_shared( HERE(), - storage_manager_->vfs(), + &resources_.vfs(), storage_manager_->compute_tp(), group_uri_, timestamp_start, diff --git a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc index df73c3e95ea0..e57b67d1ca98 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc @@ -147,7 +147,7 @@ Status DeletesAndUpdates::dowork() { // Create the commit URI if needed. auto& array_dir = array_->array_directory(); auto commit_uri = array_dir.get_commits_dir(write_version); - RETURN_NOT_OK(storage_manager_->vfs()->create_dir(commit_uri)); + throw_if_not_ok(resources_.vfs().create_dir(commit_uri)); // Serialize the negated condition (aud update values if they are not empty) // and write to disk. diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index da4b5c9c9efb..7642e87ce548 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -189,7 +189,8 @@ class FilteredData { StorageManager* storage_manager, std::vector& read_tasks, shared_ptr memory_tracker) - : memory_tracker_(memory_tracker) + : resources_(storage_manager->resources()) + , memory_tracker_(memory_tracker) , fixed_data_blocks_( memory_tracker_->get_resource(MemoryType::FILTERED_DATA)) , var_data_blocks_( @@ -396,8 +397,8 @@ class FilteredData { URI uri{file_uri(fragment_metadata_[block.frag_idx()].get(), type)}; auto task = storage_manager_->io_tp()->execute([this, offset, data, size, uri]() { - RETURN_NOT_OK( - storage_manager_->vfs()->read(uri, offset, data, size, false)); + throw_if_not_ok( + resources_.vfs().read(uri, offset, data, size, false)); return Status::Ok(); }); read_tasks_.push_back(std::move(task)); @@ -601,6 +602,9 @@ class FilteredData { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** Resources used to perform operations. */ + ContextResources& resources_; + /** Memory tracker for the filtered data. */ shared_ptr memory_tracker_; diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 122ad3864d68..1a62e36980be 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,8 +40,7 @@ #include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm { class OpenedArray; class ArraySchema; @@ -100,7 +99,7 @@ class StrategyParams { /* ********************************* */ /** - * Accessor for the resources + * Accessor for the resources. */ inline ContextResources& resources() { return resources_; @@ -175,9 +174,7 @@ class StrategyParams { /* PRIVATE ATTRIBUTES */ /* ********************************* */ - /** - * Resources used to perform operations - */ + /** Resources used to perform operations. */ ContextResources& resources_; /** Array Memory tracker. */ @@ -277,9 +274,7 @@ class StrategyBase { /* PROTECTED ATTRIBUTES */ /* ********************************* */ - /** - * Resources used for operations - */ + /** Resources used for operations. */ ContextResources& resources_; /** The array memory tracker. */ @@ -345,7 +340,6 @@ class StrategyBase { void get_dim_attr_stats() const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_STRATEGY_BASE_H diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 86bb48ba5031..08b6aa358db3 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -246,7 +246,7 @@ GlobalOrderWriter::multipart_upload_state(bool client) { for (const auto& name : buffer_names()) { auto uri = meta->uri(name); - auto&& [st2, state] = storage_manager_->vfs()->multipart_upload_state(uri); + auto&& [st2, state] = resources_.vfs().multipart_upload_state(uri); RETURN_NOT_OK_TUPLE(st2, {}); // If there is no entry for this uri, probably multipart upload is disabled // or no write was issued so far @@ -257,8 +257,7 @@ GlobalOrderWriter::multipart_upload_state(bool client) { if (array_schema_.var_size(name)) { auto var_uri = meta->var_uri(name); - auto&& [st, var_state] = - storage_manager_->vfs()->multipart_upload_state(var_uri); + auto&& [st, var_state] = resources_.vfs().multipart_upload_state(var_uri); RETURN_NOT_OK_TUPLE(st, {}); result[var_uri.remove_trailing_slash().last_path_part()] = std::move(*var_state); @@ -266,7 +265,7 @@ GlobalOrderWriter::multipart_upload_state(bool client) { if (array_schema_.is_nullable(name)) { auto validity_uri = meta->validity_uri(name); auto&& [st, val_state] = - storage_manager_->vfs()->multipart_upload_state(validity_uri); + resources_.vfs().multipart_upload_state(validity_uri); RETURN_NOT_OK_TUPLE(st, {}); result[validity_uri.remove_trailing_slash().last_path_part()] = std::move(*val_state); @@ -288,8 +287,7 @@ Status GlobalOrderWriter::set_multipart_upload_state( // uri in this case holds only the buffer name auto absolute_uri = global_write_state_->frag_meta_->fragment_uri().join_path(uri); - return storage_manager_->vfs()->set_multipart_upload_state( - absolute_uri, state); + return resources_.vfs().set_multipart_upload_state(absolute_uri, state); } /* ****************************** */ @@ -509,13 +507,13 @@ void GlobalOrderWriter::clean_up() { // Cleanup the fragment we are currently writing. There is a chance that the // URI is empty if creating the first fragment had failed. if (!uri.empty()) { - throw_if_not_ok(storage_manager_->vfs()->remove_dir(uri)); + throw_if_not_ok(resources_.vfs().remove_dir(uri)); } global_write_state_.reset(nullptr); // Cleanup all fragments pending commit. for (auto& uri : frag_uris_to_commit_) { - throw_if_not_ok(storage_manager_->vfs()->remove_dir(uri)); + throw_if_not_ok(resources_.vfs().remove_dir(uri)); } frag_uris_to_commit_.clear(); } @@ -709,7 +707,7 @@ Status GlobalOrderWriter::finalize_global_write_state() { // Write either one commit file or a consolidated commit file if multiple // fragments were written. if (frag_uris_to_commit_.size() == 0) { - RETURN_NOT_OK(storage_manager_->vfs()->touch(commit_uri)); + throw_if_not_ok(resources_.vfs().touch(commit_uri)); } else { std::vector commit_uris; commit_uris.reserve(frag_uris_to_commit_.size() + 1); @@ -720,10 +718,7 @@ Status GlobalOrderWriter::finalize_global_write_state() { auto write_version = array_->array_schema_latest().write_version(); Consolidator::write_consolidated_commits_file( - write_version, - array_->array_directory(), - commit_uris, - storage_manager_); + write_version, array_->array_directory(), commit_uris, resources_); } // Delete global write state @@ -861,7 +856,7 @@ Status GlobalOrderWriter::global_write_handle_last_tile() { void GlobalOrderWriter::nuke_global_write_state() { auto meta = global_write_state_->frag_meta_; throw_if_not_ok(close_files(meta)); - throw_if_not_ok(storage_manager_->vfs()->remove_dir(meta->fragment_uri())); + throw_if_not_ok(resources_.vfs().remove_dir(meta->fragment_uri())); global_write_state_.reset(nullptr); } diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 1cce7f7c9592..657c9db6674e 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -150,7 +150,7 @@ std::string OrderedWriter::name() { void OrderedWriter::clean_up() { if (frag_uri_.has_value()) { - throw_if_not_ok(storage_manager_->vfs()->remove_dir(frag_uri_.value())); + throw_if_not_ok(resources_.vfs().remove_dir(frag_uri_.value())); } } @@ -279,7 +279,7 @@ Status OrderedWriter::ordered_write() { // The following will make the fragment visible URI commit_uri = array_->array_directory().get_commit_uri(frag_uri_.value()); - RETURN_NOT_OK(storage_manager_->vfs()->touch(commit_uri)); + throw_if_not_ok(resources_.vfs().touch(commit_uri)); return Status::Ok(); } diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index ee8ecb0d1e71..509fdd1dcaa6 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -181,7 +181,7 @@ Status UnorderedWriter::alloc_frag_meta() { void UnorderedWriter::clean_up() { if (frag_uri_.has_value()) { - throw_if_not_ok(storage_manager_->vfs()->remove_dir(frag_uri_.value())); + throw_if_not_ok(resources_.vfs().remove_dir(frag_uri_.value())); } } @@ -732,8 +732,7 @@ Status UnorderedWriter::unordered_write() { // The following will make the fragment visible URI commit_uri = array_->array_directory().get_commit_uri(frag_uri_.value()); - - RETURN_NOT_OK(storage_manager_->vfs()->touch(commit_uri)); + throw_if_not_ok(resources_.vfs().touch(commit_uri)); // Clear some data to prevent it from being serialized. cell_pos_.clear(); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index d297410969c5..2f3a2a5fa764 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -312,7 +312,7 @@ void WriterBase::refresh_config() { shared_ptr WriterBase::create_fragment_metadata() { return make_shared( HERE(), - &storage_manager_->resources(), + &resources_, query_memory_tracker_, array_->array_schema_latest().write_version()); } @@ -613,14 +613,14 @@ Status WriterBase::close_files(shared_ptr meta) const { storage_manager_->io_tp(), 0, file_uris.size(), [&](uint64_t i) { const auto& file_uri = file_uris[i]; if (layout_ == Layout::GLOBAL_ORDER && remote_query()) { - storage_manager_->vfs()->finalize_and_close_file(file_uri); + resources_.vfs().finalize_and_close_file(file_uri); } else { - RETURN_NOT_OK(storage_manager_->vfs()->close_file(file_uri)); + throw_if_not_ok(resources_.vfs().close_file(file_uri)); } return Status::Ok(); }); - RETURN_NOT_OK(status); + throw_if_not_ok(status); return Status::Ok(); } @@ -782,11 +782,11 @@ Status WriterBase::create_fragment( // Create the directories. // Create the fragment directory, the directory for the new fragment // URI, and the commit directory. - throw_if_not_ok(storage_manager_->vfs()->create_dir( - array_dir.get_fragments_dir(write_version))); - throw_if_not_ok(storage_manager_->vfs()->create_dir(fragment_uri_)); - throw_if_not_ok(storage_manager_->vfs()->create_dir( - array_dir.get_commits_dir(write_version))); + throw_if_not_ok( + resources_.vfs().create_dir(array_dir.get_fragments_dir(write_version))); + throw_if_not_ok(resources_.vfs().create_dir(fragment_uri_)); + throw_if_not_ok( + resources_.vfs().create_dir(array_dir.get_commits_dir(write_version))); // Create fragment metadata. auto timestamp_range = std::pair(timestamp, timestamp); @@ -795,7 +795,7 @@ Status WriterBase::create_fragment( buffers_.count(constants::delete_timestamps) != 0; frag_meta = make_shared( HERE(), - &storage_manager_->resources(), + &resources_, array_->array_schema_latest_ptr(), fragment_uri_, timestamp_range, @@ -1125,7 +1125,7 @@ Status WriterBase::write_tiles( ++i, ++tile_id) { auto& tile = (*tiles)[i]; auto& t = var_size ? tile.offset_tile() : tile.fixed_tile(); - RETURN_NOT_OK(storage_manager_->vfs()->write( + throw_if_not_ok(resources_.vfs().write( uri, t.filtered_buffer().data(), t.filtered_buffer().size(), @@ -1135,7 +1135,7 @@ Status WriterBase::write_tiles( if (var_size) { auto& t_var = tile.var_tile(); - RETURN_NOT_OK(storage_manager_->vfs()->write( + throw_if_not_ok(resources_.vfs().write( var_uri, t_var.filtered_buffer().data(), t_var.filtered_buffer().size(), @@ -1160,7 +1160,7 @@ Status WriterBase::write_tiles( if (nullable) { auto& t_val = tile.validity_tile(); - RETURN_NOT_OK(storage_manager_->vfs()->write( + throw_if_not_ok(resources_.vfs().write( validity_uri, t_val.filtered_buffer().data(), t_val.filtered_buffer().size(), @@ -1189,11 +1189,10 @@ Status WriterBase::write_tiles( // requirement of remote global order writes, it should only be // done if this code is executed as a result of a remote query if (remote_query()) { - RETURN_NOT_OK( - storage_manager_->vfs()->flush_multipart_file_buffer(u)); + throw_if_not_ok(resources_.vfs().flush_multipart_file_buffer(u)); } } else { - RETURN_NOT_OK(storage_manager_->vfs()->close_file(u)); + throw_if_not_ok(resources_.vfs().close_file(u)); } } } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 801b26f3c1d0..4201aac467d2 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -174,41 +174,41 @@ Status StorageManagerCanonical::array_create( array_schema->check(config_); // Create array directory - RETURN_NOT_OK(vfs()->create_dir(array_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_uri)); // Create array schema directory URI array_schema_dir_uri = array_uri.join_path(constants::array_schema_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); // Create the enumerations directory inside the array schema directory URI array_enumerations_uri = array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_enumerations_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_enumerations_uri)); // Create commit directory URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_commit_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_commit_uri)); // Create fragments directory URI array_fragments_uri = array_uri.join_path(constants::array_fragments_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_fragments_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_fragments_uri)); // Create array metadata directory URI array_metadata_uri = array_uri.join_path(constants::array_metadata_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_metadata_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_metadata_uri)); // Create fragment metadata directory URI array_fragment_metadata_uri = array_uri.join_path(constants::array_fragment_meta_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_fragment_metadata_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_fragment_metadata_uri)); // Create dimension label directory URI array_dimension_labels_uri = array_uri.join_path(constants::array_dimension_labels_dir_name); - RETURN_NOT_OK(vfs()->create_dir(array_dimension_labels_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_dimension_labels_uri)); // Get encryption key from config Status st; @@ -241,7 +241,7 @@ Status StorageManagerCanonical::array_create( // Store array schema if (!st.ok()) { - throw_if_not_ok(vfs()->remove_dir(array_uri)); + throw_if_not_ok(resources_.vfs().remove_dir(array_uri)); return st; } @@ -362,7 +362,7 @@ Status StorageManagerCanonical::array_upgrade_version( // Create array schema directory if necessary URI array_schema_dir_uri = array_uri.join_path(constants::array_schema_dir_name); - auto st = vfs()->create_dir(array_schema_dir_uri); + auto st = resources_.vfs().create_dir(array_schema_dir_uri); RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); // Store array schema @@ -373,20 +373,22 @@ Status StorageManagerCanonical::array_upgrade_version( URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); RETURN_NOT_OK_ELSE( - vfs()->create_dir(array_commit_uri), + resources_.vfs().create_dir(array_commit_uri), logger_->status_no_return_value(st)); // Create fragments directory if necessary URI array_fragments_uri = array_uri.join_path(constants::array_fragments_dir_name); - st = vfs()->create_dir(array_fragments_uri); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); + RETURN_NOT_OK_ELSE( + resources_.vfs().create_dir(array_fragments_uri), + logger_->status_no_return_value(st)); // Create fragment metadata directory if necessary URI array_fragment_metadata_uri = array_uri.join_path(constants::array_fragment_meta_dir_name); - st = vfs()->create_dir(array_fragment_metadata_uri); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); + RETURN_NOT_OK_ELSE( + resources_.vfs().create_dir(array_fragment_metadata_uri), + logger_->status_no_return_value(st)); } return Status::Ok(); @@ -427,7 +429,7 @@ Status StorageManagerCanonical::cancel_all_tasks() { if (handle_cancel) { // Cancel any queued tasks. cancelable_tasks_.cancel_all_tasks(); - throw_if_not_ok(resources().vfs().cancel_all_tasks()); + throw_if_not_ok(resources_.vfs().cancel_all_tasks()); // Wait for in-progress queries to finish. wait_for_zero_in_progress(); @@ -464,7 +466,7 @@ Status StorageManagerCanonical::object_remove(const char* path) const { std::string("Cannot remove object '") + path + "'; Invalid TileDB object")); - return vfs()->remove_dir(uri); + return resources_.vfs().remove_dir(uri); } Status StorageManagerCanonical::object_move( @@ -486,7 +488,7 @@ Status StorageManagerCanonical::object_move( std::string("Cannot move object '") + old_path + "'; Invalid TileDB object")); - return vfs()->move_dir(old_uri, new_uri); + return resources_.vfs().move_dir(old_uri, new_uri); } const std::unordered_map& @@ -518,19 +520,19 @@ Status StorageManagerCanonical::group_create(const std::string& group_uri) { } // Create group directory - RETURN_NOT_OK(vfs()->create_dir(uri)); + throw_if_not_ok(resources_.vfs().create_dir(uri)); // Create group file URI group_filename = uri.join_path(constants::group_filename); - RETURN_NOT_OK(vfs()->touch(group_filename)); + throw_if_not_ok(resources_.vfs().touch(group_filename)); // Create metadata folder - RETURN_NOT_OK( - vfs()->create_dir(uri.join_path(constants::group_metadata_dir_name))); + throw_if_not_ok(resources_.vfs().create_dir( + uri.join_path(constants::group_metadata_dir_name))); // Create group detail folder - RETURN_NOT_OK( - vfs()->create_dir(uri.join_path(constants::group_detail_dir_name))); + throw_if_not_ok(resources_.vfs().create_dir( + uri.join_path(constants::group_detail_dir_name))); return Status::Ok(); } @@ -550,16 +552,16 @@ bool StorageManagerCanonical::is_array(const URI& uri) const { } else { // Check if the schema directory exists or not bool dir_exists = false; - throw_if_not_ok(vfs()->is_dir( + throw_if_not_ok(resources_.vfs().is_dir( uri.join_path(constants::array_schema_dir_name), &dir_exists)); if (dir_exists) { return true; } - bool schema_exists = false; // If there is no schema directory, we check schema file - throw_if_not_ok(vfs()->is_file( + bool schema_exists = false; + throw_if_not_ok(resources_.vfs().is_file( uri.join_path(constants::array_schema_filename), &schema_exists)); return schema_exists; } @@ -575,7 +577,7 @@ Status StorageManagerCanonical::is_group(const URI& uri, bool* is_group) const { *is_group = *exists; } else { // Check for new group details directory - RETURN_NOT_OK(vfs()->is_dir( + throw_if_not_ok(resources_.vfs().is_dir( uri.join_path(constants::group_detail_dir_name), is_group)); if (*is_group) { @@ -583,8 +585,8 @@ Status StorageManagerCanonical::is_group(const URI& uri, bool* is_group) const { } // Fall back to older group file to check for legacy (pre-format 12) groups - RETURN_NOT_OK( - vfs()->is_file(uri.join_path(constants::group_filename), is_group)); + throw_if_not_ok(resources_.vfs().is_file( + uri.join_path(constants::group_filename), is_group)); } return Status::Ok(); } @@ -651,7 +653,7 @@ Status StorageManagerCanonical::object_type( } else if (!uri.is_tiledb()) { // For non public cloud backends, listing a non-directory is an error. bool is_dir = false; - RETURN_NOT_OK(vfs()->is_dir(uri, &is_dir)); + throw_if_not_ok(resources_.vfs().is_dir(uri, &is_dir)); if (!is_dir) { *type = ObjectType::INVALID; return Status::Ok(); @@ -684,7 +686,7 @@ Status StorageManagerCanonical::object_iter_begin( // Get all contents of path std::vector uris; - RETURN_NOT_OK(vfs()->ls(path_uri, &uris)); + throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); // Create a new object iterator *obj_iter = tdb_new(ObjectIter); @@ -716,7 +718,7 @@ Status StorageManagerCanonical::object_iter_begin( // Get all contents of path std::vector uris; - RETURN_NOT_OK(vfs()->ls(path_uri, &uris)); + throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); // Create a new object iterator *obj_iter = tdb_new(ObjectIter); @@ -768,7 +770,7 @@ Status StorageManagerCanonical::object_iter_next_postorder( do { obj_num = obj_iter->objs_.size(); std::vector uris; - RETURN_NOT_OK(vfs()->ls(obj_iter->objs_.front(), &uris)); + throw_if_not_ok(resources_.vfs().ls(obj_iter->objs_.front(), &uris)); obj_iter->expanded_.front() = true; // Push the new TileDB objects in the front of the iterator's list @@ -815,7 +817,7 @@ Status StorageManagerCanonical::object_iter_next_preorder( // Get all contents of the next URI std::vector uris; - RETURN_NOT_OK(vfs()->ls(front_uri, &uris)); + throw_if_not_ok(resources_.vfs().ls(front_uri, &uris)); // Push the new TileDB objects in the front of the iterator's list ObjectType obj_type; @@ -870,19 +872,22 @@ Status StorageManagerCanonical::store_array_schema( // Delete file if it exists already bool exists; - RETURN_NOT_OK(vfs()->is_file(schema_uri, &exists)); - if (exists) - RETURN_NOT_OK(vfs()->remove_file(schema_uri)); + throw_if_not_ok(resources_.vfs().is_file(schema_uri, &exists)); + if (exists) { + throw_if_not_ok(resources_.vfs().remove_file(schema_uri)); + } // Check if the array schema directory exists // If not create it, this is caused by a pre-v10 array bool schema_dir_exists = false; URI array_schema_dir_uri = array_schema->array_uri().join_path(constants::array_schema_dir_name); - RETURN_NOT_OK(vfs()->is_dir(array_schema_dir_uri, &schema_dir_exists)); + throw_if_not_ok( + resources_.vfs().is_dir(array_schema_dir_uri, &schema_dir_exists)); - if (!schema_dir_exists) - RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); + if (!schema_dir_exists) { + throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); + } GenericTileIO::store_data(resources_, schema_uri, tile, encryption_key); @@ -892,11 +897,11 @@ Status StorageManagerCanonical::store_array_schema( bool enumerations_dir_exists = false; URI array_enumerations_dir_uri = array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); - RETURN_NOT_OK( - vfs()->is_dir(array_enumerations_dir_uri, &enumerations_dir_exists)); + throw_if_not_ok(resources_.vfs().is_dir( + array_enumerations_dir_uri, &enumerations_dir_exists)); if (!enumerations_dir_exists) { - RETURN_NOT_OK(vfs()->create_dir(array_enumerations_dir_uri)); + throw_if_not_ok(resources_.vfs().create_dir(array_enumerations_dir_uri)); } // Serialize all enumerations into the `__enumerations` directory diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 5b949d089083..0dad1293b5ed 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -407,11 +407,6 @@ class StorageManagerCanonical { return resources_; } - /** Returns the virtual filesystem object. */ - [[nodiscard]] inline VFS* vfs() const { - return &(resources_.vfs()); - } - /** Returns the internal logger object. */ shared_ptr logger() const; From 0991e3ceb4ebd27625cfa4d5dac4826277759970 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 17 May 2024 03:04:27 -0400 Subject: [PATCH 356/456] Migrate APIs out of StorageManager: load_delete_and_update_conditions. (#4984) Migrate APIs out of `StorageManager`: `load_delete_and_update_conditions`. [sc-46736] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `load_delete_and_update_conditions`. --- test/src/unit-cppapi-deletes.cc | 4 +- test/src/unit-cppapi-update-queries.cc | 4 +- tiledb/sm/array/array.cc | 57 +++++++++++++++++++ tiledb/sm/array/array.h | 19 ++++++- tiledb/sm/query/legacy/reader.cc | 10 ++-- .../query/readers/sparse_index_reader_base.cc | 10 ++-- tiledb/sm/storage_manager/storage_manager.cc | 51 ----------------- .../storage_manager_canonical.h | 14 ----- 8 files changed, 87 insertions(+), 82 deletions(-) diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 1f7c7a0afdfb..6aada84cda8b 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -424,7 +424,7 @@ void DeletesFx::check_delete_conditions( // Load delete conditions. auto&& [st, delete_conditions, update_values] = - sm_->load_delete_and_update_conditions(*(array_ptr->opened_array())); + array_ptr->opened_array()->load_delete_and_update_conditions(); REQUIRE(st.ok()); REQUIRE(delete_conditions->size() == qcs.size()); diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index 60e13b631d07..48c1ad268432 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB Inc. + * @copyright Copyright (c) 2017-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -203,7 +203,7 @@ void UpdatesFx::check_update_conditions( // Load delete conditions. auto&& [st, conditions, update_values] = - sm_->load_delete_and_update_conditions(*(array_ptr->opened_array())); + array_ptr->opened_array()->load_delete_and_update_conditions(); REQUIRE(st.ok()); REQUIRE(conditions->size() == qcs.size()); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f5c1cf0ca103..ade1326a8f15 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -34,6 +34,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" +#include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" @@ -50,6 +51,8 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/query/deletes_and_updates/serialization.h" +#include "tiledb/sm/query/update_value.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" @@ -105,6 +108,60 @@ Array::Array( /* API */ /* ********************************* */ +tuple< + Status, + optional>, + optional>>> +OpenedArray::load_delete_and_update_conditions() { + auto& locations = array_directory().delete_and_update_tiles_location(); + auto conditions = std::vector(locations.size()); + auto update_values = std::vector>(locations.size()); + + auto status = parallel_for( + &resources_.compute_tp(), 0, locations.size(), [&](size_t i) { + // Get condition marker. + auto& uri = locations[i].uri(); + + // Read the condition from storage. + auto tile = GenericTileIO::load( + resources_, + uri, + locations[i].offset(), + *(encryption_key()), + resources_.ephemeral_memory_tracker()); + + if (tiledb::sm::utils::parse::ends_with( + locations[i].condition_marker(), + tiledb::sm::constants::delete_file_suffix)) { + conditions[i] = tiledb::sm::deletes_and_updates::serialization:: + deserialize_condition( + i, + locations[i].condition_marker(), + tile->data(), + tile->size()); + } else if (tiledb::sm::utils::parse::ends_with( + locations[i].condition_marker(), + tiledb::sm::constants::update_file_suffix)) { + auto&& [cond, uvs] = tiledb::sm::deletes_and_updates::serialization:: + deserialize_update_condition_and_values( + i, + locations[i].condition_marker(), + tile->data(), + tile->size()); + conditions[i] = std::move(cond); + update_values[i] = std::move(uvs); + } else { + throw ArrayException("Unknown condition marker extension"); + } + + throw_if_not_ok(conditions[i].check(array_schema_latest())); + return Status::Ok(); + }); + RETURN_NOT_OK_TUPLE(status, nullopt, nullopt); + + return {Status::Ok(), conditions, update_values}; +} + const URI& Array::array_uri() const { return array_uri_; } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 64d8e74f5816..49d441cf0c0f 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -55,6 +55,8 @@ class ArraySchema; class SchemaEvolution; class FragmentMetadata; class MemoryTracker; +class UpdateValue; +class QueryCondition; enum class QueryType : uint8_t; /** @@ -99,7 +101,8 @@ class OpenedArray { uint64_t timestamp_start, uint64_t timestamp_end_opened_at, bool is_remote) - : array_dir_(ArrayDirectory(resources, array_uri)) + : resources_(resources) + , array_dir_(ArrayDirectory(resources, array_uri)) , array_schema_latest_(nullptr) , metadata_(memory_tracker) , metadata_loaded_(false) @@ -207,7 +210,21 @@ class OpenedArray { return is_remote_; } + /** + * Loads the delete and update conditions from storage. + * + * @return Status, vector of the conditions, vector of the update values. + */ + tuple< + Status, + optional>, + optional>>> + load_delete_and_update_conditions(); + private: + /** The context resources. */ + ContextResources& resources_; + /** The array directory object for listing URIs. */ optional array_dir_; diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index b789aa6392e7..119a10e902e5 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -57,8 +57,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class ReaderStatusException : public StatusException { public: @@ -278,7 +277,7 @@ Status Reader::load_initial_data() { // Load delete conditions. auto&& [st, conditions, update_values] = - storage_manager_->load_delete_and_update_conditions(*array_); + array_->load_delete_and_update_conditions(); RETURN_CANCEL_OR_ERROR(st); delete_and_update_conditions_ = std::move(*conditions); @@ -2399,5 +2398,4 @@ void Reader::fill_dense_coords_col_slab( } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index b20a46ce4ff2..db14218fbb89 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -49,8 +49,7 @@ #include -namespace tiledb { -namespace sm { +namespace tiledb::sm { class SparseIndexReaderBaseStatusException : public StatusException { public: @@ -327,7 +326,7 @@ Status SparseIndexReaderBase::load_initial_data() { // Load delete conditions. auto&& [st, conditions, update_values] = - storage_manager_->load_delete_and_update_conditions(*array_); + array_->load_delete_and_update_conditions(); RETURN_CANCEL_OR_ERROR(st); delete_and_update_conditions_ = std::move(*conditions); bool make_timestamped_conditions = need_timestamped_conditions(); @@ -1042,5 +1041,4 @@ template void SparseIndexReaderBase::compute_tile_bitmaps( template void SparseIndexReaderBase::compute_tile_bitmaps( std::vector&); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 4201aac467d2..708a76018d26 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -64,9 +64,7 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" -#include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/query/query.h" -#include "tiledb/sm/query/update_value.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/storage_manager.h" @@ -591,55 +589,6 @@ Status StorageManagerCanonical::is_group(const URI& uri, bool* is_group) const { return Status::Ok(); } -tuple< - Status, - optional>, - optional>>> -StorageManagerCanonical::load_delete_and_update_conditions( - const OpenedArray& opened_array) { - auto& locations = - opened_array.array_directory().delete_and_update_tiles_location(); - auto conditions = std::vector(locations.size()); - auto update_values = std::vector>(locations.size()); - - auto status = parallel_for(compute_tp(), 0, locations.size(), [&](size_t i) { - // Get condition marker. - auto& uri = locations[i].uri(); - - // Read the condition from storage. - auto tile = GenericTileIO::load( - resources_, - uri, - locations[i].offset(), - *(opened_array.encryption_key()), - resources_.ephemeral_memory_tracker()); - - if (tiledb::sm::utils::parse::ends_with( - locations[i].condition_marker(), - tiledb::sm::constants::delete_file_suffix)) { - conditions[i] = - tiledb::sm::deletes_and_updates::serialization::deserialize_condition( - i, locations[i].condition_marker(), tile->data(), tile->size()); - } else if (tiledb::sm::utils::parse::ends_with( - locations[i].condition_marker(), - tiledb::sm::constants::update_file_suffix)) { - auto&& [cond, uvs] = tiledb::sm::deletes_and_updates::serialization:: - deserialize_update_condition_and_values( - i, locations[i].condition_marker(), tile->data(), tile->size()); - conditions[i] = std::move(cond); - update_values[i] = std::move(uvs); - } else { - throw Status_StorageManagerError("Unknown condition marker extension"); - } - - throw_if_not_ok(conditions[i].check(opened_array.array_schema_latest())); - return Status::Ok(); - }); - RETURN_NOT_OK_TUPLE(status, nullopt, nullopt); - - return {Status::Ok(), conditions, update_values}; -} - Status StorageManagerCanonical::object_type( const URI& uri, ObjectType* type) const { URI dir_uri = uri; diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 0dad1293b5ed..52280ef11782 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -77,9 +77,7 @@ class GroupDetails; class Metadata; class MemoryTracker; class Query; -class QueryCondition; class RestClient; -class UpdateValue; class VFS; enum class EncryptionType : uint8_t; @@ -256,18 +254,6 @@ class StorageManagerCanonical { */ Status is_group(const URI& uri, bool* is_group) const; - /** - * Loads the delete and update conditions from storage. - * - * @param opened_array The opened array. - * @return Status, vector of the conditions, vector of the update values. - */ - tuple< - Status, - optional>, - optional>>> - load_delete_and_update_conditions(const OpenedArray& opened_array); - /** Removes a TileDB object (group, array). */ Status object_remove(const char* path) const; From dbce4c85f2f7809b740566c72496bddd50004bdf Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 17 May 2024 14:18:28 +0300 Subject: [PATCH 357/456] Clarify performance implication of coords check options on writes. (#4986) The documentation on https://docs.tiledb.com/main/how-to/performance/performance-tips/summary-of-factors#coordinate-deduplication had more details on the performance implications of `sm.dedup_coords` and `sm.check_coord_dups` than the API docs. This reconciles the two. [sc-44431] --- TYPE: NO_HISTORY DESC: Clarify performance implication of coords check options on writes. --- tiledb/api/c_api/config/config_api_external.h | 7 +++++-- tiledb/sm/cpp_api/config.h | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index e0d7035a519e..5bdd4d78d3d9 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -102,13 +102,16 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `sm.dedup_coords`
* If `true`, cells with duplicate coordinates will be removed during sparse * fragment writes. Note that ties during deduplication are broken - * arbitrarily.
+ * arbitrarily. Also note that this check means that it will take longer to + * perform the write operation.
* **Default**: false * - `sm.check_coord_dups`
* This is applicable only if `sm.dedup_coords` is `false`. * If `true`, an error will be thrown if there are cells with duplicate * coordinates during sparse fragmnet writes. If `false` and there are - * duplicates, the duplicates will be written without errors.
+ * duplicates, the duplicates will be written without errors. Note that this + * check is much ligher weight than the coordinate deduplication check + * enabled by `sm.dedup_coords`.
* **Default**: true * - `sm.check_coord_oob`
* If `true`, an error will be thrown if there are cells with coordinates diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 5f4d9f1b3db5..6dd3458ff452 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -276,14 +276,16 @@ class Config { * - `sm.dedup_coords`
* If `true`, cells with duplicate coordinates will be removed during * sparse fragment writes. Note that ties during deduplication are broken - * arbitrarily.
+ * arbitrarily. Also note that this check means that it will take longer to + * perform the write operation.
* **Default**: false * - `sm.check_coord_dups`
* This is applicable only if `sm.dedup_coords` is `false`. * If `true`, an error will be thrown if there are cells with duplicate - * coordinates during sparse fragment writes. If `false` and there are - * duplicates, the duplicates will be written without errors.
- * **Default**: true + * coordinates during sparse fragmnet writes. If `false` and there are + * duplicates, the duplicates will be written without errors. Note that + * this check is much ligher weight than the coordinate deduplication check + * enabled by `sm.dedup_coords`.
* - `sm.check_coord_oob`
* If `true`, an error will be thrown if there are cells with coordinates * falling outside the array domain during sparse fragment writes.
From 95893e4e7bc18f654e7cc22e8fbe184e7bf8eb28 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 20 May 2024 03:45:11 -0400 Subject: [PATCH 358/456] Migrate APIs out of StorageManager: store_metadata. (#4983) Migrate APIs out of `StorageManager`: `store_metadata`. [sc-46746] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `store_metadata`. --- tiledb/sm/array/array.cc | 4 +- tiledb/sm/metadata/CMakeLists.txt | 4 +- tiledb/sm/metadata/metadata.cc | 29 +++++++++++++++ tiledb/sm/metadata/metadata.h | 14 +++++++ tiledb/sm/metadata/test/CMakeLists.txt | 2 - tiledb/sm/storage_manager/storage_manager.cc | 37 +------------------ .../storage_manager_canonical.h | 11 ------ 7 files changed, 49 insertions(+), 52 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index ade1326a8f15..81377ec05f27 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -520,8 +520,8 @@ Status Array::close() { } else { if (query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { - st = storage_manager_->store_metadata( - array_uri_, *encryption_key(), &opened_array_->metadata()); + st = opened_array_->metadata().store( + resources_, array_uri_, *encryption_key()); if (!st.ok()) { throw StatusException(st); } diff --git a/tiledb/sm/metadata/CMakeLists.txt b/tiledb/sm/metadata/CMakeLists.txt index 2e6d29177726..76ef986ceef4 100644 --- a/tiledb/sm/metadata/CMakeLists.txt +++ b/tiledb/sm/metadata/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2023 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -31,7 +31,7 @@ include(object_library) # commence(object_library metadata) this_target_sources(metadata.cc) - this_target_object_libraries(baseline buffer constants time uri_format vfs) + this_target_object_libraries(baseline buffer constants generic_tile_io time uri_format vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/metadata/metadata.cc b/tiledb/sm/metadata/metadata.cc index b93c3a50ca81..7c3f5102a017 100644 --- a/tiledb/sm/metadata/metadata.cc +++ b/tiledb/sm/metadata/metadata.cc @@ -37,6 +37,7 @@ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" #include @@ -186,6 +187,34 @@ void Metadata::serialize(Serializer& serializer) const { } } +Status Metadata::store( + ContextResources& resources, + const URI& uri, + const EncryptionKey& encryption_key) { + auto timer_se = resources.stats().start_timer("write_meta"); + + // Serialize array metadata + SizeComputationSerializer size_computation_serializer; + serialize(size_computation_serializer); + + // Do nothing if there are no metadata to write + if (0 == size_computation_serializer.size()) { + return Status::Ok(); + } + auto tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources.ephemeral_memory_tracker())}; + Serializer serializer(tile->data(), tile->size()); + serialize(serializer); + resources.stats().add_counter( + "write_meta_size", size_computation_serializer.size()); + + // Create a metadata file name + GenericTileIO::store_data(resources, get_uri(uri), tile, encryption_key); + + return Status::Ok(); +} + void Metadata::del(const char* key) { assert(key != nullptr); diff --git a/tiledb/sm/metadata/metadata.h b/tiledb/sm/metadata/metadata.h index ec883dfdf514..d982fae1591d 100644 --- a/tiledb/sm/metadata/metadata.h +++ b/tiledb/sm/metadata/metadata.h @@ -42,6 +42,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/pmr.h" #include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/tile.h" #include "tiledb/storage_format/serialization/serializers.h" @@ -136,6 +137,19 @@ class Metadata { /** Serializes all key-value metadata items into the input buffer. */ void serialize(Serializer& serializer) const; + /** + * Stores the metadata into persistent storage. + * + * @param resources The context resources. + * @param uri The object URI. + * @param encryption_key The encryption key to use. + * @return Status + */ + Status store( + ContextResources& resources, + const URI& uri, + const EncryptionKey& encryption_key); + /** * Deletes a metadata item. * diff --git a/tiledb/sm/metadata/test/CMakeLists.txt b/tiledb/sm/metadata/test/CMakeLists.txt index 326e66e796a3..52d3c5c26b85 100644 --- a/tiledb/sm/metadata/test/CMakeLists.txt +++ b/tiledb/sm/metadata/test/CMakeLists.txt @@ -27,8 +27,6 @@ include(unit_test) commence(unit_test metadata) this_target_object_libraries(metadata mem_helpers) - # The dependency on the `tile` object library is suspect, but here for now. - this_target_object_libraries(tile) this_target_link_libraries(tiledb_test_support_lib) this_target_sources(main.cc unit_metadata.cc) conclude(unit_test) diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 708a76018d26..8d10794fd88c 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -133,8 +133,8 @@ StorageManagerCanonical::~StorageManagerCanonical() { Status StorageManagerCanonical::group_close_for_writes(Group* group) { // Flush the group metadata - RETURN_NOT_OK(store_metadata( - group->group_uri(), *group->encryption_key(), group->unsafe_metadata())); + throw_if_not_ok(group->unsafe_metadata()->store( + resources_, group->group_uri(), *group->encryption_key())); // Store any changes required if (group->group_details()->is_modified()) { @@ -877,39 +877,6 @@ Status StorageManagerCanonical::store_array_schema( return Status::Ok(); } -Status StorageManagerCanonical::store_metadata( - const URI& uri, const EncryptionKey& encryption_key, Metadata* metadata) { - auto timer_se = resources_.stats().start_timer("write_meta"); - - // Trivial case - if (metadata == nullptr) { - return Status::Ok(); - } - - // Serialize array metadata - SizeComputationSerializer size_computation_serializer; - metadata->serialize(size_computation_serializer); - - // Do nothing if there are no metadata to write - if (0 == size_computation_serializer.size()) { - return Status::Ok(); - } - auto tile{WriterTile::from_generic( - size_computation_serializer.size(), - resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile->data(), tile->size()); - metadata->serialize(serializer); - resources_.stats().add_counter( - "write_meta_size", size_computation_serializer.size()); - - // Create a metadata file name - URI metadata_uri = metadata->get_uri(uri); - - GenericTileIO::store_data(resources_, metadata_uri, tile, encryption_key); - - return Status::Ok(); -} - void StorageManagerCanonical::wait_for_zero_in_progress() { std::unique_lock lck(queries_in_progress_mtx_); queries_in_progress_cv_.wait( diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 52280ef11782..6ed3d022589a 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -378,17 +378,6 @@ class StorageManagerCanonical { const shared_ptr& array_schema, const EncryptionKey& encryption_key); - /** - * Stores the metadata into persistent storage. - * - * @param uri The object URI. - * @param encryption_key The encryption key to use. - * @param metadata The metadata. - * @return Status - */ - Status store_metadata( - const URI& uri, const EncryptionKey& encryption_key, Metadata* metadata); - [[nodiscard]] inline ContextResources& resources() const { return resources_; } From 0942e4e36d189de35bb7d3dd7f3327fb4ca33358 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Mon, 20 May 2024 17:09:56 +0300 Subject: [PATCH 359/456] Use new memory tracker for tile offsets (#4929) This PR is responsible for two things: 1. Fix the memory usage estimation for the tile offsets to match the memory tracker result 2. Switch to use the new memory tracker for offsets only. (All readers related) [sc-46275] --- TYPE: NO_HISTORY DESC: Use new memory tracker for tile offsets. --- ...it-cppapi-consolidation-with-timestamps.cc | 8 ++-- test/src/unit-dense-reader.cc | 8 ++-- test/src/unit-sparse-global-order-reader.cc | 14 +++--- .../unit-sparse-unordered-with-dups-reader.cc | 8 ++-- tiledb/common/memory_tracker.h | 5 ++- tiledb/sm/fragment/fragment_metadata.cc | 2 + .../sm/fragment/ondemand_fragment_metadata.cc | 43 ++----------------- .../v1v2preloaded_fragment_metadata.cc | 29 ------------- tiledb/sm/query/readers/dense_reader.cc | 6 +-- .../readers/sparse_global_order_reader.cc | 4 +- .../query/readers/sparse_index_reader_base.cc | 25 ++++++++++- .../sparse_unordered_with_dups_reader.cc | 4 +- 12 files changed, 57 insertions(+), 99 deletions(-) diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index 91f97861af5d..b677a6c945fc 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -672,8 +672,8 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "11000"); - cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); + cfg.set("sm.mem.total_budget", "20000"); + cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.22"); ctx_ = Context(cfg); std::string stats; @@ -722,8 +722,8 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "11000"); - cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.4"); + cfg.set("sm.mem.total_budget", "20000"); + cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.22"); ctx_ = Context(cfg); std::string stats; diff --git a/test/src/unit-dense-reader.cc b/test/src/unit-dense-reader.cc index 28d3bebb273a..f5f027c66a25 100644 --- a/test/src/unit-dense-reader.cc +++ b/test/src/unit-dense-reader.cc @@ -1126,8 +1126,8 @@ TEST_CASE_METHOD( // Each var tiles are 91 and 100 bytes respectively, this will only allow to // load one as the budget is split across two potential reads. Fixed tiles are // both 40 so they both fit in the budget. - total_budget_ = "660"; - tile_upper_memory_limit_ = "210"; + total_budget_ = "1200"; + tile_upper_memory_limit_ = "500"; update_config(); // Try to read. @@ -1243,8 +1243,8 @@ TEST_CASE_METHOD( // Each var tiles are 91 and 100 bytes respectively, this will only allow to // load one as the budget is split across two potential reads. Fixed tiles are // both 40 so they both fit in the budget. - total_budget_ = "640"; - tile_upper_memory_limit_ = "210"; + total_budget_ = "1150"; + tile_upper_memory_limit_ = "500"; update_config(); // Try to read. diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index beae5bba3bb2..0a6bf9e75f03 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -788,10 +788,10 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); } - // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment + // Two result tile (2 * (~3000 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "12000"; - ratio_coords_ = "0.30"; + total_budget_ = "30000"; + ratio_coords_ = "0.12"; update_config(); tiledb_array_t* array = nullptr; @@ -870,7 +870,7 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); // One result tile (8 + ~440) will be bigger than the budget (400). - total_budget_ = "10000"; + total_budget_ = "13000"; ratio_coords_ = "0.04"; update_config(); @@ -1207,7 +1207,7 @@ TEST_CASE( "Sparse global order reader: attribute copy memory limit", "[sparse-global-order][attribute-copy][memory-limit][rest]") { Config config; - config["sm.mem.total_budget"] = "10000"; + config["sm.mem.total_budget"] = "15000"; VFSTestSetup vfs_test_setup(config.ptr().get()); std::string array_name = vfs_test_setup.array_uri("test_sparse_global_order"); auto ctx = vfs_test_setup.ctx(); @@ -1312,9 +1312,9 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); } - // Two result tile (2 * (~1200 + 8) will be bigger than the per fragment + // Two result tile (2 * (~2600 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "12000"; + total_budget_ = "26000"; ratio_coords_ = "0.30"; update_config(); diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 95fd8f3fa566..51db6e5b93b1 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -932,7 +932,7 @@ TEST_CASE_METHOD( uint64_t data2_size = data.size() * sizeof(int); write_1d_fragment(coords2.data(), &coords2_size, data2.data(), &data2_size); - total_budget_ = "1000000"; + total_budget_ = "1500000"; ratio_array_data_ = set_subarray ? "0.003" : "0.002"; partial_tile_offsets_loading_ = "true"; update_config(); @@ -1020,8 +1020,8 @@ TEST_CASE_METHOD( } // Two result tile (2 * ~1208) will be bigger than the budget (1500). - total_budget_ = "10000"; - ratio_coords_ = "0.15"; + total_budget_ = "25500"; + ratio_coords_ = "0.06"; update_config(); tiledb_array_t* array = nullptr; @@ -1099,7 +1099,7 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); // One result tile (~505) will be larger than leftover memory. - total_budget_ = "800"; + total_budget_ = "1500"; ratio_array_data_ = "0.99"; ratio_coords_ = "0.0005"; update_config(); diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 9c64157a8191..7074d359692c 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -312,7 +312,10 @@ class MemoryTracker { */ uint64_t get_memory_available() { std::lock_guard lg(mutex_); - return memory_budget_ - memory_usage_; + if (memory_usage_ + counters_[MemoryType::TILE_OFFSETS] > memory_budget_) { + return 0; + } + return memory_budget_ - memory_usage_ - counters_[MemoryType::TILE_OFFSETS]; } /** diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 4a8a603170e7..93e1814810b6 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -2547,6 +2547,8 @@ void FragmentMetadata::load_footer( unsigned num = array_schema_->attribute_num() + 1 + has_timestamps_ + has_delete_meta_ * 2; + + // If version < 5 we use zipped coordinates, otherwise separate num += (version_ >= 5) ? array_schema_->dim_num() : 0; loaded_metadata_ptr_->resize_offsets(num); diff --git a/tiledb/sm/fragment/ondemand_fragment_metadata.cc b/tiledb/sm/fragment/ondemand_fragment_metadata.cc index c0674610fc5d..cc61c17ad308 100644 --- a/tiledb/sm/fragment/ondemand_fragment_metadata.cc +++ b/tiledb/sm/fragment/ondemand_fragment_metadata.cc @@ -174,16 +174,8 @@ void OndemandFragmentMetadata::load_tile_offsets( // Get tile offsets if (tile_offsets_num != 0) { auto size = tile_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile offsets; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } + // Get tile offsets tile_offsets_[idx].resize(tile_offsets_num); deserializer.read(&tile_offsets_[idx][0], size); } @@ -223,17 +215,8 @@ void OndemandFragmentMetadata::load_tile_var_offsets( // Get variable tile offsets if (tile_var_offsets_num != 0) { auto size = tile_var_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } + // Get tile var offsets tile_var_offsets_[idx].resize(tile_var_offsets_num); deserializer.read(&tile_var_offsets_[idx][0], size); } @@ -268,17 +251,8 @@ void OndemandFragmentMetadata::load_tile_var_sizes( // Get variable tile sizes if (tile_var_sizes_num != 0) { auto size = tile_var_sizes_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var sizes; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } + // Get tile var sizes tile_var_sizes_[idx].resize(tile_var_sizes_num); deserializer.read(&tile_var_sizes_[idx][0], size); } @@ -321,17 +295,8 @@ void OndemandFragmentMetadata::load_tile_validity_offsets( // Get tile offsets if (tile_validity_offsets_num != 0) { auto size = tile_validity_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile validity offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } + // Get tile validity offsets tile_validity_offsets_[idx].resize(tile_validity_offsets_num); if (!buff->read(&tile_validity_offsets_[idx][0], size).ok()) { throw FragmentMetadataStatusException( diff --git a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc index ea33b27b4d00..5156316161c7 100644 --- a/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc +++ b/tiledb/sm/fragment/v1v2preloaded_fragment_metadata.cc @@ -73,15 +73,6 @@ void V1V2PreloadedFragmentMetadata::load_tile_offsets( continue; auto size = tile_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile offsets; Insufficient memory budget; Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } // Get tile offsets tile_offsets_[i].resize(tile_offsets_num); @@ -119,16 +110,6 @@ void V1V2PreloadedFragmentMetadata::load_tile_var_offsets( continue; auto size = tile_var_offsets_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var offsets; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } // Get variable tile offsets tile_var_offsets_[i].resize(tile_var_offsets_num); @@ -163,16 +144,6 @@ void V1V2PreloadedFragmentMetadata::load_tile_var_sizes( continue; auto size = tile_var_sizes_num * sizeof(uint64_t); - if (memory_tracker_ != nullptr && - !memory_tracker_->take_memory(size, MemoryType::TILE_OFFSETS)) { - throw FragmentMetadataStatusException( - "Cannot load tile var sizes; Insufficient memory budget; " - "Needed " + - std::to_string(size) + " but only had " + - std::to_string(memory_tracker_->get_memory_available()) + - " from budget " + - std::to_string(memory_tracker_->get_memory_budget())); - } // Get variable tile sizes tile_var_sizes_[i].resize(tile_var_sizes_num); diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index b8a13d4d2ebc..b0967271a1c0 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -730,8 +730,7 @@ uint64_t DenseReader::compute_space_tiles_end( ThreadPool::Task& compute_task) { // For easy reference. const auto& tile_coords = subarray.tile_coords(); - uint64_t available_memory = - memory_budget_ - array_memory_tracker_->get_memory_usage(); + uint64_t available_memory = array_memory_tracker_->get_memory_available(); // If the available memory is less than the tile upper memory limit, we cannot // load two batches at once. Wait for the first compute task to complete @@ -823,8 +822,7 @@ uint64_t DenseReader::compute_space_tiles_end( // If we only include one tile, make sure we don't exceed the memory budget. if (t_end == t_start + 1) { - const auto available_memory = - memory_budget_ - array_memory_tracker_->get_memory_usage(); + const auto available_memory = array_memory_tracker_->get_memory_available(); for (auto mem : required_memory) { if (mem > available_memory) { throw DenseReaderStatusException( diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 6db4435d9049..e37612a3ff69 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -248,9 +248,7 @@ void SparseGlobalOrderReader::load_all_tile_offsets() { // Make sure we have enough space for tile offsets data. uint64_t total_tile_offset_usage = tile_offsets_size(subarray_.relevant_fragments()); - uint64_t available_memory = - array_memory_tracker_->get_memory_available() - - array_memory_tracker_->get_memory_usage(MemoryType::TILE_OFFSETS); + uint64_t available_memory = array_memory_tracker_->get_memory_available(); if (total_tile_offset_usage > available_memory) { throw SparseGlobalOrderReaderStatusException( "Cannot load tile offsets, computed size (" + diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index db14218fbb89..59c36e1f2144 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -240,7 +240,30 @@ std::vector SparseIndexReaderBase::tile_offset_sizes() { } // Finally set the size of the loaded data. - ret[frag_idx] = num * tile_num * sizeof(uint64_t); + + // The expected size of the tile offsets + unsigned offsets_size = num * tile_num * sizeof(uint64_t); + + // Other than the offsets themselves, there is also memory used for the + // initialization of the vectors that hold them. This initialization + // takes place in LoadedFragmentMetadata::resize_offsets() + + // Calculate the number of fields + unsigned num_fields = schema->attribute_num() + 1 + + fragment->has_timestamps() + + fragment->has_delete_meta() * 2; + + // If version < 5 we use zipped coordinates, otherwise separate + num_fields += (fragment->version() >= 5) ? schema->dim_num() : 0; + + // The additional memory required for the vectors to + // store the tile offsets. The number of fields is calculated above. + // Each vector requires 32 bytes. Each field requires 4 vectors. These + // are: tile_offsets_, tile_var_offsets_, tile_var_sizes_, + // tile_validity_offsets_ and are located in loaded_fragment_metadata.h + unsigned offsets_init_size = num_fields * 4 * 32; + + ret[frag_idx] = offsets_size + offsets_init_size; return Status::Ok(); })); diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 3dceedf8dba1..026538b20c03 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -252,9 +252,7 @@ void SparseUnorderedWithDupsReader::load_tile_offsets_data() { bool initial_load = tile_offsets_min_frag_idx_ == std::numeric_limits::max() && tile_offsets_max_frag_idx_ == 0; - uint64_t available_memory = - array_memory_tracker_->get_memory_available() - - array_memory_tracker_->get_memory_usage(MemoryType::TILE_OFFSETS); + uint64_t available_memory = array_memory_tracker_->get_memory_available(); auto& relevant_fragments = subarray_.relevant_fragments(); if (!partial_tile_offsets_loading_) { From 6422ef74e723271f8d2e96477a68ff1b21663b65 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Mon, 20 May 2024 16:49:03 -0400 Subject: [PATCH 360/456] Remove unused TileTuple from ResultTile::init_attr_tile. (#4990) This removes an unused TileTuple object from ResultTile::init_attr_tile added in #4705. [SC-47703](https://app.shortcut.com/tiledb-inc/story/47703/remove-unused-tiletuple-in-resulttile-init-attr-tile) --- TYPE: NO_HISTORY DESC: Remove unused TileTuple from ResultTile::init_attr_tile. --- tiledb/sm/query/readers/result_tile.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tiledb/sm/query/readers/result_tile.cc b/tiledb/sm/query/readers/result_tile.cc index 2416c77309b6..5a2c0872aaa4 100644 --- a/tiledb/sm/query/readers/result_tile.cc +++ b/tiledb/sm/query/readers/result_tile.cc @@ -143,14 +143,6 @@ void ResultTile::init_attr_tile( const std::string& name, const TileSizes tile_sizes, const TileData tile_data) { - auto tuple = TileTuple( - format_version, - array_schema, - name, - tile_sizes, - tile_data, - memory_tracker_); - if (name == constants::coords) { coords_tile_.emplace( format_version, From 133520120e0612eae9a41b39df80d4959481aa09 Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Tue, 21 May 2024 02:16:24 -0400 Subject: [PATCH 361/456] Disallow possibly-invalid reinterpret datatypes for delta and double delta compression filters. (#4992) Found by `tiledb-rs` property testing! If the delta or double delta compressor `reinterpret_datatype_` field does not divide the `input_type`, then there is a risk of an assertion failure when running the compressor. ``` tiledb/sm/compressors/delta_compressor.cc:230: static void tiledb::sm::Delta::compress(tiledb::sm::ConstBuffer*, tiledb::sm::Buffer*) [with T = long unsigned int]: Assertion `num > 0 && (input_buffer->size() % value_size == 0)' failed. ``` `value_size` is the size of the reinterpreted type, but if we have something like `u32` in the input but `u64` for the reinterpreted type, then this assertion can fail. This pull request fixes this behavior by checking the input type against reinterpret_datatype in the `accepts_input_datatype` function. --- TYPE: BUG DESC: Disallow possibly-invalid reinterpret datatypes for delta and double delta compression filters. --------- Co-authored-by: Shaun M Reed Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/sm/filter/compression_filter.cc | 6 + .../filter/test/unit_run_filter_pipeline.cc | 108 ++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index 107e0c13bf18..55c28759b8e9 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -105,6 +105,12 @@ bool CompressionFilter::accepts_input_datatype(Datatype input_type) const { input_type)) { return false; } + // We must receive an integral number of units of the reinterpret datatype + else if ( + reinterpret_datatype_ != Datatype::ANY && + datatype_size(input_type) % datatype_size(reinterpret_datatype_) != 0) { + return false; + } } return true; diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 2c225aff553d..3b5645a63032 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -1029,3 +1029,111 @@ TEST_CASE("Filter: Test compression var", "[filter][compression][var]") { tracker); } } + +TEST_CASE( + "Filter: Test delta filter reinterpret_datatype validity", + "[filter][delta]") { + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); + + auto tracker = tiledb::test::create_test_memory_tracker(); + + constexpr Datatype input_datatype = Datatype::UINT8; + + // Set-up test data. + IncrementTileDataGenerator tile_data_generator(100); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); + std::vector elements_per_chunk{100}; + + auto reinterpret_datatype = GENERATE( + Datatype::UINT8, Datatype::UINT16, Datatype::UINT32, Datatype::UINT64); + + DYNAMIC_SECTION( + "input_datatype = " + datatype_str(input_datatype) + + ", reinterpret_datatype = " + datatype_str(reinterpret_datatype)) { + if (datatype_size(input_datatype) % datatype_size(reinterpret_datatype) == + 0) { + // There is an integral number of units of `reinterpret_datatype`, + // should always work + auto compressor = CompressionFilter( + Compressor::DELTA, 1, input_datatype, reinterpret_datatype); + + FilterPipeline pipeline; + pipeline.add_filter(compressor); + check_run_pipeline_roundtrip( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); + } else { + // there may be a partial instance of `reinterpret_datatype` + auto compressor = CompressionFilter( + Compressor::DELTA, 1, input_datatype, reinterpret_datatype); + FilterPipeline pipeline; + pipeline.add_filter(compressor); + CHECK_THROWS( + FilterPipeline::check_filter_types(pipeline, input_datatype, false)); + CHECK_THROWS( + FilterPipeline::check_filter_types(pipeline, input_datatype, true)); + } + } +} + +TEST_CASE( + "Filter: Test double delta filter reinterpret_datatype validity", + "[filter][double-delta]") { + // Create TileDB needed for running pipeline. + Config config; + ThreadPool tp(4); + + auto tracker = tiledb::test::create_test_memory_tracker(); + + constexpr Datatype input_datatype = Datatype::UINT8; + + // Set-up test data. + IncrementTileDataGenerator tile_data_generator(100); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); + std::vector elements_per_chunk{100}; + + auto reinterpret_datatype = GENERATE( + Datatype::UINT8, Datatype::UINT16, Datatype::UINT32, Datatype::UINT64); + + DYNAMIC_SECTION( + "input_datatype = " + datatype_str(input_datatype) + + ", reinterpret_datatype = " + datatype_str(reinterpret_datatype)) { + if (datatype_size(input_datatype) % datatype_size(reinterpret_datatype) == + 0) { + // there is an integral number of units of `reinterpret_datatype`, + // should always work + auto compressor = CompressionFilter( + Compressor::DOUBLE_DELTA, 1, input_datatype, reinterpret_datatype); + + FilterPipeline pipeline; + pipeline.add_filter(compressor); + check_run_pipeline_roundtrip( + config, + tp, + tile, + offsets_tile, + pipeline, + &tile_data_generator, + tracker); + } else { + // there may be a partial instance of `reinterpret_datatype` + auto compressor = CompressionFilter( + Compressor::DOUBLE_DELTA, 1, input_datatype, reinterpret_datatype); + FilterPipeline pipeline; + pipeline.add_filter(compressor); + CHECK_THROWS( + FilterPipeline::check_filter_types(pipeline, input_datatype, false)); + CHECK_THROWS( + FilterPipeline::check_filter_types(pipeline, input_datatype, true)); + } + } +} From 12897936b897eebe84c294fe8b1e451da3e72baf Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 21 May 2024 11:52:00 +0200 Subject: [PATCH 362/456] Remove deprecated APIs from tests, part 4. (#4957) This removed deprecated APIs from tests. It is the final PR to remove all deprecations. [sc-46307] --- TYPE: NO_HISTORY DESC: Remove deprecated APIs from tests, part 4. --- examples/c_api/async.c | 222 ----- examples/cpp_api/async.cc | 142 ---- examples/cpp_api/png_ingestion_webp.cc | 4 +- test/CMakeLists.txt | 1 - test/src/test-capi-consolidation-plan.cc | 37 +- test/src/test-cppapi-aggregates.cc | 147 +--- test/src/test-cppapi-consolidation-plan.cc | 39 +- test/src/unit-capi-async.cc | 890 --------------------- test/src/unit-capi-dense_array.cc | 16 +- test/src/unit-cppapi-array.cc | 50 -- test/src/unit-cppapi-deletes.cc | 25 +- test/src/unit-cppapi-group.cc | 5 - test/src/unit-cppapi-update-queries.cc | 34 +- 13 files changed, 117 insertions(+), 1495 deletions(-) delete mode 100644 examples/c_api/async.c delete mode 100644 examples/cpp_api/async.cc delete mode 100644 test/src/unit-capi-async.cc diff --git a/examples/c_api/async.c b/examples/c_api/async.c deleted file mode 100644 index 8b666fcea2a6..000000000000 --- a/examples/c_api/async.c +++ /dev/null @@ -1,222 +0,0 @@ -/** - * @file async.c - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This program creates a simple 2D sparse array and shows how to write and - * read asynchronously. - */ - -#include -#include -#include - -// Name of array. -const char* array_name = "async_array"; - -void create_array() { - // Create TileDB context - tiledb_ctx_t* ctx; - tiledb_ctx_alloc(NULL, &ctx); - - // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. - int dim_domain[] = {1, 4, 1, 4}; - int tile_extents[] = {2, 2}; - tiledb_dimension_t* d1; - tiledb_dimension_alloc( - ctx, "rows", TILEDB_INT32, &dim_domain[0], &tile_extents[0], &d1); - tiledb_dimension_t* d2; - tiledb_dimension_alloc( - ctx, "cols", TILEDB_INT32, &dim_domain[2], &tile_extents[1], &d2); - - // Create domain - tiledb_domain_t* domain; - tiledb_domain_alloc(ctx, &domain); - tiledb_domain_add_dimension(ctx, domain, d1); - tiledb_domain_add_dimension(ctx, domain, d2); - - // Create a single attribute "a" so each (i,j) cell can store an integer - tiledb_attribute_t* a; - tiledb_attribute_alloc(ctx, "a", TILEDB_INT32, &a); - - // Create array schema - tiledb_array_schema_t* array_schema; - tiledb_array_schema_alloc(ctx, TILEDB_SPARSE, &array_schema); - tiledb_array_schema_set_cell_order(ctx, array_schema, TILEDB_ROW_MAJOR); - tiledb_array_schema_set_tile_order(ctx, array_schema, TILEDB_ROW_MAJOR); - tiledb_array_schema_set_domain(ctx, array_schema, domain); - tiledb_array_schema_add_attribute(ctx, array_schema, a); - - // Create array - tiledb_array_create(ctx, array_name, array_schema); - - // Clean up - tiledb_attribute_free(&a); - tiledb_dimension_free(&d1); - tiledb_dimension_free(&d2); - tiledb_domain_free(&domain); - tiledb_array_schema_free(&array_schema); - tiledb_ctx_free(&ctx); -} - -// Simply prints the input string to stdout -void print_upon_completion(void* s) { - printf("%s\n", (char*)s); -} - -void write_array() { - // Create TileDB context - tiledb_ctx_t* ctx; - tiledb_ctx_alloc(NULL, &ctx); - - // Open array for writing - tiledb_array_t* array; - tiledb_array_alloc(ctx, array_name, &array); - tiledb_array_open(ctx, array, TILEDB_WRITE); - - // Prepare some data for the array - int coords_rows[] = {1, 2, 2, 4}; - int coords_cols[] = {1, 1, 2, 3}; - uint64_t coords_size = sizeof(coords_rows); - int data[] = {1, 2, 3, 4}; - uint64_t data_size = sizeof(data); - - // Create the query - tiledb_query_t* query; - tiledb_query_alloc(ctx, array, TILEDB_WRITE, &query); - tiledb_query_set_layout(ctx, query, TILEDB_GLOBAL_ORDER); - tiledb_query_set_data_buffer(ctx, query, "a", data, &data_size); - tiledb_query_set_data_buffer(ctx, query, "rows", coords_rows, &coords_size); - tiledb_query_set_data_buffer(ctx, query, "cols", coords_cols, &coords_size); - - // Submit query - char s[100] = "Callback: Write query completed"; - tiledb_query_submit_async(ctx, query, print_upon_completion, s); - - // Wait for query to complete - printf("Write query in progress\n"); - tiledb_query_status_t status; - do { - tiledb_query_get_status(ctx, query, &status); - } while (status == TILEDB_INPROGRESS); - - // Finalize query - tiledb_query_finalize(ctx, query); - - // Close array - tiledb_array_close(ctx, array); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); - tiledb_ctx_free(&ctx); -} - -void read_array() { - // Create TileDB context - tiledb_ctx_t* ctx; - tiledb_ctx_alloc(NULL, &ctx); - - // Open array for reading - tiledb_array_t* array; - tiledb_array_alloc(ctx, array_name, &array); - tiledb_array_open(ctx, array, TILEDB_READ); - - // Read entire array - tiledb_subarray_t* subarray; - tiledb_subarray_alloc(ctx, array, &subarray); - uint64_t subarray_v[] = {1, 4, 1, 4}; - tiledb_subarray_set_subarray(ctx, subarray, subarray_v); - - // Set maximum buffer sizes - uint64_t coords_size = 16; - uint64_t data_size = 16; - - // Prepare the vector that will hold the result - int* coords_rows = (int*)malloc(coords_size); - int* coords_cols = (int*)malloc(coords_size); - int* data = (int*)malloc(data_size); - - // Create query - tiledb_query_t* query; - tiledb_query_alloc(ctx, array, TILEDB_READ, &query); - tiledb_query_set_subarray_t(ctx, query, subarray); - tiledb_query_set_layout(ctx, query, TILEDB_ROW_MAJOR); - tiledb_query_set_data_buffer(ctx, query, "a", data, &data_size); - tiledb_query_set_data_buffer(ctx, query, "rows", coords_rows, &coords_size); - tiledb_query_set_data_buffer(ctx, query, "cols", coords_cols, &coords_size); - - // Submit query - char s[100] = "Callback: Read query completed"; - tiledb_query_submit_async(ctx, query, print_upon_completion, s); - - // Wait for query to complete - printf("Read query in progress\n"); - tiledb_query_status_t status; - do { - tiledb_query_get_status(ctx, query, &status); - } while (status == TILEDB_INPROGRESS); - - // Close array - tiledb_array_close(ctx, array); - - // Print out the results. - int result_num = (int)(data_size / sizeof(int)); - for (int r = 0; r < result_num; r++) { - int i = coords_rows[r]; - int j = coords_cols[r]; - int a = data[r]; - printf("Cell (%d, %d) has data %d\n", i, j, a); - } - - // Clean up - free((void*)coords_rows); - free((void*)coords_cols); - free((void*)data); - tiledb_subarray_free(&subarray); - tiledb_array_free(&array); - tiledb_query_free(&query); - tiledb_ctx_free(&ctx); -} - -int main() { - // Get object type - tiledb_ctx_t* ctx; - tiledb_ctx_alloc(NULL, &ctx); - tiledb_object_t type; - tiledb_object_type(ctx, array_name, &type); - tiledb_ctx_free(&ctx); - - if (type != TILEDB_ARRAY) { - create_array(); - write_array(); - } - - read_array(); - - return 0; -} diff --git a/examples/cpp_api/async.cc b/examples/cpp_api/async.cc deleted file mode 100644 index f1fc91a44645..000000000000 --- a/examples/cpp_api/async.cc +++ /dev/null @@ -1,142 +0,0 @@ -/** - * @file async.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This program creates a simple 2D sparse array and shows how to write and - * read asynchronously. - */ - -#include -#include - -using namespace tiledb; - -// Name of array. -std::string array_name("async_array"); - -void create_array() { - // Create a TileDB context. - Context ctx; - - // Create domain. - Domain domain(ctx); - domain.add_dimension(Dimension::create(ctx, "rows", {{1, 4}}, 2)) - .add_dimension(Dimension::create(ctx, "cols", {{1, 4}}, 2)); - - // The array will be sparse. - ArraySchema schema(ctx, TILEDB_SPARSE); - schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); - - // Add a single attribute - schema.add_attribute(Attribute::create(ctx, "a")); - - // Create the (empty) array on disk. - Array::create(array_name, schema); -} - -void write_array() { - Context ctx; - - // Prepare some data for the array - std::vector coords_rows = {1, 2, 2, 4}; - std::vector coords_cols = {1, 1, 2, 3}; - std::vector data = {1, 2, 3, 4}; - - // Open the array for writing and create the query. - Array array(ctx, array_name, TILEDB_WRITE); - Query query(ctx, array); - query.set_layout(TILEDB_GLOBAL_ORDER) - .set_data_buffer("a", data) - .set_data_buffer("rows", coords_rows) - .set_data_buffer("cols", coords_cols); - - // Submit query asynchronously with callback - query.submit_async( - []() { std::cout << "Callback: Write query completed\n"; }); - - // Wait for query to complete - std::cout << "Write query in progress\n"; - tiledb::Query::Status status; - do { - status = query.query_status(); - } while (status == tiledb::Query::Status::INPROGRESS); - - // Finalize query and close the array. - query.finalize(); - array.close(); -} - -void read_array() { - Context ctx; - Array array(ctx, array_name, TILEDB_READ); - Subarray subarray(ctx, array); - subarray.add_range(0, 1, 4).add_range(1, 1, 4); - std::vector data(4); - std::vector coords_rows(4); - std::vector coords_cols(4); - - // Prepare the query - Query query(ctx, array, TILEDB_READ); - query.set_subarray(subarray) - .set_layout(TILEDB_ROW_MAJOR) - .set_data_buffer("a", data) - .set_data_buffer("rows", coords_rows) - .set_data_buffer("cols", coords_cols); - - // Submit query asynchronously with callback - query.submit_async([]() { std::cout << "Callback: Read query completed\n"; }); - - // Wait for query to complete - std::cout << "Read query in progress\n"; - tiledb::Query::Status status; - do { - status = query.query_status(); - } while (status == tiledb::Query::Status::INPROGRESS); - - // Print out the results. - auto result_num = (int)query.result_buffer_elements()["a"].second; - for (int r = 0; r < result_num; r++) { - int i = coords_rows[r]; - int j = coords_cols[r]; - int a = data[r]; - std::cout << "Cell (" << i << ", " << j << ") has data " << a << "\n"; - } - - // Submit the query and close the array. - array.close(); -} - -int main() { - Context ctx; - if (Object::object(ctx, array_name).type() != Object::Type::Array) { - create_array(); - write_array(); - } - read_array(); - return 0; -} diff --git a/examples/cpp_api/png_ingestion_webp.cc b/examples/cpp_api/png_ingestion_webp.cc index 56610fb7fc95..953c3cc4e9eb 100644 --- a/examples/cpp_api/png_ingestion_webp.cc +++ b/examples/cpp_api/png_ingestion_webp.cc @@ -310,12 +310,14 @@ void read_png_array( std::vector subarray = {1, array_height, 1, array_width}; auto output_width = subarray[3], output_height = subarray[1]; + Subarray sub(ctx, array); + sub.set_subarray(subarray); // Allocate query and set subarray. std::vector rgba(output_height * output_width); Query query(ctx, array); query.set_layout(TILEDB_ROW_MAJOR) - .set_subarray(subarray) + .set_subarray(sub) .set_data_buffer("rgba", rgba); // Read from the array. query.submit(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 455255d78b34..816ce9afb8cd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -125,7 +125,6 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-capi-as_built.cc src/unit-capi-array.cc src/unit-capi-array_schema.cc - src/unit-capi-async.cc src/unit-capi-attributes.cc src/unit-capi-config.cc src/unit-capi-context.cc diff --git a/test/src/test-capi-consolidation-plan.cc b/test/src/test-capi-consolidation-plan.cc index 39c5316becbf..f190a41e1e8f 100644 --- a/test/src/test-capi-consolidation-plan.cc +++ b/test/src/test-capi-consolidation-plan.cc @@ -48,13 +48,12 @@ struct ConsolidationPlanFx { ConsolidationPlanFx(); // Functions. - void create_sparse_array(bool allows_dups = false, bool encrypt = false); + void create_sparse_array(bool allows_dups = false); void write_sparse( std::vector a1, std::vector dim1, std::vector dim2, - uint64_t timestamp, - bool encrypt = false); + uint64_t timestamp); void remove_sparse_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); @@ -66,9 +65,6 @@ struct ConsolidationPlanFx { tiledb_ctx_t* ctx_c_; std::string array_name_; Context ctx_; - - std::string key_ = "0123456789abcdeF0123456789abcdeF"; - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; }; ConsolidationPlanFx::ConsolidationPlanFx() { @@ -80,7 +76,7 @@ ConsolidationPlanFx::ConsolidationPlanFx() { array_name_ = vfs_test_setup_.array_uri("test_consolidation_plan_array"); } -void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) { +void ConsolidationPlanFx::create_sparse_array(bool allows_dups) { // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 4}}, 2); auto d2 = Dimension::create(ctx_, "d2", {{1, 4}}, 2); @@ -109,35 +105,20 @@ void ConsolidationPlanFx::create_sparse_array(bool allows_dups, bool encrypt) { filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(array_name_, schema, enc_type_, key_); - } else { - Array::create(array_name_, schema); - } + Array::create(array_name_, schema); } void ConsolidationPlanFx::write_sparse( std::vector a1, std::vector dim1, std::vector dim2, - uint64_t timestamp, - bool encrypt) { + uint64_t timestamp) { // Open array. - std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - array_name_, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, array_name_, TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp)); - } + Array array( + ctx_, array_name_, TILEDB_WRITE, TemporalPolicy(TimeTravel, timestamp)); // Create query. - Query query(ctx_, *array, TILEDB_WRITE); + Query query(ctx_, array, TILEDB_WRITE); query.set_layout(TILEDB_GLOBAL_ORDER); query.set_data_buffer("a1", a1); query.set_data_buffer("d1", dim1); @@ -147,7 +128,7 @@ void ConsolidationPlanFx::write_sparse( query.submit_and_finalize(); // Close array. - array->close(); + array.close(); } void ConsolidationPlanFx::check_last_error(std::string expected) { diff --git a/test/src/test-cppapi-aggregates.cc b/test/src/test-cppapi-aggregates.cc index 5d10e7c7e6b6..e2286a879326 100644 --- a/test/src/test-cppapi-aggregates.cc +++ b/test/src/test-cppapi-aggregates.cc @@ -69,9 +69,6 @@ struct CppAggregatesFx { std::vector dim_name_values_; std::vector layout_values_; - std::string key_ = "0123456789abcdeF0123456789abcdeF"; - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; - // Constructors/destructors. CppAggregatesFx(); ~CppAggregatesFx(); @@ -79,22 +76,20 @@ struct CppAggregatesFx { // Functions. void generate_test_params(); void run_all_combinations(std::function fn); - void create_dense_array(bool var = false, bool encrypt = false); - void create_sparse_array(bool var = false, bool encrypt = false); + void create_dense_array(bool var = false); + void create_sparse_array(bool var = false); void write_sparse( std::vector a, std::vector dim1, std::vector dim2, uint64_t timestamp, - optional> a_validity = nullopt, - bool encrypt = false); + optional> a_validity = nullopt); void write_sparse( std::vector a, std::vector dim1, std::vector dim2, uint64_t timestamp, - optional> a_validity = nullopt, - bool encrypt = false); + optional> a_validity = nullopt); void write_dense( std::vector a, uint64_t dim1_min, @@ -102,8 +97,7 @@ struct CppAggregatesFx { uint64_t dim2_min, uint64_t dim2_max, uint64_t timestamp, - optional> a_validity = nullopt, - bool encrypt = false); + optional> a_validity = nullopt); void write_dense_string( std::vector a, uint64_t dim1_min, @@ -111,8 +105,7 @@ struct CppAggregatesFx { uint64_t dim2_min, uint64_t dim2_max, uint64_t timestamp, - optional> a_validity = nullopt, - bool encrypt = false); + optional> a_validity = nullopt); std::vector make_data_buff( std::vector values, optional> validity = nullopt); @@ -213,7 +206,7 @@ void CppAggregatesFx::run_all_combinations(std::function fn) { } template -void CppAggregatesFx::create_dense_array(bool var, bool encrypt) { +void CppAggregatesFx::create_dense_array(bool var) { // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 12}}, 3); auto d2 = Dimension::create(ctx_, "d2", {{1, 12}}, 3); @@ -250,15 +243,11 @@ void CppAggregatesFx::create_dense_array(bool var, bool encrypt) { filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(ARRAY_NAME, schema, enc_type_, key_); - } else { - Array::create(ARRAY_NAME, schema); - } + Array::create(ARRAY_NAME, schema); } template -void CppAggregatesFx::create_sparse_array(bool var, bool encrypt) { +void CppAggregatesFx::create_sparse_array(bool var) { // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 999}}, 2); auto d2 = Dimension::create(ctx_, "d2", {{1, 999}}, 2); @@ -300,11 +289,7 @@ void CppAggregatesFx::create_sparse_array(bool var, bool encrypt) { filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(ARRAY_NAME, schema, enc_type_, key_); - } else { - Array::create(ARRAY_NAME, schema); - } + Array::create(ARRAY_NAME, schema); } template @@ -313,24 +298,13 @@ void CppAggregatesFx::write_sparse( std::vector dim1, std::vector dim2, uint64_t timestamp, - optional> a_validity, - bool encrypt) { + optional> a_validity) { // Open array. - std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimestampStartEnd, 0, timestamp)); - } + Array array( + ctx_, + ARRAY_NAME, + TILEDB_WRITE, + TemporalPolicy(TimestampStartEnd, 0, timestamp)); std::vector a_buff = make_data_buff(a); uint64_t cell_val_num = std::is_same::value ? @@ -338,7 +312,7 @@ void CppAggregatesFx::write_sparse( 1; // Create query. - Query query(ctx_, *array, TILEDB_WRITE); + Query query(ctx_, array, TILEDB_WRITE); query.set_layout(TILEDB_GLOBAL_ORDER); query.set_data_buffer( "a1", static_cast(a_buff.data()), a.size() * cell_val_num); @@ -356,7 +330,7 @@ void CppAggregatesFx::write_sparse( query.finalize(); // Close array. - array->close(); + array.close(); } template @@ -365,24 +339,13 @@ void CppAggregatesFx::write_sparse( std::vector dim1, std::vector dim2, uint64_t timestamp, - optional> a_validity, - bool encrypt) { + optional> a_validity) { // Open array. - std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimestampStartEnd, 0, timestamp)); - } + Array array( + ctx_, + ARRAY_NAME, + TILEDB_WRITE, + TemporalPolicy(TimestampStartEnd, 0, timestamp)); std::string a_data; std::vector a_offsets; @@ -394,7 +357,7 @@ void CppAggregatesFx::write_sparse( } // Create query. - Query query(ctx_, *array, TILEDB_WRITE); + Query query(ctx_, array, TILEDB_WRITE); query.set_layout(TILEDB_GLOBAL_ORDER); query.set_offsets_buffer("a1", a_offsets); query.set_data_buffer("a1", a_data); @@ -412,7 +375,7 @@ void CppAggregatesFx::write_sparse( query.finalize(); // Close array. - array->close(); + array.close(); } template @@ -423,24 +386,13 @@ void CppAggregatesFx::write_dense( uint64_t dim2_min, uint64_t dim2_max, uint64_t timestamp, - optional> a_validity, - bool encrypt) { + optional> a_validity) { // Open array. - std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimestampStartEnd, 0, timestamp)); - } + Array array( + ctx_, + ARRAY_NAME, + TILEDB_WRITE, + TemporalPolicy(TimestampStartEnd, 0, timestamp)); std::vector a_buff = make_data_buff(a); uint64_t cell_val_num = std::is_same::value ? @@ -448,11 +400,11 @@ void CppAggregatesFx::write_dense( 1; // Create the subarray. - Subarray subarray(ctx_, *array); + Subarray subarray(ctx_, array); subarray.add_range(0, dim1_min, dim1_max).add_range(1, dim2_min, dim2_max); // Create query. - Query query(ctx_, *array, TILEDB_WRITE); + Query query(ctx_, array, TILEDB_WRITE); query.set_layout(TILEDB_ROW_MAJOR); query.set_subarray(subarray); query.set_data_buffer( @@ -469,7 +421,7 @@ void CppAggregatesFx::write_dense( query.finalize(); // Close array. - array->close(); + array.close(); } template @@ -480,24 +432,13 @@ void CppAggregatesFx::write_dense_string( uint64_t dim2_min, uint64_t dim2_max, uint64_t timestamp, - optional> a_validity, - bool encrypt) { + optional> a_validity) { // Open array. - std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, - ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimestampStartEnd, 0, timestamp)); - } + Array array( + ctx_, + ARRAY_NAME, + TILEDB_WRITE, + TemporalPolicy(TimestampStartEnd, 0, timestamp)); std::string a_data; std::vector a_offsets; @@ -509,11 +450,11 @@ void CppAggregatesFx::write_dense_string( } // Create the subarray. - Subarray subarray(ctx_, *array); + Subarray subarray(ctx_, array); subarray.add_range(0, dim1_min, dim1_max).add_range(1, dim2_min, dim2_max); // Create query. - Query query(ctx_, *array, TILEDB_WRITE); + Query query(ctx_, array, TILEDB_WRITE); query.set_layout(TILEDB_ROW_MAJOR); query.set_subarray(subarray); query.set_offsets_buffer("a1", a_offsets); @@ -532,7 +473,7 @@ void CppAggregatesFx::write_dense_string( query.finalize(); // Close array. - array->close(); + array.close(); } template diff --git a/test/src/test-cppapi-consolidation-plan.cc b/test/src/test-cppapi-consolidation-plan.cc index 02c9e373a6d8..1c0e8ae80235 100644 --- a/test/src/test-cppapi-consolidation-plan.cc +++ b/test/src/test-cppapi-consolidation-plan.cc @@ -54,21 +54,17 @@ struct CppConsolidationPlanFx { VFS vfs_; Config cfg_; - std::string key_ = "0123456789abcdeF0123456789abcdeF"; - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; - // Constructors/destructors. CppConsolidationPlanFx(); ~CppConsolidationPlanFx(); // Functions. - void create_sparse_array(bool allows_dups = false, bool encrypt = false); + void create_sparse_array(bool allows_dups = false); std::string write_sparse( std::vector a1, std::vector dim1, std::vector dim2, - uint64_t timestamp, - bool encrypt = false); + uint64_t timestamp); void remove_sparse_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); @@ -97,8 +93,7 @@ CppConsolidationPlanFx::~CppConsolidationPlanFx() { remove_sparse_array(); } -void CppConsolidationPlanFx::create_sparse_array( - bool allows_dups, bool encrypt) { +void CppConsolidationPlanFx::create_sparse_array(bool allows_dups) { // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 999}}, 2); auto d2 = Dimension::create(ctx_, "d2", {{1, 999}}, 2); @@ -127,35 +122,21 @@ void CppConsolidationPlanFx::create_sparse_array( filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(SPARSE_ARRAY_NAME, schema, enc_type_, key_); - } else { - Array::create(SPARSE_ARRAY_NAME, schema); - } + Array::create(SPARSE_ARRAY_NAME, schema); } std::string CppConsolidationPlanFx::write_sparse( std::vector a1, std::vector dim1, std::vector dim2, - uint64_t timestamp, - bool encrypt) { + uint64_t timestamp) { // Open array. std::unique_ptr array; - if (encrypt) { - array = std::make_unique( - ctx_, - SPARSE_ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimeTravel, timestamp), - EncryptionAlgorithm(AESGCM, key_.c_str())); - } else { - array = std::make_unique( - ctx_, - SPARSE_ARRAY_NAME, - TILEDB_WRITE, - TemporalPolicy(TimestampStartEnd, 0, timestamp)); - } + array = std::make_unique( + ctx_, + SPARSE_ARRAY_NAME, + TILEDB_WRITE, + TemporalPolicy(TimestampStartEnd, 0, timestamp)); // Create query. Query query(ctx_, *array, TILEDB_WRITE); diff --git a/test/src/unit-capi-async.cc b/test/src/unit-capi-async.cc deleted file mode 100644 index 1bc307aac29e..000000000000 --- a/test/src/unit-capi-async.cc +++ /dev/null @@ -1,890 +0,0 @@ -/** - * @file unit-capi-async.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2021 TileDB Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests the C API async queries. - */ - -#include -#include "test/support/src/helpers.h" -#include "tiledb/sm/c_api/tiledb.h" - -#include - -using namespace tiledb::test; - -/** Tests for C API async queries. */ -struct AsyncFx { - // Constants - const char* DENSE_ARRAY_NAME = "test_async_dense"; - const char* SPARSE_ARRAY_NAME = "test_async_sparse"; - - // TileDB context - tiledb_ctx_t* ctx_; - - // separate subarray prepared 'outside' of a query - bool use_external_subarray_ = false; - - // Constructors/destructors - AsyncFx(); - ~AsyncFx(); - - // Functions - void create_dense_array(); - void create_sparse_array(); - void write_dense_async(); - void write_sparse_async(); - void write_sparse_async_cancelled(); - void read_dense_async(); - void read_sparse_async(); - void remove_dense_array(); - void remove_sparse_array(); - void remove_array(const std::string& array_name); - bool is_array(const std::string& array_name); -}; - -AsyncFx::AsyncFx() { - ctx_ = nullptr; - REQUIRE(tiledb_ctx_alloc(NULL, &ctx_) == TILEDB_OK); -} - -AsyncFx::~AsyncFx() { - tiledb_ctx_free(&ctx_); -} - -void AsyncFx::create_dense_array() { - // Create dimensions - uint64_t dim_domain[] = {1, 4, 1, 4}; - uint64_t tile_extents[] = {2, 2}; - tiledb_dimension_t* d1; - int rc = tiledb_dimension_alloc( - ctx_, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1); - CHECK(rc == TILEDB_OK); - tiledb_dimension_t* d2; - rc = tiledb_dimension_alloc( - ctx_, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2); - CHECK(rc == TILEDB_OK); - - // Create domain - tiledb_domain_t* domain; - rc = tiledb_domain_alloc(ctx_, &domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx_, domain, d1); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx_, domain, d2); - CHECK(rc == TILEDB_OK); - - // Create attributes - tiledb_attribute_t* a1; - rc = tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &a1); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a1, TILEDB_FILTER_LZ4, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a1, 1); - CHECK(rc == TILEDB_OK); - tiledb_attribute_t* a2; - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_alloc(ctx_, "a2", TILEDB_CHAR, &a2); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a2, TILEDB_FILTER_GZIP, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a2, TILEDB_VAR_NUM); - CHECK(rc == TILEDB_OK); - tiledb_attribute_t* a3; - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_alloc(ctx_, "a3", TILEDB_FLOAT32, &a3); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a3, TILEDB_FILTER_ZSTD, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a3, 2); - CHECK(rc == TILEDB_OK); - - // Create array schema - tiledb_array_schema_t* array_schema; - rc = tiledb_array_schema_alloc(ctx_, TILEDB_DENSE, &array_schema); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_cell_order(ctx_, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_tile_order(ctx_, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_domain(ctx_, array_schema, domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a1); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a2); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a3); - CHECK(rc == TILEDB_OK); - - // Check array schema - rc = tiledb_array_schema_check(ctx_, array_schema); - CHECK(rc == TILEDB_OK); - - // Create array - rc = tiledb_array_create(ctx_, DENSE_ARRAY_NAME, array_schema); - CHECK(rc == TILEDB_OK); - - // Clean up - - tiledb_attribute_free(&a1); - tiledb_attribute_free(&a2); - tiledb_attribute_free(&a3); - tiledb_dimension_free(&d1); - tiledb_dimension_free(&d2); - tiledb_domain_free(&domain); - tiledb_array_schema_free(&array_schema); -} - -void AsyncFx::create_sparse_array() { - // Create dimensions - uint64_t dim_domain[] = {1, 4, 1, 4}; - uint64_t tile_extents[] = {2, 2}; - tiledb_dimension_t* d1; - int rc = tiledb_dimension_alloc( - ctx_, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1); - CHECK(rc == TILEDB_OK); - tiledb_dimension_t* d2; - rc = tiledb_dimension_alloc( - ctx_, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2); - CHECK(rc == TILEDB_OK); - - // Create domain - tiledb_domain_t* domain; - rc = tiledb_domain_alloc(ctx_, &domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx_, domain, d1); - CHECK(rc == TILEDB_OK); - rc = tiledb_domain_add_dimension(ctx_, domain, d2); - CHECK(rc == TILEDB_OK); - - // Create attributes - tiledb_attribute_t* a1; - rc = tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &a1); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a1, TILEDB_FILTER_LZ4, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a1, 1); - CHECK(rc == TILEDB_OK); - tiledb_attribute_t* a2; - rc = tiledb_attribute_alloc(ctx_, "a2", TILEDB_CHAR, &a2); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a2, TILEDB_FILTER_GZIP, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a2, TILEDB_VAR_NUM); - CHECK(rc == TILEDB_OK); - tiledb_attribute_t* a3; - rc = tiledb_attribute_alloc(ctx_, "a3", TILEDB_FLOAT32, &a3); - CHECK(rc == TILEDB_OK); - rc = set_attribute_compression_filter(ctx_, a3, TILEDB_FILTER_ZSTD, -1); - CHECK(rc == TILEDB_OK); - rc = tiledb_attribute_set_cell_val_num(ctx_, a3, 2); - CHECK(rc == TILEDB_OK); - - // Create array schema - tiledb_array_schema_t* array_schema; - rc = tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &array_schema); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_cell_order(ctx_, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_tile_order(ctx_, array_schema, TILEDB_ROW_MAJOR); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_capacity(ctx_, array_schema, 2); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_set_domain(ctx_, array_schema, domain); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a1); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a2); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_schema_add_attribute(ctx_, array_schema, a3); - CHECK(rc == TILEDB_OK); - - // Check array schema - rc = tiledb_array_schema_check(ctx_, array_schema); - CHECK(rc == TILEDB_OK); - - // Create array - rc = tiledb_array_create(ctx_, SPARSE_ARRAY_NAME, array_schema); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_attribute_free(&a1); - tiledb_attribute_free(&a2); - tiledb_attribute_free(&a3); - tiledb_dimension_free(&d1); - tiledb_dimension_free(&d2); - tiledb_domain_free(&domain); - tiledb_array_schema_free(&array_schema); -} - -void callback(void* v) { - *(int*)v = 1; -} - -void AsyncFx::write_dense_async() { - // Set attributes - const char* attributes[] = {"a1", "a2", "a3"}; - - // Prepare cell buffers - // clang-format off - int buffer_a1[] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15 - }; - uint64_t buffer_a2[] = { - 0, 1, 3, 6, 10, 11, 13, 16, - 20, 21, 23, 26, 30, 31, 33, 36 - }; - char buffer_var_a2[] = - "abbcccdddd" - "effggghhhh" - "ijjkkkllll" - "mnnooopppp"; - float buffer_a3[] = { - 0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, - 4.1f, 4.2f, 5.1f, 5.2f, 6.1f, 6.2f, 7.1f, 7.2f, - 8.1f, 8.2f, 9.1f, 9.2f, 10.1f, 10.2f, 11.1f, 11.2f, - 12.1f, 12.2f, 13.1f, 13.2f, 14.1f, 14.2f, 15.1f, 15.2f, - }; - void* buffers[] = { buffer_a1, buffer_a2, buffer_var_a2, buffer_a3 }; - uint64_t buffer_sizes[] = - { - sizeof(buffer_a1), - sizeof(buffer_a2), - sizeof(buffer_var_a2)-1, // No need to store the last '\0' character - sizeof(buffer_a3) - }; - // clang-format on - - // Open array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); - CHECK(rc == TILEDB_OK); - - // Create query - tiledb_query_t* query; - rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); - CHECK(rc == TILEDB_OK); - - auto proc_query = [&]() -> void { - // Submit query asynchronously - int callback_made = 0; - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - - // Wait for query to complete - tiledb_query_status_t status; - do { - rc = tiledb_query_get_status(ctx_, query, &status); - CHECK(rc == TILEDB_OK); - } while (status != TILEDB_COMPLETED); - - // Finalize query - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); - - // Check correct execution of callback - CHECK(callback_made == 1); - }; - if (!use_external_subarray_) { - proc_query(); - } else { - tiledb_subarray_t* query_subarray; - rc = tiledb_query_get_subarray_t(ctx_, query, &query_subarray); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray_t(ctx_, query, query_subarray); - CHECK(rc == TILEDB_OK); - - proc_query(); - - tiledb_subarray_free(&query_subarray); - } - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); -} - -void AsyncFx::write_sparse_async() { - // Prepare cell buffers - int buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; - uint64_t buffer_a2[] = {0, 1, 3, 6, 10, 11, 13, 16}; - char buffer_var_a2[] = "abbcccddddeffggghhhh"; - float buffer_a3[] = { - 0.1f, - 0.2f, - 1.1f, - 1.2f, - 2.1f, - 2.2f, - 3.1f, - 3.2f, - 4.1f, - 4.2f, - 5.1f, - 5.2f, - 6.1f, - 6.2f, - 7.1f, - 7.2f}; - uint64_t buffer_coords_dim1[] = {1, 1, 1, 2, 3, 4, 3, 3}; - uint64_t buffer_coords_dim2[] = {1, 2, 4, 3, 1, 2, 3, 4}; - void* buffers[] = { - buffer_a1, - buffer_a2, - buffer_var_a2, - buffer_a3, - buffer_coords_dim1, - buffer_coords_dim2}; - uint64_t buffer_sizes[] = { - sizeof(buffer_a1), - sizeof(buffer_a2), - sizeof(buffer_var_a2) - 1, // No need to store the last '\0' character - sizeof(buffer_a3), - sizeof(buffer_coords_dim1)}; - - // Open array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); - CHECK(rc == TILEDB_OK); - - // Create query - tiledb_query_t* query; - const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); - CHECK(rc == TILEDB_OK); - - auto proc_query = [&]() -> void { - // Submit query asynchronously - int callback_made = 0; - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - - if (rc == TILEDB_OK) { - // Wait for query to complete - tiledb_query_status_t status; - do { - rc = tiledb_query_get_status(ctx_, query, &status); - CHECK(rc == TILEDB_OK); - } while (status != TILEDB_COMPLETED); - - // Finalize query - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); - // Check correct execution of callback - CHECK(callback_made == 1); - } - }; - proc_query(); - - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); -} - -void AsyncFx::write_sparse_async_cancelled() { - // Prepare cell buffers - int buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; - uint64_t buffer_a2[] = {0, 1, 3, 6, 10, 11, 13, 16}; - char buffer_var_a2[] = "abbcccddddeffggghhhh"; - float buffer_a3[] = { - 0.1f, - 0.2f, - 1.1f, - 1.2f, - 2.1f, - 2.2f, - 3.1f, - 3.2f, - 4.1f, - 4.2f, - 5.1f, - 5.2f, - 6.1f, - 6.2f, - 7.1f, - 7.2f}; - uint64_t buffer_coords_dim1[] = {1, 1, 1, 2, 3, 4, 3, 3}; - uint64_t buffer_coords_dim2[] = {1, 2, 4, 3, 1, 2, 3, 4}; - void* buffers[] = { - buffer_a1, - buffer_a2, - buffer_var_a2, - buffer_a3, - buffer_coords_dim1, - buffer_coords_dim2}; - uint64_t buffer_sizes[] = { - sizeof(buffer_a1), - sizeof(buffer_a2), - sizeof(buffer_var_a2) - 1, // No need to store the last '\0' character - sizeof(buffer_a3), - sizeof(buffer_coords_dim1)}; - - // Open array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_WRITE); - CHECK(rc == TILEDB_OK); - - // Create query - tiledb_query_t* query; - const char* attributes[] = {"a1", "a2", "a3", "d1", "d2"}; - rc = tiledb_query_alloc(ctx_, array, TILEDB_WRITE, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_UNORDERED); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[0], buffers[0], &buffer_sizes[0]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[1], buffers[2], &buffer_sizes[2]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx_, query, attributes[1], (uint64_t*)buffers[1], &buffer_sizes[1]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[2], buffers[3], &buffer_sizes[3]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[3], buffers[4], &buffer_sizes[4]); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, attributes[4], buffers[5], &buffer_sizes[4]); - CHECK(rc == TILEDB_OK); - - auto proc_query = [&]() -> void { - // Submit query asynchronously - int callback_made = 0; - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - - tiledb_query_status_t status = TILEDB_FAILED; - if (rc == TILEDB_OK) { - // Cancel it immediately, which sometimes in this test is fast enough to - // cancel it and sometimes not. - rc = tiledb_ctx_cancel_tasks(ctx_); - CHECK(rc == TILEDB_OK); - - // Check query status - do { - rc = tiledb_query_get_status(ctx_, query, &status); - CHECK(rc == TILEDB_OK); - } while (status != TILEDB_COMPLETED && status != TILEDB_FAILED); - CHECK((status == TILEDB_COMPLETED || status == TILEDB_FAILED)); - - // If the query completed, check the callback was made. - CHECK(callback_made == (status == TILEDB_COMPLETED ? 1 : 0)); - } - - // If it failed, run it again. - if (status == TILEDB_FAILED) { - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - if (rc == TILEDB_OK) { - do { - rc = tiledb_query_get_status(ctx_, query, &status); - CHECK(rc == TILEDB_OK); - } while (status != TILEDB_COMPLETED && status != TILEDB_FAILED); - } - } - - CHECK(status == TILEDB_COMPLETED); - CHECK(callback_made == 1); - - // Finalize query whether successfully submitted or not - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); - }; - - proc_query(); - - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); -} - -void AsyncFx::read_dense_async() { - // Open array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, DENSE_ARRAY_NAME, &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_READ); - CHECK(rc == TILEDB_OK); - - // Calculate maximum buffer sizes for each attribute - uint64_t buffer_a1_size = 64; - uint64_t buffer_a2_off_size = 128; - uint64_t buffer_a2_val_size = 56; - uint64_t buffer_a3_size = 128; - uint64_t subarray[] = {1, 4, 1, 4}; - - // Prepare cell buffers - auto buffer_a1 = (int*)malloc(buffer_a1_size); - auto buffer_a2_off = (uint64_t*)malloc(buffer_a2_off_size); - auto buffer_a2_val = (char*)malloc(buffer_a2_val_size); - auto buffer_a3 = (float*)malloc(buffer_a3_size); - - // Create query - tiledb_query_t* query; - rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_subarray(ctx_, query, subarray); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a1", buffer_a1, &buffer_a1_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a2", buffer_a2_val, &buffer_a2_val_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx_, query, "a2", buffer_a2_off, &buffer_a2_off_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a3", buffer_a3, &buffer_a3_size); - CHECK(rc == TILEDB_OK); - - auto proc_query = [&]() -> void { - // Submit query with callback - int callback_made = 0; - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - - if (rc == TILEDB_OK) { - // Wait for the query to complete - tiledb_query_status_t status; - do { - tiledb_query_get_status(ctx_, query, &status); - } while (status != TILEDB_COMPLETED); - - // Finalize query - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); - - // Check correct execution of callback - CHECK(callback_made == 1); - } - - // Correct buffers - int c_buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - uint64_t c_buffer_a2_off[] = { - 0, 1, 3, 6, 10, 11, 13, 16, 20, 21, 23, 26, 30, 31, 33, 36}; - char c_buffer_a2_val[] = - "abbcccdddd" - "effggghhhh" - "ijjkkkllll" - "mnnooopppp"; - float c_buffer_a3[] = { - 0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, - 4.1f, 4.2f, 5.1f, 5.2f, 6.1f, 6.2f, 7.1f, 7.2f, - 8.1f, 8.2f, 9.1f, 9.2f, 10.1f, 10.2f, 11.1f, 11.2f, - 12.1f, 12.2f, 13.1f, 13.2f, 14.1f, 14.2f, 15.1f, 15.2f, - }; - - // Check buffers - CHECK(!memcmp(buffer_a1, c_buffer_a1, sizeof(c_buffer_a1))); - CHECK(!memcmp(buffer_a2_off, c_buffer_a2_off, sizeof(c_buffer_a2_off))); - CHECK(!memcmp(buffer_a2_val, c_buffer_a2_val, sizeof(c_buffer_a2_val) - 1)); - CHECK(!memcmp(buffer_a3, c_buffer_a3, sizeof(c_buffer_a3))); - }; - if (!use_external_subarray_) { - proc_query(); - } else { - tiledb_subarray_t* query_subarray; - tiledb_query_get_subarray_t(ctx_, query, &query_subarray); - rc = tiledb_query_set_subarray_t(ctx_, query, query_subarray); - CHECK(rc == TILEDB_OK); - - proc_query(); - - tiledb_subarray_free(&query_subarray); - } - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); - free(buffer_a1); - free(buffer_a2_off); - free(buffer_a2_val); - free(buffer_a3); -} - -void AsyncFx::read_sparse_async() { - // Open array - tiledb_array_t* array; - int rc = tiledb_array_alloc(ctx_, SPARSE_ARRAY_NAME, &array); - CHECK(rc == TILEDB_OK); - rc = tiledb_array_open(ctx_, array, TILEDB_READ); - CHECK(rc == TILEDB_OK); - - // Calculate maximum buffer sizes for each attribute - uint64_t buffer_a1_size = 32; - uint64_t buffer_a2_off_size = 64; - uint64_t buffer_a2_val_size = 20; - uint64_t buffer_a3_size = 64; - uint64_t buffer_coords_dim1_size = 64; - uint64_t buffer_coords_dim2_size = 64; - - // Prepare cell buffers - auto buffer_a1 = (int*)malloc(buffer_a1_size); - auto buffer_a2_off = (uint64_t*)malloc(buffer_a2_off_size); - auto buffer_a2_val = (char*)malloc(buffer_a2_val_size); - auto buffer_a3 = (float*)malloc(buffer_a3_size); - auto buffer_coords_dim1 = (uint64_t*)malloc(buffer_coords_dim1_size); - auto buffer_coords_dim2 = (uint64_t*)malloc(buffer_coords_dim2_size); - - // Create query - tiledb_query_t* query; - rc = tiledb_query_alloc(ctx_, array, TILEDB_READ, &query); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_layout(ctx_, query, TILEDB_GLOBAL_ORDER); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a1", buffer_a1, &buffer_a1_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a2", buffer_a2_val, &buffer_a2_val_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_offsets_buffer( - ctx_, query, "a2", buffer_a2_off, &buffer_a2_off_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "a3", buffer_a3, &buffer_a3_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "d1", buffer_coords_dim1, &buffer_coords_dim1_size); - CHECK(rc == TILEDB_OK); - rc = tiledb_query_set_data_buffer( - ctx_, query, "d2", buffer_coords_dim2, &buffer_coords_dim2_size); - CHECK(rc == TILEDB_OK); - - auto proc_query = [&]() -> void { - // Submit query with callback - int callback_made = 0; - rc = tiledb_query_submit_async(ctx_, query, callback, &callback_made); - CHECK(rc == TILEDB_OK); - - // Wait for the query to complete - tiledb_query_status_t status; - do { - tiledb_query_get_status(ctx_, query, &status); - } while (status != TILEDB_COMPLETED); - - // Finalize query - rc = tiledb_query_finalize(ctx_, query); - CHECK(rc == TILEDB_OK); - - // Check correct execution of callback - CHECK(callback_made == 1); - - // Correct buffers - int c_buffer_a1[] = {0, 1, 2, 3, 4, 5, 6, 7}; - uint64_t c_buffer_a2_off[] = {0, 1, 3, 6, 10, 11, 13, 16}; - char c_buffer_a2_val[] = "abbcccddddeffggghhhh"; - float c_buffer_a3[] = { - 0.1f, - 0.2f, - 1.1f, - 1.2f, - 2.1f, - 2.2f, - 3.1f, - 3.2f, - 4.1f, - 4.2f, - 5.1f, - 5.2f, - 6.1f, - 6.2f, - 7.1f, - 7.2f}; - uint64_t c_buffer_coords_dim1[] = {1, 1, 1, 2, 3, 4, 3, 3}; - uint64_t c_buffer_coords_dim2[] = {1, 2, 4, 3, 1, 2, 3, 4}; - - // Check buffers - CHECK(!memcmp(buffer_a1, c_buffer_a1, sizeof(c_buffer_a1))); - CHECK(!memcmp(buffer_a2_off, c_buffer_a2_off, sizeof(c_buffer_a2_off))); - CHECK(!memcmp(buffer_a2_val, c_buffer_a2_val, sizeof(c_buffer_a2_val) - 1)); - CHECK(!memcmp(buffer_a3, c_buffer_a3, sizeof(c_buffer_a3))); - CHECK(!memcmp( - buffer_coords_dim1, - c_buffer_coords_dim1, - sizeof(c_buffer_coords_dim1))); - CHECK(!memcmp( - buffer_coords_dim2, - c_buffer_coords_dim2, - sizeof(c_buffer_coords_dim2))); - }; - - if (!use_external_subarray_) { - proc_query(); - } else { - tiledb_subarray_t* query_subarray; - tiledb_query_get_subarray_t(ctx_, query, &query_subarray); - rc = tiledb_query_set_subarray_t(ctx_, query, query_subarray); - CHECK(rc == TILEDB_OK); - - proc_query(); - - tiledb_subarray_free(&query_subarray); - } - - // Close array - rc = tiledb_array_close(ctx_, array); - CHECK(rc == TILEDB_OK); - - // Clean up - tiledb_array_free(&array); - tiledb_query_free(&query); - free(buffer_a1); - free(buffer_a2_off); - free(buffer_a2_val); - free(buffer_a3); - free(buffer_coords_dim1); - free(buffer_coords_dim2); -} - -void AsyncFx::remove_array(const std::string& array_name) { - if (!is_array(array_name)) - return; - - CHECK(tiledb_object_remove(ctx_, array_name.c_str()) == TILEDB_OK); -} - -void AsyncFx::remove_dense_array() { - remove_array(DENSE_ARRAY_NAME); -} - -void AsyncFx::remove_sparse_array() { - remove_array(SPARSE_ARRAY_NAME); -} - -bool AsyncFx::is_array(const std::string& array_name) { - tiledb_object_t type = TILEDB_INVALID; - REQUIRE(tiledb_object_type(ctx_, array_name.c_str(), &type) == TILEDB_OK); - return type == TILEDB_ARRAY; -} - -TEST_CASE_METHOD( - AsyncFx, "C API: Test dense async", "[capi][async][dense-async]") { - SECTION("- No outside subarray") { - use_external_subarray_ = false; - } - SECTION("- outside subarray") { - use_external_subarray_ = true; - } - remove_dense_array(); - create_dense_array(); - write_dense_async(); - read_dense_async(); - remove_dense_array(); -} - -TEST_CASE_METHOD( - AsyncFx, "C API: Test sparse async", "[capi][async][sparse-async]") { - SECTION("- No outside subarray") { - use_external_subarray_ = false; - } - SECTION("- outside subarray") { - use_external_subarray_ = true; - } - remove_sparse_array(); - create_sparse_array(); - write_sparse_async(); - read_sparse_async(); - remove_sparse_array(); -} - -TEST_CASE_METHOD( - AsyncFx, "C API: Test async cancellation", "[capi][async][cancel]") { - SECTION("- No outside subarray") { - use_external_subarray_ = false; - } - SECTION("- outside subarray") { - use_external_subarray_ = true; - } - remove_sparse_array(); - create_sparse_array(); - write_sparse_async_cancelled(); - read_sparse_async(); - remove_sparse_array(); -} diff --git a/test/src/unit-capi-dense_array.cc b/test/src/unit-capi-dense_array.cc index caba13b314bc..a7a60b015298 100644 --- a/test/src/unit-capi-dense_array.cc +++ b/test/src/unit-capi-dense_array.cc @@ -58,6 +58,7 @@ #include #include +#include #include #include #include @@ -65,6 +66,7 @@ #include #include #include +using namespace std::chrono_literals; using namespace tiledb::test; using namespace tiledb::sm; @@ -889,8 +891,10 @@ void DenseArrayFx::write_dense_subarray_2D_with_cancel( REQUIRE(rc == TILEDB_OK); auto proc_query = [&](unsigned i) -> void { - rc = tiledb_query_submit_async(ctx_, query, NULL, NULL); - REQUIRE(rc == TILEDB_OK); + std::future submit_async = std::async( + std::launch::async, + [this, &query] { tiledb_query_submit(ctx_, query); }); + submit_async.wait_for(1ms); // Cancel it immediately. if (i < num_writes - 1) { rc = tiledb_ctx_cancel_tasks(ctx_); @@ -906,8 +910,12 @@ void DenseArrayFx::write_dense_subarray_2D_with_cancel( // If it failed, run it again. if (status == TILEDB_FAILED) { - rc = tiledb_query_submit_async(ctx_, query, NULL, NULL); - CHECK(rc == TILEDB_OK); + std::future submit_async = + std::async(std::launch::async, [this, &query] { + auto rc = tiledb_query_submit(ctx_, query); + REQUIRE(rc == TILEDB_OK); + }); + submit_async.wait_for(1ms); do { rc = tiledb_query_get_status(ctx_, query, &status); CHECK(rc == TILEDB_OK); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index 60c82690a217..dbe7494a486e 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -2023,56 +2023,6 @@ TEST_CASE( vfs.remove_dir(array_read.uri() + "/__schema"); } -TEST_CASE( - "C++ API: Close array with running query", - "[cppapi][close-before-read][non-rest]") { - // async queries not supported on remote arrays (REST) - tiledb::test::VFSTestSetup vfs_test_setup{}; - Context ctx{vfs_test_setup.ctx()}; - auto array_uri{vfs_test_setup.array_uri("cpp_unit_array")}; - - // Create - Domain domain(ctx); - domain.add_dimension(Dimension::create(ctx, "rows", {{0, 3}}, 4)) - .add_dimension(Dimension::create(ctx, "cols", {{0, 3}}, 4)); - ArraySchema schema(ctx, TILEDB_DENSE); - schema.set_domain(domain).set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}}); - schema.add_attribute(Attribute::create(ctx, "a")); - Array::create(array_uri, schema); - - // Write - std::vector data_w = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; - Array array_w(ctx, array_uri, TILEDB_WRITE); - Query query_w(ctx, array_w); - query_w.set_subarray(Subarray(ctx, array_w).set_subarray({0, 3, 0, 3})) - .set_layout(TILEDB_ROW_MAJOR) - .set_data_buffer("a", data_w); - query_w.submit(); - array_w.close(); - - // Open for read. - Array array(ctx, array_uri, TILEDB_READ); - std::vector subarray_read = {0, 3, 0, 3}; - std::vector a_read(16); - - Query query(ctx, array); - Subarray sub(ctx, array); - sub.set_subarray(subarray_read); - query.set_subarray(sub) - .set_layout(TILEDB_ROW_MAJOR) - .set_data_buffer("a", a_read); - query.submit_async(); - array.close(); - - uint64_t i = 0; - while (query.query_status() != Query::Status::COMPLETE) { - i++; - } - - CHECK(i > 0); -} - TEST_CASE("C++ API: Read empty array", "[cppapi][read-empty-array]") { const std::string array_name_1d = "cpp_unit_array_1d"; Context ctx; diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 6aada84cda8b..1e88faaec512 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -40,6 +40,7 @@ #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/group.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/enums/encryption_type.h" #ifdef _WIN32 #include "tiledb/sm/filesystem/win.h" @@ -67,7 +68,8 @@ struct DeletesFx { const std::string group_name_; std::string key_ = "0123456789abcdeF0123456789abcdeF"; - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; + std::string enc_type_str_ = + encryption_type_str((tiledb::sm::EncryptionType)TILEDB_AES_256_GCM); // Constructors/destructors. DeletesFx(); @@ -133,6 +135,8 @@ void DeletesFx::set_purge_deleted_cells() { config.set("sm.consolidation.purge_deleted_cells", "true"); vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); + vfs_ = VFS(ctx_); + sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::set_legacy() { @@ -142,6 +146,8 @@ void DeletesFx::set_legacy() { config.set("sm.query.sparse_unordered_with_dups.reader", "legacy"); vfs_test_setup_.update_config(config.ptr().get()); ctx_ = vfs_test_setup_.ctx(); + vfs_ = VFS(ctx_); + sm_ = ctx_.ptr().get()->storage_manager(); } void DeletesFx::create_dir(const std::string& path) { @@ -168,6 +174,17 @@ void DeletesFx::create_simple_array(const std::string& path) { } void DeletesFx::create_sparse_array(bool allows_dups, bool encrypt) { + Config config; + if (encrypt) { + config["sm.encryption_type"] = enc_type_str_.c_str(); + config["sm.encryption_key"] = key_; + } + + vfs_test_setup_.update_config(config.ptr().get()); + ctx_ = vfs_test_setup_.ctx(); + vfs_ = VFS(ctx_); + sm_ = ctx_.ptr().get()->storage_manager(); + // Create dimensions. auto d1 = Dimension::create(ctx_, "d1", {{1, 4}}, 2); auto d2 = Dimension::create(ctx_, "d2", {{1, 4}}, 2); @@ -196,11 +213,7 @@ void DeletesFx::create_sparse_array(bool allows_dups, bool encrypt) { filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(array_name_.c_str(), schema, enc_type_, key_); - } else { - Array::create(array_name_.c_str(), schema); - } + Array::create(array_name_.c_str(), schema); } void DeletesFx::write_sparse( diff --git a/test/src/unit-cppapi-group.cc b/test/src/unit-cppapi-group.cc index 44b35e654c9f..662f91515110 100644 --- a/test/src/unit-cppapi-group.cc +++ b/test/src/unit-cppapi-group.cc @@ -71,11 +71,6 @@ struct GroupCPPFx { */ bool serialize_ = false; - const char* key_ = "0123456789abcdeF0123456789abcdeF"; - const uint32_t key_len_ = - (uint32_t)strlen("0123456789abcdeF0123456789abcdeF"); - const tiledb_encryption_type_t enc_type_ = TILEDB_AES_256_GCM; - // Functions GroupCPPFx(); ~GroupCPPFx(); diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index 48c1ad268432..897eb91eb11f 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -1,5 +1,5 @@ /** - * @file unit-cppapi-updates-queries.cc + * @file unit-cppapi-update-queries.cc * * @section LICENSE * @@ -38,6 +38,7 @@ #include "tiledb/sm/c_api/tiledb_struct_def.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/query/update_value.h" @@ -91,21 +92,30 @@ UpdatesFx::~UpdatesFx() { } void UpdatesFx::create_sparse_array(bool allows_dups, bool encrypt) { + Config cfg; + if (encrypt) { + std::string enc_type_str = + encryption_type_str((tiledb::sm::EncryptionType)enc_type_); + cfg["sm.encryption_type"] = enc_type_str.c_str(); + cfg["sm.encryption_key"] = key_; + } + Context ctx(cfg); + // Create dimensions. - auto d1 = Dimension::create(ctx_, "d1", {{1, 4}}, 2); - auto d2 = Dimension::create(ctx_, "d2", {{1, 4}}, 2); + auto d1 = Dimension::create(ctx, "d1", {{1, 4}}, 2); + auto d2 = Dimension::create(ctx, "d2", {{1, 4}}, 2); // Create domain. - Domain domain(ctx_); + Domain domain(ctx); domain.add_dimension(d1); domain.add_dimension(d2); // Create attributes. - auto a1 = Attribute::create(ctx_, "a1"); - auto a2 = Attribute::create(ctx_, "a2"); + auto a1 = Attribute::create(ctx, "a1"); + auto a2 = Attribute::create(ctx, "a2"); // Create array schema. - ArraySchema schema(ctx_, TILEDB_SPARSE); + ArraySchema schema(ctx, TILEDB_SPARSE); schema.set_domain(domain); schema.set_capacity(20); schema.add_attributes(a1); @@ -116,16 +126,12 @@ void UpdatesFx::create_sparse_array(bool allows_dups, bool encrypt) { } // Set up filters. - Filter filter(ctx_, TILEDB_FILTER_NONE); - FilterList filter_list(ctx_); + Filter filter(ctx, TILEDB_FILTER_NONE); + FilterList filter_list(ctx); filter_list.add_filter(filter); schema.set_coords_filter_list(filter_list); - if (encrypt) { - Array::create(SPARSE_ARRAY_NAME, schema, enc_type_, key_); - } else { - Array::create(SPARSE_ARRAY_NAME, schema); - } + Array::create(SPARSE_ARRAY_NAME, schema); } void UpdatesFx::write_update_condition( From 51e6d3d021a19b5348d619645123cfe33a35472d Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 21 May 2024 21:11:16 +0200 Subject: [PATCH 363/456] Fix the deletes test taking too long. (#4996) The change in #4957 caused us to create a full consolidation workspace on some deletes tests which in debug mode creates a few GB of memory to be initialized. This takes multiple seconds. [sc-47836] --- TYPE: NO_HISTORY DESC: Fix the deletes test taking too long. --- test/src/unit-cppapi-deletes.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index 1e88faaec512..002ee1a53dd3 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -175,6 +175,7 @@ void DeletesFx::create_simple_array(const std::string& path) { void DeletesFx::create_sparse_array(bool allows_dups, bool encrypt) { Config config; + config.set("sm.consolidation.buffer_size", "1000"); if (encrypt) { config["sm.encryption_type"] = enc_type_str_.c_str(); config["sm.encryption_key"] = key_; From 1d0215692253adb4bcf9bbd8873eeae90b0d1937 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 22 May 2024 09:17:27 +0300 Subject: [PATCH 364/456] Fix `DetectStdPmr` for cross-compiling scenarios. (#4995) [SC-47807](https://app.shortcut.com/tiledb-inc/story/47807/detectstdpmr-fails-in-macos-when-cross-compiling) We use `try_compile` instead of `try_run` to detect whether the headers for `std::pmr` are available. This fixes errors in cross-compiling scenarios, like when building for osx-arm64 in an osx-x64 machine. Should fix failures in conda-forge/TileDB-feedstock#290, waiting for adding a patch and validating. --- TYPE: BUILD DESC: Fix configuration errors when cross-compiling. --- cmake/Modules/DetectStdPmr.cmake | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/cmake/Modules/DetectStdPmr.cmake b/cmake/Modules/DetectStdPmr.cmake index 3f96e9435490..37a6edc50f70 100644 --- a/cmake/Modules/DetectStdPmr.cmake +++ b/cmake/Modules/DetectStdPmr.cmake @@ -31,8 +31,7 @@ # binary dies with a dyld missing symbol error. if (ENV{MACOSX_DEPLOYMENT_TARGET}) - string(COMPARE LESS "$ENV{MACOSX_DEPLOYMENT_TARGET}" "14" MACOS_BAD_PMR_SUPPORT) - if (MACOS_BAD_PMR_SUPPORT) + if (ENV{MACOSX_DEPLOYMENT_TARGET} STRLESS "14") set(TILEDB_USE_CPP17_PMR ON) message(STATUS "Using vendored cpp17::pmr for polymorphic allocators") return() @@ -41,27 +40,22 @@ endif() # Otherwise, if we're not building a targeted macOS version, we just detect # whether std::pmr is available. -# -# However CMake makes this extra awesome because try_run appears to have -# changed in a backwards compatible manner. We'll just version check for -# selecting which to run. if (CMAKE_VERSION VERSION_LESS "3.25") - try_run( - TILEDB_CAN_RUN_STD_PMR - TILEDB_CAN_COMPILE_STD_PMR - "${CMAKE_CURRENT_BINARY_DIR}" - "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" - ) -else() - try_run( - TILEDB_CAN_RUN_STD_PMR - TILEDB_CAN_COMPILE_STD_PMR - SOURCES "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" +try_compile( + TILEDB_CAN_COMPILE_STD_PMR + "${CMAKE_CURRENT_BINARY_DIR}" + "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" ) +else() +# CMake 3.25+ has a better signature for try_compile. +try_compile( + TILEDB_CAN_COMPILE_STD_PMR + SOURCES "${CMAKE_SOURCE_DIR}/cmake/inputs/detect_std_pmr.cc" +) endif() -if ("${TILEDB_CAN_COMPILE_STD_PMR}" AND "${TILEDB_CAN_RUN_STD_PMR}" EQUAL 0) +if (TILEDB_CAN_COMPILE_STD_PMR) message(STATUS "Using std::pmr for polymorphic allocators") else() set(TILEDB_USE_CPP17_PMR ON) From 9b26fd97438264118be560ab7fe11c1598cd05d3 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 22 May 2024 09:41:31 +0300 Subject: [PATCH 365/456] Support `ls_recursive` in Azure. (#4981) [SC-44943](https://app.shortcut.com/tiledb-inc/story/44943/add-support-for-ls-recursive-in-azure) This PR implements `ls_recursive` in the Azure VFS. The design is based on the S3 implementation, but unlike S3, the Azure SDK calls are cleanly separated, which prevents the Azure SDK headers from leaking to the VFS header. --- TYPE: C_API DESC: Support VFS `ls_recursive` API for Azure filesystem. --- test/src/unit-cppapi-vfs.cc | 6 +- test/src/unit-vfs.cc | 13 +- test/support/src/vfs_helpers.h | 16 + .../c_api/vfs/test/unit_capi_ls_recursive.cc | 28 +- tiledb/api/c_api/vfs/vfs_api_experimental.h | 4 +- tiledb/sm/cpp_api/vfs_experimental.h | 3 +- tiledb/sm/filesystem/azure.cc | 83 ++++- tiledb/sm/filesystem/azure.h | 305 ++++++++++++++++-- tiledb/sm/filesystem/vfs.h | 40 ++- 9 files changed, 445 insertions(+), 53 deletions(-) diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index 28a803ce628d..d7a2cf2368a4 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -507,8 +507,10 @@ TEST_CASE( } } -using ls_recursive_test_types = - std::tuple; +using ls_recursive_test_types = std::tuple< + tiledb::test::LocalFsTest, + tiledb::test::S3Test, + tiledb::test::AzureTest>; TEMPLATE_LIST_TEST_CASE( "CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]", diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 7ba6cc422070..23a6aa7afff3 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -622,8 +622,9 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { require_tiledb_ok(vfs_ls.remove_dir(URI(path))); } -// Currently only S3 is supported for VFS::ls_recursive. -using TestBackends = std::tuple; +// Currently only local, S3 and Azure is supported for VFS::ls_recursive. +// TODO: LocalFsTest currently fails. Fix and re-enable. +using TestBackends = std::tuple; TEMPLATE_LIST_TEST_CASE( "VFS: Test internal ls_filtered recursion argument", "[vfs][ls_filtered][recursion]", @@ -637,10 +638,9 @@ TEMPLATE_LIST_TEST_CASE( DYNAMIC_SECTION( fs.temp_dir_.backend_name() << " ls_filtered with recursion: " << (recursive ? "true" : "false")) { -#ifdef HAVE_S3 // If testing with recursion use the root directory, otherwise use a subdir. auto path = recursive ? fs.temp_dir_ : fs.temp_dir_.join_path("subdir_1"); - auto ls_objects = fs.get_s3().ls_filtered( + auto ls_objects = fs.vfs_.ls_filtered( path, VFSTestBase::accept_all_files, accept_all_dirs, recursive); auto expected = fs.expected_results(); @@ -654,7 +654,6 @@ TEMPLATE_LIST_TEST_CASE( CHECK(ls_objects.size() == expected.size()); CHECK(ls_objects == expected); -#endif } } @@ -669,7 +668,7 @@ TEST_CASE( } std::string backend = vfs_test.temp_dir_.backend_name(); - if (vfs_test.temp_dir_.is_s3()) { + if (vfs_test.temp_dir_.is_s3() || vfs_test.temp_dir_.is_azure()) { DYNAMIC_SECTION(backend << " supported backend should not throw") { CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( vfs_test.temp_dir_, VFSTestBase::accept_all_files)); @@ -688,7 +687,7 @@ TEST_CASE( TEST_CASE( "VFS: Throwing FileFilter for ls_recursive", "[vfs][ls_recursive][file-filter]") { - std::string prefix = "s3://"; + std::string prefix = GENERATE("s3://", "azure://"); VFSTest vfs_test({0}, prefix); if (!vfs_test.is_supported()) { return; diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index 16e0c18cdee7..d1f3d9a7ac1c 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -1107,6 +1107,22 @@ class AzureTest : public VFSTestBase { public: explicit AzureTest(const std::vector& test_tree) : VFSTestBase(test_tree, "azure://") { +#ifdef HAVE_AZURE + vfs_.create_bucket(temp_dir_).ok(); + for (size_t i = 1; i <= test_tree_.size(); i++) { + sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); + // VFS::create_dir is a no-op for Azure; Just create objects. + for (size_t j = 1; j <= test_tree_[i - 1]; j++) { + auto object_uri = path.join_path("test_file_" + std::to_string(j)); + vfs_.touch(object_uri).ok(); + std::string data(j * 10, 'a'); + vfs_.write(object_uri, data.data(), data.size()).ok(); + vfs_.close_file(object_uri).ok(); + expected_results().emplace_back(object_uri.to_string(), data.size()); + } + } + std::sort(expected_results().begin(), expected_results().end()); +#endif } }; diff --git a/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc index 5792fe82fd67..453c029e2e21 100644 --- a/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc +++ b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc @@ -38,13 +38,18 @@ using namespace tiledb::test; -TEST_CASE("C API: ls_recursive callback", "[vfs][ls-recursive]") { +// Currently only local, S3 and Azure is supported for VFS::ls_recursive. +// TODO: LocalFsTest currently fails. Fix and re-enable. +using TestBackends = std::tuple; + +TEMPLATE_LIST_TEST_CASE( + "C API: ls_recursive callback", "[vfs][ls-recursive]", TestBackends) { using tiledb::sm::LsObjects; - S3Test s3_test({10, 50}); - if (!s3_test.is_supported()) { + TestType test({10, 50}); + if (!test.is_supported()) { return; } - auto expected = s3_test.expected_results(); + auto expected = test.expected_results(); vfs_config vfs_config; tiledb_ctx_t* ctx; @@ -80,19 +85,22 @@ TEST_CASE("C API: ls_recursive callback", "[vfs][ls-recursive]") { } CHECK( - tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) == + tiledb_vfs_ls_recursive(ctx, vfs, test.temp_dir_.c_str(), cb, &data) == TILEDB_OK); CHECK(data.size() == expected.size()); CHECK(data == expected); } -TEST_CASE("C API: ls_recursive throwing callback", "[vfs][ls-recursive]") { +TEMPLATE_LIST_TEST_CASE( + "C API: ls_recursive throwing callback", + "[vfs][ls-recursive]", + TestBackends) { using tiledb::sm::LsObjects; - S3Test s3_test({10, 50}); - if (!s3_test.is_supported()) { + TestType test({10, 50}); + if (!test.is_supported()) { return; } - auto expected = s3_test.expected_results(); + auto expected = test.expected_results(); vfs_config vfs_config; tiledb_ctx_t* ctx; @@ -107,7 +115,7 @@ TEST_CASE("C API: ls_recursive throwing callback", "[vfs][ls-recursive]") { }; CHECK( - tiledb_vfs_ls_recursive(ctx, vfs, s3_test.temp_dir_.c_str(), cb, &data) == + tiledb_vfs_ls_recursive(ctx, vfs, test.temp_dir_.c_str(), cb, &data) == TILEDB_ERR); CHECK(data.empty()); } diff --git a/tiledb/api/c_api/vfs/vfs_api_experimental.h b/tiledb/api/c_api/vfs/vfs_api_experimental.h index dba16811ae40..cac4966bd6b7 100644 --- a/tiledb/api/c_api/vfs/vfs_api_experimental.h +++ b/tiledb/api/c_api/vfs/vfs_api_experimental.h @@ -58,8 +58,8 @@ typedef int32_t (*tiledb_ls_callback_t)( * on error. The callback is responsible for writing gathered entries into the * `data` buffer, for example using a pointer to a user-defined struct. * - * Currently only Posix and S3 are supported, and the `path` must be a valid URI - * for one of those filesystems. + * Currently only local filesystem, S3 and Azure are supported, and the `path` + * must be a valid URI for one of those filesystems. * * **Example:** * diff --git a/tiledb/sm/cpp_api/vfs_experimental.h b/tiledb/sm/cpp_api/vfs_experimental.h index 7f0506cd0aa6..85152b4d6b2e 100644 --- a/tiledb/sm/cpp_api/vfs_experimental.h +++ b/tiledb/sm/cpp_api/vfs_experimental.h @@ -161,7 +161,8 @@ class VFSExperimental { * be included in the results and false otherwise. If no inclusion predicate * is provided, all results are returned. * - * Currently only S3 is supported, and the `path` must be a valid S3 URI. + * Currently only local filesystem, S3 and Azure is supported, and the `path` + * must be a valid URI for one of those filesystems. * * @code{.c} * VFSExperimental::LsInclude predicate = [](std::string_view path, diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 78280251897f..6ffbbffa926e 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -57,6 +57,18 @@ using tiledb::common::filesystem::directory_entry; namespace tiledb { namespace sm { +/** Converts an Azure nullable value to an STL optional. */ +template +std::optional from_azure_nullable(const ::Azure::Nullable& value) { + return value ? std::optional(*value) : nullopt; +} + +/** Converts an STL optional value to an Azure nullable. */ +template +::Azure::Nullable to_azure_nullable(const std::optional& value) { + return value ? *value : ::Azure::Nullable{}; +} + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -542,7 +554,7 @@ Status Azure::is_blob( return Status_Ok(); } -std::string Azure::remove_front_slash(const std::string& path) const { +std::string Azure::remove_front_slash(const std::string& path) { if (path.front() == '/') { return path.substr(1, path.length()); } @@ -550,7 +562,7 @@ std::string Azure::remove_front_slash(const std::string& path) const { return path; } -std::string Azure::add_trailing_slash(const std::string& path) const { +std::string Azure::add_trailing_slash(const std::string& path) { if (path.back() != '/') { return path + "/"; } @@ -558,7 +570,7 @@ std::string Azure::add_trailing_slash(const std::string& path) const { return path; } -std::string Azure::remove_trailing_slash(const std::string& path) const { +std::string Azure::remove_trailing_slash(const std::string& path) { if (path.back() == '/') { return path.substr(0, path.length() - 1); } @@ -1087,6 +1099,69 @@ Status Azure::write_blocks( return Status::Ok(); } +LsObjects Azure::list_blobs_impl( + const std::string& container_name, + const std::string& blob_path, + bool recursive, + int max_keys, + optional& continuation_token) const { + try { + ::Azure::Storage::Blobs::ListBlobsOptions options{ + .Prefix = blob_path, + .ContinuationToken = to_azure_nullable(continuation_token), + .PageSizeHint = max_keys}; + auto container_client = client().GetBlobContainerClient(container_name); + auto to_directory_entry = + [&container_name](const ::Azure::Storage::Blobs::Models::BlobItem& item) + -> LsObjects::value_type { + return { + "azure://" + container_name + "/" + + remove_front_slash(remove_trailing_slash(item.Name)), + item.BlobSize >= 0 ? static_cast(item.BlobSize) : 0}; + }; + + LsObjects result; + if (recursive) { + auto response = container_client.ListBlobs(options); + + continuation_token = from_azure_nullable(response.NextPageToken); + + result.reserve(response.Blobs.size()); + std::transform( + response.Blobs.begin(), + response.Blobs.end(), + std::back_inserter(result), + to_directory_entry); + } else { + auto response = container_client.ListBlobsByHierarchy("/", options); + + continuation_token = from_azure_nullable(response.NextPageToken); + + result.reserve(response.Blobs.size() + response.BlobPrefixes.size()); + std::transform( + response.Blobs.begin(), + response.Blobs.end(), + std::back_inserter(result), + to_directory_entry); + std::transform( + response.BlobPrefixes.begin(), + response.BlobPrefixes.end(), + std::back_inserter(result), + [&container_name](std::string& name) -> LsObjects::value_type { + return { + "azure://" + container_name + "/" + + remove_front_slash(add_trailing_slash(name)), + 0}; + }); + } + + return result; + } catch (const ::Azure::Storage::StorageException& e) { + throw AzureException( + "List blobs failed on: " + blob_path + "; " + e.Message); + } +} + Status Azure::upload_block( const std::string& container_name, const std::string& blob_path, @@ -1111,7 +1186,7 @@ Status Azure::upload_block( Status Azure::parse_azure_uri( const URI& uri, std::string* const container_name, - std::string* const blob_path) const { + std::string* const blob_path) { assert(uri.is_azure()); const std::string uri_str = uri.to_string(); diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index 489ac581e4a2..985d6d290c5f 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -34,7 +34,9 @@ #define TILEDB_AZURE_H #ifdef HAVE_AZURE +#include "ls_scanner.h" #include "tiledb/common/common.h" +#include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/status.h" #include "tiledb/common/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" @@ -64,6 +66,14 @@ class directory_entry; namespace sm { +/** Class for Azure status exceptions. */ +class AzureException : public StatusException { + public: + explicit AzureException(const std::string& msg) + : StatusException("Azure", msg) { + } +}; + /** * The Azure-specific configuration parameters. */ @@ -121,7 +131,139 @@ struct AzureParameters { const std::string& blob_endpoint); }; +class Azure; + +/** + * AzureScanner wraps the Azure ListBlobs request and provides an iterator for + * results. If we reach the end of the current batch of results and results are + * truncated, we fetch the next batch of results from Azure. + * + * For each batch of results collected by fetch_results(), the begin_ and end_ + * members are initialized to the first and last elements of the batch. The scan + * steps through each result in the range [begin_, end_), using next() to + * advance to the next object accepted by the filters for this scan. + * + * @section Known Defect + * The iterators of AzureScanner are initialized by the Azure ListBlobs + * results and there is no way to determine if they are different from + * iterators returned by a previous request. To be able to detect this, we + * can track the batch number and compare it to the batch number associated + * with the iterator returned by the previous request. Batch number can be + * tracked by the total number of times we submit a ListBlobs request + * within fetch_results(). + * + * @tparam F The FilePredicate type used to filter object results. + * @tparam D The DirectoryPredicate type used to prune prefix results. + */ +template +class AzureScanner : public LsScanner { + public: + /** Declare LsScanIterator as a friend class for access to call next(). */ + template + friend class LsScanIterator; + using Iterator = LsScanIterator, LsObjects::value_type>; + + /** Constructor. */ + AzureScanner( + const Azure& client, + const URI& prefix, + F file_filter, + D dir_filter = accept_all_dirs, + bool recursive = false, + int max_keys = 1000); + + /** + * Returns true if there are more results to fetch from Azure. + */ + [[nodiscard]] inline bool more_to_fetch() const { + return !has_fetched_ || continuation_token_.has_value(); + } + + /** + * @return Iterator to the beginning of the results being iterated on. + * Input iterators are single-pass, so we return a copy of this iterator at + * it's current position. + */ + Iterator begin() { + return Iterator(this); + } + + /** + * @return Default constructed iterator, which marks the end of results using + * nullptr. + */ + Iterator end() { + return Iterator(); + } + + private: + /** + * Advance to the next object accepted by the filters for this scan. + * + * @param ptr Reference to the current data pointer. + * @sa LsScanIterator::operator++() + */ + void next(typename Iterator::pointer& ptr); + + /** + * If the iterator is at the end of the current batch, this will fetch the + * next batch of results from Azure. This does not check if the results are + * accepted by the filters for this scan. + * + * @param ptr Reference to the current data iterator. + */ + void advance(typename Iterator::pointer& ptr) { + ptr++; + if (ptr == end_) { + if (more_to_fetch()) { + // Fetch results and reset the iterator. + ptr = fetch_results(); + } else { + // Set the pointer to nullptr to indicate the end of results. + end_ = ptr = typename Iterator::pointer(); + } + } + } + + /** + * Fetch the next batch of results from Azure. This also handles setting the + * continuation token for the next request, if the results were truncated. + * + * @return A pointer to the first result in the new batch. The return value + * is used to update the pointer managed by the iterator during traversal. + * If the request returned no results, this will return nullptr to mark the + * end of the scan. + * @sa LsScanIterator::operator++() + * @sa AzureScanner::next(typename Iterator::pointer&) + */ + typename Iterator::pointer fetch_results(); + + /** Reference to the Azure VFS. */ + const Azure& client_; + /** Name of container. */ + std::string container_name_; + /** Blob path. */ + std::string blob_path_; + /** Maximum number of items to request from Azure. */ + int max_keys_; + /** Iterators for the current objects fetched from Azure. */ + typename Iterator::pointer begin_, end_; + + /** Whether blobs have been fetched at least once. */ + bool has_fetched_; + /** + * Token to pass to Azure to continue iteration. + * + * If it is nullopt, and has_fetched_ is false, it means that th + */ + std::optional continuation_token_; + LsObjects blobs_; +}; + class Azure { + template + friend class AzureScanner; + public: /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -256,6 +398,55 @@ class Azure { const std::string& delimiter = "/", int max_paths = -1) const; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + */ + template + LsObjects ls_filtered( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false) const { + AzureScanner azure_scanner(*this, parent, f, d, recursive); + + LsObjects objects; + for (auto object : azure_scanner) { + objects.push_back(std::move(object)); + } + return objects; + } + + /** + * Constructs a scanner for listing Azure objects. The scanner can be used to + * retrieve an InputIterator for passing to algorithms such as `std::copy_if` + * or STL constructors supporting initialization via input iterators. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to recursively list subdirectories. + * @param max_keys The maximum number of keys to retrieve per request. + * @return Fully constructed AzureScanner object. + */ + template + AzureScanner scanner( + const URI& parent, + F f, + D d = accept_all_dirs, + bool recursive = false, + int max_keys = 1000) const { + return AzureScanner(*this, parent, f, d, recursive, max_keys); + } + /** * Renames an object. * @@ -577,6 +768,34 @@ class Azure { */ Status flush_blob_direct(const URI& uri); + /** + * Performs an Azure ListBlobs operation. + * + * @param container_name The container's name. + * @param blob_path The prefix of the blobs to list. + * @param recursive Whether to list blobs recursively. + * @param max_keys A hint to Azure for the maximum number of keys to return. + * @param continuation_token On entry, holds the token to pass to Azure to + * continue a listing operation, or nullopt if the operation is starting. On + * exit, will hold the continuation token to pass to the next listing + * operation, or nullopt if there are no more results. + * + * @return Vector of results with each entry being a pair of the string URI + * and object size. + * + * @note If continuation_token is not nullopt, callers must ensure that the + * container_name, blob_path and recursive parameters are the same as the + * previous call, going back to the first call when continuation_token was + * nullopt. This is not enforced by the function, which is the reason it is + * not public. + */ + LsObjects list_blobs_impl( + const std::string& container_name, + const std::string& blob_path, + bool recursive, + int max_keys, + optional& continuation_token) const; + /** * Parses a URI into a container name and blob path. For example, * URI "azure://my-container/dir1/file1" will parse into @@ -587,10 +806,8 @@ class Azure { * @param blob_path Mutates to the blob path. * @return Status */ - Status parse_azure_uri( - const URI& uri, - std::string* container_name, - std::string* blob_path) const; + static Status parse_azure_uri( + const URI& uri, std::string* container_name, std::string* blob_path); /** * Copies the blob at 'old_uri' to `new_uri`. @@ -601,17 +818,6 @@ class Azure { */ Status copy_blob(const URI& old_uri, const URI& new_uri); - /** - * Waits for a blob with `container_name` and `blob_path` - * to exist on Azure. - * - * @param container_name The blob's container name. - * @param blob_path The blob's path - * @return Status - */ - Status wait_for_blob_to_propagate( - const std::string& container_name, const std::string& blob_path) const; - /** * Check if 'container_name' is a container on Azure. * @@ -640,23 +846,86 @@ class Azure { * * @param path the string to remove the leading slash from. */ - std::string remove_front_slash(const std::string& path) const; + static std::string remove_front_slash(const std::string& path); /** * Adds a trailing slash from 'path' if it doesn't already have one. * * @param path the string to add the trailing slash to. */ - std::string add_trailing_slash(const std::string& path) const; + static std::string add_trailing_slash(const std::string& path); /** * Removes a trailing slash from 'path' if it exists. * * @param path the string to remove the trailing slash from. */ - std::string remove_trailing_slash(const std::string& path) const; + static std::string remove_trailing_slash(const std::string& path); }; +template +AzureScanner::AzureScanner( + const Azure& client, + const URI& prefix, + F file_filter, + D dir_filter, + bool recursive, + int max_keys) + : LsScanner(prefix, file_filter, dir_filter, recursive) + , client_(client) + , max_keys_(max_keys) + , has_fetched_(false) { + if (!prefix.is_azure()) { + throw AzureException("URI is not an Azure URI: " + prefix.to_string()); + } + + throw_if_not_ok(Azure::parse_azure_uri( + prefix.add_trailing_slash(), &container_name_, &blob_path_)); + fetch_results(); + next(begin_); +} + +template +void AzureScanner::next(typename Iterator::pointer& ptr) { + if (ptr == end_) { + ptr = fetch_results(); + } + + while (ptr != end_) { + auto& object = *ptr; + + // TODO: Add support for directory pruning. + if (this->file_filter_(object.first, object.second)) { + // Iterator is at the next object within results accepted by the filters. + return; + } else { + // Object was rejected by the FilePredicate, do not include it in results. + advance(ptr); + } + } +} + +template +typename AzureScanner::Iterator::pointer +AzureScanner::fetch_results() { + if (!more_to_fetch()) { + begin_ = end_ = typename Iterator::pointer(); + return end_; + } + + blobs_ = client_.list_blobs_impl( + container_name_, + blob_path_, + this->is_recursive_, + max_keys_, + continuation_token_); + has_fetched_ = true; + // Update pointers to the newly fetched results. + begin_ = blobs_.begin(); + end_ = blobs_.end(); + + return begin_; +} } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 1281ad82aead..1c63ab5d1ce6 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -514,31 +514,33 @@ class VFS : private VFSBase, protected S3_within_VFS { * the FilePredicate on each entry collected and the DirectoryPredicate on * common prefixes for pruning. * - * Currently this API is only supported for Posix and S3. + * Currently this API is only supported for local files, S3 and Azure. * * @param parent The parent prefix to list sub-paths. * @param f The FilePredicate to invoke on each object for filtering. * @param d The DirectoryPredicate to invoke on each common prefix for * pruning. This is currently unused, but is kept here for future support. + * @param recursive Whether to list the objects recursively. * @return Vector of results with each entry being a pair of the string URI * and object size. */ template - LsObjects ls_recursive( + LsObjects ls_filtered( const URI& parent, [[maybe_unused]] F f, - [[maybe_unused]] D d = accept_all_dirs) const { + [[maybe_unused]] D d, + bool recursive) const { LsObjects results; try { if (parent.is_file()) { #ifdef _WIN32 - results = win_.ls_filtered(parent, f, d, true); + results = win_.ls_filtered(parent, f, d, recursive); #else - results = posix_.ls_filtered(parent, f, d, true); + results = posix_.ls_filtered(parent, f, d, recursive); #endif } else if (parent.is_s3()) { #ifdef HAVE_S3 - results = s3().ls_filtered(parent, f, d, true); + results = s3().ls_filtered(parent, f, d, recursive); #else throw filesystem::VFSException("TileDB was built without S3 support"); #endif @@ -552,9 +554,7 @@ class VFS : private VFSBase, protected S3_within_VFS { #endif } else if (parent.is_azure()) { #ifdef HAVE_AZURE - throw filesystem::VFSException( - "Recursive ls over " + parent.backend_name() + - " storage backend is not supported."); + results = azure_.ls_filtered(parent, f, d, recursive); #else throw filesystem::VFSException( "TileDB was built without Azure support"); @@ -581,6 +581,28 @@ class VFS : private VFSBase, protected S3_within_VFS { return results; } + /** + * Recursively lists objects and object information that start with `prefix`, + * invoking the FilePredicate on each entry collected and the + * DirectoryPredicate on common prefixes for pruning. + * + * Currently this API is only supported for local files, S3 and Azure. + * + * @param parent The parent prefix to list sub-paths. + * @param f The FilePredicate to invoke on each object for filtering. + * @param d The DirectoryPredicate to invoke on each common prefix for + * pruning. This is currently unused, but is kept here for future support. + * @return Vector of results with each entry being a pair of the string URI + * and object size. + */ + template + LsObjects ls_recursive( + const URI& parent, + [[maybe_unused]] F f, + [[maybe_unused]] D d = accept_all_dirs) const { + return ls_filtered(parent, f, d, true); + } + /** * Renames a file. * From 1b4e5c288c08b2dec278f2c9fd984bc0b076b3ae Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 22 May 2024 13:56:48 -0400 Subject: [PATCH 366/456] Migrate APIs out of StorageManager: is_array and is_group. (#4980) Create a new directory, `tiledb/sm/object` to house `is_array` and `is_group`, migrated from `StorageManager`. --- [sc-46734] [sc-46735] [sc-47013] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `is_array` and `is_group`. --- tiledb/CMakeLists.txt | 1 + tiledb/sm/object/object.cc | 94 +++++++++++++++++++ tiledb/sm/object/object.h | 70 ++++++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 64 ++----------- .../storage_manager_canonical.h | 18 ---- 5 files changed, 173 insertions(+), 74 deletions(-) create mode 100644 tiledb/sm/object/object.cc create mode 100644 tiledb/sm/object/object.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 83fa0454a265..e3ce9de0f6d2 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -241,6 +241,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/types.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/serialization.cc diff --git a/tiledb/sm/object/object.cc b/tiledb/sm/object/object.cc new file mode 100644 index 000000000000..a7bb39a42b80 --- /dev/null +++ b/tiledb/sm/object/object.cc @@ -0,0 +1,94 @@ +/** + * @file object.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements standalone object functions. + */ + +#include "tiledb/sm/object/object.h" +#include "tiledb/sm/filesystem/vfs.h" +#include "tiledb/sm/rest/rest_client.h" + +namespace tiledb::sm { + +/* ********************************* */ +/* API */ +/* ********************************* */ + +bool is_array(ContextResources& resources, const URI& uri) { + // Handle remote array + if (uri.is_tiledb()) { + auto&& [st, exists] = + resources.rest_client()->check_array_exists_from_rest(uri); + throw_if_not_ok(st); + assert(exists.has_value()); + return exists.value(); + } else { + // Check if the schema directory exists or not + auto& vfs = resources.vfs(); + bool dir_exists = false; + throw_if_not_ok(vfs.is_dir( + uri.join_path(constants::array_schema_dir_name), &dir_exists)); + + if (dir_exists) { + return true; + } + + bool schema_exists = false; + // If there is no schema directory, we check schema file + throw_if_not_ok(vfs.is_file( + uri.join_path(constants::array_schema_filename), &schema_exists)); + return schema_exists; + } +} + +Status is_group(ContextResources& resources, const URI& uri, bool* is_group) { + // Handle remote array + if (uri.is_tiledb()) { + auto&& [st, exists] = + resources.rest_client()->check_group_exists_from_rest(uri); + throw_if_not_ok(st); + *is_group = *exists; + } else { + // Check for new group details directory + auto& vfs = resources.vfs(); + throw_if_not_ok( + vfs.is_dir(uri.join_path(constants::group_detail_dir_name), is_group)); + + if (*is_group) { + return Status::Ok(); + } + + // Fall back to older group file to check for legacy (pre-format 12) groups + throw_if_not_ok( + vfs.is_file(uri.join_path(constants::group_filename), is_group)); + } + return Status::Ok(); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/object/object.h b/tiledb/sm/object/object.h new file mode 100644 index 000000000000..75fc785431a3 --- /dev/null +++ b/tiledb/sm/object/object.h @@ -0,0 +1,70 @@ +/** + * @file object.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines standalone object functions. + */ + +#ifndef TILEDB_OBJECT_H +#define TILEDB_OBJECT_H + +#include "tiledb/common/common.h" +#include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/storage_manager/context_resources.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/* ********************************* */ +/* API */ +/* ********************************* */ + +/** + * Checks if the input URI represents an array. + * + * @param resources the context resources. + * @param uri the URI to be checked. + * @return bool + */ +bool is_array(ContextResources& resources, const URI& uri); + +/** + * Checks if the input URI represents a group. + * + * @param resources the context resources. + * @param uri the URI to be checked. + * @param is_group Set to `true` if the URI is a group and `false` + * otherwise. + * @return Status + */ +Status is_group(ContextResources& resources, const URI& uri, bool* is_group); + +} // namespace tiledb::sm + +#endif diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 8d10794fd88c..71616729d18d 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -64,6 +64,7 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/object/object.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/stats/global_stats.h" @@ -160,11 +161,11 @@ Status StorageManagerCanonical::array_create( } // Check if array exists - bool exists = is_array(array_uri); - if (exists) + if (is_array(resources_, array_uri)) { return logger_->status(Status_StorageManagerError( std::string("Cannot create array; Array '") + array_uri.c_str() + "' already exists")); + } std::lock_guard lock{object_create_mtx_}; array_schema->set_array_uri(array_uri); @@ -270,7 +271,7 @@ Status StorageManager::array_evolve_schema( tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY}; // Check if array exists - if (!is_array(array_uri)) { + if (!is_array(resources_, array_uri)) { return logger_->status(Status_StorageManagerError( std::string("Cannot evolve array; Array '") + array_uri.c_str() + "' not exists")); @@ -315,7 +316,7 @@ Status StorageManager::array_evolve_schema( Status StorageManagerCanonical::array_upgrade_version( const URI& array_uri, const Config& override_config) { // Check if array exists - if (!is_array(array_uri)) + if (!is_array(resources_, array_uri)) return logger_->status(Status_StorageManagerError( std::string("Cannot upgrade array; Array '") + array_uri.c_str() + "' does not exist")); @@ -503,7 +504,7 @@ Status StorageManagerCanonical::group_create(const std::string& group_uri) { // Check if group exists bool exists; - RETURN_NOT_OK(is_group(uri, &exists)); + RETURN_NOT_OK(is_group(resources_, uri, &exists)); if (exists) return logger_->status(Status_StorageManagerError( std::string("Cannot create group; Group '") + uri.c_str() + @@ -540,55 +541,6 @@ void StorageManagerCanonical::increment_in_progress() { queries_in_progress_cv_.notify_all(); } -bool StorageManagerCanonical::is_array(const URI& uri) const { - // Handle remote array - if (uri.is_tiledb()) { - auto&& [st, exists] = rest_client()->check_array_exists_from_rest(uri); - throw_if_not_ok(st); - assert(exists.has_value()); - return exists.value(); - } else { - // Check if the schema directory exists or not - bool dir_exists = false; - throw_if_not_ok(resources_.vfs().is_dir( - uri.join_path(constants::array_schema_dir_name), &dir_exists)); - - if (dir_exists) { - return true; - } - - // If there is no schema directory, we check schema file - bool schema_exists = false; - throw_if_not_ok(resources_.vfs().is_file( - uri.join_path(constants::array_schema_filename), &schema_exists)); - return schema_exists; - } - - // TODO: mark unreachable -} - -Status StorageManagerCanonical::is_group(const URI& uri, bool* is_group) const { - // Handle remote array - if (uri.is_tiledb()) { - auto&& [st, exists] = rest_client()->check_group_exists_from_rest(uri); - RETURN_NOT_OK(st); - *is_group = *exists; - } else { - // Check for new group details directory - throw_if_not_ok(resources_.vfs().is_dir( - uri.join_path(constants::group_detail_dir_name), is_group)); - - if (*is_group) { - return Status::Ok(); - } - - // Fall back to older group file to check for legacy (pre-format 12) groups - throw_if_not_ok(resources_.vfs().is_file( - uri.join_path(constants::group_filename), is_group)); - } - return Status::Ok(); -} - Status StorageManagerCanonical::object_type( const URI& uri, ObjectType* type) const { URI dir_uri = uri; @@ -608,13 +560,13 @@ Status StorageManagerCanonical::object_type( return Status::Ok(); } } - bool exists = is_array(uri); + bool exists = is_array(resources_, uri); if (exists) { *type = ObjectType::ARRAY; return Status::Ok(); } - RETURN_NOT_OK(is_group(uri, &exists)); + RETURN_NOT_OK(is_group(resources_, uri, &exists)); if (exists) { *type = ObjectType::GROUP; return Status::Ok(); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 6ed3d022589a..d96f8f2a202a 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -236,24 +236,6 @@ class StorageManagerCanonical { return resources_.rest_client().get(); } - /** - * Checks if the input URI represents an array. - * - * @param The URI to be checked. - * @return bool - */ - bool is_array(const URI& uri) const; - - /** - * Checks if the input URI represents a group. - * - * @param The URI to be checked. - * @param is_group Set to `true` if the URI is a group and `false` - * otherwise. - * @return Status - */ - Status is_group(const URI& uri, bool* is_group) const; - /** Removes a TileDB object (group, array). */ Status object_remove(const char* path) const; From ebaa63a60144148c73366b56a944158de47d8655 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 22 May 2024 16:31:48 -0400 Subject: [PATCH 367/456] Remove StorageManager::compute_tp(). (#4985) Remove `StorageManager::compute_tp()`. This also removes a lot of StorageManager variables. [sc-47579] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::compute_tp()`. --- .../storage_manager_override.h | 3 - tiledb/sm/c_api/tiledb.cc | 32 ++-- .../consolidator/array_meta_consolidator.cc | 7 +- .../sm/consolidator/commits_consolidator.cc | 7 +- .../sm/consolidator/fragment_consolidator.cc | 9 +- .../fragment_meta_consolidator.cc | 15 +- .../consolidator/group_meta_consolidator.cc | 8 +- tiledb/sm/fragment/fragment_info.cc | 2 +- tiledb/sm/group/group.cc | 6 +- .../array_dimension_label_queries.cc | 9 +- .../array_dimension_label_queries.h | 5 +- tiledb/sm/query/legacy/reader.cc | 37 ++--- tiledb/sm/query/query.cc | 17 +- tiledb/sm/query/readers/dense_reader.cc | 157 +++++++++--------- .../query/readers/ordered_dim_label_reader.cc | 18 +- tiledb/sm/query/readers/reader_base.cc | 33 ++-- .../readers/sparse_global_order_reader.cc | 48 +++--- .../query/readers/sparse_index_reader_base.cc | 30 +--- .../sparse_unordered_with_dups_reader.cc | 22 ++- .../sm/query/writers/global_order_writer.cc | 31 ++-- tiledb/sm/query/writers/ordered_writer.cc | 4 +- tiledb/sm/query/writers/unordered_writer.cc | 20 +-- tiledb/sm/query/writers/writer_base.cc | 34 ++-- tiledb/sm/storage_manager/storage_manager.cc | 2 +- .../storage_manager_canonical.h | 6 - 25 files changed, 239 insertions(+), 323 deletions(-) diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 8d123cd21fb0..666146165e2c 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -62,9 +62,6 @@ class StorageManagerStub { , config_(config) { } - inline common::ThreadPool* compute_tp() { - return &resources_.compute_tp(); - } inline common::ThreadPool* io_tp() { return &resources_.io_tp(); } diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9890faf1837c..9748f3cee07b 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -258,7 +258,7 @@ int32_t tiledb_array_schema_alloc( } // Create a new ArraySchema object - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_CREATE); (*array_schema)->array_schema_ = make_shared( HERE(), static_cast(array_type), memory_tracker); @@ -486,14 +486,11 @@ int32_t tiledb_array_schema_load( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); - // For easy reference - auto storage_manager{ctx->storage_manager()}; - // Load URIs from the array directory optional array_dir; try { array_dir.emplace( - storage_manager->resources(), + ctx->resources(), uri, 0, UINT64_MAX, @@ -506,7 +503,7 @@ int32_t tiledb_array_schema_load( return TILEDB_ERR; } - auto tracker = storage_manager->resources().ephemeral_memory_tracker(); + auto tracker = ctx->resources().ephemeral_memory_tracker(); // Load latest array schema auto&& array_schema_latest = @@ -569,14 +566,11 @@ int32_t tiledb_array_schema_load_with_key( return TILEDB_ERR; } - // For easy reference - auto storage_manager{ctx->storage_manager()}; - // Load URIs from the array directory optional array_dir; try { array_dir.emplace( - storage_manager->resources(), + ctx->resources(), uri, 0, UINT64_MAX, @@ -589,7 +583,7 @@ int32_t tiledb_array_schema_load_with_key( return TILEDB_ERR; } - auto tracker = storage_manager->resources().ephemeral_memory_tracker(); + auto tracker = ctx->resources().ephemeral_memory_tracker(); // Load latest array schema auto&& array_schema_latest = @@ -809,7 +803,7 @@ int32_t tiledb_array_schema_evolution_alloc( } // Create a new SchemaEvolution object - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); (*array_schema_evolution)->array_schema_evolution_ = new (std::nothrow) tiledb::sm::ArraySchemaEvolution(memory_tracker); @@ -3376,14 +3370,14 @@ int32_t tiledb_deserialize_array( return TILEDB_OOM; } - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); try { tiledb::sm::serialization::array_deserialize( (*array)->array_.get(), (tiledb::sm::SerializationType)serialize_type, buffer->buffer(), - ctx->context().resources(), + ctx->resources(), memory_tracker); } catch (StatusException& e) { delete *array; @@ -3441,7 +3435,7 @@ int32_t tiledb_deserialize_array_schema( } try { - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::ARRAY_LOAD); (*array_schema)->array_schema_ = tiledb::sm::serialization::array_schema_deserialize( @@ -3594,7 +3588,7 @@ int32_t tiledb_deserialize_array_schema_evolution( return TILEDB_OOM; } - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::SCHEMA_EVOLUTION); if (SAVE_ERROR_CATCH( ctx, @@ -3657,7 +3651,7 @@ int32_t tiledb_deserialize_query( client_side == 1, nullptr, query->query_, - ctx->storage_manager()->compute_tp())); + &ctx->resources().compute_tp())); return TILEDB_OK; } @@ -3751,7 +3745,7 @@ int32_t tiledb_deserialize_query_and_array( client_side == 1, nullptr, (*query)->query_, - ctx->storage_manager()->compute_tp())); + &ctx->resources().compute_tp())); return TILEDB_OK; } @@ -4226,7 +4220,7 @@ int32_t tiledb_deserialize_fragment_info( return TILEDB_ERR; } - auto memory_tracker = ctx->context().resources().create_memory_tracker(); + auto memory_tracker = ctx->resources().create_memory_tracker(); memory_tracker->set_type(sm::MemoryTrackerType::FRAGMENT_INFO_LOAD); if (SAVE_ERROR_CATCH( ctx, diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 11a77d2f16ff..69a620571637 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -126,14 +126,13 @@ void ArrayMetaConsolidator::vacuum(const char* array_name) { // Get the array metadata URIs and vacuum file URIs to be vacuum auto& vfs = resources_.vfs(); - auto compute_tp = storage_manager_->compute_tp(); - + auto& compute_tp = resources_.compute_tp(); auto array_dir = ArrayDirectory( resources_, URI(array_name), 0, std::numeric_limits::max()); // Delete the array metadata and vacuum files - vfs.remove_files(compute_tp, array_dir.array_meta_uris_to_vacuum()); - vfs.remove_files(compute_tp, array_dir.array_meta_vac_uris_to_vacuum()); + vfs.remove_files(&compute_tp, array_dir.array_meta_uris_to_vacuum()); + vfs.remove_files(&compute_tp, array_dir.array_meta_vac_uris_to_vacuum()); } /* ****************************** */ diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index 8bc3c6b9a175..c5ab1a31758b 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -117,9 +117,10 @@ void CommitsConsolidator::vacuum(const char* array_name) { // Delete the commits and vacuum files auto& vfs = resources_.vfs(); - auto compute_tp = storage_manager_->compute_tp(); - vfs.remove_files(compute_tp, array_dir.commit_uris_to_vacuum()); - vfs.remove_files(compute_tp, array_dir.consolidated_commits_uris_to_vacuum()); + auto& compute_tp = resources_.compute_tp(); + vfs.remove_files(&compute_tp, array_dir.commit_uris_to_vacuum()); + vfs.remove_files( + &compute_tp, array_dir.consolidated_commits_uris_to_vacuum()); } } // namespace tiledb::sm diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 374597efdcb9..322fcf8f4b41 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -446,12 +446,11 @@ void FragmentConsolidator::vacuum(const char* array_name) { array_dir.write_commit_ignore_file(commit_uris_to_ignore); } - auto& vfs = resources_.vfs(); - auto compute_tp = storage_manager_->compute_tp(); - // Delete fragment directories + auto& vfs = resources_.vfs(); + auto& compute_tp = resources_.compute_tp(); throw_if_not_ok(parallel_for( - compute_tp, 0, fragment_uris_to_vacuum.size(), [&](size_t i) { + &compute_tp, 0, fragment_uris_to_vacuum.size(), [&](size_t i) { // Remove the commit file, if present. auto commit_uri = array_dir.get_commit_uri(fragment_uris_to_vacuum[i]); bool is_file = false; @@ -471,7 +470,7 @@ void FragmentConsolidator::vacuum(const char* array_name) { // Delete the vacuum files. vfs.remove_files( - compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); + &compute_tp, filtered_fragment_uris.fragment_vac_uris_to_vacuum()); } /* ****************************** */ diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 47cf67ed2ca2..39f0f1d78181 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -118,8 +118,8 @@ Status FragmentMetaConsolidator::consolidate( // Serialize all fragment metadata footers in parallel std::vector> tiles(meta.size()); - auto status = parallel_for( - storage_manager_->compute_tp(), 0, tiles.size(), [&](size_t i) { + auto status = + parallel_for(&resources_.compute_tp(), 0, tiles.size(), [&](size_t i) { SizeComputationSerializer size_computation_serializer; meta[i]->write_footer(size_computation_serializer); tiles[i] = WriterTile::from_generic( @@ -187,9 +187,7 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { // Get the consolidated fragment metadata URIs to be deleted // (all except the last one) - ArrayDirectory array_dir( - storage_manager_->resources(), URI(array_name), 0, UINT64_MAX); - + ArrayDirectory array_dir(resources_, URI(array_name), 0, UINT64_MAX); const auto& fragment_meta_uris = array_dir.fragment_meta_uris(); // Get the latest timestamp @@ -202,12 +200,11 @@ void FragmentMetaConsolidator::vacuum(const char* array_name) { } } - auto& vfs = resources_.vfs(); - auto compute_tp = storage_manager_->compute_tp(); - // Vacuum + auto& vfs = resources_.vfs(); + auto& compute_tp = resources_.compute_tp(); throw_if_not_ok( - parallel_for(compute_tp, 0, fragment_meta_uris.size(), [&](size_t i) { + parallel_for(&compute_tp, 0, fragment_meta_uris.size(), [&](size_t i) { auto& uri = fragment_meta_uris[i]; FragmentID fragment_id{uri}; auto timestamp_range{fragment_id.timestamp_range()}; diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index 2849110112f6..cc4d9b3920b0 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -113,12 +113,12 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { // Get the group metadata URIs and vacuum file URIs to be vacuumed auto& vfs = resources_.vfs(); - auto compute_tp = storage_manager_->compute_tp(); + auto& compute_tp = resources_.compute_tp(); GroupDirectory group_dir; try { group_dir = GroupDirectory( &vfs, - compute_tp, + &compute_tp, URI(group_name), 0, std::numeric_limits::max()); @@ -127,8 +127,8 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { } // Delete the group metadata and vacuum files - vfs.remove_files(compute_tp, group_dir.group_meta_uris_to_vacuum()); - vfs.remove_files(compute_tp, group_dir.group_meta_vac_uris_to_vacuum()); + vfs.remove_files(&compute_tp, group_dir.group_meta_uris_to_vacuum()); + vfs.remove_files(&compute_tp, group_dir.group_meta_vac_uris_to_vacuum()); } /* ****************************** */ diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index a886a9ade66e..17ba2a5e5f7e 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -867,7 +867,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { // Get fragment sizes std::vector sizes(fragment_num, 0); - RETURN_NOT_OK(parallel_for( + throw_if_not_ok(parallel_for( &resources_->compute_tp(), 0, fragment_num, diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 81b592b3e210..3cf1403838d9 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -164,7 +164,7 @@ Status Group::open( group_dir_ = make_shared( HERE(), &resources_.vfs(), - storage_manager_->compute_tp(), + &resources_.compute_tp(), group_uri_, timestamp_start, timestamp_end); @@ -178,7 +178,7 @@ Status Group::open( group_dir_ = make_shared( HERE(), &resources_.vfs(), - storage_manager_->compute_tp(), + &resources_.compute_tp(), group_uri_, timestamp_start, (timestamp_end != 0) ? timestamp_end : @@ -758,7 +758,7 @@ void Group::load_metadata_from_storage( uri, 0, encryption_key, - storage_manager_->resources().ephemeral_memory_tracker()); + resources_.ephemeral_memory_tracker()); return Status::Ok(); })); diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc index 6812c3ee9691..c6db66ecd8da 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -56,7 +56,8 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( const std::unordered_map& label_buffers, const std::unordered_map& array_buffers, const optional& fragment_name) - : storage_manager_(storage_manager) + : resources_(storage_manager->resources()) + , storage_manager_(storage_manager) , label_range_queries_by_dim_idx_(subarray.dim_num(), nullptr) , label_data_queries_by_dim_idx_(subarray.dim_num()) , range_query_status_{QueryStatus::UNINITIALIZED} @@ -150,7 +151,7 @@ std::vector ArrayDimensionLabelQueries::get_data_query( void ArrayDimensionLabelQueries::process_data_queries() { throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, data_queries_.size(), [&](const size_t query_idx) { @@ -170,7 +171,7 @@ void ArrayDimensionLabelQueries::process_data_queries() { void ArrayDimensionLabelQueries::process_range_queries(Query* parent_query) { // Process queries and update the subarray. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, label_range_queries_by_dim_idx_.size(), [&](const uint32_t dim_idx) { diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h index 033913d3a80e..7746956ee85c 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -159,6 +159,9 @@ class ArrayDimensionLabelQueries { void process_range_queries(Query* parent_query); private: + /** The context resources. */ + ContextResources& resources_; + /** The storage manager. */ StorageManager* storage_manager_; diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index 119a10e902e5..d20883783ab3 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -518,8 +518,8 @@ Status Reader::compute_range_result_coords( } } - auto status = parallel_for( - storage_manager_->compute_tp(), 0, range_num, [&](uint64_t r) { + auto status = + parallel_for(&resources_.compute_tp(), 0, range_num, [&](uint64_t r) { // Compute overlapping coordinates per range RETURN_NOT_OK(compute_range_result_coords( subarray, @@ -616,8 +616,8 @@ Status Reader::compute_range_result_coords( // Gather result range coordinates per fragment auto fragment_num = fragment_metadata_.size(); std::vector> range_result_coords_vec(fragment_num); - auto status = parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint32_t f) { + auto status = + parallel_for(&resources_.compute_tp(), 0, fragment_num, [&](uint32_t f) { return compute_range_result_coords( subarray, range_idx, @@ -913,7 +913,7 @@ Status Reader::copy_fixed_cells( &cs_offsets, fixed_cs_partitions); auto status = parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, fixed_cs_partitions->size(), std::move(copy_fn)); @@ -937,10 +937,8 @@ void Reader::compute_fixed_cs_partitions( return; } - const auto num_copy_threads = - storage_manager_->compute_tp()->concurrency_level(); - // Calculate the partition sizes. + const auto num_copy_threads = resources_.compute_tp().concurrency_level(); auto num_cs = result_cell_slabs.size(); const uint64_t num_cs_partitions = std::min(num_copy_threads, num_cs); @@ -1105,7 +1103,7 @@ Status Reader::copy_var_cells( &var_offsets_per_cs, var_cs_partitions); auto status = parallel_for( - storage_manager_->compute_tp(), 0, var_cs_partitions->size(), copy_fn); + &resources_.compute_tp(), 0, var_cs_partitions->size(), copy_fn); RETURN_NOT_OK(status); @@ -1126,10 +1124,8 @@ void Reader::compute_var_cs_partitions( return; } - const auto num_copy_threads = - storage_manager_->compute_tp()->concurrency_level(); - // Calculate the partition range. + const auto num_copy_threads = resources_.compute_tp().concurrency_level(); const uint64_t num_cs = result_cell_slabs.size(); const uint64_t num_cs_partitions = std::min(num_copy_threads, num_cs); @@ -1888,7 +1884,7 @@ void Reader::init_read_state() { memory_budget, memory_budget_var, memory_budget_validity, - storage_manager_->compute_tp(), + &resources_.compute_tp(), stats_, logger_); read_state_.overflowed_ = false; @@ -1952,26 +1948,23 @@ Status Reader::sort_result_coords( if (layout == Layout::ROW_MAJOR) { parallel_sort( - storage_manager_->compute_tp(), iter_begin, iter_end, RowCmp(domain)); + &resources_.compute_tp(), iter_begin, iter_end, RowCmp(domain)); } else if (layout == Layout::COL_MAJOR) { parallel_sort( - storage_manager_->compute_tp(), iter_begin, iter_end, ColCmp(domain)); + &resources_.compute_tp(), iter_begin, iter_end, ColCmp(domain)); } else if (layout == Layout::GLOBAL_ORDER) { if (array_schema_.cell_order() == Layout::HILBERT) { std::vector> hilbert_values(coords_num); RETURN_NOT_OK(calculate_hilbert_values(iter_begin, &hilbert_values)); parallel_sort( - storage_manager_->compute_tp(), + &resources_.compute_tp(), hilbert_values.begin(), hilbert_values.end(), HilbertCmpRCI(domain, iter_begin)); RETURN_NOT_OK(reorganize_result_coords(iter_begin, &hilbert_values)); } else { parallel_sort( - storage_manager_->compute_tp(), - iter_begin, - iter_end, - GlobalCmp(domain)); + &resources_.compute_tp(), iter_begin, iter_end, GlobalCmp(domain)); } } else { assert(false); @@ -2112,8 +2105,8 @@ Status Reader::calculate_hilbert_values( auto coords_num = (uint64_t)hilbert_values->size(); // Calculate Hilbert values in parallel - auto status = parallel_for( - storage_manager_->compute_tp(), 0, coords_num, [&](uint64_t c) { + auto status = + parallel_for(&resources_.compute_tp(), 0, coords_num, [&](uint64_t c) { std::vector coords(dim_num); for (uint32_t d = 0; d < dim_num; ++d) { auto dim{array_schema_.dimension_ptr(d)}; diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index d5821f75db17..071b62d0f36a 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -211,7 +211,7 @@ FieldDataSize Query::internal_est_result_size(std::string_view field_name) { ": operation currently supported only for read queries"); } if (array_->is_remote() && !subarray_.est_result_size_computed()) { - auto rest_client = storage_manager_->rest_client(); + auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { throw QueryStatusException( "Error in query estimate result size; " @@ -221,7 +221,7 @@ FieldDataSize Query::internal_est_result_size(std::string_view field_name) { rest_client->get_query_est_result_sizes(array_->array_uri(), this)); } return subarray_.get_est_result_size( - field_name, &config_, storage_manager_->compute_tp()); + field_name, &config_, &resources_.compute_tp()); } FieldDataSize Query::get_est_result_size_fixed_nonnull( @@ -268,14 +268,12 @@ FieldDataSize Query::get_est_result_size_variable_nullable( std::unordered_map Query::get_est_result_size_map() { - return subarray_.get_est_result_size_map( - &config_, storage_manager_->compute_tp()); + return subarray_.get_est_result_size_map(&config_, &resources_.compute_tp()); } std::unordered_map Query::get_max_mem_size_map() { - return subarray_.get_max_mem_size_map( - &config_, storage_manager_->compute_tp()); + return subarray_.get_max_mem_size_map(&config_, &resources_.compute_tp()); } Status Query::get_written_fragment_num(uint32_t* num) const { @@ -810,10 +808,7 @@ Status Query::process() { } throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - enmr_names.size(), - [&](const uint64_t i) { + &resources_.compute_tp(), 0, enmr_names.size(), [&](const uint64_t i) { array_->get_enumeration(enmr_names[i]); return Status::Ok(); })); @@ -1716,7 +1711,7 @@ bool Query::use_refactored_sparse_unordered_with_dups_reader( } bool Query::non_overlapping_ranges() { - return subarray_.non_overlapping_ranges(storage_manager_->compute_tp()); + return subarray_.non_overlapping_ranges(&resources_.compute_tp()); } bool Query::is_dense() const { diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index b0967271a1c0..2c25f3e120e8 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -257,10 +257,7 @@ Status DenseReader::dense_read() { const auto& layout = layout_ == Layout::GLOBAL_ORDER ? array_schema_.cell_order() : layout_; auto status = parallel_for( - storage_manager_->compute_tp(), - 0, - tile_subarrays.size(), - [&](uint64_t t) { + &resources_.compute_tp(), 0, tile_subarrays.size(), [&](uint64_t t) { subarray.crop_to_tile( &tile_subarrays[t], (const DimType*)&tile_coords[t][0], layout); return Status::Ok(); @@ -388,8 +385,7 @@ Status DenseReader::dense_read() { // Compute parallelization parameters. uint64_t num_range_threads = 1; - const auto num_threads = - storage_manager_->compute_tp()->concurrency_level(); + const auto num_threads = resources_.compute_tp().concurrency_level(); if ((t_end - t_start) < num_threads) { // Ceil the division between thread_num and tile_num. num_range_threads = 1 + ((num_threads - 1) / (t_end - t_start)); @@ -446,73 +442,72 @@ Status DenseReader::dense_read() { } if (compute_task.valid()) { - RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); if (read_state_.overflowed_) { return Status::Ok(); } } - compute_task = - storage_manager_->compute_tp()->execute([&, - filtered_data, - dense_dim, - name, - validity_only, - t_start, - t_end, - subarray_start_cell, - subarray_end_cell, - num_range_threads, - result_tiles]() { - if (!dense_dim) { - // Unfilter tiles. - RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); - - // Only copy names that are present in the user buffers. - if (buffers_.count(name) != 0) { - // Copy attribute data to users buffers. - auto& var_buffer_size = var_buffer_sizes[name]; - status = copy_attribute( - name, - tile_extents, - subarray, - t_start, - t_end, - subarray_start_cell, - subarray_end_cell, - tile_subarrays, - tile_offsets, - var_buffer_size, - range_info, - result_space_tiles, - qc_result, - num_range_threads); - RETURN_CANCEL_OR_ERROR(status); - } - } + compute_task = resources_.compute_tp().execute([&, + filtered_data, + dense_dim, + name, + validity_only, + t_start, + t_end, + subarray_start_cell, + subarray_end_cell, + num_range_threads, + result_tiles]() { + if (!dense_dim) { + // Unfilter tiles. + RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); + + // Only copy names that are present in the user buffers. + if (buffers_.count(name) != 0) { + // Copy attribute data to users buffers. + auto& var_buffer_size = var_buffer_sizes[name]; + status = copy_attribute( + name, + tile_extents, + subarray, + t_start, + t_end, + subarray_start_cell, + subarray_end_cell, + tile_subarrays, + tile_offsets, + var_buffer_size, + range_info, + result_space_tiles, + qc_result, + num_range_threads); + RETURN_CANCEL_OR_ERROR(status); + } + } - if (aggregates_.count(name) != 0) { - status = process_aggregates( - name, - tile_extents, - subarray, - t_start, - t_end, - tile_subarrays, - tile_offsets, - range_info, - result_space_tiles, - qc_result, - num_range_threads); - RETURN_CANCEL_OR_ERROR(status); - } + if (aggregates_.count(name) != 0) { + status = process_aggregates( + name, + tile_extents, + subarray, + t_start, + t_end, + tile_subarrays, + tile_offsets, + range_info, + result_space_tiles, + qc_result, + num_range_threads); + RETURN_CANCEL_OR_ERROR(status); + } - if (!dense_dim) { - clear_tiles(name, result_tiles); - } + if (!dense_dim) { + clear_tiles(name, result_tiles); + } - return Status::Ok(); - }); + return Status::Ok(); + }); } // Process count aggregates. @@ -539,7 +534,7 @@ Status DenseReader::dense_read() { } if (compute_task.valid()) { - RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); } // For `qc_coords_mode` just fill in the coordinates and skip attribute @@ -618,7 +613,7 @@ void DenseReader::init_read_state() { std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max(), - storage_manager_->compute_tp(), + &resources_.compute_tp(), stats_, logger_); read_state_.overflowed_ = false; @@ -736,7 +731,7 @@ uint64_t DenseReader::compute_space_tiles_end( // load two batches at once. Wait for the first compute task to complete // before loading more tiles. if (compute_task.valid() && available_memory < tile_upper_memory_limit_) { - throw_if_not_ok(storage_manager_->compute_tp()->wait(compute_task)); + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); } const uint64_t upper_memory_limit = @@ -926,16 +921,16 @@ Status DenseReader::apply_query_condition( NameToLoad::from_string_vec(qc_names), result_tiles)); if (compute_task.valid()) { - RETURN_NOT_OK(storage_manager_->compute_tp()->wait(compute_task)); + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); } - compute_task = storage_manager_->compute_tp()->execute([&, - filtered_data, - qc_names, - t_start, - t_end, - num_range_threads, - result_tiles]() { + compute_task = resources_.compute_tp().execute([&, + filtered_data, + qc_names, + t_start, + t_end, + num_range_threads, + result_tiles]() { // For easy reference. const auto& tile_coords = subarray.tile_coords(); const auto dim_num = array_schema_.dim_num(); @@ -954,7 +949,7 @@ Status DenseReader::apply_query_condition( // Process all tiles in parallel. auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), t_start, t_end, 0, @@ -1135,7 +1130,7 @@ Status DenseReader::copy_attribute( { auto timer_se = stats_->start_timer("copy_offset_tiles"); auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), t_start, t_end, 0, @@ -1195,7 +1190,7 @@ Status DenseReader::copy_attribute( auto timer_se = stats_->start_timer("copy_var_tiles"); // Process var data in parallel. auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), t_start, t_end, 0, @@ -1237,7 +1232,7 @@ Status DenseReader::copy_attribute( auto timer_se = stats_->start_timer("copy_fixed_tiles"); // Process values in parallel. auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), t_start, t_end, 0, @@ -1337,7 +1332,7 @@ Status DenseReader::process_aggregates( // Process values in parallel. auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), t_start, t_end, 0, diff --git a/tiledb/sm/query/readers/ordered_dim_label_reader.cc b/tiledb/sm/query/readers/ordered_dim_label_reader.cc index b385649d9c6b..93df4650ed8e 100644 --- a/tiledb/sm/query/readers/ordered_dim_label_reader.cc +++ b/tiledb/sm/query/readers/ordered_dim_label_reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,8 +53,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class OrderedDimLabelReaderStatusException : public StatusException { public: @@ -253,8 +252,8 @@ void OrderedDimLabelReader::label_read() { read_and_unfilter_attribute_tiles({label_name_}, result_tiles)); // Compute/copy results. - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, max_range, [&](uint64_t r) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, max_range, [&](uint64_t r) { compute_and_copy_range_indexes(buffer_offset, r); return Status::Ok(); })); @@ -295,7 +294,7 @@ void OrderedDimLabelReader::compute_array_tile_indexes_for_ranges() { per_range_array_tile_indexes[r].resize(fragment_metadata_.size()); } throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, fragment_metadata_.size(), 0, @@ -310,7 +309,7 @@ void OrderedDimLabelReader::compute_array_tile_indexes_for_ranges() { // value for each range start/end. per_range_array_tile_indexes_.resize(ranges_.size()); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, ranges_.size(), [&](uint64_t r) { + &resources_.compute_tp(), 0, ranges_.size(), [&](uint64_t r) { per_range_array_tile_indexes_[r] = RangeTileIndexes( tile_idx_min, tile_idx_max, per_range_array_tile_indexes[r]); return Status::Ok(); @@ -323,7 +322,7 @@ void OrderedDimLabelReader::load_label_min_max_values() { // Load min/max data for all fragments. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](const uint64_t i) { @@ -736,5 +735,4 @@ void OrderedDimLabelReader::compute_and_copy_range_indexes( apply_with_type(g, label_type_); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 437cadb742e1..4c44607ee041 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -432,7 +432,7 @@ void ReaderBase::load_tile_offsets( const auto encryption_key = array_->encryption_key(); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, relevant_fragments.size(), [&](const uint64_t i) { @@ -463,7 +463,7 @@ void ReaderBase::load_tile_var_sizes( const auto encryption_key = array_->encryption_key(); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, relevant_fragments.size(), [&](const uint64_t i) { @@ -498,7 +498,7 @@ void ReaderBase::load_tile_metadata( const auto encryption_key = array_->encryption_key(); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, relevant_fragments.size(), [&](const uint64_t i) { @@ -539,7 +539,7 @@ void ReaderBase::load_processed_conditions() { // Load all fragments in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](const uint64_t i) { @@ -849,7 +849,7 @@ Status ReaderBase::unfilter_tiles( // Compute parallelization parameters. uint64_t num_range_threads = 1; - const auto num_threads = storage_manager_->compute_tp()->concurrency_level(); + const auto num_threads = resources_.compute_tp().concurrency_level(); if (chunking && num_tiles < num_threads) { // Ceil the division between thread_num and num_tiles. num_range_threads = 1 + ((num_threads - 1) / num_tiles); @@ -866,7 +866,7 @@ Status ReaderBase::unfilter_tiles( // Pre-compute chunk offsets. auto status = parallel_for( - storage_manager_->compute_tp(), 0, num_tiles, [&, this](uint64_t i) { + &resources_.compute_tp(), 0, num_tiles, [&, this](uint64_t i) { auto&& [st, tile_size, tile_var_size, tile_validity_size] = load_tile_chunk_data( name, @@ -890,7 +890,7 @@ Status ReaderBase::unfilter_tiles( // Unfilter all tiles/chunks in parallel using the precomputed offsets. status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, num_tiles, 0, @@ -981,7 +981,7 @@ Status ReaderBase::unfilter_tile( t_var->type(), array_schema_.version()); } - auto concurrency_level = storage_manager_->compute_tp()->concurrency_level(); + auto concurrency_level = resources_.compute_tp().concurrency_level(); if (!validity_only) { // Unfiltered fixed data @@ -1138,10 +1138,7 @@ ReaderBase::cache_dimension_label_data() { std::vector non_empty_domains(fragment_metadata_.size()); std::vector frag_first_array_tile_idx(fragment_metadata_.size()); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - fragment_metadata_.size(), - [&](unsigned f) { + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](unsigned f) { non_empty_domains[f] = fragment_metadata_[f]->non_empty_domain()[0].data(); auto ned = static_cast(non_empty_domains[f]); @@ -1188,10 +1185,7 @@ void ReaderBase::validate_attribute_order( AttributeOrderValidator validator( attribute_name, fragment_metadata_.size(), query_memory_tracker_); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - fragment_metadata_.size(), - [&](uint64_t f) { + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](uint64_t f) { validator.find_fragments_to_check( array_min_idx, array_max_idx, f, non_empty_domains); @@ -1199,10 +1193,7 @@ void ReaderBase::validate_attribute_order( })); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - fragment_metadata_.size(), - [&](int64_t f) { + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](int64_t f) { validator.validate_without_loading_tiles( index_dim, increasing_data, @@ -1223,7 +1214,7 @@ void ReaderBase::validate_attribute_order( // Validate bounds not validated using tile data. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, fragment_metadata_.size(), [&](unsigned f) { diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index e37612a3ff69..3abf1518d68a 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,8 +53,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class SparseGlobalOrderReaderStatusException : public StatusException { public: @@ -363,7 +362,7 @@ SparseGlobalOrderReader::create_result_tiles( if (subarray_.is_set()) { // Load as many tiles as the memory budget allows. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { + &resources_.compute_tp(), 0, fragment_num, [&](uint64_t f) { uint64_t t = 0; auto& tile_ranges = tmp_read_state_.tile_ranges(f); while (!tile_ranges.empty()) { @@ -412,7 +411,7 @@ SparseGlobalOrderReader::create_result_tiles( } else { // Load as many tiles as the memory budget allows. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { + &resources_.compute_tp(), 0, fragment_num, [&](uint64_t f) { uint64_t t = 0; auto tile_num = fragment_metadata_[f]->tile_num(); @@ -491,8 +490,8 @@ void SparseGlobalOrderReader::clean_tile_list( std::vector& result_tiles) { // Clear result tiles that are not necessary anymore. auto fragment_num = fragment_metadata_.size(); - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, fragment_num, [&](uint64_t f) { auto it = result_tiles[f].begin(); while (it != result_tiles[f].end()) { if (it->result_num() == 0) { @@ -520,7 +519,7 @@ void SparseGlobalOrderReader::dedup_tiles_with_timestamps( // Process all tiles in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { const auto f = result_tiles[t]->frag_idx(); if (fragment_metadata_[f]->has_timestamps()) { // For easy reference. @@ -595,8 +594,8 @@ void SparseGlobalOrderReader::dedup_fragments_with_timestamps( // Run all fragments in parallel. auto fragment_num = fragment_metadata_.size(); - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, fragment_num, [&](uint64_t f) { // Run only for fragments with timestamps. if (fragment_metadata_[f]->has_timestamps()) { // Process all tiles. @@ -846,7 +845,7 @@ void SparseGlobalOrderReader::compute_hilbert_values( // Parallelize on tiles. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { auto tile = static_cast*>(result_tiles[t]); auto cell_num = @@ -926,7 +925,7 @@ SparseGlobalOrderReader::merge_result_cell_slabs( // For all fragments, get the first tile in the sorting queue. std::vector to_delete; throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, result_tiles.size(), [&](uint64_t f) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t f) { if (result_tiles[f].size() > 0) { // Initialize the iterator for this fragment. rt_it[f] = result_tiles[f].begin(); @@ -1173,7 +1172,7 @@ void SparseGlobalOrderReader::copy_offsets_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -1279,7 +1278,7 @@ void SparseGlobalOrderReader::copy_var_data_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -1350,7 +1349,7 @@ void SparseGlobalOrderReader::copy_fixed_data_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -1450,7 +1449,7 @@ void SparseGlobalOrderReader::copy_timestamps_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -1517,7 +1516,7 @@ void SparseGlobalOrderReader::copy_delete_meta_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -1626,8 +1625,8 @@ SparseGlobalOrderReader::respect_copy_memory_budget( uint64_t max_cs_idx = result_cell_slabs.size(); std::mutex max_cs_idx_mtx; std::vector total_mem_usage_per_attr(names.size()); - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, names.size(), [&](uint64_t i) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, names.size(), [&](uint64_t i) { // For easy reference. const auto& name = names[i]; const bool agg_only = aggregate_only(name); @@ -1837,7 +1836,7 @@ void SparseGlobalOrderReader::process_slabs( // Compute parallelization parameters. uint64_t num_range_threads = 1; - const auto num_threads = storage_manager_->compute_tp()->concurrency_level(); + const auto num_threads = resources_.compute_tp().concurrency_level(); if (result_cell_slabs.size() < num_threads) { // Ceil the division between thread_num and tile_num. num_range_threads = 1 + ((num_threads - 1) / result_cell_slabs.size()); @@ -2119,7 +2118,7 @@ void SparseGlobalOrderReader::process_aggregates( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_cell_slabs.size(), 0, @@ -2194,8 +2193,8 @@ void SparseGlobalOrderReader::end_iteration( auto fragment_num = fragment_metadata_.size(); // Clear fully processed tiles in each fragments. - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, fragment_num, [&](uint64_t f) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, fragment_num, [&](uint64_t f) { while (!result_tiles[f].empty() && result_tiles[f].front().tile_idx() < read_state_.frag_idx()[f].tile_idx_) { @@ -2226,5 +2225,4 @@ template SparseGlobalOrderReader::SparseGlobalOrderReader( template SparseGlobalOrderReader::SparseGlobalOrderReader( stats::Stats*, shared_ptr, StrategyParams&, bool); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 59c36e1f2144..0c6574992429 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -155,10 +155,7 @@ std::vector SparseIndexReaderBase::tile_offset_sizes() { // Compute the size of tile offsets per fragments. const auto relevant_fragments = subarray_.relevant_fragments(); throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - relevant_fragments.size(), - [&](uint64_t i) { + &resources_.compute_tp(), 0, relevant_fragments.size(), [&](uint64_t i) { // For easy reference. auto frag_idx = relevant_fragments[i]; auto& fragment = fragment_metadata_[frag_idx]; @@ -403,9 +400,7 @@ Status SparseIndexReaderBase::load_initial_data() { // This is ok as it is a soft limit and will be taken into consideration // below. subarray_.precompute_all_ranges_tile_overlap( - storage_manager_->compute_tp(), - read_state_.frag_idx(), - &tmp_read_state_); + &resources_.compute_tp(), read_state_.frag_idx(), &tmp_read_state_); if (tmp_read_state_.memory_used_tile_ranges() > memory_budget_.ratio_tile_ranges() * memory_budget_.total_budget()) @@ -414,7 +409,7 @@ Status SparseIndexReaderBase::load_initial_data() { } else { for (const auto& [name, _] : aggregates_) { if (array_schema_.is_dim(name)) { - subarray_.load_relevant_fragment_rtrees(storage_manager_->compute_tp()); + subarray_.load_relevant_fragment_rtrees(&resources_.compute_tp()); break; } } @@ -589,7 +584,7 @@ void SparseIndexReaderBase::compute_tile_bitmaps( // Compute parallelization parameters. uint64_t num_range_threads = 1; - const auto num_threads = storage_manager_->compute_tp()->concurrency_level(); + const auto num_threads = resources_.compute_tp().concurrency_level(); if (result_tiles.size() < num_threads) { // Ceil the division between thread_num and tile_num. num_range_threads = 1 + ((num_threads - 1) / result_tiles.size()); @@ -601,10 +596,7 @@ void SparseIndexReaderBase::compute_tile_bitmaps( if (num_range_threads != 1) { // Resize bitmaps to process for each tiles in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - result_tiles.size(), - [&](uint64_t t) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { static_cast*>(result_tiles[t]) ->alloc_bitmap(); return Status::Ok(); @@ -613,7 +605,7 @@ void SparseIndexReaderBase::compute_tile_bitmaps( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_tiles.size(), 0, @@ -711,10 +703,7 @@ void SparseIndexReaderBase::compute_tile_bitmaps( if (num_range_threads != 1) { // Compute number of cells in each bitmaps in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - result_tiles.size(), - [&](uint64_t t) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { static_cast*>(result_tiles[t]) ->count_cells(); return Status::Ok(); @@ -733,10 +722,7 @@ void SparseIndexReaderBase::apply_query_condition( use_timestamps_) { // Process all tiles in parallel. throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), - 0, - result_tiles.size(), - [&](uint64_t t) { + &resources_.compute_tp(), 0, result_tiles.size(), [&](uint64_t t) { // For easy reference. auto rt = static_cast(result_tiles[t]); const auto frag_meta = fragment_metadata_[rt->frag_idx()]; diff --git a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc index 026538b20c03..b61595b5063e 100644 --- a/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc +++ b/tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,8 +50,7 @@ using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm::stats; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class SparseUnorderedWithDupsReaderStatusException : public StatusException { public: @@ -763,7 +762,7 @@ void SparseUnorderedWithDupsReader::copy_offsets_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_tiles.size(), 0, @@ -869,7 +868,7 @@ void SparseUnorderedWithDupsReader::copy_var_data_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_tiles.size(), 0, @@ -1241,7 +1240,7 @@ void SparseUnorderedWithDupsReader::copy_fixed_data_tiles( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_tiles.size(), 0, @@ -1444,8 +1443,8 @@ SparseUnorderedWithDupsReader::respect_copy_memory_budget( uint64_t max_rt_idx = result_tiles.size(); std::mutex max_rt_idx_mtx; std::vector total_mem_usage_per_attr(names.size()); - throw_if_not_ok(parallel_for( - storage_manager_->compute_tp(), 0, names.size(), [&](uint64_t i) { + throw_if_not_ok( + parallel_for(&resources_.compute_tp(), 0, names.size(), [&](uint64_t i) { // For easy reference. const auto& name = names[i]; const bool agg_only = aggregate_only(name); @@ -1623,7 +1622,7 @@ bool SparseUnorderedWithDupsReader::process_tiles( // Compute parallelization parameters. uint64_t num_range_threads = 1; - const auto num_threads = storage_manager_->compute_tp()->concurrency_level(); + const auto num_threads = resources_.compute_tp().concurrency_level(); if (result_tiles.size() < num_threads) { // Ceil the division between thread_num and tile_num. num_range_threads = 1 + ((num_threads - 1) / result_tiles.size()); @@ -1924,7 +1923,7 @@ void SparseUnorderedWithDupsReader::process_aggregates( // Process all tiles/cells in parallel. throw_if_not_ok(parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, result_tiles.size(), 0, @@ -2032,5 +2031,4 @@ template SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( template SparseUnorderedWithDupsReader::SparseUnorderedWithDupsReader( stats::Stats*, shared_ptr, StrategyParams&); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 08b6aa358db3..3dd4a254e196 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -326,10 +326,7 @@ Status GlobalOrderWriter::check_coord_dups() const { } auto status = parallel_for( - storage_manager_->compute_tp(), - 1, - coords_info_.coords_num_, - [&](uint64_t i) { + &resources_.compute_tp(), 1, coords_info_.coords_num_, [&](uint64_t i) { // Check for duplicate in adjacent cells bool found_dup = true; for (unsigned d = 0; d < dim_num; ++d) { @@ -426,7 +423,7 @@ Status GlobalOrderWriter::check_global_order() const { // Check if all coordinates are in global order in parallel auto status = parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, coords_info_.coords_num_ - 1, [&](uint64_t i) { @@ -477,7 +474,7 @@ Status GlobalOrderWriter::check_global_order_hilbert() const { // Check if all coordinates are in hilbert order in parallel auto status = parallel_for( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, coords_info_.coords_num_ - 1, [&](uint64_t i) { @@ -575,10 +572,7 @@ Status GlobalOrderWriter::compute_coord_dups( std::mutex mtx; auto status = parallel_for( - storage_manager_->compute_tp(), - 1, - coords_info_.coords_num_, - [&](uint64_t i) { + &resources_.compute_tp(), 1, coords_info_.coords_num_, [&](uint64_t i) { // Check for duplicate in adjacent cells bool found_dup = true; for (unsigned d = 0; d < dim_num; ++d) { @@ -874,15 +868,14 @@ Status GlobalOrderWriter::prepare_full_tiles( } auto num = buffers_.size(); - auto status = - parallel_for(storage_manager_->compute_tp(), 0, num, [&](uint64_t i) { - auto buff_it = buffers_.begin(); - std::advance(buff_it, i); - const auto& name = buff_it->first; - RETURN_CANCEL_OR_ERROR( - prepare_full_tiles(name, coord_dups, &tiles->at(name))); - return Status::Ok(); - }); + auto status = parallel_for(&resources_.compute_tp(), 0, num, [&](uint64_t i) { + auto buff_it = buffers_.begin(); + std::advance(buff_it, i); + const auto& name = buff_it->first; + RETURN_CANCEL_OR_ERROR( + prepare_full_tiles(name, coord_dups, &tiles->at(name))); + return Status::Ok(); + }); RETURN_NOT_OK(status); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 657c9db6674e..a09b08150aea 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -196,7 +196,7 @@ Status OrderedWriter::ordered_write() { // Prepare, filter and write tiles for all attributes auto attr_num = buffers_.size(); - auto compute_tp = storage_manager_->compute_tp(); + auto* compute_tp = &resources_.compute_tp(); auto thread_num = compute_tp->concurrency_level(); std::unordered_map> tiles; for (const auto& buff : buffers_) { @@ -334,7 +334,7 @@ Status OrderedWriter::prepare_filter_and_write_tiles( { auto timer_se = stats_->start_timer("prepare_and_filter_tiles"); auto st = parallel_for( - storage_manager_->compute_tp(), 0, batch_size, [&](uint64_t i) { + &resources_.compute_tp(), 0, batch_size, [&](uint64_t i) { // Prepare and filter tiles auto& writer_tile = tile_batches[b][i]; RETURN_NOT_OK( diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 509fdd1dcaa6..1c058cf3ae83 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -217,10 +217,7 @@ Status UnorderedWriter::check_coord_dups() const { } auto status = parallel_for( - storage_manager_->compute_tp(), - 1, - coords_info_.coords_num_, - [&](uint64_t i) { + &resources_.compute_tp(), 1, coords_info_.coords_num_, [&](uint64_t i) { // Check for duplicate in adjacent cells bool found_dup = true; for (unsigned d = 0; d < dim_num; ++d) { @@ -310,10 +307,7 @@ Status UnorderedWriter::compute_coord_dups() { std::mutex mtx; auto status = parallel_for( - storage_manager_->compute_tp(), - 1, - coords_info_.coords_num_, - [&](uint64_t i) { + &resources_.compute_tp(), 1, coords_info_.coords_num_, [&](uint64_t i) { // Check for duplicate in adjacent cells bool found_dup = true; for (unsigned d = 0; d < dim_num; ++d) { @@ -389,8 +383,8 @@ Status UnorderedWriter::prepare_tiles( } // Prepare tiles for all attributes and coordinates - auto status = parallel_for( - storage_manager_->compute_tp(), 0, tiles->size(), [&](uint64_t i) { + auto status = + parallel_for(&resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto tiles_it = tiles->begin(); std::advance(tiles_it, i); RETURN_CANCEL_OR_ERROR( @@ -621,7 +615,7 @@ Status UnorderedWriter::sort_coords() { DomainBuffersView domain_buffs{array_schema_, buffers_}; if (cell_order != Layout::HILBERT) { // Row- or col-major parallel_sort( - storage_manager_->compute_tp(), + &resources_.compute_tp(), cell_pos_.begin(), cell_pos_.end(), GlobalCmpQB(domain, domain_buffs)); @@ -629,7 +623,7 @@ Status UnorderedWriter::sort_coords() { std::vector hilbert_values(coords_info_.coords_num_); RETURN_NOT_OK(calculate_hilbert_values(domain_buffs, hilbert_values)); parallel_sort( - storage_manager_->compute_tp(), + &resources_.compute_tp(), cell_pos_.begin(), cell_pos_.end(), HilbertCmpQB(domain, domain_buffs, hilbert_values)); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 2f3a2a5fa764..04080324a950 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -333,10 +333,7 @@ Status WriterBase::calculate_hilbert_values( // Calculate Hilbert values in parallel assert(hilbert_values.size() >= coords_info_.coords_num_); auto status = parallel_for( - storage_manager_->compute_tp(), - 0, - coords_info_.coords_num_, - [&](uint64_t c) { + &resources_.compute_tp(), 0, coords_info_.coords_num_, [&](uint64_t c) { std::vector coords(dim_num); for (uint32_t d = 0; d < dim_num; ++d) { auto dim{array_schema_.dimension_ptr(d)}; @@ -427,7 +424,7 @@ Status WriterBase::check_coord_oob() const { // Check if all coordinates fall in the domain in parallel auto status = parallel_for_2d( - storage_manager_->compute_tp(), + &resources_.compute_tp(), 0, coords_info_.coords_num_, 0, @@ -647,8 +644,8 @@ std::vector WriterBase::compute_mbrs( // Compute MBRs std::vector mbrs(tile_num); - auto status = parallel_for( - storage_manager_->compute_tp(), 0, tile_num, [&](uint64_t i) { + auto status = + parallel_for(&resources_.compute_tp(), 0, tile_num, [&](uint64_t i) { mbrs[i].resize(dim_num); std::vector data(dim_num); for (unsigned d = 0; d < dim_num; ++d) { @@ -687,10 +684,7 @@ void WriterBase::set_coords_metadata( } auto status = parallel_for( - storage_manager_->compute_tp(), - start_tile_idx, - end_tile_idx, - [&](uint64_t i) { + &resources_.compute_tp(), start_tile_idx, end_tile_idx, [&](uint64_t i) { meta->set_mbr(i - start_tile_idx, mbrs[i]); return Status::Ok(); }); @@ -706,7 +700,7 @@ void WriterBase::set_coords_metadata( Status WriterBase::compute_tiles_metadata( uint64_t tile_num, tdb::pmr::unordered_map& tiles) const { - auto compute_tp = storage_manager_->compute_tp(); + auto* compute_tp = &resources_.compute_tp(); // Parallelize over attributes? if (tiles.size() > tile_num) { @@ -811,8 +805,8 @@ Status WriterBase::create_fragment( Status WriterBase::filter_tiles( tdb::pmr::unordered_map* tiles) { auto timer_se = stats_->start_timer("filter_tiles"); - auto status = parallel_for( - storage_manager_->compute_tp(), 0, tiles->size(), [&](uint64_t i) { + auto status = + parallel_for(&resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto tiles_it = tiles->begin(); std::advance(tiles_it, i); RETURN_CANCEL_OR_ERROR( @@ -848,8 +842,8 @@ Status WriterBase::filter_tiles( } // For fixed size, process everything, for var size, everything minus offsets. - auto status = parallel_for( - storage_manager_->compute_tp(), 0, args.size(), [&](uint64_t i) { + auto status = + parallel_for(&resources_.compute_tp(), 0, args.size(), [&](uint64_t i) { const auto& [tile, offset_tile, contains_offsets, is_nullable] = args[i]; RETURN_NOT_OK(filter_tile( @@ -861,7 +855,7 @@ Status WriterBase::filter_tiles( // Process offsets for var size. if (var_size) { auto status = parallel_for( - storage_manager_->compute_tp(), 0, tiles->size(), [&](uint64_t i) { + &resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto& tile = (*tiles)[i]; RETURN_NOT_OK( filter_tile(name, &tile.offset_tile(), nullptr, true, false)); @@ -914,11 +908,7 @@ Status WriterBase::filter_tile( assert(!tile->filtered()); RETURN_NOT_OK(filters.run_forward( - stats_, - tile, - offsets_tile, - storage_manager_->compute_tp(), - use_chunking)); + stats_, tile, offsets_tile, &resources_.compute_tp(), use_chunking)); assert(tile->filtered()); return Status::Ok(); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 71616729d18d..f2872221289d 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -395,7 +395,7 @@ Status StorageManagerCanonical::array_upgrade_version( Status StorageManagerCanonical::async_push_query(Query* query) { cancelable_tasks_.execute( - compute_tp(), + &resources_.compute_tp(), [this, query]() { // Process query. Status st = query_submit(query); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index d96f8f2a202a..bc293397da0b 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -78,7 +78,6 @@ class Metadata; class MemoryTracker; class Query; class RestClient; -class VFS; enum class EncryptionType : uint8_t; enum class ObjectType : uint8_t; @@ -218,11 +217,6 @@ class StorageManagerCanonical { */ Status group_create(const std::string& group); - /** Returns the thread pool for compute-bound tasks. */ - [[nodiscard]] inline ThreadPool* compute_tp() const { - return &(resources_.compute_tp()); - } - /** Returns the thread pool for io-bound tasks. */ [[nodiscard]] inline ThreadPool* io_tp() const { return &(resources_.io_tp()); From f6809293d55406ca20703b54a8cfcc48e316ad68 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 23 May 2024 07:19:55 -0400 Subject: [PATCH 368/456] Remove StorageManager::io_tp(). (#4998) Remove `StorageManager::io_tp()`. [sc-47814] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::io_tp()`. --------- Co-authored-by: Luc Rancourt --- .../storage_manager_override.h | 13 +++---------- tiledb/sm/query/readers/filtered_data.h | 14 ++++---------- tiledb/sm/query/readers/reader_base.cc | 2 +- tiledb/sm/query/writers/ordered_writer.cc | 2 +- tiledb/sm/query/writers/writer_base.cc | 8 ++++---- .../sm/storage_manager/storage_manager_canonical.h | 5 ----- 6 files changed, 13 insertions(+), 31 deletions(-) diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 666146165e2c..6dbddcc07d33 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -49,25 +49,18 @@ class Config; class VFS; class StorageManagerStub { - ContextResources& resources_; Config config_; public: static constexpr bool is_overriding_class = true; StorageManagerStub( - ContextResources& resources, - std::shared_ptr, - const Config& config) - : resources_(resources) - , config_(config) { + ContextResources&, std::shared_ptr, const Config& config) + : config_(config) { } - inline common::ThreadPool* io_tp() { - return &resources_.io_tp(); - } inline Status cancel_all_tasks() { return Status{}; - }; + } inline Status group_create(const std::string&) { throw std::logic_error( "StorageManagerStub does not support group creation"); diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index 7642e87ce548..db85d2b1c050 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -201,7 +201,6 @@ class FilteredData { , fragment_metadata_(fragment_metadata) , var_sized_(var_sized) , nullable_(nullable) - , storage_manager_(storage_manager) , read_tasks_(read_tasks) { if (result_tiles.size() == 0) { return; @@ -395,12 +394,10 @@ class FilteredData { auto data{block.data()}; auto size{block.size()}; URI uri{file_uri(fragment_metadata_[block.frag_idx()].get(), type)}; - auto task = - storage_manager_->io_tp()->execute([this, offset, data, size, uri]() { - throw_if_not_ok( - resources_.vfs().read(uri, offset, data, size, false)); - return Status::Ok(); - }); + auto task = resources_.io_tp().execute([this, offset, data, size, uri]() { + throw_if_not_ok(resources_.vfs().read(uri, offset, data, size, false)); + return Status::Ok(); + }); read_tasks_.push_back(std::move(task)); } @@ -638,9 +635,6 @@ class FilteredData { /** Is the attribute nullable? */ const bool nullable_; - /** Storage manager. */ - StorageManager* storage_manager_; - /** Read tasks. */ std::vector& read_tasks_; }; diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 4c44607ee041..27b3d15f4658 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -703,7 +703,7 @@ std::list ReaderBase::read_tiles( stats_->add_counter("num_tiles_read", num_tiles_read); // Wait for the read tasks to finish. - auto statuses{storage_manager_->io_tp()->wait_all_status(read_tasks)}; + auto statuses{resources_.io_tp().wait_all_status(read_tasks)}; for (const auto& st : statuses) { throw_if_not_ok(st); } diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index a09b08150aea..2139bcce3af3 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -364,7 +364,7 @@ Status OrderedWriter::prepare_filter_and_write_tiles( RETURN_NOT_OK(write_task->get()); } - write_task = storage_manager_->io_tp()->execute([&, b, frag_tile_id]() { + write_task = resources_.io_tp().execute([&, b, frag_tile_id]() { close_files = (b == batch_num - 1); RETURN_NOT_OK(write_tiles( 0, diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 04080324a950..aeb122a1164c 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -606,8 +606,8 @@ Status WriterBase::close_files(shared_ptr meta) const { } } - auto status = parallel_for( - storage_manager_->io_tp(), 0, file_uris.size(), [&](uint64_t i) { + auto status = + parallel_for(&resources_.io_tp(), 0, file_uris.size(), [&](uint64_t i) { const auto& file_uri = file_uris[i]; if (layout_ == Layout::GLOBAL_ORDER && remote_query()) { resources_.vfs().finalize_and_close_file(file_uri); @@ -1049,7 +1049,7 @@ Status WriterBase::write_tiles( std::vector tasks; for (auto& it : *tiles) { - tasks.push_back(storage_manager_->io_tp()->execute([&, this]() { + tasks.push_back(resources_.io_tp().execute([&, this]() { auto& attr = it.first; auto& tiles = it.second; RETURN_CANCEL_OR_ERROR(write_tiles( @@ -1073,7 +1073,7 @@ Status WriterBase::write_tiles( } // Wait for writes and check all statuses - auto statuses = storage_manager_->io_tp()->wait_all_status(tasks); + auto statuses = resources_.io_tp().wait_all_status(tasks); for (auto& st : statuses) RETURN_NOT_OK(st); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index bc293397da0b..dff81127d4d5 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -217,11 +217,6 @@ class StorageManagerCanonical { */ Status group_create(const std::string& group); - /** Returns the thread pool for io-bound tasks. */ - [[nodiscard]] inline ThreadPool* io_tp() const { - return &(resources_.io_tp()); - } - /** * If the storage manager was configured with a REST server, return the * client instance. Else, return nullptr. From 1b459b50f7b1385533aaace2ad52eb7a43f65010 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 23 May 2024 07:21:12 -0400 Subject: [PATCH 369/456] Migrate APIs out of StorageManager: group_close_for_writes. (#4991) Migrate APIs out of `StorageManager`: `group_close_for_writes`. [sc-46726] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `group_close_for_writes`. --- tiledb/sm/group/group.cc | 20 ++++++++++++++++++- tiledb/sm/group/group.h | 7 +++++++ tiledb/sm/storage_manager/storage_manager.cc | 18 ----------------- .../storage_manager_canonical.h | 8 -------- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 3cf1403838d9..4ba01eea7560 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -215,6 +215,24 @@ Status Group::open(QueryType query_type) { return Group::open(query_type, timestamp_start_, timestamp_end_); } +Status Group::close_for_writes() { + // Flush the group metadata + throw_if_not_ok( + unsafe_metadata()->store(resources_, group_uri(), *encryption_key())); + + // Store any changes required + if (group_details()->is_modified()) { + const URI& group_detail_folder_uri = group_detail_uri(); + auto group_detail_uri = generate_detail_uri(); + throw_if_not_ok(group_details()->store( + resources_, + group_detail_folder_uri, + group_detail_uri, + *encryption_key())); + } + return Status::Ok(); +} + Status Group::close() { // Check if group is open if (!is_open_) @@ -252,7 +270,7 @@ Status Group::close() { query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { try { - throw_if_not_ok(storage_manager_->group_close_for_writes(this)); + throw_if_not_ok(close_for_writes()); } catch (StatusException& exc) { std::string msg = exc.what(); msg += " : Was storage for the group moved or deleted before closing?"; diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index de59cde06b5b..5b1f2598d08e 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -118,6 +118,13 @@ class Group { return Status::Ok(); } + /** + * Closes a group opened for writes. + * + * @return Status + */ + Status close_for_writes(); + /** Closes the group and frees all memory. */ Status close(); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index f2872221289d..8252e6dc63d2 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -132,24 +132,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::group_close_for_writes(Group* group) { - // Flush the group metadata - throw_if_not_ok(group->unsafe_metadata()->store( - resources_, group->group_uri(), *group->encryption_key())); - - // Store any changes required - if (group->group_details()->is_modified()) { - const URI& group_detail_folder_uri = group->group_detail_uri(); - auto group_detail_uri = group->generate_detail_uri(); - throw_if_not_ok(group->group_details()->store( - resources_, - group_detail_folder_uri, - group_detail_uri, - *group->encryption_key())); - } - return Status::Ok(); -} - Status StorageManagerCanonical::array_create( const URI& array_uri, const shared_ptr& array_schema, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index dff81127d4d5..2e9fa3baad99 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -145,14 +145,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Closes an group opened for writes. - * - * @param group The group to be closed. - * @return Status - */ - Status group_close_for_writes(tiledb::sm::Group* group); - /** * Creates a TileDB array storing its schema. * From 51ba5af04b5153602d32c6258784689b2dd43c8c Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 23 May 2024 04:22:32 -0700 Subject: [PATCH 370/456] Implemented offset/length conversion functions for external sort. (#4948) Implemented functions to convert back and forth from offset arrays to length arrays, supporting both the tiledb format and the arrow format. [sc-46422] --- TYPE: IMPROVEMENT DESC: Implemented offset/length conversion functions for external sort. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/common/test/CMakeLists.txt | 2 +- tiledb/common/test/unit_var_length_util.cc | 67 ++++++++++++ tiledb/common/var_length_util.h | 118 +++++++++++++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 tiledb/common/test/unit_var_length_util.cc create mode 100644 tiledb/common/var_length_util.h diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index a7b3d4bf94e9..b696b9613d1a 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -42,7 +42,7 @@ commence(unit_test memory_tracker_types) conclude(unit_test) commence(unit_test common_utils) - this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_sort.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_view.cc unit_zip_view.cc) + this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_sort.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_util.cc unit_var_length_view.cc unit_zip_view.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_var_length_util.cc b/tiledb/common/test/unit_var_length_util.cc new file mode 100644 index 000000000000..a9489292c03f --- /dev/null +++ b/tiledb/common/test/unit_var_length_util.cc @@ -0,0 +1,67 @@ +/** + * @file unit_var_length_util.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the var_length_utils. + */ + +#include +#include +#include +#include "../var_length_util.h" + +TEST_CASE("var_length_uti: Null test", "[var_length_util][null_test]") { + REQUIRE(true); +} + +TEST_CASE("var_length_util: Test to from", "[var_length_util]") { + std::vector lengths{1, 5, 3, 2, 9}; + std::vector tiledb_offsets(size(lengths)); + std::vector arrow_offsets(size(lengths) + 1); + + std::vector expected_lengths{lengths}; + std::vector expected_tiledb_offsets{0, 1, 6, 9, 11}; + std::vector expected_arrow_offsets{0, 1, 6, 9, 11, 20}; + + SECTION("to tiledb offsets") { + lengths_to_offsets(lengths, tiledb_offsets); + CHECK(tiledb_offsets == expected_tiledb_offsets); + } + SECTION("to arrow offsets") { + lengths_to_offsets(lengths, arrow_offsets); + CHECK(arrow_offsets == expected_arrow_offsets); + } + SECTION("from tiledb offsets") { + offsets_to_lengths(expected_tiledb_offsets, lengths, 20); + CHECK(lengths == expected_lengths); + } + SECTION("from arrow offsets") { + offsets_to_lengths(expected_arrow_offsets, lengths); + CHECK(lengths == expected_lengths); + } +} diff --git a/tiledb/common/var_length_util.h b/tiledb/common/var_length_util.h new file mode 100644 index 000000000000..e53f0acad00f --- /dev/null +++ b/tiledb/common/var_length_util.h @@ -0,0 +1,118 @@ +/** + * @file var_length_util.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * This file contains utilities for working with var length data. Notably + * - conversion routing between a vector of offsets and a vector of lengths + * + * From lengths to offsets is an inclusive_scan: + * std::inclusive_scan(begin(bar), end(bar), begin(baz)+1); + * + * From offsets to lengths is an adjacent_difference: + * std::adjacent_difference(begin(baz)+1, end(baz), begin(bat)); + * + * Note that we have to special case the arrow offset format vs the tiledb + * offset format. + */ + +#ifndef TILEDB_VAR_LENGTH_UTIL_H +#define TILEDB_VAR_LENGTH_UTIL_H + +#include +#include +#include + +/** + * @brief Convert a vector of lengths to a vector of offsets, in place. + * Arrow format. In the arrow format, the length of the offsets array is + * one more than the length of the length array + * the first element is always 0, and the + * last element is the total length of the data. + * @tparam R + * @tparam S + * @param offsets + * @param lengths + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range S> +void lengths_to_offsets(const R& lengths, S& offsets) { + if (std::size(offsets) == std::size(lengths) + 1) { + std::inclusive_scan( + std::begin(lengths), std::end(lengths), std::begin(offsets) + 1); + } else if (std::size(offsets) == std::size(lengths)) { + std::inclusive_scan( + std::begin(lengths), std::end(lengths) - 1, std::begin(offsets) + 1); + } else { + throw std::runtime_error("Invalid lengths and offsets sizes"); + } +} + +/** + * @brief Convert a vector of offsets to a vector of lengths, in place. + * Arrow format. In the arrow format, the length of the offsets array is + * one more than the length of the length array + * the first element is always 0, and the + * last element is the total length of the data. + * @tparam R + * @tparam S + * @param offsets + * @param lengths + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range S> +void offsets_to_lengths(const R& offsets, S& lengths) { + assert(std::size(offsets) == std::size(lengths) + 1); + std::adjacent_difference( + std::begin(offsets) + 1, std::end(offsets), std::begin(lengths)); +} + +/** + * @brief Convert a vector of offsets to a vector of lengths, in place. + * Arrow format. In the arrow format, the length of the offsets array is + * one more than the length of the length array + * the first element is always 0, and the + * last element is the total length of the data. + * @tparam R + * @tparam S + * @param offsets + * @param lengths + * @param total_length aka the final offset + */ +template < + std::ranges::random_access_range R, + std::ranges::random_access_range S> +void offsets_to_lengths(const R& offsets, S& lengths, size_t total_length) { + assert(std::size(offsets) == std::size(lengths)); + std::adjacent_difference( + std::begin(offsets) + 1, std::end(offsets), std::begin(lengths)); + lengths.back() = + static_cast(total_length) - offsets.back(); +} + +#endif // TILEDB_VAR_LENGTH_UTIL_H From 6e7ee00c9208e49c981934bd4e101bfa15fa564d Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Fri, 24 May 2024 01:16:52 -0600 Subject: [PATCH 371/456] Bring object library `array_schema` into conformance (#5005) Previous changes have fixed the underlying problem. This PR removes the subversion of the standalone compile test for the library. --- TYPE: NO_HISTORY DESC: Bring object library `array_schema` into conformance. --- tiledb/sm/array_schema/CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 362393d2a625..598b640d1235 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -74,10 +74,4 @@ commence(object_library array_schema) attribute domain enumeration fragment time uri_format vfs) conclude(object_library) -# This is linked outside the object_library scope as an active subversion of -# the principle that object libraries must compile as standalone modules. As -# long as this directive is necessary to have to the compile test pass, we -# cannot consider `array_schema` as a properly-constructed object library. -target_link_libraries(compile_array_schema PRIVATE generic_tile_io) - add_test_subdirectory() From 409f188d545ac794a8d14f81e9f3bedb75e7ec11 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Mon, 27 May 2024 12:23:12 +0300 Subject: [PATCH 372/456] Update budget tests to fix windows nightly builds. (#5003) Fixes the nightly tests in windows after #4929. The memory budgets tests were failing because structures take more memory in ASAN debug. Adjusting the budgeting values so they pass in all build flavors. Successful run here: https://github.com/TileDB-Inc/TileDB/actions/runs/9209028209. --- TYPE: NO_HISTORY DESC: Update budget tests to fix windows nightly builds. --- ...it-cppapi-consolidation-with-timestamps.cc | 8 ++++---- test/src/unit-dense-reader.cc | 13 ++++++------ test/src/unit-sparse-global-order-reader.cc | 14 ++++++------- .../unit-sparse-unordered-with-dups-reader.cc | 20 +++++++++++++++---- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index b677a6c945fc..bb4ac9a90cb9 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -672,8 +672,8 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "20000"); - cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.22"); + cfg.set("sm.mem.total_budget", "30000"); + cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.15"); ctx_ = Context(cfg); std::string stats; @@ -722,8 +722,8 @@ TEST_CASE_METHOD( // Will only allow to load two tiles out of 3. Config cfg; - cfg.set("sm.mem.total_budget", "20000"); - cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.22"); + cfg.set("sm.mem.total_budget", "30000"); + cfg.set("sm.mem.reader.sparse_global_order.ratio_coords", "0.15"); ctx_ = Context(cfg); std::string stats; diff --git a/test/src/unit-dense-reader.cc b/test/src/unit-dense-reader.cc index f5f027c66a25..f72e45564a72 100644 --- a/test/src/unit-dense-reader.cc +++ b/test/src/unit-dense-reader.cc @@ -1126,8 +1126,8 @@ TEST_CASE_METHOD( // Each var tiles are 91 and 100 bytes respectively, this will only allow to // load one as the budget is split across two potential reads. Fixed tiles are // both 40 so they both fit in the budget. - total_budget_ = "1200"; - tile_upper_memory_limit_ = "500"; + total_budget_ = "2500"; + tile_upper_memory_limit_ = "200"; update_config(); // Try to read. @@ -1194,6 +1194,9 @@ TEST_CASE_METHOD( CHECK(a2_offsets_r_size == a2_offsets_size); CHECK(!std::memcmp(a2_offsets.data(), a2_offsets_r, a2_offsets_size)); + total_budget_ = "1100"; + update_config(); + // Now read with QC set for a1 and a2, should fail. read_fixed_strings( subarray, @@ -1243,8 +1246,7 @@ TEST_CASE_METHOD( // Each var tiles are 91 and 100 bytes respectively, this will only allow to // load one as the budget is split across two potential reads. Fixed tiles are // both 40 so they both fit in the budget. - total_budget_ = "1150"; - tile_upper_memory_limit_ = "500"; + total_budget_ = "1100"; update_config(); // Try to read. @@ -1293,8 +1295,7 @@ TEST_CASE_METHOD( 0, false, true, - "DenseReader: Cannot process a single tile because of query " - "condition, increase memory budget"); + "DenseReader: Cannot process a single tile, increase memory budget"); // Now read with QC set for a1 and a2, should fail. read_fixed_strings( diff --git a/test/src/unit-sparse-global-order-reader.cc b/test/src/unit-sparse-global-order-reader.cc index 0a6bf9e75f03..83c1e937572b 100644 --- a/test/src/unit-sparse-global-order-reader.cc +++ b/test/src/unit-sparse-global-order-reader.cc @@ -790,8 +790,8 @@ TEST_CASE_METHOD( // Two result tile (2 * (~3000 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "30000"; - ratio_coords_ = "0.12"; + total_budget_ = "35000"; + ratio_coords_ = "0.11"; update_config(); tiledb_array_t* array = nullptr; @@ -870,7 +870,7 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); // One result tile (8 + ~440) will be bigger than the budget (400). - total_budget_ = "13000"; + total_budget_ = "19000"; ratio_coords_ = "0.04"; update_config(); @@ -1207,7 +1207,7 @@ TEST_CASE( "Sparse global order reader: attribute copy memory limit", "[sparse-global-order][attribute-copy][memory-limit][rest]") { Config config; - config["sm.mem.total_budget"] = "15000"; + config["sm.mem.total_budget"] = "20000"; VFSTestSetup vfs_test_setup(config.ptr().get()); std::string array_name = vfs_test_setup.array_uri("test_sparse_global_order"); auto ctx = vfs_test_setup.ctx(); @@ -1312,10 +1312,10 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); } - // Two result tile (2 * (~2600 + 8) will be bigger than the per fragment + // Two result tile (2 * (~4000 + 8) will be bigger than the per fragment // budget (1000). - total_budget_ = "26000"; - ratio_coords_ = "0.30"; + total_budget_ = "40000"; + ratio_coords_ = "0.22"; update_config(); tiledb_array_t* array = nullptr; diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 51db6e5b93b1..56f55ffb7d2b 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -61,6 +61,7 @@ struct CSparseUnorderedWithDupsFx { std::string array_name_; const char* ARRAY_NAME = "test_sparse_unordered_with_dups"; std::string total_budget_; + std::string tile_upper_memory_limit_; std::string ratio_tile_ranges_; std::string ratio_array_data_; std::string ratio_coords_; @@ -166,6 +167,16 @@ void CSparseUnorderedWithDupsFx::update_config() { TILEDB_OK); REQUIRE(error == nullptr); + if (tile_upper_memory_limit_ != "") { + REQUIRE( + tiledb_config_set( + config, + "sm.mem.tile_upper_memory_limit", + tile_upper_memory_limit_.c_str(), + &error) == TILEDB_OK); + REQUIRE(error == nullptr); + } + REQUIRE( tiledb_config_set( config, @@ -932,7 +943,8 @@ TEST_CASE_METHOD( uint64_t data2_size = data.size() * sizeof(int); write_1d_fragment(coords2.data(), &coords2_size, data2.data(), &data2_size); - total_budget_ = "1500000"; + total_budget_ = "1900000"; + tile_upper_memory_limit_ = "100000"; ratio_array_data_ = set_subarray ? "0.003" : "0.002"; partial_tile_offsets_loading_ = "true"; update_config(); @@ -1020,8 +1032,8 @@ TEST_CASE_METHOD( } // Two result tile (2 * ~1208) will be bigger than the budget (1500). - total_budget_ = "25500"; - ratio_coords_ = "0.06"; + total_budget_ = "40000"; + ratio_coords_ = "0.04"; update_config(); tiledb_array_t* array = nullptr; @@ -1099,7 +1111,7 @@ TEST_CASE_METHOD( write_1d_fragment(coords, &coords_size, data, &data_size); // One result tile (~505) will be larger than leftover memory. - total_budget_ = "1500"; + total_budget_ = "1800"; ratio_array_data_ = "0.99"; ratio_coords_ = "0.0005"; update_config(); From 0098fd358afc942243efc4b429345dd2a4559283 Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Mon, 27 May 2024 12:53:08 +0300 Subject: [PATCH 373/456] Fix performance regression in group::dump() for `tiledb://` arrays (#5002) This [fix](https://github.com/TileDB-Inc/TileDB/pull/4582) to [38499](https://app.shortcut.com/tiledb-inc/story/38499/recursively-dumping-a-group-fails-when-a-subgroup-does-not-exist) , introduced a performance regression to `Group::dump` for `tiledb://` arrays, since now the method calls `object_type` for every member in a for loop, where every `object_type` call initiates two REST requests for `tiledb://` arrays (one for `is_array` and one for `is_group`). The fix in this PR is to let group open fail when the group is not there instead, check on the return value and log that the array doesn't exist. --- TYPE: BUG DESC: Fix performance regression in group::dump() for "tiledb://" arrays --- test/src/unit-capi-group.cc | 10 ++++++---- tiledb/sm/group/group.cc | 35 +++++++++++++++-------------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/test/src/unit-capi-group.cc b/test/src/unit-capi-group.cc index ef3afe1b70da..ff2ddca6bbab 100644 --- a/test/src/unit-capi-group.cc +++ b/test/src/unit-capi-group.cc @@ -1464,16 +1464,18 @@ TEST_CASE_METHOD( remove_temp_dir(temp_dir); } -TEST_CASE_METHOD(GroupFx, "C API: Group, dump", "[capi][group][dump]") { +TEST_CASE_METHOD( + GroupFx, + "C API: Group, dump", + "[capi][group][dump][rest-fails][sc-48099]") { // Create and open group in write mode - // TODO: refactor for each supported FS. std::string temp_dir = fs_vec_[0]->temp_dir(); create_temp_dir(temp_dir); - tiledb::sm::URI group1_uri(temp_dir + "group1"); + std::string group1_uri = vfs_array_uri(fs_vec_[0], temp_dir + "group1"); REQUIRE(tiledb_group_create(ctx_, group1_uri.c_str()) == TILEDB_OK); - tiledb::sm::URI group2_uri(temp_dir + "group2"); + std::string group2_uri = vfs_array_uri(fs_vec_[0], temp_dir + "group2"); REQUIRE(tiledb_group_create(ctx_, group2_uri.c_str()) == TILEDB_OK); tiledb_group_t* group1; diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 4ba01eea7560..77bbb6aba657 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -705,31 +705,26 @@ std::string Group::dump( for (const auto& member_entry : members()) { const auto& it = member_entry.second; - URI member_uri = it->uri(); - if (it->relative()) { - member_uri = group_uri_.join_path(it->uri().to_string()); - } - ss << "|" << indent << l_indent << " " << *it; - bool do_recurse = false; if (it->type() == ObjectType::GROUP && recursive) { - // Before listing the group's members, check if the group exists in - // storage. If it does not, leave a message in the string. - ObjectType member_type = ObjectType::INVALID; - throw_if_not_ok(storage_manager_->object_type(member_uri, &member_type)); - if (member_type == ObjectType::GROUP) { - do_recurse = true; - } else { - ss << " (does not exist)"; + URI member_uri = it->uri(); + if (it->relative()) { + member_uri = group_uri_.join_path(it->uri().to_string()); } - } - ss << std::endl; - if (do_recurse) { Group group_rec(resources_, member_uri, storage_manager_); - throw_if_not_ok(group_rec.open(QueryType::READ)); - ss << group_rec.dump(indent_size, num_indents + 2, recursive, false); - throw_if_not_ok(group_rec.close()); + auto st = group_rec.open(QueryType::READ); + if (st.ok()) { + ss << std::endl; + ss << group_rec.dump(indent_size, num_indents + 2, recursive, false); + throw_if_not_ok(group_rec.close()); + } else { + // If the group no longer exists in storage it will be listed but we + // won't be able to dump its members + ss << " (does not exist)" << std::endl; + } + } else { + ss << std::endl; } } From f27e40eabfc78adf31b7745969c25983dbe2d3e8 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 27 May 2024 14:19:55 +0300 Subject: [PATCH 374/456] Fix build errors where the fmt headers could not be found for spdlog. (#5008) Fixes regression introduced in #4881. Fixes #5006. [sc-48092] --- TYPE: BUILD DESC: Fix build errors where the fmt headers could not be found for spdlog. --- tiledb/common/logger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/common/logger.h b/tiledb/common/logger.h index f0de1df2a6c4..62fc53a37162 100644 --- a/tiledb/common/logger.h +++ b/tiledb/common/logger.h @@ -36,7 +36,7 @@ #ifndef TILEDB_LOGGER_H #define TILEDB_LOGGER_H -#include +#include #include #include From ae23511aff9c2afd377387ced29cdf2332abcb92 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 27 May 2024 07:23:08 -0400 Subject: [PATCH 375/456] Update GroupDirectory constructor to use references instead of pointers. (#4999) Update `GroupDirectory` constructor to use references instead of pointers. [sc-47557] --- TYPE: NO_HISTORY DESC: Update `GroupDirectory` constructor to use references instead of pointers. --- .../consolidator/group_meta_consolidator.cc | 17 +++----- tiledb/sm/group/group.cc | 42 +++++++------------ tiledb/sm/group/group_directory.cc | 42 +++++++++++-------- tiledb/sm/group/group_directory.h | 16 +++---- tiledb/sm/group/group_member.cc | 23 ++++++---- tiledb/sm/group/group_member.h | 17 ++------ 6 files changed, 70 insertions(+), 87 deletions(-) diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index cc4d9b3920b0..b5121b29a15b 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -114,17 +114,12 @@ void GroupMetaConsolidator::vacuum(const char* group_name) { // Get the group metadata URIs and vacuum file URIs to be vacuumed auto& vfs = resources_.vfs(); auto& compute_tp = resources_.compute_tp(); - GroupDirectory group_dir; - try { - group_dir = GroupDirectory( - &vfs, - &compute_tp, - URI(group_name), - 0, - std::numeric_limits::max()); - } catch (const std::logic_error& le) { - throw Status_GroupDirectoryError(le.what()); - } + GroupDirectory group_dir( + vfs, + compute_tp, + URI(group_name), + 0, + std::numeric_limits::max()); // Delete the group metadata and vacuum files vfs.remove_files(&compute_tp, group_dir.group_meta_uris_to_vacuum()); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 77bbb6aba657..d274a03e2242 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -160,35 +160,23 @@ Status Group::open( RETURN_NOT_OK(rest_client->post_group_from_rest(group_uri_, this)); } else if (query_type == QueryType::READ) { - try { - group_dir_ = make_shared( - HERE(), - &resources_.vfs(), - &resources_.compute_tp(), - group_uri_, - timestamp_start, - timestamp_end); - } catch (const std::logic_error& le) { - return Status_GroupDirectoryError(le.what()); - } - + group_dir_ = make_shared( + HERE(), + resources_.vfs(), + resources_.compute_tp(), + group_uri_, + timestamp_start, + timestamp_end); group_open_for_reads(); } else { - try { - group_dir_ = make_shared( - HERE(), - &resources_.vfs(), - &resources_.compute_tp(), - group_uri_, - timestamp_start, - (timestamp_end != 0) ? timestamp_end : - utils::time::timestamp_now_ms()); - } catch (const std::logic_error& le) { - return Status_GroupDirectoryError(le.what()); - } - + group_dir_ = make_shared( + HERE(), + resources_.vfs(), + resources_.compute_tp(), + group_uri_, + timestamp_start, + (timestamp_end != 0) ? timestamp_end : utils::time::timestamp_now_ms()); group_open_for_writes(); - metadata_.reset(timestamp_end); } @@ -355,7 +343,7 @@ void Group::delete_group(const URI& uri, bool recursive) { auto& vfs = resources_.vfs(); auto& compute_tp = resources_.compute_tp(); auto group_dir = GroupDirectory( - &vfs, &compute_tp, uri, 0, std::numeric_limits::max()); + vfs, compute_tp, uri, 0, std::numeric_limits::max()); // Delete the group detail, group metadata and group files vfs.remove_files(&compute_tp, group_dir.group_detail_uris()); diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc index e4107c22d421..3e01e5a1d237 100644 --- a/tiledb/sm/group/group_directory.cc +++ b/tiledb/sm/group/group_directory.cc @@ -43,6 +43,13 @@ using namespace tiledb::common; namespace tiledb::sm { +class GroupDirectoryException : public StatusException { + public: + explicit GroupDirectoryException(const std::string& message) + : StatusException("GroupDirectory", message) { + } +}; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -52,8 +59,8 @@ namespace tiledb::sm { * anywhere in the code at present, though that might change. */ GroupDirectory::GroupDirectory( - VFS* vfs, - ThreadPool* tp, + VFS& vfs, + ThreadPool& tp, const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end, @@ -66,7 +73,7 @@ GroupDirectory::GroupDirectory( , loaded_(false) { auto st = load(); if (!st.ok()) { - throw std::logic_error(st.message()); + throw GroupDirectoryException(st.message()); } } @@ -124,9 +131,9 @@ Status GroupDirectory::load() { // Lists all directories in parallel. Skipping for schema only. // Some processing is also done here for things that don't depend on others. // List (in parallel) the root directory URIs - tasks.emplace_back(tp_->execute([&]() { + tasks.emplace_back(tp_.execute([&]() { auto&& [st, uris] = list_root_dir_uris(); - RETURN_NOT_OK(st); + throw_if_not_ok(st); root_dir_uris = std::move(uris.value()); @@ -134,13 +141,13 @@ Status GroupDirectory::load() { })); // Load (in parallel) the group metadata URIs - tasks.emplace_back(tp_->execute([&]() { return load_group_meta_uris(); })); + tasks.emplace_back(tp_.execute([&]() { return load_group_meta_uris(); })); // Load (in paralell) the group details URIs - tasks.emplace_back(tp_->execute([&] { return load_group_detail_uris(); })); + tasks.emplace_back(tp_.execute([&] { return load_group_detail_uris(); })); // Wait for all tasks to complete - RETURN_NOT_OK(tp_->wait_all(tasks)); + throw_if_not_ok(tp_.wait_all(tasks)); // Error check bool is_group = false; @@ -156,8 +163,7 @@ Status GroupDirectory::load() { } if (!is_group) { - return LOG_STATUS( - Status_GroupDirectoryError("Cannot open group; Group does not exist.")); + throw GroupDirectoryException("Cannot open group; Group does not exist."); } // The URI manager has been loaded successfully @@ -177,7 +183,7 @@ bool GroupDirectory::loaded() const { tuple>> GroupDirectory::list_root_dir_uris() { // List the group directory URIs std::vector group_dir_uris; - RETURN_NOT_OK_TUPLE(vfs_->ls(uri_, &group_dir_uris), nullopt); + RETURN_NOT_OK_TUPLE(vfs_.ls(uri_, &group_dir_uris), nullopt); return {Status::Ok(), group_dir_uris}; } @@ -186,7 +192,7 @@ Status GroupDirectory::load_group_meta_uris() { // Load the URIs in the group metadata directory std::vector group_meta_dir_uris; auto group_meta_uri = uri_.join_path(constants::group_metadata_dir_name); - RETURN_NOT_OK(vfs_->ls(group_meta_uri, &group_meta_dir_uris)); + throw_if_not_ok(vfs_.ls(group_meta_uri, &group_meta_dir_uris)); // Compute and group metadata URIs and the vacuum file URIs to vacuum. auto&& [st1, group_meta_uris_to_vacuum, group_meta_vac_uris_to_vacuum] = @@ -210,12 +216,12 @@ Status GroupDirectory::load_group_detail_uris() { // Load the URIs in the group details directory std::vector group_detail_dir_uris; auto group_detail_uri = uri_.join_path(constants::group_detail_dir_name); - RETURN_NOT_OK(vfs_->ls(group_detail_uri, &group_detail_dir_uris)); + throw_if_not_ok(vfs_.ls(group_detail_uri, &group_detail_dir_uris)); // Compute and group details URIs and the vacuum file URIs to vacuum. auto&& [st1, group_detail_uris_to_vacuum, group_detail_vac_uris_to_vacuum] = compute_uris_to_vacuum(group_detail_dir_uris); - RETURN_NOT_OK(st1); + throw_if_not_ok(st1); group_detail_uris_to_vacuum_ = std::move(group_detail_uris_to_vacuum.value()); group_detail_vac_uris_to_vacuum_ = std::move(group_detail_vac_uris_to_vacuum.value()); @@ -223,7 +229,7 @@ Status GroupDirectory::load_group_detail_uris() { // Compute filtered group details URIs auto&& [st2, group_detail_filtered_uris] = compute_filtered_uris( group_detail_dir_uris, group_detail_uris_to_vacuum_); - RETURN_NOT_OK(st2); + throw_if_not_ok(st2); group_detail_uris_ = std::move(group_detail_filtered_uris.value()); group_detail_dir_uris.clear(); @@ -263,12 +269,12 @@ GroupDirectory::compute_uris_to_vacuum(const std::vector& uris) const { // Also determine which vac files to vacuum std::vector to_vacuum_vec(uris.size(), 0); std::vector to_vacuum_vac_files_vec(vac_files.size(), 0); - auto status = parallel_for(tp_, 0, vac_files.size(), [&](size_t i) { + auto status = parallel_for(&tp_, 0, vac_files.size(), [&](size_t i) { uint64_t size = 0; - RETURN_NOT_OK(vfs_->file_size(vac_files[i], &size)); + throw_if_not_ok(vfs_.file_size(vac_files[i], &size)); std::string names; names.resize(size); - RETURN_NOT_OK(vfs_->read(vac_files[i], 0, &names[0], size)); + throw_if_not_ok(vfs_.read(vac_files[i], 0, &names[0], size)); std::stringstream ss(names); bool vacuum_vac_file = true; for (std::string uri_str; std::getline(ss, uri_str);) { diff --git a/tiledb/sm/group/group_directory.h b/tiledb/sm/group/group_directory.h index a475e83177dd..7dba31ead415 100644 --- a/tiledb/sm/group/group_directory.h +++ b/tiledb/sm/group/group_directory.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -61,13 +61,13 @@ class GroupDirectory { /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ - /** Constructor */ - GroupDirectory() = default; + /** Constructor. */ + GroupDirectory() = delete; /** * Constructor. * - * @param vfs A pointer to a VFS object for all IO. + * @param vfs A reference to a VFS object for all IO. * @param tp A thread pool used for parallelism. * @param uri The URI of the group directory. * @param timestamp_start Only group fragments, metadata, etc. that @@ -81,8 +81,8 @@ class GroupDirectory { * @param mode The mode to load the group directory in. */ GroupDirectory( - VFS* vfs, - ThreadPool* tp, + VFS& vfs, + ThreadPool& tp, const URI& uri, uint64_t timestamp_start, uint64_t timestamp_end, @@ -134,10 +134,10 @@ class GroupDirectory { URI uri_; /** The storage manager. */ - VFS* vfs_; + VFS& vfs_; /** A thread pool used for parallelism. */ - ThreadPool* tp_; + ThreadPool& tp_; /** The URIs of all group files. */ std::vector group_file_uris_; diff --git a/tiledb/sm/group/group_member.cc b/tiledb/sm/group/group_member.cc index e70c2f589be4..8833d9700f9b 100644 --- a/tiledb/sm/group/group_member.cc +++ b/tiledb/sm/group/group_member.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,8 +36,15 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { + +class GroupMemberException : public StatusException { + public: + explicit GroupMemberException(const std::string& message) + : StatusException("GroupMember", message) { + } +}; + GroupMember::GroupMember( const URI& uri, const ObjectType& type, @@ -78,8 +85,7 @@ format_version_t GroupMember::version() const { } void GroupMember::serialize(Serializer&) { - throw StatusException( - Status_GroupMemberError("Invalid call to GroupMember::serialize")); + throw GroupMemberException("Invalid call to GroupMember::serialize"); } shared_ptr GroupMember::deserialize(Deserializer& deserializer) { @@ -90,11 +96,10 @@ shared_ptr GroupMember::deserialize(Deserializer& deserializer) { } else if (version == 2) { return GroupMemberV2::deserialize(deserializer); } - throw StatusException(Status_GroupError( - "Unsupported group member version " + std::to_string(version))); + throw GroupMemberException( + "Unsupported group member version " + std::to_string(version)); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm std::ostream& operator<<( std::ostream& os, const tiledb::sm::GroupMember& group_member) { diff --git a/tiledb/sm/group/group_member.h b/tiledb/sm/group/group_member.h index 19bdb7b2077e..879a9241f6d4 100644 --- a/tiledb/sm/group/group_member.h +++ b/tiledb/sm/group/group_member.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -43,22 +43,12 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { -/** Return a Status_GroupDirectoryError error class Status with a given - * message **/ -inline Status Status_GroupDirectoryError(const std::string& msg) { - return {"[TileDB::GroupDirectory] Error", msg}; -} /** Return an Group error class Status with a given message **/ inline Status Status_GroupError(const std::string& msg) { return {"[TileDB::Group] Error", msg}; } -/** Return an GroupMember error class Status with a given message **/ -inline Status Status_GroupMemberError(const std::string& msg) { - return {"[TileDB::GroupMember] Error", msg}; -} class GroupMember { public: @@ -142,8 +132,7 @@ class GroupMember { /** Is group member deleted from group. */ bool deleted_; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm std::ostream& operator<<( std::ostream& os, const tiledb::sm::GroupMember& group_member); From b227fb644a7cbd8ae1a4a449bcfda75ff9b0874c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 27 May 2024 21:49:51 +0300 Subject: [PATCH 376/456] Fix OS detection in Unix CI et. al. (#5001) [SC-48004](https://app.shortcut.com/tiledb-inc/story/48004) This PR updates the Linux-Mac CI workflow to correctly set the `TILEDB_CI_OS` environment variable, which would enable scripts to perform the appropriate preliminary actions on Linux, such as calling `sudo apt-get update`. Fix failures early in the HDFS nightly run, see https://github.com/TileDB-Inc/TileDB/issues/4869#issuecomment-2126437572 for more information. Initially, that job still failed because [GCC 13 is not available for Ubuntu 22.04](https://launchpad.net/ubuntu/+source/gcc-13), which led to using the beta image for Ubuntu 24.04, which led to errors because Python 3.8 could not be installed, which led to updating it to 3.9 but for all CI jobs. --- TYPE: NO_HISTORY --- .../workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- .github/workflows/ci-linux_mac.yml | 4 ++-- .github/workflows/nightly-test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 484558844bfe..6a6109482200 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -26,7 +26,7 @@ jobs: # Need this for virtualenv and arrow tests if enabled - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.9' - run: | set -e pipefail python -m pip install --upgrade pip virtualenv diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 80b7aaf06d3c..4f1290403097 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -55,7 +55,7 @@ on: env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF TILEDB_CI_BACKEND: ${{ inputs.ci_backend }} - TILEDB_CI_OS: runner.os + TILEDB_CI_OS: ${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'Linux' || 'macOS' }} # Installing Python does not work on manylinux. TILEDB_ARROW_TESTS: ${{ !inputs.manylinux && 'ON' || 'OFF' }} TILEDB_MANYLINUX: ${{ !inputs.manylinux && 'ON' || 'OFF' }} @@ -116,7 +116,7 @@ jobs: uses: actions/setup-python@v4 if: ${{ !inputs.manylinux }} with: - python-version: '3.8' + python-version: '3.9' cache: 'pip' - name: 'Set up Python dependencies' diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 82694da57c66..79ebf4512bdc 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -69,7 +69,7 @@ jobs: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: HDFS - matrix_image: ubuntu-22.04 + matrix_image: ubuntu-24.04 matrix_compiler_cc: 'gcc-13' matrix_compiler_cxx: 'g++-13' timeout: 300 From 907e5d3b3cf9b2ada2e4c6baa901df8c7daa3fa8 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 28 May 2024 00:53:42 +0200 Subject: [PATCH 377/456] Fix group not found issue. (#5013) After merging https://github.com/TileDB-Inc/TileDB/pull/5002 and https://github.com/TileDB-Inc/TileDB/pull/4999, dump for non existing group recursive group stopped working. This is because the error logic in 5002 expected a status code whilst 4999 changes to throw an exception. The fix changes the group code to throw a known exception which is caught at the upper level as a group not found exception. This is much cleaner that using a failed status code, which could mean many other issues. Also, taking advantage to unstatus group.cc/h. --- TYPE: NO_HISTORY DESC: Fix group not found issue. --- tiledb/api/c_api/group/group_api.cc | 13 +-- .../consolidator/group_meta_consolidator.cc | 17 +-- tiledb/sm/group/group.cc | 103 ++++++++---------- tiledb/sm/group/group.h | 26 ++--- tiledb/sm/group/group_directory.cc | 2 +- tiledb/sm/group/group_directory.h | 7 ++ tiledb/sm/group/group_member.h | 5 - tiledb/sm/serialization/group.cc | 2 +- 8 files changed, 78 insertions(+), 97 deletions(-) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 3f2246b4b021..91b66589787e 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -114,8 +114,7 @@ capi_return_t tiledb_group_open( tiledb_group_handle_t* group, tiledb_query_type_t query_type) { ensure_group_is_valid(group); - throw_if_not_ok( - group->group().open(static_cast(query_type))); + group->group().open(static_cast(query_type)); return TILEDB_OK; } @@ -123,7 +122,7 @@ capi_return_t tiledb_group_open( capi_return_t tiledb_group_close(tiledb_group_handle_t* group) { ensure_group_is_valid(group); - throw_if_not_ok(group->group().close()); + group->group().close(); return TILEDB_OK; } @@ -270,8 +269,7 @@ capi_return_t tiledb_group_add_member( name_optional = name; } - throw_if_not_ok( - group->group().mark_member_for_addition(uri, relative, name_optional)); + group->group().mark_member_for_addition(uri, relative, name_optional); return TILEDB_OK; } @@ -281,7 +279,7 @@ capi_return_t tiledb_group_remove_member( ensure_group_is_valid(group); ensure_name_argument_is_valid(name_or_uri); - throw_if_not_ok(group->group().mark_member_for_removal(name_or_uri)); + group->group().mark_member_for_removal(name_or_uri); return TILEDB_OK; } @@ -463,8 +461,7 @@ capi_return_t tiledb_group_get_query_type( ensure_output_pointer_is_valid(query_type); // Get query_type - tiledb::sm::QueryType type; - throw_if_not_ok(group->group().get_query_type(&type)); + tiledb::sm::QueryType type = group->group().get_query_type(); *query_type = static_cast(type); diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index b5121b29a15b..89677e8b8ee6 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -70,15 +70,18 @@ Status GroupMetaConsolidator::consolidate( auto group_uri = URI(group_name); Group group_for_reads( storage_manager_->resources(), group_uri, storage_manager_); - RETURN_NOT_OK(group_for_reads.open( - QueryType::READ, config_.timestamp_start_, config_.timestamp_end_)); + group_for_reads.open( + QueryType::READ, config_.timestamp_start_, config_.timestamp_end_); // Open group for writing Group group_for_writes( storage_manager_->resources(), group_uri, storage_manager_); - RETURN_NOT_OK_ELSE( - group_for_writes.open(QueryType::WRITE), - throw_if_not_ok(group_for_reads.close())); + + try { + group_for_writes.open(QueryType::WRITE); + } catch (...) { + group_for_reads.close(); + } // Copy-assign the read metadata into the metadata of the group for writes auto metadata_r = group_for_reads.metadata(); @@ -95,8 +98,8 @@ Status GroupMetaConsolidator::consolidate( auto data = ss.str(); // Close groups - throw_if_not_ok(group_for_reads.close()); - throw_if_not_ok(group_for_writes.close()); + group_for_reads.close(); + group_for_writes.close(); // Write vacuum file throw_if_not_ok(resources_.vfs().write(vac_uri, data.c_str(), data.size())); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index d274a03e2242..47e04ff83f2c 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -83,16 +83,16 @@ Group::Group( memory_tracker_->set_type(MemoryTrackerType::GROUP); } -Status Group::open( +void Group::open( QueryType query_type, uint64_t timestamp_start, uint64_t timestamp_end) { // Checks if (is_open_) { - return Status_GroupError("Cannot open group; Group already open"); + throw GroupStatusException("Cannot open group; Group already open"); } if (query_type != QueryType::READ && query_type != QueryType::WRITE && query_type != QueryType::MODIFY_EXCLUSIVE) { - return Status_GroupError("Cannot open group; Unsupported query type"); + throw GroupStatusException("Cannot open group; Unsupported query type"); } if (timestamp_end == UINT64_MAX) { @@ -126,7 +126,7 @@ Status Group::open( encryption_type_from_cfg = config_.get("sm.encryption_type", &found); assert(found); auto [st, et] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st); + throw_if_not_ok(st); encryption_type = et.value(); if (!EncryptionKey::is_valid_key_length( @@ -138,11 +138,11 @@ Status Group::open( } if (remote_ && encryption_type != EncryptionType::NO_ENCRYPTION) - return Status_GroupError( + throw GroupStatusException( "Cannot open group; encrypted remote groups are not supported."); // Copy the key bytes. - RETURN_NOT_OK( + throw_if_not_ok( encryption_key_->set_key(encryption_type, encryption_key, key_length)); metadata_.clear(); @@ -151,14 +151,14 @@ Status Group::open( if (remote_) { auto rest_client = storage_manager_->rest_client(); if (rest_client == nullptr) { - return Status_GroupError( + throw GroupStatusException( "Cannot open group; remote group with no REST client."); } // Set initial group details to be deserialized into group_details_ = tdb::make_shared(HERE(), group_uri_); - RETURN_NOT_OK(rest_client->post_group_from_rest(group_uri_, this)); + throw_if_not_ok(rest_client->post_group_from_rest(group_uri_, this)); } else if (query_type == QueryType::READ) { group_dir_ = make_shared( HERE(), @@ -187,23 +187,18 @@ Status Group::open( query_type_ = query_type; is_open_ = true; - - return Status::Ok(); } -Status Group::open(QueryType query_type) { - bool found = false; - RETURN_NOT_OK(config_.get( - "sm.group.timestamp_start", ×tamp_start_, &found)); - assert(found); - RETURN_NOT_OK( - config_.get("sm.group.timestamp_end", ×tamp_end_, &found)); - assert(found); +void Group::open(QueryType query_type) { + timestamp_start_ = + config_.get("sm.group.timestamp_start", Config::must_find); + timestamp_end_ = + config_.get("sm.group.timestamp_end", Config::must_find); - return Group::open(query_type, timestamp_start_, timestamp_end_); + Group::open(query_type, timestamp_start_, timestamp_end_); } -Status Group::close_for_writes() { +void Group::close_for_writes() { // Flush the group metadata throw_if_not_ok( unsafe_metadata()->store(resources_, group_uri(), *encryption_key())); @@ -218,13 +213,12 @@ Status Group::close_for_writes() { group_detail_uri, *encryption_key())); } - return Status::Ok(); } -Status Group::close() { +void Group::close() { // Check if group is open if (!is_open_) - return Status::Ok(); + return; if (remote_) { // Update group metadata for write queries if metadata was written by the @@ -237,28 +231,28 @@ Status Group::close() { metadata_loaded_ = true; auto rest_client = storage_manager_->rest_client(); if (rest_client == nullptr) - return Status_GroupError( + throw GroupStatusException( "Error closing group; remote group with no REST client."); - RETURN_NOT_OK( + throw_if_not_ok( rest_client->put_group_metadata_to_rest(group_uri_, this)); } if (!members_to_modify().empty()) { auto rest_client = storage_manager_->rest_client(); if (rest_client == nullptr) - return Status_GroupError( + throw GroupStatusException( "Error closing group; remote group with no REST client."); - RETURN_NOT_OK(rest_client->patch_group_to_rest(group_uri_, this)); + throw_if_not_ok(rest_client->patch_group_to_rest(group_uri_, this)); } } // Storage manager does not own the group schema for remote groups. } else { if (query_type_ == QueryType::READ) { - RETURN_NOT_OK(close_for_reads()); + close_for_reads(); } else if ( query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { try { - throw_if_not_ok(close_for_writes()); + close_for_writes(); } catch (StatusException& exc) { std::string msg = exc.what(); msg += " : Was storage for the group moved or deleted before closing?"; @@ -271,7 +265,6 @@ Status Group::close() { metadata_loaded_ = false; is_open_ = false; clear(); - return Status::Ok(); } bool Group::is_open() const { @@ -290,15 +283,13 @@ const shared_ptr Group::group_details() const { return group_details_; } -Status Group::get_query_type(QueryType* query_type) const { +QueryType Group::get_query_type() const { // Error if the group is not open - if (!is_open_) - return LOG_STATUS( - Status_GroupError("Cannot get query_type; Group is not open")); - - *query_type = query_type_; + if (!is_open_) { + throw GroupStatusException("Cannot get query_type; Group is not open"); + } - return Status::Ok(); + return query_type_; } void Group::delete_group(const URI& uri, bool recursive) { @@ -334,7 +325,7 @@ void Group::delete_group(const URI& uri, bool recursive) { Array::delete_array(resources_, member_uri); } else if (member->type() == ObjectType::GROUP) { Group group_rec(resources_, member_uri, storage_manager_); - throw_if_not_ok(group_rec.open(QueryType::MODIFY_EXCLUSIVE)); + group_rec.open(QueryType::MODIFY_EXCLUSIVE); group_rec.delete_group(member_uri, true); } } @@ -366,7 +357,7 @@ void Group::delete_group(const URI& uri, bool recursive) { group_details_->clear(); // Close the deleted group - throw_if_not_ok(this->close()); + this->close(); } void Group::delete_metadata(const char* key) { @@ -551,46 +542,44 @@ void Group::delete_member(const shared_ptr group_member) { group_details_->delete_member(group_member); } -Status Group::mark_member_for_addition( +void Group::mark_member_for_addition( const URI& group_member_uri, const bool& relative, std::optional& name) { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - return Status_GroupError("Cannot add member; Group is not open"); + throw GroupStatusException("Cannot add member; Group is not open"); } // Check mode if (query_type_ != QueryType::WRITE && query_type_ != QueryType::MODIFY_EXCLUSIVE) { - return Status_GroupError( + throw GroupStatusException( "Cannot get member; Group was not opened in write or modify_exclusive " "mode"); } group_details_->mark_member_for_addition( group_member_uri, relative, name, storage_manager_); - return Status::Ok(); } -Status Group::mark_member_for_removal(const std::string& name) { +void Group::mark_member_for_removal(const std::string& name) { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - return Status_GroupError( + throw GroupStatusException( "Cannot mark member for removal; Group is not open"); } // Check mode if (query_type_ != QueryType::WRITE && query_type_ != QueryType::MODIFY_EXCLUSIVE) { - return Status_GroupError( + throw GroupStatusException( "Cannot get member; Group was not opened in write or modify_exclusive " "mode"); } group_details_->mark_member_for_removal(name); - return Status::Ok(); } const std::vector>& Group::members_to_modify() const { @@ -628,12 +617,12 @@ uint64_t Group::member_count() const { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - throw Status_GroupError("Cannot get member count; Group is not open"); + throw GroupStatusException("Cannot get member count; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw Status_GroupError( + throw GroupStatusException( "Cannot get member; Group was not opened in read mode"); } @@ -646,12 +635,12 @@ tuple> Group::member_by_index( // Check if group is open if (!is_open_) { - throw Status_GroupError("Cannot get member by index; Group is not open"); + throw GroupStatusException("Cannot get member by index; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw Status_GroupError( + throw GroupStatusException( "Cannot get member; Group was not opened in read mode"); } @@ -664,12 +653,12 @@ Group::member_by_name(const std::string& name) { // Check if group is open if (!is_open_) { - throw Status_GroupError("Cannot get member by name; Group is not open"); + throw GroupStatusException("Cannot get member by name; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw Status_GroupError( + throw GroupStatusException( "Cannot get member; Group was not opened in read mode"); } @@ -701,12 +690,12 @@ std::string Group::dump( } Group group_rec(resources_, member_uri, storage_manager_); - auto st = group_rec.open(QueryType::READ); - if (st.ok()) { + try { + group_rec.open(QueryType::READ); ss << std::endl; ss << group_rec.dump(indent_size, num_indents + 2, recursive, false); - throw_if_not_ok(group_rec.close()); - } else { + group_rec.close(); + } catch (GroupNotFoundException&) { // If the group no longer exists in storage it will be listed but we // won't be able to dump its members ss << " (does not exist)" << std::endl; diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 5b1f2598d08e..0ba2209a1901 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -35,7 +35,6 @@ #include -#include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/enums/query_type.h" @@ -89,11 +88,10 @@ class Group { * * @param query_type The query type. This should always be READ. It * is here only for sanity check. - * @return Status * * @note Applicable only to reads. */ - Status open(QueryType query_type); + void open(QueryType query_type); /** * Opens the group. @@ -102,31 +100,25 @@ class Group { * @param timestamp_start The start timestamp at which to open the group * @param timestamp_end The end timestamp at which to open the group * @param query_type The query type. This should always be READ. It - * @return Status * */ - Status open( + void open( QueryType query_type, uint64_t timestamp_start, uint64_t timestamp_end); /** * Closes a group opened for reads. - * - * @return Status */ - inline Status close_for_reads() { + inline void close_for_reads() { // Closing a group opened for reads does nothing at present. - return Status::Ok(); } /** * Closes a group opened for writes. - * - * @return Status */ - Status close_for_writes(); + void close_for_writes(); /** Closes the group and frees all memory. */ - Status close(); + void close(); /** * Clear a group @@ -280,9 +272,8 @@ class Group { * @param group_member_uri group member uri * @param relative is this URI relative * @param name optional name for member - * @return Status */ - Status mark_member_for_addition( + void mark_member_for_addition( const URI& group_member_uri, const bool& relative, std::optional& name); @@ -293,9 +284,8 @@ class Group { * @param name Name of member to remove. If the member has no name, * this parameter should be set to the URI of the member. In that case, only * the unnamed member with the given URI will be removed. - * @return Status */ - Status mark_member_for_removal(const std::string& name); + void mark_member_for_removal(const std::string& name); /** * Get the vector of members to modify, used in serialization only @@ -371,7 +361,7 @@ class Group { bool is_remote() const; /** Retrieves the query type. Errors if the group is not open. */ - Status get_query_type(QueryType* query_type) const; + QueryType get_query_type() const; /** * Dump a string representation of a group diff --git a/tiledb/sm/group/group_directory.cc b/tiledb/sm/group/group_directory.cc index 3e01e5a1d237..202d82b5d6d6 100644 --- a/tiledb/sm/group/group_directory.cc +++ b/tiledb/sm/group/group_directory.cc @@ -163,7 +163,7 @@ Status GroupDirectory::load() { } if (!is_group) { - throw GroupDirectoryException("Cannot open group; Group does not exist."); + throw GroupNotFoundException("Cannot open group; Group does not exist."); } // The URI manager has been loaded successfully diff --git a/tiledb/sm/group/group_directory.h b/tiledb/sm/group/group_directory.h index 7dba31ead415..39cc386bc92b 100644 --- a/tiledb/sm/group/group_directory.h +++ b/tiledb/sm/group/group_directory.h @@ -44,6 +44,13 @@ using namespace tiledb::common; namespace tiledb::sm { +class GroupNotFoundException : public StatusException { + public: + explicit GroupNotFoundException(const std::string& message) + : StatusException("Group", message) { + } +}; + /** Mode for the GroupDirectory class. */ enum class GroupDirectoryMode { READ, // Read mode. diff --git a/tiledb/sm/group/group_member.h b/tiledb/sm/group/group_member.h index 879a9241f6d4..f166df4bfe38 100644 --- a/tiledb/sm/group/group_member.h +++ b/tiledb/sm/group/group_member.h @@ -45,11 +45,6 @@ using namespace tiledb::common; namespace tiledb::sm { -/** Return an Group error class Status with a given message **/ -inline Status Status_GroupError(const std::string& msg) { - return {"[TileDB::Group] Error", msg}; -} - class GroupMember { public: GroupMember( diff --git a/tiledb/sm/serialization/group.cc b/tiledb/sm/serialization/group.cc index bbb2e4e4b883..cd7dcdb235fc 100644 --- a/tiledb/sm/serialization/group.cc +++ b/tiledb/sm/serialization/group.cc @@ -282,7 +282,7 @@ Status group_update_from_capnp( if (group_update_details_reader.hasMembersToRemove()) { for (auto uri : group_update_details_reader.getMembersToRemove()) { - throw_if_not_ok(group->mark_member_for_removal(uri.cStr())); + group->mark_member_for_removal(uri.cStr()); } } From 892ef9fbcc31234903f89ef36ff38e44658ad968 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 28 May 2024 11:06:25 +0300 Subject: [PATCH 378/456] Add stat counter for when queries go over budget. (#4993) [SC-46304](https://app.shortcut.com/tiledb-inc/story/46304) This PR adds budgeting capabilities to the new memory tracking system. Memory trackers can be created with two parameters: a budget, and a function that gets called everybody the budget gets exceeded. Subsequently, the `Query` class got updated to pass its memory budget to the memory manager, and sets a callback that increments the `memory_budget_exceeded` stats counter. This will allow applications to detect whether a query went over budget. --- TYPE: FEATURE DESC: Add stats counter `memory_budget_exceeded` for when a query goes over budget. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/common/memory_tracker.cc | 33 +++++-- tiledb/common/memory_tracker.h | 96 ++++++++++++++----- tiledb/common/test/CMakeLists.txt | 4 +- tiledb/common/test/unit_memory_tracker.cc | 53 ++++++++++ tiledb/sm/query/query.cc | 24 ++++- tiledb/sm/query/query.h | 12 +-- tiledb/sm/storage_manager/context_resources.h | 4 + 7 files changed, 187 insertions(+), 39 deletions(-) create mode 100644 tiledb/common/test/unit_memory_tracker.cc diff --git a/tiledb/common/memory_tracker.cc b/tiledb/common/memory_tracker.cc index 70b992a4e5c8..0019acd55f7e 100644 --- a/tiledb/common/memory_tracker.cc +++ b/tiledb/common/memory_tracker.cc @@ -147,8 +147,21 @@ uint64_t MemoryTrackerResource::get_count() { } void* MemoryTrackerResource::do_allocate(size_t bytes, size_t alignment) { - total_counter_.fetch_add(bytes, std::memory_order_relaxed); + auto previous_total = + total_counter_.fetch_add(bytes, std::memory_order_relaxed); type_counter_.fetch_add(bytes, std::memory_order_relaxed); + if (previous_total + bytes > memory_budget_ && on_budget_exceeded_) { + try { + on_budget_exceeded_(); + } catch (...) { + // If the callback throws, undo the counter increments and rethrow. + // If we don't do this, the assert in the MemoryTracker destructor + // will fail because the total_counter_ will be non-zero. + total_counter_.fetch_sub(bytes, std::memory_order_relaxed); + type_counter_.fetch_sub(bytes, std::memory_order_relaxed); + throw; + } + } return upstream_->allocate(bytes, alignment); } @@ -189,7 +202,12 @@ tdb::pmr::memory_resource* MemoryTracker::get_resource(MemoryType type) { // Create and track a shared_ptr to the new memory resource. auto ret = make_shared( - HERE(), upstream_, total_counter_, counters_[type]); + HERE(), + upstream_, + total_counter_, + counters_[type], + memory_budget_, + on_budget_exceeded_); resources_.emplace(type, ret); // Return the raw memory resource pointer for use by pmr containers. @@ -215,7 +233,8 @@ uint64_t MemoryTracker::generate_id() { return curr_id.fetch_add(1); } -shared_ptr MemoryTrackerManager::create_tracker() { +shared_ptr MemoryTrackerManager::create_tracker( + uint64_t memory_budget, std::function on_budget_exceeded) { /* * The MemoryTracker class has a protected constructor to hopefully help * self-document that instances should almost never be created directly @@ -232,8 +251,9 @@ shared_ptr MemoryTrackerManager::create_tracker() { * Pass through to the protected MemoryTracker constructor for * make_shared. */ - MemoryTrackerCreator() - : MemoryTracker() { + MemoryTrackerCreator( + uint64_t memory_budget, std::function on_budget_exceeded) + : MemoryTracker(memory_budget, on_budget_exceeded) { } }; @@ -250,7 +270,8 @@ shared_ptr MemoryTrackerManager::create_tracker() { } // Create a new tracker - auto ret = make_shared(HERE()); + auto ret = make_shared( + HERE(), memory_budget, on_budget_exceeded); trackers_.emplace(trackers_.begin(), ret); return ret; diff --git a/tiledb/common/memory_tracker.h b/tiledb/common/memory_tracker.h index 7074d359692c..2a97957d3778 100644 --- a/tiledb/common/memory_tracker.h +++ b/tiledb/common/memory_tracker.h @@ -97,6 +97,7 @@ #define TILEDB_MEMORY_TRACKER_H #include +#include #include #include "tiledb/common/pmr.h" @@ -179,10 +180,14 @@ class MemoryTrackerResource : public tdb::pmr::memory_resource { explicit MemoryTrackerResource( tdb::pmr::memory_resource* upstream, std::atomic& total_counter, - std::atomic& type_counter) + std::atomic& type_counter, + const std::atomic& memory_budget, + std::function on_budget_exceeded) : upstream_(upstream) , total_counter_(total_counter) - , type_counter_(type_counter) { + , type_counter_(type_counter) + , memory_budget_(memory_budget) + , on_budget_exceeded_(on_budget_exceeded) { } /** The number of bytes tracked by this resource. */ @@ -208,6 +213,12 @@ class MemoryTrackerResource : public tdb::pmr::memory_resource { /** A reference to the memory type counter this resource is tracking. */ std::atomic& type_counter_; + + /** The memory budget for this resource. */ + const std::atomic& memory_budget_; + + /** A function to call when the budget is exceeded. */ + std::function on_budget_exceeded_; }; class MemoryTracker { @@ -254,9 +265,9 @@ class MemoryTracker { */ bool take_memory(uint64_t size, MemoryType mem_type) { std::lock_guard lg(mutex_); - if (memory_usage_ + size <= memory_budget_) { - memory_usage_ += size; - memory_usage_by_type_[mem_type] += size; + if (legacy_memory_usage_ + size <= legacy_memory_budget_) { + legacy_memory_usage_ += size; + legacy_memory_usage_by_type_[mem_type] += size; return true; } @@ -270,8 +281,8 @@ class MemoryTracker { */ void release_memory(uint64_t size, MemoryType mem_type) { std::lock_guard lg(mutex_); - memory_usage_ -= size; - memory_usage_by_type_[mem_type] -= size; + legacy_memory_usage_ -= size; + legacy_memory_usage_by_type_[mem_type] -= size; } /** @@ -282,11 +293,11 @@ class MemoryTracker { */ bool set_budget(uint64_t size) { std::lock_guard lg(mutex_); - if (memory_usage_ > size) { + if (legacy_memory_usage_ > size) { return false; } - memory_budget_ = size; + legacy_memory_budget_ = size; return true; } @@ -295,7 +306,7 @@ class MemoryTracker { */ uint64_t get_memory_usage() { std::lock_guard lg(mutex_); - return memory_usage_; + return legacy_memory_usage_; } /** @@ -303,7 +314,7 @@ class MemoryTracker { */ uint64_t get_memory_usage(MemoryType mem_type) { std::lock_guard lg(mutex_); - return memory_usage_by_type_[mem_type]; + return legacy_memory_usage_by_type_[mem_type]; } /** @@ -312,10 +323,12 @@ class MemoryTracker { */ uint64_t get_memory_available() { std::lock_guard lg(mutex_); - if (memory_usage_ + counters_[MemoryType::TILE_OFFSETS] > memory_budget_) { + if (legacy_memory_usage_ + counters_[MemoryType::TILE_OFFSETS] > + legacy_memory_budget_) { return 0; } - return memory_budget_ - memory_usage_ - counters_[MemoryType::TILE_OFFSETS]; + return legacy_memory_budget_ - legacy_memory_usage_ - + counters_[MemoryType::TILE_OFFSETS]; } /** @@ -324,7 +337,20 @@ class MemoryTracker { */ uint64_t get_memory_budget() { std::lock_guard lg(mutex_); - return memory_budget_; + return legacy_memory_budget_; + } + + /** + * Refresh the memory budget used by the new system. + * + * This method should be called only by Query::set_config, which gets called + * by the REST server after deserializing the query and creating its strategy, + * with a config that contains the new memory budget. + * + * @param new_budget The new memory budget. + */ + void refresh_memory_budget(uint64_t new_budget) { + memory_budget_.store(new_budget, std::memory_order_relaxed); } protected: @@ -341,27 +367,39 @@ class MemoryTracker { * * For tests that need to have a temporary MemoryTracker instance, there is * a `create_test_memory_tracker()` API available in the test support library. + * + * @param memory_budget The memory budget for this MemoryTracker. + * @param on_budget_exceeded The function to call when the budget is exceeded. + * Defaults to a function that does nothing. */ - MemoryTracker() - : memory_usage_(0) - , memory_budget_(std::numeric_limits::max()) + MemoryTracker( + uint64_t memory_budget = std::numeric_limits::max(), + std::function on_budget_exceeded = []() {}) + : legacy_memory_usage_(0) + , legacy_memory_budget_(std::numeric_limits::max()) , id_(generate_id()) , type_(MemoryTrackerType::ANONYMOUS) , upstream_(tdb::pmr::get_default_resource()) - , total_counter_(0){}; + , total_counter_(0) + , memory_budget_(memory_budget) + , on_budget_exceeded_(on_budget_exceeded){}; private: /** Protects all non-atomic member variables. */ std::mutex mutex_; + /* Legacy memory tracker infrastructure */ + /** Memory usage for tracked structures. */ - uint64_t memory_usage_; + uint64_t legacy_memory_usage_; /** Memory budget. */ - uint64_t memory_budget_; + uint64_t legacy_memory_budget_; /** Memory usage by type. */ - std::unordered_map memory_usage_by_type_; + std::unordered_map legacy_memory_usage_by_type_; + + /* Modern memory tracker infrastructure */ /** The id of this MemoryTracker. */ uint64_t id_; @@ -382,6 +420,12 @@ class MemoryTracker { /** The total memory usage of this MemoryTracker. */ std::atomic total_counter_; + /** Memory budget. */ + std::atomic memory_budget_; + + /** A function to call when the budget is exceeded. */ + std::function on_budget_exceeded_; + /** Generate a unique id for this MemoryTracker. */ static uint64_t generate_id(); }; @@ -397,9 +441,17 @@ class MemoryTrackerManager { /** * Create a new memory tracker. * + * @param memory_budget The memory budget for the new MemoryTracker. Defaults + * to unlimited. + * + * @param on_budget_exceeded The function to call when the budget is exceeded. + * Defaults to a function that does nothing. + * * @return The created MemoryTracker. */ - shared_ptr create_tracker(); + shared_ptr create_tracker( + uint64_t memory_budget = std::numeric_limits::max(), + std::function on_budget_exceeded = []() {}); /** * Generate a JSON string representing the current state of tracked memory. diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index b696b9613d1a..0bc544615a1e 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -36,8 +36,8 @@ commence(unit_test experimental) ) conclude(unit_test) -commence(unit_test memory_tracker_types) - this_target_sources(main.cc unit_memory_tracker_types.cc) +commence(unit_test memory_tracker) + this_target_sources(main.cc unit_memory_tracker.cc unit_memory_tracker_types.cc) this_target_object_libraries(baseline) conclude(unit_test) diff --git a/tiledb/common/test/unit_memory_tracker.cc b/tiledb/common/test/unit_memory_tracker.cc new file mode 100644 index 000000000000..b3be4b523fb1 --- /dev/null +++ b/tiledb/common/test/unit_memory_tracker.cc @@ -0,0 +1,53 @@ +/** + * @file unit_memory_tracker.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file tests the MemoryTracker class. + */ + +#include + +#include +#include "tiledb/common/memory_tracker.h" +#include "tiledb/common/pmr.h" + +using namespace tiledb::sm; + +TEST_CASE( + "Memory tracker: Test budget exceeded callback", + "[memory_tracker][budget_exceeded]") { + MemoryTrackerManager tracker_manager; + auto tracker = tracker_manager.create_tracker( + 100, []() { throw std::runtime_error("Budget exceeded"); }); + CHECK_THROWS_WITH( + [&tracker]() { + std::ignore = tdb::pmr::vector( + 1000, tracker->get_resource(MemoryType::TILE_DATA)); + }(), + "Budget exceeded"); +} diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 071b62d0f36a..55c29641ef2b 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -66,6 +66,17 @@ using namespace tiledb::sm::stats; namespace tiledb::sm { +/** + * Gets the effective memory budget for a query. This will be memory_budget if + * set, otherwise the value "sm.mem.total_budget" from config. + */ +static uint64_t get_effective_memory_budget( + const Config& config, optional memory_budget) { + return memory_budget ? + *memory_budget : + config.get("sm.mem.total_budget", Config::must_find); +} + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -76,7 +87,13 @@ Query::Query( optional fragment_name, optional memory_budget) : resources_(storage_manager->resources()) - , query_memory_tracker_(resources_.create_memory_tracker()) + , stats_(resources_.stats().create_child("Query")) + , logger_(resources_.logger()->clone("Query", ++logger_id_)) + , query_memory_tracker_(resources_.memory_tracker_manager().create_tracker( + get_effective_memory_budget(resources_.config(), memory_budget), + [stats = stats_]() { + stats->add_counter("memory_budget_exceeded", 1); + })) , array_shared_(array) , array_(array_shared_.get()) , opened_array_(array->opened_array()) @@ -88,8 +105,6 @@ Query::Query( Layout::ROW_MAJOR : Layout::UNORDERED) , storage_manager_(storage_manager) - , stats_(resources_.stats().create_child("Query")) - , logger_(resources_.logger()->clone("Query", ++logger_id_)) , dim_label_queries_(nullptr) , has_coords_buffer_(false) , has_zipped_coords_buffer_(false) @@ -926,6 +941,9 @@ void Query::set_config(const Config& config) { } config_.inherit(config); + query_memory_tracker_->refresh_memory_budget( + get_effective_memory_budget(config_, memory_budget_)); + // Refresh memory budget configuration. if (strategy_ != nullptr) { strategy_->refresh_config(); diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index ab983521e441..16a0e61edd25 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -924,6 +924,12 @@ class Query { /** Resource used for operations. */ ContextResources& resources_; + /** The class stats. */ + stats::Stats* stats_; + + /** The class logger. */ + shared_ptr logger_; + /** The query memory tracker. */ shared_ptr query_memory_tracker_; @@ -970,12 +976,6 @@ class Query { /** The query strategy. */ tdb_unique_ptr strategy_; - /** The class stats. */ - stats::Stats* stats_; - - /** The class logger. */ - shared_ptr logger_; - /** UID of the logger instance */ inline static std::atomic logger_id_ = 0; diff --git a/tiledb/sm/storage_manager/context_resources.h b/tiledb/sm/storage_manager/context_resources.h index 515dcdd450b3..97010b4f641c 100644 --- a/tiledb/sm/storage_manager/context_resources.h +++ b/tiledb/sm/storage_manager/context_resources.h @@ -111,6 +111,10 @@ class ContextResources { return rest_client_; } + [[nodiscard]] inline MemoryTrackerManager& memory_tracker_manager() const { + return *memory_tracker_manager_; + } + /** * Create a new MemoryTracker * From 5d6c26aeff476f3d67e82e2c92a3b9bcd954c169 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 28 May 2024 10:18:13 +0200 Subject: [PATCH 379/456] Implement consolidation memory budget variables. (#5011) This implement 3 new variables for memory budgetting for consolidation. The values are a set of 3 weights that weigh each parts of the consolidation algorithm. The buffers, the read operation and the write operation. All weights are summed and ratios are computed to divide sm.mem.total_budget. The values are: * `sm.mem.consolidation.buffers_weight` * `sm.mem.consolidation.reader_weight` * `sm.mem.consolidation.writer_weight` Note this also removes the total_buffer_size deprecated setting. --- TYPE: CONFIG DESC: Implement consolidation memory budget variables. --- test/src/unit-capi-config.cc | 8 ++- test/src/unit-capi-fragment_info.cc | 20 +++++- test/src/unit-capi-sparse_heter.cc | 22 ++++++- test/src/unit-capi-string_dims.cc | 13 +++- test/src/unit-cppapi-array.cc | 13 +++- ...it-cppapi-consolidation-with-timestamps.cc | 4 +- test/src/unit-cppapi-consolidation.cc | 4 +- test/src/unit-cppapi-fragment_info.cc | 4 +- test/src/unit-cppapi-hilbert.cc | 4 +- test/src/unit-cppapi-max-fragment-size.cc | 4 +- test/src/unit-duplicates.cc | 10 ++- tiledb/api/c_api/config/config_api_external.h | 25 +++++-- tiledb/sm/config/config.cc | 18 +++-- tiledb/sm/config/config.h | 12 +++- .../sm/consolidator/fragment_consolidator.cc | 66 +++++++++++++------ .../sm/consolidator/fragment_consolidator.h | 20 ++++-- .../test/unit_fragment_consolidator.cc | 2 +- tiledb/sm/cpp_api/config.h | 27 ++++++-- 18 files changed, 212 insertions(+), 64 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 5462ca601e36..34dca3c2fd36 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -260,7 +260,6 @@ void check_save_to_file() { ss << "sm.consolidation.steps 4294967295\n"; ss << "sm.consolidation.timestamp_end " << std::to_string(UINT64_MAX) << "\n"; ss << "sm.consolidation.timestamp_start 0\n"; - ss << "sm.consolidation.total_buffer_size 2147483648\n"; ss << "sm.dedup_coords false\n"; ss << "sm.enable_signal_handlers true\n"; ss << "sm.encryption_type NO_ENCRYPTION\n"; @@ -272,6 +271,9 @@ void check_save_to_file() { ss << "sm.io_concurrency_level " << std::thread::hardware_concurrency() << "\n"; ss << "sm.max_tile_overlap_size 314572800\n"; + ss << "sm.mem.consolidation.buffers_weight 1\n"; + ss << "sm.mem.consolidation.reader_weight 3\n"; + ss << "sm.mem.consolidation.writer_weight 2\n"; ss << "sm.mem.malloc_trim true\n"; ss << "sm.mem.reader.sparse_global_order.ratio_array_data 0.1\n"; ss << "sm.mem.reader.sparse_global_order.ratio_coords 0.5\n"; @@ -632,6 +634,9 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["sm.query.dense.reader"] = "refactored"; all_param_values["sm.query.sparse_global_order.reader"] = "refactored"; all_param_values["sm.query.sparse_unordered_with_dups.reader"] = "refactored"; + all_param_values["sm.mem.consolidation.buffers_weight"] = "1"; + all_param_values["sm.mem.consolidation.reader_weight"] = "3"; + all_param_values["sm.mem.consolidation.writer_weight"] = "2"; all_param_values["sm.mem.malloc_trim"] = "true"; all_param_values["sm.mem.tile_upper_memory_limit"] = "1073741824"; all_param_values["sm.mem.total_budget"] = "10737418240"; @@ -663,7 +668,6 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["sm.consolidation.step_min_frags"] = "4294967295"; all_param_values["sm.consolidation.step_max_frags"] = "4294967295"; all_param_values["sm.consolidation.buffer_size"] = "50000000"; - all_param_values["sm.consolidation.total_buffer_size"] = "2147483648"; all_param_values["sm.consolidation.max_fragment_size"] = std::to_string(UINT64_MAX); all_param_values["sm.consolidation.step_size_ratio"] = "0.0"; diff --git a/test/src/unit-capi-fragment_info.cc b/test/src/unit-capi-fragment_info.cc index f9e3f615f065..d3793239096c 100644 --- a/test/src/unit-capi-fragment_info.cc +++ b/test/src/unit-capi-fragment_info.cc @@ -1271,7 +1271,15 @@ TEST_CASE( REQUIRE(error == nullptr); rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); @@ -1644,7 +1652,15 @@ TEST_CASE( REQUIRE(error == nullptr); rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); diff --git a/test/src/unit-capi-sparse_heter.cc b/test/src/unit-capi-sparse_heter.cc index cd077ba0069c..070afae9a195 100644 --- a/test/src/unit-capi-sparse_heter.cc +++ b/test/src/unit-capi-sparse_heter.cc @@ -1003,7 +1003,15 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); auto rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); @@ -1356,8 +1364,16 @@ TEST_CASE_METHOD( REQUIRE(tiledb_config_alloc(&config, &error) == TILEDB_OK); REQUIRE(error == nullptr); - int rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); + auto rc = tiledb_config_set( + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); diff --git a/test/src/unit-capi-string_dims.cc b/test/src/unit-capi-string_dims.cc index 465da750acc7..59d654a3d57b 100644 --- a/test/src/unit-capi-string_dims.cc +++ b/test/src/unit-capi-string_dims.cc @@ -2125,8 +2125,17 @@ TEST_CASE_METHOD( config, "sm.consolidation.mode", "fragment_meta", &error); CHECK(rc == TILEDB_OK); rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); - CHECK(rc == TILEDB_OK); + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); // Consolidate fragment metadata rc = tiledb_array_consolidate(ctx_, array_name.c_str(), config); diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index dbe7494a486e..d11dc4894c21 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -663,7 +663,10 @@ TEST_CASE( "C++ API: Consolidation of sequential fragment writes", "[cppapi][consolidation][sequential][non-rest]") { tiledb::Config cfg; - cfg["sm.consolidation.total_buffer_size"] = "1048576"; + + cfg["sm.mem.consolidation.buffers_weight"] = "1"; + cfg["sm.mem.consolidation.reader_weight"] = "5000"; + cfg["sm.mem.consolidation.writer_weight"] = "5000"; Context ctx(cfg); VFS vfs(ctx); const std::string array_name = "cpp_unit_array"; @@ -714,7 +717,9 @@ TEST_CASE( TEST_CASE("C++ API: Encrypted array", "[cppapi][encryption][non-rest]") { const char key[] = "0123456789abcdeF0123456789abcdeF"; tiledb::Config cfg; - cfg["sm.consolidation.total_buffer_size"] = "1048576"; + cfg["sm.mem.consolidation.buffers_weight"] = "1"; + cfg["sm.mem.consolidation.reader_weight"] = "5000"; + cfg["sm.mem.consolidation.writer_weight"] = "5000"; cfg["sm.encryption_type"] = "AES_256_GCM"; cfg["sm.encryption_key"] = key; Context ctx(cfg); @@ -805,7 +810,9 @@ TEST_CASE( tiledb::Config cfg; cfg["sm.encryption_type"] = "AES_256_GCM"; cfg["sm.encryption_key"] = key.c_str(); - cfg["sm.consolidation.total_buffer_size"] = "1048576"; + cfg["sm.mem.consolidation.buffers_weight"] = "1"; + cfg["sm.mem.consolidation.reader_weight"] = "5000"; + cfg["sm.mem.consolidation.writer_weight"] = "5000"; Context ctx(cfg); VFS vfs(ctx); const std::string array_name = "cpp_unit_array"; diff --git a/test/src/unit-cppapi-consolidation-with-timestamps.cc b/test/src/unit-cppapi-consolidation-with-timestamps.cc index bb4ac9a90cb9..4bff6dbb0beb 100644 --- a/test/src/unit-cppapi-consolidation-with-timestamps.cc +++ b/test/src/unit-cppapi-consolidation-with-timestamps.cc @@ -1270,7 +1270,9 @@ TEST_CASE_METHOD( // fully removed for overlapping ranges, this test case can be deleted. tiledb::Config cfg; cfg.set("sm.merge_overlapping_ranges_experimental", "false"); - cfg["sm.consolidation.total_buffer_size"] = "1048576"; + cfg["sm.mem.consolidation.buffers_weight"] = "1"; + cfg["sm.mem.consolidation.reader_weight"] = "5000"; + cfg["sm.mem.consolidation.writer_weight"] = "5000"; ctx_ = Context(cfg); sm_ = ctx_.ptr().get()->storage_manager(); vfs_ = VFS(ctx_); diff --git a/test/src/unit-cppapi-consolidation.cc b/test/src/unit-cppapi-consolidation.cc index 47f6eb36e1e7..f6e4cbb118d8 100644 --- a/test/src/unit-cppapi-consolidation.cc +++ b/test/src/unit-cppapi-consolidation.cc @@ -125,7 +125,9 @@ TEST_CASE( // Create array tiledb::Config cfg; - cfg["sm.consolidation.total_buffer_size"] = "1048576"; + cfg["sm.mem.consolidation.buffers_weight"] = "1"; + cfg["sm.mem.consolidation.reader_weight"] = "5000"; + cfg["sm.mem.consolidation.writer_weight"] = "5000"; Context ctx(cfg); Domain domain(ctx); auto d = Dimension::create(ctx, "d1", {{10, 110}}, 50); diff --git a/test/src/unit-cppapi-fragment_info.cc b/test/src/unit-cppapi-fragment_info.cc index c60693010d2b..9173cdfee84a 100644 --- a/test/src/unit-cppapi-fragment_info.cc +++ b/test/src/unit-cppapi-fragment_info.cc @@ -879,7 +879,9 @@ TEST_CASE( // Consolidate fragments Config config; config["sm.consolidation.mode"] = "fragments"; - config["sm.consolidation.total_buffer_size"] = "1048576"; + config["sm.mem.consolidation.buffers_weight"] = "1"; + config["sm.mem.consolidation.reader_weight"] = "5000"; + config["sm.mem.consolidation.writer_weight"] = "5000"; Array::consolidate(ctx, array_name, &config); // Load fragment info diff --git a/test/src/unit-cppapi-hilbert.cc b/test/src/unit-cppapi-hilbert.cc index 318951a81260..7dbba944c5cd 100644 --- a/test/src/unit-cppapi-hilbert.cc +++ b/test/src/unit-cppapi-hilbert.cc @@ -1780,7 +1780,9 @@ TEST_CASE( Config config; config["sm.consolidation.mode"] = "fragments"; config["sm.vacuum.mode"] = "fragments"; - config["sm.consolidation.total_buffer_size"] = "1048576"; + config["sm.mem.consolidation.buffers_weight"] = "1"; + config["sm.mem.consolidation.reader_weight"] = "5000"; + config["sm.mem.consolidation.writer_weight"] = "5000"; CHECK_NOTHROW(Array::consolidate(ctx, array_name, &config)); CHECK_NOTHROW(Array::vacuum(ctx, array_name, &config)); auto contents = vfs.ls(get_fragment_dir(array_name)); diff --git a/test/src/unit-cppapi-max-fragment-size.cc b/test/src/unit-cppapi-max-fragment-size.cc index 2b2f70f9b8c2..b86a2a1937dc 100644 --- a/test/src/unit-cppapi-max-fragment-size.cc +++ b/test/src/unit-cppapi-max-fragment-size.cc @@ -243,7 +243,9 @@ struct CPPMaxFragmentSizeFx { uint64_t max_fragment_size = std::numeric_limits::max()) { auto config = ctx_.config(); config["sm.consolidation.max_fragment_size"] = max_fragment_size; - config["sm.consolidation.total_buffer_size"] = "1048576"; + config["sm.mem.consolidation.buffers_weight"] = "1"; + config["sm.mem.consolidation.reader_weight"] = "5000"; + config["sm.mem.consolidation.writer_weight"] = "5000"; Array::consolidate(ctx_, array_name, &config); } diff --git a/test/src/unit-duplicates.cc b/test/src/unit-duplicates.cc index cd10644e2a87..14dde2dfbc89 100644 --- a/test/src/unit-duplicates.cc +++ b/test/src/unit-duplicates.cc @@ -353,7 +353,15 @@ TEST_CASE_METHOD( REQUIRE(error == nullptr); rc = tiledb_config_set( - config, "sm.consolidation.total_buffer_size", "1048576", &error); + config, "sm.mem.consolidation.buffers_weight", "1", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.reader_weight", "5000", &error); + REQUIRE(rc == TILEDB_OK); + REQUIRE(error == nullptr); + rc = tiledb_config_set( + config, "sm.mem.consolidation.writer_weight", "5000", &error); REQUIRE(rc == TILEDB_OK); REQUIRE(error == nullptr); diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 5bdd4d78d3d9..0d1ef5168234 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -171,11 +171,6 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 - * - `sm.consolidation.total_buffer_size`
- * **Deprecated** - * The size (in bytes) of all attribute buffers used during - * consolidation.
- * **Default**: 2,147,483,648 * - `sm.consolidation.max_fragment_size`
* **Experimental**
* The size (in bytes) of the maximum on-disk fragment size that will be @@ -261,6 +256,26 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `sm.mem.total_budget`
* Memory budget for readers and writers.
* **Default**: 10GB + * - `sm.mem.consolidation.buffers_weight`
+ * Weight used to split `sm.mem.total_budget` and assign to the + * consolidation buffers. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 1 + * - `sm.mem.consolidation.reader_weight`
+ * Weight used to split `sm.mem.total_budget` and assign to the + * reader query. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 3 + * Weight used to split `sm.mem.total_budget` and assign to the + * writer query. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 2 * - `sm.mem.reader.sparse_global_order.ratio_coords`
* Ratio of the budget allocated for coordinates in the sparse global * order reader.
diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 01d27c788aaf..72378c06e39d 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -118,6 +118,9 @@ const std::string Config::SM_QUERY_SPARSE_UNORDERED_WITH_DUPS_READER = const std::string Config::SM_MEM_MALLOC_TRIM = "true"; const std::string Config::SM_UPPER_MEMORY_LIMIT = "1073741824"; // 1GB const std::string Config::SM_MEM_TOTAL_BUDGET = "10737418240"; // 10GB +const std::string Config::SM_MEM_CONSOLIDATION_BUFFERS_WEIGHT = "1"; +const std::string Config::SM_MEM_CONSOLIDATION_READER_WEIGHT = "3"; +const std::string Config::SM_MEM_CONSOLIDATION_WRITER_WEIGHT = "2"; const std::string Config::SM_MEM_SPARSE_GLOBAL_ORDER_RATIO_COORDS = "0.5"; const std::string Config::SM_MEM_SPARSE_GLOBAL_ORDER_RATIO_TILE_RANGES = "0.1"; const std::string Config::SM_MEM_SPARSE_GLOBAL_ORDER_RATIO_ARRAY_DATA = "0.1"; @@ -135,7 +138,6 @@ const std::string Config::SM_IO_CONCURRENCY_LEVEL = const std::string Config::SM_SKIP_CHECKSUM_VALIDATION = "false"; const std::string Config::SM_CONSOLIDATION_AMPLIFICATION = "1.0"; const std::string Config::SM_CONSOLIDATION_BUFFER_SIZE = "50000000"; -const std::string Config::SM_CONSOLIDATION_TOTAL_BUFFER_SIZE = "2147483648"; const std::string Config::SM_CONSOLIDATION_MAX_FRAGMENT_SIZE = std::to_string(UINT64_MAX); const std::string Config::SM_CONSOLIDATION_PURGE_DELETED_CELLS = "false"; @@ -307,6 +309,15 @@ const std::map default_config_values = { std::make_pair( "sm.mem.tile_upper_memory_limit", Config::SM_UPPER_MEMORY_LIMIT), std::make_pair("sm.mem.total_budget", Config::SM_MEM_TOTAL_BUDGET), + std::make_pair( + "sm.mem.consolidation.buffers_weight", + Config::SM_MEM_CONSOLIDATION_BUFFERS_WEIGHT), + std::make_pair( + "sm.mem.consolidation.reader_weight", + Config::SM_MEM_CONSOLIDATION_READER_WEIGHT), + std::make_pair( + "sm.mem.consolidation.writer_weight", + Config::SM_MEM_CONSOLIDATION_WRITER_WEIGHT), std::make_pair( "sm.mem.reader.sparse_global_order.ratio_coords", Config::SM_MEM_SPARSE_GLOBAL_ORDER_RATIO_COORDS), @@ -337,9 +348,6 @@ const std::map default_config_values = { Config::SM_CONSOLIDATION_AMPLIFICATION), std::make_pair( "sm.consolidation.buffer_size", Config::SM_CONSOLIDATION_BUFFER_SIZE), - std::make_pair( - "sm.consolidation.total_buffer_size", - Config::SM_CONSOLIDATION_TOTAL_BUFFER_SIZE), std::make_pair( "sm.consolidation.max_fragment_size", Config::SM_CONSOLIDATION_MAX_FRAGMENT_SIZE), @@ -763,8 +771,6 @@ Status Config::sanity_check( RETURN_NOT_OK(utils::parse::convert(value, &vf)); } else if (param == "sm.consolidation.buffer_size") { RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); - } else if (param == "sm.consolidation.total_buffer_size") { - RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); } else if (param == "sm.consolidation.max_fragment_size") { RETURN_NOT_OK(utils::parse::convert(value, &vuint64)); } else if (param == "sm.consolidation.purge_deleted_cells") { diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 6b9f47a16898..2fd464470e40 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -227,6 +227,15 @@ class Config { /** Maximum memory budget for readers and writers. */ static const std::string SM_MEM_TOTAL_BUDGET; + /** Weight for consolidation buffers used to split total budget. */ + static const std::string SM_MEM_CONSOLIDATION_BUFFERS_WEIGHT; + + /** Weight for consolidation read query used to split total budget. */ + static const std::string SM_MEM_CONSOLIDATION_READER_WEIGHT; + + /** Weight for consolidation write query used to split total budget. */ + static const std::string SM_MEM_CONSOLIDATION_WRITER_WEIGHT; + /** Ratio of the sparse global order reader budget used for coords. */ static const std::string SM_MEM_SPARSE_GLOBAL_ORDER_RATIO_COORDS; @@ -277,9 +286,6 @@ class Config { /** The buffer size for each attribute used in consolidation. */ static const std::string SM_CONSOLIDATION_BUFFER_SIZE; - /** The total buffer size for all attributes during consolidation. */ - static const std::string SM_CONSOLIDATION_TOTAL_BUFFER_SIZE; - /** The maximum fragment size used in consolidation. */ static const std::string SM_CONSOLIDATION_MAX_FRAGMENT_SIZE; diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 322fcf8f4b41..d2cad06f6d88 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -70,7 +70,8 @@ void FragmentConsolidationWorkspace::resize_buffers( stats::Stats* stats, const FragmentConsolidationConfig& config, const ArraySchema& array_schema, - std::unordered_map& avg_cell_sizes) { + std::unordered_map& avg_cell_sizes, + uint64_t total_buffers_budget) { auto timer_se = stats->start_timer("resize_buffers"); // For easy reference @@ -133,15 +134,12 @@ void FragmentConsolidationWorkspace::resize_buffers( buffer_weights.emplace_back(sizeof(uint64_t)); } - // Use the old buffer size setting to see how much memory we would use. - auto buffer_num = buffer_weights.size(); - uint64_t total_budget = config.total_buffer_size_; - // If a user set the per-attribute buffer size configuration, we override // the use of the total_budget_size config setting for backwards // compatible behavior. + auto buffer_num = buffer_weights.size(); if (config.buffer_size_ != 0) { - total_budget = config.buffer_size_ * buffer_num; + total_buffers_budget = config.buffer_size_ * buffer_num; } // Calculate the size of individual buffers by assigning a weight based @@ -149,7 +147,8 @@ void FragmentConsolidationWorkspace::resize_buffers( auto total_weights = std::accumulate( buffer_weights.begin(), buffer_weights.end(), static_cast(0)); - uint64_t adjusted_budget = total_budget / total_weights * total_weights; + uint64_t adjusted_budget = + total_buffers_budget / total_weights * total_weights; if (buffer_num > buffers_.size()) { sizes_.resize(buffer_num); @@ -552,9 +551,18 @@ Status FragmentConsolidator::consolidate_internal( } } + // Compute memory budgets + uint64_t total_weights = + config_.buffers_weight_ + config_.reader_weight_ + config_.writer_weight_; + uint64_t single_unit_budget = config_.total_budget_ / total_weights; + uint64_t buffers_budget = config_.buffers_weight_ * single_unit_budget; + uint64_t reader_budget = config_.reader_weight_ * single_unit_budget; + uint64_t writer_budget = config_.writer_weight_ * single_unit_budget; + // Prepare buffers auto average_var_cell_sizes = array_for_reads->get_average_var_cell_sizes(); - cw.resize_buffers(stats_, config_, array_schema, average_var_cell_sizes); + cw.resize_buffers( + stats_, config_, array_schema, average_var_cell_sizes, buffers_budget); // Create queries tdb_unique_ptr query_r = nullptr; @@ -565,7 +573,9 @@ Status FragmentConsolidator::consolidate_internal( union_non_empty_domains, query_r, query_w, - new_fragment_uri)); + new_fragment_uri, + reader_budget, + writer_budget)); // Get the vacuum URI URI vac_uri; @@ -645,7 +655,9 @@ Status FragmentConsolidator::create_queries( const NDRange& subarray, tdb_unique_ptr& query_r, tdb_unique_ptr& query_w, - URI* new_fragment_uri) { + URI* new_fragment_uri, + uint64_t read_memory_budget, + uint64_t write_memory_budget) { auto timer_se = stats_->start_timer("consolidate_create_queries"); const auto dense = array_for_reads->array_schema_latest().dense(); @@ -655,8 +667,8 @@ Status FragmentConsolidator::create_queries( // is not a user input prone to errors). // Create read query - query_r = - tdb_unique_ptr(tdb_new(Query, storage_manager_, array_for_reads)); + query_r = tdb_unique_ptr(tdb_new( + Query, storage_manager_, array_for_reads, nullopt, read_memory_budget)); RETURN_NOT_OK(query_r->set_layout(Layout::GLOBAL_ORDER)); // Dense consolidation will do a tile aligned read. @@ -681,8 +693,12 @@ Status FragmentConsolidator::create_queries( first, last, write_version); // Create write query - query_w = tdb_unique_ptr( - tdb_new(Query, storage_manager_, array_for_writes, fragment_name)); + query_w = tdb_unique_ptr(tdb_new( + Query, + storage_manager_, + array_for_writes, + fragment_name, + write_memory_budget)); RETURN_NOT_OK(query_w->set_layout(Layout::GLOBAL_ORDER)); RETURN_NOT_OK(query_w->disable_checks_consolidation()); query_w->set_fragment_size(config_.max_fragment_size_); @@ -924,21 +940,31 @@ Status FragmentConsolidator::set_config(const Config& config) { assert(found); config_.buffer_size_ = 0; // Only set the buffer_size_ if the user specified a value. Otherwise, we use - // the new sm.consolidation.total_buffer_size instead. + // the new sm.mem.consolidation.buffers_weight instead. if (merged_config.set_params().count("sm.consolidation.buffer_size") > 0) { logger_->warn( "The `sm.consolidation.buffer_size configuration setting has been " "deprecated. Set consolidation buffer sizes using the newer " - "`sm.consolidation.total_buffer_size` setting."); + "`sm.mem.consolidation.buffers_weight` setting."); RETURN_NOT_OK(merged_config.get( "sm.consolidation.buffer_size", &config_.buffer_size_, &found)); assert(found); } - config_.total_buffer_size_ = 0; + config_.total_budget_ = 0; RETURN_NOT_OK(merged_config.get( - "sm.consolidation.total_buffer_size", - &config_.total_buffer_size_, - &found)); + "sm.mem.total_budget", &config_.total_budget_, &found)); + assert(found); + config_.buffers_weight_ = 0; + RETURN_NOT_OK(merged_config.get( + "sm.mem.consolidation.buffers_weight", &config_.buffers_weight_, &found)); + assert(found); + config_.reader_weight_ = 0; + RETURN_NOT_OK(merged_config.get( + "sm.mem.consolidation.reader_weight", &config_.reader_weight_, &found)); + assert(found); + config_.writer_weight_ = 0; + RETURN_NOT_OK(merged_config.get( + "sm.mem.consolidation.writer_weight", &config_.writer_weight_, &found)); assert(found); config_.max_fragment_size_ = 0; RETURN_NOT_OK(merged_config.get( diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index aec52750d769..5b828b407141 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -79,8 +79,14 @@ struct FragmentConsolidationConfig : Consolidator::ConsolidationConfigBase { float amplification_; /** Attribute buffer size. */ uint64_t buffer_size_; - /** Total buffer size for all attributes. */ - uint64_t total_buffer_size_; + /** Total memory budget for consolidation operation. */ + uint64_t total_budget_; + /** Consolidation buffers weight used to partition total budget. */ + uint64_t buffers_weight_; + /** Reader weight used to partition total budget. */ + uint64_t reader_weight_; + /** Writer weight used to partition total budget. */ + uint64_t writer_weight_; /** Max fragment size. */ uint64_t max_fragment_size_; /** @@ -124,13 +130,15 @@ class FragmentConsolidationWorkspace { * @param config The consolidation config. * @param array_schema The array schema. * @param avg_cell_sizes The average cell sizes. + * @param total_buffers_budget Total budget for the consolidation buffers. * @return a consolidation workspace containing the buffers */ void resize_buffers( stats::Stats* stats, const FragmentConsolidationConfig& config, const ArraySchema& array_schema, - std::unordered_map& avg_cell_sizes); + std::unordered_map& avg_cell_sizes, + uint64_t total_buffers_budget); /** Accessor for buffers. */ tdb::pmr::vector>& buffers() { @@ -310,6 +318,8 @@ class FragmentConsolidator : public Consolidator { * @param query_r This query reads from the fragments to be consolidated. * @param query_w This query writes to the new consolidated fragment. * @param new_fragment_uri The URI of the new fragment to be created. + * @param read_memory_budget Memory budget for the read operation. + * @param write_memory_budget Memory budget for the write operation. * @return Status */ Status create_queries( @@ -318,7 +328,9 @@ class FragmentConsolidator : public Consolidator { const NDRange& subarray, tdb_unique_ptr& query_r, tdb_unique_ptr& query_w, - URI* new_fragment_uri); + URI* new_fragment_uri, + uint64_t read_memory_budget, + uint64_t write_memory_budget); /** * Based on the input fragment info, this algorithm decides the (sorted) list diff --git a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc index aa18fc920026..b40d8c1e831e 100644 --- a/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc +++ b/tiledb/sm/consolidator/test/unit_fragment_consolidator.cc @@ -223,7 +223,7 @@ TEST_CASE( cfg.buffer_size_ = 1000; FragmentConsolidationWorkspace cw(tiledb::test::get_test_memory_tracker()); - cw.resize_buffers(&statistics, cfg, *schema, avg_cell_sizes); + cw.resize_buffers(&statistics, cfg, *schema, avg_cell_sizes, 1); auto& buffers = cw.buffers(); auto& buffer_sizes = cw.sizes(); diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 6dd3458ff452..c9598aedb5c8 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -346,11 +346,6 @@ class Config { * The size (in bytes) of the attribute buffers used during * consolidation.
* **Default**: 50,000,000 - * - `sm.consolidation.total_buffer_size`
- * **Deprecated** - * The size (in bytes) of all attribute buffers used during - * consolidation.
- * **Default**: 2,147,483,648 * - `sm.consolidation.max_fragment_size`
* **Experimental**
* The size (in bytes) of the maximum on-disk fragment size that will be @@ -437,11 +432,29 @@ class Config { * incur performance penalties during memory movement operations. It is a * soft limit that we might go over if a single tile doesn't fit into * memory, we will allow to load that tile if it still fits within - * `sm.mem.total_budget`.
- * **Default**: 1GB * - `sm.mem.total_budget`
* Memory budget for readers and writers.
* **Default**: 10GB + * - `sm.mem.consolidation.buffers_weight`
+ * Weight used to split `sm.mem.total_budget` and assign to the + * consolidation buffers. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 1 + * - `sm.mem.consolidation.reader_weight`
+ * Weight used to split `sm.mem.total_budget` and assign to the + * reader query. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 3 + * Weight used to split `sm.mem.total_budget` and assign to the + * writer query. The budget is split across 3 values, + * `sm.mem.consolidation.buffers_weight`, + * `sm.mem.consolidation.reader_weight` and + * `sm.mem.consolidation.writer_weight`.
+ * **Default**: 2 * - `sm.mem.reader.sparse_global_order.ratio_coords`
* Ratio of the budget allocated for coordinates in the sparse global * order reader.
From d0b3435f5e46ebc88da26a9c45c507e3d7bb9b22 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 28 May 2024 13:22:55 +0200 Subject: [PATCH 380/456] Update dev history with content from 2.23. (#5015) --- TYPE: NO_HISTORY DESC: Update dev history with content from 2.23. --- HISTORY.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 492993062f74..77b6e6a2daee 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,42 @@ +# TileDB v2.23.0 Release Notes + +## Deprecation announcements + +* All deprecated APIs will be removed in version 2.26. TileDB can be built with `--remove-deprecations` (`bootstrap`) or `-DTILEDB_REMOVE_DEPRECATIONS` (CMake) to validate that projects are not using any deprecated APIs. + +## Improvements + +* Improve diagnostics when an Azure endpoint is not configured. [#4845](https://github.com/TileDB-Inc/TileDB/pull/4845) +* Do not attempt Azure shared key authentication if no account name is specified. [#4856](https://github.com/TileDB-Inc/TileDB/pull/4856) +* Clarify the documentation for the non empty domain CAPI. [#4885](https://github.com/TileDB-Inc/TileDB/pull/4885) +* Make closing a group that is not open a no-op. [#4917](https://github.com/TileDB-Inc/TileDB/pull/4917) + +## Defects removed + +* Fix wrong fallback cell order for Hilbert. [#4924](https://github.com/TileDB-Inc/TileDB/pull/4924) +* Config serialization should take into account environment variables. [#4865](https://github.com/TileDB-Inc/TileDB/pull/4865) +* Vac files should only be removed if paths removal was fully successful. [#4889](https://github.com/TileDB-Inc/TileDB/pull/4889) +* Fix C query condition examples. [#4912](https://github.com/TileDB-Inc/TileDB/pull/4912) + +### C++ API + +* `Query::submit_async` is deprecated. Call `Query::submit()` on another thread instead. [#4879](https://github.com/TileDB-Inc/TileDB/pull/4879) +* Overloads of methods and constructors in `Array` and `ArraySchema` that accept encryption keys are deprecated. Specify the encryption key with the `sm.encryption_type` and `sm.encryption_key` config options instead. [#4879](https://github.com/TileDB-Inc/TileDB/pull/4879) +* Add C++ API for tiledb_array_consolidate_fragments. [#4884](https://github.com/TileDB-Inc/TileDB/pull/4884) + +## Build System Changes + +* Vcpkg is always enabled; turning the `TILEDB_VCPKG` option off is no longer supported and fails. [#4570](https://github.com/TileDB-Inc/TileDB/pull/4570) +* Update the `TILEDB_REMOVE_DEPRECATIONS` option to exclude all deprecated C and C++ APIs from the library binary and the headers. [#4887](https://github.com/TileDB-Inc/TileDB/pull/4887) + +## Internal Improvements + +* Implement iterator facade for external sort. [#4914](https://github.com/TileDB-Inc/TileDB/pull/4914) +* Implement var_length_view and unit tests for external sort. [#4918](https://github.com/TileDB-Inc/TileDB/pull/4918) +* Implement permutation view for external sort. [#4920](https://github.com/TileDB-Inc/TileDB/pull/4920) +* Implement proxy sort for external sort. [#4922](https://github.com/TileDB-Inc/TileDB/pull/4922) +* Implement alt var length view for external sort. [#4925](https://github.com/TileDB-Inc/TileDB/pull/4925) + # TileDB v2.22.0 Release Notes ## Deprecation announcements From 695402fd0a84ba54df5db3b44ecc2ecf49c9597a Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Tue, 28 May 2024 20:32:51 +0300 Subject: [PATCH 381/456] Fix SEG in the SparseGlobalOrderReader when using a Query Condition. (#5012) The lack of this check was causing a SEG fault when reading with a query condition with the Sparse Global Order reader. More specifically, the code previously was trying to access tiles deleted from memory in some cases. These tiles were deleted as they were marked as ```unused``` during the read process. [sc-47713] --- TYPE: BUG DESC: Fix SEG faults in the SparseGlobalOrderReader when using a Query Condition. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/cpp-integration-query-condition.cc | 76 +++++++++++++++++++ .../readers/sparse_global_order_reader.cc | 11 ++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/test/src/cpp-integration-query-condition.cc b/test/src/cpp-integration-query-condition.cc index 017a04cdc05d..eaeb7b4cb0b8 100644 --- a/test/src/cpp-integration-query-condition.cc +++ b/test/src/cpp-integration-query-condition.cc @@ -1610,6 +1610,82 @@ TEST_CASE( } } +TEST_CASE( + "Testing sparse query condition with the same fragment domain.", + "[query][query-condition][sparse][same-fd]") { + Context ctx; + VFS vfs(ctx); + + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } + + // Create a simple 1D vector with domain 1-10 and one attribute. + Domain domain(ctx); + domain.add_dimension(Dimension::create(ctx, "d", {{1, 10}}, 5)); + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + schema.set_cell_order(TILEDB_HILBERT); + schema.set_allows_dups(false); + schema.set_capacity(2); + + Attribute attr = Attribute::create(ctx, "a"); + Attribute attr2 = Attribute::create(ctx, "a2"); + int fill_value = -1; + attr.set_fill_value(&fill_value, sizeof(int)); + schema.add_attribute(attr); + schema.add_attribute(attr2); + Array::create(array_name, schema); + + // Open array for write. + Array array(ctx, array_name, TILEDB_WRITE); + + std::vector dim_vals = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + // Write all values as 3 in the array. + std::vector vals = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3}; + Query query_w(ctx, array, TILEDB_WRITE); + query_w.set_buffer("d", dim_vals); + query_w.set_buffer("a", vals); + query_w.set_buffer("a2", vals); + REQUIRE(query_w.submit() == Query::Status::COMPLETE); + + // Write values from 1-10 as 7 in the array. + std::vector vals2 = {7, 7, 7, 7, 7, 7, 7, 7, 7, 6}; + Query query_w2(ctx, array, TILEDB_WRITE); + query_w2.set_buffer("d", dim_vals); + query_w2.set_buffer("a", vals2); + query_w2.set_buffer("a2", vals); + REQUIRE(query_w2.submit() == Query::Status::COMPLETE); + + array.close(); + + // Open the array for read. + Array array_r(ctx, array_name, TILEDB_READ); + + // Query the data with query condition a < 7. + QueryCondition qc(ctx); + int val1 = 7; + qc.init("a", &val1, sizeof(int), TILEDB_LT); + + std::vector vals_read(10); + std::vector vals_read2(10); + std::vector dim_vals_read(10); + Query query_r(ctx, array_r, TILEDB_READ); + query_r.set_layout(TILEDB_GLOBAL_ORDER); + query_r.set_buffer("a", vals_read); + query_r.set_buffer("a2", vals_read2); + query_r.set_buffer("d", dim_vals_read); + query_r.set_condition(qc); + REQUIRE(query_r.submit() == Query::Status::COMPLETE); + + array_r.close(); + + if (vfs.is_dir(array_name)) { + vfs.remove_dir(array_name); + } +} + TEST_CASE( "Testing read query with basic QC, condition on dimension, with range " "within a tile.", diff --git a/tiledb/sm/query/readers/sparse_global_order_reader.cc b/tiledb/sm/query/readers/sparse_global_order_reader.cc index 3abf1518d68a..e80d5792c648 100644 --- a/tiledb/sm/query/readers/sparse_global_order_reader.cc +++ b/tiledb/sm/query/readers/sparse_global_order_reader.cc @@ -1651,15 +1651,18 @@ SparseGlobalOrderReader::respect_copy_memory_budget( // Get the size for all tiles. uint64_t idx = 0; for (; idx < max_cs_idx; idx++) { + auto rcs = result_cell_slabs[idx]; + if (rcs.length_ == 0) { + continue; + } + // Skip this tile if it's aggregate only and we can aggregate it with // the fragment metadata only. - if (agg_only && - can_aggregate_tile_with_frag_md(result_cell_slabs[idx])) { + if (agg_only && can_aggregate_tile_with_frag_md(rcs)) { continue; } - auto rt = static_cast*>( - result_cell_slabs[idx].tile_); + auto rt = static_cast*>(rcs.tile_); const auto f = rt->frag_idx(); const auto t = rt->tile_idx(); auto id = std::pair(f, t); From f2d32e706a8ac761780f2d5c739b956ae5501756 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 29 May 2024 03:39:39 -0400 Subject: [PATCH 382/456] Remove StorageManager::rest_client(). (#5000) Remove `StorageManager::rest_client()`. Rename `GroupStatusException` to `GroupException`. Rename `QueryStatusException` to `QueryException`. [sc-47942] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::rest_client()`. --- tiledb/sm/consolidator/consolidator.cc | 8 +- tiledb/sm/group/group.cc | 178 ++++++++++-------- tiledb/sm/query/query.cc | 103 +++++----- tiledb/sm/query/query.h | 7 +- tiledb/sm/query_plan/query_plan.cc | 8 +- tiledb/sm/storage_manager/storage_manager.cc | 11 +- .../storage_manager_canonical.h | 8 - 7 files changed, 170 insertions(+), 153 deletions(-) diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 1a6fe6050161..527972ece84e 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -155,8 +155,9 @@ void Consolidator::array_consolidate( } if (array_uri.is_tiledb()) { - throw_if_not_ok(storage_manager->rest_client()->post_consolidation_to_rest( - array_uri, config)); + throw_if_not_ok( + storage_manager->resources().rest_client()->post_consolidation_to_rest( + array_uri, config)); } else { // Get encryption key from config std::string encryption_key_from_cfg; @@ -317,7 +318,8 @@ void Consolidator::array_vacuum( URI array_uri(array_name); if (array_uri.is_tiledb()) { throw_if_not_ok( - storage_manager->rest_client()->post_vacuum_to_rest(array_uri, config)); + storage_manager->resources().rest_client()->post_vacuum_to_rest( + array_uri, config)); return; } diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 47e04ff83f2c..b17241b0b41a 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -56,9 +56,9 @@ using namespace tiledb::common; namespace tiledb::sm { -class GroupStatusException : public StatusException { +class GroupException : public StatusException { public: - explicit GroupStatusException(const std::string& message) + explicit GroupException(const std::string& message) : StatusException("Group", message) { } }; @@ -87,12 +87,12 @@ void Group::open( QueryType query_type, uint64_t timestamp_start, uint64_t timestamp_end) { // Checks if (is_open_) { - throw GroupStatusException("Cannot open group; Group already open"); + throw GroupException("Cannot open group; Group already open"); } if (query_type != QueryType::READ && query_type != QueryType::WRITE && query_type != QueryType::MODIFY_EXCLUSIVE) { - throw GroupStatusException("Cannot open group; Unsupported query type"); + throw GroupException("Cannot open group; Unsupported query type"); } if (timestamp_end == UINT64_MAX) { @@ -137,9 +137,10 @@ void Group::open( } } - if (remote_ && encryption_type != EncryptionType::NO_ENCRYPTION) - throw GroupStatusException( + if (remote_ && encryption_type != EncryptionType::NO_ENCRYPTION) { + throw GroupException( "Cannot open group; encrypted remote groups are not supported."); + } // Copy the key bytes. throw_if_not_ok( @@ -149,9 +150,9 @@ void Group::open( metadata_loaded_ = false; if (remote_) { - auto rest_client = storage_manager_->rest_client(); + auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw GroupStatusException( + throw GroupException( "Cannot open group; remote group with no REST client."); } @@ -229,18 +230,20 @@ void Group::close() { // Set metadata loaded to be true so when serialization fetches the // metadata it won't trigger a deadlock metadata_loaded_ = true; - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - throw GroupStatusException( + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw GroupException( "Error closing group; remote group with no REST client."); + } throw_if_not_ok( rest_client->put_group_metadata_to_rest(group_uri_, this)); } if (!members_to_modify().empty()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - throw GroupStatusException( + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw GroupException( "Error closing group; remote group with no REST client."); + } throw_if_not_ok(rest_client->patch_group_to_rest(group_uri_, this)); } } @@ -254,9 +257,9 @@ void Group::close() { try { close_for_writes(); } catch (StatusException& exc) { - std::string msg = exc.what(); - msg += " : Was storage for the group moved or deleted before closing?"; - throw GroupStatusException(msg); + throw GroupException( + std::string(exc.what()) + + " : Was storage for the group moved or deleted before closing?"); } } } @@ -286,7 +289,7 @@ const shared_ptr Group::group_details() const { QueryType Group::get_query_type() const { // Error if the group is not open if (!is_open_) { - throw GroupStatusException("Cannot get query_type; Group is not open"); + throw GroupException("Cannot get query_type; Group is not open"); } return query_type_; @@ -295,21 +298,20 @@ QueryType Group::get_query_type() const { void Group::delete_group(const URI& uri, bool recursive) { // Check that group is open if (!is_open_) { - throw GroupStatusException("[delete_group] Group is not open"); + throw GroupException("[delete_group] Group is not open"); } // Check that query type is MODIFY_EXCLUSIVE if (query_type_ != QueryType::MODIFY_EXCLUSIVE) { - throw GroupStatusException( - "[delete_group] Query type must be MODIFY_EXCLUSIVE"); + throw GroupException("[delete_group] Query type must be MODIFY_EXCLUSIVE"); } // Delete group data if (remote_) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - throw GroupStatusException( - "[delete_group] Remote group with no REST client."); + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw GroupException("[delete_group] Remote group with no REST client."); + } rest_client->delete_group_from_rest(uri, recursive); } else { // Delete group members within the group when deleting recursively @@ -362,19 +364,22 @@ void Group::delete_group(const URI& uri, bool recursive) { void Group::delete_metadata(const char* key) { // Check if group is open - if (!is_open_) - throw GroupStatusException("Cannot delete metadata. Group is not open"); + if (!is_open_) { + throw GroupException("Cannot delete metadata. Group is not open"); + } // Check mode if (query_type_ != QueryType::WRITE && - query_type_ != QueryType::MODIFY_EXCLUSIVE) - throw GroupStatusException( + query_type_ != QueryType::MODIFY_EXCLUSIVE) { + throw GroupException( "Cannot delete metadata. Group was not opened in write or " "modify_exclusive mode"); + } // Check if key is null - if (key == nullptr) - throw GroupStatusException("Cannot delete metadata. Key cannot be null"); + if (key == nullptr) { + throw GroupException("Cannot delete metadata. Key cannot be null"); + } metadata_.del(key); } @@ -385,23 +390,27 @@ void Group::put_metadata( uint32_t value_num, const void* value) { // Check if group is open - if (!is_open_) - throw GroupStatusException("Cannot put metadata; Group is not open"); + if (!is_open_) { + throw GroupException("Cannot put metadata; Group is not open"); + } // Check mode if (query_type_ != QueryType::WRITE && - query_type_ != QueryType::MODIFY_EXCLUSIVE) - throw GroupStatusException( + query_type_ != QueryType::MODIFY_EXCLUSIVE) { + throw GroupException( "Cannot put metadata; Group was not opened in write or " "modify_exclusive mode"); + } // Check if key is null - if (key == nullptr) - throw GroupStatusException("Cannot put metadata; Key cannot be null"); + if (key == nullptr) { + throw GroupException("Cannot put metadata; Key cannot be null"); + } // Check if value type is ANY - if (value_type == Datatype::ANY) - throw GroupStatusException("Cannot put metadata; Value type cannot be ANY"); + if (value_type == Datatype::ANY) { + throw GroupException("Cannot put metadata; Value type cannot be ANY"); + } metadata_.put(key, value_type, value_num, value); } @@ -412,21 +421,25 @@ void Group::get_metadata( uint32_t* value_num, const void** value) { // Check if group is open - if (!is_open_) - throw GroupStatusException("Cannot get metadata; Group is not open"); + if (!is_open_) { + throw GroupException("Cannot get metadata; Group is not open"); + } // Check mode - if (query_type_ != QueryType::READ) - throw GroupStatusException( + if (query_type_ != QueryType::READ) { + throw GroupException( "Cannot get metadata; Group was not opened in read mode"); + } // Check if key is null - if (key == nullptr) - throw GroupStatusException("Cannot get metadata; Key cannot be null"); + if (key == nullptr) { + throw GroupException("Cannot get metadata; Key cannot be null"); + } // Load group metadata, if not loaded yet - if (!metadata_loaded_) + if (!metadata_loaded_) { load_metadata(); + } metadata_.get(key, value_type, value_num, value); } @@ -439,56 +452,65 @@ void Group::get_metadata( uint32_t* value_num, const void** value) { // Check if group is open - if (!is_open_) - throw GroupStatusException("Cannot get metadata; Group is not open"); + if (!is_open_) { + throw GroupException("Cannot get metadata; Group is not open"); + } // Check mode - if (query_type_ != QueryType::READ) - throw GroupStatusException( + if (query_type_ != QueryType::READ) { + throw GroupException( "Cannot get metadata; Group was not opened in read mode"); + } // Load group metadata, if not loaded yet - if (!metadata_loaded_) + if (!metadata_loaded_) { load_metadata(); + } metadata_.get(index, key, key_len, value_type, value_num, value); } uint64_t Group::get_metadata_num() { // Check if group is open - if (!is_open_) - throw GroupStatusException( - "Cannot get number of metadata; Group is not open"); + if (!is_open_) { + throw GroupException("Cannot get number of metadata; Group is not open"); + } // Check mode - if (query_type_ != QueryType::READ) - throw GroupStatusException( + if (query_type_ != QueryType::READ) { + throw GroupException( "Cannot get number of metadata; Group was not opened in read mode"); + } // Load group metadata, if not loaded yet - if (!metadata_loaded_) + if (!metadata_loaded_) { load_metadata(); + } return metadata_.num(); } std::optional Group::metadata_type(const char* key) { // Check if group is open - if (!is_open_) - throw GroupStatusException("Cannot get metadata; Group is not open"); + if (!is_open_) { + throw GroupException("Cannot get metadata; Group is not open"); + } // Check mode - if (query_type_ != QueryType::READ) - throw GroupStatusException( + if (query_type_ != QueryType::READ) { + throw GroupException( "Cannot get metadata; Group was not opened in read mode"); + } // Check if key is null - if (key == nullptr) - throw GroupStatusException("Cannot get metadata; Key cannot be null"); + if (key == nullptr) { + throw GroupException("Cannot get metadata; Key cannot be null"); + } // Load group metadata, if not loaded yet - if (!metadata_loaded_) + if (!metadata_loaded_) { load_metadata(); + } return metadata_.metadata_type(key); } @@ -523,7 +545,7 @@ const Config& Group::config() const { void Group::set_config(Config config) { if (is_open()) { - throw GroupStatusException("[set_config] Cannot set config; Group is open"); + throw GroupException("[set_config] Cannot set config; Group is open"); } config_.inherit(config); } @@ -549,13 +571,13 @@ void Group::mark_member_for_addition( std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - throw GroupStatusException("Cannot add member; Group is not open"); + throw GroupException("Cannot add member; Group is not open"); } // Check mode if (query_type_ != QueryType::WRITE && query_type_ != QueryType::MODIFY_EXCLUSIVE) { - throw GroupStatusException( + throw GroupException( "Cannot get member; Group was not opened in write or modify_exclusive " "mode"); } @@ -567,14 +589,13 @@ void Group::mark_member_for_removal(const std::string& name) { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - throw GroupStatusException( - "Cannot mark member for removal; Group is not open"); + throw GroupException("Cannot mark member for removal; Group is not open"); } // Check mode if (query_type_ != QueryType::WRITE && query_type_ != QueryType::MODIFY_EXCLUSIVE) { - throw GroupStatusException( + throw GroupException( "Cannot get member; Group was not opened in write or modify_exclusive " "mode"); } @@ -617,12 +638,12 @@ uint64_t Group::member_count() const { std::lock_guard lck(mtx_); // Check if group is open if (!is_open_) { - throw GroupStatusException("Cannot get member count; Group is not open"); + throw GroupException("Cannot get member count; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw GroupStatusException( + throw GroupException( "Cannot get member; Group was not opened in read mode"); } @@ -635,12 +656,12 @@ tuple> Group::member_by_index( // Check if group is open if (!is_open_) { - throw GroupStatusException("Cannot get member by index; Group is not open"); + throw GroupException("Cannot get member by index; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw GroupStatusException( + throw GroupException( "Cannot get member; Group was not opened in read mode"); } @@ -653,12 +674,12 @@ Group::member_by_name(const std::string& name) { // Check if group is open if (!is_open_) { - throw GroupStatusException("Cannot get member by name; Group is not open"); + throw GroupException("Cannot get member by name; Group is not open"); } // Check mode if (query_type_ != QueryType::READ) { - throw GroupStatusException( + throw GroupException( "Cannot get member; Group was not opened in read mode"); } @@ -715,9 +736,10 @@ std::string Group::dump( void Group::load_metadata() { if (remote_) { auto rest_client = resources_.rest_client(); - if (rest_client == nullptr) - throw GroupStatusException( + if (rest_client == nullptr) { + throw GroupException( "Cannot load metadata; remote group with no REST client."); + } throw_if_not_ok( rest_client->post_group_metadata_from_rest(group_uri_, this)); } else { diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 55c29641ef2b..dff7d3caeae1 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -228,7 +228,7 @@ FieldDataSize Query::internal_est_result_size(std::string_view field_name) { if (array_->is_remote() && !subarray_.est_result_size_computed()) { auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw QueryStatusException( + throw QueryException( "Error in query estimate result size; " "remote array with no rest client."); } @@ -447,20 +447,21 @@ Status Query::finalize() { } if (array_->is_remote() && type_ == QueryType::WRITE) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - return logger_->status(Status_QueryError( - "Error in query finalize; remote array with no rest client.")); + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw QueryException( + "Failed to finalize query; remote array with no rest client."); + } if (layout_ == Layout::GLOBAL_ORDER) { - return logger_->status(Status_QueryError( - "Error in query finalize; remote global order writes are only " - "allowed to call submit_and_finalize to submit the last tile")); + throw QueryException( + "Failed to finalize query; remote global order writes are only " + "allowed to call submit_and_finalize to submit the last tile"); } return rest_client->finalize_query_to_rest(array_->array_uri(), this); } - RETURN_NOT_OK(strategy_->finalize()); + throw_if_not_ok(strategy_->finalize()); status_ = QueryStatus::COMPLETED; return Status::Ok(); @@ -468,32 +469,33 @@ Status Query::finalize() { Status Query::submit_and_finalize() { if (type_ != QueryType::WRITE || layout_ != Layout::GLOBAL_ORDER) { - return logger_->status( - Status_QueryError("Error in query submit_and_finalize; Call valid only " - "in global_order writes.")); + throw QueryException( + "Failed to submit and finalize query; Call valid only in global_order " + "writes."); } // Check attribute/dimensions buffers completeness before query submits - RETURN_NOT_OK(check_buffers_correctness()); + throw_if_not_ok(check_buffers_correctness()); if (array_->is_remote()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - return logger_->status( - Status_QueryError("Error in query submit_and_finalize; remote array " - "with no rest client.")); + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw QueryException( + "Failed to submit and finalize query; remote array with no rest " + "client."); + } if (status_ == QueryStatus::UNINITIALIZED) { - RETURN_NOT_OK(create_strategy()); + throw_if_not_ok(create_strategy()); } return rest_client->submit_and_finalize_query_to_rest( array_->array_uri(), this); } init(); - RETURN_NOT_OK(storage_manager_->query_submit(this)); + throw_if_not_ok(storage_manager_->query_submit(this)); - RETURN_NOT_OK(strategy_->finalize()); + throw_if_not_ok(strategy_->finalize()); status_ = QueryStatus::COMPLETED; return Status::Ok(); @@ -662,14 +664,14 @@ void Query::init() { status_ == QueryStatus::INITIALIZED) { // Check if the array got closed if (array_ == nullptr || !array_->is_open()) { - throw QueryStatusException( + throw QueryException( "Cannot init query; The associated array is not open"); } // Check if the array got re-opened with a different query type QueryType array_query_type{array_->get_query_type()}; if (array_query_type != type_) { - throw QueryStatusException( + throw QueryException( "Cannot init query; Associated array query type does not match " "query type: (" + query_type_str(array_query_type) + " != " + query_type_str(type_) + @@ -681,14 +683,14 @@ void Query::init() { // Create dimension label queries and remove labels from subarray. if (uses_dimension_labels()) { if (condition_.has_value()) { - throw QueryStatusException( + throw QueryException( "Cannot init query; Using query conditions and dimension labels " "together is not supported."); } // Check the layout is valid. if (layout_ == Layout::GLOBAL_ORDER) { - throw QueryStatusException( + throw QueryException( "Cannot init query; The global order layout is not supported " "when querying dimension labels"); } @@ -700,7 +702,7 @@ void Query::init() { if (!only_dim_label_query() && type_ == QueryType::READ && !array_schema_->dense() && array_schema_->dim_num() > 1 && !label_buffers_.empty()) { - throw QueryStatusException( + throw QueryException( "Cannot initialize query; Reading dimension label data is not " "yet supported on sparse arrays with multiple dimensions."); } @@ -936,7 +938,7 @@ void Query::set_processed_conditions( void Query::set_config(const Config& config) { if (!remote_query_ && status_ != QueryStatus::UNINITIALIZED) { - throw QueryStatusException( + throw QueryException( "[set_config] Cannot set config after initialization."); } config_.inherit(config); @@ -1007,13 +1009,13 @@ Status Query::set_data_buffer( if (array_schema_->is_dim_label(name)) { // Check the query type is valid. if (type_ != QueryType::READ && type_ != QueryType::WRITE) { - throw QueryStatusException("[set_data_buffer] Unsupported query type."); + throw QueryException("[set_data_buffer] Unsupported query type."); } const bool exists = label_buffers_.find(name) != label_buffers_.end(); if (status_ != QueryStatus::UNINITIALIZED && !exists && !serialization_allow_new_attr) { - throw QueryStatusException( + throw QueryException( "[set_data_buffer] Cannot set buffer for new dimension label '" + name + "' after initialization"); } @@ -1157,13 +1159,12 @@ Status Query::set_offsets_buffer( if (array_schema_->is_dim_label(name)) { // Check the query type is valid. if (type_ != QueryType::READ && type_ != QueryType::WRITE) { - throw QueryStatusException( - "[set_offsets_buffer] Unsupported query type."); + throw QueryException("[set_offsets_buffer] Unsupported query type."); } // Check the dimension label is in fact variable length. if (!array_schema_->dimension_label(name).is_var()) { - throw QueryStatusException( + throw QueryException( "[set_offsets_buffer] Input dimension label '" + name + "' is fixed-sized"); } @@ -1172,7 +1173,7 @@ Status Query::set_offsets_buffer( const bool exists = label_buffers_.find(name) != label_buffers_.end(); if (status_ != QueryStatus::UNINITIALIZED && !exists && !serialization_allow_new_attr) { - throw QueryStatusException( + throw QueryException( "[set_offsets_buffer] Cannot set buffer for new dimension label '" + name + "' after initialization"); } @@ -1456,7 +1457,7 @@ void Query::set_subarray(const void* subarray) { case QueryType::WRITE: case QueryType::MODIFY_EXCLUSIVE: if (!array_schema_->dense()) { - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray is not supported on " "sparse writes."); } @@ -1464,7 +1465,7 @@ void Query::set_subarray(const void* subarray) { default: - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray is not supported for query type " "'" + query_type_str(type_) + "'."); @@ -1472,7 +1473,7 @@ void Query::set_subarray(const void* subarray) { // Check this isn't an already initialized query using dimension labels. if (status_ != QueryStatus::UNINITIALIZED) { - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray on an already initialized " "query is not supported."); } @@ -1499,14 +1500,14 @@ void Query::set_subarray(const tiledb::sm::Subarray& subarray) { case QueryType::WRITE: case QueryType::MODIFY_EXCLUSIVE: if (!array_schema_->dense()) { - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray is not supported on " "sparse writes."); } break; default: - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray is not supported for query type " "'" + query_type_str(type_) + "'."); @@ -1514,7 +1515,7 @@ void Query::set_subarray(const tiledb::sm::Subarray& subarray) { // Check the query has not been initialized. if (status_ != tiledb::sm::QueryStatus::UNINITIALIZED) { - throw QueryStatusException( + throw QueryException( "[set_subarray] Setting a subarray on an already initialized " "query is not supported."); } @@ -1553,22 +1554,23 @@ Status Query::submit() { // Make sure fragment size is only set for global order. if (fragment_size_ != std::numeric_limits::max() && (layout_ != Layout::GLOBAL_ORDER || type_ != QueryType::WRITE)) { - throw QueryStatusException( + throw QueryException( "[submit] Fragment size is only supported for global order writes."); } // Check attribute/dimensions buffers completeness before query submits - RETURN_NOT_OK(check_buffers_correctness()); + throw_if_not_ok(check_buffers_correctness()); if (array_->is_remote()) { - auto rest_client = storage_manager_->rest_client(); - if (rest_client == nullptr) - return logger_->status(Status_QueryError( - "Error in query submission; remote array with no rest client.")); + auto rest_client = resources_.rest_client(); + if (rest_client == nullptr) { + throw QueryException( + "Failed to submit query; remote array with no rest client."); + } if (status_ == QueryStatus::UNINITIALIZED && !only_dim_label_query() && !subarray_.has_label_ranges()) { - RETURN_NOT_OK(create_strategy()); + throw_if_not_ok(create_strategy()); // Allocate remote buffer storage for global order writes if necessary. // If we cache an entire write a query may be uninitialized for N submits. @@ -1578,13 +1580,14 @@ Status Query::submit() { } } - RETURN_NOT_OK(rest_client->submit_query_to_rest(array_->array_uri(), this)); + throw_if_not_ok( + rest_client->submit_query_to_rest(array_->array_uri(), this)); reset_coords_markers(); return Status::Ok(); } init(); - RETURN_NOT_OK(storage_manager_->query_submit(this)); + throw_if_not_ok(storage_manager_->query_submit(this)); reset_coords_markers(); return Status::Ok(); @@ -1977,7 +1980,7 @@ Status Query::check_buffer_names() { for (unsigned d = 0; d < array_schema_->dim_num(); d++) { auto dim = array_schema_->dimension_ptr(d); if (buffers_.count(dim->name()) == 0) { - throw QueryStatusException( + throw QueryException( "[check_buffer_names] Dimension buffer " + dim->name() + " is not set"); } @@ -2107,7 +2110,7 @@ void Query::copy_aggregates_data_to_user_buffer() { } RestClient* Query::rest_client() const { - return storage_manager_->rest_client(); + return resources_.rest_client().get(); } } // namespace tiledb::sm diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 16a0e61edd25..08dd6e022126 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -68,7 +68,6 @@ class QueryException : public StatusException { : StatusException("Query", msg) { } }; -using QueryStatusException = QueryException; class Array; class ArrayDimensionLabelQueries; @@ -870,7 +869,7 @@ class Query { } return; } - throw QueryStatusException( + throw QueryException( "We currently only support a default channel for queries"); } @@ -905,13 +904,13 @@ class Query { inline std::shared_ptr aggegate_channel() { if (!has_aggregates()) { - throw QueryStatusException("Aggregate channel does not exist"); + throw QueryException("Aggregate channel does not exist"); } return aggregate_channel_; } /** - * Returns the REST client configured in the storage manager associated to + * Returns the REST client configured in the context resources associated to * this query */ RestClient* rest_client() const; diff --git a/tiledb/sm/query_plan/query_plan.cc b/tiledb/sm/query_plan/query_plan.cc index 12df72b6dde5..a43073520873 100644 --- a/tiledb/sm/query_plan/query_plan.cc +++ b/tiledb/sm/query_plan/query_plan.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -44,8 +44,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ @@ -139,5 +138,4 @@ std::string QueryPlan::dump_json(uint32_t indent) { return rv.dump(indent); } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 8252e6dc63d2..f8f77dc78446 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -240,7 +240,7 @@ Status StorageManager::array_evolve_schema( } if (array_uri.is_tiledb()) { - return rest_client()->post_array_schema_evolution_to_rest( + return resources_.rest_client()->post_array_schema_evolution_to_rest( array_uri, schema_evolution); } @@ -486,7 +486,7 @@ Status StorageManagerCanonical::group_create(const std::string& group_uri) { // Check if group exists bool exists; - RETURN_NOT_OK(is_group(resources_, uri, &exists)); + throw_if_not_ok(is_group(resources_, uri, &exists)); if (exists) return logger_->status(Status_StorageManagerError( std::string("Cannot create group; Group '") + uri.c_str() + @@ -496,7 +496,8 @@ Status StorageManagerCanonical::group_create(const std::string& group_uri) { if (uri.is_tiledb()) { Group group(resources_, uri, this); - RETURN_NOT_OK(rest_client()->post_group_create_to_rest(uri, &group)); + throw_if_not_ok( + resources_.rest_client()->post_group_create_to_rest(uri, &group)); return Status::Ok(); } @@ -731,8 +732,8 @@ Status StorageManagerCanonical::set_tag( tags_[key] = value; // Tags are added to REST requests as HTTP headers. - if (rest_client() != nullptr) - RETURN_NOT_OK(rest_client()->set_header(key, value)); + if (resources_.rest_client() != nullptr) + throw_if_not_ok(resources_.rest_client()->set_header(key, value)); return Status::Ok(); } diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 2e9fa3baad99..c5e38a3aebdd 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -209,14 +209,6 @@ class StorageManagerCanonical { */ Status group_create(const std::string& group); - /** - * If the storage manager was configured with a REST server, return the - * client instance. Else, return nullptr. - */ - inline RestClient* rest_client() const { - return resources_.rest_client().get(); - } - /** Removes a TileDB object (group, array). */ Status object_remove(const char* path) const; From afc03ad631b4ff483c223f6205e311386cbde3f1 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 29 May 2024 03:42:51 -0400 Subject: [PATCH 383/456] Migrate APIs out of StorageManager: object_type, object_move, object_remove. (#5017) Migrate APIs out of `StorageManager`: `object_type`, `object_move`, `object_remove`. Also remove some extraneous inclusions from `StorageManager`. [sc-46737] [sc-46732] [sc-46731] --- TYPE: IMPROVEMENT DESC: Migrate APIs out of `StorageManager`: `object_type`, `object_move`, `object_remove`. --- tiledb/sm/c_api/tiledb.cc | 3 +- tiledb/sm/consolidator/consolidator.cc | 7 +- tiledb/sm/group/group.cc | 2 +- tiledb/sm/group/group_details.cc | 8 +-- tiledb/sm/group/group_details.h | 5 +- tiledb/sm/object/object.cc | 38 ++++++++++ tiledb/sm/object/object.h | 13 ++++ tiledb/sm/storage_manager/storage_manager.cc | 69 ++++--------------- .../storage_manager_canonical.h | 19 ----- 9 files changed, 79 insertions(+), 85 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 9748f3cee07b..86c7c6e5fb16 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -72,6 +72,7 @@ #include "tiledb/sm/enums/serialization_type.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/object/object.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/rest/rest_client.h" @@ -3075,7 +3076,7 @@ int32_t tiledb_object_type( tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) { auto uri = tiledb::sm::URI(path); tiledb::sm::ObjectType object_type; - throw_if_not_ok(ctx->storage_manager()->object_type(uri, &object_type)); + throw_if_not_ok(tiledb::sm::object_type(ctx->resources(), uri, &object_type)); *type = static_cast(object_type); return TILEDB_OK; diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 527972ece84e..b0e711b2b2be 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -40,6 +40,7 @@ #include "tiledb/sm/consolidator/fragment_meta_consolidator.h" #include "tiledb/sm/consolidator/group_meta_consolidator.h" #include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/object/object.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/generate_uri.h" @@ -147,7 +148,8 @@ void Consolidator::array_consolidate( // Check if array exists ObjectType obj_type; - throw_if_not_ok(storage_manager->object_type(array_uri, &obj_type)); + throw_if_not_ok( + object_type(storage_manager->resources(), array_uri, &obj_type)); if (obj_type != ObjectType::ARRAY) { throw ConsolidatorException( @@ -210,7 +212,8 @@ void Consolidator::fragments_consolidate( // Check if array exists ObjectType obj_type; - throw_if_not_ok(storage_manager->object_type(array_uri, &obj_type)); + throw_if_not_ok( + object_type(storage_manager->resources(), array_uri, &obj_type)); if (obj_type != ObjectType::ARRAY) { throw ConsolidatorException( diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index b17241b0b41a..2a2d64773dea 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -582,7 +582,7 @@ void Group::mark_member_for_addition( "mode"); } group_details_->mark_member_for_addition( - group_member_uri, relative, name, storage_manager_); + resources_, group_member_uri, relative, name); } void Group::mark_member_for_removal(const std::string& name) { diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index 8853775ca336..3c5e9174bd54 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -43,6 +43,7 @@ #include "tiledb/sm/group/group_member_v2.h" #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/object/object.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/tile/generic_tile_io.h" @@ -81,10 +82,10 @@ void GroupDetails::delete_member(const shared_ptr group_member) { } void GroupDetails::mark_member_for_addition( + ContextResources& resources, const URI& group_member_uri, const bool& relative, - std::optional& name, - StorageManager* storage_manager) { + std::optional& name) { std::lock_guard lck(mtx_); URI absolute_group_member_uri = group_member_uri; if (relative) { @@ -92,8 +93,7 @@ void GroupDetails::mark_member_for_addition( group_uri_.join_path(group_member_uri.to_string()); } ObjectType type = ObjectType::INVALID; - throw_if_not_ok( - storage_manager->object_type(absolute_group_member_uri, &type)); + throw_if_not_ok(object_type(resources, absolute_group_member_uri, &type)); if (type == ObjectType::INVALID) { throw GroupDetailsException( "Cannot add group member " + absolute_group_member_uri.to_string() + diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index 88fec9599d46..cf95bbd0838a 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -63,15 +63,16 @@ class GroupDetails { /** * Add a member to a group, this will be flushed to disk on close * + * @param resources the context resources * @param group_member_uri group member uri * @param relative is this URI relative * @param name optional name for member */ void mark_member_for_addition( + ContextResources& resources, const URI& group_member_uri, const bool& relative, - std::optional& name, - StorageManager* storage_manager); + std::optional& name); /** * Remove a member from a group, this will be flushed to disk on close diff --git a/tiledb/sm/object/object.cc b/tiledb/sm/object/object.cc index a7bb39a42b80..bdb9d429017c 100644 --- a/tiledb/sm/object/object.cc +++ b/tiledb/sm/object/object.cc @@ -31,8 +31,11 @@ */ #include "tiledb/sm/object/object.h" +#include "tiledb/common/stdx_string.h" +#include "tiledb/sm/enums/object_type.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/rest/rest_client.h" +#include "tiledb/storage_format/uri/parse_uri.h" namespace tiledb::sm { @@ -91,4 +94,39 @@ Status is_group(ContextResources& resources, const URI& uri, bool* is_group) { return Status::Ok(); } +Status object_type( + ContextResources& resources, const URI& uri, ObjectType* type) { + URI dir_uri = uri; + if (uri.is_s3() || uri.is_azure() || uri.is_gcs()) { + // Always add a trailing '/' in the S3/Azure/GCS case so that listing the + // URI as a directory will work as expected. Listing a non-directory object + // is not an error for S3/Azure/GCS. + auto uri_str = uri.to_string(); + dir_uri = + URI(utils::parse::ends_with(uri_str, "/") ? uri_str : (uri_str + "/")); + } else if (!uri.is_tiledb()) { + // For non public cloud backends, listing a non-directory is an error. + bool is_dir = false; + throw_if_not_ok(resources.vfs().is_dir(uri, &is_dir)); + if (!is_dir) { + *type = ObjectType::INVALID; + return Status::Ok(); + } + } + bool exists = is_array(resources, uri); + if (exists) { + *type = ObjectType::ARRAY; + return Status::Ok(); + } + + throw_if_not_ok(is_group(resources, uri, &exists)); + if (exists) { + *type = ObjectType::GROUP; + return Status::Ok(); + } + + *type = ObjectType::INVALID; + return Status::Ok(); +} + } // namespace tiledb::sm diff --git a/tiledb/sm/object/object.h b/tiledb/sm/object/object.h index 75fc785431a3..12876783cffe 100644 --- a/tiledb/sm/object/object.h +++ b/tiledb/sm/object/object.h @@ -41,6 +41,8 @@ using namespace tiledb::common; namespace tiledb::sm { +enum class ObjectType : uint8_t; + /* ********************************* */ /* API */ /* ********************************* */ @@ -65,6 +67,17 @@ bool is_array(ContextResources& resources, const URI& uri); */ Status is_group(ContextResources& resources, const URI& uri, bool* is_group); +/** + * Returns the tiledb object type + * + * @param resources the context resources. + * @param uri Path to TileDB object resource + * @param type The ObjectType to be retrieved. + * @return Status + */ +Status object_type( + ContextResources& resources, const URI& uri, ObjectType* type); + } // namespace tiledb::sm #endif diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index f8f77dc78446..08ccf0d7fcf5 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -41,38 +41,28 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" #include "tiledb/common/memory.h" -#include "tiledb/common/memory_tracker.h" -#include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/consolidator/consolidator.h" -#include "tiledb/sm/consolidator/fragment_consolidator.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/layout.h" #include "tiledb/sm/enums/object_type.h" -#include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/filesystem/vfs.h" -#include "tiledb/sm/fragment/fragment_info.h" #include "tiledb/sm/global_state/global_state.h" #include "tiledb/sm/global_state/unit_test_config.h" #include "tiledb/sm/group/group.h" -#include "tiledb/sm/group/group_details_v1.h" -#include "tiledb/sm/group/group_details_v2.h" #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/object/object.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/rest/rest_client.h" -#include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/sm/tile/tile.h" -#include "tiledb/storage_format/uri/generate_uri.h" -#include "tiledb/storage_format/uri/parse_uri.h" #include #include @@ -441,7 +431,7 @@ Status StorageManagerCanonical::object_remove(const char* path) const { std::string("Cannot remove object '") + path + "'; Invalid URI")); ObjectType obj_type; - RETURN_NOT_OK(object_type(uri, &obj_type)); + throw_if_not_ok(object_type(resources_, uri, &obj_type)); if (obj_type == ObjectType::INVALID) return logger_->status(Status_StorageManagerError( std::string("Cannot remove object '") + path + @@ -463,7 +453,7 @@ Status StorageManagerCanonical::object_move( std::string("Cannot move object to '") + new_path + "'; Invalid URI")); ObjectType obj_type; - RETURN_NOT_OK(object_type(old_uri, &obj_type)); + throw_if_not_ok(object_type(resources_, old_uri, &obj_type)); if (obj_type == ObjectType::INVALID) return logger_->status(Status_StorageManagerError( std::string("Cannot move object '") + old_path + @@ -524,41 +514,6 @@ void StorageManagerCanonical::increment_in_progress() { queries_in_progress_cv_.notify_all(); } -Status StorageManagerCanonical::object_type( - const URI& uri, ObjectType* type) const { - URI dir_uri = uri; - if (uri.is_s3() || uri.is_azure() || uri.is_gcs()) { - // Always add a trailing '/' in the S3/Azure/GCS case so that listing the - // URI as a directory will work as expected. Listing a non-directory object - // is not an error for S3/Azure/GCS. - auto uri_str = uri.to_string(); - dir_uri = - URI(utils::parse::ends_with(uri_str, "/") ? uri_str : (uri_str + "/")); - } else if (!uri.is_tiledb()) { - // For non public cloud backends, listing a non-directory is an error. - bool is_dir = false; - throw_if_not_ok(resources_.vfs().is_dir(uri, &is_dir)); - if (!is_dir) { - *type = ObjectType::INVALID; - return Status::Ok(); - } - } - bool exists = is_array(resources_, uri); - if (exists) { - *type = ObjectType::ARRAY; - return Status::Ok(); - } - - RETURN_NOT_OK(is_group(resources_, uri, &exists)); - if (exists) { - *type = ObjectType::GROUP; - return Status::Ok(); - } - - *type = ObjectType::INVALID; - return Status::Ok(); -} - Status StorageManagerCanonical::object_iter_begin( ObjectIter** obj_iter, const char* path, WalkOrder order) { // Sanity check @@ -580,7 +535,8 @@ Status StorageManagerCanonical::object_iter_begin( // Include the uris that are TileDB objects in the iterator state ObjectType obj_type; for (auto& uri : uris) { - RETURN_NOT_OK_ELSE(object_type(uri, &obj_type), tdb_delete(*obj_iter)); + RETURN_NOT_OK_ELSE( + object_type(resources_, uri, &obj_type), tdb_delete(*obj_iter)); if (obj_type != ObjectType::INVALID) { (*obj_iter)->objs_.push_back(uri); if (order == WalkOrder::POSTORDER) @@ -612,9 +568,10 @@ Status StorageManagerCanonical::object_iter_begin( // Include the uris that are TileDB objects in the iterator state ObjectType obj_type; for (auto& uri : uris) { - RETURN_NOT_OK(object_type(uri, &obj_type)); - if (obj_type != ObjectType::INVALID) + throw_if_not_ok(object_type(resources_, uri, &obj_type)); + if (obj_type != ObjectType::INVALID) { (*obj_iter)->objs_.push_back(uri); + } } return Status::Ok(); @@ -660,7 +617,7 @@ Status StorageManagerCanonical::object_iter_next_postorder( // Push the new TileDB objects in the front of the iterator's list ObjectType obj_type; for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - RETURN_NOT_OK(object_type(*it, &obj_type)); + throw_if_not_ok(object_type(resources_, *it, &obj_type)); if (obj_type != ObjectType::INVALID) { obj_iter->objs_.push_front(*it); obj_iter->expanded_.push_front(false); @@ -672,7 +629,7 @@ Status StorageManagerCanonical::object_iter_next_postorder( // Prepare the values to be returned URI front_uri = obj_iter->objs_.front(); obj_iter->next_ = front_uri.to_string(); - RETURN_NOT_OK(object_type(front_uri, type)); + throw_if_not_ok(object_type(resources_, front_uri, type)); *path = obj_iter->next_.c_str(); *has_next = true; @@ -688,7 +645,7 @@ Status StorageManagerCanonical::object_iter_next_preorder( // Prepare the values to be returned URI front_uri = obj_iter->objs_.front(); obj_iter->next_ = front_uri.to_string(); - RETURN_NOT_OK(object_type(front_uri, type)); + throw_if_not_ok(object_type(resources_, front_uri, type)); *path = obj_iter->next_.c_str(); *has_next = true; @@ -706,7 +663,7 @@ Status StorageManagerCanonical::object_iter_next_preorder( // Push the new TileDB objects in the front of the iterator's list ObjectType obj_type; for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - RETURN_NOT_OK(object_type(*it, &obj_type)); + throw_if_not_ok(object_type(resources_, *it, &obj_type)); if (obj_type != ObjectType::INVALID) obj_iter->objs_.push_front(*it); } @@ -847,7 +804,7 @@ Status StorageManagerCanonical::group_metadata_consolidate( } // Check if group exists ObjectType obj_type; - RETURN_NOT_OK(object_type(group_uri, &obj_type)); + throw_if_not_ok(object_type(resources_, group_uri, &obj_type)); if (obj_type != ObjectType::GROUP) { return logger_->status(Status_StorageManagerError( @@ -873,7 +830,7 @@ void StorageManagerCanonical::group_metadata_vacuum( // Check if group exists ObjectType obj_type; - throw_if_not_ok(object_type(group_uri, &obj_type)); + throw_if_not_ok(object_type(resources_, group_uri, &obj_type)); if (obj_type != ObjectType::GROUP) { throw Status_StorageManagerError( diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index c5e38a3aebdd..8ee1257c39fb 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -55,32 +55,22 @@ #include "tiledb/sm/group/group.h" #include "tiledb/sm/misc/cancelable_tasks.h" #include "tiledb/sm/misc/types.h" -#include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/context_resources.h" -#include "tiledb/sm/tile/filtered_buffer.h" using namespace tiledb::common; namespace tiledb::sm { class Array; -class OpenedArray; class ArrayDirectory; class ArraySchema; class ArraySchemaEvolution; -class Buffer; class Consolidator; class EncryptionKey; -class FragmentMetadata; -class FragmentInfo; -class GroupDetails; -class Metadata; -class MemoryTracker; class Query; class RestClient; enum class EncryptionType : uint8_t; -enum class ObjectType : uint8_t; /** The storage manager that manages pretty much everything in TileDB. */ class StorageManagerCanonical { @@ -291,15 +281,6 @@ class StorageManagerCanonical { ObjectType* type, bool* has_next); - /** - * Returns the tiledb object type - * - * @param uri Path to TileDB object resource - * @param type The ObjectType to be retrieved. - * @return Status - */ - Status object_type(const URI& uri, ObjectType* type) const; - /** Submits a query for (sync) execution. */ Status query_submit(Query* query); From 8a65644fffa86eae060129391279ae77acc7cf10 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 May 2024 11:12:12 +0300 Subject: [PATCH 384/456] Support `ls_recursive` in GCS. (#4997) [SC-47218](https://app.shortcut.com/tiledb-inc/story/47218/add-support-for-ls-recursive-in-gcs) This PR implements `ls_recursive` in the GCS VFS. Due to the structure of the GCS SDK APIs, adding a `GCSScanner` like Azure and S3 is not possible without refactoring the `LsScanner` infrastructure. Instead I only implemented `ls_filtered`, using a design that keeps the implementation in the `gcs.cc` implementation file and prevents SDK headers from leaking, which would regress build times. --- TYPE: C_API DESC: Support VFS `ls_recursive` API for Google Cloud Storage filesystem. --- test/src/unit-cppapi-vfs.cc | 3 +- test/src/unit-vfs.cc | 18 ++--- test/support/src/vfs_helpers.h | 16 ++++ .../c_api/vfs/test/unit_capi_ls_recursive.cc | 4 +- tiledb/api/c_api/vfs/vfs_api_experimental.h | 4 +- tiledb/sm/cpp_api/vfs_experimental.h | 4 +- tiledb/sm/filesystem/gcs.cc | 78 +++++++++++++++++++ tiledb/sm/filesystem/gcs.h | 62 +++++++++++++++ tiledb/sm/filesystem/vfs.h | 6 +- 9 files changed, 175 insertions(+), 20 deletions(-) diff --git a/test/src/unit-cppapi-vfs.cc b/test/src/unit-cppapi-vfs.cc index d7a2cf2368a4..2ad467363258 100644 --- a/test/src/unit-cppapi-vfs.cc +++ b/test/src/unit-cppapi-vfs.cc @@ -510,7 +510,8 @@ TEST_CASE( using ls_recursive_test_types = std::tuple< tiledb::test::LocalFsTest, tiledb::test::S3Test, - tiledb::test::AzureTest>; + tiledb::test::AzureTest, + tiledb::test::GCSTest>; TEMPLATE_LIST_TEST_CASE( "CPP API: VFS ls_recursive filter", "[cppapi][vfs][ls-recursive]", diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index 23a6aa7afff3..d2117965ec8d 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -622,9 +622,9 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { require_tiledb_ok(vfs_ls.remove_dir(URI(path))); } -// Currently only local, S3 and Azure is supported for VFS::ls_recursive. +// Currently only local, S3, Azure and GCS are supported for VFS::ls_recursive. // TODO: LocalFsTest currently fails. Fix and re-enable. -using TestBackends = std::tuple; +using TestBackends = std::tuple; TEMPLATE_LIST_TEST_CASE( "VFS: Test internal ls_filtered recursion argument", "[vfs][ls_filtered][recursion]", @@ -668,12 +668,7 @@ TEST_CASE( } std::string backend = vfs_test.temp_dir_.backend_name(); - if (vfs_test.temp_dir_.is_s3() || vfs_test.temp_dir_.is_azure()) { - DYNAMIC_SECTION(backend << " supported backend should not throw") { - CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( - vfs_test.temp_dir_, VFSTestBase::accept_all_files)); - } - } else { + if (vfs_test.temp_dir_.is_hdfs()) { DYNAMIC_SECTION(backend << " unsupported backend should throw") { CHECK_THROWS_WITH( vfs_test.vfs_.ls_recursive( @@ -681,13 +676,18 @@ TEST_CASE( Catch::Matchers::ContainsSubstring( "storage backend is not supported")); } + } else { + DYNAMIC_SECTION(backend << " supported backend should not throw") { + CHECK_NOTHROW(vfs_test.vfs_.ls_recursive( + vfs_test.temp_dir_, VFSTestBase::accept_all_files)); + } } } TEST_CASE( "VFS: Throwing FileFilter for ls_recursive", "[vfs][ls_recursive][file-filter]") { - std::string prefix = GENERATE("s3://", "azure://"); + std::string prefix = GENERATE("s3://", "azure://", "gcs://", "gs://"); VFSTest vfs_test({0}, prefix); if (!vfs_test.is_supported()) { return; diff --git a/test/support/src/vfs_helpers.h b/test/support/src/vfs_helpers.h index d1f3d9a7ac1c..9230dade9802 100644 --- a/test/support/src/vfs_helpers.h +++ b/test/support/src/vfs_helpers.h @@ -1131,6 +1131,22 @@ class GCSTest : public VFSTestBase { public: explicit GCSTest(const std::vector& test_tree) : VFSTestBase(test_tree, "gcs://") { +#ifdef HAVE_GCS + vfs_.create_bucket(temp_dir_).ok(); + for (size_t i = 1; i <= test_tree_.size(); i++) { + sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i)); + // VFS::create_dir is a no-op for GCS; Just create objects. + for (size_t j = 1; j <= test_tree_[i - 1]; j++) { + auto object_uri = path.join_path("test_file_" + std::to_string(j)); + vfs_.touch(object_uri).ok(); + std::string data(j * 10, 'a'); + vfs_.write(object_uri, data.data(), data.size()).ok(); + vfs_.close_file(object_uri).ok(); + expected_results().emplace_back(object_uri.to_string(), data.size()); + } + } + std::sort(expected_results().begin(), expected_results().end()); +#endif } }; diff --git a/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc index 453c029e2e21..24068784de4c 100644 --- a/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc +++ b/tiledb/api/c_api/vfs/test/unit_capi_ls_recursive.cc @@ -38,9 +38,9 @@ using namespace tiledb::test; -// Currently only local, S3 and Azure is supported for VFS::ls_recursive. +// Currently only local, S3, Azure and GCS are supported for VFS::ls_recursive. // TODO: LocalFsTest currently fails. Fix and re-enable. -using TestBackends = std::tuple; +using TestBackends = std::tuple; TEMPLATE_LIST_TEST_CASE( "C API: ls_recursive callback", "[vfs][ls-recursive]", TestBackends) { diff --git a/tiledb/api/c_api/vfs/vfs_api_experimental.h b/tiledb/api/c_api/vfs/vfs_api_experimental.h index cac4966bd6b7..2bb009f0c143 100644 --- a/tiledb/api/c_api/vfs/vfs_api_experimental.h +++ b/tiledb/api/c_api/vfs/vfs_api_experimental.h @@ -58,8 +58,8 @@ typedef int32_t (*tiledb_ls_callback_t)( * on error. The callback is responsible for writing gathered entries into the * `data` buffer, for example using a pointer to a user-defined struct. * - * Currently only local filesystem, S3 and Azure are supported, and the `path` - * must be a valid URI for one of those filesystems. + * Currently only local filesystem, S3, Azure and GCS are supported, and the + * `path` must be a valid URI for one of those filesystems. * * **Example:** * diff --git a/tiledb/sm/cpp_api/vfs_experimental.h b/tiledb/sm/cpp_api/vfs_experimental.h index 85152b4d6b2e..1a29742662d4 100644 --- a/tiledb/sm/cpp_api/vfs_experimental.h +++ b/tiledb/sm/cpp_api/vfs_experimental.h @@ -161,8 +161,8 @@ class VFSExperimental { * be included in the results and false otherwise. If no inclusion predicate * is provided, all results are returned. * - * Currently only local filesystem, S3 and Azure is supported, and the `path` - * must be a valid URI for one of those filesystems. + * Currently only local filesystem, S3, Azure and GCS are supported, and the + * `path` must be a valid URI for one of those filesystems. * * @code{.c} * VFSExperimental::LsInclude predicate = [](std::string_view path, diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index ccf3839394ed..540399f8f0e7 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -49,6 +49,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/logger.h" +#include "tiledb/common/stdx_string.h" #include "tiledb/common/unique_rwlock.h" #include "tiledb/platform/cert_file.h" #include "tiledb/sm/filesystem/gcs.h" @@ -551,6 +552,83 @@ tuple>> GCS::ls_with_sizes( return {Status::Ok(), entries}; } +LsObjects GCS::ls_filtered_impl( + const URI& uri, + std::function file_filter, + bool recursive) const { + throw_if_not_ok(init_client()); + + const URI uri_dir = uri.add_trailing_slash(); + + if (!uri_dir.is_gcs()) { + throw GCSException( + std::string("URI is not a GCS URI: " + uri_dir.to_string())); + } + + std::string prefix = uri_dir.backend_name() + "://"; + std::string bucket_name; + std::string object_path; + throw_if_not_ok(parse_gcs_uri(uri_dir, &bucket_name, &object_path)); + + LsObjects result; + + auto to_directory_entry = + [&bucket_name, &prefix](const google::cloud::storage::ObjectMetadata& obj) + -> LsObjects::value_type { + return { + prefix + bucket_name + "/" + + remove_front_slash(remove_trailing_slash(obj.name())), + obj.size()}; + }; + + if (recursive) { + auto it = client_->ListObjects( + bucket_name, google::cloud::storage::Prefix(std::move(object_path))); + for (const auto& object_metadata : it) { + if (!object_metadata) { + throw GCSException(std::string( + "List objects failed on: " + uri.to_string() + " (" + + object_metadata.status().message() + ")")); + } + + auto entry = to_directory_entry(*object_metadata); + if (file_filter(entry.first, entry.second)) { + result.emplace_back(std::move(entry)); + } + } + } else { + auto it = client_->ListObjectsAndPrefixes( + bucket_name, + google::cloud::storage::Prefix(std::move(object_path)), + google::cloud::storage::Delimiter("/")); + for (const auto& object_metadata : it) { + if (!object_metadata) { + throw GCSException(std::string( + "List objects failed on: " + uri.to_string() + " (" + + object_metadata.status().message() + ")")); + } + + LsObjects::value_type entry; + if (absl::holds_alternative( + *object_metadata)) { + entry = to_directory_entry( + absl::get( + *object_metadata)); + } else { + entry = { + prefix + bucket_name + "/" + + absl::get(*object_metadata), + 0}; + } + if (file_filter(entry.first, entry.second)) { + result.push_back(std::move(entry)); + } + } + } + + return result; +} + Status GCS::move_object(const URI& old_uri, const URI& new_uri) { RETURN_NOT_OK(init_client()); diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index e2d5c83441b9..f3b9ab294356 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -43,6 +43,7 @@ #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/curl/curl_init.h" +#include "tiledb/sm/filesystem/ls_scanner.h" #include "tiledb/sm/filesystem/ssl_config.h" #include "tiledb/sm/misc/constants.h" #include "uri.h" @@ -69,6 +70,14 @@ class directory_entry; namespace sm { +/** Class for GCS status exceptions. */ +class GCSException : public StatusException { + public: + explicit GCSException(const std::string& msg) + : StatusException("GCS", msg) { + } +}; + class GCS { public: /* ********************************* */ @@ -224,6 +233,32 @@ class GCS { const std::string& delimiter = "/", int max_paths = -1) const; + /** + * Lists objects and object information that start with `prefix`, invoking + * the FilePredicate on each entry collected and the DirectoryPredicate on + * common prefixes for pruning. + * + * @param parent The parent prefix to list sub-paths. + * @param file_filter The FilePredicate to invoke on each object for + * filtering. + * @param directory_filter The DirectoryPredicate to invoke on each common + * prefix for pruning. This is currently unused, but is kept here for future + * support. + * @param recursive Whether to recursively list subdirectories. + * @return Vector of results with each entry being a pair of the string URI + * and object size. + */ + template + LsObjects ls_filtered( + const URI& uri, + F file_filter, + [[maybe_unused]] D directory_filter = accept_all_dirs, + bool recursive = false) const { + // We use the constructor of std::function that accepts an F&& to convert + // the generic F to a polymorphic std::function. + return ls_filtered_impl(uri, std::move(file_filter), recursive); + } + /** * * Lists objects and object information that start with `prefix`. @@ -599,6 +634,33 @@ class GCS { */ Status is_bucket(const std::string& bucket_name, bool* const is_bucket) const; + /** + * Contains the implementation of ls_filtered. + * + * @section Notes + * + * The use of the non-generic std::function is necessary to keep the + * function's implementation in gcs.cc and avoid leaking the Google Cloud + * SDK headers, which would cause significant build performance regressions + * (see PR 4777). In the public-facing ls_filtered, we still use a generic + * callback which we convert. + * + * This has the consequence that the callback cannot capture variables that + * are not copy-constructible. It could be rectified with C++ 23's + * std::move_only_function, when it becomes available. + * + * @param uri The parent path to list sub-paths. + * @param file_filter The FilePredicate to invoke on each object for + * filtering. + * @param recursive Whether to recursively list subdirectories. + * @return Vector of results with each entry being a pair of the string URI + * and object size. + */ + LsObjects ls_filtered_impl( + const URI& uri, + std::function file_filter, + bool recursive) const; + /** * Thread-safe fetch of the write cache buffer in `write_cache_map_`. * If a buffer does not exist for `uri`, it will be created. diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 1c63ab5d1ce6..ac1a9e0dfa69 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -546,9 +546,7 @@ class VFS : private VFSBase, protected S3_within_VFS { #endif } else if (parent.is_gcs()) { #ifdef HAVE_GCS - throw filesystem::VFSException( - "Recursive ls over " + parent.backend_name() + - " storage backend is not supported."); + results = gcs_.ls_filtered(parent, f, d, recursive); #else throw filesystem::VFSException("TileDB was built without GCS support"); #endif @@ -586,7 +584,7 @@ class VFS : private VFSBase, protected S3_within_VFS { * invoking the FilePredicate on each entry collected and the * DirectoryPredicate on common prefixes for pruning. * - * Currently this API is only supported for local files, S3 and Azure. + * Currently this API is only supported for local files, S3, Azure and GCS. * * @param parent The parent prefix to list sub-paths. * @param f The FilePredicate to invoke on each object for filtering. From 474d59cab2e0921535072efa33c3677f01ceb51b Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 29 May 2024 14:10:01 +0200 Subject: [PATCH 385/456] Unstatus cache, global_state, metadata, rtree, tile directories. (#5014) This removes status from sm/cache, sm/global_state, sm/metadata, sm/rtree, sm/tile directories in favor of exceptions. [sc-48221] --- TYPE: NO_HISTORY DESC: Unstatus cache, global_state, metadata, rtree, tile directories. --------- Co-authored-by: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> --- tiledb/sm/array/array.cc | 5 +-- tiledb/sm/cache/lru_cache.h | 33 +++++++---------- tiledb/sm/filesystem/vfs.h | 3 +- tiledb/sm/fragment/fragment_metadata.cc | 8 ++--- tiledb/sm/global_state/global_state.cc | 15 +++----- tiledb/sm/global_state/global_state.h | 4 +-- tiledb/sm/global_state/signal_handlers.cc | 15 ++++---- tiledb/sm/global_state/signal_handlers.h | 6 +--- tiledb/sm/global_state/watchdog.cc | 10 ++---- tiledb/sm/global_state/watchdog.h | 8 +---- tiledb/sm/group/group.cc | 3 +- tiledb/sm/metadata/metadata.cc | 6 ++-- tiledb/sm/metadata/metadata.h | 3 +- tiledb/sm/rtree/rtree.cc | 30 +++++++++------- tiledb/sm/rtree/rtree.h | 10 +++--- tiledb/sm/rtree/test/unit_rtree.cc | 38 ++++++++++---------- tiledb/sm/storage_manager/storage_manager.cc | 2 +- tiledb/sm/tile/filtered_buffer.h | 2 +- tiledb/sm/tile/generic_tile_io.h | 3 -- tiledb/sm/tile/tile.h | 1 - 20 files changed, 81 insertions(+), 124 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 81377ec05f27..f2f228630dbb 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -520,11 +520,8 @@ Status Array::close() { } else { if (query_type_ == QueryType::WRITE || query_type_ == QueryType::MODIFY_EXCLUSIVE) { - st = opened_array_->metadata().store( + opened_array_->metadata().store( resources_, array_uri_, *encryption_key()); - if (!st.ok()) { - throw StatusException(st); - } } else if ( query_type_ != QueryType::READ && query_type_ != QueryType::DELETE && query_type_ != QueryType::UPDATE) { diff --git a/tiledb/sm/cache/lru_cache.h b/tiledb/sm/cache/lru_cache.h index 6c234a7401c0..14d14537569f 100644 --- a/tiledb/sm/cache/lru_cache.h +++ b/tiledb/sm/cache/lru_cache.h @@ -34,7 +34,6 @@ #define TILEDB_LRU_CACHE_H #include "tiledb/common/macros.h" -#include "tiledb/common/status.h" #include #include @@ -144,17 +143,17 @@ class LRUCache { * @param size The logical size of the object. * @param overwrite If `true`, if the object exists in the cache it will be * overwritten. Otherwise, the new object will be deleted. - * @return Status */ - Status insert( - const K& key, V&& object, uint64_t size, bool overwrite = true) { + void insert(const K& key, V&& object, uint64_t size, bool overwrite = true) { // Do nothing if the object size is bigger than the cache maximum size - if (size > max_size_) - return Status::Ok(); + if (size > max_size_) { + return; + } const bool exists = item_map_.count(key) == 1; - if (exists && !overwrite) - return Status::Ok(); + if (exists && !overwrite) { + return; + } // Evict objects until there is room for `object`. Note that this // invalidates the state in `exists`. @@ -191,8 +190,6 @@ class LRUCache { } size_ += size; - - return Status::Ok(); } /** @@ -235,27 +232,21 @@ class LRUCache { * Invalidates and evicts the object in the cache with the given key. * * @param key The key that describes the object to be invalidated. - * @param success Set to `true` if the object was removed successfully; if - * the object did not exist in the cache, set to `false`. - * @return Status + * @return Return `true` if the object was removed successfully; if + * the object did not exist in the cache, return `false`. */ - Status invalidate(const K& key, bool* success) { - assert(success); - + bool invalidate(const K& key) { const auto item_it = item_map_.find(key); const bool exists = item_it != item_map_.end(); if (!exists) { - *success = false; - return Status::Ok(); + return false; } // Move item to the head of the list and evict it. auto& node = item_it->second; item_ll_.splice(item_ll_.begin(), item_ll_, node); evict(); - *success = true; - - return Status::Ok(); + return true; } /** diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index ac1a9e0dfa69..a327b77bc013 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -901,8 +901,9 @@ class VFS : private VFSBase, protected S3_within_VFS { const uint64_t size = buffer.size(); ReadAheadBuffer ra_buffer(offset, std::move(buffer)); - return LRUCache::insert( + LRUCache::insert( uri.to_string(), std::move(ra_buffer), size); + return Status::Ok(); } private: diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 93e1814810b6..c43e4aae9125 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -124,7 +124,7 @@ FragmentMetadata::FragmentMetadata( void FragmentMetadata::set_mbr(uint64_t tile, const NDRange& mbr) { // For easy reference tile += tile_index_base_; - throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf(tile, mbr)); + loaded_metadata_ptr_->rtree().set_leaf(tile, mbr); return expand_non_empty_domain(mbr); } @@ -1226,7 +1226,7 @@ void FragmentMetadata::set_num_tiles(uint64_t num_tiles) { } if (!dense_) { - throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf_num(num_tiles)); + loaded_metadata_ptr_->rtree().set_leaf_num(num_tiles); sparse_tile_num_ = num_tiles; } } @@ -2053,7 +2053,7 @@ void FragmentMetadata::load_mbrs(Deserializer& deserializer) { mbr_num = deserializer.read(); // Set leaf level - throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf_num(mbr_num)); + loaded_metadata_ptr_->rtree().set_leaf_num(mbr_num); auto& domain{array_schema_->domain()}; auto dim_num = domain.dim_num(); for (uint64_t m = 0; m < mbr_num; ++m) { @@ -2062,7 +2062,7 @@ void FragmentMetadata::load_mbrs(Deserializer& deserializer) { uint64_t r_size{2 * domain.dimension_ptr(d)->coord_size()}; mbr[d] = Range(deserializer.get_ptr(r_size), r_size); } - throw_if_not_ok(loaded_metadata_ptr_->rtree().set_leaf(m, mbr)); + loaded_metadata_ptr_->rtree().set_leaf(m, mbr); } // Build R-tree bottom-up diff --git a/tiledb/sm/global_state/global_state.cc b/tiledb/sm/global_state/global_state.cc index 965c93116fea..38120c8e3c0c 100644 --- a/tiledb/sm/global_state/global_state.cc +++ b/tiledb/sm/global_state/global_state.cc @@ -60,29 +60,24 @@ GlobalState::GlobalState() { initialized_ = false; } -Status GlobalState::init(const Config& config) { +void GlobalState::init(const Config& config) { std::unique_lock lck(init_mtx_); // Get config params - bool found; - bool enable_signal_handlers = false; - RETURN_NOT_OK(config.get( - "sm.enable_signal_handlers", &enable_signal_handlers, &found)); - assert(found); + bool enable_signal_handlers = + config.get("sm.enable_signal_handlers", Config::must_find); // run these operations once if (!initialized_) { config_ = config; if (enable_signal_handlers) { - RETURN_NOT_OK(SignalHandlers::GetSignalHandlers().initialize()); + SignalHandlers::GetSignalHandlers().initialize(); } - RETURN_NOT_OK(Watchdog::GetWatchdog().initialize()); + Watchdog::GetWatchdog().initialize(); initialized_ = true; } - - return Status::Ok(); } void GlobalState::register_storage_manager(StorageManager* sm) { diff --git a/tiledb/sm/global_state/global_state.h b/tiledb/sm/global_state/global_state.h index 7879549bbdd6..224dda0972cb 100644 --- a/tiledb/sm/global_state/global_state.h +++ b/tiledb/sm/global_state/global_state.h @@ -37,7 +37,6 @@ #include #include -#include "tiledb/common/status.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" @@ -67,9 +66,8 @@ class GlobalState { * Initializes all TileDB global state in an idempotent and threadsafe way. * * @param config The TileDB configuration parameters (or nullptr). - * @return Status */ - Status init(const Config& config = Config()); + void init(const Config& config = Config()); /** * Register the given StorageManger instance. diff --git a/tiledb/sm/global_state/signal_handlers.cc b/tiledb/sm/global_state/signal_handlers.cc index 0bcaae283c39..2d74a70b495f 100644 --- a/tiledb/sm/global_state/signal_handlers.cc +++ b/tiledb/sm/global_state/signal_handlers.cc @@ -112,18 +112,17 @@ static BOOL WINAPI win_ctrl_handler(DWORD dwCtrlType) { return false; } -Status SignalHandlers::initialize() { +void SignalHandlers::initialize() { if (signal(SIGINT, tiledb_signal_handler) == SIG_ERR) { - return Status_Error( + throw std::runtime_error( std::string("Failed to install Win32 SIGINT handler: ") + strerror(errno)); } // Win32 applications should also handle Ctrl-Break. if (SetConsoleCtrlHandler(win_ctrl_handler, TRUE) == 0) { - return Status_Error(std::string("Failed to install Win32 ctrl handler")); + throw std::runtime_error("Failed to install Win32 ctrl handler"); } - return Status::Ok(); } void SignalHandlers::safe_stderr(const char* msg, size_t msg_len) { @@ -137,14 +136,14 @@ void SignalHandlers::safe_stderr(const char* msg, size_t msg_len) { /* POSIX implementations */ /* ********************************* */ -Status SignalHandlers::initialize() { +void SignalHandlers::initialize() { struct sigaction action, old_action; memset(&action, 0, sizeof(struct sigaction)); memset(&old_action, 0, sizeof(struct sigaction)); // Remember the previous signal handler so we can call it before ours. if (sigaction(SIGINT, NULL, &old_action) != 0) { - return Status_Error( + throw std::runtime_error( std::string("Failed to get old SIGINT handler: ") + strerror(errno)); } old_sigint_handler = old_action.sa_handler; @@ -155,11 +154,9 @@ Status SignalHandlers::initialize() { action.sa_flags = 0; action.sa_handler = tiledb_signal_handler; if (sigaction(SIGINT, &action, &old_action) != 0) { - return Status_Error( + throw std::runtime_error( std::string("Failed to install SIGINT handler: ") + strerror(errno)); } - - return Status::Ok(); } void SignalHandlers::safe_stderr(const char* msg, size_t msg_len) { diff --git a/tiledb/sm/global_state/signal_handlers.h b/tiledb/sm/global_state/signal_handlers.h index 7d41c220efb2..25e3a2d75396 100644 --- a/tiledb/sm/global_state/signal_handlers.h +++ b/tiledb/sm/global_state/signal_handlers.h @@ -35,8 +35,6 @@ #include -#include "tiledb/common/status.h" - using namespace tiledb::common; namespace tiledb { @@ -58,10 +56,8 @@ class SignalHandlers { /** * Initialize the signal handlers. - * - * @return Status */ - Status initialize(); + void initialize(); /** * Returns true if a signal has been received. This will only return true on diff --git a/tiledb/sm/global_state/watchdog.cc b/tiledb/sm/global_state/watchdog.cc index 1511b9563f7f..a474346797a2 100644 --- a/tiledb/sm/global_state/watchdog.cc +++ b/tiledb/sm/global_state/watchdog.cc @@ -64,14 +64,8 @@ Watchdog::~Watchdog() { thread_.join(); } -Status Watchdog::initialize() { - try { - thread_ = std::thread([this]() { watchdog_thread(this); }); - } catch (const std::exception& e) { - return Status_Error( - std::string("Could not initialize watchdog thread; ") + e.what()); - } - return Status::Ok(); +void Watchdog::initialize() { + thread_ = std::thread([this]() { watchdog_thread(this); }); } void Watchdog::watchdog_thread(Watchdog* watchdog) { diff --git a/tiledb/sm/global_state/watchdog.h b/tiledb/sm/global_state/watchdog.h index 143b003ce94b..d954b7ef1ca9 100644 --- a/tiledb/sm/global_state/watchdog.h +++ b/tiledb/sm/global_state/watchdog.h @@ -37,10 +37,6 @@ #include #include -#include "tiledb/common/status.h" - -using namespace tiledb::common; - namespace tiledb { namespace sm { namespace global_state { @@ -61,10 +57,8 @@ class Watchdog { /** * Initializes the Watchdog thread. - * - * @return Status */ - Status initialize(); + void initialize(); private: /** Condition variable for coordinating with the watchdog thread. */ diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 2a2d64773dea..d8abbff7d52e 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -201,8 +201,7 @@ void Group::open(QueryType query_type) { void Group::close_for_writes() { // Flush the group metadata - throw_if_not_ok( - unsafe_metadata()->store(resources_, group_uri(), *encryption_key())); + unsafe_metadata()->store(resources_, group_uri(), *encryption_key()); // Store any changes required if (group_details()->is_modified()) { diff --git a/tiledb/sm/metadata/metadata.cc b/tiledb/sm/metadata/metadata.cc index 7c3f5102a017..3a87eedd7609 100644 --- a/tiledb/sm/metadata/metadata.cc +++ b/tiledb/sm/metadata/metadata.cc @@ -187,7 +187,7 @@ void Metadata::serialize(Serializer& serializer) const { } } -Status Metadata::store( +void Metadata::store( ContextResources& resources, const URI& uri, const EncryptionKey& encryption_key) { @@ -199,7 +199,7 @@ Status Metadata::store( // Do nothing if there are no metadata to write if (0 == size_computation_serializer.size()) { - return Status::Ok(); + return; } auto tile{WriterTile::from_generic( size_computation_serializer.size(), @@ -211,8 +211,6 @@ Status Metadata::store( // Create a metadata file name GenericTileIO::store_data(resources, get_uri(uri), tile, encryption_key); - - return Status::Ok(); } void Metadata::del(const char* key) { diff --git a/tiledb/sm/metadata/metadata.h b/tiledb/sm/metadata/metadata.h index d982fae1591d..7ab7eb36b905 100644 --- a/tiledb/sm/metadata/metadata.h +++ b/tiledb/sm/metadata/metadata.h @@ -143,9 +143,8 @@ class Metadata { * @param resources The context resources. * @param uri The object URI. * @param encryption_key The encryption key to use. - * @return Status */ - Status store( + void store( ContextResources& resources, const URI& uri, const EncryptionKey& encryption_key); diff --git a/tiledb/sm/rtree/rtree.cc b/tiledb/sm/rtree/rtree.cc index e91a1838f792..4a6737b12904 100644 --- a/tiledb/sm/rtree/rtree.cc +++ b/tiledb/sm/rtree/rtree.cc @@ -45,6 +45,14 @@ #include #include +/** Class for locally generated status exceptions. */ +class RTreeException : public StatusException { + public: + explicit RTreeException(const std::string& msg) + : StatusException("RTree", msg) { + } +}; + using namespace tiledb::common; namespace tiledb { @@ -260,38 +268,34 @@ void RTree::serialize(Serializer& serializer) const { } } -Status RTree::set_leaf(uint64_t leaf_id, const NDRange& mbr) { +void RTree::set_leaf(uint64_t leaf_id, const NDRange& mbr) { if (levels_.size() != 1) - return LOG_STATUS(Status_RTreeError( - "Cannot set leaf; There are more than one levels in the tree")); + throw RTreeException( + "Cannot set leaf; There are more than one levels in the tree"); if (leaf_id >= levels_[0].size()) - return LOG_STATUS(Status_RTreeError("Cannot set leaf; Invalid lead index")); + throw RTreeException("Cannot set leaf; Invalid lead index"); levels_[0][leaf_id] = mbr; - - return Status::Ok(); } -Status RTree::set_leaves(const tdb::pmr::vector& mbrs) { +void RTree::set_leaves(const tdb::pmr::vector& mbrs) { levels_.clear(); levels_.resize(1); levels_[0].assign(mbrs.begin(), mbrs.end()); - return Status::Ok(); } -Status RTree::set_leaf_num(uint64_t num) { +void RTree::set_leaf_num(uint64_t num) { // There should be exactly one level (the leaf level) if (levels_.size() != 1) levels_.resize(1); if (num < levels_[0].size()) - return LOG_STATUS( - Status_RTreeError("Cannot set number of leaves; provided number " - "cannot be smaller than the current leaf number")); + throw RTreeException( + "Cannot set number of leaves; provided number " + "cannot be smaller than the current leaf number"); levels_[0].resize(num); - return Status::Ok(); } void RTree::deserialize( diff --git a/tiledb/sm/rtree/rtree.h b/tiledb/sm/rtree/rtree.h index 35fd505d7bed..557c3f92e380 100644 --- a/tiledb/sm/rtree/rtree.h +++ b/tiledb/sm/rtree/rtree.h @@ -37,7 +37,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/pmr.h" -#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/misc/tile_overlap.h" #include "tiledb/storage_format/serialization/serializers.h" @@ -136,9 +135,8 @@ class RTree { /** * Sets the RTree domain. */ - inline Status set_domain(const Domain* domain) { + inline void set_domain(const Domain* domain) { domain_ = domain; - return Status::Ok(); } /** @@ -146,13 +144,13 @@ class RTree { * if the number of levels in the tree is different from exactly * 1 (the leaf level), and if `leaf_id` is out of bounds / invalid. */ - Status set_leaf(uint64_t leaf_id, const NDRange& mbr); + void set_leaf(uint64_t leaf_id, const NDRange& mbr); /** * Sets the input MBRs as leaves. This will destroy the existing * RTree. */ - Status set_leaves(const tdb::pmr::vector& mbrs); + void set_leaves(const tdb::pmr::vector& mbrs); /** * Resizes the leaf level. It destroys the upper levels @@ -160,7 +158,7 @@ class RTree { * It errors if `num` is smaller than the current number * of leaves. */ - Status set_leaf_num(uint64_t num); + void set_leaf_num(uint64_t num); /** * Deserializes the contents of the object from the input buffer based diff --git a/tiledb/sm/rtree/test/unit_rtree.cc b/tiledb/sm/rtree/test/unit_rtree.cc index 6d81dd0c9499..8a9366948701 100644 --- a/tiledb/sm/rtree/test/unit_rtree.cc +++ b/tiledb/sm/rtree/test/unit_rtree.cc @@ -147,13 +147,13 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs_1d = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); RTree rtree1(&d1, 3, tracker); - CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); - CHECK(rtree1.set_leaf_num(mbrs_1d.size()).ok()); + CHECK_THROWS(rtree1.set_leaf(0, mbrs_1d[0])); + rtree1.set_leaf_num(mbrs_1d.size()); for (size_t m = 0; m < mbrs_1d.size(); ++m) - CHECK(rtree1.set_leaf(m, mbrs_1d[m]).ok()); - CHECK(!rtree1.set_leaf_num(1).ok()); + rtree1.set_leaf(m, mbrs_1d[m]); + CHECK_THROWS(rtree1.set_leaf_num(1)); rtree1.build_tree(); - CHECK(!rtree1.set_leaf(0, mbrs_1d[0]).ok()); + CHECK_THROWS(rtree1.set_leaf(0, mbrs_1d[0])); CHECK(rtree1.height() == 2); CHECK(rtree1.dim_num() == 1); CHECK(rtree1.subtree_leaf_num(0) == 3); @@ -214,7 +214,7 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { auto mbrs_2d = create_mbrs( {1, 3, 5, 10, 20, 22, 24, 25, 11, 15, 30, 31}, tracker); RTree rtree2(&d2, 5, tracker); - CHECK(rtree2.set_leaves(mbrs_2d).ok()); + rtree2.set_leaves(mbrs_2d); rtree2.build_tree(); CHECK(rtree2.height() == 2); CHECK(rtree2.dim_num() == 2); @@ -254,7 +254,7 @@ TEST_CASE("RTree: Test R-Tree, basic functions", "[rtree][basic]") { const Domain d2f = create_domain( {"d"}, {Datatype::FLOAT32}, {dim_dom_f}, {&dim_extent_f}, tracker); RTree rtreef(&d2f, 5, tracker); - CHECK(rtreef.set_leaves(mbrs_f).ok()); + rtreef.set_leaves(mbrs_f); rtreef.build_tree(); NDRange rangef(1); @@ -297,7 +297,7 @@ TEST_CASE("RTree: Test 1D R-tree, height 2", "[rtree][1d][2h]") { {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); auto mbrs = create_mbrs({1, 3, 5, 10, 20, 22}, tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 1); @@ -344,7 +344,7 @@ TEST_CASE("RTree: Test 1D R-tree, height 3", "[rtree][1d][3h]") { Domain dom1 = create_domain( {"d"}, {Datatype::INT32}, {dim_dom}, {&dim_extent}, tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 3); CHECK(rtree.dim_num() == 1); @@ -414,7 +414,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 2", "[rtree][2d][2h]") { auto mbrs = create_mbrs( {1, 3, 2, 4, 5, 7, 6, 9, 10, 12, 10, 15}, tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); @@ -470,7 +470,7 @@ TEST_CASE("RTree: Test 2D R-tree, height 3", "[rtree][2d][3h]") { 23, 23, 19, 20, 24, 26, 25, 28, 30, 32, 30, 35, 35, 37, 40, 42, 40, 42}, tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 3); CHECK(rtree.dim_num() == 2); @@ -555,7 +555,7 @@ TEST_CASE( auto mbrs = create_mbrs({0, 1, 3, 5}, {5, 6, 7, 9}, tracker); RTree rtree(&d1, 5, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); @@ -618,7 +618,7 @@ TEST_CASE( auto mbrs = create_mbrs({0, 1, 3, 5}, {.5f, .6f, .7f, .9f}, tracker); RTree rtree(&d1, 5, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); @@ -675,7 +675,7 @@ TEST_CASE( auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20}, {5, 6, 7, 9, 11, 30}, tracker); RTree rtree(&dom, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); @@ -745,7 +745,7 @@ TEST_CASE( auto mbrs = create_mbrs( {0, 1, 3, 5, 11, 20, 21, 26}, {5, 6, 7, 9, 11, 30, 31, 40}, tracker); RTree rtree(&dom, 2, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 3); CHECK(rtree.dim_num() == 2); @@ -882,7 +882,7 @@ TEST_CASE( create_str_mbrs<1>({"aa", "b", "eee", "g", "gggg", "ii"}, tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 1); @@ -972,7 +972,7 @@ TEST_CASE( tracker); RTree rtree(&dom1, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 3); CHECK(rtree.dim_num() == 1); @@ -1058,7 +1058,7 @@ TEST_CASE( tracker); RTree rtree(&dom, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); @@ -1165,7 +1165,7 @@ TEST_CASE( {"aa", "b", "eee", "g", "gggg", "ii"}, {1, 5, 7, 8, 10, 14}, tracker); RTree rtree(&dom, 3, tracker); - CHECK(rtree.set_leaves(mbrs).ok()); + rtree.set_leaves(mbrs); rtree.build_tree(); CHECK(rtree.height() == 2); CHECK(rtree.dim_num() == 2); diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 08ccf0d7fcf5..5dc5112dfc1e 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -94,7 +94,7 @@ StorageManagerCanonical::StorageManagerCanonical( Status StorageManagerCanonical::init() { auto& global_state = global_state::GlobalState::GetGlobalState(); - RETURN_NOT_OK(global_state.init(config_)); + global_state.init(config_); RETURN_NOT_OK(set_default_tags()); diff --git a/tiledb/sm/tile/filtered_buffer.h b/tiledb/sm/tile/filtered_buffer.h index 0a52d7e6f8aa..26f306a0b6b6 100644 --- a/tiledb/sm/tile/filtered_buffer.h +++ b/tiledb/sm/tile/filtered_buffer.h @@ -35,7 +35,7 @@ #include -#include "tiledb/common/status.h" +#include "tiledb/common/common.h" using namespace tiledb::common; diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index ad75af43f81e..dc0e72cc23fc 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -203,7 +203,6 @@ class GenericTileIO { * Writes the generic tile header to the file. * * @param header The header to write - * @return Status */ void write_generic_tile_header(GenericTileHeader* header); @@ -223,7 +222,6 @@ class GenericTileIO { * * @param header Header instance to modify. * @param encryption_key The encryption key to use. - * @return Status */ void configure_encryption_filter( GenericTileHeader* header, const EncryptionKey& encryption_key) const; @@ -236,7 +234,6 @@ class GenericTileIO { * @param tile The tile to initialize a header for * @param header The header to initialize * @param encryption_key The encryption key to use. - * @return Status */ void init_generic_tile_header( WriterTile* tile, diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index cfd1bea986c2..db057d92760a 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -35,7 +35,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/pmr.h" -#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/tile/filtered_buffer.h" #include "tiledb/storage_format/serialization/serializers.h" From 364204f28ce722043951ee7ca807003dcee90929 Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Wed, 29 May 2024 12:28:49 -0400 Subject: [PATCH 386/456] Fix output datatype for some filters. (#4988) Story details: https://app.shortcut.com/tiledb-inc/story/47328 The addition of `Filter::output_datatype` to validate filter pipelines prevented some invalid pipelines from being constructed; but it depends on the correctness of each `output_datatype` implementation to ensure that a filter can run correctly over the output of a previous filter. For the Rust bindings we have built randomized test strategies which assemble pipelines over combinations of filters which are unlikely to have appeared in the wild. This has exposed some missing or incorrect `output_datatype` implementations which allow filters which depend on certain input properties to appear in pipelines erroneously. This pull request demonstrates some such cases in unit tests using the XOR and Delta compression filters, which depend on a particular bit width of their input, and fixes the `output_datatype` implementations of filters which do not preserve the datatype between input and output. --- TYPE: BUG DESC: Fix output datatype for some filters. --- test/src/cpp-integration-filter-pipeline.cc | 49 +++++-- .../sm/filter/bit_width_reduction_filter.cc | 5 + tiledb/sm/filter/bit_width_reduction_filter.h | 9 ++ tiledb/sm/filter/compression_filter.cc | 17 ++- .../sm/filter/encryption_aes256gcm_filter.cc | 5 + .../sm/filter/encryption_aes256gcm_filter.h | 9 ++ .../filter/test/unit_run_filter_pipeline.cc | 126 ++++++++++++++++++ tiledb/sm/filter/webp_filter.cc | 6 + tiledb/sm/filter/webp_filter.h | 9 ++ 9 files changed, 217 insertions(+), 18 deletions(-) diff --git a/test/src/cpp-integration-filter-pipeline.cc b/test/src/cpp-integration-filter-pipeline.cc index 3974003b2d05..a5eae4d397d7 100644 --- a/test/src/cpp-integration-filter-pipeline.cc +++ b/test/src/cpp-integration-filter-pipeline.cc @@ -34,6 +34,7 @@ #include "test/support/src/helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/filter/checksum_md5_filter.h" #include "tiledb/sm/filter/webp_filter.h" #include "tiledb/stdx/utility/to_underlying.h" @@ -233,7 +234,27 @@ TEST_CASE( CHECK_NOTHROW(d1.set_filter_list(filters)); CHECK_NOTHROW(a1.set_filter_list(filters)); + } + + SECTION("- Multiple compressors following type conversion") { + auto compressor = GENERATE( + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA, + TILEDB_FILTER_GZIP, + TILEDB_FILTER_LZ4, + TILEDB_FILTER_RLE, + TILEDB_FILTER_ZSTD); + tiledb::Filter compressor_filter(ctx, compressor); + tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); + filters.add_filter(float_scale); + filters.add_filter(bzip); + filters.add_filter(compressor_filter); + CHECK_NOTHROW(d1.set_filter_list(filters)); + CHECK_NOTHROW(a1.set_filter_list(filters)); + } + + SECTION("- Delta cannot run on float input") { // Should throw without FloatScale to convert float->int32. auto delta_compressor = GENERATE( TILEDB_FILTER_POSITIVE_DELTA, @@ -243,21 +264,31 @@ TEST_CASE( filters.add_filter(delta_filter); CHECK_THROWS(d1.set_filter_list(filters)); CHECK_THROWS(a1.set_filter_list(filters)); + + tiledb::Filter checksum_filter(ctx, TILEDB_FILTER_CHECKSUM_MD5); + filters.add_filter(checksum_filter); + CHECK_THROWS(d1.set_filter_list(filters)); + CHECK_THROWS(a1.set_filter_list(filters)); } - SECTION("- Multiple compressors following type conversion") { - auto compressor = GENERATE( - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA, + SECTION("- Delta can run if a prior filter converts to non-float type") { + // note that this may not be a *wise* choice... but it is nonetheless viable + auto converting_filter = GENERATE( TILEDB_FILTER_GZIP, TILEDB_FILTER_LZ4, TILEDB_FILTER_RLE, TILEDB_FILTER_ZSTD); - tiledb::Filter compressor_filter(ctx, compressor); - tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); - filters.add_filter(float_scale); - filters.add_filter(bzip); - filters.add_filter(compressor_filter); + + auto delta_compressor = GENERATE( + TILEDB_FILTER_POSITIVE_DELTA, + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA); + + tiledb::Filter convert_filter(ctx, converting_filter); + tiledb::Filter delta_filter(ctx, delta_compressor); + + filters.add_filter(convert_filter); + filters.add_filter(delta_filter); CHECK_NOTHROW(d1.set_filter_list(filters)); CHECK_NOTHROW(a1.set_filter_list(filters)); diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index df47c6f05fd8..a25b2905966b 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -105,6 +105,11 @@ bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const { return false; } +Datatype BitWidthReductionFilter::output_datatype(Datatype) const { + /* if the compression did anything then the result is a binary blob */ + return Datatype::BLOB; +} + Status BitWidthReductionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, diff --git a/tiledb/sm/filter/bit_width_reduction_filter.h b/tiledb/sm/filter/bit_width_reduction_filter.h index ebcab1ec93df..b81fc2793d54 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.h +++ b/tiledb/sm/filter/bit_width_reduction_filter.h @@ -107,6 +107,15 @@ class BitWidthReductionFilter : public Filter { */ bool accepts_input_datatype(Datatype datatype) const override; + /** + * Returns the filter output type + * + * @param input_type Expected type used for input. Used for filters which + * change output type based on input data. e.g. XORFilter output type is + * based on byte width of input type. + */ + Datatype output_datatype(Datatype input_type) const override; + /** Return the max window size used by the filter. */ uint32_t max_window_size() const; diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index 55c28759b8e9..bae2a9b31e26 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -733,15 +733,14 @@ void CompressionFilter::init_decompression_resource_pool(uint64_t size) { } } -Datatype CompressionFilter::output_datatype(Datatype datatype) const { - switch (compressor_) { - case Compressor::DOUBLE_DELTA: - case Compressor::DELTA: - return reinterpret_datatype_ == Datatype::ANY ? datatype : - reinterpret_datatype_; - default: - return datatype; - } +Datatype CompressionFilter::output_datatype(Datatype) const { + /* + * If the compression did anything at all, we have a binary blob afterwards. + * Even if it didn't, the output format interleaves metadata with + * compressed data, so we can't interpret as a stream of any particular + * datatype anyway. + */ + return Datatype::BLOB; } } // namespace sm diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.cc b/tiledb/sm/filter/encryption_aes256gcm_filter.cc index fd9ffc4483b9..df91f800b1fe 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.cc +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.cc @@ -70,6 +70,11 @@ void EncryptionAES256GCMFilter::dump(FILE* out) const { fprintf(out, "EncryptionAES256GCM"); } +Datatype EncryptionAES256GCMFilter::output_datatype(Datatype) const { + /* encryption gives us meaningless bits with overwhelming probability */ + return Datatype::BLOB; +} + Status EncryptionAES256GCMFilter::run_forward( const WriterTile&, WriterTile* const, diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.h b/tiledb/sm/filter/encryption_aes256gcm_filter.h index 31c8bdb252f5..bc08097e6b09 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.h +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.h @@ -97,6 +97,15 @@ class EncryptionAES256GCMFilter : public Filter { /** Dumps the filter details in ASCII format in the selected output. */ void dump(FILE* out) const override; + /** + * Returns the filter output type + * + * @param input_type Expected type used for input. Used for filters which + * change output type based on input data. e.g. XORFilter output type is + * based on byte width of input type. + */ + Datatype output_datatype(Datatype input_type) const override; + /** * Encrypt the bytes of the input data into the output data buffer. */ diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 3b5645a63032..0e93b99d51a9 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -75,6 +75,7 @@ #include "../filter.h" #include "../filter_pipeline.h" #include "../positive_delta_filter.h" +#include "../xor_filter.h" #include "add_1_in_place_filter.h" #include "add_1_including_metadata_filter.h" #include "add_1_out_of_place_filter.h" @@ -1137,3 +1138,128 @@ TEST_CASE( } } } + +static void do_invalid_pipeline_check(FilterPipeline& pipeline) { + // Create TileDB resources for running the filter pipeline. + Config config; + ThreadPool tp(4); + auto tracker = tiledb::test::create_test_memory_tracker(); + + // Set-up test data. + SimpleVariableTestData test_data{}; + const auto& tile_data_generator = test_data.tile_data_generator(); + auto&& [tile, offsets_tile] = + tile_data_generator.create_writer_tiles(tracker); + + // in all examples this will abort with high probability + check_run_pipeline_roundtrip( + config, tp, tile, offsets_tile, pipeline, &tile_data_generator, tracker); +} + +TEST_CASE( + "Filter: Invalid XOR after compressor", "[filter][invalid-pipeline]") { + const Datatype input_type = Datatype::INT64; + + FilterPipeline pipeline; + + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, input_type)); + pipeline.add_filter( + XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} + +TEST_CASE( + "Filter: Invalid XOR after bit width reduction", + "[filter][invalid-pipeline]") { + // Create TileDB resources for running the filter pipeline. + const Datatype input_type = Datatype::INT64; + + FilterPipeline pipeline; + + pipeline.add_filter(BitWidthReductionFilter(input_type)); + pipeline.add_filter( + XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} + +TEST_CASE( + "Filter: Invalid XOR after encryption", "[filter][invalid-pipeline]") { + // Create TileDB resources for running the filter pipeline. + const Datatype input_type = Datatype::INT64; + + EncryptionKey encryption_key; + REQUIRE(encryption_key + .set_key( + EncryptionType::AES_256_GCM, + "abcdefghijklmnopqrstuvwxyz012345", + 32) + .ok()); + + FilterPipeline pipeline; + + pipeline.add_filter(EncryptionAES256GCMFilter(encryption_key, input_type)); + pipeline.add_filter( + XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} + +TEST_CASE( + "Filter: Invalid Delta after compressor", "[filter][invalid-pipeline]") { + const Datatype input_type = Datatype::INT64; + + FilterPipeline pipeline; + + pipeline.add_filter( + CompressionFilter(tiledb::sm::Compressor::LZ4, 5, input_type)); + pipeline.add_filter(CompressionFilter( + tiledb::sm::Compressor::DELTA, + 5, + pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} + +TEST_CASE( + "Filter: Invalid Delta after bit width reduction", + "[filter][invalid-pipeline]") { + // Create TileDB resources for running the filter pipeline. + const Datatype input_type = Datatype::INT64; + + FilterPipeline pipeline; + + pipeline.add_filter(BitWidthReductionFilter(input_type)); + pipeline.add_filter(CompressionFilter( + tiledb::sm::Compressor::DELTA, + 5, + pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} + +TEST_CASE( + "Filter: Invalid Delta after encryption", "[filter][invalid-pipeline]") { + // Create TileDB resources for running the filter pipeline. + const Datatype input_type = Datatype::INT64; + + EncryptionKey encryption_key; + REQUIRE(encryption_key + .set_key( + EncryptionType::AES_256_GCM, + "abcdefghijklmnopqrstuvwxyz012345", + 32) + .ok()); + + FilterPipeline pipeline; + + pipeline.add_filter(EncryptionAES256GCMFilter(encryption_key, input_type)); + pipeline.add_filter(CompressionFilter( + tiledb::sm::Compressor::DELTA, + 5, + pipeline.get_filter(0)->output_datatype(input_type))); + + do_invalid_pipeline_check(pipeline); +} diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 961209f7ecb4..412c402fc416 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -44,6 +44,12 @@ void WebpFilter::dump(FILE* out) const { bool WebpFilter::accepts_input_datatype(Datatype datatype) const { return datatype == Datatype::UINT8; } + +Datatype WebpFilter::output_datatype(Datatype) const { + /* compression produces blobs */ + return Datatype::BLOB; +} + } // namespace tiledb::sm #ifndef TILEDB_WEBP diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index cd6648a06cb4..2d6795846842 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -160,6 +160,15 @@ class WebpFilter : public Filter { */ bool accepts_input_datatype(Datatype datatype) const override; + /** + * Returns the filter output type + * + * @param input_type Expected type used for input. Used for filters which + * change output type based on input data. e.g. XORFilter output type is + * based on byte width of input type. + */ + Datatype output_datatype(Datatype input_type) const override; + /** * Runs the filter forward, taking raw colorspace values as input and writing. * encoded WebP data to the TileDB Array. From 10ab8a2d82d850671b5a933e67426b226d93764b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 May 2024 20:05:25 +0300 Subject: [PATCH 387/456] Add concept definitions for `FilePredicate` and `DirectoryPredicate`. (#5007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [SC-48083](https://app.shortcut.com/tiledb-inc/story/48083/add-concept-definitions-for-filepredicate-and-directorypredicate) This PR provides proper definitions of `FilePredicate` and `DirectoryPredicate`, as predicates that accept a constant string view and –in the former case only– a `uint64_t`. This will improve the experience when working with `ls_recursive`'s infrastructure. --- TYPE: NO_HISTORY --- tiledb/sm/filesystem/ls_scanner.h | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tiledb/sm/filesystem/ls_scanner.h b/tiledb/sm/filesystem/ls_scanner.h index 029338165806..4ba04d56f162 100644 --- a/tiledb/sm/filesystem/ls_scanner.h +++ b/tiledb/sm/filesystem/ls_scanner.h @@ -41,16 +41,33 @@ #include #include -/** Inclusion predicate for objects collected by ls */ +/** + * Inclusion predicate for objects collected by ls. + * + * The predicate accepts: + * - A string_view of the path to the object. + * - The size of the object in bytes. + * + * The predicate returns true if the object should be included in the results, + * and false otherwise. + */ template -concept FilePredicate = true; +concept FilePredicate = std::predicate; /** - * DirectoryPredicate is currently unused, but is kept here for adding directory - * pruning support in the future. + * Recursion predicate for directories collected by ls. + * + * The predicate accepts: + * - A string_view of the path to the directory. + * + * The predicate returns true if items inside the directory should be traversed, + * and false otherwise. + * + * Directory pruning is currently supported only in local filesystem, and not + * exposed in the user-facing C API. */ -template -concept DirectoryPredicate = true; +template +concept DirectoryPredicate = std::predicate; namespace tiledb::sm { class LsScanException : public StatusException { From 364cc92fff4c889269ad1afd76af5a917263241b Mon Sep 17 00:00:00 2001 From: Ypatia Tsavliri Date: Wed, 29 May 2024 21:07:46 +0300 Subject: [PATCH 388/456] Add fragment_list to array consolidation request capnp (#5020) This PR is adding `fragment_list` to array consolidation request `capnp`. This is the first step for supporting fragment list consolidation for `tiledb://` arrays. --- TYPE: NO_HISTORY DESC: Add fragment_list to array consolidation request capnp --- tiledb/sm/serialization/tiledb-rest.capnp | 3 + tiledb/sm/serialization/tiledb-rest.capnp.c++ | 42 +++++++--- tiledb/sm/serialization/tiledb-rest.capnp.h | 76 ++++++++++++++++++- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index cb48696310c0..b2af9babf3f3 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -1229,6 +1229,9 @@ struct ArrayDeleteFragmentsTimestampsRequest { struct ArrayConsolidationRequest { config @0 :Config; # Config + + fragments @1 :List(Text); + # Fragment list to consolidate } struct ArrayVacuumRequest { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ index f5770f9a81f5..df74be422db2 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/tiledb-rest.capnp.c++ @@ -9440,17 +9440,17 @@ const ::capnp::_::RawSchema s_aaeeafe1e9f3ea1c = { 1, 3, i_aaeeafe1e9f3ea1c, nullptr, nullptr, { &s_aaeeafe1e9f3ea1c, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { +static const ::capnp::_::AlignedData<55> b_f5a35661031194d2 = { { 0, 0, 0, 0, 5, 0, 6, 0, 210, 148, 17, 3, 97, 86, 163, 245, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 1, 0, 7, 0, 0, 0, 0, 0, + 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 98, 1, 0, 0, 41, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 37, 0, 0, 0, 63, 0, 0, 0, + 37, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -9460,20 +9460,40 @@ static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { 116, 105, 111, 110, 82, 101, 113, 117, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 4, 0, + 8, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 0, 0, 0, 58, 0, 0, 0, + 41, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 0, 0, 0, 3, 0, 1, 0, - 20, 0, 0, 0, 2, 0, 1, 0, + 36, 0, 0, 0, 3, 0, 1, 0, + 48, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 45, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 72, 0, 0, 0, 2, 0, 1, 0, 99, 111, 110, 102, 105, 103, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 54, 173, 17, 129, 75, 91, 201, 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 102, 114, 97, 103, 109, 101, 110, 116, + 115, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -9482,11 +9502,11 @@ static const ::capnp::_::AlignedData<35> b_f5a35661031194d2 = { static const ::capnp::_::RawSchema* const d_f5a35661031194d2[] = { &s_b6c95b4b8111ad36, }; -static const uint16_t m_f5a35661031194d2[] = {0}; -static const uint16_t i_f5a35661031194d2[] = {0}; +static const uint16_t m_f5a35661031194d2[] = {0, 1}; +static const uint16_t i_f5a35661031194d2[] = {0, 1}; const ::capnp::_::RawSchema s_f5a35661031194d2 = { - 0xf5a35661031194d2, b_f5a35661031194d2.words, 35, d_f5a35661031194d2, m_f5a35661031194d2, - 1, 1, i_f5a35661031194d2, nullptr, nullptr, { &s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr }, false + 0xf5a35661031194d2, b_f5a35661031194d2.words, 55, d_f5a35661031194d2, m_f5a35661031194d2, + 1, 2, i_f5a35661031194d2, nullptr, nullptr, { &s_f5a35661031194d2, nullptr, nullptr, 0, 0, nullptr }, false }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<34> b_e68edfc0939e63df = { diff --git a/tiledb/sm/serialization/tiledb-rest.capnp.h b/tiledb/sm/serialization/tiledb-rest.capnp.h index 4951375c0824..6a650d7bb4a6 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/tiledb-rest.capnp.h @@ -1597,7 +1597,7 @@ struct ArrayConsolidationRequest { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(f5a35661031194d2, 0, 1) + CAPNP_DECLARE_STRUCT_HEADER(f5a35661031194d2, 0, 2) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -14379,6 +14379,10 @@ class ArrayConsolidationRequest::Reader { inline bool hasConfig() const; inline ::tiledb::sm::serialization::capnp::Config::Reader getConfig() const; + inline bool hasFragments() const; + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader + getFragments() const; + private: ::capnp::_::StructReader _reader; template @@ -14428,6 +14432,20 @@ class ArrayConsolidationRequest::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Config> disownConfig(); + inline bool hasFragments(); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + getFragments(); + inline void setFragments( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value); + inline void setFragments(::kj::ArrayPtr value); + inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder + initFragments(unsigned int size); + inline void adoptFragments( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value); + inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> + disownFragments(); + private: ::capnp::_::StructBuilder _builder; template @@ -32278,6 +32296,62 @@ ArrayConsolidationRequest::Builder::disownConfig() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } +inline bool ArrayConsolidationRequest::Reader::hasFragments() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArrayConsolidationRequest::Builder::hasFragments() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader +ArrayConsolidationRequest::Reader::getFragments() const { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArrayConsolidationRequest::Builder::getFragments() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void ArrayConsolidationRequest::Builder::setFragments( + ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Reader value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline void ArrayConsolidationRequest::Builder::setFragments( + ::kj::ArrayPtr value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>::Builder +ArrayConsolidationRequest::Builder::initFragments(unsigned int size) { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void ArrayConsolidationRequest::Builder::adoptFragments( + ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>&& + value) { + ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>> +ArrayConsolidationRequest::Builder::disownFragments() { + return ::capnp::_:: + PointerHelpers<::capnp::List<::capnp::Text, ::capnp::Kind::BLOB>>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + inline bool ArrayVacuumRequest::Reader::hasConfig() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); From 248e86d122dc752be26917c80512fb47bda61e61 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 29 May 2024 14:39:49 -0400 Subject: [PATCH 389/456] Migrate APIs out of StorageManager: object_move, object_remove. (#5023) Migrate APIs out of StorageManager: `object_move`, `object_remove`. [sc-46731] [sc-46732] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `object_move`, `object_remove`. --- tiledb/sm/c_api/tiledb.cc | 4 +- tiledb/sm/object/object.cc | 45 +++++++++++++++++++ tiledb/sm/object/object.h | 20 +++++++++ tiledb/sm/storage_manager/storage_manager.cc | 38 ---------------- .../storage_manager_canonical.h | 9 ---- 5 files changed, 67 insertions(+), 49 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 86c7c6e5fb16..4dc6edd1d282 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3083,13 +3083,13 @@ int32_t tiledb_object_type( } int32_t tiledb_object_remove(tiledb_ctx_t* ctx, const char* path) { - throw_if_not_ok(ctx->storage_manager()->object_remove(path)); + throw_if_not_ok(object_remove(ctx->resources(), path)); return TILEDB_OK; } int32_t tiledb_object_move( tiledb_ctx_t* ctx, const char* old_path, const char* new_path) { - throw_if_not_ok(ctx->storage_manager()->object_move(old_path, new_path)); + throw_if_not_ok(object_move(ctx->resources(), old_path, new_path)); return TILEDB_OK; } diff --git a/tiledb/sm/object/object.cc b/tiledb/sm/object/object.cc index bdb9d429017c..2bcbbe5e9564 100644 --- a/tiledb/sm/object/object.cc +++ b/tiledb/sm/object/object.cc @@ -39,6 +39,13 @@ namespace tiledb::sm { +class ObjectException : public StatusException { + public: + explicit ObjectException(const std::string& message) + : StatusException("Object", message) { + } +}; + /* ********************************* */ /* API */ /* ********************************* */ @@ -129,4 +136,42 @@ Status object_type( return Status::Ok(); } +Status object_move( + ContextResources& resources, const char* old_path, const char* new_path) { + auto old_uri = URI(old_path); + if (old_uri.is_invalid()) + throw ObjectException( + "Cannot move object '" + std::string(old_path) + "'; Invalid URI"); + + auto new_uri = URI(new_path); + if (new_uri.is_invalid()) + throw ObjectException( + "Cannot move object to '" + std::string(new_path) + "'; Invalid URI"); + + ObjectType obj_type; + throw_if_not_ok(object_type(resources, old_uri, &obj_type)); + if (obj_type == ObjectType::INVALID) + throw ObjectException( + "Cannot move object '" + std::string(old_path) + + "'; Invalid TileDB object"); + + return resources.vfs().move_dir(old_uri, new_uri); +} + +Status object_remove(ContextResources& resources, const char* path) { + auto uri = URI(path); + if (uri.is_invalid()) + throw ObjectException( + "Cannot remove object '" + std::string(path) + "'; Invalid URI"); + + ObjectType obj_type; + throw_if_not_ok(object_type(resources, uri, &obj_type)); + if (obj_type == ObjectType::INVALID) + throw ObjectException( + "Cannot remove object '" + std::string(path) + + "'; Invalid TileDB object"); + + return resources.vfs().remove_dir(uri); +} + } // namespace tiledb::sm diff --git a/tiledb/sm/object/object.h b/tiledb/sm/object/object.h index 12876783cffe..108eb3ca0e07 100644 --- a/tiledb/sm/object/object.h +++ b/tiledb/sm/object/object.h @@ -78,6 +78,26 @@ Status is_group(ContextResources& resources, const URI& uri, bool* is_group); Status object_type( ContextResources& resources, const URI& uri, ObjectType* type); +/** + * Moves a TileDB object. If `new_path` exists, it will be overwritten. + * + * @param resources the context resources. + * @param old_path the old path of the object. + * @param new_path the new path of the object. + * @return Status + */ +Status object_move( + ContextResources& resources, const char* old_path, const char* new_path); + +/** + * Removes a TileDB object. + * + * @param resources the context resources. + * @param path the path to the object to be removed. + * @return Status + */ +Status object_remove(ContextResources& resources, const char* path); + } // namespace tiledb::sm #endif diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 5dc5112dfc1e..c804dc4a9a9b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -424,44 +424,6 @@ void StorageManagerCanonical::decrement_in_progress() { queries_in_progress_cv_.notify_all(); } -Status StorageManagerCanonical::object_remove(const char* path) const { - auto uri = URI(path); - if (uri.is_invalid()) - return logger_->status(Status_StorageManagerError( - std::string("Cannot remove object '") + path + "'; Invalid URI")); - - ObjectType obj_type; - throw_if_not_ok(object_type(resources_, uri, &obj_type)); - if (obj_type == ObjectType::INVALID) - return logger_->status(Status_StorageManagerError( - std::string("Cannot remove object '") + path + - "'; Invalid TileDB object")); - - return resources_.vfs().remove_dir(uri); -} - -Status StorageManagerCanonical::object_move( - const char* old_path, const char* new_path) const { - auto old_uri = URI(old_path); - if (old_uri.is_invalid()) - return logger_->status(Status_StorageManagerError( - std::string("Cannot move object '") + old_path + "'; Invalid URI")); - - auto new_uri = URI(new_path); - if (new_uri.is_invalid()) - return logger_->status(Status_StorageManagerError( - std::string("Cannot move object to '") + new_path + "'; Invalid URI")); - - ObjectType obj_type; - throw_if_not_ok(object_type(resources_, old_uri, &obj_type)); - if (obj_type == ObjectType::INVALID) - return logger_->status(Status_StorageManagerError( - std::string("Cannot move object '") + old_path + - "'; Invalid TileDB object")); - - return resources_.vfs().move_dir(old_uri, new_uri); -} - const std::unordered_map& StorageManagerCanonical::tags() const { return tags_; diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 8ee1257c39fb..3d1d549e4b71 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -199,15 +199,6 @@ class StorageManagerCanonical { */ Status group_create(const std::string& group); - /** Removes a TileDB object (group, array). */ - Status object_remove(const char* path) const; - - /** - * Renames a TileDB object (group, array). If - * `new_path` exists, `new_path` will be overwritten. - */ - Status object_move(const char* old_path, const char* new_path) const; - /** * Creates a new object iterator for the input path. The iteration * in this case will be recursive in the entire directory tree rooted From 505b39ab60be5a9f9345113dd7615e9d6b77195b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 30 May 2024 16:45:27 +0300 Subject: [PATCH 390/456] Improve detection of macOS deployment target, for choosing pmr implementation. (#5027) SC-48427 #4995 changed detection of `std::pmr` from trying to _run_ a C++ file referencing pmr APIs, to trying to _compile_ it, which works in cross compilation scenarios and fixed Conda failures for reasons I do not understand. However in macOS versions prior to 14 the pmr headers exist, but binaries compiled with it fail ot run. There was already a check to force using the vendored pmr in such versions but it was defective, because it checked the `MACOS_DEPLOYMENT_TARGET` environment variable, which might not exist. Before #4995, the runtime failure on these versions would be caught by `try_run`. This PR updates the detection script to use the [`CMAKE_OSX_DEPLOYMENT_TARGET`](https://cmake.org/cmake/help/latest/variable/CMAKE_OSX_DEPLOYMENT_TARGET.html) variable, which gets automatically filled by CMake either from the aforementioned environment variable or automatically. --- TYPE: NO_HISTORY --- cmake/Modules/DetectStdPmr.cmake | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmake/Modules/DetectStdPmr.cmake b/cmake/Modules/DetectStdPmr.cmake index 37a6edc50f70..cbea9237a41e 100644 --- a/cmake/Modules/DetectStdPmr.cmake +++ b/cmake/Modules/DetectStdPmr.cmake @@ -26,16 +26,14 @@ # # Detect whether polymorphic allocators are available on the system. -# Special case for macOS when the MACOSX_DEPLOYMENT_TARGET is set to anything +# Special case for macOS when the CMAKE_OSX_DEPLOYMENT_TARGET is set to anything # less than 14. For some reason, std::pmr is still detectable, but the resulting # binary dies with a dyld missing symbol error. -if (ENV{MACOSX_DEPLOYMENT_TARGET}) - if (ENV{MACOSX_DEPLOYMENT_TARGET} STRLESS "14") - set(TILEDB_USE_CPP17_PMR ON) - message(STATUS "Using vendored cpp17::pmr for polymorphic allocators") - return() - endif() +if (APPLE AND CMAKE_OSX_DEPLOYMENT_TARGET LESS 14) + set(TILEDB_USE_CPP17_PMR ON) + message(STATUS "Using vendored cpp17::pmr for polymorphic allocators because of macOS deployment target ${CMAKE_OSX_DEPLOYMENT_TARGET} < 14") + return() endif() # Otherwise, if we're not building a targeted macOS version, we just detect From 21f06ba77c96a0d7c36c14f3b545993fd89b0ec3 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 30 May 2024 18:42:26 +0200 Subject: [PATCH 391/456] Revert "Fix output datatype for some filters." (#5028) Reverts TileDB-Inc/TileDB#4988 This has shown to have issues. We are going to revert for now so that we can ship 2.24. Filed an item to diagnose this after the release: [sc-48480]. --- TYPE: NO_HISTORY --- test/src/cpp-integration-filter-pipeline.cc | 49 ++----- .../sm/filter/bit_width_reduction_filter.cc | 5 - tiledb/sm/filter/bit_width_reduction_filter.h | 9 -- tiledb/sm/filter/compression_filter.cc | 17 +-- .../sm/filter/encryption_aes256gcm_filter.cc | 5 - .../sm/filter/encryption_aes256gcm_filter.h | 9 -- .../filter/test/unit_run_filter_pipeline.cc | 126 ------------------ tiledb/sm/filter/webp_filter.cc | 6 - tiledb/sm/filter/webp_filter.h | 9 -- 9 files changed, 18 insertions(+), 217 deletions(-) diff --git a/test/src/cpp-integration-filter-pipeline.cc b/test/src/cpp-integration-filter-pipeline.cc index a5eae4d397d7..3974003b2d05 100644 --- a/test/src/cpp-integration-filter-pipeline.cc +++ b/test/src/cpp-integration-filter-pipeline.cc @@ -34,7 +34,6 @@ #include "test/support/src/helpers.h" #include "tiledb/common/common.h" #include "tiledb/sm/cpp_api/tiledb" -#include "tiledb/sm/filter/checksum_md5_filter.h" #include "tiledb/sm/filter/webp_filter.h" #include "tiledb/stdx/utility/to_underlying.h" @@ -234,27 +233,7 @@ TEST_CASE( CHECK_NOTHROW(d1.set_filter_list(filters)); CHECK_NOTHROW(a1.set_filter_list(filters)); - } - - SECTION("- Multiple compressors following type conversion") { - auto compressor = GENERATE( - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA, - TILEDB_FILTER_GZIP, - TILEDB_FILTER_LZ4, - TILEDB_FILTER_RLE, - TILEDB_FILTER_ZSTD); - tiledb::Filter compressor_filter(ctx, compressor); - tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); - filters.add_filter(float_scale); - filters.add_filter(bzip); - filters.add_filter(compressor_filter); - CHECK_NOTHROW(d1.set_filter_list(filters)); - CHECK_NOTHROW(a1.set_filter_list(filters)); - } - - SECTION("- Delta cannot run on float input") { // Should throw without FloatScale to convert float->int32. auto delta_compressor = GENERATE( TILEDB_FILTER_POSITIVE_DELTA, @@ -264,31 +243,21 @@ TEST_CASE( filters.add_filter(delta_filter); CHECK_THROWS(d1.set_filter_list(filters)); CHECK_THROWS(a1.set_filter_list(filters)); - - tiledb::Filter checksum_filter(ctx, TILEDB_FILTER_CHECKSUM_MD5); - filters.add_filter(checksum_filter); - CHECK_THROWS(d1.set_filter_list(filters)); - CHECK_THROWS(a1.set_filter_list(filters)); } - SECTION("- Delta can run if a prior filter converts to non-float type") { - // note that this may not be a *wise* choice... but it is nonetheless viable - auto converting_filter = GENERATE( + SECTION("- Multiple compressors following type conversion") { + auto compressor = GENERATE( + TILEDB_FILTER_DOUBLE_DELTA, + TILEDB_FILTER_DELTA, TILEDB_FILTER_GZIP, TILEDB_FILTER_LZ4, TILEDB_FILTER_RLE, TILEDB_FILTER_ZSTD); - - auto delta_compressor = GENERATE( - TILEDB_FILTER_POSITIVE_DELTA, - TILEDB_FILTER_DOUBLE_DELTA, - TILEDB_FILTER_DELTA); - - tiledb::Filter convert_filter(ctx, converting_filter); - tiledb::Filter delta_filter(ctx, delta_compressor); - - filters.add_filter(convert_filter); - filters.add_filter(delta_filter); + tiledb::Filter compressor_filter(ctx, compressor); + tiledb::Filter bzip(ctx, TILEDB_FILTER_BZIP2); + filters.add_filter(float_scale); + filters.add_filter(bzip); + filters.add_filter(compressor_filter); CHECK_NOTHROW(d1.set_filter_list(filters)); CHECK_NOTHROW(a1.set_filter_list(filters)); diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index a25b2905966b..df47c6f05fd8 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -105,11 +105,6 @@ bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const { return false; } -Datatype BitWidthReductionFilter::output_datatype(Datatype) const { - /* if the compression did anything then the result is a binary blob */ - return Datatype::BLOB; -} - Status BitWidthReductionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, diff --git a/tiledb/sm/filter/bit_width_reduction_filter.h b/tiledb/sm/filter/bit_width_reduction_filter.h index b81fc2793d54..ebcab1ec93df 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.h +++ b/tiledb/sm/filter/bit_width_reduction_filter.h @@ -107,15 +107,6 @@ class BitWidthReductionFilter : public Filter { */ bool accepts_input_datatype(Datatype datatype) const override; - /** - * Returns the filter output type - * - * @param input_type Expected type used for input. Used for filters which - * change output type based on input data. e.g. XORFilter output type is - * based on byte width of input type. - */ - Datatype output_datatype(Datatype input_type) const override; - /** Return the max window size used by the filter. */ uint32_t max_window_size() const; diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index bae2a9b31e26..55c28759b8e9 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -733,14 +733,15 @@ void CompressionFilter::init_decompression_resource_pool(uint64_t size) { } } -Datatype CompressionFilter::output_datatype(Datatype) const { - /* - * If the compression did anything at all, we have a binary blob afterwards. - * Even if it didn't, the output format interleaves metadata with - * compressed data, so we can't interpret as a stream of any particular - * datatype anyway. - */ - return Datatype::BLOB; +Datatype CompressionFilter::output_datatype(Datatype datatype) const { + switch (compressor_) { + case Compressor::DOUBLE_DELTA: + case Compressor::DELTA: + return reinterpret_datatype_ == Datatype::ANY ? datatype : + reinterpret_datatype_; + default: + return datatype; + } } } // namespace sm diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.cc b/tiledb/sm/filter/encryption_aes256gcm_filter.cc index df91f800b1fe..fd9ffc4483b9 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.cc +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.cc @@ -70,11 +70,6 @@ void EncryptionAES256GCMFilter::dump(FILE* out) const { fprintf(out, "EncryptionAES256GCM"); } -Datatype EncryptionAES256GCMFilter::output_datatype(Datatype) const { - /* encryption gives us meaningless bits with overwhelming probability */ - return Datatype::BLOB; -} - Status EncryptionAES256GCMFilter::run_forward( const WriterTile&, WriterTile* const, diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.h b/tiledb/sm/filter/encryption_aes256gcm_filter.h index bc08097e6b09..31c8bdb252f5 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.h +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.h @@ -97,15 +97,6 @@ class EncryptionAES256GCMFilter : public Filter { /** Dumps the filter details in ASCII format in the selected output. */ void dump(FILE* out) const override; - /** - * Returns the filter output type - * - * @param input_type Expected type used for input. Used for filters which - * change output type based on input data. e.g. XORFilter output type is - * based on byte width of input type. - */ - Datatype output_datatype(Datatype input_type) const override; - /** * Encrypt the bytes of the input data into the output data buffer. */ diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 0e93b99d51a9..3b5645a63032 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -75,7 +75,6 @@ #include "../filter.h" #include "../filter_pipeline.h" #include "../positive_delta_filter.h" -#include "../xor_filter.h" #include "add_1_in_place_filter.h" #include "add_1_including_metadata_filter.h" #include "add_1_out_of_place_filter.h" @@ -1138,128 +1137,3 @@ TEST_CASE( } } } - -static void do_invalid_pipeline_check(FilterPipeline& pipeline) { - // Create TileDB resources for running the filter pipeline. - Config config; - ThreadPool tp(4); - auto tracker = tiledb::test::create_test_memory_tracker(); - - // Set-up test data. - SimpleVariableTestData test_data{}; - const auto& tile_data_generator = test_data.tile_data_generator(); - auto&& [tile, offsets_tile] = - tile_data_generator.create_writer_tiles(tracker); - - // in all examples this will abort with high probability - check_run_pipeline_roundtrip( - config, tp, tile, offsets_tile, pipeline, &tile_data_generator, tracker); -} - -TEST_CASE( - "Filter: Invalid XOR after compressor", "[filter][invalid-pipeline]") { - const Datatype input_type = Datatype::INT64; - - FilterPipeline pipeline; - - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, input_type)); - pipeline.add_filter( - XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} - -TEST_CASE( - "Filter: Invalid XOR after bit width reduction", - "[filter][invalid-pipeline]") { - // Create TileDB resources for running the filter pipeline. - const Datatype input_type = Datatype::INT64; - - FilterPipeline pipeline; - - pipeline.add_filter(BitWidthReductionFilter(input_type)); - pipeline.add_filter( - XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} - -TEST_CASE( - "Filter: Invalid XOR after encryption", "[filter][invalid-pipeline]") { - // Create TileDB resources for running the filter pipeline. - const Datatype input_type = Datatype::INT64; - - EncryptionKey encryption_key; - REQUIRE(encryption_key - .set_key( - EncryptionType::AES_256_GCM, - "abcdefghijklmnopqrstuvwxyz012345", - 32) - .ok()); - - FilterPipeline pipeline; - - pipeline.add_filter(EncryptionAES256GCMFilter(encryption_key, input_type)); - pipeline.add_filter( - XORFilter(pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} - -TEST_CASE( - "Filter: Invalid Delta after compressor", "[filter][invalid-pipeline]") { - const Datatype input_type = Datatype::INT64; - - FilterPipeline pipeline; - - pipeline.add_filter( - CompressionFilter(tiledb::sm::Compressor::LZ4, 5, input_type)); - pipeline.add_filter(CompressionFilter( - tiledb::sm::Compressor::DELTA, - 5, - pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} - -TEST_CASE( - "Filter: Invalid Delta after bit width reduction", - "[filter][invalid-pipeline]") { - // Create TileDB resources for running the filter pipeline. - const Datatype input_type = Datatype::INT64; - - FilterPipeline pipeline; - - pipeline.add_filter(BitWidthReductionFilter(input_type)); - pipeline.add_filter(CompressionFilter( - tiledb::sm::Compressor::DELTA, - 5, - pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} - -TEST_CASE( - "Filter: Invalid Delta after encryption", "[filter][invalid-pipeline]") { - // Create TileDB resources for running the filter pipeline. - const Datatype input_type = Datatype::INT64; - - EncryptionKey encryption_key; - REQUIRE(encryption_key - .set_key( - EncryptionType::AES_256_GCM, - "abcdefghijklmnopqrstuvwxyz012345", - 32) - .ok()); - - FilterPipeline pipeline; - - pipeline.add_filter(EncryptionAES256GCMFilter(encryption_key, input_type)); - pipeline.add_filter(CompressionFilter( - tiledb::sm::Compressor::DELTA, - 5, - pipeline.get_filter(0)->output_datatype(input_type))); - - do_invalid_pipeline_check(pipeline); -} diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 412c402fc416..961209f7ecb4 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -44,12 +44,6 @@ void WebpFilter::dump(FILE* out) const { bool WebpFilter::accepts_input_datatype(Datatype datatype) const { return datatype == Datatype::UINT8; } - -Datatype WebpFilter::output_datatype(Datatype) const { - /* compression produces blobs */ - return Datatype::BLOB; -} - } // namespace tiledb::sm #ifndef TILEDB_WEBP diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index 2d6795846842..cd6648a06cb4 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -160,15 +160,6 @@ class WebpFilter : public Filter { */ bool accepts_input_datatype(Datatype datatype) const override; - /** - * Returns the filter output type - * - * @param input_type Expected type used for input. Used for filters which - * change output type based on input data. e.g. XORFilter output type is - * based on byte width of input type. - */ - Datatype output_datatype(Datatype input_type) const override; - /** * Runs the filter forward, taking raw colorspace values as input and writing. * encoded WebP data to the TileDB Array. From 4455a3cd8d0082e2a9ceab5be26b43b441a81282 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 31 May 2024 05:37:02 -0400 Subject: [PATCH 392/456] Remove StorageManager::logger(). (#5024) Remove `StorageManager::logger()`. [sc-48294] --- TYPE: NO_HISTORY DESC: Remove `StorageManager::logger()`. --- tiledb/sm/c_api/tiledb.cc | 4 +- tiledb/sm/c_api/tiledb_filestore.cc | 12 +- tiledb/sm/storage_manager/storage_manager.cc | 103 +++++++++--------- .../storage_manager_canonical.h | 3 - 4 files changed, 57 insertions(+), 65 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 4dc6edd1d282..5824c5dee92c 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -226,7 +226,7 @@ capi_return_t tiledb_log_warn(tiledb_ctx_t* ctx, const char* message) { return TILEDB_ERR; } - auto logger = ctx->storage_manager()->logger(); + auto logger = ctx->resources().logger(); logger->warn(message); return TILEDB_OK; @@ -1722,7 +1722,7 @@ capi_return_t tiledb_subarray_alloc( (*subarray)->subarray_ = new tiledb::sm::Subarray( array->array_.get(), (tiledb::sm::stats::Stats*)nullptr, - ctx->storage_manager()->logger(), + ctx->resources().logger(), true, ctx->storage_manager()); (*subarray)->is_allocated_ = true; diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 137ac504787e..468893875497 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -281,7 +281,7 @@ int32_t tiledb_filestore_uri_import( tiledb::sm::Subarray subarray( array.get(), nullptr, - context.storage_manager()->logger(), + context.resources().logger(), true, context.storage_manager()); // We need to get the right end boundary of the last space tile. @@ -305,7 +305,7 @@ int32_t tiledb_filestore_uri_import( tiledb::sm::Subarray subarray_cloud_fix( array.get(), nullptr, - context.storage_manager()->logger(), + context.resources().logger(), true, context.storage_manager()); @@ -433,7 +433,7 @@ int32_t tiledb_filestore_uri_export( tiledb::sm::Subarray subarray( array.get(), nullptr, - context.storage_manager()->logger(), + context.resources().logger(), true, context.storage_manager()); uint64_t subarray_range_arr[] = {start_range, end_range}; @@ -550,7 +550,7 @@ int32_t tiledb_filestore_buffer_import( tiledb::sm::Subarray subarray( array.get(), nullptr, - context.storage_manager()->logger(), + context.resources().logger(), true, context.storage_manager()); uint64_t subarray_range_arr[] = { @@ -612,7 +612,7 @@ int32_t tiledb_filestore_buffer_export( tiledb::sm::Subarray subarray( array.get(), nullptr, - context.storage_manager()->logger(), + context.resources().logger(), true, context.storage_manager()); uint64_t subarray_range_arr[] = { diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index c804dc4a9a9b..20307992660e 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -72,6 +72,13 @@ using namespace tiledb::common; namespace tiledb::sm { +class StorageManagerException : public StatusException { + public: + explicit StorageManagerException(const std::string& message) + : StatusException("StorageManager", message) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -128,15 +135,14 @@ Status StorageManagerCanonical::array_create( const EncryptionKey& encryption_key) { // Check array schema if (array_schema == nullptr) { - return logger_->status( - Status_StorageManagerError("Cannot create array; Empty array schema")); + throw StorageManagerException("Cannot create array; Empty array schema"); } // Check if array exists if (is_array(resources_, array_uri)) { - return logger_->status(Status_StorageManagerError( - std::string("Cannot create array; Array '") + array_uri.c_str() + - "' already exists")); + throw StorageManagerException( + "Cannot create array; Array '" + array_uri.to_string() + + "' already exists"); } std::lock_guard lock{object_create_mtx_}; @@ -225,8 +231,8 @@ Status StorageManager::array_evolve_schema( const EncryptionKey& encryption_key) { // Check array schema if (schema_evolution == nullptr) { - return logger_->status(Status_StorageManagerError( - "Cannot evolve array; Empty schema evolution")); + throw StorageManagerException( + "Cannot evolve array; Empty schema evolution"); } if (array_uri.is_tiledb()) { @@ -244,9 +250,9 @@ Status StorageManager::array_evolve_schema( // Check if array exists if (!is_array(resources_, array_uri)) { - return logger_->status(Status_StorageManagerError( - std::string("Cannot evolve array; Array '") + array_uri.c_str() + - "' not exists")); + throw StorageManagerException( + "Cannot evolve array; Array '" + array_uri.to_string() + + "' not exists"); } auto&& array_schema = array_dir.load_array_schema_latest( @@ -277,9 +283,8 @@ Status StorageManager::array_evolve_schema( Status st = store_array_schema(array_schema_evolved, encryption_key); if (!st.ok()) { - logger_->status_no_return_value(st); - return logger_->status(Status_StorageManagerError( - "Cannot evolve schema; Not able to store evolved array schema.")); + throw StorageManagerException( + "Cannot evolve schema; Not able to store evolved array schema."); } return Status::Ok(); @@ -288,10 +293,11 @@ Status StorageManager::array_evolve_schema( Status StorageManagerCanonical::array_upgrade_version( const URI& array_uri, const Config& override_config) { // Check if array exists - if (!is_array(resources_, array_uri)) - return logger_->status(Status_StorageManagerError( - std::string("Cannot upgrade array; Array '") + array_uri.c_str() + - "' does not exist")); + if (!is_array(resources_, array_uri)) { + throw StorageManagerException( + "Cannot upgrade array; Array '" + array_uri.to_string() + + "' does not exist"); + } // Load URIs from the array directory tiledb::sm::ArrayDirectory array_dir{ @@ -333,33 +339,25 @@ Status StorageManagerCanonical::array_upgrade_version( // Create array schema directory if necessary URI array_schema_dir_uri = array_uri.join_path(constants::array_schema_dir_name); - auto st = resources_.vfs().create_dir(array_schema_dir_uri); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); + throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); // Store array schema - st = store_array_schema(array_schema, encryption_key_cfg); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); + throw_if_not_ok(store_array_schema(array_schema, encryption_key_cfg)); // Create commit directory if necessary URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); - RETURN_NOT_OK_ELSE( - resources_.vfs().create_dir(array_commit_uri), - logger_->status_no_return_value(st)); + throw_if_not_ok(resources_.vfs().create_dir(array_commit_uri)); // Create fragments directory if necessary URI array_fragments_uri = array_uri.join_path(constants::array_fragments_dir_name); - RETURN_NOT_OK_ELSE( - resources_.vfs().create_dir(array_fragments_uri), - logger_->status_no_return_value(st)); + throw_if_not_ok(resources_.vfs().create_dir(array_fragments_uri)); // Create fragment metadata directory if necessary URI array_fragment_metadata_uri = array_uri.join_path(constants::array_fragment_meta_dir_name); - RETURN_NOT_OK_ELSE( - resources_.vfs().create_dir(array_fragment_metadata_uri), - logger_->status_no_return_value(st)); + throw_if_not_ok(resources_.vfs().create_dir(array_fragment_metadata_uri)); } return Status::Ok(); @@ -371,8 +369,9 @@ Status StorageManagerCanonical::async_push_query(Query* query) { [this, query]() { // Process query. Status st = query_submit(query); - if (!st.ok()) - logger_->status_no_return_value(st); + if (!st.ok()) { + resources_.logger()->status_no_return_value(st); + } return st; }, [query]() { @@ -432,20 +431,20 @@ StorageManagerCanonical::tags() const { Status StorageManagerCanonical::group_create(const std::string& group_uri) { // Create group URI URI uri(group_uri); - if (uri.is_invalid()) - return logger_->status(Status_StorageManagerError( - "Cannot create group '" + group_uri + "'; Invalid group URI")); + if (uri.is_invalid()) { + throw StorageManagerException( + "Cannot create group '" + group_uri + "'; Invalid group URI"); + } // Check if group exists bool exists; throw_if_not_ok(is_group(resources_, uri, &exists)); - if (exists) - return logger_->status(Status_StorageManagerError( - std::string("Cannot create group; Group '") + uri.c_str() + - "' already exists")); + if (exists) { + throw StorageManagerException( + "Cannot create group; Group '" + uri.to_string() + "' already exists"); + } std::lock_guard lock{object_create_mtx_}; - if (uri.is_tiledb()) { Group group(resources_, uri, this); throw_if_not_ok( @@ -481,8 +480,8 @@ Status StorageManagerCanonical::object_iter_begin( // Sanity check URI path_uri(path); if (path_uri.is_invalid()) { - return logger_->status(Status_StorageManagerError( - "Cannot create object iterator; Invalid input path")); + throw StorageManagerException( + "Cannot create object iterator; Invalid input path"); } // Get all contents of path @@ -514,8 +513,8 @@ Status StorageManagerCanonical::object_iter_begin( // Sanity check URI path_uri(path); if (path_uri.is_invalid()) { - return logger_->status(Status_StorageManagerError( - "Cannot create object iterator; Invalid input path")); + throw StorageManagerException( + "Cannot create object iterator; Invalid input path"); } // Get all contents of path @@ -711,8 +710,8 @@ Status StorageManagerCanonical::store_array_schema( for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { auto enmr = array_schema->get_enumeration(enmr_name); if (enmr == nullptr) { - return logger_->status(Status_StorageManagerError( - "Error serializing enumeration; Loaded enumeration is null")); + throw StorageManagerException( + "Error serializing enumeration; Loaded enumeration is null"); } SizeComputationSerializer enumeration_size_serializer; @@ -737,10 +736,6 @@ void StorageManagerCanonical::wait_for_zero_in_progress() { lck, [this]() { return queries_in_progress_ == 0; }); } -shared_ptr StorageManagerCanonical::logger() const { - return logger_; -} - /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ @@ -761,16 +756,16 @@ Status StorageManagerCanonical::group_metadata_consolidate( // Check group URI URI group_uri(group_name); if (group_uri.is_invalid()) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate group metadata; Invalid URI")); + throw StorageManagerException( + "Cannot consolidate group metadata; Invalid URI"); } // Check if group exists ObjectType obj_type; throw_if_not_ok(object_type(resources_, group_uri, &obj_type)); if (obj_type != ObjectType::GROUP) { - return logger_->status(Status_StorageManagerError( - "Cannot consolidate group metadata; Group does not exist")); + throw StorageManagerException( + "Cannot consolidate group metadata; Group does not exist"); } // Consolidate diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 3d1d549e4b71..6b68315c63ce 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -309,9 +309,6 @@ class StorageManagerCanonical { return resources_; } - /** Returns the internal logger object. */ - shared_ptr logger() const; - /** * Consolidates the metadata of a group into a single file. * From 73dcd88850f9675108773849758645ba84f18b60 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Fri, 31 May 2024 12:23:05 +0200 Subject: [PATCH 393/456] Bump dev version to 2.25.0. (#5031) --- TYPE: NO_HISTORY DESC: Bump dev version to 2.25.0. --- tiledb/doxygen/source/conf.py | 4 ++-- tiledb/sm/c_api/tiledb_version.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tiledb/doxygen/source/conf.py b/tiledb/doxygen/source/conf.py index b39f626ca33d..c567b5461ba1 100644 --- a/tiledb/doxygen/source/conf.py +++ b/tiledb/doxygen/source/conf.py @@ -76,9 +76,9 @@ author = 'TileDB, Inc.' # The short X.Y version. -version = '2.24' +version = '2.25' # The full version, including alpha/beta/rc tags. -release = '2.24.0' +release = '2.25.0' # Breathe extension configuration. doxygen_xml_dir = os.path.join(TILEDB_BUILD_DIR, 'xml/') diff --git a/tiledb/sm/c_api/tiledb_version.h b/tiledb/sm/c_api/tiledb_version.h index f5b57af5d7bf..cc75ba3e994d 100644 --- a/tiledb/sm/c_api/tiledb_version.h +++ b/tiledb/sm/c_api/tiledb_version.h @@ -27,5 +27,5 @@ */ #define TILEDB_VERSION_MAJOR 2 -#define TILEDB_VERSION_MINOR 24 +#define TILEDB_VERSION_MINOR 25 #define TILEDB_VERSION_PATCH 0 From da47362d7c11d9a17a403a7d7165733cf5049a63 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:04:45 -0400 Subject: [PATCH 394/456] Migrate APIs out of StorageManager: store_array_schema. (#5019) Migrate APIs out of `StorageManager`: `store_array_schema`. [sc-45794] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `store_array_schema`. --- tiledb/CMakeLists.txt | 1 + tiledb/sm/array_schema/array_schema.cc | 1 - tiledb/sm/array_schema/array_schema.h | 8 +- tiledb/sm/array_schema/auxiliary.cc | 129 ++++++++++++++++++ tiledb/sm/array_schema/auxiliary.h | 72 ++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 85 +----------- .../storage_manager_canonical.h | 11 -- 7 files changed, 212 insertions(+), 95 deletions(-) create mode 100644 tiledb/sm/array_schema/auxiliary.cc create mode 100644 tiledb/sm/array_schema/auxiliary.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index e3ce9de0f6d2..ea9659ba2e95 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -149,6 +149,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/array_schema.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/array_schema_evolution.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/attribute.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/auxiliary.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension_label.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/domain.cc diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index b0fbc42f51b9..cb32bc3a2924 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -54,7 +54,6 @@ #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/integral_type_casts.h" #include "tiledb/sm/misc/tdb_time.h" -#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/tile/generic_tile_io.h" #include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/type/apply_with_type.h" diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index 8c360db43071..ba300d398b5e 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * @copyright Copyright (c) 2016 MIT and Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a copy @@ -46,8 +46,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Attribute; class Buffer; @@ -744,7 +743,6 @@ class ArraySchema { void clear(); }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_ARRAY_SCHEMA_H diff --git a/tiledb/sm/array_schema/auxiliary.cc b/tiledb/sm/array_schema/auxiliary.cc new file mode 100644 index 000000000000..0d05fc878b18 --- /dev/null +++ b/tiledb/sm/array_schema/auxiliary.cc @@ -0,0 +1,129 @@ +/** + * @file auxiliary.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements auxiliary functions to support the ArraySchema class. + * + * This file exists solely as an intermediary solution to resolve build issues + * with `capi_context_stub` when migrating `store_array_schema` out of + * `StorageManagerCanonical`. At present, there is an interdependency chain + * with `generic_tile_io` which must be resolved before this function can be + * placed into `class ArraySchema`. As such, this file is _intentionally_ left + * out of the `array_schema` object library and _must_ remain that way. + */ + +#include "tiledb/sm/array_schema/auxiliary.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/storage_manager/context_resources.h" +#include "tiledb/sm/tile/generic_tile_io.h" +#include "tiledb/sm/tile/tile.h" + +namespace tiledb::sm { + +/* ********************************* */ +/* API */ +/* ********************************* */ + +Status store_array_schema( + ContextResources& resources, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key) { + const URI schema_uri = array_schema->uri(); + + // Serialize + SizeComputationSerializer size_computation_serializer; + array_schema->serialize(size_computation_serializer); + + auto tile{WriterTile::from_generic( + size_computation_serializer.size(), + resources.ephemeral_memory_tracker())}; + Serializer serializer(tile->data(), tile->size()); + array_schema->serialize(serializer); + resources.stats().add_counter("write_array_schema_size", tile->size()); + + // Delete file if it exists already + bool exists; + throw_if_not_ok(resources.vfs().is_file(schema_uri, &exists)); + if (exists) { + throw_if_not_ok(resources.vfs().remove_file(schema_uri)); + } + + // Check if the array schema directory exists + // If not create it, this is caused by a pre-v10 array + bool schema_dir_exists = false; + URI array_schema_dir_uri = + array_schema->array_uri().join_path(constants::array_schema_dir_name); + throw_if_not_ok( + resources.vfs().is_dir(array_schema_dir_uri, &schema_dir_exists)); + + if (!schema_dir_exists) { + throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); + } + + GenericTileIO::store_data(resources, schema_uri, tile, encryption_key); + + // Create the `__enumerations` directory under `__schema` if it doesn't + // exist. This might happen if someone tries to add an enumeration to an + // array created before version 19. + bool enumerations_dir_exists = false; + URI array_enumerations_dir_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + throw_if_not_ok(resources.vfs().is_dir( + array_enumerations_dir_uri, &enumerations_dir_exists)); + + if (!enumerations_dir_exists) { + throw_if_not_ok(resources.vfs().create_dir(array_enumerations_dir_uri)); + } + + // Serialize all enumerations into the `__enumerations` directory + for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { + auto enmr = array_schema->get_enumeration(enmr_name); + if (enmr == nullptr) { + throw std::runtime_error( + "Error serializing enumeration; Loaded enumeration is null"); + } + + SizeComputationSerializer enumeration_size_serializer; + enmr->serialize(enumeration_size_serializer); + + auto tile{WriterTile::from_generic( + enumeration_size_serializer.size(), + resources.ephemeral_memory_tracker())}; + Serializer serializer(tile->data(), tile->size()); + enmr->serialize(serializer); + + auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); + GenericTileIO::store_data(resources, abs_enmr_uri, tile, encryption_key); + } + + return Status::Ok(); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/auxiliary.h b/tiledb/sm/array_schema/auxiliary.h new file mode 100644 index 000000000000..85919a9e8704 --- /dev/null +++ b/tiledb/sm/array_schema/auxiliary.h @@ -0,0 +1,72 @@ +/** + * @file auxiliary.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines auxiliary functions to support class ArraySchema. + * + * This file exists solely as an intermediary solution to resolve build issues + * with `capi_context_stub` when migrating `store_array_schema` out of + * `StorageManagerCanonical`. At present, there is an interdependency chain + * with `generic_tile_io` which must be resolved before this function can be + * placed into `class ArraySchema`. As such, this file is _intentionally_ left + * out of the `array_schema` object library and _must_ remain that way. + */ + +#ifndef TILEDB_AUXILIARY_ARRAY_SCHEMA_H +#define TILEDB_AUXILIARY_ARRAY_SCHEMA_H + +#include "tiledb/common/common.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +class ArraySchema; +class ContextResources; +class EncryptionKey; + +/* ********************************* */ +/* API */ +/* ********************************* */ + +/** + * Stores an array schema into persistent storage. + * + * @param resources The context resources. + * @param array_schema The array schema to be stored. + * @param encryption_key The encryption key to use. + * @return Status + */ +Status store_array_schema( + ContextResources& resources, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key); + +} // namespace tiledb::sm + +#endif // TILEDB_AUXILIARY_ARRAY_SCHEMA_H diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 20307992660e..d77306e52e88 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -45,6 +45,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/auxiliary.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/enums/array_type.h" @@ -211,9 +212,9 @@ Status StorageManagerCanonical::array_create( (const void*)encryption_key_from_cfg.c_str(), static_cast(encryption_key_from_cfg.size()))); } - st = store_array_schema(array_schema, encryption_key_cfg); + st = store_array_schema(resources_, array_schema, encryption_key_cfg); } else { - st = store_array_schema(array_schema, encryption_key); + st = store_array_schema(resources_, array_schema, encryption_key); } // Store array schema @@ -281,7 +282,8 @@ Status StorageManager::array_evolve_schema( // Evolve schema auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - Status st = store_array_schema(array_schema_evolved, encryption_key); + Status st = + store_array_schema(resources_, array_schema_evolved, encryption_key); if (!st.ok()) { throw StorageManagerException( "Cannot evolve schema; Not able to store evolved array schema."); @@ -342,7 +344,8 @@ Status StorageManagerCanonical::array_upgrade_version( throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); // Store array schema - throw_if_not_ok(store_array_schema(array_schema, encryption_key_cfg)); + auto st = store_array_schema(resources_, array_schema, encryption_key_cfg); + RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); // Create commit directory if necessary URI array_commit_uri = @@ -656,80 +659,6 @@ Status StorageManagerCanonical::set_tag( return Status::Ok(); } -Status StorageManagerCanonical::store_array_schema( - const shared_ptr& array_schema, - const EncryptionKey& encryption_key) { - const URI schema_uri = array_schema->uri(); - - // Serialize - SizeComputationSerializer size_computation_serializer; - array_schema->serialize(size_computation_serializer); - - auto tile{WriterTile::from_generic( - size_computation_serializer.size(), - resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile->data(), tile->size()); - array_schema->serialize(serializer); - resources_.stats().add_counter("write_array_schema_size", tile->size()); - - // Delete file if it exists already - bool exists; - throw_if_not_ok(resources_.vfs().is_file(schema_uri, &exists)); - if (exists) { - throw_if_not_ok(resources_.vfs().remove_file(schema_uri)); - } - - // Check if the array schema directory exists - // If not create it, this is caused by a pre-v10 array - bool schema_dir_exists = false; - URI array_schema_dir_uri = - array_schema->array_uri().join_path(constants::array_schema_dir_name); - throw_if_not_ok( - resources_.vfs().is_dir(array_schema_dir_uri, &schema_dir_exists)); - - if (!schema_dir_exists) { - throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); - } - - GenericTileIO::store_data(resources_, schema_uri, tile, encryption_key); - - // Create the `__enumerations` directory under `__schema` if it doesn't - // exist. This might happen if someone tries to add an enumeration to an - // array created before version 19. - bool enumerations_dir_exists = false; - URI array_enumerations_dir_uri = - array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); - throw_if_not_ok(resources_.vfs().is_dir( - array_enumerations_dir_uri, &enumerations_dir_exists)); - - if (!enumerations_dir_exists) { - throw_if_not_ok(resources_.vfs().create_dir(array_enumerations_dir_uri)); - } - - // Serialize all enumerations into the `__enumerations` directory - for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { - auto enmr = array_schema->get_enumeration(enmr_name); - if (enmr == nullptr) { - throw StorageManagerException( - "Error serializing enumeration; Loaded enumeration is null"); - } - - SizeComputationSerializer enumeration_size_serializer; - enmr->serialize(enumeration_size_serializer); - - auto tile{WriterTile::from_generic( - enumeration_size_serializer.size(), - resources_.ephemeral_memory_tracker())}; - Serializer serializer(tile->data(), tile->size()); - enmr->serialize(serializer); - - auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); - GenericTileIO::store_data(resources_, abs_enmr_uri, tile, encryption_key); - } - - return Status::Ok(); -} - void StorageManagerCanonical::wait_for_zero_in_progress() { std::unique_lock lck(queries_in_progress_mtx_); queries_in_progress_cv_.wait( diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 6b68315c63ce..0e5571b7c8bb 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -294,17 +294,6 @@ class StorageManagerCanonical { */ Status set_tag(const std::string& key, const std::string& value); - /** - * Stores an array schema into persistent storage. - * - * @param array_schema The array metadata to be stored. - * @param encryption_key The encryption key to use. - * @return Status - */ - Status store_array_schema( - const shared_ptr& array_schema, - const EncryptionKey& encryption_key); - [[nodiscard]] inline ContextResources& resources() const { return resources_; } From ef14b41602b3b74210b4f2aa4523da3f6462e7a1 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:06:51 -0400 Subject: [PATCH 395/456] Migrate APIs out of StorageManager: group_create. (#5018) Migrate APIs out of `StorageManager`: `group_create`. [sc-46733] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `group_create`. --- tiledb/CMakeLists.txt | 1 + tiledb/api/c_api/group/group_api.cc | 3 +- .../storage_manager_override.h | 4 -- tiledb/sm/CMakeLists.txt | 1 + tiledb/sm/group/CMakeLists.txt | 3 +- tiledb/sm/group/group.cc | 43 ++++++++++++++ tiledb/sm/group/group.h | 9 +++ tiledb/sm/object/CMakeLists.txt | 35 ++++++++++++ tiledb/sm/object/object_mutex.cc | 42 ++++++++++++++ tiledb/sm/object/object_mutex.h | 56 +++++++++++++++++++ .../object/test/compile_object_mutex_main.cc | 34 +++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 44 +-------------- .../storage_manager_canonical.h | 11 ---- 13 files changed, 227 insertions(+), 59 deletions(-) create mode 100644 tiledb/sm/object/CMakeLists.txt create mode 100644 tiledb/sm/object/object_mutex.cc create mode 100644 tiledb/sm/object/object_mutex.h create mode 100644 tiledb/sm/object/test/compile_object_mutex_main.cc diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index ea9659ba2e95..977bdf225879 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -243,6 +243,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object_mutex.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/serialization.cc diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 91b66589787e..177ddc873293 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -81,7 +81,8 @@ capi_return_t tiledb_group_create( tiledb_ctx_handle_t* ctx, const char* group_uri) { ensure_group_uri_argument_is_valid(group_uri); - throw_if_not_ok(ctx->storage_manager()->group_create(group_uri)); + throw_if_not_ok( + tiledb::sm::Group::create(ctx->resources(), tiledb::sm::URI(group_uri))); return TILEDB_OK; } diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 6dbddcc07d33..66017e40dba7 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -61,10 +61,6 @@ class StorageManagerStub { inline Status cancel_all_tasks() { return Status{}; } - inline Status group_create(const std::string&) { - throw std::logic_error( - "StorageManagerStub does not support group creation"); - } inline Status group_metadata_consolidate(const char*, const Config&) { throw std::logic_error( "StorageManagerStub does not support group metadata consolidation"); diff --git a/tiledb/sm/CMakeLists.txt b/tiledb/sm/CMakeLists.txt index 8f875d7c66fe..635f09ddd41b 100644 --- a/tiledb/sm/CMakeLists.txt +++ b/tiledb/sm/CMakeLists.txt @@ -39,6 +39,7 @@ add_subdirectory(fragment) add_subdirectory(group) add_subdirectory(metadata) add_subdirectory(misc) +add_subdirectory(object) add_subdirectory(query) add_subdirectory(query_plan) add_subdirectory(rtree) diff --git a/tiledb/sm/group/CMakeLists.txt b/tiledb/sm/group/CMakeLists.txt index fbd09e21dba6..6ae37a84feef 100644 --- a/tiledb/sm/group/CMakeLists.txt +++ b/tiledb/sm/group/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,7 @@ include(common NO_POLICY_SCOPE) add_library(group OBJECT group_directory.cc group.cc group_details.cc group_details_v1.cc group_details_v2.cc group_member.cc group_member_v1.cc group_member_v2.cc) target_link_libraries(group PUBLIC baseline $) target_link_libraries(group PUBLIC buffer $) +target_link_libraries(group PUBLIC object_mutex $) # # Test-compile of object library ensures link-completeness diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index d8abbff7d52e..62052a9523ba 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -46,6 +46,8 @@ #include "tiledb/sm/group/group_member_v2.h" #include "tiledb/sm/metadata/metadata.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_mutex.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/storage_manager/context_resources.h" @@ -83,6 +85,47 @@ Group::Group( memory_tracker_->set_type(MemoryTrackerType::GROUP); } +Status Group::create(ContextResources& resources, const URI& uri) { + // Create group URI + if (uri.is_invalid()) + throw GroupException( + "Cannot create group '" + uri.to_string() + "'; Invalid group URI"); + + // Check if group exists + bool exists; + throw_if_not_ok(is_group(resources, uri, &exists)); + if (exists) { + throw GroupException( + "Cannot create group; Group '" + uri.to_string() + "' already exists"); + } + + std::lock_guard lock{object_mtx}; + if (uri.is_tiledb()) { + StorageManager storage_manager( + resources, resources.logger(), resources.config()); + Group group(resources, uri, &storage_manager); + throw_if_not_ok( + resources.rest_client()->post_group_create_to_rest(uri, &group)); + return Status::Ok(); + } + + // Create group directory + throw_if_not_ok(resources.vfs().create_dir(uri)); + + // Create group file + URI group_filename = uri.join_path(constants::group_filename); + throw_if_not_ok(resources.vfs().touch(group_filename)); + + // Create metadata folder + throw_if_not_ok(resources.vfs().create_dir( + uri.join_path(constants::group_metadata_dir_name))); + + // Create group detail folder + throw_if_not_ok(resources.vfs().create_dir( + uri.join_path(constants::group_detail_dir_name))); + return Status::Ok(); +} + void Group::open( QueryType query_type, uint64_t timestamp_start, uint64_t timestamp_end) { // Checks diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 0ba2209a1901..bdc703adee78 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -80,6 +80,15 @@ class Group { /** Destructor. */ ~Group() = default; + /** + * Creates a TileDB group. + * + * @param resources The context resources. + * @param uri The URI of the group to be created. + * @return Status + */ + static Status create(ContextResources& resources, const URI& uri); + /** Returns the group directory object. */ const shared_ptr group_directory() const; diff --git a/tiledb/sm/object/CMakeLists.txt b/tiledb/sm/object/CMakeLists.txt new file mode 100644 index 000000000000..d04ee3fe69e0 --- /dev/null +++ b/tiledb/sm/object/CMakeLists.txt @@ -0,0 +1,35 @@ +# +# tiledb/sm/object/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +# +# `object_mutex` object library +# +commence(object_library object_mutex) + this_target_sources(object_mutex.cc) +conclude(object_library) diff --git a/tiledb/sm/object/object_mutex.cc b/tiledb/sm/object/object_mutex.cc new file mode 100644 index 000000000000..b43e2b767a2f --- /dev/null +++ b/tiledb/sm/object/object_mutex.cc @@ -0,0 +1,42 @@ +/** + * @file object_mutex.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines an object mutex. + */ + +#include "tiledb/sm/object/object_mutex.h" + +namespace tiledb::sm { + +/** + * Mutex that protects the atomicity of object operations. + */ +std::mutex object_mtx; + +} // namespace tiledb::sm diff --git a/tiledb/sm/object/object_mutex.h b/tiledb/sm/object/object_mutex.h new file mode 100644 index 000000000000..47dad744dec8 --- /dev/null +++ b/tiledb/sm/object/object_mutex.h @@ -0,0 +1,56 @@ +/** + * @file object_mutex.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares an object mutex. + */ + +#ifndef TILEDB_OBJECT_MUTEX_H +#define TILEDB_OBJECT_MUTEX_H + +#include + +namespace tiledb::sm { + +/** + * There was previously a mutex member variable in `StorageManagerCanonical` + * for providing thread-safety upon the creation of TileDB objects. This header + * defines a standalone mutex to preserve this legacy behavior. The mutex is + * intended to act as an interim solution as APIs are migrated out of + * `StorageManagerCanonical`, but will likely eventually be adopted to protect + * the atomicity of all object operations. + * + * @section Maturity + * Once the migration of APIs out of `StorageManagerCanonical` is complete, + * an audit should be done to determine the true merit of this header. + */ +extern std::mutex object_mtx; + +} // namespace tiledb::sm + +#endif // TILEDB_OBJECT_MUTEX_H diff --git a/tiledb/sm/object/test/compile_object_mutex_main.cc b/tiledb/sm/object/test/compile_object_mutex_main.cc new file mode 100644 index 000000000000..11cab503fe1b --- /dev/null +++ b/tiledb/sm/object/test/compile_object_mutex_main.cc @@ -0,0 +1,34 @@ +/** + * @file compile_object_mutex_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../object_mutex.h" + +int main() { + std::lock_guard lock{tiledb::sm::object_mtx}; + return 0; +} diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index d77306e52e88..b7dc00ca8e19 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -59,6 +59,7 @@ #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" #include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_mutex.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/storage_manager/storage_manager.h" @@ -146,7 +147,7 @@ Status StorageManagerCanonical::array_create( "' already exists"); } - std::lock_guard lock{object_create_mtx_}; + std::lock_guard lock{object_mtx}; array_schema->set_array_uri(array_uri); array_schema->generate_uri(); array_schema->check(config_); @@ -431,47 +432,6 @@ StorageManagerCanonical::tags() const { return tags_; } -Status StorageManagerCanonical::group_create(const std::string& group_uri) { - // Create group URI - URI uri(group_uri); - if (uri.is_invalid()) { - throw StorageManagerException( - "Cannot create group '" + group_uri + "'; Invalid group URI"); - } - - // Check if group exists - bool exists; - throw_if_not_ok(is_group(resources_, uri, &exists)); - if (exists) { - throw StorageManagerException( - "Cannot create group; Group '" + uri.to_string() + "' already exists"); - } - - std::lock_guard lock{object_create_mtx_}; - if (uri.is_tiledb()) { - Group group(resources_, uri, this); - throw_if_not_ok( - resources_.rest_client()->post_group_create_to_rest(uri, &group)); - return Status::Ok(); - } - - // Create group directory - throw_if_not_ok(resources_.vfs().create_dir(uri)); - - // Create group file - URI group_filename = uri.join_path(constants::group_filename); - throw_if_not_ok(resources_.vfs().touch(group_filename)); - - // Create metadata folder - throw_if_not_ok(resources_.vfs().create_dir( - uri.join_path(constants::group_metadata_dir_name))); - - // Create group detail folder - throw_if_not_ok(resources_.vfs().create_dir( - uri.join_path(constants::group_detail_dir_name))); - return Status::Ok(); -} - void StorageManagerCanonical::increment_in_progress() { std::unique_lock lck(queries_in_progress_mtx_); queries_in_progress_++; diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 0e5571b7c8bb..9c70fbf39b5f 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -191,14 +191,6 @@ class StorageManagerCanonical { /** Returns the current map of any set tags. */ const std::unordered_map& tags() const; - /** - * Creates a TileDB group. - * - * @param group The URI of the group to be created. - * @return Status - */ - Status group_create(const std::string& group); - /** * Creates a new object iterator for the input path. The iteration * in this case will be recursive in the entire directory tree rooted @@ -373,9 +365,6 @@ class StorageManagerCanonical { /** Mutex protecting cancellation_in_progress_. */ std::mutex cancellation_in_progress_mtx_; - /** Mutex for providing thread-safety upon creating TileDB objects. */ - std::mutex object_create_mtx_; - /** Stores the TileDB configuration parameters. */ Config config_; From 61686a39e93a6bf66042cace703ef16a09d48318 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 3 Jun 2024 19:14:39 +0300 Subject: [PATCH 396/456] Do not allow adding members with relative URIs in remote groups. (#5025) [SC-39197](https://app.shortcut.com/tiledb-inc/story/39197/failed-to-write-group-with-relative-uri-to-rest) Adding group members with relative URIs currently fails at the time of making the REST API call when closing the group, because the REST server has not yet implemented the logic to get the group's absolute path. This PR improves experience by failing earlier, at the time of adding the member, if it is on a remote group and has a relative URI. --- TYPE: BUG DESC: Fail early when trying to add members with relative URIs in remote groups. --- test/src/unit-cppapi-group.cc | 38 +++++++++++++++++++++++++++++++++++ tiledb/sm/group/group.cc | 10 ++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/test/src/unit-cppapi-group.cc b/test/src/unit-cppapi-group.cc index 662f91515110..bf8c8ab916ca 100644 --- a/test/src/unit-cppapi-group.cc +++ b/test/src/unit-cppapi-group.cc @@ -1002,3 +1002,41 @@ TEST_CASE("C++ API: Group close group with error", "[cppapi][group][error]") { cleaner(); } + +TEST_CASE( + "C++ API: Group with Relative URI members, write/read, rest", + "[cppapi][group][relative][rest]") { + VFSTestSetup vfs_test_setup; + tiledb::Context ctx{vfs_test_setup.ctx()}; + auto group_name{vfs_test_setup.array_uri("groups_relative")}; + auto subgroup_name = group_name + "/subgroup"; + + // Create groups + tiledb::create_group(ctx, group_name); + tiledb::create_group(ctx, subgroup_name); + + // Open group in write mode + { + auto group = tiledb::Group(ctx, group_name, TILEDB_WRITE); + if (vfs_test_setup.is_rest()) { + CHECK_THROWS_WITH( + group.add_member("subgroup", true, "subgroup"), + Catch::Matchers::EndsWith("Cannot add member; Remote groups do not " + "support members with relative " + "URIs")); + } else { + CHECK_NOTHROW(group.add_member("subgroup", true, "subgroup")); + } + group.close(); + } + + if (!vfs_test_setup.is_rest()) { + auto group = tiledb::Group(ctx, group_name, TILEDB_READ); + + auto subgroup_member = group.member("subgroup"); + + CHECK(subgroup_member.type() == tiledb::Object::Type::Group); + CHECK(subgroup_member.name() == "subgroup"); + CHECK(group.is_relative("subgroup")); + } +} diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 62052a9523ba..1db111d5685d 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -616,11 +616,19 @@ void Group::mark_member_for_addition( throw GroupException("Cannot add member; Group is not open"); } + if (remote_ && relative) { + // Relative URIs are supported in the capnp serialization format, but the + // REST server has not yet implemented the logic. + throw GroupException( + "Cannot add member; Remote groups do not support members with relative " + "URIs"); + } + // Check mode if (query_type_ != QueryType::WRITE && query_type_ != QueryType::MODIFY_EXCLUSIVE) { throw GroupException( - "Cannot get member; Group was not opened in write or modify_exclusive " + "Cannot add member; Group was not opened in write or modify_exclusive " "mode"); } group_details_->mark_member_for_addition( From a872f2959b957b673fdd56714414dedf98bf3b4b Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 3 Jun 2024 17:36:23 -0400 Subject: [PATCH 397/456] Migrate APIs out of StorageManager: array_create. (#5036) Migrate APIs out of `StorageManager`: `array_create`. [sc-44985] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `array_create`. --- test/src/unit-enumerations.cc | 2 +- test/src/unit-request-handlers.cc | 4 +- tiledb/sm/array/array.cc | 99 +++++++++++++++++++ tiledb/sm/array/array.h | 15 +++ tiledb/sm/array/test/unit_consistency.h | 8 +- tiledb/sm/c_api/tiledb.cc | 22 +++-- tiledb/sm/query_plan/test/unit_query_plan.cc | 7 +- tiledb/sm/storage_manager/storage_manager.cc | 96 ------------------ .../storage_manager_canonical.h | 13 --- .../sm/subarray/test/unit_add_ranges_list.cc | 5 +- 10 files changed, 138 insertions(+), 133 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 4ed163239caa..11c9ed1427ac 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2879,7 +2879,7 @@ shared_ptr EnumerationFx::create_schema() { void EnumerationFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); + throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); } shared_ptr EnumerationFx::get_array(QueryType type) { diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index ff0c59655638..17c5287909fb 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -357,7 +357,7 @@ RequestHandlerFx::~RequestHandlerFx() { void RequestHandlerFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); + throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); } void RequestHandlerFx::delete_array() { diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index f2f228630dbb..0080ee73bba4 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -39,6 +39,7 @@ #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/auxiliary.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/crypto/crypto.h" @@ -51,6 +52,8 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_mutex.h" #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/rest/rest_client.h" @@ -174,6 +177,102 @@ const EncryptionKey* Array::encryption_key() const { return opened_array_->encryption_key(); } +Status Array::create( + ContextResources& resources, + const URI& array_uri, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key) { + // Check array schema + if (array_schema == nullptr) { + throw ArrayException("Cannot create array; Empty array schema"); + } + + // Check if array exists + if (is_array(resources, array_uri)) { + throw ArrayException( + "Cannot create array; Array '" + array_uri.to_string() + + "' already exists"); + } + + std::lock_guard lock{object_mtx}; + array_schema->set_array_uri(array_uri); + array_schema->generate_uri(); + array_schema->check(resources.config()); + + // Create array directory + throw_if_not_ok(resources.vfs().create_dir(array_uri)); + + // Create array schema directory + URI array_schema_dir_uri = + array_uri.join_path(constants::array_schema_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); + + // Create the enumerations directory inside the array schema directory + URI array_enumerations_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_enumerations_uri)); + + // Create commit directory + URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_commit_uri)); + + // Create fragments directory + URI array_fragments_uri = + array_uri.join_path(constants::array_fragments_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragments_uri)); + + // Create array metadata directory + URI array_metadata_uri = + array_uri.join_path(constants::array_metadata_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_metadata_uri)); + + // Create fragment metadata directory + URI array_fragment_metadata_uri = + array_uri.join_path(constants::array_fragment_meta_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragment_metadata_uri)); + + // Create dimension label directory + URI array_dimension_labels_uri = + array_uri.join_path(constants::array_dimension_labels_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_dimension_labels_uri)); + + // Get encryption key from config + Status st; + if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { + bool found = false; + std::string encryption_key_from_cfg = + resources.config().get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + resources.config().get("sm.encryption_type", &found); + assert(found); + auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st_enc); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + throw_if_not_ok( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + } else { + throw_if_not_ok(encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + static_cast(encryption_key_from_cfg.size()))); + } + st = store_array_schema(resources, array_schema, encryption_key_cfg); + } else { + st = store_array_schema(resources, array_schema, encryption_key); + } + + if (!st.ok()) { + throw_if_not_ok(resources.vfs().remove_dir(array_uri)); + return st; + } + + return Status::Ok(); +} + // Used in Consolidator Status Array::open_without_fragments( EncryptionType encryption_type, diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 49d441cf0c0f..08ed27545580 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -370,6 +370,21 @@ class Array { * serialization in pre TileDB 2.4 */ const URI& array_uri_serialized() const; + /** + * Creates a TileDB array, storing its schema. + * + * @param resources The context resources. + * @param array_uri The URI of the array to be created. + * @param array_schema The array schema. + * @param encryption_key The encryption key to use. + * @return Status + */ + static Status create( + ContextResources& resources, + const URI& array_uri, + const shared_ptr& array_schema, + const EncryptionKey& encryption_key); + /** * Opens the array for reading at a timestamp retrieved from the config * or for writing. diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index c0b56da2e643..69dc74761bfa 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -124,11 +124,7 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - Status st = sm->array_create(uri, schema, key); - if (!st.ok()) { - throw std::runtime_error( - "[WhiteboxConsistencyController] Could not create array."); - } + throw_if_not_ok(Array::create(sm->resources(), uri, schema, key)); tdb_unique_ptr array(new Array{uri, sm, *this}); return array; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 5824c5dee92c..f9e28b05b116 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2562,8 +2562,8 @@ int32_t tiledb_array_create( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the array - throw_if_not_ok(ctx->storage_manager()->array_create( - uri, array_schema->array_schema_, key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key)); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2581,8 +2581,11 @@ int32_t tiledb_array_create( } // Create the dimension label array with the same key. - throw_if_not_ok(ctx->storage_manager()->array_create( - dim_label_ref.uri(uri), dim_label_ref.schema(), key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), + dim_label_ref.uri(uri), + dim_label_ref.schema(), + key)); } } return TILEDB_OK; @@ -2630,8 +2633,8 @@ int32_t tiledb_array_create_with_key( key_length)); // Create the array - throw_if_not_ok(ctx->storage_manager()->array_create( - uri, array_schema->array_schema_, key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key)); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2649,8 +2652,11 @@ int32_t tiledb_array_create_with_key( } // Create the dimension label array with the same key. - throw_if_not_ok(ctx->storage_manager()->array_create( - dim_label_ref.uri(uri), dim_label_ref.schema(), key)); + throw_if_not_ok(tiledb::sm::Array::create( + ctx->resources(), + dim_label_ref.uri(uri), + dim_label_ref.schema(), + key)); } } return TILEDB_OK; diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index a0168e896c4b..6246d1c6f758 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB Inc. + * @copyright Copyright (c) 2023-2024 TileDB Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -95,10 +95,7 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - Status st = sm_->array_create(uri, schema, key); - if (!st.ok()) { - throw std::runtime_error("Could not create array."); - } + throw_if_not_ok(Array::create(resources_, uri, schema, key)); tdb_unique_ptr array(new Array{uri, sm_.get()}); return array; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index b7dc00ca8e19..e8a58c3e27c2 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -131,102 +131,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::array_create( - const URI& array_uri, - const shared_ptr& array_schema, - const EncryptionKey& encryption_key) { - // Check array schema - if (array_schema == nullptr) { - throw StorageManagerException("Cannot create array; Empty array schema"); - } - - // Check if array exists - if (is_array(resources_, array_uri)) { - throw StorageManagerException( - "Cannot create array; Array '" + array_uri.to_string() + - "' already exists"); - } - - std::lock_guard lock{object_mtx}; - array_schema->set_array_uri(array_uri); - array_schema->generate_uri(); - array_schema->check(config_); - - // Create array directory - throw_if_not_ok(resources_.vfs().create_dir(array_uri)); - - // Create array schema directory - URI array_schema_dir_uri = - array_uri.join_path(constants::array_schema_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); - - // Create the enumerations directory inside the array schema directory - URI array_enumerations_uri = - array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_enumerations_uri)); - - // Create commit directory - URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_commit_uri)); - - // Create fragments directory - URI array_fragments_uri = - array_uri.join_path(constants::array_fragments_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragments_uri)); - - // Create array metadata directory - URI array_metadata_uri = - array_uri.join_path(constants::array_metadata_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_metadata_uri)); - - // Create fragment metadata directory - URI array_fragment_metadata_uri = - array_uri.join_path(constants::array_fragment_meta_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragment_metadata_uri)); - - // Create dimension label directory - URI array_dimension_labels_uri = - array_uri.join_path(constants::array_dimension_labels_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_dimension_labels_uri)); - - // Get encryption key from config - Status st; - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - config_.get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - config_.get("sm.encryption_type", &found); - assert(found); - auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st_enc); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - RETURN_NOT_OK( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); - } else { - RETURN_NOT_OK(encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - static_cast(encryption_key_from_cfg.size()))); - } - st = store_array_schema(resources_, array_schema, encryption_key_cfg); - } else { - st = store_array_schema(resources_, array_schema, encryption_key); - } - - // Store array schema - if (!st.ok()) { - throw_if_not_ok(resources_.vfs().remove_dir(array_uri)); - return st; - } - - return Status::Ok(); -} - Status StorageManager::array_evolve_schema( const URI& array_uri, ArraySchemaEvolution* schema_evolution, diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 9c70fbf39b5f..f2c7d55127d5 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -135,19 +135,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Creates a TileDB array storing its schema. - * - * @param array_uri The URI of the array to be created. - * @param array_schema The array schema. - * @param encryption_key The encryption key to use. - * @return Status - */ - Status array_create( - const URI& array_uri, - const shared_ptr& array_schema, - const EncryptionKey& encryption_key); - /** * Evolve a TileDB array schema and store its new schema. * diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 61ea4bda2c97..86e4ec1a7246 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2021 TileDB, Inc. + * @copyright Copyright (c) 2021-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -91,7 +91,8 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { tiledb::sm::Array a(tiledb::sm::URI{"mem://junk"}, ctx.storage_manager()); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); - CHECK(ctx.storage_manager()->array_create(a.array_uri(), sp_as, ek).ok()); + CHECK(tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek) + .ok()); CHECK(a.open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION, From 18cfeb16afd2d6baf86fcd20a6035ad1c7982cc3 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:51:36 -0600 Subject: [PATCH 398/456] Correct defective return value in `Posix::ls_with_sizes` (#5037) PR #2671 introduced `ls_with_sizes`. There was a defect in `class Posix` where an empty directory returned `nullopt` instead of an empty vector. This PR corrects that defect. I audited the original PR for possible related errors. This was the only one. [sc-48646] --- TYPE: BUG DESC: Correct defective return value in `Posix::ls_with_sizes` --- tiledb/sm/filesystem/posix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index 58b4f0ab1d5f..4b8011f2510c 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -311,7 +311,7 @@ tuple>> Posix::ls_with_sizes( struct dirent* next_path = nullptr; DIR* dir = opendir(path.c_str()); if (dir == nullptr) { - return {Status::Ok(), nullopt}; + return {Status::Ok(), std::vector{}}; } std::vector entries; From 6300fbff3be04be5a8a0fe25c16147d75f64cc14 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Tue, 4 Jun 2024 00:41:13 -0700 Subject: [PATCH 399/456] Reorganize utility classes. (#5032) In order to keep from cluttering common with some of the various view and other utility classes, moved those classes to common/util subdirectory and the supporting unit test files to common/util/test. Created corresponding CMakeLists.txt in common/util and common/util/test. [sc-48619] --- TYPE: NO_HISTORY DESC: Reorganize new utility files contributed by external sort development into common/util. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- .../tiledb/common/dag/utility/range_join.h | 2 +- .../dag/utility/test/compile_utils_main.cc | 2 +- tiledb/common/CMakeLists.txt | 1 + tiledb/common/test/CMakeLists.txt | 5 -- tiledb/common/util/CMakeLists.txt | 30 +++++++ .../common/{ => util}/alt_var_length_view.h | 0 .../{arrow_proxy.hpp => util/arrow_proxy.h} | 6 +- tiledb/common/{ => util}/iterator_facade.h | 2 +- tiledb/common/{ => util}/permutation_view.h | 0 tiledb/common/util/print_types.h | 52 ++++++++++++ tiledb/common/{ => util}/proxy_sort.h | 0 tiledb/common/util/test/CMakeLists.txt | 81 +++++++++++++++++++ .../test/unit_alt_var_length_view.cc | 0 .../{ => util}/test/unit_iterator_facade.cc | 0 .../{ => util}/test/unit_permutation_sort.cc | 0 .../{ => util}/test/unit_permutation_view.cc | 0 .../common/{ => util}/test/unit_proxy_sort.cc | 0 .../common/{ => util}/test/unit_sort_zip.cc | 0 .../{ => util}/test/unit_var_length_util.cc | 0 .../{ => util}/test/unit_var_length_view.cc | 0 .../common/{ => util}/test/unit_zip_view.cc | 0 tiledb/common/{ => util}/var_length_util.h | 0 tiledb/common/{ => util}/var_length_view.h | 0 tiledb/common/{ => util}/zip_view.h | 0 24 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 tiledb/common/util/CMakeLists.txt rename tiledb/common/{ => util}/alt_var_length_view.h (100%) rename tiledb/common/{arrow_proxy.hpp => util/arrow_proxy.h} (94%) rename tiledb/common/{ => util}/iterator_facade.h (99%) rename tiledb/common/{ => util}/permutation_view.h (100%) create mode 100644 tiledb/common/util/print_types.h rename tiledb/common/{ => util}/proxy_sort.h (100%) create mode 100644 tiledb/common/util/test/CMakeLists.txt rename tiledb/common/{ => util}/test/unit_alt_var_length_view.cc (100%) rename tiledb/common/{ => util}/test/unit_iterator_facade.cc (100%) rename tiledb/common/{ => util}/test/unit_permutation_sort.cc (100%) rename tiledb/common/{ => util}/test/unit_permutation_view.cc (100%) rename tiledb/common/{ => util}/test/unit_proxy_sort.cc (100%) rename tiledb/common/{ => util}/test/unit_sort_zip.cc (100%) rename tiledb/common/{ => util}/test/unit_var_length_util.cc (100%) rename tiledb/common/{ => util}/test/unit_var_length_view.cc (100%) rename tiledb/common/{ => util}/test/unit_zip_view.cc (100%) rename tiledb/common/{ => util}/var_length_util.h (100%) rename tiledb/common/{ => util}/var_length_view.h (100%) rename tiledb/common/{ => util}/zip_view.h (100%) diff --git a/experimental/tiledb/common/dag/utility/range_join.h b/experimental/tiledb/common/dag/utility/range_join.h index db9a660f2e83..5c0411d76fc2 100644 --- a/experimental/tiledb/common/dag/utility/range_join.h +++ b/experimental/tiledb/common/dag/utility/range_join.h @@ -73,7 +73,7 @@ #include #include #include -#include "tiledb/common/arrow_proxy.hpp" +#include "tiledb/common/util/arrow_proxy.h" #include "external/include/span/span.hpp" diff --git a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc index b158e690c650..3b57acf679d5 100644 --- a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc +++ b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc @@ -31,7 +31,7 @@ #include "../range_join.h" #include "../spinlock.h" #include "../traits.h" -#include "tiledb/common/arrow_proxy.hpp" +#include "tiledb/common/util/arrow_proxy.h" int main() { } diff --git a/tiledb/common/CMakeLists.txt b/tiledb/common/CMakeLists.txt index 3c8fbfb987b5..9cb5b1952ceb 100644 --- a/tiledb/common/CMakeLists.txt +++ b/tiledb/common/CMakeLists.txt @@ -46,6 +46,7 @@ add_subdirectory(interval) add_subdirectory(random) add_subdirectory(thread_pool) add_subdirectory(types) +add_subdirectory(util) get_gathered(COMMON_SOURCES) set(TILEDB_COMMON_SOURCES ${COMMON_SOURCES} PARENT_SCOPE) diff --git a/tiledb/common/test/CMakeLists.txt b/tiledb/common/test/CMakeLists.txt index 0bc544615a1e..034cd452920d 100644 --- a/tiledb/common/test/CMakeLists.txt +++ b/tiledb/common/test/CMakeLists.txt @@ -41,8 +41,3 @@ commence(unit_test memory_tracker) this_target_object_libraries(baseline) conclude(unit_test) -commence(unit_test common_utils) - this_target_sources(main.cc unit_alt_var_length_view.cc unit_iterator_facade.cc unit_permutation_sort.cc unit_permutation_view.cc unit_proxy_sort.cc unit_sort_zip.cc unit_var_length_util.cc unit_var_length_view.cc unit_zip_view.cc) - this_target_object_libraries(baseline) -conclude(unit_test) - diff --git a/tiledb/common/util/CMakeLists.txt b/tiledb/common/util/CMakeLists.txt new file mode 100644 index 000000000000..15d75d4b447e --- /dev/null +++ b/tiledb/common/util/CMakeLists.txt @@ -0,0 +1,30 @@ +# +# tiledb/common/util/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +add_test_subdirectory() diff --git a/tiledb/common/alt_var_length_view.h b/tiledb/common/util/alt_var_length_view.h similarity index 100% rename from tiledb/common/alt_var_length_view.h rename to tiledb/common/util/alt_var_length_view.h diff --git a/tiledb/common/arrow_proxy.hpp b/tiledb/common/util/arrow_proxy.h similarity index 94% rename from tiledb/common/arrow_proxy.hpp rename to tiledb/common/util/arrow_proxy.h index 5a8cc0c23e19..1591b2555b9d 100644 --- a/tiledb/common/arrow_proxy.hpp +++ b/tiledb/common/util/arrow_proxy.h @@ -41,8 +41,8 @@ * */ -#ifndef TILEDB_ARROW_PROXY_HPP -#define TILEDB_ARROW_PROXY_HPP +#ifndef TILEDB_ARROW_PROXY_H +#define TILEDB_ARROW_PROXY_H template struct arrow_proxy { @@ -55,4 +55,4 @@ struct arrow_proxy { template arrow_proxy(T&&) -> arrow_proxy; -#endif // TILEDB_ARROW_PROXY_HPP +#endif // TILEDB_ARROW_PROXY_H diff --git a/tiledb/common/iterator_facade.h b/tiledb/common/util/iterator_facade.h similarity index 99% rename from tiledb/common/iterator_facade.h rename to tiledb/common/util/iterator_facade.h index be97d52e4417..5131b7f3b5f1 100644 --- a/tiledb/common/iterator_facade.h +++ b/tiledb/common/util/iterator_facade.h @@ -42,7 +42,7 @@ #include #include -#include "arrow_proxy.hpp" +#include "arrow_proxy.h" namespace detail { diff --git a/tiledb/common/permutation_view.h b/tiledb/common/util/permutation_view.h similarity index 100% rename from tiledb/common/permutation_view.h rename to tiledb/common/util/permutation_view.h diff --git a/tiledb/common/util/print_types.h b/tiledb/common/util/print_types.h new file mode 100644 index 000000000000..e7d2161800e6 --- /dev/null +++ b/tiledb/common/util/print_types.h @@ -0,0 +1,52 @@ +/** + * @file print_types.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements a compile-time debugging utility for investigating + * the specific types of objects. + * + * Based on utility from NWGraph. Author Luke D'Alessandro. + */ + +#ifndef TILEDB_PRINT_TYPES_H +#define TILEDB_PRINT_TYPES_H + +template +struct print_types_t; + +/* + * Print (as compiler error message), the types of the + * variadic argument list. E.g., + * print_types(foo, bar, baz); + */ +template +constexpr auto print_types(Ts...) { + return print_types_t{}; +} + +#endif // TILEDB_PRINT_TYPES_H diff --git a/tiledb/common/proxy_sort.h b/tiledb/common/util/proxy_sort.h similarity index 100% rename from tiledb/common/proxy_sort.h rename to tiledb/common/util/proxy_sort.h diff --git a/tiledb/common/util/test/CMakeLists.txt b/tiledb/common/util/test/CMakeLists.txt new file mode 100644 index 000000000000..471ad68ac893 --- /dev/null +++ b/tiledb/common/util/test/CMakeLists.txt @@ -0,0 +1,81 @@ +# +# tiledb/common/util/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test unit_alt_var_length_view) +this_target_sources( + unit_alt_var_length_view.cc +) +conclude(unit_test) + +commence(unit_test unit_iterator_facade) +this_target_sources( + unit_iterator_facade.cc +) +conclude(unit_test) + +commence(unit_test unit_permutation_sort) +this_target_sources( + unit_permutation_sort.cc +) +conclude(unit_test) + +commence(unit_test unit_permutation_view) +this_target_sources( + unit_permutation_view.cc +) +conclude(unit_test) + +commence(unit_test unit_proxy_sort) +this_target_sources( + unit_proxy_sort.cc +) +conclude(unit_test) + +commence(unit_test unit_sort_zip) +this_target_sources( + unit_sort_zip.cc +) +conclude(unit_test) + +commence(unit_test unit_var_length_util) +this_target_sources( + unit_var_length_util.cc +) +conclude(unit_test) + +commence(unit_test unit_var_length_view) +this_target_sources( + unit_var_length_view.cc +) +conclude(unit_test) + +commence(unit_test unit_zip_view) +this_target_sources( + unit_zip_view.cc +) +conclude(unit_test) diff --git a/tiledb/common/test/unit_alt_var_length_view.cc b/tiledb/common/util/test/unit_alt_var_length_view.cc similarity index 100% rename from tiledb/common/test/unit_alt_var_length_view.cc rename to tiledb/common/util/test/unit_alt_var_length_view.cc diff --git a/tiledb/common/test/unit_iterator_facade.cc b/tiledb/common/util/test/unit_iterator_facade.cc similarity index 100% rename from tiledb/common/test/unit_iterator_facade.cc rename to tiledb/common/util/test/unit_iterator_facade.cc diff --git a/tiledb/common/test/unit_permutation_sort.cc b/tiledb/common/util/test/unit_permutation_sort.cc similarity index 100% rename from tiledb/common/test/unit_permutation_sort.cc rename to tiledb/common/util/test/unit_permutation_sort.cc diff --git a/tiledb/common/test/unit_permutation_view.cc b/tiledb/common/util/test/unit_permutation_view.cc similarity index 100% rename from tiledb/common/test/unit_permutation_view.cc rename to tiledb/common/util/test/unit_permutation_view.cc diff --git a/tiledb/common/test/unit_proxy_sort.cc b/tiledb/common/util/test/unit_proxy_sort.cc similarity index 100% rename from tiledb/common/test/unit_proxy_sort.cc rename to tiledb/common/util/test/unit_proxy_sort.cc diff --git a/tiledb/common/test/unit_sort_zip.cc b/tiledb/common/util/test/unit_sort_zip.cc similarity index 100% rename from tiledb/common/test/unit_sort_zip.cc rename to tiledb/common/util/test/unit_sort_zip.cc diff --git a/tiledb/common/test/unit_var_length_util.cc b/tiledb/common/util/test/unit_var_length_util.cc similarity index 100% rename from tiledb/common/test/unit_var_length_util.cc rename to tiledb/common/util/test/unit_var_length_util.cc diff --git a/tiledb/common/test/unit_var_length_view.cc b/tiledb/common/util/test/unit_var_length_view.cc similarity index 100% rename from tiledb/common/test/unit_var_length_view.cc rename to tiledb/common/util/test/unit_var_length_view.cc diff --git a/tiledb/common/test/unit_zip_view.cc b/tiledb/common/util/test/unit_zip_view.cc similarity index 100% rename from tiledb/common/test/unit_zip_view.cc rename to tiledb/common/util/test/unit_zip_view.cc diff --git a/tiledb/common/var_length_util.h b/tiledb/common/util/var_length_util.h similarity index 100% rename from tiledb/common/var_length_util.h rename to tiledb/common/util/var_length_util.h diff --git a/tiledb/common/var_length_view.h b/tiledb/common/util/var_length_view.h similarity index 100% rename from tiledb/common/var_length_view.h rename to tiledb/common/util/var_length_view.h diff --git a/tiledb/common/zip_view.h b/tiledb/common/util/zip_view.h similarity index 100% rename from tiledb/common/zip_view.h rename to tiledb/common/util/zip_view.h From 0130764b4ba0fe68b2c4a6c637b4f7e3ac65bfeb Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:02:16 +0200 Subject: [PATCH 400/456] Revert "Enable array open v2 and query v3 by default." (#5042) Reverts TileDB-Inc/TileDB#4974. --- TYPE: NO_HISTORY DESC: Revert "Enable array open v2 and query v3 by default." --- test/src/unit-capi-config.cc | 12 ++++++------ tiledb/api/c_api/config/config_api_external.h | 10 +++++----- tiledb/sm/config/config.cc | 4 ++-- tiledb/sm/cpp_api/config.h | 10 +++++----- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 34dca3c2fd36..ca4269d88a8b 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -239,8 +239,8 @@ void check_save_to_file() { ss << "rest.retry_initial_delay_ms 500\n"; ss << "rest.server_address https://api.tiledb.com\n"; ss << "rest.server_serialization_format CAPNP\n"; - ss << "rest.use_refactored_array_open true\n"; - ss << "rest.use_refactored_array_open_and_query_submit true\n"; + ss << "rest.use_refactored_array_open false\n"; + ss << "rest.use_refactored_array_open_and_query_submit false\n"; ss << "sm.allow_separate_attribute_writes false\n"; ss << "sm.allow_updates_experimental false\n"; ss << "sm.check_coord_dups true\n"; @@ -578,13 +578,13 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { CHECK(rc == TILEDB_OK); CHECK(error == nullptr); rc = tiledb_config_set( - config, "rest.use_refactored_array_open", "false", &error); + config, "rest.use_refactored_array_open", "true", &error); CHECK(rc == TILEDB_OK); CHECK(error == nullptr); rc = tiledb_config_set( config, "rest.use_refactored_array_open_and_query_submit", - "false", + "true", &error); CHECK(rc == TILEDB_OK); CHECK(error == nullptr); @@ -612,8 +612,8 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["rest.load_metadata_on_array_open"] = "false"; all_param_values["rest.load_non_empty_domain_on_array_open"] = "false"; all_param_values["rest.load_enumerations_on_array_open"] = "false"; - all_param_values["rest.use_refactored_array_open"] = "false"; - all_param_values["rest.use_refactored_array_open_and_query_submit"] = "false"; + all_param_values["rest.use_refactored_array_open"] = "true"; + all_param_values["rest.use_refactored_array_open_and_query_submit"] = "true"; all_param_values["sm.allow_separate_attribute_writes"] = "false"; all_param_values["sm.allow_updates_experimental"] = "false"; all_param_values["sm.encryption_key"] = ""; diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 0d1ef5168234..5e0e8c6b3121 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -680,13 +680,13 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * with the open array
* **Default**: true * - `rest.use_refactored_array_open`
- * If true, the new REST routes and APIs for opening an array + * If true, the new, experimental REST routes and APIs for opening an array * will be used
- * **Default**: true + * **Default**: false * - `rest.use_refactored_array_open_and_query_submit`
- * If true, the new REST routes and APIs for opening an array and submitting - * a query will be used
- * **Default**: true + * If true, the new, experimental REST routes and APIs for opening an array + * and submitting a query will be used
+ * **Default**: false * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
* **Default**: 524288 (512KB) diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 72378c06e39d..69ce8565a852 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -94,8 +94,8 @@ const std::string Config::REST_CURL_VERBOSE = "false"; const std::string Config::REST_LOAD_ENUMERATIONS_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; -const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "true"; -const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "true"; +const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false"; +const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "false"; const std::string Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES = "false"; const std::string Config::SM_ALLOW_UPDATES_EXPERIMENTAL = "false"; const std::string Config::SM_ENCRYPTION_KEY = ""; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index c9598aedb5c8..02d8105bc597 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -857,14 +857,14 @@ class Config { * **Default**: true * - `rest.use_refactored_array_open`
* **Experimental**
- * If true, the new REST routes and APIs for opening an array + * If true, the new, experimental REST routes and APIs for opening an array * will be used
- * **Default**: true + * **Default**: false * - `rest.use_refactored_array_open_and_query_submit`
* **Experimental**
- * If true, the new REST routes and APIs for opening an array and - * submitting a query will be used
- * **Default**: true + * If true, the new, experimental REST routes and APIs for opening an array + * and submitting a query will be used
+ * **Default**: false * - `rest.curl.buffer_size`
* Set curl buffer size for REST requests
* **Default**: 524288 (512KB) From c303e3c55c07799ee3f22816d36de0118cacf347 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 5 Jun 2024 14:04:19 +0300 Subject: [PATCH 401/456] Complete migration of deprecated APIs in tiledb_unit. (#5016) [SC-48173](https://app.shortcut.com/tiledb-inc/story/48173/finish-work-to-build-tiledb-unit-with-no-deprecations) This PR finishes migration of deprecated APIs in `tiledb_unit`. One use in `unit-cppapi-array.cc` was trivial, and the other uses were in `unit-capi-serialized_queries.cc`. They were migrated as well, and the file's twin `unit-capi-serialized_queries_using_subarray.cc` was deleted. --- TYPE: NO_HISTORY --- test/CMakeLists.txt | 1 - test/src/unit-capi-serialized_queries.cc | 95 ++- ...-capi-serialized_queries_using_subarray.cc | 749 ------------------ test/src/unit-cppapi-array.cc | 3 +- 4 files changed, 71 insertions(+), 777 deletions(-) delete mode 100644 test/src/unit-capi-serialized_queries_using_subarray.cc diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 816ce9afb8cd..f66e8c7c0455 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -252,7 +252,6 @@ endif() if (TILEDB_SERIALIZATION) list(APPEND TILEDB_UNIT_TEST_SOURCES src/unit-capi-serialized_queries.cc - src/unit-capi-serialized_queries_using_subarray.cc src/unit-QueryCondition-serialization.cc src/unit-curl.cc ) diff --git a/test/src/unit-capi-serialized_queries.cc b/test/src/unit-capi-serialized_queries.cc index 82790a7f75bf..e7413d6a5f11 100644 --- a/test/src/unit-capi-serialized_queries.cc +++ b/test/src/unit-capi-serialized_queries.cc @@ -118,6 +118,22 @@ struct SerializationFx { REQUIRE(loop_num->second > 0); } + static void check_subarray_stats(int dim0_expected, int dim1_expected) { + Stats::enable(); + std::string stats; + Stats::dump(&stats); + // Note: if these checks fail, use Stats::dump(stdout) to validate counters + CHECK( + stats.find( + "\"Context.StorageManager.subSubarray.add_range_dim_0\": " + + std::to_string(dim0_expected)) != std::string::npos); + CHECK( + stats.find( + "\"Context.StorageManager.subSubarray.add_range_dim_1\": " + + std::to_string(dim1_expected)) != std::string::npos); + Stats::disable(); + } + static void check_delete_stats(const Query& query) { auto stats = ((sm::Reader*)query.ptr()->query_->strategy())->stats(); REQUIRE(stats != nullptr); @@ -177,7 +193,9 @@ struct SerializationFx { Array array(ctx, array_uri, TILEDB_WRITE); Query query(ctx, array); - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -218,10 +236,10 @@ struct SerializationFx { Array array(ctx, array_uri, TILEDB_WRITE); Query query(ctx, array); - Subarray sub(ctx, array); - sub.add_range(0, subarray[0], subarray[1]); - sub.add_range(1, subarray[2], subarray[3]); - query.set_subarray(sub); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.add_range(0, subarray[0], subarray[1]); + cppapi_subarray.add_range(1, subarray[2], subarray[3]); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -356,7 +374,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -365,6 +385,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -393,7 +414,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -407,6 +430,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -456,7 +480,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {3, 4, 3, 4}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -465,6 +491,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -500,7 +527,9 @@ TEST_CASE_METHOD( std::vector a3_data(60); std::vector a3_offsets(4); std::vector subarray = {3, 4, 3, 4}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); auto set_buffers = [&](Query& q) { q.set_data_buffer("a1", a1); @@ -575,7 +604,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -625,7 +656,9 @@ TEST_CASE_METHOD( std::vector subarray = {1, 10, 1, 10}; query.set_layout(TILEDB_GLOBAL_ORDER); - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx_client, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -634,6 +667,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -667,7 +701,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_coordinates(coords); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); @@ -677,6 +713,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -699,6 +736,9 @@ TEST_CASE_METHOD( "[query][dense][serialization][rest]") { create_array(TILEDB_DENSE); write_dense_array_ranges(); + if (!vfs_test_setup_.is_rest()) { + check_subarray_stats(1, 1); + } SECTION("- Read all") { Array array(ctx, array_uri, TILEDB_READ); @@ -710,10 +750,10 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - Subarray sub(ctx, array); - sub.add_range(0, subarray[0], subarray[1]); - sub.add_range(1, subarray[2], subarray[3]); - query.set_subarray(sub); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.add_range(0, subarray[0], subarray[1]); + cppapi_subarray.add_range(1, subarray[2], subarray[3]); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -722,6 +762,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -746,10 +787,10 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {3, 4, 3, 4}; - Subarray sub(ctx, array); - sub.add_range(0, subarray[0], subarray[1]); - sub.add_range(1, subarray[2], subarray[3]); - query.set_subarray(sub); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.add_range(0, subarray[0], subarray[1]); + cppapi_subarray.add_range(1, subarray[2], subarray[3]); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); @@ -758,6 +799,7 @@ TEST_CASE_METHOD( // Submit query query.submit(); + REQUIRE(query.query_status() == Query::Status::COMPLETE); // The deserialized query should also include the read stats check_read_stats(query); @@ -781,10 +823,10 @@ TEST_CASE_METHOD( std::vector a3_data(60); std::vector a3_offsets(4); std::vector subarray = {3, 4, 3, 4}; - Subarray sub(ctx, array); - sub.add_range(0, subarray[0], subarray[1]); - sub.add_range(1, subarray[2], subarray[3]); - query.set_subarray(sub); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.add_range(0, subarray[0], subarray[1]); + cppapi_subarray.add_range(1, subarray[2], subarray[3]); + query.set_subarray(cppapi_subarray); auto set_buffers = [&](Query& q) { q.set_data_buffer("a1", a1); @@ -796,7 +838,6 @@ TEST_CASE_METHOD( // Submit initial query. set_buffers(query); - // Submit query query.submit(); // The deserialized query should also include the read stats check_read_stats(query); @@ -861,7 +902,9 @@ TEST_CASE_METHOD( std::vector a3_offsets(1000); std::vector subarray = {1, 10, 1, 10}; - query.set_subarray(subarray); + Subarray cppapi_subarray(ctx, array); + cppapi_subarray.set_subarray(subarray); + query.set_subarray(cppapi_subarray); query.set_data_buffer("a1", a1); query.set_data_buffer("a2", a2); query.set_validity_buffer("a2", a2_nullable); diff --git a/test/src/unit-capi-serialized_queries_using_subarray.cc b/test/src/unit-capi-serialized_queries_using_subarray.cc deleted file mode 100644 index cda100807327..000000000000 --- a/test/src/unit-capi-serialized_queries_using_subarray.cc +++ /dev/null @@ -1,749 +0,0 @@ -/** - * @file unit-capi-serialized_queries_using_subarray.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2017-2023 TileDB Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * Tests for query serialization/deserialization using separate subarray. - * - * @note This module tests the capi by using the cppapi entities which are built - * on the capi functionality. - */ - -#include -#include "test/support/src/helpers.h" -#include "test/support/src/vfs_helpers.h" -#include "tiledb/sm/c_api/tiledb_serialization.h" -#include "tiledb/sm/c_api/tiledb_struct_def.h" -#include "tiledb/sm/cpp_api/tiledb" -#include "tiledb/sm/query/legacy/reader.h" -#include "tiledb/sm/query/writers/writer_base.h" -#include "tiledb/sm/serialization/query.h" - -#ifdef _WIN32 -#include "tiledb/sm/filesystem/win.h" -#else -#include "tiledb/sm/filesystem/posix.h" -#endif - -#include -#include -#include - -using namespace tiledb; -using ResultSetType = std::map; - -namespace { - -template -bool check_result(const T a, const T b, size_t start, size_t end) { - auto a_exp = T(a.begin() + start, a.begin() + end); - auto b_exp = T(b.begin() + start, b.begin() + end); - return a_exp == b_exp; -} - -template -bool check_result( - const TResult a, - const TExpected b, - std::optional start = nullopt, - std::optional end = nullopt) { - TResult b_typed; - if constexpr (std::is_same::value) { - b_typed = std::any_cast(b); - } else { - b_typed = b; - } - if (start.has_value()) { - assert(end.has_value()); - return check_result(a, b_typed, *start, *end); - } else { - return check_result(a, b_typed, 0, b_typed.size()); - } -} - -struct SerializationFx { - test::VFSTestSetup vfs_test_setup_; - tiledb_ctx_t* ctx_; - Context ctx; - const std::string array_uri; - - SerializationFx() - : ctx_{vfs_test_setup_.ctx_c} - , ctx{vfs_test_setup_.ctx()} - , array_uri{vfs_test_setup_.array_uri("testarray")} { - } - - static void check_read_stats(const Query& query) { - auto stats = ((sm::WriterBase*)query.ptr()->query_->strategy())->stats(); - REQUIRE(stats != nullptr); - auto counters = stats->counters(); - REQUIRE(counters != nullptr); - auto loop_num = - counters->find("Context.StorageManager.Query.Reader.loop_num"); - REQUIRE((loop_num != counters->end())); - REQUIRE(loop_num->second > 0); - } - - static void check_write_stats(const Query& query) { - auto stats = ((sm::Reader*)query.ptr()->query_->strategy())->stats(); - REQUIRE(stats != nullptr); - auto counters = stats->counters(); - REQUIRE(counters != nullptr); - auto loop_num = - counters->find("Context.StorageManager.Query.Writer.attr_num"); - REQUIRE((loop_num != counters->end())); - REQUIRE(loop_num->second > 0); - } - - static void check_subarray_stats(int dim0_expected, int dim1_expected) { - Stats::enable(); - std::string stats; - Stats::dump(&stats); - // Note: if these checks fail, use Stats::dump(stdout) to validate counters - CHECK( - stats.find( - "\"Context.StorageManager.subSubarray.add_range_dim_0\": " + - std::to_string(dim0_expected)) != std::string::npos); - CHECK( - stats.find( - "\"Context.StorageManager.subSubarray.add_range_dim_1\": " + - std::to_string(dim1_expected)) != std::string::npos); - Stats::disable(); - } - - void create_array(tiledb_array_type_t type) { - ArraySchema schema(ctx, type); - Domain domain(ctx); - domain.add_dimension(Dimension::create(ctx, "d1", {1, 10}, 2)) - .add_dimension(Dimension::create(ctx, "d2", {1, 10}, 2)); - schema.set_domain(domain); - - schema.add_attribute(Attribute::create(ctx, "a1")); - schema.add_attribute( - Attribute::create>(ctx, "a2").set_nullable( - true)); - schema.add_attribute(Attribute::create>(ctx, "a3")); - - Array::create(array_uri, schema); - } - - ResultSetType write_dense_array() { - std::vector subarray = {1, 10, 1, 10}; - std::vector a1; - std::vector a2; - std::vector a2_nullable; - std::vector a3_data; - std::vector a3_offsets; - - const unsigned ncells = - (subarray[1] - subarray[0] + 1) * (subarray[3] - subarray[2] + 1); - for (unsigned i = 0; i < ncells; i++) { - a1.push_back(i); - a2.push_back(i); - a2.push_back(2 * i); - a2_nullable.push_back(a2.back() % 5 == 0 ? 0 : 1); - - std::string a3 = "a"; - for (unsigned j = 0; j < i; j++) - a3.push_back('a'); - a3_offsets.push_back(a3_data.size()); - a3_data.insert(a3_data.end(), a3.begin(), a3.end()); - } - - ResultSetType results; - results["a1"] = a1; - results["a2"] = a2; - results["a2_nullable"] = a2_nullable; - results["a3_data"] = a3_data; - results["a3_offsets"] = a3_offsets; - - Array array(ctx, array_uri, TILEDB_WRITE); - Query query(ctx, array); - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - - // The deserialized query should also include the write stats - check_write_stats(query); - - return results; - } - - void write_dense_array_ranges() { - std::vector subarray = {1, 10, 1, 10}; - std::vector a1; - std::vector a2; - std::vector a2_nullable; - std::vector a3_data; - std::vector a3_offsets; - - const unsigned ncells = - (subarray[1] - subarray[0] + 1) * (subarray[3] - subarray[2] + 1); - for (unsigned i = 0; i < ncells; i++) { - a1.push_back(i); - a2.push_back(i); - a2.push_back(2 * i); - a2_nullable.push_back(a2.back() % 5 == 0 ? 0 : 1); - - std::string a3 = "a"; - for (unsigned j = 0; j < i; j++) - a3.push_back('a'); - a3_offsets.push_back(a3_data.size()); - a3_data.insert(a3_data.end(), a3.begin(), a3.end()); - } - - Array array(ctx, array_uri, TILEDB_WRITE); - Query query(ctx, array); - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.add_range(0, subarray[0], subarray[1]); - cppapi_subarray.add_range(1, subarray[2], subarray[3]); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - } - - void write_sparse_array() { - std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - std::vector d2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - std::vector a1; - std::vector a2; - std::vector a2_nullable; - std::vector a3_data; - std::vector a3_offsets; - - const unsigned ncells = 10; - for (unsigned i = 0; i < ncells; i++) { - a1.push_back(i); - a2.push_back(i); - a2.push_back(2 * i); - a2_nullable.push_back(a2.back() % 5 == 0 ? 0 : 1); - - std::string a3 = "a"; - for (unsigned j = 0; j < i; j++) - a3.push_back('a'); - a3_offsets.push_back(a3_data.size()); - a3_data.insert(a3_data.end(), a3.begin(), a3.end()); - } - - Array array(ctx, array_uri, TILEDB_WRITE); - Query query(ctx, array); - query.set_layout(TILEDB_UNORDERED); - query.set_data_buffer("d1", d1); - query.set_data_buffer("d2", d2); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - } - - void write_sparse_array_split_coords() { - std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - - std::vector d2 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - - std::vector a1; - std::vector a2; - std::vector a2_nullable; - std::vector a3_data; - std::vector a3_offsets; - - const unsigned ncells = 10; - for (unsigned i = 0; i < ncells; i++) { - a1.push_back(i); - a2.push_back(i); - a2.push_back(2 * i); - a2_nullable.push_back(a2.back() % 5 == 0 ? 0 : 1); - - std::string a3 = "a"; - for (unsigned j = 0; j < i; j++) - a3.push_back('a'); - a3_offsets.push_back(a3_data.size()); - a3_data.insert(a3_data.end(), a3.begin(), a3.end()); - } - - Array array(ctx, array_uri, TILEDB_WRITE); - Query query(ctx, array); - query.set_layout(TILEDB_UNORDERED); - query.set_data_buffer("d1", d1); - query.set_data_buffer("d2", d2); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - } -}; - -} // namespace - -TEST_CASE_METHOD( - SerializationFx, - "subarray - Query serialization, dense", - "[query][dense][serialization][rest][sc-40489]") { - if (!vfs_test_setup_.is_rest()) { - SUCCEED("sc-40489: this test passes on remote arrays only"); - return; - } - - create_array(TILEDB_DENSE); - auto expected_results = write_dense_array(); - check_subarray_stats(2, 2); - - SECTION("- Read all") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(500); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {1, 10, 1, 10}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Check stats before serialization - check_subarray_stats(3, 3); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - // Check stats after serialization - // #TODO Revisit after Stats serialization is reworked - check_subarray_stats(5, 5); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 100); - REQUIRE(std::get<1>(result_el["a2"]) == 200); - REQUIRE(std::get<2>(result_el["a2"]) == 100); - REQUIRE(std::get<0>(result_el["a3"]) == 100); - REQUIRE(std::get<1>(result_el["a3"]) == 5050); - } - - SECTION("- Read all, with condition") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(500); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {1, 10, 1, 10}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - uint32_t cmp_value = 5; - QueryCondition condition(ctx); - condition.init("a1", &cmp_value, sizeof(uint32_t), TILEDB_LT); - query.set_condition(condition); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - // The deserialized query should also include the write stats - check_read_stats(query); - check_subarray_stats(5, 5); - - // We expect all cells where `a1` >= `cmp_value` to be filtered - // out. For the refactored reader, filtered out means the value is - // replaced with the fill value. - auto result_el = query.result_buffer_elements_nullable(); - if (test::use_refactored_dense_reader()) { - REQUIRE(std::get<1>(result_el["a1"]) == 100); - REQUIRE(std::get<1>(result_el["a2"]) == 200); - REQUIRE(std::get<2>(result_el["a2"]) == 100); - REQUIRE(std::get<0>(result_el["a3"]) == 100); - REQUIRE(std::get<1>(result_el["a3"]) == 110); - - auto null_val = std::numeric_limits::max(); - for (uint64_t i = 5; i < 100; i++) { - REQUIRE(a1[i] == null_val); - REQUIRE(a2[i * 2] == null_val); - REQUIRE(a2[i * 2 + 1] == null_val); - REQUIRE(a2_nullable[i] == 0); - REQUIRE(a3_offsets[i] == 10 + i); - REQUIRE(a3_data[10 + i] == 0); - } - } else { - REQUIRE(std::get<1>(result_el["a1"]) == 5); - REQUIRE(std::get<1>(result_el["a2"]) == 10); - REQUIRE(std::get<2>(result_el["a2"]) == 5); - REQUIRE(std::get<0>(result_el["a3"]) == 5); - REQUIRE(std::get<1>(result_el["a3"]) == 15); - } - - REQUIRE(check_result(a1, expected_results["a1"], 0, 5)); - REQUIRE(check_result(a2, expected_results["a2"], 0, 10)); - REQUIRE(check_result(a2_nullable, expected_results["a2_nullable"], 0, 5)); - REQUIRE(check_result(a3_data, expected_results["a3_data"], 0, 15)); - REQUIRE(check_result(a3_offsets, expected_results["a3_offsets"], 0, 5)); - } - - SECTION("- Read subarray") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(500); - std::vector a2_nullable(1000); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {3, 4, 3, 4}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - check_subarray_stats(5, 5); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 4); - REQUIRE(std::get<1>(result_el["a2"]) == 8); - REQUIRE(std::get<2>(result_el["a2"]) == 4); - REQUIRE(std::get<0>(result_el["a3"]) == 4); - REQUIRE(std::get<1>(result_el["a3"]) == 114); - } - - SECTION("- Incomplete read") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(4); - std::vector a2(4); - std::vector a2_nullable(4); - std::vector a3_data(60); - std::vector a3_offsets(4); - std::vector subarray = {3, 4, 3, 4}; - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - - auto set_buffers = [&](Query& q) { - q.set_data_buffer("a1", a1); - q.set_data_buffer("a2", a2); - q.set_validity_buffer("a2", a2_nullable); - q.set_data_buffer("a3", a3_data); - q.set_offsets_buffer("a3", a3_offsets); - }; - - // Submit initial query. - set_buffers(query); - query.submit(); - check_subarray_stats(5, 5); - - REQUIRE(query.query_status() == Query::Status::INCOMPLETE); - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 2); - REQUIRE(std::get<1>(result_el["a2"]) == 4); - REQUIRE(std::get<2>(result_el["a2"]) == 2); - REQUIRE(std::get<0>(result_el["a3"]) == 2); - REQUIRE(std::get<1>(result_el["a3"]) == 47); - - // Reset buffers, serialize and resubmit - set_buffers(query); - query.submit(); - check_subarray_stats(7, 7); - - REQUIRE(query.query_status() == Query::Status::INCOMPLETE); - result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 1); - REQUIRE(std::get<1>(result_el["a2"]) == 2); - REQUIRE(std::get<2>(result_el["a2"]) == 1); - REQUIRE(std::get<0>(result_el["a3"]) == 1); - REQUIRE(std::get<1>(result_el["a3"]) == 33); - - // Reset buffers, serialize and resubmit - set_buffers(query); - query.submit(); - check_subarray_stats(9, 9); - - REQUIRE(query.query_status() == Query::Status::COMPLETE); - result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 1); - REQUIRE(std::get<1>(result_el["a2"]) == 2); - REQUIRE(std::get<2>(result_el["a2"]) == 1); - REQUIRE(std::get<0>(result_el["a3"]) == 1); - REQUIRE(std::get<1>(result_el["a3"]) == 34); - } -} - -TEST_CASE_METHOD( - SerializationFx, - "subarray - Query serialization, sparse", - "[query][sparse][serialization][rest]") { - create_array(TILEDB_SPARSE); - write_sparse_array(); - - SECTION("- Read all") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(1000); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {1, 10, 1, 10}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 10); - REQUIRE(std::get<1>(result_el["a2"]) == 20); - REQUIRE(std::get<2>(result_el["a2"]) == 10); - REQUIRE(std::get<0>(result_el["a3"]) == 10); - REQUIRE(std::get<1>(result_el["a3"]) == 55); - } -} - -TEST_CASE_METHOD( - SerializationFx, - "subarray - Query serialization, split coords, sparse", - "[query][sparse][serialization][split-coords]") { - create_array(TILEDB_SPARSE); - write_sparse_array_split_coords(); - - SECTION("- Read all") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector coords(1000); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(1000); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {1, 10, 1, 10}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.set_subarray(subarray); - query.set_subarray(cppapi_subarray); - query.set_coordinates(coords); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el[tiledb::test::TILEDB_COORDS]) == 20); - REQUIRE(std::get<1>(result_el["a1"]) == 10); - REQUIRE(std::get<1>(result_el["a2"]) == 20); - REQUIRE(std::get<2>(result_el["a2"]) == 10); - REQUIRE(std::get<0>(result_el["a3"]) == 10); - REQUIRE(std::get<1>(result_el["a3"]) == 55); - } -} - -TEST_CASE_METHOD( - SerializationFx, - "subarray - Query serialization, dense ranges", - "[query][dense][serialization][rest]") { - create_array(TILEDB_DENSE); - write_dense_array_ranges(); - - SECTION("- Read all") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(1000); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {1, 10, 1, 10}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.add_range(0, subarray[0], subarray[1]); - cppapi_subarray.add_range(1, subarray[2], subarray[3]); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 100); - REQUIRE(std::get<1>(result_el["a2"]) == 200); - REQUIRE(std::get<2>(result_el["a2"]) == 100); - REQUIRE(std::get<0>(result_el["a3"]) == 100); - REQUIRE(std::get<1>(result_el["a3"]) == 5050); - } - - SECTION("- Read subarray") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(1000); - std::vector a2(1000); - std::vector a2_nullable(1000); - std::vector a3_data(1000 * 100); - std::vector a3_offsets(1000); - std::vector subarray = {3, 4, 3, 4}; - - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.add_range(0, subarray[0], subarray[1]); - cppapi_subarray.add_range(1, subarray[2], subarray[3]); - query.set_subarray(cppapi_subarray); - query.set_data_buffer("a1", a1); - query.set_data_buffer("a2", a2); - query.set_validity_buffer("a2", a2_nullable); - query.set_data_buffer("a3", a3_data); - query.set_offsets_buffer("a3", a3_offsets); - - // Submit query - query.submit(); - REQUIRE(query.query_status() == Query::Status::COMPLETE); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 4); - REQUIRE(std::get<1>(result_el["a2"]) == 8); - REQUIRE(std::get<2>(result_el["a2"]) == 4); - REQUIRE(std::get<0>(result_el["a3"]) == 4); - REQUIRE(std::get<1>(result_el["a3"]) == 114); - } - - SECTION("- Incomplete read") { - Array array(ctx, array_uri, TILEDB_READ); - Query query(ctx, array); - std::vector a1(4); - std::vector a2(4); - std::vector a2_nullable(4); - std::vector a3_data(60); - std::vector a3_offsets(4); - std::vector subarray = {3, 4, 3, 4}; - Subarray cppapi_subarray(ctx, array); - cppapi_subarray.add_range(0, subarray[0], subarray[1]); - cppapi_subarray.add_range(1, subarray[2], subarray[3]); - query.set_subarray(cppapi_subarray); - - auto set_buffers = [&](Query& q) { - q.set_data_buffer("a1", a1); - q.set_data_buffer("a2", a2); - q.set_validity_buffer("a2", a2_nullable); - q.set_data_buffer("a3", a3_data); - q.set_offsets_buffer("a3", a3_offsets); - }; - - // Submit initial query. - set_buffers(query); - query.submit(); - REQUIRE(query.query_status() == Query::Status::INCOMPLETE); - - auto result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 2); - REQUIRE(std::get<1>(result_el["a2"]) == 4); - REQUIRE(std::get<2>(result_el["a2"]) == 2); - REQUIRE(std::get<0>(result_el["a3"]) == 2); - REQUIRE(std::get<1>(result_el["a3"]) == 47); - - // Reset buffers, serialize and resubmit - set_buffers(query); - // Submit query - query.submit(); - - REQUIRE(query.query_status() == Query::Status::INCOMPLETE); - result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 1); - REQUIRE(std::get<1>(result_el["a2"]) == 2); - REQUIRE(std::get<2>(result_el["a2"]) == 1); - REQUIRE(std::get<0>(result_el["a3"]) == 1); - REQUIRE(std::get<1>(result_el["a3"]) == 33); - - // Reset buffers, serialize and resubmit - set_buffers(query); - // Submit query - query.submit(); - - REQUIRE(query.query_status() == Query::Status::COMPLETE); - result_el = query.result_buffer_elements_nullable(); - REQUIRE(std::get<1>(result_el["a1"]) == 1); - REQUIRE(std::get<1>(result_el["a2"]) == 2); - REQUIRE(std::get<2>(result_el["a2"]) == 1); - REQUIRE(std::get<0>(result_el["a3"]) == 1); - REQUIRE(std::get<1>(result_el["a3"]) == 34); - } -} diff --git a/test/src/unit-cppapi-array.cc b/test/src/unit-cppapi-array.cc index d11dc4894c21..287a36c187b3 100644 --- a/test/src/unit-cppapi-array.cc +++ b/test/src/unit-cppapi-array.cc @@ -1984,7 +1984,8 @@ TEST_CASE( // Read from upgraded version Array array_read(ctx, old_array_name, TILEDB_READ); - std::vector subarray_read = {1, 4, 10, 10}; + Subarray subarray_read(ctx, array_read); + subarray_read.set_subarray({1, 4, 10, 10}); std::vector a_read; a_read.resize(4); std::vector d1_read; From 18018e1c9aab965020f5b165916a577f923eb11b Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 5 Jun 2024 07:04:49 -0400 Subject: [PATCH 402/456] Migrate APIs out of StorageManager: array_evolve_schema. (#5038) Migrate APIs out of `StorageManager`: `array_evolve_schema`. [sc-44986] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `array_evolve_schema`. --- test/src/unit-enumerations.cc | 8 ++- tiledb/sm/array/array.cc | 65 ++++++++++++++++++ tiledb/sm/array/array.h | 16 +++++ tiledb/sm/c_api/tiledb.cc | 7 +- tiledb/sm/storage_manager/storage_manager.cc | 66 ------------------- .../storage_manager_canonical.h | 15 ----- 6 files changed, 91 insertions(+), 86 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 11c9ed1427ac..97a24cd5960d 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2064,9 +2064,11 @@ TEST_CASE_METHOD( auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); - auto st = ctx_.storage_manager()->array_evolve_schema( - array->array_uri(), ase.get(), array->get_encryption_key()); - throw_if_not_ok(st); + throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + ctx_.resources(), + array->array_uri(), + ase.get(), + array->get_encryption_key())); // Check that we can not rewrite the query condition. array = get_array(QueryType::READ); diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 0080ee73bba4..3d9cee782b7c 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -165,6 +165,71 @@ OpenedArray::load_delete_and_update_conditions() { return {Status::Ok(), conditions, update_values}; } +Status Array::evolve_array_schema( + ContextResources& resources, + const URI& array_uri, + ArraySchemaEvolution* schema_evolution, + const EncryptionKey& encryption_key) { + // Check array schema + if (schema_evolution == nullptr) { + throw ArrayException("Cannot evolve array; Empty schema evolution"); + } + + if (array_uri.is_tiledb()) { + return resources.rest_client()->post_array_schema_evolution_to_rest( + array_uri, schema_evolution); + } + + // Load URIs from the array directory + tiledb::sm::ArrayDirectory array_dir{ + resources, + array_uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY}; + + // Check if array exists + if (!is_array(resources, array_uri)) { + throw ArrayException( + "Cannot evolve array; '" + array_uri.to_string() + "' does not exist"); + } + + auto&& array_schema = array_dir.load_array_schema_latest( + encryption_key, resources.ephemeral_memory_tracker()); + + // Load required enumerations before evolution. + auto enmr_names = schema_evolution->enumeration_names_to_extend(); + if (enmr_names.size() > 0) { + std::unordered_set enmr_path_set; + for (auto name : enmr_names) { + enmr_path_set.insert(array_schema->get_enumeration_path_name(name)); + } + std::vector enmr_paths; + for (auto path : enmr_path_set) { + enmr_paths.emplace_back(path); + } + + auto loaded_enmrs = array_dir.load_enumerations_from_paths( + enmr_paths, encryption_key, resources.create_memory_tracker()); + + for (auto enmr : loaded_enmrs) { + array_schema->store_enumeration(enmr); + } + } + + // Evolve schema + auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); + + Status st = + store_array_schema(resources, array_schema_evolved, encryption_key); + if (!st.ok()) { + throw ArrayException( + "Cannot evolve schema; Not able to store evolved array schema."); + } + + return Status::Ok(); +} + const URI& Array::array_uri() const { return array_uri_; } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 08ed27545580..d239dddc33ac 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -52,6 +52,7 @@ using namespace tiledb::common; namespace tiledb::sm { class ArraySchema; +class ArraySchemaEvolution; class SchemaEvolution; class FragmentMetadata; class MemoryTracker; @@ -363,6 +364,21 @@ class Array { opened_array_->set_array_schemas_all(std::move(all_schemas)); } + /** + * Evolve a TileDB array schema and store its new schema. + * + * @param resources The context resources. + * @param array_uri The uri of the array whose schema is to be evolved. + * @param schema_evolution The schema evolution. + * @param encryption_key The encryption key to use. + * @return Status + */ + static Status evolve_array_schema( + ContextResources& resources, + const URI& array_uri, + ArraySchemaEvolution* array_schema, + const EncryptionKey& encryption_key); + /** Returns the array URI. */ const URI& array_uri() const; diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index f9e28b05b116..96c8a216b72d 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3016,8 +3016,11 @@ int32_t tiledb_array_evolve( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Evolve schema - throw_if_not_ok(ctx->storage_manager()->array_evolve_schema( - uri, array_schema_evolution->array_schema_evolution_, key)); + throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + ctx->resources(), + uri, + array_schema_evolution->array_schema_evolution_, + key)); // Success return TILEDB_OK; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index e8a58c3e27c2..07aeb4fa4648 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -131,72 +131,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManager::array_evolve_schema( - const URI& array_uri, - ArraySchemaEvolution* schema_evolution, - const EncryptionKey& encryption_key) { - // Check array schema - if (schema_evolution == nullptr) { - throw StorageManagerException( - "Cannot evolve array; Empty schema evolution"); - } - - if (array_uri.is_tiledb()) { - return resources_.rest_client()->post_array_schema_evolution_to_rest( - array_uri, schema_evolution); - } - - // Load URIs from the array directory - tiledb::sm::ArrayDirectory array_dir{ - resources(), - array_uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY}; - - // Check if array exists - if (!is_array(resources_, array_uri)) { - throw StorageManagerException( - "Cannot evolve array; Array '" + array_uri.to_string() + - "' not exists"); - } - - auto&& array_schema = array_dir.load_array_schema_latest( - encryption_key, resources_.ephemeral_memory_tracker()); - - // Load required enumerations before evolution. - auto enmr_names = schema_evolution->enumeration_names_to_extend(); - if (enmr_names.size() > 0) { - std::unordered_set enmr_path_set; - for (auto name : enmr_names) { - enmr_path_set.insert(array_schema->get_enumeration_path_name(name)); - } - std::vector enmr_paths; - for (auto path : enmr_path_set) { - enmr_paths.emplace_back(path); - } - - auto loaded_enmrs = array_dir.load_enumerations_from_paths( - enmr_paths, encryption_key, resources_.create_memory_tracker()); - - for (auto enmr : loaded_enmrs) { - array_schema->store_enumeration(enmr); - } - } - - // Evolve schema - auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - - Status st = - store_array_schema(resources_, array_schema_evolved, encryption_key); - if (!st.ok()) { - throw StorageManagerException( - "Cannot evolve schema; Not able to store evolved array schema."); - } - - return Status::Ok(); -} - Status StorageManagerCanonical::array_upgrade_version( const URI& array_uri, const Config& override_config) { // Check if array exists diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index f2c7d55127d5..c603ef66f434 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -64,7 +64,6 @@ namespace tiledb::sm { class Array; class ArrayDirectory; class ArraySchema; -class ArraySchemaEvolution; class Consolidator; class EncryptionKey; class Query; @@ -135,20 +134,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Evolve a TileDB array schema and store its new schema. - * - * @param array_dir The ArrayDirectory object used to retrieve the - * various URIs in the array directory. - * @param schema_evolution The schema evolution. - * @param encryption_key The encryption key to use. - * @return Status - */ - Status array_evolve_schema( - const URI& uri, - ArraySchemaEvolution* array_schema, - const EncryptionKey& encryption_key); - /** * Upgrade a TileDB array to latest format version. * From a64bfabacf78f86f63abb59a4a75bf2df469c36d Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:18:24 -0400 Subject: [PATCH 403/456] Migrate APIs out of StorageManager: group_metadata_consolidate. (#5040) Migrate APIs out of `StorageManager`: `group_metadata_consolidate`. [sc-48743] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `group_metadata_consolidate`. --- tiledb/api/c_api/group/group_api.cc | 4 +-- .../storage_manager_override.h | 4 --- tiledb/sm/group/group.cc | 26 +++++++++++++++++++ tiledb/sm/group/group.h | 16 ++++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 25 ------------------ .../storage_manager_canonical.h | 13 ---------- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 177ddc873293..ae6b8bb280e6 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -576,8 +576,8 @@ capi_return_t tiledb_group_consolidate_metadata( ensure_group_uri_argument_is_valid(group_uri); auto cfg = (config == nullptr) ? ctx->config() : config->config(); - throw_if_not_ok( - ctx->storage_manager()->group_metadata_consolidate(group_uri, cfg)); + throw_if_not_ok(tiledb::sm::Group::consolidate_metadata( + ctx->resources(), group_uri, cfg)); return TILEDB_OK; } diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 66017e40dba7..2fa240c46c2d 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -61,10 +61,6 @@ class StorageManagerStub { inline Status cancel_all_tasks() { return Status{}; } - inline Status group_metadata_consolidate(const char*, const Config&) { - throw std::logic_error( - "StorageManagerStub does not support group metadata consolidation"); - } inline Status group_metadata_vacuum(const char*, const Config&) { throw std::logic_error( "StorageManagerStub does not support group metadata vacuum"); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 1db111d5685d..c9803b67675b 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -36,6 +36,7 @@ #include "tiledb/common/memory_tracker.h" #include "tiledb/common/stdx_string.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/query_type.h" @@ -573,6 +574,31 @@ void Group::set_metadata_loaded(const bool metadata_loaded) { metadata_loaded_ = metadata_loaded; } +Status Group::consolidate_metadata( + ContextResources& resources, const char* group_name, const Config& config) { + // Check group URI + URI group_uri(group_name); + if (group_uri.is_invalid()) { + throw GroupException("Cannot consolidate group metadata; Invalid URI"); + } + // Check if group exists + ObjectType obj_type; + throw_if_not_ok(object_type(resources, group_uri, &obj_type)); + + if (obj_type != ObjectType::GROUP) { + throw GroupException( + "Cannot consolidate group metadata; Group does not exist"); + } + + // Consolidate + // Encryption credentials are loaded by Group from config + StorageManager sm(resources, resources.logger(), config); + auto consolidator = + Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); + return consolidator->consolidate( + group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); +} + const EncryptionKey* Group::encryption_key() const { return encryption_key_.get(); } diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index bdc703adee78..046c9dc1fd20 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -238,6 +238,22 @@ class Group { */ void set_metadata_loaded(const bool metadata_loaded); + /** + * Consolidates the metadata of a group into a single file. + * + * @param resources The context resources. + * @param group_name The name of the group whose metadata will be + * consolidated. + * @param config Configuration parameters for the consolidation + * (`nullptr` means default, which will use the config associated with + * this instance). + * @return Status + */ + static Status consolidate_metadata( + ContextResources& resources, + const char* group_name, + const Config& config); + /** Returns a constant pointer to the encryption key. */ const EncryptionKey* encryption_key() const; diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 07aeb4fa4648..74c21f0055fc 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -478,31 +478,6 @@ Status StorageManagerCanonical::set_default_tags() { return Status::Ok(); } -Status StorageManagerCanonical::group_metadata_consolidate( - const char* group_name, const Config& config) { - // Check group URI - URI group_uri(group_name); - if (group_uri.is_invalid()) { - throw StorageManagerException( - "Cannot consolidate group metadata; Invalid URI"); - } - // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources_, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { - throw StorageManagerException( - "Cannot consolidate group metadata; Group does not exist"); - } - - // Consolidate - // Encryption credentials are loaded by Group from config - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, this); - return consolidator->consolidate( - group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); -} - void StorageManagerCanonical::group_metadata_vacuum( const char* group_name, const Config& config) { // Check group URI diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index c603ef66f434..43f7c869bba6 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -262,19 +262,6 @@ class StorageManagerCanonical { return resources_; } - /** - * Consolidates the metadata of a group into a single file. - * - * @param group_name The name of the group whose metadata will be - * consolidated. - * @param config Configuration parameters for the consolidation - * (`nullptr` means default, which will use the config associated with - * this instance). - * @return Status - */ - Status group_metadata_consolidate( - const char* group_name, const Config& config); - /** * Vacuums the consolidated metadata files of a group. * From 94a2d982964a4b896014678fd565577fc74ae3b1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 5 Jun 2024 23:50:23 +0300 Subject: [PATCH 404/456] Remove pin to MSVC toolset version in CI. (#5047) [SC-48973](https://app.shortcut.com/tiledb-inc/story/48973/revert-pins-to-msvc-toolset-14-39) GitHub Actions is starting to update the MSVC toolset version in CI, causing [occasional failures](https://github.com/TileDB-Inc/TileDB/actions/runs/9386900730/job/25848568717). This PR removes the pin to version 14.39 added in #4759, hoping it is no longer necessary. --- TYPE: NO_HISTORY --- .github/workflows/build-windows.yml | 4 ---- .github/workflows/unit-test-runs.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 9b62cff059db..aebad0e24cc1 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -131,10 +131,6 @@ jobs: uses: TheMrMilchmann/setup-msvc-dev@v3 with: arch: x64 - # By default Visual Studio 2022 chooses the earliest installed toolset - # version for the main build and vcpkg chooses the latest. Force it to - # use the latest (14.39 currently). - toolset: ${{ matrix.os == 'windows-2022' && '14.39' || '' }} # This must happen after checkout, because checkout would remove the directory. - name: Install Ninja uses: seanmiddleditch/gha-setup-ninja@v4 diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index dea85a3fea47..0b1bfce01f39 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -32,10 +32,6 @@ jobs: if: ${{ startsWith(matrix.os, 'windows-') }} with: arch: x64 - # By default Visual Studio 2022 chooses the earliest installed toolset - # version for the main build and vcpkg chooses the latest. Force it to - # use the latest (14.39 currently). - toolset: '14.39' # Configure required environment variables for vcpkg to use # GitHub's Action Cache From dfe84f64288cbc1cff412792d088273fcff5e0e8 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Thu, 6 Jun 2024 02:20:52 -0400 Subject: [PATCH 405/456] Migrate APIs out of StorageManager: array_upgrade_version. (#5039) Migrate APIs out of `StorageManager`: `array_upgrade_version`. [sc-44987] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `array_upgrade_version`. --- tiledb/sm/array/array.cc | 77 +++++++++++++++++++ tiledb/sm/array/array.h | 13 ++++ tiledb/sm/c_api/tiledb.cc | 6 +- tiledb/sm/storage_manager/storage_manager.cc | 74 ------------------ .../storage_manager_canonical.h | 12 --- 5 files changed, 94 insertions(+), 88 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 3d9cee782b7c..1deb57ecf1b4 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -2124,6 +2124,83 @@ const ArrayDirectory& Array::load_array_directory() { return array_directory(); } +Status Array::upgrade_version( + ContextResources& resources, + const URI& array_uri, + const Config& override_config) { + // Check if array exists + if (!is_array(resources, array_uri)) { + throw ArrayException( + "Cannot upgrade array; Array '" + array_uri.to_string() + + "' does not exist"); + } + + // Load URIs from the array directory + tiledb::sm::ArrayDirectory array_dir{ + resources, + array_uri, + 0, + UINT64_MAX, + tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY}; + + // Get encryption key from config + bool found = false; + std::string encryption_key_from_cfg = + override_config.get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + override_config.get("sm.encryption_type", &found); + assert(found); + auto [st1, etc] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st1); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + throw_if_not_ok( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + } else { + throw_if_not_ok(encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + static_cast(encryption_key_from_cfg.size()))); + } + + auto&& array_schema = array_dir.load_array_schema_latest( + encryption_key_cfg, resources.ephemeral_memory_tracker()); + + if (array_schema->version() < constants::format_version) { + array_schema->generate_uri(); + array_schema->set_version(constants::format_version); + + // Create array schema directory if necessary + URI array_schema_dir_uri = + array_uri.join_path(constants::array_schema_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); + + // Store array schema + throw_if_not_ok( + store_array_schema(resources, array_schema, encryption_key_cfg)); + + // Create commit directory if necessary + URI array_commit_uri = + array_uri.join_path(constants::array_commits_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_commit_uri)); + + // Create fragments directory if necessary + URI array_fragments_uri = + array_uri.join_path(constants::array_fragments_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragments_uri)); + + // Create fragment metadata directory if necessary + URI array_fragment_metadata_uri = + array_uri.join_path(constants::array_fragment_meta_dir_name); + throw_if_not_ok(resources.vfs().create_dir(array_fragment_metadata_uri)); + } + + return Status::Ok(); +} + Status Array::compute_non_empty_domain() { // Keep the current opened array alive for the duration of this call. auto opened_array = opened_array_; diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index d239dddc33ac..2e2ccad56e3f 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -1046,6 +1046,19 @@ class Array { return resources_.rest_client(); } + /** + * Upgrade a TileDB array to latest format version. + * + * @param resources The context resources. + * @param uri The URI of the array. + * @param config Configuration parameters for the upgrade + * (`nullptr` means default, which will use the config associated with + * this instance). + * @return Status + */ + static Status upgrade_version( + ContextResources& resources, const URI& uri, const Config& config); + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 96c8a216b72d..26249ea91b64 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -3071,8 +3071,10 @@ int32_t tiledb_array_upgrade_version( } // Upgrade version - throw_if_not_ok(ctx->storage_manager()->array_upgrade_version( - uri, (config == nullptr) ? ctx->config() : config->config())); + throw_if_not_ok(tiledb::sm::Array::upgrade_version( + ctx->resources(), + uri, + (config == nullptr) ? ctx->config() : config->config())); return TILEDB_OK; } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 74c21f0055fc..30bec13652dc 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -131,80 +131,6 @@ StorageManagerCanonical::~StorageManagerCanonical() { /* API */ /* ****************************** */ -Status StorageManagerCanonical::array_upgrade_version( - const URI& array_uri, const Config& override_config) { - // Check if array exists - if (!is_array(resources_, array_uri)) { - throw StorageManagerException( - "Cannot upgrade array; Array '" + array_uri.to_string() + - "' does not exist"); - } - - // Load URIs from the array directory - tiledb::sm::ArrayDirectory array_dir{ - resources(), - array_uri, - 0, - UINT64_MAX, - tiledb::sm::ArrayDirectoryMode::SCHEMA_ONLY}; - - // Get encryption key from config - bool found = false; - std::string encryption_key_from_cfg = - override_config.get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - override_config.get("sm.encryption_type", &found); - assert(found); - auto [st1, etc] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK(st1); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - RETURN_NOT_OK(encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); - } else { - RETURN_NOT_OK(encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - static_cast(encryption_key_from_cfg.size()))); - } - - auto&& array_schema = array_dir.load_array_schema_latest( - encryption_key_cfg, resources().ephemeral_memory_tracker()); - - if (array_schema->version() < constants::format_version) { - array_schema->generate_uri(); - array_schema->set_version(constants::format_version); - - // Create array schema directory if necessary - URI array_schema_dir_uri = - array_uri.join_path(constants::array_schema_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_schema_dir_uri)); - - // Store array schema - auto st = store_array_schema(resources_, array_schema, encryption_key_cfg); - RETURN_NOT_OK_ELSE(st, logger_->status_no_return_value(st)); - - // Create commit directory if necessary - URI array_commit_uri = - array_uri.join_path(constants::array_commits_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_commit_uri)); - - // Create fragments directory if necessary - URI array_fragments_uri = - array_uri.join_path(constants::array_fragments_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragments_uri)); - - // Create fragment metadata directory if necessary - URI array_fragment_metadata_uri = - array_uri.join_path(constants::array_fragment_meta_dir_name); - throw_if_not_ok(resources_.vfs().create_dir(array_fragment_metadata_uri)); - } - - return Status::Ok(); -} - Status StorageManagerCanonical::async_push_query(Query* query) { cancelable_tasks_.execute( &resources_.compute_tp(), diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 43f7c869bba6..def3a5f6bfc5 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -134,18 +134,6 @@ class StorageManagerCanonical { /* API */ /* ********************************* */ - /** - * Upgrade a TileDB array to latest format version. - * - * @param array_dir The ArrayDirectory object used to retrieve the - * various URIs in the array directory. - * @param config Configuration parameters for the upgrade - * (`nullptr` means default, which will use the config associated with - * this instance). - * @return Status - */ - Status array_upgrade_version(const URI& uri, const Config& config); - /** * Pushes an async query to the queue. * From 2593e54a11014cdbe21e4eea5f65e3ec16547362 Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Thu, 6 Jun 2024 02:59:11 -0400 Subject: [PATCH 406/456] Prevent constructing or setting attribute with invalid cell_val_num. (#4952) While working on the Rust API I observed that the following code would error when `datatype == Datatype::Any`: ``` AttributeBuilder::new(context, field.name(), datatype)? .nullability(field.is_nullable())? .cell_val_num(cell_val_num)? ``` the error arises in the call to core function `set_cell_val_num`, which errors out if the datatype is `ANY`, even if you set to `constants::var_num` which is required for `ANY`. From there I noticed that it is also possible to construct an Attribute with an invalid cell val num, e.g. construct an `ANY` attribute with a non-var cell val num. This pull request adds the `check_cell_val_num` function and moves all associated error-checking logic into it, then calls this function from all of the `Attribute` constructors as well as `set_cell_val_num`. --- TYPE: BUG DESC: Prevent constructing attribute with invalid cell_val_num. --- tiledb/sm/array_schema/attribute.cc | 62 +-- tiledb/sm/array_schema/attribute.h | 28 +- tiledb/sm/array_schema/test/CMakeLists.txt | 3 +- tiledb/sm/array_schema/test/unit_attribute.cc | 356 ++++++++++++++++++ 4 files changed, 417 insertions(+), 32 deletions(-) create mode 100644 tiledb/sm/array_schema/test/unit_attribute.cc diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index ad4b55d5c0dc..e479ddb73672 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -57,6 +57,8 @@ class AttributeStatusException : public StatusException { } }; +using AttributeException = AttributeStatusException; + /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -81,25 +83,11 @@ Attribute::Attribute( , name_(name) , type_(type) , order_(order) { - set_default_fill_value(); - - // If ordered, check the number of values of cells is supported. if (order_ != DataOrder::UNORDERED_DATA) { ensure_ordered_attribute_datatype_is_valid(type_); - if (type == Datatype::STRING_ASCII) { - if (cell_val_num_ != constants::var_num) { - throw std::invalid_argument( - "Ordered attributes with datatype '" + datatype_str(type_) + - "' must have cell_val_num=1."); - } - } else { - if (cell_val_num_ != 1) { - throw std::invalid_argument( - "Ordered attributes with datatype '" + datatype_str(type_) + - "' must have cell_val_num=1."); - } - } } + validate_cell_val_num(cell_val_num_); + set_default_fill_value(); } Attribute::Attribute( @@ -121,6 +109,7 @@ Attribute::Attribute( , fill_value_validity_(fill_value_validity) , order_(order) , enumeration_name_(enumeration_name) { + validate_cell_val_num(cell_val_num_); } /* ********************************* */ @@ -303,22 +292,41 @@ void Attribute::serialize( } void Attribute::set_cell_val_num(unsigned int cell_val_num) { - if (type_ == Datatype::ANY) { - throw AttributeStatusException( + validate_cell_val_num(cell_val_num); + cell_val_num_ = cell_val_num; + set_default_fill_value(); +} + +void Attribute::validate_cell_val_num(unsigned int cell_val_num) const { + if (type_ == Datatype::ANY && cell_val_num != constants::var_num) { + throw AttributeException( "Cannot set number of values per cell; Attribute datatype `ANY` is " "always variable-sized"); } - if (order_ != DataOrder::UNORDERED_DATA && type_ != Datatype::STRING_ASCII && - cell_val_num != 1) { - throw AttributeStatusException( - "Cannot set number of values per cell; An ordered attribute with " - "datatype '" + - datatype_str(type_) + "' can only have cell_val_num=1."); + // If ordered, check the number of values of cells is supported. + if (order_ != DataOrder::UNORDERED_DATA) { + if (type_ == Datatype::STRING_ASCII) { + if (cell_val_num != constants::var_num) { + throw AttributeException( + "Cannot set number of values per cell; Ordered attributes with " + "datatype '" + + datatype_str(type_) + + "' must have `cell_val_num=constants::var_num`."); + } + } else { + if (cell_val_num != 1) { + throw AttributeException( + "Ordered attributes with datatype '" + datatype_str(type_) + + "' must have `cell_val_num=1`."); + } + } } - cell_val_num_ = cell_val_num; - set_default_fill_value(); + // check zero last so we get the more informative error first + if (cell_val_num == 0) { + throw AttributeException("Cannot set zero values per cell"); + } } void Attribute::set_nullable(const bool nullable) { diff --git a/tiledb/sm/array_schema/attribute.h b/tiledb/sm/array_schema/attribute.h index 830e63fd552f..3b968f696549 100644 --- a/tiledb/sm/array_schema/attribute.h +++ b/tiledb/sm/array_schema/attribute.h @@ -56,7 +56,17 @@ class Enumeration; enum class Compressor : uint8_t; enum class Datatype : uint8_t; -/** Manipulates a TileDB attribute. */ +/** + * Manipulates a TileDB attribute. + * + * @invariant + * + * A valid `cell_val_num` depends on the Attribute datatype and ordering. + * For `Datatype::ANY`, the only valid value is `constants::var_num`. + * If the attribute is unordered, then all other datatypes support any value. + * If the attribute is ordered, then an Attribute of `Datatype::STRING_ASCII` + * must have `constants::var_num`, and all other datatypes must have 1. + */ class Attribute { public: /* ********************************* */ @@ -84,6 +94,8 @@ class Attribute { * @param type The type of the attribute. * @param cell_val_num The cell number of the attribute. * @param order The ordering of the attribute. + * + * @throws if `cell_val_num` is invalid. See `set_cell_val_num`. */ Attribute( const std::string& name, @@ -102,6 +114,8 @@ class Attribute { * @param fill_value The fill value of the attribute. * @param fill_value_validity The validity of fill_value. * @param order The order of the data stored in the attribute. + * + * @throws if `cell_val_num` is invalid. See `set_cell_val_num`. */ Attribute( const std::string& name, @@ -237,9 +251,13 @@ class Attribute { void serialize(Serializer& serializer, uint32_t version) const; /** - * Sets the attribute number of values per cell. Note that if the attribute - * datatype is `ANY` this function returns an error, since `ANY` datatype - * must always be variable-sized. + * Sets the attribute number of values per cell. + * + * @throws AttributeException if `cell_val_num` is invalid. See class + * documentation. + * + * @post `this->cell_val_num() == cell_val_num` if `cell_val_num` is + * valid, and `this->cell_val_num()` is unchanged otherwise. */ void set_cell_val_num(unsigned int cell_val_num); @@ -306,6 +324,8 @@ class Attribute { /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ + /** Called to validate a cell val num for this attribute */ + void validate_cell_val_num(unsigned cell_val_num) const; /** Sets the default fill value. */ void set_default_fill_value(); diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index efb004a35ee4..41d6018b91dc 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021-2022 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ commence(unit_test array_schema) this_target_sources( main.cc unit_array_schema.cc + unit_attribute.cc unit_dimension.cc unit_dimension_label.cc unit_domain_data.cc diff --git a/tiledb/sm/array_schema/test/unit_attribute.cc b/tiledb/sm/array_schema/test/unit_attribute.cc new file mode 100644 index 000000000000..5aa4f4b6df41 --- /dev/null +++ b/tiledb/sm/array_schema/test/unit_attribute.cc @@ -0,0 +1,356 @@ +/** + * @file tiledb/sm/array_schema/test/unit_arribute.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2022-2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file contains unit tests for the attribute class + */ + +#include + +#include "tiledb/sm/array_schema/attribute.h" + +using namespace tiledb::common; +using namespace tiledb::sm; + +// Datatype::ANY attribute is var-sized by default +TEST_CASE("ANY Attribute default cell val num", "[array_schema][attribute]") { + Attribute a("a", Datatype::ANY, false); + CHECK(a.cell_val_num() == constants::var_num); +} + +// Pass non-default cell val num to constructor +TEST_CASE("ANY Attribute cell val num validity", "[array_schema][attribute]") { + SECTION( + "Constructor: default cell val num is `constants::var_num` for " + "`Datatype::ANY`") { + CHECK( + Attribute("a", Datatype::ANY, false).cell_val_num() == + constants::var_num); + } + + FilterPipeline empty_pipeline; + ByteVecValue empty_fill; + + SECTION("Constructor: `constants::var_num` is allowed for `Datatype::ANY`") { + CHECK_NOTHROW(Attribute( + "a", Datatype::ANY, constants::var_num, DataOrder::UNORDERED_DATA)); + CHECK_NOTHROW(Attribute( + "a", + Datatype::ANY, + false, + constants::var_num, + empty_pipeline, + empty_fill, + 0, + DataOrder::UNORDERED_DATA)); + } + + SECTION( + "set_cell_val_num: no error when redundantly setting " + "`constants::var_num`") { + Attribute a("a", Datatype::ANY, false); + REQUIRE(a.cell_val_num() == constants::var_num); + REQUIRE_NOTHROW(a.set_cell_val_num(constants::var_num)); + CHECK(a.cell_val_num() == constants::var_num); + } + + const unsigned int invalid_cell_val_num = GENERATE(0, 1, 2, 5, 10, 100, 1000); + + SECTION( + "Constructor: only `constants::var_num` is allowed for `Datatype::ANY`") { + CHECK_THROWS(Attribute( + "a", Datatype::ANY, invalid_cell_val_num, DataOrder::UNORDERED_DATA)); + CHECK_THROWS(Attribute( + "a", + Datatype::ANY, + false, + invalid_cell_val_num, + empty_pipeline, + empty_fill, + 0, + DataOrder::UNORDERED_DATA)); + } + + SECTION( + "set_cell_val_num: only `constants::var_num` is allowed for " + "`Datatype::ANY`") { + Attribute a("a", Datatype::ANY, false); + const auto cell_val_num = a.cell_val_num(); + REQUIRE_THROWS(a.set_cell_val_num(invalid_cell_val_num)); + CHECK(a.cell_val_num() == cell_val_num); + } +} + +TEST_CASE( + "Non-ANY unordered Attribute cell val num validity", + "[array_schema][attribute]") { + // Includes all datatypes except Datatype::ANY which is covered above + const auto attribute_datatype = GENERATE( + Datatype::FLOAT32, + Datatype::FLOAT64, + Datatype::INT8, + Datatype::UINT8, + Datatype::INT16, + Datatype::UINT16, + Datatype::INT32, + Datatype::UINT32, + Datatype::INT64, + Datatype::UINT64, + Datatype::STRING_ASCII, + Datatype::STRING_UTF8, + Datatype::STRING_UTF16, + Datatype::STRING_UTF32, + Datatype::STRING_UCS2, + Datatype::STRING_UCS4, + Datatype::DATETIME_YEAR, + Datatype::DATETIME_WEEK, + Datatype::DATETIME_DAY, + Datatype::DATETIME_HR, + Datatype::DATETIME_MIN, + Datatype::DATETIME_SEC, + Datatype::DATETIME_MS, + Datatype::DATETIME_US, + Datatype::DATETIME_NS, + Datatype::DATETIME_PS, + Datatype::DATETIME_FS, + Datatype::DATETIME_AS, + Datatype::TIME_HR, + Datatype::TIME_MIN, + Datatype::TIME_SEC, + Datatype::TIME_MS, + Datatype::TIME_US, + Datatype::TIME_NS, + Datatype::TIME_PS, + Datatype::TIME_FS, + Datatype::TIME_AS, + Datatype::BOOL, + Datatype::BLOB, + Datatype::GEOM_WKT, + Datatype::GEOM_WKB); + + DYNAMIC_SECTION( + "Constructor: zero cell val num is invalid for datatype " + + datatype_str(attribute_datatype)) { + CHECK_THROWS( + Attribute("a", attribute_datatype, 0, DataOrder::UNORDERED_DATA)); + } + + const unsigned int valid_cell_val_num = + GENERATE(1, 2, 5, 10, 100, 1000, constants::var_num); + + DYNAMIC_SECTION( + "Constructor: default cell val num is 1 for datatype " + + datatype_str(attribute_datatype)) { + CHECK(Attribute("a", attribute_datatype, false).cell_val_num() == 1); + } + + DYNAMIC_SECTION( + "Constructor: all nonzero cell val nums are valid for unordered " + "attribute with " + "datatype " + + datatype_str(attribute_datatype)) { + CHECK( + Attribute( + "a", + attribute_datatype, + valid_cell_val_num, + DataOrder::UNORDERED_DATA) + .cell_val_num() == valid_cell_val_num); + } + + DYNAMIC_SECTION( + "set_cell_val_num: all nonzero cell val nums are valid for unordered " + "attribute " + "with datatype " + + datatype_str(attribute_datatype)) { + Attribute a("a", attribute_datatype, 1, DataOrder::UNORDERED_DATA); + REQUIRE(a.cell_val_num() == 1); + REQUIRE_NOTHROW(a.set_cell_val_num(valid_cell_val_num)); + CHECK(a.cell_val_num() == valid_cell_val_num); + } + + DYNAMIC_SECTION( + "set_cell_val_num: zero cell val num is invalid for datatype " + + datatype_str(attribute_datatype)) { + Attribute a( + "a", attribute_datatype, valid_cell_val_num, DataOrder::UNORDERED_DATA); + REQUIRE(a.cell_val_num() == valid_cell_val_num); + REQUIRE_THROWS(a.set_cell_val_num(0)); + CHECK(a.cell_val_num() == valid_cell_val_num); + } +} + +TEST_CASE( + "STRING_ASCII ordered Attribute cell val num validity" + "[array_schema][attribute]") { + const unsigned int invalid_cell_val_num = GENERATE(0, 1, 2, 5, 10, 100, 1000); + + FilterPipeline empty_pipeline; + ByteVecValue empty_fill; + + SECTION( + "Constructor: anything other than `constants::var_num` is invalid for " + "increasing " + "STRING_ASCII attribute") { + CHECK_THROWS(Attribute( + "a", + Datatype::STRING_ASCII, + invalid_cell_val_num, + DataOrder::INCREASING_DATA)); + + CHECK_THROWS(Attribute( + "a", + Datatype::STRING_ASCII, + true, + invalid_cell_val_num, + empty_pipeline, + empty_fill, + 0, + DataOrder::INCREASING_DATA, + std::optional())); + } + + SECTION( + "set_cell_val_num: anything other than `constants::var_num` is invalid " + "for increasing STRING_ASCII attribute") { + Attribute a( + "a", + Datatype::STRING_ASCII, + constants::var_num, + DataOrder::INCREASING_DATA); + REQUIRE(a.cell_val_num() == constants::var_num); + REQUIRE_THROWS(a.set_cell_val_num(invalid_cell_val_num)); + CHECK(a.cell_val_num() == constants::var_num); + } +} + +TEST_CASE( + "Non-ANY, Non-STRING_ASCII ordered Attribute cell val num validity", + "[array_schema][attribute]") { + // Datatype::ANY and string datatypes intentionally omitted + const auto attribute_datatype = GENERATE( + Datatype::FLOAT32, + Datatype::FLOAT64, + Datatype::INT8, + Datatype::UINT8, + Datatype::INT16, + Datatype::UINT16, + Datatype::INT32, + Datatype::UINT32, + Datatype::INT64, + Datatype::UINT64, + // STRING_ASCII omitted due to different semantics, see other TEST_CASEs + // Other string datatypes omitted as they are not valid for ordered + // attributes. See `ensure_ordered_attribute_datatype_is_valid`. + Datatype::DATETIME_YEAR, + Datatype::DATETIME_WEEK, + Datatype::DATETIME_DAY, + Datatype::DATETIME_HR, + Datatype::DATETIME_MIN, + Datatype::DATETIME_SEC, + Datatype::DATETIME_MS, + Datatype::DATETIME_US, + Datatype::DATETIME_NS, + Datatype::DATETIME_PS, + Datatype::DATETIME_FS, + Datatype::DATETIME_AS, + Datatype::TIME_HR, + Datatype::TIME_MIN, + Datatype::TIME_SEC, + Datatype::TIME_MS, + Datatype::TIME_US, + Datatype::TIME_NS, + Datatype::TIME_PS, + Datatype::TIME_FS, + Datatype::TIME_AS); + + FilterPipeline empty_pipeline; + ByteVecValue empty_fill; + + DYNAMIC_SECTION( + "Constructor: cell val num 1 is valid for increasing attribute with " + "datatype " + + datatype_str(attribute_datatype)) { + CHECK_NOTHROW( + Attribute("a", attribute_datatype, 1, DataOrder::INCREASING_DATA)); + CHECK_NOTHROW(Attribute( + "a", + attribute_datatype, + true, + 1, + empty_pipeline, + empty_fill, + 0, + DataOrder::INCREASING_DATA, + std::optional())); + } + + DYNAMIC_SECTION( + "set_cell_val_num: cell val num 1 is valid for increasing attribute with " + "datatype " + + datatype_str(attribute_datatype)) { + Attribute a("a", attribute_datatype, 1, DataOrder::INCREASING_DATA); + REQUIRE(a.cell_val_num() == 1); + CHECK_NOTHROW(a.set_cell_val_num(1)); + CHECK(a.cell_val_num() == 1); + } + + const unsigned int invalid_cell_val_num = + GENERATE(0, 2, 5, 10, 100, 1000, constants::var_num); + + DYNAMIC_SECTION( + "Constructor: cell val num " + std::to_string(invalid_cell_val_num) + + " is invalid for increasing attribute with datatype " + + datatype_str(attribute_datatype)) { + CHECK_THROWS(Attribute( + "a", + attribute_datatype, + invalid_cell_val_num, + DataOrder::INCREASING_DATA)); + CHECK_THROWS(Attribute( + "a", + attribute_datatype, + true, + invalid_cell_val_num, + empty_pipeline, + empty_fill, + 0, + DataOrder::INCREASING_DATA, + std::optional())); + } + + DYNAMIC_SECTION( + "set_cell_val_num: cell val num " + std::to_string(invalid_cell_val_num) + + " is invalid for increasing attribute with datatype " + + datatype_str(attribute_datatype)) { + Attribute a("a", attribute_datatype, 1, DataOrder::INCREASING_DATA); + REQUIRE(a.cell_val_num() == 1); + REQUIRE_THROWS(a.set_cell_val_num(invalid_cell_val_num)); + CHECK(a.cell_val_num() == 1); + } +} From 0fafe92173197288defadaf753a5e0871409fe5c Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 6 Jun 2024 00:02:01 -0700 Subject: [PATCH 407/456] Add simplified C++23 `chunk_view` (#5035) This PR adds a class to provide a subset of the functionality of the C++23 `std::ranges::chunk_view`. This view divides an underlying view into a range of equal-sized chunks (the last one may be shorter than the others). A complete description of `chunk_view` can be found on cppreference.com, which also includes "exposition only" aspects that point to particular implementation approaches. We did not adopt those approaches in most cases, though the public API of the class matches the spec, for the functions that we did implement. This view will be used to partition the original input data from the user query into, well, chunks that will be individually sorted and then written to a TileDB array as a data block. Data blocks are of fixed and uniform byte size that will contain a chunk smaller than or equal to its byte limit. * The class assumes a` random_access_range`, which is what we have in our buffers that we are chunking, so I didn't try to generalize any further than that. * The class provides a complete iterator interface via `iterator_facade` as well as a complete view interface via `view_interface`. Aspects of those interfaces that rely on C++23 are pended. * Unit tests validate that `chunk_view` and its iterator meet the expected `std::ranges` concepts. * The unit tests do not yet test composition of this view with other views (our custom views as well as other C++20 views). This will be fairly extensive and the subject of its own PR. * One todo would be to create a variable chunk class view that chunks up the underlying view into variable-sized pieces, the better to fit into the data blocks. However, the data blocks will be fairly large, so there should not be too much internal fragmentation. We can always manage internal fragmentation by making the chunks some fraction of the block size, so even if there is some variation in chunk size, they can be well-packed into the data blocks. --- TYPE: IMPROVEMENT DESC: Implementation of a `chunk_view` class to provide a subset of C++23 chunk_view, suitable for supporting external sort. --------- Co-authored-by: Luc Rancourt --- .../tiledb/common/dag/utility/range_join.h | 2 +- .../dag/utility/test/compile_utils_main.cc | 2 +- tiledb/common/util/CMakeLists.txt | 1 + tiledb/common/util/alt_var_length_view.h | 4 +- tiledb/common/util/block_view.h | 43 ++ tiledb/common/util/detail/CMakeLists.txt | 30 ++ tiledb/common/util/{ => detail}/arrow_proxy.h | 2 +- .../util/{ => detail}/iterator_facade.h | 4 +- tiledb/common/util/detail/test/CMakeLists.txt | 33 ++ .../{ => detail}/test/unit_iterator_facade.cc | 2 +- tiledb/common/util/permutation_view.h | 4 +- tiledb/common/util/print_types.h | 47 ++- tiledb/common/util/proxy_sort.h | 2 +- tiledb/common/util/test/CMakeLists.txt | 53 ++- .../util/test/unit_alt_var_length_view.cc | 2 +- tiledb/common/util/test/unit_block_view.cc | 38 ++ .../common/util/test/unit_permutation_sort.cc | 10 +- .../common/util/test/unit_permutation_view.cc | 4 +- tiledb/common/util/test/unit_proxy_sort.cc | 2 +- tiledb/common/util/test/unit_sort_zip.cc | 6 +- .../common/util/test/unit_var_length_util.cc | 2 +- .../common/util/test/unit_var_length_view.cc | 2 +- tiledb/common/util/var_length_util.h | 2 +- tiledb/common/util/var_length_view.h | 4 +- tiledb/stdx/CMakeLists.txt | 1 + tiledb/stdx/DIRECTORY.md | 11 + tiledb/stdx/__ranges/CMakeLists.txt | 30 ++ tiledb/stdx/__ranges/chunk_view.h | 282 +++++++++++++ tiledb/stdx/__ranges/test/CMakeLists.txt | 39 ++ tiledb/stdx/__ranges/test/unit_chunk_view.cc | 389 ++++++++++++++++++ .../__ranges}/test/unit_zip_view.cc | 15 +- .../{common/util => stdx/__ranges}/zip_view.h | 21 +- tiledb/stdx/ranges | 47 +++ 33 files changed, 1064 insertions(+), 72 deletions(-) create mode 100644 tiledb/common/util/block_view.h create mode 100644 tiledb/common/util/detail/CMakeLists.txt rename tiledb/common/util/{ => detail}/arrow_proxy.h (97%) rename tiledb/common/util/{ => detail}/iterator_facade.h (99%) create mode 100644 tiledb/common/util/detail/test/CMakeLists.txt rename tiledb/common/util/{ => detail}/test/unit_iterator_facade.cc (99%) create mode 100644 tiledb/common/util/test/unit_block_view.cc create mode 100644 tiledb/stdx/DIRECTORY.md create mode 100644 tiledb/stdx/__ranges/CMakeLists.txt create mode 100644 tiledb/stdx/__ranges/chunk_view.h create mode 100644 tiledb/stdx/__ranges/test/CMakeLists.txt create mode 100644 tiledb/stdx/__ranges/test/unit_chunk_view.cc rename tiledb/{common/util => stdx/__ranges}/test/unit_zip_view.cc (96%) rename tiledb/{common/util => stdx/__ranges}/zip_view.h (96%) create mode 100644 tiledb/stdx/ranges diff --git a/experimental/tiledb/common/dag/utility/range_join.h b/experimental/tiledb/common/dag/utility/range_join.h index 5c0411d76fc2..ba6166dd789f 100644 --- a/experimental/tiledb/common/dag/utility/range_join.h +++ b/experimental/tiledb/common/dag/utility/range_join.h @@ -73,7 +73,7 @@ #include #include #include -#include "tiledb/common/util/arrow_proxy.h" +#include "tiledb/common/util/detail/arrow_proxy.h" #include "external/include/span/span.hpp" diff --git a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc index 3b57acf679d5..eafc4322116c 100644 --- a/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc +++ b/experimental/tiledb/common/dag/utility/test/compile_utils_main.cc @@ -31,7 +31,7 @@ #include "../range_join.h" #include "../spinlock.h" #include "../traits.h" -#include "tiledb/common/util/arrow_proxy.h" +#include "tiledb/common/util/detail/arrow_proxy.h" int main() { } diff --git a/tiledb/common/util/CMakeLists.txt b/tiledb/common/util/CMakeLists.txt index 15d75d4b447e..f37bd2951d48 100644 --- a/tiledb/common/util/CMakeLists.txt +++ b/tiledb/common/util/CMakeLists.txt @@ -27,4 +27,5 @@ include(common NO_POLICY_SCOPE) include(object_library) +add_subdirectory(detail) add_test_subdirectory() diff --git a/tiledb/common/util/alt_var_length_view.h b/tiledb/common/util/alt_var_length_view.h index bb16b7d41609..b80a80f957ff 100644 --- a/tiledb/common/util/alt_var_length_view.h +++ b/tiledb/common/util/alt_var_length_view.h @@ -1,5 +1,5 @@ /** - * @file alt_alt_var_length_view.h + * @file tiledb/common/util/alt_var_length_view.h * * @section LICENSE * @@ -56,7 +56,7 @@ #include -#include "iterator_facade.h" +#include "tiledb/common/util/detail/iterator_facade.h" /** * A view that splits a view into subranges of variable length, as delimited by diff --git a/tiledb/common/util/block_view.h b/tiledb/common/util/block_view.h new file mode 100644 index 000000000000..f4d9383af16a --- /dev/null +++ b/tiledb/common/util/block_view.h @@ -0,0 +1,43 @@ +/** + * @file tiledb/common/util/block_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements a zip view for zipping together a set of ranges. + * It is intended to implement the zip view as defined for C++23. From + * https://en.cppreference.com/w/cpp/ranges/zip_view: + * 1) A zip_view is a range adaptor that takes one or more views, and produces + * a view whose ith element is a tuple-like value consisting of the ith elements + * of all views. The size of produced view is the minimum of sizes of all + * adapted views. 2) zip is a customization point object that constructs a + * zip_view. + * + */ + +#ifndef TILEDB_BLOCK_VIEW_H +#define TILEDB_BLOCK_VIEW_H +#endif // TILEDB_BLOCK_VIEW_H diff --git a/tiledb/common/util/detail/CMakeLists.txt b/tiledb/common/util/detail/CMakeLists.txt new file mode 100644 index 000000000000..87855c545e98 --- /dev/null +++ b/tiledb/common/util/detail/CMakeLists.txt @@ -0,0 +1,30 @@ +# +# tiledb/util/detail/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +add_test_subdirectory() \ No newline at end of file diff --git a/tiledb/common/util/arrow_proxy.h b/tiledb/common/util/detail/arrow_proxy.h similarity index 97% rename from tiledb/common/util/arrow_proxy.h rename to tiledb/common/util/detail/arrow_proxy.h index 1591b2555b9d..c38129bbe801 100644 --- a/tiledb/common/util/arrow_proxy.h +++ b/tiledb/common/util/detail/arrow_proxy.h @@ -1,5 +1,5 @@ /** - * @file arrow_proxy.hpp + * @file tiledb/common/util/arrow_proxy.hpp * * @section LICENSE * diff --git a/tiledb/common/util/iterator_facade.h b/tiledb/common/util/detail/iterator_facade.h similarity index 99% rename from tiledb/common/util/iterator_facade.h rename to tiledb/common/util/detail/iterator_facade.h index 5131b7f3b5f1..a6334034e7ee 100644 --- a/tiledb/common/util/iterator_facade.h +++ b/tiledb/common/util/detail/iterator_facade.h @@ -1,5 +1,5 @@ /** - * @file iterator_facade.h + * @file tiledb/common/util/iterator_facade.h * * @section LICENSE * @@ -42,7 +42,7 @@ #include #include -#include "arrow_proxy.h" +#include "tiledb/common/util/detail/arrow_proxy.h" namespace detail { diff --git a/tiledb/common/util/detail/test/CMakeLists.txt b/tiledb/common/util/detail/test/CMakeLists.txt new file mode 100644 index 000000000000..c815cd54a4a4 --- /dev/null +++ b/tiledb/common/util/detail/test/CMakeLists.txt @@ -0,0 +1,33 @@ +# +# tiledb/util/detail/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test iterator_facade) + this_target_sources( + unit_iterator_facade.cc +) +conclude(unit_test) diff --git a/tiledb/common/util/test/unit_iterator_facade.cc b/tiledb/common/util/detail/test/unit_iterator_facade.cc similarity index 99% rename from tiledb/common/util/test/unit_iterator_facade.cc rename to tiledb/common/util/detail/test/unit_iterator_facade.cc index 4ceafc0e88d8..cd0f4e0b5537 100644 --- a/tiledb/common/util/test/unit_iterator_facade.cc +++ b/tiledb/common/util/detail/test/unit_iterator_facade.cc @@ -40,7 +40,7 @@ #include #include #include -#include "../iterator_facade.h" +#include "tiledb/common/util/detail/iterator_facade.h" TEST_CASE("iterator_facade: Null test", "[iterator_facade][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/permutation_view.h b/tiledb/common/util/permutation_view.h index 9fa2274c940e..44ef010486de 100644 --- a/tiledb/common/util/permutation_view.h +++ b/tiledb/common/util/permutation_view.h @@ -1,5 +1,5 @@ /** - * @file permutation_view.h + * @file tiledb/common/util/permutation_view.h * * @section LICENSE * @@ -50,8 +50,8 @@ #include #include -#include "iterator_facade.h" #include "proxy_sort.h" +#include "tiledb/common/util/detail/iterator_facade.h" /** * A view that creates a permutation of the underlying view, as determined by diff --git a/tiledb/common/util/print_types.h b/tiledb/common/util/print_types.h index e7d2161800e6..6222aa6ab2ce 100644 --- a/tiledb/common/util/print_types.h +++ b/tiledb/common/util/print_types.h @@ -1,5 +1,5 @@ /** - * @file print_types.h + * @file tiledb/common/util/print_types.h * * @section LICENSE * @@ -28,9 +28,50 @@ * @section DESCRIPTION * * This file implements a compile-time debugging utility for investigating - * the specific types of objects. + * the specific types of objects. It is the equivalent of a compile-time + * `printf`, but for types and is useful for debugging compile-time issues + * related to templates. It is intended for development use only and should + * not be included in production code. * - * Based on utility from NWGraph. Author Luke D'Alessandro. + * As a compile-time "printf" for types, it will generate a compiler error + * whose message will contain the expanded types of the variables passed to it. + * Note that the `print_types` takes *variable* as argument, not actual types + * or template parameters. The programmer will need to inspect the error + * message to determine the types of the variables. + * + * @example + * + * // See https://godbolt.org/z/3q341cs57 for running example + * #include + * #include "tiledb/common/util/print_types.h" + * + * template + * void foo(const S& s, const T& t) { + * // Note we pass s and t, not S and T + * print_types(s, t); + * } + * + * int main() { + * foo(1.0, std::vector>{{1, 2}, {3, 4}}); + * } + * + * This will produce an error message like the following (gcc 12.1): + * :18:31: error: invalid use of incomplete type 'struct + * print_types_t >, + * std::allocator > > > >' 18 | return + * print_types_t{}; + * + * Or like the following (clang 16.0): + * :18:10: error: implicit instantiation of undefined template + * 'print_types_t>>' return + * print_types_t{}; + * + * The types of the variable `s` and `t` are contained in the template + * arguments shown for `print_types`. In this case they are respectively + * `double` and `std::vector>`. + * + * This file is based on the `print_types` utility from NWGraph. + * Author Luke D'Alessandro. */ #ifndef TILEDB_PRINT_TYPES_H diff --git a/tiledb/common/util/proxy_sort.h b/tiledb/common/util/proxy_sort.h index 01fa5b240bb9..397df531c181 100644 --- a/tiledb/common/util/proxy_sort.h +++ b/tiledb/common/util/proxy_sort.h @@ -1,5 +1,5 @@ /** - * @file proxy_sort.h + * @file tiledb/common/util/proxy_sort.h * * @section LICENSE * diff --git a/tiledb/common/util/test/CMakeLists.txt b/tiledb/common/util/test/CMakeLists.txt index 471ad68ac893..9b00d92d26a8 100644 --- a/tiledb/common/util/test/CMakeLists.txt +++ b/tiledb/common/util/test/CMakeLists.txt @@ -26,56 +26,51 @@ include(unit_test) -commence(unit_test unit_alt_var_length_view) -this_target_sources( - unit_alt_var_length_view.cc +commence(unit_test alt_var_length_view) + this_target_sources( + unit_alt_var_length_view.cc ) conclude(unit_test) -commence(unit_test unit_iterator_facade) -this_target_sources( - unit_iterator_facade.cc +commence(unit_test permutation_sort) + this_target_sources( + unit_permutation_sort.cc ) conclude(unit_test) -commence(unit_test unit_permutation_sort) -this_target_sources( - unit_permutation_sort.cc +commence(unit_test permutation_view) + this_target_sources( + unit_permutation_view.cc ) conclude(unit_test) -commence(unit_test unit_permutation_view) -this_target_sources( - unit_permutation_view.cc +commence(unit_test proxy_sort) + this_target_sources( + unit_proxy_sort.cc ) conclude(unit_test) -commence(unit_test unit_proxy_sort) -this_target_sources( - unit_proxy_sort.cc +commence(unit_test sort_zip) + this_target_sources( + unit_sort_zip.cc ) conclude(unit_test) -commence(unit_test unit_sort_zip) -this_target_sources( - unit_sort_zip.cc +commence(unit_test var_length_util) + this_target_sources( + unit_var_length_util.cc ) conclude(unit_test) -commence(unit_test unit_var_length_util) -this_target_sources( - unit_var_length_util.cc -) -conclude(unit_test) - -commence(unit_test unit_var_length_view) -this_target_sources( +commence(unit_test var_length_view) + this_target_sources( unit_var_length_view.cc ) conclude(unit_test) -commence(unit_test unit_zip_view) -this_target_sources( - unit_zip_view.cc +commence(unit_test block_view) + this_target_sources( + unit_block_view.cc ) conclude(unit_test) + diff --git a/tiledb/common/util/test/unit_alt_var_length_view.cc b/tiledb/common/util/test/unit_alt_var_length_view.cc index dd0d75a37b35..50164071e2e1 100644 --- a/tiledb/common/util/test/unit_alt_var_length_view.cc +++ b/tiledb/common/util/test/unit_alt_var_length_view.cc @@ -33,7 +33,7 @@ #include #include #include -#include "../alt_var_length_view.h" +#include "tiledb/common/util/alt_var_length_view.h" TEST_CASE( "alt_var_length_view: Null test", "[alt_var_length_view][null_test]") { diff --git a/tiledb/common/util/test/unit_block_view.cc b/tiledb/common/util/test/unit_block_view.cc new file mode 100644 index 000000000000..dd800ee96c8a --- /dev/null +++ b/tiledb/common/util/test/unit_block_view.cc @@ -0,0 +1,38 @@ +/** + * @file unit_block_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the block_view class. + */ + +#include +#include "tiledb/common/util/block_view.h" + +TEST_CASE("block_view: null test", "[block_view][null test]") { + REQUIRE(true); +} diff --git a/tiledb/common/util/test/unit_permutation_sort.cc b/tiledb/common/util/test/unit_permutation_sort.cc index 41b374562901..23bf78abf15c 100644 --- a/tiledb/common/util/test/unit_permutation_sort.cc +++ b/tiledb/common/util/test/unit_permutation_sort.cc @@ -36,11 +36,11 @@ #include #include #include -#include "../alt_var_length_view.h" -#include "../permutation_view.h" -#include "../proxy_sort.h" -#include "../var_length_view.h" -#include "../zip_view.h" +#include "tiledb/common/util/alt_var_length_view.h" +#include "tiledb/common/util/permutation_view.h" +#include "tiledb/common/util/proxy_sort.h" +#include "tiledb/common/util/var_length_view.h" +#include "tiledb/stdx/__ranges/zip_view.h" TEST_CASE("permutation_sort: Null test", "[permutation_sort][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/test/unit_permutation_view.cc b/tiledb/common/util/test/unit_permutation_view.cc index bead37c87051..00a9b38805a7 100644 --- a/tiledb/common/util/test/unit_permutation_view.cc +++ b/tiledb/common/util/test/unit_permutation_view.cc @@ -33,8 +33,8 @@ #include #include #include -#include "../permutation_view.h" -#include "../var_length_view.h" +#include "tiledb/common/util/permutation_view.h" +#include "tiledb/common/util/var_length_view.h" TEST_CASE("permutation_view: Null test", "[permutation_view][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/test/unit_proxy_sort.cc b/tiledb/common/util/test/unit_proxy_sort.cc index 1704d7d0e9bf..8ddbbc24a68c 100644 --- a/tiledb/common/util/test/unit_proxy_sort.cc +++ b/tiledb/common/util/test/unit_proxy_sort.cc @@ -35,7 +35,7 @@ #include #include #include -#include "../proxy_sort.h" +#include "tiledb/common/util/proxy_sort.h" TEST_CASE("proxy_sort: Null test", "[proxy_sort][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/test/unit_sort_zip.cc b/tiledb/common/util/test/unit_sort_zip.cc index cc5f67187207..2f1c829be7ab 100644 --- a/tiledb/common/util/test/unit_sort_zip.cc +++ b/tiledb/common/util/test/unit_sort_zip.cc @@ -33,8 +33,8 @@ #include #include #include -#include "../alt_var_length_view.h" -#include "../zip_view.h" +#include "tiledb/common/util/alt_var_length_view.h" +#include "tiledb/stdx/__ranges/zip_view.h" TEST_CASE("sort_zip: Null test", "[zip_view][null_test]") { REQUIRE(true); @@ -207,7 +207,7 @@ TEST_CASE("sort_zip: mini sort zip view", "[zip_view]") { TEST_CASE("sort_zip: range sort zip view concepts", "[zip_view]") { using VI = std::ranges::iterator_t>; - using ZI = std::ranges::iterator_t>>; + using ZI = std::ranges::iterator_t>>; CHECK(std::forward_iterator); CHECK(std::indirectly_movable_storable); diff --git a/tiledb/common/util/test/unit_var_length_util.cc b/tiledb/common/util/test/unit_var_length_util.cc index a9489292c03f..fe54dee1947b 100644 --- a/tiledb/common/util/test/unit_var_length_util.cc +++ b/tiledb/common/util/test/unit_var_length_util.cc @@ -33,7 +33,7 @@ #include #include #include -#include "../var_length_util.h" +#include "tiledb/common/util/var_length_util.h" TEST_CASE("var_length_uti: Null test", "[var_length_util][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/test/unit_var_length_view.cc b/tiledb/common/util/test/unit_var_length_view.cc index e0d0e2026ae4..77bdb7780ebd 100644 --- a/tiledb/common/util/test/unit_var_length_view.cc +++ b/tiledb/common/util/test/unit_var_length_view.cc @@ -33,7 +33,7 @@ #include #include #include -#include "../var_length_view.h" +#include "tiledb/common/util/var_length_view.h" TEST_CASE("var_length_view: Null test", "[var_length_view][null_test]") { REQUIRE(true); diff --git a/tiledb/common/util/var_length_util.h b/tiledb/common/util/var_length_util.h index e53f0acad00f..1e08e25f5f74 100644 --- a/tiledb/common/util/var_length_util.h +++ b/tiledb/common/util/var_length_util.h @@ -1,5 +1,5 @@ /** - * @file var_length_util.h + * @file tiledb/common/util/var_length_util.h * * @section LICENSE * diff --git a/tiledb/common/util/var_length_view.h b/tiledb/common/util/var_length_view.h index 2054f1ef68bc..3dc2ce108ece 100644 --- a/tiledb/common/util/var_length_view.h +++ b/tiledb/common/util/var_length_view.h @@ -1,5 +1,5 @@ /** - * @file var_length_view.h + * @file tiledb/common/util/var_length_view.h * * @section LICENSE * @@ -47,7 +47,7 @@ #include -#include "iterator_facade.h" +#include "tiledb/common/util/detail/iterator_facade.h" /** * A view that splits a view into subranges of variable length, as delimited by diff --git a/tiledb/stdx/CMakeLists.txt b/tiledb/stdx/CMakeLists.txt index b5bc6bca4758..31080528e4a6 100644 --- a/tiledb/stdx/CMakeLists.txt +++ b/tiledb/stdx/CMakeLists.txt @@ -28,3 +28,4 @@ include(common NO_POLICY_SCOPE) add_test_subdirectory() add_subdirectory(synchronized_optional) +add_subdirectory(__ranges) diff --git a/tiledb/stdx/DIRECTORY.md b/tiledb/stdx/DIRECTORY.md new file mode 100644 index 000000000000..8b0a8141bcfd --- /dev/null +++ b/tiledb/stdx/DIRECTORY.md @@ -0,0 +1,11 @@ +## stdx + +This directory contains local implementations of C++ standard library components that have been introduced in recent versions of the standard, but that are not included in the standard library shipping with one or more of the compilers supported by libtiledb. + +### `ranges` + +There are two views standardized as part of C++23 that we provide a subset for: `zip_view` and `chunk_view`. The file `tiledb/stdx/ranges` is an include file (mimicking "`ranges`" in the standard library) that provides these two views. The implementations of these views are included in the `__ranges` subdirectory. (This naming convention mirrors that of LLVM.) The include directives for the implementations are surrounded by the appropriate `version` preprocessor directives so that once `zip_view` and/or `chunk_view` are available for a given compiler, the standard implementation will be used instead of our local implementation. + +### `thread` and `stop_token` + +These are copies of Josuttis's model implementation. \ No newline at end of file diff --git a/tiledb/stdx/__ranges/CMakeLists.txt b/tiledb/stdx/__ranges/CMakeLists.txt new file mode 100644 index 000000000000..14947b9244ee --- /dev/null +++ b/tiledb/stdx/__ranges/CMakeLists.txt @@ -0,0 +1,30 @@ +# +# tiledb/stdx/__ranges/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) +include(object_library) + +add_test_subdirectory() \ No newline at end of file diff --git a/tiledb/stdx/__ranges/chunk_view.h b/tiledb/stdx/__ranges/chunk_view.h new file mode 100644 index 000000000000..ba892d95b7b4 --- /dev/null +++ b/tiledb/stdx/__ranges/chunk_view.h @@ -0,0 +1,282 @@ +/** + * @file tiledb/stdx/__ranges/chunk_view.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Simplified implementation of a chunk view for C++23. This is a view + * that splits a view into subranges of uniform length, as given by the + * constructor argument `chunk_size`. The size of the last chunk may be less + * than or equal to `chunk_size`. The number of chunks is determined by + * `detail::div_ceil`. + * + * @todo Implement the full C++23 chunk view (or wait for C++23 or reference + * implementation without GPL) + */ + +#ifndef TILEDB_CHUNK_VIEW_H +#define TILEDB_CHUNK_VIEW_H + +#include +#include +#include "tiledb/common/util/detail/iterator_facade.h" + +namespace stdx::ranges { + +namespace detail { + +/** Helper template, API from cppreference, sans "__" prefix */ +template +constexpr I div_ceil(I num, I denom) { + // In cppreference: + // I r = num / denom; + // if (num % denom) + // ++r; + // return r; + return (num + denom - 1) / denom; // This should be a little faster +} +} // namespace detail + +/** + * A view that splits a view into subranges of uniform length, as delimited by + * a chunk size value. The resulting view is a range of subranges, each of which + * is a view into the original data range. The iterator over the chunk_view + * is a random_access_iterator that dereferences to a subrange of the data + * range. + * + * @tparam R Type of the data range, assumed to be a random access range. + * + * @todo R could be a view rather than a range. + */ +template +class chunk_view : public std::ranges::view_interface> { + // Forward reference of the iterator over the data range data + template + struct private_iterator; + + /** The type of the iterator over the chunked data range */ + using data_iterator_type = std::ranges::iterator_t; + + /** The type that can index into the chunked data range */ + using data_index_type = std::ranges::range_difference_t; + + /** The type dereferenced by the iterator is a subrange */ + using chunk_type = std::ranges::subrange; + + /** The type of the iterator over the chunk view */ + using chunk_iterator = private_iterator; + + /** The type of the iterator over the chunk view */ + using chunk_const_iterator = private_iterator; + + public: + /** Default constructor */ + chunk_view() = default; + chunk_view(const chunk_view&) = default; + chunk_view(chunk_view&&) = default; + chunk_view& operator=(const chunk_view&) = default; + chunk_view& operator=(chunk_view&&) = default; + + /** + * Constructor taking ranges for the data range, along with size of the + * chunks. This is the signature from cppreference, but having an argument + * type for `chunk_size` that could be negative does not seem right. + */ + chunk_view(R& data, std::ranges::range_difference_t chunk_size) + : data_begin_(std::ranges::begin(data)) + , data_end_(std::ranges::end(data)) + , chunk_size_(chunk_size) { + } + + /* + * Define the necessary members for view_interface. + */ + + /** Return iterator to the beginning of the chunk view */ + constexpr auto begin() { + return chunk_iterator(data_begin_, data_end_, chunk_size_); + } + + /** Return iterator to the end of the chunk view */ + constexpr auto end() { + return chunk_iterator(data_end_, data_end_, chunk_size_); + } + + /** Return const iterator to the beginning of the chunk view */ + constexpr auto begin() const { + return chunk_const_iterator(data_begin_, data_end_, chunk_size_); + } + + /** Return const iterator to the end of the chunk view */ + constexpr auto end() const { + return chunk_const_iterator(data_end_, data_end_, chunk_size_); + } + + /** Return const iterator to the beginning of the chunk view */ + constexpr auto cbegin() const { + return chunk_const_iterator(data_begin_, data_end_, chunk_size_); + } + + /** Return const iterator to the end of the chunk view */ + constexpr auto cend() const { + return chunk_const_iterator(data_end_, data_end_, chunk_size_); + } + + /** Return the number of chunks in the chunk view */ + constexpr auto size() const { + return detail::div_ceil(data_end_ - data_begin_, chunk_size_); + } + + private: + template + struct private_iterator : public iterator_facade> { + using value_type = V; + using reference_type = V&; + + /** Default constructor */ + private_iterator() = default; + private_iterator(const private_iterator&) = default; + private_iterator(private_iterator&&) = default; + private_iterator& operator=(const private_iterator&) = default; + private_iterator& operator=(private_iterator&&) = default; + + /** Primary constructor */ + private_iterator( + data_iterator_type data_begin, + data_iterator_type data_end, + data_index_type chunk_size) + : data_begin_(data_begin) + , data_end_(data_end) + , current_(data_begin) + , chunk_size_(chunk_size) { + } + + /************************************************************************* + * Functions needed for iterator_facade + * Here we just supply the minimum needed to make the iterator work + *************************************************************************/ + + /* + * The critical function for defining the iterator as it determines the + * value type, etc and also does the dereference. In the case of the + * chunk_view, the value type is a subrange of the data range, and + * the subrage return value must be a proxy object, which we return by + * value. This is fine, since we can't assign a subrange to another. + * We do, however, want to be able to modify the contents of the subrange. + */ + constexpr value_type dereference() const { + if (data_end_ - current_ < chunk_size_) { + return {current_, data_end_}; + } else { + return {current_, current_ + chunk_size_}; + } + } + + /** + * Advance the iterator by n. Note that we don't move past the end of the + * data range. + */ + constexpr auto& advance(data_index_type n) { + if (data_end_ - current_ < n * chunk_size_) { + current_ = data_end_; + } else { + current_ += n * chunk_size_; + } + return *this; + } + + /** Return the distance to another iterator */ + constexpr auto distance_to(const private_iterator& other) const { + return other.current_ - current_; + } + + /** Compare two iterators for equality */ + constexpr bool operator==(const private_iterator& other) const { + return chunk_size_ == other.chunk_size_ && current_ == other.current_; + } + + /** Flag to indicate that the iterator is not a single pass iterator */ + static const bool single_pass_iterator = false; + + /** + * Iterator to the beginning of the data range. This is a copy of the + * parent data_begin_. We maintain the copy to avoid an indirection through + * the parent pointer. + */ + data_iterator_type data_begin_; + + /** + * Iterator to the end of the data range. A copy of the parent + * data_end_. + */ + data_iterator_type data_end_; + + /** The current location of the iterator */ + data_iterator_type current_; + + /** The chunk size. Also cached from parent. */ + data_index_type chunk_size_; + }; + + /** The beginning of the data range */ + std::ranges::iterator_t data_begin_; + + /** The end of the data range */ + std::ranges::iterator_t data_end_; + + /** The number of chunks in the chunk_range */ + std::ranges::range_difference_t chunk_size_; +}; + +/** Deduction guide for chunk_view */ +template +chunk_view(R, I) -> chunk_view>; +} // namespace stdx::ranges +/** + * Define "chunk()" cpo for creating chunk views + */ +namespace _chunk { +struct _fn { + // @todo Should this take a viewable range? + // template + template + constexpr auto operator()(T&& t, I i) const { + return stdx::ranges::chunk_view{std::forward(t), i}; + } +}; +} // namespace _chunk +inline namespace _cpo { +inline constexpr auto chunk = _chunk::_fn{}; +} // namespace _cpo + +namespace stdx::views { +using _cpo::chunk; +} // namespace stdx::views + +namespace stdx::ranges::views { +using _cpo::chunk; +} // namespace stdx::ranges::views +#endif // TILEDB_CHUNK_VIEW_H diff --git a/tiledb/stdx/__ranges/test/CMakeLists.txt b/tiledb/stdx/__ranges/test/CMakeLists.txt new file mode 100644 index 000000000000..554e291c37a6 --- /dev/null +++ b/tiledb/stdx/__ranges/test/CMakeLists.txt @@ -0,0 +1,39 @@ +# +# tiledb/stdx/__ranges/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test chunk_view) +this_target_sources( + unit_chunk_view.cc +) +conclude(unit_test) + +commence(unit_test zip_view) +this_target_sources( + unit_zip_view.cc +) +conclude(unit_test) \ No newline at end of file diff --git a/tiledb/stdx/__ranges/test/unit_chunk_view.cc b/tiledb/stdx/__ranges/test/unit_chunk_view.cc new file mode 100644 index 000000000000..35afb0fe28fe --- /dev/null +++ b/tiledb/stdx/__ranges/test/unit_chunk_view.cc @@ -0,0 +1,389 @@ +/** + * @file tiledb/stdx/__ranges/test/unit_chunk_view.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the chunk_view class. + */ + +#include +#include "tiledb/stdx/__ranges/chunk_view.h" + +TEST_CASE("chunk_view: null test", "[chunk_view][null test]") { + REQUIRE(true); +} + +// Test that the chunk_view satisfies the expected concepts +TEST_CASE("chunk_view: Range concepts", "[chunk_view][concepts]") { + using test_type = stdx::ranges::chunk_view>; + + CHECK(std::ranges::range); + CHECK(!std::ranges::borrowed_range); + CHECK(std::ranges::sized_range); + CHECK(std::ranges::view); + CHECK(std::ranges::input_range); + CHECK(!std::ranges:: + output_range>); + CHECK(std::ranges::forward_range); + CHECK(std::ranges::bidirectional_range); + CHECK(std::ranges::random_access_range); + CHECK(!std::ranges::contiguous_range); + CHECK(std::ranges::common_range); + CHECK(std::ranges::viewable_range); + + CHECK(std::ranges::view); +} + +// Test that the chunk_view iterators satisfy the expected concepts +TEST_CASE("chunk_view: Iterator concepts", "[chunk_view][concepts]") { + using test_type = stdx::ranges::chunk_view>; + using test_type_iterator = std::ranges::iterator_t; + using test_type_const_iterator = std::ranges::iterator_t; + + CHECK(std::input_or_output_iterator); + CHECK(std::input_or_output_iterator); + CHECK(std::input_iterator); + CHECK(std::input_iterator); + CHECK(!std::output_iterator< + test_type_iterator, + std::ranges::range_value_t>); + CHECK(!std::output_iterator< + test_type_const_iterator, + std::ranges::range_value_t>); + CHECK(std::forward_iterator); + CHECK(std::forward_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::bidirectional_iterator); + CHECK(std::random_access_iterator); + CHECK(std::random_access_iterator); +} + +// Test that the chunk_view value_type satisfies the expected concepts +TEST_CASE("chunk_view: value_type concepts", "[chunk_view][concepts]") { + using test_type = stdx::ranges::chunk_view>; + CHECK(std::ranges::range); + + using test_iterator_type = std::ranges::iterator_t; + using test_iterator_value_type = std::iter_value_t; + using test_iterator_reference_type = + std::iter_reference_t; + + using range_value_type = std::ranges::range_value_t; + using range_reference_type = std::ranges::range_reference_t; + + CHECK(std::is_same_v); + CHECK(std::is_same_v); +} + +std::vector v10 = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; +std::vector v11 = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0}; +std::vector v12 = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}; +std::vector v13 = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0}; +std::vector v14 = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0}; +std::vector v15 = { + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 7.0, + 8.0, + 9.0, + 10.0, + 11.0, + 12.0, + 13.0, + 14.0, + 15.0}; +std::vector v16 = { + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 7.0, + 8.0, + 9.0, + 10.0, + 11.0, + 12.0, + 13.0, + 14.0, + 15.0, + 16.0}; + +// Simple test that a chunk_view can be constructed +TEST_CASE("chunk_view: Constructor", "[chunk_view]") { + [[maybe_unused]] auto c = _cpo::chunk(v10, 2); + [[maybe_unused]] auto d = _cpo::chunk(v10, 3); + [[maybe_unused]] auto e = _cpo::chunk(v11, 2); + [[maybe_unused]] auto f = _cpo::chunk(v11, 3); +} + +TEST_CASE("chunk_view: Constructor + size", "[chunk_view]") { + auto c = _cpo::chunk(v10, 2); + CHECK(c.size() == 5); + auto d = _cpo::chunk(v10, 3); + CHECK(d.size() == 4); + auto e = _cpo::chunk(v11, 2); // 2, 2, 2, 2, 2, 1 + CHECK(e.size() == 6); + auto f = _cpo::chunk(v11, 3); // 3, 3, 3, 2 + CHECK(f.size() == 4); + auto g = _cpo::chunk(v12, 2); + CHECK(g.size() == 6); + auto h = _cpo::chunk(v12, 3); + CHECK(h.size() == 4); + auto i = _cpo::chunk(v12, 4); + CHECK(i.size() == 3); +} + +TEST_CASE("chunk_view: Iterators begin and end", "[chunk_view]") { + for (auto&& v : {v10, v11, v12, v13, v14, v15, v16}) { + auto a = _cpo::chunk(v, 2); + auto b = a.begin(); + auto c = a.end(); + auto d = a.begin(); + auto e = a.end(); + CHECK(b != c); + CHECK(d != e); + CHECK(b == d); + CHECK(c == e); + } + + for (auto&& v : {v11, v12}) { + auto a = _cpo::chunk(v, 2); + auto b = a.begin(); + CHECK(b == a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b != a.end()); + + ++b; + CHECK(b != a.begin()); + CHECK(b == a.end()); + } +} + +TEST_CASE("chunk_view: Iterators ++ and --", "[chunk_view]") { + for (auto&& v : {v10, v11, v12, v13, v14, v15, v16}) { + auto a = _cpo::chunk(v, 2); + auto b = a.begin(); + [[maybe_unused]] auto c = a.end(); + auto d = a.begin(); + auto e = a.end(); + while (b != e) { + ++b; + d++; + CHECK(b == d); + } + } + + for (auto&& v : {v10, v11, v12, v13, v14, v15, v16}) { + auto a = _cpo::chunk(v, 2); + auto b = a.begin(); + [[maybe_unused]] auto c = a.end(); + auto d = a.begin(); + auto e = a.end(); + while (b != e) { + ++b; + --b; + b++; + d++; + d--; + ++d; + CHECK(b == d); + } + } +} + +TEST_CASE("chunk_view: Iterators + 1", "[chunk_view]") { + for (auto ch : {2, 3, 4, 9, 10, 11}) { + for (auto&& v : {v10, v11, v12, v13, v14, v15, v16}) { + auto a = _cpo::chunk(v, ch); + auto b = a.begin(); + [[maybe_unused]] auto c = a.end(); + auto d = a.begin(); + auto e = a.end(); + + while (b != e) { + auto x = b; + b += 1; + if (b == e) { + break; + } + CHECK(b != d); + CHECK(b == d + 1); + CHECK(b != d + 2); + d++; + CHECK(b == d); + CHECK(b == (x + 1)); + } + } + } +} + +TEST_CASE("chunk_view: Iterators - 1 - 1", "[chunk_view]") { + auto a = _cpo::chunk(v11, 3); + auto b = a.begin(); + auto c = b; + auto d = a.begin(); + auto e = a.end(); + + CHECK(b == c); + CHECK(c == b); + CHECK(b == d); + CHECK(d == b); + CHECK(c == d); + CHECK(c == c); + CHECK(d == c); + CHECK(c == a.begin()); + CHECK(b != e); + CHECK(b == c); + CHECK(c == b); + + auto x = b; + ++b; + auto y = b; + + CHECK(c != b); + CHECK(c == (b - 1)); + CHECK(b == (c + 1)); + CHECK(c == x); + CHECK(c == y - 1); + CHECK(y == (c + 1)); + CHECK(y == ((b - 1) + 1)); + + --b; + x = b; + + CHECK(c == x); + CHECK(x == b); + CHECK(c == b); + CHECK(b == c); + CHECK(b == a.begin()); +} + +TEST_CASE("chunk_view: Iterators - 1", "[chunk_view]") { + for (auto ch : {2, 3, 4, 9, 10, 11}) { + for (auto&& vx : {v10, v11, v12, v13, v14, v15, v16}) { + auto a = _cpo::chunk(vx, ch); + auto b = a.begin(); + [[maybe_unused]] auto c = a.end(); + auto d = a.begin(); + auto e = a.end(); + + { + auto dh = 9; + auto a = _cpo::chunk(vx, dh); + auto bb = a.begin(); + auto dd = a.begin(); + CHECK(bb == dd); + + ++bb; + + if (bb == e) { + break; + } + --bb; + if (!(bb == a.begin())) { + [[maybe_unused]] auto asdf = 1; // to allow debugging breakpoint + } + CHECK(bb == a.begin()); + } + + while (b != e) { + CHECK(b == d); + + ++b; + if (b == e) { + break; + } + --b; + + // CHECK(b == d); + b++; + d++; + CHECK(b == d); + } + } + } +} + +TEST_CASE("chunk_view: Iterators", "[chunk_view]") { + for (auto&& v : {v10, v11, v12, v13, v14, v15, v16}) { + for (size_t i = 1; i <= v.size(); ++i) { + auto a = _cpo::chunk(v, (long)i); + auto b = a.begin(); + CHECK(b->size() == (unsigned)i); + } + } +} + +TEST_CASE("chunk_view: Iterators values", "[chunk_view]") { + auto a = _cpo::chunk(v10, 5); + auto b = a.begin(); + + CHECK(b->size() == 5); + CHECK((*b)[0] == 1.0); + CHECK((*b)[1] == 2.0); + CHECK((*b)[2] == 3.0); + CHECK((*b)[3] == 4.0); + CHECK((*b)[4] == 5.0); + ++b; + CHECK(b->size() == 5); + CHECK((*b)[0] == 6.0); + CHECK((*b)[1] == 7.0); + CHECK((*b)[2] == 8.0); + CHECK((*b)[3] == 9.0); + CHECK((*b)[4] == 10.0); + ++b; + CHECK(b == a.end()); +} diff --git a/tiledb/common/util/test/unit_zip_view.cc b/tiledb/stdx/__ranges/test/unit_zip_view.cc similarity index 96% rename from tiledb/common/util/test/unit_zip_view.cc rename to tiledb/stdx/__ranges/test/unit_zip_view.cc index 6abc6b8d0992..b0148d001aaa 100644 --- a/tiledb/common/util/test/unit_zip_view.cc +++ b/tiledb/stdx/__ranges/test/unit_zip_view.cc @@ -1,5 +1,5 @@ /** - * @file unit_zip_view.cc + * @file tiledb/stdx/__ranges/test/unit_zip_view.cc * * @section LICENSE * @@ -34,8 +34,8 @@ #include #include #include -#include "../alt_var_length_view.h" -#include "../zip_view.h" +#include "tiledb/common/util/alt_var_length_view.h" +#include "tiledb/stdx/__ranges/zip_view.h" TEST_CASE("zip_view: Null test", "[zip_view][null_test]") { REQUIRE(true); @@ -66,7 +66,8 @@ TEST_CASE("zip_view: Should not copy", "[zip_view]") { /** Test that the zip_view type satisfies the expected range concepts */ TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { - using test_type = zip_view, std::vector>; + using test_type = + stdx::ranges::zip_view, std::vector>; CHECK(std::ranges::range); CHECK(!std::ranges::borrowed_range); @@ -86,7 +87,8 @@ TEST_CASE("zip_view: Range concepts", "[zip_view][concepts]") { /** Test that the zip_view iterator satisfies the expected concepts */ TEST_CASE("zip_view: Iterator concepts", "[zip_view][concepts]") { - using test_type = zip_view, std::vector>; + using test_type = + stdx::ranges::zip_view, std::vector>; using test_type_iterator = std::ranges::iterator_t; using test_type_const_iterator = std::ranges::iterator_t; // using test_type_const_iterator = decltype((const test_type){}.begin()); @@ -152,7 +154,8 @@ TEST_CASE("zip_view: Iterator concepts", "[zip_view][concepts]") { // Test that the zip_view value_type satisfies the expected concepts TEST_CASE("zip_view: value_type concepts", "[zip_view][concepts]") { - using test_type = zip_view, std::vector>; + using test_type = + stdx::ranges::zip_view, std::vector>; CHECK(std::ranges::range); using test_iterator_type = std::ranges::iterator_t; diff --git a/tiledb/common/util/zip_view.h b/tiledb/stdx/__ranges/zip_view.h similarity index 96% rename from tiledb/common/util/zip_view.h rename to tiledb/stdx/__ranges/zip_view.h index c2f453a7be88..de1272820c3b 100644 --- a/tiledb/common/util/zip_view.h +++ b/tiledb/stdx/__ranges/zip_view.h @@ -1,5 +1,5 @@ /** - * @file zip_view.h + * @file tiledb/stdx/__ranges/zip_view.h * * @section LICENSE * @@ -58,8 +58,8 @@ #define TILEDB_ZIP_VIEW_H #include -#include "iterator_facade.h" - +#include "tiledb/common/util/detail/iterator_facade.h" +namespace stdx::ranges { /* * @todo Should this take viewable ranges? E.g., * template @@ -239,7 +239,7 @@ class zip_view : public std::ranges::view_interface> { /** * Dereference the iterator -- the critical function for defining the - * iterator sinc the facade bases many type aliases and other functions + * iterator since the facade bases many type aliases and other functions * based on it and its signature */ reference dereference() const { @@ -323,7 +323,7 @@ class zip_view : public std::ranges::view_interface> { /** The ranges being zipped */ std::tuple ranges_; }; - +} // namespace stdx::ranges /** * Define "zip()" cpo for creating zip views */ @@ -333,14 +333,23 @@ struct _fn { // template template auto constexpr operator()(T&&... t) const { - return zip_view{std::forward(t)...}; + return stdx::ranges::zip_view{std::forward(t)...}; } }; } // namespace _zip + inline namespace _cpo { inline constexpr auto zip = _zip::_fn{}; } // namespace _cpo +namespace stdx::views { +using _cpo::zip; +} // namespace stdx::views + +namespace stdx::ranges::views { +using _cpo::zip; +} // namespace stdx::ranges::views + /** * Define "swap()" for tuples of references * diff --git a/tiledb/stdx/ranges b/tiledb/stdx/ranges new file mode 100644 index 000000000000..949b11b363c4 --- /dev/null +++ b/tiledb/stdx/ranges @@ -0,0 +1,47 @@ +/** + * @file tiledb/stdx/ranges + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file provides a single header for ranges / views that we have + * implemented in anticipation of their availability in C++23. + */ + +#ifndef TILEDB_STDX_RANGES +#define TILEDB_STDX_RANGES + +#include + +#if !defined(__cpp_lib_ranges_chunk) || __cpp_lib_ranges_chunk < 202202L +#include "tiledb/stdx/__ranges/chunk_view.h" +#endif + +#if !defined(__cpp_lib_ranges_zip) || __cpp_lib_ranges_zip < 202110L +#include "tiledb/stdx/__ranges/zip_view.h" +#endif + +#endif // TILEDB_STDX_RANGES From 804d7fb81ec86c6a135711b41fbc832797f31bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Thu, 6 Jun 2024 09:06:22 +0200 Subject: [PATCH 408/456] Libfaketime cmake version support. (#5049) `libfaketime` CMakeLists.txt doesn't support older versions of CMake. [sc-49009] --- TYPE: BUILD DESC: Backwards compatibility with older CMake versions for libfaketime. --- ports/libfaketime/CMakeLists.txt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ports/libfaketime/CMakeLists.txt b/ports/libfaketime/CMakeLists.txt index 08e01c4f6ebd..a5d6f20858c1 100644 --- a/ports/libfaketime/CMakeLists.txt +++ b/ports/libfaketime/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.20) project(faketime VERSION 0.9.10) @@ -21,18 +21,11 @@ target_include_directories(${PROJECT_NAME} file(GLOB_RECURSE PUBLIC_HEADERS "src/*.h") -target_sources( - ${PROJECT_NAME} - PUBLIC - FILE_SET HEADERS - BASE_DIRS src - FILES ${PUBLIC_HEADERS} -) +install(FILES ${PUBLIC_HEADERS} DESTINATION include) install(TARGETS ${PROJECT_NAME} - FILE_SET HEADERS RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) \ No newline at end of file +) From 2dc4fb7521de64c23b7e70655b522596706d28c9 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 6 Jun 2024 10:07:16 +0300 Subject: [PATCH 409/456] Remove statuses from `ls_with_sizes` and do not mask failures on POSIX. (#5043) [SC-23179](https://app.shortcut.com/tiledb-inc/story/23179) [SC-48851](https://app.shortcut.com/tiledb-inc/story/48851) [SC-48854](https://app.shortcut.com/tiledb-inc/story/48854) This PR updates the signature of `ls_with_sizes` in the `VFS` class and the specific VFS implementations[^1] to indicate errors by throwing exceptions instead of returning `Status`. On top of that, `Posix::ls_with_sizes` was updated to fail if `opendir` failed with an error code other than `ENOENT` (see also https://github.com/TileDB-Inc/TileDB/pull/5037#discussion_r1624775918). Also in `Posix::ls_with_sizes`, the pointer returned by `opendir` was updated to be placed inside a smart pointer, making sure it gets freed. This caused errors in the `find_head_api_violations.py` script, which were fixed. [^1]: HDFS was not updated to minimize risk. --- TYPE: BUG DESC: Do not mask failures when listing a directory fails on POSIX. --------- Co-authored-by: Luc Rancourt --- scripts/find_heap_api_violations.py | 7 +- test/src/unit-vfs.cc | 8 +-- test/support/src/vfs_helpers.cc | 14 ++-- tiledb/sm/array/array_directory.cc | 4 +- tiledb/sm/filesystem/azure.cc | 21 ++---- tiledb/sm/filesystem/azure.h | 3 +- tiledb/sm/filesystem/filesystem_base.h | 4 +- tiledb/sm/filesystem/gcs.cc | 22 +++---- tiledb/sm/filesystem/gcs.h | 3 +- tiledb/sm/filesystem/mem_filesystem.cc | 18 ++---- tiledb/sm/filesystem/mem_filesystem.h | 14 +++- tiledb/sm/filesystem/posix.cc | 88 ++++++++++++++++++++------ tiledb/sm/filesystem/posix.h | 4 +- tiledb/sm/filesystem/s3.cc | 30 ++++----- tiledb/sm/filesystem/s3.h | 7 +- tiledb/sm/filesystem/vfs.cc | 64 +++++++------------ tiledb/sm/filesystem/vfs.h | 3 +- tiledb/sm/filesystem/win.cc | 15 ++--- tiledb/sm/filesystem/win.h | 3 +- 19 files changed, 167 insertions(+), 165 deletions(-) diff --git a/scripts/find_heap_api_violations.py b/scripts/find_heap_api_violations.py index 1bc28dd84638..044dcad04469 100755 --- a/scripts/find_heap_api_violations.py +++ b/scripts/find_heap_api_violations.py @@ -92,8 +92,7 @@ # Contains per-file exceptions to violations of "make_shared". make_shared_exceptions = { - "*": ["tdb::make_shared"], - "*": ["tiledb::common::make_shared"], + "*": ["tdb::make_shared", "tiledb::common::make_shared"], } # Match C++ unique_ptr objects. @@ -103,7 +102,7 @@ unique_ptr_exceptions = { "*": ["tdb_unique_ptr", "tiledb_unique_ptr", "tdb::pmr::unique_ptr"], "zstd_compressor.h": ["std::unique_ptr ctx_;", "std::unique_ptr ctx_;"], - "posix.cc": ["static std::unique_ptr cwd_(getcwd(nullptr, 0), free);"], + "posix.cc": ["std::unique_ptr", "static std::unique_ptr cwd_(getcwd(nullptr, 0), free);"], "curl.h": ["std::unique_ptr"], "pmr.h": ["std::unique_ptr", "unique_ptr make_unique("], } @@ -148,7 +147,7 @@ class Violation: def iter_file_violations(file_path: str) -> Iterable[Violation]: - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: for line_num, line in enumerate(f): line = line.strip() for checker in violation_checkers: diff --git a/test/src/unit-vfs.cc b/test/src/unit-vfs.cc index d2117965ec8d..1999bd8e5d5f 100644 --- a/test/src/unit-vfs.cc +++ b/test/src/unit-vfs.cc @@ -339,9 +339,7 @@ TEMPLATE_LIST_TEST_CASE( std::string s = "abcdef"; require_tiledb_ok(vfs.write(ls_file, s.data(), s.size())); require_tiledb_ok(vfs.close_file(ls_file)); - auto&& [status, opt_children] = vfs.ls_with_sizes(ls_dir); - require_tiledb_ok(status); - auto children = opt_children.value(); + auto children = vfs.ls_with_sizes(ls_dir); #ifdef _WIN32 // Normalization only for Windows ls_file = URI(tiledb::sm::path_win::uri_from_path(ls_file.to_string())); @@ -599,9 +597,7 @@ TEST_CASE("VFS: test ls_with_sizes", "[vfs][ls-with-sizes]") { require_tiledb_ok(vfs_ls.write(URI(subdir_file), s2.data(), s2.size())); // List - auto&& [status, rv] = vfs_ls.ls_with_sizes(URI(dir)); - auto children = *rv; - require_tiledb_ok(status); + auto children = vfs_ls.ls_with_sizes(URI(dir)); #ifdef _WIN32 // Normalization only for Windows diff --git a/test/support/src/vfs_helpers.cc b/test/support/src/vfs_helpers.cc index 326cc583b5dc..2dcb9fda794c 100644 --- a/test/support/src/vfs_helpers.cc +++ b/test/support/src/vfs_helpers.cc @@ -451,12 +451,16 @@ VFSTestBase::VFSTestBase( } VFSTestBase::~VFSTestBase() { - if (vfs_.supports_uri_scheme(temp_dir_)) { - bool is_dir = false; - vfs_.is_dir(temp_dir_, &is_dir).ok(); - if (is_dir) { - vfs_.remove_dir(temp_dir_).ok(); + try { + if (vfs_.supports_uri_scheme(temp_dir_)) { + bool is_dir = false; + vfs_.is_dir(temp_dir_, &is_dir).ok(); + if (is_dir) { + vfs_.remove_dir(temp_dir_).ok(); + } } + } catch (const std::exception& e) { + // Suppress exceptions in destructors. } } diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 48c0a65e49e9..e031ed5f4a97 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -583,9 +583,7 @@ const std::set& ArrayDirectory::dir_names() { } std::vector ArrayDirectory::ls(const URI& uri) const { - auto&& [st, opt_dir_entries] = resources_.get().vfs().ls_with_sizes(uri); - throw_if_not_ok(st); - auto dir_entries = opt_dir_entries.value(); + auto dir_entries = resources_.get().vfs().ls_with_sizes(uri); auto dirs = dir_names(); std::vector uris; diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 6ffbbffa926e..7b45be982fae 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -585,32 +585,26 @@ Status Azure::ls( const int max_paths) const { assert(paths); - auto&& [st, entries] = ls_with_sizes(uri, delimiter, max_paths); - RETURN_NOT_OK(st); - - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(uri, delimiter, max_paths)) { paths->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> Azure::ls_with_sizes( +std::vector Azure::ls_with_sizes( const URI& uri, const std::string& delimiter, int max_paths) const { const auto& c = client(); const URI uri_dir = uri.add_trailing_slash(); if (!uri_dir.is_azure()) { - auto st = LOG_STATUS(Status_AzureError( - std::string("URI is not an Azure URI: " + uri_dir.to_string()))); - return {st, nullopt}; + throw AzureException("URI is not an Azure URI: " + uri_dir.to_string()); } std::string container_name; std::string blob_path; - RETURN_NOT_OK_TUPLE( - parse_azure_uri(uri_dir, &container_name, &blob_path), nullopt); + throw_if_not_ok(parse_azure_uri(uri_dir, &container_name, &blob_path)); auto container_client = c.GetBlobContainerClient(container_name); @@ -624,9 +618,8 @@ tuple>> Azure::ls_with_sizes( try { response = container_client.ListBlobsByHierarchy(delimiter, options); } catch (const ::Azure::Storage::StorageException& e) { - auto st = LOG_STATUS(Status_AzureError( - "List blobs failed on: " + uri_dir.to_string() + "; " + e.Message)); - return {st, nullopt}; + throw AzureException( + "List blobs failed on: " + uri_dir.to_string() + "; " + e.Message); } for (const auto& blob : response.Blobs) { @@ -648,7 +641,7 @@ tuple>> Azure::ls_with_sizes( options.ContinuationToken = response.NextPageToken; } while (options.ContinuationToken.HasValue()); - return {Status::Ok(), entries}; + return entries; } Status Azure::move_object(const URI& old_uri, const URI& new_uri) { diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index 985d6d290c5f..c1889da60dca 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -392,8 +392,7 @@ class Azure { * @param max_paths The maximum number of paths to be retrieved * @return A list of directory_entry objects */ - tuple>> - ls_with_sizes( + std::vector ls_with_sizes( const URI& uri, const std::string& delimiter = "/", int max_paths = -1) const; diff --git a/tiledb/sm/filesystem/filesystem_base.h b/tiledb/sm/filesystem/filesystem_base.h index ff54707376f3..cf8f53dc0981 100644 --- a/tiledb/sm/filesystem/filesystem_base.h +++ b/tiledb/sm/filesystem/filesystem_base.h @@ -115,8 +115,8 @@ class FilesystemBase { * @param parent The target directory to list. * @return All entries that are contained in the parent */ - virtual tuple>> - ls_with_sizes(const URI& parent) const = 0; + virtual std::vector ls_with_sizes( + const URI& parent) const = 0; /** * Renames a file. diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index 540399f8f0e7..e5381ff998a8 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -477,32 +477,27 @@ Status GCS::ls( const std::string& delimiter, const int max_paths) const { assert(paths); - auto&& [st, entries] = ls_with_sizes(uri, delimiter, max_paths); - RETURN_NOT_OK(st); - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(uri, delimiter, max_paths)) { paths->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> GCS::ls_with_sizes( +std::vector GCS::ls_with_sizes( const URI& uri, const std::string& delimiter, int max_paths) const { - RETURN_NOT_OK_TUPLE(init_client(), nullopt); + throw_if_not_ok(init_client()); const URI uri_dir = uri.add_trailing_slash(); if (!uri_dir.is_gcs()) { - auto st = LOG_STATUS(Status_GCSError( - std::string("URI is not a GCS URI: " + uri_dir.to_string()))); - return {st, nullopt}; + throw GCSException("URI is not a GCS URI: " + uri_dir.to_string()); } std::string bucket_name; std::string object_path; - RETURN_NOT_OK_TUPLE( - parse_gcs_uri(uri_dir, &bucket_name, &object_path), nullopt); + throw_if_not_ok(parse_gcs_uri(uri_dir, &bucket_name, &object_path)); std::vector entries; google::cloud::storage::Prefix prefix_option(object_path); @@ -515,10 +510,9 @@ tuple>> GCS::ls_with_sizes( if (!object_metadata.ok()) { const google::cloud::Status status = object_metadata.status(); - auto st = LOG_STATUS(Status_GCSError(std::string( + throw GCSException( "List objects failed on: " + uri.to_string() + " (" + - status.message() + ")"))); - return {st, nullopt}; + status.message() + ")"); } if (entries.size() >= static_cast(max_paths)) { @@ -549,7 +543,7 @@ tuple>> GCS::ls_with_sizes( } } - return {Status::Ok(), entries}; + return entries; } LsObjects GCS::ls_filtered_impl( diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index f3b9ab294356..a89c46a21c96 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -268,8 +268,7 @@ class GCS { * @param max_paths The maximum number of paths to be retrieved * @return A list of directory_entry objects */ - tuple>> - ls_with_sizes( + std::vector ls_with_sizes( const URI& uri, const std::string& delimiter = "/", int max_paths = -1) const; diff --git a/tiledb/sm/filesystem/mem_filesystem.cc b/tiledb/sm/filesystem/mem_filesystem.cc index 329f262ed16b..2e5f97d4720a 100644 --- a/tiledb/sm/filesystem/mem_filesystem.cc +++ b/tiledb/sm/filesystem/mem_filesystem.cc @@ -374,18 +374,16 @@ bool MemFilesystem::is_file(const std::string& path) const { Status MemFilesystem::ls( const std::string& path, std::vector* const paths) const { assert(paths); - auto&& [st, entries] = ls_with_sizes(URI("mem://" + path)); - RETURN_NOT_OK(st); - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(URI("mem://" + path))) { paths->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> -MemFilesystem::ls_with_sizes(const URI& path) const { +std::vector MemFilesystem::ls_with_sizes( + const URI& path) const { auto abspath = path.to_path(); std::vector tokens = tokenize(abspath); @@ -398,9 +396,7 @@ MemFilesystem::ls_with_sizes(const URI& path) const { dir = dir + token + "/"; if (cur->children_.count(token) != 1) { - auto st = LOG_STATUS(Status_MemFSError( - std::string("Unable to list on non-existent path ") + abspath)); - return {st, nullopt}; + throw MemFSException("Unable to list on non-existent path " + abspath); } cur = cur->children_[token].get(); @@ -410,11 +406,9 @@ MemFilesystem::ls_with_sizes(const URI& path) const { assert(cur_lock.owns_lock()); assert(cur_lock.mutex() == &cur->mutex_); auto&& [st, entries] = cur->ls(dir); - if (!st.ok()) { - return {st, nullopt}; - } + throw_if_not_ok(st); - return {Status::Ok(), entries}; + return *entries; } Status MemFilesystem::move( diff --git a/tiledb/sm/filesystem/mem_filesystem.h b/tiledb/sm/filesystem/mem_filesystem.h index dec513176f81..c03e1a2305df 100644 --- a/tiledb/sm/filesystem/mem_filesystem.h +++ b/tiledb/sm/filesystem/mem_filesystem.h @@ -37,6 +37,7 @@ #include #include +#include "tiledb/common/exception/exception.h" #include "tiledb/common/macros.h" #include "tiledb/common/status.h" @@ -52,6 +53,14 @@ namespace sm { class URI; +/** Class for MemFS status exceptions. */ +class MemFSException : public StatusException { + public: + explicit MemFSException(const std::string& msg) + : StatusException("MemFS", msg) { + } +}; + /** * The in-memory filesystem. * @@ -139,10 +148,9 @@ class MemFilesystem { * Lists files and files information under path * * @param path The parent path to list sub-paths - * @return Status + * @return A list of directory_entry objects */ - tuple>> - ls_with_sizes(const URI& path) const; + std::vector ls_with_sizes(const URI& path) const; /** * Move a given filesystem path. diff --git a/tiledb/sm/filesystem/posix.cc b/tiledb/sm/filesystem/posix.cc index 4b8011f2510c..6e0b678941b3 100644 --- a/tiledb/sm/filesystem/posix.cc +++ b/tiledb/sm/filesystem/posix.cc @@ -58,6 +58,69 @@ using filesystem::directory_entry; namespace tiledb::sm { +/** + * A class that wraps a POSIX DIR* and closes it when it goes out of scope. + */ +class PosixDIR { + public: + DISABLE_COPY_AND_COPY_ASSIGN(PosixDIR); + DISABLE_MOVE_AND_MOVE_ASSIGN(PosixDIR); + + /** Destructor. */ + ~PosixDIR() { + if (dir_.has_value() && dir_.value() != nullptr) { + // The only possible error is EBADF, which should not happen here. + [[maybe_unused]] auto status = closedir(dir_.value()); + assert(status == 0); + } + } + + /** Returns the wrapped directory pointer. */ + DIR* get() const { + return dir_.value(); + } + + /** Returns if the dir is empty. */ + bool empty() { + return !dir_.has_value(); + } + + /** + * Opens a directory and returns a UniqueDIR if it exists. + * + * @param path The path to the directory to open. + * @return A UniqueDIR if the directory was opened successfully, or nullopt if + * the directory does not exist. + * @throws IOError an error occurs while opening the directory. + */ + static PosixDIR open(const std::string& path) { + DIR* dir = opendir(path.c_str()); + if (dir == nullptr) { + auto last_error = errno; + if (last_error == ENOENT) { + return {}; + } + throw IOError( + std::string("Cannot open directory; ") + strerror(last_error)); + } + return {dir}; + } + + private: + /** + * Internal constructor that gets called from the `open` method. + * + * @param dir The directory pointer to wrap. + */ + PosixDIR(optional dir = nullopt) + : dir_(dir) { + assert(dir != nullptr); + } + + /** The wrapped directory pointer. */ + optional dir_; +}; + Posix::Posix(const Config& config) { // Initialize member variables with posix config parameters. @@ -305,18 +368,17 @@ void Posix::write( } } -tuple>> Posix::ls_with_sizes( - const URI& uri) const { +std::vector Posix::ls_with_sizes(const URI& uri) const { std::string path = uri.to_path(); struct dirent* next_path = nullptr; - DIR* dir = opendir(path.c_str()); - if (dir == nullptr) { - return {Status::Ok(), std::vector{}}; + auto dir = PosixDIR::open(path); + if (dir.empty()) { + return {}; } std::vector entries; - while ((next_path = readdir(dir)) != nullptr) { + while ((next_path = readdir(dir.get())) != nullptr) { if (!strcmp(next_path->d_name, ".") || !strcmp(next_path->d_name, "..")) continue; std::string abspath = path + "/" + next_path->d_name; @@ -333,22 +395,12 @@ tuple>> Posix::ls_with_sizes( entries.emplace_back(abspath, size, false); } } - // close parent directory - if (closedir(dir) != 0) { - auto st = LOG_STATUS(Status_IOError( - std::string("Cannot close parent directory; ") + strerror(errno))); - return {st, nullopt}; - } - return {Status::Ok(), entries}; + return entries; } Status Posix::ls( const std::string& path, std::vector* paths) const { - auto&& [st, entries] = ls_with_sizes(URI(path)); - - RETURN_NOT_OK(st); - - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(URI(path))) { paths->emplace_back(fs.path().native()); } diff --git a/tiledb/sm/filesystem/posix.h b/tiledb/sm/filesystem/posix.h index a97e1a0f678f..277735241ad8 100644 --- a/tiledb/sm/filesystem/posix.h +++ b/tiledb/sm/filesystem/posix.h @@ -260,8 +260,8 @@ class Posix : public FilesystemBase { * @param uri The parent path to list sub-paths. * @return A list of directory_entry objects */ - tuple>> - ls_with_sizes(const URI& uri) const override; + std::vector ls_with_sizes( + const URI& uri) const override; /** * Lists objects and object information that start with `prefix`, invoking diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index 708716f8891f..b7cc0f31d81f 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -588,8 +588,7 @@ void S3::write( } } -tuple>> S3::ls_with_sizes( - const URI& parent) const { +std::vector S3::ls_with_sizes(const URI& parent) const { return ls_with_sizes(parent, "/", -1); } @@ -868,27 +867,22 @@ Status S3::ls( std::vector* paths, const std::string& delimiter, int max_paths) const { - auto&& [st, entries] = ls_with_sizes(prefix, delimiter, max_paths); - RETURN_NOT_OK(st); - - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(prefix, delimiter, max_paths)) { paths->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> S3::ls_with_sizes( +std::vector S3::ls_with_sizes( const URI& prefix, const std::string& delimiter, int max_paths) const { - RETURN_NOT_OK_TUPLE(init_client(), nullopt); + throw_if_not_ok(init_client()); const auto prefix_dir = prefix.add_trailing_slash(); auto prefix_str = prefix_dir.to_string(); if (!prefix_dir.is_s3()) { - auto st = LOG_STATUS( - Status_S3Error(std::string("URI is not an S3 URI: " + prefix_str))); - return {st, nullopt}; + throw S3Exception("URI is not an S3 URI: " + prefix_str); } Aws::Http::URI aws_uri = prefix_str.c_str(); @@ -914,11 +908,10 @@ tuple>> S3::ls_with_sizes( auto list_objects_outcome = client_->ListObjectsV2(list_objects_request); if (!list_objects_outcome.IsSuccess()) { - auto st = LOG_STATUS(Status_S3Error( + throw S3Exception( std::string("Error while listing with prefix '") + prefix_str + "' and delimiter '" + delimiter + "'" + - outcome_error_message(list_objects_outcome))); - return {st, nullopt}; + outcome_error_message(list_objects_outcome)); } for (const auto& object : list_objects_outcome.GetResult().GetContents()) { @@ -946,16 +939,15 @@ tuple>> S3::ls_with_sizes( Aws::String next_marker = list_objects_outcome.GetResult().GetNextContinuationToken(); if (next_marker.empty()) { - auto st = - LOG_STATUS(Status_S3Error("Failed to retrieve next continuation " - "token for ListObjectsV2 request.")); - return {st, nullopt}; + throw S3Exception( + "Failed to retrieve next continuation " + "token for ListObjectsV2 request."); } list_objects_request.SetContinuationToken(std::move(next_marker)); } } - return {Status::Ok(), entries}; + return entries; } Status S3::move_object(const URI& old_uri, const URI& new_uri) const { diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 8c074479aee2..fbb2d18c12a4 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -791,8 +791,7 @@ class S3 : FilesystemBase { * @param parent The target directory to list. * @return All entries that are contained in the parent */ - tuple>> ls_with_sizes( - const URI& parent) const override; + std::vector ls_with_sizes(const URI& parent) const override; /** * Disconnects a S3 client. @@ -889,9 +888,9 @@ class S3 : FilesystemBase { * @param prefix The parent path to list sub-paths. * @param delimiter The uri is truncated to the first delimiter. * @param max_paths The maximum number of paths to be retrieved. - * @return Status tuple where second is a list of directory_entry objects. + * @return A list of directory_entry objects. */ - tuple>> ls_with_sizes( + std::vector ls_with_sizes( const URI& prefix, const std::string& delimiter, int max_paths) const; /** diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 3b3295731922..3b74fcd3a871 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -222,10 +222,8 @@ Status VFS::dir_size(const URI& dir_name, uint64_t* dir_size) const { do { auto uri = to_ls.front(); to_ls.pop_front(); - auto&& [st, children] = ls_with_sizes(uri); - RETURN_NOT_OK(st); - for (const auto& child : *children) { + for (const auto& child : ls_with_sizes(uri)) { if (!child.is_directory()) { *dir_size += child.file_size(); } else { @@ -703,90 +701,74 @@ Status VFS::is_bucket(const URI& uri, bool* is_bucket) const { Status VFS::ls(const URI& parent, std::vector* uris) const { stats_->add_counter("ls_num", 1); - auto&& [st, entries] = ls_with_sizes(parent); - RETURN_NOT_OK(st); - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(parent)) { uris->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> VFS::ls_with_sizes( - const URI& parent) const { +std::vector VFS::ls_with_sizes(const URI& parent) const { // Noop if `parent` is not a directory, do not error out. // For S3, GCS and Azure, `ls` on a non-directory will just // return an empty `uris` vector. if (!(parent.is_s3() || parent.is_gcs() || parent.is_azure())) { bool flag = false; - RETURN_NOT_OK_TUPLE(this->is_dir(parent, &flag), nullopt); + throw_if_not_ok(this->is_dir(parent, &flag)); if (!flag) { - return {Status::Ok(), std::vector()}; + return {}; } } - optional> entries; + std::vector entries; if (parent.is_file()) { #ifdef _WIN32 - Status st; - std::tie(st, entries) = win_.ls_with_sizes(parent); + entries = win_.ls_with_sizes(parent); #else - Status st; - std::tie(st, entries) = posix_.ls_with_sizes(parent); + entries = posix_.ls_with_sizes(parent); #endif - RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_s3()) { #ifdef HAVE_S3 - Status st; - std::tie(st, entries) = s3().ls_with_sizes(parent); + entries = s3().ls_with_sizes(parent); #else - auto st = Status_VFSError("TileDB was built without S3 support"); + throw BuiltWithout("S3"); #endif - RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_azure()) { #ifdef HAVE_AZURE - Status st; - std::tie(st, entries) = azure_.ls_with_sizes(parent); + entries = azure_.ls_with_sizes(parent); #else - auto st = Status_VFSError("TileDB was built without Azure support"); + throw BuiltWithout("Azure"); #endif - RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_gcs()) { #ifdef HAVE_GCS - Status st; - std::tie(st, entries) = gcs_.ls_with_sizes(parent); + entries = gcs_.ls_with_sizes(parent); #else - auto st = Status_VFSError("TileDB was built without GCS support"); + throw BuiltWithout("GCS"); #endif - RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_hdfs()) { #ifdef HAVE_HDFS - Status st; - std::tie(st, entries) = hdfs_->ls_with_sizes(parent); + auto&& [st, entries_optional] = hdfs_->ls_with_sizes(parent); + throw_if_not_ok(st); + entries = *std::move(entries_optional); #else - auto st = Status_VFSError("TileDB was built without HDFS support"); + throw BuiltWithout("HDFS"); #endif - RETURN_NOT_OK_TUPLE(st, nullopt); } else if (parent.is_memfs()) { - Status st; - std::tie(st, entries) = - memfs_.ls_with_sizes(URI("mem://" + parent.to_path())); - RETURN_NOT_OK_TUPLE(st, nullopt); + entries = memfs_.ls_with_sizes(URI("mem://" + parent.to_path())); } else { - auto st = Status_VFSError("Unsupported URI scheme: " + parent.to_string()); - return {st, std::nullopt}; + throw UnsupportedURI(parent.to_string()); } parallel_sort( compute_tp_, - entries->begin(), - entries->end(), + entries.begin(), + entries.end(), [](const directory_entry& l, const directory_entry& r) { return l.path().native() < r.path().native(); }); - return {Status::Ok(), entries}; + return entries; } Status VFS::move_file(const URI& old_uri, const URI& new_uri) { diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index a327b77bc013..b3af46c60503 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -506,8 +506,7 @@ class VFS : private VFSBase, protected S3_within_VFS { * @param parent The target directory to list. * @return All entries that are contained in the parent */ - tuple>> ls_with_sizes( - const URI& parent) const; + std::vector ls_with_sizes(const URI& parent) const; /** * Lists objects and object information that start with `prefix`, invoking diff --git a/tiledb/sm/filesystem/win.cc b/tiledb/sm/filesystem/win.cc index 265768f5fd68..f3ead73063a7 100644 --- a/tiledb/sm/filesystem/win.cc +++ b/tiledb/sm/filesystem/win.cc @@ -45,6 +45,7 @@ #include #include +#include "filesystem_base.h" #include "path_win.h" #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" @@ -334,18 +335,14 @@ bool Win::is_file(const std::string& path) const { } Status Win::ls(const std::string& path, std::vector* paths) const { - auto&& [st, entries] = ls_with_sizes(URI(path)); - RETURN_NOT_OK(st); - - for (auto& fs : *entries) { + for (auto& fs : ls_with_sizes(URI(path))) { paths->emplace_back(fs.path().native()); } return Status::Ok(); } -tuple>> Win::ls_with_sizes( - const URI& uri) const { +std::vector Win::ls_with_sizes(const URI& uri) const { auto path = uri.to_path(); bool ends_with_slash = path.length() > 0 && path[path.length() - 1] == '\\'; const std::string glob = path + (ends_with_slash ? "*" : "\\*"); @@ -389,7 +386,7 @@ tuple>> Win::ls_with_sizes( } FindClose(find_h); - return {Status::Ok(), entries}; + return entries; err: auto gle = GetLastError(); @@ -401,11 +398,9 @@ tuple>> Win::ls_with_sizes( offender.append(" "); offender.append(what_call); } - std::string errmsg( + throw IOError( "Failed to list directory \"" + path + "\" " + get_last_error_msg(gle, offender.c_str())); - auto st = LOG_STATUS(Status_IOError(errmsg)); - return {st, nullopt}; } Status Win::move_path( diff --git a/tiledb/sm/filesystem/win.h b/tiledb/sm/filesystem/win.h index e5d3ded22069..a53c1c038363 100644 --- a/tiledb/sm/filesystem/win.h +++ b/tiledb/sm/filesystem/win.h @@ -161,8 +161,7 @@ class Win { * @param path The parent path to list sub-paths. * @return A list of directory_entry objects */ - tuple>> - ls_with_sizes(const URI& path) const; + std::vector ls_with_sizes(const URI& path) const; /** * Lists objects and object information that start with `prefix`, invoking From bc4ec51a689337a16302d94f6ec40ab8809704dd Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:21:02 +0300 Subject: [PATCH 410/456] Add tests for partial tile offsets loading. (#5004) Adding complete tests for the partial tile offset loading feature. Remodeled one of the existing tests [sc-46969] --- TYPE: NO_HISTORY DESC: Add tests for partial tile offsets loading. --- .../unit-sparse-unordered-with-dups-reader.cc | 208 +++++++++++++----- 1 file changed, 156 insertions(+), 52 deletions(-) diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 56f55ffb7d2b..470bbfb468aa 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -68,6 +68,7 @@ struct CSparseUnorderedWithDupsFx { std::string partial_tile_offsets_loading_; void create_default_array_1d(); + void create_large_domain_array_1d(); void create_default_array_1d_string(int tile_extent = 2, int capacity = 2); void write_1d_fragment( int* coords, uint64_t* coords_size, int* data, uint64_t* data_size); @@ -236,6 +237,27 @@ void CSparseUnorderedWithDupsFx::create_default_array_1d() { true); // allows dups. } +void CSparseUnorderedWithDupsFx::create_large_domain_array_1d() { + int domain[] = {1, 20000}; + int tile_extent = 2; + create_array( + ctx_, + array_name_, + TILEDB_SPARSE, + {"d"}, + {TILEDB_INT32}, + {domain}, + {&tile_extent}, + {"a"}, + {TILEDB_INT32}, + {1}, + {tiledb::test::Compressor(TILEDB_FILTER_NONE, -1)}, + TILEDB_ROW_MAJOR, + TILEDB_ROW_MAJOR, + 2, + true); // allows dups. +} + void CSparseUnorderedWithDupsFx::create_default_array_1d_string( int tile_extent, int capacity) { int domain[] = {1, 20}; @@ -914,76 +936,158 @@ TEST_CASE_METHOD( TEST_CASE_METHOD( CSparseUnorderedWithDupsFx, - "Sparse unordered with dups reader: tile offsets forcing multiple " - "iterations", + "Sparse unordered with dups reader: tile offsets partial loading", "[sparse-unordered-with-dups][tile-offsets][multiple-iterations]") { - bool set_subarray = GENERATE(true, false); + bool enable_partial_tile_offsets_loading = GENERATE(true, false); // Create default array. reset_config(); - create_default_array_1d(); - - // Write two fragments. - std::vector coords(100); - std::iota(coords.begin(), coords.end(), 1); - uint64_t coords_size = coords.size() * sizeof(int); + create_large_domain_array_1d(); + bool one_frag = false; - std::vector data(100); - std::iota(data.begin(), data.end(), 1); - uint64_t data_size = data.size() * sizeof(int); + SECTION("- One fragment") { + one_frag = true; + } + SECTION("- Multiple fragments") { + one_frag = false; + } - write_1d_fragment(coords.data(), &coords_size, data.data(), &data_size); + // Write fragments. + if (one_frag) { + std::vector coords(100); + std::iota(coords.begin(), coords.end(), 1); + uint64_t coords_size = coords.size() * sizeof(int); - std::vector coords2(100); - std::iota(coords2.begin(), coords2.end(), 101); - uint64_t coords2_size = coords.size() * sizeof(int); + std::vector data(100); + std::iota(data.begin(), data.end(), 1); + uint64_t data_size = data.size() * sizeof(int); - std::vector data2(100); - std::iota(data2.begin(), data2.end(), 101); - uint64_t data2_size = data.size() * sizeof(int); - write_1d_fragment(coords2.data(), &coords2_size, data2.data(), &data2_size); + write_1d_fragment(coords.data(), &coords_size, data.data(), &data_size); + } else { + std::vector coords(100); + std::iota(coords.begin(), coords.end(), 1); + uint64_t coords_size = coords.size() * sizeof(int); + + std::vector data(100); + std::iota(data.begin(), data.end(), 1); + uint64_t data_size = data.size() * sizeof(int); + + write_1d_fragment(coords.data(), &coords_size, data.data(), &data_size); + + std::vector coords2(1000); + std::iota(coords2.begin(), coords2.end(), 101); + uint64_t coords2_size = coords2.size() * sizeof(int); + + std::vector data2(1000); + std::iota(data2.begin(), data2.end(), 101); + uint64_t data2_size = data2.size() * sizeof(int); + write_1d_fragment(coords2.data(), &coords2_size, data2.data(), &data2_size); + + std::vector coords3(5000); + std::iota(coords3.begin(), coords3.end(), 1101); + uint64_t coords3_size = coords3.size() * sizeof(int); + + std::vector data3(5000); + std::iota(data3.begin(), data3.end(), 1101); + uint64_t data3_size = data3.size() * sizeof(int); + write_1d_fragment(coords3.data(), &coords3_size, data3.data(), &data3_size); + + std::vector coords4(10000); + std::iota(coords4.begin(), coords4.end(), 6101); + uint64_t coords4_size = coords4.size() * sizeof(int); + + std::vector data4(10000); + std::iota(data4.begin(), data4.end(), 6101); + uint64_t data4_size = data4.size() * sizeof(int); + write_1d_fragment(coords4.data(), &coords4_size, data4.data(), &data4_size); + } - total_budget_ = "1900000"; - tile_upper_memory_limit_ = "100000"; - ratio_array_data_ = set_subarray ? "0.003" : "0.002"; - partial_tile_offsets_loading_ = "true"; + total_budget_ = "3000000"; + ratio_array_data_ = "0.002"; + partial_tile_offsets_loading_ = + enable_partial_tile_offsets_loading ? "true" : "false"; update_config(); tiledb_array_t* array = nullptr; tiledb_query_t* query = nullptr; - // Try to read. - int coords_r[200]; - int data_r[200]; + int coords_r[16100]; + int data_r[16100]; uint64_t coords_r_size = sizeof(coords_r); uint64_t data_r_size = sizeof(data_r); - auto rc = read( - set_subarray, - 0, - coords_r, - &coords_r_size, - data_r, - &data_r_size, - &query, - &array); - CHECK(rc == TILEDB_OK); + auto rc = 0; + + // Case 1: Read only one frag. Should be ok for both cases of partial tile + // loading Case 2: Read multiple fragments with partial tile offset loading. + // Should be ok + if (enable_partial_tile_offsets_loading || one_frag) { + rc = read( + false, + 0, + coords_r, + &coords_r_size, + data_r, + &data_r_size, + &query, + &array); + CHECK(rc == TILEDB_OK); - // Validate the results. - for (int i = 0; i < 200; i++) { - CHECK(coords_r[i] == i + 1); - CHECK(data_r[i] == i + 1); - } + // Validate the results. + int elements_to_check; + if (one_frag) { + elements_to_check = 100; + } else { + elements_to_check = 16100; + } - // Check the internal loop count against expected value. - auto stats = - ((SparseUnorderedWithDupsReader*)query->query_->strategy()) - ->stats(); - REQUIRE(stats != nullptr); - auto counters = stats->counters(); - REQUIRE(counters != nullptr); - auto loop_num = - counters->find("Context.StorageManager.Query.Reader.internal_loop_num"); - CHECK(2 == loop_num->second); + for (int i = 0; i < elements_to_check; i++) { + CHECK(coords_r[i] == i + 1); + CHECK(data_r[i] == i + 1); + } + + // Check the internal loop count against expected value. + auto stats = + ((SparseUnorderedWithDupsReader*)query->query_->strategy()) + ->stats(); + REQUIRE(stats != nullptr); + auto counters = stats->counters(); + REQUIRE(counters != nullptr); + auto loop_num = + counters->find("Context.StorageManager.Query.Reader.internal_loop_num"); + + if (one_frag) { + CHECK(1 == loop_num->second); + } else { + CHECK(9 == loop_num->second); + } + + // Try to read multiple frags without partial tile offset reading. Should + // fail + } else { + rc = read( + false, + 0, + coords_r, + &coords_r_size, + data_r, + &data_r_size, + &query, + &array); + CHECK(rc == TILEDB_ERR); + + tiledb_error_t* error = NULL; + rc = tiledb_ctx_get_last_error(ctx_, &error); + CHECK(rc == TILEDB_OK); + + const char* msg; + rc = tiledb_error_message(error, &msg); + CHECK(rc == TILEDB_OK); + + std::string error_str(msg); + CHECK( + error_str.find("Cannot load tile offsets, computed size") != + std::string::npos); + } // Clean up. rc = tiledb_array_close(ctx_, array); From 1326ed40a97b816e566ec17d0b11be8395e95bb6 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Fri, 7 Jun 2024 04:23:28 -0400 Subject: [PATCH 411/456] Remove Status_StorageManagerError and Migrate group_metadata_vacuum out of StorageManager. (#5034) Remove `Status_StorageManagerError` and its occurrences. Related subsequent fixes: * Remove `Logger::status_no_return_value` and replace occurrences with `Logger::error`. * Remove `Status` and `std::optional` from returned tuple of `Array::open_for_writes`. * de-`Status` `Context::init_loggers`. * Migrate `StorageManagerCanonical::group_metadata_vacuum` -> `static Group::vacuum_metadata`. [sc-48630] [sc-48639] --- TYPE: NO_HISTORY DESC: Remove `Status_StorageManagerError` and Migrate `group_metadata_vacuum` out of `StorageManager`. --- tiledb/api/c_api/group/group_api.cc | 2 +- .../storage_manager_override.h | 4 -- tiledb/common/exception/status.h | 12 +--- tiledb/common/logger.cc | 6 +- tiledb/common/logger.h | 15 +---- tiledb/sm/array/array.cc | 66 +++++++------------ tiledb/sm/array/array.h | 11 ++-- .../consolidator/array_meta_consolidator.cc | 2 +- .../sm/consolidator/commits_consolidator.cc | 2 +- .../fragment_meta_consolidator.cc | 2 +- .../consolidator/group_meta_consolidator.cc | 2 +- tiledb/sm/group/group.cc | 22 +++++++ tiledb/sm/group/group.h | 15 +++++ tiledb/sm/query/writers/unordered_writer.cc | 2 +- tiledb/sm/query/writers/writer_base.cc | 2 +- tiledb/sm/storage_manager/context.cc | 56 +++++++++------- tiledb/sm/storage_manager/context.h | 3 +- tiledb/sm/storage_manager/storage_manager.cc | 30 +-------- .../storage_manager_canonical.h | 11 ---- 19 files changed, 115 insertions(+), 150 deletions(-) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index ae6b8bb280e6..179ad5229beb 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -587,7 +587,7 @@ capi_return_t tiledb_group_vacuum_metadata( ensure_group_uri_argument_is_valid(group_uri); auto cfg = (config == nullptr) ? ctx->config() : config->config(); - ctx->storage_manager()->group_metadata_vacuum(group_uri, cfg); + sm::Group::vacuum_metadata(ctx->resources(), group_uri, cfg); return TILEDB_OK; } diff --git a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h index 2fa240c46c2d..05708ac1f9e6 100644 --- a/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h +++ b/tiledb/api/c_api_test_support/storage_manager_stub/storage_manager_override.h @@ -61,10 +61,6 @@ class StorageManagerStub { inline Status cancel_all_tasks() { return Status{}; } - inline Status group_metadata_vacuum(const char*, const Config&) { - throw std::logic_error( - "StorageManagerStub does not support group metadata vacuum"); - } inline Status set_tag(const std::string&, const std::string&) { return Status{}; } diff --git a/tiledb/common/exception/status.h b/tiledb/common/exception/status.h index bc7eaae277e7..5fe546a643a2 100644 --- a/tiledb/common/exception/status.h +++ b/tiledb/common/exception/status.h @@ -5,7 +5,7 @@ * * The BSD License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * Copyright (c) 2011 The LevelDB Authors. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -60,8 +60,7 @@ #include "tiledb/common/common-std.h" #include "tiledb/common/heap_memory.h" -namespace tiledb { -namespace common { +namespace tiledb::common { #define RETURN_NOT_OK(s) \ do { \ @@ -250,10 +249,6 @@ inline Status Status_Ok() { inline Status Status_Error(const std::string& msg) { return {"Error", msg}; }; -/** Return a StorageManager error class Status with a given message **/ -inline Status Status_StorageManagerError(const std::string& msg) { - return {"[TileDB::StorageManager] Error", msg}; -} /** Return a FragmentMetadata error class Status with a given message **/ inline Status Status_FragmentMetadataError(const std::string& msg) { return {"[TileDB::FragmentMetadata] Error", msg}; @@ -433,7 +428,6 @@ inline Status Status_TaskError(const std::string& msg) { inline Status Status_RangeError(const std::string& msg) { return {"[TileDB::Range] Error", msg}; } -} // namespace common -} // namespace tiledb +} // namespace tiledb::common #endif // TILEDB_STATUS_H diff --git a/tiledb/common/logger.cc b/tiledb/common/logger.cc index 2c7e5235662a..6b0250449574 100644 --- a/tiledb/common/logger.cc +++ b/tiledb/common/logger.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -147,10 +147,6 @@ Status Logger::status(const Status& st) { return st; } -void Logger::status_no_return_value(const Status& st) { - logger_->error(st.message()); -} - void Logger::trace(const std::string& msg) { logger_->trace(msg); } diff --git a/tiledb/common/logger.h b/tiledb/common/logger.h index 62fc53a37162..dde199fb4b50 100644 --- a/tiledb/common/logger.h +++ b/tiledb/common/logger.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -48,8 +48,7 @@ namespace spdlog { class logger; } -namespace tiledb { -namespace common { +namespace tiledb::common { /** Definition of class Logger. */ class Logger { @@ -315,13 +314,6 @@ class Logger { */ Status status(const Status& st); - /** - * Log a message from a Status object without returning it. - * - * @param st The Status object to log - */ - void status_no_return_value(const Status& st); - /** * Log an error and exit with a non-zero status. * @@ -457,8 +449,7 @@ inline Status logger_format_from_string( return Status::Ok(); } -} // namespace common -} // namespace tiledb +} // namespace tiledb::common // Also include the public-permissible logger functions here. #include "tiledb/common/logger_public.h" diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 1deb57ecf1b4..dc7ca4fc6e81 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -585,12 +585,11 @@ Status Array::open( array_dir_timestamp_end_, ArrayDirectoryMode::SCHEMA_ONLY)); } - auto&& [st, array_schema_latest, array_schemas] = open_for_writes(); - throw_if_not_ok(st); // Set schemas - set_array_schema_latest(array_schema_latest.value()); - set_array_schemas_all(std::move(array_schemas.value())); + auto&& [array_schema_latest, array_schemas] = open_for_writes(); + set_array_schema_latest(array_schema_latest); + set_array_schemas_all(std::move(array_schemas)); // Set the timestamp opened_array_->metadata().reset(timestamp_end_opened_at()); @@ -606,12 +605,11 @@ Status Array::open( array_dir_timestamp_end_, ArrayDirectoryMode::READ)); } - auto&& [st, latest, array_schemas] = open_for_writes(); - throw_if_not_ok(st); // Set schemas - set_array_schema_latest(latest.value()); - set_array_schemas_all(std::move(array_schemas.value())); + auto&& [latest, array_schemas] = open_for_writes(); + set_array_schema_latest(latest); + set_array_schemas_all(std::move(array_schemas)); auto version = array_schema_latest().version(); if (query_type == QueryType::DELETE && @@ -1842,19 +1840,15 @@ Array::open_for_reads_without_fragments() { } tuple< - Status, - optional>, - optional>>> + shared_ptr, + std::unordered_map>> Array::open_for_writes() { auto timer_se = resources_.stats().start_timer("array_open_write_load_schemas"); // Checks - if (!resources_.vfs().supports_uri_scheme(array_uri_)) - return { - resources_.logger()->status(Status_StorageManagerError( - "Cannot open array; URI scheme unsupported.")), - nullopt, - nullopt}; + if (!resources_.vfs().supports_uri_scheme(array_uri_)) { + throw ArrayException("Cannot open array; URI scheme unsupported."); + } // Load array schemas auto&& [array_schema_latest, array_schemas_all] = @@ -1867,31 +1861,21 @@ Array::open_for_writes() { auto version = array_schema_latest->version(); if constexpr (is_experimental_build) { if (version != constants::format_version) { - std::stringstream err; - err << "Cannot open array for writes; Array format version ("; - err << version; - err << ") is not the library format version ("; - err << constants::format_version << ")"; - return { - resources_.logger()->status(Status_StorageManagerError(err.str())), - nullopt, - nullopt}; + throw ArrayException( + "Cannot open array for writes; Array format version (" + + std::to_string(version) + ") is not the library format version (" + + std::to_string(constants::format_version) + ")"); } } else { if (version > constants::format_version) { - std::stringstream err; - err << "Cannot open array for writes; Array format version ("; - err << version; - err << ") is newer than library format version ("; - err << constants::format_version << ")"; - return { - resources_.logger()->status(Status_StorageManagerError(err.str())), - nullopt, - nullopt}; + throw ArrayException( + "Cannot open array for writes; Array format version (" + + std::to_string(version) + ") is newer than library format version (" + + std::to_string(constants::format_version) + ")"); } } - return {Status::Ok(), array_schema_latest, array_schemas_all}; + return {array_schema_latest, array_schemas_all}; } void Array::clear_last_max_buffer_sizes() { @@ -2326,12 +2310,10 @@ void Array::ensure_array_is_valid_for_delete(const URI& uri) { void ensure_supported_schema_version_for_read(format_version_t version) { // We do not allow reading from arrays written by newer version of TileDB if (version > constants::format_version) { - std::stringstream err; - err << "Cannot open array for reads; Array format version ("; - err << version; - err << ") is newer than library format version ("; - err << constants::format_version << ")"; - throw Status_StorageManagerError(err.str()); + throw ArrayException( + "Cannot open array for reads; Array format version (" + + std::to_string(version) + ") is newer than library format version (" + + std::to_string(constants::format_version) + ")"); } } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 2e2ccad56e3f..dc955493a85a 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -1200,19 +1200,18 @@ class Array { std::unordered_map>> open_for_reads_without_fragments(); - /** Opens an array for writes. + /** + * Opens an array for writes. * * @param array The array to open. - * @return tuple of Status, latest ArraySchema and map of all array schemas - * Status Ok on success, else error + * @return tuple of latest ArraySchema and map of all array schemas * ArraySchema The array schema to be retrieved after the * array is opened. * ArraySchemaMap Map of all array schemas found keyed by name */ tuple< - Status, - optional>, - optional>>> + shared_ptr, + std::unordered_map>> open_for_writes(); /** Clears the cached max buffer sizes and subarray. */ diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 69a620571637..6cf4f6f110ed 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -120,7 +120,7 @@ Status ArrayMetaConsolidator::consolidate( void ArrayMetaConsolidator::vacuum(const char* array_name) { if (array_name == nullptr) { - throw Status_StorageManagerError( + throw std::invalid_argument( "Cannot vacuum array metadata; Array name cannot be null"); } diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index c5ab1a31758b..2544007ff71d 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -103,7 +103,7 @@ Status CommitsConsolidator::consolidate( void CommitsConsolidator::vacuum(const char* array_name) { if (array_name == nullptr) { - throw Status_StorageManagerError( + throw std::invalid_argument( "Cannot vacuum array metadata; Array name cannot be null"); } diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 39f0f1d78181..19c7a9bae44c 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -181,7 +181,7 @@ Status FragmentMetaConsolidator::consolidate( void FragmentMetaConsolidator::vacuum(const char* array_name) { if (array_name == nullptr) { - throw Status_StorageManagerError( + throw std::invalid_argument( "Cannot vacuum fragment metadata; Array name cannot be null"); } diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index 89677e8b8ee6..7436a2a462f9 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -110,7 +110,7 @@ Status GroupMetaConsolidator::consolidate( void GroupMetaConsolidator::vacuum(const char* group_name) { if (group_name == nullptr) { - throw Status_StorageManagerError( + throw std::invalid_argument( "Cannot vacuum group metadata; Group name cannot be null"); } diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index c9803b67675b..e2cca0923f5f 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -599,6 +599,28 @@ Status Group::consolidate_metadata( group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); } +void Group::vacuum_metadata( + ContextResources& resources, const char* group_name, const Config& config) { + // Check group URI + URI group_uri(group_name); + if (group_uri.is_invalid()) { + throw GroupException("Cannot vacuum group metadata; Invalid URI"); + } + + // Check if group exists + ObjectType obj_type; + throw_if_not_ok(object_type(resources, group_uri, &obj_type)); + + if (obj_type != ObjectType::GROUP) { + throw GroupException("Cannot vacuum group metadata; Group does not exist"); + } + + StorageManager sm(resources, resources.logger(), config); + auto consolidator = + Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); + consolidator->vacuum(group_name); +} + const EncryptionKey* Group::encryption_key() const { return encryption_key_.get(); } diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 046c9dc1fd20..1480178bb5b0 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -254,6 +254,21 @@ class Group { const char* group_name, const Config& config); + /** + * Vacuums the consolidated metadata files of a group. + * + * @param resources The context resources. + * @param group_name The name of the group whose metadata will be + * vacuumed. + * @param config Configuration parameters for vacuuming + * (`nullptr` means default, which will use the config associated with + * this instance). + */ + static void vacuum_metadata( + ContextResources& resources, + const char* group_name, + const Config& config); + /** Returns a constant pointer to the encryption key. */ const EncryptionKey* encryption_key() const; diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index 1c058cf3ae83..b67b9b1b8a6e 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -273,7 +273,7 @@ Status UnorderedWriter::check_coord_dups() const { return Status::Ok(); }); - RETURN_NOT_OK_ELSE(status, logger_->status_no_return_value(status)); + RETURN_NOT_OK_ELSE(status, logger_->error(status.message())); return Status::Ok(); } diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index aeb122a1164c..be6a4ada95e0 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -345,7 +345,7 @@ Status WriterBase::calculate_hilbert_values( return Status::Ok(); }); - RETURN_NOT_OK_ELSE(status, logger_->status_no_return_value(status)); + RETURN_NOT_OK_ELSE(status, logger_->error(status.message())); return Status::Ok(); } diff --git a/tiledb/sm/storage_manager/context.cc b/tiledb/sm/storage_manager/context.cc index 7377d6c9011c..8e6a38a78111 100644 --- a/tiledb/sm/storage_manager/context.cc +++ b/tiledb/sm/storage_manager/context.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,8 +37,14 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { + +class ContextException : public StatusException { + public: + explicit ContextException(const std::string& message) + : StatusException("Context", message) { + } +}; static common::Logger::Level get_log_level(const Config& config); @@ -67,7 +73,7 @@ Context::Context(const Config& config) /* * Logger class is not yet C.41-compliant */ - throw_if_not_ok(init_loggers(config)); + init_loggers(config); } /* ****************************** */ @@ -111,9 +117,10 @@ Status Context::get_config_thread_count( config.get("sm.num_async_threads", &num_async_threads, &found)); if (found) { config_thread_count = std::max(config_thread_count, num_async_threads); - logger_->status_no_return_value(Status_StorageManagerError( + logger_->error( + "[Context::get_config_thread_count] " "Config parameter \"sm.num_async_threads\" has been removed; use " - "config parameter \"sm.compute_concurrency_level\".")); + "config parameter \"sm.compute_concurrency_level\"."); } uint64_t num_reader_threads{0}; @@ -121,9 +128,10 @@ Status Context::get_config_thread_count( "sm.num_reader_threads", &num_reader_threads, &found)); if (found) { config_thread_count = std::max(config_thread_count, num_reader_threads); - logger_->status_no_return_value(Status_StorageManagerError( + logger_->error( + "[Context::get_config_thread_count] " "Config parameter \"sm.num_reader_threads\" has been removed; use " - "config parameter \"sm.compute_concurrency_level\".")); + "config parameter \"sm.compute_concurrency_level\"."); } uint64_t num_writer_threads{0}; @@ -131,9 +139,10 @@ Status Context::get_config_thread_count( "sm.num_writer_threads", &num_writer_threads, &found)); if (found) { config_thread_count = std::max(config_thread_count, num_writer_threads); - logger_->status_no_return_value(Status_StorageManagerError( + logger_->error( + "[Context::get_config_thread_count] " "Config parameter \"sm.num_writer_threads\" has been removed; use " - "config parameter \"sm.compute_concurrency_level\".")); + "config parameter \"sm.compute_concurrency_level\"."); } uint64_t num_vfs_threads{0}; @@ -141,9 +150,10 @@ Status Context::get_config_thread_count( config.get("sm.num_vfs_threads", &num_vfs_threads, &found)); if (found) { config_thread_count = std::max(config_thread_count, num_vfs_threads); - logger_->status_no_return_value(Status_StorageManagerError( + logger_->error( + "[Context::get_config_thread_count] " "Config parameter \"sm.num_vfs_threads\" has been removed; use " - "config parameter \"sm.io_concurrency_level\".")); + "config parameter \"sm.io_concurrency_level\"."); } // The "sm.num_tbb_threads" has been deprecated. Users may still be setting @@ -156,9 +166,10 @@ Status Context::get_config_thread_count( if (found) { config_thread_count = std::max(config_thread_count, static_cast(num_tbb_threads)); - logger_->status_no_return_value(Status_StorageManagerError( + logger_->error( + "[Context::get_config_thread_count] " "Config parameter \"sm.num_tbb_threads\" has been removed; use " - "config parameter \"sm.io_concurrency_level\".")); + "config parameter \"sm.io_concurrency_level\"."); } config_thread_count_ret = static_cast(config_thread_count); @@ -210,16 +221,16 @@ size_t Context::get_io_thread_count(const Config& config) { std::max(config_thread_count, io_concurrency_level)); } -Status Context::init_loggers(const Config& config) { +void Context::init_loggers(const Config& config) { // temporarily set level to error so that possible errors reading // configuration are visible to the user logger_->set_level(Logger::Level::ERR); const char* format_conf; - RETURN_NOT_OK(config.get("config.logging_format", &format_conf)); + throw_if_not_ok(config.get("config.logging_format", &format_conf)); assert(format_conf != nullptr); Logger::Format format = Logger::Format::DEFAULT; - RETURN_NOT_OK(logger_format_from_string(format_conf, &format)); + throw_if_not_ok(logger_format_from_string(format_conf, &format)); global_logger(format); logger_->set_format(static_cast(format)); @@ -227,18 +238,16 @@ Status Context::init_loggers(const Config& config) { // set logging level from config bool found = false; uint32_t level = static_cast(Logger::Level::ERR); - RETURN_NOT_OK(config.get("config.logging_level", &level, &found)); + throw_if_not_ok(config.get("config.logging_level", &level, &found)); assert(found); if (level > static_cast(Logger::Level::TRACE)) { - return logger_->status(Status_StorageManagerError( + throw ContextException( "Cannot set logger level; Unsupported level:" + std::to_string(level) + - "set in configuration")); + "set in configuration"); } global_logger().set_level(static_cast(level)); logger_->set_level(static_cast(level)); - - return Status::Ok(); } common::Logger::Level get_log_level(const Config& config) { @@ -260,5 +269,4 @@ common::Logger::Level get_log_level(const Config& config) { } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/context.h b/tiledb/sm/storage_manager/context.h index 5ef63c33ccf7..61002a156ec3 100644 --- a/tiledb/sm/storage_manager/context.h +++ b/tiledb/sm/storage_manager/context.h @@ -202,9 +202,8 @@ class Context { * Initializes global and local logger. * * @param config The configuration parameters. - * @return Status */ - Status init_loggers(const Config& config); + void init_loggers(const Config& config); }; } // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 30bec13652dc..0bf601483aa7 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -136,11 +136,8 @@ Status StorageManagerCanonical::async_push_query(Query* query) { &resources_.compute_tp(), [this, query]() { // Process query. - Status st = query_submit(query); - if (!st.ok()) { - resources_.logger()->status_no_return_value(st); - } - return st; + throw_if_not_ok(query_submit(query)); + return Status::Ok(); }, [query]() { // Task was cancelled. This is safe to perform in a separate thread, @@ -404,27 +401,4 @@ Status StorageManagerCanonical::set_default_tags() { return Status::Ok(); } -void StorageManagerCanonical::group_metadata_vacuum( - const char* group_name, const Config& config) { - // Check group URI - URI group_uri(group_name); - if (group_uri.is_invalid()) { - throw Status_StorageManagerError( - "Cannot vacuum group metadata; Invalid URI"); - } - - // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources_, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { - throw Status_StorageManagerError( - "Cannot vacuum group metadata; Group does not exist"); - } - - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, this); - consolidator->vacuum(group_name); -} - } // namespace tiledb::sm diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index def3a5f6bfc5..f024abd2f765 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -250,17 +250,6 @@ class StorageManagerCanonical { return resources_; } - /** - * Vacuums the consolidated metadata files of a group. - * - * @param group_name The name of the group whose metadata will be - * vacuumed. - * @param config Configuration parameters for vacuuming - * (`nullptr` means default, which will use the config associated with - * this instance). - */ - void group_metadata_vacuum(const char* group_name, const Config& config); - private: /* ********************************* */ /* PRIVATE DATATYPES */ From 9116d3c95a83d72545520acb9a7808fc63478963 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Fri, 7 Jun 2024 19:26:53 +0300 Subject: [PATCH 412/456] Add CurrentDomain API support. (#5041) This PR adds the foundation for the CurrentDomain API: - format change in array schema and disk serialization - plumbing so you can create/open an array with current_domain data using sm APIs - array schema dump extension - test coverage To be done C and CPP API that wrap APIs defined here. [sc-42489] --- TYPE: FEATURE DESC: Add CurrentDomain API support. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- format_spec/FORMAT_SPEC.md | 2 +- format_spec/array_schema.md | 1 + format_spec/current_domain.md | 15 + test/src/unit-capi-array_schema.cc | 6 +- tiledb/CMakeLists.txt | 2 + tiledb/sm/array/array.cc | 7 + tiledb/sm/array_schema/CMakeLists.txt | 18 +- tiledb/sm/array_schema/array_schema.cc | 63 +- tiledb/sm/array_schema/array_schema.h | 24 + .../sm/array_schema/array_schema_evolution.cc | 35 +- .../sm/array_schema/array_schema_evolution.h | 18 + tiledb/sm/array_schema/current_domain.cc | 243 ++++++ tiledb/sm/array_schema/current_domain.h | 209 ++++++ tiledb/sm/array_schema/ndrectangle.cc | 145 ++++ tiledb/sm/array_schema/ndrectangle.h | 188 +++++ tiledb/sm/array_schema/test/CMakeLists.txt | 1 + .../test/array_schema_test_support.h | 4 + .../test/compile_current_domain_main.cc | 37 + .../test/compile_ndrectangle_main.cc | 37 + .../array_schema/test/unit_current_domain.cc | 701 ++++++++++++++++++ tiledb/sm/c_api/tiledb_enum.h | 4 + tiledb/sm/enums/current_domain_type.h | 80 ++ tiledb/sm/fragment/fragment_metadata.cc | 37 +- tiledb/sm/misc/constants.cc | 11 +- tiledb/sm/misc/constants.h | 9 + tiledb/sm/serialization/array_schema.cc | 6 + .../serialization/array_schema_evolution.cc | 5 + 27 files changed, 1870 insertions(+), 38 deletions(-) create mode 100644 format_spec/current_domain.md create mode 100644 tiledb/sm/array_schema/current_domain.cc create mode 100644 tiledb/sm/array_schema/current_domain.h create mode 100644 tiledb/sm/array_schema/ndrectangle.cc create mode 100644 tiledb/sm/array_schema/ndrectangle.h create mode 100644 tiledb/sm/array_schema/test/compile_current_domain_main.cc create mode 100644 tiledb/sm/array_schema/test/compile_ndrectangle_main.cc create mode 100644 tiledb/sm/array_schema/test/unit_current_domain.cc create mode 100644 tiledb/sm/enums/current_domain_type.h diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index c4def7a9f782..958941d7c5e0 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -4,7 +4,7 @@ title: Format Specification **Notes:** -* The current TileDB format version number is **21** (`uint32_t`). +* The current TileDB format version number is **22** (`uint32_t`). * Data written by TileDB and referenced in this document is **little-endian** with the following exceptions: diff --git a/format_spec/array_schema.md b/format_spec/array_schema.md index 108edbc857f3..04a0e11974ec 100644 --- a/format_spec/array_schema.md +++ b/format_spec/array_schema.md @@ -54,6 +54,7 @@ The array schema file consists of a single [generic tile](./generic_tile.md), wi | Label 1 | [Dimension Label](#dimension_label) | First dimension label | | … | … | … | | Label N | [Dimension Label](#dimension_label) | Nth dimension label | +| CurrentDomain | [CurrentDomain](./current_domain.md) | The array current domain | ## Domain diff --git a/format_spec/current_domain.md b/format_spec/current_domain.md new file mode 100644 index 000000000000..d497fca5f94a --- /dev/null +++ b/format_spec/current_domain.md @@ -0,0 +1,15 @@ +--- +title: Current Domain +--- + +## Main Structure + +The array current domain is stored within the [Array Schema](./array_schema.md#array-schema-file) +file. If a current domain is empty, only the version number and the empty flag are serialized to storage. + +| **Field** | **Type** | **Description** | +| :--- | :--- | :--- | +| Version number | `uint32_t` | Current domain version number | +| Empty | `uint8_t` | Whether the current domain has a representation(e.g. NDRectangle) set or not | +| Type | `uint8_t` | The type of current domain stored in this file | +| NDRectangle | [MBR](./fragment.md#mbr) | A hyperrectangle defined using [1DRange](./fragment.md#mbr) items for each dimension | diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index f4d520671378..5166195b92aa 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -921,7 +921,11 @@ void ArraySchemaFx::load_and_check_array_schema(const std::string& path) { "- Cell val num: " + CELL_VAL_NUM_STR + "\n" + "- Filters: 2\n" + " > BZIP2: COMPRESSION_LEVEL=5\n" + " > BitWidthReduction: BIT_WIDTH_MAX_WINDOW=1000\n" + - "- Fill value: " + FILL_VALUE_STR + "\n"; + "- Fill value: " + FILL_VALUE_STR + "\n" + "### Current domain ###\n" + + "- Version: " + + std::to_string(tiledb::sm::constants::current_domain_version) + "\n" + + "- Empty: 1" + "\n"; + FILE* gold_fout = fopen("gold_fout.txt", "w"); const char* dump = dump_str.c_str(); fwrite(dump, sizeof(char), strlen(dump), gold_fout); diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 977bdf225879..c541db76a583 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -154,6 +154,8 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension_label.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/domain.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/enumeration.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/ndrectangle.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/current_domain.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer_list.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/c_api/api_argument_validator.cc diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index dc7ca4fc6e81..9737d2e5ba5a 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -40,6 +40,7 @@ #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/auxiliary.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/crypto/crypto.h" @@ -264,6 +265,12 @@ Status Array::create( array_schema->generate_uri(); array_schema->check(resources.config()); + // Check current domain is specified correctly if set + if (!array_schema->get_current_domain()->empty()) { + array_schema->get_current_domain()->check_schema_sanity( + array_schema->shared_domain()); + } + // Create array directory throw_if_not_ok(resources.vfs().create_dir(array_uri)); diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index 598b640d1235..2c0745ff4a6a 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -65,13 +65,29 @@ commence(object_library enumeration) this_target_object_libraries(buffer constants seedable_global_PRNG) conclude(object_library) +# +# `ndrectangle` object library +# +commence(object_library ndrectangle) + this_target_sources(ndrectangle.cc) + this_target_object_libraries(constants domain) +conclude(object_library) + +# +# `current_domain` object library +# +commence(object_library current_domain) + this_target_sources(current_domain.cc) + this_target_object_libraries(ndrectangle constants) +conclude(object_library) + # # `array_schema` object library # commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain enumeration fragment time uri_format vfs) + attribute domain enumeration fragment current_domain time uri_format vfs) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index cb32bc3a2924..4b006374087c 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -37,6 +37,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" @@ -104,7 +105,9 @@ ArraySchema::ArraySchema( memory_tracker_->get_resource(MemoryType::DIMENSION_LABELS)) , enumeration_map_(memory_tracker_->get_resource(MemoryType::ENUMERATION)) , enumeration_path_map_( - memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) { + memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) + , current_domain_(make_shared( + memory_tracker, constants::current_domain_version)) { // Set up default filter pipelines for coords, offsets, and validity values. coords_filters_.add_filter(CompressionFilter( constants::coords_compression, @@ -141,6 +144,7 @@ ArraySchema::ArraySchema( FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , uri_(uri) @@ -165,7 +169,8 @@ ArraySchema::ArraySchema( memory_tracker_->get_resource(MemoryType::ENUMERATION_PATHS)) , cell_var_offsets_filters_(cell_var_offsets_filters) , cell_validity_filters_(cell_validity_filters) - , coords_filters_(coords_filters) { + , coords_filters_(coords_filters) + , current_domain_(current_domain) { for (auto atr : attributes) { attributes_.push_back(atr); } @@ -256,6 +261,7 @@ ArraySchema::ArraySchema(const ArraySchema& array_schema) , cell_var_offsets_filters_{array_schema.cell_var_offsets_filters_} , cell_validity_filters_{array_schema.cell_validity_filters_} , coords_filters_{array_schema.coords_filters_} + , current_domain_(array_schema.current_domain_) , mtx_{} { throw_if_not_ok(set_domain(array_schema.domain_)); @@ -737,6 +743,9 @@ void ArraySchema::dump(FILE* out) const { fprintf(out, "\n"); label->dump(out); } + + // Print out array current domain + current_domain_->dump(out); } Status ArraySchema::has_attribute( @@ -804,6 +813,7 @@ bool ArraySchema::is_nullable(const std::string& name) const { // dimension_label #1 // dimension_label #2 // ... +// current_domain void ArraySchema::serialize(Serializer& serializer) const { // Write version, which is always the current version. Despite // the in-memory `version_`, we will serialize every array schema @@ -871,6 +881,9 @@ void ArraySchema::serialize(Serializer& serializer) const { serializer.write(enmr_uri_size); serializer.write(enmr_uri.data(), enmr_uri_size); } + + // Serialize array current domain information + current_domain_->serialize(serializer); } Layout ArraySchema::tile_order() const { @@ -1422,6 +1435,15 @@ shared_ptr ArraySchema::deserialize( } } + // Load the array current domain, if this is an older array, it'll get by + // default an empty current domain object + auto current_domain = make_shared( + memory_tracker, constants::current_domain_version); + if (version >= constants::current_domain_min_format_version) { + current_domain = + CurrentDomain::deserialize(deserializer, memory_tracker, domain); + } + // Validate if (cell_order == Layout::HILBERT && domain->dim_num() > Hilbert::HC_MAX_DIM) { @@ -1471,6 +1493,7 @@ shared_ptr ArraySchema::deserialize( FilterPipeline( coords_filters, version < 5 ? domain->dimension_ptr(0)->type() : Datatype::UINT64), + current_domain, memory_tracker); } @@ -1791,4 +1814,40 @@ void ArraySchema::generate_uri( array_uri_.join_path(constants::array_schema_dir_name).join_path(name_); } +void ArraySchema::expand_current_domain( + shared_ptr new_current_domain) { + if (new_current_domain == nullptr) { + throw ArraySchemaException( + "The argument specified for current domain expansion is nullptr."); + } + + // Check that the new current domain expands the existing one and not shrinks + // it Every current domain covers an empty current domain. + if (!current_domain_->empty() && + !current_domain_->covered(new_current_domain)) { + throw ArraySchemaException( + "The current domain of an array can only be expanded, please adjust " + "your new current domain object."); + } + + new_current_domain->check_schema_sanity(this->shared_domain()); + + current_domain_ = new_current_domain; +} + +shared_ptr ArraySchema::get_current_domain() const { + return current_domain_; +} + +void ArraySchema::set_current_domain( + shared_ptr current_domain) { + if (current_domain == nullptr) { + throw ArraySchemaException( + "The argument specified for setting the current domain on the " + "schema is nullptr."); + } + + current_domain_ = current_domain; +} + } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index ba300d398b5e..a07101cf19a8 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -56,6 +56,7 @@ class DimensionLabel; class Domain; class Enumeration; class MemoryTracker; +class CurrentDomain; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -120,6 +121,7 @@ class ArraySchema { * @param cell_validity_filters * The filter pipeline run on validity tiles for nullable attributes. * @param coords_filters The filter pipeline run on coordinate tiles. + * @param current_domain The array current domain object * @param memory_tracker The memory tracker of the array this fragment * metadata corresponds to. **/ @@ -141,6 +143,7 @@ class ArraySchema { FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, + shared_ptr current_domain, shared_ptr memory_tracker); /** @@ -585,6 +588,24 @@ class ArraySchema { std::optional> timestamp_range = std::nullopt); + /** + * Expand the array current domain + * + * @param new_current_domain The new array current domain we want to expand to + */ + void expand_current_domain( + shared_ptr new_current_domain); + + /** + * Set the array current domain on the schema + * + * @param current_domain The array current domain we want to set on the schema + */ + void set_current_domain(shared_ptr current_domain); + + /** Array current domain accessor */ + shared_ptr get_current_domain() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -700,6 +721,9 @@ class ArraySchema { /** The filter pipeline run on coordinate tiles. */ FilterPipeline coords_filters_; + /** The array current domain */ + shared_ptr current_domain_; + /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 560db92f149f..2cafe3180a09 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -39,6 +39,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/array_schema/enumeration.h" @@ -91,6 +92,7 @@ ArraySchemaEvolution::ArraySchemaEvolution( enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , attributes_to_add_map_( @@ -101,7 +103,8 @@ ArraySchemaEvolution::ArraySchemaEvolution( , enumerations_to_extend_map_( memory_tracker_->get_resource(MemoryType::ENUMERATION)) , enumerations_to_drop_(enmrs_to_drop) - , timestamp_range_(timestamp_range) { + , timestamp_range_(timestamp_range) + , current_domain_to_expand_(current_domain) { for (auto& elem : attrs_to_add) { attributes_to_add_map_.insert(elem); } @@ -175,6 +178,11 @@ shared_ptr ArraySchemaEvolution::evolve_schema( schema->generate_uri(); } + // Get expanded current domain + if (current_domain_to_expand_) { + schema->expand_current_domain(current_domain_to_expand_); + } + return schema; } @@ -370,6 +378,30 @@ std::pair ArraySchemaEvolution::timestamp_range() const { timestamp_range_.first, timestamp_range_.second); } +void ArraySchemaEvolution::expand_current_domain( + shared_ptr current_domain) { + if (current_domain == nullptr) { + throw ArraySchemaEvolutionException( + "Cannot expand the array current domain; Input current domain is null"); + } + + if (current_domain->empty()) { + throw ArraySchemaEvolutionException( + "Unable to expand the array current domain, the new current domain " + "specified is empty"); + } + + std::lock_guard lock(mtx_); + current_domain_to_expand_ = current_domain; +} + +shared_ptr ArraySchemaEvolution::current_domain_to_expand() + const { + std::lock_guard lock(mtx_); + + return current_domain_to_expand_; +} + /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ @@ -380,6 +412,7 @@ void ArraySchemaEvolution::clear() { enumerations_to_add_map_.clear(); enumerations_to_drop_.clear(); timestamp_range_ = {0, 0}; + current_domain_to_expand_ = nullptr; } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index bc2160698f14..bdec153e483b 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -55,6 +55,7 @@ class Domain; class Enumeration; class MemoryTracker; class ArraySchema; +class CurrentDomain; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -79,6 +80,7 @@ class ArraySchemaEvolution { * @param enmrs_to_add Enumerations to add to the schema. * @param attrs_to_drop Attributes to remove from the schema. * @param timestamp_range Timestamp range to use for the new schema. + * @param timestamp_range CurrentDomain to use for the new schema. * @param memory_tracker Memory tracker to use for the new schema. */ ArraySchemaEvolution( @@ -90,6 +92,7 @@ class ArraySchemaEvolution { enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, + shared_ptr current_domain, shared_ptr memory_tracker); DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); @@ -188,6 +191,18 @@ class ArraySchemaEvolution { /** Returns the timestamp range. */ std::pair timestamp_range() const; + /** + * Expands the array current domain + * + * @param current_domain The new current domain to expand to + */ + void expand_current_domain(shared_ptr current_domain); + + /** + * Accessor for the current domain we want to expand to + */ + shared_ptr current_domain_to_expand() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -225,6 +240,9 @@ class ArraySchemaEvolution { */ std::pair timestamp_range_; + /** The array current domain to expand */ + shared_ptr current_domain_to_expand_; + /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/current_domain.cc b/tiledb/sm/array_schema/current_domain.cc new file mode 100644 index 000000000000..9aad114d5a1e --- /dev/null +++ b/tiledb/sm/array_schema/current_domain.cc @@ -0,0 +1,243 @@ +/** + * @file current_domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class CurrentDomain. + */ + +#include +#include + +#include "tiledb/common/memory_tracker.h" +#include "tiledb/common/random/random_label.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" +#include "tiledb/storage_format/serialization/serializers.h" + +#include "current_domain.h" + +namespace tiledb::sm { + +CurrentDomain::CurrentDomain( + shared_ptr memory_tracker, format_version_t version) + : memory_tracker_(memory_tracker) + , empty_(true) + , version_(version) { +} + +CurrentDomain::CurrentDomain( + shared_ptr memory_tracker, + format_version_t version, + shared_ptr ndr) + : CurrentDomain(memory_tracker, version) { + set_ndrectangle(ndr); +} + +shared_ptr CurrentDomain::deserialize( + Deserializer& deserializer, + shared_ptr memory_tracker, + shared_ptr domain) { + auto disk_version = deserializer.read(); + if (disk_version > constants::current_domain_version) { + throw std::runtime_error( + "Invalid current domain API version on disk. '" + + std::to_string(disk_version) + + "' is newer than the current library current domain version '" + + std::to_string(constants::current_domain_version) + "'"); + } + + auto empty = deserializer.read(); + + if (empty) { + return make_shared(memory_tracker, disk_version); + } + + CurrentDomainType type = + static_cast(deserializer.read()); + + switch (type) { + case CurrentDomainType::NDRECTANGLE: { + auto ndrectangle = + NDRectangle::deserialize(deserializer, memory_tracker, domain); + return make_shared( + memory_tracker, disk_version, ndrectangle); + } + default: { + throw std::runtime_error( + "We found an unsupported " + current_domain_type_str(type) + + "array current domain type on disk."); + } + } +} + +void CurrentDomain::serialize(Serializer& serializer) const { + serializer.write(constants::current_domain_version); + serializer.write(empty_); + + if (empty_) { + return; + } + + serializer.write(static_cast(type_)); + + switch (type_) { + case CurrentDomainType::NDRECTANGLE: { + ndrectangle_->serialize(serializer); + break; + } + default: { + throw std::runtime_error( + "The current domain to serialize has an unsupported type " + + current_domain_type_str(type_)); + } + } +} + +void CurrentDomain::dump(FILE* out) const { + if (out == nullptr) { + out = stdout; + } + std::stringstream ss; + ss << "### Current domain ###" << std::endl; + ss << "- Version: " << version_ << std::endl; + ss << "- Empty: " << empty_ << std::endl; + if (empty_) { + fprintf(out, "%s", ss.str().c_str()); + return; + } + + ss << "- Type: " << current_domain_type_str(type_) << std::endl; + + fprintf(out, "%s", ss.str().c_str()); + + switch (type_) { + case CurrentDomainType::NDRECTANGLE: { + ndrectangle_->dump(out); + break; + } + default: { + throw std::runtime_error( + "The current domain to dump as string has an unsupported type " + + current_domain_type_str(type_)); + } + } +} + +void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { + if (!empty_) { + throw std::logic_error( + "Setting a rectangle on a non-empty CurrentDomain object is not " + "allowed."); + } + ndrectangle_ = ndr; + type_ = CurrentDomainType::NDRECTANGLE; + empty_ = false; +} + +shared_ptr CurrentDomain::ndrectangle() const { + if (empty_ || type_ != CurrentDomainType::NDRECTANGLE) { + throw std::logic_error( + "It's not possible to get the ndrectangle from this current domain if " + "one isn't set."); + } + + return ndrectangle_; +} + +bool CurrentDomain::covered( + shared_ptr expanded_current_domain) const { + return covered(expanded_current_domain->ndrectangle()->get_ndranges()); +} + +bool CurrentDomain::covered(const NDRange& ndranges) const { + switch (type_) { + case CurrentDomainType::NDRECTANGLE: { + for (unsigned d = 0; d < ndranges.size(); ++d) { + auto dim = ndrectangle_->domain()->dimension_ptr(d); + if (dim->var_size() && ndranges[d].empty()) { + // This is a free pass for array schema var size dimensions for + // which we don't support specifying a domain. + continue; + } + + if (!dim->covered(ndrectangle_->get_range(d), ndranges[d])) { + return false; + } + } + return true; + } + default: { + throw std::runtime_error( + "Unable to execute this current domain operation because one of the " + "current domains passed has an unsupported type " + + current_domain_type_str(type_)); + } + } +} + +void CurrentDomain::check_schema_sanity( + shared_ptr schema_domain) const { + switch (type_) { + case CurrentDomainType::NDRECTANGLE: { + auto& ndranges = ndrectangle_->get_ndranges(); + + // Dim nums match + if (schema_domain->dim_num() != ndranges.size()) { + throw std::logic_error( + "The array current domain and the array schema have a non-equal " + "number of dimensions"); + } + + // Bounds are set for all dimensions + for (uint32_t i = 0; i < ndranges.size(); ++i) { + if (ndranges[i].empty()) { + throw std::logic_error( + "This current domain has no range specified for dimension idx: " + + std::to_string(i)); + } + } + + // Nothing is out of bounds + if (!this->covered(schema_domain->domain())) { + throw std::logic_error( + "This array current domain has ranges past the boundaries of the " + "array schema domain"); + } + + break; + } + default: { + throw std::runtime_error( + "The CurrentDomain object has " + std::string("an unsupported") + + "type: " + current_domain_type_str(type_)); + } + } +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/current_domain.h b/tiledb/sm/array_schema/current_domain.h new file mode 100644 index 000000000000..b73f3d2314d9 --- /dev/null +++ b/tiledb/sm/array_schema/current_domain.h @@ -0,0 +1,209 @@ +/** + * @file current_domain.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class CurrentDomain. + */ + +#ifndef TILEDB_CURRENT_DOMAIN_H +#define TILEDB_CURRENT_DOMAIN_H + +#include + +#include "tiledb/common/common.h" +#include "tiledb/sm/array_schema/ndrectangle.h" +#include "tiledb/sm/enums/current_domain_type.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/storage_format/serialization/serializers.h" + +namespace tiledb::sm { + +class MemoryTracker; +class ArraySchema; +class Domain; + +/** Defines an array current domain */ +class CurrentDomain { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** No default constructor*/ + CurrentDomain() = delete; + + DISABLE_COPY(CurrentDomain); + DISABLE_MOVE(CurrentDomain); + + /** Constructor + * + * @param memory_tracker The memory tracker. + * @param version the disk version of this current domain + */ + CurrentDomain( + shared_ptr memory_tracker, format_version_t version); + + /** Constructor + * + * @param memory_tracker The memory tracker. + * @param version the disk version of this current domain + * @param ndr the NDRectangle to to set on this instance + */ + CurrentDomain( + shared_ptr memory_tracker, + format_version_t version, + shared_ptr ndr); + + /** Destructor. */ + ~CurrentDomain() = default; + + /* ********************************* */ + /* OPERATORS */ + /* ********************************* */ + + DISABLE_COPY_ASSIGN(CurrentDomain); + DISABLE_MOVE_ASSIGN(CurrentDomain); + + /* ********************************* */ + /* API */ + /* ********************************* */ + /** + * Deserialize a CurrentDomain + * + * @param deserializer The deserializer to deserialize from. + * @param memory_tracker The memory tracker associated with this + * CurrentDomain. + * @param domain The array schema domain. + * @return A new CurrentDomain. + */ + static shared_ptr deserialize( + Deserializer& deserializer, + shared_ptr memory_tracker, + shared_ptr domain); + + /** + * Serializes the CurrentDomain into a buffer. + * + * @param serializer The object the array schema is serialized into. + */ + void serialize(Serializer& serializer) const; + + /** + * @return Returns the type of current domain stored in this instance + */ + CurrentDomainType type() const { + return type_; + } + + /** + * @return Returns whether this current domain is empty or not. + */ + bool empty() const { + return empty_; + } + + /** + * @return Returns the format version of this current domain + */ + format_version_t version() const { + return version_; + } + + /** + * Dump a textual representation of the CurrentDomain to the FILE + * + * @param out A file pointer to write to. If out is nullptr, use stdout + */ + void dump(FILE* out) const; + + /** + * Sets a NDRectangle to this current domain and adjusts its type to reflect + * that. Throws if the current domain is not empty. + * + * @param ndr A NDRectangle to be set on this CurrentDomain object. + */ + void set_ndrectangle(std::shared_ptr ndr); + + /** + * Throws if the current domain doesn't have a NDRectangle set + * + * @return Returns the ndrectangle set on a current domain. + */ + shared_ptr ndrectangle() const; + + /** + * Checks if the argument fully contains this current domain. + * + * @param expanded_current_domain The current domain we want to compare + * against + * @return True if the argument is a superset of the current instance + */ + bool covered(shared_ptr expanded_current_domain) const; + + /** + * Checks if the arg fully contains this current domain. + * + * @param expanded_range The ndrange we want to compare against + * @return True if the argument is a superset of the current instance + */ + bool covered(const NDRange& expanded_range) const; + + /** + * Perform various checks to ensure the current domain is coherent with the + * array schema + * + * @param schema_domain The array schema domain + */ + void check_schema_sanity(shared_ptr schema_domain) const; + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** + * The memory tracker of the CurrentDomain. + */ + shared_ptr memory_tracker_; + + /** The type of the current domain */ + CurrentDomainType type_; + + /** A flag which enables or disables inequality comparisons */ + bool empty_; + + /** The ndrectangle current domain */ + shared_ptr ndrectangle_; + + /** The format version of this CurrentDomain */ + format_version_t version_; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_CURRENT_DOMAIN_H diff --git a/tiledb/sm/array_schema/ndrectangle.cc b/tiledb/sm/array_schema/ndrectangle.cc new file mode 100644 index 000000000000..009f844b28b9 --- /dev/null +++ b/tiledb/sm/array_schema/ndrectangle.cc @@ -0,0 +1,145 @@ +/** + * @file ndrectangle.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class ndrectangle. + */ + +#include +#include + +#include "tiledb/common/memory_tracker.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" + +#include "ndrectangle.h" + +namespace tiledb::sm { + +NDRectangle::NDRectangle( + shared_ptr memory_tracker, + shared_ptr dom, + const NDRange& nd) + : memory_tracker_(memory_tracker) + , range_data_(nd) + , domain_(dom) { +} + +NDRectangle::NDRectangle( + shared_ptr memory_tracker, shared_ptr dom) + : memory_tracker_(memory_tracker) + , domain_(dom) { + range_data_.resize(dom->dim_num()); +} + +NDRange NDRectangle::deserialize_ndranges( + Deserializer& deserializer, shared_ptr domain) { + NDRange nd; + auto dim_num = domain->dim_num(); + nd.resize(dim_num); + for (unsigned d = 0; d < dim_num; ++d) { + auto dim{domain->dimension_ptr(d)}; + if (!dim->var_size()) { // Fixed-sized + auto r_size = 2 * dim->coord_size(); + nd[d] = Range(deserializer.get_ptr(r_size), r_size); + } else { // Var-sized + uint64_t r_size, start_size; + r_size = deserializer.read(); + start_size = deserializer.read(); + nd[d] = Range(deserializer.get_ptr(r_size), r_size, start_size); + } + } + return nd; +} + +shared_ptr NDRectangle::deserialize( + Deserializer& deserializer, + shared_ptr memory_tracker, + shared_ptr domain) { + return make_shared( + memory_tracker, domain, deserialize_ndranges(deserializer, domain)); +} + +void NDRectangle::serialize(Serializer& serializer) const { + for (unsigned d = 0; d < range_data_.size(); ++d) { + auto dim{domain_->dimension_ptr(d)}; + const auto& r = range_data_[d]; + if (!dim->var_size()) { // Fixed-sized + serializer.write(r.data(), r.size()); + } else { // Var-sized + auto r_size = r.size(); + auto r_start_size = r.start_size(); + serializer.write(r_size); + serializer.write(r_start_size); + serializer.write(r.data(), r_size); + } + } +} + +void NDRectangle::dump(FILE* out) const { + if (out == nullptr) { + out = stdout; + } + + std::stringstream ss; + ss << " - NDRectangle ###" << std::endl; + for (uint32_t i = 0; i < range_data_.size(); ++i) { + auto dtype = domain_->dimension_ptr(i)->type(); + ss << " - " << range_str(range_data_[i], dtype) << std::endl; + } + + fprintf(out, "%s", ss.str().c_str()); +} + +void NDRectangle::set_range(const Range& r, uint32_t idx) { + if (idx >= range_data_.size()) { + throw std::logic_error( + "Trying to set a range for an index out of bounds is not possible."); + } + range_data_[idx] = r; +} + +void NDRectangle::set_range_for_name(const Range& r, const std::string& name) { + auto idx = domain_->get_dimension_index(name); + set_range(r, idx); +} + +const Range& NDRectangle::get_range(uint32_t idx) const { + if (idx >= range_data_.size()) { + throw std::logic_error( + "Trying to get a range for an index out of bounds is not possible."); + } + return range_data_[idx]; +} + +const Range& NDRectangle::get_range_for_name(const std::string& name) const { + auto idx = domain_->get_dimension_index(name); + return get_range(idx); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/ndrectangle.h b/tiledb/sm/array_schema/ndrectangle.h new file mode 100644 index 000000000000..84a4c1a2a817 --- /dev/null +++ b/tiledb/sm/array_schema/ndrectangle.h @@ -0,0 +1,188 @@ +/** + * @file ndrectangle.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class NDRectangle. + */ + +#ifndef TILEDB_NDRECTANGLE_H +#define TILEDB_NDRECTANGLE_H + +#include + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/current_domain_type.h" +#include "tiledb/sm/misc/types.h" +#include "tiledb/storage_format/serialization/serializers.h" + +namespace tiledb::sm { + +class MemoryTracker; +class Domain; + +/** Defines a NDRectangle */ +class NDRectangle { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** No default constructor. */ + NDRectangle() = delete; + + DISABLE_COPY(NDRectangle); + DISABLE_MOVE(NDRectangle); + + /** + * Constructor + * + * @param memory_tracker The memory tracker. + * @param dom Array schema domain. + * @param nd The vector of ranges per dimension. + */ + NDRectangle( + shared_ptr memory_tracker, + shared_ptr dom, + const NDRange& nd); + + /** + * Constructs a NDRectangle with dim_num empty internal ranges + * + * @param memory_tracker The memory tracker. + * @param dom Array schema domain. + */ + NDRectangle(shared_ptr memory_tracker, shared_ptr dom); + + /** Destructor. */ + ~NDRectangle() = default; + + /* ********************************* */ + /* OPERATORS */ + /* ********************************* */ + + DISABLE_COPY_ASSIGN(NDRectangle); + DISABLE_MOVE_ASSIGN(NDRectangle); + + /* ********************************* */ + /* API */ + /* ********************************* */ + /** + * Deserialize a ndrectangle + * + * @param deserializer The deserializer to deserialize from. + * @param memory_tracker The memory tracker associated with this ndrectangle. + * @param domain The domain from array schema + * @return A new ndrectangle. + */ + static shared_ptr deserialize( + Deserializer& deserializer, + shared_ptr memory_tracker, + shared_ptr domain); + + /** + * Deserialize a ndrange + * + * @param deserializer The deserializer to deserialize from. + * @param domain The domain from array schema + * @return A new NDRange. + */ + static NDRange deserialize_ndranges( + Deserializer& deserializer, shared_ptr domain); + + /** + * Serializes the NDRectangle into a buffer. + * + * @param serializer The object the NDRectangle is serialized into. + */ + void serialize(Serializer& serializer) const; + + /** + * Dump a textual representation of the NDRectangle to the FILE + * + * @param out A file pointer to write to. If out is nullptr, use stdout + */ + void dump(FILE* out) const; + + /** nd ranges accessor */ + const NDRange& get_ndranges() const { + return range_data_; + } + + /** domain accessor */ + shared_ptr domain() const { + return domain_; + } + + /** + * Set a range for the dimension at idx + * + * @param r The range to set + * @param idx The idx of the dimension given the array schema domain ordering + */ + void set_range(const Range& r, uint32_t idx); + + /** + * Set a range for the dimension by name + * + * @param r The range to set for the dimension specified by name + * @param name The name of the dimension + */ + void set_range_for_name(const Range& r, const std::string& name); + + /** + * Get the range for the dimension at idx + * + * @return The range of the dimension + */ + const Range& get_range(uint32_t idx) const; + + /** + * Get the range for the dimension specified by name + * + * @return The range of the dimension + */ + const Range& get_range_for_name(const std::string& name) const; + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The memory tracker of the ndrectangle. */ + shared_ptr memory_tracker_; + + /** Per dimension ranges of the rectangle */ + NDRange range_data_; + + /** Array Schema domain */ + shared_ptr domain_; +}; + +} // namespace tiledb::sm + +#endif // TILEDB_NDRECTANGLE_H diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index 41d6018b91dc..987264201750 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -38,6 +38,7 @@ commence(unit_test array_schema) unit_dimension_label.cc unit_domain_data.cc unit_tile_domain.cc + unit_current_domain.cc ) this_target_link_libraries(tiledb_test_support_lib) conclude(unit_test) diff --git a/tiledb/sm/array_schema/test/array_schema_test_support.h b/tiledb/sm/array_schema/test/array_schema_test_support.h index 2c751fe1bfd0..3f7b05dae9cc 100644 --- a/tiledb/sm/array_schema/test/array_schema_test_support.h +++ b/tiledb/sm/array_schema/test/array_schema_test_support.h @@ -76,6 +76,7 @@ #include "tiledb/common/common.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/enums/array_type.h" @@ -329,6 +330,9 @@ class TestArraySchema { FilterPipeline(), FilterPipeline(), FilterPipeline(), + make_shared( + tiledb::test::create_test_memory_tracker(), + constants::current_domain_version), tiledb::test::create_test_memory_tracker()) { } diff --git a/tiledb/sm/array_schema/test/compile_current_domain_main.cc b/tiledb/sm/array_schema/test/compile_current_domain_main.cc new file mode 100644 index 000000000000..0b2ab467ed10 --- /dev/null +++ b/tiledb/sm/array_schema/test/compile_current_domain_main.cc @@ -0,0 +1,37 @@ +/** + * @file compile_current_domain_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tiledb/sm/array_schema/current_domain.h" + +int main(int, char*[]) { + try { + tiledb::sm::CurrentDomain(nullptr, 0); + } catch (...) { + } + return 0; +} diff --git a/tiledb/sm/array_schema/test/compile_ndrectangle_main.cc b/tiledb/sm/array_schema/test/compile_ndrectangle_main.cc new file mode 100644 index 000000000000..d33ddd34f2f7 --- /dev/null +++ b/tiledb/sm/array_schema/test/compile_ndrectangle_main.cc @@ -0,0 +1,37 @@ +/** + * @file compile_ndrectangle_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tiledb/sm/array_schema/ndrectangle.h" + +int main(int, char*[]) { + try { + tiledb::sm::NDRectangle(nullptr, nullptr); + } catch (...) { + } + return 0; +} diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc new file mode 100644 index 000000000000..204d4aea355b --- /dev/null +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -0,0 +1,701 @@ +/** + * @file unit-current_domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023-2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the CurrentDomain. + */ + +#include + +#include "test/support/src/mem_helpers.h" +#include "test/support/tdb_catch.h" +#include "tiledb/common/memory_tracker.h" +#include "tiledb/sm/array/array.h" +#include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/storage_manager/context.h" + +using namespace tiledb::sm; + +template +class CurrentDomainFx { + public: + CurrentDomainFx(); + ~CurrentDomainFx(); + + shared_ptr create_current_domain( + const NDRange& ranges, + shared_ptr schema, + shared_ptr ndrectangle = nullptr, + bool empty = false); + + void check_storage_serialization( + shared_ptr schema, const NDRange& ranges); + + storage_size_t calculate_serialized_size( + shared_ptr current_domain); + shared_ptr serialize_to_tile( + shared_ptr current_domain); + + shared_ptr create_schema(); + + shared_ptr create_schema_var(); + + void create_array(shared_ptr schema); + shared_ptr get_array(QueryType type); + shared_ptr get_array_directory(); + shared_ptr get_array_schema_latest(); + void check_current_domains_equal( + shared_ptr s1, shared_ptr s2); + + void rm_array(); + + shared_ptr memory_tracker_; + URI uri_; + Config cfg_; + Context ctx_; + EncryptionKey enc_key_; +}; + +template +CurrentDomainFx::CurrentDomainFx() + : memory_tracker_(tiledb::test::create_test_memory_tracker()) + , uri_("current_domain_array") + , ctx_(cfg_) { + rm_array(); + throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); + memory_tracker_ = tiledb::test::create_test_memory_tracker(); +} + +template +CurrentDomainFx::~CurrentDomainFx() { + rm_array(); +} + +template +void CurrentDomainFx::rm_array() { + bool is_dir; + throw_if_not_ok(ctx_.resources().vfs().is_dir(uri_, &is_dir)); + if (is_dir) { + throw_if_not_ok(ctx_.resources().vfs().remove_dir(uri_)); + } +} + +template +shared_ptr CurrentDomainFx::create_current_domain( + const NDRange& ranges, + shared_ptr schema, + shared_ptr ndrectangle, + bool empty) { + // create current domain + auto current_domain = make_shared( + memory_tracker_, constants::current_domain_version); + if (empty) { + return current_domain; + } + + if (ndrectangle == nullptr) { + // create ndrectangle + ndrectangle = make_shared( + memory_tracker_, schema->shared_domain(), ranges); + } + + // set ndrectangle to current domain + current_domain->set_ndrectangle(ndrectangle); + + return current_domain; +} + +template +void CurrentDomainFx::check_storage_serialization( + shared_ptr schema, const NDRange& ranges) { + auto current_domain = create_current_domain(ranges, schema); + + auto tile = serialize_to_tile(current_domain); + REQUIRE(tile->size() == calculate_serialized_size(current_domain)); + + Deserializer deserializer(tile->data(), tile->size()); + auto deserialized = CurrentDomain::deserialize( + deserializer, memory_tracker_, schema->shared_domain()); + + check_current_domains_equal(deserialized, current_domain); +} + +template +void CurrentDomainFx::check_current_domains_equal( + shared_ptr s1, shared_ptr s2) { + REQUIRE(s1->empty() == s2->empty()); + REQUIRE(s1->type() == s2->type()); + REQUIRE(s1->version() == s2->version()); + REQUIRE( + s1->ndrectangle()->get_ndranges() == s2->ndrectangle()->get_ndranges()); +} + +template +storage_size_t CurrentDomainFx::calculate_serialized_size( + shared_ptr current_domain) { + storage_size_t num_bytes = 0; + + // uint32_t - version + num_bytes += sizeof(uint32_t); + + // bool - empty current domain flag + num_bytes += sizeof(bool); + + if (current_domain->empty()) { + return num_bytes; + } + + // uint8_t - type + num_bytes += sizeof(uint8_t); + + auto ndrectangle = current_domain->ndrectangle(); + for (auto& range : ndrectangle->get_ndranges()) { + if (range.var_size()) { + // Range length + num_bytes += sizeof(uint64_t); + // Minimum value length + num_bytes += sizeof(uint64_t); + } + num_bytes += range.size(); + } + + return num_bytes; +} + +template +shared_ptr CurrentDomainFx::serialize_to_tile( + shared_ptr current_domain) { + SizeComputationSerializer size_serializer; + current_domain->serialize(size_serializer); + + auto tile{WriterTile::from_generic(size_serializer.size(), memory_tracker_)}; + Serializer serializer(tile->data(), tile->size()); + current_domain->serialize(serializer); + + return tile; +} + +template +shared_ptr CurrentDomainFx::create_schema() { + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); + + auto dom = make_shared(HERE(), memory_tracker_); + + auto dim = + make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dim2 = + make_shared(HERE(), "dim2", Datatype::INT32, memory_tracker_); + throw_if_not_ok(dim2->set_domain(range)); + + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(dom->add_dimension(dim2)); + + throw_if_not_ok(schema->set_domain(dom)); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + throw_if_not_ok(schema->add_attribute(attr1)); + + return schema; +} + +template +shared_ptr CurrentDomainFx::create_schema_var() { + auto schema = + make_shared(HERE(), ArrayType::SPARSE, memory_tracker_); + auto dom = make_shared(HERE(), memory_tracker_); + auto dim = make_shared( + HERE(), "dim1", Datatype::STRING_ASCII, memory_tracker_); + auto dim2 = make_shared( + HERE(), "dim2", Datatype::STRING_ASCII, memory_tracker_); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(dom->add_dimension(dim2)); + + throw_if_not_ok(schema->set_domain(dom)); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + + throw_if_not_ok(schema->add_attribute(attr1)); + + return schema; +} + +template +void CurrentDomainFx::create_array(shared_ptr schema) { + throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); +} + +template +shared_ptr CurrentDomainFx::get_array(QueryType type) { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); + return array; +} + +template +shared_ptr CurrentDomainFx::get_array_directory() { + return make_shared( + HERE(), ctx_.resources(), uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); +} + +template +shared_ptr CurrentDomainFx::get_array_schema_latest() { + auto array_dir = get_array_directory(); + return array_dir->load_array_schema_latest(enc_key_, memory_tracker_); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "Create Empty CurrentDomain", + "[current_domain][create][empty]") { + auto schema = create_schema(); + auto current_domain = create_current_domain( + schema->shared_domain()->domain(), schema, nullptr, true); +} + +using FixedVarTypes = std::tuple; +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Create CurrentDomain", + "[current_domain][create]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 1000}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + auto current_domain = this->create_current_domain({r, r}, schema); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check disk serialization works", + "[current_domain][serialization]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 1000}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + this->check_storage_serialization(schema, {r, r}); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "Check array create throws if current domain exceeds array schema domain " + "bounds", + "[current_domain][create][out-of-schema-domain]") { + auto schema = create_schema(); + + auto dom = make_shared(HERE(), memory_tracker_); + auto dim = + make_shared(HERE(), "dim1", Datatype::INT32, memory_tracker_); + int range[2] = {0, 1001}; + throw_if_not_ok(dim->set_domain(range)); + auto dim2 = + make_shared(HERE(), "dim2", Datatype::INT32, memory_tracker_); + throw_if_not_ok(dim2->set_domain(range)); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(dom->add_dimension(dim2)); + + auto current_domain = create_current_domain(dom->domain(), schema); + + // It's fine if an incorrect current domain doesn't throw here, sanity checks + // on schema are performed during array creation when we know schema domain + // can't be changed anymore. + schema->set_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "past the boundaries of the array schema domain"); + + REQUIRE_THROWS_WITH(create_array(schema), matcher); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check array create throws if not all dims are set", + "[current_domain][create][all-dims]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 1000}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto current_domain = this->create_current_domain({r}, schema); + + // It's fine if an incorrect current domain doesn't throw here, sanity checks + // on schema are performed during array creation when we know schema domain + // can't be changed anymore. + schema->set_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "schema have a non-equal number of dimensions"); + + REQUIRE_THROWS_WITH(this->create_array(schema), matcher); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check array create throws if current domain has an empty range", + "[current_domain][create][no-empty-ranges]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 1000}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto ndrectangle = + make_shared(this->memory_tracker_, schema->shared_domain()); + ndrectangle->set_range_for_name(r, "dim1"); + + auto current_domain = + this->create_current_domain({r, r}, schema, ndrectangle); + + // It's fine if an incorrect current domain doesn't throw here, sanity checks + // on schema are performed during array creation when we know schema domain + // can't be changed anymore. + schema->set_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "no range specified for dimension idx"); + + REQUIRE_THROWS_WITH(this->create_array(schema), matcher); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "Check NDRectangle verifies bounds for dim indices", + "[current_domain][ndrectangle][index-bounds]") { + auto schema = create_schema(); + + auto ndrectangle = + make_shared(memory_tracker_, schema->shared_domain()); + + auto matcher = Catch::Matchers::ContainsSubstring("out of bounds"); + auto matcher2 = Catch::Matchers::ContainsSubstring("Invalid dimension name"); + + std::vector rdata = {1, 2}; + auto r = Range(rdata.data(), 2 * sizeof(int32_t)); + REQUIRE_THROWS_WITH(ndrectangle->set_range(r, 2), matcher); + REQUIRE_THROWS_WITH(ndrectangle->get_range(2), matcher); + REQUIRE_THROWS_WITH( + ndrectangle->set_range_for_name(r, "nonexistent"), matcher2); + REQUIRE_THROWS_WITH(ndrectangle->get_range_for_name("nonexistent"), matcher2); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check writing/reading current domain to/from array end to end", + "[current_domain][create][end-to-end-current_domain]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {10, 59}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto current_domain = this->create_current_domain({r, r}, schema); + + schema->set_current_domain(current_domain); + + this->create_array(schema); + + auto opened_array = this->get_array(QueryType::READ); + + this->check_current_domains_equal( + current_domain, opened_array->array_schema_latest().get_current_domain()); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Schema evolution, evolving to bigger current domain works", + "[current_domain][evolution][simple]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + std::vector rdata_expanded; + Range r_expanded; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + rdata_expanded = {"ABB", "ZZZ"}; + r_expanded = Range(rdata_expanded[0], rdata_expanded[1]); + } else { + schema = this->create_schema(); + rdata = {1, 50}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + rdata_expanded = {1, 55}; + r_expanded = Range(rdata_expanded.data(), 2 * sizeof(int32_t)); + } + + auto current_domain = this->create_current_domain({r, r}, schema); + schema->set_current_domain(current_domain); + this->create_array(schema); + auto opened_array = this->get_array(QueryType::READ); + + auto ase = make_shared(HERE(), this->memory_tracker_); + auto orig_schema = opened_array->array_schema_latest_ptr(); + + auto current_domain_expanded = + this->create_current_domain({r_expanded, r_expanded}, orig_schema); + + // Check current domain expansion doesn't throw + ase->expand_current_domain(current_domain_expanded); + auto evolved_schema = ase->evolve_schema(orig_schema); + + // Persist evolved schema and read it back, check it shows the correct + // rectangle + throw_if_not_ok(Array::evolve_array_schema( + this->ctx_.resources(), this->uri_, ase.get(), this->enc_key_)); + + // Read it back + auto new_schema = this->get_array_schema_latest(); + + REQUIRE( + new_schema->get_current_domain()->ndrectangle()->get_ndranges() == + NDRange{{r_expanded, r_expanded}}); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Schema evolution, contracting current domain throws", + "[current_domain][evolution][contraction]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + std::vector rdata_contracted; + Range r_contracted; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + rdata_contracted = {"ABD", "ZZZ"}; + r_contracted = Range(rdata_contracted[0], rdata_contracted[1]); + } else { + schema = this->create_schema(); + rdata = {1, 50}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + rdata_contracted = {1, 45}; + r_contracted = Range(rdata_contracted.data(), 2 * sizeof(int32_t)); + } + + auto current_domain = this->create_current_domain({r, r}, schema); + schema->set_current_domain(current_domain); + + auto ase = make_shared(HERE(), this->memory_tracker_); + + auto current_domain_contracted = + this->create_current_domain({r_contracted, r_contracted}, schema); + + // It's fine if a contracted current domain doesn't throw here, sanity checks + // are performed when evolution is applied and the schema is available. + ase->expand_current_domain(current_domain_contracted); + + auto matcher = Catch::Matchers::ContainsSubstring("can only be expanded"); + REQUIRE_THROWS_WITH(ase->evolve_schema(schema), matcher); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "Schema evolution, empty current domain evolution not allowed", + "[current_domain][evolution][empty_new]") { + auto schema = create_schema(); + + auto ase = make_shared(HERE(), memory_tracker_); + + auto empty_current_domain = create_current_domain({}, schema, nullptr, true); + + auto matcher = Catch::Matchers::ContainsSubstring( + "the new current domain specified is empty"); + REQUIRE_THROWS_WITH( + ase->expand_current_domain(empty_current_domain), matcher); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check you can evolve from an empty current domain schema", + "[current_domain][evolution][empty_existing]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 50}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto ase = make_shared(HERE(), this->memory_tracker_); + + auto current_domain = this->create_current_domain({r, r}, schema); + + ase->expand_current_domain(current_domain); + + CHECK_NOTHROW(ase->evolve_schema(schema)); +} + +TEST_CASE_METHOD( + CurrentDomainFx, + "Check array evolution throws if new current domain exceeds array schema " + "domain bounds", + "[current_domain][evolution][out-of-schema-domain]") { + auto schema = create_schema(); + + auto ase = make_shared(HERE(), memory_tracker_); + + std::vector rdata = {1, 1001}; + auto r = Range(rdata.data(), 2 * sizeof(int32_t)); + auto current_domain = create_current_domain({r, r}, schema); + + ase->expand_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "past the boundaries of the array schema domain"); + REQUIRE_THROWS_WITH(ase->evolve_schema(schema), matcher); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check array evolution throws if not all dims are set for the new " + "current domain", + "[current_domain][evolution][all-dims]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 50}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto ase = make_shared(HERE(), this->memory_tracker_); + + auto current_domain = this->create_current_domain({r}, schema); + + ase->expand_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "schema have a non-equal number of dimensions"); + REQUIRE_THROWS_WITH(ase->evolve_schema(schema), matcher); +} + +TEMPLATE_LIST_TEST_CASE_METHOD( + CurrentDomainFx, + "Check array evolution throws if new current domain has an empty range", + "[current_domain][evolution][no-empty-ranges]", + FixedVarTypes) { + shared_ptr schema; + std::vector rdata; + Range r; + if constexpr (std::is_same_v) { + schema = this->create_schema_var(); + rdata = {"ABC", "ZYZ"}; + r = Range(rdata[0], rdata[1]); + } else { + schema = this->create_schema(); + rdata = {1, 50}; + r = Range(rdata.data(), 2 * sizeof(int32_t)); + } + + auto ase = make_shared(HERE(), this->memory_tracker_); + + auto ndrectangle = + make_shared(this->memory_tracker_, schema->shared_domain()); + ndrectangle->set_range_for_name(r, "dim1"); + auto current_domain = this->create_current_domain({}, schema, ndrectangle); + + ase->expand_current_domain(current_domain); + + auto matcher = Catch::Matchers::ContainsSubstring( + "no range specified for dimension idx"); + REQUIRE_THROWS_WITH(ase->evolve_schema(schema), matcher); +} diff --git a/tiledb/sm/c_api/tiledb_enum.h b/tiledb/sm/c_api/tiledb_enum.h index 9fb10866c03e..016809bc5d92 100644 --- a/tiledb/sm/c_api/tiledb_enum.h +++ b/tiledb/sm/c_api/tiledb_enum.h @@ -132,3 +132,7 @@ /** application/pdf*/ TILEDB_MIME_TYPE_ENUM(MIME_PDF) = 2, #endif + +#ifdef TILEDB_CURRENT_DOMAIN_TYPE_ENUM + TILEDB_CURRENT_DOMAIN_TYPE_ENUM(NDRECTANGLE) = 0, +#endif diff --git a/tiledb/sm/enums/current_domain_type.h b/tiledb/sm/enums/current_domain_type.h new file mode 100644 index 000000000000..38c9f8492b91 --- /dev/null +++ b/tiledb/sm/enums/current_domain_type.h @@ -0,0 +1,80 @@ +/** + * @file current_domain_type.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This defines the TileDB CurrentDomainType enum that maps to + * tiledb_current_domain_type_t C-API enum. + */ + +#ifndef TILEDB_CURRENT_DOMAIN_TYPE_H +#define TILEDB_CURRENT_DOMAIN_TYPE_H + +#include +#include "tiledb/common/status.h" +#include "tiledb/sm/misc/constants.h" + +using namespace tiledb::common; + +namespace tiledb { +namespace sm { + +/** A current domain type. */ +enum class CurrentDomainType : uint8_t { +#define TILEDB_CURRENT_DOMAIN_TYPE_ENUM(id) id +#include "tiledb/sm/c_api/tiledb_enum.h" +#undef TILEDB_CURRENT_DOMAIN_TYPE_ENUM +}; + +/** Returns the string representation of the input current domain type. */ +inline const std::string& current_domain_type_str( + CurrentDomainType current_domain_type) { + switch (current_domain_type) { + case CurrentDomainType::NDRECTANGLE: + return constants::current_domain_ndrectangle_str; + default: + return constants::empty_str; + } +} + +/** Returns the current domain type given a string representation. */ +inline Status current_domain_type_enum( + const std::string& current_domain_type_str, + CurrentDomainType* current_domain_type) { + if (current_domain_type_str == constants::current_domain_ndrectangle_str) + *current_domain_type = CurrentDomainType::NDRECTANGLE; + else { + return Status_Error( + "Invalid CurrentDomain type " + current_domain_type_str); + } + return Status::Ok(); +} + +} // namespace sm +} // namespace tiledb + +#endif // TILEDB_CURRENT_DOMAIN_TYPE_H diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index c43e4aae9125..826b15993361 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -40,6 +40,7 @@ #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/ndrectangle.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/fragment/fragment_identifier.h" @@ -2158,24 +2159,9 @@ void FragmentMetadata::load_non_empty_domain_v5_or_higher( char null_non_empty_domain = 0; null_non_empty_domain = deserializer.read(); - auto& domain{array_schema_->domain()}; if (null_non_empty_domain == 0) { - auto dim_num = array_schema_->dim_num(); - non_empty_domain_.resize(dim_num); - for (unsigned d = 0; d < dim_num; ++d) { - auto dim{domain.dimension_ptr(d)}; - if (!dim->var_size()) { // Fixed-sized - auto r_size = 2 * dim->coord_size(); - non_empty_domain_[d] = - Range(deserializer.get_ptr(r_size), r_size); - } else { // Var-sized - uint64_t r_size, start_size; - r_size = deserializer.read(); - start_size = deserializer.read(); - non_empty_domain_[d] = - Range(deserializer.get_ptr(r_size), r_size, start_size); - } - } + non_empty_domain_ = NDRectangle::deserialize_ndranges( + deserializer, array_schema_->shared_domain()); } // Get expanded domain @@ -2725,20 +2711,9 @@ void FragmentMetadata::write_non_empty_domain(Serializer& serializer) const { std::vector d(domain_size, 0); serializer.write(&d[0], domain_size); } else { - // Write non-empty domain - for (unsigned d = 0; d < dim_num; ++d) { - auto dim{domain.dimension_ptr(d)}; - const auto& r = non_empty_domain_[d]; - if (!dim->var_size()) { // Fixed-sized - serializer.write(r.data(), r.size()); - } else { // Var-sized - auto r_size = r.size(); - auto r_start_size = r.start_size(); - serializer.write(r_size); - serializer.write(r_start_size); - serializer.write(r.data(), r_size); - } - } + NDRectangle( + memory_tracker_, array_schema_->shared_domain(), non_empty_domain_) + .serialize(serializer); } } diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 7c575f77b532..8feb0542c065 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -691,7 +691,7 @@ const int32_t library_version[3] = { TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH}; /** The TileDB serialization base format version number. */ -const format_version_t base_format_version = 21; +const format_version_t base_format_version = 22; /** * The TileDB serialization format version number. @@ -725,6 +725,15 @@ const format_version_t enumerations_min_format_version = 20; /** The current enumerations version. */ const format_version_t enumerations_version = 0; +/** The lowest version supported format version for CurrentDomain API. */ +const format_version_t current_domain_min_format_version = 22; + +/** The current CurrentDomain API version. */ +const format_version_t current_domain_version = 0; + +/** The NDRectangle current domain */ +const std::string current_domain_ndrectangle_str = "NDRECTANGLE"; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ const uint64_t max_tile_chunk_size = 64 * 1024; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index 4e23f45eabaf..c675b8de4d35 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -709,6 +709,15 @@ extern const format_version_t enumerations_min_format_version; /** The current Enumerations version. */ extern const format_version_t enumerations_version; +/** The lowest version supported format version for CurrentDomain API. */ +extern const format_version_t current_domain_min_format_version; + +/** The current CurrentDomain API version. */ +extern const format_version_t current_domain_version; + +/** The NDRectangle current_domain */ +extern const std::string current_domain_ndrectangle_str; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ extern const uint64_t max_tile_chunk_size; diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 5f3bc76d3a7c..dda5ccf47b9a 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -40,6 +40,7 @@ #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" @@ -912,6 +913,8 @@ Status array_schema_to_capnp( } } + // TODO: to add actual wire CurrentDomain (ch48253) + return Status::Ok(); } @@ -1151,6 +1154,9 @@ shared_ptr array_schema_from_capnp( cell_var_offsets_filters, cell_validity_filters, coords_filters, + // TODO: to be changed to actual wire CurrentDomain (ch48253) + make_shared( + memory_tracker, constants::current_domain_version), memory_tracker); } diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index 36a3c329fec9..b21c18accdab 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -41,6 +41,7 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" #include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/current_domain.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/array_schema/enumeration.h" @@ -152,6 +153,8 @@ Status array_schema_evolution_to_capnp( timestamp_builder.set(0, timestamp_range.first); timestamp_builder.set(1, timestamp_range.second); + // TODO: to add actual wire CurrentDomain (ch48253) + return Status::Ok(); } @@ -215,6 +218,8 @@ tdb_unique_ptr array_schema_evolution_from_capnp( enmrs_to_extend, enmrs_to_drop, ts_range, + // TODO: to add actual wire CurrentDomain (ch48253) + nullptr, memory_tracker)); } From b3b2e7f0ba8997e746e642e6b889804a0207d80a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:09:10 -0400 Subject: [PATCH 413/456] Remove tiledb/common/thread_pool.h (#5055) Remove `tiledb/common/thread_pool.h`. Replace all inclusions this file with `#include tiledb/common/thread_pool/thread_pool.h`, which was the body of the removed file. [sc-48271] --- TYPE: NO_HISTORY DESC: Remove `tiledb/common/thread_pool.h`. --- test/src/unit-s3.cc | 2 +- tiledb/common/thread_pool.h | 29 ------------------- .../thread_pool/test/unit_thread_pool.cc | 4 +-- tiledb/common/thread_pool/thread_pool.cc | 4 +-- tiledb/sm/array/array_directory.h | 2 +- tiledb/sm/filesystem/azure.h | 13 ++++----- tiledb/sm/filesystem/gcs.h | 4 +-- tiledb/sm/filesystem/path_win.h | 4 +-- tiledb/sm/filesystem/s3.h | 4 +-- .../sm/filesystem/s3_thread_pool_executor.h | 10 +++---- tiledb/sm/filter/filter_pipeline.h | 10 +++---- tiledb/sm/global_state/global_state.cc | 12 +++----- tiledb/sm/group/group_directory.h | 2 +- tiledb/sm/misc/cancelable_tasks.cc | 10 +++---- tiledb/sm/misc/cancelable_tasks.h | 10 +++---- tiledb/sm/misc/parallel_functions.h | 10 +++---- tiledb/sm/rest/rest_client.h | 2 +- tiledb/sm/serialization/query.h | 10 +++---- tiledb/sm/serialization/query_plan.cc | 4 +-- .../storage_manager_canonical.h | 2 +- .../sm/subarray/relevant_fragment_generator.h | 12 ++++---- tiledb/sm/subarray/subarray.h | 4 +-- tiledb/sm/subarray/subarray_partitioner.h | 10 +++---- tiledb/sm/subarray/test/unit_range_subset.cc | 4 +-- 24 files changed, 63 insertions(+), 115 deletions(-) delete mode 100644 tiledb/common/thread_pool.h diff --git a/test/src/unit-s3.cc b/test/src/unit-s3.cc index dec8e72dbe52..922507ced52f 100644 --- a/test/src/unit-s3.cc +++ b/test/src/unit-s3.cc @@ -35,7 +35,7 @@ #include #include "test/support/src/helpers.h" #include "test/support/src/vfs_helpers.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/s3.h" #include "tiledb/sm/global_state/unit_test_config.h" diff --git a/tiledb/common/thread_pool.h b/tiledb/common/thread_pool.h deleted file mode 100644 index fae0b874725c..000000000000 --- a/tiledb/common/thread_pool.h +++ /dev/null @@ -1,29 +0,0 @@ -/** - * @file thread_pool.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "tiledb/common/thread_pool/thread_pool.h" diff --git a/tiledb/common/thread_pool/test/unit_thread_pool.cc b/tiledb/common/thread_pool/test/unit_thread_pool.cc index 9af3936d21ed..5220f9be8a99 100644 --- a/tiledb/common/thread_pool/test/unit_thread_pool.cc +++ b/tiledb/common/thread_pool/test/unit_thread_pool.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ #include #include -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/misc/cancelable_tasks.h" // Fixed seed for determinism. diff --git a/tiledb/common/thread_pool/thread_pool.cc b/tiledb/common/thread_pool/thread_pool.cc index 19d44a957356..b026262a3c34 100644 --- a/tiledb/common/thread_pool/thread_pool.cc +++ b/tiledb/common/thread_pool/thread_pool.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,7 +36,7 @@ #include #include "tiledb/common/logger.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" namespace tiledb::common { diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index 5509d1900d77..3016109a8b85 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -34,7 +34,7 @@ #define TILEDB_ARRAY_DIRECTORY_H #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filesystem/vfs.h" diff --git a/tiledb/sm/filesystem/azure.h b/tiledb/sm/filesystem/azure.h index c1889da60dca..f19e37797537 100644 --- a/tiledb/sm/filesystem/azure.h +++ b/tiledb/sm/filesystem/azure.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,7 +38,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/filesystem/ssl_config.h" @@ -58,13 +58,11 @@ class BlobServiceClient; using namespace tiledb::common; -namespace tiledb { - -namespace common::filesystem { +namespace tiledb::common::filesystem { class directory_entry; } -namespace sm { +namespace tiledb::sm { /** Class for Azure status exceptions. */ class AzureException : public StatusException { @@ -925,8 +923,7 @@ AzureScanner::fetch_results() { return begin_; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // HAVE_AZURE #endif // TILEDB_AZURE_H diff --git a/tiledb/sm/filesystem/gcs.h b/tiledb/sm/filesystem/gcs.h index a89c46a21c96..9df5fefa4f65 100644 --- a/tiledb/sm/filesystem/gcs.h +++ b/tiledb/sm/filesystem/gcs.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,7 +39,7 @@ #include "tiledb/common/rwlock.h" #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/curl/curl_init.h" diff --git a/tiledb/sm/filesystem/path_win.h b/tiledb/sm/filesystem/path_win.h index b7f5c429d5db..872b949accbf 100644 --- a/tiledb/sm/filesystem/path_win.h +++ b/tiledb/sm/filesystem/path_win.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,7 +39,7 @@ #include #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "uri.h" diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index fbb2d18c12a4..64482a953b46 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,7 +41,7 @@ #include "tiledb/common/filesystem/directory_entry.h" #include "tiledb/common/rwlock.h" #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/platform/platform.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" diff --git a/tiledb/sm/filesystem/s3_thread_pool_executor.h b/tiledb/sm/filesystem/s3_thread_pool_executor.h index e1fa43507bd8..c2adfb10a9bb 100644 --- a/tiledb/sm/filesystem/s3_thread_pool_executor.h +++ b/tiledb/sm/filesystem/s3_thread_pool_executor.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2019-2021 TileDB, Inc. + * @copyright Copyright (c) 2019-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -40,12 +40,11 @@ #include #include -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class S3ThreadPoolExecutor : public Aws::Utils::Threading::Executor { public: @@ -94,8 +93,7 @@ class S3ThreadPoolExecutor : public Aws::Utils::Threading::Executor { std::condition_variable cv_; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // HAVE_S3 #endif // TILEDB_S3_THREAD_POOL_EXECUTOR_H diff --git a/tiledb/sm/filter/filter_pipeline.h b/tiledb/sm/filter/filter_pipeline.h index b8a521d451f7..66c6093803f1 100644 --- a/tiledb/sm/filter/filter_pipeline.h +++ b/tiledb/sm/filter/filter_pipeline.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2022 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -39,7 +39,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/filter/filter.h" @@ -50,8 +50,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Buffer; class EncryptionKey; @@ -371,7 +370,6 @@ class FilterPipeline { ThreadPool* const compute_tp) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_FILTER_PIPELINE_H diff --git a/tiledb/sm/global_state/global_state.cc b/tiledb/sm/global_state/global_state.cc index 38120c8e3c0c..d6287c49409f 100644 --- a/tiledb/sm/global_state/global_state.cc +++ b/tiledb/sm/global_state/global_state.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,7 +37,7 @@ #include "tiledb/sm/storage_manager/storage_manager.h" #ifdef __linux__ -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/filesystem/posix.h" #include "tiledb/sm/misc/utils.h" #endif @@ -46,9 +46,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { -namespace global_state { +namespace tiledb::sm::global_state { GlobalState& GlobalState::GetGlobalState() { // This is thread-safe in C++11. @@ -95,6 +93,4 @@ std::set GlobalState::storage_managers() { return storage_managers_; } -} // namespace global_state -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::global_state diff --git a/tiledb/sm/group/group_directory.h b/tiledb/sm/group/group_directory.h index 39cc386bc92b..be5aec059d05 100644 --- a/tiledb/sm/group/group_directory.h +++ b/tiledb/sm/group/group_directory.h @@ -34,7 +34,7 @@ #define TILEDB_GROUP_DIRECTORY_H #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filesystem/vfs.h" diff --git a/tiledb/sm/misc/cancelable_tasks.cc b/tiledb/sm/misc/cancelable_tasks.cc index 09e59debc229..960f98da05dd 100644 --- a/tiledb/sm/misc/cancelable_tasks.cc +++ b/tiledb/sm/misc/cancelable_tasks.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,12 +31,11 @@ */ #include "tiledb/sm/misc/cancelable_tasks.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { CancelableTasks::CancelableTasks() : outstanding_tasks_(0) @@ -101,5 +100,4 @@ Status CancelableTasks::fn_wrapper( } } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm diff --git a/tiledb/sm/misc/cancelable_tasks.h b/tiledb/sm/misc/cancelable_tasks.h index e586cc4a07eb..fc38c39d65cb 100644 --- a/tiledb/sm/misc/cancelable_tasks.h +++ b/tiledb/sm/misc/cancelable_tasks.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,12 +38,11 @@ #include #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class CancelableTasks { public: @@ -102,7 +101,6 @@ class CancelableTasks { bool should_cancel_; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_CANCELABLE_TASKS_H diff --git a/tiledb/sm/misc/parallel_functions.h b/tiledb/sm/misc/parallel_functions.h index a30b64234688..4cfc6ff141a7 100644 --- a/tiledb/sm/misc/parallel_functions.h +++ b/tiledb/sm/misc/parallel_functions.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -33,7 +33,7 @@ #ifndef TILEDB_PARALLEL_FUNCTIONS_H #define TILEDB_PARALLEL_FUNCTIONS_H -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/global_state/global_state.h" #include @@ -41,8 +41,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** * Sort the given iterator range, possibly in parallel. @@ -401,7 +400,6 @@ Status parallel_for_2d( return return_st; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_PARALLEL_FUNCTIONS_H diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 72fb1ad20df6..0f7cfae63f5f 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -37,7 +37,7 @@ #include #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/group/group.h" #include "tiledb/sm/serialization/query.h" #include "tiledb/sm/stats/stats.h" diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index a5ad85c03482..6f6ede1e0596 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2022 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,7 +36,7 @@ #include #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/subarray.h" @@ -47,8 +47,7 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class Buffer; @@ -282,7 +281,6 @@ Status query_from_capnp( #endif } // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SERIALIZATION_QUERY_H diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc index 9c5caea9a5fd..6b604566cbe3 100644 --- a/tiledb/sm/serialization/query_plan.cc +++ b/tiledb/sm/serialization/query_plan.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -41,7 +41,7 @@ #include "tiledb/sm/query_plan/query_plan.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/layout.h" diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index f024abd2f765..67ab83edb02a 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -48,7 +48,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/enums/walk_order.h" #include "tiledb/sm/filesystem/uri.h" diff --git a/tiledb/sm/subarray/relevant_fragment_generator.h b/tiledb/sm/subarray/relevant_fragment_generator.h index 362c7038c632..769995d8eefd 100644 --- a/tiledb/sm/subarray/relevant_fragment_generator.h +++ b/tiledb/sm/subarray/relevant_fragment_generator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -34,14 +34,13 @@ #define TILEDB_RELEVANT_FRAGMENTS_GENERATOR_H #include "tiledb/common/common.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class OpenedArray; class RelevantFragments; @@ -151,8 +150,7 @@ class RelevantFragmentGenerator { /** Reference to the subarray. */ const Subarray& subarray_; -}; // namespace sm -} // namespace sm -} // namespace tiledb +}; +} // namespace tiledb::sm #endif // TILEDB_RELEVANT_FRAGMENTS_GENERATOR_H diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index c9e5d89c7769..453dd13f51a7 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2023 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -36,7 +36,7 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/config/config.h" #include "tiledb/sm/enums/datatype.h" diff --git a/tiledb/sm/subarray/subarray_partitioner.h b/tiledb/sm/subarray/subarray_partitioner.h index e3df1540eb1c..f7458f4f7b33 100644 --- a/tiledb/sm/subarray/subarray_partitioner.h +++ b/tiledb/sm/subarray/subarray_partitioner.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,15 +37,14 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/stats/stats.h" #include "tiledb/sm/subarray/subarray.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { /** * Iterates over partitions of a subarray in a way that the results @@ -565,7 +564,6 @@ class SubarrayPartitioner { ByteVecValue* splitting_value) const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_SUBARRAY_PARTITIONER_H diff --git a/tiledb/sm/subarray/test/unit_range_subset.cc b/tiledb/sm/subarray/test/unit_range_subset.cc index fadf46b60bb7..18d53e456367 100644 --- a/tiledb/sm/subarray/test/unit_range_subset.cc +++ b/tiledb/sm/subarray/test/unit_range_subset.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,7 +32,7 @@ #include #include "../range_subset.h" -#include "tiledb/common/thread_pool.h" +#include "tiledb/common/thread_pool/thread_pool.h" #include "tiledb/type/range/range.h" using namespace tiledb; From f99c3f51adc684b894c2df8e2aaf3b4d7fba3b02 Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:14:24 +0300 Subject: [PATCH 414/456] Fix partial tile offsets loading testing in Windows nightlies. (#5058) Fix nightly failures for partial offsets testing by slightly increasing the budget. Failure here: https://github.com/TileDB-Inc/TileDB/actions/runs/9410904346/job/25923315811#step:7:4571 --- TYPE: NO_HISTORY DESC: Fix partial tile offsets loading testing in Windows nightlies. --- test/src/unit-sparse-unordered-with-dups-reader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit-sparse-unordered-with-dups-reader.cc b/test/src/unit-sparse-unordered-with-dups-reader.cc index 470bbfb468aa..3b73642ea87c 100644 --- a/test/src/unit-sparse-unordered-with-dups-reader.cc +++ b/test/src/unit-sparse-unordered-with-dups-reader.cc @@ -1002,7 +1002,7 @@ TEST_CASE_METHOD( write_1d_fragment(coords4.data(), &coords4_size, data4.data(), &data4_size); } - total_budget_ = "3000000"; + total_budget_ = "3300000"; ratio_array_data_ = "0.002"; partial_tile_offsets_loading_ = enable_partial_tile_offsets_loading ? "true" : "false"; From 3bb06997df6914ac661e5aa9a2472d84f2440fcf Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 11 Jun 2024 01:24:15 +0300 Subject: [PATCH 415/456] Add temporary mitigations for CI failures in `windows-2022`. (#5057) [SC-48975](https://app.shortcut.com/tiledb-inc/story/48975) Visual Studio 2022 17.10 has been recently released and one of its changes to MSVC is that the constructor of `std::mutex` became `constexpr`. According to MSVC's binary compatibility guarantees, a program must use a Visual C++ redistributable version at least as new as the toolset that compiled the program. However, due to [a defect](https://github.com/actions/runner-images/issues/10004) on the GitHub Actions `windows-2022` image, an old redistributable version was picked, causing weird failures in CI. This PR fixes these failures, by defining the `_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR` symbol, which reverts the change to `std::mutex`. Due to unexplained errors, I could not do that for the standalone unit test workflow, and instead I changed that to use the `windows-2019` image which does not exhibit this problem. This PR should be reverted once the issue on GHA gets fixed. --- TYPE: NO_HISTORY --- .github/workflows/build-windows.yml | 1 + .github/workflows/unit-test-runs.yml | 8 +++++++- ports/triplets/x64-windows-asan.cmake | 4 ++++ ports/triplets/x64-windows.cmake | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index aebad0e24cc1..e35f76a9f6b5 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -61,6 +61,7 @@ jobs: TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' + CXXFLAGS: '/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR' # https://github.com/actions/runner-images/issues/10004 steps: - name: 'tiledb env prep' run: | diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index 0b1bfce01f39..baa5fb1762b0 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -10,7 +10,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + # Temporarily reverting to windows-2019 until https://github.com/actions/runner-images/issues/10004 gets fixed. + os: [macos-latest, ubuntu-latest, windows-2019] fail-fast: false name: Build - ${{ matrix.os }} timeout-minutes: 90 @@ -53,6 +54,11 @@ jobs: shell: bash if: ${{ !startsWith(matrix.os, 'windows-') }} + - name: 'Print env' + run: set + shell: cmd + if: ${{ startsWith(matrix.os, 'windows-') }} + - name: 'Build standalone unit tests' run: | cmake -S . \ diff --git a/ports/triplets/x64-windows-asan.cmake b/ports/triplets/x64-windows-asan.cmake index d19486d2cfed..2d1e356817c9 100644 --- a/ports/triplets/x64-windows-asan.cmake +++ b/ports/triplets/x64-windows-asan.cmake @@ -7,3 +7,7 @@ set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) # bigobj is needed for capnp. set(VCPKG_C_FLAGS "/fsanitize=address /bigobj") set(VCPKG_CXX_FLAGS "/fsanitize=address /bigobj") + +# https://github.com/actions/runner-images/issues/10004 +set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") +set(VCPKG_CXX_FLAGS "${VCPKG_CXX_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") diff --git a/ports/triplets/x64-windows.cmake b/ports/triplets/x64-windows.cmake index c87b4f5dd385..eb54e7f5a55f 100644 --- a/ports/triplets/x64-windows.cmake +++ b/ports/triplets/x64-windows.cmake @@ -2,4 +2,8 @@ set(VCPKG_TARGET_ARCHITECTURE x64) set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_LIBRARY_LINKAGE static) +# https://github.com/actions/runner-images/issues/10004 +set(VCPKG_C_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") +set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") + set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) From 14274ab459bcde282f0e0c027ae9506f73d9b53d Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Tue, 11 Jun 2024 04:13:10 -0400 Subject: [PATCH 416/456] Remove GHA REST CI. (#5061) The authentication for GHA REST CI job has expired causing recent failures. REST CI was migrated to GitLab in #4740, so this PR removes running REST CI on GHA. The GHA REST CI job only runs on push (merge) to `dev` or release branches so I merged on my fork to kick off a full CI run without this job to test. https://github.com/shaunrd0/TileDB/actions/runs/9451126638/job/26031334343 [sc-49272] --- TYPE: NO_HISTORY DESC: Remove GHA REST CI. Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- .github/workflows/ci-rest.yml | 43 ----------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/workflows/ci-rest.yml diff --git a/.github/workflows/ci-rest.yml b/.github/workflows/ci-rest.yml deleted file mode 100644 index 327788a1208a..000000000000 --- a/.github/workflows/ci-rest.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: REST CI - -on: - workflow_call: - workflow_dispatch: - inputs: - ref: - type: string - required: false - - push: - branches: - - dev - - release-* - - refs/tags/* - -jobs: - rest-ci: - runs-on: ubuntu-latest - - steps: - # For easy access to lookup dispatched CI job. - - name: Print URL for REST CI actions - run: echo https://github.com/TileDB-Inc/TileDB-Internal/actions - - # If this workflow fails on the remote repository, this CI job will also fail. - - name: Workflow dispatch to REST CI - id: trigger-step - uses: aurelien-baudet/workflow-dispatch@v2 - env: - TILEDB_REST_CI_PAT: ${{ secrets.TILEDB_REST_CI_PAT }} - # Skip if no PAT is set (e.g. for PRs from forks). - if: env.TILEDB_REST_CI_PAT != null - with: - repo: TileDB-Inc/TileDB-Internal - # Trigger workflow on TileDB-Internal at this ref. - ref: "main" - workflow: full-ci.yml - token: ${{ secrets.TILEDB_REST_CI_PAT }} - # Pass TileDB core ref to test against REST. - # github.head_ref will only be set for PRs, so fallback to github.ref_name if triggered via push event. - inputs: '{ "tiledb_ref": "${{ github.event.inputs.ref || github.head_ref || github.ref_name }}"}' - wait-for-completion-timeout: 90m From 875f8f45d77a8fcafbcb1e06faf88c9a9e569e8c Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 11 Jun 2024 11:14:00 +0300 Subject: [PATCH 417/456] Support setting S3 storage class. (#5053) [SC-48305](https://app.shortcut.com/tiledb-inc/story/48305) [SC-48552](https://app.shortcut.com/tiledb-inc/story/48552/arrays-support-s3-storage-classes) This PR adds a config option to support setting the storage class of the objects that get uploaded to S3. This allows users to seamlessly take advantage of features like [Amazon S3 Intelligent-Tiering](https://aws.amazon.com/s3/storage-classes/intelligent-tiering/). --- TYPE: CONFIG DESC: Add `vfs.s3.storage_class` config option to set the storage class of newly uploaded S3 objects. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- test/src/unit-capi-config.cc | 4 ++ test/src/unit-cppapi-config.cc | 2 +- tiledb/api/c_api/config/config_api_external.h | 16 ++++++ tiledb/sm/config/config.cc | 2 + tiledb/sm/config/config.h | 3 + tiledb/sm/cpp_api/config.h | 17 ++++++ tiledb/sm/filesystem/s3.cc | 57 +++++++++++++++++++ tiledb/sm/filesystem/s3.h | 8 +++ 8 files changed, 108 insertions(+), 1 deletion(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index ca4269d88a8b..5f8abc0260d8 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -340,6 +340,7 @@ void check_save_to_file() { ss << "vfs.s3.requester_pays false\n"; ss << "vfs.s3.scheme https\n"; ss << "vfs.s3.skip_init false\n"; + ss << "vfs.s3.storage_class NOT_SET\n"; ss << "vfs.s3.use_multipart_upload true\n"; ss << "vfs.s3.use_virtual_addressing true\n"; ss << "vfs.s3.verify_ssl true\n"; @@ -739,6 +740,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["vfs.s3.connect_scale_factor"] = "25"; all_param_values["vfs.s3.sse"] = ""; all_param_values["vfs.s3.sse_kms_key_id"] = ""; + all_param_values["vfs.s3.storage_class"] = "NOT_SET"; all_param_values["vfs.s3.logging_level"] = "Off"; all_param_values["vfs.s3.request_timeout_ms"] = "3000"; all_param_values["vfs.s3.requester_pays"] = "false"; @@ -812,6 +814,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { vfs_param_values["s3.connect_scale_factor"] = "25"; vfs_param_values["s3.sse"] = ""; vfs_param_values["s3.sse_kms_key_id"] = ""; + vfs_param_values["s3.storage_class"] = "NOT_SET"; vfs_param_values["s3.logging_level"] = "Off"; vfs_param_values["s3.request_timeout_ms"] = "3000"; vfs_param_values["s3.requester_pays"] = "false"; @@ -880,6 +883,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { s3_param_values["connect_scale_factor"] = "25"; s3_param_values["sse"] = ""; s3_param_values["sse_kms_key_id"] = ""; + s3_param_values["storage_class"] = "NOT_SET"; s3_param_values["logging_level"] = "Off"; s3_param_values["request_timeout_ms"] = "3000"; s3_param_values["requester_pays"] = "false"; diff --git a/test/src/unit-cppapi-config.cc b/test/src/unit-cppapi-config.cc index d84e9996b1b6..b6e5d6d1151f 100644 --- a/test/src/unit-cppapi-config.cc +++ b/test/src/unit-cppapi-config.cc @@ -75,7 +75,7 @@ TEST_CASE("C++ API: Config iterator", "[cppapi][config]") { names.push_back(it->first); } // Check number of VFS params in default config object. - CHECK(names.size() == 68); + CHECK(names.size() == 69); } TEST_CASE("C++ API: Config Environment Variables", "[cppapi][config]") { diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 5e0e8c6b3121..161a5a75ae6e 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -569,6 +569,22 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * The server-side encryption key to use if * vfs.s3.sse == "kms" (AWS key management service).
* **Default**: "" + * - `vfs.s3.storage_class`
+ * The storage class to use for the newly uploaded S3 objects. The set of + * accepted values is found in the Aws::S3::Model::StorageClass enumeration. + * "NOT_SET" + * "STANDARD" + * "REDUCED_REDUNDANCY" + * "STANDARD_IA" + * "ONEZONE_IA" + * "INTELLIGENT_TIERING" + * "GLACIER" + * "DEEP_ARCHIVE" + * "OUTPOSTS" + * "GLACIER_IR" + * "SNOW" + * "EXPRESS_ONEZONE"
+ * **Default**: "NOT_SET" * - `vfs.s3.bucket_canned_acl`
* Names of values found in Aws::S3::Model::BucketCannedACL enumeration. * "NOT_SET" diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 69ce8565a852..4a5e677295c8 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -219,6 +219,7 @@ const std::string Config::VFS_S3_CONNECT_MAX_TRIES = "5"; const std::string Config::VFS_S3_CONNECT_SCALE_FACTOR = "25"; const std::string Config::VFS_S3_SSE = ""; const std::string Config::VFS_S3_SSE_KMS_KEY_ID = ""; +const std::string Config::VFS_S3_STORAGE_CLASS = "NOT_SET"; const std::string Config::VFS_S3_REQUEST_TIMEOUT_MS = "3000"; const std::string Config::VFS_S3_REQUESTER_PAYS = "false"; const std::string Config::VFS_S3_PROXY_SCHEME = "http"; @@ -482,6 +483,7 @@ const std::map default_config_values = { "vfs.s3.connect_scale_factor", Config::VFS_S3_CONNECT_SCALE_FACTOR), std::make_pair("vfs.s3.sse", Config::VFS_S3_SSE), std::make_pair("vfs.s3.sse_kms_key_id", Config::VFS_S3_SSE_KMS_KEY_ID), + std::make_pair("vfs.s3.storage_class", Config::VFS_S3_STORAGE_CLASS), std::make_pair( "vfs.s3.request_timeout_ms", Config::VFS_S3_REQUEST_TIMEOUT_MS), std::make_pair("vfs.s3.requester_pays", Config::VFS_S3_REQUESTER_PAYS), diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 2fd464470e40..3dd6fa760f1e 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -561,6 +561,9 @@ class Config { /** The S3 KMS key id for KMS server-side-encryption. */ static const std::string VFS_S3_SSE_KMS_KEY_ID; + /** The S3 storage class to upload objects to. */ + static const std::string VFS_S3_STORAGE_CLASS; + /** Request timeout in milliseconds. */ static const std::string VFS_S3_REQUEST_TIMEOUT_MS; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index 02d8105bc597..cd1c2110ad9c 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -744,6 +744,23 @@ class Config { * The server-side encryption key to use if * vfs.s3.sse == "kms" (AWS key management service).
* **Default**: "" + * - `vfs.s3.storage_class`
+ * The storage class to use for the newly uploaded S3 objects. The set of + * accepted values is found in the Aws::S3::Model::StorageClass + * enumeration. + * "NOT_SET" + * "STANDARD" + * "REDUCED_REDUNDANCY" + * "STANDARD_IA" + * "ONEZONE_IA" + * "INTELLIGENT_TIERING" + * "GLACIER" + * "DEEP_ARCHIVE" + * "OUTPOSTS" + * "GLACIER_IR" + * "SNOW" + * "EXPRESS_ONEZONE"
+ * **Default**: "NOT_SET" * - `vfs.s3.bucket_canned_acl`
* Names of values found in Aws::S3::Model::BucketCannedACL enumeration. * "NOT_SET" diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index b7cc0f31d81f..25315586edee 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -81,6 +81,13 @@ using tiledb::common::filesystem::directory_entry; namespace { +/* + * Functions to convert strings to AWS enums. + * + * The AWS SDK provides some enum conversion functions, but they must not be + * used, because they have non-deterministic behavior in certain scenarios. + */ + Aws::Utils::Logging::LogLevel aws_log_name_to_level(std::string loglevel) { std::transform(loglevel.begin(), loglevel.end(), loglevel.begin(), ::tolower); if (loglevel == "fatal") @@ -157,6 +164,47 @@ Aws::S3::Model::BucketCannedACL S3_BucketCannedACL_from_str( return Aws::S3::Model::BucketCannedACL::NOT_SET; } +/** + * Return a S3 enum value for any recognized string or NOT_SET if + * B) the string is not recognized to match any of the enum values + * + * @param storage_class_str A textual string naming one of the + * Aws::S3::Model::StorageClass enum members. + */ +Aws::S3::Model::StorageClass S3_StorageClass_from_str( + const std::string& storage_class_str) { + using Aws::S3::Model::StorageClass; + if (storage_class_str.empty()) + return StorageClass::NOT_SET; + + if (storage_class_str == "NOT_SET") + return StorageClass::NOT_SET; + else if (storage_class_str == "STANDARD") + return StorageClass::STANDARD; + else if (storage_class_str == "REDUCED_REDUNDANCY") + return StorageClass::REDUCED_REDUNDANCY; + else if (storage_class_str == "STANDARD_IA") + return StorageClass::STANDARD_IA; + else if (storage_class_str == "ONEZONE_IA") + return StorageClass::ONEZONE_IA; + else if (storage_class_str == "INTELLIGENT_TIERING") + return StorageClass::INTELLIGENT_TIERING; + else if (storage_class_str == "GLACIER") + return StorageClass::GLACIER; + else if (storage_class_str == "DEEP_ARCHIVE") + return StorageClass::DEEP_ARCHIVE; + else if (storage_class_str == "OUTPOSTS") + return StorageClass::OUTPOSTS; + else if (storage_class_str == "GLACIER_IR") + return StorageClass::GLACIER_IR; + else if (storage_class_str == "SNOW") + return StorageClass::SNOW; + else if (storage_class_str == "EXPRESS_ONEZONE") + return StorageClass::EXPRESS_ONEZONE; + else + return StorageClass::NOT_SET; +} + } // namespace using namespace tiledb::common; @@ -200,6 +248,7 @@ S3::S3( s3_params_.requester_pays_ ? Aws::S3::Model::RequestPayer::requester : Aws::S3::Model::RequestPayer::NOT_SET) , sse_(Aws::S3::Model::ServerSideEncryption::NOT_SET) + , storage_class_(S3_StorageClass_from_str(s3_params_.storage_class_)) , object_canned_acl_( S3_ObjectCannedACL_from_str(s3_params_.object_acl_str_)) , bucket_canned_acl_( @@ -511,6 +560,10 @@ void S3::touch(const URI& uri) const { if (!s3_params_.sse_kms_key_id_.empty()) put_object_request.SetSSEKMSKeyId( Aws::String(s3_params_.sse_kms_key_id_.c_str())); + // TODO: These checks are not needed since AWS SDK 1.11.275 + // https://github.com/aws/aws-sdk-cpp/pull/2875 + if (storage_class_ != Aws::S3::Model::StorageClass::NOT_SET) + put_object_request.SetStorageClass(storage_class_); if (object_canned_acl_ != Aws::S3::Model::ObjectCannedACL::NOT_SET) { put_object_request.SetACL(object_canned_acl_); } @@ -1554,6 +1607,8 @@ Status S3::initiate_multipart_request( if (!s3_params_.sse_kms_key_id_.empty()) multipart_upload_request.SetSSEKMSKeyId( Aws::String(s3_params_.sse_kms_key_id_.c_str())); + if (storage_class_ != Aws::S3::Model::StorageClass::NOT_SET) + multipart_upload_request.SetStorageClass(storage_class_); if (object_canned_acl_ != Aws::S3::Model::ObjectCannedACL::NOT_SET) { multipart_upload_request.SetACL(object_canned_acl_); } @@ -1748,6 +1803,8 @@ void S3::write_direct(const URI& uri, const void* buffer, uint64_t length) { if (!s3_params_.sse_kms_key_id_.empty()) put_object_request.SetSSEKMSKeyId( Aws::String(s3_params_.sse_kms_key_id_.c_str())); + if (storage_class_ != Aws::S3::Model::StorageClass::NOT_SET) + put_object_request.SetStorageClass(storage_class_); if (object_canned_acl_ != Aws::S3::Model::ObjectCannedACL::NOT_SET) { put_object_request.SetACL(object_canned_acl_); } diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 64482a953b46..deb37c66249f 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -224,6 +224,8 @@ struct S3Parameters { sse_algorithm_ == "kms" ? config.get("vfs.s3.sse_kms_key_id").value() : "") + , storage_class_( + config.get("vfs.s3.storage_class", Config::must_find)) , bucket_acl_str_(config.get( "vfs.s3.bucket_canned_acl", Config::must_find)) , object_acl_str_(config.get( @@ -326,6 +328,9 @@ struct S3Parameters { /** The server-side encryption key to use with the kms algorithm. */ std::string sse_kms_key_id_; + /** The S3 storage class. */ + std::string storage_class_; + /** Names of values found in Aws::S3::Model::BucketCannedACL enumeration. */ std::string bucket_acl_str_; @@ -1377,6 +1382,9 @@ class S3 : FilesystemBase { /** The server-side encryption algorithm. */ Aws::S3::Model::ServerSideEncryption sse_; + /** The storage class for a s3 upload request. */ + Aws::S3::Model::StorageClass storage_class_; + /** Protects file_buffers map */ std::mutex file_buffers_mtx_; From 7f7b15b78624a8fb7897ddfbc199321fcf10eb0c Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:55:42 +0200 Subject: [PATCH 418/456] Dense reader: improve memory consumption for tile structures. (#5046) This change significantly reduces the memory usage for tile structures (result space tile and tile subarray) by doing three things: - Only create the structures for the tiles that will be processed in an internal loop. - Using a structure to store the tile subarray with a smaller memory footprint. - Adding estimation for the tile structure size when determining the tiles to process for an iteration. For the first point, a new class was added: `IterationTileData` to store the tile structurres for an iteration. It is held in a shared pointer so that the compute task can take tile data to finish processing tiles in a thread safe/efficient manner. A problem that arose here is that the tile subarrays were used to get the number of cells when computing the tiles that it can process in an iteration. Since the tile subarrays are heavy to compute, and should be computed in a parallel manner, this posed a problem. To solve this, a new method was added to Subarray to return the cell count of a particular tile (`Subarray::tile_cell_num``). Since the method doesn't allocate any memory, it can run significantly fast and there are no noticeable performance degradation, even in large queries. With this method, we can compute the tiles to process and then compute the tile subarrays later once we know the number of tiles for an iteration. For the second point, a new class was added: `DenseTileSubarray` to store the tile subarrays. This class takes much less memory than `Subarray` and is also much fater to create in memory. Since this is for dense only, the ranges can be types, which also uses less memory as we don't need to store a size. Finally, the computed ranges for a tile can be used in `TileCellSlabIter` as a const reference, which surely improves performance and reduce small memory allocations. For the third bullet point, `compute_result_space_tiles` now tries to estimate the memory for the tile structures as it can be significant when the tile size is small (ie: 1000 cells). I've also changed the code to take into account the filtered data size when computing memory against the memory budget to improve our accuracy. Note that this PR extracted `ReaderBase::compute_tile_domains`, which is common code for the new dense reader and the legacy reader to compute the result space tiles. The rest of the code was split between the legacy reader class and the new dense reader class as their implementations diverged significantly. Finally, I've made a small change to release the filtered data as soon as unfiltering is done to limit the memory footprint even further. [sc-38387] [sc-48388] --- TYPE: IMPROVEMENT DESC: Dense reader: improve memory consumption for tile structures. --- test/src/unit-Subarray.cc | 3 + test/src/unit-cppapi-consolidation.cc | 1 + tiledb/sm/query/legacy/reader.cc | 77 ++++ tiledb/sm/query/legacy/reader.h | 43 +++ tiledb/sm/query/readers/dense_reader.cc | 425 ++++++++++++++-------- tiledb/sm/query/readers/dense_reader.h | 149 +++++--- tiledb/sm/query/readers/reader_base.cc | 157 +++----- tiledb/sm/query/readers/reader_base.h | 53 +-- tiledb/sm/subarray/subarray.cc | 96 +++-- tiledb/sm/subarray/subarray.h | 91 ++++- tiledb/sm/subarray/tile_cell_slab_iter.cc | 38 +- tiledb/sm/subarray/tile_cell_slab_iter.h | 38 +- 12 files changed, 737 insertions(+), 434 deletions(-) diff --git a/test/src/unit-Subarray.cc b/test/src/unit-Subarray.cc index e24cbdd243b2..3a8932cbcedd 100644 --- a/test/src/unit-Subarray.cc +++ b/test/src/unit-Subarray.cc @@ -317,6 +317,9 @@ TEST_CASE_METHOD( CHECK_NOTHROW(cropped_subarray.get_range(1, 1, &range)); CHECK(!memcmp(range->data(), &c_range_1_1[0], 2 * sizeof(uint64_t))); + auto tile_cell_num = subarray.tile_cell_num(&tile_coords[0]); + CHECK(tile_cell_num == cropped_subarray.cell_num()); + close_array(ctx_, array_); } diff --git a/test/src/unit-cppapi-consolidation.cc b/test/src/unit-cppapi-consolidation.cc index f6e4cbb118d8..d643711359a1 100644 --- a/test/src/unit-cppapi-consolidation.cc +++ b/test/src/unit-cppapi-consolidation.cc @@ -226,6 +226,7 @@ TEST_CASE( Context ctx; Config config; + config.set("sm.consolidation.buffer_size", "1000"); FragmentInfo fragment_info(ctx, array_name); fragment_info.load(); diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index d20883783ab3..f1ea3df97813 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -142,6 +142,66 @@ Reader::Reader( check_validity_buffer_sizes(); } +/* ********************************* */ +/* STATIC FUNCTIONS */ +/* ********************************* */ + +template +void Reader::compute_result_space_tiles( + const std::vector>& fragment_metadata, + const std::vector>& tile_coords, + const TileDomain& array_tile_domain, + const std::vector>& frag_tile_domains, + std::map>& result_space_tiles, + shared_ptr memory_tracker) { + auto fragment_num = (unsigned)frag_tile_domains.size(); + auto dim_num = array_tile_domain.dim_num(); + std::vector start_coords; + const T* coords; + start_coords.resize(dim_num); + + // For all tile coordinates + for (const auto& tc : tile_coords) { + coords = (T*)(&(tc[0])); + start_coords = array_tile_domain.start_coords(coords); + + // Create result space tile and insert into the map + auto r = + result_space_tiles.emplace(coords, ResultSpaceTile(memory_tracker)); + auto& result_space_tile = r.first->second; + result_space_tile.set_start_coords(start_coords); + + // Add fragment info to the result space tile + for (unsigned f = 0; f < fragment_num; ++f) { + // Check if the fragment overlaps with the space tile + if (!frag_tile_domains[f].in_tile_domain(coords)) + continue; + + // Check if any previous fragment covers this fragment + // for the tile identified by `coords` + bool covered = false; + for (unsigned j = 0; j < f; ++j) { + if (frag_tile_domains[j].covers(coords, frag_tile_domains[f])) { + covered = true; + break; + } + } + + // Exclude this fragment from the space tile + if (covered) + continue; + + // Include this fragment in the space tile + auto frag_domain = frag_tile_domains[f].domain_slice(); + auto frag_idx = frag_tile_domains[f].id(); + result_space_tile.append_frag_domain(frag_idx, frag_domain); + auto tile_idx = frag_tile_domains[f].tile_pos(coords); + result_space_tile.set_result_tile( + frag_idx, tile_idx, *fragment_metadata[frag_idx].get()); + } + } +} + /* ****************************** */ /* API */ /* ****************************** */ @@ -1475,6 +1535,23 @@ Status Reader::process_tiles( return Status::Ok(); } +template +void Reader::compute_result_space_tiles( + const Subarray& subarray, + const Subarray& partitioner_subarray, + std::map>& result_space_tiles) const { + auto&& [array_tile_domain, frag_tile_domains] = + compute_tile_domains(partitioner_subarray); + + compute_result_space_tiles( + fragment_metadata_, + subarray.tile_coords(), + array_tile_domain, + frag_tile_domains, + result_space_tiles, + query_memory_tracker_); +} + template Status Reader::compute_result_cell_slabs( const Subarray& subarray, diff --git a/tiledb/sm/query/legacy/reader.h b/tiledb/sm/query/legacy/reader.h index 6cec8f488a7a..2c43922cf856 100644 --- a/tiledb/sm/query/legacy/reader.h +++ b/tiledb/sm/query/legacy/reader.h @@ -76,6 +76,35 @@ class Reader : public ReaderBase, public IQueryStrategy { DISABLE_COPY_AND_COPY_ASSIGN(Reader); DISABLE_MOVE_AND_MOVE_ASSIGN(Reader); + /** + * Computes a mapping (tile coordinates) -> (result space tile). + * The produced result space tiles will contain information only + * about fragments that will contribute results. Specifically, if + * a fragment is completely covered by a more recent fragment + * in a particular space tile, then it will certainly not contribute + * results and, thus, no information about that fragment is included + * in the space tile. + * + * @tparam T The datatype of the tile domains. + * @param domain The array domain + * @param tile_coords The unique coordinates of the tiles that intersect + * a subarray. + * @param array_tile_domain The array tile domain. + * @param frag_tile_domains The tile domains of each fragment. These + * are assumed to be ordered from the most recent to the oldest + * fragment. + * @param result_space_tiles The result space tiles to be produced + * by the function. + */ + template + static void compute_result_space_tiles( + const std::vector>& fragment_metadata, + const std::vector>& tile_coords, + const TileDomain& array_tile_domain, + const std::vector>& frag_tile_domains, + std::map>& result_space_tiles, + shared_ptr memory_tracker); + /* ********************************* */ /* API */ /* ********************************* */ @@ -111,6 +140,20 @@ class Reader : public ReaderBase, public IQueryStrategy { /** Returns the name of the strategy */ std::string name(); + /** + * Computes the result space tiles based on the current partition. + * + * @tparam T The domain datatype. + * @param subarray The input subarray. + * @param partitioner_subarray The partitioner subarray. + * @param result_space_tiles The result space tiles to be computed. + */ + template + void compute_result_space_tiles( + const Subarray& subarray, + const Subarray& partitioner_subarray, + std::map>& result_space_tiles) const; + /** * Computes the result cell slabs for the input subarray, given the * input result coordinates (retrieved from the sparse fragments). diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 2c25f3e120e8..497af0c3ebbf 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -244,25 +244,24 @@ Status DenseReader::dense_read() { *reinterpret_cast(domain.tile_extent(d).data()); } - // Compute result space tiles. The result space tiles hold all the - // relevant result tiles of the dense fragments. - std::map> result_space_tiles; - compute_result_space_tiles( - subarray, read_state_.partitioner_.subarray(), result_space_tiles); + // Cache tile domains. + const auto&& [array_tile_domain, frag_tile_domains] = + compute_tile_domains(read_state_.partitioner_.subarray()); - // Compute subarrays for each tile. + // Compute number of cells for each tile. const auto& tile_coords = subarray.tile_coords(); stats_->add_counter("num_tiles", tile_coords.size()); - TileSubarrays tile_subarrays{tile_coords.size()}; - const auto& layout = - layout_ == Layout::GLOBAL_ORDER ? array_schema_.cell_order() : layout_; - auto status = parallel_for( - &resources_.compute_tp(), 0, tile_subarrays.size(), [&](uint64_t t) { - subarray.crop_to_tile( - &tile_subarrays[t], (const DimType*)&tile_coords[t][0], layout); - return Status::Ok(); - }); - RETURN_NOT_OK(status); + std::vector tiles_cell_num(tile_coords.size()); + { + auto timer_se = stats_->start_timer("compute_tiles_cell_num"); + auto status = parallel_for( + &resources_.compute_tp(), 0, tile_coords.size(), [&](uint64_t t) { + tiles_cell_num[t] = + subarray.tile_cell_num((const DimType*)&tile_coords[t][0]); + return Status::Ok(); + }); + RETURN_NOT_OK(status); + } // Compute tile offsets for global order or range info for row/col major. std::vector tile_offsets; @@ -272,9 +271,9 @@ Status DenseReader::dense_read() { tile_offsets.reserve(tile_coords.size()); uint64_t tile_offset = 0; - for (uint64_t i = 0; i < tile_subarrays.size(); i++) { + for (uint64_t i = 0; i < tiles_cell_num.size(); i++) { tile_offsets.emplace_back(tile_offset); - tile_offset += tile_subarrays[i].cell_num(); + tile_offset += tiles_cell_num[i]; } } else { for (uint32_t d = 0; d < dim_num; d++) { @@ -354,33 +353,53 @@ Status DenseReader::dense_read() { // identified. ThreadPool::Task compute_task; + // Allow to disable the parallel read/compute in case the memory budget + // doesn't allow it. + bool wait_compute_task_before_read = false; + // Most of the computations in this loop are moved to a seperate thread so // that we can reach the next batch of tiles while processing the current one. // for all tiles: - // if not enough memory: - // wait(compute_task) + // compute tile batch // read qc attributes // wait(compute_task) // compute_task = proccess qc attributes - // read all attributes - // wait(compute_task) - // compute_task = proccess all attributes + // for all attributes + // read attribute + // wait(compute_task) + // compute_task = proccess attributes + // end for + // end for while (t_end < tile_coords.size()) { stats_->add_counter("internal_loop_num", 1); - // Get result tiles to process on this iteration. - t_end = compute_space_tiles_end( - names, - qc_loaded_attr_names_set_, - subarray, - t_start, - result_space_tiles, - tile_subarrays, - compute_task); + // Compute result space tiles. The result space tiles hold all the + // relevant result tiles of the dense fragments. + auto&& [wait_compute_task_before_read_ret, result_space_tiles] = + compute_result_space_tiles( + t_start, + names, + qc_loaded_attr_names_set_, + subarray, + tiles_cell_num, + array_tile_domain, + frag_tile_domains); + t_end = t_start + result_space_tiles.size(); + wait_compute_task_before_read |= wait_compute_task_before_read_ret; + + // Create the iteration data. + shared_ptr> iteration_tile_data = + tdb::make_shared>( + HERE(), + resources_.compute_tp(), + subarray, + t_start, + t_end, + std::move(result_space_tiles)); // Add the number of cells to process to subarray_end_cell. for (uint64_t t = t_start; t < t_end; t++) { - subarray_end_cell += tile_subarrays[t].cell_num(); + subarray_end_cell += tiles_cell_num[t]; } // Compute parallelization parameters. @@ -391,18 +410,23 @@ Status DenseReader::dense_read() { num_range_threads = 1 + ((num_threads - 1) / (t_end - t_start)); } + // Wait for the compute task to finish before we read more tiles. This is to + // prevent using too much memory when the budget is small and doesn't allow + // to process more than one batch at a time. + if (wait_compute_task_before_read && compute_task.valid()) { + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); + } + // Apply the query condition. auto st = apply_query_condition( compute_task, subarray, - t_start, - t_end, qc_loaded_attr_names_set_, tile_extents, - tile_subarrays, + tiles_cell_num, tile_offsets, range_info, - result_space_tiles, + iteration_tile_data, num_range_threads, qc_result); RETURN_CANCEL_OR_ERROR(st); @@ -429,10 +453,15 @@ Status DenseReader::dense_read() { name, qc_loaded_attr_names_set_, subarray, - t_start, - t_end, - result_space_tiles, - tile_subarrays); + iteration_tile_data, + tiles_cell_num); + + // Wait for the compute task to finish before we read more tiles. This + // is to prevent using too much memory when the budget is small and + // doesn't allow to process more than one batch at a time. + if (wait_compute_task_before_read && compute_task.valid()) { + throw_if_not_ok(resources_.compute_tp().wait(compute_task)); + } // Read and unfilter tiles. std::vector to_load; @@ -449,37 +478,36 @@ Status DenseReader::dense_read() { } compute_task = resources_.compute_tp().execute([&, + iteration_tile_data, filtered_data, dense_dim, name, validity_only, - t_start, - t_end, subarray_start_cell, subarray_end_cell, num_range_threads, - result_tiles]() { + result_tiles]() mutable { if (!dense_dim) { // Unfilter tiles. RETURN_NOT_OK(unfilter_tiles(name, validity_only, result_tiles)); + // The filtered data is no longer required, release it. + filtered_data.reset(); + // Only copy names that are present in the user buffers. if (buffers_.count(name) != 0) { // Copy attribute data to users buffers. auto& var_buffer_size = var_buffer_sizes[name]; - status = copy_attribute( + auto status = copy_attribute( name, tile_extents, subarray, - t_start, - t_end, subarray_start_cell, subarray_end_cell, - tile_subarrays, tile_offsets, var_buffer_size, range_info, - result_space_tiles, + iteration_tile_data, qc_result, num_range_threads); RETURN_CANCEL_OR_ERROR(status); @@ -487,16 +515,14 @@ Status DenseReader::dense_read() { } if (aggregates_.count(name) != 0) { - status = process_aggregates( + auto status = process_aggregates( name, tile_extents, subarray, - t_start, - t_end, - tile_subarrays, + tiles_cell_num, tile_offsets, range_info, - result_space_tiles, + iteration_tile_data, qc_result, num_range_threads); RETURN_CANCEL_OR_ERROR(status); @@ -708,38 +734,32 @@ DenseReader::field_names_to_process( return {names, var_names}; } -/** - * Compute the maximum tile coords that we can process on this iteration to - * respect the memory budget. If the available memory is less than the tile - * upper memory limit, waits for compute task to complete before proceeding to - * the new iteration. - */ template -uint64_t DenseReader::compute_space_tiles_end( +tuple>> +DenseReader::compute_result_space_tiles( + const uint64_t t_start, const std::vector& names, const std::unordered_set& condition_names, - Subarray& subarray, - uint64_t t_start, - std::map>& result_space_tiles, - const DynamicArray& tile_subarrays, - ThreadPool::Task& compute_task) { + const Subarray& subarray, + const std::vector& tiles_cell_num, + const TileDomain& array_tile_domain, + const std::vector>& frag_tile_domains) const { + std::map> result_space_tiles; + // For easy reference. + const auto dim_num = subarray.dim_num(); + const auto fragment_num = (unsigned)frag_tile_domains.size(); const auto& tile_coords = subarray.tile_coords(); - uint64_t available_memory = array_memory_tracker_->get_memory_available(); - - // If the available memory is less than the tile upper memory limit, we cannot - // load two batches at once. Wait for the first compute task to complete - // before loading more tiles. - if (compute_task.valid() && available_memory < tile_upper_memory_limit_) { - throw_if_not_ok(resources_.compute_tp().wait(compute_task)); - } - - const uint64_t upper_memory_limit = - std::min(tile_upper_memory_limit_ / 2, available_memory); - // Keep track of the required memory to load the result space tiles. - uint64_t required_memory_query_condition = 0; - std::vector required_memory(names.size() - condition_names.size()); + // Keep track of the required memory to load the result space tiles. Split up + // filtered versus unfiltered. The memory budget is combined for all + // query condition attributes. + uint64_t required_memory_query_condition_unfiltered = 0; + std::vector required_memory_unfiltered( + names.size() - condition_names.size()); + uint64_t required_memory_query_condition_filtered = 0; + std::vector required_memory_filtered( + names.size() - condition_names.size()); // Cache which fields are for aggregation only. std::vector aggregate_only_field(names.size() - condition_names.size()); @@ -748,63 +768,150 @@ uint64_t DenseReader::compute_space_tiles_end( aggregate_only_field[n - condition_names.size()] = aggregate_only(name); } - // Create the vector of result tiles to operate on. We stop once we reach the - // end or the memory budget. The memory budget is combined for all query - // condition attributes. - uint64_t t_end{t_start}; + // Here we estimate the size of the tile structures. First, we have to account + // the size of the space tile structure. We could go deeper in the class to + // account for other things but for now we keep it simpler. Second, we try to + // account for the tile subarray (DenseTileSubarray). This class will have a + // vector of ranges per dimensions, so 1 + dim_num * sizeof(vector). Here we + // choose 32 for the size of the vector to anticipate the conversion to a PMR + // vector. We also add dim_num * 2 * sizeof(DimType) to account for at least + // one range per dimension (this should be improved by accounting for the + // exact number of ranges). Finally for the original range index member, we + // have to add 1 + dim_num * sizeof(vector) as well and one uint64_t per + // dimension (this can also be improved by accounting for the + // exact number of ranges). + uint64_t est_tile_structs_size = + sizeof(ResultSpaceTile) + (1 + dim_num) * 2 * 32 + + dim_num * (2 * sizeof(DimType) + sizeof(uint64_t)); + + // Create the vector of result tiles to operate on. We stop once we reach + // the end or the memory budget. We either reach the tile upper memory limit, + // which is only for unfiltered data, or the limit of the available budget, + // which is for filtered data, unfiltered data and the tile structs. We try to + // process two tile batches at a time so the available memory is half of what + // we have available. + uint64_t t_end = t_start; + bool wait_compute_task_before_read = false; bool done = false; + const auto available_memory_iteration = + array_memory_tracker_->get_memory_available() / 2; while (!done && t_end < tile_coords.size()) { + uint64_t t_num = t_end - t_start + 1; const DimType* tc = (DimType*)&tile_coords[t_end][0]; - auto& result_space_tile = result_space_tiles.at(tc); + + // Create a result space tile. + ResultSpaceTile result_space_tile(query_memory_tracker_); + result_space_tile.set_start_coords(array_tile_domain.start_coords(tc)); + + // Add fragment info to the result space tile + for (unsigned f = 0; f < fragment_num; ++f) { + // Check if the fragment overlaps with the space tile + if (!frag_tile_domains[f].in_tile_domain(tc)) { + continue; + } + + // Check if any previous fragment covers this fragment + // for the tile identified by `coords` + bool covered = false; + for (unsigned j = 0; j < f; ++j) { + if (frag_tile_domains[j].covers(tc, frag_tile_domains[f])) { + covered = true; + break; + } + } + + // Exclude this fragment from the space tile if it's covered. + if (covered) { + continue; + } + + // Include this fragment in the space tile. + auto frag_domain = frag_tile_domains[f].domain_slice(); + auto frag_idx = frag_tile_domains[f].id(); + result_space_tile.append_frag_domain(frag_idx, frag_domain); + auto tile_idx = frag_tile_domains[f].tile_pos(tc); + result_space_tile.set_result_tile( + frag_idx, tile_idx, *fragment_metadata_[frag_idx].get()); + } // Compute the required memory to load the query condition tiles for the // current result space tile. - uint64_t condition_memory = 0; + uint64_t unfiltered_condition_memory = 0; + uint64_t filtered_condition_memory = 0; for (const auto& result_tile : result_space_tile.result_tiles()) { auto& rt = result_tile.second; for (uint64_t n = 0; n < condition_names.size(); n++) { - condition_memory += + unfiltered_condition_memory += get_attribute_tile_size(names[n], rt.frag_idx(), rt.tile_idx()); + filtered_condition_memory += get_attribute_persisted_tile_size( + names[n], rt.frag_idx(), rt.tile_idx()); } } - // If we reached the memory budget, stop. Always include the first tile. - if (t_end != t_start && required_memory_query_condition + condition_memory > - upper_memory_limit) { + // - We always include the first tile, so if t_end == t_start, we don't + // stop. + // - We stop if the in memory tile size (unfiltered) is larger than the + // upper tile memory limit. + // - We also stop if the estimated memory usage (filtered + unfiltered + + // tile structs) is greater than our available memory budget for the + // iteration. + if ((t_end != t_start) && + (((required_memory_query_condition_unfiltered + + unfiltered_condition_memory) > tile_upper_memory_limit_ / 2) || + ((required_memory_query_condition_unfiltered + + unfiltered_condition_memory + + required_memory_query_condition_filtered + + filtered_condition_memory + t_num * est_tile_structs_size) > + available_memory_iteration))) { done = true; break; } else { - required_memory_query_condition += condition_memory; + required_memory_query_condition_unfiltered += unfiltered_condition_memory; + required_memory_query_condition_filtered += filtered_condition_memory; } // Compute the required memory to load the tiles for each names for the // current space tile. for (uint64_t n = condition_names.size(); n < names.size(); n++) { - uint64_t tile_memory = 0; + uint64_t tile_memory_unfiltered = 0; + uint64_t tile_memory_filtered = 0; uint64_t r_idx = n - condition_names.size(); // We might not need to load this tile into memory at all for aggregation // only. if (aggregate_only_field[r_idx] && can_aggregate_tile_with_frag_md( - names[n], result_space_tile, tile_subarrays[t_end])) { + names[n], result_space_tile, tiles_cell_num[t_end])) { continue; } // Compute memory usage for all result tiles in the space tile. for (const auto& result_tile : result_space_tile.result_tiles()) { auto& rt = result_tile.second; - tile_memory += + tile_memory_unfiltered += get_attribute_tile_size(names[n], rt.frag_idx(), rt.tile_idx()); + tile_memory_filtered += get_attribute_persisted_tile_size( + names[n], rt.frag_idx(), rt.tile_idx()); } - // If we reached the memory budget, stop. Always include the first tile. - if (t_end != t_start && - required_memory[r_idx] + tile_memory > upper_memory_limit) { + // - We always include the first tile, so if t_end == t_start, we don't + // stop. + // - We stop if the in memory tile size (unfiltered) is larger than the + // upper tile memory limit. + // - We also stop if the estimated memory usage (filtered + unfiltered + + // tile structs) is greater than our available memory budget for the + // iteration. + if ((t_end != t_start) && + (((required_memory_unfiltered[r_idx] + tile_memory_unfiltered) > + tile_upper_memory_limit_ / 2) || + ((required_memory_unfiltered[r_idx] + tile_memory_unfiltered + + required_memory_filtered[r_idx] + tile_memory_filtered + + t_num * est_tile_structs_size) > available_memory_iteration))) { done = true; break; } else { - required_memory[r_idx] += tile_memory; + required_memory_unfiltered[r_idx] += tile_memory_unfiltered; + required_memory_filtered[r_idx] += tile_memory_filtered; } } @@ -812,27 +919,54 @@ uint64_t DenseReader::compute_space_tiles_end( break; } + // Insert the result space tile into the map. + result_space_tiles.emplace(tc, std::move(result_space_tile)); t_end++; } // If we only include one tile, make sure we don't exceed the memory budget. + // We can also let a tile through if it fits in the available budget but not + // in the iteration budget. In that case we just disable processing tiles as + // others are read so that only one batch is processed at a time. if (t_end == t_start + 1) { const auto available_memory = array_memory_tracker_->get_memory_available(); - for (auto mem : required_memory) { - if (mem > available_memory) { + for (uint64_t r_idx = 0; r_idx < required_memory_filtered.size(); r_idx++) { + uint64_t total_memory = required_memory_filtered[r_idx] + + required_memory_unfiltered[r_idx] + + est_tile_structs_size; + + // Disable the multiple iterations if the tiles don't fit in the iteration + // budget. + if (total_memory > available_memory_iteration) { + wait_compute_task_before_read = true; + } + + // If a single tile doesn't fit in the available memory, we can't proceed. + if (total_memory > available_memory) { throw DenseReaderStatusException( "Cannot process a single tile, increase memory budget"); } } - if (required_memory_query_condition > available_memory) { + uint64_t total_memory_condition = + required_memory_query_condition_filtered + + required_memory_query_condition_unfiltered + est_tile_structs_size; + + // Disable the multiple iterations if the tiles don't fit in the iteration + // budget. + if (total_memory_condition > available_memory_iteration) { + wait_compute_task_before_read = true; + } + + // If a single tile doesn't fit in the available memory, we can't proceed. + if (total_memory_condition > available_memory) { throw DenseReaderStatusException( "Cannot process a single tile because of query condition, increase " "memory budget"); } } - return t_end; + return {wait_compute_task_before_read, std::move(result_space_tiles)}; } template @@ -840,11 +974,10 @@ std::vector DenseReader::result_tiles_to_load( const optional name, const std::unordered_set& condition_names, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, - std::map>& result_space_tiles, - const DynamicArray& tile_subarrays) const { + shared_ptr> iteration_tile_data, + const std::vector& tiles_cell_num) const { // For easy reference. + auto& result_space_tiles = iteration_tile_data->result_space_tiles(); const auto& tile_coords = subarray.tile_coords(); const bool agg_only = name.has_value() && aggregate_only(name.value()); @@ -854,6 +987,8 @@ std::vector DenseReader::result_tiles_to_load( return ret; } + auto t_start = iteration_tile_data->t_start(); + auto t_end = iteration_tile_data->t_end(); for (uint64_t t = t_start; t < t_end; t++) { // Add the result tiles for this space tile to the returned list to // process unless it's for an aggregate only field and that tile can be @@ -861,7 +996,7 @@ std::vector DenseReader::result_tiles_to_load( const DimType* tc = (DimType*)&tile_coords[t][0]; auto& result_space_tile = result_space_tiles.at(tc); if (agg_only && can_aggregate_tile_with_frag_md( - name.value(), result_space_tile, tile_subarrays[t])) { + name.value(), result_space_tile, tiles_cell_num[t])) { continue; } @@ -883,17 +1018,16 @@ template Status DenseReader::apply_query_condition( ThreadPool::Task& compute_task, Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, const std::unordered_set& condition_names, const std::vector& tile_extents, - DynamicArray& tile_subarrays, + const std::vector& tiles_cell_num, std::vector& tile_offsets, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const uint64_t num_range_threads, std::vector& qc_result) { auto timer_se = stats_->start_timer("apply_query_condition"); + auto& result_space_tiles = iteration_tile_data->result_space_tiles(); if (condition_.has_value()) { // Compute the result of the query condition. @@ -910,10 +1044,8 @@ Status DenseReader::apply_query_condition( nullopt, condition_names, subarray, - t_start, - t_end, - result_space_tiles, - tile_subarrays); + iteration_tile_data, + tiles_cell_num); // Read and unfilter query condition attributes. shared_ptr> filtered_data = @@ -926,11 +1058,10 @@ Status DenseReader::apply_query_condition( compute_task = resources_.compute_tp().execute([&, filtered_data, + iteration_tile_data, qc_names, - t_start, - t_end, num_range_threads, - result_tiles]() { + result_tiles]() mutable { // For easy reference. const auto& tile_coords = subarray.tile_coords(); const auto dim_num = array_schema_.dim_num(); @@ -943,6 +1074,9 @@ Status DenseReader::apply_query_condition( RETURN_NOT_OK(unfilter_tiles(name, false, result_tiles)); } + // The filtered data is no longer required, release it. + filtered_data.reset(); + if (stride == UINT64_MAX) { stride = 1; } @@ -950,8 +1084,8 @@ Status DenseReader::apply_query_condition( // Process all tiles in parallel. auto status = parallel_for_2d( &resources_.compute_tp(), - t_start, - t_end, + iteration_tile_data->t_start(), + iteration_tile_data->t_end(), 0, num_range_threads, [&](uint64_t t, uint64_t range_thread_idx) { @@ -965,7 +1099,7 @@ Status DenseReader::apply_query_condition( range_thread_idx, num_range_threads, subarray, - tile_subarrays[t], + iteration_tile_data->tile_subarray(t), tile_extents, result_space_tile.start_coords(), range_info, @@ -1097,20 +1231,18 @@ Status DenseReader::copy_attribute( const std::string& name, const std::vector& tile_extents, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, const uint64_t subarray_start_cell, const uint64_t subarray_end_cell, - const DynamicArray& tile_subarrays, const std::vector& tile_offsets, uint64_t& var_buffer_size, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const std::vector& qc_result, const uint64_t num_range_threads) { auto timer_se = stats_->start_timer("copy_attribute"); // For easy reference + auto& result_space_tiles = iteration_tile_data->result_space_tiles(); const auto& tile_coords = subarray.tile_coords(); const auto global_order = layout_ == Layout::GLOBAL_ORDER; @@ -1131,8 +1263,8 @@ Status DenseReader::copy_attribute( auto timer_se = stats_->start_timer("copy_offset_tiles"); auto status = parallel_for_2d( &resources_.compute_tp(), - t_start, - t_end, + iteration_tile_data->t_start(), + iteration_tile_data->t_end(), 0, num_range_threads, [&](uint64_t t, uint64_t range_thread_idx) { @@ -1146,7 +1278,7 @@ Status DenseReader::copy_attribute( tile_extents, result_space_tile, subarray, - tile_subarrays[t], + iteration_tile_data->tile_subarray(t), subarray_start_cell, global_order ? tile_offsets[t] : 0, var_data, @@ -1191,8 +1323,8 @@ Status DenseReader::copy_attribute( // Process var data in parallel. auto status = parallel_for_2d( &resources_.compute_tp(), - t_start, - t_end, + iteration_tile_data->t_start(), + iteration_tile_data->t_end(), 0, num_range_threads, [&](uint64_t t, uint64_t range_thread_idx) { @@ -1205,12 +1337,12 @@ Status DenseReader::copy_attribute( tile_extents, result_space_tile, subarray, - tile_subarrays[t], + iteration_tile_data->tile_subarray(t), subarray_start_cell, global_order ? tile_offsets[t] : 0, var_data, range_info, - t == t_end - 1, + t == iteration_tile_data->t_end() - 1, var_buffer_size, range_thread_idx, num_range_threads); @@ -1233,8 +1365,8 @@ Status DenseReader::copy_attribute( // Process values in parallel. auto status = parallel_for_2d( &resources_.compute_tp(), - t_start, - t_end, + iteration_tile_data->t_start(), + iteration_tile_data->t_end(), 0, num_range_threads, [&](uint64_t t, uint64_t range_thread_idx) { @@ -1248,7 +1380,7 @@ Status DenseReader::copy_attribute( tile_extents, result_space_tile, subarray, - tile_subarrays[t], + iteration_tile_data->tile_subarray(t), global_order ? tile_offsets[t] : 0, range_info, qc_result, @@ -1316,25 +1448,24 @@ Status DenseReader::process_aggregates( const std::string& name, const std::vector& tile_extents, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, - const DynamicArray& tile_subarrays, + const std::vector& tiles_cell_num, const std::vector& tile_offsets, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const std::vector& qc_result, const uint64_t num_range_threads) { auto timer_se = stats_->start_timer("process_aggregates"); // For easy reference + auto& result_space_tiles = iteration_tile_data->result_space_tiles(); const auto& tile_coords = subarray.tile_coords(); const auto global_order = layout_ == Layout::GLOBAL_ORDER; // Process values in parallel. auto status = parallel_for_2d( &resources_.compute_tp(), - t_start, - t_end, + iteration_tile_data->t_start(), + iteration_tile_data->t_end(), 0, num_range_threads, [&](uint64_t t, uint64_t range_thread_idx) { @@ -1342,7 +1473,7 @@ Status DenseReader::process_aggregates( const DimType* tc = (DimType*)&tile_coords[t][0]; auto& result_space_tile = result_space_tiles.at(tc); if (can_aggregate_tile_with_frag_md( - name, result_space_tile, tile_subarrays[t])) { + name, result_space_tile, tiles_cell_num[t])) { if (range_thread_idx == 0) { auto& rt = result_space_tile.single_result_tile(); auto tile_idx = rt.tile_idx(); @@ -1359,7 +1490,7 @@ Status DenseReader::process_aggregates( tile_extents, result_space_tile, subarray, - tile_subarrays[t], + iteration_tile_data->tile_subarray(t), global_order ? tile_offsets[t] : 0, range_info, qc_result, @@ -1409,7 +1540,7 @@ Status DenseReader::copy_fixed_tiles( const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, const std::vector& qc_result, @@ -1605,7 +1736,7 @@ Status DenseReader::copy_offset_tiles( const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t subarray_start_cell, const uint64_t global_cell_offset, std::vector& var_data, @@ -1783,7 +1914,7 @@ Status DenseReader::copy_var_tiles( const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t subarray_start_cell, const uint64_t global_cell_offset, std::vector& var_data, @@ -1864,7 +1995,7 @@ Status DenseReader::aggregate_tiles( const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, const std::vector& qc_result, diff --git a/tiledb/sm/query/readers/dense_reader.h b/tiledb/sm/query/readers/dense_reader.h index 5d54af461242..f3ccd3fbefa5 100644 --- a/tiledb/sm/query/readers/dense_reader.h +++ b/tiledb/sm/query/readers/dense_reader.h @@ -38,7 +38,6 @@ #include "tiledb/common/common.h" #include "tiledb/common/status.h" #include "tiledb/sm/array_schema/dimension.h" -#include "tiledb/sm/array_schema/dynamic_array.h" #include "tiledb/sm/misc/types.h" #include "tiledb/sm/query/iquery_strategy.h" #include "tiledb/sm/query/query_buffer.h" @@ -56,28 +55,80 @@ class Array; /** Processes dense read queries. */ class DenseReader : public ReaderBase, public IQueryStrategy { /** - * TileSubarrays class to store tile subarrays inside of a dynamic array with - * custom destructor. + * Class that stores the tile data for an internal loop of the reader. */ - class TileSubarrays : public DynamicArray { + template + class IterationTileData { public: - TileSubarrays(uint64_t n) - : DynamicArray( - n, - tdb::allocator{}, - Tag::NullInitializer>{}) - , n_(n) { + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + IterationTileData() = delete; + + IterationTileData( + ThreadPool& compute_tp, + Subarray& subarray, + uint64_t t_start, + uint64_t t_end, + std::map>&& result_space_tiles) + : t_start_(t_start) + , t_end_(t_end) + , tile_subarrays_(t_end - t_start, subarray.dim_num()) + , result_space_tiles_(std::move(result_space_tiles)) { + auto& tile_coords = subarray.tile_coords(); + throw_if_not_ok( + parallel_for(&compute_tp, 0, tile_subarrays_.size(), [&](uint64_t t) { + subarray.crop_to_tile( + tile_subarrays_[t], + (const DimType*)&tile_coords[t + t_start][0]); + return Status::Ok(); + })); + }; + + DISABLE_COPY_AND_COPY_ASSIGN(IterationTileData); + DISABLE_MOVE_AND_MOVE_ASSIGN(IterationTileData); + + /* ********************************* */ + /* PUBLIC METHODS */ + /* ********************************* */ + + /** Returns the tile start index. */ + inline uint64_t t_start() { + return t_start_; } - ~TileSubarrays() { - // Delete the tile subarrays. - for (uint64_t i = 0; i < n_; i++) { - (*this)[i].~Subarray(); - } + /** Returns the tile end index. */ + inline uint64_t& t_end() { + return t_end_; + } + + /** Returns the tile subarrays. */ + inline DenseTileSubarray& tile_subarray(uint64_t t) { + return tile_subarrays_[t - t_start_]; + } + + /** Returns the result space tiles. */ + inline std::map>& + result_space_tiles() { + return result_space_tiles_; } private: - uint64_t n_; + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Start tile to process. */ + uint64_t t_start_; + + /** End tile to process. */ + uint64_t t_end_; + + /** Tile subarrays. */ + std::vector> tile_subarrays_; + + /** Result space tiles. */ + std::map> result_space_tiles_; }; public: @@ -190,18 +241,30 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::unordered_set& condition_names); /** - * Compute the result space tiles to process on an iteration to respect the - * memory budget. + * Computes the result space tiles based on the current partition. + * + * @tparam T The domain datatype. + * @param t_start The start tile index in the tile coords. + * @param names The fields to process. + * @param condition_names The fields in the query condition. + * @param subarray The input subarray. + * @param tiles_cell_num The cell num for all tiles. + * @param array_tile_domain The array tile domain. + * @param frag_tile_domains The relevant fragments tile domains. + * @param iteration_tile_data The iteration data. + * + * @return wait_compute_task_before_read, result_space_tiles. */ template - uint64_t compute_space_tiles_end( + tuple>> + compute_result_space_tiles( + const uint64_t t_start, const std::vector& names, const std::unordered_set& condition_names, - Subarray& subarray, - uint64_t t_start, - std::map>& result_space_tiles, - const DynamicArray& tile_subarrays, - ThreadPool::Task& compute_task); + const Subarray& subarray, + const std::vector& tiles_cell_num, + const TileDomain& array_tile_domain, + const std::vector>& frag_tile_domains) const; /** Compute the result tiles to load for a name. */ template @@ -209,24 +272,20 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const optional name, const std::unordered_set& condition_names, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, - std::map>& result_space_tiles, - const DynamicArray& tile_subarrays) const; + shared_ptr> iteration_tile_data, + const std::vector& tiles_cell_num) const; /** Apply the query condition. */ template Status apply_query_condition( ThreadPool::Task& compute_task, Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, const std::unordered_set& condition_names, const std::vector& tile_extents, - DynamicArray& tile_subarrays, + const std::vector& tiles_cell_num, std::vector& tile_offsets, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const uint64_t num_range_threads, std::vector& qc_result); @@ -246,15 +305,12 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::string& name, const std::vector& tile_extents, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, const uint64_t subarray_start_cell, const uint64_t subarray_end_cell, - const DynamicArray& tile_subarrays, const std::vector& tile_offsets, uint64_t& var_buffer_size, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const std::vector& qc_result, const uint64_t num_range_threads); @@ -277,14 +333,14 @@ class DenseReader : public ReaderBase, public IQueryStrategy { * * @param name Name of the field to process. * @param rst Result space tile. - * @param tile_subarray Tile subarray. + * @param tile_cell_num Tile cell num. * @return If we can do the aggregation with the frag md or not. */ template inline bool can_aggregate_tile_with_frag_md( const std::string& name, ResultSpaceTile& rst, - const Subarray& tile_subarray) const { + const uint64_t tile_cell_num) const { if (array_schema_.is_dim(name)) { return false; } @@ -304,8 +360,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { // Make sure this tile isn't cropped by ranges and the fragment metadata has // tile metadata. - if (tile_subarray.cell_num() != rt.cell_num() || - !frag_md->has_tile_metadata()) { + if (tile_cell_num != rt.cell_num() || !frag_md->has_tile_metadata()) { return false; } @@ -329,12 +384,10 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::string& name, const std::vector& tile_extents, const Subarray& subarray, - const uint64_t t_start, - const uint64_t t_end, - const DynamicArray& tile_subarrays, + const std::vector& tiles_cell_num, const std::vector& tile_offsets, const std::vector>& range_info, - std::map>& result_space_tiles, + shared_ptr> iteration_tile_data, const std::vector& qc_result, const uint64_t num_range_threads); @@ -356,7 +409,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, const std::vector& qc_result, @@ -370,7 +423,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t subarray_start_cell, const uint64_t global_cell_offset, std::vector& var_data, @@ -386,7 +439,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t subarray_start_cell, const uint64_t global_cell_offset, std::vector& var_data, @@ -403,7 +456,7 @@ class DenseReader : public ReaderBase, public IQueryStrategy { const std::vector& tile_extents, ResultSpaceTile& result_space_tile, const Subarray& subarray, - const Subarray& tile_subarray, + const DenseTileSubarray& tile_subarray, const uint64_t global_cell_offset, const std::vector>& range_info, const std::vector& qc_result, diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 27b3d15f4658..cff2edb153b8 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -101,66 +101,6 @@ ReaderBase::ReaderBase( } } -/* ********************************* */ -/* STATIC FUNCTIONS */ -/* ********************************* */ - -template -void ReaderBase::compute_result_space_tiles( - const std::vector>& fragment_metadata, - const std::vector>& tile_coords, - const TileDomain& array_tile_domain, - const std::vector>& frag_tile_domains, - std::map>& result_space_tiles, - shared_ptr memory_tracker) { - auto fragment_num = (unsigned)frag_tile_domains.size(); - auto dim_num = array_tile_domain.dim_num(); - std::vector start_coords; - const T* coords; - start_coords.resize(dim_num); - - // For all tile coordinates - for (const auto& tc : tile_coords) { - coords = (T*)(&(tc[0])); - start_coords = array_tile_domain.start_coords(coords); - - // Create result space tile and insert into the map - auto r = - result_space_tiles.emplace(coords, ResultSpaceTile(memory_tracker)); - auto& result_space_tile = r.first->second; - result_space_tile.set_start_coords(start_coords); - - // Add fragment info to the result space tile - for (unsigned f = 0; f < fragment_num; ++f) { - // Check if the fragment overlaps with the space tile - if (!frag_tile_domains[f].in_tile_domain(coords)) - continue; - - // Check if any previous fragment covers this fragment - // for the tile identified by `coords` - bool covered = false; - for (unsigned j = 0; j < f; ++j) { - if (frag_tile_domains[j].covers(coords, frag_tile_domains[f])) { - covered = true; - break; - } - } - - // Exclude this fragment from the space tile - if (covered) - continue; - - // Include this fragment in the space tile - auto frag_domain = frag_tile_domains[f].domain_slice(); - auto frag_idx = frag_tile_domains[f].id(); - result_space_tile.append_frag_domain(frag_idx, frag_domain); - auto tile_idx = frag_tile_domains[f].tile_pos(coords); - result_space_tile.set_result_tile( - frag_idx, tile_idx, *fragment_metadata[frag_idx].get()); - } - } -} - /* ****************************** */ /* PUBLIC METHODS */ /* ****************************** */ @@ -1056,7 +996,7 @@ uint64_t ReaderBase::offsets_bytesize() const { } uint64_t ReaderBase::get_attribute_tile_size( - const std::string& name, unsigned f, uint64_t t) { + const std::string& name, unsigned f, uint64_t t) const { uint64_t tile_size = 0; if (!fragment_metadata_[f]->array_schema()->is_field(name)) { return tile_size; @@ -1077,11 +1017,34 @@ uint64_t ReaderBase::get_attribute_tile_size( return tile_size; } +uint64_t ReaderBase::get_attribute_persisted_tile_size( + const std::string& name, unsigned f, uint64_t t) const { + uint64_t tile_size = 0; + if (!fragment_metadata_[f]->array_schema()->is_field(name)) { + return tile_size; + } + + tile_size += + fragment_metadata_[f]->loaded_metadata()->persisted_tile_size(name, t); + + if (array_schema_.var_size(name)) { + tile_size += + fragment_metadata_[f]->loaded_metadata()->persisted_tile_var_size( + name, t); + } + + if (array_schema_.is_nullable(name)) { + tile_size += + fragment_metadata_[f]->loaded_metadata()->persisted_tile_validity_size( + name, t); + } + + return tile_size; +} + template -void ReaderBase::compute_result_space_tiles( - const Subarray& subarray, - const Subarray& partitioner_subarray, - std::map>& result_space_tiles) const { +std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray& partitioner_subarray) const { // For easy reference auto domain = array_schema_.domain().domain(); auto tile_extents = array_schema_.domain().tile_extents(); @@ -1102,19 +1065,9 @@ void ReaderBase::compute_result_space_tiles( } } - // Get tile coords and array domain - const auto& tile_coords = subarray.tile_coords(); - TileDomain array_tile_domain( - UINT32_MAX, domain, domain, tile_extents, tile_order); - - // Compute result space tiles - compute_result_space_tiles( - fragment_metadata_, - tile_coords, - array_tile_domain, - frag_tile_domains, - result_space_tiles, - query_memory_tracker_); + return { + TileDomain(UINT32_MAX, domain, domain, tile_extents, tile_order), + frag_tile_domains}; } bool ReaderBase::has_coords() const { @@ -1263,38 +1216,22 @@ void ReaderBase::validate_attribute_order( } // Explicit template instantiations -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; -template void ReaderBase::compute_result_space_tiles( - const Subarray&, - const Subarray&, - std::map>&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; +template std::pair, std::vector>> +ReaderBase::compute_tile_domains(const Subarray&) const; template tuple, std::vector> ReaderBase::cache_dimension_label_data(); template tuple, std::vector> diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index d9a859d13e81..01f59d004548 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -174,35 +174,6 @@ class ReaderBase : public StrategyBase { /* STATIC FUNCTIONS */ /* ********************************* */ - /** - * Computes a mapping (tile coordinates) -> (result space tile). - * The produced result space tiles will contain information only - * about fragments that will contribute results. Specifically, if - * a fragment is completely covered by a more recent fragment - * in a particular space tile, then it will certainly not contribute - * results and, thus, no information about that fragment is included - * in the space tile. - * - * @tparam T The datatype of the tile domains. - * @param domain The array domain - * @param tile_coords The unique coordinates of the tiles that intersect - * a subarray. - * @param array_tile_domain The array tile domain. - * @param frag_tile_domains The tile domains of each fragment. These - * are assumed to be ordered from the most recent to the oldest - * fragment. - * @param result_space_tiles The result space tiles to be produced - * by the function. - */ - template - static void compute_result_space_tiles( - const std::vector>& fragment_metadata, - const std::vector>& tile_coords, - const TileDomain& array_tile_domain, - const std::vector>& frag_tile_domains, - std::map>& result_space_tiles, - shared_ptr memory_tracker); - /** * Computes the minimum and maximum indexes of tile chunks to process based on * the available threads. @@ -659,21 +630,29 @@ class ReaderBase : public StrategyBase { * @return Tile size. */ uint64_t get_attribute_tile_size( - const std::string& name, unsigned f, uint64_t t); + const std::string& name, unsigned f, uint64_t t) const; + + /** + * Get the on disk size of an attribute tile. + * + * @param name The attribute name. + * @param f The fragment idx. + * @param t The tile idx. + * @return Tile size. + */ + uint64_t get_attribute_persisted_tile_size( + const std::string& name, unsigned f, uint64_t t) const; /** - * Computes the result space tiles based on the current partition. + * Computes the tile domains based on the current partition. * * @tparam T The domain datatype. - * @param subarray The input subarray. * @param partitioner_subarray The partitioner subarray. - * @param result_space_tiles The result space tiles to be computed. + * @return array tile domain, fragments tile domains. */ template - void compute_result_space_tiles( - const Subarray& subarray, - const Subarray& partitioner_subarray, - std::map>& result_space_tiles) const; + std::pair, std::vector>> compute_tile_domains( + const Subarray& partitioner_subarray) const; /** Returns `true` if the coordinates are included in the attributes. */ bool has_coords() const; diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index 3f853876b147..cd5121421dec 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -825,17 +825,52 @@ template Subarray Subarray::crop_to_tile(const T* tile_coords, Layout layout) const { // TBD: is it ok that Subarray log id will increase as if it's a new subarray? Subarray ret(array_, layout, stats_->parent(), logger_, false); - crop_to_tile_impl(tile_coords, ret); + crop_to_tile_impl(tile_coords, ret); return ret; } template void Subarray::crop_to_tile( - Subarray* ret, const T* tile_coords, Layout layout) const { - new (ret) Subarray(array_, layout, stats_->parent(), logger_, false); - Subarray(array_, layout, stats_->parent(), logger_, false); - crop_to_tile_impl(tile_coords, *ret); + DenseTileSubarray& ret, const T* tile_coords) const { + crop_to_tile_impl>(tile_coords, ret); +} + +template +uint64_t Subarray::tile_cell_num(const T* tile_coords) const { + uint64_t ret = 1; + T new_range[2]; + bool overlaps; + + // Get tile subarray based on the input coordinates + const auto& array_schema = array_->array_schema_latest(); + std::vector tile_subarray(2 * dim_num()); + array_schema.domain().get_tile_subarray(tile_coords, &tile_subarray[0]); + + // Compute cell num per dims. + for (unsigned d = 0; d < dim_num(); ++d) { + uint64_t cell_num_for_dim = 0; + for (size_t r = 0; r < range_subset_[d].num_ranges(); ++r) { + const auto& range = range_subset_[d][r]; + utils::geometry::overlap( + (const T*)range.data(), + &tile_subarray[2 * d], + 1, + new_range, + &overlaps); + + if (overlaps) { + // Here we need to do +1 because both the start and end are included in + // the number of cells. + cell_num_for_dim += + static_cast(new_range[1] - new_range[0] + 1); + } + } + + ret *= cell_num_for_dim; + } + + return ret; } uint32_t Subarray::dim_num() const { @@ -1903,7 +1938,7 @@ void Subarray::compute_relevant_fragment_est_result_sizes( (*result_sizes)[n].size_var_ += tile_var_size; if (nullable[n]) (*result_sizes)[n].size_validity_ += - tile_var_size / attr_datatype_size * + tile_size / constants::cell_var_offset_size * constants::cell_validity_size; } } @@ -1944,7 +1979,7 @@ void Subarray::compute_relevant_fragment_est_result_sizes( (*result_sizes)[n].size_var_ += tile_var_size * ratio; if (nullable[n]) (*result_sizes)[n].size_validity_ += - (tile_var_size / attr_datatype_size * + (tile_size / constants::cell_var_offset_size * constants::cell_validity_size) * ratio; } @@ -2651,8 +2686,8 @@ bool Subarray::non_overlapping_ranges_for_dim(const uint64_t dim_idx) { return true; } -template -void Subarray::crop_to_tile_impl(const T* tile_coords, Subarray& ret) const { +template +void Subarray::crop_to_tile_impl(const T* tile_coords, SubarrayT& ret) const { T new_range[2]; bool overlaps; @@ -2676,9 +2711,9 @@ void Subarray::crop_to_tile_impl(const T* tile_coords, Subarray& ret) const { if (overlaps) { ret.add_range_unsafe(d, Range(new_range, r_size)); - ret.original_range_idx_.resize(dim_num()); - ret.original_range_idx_[d].resize(i + 1); - ret.original_range_idx_[d][i++] = r; + ret.original_range_idx_unsafe().resize(dim_num()); + ret.original_range_idx_unsafe()[d].resize(i + 1); + ret.original_range_idx_unsafe()[d][i++] = r; } } } @@ -2749,25 +2784,38 @@ template Subarray Subarray::crop_to_tile( const double* tile_coords, Layout layout) const; template void Subarray::crop_to_tile( - Subarray* ret, const int8_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const int8_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const uint8_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const uint8_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const int16_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const int16_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const uint16_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const uint16_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const int32_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const int32_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const uint32_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const uint32_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const int64_t* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const int64_t* tile_coords) const; template void Subarray::crop_to_tile( - Subarray* ret, const uint64_t* tile_coords, Layout layout) const; -template void Subarray::crop_to_tile( - Subarray* ret, const float* tile_coords, Layout layout) const; -template void Subarray::crop_to_tile( - Subarray* ret, const double* tile_coords, Layout layout) const; + DenseTileSubarray& ret, const uint64_t* tile_coords) const; + +template uint64_t Subarray::tile_cell_num( + const int8_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const uint8_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const int16_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const uint16_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const int32_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const uint32_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const int64_t* tile_coords) const; +template uint64_t Subarray::tile_cell_num( + const uint64_t* tile_coords) const; /* ********************************* */ /* LABEL RANGE SUBSET */ diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 453dd13f51a7..59cc88955dde 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -93,6 +93,80 @@ class FragmentMetadata; enum class Layout : uint8_t; enum class QueryType : uint8_t; +template +class DenseTileSubarray { + public: + /** + * Stores information about a range along a single dimension. The + * whole range resides in a single tile. + */ + struct DenseTileRange { + /** The start of the range in global coordinates. */ + T start_; + /** The end of the range in global coordinates. */ + T end_; + + /** Constructor. */ + DenseTileRange(T start, T end) + : start_(start) + , end_(end) { + } + + /** Equality operator. */ + bool operator==(const DenseTileRange& r) const { + return (r.start_ == start_ && r.end_ == end_); + } + }; + + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + DenseTileSubarray() = delete; + + DenseTileSubarray(unsigned dim_num) + : ranges_(dim_num) { + } + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Returns the orignal range indexes. */ + inline const std::vector>& original_range_idx() const { + return original_range_idx_; + } + + /** Returns the ranges. */ + inline const std::vector>& ranges() const { + return ranges_; + } + + /** Returns the orignal range indexes to be modified. */ + inline std::vector>& original_range_idx_unsafe() { + return original_range_idx_; + } + + /** + * Adds a range along the dimension with the given index, without + * performing any error checks. + */ + void add_range_unsafe(uint32_t dim_idx, const Range& range) { + ranges_[dim_idx].emplace_back(range.start_as(), range.end_as()); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** Stores a vector of 1D ranges per dimension. */ + std::vector> original_range_idx_; + + /** Stores the ranges per dimension. */ + std::vector> ranges_; +}; + /** * Interface to implement for a class that can store tile ranges computed by * this class. @@ -508,6 +582,11 @@ class Subarray { const void* end, uint64_t end_size); + /** Returns the orignal range indexes to be modified. */ + inline std::vector>& original_range_idx_unsafe() { + return original_range_idx_; + } + /** * Adds a range along the dimension with the given index, without * performing any error checks. @@ -807,7 +886,13 @@ class Subarray { * the input `layout`. */ template - void crop_to_tile(Subarray* ret, const T* tile_coords, Layout layout) const; + void crop_to_tile(DenseTileSubarray& ret, const T* tile_coords) const; + + /** + * Returns the number of cells in a specific tile for this subarray. + */ + template + uint64_t tile_cell_num(const T* tile_coords) const; /** Returns the number of dimensions of the subarray. */ uint32_t dim_num() const; @@ -1556,8 +1641,8 @@ class Subarray { * tile with the input coordinates. The new subarray will have * the input `layout`. */ - template - void crop_to_tile_impl(const T* tile_coords, Subarray& ret) const; + template + void crop_to_tile_impl(const T* tile_coords, SubarrayT& ret) const; }; } // namespace tiledb::sm diff --git a/tiledb/sm/subarray/tile_cell_slab_iter.cc b/tiledb/sm/subarray/tile_cell_slab_iter.cc index 34db084a8807..0d0cbfed671f 100644 --- a/tiledb/sm/subarray/tile_cell_slab_iter.cc +++ b/tiledb/sm/subarray/tile_cell_slab_iter.cc @@ -58,7 +58,7 @@ TileCellSlabIter::TileCellSlabIter( const uint64_t range_thread_idx, const uint64_t num_range_threads, const Subarray& root_subarray, - const Subarray& subarray, + const DenseTileSubarray& subarray, const std::vector& tile_extents, const std::vector& start_coords, const std::vector>& range_info, @@ -66,8 +66,11 @@ TileCellSlabIter::TileCellSlabIter( : num_ranges_(root_subarray.range_num()) , original_range_idx_(subarray.original_range_idx()) , range_info_(range_info) - , layout_(subarray.layout()) - , dim_num_(subarray.dim_num()) + , layout_( + root_subarray.layout() == Layout::GLOBAL_ORDER ? + cell_order : + root_subarray.layout()) + , dim_num_(root_subarray.dim_num()) , global_offset_(0) , pos_in_tile_(0) , dest_offset_row_col_(0) @@ -75,13 +78,9 @@ TileCellSlabIter::TileCellSlabIter( , end_(false) , last_(true) , global_order_(root_subarray.layout() == Layout::GLOBAL_ORDER) - , mult_extents_(subarray.dim_num()) + , ranges_(subarray.ranges()) + , mult_extents_(root_subarray.dim_num()) , start_coords_(start_coords) { - if (!init_ranges(subarray).ok()) { - throw std::logic_error( - "Could not initialize TileCellSlabIter in constructor"); - } - init_coords(); init_cell_slab_lengths(); @@ -287,27 +286,6 @@ void TileCellSlabIter::init_coords() { } } -template -Status TileCellSlabIter::init_ranges(const Subarray& subarray) { - // For easy reference - const auto& array_schema = subarray.array()->array_schema_latest(); - auto array_domain = array_schema.domain().domain(); - uint64_t range_num; - const tiledb::type::Range* r; - ranges_.resize(dim_num_); - for (int d = 0; d < dim_num_; ++d) { - subarray.get_range_num(d, &range_num); - ranges_[d].reserve(range_num); - for (uint64_t j = 0; j < range_num; ++j) { - subarray.get_range(d, j, &r); - auto range = (const T*)(*r).data(); - ranges_[d].emplace_back(range[0], range[1]); - } - } - - return Status::Ok(); -} - template void TileCellSlabIter::update_cell_slab() { // Compute the cell slab length. diff --git a/tiledb/sm/subarray/tile_cell_slab_iter.h b/tiledb/sm/subarray/tile_cell_slab_iter.h index cf898e8f3160..ba92d6554189 100644 --- a/tiledb/sm/subarray/tile_cell_slab_iter.h +++ b/tiledb/sm/subarray/tile_cell_slab_iter.h @@ -64,32 +64,6 @@ struct RangeInfo { template class TileCellSlabIter { public: - /* ********************************* */ - /* TYPE DEFINITIONS */ - /* ********************************* */ - - /** - * Stores information about a range along a single dimension. The - * whole range resides in a single tile. - */ - struct Range { - /** The start of the range in global coordinates. */ - T start_; - /** The end of the range in global coordinates. */ - T end_; - - /** Constructor. */ - Range(T start, T end) - : start_(start) - , end_(end) { - } - - /** Equality operator. */ - bool operator==(const Range& r) const { - return (r.start_ == start_ && r.end_ == end_); - } - }; - /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -99,7 +73,7 @@ class TileCellSlabIter { const uint64_t range_thread_idx, const uint64_t num_range_threads, const Subarray& root_subarray, - const Subarray& subarray, + const DenseTileSubarray& subarray, const std::vector& tile_extents, const std::vector& start_coords, const std::vector>& range_info, @@ -217,7 +191,8 @@ class TileCellSlabIter { * ranges, after appropriately splitting them so that no range crosses * more that one tile. */ - std::vector> ranges_; + const std::vector::DenseTileRange>>& + ranges_; /** Saved multiplication of tile extents in cell order. */ std::vector mult_extents_; @@ -244,13 +219,6 @@ class TileCellSlabIter { /** Initializes the range coords and the cell slab coords. */ void init_coords(); - /** - * Initializes the ranges per dimension, splitting appropriately the - * subarray ranges on tile boundaries. Eventually the produced `ranges_` - * should not never cross more than one tile. - */ - Status init_ranges(const Subarray& subarray); - /** * Updates the current cell slab, based on the current state of * the iterator. From 5cf6377915baa35b9be4b7191d5117e3c5cc697a Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 11 Jun 2024 06:03:51 -0400 Subject: [PATCH 419/456] Eliminate usage of StorageManager in class Array. (#5050) Eliminate usage of `StorageManager` in class `Array`. [sc-48959] --- TYPE: NO_HISTORY DESC: Eliminate usage of `StorageManager` in class `Array`. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- test/src/unit-Reader.cc | 4 ++-- test/src/unit-average-cell-size.cc | 2 +- test/src/unit-enumerations.cc | 10 ++++----- test/src/unit-request-handlers.cc | 2 +- tiledb/sm/array/array.cc | 10 ++++----- tiledb/sm/array/array.h | 12 ++++------- tiledb/sm/array/test/unit_consistency.cc | 9 +++----- tiledb/sm/array/test/unit_consistency.h | 12 +++++------ .../array_schema/test/unit_current_domain.cc | 2 +- tiledb/sm/c_api/tiledb.cc | 14 ++++++------- tiledb/sm/c_api/tiledb_filestore.cc | 10 ++++----- .../consolidator/array_meta_consolidator.cc | 4 ++-- .../sm/consolidator/commits_consolidator.cc | 2 +- .../sm/consolidator/fragment_consolidator.cc | 12 +++++------ .../fragment_meta_consolidator.cc | 2 +- .../array_dimension_label_queries.cc | 2 +- tiledb/sm/query_plan/test/unit_query_plan.cc | 2 +- .../sm/subarray/test/unit_add_ranges_list.cc | 2 +- tools/src/commands/info_command.cc | 21 +++++++------------ 19 files changed, 59 insertions(+), 75 deletions(-) diff --git a/test/src/unit-Reader.cc b/test/src/unit-Reader.cc index 2b68f1350b73..bbc3ba3a669e 100644 --- a/test/src/unit-Reader.cc +++ b/test/src/unit-Reader.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -164,7 +164,7 @@ TEST_CASE_METHOD( std::unordered_map aggregate_buffers; std::optional condition; ThreadPool tp_cpu(4), tp_io(4); - Array array(URI(array_name_), context.storage_manager()); + Array array(context.resources(), URI(array_name_)); CHECK(array.open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0) .ok()); Subarray subarray(&array, &g_helper_stats, g_helper_logger()); diff --git a/test/src/unit-average-cell-size.cc b/test/src/unit-average-cell-size.cc index a5a7a5f07c20..7190374d37ba 100644 --- a/test/src/unit-average-cell-size.cc +++ b/test/src/unit-average-cell-size.cc @@ -207,7 +207,7 @@ struct CPPAverageCellSizeFx { uint64_t a2_size, optional a3_size = std::nullopt) { auto array_for_reads{make_shared( - HERE(), sm::URI(array_name), ctx_.ptr().get()->storage_manager())}; + HERE(), ctx_.ptr().get()->resources(), sm::URI(array_name))}; REQUIRE(array_for_reads ->open_without_fragments( sm::EncryptionType::NO_ENCRYPTION, nullptr, 0) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 97a24cd5960d..e8aee8b2df7f 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -1093,7 +1093,7 @@ TEST_CASE_METHOD( EnumerationFx, "Array - Get Enumeration Error - Not Open", "[enumeration][array][error][not-open]") { - auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + auto array = make_shared(HERE(), ctx_.resources(), uri_); auto matcher = Catch::Matchers::ContainsSubstring("Array is not open"); REQUIRE_THROWS(array->get_enumeration("foo"), matcher); } @@ -1140,7 +1140,7 @@ TEST_CASE_METHOD( EnumerationFx, "Array - Load All Enumerations Error - Not Open", "[enumeration][array][error][not-open]") { - auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + auto array = make_shared(HERE(), ctx_.resources(), uri_); auto matcher = Catch::Matchers::ContainsSubstring("Array is not open"); REQUIRE_THROWS(array->load_all_enumerations()); } @@ -2464,7 +2464,7 @@ TEST_CASE_METHOD( throw_if_not_ok(cfg.set("rest.load_enumerations_on_array_open", do_load)); Context ctx(cfg); - auto a1 = make_shared(HERE(), uri_, ctx.storage_manager()); + auto a1 = make_shared(HERE(), ctx.resources(), uri_); throw_if_not_ok( a1->open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); REQUIRE(a1->serialize_enumerations() == (do_load == "true")); @@ -2472,7 +2472,7 @@ TEST_CASE_METHOD( a1->array_schema_latest_ptr()->get_loaded_enumeration_names().size() == 0); - auto a2 = make_shared(HERE(), uri_, ctx.storage_manager()); + auto a2 = make_shared(HERE(), ctx.resources(), uri_); ser_des_array(ctx, a1.get(), a2.get(), client_side, ser_type); @@ -2885,7 +2885,7 @@ void EnumerationFx::create_array() { } shared_ptr EnumerationFx::get_array(QueryType type) { - auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + auto array = make_shared(HERE(), ctx_.resources(), uri_); throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); return array; } diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index 17c5287909fb..b4702f87f30f 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -369,7 +369,7 @@ void RequestHandlerFx::delete_array() { } shared_ptr RequestHandlerFx::get_array(QueryType type) { - auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + auto array = make_shared(HERE(), ctx_.resources(), uri_); throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); return array; } diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 9737d2e5ba5a..d0921e92d9d3 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -58,7 +58,6 @@ #include "tiledb/sm/query/deletes_and_updates/serialization.h" #include "tiledb/sm/query/update_value.h" #include "tiledb/sm/rest/rest_client.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/sm/tile/generic_tile_io.h" #include @@ -88,10 +87,11 @@ ConsistencyController& controller() { /* ********************************* */ Array::Array( + ContextResources& resources, const URI& array_uri, - StorageManager* storage_manager, ConsistencyController& cc) - : array_uri_(array_uri) + : resources_(resources) + , array_uri_(array_uri) , array_uri_serialized_(array_uri) , is_open_(false) , is_opening_or_closing_(false) @@ -99,11 +99,9 @@ Array::Array( , user_set_timestamp_end_(nullopt) , array_dir_timestamp_end_(UINT64_MAX) , new_component_timestamp_(nullopt) - , storage_manager_(storage_manager) - , resources_(storage_manager_->resources()) , config_(resources_.config()) , remote_(array_uri.is_tiledb()) - , memory_tracker_(storage_manager->resources().create_memory_tracker()) + , memory_tracker_(resources_.create_memory_tracker()) , consistency_controller_(cc) , consistency_sentry_(nullopt) { } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index dc955493a85a..9628ef810a08 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -45,7 +45,6 @@ #include "tiledb/sm/crypto/encryption_key.h" #include "tiledb/sm/fragment/fragment_info.h" #include "tiledb/sm/metadata/metadata.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -302,8 +301,8 @@ class Array { /** Constructor. */ Array( + ContextResources& resources, const URI& array_uri, - StorageManager* storage_manager, ConsistencyController& cc = controller()); /** Destructor. */ @@ -1064,6 +1063,9 @@ class Array { /* PRIVATE ATTRIBUTES */ /* ********************************* */ + /** TileDB Context Resources. */ + ContextResources& resources_; + /** The opened array that can be used by queries. */ shared_ptr opened_array_; @@ -1123,12 +1125,6 @@ class Array { */ optional new_component_timestamp_; - /** TileDB storage manager. */ - StorageManager* storage_manager_; - - /** TileDB Context Resources. */ - ContextResources& resources_; - /** The array config. */ Config config_; diff --git a/tiledb/sm/array/test/unit_consistency.cc b/tiledb/sm/array/test/unit_consistency.cc index 212a24dfbc40..1aeb1522eb90 100644 --- a/tiledb/sm/array/test/unit_consistency.cc +++ b/tiledb/sm/array/test/unit_consistency.cc @@ -172,10 +172,9 @@ TEST_CASE( Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, make_shared(HERE(), ""), config); // Register array - tdb_unique_ptr array = x.open_array(uri, &sm); + tdb_unique_ptr array = x.open_array(resources, uri); REQUIRE(x.registry_size() == 1); REQUIRE(x.is_open(uri) == true); REQUIRE(tiledb::sm::utils::parse::is_element_of(uri, uri) == true); @@ -198,7 +197,6 @@ TEST_CASE( Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, make_shared(HERE(), ""), config); std::vector> arrays; std::vector uris = { @@ -207,7 +205,7 @@ TEST_CASE( // Register arrays size_t count = 0; for (auto uri : uris) { - arrays.insert(arrays.begin(), x.open_array(uri, &sm)); + arrays.insert(arrays.begin(), x.open_array(resources, uri)); count++; REQUIRE(x.registry_size() == count); REQUIRE(x.is_open(uri) == true); @@ -241,10 +239,9 @@ TEST_CASE( Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, make_shared(HERE(), ""), config); // Create an array - tdb_unique_ptr array = x.create_array(uri, &sm); + tdb_unique_ptr array = x.create_array(resources, uri); // Open an array for exclusive modification auto st = array->open( diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 69dc74761bfa..426fcfabb533 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -45,7 +45,6 @@ #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/layout.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include "tiledb/storage_format/uri/parse_uri.h" using namespace tiledb; @@ -99,7 +98,8 @@ class WhiteboxConsistencyController : public ConsistencyController { * Warning: This does not clean up leftovers from previous failed runs. * Manual intervention may be required in the build tree. */ - tdb_unique_ptr create_array(const URI uri, StorageManager* sm) { + tdb_unique_ptr create_array( + ContextResources& resources, const URI uri) { // Create Domain uint64_t dim_dom[2]{0, 1}; uint64_t tile_extent = 1; @@ -124,15 +124,15 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - throw_if_not_ok(Array::create(sm->resources(), uri, schema, key)); - tdb_unique_ptr array(new Array{uri, sm, *this}); + throw_if_not_ok(Array::create(resources, uri, schema, key)); + tdb_unique_ptr array(new Array{resources, uri, *this}); return array; } - tdb_unique_ptr open_array(const URI uri, StorageManager* sm) { + tdb_unique_ptr open_array(ContextResources& resources, const URI uri) { // Create array - tdb_unique_ptr array{create_array(uri, sm)}; + tdb_unique_ptr array{create_array(resources, uri)}; // Open the array Status st = diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc index 204d4aea355b..077ddc0a0c60 100644 --- a/tiledb/sm/array_schema/test/unit_current_domain.cc +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -266,7 +266,7 @@ void CurrentDomainFx::create_array(shared_ptr schema) { template shared_ptr CurrentDomainFx::get_array(QueryType type) { - auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + auto array = make_shared(HERE(), ctx_.resources(), uri_); throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); return array; } diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 26249ea91b64..c1499629a317 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2164,7 +2164,7 @@ int32_t tiledb_array_alloc( // Allocate an array object try { (*array)->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (std::bad_alloc&) { auto st = Status_Error( "Failed to create TileDB array object; Memory allocation error"); @@ -2224,7 +2224,7 @@ int32_t tiledb_array_delete(tiledb_ctx_t* ctx, const char* uri) { tiledb_array_t* array = new (std::nothrow) tiledb_array_t; try { array->array_ = make_shared( - HERE(), tiledb::sm::URI(uri), ctx->storage_manager()); + HERE(), ctx->resources(), tiledb::sm::URI(uri)); } catch (std::bad_alloc&) { auto st = Status_Error( "Failed to create TileDB array object; Memory allocation error"); @@ -2312,7 +2312,7 @@ capi_return_t tiledb_array_delete_fragments_v2( tiledb_array_t* array = new (std::nothrow) tiledb_array_t; try { array->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (...) { delete array; array = nullptr; @@ -2382,7 +2382,7 @@ capi_return_t tiledb_array_delete_fragments_list( tiledb_array_t* array = new (std::nothrow) tiledb_array_t; try { array->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (...) { delete array; array = nullptr; @@ -3372,7 +3372,7 @@ int32_t tiledb_deserialize_array( // Allocate an array object try { (*array)->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (std::bad_alloc&) { auto st = Status_Error( "Failed to create TileDB array object; Memory allocation " @@ -3526,7 +3526,7 @@ int32_t tiledb_deserialize_array_open( // Allocate an array object try { (*array)->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (std::bad_alloc&) { auto st = Status_Error( "Failed to create TileDB array object; Memory allocation " @@ -3707,7 +3707,7 @@ int32_t tiledb_deserialize_query_and_array( // Allocate an array object try { (*array)->array_ = - make_shared(HERE(), uri, ctx->storage_manager()); + make_shared(HERE(), ctx->resources(), uri); } catch (std::bad_alloc&) { auto st = Status_Error( "Failed to create TileDB array object; Memory allocation error"); diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 468893875497..8ee15e6913ee 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -212,7 +212,7 @@ int32_t tiledb_filestore_uri_import( // Sync up the fragment timestamp and metadata timestamp uint64_t time_now = tiledb_timestamp_now_ms(); auto array = make_shared( - HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + HERE(), context.resources(), tiledb::sm::URI(filestore_array_uri)); throw_if_not_ok(array->open( tiledb::sm::QueryType::WRITE, 0, @@ -399,7 +399,7 @@ int32_t tiledb_filestore_uri_export( } auto array = make_shared( - HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + HERE(), context.resources(), tiledb::sm::URI(filestore_array_uri)); throw_if_not_ok(array->open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION, @@ -502,7 +502,7 @@ int32_t tiledb_filestore_buffer_import( // Sync up the fragment timestamp and metadata timestamp uint64_t time_now = tiledb_timestamp_now_ms(); auto array = make_shared( - HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + HERE(), context.resources(), tiledb::sm::URI(filestore_array_uri)); throw_if_not_ok(array->open( tiledb::sm::QueryType::WRITE, 0, @@ -582,7 +582,7 @@ int32_t tiledb_filestore_buffer_export( tiledb::sm::Context& context = ctx->context(); auto array = make_shared( - HERE(), tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + HERE(), context.resources(), tiledb::sm::URI(filestore_array_uri)); throw_if_not_ok(array->open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION, @@ -642,7 +642,7 @@ int32_t tiledb_filestore_size( tiledb::sm::Context& context = ctx->context(); tiledb::sm::Array array( - tiledb::sm::URI(filestore_array_uri), context.storage_manager()); + context.resources(), tiledb::sm::URI(filestore_array_uri)); throw_if_not_ok(array.open( tiledb::sm::QueryType::READ, diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 6cf4f6f110ed..52c8356f3afa 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -69,7 +69,7 @@ Status ArrayMetaConsolidator::consolidate( // Open array for reading auto array_uri = URI(array_name); - Array array_for_reads(array_uri, storage_manager_); + Array array_for_reads(storage_manager_->resources(), array_uri); RETURN_NOT_OK(array_for_reads.open( QueryType::READ, config_.timestamp_start_, @@ -79,7 +79,7 @@ Status ArrayMetaConsolidator::consolidate( key_length)); // Open array for writing - Array array_for_writes(array_uri, storage_manager_); + Array array_for_writes(storage_manager_->resources(), array_uri); RETURN_NOT_OK_ELSE( array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length), diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index 2544007ff71d..d0859aa9582f 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -68,7 +68,7 @@ Status CommitsConsolidator::consolidate( // Open array for writing auto array_uri = URI(array_name); - Array array_for_writes(array_uri, storage_manager_); + Array array_for_writes(storage_manager_->resources(), array_uri); RETURN_NOT_OK(array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length)); diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index d2cad06f6d88..93d224f6ed34 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -209,14 +209,14 @@ Status FragmentConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - auto array_for_reads{ - make_shared(HERE(), URI(array_name), storage_manager_)}; + auto array_for_reads{make_shared( + HERE(), storage_manager_->resources(), URI(array_name))}; RETURN_NOT_OK(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing auto array_for_writes{make_shared( - HERE(), array_for_reads->array_uri(), storage_manager_)}; + HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; RETURN_NOT_OK(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); @@ -321,14 +321,14 @@ Status FragmentConsolidator::consolidate_fragments( auto timer_se = stats_->start_timer("consolidate_frags"); // Open array for reading - auto array_for_reads{ - make_shared(HERE(), URI(array_name), storage_manager_)}; + auto array_for_reads{make_shared( + HERE(), storage_manager_->resources(), URI(array_name))}; RETURN_NOT_OK(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing auto array_for_writes{make_shared( - HERE(), array_for_reads->array_uri(), storage_manager_)}; + HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; RETURN_NOT_OK(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index 19c7a9bae44c..b67262b34942 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -70,7 +70,7 @@ Status FragmentMetaConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - Array array(URI(array_name), storage_manager_); + Array array(storage_manager_->resources(), URI(array_name)); RETURN_NOT_OK( array.open(QueryType::READ, encryption_type, encryption_key, key_length)); diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc index c6db66ecd8da..6330d25d098a 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc @@ -359,7 +359,7 @@ shared_ptr ArrayDimensionLabelQueries::open_dimension_label( // Create the dimension label. auto label_iter = dimension_labels_.try_emplace( dim_label_name, - make_shared(HERE(), dim_label_uri, storage_manager_)); + make_shared(HERE(), storage_manager_->resources(), dim_label_uri)); const auto dim_label = label_iter.first->second; // Open the dimension label with the same timestamps and encryption as the diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 6246d1c6f758..74fb4b8449cd 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -96,7 +96,7 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { // Create the (empty) array on disk. throw_if_not_ok(Array::create(resources_, uri, schema, key)); - tdb_unique_ptr array(new Array{uri, sm_.get()}); + tdb_unique_ptr array(new Array{resources_, uri}); return array; } diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 86e4ec1a7246..9f85da94488b 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -88,7 +88,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { CHECK(sp_as->add_attribute(sp_attrib).ok()); tiledb::sm::Config cfg; tiledb::sm::Context ctx(cfg); - tiledb::sm::Array a(tiledb::sm::URI{"mem://junk"}, ctx.storage_manager()); + tiledb::sm::Array a(ctx.resources(), tiledb::sm::URI{"mem://junk"}); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); CHECK(tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek) diff --git a/tools/src/commands/info_command.cc b/tools/src/commands/info_command.cc index 06418ed6b597..8d0af6350b97 100644 --- a/tools/src/commands/info_command.cc +++ b/tools/src/commands/info_command.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2018-2021 TileDB, Inc. + * @copyright Copyright (c) 2018-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,15 +42,13 @@ #include "tiledb/sm/enums/encryption_type.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/fragment/fragment_metadata.h" -#include "tiledb/sm/storage_manager/storage_manager.h" #include #include #include #include -namespace tiledb { -namespace cli { +namespace tiledb::cli { using namespace tiledb::sm; @@ -119,11 +117,10 @@ void InfoCommand::print_tile_sizes() const { Config config; auto logger = make_shared(HERE(), ""); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, logger, config); // Open the array URI uri(array_uri_); - Array array(uri, &sm); + Array array(resources, uri); THROW_NOT_OK( array.open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); EncryptionKey enc_key; @@ -193,11 +190,10 @@ void InfoCommand::print_schema_info() const { Config config; auto logger = make_shared(HERE(), ""); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, logger, config); // Open the array URI uri(array_uri_); - Array array(uri, &sm); + Array array(resources, uri); THROW_NOT_OK( array.open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -211,11 +207,10 @@ void InfoCommand::write_svg_mbrs() const { Config config; auto logger = make_shared(HERE(), ""); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, logger, config); // Open the array URI uri(array_uri_); - Array array(uri, &sm); + Array array(resources, uri); THROW_NOT_OK( array.open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -287,11 +282,10 @@ void InfoCommand::write_text_mbrs() const { Config config; auto logger = make_shared(HERE(), ""); ContextResources resources(config, logger, 1, 1, ""); - StorageManager sm(resources, logger, config); // Open the array URI uri(array_uri_); - Array array(uri, &sm); + Array array(resources, uri); THROW_NOT_OK( array.open(QueryType::READ, EncryptionType::NO_ENCRYPTION, nullptr, 0)); @@ -560,5 +554,4 @@ std::vector InfoCommand::mbr_to_string( return result; } -} // namespace cli -} // namespace tiledb +} // namespace tiledb::cli From b0fce925a9e270406e8134e12da2a6507281527f Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 11 Jun 2024 08:36:37 -0400 Subject: [PATCH 420/456] Remove StorageManager from the FilteredData constructor. (#5063) Replace `StorageManager*` with `ContextResources&` in the `FilteredData` constructor. [sc-47813] --- TYPE: NO_HISTORY DESC: Replace `StorageManager*` with `ContextResources&` in the `FilteredData` constructor. Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/sm/query/readers/filtered_data.h | 8 ++++---- tiledb/sm/query/readers/reader_base.cc | 2 +- tiledb/sm/query/readers/reader_base.h | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tiledb/sm/query/readers/filtered_data.h b/tiledb/sm/query/readers/filtered_data.h index db85d2b1c050..455fc71a2b04 100644 --- a/tiledb/sm/query/readers/filtered_data.h +++ b/tiledb/sm/query/readers/filtered_data.h @@ -36,7 +36,7 @@ #include "tiledb/common/common.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/common/status.h" -#include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -158,6 +158,7 @@ class FilteredData { /** * Constructor using a sorted list of result tiles. * + * @param resources The context resources. * @param reader Reader object used to know which tile to skip. * @param min_batch_size Minimum batch size we are trying to reach. * @param max_batch_size Maximum batch size to create. @@ -171,11 +172,11 @@ class FilteredData { * @param var_sized Is the field var sized? * @param nullable Is the field nullable? * @param validity_only Is the field read for validity only? - * @param storage_manager Storage manager. * @param read_tasks Read tasks to queue new tasks on for new data blocks. * @param memory_tracker Memory tracker. */ FilteredData( + ContextResources& resources, const ReaderBase& reader, const uint64_t min_batch_size, const uint64_t max_batch_size, @@ -186,10 +187,9 @@ class FilteredData { const bool var_sized, const bool nullable, const bool validity_only, - StorageManager* storage_manager, std::vector& read_tasks, shared_ptr memory_tracker) - : resources_(storage_manager->resources()) + : resources_(resources) , memory_tracker_(memory_tracker) , fixed_data_blocks_( memory_tracker_->get_resource(MemoryType::FILTERED_DATA)) diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index cff2edb153b8..4059ec13b02e 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -573,6 +573,7 @@ std::list ReaderBase::read_tiles( const bool var_sized{array_schema_.var_size(name)}; const bool nullable{array_schema_.is_nullable(name)}; filtered_data.emplace_back( + resources_, *this, min_batch_size_, max_batch_size_, @@ -583,7 +584,6 @@ std::list ReaderBase::read_tiles( var_sized, nullable, val_only, - storage_manager_, read_tasks, memory_tracker_); diff --git a/tiledb/sm/query/readers/reader_base.h b/tiledb/sm/query/readers/reader_base.h index 01f59d004548..8db390e6a556 100644 --- a/tiledb/sm/query/readers/reader_base.h +++ b/tiledb/sm/query/readers/reader_base.h @@ -47,7 +47,6 @@ #include "tiledb/sm/query/readers/result_cell_slab.h" #include "tiledb/sm/query/readers/result_space_tile.h" #include "tiledb/sm/query/writers/domain_buffer.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/subarray_partitioner.h" namespace tiledb::sm { From 381fd4fc1d6743cf4a3ebdf840d13b1eee233994 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 11 Jun 2024 16:45:10 +0300 Subject: [PATCH 421/456] Add CMake option to disable auto-downloading of vcpkg. (#5048) [SC-48929](https://app.shortcut.com/tiledb-inc/story/48929/add-cmake-option-to-disable-auto-downloading-vcpkg) This PR adds a CMake option to disable auto-downloading of vcpkg, as requested in https://github.com/msys2/MINGW-packages/pull/21081#issuecomment-2149716403. There is already an environment variable that serves the same purpose, but the CMake option has the advantage of being cached across configures. I also updated the error message when `TILEDB_VCPKG` is disabled, making the migration path more clear. Validated by configuring locally with this option specified, and observing that vcpkg was not downloaded, and `find_package` calls failed. --- TYPE: BUILD DESC: Automatic downloading of vcpkg can be disabled by enabling the `TILEDB_DISABLE_AUTO_VCPKG` CMake option, in addition to setting the environment variable with trhe same name. --- cmake/Options/BuildOptions.cmake | 3 ++- cmake/Options/TileDBToolchain.cmake | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 4bc86d5f7330..8415c3b02295 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -34,6 +34,7 @@ option(TILEDB_LOG_OUTPUT_ON_FAILURE "If true, print error logs if dependency sub option(TILEDB_SKIP_S3AWSSDK_DIR_LENGTH_CHECK "If true, skip check needed path length for awssdk (TILEDB_S3) dependent builds" OFF) option(TILEDB_EXPERIMENTAL_FEATURES "If true, build and include experimental features" OFF) option(TILEDB_TESTS_AWS_S3_CONFIG "Use an S3 config appropriate for AWS in tests" OFF) +option(TILEDB_DISABLE_AUTO_VCPKG "Do not automatically download vcpkg. Ignored if CMAKE_TOOLCHAIN_FILE or ENV{VCPKG_ROOT} is set." OFF) option(CMAKE_EXPORT_COMPILE_COMMANDS "cmake compile commands" ON) @@ -49,7 +50,7 @@ if (DEFINED TILEDB_STATIC) endif() if (NOT TILEDB_VCPKG) - message(FATAL_ERROR "Disabling TILEDB_VCPKG is not supported.") + message(FATAL_ERROR "Disabling TILEDB_VCPKG is not supported. To disable automatically downloading vcpkg, enable the TILEDB_DISABLE_AUTO_VCPKG option, or set ENV{TILEDB_DISABLE_AUTO_VCPKG} to any value.") endif() # enable assertions by default for debug builds diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 40befe0460ae..2a566b58f58b 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -41,7 +41,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "Vcpkg toolchain file") - elseif(NOT DEFINED ENV{TILEDB_DISABLE_AUTO_VCPKG}) + elseif(NOT (DEFINED ENV{TILEDB_DISABLE_AUTO_VCPKG} OR TILEDB_DISABLE_AUTO_VCPKG)) # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # To help with resolving conflicts, when you update the commit, also update its date. From adc7d9d9d1ee1ebcdb839add78ce454d92a7a3b8 Mon Sep 17 00:00:00 2001 From: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:09:44 -0600 Subject: [PATCH 422/456] Isolate tests failing under windows-2022 runner. (#5066) This PR rewrites a test case that's currently failing with an undifferentiated SIGSEGV error into two test cases, with each test vector in its own DYNAMIC_SECTION. [sc-49381] --- TYPE: NO_HISTORY DESC: Isolate tests failing under windows-2022 runner. --- tiledb/sm/filesystem/test/unit_uri.cc | 85 ++++++++++++++++----------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/tiledb/sm/filesystem/test/unit_uri.cc b/tiledb/sm/filesystem/test/unit_uri.cc index 8e7480164c13..a2a40e7c29ae 100644 --- a/tiledb/sm/filesystem/test/unit_uri.cc +++ b/tiledb/sm/filesystem/test/unit_uri.cc @@ -184,44 +184,61 @@ TEST_CASE("URI: Test schemes", "[uri]") { CHECK(URI("tiledb://namespace/array").is_tiledb()); } -TEST_CASE("URI: Test REST components", "[uri]") { +TEST_CASE("URI: Test REST components, valid", "[uri]") { std::string ns, array; - CHECK(!URI("").get_rest_components(&ns, &array).ok()); - CHECK(!URI("abc").get_rest_components(&ns, &array).ok()); - CHECK(!URI("path/to/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("/path/to/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("file:///path/to/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("s3://bucket/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("azure://container/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("http://bucket/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("https://bucket/dir").get_rest_components(&ns, &array).ok()); - CHECK(!URI("hdfs://namenode/dir").get_rest_components(&ns, &array).ok()); + struct test_vector { + std::string uri; + std::string ns; + std::string array; + }; + const test_vector valid_rest_uri[] = { + {"tiledb://namespace/array", "namespace", "array"}, + {"tiledb://namespace/array/uri", "namespace", "array/uri"}, + {"tiledb://namespace/s3://bucket/dir", "namespace", "s3://bucket/dir"}, + {"tiledb://namespace/azure://container/dir", + "namespace", + "azure://container/dir"}}; + constexpr size_t N = sizeof(valid_rest_uri) / sizeof(test_vector); - CHECK(URI("tiledb://namespace/array").get_rest_components(&ns, &array).ok()); - CHECK(ns == "namespace"); - CHECK(array == "array"); - CHECK(URI("tiledb://namespace/array/uri") - .get_rest_components(&ns, &array) - .ok()); - CHECK(ns == "namespace"); - CHECK(array == "array/uri"); - CHECK(URI("tiledb://namespace/s3://bucket/dir") - .get_rest_components(&ns, &array) - .ok()); - CHECK(ns == "namespace"); - CHECK(array == "s3://bucket/dir"); - CHECK(URI("tiledb://namespace/azure://container/dir") - .get_rest_components(&ns, &array) - .ok()); - CHECK(ns == "namespace"); - CHECK(array == "azure://container/dir"); + for (size_t j = 0; j < N; ++j) { + auto x{valid_rest_uri[j]}; + auto uri{x.uri}; + DYNAMIC_SECTION("\"" << uri << "\" valid") { + CHECK(URI(uri).get_rest_components(&ns, &array).ok()); + CHECK(ns == x.ns); + CHECK(array == x.array); + } + } +} + +TEST_CASE("URI: Test REST components, invalid", "[uri]") { + std::string ns, array; + + const std::string invalid_rest_uri[] = { + "", + "abc", + "path/to/dir", + "/path/to/dir", + "file:///path/to/dir", + "s3://bucket/dir", + "azure://container/dir", + "http://bucket/dir", + "https://bucket/dir", + "hdfs://namenode/dir", + "tiledb:///array", + "tiledb://ns", + "tiledb://ns/", + "tiledb://", + "tiledb:///"}; + constexpr size_t N{sizeof(invalid_rest_uri) / sizeof(std::string)}; - CHECK(!URI("tiledb:///array").get_rest_components(&ns, &array).ok()); - CHECK(!URI("tiledb://ns").get_rest_components(&ns, &array).ok()); - CHECK(!URI("tiledb://ns/").get_rest_components(&ns, &array).ok()); - CHECK(!URI("tiledb://").get_rest_components(&ns, &array).ok()); - CHECK(!URI("tiledb:///").get_rest_components(&ns, &array).ok()); + for (size_t j = 0; j < N; ++j) { + std::string x{invalid_rest_uri[j]}; + DYNAMIC_SECTION("\"" << x << "\" invalid") { + CHECK(!URI(x).get_rest_components(&ns, &array).ok()); + } + } } TEST_CASE("URI: Test get_fragment_name", "[uri][get_fragment_name]") { From 7ca1c0a2b3ee77590eaf5b62a152738fa474bb71 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:15:05 -0400 Subject: [PATCH 423/456] Eliminate usage of StorageManager from class Group. (#5062) Eliminate usage of `StorageManager` from class `Group`. [sc-49279] --- TYPE: NO_HISTORY DESC: Eliminate usage of `StorageManager` from class `Group`. Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/api/c_api/group/group_api.cc | 2 +- tiledb/api/c_api/group/group_api_internal.h | 6 +++--- .../sm/consolidator/group_meta_consolidator.cc | 6 ++---- tiledb/sm/group/group.cc | 18 ++++++------------ tiledb/sm/group/group.h | 18 ++---------------- 5 files changed, 14 insertions(+), 36 deletions(-) diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index 179ad5229beb..ba4b15371c29 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -100,7 +100,7 @@ capi_return_t tiledb_group_alloc( "Failed to allocate TileDB group API object; Invalid URI"); } - *group = tiledb_group_handle_t::make_handle(uri, ctx->storage_manager()); + *group = tiledb_group_handle_t::make_handle(ctx->resources(), uri); return TILEDB_OK; } diff --git a/tiledb/api/c_api/group/group_api_internal.h b/tiledb/api/c_api/group/group_api_internal.h index 1df1977b94ae..25c9d7af4607 100644 --- a/tiledb/api/c_api/group/group_api_internal.h +++ b/tiledb/api/c_api/group/group_api_internal.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -51,8 +51,8 @@ struct tiledb_group_handle_t tiledb_group_handle_t() = delete; explicit tiledb_group_handle_t( - const tiledb::sm::URI& uri, tiledb::sm::StorageManager* sm) - : group_(sm->resources(), uri, sm) { + tiledb::sm::ContextResources& resources, const tiledb::sm::URI& uri) + : group_(resources, uri) { } [[nodiscard]] inline tiledb::sm::Group& group() { diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index 7436a2a462f9..ccb6099bb4a5 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -68,14 +68,12 @@ Status GroupMetaConsolidator::consolidate( // Open group for reading auto group_uri = URI(group_name); - Group group_for_reads( - storage_manager_->resources(), group_uri, storage_manager_); + Group group_for_reads(storage_manager_->resources(), group_uri); group_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_); // Open group for writing - Group group_for_writes( - storage_manager_->resources(), group_uri, storage_manager_); + Group group_for_writes(storage_manager_->resources(), group_uri); try { group_for_writes.open(QueryType::WRITE); diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index e2cca0923f5f..7701b7b6009a 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -66,13 +66,9 @@ class GroupException : public StatusException { } }; -Group::Group( - ContextResources& resources, - const URI& group_uri, - StorageManager* storage_manager) +Group::Group(ContextResources& resources, const URI& group_uri) : memory_tracker_(resources.create_memory_tracker()) , group_uri_(group_uri) - , storage_manager_(storage_manager) , config_(resources.config()) , remote_(group_uri.is_tiledb()) , metadata_(memory_tracker_) @@ -102,9 +98,7 @@ Status Group::create(ContextResources& resources, const URI& uri) { std::lock_guard lock{object_mtx}; if (uri.is_tiledb()) { - StorageManager storage_manager( - resources, resources.logger(), resources.config()); - Group group(resources, uri, &storage_manager); + Group group(resources, uri); throw_if_not_ok( resources.rest_client()->post_group_create_to_rest(uri, &group)); return Status::Ok(); @@ -369,7 +363,7 @@ void Group::delete_group(const URI& uri, bool recursive) { if (member->type() == ObjectType::ARRAY) { Array::delete_array(resources_, member_uri); } else if (member->type() == ObjectType::GROUP) { - Group group_rec(resources_, member_uri, storage_manager_); + Group group_rec(resources_, member_uri); group_rec.open(QueryType::MODIFY_EXCLUSIVE); group_rec.delete_group(member_uri, true); } @@ -808,7 +802,7 @@ std::string Group::dump( member_uri = group_uri_.join_path(it->uri().to_string()); } - Group group_rec(resources_, member_uri, storage_manager_); + Group group_rec(resources_, member_uri); try { group_rec.open(QueryType::READ); ss << std::endl; @@ -927,7 +921,7 @@ void Group::load_group_from_uri(const URI& uri) { uri, 0, *encryption_key(), - storage_manager_->resources().ephemeral_memory_tracker()); + resources_.ephemeral_memory_tracker()); resources_.stats().add_counter("read_group_size", tile->size()); @@ -952,7 +946,7 @@ void Group::load_group_from_all_uris(const std::vector& uris) { uri.uri_, 0, *encryption_key(), - storage_manager_->resources().ephemeral_memory_tracker()); + resources_.ephemeral_memory_tracker()); resources_.stats().add_counter("read_group_size", tile->size()); diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 1480178bb5b0..2445d876fa3d 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -42,7 +42,6 @@ #include "tiledb/sm/group/group_directory.h" #include "tiledb/sm/group/group_member.h" #include "tiledb/sm/metadata/metadata.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -60,22 +59,12 @@ class GroupDetailsException : public StatusException { class Group { public: /** - * Constructs a Group object given a uri and a ContextResources reference. - * This is a transitional constructor in the sense that we are working - * on removing the dependency of the Group class on StorageManager. For - * now we still need to keep the storage_manager argument, but once the - * dependency is gone the signature will be - * Group(ContextResources&, const URI&). + * Constructs a Group object given a ContextResources reference and a uri. * * @param resources A ContextResources reference * @param group_uri The location of the group - * @param storage_manager A StorageManager pointer - * (this will go away in the near future) */ - Group( - ContextResources& resources, - const URI& group_uri, - StorageManager* storage_manager); + Group(ContextResources& resources, const URI& group_uri); /** Destructor. */ ~Group() = default; @@ -443,9 +432,6 @@ class Group { /** The group directory object for listing URIs. */ shared_ptr group_dir_; - /** TileDB storage manager. */ - StorageManager* storage_manager_; - /** The group config. */ Config config_; From 18f02243959e45a346cdb056ac2bb687d548697d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 12 Jun 2024 00:10:23 +0300 Subject: [PATCH 424/456] Add current domain to the storage format history. (#5076) Add the current domain to the storage version history in format_spec/FORMAT_SPEC.md. [sc-42489] --- TYPE: NO_HISTORY DESC: Add current domain to the storage format history. --- format_spec/FORMAT_SPEC.md | 1 + 1 file changed, 1 insertion(+) diff --git a/format_spec/FORMAT_SPEC.md b/format_spec/FORMAT_SPEC.md index 958941d7c5e0..e7750a9b9e23 100644 --- a/format_spec/FORMAT_SPEC.md +++ b/format_spec/FORMAT_SPEC.md @@ -36,6 +36,7 @@ title: Format Specification |19|2.16|[Vac files now use relative URIs](https://github.com/TileDB-Inc/TileDB/commit/ef3236a526b67c50138436a16f67ad274c2ca037)| |20|2.17|[Enumerations](https://github.com/TileDB-Inc/TileDB/commit/c0d7c6a50fdeffbcc7d8c9ba4a29230fe22baed6)| |21|2.19|[Tile metadata are now correctly calculated for nullable fixed size strings on dense arrays](https://github.com/TileDB-Inc/TileDB/commit/081bcc5f7ce4bee576f08b97de348236ac88d429)| +|22|2.25|[Add array current domain](https://github.com/TileDB-Inc/TileDB/commit/9116d3c95a83d72545520acb9a7808fc63478963)| ## Table of Contents From 5e387302c2323b97085e1142a13daac7b548ce93 Mon Sep 17 00:00:00 2001 From: Ryan Roelke Date: Tue, 11 Jun 2024 17:12:01 -0400 Subject: [PATCH 425/456] Fix write queries using sm.var_offsets.extra_element=true. (#5033) Story details: https://app.shortcut.com/tiledb-inc/story/48614 The write query uses `coords_info_.coords_num_` as a stand-in for the number of cells in the write query input. This value is computed as `*buffer_offsets_size / constants::cell_var_offset_size`, which makes sense for the traditional tiledb input where the number of offsets is the same as the number of cells. The configuration `sm.var_offsets.extra_element`, however, breaks that assumption. This option is useful for arrow compatibility but also simplifies upstream code - we want to use it in `tiledb-rs`, for example. When this option is used, `coords_num_` must be adjusted accordingly. Previously it was not, and now it is. The added unit test demonstrates that if `num_coords_` is not adjusted, then the unordered writer finds an extra "empty" element at the end. This results in a duplicate coordinate if there is an *actual* empty element. In addition to the unit test, the branch trying to use this feature in `tiledb-rs` passes its property-based testing when linking against this branch of core. --- TYPE: BUG DESC: Fix write queries using `sm.var_offsets.extra_element=true` --- test/src/unit-cppapi-var-offsets.cc | 104 ++++++++++++++++++++++++++++ tiledb/sm/query/query.cc | 11 +++ 2 files changed, 115 insertions(+) diff --git a/test/src/unit-cppapi-var-offsets.cc b/test/src/unit-cppapi-var-offsets.cc index 6c5020f513df..a5a9825fbef8 100644 --- a/test/src/unit-cppapi-var-offsets.cc +++ b/test/src/unit-cppapi-var-offsets.cc @@ -1807,3 +1807,107 @@ TEST_CASE_METHOD( } } } + +TEST_CASE_METHOD( + VariableOffsetsFx, + "C++ API: Test sm.var_offsets.extra_element: sparse array with string " + "dimension", + "[var-offsets-dim][extra-offset][sparse][rest]") { + std::string array_name = + vfs_test_setup_.array_uri("test_32bit_offset_string_dim"); + auto ctx = vfs_test_setup_.ctx(); + + /* + * Use the `sm.var_offsets.extra_element` option on the write + * side and read side respectively and make sure that we can + * read back the same logical data and offsets that we put in. + */ + + const std::string data = "abcdefghij"; + + // Starting offsets of each value; does not include "extra offset" (added by + // write or read if needed) + const std::vector data_elem_offsets = {0, 0, 2, 5, 6}; + + auto do_write_extra_offset = GENERATE(true, false); + auto do_read_extra_offset = GENERATE(true, false); + + // Create and write array + { + Domain domain(ctx); + domain.add_dimension( + Dimension::create(ctx, "dim1", TILEDB_STRING_ASCII, nullptr, nullptr)); + + ArraySchema schema(ctx, TILEDB_SPARSE); + schema.set_domain(domain); + + tiledb::Array::create(array_name, schema); + + auto array = tiledb::Array(ctx, array_name, TILEDB_WRITE); + + Config config; + + std::vector write_offsets = data_elem_offsets; + + Query query(ctx, array, TILEDB_WRITE); + if (do_write_extra_offset) { + config["sm.var_offsets.extra_element"] = "true"; + query.set_config(config); + write_offsets.push_back(data.size()); + } + + query.set_data_buffer("dim1", (char*)data.data(), data.size()); + query.set_offsets_buffer( + "dim1", (uint64_t*)(write_offsets.data()), write_offsets.size()); + + query.set_layout(TILEDB_UNORDERED); + query.submit(); + query.finalize(); + array.close(); + } + + // Read contents back + { + Config config; + if (do_read_extra_offset) { + config["sm.var_offsets.extra_element"] = "true"; + } + + const uint64_t expect_num_read_offsets = + data_elem_offsets.size() + (do_read_extra_offset ? 1 : 0); + std::vector read_offsets(expect_num_read_offsets, 0xFFFFFFFF); + + std::string read_data; + read_data.resize(data.size()); + + auto array = tiledb::Array(ctx, array_name, TILEDB_READ); + Query query(ctx, array, TILEDB_READ); + Subarray subarray(ctx, array); + subarray.add_range(0, std::string(""), std::string("zzzzz")); + query.set_config(config); + query.set_subarray(subarray); + query.set_data_buffer("dim1", (char*)read_data.data(), read_data.size()); + query.set_offsets_buffer( + "dim1", (uint64_t*)read_offsets.data(), read_offsets.size()); + + query.submit(); + + REQUIRE(query.query_status() == Query::Status::COMPLETE); + + const auto results = query.result_buffer_elements(); + const uint64_t num_read_offsets = results.at("dim1").first; + const uint64_t num_read_bytes = results.at("dim1").second; + + CHECK(num_read_offsets == expect_num_read_offsets); + CHECK(num_read_bytes == data.size()); + CHECK(read_data == data); + + const std::vector read_offsets_starts( + read_offsets.begin(), read_offsets.begin() + data_elem_offsets.size()); + CHECK(data_elem_offsets == read_offsets_starts); + + if (do_read_extra_offset) { + CHECK(num_read_bytes == read_offsets.back()); + } + } +} diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index dff7d3caeae1..f9bb85d6d001 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -1231,6 +1231,17 @@ Status Query::set_offsets_buffer( // Check number of coordinates uint64_t coords_num = *buffer_offsets_size / constants::cell_var_offset_size; + + const bool offsets_extra_element_ = + config_.get("sm.var_offsets.extra_element", Config::must_find); + + if (offsets_extra_element_) { + // the offsets buffer has `ncoords + 1` elements so that each coordinate + // is given by `[offset[i], offset[i + 1])` instead of using the length + // to determine the last + --coords_num; + } + if (coord_offsets_buffer_is_set_ && coords_num != coords_info_.coords_num_ && name == offsets_buffer_name_) return logger_->status(Status_QueryError( From ca9b6b60d1e6f8f52a1b6a88d348e2c3290ad999 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 12 Jun 2024 03:13:00 -0400 Subject: [PATCH 426/456] Add v2_24_0 arrays to backward compatibility matrix. (#5075) Add v2_24_0 arrays to backward compatibility matrix. [sc-49171] --- TYPE: NO_HISTORY DESC: Add v2_24_0 arrays to backward compatibility matrix. --- .github/workflows/build-ubuntu20.04-backwards-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml index 6a6109482200..b3d290bc904e 100644 --- a/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml +++ b/.github/workflows/build-ubuntu20.04-backwards-compatibility.yml @@ -59,7 +59,7 @@ jobs: - ubuntu-20.04 # Note: v2_1_0 arrays were never created so its currently skipped # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION - tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0"] + tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0"] timeout-minutes: 30 name: ${{ matrix.tiledb_version }} steps: From 9c10ba8599b368a9bade85cccf29e801a2c0d5f1 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 12 Jun 2024 14:04:21 +0300 Subject: [PATCH 427/456] Move the `check` CMake target to the top-level directory. (#5054) [SC-48054](https://app.shortcut.com/tiledb-inc/story/48054/every-tests-should-be-run-in-windows-debug) Before, the build target `check` was defined in the `tests` subdirectory, which caused it to run only tests in that directory (`tiledb_unit`, `tiledb_regression`, and `test_ci_asserts`). By moving it to the root `CMakeLists.txt`, it runs all tests, including the unit tests on `tiledb/sm`. This will consequently make all tests run on nightly builds. --- TYPE: NO_HISTORY --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- CMakeLists.txt | 10 ++++++++++ test/CMakeLists.txt | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 438eb39f39e3..c178e9829f7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -583,6 +583,16 @@ if (TILEDB_TESTS) endif() add_subdirectory(test/regression) + + # Add custom target 'check', which builds and runs all tests. + # This includes both the main test suite in the 'test' directory (consisting of + # tiledb_unit, tiledb_regression and other tests), as well as the standalone + # tests in subdirectories of the 'tiledb' directory. + add_custom_target(check + COMMAND ${CMAKE_CTEST_COMMAND} -V -C $ + DEPENDS tests + USES_TERMINAL + ) endif() # ------------------------------------------------------- diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f66e8c7c0455..00029343643b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -397,13 +397,6 @@ if(TILEDB_HDFS) endif() endif() -# Add custom target 'check' -add_custom_target(check - COMMAND ${CMAKE_CTEST_COMMAND} -V -C $ - DEPENDS tests - USES_TERMINAL -) - # CI tests add_subdirectory(ci) if(WIN32) From ea131acafef32517e1bbdbe75a43a858dfce120b Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 12 Jun 2024 09:27:00 -0400 Subject: [PATCH 428/456] Eliminate usage of StorageManager from class Subarray. (#5077) Eliminate usage of `StorageManager` from class `Subarray`. [sc-49398] --- TYPE: NO_HISTORY DESC: Eliminate usage of `StorageManager` from class `Subarray`. Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/sm/array/array.h | 5 ++++ tiledb/sm/c_api/tiledb.cc | 3 +- tiledb/sm/c_api/tiledb_filestore.cc | 30 ++++--------------- tiledb/sm/query/writers/writer_base.h | 7 ++--- tiledb/sm/serialization/array.cc | 1 - tiledb/sm/serialization/array_directory.h | 13 ++++---- tiledb/sm/subarray/subarray.cc | 25 +++++----------- tiledb/sm/subarray/subarray.h | 10 ++----- .../sm/subarray/test/unit_add_ranges_list.cc | 6 +--- 9 files changed, 29 insertions(+), 71 deletions(-) diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 9628ef810a08..c1e6ce1a9b34 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -115,6 +115,11 @@ class OpenedArray { encryption_key_->set_key(encryption_type, key_bytes, key_length)); } + /** Returns the context resources. */ + inline ContextResources& resources() { + return resources_; + } + /** Returns the array directory object. */ inline const ArrayDirectory& array_directory() const { return array_dir_.value(); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index c1499629a317..7b10b8f755e2 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1723,8 +1723,7 @@ capi_return_t tiledb_subarray_alloc( array->array_.get(), (tiledb::sm::stats::Stats*)nullptr, ctx->resources().logger(), - true, - ctx->storage_manager()); + true); (*subarray)->is_allocated_ = true; } catch (...) { delete *subarray; diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index 8ee15e6913ee..cc2d4d1ce031 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -279,11 +279,7 @@ int32_t tiledb_filestore_uri_import( std::vector buffer(buffer_size); tiledb::sm::Subarray subarray( - array.get(), - nullptr, - context.resources().logger(), - true, - context.storage_manager()); + array.get(), nullptr, context.resources().logger(), true); // We need to get the right end boundary of the last space tile. // The last chunk either falls exactly on the end of the file // or it goes beyond the end of the file so that it's equal in size @@ -303,11 +299,7 @@ int32_t tiledb_filestore_uri_import( tiledb::sm::Query query(context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray_cloud_fix( - array.get(), - nullptr, - context.resources().logger(), - true, - context.storage_manager()); + array.get(), nullptr, context.resources().logger(), true); uint64_t cloud_fix_range_arr[] = {start, end}; tiledb::type::Range subarray_range_cloud_fix( @@ -431,11 +423,7 @@ int32_t tiledb_filestore_uri_export( do { uint64_t write_size = end_range - start_range + 1; tiledb::sm::Subarray subarray( - array.get(), - nullptr, - context.resources().logger(), - true, - context.storage_manager()); + array.get(), nullptr, context.resources().logger(), true); uint64_t subarray_range_arr[] = {start_range, end_range}; tiledb::type::Range subarray_range( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); @@ -548,11 +536,7 @@ int32_t tiledb_filestore_buffer_import( throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray( - array.get(), - nullptr, - context.resources().logger(), - true, - context.storage_manager()); + array.get(), nullptr, context.resources().logger(), true); uint64_t subarray_range_arr[] = { static_cast(0), static_cast(size - 1)}; tiledb::type::Range subarray_range( @@ -610,11 +594,7 @@ int32_t tiledb_filestore_buffer_export( } tiledb::sm::Subarray subarray( - array.get(), - nullptr, - context.resources().logger(), - true, - context.storage_manager()); + array.get(), nullptr, context.resources().logger(), true); uint64_t subarray_range_arr[] = { static_cast(offset), static_cast(offset + size - 1)}; tiledb::type::Range subarray_range( diff --git a/tiledb/sm/query/writers/writer_base.h b/tiledb/sm/query/writers/writer_base.h index 10a4464f5c9f..4f085c1b7c89 100644 --- a/tiledb/sm/query/writers/writer_base.h +++ b/tiledb/sm/query/writers/writer_base.h @@ -45,13 +45,11 @@ #include "tiledb/sm/query/strategy_base.h" #include "tiledb/sm/query/writers/dense_tiler.h" #include "tiledb/sm/stats/stats.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/tile/writer_tile_tuple.h" using namespace tiledb::common; -namespace tiledb { -namespace sm { +namespace tiledb::sm { class Array; class DomainBuffersView; @@ -501,7 +499,6 @@ class WriterBase : public StrategyBase, public IQueryStrategy { bool remote_query() const; }; -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm #endif // TILEDB_WRITER_BASE_H diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index 0791ebb1503e..8ec41e05ffca 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -50,7 +50,6 @@ #include "tiledb/sm/serialization/array_directory.h" #include "tiledb/sm/serialization/array_schema.h" #include "tiledb/sm/serialization/fragment_metadata.h" -#include "tiledb/sm/storage_manager/storage_manager.h" using namespace tiledb::common; using namespace tiledb::sm::stats; diff --git a/tiledb/sm/serialization/array_directory.h b/tiledb/sm/serialization/array_directory.h index ec884f7ee826..c45ade4e9b60 100644 --- a/tiledb/sm/serialization/array_directory.h +++ b/tiledb/sm/serialization/array_directory.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -42,13 +42,12 @@ using namespace tiledb::common; -namespace tiledb { -namespace sm { - +namespace tiledb::sm { class ArrayDirectory; enum class SerializationType : uint8_t; +} // namespace tiledb::sm -namespace serialization { +namespace tiledb::sm::serialization { #ifdef TILEDB_SERIALIZATION @@ -78,8 +77,6 @@ void array_directory_to_capnp( #endif -} // namespace serialization -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::serialization #endif // TILEDB_SERIALIZATION_ARRAY_DIRECTORY_H diff --git a/tiledb/sm/subarray/subarray.cc b/tiledb/sm/subarray/subarray.cc index cd5121421dec..a529980d116a 100644 --- a/tiledb/sm/subarray/subarray.cc +++ b/tiledb/sm/subarray/subarray.cc @@ -57,7 +57,7 @@ #include "tiledb/sm/rest/rest_client.h" #include "tiledb/sm/rtree/rtree.h" #include "tiledb/sm/stats/global_stats.h" -#include "tiledb/sm/storage_manager/storage_manager.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/subarray/relevant_fragment_generator.h" #include "tiledb/sm/subarray/subarray.h" #include "tiledb/type/apply_with_type.h" @@ -95,15 +95,13 @@ Subarray::Subarray( const Array* array, Stats* const parent_stats, shared_ptr logger, - const bool coalesce_ranges, - StorageManager* storage_manager) + const bool coalesce_ranges) : Subarray( array->opened_array(), Layout::UNORDERED, parent_stats, logger, - coalesce_ranges, - storage_manager) { + coalesce_ranges) { } Subarray::Subarray( @@ -111,15 +109,13 @@ Subarray::Subarray( const Layout layout, Stats* const parent_stats, shared_ptr logger, - const bool coalesce_ranges, - StorageManager* storage_manager) + const bool coalesce_ranges) : Subarray( array->opened_array(), layout, parent_stats, logger, - coalesce_ranges, - storage_manager) { + coalesce_ranges) { } Subarray::Subarray( @@ -127,14 +123,11 @@ Subarray::Subarray( const Layout layout, Stats* const parent_stats, shared_ptr logger, - const bool coalesce_ranges, - StorageManager* storage_manager) + const bool coalesce_ranges) : stats_( parent_stats ? parent_stats->create_child("Subarray") : - storage_manager ? - storage_manager->resources().stats().create_child("subSubarray") : - nullptr) + opened_array->resources().stats().create_child("subSubarray")) , logger_(logger->clone("Subarray", ++logger_id_)) , array_(opened_array) , layout_(layout) @@ -143,10 +136,6 @@ Subarray::Subarray( , relevant_fragments_(opened_array->fragment_metadata().size()) , coalesce_ranges_(coalesce_ranges) , ranges_sorted_(false) { - if (!parent_stats && !storage_manager) { - throw std::runtime_error( - "Subarray(): missing parent_stats requires live storage_manager!"); - } add_default_ranges(); } diff --git a/tiledb/sm/subarray/subarray.h b/tiledb/sm/subarray/subarray.h index 59cc88955dde..5af504be2785 100644 --- a/tiledb/sm/subarray/subarray.h +++ b/tiledb/sm/subarray/subarray.h @@ -43,7 +43,6 @@ #include "tiledb/sm/misc/tile_overlap.h" #include "tiledb/sm/misc/types.h" #include "tiledb/sm/stats/stats.h" -#include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include "tiledb/sm/subarray/range_subset.h" #include "tiledb/sm/subarray/relevant_fragments.h" #include "tiledb/sm/subarray/subarray_tile_overlap.h" @@ -368,8 +367,7 @@ class Subarray { const Array* array, stats::Stats* parent_stats, shared_ptr logger, - bool coalesce_ranges = true, - StorageManager* storage_manager = nullptr); + bool coalesce_ranges = true); /** * Constructor. @@ -388,8 +386,7 @@ class Subarray { Layout layout, stats::Stats* parent_stats, shared_ptr logger, - bool coalesce_ranges = true, - StorageManager* storage_manager = nullptr); + bool coalesce_ranges = true); /** * Constructor. @@ -408,8 +405,7 @@ class Subarray { Layout layout, stats::Stats* parent_stats, shared_ptr logger, - bool coalesce_ranges = true, - StorageManager* storage_manager = nullptr); + bool coalesce_ranges = true); /** * Copy constructor. This performs a deep copy (including memcpy of diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 9f85da94488b..063ffccaa1e4 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -102,11 +102,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { // The Subarray used to test add_ranges_list. tiledb::sm::Subarray sa( - &a, - &test::g_helper_stats, - test::g_helper_logger(), - true, - ctx.storage_manager()); + &a, &test::g_helper_stats, test::g_helper_logger(), true); // Add ranges // NOTE: The type used here for the range needs to match the type used for the From 4d008c06124cdd65e5ab3ef386644d00d15afa51 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 13 Jun 2024 12:57:38 +0300 Subject: [PATCH 429/456] Use Ninja on macOS CI. (#5069) [SC-49308](https://app.shortcut.com/tiledb-inc/story/49308/macos-runners-significantly-slower-after-pr-5001) This PR updates all macOS CI jobs to use Ninja instead of makefiles, which was observed to fix unexplained significant performance regressions. Certain other scripts were updated to be generator-agnostic by invoking `cmake --build` instead of `make`. Nightly builds were not updated to keep testing coverage with makefiles, and the standalone unit tests were not updated because their performance has not regressed. --- TYPE: NO_HISTORY --- .github/workflows/ci-linux_mac.yml | 18 +++++++++++++++++- .github/workflows/full-ci.yml | 3 +++ scripts/ci/build_benchmarks.sh | 2 +- scripts/ci/build_libtiledb.sh | 12 ++++++------ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index 4f1290403097..d33495ab7ab6 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -51,9 +51,20 @@ on: description: 'Enable manylinux builds' required: false type: boolean + vcpkg_base_triplet: + default: '' + description: 'Base triplet for vcpkg' + required: false + type: string + cmake_generator: + default: 'Unix Makefiles' # TODO: Ninja is much better than makefiles. Should it be the default? + description: 'CMake generator' + required: false + type: string env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF + CMAKE_GENERATOR: ${{ inputs.cmake_generator }} TILEDB_CI_BACKEND: ${{ inputs.ci_backend }} TILEDB_CI_OS: ${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'Linux' || 'macOS' }} # Installing Python does not work on manylinux. @@ -63,7 +74,7 @@ env: CC: ${{ inputs.matrix_compiler_cc }} CFLAGS: ${{ inputs.matrix_compiler_cflags }} CXXFLAGS: ${{ inputs.matrix_compiler_cxxflags }} - bootstrap_args: "--enable-ccache --vcpkg-base-triplet=x64-${{ startsWith(inputs.matrix_image, 'ubuntu-') && 'linux' || 'osx' }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" + bootstrap_args: "--enable-ccache --vcpkg-base-triplet=${{ inputs.vcpkg_base_triplet || (startsWith(inputs.matrix_image, 'ubuntu-') && 'x64-linux' || 'x64-osx') }} ${{ inputs.bootstrap_args }} ${{ inputs.asan && '--enable-sanitizer=address' || '' }}" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' SCCACHE_GHA_ENABLED: "true" @@ -119,6 +130,11 @@ jobs: python-version: '3.9' cache: 'pip' + # This must happen after checkout, because checkout would remove the directory. + - name: Install Ninja + if: inputs.cmake_generator == 'Ninja' + uses: seanmiddleditch/gha-setup-ninja@v4 + - name: 'Set up Python dependencies' if: ${{ !inputs.manylinux }} run: | diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index e09438bd48a4..9f5d940cfaf9 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -58,6 +58,7 @@ jobs: with: ci_backend: S3 matrix_image: macos-12 + cmake_generator: 'Ninja' # Use Ninja due to performance issues. timeout: 120 bootstrap_args: '--enable=s3,serialization,tools --enable-release-symbols' @@ -66,6 +67,7 @@ jobs: with: ci_backend: GCS matrix_image: macos-12 + cmake_generator: 'Ninja' # Use Ninja due to performance issues. timeout: 120 bootstrap_args: '--enable-gcs --enable-release-symbols' @@ -100,6 +102,7 @@ jobs: with: ci_backend: AZURE matrix_image: macos-12 + cmake_generator: 'Ninja' # Use Ninja due to performance issues. timeout: 120 bootstrap_args: '--enable-azure' diff --git a/scripts/ci/build_benchmarks.sh b/scripts/ci/build_benchmarks.sh index a3de85b57ab1..8e4e1f27e587 100644 --- a/scripts/ci/build_benchmarks.sh +++ b/scripts/ci/build_benchmarks.sh @@ -30,5 +30,5 @@ pushd $GITHUB_WORKSPACE/test/benchmarking mkdir -p build cd build cmake -DCMAKE_PREFIX_PATH="$GITHUB_WORKSPACE/build/dist;$GITHUB_WORKSPACE/build/vcpkg_installed/$VCPKG_TARGET_TRIPLET" ../src -make +cmake --build . popd diff --git a/scripts/ci/build_libtiledb.sh b/scripts/ci/build_libtiledb.sh index 17adb2388b96..f94bf3e10f72 100644 --- a/scripts/ci/build_libtiledb.sh +++ b/scripts/ci/build_libtiledb.sh @@ -31,13 +31,13 @@ set -xeuo pipefail cd $GITHUB_WORKSPACE/build -make -j4 +cmake --build . -j4 -make -C tiledb install +cmake --build tiledb --target install ls -la -make -j4 -C tiledb tiledb_unit -make -j4 -C tiledb unit_vfs -make -j4 -C tiledb tiledb_regression -make -j4 -C tiledb all_link_complete +cmake --build tiledb -j4 --target tiledb_unit +cmake --build tiledb -j4 --target unit_vfs +cmake --build tiledb -j4 --target tiledb_regression +cmake --build tiledb -j4 --target all_link_complete From 47dde510d2d9eef186ba546d96d8fcc115116834 Mon Sep 17 00:00:00 2001 From: Shaun M Reed Date: Thu, 13 Jun 2024 06:06:51 -0400 Subject: [PATCH 430/456] Fix segfaults in WebP queries ran in parallel. (#5065) The WebpFilter calls `WriterTile::set_max_tile_chunk_size` to override the maximum tile chunk size in an attempt to avoid further chunking data within each tile, but because `WriterTile::max_tile_chunk_size_` is static this doesn't work as intended when we're running multiple WebP queries across threads. It's possible for one thread to override this value while it's being used by another running query and this results in incorrect chunking for WebP data, potentially causing segfaults from libwebp reaching out of memory bounds. The WebP library was reaching out of bounds because after another thread overrides `max_tile_chunk_size_` we don't account for the smaller chunk's dimensions when calling WebP APIs and still pass in the same height, width, and row stride to [WebP API calls](https://github.com/TileDB-Inc/TileDB/blob/dev/tiledb/sm/filter/webp_filter.cc#L144). This PR removes the use of `WriterTile::set_max_tile_chunk_size` and updates `FilterPipeline::use_tile_chunking` to return `false` for WebP, which IIUC should have the behavior that was initially intended and fixes segfaults seen on REST. [SC-48697](https://app.shortcut.com/tiledb-inc/story/48697/rest-segfaults-running-webp-multiple-queries) Limitations: If a WebP array exists that was impacted by this during ingestion and it did not produce a segfault, that array will still be read back using chunking. AFAICT there is no risk of segfault for this case since the decoding APIs in WebP do not depend on the caller to specify buffer dimensions, instead WebP will decode and provides to the caller the width and height of the decoded image. On the next write to the array there will be no error or risk of segfault and the new tiles will not use chunking moving forward for reads or writes. --- TYPE: BUG DESC: Fix segfaults in WebP queries ran in parallel. --- test/support/src/whitebox_helpers.h | 48 ++++++++ tiledb/sm/filter/filter_pipeline.cc | 3 + tiledb/sm/filter/test/CMakeLists.txt | 1 + tiledb/sm/filter/test/filter_test_support.cc | 4 +- tiledb/sm/filter/test/tile_data_generator.h | 3 +- .../test/unit_bit_width_reduction_pipeline.cc | 12 +- .../filter/test/unit_bitshuffle_pipeline.cc | 6 +- .../filter/test/unit_byteshuffle_pipeline.cc | 6 +- .../test/unit_positive_delta_pipeline.cc | 8 +- .../filter/test/unit_run_filter_pipeline.cc | 2 +- tiledb/sm/filter/test/unit_webp_pipeline.cc | 110 ++++++++++++++++++ tiledb/sm/filter/webp_filter.cc | 1 - tiledb/sm/tile/tile.cc | 4 - tiledb/sm/tile/tile.h | 12 +- 14 files changed, 188 insertions(+), 32 deletions(-) create mode 100644 test/support/src/whitebox_helpers.h create mode 100644 tiledb/sm/filter/test/unit_webp_pipeline.cc diff --git a/test/support/src/whitebox_helpers.h b/test/support/src/whitebox_helpers.h new file mode 100644 index 000000000000..bf9f170d26a8 --- /dev/null +++ b/test/support/src/whitebox_helpers.h @@ -0,0 +1,48 @@ +/** + * @file whitebox_helpers.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Helpers to provide whitebox access to TileDB internals for testing. + */ +#ifndef TILEDB_WHITEBOX_HELPERS_H +#define TILEDB_WHITEBOX_HELPERS_H + +#include "tiledb/sm/tile/tile.h" + +namespace tiledb::sm { + +class WhiteboxWriterTile { + public: + static void set_max_tile_chunk_size(uint64_t size) { + WriterTile::max_tile_chunk_size_ = size; + } +}; + +} // namespace tiledb::sm + +#endif // TILEDB_WHITEBOX_HELPERS_H diff --git a/tiledb/sm/filter/filter_pipeline.cc b/tiledb/sm/filter/filter_pipeline.cc index df39f64f823b..401b31f5b8eb 100644 --- a/tiledb/sm/filter/filter_pipeline.cc +++ b/tiledb/sm/filter/filter_pipeline.cc @@ -45,6 +45,7 @@ #include "tiledb/sm/misc/parallel_functions.h" #include "tiledb/sm/stats/global_stats.h" #include "tiledb/sm/tile/tile.h" +#include "webp_filter.h" using namespace tiledb::common; @@ -644,6 +645,8 @@ bool FilterPipeline::use_tile_chunking( } else if (version >= 13 && has_filter(FilterType::FILTER_DICTIONARY)) { return false; } + } else if (has_filter(FilterType::FILTER_WEBP)) { + return false; } return true; diff --git a/tiledb/sm/filter/test/CMakeLists.txt b/tiledb/sm/filter/test/CMakeLists.txt index edfc14b9068a..3bd3c12392c0 100644 --- a/tiledb/sm/filter/test/CMakeLists.txt +++ b/tiledb/sm/filter/test/CMakeLists.txt @@ -54,6 +54,7 @@ commence(unit_test run_filter_pipeline) unit_encryption_pipeline.cc unit_positive_delta_pipeline.cc unit_run_filter_pipeline.cc + unit_webp_pipeline.cc unit_xor_pipeline.cc ) conclude(unit_test) diff --git a/tiledb/sm/filter/test/filter_test_support.cc b/tiledb/sm/filter/test/filter_test_support.cc index ec1b2f4803c4..f517c5b86509 100644 --- a/tiledb/sm/filter/test/filter_test_support.cc +++ b/tiledb/sm/filter/test/filter_test_support.cc @@ -42,12 +42,12 @@ SimpleVariableTestData::SimpleVariableTestData() : target_ncells_per_chunk_{10} , elements_per_chunk_{14, 6, 11, 7, 10, 10, 20, 10, 12} , tile_data_generator_{{4, 10, 6, 11, 7, 9, 1, 10, 20, 2, 2, 2, 2, 2, 12}} { - WriterTile::set_max_tile_chunk_size( + WhiteboxWriterTile::set_max_tile_chunk_size( target_ncells_per_chunk_ * sizeof(uint64_t)); } SimpleVariableTestData::~SimpleVariableTestData() { - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } /** diff --git a/tiledb/sm/filter/test/tile_data_generator.h b/tiledb/sm/filter/test/tile_data_generator.h index 1caecc356830..284b868c9f4a 100644 --- a/tiledb/sm/filter/test/tile_data_generator.h +++ b/tiledb/sm/filter/test/tile_data_generator.h @@ -37,6 +37,7 @@ #include #include #include +#include "test/support/src/whitebox_helpers.h" #include "tiledb/sm/tile/tile.h" using namespace tiledb::common; @@ -139,7 +140,7 @@ class IncrementTileDataGenerator : public TileDataGenerator { } ~IncrementTileDataGenerator() { - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } uint64_t cell_size() const override { diff --git a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc index ceb2262c7494..e74eca6ca0aa 100644 --- a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc @@ -269,7 +269,7 @@ TEST_CASE( auto tile = make_increasing_tile(nelts, tracker); auto offsets_tile = make_offsets_tile(offsets, tracker); - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); CHECK( pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) .ok()); @@ -335,7 +335,7 @@ TEST_CASE( } SECTION("- Window sizes") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { @@ -362,7 +362,7 @@ TEST_CASE( } SECTION("- Random values") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); std::random_device rd; auto seed = rd(); std::mt19937 gen(seed), gen_copy(seed); @@ -401,7 +401,7 @@ TEST_CASE( } SECTION(" - Random signed values") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); std::random_device rd; auto seed = rd(); std::mt19937 gen(seed), gen_copy(seed); @@ -460,7 +460,7 @@ TEST_CASE( } SECTION("- Byte overflow") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); auto tile = make_shared( HERE(), constants::format_version, @@ -493,5 +493,5 @@ TEST_CASE( } } - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } diff --git a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc index 181b892b22eb..665fc405bef5 100644 --- a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc @@ -143,7 +143,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); CHECK( pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) .ok()); @@ -161,7 +161,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { } SECTION("- Indivisible by 8") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); @@ -194,5 +194,5 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { } } - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } diff --git a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc index c21631bfc4c8..7bd576815e40 100644 --- a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc @@ -141,7 +141,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); CHECK( pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) .ok()); @@ -159,7 +159,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { } SECTION("- Uneven number of elements") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); const uint32_t nelts2 = 1001; const uint64_t tile_size2 = nelts2 * sizeof(uint32_t); @@ -192,5 +192,5 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { } } - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } diff --git a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc index d2239632ee1a..5242b882fd28 100644 --- a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc +++ b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc @@ -173,7 +173,7 @@ TEST_CASE( auto tile = make_increasing_tile(nelts, tracker); auto offsets_tile = make_offsets_tile(offsets, tracker); - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); CHECK( pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) .ok()); @@ -240,7 +240,7 @@ TEST_CASE( } SECTION("- Window sizes") { - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); std::vector window_sizes = { 32, 64, 128, 256, 437, 512, 1024, 2000}; for (auto window_size : window_sizes) { @@ -271,7 +271,7 @@ TEST_CASE( auto tile = make_increasing_tile(nelts, tracker); auto offsets_tile = make_offsets_tile(offsets, tracker); - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); for (uint64_t i = 0; i < nelts; i++) { auto val = nelts - i; CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); @@ -282,5 +282,5 @@ TEST_CASE( .ok()); } - WriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); } diff --git a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc index 3b5645a63032..b543a3b08a93 100644 --- a/tiledb/sm/filter/test/unit_run_filter_pipeline.cc +++ b/tiledb/sm/filter/test/unit_run_filter_pipeline.cc @@ -293,7 +293,7 @@ TEST_CASE( SECTION("- Multi-stage") { // Add a few more +1 filters and re-run. - WriterTile::set_max_tile_chunk_size(80); + WhiteboxWriterTile::set_max_tile_chunk_size(80); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); pipeline.add_filter(Add1InPlace(Datatype::UINT64)); diff --git a/tiledb/sm/filter/test/unit_webp_pipeline.cc b/tiledb/sm/filter/test/unit_webp_pipeline.cc new file mode 100644 index 000000000000..27a74a7b9751 --- /dev/null +++ b/tiledb/sm/filter/test/unit_webp_pipeline.cc @@ -0,0 +1,110 @@ +/** + * @file unit_webp_pipeline.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This set of unit tests checks running the filter pipeline with the webp + * filter. + */ + +#include +#include "filter_test_support.h" +#include "test/support/src/mem_helpers.h" +#include "test/support/src/whitebox_helpers.h" +#include "tiledb/sm/filter/webp_filter.h" +#include "tiledb/sm/tile/tile.h" + +using namespace tiledb::sm; + +TEST_CASE("Filter: Round trip WebpFilter RGB data", "[filter][webp]") { + tiledb::sm::Config config; + auto tracker = tiledb::test::create_test_memory_tracker(); + + uint64_t height = 100; + uint64_t width = 100; + uint64_t row_stride = width * 3; + auto tile = make_shared( + HERE(), + constants::format_version, + Datatype::UINT8, + sizeof(uint8_t), + height * row_stride, + tracker); + + // Write full image in a single tile with chunking enabled. + std::vector data{0, 125, 255}; + std::vector expected_result(height * row_stride, 0); + for (uint64_t i = 0; i < height * width; i++) { + // Write three values for each RGB pixel. + for (uint64_t j = 0; j < 3; j++) { + CHECK_NOTHROW(tile->write(&data[j], i * 3 + j, sizeof(uint8_t))); + expected_result[i * 3 + j] = data[j]; + } + } + // For the write process 10 rows at a time using tile chunking. + // The row stride is 300 bytes, so the tile chunk size is 3000 bytes. + uint64_t extent_y = 10; + WhiteboxWriterTile::set_max_tile_chunk_size(extent_y * row_stride); + + FilterPipeline pipeline; + ThreadPool tp(4); + float quality = 100.0f; + bool lossless = true; + // NOTE: The extent_y parameter must respect chunk size or risk OOB access. + // This sets WebpFilter::extents_ which is passed to WebP API during encoding. + // If we set this to `height`, webp will reach past the end of chunked data. + pipeline.add_filter(WebpFilter( + quality, + WebpInputFormat::WEBP_RGB, + lossless, + extent_y, + width * 3, + Datatype::UINT8)); + bool use_chunking = true; + CHECK( + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp, use_chunking) + .ok()); + + // Check the original unfiltered data was removed. + CHECK(tile->size() == 0); + CHECK(tile->filtered_buffer().size() != 0); + + // Read the full image back with chunking disabled. + // NOTE: For the read case, WebP decoding APIs don't require height and width. + // Instead, WebP takes references to these values during unfiltering and sets + // them to the correct values after decoding is finished. + WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); + auto unfiltered_tile = + create_tile_for_unfiltering(height * row_stride, tile, tracker); + run_reverse(config, tp, unfiltered_tile, pipeline); + + for (uint64_t i = 0; i < height * row_stride; i++) { + uint8_t value; + CHECK_NOTHROW(unfiltered_tile.read(&value, i, sizeof(uint8_t))); + CHECK(value == expected_result[i]); + } +} diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index 961209f7ecb4..fe4ea38d9c10 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -381,7 +381,6 @@ void WebpFilter::set_extents(const std::vector& extents) { throw StatusException(Status_FilterError( "Tile extents too large; Max size WebP image is 16383x16383 pixels")); } - WriterTile::set_max_tile_chunk_size(extents_.first * extents_.second); } template void WebpFilter::set_extents( diff --git a/tiledb/sm/tile/tile.cc b/tiledb/sm/tile/tile.cc index fd0c1ec1c935..afec0f51b367 100644 --- a/tiledb/sm/tile/tile.cc +++ b/tiledb/sm/tile/tile.cc @@ -98,10 +98,6 @@ uint32_t WriterTile::compute_chunk_size( return static_cast(chunk_size64); } -void WriterTile::set_max_tile_chunk_size(uint64_t max_tile_chunk_size) { - max_tile_chunk_size_ = max_tile_chunk_size; -} - /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ diff --git a/tiledb/sm/tile/tile.h b/tiledb/sm/tile/tile.h index db057d92760a..343aaa3ce3ca 100644 --- a/tiledb/sm/tile/tile.h +++ b/tiledb/sm/tile/tile.h @@ -359,6 +359,11 @@ class Tile : public TileBase { * Tile object for write operations. */ class WriterTile : public TileBase { + /** + * Allow access to max_tile_chunk_size_ for testing. + */ + friend class WhiteboxWriterTile; + public: /** * returns a Tile initialized with parameters commonly used for @@ -380,13 +385,6 @@ class WriterTile : public TileBase { static uint32_t compute_chunk_size( const uint64_t tile_size, const uint64_t tile_cell_size); - /** - * Override max_tile_chunk_size_ used to process tile chunks in parallel. - * - * @param max_tile_chunk_size The maximum chunk size. - */ - static void set_max_tile_chunk_size(uint64_t max_tile_chunk_size); - /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ From fc1ff69129dd6ec0a1bfa434b68247af98d9c7eb Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 13 Jun 2024 13:12:27 +0300 Subject: [PATCH 431/456] Remove error statuses from inside `parallel_for`. (#5064) [SC-49251](https://app.shortcut.com/tiledb-inc/story/49251/remove-status-from-all-parallel-for-s) As preparation for removing `Status` from the thread pool and parallel functions, this PR removes returning statuses _in the bodies of functions passed to `parallel_for***`_. Specifically: * Calls to `RETURN_NOT_OK` were replaced by calls to `throw_if_not_ok`. * Returning statuses was replaced with passing them to `throw_if_not_ok` and returning `Status::Ok()`. * Calls to `RETURN_CANCEL_OR_ERROR` were replaced by calls to `throw_if_not_ok` and calls to `StrategyBase::throw_if_cancellation_requested()`, a new method that throws if the query has been cancelled. --- TYPE: NO_HISTORY --- test/src/unit-backwards_compat.cc | 4 ++-- tiledb/sm/array/array.cc | 6 ++--- tiledb/sm/array/array_directory.cc | 22 ++++++++++--------- tiledb/sm/filesystem/azure.cc | 2 +- tiledb/sm/filesystem/gcs.cc | 2 +- tiledb/sm/filesystem/vfs.cc | 8 +++---- tiledb/sm/filter/filter_pipeline.cc | 4 ++-- tiledb/sm/fragment/fragment_metadata.cc | 2 +- tiledb/sm/query/iquery_strategy.h | 8 +++++++ tiledb/sm/query/legacy/reader.cc | 13 ++++++----- tiledb/sm/query/query.h | 8 ------- tiledb/sm/query/readers/dense_reader.cc | 17 +++++++------- tiledb/sm/query/readers/reader_base.cc | 7 +++--- .../query/readers/sparse_index_reader_base.cc | 15 +++++++------ tiledb/sm/query/strategy_base.cc | 7 ++++++ tiledb/sm/query/strategy_base.h | 5 +++++ .../sm/query/writers/global_order_writer.cc | 4 ++-- tiledb/sm/query/writers/ordered_writer.cc | 10 ++++----- tiledb/sm/query/writers/unordered_writer.cc | 4 ++-- tiledb/sm/query/writers/writer_base.cc | 8 +++---- 20 files changed, 88 insertions(+), 68 deletions(-) diff --git a/test/src/unit-backwards_compat.cc b/test/src/unit-backwards_compat.cc index e72bd4a161ed..6839dd9d4184 100644 --- a/test/src/unit-backwards_compat.cc +++ b/test/src/unit-backwards_compat.cc @@ -150,8 +150,8 @@ TEST_CASE( std::string array_uri(arrays_dir + "/dense_array_v1_3_0"); REQUIRE_THROWS_WITH( Array(ctx, array_uri, TILEDB_READ), - "[TileDB::Array] Error: [TileDB::ArrayDirectory] Error: Reading data " - "past end of serialized data size."); + Catch::Matchers::EndsWith( + "Reading data past end of serialized data size.")); } template diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index d0921e92d9d3..59091561a59a 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -740,11 +740,11 @@ void Array::delete_fragments( auto vfs = &(resources.vfs()); throw_if_not_ok(parallel_for( &resources.compute_tp(), 0, fragment_uris.size(), [&](size_t i) { - RETURN_NOT_OK(vfs->remove_dir(fragment_uris[i].uri_)); + throw_if_not_ok(vfs->remove_dir(fragment_uris[i].uri_)); bool is_file = false; - RETURN_NOT_OK(vfs->is_file(commit_uris_to_delete[i], &is_file)); + throw_if_not_ok(vfs->is_file(commit_uris_to_delete[i], &is_file)); if (is_file) { - RETURN_NOT_OK(vfs->remove_file(commit_uris_to_delete[i])); + throw_if_not_ok(vfs->remove_file(commit_uris_to_delete[i])); } return Status::Ok(); })); diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index e031ed5f4a97..189bdf20dfd1 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -177,7 +177,10 @@ ArrayDirectory::load_all_array_schemas( array_schema->set_array_uri(uri_); schema_vector[schema_ith] = array_schema; } catch (std::exception& e) { - return Status_ArrayDirectoryError(e.what()); + // TODO: We could throw a nested exception, but converting exceptions + // to statuses loses the inner exception messages. We can revisit this + // when Status gets removed from this module. + throw ArrayDirectoryException(e.what()); } return Status::Ok(); @@ -316,13 +319,12 @@ void ArrayDirectory::delete_fragments_list( // Delete fragments and commits throw_if_not_ok(parallel_for( &resources_.get().compute_tp(), 0, uris.size(), [&](size_t i) { - RETURN_NOT_OK(resources_.get().vfs().remove_dir(uris[i])); + auto& vfs = resources_.get().vfs(); + throw_if_not_ok(vfs.remove_dir(uris[i])); bool is_file = false; - RETURN_NOT_OK( - resources_.get().vfs().is_file(commit_uris_to_delete[i], &is_file)); + throw_if_not_ok(vfs.is_file(commit_uris_to_delete[i], &is_file)); if (is_file) { - RETURN_NOT_OK( - resources_.get().vfs().remove_file(commit_uris_to_delete[i])); + throw_if_not_ok(vfs.remove_file(commit_uris_to_delete[i])); } return Status::Ok(); })); @@ -937,7 +939,7 @@ ArrayDirectory::compute_fragment_uris_v1_v11( if (stdx::string::starts_with(array_dir_uris[i].last_path_part(), ".")) return Status::Ok(); int32_t flag; - RETURN_NOT_OK(this->is_fragment( + throw_if_not_ok(this->is_fragment( array_dir_uris[i], ok_uris, consolidated_commit_uris_set_, &flag)); is_fragment[i] = (uint8_t)flag; return Status::Ok(); @@ -1076,11 +1078,11 @@ ArrayDirectory::compute_uris_to_vacuum( auto& tp = resources_.get().compute_tp(); auto status = parallel_for(&tp, 0, vac_files.size(), [&](size_t i) { uint64_t size = 0; - RETURN_NOT_OK(resources_.get().vfs().file_size(vac_files[i], &size)); + auto& vfs = resources_.get().vfs(); + throw_if_not_ok(vfs.file_size(vac_files[i], &size)); std::string names; names.resize(size); - RETURN_NOT_OK( - resources_.get().vfs().read(vac_files[i], 0, &names[0], size)); + throw_if_not_ok(vfs.read(vac_files[i], 0, &names[0], size)); std::stringstream ss(names); bool vacuum_vac_file = true; for (std::string uri_str; std::getline(ss, uri_str);) { diff --git a/tiledb/sm/filesystem/azure.cc b/tiledb/sm/filesystem/azure.cc index 7b45be982fae..2a4bc8b358d6 100644 --- a/tiledb/sm/filesystem/azure.cc +++ b/tiledb/sm/filesystem/azure.cc @@ -841,7 +841,7 @@ Status Azure::remove_dir(const URI& uri) const { std::vector paths; RETURN_NOT_OK(ls(uri, &paths, "")); auto status = parallel_for(thread_pool_, 0, paths.size(), [&](size_t i) { - RETURN_NOT_OK(remove_blob(URI(paths[i]))); + throw_if_not_ok(remove_blob(URI(paths[i]))); return Status::Ok(); }); RETURN_NOT_OK(status); diff --git a/tiledb/sm/filesystem/gcs.cc b/tiledb/sm/filesystem/gcs.cc index e5381ff998a8..c262baab238e 100644 --- a/tiledb/sm/filesystem/gcs.cc +++ b/tiledb/sm/filesystem/gcs.cc @@ -439,7 +439,7 @@ Status GCS::remove_dir(const URI& uri) const { std::vector paths; RETURN_NOT_OK(ls(uri, &paths, "")); auto status = parallel_for(thread_pool_, 0, paths.size(), [&](size_t i) { - RETURN_NOT_OK(remove_object(URI(paths[i]))); + throw_if_not_ok(remove_object(URI(paths[i]))); return Status::Ok(); }); RETURN_NOT_OK(status); diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 3b74fcd3a871..7b2357d1eea3 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -432,9 +432,9 @@ void VFS::remove_dirs( ThreadPool* compute_tp, const std::vector& uris) const { throw_if_not_ok(parallel_for(compute_tp, 0, uris.size(), [&](size_t i) { bool is_dir; - RETURN_NOT_OK(this->is_dir(uris[i], &is_dir)); + throw_if_not_ok(this->is_dir(uris[i], &is_dir)); if (is_dir) { - RETURN_NOT_OK(remove_dir(uris[i])); + throw_if_not_ok(remove_dir(uris[i])); } return Status::Ok(); })); @@ -486,7 +486,7 @@ Status VFS::remove_file(const URI& uri) const { void VFS::remove_files( ThreadPool* compute_tp, const std::vector& uris) const { throw_if_not_ok(parallel_for(compute_tp, 0, uris.size(), [&](size_t i) { - RETURN_NOT_OK(remove_file(uris[i])); + throw_if_not_ok(remove_file(uris[i])); return Status::Ok(); })); } @@ -494,7 +494,7 @@ void VFS::remove_files( void VFS::remove_files( ThreadPool* compute_tp, const std::vector& uris) const { throw_if_not_ok(parallel_for(compute_tp, 0, uris.size(), [&](size_t i) { - RETURN_NOT_OK(remove_file(uris[i].uri_)); + throw_if_not_ok(remove_file(uris[i].uri_)); return Status::Ok(); })); } diff --git a/tiledb/sm/filter/filter_pipeline.cc b/tiledb/sm/filter/filter_pipeline.cc index 401b31f5b8eb..d14d80e9c9a1 100644 --- a/tiledb/sm/filter/filter_pipeline.cc +++ b/tiledb/sm/filter/filter_pipeline.cc @@ -358,11 +358,11 @@ Status FilterPipeline::filter_chunks_forward( std::memcpy((char*)dest + dest_offset, &metadata_size, sizeof(uint32_t)); dest_offset += sizeof(uint32_t); // Write the chunk metadata - RETURN_NOT_OK( + throw_if_not_ok( final_stage_output_metadata.copy_to((char*)dest + dest_offset)); dest_offset += metadata_size; // Write the chunk data - RETURN_NOT_OK(final_stage_output_data.copy_to((char*)dest + dest_offset)); + throw_if_not_ok(final_stage_output_data.copy_to((char*)dest + dest_offset)); return Status::Ok(); }); diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 826b15993361..8afe38d9eb97 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -770,7 +770,7 @@ std::vector> FragmentMetadata::load( FragmentID fragment_id{sf.uri_}; if (fragment_id.array_format_version() <= 2) { bool sparse; - RETURN_NOT_OK(resources.vfs().is_file(coords_uri, &sparse)); + throw_if_not_ok(resources.vfs().is_file(coords_uri, &sparse)); metadata = make_shared( HERE(), &resources, diff --git a/tiledb/sm/query/iquery_strategy.h b/tiledb/sm/query/iquery_strategy.h index 059cfcdc95df..174c2bbb8b71 100644 --- a/tiledb/sm/query/iquery_strategy.h +++ b/tiledb/sm/query/iquery_strategy.h @@ -42,6 +42,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for query status exceptions. */ +class QueryException : public StatusException { + public: + explicit QueryException(const std::string& msg) + : StatusException("Query", msg) { + } +}; + class IQueryStrategy { public: /** Destructor. */ diff --git a/tiledb/sm/query/legacy/reader.cc b/tiledb/sm/query/legacy/reader.cc index f1ea3df97813..c7582146f20f 100644 --- a/tiledb/sm/query/legacy/reader.cc +++ b/tiledb/sm/query/legacy/reader.cc @@ -581,7 +581,7 @@ Status Reader::compute_range_result_coords( auto status = parallel_for(&resources_.compute_tp(), 0, range_num, [&](uint64_t r) { // Compute overlapping coordinates per range - RETURN_NOT_OK(compute_range_result_coords( + throw_if_not_ok(compute_range_result_coords( subarray, r, result_tile_map, @@ -591,12 +591,14 @@ Status Reader::compute_range_result_coords( // Dedup unless there is a single fragment or array schema allows // duplicates if (!single_fragment[r] && !allows_dups) { - RETURN_CANCEL_OR_ERROR(sort_result_coords( + throw_if_not_ok(sort_result_coords( range_result_coords[r].begin(), range_result_coords[r].end(), range_result_coords[r].size(), sort_layout)); - RETURN_CANCEL_OR_ERROR(dedup_result_coords(range_result_coords[r])); + this->throw_if_cancellation_requested(); + throw_if_not_ok(dedup_result_coords(range_result_coords[r])); + this->throw_if_cancellation_requested(); } return Status::Ok(); @@ -678,13 +680,14 @@ Status Reader::compute_range_result_coords( std::vector> range_result_coords_vec(fragment_num); auto status = parallel_for(&resources_.compute_tp(), 0, fragment_num, [&](uint32_t f) { - return compute_range_result_coords( + throw_if_not_ok(compute_range_result_coords( subarray, range_idx, f, result_tile_map, result_tiles, - range_result_coords_vec[f]); + range_result_coords_vec[f])); + return Status::Ok(); }); RETURN_NOT_OK(status); diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index 08dd6e022126..eb23873819c9 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -61,14 +61,6 @@ using namespace tiledb::common; namespace tiledb::sm { -/** Class for query status exceptions. */ -class QueryException : public StatusException { - public: - explicit QueryException(const std::string& msg) - : StatusException("Query", msg) { - } -}; - class Array; class ArrayDimensionLabelQueries; diff --git a/tiledb/sm/query/readers/dense_reader.cc b/tiledb/sm/query/readers/dense_reader.cc index 497af0c3ebbf..dae9a3570ef4 100644 --- a/tiledb/sm/query/readers/dense_reader.cc +++ b/tiledb/sm/query/readers/dense_reader.cc @@ -1141,7 +1141,7 @@ Status DenseReader::apply_query_condition( *(fragment_metadata_[frag_domains[i].fid()] ->array_schema() .get())); - RETURN_NOT_OK(condition_->apply_dense( + throw_if_not_ok(condition_->apply_dense( params, result_space_tile.result_tile(frag_domains[i].fid()), start, @@ -1172,7 +1172,7 @@ Status DenseReader::apply_query_condition( return Status::Ok(); }); - RETURN_NOT_OK(status); + throw_if_not_ok(status); // For `qc_coords_mode` just fill in the coordinates and skip // attribute @@ -1273,7 +1273,7 @@ Status DenseReader::copy_attribute( auto& result_space_tile = result_space_tiles.at(tc); // Copy the tile offsets. - return copy_offset_tiles( + throw_if_not_ok(copy_offset_tiles( name, tile_extents, result_space_tile, @@ -1285,7 +1285,8 @@ Status DenseReader::copy_attribute( range_info, qc_result, range_thread_idx, - num_range_threads); + num_range_threads)); + return Status::Ok(); }); RETURN_NOT_OK(status); } @@ -1332,7 +1333,7 @@ Status DenseReader::copy_attribute( const DimType* tc = (DimType*)&tile_coords[t][0]; auto& result_space_tile = result_space_tiles.at(tc); - return copy_var_tiles( + throw_if_not_ok(copy_var_tiles( name, tile_extents, result_space_tile, @@ -1345,7 +1346,7 @@ Status DenseReader::copy_attribute( t == iteration_tile_data->t_end() - 1, var_buffer_size, range_thread_idx, - num_range_threads); + num_range_threads)); return Status::Ok(); }); @@ -1375,7 +1376,7 @@ Status DenseReader::copy_attribute( auto& result_space_tile = result_space_tiles.at(tc); // Copy the tile fixed values. - RETURN_NOT_OK(copy_fixed_tiles( + throw_if_not_ok(copy_fixed_tiles( name, tile_extents, result_space_tile, @@ -1485,7 +1486,7 @@ Status DenseReader::process_aggregates( } } } else { - RETURN_NOT_OK(aggregate_tiles( + throw_if_not_ok(aggregate_tiles( name, tile_extents, result_space_tile, diff --git a/tiledb/sm/query/readers/reader_base.cc b/tiledb/sm/query/readers/reader_base.cc index 4059ec13b02e..580cc22c075f 100644 --- a/tiledb/sm/query/readers/reader_base.cc +++ b/tiledb/sm/query/readers/reader_base.cc @@ -817,7 +817,7 @@ Status ReaderBase::unfilter_tiles( tiles_chunk_data[i], tiles_chunk_var_data[i], tiles_chunk_validity_data[i]); - RETURN_NOT_OK(st); + throw_if_not_ok(st); unfiltered_tile_size[i] = tile_size.value(); unfiltered_tile_var_size[i] = tile_var_size.value(); unfiltered_tile_validity_size[i] = tile_validity_size.value(); @@ -836,7 +836,7 @@ Status ReaderBase::unfilter_tiles( 0, num_range_threads, [&](uint64_t i, uint64_t range_thread_idx) { - return unfilter_tile( + throw_if_not_ok(unfilter_tile( name, validity_only, result_tiles[i], @@ -846,7 +846,8 @@ Status ReaderBase::unfilter_tiles( num_range_threads, tiles_chunk_data[i], tiles_chunk_var_data[i], - tiles_chunk_validity_data[i]); + tiles_chunk_validity_data[i])); + return Status::Ok(); }); RETURN_CANCEL_OR_ERROR(status); diff --git a/tiledb/sm/query/readers/sparse_index_reader_base.cc b/tiledb/sm/query/readers/sparse_index_reader_base.cc index 0c6574992429..cf4526143108 100644 --- a/tiledb/sm/query/readers/sparse_index_reader_base.cc +++ b/tiledb/sm/query/readers/sparse_index_reader_base.cc @@ -678,7 +678,7 @@ void SparseIndexReaderBase::compute_tile_bitmaps( { auto timer_compute_results_count_sparse = stats_->start_timer("compute_results_count_sparse"); - RETURN_NOT_OK(rt->compute_results_count_sparse( + throw_if_not_ok(rt->compute_results_count_sparse( dim_idx, ranges_for_dim, relevant_ranges, @@ -738,7 +738,7 @@ void SparseIndexReaderBase::apply_query_condition( // Remove cells with partial overlap from the bitmap. QueryCondition::Params params( query_memory_tracker_, *(frag_meta->array_schema().get())); - RETURN_NOT_OK(partial_overlap_condition_.apply_sparse( + throw_if_not_ok(partial_overlap_condition_.apply_sparse( params, *rt, rt->bitmap())); rt->count_cells(); } @@ -755,8 +755,9 @@ void SparseIndexReaderBase::apply_query_condition( // Remove cells deleted cells using the open timestamp. QueryCondition::Params params( query_memory_tracker_, *(frag_meta->array_schema().get())); - RETURN_NOT_OK(delete_timestamps_condition_.apply_sparse( - params, *rt, rt->post_dedup_bitmap())); + throw_if_not_ok( + delete_timestamps_condition_.apply_sparse( + params, *rt, rt->post_dedup_bitmap())); if (array_schema_.allows_dups()) { rt->count_cells(); } @@ -766,7 +767,7 @@ void SparseIndexReaderBase::apply_query_condition( if (condition_.has_value()) { QueryCondition::Params params( query_memory_tracker_, *(frag_meta->array_schema().get())); - RETURN_NOT_OK(condition_->apply_sparse( + throw_if_not_ok(condition_->apply_sparse( params, *rt, rt->post_dedup_bitmap())); if (array_schema_.allows_dups()) { rt->count_cells(); @@ -801,12 +802,12 @@ void SparseIndexReaderBase::apply_query_condition( *(frag_meta->array_schema().get())); if (!frag_meta->has_timestamps() || delete_timestamp > frag_meta->timestamp_range().second) { - RETURN_NOT_OK( + throw_if_not_ok( delete_and_update_conditions_[i] .apply_sparse( params, *rt, rt->post_dedup_bitmap())); } else { - RETURN_NOT_OK( + throw_if_not_ok( timestamped_delete_and_update_conditions_[i] .apply_sparse( params, *rt, rt->post_dedup_bitmap())); diff --git a/tiledb/sm/query/strategy_base.cc b/tiledb/sm/query/strategy_base.cc index 878c62eb8ea1..1099c4eb8a79 100644 --- a/tiledb/sm/query/strategy_base.cc +++ b/tiledb/sm/query/strategy_base.cc @@ -36,6 +36,7 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/query/iquery_strategy.h" #include "tiledb/sm/query/query_buffer.h" namespace tiledb { @@ -101,6 +102,12 @@ void StrategyBase::get_dim_attr_stats() const { } } +void StrategyBase::throw_if_cancellation_requested() const { + if (storage_manager_->cancellation_in_progress()) { + throw QueryException("Query was cancelled"); + } +} + std::string StrategyBase::offsets_mode() const { return offsets_format_mode_; } diff --git a/tiledb/sm/query/strategy_base.h b/tiledb/sm/query/strategy_base.h index 1a62e36980be..2e029d0be119 100644 --- a/tiledb/sm/query/strategy_base.h +++ b/tiledb/sm/query/strategy_base.h @@ -338,6 +338,11 @@ class StrategyBase { * the query. */ void get_dim_attr_stats() const; + + /** + * Throws an exception if the query is canceled. + */ + void throw_if_cancellation_requested() const; }; } // namespace tiledb::sm diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index 3dd4a254e196..4a637e877949 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -872,8 +872,8 @@ Status GlobalOrderWriter::prepare_full_tiles( auto buff_it = buffers_.begin(); std::advance(buff_it, i); const auto& name = buff_it->first; - RETURN_CANCEL_OR_ERROR( - prepare_full_tiles(name, coord_dups, &tiles->at(name))); + throw_if_not_ok(prepare_full_tiles(name, coord_dups, &tiles->at(name))); + this->throw_if_cancellation_requested(); return Status::Ok(); }); diff --git a/tiledb/sm/query/writers/ordered_writer.cc b/tiledb/sm/query/writers/ordered_writer.cc index 2139bcce3af3..f778b43e6ef7 100644 --- a/tiledb/sm/query/writers/ordered_writer.cc +++ b/tiledb/sm/query/writers/ordered_writer.cc @@ -337,21 +337,21 @@ Status OrderedWriter::prepare_filter_and_write_tiles( &resources_.compute_tp(), 0, batch_size, [&](uint64_t i) { // Prepare and filter tiles auto& writer_tile = tile_batches[b][i]; - RETURN_NOT_OK( + throw_if_not_ok( dense_tiler->get_tile(frag_tile_id + i, name, writer_tile)); if (!var) { - RETURN_NOT_OK(filter_tile( + throw_if_not_ok(filter_tile( name, &writer_tile.fixed_tile(), nullptr, false, false)); } else { auto offset_tile = &writer_tile.offset_tile(); - RETURN_NOT_OK(filter_tile( + throw_if_not_ok(filter_tile( name, &writer_tile.var_tile(), offset_tile, false, false)); - RETURN_NOT_OK( + throw_if_not_ok( filter_tile(name, offset_tile, nullptr, true, false)); } if (nullable) { - RETURN_NOT_OK(filter_tile( + throw_if_not_ok(filter_tile( name, &writer_tile.validity_tile(), nullptr, false, true)); } return Status::Ok(); diff --git a/tiledb/sm/query/writers/unordered_writer.cc b/tiledb/sm/query/writers/unordered_writer.cc index b67b9b1b8a6e..b8cefabba39b 100644 --- a/tiledb/sm/query/writers/unordered_writer.cc +++ b/tiledb/sm/query/writers/unordered_writer.cc @@ -387,8 +387,8 @@ Status UnorderedWriter::prepare_tiles( parallel_for(&resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto tiles_it = tiles->begin(); std::advance(tiles_it, i); - RETURN_CANCEL_OR_ERROR( - prepare_tiles(tiles_it->first, &(tiles_it->second))); + throw_if_not_ok(prepare_tiles(tiles_it->first, &(tiles_it->second))); + this->throw_if_cancellation_requested(); return Status::Ok(); }); diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index be6a4ada95e0..5f5c65554b72 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -809,8 +809,8 @@ Status WriterBase::filter_tiles( parallel_for(&resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto tiles_it = tiles->begin(); std::advance(tiles_it, i); - RETURN_CANCEL_OR_ERROR( - filter_tiles(tiles_it->first, &tiles_it->second)); + throw_if_not_ok(filter_tiles(tiles_it->first, &tiles_it->second)); + this->throw_if_cancellation_requested(); return Status::Ok(); }); @@ -846,7 +846,7 @@ Status WriterBase::filter_tiles( parallel_for(&resources_.compute_tp(), 0, args.size(), [&](uint64_t i) { const auto& [tile, offset_tile, contains_offsets, is_nullable] = args[i]; - RETURN_NOT_OK(filter_tile( + throw_if_not_ok(filter_tile( name, tile, offset_tile, contains_offsets, is_nullable)); return Status::Ok(); }); @@ -857,7 +857,7 @@ Status WriterBase::filter_tiles( auto status = parallel_for( &resources_.compute_tp(), 0, tiles->size(), [&](uint64_t i) { auto& tile = (*tiles)[i]; - RETURN_NOT_OK( + throw_if_not_ok( filter_tile(name, &tile.offset_tile(), nullptr, true, false)); return Status::Ok(); }); From 081f5ec9ccb76367f943c15b4d7fc5f234d036d5 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 13 Jun 2024 03:48:43 -0700 Subject: [PATCH 432/456] Tests to exercise combinations of views for external sort. (#5051) This PR provides the file `tiledb/common/util/test/unit_view_combo.cc` that includes a set of tests of various combinations of the view classes that have been developed for external sort. In particular, we have tests to `chunk` a `chunk_view`, `var_length_view`, and `alt_var_length_view`, to `chunk` a `permutation_view` and permute a `chunk_view`, and to `chunk` a `zip_view` and `zip` a `chunk_view`. The tests effect these compositions by defining a range, taking a view of it (by calling, e.g., `chunk()`), and then calling another view adaptor. We do not intend to support composition by `operator|()` at this time. [sc-48753] --- TYPE: NO_HISTORY DESC: Tests to exercise combinations of views. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/common/util/permutation_view.h | 6 +- tiledb/common/util/test/CMakeLists.txt | 5 + tiledb/common/util/test/oberon.h | 375 +++++++++++++++++++ tiledb/common/util/test/unit_view_combo.cc | 349 +++++++++++++++++ tiledb/stdx/__ranges/test/unit_chunk_view.cc | 42 +++ 5 files changed, 774 insertions(+), 3 deletions(-) create mode 100644 tiledb/common/util/test/oberon.h create mode 100644 tiledb/common/util/test/unit_view_combo.cc diff --git a/tiledb/common/util/permutation_view.h b/tiledb/common/util/permutation_view.h index 44ef010486de..5828448829f4 100644 --- a/tiledb/common/util/permutation_view.h +++ b/tiledb/common/util/permutation_view.h @@ -143,13 +143,13 @@ class permutation_view : public std::ranges::view_interface< data_begin_, index_begin_, data_end_ - data_begin_}; } - /** Size of the permutation view */ + /** Size of the permutation view. */ size_t size() const { return data_end_ - data_begin_; } - /** Accessor */ - auto& operator[](size_t i) const { + /** Accessor. Note use of decltype(auto). */ + decltype(auto) operator[](size_t i) const { // More general? return *(data_begin_ + *(index_begin_ + i)); return data_begin_[index_begin_[i]]; } diff --git a/tiledb/common/util/test/CMakeLists.txt b/tiledb/common/util/test/CMakeLists.txt index 9b00d92d26a8..60b71e92dc8f 100644 --- a/tiledb/common/util/test/CMakeLists.txt +++ b/tiledb/common/util/test/CMakeLists.txt @@ -74,3 +74,8 @@ commence(unit_test block_view) ) conclude(unit_test) +commence(unit_test view_combo) +this_target_sources( + unit_view_combo.cc +) +conclude(unit_test) diff --git a/tiledb/common/util/test/oberon.h b/tiledb/common/util/test/oberon.h new file mode 100644 index 000000000000..c1053d584c07 --- /dev/null +++ b/tiledb/common/util/test/oberon.h @@ -0,0 +1,375 @@ +/** + * @file oberon.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Oberon's monologue from a Midsummer Night's Dream, Public Domain + * The verses of iambic pentameter are divided into 10 syllables, which don't + * seem quite like 10 syllables, but I'm a computer scientist, not a poet. + * + * I know a bank where the wild thyme blows, + * Where oxlips and the nodding violet grows, + * Quite over-canopied with luscious woodbine, + * With sweet musk-roses, and with eglantine: + * + * This file provides different representations of the text, for testing. + * Included are + * - contiguous strings, along with the partition vector delimiting the + * positions of the syllable breaks + * - vectors of strings, along with their expected self-sorted order + * - vectors of vectors of characters, along with their expected order + * + * In addition to "self-sorted" order for each verse, we also provide expected + * results for verses 2, 3, and 4 when sorted by the order by which verse 1 (the + * first verse) is sorted. Note that in those cases, verses 2, 3, and 4 will + * not be in self-sorted order. + * + * Note that although the syllable breaks are not at the same position for each + * of the verses, each verse has the same number of syllables. + */ + +#ifndef TILEDB_OBERON_H +#define TILEDB_OBERON_H + +#include +#include + +namespace oberon { +/* + * As contiguous strings, along with the partition vector delimiting the + * positions of the syllable breaks. + */ +std::string cs0 = "Iknowabankwherethewildthymeblows"; +std::vector ps0 = {0, 1, 5, 6, 10, 15, 18, 20, 22, 27, 32}; + +std::string cs1 = "Whereoxlipsandthenoddingvioletgrows"; +std::vector ps1 = {0, 5, 7, 11, 14, 17, 20, 24, 27, 30, 35}; + +std::string cs2 = "Quiteovercanopiedwithlusciouswoodbine"; +std::vector ps2 = {0, 5, 9, 12, 13, 17, 21, 24, 29, 33, 37}; + +std::string cs3 = "Withsweetmuskrosesandwitheglantine"; +std::vector ps3 = {0, 4, 9, 13, 15, 18, 21, 25, 27, 30, 34}; + +/* + * As vectors of strings ("vs"), along with their expected self-sorted order, + * and their order as sorted by vs0. + */ +std::vector vs0{ + "I", + "know", + "a", + "bank", + "where", + "the", + "wi", + "ld", + "thyme", + "blows", +}; + +std::vector vs0_expected{ + "I", + "a", + "bank", + "blows", + "know", + "ld", + "the", + "thyme", + "where", + "wi", +}; + +std::vector vs0_by_vs0_expected{ + "I", + "a", + "bank", + "blows", + "know", + "ld", + "the", + "thyme", + "where", + "wi", +}; + +std::vector vs1{ + "Where", + "ox", + "lips", + "and", + "the", + "nod", + "ding", + "vio", + "let", + "grows", +}; + +std::vector vs1_expected{ + "Where", + "and", + "ding", + "grows", + "let", + "lips", + "nod", + "ox", + "the", + "vio", +}; + +std::vector vs1_by_vs0_expected{ + "Where", + "lips", + "and", + "grows", + "ox", + "vio", + "nod", + "let", + "the", + "ding", +}; + +std::vector vs2{ + "Quite", + "over", + "can", + "o", + "pied", + "with", + "lus", + "cious", + "wood", + "bine", +}; + +std::vector vs2_expected{ + "Quite", + "bine", + "can", + "cious", + "lus", + "o", + "over", + "pied", + "with", + "wood", +}; + +std::vector vs2_by_vs0_expected{ + "Quite", + "can", + "o", + "bine", + "over", + "cious", + "with", + "pied", + "wood", + "lus", +}; + +std::vector vs3{ + "With", + "sweet", + "musk", + "ro", + "ses", + "and", + "with", + "eg", + "lan", + "tine", +}; + +std::vector vs3_expected{ + "With", + "and", + "eg", + "lan", + "musk", + "ro", + "ses", + "sweet", + "tine", + "with", +}; + +std::vector vs3_by_vs0_expected{ + "With", + "musk", + "ro", + "tine", + "sweet", + "eg", + "and", + "lan", + "ses", + "with", +}; + +/* + * As vectors of vectors of characters ("vvc"), along with their expected + * self-sorted order. + */ +std::vector> vvc0{ + {'I'}, + {'k', 'n', 'o', 'w'}, + {'a'}, + {'b', 'a', 'n', 'k'}, + {'w', 'h', 'e', 'r', 'e'}, + {'t', 'h', 'e'}, + {'w', 'i'}, + {'l', 'd'}, + {'t', 'h', 'y', 'm', 'e'}, + {'b', 'l', 'o', 'w', 's'}, +}; + +std::vector> vvc0_expected{ + {'I'}, + {'a'}, + {'b', 'a', 'n', 'k'}, + {'b', 'l', 'o', 'w', 's'}, + {'k', 'n', 'o', 'w'}, + {'l', 'd'}, + {'t', 'h', 'e'}, + {'t', 'h', 'y', 'm', 'e'}, + {'w', 'h', 'e', 'r', 'e'}, + {'w', 'i'}, +}; + +std::vector> vvc0_by_vvc0_expected{ + {'I'}, + {'a'}, + {'b', 'a', 'n', 'k'}, + {'b', 'l', 'o', 'w', 's'}, + {'k', 'n', 'o', 'w'}, + {'l', 'd'}, + {'t', 'h', 'e'}, + {'t', 'h', 'y', 'm', 'e'}, + {'w', 'h', 'e', 'r', 'e'}, + {'w', 'i'}, +}; + +std::vector> vvc1{ + {'W', 'h', 'e', 'r', 'e'}, + {'o', 'x'}, + {'l', 'i', 'p', 's'}, + {'a', 'n', 'd'}, + {'t', 'h', 'e'}, + {'n', 'o', 'd'}, + {'d', 'i', 'n', 'g'}, + {'v', 'i', 'o'}, + {'l', 'e', 't'}, + {'g', 'r', 'o', 'w', 's'}, +}; + +std::vector> vvc1_expected{ + {'W', 'h', 'e', 'r', 'e'}, + {'a', 'n', 'd'}, + {'d', 'i', 'n', 'g'}, + {'g', 'r', 'o', 'w', 's'}, + {'l', 'e', 't'}, + {'l', 'i', 'p', 's'}, + {'n', 'o', 'd'}, + {'o', 'x'}, + {'t', 'h', 'e'}, + {'v', 'i', 'o'}, +}; + +std::vector vvc1_by_vvc0_expected{ + {'W', 'h', 'e', 'r', 'e'}, + {'l', 'i', 'p', 's'}, + {'a', 'n', 'd'}, + {'g', 'r', 'o', 'w', 's'}, + {'o', 'x'}, + {'v', 'i', 'o'}, + {'n', 'o', 'd'}, + {'l', 'e', 't'}, + {'t', 'h', 'e'}, + {'d', 'i', 'n', 'g'}, +}; + +std::vector> vvc2{ + {'Q', 'u', 'i', 't', 'e'}, + {'o', 'v', 'e', 'r'}, + {'c', 'a', 'n'}, + {'o'}, + {'p', 'i', 'e', 'd'}, + {'w', 'i', 't', 'h'}, + {'l', 'u', 's'}, + {'c', 'i', 'o', 'u', 's'}, + {'w', 'o', 'o', 'd'}, + {'b', 'i', 'n', 'e'}, +}; +std::vector> vvc2_expected{ + {'Q', 'u', 'i', 't', 'e'}, + {'b', 'i', 'n', 'e'}, + {'c', 'a', 'n'}, + {'c', 'i', 'o', 'u', 's'}, + {'l', 'u', 's'}, + {'o'}, + {'o', 'v', 'e', 'r'}, + {'p', 'i', 'e', 'd'}, + {'w', 'i', 't', 'h'}, + {'w', 'o', 'o', 'd'}, +}; +std::vector> vvc3{ + {'W', 'i', 't', 'h'}, + {'s', 'w', 'e', 'e', 't'}, + {'m', 'u', 's', 'k'}, + {'r', 'o'}, + {'s', 'e', 's'}, + {'a', 'n', 'd'}, + {'w', 'i', 't', 'h'}, + {'e', 'g'}, + {'l', 'a', 'n'}, + {'t', 'i', 'n', 'e'}, +}; +std::vector> vvc3_expected{ + {'W', 'i', 't', 'h'}, + {'a', 'n', 'd'}, + {'e', 'g'}, + {'l', 'a', 'n'}, + {'m', 'u', 's', 'k'}, + {'r', 'o'}, + {'s', 'e', 's'}, + {'s', 'w', 'e', 'e', 't'}, + {'t', 'i', 'n', 'e'}, + {'w', 'i', 't', 'h'}, +}; + +std::vector perm0 = {0, 2, 3, 9, 1, 7, 5, 8, 4, 6}; +std::vector perm1 = {0, 3, 6, 9, 8, 2, 5, 1, 4, 7}; +std::vector perm2 = {0, 9, 2, 7, 6, 3, 1, 4, 5, 8}; +std::vector perm3 = {0, 5, 7, 8, 2, 3, 4, 1, 9, 6}; + +} // namespace oberon +#endif // diff --git a/tiledb/common/util/test/unit_view_combo.cc b/tiledb/common/util/test/unit_view_combo.cc new file mode 100644 index 000000000000..4af12a3cc2d2 --- /dev/null +++ b/tiledb/common/util/test/unit_view_combo.cc @@ -0,0 +1,349 @@ +/** + * @file unit_view_combo.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements tests to exercise combinations of different views, most + * notably, combinations of various views with `chunk_view` + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +// The null test, just to make sure none of the include files are broken +TEST_CASE("view combo: Null test", "[view_combo][null_test]") { + REQUIRE(true); +} + +TEST_CASE("view combo: chunk a chunk view", "[view_combo]") { + size_t num_elements = 32 * 1024; + size_t chunk_size = 128; + size_t chunk_chunk_size = 8; + size_t num_chunks = num_elements / chunk_size; + size_t num_chunk_chunks = num_elements / (chunk_size * chunk_chunk_size); + + // Make sure we don't have any constructive / destructive interference + REQUIRE(chunk_size != chunk_chunk_size); + REQUIRE(num_chunks != chunk_size); + REQUIRE(num_chunk_chunks != chunk_size); + REQUIRE(num_chunks != num_chunk_chunks); + + // Don't worry abount boundary cases for now + REQUIRE(num_elements % chunk_size == 0); + REQUIRE(num_elements % (chunk_chunk_size * chunk_size) == 0); + REQUIRE( + 4 * chunk_size * chunk_chunk_size < + num_elements); // At least four outer chunks + + std::vector base_17(num_elements); + std::vector base_m31(num_elements); + std::iota(begin(base_17), end(base_17), 17); + std::iota(begin(base_m31), end(base_m31), -31); + + REQUIRE(size(base_17) == num_elements); + REQUIRE(std::ranges::equal(base_17, base_m31) == false); + REQUIRE( + std::ranges::equal(base_17, std::vector(num_elements, 0)) == false); + + auto a = _cpo::chunk(base_17, (long)chunk_size); + + SECTION("Verify base chunk view") { + for (size_t i = 0; i < num_chunks; ++i) { + auto current_chunk = a[i]; + for (size_t j = 0; j < chunk_size; ++j) { + CHECK(current_chunk[j] == base_17[i * chunk_size + j]); + } + } + } + + SECTION("Verify chunked chunk view") { + auto b = _cpo::chunk(a, (long)chunk_chunk_size); + for (size_t i = 0; i < num_chunk_chunks; ++i) { + auto current_chunk = b[i]; + for (size_t j = 0; j < chunk_chunk_size; ++j) { + auto inner_chunk = current_chunk[j]; + for (size_t k = 0; k < chunk_size; ++k) { + CHECK( + inner_chunk[k] == + base_17[i * chunk_chunk_size * chunk_size + j * chunk_size + k]); + } + } + } + } +} + +std::vector qq = { + 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0}; +std::vector rr = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0}; +std::vector pp = {0, 2, 7, 10}; +std::vector nn = {0, 3, 6, 10}; +std::vector oo = {2, 0, 1}; +std::vector mm = {1, 0}; +std::vector ll = {0, 1}; + +struct subrange_equal { + template + bool operator()(R&& a, S&& b) { + if (size(a) != size(b)) { + return false; + } + for (size_t i = 0; i < size(a); ++i) { + if (a[i] != b[i]) { + return false; + } + } + return true; + } +}; + +TEST_CASE( + "view combo: chunk var_length_view and alt_var_length view", + "[view_combo]") { + auto check_chunk_view = [](auto& a) { + CHECK(a[0][0][0] == 21.0); + CHECK(a[0][0][1] == 20.0); + CHECK(a[0][1][0] == 19.0); + CHECK(a[0][1][1] == 18.0); + CHECK(a[0][1][2] == 17.0); + CHECK(a[0][1][3] == 16.0); + CHECK(a[0][1][4] == 15.0); + CHECK(a[1][0][0] == 14.0); + CHECK(a[1][0][1] == 13.0); + CHECK(a[1][0][2] == 12.0); + + CHECK(std::ranges::size(a) == 2); + CHECK(std::ranges::size(a[0]) == 2); + CHECK(std::ranges::size(a[1]) == 1); + CHECK(std::ranges::size(a[0][0]) == 2); + CHECK(std::ranges::size(a[0][1]) == 5); + CHECK(std::ranges::size(a[1][0]) == 3); + + CHECK(std::ranges::equal(a[0][0], std::vector{21.0, 20.0})); + CHECK(std::ranges::equal( + a[0][1], std::vector{19.0, 18.0, 17.0, 16.0, 15.0})); + CHECK(std::ranges::equal(a[1][0], std::vector{14.0, 13.0, 12.0})); + + CHECK(std::ranges::equal( + a[0], + std::vector>{ + {21.0, 20.0}, + {19.0, 18.0, 17.0, 16.0, 15.0}, + }, + subrange_equal{})); + + CHECK(std::ranges::equal( + a[1], + std::vector>{ + {14.0, 13.0, 12.0}, + }, + subrange_equal{})); + }; + + auto u = var_length_view(qq, pp); + auto a = stdx::ranges::chunk_view(u, 2); + check_chunk_view(a); + + auto v = alt_var_length_view(qq, pp); + auto b = stdx::ranges::chunk_view(v, 2); + check_chunk_view(b); +} + +TEST_CASE("view combo: chunk a permutation view", "[view_combo]") { + /* See unit_permutation_view.cc for validation of the permutation */ + auto w = var_length_view(qq, pp); + auto x = permutation_view(w, oo); + + auto a = stdx::ranges::chunk_view(x, 2); + CHECK(std::ranges::equal( + a[0], + std::vector>{ + {14.0, 13.0, 12.0}, + {21.0, 20.0}, + }, + subrange_equal{})); + + CHECK(std::ranges::equal( + a[1], + std::vector>{ + {19.0, 18.0, 17.0, 16.0, 15.0}, + }, + subrange_equal{})); +} + +TEST_CASE("view combo: permute a chunk view", "[view_combo]") { + auto w = alt_var_length_view(qq, pp); + auto x = stdx::ranges::chunk_view(w, 2); + auto a = permutation_view(x, ll); + + CHECK(std::ranges::equal( + a[0], + std::vector>{ + {21.0, 20.0}, + {19.0, 18.0, 17.0, 16.0, 15.0}, + }, + subrange_equal{})); + + CHECK(std::ranges::equal( + a[1], + std::vector>{ + {14.0, 13.0, 12.0}, + }, + subrange_equal{})); + + auto b = permutation_view(x, mm); + + CHECK(std::ranges::equal( + b[0], + std::vector>{ + {14.0, 13.0, 12.0}, + }, + subrange_equal{})); + + CHECK(std::ranges::equal( + b[1], + std::vector>{ + {21.0, 20.0}, + {19.0, 18.0, 17.0, 16.0, 15.0}, + }, + subrange_equal{})); +} + +// #define print_types + +#include +TEST_CASE("view combo: chunk a zip view", "[view_combo]") { + auto z = zip(qq, rr, rr); + + // Verify what zip_view returns + CHECK(std::get<0>(z[0]) == 21.0); + CHECK(std::get<0>(z[1]) == 20.0); + CHECK(std::get<0>(z[2]) == 19.0); + CHECK(std::get<0>(z[3]) == 18.0); + CHECK(std::get<1>(z[0]) == 1.0); + CHECK(std::get<1>(z[1]) == 2.0); + CHECK(std::get<1>(z[2]) == 3.0); + CHECK(std::get<1>(*(z.begin() + 3)) == 4.0); + + CHECK(std::ranges::equal( + z, + std::vector>{ + {21.0, 1.0, 1.0}, + {20.0, 2.0, 2.0}, + {19.0, 3.0, 3.0}, + {18.0, 4.0, 4.0}, + {17.0, 5.0, 5.0}, + {16.0, 6.0, 6.0}, + {15.0, 7.0, 7.0}, + {14.0, 8.0, 8.0}, + {13.0, 9.0, 9.0}, + {12.0, 10.0, 10.0}, + })); + + auto a = stdx::ranges::chunk_view(z, 2); + CHECK(std::ranges::size(a) == 5); + CHECK(std::ranges::size(a[0]) == 2); + + CHECK(std::get<0>(a[0][0]) == 21.0); + CHECK(std::get<0>(a[0][1]) == 20.0); + CHECK(std::get<1>(a[0][0]) == 1.0); + CHECK(std::get<1>(a[0][1]) == 2.0); + CHECK(std::get<0>(a[1][0]) == 19.0); + CHECK(std::get<0>(a[1][1]) == 18.0); + CHECK(std::get<1>(a[1][0]) == 3.0); + CHECK(std::get<1>(a[1][1]) == 4.0); + + auto&& [b, c] = a[0]; + auto&& [d, e] = a[1]; + (void)d; + (void)e; + + auto&& [f, g, n] = b[0]; + auto&& [h, i, o] = b[1]; + auto&& [j, k, p] = c[0]; + auto&& [l, m, q] = c[1]; + + CHECK(std::get<0>(b[0]) == 21.0); + CHECK(std::get<1>(b[0]) == 1.0); + CHECK(std::get<2>(b[0]) == 1.0); + CHECK(std::get<0>(b[1]) == 20.0); + CHECK(std::get<1>(b[1]) == 2.0); + CHECK(std::get<2>(b[1]) == 2.0); + CHECK(std::get<0>(b[1]) == 20.0); + CHECK(std::get<1>(b[1]) == 2.0); + CHECK(std::get<1>(b[1]) == 2.0); + CHECK(std::get<0>(c[0]) == 19.0); + CHECK(std::get<1>(c[0]) == 3.0); + CHECK(std::get<0>(c[1]) == 18.0); + CHECK(std::get<1>(c[1]) == 4.0); + CHECK(f == 21.0); + CHECK(h == 20.0); + CHECK(j == 19.0); + CHECK(g == 1.0); + CHECK(q == 4.0); + + CHECK(b[0] == std::tuple{21.0, 1.0, 1.0}); + CHECK(b[1] == std::tuple{20.0, 2.0, 2.0}); + CHECK(c[0] == std::tuple{19.0, 3.0, 3.0}); + CHECK(c[1] == std::tuple{18.0, 4.0, 4.0}); + CHECK(c[2] == std::tuple{17.0, 5.0, 5.0}); + + // a[0] is the same as the first two elements of z + CHECK(std::ranges::equal( + a[0], + std::vector>{ + {21.0, 1.0, 1.0}, + {20.0, 2.0, 2.0}, + })); + + CHECK(std::equal(b, b + 1, std::begin(z))); + CHECK(std::equal(c, c + 1, std::begin(z) + 2)); +} + +TEST_CASE("view combo: zip a chunk view", "[view_combo]") { + auto c = chunk(qq, 2); + auto d = chunk(rr, 3); + + auto z = zip(c, d, d); + CHECK(size(z) == 4); // 3, 3, 3, 1 + + CHECK(std::ranges::equal(std::get<0>(z[0]), std::vector{21.0, 20.0})); + CHECK(std::ranges::equal( + std::get<1>(z[0]), std::vector{1.0, 2.0, 3.0})); +} diff --git a/tiledb/stdx/__ranges/test/unit_chunk_view.cc b/tiledb/stdx/__ranges/test/unit_chunk_view.cc index 35afb0fe28fe..23cf7d82b3ae 100644 --- a/tiledb/stdx/__ranges/test/unit_chunk_view.cc +++ b/tiledb/stdx/__ranges/test/unit_chunk_view.cc @@ -30,6 +30,8 @@ * This file implements unit tests for the chunk_view class. */ +#include + #include #include "tiledb/stdx/__ranges/chunk_view.h" @@ -387,3 +389,43 @@ TEST_CASE("chunk_view: Iterators values", "[chunk_view]") { ++b; CHECK(b == a.end()); } + +TEST_CASE("chunk_view: Larger vector", "[chunk_view]") { + size_t num_elements = 8 * 1024; + size_t chunk_size = 128; + size_t num_chunks = num_elements / chunk_size; + + // Don't worry about boundary cases for now + REQUIRE(num_elements % num_chunks == 0); + + std::vector base_17(num_elements); + std::iota(begin(base_17), end(base_17), 0); + + REQUIRE(size(base_17) == num_elements); + + REQUIRE( + std::ranges::equal(base_17, std::vector(num_elements, 0)) == false); + + auto a = _cpo::chunk(base_17, (long)chunk_size); + + SECTION("Verify base chunk view") { + for (size_t i = 0; i < num_chunks; ++i) { + auto current_chunk = a[i]; + for (size_t j = 0; j < chunk_size; ++j) { + CHECK(current_chunk[j] == base_17[i * chunk_size + j]); + } + } + } + + SECTION("Verify base chunk view") { + for (size_t i = 0; i < num_chunks; ++i) { + auto current_chunk = a[i]; + for (size_t j = 0; j < chunk_size; ++j) { + CHECK(current_chunk[j] == base_17[i * chunk_size + j]); + current_chunk[j] = 0; + CHECK(current_chunk[j] == 0); + } + } + CHECK(std::ranges::equal(base_17, std::vector(num_elements, 0))); + } +} From 5577973dc58c5f0f81e78e0713f72794abade234 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Thu, 13 Jun 2024 09:08:12 -0700 Subject: [PATCH 433/456] Sort individual chunks in chunk_view (#5052) This PR comprises the test file `unit_sort_chunk` that tests that individual chunks of a `chunk_view` can be separately sorted. It tests a `std::vector` and a `zip_view` of `std::vector` and `alt_var_length_view`. [sc-49444] --- TYPE: IMPROVEMENT DESC: Tests that the chunks in a `chunk_view` can be separately sorted. --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/common/util/permute.h | 89 ++++++++++ tiledb/common/util/test/CMakeLists.txt | 6 + .../common/util/test/unit_permutation_sort.cc | 124 ++++++++++++++ tiledb/common/util/test/unit_sort_chunk.cc | 158 ++++++++++++++++++ tiledb/common/util/test/unit_sort_zip.cc | 2 +- 5 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 tiledb/common/util/permute.h create mode 100644 tiledb/common/util/test/unit_sort_chunk.cc diff --git a/tiledb/common/util/permute.h b/tiledb/common/util/permute.h new file mode 100644 index 000000000000..3ef151930d10 --- /dev/null +++ b/tiledb/common/util/permute.h @@ -0,0 +1,89 @@ +/** + * @file tiledb/common/util/permute.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#ifndef TILEDB_PERMUTE_H +#define TILEDB_PERMUTE_H + +#include +#include +#include +#include + +/** + * Permute an array in-place according to a given permutation array. + * @tparam Vector + * @tparam Perm + * @tparam Done Should generally be `std::vector` + * @param b Array to be sorted + * @param perm Array specifying the permutation + * @param done Array of flags indicating whether an element has been processed + * or not. All elements must be cleared (set to false) on entry. Note that it + * will be modified while the function is executing but on return, all the + * entries will be cleared. + */ +template < + std::ranges::random_access_range Vector, + std::ranges::random_access_range Perm, + std::ranges::random_access_range Done> +void permute(Vector& b, const Perm& perm, Done& done) { + using flag_type = std::remove_cvref_t; + constexpr flag_type zero_flag{0}; + constexpr flag_type one_flag{1}; + + // @todo Is there a better way to track done-ness? Hi bit of perm? + for (size_t i = 0; i < perm.size(); ++i) { + if (zero_flag != done[i]) { + done[i] = zero_flag; + continue; + } + + size_t ix = i; + size_t px = perm[ix]; + while (!done[px]) { + // @todo Find with ADL??? + std::swap(b[ix], b[px]); + + done[ix] = one_flag; + ix = px; + px = perm[ix]; + } + } +} + +template < + std::ranges::random_access_range Vector, + std::ranges::random_access_range Perm> +void permute(Vector& b, const Perm& perm) { + std::vector done(perm.size(), false); + permute(b, perm, done); +} + +#endif diff --git a/tiledb/common/util/test/CMakeLists.txt b/tiledb/common/util/test/CMakeLists.txt index 60b71e92dc8f..0d6787bbf13a 100644 --- a/tiledb/common/util/test/CMakeLists.txt +++ b/tiledb/common/util/test/CMakeLists.txt @@ -50,6 +50,12 @@ commence(unit_test proxy_sort) ) conclude(unit_test) +commence(unit_test sort_chunk) + this_target_sources( + unit_sort_chunk.cc +) +conclude(unit_test) + commence(unit_test sort_zip) this_target_sources( unit_sort_zip.cc diff --git a/tiledb/common/util/test/unit_permutation_sort.cc b/tiledb/common/util/test/unit_permutation_sort.cc index 23bf78abf15c..a4d2c87169ba 100644 --- a/tiledb/common/util/test/unit_permutation_sort.cc +++ b/tiledb/common/util/test/unit_permutation_sort.cc @@ -38,10 +38,13 @@ #include #include "tiledb/common/util/alt_var_length_view.h" #include "tiledb/common/util/permutation_view.h" + #include "tiledb/common/util/proxy_sort.h" #include "tiledb/common/util/var_length_view.h" #include "tiledb/stdx/__ranges/zip_view.h" +#include "tiledb/common/util/permute.h" + TEST_CASE("permutation_sort: Null test", "[permutation_sort][null_test]") { REQUIRE(true); } @@ -339,3 +342,124 @@ TEST_CASE( } } } + +/** + * This function is set up to time various ways of sorting a set of arrays + * according to how one of them is ordered. It compares the time of sorting + * with a proxy sort and in-place permutation vs in-place sort. + * + * Doing in-place sort with the zip view is the most efficient (especially when + * parallelized. + * + * To do the timings, access a version of timer.h (e.g., from the experimental + * dag hierarchy and uncomment the timer related lines. To do the timing of + * parallel zip sort you will also need to use a parallel sort algorithm. You + * may also want to increase the size of N to 20'000'000. + */ +TEST_CASE("permutation_sort: time", "[permutation_sort]") { + uint32_t seed = Catch::rngSeed(); + size_t N = 2'000'000; + std::mt19937 g(seed); + + std::vector perm(N); + std::iota(begin(perm), end(perm), 0); + + auto init_19 = std::vector(N); + std::iota(begin(init_19), end(init_19), 19); + std::vector shuffled(init_19); + + std::shuffle(shuffled.begin(), shuffled.end(), g); + std::vector sorted0(shuffled); + std::vector sorted1(shuffled); + std::vector sorted2(shuffled); + std::vector sorted3(shuffled); + std::vector sorted4(shuffled); + std::vector sorted5(shuffled); + std::vector sorted6(shuffled); + std::vector sorted7(shuffled); + std::vector sorted8(shuffled); + std::vector sorted9(shuffled); + + CHECK(!std::ranges::equal(shuffled, init_19)); + + // Different numbers of vectors have different performance characteristics + // auto z = zip(shuffled, sorted0, sorted1, sorted2); + // auto z = zip(shuffled, sorted0, sorted1, sorted2, sorted3, sorted4); + // auto z = zip(shuffled, sorted0, sorted1, sorted2, sorted3, sorted4, + // sorted5, sorted6); + auto z = + zip(shuffled, + sorted0, + sorted1, + sorted2, + sorted3, + sorted4, + sorted5, + sorted6, + sorted7, + sorted8, + sorted9); + + std::vector done(N, 0); + + SECTION("separate sort") { + std::ranges::sort(shuffled); + std::ranges::sort(sorted0); + std::ranges::sort(sorted1); + std::ranges::sort(sorted2); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + SECTION("proxy sort permute") { + proxy_sort(shuffled, perm); + + permute(shuffled, perm); + permute(sorted0, perm); + permute(sorted1, perm); + permute(sorted2, perm); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + SECTION("proxy sort permute zip") { + proxy_sort(shuffled, perm); + + permute(z, perm, done); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + + SECTION("zip sort") { + std::sort(z.begin(), z.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } + +// Enable if pmergesort is available +#if 0 + SECTION("zip par sort") { + + pmergesort(z.begin(), z.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + + CHECK(std::ranges::equal(shuffled, init_19)); + CHECK(std::ranges::equal(sorted0, init_19)); + CHECK(std::ranges::equal(sorted1, init_19)); + CHECK(std::ranges::equal(sorted2, init_19)); + } +#endif +} diff --git a/tiledb/common/util/test/unit_sort_chunk.cc b/tiledb/common/util/test/unit_sort_chunk.cc new file mode 100644 index 000000000000..309ba2562e91 --- /dev/null +++ b/tiledb/common/util/test/unit_sort_chunk.cc @@ -0,0 +1,158 @@ +/** + * @file tiledb/common/util/test/unit_sort_chunk.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements tests for sorting chunks in a chunk_view over a vector. + */ + +#include +#include +#include +#include +#include +#include "oberon.h" +#include "tiledb/common/util/alt_var_length_view.h" +#include "tiledb/stdx/ranges" + +#include "tiledb/common/util/print_types.h" + +TEST_CASE("sort_chunk: Null test", "[sort_chunk][null_test]") { +} + +/* + * Test that sorting each chunk in a chunk view will sort blocks of elements in + * the underlying vector. + */ +TEST_CASE("sort_chunk: std::vector", "[sort_chunk]") { + size_t N = 1024; + size_t chunk_size = 16; + size_t num_chunks = N / chunk_size; + + std::vector v(N); + std::iota(v.begin(), v.end(), 17); + std::ranges::reverse(v); + std::vector expected(v); + + /* Sort blocks of underlying vector */ + for (size_t i = 0; i < num_chunks; ++i) { + for (size_t j = 0; j < chunk_size; ++j) { + std::sort( + expected.begin() + i * chunk_size, + expected.begin() + (i + 1) * chunk_size); + } + } + + /* Create chunk view of v */ + auto cv = stdx::views::chunk(v, (long)chunk_size); + CHECK(std::ranges::size(cv) == (long)num_chunks); + + /* Sort each chunk */ + for (auto&& cu : cv) { + std::sort(cu.begin(), cu.end()); + } + + CHECK(std::ranges::equal(v, expected)); +} + +TEST_CASE("sort_chunk: zip_view of std::vector", "sort_chunk") { + size_t N = 1024; + size_t chunk_size = 16; + size_t num_chunks = N / chunk_size; + + std::vector v(N), w(N + 5); + std::iota(v.begin(), v.end(), 17); + std::iota(w.begin(), w.end(), -13); + + auto z = stdx::views::zip(v, w); + + std::ranges::reverse(v); + std::ranges::reverse(w); + std::vector xv(v), xw(w); + + /* Sort blocks of underlying vectors */ + for (size_t i = 0; i < num_chunks; ++i) { + for (size_t j = 0; j < chunk_size; ++j) { + std::sort(xv.begin() + i * chunk_size, xv.begin() + (i + 1) * chunk_size); + std::sort(xw.begin() + i * chunk_size, xw.begin() + (i + 1) * chunk_size); + } + } + + /* Create chunk view of v */ + auto cz = stdx::ranges::chunk_view(z, chunk_size); + CHECK(std::ranges::size(cz) == (long)num_chunks); + + /* Sort each chunk */ + for (auto&& cy : cz) { + std::sort(cy.begin(), cy.end()); + } + + CHECK(std::ranges::equal(z, stdx::views::zip(xv, xw))); + CHECK(!std::ranges::is_sorted(z)); + CHECK(!std::ranges::is_sorted(xv)); + CHECK(!std::ranges::is_sorted(xw)); +} + +TEST_CASE("sort_chunk: zip_view of vector and alt_var_length_view") { + auto ob1 = alt_var_length_view(oberon::cs1, oberon::ps1); + REQUIRE(size(ob1) == 10); + std::vector v(10); + std::iota(v.begin(), v.end(), 17); + std::ranges::reverse(v); + + auto z = stdx::views::zip(v, ob1); + auto cz = stdx::views::chunk(z, 2); + CHECK(size(cz) == 5); + CHECK(size(cz[0]) == 2); + + for (auto&& cy : cz) { + std::sort(cy.begin(), cy.end(), [](auto&& a, auto&& b) { + return std::get<0>(a) < std::get<0>(b); + }); + } + + // 26, 25, 24, 23, 22, 21, 20, 19, 18, 17 + // -> 25, 26, 23, 24, 21, 22, 19, 20, 17, 18 + // Where ox lips and the nod ding vio let grows + // -> ox Where and lips nod the vio ding grows let + CHECK(std::get<0>(cz[0][0]) == 25); + CHECK(std::get<0>(cz[0][1]) == 26); + CHECK(std::ranges::equal(get<1>(cz[0][0]), std::vector{'o', 'x'})); + CHECK(std::ranges::equal( + get<1>(cz[0][1]), std::vector{'W', 'h', 'e', 'r', 'e'})); + CHECK(std::ranges::equal(get<1>(cz[1][0]), std::vector{'a', 'n', 'd'})); + CHECK(std::ranges::equal( + get<1>(cz[1][1]), std::vector{'l', 'i', 'p', 's'})); + CHECK(std::ranges::equal(get<1>(cz[2][0]), std::vector{'n', 'o', 'd'})); + CHECK(std::ranges::equal(get<1>(cz[2][1]), std::vector{'t', 'h', 'e'})); + CHECK(std::ranges::equal(get<1>(cz[3][0]), std::vector{'v', 'i', 'o'})); + CHECK(std::ranges::equal( + get<1>(cz[3][1]), std::vector{'d', 'i', 'n', 'g'})); + CHECK(std::ranges::equal( + get<1>(cz[4][0]), std::vector{'g', 'r', 'o', 'w', 's'})); + CHECK(std::ranges::equal(get<1>(cz[4][1]), std::vector{'l', 'e', 't'})); +} diff --git a/tiledb/common/util/test/unit_sort_zip.cc b/tiledb/common/util/test/unit_sort_zip.cc index 2f1c829be7ab..6ef60e671c82 100644 --- a/tiledb/common/util/test/unit_sort_zip.cc +++ b/tiledb/common/util/test/unit_sort_zip.cc @@ -1,5 +1,5 @@ /** - * @file unit_sort_zip.cc + * @file tiledb/common/util/test/unit_sort_zip.cc * * @section LICENSE * From b0650721feec979468e3c3ebc929f8f02d3c2f28 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 13 Jun 2024 19:31:55 +0300 Subject: [PATCH 434/456] Revert "Add temporary mitigations for CI failures in `windows-2022`." (#5082) [SC-49300](https://app.shortcut.com/tiledb-inc/story/49300/revert-temporary-changes-to-fix-windows-2022-failures) Reverts TileDB-Inc/TileDB#5057 Deployment of a new `windows-2022` GitHub Actions runner image has finished. This PR reverts the temporary mitigations added in #5057, to see if the failures will be fixed. --- TYPE: NO_HISTORY --- .github/workflows/build-windows.yml | 1 - .github/workflows/unit-test-runs.yml | 8 +------- ports/triplets/x64-windows-asan.cmake | 4 ---- ports/triplets/x64-windows.cmake | 4 ---- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index e35f76a9f6b5..aebad0e24cc1 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -61,7 +61,6 @@ jobs: TILEDB_WEBP: ${{ matrix.TILEDB_WEBP }} TILEDB_CMAKE_BUILD_TYPE: 'Release' VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' - CXXFLAGS: '/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR' # https://github.com/actions/runner-images/issues/10004 steps: - name: 'tiledb env prep' run: | diff --git a/.github/workflows/unit-test-runs.yml b/.github/workflows/unit-test-runs.yml index baa5fb1762b0..0b1bfce01f39 100644 --- a/.github/workflows/unit-test-runs.yml +++ b/.github/workflows/unit-test-runs.yml @@ -10,8 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # Temporarily reverting to windows-2019 until https://github.com/actions/runner-images/issues/10004 gets fixed. - os: [macos-latest, ubuntu-latest, windows-2019] + os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false name: Build - ${{ matrix.os }} timeout-minutes: 90 @@ -54,11 +53,6 @@ jobs: shell: bash if: ${{ !startsWith(matrix.os, 'windows-') }} - - name: 'Print env' - run: set - shell: cmd - if: ${{ startsWith(matrix.os, 'windows-') }} - - name: 'Build standalone unit tests' run: | cmake -S . \ diff --git a/ports/triplets/x64-windows-asan.cmake b/ports/triplets/x64-windows-asan.cmake index 2d1e356817c9..d19486d2cfed 100644 --- a/ports/triplets/x64-windows-asan.cmake +++ b/ports/triplets/x64-windows-asan.cmake @@ -7,7 +7,3 @@ set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) # bigobj is needed for capnp. set(VCPKG_C_FLAGS "/fsanitize=address /bigobj") set(VCPKG_CXX_FLAGS "/fsanitize=address /bigobj") - -# https://github.com/actions/runner-images/issues/10004 -set(VCPKG_C_FLAGS "${VCPKG_C_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") -set(VCPKG_CXX_FLAGS "${VCPKG_CXX_FLAGS} /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") diff --git a/ports/triplets/x64-windows.cmake b/ports/triplets/x64-windows.cmake index eb54e7f5a55f..c87b4f5dd385 100644 --- a/ports/triplets/x64-windows.cmake +++ b/ports/triplets/x64-windows.cmake @@ -2,8 +2,4 @@ set(VCPKG_TARGET_ARCHITECTURE x64) set(VCPKG_CRT_LINKAGE dynamic) set(VCPKG_LIBRARY_LINKAGE static) -# https://github.com/actions/runner-images/issues/10004 -set(VCPKG_C_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") -set(VCPKG_CXX_FLAGS "/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR") - set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) From 74564af7f5e8f51cf4a9174ac728066a91ac051d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 14 Jun 2024 15:39:05 +0300 Subject: [PATCH 435/456] Add the directory to the tiledb library on PATH in `tiledb_regression`. (#5088) After https://github.com/TileDB-Inc/TileDB/pull/5054, `tiledb_regression` started to run in the nightlies. `tiledb_regression` directly links to the `tiledb` target, which on Windows when building a shared library, is not found by default. This PR updates the `tiledb_regression` CTest test to append the path to `tiledb.dll` to PATH so that we don't have issues running the nightlies anymore. [SC-48054] --- TYPE: NO_HISTORY --- test/regression/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/regression/CMakeLists.txt b/test/regression/CMakeLists.txt index 87d6377ed5b6..99bf3f358af6 100644 --- a/test/regression/CMakeLists.txt +++ b/test/regression/CMakeLists.txt @@ -86,3 +86,8 @@ add_test( COMMAND $ --durations=yes WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + +if (BUILD_SHARED_LIBS) + # Add the location of the tiledb shared library to PATH, to help Windows find it. + set_property(TEST tiledb_regression PROPERTY ENVIRONMENT_MODIFICATION "PATH=path_list_append:$") +endif() From 93220d7c3041fcd05fdbf8a45cc9969f47aaf763 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Fri, 14 Jun 2024 05:52:30 -0700 Subject: [PATCH 436/456] Implement actualize for alt_var_length_view (#5087) This PR implements a function `actualize` that * orders the underlying data of an `alt_var_length_view` according to the permuted subranges * sets the subranges in the `alt_var_length_view` to delimit the new var-length element * stores the size of each subrange in an index argument After calling `actualize`, the `alt_var_length_view` will "look" exactly the same, i.e., iterating through it before and after will give exactly the same results (even though the underlying data range has been reordered). Actually ordered (instead of virtually ordered) var-length data, along with an array of the corresponding sizes, will be needed for saving phase I data to a temporary array. [sc-49496] --- TYPE: FEATURE DESC: Implement actualize function that orders data underlying `alt_var_length_view` --------- Co-authored-by: Luc Rancourt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> --- tiledb/common/util/alt_var_length_view.h | 35 ++++++++++++++ .../util/test/unit_alt_var_length_view.cc | 48 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/tiledb/common/util/alt_var_length_view.h b/tiledb/common/util/alt_var_length_view.h index b80a80f957ff..c61e7378902c 100644 --- a/tiledb/common/util/alt_var_length_view.h +++ b/tiledb/common/util/alt_var_length_view.h @@ -351,4 +351,39 @@ alt_var_length_view(R, R, J, I, I, K) template alt_var_length_view(R, R, J, I, I, K, L) -> alt_var_length_view, std::ranges::subrange>; + +/** + * Actually reorder the data underlying an alt_var_length_view (un-virtualize + * the permutation). Upon return, data will have the sorted data, offsets will + * have the SIZES of each subrange, and the subranges will be "in order" + * @tparam S Type of the subranges + * @tparam R Type of the data range + * @tparam I Type of the index / offset / size range + * @tparam B Type of the scratch buffer + * @param subranges The alt_var_length_view + * @param data The data range underlying the alt_var_length_view + * @param offsets The offsets range to be written to + * @param buffer Scratch space used for reordering, must have enough space to + * hold all of the data + * @return + * + * @todo Make this a member of alt_var_length_view + */ +template < + std::ranges::forward_range S, + std::ranges::forward_range R, + std::ranges::forward_range I, + std::ranges::random_access_range B> +auto actualize(S& subranges, R& data, I& offsets, B& buffer) { + auto x = buffer.begin(); + auto o = offsets.begin(); + for (auto& s : subranges) { + std::ranges::copy(s.begin(), s.end(), x); + auto n = s.size(); + s = std::ranges::subrange(x, x + n); + x += n; + *o++ = n; // x - buffer.begin(); + } + std::ranges::copy(buffer.begin(), buffer.begin() + data.size(), data.begin()); +} #endif // TILEDB_ALT_VAR_LENGTH_VIEW_H diff --git a/tiledb/common/util/test/unit_alt_var_length_view.cc b/tiledb/common/util/test/unit_alt_var_length_view.cc index 50164071e2e1..b76e6a0e99d1 100644 --- a/tiledb/common/util/test/unit_alt_var_length_view.cc +++ b/tiledb/common/util/test/unit_alt_var_length_view.cc @@ -603,3 +603,51 @@ TEST_CASE("alt_var_length_view: Sort", "[alt_var_length_view]") { } } } + +TEST_CASE("alt_var_length_view: Sort and actualize", "[alt_var_length_view]") { + std::vector r = { + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0}; + std::vector s(r); + std::vector o = {0, 3, 6, 10, 12}; + alt_var_length_view, std::vector> v(r, o); + + SECTION("Sort by size, ascending") { + std::vector> expected = { + {11.0, 12.0}, + {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0}, + {7.0, 8.0, 9.0, 10.0}, + }; + + std::sort(v.begin(), v.end(), [](auto& a, auto& b) { + return a.size() < b.size(); + }); + + CHECK(v.begin()->size() == 2); + CHECK((v.begin() + 1)->size() == 3); + CHECK((v.begin() + 2)->size() == 3); + CHECK((v.begin() + 3)->size() == 4); + + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + + // Check the underlying data has not changed even though sorted + CHECK(std::ranges::equal(r, s)); + std::vector scratch(size(r)); + + actualize(v, r, o, scratch); + + // Check the underlying data has changed to the expected sorted order + CHECK(std::ranges::equal( + r, + std::vector{ + 11.0, 12.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0})); + + // The alt_var_length_view should still "look" the same + CHECK(std::ranges::equal(o, std::vector{2, 3, 3, 4, 12})); + for (auto&& i : v) { + CHECK(std::ranges::equal(i, expected[&i - &*v.begin()])); + } + } +} From d32d9aff9937e90b430eae0bd6ad5c6224ec6f4e Mon Sep 17 00:00:00 2001 From: Dimitris Staratzis <33267511+DimitrisStaratzis@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:13:01 +0300 Subject: [PATCH 437/456] Fix exceptions with message: ```unknown exception type; no further information```. (#5080) This PR migrates from ```throw Status_*Error()``` to ```throw *Exception()```. This was causing some exceptions to be silenced because the previous implementation was not handling them correctly resulting in the display of the generic error message: ```unknown exception type; no further information``` To achieve this migration I had to create the following Exception classes: ```FragmentInfoException```, ```QueryConditionException```, ```ArraySchemaSerializationDisabledException```, ```ArraySchemaSerializationException```, ```ConsolidationSerializationDisabledException```, ```ConsolidationSerializationException```, ```EnumerationSerializationDisabledException```, ```EnumerationSerializationException```, ```QueryPlanSerializationDisabledException```, ```QueryPlanSerializationException``` [sc-48757] --- TYPE: BUG DESC: Fix exceptions with message: ```unknown exception type; no further information```. --- tiledb/sm/array/array.cc | 8 ++-- tiledb/sm/fragment/fragment_info.cc | 11 ++++- tiledb/sm/query/ast/query_ast.cc | 5 ++- tiledb/sm/query/query_condition.h | 7 +++ tiledb/sm/rest/rest_client.cc | 13 +++--- tiledb/sm/serialization/array_schema.cc | 57 ++++++++++++++---------- tiledb/sm/serialization/consolidation.cc | 52 ++++++++++++--------- tiledb/sm/serialization/enumeration.cc | 52 ++++++++++++--------- tiledb/sm/serialization/query_plan.cc | 52 ++++++++++++--------- 9 files changed, 160 insertions(+), 97 deletions(-) diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 59091561a59a..45cbe3a41bca 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -386,7 +386,7 @@ Status Array::open_without_fragments( if (remote_) { auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw Status_ArrayError( + throw ArrayException( "Cannot open array; remote array with no REST client."); } /* #TODO Change get_array_schema_from_rest function signature to @@ -675,7 +675,7 @@ Status Array::close() { set_metadata_loaded(true); auto rest_client = resources_.rest_client(); if (rest_client == nullptr) { - throw Status_ArrayError( + throw ArrayException( "Error closing array; remote array with no REST client."); } throw_if_not_ok(rest_client->post_array_metadata_to_rest( @@ -699,7 +699,7 @@ Status Array::close() { opened_array_.reset(); } catch (std::exception& e) { is_opening_or_closing_ = false; - throw Status_ArrayError(e.what()); + throw ArrayException(e.what()); } is_opening_or_closing_ = false; @@ -1159,7 +1159,7 @@ Status Array::reopen(uint64_t timestamp_start, uint64_t timestamp_end) { set_array_closed(); } catch (std::exception& e) { is_opening_or_closing_ = false; - throw Status_ArrayError(e.what()); + throw ArrayException(e.what()); } is_opening_or_closing_ = false; diff --git a/tiledb/sm/fragment/fragment_info.cc b/tiledb/sm/fragment/fragment_info.cc index 17ba2a5e5f7e..6cfee3597b47 100644 --- a/tiledb/sm/fragment/fragment_info.cc +++ b/tiledb/sm/fragment/fragment_info.cc @@ -48,6 +48,13 @@ namespace tiledb::sm { +class FragmentInfoException : public StatusException { + public: + explicit FragmentInfoException(const std::string& message) + : StatusException("FragmentInfo", message) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -182,7 +189,7 @@ Status FragmentInfo::get_total_cell_num(uint64_t* cell_num) const { const std::string& FragmentInfo::fragment_name(uint32_t fid) const { ensure_loaded(); if (fid >= fragment_num()) - throw Status_FragmentInfoError( + throw FragmentInfoException( "Cannot get fragment URI; Invalid fragment index"); return single_fragment_info_vec_[fid].name(); @@ -936,7 +943,7 @@ Status FragmentInfo::load(const ArrayDirectory& array_dir) { void FragmentInfo::ensure_loaded() const { if (!loaded_) { - throw Status_FragmentInfoError("Fragment info has not been loaded."); + throw FragmentInfoException("Fragment info has not been loaded."); } } diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index 16dfad7c9550..d49f68dd9595 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -33,6 +33,7 @@ #include "query_ast.h" #include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/misc/integral_type_casts.h" +#include "tiledb/sm/query/query_condition.h" using namespace tiledb::common; @@ -317,8 +318,8 @@ Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { (op_ == QueryConditionOp::IN || op_ == QueryConditionOp::NOT_IN)) { for (auto& member : members_) { if (member.size() != cell_size) { - throw Status_QueryConditionError( - "Value node set memmber size mismatch: " + + throw QueryConditionException( + "Value node set member size mismatch: " + std::to_string(cell_size) + " != " + std::to_string(member.size())); } } diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 8ef3df48b018..6738063922b2 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -47,6 +47,13 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +class QueryConditionException : public StatusException { + public: + explicit QueryConditionException(const std::string& message) + : StatusException("QueryCondition", message) { + } +}; + class FragmentMetadata; class MemoryTracker; struct ResultCellSlab; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index c291323f39ea..8c7b8f177489 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -318,7 +318,7 @@ shared_ptr RestClient::post_array_schema_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting array schema from REST; server returned no data."); } @@ -676,7 +676,7 @@ RestClient::post_enumerations_from_rest( const std::vector& enumeration_names, shared_ptr memory_tracker) { if (array == nullptr) { - throw Status_RestError( + throw RestClientException( "Error getting enumerations from REST; array is null."); } @@ -721,7 +721,7 @@ RestClient::post_enumerations_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting enumerations from REST; server returned no data."); } @@ -736,7 +736,8 @@ void RestClient::post_query_plan_from_rest( // Get array const Array* array = query.array(); if (array == nullptr) { - throw Status_RestError("Error submitting query plan to REST; null array."); + throw RestClientException( + "Error submitting query plan to REST; null array."); } Buffer buff; @@ -779,7 +780,7 @@ void RestClient::post_query_plan_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting query plan from REST; server returned no data."); } @@ -1650,7 +1651,7 @@ RestClient::post_consolidation_plan_from_rest( &returned_data, cache_key)); if (returned_data.data() == nullptr || returned_data.size() == 0) { - throw Status_RestError( + throw RestClientException( "Error getting query plan from REST; server returned no data."); } diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index dda5ccf47b9a..1816fae07101 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -77,6 +77,21 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class ArraySchemaSerializationException : public StatusException { + public: + explicit ArraySchemaSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][ArraySchema]", message) { + } +}; + +class ArraySchemaSerializationDisabledException + : public ArraySchemaSerializationException { + public: + explicit ArraySchemaSerializationDisabledException() + : ArraySchemaSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; #ifdef TILEDB_SERIALIZATION @@ -357,7 +372,7 @@ tuple>> filter_pipeline_from_capnp( void attribute_to_capnp( const Attribute* attribute, capnp::Attribute::Builder* attribute_builder) { if (attribute == nullptr) { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "Error serializing attribute; attribute is null."); } @@ -770,7 +785,7 @@ void dimension_label_to_capnp( if (dimension_label.uri_is_relative()) { dim_label_builder->setUri(dimension_label.uri().to_string()); } else { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "[Serialization::dimension_label_to_capnp] Serialization of absolute " "dimension label URIs not yet implemented."); } @@ -797,7 +812,7 @@ shared_ptr dimension_label_from_capnp( auto is_relative = dim_label_reader.getRelative(); if (!is_relative) { - throw SerializationStatusException( + throw ArraySchemaSerializationException( "[Deserialization::dimension_label_from_capnp] Deserialization of " "absolute dimension label URIs not yet implemented."); } @@ -1862,18 +1877,18 @@ void serialize_load_array_schema_request( break; } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema request; exception " + std::string(e.what())); } @@ -1907,17 +1922,17 @@ LoadArraySchemaRequest deserialize_load_array_schema_request( return load_array_schema_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema request; exception " + std::string(e.what())); } @@ -1963,18 +1978,18 @@ void serialize_load_array_schema_response( break; } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error serializing load array schema response; exception " + std::string(e.what())); } @@ -2012,17 +2027,17 @@ shared_ptr deserialize_load_array_schema_response( return load_array_schema_response_from_capnp(reader, memory_tracker); } default: { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ArraySchemaSerializationException( "Error deserializing load array schema response; exception " + std::string(e.what())); } @@ -2081,26 +2096,22 @@ Status max_buffer_sizes_deserialize( void serialize_load_array_schema_request( const Config&, const LoadArraySchemaRequest&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } LoadArraySchemaRequest deserialize_load_array_schema_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } void serialize_load_array_schema_response( const ArraySchema&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } shared_ptr deserialize_load_array_schema_response( SerializationType, const Buffer&, shared_ptr) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ArraySchemaSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/consolidation.cc b/tiledb/sm/serialization/consolidation.cc index 7efababc271d..e6c06c1ecafe 100644 --- a/tiledb/sm/serialization/consolidation.cc +++ b/tiledb/sm/serialization/consolidation.cc @@ -53,6 +53,22 @@ namespace tiledb { namespace sm { namespace serialization { +class ConsolidationSerializationException : public StatusException { + public: + explicit ConsolidationSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][Consolidation]", message) { + } +}; + +class ConsolidationSerializationDisabledException + : public ConsolidationSerializationException { + public: + explicit ConsolidationSerializationDisabledException() + : ConsolidationSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION Status array_consolidation_request_to_capnp( @@ -269,18 +285,18 @@ void serialize_consolidation_plan_request( break; } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan request; exception " + std::string(e.what())); } @@ -310,17 +326,17 @@ uint64_t deserialize_consolidation_plan_request( return consolidation_plan_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan request; exception " + std::string(e.what())); } @@ -360,18 +376,18 @@ void serialize_consolidation_plan_response( break; } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error serializing consolidation plan response; exception " + std::string(e.what())); } @@ -401,17 +417,17 @@ std::vector> deserialize_consolidation_plan_response( return consolidation_plan_response_from_capnp(reader); } default: { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw ConsolidationSerializationException( "Error deserializing consolidation plan response; exception " + std::string(e.what())); } @@ -433,26 +449,22 @@ Status array_consolidation_request_deserialize( void serialize_consolidation_plan_request( uint64_t, const Config&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } uint64_t deserialize_consolidation_plan_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot deserialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } void serialize_consolidation_plan_response( const ConsolidationPlan&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } std::vector> deserialize_consolidation_plan_response( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot deserialize; serialization not enabled."); + throw ConsolidationSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/enumeration.cc b/tiledb/sm/serialization/enumeration.cc index 197a2d8834d5..99f771fc87b4 100644 --- a/tiledb/sm/serialization/enumeration.cc +++ b/tiledb/sm/serialization/enumeration.cc @@ -48,6 +48,22 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class EnumerationSerializationException : public StatusException { + public: + explicit EnumerationSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][Enumeration]", message) { + } +}; + +class EnumerationSerializationDisabledException + : public EnumerationSerializationException { + public: + explicit EnumerationSerializationDisabledException() + : EnumerationSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION void enumeration_to_capnp( @@ -198,18 +214,18 @@ void serialize_load_enumerations_request( break; } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations request; exception " + std::string(e.what())); } @@ -239,17 +255,17 @@ std::vector deserialize_load_enumerations_request( return load_enumerations_request_from_capnp(reader); } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations request; exception " + std::string(e.what())); } @@ -289,18 +305,18 @@ void serialize_load_enumerations_response( break; } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error serializing load enumerations response; exception " + std::string(e.what())); } @@ -333,17 +349,17 @@ deserialize_load_enumerations_response( return load_enumerations_response_from_capnp(reader, memory_tracker); } default: { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw EnumerationSerializationException( "Error deserializing load enumerations response; exception " + std::string(e.what())); } @@ -356,29 +372,25 @@ void serialize_load_enumerations_request( const std::vector&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } std::vector deserialize_load_enumerations_request( SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } void serialize_load_enumerations_response( const std::vector>&, SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } std::vector> deserialize_load_enumerations_response( SerializationType, const Buffer&, shared_ptr) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw EnumerationSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION diff --git a/tiledb/sm/serialization/query_plan.cc b/tiledb/sm/serialization/query_plan.cc index 6b604566cbe3..9170522b31f6 100644 --- a/tiledb/sm/serialization/query_plan.cc +++ b/tiledb/sm/serialization/query_plan.cc @@ -53,6 +53,22 @@ using namespace tiledb::common; namespace tiledb::sm::serialization { +class QueryPlanSerializationException : public StatusException { + public: + explicit QueryPlanSerializationException(const std::string& message) + : StatusException("[TileDB::Serialization][QueryPlan]", message) { + } +}; + +class QueryPlanSerializationDisabledException + : public QueryPlanSerializationException { + public: + explicit QueryPlanSerializationDisabledException() + : QueryPlanSerializationException( + "Cannot (de)serialize; serialization not enabled.") { + } +}; + #ifdef TILEDB_SERIALIZATION void query_plan_request_to_capnp( @@ -178,18 +194,18 @@ void serialize_query_plan_request( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan request; exception " + std::string(e.what())); } @@ -225,17 +241,17 @@ void deserialize_query_plan_request( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan request; exception " + std::string(e.what())); } @@ -275,18 +291,18 @@ void serialize_query_plan_response( break; } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error serializing query plan response; exception " + std::string(e.what())); } @@ -318,17 +334,17 @@ QueryPlan deserialize_query_plan_response( return query_plan_response_from_capnp(reader, query); } default: { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; " "Unknown serialization type passed"); } } } catch (kj::Exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; kj::Exception: " + std::string(e.getDescription().cStr())); } catch (std::exception& e) { - throw Status_SerializationError( + throw QueryPlanSerializationException( "Error deserializing query plan response; exception " + std::string(e.what())); } @@ -338,26 +354,22 @@ QueryPlan deserialize_query_plan_response( void serialize_query_plan_request( const Config&, Query&, const SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } void deserialize_query_plan_request( const SerializationType, const Buffer&, ThreadPool&, Query&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } void serialize_query_plan_response( const QueryPlan&, const SerializationType, Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } QueryPlan deserialize_query_plan_response( Query&, const SerializationType, const Buffer&) { - throw Status_SerializationError( - "Cannot serialize; serialization not enabled."); + throw QueryPlanSerializationDisabledException(); } #endif // TILEDB_SERIALIZATION From f23563325e1b2967d26f2456ce99ed6f43a1db0c Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:51:29 +0200 Subject: [PATCH 438/456] Unstatus run_forward. (#5089) This removes status from the run_forward functions in the filter pipeline. [sc-49510] --- TYPE: NO_HISTORY DESC: Unstatus run_forward. --- .../sm/filter/bit_width_reduction_filter.cc | 30 +++++++-------- tiledb/sm/filter/bit_width_reduction_filter.h | 4 +- tiledb/sm/filter/bitshuffle_filter.cc | 18 ++++----- tiledb/sm/filter/bitshuffle_filter.h | 2 +- tiledb/sm/filter/byteshuffle_filter.cc | 16 ++++---- tiledb/sm/filter/byteshuffle_filter.h | 2 +- tiledb/sm/filter/checksum_md5_filter.cc | 17 ++++----- tiledb/sm/filter/checksum_md5_filter.h | 2 +- tiledb/sm/filter/checksum_sha256_filter.cc | 17 ++++----- tiledb/sm/filter/checksum_sha256_filter.h | 2 +- tiledb/sm/filter/compression_filter.cc | 37 ++++++++++--------- tiledb/sm/filter/compression_filter.h | 2 +- .../sm/filter/encryption_aes256gcm_filter.cc | 22 +++++------ .../sm/filter/encryption_aes256gcm_filter.h | 2 +- tiledb/sm/filter/filter.cc | 6 --- tiledb/sm/filter/filter.h | 10 ++++- tiledb/sm/filter/filter_pipeline.cc | 35 ++++++++++-------- tiledb/sm/filter/filter_pipeline.h | 3 +- tiledb/sm/filter/float_scaling_filter.cc | 20 +++++----- tiledb/sm/filter/float_scaling_filter.h | 6 +-- tiledb/sm/filter/noop_filter.cc | 7 ++-- tiledb/sm/filter/noop_filter.h | 2 +- tiledb/sm/filter/positive_delta_filter.cc | 28 +++++++------- tiledb/sm/filter/positive_delta_filter.h | 4 +- .../sm/filter/test/add_1_in_place_filter.cc | 8 ++-- tiledb/sm/filter/test/add_1_in_place_filter.h | 2 +- .../test/add_1_including_metadata_filter.cc | 28 +++++++------- .../test/add_1_including_metadata_filter.h | 2 +- .../filter/test/add_1_out_of_place_filter.cc | 16 ++++---- .../filter/test/add_1_out_of_place_filter.h | 2 +- .../sm/filter/test/add_n_in_place_filter.cc | 7 ++-- tiledb/sm/filter/test/add_n_in_place_filter.h | 2 +- tiledb/sm/filter/test/filter_test_support.cc | 24 +++++------- .../sm/filter/test/pseudo_checksum_filter.cc | 14 +++---- .../sm/filter/test/pseudo_checksum_filter.h | 2 +- .../test/unit_bit_width_reduction_pipeline.cc | 32 +++++----------- .../filter/test/unit_bitshuffle_pipeline.cc | 12 ++---- .../filter/test/unit_byteshuffle_pipeline.cc | 12 ++---- .../sm/filter/test/unit_checksum_pipeline.cc | 5 +-- .../filter/test/unit_encryption_pipeline.cc | 6 +-- .../test/unit_positive_delta_pipeline.cc | 21 ++++------- tiledb/sm/filter/test/unit_webp_pipeline.cc | 4 +- tiledb/sm/filter/test/unit_xor_pipeline.cc | 6 +-- tiledb/sm/filter/webp_filter.cc | 8 ++-- tiledb/sm/filter/webp_filter.h | 6 +-- tiledb/sm/filter/xor_filter.cc | 22 +++++------ tiledb/sm/filter/xor_filter.h | 4 +- tiledb/sm/query/writers/writer_base.cc | 6 +-- tiledb/sm/tile/generic_tile_io.cc | 4 +- 49 files changed, 245 insertions(+), 304 deletions(-) diff --git a/tiledb/sm/filter/bit_width_reduction_filter.cc b/tiledb/sm/filter/bit_width_reduction_filter.cc index df47c6f05fd8..22012bcac192 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.cc +++ b/tiledb/sm/filter/bit_width_reduction_filter.cc @@ -105,7 +105,7 @@ bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const { return false; } -Status BitWidthReductionFilter::run_forward( +void BitWidthReductionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -155,9 +155,9 @@ Status BitWidthReductionFilter::run_forward( case Datatype::TIME_AS: if (tile.format_version() < 20) { // Return data as-is for backwards compatibility - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } return run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); @@ -169,14 +169,14 @@ Status BitWidthReductionFilter::run_forward( case Datatype::UINT8: default: // If bit width compression can't work, just return the input unmodified. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } } template -Status BitWidthReductionFilter::run_forward( +void BitWidthReductionFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -207,24 +207,22 @@ Status BitWidthReductionFilter::run_forward( } // Allocate space in output buffer for the upper bound. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); buffer_ptr->reset_offset(); assert(buffer_ptr != nullptr); // Forward the existing metadata - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); // Allocate a buffer for this filter's metadata and write the header. - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&total_num_windows, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&input_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&total_num_windows, sizeof(uint32_t))); // Compress all parts. for (unsigned i = 0; i < num_parts; i++) { - RETURN_NOT_OK(compress_part(&parts[i], output, output_metadata)); + throw_if_not_ok(compress_part(&parts[i], output, output_metadata)); } - - return Status::Ok(); } template diff --git a/tiledb/sm/filter/bit_width_reduction_filter.h b/tiledb/sm/filter/bit_width_reduction_filter.h index ebcab1ec93df..094f4e19c51a 100644 --- a/tiledb/sm/filter/bit_width_reduction_filter.h +++ b/tiledb/sm/filter/bit_width_reduction_filter.h @@ -113,7 +113,7 @@ class BitWidthReductionFilter : public Filter { /** * Reduce the bit size of the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -192,7 +192,7 @@ class BitWidthReductionFilter : public Filter { /** Run_forward method templated on the tile cell datatype. */ template - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const tile_offsets, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/bitshuffle_filter.cc b/tiledb/sm/filter/bitshuffle_filter.cc index ae3f23fb274c..067e008b68fd 100644 --- a/tiledb/sm/filter/bitshuffle_filter.cc +++ b/tiledb/sm/filter/bitshuffle_filter.cc @@ -58,7 +58,7 @@ void BitshuffleFilter::dump(FILE* out) const { fprintf(out, "BitShuffle"); } -Status BitshuffleFilter::run_forward( +void BitshuffleFilter::run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -68,39 +68,37 @@ Status BitshuffleFilter::run_forward( auto tile_type_size = static_cast(datatype_size(filter_data_type_)); // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); // Compute the list of parts to shuffle std::vector parts; - RETURN_NOT_OK(compute_parts(input, &parts)); + throw_if_not_ok(compute_parts(input, &parts)); // Write the metadata auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Shuffle all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); if (part_size % tile_type_size != 0 || part_size % 8 != 0) { // Can't shuffle: just copy. std::memcpy(output_buf->cur_data(), part.data(), part_size); } else { - RETURN_NOT_OK(shuffle_part(tile, &part, output_buf)); + throw_if_not_ok(shuffle_part(tile, &part, output_buf)); } if (output_buf->owns_data()) output_buf->advance_size(part.size()); output_buf->advance_offset(part.size()); } - - return Status::Ok(); } Status BitshuffleFilter::compute_parts( diff --git a/tiledb/sm/filter/bitshuffle_filter.h b/tiledb/sm/filter/bitshuffle_filter.h index c1f9ca354cc1..da663c285855 100644 --- a/tiledb/sm/filter/bitshuffle_filter.h +++ b/tiledb/sm/filter/bitshuffle_filter.h @@ -87,7 +87,7 @@ class BitshuffleFilter : public Filter { /** * Shuffle the bits of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/byteshuffle_filter.cc b/tiledb/sm/filter/byteshuffle_filter.cc index f92f46a9b6e6..cd07c218c9e9 100644 --- a/tiledb/sm/filter/byteshuffle_filter.cc +++ b/tiledb/sm/filter/byteshuffle_filter.cc @@ -58,7 +58,7 @@ void ByteshuffleFilter::dump(FILE* out) const { fprintf(out, "ByteShuffle"); } -Status ByteshuffleFilter::run_forward( +void ByteshuffleFilter::run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -66,7 +66,7 @@ Status ByteshuffleFilter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -74,23 +74,21 @@ Status ByteshuffleFilter::run_forward( auto parts = input->buffers(); auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Shuffle all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); - RETURN_NOT_OK(shuffle_part(tile, &part, output_buf)); + throw_if_not_ok(shuffle_part(tile, &part, output_buf)); if (output_buf->owns_data()) output_buf->advance_size(part.size()); output_buf->advance_offset(part.size()); } - - return Status::Ok(); } Status ByteshuffleFilter::shuffle_part( diff --git a/tiledb/sm/filter/byteshuffle_filter.h b/tiledb/sm/filter/byteshuffle_filter.h index ef25bf97214e..88bb03c432e1 100644 --- a/tiledb/sm/filter/byteshuffle_filter.h +++ b/tiledb/sm/filter/byteshuffle_filter.h @@ -79,7 +79,7 @@ class ByteshuffleFilter : public Filter { /** * Shuffle the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/checksum_md5_filter.cc b/tiledb/sm/filter/checksum_md5_filter.cc index 9fa157c0af24..ce1b7d416dd1 100644 --- a/tiledb/sm/filter/checksum_md5_filter.cc +++ b/tiledb/sm/filter/checksum_md5_filter.cc @@ -60,7 +60,7 @@ void ChecksumMD5Filter::dump(FILE* out) const { fprintf(out, "ChecksumMD5"); } -Status ChecksumMD5Filter::run_forward( +void ChecksumMD5Filter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -68,7 +68,7 @@ Status ChecksumMD5Filter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Set output buffer to input buffer - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Add original input metadata as a view to the output metadata throw_if_not_ok(output_metadata->append_view(input_metadata)); @@ -82,17 +82,16 @@ Status ChecksumMD5Filter::run_forward( uint32_t part_md_size = Crypto::MD5_DIGEST_BYTES + sizeof(uint64_t); uint32_t metadata_size = (total_num_parts * part_md_size) + (2 * sizeof(uint32_t)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Checksum all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); + throw_if_not_ok(checksum_part(&part, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(checksum_part(&part, output_metadata)); } Status ChecksumMD5Filter::run_reverse( diff --git a/tiledb/sm/filter/checksum_md5_filter.h b/tiledb/sm/filter/checksum_md5_filter.h index 54462895511d..4faf0817300f 100644 --- a/tiledb/sm/filter/checksum_md5_filter.h +++ b/tiledb/sm/filter/checksum_md5_filter.h @@ -84,7 +84,7 @@ class ChecksumMD5Filter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/checksum_sha256_filter.cc b/tiledb/sm/filter/checksum_sha256_filter.cc index 2a6fbdf89c10..e91185e688ad 100644 --- a/tiledb/sm/filter/checksum_sha256_filter.cc +++ b/tiledb/sm/filter/checksum_sha256_filter.cc @@ -60,7 +60,7 @@ void ChecksumSHA256Filter::dump(FILE* out) const { fprintf(out, "ChecksumSHA256"); } -Status ChecksumSHA256Filter::run_forward( +void ChecksumSHA256Filter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -68,7 +68,7 @@ Status ChecksumSHA256Filter::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { // Set output buffer to input buffer - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Add original input metadata as a view to the output metadata throw_if_not_ok(output_metadata->append_view(input_metadata)); @@ -82,17 +82,16 @@ Status ChecksumSHA256Filter::run_forward( uint32_t part_md_size = Crypto::SHA256_DIGEST_BYTES + sizeof(uint64_t); uint32_t metadata_size = (total_num_parts * part_md_size) + (2 * sizeof(uint32_t)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Checksum all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); + throw_if_not_ok(checksum_part(&part, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(checksum_part(&part, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(checksum_part(&part, output_metadata)); } Status ChecksumSHA256Filter::run_reverse( diff --git a/tiledb/sm/filter/checksum_sha256_filter.h b/tiledb/sm/filter/checksum_sha256_filter.h index fdc79cb15470..2b47d04c6006 100644 --- a/tiledb/sm/filter/checksum_sha256_filter.h +++ b/tiledb/sm/filter/checksum_sha256_filter.h @@ -85,7 +85,7 @@ class ChecksumSHA256Filter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/compression_filter.cc b/tiledb/sm/filter/compression_filter.cc index 55c28759b8e9..458437fe469a 100644 --- a/tiledb/sm/filter/compression_filter.cc +++ b/tiledb/sm/filter/compression_filter.cc @@ -247,7 +247,7 @@ Status CompressionFilter::get_option_impl( return Status::Ok(); } -Status CompressionFilter::run_forward( +void CompressionFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -256,22 +256,24 @@ Status CompressionFilter::run_forward( FilterBuffer* output) const { // Easy case: no compression if (compressor_ == Compressor::NO_COMPRESSION) { - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } - if (input->size() > std::numeric_limits::max()) - return LOG_STATUS( - Status_FilterError("Input is too large to be compressed.")); + if (input->size() > std::numeric_limits::max()) { + throw FilterStatusException("Input is too large to be compressed."); + } if ((filter_data_type_ == Datatype::STRING_ASCII || filter_data_type_ == Datatype::STRING_UTF8) && offsets_tile) { if (compressor_ == Compressor::RLE || - compressor_ == Compressor::DICTIONARY_ENCODING) - return compress_var_string_coords( - *input, offsets_tile, *output, *output_metadata); + compressor_ == Compressor::DICTIONARY_ENCODING) { + throw_if_not_ok(compress_var_string_coords( + *input, offsets_tile, *output, *output_metadata)); + return; + } } std::vector data_parts = @@ -283,9 +285,10 @@ Status CompressionFilter::run_forward( 2 * sizeof(uint32_t) + total_num_parts * 2 * sizeof(uint32_t); auto num_metadata_parts = static_cast(metadata_parts.size()); auto num_data_parts = static_cast(data_parts.size()); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Allocate output data uint64_t output_size_ub = 0; @@ -295,18 +298,16 @@ Status CompressionFilter::run_forward( output_size_ub += part.size() + overhead(tile, part.size()); // Ensure space in output buffer for worst case. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); assert(buffer_ptr != nullptr); buffer_ptr->reset_offset(); // Compress all parts. for (auto& part : metadata_parts) - RETURN_NOT_OK(compress_part(tile, &part, buffer_ptr, output_metadata)); + throw_if_not_ok(compress_part(tile, &part, buffer_ptr, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(compress_part(tile, &part, buffer_ptr, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(compress_part(tile, &part, buffer_ptr, output_metadata)); } Status CompressionFilter::run_reverse( diff --git a/tiledb/sm/filter/compression_filter.h b/tiledb/sm/filter/compression_filter.h index 4925e79610e2..88d30e8dbb28 100644 --- a/tiledb/sm/filter/compression_filter.h +++ b/tiledb/sm/filter/compression_filter.h @@ -126,7 +126,7 @@ class CompressionFilter : public Filter { /** * Compress the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.cc b/tiledb/sm/filter/encryption_aes256gcm_filter.cc index fd9ffc4483b9..a485e1870275 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.cc +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.cc @@ -70,18 +70,19 @@ void EncryptionAES256GCMFilter::dump(FILE* out) const { fprintf(out, "EncryptionAES256GCM"); } -Status EncryptionAES256GCMFilter::run_forward( +void EncryptionAES256GCMFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { - if (key_bytes_ == nullptr) - return LOG_STATUS(Status_FilterError("Encryption error; bad key.")); + if (key_bytes_ == nullptr) { + throw FilterStatusException("Encryption error; bad key."); + } // Allocate an initial output buffer. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -95,17 +96,16 @@ Status EncryptionAES256GCMFilter::run_forward( Crypto::AES256GCM_IV_BYTES; uint32_t metadata_size = 2 * sizeof(uint32_t) + total_num_parts * part_md_size; - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&num_data_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok( + output_metadata->write(&num_metadata_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&num_data_parts, sizeof(uint32_t))); // Encrypt all parts for (auto& part : metadata_parts) - RETURN_NOT_OK(encrypt_part(&part, output_buf, output_metadata)); + throw_if_not_ok(encrypt_part(&part, output_buf, output_metadata)); for (auto& part : data_parts) - RETURN_NOT_OK(encrypt_part(&part, output_buf, output_metadata)); - - return Status::Ok(); + throw_if_not_ok(encrypt_part(&part, output_buf, output_metadata)); } Status EncryptionAES256GCMFilter::encrypt_part( diff --git a/tiledb/sm/filter/encryption_aes256gcm_filter.h b/tiledb/sm/filter/encryption_aes256gcm_filter.h index 31c8bdb252f5..548726564053 100644 --- a/tiledb/sm/filter/encryption_aes256gcm_filter.h +++ b/tiledb/sm/filter/encryption_aes256gcm_filter.h @@ -100,7 +100,7 @@ class EncryptionAES256GCMFilter : public Filter { /** * Encrypt the bytes of the input data into the output data buffer. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/filter.cc b/tiledb/sm/filter/filter.cc index bbba192c075f..81d39a703128 100644 --- a/tiledb/sm/filter/filter.cc +++ b/tiledb/sm/filter/filter.cc @@ -41,12 +41,6 @@ using namespace tiledb::common; namespace tiledb { namespace sm { -class FilterStatusException : public StatusException { - public: - explicit FilterStatusException(const std::string& msg) - : StatusException("Filter", msg) { - } -}; Filter::Filter(FilterType type, Datatype filter_data_type) { type_ = type; diff --git a/tiledb/sm/filter/filter.h b/tiledb/sm/filter/filter.h index 0ceecc737a36..910b6ab11bb6 100644 --- a/tiledb/sm/filter/filter.h +++ b/tiledb/sm/filter/filter.h @@ -53,6 +53,13 @@ enum class FilterOption : uint8_t; enum class FilterType : uint8_t; enum class Datatype : uint8_t; +class FilterStatusException : public StatusException { + public: + explicit FilterStatusException(const std::string& msg) + : StatusException("Filter", msg) { + } +}; + /** * A Filter processes or modifies a byte region, modifying it in place, or * producing output in new buffers. @@ -137,9 +144,8 @@ class Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data * @param output Buffer with filtered data (unused by in-place filters). - * @return */ - virtual Status run_forward( + virtual void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/filter_pipeline.cc b/tiledb/sm/filter/filter_pipeline.cc index d14d80e9c9a1..1f5e84e75c12 100644 --- a/tiledb/sm/filter/filter_pipeline.cc +++ b/tiledb/sm/filter/filter_pipeline.cc @@ -269,13 +269,13 @@ Status FilterPipeline::filter_chunks_forward( f->init_compression_resource_pool(compute_tp->concurrency_level()); - RETURN_NOT_OK(f->run_forward( + f->run_forward( tile, offsets_tile, &input_metadata, &input_data, &output_metadata, - &output_data)); + &output_data); input_data.set_read_only(false); throw_if_not_ok(input_data.swap(output_data)); @@ -382,13 +382,13 @@ uint32_t FilterPipeline::max_chunk_size() const { return max_chunk_size_; } -Status FilterPipeline::run_forward( +void FilterPipeline::run_forward( stats::Stats* const writer_stats, WriterTile* const tile, WriterTile* const offsets_tile, ThreadPool* const compute_tp, bool use_chunking) const { - RETURN_NOT_OK( + throw_if_not_ok( tile ? Status::Ok() : Status_Error("invalid argument: null Tile*")); writer_stats->add_counter("write_filtered_byte_num", tile->size()); @@ -404,25 +404,28 @@ Status FilterPipeline::run_forward( // Get the chunk sizes for var size attributes. auto&& [st, chunk_offsets] = get_var_chunk_sizes(chunk_size, tile, offsets_tile); - RETURN_NOT_OK_ELSE(st, tile->filtered_buffer().clear()); + if (!st.ok()) { + tile->filtered_buffer().clear(); + throw_if_not_ok(st); + } // Run the filters over all the chunks and store the result in // 'filtered_buffer'. - RETURN_NOT_OK_ELSE( - filter_chunks_forward( - *tile, - offsets_tile, - chunk_size, - *chunk_offsets, - tile->filtered_buffer(), - compute_tp), - tile->filtered_buffer().clear()); + st = filter_chunks_forward( + *tile, + offsets_tile, + chunk_size, + *chunk_offsets, + tile->filtered_buffer(), + compute_tp); + if (!st.ok()) { + tile->filtered_buffer().clear(); + throw_if_not_ok(st); + } // The contents of 'buffer' have been filtered and stored // in 'filtered_buffer'. We can safely free 'buffer'. tile->clear_data(); - - return Status::Ok(); } void FilterPipeline::run_reverse_generic_tile( diff --git a/tiledb/sm/filter/filter_pipeline.h b/tiledb/sm/filter/filter_pipeline.h index 66c6093803f1..b12339784790 100644 --- a/tiledb/sm/filter/filter_pipeline.h +++ b/tiledb/sm/filter/filter_pipeline.h @@ -221,9 +221,8 @@ class FilterPipeline { * @param compute_tp The thread pool for compute-bound tasks. * @param chunking True if the tile should be cut into chunks before * filtering, false if not. - * @return Status */ - Status run_forward( + void run_forward( stats::Stats* writer_stats, WriterTile* tile, WriterTile* offsets_tile, diff --git a/tiledb/sm/filter/float_scaling_filter.cc b/tiledb/sm/filter/float_scaling_filter.cc index f11d4703f52b..c2287c4a60b7 100644 --- a/tiledb/sm/filter/float_scaling_filter.cc +++ b/tiledb/sm/filter/float_scaling_filter.cc @@ -62,7 +62,7 @@ void FloatScalingFilter::serialize_impl(Serializer& serializer) const { } template -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -70,9 +70,9 @@ Status FloatScalingFilter::run_forward( auto input_parts = input->buffers(); uint32_t num_parts = static_cast(input_parts.size()); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // Iterate through all the input buffers. for (auto& i : input_parts) { @@ -80,8 +80,8 @@ Status FloatScalingFilter::run_forward( assert(s % sizeof(T) == 0); uint32_t num_elems_in_part = (s / sizeof(T)); uint32_t new_size = num_elems_in_part * sizeof(W); - RETURN_NOT_OK(output_metadata->write(&new_size, sizeof(uint32_t))); - RETURN_NOT_OK(output->prepend_buffer(new_size)); + throw_if_not_ok(output_metadata->write(&new_size, sizeof(uint32_t))); + throw_if_not_ok(output->prepend_buffer(new_size)); // Iterate through each input buffer, storing each raw float as // an integer with the value round((raw_float - offset) / scale). @@ -90,18 +90,16 @@ Status FloatScalingFilter::run_forward( T elem = part_data[j]; W converted_elem = static_cast( round((elem - static_cast(offset_)) / static_cast(scale_))); - RETURN_NOT_OK(output->write(&converted_elem, sizeof(W))); + throw_if_not_ok(output->write(&converted_elem, sizeof(W))); if (j != num_elems_in_part - 1) { output->advance_offset(sizeof(W)); } } } - - return Status::Ok(); } template -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -126,7 +124,7 @@ Status FloatScalingFilter::run_forward( } } -Status FloatScalingFilter::run_forward( +void FloatScalingFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/float_scaling_filter.h b/tiledb/sm/filter/float_scaling_filter.h index 518fe0ed38b1..c50a2663a901 100644 --- a/tiledb/sm/filter/float_scaling_filter.h +++ b/tiledb/sm/filter/float_scaling_filter.h @@ -106,7 +106,7 @@ class FloatScalingFilter : public Filter { * stores it as integers with the value round((raw_float - offset) / scale) * with the pre-specified byte width. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -167,7 +167,7 @@ class FloatScalingFilter : public Filter { * Run forward, templated on the size of the input type. */ template - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -177,7 +177,7 @@ class FloatScalingFilter : public Filter { * Run forward, templated on the size of the input type and byte width. */ template - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/filter/noop_filter.cc b/tiledb/sm/filter/noop_filter.cc index 26a36abe6e02..b5beab218fd1 100644 --- a/tiledb/sm/filter/noop_filter.cc +++ b/tiledb/sm/filter/noop_filter.cc @@ -55,16 +55,15 @@ void NoopFilter::dump(FILE* out) const { fprintf(out, "NoOp"); } -Status NoopFilter::run_forward( +void NoopFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status NoopFilter::run_reverse( diff --git a/tiledb/sm/filter/noop_filter.h b/tiledb/sm/filter/noop_filter.h index 3e5bbc87f555..e2546dea669d 100644 --- a/tiledb/sm/filter/noop_filter.h +++ b/tiledb/sm/filter/noop_filter.h @@ -59,7 +59,7 @@ class NoopFilter : public Filter { /** * Run forward. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/positive_delta_filter.cc b/tiledb/sm/filter/positive_delta_filter.cc index 9b469efe16c2..f308cc788c2c 100644 --- a/tiledb/sm/filter/positive_delta_filter.cc +++ b/tiledb/sm/filter/positive_delta_filter.cc @@ -68,7 +68,7 @@ bool PositiveDeltaFilter::accepts_input_datatype(Datatype datatype) const { return false; } -Status PositiveDeltaFilter::run_forward( +void PositiveDeltaFilter::run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -131,22 +131,22 @@ Status PositiveDeltaFilter::run_forward( case Datatype::TIME_AS: if (tile.format_version() < 20) { // Return data as-is for backwards compatibility. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } return run_forward( tile, offsets_tile, input_metadata, input, output_metadata, output); default: // If encoding can't work, just return the input unmodified. - RETURN_NOT_OK(output->append_view(input)); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output->append_view(input)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + return; } } template -Status PositiveDeltaFilter::run_forward( +void PositiveDeltaFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -174,23 +174,21 @@ Status PositiveDeltaFilter::run_forward( } // Allocate space in output buffer for the upper bound. - RETURN_NOT_OK(output->prepend_buffer(output_size_ub)); + throw_if_not_ok(output->prepend_buffer(output_size_ub)); Buffer* buffer_ptr = output->buffer_ptr(0); buffer_ptr->reset_offset(); assert(buffer_ptr != nullptr); // Forward the existing metadata - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->append_view(input_metadata)); // Allocate a buffer for this filter's metadata and write the header. - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&total_num_windows, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&total_num_windows, sizeof(uint32_t))); // Compress all parts. for (unsigned i = 0; i < num_parts; i++) { - RETURN_NOT_OK(encode_part(&parts[i], output, output_metadata)); + throw_if_not_ok(encode_part(&parts[i], output, output_metadata)); } - - return Status::Ok(); } template diff --git a/tiledb/sm/filter/positive_delta_filter.h b/tiledb/sm/filter/positive_delta_filter.h index 01fcb3e51d8a..57445b2f3a6c 100644 --- a/tiledb/sm/filter/positive_delta_filter.h +++ b/tiledb/sm/filter/positive_delta_filter.h @@ -104,7 +104,7 @@ class PositiveDeltaFilter : public Filter { /** * Perform positive-delta encoding of the given input into the given output. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const, FilterBuffer* input_metadata, @@ -154,7 +154,7 @@ class PositiveDeltaFilter : public Filter { /** Run_forward method templated on the tile cell datatype. */ template - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const tile_offsets, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.cc b/tiledb/sm/filter/test/add_1_in_place_filter.cc index f4c9425fade2..d5683a123d46 100644 --- a/tiledb/sm/filter/test/add_1_in_place_filter.cc +++ b/tiledb/sm/filter/test/add_1_in_place_filter.cc @@ -44,7 +44,7 @@ void Add1InPlace::dump(FILE* out) const { (void)out; } -Status Add1InPlace::run_forward( +void Add1InPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -52,7 +52,7 @@ Status Add1InPlace::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); output->reset_offset(); uint64_t nelts = input_size / sizeof(uint64_t); @@ -63,9 +63,7 @@ Status Add1InPlace::run_forward( } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status Add1InPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_in_place_filter.h b/tiledb/sm/filter/test/add_1_in_place_filter.h index 67b33f33de67..ec64e9801cd4 100644 --- a/tiledb/sm/filter/test/add_1_in_place_filter.h +++ b/tiledb/sm/filter/test/add_1_in_place_filter.h @@ -52,7 +52,7 @@ class Add1InPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.cc b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc index dfc344933076..cd74e5b636b9 100644 --- a/tiledb/sm/filter/test/add_1_including_metadata_filter.cc +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.cc @@ -44,7 +44,7 @@ void Add1IncludingMetadataFilter::dump(FILE* out) const { (void)out; } -Status Add1IncludingMetadataFilter::run_forward( +void Add1IncludingMetadataFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -57,46 +57,44 @@ Status Add1IncludingMetadataFilter::run_forward( md_nelts = input_md_size / sizeof(uint64_t); // Add another output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size + input_md_size)); + throw_if_not_ok(output->prepend_buffer(input_size + input_md_size)); output->reset_offset(); // Filter input data for (uint64_t i = 0; i < nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } // Finish any remaining bytes to ensure no data loss. auto rem = input_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Now filter input metadata. for (uint64_t i = 0; i < md_nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input_metadata->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input_metadata->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } rem = input_md_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input_metadata->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input_metadata->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Because this filter modifies the input metadata, we need output metadata // that allows the original metadata to be reconstructed on reverse. Also // note that contrary to most filters, we don't forward the input metadata. - RETURN_NOT_OK(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_size, sizeof(uint32_t))); - RETURN_NOT_OK(output_metadata->write(&input_md_size, sizeof(uint32_t))); - - return Status::Ok(); + throw_if_not_ok(output_metadata->prepend_buffer(2 * sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&input_size, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->write(&input_md_size, sizeof(uint32_t))); } Status Add1IncludingMetadataFilter::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_including_metadata_filter.h b/tiledb/sm/filter/test/add_1_including_metadata_filter.h index c508caff401d..93b62d919eb2 100644 --- a/tiledb/sm/filter/test/add_1_including_metadata_filter.h +++ b/tiledb/sm/filter/test/add_1_including_metadata_filter.h @@ -54,7 +54,7 @@ class Add1IncludingMetadataFilter : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.cc b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc index aa7c7f475394..78cc89d89ba4 100644 --- a/tiledb/sm/filter/test/add_1_out_of_place_filter.cc +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.cc @@ -43,7 +43,7 @@ void Add1OutOfPlace::dump(FILE* out) const { (void)out; } -Status Add1OutOfPlace::run_forward( +void Add1OutOfPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -54,28 +54,26 @@ Status Add1OutOfPlace::run_forward( auto nelts = input_size / sizeof(uint64_t); // Add another output buffer. - RETURN_NOT_OK(output->prepend_buffer(input_size)); + throw_if_not_ok(output->prepend_buffer(input_size)); output->reset_offset(); for (uint64_t i = 0; i < nelts; i++) { uint64_t inc; - RETURN_NOT_OK(input->read(&inc, sizeof(uint64_t))); + throw_if_not_ok(input->read(&inc, sizeof(uint64_t))); inc++; - RETURN_NOT_OK(output->write(&inc, sizeof(uint64_t))); + throw_if_not_ok(output->write(&inc, sizeof(uint64_t))); } // Finish any remaining bytes to ensure no data loss. auto rem = input_size % sizeof(uint64_t); for (unsigned i = 0; i < rem; i++) { char byte; - RETURN_NOT_OK(input->read(&byte, sizeof(char))); - RETURN_NOT_OK(output->write(&byte, sizeof(char))); + throw_if_not_ok(input->read(&byte, sizeof(char))); + throw_if_not_ok(output->write(&byte, sizeof(char))); } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status Add1OutOfPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_1_out_of_place_filter.h b/tiledb/sm/filter/test/add_1_out_of_place_filter.h index 21065a002f65..c39d70c9d086 100644 --- a/tiledb/sm/filter/test/add_1_out_of_place_filter.h +++ b/tiledb/sm/filter/test/add_1_out_of_place_filter.h @@ -52,7 +52,7 @@ class Add1OutOfPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.cc b/tiledb/sm/filter/test/add_n_in_place_filter.cc index 15fa7df2f856..6452ae19f9bc 100644 --- a/tiledb/sm/filter/test/add_n_in_place_filter.cc +++ b/tiledb/sm/filter/test/add_n_in_place_filter.cc @@ -45,7 +45,7 @@ void AddNInPlace::dump(FILE* out) const { (void)out; } -Status AddNInPlace::run_forward( +void AddNInPlace::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -53,7 +53,7 @@ Status AddNInPlace::run_forward( FilterBuffer* output_metadata, FilterBuffer* output) const { auto input_size = input->size(); - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); output->reset_offset(); uint64_t nelts = input_size / sizeof(uint64_t); @@ -64,8 +64,7 @@ Status AddNInPlace::run_forward( } // Metadata not modified by this filter. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - return Status::Ok(); + throw_if_not_ok(output_metadata->append_view(input_metadata)); } Status AddNInPlace::run_reverse( diff --git a/tiledb/sm/filter/test/add_n_in_place_filter.h b/tiledb/sm/filter/test/add_n_in_place_filter.h index 97cba2c3c18c..b5c5e3069c59 100644 --- a/tiledb/sm/filter/test/add_n_in_place_filter.h +++ b/tiledb/sm/filter/test/add_n_in_place_filter.h @@ -52,7 +52,7 @@ class AddNInPlace : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/filter_test_support.cc b/tiledb/sm/filter/test/filter_test_support.cc index f517c5b86509..907940fd0138 100644 --- a/tiledb/sm/filter/test/filter_test_support.cc +++ b/tiledb/sm/filter/test/filter_test_support.cc @@ -68,13 +68,11 @@ void check_run_pipeline_full( const FilteredTileChecker& filtered_buffer_checker, shared_ptr memory_tracker) { // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); + pipeline.run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); @@ -120,13 +118,11 @@ void check_run_pipeline_roundtrip( const TileDataGenerator* test_data, shared_ptr memory_tracker) { // Run the pipeline forward. - CHECK(pipeline - .run_forward( - &dummy_stats, - tile.get(), - offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, - &tp) - .ok()); + pipeline.run_forward( + &dummy_stats, + tile.get(), + offsets_tile.has_value() ? offsets_tile.value().get() : nullptr, + &tp); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.cc b/tiledb/sm/filter/test/pseudo_checksum_filter.cc index c65a7c7308ef..0b12a02093c3 100644 --- a/tiledb/sm/filter/test/pseudo_checksum_filter.cc +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.cc @@ -43,7 +43,7 @@ void PseudoChecksumFilter::dump(FILE* out) const { (void)out; } -Status PseudoChecksumFilter::run_forward( +void PseudoChecksumFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -54,24 +54,22 @@ Status PseudoChecksumFilter::run_forward( auto nelts = input_size / sizeof(uint64_t); // The input is unmodified by this filter. - RETURN_NOT_OK(output->append_view(input)); + throw_if_not_ok(output->append_view(input)); // Forward the existing metadata and prepend a metadata buffer for the // checksum. - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(sizeof(uint64_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(sizeof(uint64_t))); output_metadata->reset_offset(); uint64_t sum = 0; for (uint64_t i = 0; i < nelts; i++) { uint64_t val; - RETURN_NOT_OK(input->read(&val, sizeof(uint64_t))); + throw_if_not_ok(input->read(&val, sizeof(uint64_t))); sum += val; } - RETURN_NOT_OK(output_metadata->write(&sum, sizeof(uint64_t))); - - return Status::Ok(); + throw_if_not_ok(output_metadata->write(&sum, sizeof(uint64_t))); } Status PseudoChecksumFilter::run_reverse( diff --git a/tiledb/sm/filter/test/pseudo_checksum_filter.h b/tiledb/sm/filter/test/pseudo_checksum_filter.h index 96197b7ad4c7..2fee1dffe1c1 100644 --- a/tiledb/sm/filter/test/pseudo_checksum_filter.h +++ b/tiledb/sm/filter/test/pseudo_checksum_filter.h @@ -52,7 +52,7 @@ class PseudoChecksumFilter : public tiledb::sm::Filter { void dump(FILE* out) const override; - Status run_forward( + void run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, diff --git a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc index e74eca6ca0aa..9d2717548445 100644 --- a/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bit_width_reduction_pipeline.cc @@ -54,7 +54,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -103,7 +103,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -139,7 +139,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -176,7 +176,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -205,7 +205,7 @@ TEST_CASE("Filter: Test bit width reduction", "[filter][bit-width-reduction]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -270,10 +270,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -343,10 +340,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); pipeline.get_filter()->set_max_window_size( window_size); - - CHECK(pipeline - .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -384,9 +378,7 @@ TEST_CASE( CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -444,8 +436,7 @@ TEST_CASE( constants::cell_var_offset_size)); } - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), &offsets_tile32, &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), &offsets_tile32, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -476,10 +467,7 @@ TEST_CASE( } auto offsets_tile = make_offsets_tile(offsets, tracker); - - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc index 665fc405bef5..507e7c0967ac 100644 --- a/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_bitshuffle_pipeline.cc @@ -53,7 +53,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { pipeline.add_filter(BitshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -84,7 +84,7 @@ TEST_CASE("Filter: Test bitshuffle", "[filter][bitshuffle]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); @@ -144,9 +144,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { SECTION("- Single stage") { WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -178,9 +176,7 @@ TEST_CASE("Filter: Test bitshuffle var", "[filter][bitshuffle][var]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc index 7bd576815e40..735430f47596 100644 --- a/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc +++ b/tiledb/sm/filter/test/unit_byteshuffle_pipeline.cc @@ -51,7 +51,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { pipeline.add_filter(ByteshuffleFilter(Datatype::UINT64)); SECTION("- Single stage") { - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -82,7 +82,7 @@ TEST_CASE("Filter: Test byteshuffle", "[filter][byteshuffle]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK(pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); @@ -142,9 +142,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { SECTION("- Single stage") { WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -176,9 +174,7 @@ TEST_CASE("Filter: Test byteshuffle var", "[filter][byteshuffle][var]") { CHECK_NOTHROW(tile2->write(&i, i * sizeof(uint32_t), sizeof(uint32_t))); } - CHECK( - pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile2.get(), offsets_tile.get(), &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_checksum_pipeline.cc b/tiledb/sm/filter/test/unit_checksum_pipeline.cc index c7e667abf8dd..f9877eab22c3 100644 --- a/tiledb/sm/filter/test/unit_checksum_pipeline.cc +++ b/tiledb/sm/filter/test/unit_checksum_pipeline.cc @@ -56,7 +56,7 @@ TEST_CASE( ThreadPool tp(4); ChecksumMD5Filter md5_filter(Datatype::UINT64); md5_pipeline.add_filter(md5_filter); - CHECK(md5_pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + md5_pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -76,8 +76,7 @@ TEST_CASE( FilterPipeline sha_256_pipeline; ChecksumMD5Filter sha_256_filter(Datatype::UINT64); sha_256_pipeline.add_filter(sha_256_filter); - CHECK(sha_256_pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp) - .ok()); + sha_256_pipeline.run_forward(&dummy_stats, tile2.get(), nullptr, &tp); CHECK(tile2->size() == 0); CHECK(tile2->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/test/unit_encryption_pipeline.cc b/tiledb/sm/filter/test/unit_encryption_pipeline.cc index ed0ed9a1ed93..84c32c9b3794 100644 --- a/tiledb/sm/filter/test/unit_encryption_pipeline.cc +++ b/tiledb/sm/filter/test/unit_encryption_pipeline.cc @@ -54,7 +54,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { pipeline.add_filter(EncryptionAES256GCMFilter(Datatype::UINT64)); // No key set - CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK_THROWS(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp)); // Create and set a key char key[32]; @@ -64,7 +64,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { filter->set_key(key); // Check success - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -79,7 +79,7 @@ TEST_CASE("Filter: Test encryption", "[filter][encryption]") { // Check error decrypting with wrong key. tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); key[0]++; filter->set_key(key); diff --git a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc index 5242b882fd28..4b02fdff0c0b 100644 --- a/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc +++ b/tiledb/sm/filter/test/unit_positive_delta_pipeline.cc @@ -53,7 +53,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { SECTION("- Single stage") { auto tile = make_increasing_tile(nelts, tracker); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -101,7 +101,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { pipeline.get_filter()->set_max_window_size( window_size); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -123,7 +123,7 @@ TEST_CASE("Filter: Test positive-delta encoding", "[filter][positive-delta]") { CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK(!pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + CHECK_THROWS(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp)); } } @@ -174,10 +174,7 @@ TEST_CASE( auto offsets_tile = make_offsets_tile(offsets, tracker); WhiteboxWriterTile::set_max_tile_chunk_size(80); - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); - + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -249,10 +246,7 @@ TEST_CASE( pipeline.get_filter()->set_max_window_size( window_size); - - CHECK(pipeline - .run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); @@ -277,9 +271,8 @@ TEST_CASE( CHECK_NOTHROW(tile->write(&val, i * sizeof(uint64_t), sizeof(uint64_t))); } - CHECK( - !pipeline.run_forward(&dummy_stats, tile.get(), offsets_tile.get(), &tp) - .ok()); + CHECK_THROWS(pipeline.run_forward( + &dummy_stats, tile.get(), offsets_tile.get(), &tp)); } WhiteboxWriterTile::set_max_tile_chunk_size(constants::max_tile_chunk_size); diff --git a/tiledb/sm/filter/test/unit_webp_pipeline.cc b/tiledb/sm/filter/test/unit_webp_pipeline.cc index 27a74a7b9751..c8013e5d4a60 100644 --- a/tiledb/sm/filter/test/unit_webp_pipeline.cc +++ b/tiledb/sm/filter/test/unit_webp_pipeline.cc @@ -85,9 +85,7 @@ TEST_CASE("Filter: Round trip WebpFilter RGB data", "[filter][webp]") { width * 3, Datatype::UINT8)); bool use_chunking = true; - CHECK( - pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp, use_chunking) - .ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp, use_chunking); // Check the original unfiltered data was removed. CHECK(tile->size() == 0); diff --git a/tiledb/sm/filter/test/unit_xor_pipeline.cc b/tiledb/sm/filter/test/unit_xor_pipeline.cc index 51ea0c0e6df2..3f4e18a17ea7 100644 --- a/tiledb/sm/filter/test/unit_xor_pipeline.cc +++ b/tiledb/sm/filter/test/unit_xor_pipeline.cc @@ -113,7 +113,7 @@ void testing_float_scaling_filter() { ->set_option(FilterOption::SCALE_FLOAT_OFFSET, &foffset) .ok()); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); // Check new size and number of chunks CHECK(tile->size() == 0); @@ -177,7 +177,7 @@ void testing_xor_filter(Datatype t) { ThreadPool tp(4); pipeline.add_filter(XORFilter(t)); - CHECK(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); // Check new size and number of chunks CHECK(tile->size() == 0); @@ -293,7 +293,7 @@ TEST_CASE("Filter: Pipeline filtered output types", "[filter][pipeline]") { } ThreadPool tp(4); - REQUIRE(pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp).ok()); + pipeline.run_forward(&dummy_stats, tile.get(), nullptr, &tp); CHECK(tile->size() == 0); CHECK(tile->filtered_buffer().size() != 0); diff --git a/tiledb/sm/filter/webp_filter.cc b/tiledb/sm/filter/webp_filter.cc index fe4ea38d9c10..58b1e41a46ed 100644 --- a/tiledb/sm/filter/webp_filter.cc +++ b/tiledb/sm/filter/webp_filter.cc @@ -54,7 +54,7 @@ namespace tiledb::sm { * a program crash. */ -Status WebpFilter::run_forward( +void WebpFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer*, @@ -101,7 +101,7 @@ using namespace tiledb::common; namespace tiledb::sm { -Status WebpFilter::run_forward( +void WebpFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -111,7 +111,7 @@ Status WebpFilter::run_forward( return run_forward(input_metadata, input, output_metadata, output); } -Status WebpFilter::run_forward( +void WebpFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, @@ -212,8 +212,6 @@ Status WebpFilter::run_forward( throw_if_not_ok(output->prepend_buffer(enc_size)); throw_if_not_ok(output->write(result, enc_size)); } - - return Status::Ok(); } Status WebpFilter::run_reverse( diff --git a/tiledb/sm/filter/webp_filter.h b/tiledb/sm/filter/webp_filter.h index cd6648a06cb4..7e9a8b31a9dc 100644 --- a/tiledb/sm/filter/webp_filter.h +++ b/tiledb/sm/filter/webp_filter.h @@ -171,9 +171,8 @@ class WebpFilter : public Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data. * @param output Buffer with filtered data (unused by in-place filters). - * @return Status::Ok() on success. Throws on failure. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -189,9 +188,8 @@ class WebpFilter : public Filter { * @param input Buffer with data to be filtered. * @param output_metadata Buffer with metadata for filtered data. * @param output Buffer with filtered data (unused by in-place filters). - * @return Status::Ok() on success. Throws on failure. */ - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/filter/xor_filter.cc b/tiledb/sm/filter/xor_filter.cc index 488ffc7fe466..b6ecdc6c36d5 100644 --- a/tiledb/sm/filter/xor_filter.cc +++ b/tiledb/sm/filter/xor_filter.cc @@ -77,7 +77,7 @@ Datatype XORFilter::output_datatype(tiledb::sm::Datatype input_type) const { } } -Status XORFilter::run_forward( +void XORFilter::run_forward( const WriterTile&, WriterTile* const, FilterBuffer* input_metadata, @@ -105,25 +105,25 @@ Status XORFilter::run_forward( input_metadata, input, output_metadata, output); } default: { - return Status_FilterError( + throw FilterStatusException( "XORFilter::run_forward: datatype size cannot be converted to " "integer type."); } } - return Status_FilterError("XORFilter::run_forward: invalid datatype."); + throw FilterStatusException("XORFilter::run_forward: invalid datatype."); } template < typename T, typename std::enable_if::value>::type*> -Status XORFilter::run_forward( +void XORFilter::run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, FilterBuffer* output) const { // Output size does not change with this filter. - RETURN_NOT_OK(output->prepend_buffer(input->size())); + throw_if_not_ok(output->prepend_buffer(input->size())); Buffer* output_buf = output->buffer_ptr(0); assert(output_buf != nullptr); @@ -131,18 +131,16 @@ Status XORFilter::run_forward( auto parts = input->buffers(); auto num_parts = (uint32_t)parts.size(); uint32_t metadata_size = sizeof(uint32_t) + num_parts * sizeof(uint32_t); - RETURN_NOT_OK(output_metadata->append_view(input_metadata)); - RETURN_NOT_OK(output_metadata->prepend_buffer(metadata_size)); - RETURN_NOT_OK(output_metadata->write(&num_parts, sizeof(uint32_t))); + throw_if_not_ok(output_metadata->append_view(input_metadata)); + throw_if_not_ok(output_metadata->prepend_buffer(metadata_size)); + throw_if_not_ok(output_metadata->write(&num_parts, sizeof(uint32_t))); // XOR all parts for (const auto& part : parts) { auto part_size = (uint32_t)part.size(); - RETURN_NOT_OK(output_metadata->write(&part_size, sizeof(uint32_t))); - RETURN_NOT_OK(xor_part(&part, output_buf)); + throw_if_not_ok(output_metadata->write(&part_size, sizeof(uint32_t))); + throw_if_not_ok(xor_part(&part, output_buf)); } - - return Status::Ok(); } template < diff --git a/tiledb/sm/filter/xor_filter.h b/tiledb/sm/filter/xor_filter.h index e6decfe71539..be118f7c6017 100644 --- a/tiledb/sm/filter/xor_filter.h +++ b/tiledb/sm/filter/xor_filter.h @@ -90,7 +90,7 @@ class XORFilter : public Filter { * element in the part, and then the differences of each consecutive pair * of elements. */ - Status run_forward( + void run_forward( const WriterTile& tile, WriterTile* const offsets_tile, FilterBuffer* input_metadata, @@ -119,7 +119,7 @@ class XORFilter : public Filter { template < typename T, typename std::enable_if::value>::type* = nullptr> - Status run_forward( + void run_forward( FilterBuffer* input_metadata, FilterBuffer* input, FilterBuffer* output_metadata, diff --git a/tiledb/sm/query/writers/writer_base.cc b/tiledb/sm/query/writers/writer_base.cc index 5f5c65554b72..d03571115534 100644 --- a/tiledb/sm/query/writers/writer_base.cc +++ b/tiledb/sm/query/writers/writer_base.cc @@ -899,7 +899,7 @@ Status WriterBase::filter_tile( } // Append an encryption filter when necessary. - RETURN_NOT_OK(FilterPipeline::append_encryption_filter( + throw_if_not_ok(FilterPipeline::append_encryption_filter( &filters, array_->get_encryption_key())); // Check if chunk or tile level filtering/unfiltering is appropriate @@ -907,8 +907,8 @@ Status WriterBase::filter_tile( array_schema_.var_size(name), array_schema_.version(), tile->type()); assert(!tile->filtered()); - RETURN_NOT_OK(filters.run_forward( - stats_, tile, offsets_tile, &resources_.compute_tp(), use_chunking)); + filters.run_forward( + stats_, tile, offsets_tile, &resources_.compute_tp(), use_chunking); assert(tile->filtered()); return Status::Ok(); diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 976f333291c5..7ebaef581d7e 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -196,8 +196,8 @@ void GenericTileIO::write_generic( // Filter tile assert(!tile->filtered()); - throw_if_not_ok(header.filters.run_forward( - &resources_.stats(), tile.get(), nullptr, &resources_.compute_tp())); + header.filters.run_forward( + &resources_.stats(), tile.get(), nullptr, &resources_.compute_tp()); header.persisted_size = tile->filtered_buffer().size(); assert(tile->filtered()); From 7b4f4030106bbf2634e4dff3ce241db4a8b66e6d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 17 Jun 2024 17:52:41 +0300 Subject: [PATCH 439/456] Refactor handling of embedded `magic.mgc`. (#4989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [SC-25167](https://app.shortcut.com/tiledb-inc/story/25167) [SC-47655](https://app.shortcut.com/tiledb-inc/story/47655) [SC-47656](https://app.shortcut.com/tiledb-inc/story/47656) [SC-47657](https://app.shortcut.com/tiledb-inc/story/47657) [SC-47658](https://app.shortcut.com/tiledb-inc/story/47658) This PR overhauls the facilities to embed and load the `magic.mgc` file that is needed by libmagic: * The most important change is the removal of `magic_mgc_gzipped.bin.tar.bz2`. This file contained a copy of `magic.mgc` that was compressed, converted to escaped C characters, packed and compressed again to take less space, and stored in source control, so that at build time to get unpacked and `#include`d in `mgc_dict.cc`. Because this file was being prepared ahead of time by a manually invoked C++ program, this approach had the disadvantage that it tied the Core to a specific version of libmagic. This was made evident in #4673, where just updating libmagic was not enough; we also had to update `magic_mgc_gzipped.bin.tar.bz2`. What we do now is rely on CMake to find `magic.mgc` and perform its entire preparation at build time. The C++ program was rewritten to be a CMake script, which makes it much simpler and enables it to run on cross-compilation scenarios. The script accepts the uncompressed `magic.mgc` file, compresses it and produces a header file of the following format: ```cpp static const unsigned char magic_mgc_compressed_bytes[] = { 0x28, 0xb5, 0x2f, 0xfd, … }; constexpr size_t magic_mgc_compressed_size = sizeof(magic_mgc_compressed_bytes); // Editorial note: we used to prepend the decompressed size at the start of the // binary blob, but this was non-standard and could not be easily done by CMake. constexpr size_t magic_mgc_decompressed_size = 7041352; ``` * The algorithm to compress `magic.mgc` was changed from gzip to zstd, resulting in a 17.9% reduction of the compressed size (from 333067 το 273500 bytes). * Tests for `mgc_dict` were also updated to use Catch2, and were wired to run along with the other standalone unit tests. * This necessitated to make an object library for `mgc_dict`, which was done as well. Validated by successfully running `unit_mgc_dict` locally. --- TYPE: BUILD DESC: Improve embedding of `magic.mgc` and allow compiling with any libmagic version. --- ports/libmagic/CMakeLists.txt | 4 + tiledb/CMakeLists.txt | 27 +- tiledb/sm/compressors/CMakeLists.txt | 33 +- tiledb/sm/compressors/util/gzip_wrappers.cc | 100 ---- tiledb/sm/compressors/util/gzip_wrappers.h | 67 --- .../util/tdb_gzip_embedded_data.cc | 214 -------- tiledb/sm/compressors/zstd_compressor.cc | 30 +- tiledb/sm/compressors/zstd_compressor.h | 14 +- tiledb/sm/misc/CMakeLists.txt | 57 +-- ...generate_embedded_data_header.script.cmake | 72 +++ tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 | Bin 322448 -> 0 bytes tiledb/sm/misc/mgc_dict.cc | 61 +-- tiledb/sm/misc/mgc_dict.h | 63 +-- tiledb/sm/misc/test/CMakeLists.txt | 8 + tiledb/sm/misc/test/compile_mgc_dict_main.cc | 34 ++ tiledb/sm/misc/test/unit_mgc_dict.cc | 468 +++++++----------- 16 files changed, 390 insertions(+), 862 deletions(-) delete mode 100644 tiledb/sm/compressors/util/gzip_wrappers.cc delete mode 100644 tiledb/sm/compressors/util/gzip_wrappers.h delete mode 100644 tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc create mode 100755 tiledb/sm/misc/generate_embedded_data_header.script.cmake delete mode 100644 tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 create mode 100644 tiledb/sm/misc/test/compile_mgc_dict_main.cc diff --git a/ports/libmagic/CMakeLists.txt b/ports/libmagic/CMakeLists.txt index a6248bdf871a..3d731affe3d0 100644 --- a/ports/libmagic/CMakeLists.txt +++ b/ports/libmagic/CMakeLists.txt @@ -91,6 +91,10 @@ set(targets ${targets} libmagic) target_link_libraries(libmagic PRIVATE PCRE2::POSIX) +if(WIN32) + target_link_libraries(libmagic PRIVATE shlwapi) +endif() + target_include_directories(libmagic PUBLIC "$" diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index c541db76a583..98e4eda41f35 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -169,7 +169,6 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/gzip_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/lz4_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/rle_compressor.cc - ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/util/gzip_wrappers.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/compressors/zstd_compressor.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/config/config.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/config/config_iter.cc @@ -440,33 +439,10 @@ target_link_libraries(TILEDB_CORE_OBJECTS INTERFACE object_store_definitions) ############################################################ # provide actions/target for preparation of magic.mgc data for embedding/build -set(MGC_GZIPPED_BIN_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/..") -set(MGC_GZIPPED_BIN_OUTPUT_FILE "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}/magic_mgc_gzipped.bin") -set(MGC_GZIPPED_BIN_INPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/sm/misc") -set(MGC_GZIPPED_BIN_INPUT_FILE "${MGC_GZIPPED_BIN_INPUT_DIRECTORY}/magic_mgc_gzipped.bin.tar.bz2") +find_package(Magic_EP REQUIRED) -add_custom_command( - OUTPUT "${MGC_GZIPPED_BIN_OUTPUT_FILE}" - DEPENDS "${MGC_GZIPPED_BIN_INPUT_FILE}" - COMMAND ${CMAKE_COMMAND} -E tar x "${MGC_GZIPPED_BIN_INPUT_FILE}" - WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" -) -add_custom_target(gen_mgc_unarch ALL - DEPENDS ${MGC_GZIPPED_BIN_OUTPUT_FILE} -) add_dependencies(TILEDB_CORE_OBJECTS gen_mgc_unarch) -add_custom_target( - update-embedded-magic-data - COMMAND "$" < "${libmagic_DICTIONARY}" "${MGC_GZIPPED_BIN_OUTPUT_FILE}" - # need to work in 'local' directory with no prefix paths so no paths are included in archive - WORKING_DIRECTORY "${MGC_GZIPPED_BIN_OUTPUT_DIRECTORY}" - COMMAND ${CMAKE_COMMAND} -E tar cvj "magic_mgc_gzipped.bin.tar.bz2" "magic_mgc_gzipped.bin" - COMMAND ${CMAKE_COMMAND} -E copy "magic_mgc_gzipped.bin.tar.bz2" "${MGC_GZIPPED_BIN_INPUT_FILE}" - DEPENDS "${libmagic_DICTIONARY}" - COMMENT "Re-generate ${MGC_GZIPPED_BIN_INPUT_FILE} for embedded magic.mgc support" -) - target_include_directories(TILEDB_CORE_OBJECTS PRIVATE "${TILEDB_CORE_INCLUDE_DIR}" @@ -608,7 +584,6 @@ find_package(LZ4_EP REQUIRED) find_package(Spdlog_EP REQUIRED) find_package(Zlib_EP REQUIRED) find_package(Zstd_EP REQUIRED) -find_package(Magic_EP REQUIRED) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE BZip2::BZip2 diff --git a/tiledb/sm/compressors/CMakeLists.txt b/tiledb/sm/compressors/CMakeLists.txt index b2c694c3e05f..6df47f697e88 100644 --- a/tiledb/sm/compressors/CMakeLists.txt +++ b/tiledb/sm/compressors/CMakeLists.txt @@ -4,7 +4,7 @@ # # The MIT License # -# Copyright (c) 2021-2022 TileDB, Inc. +# Copyright (c) 2021-2024 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -34,7 +34,7 @@ commence(object_library compressors) this_target_sources( bzip_compressor.cc dd_compressor.cc delta_compressor.cc dict_compressor.cc gzip_compressor.cc lz4_compressor.cc - rle_compressor.cc zstd_compressor.cc util/gzip_wrappers.cc) + rle_compressor.cc zstd_compressor.cc) this_target_object_libraries(baseline buffer) find_package(Bzip2_EP REQUIRED) find_package(LZ4_EP REQUIRED) @@ -43,33 +43,4 @@ commence(object_library compressors) this_target_link_libraries(BZip2::BZip2 lz4::lz4 ZLIB::ZLIB ${ZSTD_TARGET}) conclude(object_library) -# -# tiledb crude gzip archiver initially for embedding magic.mgc -# -add_executable(tdb_gzip_embedded_data) -target_sources(tdb_gzip_embedded_data PRIVATE - util/tdb_gzip_embedded_data.cc -) - -target_link_libraries( - tdb_gzip_embedded_data PUBLIC - filter - compressors - ) - -target_include_directories(tdb_gzip_embedded_data PRIVATE - $ -) - -# Link to Threads::Threads if library or flag needed to enable threading. -find_package(Threads REQUIRED) - -if (CMAKE_THREAD_LIBS_INIT) - target_link_libraries(tdb_gzip_embedded_data PRIVATE Threads::Threads) -endif() -# On Linux, must explicitly link -ldl for static linking to libzstd. -if (NOT WIN32) - target_link_libraries(tdb_gzip_embedded_data PRIVATE dl) -endif() - add_test_subdirectory() diff --git a/tiledb/sm/compressors/util/gzip_wrappers.cc b/tiledb/sm/compressors/util/gzip_wrappers.cc deleted file mode 100644 index cb8912e34078..000000000000 --- a/tiledb/sm/compressors/util/gzip_wrappers.cc +++ /dev/null @@ -1,100 +0,0 @@ -/** - * @file gzip_wrappers.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file defines gzip_compress and gzip_uncompress, routines - * that slightly simplify use of the gzip compression routines. - */ - -#include "gzip_wrappers.h" - -#include - -void gzip_compress( - shared_ptr& out_gzipped_buf, - const void* in_bytes, - uint64_t nbytes) { - assert(out_gzipped_buf.get() != nullptr); - /** - * buffer format: - * uint64_t uncompressed_size - * uint64_t compressed_size - * uint8_t [compressed_size] - compressed data - */ - uint64_t overhead_size = 2 * sizeof(uint64_t); - tiledb::sm::ConstBuffer const_in_buf(in_bytes, nbytes); - // Ensure space in output buffer for worst acceptable case. - if (!out_gzipped_buf->realloc(nbytes + overhead_size).ok()) { - throw std::logic_error("gzip output buffer allocation error"); - } - out_gzipped_buf->reset_size(); - out_gzipped_buf->advance_size(overhead_size); - out_gzipped_buf->advance_offset(overhead_size); - tiledb::sm::GZip::compress(9, &const_in_buf, out_gzipped_buf.get()); - - // TODO: Handle possibility that 'error' is just 'not enuf buffer', i.e. - // unable to compress into <= space of in_buf (currently that is not - // returned as a distinct error from Gzip::compress()) - - // adjust to size actually used - out_gzipped_buf->set_size(out_gzipped_buf->offset()); - uint64_t compressed_size = out_gzipped_buf->offset() - overhead_size; - uint64_t uncompressed_size = nbytes; - // return next 'write()' position to beginning of buffer - out_gzipped_buf->reset_offset(); - // write sizes to beginning of buffer - throw_if_not_ok( - out_gzipped_buf->write(&uncompressed_size, sizeof(uncompressed_size))); - throw_if_not_ok( - out_gzipped_buf->write(&compressed_size, sizeof(compressed_size))); -} - -void gzip_decompress( - shared_ptr& out_buf, const uint8_t* comp_buf) { - /** - * Expected compressed buffer format: - * uint64_t uncompressed_size - * uint64_t compressed_size - * uint8_t [compressed_size] - compressed data - */ - uint64_t expanded_size = *reinterpret_cast(comp_buf); - uint64_t compressed_size = - *reinterpret_cast(comp_buf + sizeof(uint64_t)); - - const uint8_t* comp_data = comp_buf + 2 * sizeof(uint64_t); - - out_buf->resize(expanded_size); - - tiledb::sm::PreallocatedBuffer pa_gunzip_out_buf( - out_buf->data(), - expanded_size); // the expected uncompressed size - - tiledb::sm::ConstBuffer gzipped_input_buffer(comp_data, compressed_size); - - tiledb::sm::GZip::decompress(&gzipped_input_buffer, &pa_gunzip_out_buf); -} diff --git a/tiledb/sm/compressors/util/gzip_wrappers.h b/tiledb/sm/compressors/util/gzip_wrappers.h deleted file mode 100644 index bafacfc7cf30..000000000000 --- a/tiledb/sm/compressors/util/gzip_wrappers.h +++ /dev/null @@ -1,67 +0,0 @@ -/** - * @file gzip_wrappers.h - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - * @section DESCRIPTION - * - * This file declare gzip_compress and gzip_uncompress, routines - * that slightly simplify use of the gzip compression routines. - */ - -#ifndef TILEDB_GZIP_WRAPPER_H -#define TILEDB_GZIP_WRAPPER_H - -#include "../gzip_compressor.h" - -#include "tiledb/common/common.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" -#include "tiledb/sm/misc/types.h" - -/** - * gzip compress data from in_buf into a simple format in out_gzipped_buf - * that can be passed to gzip_uncompress to retrieve the uncompressed data. - * - * @param out_gzipped_buf The returned buffer with the compressed data. - * @param in_buf The input buffer containing data to be compressed - */ -void gzip_compress( - shared_ptr& out_gzipped_buf, - const void* in_bytes, - uint64_t nbytes); - -/** - * gzip decompress data from inbuf into linear series of bytesw in outbuf - * that can be passed to gzip_uncompress to retrieve the uncompressed data. - * - * @param out_buf The input buffer containing data to be compressed - * @param comp_buf The buffer with the compressed data to be decompressed - */ -void gzip_decompress( - shared_ptr& out_buf, const uint8_t* comp_buf); - -#endif // TILEDB_GZIP_WRAPPER_H diff --git a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc b/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc deleted file mode 100644 index a3384443fb24..000000000000 --- a/tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc +++ /dev/null @@ -1,214 +0,0 @@ -/** - * @file compile_filter_main.cc - * - * @section LICENSE - * - * The MIT License - * - * @copyright Copyright (c) 2022 TileDB, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include "../bzip_compressor.h" -#include "../dd_compressor.h" -#include "../dict_compressor.h" -#include "../gzip_compressor.h" -#include "../lz4_compressor.h" -#include "../rle_compressor.h" -#include "../zstd_compressor.h" - -#include "tiledb/common/common.h" -#include "tiledb/common/scoped_executor.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" -#include "tiledb/sm/misc/types.h" - -#include "gzip_wrappers.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#endif - -int main(int argc, char* argv[]) { - (void)sizeof(tiledb::sm::GZip); - - const int seg_sz = 4096; - char fbuf[seg_sz]; - auto filter_stg = make_shared(HERE()); - auto inbuf = make_shared(HERE()); - auto zipped_buf = make_shared(HERE(), &*filter_stg); - uint64_t nread; - uint64_t cntread = 0; - - auto infile = stdin; - auto outfile = stdout; - - // note: If stdout used ( '>' ), compressed data subject to being - // intermixed with any application output, corrupting the compressed - // output stream. - - if (argc > 1) { - outfile = fopen(argv[1], "w+b"); - if (!outfile) { - fprintf(stderr, "Unable to create file %s\n", argv[1]); - exit(-2); - } - } - auto closefile = [&]() { fclose(outfile); }; - tiledb::common::ScopedExecutor onexit1(closefile); - -#ifdef _WIN32 - // need to be sure in/out are in binay mode, windows default won't be!!! - if ((-1 == _setmode(fileno(stdin), _O_BINARY)) || - (-1 == _setmode(fileno(stdout), _O_BINARY))) { - fprintf(stderr, "failure setting stdin/stdout to binary mode!"); - exit(-1); - } -#endif - - do { - nread = fread(fbuf, 1, sizeof(fbuf), infile); - if (nread) { - throw_if_not_ok(inbuf->write(fbuf, nread)); - cntread += nread; - } - } while (nread == sizeof(fbuf)); - - tiledb::sm::ConstBuffer const_inbuf(&*inbuf); - // Ensure space in output buffer for worst case. - if (!zipped_buf->prepend_buffer(inbuf->size()).ok()) { - printf("output buffer allocation error!\n"); - exit(-3); - } - tiledb::sm::Buffer* out_buffer_ptr = zipped_buf->buffer_ptr(0); - assert(out_buffer_ptr != nullptr); - out_buffer_ptr->reset_offset(); - tiledb::sm::GZip::compress(9, &const_inbuf, out_buffer_ptr); - std::cerr << "sizes input " << inbuf->size() << ", compressed " - << out_buffer_ptr->size() << std::endl; - if (0) // leave available for future possible diagnostic need - { - // save compressed data to allow examination from filesystem - auto outgzip = fopen("magic-mgc.tdbgzip", "wb"); - if (!outgzip) { - fprintf(stderr, "Unable to create magic-mgc.tdbgzip!\n"); - exit(-21); - } - if (fwrite(out_buffer_ptr->data(0), 1, out_buffer_ptr->size(), outgzip) != - out_buffer_ptr->size()) { - fprintf(stderr, "write failure magic-mgc.tdbgzip!\n"); - exit(-22); - } - fclose(outgzip); - } - auto tdb_gzip_buf = make_shared(HERE()); - - unsigned maxnperline = 128; - auto addbytevals = [&](void* _pbytes, uint64_t nbytes) { - uint8_t* pbytes = reinterpret_cast(_pbytes); - static unsigned cntout = 0; - for (auto i = 0u; i < nbytes; ++i) { - fprintf(outfile, "'\\x%02x',", pbytes[i]); - if (++cntout > maxnperline) { - cntout = 0; - fprintf(outfile, "\n"); - } - } - }; - addbytevals(&cntread, sizeof(cntread)); - throw_if_not_ok(tdb_gzip_buf->write(&cntread, sizeof(cntread))); - uint64_t compressed_size = zipped_buf->size(); - addbytevals(&compressed_size, sizeof(compressed_size)); - throw_if_not_ok( - tdb_gzip_buf->write(&compressed_size, sizeof(compressed_size))); - auto nremaining = zipped_buf->size(); - // vs19 complained... - // A) could not find raw string literal terminator - // for almost identical error, though not quite same cause, can see - // https://developercommunity.visualstudio.com/t/bug-in-raw-string-implementation-converning-eof-02/254730) - // vs19 16.11.3 was in use with tiledb failed #include of raw literal file - // B) that (all data as) one string was to large a string - // C) many shorter strings expecting concatention were causing - // compiler heap memory errors (100+GB) - // D) caused warnings trying to do direct 0xXX numeric values - // end result format is comma delimited '\xXX' characters with periodic line - // splits - while (nremaining) { - auto ntowrite = nremaining > seg_sz ? seg_sz : nremaining; - // TBD: error from ->read()? - if (!zipped_buf->read(fbuf, ntowrite).ok()) { - printf("ERROR reading from compressed data.\n"); - exit(-7); - } - if (!tdb_gzip_buf->write(fbuf, ntowrite).ok()) { - printf("ERROR writing compressed format buffer."); - exit(-11); - } - addbytevals(fbuf, ntowrite); - nremaining -= ntowrite; - } - - // brief sanity check that wrapper compression matches unwrapped compression - shared_ptr out_gzipped_buf = - make_shared(HERE()); - gzip_compress(out_gzipped_buf, inbuf->data(0), inbuf->size()); - if (out_gzipped_buf->size() != tdb_gzip_buf->size()) { - printf( - "Error, compressed data sizes mismatch! %" PRIu64 ", %" PRIu64 "u\n", - out_gzipped_buf->size(), - tdb_gzip_buf->size()); - exit(-13); - } - if (memcmp( - out_gzipped_buf->data(0), - tdb_gzip_buf->data(0), - out_gzipped_buf->size()) // tdb_gzip_buf->size()) - ) { - printf("Error, compressed data mismatch!\n"); - exit(-17); - } - - shared_ptr expanded_buffer = - make_shared(HERE()); - - // brief sanity check the decompressed()d data matches original - tdb_gzip_buf->set_offset(0); - gzip_decompress(expanded_buffer, static_cast(tdb_gzip_buf->data())); - if (expanded_buffer->size() != inbuf->size()) { - fprintf(stderr, "re-expanded size different from original size!\n"); - exit(-29); - } - if (memcmp(expanded_buffer->data(), inbuf->data(), inbuf->size())) { - printf("Error uncompress data != original data!\n"); - exit(-21); - } - - return 0; -} diff --git a/tiledb/sm/compressors/zstd_compressor.cc b/tiledb/sm/compressors/zstd_compressor.cc index 203d1c0bac31..2ac2158b7ffc 100644 --- a/tiledb/sm/compressors/zstd_compressor.cc +++ b/tiledb/sm/compressors/zstd_compressor.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -92,10 +92,6 @@ void ZStd::decompress( ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer) { // Sanity check - if (input_buffer->data() == nullptr || output_buffer->data() == nullptr) - throw ZStdException( - "Failed decompressing with ZStd; invalid buffer format"); - if (decompress_ctx_pool == nullptr) { throw ZStdException( "Failed decompressing with ZStd; Resource pool not initialized"); @@ -104,13 +100,25 @@ void ZStd::decompress( ResourceGuard context_guard(*decompress_ctx_pool); auto& context = context_guard.get(); + decompress(context, *input_buffer, *output_buffer); +} + +void ZStd::decompress( + ZSTD_Decompress_Context& decompress_ctx, + ConstBuffer& input_buffer, + PreallocatedBuffer& output_buffer) { + // Sanity check + if (input_buffer.data() == nullptr || output_buffer.data() == nullptr) + throw ZStdException( + "Failed decompressing with ZStd; invalid buffer format"); + // Decompress uint64_t zstd_ret = ZSTD_decompressDCtx( - context.ptr(), - output_buffer->cur_data(), - output_buffer->free_space(), - input_buffer->data(), - input_buffer->size()); + decompress_ctx.ptr(), + output_buffer.cur_data(), + output_buffer.free_space(), + input_buffer.data(), + input_buffer.size()); // Check error if (ZSTD_isError(zstd_ret) != 0) { @@ -119,7 +127,7 @@ void ZStd::decompress( } // Set size decompressed data - output_buffer->advance_offset(zstd_ret); + output_buffer.advance_offset(zstd_ret); } uint64_t ZStd::overhead(uint64_t nbytes) { diff --git a/tiledb/sm/compressors/zstd_compressor.h b/tiledb/sm/compressors/zstd_compressor.h index 9db1c63cf74c..23a33bf4c80c 100644 --- a/tiledb/sm/compressors/zstd_compressor.h +++ b/tiledb/sm/compressors/zstd_compressor.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2017-2021 TileDB, Inc. + * @copyright Copyright (c) 2017-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -119,6 +119,18 @@ class ZStd { ConstBuffer* input_buffer, PreallocatedBuffer* output_buffer); + /** + * Decompression function. + * + * @param decompress_ctx_pool decompression context + * @param input_buffer Input buffer to read from. + * @param output_buffer Output buffer to write the decompressed data to. + */ + static void decompress( + ZSTD_Decompress_Context& decompress_ctx, + ConstBuffer& input_buffer, + PreallocatedBuffer& output_buffer); + /** Returns the default compression level. */ static int default_level() { return default_level_; diff --git a/tiledb/sm/misc/CMakeLists.txt b/tiledb/sm/misc/CMakeLists.txt index db755e822aa5..0e22dde09a09 100644 --- a/tiledb/sm/misc/CMakeLists.txt +++ b/tiledb/sm/misc/CMakeLists.txt @@ -26,6 +26,8 @@ include(common NO_POLICY_SCOPE) include(object_library) +find_package(Magic_EP REQUIRED) + # # `cancelable_tasks` object library # @@ -41,6 +43,30 @@ commence(object_library constants) this_target_sources(constants.cc) conclude(object_library) +# +# `mgc_dict` object library +# +# The file is used as a header but we don't name it .h to avoid it being picked up by IDEs +set(MGC_GZIPPED_H_OUTPUT_FILE "${PROJECT_BINARY_DIR}/magic_mgc.zst.h_") + +add_custom_command( + OUTPUT "${MGC_GZIPPED_H_OUTPUT_FILE}" + DEPENDS "${libmagic_DICTIONARY}" "${CMAKE_CURRENT_LIST_DIR}/generate_embedded_data_header.script.cmake" + COMMAND ${CMAKE_COMMAND} "-DINPUT_FILE=${libmagic_DICTIONARY}" "-DOUTPUT_FILE=${MGC_GZIPPED_H_OUTPUT_FILE}" -P "${CMAKE_CURRENT_LIST_DIR}/generate_embedded_data_header.script.cmake" +) +add_custom_target(gen_mgc_unarch + DEPENDS ${MGC_GZIPPED_H_OUTPUT_FILE} +) + +commence(object_library mgc_dict) + add_dependencies(mgc_dict gen_mgc_unarch) + # change to `this_target_include_directories` when available + target_include_directories(mgc_dict PRIVATE "${PROJECT_BINARY_DIR}") + this_target_object_libraries(buffer compressors) + this_target_link_libraries(libmagic) + this_target_sources(mgc_dict.cc) +conclude(object_library) + # # `math` object library # @@ -72,34 +98,3 @@ commence(object_library time) conclude(object_library) add_test_subdirectory() - -# -# `mgc_dict.*` tests are declared in this directory for the moment. -# -# See also `/tiledb/CMakeLists.txt` for related targets `gen_mgc_unarch` and -# `update-embedded-magic-data` -# -if (TILEDB_TESTS) - # simple unit test of magic.mgc embedded data vs external data - find_package(Magic_EP REQUIRED) - if(NOT EXISTS ${libmagic_DICTIONARY}) - message(FATAL_ERROR "Failed to find libmagic 'magic.mgc' file.") - endif() - add_executable(unit_mgc_dict EXCLUDE_FROM_ALL) - target_sources(unit_mgc_dict PRIVATE - test/unit_mgc_dict.cc - mgc_dict.cc - ) - target_link_libraries(unit_mgc_dict PRIVATE - compressors - filter - libmagic - ) - - target_compile_options(unit_mgc_dict PRIVATE -DTILEDB_PATH_TO_MAGIC_MGC=\"${libmagic_DICTIONARY}\") - target_include_directories(unit_mgc_dict - PRIVATE - ${CMAKE_CURRENT_BINARY_DIR} - "${libmagic_INCLUDE_DIR}" - ) -endif() diff --git a/tiledb/sm/misc/generate_embedded_data_header.script.cmake b/tiledb/sm/misc/generate_embedded_data_header.script.cmake new file mode 100755 index 000000000000..a5e4d78c4ecc --- /dev/null +++ b/tiledb/sm/misc/generate_embedded_data_header.script.cmake @@ -0,0 +1,72 @@ +# +# tiledb/sm/misc/generate_embedded_data_header.script.cmake +# +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +if (NOT INPUT_FILE) + message(FATAL_ERROR "Must specify INPUT_FILE") +endif() + +if (NOT OUTPUT_FILE) + message(FATAL_ERROR "Must specify OUTPUT_FILE") +endif() + +cmake_path(GET INPUT_FILE FILENAME INPUT_FILENAME) +cmake_path(REPLACE_FILENAME OUTPUT_FILE "${INPUT_FILENAME}" OUTPUT_VARIABLE compressed_file) +set(compressed_file "${compressed_file}.zst") + +string(MAKE_C_IDENTIFIER ${INPUT_FILENAME} INPUT_VARIABLE) + +message(DEBUG "Compressing ${INPUT_FILE} to ${compressed_file}") +file(ARCHIVE_CREATE OUTPUT "${compressed_file}" PATHS ${INPUT_FILE} FORMAT raw COMPRESSION Zstd + # Level 12 was found to have the best balance between compression ratio and speed + # but is available in CMake 3.26+. + COMPRESSION_LEVEL 9 +) +file(SIZE ${compressed_file} COMPRESSED_SIZE) +message(DEBUG "Compressed size: ${COMPRESSED_SIZE} bytes") + +message(DEBUG "Reading compressed data from ${compressed_file}") +file(READ "${compressed_file}" compressed_data HEX) +file(REMOVE "${compressed_file}") + +message(DEBUG "Writing embedded data to ${OUTPUT_FILE}") + +# Split the data into lines of 128 byte literals. +set(MAX_CHARACTERS_PER_LINE 128) +string(REPEAT "[0-9a-f][0-9a-f]" ${MAX_CHARACTERS_PER_LINE} characters_per_line) +string(REGEX REPLACE "(${characters_per_line})" "\\1\n" compressed_data "${compressed_data}") + +# Convert the hex characters to C-style hex numbers. +string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1, " compressed_data "${compressed_data}") + +file(SIZE ${INPUT_FILE} DECOMPRESSED_SIZE) + +file(WRITE ${OUTPUT_FILE} + "static const unsigned char ${INPUT_VARIABLE}_compressed_bytes[] = {\n" + ${compressed_data} "\n" + "};\n" + "constexpr size_t ${INPUT_VARIABLE}_compressed_size = sizeof(${INPUT_VARIABLE}_compressed_bytes);\n" + "constexpr size_t ${INPUT_VARIABLE}_decompressed_size = ${DECOMPRESSED_SIZE};\n") diff --git a/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 b/tiledb/sm/misc/magic_mgc_gzipped.bin.tar.bz2 deleted file mode 100644 index d23e06bc82aca192b974977acb1ae32599d28094..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 322448 zcmV(@K-RxPT4*^jL0KkKSs4*;o(1CcUyx7`K!tzg00;mC|D#?|1OQ@=Uu{aFsZU; z=xS2aB(;zd6jxro-sRREwziDvhz9R_+P3Y{-r9F#sZ^HQ?&Fc(+@|$zqq7BSw6AvP zyR+B3-P^e4vJ|N++j{G#tzGN8vE8VuyA5pHx!r20y^n6!QmfXLvn{uK4m|eu#@=_k za+Edic6EZ1Pod$l>T(Vc3E ztca?f#qH}SdFbWs*0?)84c_;hJz!L*D&5|mG+-r0&tAOuy4}*HI-q4PZRwGAumOd)<3gF4&Jc&Qxi($7OF?D$(7gB$W2|XPa%M>e#zA z+b?!?do*3P+hsRefYW=jy7#v(#)5BlUb*K^+uf~iTUiN3Rd!dZ_p+*q*SN~Pwc5-t zcXPXy-JR_9J+-HmIeJjiS~vm1lFFy*=zjw(Z@5-JEN7&i8L;uASZ47q)ev?9Q_7 zx4WS4XM0^a)vZwC$FmjNO?2v-G|;wtdg@(n$EO{ds+H-Qvvkr4JG*lfLv6PAyIcxO zuxe3MRqtKhuvKzrO!VE`i)U`#m3uHLBvm>$Oo$4t4SHv`Pi9+ZcREUmF0-Yp9@wNR z6yDc6-uGtR?>%R|j(52D000000000000000000000005)ddcopWy}I^yHJe*}>$>#owTv$r~%Hruvt?(XiLcL$w9~&?7hx5+hXmu?qTWN^Nj+xx!sz2 z3eLk-+1+<;_1^Q`JIwCgYj9bQO_N*O-JLw_G`oAN1+A}K z9?g5b**)34X@;rx_pE#GbY9wn-R@-b(=FZEmuuc`(9~HdQ%iTe_1CFx?t+%y_V>N- zc}ZTqb6vJvcIfX|-p=;RR$$wmS8lZ0WZk%zrm1_qpk5ow)eg2*;4ej?cTe^dCy+Ya^<}pt#xXlS?(8iz21(UUhVGAz1DWO zd(^ug#@pSIt<7$`ZQGvR#=JEfV!Fp}bDf-b16tQCJ%+p3-u8C)yn9b|?Y9bYsarBG zJ9Ocq`gPd*xSVslr)S>xt?kR+d!&x$aJ8j#d$VV{?r~RZJ#Nco*SoFa+C6tSymu<@ zo!#1Q`_D%bdr{Rowzk^Z<)zu3lIwSO-qW+z-N$l`(eBhc*E)7{Cw1-6?!#WG?7Fn= zw63kAvEALfI?s5^s`k4a_jeADRoezTy5(RC^0B>mILW^Lsb0 zyLki9BWpg z({FCw*S)RHT2+O#z1!2*yPR~_SFUen?=`#ECo|sg=eF~=EzYgNM%Qn9-FG<-t*!3f z&v$LFZtr?DrLnu7TW)%LUE7`8z3n?U+Bm2z3$zL_045-Oi{@24-{`@dE{=+R008~@Y0ukf6_ zGs}FF{-iu@{cfP7uM`?fRz>hDYoe-L{1K?`?_03KA~W|f6Yck4=m$Y-ESnx*Z0+cI2hRw=%E0c!vN*3a+=6iB=`A*e zp5jeuY8%81W*&!Ctxd~zCffq9ByGPKY9-U8mIq?6--^xs)drUv_X7CyNt;v$z+h?` zW3snFnao|$7=sNV!RQVJ2ziyu)lD<1LS8{GxGXY#)0VbL^CLoCX72Rllb5(*_Q6nR zEs%UXOD)JkCxJb~Y1fI^!`$O@u&_};t0=z5hr*IXOOk6lYd!BHf)G~rg)Ks1v&6Zc zodYYbAi0rq8}KcuE5W?;SnqNuj@NE&u;~F|hiM5_Yed`(6Js6AGRC%voJl+yHPRW@ zRT^s_L_lb@Fg4A#2suLc8evnpiB_14zO;B5povj~ngl5zQ=N^dHhg@g$e5}k8{9to zBzVP`5G<)P|evep%WMc(m&CffA<)!T7H_6O{73aO8CkBFAiWD?)kP?M_Z8l8_uoXRT~wv+Q(& z>&iU#wCQmRNJ<6a+_XeA+wiZT_#UT(7i2m8?ei4KI!oILPY*cHsyXIJZBR^1UkpiPB{Cgk6~t<TX=hC(=7|Bm zIbe4dZVQ`MJ>Z=4qoC=Ait#4F&TIQfZunw!iMX{o$Jliz3r`#p>5xbQ|7pSRjL;G6 zgGZv%xvUSWxU{ZX+S@#1wBjw?7Vs#n1^CP-$Dbwb?~HlVst3sot>7u1FhGT~u6FV{ z?~KSo%W2heBCQ+3_F>D0My%A%0t0<$cK%_7cQCFjcsG#q!Gm#Z{Pw*}S(4^2?~{ZY zR@?|d5n51o$neQ*)GVHEojj%r8MjYLeKw&uA-NLRT-)IY)NHd}$A`k1^BVqx+8#(e zSz=BeeWnpKCng*%(JX`gxoJzIE9nTJ?1rJ_CQn`Y9Km)y-4QxN7UX&>d?L%)>1pV+ zE6C8S@I~r1$&26~WNDD;LSVjHcSvyjzVoD537{enU?BkCUQQQUA}mFyBza5bQdd)? zA0?FltMjYhXe{?C)_r5=^YNTu&!Ko7(g-44@JGa9gW0;lITP)hgiUDCcEpPMX^>nN z?RfKCkBCdO7sp-8uPSZ^1rqkPo}TOl$R#s*9b?{en3YI|e792KOi z96duKrV;8}FpPPFB@pB$D`9cyFsTZWw=znuMVMmY>OWokQB9?YR16FZ_$YXw-<<4O zEkUk#nNqW*!F6_a2=2hCbZ6j);s9&~%G32HR?}Doz=vSinvCYyTWZ5uN2_$3+B_zl zGfvkv_U zQ1wAVpKU^L5`HAp-wePjCY^C;{@x@Rf~&?sLA`hth+)#&?yKK5Fq2c$lSmwDJLHBg zy>+b;oXfoNs%Qbx=gkgQE5cGVYz82>Hb83v?J_}#;5PIWax;E{8|uYuSP)jcLX}9M zoOx<;HF>q_VeVFMcicz6nm$)kNDB(ol`?L9aTXnQUk*PKTaIx2EGZz;}i|%VOVUr?4P4lAfdn*EHf$GL_O77)6a`j^hE-YDh6$+DU00@ zD{aK>d~ks#2988R8ccDz-*TIE?7*?}IDMv#zYmsso>a6Esv)=!lR!h?3GD_8(KIi;oIOGNgc}F9uY#^6#4M12|5> z$y@<3_aPbdprGZPvJ+_ox9PUruXg_0#VIFQ?wKk~Vzp%abWG_Nx*(r5S4QOm0em#`TU0^ov@=V(8$ws3&}Q zkcqM=#vdbD(XpJ6V_9-JwZ}2F#31&juO4B!3VvDV-(50^Si>E~j}!JsCJ@QqFH(|4 zfr?n%wu`7b8JHpr@Rm~&+YQl4$wdi;5nYsp85)~?swX^f4Gw3l6pYaj%TXjEwwtNf z)J{hFhQb0UGUQQLjuDr;ykvBvcl4ztLW{47SSmP zBcnz_vfUK7!p=~XkAT(!2!qcbejOGdl#hOtO zAnIdSgD#B@A_3M7SkuZ=mum~we+U(OPmk@y;y9|n0 zs)XqXq;R18xb3~-*yP_m%Zpb@Nj1dTL(1*vBdYeIG>*KU@P!~NL>C>gGl2bZlZLNg z%)ld=Pf6jq60E-AVu=I>rUgtX@qw#K_-w5SkrC))Yh>}$Cba|-48``W#z$gOAqhJz>@K{VHJQ*}^WtH>Xu!GL3wW;zjFYr_xq#-l5R%^W zj`USuDQN3S3>vLwhPa@>sG+RKlaM`H;sn&vVLD#Hu@0?mH!;RBkSkIG0pOmYO?f<= zbHAxUsUx9f94QEA3|=*)4jw@(LDta99R9&$u?^d)9hou)8Ca*Jjjy51vrh)+jnFTU z7l>p=1~3<$lIT%KD%NNtLqIx{6?@H>5CWAtD80 z@yMLWON#5K%B0p+y8)0!XR`6(uo!HnPKsU$UrWft zp$;mT-hBPG2L$O|3+Z>dA}>eDrrPIa&WO1)^eCxVLFH_61Egh*V=)N$)>EAqN!p`!lgX`j2Sr0j0&?^enSxlsvKz)$6 z#buV(+>R&V&ROvw8A#a(NT~>8k362FOU)rnA>oAkDmW*->Ar-Uc$$w2Gmb>m;<>*7 z)meqaXSywv;I8cCj2A%ZAWyu`u6<0Ea%BjqLUV|_PE9r1C3sYpK}tng*8Y_TyJHJY zlPOj-6GfBdpuP*izi=rkSwi|e3FVV(v}jc^6+<4mz8oCW^^DN^a~}z$tLNYi&UF)C z-hhVz>yk|AbLP5J>4ZXjHN0LY+e>oVZ$epgPb+U`J5LWPBCpz+5XpGLQ~ zH|T+8$e8VmXYyf|pCqt?nD-=Qg_5ldeV1@xxP;dsbgXR#^{LGa1A87xwQ`0jFS7XO zVt7hzxI|NnOjJQkI>8;cUMlT0WjQXG{-V3U>A_jmO2t+Zgy&qJaMSP(I{1-vq-qH} zfQ_@0wL>h`k3>saQ*@kZ(Vss1gpg76=o0I6dtEtjJ=*l^WZXvnej2_ zpL!))#M#PJ%ZC>d*Nw?Eopa{(NyL7R)P2* z-IZBpi4K}eu2Z{?r<(+HYhfOQ|ensE%j0p%k?aaN*U39JQvZ7g?6xom@8E z9duGUK?T({&%RB_o%Jl|fb>>0p48(mlt&CSr9w-SbTxdSKvJM3yQjAIi{l>lhP5+r zkg1DGmDenbN(v5Y3RhKMj$u(s3WwYn$X=V`sJ$=fhPfPRy^KdHgD#RDBr?V$b&M`}Zp1nr`@%Rb zlj0y{5tgONh$B*y9w<#_GLqpdes+NLI^Nvq5HGAUM1V`hLWW;h2q&fyJ=l=;Hu6() zl_M?79&;PO5`z)nuW3h#DeGs-zJMcA=ZcNE)A&fFQt4=iTr0-$L8AWx(O1e>D$t~J zk88vWfiB}D)h#%(vJ?%dEDU@LhcAInJ-(9(?`4(R1d~F!t}vS~yQF=3SwQ&7RSmdj zKM3lwT*m={)8>0>oUR``Ra|di+7eqgvxJG<5kkoUPN85ersc>9#^WJWP^wCJ@D`3* zBH*dB080&toVo66MqR0cQ0vzZIN85VC8%1?hn-2evGAy=++?k>J{zFqu+lZ`LMy)| z?JQOku_}>G)|*p<9-L2D5d_Qlek>R5G3t>u?nQ`ZY#wQIUt+O&_M9|?T8%|?Hl2~W zkQ}$SBo8v}GGy)#Ex*^2F+P(#+BG6n1H+4@WKz2vxQ{E1-MUL*~5l z1>a8StaIa1cZt$sy~NQVm_tEgUg(v;-CucPcJ#4r0CvyGewheLXGh^2BWNh&yOyK_ z)6NeFI(f#eVHxcMZF^1Cj3!&J3Q_X7jaiir3Xo^MJj8lbv^mO>XuCfJg*z$5C_Tnn z)KtM~kt)O$hoiWLGd59V6-^q4WKCwES|-~li%%m5B5ZwagWDksb5QI(Dn3-%n@X4| zbENQ~LTSEm!P$a!$QaPhc~2?H&TP5eol05P#$14tjyjtzf;J%opM72p7_xQ)%B{E# zV5afugPl~{EL_kv_h*EEX+8!Qo{a35X_7ZMGkg4lH~n1sM5YIfg7$=9o`V4&FX_4cqdFmS zpqMdhI&f4^L-egww<*Su>quzfA*Dlk1T{=s&KnKgk`G1^?mj(6-qK9E@*4UX82#)^ zk`2RqOIuBvl|AkSQY2mseK5JAuU(_itrg0KVCyH(hG)Y^YryAhA#mI+bo~sXH>GW| zLAq?wPHGz~-wS;-TBj4ue!EC+CbS+Yy$?0*wDM@kKQv{n_on0~6B%vtDcC`XG(u(C zgm`@5>N2>e!b>7sZ4#}P2vLyZEk#RQ;;mv%AR5R_3zVFA-N}6NIHu7C0t|rl zurc{W?}f}HTU6XF8gk{spc9uDBa>9_NTodx4TO8u?1O}j8o`Q=Iv1uAq{zc5aIwsp z8&Z)$1sJ~3^s;Wa%#dEV#ZVsiB?LhQikU4}0z(s_d}2!m4x~(m0)WG=EW?@4WyCcL zTWQb-v3~ai+70GCM6V~k;Jm}4fI(}awdv|>c7kMk5&X2pA|Z zx%B2Tj{*UBc5pijUxiRGdD?CLEs=T6FXg=tH?KuZE}_IOzyw4bo>@`&H=g4iEH&HA z_l{&`K;|^C1ky#zRG$MjS_6S}75} z#Fvk)bBjKC6!VXcV-gO!CMGHj0>K=q({+$uobx(TK#Oev%&a826FPhC7E$T4=IWKuqNS4&1fk9u>qsAP7b z2<8uHNZ#8ALol@2U)@XC6HXEjj~5~W{HXKd-F&N&v8f#-;n?kPAp=A*^+pl!!Hn3( z2dPhbmB^0|j2`vEnhL5kpf8x4#ur@Iyhy6PC$#tE0`%qxg!e8u<@~gqXgO4r-_-o+P^Bz8xIN@@p_Ft|8d7nWrQ9|v7Sp*-r|<8V0MWL5X7H@IwOGNwTiB8AJhK%gqu1o2GM>ftEm z)uA9(Tm-%4OFy3;jC`L&@dVf2yj&aK9H3h<~=OPVwIACZ%yMV_KHf4EObIQ12JuulQ#15~SAm1jaOqLR=l4U$G;v?KHqh!4;< z?V>2PeJ89QS(7X@4${Y=TqCgqqPs(XA)Oss0O2eW*#(4+TJ5yBcvwJ6x?@(F63wN^ zhTvNI>S&ZUi&eadQN0U8-XJ-8JSJoW7tazrrWY&B&Xb3_&j_u$?QhfQ%f4$@MBqPX zaC@xju{vbBedN9HWwfm#X)--3$m4)G+8!|^Lm5XY{RoHcDK)+8Rj6xJP$-n3av0~< zdXse$$EHhDP(Wle_fcpJu9Z9v5EknkM(y_K5fhhL>x=^}@rSLDS1{kMU|FEO99yK~ zt`E28^IT*IG{f0J)UZRBYD#qI5oImx-?iti6x9%e+eL28LubO&+t!n7=s|i@4vCdq zVCm`w2eNfF24juDA5Ka?R67*YQ=1}N zr`njlXjQc${x&-sRFTf9TZNSgV3AAlvbHoft+?{ZuuV$jyCetB69INF&K5mKE%OvYnT#fxB>sHVdD0JRZ_dEe#3MCMP zJ-CCbf+WVEWuWRSkRW&mE}Jnp&i`K0Nb_(qVcryL!fO zuaqw7p;eY>-Vlm}SRrUlh7^@$k_2~WklGAxA49f`aSjeo2~{bW+)DC-kgdO45vPgL zH3=&Q4ZC4k5uy?*o5KS~r9*La7ZkR0}?#(3;OK*+I2SkK^sO-i}2w^0n%{G_ZTe# zDcRqMaaxpa?--|?5#G+YVbDxO4Dmf)e#X3@KUaXjNHpo74CtT>C=$Ldm5Jz0!k-~P zj8H|>*{j@044v}Zhbzn3 zv}0V`4KL0su5Wv1F9zyt#7MXS)F=W_994 z;8ytr3Cl0-yr+zHl%z}rtm{IOht%t{<8;L*J7cCrEY^ATEk{*VyXogx>$FE@Xx;c| zH0^|&9}yR9+w@nxJI}m&Y$fyCxN+!EV-}Jx1ntje0u{~Ay#WDh>bn7#s>Ea-U4zlf zF^cjCc{MJ&MHfuc*t)p(M)r8%xIJ*jAjNH~P-Lm#m3M~dvD5D@aQ#F@WWl7lurCZJ zSH0&IlGWg17@H-iyBZ>4qzjK$W>h6N6=0dFudPt8qMGFF)d$_?&rIM7wAINHv>0)y z56`e6D#l9lL>j3I_W<8O_F;sZ2Mh3mkr@GkTA{Gs#LN)u$Fjpk0$jN}w%@ng$3o%fFz2Ba z0_gdVFo~>|O4PV21>V%aYHc?ptP!qP&6(0-1??6~Ydd7;%In3V<#mA?(L1$=4>S=3 zxYGj)a8!VYUV`k1_(H zn|bVvseqcQq}` zKw-0$8VZSVv6d=$HQcJ|*AkfWc20}hJeW>aK*QvJwEz*1^l+il`psy&#I z@Ph~_8E`Oy+{Y}qgcyX{EqI``2TUn=_}-HKo@k=!Z70*&p(lqH=R%szAQ3$hZnQbe zciRoc*0l<7=&Ks&+o7E(BC_L{+;@oz^ipQ319?Wab;wYp0%Kxa4^J1F+Ypc@52|;& zT0mGR3n1hPPiH%~PbaUTe87tochSHqCL*;l zK%DuCghU7d%{<3#_bo?UN}wD9CzlB?BioNqZF7N|fSGQh>}7&t_mdt0Hbd=o9`96t9!?d z>VOu2S@h3}hK1PFV3Ozr7F#YC1kG$DOf@;t-Sp_)t~D->>52>5T`8|o>^HS$)UpQ5<06{TFPgBUFp(>9hVR9r=1 zxPn2Ov|0F3Jf7&;hVf*@k$CW$FM0ehCCr+&dL5TsIe}e@w-KR4z_B-=duCq5&aptC zmdNGmVV@eVa7)(t~coV3}WND+FR*Wf>pN{cZtuL%d+3O*h6jF$w8;au9m z(E2`bHPz72rY=0;H3%Zutc~)J&5$7guKFJJmg=;HY_Eo4@%hWHg7MnoFNl7eV|tYA zQ(DxH0-ZH7^(aWflgd3KWVuKb(feo1!6nHO7;_j^x}(_`T*VM&S{G?{J@p*6z7m40 z2#$0%S6cBVv(^>PTQ%f{xI7OFpDC_M=HS^|-mQ?UAeYpjbh*LcP;_B1skwxR6RY09 z$EE6NHY$Y$v?WHdB+!n^mU8Oa=qFFhKBu5FoT`-B%?L>+$L&foU6;y14g&&uqVS&w zZ(TUEcs)W2MyQ;GQ~1TIa6sg<%-uT@11#JxUfDiUztU%_aec#mD?!v{r*R<5bmVX5 zBr`l383Q+aS*hLILNd58lAyd;WDSZ%svB-rlUFcU#_FKbuBWml9!_gV_TCT^bt_IQ z>$w=J508+GKP`uS91cZn2G-|lPaS9knv@hCrz z&vT>-Qdx#a=)@3c>DBOFojs#GKMJ7j_Uh36eDV3XO#+KTxzuBk+J!A9S^%rghDJfw z*-o=mx<|C-YYrGo-6n9csD;qPz0oL=ZlHwPnM%`CycQw0tIdV>sZfrFr7ppXJ(ke~ zS0{DD;)MKTdLX93LY7T*;hwLWwwk&wSLawrv|dLM(})v^m=h;W+EA!!T``KLc6QV) zQN0|3`~wc%q#JM|LDJ2{)!fxO9%HWQf$8C2Xo0!qO}=%J(Vte6bWM_RzPdMHX6%$~ zC0$JVxKk3Kkdc6*?r5eC%~L_Ri^4}KTm;fsxQcA=UfBt zv|Tk>1@8sXh&taClv); z@&$Tna;oFkJ7^!6A_2Rj^>xw_i`b@9QJk3uyfQsYbDFe576mK$cB`Z&Ms`eE&dll8 zm2@!?cusH1=b1$2lA%F0U?*3~gBLkI2(6v8O@2~y>xl`any*}zU7Fabw3GaA>Af$$ zp20I7Y=xUZC7%J&`Nin_B{~v^D_INO*;HN5%a%e;Rw5ouflhuvpzY-Lu9fK)W#oB$ z=U`kpFQHD$9$@R`vEBuRJ;zlvKJX^Tw*ZDmYr1nl)`P-YU1C&lmfb{Bj<6x-fn8ez zvAu4M+VW6*@(3fNr;!F_p6DT#Wf=~#Dq^GUG)q~&aNEX|84>gN5tQ-OX4MKAIn#^c zuqwE!;Q}sZ;p`2+xL{1PZjM-Uo2;fr!%$nm_kP9~bG@ZH^#@^XOq(gRgI8hzxx_o( zF*4YBaa=K_1YWO5aN8QgKAr0tw~|^_mRk0h4SD-l*gJl%?o%GXj}wX_%PX`@jlN@4 zW5*7W%-j-zSPu`ywZj;=Evon!WVgDLDlrMH~#JmE$pn zma@d$F=AiSw+0a?g`);kT2N*kz;+~?z2;NUIiVzUS9ib*hp zs4$ZJi3X-ry<^M|ZrK>?8CZa56#NYHShRFE zW+yfz>=JMz9=Behdl920Y()_}LRe&LwUwUWlpDobv*e z2Ny18xLDY3c!SkJ$7no`B`;0ONp3`yTJ;*ZE!$i;<9y{pu6rYezD41j`3huNM1Nfc z+z6eHXE?&^dQ9YnTYl742Q_#O zxipVWl92eljv;;K-oEDE*11vVet;6sHJj+r+oQ^sSDO|7tL;A9RAh*)SJndK-NjSW zeCOoRjeexBKez{W>`87 zM|*Z&-Ypd$i^AC1V(~|)`bg~rl?+AeIDtHV(Au@wXgc8m9;!V1>Qp2WgVP4cBzBn}UQkMrcuSu6l+_ z(VG$g;MbBom(soB^_a$8Q`VnTmefwjMo})k=5P}O*sY$#IzfH2$0b`la+rXb$>wc; zHF7t0RLB~NolXR%dqQGshV)+SuXMM9KF!W$VbmU=yvdpAg1|`_PA=IH6mv3|`qz)U z()}3?Tq^TCf})tARY||Tq&HM(ARmId=gf?{3Jc>{cw3&*3E{t(b6PcEAzE7DVkA(& zd0Mng`b>;A&~3rY`RuB-uV*hy0?eha;pA~M%<#>;Rx-94)go>wrS3}E+fZwfJCBm^ zK}~$P+3v~C9-npBqOY^Z$&WfNxHSgWgB+#qv9f?^-5zc~2$thpKFW3hJ(ieUlgEU3 zZNiur=nz-}nm7|XQCzm15Wqyfg$wxk9V6Co1J*-;!#S~roQG;cC+|uev){Ox+b?zt zLdT?1n*{Q%COtG{)7@R-5L}ch8zz)Gx>;W|69G6(edf6)1ft_*yavYuf>jP_ZTGhi z)N!s6$h~|Wwy6W*;7s}DHzD*Tu_gP7ws?CkBNvd;-4_MLiN#@)5YdVg>=l?r?9Mu}NNO*1 zkWD6?UKAG)TJJVaUa4)&+FgeM#gri@#OB>^Nh?L|KFOERL~mk_^-%G|cR#CwFf)(B zjW)pb!+c=46qKjP{sx%u3hP&&yAFXLYX_QaLn?Idn@wk}AS29Jumc>*1#R3f<80Fy zd<7JFZTH(mLU?A4R-<5Jx=DU@DHY43y4rJ5{n(1Bq4CUgBjmllNW<0{M1dUSj;wB3 z+5!|duF!Szij@8FK zhq$U*(S;V|bhzXQ>U%*>`T_4Ai9Bsnv5|xZ9J55p?gSMC0`w~izNGXUJF&>22wdU? z>d2a#+KBrB^&u>cd>0b?<}o@5Ec8CmBk+@yO3idPVsZ^jeEYUnohtds?mVlty$6hs zL1Gm2V+hl6d3^2BJv#6Z(zQHbXYpyXyl$hm0j*PA=?`Aa^ zXdFQoMvV&!zNum0rat<9tbGpph&SFUH9y4s;7&eY`@QsM% z@?)8!^^Zb)sLzAQDQh5tM#**(7CDM|amD6_d{RQsQo4rJywHsw2up8QrbtD$gPjik zjSr;jZr>P%t$%9FBS404S7poV+NU3mLEfd1b%Muy9{6BMZ4CLfGE(1wJuDmkA!LV5In)_cQ20Z5`xHcroZ0zn|QF<1rdNjXdwe#O{L6BnVJOjsJ3=^ge3iXmm-8?)N2b` ziXde4F5dAVv=ea1kg9hXeFcarhBqMxzMfyr;hcPBB)0JU@bO#>{G-EbZe8O$;?SFX z2Hh-aLXIw;kvo4_UmtkI*<|FZrXA{hD~MY=Z+p2^H=0e9%#il;&wBY3HPbszFsp5s7qErqd>T^J>dYuL|Y;O*mk(=vQ zfl8eE4^XaO8y+q+uGeEVy}`)HJ}heo`t@iqHuk}WtELis&?mG|67Id!8g;pbw0n4D z8hd)a58|Hq*z$Zrc0+Iu&UtovsMc*@=Ep|HXfV+ICU1P~?Eb_6$Y`^KyFS~KoT(*u zhtdi(u4wILj!qVeS1`t>o+$M4Xh;(E(Lv!mb+fWMuqsVk^F+vLM0t&qwzvoxy?yRvDFc#nd3CFzOc>nc*!i1u+pBuqcyWn3hMsjuM-mT3Z4X8wOT|j#@XuoBh~22 zd5vD8aNCjx9Y>3bR|c|ZP#gA|DYN6vltQPO3nX%JdspaaATYPSDDW0_UDuw{Al$%_ zDRa|<*<-=FvQV$n8`MdqF>jBH{FN=D%!22*u1WKj874Llor@0 z-gOZfD93A+zj{b(8%fma7AIRQyZUu8bp$PSkwHu~wS>b?29{o(BUh#Z&3Y3}%a65C9vB>ie?4^?la84v#I21`o}&JG zf0S}3-7#a*IQ_J@sDSnfqM^69P8F7ZdiM(WYtp-*MXOv3J{L_10q4k@vMPd_E+ixr zt6h;1$dL0npA&DO979l{gx6NLX*l#4NmkLN&V!t-Tc}}FbI|M5gqx{nu+V{ee7pHD z%3j1*4Yvm8gP6=+OE{M+m{%BS_PN2|#Gzsu)~`|zm1pms>L41^CG0$hPct5dSAJ4v zLT+|b;)LpnBfF2N66BYc=pju|?X^e^FAPgASmlf!>`mVa%8Sj=T6#px=V=bBpc`p` z*b!hsiUtL(+}&ZD3F@wd?_tMqK2I~ct1BvcnF>`k_X#?eD13|G(YDPEz6oBdi!A{R z9QgIC%OYEIr)k{j$C$=Ur)WiyS8LVlXj9g|hNCX_xDd6N)I!scE!3gaMHXsQ@%S*aw3-z5eLeyVWXI{LrA_?a3&5z9vRouImt0}n;p%- zI{W7?G=o6(SC^(rG>E0I%i_j%I&+8v5kqH9Okng;KU&&9ydljydD8N3rGWuEh!J7b z#Ed8EJ|~LFd+lhk#$Zr`6NMt=0VQv3cq+$+@f#2^44jVnyT}PsMpH5v6euXkBYq z*`vmRZ33_*NQPa?EnqPRLK1uob={UdYy)dUC0>~91vBWC2F+IWK|6J~q-@{Yao8E) z+ugDSosQQkaO!49S}bw$j^65#7hyeE~O zeXT|A8p%{E-X7y%9X-vFAj2CHI3V!E_M=H5nViLAo`e?alC`qp*l5X6ptkcx8ZxdZ zD{`|T!e_0J!3u|I9*=r#n3AGbf-&Amsz$-mnb~>$QU*fS?oT3SjMm^oxwho5w&j7} z6yRlu7pP@u%#M`h)Gw!AA$O)Qq(-8KYK@g`oZM z=#ZP=O$s@Icp-a~SWJ3Up+TEt9tzx2P^EkC7W(FO;sAHg!V^bxG>n*d*fRq|)xyp) z4^M|2%3_)Ks*{bL!OQX%m zGR}L^y0EDo)aJaFO>ZF<^>d4Y>ttuy;n3^2D1NzV!~2VSZR(SLY}k&5!bsD%5X zJvSvy6S-?-81^M++&*l_pcAOG>$0-(@sSlL79FqBm!e78dqE9=ubVl+YTz@VCaaFT zJ+k+mC~I&*uXM8YGq?DImdvuacBt0C>9#L9^O+o22X%862g7IlFtti)Q_T`mSZnQiCSloSE8)S0pcG)VoSo+A66i4Gy@)R(6f_E*BY=EO~ zbv)E07#ua}8+8YNe4Bt`x5kqpwbKS}@yB5ImXI%Pw~Ib)(o<-zE>a-%h4@bAG|X+- zORyQG&LFVD0bNs$>H@&t%;*}io*bXAJbj*O5yl0p@LB~ZUpBQV_fi?lh-F4_^o=CL!3Ut8&5_URGnR=NJnahZ>3D;K!jaZm0&ivXYmOvWZ80 zwGiQEMSB6)p&9ESRj1kNyz+7N|5CPn)qKLHmswju3LOAn#-ZNGV2Y#2S^{bpL4<|l z47eyZia4$vL|~TuekG8*gzdS;u8;3 z?$?5HUbyPaT9!PNQROqE+)ndV)?lP?0!c{YLg**e_C4Qg>2OjNJ8PK-b{9T_@W)`n z>$*?43QwTc`Q7Q1>4cUPh*2v^Udo6`BWsHmvR>e0E-tVZpyx})B%-Lu0f6k`=SNT+fSxWK@kyIPsN z*yq!@qwRGPYFfDPmirEn7YhB^mM5Q1){Z&FyW<;+)PyG9p7+^G;MlCQr1`CIT@z=a z#18q=a5g46yOTx9>ON6;4;2ovtgW03rK@bKl=6Xv2r!U=17!32vz29ixmk?9Wd|1s z;s#Bdbm_wJA=id@_6&m%ZhjR*4#G>ZgAJu@<33q4G%PS{Dw(@duvsrJGN^_S7VrtU zGj(zZFbtPg#5hjNrI%zqvn6IpeK#CV!{#LgR}a9K+@C}efejKCpxT-VL1Tvj3b%ky2n`zS|%9IS!JK$$wy{e4k5 z6@)&uH>$A}vZo5W+>0+hgosZ*4?y$Sj^XZ;>Z&>lP*Hn@eSG3{^nh zWZI`qBY9;IhmTjDZ90Nvzd<2S=49I zdJ@OSn{6h}Ud>*dH=@yZ1wOja>k=NDNwH0-I7gNZ?-gU6T~4dB=3A`na++W@`w8G2 zDtJ*3l6}zjcMG~WI%K5?mQ7DLeR7OtLB69B;E+yo-NlgL7pM}ptYgXI4z$v;t;3;` zexug3jta)E&AW z=xBjdPMN*%8el;^p4*6+4UQDe8jJV^*{8bos6~i@c=~{%HUQJ?A(R|+&TMZKP1LWw zqEuf;KHbLEDTHRWFcn84p7>ZfJ><9bzQu^Nvgc`FkVgqNsnE6^0Z?f92-%wnQ}1lN zP(87pjK;A_`rv?q!8gG15^hU07YHc@&C9dio%BuCm{=lnss+BoyFMJ6wanhqYc`i7 zpS>vE;$!ZexDO7?F%j6)@bJJyW)%!>k7i3bwi?!0`?>6!xX(2~7|e#v7&tO3ZMMRF z?ki&KyIJmKYFkdc@X)x?c2mM^lKx`SeE_$JGO6?ft=Hqfh4^zBZ8}uFi*&v3e7|RcKZ{??TfrBM#$+-Be|S z&$*Di#SO5?u{{kby%M^*1RJk3IVj4VyFjh5K(x~A1cW657`O?9iM zYE&pdHE7gtE5#&3n%!zo8uzKD7*5_e<;RPnn6A{BfXbIjnvSq-daz>95>;84nW>qP zJ*E?b9F`Y0aUL}`$82U+G|t^$F&&*JBXP$UhEptN!Z?-dBME}g*bzcUdxH}?YL;cd zn#+>b$QFwbaS65grPv&sU)GU3YHR97R?A{C9vlu7e3&l65F}Ov?d4<1Hwr~KD7o89 zZSv_WsF`)2bv4GNDvV>;;kE;FA4x^NHk@eJLGJ8kxOGzYsnu@8w^drSjRqKOZlJ3Y z6moNNAA~uBFssJsJ~Nxsj~xjU2EP{sEKB`By#`5AK>`Xq;+HYp@s5W#E0+kV+%-j$ z4W<$4mp;)l0;fna_&tmmW27Tn9OgfHcHcRYk-5=-3)fA`3E|fPJx6v6>Pg>HT;Vsc zV0ay?Uh$^bBtqL15g!|#%*}7a1{Yp2=iPR>R-Mttfa<<{xPMQc)a9wrcCQ0XcT2o} zvWDa3z??4gc3N0o#b`Db-P(H{{CGm(b;0Qh;a{AE_^zj;U!2JELZV=@(uI>rxd>|c8uFfJHAmqa@l8q+BN5xVuJN|1BQoc(+H$vCiN`g< z-e5$E8y1ky25w9l)tHskc3t*^hn2C*xg<#xH<7h>UshgJ880if5Qa(}nmMk(NM2}AR*IJb z5-Z#%)CSeBsjNm2Sbbfx#~AW%EF~nEcHS(y9Rzpb=*AiQ1|(!zO?ZBed@3D|E-H7M zhVFcIL>R`Uv1#3ag7wzF(3Y1e8>zNFBx3ba+-Hsaj);mb_^<<{>Fc zV&Bel?M}4RETP`iZlcSZrz_Y37LAnOj@TL0#KelCfWGncg89 zhK-6HK;j-GbFQ3-I-C&giMJJZ5H1yj1z7Q$OPG7+E9!4hS*w&l7$$FFu=B1Q-5bzIVF85lpu0rSm2(*H99b9!gbH~z@_2!x2+3VZE}Cmo z9{id^ZIbs=#u_@M-4Hzph&GN)U{{w@d!sx&fsUI9l=u;BQVPC-@>)ZXW@HyD?!gLZns#Sj;!07(Dt&uD@DqJ6aGQzb%Eo}rx?z!|7z+ERzBV`$1aee?;Oo8h29#Zf?r)eRIb+0# z>NemeSBMzmItaZ{pz~+XXyA;jWTP>K%7VKQOlpg>bOzWDP~jP2lC_~(WDr$% zIkJ~j+|*M3oLKPpGxv|15_ql6QEn$Dq!bHodlcAwTYoHhQ-qT**`%b{MeSf zYS^VNC%5X?DsJ>L&S5`(QzHsHHB?B_`)oL z*fPW}w4ouo({Gg|W33s5Y>F_c0)Sw|yy<>mKXi1bMjMOc^O6>@lC#5DYHjXkgh-#S zhkm0Gt`|JB#Br@w9$;N`-zn6_wCp8@y=y3|j|?)Pmz)*`5;fw$Bk)qH4{ry1t$fGM zd=yV~i5=CPoQZrl+|hRA(#0pv{iA5i8kcY!X?D?YJ~SOo;;D+CCA{2H;StI-Bqx{Z zzBfZ=9(iSR9hvR|?FM9Jqj1z-7W~gO<9&}l?)--Qca#b4u$o!&zcK)blTmXAHn_U@ zR6yIek}hu|w}R>w)vYrQ4r`W(tOUA4lEN;7aL*;86>dzfo5TTB)qHrUASGqU6^3De zD=&EP^sBTH<|wk09PmdRKWN$>?;TNASGhJg&1`hDyy`Dq!YxhYkT}Rq^PX6WmaYS^Ieg;vZvnmKpKDiK z(;No(O~TiS12>lxm|pzv`7!A$Hf-efEsxjet)=m%0^K?6*xrSErHe@ep!R}Z{02n& z0>Wv57%}5$zY~(!!pzI>w^9al&ib4&mq)W9X{9XD3Uu3LjUZ=|3NX`0DsBd+Mso?c z7Q#4@#yXtEH!VnZ8Tp2DvZ-@w*fr3=_7{caCfo)20P*23KJ^%Zk1YhL3tuQ>vZ#vj zZng1Ad#vdBY^^YApxeU0e6IN48W@-(JXNP=PJ_|4ff z6ig7jJoS^dX>I2|i`@l|6oUqu9kwxGm8FCuHd^kS6YJjt!;zuK80Kfy4PuY(xSxep zs)~Y-3|SlPbLU}QM0OR)gC8t;H5p1dS1d^4X8q+LwtgcC9>kvld4g8ikS?*iixId_W4%;K%xZ-i01m?7HDNG7p}X$WwN5-Rpo2rEL? zyu4Ed={OVbR7lGzC%;foaHTYWc9Eah_$=%)fG?I-jiIvwGbrGrAvWP<=C&YNW*46F zUTp!n;L^~H z2Zg65!JW+qa%a(ZW`n?0d_H-eOL>Jr9w>hKP1rXMM{i`oSg@MOhndHd)LBF#@;yIX zW5dYMO~HzILypjcAn(cEfKGTs*}Z|=h+ZYQ#tt*M#$Flb;%*rR#<$NCdD3FIG%v)8 zy0yu33YsCdctN@Z6p;BnA47?crBp1h9-AtxH`j2BYR7G-A|_TGD#bAY+C^MiF`kS; z!x%ZAKPT7cLKTNDqnK(K@5>`d)pB&<}gJHppuSJJZ=L}wz<~>;GvDnHMM6B${pvY z1V2lWt6>}2Q;P}{6OfITCVy#Ky1-NQ$b{hFJtS_FBg7QyO8E^^&csBxf(oqw#5=3O zx=n%=muy6Z_C{}eDM)lBHD7KMM{{d_H`W0u{e9y5hEARspgj4I5Ouh@S}$?x`s>xy zm~Ha8Yvf64iHFreUFcOLj52DVb}Je}5XDY2J1Z5$P6aV9vHCQH2I*tr`W$u6LXOL; znK~l|B)81PG>1N&`#8v@O4xQ$h z;5;zw!$6vj%cX|)#PE?pWLq-Y-`nMM0}`Z#c!=e>#0o`bXqymW`Tl5eF82B8?}V&Z z0YvHH1QhAOgsy|(N*(PfaH!Hq$XlpHJ-Xc0-gyN&XbVw?>YHCtaGKm7UZqhFs!{sGxRw~}oyUr7YcJ3S z!;Tz+G+K!0r{IBDyM(sEe-?FFccI$^1n73sB%IUQA65cZf@X#cT{W#_n)|20>)Z}Th>?0{}Ci}cTiR45Y#0uY~DYDE4CQb1&(`}d!Ej)az z6cfm3E_3R6J+WJMj*^-%noo(!R@{MHs4q=ant^ z_lbBLEbF%*%FBn+;p!}~*ai8~dWaf*cTPhiOr)iTSa5+2!GR;V=*Jr-XbY>P3ZuZI zO2*i$7Z;GqM%Ibi=XR$+H4y+gNvYM%QCl!u7tXbrXEf2B*{XK)aikNV9MrQZ7D_l+2YSLpSLy>IMu_V4lTX9N-i^K0~yMoZ*JS(F)$q?wJ9^ zx$sXp!q_;6v=H$Qxa_$-rJZcdnpyTFskP(H1Cp!8e<>X)RBq%Bf&^*m-8nCkbii{O zAO&{t!tYi%#7)7s+`t;^=pw_9g0O#|EF5fxYHEYtn#$cw-xg|7Zo=qZ2Y3PthZXq?M@u^*|2|9P7+j(xY!=zIy$**^Oig0V) zl+M1?0}AxQjIeJ^ZOzMzM(*{BtOL-rTRQN16|i0g2hba`?0GAh9k+)g%i=}&Y&in! zuI9=#_~;uE9LBKOX+65n4$Z_`&GKIC4Dr>J-si`@Y7?h#72zQ;b-n7PvG7t6n0!N-AUu3AXKZH(#EF-8bB_dV#Hcf~r{ylK zlv4KIxSFqKO&eL8-K5`$9dR{Z7g-s#le2Y7Uoybe^+5VxeSZ-6+0<`+!B?rmDQZ!~ z&|e#G7Bd_Um{@nPpVslJ6-EHMbR@D5Y7Z%0-Gkz>4w<=;cokSBF}p}$y=Y4kf+t#r zu%9!?Kn+wDib&wg#i(~Zyh}1T{j^e|hL}qexqIkS+H)R^ZUc=zkuNZiN>%165v0jp z6)_2#!Kf{#qap|bpA;(KG%qEhcQ`2I91$==1w=!l1G$oDhl0d>Y#GHwygY|Njxyfb z1<6*6z7qwMq0*7CDZ?vs!y7sKt=9V%X2AsRfY;E$g9YK$48DLU z1VLs2=oSbuM?KP9dJEl3 zx#snngrV4Ta2mF>%+H!AZ{zY3j~0b6@CxZgj6&Z~yDI!f$76;K>w@5$-P|B>hq)zb z`Wd{W`j{S3#yFl3_Z>=|wv#`T6tuSnHpO-jv{qIG`&LBmr=#qehZ8aKBs z9=Pf0xF|hRGT>}qaqzr-y&iui(R`Y%Sx5TS#4)f&)%5a7M8tOF*F*?JZ4z&dNT@;Z zOm^Roy(~=VfgMZp9nf2ni(Y~v8VTgDGWDm9)((s{&QzkGgEenp2|%J7v6*y>j}vU5 zt{iORa7b7VR70)B7NMZB{x$W;^7+P!tTWIOouTYivyf!g2wt=haZsGoF2XKE3f4Is zfFKZ{^>p5@X4Gy^t^AL=8U3MbpefU4_o03&QDoBs)0LTi!~8x5#fmd4sg~(FcjP zp9`Q?N2R+DV3~dG)b|?M#9qFMVf^8a9uBVNx?Ck_NKldN4b|T&X`i2_*Lx?;#z}L` zJ;&vF(t09akKr0XVjtF_5Rf9@*!^NRBxP(DlhzT~v}w=i4<I5GH$pF~}Y$E#KZu$FM(h5>~ zd$XMaK#x;9q_@;cI|w``bP`uJD=>>!&Fh;29eBW;aq|3+NV-ir>3V>~h!x|w3A%0M zg~-ruvYwB7p;_{5y=hLPb-ctO$G&!7Xxo!o?N4^Y!+S$*`P7|bs~ZmtGn{kdS0E}K z>@F}wY&95zF%gO&-J91SORVZ9BcQ46N-YI?rONww=x4>ou)}5anbR17_!(E z0*t3l0t?G>;f%G2HdT%(#*kNQ_0d^k5T7d?PHNj!k7C&PnjGd=5f&GP0^%rui)Mbi zN*0_&Pf_9LpNq+ZEG=FcOlHnlgluje^n6Gj`x@@+t*1N?hwfo{?-zL=86%rSLe%kX zG!oZVhGC1e&8;DFB>Q#WV6%bs+yV?7g`o+%ziKXV^zRro`is$Zb@Y@ zy7su=BB^l{L!?$ovw>uiM9%iDfsiwZV5NG5SOZC>~9&Oh5#^ul3b(cNF*!2F-WKBjEakfKHpc%{Vq|%a`{wAm_OuX0uDU zf?SO~?wC|A(ge}P`T{63&+jG6z1)l3@Fs~o<1`84<7ZO^(pJ^Se;vIsVP}?Z-T3>9 zV$906tMS~Tf%bzHAV;l&vJ|4Z*iCOPS<9NV2L=PzDo89Nt=e%tK9E+(fF^?>(%&XT zC3x-Dm*rvLDL}A~cU5XJu)K4|{Ld$l1${A?hT+l{W+8Y@)0G5*V@EJ3F&koyV3wETecVw9c;8gHW!R11`6cJNve->l(cdebKmVsiYl8L^?)# zh0pD)aEq#>APcslQ%6ogGQ_t63|1~Wz}PcVi(|wC?Kgau;x<1`*u-4$JFgz#Ix#r7 z>mqB_uTjHbWskIiR-gzOflXx`2Z7xx8Ow*_B{}BO1en_wjR7p2b`*?gdyxnNq4GQ< zVS_8#c{N1Sq!Q1CaB<>S&IC&TN-c%7rb0hmp zy>Z=fH4DChph?ME@fCJln9;ASK?@6w$KE2DlXR zD8v)t99D0cc&B^mxj;>r)xGwKx><`k24;*v&}blK`AF{fIcv8wYA2~T-g&;dLeq55 zb33-PORtaSfT$?x`(i91ZY#&zTJujoOo{s4Py&*vB9hjI5mu1&R zfuSXd0zIV8kry#0k{%5c@Q7NJVCcDvzh=wU13E0q109Q4dmL+!STvBmhGx@A*dkLC zR1I@>zBf`4Dw^kQG6K+Fi53GiE!a#Oa6W$~9`&`o?CRV7m}kIB;=z$F}tKuaS^O#;+|bxg+PWSiWZ2%-xpVZJiFxt};0 zZD_54G57NxrE=3 zB3UF0c7bcVV|*LGGjwwmhk+xLlM!=LGg${?(H{2f4ao`e>F~L}((A7Wj*WJnBV^ex3FK=*)z?0u-c-TO6gC`yev>*=*M4s~cnH!?HGV#RKSfXl2% zaUU<&gAKTFkF-3G53XN;957%nfp|xdc zxO6~BaXssQpcoV>Wld!yfHu%+M_VWO4dc3l_MPc#R#pbBFjSEG<^Tjtdc0kZHA8Cka`rb_`G zFo5-(gZnLKjH9}3k0gyF5U^!h2tD56>jrv4x`4$Va&kcvYQ>ky;U3lkJMKnx;N>e% zH4t5~hPvKn+!<=FN{iLm?lUFlj8*2a%eCfZ5_w7{5X!k$sdZ}DXnq2=J*C~a2!OOg^=(BV!_w7s z?pj&!XbRHpU}5HvDdXM8FQAaVqB(Fi9lGd!JL@&@rbR@xC;=pi4oS~=rQd#-$%b1d zx6V$f0HlCSnNCI+gjF8WtEA-5 zofV)CF7Hb55V|}AHSH5+D;lK|Y{v7U79}0M-hm}b7IwL~>|^9+O1yt2wvJW)_GsNJiP zCSb1${d~|jeD@$Ne#*zwLgcKj@`R6@P+uH@T^K+t_^raY?PJBSgm5u7{wvrftQjO{cP=j#b;_ zKVQ1@ZHXEITCh|YW@icAh!9-Oi*v#g**nda{sbl+a-{n-$ZJKSl-LEPE|w zXJ5JCAo&Tia^y7l+E1Az;BNrU3L@cYK&0U2&J|D=CQ!1)mtthq>oMFIN_zVgG+;?P zp3!RNZ`tUi^RWF~i*)dSuVGS$P%8It3MoWn>M`aO@pY87q&|2shV5xrWtz;PeCh~g zY@Xji7V$)?ogvq?tQM5%1E!khJ3-jcAfB~A!gVP7Dm_mxdRczbmm2;&ThrMTp z14U_gylCYDr=FWx#~`K6gHxi@);y}tDG|gXPnigHTfRBcp>G8&jxHQtJK1T^!BmD~3cP%WG6N*jgs1>3<|NMb6o z5T0Ntkd=-2`26ggIM20i%f4-DvM1v6>+x<2aXV%a6n zTycmU6AJgEhvw2*0s)e^C}OImHJ{BOJok^rMKbbT)rqra)^{KS3E=`3 zRmusPrXJ;FcytRWCJjeZ?<&sRE>LL(_ZlYa`z5EVsbu$Epv*_c91sFh(mS#RsXQ3L z#?4E%=xd?xqqE(aZhFxjJ|W%(dz%Gul)|`fg=Og+g%eGi6$S%v-;ydC`W(Lf#RHw= zL}r$B!w61ilLWR_)O*~?9#PxFYZfnWitAaaRoA%vUi4qHi{sdR7V?9_5Vwp!Mx35z+!rqBqaU}zeD&BuXuS(dsCCtQ*nlxrQgaZ^8nI4kJ!Vc@7s27ksdyj() zV{gu<&j@Z&Grn(Dk}^)Zhq*2{Qm2?YVe>naxwq58LVc)x(N2MT0py{5sY>k#L)XP> zj+odS1^+xQKPW-I?d3zbM0TGUyB+HR>-wUCr3PWWVYIl*i}PfgmG>fUy?)PSg?^%! z&LBMYa7VhLp^j9jjO;o)JT2z|&wyF+cDGm?>0220Ysm4VM-mfZ!kD?DTo(8jIc2H@ z#>_@G{R)i3Mbg${wZ^Wh?SUelYNP2{6{QN|0bW7Vy7z|RY39~`F&8AU5%d-`ZLc&% zxXhG=o2L01bt9GzY%mVB$g~Y*n#t%)OmGUt9@f^cn5YdnF+DTEU)DnB)I+7jI$N&sBA`K;mDuOY40oQJ6=o0Fap zu4f_ye0BhWI9hHkEp;VM^(1K)7M%;2C+*xOFg~SsZ-#=6UXv*7-4b6(#{kooq{q_` zkgOx$N*Ccuwxw^XibjZArZ9ExUmu!{H)Y2u8jjY=Lv~33cuSTG%WEu_*ec5NJb<;u z;vue|bx8n7&}?!h35MuaSbE))rH?Ks+r96Y~uT(^DCf(z7#`T^Mv5o5x_qV|}+a5>v3>hX?q%6Iu z5?hW2^ZQQa(z^E=EHbL55X^yx0r;gyGca*dsb4mP99SxPSRWmkSqqxC*_C7&lQ%k- zes;MCtp+aaC_=0S92pH*+a9wqzm}bA&XZ$Xgb>XSo#W$%Y}71icbKqn$0O{nvSVO} zP)%nQ9(+~ONZO>kt>0_ZIO7WsO9G7yg6%{S@xaLyyXHl(+~JHcR<@~Wc)JetcoJCP z(*5ry&ONi2GGBpdMS0Vs5mUj#-iO=>lKVQZDtUK&)(P!~v-Tv=Si6Dt@ERvSC=F|{ zrP9rSsS9>tD&@yb7%RJgmP-evG15{%rpf6$irSz~tEB+o#o(Yag|Zg;oLV!xXs*bH zwP4G-(mWfz4opQ>UompM4bk@K&s>vF5ZnaMXQO ztCJM$242;|4jPo9m8ijPwSF$E4UEqaUdzwXVC%2jTie1IJ(diP-Yp4|Q1l_3tlUsA z;-bZKg(jkDG#!%OS>YxGt$?r>ZK4Q2%*ZvpP#$YMv&}Bwaf~XOTOO!7!_jyPohsp+ zPt(g1X{X+jK;tRn>y)0icUT%$$q|;M&9y^}N!AXchaAqDMQRCLreB@@aC6|@;2cb+aeWceiCJ}_v?2#FLFo?s8 zR%jtMn%*amBXW)t%^RIU=(k_3*eMF7PU8Dy%0e-WNS@}_GLm}ic16BNU%8Uc!|so% z=x`vTlBy2zE%!_gBq{|~FL3I%%CMHS&E?6I7P0gMJbAda1h!`SFMOo`R)ZzP;HJUj zWW60WqUB++`UFb@PA{;XzAoK{busXP3&GS~>lsFB!Ep&pagJ#z&EVVuZikLh80LwE zrL!O?6R72#0>lVXAFg)+E<|L|goPC&te9>XW-<4GMQFy`&W)!1x4eXhrA&$o9Yw{k z+*o|+(~w3=ebfs7O~!SE$ZLaH;te29E0JxUhl0Q|9H;A}t-BWVfU|){YZA^7XaX}B z9l#313dBJDf|G(gRX! z9LeOOwIadrT^tOtcu-^w#}3S51@>Kgu7ipjyWB7bR#)nY;36wmw8oyqK2Lc=PRsys zVMA}z%b}c$-)c!8H7=7V#S6VP+X+^JpLb+V^iopFX%(_0x&1PJny`@EL| z5=TMWq|MiQs3k}G8|B=is46UBeWbH(4+EpB$yAkGU0%+fPwu6i zFs>8mCBJRWVZv`Q?!Y{K5*a&44q={@Qt zs8dyxjO2;GWdbDodHndcYho`4XaSEm8mxPKChvwjPsggk@! zM@8-uJs#J7>1GSMF)AgHJKhAv%MP(*5IlCBT5x2-eDW%4jnG5rc|;a`!&*{P(Cr{O zSy(kj%c-(CV~uLW1jv9wO5tb8qVw#!v#{wx5;4N;XNobH zX3v`1O}m*^K>1cnd{OR5&oobVG>oYqqhz*daa_azk!bc|YRb&cpy!c`mFkej;i~V9 zVvQQ;9|VB5EC|?uZ&z9CjjDxy+_H8^0)7F*e{Lf{M;``U5MnDtL=U-S7AjLT4Ed7C zHXw3ye|aC!M}%pds&42w8T)rtG&&0~TB1_~&BQNUZ$F$2LZuTSc>qMEsiA+V2$b4kSY3zlTLt?&AF&Gkz?mRW6Zb9HML3uTE*a@4O zz4?-;=MWz5=divgi%mS|Fz_y6rzr*p1%%BNS|d_H&VfE)o7QET9LmFmVAvLl8Db>_ zd~Gu@#306&d${)vuWJkvNNsYCWV<)IPfE_FXza7XT<9M;@~y~+y5mb{51a0B%0$#2Fgj;Fv1PU}h`KE(}!+Kw0VEK~0)I4Ds%C)?- zqjqtCP9ptU6!u+nUwWVj`K-b1Cv$4j7m?O9B2G;BVk)18ki1^PKY&z^F@jO{_4DR{ zeDZT0B<6&~hc)lkBh< z4D30Bs9U}HfMYd|+$AFch&mn{*v){tP5P3U!S~8f`85 zHTyXU8R78(l*J*6t}s|7HfDa?1Q?v`=&MT(m)&L%3nM=Ex}zkH0N~G>N4MN*859Xu zQ2ioK#K{qOGp>QE$epL-3fX2^~54m!r!wVKrCISk4heseEvqBb9E_L|3=r z-sovF;vmClwp6IE5+swa*pL{@CG;VCv{GYdmokB4!py+%PhU^e6#OH}lG*ij>}3no z*z4Fsk+qfIdVx0@lc?6VVAOpZRWH-&snSG{rs>=E1EytLrBirJgb^-s*kSA@QdSMe zVvpWIj{~J^-keBv6&^bv{5~ndQM$_0s1gOoEZXD3C*bSK2^2K2=tzDitRfsIA*35c6~bz%ZLsxx!O8 zC|J0tMl=g=XhcP}R)N@oRBiL+bP1Yp=Q?Kus^ITPAvtc3>`U6?h)Hw&BF3 z+l6cZXU#K5^5S}*-6nmvSoaFBx=*aU=w?-j5wsyEx&Z)eQ+CzEyC`ctCkWRMelw6E zB`67MYly>amKl6mu?qXb{>bem3dWJoRR}I)PZ?xXY z8&%{@eEL%=`lE9&mN1b0V)rI$PS6BlH?wsBILq5zP#)MrB`jU7HB>AO0gzBOMIhE% zOHwSpQ;b)V z&s&$gCJlB72}Wk>6xmf2SYe5{mtO?oNpX#^Sfh=3H^_~7-O)}^#0@s2>V@VzLQ7Cr z)Z*#8Bf3+&Ai}+r8*9p#6O>pG^a2K`B&GyBs5zX9%ME3j@Cn;IYtgGsH|x7@vV-Z_ z^U*5DQ!-rmmfw%$E5c>lMI`uo)cG%@UJ7jy3WvGP&dcJlmI>6A4n>x;APz2AM& zy%%UuMq_JBpR~Op?uRbS)ubi^AwP7jI{@*dn%3W9LHpa7(6t}}8oRU3rHNlo>uJ(X zz1Bs^(3Kb0**ii77l@>@*xh^zr=;j919u9}9+ZT+?uHf zRqFyPg_WN>i4w~7PuQWzs?V?+?q!Y}pi!g0An7O%C)QS)YwuLdH3oWgVkQn6DYmbXpnA=*$ zv=Qnv%4#u361>YPRq8AEU^lcWttPQYLz0r}U9z40NJF=C0;4+fYT!Ga8Vtz&e!I+ww%~O{g->*LVQUgo;ymL*9R&UY)0bonr`6HB z7d4UTB*!iS*V``~J#l8vD_MpFZ;Pddr<%^}5t`Gg=v5L2*8sF_ER^yVbss#QO|D;+ zQg(PqIehA;Im>gDy`02xkz8vm6GneMTh?3%>m;Hj1n$gHDPEexdQX&>q8^xxF2rmn zvlyxZc8eQm^xmgh`Ml+##HqqdyJXulM6y`8!0^_Y*R->mI{^dCu~%LN?6ftkk02pbV4d^%pN zh?2zi8uo%f(o-}R*t#@LF4eq+co-b1)j{*;j)Rj6`KvRj0Z?}7iqgIH({y?(Q(t0) zd|4CB168^V;RzTaoZ}BdOVD9Aopq}OT!MSgCpCdj0KGm8D`XMW=j?U7z#VdJHB5?v zcQy3XUbd~I$Et!SEjdUPjMyATK-R>IQ{rYY!2s*jNW}&{GPdn*H+%DjCdHHZw|4dJ zNfjPKmk1b~j_ntCqQYJF-5_!gRpxLMvMhLmHXp;U)Yg~?)!2Hfap?(J!7Z^_czvg| zjfc_uUUKy7ToGBWc!S1hiHLt_ls(?ZLfUIoiOs2Cbq`t`E2gca>Q@1we27zJ;duE<=J@_WekGB za%Rz<%q5R;ZcuTTqs`Ep=Bq1SL%FqNm`+#rRNuH1g)-xIu(Z07ltSYKNpd1x-No8D!{pYt!TbfLKD2;jtx}tjfO}DwW#s4hayFKA}ZByXCwV7QuF15OQgoI{uwx zffg+=X|+amG+o6?=pqYEE8*YD8TARXI!+ zwnBt3u|d9Tx@eopTkgM0CfAz#&yu_oa?^knD&7^Gi!qbbA05~*u~ikzeG)1b7vh_7 zoCykiWDwHXxAK!rmkL;EZXyyjIdIsFDz=2GcM_WOO2^*^DIsWxiMY?>p^?;WN?Tzl zoMkp{AV#mE5157X2JR1xF!YR>&W<=TL4uZIGZBNB^|D=3jyUuxcd_JKo;n&F=6Xs< z79}dAPAYV4$-sT<+j-I$eNcg~ee^9`kGCzMok0BhtqOgmCoD$Ze6|_johO$e7YS3a zx9WO0lwi(Qq~{19q3=LLX-2YcsH$;E{7`PS^$jQAiK7)5b6@lgMdd&N^m%B?hEVqLM2b51=-JP0AbjI z#TVOlJ=_WJAa)9Pxg}WDeKkF4sk%HA649zHyTkW}OIo}!A-wU+o`%}-rUYzZS}?(I zF!wln+Jq3ASZ*G*D8?=*679o#+lH4X06{>$zqK$%@o8@ z+3xU&VBTmb!6X1?V9D3I&%@#RFD)yF(dsLLp>yqvEz40DYN@bXQs}QOxqFL!#$-#Q zZ7Xl#5jv|w>uRo{Pl^biOOJq8 zPi!2$vLfwLsU9x9tMPQ18d(BAB%{TvHFuNGfWJ_x(CcZ3J6;LeD9HHEh+b_Syr_lI zdBLo39~;7q1$IG*YbAL#@cs>`e7LA7i`i&pLr$WbVI+#Y$9pA6oF0!6Ue) zM8Kfg6<}piolvyJ*wp5ta9}d<&AF*OO0INnmW3EmmB-re)#kCk2K6L%p*kGRcb1qW zHtg-TUUv14acHg(lS8Z)5PfE9BG@l_-hlyvWIMg}W>7J`Er7}`qu}%5)V|n`wu8oG z-icL+d$_FPn!+?AOen*ZE0u$go1-Vad(biE){U?QUxzkq|+aIRLAv{5y?#KVR16Eb08uia};&mdSK=F!QS9_S?D zaD|pjW*7F<}F{r|OeDj`(lVn!znvm#8j^f52s3W&95tHPuE{*c%yd!%ImKFpXEDt9D z;~EC_3thAI?lF)%!g5mJ^#jpH?_rEJq0G!Plv0&Wc}=Li3DDbef+CMfN}>2!+t>MS1aRef#I6(_$I>7}nD>yhR_PH+wxxt&&gb4br!IF^|U9_a6-p8QY z@3Pz_KzK$2#!X}_qV)Yx4Ex}fiU#(SFKgLw`If3a=ezRckuqM1SgT%?_cHj;W1;gB zhznrK0$4k?9CkQIVZAg{XLn`RxWVD)+v8IOW*)AKT!b15)9r3OHy0S9J~Q!c43l-w zoh}d}08u?;&|hB&focq~SES<7FdKP%q#<0^9~QDQ#csXc;gFHwh_?lB(E7LBCCxVH zZWFr<_sW@w1<7~bXC+);P5S}Ga?-uU-U`1JI0}mwU4*tJt)phral@%GjTFpIk&N_g zaiMoi+kz;YsLPL^(hkrdLt!Gq6wV}Mo!zAvuvBdYA$(n+#R9b?!#b1OKEUK_R};bl zbmTy3E{IQ~Gp$T9iBhSdMN}<63dV^y*NxZ zqy!;fL;+rAD%o5nsvuKyc=ep9nOLeS5EPdq>5Jf>f-NLi;fAzw0UwJ*UOmz1&DeU+J)+(T} zx}4{F$Bqu$Z4-a|5=G*>`k!+hQSR zioFi(92_ygRz0|(cjeE7@_N`VW9RBmnK)_z?HLl z5O?Hz8+vwfV37t~9&)Oip02DoVl_ercR`!lO0fqque|hSN4tRtJAqe+?h?TiB9wLJ z0L`!^pch*av^~#?+d&|b99RXo%hU-KqQq+f2UySNrr_i*GZI2amG_NbC3OO{LYd{~f(!6p2wg}V)@4_~w7f*QND*NG_U7tx7DUlgGkC}j z!b_FYFHXKcyN(_fHBh91JjWe_>>Aa^)3Q>_>IF_ySyB>m`9gHF4*kR^-9tR40@d0F z-I;ASltVPR%AksEXY20O5|ei{z(TObyAjJ-==3bVJ+o#fx4_tj`4;uBtdUpFgGx7S zA(yn~i3>|C9f9k!(9VV0HkV6P^Mfh3l>l37{p^T8~3a}`i! z?W_i}OoPpWA;;GdngvFcCF|BX?|3-Gn!s_in^-JmY$oU5aMTD+vHJ`W-)*Rxs;azH z#V$EM5N_o#hp~IU1WWV5!&!2lygiX?T{}&^jt``G_wPlTaA9zSsO)Q|yU9U{t6)fq znoROH3lar+U0ou?tVEXvv^XvziBQ&bb=wpblx*@gS6&}+;7FExc+Ut z7xfU;;o7lH)|&ClS?wxcGX`c)qxM_0l2?HKRHBre_e9ceDz8ym_*yt? zHm|oYdCv@3Z740nU?Kk*S&}f1YR;S?(@p< zxeITQ2Hc{y(3LM=w|eFG+WaqmiQ@kSXBdrSaz{I67~H zYJ8pI*{mIa-zOe7%iknX4b`?!m)c+2oSzQlJOOy{(jt0&Iu92v^-dvsM%4mfIkkF+ zGM}skC>~3|ydL2cdc$9e&XH(cJIL`q8qhtUwo`Ory0r`mKUgR`$1-s(uH`!xrs~@&=O=volu;#ibS{ z_Kdo&l{^F%VNw^PajxGdld8~GD6=aT$RgV^sDdBf<_QUxwQSHq_3Oe)}pH zL)z$758?N*^1PPoCy)hCPHtUz{V4s*%_$y9zEVMPwm#Qkqa6!<7nQ9VA*>}cTZOjtnb z)zl0HdOXTN$L-j#N{$)Yj64Os=|pZ}L|b;vhJppKSU(ugCvNH;c zfZ_`ate1gHMfRm9XH{}?<2f%?gbo)+dpPo`G>9M&)dNz^3D(UNw-2)M z)3Mmo3$ck4!x1TrD`onEFSi%QaeyG@MFt*mD;&@6wHe6g1Ur9PTnYYl#o@9n-`oOj``@+{q1Kz7?KUw zUDYuffWdI51+A8E64XrQ_~m-$w4n0o%rg(#V5V|yE>IVKQCd(8-)3ECG3{@RtK+dfZ6(0sE6I4KkAmy*5`p0FJ_mE2^3 z^Uq{CQkEHdsbNYr2Dwhhc1su@XN(*`2FHtWdITI1a1MK9=F&dB2~%Bf=R1x*mX3xUjV}s24|{Qp3uJ7PVN1jVwT%+}+1| z3g-0`gF^J@pA}njPi`R7txl0_ahU@=;+UsRQ|V||+XDOF3dCh+2usO@fiw@E#ic|vqph99Lh?vP=;2}$tjfK`t)bLAk!*%LuSYiLUBd%wGV;3 zV&&ap7!8bcUKu2c4aqsEi9rxk(gi6wz=fGqH|!a^D$sl8P<$QkqIu1X1VY2|zJ*xm zhevoN?{(Q<26?x!W{Mj&FmcU5S(LpD!)XHO3IzV0nFvXxa1r;WrtXoID99CV!cPr7 zUcQAUQpT^jpQ&TgpZnkUTz{Hfx$xz5_%QWsZeCqvK zXyT4|duM~Fo8&SdH?bJ0tn5U(CTl>pEx9r%;HXGt>EON(Gx+rAvcm-Jy=+y3@W&N8 z*e2zjLVS7@`e?59VS?*QiL7RW4eaLzAB0xl-F%NC^vKEq4|pa>e$x3VTqYx9kHOs( z+=fplxXt*Iu=!CVo@Km)xwugmMM7g^c9I-WLvV%M1#W{Tkv6ETcMKk~d4<(govlo> z0at=;*mn^*ReiWg0ea%MfY@DBp($g@@4~yjen?PO8AM zr&~e`u`77DH_BS}ph@tC(Uws=bt^jPvRmfz7LuXr#U4rF0U?k?KHs{RhtZmMR$K0J z&3U!P?<7pq&v;}@yy};xqrq1qCCc1z*K!VMo(#WdQO}H+-z{yJS~Y4g<#{ghF|*!D z8Byl9nIQ=c@Ce&x4hZH>%GJ89hVO-cN#Dq*tP{)F88`Gu&SUa?1+jIV-6TwoM&U!fVGi#$)Rb<3$4MvG3bMQ zglXfwJa`36)=A}OIaT_wG*a-1^QbE2m1}Ic4NYNGDPu1Eo{mGNwpYJ^*|{c~8-smg zE~KcrI_~^89Y&^t0pt#HaaY`cYsK`^@@W}rh6bw4mEc8(HVC^+Wi5aR;hpITQ^mhV zNqCZwz*YIg<_Pw0tz|^;7Scz_V1~~?5zXGwGzqfH-l#649=DUxAqwoy5^0LYv|nc1 zL(>wJpCtkP6%mL{8zY9E@PN?;A1g|{n}K-{X%lR6=IhOrpvUI+ub=il$oW^ zQM~DbAp>JuXyQau$Ztj5Kz{czY$fASRgOS!*MU|9s6Pz64r3tL&@_VCKJPNka?pB7 zMSqE0U9AOJ4xvp#y{_(xpd4(s&1HPzP&w4<`SKT6>bHIZAUeuYpxH-gr{#&gHaGDE zXAX#i>t`x)x&nsq9AJeXL=}OMuOX<06%xh5mD!0U1^MJmB|{ERjwp8>CZ(-Zx}6WL zOiF%-b&}R2^UUv007TxUwGsiHUReR%hYt&HK-$HtXFK`=9RgNWtxF}@{pe+<3_Y%K z6;!e3eu?;MEzvjZy%=7=_@|38Ek#Mqe$onyG6NJ`v#!j&SxX0;UZE)QoRvXzUq>Ey zX~~vpXB9(YiAh>o9<99dJ6d)IT`bcU_Q4qvK>-GA&@X3JQ_5?!v^Qu&&_T_y-ZJ}T zp9~m$Ab#Tt-V-b77v03aV;~QK4v^ky@|TwbK737+M&j>F7M_+2o*jwm>+0Am#Mnrk zwv8lE--=h;%nICc_TZDW*6u2PtWe&jKt)9@Z?}EFSha|C?e!KLb?70&F&iQ~&1He{o z)dCUL3(a1EqR$l$twC8=V5l#>{Nw$Lw~dW?o4&<4r6b1nctj{x|O)TkY- zNi=dj&xl;c(;HWc_$SX)8U+Gm9pKk|;?$|@rQ|B^^`SNfZbo*E-QIy0%nO%z(P>w{ z+727u2V3WUGLo`C8Pbuzfu5H|ffX&fL zix+0z@|qjVZL|aSf(nv-vALRRGKSvau5ht}6C0W2>tXgY;3^9hqPD1Sb0?YAJn;dB z9T-rPt(gjl2A*6%@UY)cC=H0Odf!PoR_4p+0x>Zc5G>``6vT$azCu}^sFAs5zpfk8 zODn?mbBi*vGLSRn%JYU&F~!_jowVTPft7%jO7)C)oanPyu2B)xJ zr5xPt0M9_=@$SzTuhQCAjAKb})fTjjnqv%n%(qxkGOK8?QgP`lzA#rZ_SSq7Gur2J z<$RLO&1%SvNouboI&hmm4yi1hPBq1hpeU{Csv5RaNHyBS!NuAuEqQ6!d}xZf6`Ibg zgCpmyzc!f~a@|Xjw}@L*IRo0de&HFHedE-%y?4{@WOuINWB~+{lW7Es3-tA@!2;W0 z=@$vncoL*NECUzo+BVTG^7#j@qRBTXpmn<&SQt8;<6iQHofl)nvrwga; zj_W4}Ah$%6=#~)}aH{)U?Evgf4+aA*QzKL5ktyIHUmQO~#4WZ}xQ(qaQJ3_~$t%X6#BBv!*8{W8{XiDSG zy`)>()m{cB3vOBIU|^OM2+0Hl1?CG(0*lImv3p3cibfswCCwO*c6r+jLU<_#OhK|6 z7Vf$l2e{0X>qJ3n<#cYq`7lUb*HLaqAlBnfOODQ&!-3c6!494_!m;+GT>QwecBO*O zV+8l2BML*WED~%Mfi9SrkFira%i=UDKA9(x)*_dnIx1WZfRKAZsJBES?IUP`aZn68 zF&{$~VhTm}yP0jsPX^*h^_0zOuUpz{bR&Af&oFTW76D`A!h4u-pp%oP<1dU(Sgu) ztw`tat;^kE*Zt~MTODg{lvm?UrskXHth&Gl>{6x zIB-^4tAuHYl3q4wRo0TBT&^&#{S&}#1CL5Ccf8HH=DC$WkVrJ3;VDloYlnjwAbRoSmC@Cnz` z9HeMW=u1{)Htw|e3);Q~^`cNnAE-I?uD63Fr+|bV55dM8fH56OV7m6cRkUX!(5y`5 z-A*-q%t|Y2e{Nl~NrQ0CNp|mk*I?#QG4}5*N3~ z7j22oyV*Pl&`!7+(h22Ewo~6MXigprdyUHR5Jq~Z7l+HvS>mE& zD}FLDCpIJSWnI=#GTygCr&8Z0kb3qScO8kl(guT|6zByh9b?ek&I8X0nF$F4;xOa2 zA#SOzyIKj#05jGnPd?$t%+N$H+3i$TS%RMVeE|@vI*Fx53>Du;-8@q;RmauNaJlKk z9k)|#ZFMsRA(>kgJr1xgS_Y44t`~5b2p6H5l2wU9e40cem7Qw#T{q*++%tuZ;26=5gXnbM0v#Zg1tn)I5{KPwsK1yxN*vYxBak|9#p46fD& zDMiRISCxFHh|f4~DrT!j53|s%f?jkGfaQ9mJ~EmN*t&DR;q`N*_&X7KCE z<&Y5b>DWyoOfa{xs_RdPVtd#=-FH1uP82sDx$doy?vm{vE-t39z>q%@j(THfyxZ3F@VKm3o(H7gZ_|C6Mh1{A~p}x9pWwyh(pq~#` zTV427ImF7=uqBvtw1)4Io?oFOL-@C2*ml>Kyorq?#akHXJyH}MHRz1=(yQ#)=^C>d zm~fQ?7Ql_qt4XBS1*EGRtB`6qOWLhj!LHaG9p>-bU1KVz) zpvf{z+D2I?vX@a7u@};SDt#b+X-aYLHB>oUWu32^lLJ7!pqmT48_lsKi%%fzAqyRU zcZ;@P2e%A5w1|fS&{^>Cg`OMf;FA)x=VTQv8y? zYn_&EqMwPl-68wIF05Fs96LLPIB|`00F*+~9k+s)Ily|je$m+JP(X?flH8-OT!&YM z!KEzyb3HN8PCRQO1kcEldf7%T>QmK~BoMKuPaD^PbYMXQ?eh3=C#l+MilzzIYaVfw z(IMz^{SlqG^oEIsHoP4jez^om$1Vp`a*?~3WvS$Ni+!t`T|XE%rjR1D>_D38K3vV* zbB7#)g%q=q=^N47Z!eCX-=4Fnq3V*;98zbTw-`*ggSmkExOsR~pn+ z4BL+5&iu6g%XPuj=#@3dvBIDo7H=BIw~F8aUOHx5(sH@3)9_YScZ5{+50tn-s2LOC zu!xIvLLg?`MAGQw>#WMm!(bkCZVuHn$uptx7+c+-boDZ+&L|S~ku)RGIrj_(*PQ4V zqpLS7X{{jw3sEX5ez7)FR~bMm>XC|cTU7BeLx$(6!tHO^In6PG*~Qy_FR|GaI!l9A zP{2uQRYtQR4jfmCSdOKZM&LxvWl2`jC0e_T$rAEha%UuKq7luxYT=B$epPq8u$bQ| zX(vFd;RVUuSPQqWyidc4b*wsQFv9m9P`glLP~`&6l2(=Nuf6F=w6Go#7ON^*P%0ag zY<3hirSYDe?9;vVqe2u|LjsXzE4JIl)L1uQ_~mpCfOlFYgncvf+#lVdCNrTSl_cz4780C5mD!}PGhw?B=EroV$|U6z*-ZCD905Tf#ldteo)AXlb{+3lG743IkBi2JtBcSHlkID>brHZ*QA#X>gT>`xggE$OMriQ#(WC zx4o0_@Jx>`9eB_ey)P|iRV4~I_D#02jMnakGYamto`paegP!Dmz;?MvazPdP={$x8 z&uuVg7y^z=Gt~aHD}=LC_e_<}dFH$gGLw|rGs!A6?kj@yDixO~t?3fAcUL}BoKrhp z_S4WI9L;1sxGK1yAS41fw>4L4}0mZMNIUqMA5mG)LcfQ)C#m% z?3H!{rKWt^c_4A6Gw?`MmpL*gIam*rUst3tj4_-nE0{2=xr52~4cz+e`aC4Bkr5`7 z$x1ioDZZ%!G6mtf0wKZJH17+6G^mK3VYV${%CCmyk6me!ts6+@GT7~MoWDM{t6wN< zbo7&VvU68&ZPeBVpWLw3cdk!-_i2vg8Q{x;-d9^fCnRm1BTv|CT1k8vbrw0me?emIc z=9cze2U{P{8dSpsFo3&}l~B^%AD#xq^y6W!&x*)KpBo!g4Aj5~Wn zX>x`}6aMqQtTgXyHN%EdJ4r8Revn2ttECk)q>9YkVHLaZczob@`vE!@q z)a2aLcL39+z>kDt4%mSd&yFLiFneyXY1a>#K)TF)I0uLxak6=kU~HX1Ok_)K@v{@V zrG$3|5=CBr}C9I+Di^#3AxNik5 zL3l{slBi}0ZicAIrC9Qrl#05B9YkIxa_J4F-xZtYp$8D5gf2M)UNDP;UyPOP(Y$WD zfZL$;BJ`{V!(`5Vc+faK)oRfGKY2K%A5Kc2(r2=$oM7itsX29{u$;(M?jIEwgkyUY z*obL|jy@%==i3Jj*_9@!L=o0ARi!%!9LzkW^(OOWtUdVfBf*a+2|D{w?RhIA5LXBp z4Ya{$$MSo@jTknt6Md#b<;iXbw$L`~o>z&z=0~l;;Fncp4mYtmmZ4~EvK#CM^+aQB z#8(Z@8p7ig$EtHXO^#)}E1okKYvLEeP8NXbZV6?$eu&1;LN&FcUE*|QgR+pHBdO+t}F-Yt3qLLSPnWC{5U&5&Ts5k#0fWL-`5Zj7QyWych|`v7(u(ZEdbB z&otr>H-&msQW6_7+!!(D6ziKRE8$c1dMD_hFGKC!=bX#IC08zW&_y~uBL?w7b_fb3 zo=c6=Ba-G#Fpsom4N?w*{rj;_@g^2R$@ zTDT93DO3*7!a;a#fL}_>jkf~Atvt*$;e7+3VhibcPdR5ihdk{sa+R*hAw9zSQ+aK! z7~EC1+;h_DrdM+}J2)V+L8!MAV^MDaTN<$^+BE))!G#qS>lHHb1-y!$)?`!8<{rN4 z0q{_$2=Ky8W_hK6SXJ!yJsH-hGmae+tnRHodgKlz{Iw4v!)So+G6f+19-V zCD5*Ucilu|=?QRj?t_?lWQ_L6+Nu~NiO>=Q9$hJ_*(lmUGmH0l^n_h|VvDJJe%^@F zo!lbeSp#HowQ0LIgC_}jEN>|jd#b9>(qQYj&8X`|Z{gRtxQ?#Q_GOVJ`K+OL#LnlC zUDg*c$Y?|tGZsv;Dq@p+$QKup3yujo*1F;EQ_*N20l6l*J20z1R&jw2&D;=pp?w~5 zE8Ps@H5uLRq5Axf#}KuP8aS&iZlIg2a8Ta)Fto9J>NQYvF}l+j>rS@0)x581caNou zhT<))IgFdC11wxH1>|fEu=g0SyR2%{=_E1u7uKC1<$&+UX;(@XTOdPeGADOniMP#I z#@BntCOfqG$#VG97PoCdXqtqeQFE(cgVKr@n`3FjguE&# zO>3pN!&36Ls`S)!y_T4YX2{SnmIKn(`_fz&e$M4uuVYLZ2$6DL8q6CBZh@_pj56aN zB2evztf2q}i|?!6_A6Q^ zU>NO%Lk{FKk-7FG-k(TRNr z^-(Zjl0*W8K(lbpX$ldb;1dqrZns|Z<3i>KtBsKvF44_=WXXXT%q?5k35oAJ zWFd}dDR+i1oEDh-2Zsgsz+;yiD4@n0g6QjrE1>Uus5rpsXW08UI`RoktkYF8`^4wC zO^eS7diUIh3#T_&DvqR0+_;X`4L!~W0THFfxtc9?Ng!M;nUe?yQi}zVr(o+!(>r*u zw+Ss8HR2YG;s_dCuw_TUUkNZS@nNGwXxb#U5Z(D6Oqkv17noIaG;GZb4yY+vR-pXb zM!g8?J>iJvow-UJc{~iDJ%R&4F2vo&O=(9*5S1kxT9`hShIT~olz_{WT(~~RVwF~r zYoy))om_+IRk}3dik2k5InFvyc}>`G-M@N z-d;qQ3jp8XWlW$lEuW9Q6hX)a!vy`z zJ{Z}9fuVeL&EU~N8i%tTASHfi_18sEzmq9IL{t*@2Jv`vYP?9mMMX*z9}B+*7OMzt zN@{7_7M(brA2b!C4|FB{jehmla4?%rfsBF1;Sw;xAK99q1it7MV;j z@Lsb``<+)EbOpPBoz(>W$2z?nJBlQW+3h+Qokmv1R?`WeZn&b-1)IJ`DqF1bf^Y(< zc7o~#IvKxYoTvf?PW6?(jYwC0Jgl#1)E45`8LJ{4vbqD3b?VnwO9~(q-!f#un<#Wp zZ(^pmNJzK5n6qX?*HE6c;^4Upa~>39QZ`A|=7?Q4pE+Brt~7-4z=YcA9;D#aUYg)V zXPEN*!({E;@u5s9M3}_+Z59_A?I6PFT-YEdI2t#^a!&@A#9d=bldmO;SRnxE*~6&u zu4LiFl$mG~Nnp^EASGimH1kyS(Hgg_0n!i)*lM#PV0!gy(!L`gCNkr`cJZjuB1Xbr zqqpVAZLwGb5^?Dq^AidPr&ZVOpiTG^HRh4$r6 z8wtU{dGLHB6n6{@-C>0Y%SQ(R~nGb`qUU@xaUv3s<;=~A)o&|?cy+gLRC3ji| z<%m-$iJjAk`+Dz{PC~I}gRf=53=Jf{vk1NBjo^V{jypAdP$AkI@Tvmvwy9`{7#E$P zHFn_0%KaKU)ah$N6uKO}oE@0C602hynucx=b`{YG$(%MdcGku+s?1fJ$bgRmvrl@w zN_K0_0HlRaQY4qGipwXpWdhypmD+-DKQBrzy2Ode=af18H|lt>!n75IpJJ+8E7kdt zmsfEMERQ&0ktnDP!$uQY_LTvpp0cAb{Fcqh1yN&;n4#N>JYQGVRTw5i zGy~;B@aCbFuAJeAbX{elu7S||)RBs4?l)HY}T*>17i=}{q#Ll zHK-KQJD6s3cdJl5q2uh=-1O8~Eb*J#$UyB`$_X{)#f58~im1T7Fgzo_P&!wfX&lcr z#i^#9;Az)mu&OJPX;rdLi+hc5dh@OLL|TSxp}}t66S!ob!>$z*;A`V=K60?lkyEo3 zWxNe6;iJ}Q(I|=xl*Ey*#f}XMvCgoov}`TR&1;M6me#%AQfn6GWh0%V+|kxvKG<;V zv{XIvl^~k8o(>X7Dm>I}Z+HUF$4TQ#+cP3mDP6Y1?}fe_`=}K+3{*c5(a^@kkZ-fE zC^#N252-wB;lu>kM$Ev<0>16USXboXEUoIm_IXp;Rj)3acstu$<-mSHX=?B(Im0he zRf*X>h7ZahZ)Z9-Eg_{w7<=|sn#e5&uAFOl78pqq*7>TGH`mnn0CJVIl7qT0-CGj} zY(O=~m1w-y@USK3vaqII-~!?<2ySa7OHrGYyw0x!*92ud9X`L1W7b0oDv+m7&_I+%kh2Dp|QATqbeP{7;E%QA;)C!taT-m zz^Q71!^`D2UyzBVDu#za2TRT4?MM#C#7nH*MHCz>t*)-^^{$vFA+t-pP9J&oK54e5 z-?IEZ=t8?;3ik>(Z1dG5WUFJ>*NcdE-4nS`Lg#h+vH*OlZ!B~n_-}(U?>eG6WSdsS zsys_^(LwuWG5=j*QVv7W&vD~^o(#uzzMr%#BPYlh33nwgRk=ng>aSi`SJ#H-667~l zB(Q+o(Y7^%Rb8#r(X75!7q z1V_6V^C3q>tG?np&r4?%RmsUDVxnj}gG`l_7R@I2WFkjxl~f9{zSTJd3W}_pvOG$i zq0UjiUvN{qWCMELC|oaej-`zsGA{jHdO~S)D0%h!i*;7FsYJHvvTo5!-xn4?0xC9X z(r{m?77Q$o6U2hug_QwE_M_LRPr4>j?ur0QE%TocMK<7OY6uJ~&EE2s9kG(zyc6AK zI{ff7U^0zxv&q3DfY`5JmHQemW`6d7sBB8rMEbsVT|!qwd4Fa&?xjQENY0juoE1wd9%ES0C{+9mg~4a zP>Z{1Q}SQ49x#;tR(0q@(+=D{)iz6Iqj&3APf61SQ^q3rPaDBuq{0=6o<{SZYTJ;8 zpJhhv5Ohv;jwiV^?qWlU0@0&wquvM^@vE0OF4>{|r6-6R6*39D@j&e_aHR`U7~8Rg zPxAamPI0u0rpzYJnXv`eyySENPum4khrq_uA~)V$i>rBw&pqyR+WOaz(dig~<0!nU zhfj7&*#}TbZra&+MhVUNJ;pn(%{vYr(B`R$OXn{UVDjIu&n;;zP79B-m}2Ug@JZSr=YSWpx=-RviPYi* zl9hPnQE+E;<&RPEcvpP!>}*DTOfR8+LM2y5=s3ih3j09$BCnwCrLO0KROD)`6q!|1 z1RXR~S;J=R$5WB-MK(aNOj0n%$Ci-P<;z#-+c1C~-0on2Yz<&~ zAd2MHQ00wQOwc;KD~O!sNo-7!@_8I&!pUr6Ymr9mm$rg%3SB0-8vTSl>vCU{ivq+1 zO+noo(cvc<3{6WB1zjjuJ%a1tt{UF7@QPW+qJ{Mmfs)SdtVeQf)*!Bf+0QKQ&)CK} zLV)9JIFZyjA`jOgID_t=1f<7$IHg>G`-%QdwY6ZphaoGrjG)&-eW8(@BO^D?DicYR zzHn1(P&OcGeYBMX z6|jK{4O6|J7X%dzTP+qD97<}pp~fnFlbb1Tt^RU!nzVWMHM!s+HE0oiJP!QZ-& z+Chhq@L8zMm?@2jhZ91qhT^P1CPKQ1$tX|G6b;cF(`0Ev0#V9OuT90`-6HFNlmW38 z_(9EJex~^nk6?y`io(^{L4bvbB>ZGGrSg(fH*Tn`U1sa!UTF#lHnRr7j^eMh@84bg zHG{BJyvnuoNU`On4#B1ttpeE0g?RQs^BJtUjLu|l!+%B@2{WOK)$Tm?qmGRbyL74o zK|*%Xq}&eIt({s8qgj#MhV_p-yd7}?EV7kp5D_5L2(2R4+gj~~o7AFx><%kevm|#u z-a_a~0$CeD!k$$`n1x*xEs%1uYufrmQ%44Z79SBER1OS^NMg`e40r?-8J{Q?Z{!&0 zOJ}Ny^4G2&06O-L6=k2R6B-kF;S4`2-$Bgu65(FAkIIC5>f?CG5kr7lV$VlqQddeT zRm>HEaA0ME4{eoI#@Dvh-#(T(GF*7xlS~M_pevD%K4adyey1?AtsX}tAmzu`QCyzO znJ=tbH1r0KM@c^5ZKlM&c;w$uKgNT5IU0%9#k)5nHzUBfyG0d5vdt zAUnNNk{s_{TLc5$1Tvr`nN~ifj2s`R2P@hn)^ZH*7Cg5MQ@lj9q5x*ix3wG@-Qu>z zcJD-%k9%JxpdlfXBlRB!U6}XQfO%>8NfNK)Dc7UKD8-vl_!8U>10(yTKK4vR-0Dhz zpI3_d0r1I-#VuguObw2D7=5zBX~9Wj)oGbNYK0pqUYWyP^N2<*N{=&~bcduT%&-zh zN19NS3tP8Jzc@152-e%Ddu|NgfKq^^s4X{T+}mK}RhNSvS_&*>;!NAXM5~<26L5v7 ztdUOwSRm(KS>~sK>!J0kQOCAw&pGQevZ(7^_)Vu=SSJbBEdtYftM?YrL?_#ut%f&( z*LxM%%HE~V)i+LgY%oef0>|DP>s&cdu44DMao5X2RBffK=k&{TP0rwg{C#AwK>8})Wekgf=dGK{Q)V(GL_aV$ zb}5!j977p~zB3?*fUXYjk2z0O0$CmK-w&U_OSJS z<9n$uIF-7K`q&{w0ZyJO7|KAXBZqek?u*5KzSbpnxStC2R6zT0buI3#YZVv-gh9dw z0YdRs1q_MNK}YP84xkyLb+yuOxO3{mF%eYT;oV#gaJ+=%I591IxuEAo*t;$~#AjAK2ThyU zC7RNBD%g1`dz|-R5CQi&lP4=>z&~qzN>&?~1kxOpG`pdrm{1WjCDSQqQ#Po*GH*fQ z*(k(r%qfEGR-!y@;dqd+i#tZ4;Mk2n2}bp7n0P|8>B`|^gfEC?VC^)JRAP7(SF|lj z?gfUdF>EuAYObspTW$A2lU7#}v<&llVZxh;2g6cT7Gc~X_7O#_PfS`#_a-g;$ZgDU z2B_?eQOp8Q!UKJsB;Ix=lt;aM(#Nz#L6<5v^5-Ooq_h=Uc5G<87EKODn% z#R(O?5yfYi)DgTZZR9nVbVrI-^TAl_6EZ}Nf~o0J7@I3znHr1nZ=1MSypAlPV`srR z0XRkMy$>#pM-uCq_A+sER#y-tpB;ki>_~Gy&bLy@lTePM_Q7z1Sw}yXX~DW9JF6vp z###gq(9F5S0*7IVZ&!;R;sbhmW(G1*g_5)KoYPheM+gh7;njLLYXYPB>XIQZA&9T_&LWY@8PSE9CB=Kx*n znzA++Ob+$GXXVdfr_S0KQi@doW#qbOm<`S)r*sZ^7gbk+qo_BNGSvj-VFuvZO5_TB z=c5qvdTtk$89DRI*Gn!Z3Fv;3pMg2vk7IN)u;s7KL{85X9n8~-w(RNa;6r$~o&p`(KdFVzd z@{!mwp-+Xk&A1h7Lz1>wSAR_=!agPxQYlRro=48?c`RvE2q*UPx^JFc4U2-_=Zidb zd-j@#(@53C&7oRri`*VuDr0`x2~~~jK3m)UF4b&dzdNZT127{Awbz)8UEh`*KM0-}pLAIf2)lh^L z{49ksPN7ZX!IKTXo^i9V?jW*`VCEseSo0mEvb4uf#Y0u2x{d86Z=miylf}rT0FZM9 zUG5E$Oz08S+5z`WC%hF2T&pfDKvsWhsf4J?^cBI;TQG#1k6^bI?#_5Y655r?j5O>T zx0khQQpge}@kMp~ht&Qa#Rd&RP*Kp^@c5B-NWx+U!FKxA^~)R$U!}2DYZK}7SulpP z?detmXBl}@DT#%6i>8TuC=7Z6HByb&tmbKG_Q`jWkXGEHe2NF8gp9-DgC+nsK*+zQ zp~#hxjkeco$>()hjv!P9v6E3j_K0=stGFlD-QH~-E4R1;kbZ*YYM&OXC=}Z%68#gt zo3T2Pn zl%H2zU0r~LVayP-5W)7S8%Q(Tki3Re-G@3?jRu^E7Xww)=U+69R9Nw-*vp+4&caS-*C!j-nFM;kEZ5;eUMP&lM&wBzm2FT)}& z)^&o5>e;$I&fLB^y}TX_sDEXCWGjuR&26U?^>M)SD1@ch3JH^*J#ml)^dZc#ba9Sw;$C1E zlPTuA;K4Mv2S$Mdm{yb<&07%O>mhv`%alPpB)*`_oF51=1kNXdx0HEZq_GQ@61Kp= zd){+|XE`*25RGEWB=OYW$J%GSlR1$Nx#{$BKEEMY7!hw&B@u; zVi}jcOJScVO~5woq#%&?7X02|JxMOZ@#)c}EhE>n4#uxd1@}SWb_z~(lUg|Lv%iu!_nR4^QWf? zqP@5_*KJ`yxC-7Z@@!|_EGMWiIpVu$Du5OPB&a30I(e()s56>}5|lELRWWRzR?9>x zmToixapk{@d7qC_MndCpw+GJtYMwCzhq$$=^h2{>fO8084&;kCdhRA1{;{$WZx%3H zjF(VmIlQ1wp;Kk0Rb^4xz=0B(fg-6O+0@O-qEuc{?yCq5_86!^RpJ=Xxj>b2;F*_rE|ob7ppruVTULU5swAK|G*_IJtW&v}fafe8nU+Go?IDI7h} zTGq!&MmJzXOn4>X9wZ-Ok)SDKFx@Qe`Aq0?AH$`K&PG*(N4~vaxnrYuIdh~lI?W=h z!jo>ZrMnTUV3H^HJV-NTP6;6@g)UGiGvsq83Jv4nP`0o72ImEd^&bQRFDHz1X^5;`1`uUw5#3&&q6us{1rF zBG!4z=AMlz%n6r>hvrSoT75J^%^A(^9JDMlgi?C2jAyNU0%K=EtCawK;Z7hrB}Gc< z7f}g#BD-2`;Z5g@GYsZcdAA@SmNB%bvfAa|O}V}|-Z>&=v~9Ps13JtnF3QRN7}DBg z1e5t4{h*iyWg{#bE}&)Y&NswMa8Vg)+1nt~#nOaKorSCBv2{gwxw)+oL*Da&IK}5G zJC$TiAA*k;CT%*$PdR|ngy~-nIXBdnF5ow^%j(J^@p$2EbhS3&`b&QF?^nJY@|Mft zxn_D7CIVu{j1il~av0_JyJle}!fT`!t$j`@a}tX^oQbRk!9EUuF5j?nTIdGdQQwd# z_T*|81rJTu7}!9~~IgU&;o3TjeZj-m8rY@kmTd%+%aoGMJ^eW!#=HDyIe1uMukM7R$0^?{cOsGR`7+ ztQE+eMC5nu0W!6GX(W6K5P+a{ks!`dVCvG$w8D)jRjy$d>r+!|jAS&f)U4~W+3WDL z5o^o5<*wzsbV93LlZ6IGVGF=|>6Tp0tmhoGZR3eO34YvPZj3ehe>esKFslvsJLvy<90u9^^U-Kvh_Qt>C` znCu`HMBA#fCh+XbYvqCnu;et_<@6TrA9!~h?}CXdprLD+@;gHNI1z^9NV({_XdTs9$p#&6 zNAXT>8IUR5V;x2B2o?=cEk4 zz_#vi0a}n8t%4kQC^@@xOScQ`Pd&2(5$T|CK~E{L(l^WkaofNZPaYSqJ0kaBTM>0} z3#9$7;u#a+d?98pu)r)W2-&-ABRh^K!#j^RZI2Xvd2<`EJo0cgL0q6J*f_xq-&6X1 zfmq=QPG>&cE4iHmEZu)gw#=VmVyeA9Sl0=Q+N@@&Lue@a5|udeHGa_1$AXi+Q|n#N zL#LnJSpJv#-Kr568|~N;j@&vJ>oVgVWpqi~OfOCGM&0$;N}4z;xg=mOmipL`C`tHM z`L*+MJhwYd5q4!9N=$@>?%GT7?Q|Q1j~bOs%MG|P1XgZOGser6aC72!nUd<%5Ip1t zk*7kibEM3yS(?Sx1>r=i_^d;8392cNaNN8ZS4**{WKo-Zhd_y7Qc4Ihi{^e(Oj^5a z-8!fjA`v)CdfEbQ1Gbu*yz1oyR34kdkyXpktUJhAi4nPiq&8Kt2~*7i`P@5p%B5R7 z9O2}KPF^d~X_EH{Ti2-PbDFi~jz1MEIG;;}K)o=3P-m%=@DZiP)Q6>Z#1WROOS!{rScZwb@799W&<37x(wMh{N%$kF@s%aJQ17N zXlFoF3kt z0iO_iK&%#W#b3MGx1AazX2?O~;p0c`=BmrjqG(!dz`o$Ck-Z`|7(-gXef3mublsWYckHzVSEn_5(dut-iB2d4GMjZ7jHYODcSJYgspb>% z${gp~{A)~r%~4GcI;p`?NF_T+*0!!-<+xn1$N-Hw=uS6wEPUF%1c*U z(ruu5i@X`nW)Lz~joL;#hb&jns>KTY;_aZUhUin;NP0ea+m-l$ZT&T))Cz(d5Iy1$ zj|2?`u*>$*1+E5~7$|mLF+l(=cIdXqDaVh7Oyaij!@6=5JAG#ZK(6VD6|N;2T=+*I zjv&P{*(Cmgn!8xv9EO!)5=`0}JER$8ig;|J3!0_|w3d&>!W{H@fMMo{Sq!V~^-lMiDoYOgq=t?M`e7sL-x4g=E*V$__1tBm?f_$8aT*` z4?5Ok;e=9-Z9#zIjGC=_vP#i+FL^$-6=-6j`>W!sQ#jx`T(AIE-)Fala+G-4#TW zyk*y5@j%T=F3J-cPhr}#eX68Qa-D63Esd!{#)dH^vXsQXGrywl=R$MCdc%;f>p7w4 zys=BgFJb!@?NTV4E?J(hF2r>^1SP&HNDLC!8s}N@GEE;|0P&;HyM(d}QS&aebzEcx z0ZG}P*%{94r2LoXH8EaaRG^tX;{8w;mb9hPihI5lsu6FFq?ChU2k}#xOJOj3q6j4; zjzPHUbshx^#0TBzemB##m*iFs9OR2?J6@lQI&bYxF;tEdgrP(!VZC?+1xJ@oCUo*1 z*+6JmtO&aLX^N}W&4;@SY&V-S5LMxn+G!nOy_uo6kQF`7Cad)2apHvItqb+d6t!xx zV6cfg4T-1-C~TjBOf#@xLBT*zea@*;?!M=w*FZZ)T~X5I>QAjNi+wDI0@z-Tn9S7~ zhpKpn>oWIQfrPCxOR{i>1eqSTO29Rn0ZP%OyuD_cUx{fBq;?ASwk6!9E!xIc5l++_ z$S=v}3|ls%$nNWfe~E$jK@R6ya75`vx~qZUVVxAMS!fXJZY3vMCGszH=MLBlD)SH# zsd?7BBfFmsFgCM+bT`fRK)!jQ5;S?W^Q?BUhcnW(#`uq9A$oQb{;1HW%$Blc7w@y@VbPco68dHz>?yX zjpaq7julIR0}qI?-1=WP>zv zXYFdwhqG1anzE+m1585>lrsN4axb@gmXfvv=mai?A>yt~d+1;~rzWY4+HA&t*OQMsD4Up?z ztv==ipw8)Rv^&}2g1tay4Zk2Ophb63H!2GHA5yb0rzn1TID9Wk@1QUg?~j$i0{G&EdLpLNj8E)TCY_ z!ipAng4M^_%rAOvg|Nv{!AZ})=cKc_#!@KtU#o=$NqZgLP!9}5#aLEi=ET*x-uMTZ zq0?ik)qQ$4yf2;(+)_NNTrW%n9pW<^L0}MW_?Lzn9?XOgGgd9U@}>wJ8$(Wj=uYFM z_^MyY1y6r) zN6QxIB+pV5v@IGgG$r*V%o`c@*Q(UQ5~gegUfeaJ{f`9V_!}xTWL>4~l{44&!{Qix zeLGC_EJcyeymd+l0>{PN9X$leq*q#~zCdNH2f>Rg*eQ~dRzo0UtzG7+xN+DvzYV$= zqn#;UA`QxM*^bW1KntgWxM}070!*_cWTq{)jcDEk9Ihx7GeZfBDk{$`y84v$&TaxB zqZolkc^`+up);7xspnPJ%4_5ooRfmeF@cU1hks5};(d|EqjEwt)a)7ey3EFNOfkF8 zupuxGv=|oN%;zWzzjB#}hl3$27pa6pCj$XXdd8Aq3CUYVV?^~j!-HBKP!`WT$-}E% z{OW;bub%-IG?2S-!Ky6~FP%FF4(byjXBrL$EJB8l+|0|Pe3q2LUSN_YEDD){CJMqjl@KYE_2YG7&q$ne+H`Xq8wuCfxCfld#?qhXAg5L?5BH8U4~QLeeyJ z-eH<73HMF>woG+kT^n91{s5HROZPgJNrx3&Zkkpe(iNKxB9XXPR0%kjcZIFX6G#Oa zquN$)uPgvH?t^MtjyFpYn{TawAk!usDv`-d{O`90ioCTbhXVSkC*)CD+O0i-g>?~0k84N-#8YSY_>-FRmNdqv6D31Zw`tnEJPpOny#Z{nuZ(0m?|C#2x2?Z63v*l0W!RSN+dCV(`8SBDKeDt%Bh zGgAY~LR@KN$*8NPXjp(O9A+y9;Kgz!SfC}D&S4kz&v9WUyq{{PRWs}?dgo`=slCE9 z{7|}_?L$$48D?R|b&_&sI5Zj38L9UuvR)#?c4HA8Jnba9LbvTP`|&?#sc*zj~=Nldmi-=3!okSaw|1;@bYinJqVmRPNB z{Ayd=w>VDZF^B`f(_nbC_Bf@+&WrD*m5{5e+m1(v&Dq;XTyHAlcaZ0y`PsA-VbYtRt$VEox&_Xx7| zqj%JVf?hZTjmC24Pq)BcqIV~867I7pjaP@hyrGFK1?D;h&qjqbHRp|j%RvxY1JF`> zIasK*-(o{W&99H|y(v=CeDx zh>0!xyfngVilvEFQCzkc51)J`9qe%0-07y64xdF`RGfpIz{y>UA{IacuPFJbLD-I- z%q`WcjXcACHfAU0<)Po4BwY6lXN$2{#;w*%tTC{(p%HTdWnR}nq#CRN@NHP;dhw>) zmQs1(HcoJ#Yy{l0X*`^p9(4Cy-9e_1H>SKmDj|9xp}Aag8im6{w}}>GmY-1h>?k1c zAc&SquQ*++npmBO2ban~UqqER9CPSLj@baCx;935*xY?SDlwNC8|c$8*eo4BxWZbx z72?NRdKCTMD>;%i5!nllP3@Z!q0^Qp;fka=awj}RCFQ1cL*Mk zzI>PNxaM3aH3HvUmeaMiXd}Jj;#puL!DYT9f`zI06p|#Nb*-hO+ z0Stv=7lUt&I$M`h;}Y6Hjns2xtLhe|*tHWRJ;5O!tMJC0;J-PX!k6jv#a?`mX2%C6 zR*}$PO0x99Jny2$@{+U&qwRM+hNHJ9$GD}p1SqAKCCk9P5D^HOYVJ&GdUp+XPJ2s6%iq|S+e15c+c$K-p9>w20JB`0c(V=PxQRD@@|9L1&)nED{l-rz?7S@|gzqyI9%o04h73S;{p2bI+5TB(Jj& z>-ynwrt~_3@b_Ks+~N}5!efbx=Nrrg7FP%hmY*dwL!OJ&#ftVBC57kZ_j>(*U2 zx8bO@P*CUea!Ao`lJkXxgc;;ZwL5tATDuF0IgL+g zFDTQq&7pc^xEjWReN_BCR{ZoJMoFf=AG}FQs`1D>xv^Y)^x3;)_F1{>4`)};U7avs z&mvaRf@Ua_2w!4AKRiZg6#KPn#<;3k%)}fpmm!+r4AqwLvAB-T#hf0QKw5BOTwFv51|E8{p|kKRR5?hEZv{Mpz9%w^rfCi=FeAE`2@27-DcuhA zE0=c(FAmpwrJ=!*##&3*(YlN2##^;qH?%6w`9Vx8%Xq-^v>4nzp>U&W<{pqhr{g)+ zu*`gNWT8w;?-$Z*Drkq;%Zm3P)h|0IebJ=2fir|JFlw$eB3QU zG2j-?TK11)=ryE&M>E^WvQY0u4>K~&Jl80k(vu494Ag*{uuS)P(~Qk9Eal^jMD$Kk z>J_+K3ep)-ZMz#B!TuAbHOO$>0;@%`@>Z!mg0e474($`~PJuBB&J}f}0|gMMgWErE zL)*C*gz(ak^gY0J5S&mP=i3`Dnw6T|M|wb7je8)x$sbZj>gYT4qIpMiXC0SKSoo)S ztCwo>J+@=!doyydc2>Y+d3<7@I`oEN-J!!Sp%>GHfD5 zAQXbQM?4iM6tMbtW!%9Nvg%gA-o4~mth~=^Cb=~}r*TUMzJ@+P*c3Zf7uY$=D#U+I z<`YIq!NNTAmbec)l|X5a_E)RQOK#ci12V)*FykTrSwHx9f1qj7Ox8 zEnRvqhX9Qcm9~f$9Zf}&n!#2(l(y??{JR49-V^Res_a8Y8{$lYCAkTSR1PN}8%eE9 z0?>tN+l0enmPqUwRstoZtQQF`CK*I+=^FJ35}VYNj>k4;FuGG*&w6@p@f1s4#dse{ zn0MBJSS$*SmL;1b+=hK9${p>VQX-~j?x$me)^I_w;(HQ7*=+T$^z{8H+)EM7kUbS_dyh~``xy}EYefwA** zI9~V?>}Y)ktyWLV!#8kfWaPG}o=ymn?vZpctiCaRg?uk#iz+lPc>{6Vs-$daQ)6!M z1*;P{cvueRF0F4x85~GV@1j**`?xvN+6#RR+153i-l>)_^U)O~MOtNm#+4VWN!fGV zaAa0u^VpaqTBJ!AYf@#8kPxG{FVBax`Ml$(gjDXe{0Ds$kle)$iwGT!lP#CpwUJoh zwC5FU$KW_(%dZ3hzN?dsrX7mgr3M;-TEDgn!9A>4uB}eMfPl^=Wgz^pK}Q3p*!0fz zSwY7z3DH1XK8!p9mgx=!3Ukcyg%?c8q}pTOf(b~43=;u(D|mpJ>_q`e@jyjGJzef) zb#v$!r&7YYKv}4fqmk=(=d=st_x2b;fN^^CT(wBg$@9Y4;ei(AP9vJm`-tk z3~i%8Q2G_$nJdMeqA-ea6Yp}dhm$xWyr{Hf+$uSs>)z^?-3KR=9Yu@pUm`vgP!Tdq zu*3mYFH>V8xO$4aUy2jy!S{AuLyfl6F8uU_#QEtZHi8*JTuV){-qO{WGjZRZ=s@3> zMK~>7BT1;}f~`fcrOnnIGk2ai(v&5Pd*@fgUlnb&ipkM3@~3!joaQ_jQ!rdyd9Rs? z@*j^H`MpV=;;0=(NgJc?a7+U!Pn*QqK7{i2@Zotv0tNB|$aX!Qun<7L=e`Es5am6w zZCzQA73&J(Z@f+jR|!zh4KH2iD6-AnqDxOtcyqxgvqe@DLyq4iX$xJCE+eb2TueP_ z2MUv^b7Fu`9&0O`_UX7VAn>-WXEhf+-up8yR`aPu9R~5b0J|*UK&adw#>%VTLKX=i zLbFN=3(Y5S*f94T-&`;xn3YsCgD}wFg}9ThNmR?Wp{Y0#mn|!`B9~zorle3YIiG@6 zX#}Xdu!JKc6hs_$Esm=1m@cF(RQOMpaRxWe%rf`kDz}|qUu%qv3i|1I9n-V$rv#?I zaVHqdTKLEc!Nb>QX2+{nGKvDv8DMqpp}C`m=H#o?HKm1)OuRyHnksgK^DH?+@070u z?+xP(Um{$yIFPkt1+Nlx;wW3FouztWuRc1bo>1;?SVL$lq}C@IVe?IKDng)vRu7yq6@n2vz5coWglGDzdq;H8%OiOOw>7piX}j)H@X(2TI*+y+mO$ zd|1yLyu1qz9nZ3!y}mn_12Z;=XH7p|Ftjcut6XXjP%gyHX(|@f&xV*Rq@j3HA8LB+ zN_$l($7U9DK2U)n7ifxIjSpquSl*d%9YjDv1Q|zi67my>B9KH<=M;Mvrr&opJY(M1 zBo7c#!xZyEuqoG4TQM~7&E51;(ZWKIjn(2+CK4;sgBL3|7uBlrVB00S8LdAdb@SmX zI9r%$lH*s>B6pqT!1JR;sO>O|QNsnDQE_=XWSdhE*4abr^Idh5-JY_6IqNvWZWX)0 zPZS8;-*$y>fEkVA+Pg;6^c|Ogb90Rb;deq8*B#I-LA-W#aa`Rzec)j7 zR!={s-_|H$453eS06}?!(54W5&&>^_(cS~R*1sqr3MK29@uxs#AdLunt`XJAh`Q~- zvDF8)rzDc$QO?WQ8I-I_f!;_eXyaxwqa*`ctnR;K#jwv?h)woP(EWYP=DtY9$+scF zKQtrLY#`hmiZ%y;QuW#|BQU_bgbaP5f>oLIdwiEt>kxf`XSutUfK0-OSU`fRNHB#> zLWJ)L0 z?ch(4{K-j!XIfbBt^xuD-AM^L{+U{3OI0v4s&|oV;>t%8kg6Ok`pGVz9nO4_H< zI1xJ;p-tsySYD7w>m{TMuVu(t98W1E%1%z14y;Q!q{kfjE<;OaF)z(VvK%*Y8yR@7 z2ZMD2oZhW+@$Iy7GWc3h9PgU+g7{ytLTtk45(KKwvqbAW78&yD!Ru*JoVc*Z&Tq*G zWnO!si^)s}VMnU;=Fp18IolPOE6F%%A>HaV0gyT?H%ipK3p#@)1h@x;s+$4VVQ5%E z-ql%)x0?pd81DvZE&Tcr^tF@<5$4!e-8o%U;Y4vuC-JoU-i=-Bl9jVGx;KcdwVZ%s z*-#+gK)852LMMs@5pv?PHDbzbE7xnnx8e+^ZbPwQx@KN)0buCwPYB4fr?^jAgy45V z>%HESn3|iz)JJ!}SWxyb#aUI(?;kYF8IW!2iVoNyrl`J3_#&;8Ooih*)6S6R)YwR- zBcQ73H`f(dPxioC$j7DZ546!|f@VYvJ`ob~L0X`nU34|8+_kWgJ71JEmeRHJ?rh}R$~YuPdZOXyQw6rvzq@a)!p#;70j&Lq{I>c;VRjETYkHI zXECx-r;eXf$v1XYk-9)Yr+SBV8{XQaC#J+KKykx>=&Q)(bn?VZ3f$8UVaeA zCicQIi4euvcx!~=rs4!k(+o}ajtFap1`=Tfv=DeQJv+Z4?6=tFJKHdc~5)H@XCzO{8SmHh`HnyWYHJ(~>F+;Zm3| zSHRuKJCREsn~>e@11y{@6QUGHO5j!s78#+n&^5R_owI(5Y^+JDV6K%OIU`DH78y2F z$w8T@H$Lj9P!YE6MUCb;QzFdR%V$+ntkHr4ybAQ6w)3({(+g?OUJ&d?l@K7QoTta$ zG3@3@rM}8Jn2Og8u9D0pbXKW1mOJk-U5Bl-;G4WI<02a0#HE9B#tGxEH_9W?nR%`w zdP&7w%>A06q7$}A!vfmdMd8-ywc*H~3t2EYP@jW?xL_I}1rimNF4s$PW^u2w*pk;`_w+3ZKTl+_2He4~dyC z>S+m}<3*6JNWGUGW8+;rIccy;@d5pCCN{oTrD0lT;75z!I1W90s~1?B_vgh_prm+y zQdO1;Ta7iGd6nq&C519NV8uKs%Ue`nB~}l`gUj6DeArGFe)VULEb`1aE*~lgM>Ev9 zh~=s0A52j6Xb|kDxiZT7$Ba2<&L38NaX2_2*wLC>aR@>foe(4TfTqm%otETXj)+k^ zd1l13NmzZu5l(r_;i8ub6vkThu4Xuf&vaZkPDvuY0y;Du+6sp%LEM!K%~J zO+u)-eIPVn9rMpckjbzc(+_gXgV>0}^F3{!y;jSlK?u(`z7Y`g+Tr_xp_PP4WtbKy zeT7noyV?;^ni9(*pe~pZYGY{Z;746eJrURqd)iR5SW0ha;v6j?s!fo3KX{RxO$==Iq7c4hrSuHsGkX^Lwx~~U>Ky-I_ z`o4HsH;`S*lc*{;@?2RhJGGPNsisz}LxVQgn5qz?Q66%{YtBexdeK_czIA5S;y~M2 zI08iz)tV{6ap+juZetn6eZx^*SoG_&l}K>4C{Yk-GoJfaL0lf|yI!m=`fc!8W{(pP z1_@bghU&e22xZC&P3G7%Zu57jE>ZEKWjKy&hm^XTuDG%9+-;pNMG7iOdPTF=)KIqx z9%A<74rulUijgL16NZn+w`dpMDkyaiCX5qS2t6Lw@Rsj_-EAliqA=NZqy|8+{y6wM z;fc(x!Uo}C2>Cv2EDfe_8AvM;dbO-kZ;Vr1))FTs%705=dCpDe4+tM}hgJ+CfxSH6Y z+6p{2tJEC9SyzKWdDorc4BJc12|jn!76tm334Qi~62k zN?%IWhSaUKOIUz_y)vXV(F)WdnWnyx7iN?}5c`rT*~F`+?lIBiUx98E6?{4)#uXkT zIF&j`N9h3@LR42)C$z9KQL3hYg!Qklcitc;DOL~|6pn$ea*bB;ZmApBZLK1fU$lqL z!3`txjg{2JGCV$H4{y{|#Ankx@5%*VbCShvsmy->t-WcaTbnawNtR|9$C83 z5;j8BC?{+c?|_O41Hvqgwjv5rCOjNxQZNMR-EH7SR2BGUj`e*QcaeaL5ov`JJmKIs zkk@IRiuCQQ7>{{VqJ|y^&g#=Lnmf=en&{O9F%l0vC5nk@1N|*z9AZ_g>b5YD9u&Bg z(h9^GajbO1nmNEgZ7_qG4`(g$s#_nqF(0IC#4p>j9(|s$PaUXn?^4wUjHK?d@dI2M zlb>qz*h~t$i)Q8!dN6?25$H4vv<#rVXE`4zxtA-mv*XQyH8>W`7-ONl ztB_S;c3n7m(9PL_O$=YE@nW{kb#(M+NtJLx=ReZ zP_d+{ED|ZeC(;4x1L6v}Z?GWdg7|nNn>n z7*<2sjuk|a*s#J%luv9GK^_Q4J3ev^3RXGeC7*KA2AJa3=Z#iN)#LGII1~nVR~Ybf zSPR#NmhF+cauL;sxUS!CiBSH&{85QUeO8rR_~U7{zW}R9ibg!M;v89v-cMQ#KCe2r zixCAGe8&RKo2qi;x{*N8K<9=yT%@DLr7(6I>EX~4-GIY$r#G9C-MLAD2?%3>kw(fs z@K;v`deFBR*sc8VQ&32inUHzaK-utId7#I_(-fyZp|YCH0s%7QZHmyrBPq6QVl zcNzC()b>DTUbiMB(AithZFjIKgg}-}cFpRvr~NwQ%?vher~Sby!}X8 zsrq&jhGZsF)a)e3Q~|GMLLw5A=2UaQ;Axer=*4H^N4n#xirA;61}_L*aG$!#5nC+~ z{i&W1HLX2J3Tu_EZw;}hS9I*|Zl#d)%X1L%g3?T~`6Rr9&kK4~@h7==m>n^fHzg0V zTEvLLWKhrAtJV#TJK|O04A;FBDDsX18*GCps7NvU&22)frJ9>B(QDMzHYGfZma{M7 zdH|*&#H!{ImrTNJC!t!ZIx@n0w|)yp%*CK83v3+o6_u#mB*%c~?J?|sO+&7Sy9=2$ zB2oD7uiI=M^`mxHj?hAF$RxD-LSTD|=M3J(IJHdj8qq{dVwvgvbSD6=*f3yfuef&( z>&*qZ5Th;$QB1St?a=AZf|%Wwgq3d|6FI!5Q#yrGPQ%G*UiZErJZ~xQcc2~LgX%9k zUt4toLfpA>Ee}3x?a5Krx1%@Pn`*&maM@nA%=LRC*5rc-vW+sK$>4~{?M2to1r?Nm zRUyw8pSG>KOid(0TFH!11gVMq*3O%71vuDLF?Acg(|Bt$-L*if+*}9;?mJh>6IVO| z%)mvvhNTLCCgVW?#IK!r)h%jFDv zj1u2NNF;{J+q_Sz{W0vAo@BEO%Z)YhM~^_4UoTU%=5Cf%u`%DjWkz*LGL$bTUE9^*-+f#2Ej>mZeDsAEO*3K z-Ue=dgmg1?TQSvSJL18V;5vZpx^ouf6YbL=Am|Gfs4(EQrb$zW4B#&zJ$04=rWZxS zyZ5gf&>)i%PF=hcHHgwn5KdMgfmBo|!Fdd2PNOafsk#Fvd9j+N6%>01V{IqB(GO>L ztB)S4t2Z-k8+4%aKcequ>2}##-O}+M*iYH%-sJbozV|$*a#bKquVlQ1FLIFQN+wx~vIbGo2dVo}w3Ij-(R-2g%MVec=xNS-tSvqd3O2z!JK8F|f;2$Bc z0F7^#CX)K!2g6==^|dh<*n%)XO$$~@YkSKJkd-b4!&Ai~E8+{yB5lq&YLk`XuIU-zCsli+jnPD!mevc-Nngy<4`e9HIaqpHN~$$xn=a{Znx!J zyDMnCWxy6iI_7cD2ROqZj?_}ZZfJw>;~ho}Fde)sO>X$423X`K*PemHqRv@7RULZD z8yFGE+=q1wCuum2T`ngypz>$YE#$xR@+}=kVR4 z?~R4;H@e3D?eG$9Zz?SM8kA*MH$^p;(r0f*q9eR%lP{`99Y{wrwyudXU9yEFoQk1$ z5obthbf_%UpDST}zbcC^Jom4?JJ6TzVptWo5}X<3JSwoRXrpDYoron)odJCtZB>_? zIvcr=9@Dd~0t+Qls8)MrpD`?gkC3-}D?OZZ9Znx+Ahw5)J$pAwZff|=iA+i?>!?l0 zw*H}P3h~_`#N@EDngn~fCIw9)Ep(9%taf#c%9^{Laxf;~Y?-SD#^b5(pagJq5kXMw z=sDZJ#EP>T&ckc#M$?`;BZiFtz736grIz=1BVPZc@w#FRRHw{fHi3R7f#S0R^|)X{Qg+IaBl_hMr$`MWdO{k^C>+xRk5G+l-Jec8 z%4!iNPiU9LcniG>E#bfzs6SdPqSkQ@Q+l1;V%)EA4Y&iwJPtH6`b0CK0#i6tC4_2l z{eHkbiFE~n)b_hk2u=us1(T#=v-rzyCybu0mdL`kI50cJg#?NSS(WJH-S!|RUD8*< zht(TzVT{enhTt}UyEP%*WQP`oB5H5RMqJl6reOSf5~{%=M!|OTCr?f+Igsq4c;xg6 z(hnXmGv@MZ1o6ue1tPDAEt(#Mjd*lC#`8L6YCYlsq8Cm(LA;TzMVs6cvF@|?Y^fe& zlGv2HaCX-S=Yz8}AFwg`z^MkVN$uK|$R2}OnD?ofe(rD_(5-xR+-oa4Yth8&seer2 zD^~aTa1A`kRFBdUnSc$OPY#`Ki3N$lIxNL98MTNNODoKyP~x!cFwmT zG+H{+kWYcj1h_6eDoYv#0M}C*bUk|%j`VJ| zPefoV1zQb=;xT0~ImxWan}S#NoOSA7eEJ4(Yc9PPa(Ig&N$w<;GfR*Q*Uh@Xy+j$7 zr`=*H1sN^LW(Be?k8`(DwQUuQ&dqJv&3L!5DN5~Mp*%wfp>&XW0R@QgKAu%+NHBO| z(&YleeggU+hOe-c+@)*;b9)7m$}SC6&Wh?%1TeutM|g@)y+Kc99O&{k7$%ZA3^k~k zQRIv$Qtha>1#mnm!gY8g-0Bl}Cg@sKw71a?G^BaSu-Vq#=rnM>=Bs<`=z*ecGgOkN z!WmiST2tVuCJ-Y|HM`3#P;gu@v1Q2_mAbn1B4&q+N9AqOAq+Pz=F$`g+2u1SS;EG> z%?S#A8<%7b_Qss+6ee~zxoS&`k6MM3#fHj$-tuw;%!_p%>SBgQ6!ZZeGERt+$dK1L z#nQKM+=IZdh?CB3a#q5+XlZHeC$)w}?2}6##wBnvn(xIFAQR}pc4lO^KNSG$!Mnsg)#lI7`GB&;ULe(^L*1Oc8pT`Ymg9r6O*u8yL;@~*BbW< zC#%Xd8js$-O~KA~ITLmVHd$Rf;VlQHoqsQ|vQ`@4&xV&?+!J~OK`#;aG<%Y3z7}2_ zvvlvUt)pOvhD3@0?CR;C9HR(UGE9d3xQ|QrHw_((GEG#zLIv}}UV9VFGwO7*L&=z- zH1=Erip^Z*jymgV&{5i1=6cUi0UYkmc*n-B%kSBOy81eJByx^*uP$t$)9!a$vrFq{ zjgo3J)x81>&SfsUb7K!0o*C@#c$K!uCC5-}dWHO68tMrYJZ^O6B=q0WjKzzoi=CPk z?Boe@#%b-tVO9^VWGkw12P>4qmJyqFdrr)^U{0v{TU1xz7OCDPid2F?x^1Hsf`5UQ6O>jE-m$Fkxnila)Tiq}ug{ zQWL@#MuG5N%(r1PSY_>Ol#41SYO{a@OL8dj&z>n%3T+Dc#Y6yBAZ2&dE~O)?6RoL@fcA>iq_|5nNN}(_9=YITkbj4mC>E9O}C8RtgkT~ zmBgAD=X$6tM3!pga%LJ{Ej9fxmeuhVr>rp!6fmnbAF*+r7TwO9foem3P$5Qqky(t_ zD6_sh6(<-e``dI2Vw3}QAgHfN7u-T)qDd@l^JjXZ;JsuE0hv%5%c+vikmHsEIz8v1 zhT}m>-eOS)rV!6TD%IsDC%E?$$S>_t;OcdmWd;v-qyXfHMPRoVlazedR%woF<}4l! z*aRX=o=9jaNg6FuNL(6kvSScgZCTnEhqs!GrP+@KSyRqNjKbq^Kge>|sBnA~>SA#< z4KQA@jkNTZ-Ax&ePE5y%BsIdZoL_wx&SJjT7~bQ{?ct&o5qtK+q2M;}AQ080BS;!` z0WlB@PG487wD5w~2M%sQK`#ilKwmexMq)1;QEyCSsw7^?hA??YjzZBUTiyYCeQW7Y zBjJIqn0We@K4DR}ac2zM9!=+JNgs&7v)oP!*}S6^$5#ws$QB`cQHlbD_^Wb90ul5PozOCV?-fM?3MA~$sA+iEfjX4piNj3OhJ(tPJBb(gug-f(;gOhNa;qsB?4trj)OlUlFOL(lBKdHqd*Y|n$Zg!>k&5j@gWMu$l)Wkk#O@{_SudL`z)oOo7!xmPR=3B;z`<(dMUyH`m&+0 z1aA@ukZ}(~c{$AWM=qJ2H4+E0NDdstt(OKuL03Cr%!hEuPS-6RSTlz5gG5f{?hYOy z^60HfoqVi=8jL=>lDpXD z^LdY-i+M z7C{QI6jn@creJC^1SA?zZR$#UFm4WBDF4I<$q(=ppyhEG27q0=KRQ!hMMoG<7= zoLVW0FA?Z1)>E%>2_=qrTMkrURY33{E2llkA%2=ySOuziluaVnxIZh-bs0*j-e;~? zez9w^HDQM#bB!y+n6}H-Ol3|`&z)RQk*4wN(<_x|{g9O%8 z5m;_-LgTflq~(TzmJ)0?yMu}>4>yn@A|ZUR&e+BoUi&Unsuz`9n^}Mv*G949ijEm& zt|g;P1$O{xK$gF}iV~q;S*}kJ#zzn^tGuh`@W8ZY6FP^FY>@WSA$gLrjf0)6OJB^z zQNF`$kQpn@phkBTi$=Z(phYP=Cf$Y#(DY9-v|K<~c08C(QL>bV)PgigaLp*`Z4wMS zYY7BerL%T>)EXcv74+ajKO%C?*m4#M$37i9rjSN>!$y<@(n2Mc<2d)Uj~%;N>b2l@ z#aXlkLR5EOc==EUQf#P2^m4MhuSagK2ak!F$+G$DgQdGo=XkFsT=5w*#L+7zpNCXd zIda9rvP|wl`FH5Y<|KsYSj_}5jRwU@TVP4XTz?`~OR56di4CF;km}PrqeSM_s}MAd z>J5mM+-ae0&|=SoU?y3yW$lvpu3nw{ak={Q@#~Yrx5uyDQvNzaWKD!NQM4OKn-Vsn zwy7911Z3|+XL`{gcRcuzkt)!N~ zHc_&vw1a3{RTjdwq+3K=Rkp!fB^yPwDA`2Xs%-Ip=_jVl59e4LuE$QP}wSN39&}V8!1~vwpPU^)k?`UwxnB9HiB#mYMVs1C^o6JNZ3nh4TZFeWhHG!(XmL{sM&25wFcE& zNwSr(3u=yiRV}Iu zX|*M7A~qGZY$VxRB`vB3)T3%v)HalDNUf@ks-d+;)eWSLs@W~1TSaWDrqNB2ZL*t0 zEtM+T3uzldHlntbHY;Lmg=|$elGz(Ihn*|ebAsjxO%0NSeB zB{ov6g(k?_rL?Hp8$)GQ(v6a>pskeJp|u-S8&hQqWZIIpl`X2a6||_@iM2r5MQkR- zt)pnO0=A0SrodZEYMUDcHj8AI)l_Vh*+sOiqAh@JB{n9+Hib>8O_G~Y8#Y@jU^bB1 zL9~IjR@AnL*lZgVEt^)*Hh|hyZB;f%+9=qzQ)(jGfwHQW#WqzfsJ5z>&|5`YNNl5I zjj6Jh%B8VZ)Han3gJemtjiEMFwxn#J+EK8!Luo5v6KILD*rM4sp|XV78&svRhSFP7 z4T^0=TPTLbQ)O);+DdICZ4tFCr7eN6Lt+i8NwHP1O{Cb9Wkt1BZ4$Psnr7e^rWZAM)X-jHVHUP~^8v|)Z z)TYojRBc4s7SyX^4WyeP8xd|H*$tw$MX-j%2FX)uq}Zm(MX@Sv6xxlGYKgQ-vPrcqt(2D4Q)-)3n?$w} z*c&P~ked=lWks(7RXJZEu|YRg|wx#P;Dq|6|pAOR?5 zwN0p|(pw_h6J-sl7ST&#TWl#C3R^0xVM}TwWD8`QLN=Y*5%y zwI;|zYLy#REtRqr_|-=kiLy45Z5G5CiEM$iQrJeon@F~*TM2B4whGuuwHDEBAlj_T zTPUrF*e$9Jk+n&&64@%YsHW1JQnsizqiHKODz;Td!E8%qM#k34Y$({Nu-cPkn@UZB zHc>W$wuH8n*;Lw%g$!IzxxM{8kK zBt7O#8m2;CjiGA(!SHz}r61nzhX8z4gvc7$8Jc zmi7IUbe{m2G%*zw#1SjYT=&ykm&*Qll#Zq{_I86u9Ha#1Le1o5@?Q19OI}|JEKh-w zvG5IQWdrtRpWB1o+=pxE<*b)Dxc$&62b7ZkX)!32oJ(PGYJs0;}YL& zH*@;-ywerTcJN#Qh!`?;&0-!l?Vj>tRz@4RDJsc`xQe}u7su&75MPrTxv#s2>ht86 zqtUDi$T@n+GNJtA+rvGeK?fU^Ke0-^#sC6HMhHIy)rk6&unw#?UCJ}A2_2H| z$UHwy4kQ)`--La(;lV~wnE~XDevde)I#C5m>7UBM;OcV?uH#casd4L!(H6ec%$r@*WrLux|ADWjORVi5jwVbaNZ=|-MQ7>z-; zD;gDY;X1ISodbd5B=1$@SZ*zjsz`!)R{D?CR*%iAXcyd@Los4-U@5-_ft^WRA@RnAU%vS4oBwurr4+7LxI%3+ngk=Q|p%U+W# zqPwMcS~wv%QebqsiPuvEJf5LF@IJK$uRWhsoSgbx*<%!;(m~36rEPK6XCK%^pQboC zM+?#7!n0~A?%hl0JL+=t4~t7O;1s&Q$T%YY_7vslSz~)ijmI%WgDkzf!*Q3_IDmB}+2{lx;J<$Ap zE87X12?Hp{my+4HkHN#m^nBt2#d1U;B))YGKq09LI-*}(Hyg25fh7dISvN{JYY;Yj z6;DW-5}CG_rg1%8mC(#7&U^#sZ9AQH|yC9Xm2^pSya_fjR{m1>T(wyCA!k;iF*gWm$YIa<9Ef>8W|0FY84!T^Ke;rbml zMTI`AAOjzae)z-%Wsm_<%ou$3G7VHkfngM@rf6VZnFGECn-_l=BT^CiH`lmZSvT^Pq+^Fjv`R zTC~++fK#atesm1L73C0zToYOyF6$-V1)>}E9tDrj)$Y_rFzW~pjq12n1h6TlaKqMs zNJb!xlw4KpsBjxDE~Tv0RfVWd?P zM5FVKGL$itO!6i3sDDLec5j>kxqCEYPT#wqUe;H0s5?oBrf4$O6eT8AK+P_?fjIj_ zKtz_GuV5f7EUff#{G|ISb8bIZ3)!;R$29e~JdfMvUqhd~(=6!q?)GC39jc!M2BFB+ zalVg*Sqof1@8w32Sio(4&#Lm#sVJ<3`MvJU0=iC__1Vs9h9c;ypJTyCn{I-? zGlrY!P7Fge0VuL9#DHBFC=tXMQ<1E>BFl)xH!Msosjfzqo0wz#d2`l*`9~<&qR&ATL zbG|yAk6e)vu3VT~@s8FEIK&vUKnVywqNmvo#qiHLZvdf===|!5si$M-c%)TG z@irMjv6UGIPl&A8$+0ZST?voiwt7m0ak{RaWpU@k@N*m?L;d5x#X4Z@CJke_W{ zh3mPWOXfW~+o#oK5A0~whlhu?Z^U&7h12-6&XZ*PfL(YN9eoAH_-3kmGmsG}d(|;w zUKEW2k-!}b=e3+Hs}A!7RXPrcD{iT=sRN zngua)z%1G}8y9I{EA23uhm%j8suR2DZ&85HX102!>@}3fq?(%G8X*k3IqC;2CwF&5jL9-YJ5<5CTA{o>WY(FP=u#E}s7rXCGL;{R zqkD)i?gzHS_Hocig*#GJg%Hhnc-Dd)JmAF`C1To}+8rA&Nxd!Y@FoNf?f%kMT9QRW zl(HFIk9~zKg}Wy6n6T+>Aqq0LA{?Jswtg_B*Yn)858K7Ejo%1B?`Zr!f};EbIjyEj zJT9t^zo5;+FU%|RqP|3+gQNK*tLWV{BGWG!5HLvqfd?WgRPTVK-l@WX#n^)4&a!hE4n1hH&zzfg{Bi;Gb0~b zW7Ollo-{lW?%&9$wyS=)=9YC5WF@$SRN@P`1NRU(W;Ghrz=A)VVr!?KtR<1hGzlQ{ zamuT)F9e~)pdd(E@9n%{^`lU6+#(Z$cp=|d%cOX~N3SQ|@yq~~a`=$2=8k!*gTtUa zi2F1qhwOOs4!j35^e~PC72~GMWsP=nDVr7q82mMB%U8aD$<4m*1spyrnHdCSB9qFe zhLi9kBP4cPCm)?vavibk&y|rCn3#q_ir)_{w+fPxPE7tnMKBYRpDU)CD!$M%1vl6D zHjv~aF+vu(0vO!Ui2MW;g;%0{1ErN-Uv06v{bLOSSI#SVDzSOczcE{**JF=0V4+*) zcICNo6ES_|p$-IsG7OZkSb)yixEwo2b*s*OU99vx5OAfqFqEwp7{KzyskN5Cu@sEr zaxnCW7$=FE20Nf;7z7P0=Hlb!$$6F>h*2H%Ku%+xEjnWArwv_je2|0X}I&+30k{*a^>hh8t}y3@PXNa56oLX-spnxz$!`wC!RWw0W$Z4KJ^rYO

@I_g8pdMpw=w7wA1)l!BtV)fQT7uQwgD<_`eC;E^P0x_s z!c0poa_T#NtO;~a0?e3OcbV1AXqZ*{<|V+KIXMDESiU9)!yx3f!oGNGR;Vc!bLo`f z_-TaHl~OB&{O)dKu1{~dmq14ijjYZX5?8gCJ5uS?4On}72sBO`EhA^XvD{a6!RXK; za*i6iWCR9^GubdtJnDTJOb}4n$ID>=;K_Np*^tbTgt#U~LQ<{Z9~%UU zZ`2k=z_cT`)v7D?JL5^>8E1qq$w=_3OR_es6+d{I{8IS>d{k(DFbHKUPm1P2rq~TG zLP*@bFG}7W$ij4{UCJ+vSG`8agT1rQfD|!pLdT$TG^^NOb_S{aB&P|d4k$s37--${BA0N zW5xDH9ZOe}yX59yM)aP1a@Xl^47go0YD2NZ;nD{QEx7Xn3@2ZKP31c*F=+IFT$azt zBqS07dl7L9jY*L!XU#Zc@(gC(x_O_jghZih#ST88m1!-c`%SHjkAM;#p)(*fG&1r0 zNM4otOo!ERXLs<_uKl*cp&mk-J1#r}ppia$nUEgR%JR)9zVf?bI0j*0B``=(Dzqx? zI3+Qm$>-6zZh$MPec~E27re`$o$=~KpvXd&Ob-2Ur!!60#2OV=Lm}CKqIS_BwB$}6 z>QTDe1EK<9IE`pVPSdJyTC=cy!0Nzs{82cb`NB06Jec^V@mK*@w@X6W(TC*1`34d` zKZAgX;OCc-N}I|@br!1c9jKMQ&J+wLUEhf6(R&u=8uj%`QLkip8XWm$*2)$NhsG-W zRxJrDO1#o~B!xho&X%46_ju>9^g*j@qQ_=rZBOHTw3Ti>`e}<$2U?IOI-?+NKkfG; z~Y1m7*V@80tuVkW=U=BgHY>;~QS=%VGBG@4 z#_B6n#Qhj~s*)xUfMx{MKzx#@}hy)vr}r%s>7-cU(q=xSHhEa`VCQ~(gbr66xN}s zN^l{mM6GG=JUPfezN02+Qqi(=8=~q%NfE4>F9BW+yzJoaebQ1TAxVc411&tUb8rcKTOAIZA&F^Z zimb3yZB7Hjpc*d?Te?9ibp6n z!j*4gv(BiofGZ5gvL1kOP~_j2Si}TqinxJ~Ly(Ohq!~Ka;(4ONqwEWH><))J?QNb6XIEd&VG3-Nex^>9NZ1UExH@n^3nvjLo z)D%1>P{5v~-%}U|n!_qx+!QI45YlI>uS$BuIx9Z&UXRz@kP}MA$_*~viuOCVDSJ(5 z{^{2}C3Um5L**YWN?1m=7o_C|`S4sQ7&5UjOmWW7Rzx+#C{TOUV8iEdM7IL^z-m|= zZf^B?h;$i_#8ruE)(kh#%vsA`1SYjC9Fo!RZx0Vt50E7V*w!qXEL-`OFPQ{b&Z0_Yzz{jpTGB5N z6lIc>(^OgDk?rQ-Xdc>c))cEAGPvkGD%EMGF=Vh z%b#3i8|TQaecMyNCJp-3fhV6=Grgsv)1@}jmnl#i;|u8PV7-GHZT=EecJy=DvPyvtuZgent( z^<#jDj@#F^MgpbE{uST@mrsmI+rt7Ucovmh+X%c^XZ14-{1Q$0D zf!08mQ($aW4dDwYFiUc9v@VkFb-sg_ca(Vlqz(CR+AH9G>ij(N+5tvg>tPxlUM7CiJ@VWd{K{@VA>oAlcv2g{`U>}~uenRY5(e?30yvQggU5Q&%#tKLgCeMZVnh)D_;7)ZpAe>S;3_3#` zE|E|!qri`@h0NztwShW7oVvGuOLe+1^IMa}izP_B#}pN{ZW&8gOK&z&+}USAt%lO0 zR6@#yt>g&MzJUetZu8QHP8&HmxX{Asg9w{$Rca$7s~#KCGV<5FQxorZmV_`_gm5Wb z>R6bB3bii6>SZ0H@%GHWX~gOGhRO`RiG;lwdFp=_h}o^76+Qds%wh@W+V-fRi3N?x z-?f}ULeIECkzu*Mg*CD!rsQ}czhWcFJ9INTJi%x;-dc5|!7B@ZQQ{qSgo!iR?XNBNqCzR^;Yvp-41l9YQk+^SQZvM%?Er60>5E70_8V`^wczPf(8fwT+r5sCo4jR6^s^ZZ{(;wVR8= zmZ)ChWo8Vwo}(6nc(?B5Iy_e`Swe{)aVPcbaUC0w-=WnSg!X3vlMrR#BpJ!F!}7ALD^>!oxMB+pTF}w6?z-O)(;+|%evENZ`dL|2 zAb0wz453nrXYR-_a zPa`9snaoD$0`KCD38ye5qBpKEh~VW_b1abSxTT8M^e0LrKTdk&A;H`wJL(CJT8PxC z%UW^Rxq>tHJ|I)V31P2?A>6{nZpIicjQZTL`5~nK)hMumu`tezJQVRv2_k9xe!F++ z)K^-LFHw%mZUMJ_e&dfG^nhR8^8GI=9pM^uae z)X9gHWgUIckO;V#q9&PTb`?pv~dR*q7!<=gGfFL@o0w+$|2$T zkySoUFQZVT$a+2@2ugfEoXDul;6l~6x=8>{g>N9DP0W-}?j)$BJ-`4E&Y zNPO4D;hWQQ5rmWPGJ3RN;^L;)CI4M> zk+>BDRs-aP#Y#-AUit(kG7)^@R3%b1hG)Xz?7=71&GVTQ(Wg;+NpG|`w@yuAb~evC zsPDRM5v{X=L?-G!dxq0{H0UQAK37(e;lUttVT^=J9vto#VI}yI*;=>5mX=0*-aoT0 z<0X27oQhds9un+FRJ_2Bswa4J?JkazO%~oL{)h2 zcCi-4VGnzuiJLvfj0E9Y%CAwG*`V9XJbE1A&f{^CiJZ0A#5{Co}#Z+%~ zQ#T>SK;eVw&1W9)#*Dy3QC3v~cs10LXq{F=}2Sb7ohp-SaP_hwtI?gnrrLYKIZ{I}ij8t#D z#YJ6wLqkc*6n5j7nhVV0bjZ3!hp;Ay++2f>Diwv#5We==yopbc)qfh`sUk&FdAqZ(j_}+acCFVLo2; zaw8XR0|2QEKEdJ9QHLWYHW?TMCFwB43S=^vy85__?#Q-1lo!11<>$XBm zV?GqbXU0IR=oCtGk4%f06y$J8-fNxA(@nl$sD`z;HnOi!kDkDQoCFYKl2i+7^s)d4 zpa&lT=FJtO#ACwLf3_(b`$>t7wUuJ@CXOljK(NiYp@#vjqBPC_x-0N(aC?I*r zq>3MOb#l!H8>@D*@5)&-Av{y5UrBIu{DlaaW(L4RWo}N#mP=yz8r#sTz?C7b1RWZ8 z4{D4A3LLtT8Y&=56bbgi4>2^v_#n4vEsEBFLd#PTlf?J$uhSd^lH?S2!>RebNJ<4% zXOfZ1Np#_nRL2U@Vf5pGXlhmE1YP92>`0j4dS6C@5OvlPK*m zFnk1YW$zXVMFFhPO}8{#-NiNMg(h zAWtJcq=?k2jPN%IddWrG6}U2%axSix^xptk7O}?fs=e8wl~oh*48ft4!--x4l9)CC zp^MUid~dql=F8L;)Lzel{Q|Cvjps6ymf$@_PCYblTNy5a>I$lb`1!0HnJ^1WR5e1j z0o&DnUIgUSlVbYPh$Yr>O@~FF63IZi_4|Sr1gex-Va?t{Vc` z%a98G8C+wYw6uk!@WbyQcx|-Z=tt}$UgXXAFk?k#N{8P@Muyz z(!mtgRl_jWgE|@zZFD4n&)Mj2hZ5Yktza>|k=k+)BnQw-VHJ=ba&0cXh9VLXq=3F( zBL|1zrmn-J#6A{As$&+3X(?9sLhf|4dw3WC!dC0<+X~LtO^#a5$#>82-;~-Reri-r z=n+37ki&0gKWB^GCU*rYP!uWb@1nx06z-b2bdjh!-O`MR~`419W0WC1N<*N6Q5`?2<(u zk$*MA>`UXBaBa#G1=@h&lO}WUWz8d1`qfX!F#wI$&~l>fGU92ZS_w5VAl{X0>lmLQXD`~6D{!u zNm18I&G^NvQ#tE#y!sM|&U~Q(MD#|zmgJlUz2x4-?`Thlay%P2B?I#fi>>ENS^UEN z^YIb+$zs7tsDM>qVAE~mTF=Lt5d=(X3qgD_HAf334yuq#k;&!z9y6B(TGjA$yajNQ zLDDD&#lkfR(Z&Ti#KlVHN04#9c+1~UJwTld$!?Yj-Y;00W?k5zYEk)KxxLTfgzMbx zlrh4U!rRVyH$hsUq4ug4E5HkeK38o__NGruFH%XMDpjPT%*#PAS_42ALj@(tVDdfD z3$apA13P4(aBM{nQv<|OSY8A)VAhvt4LSSfwY6bJ43r7szJkvbjLa61Q;*jEW#WjTP{JX% zN8{Wm)j;6GLtTZSALil6Z@lRVO1=A}P)!TcRa?y|%=+sjF91~HyKCAy^-1D@@ste+ z8cRpF4DON4dBAtyV1fxvnZ}2ru_a%F zBbVVWgorPcMd#g2umV)nvB;kISWG9lz=D~OhJ-xxAj8^hYr3PU-Uev_Z#4!`Ii!U) zYgpm?3PeeP*LB!9C_C5Ns~#(4m+2rL0WByG7Sz5!Yc*ULpN8PN>nEvCZ^j+wr)_H0 zEfn&UDvf}FiHqz*h@Wv9Xe)XJQ(wVmgH6g9N>qo0hg_rs!Zi zj3SV-I=~Jn3UsxaN?3SBWF~j=IV>W^$b0&rpwNN1fTwoG_iH1{XUdtshvY>A>6Kx? zABNc(5XGN8HzQ>~M`K5ij)!zbqL-F}g&0FpzBCz^IKLmDW*U3j8}CcyPzu2Uz1&iK zpv9tKg072yG1+#s;$tVDWtXWi2uq_VI29bHWz@$uo!WpD2Mq7WV(PBPAd8XPSd^9G zmQ}16p1VL38<5+GQ+6_~xvwGuZHQlwdP2|2G|am;x^s8}ETtG%$t`o)INzOrGWtTI z*N5T=inHC+`Z-pxW{$C?HUQ*b?iCB06~Y-^b6NwOeSGrq%bT}f&%w(`zG1fVGTQJL=R!i%k%(Rc zY6Pkx0>6fy+$xZ3`c+G8(AaeA)wt3)C{wX-b-HO=I1>G{?0gt_l^%BqSzZE$+Y{pD zB+-~4>IvYuVj1w(MN}TvIeTU`P?%!U$5nfNUaTA_5 zvSbbN=pH7fz8nlif_2t_;CVs3S@*$?Js>fR^#h< zUy}1Qz0uS`B?PTbn!p*x>Z&t(S* z%p;%9huh0N8s^E?gQF}8JIh|bY>vFgJd6r;WJ!oW6QpKM74wH9tuB>1s#lj0bPwK% z6SK71Z)4HqP%!EgeXZ`A(){(n1FGs2=Zh)?Ud$f&eh@ZQcM?k98XlL_K1CmE-;`QY z5akJU4h>g9U}X^^r|JP8}q<6{@fpDiUin?&~arwx@XcEl9)y-C5&F=QjO z1?3GDl9wVCyY*DK4y+zqMLGIZ@bZs~7=-|^+3k9%h~3nZhN|pw+r~SBl)qbnZF*ga zf_$A@6E7t&`))FNGGX0-XoSNm9CpJ&N^>(<-KEkB{&VtxC)mnuje?YG_NuEBsiF4< zD5(7YI`;;A)0gjpaB;Dc3|M0HXj{!sZ+3*ZBLmu>7%#m5YiS{khqX8Y7hIz3-$SBs z%x4bdcwoTmqEh+7U)-2b$;Imx&a`Nv^tV$46;;-v-<&z}3Z%?dL8VLHa1LCm&nCnz4GgJ04qNLWcJc|0 zmMai*aM-@B&_QYS zrm_13WvU`#Kw*pS?ga3}CuEJigaIE>V$srjQM%q~1A*=0)|DptQ4Q~-8!_S(RtRd$ z4DSs<(W)Aqsj>=Tsnr+GRl6gNoDr$x@g9%QWEl_SEh#XgF#;1(3OpnrkZsX&H}$S5 zfYxbdfrz(hK)Sj~RF`I{0PezvQ;V3R%HKpL4|0f$FL2oVLtsG^oE6AoMDgD66MRw~ za5r3Y%=WosSRcG;%F71oLcB3cPl_q+3`4x7)xJ*K?spl6;haeqKCOYIVow1zKLFRv z+=kvGe!pKLF9li=eboS7o`8P^btdm@tU0tmxdR`vo1oD31%Bad=gRB?%qZ8>3YN(= zE{|Aoj}k$A+IL9OQbQ4 zZ7n}~%%PAn!k~kpOCnuXt(JTtU3Cl7khbD7&j(|?!KwrEQb}11JMVSsiuy$wj3`U% zG1Hf$k%AzQ*!gO^HFWwme2Ei;#;Ui+8nx9f43pa?Qnv)-PbqSmGmC_yLCo5|Ih9yj zpL$tjXB{}?92|s@`ZX{!5l8K`v|->x+`lw&n8VgIENPhn@^zE|MyxnwNzbTqz_pyQ zy&HMn!o!%Wmi!0;AsBiM4u}|P3A-D3E=3fJ3&NxIs5iXGUqN*tE3peHCp`k~SJAEr zS?Hq&(isqDZJ-r=8r`z0`U0blP`+}3#a7d_3A(poX@>Z@kcT?dJz4gft~d#H#VU;< z$b#ow^Af7I#h*@ZHeP0yriU}7b5Bb}H9NT62|q2E+trk}0x<%YIzx&Y8$y3Q;m8VM zmaD;9Wlvd=4;iUK}^DhzHagdDu@!cO?)Y@}R|;`wNdtAfWy} zjb=9{grI|(?cYN}pzJG<5vXSIoyoXLg=eYA3@05P=H!*A>X)=R$&*e$SOai*2&Q17 zi>y7ht^jKBS9IH!u8(CW=Rs-ldsKI62zD()dlHUtMek#(G8<%mAlCS=&&v+_@PX`6 zy&SWyNmYt&TrHtV1Ka_%E?W7y&AJD}HRaA!t1y9a3eJM-EXBCx2YqK*`4BFyDJ6qa zL@*);)`hX;t?cIRpGUJAMhl{4b5_YzR?zCTe1*5+k?Cz6cQrS#G9I^4UzBPhS~gxt z5$us`CqxWF(Q#7rxa)CBg;hj#LcZ%t!u~SRN54wTv9baX=j=nE>o8=b4T6iYC33VR z+s9Nq3N8sUr+%Z8fsyPdvgUO#~!}rK!q;h{L_?VaF@%#X>c#v#l zW=YacfN?k%x0|n#gEM>63zyVc1MVLPgKu~*kzs5Hy3p)$u^UZRSoPqHELr32*U%9! zqt{dEdl2fU%y&MC6q{`20z&+UGy(%c@b8WfxQTqr3A%H;cU|MxDQGGX7uZXM_tFE| z`RmCYV!UHYVJ^FugHs8T2nZTmG_O1+ftN8uS7R%5`WqGf2Ur!~ZwGwtLj|K#ARx#= zRC_A0S~M5D-!U^^GO$u0fBbFbOutjdU^LN90 z=(9vI&C`Mc8?n0*B;$4RbU1pL4qPR0yq z1Tx}OM6Vj3IF3IL0L9@z%W_d;Lh-TW>UT&Nu2ylHQ8%MUBC^E&>SSgkH`3)O?|s?& zh)wIIaVJpcZ?A52*KSaJX#SI|SBTDlq%<&{9OfRE$T8<&+vkGP<$ zADJb2M)MF(E0jj<)vG!~+YBzE_KM9muWaC$5Dlr~+(KiLSt}Q%fMu4*O@JeRa0*x! zr^Hmr-Qx-H{GC)qRRgN7LUoqOMhV4qZKhP@!}$^dE({!k;&Raa*g*mkA-z?0W(ZlY zVy^q%Cs#b!7&3m}m=hkH&RAvnOQ5Ga*!i)I%y>hMLb=n0M@?MklbdHHe}Sar*s za4(pVv1Z(oLf2(DL>Jo{#1K?u606KxR|Xm_mS{KG71?M#_OKmYK;BKjdq>Zz9E2Vj z;c3&|fbpP<%KcWmvIS6o3>3?poLfASdLvbo&lo z6r0+st`Pn=ac!~be{AY~r|0M!p7TI(zoF#>B`OUQ(44Srix z^}@H$v@lnic_6HcfRABfFaibfU6HB86 z`NhWZ7&vU&9K`pBrZ>qEix(g&1XMld?qSQY%YKs`f}FC*46kP)VHhU7l211IATU56 zz^O}#=SS019p+t2p!26kB{lK)pwB_pCUYO0WO9hTn@aS1Yhd9S(QV&|{l~cG@=4{X zUx5!ms9Hz}Ze9^b;1_{59Tlu0DG|=?#Ay}al)nsheeIWj&Wnh&eWxvl~za0aB7f*gdj#AGeQg0r<(9-m6Edx~Q4II_bNIZMI8 zH$vE?f{kC2wz|Z~sz@PGP|hERCOPn?BBVRdyrno+8_KuneF8WUuAyPVsFH;_9Bz>d zwUb0vCn%EH`K=V)U>5~XV)KD*G6Lf0T3WzQBm|am6rW^gfUSrNAhcBSG3;8 zIk;WJtSChyD1r~B=+0MQF{oFqxOiX4GdhGIj!A;%_3 zHi2IDz*IwWvS{=ojcB;uEA+BOO~G@Mh7*Wzvlf%oYjA}M2xJ+V_(ISN3*bYOB$VG- zM(xDYbxkgf3*}41h!gUQtm?SxcO0;Guc-i|d|BNSkSr0qRqJsf-6&#=KZ~p({vebp zV;`(hprP?wiyT(2J2kPn2Dx&IinYgATA~bR)MnWn1wT7h3OKs;-@zft3LX}(kVpqG z1?eT1P?Ev!{YV9B%6s1QRCQyqh2pO&^7lY(rh?_yRRc?#+Q{2_Wx;P7!~*ek_NAl- z2xMva)J&xt8x1IUq`h7u>s>F2xBz2mL)X(~H0I{83rsK2&AO)WrHxmdRcCiHtb-*w z3g(gPUp2(gwC=)@eH`G0J-H8phNCQ4sg_$+OGLEyRI@g4VnX~#VlcRIVyQ|G@PzXX z+-`S`#2k4Xia@U}k6pl92pUL5L)*qVY@o5IcMfUCfmV<276=;mDW|3%oTxDx@|o^t z(b|ia@}>Y)cec*?<3rWMUKzL4`8Ee}`8D_~fU%L>mHIw;X*4v$mxaaYcNfY<+GR(H ztam+vJ=p8mh;9}e0XYrx&}56xi@`A4fC95GaS6_Wq#$#}*Hb3V!;90M=iWj6L%E4S967M2dy~=h;6yj$o)=|8lGLt8XQ{=VTW06jp$zpR{DtGq@bV6e(>3?XDzy3VYMq+Id>h_)EW;%5%I zA4)J~EHxFu_jSJoB93v{wDWl`#)tz|Ziv4RQ=T>UzUkq~t71XiDheVL;qv0~upHq8 zZuV>Uj1qSe;}V>tD5T-^-Nuk6Fnq45!C}Gc#9T%Qn|^0r(z^Gg#G}S7%p#D0?3#vw zA$&{NBiDF^H!XN81|`%6D0)QYVUgexS`p28SzE6?nmcYS5x*=CR73-hOIrOl__V1@ zx%Cl9yt`YmneI)d!NYq^_8{2Ns~t;1Q=L&`$lkAEnn3OlK#A-Mpsei;)iBumzhjj? z0nA=e+CwF#@Q@K zr@Z4!yispPkhZMoyI?GY-Syqh$-kq*Fr?hywe>+`l9hL0Az@w_^Nur@wI^f-;Z< z)Yy-=a>s|7_($J*R?MVWfFx^uYKk4^s?=zs_MGy4=k&_M=qH1983?q4UFgo+=c0O4 z#|sV=Km>D9!ElDmIJBUn8muZ5JTQ6K;S(Le(K5?D{*}b;g_rmE#w<0IYs9r z<*jqfIgNg=hf~J2u-1#BuMjKse(wVI8?W95h6P51p>AQdHeZ;ci<}Nt(!wf8X!%$NozJFyA<&~eutXWB^Ds-+5o8&sodesF4uIzZ z!?{S%{?qz)m}wvAFJ1W$P#^@ID5ED!pC+v08?zrF)2BmvS2-rysp(>` z6g(Uzfwim&12m(87ez)P3w++DHecT0>h}tqxIxvc14AVJ=V!IC=>=Yf*0@9BWqbwW5~*y2BKoi_<9zc|%YG6(16lOirhbzGbKGIL4n1%a%7LzDSEM4Hr4{O&uDPLWnL9t{hdT~&3 zlG9k@b`B{{w#T>0bE9(N{=_3YNDOdK(^v_NT~;O?<|bd8*~J`6LY;xWDH1x{no`00 zgweh~67cfu`do9ss0T1?dlfnlcVWZY(~#5P_xV0$K#9u;F{x3Cpj1>^h$T{bT{S6Y z5==ZKX}&lW#)Cf-vEG>>mqMyieZvN3Vh%mksmNxER5COmq9g1jAD}~u8|iX)_)KVC z4;4Ww&g4pa$0}>m!6WdB1i8MISUMA;>igY|&iX6gUzfOnMdn4(+~kM!AjB(x`RGQ^ zm^=u*TX~Aoa(rrVtFzCc&MFUg&*pO<5mzr#Nkk@7`%0qJMoDWJCZmHTu#Zk&(Nh}C z8WpLzR}4A|6$Y=~KOO1lvZWGTHJ-N6>Rd;&ov!>%-EBJlBH(o1Fe6$@`my@nyw%Zr z5#vjidL|Ag!&s%G&FS&l62zCnm^_|I<8EXg8Q|IKTScZ9aPSUZW6QboQ)6412ksq>X3JfyfwbAUgR+C$VBiez#8H>Keqr)~XLlnGGn4pm#85vu_wn z7LZ^Eg^^3{Zcq@uWt)YS`1ia=AuyPXRE3m+hPm*tQcq?j(6z+WYa6NciK(<2N0W8a zb!skT{fP|i&OBX*;E>HDZC{)B){wY3mu7>V@lyE=kw10$n|AlHo@~wr=(`FzJ<~jB z$Gr_HwbK)GZUx_YU@O!^JI+nC-;YjmVNOUIH+E^6APXMk2NEL$#9OzyTqe84LAz%j z7}%0GEe!|Md?$D}@s6c>MEK1REM`SGGuHR<1rg5?L6L$Ad{a>sxbfl#>-N`!V(*$q z>wQG>SVdBTkGRQ7rSNm??@oGH3p77fAgPod8O!*lLdsQ}6Yy&(+=C;|?Pw0)g#?+y z3P2Q5goIR4nEteds1Dck-F+KZLbMu09?yHLtD;Z@E6%g?YqbBAkIcU?fnM8$t^5^H z08q^j+`*0%vo?@BvQ7vNp$4i1cC*2Z$(6 zcmYT>>Mn@t!1g*QoE$fGTH8x_ozNL()hxp7dlAH68ihDZ z2{IUH;zNUZvg(21T}IG;Mqfd6HxG@bGQ;F>pO6`a6J);<@Yy@4wvKszFTx!Vnd^XrL*6)NdJuhQXgqtJ@ArNMA#FY-GH!;{3vP=O4( zs%F$!C24rASzU#_^;`SWlVtVHL3PR5p7C=j=s6%SKm-fx&H~kFpyrLyTSz?DPl<#V z&KQ`f6_3zE1y2jjA%Pxnh6GTmc=iw&A~-aSiE%M-?d;21mtTl9QqtbHnc8a(mt`lX zFBTnU9ri-X+wLQV=BbUA7TsQivW^IO_fynd6PH4?Gu>u3gY^{~DO1Bw15kmu)}Av< zDl$3c{W9t8b1pE^_{w7f(e2xa(4u*t-iHL-#S1}a_fD1$+X<&z*X@Z3_lN2s6IDEO zEuZ$UR+58K2Vv9Xm$wkyFE)CY;T!?+SdAp&#}Vb&IuNmkD3axlMFfhC^yAv2o-x;{ zcrir^5X(}jeHO_c$)5sodPvN0O^Yz1+#P3AjRKbK$C(`70PGBk>S#5YSv(^YnM-gj zBks(8hjXE)h+IUC`LxN35$wHvU#YQNBdeAljJK9oC`D1w0ZB$;3aDO*R$wGY<}MMD zmH!|gxTO%NR0Ht&Nk}RWLNutNEn9&)P%I%7j1od55v+zE6hEd~(}Fv9z52PB{}I%1 zbWw=AOrpU;%809~Z5UzgX3@qe27O9gAQNRwwo%aO3P8mQ;f1cl zg%`O8+>gk$rT~mW;pM<1h(hg)uT2-3I42*Ndj(wa%VL1v4?kx zHlDCZA_G*$`mhtYBqwgYes3NpRrqdbB{b)?8ka?(t#V^-c|T(^^X3UAq2=z>nW+yT zeC-h8ma@2`@T`iOzD>#$Fn-2Xi<2m1o_HIBMVR<%w=AO}w8$4wpQA~gGc!@y(gOp7 zmMM(UuDH5*E7slM-&cB>n^i7vW3gcdd|w&AaNllGTOwE+a~+^LJWbiWXoe_a2W6i? zp%e+CRYqh(S8!NT`dsr^R!@b#jLSB_WTKY3=5;xV*}|g&+lfOx@%ot*n~X2Ca_X5J zv&#ss(A_kZ5m_%#1Wg#4G*G8rsfvPu(2Z_VyW#B&n2f=ec!*6saIp|Wt~xJLw`5+_ zYUTtCFJCLLPecY^UpfP@lHyM8i}7)KGKr}Zgn@mHC@gC zk=C#-tG3Df{6yge(ofK#-Kt1lvK@AD%&pfo4+5ZEO&teqpKOPhpBX}Ey6~9Nyc@+j*9={WM5Eqdj zEZ^#MY#zsgii()OKX*k5K-s>jkd@dfpyQUN7{T+hCX}sWME$b->Xcl$x_(5eGZqLb z6;zaTha>DCL?vL|3Ovp8JtwQ&CX@{#V6D$KDSb2@AC{P)OGZ(T zN^*mK&3};MgFlT?A%wXB$$4a-A|#Q$t#|hNv$MbL2Rn zmWis3Noncv%|9AszXk24L+B3eO20wNndgYDLQR+@%Go?aKW+L9wMU=4KI}w?PTtCE zHY(P;M^e@$(KF_t)CXYF3OSHm?L#IU0yrC-K)43kzDmF+Mm;6s%h6L#oyNLGdJhLH zI4A|!qlG(J-a}$LOH%UwexoO}2iI!@$Pkji!lo&d4tOUWxM{p;LS>*lx<#{cZZ}Dt z%?2HC@bATlEdwyU=s<~K`rF#;GjtbYdqOci27eXT!Nvq0hLKmICK`@J^2moF=*nRz zsM#_E%SzobFQcv0Pa;h_#+ScGS+&);XvWfnRI~&@)0a(*3$q!Dycnr30x#V_EjT$K zd;{`d5oeFPXz2i(_58?JI?&&%UMTu^sFLP=Hw<++R2d22xdb?9MiNLBgQtS33rJXB z2r77|mL775JXswreR;2(@-UU_<-nHiao}hwJ)DdMOwz9!ngm6;0&k@43hhF#%CyYP zP(o}&u}RlluVI7IfP5;;?OpN-w%D-fyZr8)nDFl8u?5|p!w?AT0MlxXOM&K zuFri|{C|LWw!;}SP!Ol4BS4$U)XN-9M|?^T3n_}U(nL5n!oN`v14wEG4#BhHc_1&j zhtFeZm=~u7&jg}V5!X}5DRROU=EZi$GO5(tMg)HRzUG3QgH-Iwu-qQxw0KWHVaJO4 z9c?dVs!v;J3bE@X`+8lKSsoHCmksFzm7VW|nP$9Tyr`cyPClwk!SGyFd7SMpbW*RY=;;ad*sdZw4a2xc%SI+t=hNnxyJ@Vnzd` zsB{^l=V7X$%7OAGl0zR^DLUiwEmZK#>F;L7t}Jsv?XW)BV+{?Zm%O&(IicW0)guun zs*VRqbwQo3;KK={m5P!gDBzBNMTXm3n^VybczYlzDz+wMkY zjO>uCcKk-TWwL>5TJY5&8<+oUr1=CKww9EybPN}Wnt6Nz>8JU zS)D~e^tfXj%0ajDk z!4|quU60PacT}wpc-u9qJG$4Ri~}|kINlJP2qM}*#P05<5f?&$jp* zH7;UPZb7I?xbkgfPOci{lP3g=s`J2ZexP=F`pALyql0^Go!$4Lx_+1|weXX9e$d0;MDUKBqu+V1QmVG!bK;R@NMAT`;Y^ zx6*jxZN3=e!VfIlyGt?n(PX>n;k-Jgy9dPe1 z#JLsG30Q)(byLX_=riVJbl_(p*T|v}w+T6Hikh3@$_Tp#3;t`;IA>NMT`|kfJK)56!4*klAmpcnKDen7^xYyFT=u| zObsD7KxXg{JU<0GUP|aPm{{$F zvcqdcNFGd}JCp{HW#`CKZ_e%w-n|Y+gZ1L+xHlZ!Eo}OIr?FPdJf$IyCWJP+8v#tI zN=-n3!*on562^dVw-&@x13=bnRWi?wr&U|Pb6&1AMrItxf_l{sQnn3ZsoO!IDG<;D zAEXczbqG;0(#h%jj-RrDeQ;M(LsojVlIU+zeV?jz%Hw&h-ecy4UapXk9eI-1_|g~A za&3Hw3GCy1$cjKHXC5+)U@{@0&qb{CGMG?6?OeCbGkT02VN{zi+pTn5`Ym-Yer+0a zg0RqLl%itZQMU`VOy)0_bULZn^kWTuE{5Q4!!^3svn^w9Dad{$^h-sloMyYr*ERDM zb(Z7CGj(g7KY&?M2rNQ%c9IweiD=C3Mg~@2g>*isQ|?&eH8EnR%mOuVD1=qb=u(HK zg6q#lOAu@MMkl#V@}!!>oWimvoQ^W07wdsVs%ix8*j;&Z9F|0V4yj3GCM}N6+S#Mn77uc*n08=d_`~SD z9v#szz}RNQ>lgBSB&>wY!!J&QO1aMLb`nHf=OHr=W?1g*3~{F0ih^?AX`?W$BiMO| zZqsom(9P$mUZVS5v{s8B7T$_3egk-!(P9Hs>sn2SiZIWKQXpA#5NP+y95;)kN~3(L zjVSU^P<5Mef&@9o#Lv^g;!9 zEu8m$Rx$$>msH7L0vS(@SfWsYzpfM`8JJsSN&`^}t13Z=?=6{D`gb%;-)NrWg2>qX z5fxKSGb=9-A|f3V;tH<5+Q}In|m5-@c5nj;;~Ah+?(^w zFOej12gpLizjv=CyrLjYv3;P8N!NK?WVhwP>zEyvcPZ!Si|QRNq>F6zg^v5RzFcm+ z1;RrHwb8Y>xO9I()M3Ea^jBf1+#S;#jYv|S2J*zZ+`C(quxOJxZ~_h~5P@*&y9?jF zvB5OF*gFcWA`QpHFOS$2u9*+U1VXC}c%K8-3xj&oVN{+z)?j-j7z>dwnsrzfM2anu zbL;RSA=kUr8}!nzo)jOf`OqkZor!6PeNiCXjwZl$z(Zmvg)izK2$N(dKDSTjb;eHsK;JjX!Hbk8)V<5M6yX ztfWTjd^$tb-FA(=R&fh(=*?WRLscbmBxt!v@=e3=#nNdMRwCxo!xw9qD2e2}fExWj^pNm!ur2WpA$Tu-^rsWI^Wa zfUUvOU>GN%iBwl?7kNIpM!pcI_frx_%aL<&=2SWDQhnqqH3NX~SPTJ=bj8ZK$BdL^ z>g@AIT@D07rH?He?6OfA5xJ@ZNMo5)(fU0zY`UI0qCsK}lPSdd;Qi#e<~~oG-5b{X zcsI?Dhs6Y=ylZu^S0>?40jn1>W1`~h$LPm}8l05JafZEfgR}yuxSQtT`*Yz~=1RQ)`+^`4V#{3B6BN#i6fERuCa(CTNwO`J+0+1S*V)7X|oc*bHzlTov6_ z4NB!Fd=iQ+Lm*?~y4l~NR96Qa!sln3hcJn#?MfPzE{6#FBGf%^stk9Xx~tKTMAh0m zZn+NKG^ZeW(DlqK3+)Vo*qTT2N{E7)d2|^UDYa)PFWW&(km*8JSFO zIaUZ=poru<>XqC%LSHreRE&n*A9|Mg2_za2f_mq4Y&cUr^cPt9Lp4l$xjC+iRl1<~ zLTO^62bynH+^8-)LPg$NSFXGhQzo$rcRBQA11ApDgTf+OGYF%=q$`g>EjSEmEEiZyR}RcxnB~M`E5)=pTKWestU$ziK%|f(2i&Sw`$aDFlR%m0 zks#TAMl~C3OTrt(o5$QEmVi6`ElgtscyN~1!n8Lj-e7S(Yc~s1jA(wG@K_25U;>?F zd%E*fb9xXCw>3hi&ZVMhK4)IQVlu*21>kxgpHrO4ViAad zNMegPLqU6)G7?*rsLVmyLki~PTI}b6>uah6(}Xo5oiZi$m!uv~9KmAZH!e2u7!t9R z=$9lTvrtGtWbmC!Fr+F%>eB%c)7IU{i^i$RM1Fnq!UHX*aEwDds+dGphmGx?V*yHx z3*MS|446Ez7fK{oyNM_F9ndr38jd3`-NZgFSMo5~ai$})TRf;4Bg5)(_6RhVE<{I= zU&z3i=RG{&JZG5!8`eEhj^v~9sYqyoHG=DVNN1^L`KhNk!ZHX ziHkX&Xfb|yd-JZ#(`E2@y@73R9wP&uU7E0z4{Acqz%^GGis8RPch?T67bF=DC6$GJ z=5JzZKX!(SgXjZ$ra}YvcZV5Hizh~Dw3LKasHTQoF2!uOk)q%rAR-_lgd8DpNR$+& z1r}w@Ln(sP4>8H0FMwZghlT8d$276Bz}JmpX$2nfM3zy`xq_*1F19oW4&4ds@>m@OT!H1ZK|DEfg9a<_YPDPlY>P{g!Wu z?G#k5d;}-C-rH}ZjD`U@#R&bFVA+ew^hnvBCQv5))znD3c2(}8+cAJN$*G4i(aBV* z1!T?X$ZQdTG7e<+d6B1}L9NB0m7gvt_9tYy81Qt;)rR|Jn|z7`yATZoUW?w&4iI~j z(nA0@V{Zu9VvWV|B2oN+!MO@=_aHRy(>El;fSvjz$CLhWGT6vcwx1S(YJzfL|`z$EqQ8_oGi3}bZnX%Ury?M>m zU0Jo6YEm0TMj-N4kbS(2nLH)#VMPbsK?$(}>?bz+*LEI$Q1O8XgakNVjN9;o1NfGS zdi3oHhDc(IpUR%gW`R$ZLtarTn+nX^V!YCgmVg*&$kgJNbUpLkj0l3>HF zW!dshKw8*0BMYv`)m`pR~dz>{EYLxkXj zWF_JHD_%H&zJk~tJ7N=+>#L4;7Lk{QUm66MMJ|Bn(_kRF0z@yj(cTG7rF5Zg>gN{P zN!pKEVO2o`fe)aZ^!|*GL9=UuiHIa~>oIJ*35bfdv4=4k$2JS71YM=Xr1TD52Tw8H z1@=*n4qqm=istx~sE34Q(XVxD>8ZOhd4ulDw8WPc6C4L+y^b@Cr=wCncwa zWRSN2go*Uq-#5#>#}H5$7|sBK^LJIYOg}*gJeMX&AyLud7rTuHU(y6R^X=Z!p%{p5 zM_tHRZWJ&bBnc_Uk7)vgJ89)bhT|A0^j5BH3JK|q+C8Ib@$VOs9XOe8+iL5Wd-gR-H*2+Q$ttYCS zxoe9Zb93jp2pWKne9%kyuGcHQ9ZClT)TFCsKzgXs zU9^C*?46!rW4>VC)SEMr8Pa}OQzIw_Pvx7S;kaPRo@(`7i;duE(*JqW5@$Z?ayz@t#kn zl2h%410@iM3GQ`U*Ea;7Gw?XNMy_%-LxT(*dSf|bF4g>6e7Jx<7n>)j@&%qxa3S4uO6Wt~xzmj8J}|A)%fxAoC*Lv-yl@M9+iCPr!XAqq+>( z!wYHSWhDnV718gzn>Sw9Mz@%t_sYrVr228gHQurV(w``a65|e6!ch3Ip;Dm;pi46) zKv+PZD_zN8Zr)lJkUB{Vq}}lL6#;y$i&cIpsa5x!SV%WMF}SPcandQErV*A813wSh zM83&Pnmr?1VS;dUAUl3ZfqUt1gi<#ShZHoVxj}<&kf%nBCe0?E442zd#*BPZQ`L4^ zG74aD7g394>w4xcK&+YAZ`MW&bqV5Ol6Yqmb_fImcb?E(E*Nh)TAB?D?x)jrlt?@R zRL5CNxiZl0#XV%7OzAqxl$WTgPz)Bz$4dNxaJoRQ=KRI<8=2dV^xbSE zL$3qLd+C2hz%Fz-nXTeedwEot7EGkcx>ZKSN;&9K_lS^sgMDP8QUTSu zpQ!qj?IFxCv0EuZ9MF3LS$KFPDlNt?G%mvO#NeX63@XYi!)dVaI2ThV&c6mzc3A*4vq-fm&#Nx zv5V1ntuDn-Dk`dh^X<0V_b2g(?0Q3|N3Nvf(enLdE81+ltS;zKj8ajNnVaRlrr5`M zdFV74oyiv7$W^Z89UZKRkXC4RsJ+H)hS(^dAvtOStF=#OF{ka949g&_-fh7Vy`V}l zFbMif8s}<_t=b&h!6$MIzDop=p3)97uY-Nf&_3K!*M;fmCLV3mHgKM%hp0N) z&^xD+k?pWTm4Ugv4ZY|B_5`{^{3{Z0uI>y(qw$N-jTne1sPK%CTWPR`5UAGa{nasY ziUu!J&Jp^djFyNJTf_<|W&oz3M#bLD2By>$n6u7ort2pV`8!vU!Lprc|&vsqnrRLAIUU zpvK`JXVY_>ix(Qop;kn;x;fTbbfh5y422RFHz$1XGL#xR>QsiDT;^n>iW~TvG~>Ho zg@=Z;P$kMihouhX_cs~erZIgEPNnJ>M5R47SV7QIBhF7f%L z2P;=IR3sR1x*2MSs)WF-W?HsS*=2xdPeyQk4KgMfaV=+9&yv2(Nya~$AucD` zhmC~Z_NE@)XaV5pQ_2zDOQeYBs~o246(K28?rcuyo1A)`XlyyNVZdOa=fjJAK!c7bBtsgWsrIhm6#iaX)V+QrEg(S zt@XGbmYLD_Uxc_$SU~Ri=o$&)=JA_82c3IfoMI!ef2y^iL$dG`bD&acb}Vplbndad z4ZL#mqL~jP>&ompQ>hcw_P2*FA+>{=*dwt;E#{_om);9YRbQ9CBBB@p-^>P`j$n{N zp<&K75J(59RVxWkkw~0#ZM@|qGCrO)Pd_{ph%Jr)^&pYO#onKeUA=0aVcFUqCyJ1c zj-p=f4=ylBg^Vs7>_?%M9#fcsw&PX7ct%q#E9pN<#{={BqEX$?k9= zw5bo(<&JVfwF6Hn>II%E+Flu`91QQELfgSHXEX>inlEz56K?2u+@;B5PY2{Br>Y5^iCfQgT{apy4cbBFqUm<{1d>Q8=MY3^}d_6e9K8 z%l4fGtjs}|Fct!ikWh^AgY#4Z$yjzuKEN1p6Xzo|)Ov({)c9KcXhK7`LWV`eiaDGT z!xjTc_!h8oDKs)V7J>5tULwlG5m^IE0N2=~Zc?7^wD=c@-nguL0F*_EC#pXrsf@T} zZRb38db8qze!cXROg%d2B>^Y#Pn9`_*&3cBd8xf_R&p++MA=E?#syx`DoZ729%-3Ykhh1PM8LC$8Y&|(^r^Xy zJ|rnf&sgrBFl^MsuPiT=ACXa=su&pTu1a*HB3v=OPDjY8vri&7vq|L=5Pid8G&ZWQ z#q61HOGi|OF~*-a+TJ-0?T2mb>UoHbuRlu_q!^kNfb$_$`JnRXET`=}>M(LCFL4Fi z;O(ikiVcblL3#B0^yqS9dK!mB_rc}fqpUT{o1PHKe@oaKnmo{JATSiD$T?tQ+s84O zn=clbc*iepCCQ^ys7}v>#mTlm8`c7|17s51LC+@bUN$q1fvhERozEivkXVHZ5aW|M zvE}=4X7wXM8*Lt@J=D?Z=Z#oVi_&dQF~owkA-^^o3sHTpHY0k;j$O(yXaco<7#XL^ zA&@m!8wx9=A8T~w5N=jM&$?$sJUA&_ z_@viBI7-t#U7^eP)Z=X~PEZL}BkmR#ygd$J6_eu#G0lND!B>nA;F> zaE4Q0YEf!bSisH0< zbxr2&q8{x6>Pl$C28A2}@CX}mLJ;zj1d04KuOQ)%0Vt8Z%)$Wob9qCSCBy{lHi11g z#f}-2Y4t8ZSXY6sU(*dZGjxl`P*dp8RAyCRO1{_*MS6+eRbMGZc{68<^iS?Ugo=0I zZa5dYv(-(F+TN%IX%BHLsRsv$wc=O>h7jaw$OYsS;3=Qb;UF{bq1PtmJq+Mz2T(s< z#GgK1atovtT^B8lI5f7vXpo`7{Ru|6sW`HtrGqkxNM-XuqwlJ!*FchDb@#%(Hzj9; z8rqsTFMfVZ3}A_@FgOSwgdqcsH85)kFbphnaEYLFnP{YRno)hDDNpVO@ema49Pu&x zy~)pkWs5w3!A>WNLhx+H619$Kz28D63(FFxxn5HsK~3-kY6Jmx$^&yHl*? zb!tU-!K@<@j+&&u+;~=jG7nW!zLoe;}LCkqh8PeeFw zB9MoDv3Ix*KW#l^u26bIAv^GY(P9EtscZ#s`FNK8kCa8#1H)@@2bq}SIC&QwM8^VA zM;zZ98jPFMtU#rgViXp!Ln$-asz~DQGh0eU@;aQL?g?0l>Pf#lUFhLd-~~O~BBZP6 zs0)JRopzI?Bw$%n^R$ZfJj^$*HEUoZgeb#oG;1uT?$6bip19d?Razp^?3+)jTgzSi zRd%Q?s?-u>xb>xWUe>VBw+ydG0x+2+L3lwL25C8_v(KkRj$1 z9wJX!oK_Z83XB-QyG4gSB1-y2`z(B=(kDI3ywkyra}r-Y(7Am!;D~z$;tW-acTp39 z5}1r4a{zFqiz);e_5uf}lHSs5QC_(Dh#D(uP9l)g%$5(v_7yCuD*S+xV+t5yH#n+4|0sz4M40ytyz7{dmL z5Qiu|gd+&_NCfyr$-cWPJ6I2?lLEn=h+-C6BSry5prmSOHWEpuCn^2n?5^<-WQIQICfG2KbIT6E?Ugo2UJcl&y~;@Av`5~uj8};zn9Oa!} z9oj(r1=~`pHeYY)1q7q7>!Wy+OID$ZJf^@Zi6IzK|%r(42zLWdd&wM!ZLu1UGzZ=t2k**`MldITe6euVV zRQUEF9tBz8nMjV!rlEqWOAfa(2Xwa`O1ts(8Oe}(Pv!@i@ZfOMp@?OvnvxKZsRP>> zASjTqq|iT2Jtle6bZ!GTnFt}Chlv;~c&CY)2L;=XgbyCweypR9mD#MjSP?E-T|L*c zuvUd_wL}2F1-tWo_H~1}LUja-xes~nqXbYH<(tSL{Zf@gkfa`gmUkmxH78x?i1CDT zIqg9KUgZizG7Fq##8LSBPV>-#P?mK^a#l@*6^xb3lh?uQH$lcn zKQS3MExLuO#~^;rQ0`UkC2r|LpN4rA8yxet_nSBJ|1jqX$@k8&?GGNzv32b2}eskXUi;-@`2 zUZGr3TGN(hm;;K8z>s+hrWN&pZf0iu zyegfnMAVw}yl>?sf^S3+cxx6jQohRXT|VjKM0VGymx1^S%ea*gBd7J!!?Q;zIbwK< zy9}XQHcGHE18_+(S0ncbh6xmn!k!@`k|us~VJqh=bq&o%>n5U>U(uXvN}m-0&{ctI z3OQbBSBwdDD|)Gv<9;A0Ew51@qk(3X6HHRr?*Q`n)Y!@>4ePFkSYq7ACqTtbXJL!? z2m(n$(})h!8yTG$A5g@DtOmFn4LciS_D#HpI`IgVYC8O8XoPoH<@rX(cTekb(_MbW zhG#&ZGEq(TYUrTc%1K0V&CA^iG~dL5*RbG}!aObFy+&8St@l@hoRqYWBBdvY7o${6 zFc|S11fWL*+mQq*XKi|NnAhx!>`q3})et~gMe@GN7WF*En zVk{VXGzdZt0z-pmM!13)>f%J+>KYX0k~zU8*HNh+4@KFelZHbHILt$XgsNnAN7YaU zhj(=VjMCDG5@9E>6ofS=4D@5ORMf(y*W|ZnM}?)l4TW4n?dNqD2PB80810Nr;q?@p z1-wu1kX1Z9wVku}T>ZvW@%);;R0 zn3%s0G84_O-eVjvM`%<9ei425#6(|wMd|$2G-c~B{QNJ>ou&-YeYsuen=pGov*Ym! zyai~a?n-PT8@*e8A)*}qF7X4%O^b@JhFLasrpyT1=ty0e5Dgd|oNd1b^&@$T{vO+~ z=6x&r67;4J(P6@EK!F|XOH*Xx)piS@)j~f3ouL9|Hgi9VTrMyZn%TmA=)Ca?KG@ls zQw*RGfzdUW!AtZEX2Cmx2LoEZS`mO~ga|AYaVX}K2x@1L6g$}f=RA31r!#l!G*IU2 zV)9PIln7tgz7xucuR>V?QQc>pxx~cErvZUl`Am8;_!e!Cr#$8Cz{FR<0aWV{2&J~X znld?Ls9P_nz`G7;aTPsTYZV}ahhR+E^q_a8M43c6;~_*9+rU)1oC>^V42|xhXDGYu z30zT?CE3(z)I#VYOdLFENeFw|Zch!b80MvJEPRID?-tSABIUp?t~VFrlYo>Q*W!L5 zY5jf$!SIgyT4%nN9aK?+dgf;$^q6`?RE>^;`5Nv=LB^?6NLqY=x=QfIHta3a=khOF zS0SK1r-in;DeP;;c$A3W6a2NjoFLi+489^1dm0sTQLl?yOYtc}9?WlOHp{dA(T=IwoW7~zXxJ_ImaOHY_gAQ`SE zqv0q-Xe6=5lI4g+WSmdIxw6k&siO#5%yPQCkxl+3j`MUtEQy0`ZEQWn%Q` zyU8fCX+j5&vJ2^RI7b7xca)=#-pdeNhlW~Qj!!7Nd@UZl9Oi70PSp)ofn|EV$n`v^ z=zyJlzEL#hx>k#0dO%_NZxIsop$g03sI?g66=AL%Qe;QL8B6+1z(x>^WzT6Im*sFB zG)CUMI&W-h>DcHl zE@z$;1?JVpYd1!I9+XfO7xJO?2~@=f0|TPtNy-vSiSkRNnL8e_YUy$yEy-D+LDp`D zc%#?J$-G?$V)!1U$=Wb^kfAqVN5o_9>KYj81)|?qoIJpjhNI)7_fE4kNvx@v??a)e zxCjy#)-BCqY1?M?_BV1tlkQ zNt2g*AnS}DY5XW}O)DM~`2xx&AGM|}%HC5$7FH_kn=AlMK(W6`4^S_JlAa}P;gBD= z?(Le&ERZvieAz)_6UjiUkwFF&p=q=;8u~f+lmw8#tZ+QYYfoW*&E-~n3Y?H`t#Hn9 z&urPUoUhaEy<-oat6c}aYrfEpxNARM!x@~zag1@^O?FSZutvTjPir!--1`3`=E}=> zLM)&}MIb<~gg~g7lGos0C{pdS)SfiPmN20+D~A}#fdHyMN2e-RSgq_KP5JyQN`hcv zg~{s(+k)$U6ABG1qIN>rqa9A{8pK%f*ei4?F+jkAoq;M?f`~^%PC~_Xox6%Ey*MJf zh4*izfsrj_9)Yb|hhr$x$xHd7Ir8E`OC!qvqOhv$zKfv4o#O%S(*%@C<1LH=l@tum zaQRW1Syf!QoJ{T^rCs;A^vj;GsgI6j%%~&UQQU1hy}|^Vrb{wmTTr|!o~%1@QB%Mc zEU1%m?ehK_9e7B9zn}MXZ?-Tjv(gCQD0_DalG0H~FoB#idD>ECSQzlk#Hmd1U}J}3 z!d|5mM%-(=uw>1#+w0O3Fs=7!%e;^9s7i|FFJuOSFlj@ zPF)O7vfG#tNkoi@9@~!n(P&#ZwKMR%W;Lq>>McT;hLn=pqS{DyHAHzv3A`!XH|?0{ULEpxMIyl_G~WFKVMebMK{Ebi+Wyh}^J3 z@Prx-gNUgp;|&nHR1RV0S6J4oGEeGVwEDD=1Uy6z)=ii33*c63k_GRE1#xWQqsl0Ru zs5G@Z&W>2Ll(pqjc=&9W4AVMR9$|vf&;-ij7H2|S2S{Gr`fU0|9mys1sC=QutgR{x zeCbzztkVz4z!g$eib&-vVu-a`J4=1aQbQWBvqe`ZO`TRJdo)xc;kXwFvdYhpIlnx7 z57IU4NzS?O5>42txN=(3!);~tZ%a68NE`$t$^p-Tg~KWjB##)myjE9}Cft0b%~{2c z4&pMhl=y;>4#xqar-cSEO(1awVstVT{pl#7=V#D6UR%+^>uRvnJVi?gfC<|bYuM(RaK~df5_LJsMZZqTi$tNA>t}D zD+N2>ON$1=L7)MO7nWO_yizn=c(k6Tm6cRf?@Cd)MGX6`dKNKk-|NWImrlKk^@3!% zs6AvvvgmFBk_HW!acf1{r&WhM{tZ(Q197waBes|k#5V0oi2-4NVf)OJ7cy;e)Ixpb z3f1o;utyOaMmbfNeMIO9Q3Q48nrdxU$>jq+J6I(iw3{dy5rTUL!j|`nk8OhOm%y{D zSlkV1(@Py1*gIGv^WuQvxbH+aks7UMl616ho)hPbOk;C}1W1Ti$vy`}CxOwppBKn_ zWhe1zt&sqlU|Y(b3-09C7^EfiN1N#s_kk`Rh{|(CgEZLonKKYaz>8A&$g#d(BPbxfwA~qm0mN7yPBJ#!5 z99-lXMSI{oQa^NEmxNh&NNcO+d@)r9hD&}Mf(#SkF9nX`WLcF!LkW54UABf;{%Z8f`#J~) zeZfU$)Au5j90B#9&Bf$f;JGiTrYatY=mI)R-7y1RnbAR40`a~wRm|+lf*sn!W2&v%xORtA?)9+CblmoWj(Ggn1TM48EB| zKZt4X%ryR!DKafdwMT_*=^A!41%>w}Kez>IsyIARE2^(thGUQj?%J-Hd(IaeNb0pm zo~Z_D72w&Fj#CI>GXf@&!UqE#1Tk>aK@ShX67i0VjyX?h?0}o#A_v5*NwETdh(Kjc zc7aS{ZpPFcg!Wl(d*;|!cgpuLRL^Kg3nP(}&j&BWFva5v*t2Puq#_IVnKr<{NH+Rt z2dUhv$_Ub;0us_wZ^U+_P9agarA`B~fl=HpSs>skbrG{v!<=xL=wWtG428S2iyGP& z+!!UI1ueXveZmZ^TpS=F8wMB5yi7NCKE?93f{wlcATkx=gj8S3I8YP{s{I8~w3Jy; z$;kk#56X_K1zFRXN}TAG-zk+wMOX+Fy%t6YURfX^m4ab7u{TM64)sLOx&;2N8n)Nvl`bEZJ6qQxX2ejOm7$;J~G8{0_Sx5z2iscHrDi6e+wr|j`yCkYBc%fAy5!n}6 zy6Qr|P_ao-3}7!p5?L6cssI%g7GgelkBT~`H%Vg*ip>9%hE$76?^G%>F|kp8n0^Bc z`&w?Ivz*b5p9RBVX;l`2?SxDSJsN7b1Ovh&NLfm)C93}TmHqi@V6qBL2pN%B1aN^7 zRgq}^C-*=q!>Xh-lD(n%)eaBTsT{{Mazv%p%l^w0&Nd1jlvn(YqliBwsuU~uC(EFi zQ9qO8Av{Z^)c8@igGW;8&LD@jZ;Zo7@44%vzZWdJXPvXw`SsJ|zn;8&{gefI^W)#& z)Z5wJ!!TmM1fVY8&DLseu|d1VtZPo~W%K2q~Hr z0bB6>&wU_;#b!uqVYFK0T-kuo&N8(}fbHqK)q-&{xc78eu`ZHzK<2FL9dfG0)n6{r zV;xbbr0N8Sq}NgF6Rr7!jtM`V{fy0qg6(Tt#Y8xWGlSW17oL4^QQN!YogEjG;_%Gz zd(G4aim6-cB_PK2zN~8Yw59v3J~qC78|AH@z3C>cUNCe1-(Iik z?k|k6Ot)kaIQmpN-qs{T&ZvV9*t#j&!!HlMQaZgLUaqW-7(xKZvolshaFs+55rbX@ zTj3#JmvTjhGk6DjRbIV#H8BPjxH+W;;6xC7ppt>6hbpCmRSJ_Pm%N~mEG-635XX7# zv>i+m4v)TzH%w~8AP(qW!jiG|xLYh*J@>dR$fYG1#H|4S4GJrhY+5+y!KX7>@@c%~SjR10$6k=|OWRQnhemmWTjH8MaRk4m86zFMhy ze0J{cren)IW1V(%o_%341fPgIS?+{9@aL+k4z^hZx)^~|cVo`QhRv)f)5?RxkRE74 zEU|BWB`6p_M7$L;ctO=E0u(@6HrPc~q%F6(fi};*xMReh72i_9bn;ZE=-EA?40{ADX+F$AF;@smuk#Fpj>8zNA79l=Ew zd(+-5@p+pV&_M-oRr-gI(GM45`M^kl&}jm9EYwb?zj=h?Z5XA9 z3bxTgeb8mS2V)Ua-gZ+CKTpGIGT6?*T^Yi1B{lfoPQRO zD!bUIBGL*Lc?P|BuzL1(WC~v54(QaZ4G?CH$LKevn#^p9NUz_Gj zLo&yf7)q4r`Km_v^&V-bx-~L3dsd_g$h~J&D%$2Dj&};b$gT-YRh;u$l&Oz-GYJ6- zpaK!75UB7ws0#K1=KP1Iw->u>>eQXKI0(4(E_Ps6n7zOH?-4i z%A2jwkGrG6L+T;|>z&H%FC@!}y68mIQx)*1B`YO!K)b_cZedN9#fI$Wvn1RmmCIS? zo!GOhvMk_YdS*vr0hn|gQcmt!d?QllH;ll4A({_`oaR8FIWu7 zYfJYjq{F3A4L_F{_-K}?F1vN%yIq$+u!*7)Ws7FdY)WsEUV`G&!tcuqxHIQFvfsjW;eTL$;Pgi3v&ox#tBx%Lc z#Lgj2%qMKE({4eKN-j8wX(F5^c1r_~2W?33+oRGWUimGS4$cS>l?rSO04V5yC2lEpkavYrUg+jo zBZG~|Zkk9U&>>`kkj4!YTaq+J%34MdV|Yl$0y%O0HU;R@+hQLm z2Op*_(V8lk1Ug8Xt$be7@)5xO+LJ)h7a%)jeSRd3zcJLsgpItS`?j8oZu;%&uLR0P z@a7?@ReNZYkkN2%0R&GrflpNMxO<1*oEE|O&Cf*;*cH+jK(>v2a3`m^G>qC|jP`Mq z{qzYW@-sG6Vdclfmu#c%p_P$wwAF7dJ_eWoCWWW*og^uued=1*Qh6jPaVN&`W(Pe5 zqG2F^JSl-1xMSX|3ys=FbOx^QH-bDd8j01X=O{xk0Q=NJVN8LX4emY(z9RC3-@kmJ zDTYJxE+iW97t%ingFTIqnu>u5344| zt(*3viw=@ZS25g^u`pQpPles3RL+e>A7jRptMW!Y`*?7vP^&u{U_`o@d8!D;Z9oOl z?MOf~3*-}W2znV(P+KpC?;K&ka-lAUinvW|@Kp#rE+mX*!)I71k;9IIW5rNQ0A+!i zhMhjb&$eZ&pqJ|4{1H!(6~EQ0xjEvN}Y26-&ScTbpYEInO%vy+js%}|Gi z(``_d2BEl&z2I4t@^l2zXoF=XDFr0@mut~eZcu7Owo-(@SV-H{-1=w9M>)bckc5sK z@Fq8m|9l4mM4V5?0tAS{hCF^J#k#WW%dmX}TM*;{d@EX%O>iOzerff?#7C`C^E=5I zj#O=pV21eJNQ$1NF*$ph{WOUWPbBHcMoM}|0!55~(i$u$4}oHy852XLBrz46!|8({ z@C?WaTu;aycD}}|@*Y#E)e6_kL!tMGq3luno})8h>>esqtp@rG$6Jw*kBYm~FK4CX zz6~^?VnTt$gKO zQ8DM2sq-7iWrvgDJ9)EUeP?!1(O#k2_~;DYqS-{VkGa^TTBjOXyLC2=j3Vtv5T}MF z9Ks$xh3ue7!A;^G)XJ2vQZJOlqCwoQ2Z<1lRgJ?K{@5AX1{Bi^rPW&jGly!R!{kLo z8k#aZS??8G3c(OBaO`?+>n8A)!D!abBBDw97{n|;Sh)4g`NHe8)x7cm$=Xz59$AjKXDs2JXOaauZkx_Kt`wL6&H^z0#$~ylqiHV6fLtt^< zGH<$B`A3LgzCpUf`>+*gm%a^ZwY=pt+U^w2^k)j9`2yD#+mFj)q(H#(oZQxdVFl1) zkg&5vsIL_4^f^J5DbuLu$D;M`8_)5@o7n~ zrJOILuy6|6*<;{1EEPJRgBu$W9DEZ~= z!a$6eRAdUz>m2YdL1IEs-O}>EIAW-46dt)SPZZq5P$Ovk^=%8NJ*(>LUGnIXQ;0m2 zXlO*#xE(VQD&0+t=6lQSvLu(a5+rnN6z*Or-ykUA+snCB6^U(P1E7t_AQw12`{FqW zUN$mso)m$F2x^9Z6zZw{CWwd(lx|$g)fF58lyr~B!2(uys4i{SHA`4<@2%- zZfi$m0(L+UnEs0}_bZ|fW864bF0Z@BluH*7)dvK1$UAf0IR!AWHRq;KFIJ78U@@_* z3C|8TLrI-OpcS@ngRI%cSozraOBaOSMEUh(lDjPqH;=^snTwX<&^L>OVRt8}>*nvRI}? zg{qi<{3}n_l53Q1vT$cp4&^Fz@4(uO;$z6PAS4s5pWey1N->0yXy*i};2jX;!jaHs zL9^L}Qche+L-59(H!OL9YPGh9@~07kxf-aA;F*co-Q@3yMOajoA-J83T@4W#inq;< z;85ZCMTi)~Sv(U=20nW5>*I^#b>r@#O22#aN+~M6IOp^0mOY5x{q6oDseaGue-a$I z%f**&;`P4vK2+A6c7&#-Ajyq{xa!5U-bcGN&G%#?QK-lwSr>-TGjEz?P35R~x==Fp zqJ{Yi-z=|^=SD&-R?3~}BN!fSx;5_O4mOrdn3%0BGgap{)!`JhBDq$ozBuFd)s8?! z{j^4!sD37ec^!x3p5QKSK&{(N>HwKGTS zv=-+2Oy|PcW>v99YY3ufNv6*r*@k704q;e2J0mrzn;ap9K5(?hOMt9a>N^;PbW1?+ zJX)}L)i|N7bbTj^dme<)i5_oSz~grsK-7Xc5jDu#I<+smls%0|RdZqQBc$&oh_P0x z%y(Ei2R`1Vdxci9@pHB>L{@l2(ie*RI$eZh530P#ZEiNC8R3mne(cMo29a7iDH1#* z5(H$QiNM6a#BleX=Nx2MWWNA6(t?D9v}&eDDBTSUQ65eWNaq1ZmLTFbWcSkHrHcv< zEFJA3(~T>QyBP%f6IR=_b3$(W^zSV!BKH?;glBEu0oOJ9kg_qEaqN^Z*r&lKx4|fd zNeYMIiz_4;vN3T}Fm&BU2GNXealAU`39cmQG)RtVCgseb4Z9BWZvc2i2e0kdtlXD| z)Gde%M!1ORL}mm(zMVZjXBx>M93UPlk`2cO+Ry@qp0b)hobtt=2n`u7P>-mjmc}q{ zOh7L`i6%;|w{PX09i~~S2N`+NwGREW@ay>mKnr>BqSuHTMB)!|LiU$om#=852V3&<3)VuK10yme9uQ#8*6o>ZG&1oHkeyPZn}H9n$FtY4T$W2a63XIk zI(cpC&@;sC&kMf|l4-V$hunf^_C)EhF21a!8DSYUa0k?B^l>y|S6hUG+2+Mh5qFQC zeEkWbVRmY((#@nIF^DF?M(BbxHnf7XyBtvB9XJ(mc?%&2S$OJC?PeaVKp$yG7JAAI z_$bd+6Vn?!44m|Dg-#9$Lkby1t3oj(At*{`sx1%YaYO3jF-IRGlv(sDLp~UpCo|LkFBZb-cTa|$zJA$?trt@$8__?KuNN$Af{3_ zBRF9B76=x)S=4_s5p#{8$$2k5b2qpNNtXffVkm2cQ8sB!8y)O1nQ)Ou7g#fDRgJy9 zB!X%1@G4>u4ESx<{?`q`MYUDFqclKxknj&tb?tQt~aX zW1OBFi*5S-lWY=Du{Medvyy|8DB=Sx0Yj||P6>3*SBG#_qrkas*fNaM`~?5 z0ZNw)xV)z(?S|E(1{PJL=@IC7;npeDAS>MsgQv7EqC`G@d|f1dx~z5eDLLMRIV`q% z;hdyHeQ=LU>j=L;GaK6{PaKE><;$0igD9<^YGNlK3&~#OiaEf1zg-HzEGe|>#rDb7 zAToizYp5rAd#41df&(;^Ja4o$jAMJAmB(nWo7O0z@F$R#bKbagW{5E1(D&VhhCBQ<{fsxoCMYQ`lJu-n+e`*1A3<m=jkP*U=B>er~7g-p_ZSLNAJ z4#>R$9E%qnxLXqtHMua?0G#e9hY=BZFxueo5nG`46Qoxd+j>#hq5b3c1PjA9d>R9F zHU_Sn2e+8)S3Kw-qr8Xk-P~g+g#hv^$YDr??!Zwxoc}|EI@@a-yfJ&#;k1@27&5#vvo(czcJb1>982Bn`&|x?Z1gl9U-3VGrL<*?EpDMlNnJwPibCw-YEVbKC=J-JPSbHrY zdxG9QqcoGa!%EaQ0YH{Q)9O0Y9jQr6QBrb+)I}I0V!>&8YNsQn5JAJL=Ue7EzO|h& zcGt@fG3P8|^S5$QOOA|S_;H{R#M{Ob*TX1kgHQk_CPE>H4NhgS7|J5C0~df!_bo!+ zs{)l}Xd>2iTEW~N5{$*7zHn$$B%IBue*`Pt_2VdlC6Ea4 zAc>v6TQ|79?Cv;WWOw14%16a9$ILm>l8fwvtdX;!UUDhF%zy*y5#H$IwpyoQRdGXFpC64)Vxdvc&5$q1ye|zu zoH001d+bpLhJ89VcIM-aMFduCTj{z=$%gBalfaIv$$L$9Pl2+CJPtNI@t; zksU~WfggwAiMvSWBE*qFR}g{WxET|1sfTj|u1>53HH+*UPG_NGd9WQpGfV0k0%X*D zU6^9}dh2k)!@xS|ggHtKzFy0H6_lH2ZLP*G&6KmA6P_sx&z1JY_$sahZ%*T7;2i)+ zkx)&7$sm;>N%aeQ&3;#fW&(4TOkYJkm?Sa=WPc&WN^t>ZlqVKTfj8NU+JY@z*-_ei z1>)fY1A;=ZgsaCCiG-ESYZW;Nv6WDusE{D0CX;mx|9gs&R)BPaOti^L-HC$?B^K@HYH|Y1F4HWKc(+762xzs4TLNUToa?2haM$%a!&wYz}56>H=*89f>(4ZBv*A*$r`Vhc}&pi@j&w57DJBE;3yut-Nn zUE#Hys+~~P3iLdYx+xYf3*v>Z}wR;L}d&OcI;rrkG9FLGw6r zsgAZCqQb5i8e?N&NjH}nuzEq{lFPMwwVdGxUgb{iHNC8S$g+r*a}4+7PNM1Pc*GUy z)_;!TEWo|OS0t2LEA~8e@#U7x-lg^G??a&m zg)yxiT}sEuuT=<2-f(hnD)dSGmq z0S*Oo@qRbvWMC8&EDj);1I~Xc>i!GGM_TAG$3i!+9F{ru|3UYgAFV zDe}b%T6rz*C`a=n{EEupBOwKcm{FMZZnqG52u60!e7zRmD`6Rm;UJK)3G#!8h$#IM zgv5PRasq!(#rNNQUlD!x`TvUC-yk71aG`X!M$KCRgRkh|Lq8qP!v9vfp$!lEGky@v z#2N|6f}@n0Xw?p0C<^McpLUw&2p*!1Xh9;C^28R@ChRYWf(!&BVN3~lwg9%F0M35B zx8S%2nQl{S#w6&+ApB+%j!RJCjsBk^D*Cj}<Zxe6kcQM^~6gA1DR-DtEQfhtsy9!f@QUAGlp++C5A?f9oY1V#up|vI1G*FiB*7t zsC`Rn!9O#H2pZ5334pGj%piebrerv1mA`qaY`16_|GzOjaoSvoTrR`o^- zs=p7{=;Dk{lNwuxJ3wN9YPv=n7-$rFE|brtR^syC#3VpPaGxM{2^<#Bk_2c{!uFw@ z&q!9OK-a-sZpTN;@V>*97^W12VNXyvzM6AZ+4w=ED>%!nHz!BT(i+5WBN~91HuIy( zgx#kbwgH0q0B)oLA$by!Bo+1>oEwI2&ko(tRhYIK8OKshn2v=m}D*+ z?MzZJ=fVxec2rnl*jejt?)vUipa$7(*t+36^f%Hxu>4T4tg2VWj$o(+=!hxu@{ZsJ z=T`8Ii8gkFRG&1!Yme8`)&^Tb4axfO&>&tRc}kDz0M-)Zbj!m60t1v>2-Z=Wa{&&5 zHa}pvQN5XSXMq`Eql-C1rlNi3s#1h0SoNtFb4v&~k+xbb*c&~4yX-vvA0guTd_WKj zFFnks53y@>g%N&me3JkZ8Nw`379xiR?CrKGeUjc(zTlf&o3I+QAX8Nb3@fNG7CsyT z9gJ}#u5VoB9j6UY(6`LL9=ek31PE~`4UT4Enha7G!;_%3H-nD-y5*c)rV@EfXc7i@ z<%e)B?#coKGws*v3rdcJ>0L9%8$%0{`V)B9Rf1cW&m$5=5t+;^C4W+Rqbyza|`!Qd3wN6$I9AN#6mW=o;N0_;c1QW zD~T$&kmWDrnnm;`vR?#Ghgn(8JiI5BA#p4!6dX2o@rBGze*2h?wRs*Z7VSpfh^0q( z&3to7@uU5w#m69K%q%-*9_5QzH&zuLLzjMZIv1!~YIRC9W78#@GFHAXa-N;y;K*Jm zkt*V_P*#<9oN4VBhOQ;&BXOm%F=rwDKLXg#&FgTvd=xy=3%nOB5DMwrm+zoFa$DmV zd2^(bEL6FUaPe1Hi@3hoh`fOjFLlNq7RgjYJUE{#1PAWHX&4OmYG6C6c zdp=v|no;Md8kQB0L%^r z9f7se&q2y+0lS%TkUE;78(K2EY@!<*4kl}dmEG0ABX|{RVIn;&9L2qC~9?&pu5Pfjm^qv@8k2>H; zsSAAK)DAj=Agb`&QnHB&N^dBQpmjcgoGYDY#7LkU>RtH1e0%ZAii*wJqe-7%PpPY7 zeH`X*+!s=U)ul$OqdE?AtQ!wV%*0U9iXPerQ(||=sOhwT0|2}yxz6Z7@V3iv;O&gY zfz&y((6+FOPu^f*wb_axS0pIsNqWeJ!9GJYEcG#=(7H?*kU21CqnTjZtJxwk9(6Q< zmNu}``RE@M9hy8Q81(`2tV8SYjLm6Sc*~Qv7^VE6$1Kb+UoowI>tkgSa{vOBK~3{2 z@M;y?ur<8%v|SpwB{|@c=<-^l+3fs_DE=Bf~wwtFUk+Fby_0iY*3A zPOv82Cb1BOsjMpeo?#bc>L8V{Xp^c{7w~3&3Ctq{OOou_P7zz~F94zDo zB!rL=jGoLzd+oj-j`CyF1`}>H_dn~)j-ARL|-qn3rvWi9?wa&u)N~Hu0cDg2wgDTuq4ZME>NuY3k;Q)4wIk? z8ALT$fwn<_h(}DjlwJ28dsEe6W|}x?^J(=TFUg+PF`_<~3kT^>mJx|`&RLj1`*sf{ ztTro!7J|VB*8u{0k=Jb{LWcF#{mpiGzAM%$6y>>e@x>E3Q=ah)*m~v$SgWjX;%y5W zt(s;lg9PG(k{)5*c?gHcCZ=1q^L;C^JqA^tV~38jzlE;Kiej4Vr_M9gTN~?zRyCN4 zBGBnMnmSmDMBf1Sn-+n_c!5uqBpda-H$HKq#$l~}Y$1ERXLqCaAY|yvsUJy@&_97U z)D%J_CMn%k8Zs!3&AcrD$EQJRQ6Mcuqw7r`)&_o!z*8anOAwQ>O#J+d`HE`n55-Z< zS1vp|G%Y}Q!Z&^*CrTLRHPZM%q%Nu!!r%$1K0p#AKd}V}6h=ZoKb2mScx+uZ&{tZ?L0U3P)0CU%C%6Ed}iLgCp;*7WDe5OxHK2+8(WHV?^G{l znm}sW#O0_Ul}A>Ow~RXDGc9?u2-qB#fqc;>y<491UO*j^9S+jvbUg6r!hLUsV8t$6 zi8)~B?(RRQF952juDOg`Oqk$-0?GS!4L#<6Yn$s?eT!y$Chu9hIJLz;cVy%sg+&gU%?$473%t+jNV*Xmqf}RnS$@WASZTNVy7*X^2jTc!f=iE3 zp-F{=gcMO#K$GhjcrrO-oV)c1L%>f3v81r!2*(+#K_g@MTNFJ-g)_rZeb?pVy~#!7 z>%5kElQy%(OCXC!h)|!k#UcfCce5xisRES-SxF;n*fl^vE)>&ya{05zYDM7*f_2j& z3BjQU59*y?6++sG@WXQyj~rFji0p5exKsr4>r|GiXSA|>HRe?jS$wkRWyA36biJtxg-B5WAhO*tV9wdsQ)vvi9W zbbiEhrusF#4VElf-MtdzY!vEcrvu*2QB-pU5Df$8b7K0K%Mt*$N-&vuUVK~^$Mk_+ zQrM)egC)WPO)9%Asfl<)XO~=MKTSdNrO{=NFPt0@vEwo)p=`y+@u#^ zzFI#?%*h?FpIaPrNul41n1cIA%j*gQRux897a$j@(NSxo){F4130qLfWUP=t73c~C zkBDUHPG#6I)#Jn?x)hi8Za}TS7I*hyh^aicVhm^h1WAA$A6q5ZZ^JEaQ49 zX;x+=!Ird92>Af3YW+|`L?RyH%iRzZ=BN~XWhw}C2EMK+U}4IeLWk=hdC#HHXr75h zge3{;YggbV5&^KaVb%dkxdkzAPd7`SF=MD$j-vj4*}B&KO*paIy5D)Ym_w&9sE2^z znifLtk`trW^>ma=wLcpdtC=0}2D#2=l0Y+zz-c!ZF9`m9XF{jiP^xQH6BTfCFJ&T1 zp;Zs4b-A{-5+xnq80?-+wmGl~e0T1ZMi`o+@wvAXpkBsmrB<&EdQS(28yP0rN#GOV zgdM^@SKjLO&`9>WNWS~uKVFfCo!ue6d2o6e0D+e(V>&0Qh(k0Esct_SEqho>f{Em4 zhL0A_`vBN=HX3*4;wP9!9yK9Brizh_GcGDh?3rKbMCRc4mr`}9nTWr@9V6a5zeVf_MSKnx^4CexuBZ17X zB;KcQ=~Azc?~PpM6_F* zqw6aws)}5dq$px;1W-`wbT3&jKrSGS9V?3qy$+AaAY;R0BdBTPRr`tnp#src#L_hW zxgCbiA$U`e<=U6H33HQ)2uW!^8L3wUrJ|J^74Ig->U%IfvR|^ERfQNX#Qd2p5V`Hv zV}@WKaoo4dQcK0LksNoIi-8`3D5R=#WB}z94*c+jTNEHeW&FyMpBRlQkRGTSR1+Xs zfk9=BO=-a1kmHb<6s7u|4HgIrP$>J}4tN6%>eibW2g2C9g7Uu8_*s!rdC=h*=S324 z2Ea~9FshsYrU=A3FlZoIs4Y3`TQsuOmE!diy{4?+zhq9BBOu(CcQ!{o za+%zkI&5fvJfmTYM7cE1uqC*LEEgQM-x6_%)2ZUMeqmkp-lJ z#c-@yQLcjY9RUSkfcwTsud@aY0JaA0)VaP2*1lRgo#F0BRC4BeNXy)ENmQ7@JcNb7 zdllYrER$7G6bEhiI*gC42S-?#23+zv5bIlK*?7=VT!&308Mp#)oycY7Xatb%AYo`P zfy7g=kbq=V%GiR*mj>!4sXD&-7OnY!+L*&59jP%^M#uuPAJL@<{_EvER&=Sf80%Zt-wBh8(v>$fPUHaK(tfvXw z%&LbsjL4}1@&wPJ9<5T<6HQb)zhFX;5IG^fS=WS&I@g&yM&~a$e0J4jJC7 zZL=qaOLzLPwXEcR@CXYK1ek$%&}j;*(w%tmej%%|4jtY}pkX4&;Zza(1i4jgSnQ0^ z{AkuzG&DaU!173kdsSV7O(g?h^efts8A-h#&0x*Eh0eN?>hu+; zsj|kEa73m^WDZS@9f?jrNmN}K;|;%7Hm{#oQz=^YI<~8vH+cg1K*ehsa+;m#_m&w2 zsRYn4=G$0Ms0Pdkks^a_0=%=19>NZ*L43Xp&b~DY;6!d5NCbmCs12YdIs-g|(fcHm zTrf~?Raji|xbPUS;Rb3drq%;=3A%R+W1WwlL{bWD3Px`J3kDcIbPZfaaA1o^zA=a) zgo(638_*6k2lEg}X;Pqt_N6fx`%IMVj}N9%QDdoH>h#94narcIIOCbDPNAh8liiq* zq0udso*jf`-U`)95`}@jiqnM1qg5_G#0jKa5}_c4KPb*KqP(GvcZ(rJY+SfnTk7I+ zr>zTba%&RwoF7aA*Ar`S4b1b+wxb7wuaCo!K|}e?k(?`qiIPfQ``w_>>7D0OXVpvJeSlq{RV4ggLGEA`FLM z^AF8!RQn;t261$)SsojlWEL4RJ4s+MUZG$!1ID-o!gTwSm8krK9JN)6xQ0WW_&0b$ z?e6Pafy0O1HAWR=@RfS2YWdw*o+yYwgm)*0wIa*)4mQtVm`s_22TkRC6`JO zM^u`NXqI8FKzfq9NnxYwYdE-T8KpZzJNPgVq1W9l0{tqi#TGXDa3;lqH78;W@70_j ziM!y>SE%W$u!nfGWv(-G>%m-Z*ub$W0*Bs>O`l=hgw}D2U5Of5;+6% zu7O{arM~y-gb`V3=d$v8vedEXM#K&nEIdC~u02kOxa<}doon^quWzuRR7eJWNN&c$ zq9ih5hIFjMOsiD)80>nyw?~lkJvB}pz<^D>v>PFeL>pAUOh~@z%Q^{D&J-Z4uA4bp zECus+F7w0qZZ)NTp@!^i;J4d&Fl<-qSOdVoFS=jVZd_kRka&ylgk)%RcXUh~Tv136 zMl#aLfPK*leCrK#gmS*s3jy?#;ImFwQ7BZLv1aY4b3l|upxqeA60kZ{Q*o$a$r$p0 zBbV6FLQ4M>0SV|3h>#J92f{c&K!w>UEME(Bg$t&l#NJ$CJLr(bHXh5c)^%zVMq_a0zXu74WZ#6xVVb1YU}{Z zCAr3ZY~utE)=TLeIu!Epi5|_g1&wvW127!aFiAYJkkn_isNNmtbyOqwV11*KNxQA>mf3W%QBLjMoJAG@@D;bj4SpZzVEJeEU{`rf~cTU|Ap(cuf!tw zAn9=Zw|GfY=kfPoP8C!`aw{gm2zcK;+oeuw6f=~H5(4Xft-5R=F;T`<1En#cQG4}U z#of$EYfEwwdj^jgS|m0C$pZt6dc5ssctLXSNejI&O~-H8mGj(&->YRT&J3ZG)WoT>x-^#2oFQ8S<)f_&|as zQP({Y=oT&Bwj=G-9XjLyr7wUmSc~TnvB@JEFOdG>?;-A&K`xV96!|0u?2}CBDn-Ca zx|x`f@2LHu@-*#?N3T{ouCcKR^sNEeE|f#L_5f3(>cr?-e)q6#+EKcfd87Bv3Dopf zKL=>&D%)3_uo8+TL4a6_ZI!KwoceV_fI`|m2p&!L8b3L}`er+B?K<$&iLCH`S1$c;p$zR22!3f2R2)H4bwmpf3eh;#)uLP3{A4g2XzEi#1z zN4~sZ!TSg(!vxccY|f8i(4I#XJ}v~KMaL>8Y`I)Le}KKcx{E(@O?@!cjEX?I=I!R0 zc%G7(8D2>7PHVkG5MKbh_V>K>2}J2>CA?C>SgnXak9X_@5IP{b0ss*Ug31Rrrk?`* zq6IkpB%9h2Lwc&h(z9hij|-yL8G!bZ%LOK|te#4?DB?s~Q&^WrdLwl4t|QiHo*~(3 zNyZ-ylyw6;cn=x_fIdV`Ibk-5XC4xE0jj_#a4_|au~E1P&K~uq8$*=WzC)CYD$~~X z1g>e{p^_d5PIVMwRNry+(8EDKR8=_s6tiEb z3y!2O!L!`wO>ILdnzXcf!mf5-A;l8;xz#*MNkzDAW9Y6QWL9~Ckmh};S`<)Kk+kc|cVDJ~-ra*|{gXEd$}g<%<$X#{~$q?gxp zUTJ(N1Mz9`G;aa)dAY&s)cPCqGjLl#6XmueLOw2l=FC7*Q|Z163fo`LY9IA)_rj;Wvw3KOo{Zi8_WsiR^T^=%k10~SuNm~&IPt>SYtkBGi5S! zVq^7=T2Erdjoag)w4kg|^tNSDN;*h^WYaH0aPJ-`;GWF(8^yen@P+dqV|Oiw2)pyg zhJ+E5)-2&^C!9E-L-0t^x`&kJSDEZKdi%qU2&yALk=kAw1SP^Rzz4kIwKmlESEUs# zUl}U6g5`?VcC9Un9(mbkJ;surINHWX4xLoDeKC62gk_rUJ|s3HGBwjro*Ii77y@{x zQV2e_OYr5R-cSw;3=UlYDIZn)C4^~}ip8Qe;AlFg4)+sJ!56R=-tLtHnQQ(TYuQpoPSfHXb*XXIZVK&PO1)cZh~0q-Ti0)2>yN zQNj>CK$7^(sl%1IY`m0P%LOd7c zzis|003vz%-3g`0)^p8m7c&BGzx04-S}Mkb^_r75jk0D zDPXmg4hD!CQ`4%HDmx;-bYZF0!ZocIOS^@gBWYnd8F+;id0h_M z244_s5DA2lGn(3}IN6wTcS+d1)5^9uMA`Xh)Yi&WO69QvjUESS_%A%j5+Z%gQl&2H zk(BzWuLVN7=1vz)UfR`goal9Ypp?xJ&#Cf&ib#fNkf>HH6Nh*N=36pOMl)0~rq6bn z%?*+Pl{TCktxFt=4b&wp1#j{PsyU#v-wav4KFcqVC1v`vx5E1%PYJ|*A zWxoo?Fq>U@Zc?AWG4Q+21H_?WS+bc{7~Th4s*d?bYHdw?l<%T#(@kAKHPh)yFj08h z*ucmQj`JiG902k5aJaQb+O?2IR2M$tXErS>&ZnBktsD;r>o(BHptlU8Xi%;p@-T>m zQJct};JPA2_1X)Z_loh<)cjH@WK=+a!_ThDy<_AHtudVlAaPW3CbjxCvWplRYhA9! z&V=y$9UYBvT+mboq%3T1`j~HMDi!G}4FOydI9+Ao7oG3Kro^s72ujP~u@>O^u2u%( zq9lYH%cT2kK?HTf^2wQ1?tCj%Djj}7GnbAS^n|sq0{K`L}v`KX-Y{v1M3S;6V7d!&bhQyZ;q|R{w zta3*dT_w{*zLA+p3nUN{x-yPo;;2=aQ3f_MiW@gla2DhrC<57Y4A5yFuwb$M(_1O9 zhi8zgWN>UoIu*{U4pfuPOfczG3~ZNl>0bxpq~<+rnq9Tw#fk?TmOWV&Nj;@wumohhFY7lwL(dGr*TGB}58(m7a^kU;8DdW$ zgj#}~EmSz_VX|1t9L9~amwm{b`Bp4hU(3p!`Sp9FW^<{9H(e+pSg1%|z^HjnRQeB?ibZj2 zKrg%AWQFZxz(OuQx7PZkh@^o6Jwz2BJxCFP)vufzQZ+JJ?efrlG8?*I4nlbM*RqVw zG!yRRzLLJ6PD0H(rJ*vyBOZ+k8TeSZhwVNvlc}VJb4K7Am$zVh-gz2*9BdSlg&{@{HL> zh=T;a!jXst226KQadlu3GFnwZ0CR!*`gcO<^Ti~TUK`asn5yT9e`b-Fg}4+}1cgE~ zt!7QbrnjvyU<}0rAn(zAit6!#ZS`mB2&S-{QQGL(Ll%7?+#&IG^Zav6vc&{945C)L zoI9mw+|zZ^kEh^iQaR7Y3Y^`v?(8Va=fYg)37@%U^sW%k(lPOYG9C3j;S+^%5c-46#Z zhkJ9!GD*(>!K`+LEB63eXC;GOIWu^Q-ZFb8VoaqKb5x$4!bzhbLL|D>W^W$JA*o(v zxHe*CYvyUMUFD-!s72m+q0A{It3<)e!YIsvl0w-iS%wf4fL{^I4bwtf0vwj3hjD<7 z`3VNmUdJlf&4_}VK;(E~w&q`GjpS}N@~#*guL0PLDb)f-`93^OWlNnuB(ZlPfZ0QQ ztAAnF%i>dWvGT;!d%7c>>3-T!DZW3XAB-m?jY56B|@}Lu#9_h79O<-rnT9`=V+Xa&>p<393=8)9=r03 zb2AN1M4om_NIVz8%ZFX}4iigQCxqy;*TSHt4*I&~H(4KhFy=rR%@U5@8n-uNwnC9i zrzJ6fxFFz%=$E2Hu#hXA6p0u$;)(0Ng&seZR~$6~HDAgI;M&H1hnHZWokS$E(|OmA zkQtXw#ZMxGM~w`v|ibQQ_7VMp2$2?`#DJ_gwevj$}R^5jA5gGmC{PP)>}V~%K} zrdI_U0+Z|ay*2`1HmMifMmkH~7HMu?kVzt`L)WH#y}*~W9j84}8tqb8`H+5L{UEiLlFk8Ze^z=XKgFeWFptdKz+S8EWJzA_}z*w51rP z;>t71(7v^Tm>1nj<9yXTUj{vDQoBJR8vs?ma9$-@R+kT2xKSUjt6ys z?)+eXbMIqkg;6uLY>~D}k8-uSyiHn>NY%Yt!3{AHcR<xNpvg8FC_*FAxXv?y!4n~GeInRqJc5RGy6z5ToDDv2P*x6>7y{M$>bP%(Ja^_fw^@g=TjtX zRv0=T%F{Z{2zR`&_#I99WK)+h3LL(QwdfA$IJZ#gdPe5x~1t7Jm` zXoPJuA*01z;C5;Vp21_b95@LzlUalIQ=tZFrB~EEUnkRwh#)`XmH`z3Z&?1 zT#_EuxDu*k7#|CF9LsfG>&Kg>Msy)8BWqCjF_|0=rRN27hYIe-y51`($@t9DCBze2 z?uf{)f=f;hU~y^ez9gb@KwUsk;o1>Xu`aWNrRwwim0o00u(#K_Gz&>3eVYA z<%AVWDzhsTteJ(mNy|9R>bh~16PILDXw%#tTRlDxg_sLEOvFKE58?ww<#d3qv5{YF zbuV2|*Mi-k#*!c+hD!*08@X}k9zL~esZn}0TIC?2cw*XLLL;jCOiH?@Zl{HUTsU({ zd6a@w)sMiCj;PU;IACFKV{!+XJxl3*M53=7v3IT+eSwY5&X~AytqNGd3PP16AgW*@ z52WN!FEB2FIX3NSILb~CGgHgwyS&EV8VZ&<$h5r; z-YJkF%BFQ5_Syr8H5Oget#d$H!u_~yKI8Cxi-x9Vx)$gv5QR@k7z*M{rdY3*lsY3% z%s`DD0TefrVHY#f#H#O#C|GXoDv2D%6L8Qv!K+D{veV7u&UQl8BbaOmG+d8k1d?5( zN3s^Y-K5xYNj)s~_)Tua-GK+}lr=)2cz{@GvxaIK+{}yRx3t?%zKzQ?^4jc_+U+Ve zB$o@1)&td$qM_mV*B6<-5ysPzkaKti5UW|xdJu5UArKHp`;lZ!ToLGPWFQUQ>8;6! z#4r;rLN@BY$Y!mc=N(?m>=5vh=^qGhE_++T<`r7PR^a|Fqqp^fO|@bVKozc$pnKie zU&3B1)ius9td~q)4EYu&I<~v%zi$-tirZtx>7^JCY|s$7J(~}7E!;EFxH;RmtoZI? zRj`}q(A6*_dH}lyM~Hmtdd^$X`nxE*pv-e^PH&US`qR>;4JFK!v?s*wA~0q1M6@f6 zjvV%=K1#NfzEz`{sJ%FX1C~XZ8t8%ri$um-Vg`r@b;$ud6l-e6->CQn)Z9{UhX5ds z-e;lMC_3G<*0Sv3K=HnSGAr>7DU*f3O6tDm5s7zkmAV!(z)nR-MENQ8EnejOO)N7{ zgtaE6Oaj6{RUa`eWgg#N{OLE(Xo=34BqLDv0%Z3V!vw`Hlpn);=%!hFQ*Tgxjky*J zEW%oIr1$r6*f?A`ISOCPnmTr_D_Lo6qcJO*{||(;*8R`?0}G1PDNX0aq)T?I{IjN_ITVft9NiVo^3($Rz@i@-I1d9nE1m+?Y!cbCM1@)H71kD>TIN zyfwIJ+?Bc zI=VJ?lGypSNH024_R9zn(7ln;siD*I0#S%V+g_h+#EGT%70dBkz?zB>hDt;?kA&35 zeiK`u$N*;5;g(VP{Inz^l5HC5hz4~R$jn{R)oAjNx~_p9lhbFN4$?E5!?#CLx;vd5R?Z& z18#v=>JfQ%W(jLd9G66fVz9oPwKV}gL|=Iq(RH5nq`kRVF_1BD!LU;NaGrHgs$~k zddrQY&_}Y|b;J$=56i)-$cZNgvU-}%?9KGXx?oEe4uG9KYN>`nO%#>lHkCs_A&a@6 zW%H2_R%q)G;VKlf$95g2XQPJFn1%c!z^s5dOorUO7;mM`N5t=IM^R^~lu8VeCb@o! zlK4jqEFo7U&=cO%kDTX)KT0w-nJqCZvJ!ahjS~2v*qs1=REDvSMM(#URLaom&$%G9 zZmUXEAkbW{;xN1+VgwmdfkMIwtV#*+!*VW-iG>f&Aj8kh3UoJwj0Ym%M(gc@?n>TFhCb`ncQO`ha5;|Y*iJ%%q^AYaPT zQk*3}3xhda^I18gU=5`v<3g`%QT-jyL$u%|_Lhl3s@^N5Ta=k#SpJ6OxzmYfqS34; zO=7X}nx95lk-Lr+9f+w4Fmh2)G6LoYQvF0k;qeqIS&Oo>J+4I|9pNjFBK)z&St?LX zp&3H4B&ZZ94JyW`l==?2hLs;^I7a+J;MXC=;O+x)#X6w2)eCYC-g}M@Cz;f*Obf(= zqM{B?xmw~r`AjIBOUU5;nz>6ogfnCUzmx(U&?4F#&X z4iF!d1Ot{2m65+jb`c1;&W*?x_b$QA1p^XE3Q9VYaLV?)2w zFyC+z5!$)q(c>K%i$U3%!{@JS>3BsDX0Mub{|$us?=)Ad!gRQ zPEVm!mKoyGr}XN->R=psaD0~*(ZHz)#P_@~$&#V;vU{fY?Anpz@}r7|Ac`BAPJsDwkep6@&1JR`j+gj-ym8xF&8=O;dE`Y&b`35~>sJ{bk!RbaL{CyPe}L27BmmdxqP<;TNnhX)DHoy1sd8<)8$ZgB2XCH zunF=k(tBuHct&2NB{D7;5W4Vb)H_h(K_Yex8%}wV3KABRh7Ds+wVYl6OH|@Jpg>q6%Bx0`>Ub&R=ddu37*K_hS0j$n?ts@^nQGGOCDOUltIq@_$dWI7 zv%8>mdNQ)1_U(PpsY>FS<;G2-vcwY)z1B=1WyH$hokhLS1%IpPnR1Z7ngxFK5@ z9!nLiStD-37NEib1UGHvxl6*mZBormBzCf(QKrtSZH`odTs}C5_+Mi25U~P@dVaU3EaVF5g^jb{9wDZxdX6Z-He|0H{ zXo$C3D+q;kqYW8a8l;Y{;RRqXcE-mNvD}gKu&X74_pz*PeEi-JcEq_Xstd}ti+=gY zWviuKlZYPt8N?$2s4OlMdew5^*_le1b;_eMg1!!j85tVCCHgu-`<@Ye1Ms_m;PZl3 zwV@mroyt0>n4y+=2Ggap1RCSgBD^S2b7k}l56d=t=tYe_)-`5VrxVkgyFoKoO|`WptnjgJ2L3bf?#;O0R~_pr}Ca5PPc5j_;vXfJa#kL0Mr|q}+&>KI7?Hln*a)%0D%nDFHro?%XV2?#lu#kw zeDtM3Np89dPW1hZ8g`#)p7+xMqeBj1etG~ab4qBVwX!}iQ-xGh=Ot>o>Dh$Le#wTJ zs7ERY3Fo*<8gVs>p0sBrNisST2;udeXL@4ki9sS|45x#Gpl+wSDOS`GccJanIu<({ z4FTw+LwRZ;TpK6`@>4*r$-XR^x`=YIQuRv$p(w2uhun(nplR-cxip>1hhT6}m~igR z)5=#OqLP8saSYs@C1)$e0uyxePvSVd3BqcSxcbMMe4s0KE89cluNl=JjlmE=D;*le zMU@Yph*Y<2cxDdKgWM!I%Cg5I#b>cmq(!4#G1B|1sTWPh(ld5jnRoQD0%2nk)Y&2+ z+stGgCkQzTdio6gVN&WMWj?Q!K?oobcVsnv4L88_(b7GzHqWA%FYKONsaxn?uvAr5 zR87i-aILTk2&E-^SUwUyyD)a??bPC@%Mq8Gtkxn0NpuSk%o^7gXZw^*k9O<`SyAhQWE+zf~{P z*f>4Irw@;oj)CJWA<}c-Y(v{a*z1rwSpg1!Z8VKi1u%qGg5p@*OMVtK-`Q9 z#ycU@G+UJq<-8ps7IddgSATM1r1ap@tvoSN%dM5m2zhkIpz8E0X=D^5YDjcM-7pX_ z1UmuJN965Z+VwGsI(98RnFiF-LQ>-26dz9uw}ixbvsJ}`MgUoXge?KR00K~84$`Jc zY#8yLcavKyV6nYktkwtSjj{owLP8?&zHV)(a}>G5(R09*m(cDCl2uIyH(oD09~h#A zz=jM6f}n9F51qF{^sfbmfK#db#TkuI^%nQ5q==i4zB5sDNq#7?|0W}EtHqF z!Am%x=HH|eDG85QoeT^P>RBC+3`P)&sa8yuQI2S~7pBI@3G}7$5uHf^^x9#8M*Q3` z4Ral*ZDqc7nyJ9K^2v($*|L)BGX`djC0EjM@m&d4tnpn<0jXf82DJIThf`;AC^RGp z*~LxQ5gX>srb0`cl>VTWj<4c#Qn{}XiB&6AXYpa=P z8Hg*=Hzxk6SpKvqtXtJ=W$6&cW|R{V_=-7kFx9UCTn&O z`K=X-k|&gysF&}+w!0mxsg+tT5ANIh-6;e zGndhYeM^{(?TgCaV(sDzkH|`RflYyKG-Ve&A>K-^NYuMqMOfQFlfMYgY{3cVvE9lB zg~AbZ5t9!HShW>d)Y34C0J2&nz3l3>K>$IRnKSztoc5J{!>Da`+Z_eDP$d$;RlvUG zj+D}VYp#dTtMpaxKR|lpok5NXvXWvg~ANDd4fW@8))9#b4G(y@N7(VZ%vxeW*!hS zt5(F{KnjVYo|H+1LkKMgjNN#hnv4zd$>e}?-6ywo(5%OYMMZ?Q1dxd`N&sr$Ov0jS z%QDTau4ep?Rm|*qZ%Qi9R+qfNT<$~dV7!DSrd7|OLiymGZqRUzYsWxS$xb_*l%!JE;MWyGAcMCx!A|94R{#HkZsE4Wea5&S9N}V zyX^0hl-xrD2suK0Fn;F*T8rIwbdFxJv1o-EQ&wTnQ+ftu6O{O_2p`_mTG@+u7FwO~ z#ujAhgGV*zhv;W4CIv9gDa;)C=6NK$Ma;t0_L_}hNg;NY<&e_>#dPRZ97=DDkjTA` z#T2`US1OH2#bjlE_M-h0ecBBHR7YiKJ#q*U>oCWQ^tlTIWR`eD7*0pS& z@^%49-hPG*VD#Tatd-p}=`4WqqS&H`IJPBsX?p~F(<2n@a}w$-?MA1V$=P8HVAyxg z*`Oo2g%^e_U6OBZK)iP#K?6h3=wGPo=49q$i>2Z(xK%h*EvuBf6~4l83UZ)E(3<1=P8eD^tub3bItKRIXf`dBsDW5 zjwTwr9++O$j0?3K^t};df?}-P;?(t>C1R<`mH6n&6;6Q!FL((WTJ>_&x`zQFNG^h4 zw6u|i27|;(c-^tOiM|jclN(i_I{n$zOm;0*s->3F$XCRzXk~x3SO+w+?FuKVkJV)> zc*w{(t8oZc)I`;OmZ>s_FlJKd=q;t^vtoG#BDH;RnP*G6ZO@pNmwpQ zEeem`yc^@R_N6PzrSWc*rKyp-W)bg5S@X^p7$MR^O+|)NquN6!pHeLGv6|x*KqEbc zBR586x^!b>2=N?rqVo*)CJI-zxi4AE5{ZXm+qt$q*t`OKH|~)7wUjm(9IvOdy zCA>(Kl?zzz@T=Ad2(-GTDL8Recd2p{zH+X`JnF~ifZ)f@;d zre{5ibgU&)RzKG%uH{FQstS{AR=`FRH}9fJK?sh$FAW29XcbJjXh?b#p$DR{?}o+A zz-lf#0eyj@@YeIInl|sP>r*wPK&M^@WDCBD;F|_09S}&k3q?_Y^fIa!Dc?1*1Tc+i zJNHT2O<_v$APeSu8ny_`?MRZWQwkgvD8WO5Q71l2VAKL82y#zegD8tgD>Q*3)XkKf z7~`-~?3j8;V%i{8^5kN}Wwg!rx(&m8UVxCUqV&N72>L^>s`7qVmF+tW3t7ure3UkP zAZT32(6N6!IeTq+k4NY zy_CZX`ebir#$AeqMh)i;AU1D-kxc;o7#Bs3Dn8r1m|$7w>u-wLpVm&IJOns0VXA@8 z)Ix@%numF|NV`j(FowvJFAH4QQwNW`R;h-q+v+*pg5O~~mjw1Nv{d}8x3+`Zuj(-5 z_iqs7flB)`gj`TWjySVyZ5EdNd}+`t%JRZ1^;nGfP@lyomN$##@QHm|@x%o_nNR^f zbJAHI-BBlIHYc<gdlK0!E(HK4iV)LvRcw^cckf=gX&oGWGM1J=C zMdy~ax%HHY7o*|JV$w#A&?Ky2y7!TGvxR{w=Ji(wN zystJHfmqmrr2A(io^nDPAYy9Wqx z;MCTiR+^DLWH^CLoZ#gUrg&p3phorruATusW}#CQ#CjA1h@sFuYCz5phQT%(Y^Xhc zJnaN~*JY+OB{8aMdUTplY!-B4i)Mw1aXfXb)l7o7mKhbB9q>X28|M)6GOH0KB2{p| zqQ={_V|-3_Dm+XDbY!uc>(Y7^vZkzPC}TXpb(ij8mVgUvW3lO=jOW@&&IO_k#4y8T zh_hft)_Jo>y$vH;9hrf!3lJ#yhraSZl+@1$WLELzB}?Fc21PDK_9C|`!|5kEo|UGT zK#OS{HyD=7ESuU9k_Sry*kGyPi!uB-R#prgwz=_(iZTe;CU}+#@;2MwFHkQl$*d0y zE-0vu=X|!(+A%p6L{bxD?+~tm`ro2u8uxO-p9B*wk`bxIyG$xo6?IBfuvK%PRU9}w z9nUq`4?dNgXjI>^Es=gSq@dpHFfwD$5>6I!U zku@58a7hjarp!Nj?oc*Lgvwf27Zp!>@3E$_Y~#pHkBCa-l%Kl4JqR97$D}>s$g{o! zjD6Cu&|kC4>bi=tXnELd=NJ5UL?4+q9-h^m*?6wH2?`yDmR zF1l@=X0M5b^F2!w@yC^TBWbtQYcN75#0oFYu+h@kMrPSu)txaSiR_tqjfT%3A6~R( zh+Ta~K0|G7KZ^UJa3a(TWVdf{Xt0pk6K)$1(7FiaZv4}>8g-_t8!67vA-q6%LRMdT zK(zAlUn_yv1vsjNm64TIGdWaMO#LAebCfg>c8r&bJ>Z^8QB@e(rbem}yYH{BwUVNRB}%=17rM8Dx3SDx1#PhcjtfJ!O`q2rsp+#e|mcK6v7% zY@&`mJi?BR>u`g35Z6S^VtTGfCc{gOPf<#%Qb%Zp%tK?_@iT}J%Mgmf@U%3vAZTa2=2+H%p$}|Ev#v@(dvX(Zg~x%7EZh{vBzoj;ERqpB96 zoSwn46GaTY0O0MwEmpG`C*^X$#CCOke~SCnni$7u$Rqb@2zfq}Vm!i#3-eg@fiy(m zGbsH-B8ddLnkb~Gt~GmRjMplqN~f66zSjfr^9OpC+umeEQpTcd`|9MdaX|%8#sf?X z!8MMY;f^4%o!-#0hmmHRQYIsv2Z8|w9i#{ZGPHA5XR4^DX2APG+WV@|r6PRM0E4#} zgl4Rtgvc+Xq2011!fCvO>w+acDb6(QHCMO9v*p`1K86S9VLFC!Z2<;pmNf-|5aT81 z$rBd7Qq^)!dP=HfLuQh>7tXBSlWh$*k>2qgD|%IvhHNT6@r5+NcPg|h9~&!IwfZ3% zM?CB~z;W1%qr?WJxuU2e+zdCuZ+k9`g-*k5>4;SNk5 zMPieZ?F&{K=%^nJDK%sFQj;d7{VT1{Zo{Qbz39E~#K_%g34=vs3Ux@A{yu^1;i#=CI# zN$OkScWe^A=nH`gFg;&BKoAHmRb?S>WqR5II^I3hBPPe@`sv-6@h<4=fLdJ^ywY)2 zdV*qZh0;LQDn|~-t&4F#6J1C!&j>NI$=t6A6mhYJ6o*5$0J8{X22+3tmDz_$2{m+& zfM2B5xBnhm7P(~P0DKeM}tB&lUeY|{}Ch;-@ zlm~mKNCRe!XKaVfuuwUvXK*(}e#f+zr93m~1p%Q93w3+M%1OW(CP*;atPd~_B6AAr zcW`>2ioAzDc~XIqYY!I5ti`V82##A~uaAf=B4C*Y0%3a|3xi4NXTc2+8d8-pxooR~ zk3ri;q;R`va1v%lwC{k9e3z*1rP>xYS1K`58!EX-lA!pj+~6489?njz+j>!}fQDv<(DS1ZE8hh`b0j!}_y zjrB?zjYe*tde)WR6k2doRoqhcXA%ln;5;a*jAy}W3(eAivyF=t15BdvXaxc&%sQN_ zPkw>IOLCpdl|LKW8dR7!hiAJMc6WFV-@=CON|#)KRawvT?WGXZw2ZMYA!AEQ@?kQ; z3Ai4~6!)RnWQO?j&dccH=x|M! z5|Dw`WUhZZ^Dj-|N3B4{n;8~d(`wyGBoz^5kcLPUtvOwTwjT0X!mn}4ugjrYx|yz~ zXIVp9nb7L7`+C!gb7p6_HaYT;89E{=w#`Z9$t(MhBLN z!p-8wn7Sw2NJ`lm?NNaw!l|fy7?Ue4Qjm#%KzRWz^h=|d=%PsD9St_I-dPdsvrR)mW+s9_4`=LGnH(i~o|!Wb+(P6XA2;$D)F$$~zQYFp=R zxQyG&Lt$gZ(bOtT%=R>($mu*$j$y8?X$jG}l;E;NViPQ5lM`!IzcmOG+3137k_*_} zlZBzWh%f-uMRdUT8Bq~jKay4XG46iY3^T=mqyndOr)GHqM%ZA(Qvz_AGs`x@Z?3<6 z3h4So=amN^yj?=0q;|Gn5g?hibDa$h)lt>#(y42OQKHCMO>W6E8Mm*@+<(D!{Jts0 zz?08I`%+U>6X?X1l&(22CpjUhLp#4q!v#hIs)DJbll7eU8H#tRmw9e}r)6@c49cm6 z3O@oSTX6}2%Kg^zhgsfOSTAv2Y#~^Kfr}KO@?^>)i!I_4gsm*SFVr5+YDV&Dxm`Z7 za(2Gy1%^yD4Q@DS9y+6+Krxvxu|^%x5c0`lkfN||-T*qDF^@p#_j$euWo%Rp#@Xg|CJt zTf0xJpXFA-IxDcKvDfxY94#0Cqn?#^#>gZBE3^{u=)tCFsf5+CiAsb@_V+5BSd|{K zv!-(<#;q{R9iCqv$H15CpE7zNu?)KW`fR0zni=kanADdI!M0e1+Zk{zE zoY2Ugi;3bKH5aLgD`=>y(HRw#<$gjO6E;G4Y6&WODW7P%we~H_0WaIq?(d?tzqP?TXTC5YIYljdWmifI&j9VSz_m#bB|RK~d8QF*kKzYgaTcITz`idw z6w4?GZy%Abt$|UTx#+?v3Ir)7hWZnnW35F37Mq6K%~sCD85k@i@L_9Ace@yX zg~Hj;n-Cd<0Uc&&F0sVnFeD`k>~y+9FTo3f{!1zA# z%)F3^(}2M}~K zPj-7z-ncIoxdU$%NH`=m>Twqap~j=>S!-Ll96sC2&Dcd&%BqAy zDyZw$cezq2aL+_px}QY@$mg?bC^EREu*>e?rs1KftX%BhV=)gH7^XR?!`ag+rC>Gf zof#!oN)VK*SUcMyh$Q>HMUN_Rx_(z&bP?cr9Zou|s#qJWRM=9N^cqsmgwhXD0*SA1 zKoBrH=N-BeapKyeq^UTEa)K$oQouq%leaEt@eiyNM2A8p)D^m4#I4(tZ>s_+ph9D? zP}x0$4dY1@2Xm#Z@1U8ro2Fp9ktzLD>F2+!!`O=^di- z`iWvxwe-=oZZHR94x_Lo%Cm8xV_C@MUH6N9hImfq=99!hfm)109kA{rdm%ciElsOv zSQjCao9{M0b8C+pf{WqA>wYp;bu>ezaeI12gg z>ktq+I7+~_FHpo+4h(n(->$u7?|4#zjZv{#9++<)%5gK5^!5T3MG8EoR%tdBJpI*# zTN3O~M=}jR$qz=gP?U#A8HmVkxoqE5(!ZC3fV=6R%Trw<J!B(;hd0cz=cT|msbiEEYU#xuKf$VH~>jNw!cxUw3D_#IY(w;&8C$? zK@P@EkZIHj0;4CP9wKB+^rI^XScgL#SrE#hAyMVd z^z0O|9lzW&5;e^Yg*73=`|U#sTQZL7+L3O-l-Lk|Axo47V1#IVMS6b6!cECs?<#zs zS|!L1X5kdZ#C+VXR@FfU-8xBDq%s09o?>O5Awkl%Gg1;SiHHI}F#*FR`}5)=_=t<* zpU3O{{(b)+uRqiD-M=4?$KfcXs`&oj{KW-inN4IJm9}gOSIKSw06+o}|9}7x`~U#~ ziZ1(`o`Aylce(exS3cU=yO{0WsaD<1yPWN&+p$l)_W*k{GXjJWExjGq^`&g~?z9@U zy*C#04M?uF>jVfDbYdWFt6HS(*Dq!gCG2{2>qX{jy6e-|TIrQu#-g5>bkw1Fu~92* zHe`|)vDYQB=;DW?RVaw4rCK#03btPEsa<=Rl=jjAwkvxc_OC+}Pc2UN>XGd8wQSJP zwT|81u_^A(BBayPtJg*wxLV?P+SND?6N?+t_bf zkan+kZuZ{nG>Z4PRO~VZCbUxNqCMX6#-XQqPJ~zz1*5rJ;tupwqQs~G|^tnde^p+Cq^hbXwKR= zgWBd7XzNZTb?M}E-MuGUr@Xz?;wlrfM5y)+0H{)@YuvjY&Z*bCk6u@8ZJQ+ebmDI7wy-+%%X8eV z+f=FBYZGg+)w4xXyEQ=`#;d&Nc6xCms_tp7>IZW@y>j-)ZC9)oZF^yMGLlJ4Z7*3$ zq)X7!od5@3hlyUzMN->U)~<@SmF=5qhel9zYEf%T){yq;bJKvRt(&7t^l3>bHp`~2 zvRgN8)-4*TU6|e6uIzF)Y1UgcQ7H4=>UuYA4Vy)}=Jp+}RU=yMq>A3%>CO!;o}6Bp zLXF#zAU&0fv!`@v4YRY;q^|dF%95LFrd3o<)tdKYuWXgOh1#O2HK>a9rEQR{ja3So z>)vXoY?N*5wTGdvY&OT3FsWX*w?WBwxNx*$YeQ`G?l){Br?zrrHM2dtnOemr+Lh}d z_HM9uz2QU8ZtqT#G-mbBfB*mh0000000000000000000Mk9xD6xw_u-h4X#$&Z=JP zx8HBCZL&LS?2)Iw$Gpw5@wHOPuYK<3j6L_qT;FWFVIn)XS60g1?d#gxOWV1%)i`Ud zmuXr%qh{N?M|0ff^IcY~lxw}6=I-vQG0E7VRkn(~z2}an zx$iZ2`)6f|Pn~x=Gq$4L+_Ph+*F^T!dlRnQ!`H7pomlr5b!xixy>-a!cQg~NJ7o#F zZ@YT(*yWaO}H@)PCySl5Y(=To)^`qWt-Pf~coNsIGx6Ti&2!^v} z`|Z0D+jj2t?;X|ecX`&7^zPl=u97A1XA8Tz)^*j*ta^3c+r4|b$>mp}lWkqu&1FSN zJ8ppPP22$0)-}6ZOzd^bMzOkXj>gTGbFSLb*1KX|CdS*j>w8AUvAp%T3cdGq z`g`f!>3Og`(|YS#_6t+p?(X*N>>9*lY}ebBc4wQq`@3fJuRYv6r~}uocJ^lHy`HLM zYOP^e7T(>nt+MT@r%T(>+jq5g_Zz-l*`9s5+j`!{D|fv1y>wa)UfH*Hd$(S*+1~c; zp4?q!7VWz>*|jQewRKsuZR_64?(dcCNx7-5&%D*+=-tS&7 zTWYJ_!>+*IhkMP<*`U{_k9%;djcuE{UG1%pce}eakzaOl({E>TYLRE$bJx3jxOwMk z-9&Bf-rlXf&aZo&Z+-Kgs~Tzd+n!#X)w=EMd#B%d+P>Y&zM2=iId12@rDpPQoxODR zX7sV{?^fH+R?XYjU26ASYr8V7?!E7CHSc@0D7(7nr(1Vx-tTW5*1ER2cVeBg_U_v^ zyQ_+-ZsF4BUCv(ajQ5A0w|7q0H##dqR^+c|VyDhzK?RUHE zS)qA&IZCuZLygeax z%eyyT-sh{Wt~Xqji=?`9;wj()Vx>k1r+@(|CzL=g86A)lZp=gLT=AS<)n#O% zX#|&ErLcA7GnmteO<%mM z6((TfDOl#X;C7u7n)kDCg&%2rcZFNV8QJRc=P=)s(#>rIEOPHRj0!6WK?-T?uq@j~ zZa13^Ifge^&I8^~4H&ImwnNr&(KZ9`}wy zQUr6sac&lNf||C7g0dh`3b3OuQ8hQ%hMhZGeY#JDL!)+Svz&8G49o^miPOi3Np94I zIFbTwtKRX)pQB1v$BBq?_B*Ox$f3=KK~E?qNJOx-Oy0TbTnmXW%{W`&c8xwOdPy$j zKX#4Q9l&a`aONXam@(qMN0JAb!Y>UeuhC297q}C>Z83JIuGs?yxCIIxVl@zyFzWYv z1BH-N7c*Kkg{Gn8cUdr6>Eb5jbJF!Kn1)Z50s*dKu#a;gMZF@UNPJAoczkW$%W%y=fG(WSUirAQ-dCxl<_{<^jZY9%FhB?(2$v8Y#RMV~a&e*c zDU``0RgHJ;A)%}tCa({{dU26!w-rlJtSx(1`HxJA9Z~i#-w;l$dx%RTfXX4d{w@_h zIMC?iDupV8<&YI0oQ3RI*K4`?v(L^ry628FHyv@;J9Wc@m);jSx#6>Mfq+V6r3gl0 zk(iU`98yobeq_Vxp>ePfEWVXMN^)|L)8wBMo26F(KXrP&3?|xrZ@?#6F2qB3LR}r9 z3(cqD4NLSmF}?K;3yA>{2!IM|9y#H<)tIfZPH5P!IZeOURnLtNHYS$OXma?(NO1); zrj9EDViPqdj!N0BGQ=R)cT3mCAO;{K=`XBkLa)Q>U{#B1MUoK^TcD0H(UDP6mB>ym zkn2TX=6;a-BrcZVGQy&TK#~MF2C(RK^%kBU%~g>~6bjxutWJ;(QOS9m9> zYDfrjR0EObM2jCL0RDzkjKj>XkQZ6((k6xU_-i)10GtGdJx&Iu1k+qBbYDjjJgJHt zd3W-!1ud2ueF3-%oHwo)M)-kcS4JWN7b?Z*DfKU?$&)}pG>8~NyzX?lc_UUy`QxEo+n|_&BJqb(6NGbeJX`bMh#SB|WJ0Ft?nPiLRKi)A zH{P3tWS@hWyfp+IFf!+go-LGkkp$a|Q2`Ml1f?r#K=}eH{UHJ;7T4}H@%yH1gUxM* zb6}b-52pEzi7ce+HyK!%H@Q(GPJ|yctGKkio{f8Fbp?3xk7ksEyoeH%39X~2q-3W> zX;DPdp>JI9w0GRe)DS$yy9T5l(y|A-` zIro(**2i+Q*YuZCI}<;#V71n{s@g1rixJ2wD45a(6;9cyfMh_%rK4XF>V=VVgD&xw zwTv)K->e&m5uq0|uC3qd5K-`eV229<2#|>tPN-`X3;{mX!0DJ^-kBBZTt4xjn^G6x$)lzv zILQ85-z3LWFQ%x;a?_1p5Mq`a4N0U5SC4PnZIIa~! zWKlxDR0Q0F?$BD2fiQ4hIT4Zut8yq43eG)ZQS63xA3b42mvJW%EkYiU6t*YQTJyT0 zOl%Ci(WRBLXhD^^AO!6cXf$AoH;aBARD+==7!kbCyVQ4@y*|Y4AB{GAN~GaqRqk-% zhuyF-IfdTnj@*8u-f8Aey=HwyxNQs$T}Uo~Q^N66ibO_{AR6?yHv<^CrcZLOF>nO| z>EYE(&vRxgrmaPT%3Y4=JWm`r!HIM^j^=zGl`!^St#;O;1}jCvS}6!J@oI{*BvABD znqCF;85az|evw&=w0L0x+ccReL>B_Xy7gPeKpYLt{9c)bnhmLR+w zWqPI1wwxsgTev5D_s0!iO=}$6ut(W69AQ4F3(PBjB6Cs+4=SqiVvHmC*=3p6seLVU z#g@PfT$(XZ5(|xosxjn=Ljm-zJkFY23Ej9u*X&%vpLU;t?dQ6d9tnmAv&8X0LCis7 z87OHH+V6ThdrZ0V{2gTtuYiTbM&&q0p2R1;Kp8B|s88FbLaIU_MM(m)-+F|&+CEm# z$`+8PN?OiIN7R-Krr0I8eH!3vbee%Pl09P8m}}zlIG=BV`>3cb2mml8*eD<`l{LJI zS8CDjyQUoAm`w{VfDyxP7)N)&wqH_G6->XCT?K7GnYs&OaFtU!S1wJ3Z!8qZgsJi2 z9L6M?ZZ$}0?Cc4x=n>kW4@{fOjxemGrYi9u9s?&YxR2(ct^oCZtE5r0~Fyrx16l!+!(( zB0mfe5Q2yjmIG-Qwk7WRSp9Z=*pkr4-F-w0l?#BFVl!$}LD51&0dB`Zv1D;R6j}EZ0da#Ut_N%y zR#}UDaH-OFvDhngu`CuPJfLjAMPqEW9_|*Lng>!TUJTqkrLKUBfy1h7Nr+U2!8M*K zS`sYG>y*z=W|dLK#-|td6M1VNM`J-j1MxuV!5a5HOU4Zt*T)Q15SdSt*a409*uvn2 zo^zCZS;gQp+H21b%qj!1iO8b>g)d}q9N|RnzF-i1rY36j9N?98AEiKPIE;4)WoLai z;9|z8>~zbi!I$mhkAvB=NEuZ15)F|fqb#uc$d0HRu&sQ|N*ffOb8^CF8c>PU1hbA& zK2&^mpQz2JZp9|cmZ+sH!iAJV2NS_4I9V}49@c)5VFJlngNbz?Hsl#Lu_UM??6ho7R1=NzvCX)~)9kQ9P5a{(5E;E$k)7GB7$6SR1c)rP8@?-y zgN21K@Vuh6fZGz~1<)hThJpeR5|dXFEsAB!ED$7=F$ai;rqJpP1xyM(sBS|#u(?l$ zbqe?QN+w9dw-0>FtL_r6l`vY=>&dWaFfoxOVFcPKZ%U)^k_~4EW^NKk(EoJ0FU0np{JXj zzP^xJ0IM?F$%EB!s;SAN?-5PU7nM5rsFKXUY6DIAlHa*<@y5p8{3VQXc_5}GGA_kf zD-w_!6m8MF7;Sc%Jv0!R=X_E%r_+bTteme1Ed_uk6LSy~C1D7{J^0#P$CbB6cUFPS zt~(78e8UIo2)n2gEzTuqs@;XS8E(oV23!My?x0I-?^D@xM-s8F^tQ5#FReGcD|ha1 zE;Cw*IeDfj5EN~}7tP;1r~zW0SuezRC;-MzW^bq9mV*ULDNFZzY)b&T@*a$vh_zKt zO+RFuQLK<3gJWXkHn+B$+^Ld-XDWt4)9Mxfmb!nMb;DC zSG6rdC@|zBb2N*@!Eii$NM){V@RT7X$vuab?#81$_1b%~b-#+~>D1_1QT9q%gP(`X zww+`HRyq?s?h6u>lRXH)EA>^5A#z;;y&8#b7&2Bw*c`dG_{TKgIM1t#BgxuJ2A&k) zgwpf`f^drY_n$n=P~;RmjiB3m;p|vG?SaC*U5n)7D4~q#LJd*_a*)N&HBS~yGU~Ma zwdPur6^zNiz60w-NU6P*nA_2zfM5g=5daXfSp^7nc_4Iw0ysc~q*TpEX;Kmqeqa$* zA;NBv%1?XDvPvrA%1YpePWeD=pExHJY@F|Ob$OMjZ3&&?p#8g^okwVbIy5+$$GPSK zoi~_dIBLyj8hEoO1?uXNbKP9VS);Uvky}u|7EYtCoc2;`Yk@5BTegiwe#;N1zAv<( z>Ch?MqKXW!erRCV(!eAM0SWciOqC_oJ~3Ge^zqG^hg^=x(e|`%=yaaJljNa|*`U84 zCoOHn=BwZ)dHuo+0t54bbz0%82!T@Q@+EJR(*Z+E)ZP`m-*{k%<(D$WWPMSN#Q?8o zcH12RIeWUybE=RbBLcNyIs;SDMzfD)=*-}imm{*_H!K8VzfkYH#SMJ%MyTSx*VCD0 zBps2#*Ex>wBVn8$Zr$^B6n5Y>Ah0(?1cwqeU@D=jP5JDqMWBNE+l<_yPD8avnZ~Di z=D3Lx{ZDmJs3e;subpHmOlGw~*dp^i$8F?`=J!d+0pLT7zfeF6oo2fskmlgVLxSu0 z{BAuiNW8t_V!Bn9$la?9?M?D`e+lxheoP0IEKjG*+OXo z(>$4c>=MwCks9@%F(Z+5V4sKtEIi$Q&RZmc-l7Wk*M$)Zjd*m08aOD7^_GNfxuKJZ zvH;H#sxj_dl_>itDuRN`4rW4jo@`gZU_VIpIH}6g@W^D7ou&5dqXzsS4NYWTU~9?* z#n5>|r^rux3>Y9G+eWRJc3E_U3VteHjzTM7-FfCyR9S`Gc2fQea?p$p;bLLYg1|7e zTVq$Kh?8{1m39imTk$<{6?A;DC5Y1QtG@`_lzrNS+?MNtF%LTN9i%B9P@SzYFlgv7 zA(yTIPdNlOl;%4zprHDP;g!GO8oCRgtY#ApnJk+~1tj1Of?#qb@JSLQ8Q1E9LnpPK` z>0giTEL^{PcY-=7>L+ARi+TCPvIecw!{QTUwkAR+WTNEWZ9<$>DCX9y@KdhMmGz0J zQNg~bB|SVb(ZPgkh`Hn(shlYqUkX{#lX0+bn%8n+uGDhZKA{vd^>${5sQMifUJHy~L;A0x9uwT%nfFLg- zCs*bjd|f@<&tVt4Nyra%2n2!0>l75sKe@HGrtogY1cmv7jhsUKU6r>Gy4Ri7gnA~P zCg9m)Q?q{-+PORBs0#Om#CYoywnc!U6uc+k)mGKedu$(S(!6kq32u8BoH+v4DOiBu znJjF_gd%$jnWNRD!Wcx02V<#q9|k=is8}zc2}vgNN33}o^y{gD8qG7!kx6t1yQrl~MH% zECgslizmeER!RchE)qD+o!sI;f&vwlt8kE!Xf~L2sZE(YIqmang&(T?;8TuH`Yg(h zS{f#iU42-?4l!liemsenNhn1)#O0io^PiF>bDS((W7na8(~izi!g7oWallv_qG%T# zUk`|mS*&`KkXY785*=K8RjJ#?6^s`o=+@hs2C2qIJW7lN0~+XQy|}bUX+(1 zKJdr8M@N&{z0uplM~>l%Gz{u>md~m~7XBkt;%pZ>F-Vu$GI8XOQzR=#dZwm;o$C(^ zH?BSp4;B!3_|>`MfNLbF9PAg-9S=7mBVOi?e>0m02!e>Pr!^3S>3t^J@9Th7xWw)k z`e+XU9kNR9V7J>d$R^gppJgr*+cd+?W>pdh`r^q-0?j_kCJ>=ULy|;ex_Qc+8xzW| zf^gXjc>6Wd2z4BQY^Pg5)@)0$7|khHt~_+o`yN_k0U74Sz(==&zfK|*rLr|n=Dtj@ zNMscn!*`Q>#Prc0xESGB zJ*Kp%W~JS4hpkC8g)ShZ+}Djet}z7`dBhT!%7!KhV4m?xGS?Yg0yz?5qvcwH@sb~| zpu&WNt>vbRXA~dY=0VgZ)GUPX_dJ?+@V2f6%oC&F1}|Z@is>+6tf9Wh`I!nG^n=E> zAdE=!&D5Bul|;rDfO+5}?8(9nH`B@~ibP2;%Po=}h`U5AuCSGJ#Q4n!IJKLc^TBO6 z4qIlDF9Y3LOp&32w1*P(jMfj$bEd9rCyecvF=Nb)g;<5nDIlnyC@iZ8!d^x(KTLJj zmrlNri`b3T@Ji~bw3~If0o}-x+9+|X=o%I zPN&?MZqTa8P$;IJB=uQ+_GM2DE%j-kPe^i7c!lMCE8x*1GxFzlPqueFMn}4M1@qiE zrK6kX-N@e!MTLm)&Io|Vk&XH3+&jn>M8P!KH-%L0h8a2M-#8P=Tyv%6nR(Z;h9OL& zg2`|t;s~0I8^i;U)T5H;>+$AIVn+O?FM3eQX#`?;P)6$_6f*F}BdD}3atz*|vI(0y zeyH_AflRi{$i_R7rQJ2-43bk*xO4fCZnJtHGDqB+6C5sp+00k{cERHm#b~cMl-3@bIJ1a7nwo8R91?lv^S_!I@Lp({ZtrlgT zAdyV|!RzMjF>&oX)Xv+)Up;{IOsLxhsVMU|j9}f<%xRDu8PLgsF~_H+alx774A9xF zC|w249_P%Xn1S|GxDq2ftgk9emEI+@eSHWMY~>;|+rp|-p%&^?q3+JP!A!)C>=hG~ zhw;9EP6TkNJou~4LWDIBX-8p@eyr`W+Pn| zMuH}JeubVc+6+My#wj3qe4=P1?f12?GK%2eBpDNeJPsF+$kgj#VURPgLP;X(5-w+0 zM(kE#b=@~HrPvw`c`%o>5yt4Shdl!6IgovcR67;vtY8_{8o6`uk&<@c=$#1o*&l~p zNNd?j!NJC6WavWqIONrM=Kck@0YQ6QzjGeJE!^JAhc){1w?*^}V{fU&Jeq0?u=s7` ze9^b_o1xpNgDu^c?%vC)%R7*9p^r|z3Z?;*N2Tl=#)E)Kc(3v4g6v-V&XbfvohPFe z8-n+%vB`&&<9+KrUQLH4MNbSOn$sg-gSF`3zbjKuYo2K7Ph&E0(yhWHbK6PC*y}F4 z_VJHB5f?9Xolav;=+>_=T8*4oEse}$R}OUI?uH;CoIPt$W0AnE4rZJ z=i(e5sW9p(g8MnL8e-R8GoIs5LG$c zaV_HDI3TNa3@A>>exV$0rMG;rcZ4N^?LLEx%THFyO?{D7ojf;Hhga2a^$~Ea&9@00 zvKFhNL!{E28ebAF@&vm)6shK}B%6Wn5e=SQhQ_on3{&6=;Je9q3y>9ah((p^FDh_H z0L318tLGpUe4}{k+-EUVj3o-v4TiCLb*A}#tc%SmX%miybQF!V9`9jd!@C$iy{rrqHQM;nRq-t3ED?lM>L*l<;cnfMOH#JWQ20z~&C)d9EI zq*8sM*de7AYkW~liGs5Hu<6zZi4`QfB7)kmt-!X7+9#mfc|gWrp5`*fJe?KDr>os4 z(U=-ekzro}VVp~{yCR>9R<4F86F}&RhqoOcVj)jS_fna1yJ|w{mo?@6cK3P6h`UDF zApC~D=C^bShUhxx@NH=zWs*Qu)rGlB#@uUD^(j`#kX_z_hS1_O9WN#3%B@0j!@r%c zKyZP`fVQ0FgRSOFdaQ6?aWfpafuVbR{9Od_z}B;{f}s{%^wxTxJ(B6gQy@Xgv;v^D z#Lx@#(GAQqe7(;gb4b=M7WShXSRO#;1J>Puj=+(61t0~GPIsx6EoC-chPy8^vAfts z3DPGQEj4|nxIBo7@~YWe2+F@?bv0HA90x(3)5uSNfusTLxi@>uAyJH37?0f`V4?%@ zIG(EoiVPG=A0?h;LwU6@iedM4Owt4^P)v-Ege~zRalHPuLtA5De4d$OmrpLl_Rn5i zZ%9X(2iGsnkw^1Kh_Jc{&+X2I1BWF#+oCK7$&p}ijJ%s)P0EId6#N9uyc}7WbA*lN znv-Vp5EHg&bZqrcguoSig^S*`sTn5%c4@Rep>by}?`?cfr` zJy~MM#V9+_U{8H3@wk{oGm3{GRgMPJv0;w2mdW%jF59*G#XwpDLpgHz6iDMQycR7z zy!JCUEs@zvs}r;?iDki0YByYJIHfySfMw7HCoMKb@?i*8bE4^$%RuTypdQ)gXMoBx zJ$r?c>{`i-i`@$Imv;mf1`1;CouSWkv}`>cr|Z}rZG6fPRE}Jgd_#ocOUs)V*1*gs z+t5#Z96l7CCU~>+6|FT?CrJ1z1xMFBy*Bjm2O?yR@fCWTGx@-wlfqEt5$?qr5gn`P zAQZ9{ok&yBouSwQt2+7*Z+^tCC@zG(acM_$jDR$<&30wLDvfrB(=N;Ih9j{W+vBJ5cQ|SRG@yz zK=x^o2adH_c~Xc^rB*c#&zNS9P+V9mXnBqD2E52A&b?TGHjP9tE3Bg%9%{4($^*Gb z3mG>AdVvkay0W}@cp+B={;-iqABo6-jXRV?W{z&6h#xW};9l=+ttVA5gKD|3!HYg} zuX-JsQ22=!6~d)f!8|XIhY>k}eQpxnZI=pm#n*UUMe#Zs3MEZu<~%-arX81B^4&MWGz z)9{`nje})pf&Ib_DvP%c8f$okhPuhkDh<5bS6D`5#<7L7q|FYI2U%pm=(uw7oTj*X$Rw&6PI}WRTOwHmh!lwNl0(8^cqG;gwU3^jLD+G|N$sjL ziGvLZ&l9`+G3^-yos#8yEH`@_ewhLpX$u3|%8*|Q91_I#&Vu#k20)^SP{ux1R}#Uw z3+Rtfzjfc)Cb(@d>&gNJibj6vRhk~wkxOh-t4I$4ba?rsbrRjTj5H6(6?+}oFat~k zSEh9k_gXbmN$O~8K^$`QySE+(G+$0Y68jM+jTkkLS_2F-W0jUK*3~YqKVp)Htrmnr z_83zfY;TUzW9JqQr#Xhe!LBSIcSQo)2%S>lx9o!h1>;%jYJ3^r8a;zp))K5{8Ipr~ zxMqpsjNZPSsCQA$Y5K!oNhxp#F9PVnKylX_=G@<~9}DFTpBf+tN0JQcuvYu#pEU?5 zEYafm{gar(TyzV*FJNhkaQBd7w3-0D65cO2R~g}N8k|N?tf>+N=dSz$wxoP-49zXc zO&Mxc-X}xKu^0^FXS^Q|Vy&+_hQ2~gtOE#9Inj_v=!?m0%3Pr#G{`h+gF1(?O(<%f z0976?R=*8;??`P9D7Q0TC=^`t2M=60%e}ud&{?g^5Z^=PEK_ZQg3?-dNgL>*ihhe2`RU2 zs-R~h5}$eF39o!Yji};bZAz2{G`)URAGCHE2L@%c0jcA8n!@1UmV}D=f|77=NW4nS z#FGNLj7uvUN`@74EO=jS%k(P< z!km|C0dZ)-2vwQURuEI$%(aTTuZ?&Y0UVSr&`Ae+j&@t6f`x;Lf-e<-YAk1^CME3v zTE*EEb>Xq*f?j-enVNEi?-};)?l0afd}-9UvCx^0Bge3TlMbw6oU{TKeGE?cWy#37 z(Bs+^iG{0Y(U}QRSm45wl{_)9v+Z9d7qO?rbG$%HXl|DH$+dcs3CVMn zHZ2gw5<-PnMm;6O;{yhBjP-9lf>({X3$>Ct1fsq2`0-k_ww0Lkh$|>Itx%sJmL2mI91fk4=RCfE>Pu9; z6+vVxAi)X_PN4PlVICl4LR>+j*u;^9zzakOFoSx4aN5HU!C;k_f|ORS)mV{cI3iG| zdu>P)JtWK^xO#WW_n?&b1sg%XBM7{5RWP31smiVH9CG_NlZ7G`TC%&xlDK`WrPath z?(@v?lF8o4IbF(uP(ZnZSiMV0u~)XGUo$ZUbR$$@aZ4clG8oR4ydmQp=X7+p80Tbf zjTuBEJMI;&3T`20S}uVfUDLWKiD9}WhC?^pffISrt$Jx5sT`9C6wk!*Q`- z-K6_jLV%3l#(hjSlYt>1UR5~D;hb`S)!(usTC7r@^jsxC;xc(j)|8Cw)b3KnXp7K9 z{$5(!o&x(pr@pDY^s9x(Df@g40v)v#&%6#rM&xyhhbmJKUp!VKlf1w>U^`NxR`* z;>85?Hwn8Ryv5s{VVa_15=lq(2tiLT552fx_;OM)CxD7CmWojkRuZRvwd>@9u0^5} zay3mTV08wFy0%xq@D@p1&fum~@`PxLae*MVbQ}*;CgNy5qFYBzPdMI2a8x6q52i6( zi-<%zL6%&*W$1W9tm(H+i#)O79LTNoS=)=;23O7Zm?}(I?3~=TA$vW0nBNd7$eA41 zY(JWa*}!6LHFzP`vrCzn_%Y7(1gK&)smzsz?A-$tYn`4^EwETTb1WsQ0=(3WJ>s^- zOyVNGpcgTK%)yId?yxQKYB-Mi-MqF21;N}NZBn7z0kFzcZ5y^71La>+=&N}X9SUYb zw&B*hv-9;Rm@u+Dm)ia5G=l)SI7EyQtTJb_ENyfP3%b^)eL?qF?ip~-P0f?G^b-@! z)zA$PB6-G+EQ(v=DQ|bN8<4qO8QjKQVs}@7)SRA5rAqM@0N7jGYOe$p#8shCY1)Lf zwLny6p%vK6)hCYWc2d#m95V3ZuN2xs2CHkf;+18~)RGHBAz1x31I1lXLjXP_83Ji# zu*$X-ERE(VE#?eU>b;3%3quyvD#d#2Qx*E-ObutjYWy--2}mqW!$N5Kx5!F-(@Nrv_ChsP6i`u>I5v8rn#)zNig$Q+!-i>#F8QqpB&6Z!LY~20Hv-H|qo=FHj}&1K zM||n$OIyVVm+w1tNt2b$C?UbmVcZQ#4U&0ht3+P2nC)>NLmy1GJZK89m8@v8fV!`2 z<5M1XJw`*KxkDCsm5g2lo4`sv8{TuAQ-_Cb7ivCYL|nDLkrc@W@ppDi_69D_f~%2% zhM0>R8=Ib_$(ckDtzcXwwMR%(J4Kx44LMobmv1{0VA`s#4YljANY`>%>D^(aO2|gw z4aNxenBWnyF$yiFW=aXHVRU278fgR$syY#eDgpY>okOj!5JvMtra^Z~ydwB$<7<}y z;O2I+MmYc*$LTxmbCD670o_b!qoOPm^t_8WYH5Yd_suYg7fyuq6EyOfw|c)bPhNcS3d5CbW5 zNb%N)gDwS48{xX@mji@bYeXwKBbK1#Mf@!>ZKHX7s`Qxk27-R!$@d&(DC|NNChe7E zb~|L&bJIS&*kUl!#sPx~uu|5qtvk5$%;F5lE*PuQ3Sh;_Pg>FC>Tn`M!sfP=p)u>v zIQGUYJES&vUO)W2qlPNmEynY9`!iSoaPQe73$E z6}Zj4miC?3)|xuv3)dZ&o*zeHa*Rj}c#Qzp%&e?HH!Uid@jTflfl~`14he#yq$23! zuBJB_6du%S?R3$~i3?}CqpccZ-karLb%<(-+tExSZD>kSfcS@lGtm*9j~@)cYDKQv z4G@hgaVYc^*yR>XZZn=#`#Tq5f1 zn}V#tNs#S^FxrM3C4<9!^iQPw!ICtq~+DcfwzxzRRoGa zPqOB4u?>7cs)8du0nFfBIleG~A0|201)#t^b33IZ-)E7ei_oYcJ3^J^J4%PjxT#VG^{=MBpnNtqCP~XeY!K!Du7LLoDNeaWg$l;O0`jdog@YW&&%)oGjMM|OL=NI7Osf8N{^Y@D3vw{h-Wld+Gga~yGw&B|y z7>b*=Y(s7yG6&W?9fzQfUvE{iJk?6?9Ew1X0g9E`$7$;~s;r6)Z!Q_OGZ@!9Sfe!d zH)FTNEQe5*X77E6BG9ejCVX09Kv}-Qkp)9eyE*}W0k9d{cO@+3G|+hD29U7|ws(2l z@ddk=2+Mewx5sJN_XJUEi3n`FO`IO`*f4X!95mu|PHeV84c3V}{SRK`^Dh9PERCZp z0S>TWwjvN5Emm@ON$Jl067TDArz)R&D)?9ySJqmqI0*aPnvUkpwyu!tC5u*Y!N#EBdOB`r^)Gu8s9w>MjEsytp-2lg+?!kj z!%6QGBg&hgr)RZ7?G}Loz{8=Dh=rA|V5f~nJgHnI*%R&_a|9ifQG)|iG<`#MdPRE3 zE%U(BQ9w%0U>exmAw@Q)6T^3m2p7vXM(Jq$7Lw^8;__iZveku5%GnmqoW$JdqtC#2 z0x`nG1!J^j)ybyzndQ9BOOAm;@VU zl|-x@k01>J>9IM=x6zzhgLz+3^-wFopkb0|=#1>kd4_ymxtQ*J4YAio&rxU9t*Y_~ zd@{sfKzy)ak+WlRz`&Rf?*P=q08VNZ!0KJ=t6Mx&1tww=P=)$@(%cZTh@r~%=GL_r zSP&>ng3J!r45o1hy+c1|380fDtCdyKTtS*nfcf=Ls_HWEeu1<`71!g%DaNuud6Q!z zR!clv94&_A1P@9anwM_`L!Fccia?a2Ud;2r3_J|J={J~@Up_KxjWVqD+R3U8gyxf| zP9gYxJ@8%G8KZG)=ysu>DuK$yEf(CJ8#zaMI9>|e*T8BVeyI5%a4lt3W+h*!GX3NjZf6nOJiv3*MoV!2r9xI>HAsLSaK+G*xKK z3eH!oVeCR*&<4Ez5u|Sfd1RgUD17;N{!Zm+HAQyi=5u4cF#o9KjT#Z=md3d#;m|QaHgIr$t4Nz5(8sM`*q`6fW zsb5Guys69rqE)MU_P*ZAi6ZS^ulXF5e3ZWbQDsix)e?G z_PnK->>nAIbBhF*ZSdJO`Z^Hi3)*2dR~TX z8&Wsp^YOahyWznh_P?HApQ@SXIk;aMIi0m-L7I4h+Bm#mICPw8>s@eI=Y7hEu%^VU zZ8Y>BYcP?)4X%C0b%fkh&_G{Tyx)AxhY-x@``Z!|e_^ixCA7!uUMQ zZs5;sAltLG)eNKz<)2+Y6{<9B^R#Ny$Eiy4v z6-FGegb^SlcL&vn-g3p42FS|G%E@0fokqsvZNTYStBES0`dRWI6S0$+tF|6>0cikRyxlUUb?rIWOd;*|G7>OSrWhY9ZPCND4(AtwBt2bzD)3(vIoXH< z1az?g*C&SPkD~JfZ3M*Aj|9b~mpT`@P?4ODT8psTM#IDbE!iuSu}n%7I}40-r5o84 zPA!Pt+6)rqiP=7sd^6I2M6ReeiEiJ8YE^_gr<1~f-!UN|O45;FKRzB3Tn%edz)lViSLz|GkmyPjp-TJ%;Hz4QT&XbZ5 z?X3CU&27>{E!K3g$4hzc@s^ zmPwm%I^vD(V$Moayv1wjg*LL>qiZ{Ct2^3;1#zGTgDcBs3p9xG)0zY+ub8mwtj_3; zjWiz@5itOUUJttyF6NL>GB-%-Qz2}+KwDV#15G0d3qZUN+$*W)a`gBK$tJatI_fM9 zy47{ZiCDnF#htd2!#gNvYW`MmaNUhEYS67t#!5=DggWL^_0I$0IKtV2=RL^5niU8x zHs*Z5!B8Xz%~I0)0Hk2GtK3dN(DxEF{Jy)XRh6=1Qr?rB*p6dTdywK4XlODQ^3hNT zn6+<;NVj(0HtO3!yA(&Kgy?+$Oukedt&&hQA~|B~!!bF;H%>RSz`*%g)SusF_{ z&wCfRGDwt`b(7M>?IN1wEA6-BAj8q5%LFhSluR-LLbxnQ1_<>uxDBYzZNPGv$|pTA zJes;#J@6f;2LyY%&5AQGtos7-=w9@#iTM*0A=k>tNN0Eh z?v%!!?>T_ykqHZ`DV~;63eV1Zp;qY(mJuV*xq8=)$QpKa#VD`kT3zrXQk}~uBUTlt zdNJNPXA2dp_(MY3fcSOSekw<|&xRz0L6YdJsX7t6%n+OdCM~%RpmeABidfzy!`#-* zYz#oQ~oeLn; zf-M&fE@d1WETVa2I-fN;@`k<3=I+gE77`b*qw0@BdfUR6m=zyx`k`z<729XU-Xt21 z0Jc6)5D*9m{hOy&uvRBj&dDc27K1FEJVwduo9n8jPcm#pH%r?HS>k$osjnt=%s50k zM5D)Hism@E#GUZRO{=8GvZ4epkuTBZ<>iaa4)v|C3<3;^fVmcMnj&|Edp!B!F_Obl z+2=vbNCigLm>2`YbH?1gykzLHL1&)5h~R$O@vjYEg!1HEjW>aWFj!wrPL%u&nIQE+ zdrF3nO;@q#5E&gHT|zrcb|Ix7i|Ujo4qdB*Xxc_z6~K5JxaJ2@VNMmOP~8!aqT-8s zZ;6Vb>(eMmZJ3^1FEBn;^~9*AdN~OTy9|}Y7+7Xs58WuD9Am@g@KQ2f!R`XEh`5jm z?3Ls|A!D$kX#_7Mk6cTbsSUGwd&)S`$2xh24ulbGs6#SLBnZ%roErMm%TZbx6%sV6 zlMwA9BA9ufZh)*l5s%4yYgkt&fEYLm89YAZsWGP5jBVlwr zJq<7$67JWkze?6`%^c5|%+Xd=6bW0E)UtI!B{NMDs*%s^b0(hcNz|XfuhnNXVXJFOMN6FY*Px_&43b^a8@J3 zm}D-IbVz$O+iL3oJ3z$0v{Dc?!G^xQ@@>zp3GviuV?kiYoX7i8m*SHS`s7Fm4%5A# zIS>jijVWCi_6GCjDTa;+0g4H6KlO?U{IO^8D{}Bvo0a}{zFl0 zCG%DH(Os8A;Tf*_lzD3A{-bqbt;oUX6<0#QaD}o85W7Z5T}xUN*HiQ6!DHL1Ne1=0 zY1Sp_Sjnf8H%pEev8eV>E8fFaIlD8&@J1JUdR!%}(T4C|-Hnot3L4i;4T%>EYuV9D zUMsn@N8ZLCsV;`mkQUql*;qHMX$=3kT> zZr95wL(z!wY}O6H-qI2rmR4BJ1`*aNADk8DLY+=1q!#>CXD+tPX{TZjE6C$Y1Py!2 z(`UP?xODipF7HcTdJw|3^1<#1*wm-b7KF@!xTC{;d%}IYgHlPs+8!oDKeB406yUY; zLRU`05o>~zCO?hMUCnuIwi}+l$P%Hp?da3XPy{{{*eQ2r0}piL5{YZaIjauZ-q&kW zF2T#(Y%rbS$`FU^mF1{FQ;?mDBt+D8B2m5W&qIkJ2Kb`EnjPHNWY95ir7fCQ z7NP`$YzYgKMrcyf7ZjxmY&tl;eXyAAGGhANGDO!37fw)t_&Oj7C1a^KCOq>5KD9-g z$m;O>2VYY#H4I*T?-Ri}zGoL<_h}e0&zQr*)>+bQJ-m5@0xcMwth7Rel%+7$mA0~C zcjU|KYfF|Qs2UcM39XN`b-0wI{B{oDJ6iToJe9gMp1wR|8v-E(y{(N$3dUrBwY#Ui z!RsLofX`kq=PV#l{o$RGsY%2d)_c@&)v@;FKd)D++~LL93Z`>+U8;gdo~MQi^5rYq z;p=lTVNHO?663Js0urlLBc+D%74>pBtri&(j#tiihY+h`h0bB>_j(?R`rkNPHdB$? z&MwIu+?^5?>QWhD@KT5w4A)Zci#%Q!8%G|H^H^^bw33-NL08-{W(no6g-h>6Eb@?f zpfM$wZ{)MSdziB@;^ymo@J-g;?2L~M-Y120XclW)$3$z!b*Bej!Bev0 zFttVn*mFy7WAwAnd)&*}OjfHhLL|ea+Tv5a&mAs{l?t1o7@z)WBewi|~h zc9wm&Z!zX*CFPR6q$#tRlAYM0&~MdkiYbI(SZ0^7K{Y1LEmz6DT{at@dUhe3V%;_I zs#505s48aPCF$n|dLcTkFr0%6!{i{4Mim(|#OKSF+7xw=aGR!i4}D_sq?+^@1~YE7 z_f7N-J?#ef3q3C65i30TY9Ua_lA^MEvqmS$yfSO1cJP6i(`_hoY*#9L^~7mdx>jKC zTFxG_Gq7f1Zjkkt(in+Z47c3Efy?d84Di(x;lO#DgqVA#ev!dUp9pFP8COVVmD|6R z&eEuDQK-z-U1u&~cTWcL0RaJ?7nTb-!0dhNYa-qmEGos|-&&X@l)K`R#DanajtwP? zVM-*md!}49Vp=cksjOwKgL^T4A3$S=3)}D$25*Wv3E}&OU%Wot|IwLVqA_CGO%YNM5(eQx($Nhvrp$3Xd@kQd$i! za9S^0V;$5pwa(i&^ z1z4lz!iWjN~1BTH36kCjT8T1T{j z2I2%AcFJX_@~~{0NF2nZ@=u-7njx0bg%RpEFeYIjfTZ`vlp~VWf*VDLffP8>R|Fzd z)XxP22uHUep|*0;%#L8}Vxj!oUOu0V9tDUa4JOtxgWM%ayw)Ns=B251lRIfSalV79 zc8M^a=~<;-Shc!P8tI|}om{jrwTzczi1ErfYdi#x5yGOZYFXn-$7g}D6?N_mc52XK zy5~b%DMI+@OC9VSJ--_;T>(5SGS`N&m~F)&m*8_Pz#~f|N)TC9COakV3UG*Y;mJ#Z z66szaMYv!Vpigkb$loGK5OC3(+r(8PTq7Haba{w$XLhVk6V~joh6mHq^c8(y=Lp2m8H6pfSub^ zBSa09#vSO#pcphxjzkzuh!o% zq7odN5q?frvD_8N9JHLa7*$J`&mjF+VFX$ZCt_8QV##&aC(~YAizAj|9D|j-HgqVb{nTf2%O>Cm z$zyMA+aw5GI|Q2LHhatl4z?Ax5Z^BLauA8zhM<5e4BuSfb)nKaTQU`h-eC(kJCbxQ z-N`@GNoZg+&oI1TlI(AGq>_Yb7Nu5#FLe+bE<2RrE#y#1I6>m-!k98QoCV&~X2h2X z@g@wrf#CSbsnOQzZuE~(zNpM)?kM(8PE~mvF4>;>?L|O5NE1+wAlBlX8i%OKxFkng zOt7{vHr!GW!Z%!s!p}-?#g>xq6v>X+RV&9M939onf@=GC5V!1gwE;@skide`iwi)d z`!z?W(3Vha936r(9LAdAfO9PCbpl2R(L>2ue2G9&m*z0^@S2${Hv?={q@sjl4l^3h zj-*Y_Gv!q}!WmJ}TlXaiIJ139Xf0o6CNk#DJrNqZ8Ksie;W+dbj?C&M*?My$5sk1N zD8z&tu&bR&q(Iq*LrVu=v~uq7(B;Ll*vq<3InOfeSJo%!v9!M=m@Rth`rI#3r(q5* z+29@URWmh>OK#-}YU9;s+YKmDvQRWTiSdGoNn1kDta}$s^<|t(JmJJ>mob96I?3|d zVo?3=?vpQMN{LPeg3(}>u!*cOeDG<9eYY;s1sB=5cC2E(X=zzPkrXrqh->bs#92+(v5I2mJ_4m zW#DD&vYi4%UZO=7JOg=~LGi<_%u?Vk`^vW4YcY>Et})0&frcPjta=4AS0F~}J!k1@ zlZ@GcNEK14iH%*8B|MjR3T+&lZH{^+U|Ozg*4~WkZnvG{%6Jx?v>5h8nt{T!^ksSp zCi4p?lyQCD)AmW)ykl@@t|V$h(J2Ya4_T*x_%-ti4FbWX~3nJ|;m8Pk#YB#M)iDMoL9JZ*lb;i2| ztD7LFGlSr_o)nUKn;R?aDWt=@i(tE0(;If=GNzot(5`t*Fzz2euG7iLgD@F_V4O@j zoV<4$x)33-a`mzYY$$US7-ilh(i!j?vc9|zKW5m@50nZi^#mgxr+QJO>C9a32_MCj zn075Bd~10#vwqNH8)@*cS8!qVi}#N`nzC(1SkrWyE>Bv}s(U07hP9I6R7?;wg0gA1 zaQ4O+G9>e)<574%%xUrmL@TwAR=WL$qicqSkSR2Z>`Ri;j%^c{YIY8Uq*uvcp3k(|}ivr0&7IbqrbD73A zA%%D(uL^ID*r>}nL{u}}9$a!sA1k#eQ9i6RvW80;|kxn@fPW;owCRckWc~ z#s((@+0~+I{x2lSmnlw5oiR;hpsCyu!VJ81v_8u+4jGntL2jvXrzvTu?-|58C*Mtk zV#!1v2?Mg_DK3n{=~@#A$?roiv&xgHFW$oW5a?iNUTg~w(c9R}87&>!LQa^>3RNj` ztx~NguMCnPQeshnOO0v`k(iZ&e17SC!afp#P`?8}7~#+q~2mJGo>Qg5XvVqykX5tcZGy=p#hY zvRF_iwK8*jEhwRY=&7Ti;=e---9eV3FgAQI_mTmHtL;8 z5fx=k^AN)a4BZbqpM`SE#ze`wHq6v~wS>|kj_ApzoO+yyR) z9hBe#n5OU!Ow-)H$GY>Hd*a74*2eIopK0DmTZwoDUf~BoN6V_{K%LUFBdJmC>1=m- z9_M!-Mel|6xCyQAt-1!M>$u`>{KAIY9IwHsbGO-DD>pvu;sLz5p~XQhiLYD zQ18K5rEq(yd{aWb9#0VJ6ng_8hRsYx9zFw$ENSv6{OwATz6pyoD|4jThRb-9gZj#^ z)vXIT#2EfQw@PU$8nHZ!MTx@;+I!%9PI?#@4MzfeDv%Qwq70p~B%suUPc~PFFjseZ z_1GZWiQ`0UX8P^Oxj@Ur#soFi&rl{+sScT3yJCz$+@3fY94)vtazi%k$mbi3$7sTp zYa@`U7tT?-T3Y5IO;)CTfO){lUrAhqI$ZE1MT+zJwd&6bbX8=!L?nuf19Pr>(4MPp zADHup3J&38wVaUmlKIqbN8MZU3`}I;92E~}jida}UDAzH#tae-p?3iiEMUb;lT>xd ziV>s|i*MysBqgI}Woi}D7J(*6WYUUAK6wyu#ohGSm$oie0lKLQ@@D25`*%zWFQNia zYKFu~kV`zxK~-t_6(Z0R7$b0~6}93W0`r|-`_>wPM&zrwCH<1?dF~u;xDorfJEm}Z zkkRL%q!npPttnl>Qqa^Xd3)2&XscjObUH7Ridl^c9s-KqrX4RL2%5NC=m?aNCAoDT z@7-=jEB3)bKH43O0pQg_C#iM>%9Cl3%9VD;@OeNtV=ZZy$DaFku$D>rMN+MYZ3DS; z%hi;yFo&_2+@u~&E!a?D0zrse$Fd^*aiR^{PXYvW$O<|~rQX}!q|xGF_q#D6vZ8%4 zv{*d6s01f{h$AH&o_0}mH;5XH%k2c|(^AdZn!INNcIUS!pn8Ys907;T9)D*j_irQ& z%R*5f5uxqO=~KBDAuKG9t6MMzedQcmE5=$j3bs;cp37__35Tk;&pK43qjxr$BiW`b zB?n9>v)4$`?oDNZAZswBrNk^>mv)1CHO|62i@p=O^CHunuiW*ZO92nD?zqSEa#Dm}mkh!q z+nXgljJy&gvMVns%2-chK8cgV?ock{e|R;j%yp^wZQXO;k~l$x4Hei5A=Gku4d(FQ zKJRd5qCAxB>0hdLU>4eW8sw(EsJPj!49M6ssh1wq z&JL>J%G_;%jvG}1F?$*56~_-Afv;X&frbZ&!tm%f6k?OatQ`YEkSmrh$^cFjrODgx6E+p|F-b@Y5M$glxwtI`x&LqSw0?j;a^1 zbPDgtvVDL7&xp)=>OCtMoKEo z(zUJEIf4;}zT5_y4mc*qs_@TjO}Q`Av9(7J#=Iew@d)nuvOA-YJ!iocB8>{7C79zO zfJV&?;Bd@{*DW8Yo&A`@&}%72phB7KQ}NeFBV#V9VqKqxYznH|G&{xSIKv?wdUw1! zoS~al!hw8>IqwDFtXIevWNaeTylbNXT(wglFkvU(44?=YFxE@x>+2Advz%sX$YQ}u zM)GsffR9ZugbUrQSE&5z7N`mW^;J5vML@*8n4W+vWIGlPa7v!!U2fD`@m2eTR5V3KmF>O2;X-zr z93&nNVDLkh0R*fG3=vXT&e9GjYp^UW(b#39Gmcv*=dA&yjWCLp!S;j80-)LizV>6X z5?3>DqJ~m%Vem4l7g&bJv6{l*X8;;*TT&fDmLc$k=BIJ-0}&S z^J~Ojd}S_oE}TR{Kx2{GZo{z$ATV3^Gw&t5Xd1vlTPK8mlq@~nc>J-q`y(e-IES&bS{WT}S%2X^Ca6wK0t0ZWBr_K%x;8LHrXAwzt8Ly$tE z{TpiU<&D5_s^2W08QQ)YD2b|vizbj%$*w|syO&u;A@JUq4O)W}aAK6wNgK(&kh!~1 zU$loVONT3xXV6pv^tUX~7%hOqTuT|{-SeqCmkSewj6+M%`o|!!2WG<&SS4KF3iPwI z!x2YGk;)W|a4YR0FRCx7730Q(UMdG>j6*q|`Th;Qc^M+}$IHpUA-0(OLE2mJ-PdRs zIq3775dpr?oN9m?5{FJPdj%b)ZuD8R&jbcs_f+DA)rN17kcQZhYV2T@Ylv~X=}ztP zvIyqfa72O&uS+C2s;UXun}YBpdN+kvb2u6{c6X1v!<+|TlP5W{wK2t136mukX%H0D zElDmj$y)7P3eJ$$e`XFu5?ewV>|xPzOZRGfLVR&K?lH@;YObVlUv?Yn!&`?l;gLSS zb~3jy!lyeAm(ZJ5j|3M2CDMa~Agv~&JA)I>Zce!PE=7*3wF(Mm2k7YbNzs*I7`lY+ zdTE}{QStS`0eXVpoT5Lf%KOkvo80jtiG%WWz1O$}bAapx<7`B+`w-zB<^iLrt*qBL|US?U^xWs1#_q%|&x&fgXE)OLf$>|A% zLPQSmIlgg;=77xaJmn9IyaT+|z}dWIix)ZDmf-pDJtAbArKg74EVx-o#6I4!W#u^f z@#i0~AWg~bwr+e|zU4fgGqD1_t%iAdYK7{-m1ai~3S{8w27DTsIOkTZBw)Usgx1?glUk-6=Ajn@H(T%c!jYO*cg>2RhRSV&*Jip^^4F zsS(NwR9^f>g^-n#=CcyPQ`zT^1S4WWD1?hp;@R3L!ePMVdyr*Za)Mx_AU?^e>e}Mb z6})D}cyK~N@po}yd78UKA<)Yu$ZWHnk-4)a^3vT`6~+V&SZ+(JFcKW?UgkGhNqgkW zJDagYGS;MdBSf8Rrl002J~*kj=YVwawMRtq)wax8ayzq)w@ZS8Byo~q&DS{- zHbIzT;f~Yl?^=p&sN~U(2AB|oz!It-rf@rr2}C`FOsbpB zvFM?I5>^YgZhOdjT$W6#?h}<1I`V?9O9~x7wH@|7In=#`W$DKOxr^|)btQ9@5DiuJ zv3QpPA)Oy>GKymeMulY#LPoQ2tb;aw@W#c=%CzV48DcYo^r!n47pO6up)M{UL zi0SkmRl(`~6@YYBY~OI1wG#VT0mzp8bSp5oHN;B`0wCwXL2Del+T@{X(c+@Kcdc(w zI1jOt+KtW)Fxc7}fg^FEjkyic^l))qCV1c39yxn@6Yo>R2qS^yOhlh4I~NyYDtdVD z&6%K_E4EU;*$!&!_Esha+U7K!pQR8k&_o%|TFJm^8PL7dFK8N$aZ0O)cXPZGlbJBP zED`7%25750J#E`0F1j6;H`#6=lN{(z2!iB_<8W`fajf`6!``LgJJoru*itI_ASQ-+ z0-qZ9Ua$qxN7!o~3G~ep+Y+j*Bwqw&w)biZg+V^9BMZ_Ofp@!#)conzmG<}$>`BV$ zBIBaj-(1gK!*=}R5|wtm42E>II1id{$T_BP!Owff!iiy!l_4b^g>X>e6H0gW zo1DUL*Q14IQYy6aumGU2Ap46uQ{hKvwR90S;&LA0V@=l~24}TRv3f|tqV>)M!?o@K zC}3$(YOd}Rx{ivx-oE7vW5o*o9GD(Q>s?ODBxNPz85oRxrx~WGEEV-Q=Yh>*)FaF- zy@0Ok3VwS%3|evmSF9AmDQ#+``_P6nAObcS(|TKOHrD!wm|2zzu1c}VF+z(03&C9t zxho(qJz)k?ntE2Gt!r)zb#iJ4_iD}|v+8`MLMUAz*SsACFM95@c%<8AAmWz7K{F|r zh)C5er^xT;8{7=SkbA>QI-e%Iccra8;Hk~jY8E*T<_HBw=76oH*nKh9} z4zuT-fQOBLYmjre!5~580G=%_k+GS%@0J8Di#vo5(mSm5^2s1;A*?L(z*8L6&N?_VjyW3;S5uF+F-@33f)Biuu+XkBAgW%3I4 zN5xyNP!MDSfus}*09%2v7g9P%juW=qn2npbu|rHW7mFdx9R4zzGs3R2@n;htW|##C zY7XqDB6Wd00#8vh&Q=zbM-q;~IY!@8ItJ`@WDyZ1%{6ZC5# z0LXAvNm;h!``gPEQ4LFw4`!M?LE&)e4usa=65cr=Z(M4CkRPF0sx%wd+`u(KNTh*b zqco6wC%c5*sP1S$E43wIpq=KhrJ~xHCLsVt_XTHSK%qJK;nc~<&Vk}|L^-EZW`%6K z6gM@|P@Sz`(ETZ+WY1O@uvE8Z1SO;tCV_&}a#iy!re)x%EY~E>oB)jzbu$Aqno3$W zPVq~dU9A|_3!??NK(}?*Cg|u^nj%_qOi}a!wG_9C85nY$T0uB3J_UgH37Iga7u~ez zCE^rjn9lYeOqf@Wo3gP(LOvBokg#5crZ3_i#Y6fVR6&K3IB2D-u4O_V+ zLLCmVmeND)mUw6*4%d%Ez3km@(V!@yL7?pnLao$?5-EYZMee*-p|r#=1?F7~;s;I- zSeudg4viRCOj(OLD@fLsa;7eg&k^3Mdc}}DQLwizEu1WL!>I~;!&b)wthF4g4kx~3 z-EtayAzV{fC@EQSeT$6vX2`O)r_o-}?!d$}P4W)XPLH^6>9u985a_%|TWv|)MO^9B z$_dbDq8a(B{R2oD_1wLyTV2X$dhMaX@6sIVAt0m#Cwcfoz&GNwU;j^?=d&IV*&46Yn zqs#I;+JPt*b1OK8xbW89yqhG+$vA;aYHI}(AXRRaD!$dZQOubhIaZHZM2Sg~$TeJf zI#QLC`87}HFqUdm)p+fU&tH^GiUW9;jqxDPoUfrgF<4G2J>u2b;OaL5-10_|y>6Kc z%3)F=5{HFvbT%kWWz~R#gR|8t{VBoS=|ES-vu$E<4TT0)Nzbv!PcmY&&uY+DM^dEP zdBvf2z{R1R?{$diGO9B^cKW*q-D<4|G#DE=Zh%(O$H-(ieuH`6o%t45A2=w-!-rFL zSim&YtyE&9nJ{Sdh8JcTS?Ls?C1m?ibioBJvc&+tR%~`UA<*|%s++R8bn$d5VS}t# zQjtV;&d8kRxk*x_OIciE=CWQ_BBE77*xkvot! zK?<05ag;1GP3@HJX50hE;3TbHaOvbbWrS)yYN*l9S~5eP@wng-zUbVMN+oijkm}uI z?M@DnOmsjFUfD06@rs)17RXoHuGD{nJT_;GUu$M)hOEfX@#LHtJU$-D4uuvK?NKs`Qv5>Zs=4 zAW;g>s@Z|;vqe2^n-o?hdgp7wg*%FfLGtyDw}=QR(arb|1e|S2tUP|9NpT|c3~VqB zCS;G3&~8#%WnqtqCwqo;M(lqgr-ejRSQ9*(2p>*l@G`d0w?4m=$}e#=apT9a(j z^5J4Rk?Ed~W-KuQ9zY`TiwFjy2!W2c*S1yn#gR`KpK{|y8!vKKQ5NB1O-G<>+KT68e1zu@cfRV7%M|SB~;1?5r%OVX1mCjf%Tn@0MoV! zIy}*^SC_8D2TB;ea=MqAJ(J&#$xB%Gm`yJ+emCLt?l^^c@Kt?&c^4%z94)}Cj}N0u zA#viYZxfRzstGArED zW0u6~N#bUl+LRzR2k>XKK?qPvQ9-HWTMk1yuGWec#b8BLEki{NVH~eHt+-Mp(9_-C zho7I;1i`YR67FHS`l8%2_$^L#)hZy#r*Ct%EvlEK0&KY_SvJ@xt}Ag-#C6|6;HGey_#ukK2q}XW2fC4IrYqO2f}kQI&Lszs{C_7$g3FH3M;0-* z8S|EPNXNtOJ26Z>H%(j+;sOo=HV0J9^Q7WZcdH#TSHolDnobB@XX1Di3}@g)$HioZ zmQiRa5Dy4>Dc!`%%O^ZMQ>R_nNzM(G;CJ7U%+(%LV@I;;@Q~VTR6RjM=5(0B>&;i{ zzE?tFm`vm1WJ$HsW8x(QUh+Lm_Lk*>^$!ad7)`_ieA`{IeL2^!HpcXjSvaH4JQiU+ zJipt;l_eyMamL9>neQ?ztNN5gcCm(GJ)^wU#sHySjw&ynS^>laOntnX>q_Td4M&)f z=w)QG4yD6W8@XCK@*OnR*cqhyjD?_vxU_!Hho{*u*dYcfB32~64oB``>~{u!KM%W4 zB&^9aQP!ZQO=cAcg5@)QD~!x&qqiCmA2YU~*qd&3lODbgUaAyVE~CQ^(&GZf zjRN!q*lUawr=f><<2rfy6ptKtq}>jQ7)h)^D07L#;d#7(o4c{@fKrEtkmM&!NftjJ zEoqFch=#^t)Am#okCtbUbb4$n(XRLno#7Tu;je19PJPD?PltZN@_0v^JyoxQpYe_(_ zi8xF#vqYNm`J9TUiI^X(!w%HPIGNg4CIZMo;hWfI>Bxv)yPfH$7bpg~fd+y>ZQ9#M z4pH1J>dV5q&ml>IOVwV9XF5MnaP&QJX^uOWTDTqv8p8OmV#2sRc>p}bwDh$$nE70B zrGPf%D0#CkNLLUPe5{mg1?$8&wt)mxfUyu>!}B!5h9#g6vXeDxMPtPmtdaZjVs#K-cOlE;gW7{!r+psYXeo&&}fyu~n7-97A-geG;JQh=dteX!kwo850_ zWAm(Yce9s3s}PcKA_m(53(G|3)7*4swwgNXeP%$Zm|1TxUqe(QRX}1tu51r#{gnO= z#1>oz4D#vQ4lsoP=Dg7t#&ezK2`-AziRggS8XvP6qk9Bt9gPgfkfJE=4X74va*Qdc zQqwNYk8{w(JDED>^fbyCX&6UnPb&!Pc9Icf22Q;;)>65$fkm2~I#tcry~5*!1A1f9 zxy>)(L6#bc=qA@m!)DWWLif^|oyE)6HJZY~t(DT_59NYf74U{2gxgn@Y8wTuuDcq& zfH~X(=R>H(V4X2}>K=FAd+@5I9uJmteG{WSjZLjg99II7u4b4Dp)Sx|8D5bXKb4w% z29Je?{R?p@oEEqdp=NopEv*xoG2nGqea@@9ivB< zajuw5VaVCLRmm@g#Q;pYVfDuWCPHBVd}nsKi-h{YkmF{_=7LU)NxbF>TEy^;N$PMV zlGEtmoiwB#9jvHc2C43GDgFuYD6)c!A6U4Bkj{oU`F#AR49?sht+cl#D+cvaih^%2 zX?8A9!1nrG`QS`SuOFdh?}pObwzt?IZU;0W*9&8YEsbC$MF@#mVZvmnAbDza4Lp3f za8FPP=A2x_wSat8C0!|L_hO}0oZ?hLkab%4vqNj!QD=~loTQ=&cr0#7sH6i!wM!vF z@Y=>bWq6Iz=rFY6>V{3>1|qm7S(f|3!2zY8Z8)2y_ap4flS5*AGN^Yd-h4iU+ZD-_ z+puyH;kLXD#qhhh=`B}yMX{5J;1a@o#F(ukDwdpm!&gn>mB5Oz!Z)FWYkF?P=yZ^z z-eNMF-8!riCm%4Kb%uuc_1U(_NxM#<>j)G$i2=fS-&@fsZsXf?s7q;Uu`o>=l8PXdsCA#^&47tiZz(=ybFvwuPj@B}$Bs!j!kg;1RGL;<)PX zL(!Q8h?r)I-Hx{hPT}NY`K44aH=n5B*A|i$2Y3~J$Mk6|wbi=G= z*k`-PL|}3^BF{3_4p(CM-LVK$EZl2YSi)3bDqc*0;;rfSGRx!Q_IEJ$^7Jng{7P*Y zSl$t+^gJdacUNeg<5Ge#pSbZ@Ms75zQ&U3w$;(ZFB)e3Qsa^*SQ|Ya#7N}DQa&ol> z>{55)g{<-LNObiLO5N&^VT+5*vcj331S_DUKT*)hwhOVq?1=h`Zi3&49ndq<**@|I zDT+R-uGw#Bzqj1bd?6w-q>Q$Ep#Ah8VB3NlM72gLE3t-cLlOnF;{Y za|*m=sEl-G77KVVlj=NE2(*37>ncF2tFpAea2gL^yK|Lzq$Cm(hf&`L?xOl4>ZLlZ zR5{mh=gu6ZLR&HGVmi@uSP?LfK!Q2)<~31qkE7M)EKaad5XQq}Hh-0~)p-#$ja80l z<`~W_gfSu|j#MGmc0%Svh(o)TL5P)e?KuwNn^p#x85gwUx{K=XE|$QVpn@v|gB&~= zU35~VIX?5oTqsrzp_O|x(*>E z>(+7XHKYt$2SU|Nvom8X(&fFy1Nn@7AmGd_qF=oaX<+rXt5fQ36>O;$Me*)Q?lH6n%!^3lFX3GcS-g(>0L&0tG90tNJ zP&*-hXL9uRhAIj2PNr@;U<(^eUO`$B^VqwbETCX9vKAbslOD&VvI%tz*qt6hemW8m z@E8lfR6giNUSvjWAg))+Yjewr z1z;dnKosEOhD>rfY|jYCXpr%cuD2Rc?8@#+vHs6xpzOIg*HHWd%I}oyN_|SeG$6p* zay_>17xj7TRM|UWlo5^ka28~#BQ~o;JTKKgd260$uxZ5?QvnVrEWZI&{i6gu@VJ)D zsnX7Nh(DCFE$pZcgGhYia2YR&XemIu=6FTynhpp?kb_SY0_NJ)md+ofZeS6!wP4q) zU=|u@yh*o!y@?*Jq_~IS!7ajPTd$zWutZdeTb(2&7z{@y^^W^>N*kRv{255(+oIy0 zjtA{>9Vn~G@tBU0AWk^0-EbcQIb(~p^bEcRVd~aAf!w8dr4AX9&0yTBc3a4IjxjYu z0q$EN%!!(uJYhP}px`t-9}0((!ZZL0wAZ$zF@A_}S5^0Yt4Hx1y>qQmaTrU2%p@7^ zax>EeUuVqnRlcCxtVjcKUV@7xj&?N=*&!gJJB7`|!Eq0z!Uh+$e}1u!1VIwUa5 z2#Sg>g1Bcs_aheC+{Sch)XApKa^oLjU|3}P&|!MUY9SPOb%4Slr+P4@?>ARRXP-Ln zDCzD`wB)w~AJCm#oF~%~I4Qi#=tT*?5MOS-@n6ik7A*rmIP}wvvKkRe1PB>;i&XY! zhly|$b|&i}lDxLhu{jb@1&rcN#I#xi-(n6T<50eo#f+vPWk{XCfv!I(sA*5Gbs?D6 z1jL*?jm{K}B2G75?y9cu8cC>bP`0b1u8grNKW8<7<^`9an`gsfMaq$@=}rxo$dUy; z*G;}56$)Qna6XxbJQO~tFxv=HkT`)DsIWSXyYx;iEm=yjyYkC7IWedqK-ufVIGag> z>>H9Gbf>+vT|O3AHWq=AfLTZnGaB%Cc-uj_;|Tnk7#cht2nlM{bkfkpux~4mVE4lGX=*KiH;#Ed!`&?s4N6`$O21JLtg(zHn#Bd?0KjVcr8F``^{s8S_h zp#;S)fkt2sw({f;CAGg}@aQ%apQCaYFhxwwisJc>y+Hz?@MztA)KV#|1?{+@JcWKV zR%Ran^cH~?Lx}~;X)iRIEpsOw*=Vffc-dq+_QS!?!egV}K#3j}LzYQ}0S392tpcl{ z!E;;Nn50u}74bv{^}BQ#He1JPSr(nByM+7b*=S9&@Hkts=LQbADtte3)~wRL?QV2W zMUpwhyZJO*c&K11;%@?Z-EI3YE$5hI#P(2z+n|+G1rJ?~WV_XCV%tNH(8_ytmniDO z&YFWd-Z%-CS6mY}GSX)b){U_PL7h}=G$2{*a8(~sFzL<{sZy!tcIFZRAS0bNQ&8IbZr&=P^fBz(Bt26hjaoRGa6YoZ*xEiLC1XaZ7eFRU_d9EY zu2KTjl(NSLbr|CYfm`26f~mhB=;&T8)l%?1gR48pL$PL+e08=9V%r6mKjW=Arvh;fIyoNj6` zYJf4Fr!#cn80adI4isma=0Qnl?v&`e(il?bx|FF(9yI1VJ_L%pKZu7fpt;EwyHTuU4n)S(z`UPDwY(Q7 zKOC6o+v}%2o8Gh??v6{Blmp^W0kf!Onwf;4LVBH0VPZ_%8DQ6X@46hTsKz=_`=XBL zvlj0@yFz72<7sWS?j~#4_1(4cpGaG4_MfY$E&MeC!N|&HnKnf<5pac4^J9$ag26Yi ziU&;{RKXTgHgXQUf%r&A?5@J2JDOnkO0soJWzaczK>YV;cNJLTO`c^oa~?=@PT1Ls zdjPk?7MvA{&9LD(fJsYU_NH!}@uWKR_2$&cm_zoOW|DgV(54k3$wDJO8b6y1`YjYh zoRlXhiqFSdePJR=lc8)u!r3vYqaT&E=Y&B|d)T_Z(?0mgblQ&^VsdA$qK^`UuBDr& zkGI#4ZGUihr3n1=b_6$iuNFHDnFdrZlVKDyVM1MpLqUu}2y0(fOuedCi*Wi4`*kZD z@Of&ryeU#?aR^clz$(C#fso>ZdL^JCvMN}|3%olNGu$DeS~b+&vBY#$^#Xk@6Ta|o znDNjYKVw z?C~?c9tP_{h~Az=j@6?M8ENw&hM#P_GH!KA#^f={wMODRUM+Owleh>xfNOxJdKNrw z_gv6Vu@>6ONEtEZW#5wVv?x70TY~rl)fZG3ttU}H2FXHFkH(pCWYi#+4;gp7V2!yS zHd^2nt}fk-a@}Ofe77~I#Ie!01diNEApQ?~CO5Q#ZTh&_2@0nv=h;9)D&mVOLaIB9 zRN|hU+<4>T-@?r<{I40@v(89+(R8EhI)hOUYfpMp!OVo+Q>TFL&_GN~gqCRE7h*6w zv_(i3?bFs)hWSFy1hP?nb^2Fv89PpPQ+7a(3KiUD#|5~bP%yWB-WtZM$Iy&k;5yf0 zN=0IXmX4*d0ZD0ERa~Zu62{mXIdBz{K*xLx8)@1m7~AV1PeoMo6k;B?RV{Ik4c;zY zD`6vfZrBjWe8o1#g)tEo(VA)uaY#@?2t#PzTcP_%8E|{6%teXgQ4l5EQ$rhnF*epj zmLYM2z75{-^MQ+m*j`c^rxmVG(YDTX2X1zQ|Y+fe0I(Ten^)9eMsnCj+kc|OZ>D#Yb3>W2+gk~zbR5j>!5nMV`XMhBwl zFQ}f)Jx;J} zH01&AAZR6AS1K@oAvr+MVm^ZMC5F_+i>J5?B_? z3^SpE>yE9H#CE0+u0UQH_O&iy)iEWEHfJmVK|sF0ERZ-+S* zOhs7qge7n`=RpKV<{E*dGaP8_=|qE@T@3Vk{gio=(Rjh9Kp}7d!)PE^_VPzM!U(Bb zt33)IfxUqTjZvfovSVbEBLECE9MnoI9*)Z!(5anv7X&XRPPMTU$FWjZ-J1gB(0dte zPUlXfA`-h)bM8gpU%IT*nqJTi81p<%!yKF?d1dAaQ*6(0mVG!VtkrYJsJSX2wuR4Z*2@GeqMB@v87hjE-So9fv((^967u)8?1Dd>WI|0v; zM`Ceds-i5#s6cs?%PIjCh?o(X#L2XTnuV`>uCV+FH31|Na9SaL5(?xg)M0>*A1Tpl zdJn2+&2=uF<~i43Zy(b~&+e2Zl5SEoykhY2KalMBEh(A9NYYrD>TP@pVDZax`8jSD z$yC>8)^4?^c<{{NjB!+rjxYlZ$9;KX3)P0cX-<%1c8y7}VPFOc zB2r6n2M`$>3<+D74BDZ##>%W+Nj3;LR&n!s4?TTSP&=JGjs@%$;#In8#z=T;ku{gy zJugdT?+1}2(TwJ1CYQCOL`~7c*_zd}4^%$>a7bOhgH zWw!E8(&V>-B&!kXMqgMY4f$KwhCQpi!YIpT1j>XydUrRX(R|+HRvsIj(!5OO9!%wG zhI$xu4M`FMA;dk845wIV$mfqUTT;mK?)=Y4$J>!^>DyuJxMnr0_o|?)h*kPWRZpuq zRcIC>z@FRFM)b8kR6MwJz)_zUt`Y_&D(=v$3=4HRE`b@<@xn0aS zFJrBauy|5cg35=k=E84-HJTTL$_EWWnm2w95uS31Dp?h?@UorVy9p!|5eA+n!uly= z!#yk5SeD_L&J5>1Q5#|spVLp`HCGO{T%{0ci;a|I*rKp);5+jrZ+yye+0{0QOCVI_ zfUfD!Wkfa|zDD-o0;F?^kxN3hL-0+|L)r_->`D6k&AFP^%|+0lh$QUcjr1@Fyi^3y96v*2v4mR?JJTXjs87X?4;*xGStRyHrtMg=gfG&d< z-O9+w)OBjeHnG*F=;+epUvy^RHba>y82BGkd$F)YZud?JjdxX;}7#G1r z3SLkYKPC`m+W@>aXE^Z4R+nE6DkwfrszJD*&;$n%hXk9kB30lpTs$X@o$7#VqCu%u%Ufg0F*)5R!an zs5V$^K9x`?R1lCG=OyoM=VYop+wJS?1xX7`)#0ENMcmnJ6q_>EqtXg_-FlcJ?hp6DbKtDDZ|bSq?a!bGK_$i2%4K&WjdTHgun)cJNeNYac& zI}DBt4#Rn3R7`XhLK5QIOZS{7vA%7W9^?!$ZFj(-obAQ4*EnP0z~RG>hm@!T7l3eyo8cD5M2-j> zH@ssK^-LLKTjv2Sd^Fsv@y6!8LX`ybdRcwi#ZHQ=;63pMkPOBJWJPY@i5!%1c_cm3 z0$~#fheG2adJiMhCz6c-K%%{eGfb0w*OEncl%ne4+f~40WD2gn2VU%dH*U)Obg?P( zy963(vVzbqZuRAbXFYaKpk{G52g{b1VLbAOh!pW`=}}GM6q}a0yK^{+HwZL*3@%TZ zNu^fDj8zx%T)L~ig&^?;;;7G~RuBX<%sAoD7`jPIUKU}DePgwrTfAXd$Cd&TSW*KZ zHcm;_ECsdA&1u}?w+POH=2&gxo36V)=(Y&LxqISfGqA9W5aVZrqX$HR%ybd*^g`!Fm3kB{DKtoVw#%15X)DH_bSRG+K1s)1%UkC52W-T* zAfLASPA!GT7Kd}oYTowrnOm;Eg1e9%1}%1gY5ub|4pH|iSQccnH5Sn*+&Vi2_s;bB z&2oG|QnMLvyaOIBwzgD<&c6>Yfe^+6ds02pKG8!gIT*}6 zc}Dq{H3fT55bmkUVHAcRPJ;vrVuAMHB4h{_Da*4HJk`pI`QR6m!7+p`knb)rP}8cB ze01_FZ@o|uoGAX$gsu;b%dWjJodQJshu*i;?^}0m`uYb6s{w*T9Oe6{@n}u0*eI)T z;biwUtF30AgSW}01a?ciHjd%v)*1vrtz;;O4e>VzZPkDq3n;7FZP^YVM?J|G8Vwy= z-o-&C-NX%ui5i5K%dNEmP_^{A1Bvk=rfFvT*eM_`89G3Bc-Pfs#Xm@2OLK}L2J zAlyOEF<#{(AZ@+OJ;AYE)|E;`BCIRk^V~>j77+{xKnvF#^P1gAr#3K5ydmyd+K%($ zp&oh;b=jJ+giy6rm{ta`N<5-0;fF3}bh8nC3}TUCDhi|24QFT))bd97vt_d*)tUY1|fw6`R9y0~p!Y%t~~f*TDTo#}A!^ucCa`Na_JwRHAm&Q<6x@UwL| z%8LOe5R~^lTx#5jFqv{TvD9D`2(0CDsQp>>w6AFcVo)=Abg&ZEqYeiwS_M(OadMp* zgdD6PQ==*hAb>U+-P95+msMaY))S^8rr^DW=Sf=vE5+#%IRE1awFA4z2 za{(G2RSUSv@g0j3qP#G!D3PMc=rG~$6{>)oK4%|8KZN+9J&-F)1{+yEFf&+~H-T}G zPP2@YfyndPlMW*18%tzdfmTc4Xst?>%|a`g!U*|o8Tgn+-aIrrA%?ahcbt78tQnYk z3u-lFMc&8f&o%jIQ^I;taT1(}_@OS&qkWS(#m{EjPWRxrO13K;5*z_Xe5~F340{g2 zIJD@viQU+kIBOLRmC%nVdG9!jbA^wy2M!~oi#K*t1Fb$+4QeI}?k3%~WFsss%7-=_ z>hG{^p*KkNNd8Y3A_O3f5>e`1PPS(pg{Cd!FpzdAF5#KmI*%Q<-BBdyj=4n7>U(ua zJ1BblBnCvleIg1kKucmi3dz1$ByDiV6pC1aW4(PWOUT-4M>^J`u_GmC#8-{S*sNYd zY#d@^N6VAk9Lo-vC_~T_Es3h<1=Ry*gTAz7NB7BGoc|-S7Wou$TGR{5WU#3A9I1i| zBs`qCq+I(s&-p0K^CY|8{1YB1ZAw?S<^O)|C|eU$JgIbc}OW?k`+42*JLAgy?6ZbhJ? z*$->UR8_aM+SbSI?gWQ!lmP~g>CPUgGbW+JJJ=1`AEqiAn1&+rxH{tz)7-ZzQ1>K8(VEiTRU)+rU)4DOe|Qq*RJzf z6V_2pMb5x&Vp0}-z2xrQ<)P=0=8h#zNZY%qjXRdLEy<1f@iQ#?921zBqOXy+>YlKj zKUmb%anHD_s5n7LV>uIy^djemi6XsWDRnb9y7H1Hc^OL9f|ETxscQoSJ?-uvoGL-p zbIBsr5-_b1mxW*Do|z!EmMmx~z}T}aW%9PAf+M`_luiQH+$+&{`1Uv6?u(P9D;^&`UE z#Jvjky3mzpiHNxXG2T7U(l5*Og`+2Z^fq559@r`!%meYiRC9}L{<$<@#e1Q3hkvd$ z3cN&jiPB$So4`g<6e8JfP@zfR4alg(46~~Xj26;a-Kg(R=R+Zur-n)%Kk(I}!`7$@O_o&o2RX$gr$Z{bhSj z1Vz8m67|Zv2EAn6&o7KL@wL7L|;1%YMY(R17#`QUGE7*1t!V5 z)1PSA=CKu1(9&6<4De~(cg7a35pP?G1b6Ey6!M|b-xxtJc?Xlg-j#u}l^H6##{1kT zcCCd-Jz&JiG<9qj8Bdbb^)h$ot;VFxfRJ;?+w&QA_96L0Y;jnIz=`54O}hPw4F`i@ zw;YeMGYU0iOIHo-U4=1s2qiuqmO*)kJLGr*_aZZD02ESc^n07p%FO(B(jIO-Z&u7- zuW0lzMwV4>4b2JC>m@=9`xxuBdnv^8E&_X<#APt9K$Y<~mhwG)5cZ%rW7Oz}N&^@+ zzQ2RL%ekN@?(@ftm84kEBN>s)W1jfH__I*Ec|{=MqTzsaT?D38)M9g%=_LAlH%%kb z@0o)iApOEL#{&Mam>UFiPOZ@OF;drH%N|#mV>)UlEgGtN*f-^&5FRNT3bIv6BqN=z z6DU&)-jkJNNNiOP7~=+#Gj?^byzKhm9c@_lt+>0DL;5;j#DHjMRhsL6CEg3DSTT>g=Z6>!(87j#?!>rbQP%`+PeK5`SGeJ zsCb)pf)O`C+V6_O<@Q2a#=uS^c=lr7oKR@@Fqg4EXS9}UNggbEJyHEaB|>`gs)w{VzHQ2BsObZ+3jt{ z^j|Nw2oy?|Lc)Vdv;{aWOX%8|-qX@`vdr&Vc!B}GVNJR(H&v@R>L53WFfm7jlo~T< za=lT$8;cvXk-FLxI6En$Jk&XmN$ORS@RUXZfg z-WV!Empqk?(ugjr>uA2}5eC^LXQpnq#Ume15^G+W#i7vT_Bh)@P8u1($a^69iX_Wu z=zVPR&IypBIn7D3JOPL%EW_7AD*JJqo}GBhF8z=NcPkfzU9Kiw%4Ar`t7O1fz#ZmR z#y-hQP(O&q(7?BK?_>mx%-n%1(wVy+-vU@o^|p4U_8WZ_o023lH;mR za0`0VLK1#Y4IU9Pi>u>OMB|D$yK7FPbpp9{IOul5Z7I3FqwI=DSynMwj(2y#uLVKL zQMK_WHuj*x#CUUV;j(2%oVRC7xt)*fEYs;3%;6dP&>fkWF)LBQI3e#jU0KoUx|97= zQ)AWX&xd5yxj~7@6^nT+yt%HyYDx`}8*8Oi7CH!55Dl=qShJWiHSD{#!CS|?p?N`i zawkf5b9EvOEWwpM2-E34Ss;%RpOYDxC$@_cYJ6Lz;+KBK*%cQfA6sg_D9$$pz zM5+ciQJ7dWCO+PnR!kR!d!u)9K^ugJTq3N<@iHey-o)zDk0KT0wysB}5o+0HR-8o?GGQO=p2qYfuhXr*Q0K(8{me(;|VYeoUt% zmgt&&@3?R@Z+RoZ4=c5?EUcZ6aON0gUO%$Bn$HAr!iN3_tZoLrH@!#O^Vr@jxJE9E zYzYG3uN9D?CHBkH;^0?u8^QxoP4K>D0oA-8UG*lA;W*ldQkwPNHC0ym3{;GOdU8Ww z(5)XU;K`PVzf$;aDY(zy1kfZB#?+sHHoKD#(JtnRTPvz?==#=iDs{b@HD_Qk_icp; zi_|DC5LX1F%n9B_EO9_+#JbTS-UNOoEaM02))!r3*BHn=EgRsMk74v@ z%cL9WMVByD2#TN+kyi-s3ghho+Co|=RJ||G!G6jrqu?kg@G=7P&OA)@qR|GXTw0PC z*^Nv=tqbV{Xuo57z92;nmP$9}s%!#LmpkEjp0}#< ziL4vfHAbg+@T#@m-kbqM<494h3ZJ90l)(St9Gt_9^4EUqqaoZ=T$J zx)ir5%Fyi~V_~K1DsJ*KD-NJrMU3UR)FQVXo@5cF(BYT@tA>i%`+?r6MXmhm6jB7F zUO-O|LuYj2E;jI+i!Q@dqp65+-8Ax91VaS-6sex|JJ~v-K~Qv>I2_dP9UOWIXuxWr z)Cz{StT&3j(UD`#+d~zcnhJ1n4H*y{wlaj3X{FcSN6A&%cI00$M1!yzn?;y(=H(1_ z1EF+`1fVITI|X4ju^)JkVA!&Ig`dc?#G8++>zUlwE0vCXMMD=&(G4n8Unwvr2VZoH z)@haH*Ri5b3_zr)4Q%GQyz!yz4H_!>>1_@#2@2qFB0%<}dE{wJ0IrYCkYrKkFuTIi zRhExwXPBe0L~vRg&CXaVu&#dq9UUSU;{|(E=YL5~4)fO&5(W%rJ%&6`V-m*_W%%n0 zg|#z|T_`T#mvb_yeC@Y{ni_l#8O7!Ka*}JoDzS0#pAV$HDRfrcmGNw}I@kr5=^L-{ zU~$<9PDV*YDYJ3v9eFT3fUl*FmTAwH*KxcUCjPgJOGkF;5U8TZ9}Z+zXBohQO8UuE z77fMBIYJlOnMAt4kJtCGk}skDD!@5W&gTzy;y4KRv3<68-w^p;WHb%G65z2 zKM)=pA3*0_!qteFES-UlwKTb!bL9~a0jiqevaz8(P;#{eKg~8e2CO?X?j@NYYAzCq z1Pv-*O9H1J^FS2(Ogsk&Wi7kG%{bUE9MDh2KKAE-j#5Y4qre>@ECPi z3Id>yqhGhM(9Ae(3nX$Dz<6U}>@=L_p!l8Ri~&M;)bm@1=5Z^#sij)O;cmxGtSo?q ziY2xkN-PT)6u4&P4(aKsre8Use~6FmUQ0StSi zk4+5_dTL&VkZ!MDzN}*G2^=%Ucd?rew-pT9#YgHH$|7%udfmc%7<0~zAYKkR_k($2 zZ61SKWFmCwZ;hc@=JaLPSDx4pt0(G7>@nUoYzo`;A?#M4JHjT481v$7W6Y;chhd$A zqWhgJkz>7b7vu5itnX;ubnwB3q3QXFhq6Vd3hMai%gqDBi+F<@i+)0A&=oO4j>w}o zlLc^U@of?4N`pm1^Jd#eoT8hGoK{kbb=+-SldDFHL4dl1FOs1^of?NLJz>05(~N7) zS_cSjzP+!i+5lzWw1u-ljZ3QzOj4F!0Wz++JaNl=n8mTI5y8fb%kO)xPEt=0j`sm# zJ+%HOF!nn>;|&Y39U(0U34`0EJME?QWZ2TL&5u`0;Tc#1ArAGqm^|E=_0k}rTM2Jh z6K4vFd)Tom4Y%E4b~wpJf>%*Uj#8%NP^5Nz+Da7;NbCc4X(}gV$JdNj<8EXbd29AH z$UiiMVFli8l*lQQSTxG!5)BfjJCpK;lhX>S9@hpwgFYHjs_46g=NIGhpPIxD^B$R-*b_fr`*fv}tT z6E+(Iu-=*XjFE$MVRDkv4XEu;39{{{Q$i4QP{P>)nJiUcod*FvG6lTFVlmzAlSnH& z%ymUZDu?*A?dSu*RVkY*G-3$6kcGk*Uc0>I7t0xFe)Zbf5LAGmdL>E_xwni%J1Z0B z1q5hB_FPKIaZGlWGB*A_jFl_wjn^~t@XR#UX%&!< zLL0y(29#TqV6xC-Q+c@0)+~#*ns|BT-znjRhIV=`3k`wdMS$(BRl}6(x{Mj&0!n!= ziee1ry<2@ub?cLucDYt=c3}$!z)A&4zEM!;71Ow-e}qg37Er5lO}Pbga@8v^XXuW2 zP1)@6la+MqxMl^EUPej*y(HY~4yI^dRIxKNyDdZQJQUv`cZ*0IP4>m=!N<)+ed;dO z`PZrh8$>GaFc45X)60ZcxX4u%6Yn^Y+6DsIS71g#z{n9@BArbH(=^j@9Qbbea&+W@ z`;(oZ-8Lr6r9lMul}Vb$*MaLzJGpY1uAxM*J ztqwPoSjy@RqfIPQ-dMrniXUMGbt`h9rd!&d)C=8=x}#`xU`aI_R9=TvlRoB) zG@xP1@+uRH{#S8-}3R4rH*zMm!4k~}6`Q97TL zhpugndpR29jif?wGY@AeCF_z^RFb20pxP>JPA3I{i+c;u+(OoSa>d{X%ry!Z6kP5re z-CJhU=oLxZyUX+!D3J1WjVMftksG11ki3(pX>D^hj9jKbbX9rL2+Na`1I@T6MX&;k zI;$%;bZ;6r!NjvrT;)U=e{JQ{)P~AaL}N-hLn?iWoJB73kU4eg&|C;;nt|nXxNY1@ zaPX$6Q)60!h(3~Z+to%391qx$eX@(!$&8jS9%atga~LD9Dvc;RPmW7CqvN{90=J)X zzIVH&@DpVL?#R5?viLGsjRH5O+SRK>gT6GAg=ChE9PH##Vd*1FBDxTOfk@Kl82p9pyqZ-`bX)QL$d_L1hRef^z-SFZu z$H|{A+4nMCfqO%#F{5Kbqv68Xb_X{@v`#UtRXILYE^z82_mdn%Uq^1Id6I-Reb5sL z!rB_%$?~f^lv4nPC8xzlG#HyRQijqV#&*&rbblbgT8mI}n)uf5A{SQ;r|34LRl&6& z&U^Mo$y>}SO6euMGHE*fi|>~yR~#V zN7bQ%ER%N5R1>K^GTvt&Dk-TM_139pk+wv7GGsBdt#>HZ9QH8!FsXoUYksrd9`52~UU~ zHa#OKsnAXb!4x1pfKZ;3pHv4=4Njmh1at{gPWibz5(RzTGT8<~T4I5(z3 z8w{MOq;l%&MvW)GFZ>p)$EAI=VdMC7 z1{in9xm-6X&ec-*1U9B%E>tx1aL0>O!9L^J!b389!?)?^mT=?L(*nz=fvY^qG3 z51Lvu2q1uODeX#tw@lSnNm-`&+NOmzvk<~l+?X0 zx%mN~1_@jW9q5!u#uGM-AJ4SKWIg%fq8nsc(lZ3JbnqN^nP1 zWxOEjqa>3rCE<=;o{!b?;S{|VeEy^ybp&u%6EQa`G*MHqvFEN1c>-t&@zSr{Rq6nu zE)n0x{+Kp!u$~K!rDJku4!Y@5<7FGZy^9YF6e3HT_@D>w2%T`C)25fpx=>Qn71WOs zcu6N4;o{O zkh7iF0_QX%51!4u`)QcOvh-6iIL0QqBw z*9_?`C3$4vtB%QLbPI#aOzrF*53aM33BqY@mnzpWQ`QHas%d7;ePNbP>QWHpBRUI3 zz_Al4%tVqH%Ec9Kafhihfn*@zzHJ|$uSmlC84ji`ZP(r8l3xtvO>j)Oz;fjzrdgpt zkzPBr0x*|CfTG-@)2CRiDGP+R$k=iGo(rDV2LewA^G$kw?JR4h;O z>O=z@e2bul9m`eia3P~46x3&2aK31bABn{MyUs;p5ie?m(`PuoQ!v3pMfqGmGn64( zoT?Q|YNcgNTk&I6Mw$zuydlooa^3y5ki_}9z#$k@R^3{aS^&Y0^@w5*0(E-%PGQ30 zQAozez6X~f=j(!K?JW)2`fa>3iH&QyOX4iL81ilenU%Q5YKxdJI*G#)V9iJ%zK()v z-iTHpg5w3Zx~Tad7H`3@;0*UGB1e779MF9onA&{puBmXi6js|T=bnko4zOF7&ssUD zFw*!O&B0yonsLD7qup+3?Nb<)9UQqx@}Tt^Dsrov8SYDXG_>Ft@-6|ANV&;eDtoC3 zkM5EHeI&`U5Kon?gWFRS>UJEp>c*e22ja=?5aAghsVP~-u|iTP`ze%AL4*M%%qQEb zIhY=h!*Hq-?h6~5=fdK$77J=Pp?W$6ha>?oSlt}!A3!kh8ib|X4^|+aWiZit#x^i+ zob=xmd?@yK=6s+}CG;3bPe_Xaa<{0^T?IYdubh)gAWltPTH;HWzbFU~Rx4rDUU{UY zpD$AVyHKT(@<%c762=+}u}g$4T`YB<3}m$NR9;-s5pTJ~LlQXT-&7Xs_NNHo9>n_gVIuJf75DbV&(<+qRzhH`_-VOvU; z%GT$zmFqWPQ_J=^4wl;lVf4P@u3(u5i|roVrcZeIDS?2jX8}bmSyRO*yo$XSMzOcq zjO7&OKtl6R<{+7js6ys9K^%AOj7wvSyu9c+ymYNUL(DZ4Z;~$%G6Bk*CDj&GDi1u@ zBO8!l-Ec6OX5riCttn@TDWog%&<-9D;K$Ax*j#~Ws%em?#0cUQB+~_1q8g!vo7xB= z6PWv;o{*PE)-bh93EcY{q>l89#-uKkU?WtWG2)Lloa0^Syn-#I&hcc!De2%VME3zv zCxxEQ*pNYtI~?LE;uCP7jG*Y3(dQUA)Im#o?t|7k99fbL8Zxgqf+PF)F$XMm8oZ+EmUcyL)X219a%S`Y{b_4 zRz)FBZ=9U39xkT!n551kV^&~ha?d;cNuJeen*oah1Fx(mI}x7wJ#LnxOadBEK@`Iou!^Meddb~O%a1q0bWJsK+GX{PH%nJ$Yf zkm&35oSzziv$v)ajW014+@z>*n79>&g_`GM zV|okCPCE2TV~@4MsYNBiAE@2)t?IgKtC?90_<- zTELt`oGY_Zw|qT>qW$3w*KQ0t=WM31NK9WQKJcq-fm)8&0w{S&a16D@m~khxFs9R~ zQC^}v@1xcpdKH%3Z?}u>OU=VTiIW7*@iumJf%9NO>I{#RUkE2WWKxjXgtzgLNzCV_ zNS)~~y~^pN50actku1*vz(RWDATtEmTpr54Hi{gUEy(9bB)L0LC_?cJQ+qVhYJfv@ z8vt-u1tgUzd0Hc<8DtS0RoZyC3p0OY5k6ls*R5UjP+U8HUjA12k#8tj zBAT}E&?0j^-icE9lAX$jaG$Gei^8^44nIUJvgjHe_YM@#pQJhOfV$VO;Ue<}tVfXq zhlNaFPzgct&^xb7Ye(jD9X;{{QSD8arz+k%rTl^>y@GP*a}n++@qm=&O$!F&AmY#& zRTzhmYI)y>P=ghs(i0}E_Plq*WS*eqJ%Z=xf}%SQWigv{U8~|p%_XR3)o>QRNt=*&r^_H|*jChc;vQ!OKJ?2XFmNPRk>t3>V+41n zTs^`Ai-C6&1j-oM`McKx0n?b~>houK5LdBfQcP zA2nnwlN5KY5TQv{r7OKjAliicx`!Cfl$!Jw!R?SN0ktF)w6mO}V40unAc6^khy z7QbfOeOb%TV!BX{m_?im^vO_{)TQ^Gn#pjmf|=254s|+#TX*F6aFD=8csM~u!_c(9 zZ3Hrf%4D`Ssl)WD1Yecos8gJ1;6GO3w;VnQ*~ebKx;NJYvfI+cDZ|zBBpCwA&>FBV z211BW&0&14&9qC~P=&sPMr`ny+I`h5RN`WtE9HPS$8}jDDln2Ue2iYQ_X&#f@N8VL ziEj^2aGjB(k)MS_62>5-y9hleD!m+V9`sLWTMLPdS-v*ezN7VDHZ&=4n8W$!*(fdEXCcQdfguj z-YaS(p+6=p&@WHZ$#M}y1}s`AfO5&j+1*D6^eNCpO|vh1(mWf88c*a^BQbl`updPi zB|%Y;V1kd{NduKG!ooCeLHRYp--ppSjW;HsqtPM>Vr~`LVBE}adA6NVVkjO*BXWG^ zf!1Kr>CH>5DKH=ANQd^Uim7~WrJJmz=e9GWIG`p|nQ(w5~kWIPBzqQWjl$ znPUlW2JrDX#?~^RWWnTV4kXvk3iL;ktJnrZMh?@e2JgLMNbogWlJfVX1Egd!!Dvhl(EDQ`q^Me7pVM8upOz_|(8(!vn zPjs7e(95)nUT_r6H`|(w(&JiSeeDXtp}2TxFnw@6J;-bM^;Jjmx_3KE*S*Qc(^$S z-!PMfT1tBPJQd1b5hXda@#~IZ9f>H=Ubm~wgCYlHpw*?r!z*Ktd9g;y7cxY1KaPh#7P+sR|=G(ZQwRkZ2`sd zh@H;T>e*u?JH6=(NNl|ebxKq)Tg%~jeGo(kbmc}FL-CFKp>czEECr0ralrP$#dfeF z+lVFRE8S8GJHoLnMj|63H)n4?Fsp&FSkkU6QA)5(AQ66|QtW(oDmfy@iyPUR??H|^NzB*%jC817tHp$ez8SYUD0?p`g< zIR>>gt~0!#J39EJCm865tOn~7nGdAAE1;-qp2(MQg=}B{!u(k<|GN}%+?si8?gevhw2gbS4QE}H@6)a%lD2hD?h$xSKq}#->E5UjQ zSqBW75}+Mtj(1>^`FW)0-F^*V6x*nv{NQmzXe9N*GE%in79G8)p!qL}?=|$SarrzG zB=4LV3?2fc5iv-NQM&LM115r5^y3vq&4tHmq%F^kojmeDTd6?us~X2w)UN|F6>xKd z+H4cZJ}C^cQ_K_;pqMWPq8}sP4`OPFGE3oS((Y%p6-${G%+Ax?lU-JTritPt)~&9D z9w2t2ak?d0WK;_S@Ue$FWzG+^yltO;Jl*-p_*zMXQ!mv+3KI2J?RAnBscbRcrruFf z6+Ol_fv>5n0;wAk$8rknJ?w1P&|TPG#v_CGY(&^__Hx?}=#%&FLpBF#bX{1&R zh1gM|&C-zP!YPxnV^_VQISQFWLvef;a~|H#s@57`CAK?*$ms-`!Ub0A9-VpFBx78P z__FgJe1}iK!Gx=;NqaPKAi4~r+>|xOS+(NT@{Y2&&?`)?-!h_r`=hzB%Qw4d@)IW> zgfI|dJQ~<<0{?q}-MmoqYqMeLp7CvhQz9bc5`l*@Sc2%Fanb11i|VEcBV+K|i0F|| z!BV+^k&-v+4@ZS+aHf+1Zx~(fUN3ZFj!;BcUdl`v!y<7!z{$%BNaNzGhbOGKHXA+l z;$u+}uz|@rh})W`%iB*2UKR;tz~c~RL=8HSun@(IHiMlwi7(*SNfZ z3)a?boQ{@YYZkvbJ^cVVjvf{4&CbgfADKI;<;BF!Vnl}ae9VTYg{pY&e!2F0^FyBOFqNdnO=6P^!3kf0VaNA$^FcGXUG;;V~VOCXNRjz+S8ao$`;=(@yFNWDTZb16fKwjaOo@$2|5g zPU1K+u3U=haIEp=-LuolQq zBzgTA3*OEPV+n3Z-%&oIw68t^!DXUS;4$M8i8~P?#=GsQ@%RRcW>hl%TwW_2l-|_z zX0;4iy4h}^{4Yvd1k0TK32X-RS69L?6b%n^cf~uM4d;D4&pt7nbHeB-{kCo zn80*ro=i}TDJ5s|_Cnw(8n71<`qe;DChkgTkcZ>1*@hrCJ0|PfP;T#8LT6 zOGF&Tuu6D6UQ;8dR+%WlvYybLj66j6!y zh;^r>dF|oy=I5a-_Cm76=P!xp#sl_5C|WgKJHT*b2zW&s;R}4L>-Yw(<9n198S~ab zK7HBT&lz`hm6oj3R!!y|xN*aVb>s}-FmiX9Meoav*vR!=ZGw`0=BH7(8DN0si@hK< zXW9>1y&xld1KFf0nGTZ0y)*|^m2QSLQuewLIOM>X)RhB`78%>qmgIaNdjQv% zyPSZ+o^2R4Z*gt&Fk zE5$DUYQk3`%C;dzm0s1&Ar9Gj!8hfKi_0^QeTteo~KI?U23gQQo z9oJT|4JBnoOkOPa$Ep|v1SxyyEiCTRZQI zg-Wy>*+(;aeM+fZ+N0qNa5x3#4zZengiGP_N+h_MS@K5TE3yxPm2Be5J8UD!CvRII zSw(f}YPUN{97!v_Hc+2QDP=Ei{u47o2NScM!Cy-?QgCxQ*ys+TSp72;5;-H6RD28=g5JAwdV>}z+`K7>mK!QaA{8%F4 z;cq+6MA->oX_t_?VbB!@;`9~_HP-E^b72bt*;IYea@pg-a>!il+rDgCZL7a84f?x+ ztItQCR6$70uohv+LL~zAcL}c8T!6~hF*PP@Ny%C;N(EHlsn+g5yV}5IIqs}`4PB1D zU8pMPJ8BeF?0s8#%}0^gBZiz;s}hoDiDC~dkXR40qTR*B$0FSdMXmi`jS%3fn&Y7j)W zWdl1&hKr^ZLm`G_iff3n>C-R1q``kjuy0kc!yg_jc4I)^lH~AaB~e-Kpxe_od2VJA zU^%)9n!+pK#3$a)1&q7EpcXqcVSi@0q4F#CE2cg%s5_i`iYPgr_S zaxM`?0oYC$u>G^wfML_Janr>joO4nOkE407er6>w;uu{X?-RR3=Zn{cr2ll(yQv&U zOGg8eP$JknH>|Y`E(nacFSu!N_BpAYy_EbDiYsbJZv;!H0P_eJue`6PDz|K5W&m11 zrN55pWDji-y?bNF++$W~SF;J8qIckWz-7J+q90F9Fd-BHOW9DVVZ8KqVfzB*vX_~@ zNKj3PuSo!&G?Hl{ut6+h`Je9QDrwD-P3zaU$|m*K#-Si!_-w}7z(-M};6^G8UO$?d zcI4Z1fxVvY9dnYd0y#wY?s`ac9p>tjX^vGZOG5c&(iICm0u#-%-S*>ov&P+sBEhA} zKJizdPBEuiMsMAsq)8zjiQsCz!H5?aJDm#9YUM9D=~d;cTXigExt#~%-pp{PR?b;`ow0bVm9>_HMZ=B@4Iy2E|EQxXnDARsd9cz%j4 zk{iFUfWw-)03r*hvxJio4(O*mq`GoW1dlfYK+2a*>*AGaTyYli zqB;fScnFV@sKOP_R5yh^MGUUCj1BdKVyvo+xIZ1)8|}V9>fzc9yyBL)cXWCQGO9y! zp=oM;6}7@S)*>o%;;^;1ur_)v9*E6pD)8s2lm;@xSAcMP7wx0tAcj7-(<>G=KV9^v zYUwdQQ}KBlB+RUPk=drWc27i8jI78gPh7pyN|wEVR^8VtBR4V?;X3f@L6yB^y89aK z)bLYR(kx8uOs+15(n)J|!2W(Yz!4LCDr4i2n!{V%G8&l%QldL$$J>Oa2jqn2_dhT- z0GT&*1$mPSf)K)Oo~paray;dgNPHW>Dp+f_qANhaZ7;<~=E132=)hi(&3VR9tM-30|t(EW*30yC4v=cKH+pxD_T{wIO=X0bjnbX+$}P2ZaQUZpz9#vPN#?l{?;_pt{W4oIX&6d zF+w_*e1CMJZRx5*2Md?Pm_c5pFtO}da}8DOj2+7BxbwB;=+LXqs~Avwo=H%-q`kXf ze?~3|r*&MouTI*UIJuXnv{l-%C1j*Wm0wpu0C$$n(>mHEL(m%llaA>$aKS*6lURkE zB}@!zGMrp@+@#ueHBaf;th%~B!$q{S1Q%%CiEJh!Akg?mOK}|6s@WmPOoo^a)0%xzen*oiQg~Mz3bE5`Fr^kyK`bl4%1N3e zW#hnsMXr=OS=X|{AeqgyYtq2^C6@CqhpI@-Wl=A_-h11ilxq zP4usCl4BeUrX=KR3PM}2R|8U)!md?) zP79k?*w&^p%{}!d(XqvxNxpUI4_XP{WuuilWY-QcT>zGb?B{Oj)VHjg%ZSHL&ZO&; zD8v=S6`{GxNai*I+LxWzCKCYs(<8xofFO69Nfp81UoUr}>y8Ahc~7bEVc4{7X)0|@iT$=bUCSqmTXmR@0VvfIX)5+D&^2K&3N-! z6UB8BP`Kw8B^r56LrTtZwkN2=bhWE(w^P8h=R@gCiz8(4H#BrPUWoiXVj75iL>b3) z2C8Qn*^O5em!~9oAgo!-HqcE?9dRKXN7*4>(JY!nC%N78G5uZboGd`2TaH_gaX2MV zo~lTntjHP0v^_S&MwIa;yMbz|aocv*m>zrzPFD28b!9`GtDc6$$v}#1(+RavIFU#j zp=?OR#ya@D*2D-p^fk3aiMoAUrel7C9Z3QTc)I*B#wVue>piv)eHXrD6vzj3tSs4W zLxFXM(WyTe><5K5Lusw2W^p9aJ(!JW&&(W-Go#Lg?pt^WrCH)5%VnyQ7Fu&Y=t}C&Qj2}b= zECn?in%+Bj6IxIhLDND#Al}0pC0HgINVl!^50uU4QQ3I9t4q*+Y9*sc24+Ifib;n) z!<;A-4*ZAgq&ueW(`os)sy&FixzSiE0G=6PjA=3>vvD?Umci=4o3VbrjmzqxOjq^L z(hePq@Uri8<8+p~$)y?M6i*u!&yFSVJ!>7<4^XUXKDV=BC9%A!5LPTQkmNfS1;_~Y zdE9(PrN|UxuqItVn-9zl?^K_YqMH$*Ma}mN6PFtP>VQr~$aq!@aT%=BwI`@X_0tV> z6NKBR9sEZ<0TJ1cPRmKg0*blvPPMc~^0LnFU=J}Np$nZH3|(W^TnaC1E2{LRpmSLj z6MF1Yp^zZPp6s)YR%kpbqMJtG#Yn&d%Kh_#S~^;b zBp+8mL*+yw)rG3bGxpg7Db)ql3FBGiaX9MKRIfRz$#CW|5*qsM0;3SiaSs^Iv0uxK zW4Y(^UA+emlTF`3(Y|ElA$s3nIXPZJ&%rDm+k==ki&j#>wl zLW6GNhvmgpkQLF)zaMP5%d9M9+fC%3V`Bq|H7qi`dnzTCK|Rlk3T_1}1Fg6nHpj=P z>{D%EfXFq2wx$E*AiQe{vO{2T2a5?B?lErJMl8wmy`0eaydiKCltD?5kjg&HWg4TG zjP!s7bATQK1bFk>oLmcn7Fl=c-ZHeF({P7uo-X(RBI|c*qhd_Uc~Nxr-nSl{xI)8< z@YCBEl9g=Qj=n194#Z5zCzTq`e3)$2wEa4m9eEtoax5h&h+`5m?M%n*6@r+aqV!du zxlUlPw@&E4x48lR;9iX()o%lRsPRBB8@1zp;yA%U60q^H#&KU|IFae0tAbFFZBOar(WnIcER4bHg zW=2rME>6G`ghV?vXAH&m3G5}#b3yh3eZp1ZFv%ggaUF%sPbh@XiP1K~(&3SAI-pmA za`@&!^9gQH@+Mcbsvb2!=1Zcvwd z?~IR*j7&P*y?sT}B0deOiqW=s(_eF##frsqllT<@+zAachoW9!ZFZGc$55r3t2T$Y zo^Tu~>aCXO@`BN0E6PM0bJQ7#R{Z{UZ7$`Oo4jo6TlM6X=5QL*82*QRK+fe_Re_eS%x&YB>iOT_N(%@a*wGX z;H;(3MIr`}55qva9lY(-8Hygv1QG<=A#Wd9T$2d_b#x?FvNxruLt+<6Bs!uYf@#Z? zBy#yHuU>$zuM?1=qz^7LWC;q*aBD@=1fGWSuZ8yw&_9#imm7lVM`6k-Sr zh1gVcO9F#dmliVDtn3tr$ZkT3?SKzou)5ud-XS{e&{1BXM}R#MsyQujUIocx+Jl!- zQX68AYE+t`zr7+MWQGlD!={Cd)D2LCf}=o8p}6q_e)aL~!nh=x5=u3`9b?uw=vbh1 zFSxLIAqR-uZ#-6SD;Rs;85ScL(FQI)2fgDO&Z>Ii0YqhKFc@)eMiy4!BtU*CKp+oM zb)Hju&dRd;uIS4|SZOOtW1UIp!+8We}TgW2^fdpUlR&%J!)CL+ErI zT1=eG;XR%lsoabT+LgexF3l)q#jy^92ZoP3q!Xe$69QR`5{)vcxJSyTnBIyYi=A=6 z6}B3snB?jgurPO^e3d*TXnl?~m?as-E`Zzjco(b+w$QA27?3uw?vQ&%`8UHcgRjd@Wp3fGx&G-ueO)*K6?wW9iS)!vn13w->h*~mkhhe^mrC6c;={Wf+m1avKqqugB#6Qb zWi%mr%yRiovaD`tigQ+!qOf*g<= zB>}E%Kwwy{R-`2|CEm+vkTie+!I2KcE-ETQoIrlI(%*^lW*r1)s%V5EyeDl3VNFK( zjJHTV*`dyNNc~zDz%$a?EKeWkCvJmvk$@#5ciZAC9*Adw;sFM`K-2Nx3~+g=w+(d zMuC=JrX_rGt<$-$nak4hCS>K!yBDL_+--`J0PVtTXohfgU6*cnl#1wa`jr~k93mV- zZcYb%siq=8P{dukdPswze%ZB$b-e@p~~952?iPPvgF~GVhM&JWW==g z#*-i2%o3zKt6JGj48fO&MP2QaX_|ZR)dgY5@6O`$?PYR%VpVaY*LKf#~qHw_FuSd^8>d@T0 zs+?qU-iV7_)Uj)1m%eXsl`>5{ayjlThm^uT<-G?s5_NXv>f-_#DtFM%WuO%g4tQY4>}B+3dpT`M7H7MVd=O!4 zxCP4FqOC(f;0B@sad`C(8eMlm#JoH?qIm5ia!J}KP$K<>>!s7_?WmGdLISeN8gZ(5 z?Lw!pie;nYKP~{2$3^)CsxLuxwDHOzfv*o(jaXoCeT5viNhZW@$Cu6&dEU108PcW4 zI|W900b(429^o#Pa?wZ&!U#Fg2M<29>U>I4nm9;-gK!ovY*NHxE&3QfbZvZk>#gqS zVc4MR&YiG?0-@*7r$*Lc+Tn$p9`56#$yIM2->Ya!C}zlakx%3XUP@_(SJu;dz-xBC zt4ZGZaYlUN!U?2k3D*3NCSv)Qj*}M zE$gME5ZP&_zBgcsVq6-c2N&t%sWpM59vA48-Mq7IiYhHbKL)8zePiRKRNya1)s)TK zgR{qaFE$_>>#9a|X^b0C73y@a3FT0b6E6*NFnH+};Lb#pm$B-)=_g1C;Gwzc4$6Np z)GelCRa#?>W4Tit*OhB4`OqyKWSG~ySMK?SV&w5%0V-_$;S&0TvigksU zs_a`hTVRbqi^?8KxzyAUu~VL^PM#Zl`pj3WV+1u(ZR=f0afEOZFhP1d3=9qA?KZ@F zvKkI+?G>m~&vCd5L-eU_#kV*>(?<_m3qn+( zaA;vJ-kweaj6*q99=uEkhDgmebEZ(8eW})j$<%uVI$1=Ge6OHF95xpSdjq9}R)mea zk_)qfRZ;`RszpU%7tv`vsWacQj-+@Aq>P6QmqO>?-%VDb*RfQ(>n zeUW4_l-E&nH%UC_a5=S%yTmN*XcHLD6cbLyp;+!TeB(udRfFP3%(J!}6t)C3@-QL# zLqxr1%o3X>!Lr!C8mAR3!53Z{Pbg5K)kw9+MMC^yxWvo!n#=j1$V?D z9e`4x3|_`jfqRwwKG$f>v7)pRQ3DJy#0HsI5HJ+W=zfN1Zz6OP4mg+?C5}3ZH4JYh zfe+rF5StVPXr9#Wz8b$2OxqlAJK=UKOe;)lFfFJdq08J|7WcVZvAsc#e!%YH6cz0g zN&#;MAp}NVF2v>6qL#`tBRADnTlWH9Nra6G{>LuBXWrZihU0pRd+ORVzzU5PZonJ}Uf zRV|IdP*6R%ElI~>^o;n=^M@hAa;SE?n5ZKUOWA_T%rrUz^;fgr;*N`Yx4J5tS&P0x z323f9#CPWiR$^r^eibSd7-p1NoS1S{`Lv?dZ6pN==z zroVjvX5cBYKp+-S24k>EK{dio5Tm?_Qy4zU^FIldHB^w`9KC?KlO>QwZ8kn177-&E z28JJbG*RU#&h9B4v9>%7HfRnYUJEWr7S@2V=uEFzw5DjQ#~7i+Pz%T>+#a1$yVn>q z5rP|!F)tw2%5w`RIxUb}w})+|v?(Ri+VH2SW@E2D;(N=`ED=bNZG#3%9JLMO3?x5P z#S~cXlCBMfB?6YfyHq$okzRm17*P~v(^}Ri3=zmusEk%WgP;JYSB+YI=yQvW=TK6H^lw1 zoRnTwRRNzg@n^6(28HXppbfbn?QlO4aC{|I5sRwQUR3ZQFIgut zW~&XksdPv@YJ<8>osc3iP(PC%Bx)e0{%J0XosHE6JUAM_z2{u-)~xYH4%{AW^q7sj z$YD<6;d;F=x^8sWm!Z#7t;Ev2a_Ua?C&6dbwxLnu>U|hKptWuyWnH8doUMkPmm=_( zY62P8n}@;{Z;xFM$f5z`yu&m-C6>uInHnH_80$P-wGiP!e;%lBo%I%IMW!P=skn8n zO&iw4R|WTjg6bB5g^}1n23xf|ZH+iZS&YjZ2so9})K3~b?U-L0M3`(%FxjvG{31UH z_m0U;1*X1DS&8Mc4I}r21cfv87oK!Y&_pK9*_!%7uGJ! z#_Hg1)oTnXqRT=uQY=NV)nrOa**WP|!ih8lxz~kB;A5cy6%sDhWC4o~Ndn!pwLoFD z!U`6J5ECF#0|D4CyxuraE>~B+ssR$+MZLin#$7SUA-lG(V(AYG-Vk3Azh>Hnrk3td zqX|hcpV0%UE00IZj8)z|HK}4d`@)E`An9n!&-A>aT}!FFlq|wFCq8SyRa=VrkD;L9~bz95yw6!g40gRY zQYx7mQPqpP=4pK>)B7cOOU8=rdIt$?xQF6(I8jiHO)pkVRylEe=JnyKKHm_)Yo8Vc z9@AL)1SZ!Wo>&Jvh5&VU4WsOZcnqd=>^bnn#Bvu!M&$ebqvfCaR zRVQ*h@EAbWiA-pFUd&0+;;Oe_DIf|d1;+^j^4{d!jNs8>26k?d6ubGue2wskaSl=B zIt>U|Lxc;i9gLTWP%QC-s9&re`_T6UZ2)^vIN_2bzNQ zFRI{(N!wLc0dQv^pa;T1R8KlmKKsis0XQ_?6AkNbNA5mkp~*{# zb^&`q-72-oT^MW` z!}hnMWp7OZJxQw8un+?JW;~<}Vx#Hp_=E?Fv&=fr2wu5+N8mkM8_%dojMEBda6u7> z@MvOfu4_Zkqc!gGLTMAw49jtf=5XuLm956uLk8)6gecuU-b2ErW<)jLZT7|IVlCcG zjR3>|xVDI>VNeb#$B$H&FAb&6NZmpK+uNG33pc7dCJamEp}(?Hek6B36#@EZCP+xb zX4dscs4J2|#eF=Zw{P7xL!PE|=NreG7|`DG!K|YTDdT7k5x7B=xz1u3*5|JEw+vRL z0x6uws>;TMjPml~tRXK9XkVo&6hO=u57ZrxKy?gNMjO>sty|1DIHneu*Lq(ohGSICy9UqaLl(nIR z0rS9b2w~Z1MLQ)5k5}HCKSukKXq%Z5)E%^MK_(Rx7F3+Xa_<>YS*;kiA+e6YtN;hl z4talB>|dx#3W>!GDrnGg(bg_-J(1Fv4$oj%gW*=DnbJMJWjYE;15HTA&W-RGAj`Y* zR(S7Ug&U;qEIqQ$E~v|f)5QRU<5ldK&Y-sU7tosDGs#pgnK7pvgdvnGm zAi(83t?Ta7dMU?HMi=o9r^o_=?3;lblp#6o84c$QTkjcATDHl;m87u*rE}VmA*C4$ z;M@>GIhO^mc(JTGdRHGB`%JIp^Ta7E4h;p4=GCkC8)}WuI~0aYa-~;j%orRyN=8ga zIm<84>~N>KlfM`&pu9I_3AD33;7kn4gPV~D1^H-6XXz|aF$`w`Yonpgu3Dgu2j)Mt zaML=dGsBG(k(O%RIWL(RU~{Bc6hlL^9Ty3BaVQJ#Qx=T3Ook9l%AB1QDJA08M_ak# zftdw#=a(iG-NQ{2-HPCKAaz1fhHLQUWQ=-shJg)=uQX^dUD6;E9mLYdVA9G?IN;E zN^lJwaAbv;$JJoZ>sHabG#qBmNMFGkT8rCW16>}c_0l=aeew@9 z!5}jepDcNCya^aE2usYFa?^o2AmRo^HB2p1LwB36>Mf;#>Bx| zZ40DN8vW$FtWLvugYVPF;2>$kv$!t`!FW_YxVUcQTF0S0s`{k+HklYVK4QGZ5nwB& zFGyyx&q7k25{eOe5-X)g%$G$-t0inex6-MX-Ah#A3z*JalPU^Q1zER3TPDI;RM}ag z+i`3m)Wdm^Ie|_IR?xhs>8A@`L|!NjHB3_ZeC10eaSBV3oWYO`Hv`AXJuhR;2!2iS zHb`I@4A?BXUv!4^xtX1msUO$`K10Qaj?fg5uVZ)leqp%UuVBbCpFbU`jPC(ZPoGV7iXP)PYE z=YWc<#%{Dlta|>;%$M%JS}+=c@Q;N9mS*`W(el21D{)XS4VWWPRNESzUeBIamC`f- zMuK;!E3&~b`%)rjyGXWre^rjXJ>Oq$a(Tc)*U#uG4BU{W%EYYp8?**?%2%YSb8R!3 zQ6`R1*z$C{mB5CHw<^7G)tlLQN)DABc<@CAIN+o8dKsX6cB0s!VmrxA%Ww-^KOP?Y zc0Bjm1t$>fjQ)L5?SbXQ^`0H=_E1J+OzjE>JSUF~=)72T+|k5AU5DCPURh!GBWvv9 z_n1vjE>E|uvWsEPN;>X&@UR7?WsQ0i}78>%54MgUb5j=ui z%^3|2mzy-GX7(`2d=%lvL*sHKO)T`)w$D0CV-s@AwPj0XyjZhlJ!4o$ZpB;qjzwi1 z#G{9>`sbWj6%lwmN#xlc8;TL0D`|L#$MmHj6WU^~;OtvAMZvmzf_}*tIrVcQQ%=o> zmF78Uq3u(s-b(2RNdu~4>#`{+Z(hiR2TPjBj{^o-V|TqhVj**X4a^X!H#_x6meLqJ z?Y2utqsY2hQX+Be2kLn`o$a-W-Me5Xt2fC(yMK}X&UoYiMdC9Htj?Z60pj)?bK46q zw>(hi>&H=Ej`CnBl9Zk$aG%h_Z*KxK<}p(RGzP9rs|Qsev0a)47Kj8@AmiB*ZMk4V zwiFYkng%fvm*EI@rYWyRtUWzwyfIckm}YR~Ixg}PIxn;l{w7w7m`DY@xHl0z#avc7 zv4kQv>6n|9plJ6w_i$?qR$R6XCfg%P>C7aZOpCH(XH923b259UOV_Iyod)6N;}RW2 zxKYRykg@k_s~KGM2s(M+O$_9 zBWbNlqsj}{pMxtk+R*FEXTiYOs=yL63!RE9q`P2P(Ls~g58%Oq2=W3=jD`$FmtB?1 z!4o!hyz|~DS;VwEs*ORzkp6nCP-KMYR+JuD8|GO!#5<}mQBjCH8L1(>4taQ@TyzW) zaED)>TMi+p{Zyt9JY*fKy9JM>8XGx8gqNbm;ESgmxDyhl>X#2>w`hPWe=W6$Dfpho zS!2#qDrjaLG8oQOsM0n{TOiZfdrzChsn;H%t7&4P=6!5r;Ebf$Bdm*NzH8|ssq^{S z-H54xLyR@2Hc4TE^cD8|BwAFKHJ(Y|X?L}DOwS%NWWyQ>g1@q%nPhr}4oj41OE{r1 zR$i$OiS(KVpD);*0D&r^>xzl-ICNxPu8m)N=Itjed7rn3SI@A#DTRXrmUdAA;#=Z8 z?~#2B1DO%np9wfZK;?juLG?{to_6-O&mv8#2tg!qEzSdmI!lv6a=i8Gh_8u`YTJ6S z-Dv(BEQ^M{k)+aNVTKDf1S>T%(oPj#ff*Ac)0$XNg9Lz@*!Du3Wm{XQ4qWCSK#ny` z9*fULcF{7vf}rTVlnq1~I3r#_LuBn00kGU~aS`vtw3HP^Wqg4zDkb$8oz*b8Tj>h& zb~L1Af~E}Tx>(UO>R=`)Dy1yw)8cr{bL)F|i-sX|3Y+aVF9d0Z+^FN1vD{%gZzjsz z1VVM&BE2;9z4Ob)~3?3*oqNz#I}p08$3?;7egN`4l<4A_>{yTqqjn z1n?+}^{N)AsGKT^f|lWgX+ev3#cZQ+Y`5B&q6*`&<{lvK5&J&SeyA|<#i{H4jBQsC7Byhr<_=x&J{*gXFbR9NVtw0E1#LeP?=XziJ7V1QLm#3K95}FgHkx6 z&cv+SW&)9Lw6YVO14`3ZFQ0Ffrg{Wm6ONf%=Q^mAM?RBQ&|{|yavDxhuWx{f%KFLX z0hB1TqPxVlQ#Tma4mTnjkV^A3BlWR$rpoJ5SIi@MyY5HYs(E*uTB!fOo}b^M}RpeTEauC-ZOy`Q^oxSCEsq-o5; zL$zgQhI^MX56;4W052164_kdE4?Eaz;s`rAcVXpCh;gV}=}?Q0i(nqaKBCf_2~&k% zIdj*@>xg}r55??M4DVKHC2!WqoD2kN;tg!g6P~5xK`~&n9;LV^6{7Q)??NPr_Yc^X zXkyW61_7-Ps9ED%B4NknBV2sLqwaUoOYbigcUbo8n4bC_Vo)G7ymWpbgKkM@Zx z$H+OEGu*+}O*Fc3OUsrW?&7sXnQ4EmsDo2EvI41kZCd@Dc>U@U@*s?#lebJdK;?E; zTS|`eHT}FcE#liebyw9Y5YU3nVlO(p(Q2wh9v{4YQ)D?tu!Bp2;e!gpFgj^7%ce!8 z<4~}isfD;;2ZV;A`&NY8KQzN(av}X@DKpHe2)?`Eb&qI7^gk3{4>6@XO?CsASV3aP zxefcY@$WI`rC9dHpUVwp3$*lvz--(H%?czeSlNwIt`~QlRh^c*M-4nTIEqY_?&~o+ zH4Q~J>O@Q})m+whajTl_JloC#wRTKzv!Dy5SB$o8!6RtMOsze>__oj1md=u4fZBzf zz^SLQ{LMP^6_>T?$!sZN=c5!JvanR9XAaIz_m4g{xa@aD~1BhjJC=EpKB{KD4ut@tuvSm_wtiCA+w)=c6h1;PNDi_V; z#EC`R^Ve}Pnoy7}eyz*#5>nfJ17j&w12~5QBhy;Qm^dmT9Y}FkEHs$LSUPokB*#@z?3XKM6gawH_OtTBXqJ?~?T8K4kx<$Q)KpxtXD8~0@mZ@4iu zjx${&yl0j0IJJ0#xV~JjC|DuT+dECDUQFAG4lJs70TVhHiMl@Np+dSL66@g=4O%e3 zo6iOCh98Ys8ceNUA%?=D4p_4A+ze{k-JX%Kt=dhaly*_>860~o`sz!ay!N*YnT#ok z>j@)h_1ot_h$5+~@hy_ylYHlh<|EE4KEXx@feH7G*CCG>Qxn{wQmBzw=2afk*As5E=Y;AD5Q}6B4KudqL+|p zc2eYeOrSx++yz1+F{Bk`i)C;S6foT2xnU)q`4xubp&ul+?CHIZxQ#fDhBoJmL&^+g z%B}IifFD9A#Rl`61bu)iLe$Ht3NVAg>uF*+hR@1IO3E_ zP&BgEu}7rTbN50Gvl+jS(4TeCjxH1TA+4A>E&=j+Bhst`OGx@qGJ;l|zRP zj7C0sMu9U^r4DG?SyygUuWQ7Qb$47PplTCc(yfYZ)eJ2Ik&z5HtD#Yx?fk^8fNr9v zIx22KHS^D@>8GoyhSWP%W#Iig631>5F94lQ#$J2_b?>T}i`1P(Qhq}2?wUUezeJyy zxB|sF(1~>FLx_xY60%ng`bl8rnOhll)!3peNMPAuDeD*u$bNqgU5LnNK-L(zJ-uJr zg;?qv?Wjt=E9EkyHXP7a8(i$5}d#na0pD*~thM zO0LuuNuE35V{=}5*pXOB6DwgiE-EyS7}kpv=BK>5nQ}7h-!<+Z*MLH zgB-%xLiDMsS%y+_;Ves(!!U-rFrLijiXD*%NlIwJKJvi6Bzw|N4@#X^RXJ3I3?1hL zuwJz4G4^oBfzD)HpPl}c1;YK^>3Mfs1Zqq(@_etVY%vfj?`hk+w)n*IjT9x=5*M?M zPiSXHOSQ0x&k}^`TTUhAHi(M7t%iJFwuMX%qE6Lxb5A+N&uQkkHzU2zOcDXRUijgw zWMVq=Fm!52yE2f)oQm6vgHV&!N9`KtQ z_l+XOFcEFHcyO~VhjbNjfa{z@B5!P+ftOEUv`$f|9nh>t7=w$-7e56KC@%p>HlB!0 zge!L5j3MA4XuqAcGj&@{qS)=CJ=Rwwb==J;yFn3d;7E30LET2BYf}ND+gQG-P)uH&f$!RBjKI90VJiMT{Cy0=CZdf?#{{^f^5wCfb5V`0eISctGtTX<&GFa#y7VRV$Uu{m!s&K z8Fw-yys#1)QB5mNTo@Th!>GqW4z_T~77*6kIvTT*79(FT2=EygJSrUy@EzmBr$M#n zdsKY%#G7$8p>DU+bVQ=C&a>-`rBmV@nsI8)$F?h8Z%0~;P)zmM%Y#0QK24tGu;P&! z%qepeg?EE#MJ4fZ2*ZraYhbT-n!7F{8-@|>%)KXj95H-aLl8kB?mxSf{>TXkcH}O+ zNXXc|OM2n0^brJnRCcFNN9B>)UWE6D6RW&H;9M7dC#%tt#}jg40@&9Hcw22jTBD1`C5 z9dq{@ryp*3M}ZsNOJ-#}Mm5J&JtH+PK|z8PV9OnIC2AGSimOF)#;K3T%gBK_-duC* zk#%JPQtLkrF=Ttv`ygp@VE4!ecxRG%=^k?RFlXGkc?<>S`N9!*XV>`cr%aqHYDLSq){YQkU^AP{bbPB}K7S^1_@Qmiy{ zdNfafeC6P@>Ay8h3)6&=s)myf8@wQr3hLD?hI_El3hst?%*rbHo<#-BOENQKx6u#d z=!1JkGU*FH2VBq%(7>PsKonoEwt%b`m>^QSv1q|VcH)5WYGVqJCGLxBkVeXVW7_z0 zGaB?2$wCg3P2La@U=-U<_rRhPcoFGSXfjsAh2yFrnTB1+8yj3|j|<{4fjzM#a24rD z#^&V}VEB5Q(=t3~r9ft31YYp;tv6x}Xmo=J%)blFoeu}BROi7JAqHF?(?Ch0K;l*q zl%ijx&R|SGRJrRWE<2X!N_pb=Za!H!-j0f4qe#&Es`W=mEs)|IFScN~X1CrVPq5KsJ3^ig^ z(tyWjLlI=sg{6q6Nw~}1;tiXW(d%_0mWC)_GLRKU;0uX-&R#7+r3+I%61H%50g(VY zBb9RVv%7dItP;FOCIStX zrJKYzr9CT?jlMq+N3%_#c(cDT?%=j)Y8AmjDpmh9O z?q70&S?4_zD%3efdK=Ou1KXepU71fs>#)<|0@?2->_@r@yv9kwo-enk6-c*f;7+>| zkdhLiix@3O+9C#gw*ebVKKT6ZFdlkPaGRv)-s3|Ont`zb0k4R+2W!DxUeF@? zgW)mMqyc157Ap1Zb=Z2%k-T zOObtuu8x_?Hr+eYqGsp?DtC)nUB3wjawmO~@%yNmZv6?Ls{X)jyONk1RJbZ^cfw&|$>` z*4ZeQQr3punNu8N(tgXdn_cL;?Z9j8ER^qd4#x%SCXK+E)Lmy<4Mg)Bo3sRcne&9v z3W*CLl|?czpdyH;t^S6G!3?9tZ)O0FiZ2S^0tIFChA!WSs{5x$A{bAQq{{~>M}TAY zKIK0OBH0qPNk}O1!Ftr#26NBBRCO_^YTN{NQ20=|Y=^>Lgs!cH@n;vQXi&uOq$JI` z!3BwoBSg4JJChqeTQ9ev=z_>=IBeH(-lNz(xIV4eT5N@Fz&uib!9SGM(7E}Q`+Wj_ zzA1<&VKcg}cpH7(lsL&LnaNLKo2-khnPSI-!2+;rr87Kfo_Hh@w56L7SCT$wpUn~8 z=;60xMC*%#f>)y?Ze5j$Jp=&@z>D_w0`beKQw9m@`ByAa4`hzTpXM7njuCTGTBmO2 ziq{<;g(Fs^JkD}OPXcq^hA~-*O~*H3YL+=H47lp2goL81>bLBWheZr?dqPNjif7HZ zUmp&10R+Hpn(9!WccOA>U~Lcs1P@4(468hT@#5l;RA(lO`5YgMV*F3lJs3KK2R%Gn zHF$_RW#;oUkDkrCUZr&#P}FaqP^Emzj53#+Wfol#8v&0CI&VD+6UKL(cUnW5P&|I~ z*AHqyrU$hZFE~7R&JHnLo)d+%+2Fh-vBOC`@Lb4G1~iT!9!XO69{JzJp!q;jw?HP zQ1eAK=)L&MIBC@Avu&8^$EAv#f#~m8sG`oUs%4&*TEAT?092$(L8)!&zQIX*a z6~8rgPLMrwUP*dvw3i0jfmjAhuW!{kB5gD_uEjK+tEQP9oq$kP7XAu>%tpjjUtDnu zN#@^o-lq=R9wxXK$3%$_;R0oYg5{MlO=l~ZhpeD=rEUuSjqQT)hL_7pf=tmemjFFM z2qbl*+_~lxo-DUotbVyrd9OoG=^Hdty~==!2rp!O?iuK`#Be@dGG#m5!0Q6h@irNc zF~ShnmpJ${j(nSzO|f2^$$HpmWV}5AqJO=S9RVr=KCf`MF$*+G#@Y4(=C_L}06>o` z439&^r57s#6IumDYdt&!55pKPlOUJaLJK5z!3*X~oHuHLHy8JaWDAcIxbm+dHeoY` ziC@9snR83JOV=5fh^QV~S z8c_ZR6FSrhsAdbwj>+3xa=~3?hfGIVZV{^0nc+sd_x7p<(UNWPh;@d|{pzi!O)W_A~}et00Y21m^@jg51Rj z?)vTU<(tJA3QV(XS4!^rDK7TuTm2xS7v6nI7QDQ|=roOyX)l;nV1mxVq5_u$Lj`8tLJ4gVG=6OlvAct6wyg56Xc5s*1ioZg^6NAxw{r{4m5O@~fRfgk{-@($jp)9lmct zjflyl>30s5&5{cDX^%E9%u2(TdU5mmU({G2Q30^Z@Kq*)d5Va1r3(kD4w^y)s%#wP zcH?!(q9pTWj((~e$efB6@YzK+N^Z}L%y97p->|iH{a(KNLp0xA!jVG6^zG|56ph*HxLW$L* zdR9e;1RSf(DqVq_(EJcv?=(O?;yPE=m9t%(n9z)wuycg?DdjwahWdqU-aS%s6e^sZ zNum<3)l`}$7VM1;kJ*Yn7D8ra-lx7Dy-;hx5HO}Ah6P%#8V6ROsKd4C(({)wE7k9_ zfQm02@e}M-7ksBZ9h6s@el(td{)ZSi&D z4v#sY)K<0)D@ubwiU9TQwOJ#eln&nCWo^8#gmQ$fnLJwPW7%N_jyVam7-u`Mv~KZk z@PTZ>Zgjs+M-Rp>oBUAfN&*a-#SM~eu|~-z#Z94Xwk5SoWwlh=*(GgI zY!R?l#0`qsL2OC1fv}qaY@oJQ)E30rs@kfys2c*(T zZ3fhZw1Z_*+NzC#O{qrGCe*8CHiX#|YK@3Cl-i`(RkcfGNwBt{*-L3FWQnqp+9|TO zgti9MR>@7OEtN@Z*-L1tu$8eE)fUPvh;2o(Ce)*3Eu`9$Xf}bcMYT(0meQ@NnVlQkznn2{xqID`i`40kWfM7S&q?u}zfO8%mo>wxl*nqiCXRk+9e( z+NEriwkkHDY*cL$wyK*cEvj2m*w}2U*+H}`X>BF6t*F?AvZmD*(k+o&NLxxxq?;uh z3O1l^D`=ZVR@AmCt*WW9QrRP7HlVg6XsX(V!nQ4vHUY2}#j>i|4S?DxHVJJdY^tr5 zR>HPbY)fReEuf8%wl*SdR1KA#Y(cbE#Iq4?RLn-vR?}fiX-Tp{wN%=vu&t^aO`~X|WUY#AR?;TP zLtxs5%55pKQf85`O^T+}hRI86N;Xw3h&GL?Y>{k|Y=dP@s20g9YNpDj#G6f(O|6M+ zRc#Am*=&hy1+*=JTNSVt%Bi(dY>?QKX>CcdTPkfUWkYJIv?AIhY^d6mY>~BFRJNr} zlBU&DX>COWl6N8+E&m_ zir9l_Ccz_Un^Id9R?URAky}x5o8&$LeWi6^xWo#&I z654G6uvWoaK&_(KlVNQUu@h+xg*K9HC9<0a#ce6IM%37YYNKi<#5RetrpmTSHcD+1 zWhyo)Q)I1@wyLeFt7#3XErDAqn@Vj`TPmvB7R5HITS}(MCc!pJn^HD|WUY#uDK-Yn zX$smUw1&ba)mub1l}g$xY9`2)v9gre6J*$%RBcfJR?6CJgJ?rwHmS0uHma?W z8%t!3sHVwQz?Ri3H5&ppr8bHiCe=-pZAi8wYEs&jHd1Vs#TLm?vIVpYV%ntIq}f)= zro@{>wkFCOQ7xshRkTfnrqxE$t7?|SY^a+gZA95iYExx4rq#6~*htt*VwTjV)SCjf zl(r3(ZC2EV)Y?ThC9qc1hR~+arfRklHX_+l+JxFbwn=QMEvi!4DA=L3D%gW*Ce)_Y zOJZ$BY!z%lvL?c)*b8YU)V7p1f^7wCErlk@O{&@mZ6?Zvu&s$(RU1*Vs@kb+DB7Uf zM7D)ZsT)N$pxQ>zY&NKyD%!1zRkbBHC~Sz?6|#uhsM<}S*p+Qkmcq6Ywi4Q^*&9){ z4U|o)Z6j(MQZ|ENnYoCe4y%#in*}ze z$X3)=!i|L+61EYr4T##QwN0gMTPU_vn^h}nl-erVt)iO|wAia8%ZsZ zO@%h3wpBKTY)z0=v=Z3^XtqeUs@k?C8zQ!;OKRAQV6BR6N;XEyQ)o@9me89@EsdZy zD`J~LY(r>lfZA1CRGSHGm29hI+NuqTY*S*DY)INF|KgG=kNuTZ3Z1|2zt8*qamO9M z-<|sF{&VO1`>%Wc|M$Os`tkohZu|ZkMRS3FxB_zumbZ(c18N+@T8;<>&5&y&tz+Im zS{;j0hQ3)(c{5qx40XcI@r3|LkeIza4O*RcOX6mo98n)uf{enamW}bJSQ?XbgKgQ1$OZZFkUX=p#hnP zOlUc%>L5uh9h;ET;fMH)xiR)r1}rP`oW2%ycD>1X;3fD+8! z%iuu(mZ$4qH%?vc(4t5VwqLfL@z^hT$xVoHSU{C0R%+#kBDm3_oM|xWxTGGD##9YJ z`4p6g?aPYtRjziq(duT`H!q3LrbRflLb}sO(t|2@lgZH#8e$E8iq@ti#|@0EV-`EJ%WrMQ^reY`$9(gkHgzd`{$D^TWm* z<72O0c;3SBM>N0#6^9UImcm>gZ(Z7!=u z5R`Jr@q>n%R#Znq%$YPaAR`17xfa9=!u)J47Mxdj!GI)|eK%f&lrkL_4Ro#1n`p9t zRi4sb%o)jqUL?f}8zOd|#1XrnB@1|&XC?KH3)sPrFYbo|5IP2MoPO&7X)|u~3Vi|r z#gd-_%8)4if+7sW3_lOFjubNemnea1fOLC99qVsaM(@iDLIsFhwg^0Bl(^m)Z5!^xsd_89TfJ}*wO5F~y~V4uN`y+7en96cqwI5Fe6kqP z;4S4^rIK`}bF{;w!V6DG8hp-O1%NNb*kGH9@M_X4&L3Q$m7~eNif+5E~v8$2%9!`v?FV5dH3qKpKN1)`Q@HvH|1h<*dS=oz`MiM5zf0YqO z1V>CViO%zg-(OX4sA?;oG{Gq$JP;$nlWS&{X{6#})L?`xgdidz;RRob2|F8^KSH7_ zG18><>(XKm+z%zWPwP)F^Dx< z0+N=>)O;7sqUKqxv68$h9phXRh2A8Y%NrkMBPucx>sea9N3YB0xX^{AL3<0cIE|2o z>CUOCekov)Mf$`H$f+#^My~Yb9Hx$b$T`<(z1vXQ>JJnV6_^iRKV`HxgBXmDb3Pem z0fkMfZBsD^)U^OaZ(n$(`(^5qjJB;izVC=ToD4<+1H=v*ClzPP8;6s%iPbqSnSJC= z0!VCRa8+j?7MB}=U=tS{0>fD`=aNn>h%s^XKoy4g9h&jZ z?u-{M+_5xN2T^p1LasjNVAFh9ho!%knP<|wR0(Blhs`_)L1J0ApL`S`;4H$0Y6=|& z14N&Eq5wE}u}+*e+{)m*nUM03J#xJzn^n~F z8o~z^V#XR0N%MEokvh0Vwv-Hr_JuSZbkY)|l7$J}VPo1Wtq*!2oI<6&T-Muh#h}Dj z>Js?JpkJBBM?}Q+p3yY~EF4rwL#{YP3Cj1@_i3DFzF3saModi<6BGq~x(M{TLGW}- zh6*|zA!U9G@@a0k{v3YE1?Euln8DAPGg_-RYt-Ez6n24t?uV<~3GoW)G!EvQ3yN01 zF~PHRYj%XNWH|3j1Q5@G)%iJw zE<}|vl+dQKbFqaBy?Kld%$`ZnvFX5qC{lWKuX;sHML`FJd^*`JTD{=YFS@M$XLAi2 z0R{y4hqU-`${)++FHpDCAQ;_!W1Ht98YnQf(tViu*W|_21R(>j$vFk&3iGLhyf`eYESYCmEECOLt6?=h+`?O*Hc z9stDLWK*gUETA&!pm;LRt4)_RH(^sZTRtgj^;4YP$v%fm_*qOH$=6I07zEDKvS?gF z7!-i8$BYCyuzUlxer(706Qv=^+{V;^i1v}hDTk2Xt<|OC%N38{1rR`_uG}3To#&(p zLkOTCqDV2x0fsBk9b*~e78?jX2J9)K1(xFPwm>1}btf6qX#>>lvyf3+V|;QoM#OV_ zEMy3$6-4X}*yA5Mv3&l04aG(#o{BN07v`kUeK1k zl#hB3q0xJB0R0IKhL$E|q(9FLLF-0MKu#Hm5$QklJ2Uu8N zkgv#?;H|hSiVEP;N+ETRl|sY}L+_BJ!VI|pmhTvwP*84*oMY&z!4G^H?zL270|J0| zUHY2b7DpzS-hCl?PzT4gKR0@Zqg)|rJtfu|r56WR(0dix3r%t+0)3eUMt(7DR;U4Q zB()KcwIEG6rc-p9;D|LLxEHK_6jihGO!U_csrrN=fVfR(?d|9I9!3S+x{g~*Wt$Q9 zU`!Ci@Uw(H$G%8RS>jYQuq_99YsNAriS*0NRvtp_cuNn(M_`82nN2#C3DcqLYY}waa?wSJD(vW zK1)zHHNc^(QmXq9V*QumRjEHMhAx&=;^c>5CHVBMJS$IgX3-_NeH38yw$(OUq=Y&O zApB?IK@SNC0QJM1am%i4`Nh_AZNIk~@wnHH+-&j3ZhFrA1t&8yGjEO67$F&?1!E%! z`3z6<69Cr4oo=?*T3@flpag6OB-jCd5VA@ja*(L!$%6y;IUDholmO7oH?i#BLb%d} zlb{S9PB4df;M><6HoR+%ZMNKQzVU49*BpA`chmDY*IBk}ec@2vN^alCZlzwi?0eVFzj^zxKoQHo@4f&44}b>`>k8{nNXl#pS962J<$Kbk!*2!Dr@xk{=BE zpe+6}3+$PRXm-q*!cl;JtB52fhhP(kZaYDr6sYhK`!%(wM?DRFkEU15X37qUNuHK$ zZ4DoDz;L}oQ7)cxX<@qh?L#gB4#fvqhH*evtt3to?ffxwJOT{RudWz$VPxxbJ6z*H z6)$7kiDh(sUW#y#k3b(5KUVEy%j3H;2L;e%eX90S!-aGw>+R2#U*dR#8-&cpspcT| z8GJD7frBw27`#K>B|^mN&l<&eG(RB78Fek^mNUvh=942Rn-=8`XMo*QmN5@O_I9g2 z+aW~?8CScS5$GaE9`{sOLh`E-5>-LpFF0eg%^iw z^AsK@akX+%C{=l_!es-yEPCxs=bZ>jE%qrDRx%XfZ)r=s31|=CKV}*+sXk&nq+GKP zD(EA>r2;jb|I)NaNnl)bGJ((nD-$Rc&p;L*X-Wn9y zTcAHycnjI^76*>opC6~g5t0IkVI4~9{dh+`eU`XBdzx*O51{6O8XDpXqNonu@a$g# z^}Bqa#x8Q@UV=QAv78ADUIH~6LQ_+JH$sAdoQV6A?j|Wa#xGK)NEPPzdwNK2vSlk+ zGdAS3!^_}0UE;lYszO@FW65G*^F>|;_N(Cue8(zaB11}nTaES7(}nvchC~o0uO5D* z){qkIni5bqGKK6O-IfQnZEhs)>0deb!ngyWAs zkTve)(b$B1{vgf~IBr~~QLgCHlpry~32E^RsPOnNkHpoSlbE0( z1>O`06%wPBOWCObn8!JwP7@VrFE|81uP_H}!q9A1GtuwGeQn00gUrhw;=VG(18=w5A56N%o?I5*_2N&>r!N)We1k{>m#^vE?e zivSd7YsuhT(~ct3%e<}7Z@WagCh-FubP$VeOE3yh8%S%`uKNDz5#$kd;_1wjJiX5~ z2nmE^B(jR2=75dnsd@oB+Qx8UnZs#MMjn)GRBUD99t_|+nb7Fk$-KnO2sQE*3T(v9 zOne{#!W~4OZ&`~>_WH9xqNobB3)3!Mq(>i5?@9M8Yme|AZqGCX(Ti%nb7s?Z2Am@EXMcK^uqo~C6Sj?%I(U=5@Jj@TzC^yh&pGoM2y8pJ zigK#IIB8_Fb7LQYn1Mxjs#M%YmI`S&EbrNv?cmMW3)q@bmJ9eFAZoPMNQg?!ro)C_ zv~qVBd1*JIoZTv)3|{#x46uG7NTkAN_#St?mu0oopa?D9+!q(&VP4rR@acNtKqd?C z3FsHdAmbS5t5q$?po&03S!qtk?6fJW0*cKO5J6Q>l~E^Sipw(=nty}A!{W&UQ`u#u zM0NNmefE#cAP=ifXGneM%w)3%svaZa_BeY)V+Tf8=Gv<{C^K-?f;P5Xp#qXf7pyY*n{5Vpr9k5wDis3~Gxrn?pMM^vaGR%VI6WtRgNeX37BZD%8? zk6MW*KFM6)fe@G??Q0yx^>pEpUbdP(E`vh?k`nFvj*_YhE3g6hrQw6F``p9Ur{7bI zAmy36?H9U&>M3+RwS!1ZiV`pqK2d_#y zmjpoEcB_E|FkVYM7b+@w+$Du)lw0G@i=w1wz1}}$I$5uZZVrnZ1L<8VsYd}2uKLmQ zbRjn4=P51}NxM6D7|++WLdT*6(_+m{g!Cn(FtY%-6`)o*5-LDh1t^5x!ff|yDwt)( z%f@ze?06sH`9Xk8hw4|`!z3;+AQlPycq#8Y5T zLh_uD7laTjqYp3)qzZP(kyKwst^I({yg}sQ%mw;Wv?)RLO;tAlK^>0->}U(8LL*tZ zzQ}CzPELqv5ZRa1f>D>`1`1O-m@6c>;Y6P)&4dLy`)~&*F@;>6J@s8>HyU|#<`@TX zT7gL}u;WXm!-Q1vkbdDiwsAkMV8Mdav_%<co!y@&RwV&)0b8Pj&e=NdKFnrNr;BuM-h>sC(xm{?=E^`Uri5VAVKD0p zj$sQaEl}$i5Zi4PQYP+sVHM94U`7f*QjJV0!L-z`G_UV)at&~0rh}5RoKzo865W0% zVeW$Z^M=!M#;omhY|I+?Q-!wMF;K2cb^SwX9$}#c%ujStkUEsZMK~++o7TC2WOp7$ z2vpDBHaX};#npFc1=-1sOCZA5`>@HopIL%>lLnC0*VHd8A=%mDoUgLD!>qh(XdXgk zZdi)ay%=TGC|cgTTAO+R6cnSEcN$PLgJA)QHUt{OUQ&|F6wDmu!xwQMdP2pCg6Z2t z{nYfArOi*H<@SmKP~Olo%Ff8$B@M7_suFTrT(63WekdlgoLhh(;mDn^SdxNZB*CuW z&UHf;(4_>^aG`x9?jXgn!^HHo2ZH+OLW6m8q)VNXG}d(dJLRk}Owy+BH_s+hMK*xd zFA^<%C9}c@5Hr$D$x|Y3KBrEOT{-foln~SHBR&0MP>;;V!8ekdR;k#G%LME)u5KfJZia99vD3*5hGn1RUKl=OIHDT;8E` zBS^}7TH8p2l=gc-l~f?|sLXoyp;*wD!bz9r1rYAR_e*Dzl4ML!S!n_HTBOf3Afdd_ z25m#hLT(oYH=sO3WkB`^p(OJUX(-JPYV4PVaalg>~kjvkJ41W&5b&n*^bX%b&z6@ zDZ62PG>1#*c_C3OFz9gH1dohJ?C6k$2+9ouF=B{I0U#V7)W(%hb|4?Dyf~_BWN^oF zI|&OC>`a8|A+FpNdI?5D!Cep%0`IPmkq#D+$7s=g-wDUWNTV%i@{elRfC}7rJ z9T*U)DK5c-MG*&<$vnZ{nr;ZK8hya@@}ckW|WKXEY+Q9!c@%+?YWQAZdsmRk?L47052dNF81@BCj;^sNo!{ z0%C)jZPClIPiDy&<>#VikV?=NMq-k*kS`XtK4q$nuvAq{P^nJ{dm$B9U}^!%s)vXc zKZrmG%k3YLDE_hd1b$iWwVWWRu7&xg~ zd;#!$A8grv{rZxXpYpwM_{GdWZ55d#Vt!k(E_PXQh0AGbC#}Y`OZ+&%q}bU&ew9k> z=E|ozU|(VUJIG{!vR*7sN=~6$;AA8>iJz6hOh^LgSd}LCtI!8>x6;TS;&=hHo&_&L=s`$7Y+Un~?QH z1Z4a|2iSg|U}(sS8-l8SC!n7w9My9YjQcGssX+W)))(b_In5AS< z^EV{Kz^KDe$zOv`jHlEU8i>_~=fu2yea>1riS*!bq~sXe(0*|No7T!&)Hyy~Be zT!Dd9921kzy--6;<(*IRx<(U?ZEsFle05mH*(kLr?FT!pK|AV2vnoQThPlfcBcF8N zsqhKNZq}dJ7YUdS^=w0*5P%fHFJF%c&jl^G5C@+`HaNotG_sgT!`nRLlE>CjJPqMl zGzZHO69RLcRxnO~P3ipMh*jDfkBTc!6WYC59j+R%%AJ#u^uSFUp%1iC*sT1%DnvBP zB8noW>2h$_-lGA0pwqPo5?wH}_=O&{EG30>fbuQyFFnN$lFZ?VNfgQhOAW;!{2unV zKot!W2#9Hs2ZEsiYG(0<`OuHbmV|0c0*Gf#1f6n&3CQ-#aO(|PqrTmX=NN2iTZ=T9 zLV9&9!M%w8C;<$-7}Es&Z;hPnxRM1hB3xrDRkI>2 z`LYJM^dTvMGOH(i#>1l!jp-Jd7UM-_L##OKeDGytLi&xa7$%Ejq2^nbVPa$}Vc~)l zHF>xZ_(ceQ`iJH~&DTCMb3CH^idyA@0j?^qoT=oJFVn(}rp!Rg$kxeCC=M$z5FZ@|hf)mq-GuhZ$0QpW ztUU^M86lt}R0*PnAg2}OsjLLS;8vO~4_hD}UNLeCh;>M9FVh4g9aY+5kvcRqJ~zC| z8&o~O@48_<1Wdq>1uO;v!WVeBo8o2W7I{YZo1KCNE`5aSMz{~UKAY~Kh@1u-3VVs7 zD1Syj24i4?fqA1xwxyM3+*}phiH#)2L_^PJat*K`*hPEnOK!mz!3^im7+PZ9JY)Jc zBWZx63*LkH94gg|eH5tTJcj-EXCb(2U{m9mYz!g&L6 z(dqbAQTi^-9LLj+?-7&Im~IQ>ToSB%9oHfib(G`=kw$LH%LJko=OXLb*8`1&&XX8; z8h$lj7X%c60M(ePNp1|V0amc^$lH)#xck_!Gqp=8;tU6aD(FE_EaLZ9kz+~vP(hzo zO6@+uk+7Y}wzk3uT&|`NJpuqwP;YJEfgyIJi6?aF#@YK`UTAlO7OXfzJ{+{^8(bfB zpiWA4r4XyIM2gMBIdg02SOKfZzCD4g8rO%=d_Y}m=B%wZxI}I&oH}bvO=o*gh z3O|JQ?YBaHzV__F$|s?m^m(^OzEf^?u^0iTG?TN!hp=1;G%aYSGl;R z9>|lJ2^!j8T-`C_r=E(m+sboQx_%2#n;8$pFCaj+zQG9+mDiPubzdoQj$b8A?BVQr=qCEGLs9prQoVl5Qeb;P#XCva|Rb=-~*L#oej0xkGS zcyh4VNN{yvup&a{?@C3EF`S4B#a>SM{gu~&SzHt31o1^-0g5tUOM}SU_f9C%glv4_VzVH~27hk!9t50JH^65Nc=G3k6gP<52%sh z1b0kFS567(+42<&l*@$IE9EpBypVAo!7R6=-8=CrNJ=~GXvjLPF-6hR1(9dQB_GYt zy-2Ill#6_KD~rWgLg8&?N~B9tqawW*r=4-Ur@VNlfNF9BS8CH`fhwA_rQ&Vg7mke? zi9@vUW)&`!#p3i9DCdXL&X*dATLXZ_vFf4LwWMQUAi@!+5MtFd3?EvX!JaD9%NGi0 zm`g8@ud$^)Y0;dQ&E`?R2qZk&w}Y3SmKb0f92%IHT{Qu)^VezuxDYgGbp&CcIHRde zg)?OS*R*0Fa%7{3W)udq`=*JMN-(U!xog{Tb&;hNjF`{wOb1pq@frp6gl2VV6Bz0dFiGJ>0sK~TP@x?7V&Mks<_SuZRBw6=?V|p zB>5lbfK>fk1f|H^H$E59gfJ%oRE7%Tsds?P5(`n-T`0_~Hbcl=Fc;UDQ5*xL4ky_>~~^4qKc2|Drk?DZBM1=;nMi{6P zRophMG4Y2d+2Jfr2yn2bl|}^GHd^jbCLDle+ay|cc+vB*lbV^lM2_Elsc7 z63NM#rvwRLB^enX6vZAT6ekw!S4=nFyro^H3wu|pT6kWuRraa^%h)t}!1bRr6;jg0 zsRM2-j;;G~g6*^zbfzzoI*zQ64A^vNf}%=?P5B(J)VftSIQSN8SG8o%LYgQM z#0RYI<_ELM73NQYA);H#1$v66918-hYIr(@3_=aZ$RVeUIgtfwT-U4mme8Ib~oH<8riEHAcs6Ng@0nYhw zoER%<$kT-824S{Xc1pMmn!G%Vg?8ddfSD9md*ocmG@A;c*?RFEySclSK_c}z1`ft7 z);d@ah=8Y6Yhz2YH3}PVF$A&>HtCDha8ZOdov`$r=E~p>cJbUy4m2x4Nv|ZgvUhR} zAd1D+5jz6R&?_Jyu)Lkj@jGE)XLx(m1NHD&{>caQ2#hhb z0S=N{JPr-_BwVz;*s1O{D5*mUX#CsOJ4!ohCLwPzk9}5f)>%q&Cfig?j737IysFZDAH7`PiVL?98vN0OspGM)q|*hznX|RzeEWbv*NYX-#VL1ST%- z%@Y~My!&s1*c+y!1QCs|9!-o7 z`UiDw=!HrKvFRLO5T82n(TXN)I+1+S3kW_K$I@TA#5j)oJnJpankDzeg{$U~BB8{5 z5Gxb|hC0FPqWMMn5`4DTELdP(UZ^BTN5KjK9Fr3m#eV_m(zwhAG9t)4&g1V}x#(wk zC?IMmYMYg|S-Xv`Rd*A*i|e|2#QB6P&?ZI#6$ibTFLZ=KY%==64!+UT&rz_hH`7bW zVtKyA*#*>xQucPP^H@UDsS>mJne~9pRHl*%s)5cP=-W*itar!%!j#EezKv zpi(wo+s|N0>K*S2Rvm(r&u1;LV+pXzYZq&c25g4#OBD_Smr|9dz$>{KB}Q-o zUpo9(HMSNmS+2b+J$sKw(;KFAuzEOns5L|4@?H4=FJ+o1ei`~tl!YtNgoJcpo)Y`k z;#1ZQ+7v7At)m#?mjef`uB=7mtuj%cPKH8n&==%akyKej!OIlpEUmDK@}W1mdQC=W zHbUNIl$s9D*x5Z#S{7^Ku^^{Ix>Q!+YLu-*-Y~m~X`u#n%7|3UR~1UVViMyT?PN-H z#~*J|?G0)4w7M#hbXv^bh;#0UMtI_bxK~HQ#!@?mc?Tx(7&>n|wW$+~sCq#FILgsv zB}kN@N!m;Y7Z%*6>yp4Yv3iJ}JquEquRdd+atyQQ7$}*CH6uA{mx?t~L6q49@7*EX zpv&v>O&`OmydM|`m(uLQ)bI~{+4@DZwRit$t z5*FacH%|b{nu5dXz&U{f!EEjt(&Wp*X`%D?1qYz`PDGolSh;Ds4H7H`Jb`-+2p+j1 zJIx-dK?xJJR7;k13v(|jPc-QtAq@q1S%IYv${7ZGlmm#DF;JlcTH?Yz z61;7h(==JCMMCyd%2(m(S*REg(ShfbezUei>IHI2(k>QL06YRA!}y#qgVwqNY6Ljd zK#XO8S{7STlJh8W>PRIaDe3xD6)j{T@n$Us^T*+a>7fvs7cm^BE^b*Dx-ez3C_+vK zG1Y`RkQSyr-MS&1qT~)u4S@yPl~sUOIMW6g+s7gwH5rimnR`XkV73U7o&y%gC)ua$5ya`mP@|-G zN5Soch)K7+lE}-_hYNOTXXz#O7&Mw?2?R9*?^Iu?hXVYdRU5eKu|G*j()Pwu6;AW4 zObFR~nw}!$Qduac6d5iQvZ8?<1vd1vyb!_C!*eHIPzxb`6Pi}kg&Q72;pGado))&H zXF7D%q+{zs2&RZ#z91eAsgO&sYX{=t@;fv)WW&Ob8&o(2GYp{SeI33U>4$Nct8J-G zoo7-qUVF((m??fuvDFy#fV}n4&oNT4ZF==9Z;B6;R|p;ir>`B^T137}!s8qdYEmK6 zgc$>*VLf(%kdDlPi9=JagPJL@?dskG@esFQNm3=LSS$(Mn9@9ZbD}V+6SN_Ti_Hl9 z7J$AMIoYE{<=I8p$n$rBkK4}xY;Yj&!InV%8l$T7Gox2Bh|Uv^%n00$uf^mO23-ql z=~-c@xXRvRh$D=@P^X38pw=-Gj0wW_U#n(IjH7J%4mmUz=L1)$Ft;};IUXIfX5nXpDk?APl5zeCnCMX16I^fB*xEA!72F{7S}GNpiE;Bp|#c&XOhLfuNBP z^#MN;3uSXL*}iL1_b$;xC_cV=-va-x5UyeUu_px68x$U z_GJ-41cl^{>E;3#bKdamevG^*sj~F+u*=Rfa4|=M=IPv(2e%L(L}Ubtrfp%LY@{8I z_!I?{As)9== z3F+$cjNIAhi{*6n@Fg5jfdaF%>IWrK>t^dsjiJ`hZ}wg$8i z8F;-x(H`)wHCE=|Fa#%2((B@e=%@>tv%w+Pxe8V$RvRHNmh-LmCmEDKoXuxLkxNkx zh;KUI5!O)sMYq#ili{#UFb84?Kp2F-(%EY&u{ zqV7bLhfdOU%RJY+uvI`o_V@ehh z>1Ixcx@wjJTI7!N9grOE7A(x=zKC{&4pb+YvsHqeas z6Sn|iLu}RA$m$471C57Yu&Mp$@F{Bhm#s1dh8?j!36aO+wgb0Pt)781q$|rL1hVQw z;BN7t^D&5J6`O|Vjkn92DJ$tCZOFXjp#d!MEJP&(!y2bGq&6x{rc0kmFJY;K*6Ee$ zzJzPl+fQ}{CW6W-6qcyPwI>P)z&saHowV5;<(aw^B!Y<8pldQlc{p>u${r>WbPfyk z_KukuNYY!DQ^Q`k0D#bqZbAo;7agh6nQVP~oUDK~%M6zRJZ>0aXnq56acoBeyCo|+BWpo|`S4;Mx5}pE2jev|VE$a-56&{9Vk}@1Y@bb=5VKGF(#u$}~ z-83duG~3ZA6d^aKj0KEGoi>s;ai|s-BRu1vhJ_l6t_kUj%KZNRj07 z(tJUu>F3l-t<~L)ol$>bxV}WHpPEyzo4m}CCBUN&x5PfP0yNxwVpVl^m>|e9IUewH z+u5^{Uq<9^U?Jmb30)j^4?%;C;E1k`$RTP;#W6UFNMUn5wDS1JT1JmVA{5@#!P1VE z>hhO8X6}uLJdBZLm?Z5Vy0Pg2NYqe8ZiF?Cx`Q!&!>4mt-(vX_H(apRa}1%G7PDa* zwx$=!#vsZ#w+cU7MI&-+%51(`HpNX ztZkp9n8=9=I6mwNzR89^9Tu{pW$W23-@?V6;at!f#R9v>1j>fttS}gOxr^=)O_P z4a-UxJ^{+Bq{%B&V`ajP;>8E5aS{=*i?e}evD3q41;%v1+La?hB`0DECpLD22BD## zaz{{AF6UUem6PCaZcKIxhIzd7%AM?Rz=7>z_h9M!DUp>Lgv}1Bo&4?o!s-@iL^y?gxp-+w)J`~FHrU7zpKr&h0v zbNdZ zZW(NQ$`NH8DP&Z~GamA`Ui!$yAvC)o^aLW1Qstq7Vfv7`oV7T(ms=m+(q^*it`LO0 zEQ#uDI|-a{n*nA&qu%SrU#J6Vqm)kPY`UT%I|o5=1@-1@W7BCM zQi;W{S>L4>6{`y}o@q;xAzIMLQZyF5mAue%W402@EDNN%z=SwcgB%4s@5@M`YCXQ| z^r?_aEDX3omz|spb%2i!$$})r$yWa`II$G9nD@^)#ol;*1_kvg)*KZ=Gcu2ag!+ZN zH&g|tT*Ps70)U9~JgBbk^*_8X%n`VHHf4ZLh(c{~Y~5`j#-Cj?9i#0)#k3|T6DF3E zMqWRwNh)K2AU3IH>oYWzsb(;pjvj!nIK>!D0+ zz3;Vu8`ko5wo;0H&n{Heh!SXK0p<-LKebAxEVgpjY#Ievd<+xTTB6&~Xn5!;?TBa) zN^I#O+efL3$O$eQ8Vu_JnFqwvj#io#TOsg68^^h7A?pUm6f@tDvR221L+LoVRb@8Y zO?gX17~!cYvD`d?XSi^lDvHB9?|Nfh!`2W-i3NQp5Hym{ULFB=&tZWi5`@LT+sJm7 z_SIa;R#-F&v>bz#Cnaxsya|NNxL{yoBBsOy84gN_8O@>tU}OTmt(DpjRh?nr?qZn3 z6T};k(b3|O=P2`&U1`886%b%#7cMqLdEJ{Gx6{+oJ2B>;QZze_5!!1p3ebMx5);s3 zIEFg$V=fh=G%$ZaF?@tnBMQ0%LxULKS{H{=16#G(L{jR;>@x#gj+zue$)s|fgywt| z_R2s3Ty~sr-+J`8hl3e?dKOq#JTn&#vBT0AOqPk*N)SsF9|(k9K53jVQn`Q(%#}h0 z?-o$aCslPaJp|@5swT+20h6?ZZXtt2_OT32OHa&4kXcq8t1 zrkAin%-heiVOvHxfw+cdQjr^t7^8>M&kv9SUu`TPX;>A1G&}2)q%e<4Q8DC1l;bS-137dQz0xHQ1qX z#AUbaZBpq+aQVGXdET4_xIkDCWUZvKbb1e*R!TMqFLSAk8(`+J4R#tge)h#!HA6|J zrgR>CZ+pT*JO>_g36-wKHyK!AS|MNqn=~!F1Ra}&F3Tm+Z6p(bi^pP(@ z$i}4n?`6U^7CRj5{T%V27N^U_X@x~#u8-&1(poW4x2A$GB-{&H?+$x%o$&|-_<$aR zz+q$Y3uz#jpncB&dKJa50nY)7h-V&RE=IE`%wqhOWvs)p8O&kc4OF+y?^#jNh9r-I z^xBT{1*|*E%$bSfX;jxiNA|%dIaS)3C|tm!Vh0sSb+?{egka~X2aic<+!SJtjvC~J zh-iYf8`#8;VB2;Cj3cczN^YoO*O{+(WrIMc#9|ddrd1&{y$9vrIj%d1Q0Zick1sn3 z*eeAin+Rr|5<=@Z>sr!Y;pf)gBgqU}rnReeaPGGn-3in^BKw$9*X%9i;~;Fa(5+_{ z$BN7v;b_{n!DxZ>n4FOxb# zGXc!9n;kyj(HDv)omNi(T0;XBO7V_>uF#**Heg6pQiv_E`0sv^o$N`>1AYnNVwhMC z$zka~k8BtQ+d&zmz`1M=w*0{zVjh5QnRou!@n;L&^5&*hJE&EM(kIt54M^t%n zG))Bq3(v~c!+=9aarht>nuy&Gy-&)f%{@zca@si6ZKV;K`0CgUJ7y8N zi&d7)(_nW4ZmRMYPPpeD$`Qjl0TG7wre~*DPK0YuK%YMO1ST-LRuv3}R!tr_MrF(% z(mB@~X6#(WOL)d}Xvncm!AXA(0l+N9NPC;On4~z3R_V_M*t;pvH`k5pbGYuT+|lN_ z+sEURHy)i?r5Za1Ay@8e@R=6C_1Jc5(z%za)$;U?=rfgF*###nG;7h;F|=pTw~-8r zyRNZWHlpsWx3o)dj7@)T+aQkdONOCnOCyhQ`MMPlOj`O9-o$r2o2VQj44LBO zI)kfa*!Km~30*24z-Q(klZ#so#q@%&(V@jG-!${gzLJIJ_1?cCQB6FV4vA)=F z*P#(JGXgfa<6wV=&D4bT>yGp+>ki%;64Vq2;p8D4Sx_Q0GS@&D7-UYi#T~##_0$%> z4Xzkd(dg2GfT=1B105w7=ui=i$2%F_K9N6}sRh-XD<%4o14jTt3PMehU10vPAn7Nk zHj_(HKB{hlg-$EWUbmybd;#**OSP1rHc@)w&;o+XjiQW_@i=dOAmCFCC-eR2TT_Sy zV(yZ%`sA^7fc0)3@Vak&K7=vk;%$`1n0Cj=-f-Zk&?u;@Su6*N?|EDjNT^M{CUe?M zW8Gy@4bx{^CIFk}tIaGKH&lnvG+vj_2ifQ}qy;IRuo@VcdfKdWE!UA%w$>hf;mG`R!u%4g|RiGE|(^ zVT=)BIciEnPR(uS0eJj)Wz9si#nrKDX2!?UD)JO@Felkr7&=oHHZe%3%Mso|ZC$5K zAF!fjY`+Udmw0lOu*;5`JMN6Pa@~QurQO(;yekscth1CMsH$M+^hkNIS`?14LXtBF zhE6IG@tr0k6vnAEY`6$3LPjnKI>NKL{63}v#f!XZeuxls2n@0>0^8AzyA^rqLB5_q zHk_6T#!Fw51Yq#7@!~<$2S7s)7#nvw3pPYz_lL@KHBBuC{0u)cu7jAYl zd?e~d{6^T`oe;)t{YSALF+Of~gk|fS#8@Dj<21tsLtkpd_E1!D`Pna|pjDNkHxUPQ zfmbbJ44VjeB)*_fAQm{A9I~_qsz)veN=eNrGbaI=rUbW(F0UFLUFVQhxV=m(47fy! zkb8EmqR=^|Do8CrPn%mf6AL`ygFb(R!1a)LL?D;%q7G_8`4pY~rVC|H(ps<=pol5i zA?>y@E*?(`8URIabU{>=wPi$zM|o^l=zzRi-sm50T&wQcRq*4qsxQpWQPEu-{DoC4 zBN_<3Rd<+y_@(SZK2VYH^3m}HEwei&a8Mv$fd?bt_zJ51A_^+_qnhh{Q=<3{Iunjt6-VQB$5$@fhIQAjeDlV$XK$Q5 z_1|^YZY#BxSq#wz3x zqEWW!HS*z(%vbl=kIY3Z3&9WI6ASW#P%W#fEg7+UnM*YuY4L7gJ2bG2lH zu5IyzB7ZKWi7g;WfV(!J88Ni<7o~g686^Cd@ZfupxqPv8XStXmZzcrz(mrm+3gCIU z95Chd!06AjHx<;%WYp2SV?I1Q6v??h2b5;OW$Lt}w;ZpAD*@fAb z(7ps+l?zGoX|1VQ7ja*)tz4hafMAtRBzC|iD|&rIN{e#%dz_E8H|(AFs$gFah6;jk zz#x69ZPfvgg^}SZkxYvk%pkuerFr&GlSw4YXsn(*5Y??~A}9)H83)Q&7Znvz(3v!9 zFvJ9Eh1>4KQx)C#SsxSbgj#3DqRUTkWym=O?)q|2oUm-uz^53Ln4>)cNDElvo?~!* z5)_`Pp*T+>ESoCq5T1^LWou+&zWGCLNH_*nln0k|@g>*DkJU!e&6|dw}Gz#Rp`Q#U>~Kgv9Y9?X@s8roD=h3-WJ;en z@qUD5PSK9#I;mDzRCHTgx8w`4eP1EQF?(?PG>=ox7aeI%r-F3O7CupR)*3U9a$AMX z7_v3Y_5<7oAVhn52bQ<5O|TG{y4lI(Nu_?jhHgUzY0o7k_cajA7;Dd*a7?_rH;t1t z)=pi%Ua%z^%H@?^KGq=XU`?GxE%J=`P`ip#zOmddtp$oCN@DHf8!RP3Y42UAS{<=5 z_Zz1$FP(4@Lk94^1EvB#4hxA;$j8;hw|*9HD<$Jgw%+^%;#Rw=0J7NZ_Wvq~uslq% zLs6nod5&vq7`Z}#+|9iIWg{|zwhwfXfVBgKZ7dSdL$LE`P7`>1H$0Sq+%I{H=jG(C z4f6SIL3Iz5>D%8lt9Y34DSFKzUh94-58FRtdf>enc+EjA@Y&ZPLHk!^(lgKK2GgT} z;!JER3wy|+G22TK#MqNp^}GTfsY<~OXyTPjXhHbk3Bozwd4||gxNWVVwd6@%<;@=y zTFXLPRVBj^J|39XZUhE&@a#ZR88fCyIEDOiZBu49}cCI7{#wSAEs# z+eDKe*gpqw2@az50|6e^rT3o;UunlEj2uRNk6`MIlP9gtMdrr@7HB#O6DykHgG}kCE&SrdaTu=W=uVd2^!L2<8D_Xue6)7xOGvygU}39vkHSC`$GO!{dH?PoxPqW#^? zZ)M{t??p0)euBMKUjli%>_LE3gf|!pDPj&FmO-NThzUt+DWv&$3p{&>ev}MU3=FEF zi>Trx-sYOx!9q+K;L=N-qdf6t8;wyGJ^3V5*Inio9!i{SfqXzR+TDhdaG^yuS&XlR zPj*cRy%QF?H53MjRpCN%NXs&uQmUT7irBYO>$pfVo-}U|=H5oxm(Re6FM{|Wz_^nM zm2r=>9@YiL07t_^Hm2~C&@fS7Er@zHhA^d95F}AZ?!iwt0?=ik8N<=b4XpV#{%*=d zs&2M9;taw&O-3avLdV9Gq^%l^9mq$QO{^qjpfkeYYWa8Ty43IK)m(uF3 z_6~IzloKS?m5UqOD28lY8%&%+(F+$*Ek0TvNw6}JpOTW+ASrf2j;W>j4xAyC`WzpY zdCz-xZ(7>aj8ZMs>4zJqJbq(K5)5ruiYN|uLfRWy)&bJUHx6xUfeQ3|+IO#yYW*>D z5tfKpRvAjYRz%B`xr|y2n}$(Yxc{pfX zI^NAg*U#%T1Pf4%Iz$K;WCQftB|RuJFgVU+yI0O$2@&{Lj5B1fIcJ9FSPcSk@iN8T z0EZz8of0F*Ya!PgTIUVxq;`d_y})#~dLme^#P~Hupfh=KeLP+`zV}RIIL%uwFSjPz zz-U|%l{6FtVJ~}gHtLWl@_Ng%#Z)j9g^r_pN0gkexK`TMMoS^+H#6KX&WnJL%@+af zH$fa*F<{(L>H&w^M&Z~qo3>(I+4ei`9&dreZH)oQPPH^^ZuA%zi^@svm(J&rxu0?k z6R4lMlG2jB5R+}*I}CoG2cy}~6bU^7{~xnLjK0A>;BOO_W?48Se-BhA7pK$C9hl?{ z)(M&-k_BI$mRm!4{VOn1lm8~u> z+&e~Abw?w$$w$3(*~845+4k3`EC{W^#SSzjSkejknU@biTZLO@dlZF^7uJwq_|(KD z1{uYaPD<5^1TIH%c`kCe9xF-_yhAyXBNWm{T50SmI9=V*=p)ML`dGTFL!}H@>1LGB zL1bd9Fhi)qUdb3y2)Gj+BXpHbQ_bWVpjz?(Hh`Jd!waf{p$u~ohP(@)h)4Q0t><`chBl_`$l^*?5eUB{OuFLK1J# z5HO8LP^mqo%b}R4jSkZ8Q%Y)M3smgaQFtYlC`qp;b*EtQ5Ym}@yfJGM1Pajuii`;D ztwIe@fTxuy%RYcxr+0g16J%>A5oZ(@6vHryQGBWpJcQ$A6?eGMd8hAL*j{RcDC?Ho zx%l8ma*}77xlD&yh}3a&eF6xHGHcAj6KB?`$_j)SSY+(J(3C0-&8D4__A!KJU5gn3 z)roauM4>~|ZS27L*Wm|oT(!R0tU9gE23B{(@1u3Iq4ly{Pgkk~>R4hX3f&t9IM{VJ z#R3IlCtzh(Zu~m-=R`w679sByKwa&agGam*!ryp5ryUTK6LC0T!l4+ZJ8Lu~`%T4@_c(5; zYDq$(yv+kNmA+ujGs=YxyyE0Wkado7S;?dnip{Wz3y*9V`rN{tc!=>3z(kpgacmHB zJQ%Lc!)YnCwi>4g##e^gakk=I3g@_FmN`CBHGcS64@5dLR!L44l~z?iSo!*Mljd|Kj|L?j%W_~)P4&)Ys8zs6(qM|F>8}@(QjXqm*rN`s zc<(vn2t?6w91yKwf*Xo)keS`3s#QZJdmuR3gal6*Thy{yDm8_2^T~-SjEl+!yVzdI zc@RTu9VaW?76y=Z9SR5Ka1iMngGF+nQXWh@;a?-mtLGUrKcUe?#ccp^XSEc!opEyz z!Gto%OIn3^Zb)5YC&J5N;}_(vVr%l^Fv|Bx7s!O}YmLzQg1hKwh_oKw@YDgN6?UNu zHFX&ofw=5gcf~{Rke}KunFq92v=Nvgf%LD6oRR-Xxo|Qw&6shKELU!5%M2foW@hq5iBFvK6j;x= zpHlYPL3q~WGs2Kbs^d&WogCL&a-got7N@u|O2x*}IF_L5PSwJ*Lx5a3>8Wc7A@%sT zCOX4vvq6q0*N@uz+`;W_xb;|ST3CYdQXZ(NJqf?yf#O03>u?xWt1q999O4qd3RHG(o^-q=OyE1gxe# zBlGXf0!(sdZ;7}QYWY2@RNr{@BhKHa2X~Yk(RZglRlh zY7?^)N^&9cgJ2gQqdR=D9(N<4m&(h=^5U&_QS}OxLFMiUgQ1USG>ftam5yW-p zU5#_vFsNf+wL_S)6K@k{x1ZW86jB7Y%&e5k@SwI=eI$#ej~G|&dmo7%fdaV)F&O#A+b2x*xM_j@v9>w&s^{L?6w^2v#U#XQ%_1l# zkf6g^*{zhv_UtqWAvoO_PZA$j&oR<~``I%G)IO;^m{5^vz+?bzy5pD#!jQTvA74|j zyQCHC>Oi{}&?*8I#BEA7R>$tsk;B}(s_#tGt~+^_K_<3&omUoRy^et3i3(Au`1rQ>VpjwWi45Qh25}~e7>G0G4smv+N2QH3*+pq zjuh~-FxZNX zlp!C@MX~L-(z?LJ-csi>p-~4`uSmp)HezBq!gj=@5R?FENem5;6PSf^rM2O3f!|uY zBhqyJ0Z)UFY%gHG19xE`)iE!C_&(Cyi)~ z(EPI+arXfR+pR<~ySH^;3Z)X&K9LJdAjAOXJT2OwQ1!08^)OcIXCQ1*>Sj3o-fp3^^gj!q~Sa@=; z1nCqeavMlW(GGJ#11xZ`WP8itq*K;0TnV29%4-uYT^!_?2x>`&D0Uw5GKBsXY9{Z=eTzJrB1!QqcavZIkn^kVjDx&Y= z5}-+8POEVwm0l!pfMrq}3mm$#4H_IsihENavGAQxks_E0t#t9(rJ2pbY>Hk-18h2Z zAg+#!A?8v^DE)j?;I^PC0V9KxWLs!TOmycy9;`_cY!pWXIUB_5CKf%Sx>YOSEmpGC zVQXUzG;6G8z!Oy=sjwFUi7JC$72WMkCIrB*tP0XI89g6>sXG77?9y?|(2>860KBjIUj7@i< zOau>8t2qqtp^U{~KXFd4k48BFWSr>G9ejGdb3$hGquN5ITQ!bE=jv#e2mY~I;b3)SP(AGXFob+4!lx|w> z-65Uwu}{e2>2i{|x=N+AYfUoZbernk&XgG_cqbvKE8II6L$7a{euBwC9FSR<*8$X( z1%Z^F2l~gIqWowmj~rojehGP`MtyB6CdTRL4@5a0sa;#AraF?CLu3iqX%Nrl@z*%@ zmi4;cH)YywbPlFq&jEA>q%AQyQ0V_RdO~zo*OFfgB^7 zMyG_6Qz(kVCpls;Hp!AwLpJazU8`g)%}t z49$+<@(`F?u%#7?San3#%D@?5aZ^aOW88Bw4dKwkAlx!kFuZg!?XrC+szFfv>{84e z7%BjzostFZ)wUya@Ht@Qn)p^^C2Gv?7n{)ndU%l#7U4j#3tC5fQ!#X5KZG>hYu-Oa(#e4?5 zoe^4A8W~v;&y)o*RhBv!frRXl^tk;bUoHWC0t7HGA!gJ@KVMqfptYqqJ7Z#$6E!kf zVvcR&&Tg`r{c@d0AZHQm)9p8@0x(TGo3yXYQN>|@wE zPD9UdtIm!Zy8*~*rziutRFDs8R_413vEjIujCl3499g#|+X$>;fRQn!f?l78=fZ$C_N^&#N#?yhKi3C=%1MH16AcMLQ zD8u1idN|*&q8Ep=#MMfcCqSOXTQ4H#HUdemLKP*WdYh2wlILxOtzUXZGoSjk z`e$DdSCVzwPfCJ$B;QM;tBf8)_NAXaf1uP-=ASaD7>>~zfbw4Z(3jJX9 zEz0dgLNl;y4!T(;`gSBd{_$i>8Jm^n-CTpX zq%MWAs*{y@tQtrGLK+gVUlrL78`9hh4l<00JOP_LU|98cN_|;a>pf?F(A(Ncx91o> z6vY^?0UcTD+y^O}$+k`-s_jg{!SRnw;di%F$L zK-&7@YTbj8zyLwuSw>b!Fho&d%@-lK8j_S~n#}Da7m8fGaW)3>Te*XEt>hf`R&KF; z#Kl4vI89Za%x*2sRy9!ulnvTop$iWYKgHr0;~`;I$IRj)x&f3IR9X&f_A?F5g>#Od z6gam=2XEEuTQbq?nN#4faN2_j7je-%DMDp+gQgL4_@@Tzn*D~CH;dAUfIu=kCQ_hZ zau^76FzDXqyzPJwET=*5qq~b$ihh(eA|XLxLm9rtEi1uFz51|W>qQW^ zgQa1;=v5jWu*Q`U^k`H0jw zt+mI{j3|6Vm+CRd`67%h3Wrw-dg0JOYh72ELb+5IWW$eMdKri+SYx#s+FtRY@2nMR z8sz=#9$&4wVzeVrf_)gyI8oFRe7`^+c1ClOns9^(7O9~}PUZC>OxW_Wu;;Ag-@bq! zzNzg8U#0RzM+mJ(DeF2yD{H5Jqj6Q9~FWV*8+T(z`Uu%<CutoqX$y^5I9(ykr;#+5y*@M zYssCQfV$AsXI)XeZhl%Qd^a?LY>IE&ZUu%IAhDz-)2QX=ycaU%UrMl8!ZY8Q9__%B z>Ja%thWt`-5({SM*Vvd3eiFly&eylIHOJJx4`afM06au*^hF^ji+ zvpAhq=(t-K9g0oVzI&dh$1NJe&B6u9;3}emol=d&ef30_(srn{$;_DHNlc%|TpT)^ zi<*JC1cKd^Hy{AQ6{#xa(jbYbnV6&IrB90aXr3|0B9Yu48>=34(xfm!JUrWapRg5p zDmYQqOCa>Aa7w1fj+T0H{Js_-j+==mMBc4Z3L!lJFNpe{Ynrq@=;!+eF4?j4!c7*bge$QC)XkLEgxo0 z5REIYx)G1+uW8^M0fnv#ws#}}H<%W<$+NySZImu5GSk{u#%^SICIi}mKWj+9 zF@FbCGA3J%&6CO&^M6!zwU;gBLzlu^)~d;LC)jwM(D`wSpmy7NdaNMyTe2U%B&^u< zda~D&jSQMC4Oh`q0vYfw52Knsrf_UfsD@ZS1&N)&f=XB-1p&SP!m zR)a0D1-MQco@IfIykcH9UR|Ik)9rac!o!n!Yr2s!9vASIPRU-aygpWExFg+q^IY?Z zS^!X~me-BjBnLs$934=`&jsyit{p&2)-D>f$X&dAEaB=Ouu%SD0u>Yx5zrE;woa6X z03&U_T}3mZ*lf)tRoMa%y?Ne+y5#uY&ob)g;3fcEphfwGB$A_j&i&E41*%1xA`J_u zwR2r8w%Vq}M7x-Tr+P39Tt*3)?q_8g#%D2QB8Z>ly+uB(}x@{xmDzfhCTVR2DzGz9E`7;%dr2!{N3 zn8Jv7V6K6E!ANK8JQI(T^EXhELXxXHLz0K+(9W%L`MG*w%1HcwL-HM=9=?WFe>0qN57^5&%W<@+B1$Vxj z1Gxz?tBEy_DO>`^7*;|Z89j#}th-`Uw>+C+jw9*W5@R8eBV7n6sT*|WOXNqQEJE37 zcy8|S;E^l`>|l$`QaE# z5P=?A7<6dD2U!ga{9+Sl;E>m)bD4{q7#0w-B+%m?iy23dfWzD;vH=AkGXz>1NBi0* zW~nSzrRF4GJezhd1Z6$2g<->i1TA|;vTv|$V%J(WPR**GAbE#?gdoj*1tAia4O9x^ zyu(nB4e3e{HWZ7dqE}}6P6c92oqM9hdG=vKXdE40JdpX*N&u5>C_ZnS=?UglmQ}}? zh9I(635NSJ-uus82BIyIujsZrpq@6q>iKH2C5&EoiBTatmjS{Y3W}V8(j77BU4tfy zO>$qD8L#L$CP@{fOv>!^-Yqc|utX1SRaZ4lo;(CR`jaw%wnI`xCOS2q{bQuO)Uc#Y z7M21Vqta*&Ynm{2YKZ<>$T1AkNc2t&g@p=OjzCaAU<((mEklfB9A!W)vA{Cyd{br{ zPbUki@F9u8wkOV4?MhBbnQnEQ14n;OU zHGG$(mbSP4<=3deb0vqMHavRE;jmJ`TW1bJ5pr%31D9mQE~o zjmYJJHif!&{F!92a)K2{JEVCyEt1zU%d(p#;!IPox*NX@E7-BHX+}* zIIXuklo4=41U=XaMUriAR~*7*GI3lF7wH7tOA1%g-W|$3wl|*kLYPvpl&II6U#(Sw z{O8%$0pQIljj@II7z2oSPsJcY=5lWjBO%Wfk_&r2PF+!FoQHV z!upfSU>v#OLVRa?Qs$9D+*uvzz9?^Ne6;Fp>xdJI?=Gwc<6gaELU2>K4<(+pduU1{ zb)_*AHNvVJf<>0&06@vsRrzb)qvfe5m4s;163a7$E1NNKEZ*~F_ui~9Um<~p6%Vx4 zn1$8Tt>iJA)p8FPrbzWQtFncJ6d?jbK$c$bv^gwR+kk1KD%bh)4+*jSlTbeh-HLR!x@iNP?? zQM{*0mwi%UyWu52git(j$hMqMm^wQF|sY@+ zml|R;*QOyky0H|k>^jZT`6|*W_c%iWzDXuj5X|SkSv!T+wuMo!X(_koLaLaG9d=V| zHXVd{l0Zgfe%8ZuOv@xDf}{+6Nj@}RImp`z9RiSjV03E*Jl<6@_gA8Z1-qr0p4Xj7 zIR3Do2vn{nI1DuU*qri|I;r@|9NQKk7si7ZxoIw@eM0_f>VxpXh9VeH7eOA7;M7S~ zzt-KJnkwL_qLAl)B4GtQ!wY(eg}PG-7c50IHxwUo*bIYeF=tnYXKf1RJ0l}Aq+VoZ zA>Dw?9Y_)vNFzpDIv}jl zLO#G%Xtw5`Q@|6`1`41Y#D4?AUQoPz{N8r4PJ;(SPhlU&_MuROrRqNtF&!gUCZ*SH3_R-qR-!*Ls@aNiF=5)s%X~2ae26ZRFQvlv zSRtcB6NWkkKp;{NiiM|#qbjQ|n-`}^RxN&k3LGpCXJ?=zTWC(=HrYOlsBxV#-+kf_ zU+78P;p_64Z-*UTy+PG})r1LbXUq4JFha|^3g&?V6LS2&XEH%C+^_~&rf#Jn$-N1} zM1s*n6snX;kT`KzrHO=Xy_la6(;>Z7BrwkyMJ=aHGGrT%ISQiz0t6&+-cvC#?%O)o z!XOdsEK2w!KYecWp*X2!M+T)rKz*isx8rmomD|aqyY0Wy&r6t&?e9 zf>h>aj}Ry+m2@%P;}@tii;imPz0sMUM*1k=$^&+y4YFM)bg6B?6*WiP$_rriDD6n~ zTR8kXx2&KU>j5?fNQuFs9Fx@KlHmwlfvM#!ycQB2rpu zXhJ#Lqpl?k=awmytdcBbF$l}rSU(Wr+4r3JgB8xa`U1_e2%#xHAaG&tpGwr1k{S*% zdKBtwSS~?>6rv-QuvtOF<3K0GQuz-(q7cKKo2?O^{lODdE zo_bW^Ei=|iJv+d=vk7PhZr2XxI<=k;U3rFrObduuWy++fY>zvs4v38@#SDPk7Y=L9BFEcPDW+k-Y-!` zE@Tiqj#YD7IvsFAvh;{u5gW#??7-ll{c8pq!=7)RYoqIQ`-V<-eH5fXh53h%u}GCO zlVZ3HR5}@esXK|W>JDvHi@gWkmse1AJy#4raWOtC;RN=?sss2}X)@|kn?nmoOUps! zBRlp)gS}wt81~ck_Z*wkn7`dJqIj!a>M>F!RM0z~;QiW)*r?j8>wvx`Ilp zV>D zN+p1_*)A~n5|hCCgd24Q6|-=?1JblmS9JWfRF!h~EU+o!LKcrrvUpdWfCHPE0oc9? z4D=c^2$2Iw^t$%>$%3%EmQ+?cWCuqzgG^_2lcOIzxXwc$qgzKx+^y!1yL(Aw6lP{- z(mvop%VDB-BH0#FlM6a75#Z42J@Nqrhh`Tfw9+6rqJuutbH6G5@ysqYUkg|+E08`h z(Sc$EsFx9?K0Qlaf@q#@5v(B#@hz9aLb47jTR}8hVafx~f)!r6ww+-;8?LM*yVVX} ze({H^qf`}7gOtm1cdjj`#k__MXT2`rE>vYaG1J)CB=M%M<`aOkt zlQ`CKs4Oo?OE_V73tN7>hcCh48uGk8^}Qby9McnveY|Amvym$^q-EbO-QeOmu#C8h z7A25chnrjFNbe>g?jI}8__n_KuNN%B)88T6quYOed8hv10YHWXa^mi4jV5BLk7wSI0Zu)tl^vHZLilFrwL?W zg)*^1l-#?cB$T*T+RFhH!Nc-sS5+k7AWhFJ5SWGtIiMja3^Myxc zWQ_3)gkvKlfvA>)58rYZ(Y9fZhP21$FS6+V5deXaD?<<*2?&u9x0LkIG|FzN1_z6~ zL$;G1(!%l>uUo~5)uGvJo{yz{-ecFNE;YOib`K0#V{aN}`UQIzTPRpqu+@;lay32J z5}#?gQvnh4HDsNPf{+!u!;EbX^-tGHN>!30{DRYxmj*mo?5gCa3RzO7HZn2 zw|HOg0P^hg=eWY4IGgdJk}kg0rJ*Nr0xlLiR#Mw37u@IF{MFxV>v+r7?dSKMyY0Jo z`T6s&E`Im@ckSDn_XQDw`u<=C>Pu$^F=S{!gD8SE8(EuM%S}{%A|{{PkiIRT((LP)eaKNjC%*u4Bcdx73LE?`{DZh8{`u z$gq@^nv2LN;0U|EEG!y$MNj2kPatR4Gkj}E>C&+Ra zZj-W0qIZ1vYE(h=KcEyVZ9E4YL1Uav@;|Sp(uCtz3Bet#!%Vb}{qh!j!X+>@mPt9t}qA)P@@-AUZ2J+*Qg$!Tddl54XitHW}48n%=lzRM-QE$s^exu$DW!gYl*+8qz z;e`(vQMys2FBfg+ss$8vhp&FyJnc&|IzwP~-SPn#fKmZag&?tjCQSP^m0fM`#0Hr} z4Ji+Ihh##Lov5~Ez5~RFl*8RKvH;PcTzmw<`Fz!K(ZvCDv9}U_O%j(RjyY!CcbMU>Y>(Byd+ZV8ma*GCI%pv3=7Vm;`0KiA@r$=pe zzKL=cmz4vg7#VOmK_6P7=g}z0Q+Oi0NFW1>W;WmeML@d0Ko=wuJBD!@7=gkf3t%K7 z3{re?2SoU+Wofn+DvsGNVkB>xC6=(R#9%T%GFDXstpbnMnSV*&JhUNLldS`DCV7(R zxVxl^f)nd)yCvx34FmB&Jfw&ShX{B;!5pwCCqm(a5dI?(`|+Q=+MXXn-&nfLy*Y2h?V)1mnanLBW=2{Q#YVqSWY9 zE6lzeT=6Az36@#WJt;LF(D8*}5O{tJ5gk&VJkih)?vsmuan}jpU=~`lMXo};5nCdu zjrTEARcPp9*mmhXv|zqP=Kx&;rQqe2_f)P(L=21RWuA%xfG>8bTOAR>7GHx^dXE+{ z_EYYFP(V%Vst_%62?X)CiraCqrR+Mo0z+`fN+M%n$5~Jcfl86=)x`q+9F%?}qrQY& zOBm_NQ+#XR)#~R!>XK*;6*^8IVrU~7$T0cwU+W`SCV_BkgKv^ zEc_egKFSKo3lV8OL_yC?mGv~o)@mDdR5L+l{oFKy#o)ibMjwOR(as>*A9;cmvJs}C zXM4o+%(}Hq&};Ni7^5tZq0qOrVB#0nZ4KHUd~CWag!3<&vS(B#(31^(yMDqt{VIVK zmDK4`vrqGJ$RgC0MdT*QR#myBS=8Ou5MLHS0dy%G>ebI4WS={RLL8M?wl6+MFC87| zmUS7+NshNS1OaCee9$RD5EJc@(;-p{MudEP$X2d77RVdG= za6IwKwuY=D<7{>V*H}A|sDZ-KrUI3EaI7W?VRcbX8((VULN378mr zBV^G(yG=liTaS^F#L>iY#j)_4F`tL#-^46zowwGN)kKlz%EGFvzpgo$y$KI zPq@7ZE_Ua~cO%mMT-I*ohn8U(b@o||`G9!n!%FGB&MIaSJRrzlNf0j{`9s11eC80A z3zR-Z)ln8yI`b%eFV2BbL>$MWX&Vayq@mVIKp+M8VYO!~5}1G#4=M!*Uv}e(BbC_- zAubhzcVg;c%|@cd4X(5|g!1(;l<#!=$p*^CZMP=bt94+40$~iv)xJ6w_7jk$ zc8umA6FCDvuLE39qz;W!H{0qISMd9?<&_wGs7kL;x*i z6>#*?+<_)Vyt_f{nbMR#Kz1ZvOc%xm1AurTa3ZM4Mwzw{u&{uXQYC^jDk?JsYO@Gk z{$Q{|?#nVsFc}+Rk?e=E3N~i&G%lO*+I?yv=sUh|h(i>B zsDhSIkQik45x#~FfQJE@1fj*<&i8P^n_b+Q+tYG=np|pit;e5TeciXO-rT+Ktnt48 zl!-6zaFT>$B3w}_NGKtSzo7qgyvQ22L;h16dXIYYRy*@oOc7!nMg=)Chqe2(iG+;I zJwyP}1|7ZH9T2eE)3%HV2_vwlkd%s~H-{McQecYPGitdZT!e8UEz*(UZf{>3&$oO* zxG#su*Tcu}z9<12kTLS=2{O>ON(X>%x7zpir|w)PSx*wi1LGdQ(@w?vRXwG z`o1OfajeZy;Mrh`kR+$4w@63D9|(Y+nYc6O&aV5h_MFd9qbuR& zPZkddrk!cBfvLdF9jJJhw>@bAgY_qYof9g=X4ZbTf^`(R03}fGI${Juf?9<%JudLq zX`Z{)QlAKHq+GY2l`e7ESzKscP1)4_Dl z)vipE3mKX`gnawKg~Uy8gqlU75Muq?RdLC8>)}1i-S~Z^Y3Hug#R9yii#R8c{VF1g zZ!l+=6%ppE^QN=dxIk`f(lYt+)>+FHeTsF&xFwYhO@jWF1F;L(a(wm*Y)PI8)7M-s z!I)oZ&9#RH`^od6x?uc5!>d?_1!Qg#x)?!>F&ZH6YGpi=w-V-hfuaHk!0RbI+Rp3f8oXg4W+1}vwA#w@^r zAwn0k9j|IhOi5ySfu1G9JB|`kOP^>sX3kBc_nMVX;=}z;I7cW*ItbII8;gkFyLCo2 z03x7K=XppR(WLD}=RlQ0fvIiJlF&_hgzmfYPnin1h>Kl13G$trX1aJU1#cFCQfUtP zVV9m9vDBj**G>(jY~oaJ?W-Z8+E%Nh;8{TMHnvPWDk;2BKU@0qL&`8iu0`e);EPBB z>fw|FHZ>C#nK^ZdGWI!OlPi|WIga;BU?XAi--Jj? z37(w9Z78KI@VIDDJfeScLR$_Zy*L&L=ga0UQE3D7BFeU&%(679mbeusu}4(A)VM2h z*cF@{gcVM5Ae_Qlf?)7S!sC~VRIfGd8u!0j;z(h!Fhj zpL6;Qa4ojLOB<+C0|S|Xa|m0oAEb=DK;=*64lSdO%%;Wlm$b=kR0+A>S_|wHC1OF@ zgb>9EhT+Cx`T&L}V<%z2mvHLZO)>0M`_VUarWy!j(m?^7oo1mCa_vhFv+1&qejifuUO$WNbHz8aOtcUQ@ zC#Dp3Qw3TmGu#IJ;o)029pX|dlf z@#3T7M3~2mg5Izl*xI$(OoAahY&vaoU^%Df(H2yooFf?(oZB3V9rA}}8{{8^k*I>5 zkRYgXhkQITg_X$c73^;%U|NO_y~*v(G3v)|jO9<>H&~K1Y?g5*A`gWVoPNP-e3BgW zD0mZL;rLIyd{I)D8VoTBqO<6x#jYkH~?tf|pHEzF|3Bu7)Ov(qz56&2z^^z?;K z3xywFTNhBd0YlG+_KN%@JBR|_1vT8(33_KUs*Gb{_&zuw0%G$<9IX%1G9khiAmg&> zGo0XF%w~zZM^rK3$OwY&BWb7$;A4RRLRnx?vGYLfssM(`2^^h2Iw5MF7-1%^h^sii z4znZSU`B<}+-VQout4M8;Wp+Z*0%S@cu0Qy4T7XdEf-AS#Bs76H_$!G=8B!9M28rzoTgmM zS_!}$U?dzkrHny?ox~t>QBoD<*O79Cd9jQVWbk>Ov`f~TV&F67B<{;@ZS4|K)oy|8 zVAsi;OCD;$h){_s2lJt3!iLjQQ0sH07V--cn8xNghMSw|#l*!O_F!Ntn>$K6W)-eH zKaP4PRa93`?p5PW9b_^V02rvW0O1A`AA#od^eM)xmPcNJ?yz#Y0DFG4YEIbX#G6p7 zS;-3oqvJt}AQ}%a72wX|a|sm5vLmjIgkKipO_nTRLwpokY_ndlv#}Pw3Q3jw!VeO0 zu$oZR8Gehi9u)%QdBrHs&smZamBSt)GtmRba?tFqjPhXZ3b(K@8Lx#nGQ)-ds`Ny= zR|#>t`(ASJ#P!fmE_IH1q>ShYLj~8tX0$59@|x8`nJ=COczQsLI2HAD4FfXsPWrJ11GMHSMPik#Wz0kG~thH~R1J;X9pK}SGceWU}#Q8>bT!|y2Q zlrs@pT49Bb4Go_zJ$p!lT5Sfrf#7oaEl}y94WUu4Sw>DT zZJg;H0JACx?yVh_Q4~64KDnyoU&IoYE>Lqc7m=w9-0(hBWFGNajWfHor4XbmGbMp#ej7e-FK^4uRA)6q^#gQWtv5nK|YaoA`_ zf@m=m!Q0C^^`^8SJ;;qAjn9sv&!kI)9dcQB^D3g^o^0`PWazrj z+fW!E><1qUqvDQuQJZWqCf`bN8_*&O5=q-{>3t9gEaS6de8Vy@P++Fn0SSh~`->6? z(g;e}9}kfboX85=bF36q5wYjDne-hjA#7Cn1Qj?21fkQc+y_0hH!&+~kt;h4XuV&h z>Qd5G6ptsO@)&2Fe4t8CjZR1l(e<}DQAP23ywPs)j*0V%*#QLwsz{JTRVGq(V)OUF zKY2ofaxju>^pSMN_Gf*4Cn?ogiL_JNU2%LB1c6nr0MFEk?sC6wc6eSQBLxlofKxqleLC<|PZ{1oorGOR8$N4B9XkuM<)-f_v}+yWsNV5>zDhN+iyhZq7U5s>kQBwqo*>yT)=BW8DWm}OxQ z7e!ckk?=thIOz!y(tIy)7$jUt*x4m+TbpyZ@t`;eDw(+sEbkO8NL9<5-yYrS=!Ff? z5~ofR=-3$^n!ZS82NOYQaQvHCb)6>S6vW*NFw2Da*hn)iLlzHvogq{t)|~kGc8BXB z^)mbim5OrW;w~YggU&(Edci7<5*t_ut$EgCGnlY#2C)ttmNHJ&k6UMGp3hkY2Rr$t zZ+Z3*!ND01X-M?-`ucTM03b|5R0@^uBI$CVJpSyX8qoC5AxD}eMFWo|1B(tp(Rs+) zr=JM&QA{>ws8<5(*27p*7Lj+N{rN0)Y{IO$tc}IT(pLO%g;)ejJ#7u-zZqOR09=qw z&rZ^V548{EkwUXWP*mB!T}h;;O!OaF0?>2lXZT7lvZo18brmHK?Y~a#*r3X0BR9-m zys(0|C0vsNH&_UPC-@5E?iLcOZqd#TkPlfy?c6mvqy+llZbEr}6(lbqhYtktFGenM zAg27HEA555>yv`ju?vlJ92%V{QHauvQyQhC-DP6!Nwfs)IhJ+5u9-Zl$O8%pI&=`P7>UqLQ0iG~gY-oWy^3cS}P1#@CNx!_g; z?gq&>8&&ifY)-dVtIxYTTKX)a#5dDyJC>b`>c0VqXzEO%7o|vZg%nm06+NA^5LQqi zY1rvYj8dm1gM>R095$lq=Y^S_i)bzVf$JWkD92ard z;N#M023^VIAS_i$=0su{h+uq=Q2~y|=c)qVZQZnYbX&_D(J!E*q_!Y{6K7tx39d~> zr$AzoI<92~N>y>mFBtifHWEXxNo|IiK;jo%G#gWs3DFO$p2C_a`r_`aC%G+E8nE$! znkx(3wbsktj3oh-!gxgJtRRN&!J4og0&rO%DPWx1fhPLyYl@Bz&siRD%SVo+!1COT z_H9>xZB;McE^BpsxUB)6lc&oB3{!qA8IiWI2n2`13^t06#w-P5A2S@0sM6mCF!p&h z%DVUC`>qZB-<|i4et!IW^WOK@p8WOqI_=~9l1jhjB&4eO_ut+*@$1jW9z1`$Z|}R; zjr;5KulM}**In^KoZw3Af!I5)ZB1$ze=S$`-@}m8!rx#^mfft@D1#wf?umrm8YCSEl_&2Qy}GiZgnPh5}l#j?vMRMXd@SDx+ zh_kjs#bl1|M_?_mJd-iAeTPOpfk|IP1NIFyIuyPRnPmBGep`1CQOZTr03-+5iG=45 z5}Xwkk&3V@Kz zMn=R6Y{H+$hn-iigW)25`7p{}d&%k$oGiKP&i5WZYN!>Fz?Ka zrc?|UUq?-MhSvaj^xlIfwHwc78JKd+cfjB`48jDV_m}AeCX$!n8<@gTqkNSH$yCje z328!#65LP##k`i64y>Nez^P<0!9zK>=hK$*3maoZvPPiNf}pxHi6;PLT}<7r;Yre9 zUD1$9dP5;B(gL6ieR?vqUE+O8u$wli&M21CiKm-)R=wP`$KHIh;j5w@R2|$#GOFl@i7e6(usJ>&VkQGa8_C`AqUchyXxpl#*IFldrVkOFTLzY=suHCxsPShE;yMBYx*bs2_#dyQQ1QH-dMYMRt7tX2PHPZOU!tq9+=X9l{ zIZXo0mheVIz`PTGQS_&hIb$MH3(wHf$;>lo(5gk=A)ShH9-Imm#nMZzMBg3_ZL}PN zTA1K@;^j%xK%E#{eu?+Hu}nPAwrC~qDy7M~728Z;t-P}{AYW2-G5K=>Oc&8gAk7F} zymx-U#a!J+Bqpw$^1!qR6B}w4=%VyKQxJF_dl$1ne4WZ7Vv*{QCTWz3SB%EF<16kX z=oZZbfe(F=TPwjn+qYNCBxzA6L^;}5>Ik%J_84kmQs$%hV%xynUu)-$JDnJ$QJr5Y zDN4$n1V@{pj9l)BDj$vjMIbsJd!oBBJBn9Jf~k>8GHj3;GGlEv`?G?QNG{eaCLb99 zYD==A#?YKm23SF?TH5NNb-kkT-X3bWJI5 zLgz*p;jo8?ed3kTVVY%08XRm?14HXymS+6S8YBts!7)i9>X8EywZPrsL&s@DGnxsr zWcXprNdreH5Y4bN|KtK`qLZ>NJ zrI}COnEQ^|IvCM49dozTeJXPlv% zO57q1DFj~_7=;Mcg5zG@o_f#|T9Tex*9VSX8ATY!T)#!MB#x8C+p+x^5budlJaBcE zqlF#p!j5`Srej6nhT}XF>X?Z$ZCbmUKP^tyhBB&wjEd>zC`+tgMI%Up2Y6M}=#YRf zAv&MPLKXscEvqCo))5BAbiCJid@uvcwrn??Ds(uOJ{2r7IU2)POa_}b)ywe?R3B$$ zXdYc*TJx{C<9*8^Bs2=}QdpeLBG}4^GpBoEG=crJ5xL%Kfwmq4#ZDkJRWPSi=oup2 zu@5VYyWT*(5Qcb!WP-;u!hYkKHsNH}laiU4j6k11B4E{kWq)mhw!cOQtHVh2tY|pA zIBG>tVZ7<}cVC`O1kyyb!-*y5t`wf7=2ee%>x6?P`!8gDa{W$!zbC2jTITj(+TETGbS@SBci@~j0XcqfS;4uL1a1$ zmW9%ul|`U95T?LYg^p(0G5Wj&oDwGNBp9Q7m zECRhZ(b5zvSiW&0q$0_;HS|k`V%NC19T4eURr}cnbQ(U1#%d$IB_CTAH(}tmun-=V zNg*XndLUXbmjcak)?(Z%ZpJ)5m&8b|CfTREZl#w2;YacXpcruNGt6F@2$_iuw8P?v z-))D76bU$EIN}(`v%Jl|U<-2L7`!3KLTX%gIy92YsprD1jRNS^LB1gypfqDwt2^<% z&M(-Gkbwu0;MEGxm^@G|lFU(I5CE-$1h;}L4aSS;q_&~#Z7a+M9Z1x1iJE%kBK2W{ z&rdveW=4#cgoQDYaib*rvzuU}F}cEUOBjYwAVjbz%T#VnaJ=BXuFxR|j%!mg^YJ@@ zfR7l}GoKSZ72;5VXST1#_Q?G@wgqjQ>@?Xs;j08o;6+7|K(GjYW6OL}s#9($H3HxSHjT3=<4E0Z8o zZwo72JS{QZbKx8X7BW%qJfN|Yi~?z2H$+Y%LsKnSmE*V)do={6&$E|{-tV!oqvgb4 zIkM@3w?q^Osykz4!d)=S@DS-}*=36>E@NOr5g=2mpmu41aAe9DFob!Kgj&oG#n}d+ zEv{=ieYpIZ{`#QprM7_zZ+?eOhVWG4jD;)RRSSr~7(|UoGam7L;WfS3oh8<0Ayli^ ztdmnO493WcG-+oSux=_`_(6!^$~;n((vUROYNR!@%o2@7H9aA8GgjV5^O7(rcV zYC;a>wz8agVwnIVL#Be*Mv!QwmAiemTk-)2*VwU;4K^MHnk`|jNj>OfOX$&@q>x0# zH)4y%&@2^+P3^{IcMwb?(YIy=p)wT zEofr%TTi>s?c#(f6*A%mY-J5M+{Ge@6=06xHmbrG>~Ym0P%PB&rvgMEV2i=t1m4&~ zVlEh(OU6r-v_Dwt*_mEf!rr#q%}WVkSsr6c8`1Cyac~32wO%{ZD#N|u)B=?wFM-k6 zd0n`ApoC7XzkR3|T;|-6Yh#7gS3+(CYMwe0$vh_XiQBFxR6*934BP72fV$6ljwozspxhIOUfu>oZW00}3`@urtZ z6DEl0iF2J3;zPzz%dQ;yq@v0QK}U7F%vnMSC!5^NB$TT(s)7Vq$(|t^!TZKUEP8z} zb88<3DweyknapjXA#m>Hyb(62(ln{N5J&d$-{R}GITnS;t{h$v@Cwjmo$FqBV!agBc775$kdXjR z&S1d^-jwPwlHwsfm-Y}%kAXW0m_yrGh#+Ga*+!x-wUX>vM~7RtFX;mNjDY?Zvw^h) zG>!*tA;qP1-ltE<2(jnB$8-?5bpR3}o6q6JMSlPS!2}3hNs4lyXI)VB`!^j;3`oq} z*mBbLNwYOP8t-|gZ&C^2P`pGH0;tb&dk8Vh73%=}nD{=%C4gJPKxM5XAvBTE20((O zHzEg%VZVWq%i>G(obg4T=sfWcwfZ`=g+^6TLK;AzI+t}z#N&jbLJH=VOBfIz zO@=;No3x)?GbJBojr=qMrgiXxgpgto1q&%55KzBS(+KxwF~ATUfqert^*kWGtwsqU zJhrCW)}rLJpfzdngeQ#nxL&>{6%-Ggo`QLR2GS6@EmLs%YCgTeN^Pn_Y*LzCit?V+ z(n9^Ao(h8-shM4O1}$TMA@%kXxn5ys0%6Cpq(M;@a|aRa`Pgw`$OtGn+uW}EpeO76R_b|S0{tux`{`(WK0tV5va^MqiEt*oniK`Lmzo5ot;9saX1(t zl66m_UuMPIZ@tM!o80*6k|$xP zP`@9e;AJK^REwQXubdIrTU+XZ48#KGn|9_S#T^#X+%C;X)^{yV*VXhGKy^iyKp@n$ zvxK-4q40ov8kt^et9M>DV1vbp*?`hcnc1DEWK@K{mnKGW`d%S;=!!JupiuH)HOke5xMbz`0RWsPk~L>!5M2kbU9*H-G_Xfws%9!Ue~^!|_)A3)o+E?h2zAp#_oyY!(s zA4Q|H6k6k$Lc#wv%)y(90pKC>(vO#D*tuO95A%gA^@Ym71y75Xx( zu>8d%bRLP19|3#_BhgCU-SY0`#6;~|3M8ah+chdRj;`o1pk!9(DgL@*Dg@&-%LZ;| z1R{H1V1njo?970w?Qw=n?_vWoYzwp0@^LU zX85Xiup&u>yCav8zK@Mxp5oOGK^YEMTTexi2Wt@bwz>DOPKJ?0=7#JxHiN< z#~Ff-jD}Z=fzL_uAfLS@tj_);?rTxv2ca*sNH;7hsF+>OhS%u;0MHXS?79~#FMJ1h zr3hkcl5NuA3Cxj+*JK<^p)zQ^_;PHy)Kw3F?w3vJ%K|(C(3VQ?qB1mSMVUNEDp>A) zyc|wBU@xo0B%|wtF}lK{!({TU2p2OPKS0G>a8r{y5?mmwBS;b;n*0Zn}zi#rq@>(B%mB6@m$3*9>gsPV*is6*>i8(yY`$y=EW zpaoUf!&{bVbaS1eIpqFPyWV z+9YObNkt^aS8-m55%$`A4k;YUR7AFWB!4E*)#aTC1W#EDSUVl0c!BVLn$LAF&D$h_I+o1;Yz1 z5NN@k5=qM#q;&080rr_(;B06(LGT(?ZA7Gz0Y1$kPAQksDenZBIfSaJN`QjT0mfBv z^jrPJU7pMZOQ7w!Wt8q%&JK}os5XhfRS=pPX^1^Iq$>gfMg@BDnl?VQ`6YhfWFS3Z zacx$}1&FL^z22wHd8G-iKxF1NMWt}>5CP4A4ngWta73ePZ z{Y`JSR0X{&E6sSix#D7W(u&_z5#6HO#R8@$=m&xMJy2H2Skv?ZQ^1!3ExkyLqo&p2 zCTd-TGm|wLOK5D^L0;WKDFPZLu6Aa~QZf|7^078kN|FuJG$wi=I6$wF!h(3XOaJx?u z&h#*Xp%E?vEmH&47gxkbdXBMDH|T^i&|WkVeZLXd(+|`|eR%k+VgQNb$%e6E#8YE? zmHH;k81rxi3ai-w!GZ~iCCdT>{OBalkPgw`nF&y4^_k`rKg1!SDIuNF( z2OM%oCn_<>25*ztS1VV2Vnv~=H$XVRK@lD#0U}Q#qH3R*qS@|KsY8*5g>mjnsvCsD z0j{~72P4aRF&MPCFLBk)1T z)o0Qo$I2uKA-Ma_&H_3v0&pO5K$76scL}i~EFsq*s4{?h_+hO*7y^2p(K~jm`)rAnPU+_v7Uy9x$$u9UWe8ls>a6Y@sD_}+ zNryr`jmm3d>@{5bDgLk*2Rww6SP_xNK}kU_yxXI(aoKG^Z&#J0rdg z#4qiTCAL&y9+iE@oe$h-sChf2kmmVUIHcmBZjj(-*S=L+nnObujJ}~IHDQ$y;tUYj z9`lDhhdUe0`uQnDaWPfc_}vWgLSM!OYe5awR3{!Lk4Er3v4qpV8?Onetw@B~MqQMH zvQ-({7i-gmV@a{(z!a}Zz16R&)y>F90*MnXG3B=Jf3DyEkJ;@Y~jXoinT zesy6g{3D~fpDQ>M3nLUl+z8W|>OG6oLXQLnmuMHv%Xks#(I#^8n<`g|YzanBcZF9tCZ-`Msj7BXd@CtT{GO>I?;>`tY;Z0txJ^ zAYE{70#okBVDVdY_{AKZ2HacOR)Lh9f2j|&=}{1`s$NKyF8+NrQz1RjDX4*nHoKQD zG1I`Ck-eZW3nI`MDu*3>Zl5Uz2ty+}m4OA68_&HQqlsxyymWF?m3&*y1egO7K+eRk zgR%t5x$PfyCi(JC(N7=4%dIfVa(LLMsH`J#Lr|IF!eHvmdmta86PYkN9n;~l#T`7Ha>|3#*+6PM$!pbg1-H{9+{sG=nm0O#q#B2X z2g9jb2-a04FbPXTl`AF_mX%Li^_I@2;|74z3~Q0CZXGZ_YOiVHdQslxIAjNz#_j(5 zERJm}IRkP{f%gmtm9oX>k6sv;SrwmaPp3S>eZle%Pq)>WXX^KCC{ zk16UZz}G;G<;{waA8aZ06)6iI#&2+2t^tu_$AF{T zR19WwGw)4>3kFf$91Gr7&Gr~Dh|U(qSLbDAtcsaJ zsA1M60*Als@>38M1hnN^akBO%t=@&RV5H4oX4 z;o+0vdnIyPCD2{(wX(UHw(^;-#vG0WvToE_sN)^^xCc+Kd&nxLE;FV~74DUUHVL3L zI(Z+k#@3Gl$!oJwBJb@>PK(z0Kwt_pK zOEsw)#|O~1X0%>-78i*Nj$^@VqFo^2(e~$;Wt>=z{?(igTZ#_eeEZvvee~^{ao_Xr zPu91s#rK{kZn^UJo%bJ{ZMIrqmy^IXNLQCI%7Bg}jpnw51f5+JvtN)j?8s<9^;U6J zrjfqf$J=P86S~+WU0Av?b^*I`Aq^O~rn7;X2^ovLmxDaGLT`Zt6ha;m2Z_cVjEeB7 zTq_ZK;=D`0EA0p#^civiPrF4VE&)u~u){DBkWQ(a#YDKmn<8Cg0Yi(`oF>`lRQNmv z(8dBGQ9x^+-cJjLrIJ;LlYCO+8C{5_L~{1`>+{~Be&YO#C`+OQ7y&BIRIEXS1B{JH z$KbSDN=R14Wt1J&6^P;Qcp-X+JDhWo@j7MSeop|0PPJx=cX%ttr-wu$06}n!qV)D1 zhlhIy4R!@E1}aY>iaWlOI?@vSxh{!ut7Jiex)W-MU=%W=BJq(N1@N+9NNvOqopF6- zAZKe7SFWkblH^wmZVyT7D9~yvh%W~2H&kIP)+U^Q&Y+lOkQ)qQ2;Ddf-;q*waTrwE zw6RSrP(_F!SO!EE$i-;8K90Wu+(;Tdql$`#Fcb!(9S1&2lt9XeAaa6zypwD9%BinX zmR_zUAy8R)(h#f+Q!nEx%gpubfLLAmeb>sEJ);yeH}|I^%`(aA>V2vdc__|QJXB^q zJoIE?X`*aU7cKz@ z@k&Y67tcaK08@G1p&rV$vSz3d-w-%ayNUc|v{~UTKTtq`b&xlpb3x6`XG0qlbpDQk zYLUyv3y9(j37NOU=IPK;9%9lcJ|$jttXWeE0RW76OaTL}q!B>}5a59VxiZ>f6$UG* zxQ(*Aszs2)BIfbdvM#ty<&-v#H$u^uIl%0oAG3`WHB8(aHG&C7xLP7aBtwT3NbWm| zplMe7ixygMq(ddgev%@Q19eC9?g+vxCpin$st?AH`Vb6(c|Du=nXiVZ0J69%{ADW@h)pn|JTkEWGHO{ANfd%s-ibEXnh z;a0CY6h3>cwmVM?EPXNaTU?RuUt8E?*|7o{ifH))U~uGd#AN=iE1=J+h&~Y!JRJl0 zEnlGTL>%74{`kwzp8|_EQOc?%4o;bY;Ib?N3|lh0*Qa^JMApM;8_+v|&;?57si3Zd z(c{a(>K#fdBqLFq_?uS&p@LawqZmxq+nIi;BcZ66X269dTz7_fK=4L)rXDVRor|#L zz-SyW?aWE3m_#5?d}%i*J`3?Eh2ean??;~q7`WF~);|NA454_gk;~k9JL)QhL@ULq zD=L@Uge%B|)Hz(D{&*r#ewEt5*98d(K~$4Jx}C?e{1?fbn9HoQ%-7y~LA2>3XA}jq z_NLNXM+QZCV&$d)>ZA!$m+!YN>STqW*5%P8Uy9dN)q=8adn3yIHOB14xO#FWoo`|$ zWk#45NoRsHrSaXTah~Q(5eYA*qF3M$6I_iaZ${e4Fg83GP_N3dPztqRpwN7g#^AXo z6>BCXH(-6mqIl0{G8LVVhwm z1s+azH@rU8HRujtE~uDO#=A~DDIbGZ5AS%8h$LIbiP#t>_CAK@>k&V7wgBar2`bZ` zWeo7CxZ#Kp_VDPv_)P``-|GqxG*RIPvS+h=X|igwwL>$YRp!vC{+R4H4xljxM zf@qbLiTofk!5MBU4ZH{98d#Mj|52u2l8XV|Q&?YAKu+XGyY zOuYM~{zB;&R(cR4P(lgIA69X;Y^;L-gK>$lSWLk@svIPauVq>kKL-vW5U_&S`%0W& zFW5v%7;k{#r8@{C?(Sg4wc|BovFUQa+fD6g`=I2XsUu^1Rgl028G<=#5RVkg9-L zeZ80g5WV}#wn@a6_Cw#bsCeMmOnip#l@P+C7qJP+nfjrS2d$JBiO3smPRokNBaL!@ z5Tqhmo)2}L6PYRfz?6PvL-#}aALf|}3{SbNEv`QZ_MQ!HCwV8XX!OWsGOJ-{Wl--g zdJ^j~gy64w^H>~QUE?k~nivnwOVc>>oDxKB!l>{~6@E9q>%iY1HtB{+goXfg{OF|oEX4eo$2@G#0uEeOucsy5Wkue>))3i4f^cupCiE zR39Ruz(i8+Z1XxkW|}FP0Dz6HnR1_n$3z20O6sCR>Qtpfb^FGmO^CgREQMXy=@Hd= z;_@V+t=G-(KX;t#-xJNhH-9dS<;L&lzhB+IIqRA2_n>Me`RIzz6i}E7oMXy}b&bgr z)aLM9tI=L5Xz~zSMC~ox)b`s}?sru&K*2(gCIu>SLf>&OEBPJ>9WlvXb4F6vd2Ua5=S7!qMq`^f<1N*F5P7F~n!QdscXg|$UmRO0)O zfxfjVBZ@ZqTn%%;mJxPFcKQ9>?24v~4*PAKG`Lq;kVR@57$+kMUELHaHHjc5@twZ6 za_Y3zwn@5+P_ck|CmSwz8EL>)IK7>*>np{;#`uDG*nVQ~S@GkW(z|GLt~&!w3-KV} zg9g^IfXx&%&WNl_SA|D`FE}KJA?-mi?Upi2ncLSFs+o*@if>7C+GZ3Yg7v4c83~X< z;F4hreU#o(ZJ0}UhP&fc0Yms{LI~~7g{7<5w0kMT3*jAJVWKU0qi_}ubiIMFp&y(; zaJ0#UIj)o95)=ExXid>IYysPXA(@_^Ja$)-_N6l8d!RW{2{?Kp6r_CpgQ}`7b~wVg zx$hr<;di1qwj0jE}xh;%C`p7WTc5l%H9IxAK3PzFS;@*yZ8 zDGW)CljxCorN%^#Yx>LJ;SB+23oJ;ob}xYrM_KA(ik&`z0r+t^Z9e<5!e!gq1fsKx zt-3k`;7FomP}-xY6*KOi0hNz7V9bP=n`YG!-sqBF%mv)<`i|mQBNsu zmGla7!`n@LG&;MBC0Ow2Hak9XCj7@-^n4n1i1}0-1k1AyW5$!9Tw>Rox*J9;WDLZa zEAzL{Kh7@3RGXlRGLwc*o}M0+TPK(aM~BA&gF|9=1K|N2ZidHHBMZjYpt=SautXTY zcBCGeO&OQkAcKP{Kp7PD9j4&Cv!RXP6wSaZEo7;+J2;0;R8_v`#)SmVf%D+n{mMab zARGFj}6L&9sgzOM2BI7Wq)uM_I+$Su}a?<9UA;2f44otE#DdBYzu;Ctq zdt-K9=(F5Eu&O%zgk1vPWCY5rTtAUON(k_A*jw82g>!aK%NPUCumL8bwwke)&VcN& zWfnbqJe!&}1Iosvi6Lm^vzIOUxrTs>;=#&|?*+9-L!YiWMZi1a$_LhLc5S5U|WEwp9MjXM=a z3!;#luo62(`21kSV0`u}i?N5OffNa|hcd8kTgk_#4IC{JP2qVy!vaQX^=ME&IkyzU zK(A9xNy-6>H&O>Nr>@cBiUY7vA4h={{GBt$sBd^FyL-QduLG1wt z8y-~N%tF%xNK)uQM79cnJMtyGzPnI_Qc~V1^%Eb?0wDFWH?TssC@2@8q^Lp_ke|4r z6&Mc>V<%3L+Sc`DG_H?ek@7-Fii3P}!#Lhs_~X;RrR!X0XZIUk|9IoJzc;PA z{pSA^0Dbr0Cqa96phNNJci}ZC#LvL6)mMgE+Hl`9QCYBz-T8*=8`KTon(>B+u6McE zA&24zRA4eJ>$qzeWPu@}aS@>hmXa_;A@TGr5OBtw4$M~ENY7KsUk`S_LKm`P7p&{^ zcPUh}iKyYZ?>W&H8=(Ao)Ic+Qls*^>xL`83c8DkCT_`eAm=#3CWV6X~78O0{s%w1% zeL&VtBnBMt0c8x70&N?b(E9#bmzz7bz??sL^r`_ej&KIHWLP}&^#`Mq^t_lt&$E?I zVb{(f?HJzAX)0w7gCh4_C@|9@>Zg*KPLLaptZVj+F@(*D1go^N4}=s~1_f-mSVTq! z%!aacs{sz4(H*+{z%Eh@p!1=M;((|}W&6xkG{h7-Cz3kY34hYG4&#f zg_q1x)(LDRS2ScxEDx%xQJitu_;|nw8_z58=A-j3fa$%Eq3b!t#xBNQFpB846)CB7 zp;`&^C>)iDYg0?#i$H|{tV4N$Tpw8M7=*=kLf%K|~( z;nIXI7ZI>>3kcR^8B-!d5OrN~wm)pufi+~B0I?QJGHpS5NG=>Y$!s*^fmF2u1Q~$M zVmt$@5*AM5MyXbG|?-zZahK^<_sZ6&`K0xcD;~hF;az6j5u&?^iFhcH*vDC zG`*6aWBRS4BUu%XSFsT%-O1HdJ0Eh#GI45DG*RcJv-bB(R@&o=x= zo~xirAVpe`4Zw*JhQ#tm>3s-8il(V zirRb5%H0;_823cjNWXweQaI~#DfFyGT3cySskT@Lf!OH&0Wgg44gz3M6Fg|*wZI7w z#3*^VWIW{@phGcH3XSZ$kQULT&RB0IixcF-qjzBTv6VnN8XIx@4ks^~KrI>sZ} zh3{*k#(Gg`%=v}#VSF$qBO}Ecm(g2yu)KgJt&k+rMrhBn$1wp9rekgJ6O5KSVUl1Z zBrXnzf-QrMyOamC^myTfwRskx!a*hQm4>7kbrj+PJERzIwY2VQVq~Z?ta2wb0)+bo zSnzjp_B8Je(Zs@y3q#;yqQ$P*$0^n^Kb zGi*TLy8PAUbyaGLZ5z8_)Hi08S;bKTLwz!W335>a6!V5HZjdFc{NOE}*vMf6vx5W) z=f%l+VqJSCPMYsaM7nl>+R#cbCXmY0I`}7L9O`e}0m5L!La|KnHg=K{8fknpTb)|? z`mk?g3Isb{IV64QT_gDXmgI)7HNeYozd=9^qLVFfA+$p#!?jIu`g}J<%z>PVe9t|j zHeiV?8(3Ub#<9;_+qxs;`vEtZHCRRG$t_unV^%lq_HE2INr_CLJ2e0YF%A^kTN;*e z=Qo$d%&!m2*EOIm=TEyisA^YgEF*A%HoV5`r{7chObGCzNFh!i3Rzc&AyYj!EAfV- zmx_E&%TE(wsi1VlO^-<~o0J)f5#Uavpo(GIaMplvP>_iU4bDyFeYvzfKTTre0*@^m z;?})U*61cW#o{XQ3zZ#i(<1H1l^v{O1H<%$ovluKI8&OFpxSxM*-8px41%WoUrM^P zK_ju$=ZMyr+p!bPZVpQfkf20$O|5B_$_$51_iV`b zs8gqSp9X(S-`r^T0yBzlI2x{Czhp@v;FAQl5(Po&a*#<}^@{FOrxxhz^!tyE%{v;_ z0HV2#9HF>t6wn8i!{l(Jw^zch6t9R2$MlGAm9{dw*cl}06MeT7FHKtOVzD$P;7O*u@|^?_Qqg$NZSv6*{B#k zwYXN%645;%FNQ{pvsyqf!XFZZeQ$l7x_a4kb=t`x_5nHk7`bl;))gdV%0^s_JYgbn zmHn}!tEFkOoF75aF>9fn zXHj~&BkFX~Kd^T*2Eq4`u+YI)&QOO}XH{-v)o+oP#u_?FvN{akwI?lV+&~%}klk1y z^7Q{gL^Ew-67tl%_l1nO3~<*lk8CR(3`B~EB{LsQykgU$l*{be?>f*Sl8uiiUe`)# z8o5yj52}P6;`WC<6=`4*QuRz7>7eZR!?e`>gR442TLgQ6YnJFO*yCsgW+Xn~HpTO4 zgBg7tE-=_A5J9w&5ID1I6ZK$Bo-ePb#52<>lvXGiS7RlL-K{}n zHOuH6Cx(?n~-M` zvt<#K#2-bm1!Y6Ew)XLy$<=;h5uma2hJy#Wjy@xOM#bXnCnKJ-d`Ha*H1Z^5_t4%-8svkp=||sxs~OgVwxKWUp^k1rY9&} zo;_Vdse%3C4r+ilZF>+Q0MJz`CGlDzyOT-@LGZ_!US#59gx7q zLAeQLC8L;80tRHGL-?jZ)P<%uPI4-YX!%>b?Qh}G-*+I~gOK#v8A0r$ zgjp~hZ=;d%QIeX5F7UfN#10%*wDWW|`p_3+E$)g9oixI5AScE`tr7xJ07@mBT2Br#mR%OZ}nGQJnNXW^!i8V+SEnnAvs) z6Dkib^@dumWoOKn(7dJ1qCjBpO42%3fjPww!6d8>Azunq+2E}_1MVXPhz+c%lud(~ za2W$)@73xdvdDB06w7{Og1Md+w{u{E8qh!b(GSyz;LNAxqNAKfW|td%jk z_)gsiPU4ijbsM-5?#$I##O%L8NAg2N?kpbZPC&Q{zOM?V9vD9o1$!}S7c+;0JmN=j zj1SBa&KNTH2WrW3aga?`7a}0Lz;y5?LIO(yk4Nh=a2#9^p4PF`z*4h2G-)agqpDG|Q&(qY2)+ z7Kb^JgywY@1l$kzyMQ}@lHi6Fs}`n2ymS*d4qtb#D=3bRC;lSdVn)lP(+G3jWP}&gJXE132jP=KK_wJ$vhz`c??^zj0EM>} za?-Gqv9Zr!qi>(vok#Rz6FKs1;=;y!sY;;eSxQX?Rh4?SrQv(x8->5W-vLzEg?tqfy&b_%77F zA|V5WR8NMZf>7Zi?=W{bNvH#AsX^tF1_w(Az$XppGYq;#(3BDfDl--Mf>Xn(_KrCN zj0-m1uH;~Z7hB_qCpf9X;UEnW8hODz%Yb&m4zeMgV3?$0IhHDmL_5anij~gIgxO3% zpLh(X=N~VcxotM~z@aKZ!<(vmy6*LEL_4#EZ(FWPdz86zmg<*)w2+ZTGV*BJR$0)F zJAlavNWVvCC8Ft zRR#K3Y-=CQcA$k1+8{j{ewR3jMA{aE)o2;{3XA3yT%IWfX9l=gdW(K51G7eDEQGrp z@d(?JJOQGpZ%a2*a;Uun`RPIWv#B+UanfW6J!8}%92mE75hUwIQQf%DtnD0%+fa>E zwOr+P{^|`}CNzY5GF%inCM;|zz~`p=#FUjz<{}pyp4)kP3d|5mP{m|dQWoW~gSs)d zEy=-oq6^#l$@5^0WR;$ZS5%eo(M0mf{A?gq>UIS-r3v;dMN}IHl$xS&CH4-L%N6{! z*nHJ<=Y*d3B?col;Z^kb)90+K~?I<&8E%`L*6BS zRFUXOsD}rl$r<<=S_;>&zTs&QHA^qN&v?uLSA3BQZPwbR)e%-!)fX$6&UNvX_KUS` zAXc~D;pPCJI}~KYqY@*U*RJcpEXN4?!AqQsRGE4ctoPc5M{jE)q*r_;Jya>O*TgsFk+azxWP&p z08%9o+cIv7a4Xbnx7(3;8dH-OCagq@pzh(Wmog0ketJq=2oNw6Rj9J-3{D)WqRp!o zUn!RSWc_hqH^7-l4zsv3(}FOB7{szTHE?gGZcHK|(7z|wE4vPyVPdqbA&`#(5H=o^ zX!F6=FIk0mQGpOggGj*LNYE9$L!6|m?_(QRb43W`?{Y6#f}R262?U6QxjLiq)R=?` zbOg60>4W;vA!Stxs;jF&94lmo>g=v81yBTx^~TFpVv9JNa&2txMWllY62z(*1yG{t zL*%17rqej+*6hiOGx#3|!S~+)d;nf+v+HV2ve_Zo;*m!E^Ds!jAkAXh_>z>B>fG@4 zHOq#q+IEM3Q_JBGDPXUxfu6v7uPkT}m2T&kb;d{U1DI<<2_Trk!!g&;7q?9nhN&)w zQgPHPMTjH@Z-4~Gzccu9SeU`5Qk3MNy`@BkPsC?JGmt^UpTjlXHRvp^sn-|*hRQAE z&4}(k9;S%T34|6R#Xhx3Yrva+G5X4Swv6TNg0Ts5GtX1*Ec=)n7L2h(DSa9 z8X6eND4_6r)@lzWW(jJLt>dL`8sXb@3=l}lTk;B%b z8f4@20k-=Zg{sBrX(ZC>~EdJ(+CJ2;_{G) zjcG#6@F{*)GMXUmtT6OZjK)C6{7YW~T zXD&o86D8O*P(s2Ba#%d1VDHQGu_Q5dFQk~8Uzb~l?oz{UVji~v#pMB78h1qymeDhS zL(Zyu($Ex9OvG`55c+;ecp<21JOD|^_7f2hEsc1Y>Z3I+VKi(PdGtuzI$YIVKd(i@6|yVtr;WVaA&Ly=_?C=%Jd zyo>vr0$J)pOJG6?P!+8qZ1X5P^J;FVW7JS7Yc@LP-Z3v>fFb}Yx~&KctHbo+W$FD` z>rP)y_HjgkTZp}_m zk>t^A!%Qg2{tQ6@Gm(1&teiQaYG!80+@6;M9`L&e0wH}u#zO={LPFmK&E1lQJ_l3> zhM`h+AX<@)hPsRau;_yaY!kCk3$28pVjY8MW(6eRBmf^sJCZp)=P& zXTKm&kuL0lxzRmKyz#Y?w{bv&jtjZRmG!*BEYLO~+!BND9rj_vR9t*=kjY1=6k&3Q zHD~Y*c=bpEl$>NG1~5D|Oc$I)N|$+d`@7+$ zOI1?|4*(GBwk{HRAB)cZRdS&!AQj=L#i7uIOAx^fa%K|ga&4Dfgv0wP?tWBhz)XR5 zG-^?YWTBJu1fB)C58Ete>PCrT2$~>+Lh6RL!D7=4v3EwwZ&j5h3NYpDz657@`jNe@ zhCt)m739MN)bQOfLl0Ka3woVNIDr+hI75VD0(!ZRa)uBJfG~7-uy`73DbOVWXpg*|(+yVlNc- z9_B%>k>Zl&iW1(=l*sT@?hTT|he76NUKEl*s?XvYkc z#Khn#tg5Sua_Ib*Qx2o%kcO#qBVN5Goc&eZLKFec*TCx&9^c%^GczqU@=fnTZ7Ryb zb}CH;In-a1+k@zX1_1^e#>dtM0fe1NEJ`L}^!Y%t!m?qB)NvBM*>TargFHm)LE#F% z2G?U7WV0MC@FKDB1hW00Yt{;#jIXZ6e1t@CDs&rGUQW-Jc|tg%jqj}Dh0eo7pLAMO z1=va;R9e_>w+YbQbQfbOtc{}u>f;9vYPi>l?4xn<22|zA+&Bz(0kldAnXi=+6+R58 z(z``C*u1u*_b*kU06)tG>jSN5BuZX0+&)t!4iCW)ccY?2qDs)ArlSHB98pB^Qe-S9 z(u%qjF3HEHh52o@c^Xtgom%z?UBm*5sHiHUD5K^qL3~*LSsfn5QGk#!`!zkcFgRL! z7jV2U1)>1OZqUdrA}HTYCAcsOKC>|V3wSkZa|gpCFh&D(a+p9lh7YhBftFTjRJ3$9 zX#(#cX|=WSW?wa^wMo*#DhP+W#*3iP?EDx$!O?p2penr89Jo+u;P;nv*1R!r2TWv9Q5*-*cYo?NyyO%4+H79HFgXOnOMfqyX^$Rqx zbeZOdu6}~dvXueh#vlP1%S77Tn)d5kVHJ{KAnOd=s#qvrT7_dlbr9SiS{Wtn_1B%& zwgbJZA}YFqNDy9$;$@dob}{U!I=rNrJdiW-^LNs8TW=P~=#n zq43)22uTDB_`Wg75=^T3#(ihDYLn#y!2%%7M}cPfzI8;y%hEZ+D0n6%MD1axXpm`< zLNZ!#VNl^ah|MvE#zQS-eN0eqYRloM?gM)RoM$m}Odpz^eb{c}N4bQ!(4*mUQJvQB zJ(V1Q{d!OLu_$()+g!r;+>CH2bTCUQPi$$=B{K(FT6J{T2*3h7&|QS^d9AsAVZ_c? z(u*&EH9XYTWG6KbDdO1UrQkKs5iFB*&?3M>&xs3ADZm|;<=bMNTfs*k6l6TITZU5F z>?{|Su|acf$?TsI;j~?Xa)PVykmPJ+=1pymi_AnVx?b)0GwFM`0<(&Y^(^5E9_Q<6zniKZ*c^1?s6?m2o9(pm93o=gF}Ap9tk(HIU? zdU3rO#LZTrz`y_-l(fRA{3_Y@B#lT(HM7v(+Lg?n<%JYUCkQtX0tO);926%8WTzPq z&Jyiz)-58Aa$hD~1E4kNJ(QbnEjWi;gKZOdVjpZk+=x`93sVD%g0)SBB7%BZya~CC z=PR6>YcVAd3Zc@D-3_YAzO;BMxVIO)xRj+=4~QJXJvn8}l0Js%wN*7}mi#&9eS_9t zj`w62K|@l}Dl5`QVyS{Q%>f}sd7amSBPm68b1NO#r+3eWnhCJ2n-(umIqJk~1C8Z= z_p>CF=Lx>Wq>}Ej!1DXL&}Z$ZG*Juz^Cpvv1q?9~LD9@>$>_e8)tPC?9@c6gf2%`Jh8tQdw)@E(< znoeF_IEisBWf~C3hh~wjlGx!`v))3B@ql?I6Q79@-t*1+g8pV6Gw2WbF$0)FYBXJB@g^34#;Hm)?2;f;orb0MXzN4(^XO z(aJLunu)$TQl+vrRYDsvFg1s|_4tr%tZ^(=`t%bAL)n6Yx9f)lE5O53=0-p%?|`k! zkPR_Ye0coR)@=HI+PA3!b0Q3juwRr0MZujdt_NPCfFMP1ITZ--gp&+6+HKjO>xKe_ z?7wmDab{Xn1xqKX-AY3rq)>MOBi!$ctDs66Oo8V(NwL-ThGL(1hX`Y=7aEzgE|OP3 zJY1bhJWXIaD>sVIjH}wQ_k8JNt3&N_fl;~fS9?3AkRfQ7AXpgCA%mRhRJc&kDE@l| z0!a~q`N@dKrP25Z#IgA7oCS%NB;qN>Ypl6U&hA zS$>0TLpH(6BT|KQ#PMg^0*!FPM-CZy_8{YK9v9-F<483NN{E13Xt2grQVIkm6g52L zMWZ$i?6fOTJV9U@?dPAD-g0`Qq1Hf!Nk<6KqZj8NX-~yb9XO$=9>`r&qE<*A@iGT| zluE^S&?%8bw4gV%T){j^uY$502&PlX5scj^uvY=#J3KH-f;AE#11&2(!yLfm9@ZuS zCq*0gW<1ldM9611C5u^mRS`_!qI{LqechuBM2Dxm-nK?plkA@|YMQ(*Cl-~^Ngc(p z$28R`OGv_NLXL$EK=&LBEYw_V8RIy10*l6a) zA&Q4ML%_qndgmixa$~$8H!suRbM8NSd{cetLB9Y~?2k)&i& zCHpX{SD+r*WPRRBydeXotRyf^I7$d1ey!^1p*uefSw$)eKTpXIk*uWWvKD*kByuRm zv9|0yIv~UH8oz>^?#jh_6?aBCtEqgqDF?Vq2i5{rs++yKMw}=siML2pN)~(GoyLuw z5W`5oF41jfQpjDZ`&(@tlUPcGTW)m>;pSGJ+>!x8vLbCLOcork zUQXEn=M|wls3vU+P=bJZ!PXWHK{)(z$e1U1Ha(>}p(wa1)TbDhf!I>G4hRzkiD-O5 zVbH|I!ASDiaQ&{m^4VpKT!b)?+2>% z<#8l@R2(P3aIf_0#-I@BQ98gytfrlcZXXk%Aje@C*hzuaV0lzyn8;LxtmQ+UgKh42 zKBmBz^kn@5iEu}Tagf+prSA$CnCmSoC=8Q3SR`m%NgVH%8+zs75QdH%#%RG@?Ejngv{C z;m&ZT728I2X`qj47o`1wM*G`up`k(=svRHFWr7C7JP<-`+Z3=g9R&`EBnn&{OdnL) z#mkE{vo$YJ?-cH>AK}*1y;s3L6HkTT?GZY=D>5sT;BN8+tw2JO)bphd3>K8Mjbb8T zx!p3tT%J`7t`iiHpw-r~1m`?D>7=fF-;ix1kcQRF3$(+hPlZf@Dq3^f1i_vKBmRsn^oVd7d+LmVA_%@nLtbn(c3AQJBeMrbn5P#X58pRi*($VsPe1q7A1s zRBXt=fvOEvl?M2p0I*$N1T;~u*LIjjsK|V3wWvpBr+b%2rG2deSbz_-H9X`|NTyMW zBkBX^FJxRxiCRL%NwCAx5-s^p#RpwTWzWN>$IXl9@in^& z5wX(>LU22tSG(<&S|;kG-Oo3@$uhpU?2x;%5?!raz^rPi9vb{(a0 z5)imBS(gE?G^#Tzh--s`k_-GIaQx~R{-%m>ngbeFv_0$!To@4>f>ii@tFevG4K5*X zlX`*?NIq!3A@xBP$Z?|*pr`NhYL{v=f#_kX+h{`>p$uYwqGe2`qO z_le(+OI#y$$WH{T0{;}pw-97C%VPMIQo8_HLUGf8QH@-kH%*^{OjLp}Gr&7pFOuK;fXNy>tdj@sJ?~$-|IEA$+Ble6k`LnX2&8nCrmv@TGNxZ77In zKKSslh8*l@ht8MFol2+(TTN(*ZQ~u%%;k12h$ujRAi{{ULD(KL@z$fR9ta~OKZDxV z*d{96c-$#GGts+pp~OL+=oJqC5)VP<(XO|pNQq1Pse8QVeAr1$E>;o*1|5M$Om&U4 zs1;$j6F9nAwGE;$Qzu{pl9n>7*8%a_P4 zS8qZCy_Y9aK_PWDK`>M?x=Thb3@t#});s6Gk}4UrXQW2qgY{BN7R7E11e?bCxbaI) zaUCqfuEb}Jp@_k_KPnia*RlYGEj`trVo;*Fgc1)=f z);^OH0Ga|ZvZ-8mR^ft2md4sK%(`u{qMLm)BWxuHK=u)B*G3xCwH44PWz+_a8yYH53V_#zgUVfRLzw3B3}Q~1DLh$K zbyn3#8!q2%&=plSucT2}g>NovNM;llwS3TwWA|j+0~w9ML+lF5uQ=;3nm4*S%qjAy zmfP<@8b%snOzP6-am%~1`XJqgKC`m9azYGojAzmFwT*{@rA$Sdgu&-0Lu#(m6tufl zp{^D&=m~6KZM0l^LXql48%-ApDiVw3;k)X=C4_*JC{h6pG6BlQ#;OMvI_&DzzpR~D zK0Xun$vbeoBgwCTFgIX%VeD7Y+Tnb3DuyGjR2@xPOpHyK5a`BGfj4MPUNVX+1y8HO zEMcRGih+x^C-BAT82IMx?sQxOPoyHtiuoP5+ zpkz?u(d9{3k_#!%w}nnDTp5yWsa-TWv>y?5Ep|{9%?uavzJ`znD~f+cF&G8XeLC*NEsX~+n31{ zO1kURS7uJPVUiNHU8*YxxKL_lwvOES!ub9+?77XUVGY8I3TIk2 zGw`w5I7xc=zPOR{-4MAN0XJ+tEj@r_2F_^XWPSy&i0!U}_PoeOy- z0z0D85tRzYC{F`*4+R8-KW$l-aX?)fnX!CW>&&(?N5$No5WR>$OTvH^!!0aG`4Ni5 znwnm)bkJ*>EhMj|R9>`)R@A)pP9C*-@HEFLP48u^u+dyEn6bngSeHr@WPsIH<*p3d z!=8=WlkJZbRaW=Dr99LC}6>$=0Yot#$BSX(|VEhu*#B+;c(Q$RkWaIK)zJ=@7cq&5K8fiFgEPXLWIz^ zg~8#&LB|ny0~{4=P)3DR!6#}Z49oRVk)7Qhju$}ZZi4aAGh1b3UeGBUO~8eF<(hbK z^C%R=Kf5=un2H^Wl?`<>p^HaTUk9nh5ZsZGY%`VK)^JK>6f2Ca0g!>s=#kSY-V|

!gs45JGV@a1yR!ASaze5^$V( zx`L6$x6{YEmML4xB`4y#AjC^5a~`5Fp^)<=^lr3SOeWa?pRv(m(8vXN$CiVmYLz~& zq8`WUfX5eHaq$`!i|Hm6TPejiD6U@cJ(H@;fR#aIkJr9r;{`?)Ei*>;b)|#$O@gd( zvUnP;MorHQkU(g`utjT9=LjmG`LnRt1elz>GPNK!rW=lbUHn0&Rmyi|QMrJzl&FQ= z#Hxf8aG8dk3dW0EH|Dqb#7KY%fa zrc=ww5bQE0fV?3yStSRDVHZI-N+R^YLF`*(buA`bm%7M>jSVc{U}W8KyU|2q+B(M3 zmfc{dhdBgt(Bx|6^<&6pL??g>vPig8wmU(lscLuHg^vA++`C}L#xyRVOYTVtZy_Md1#@im#MZJXS@#1Jq#0Xc zKNt?nY{1(ulVwjh`^{2eDxhK6cD`vjNLs9p zT{pJ;JZ(X#t1BioTX+w2v?poxdcXV>~0ByvN2Uu)kc&N&ja78iWWtta&{@;tU=9HJ=cj^3UOBQ z3E|UDP$v7rqHALPn-;7>9y2_5lVL6`M`fPMwJ0THHW?Y$*g{w8E2~>ya> zOVJ^+k*h^MjWx45o)}t{XOMQrbZ@0H`fg!1=kmT!zK8r$aVp57r zO~ZWpU!l|1Cw*ilOtavkKG}qv$Od%-2@3fWZ9`9thp-=l7gyX6yn#X9EZ6WlGzJl^ zc+#(iX3s^smj@z<_mQES-td~sqFMx45MI2e+-mOwsh!BM* zQQ&mLG2>gnR)lK?L2IhHL!iM)le(ouRD65^5D2p#DOKo}Lp)UGD7H7Urwb^2T@}?8 z*~}H)C3Og5v3MoCm!x+gQ9>v?X$c1#BsE@;eC3uT)=Vidh*LrLvo`jgT|iI=AHqzEu3$8PgJZBjeDn|_A{Ws(?Nl!|>8r_FR_M?H zkC)Y;;(9m|y}=Q%as)${8uHtFk={3j7Fnh?2-61TpKN=qT7!$JkCGdNTID8PW0k0< zC*|-k)Mck7GIcJjr+YR&uLzLVfxwrPAz^~Egj}yVS5LbbjHnwM%r=&!8RXKN z6dLTpVD^!c$gv;@i9pPPZKd>P9Z;k0JFV*cbg&dvAkA;-Na6a-m}uxZNh84mZnb=d zuOU@~Y``IfKBy4TG@Q67=Hwq@=L|d@6b@uCAXsH0zD8u{tVN+q#8MzwtjZVY#WEg{ zSyG>=&0In%Y+f2`G3`bGfLJ5LZs1|!!N>xfIywvxhu*eDB-is1J<+w!z^UjMZ49GM zA5~>eIhVRO%oQ_4PAMcUzQjye6DB)Xc!1Ljs4y5rlBCB9j|rT1)lP)#FS~PFIFXs6 z$mU_sx?D}?LwZ9zag$RqHorzDL3p;Q!htE_HS?6JMq6PT2jx957a<;68>R1Of|nFr zbwN`kTb^EJ(Jv%8QjMRSxkAuq&VHf?ccxJliFY*ma}PS2R75a9`5c)Uat{NpU3;)O zRA(-F0e~pM1P&<*XWkONQoKAdjJ@tX3x{6v<&7eaPk;zxc;oZ$RtwySByrjb^+UH> z?+&EvEA49p?!7}dx3aa$dN8*lvC$O>>9$zbX2RU@)V8JboRSd>^_uAc()9N(hPNQJ!PVtPthT4h$)l5h!!_WlbBlzjUwd@k`bx45DvwD{zZr6SAp088vik1(J4xL6?6<`$t&6ostxFL( zaycb!fpFw28Wa#wB`Fee$-ZOheYa|VnMsaCTZjo@Uf(f$E=0l7)z+SoWT~Mu7TFA# z4M0@W;D+G^p`gZiTChH~iEg(SIBJxJu$qo;ir?j(1nwu44Ko(<9ERp_G8L$1qjYCX zE)6z4Qit4D94#RaH&-_A+kUYN8Vah#dU zX$w}2I)3VSZhWxBMx677^c|FyRRiK=ORKK?sAb2NO1SbDh<&$;TUwl+S`cvy><)M6 zSGpn8M#ZP-)IoMgDN>s5$vvB3tFuQO=3&XcDmyyvlrsq<2cW7}u8O9Ifj+>y=tyUS z@o$>nooHXQa|!x5kY=!q9Mv=vfpD6$M+o72agY)s;o&PN1t2w)e{gpE!@&P{(d7;x zt2ejcu4+s9aFgU5o3!xL0F|e?`*All%IMC@;3Ufmcrd{Nrh$UB-RLEZA>S~zN#k(a znuGxy6mIj-V0j>YU2{HW@Gxa0b&3eEF_Fdz$hPF}L!&_nf`~9&_ueoLrcB^k_jQPb z`x4`Y=2A8Xxe&axws33hWe3+<3qU66X_F#`4RVXY#KPho z)@xtfj`U3u)$xZ3p(v!A_F^;K42+dOgaSHL+YWM`vhMVUA&ce-#pAZUB%WOw_=q1f%pI|yPZ;p3^rxz^S zW~YJ|cGr?j1NCZE324}ijOW(o6RildNZuR@DaO^lVXFK=dc1UtI&xWV1IeE$ac>-V z1+9mstOXsJ@U6%h4ab8ojSF9NBilN*2OQS-NQW`*AGwka-(q1iu4)r*=TfQ_KRD=RgDP(Ir9B2$4jZ5q;ni8W5`p4G`mytZTvg+4xynI@`rjDEG_Q zIX6`m9a@IQJO5Owm?&uF^MWL`A7}>1(OR9za%8>nI!u* zqU=&IlkvL^1?en%PF!L`QV%B>vu>EP(7}jh*SsXcpQdO~RE$<@0QUf^e$mTqLm} zEkQ|g%!`yk1tk0^)^dh%2Tv-&2f+x$FBqw9N#JW0kc7}04DxQ=@VzbU{fypOn|xoH zQ$eb3p$_AKTFR<(@XR|b-sJWJ*BlTJ>NdbS}vI8G4A{DVObpBHXQ%!Nvmswh66 zUjBGx1UN`|Le|fw&RwAg^tuKi!O2gG4K)baIROHot(H#&(l5+_AdWjBxjaz$E|2p% zj*;$|Uz{7lh!qOGY<3|{(=(kTif1ZqMn_ubsp?Xm#*ga((B!4w&~luUGCoS_k&d_ zReJ6<9knaluJ*N-W3b-C+Z^|DxT1FZZ>QgD(Y>oY_7`r|t6qBtd(~H`mwVm3-o0tr zdOh3P!uQ*Y-sgR5T&~jAt!A;aI>}A$be-+F+{brtH0z!2=cm2Al8!SvuJgT6YsJjo z_G7(tXCtS3Y11mb?AJ=!o7<`?Tdl8eI@?zFy7um)k@bv&hK)b){HG7<^ zyK*CA5Ve!DP?xg@z0T0}Ztk|TZ+2a?+f^FcWwX=X%7?6L)86&gw(Zrnu5Cq3ZF_Fb z9?eg0cJ?w;+KfHdb=xU+x31mKUE9;UrNK%I%FSNb9&VGp#@2gzy?g)w00000000lN z?$Y`0``g(lQ&8=00)hE&iyZoS*C>$~aKw|l+r*H69W&28@W_nYmKy>-5B zt+L+lz3*>pw?*6So_WZ09=ES^2j0(5cW-uj`y1}H?DJc9ce}mi?)IMQre^ay?a#ND z-)<*9+vlyhH@V*C_j}bn?{{*qxlKB&ySjVb-l@Cio4n-cb$6caXMNv&Z@%u^tv%tR zZhPHcx7JnLyT12Jvf1un>p_b5y_Y=Y-MhZK+uvt;^=|97>U-WU-t6w3b#ea7t-P?CQ`R@BU^||a<&eG}a z-5K-U?TfqXsoT}=_mW+ytx@ggw{4B>RP!CpuYBivd}g_-p5Ae8@~piw_p`gW>)D>&E8Ew(*KGr4_hYWFd?KCz zm@1JO1XI8Ol&AoTpMjDxnyMoK6!D-VB`RnjihCeTf~brF6%Yi)DrlKM@M(o)KwbLx z|G&%4ajp0J!Rzg|cP%|$9#+P6vGPuhPYZfaF7_50?Cb-5it(+5s zNW~RV7BF>&1yx{D9g!~9X&~ib(3Kz+KadUOyauFKX)I_*0$qZk%>n@0(vRvpG@W0P z8B2$!W@hVto@X$cd_BtkkR z_7(~JA{}T6i%)i718eVC*CQBrVz{0xm0m9bXG8hi-;^-rYuvoNj7Za?MeS8n1vrOkkBzE6WE|n6zP>Mg=)6&bg=d z*(W(h*Kj@xxD%a?)8s9Mr_P9ttyBd08Brf(;8IX3DyN{^1$AbJyhai+NqUel?M1>$ zHO%%Ss;w=NG>y?CmlJz;IPTqO>Q6;hYOqr0e7{mr*C!sLdPGJ69jLx5== z%NPo2l}5qdl*=H@y&(=jkHO7m0bn>F0x0O?C`T%GK;d%8g=rV_EUtz8yWr`_I9Xv+ zrq(9rxRo+vuw1hP2{t((!hLXeo4V9jZKyU_z2MBwJwWP(*JxZjSX>a2IEb~2aiR;$ zzJ-l0=eaj1v3)e0xs+2<)EGQQJyCe)MX6NCPE*s3BUK4ClIdO9mK4*JpBYF8uf?3$ zX_WY;2t>=pyN)0tBTx@=Xf#@QR~?r15aC0*{k6 zm5)C3qRGAE5z9R-I7MKWEUGva(aXY?%>3NDG4vuK5?1r0Z${!g9#hi`p z7$%wVl-xHurUQCy`cSN&ElsMqRCx0u%jFz)xq%3L1fK0*v;zW&@7F`Zay0c!9zZY@ zaqJh$K_EDhkZu5cerEQ%W8#Ht_?qb85997sIKV;GyR5aS2wok3K)$@3(E+O&h-igG z*I|z@K~)=^KB}1X-Ah^=vd9%aZr55O-t-ZrtgDbdzg9xjbAPEN| zK01i_EX8PuN0V=?fopIafF-(vp8{!;B|iCYz#X`pn+DsyZa(W3^9zo36zN+TKFB+O zZ!+SC5S?H$!C`@qPAMtWqRbGZ6UB$FvZxRE zlz&;_op$XetX!z?DT14KHt7qS%&z*tK)`inJ|aM4Dssg(7eIqiN*~)Ra6=3D=F9 zg=&5h39(V^UxD^!Le^__LV+MjQS!N3_Kbp%s^FQT3osOkjwPsv+d(9RC9#HnM9gPl zx*ic-i?xAuyMqTW9n6LuXeqyq=N_RT$B?Q^?!YVV2B>=;=S)$XqO8YbLjn=26(m#{k{Sdfx^)49e=O;Ae1+G7)C>a5y|D`1WkLmEogwAUR)dovBz zl(;pm=^I?;RB2%f;9ms^3Ub}2D7RdjIjX0)wMNsR6+wdiL z4TwHe+Z~N9g!ZjaKmsYVq`lf(`pONhVY3Pa)#ql-sqPX}R*u&zpIV%v5@|GU3=*s1 z=?M2&DwEzU0D#Rwh^H8N8IX;GSpBrFaL+*3*o==MD0zj_qvo7Ne<0X4Wo;KrL3$pa z-5cqRcRRbY>k?2+9#rg#Rg1U{cNU=`Q;>{a+r)Q`Pae(_-n8WKNJv4Tf}7`3Fe46w zY?d7b#3j=V!35ekhGp7$AOt~pf;D5!K}e|M$G73OhzK;$*&?MeX>FNPyh*EX39VV{ zOoqi-K>X~dF z-FC+^&$Y_Tof~pmuBMEXI%`AYezO3@Ndyv#5g%!8nbkW6n9!uks(yUD8dxEzQ1QwE z1A9Buqn;;7rRxm`VNvNcBS@2h-)T$i3rOo8! z=7L#GuRVtw;BAoe3gjDc&5m>0cePItEF<*ZZAsdWPMI`9D)Rj@^6z4SGh))TRf{mK z83i0c62@eHgTn6aT9ra*=CEpUnT!Nd346E-%k?X{Ajucwn{Yv5F@ao27!uUy3N+4g7)OAn$Fb+$9gdU0*b~5RHb) zI8$@eP;EW3gE5t&2!jYX{S#_AYqQI^Fs_DRSI+c16RC&cd9Him!aU>3dL4_0$VO>M ze#U9J5U3|@rdPhA!HdYAP)+ZL7@3VP0m>mulcAy4+{`I_w@OQm;l)>GaGlynnqa={ z)s*O^lOh)u2_yDHEy*%+UNW$|A`3FuX|n^l;)PvT8pSn&u?+$+QKkhj8_{QOB@DN$ ziMhc@d)5+7$4`mApkniXw70k;z3bSYbwwP3X~aRePoXW>@K!z$de6@J9M5vX{%Lz zu3U;uE$Kn-WOz~QuocVD&7?THKq7E~zUx;>Fi^{R!JMU&Q%s2nNY+cn!P%@evqSNZ zovd-)qw(GsV zKBsaZtS#X&tF=C#6w}BZSaAW<`4CkBFidsb%P7sLcyLs9ru`lBqdFO27Cb2`dOSe@T zeVv}`Ce#4H!4?-*Po!=l4Y)=!R#?8n$op4V^61Q~F;!JdxuO-D-FHL`Sy=||gArn5 z zauER!QXt{?$s)RstwTV)duD62<1Ph*kk^Bdy^USnULPuxwR|MezG9W`%J4uWmFab_ zbT$NNpMfC4Bj-iYWf2cb_PUdkz9Bsxpg0?X<59bJ7|>T6Xr>=4Ci|$w{HX>!4eQ2U zS3{LB-c0HVmqLl5#~V%OP}HEDOj~e5Gl2A%7|7g7l7)F6=@(A!60qTKRkv7f6|GNo zWVBQTu4!=2dJ}nuNtSMrx@fww%KX{8QIr2qf`@nk)Rm9cr`1> z5y%M<>?3`xeLK`9gBDBKRvL}D$4Q~rbFvxGCMK0-kX|e(#ZV`rsD{@Sre()~*Ojs~ z>S3uca)_|*!gW#=tb-#@LU~*;bQ-&T!>|vxz3S7Aqe~7sAa|GIl3#8I9of+Al&K`% zVI-5{vHI#t?)=lyYE&Wbdf8yiv)dsbfYWB8uj}x0^o1$88ZBWX*%RK1-FfOQ{&e%Q ztDOfoLw4tsiuuF4(^?dWJ|*r}!nD>(DhDpa7s)yvmHXhe$44^WR-$1RbXM;(2ZWLF zF(5KsD?ul08OIKv7u(J*(&)-4*2B^=&Tp!CIm{50KDBxWl|Pl++-}q%AT0tnoc25f zs=-4;_pS!HK>UfQDWKVHd<9z*Ej5i-JG97Xrt&O`z}eH7_Bd634VJvTO@XMLWJ{1Q zT1lhisimmc`SRV_@09NLy^eL&H&Yx%wyBGqEH?yIMSFL%ILf)(j*hA5WOCh5rd!0F z!_B&DaGV0HP$Pa1?o8R^byL}ZcAJvweyj9VrgOz3ERQ(NM27sZF98Xq!86-dQGy3fs>$1vd-xP zzj?1VRQ2`Lo-Ql8+)W$`XJDx76nqQ2nzr?ssy0q`dwf5<49< zo$nvM0A_>u_H^<7Co9An&|h&wkp_g)lO}66yes`fcz0N)KF|t)T&@%Kz3-o!g`F^W zH_2bG0%LnONr*I?JRBa3-D)o5S5P34)I1{-aiN_BkubWktt1{^nMPttpD}P0Ffc*- zgf#3A)4=Tp4yTQ2D^E?r=k(;p7!%{xO0w_`ZKL^KMMEb>G<>9@kQidCx$80%HV%%B zy-A0&ZoN~`KFLIPsJh4&DiG{;wxHMDt0WzA6L&E-=xqZl*g9<$3)|5z^Yx4PBI}FT zsQ69D4ptn^dHBFujHqP@Fl1DI$~gv~7raR!1{S|48g#H2F*g!UDg{S4!>(R6TT}Mr zv-hE=twrZO;rL-Ra11%unkmtYJpwg(axFlpkc6hB0v!UTLIIT+4kQ}RHY=W2b0kD* z=+YkP?G|%ba9iFEns25}F?eKajcWD*_$0?`%YO7-7>?5s(e`R}L{~=nvWHFWuJIq} z>fg+PDo!z>`PUj&5_(X-x;xRx;W&K>j1u>fzbQgbBJ;@kidcLy!zVwCzQ<4oYdJ8m zfHGmJqi3(%3hg7}w|X%ORi^c&M?}PpCJ2@~wlyU07keY-GJmE@o;D`t4ff-Y1tInt zpXAXg_E|-vN`-}C{#1EvV$Q%^-R$kIARb>MGP4}3PZ3n?t-uzDBF!0ZyG zuIpu>G9H(-U_dbyN6em^wuV92l=D*I^ysqW;&suj`8v(W72wt0Np16ac)SV`OGG^b z!0m}^M7x_}`i0#?MNf>$P?6>9d<2ESP|2*5=nNw)3Z&0MrEbrTIacXs6!;HgCAy8U zVLj68AxCWT>n~E0xb&U?5is?nT>LcvM=%cfIk8ppMf`PuNr2%{(qzwXS#nZmz$XFw z>jEp1TOm6XzT9eJyv)kzp@(uAjwCdPg#acZ(rL>Z)O7Ib9|o2Y*KI~3K2%Grbbym+ zG~|P4$NTA$+=8`GsETAk!OE2L?Tq)-yDP`;JHmK(ZAON!8<2|P{hl0yX3M9c7Ukop zpg{DArwPotI2%VUxYctCsMn7m%NBIpRQrn$Xsp)&QiC#0J6L1M{mD%hTwJ zWosPj@xbXpCv{uh50pWPt4Ld*!J!b{xla#mW(9#mpDn7aQZY4Da+hLQqImT8FDjT| zYp0EM0q>}E#vH~x zmJKP*njbOB zd>dp8s+&YLEOD0XrJOBMDG^I(;_%fIb?LG7O%e}!*viB*j{GZd1fbj5s_$BWP4)1+ z;i|uuopH*GF%Lfo9wVD|O-j3&n#}ogXG9MIL@wQcV0JBM zCm)a*3#OI}WZex|#nF`RGRLUO1`pycy`ghy2c)>hbF%+QmgN#D3{-o)2BRi>P> zQsRcYS(l(#lZ}ore0+AERYp+{M)hfC(D#Xrf*y&JuuxySNu39ga)e?F5l_zGr|O`_@R6(NO6HIf|SwH$N2`l0@_mwRyQVB{AQ+5$LSs=ufJF)Nw`O;U>$;K9P>rJpi{QA z?5offr;=jASse#QqOyh0L~?-u?h+dSosPPa~( zixwb*Z)*5>R7#A>U0QFCt;Bd`V=_@!u1cJfe6;bdSAAHrsUfeE5ba#0MHW$PV>;}!E5>>03lYS9 zUu7OFSIQ$G*nlT0w^;n2+Y;B`$9InrEkU63}RBdtgMu6~Ve8 z6-q%ai|_|@{&z^1T&rbPE83oXt{wJ%B~)F2d`M*-EF7{K1*a)TWH}v^G7mV*8Q3f<}f@6c`Pd=quAzfw~8QUag>5%4&=e z=5tsm;gb`>1j_+i!k8RCEnlMM+Pj`ESrYM5W)^6i#-M`SwZQ|Wt~SXyEjA_;k3z3j zQXqp^Subh*FK&mEUwBtGN9tdb8{O`9KFEKRNvQ}iNm4y(!||}b?AEa*%Ati5o;$@4IdJ) zFA^NNi7l>(R!pz8?7sUhDKs0PB?@_+ zupFXc-i#to2X1nxY=)_SU$}xIgR8WQ=+@jJoZ6=5%Vf@dxAn9;KMzG288aYqJJg14 ziiM?hjgA{xI8IGKoaCtHo;SKs5iC-Gg?FvB8h#;581TL=sTqK|slV3Q(i{zG0m1KW zPT_7I5zhqtVRbFKU(KQ?emKG3lk)54w!`A`gLUzUJ% zTqh+}y@YaWO+r#Q(V4S_UJ4JixSLS_bYsb+*L5o zrCjn4Iiz}R+@jXdDk6cJ2~;V|($TT56pY1G2LXDmjm5-PzbN;qJ0&6RN)r6E44#P2 z&2~pKb{sm8T!7cgjI)~(9`$m#-nBxFO_PM52dT%4VDHSxD4Hd3cUqS=E47PV=SWzgQ zK$77?g$`e{5o9RqB(g;NYh2aL58c5da<2Jl#lu?O1RrGK$G**0ZW(!mWdoW8?JLuJ zzi6!EUE!CJ^E6jILO1Q`BB>XxNJW+SJ|F=PgbJ2Ts!DDTBAnSL6fra820wcbPhNVD zv#)`#c|G5ge>mc=)9$(G-p9YUH97D&H}2!Poy*M>>a+ISTj_Hb-O#Kp<#^|HLt%w6 zyxFExn@zEyptx5&N|zVZYR}7)F=Or%=mH_$wKJvZZu(OYq@a;rbJ)0if`YtSjUB?p zVyT5@fL~03!njhxHf2H&J9VuM8krTqr^Dwr6d4D?$0HXwcQ<4ba>YxM+z%i)|MiksZ3BXX!m5UGl93bX{n<4(^jg{=GOPZLC0$TDB7*) zkqsULK~E@z^{;4in^T0i*vD6=0UX8;C#$unY|Kf4 z2|%n~)y4O!UFHLxo$n5x+6@O!sEd^*B?Gq(gZOlL2;3gTS70M)RtE&uRiqqifrOW- z_)?OWWGU*zEfV=GcPL1yLR8j8j>a>>R7E9l(zS08>w+t5meYG9csHjhx`Bdis+>h> z6w+C)E16vtQ+q=s==pHeWEwSt9*yivBem7q=Uxn+na8OB>v5chZ=_fv2+rnR=VEF}*p8VCGV_&9 zfS)>?E0*bYdIvaVpeh4t1H22vV~le^ebD35ku<~hHnKiMdXbj^v?6>Pqimj?;@RT&E_eV$!a0;F@=NAv7_Y0 ziXlO)_*W0W^HRhh%Yt*$eLKVE40V2AQuNjgqoC`M6?VeP13m!Hch6`YK4)72}> zuQ8t4)@oA9+_JrK!Vf5F1t@M)sIk2c3RZ*05Eu_2%}q;2hBn058oF3E5?3Q3P)jhB zvn5>y!(2_a*xMc-rEm*#)s8tTQa$f0^UCMay6(k`*a(z0G|d-P*7`=_`lQ>Ain$(G zg5k>?yH1FBvtN77Bukc5R?}DJ^OHAi!BznoFde94%S~!iLbE-m)#4S^Z4j~r*&*+M zMnx*|bY~_u!8ArnLZOG^zj>*GimvueuIG=pADoBrm93Slq?7C(ix?=wxK)C-4os9c z8Jfx4ifFHCjNUMU9eVuI^0(Z_?G^1uy^~I?yYL-VIC()Tj5p2Rp&^WDTw}3_0JBhp zz~&(MbsQxRYH1{?4KzR)SdF6W#o7_nQ!xV6dqv?W@wtZ7dVwy4Q|}PVvou%zbz@^k~}qAgD`YThg-IHRZT1Q~72cI5{;FpfD;fu=BY)<8EJfM@5< z4v=6vr4+w+f>rr!*04Z|a@|#dDoOP=j@>13wejwtMEQkg`VMo`-qOyj)(=t&uzK9( zfI!(Nl0n*JF*RO%GO4Zi(;yCAYt2w&JvwFPDhFoGRJin+J)G&7bG{Z*c`#GW!A|hw;PB!QzTMX z!b5K`kJ4dR8Wr?BmaM608t0~RbeocKpk>i-Wylkx0kAyP!mr4)fRQE9ORVlDK zi-R2bLaqwu+HKID%L;^PUZjA=omX_pq`-U(Hp?m~n?Dtt1-_o(KwRy8^0 zN~a8Wc2FY{?43=Cf*}-=G_ur**7seWGJa&)fSWIPAJTm4K7b_}BY@{s(Urt^ptfkNx0KMv#mJ3As{e=?$8ei6ivgRs9`k5or^@|Qzy?UZxA@|c~$1r zcSli`6*C=FnGg|fLcG`uL%$d4d_a@*by0i zD7Ms2Mz}5ZK&WieRX~bUeezqR9xtrkib`ovV+d?Ua?#U;mvTzVE?>$LAsGswtgtXH z7e95=9Yva0k4O$7W(jkwK#!E}9QoB>z63$V3quVFGJ%q~+!e+lvTwo_lVeb4ou#n` zs}*+_<7LlGLx59Y&H)1su#VhFa8I+FDG!~sYvRRa)4PUzB7!BuIw|Q(g>Dskm%dzu z!sH5<`@nsaK<-`gG#}f6OV^*e97UemWRQx`NDsYR6ZGcOI!{jkSkEZZCs|%}?TJjfpDWPaFABk2b+n zuQEJ0zs18X3A)*2*8&S?8&3MI3pL|jf`!tc5aso1gF$H6ltT&uJ) zsVC#S-krj7U?3BTa9U7mD&sV~IT#ruMkQyRlSf9ir-`nC=+2?n>)Qx?QPLDyQf-)8 zZ5~eGAhITBe|M!;+tkKug0Tb}J51h{tY?)%m$d`c%M{vM+=Z^PgkpfE{%Sejd7~Wi z^S;5D_SYH5h7GmN=U01Y_Z)Gjhov5!n3aWf+BC)E*(!r}cA{5=$d@=FCq#S#>MMr< zu22PUDY?aPvjfr?A#@PQwRj8c&xGI?1375I{oFNd1S9qCcm_I=GB^gqFOs=Af@!c4 zfq`M>>Dw7KC_PS*@reREUC*h-#0@yldnhKHn zqO6NQBjh&zb^GLmAr4~{US;jwB;f0uh(9Kq>0(!;QKsF29?MP`9b5O!tyJXD@5u-b z^e@)Vk(@x(g5}+Qz(d=zAv>s`t&v!?43nZ!WNP!+w;C%|Ogvph>txS%@<*;2Abna# zH<@=;bci*?!R|Fy;X}fGeyu}FoVnC#+q!P`p{XZ^p*^{tBt!U0}NeNE!o zR`FhWp%RGNY7m6ELx@0e#p|P$y=ijrD)NeFV)JnOuDWxSdPf^BcvKuZ5Ef0~scXCB ztiwy;^+-KmSU65z=(=V+6Ji76wFdJYinlnS`XugQq}T)qjVm&o9=wM(=4*+D?Uw@P zDy1S5H&K2%x`ZPSacyqXs4ys+0uz)3$DaDx-Z=(Yrf*{S_ZMU@ZM=wFE^8ucU}_2sMNfw0%R)gqzs!|C@(YDnQVGd*&;AkmXrhPGqG{E;RX!!!M937 zM&YP>T%$zg3~w1>S)VzStqS&~k_={*CVE1ao4VD6Vy|T?j|DSu>}6)c{7^r15G{cR zsW5B2$y8iEBMFPLq4{UM4k0wEvNDtJCV_K4y^~;CuF6p1N-NS{WsOYCFf+N+tr*Zc zjaM8%yZWysy%H(+6vJ%`F(-IP4h+2S@u7>}J9v|tr3PlA6YW-!e6$1EMAt?hdg6*U z$0!2ckdv(*RfXfb5Qod05>bB1R_o{49cpWO%u5%eq_dh-AI(F zeG!4G&Qehxu|HlBScBACdHqu)Kyd#WDBu>dx*ucB~wZJcP#tsV|GJ1OzW3 zSEXm9=!HwI;v|KJ+F(?!MuaIKZz(O=3e5H`B2(FOXsZ5!DZdBv%z{o+N)YU#p)z=$ zA-p#qJ@_Fi%3;=sr@=~!GaP}yKMIGWQ)6R2=N4tA$bhE|>u|gvM2in|cX&OWj*8nc z;OFWppoI?y4pU%R51-2qkU-w%&y}AJhjl9~Md^^95QCofA6p4sMHk*ldX2%pYodD8 zg648tW)mL}ZQbI_-i?nTQU#kq{TS}y)GG{Rs(y~Z_4Fj-h?i09xv;A5Vkxph2!+Fx zS_L?0G&%Jq8Z6vdIkr~&Ru&}hiHkEylj*)BJ6iL{YFX2Zi9p>1m6r;tzjSyTV%%x8 zZ5}@*(^+|_Ibwy45!4CgDm|4WGSzOPU9(Htw{@dr#C8ZMFN!{9atnP1R5ok+q!|ND zh6lt7LQGYBw`%H|yv-4*4y_|l369~kuFK<_>3VxlbuyEHfpZYH>}`f}ce^W#b85-G zFe1ExhtucfyIk77A~2M=FjQ_|J~SlD%b+MKgCTjfCGKWrk~Jzr@JS9S9|MFpL?Ehc zwTK&A6*lCCuWG(neU`i>mJl{{rd81a7d{ey7**U2z$t3|JgIa+1g+ZWmvD(MC1>cF zB@xE$K#&m2xr!`5Zx^Er3`WV#G&sW#jg^o?6yH)GRM=kPS(fW{^Y4P?8RSb%J&LDZ zd-Q_dbLeq!Wx8{5;bEtPjUb3JY+LQh!WIq1os&8elYN)ldZ`z=uE@TW=!=|sec# z2UQgBy93fNvK+V@XlOv*2o6$=)1e_>D1q@m)N^$kwKtHLa`S<=gu6?s)b>}+c~tq# zBPx8lj|EUXHPh%L1r{H2T((K+;G&EoqWM@DaOAV2lBFu#F@@Q}#i4lFiQj$MHr~sK z8d!pc8f?i*#OTYH%J!SeGf5vexP)?J2UYQiQp^;>5QT~qN~d0~@F114;>VTx(mm_K zdvOf1yDr%Ek82L^(ner2)ka63tlAvN!~{<$u& zFKz>4!Aa|5p)!$9t9-7xd98Q4);{Y|u{snbge(+_;272=n-$6enIVz) zLYav!aA@f0N%(1yd02jpM;wd}``dvLuAwRGnS;>IV>P&iM0Rrq+0DzjdVEsry~A5% z*24q6p$U%mrGO>Q3EEgoYtn{cH8Wf} zOF=@eW$2CdEF=cjU2vYlmNrwcEzZt{g-OMfzE(lEkCm#RW=TqbxNv>}_&6KbIHbfH zcM0%RO+4ey1{g5nv2E)pqq&8ld^SNar+7#mt9v^Ih;4vD!?avKC6uva!LosT7Ij?s zlFCw!zN>|=Tjc5XgxIVdlU~ItB(x{q#MWfD#I({-((bi!E_g2FDvpyP6au*K8q&Rp zfIHjP-4@~K25p`aMToZFLBe1x`V(+`2 z7|-2Ce%bVnUc^l`)tq5qjBQOI%BZi2j(~zjfXWBL9hE_pZQ*Gt6%cXqG5yzMei67b z$7^14QEvoV+SbucUYX@)TYTfaQi@G-xOn13fN!SMUOBk8T852WDCL+S+D)N(jzVtH zi5}XFW$(EYE6tEDrFlL~AeY%V7L$r)k@G*A0PeWna2*jc8D?)gs-bqW^F8#94mhu0 z45_KM9(y*O<&}RC)L0J54&Hzf`uafS(4uYm1R5Ess_?mkpg7IuW|7K8uSGi`eLa@o zMRlj6ML(3+tN@$VXtdShlBhRCEWL@fhqX)9e&o_d8I%o*>ubHhG)HATzmP<1MnQLu z-MVKs_`yI>fFy=BMhA5Q3Xf!mP_-5ccDZ3D}9QPjzAXe=)MP^zu(D3-_ zuq+_`Ajc*kAzQs@x!j>nsOm{sRK$_0hE`u>PiheADEfo-pd(!Rx2R4(ytuYtA_Oa! zCX#8Ti_XLI68WqV@LhU%&J4vWLx&z|fT;@brCT(z>XwBm8jlBM-SO%+TiZ+f8V9Qu zchg`({1w!4_^qpnO94=#^b~N8QA3Dx;|Y^LGa!~?<`QKV9_<4aj%(7sL$gx(cadp#7uP>Yv0E$h%$s4ARh!^x0O3WM&oYbc{! zV*-V%fL^?T8n-we@}RQs2e>>$9uVR32;qYt#2Qp#9KfA${fphMCw>eO79l@BWCPXW zE@uIe^0u}Qnqk!$I452@CZertsOd{UpdcBA!_}u=5Hpc6hpNM@DmhgTW-xWMU^&V;@|Qldal&x;joqSnY?Z6!iF>E(QIaJqL@ZihFet9j zz;X1`iP#V$H`Qz7;%b!^?Um;hpv%+M^%&IAor5nH@456D1K7@+N7W#(8~ENRZ9;O4~i3UZ_Dw`9P?0D`*>Quz)PF*inR zTLVstD;4g<&1~(*ojfNDsKN_NS3b@1FDFE5|z^qO(&_H}s>h|DzWst&6K4JJG$SgzCRJoVF zsGgvxhzp}`v$?W<@z_3Q8R$x-6t956-vbPZ<)4rF7d6fNx_zDnzyQB;FU)P%8pt*@xa?@;fXgs=FvT1#`&6jYvZ%_JLRqK}I5Y2C(Roy0Pr06u*#;hcnsG8wPkA}Ue@`|WV7I13|5Ke=08HR>F$sF}o z+Ac36EaSk8GWc8*lBQ+&FVdv4XT!-DS6Lz;6LEXD zl_cI#o0!>bRgQSA?M>yPsxOqIMNWq!!)u0>S9y`Ar3cKlJ?XBT5tiI}eo&Mr)Lnc# z4mW9-JRKevL5Z^SGtFWN=odu#A|tSrL2GNM=V2FF&VvmIki@MI4&}=B`&&Fj1 zWpGkdm@)c2>E+GPwfOk7!jLSmaBOu=wuN0>UaVj=5fr)i3Zebemb_F#M@|kVcdyq# zNwNw>BV&Bi=xl?7ya5X<4nDU%7va3&IJpBOL_r2IzV~$ACN^QRBt})?ZZTJk8KmQQ zQjVQ=Jbeoa4hbw5J&@5S^D?v82-@oRs~viYGsBxuVRYI{!3UNQQca4OM>8X@$bQA>K3gQr3gjkyzg5!B*i1zdxkscj)6tY{3q3vuQiv?nBi($}(hLvhms z+1c=$<~6O47(&Uy%)a-fut*PQn$cQ+VSOuhvv+G8grw;~s8!M-yg9XUB0Ag3h0PAC z%~vs`M$9;f+S;#xL(5Q#aF8%2Q>}e)Um4(?7cLiLBTRYB`H;?E>wJ~tYib{KbZ1u1 zh(hGL`>3luA_67Vo)TQe<=l{rrmngcM`#~(8ee&qJ?*u8juIK>mdPG;%3ix0UrJV} z52Qmb|<4WWvN8(kG=?*;9om$Tlc zB02M=WrAKf<+n5f%IlJK9U9eTDRSGUax)q;0c|6JBj$7g=0x+vO|%X!(&-npQv=b{ z+<$yS17W04Wo;VkqFH6M{%#ca5H_?O1>S7zU5!*SQk?*rM+&HmD7-tNcl4zCj^WJ} z>k-nIi!=dlQu?1}jJdwUx>-VK`{kvjNw@YyH`-H+Xb|b)`awbKB`-gCsYd)U+Zo?1NlzK-XoJw&Sy1*AP4~KEV z2Me#22CXVX58J|fh?IEqSZY;Sh)ewbjJGG{3yj00ar1j_fr~tg}!(ncRES7;A91;NW=-pifCt`DrL_ z2>mfYA1k*f7Y6n&OT~3*?Mdw9J;6}FUyoINQo=1kd(H^BBg7d*eG`}<>&cPlcR`Yq zB=0V)UfBi(EIajfut^XK38%Aak3EiT5nc&iu5uN20<8sZZ6UQz3*KM??? zqj_gg!)kgS_Wr)5pdA{8Q!B^j;2n}E({*yxC5i~j@$329*tnBzFpSe9WGdybGU;O~ zw<66Tc<&^*-W8B^=*LL_c0h^0DKV#v0K$(4a633V?Bd7~mu4}t3j*m-`{M_p18y|+XsR;suVY04 zV1TuSO(Gv$7*1EUtbS6H?IwMI6R(9b@Dy4~)R{#teXIfjKp!OglHZqEU!R(WcaILA z6}td-7FaWA^BKC_?D0IV&6xF8@2?_K6R&7s+9qTzZ-|FYfItC6A!$MMz=tKMVL+3e z_XZGD1yp z?^w21MW1+?CZ!By7%Y@d_SwV>Ogg%?oDPm5_(i}1y+__ zt_^Q;r2Ed3O!ZJAsXVvNB}!0ckE&hD&Y){StSAIs%zfSHF75?*yA$R6V*(A8uz1vX zt`elJVXSqE7cCasAos8na~O@M;iEqtBRKb{hb_Ifs(PO63f5p60Io2R23~X6+$^DK z#(k4qqsPQ7C^i*jFIx;rH3~8EQgL}`uj<-o?hRh)9}fapv)JIIVe1FsNTKm19`Y)= z3c(7On(vuAS)s7Z!6zFj@;H;L!o%$91Q7Al{4tT9E|`qzRP~Doy}@IFAbaxreO!^%+%oEUrnVJ%pv1K|Z6KtDr>Y<$TylRX#Stt+TBK zdI9l`x82%0Fm#5LHPd0F{DK2w~4eJig zhg+CJe5AX%q6~g}R^7s=&U^J|3iJ2cJo~M@S2d=zX3DC)Q%JSB? z@JlJ>B~lr+iOT4%sOO5G%w>pMPAwQN49$g=IFn&*#HRHW2&w{dZ#j9M25yLf-NVfB z{UKbeicUvPkKHOfHSW4;VDJ@pTgllp70;^dV|>iE=1b(X>=1)#zHqbZfU!9OT&Dz_ zIrbRM+f`tbM;2o{M2JwxvXiOe=`Uz=QN+4z^KMIqDTerhI#1nz2xEpa*>5nRS+mNQ zZBUz;4RySrs6Mhpv5;#$Lfi}j34$YW&E$b9$IWd^H^y_aQ{ko*6ASIP4xWdBCob=i zJ!lhf@j<7Y-mMF?VVa>r=Zz$w^j7dbV8x7aNDop&JJBSlv*<+^3YKK6%$kMEc?tOz z$igH+Q;8Q0Xc@q$3%E7vnf9}k4)GI^Cd5c~gAe8K7qJ@|yHR4mhNccHV<{5SrCUz& zkKS(1uDK6($bms6HbFRWOdM3IXdMeQ9`YO3TufP4V$RuBr7nfTP6CiY4`WywRvMWw zxcs^4z3Y^+px%vUXr-u(qqC+5l6ty|@RvQG8@GPL1$0K*DlFQntG#365G7#?!|&NSdeau+IchrQuGY9+cOKZfTyetGGuFG2gU47y z*NjW`DO_w~UVD^hQbUYuMg(LE)^cc5(Wvo(!ZjhDSJAK(p(WOj7@uN_vJ1cHZTHj6 zB4(60NCDa1b^ER8iNB@X#{pa0_@yQAhw7U1ys+f0J?6GMt2ujBmUJ2;I&B8-Icex) zfuU70!b+nS^F$jO4hs!s;E=uMXpd7s;x}l=kH31+h{*9lfsV{8&3h%#U9tut!L#ZV zGVQur3Ww4W0!KpPO;vnyZ_o6*;+5`-yb=PRr@2ISD9{|io_u7e z)~7-f-Qe>TQHb?!jr(cyb&v<%@I5aY{X|X^)4p&yTFA&9!t~8tXDG<^l9codsavp! znEQq7Nv(5_-6D+c1pVo2HK5014PJK{naAOQvpelW&EyA?!NW8k0=ryvjVd%7j>|h^ zWZvn~RTY14wc$-FMj6rfjj-Hx_3GKD&W(Ys2<}O3CAV(gveBCN9Mo9uc054Fn1x_v zT;DgZAxH<{hGwa6F0#8`KJI= zCqxh24igxQH+@y(HNw?Dn@?$PhN{Y*^A-{`j7&2aMi4EaRBg|st%wAKHibhhx*i&n z?V>d)6D`o4wG~ZVtc9WOT3n9bbYa*e9j^nNAzia;A}DD0?^+c|K#L-sdoJ%36IJB~ z`UZyUOEekdNvFaSqeR6#Hvb(yEop=}(D)T#syN4m>cy=DI>#M}c}1?F(wHtAYKxeb zF_^KT`mpYzcg1`hI1e$$?!bHjYY@K71!CbfOZ#fOxs(TcJZ=Rm@U@qt zV2zmTCD;cAO|X#Rfh>`D;;gEIjrNi35Ws?b!-K86pHzCu1a~2NjHd z)GsU12Cp`i;Lj$rQHwZH@WE<^Yih#`YNMhHpgjR7&@WxMf}=Bd_+QHPhg&R4)3L(L z;OiQ+vu`M!ILEtSGAi*L*JalU3mdJHgG9VMo+T#>0P!xLn8fKQ=L$Spx#zdkiQ-F( zZ49Keb8qA-$;8LZ9_<>zQQqof6;2oW_qVRelrBU^5)6$xL+Yaks_4yEDRF5q1kLqn z7gm*Xb_2u$KEg^@yUq+;Ga0uI-R(NZx{(d$Mq`&b6E{PEAkAQHa5oMj;RGw78P>tR zG;vvgW!qroUIhj%mOooFEfQsl3J%Xw7-Sf!ERHE{1ZFjRoGPaE^m;%L>_*?ZhSzG( zvSyhPyr9zeRD`a6#RTra<|@=S-EBrMQLKXPwYbDFgVSn)fUO2AR7Fg5`DVSmy_39D zDJ%oqL(pr>ebeKKN6oTP6_+G_HgQPV%jic1^VkZ|oaSZRO^<@&ZwD|aim?Z(q-WtA zu<4qhFl;Fz4D#2^YBIdW2 zkA3ECAsXivveApAZKX{LF%&MBVasPX5(+lKOU9st$H*5OgW?8m#a{KcnyHakt!_Kn z6tGX4kbr}c+mwVUt<^}Eq5NA>;YouA##(B~iXB!GKvJGdKkf@r_Ry`C$z_LrWveAM}_};5& zCQ&5%LFpgW*SV)Bw`+r=y>QVI$238v5ZbaUkr_Ni-XR+dk$A8e(Qhf-8-bB(w?{ee z2Z@mzb)N6nrz)d@LWApH0mPM_G3we)G)?)qsHi=N!=@=S+H(tr4f=DfJfpo`jO|#| zrUn|Om$w#EPfd8_k5R2OT{ZQP2%yx|85>8G&&*BT6c2#Uo0J!Z`(qepzOI^@D$x@= z2v_L?JPIu`nD4B|yq@s0%S&`mK0-J8%{#7~35|XebPJ{)m4)bfs7$iLK|_uN;Z03bgI$BJXxI$)_?3u4^sZygIOjuXrbk_M!Wa*HpzAr7E|)hAm3 zuU~grRIv`Q)rD(-fPoNfry01&b2Z(21#BlUmA4Wy0D!S6`&KiWjA3AtQ!u8SsD^kB z^+jcn_|i4YLNVcIZA3jnr5?}BB`c0W%0Xd% z;9cPHLFb!0*`Rbao9P?{lrB8XADiI@vj!HPA8wQg`AR*V))C29VK7-L)S=2v3Y;HT z!BM^B%x!UU5P>Ebdu~w zU~XB&CX}7ORKMD|$Qo4{0Sq=f<*d?eQPBxfk6>#tP`|MaTaa=S7DdDh3~K5EtH~zP zpVPqaWgvAdpfq#!j#f@oR|{!-ag@EQau=_gdqB%A<5~eR5#G2lC$Jr96-6X~N%v(# zyOyagr>cvX9Yb$~8?y-j+$Rd$h`us3Zhk+Qe&$mP78h#JNfA;G^5W&&Wk^IShM_1u zoZjA7e6=;qLRB=*(Cm|Uri;p%Ut*atbS4zsP=i?Z7lP#|#qdGnnSgpAhlDrW3>*== zNRWXrl-4y$J$?yqWEP+yA=RP;;xMo-4};EKW$&FC+0u?H7L2N10kmgKRBK4` z7o4WLIUhMcNeI5S5nuKIEVsau$3*x z?@A4BosUrgaUlh=v2A+%@Kxyqb-bsGf;wmEhNX0iFjvCy)WV*UilrKitMj51;=jtg zDA3U>rnrKkG2+{Br`NqQQCV-U4q&`D?|8*)fgI06A-xtwsh(x#7!9E!vbB5FtOIQ6 zy9{|xV^D!Z?r?;-M@Ar=jV;@u546hn87S{voPB~+Ilx7-<|a^}ebO`ZSc!;RAxOrK zt;2{I*BS49Wr10mU?6rehmBa3h|DOa!qBEdE~%hbs;TpxEz8+@zPJ0JGEhUEU$TUX z?7rFJKVQmDuZg0`htXr1j6Gnt6^wvVHKA-3!Jflcqn<@k=nRpm)B58#TFlga?hchbq!1iJ z^72Fm57DR`JE z>a!CT(u6j6C(xs{vVd!H*ySsHR;71{TIF17Z%=M+m2<2y*={vLIrG}5ao$-k+U96$ z4He96`C{86g(2)92|)ADRbO(f+3+AWOVG=?Y3hX$#?90SxK(d*jby8mD_b-f3A+u# zp&ml)^(oK8I-txbKa9zYgQc5oz535Ij12PB$hedRC+bbO3rG>^EeQ8M^He(3nXf7* z7FQ5@GV$EOTc+aCZYNnYJ;Ro-7;Q3GPZBk~msQnYt!`bEo0*?V3#bd^OR7Eadj_lv z83AabhwI%EPNb-q(3N?5wi;YCf_oKbk=##v+6r&9(UwGcMifN3Iu*;2aDW8)lG5ZM z3yhwi$X3$_(5n;F)>_Rmncj{9grP?AhOK9~ctRJW(8RMdhSh2+;l)-H(UsIuz7 z+P!Bs+KbWkG6WA-(p#wk8OoOss1B{>@gW7gjT=p=468$!prO?*5%l3vRI_SgK#c+x zrtH!*KI>mB*Q1wQLSVl0Hb|mYfhy23bn?~@)G!hq_>~h4K{Tl+N$IT!OOXR%j4o+} z>_Hfxs=QRrcN(#_*~AR*YFZ0$Qz;)@?#rAJ@}?MF2zRYBvdP$R*vgZ$mF2kAX?1%e zi}+$&fEyJdiVz|s_ZFUnpbJ;!jO;UL)uW2_T>{q!s=U`JMVX?wrI@?`Xpi?GvorDi z?Krx3vB}RB3z@Zvwf%K`%Hy;wNZTQ7g)F`8iv9fFX834an8flAXJmzb;EmPyQwe)@65>F6134UhF-|l zi<1=wh&x09sdiDTZCA(It*(4uo$+OHG}>Itk6>^O^OuT<(Wd*C>}YzHXOje_Xh}$h zK|-taoykfHmyOiBrfu*KYsW$+ocae_>DM|`P%b78vJbwDm4;cNm^vd1i5rpTlhM=P zYV~2~iVF}0WSz~)VY{w*l=$#`{h%bFidl*qmZkPr|k-CovRl$$yj z=t4CgMgAD!h>HYTzX z1PdmNJ1Mtx6X)+P&g}h01Hi(4mN8?rSsX$4RESw5-g&>t8`5PXt``;O0YNhe6 z4FVfZln`%S_Kl+MnsST+&Qkx7j;LD0#S2OD<^}~rcyL6 z1O|eedB%0uUpIJp2tJj1gzm_j?+LMbX|B4#CJaJptq(c!KHYRL2P>zsqISu*dNoW8 zYeaz%iwuNrxNfObJK9)Xnpl?k#`KgI__prJ{C>+hyJB??unMDJFaBp7WM>3|s=;USFflYIe7#xQ_aVSGQej9?l>{ERIfQhy%4E z3gJqQfR8?EGW3``As}su^y+d!wydcm0OtY>EwOl7Utz>W?t-6~N$uwMmV=}yxWy3$ zml@Zr^|WrA1hpU-3{I~~+;1a&HDz#b1fnW9pxiT&upH%`zH|{_iFc&Yi2>JITLsu0 zmjdQC*PFdE5?AJaY_Ne9Wxj|w1~{rVbk7zjDlr(vfxKUruTg9{+XgWfp zZ2asUjA*<#TY~nLJ_SD1qe+cYhct8<=vEtE7#oYI8z%7JNUX}&n8%m9(kSgD9u6Sq zs+q?=;v>IIPB0mYrR~FM zW<;UMh}qsfgVmcCWQaZF_mhAt+z`O4HO^t+a=o0t8A*!|aLfc(EaMJ%Obd2}2+=L= zjUR0iY1{FEc{kc_AwcY~YLvaSBse2APXXa5P_c@AOoHoykX0q>3V!lf_CETG^VkHg zTj}>B436wdmvqOG@Pe#|s&rNWxNVe#DH}IpHlpxg%f)3xUM}-+b7WA+p=37+cbwo& zjzt$O#(6K0Nl{b~sy>X`Q$U_CXnOHc`gJFIge;Xo;F-9k=e<%M=P2QVo-TJx@N^{m zvVd{Ma1c+bNJZ z-Rww(sMG;j_13M{!V*TMoIr(w%x$rd=Ypq06pOjN0=PCciA<@;YNQ||TAhDkQ@HS) z*cOf}CIXb_CSYl(=K{0U9Ol}078VgZ3sa06r2?%3Rv^xEn?q`?a@azajNy84t~6X6 zvcPrLRjyrvhQ&`g*mUYE@2D0{DHA}_4Si9&jZdBf6j_w^f_XZzR)evSnRs*712Aaf zRY9u2)B%HYMD_a;ZJZ3}1JNy{TxURc!G~Lu#i?C6Wi(oc1__)h6~*Z?V**4bH-ANp zsnU>6nq{03bAb%UvkZ?Fokrj@a&2}lE;X%lEoiA3xtx-NBndA|4S;lZqhj``k^``9 zxyMn=1_Ju=kM5((tvvE^A1S}6LA>tTVjQv+LcNZP#v{?M*ymDXCX-cZ4Ce(`h5S0Q z5R$vDJ=6wLEd$~_8rD`Aw`Q1eful$!6|gGL2u;}td9=uHBt<-ytM5J@p z3qw}rN85ua`D=EidEu#uVK*Tfof7LfQ(BhluY;+i(- zt7P*6M1?d|i0J~>f1nVNIILL{V8!e3CX8JL10~LRYP^@d(F$@|uHH%$Fl5kc)YBae;ocoy9fZ5nGrCvZ zzB>lLJZHN?4EdoBZUIR`GTTS^(4T? zgp=U`eQw*jEjh0h#hpW@KaRYJ z=9X_G$6X6S0uy+oTGlEnC>pj4CSOu^D)439uGXtL4w}e^t+2vynCLQ;gcx8!M%0A7 z%eeDnLT(|o;bdtte$gI0U4JL4MXl9Ewr;k#D^bHDfj$xF(LJ-)bmzcT;Rs$A5FTN) zAkQTzWjdwzVGqYs*%hgBhw9aKUYg;(@p+~?2pMCOqE$FuTQL%>Rk>D%{v&t3Q7)bu zMPDP5p~rZ^#HCYIBC(f35`Iv%;Sm60u=SkcU%7js$~4J?dyQ!eoC=`5Pt)kOgjnpU z2Uyauw@Szza3p9;x!Ro9{jFBNid$OW8M_8kR4EYDzNhZTQoy^+XAA+@M$FmCMO6dU zgEA}0#rSL?e4f^*=U1G>7~%Olviv6*SWo`*)u>8M!t!PQ--m_@nMy z3;S=uTzds#v5|KoSGm<3UsO*LIW%trPcY!XTZV9cjxomK`NxZN0lkzI)89=2M!+=Y7lx@F2Xkd|GzO z3xrNEwJan-dl2&so;-R^J~jv6ZmRb*4i&KUyQ^TRQ)#lj8zYdPW9w2XHdE#|2FEQ6 zQ!rAMQIr7vG9z{iqj+ipV{Pp#z<_6>@QM``RP{aR$s$zBU%ka}#n2G`Dv7Ft10ZHz zQWgf(d@G44)_^ao?{V-ofR4g^UdxWX1J)c-Y=n4tbm_IknKEwz?f6x_ctmO;)5meO zm$M#z%>fy%aXt7#1$LIDx!4u{iWbc)XalgBVAb}*hlWneg+P!?o8U`qF_VS?kU0#6 z8d{XBNX5PZ4_(U*AS4b0qQ2-Y)5O9-Vx>)STI-T3zv;{O%{fGAqkjaf| z2buJcrW4R>0->(h=#e^dkZI=2Fc9sEnv_mndV`<>8B5fF8lZ3#FzJ)-l~)4$%T8|O zUvw@PMt*lhCwRH(1WQ8GR7=IAkx5psCO*eRMm$*bd6b0~GPQPT?93ZTx-APT2LaTj zdivsLh;2iYkoAV{!pSJ&A(FZz=q{PFR|j2(joDJU)j-`F-v_mJJCiX^J@WOX(*+oV z(lI85Ha|kGzOjANvT3xhVo-7xWy3=|`=b^fFU?>R3UpoY(ylJGCvAb+5*M*Z(^wxQ zV!d7~+i#MT`d8PF2(VvD+xL*lT6>r&x|nuf2?TluSH?CyGqtz$^U6Q!SKx6e$7hriVji1GPw$)IubE2dwZHQJ+Su){%A_5)(0ZUj= zM;F^ma~QE!e3P?hhg#oAr0<0V&iIWI*%19QaODpE(roXjHCi8oc?X4^^?_8d5KT;(O* z*9Ni%rEchnC_OfcfjPVivaCLb1-9YRR9Vplf$@Nv9<79eS95>nB zD3-HqXB|11LJ>ok>QU_y9YgjRLZYrY(5*GmZR~w15p%2y1qXW}T9{(PGxW4j;&u_2 zUwClnq+6V1SU&RCG{v9Ljj~X29g&$7;ABD(Es{Y!=E&<|fmt538Iq zzXuk>%!OePAx=8nNc6Y5QfIt7q2v3=mU&{FniW%uNgrO+_rO7rNft~~E{N8|I>o<2 zolUu=Stk&{aLfzcc4Av~cYF$zLKlZ!n&bs-32XMR)YLPu9Kd->1*4@TGHM4xcI)n* zG02FJn6#g%j!kEC?xoUXY@1nsgSfE>NQ6qXB3p&$Whh|mry-@;oYGAMtc!fd`7C{! zQx-%;jBM+rAB{nNOv`|T!F^m`J1M9MfX70wa+(pnfUMRBJ&ZS#&^y?&k3KSX35B@k)@Vr_A1md7nIMv9-UogYOc42!ULmm@{-b zZN(*VQYH0XDexzy)Li4{wn^CPqY&XjS1lpKIs$df8`5?$86>zuISPzTv<`_iLtC+V zq@X4lSMgg_V9LXl93f3r+kB%0Rgft=O~KP5nc8Pi)mG^aLZBj{LE*@Q4!BI#z{YE3 zmOY#X3-SqZfRA9jfnl8orQQp$XqbGP98|rS^We2?g^<@E;-a| zOWbIolmvsYWi`R`tW&5+^RSN&n$h4m4G=%EZX?dDdeo2w!|LOv9u-CO)?W{x(y~6< zXGX?vc}lp!^Q`zGnLMWzZ(#c2Y3Zy0 zgi@9WE;;2Ll?P*RCh1;(J_6 z-Dd1=sX?GzKw*?>n)B09tAaOpb0~cobqK>x)Bpi^#FwH%(nRrsQe}J`eAx zUNmrJIOlXL8i)j8^Vd(-Aep_@JieP7;un#`Yko&CbH(a~OkI$v`v&}Eo3y+VRWS?a zB?5pZcN`jGR$H23Fl9^h&c!*5g$^Tcje0pj;x(R8xWl}H6b_@M@DPOf;}UXa-mArE zoow4Y{g;KDqd<=K`lY(kwueTz1yCOH`~cGz?y{_k8}<8hX82p_TQc z?9D|3fl8k1DhpK!CU_z|4|MmR|HSM~Gj1sbFKDlzC+QeuF zATG=B4q9zZBHv|J``rp*r&-lwXfl0bv;&H1GH|r?OrorVHe?15;5Ml!9-%QpL6ahE zyg^MR|!txw%f6bMmU z6YsW=RubZN7bPCgECR$e%0#DfEa@i?o+zMn4;hP0nrt67rd@r>nM0vPV-)5KI&dsg zh*Am&M@Lf;NPK>VJ7!;u!t!bZxpkl2ubSxyQMptIg3xD1xa5pk;7s+WP>+X~0mD%D zp>NvqoK|XCvnjDB``W;~K@r!lYaA<*1t;qqS5QdYUAMn#Zk+;o=5J7x$$8 z3>r=L5R)EF;WaoqLeA5sxl0zO4}}X9ric__zK$Xi905JIp*w9~q?a<}bl_C{tjTvu-&xhs*6-cK`qu~Z zk_=!`2UM7gHz?StKx6`l&o+a{%{qdD6M{*xQ$wKuc9G;a#A`R}v5Ds7-6q&qB#okv z9Cdl`19V{`FzlwT`7gXE2F!~ZB9H~eqNK@}7#q0Zb|nCyKATsh;mPuJ>v7Eb0ao-? zq~O5@g`G?}EX$f!m_3Bv86^bgVO_?71@S`VHZCu!eOY66W0cArXGzjX?}6(CZDj-$$s5@+t#4_|U!g(yud(E;Z}ffxsN7@X$lNT;vt-1W*|E; z%1ngk%g!e6kkHx=2e380-ZT3=TItEd-u12pgf`~B*GPW)TSXskY22P@Xp31L>?)O> z0yjm*hTpmJ$7}1->K*Vz5E%T+(`q!X@M;afb@hW4F$Me2qLY0qOpxYnT|N%36lXd1 z)?9JjR|WceCdDZ=Sv$_WX4OM#q#l%@F1!UN;5+F#whDR{p$s8=2UfDbbBJNxMoUE_ zDh~{(Z)H}{Q>Hc)4cewlQiydJZGqO=!`xf@v~pl89fF*Xk{*u}erZ;6E2YnR+I5)GUM^v;2W18<~aaMCZj~@F~ zjq5?QTSt$Y6-VI^b}cd~yF?8 z96+I3)nkRDu3Fhs<~ni*-ltrj-sH0zuGHFWa@YNAiwnw#4A7+7R;* zs2g^aR3UK66|}n!wfd*R+g@>^BLxp^fqE7KhTqejRYoXko-0epEmLTS*u4?)%Za5P zCwrfQE&F{@k&jB#{tiSnMtrr|dQ^bBG=KqxPBTm)be z7Q_rP%t}W)ib8PM%#x(qPkwn0A#rIA+qDk>qUy~lRKtcKr`Gb~E!hT>kCe(XEQ&8L zL+OZ?HmI1k{p&pB$V_h+&Fm%0T8NfbS07_IRE)&I*z@m86PJ~}Z6rxj>1RBCaL`Dl zW+SYeDj<)b=9aJ`Vf{xEwPv&!`azkvR&`~*_(_D#w-JI>S6?F8We)mN-BJqX$Kfl} zMY@7)w8&pL6n&GR$9O+)#y@BIK>u=u3XLjKq7KqEp> zDii=vDpDv?14soRh)^m}f)oV;3rdPm&@CV+Pz@@IQVBqUP>2*F0YE5}f{iGpBBD}& z6o3r^B?yHgg#r`;Ql$zs0sv5oP$<$0(kN1cKmZhgAxNq;4Js8vr3$nn1prcl6oEiA z3dE~GgGwtx2}%f3DwF_?AguriKqUYJN(o2>2#Z233ZY636rn;OPz5RtD2*zkKr{l2(1F4K#CL%C@msTAxKh_qD1``6h_p1V0*wHT z2q4m}AR|JJA|pzaBA`-aX5$TDkPtF~E#pV?C4=Hz>n-^o-n zI8Zm`Go*^2BRWlsiSzkMjSYl`FK!U}<0uGrzZ1VZ6Q2y=r`h&+%sn;V? zLt8f15$CQx!R4jwB&EdyZGCVGFQlOoTGiaUjU3{x&6e*Uq`y*faJ6 z3HtB$*~YlQ>AzVIXL(FWX(9zj*3zr2u8>_i{1+_&pHmS=e0v@T;B<4)!Q@-eG$K7aqr$1=%I>%Sg8k_Z=xSzAPc8BeF$+r7zlp8WIr*p9e zgtQJxh`zvb&R{5#3QqD2s4<4Q2$zO)PvZ>|2u!|W*el*Yc9rvSSIa*EtQA4#F!SE# zO*X<9;{p0^jXH2nvvgrN05_2C(-`&(>@V8=$eTdE-F2_J99#2^{?rX8*-dxF9?Ql$ z??cbr{_NX+E_Bc%#h(tHHU`@mK7`=< zKIh!y>>KsInjTMb0#Ix$A_&LwnFgB}URDf;bJ#QMp2tj6o(6xak~YD4j`4%VO-j;3;5Q4G45*d5mv-{mA0nfL=Sv9Roc`~>dwky;>())7+d#m3 zhaO-KZMCtr*&fIpSk4c09ND;?xxw$aLbF_L_H_G9ckk{0Z^3xX_tD4jH3~_^s2ryg5n8JA{426i0XIg5f@gg97V9DwqXGSRA z(br^C@e|EX`0rl(-+j@gC^(Ek+%)R}qy}XA6JM}85u9Obc*S3L$IG2z-SR`JPdfKFxw(Ub{)ix{-}krfU)PVBfxcqT$soX> zcbwh$`t}CA3zX1?5r8{I&A{SJD^||=CKY(HtO8nm%NvlveG>JyIe|Snc-CLY$*`p;cBl;jvti!Q^xvWD)%6y(;+X139JVn9nL*DY# z)B-__j`bp1#eK~*D=?#VMmVBwEQx zDfWkn1oS(i$n=hi?=PUba|lP{MNPH)p!^GZ7qLK2ER>Z3sN)}~ z)Yi<|B>ES`BELT2e6G_N8te&eaKmq$Z2ct&bz|eaH7Lhms9DDHKDwS6icGhAgr~}x z>Ni}td#fuD)%G(1>cR2GFRRRXUNT);xtCs)$$CbwB%&+6%f+(=-X&zvHg8`*@}^+? z!*Dkkkxv8ky~68yf)mNEhG&&C`pE(PhFH!>WQT@l?AzSCmdw`)dDD_V9cm+nBf14s z3@ZAO9PP#~)sMPS7$I4Hg9I7`S^x3bCZQr&X^jsj669fA58Bx?Xd>eb*|pGjzf29N7= z7Gw=EPYw|w1H|P~IpP{`A1hushcMCNkP8k4L@)p%7sz!126lZkRz1+8!oHL>AbMKl z>H?u=lN&Fd8r~(1k4n%S@UHARc(m z=5n)_5FA?!si2kdf=?b{&kFX}99`I}>X6dn+}~g^S%L^aEML*4G@}udSP>NfwNzZO zmTxQ?1cF)LLbB%_hjU6YjIxF)2)?e=5*8O>)$Lhp0NkIaH%+vik;o@J7P%3TQ8V_c ztfc%Mi?52%KBuL?8iDKN)Mo-aOVg_qx{kV1v>}jH5okqhm;n`Z6;RM=6-R2T>Plsg zTe&|e=B-KI>N`oX(KM&2W@LB+fPDRX+&M~@maZ1@gaR(od$7VSz@}MlS}`QVUg>~G zs;P18kO3ctfUJlJ$xWzSv>#?n4ksB@vhl(MA60eDO;(c%Vj;jf_uIYo)cdx3bi>to z$<4&flAi(9nL+UN`{0@nfboa`J;dVqajNI@nW;w{9=txFj)1G<1Sn(lVGV^oq_Nd{ z7E~3Z4w;LRYqPe#s^%wDjM?Ctm$v}VtXIwv`?u3-^MnN5U;<3lZ$A)y9Wc&Gxo;sB zLs`ip5cJ_MrlXQnG$u1r-B%jg1b)IRG!x?9@$p}r1Yd|38`h24t#CSlA3I_YE)dxX zv-aEWI47vP>wn){a&_J|$@|#zzcO<_ZG_s+rk9!rdSlDE)-*V!bONbRtBL2Srt`ul zb_?1ci-%uzi#G^-lxS;YSU99b^IJ+>AgO z6$ccNMh^ILH`a53nMXm1*4(uxa>iz%)mCdrp*>YWbx8PNBHjQ~K&-#L={0FMuVIpb zf#%eG8)W>6v5>FUd^zuHlQkaere5Q}I@Hmu6i`~pwmkjUTO8Zm^Me{-(*f@K5o;eu zF~@bre%~5roc=FOtdy0nh(Rm`zo9l$MpbKzt>(J~*J{?x7HZWq2aS7DqtK^<9ispK>WS$S7XezjSmS-Zczz?UvM+~giTTkWdRyms+M zzlm!(c|?8>M)lkaz__IF1`C|9YJrcq!>!U9O}1@)%4JteMQ(e15zt?5iwtGEo?0qh&0L+LWAjlWoNF&LKXrUc-gVvB1NmV=?XK*AaBWcOb49*Dl#{lX=W>c-9xY|M$z~*6 z6(||HvqrcXac<{S(-6+#3`XXJXXjhFyig^#Vzj7D0E4uguw0-iws8z zp3~(Nh3jb2=#yJj;W8Y{QV$J7G_QnymU#kTSetG~s7aDfIhcpk&4+vqP&*$ITJX6d zhRW>+wVRKbN84k5@Q&@x=EyJy8JKtm3urtN6R^6*AXJzq@Px&cq1H(Ok|@Z=W{%5% z5iv)Bl-4q)wi4rC6{(f=hdU?ocw>Vi2LoCkQB~)($;|cQbY%IX&O8K51_Z()l7?5` z1Smq;Iv*=al_%8Tw{vb#eP*C)uugiV!G}++gB~HVe-J#T(*43N36Y7Pce(ek$DZ4F z&pg;PFlOR(sOXb>jDFWnbI*!W%Y~SP9pU2z5H#z&(^1n9E1$ChIAo>6u?7jvOKEE< zG7BiCp|~6mODrWa)fr{1iAww!&LuT2?@nKxED5s6>IJKdQev#f>tb=3I*9xyDPUti z8y25N1;4r%_H`Y~U?>mv3PL4xP*!el7tcG#*PB8Beoojb{OYf_^YJ57Fog(1MRRa)M|moPvOTFbX;w`?6d-~(3u=1ld3);*?p-v^lACdHa8L*8-Cj&fscXl8kI<2q}` zPP)!M;Fm~KoSYHya`0&Mrbv%=^()9)l2{XC8VL6D+h(4eZ6I9mLm?_6%~R==B9gpE z_#u{jMdyYYVaUtMR%x+fP^xzizLRBw7sAx~VP2koWXMpv0Iiaik+HGi^vwqS%wIr( zS#XD{Hong}!Hq5X_iuB*?TlZ(!@0J3n4^vIWW}%Vz0QyuM*HC3Y#LbSjF;~0&hef8 zGl;a)IsR2dQ})K%ZSlUn*fWD0UHQI!eA|CM>wR?VUkZOb69a@dCkZCWZF`9nr68IBSUCc+b$BYlU#)~^W;@qjk$Yf5fF7(mE1VM?_C zbkpoc$IPo;snG`x)t1R86haP>#w`CIMr7m&*lQ6d1q&`pL>EGwHu&mc+(370JCfE~ z>TD!LYWwRTS0-K8`W=#NZD8ZzIrSL$-N6nWPlq}l6bgn>mm&(Xsk8L_`+Pjzp1h_! z^bZ^f0Lfwq4Dy263N4BwWrqVmz9i_nuy|CkFd+DRywlrmBH~m;ft$|yE;iYG6Kju3 z7U7fXKUyHD45Ks7cV30ZC|$0$kqvGht!h4Coz72E z&hZVJ;Ddpdd1wegaL6PGFBl4e1;Dxi;gO&c${9kbDIxe`6eCc?Z(r;TtR89i6LBLw zK5a4+n`v^0ZRo^!57w_Q#OZBjsxE;Rqa2{3i6aMn)Uyre)sig(zZv*d`gzRd_XYB& z5!&-i)p12)cZ#^Ed1l+DWMcQCvS7nkgrMEU_zQfvL*xrbYu|M>=pR7<-yPXo07fOp z8w`tJIC#`onGLtC>O@>bXX%=4-mTYZr)^10wPx+%f3$f`n}e7TD_R%FhhFg=h`Y~0 zkCwTn^JAwm*|m&fuolKv6I{U1Fh0^dBE_C%*dTMWMXp z0|LPCS0uezRblBSDs=&LddaxN_L0{T1xmKKEXcP!LsQ_rjE;d#3uh`kcGMFwYJ_|~ z(1{>$@IGPO=f#e5h@doNlaaLGDwtbAZzi7N==BG zf{D%ba+cdMKAKrUA7D}Sgi%W14FNR`kHd-ML{0H2JCPbn4erZyOtu5`>Y${p#O_@t z;B}K2PkJ7KdGBpp{AWOcK)PZ8dM9K)OB51Ls)9>_7f*7L$?urvABO=*b{MUEGQ+s0 z3jOt30|%xL=1;JXDTfoX?F{xy-BpE2ZUgNIXSN<0;xzc(NT`pdk8QGp# zAa9P8$3nXW!2+er7R^V%VIw2+2q=Y;nk{h-9v#kE;<{^q!&LEt;6fM_ZY1023J|Wu zfn>!8civ)&3>};wF^Crwn<1ldBa5jpjL{o_gl19312;+^w(pUrM_*H@K;NAWJwU?I z%_MO<>b{S)jtgWvy@BeDLEn$5Ap_GR%#`jW$L8Fyo@@$;tNZ3j`KYZ@)u4VXHs~4e zNG?5KvH>Q2#lLTulya_c<%yI9Lntocdn|$BLeLjtT#ZA?f*I2s)`Rt^6t1>2s2U7O zXi}DFy(4t0F!@4|eY+c$H+nDiZ(j{qmVnQ^6WzHz$Yt+Ea=R6Wc^O1yeM&G@JV2(K ztpQEbDxHr69lDY1drCbWf{)x)-i=dCM$b`#u2mAkqszfm)Lcd$$6u1+WAQ_Pm{fQRVQkxzr_&xuP-h)m&v zS@;P%MJTo;Z7#}31GG<@QGW4)+9N6GI{+uHo0gI1gGj0Co^p&RBGW~gtm*Nq?1QY( z421gvsmEB3AG$d)ZAFWEa5EA(qT#JE**BysAe57>FiQhz3e&WDJA7dkueyraDUBSy z!2=Zp5XPcFTG_XucBhz!n+%jXF4mbrLLa^n`O8cYR6~VP&*x-1;tj6nii?aKTX&Cn z#_(0ux4Y-weeVZXnA;j)7a+_7i6w$-j$SVib&V-^-#}ad^#`Ag+&&wgl(=%W*`Y@R zA{NA{!bUuu4Y`AlK9R_ATq7d-3OcJj8CG{tw}8}X<#ZxK{cH_kpn;1BikFN6L=`k2 zj9??Q&>(o~I?uBNHswdON_qpfSFa~JUY|^_OtOe8IOzfwQcb*>K3{6c#Td;>k=c|D zN-@d@Od)mkuYp(R(54A_@WLX=bvCfvk=R@u|dh) zJsff+5hpEP1!2(xzPK$6I2GX!Oj$settm}??GSX)byO1T!x0H1TUdTRo(w0m>DL2&iV64-mgnZi&&1G>R(li1pk< zbobGIlEn%;CDktrCNaKcK=HhMypX>3<%R_7cjNTOH0BI|<##@Q0l=&m5QLtMWvn+_ zt;`2%6pJ-5yh0&BJECo?%c|R~*eI{M0(b;g5^8s$l?Au1v~b*ys6t}7oOH6ojA>I9 z68g#2jp442N1Mz-rq`;34;WLzV`IA9g@>_;GdAMhU^2Y$lZsk&haAqW+q$p`nfcd& zay8Oss*OY?(}u}IpH-{eWn!9^I|@jTUJ-z3QI}R(6*0oAE114jK-RpsPj@n>vYSwP zKo8nCRkH#n#Wjtd!zn1Tw&a$O+rfipAZf06OWLCtidK?HT~60qf*N+?LUF_l7K6qR z^Rcn0K4Uj9rMMu~fVH9PtY(pH(EXQ}WqI9!S(%-K!XoU;UiZ@?jGqdMNm zK!tB?hDM{yx<)`J?MX>#p2kD!SZ#L(@i-276?M7W>RE{&n61eJbF`Q4U@O?f$*)<{ zAGz$`tnM1xappg5=GZo{agIsb>6euSfKXxWDCLkWZbQ^&N_y*b)5^H46$bF?)3hWa zZI9BDOxfBj*Q(j?3ndd!D%h(N>jON+cqMPDniMJ%-FD6oz_!|(e_0SA(DROZ4PM*W z9+EaBpB%c1rYobBn@6pqj>hS1ZD(%-{F&;dT^w=%5-*5ALxXZrXf<1hRbs}wf&19h zGiG2M9tEm_RC#I9@E=fc>)3*3(Kd(`Sa^d|&+Z{fh7B97NpCY)QqrlK;Matnx+d6A zFo4)daU5t+^V5ml9XWkTK{~o2WM(57b1cUc%h!g?N0B}>jxsVTYZ?KXMKsS9GT@w3 z5=)rNKu!)2ZxH2~E+`V6z>V30uPEgZ`WB}p)Y@l*0+^zrrwqZSDE?K|vD$H(jf5e7lQmznLZGGNUR$d8GQ2Z zN{&=ucUEHmHFF4CKl3j}F|r64MU( z>szD5fT>Z+7$_6-Nr@;V#R7D|wOo)e5K3p2)#dBrL}Q>cnv_`B#`;wiiUh_-l^MlJ zmc>AI;?7BOqw;6dn?VyGa9SY@vTE$>DiWyMZ(9l7%a#nNO`H{!f{U{!D5FB?>hE)7 zAR1qILOB@Q4Smk-cPZ1wAgK{O8~~`TT-7-A1}M-#K*vICB_mCRYmf=U%F?&R5nONO z(PU;`gr~S^NAXnH!-r?9_SrO0Xe{+as%dFQkrMz!Q-f&=ipKcipJ|a()k*U}EwxDL zJ+^G)J$Sg4YZ{^Mg=FfwV@I+WD0C2}&1qGXE__lFA~{-SkXB6Nrv;cA(Am`wAFK|* zxaJp3e0FGgI$Td~A15S*SR=Y)K==f9IZmc>C7pvW9muN*#g7Qu+A6GotD-*mmGlu^ zd7ie{jSOv%c+#YE* zi&e3Y|HL^1{h($?}Y5m#v4OO)U$I?kVNs2178NdYcreEVZ?9J#IaK6*tzT zjBkB()e;JewLxw5cqlw-+X~H_?!x#D#QNBC1A9n$oj!>MLuA&agHC6GhC9@dxP;pn zd*Ic?tMUp!u9v_Tx>4^6wUgBJsh#1v)?S@sE9+9IFcI%BGBZn{5vlo^fKa>A#Jv5S zpAQiyowC)P&PQ-AS3_`P4THoDOr?Swe=cSq38g%FkCdVaAZRss)W~sxaud+x4{|cb z3JpiX1|%24f}2D^tm8D{+Bp-n)Ks7%1m#qFXS!jhmPftgHY322XR$$Tw+LK`bX&y`g}!UR(1eZ~)BSep#(fL{})DOhsY&^(HCr zF@V7DFWT?LXi45=IE;bXWm+43mtq}Istu*ed_0$s(HfQ~if9Ehl`pkk2j$TMd?XNW zOp9Cc6BSWoAci&QRHC$6oC>~k@2GGRP`>Y~c@`E`sF|m@NwtE^rGuc#VC@`zgYVjL zaSl|QicZdnb5f!V7v}+T&EqNYu)?vWb&4s>mK?IF+{d+tB_(`B5I+$YgjQ`pAwnk; zy1JZ_O2!7aOSPmTnFX(v)m6~YC|A(X#5fF$kDY}>hfFEg?y%y8xL&l-ghwhl+C@i{ zAu=J}?mHIW?zF-Q-a&ezJsFj5GOa{b5Eu4v~xPXiP2$!4_O*1ZD5p$tC5NB`bdxsV1vAZ(>=qT#1E>-*nh$#>j@;BAPQKbFid*tpErdTz#qHCesrU_pd+LYE5o zn9V|YwVjHa$?J}y00KtB%^nNB*}SwH+Pdwvy~Brqiu^$UAVpgeaeY4ceXMDEyB6qnM zhk7`}tPJI-87hdU%D9HoCIV1stxp|=5r9Ji1_Ra5{T<+$Ft-<=OCh~W>&MYaa{4ED z=z~6#gSx!FaveQkUB_ML?YI1CeFy07a|V$0rZ(HWai)FWZ^yUYTO*({zg`Y=UL+f5 zz{ucF1(X&^bjpp;ogmPNPA9_8_Y zp{QfM130YxOh++MHQ|t_S4xK-5?;Es1~^gHDu9}mK0y?-9PYrRu)t|Hy{l^M zP1agJBqQrm-hB~OrA39@JkC$gW5gNX6l6d%h9oGF(3ip?)b}Vco|GFt_BQjEWl?4% zFbIqCkxJ3ZfZa}?3}>L(OV8GT(ko)8d%b@m!H3q8g6~CM^QL#-j(E+TPjjvBeQnnN zX!CD%`~oN!)biODql)5^+wvh#>sW(E691#wl<9w+8RTE^r;;TzfoLOxyvs7aNb@~q zhf>oDjLP3);6CZefsS&be4-me!ZSqPF9X^l>3yL~i-ncOL$v%xrvyyEG$|1~)$CUTn5)$;>@--GG72pr)#BiIloDrXsY5*yIVc6pD(aS;8D0J`UoXuP$Hzz_azlN ziG2G+%NTjzHWFh^V7b7Y0=KpCFMgr7x3RKc5gS-vL)2UulPY81Rf0zW`2)ImBv|I! zCPnNW&o+Uqt*S40ji4&fb2LR`GWg*fB0?(HA;Nhilub+K!!nW_Lt8T|HU}B((qo?c zmkhglBz~O8$17V9eb9(u$Vzs~OR{S!>34fh^Reqnp3Ait7`FMr!-zfOdIFU>ooRI-N#EMNpLFq#^)9TH|?QGXa?D3v~>75j5=$K7}yIKFUe zH>16Ur~w{S5l*+eu55m9Y+??cpV-8Mak3P0S$#@0A5TxwO+SG{{D z)@y@4GZ*6AwL#;N=qtCbhv43Ptc90{A-QZwwX~N(p&ip?*4S&xcUnB&FeNG|?TZS{ z7ZBsZ>USaJ>5s2qC%cm(0-rOtdPC9#q?{9HlAOy{yc#?6s81a(JmRlP>PF&%e#d|@ zW#hL$H6hJ;PVob*Uz}v3@W!qn{N)NysY~oE(mSNAECMEYWR#lZVqFLo1kkT%IP3`g zWmWAK0eAw|P91Ep+NJ&PN3n8`5B{SCn{mX<{^SJI|KG!^ek? zXhTiGvbt{4mEg2dY2s|A_E9Zq#E~A}$6TvP7!kV*0+4b7`*ymA zA4bHq8{u$lMO+-6su2d#aS2>@NXn#ywA^W8er?L|3hj4H1r;H8ycbe$sLX>*gcV`v z1Xf#Og`=l2r0*}H*P&JH*89{`5A6(dt6-l9IWjK-hf+2Z#E;Y-M^D zt%oCgUiNy1l#L^)8QK3+qxW2m}T{CsLIO*#Un zG;u3KOD9fCnt-U01*!F-zYeh(zKOLlk%N=(yS=17#M6A|L)8Wlg`hqnzTaX#g4N@V zD6>IAIrI&Bx5qxl`oEnHrx;TZJS=_~&5rodey9$)u;YT zjdv&Xy8)BW%BMpgyfl&uq~eWnnba~;kZum$Rpvs3s-lH%#sssGrJ9gYnIWME8W0LF zZzsZK!h?t7wF~kiT^g-)jH@csgr=Da*-~{{{h);OCZ`avQ2S%PcYv80HoZ=9(Mvsw ziM8;L)d0YqNAOYS{rVNcKTyf^Ix``CDHw&>X8wm`9xqd<5^(&0;2?B5RDj{cLIU>6DS?RK(oQ2J zqi#5;{15f2Sx-b7PKY8|C(1bIko=PS?r8vtL?Y!MhlU4rYE=}zi|tEmH4}$Qt;;L~ zAsj3V!EuC$Ky%*2P-~^&Zwugp2?>n|DC(q&vVqvLrQ8)%(cuZ2(PZHB6XbPl_Oy8M zHj4<;dU2DlQo2;GhEX}N%9g3@7XbRJsj)W5-2lyR%OKKv%7E>l1S&Qw=N55NF`FUD zyoDKhh7HYDnF(F%QM0h2%MDP-X$0Wq*PR?lfOUb@L`2lazV8Y%Z!9PD!0ZWQ4rL1sWMXRIeR@ zmk!9nWkE(vTef**^=zcbUb;D=*=zu2dWwA!C<(&sNGE~U_i7STml%pxlu$}oT>yIb z>b2aBdiz|=DN&GQj`4#~y(k9yTKYZ6FOIj=YleF)7~IxVp(wP_}vTUl4HIX#584 z;5$(f5l1j)d@diTb`YhJ?j{=6PsR;NS3T^)!VQn2fUQ@6SdXNGWP>Q(s}~{%&Qu`% z(R3KK zEa_^6b%Pbx6tRlYw$E~^g+n6mz|7_ue4LH)3rPTxnGa#$@H4*0+<1<{DC4HqI%On! zG8uJ|7l|Mm!d!4DNYX{QfDW(-{qoAaFf$G05wcH~MiXI*&wD)N{Y&MwOj1eLd3KIB zqka?%wBo7T+a|?a4a1i-p|2Arv0p>#rO8qh1Kla-Ha>tEg@j4h6j&P}f-Ds+2!Ov~ zWc8qU#G3%x+=SDF?`7Gbe9LdXFRB&HqWsHAwKzoBe1RtSVQE<6S*c$KQXnUW=q`-M zQr+Onl*NPa;0UVfs=deT9(xC((IeD)()@J0&Ud`)opNdVe^2Q&pWNe^H{yPXVD>mx`fs5izfYpK?p+anmrxLVIW(5dLeSEZ~Q)pfx>yi~vDxpRjLJ$L81jB4CGEuE7CTY zMeaJ)P^LkwO6YJkuO;&P})d@r|pSOEz%e z+y&DOvL2`~9ckycpGJ>Wkq(^58dUe4Ptt53P{8#PX#w(L2M8|hS?WfY(1OLs+DslN zS}Hc?``tzW!BT%|m%yPuKQtai6L(Gg1LeNhbv<#XU?62M>+V4Lyg*~~t^7h5qs&9} zpvljTSC@$qnz@8fe)W&WI3Y1592;OryzLeBv#RGPBtEW6v6_5&Rcdg#;I`>ZWFn9XbG+^(ofzhvYg9&oo)~=fZfppG+rSso-u5jOjcGNkn z$P}Fu?||Y7eLOqjcFvWIHxsEa2i@t0c`I=gI;D1?pl@W$jVgxIl68@qLF#{k51TmY zMn*}5V0i4Uda%G9q@oLD2rMdtZtM-&zhMlLrO}wOs2rMvYN`r_az|B`ClA;@asz5% zA@?n+T!A1O5Exk6+S6L0nsWQv0upj}vZxEhr#~g#rZr)TdNqR&H5ML(aR7uB&43`N z`T{suM@pc?AD6rA5he#E;mT}Nc=pr2c>4***!`2;13G)Ir#Nk~sUFGB`e)wx*McGm zNtJ#+p%6S|-8=*1l`O9YHYWCZ8rVY-AI)l^hBF)S5b8Shy--!;BR-h~vs$M{45-U8 zykN+(>M7kB!5fBL5UQ&93P2f8ndyP@JBF-!gM}A5@hytOb4-!26)!Z5ln&=!{jl;Othl?doON*e);Sa%Nky-F!>H2dC=R^Q*S{N={?OK zzW3dS=#OqQde7@*0*S&Je^H7apA?B|CPLlMP9T z$Ft-MhZ9`#)MF!eVZe(Bb$h6hE?fZyFuoQbv`q!344^L$E(wC7{m>D;jTmt(PXl7# z(HuDr&*)E`QI!$(7Hh%f91Qfurdd2zQ-%|@I$J>z4%rJw4J05iZBD@1`Q_ueS%&XZ ztal@j9_U;p4QKpwj&a!*oMEG=Ex&WWyEO0 zdEt51`Wk1wy#2S|oTbvhNU4<0e2%%Nj8*6{qZ3IR!6HTJ(SB8yd18NJuX4OQ%iN+P z5(FU))icKQ&n)6Y4Ob$H2`*T{Eo1~NUTV0i?Lzl@HB!R7wdIM)CEpZ5^=?RnaS}|# z!_{w{dHV)OpU&=&B^}y$ffV=gIE2y3c2M!2c+8-q?d#GU7s0{+SZtz0{Dekf>#zuHs$1bPSUg| zNwYAef=p`?W3>3~iPeo68%w9RI1vD#D9{Blx51mL8#0M@8{cjy=JhP7S}EH2gHKB$ z8{-=@Fa@`^odS_(F{DG#q41objqRN1=Gi*{cVi|9W*LBU(fvqGSz{EQ6#H!KbGJ?* zl2o4y+ERl;4y9$&JY$j!*pG*}3YDZV9YOB)+qi`OU$FiKz?2;=!7wC6`AA_y{7a33 zk;&mlfNA+zQG1LSHgt~cCQ8UzP(nm@12Uv3ZI@bdt`$-gt;7+}7_bplDJv4PB_u&z z?bS+v5d!&5Fg1wRlL;^KaekHLYsrrK_Q$=6#AX7WGv1ni3Mi?^U$^ILWW)+$-_^1u zD{eoT|107ch>DT^n^B!HgVjuS{Iunua;k~a@V+?!w(^uEDfB7}jf9*|8Ec=P=@C_q ze&PkO`o!I8+f8Xgu_F&)d(yrA63 z&L+|`6E&o8HaWJIBk-*ejoZkE%+%8cukk79Y@gLGzE*`uWJS(Bx?3);S#wOZ!H+9y zkT=9fM@XQ|Z3C-bk5vt+$%X6ilgLaDwZtV8u-0gGK2#;+IaI>{KG|NR&|S0{QzzQD z=%t1b?4sz=!tKx{h*y1}ELzGaYv31Rmi4ElV72%nmAEWJ7R1=iyjV`=9Hk}@c)4t( zw_=l@P?a*;?;Bm)Hx`U4I1>h10mXbL#*+iHW2h%rpj4Me-N%qkvjkDiIpaF4tOiJ2 z!@H@=DnUj!&m21sfI(3Ict>a(JzC-pDV3grc=SA zR76us(@Gpv6ujlU6KMqD?h=@i8)r!#gX4@qOjEiLBq>^bn6L*@Rvi00cs)oUGYwBO ztMOWzp{Oc_(~}ciodLKV`IYcH?EQ&Oc^Oa>jXNC7327P@?-v;uA)Nz?jO_v9W68`EJNb&$OcK=#Ol@Q>`%dJ+up?KLnF@sWA~*%;?w>Rj@H(fQq34 zgFhUerYt1UR9=IFjrh7)?wO{^Dj12=xi0>EOv6p6F7ZFQKwLE4(`{CfaZ;(qja`65 z>XiL(VlRM}J%fT2Izw5Jo<PP=<2zKll;>UhbjVB2-QgS1a@|pBqhcd$0EnM2yup78*LLGnqQW)p zi{abU`QwTz(mG@$g!%9nCXj)*8hI5mBv2@CCTYqSDab{kq*b6~o!*N&rtWW3Qob@X z1JU=JU4n6-EyMt25IMR9{h7&r3h6*K?cUvZ6w_CwGGc|Ru(3;OCQoV)oFl=5Y1a5d zTo^bY1U*p+@i_P|mS$fZu+&j=DR>!*75(0!0?aMiRGRlu->Z+#%ME;RJAqhMtX{fA zOYn(vJ;aPjl?(R2G4sD*krBonC^VAP*1FVS#S2 z2Hbgo)q?bHsM!Fls6a}@lcoCz`(=07gS6p-m70?EGAPguxK9Hv$%>Hn968u zagbrw*VG^uDXe#dz{q%ns;a*UC<{2OBlb|LtDkk<-9RwZFzbqtbQ_v=g#*p_UujO^ zLaKOz0md}24M;&$UkpKaF~WMG-tF_mUaIu=l;N&ZVsQs)PUz<*Z**B55mh?5IxGbM zn?^X|)Oxgy?Xq^uqbe;|&Dkr-XkXA5t>tWp8d(G9D4Y_pryG})_z=+Q4L)inWe_-1 z1++#{fqhwtxiYV-r_$#g+WD9WM^IG)PqtR^lMG{ zuPR{Ecu|Y%>aQS&g$nNcFuA8vCOIAfk#2q$+Fkp?z_%I8Tamo;X|$z>O3W$Qj9L}l zwsIyC`yDB~`AW*4aJ2`(jXBRxC|>G@}#@y8CxZY}r%p1?&jYB;6b5K&5zkh2F5GOk{+6T3%+7R%s=ABtGQ z6qX$+5d<%YH+cT}R%^4*;rro12meSa3;rM?s0;jk*Y|wm{QaiC=kysruiyoK z{T*_P{&hc%Cv27T5JqHQx!O^)pu8QW2ry;fYw{b8Zi9@)eRt)-N2UmyL0*00pL+~s zF$#~LaS18QHwAOHP%k~kddsX$6O90WFv}ka}y2ui>XQEjvA;h&;E0CG8(}Dyk z0Q;QXq=<|5P!1zKK+}y5wgJ*Hj|I9db>pEsmmpH`mJM~fEjdao0MKg}F>NTb)u2kc zSxm_UVsPFbJMxJ-gLvWg=1_QcNI%DPRSv|2)JM(I)3RF9ly#P61vEvj6OU=`t%UDk z9i{Qkby>y$C|qv?j+uBmz^bVc)*INY)>rKz+?(>y%(rW&>k>1(yV<>Z0E9+Wbu3=5 zYdqr@*$ua#MysQzrpm_vR!%62q(&fo_%kxg!c9P_v(~?G_Ds;5*Wn6~RF_6xBQ21o zPL7OdQ)&t-1R+E@E9n@uaXcOho5v|NjWDa6tUU?9dVvKoK??*`Q+@ORm~H_)inq_! z&uR_t6L@U>s7xFTXG^#33KQ#oFyHRK#XvlU>x*X0Bz3ZXJ-%TeKDt!~Hku0d_ zC67`V1KJb|Pq8s=y0_8I-TLEdLHE8=C47h672CoMilB8GWV27K@ zuLBloW_;ofu-5g$pN3x?Epx(%Gx7Zu4_!qm9a=g$j=P|}^J$sPAr*NFA{QYa4t!%v z*!mt?6$W4zAVG9$P_kfy#A-#!hgPlh1KBDS58Fpwz?-*b(bTtlYk(yS0^Cwd8MNL3 z@Li4BdV)23&u_&mbeZ=L$RrHO8^uU?GiJPNjp4vzVCU0qoRZc;u zDypjmC6QuFM)S_~rTItTu0$UWHz~AlU64;9UR$LwiVU8>Z=VJV*!e!@5Wl*#M3v zG^q(g^mYKABzl;7k`%hlRC8U`;za`x7=*jyr@p`TBZn5(WB$ypKKaETT@)#LkQ|D2t{@Om)B{am@&Ekk6WwD3Xm6 zxf6Mw5|wtLn;}*y5032MMO+VwNp92L+C_#5k2>3{nDcCL_6~Hd&%_#D;=^Mf=@k`C z6~S>ZG*bejv+=mKneH+_Hp;N>QtGSs?B5dP@Jl$#w;te(=r5-9V@zGx5UwX80;5sM zL|wXvuolIsK!gy91zJGIKnmX#q{I404QTP5^S0V& z>b*U|!4`GMefR8_?S9w4dcnusoV{S;aE_-eXpoPH;(0c}YMmDYZqZ^~#*`q4r^h!? zbPdn(%FH97Va(+>UzK2jj)nlx{u6}2UUU=t(nIVlRkyo!`VWqQt!Lv<9!q8MZc>Kh zS<4Pv2kJXx>lQ{RIIl3R!_@pmXd3 znLnclmQ++hLzq2Wns5l=*wzUn6?_6Lh)ag15$*#p5eu1ECPt`5Nf0Rz_RKV?NMJP> zRo_dcL_bi-yNCl~%Sb{maPjisL&kl*EI3_UD*k3Y4F!AhmYbXD-=56Z=2Sq!#1g`a4FG%nwy4Hg#di40RU*O z+&Zt-!CauZfjnfNu`nz3EoXGv0erepc40s)$>mH%=4}0@K@VnJz0S-MN>+Rze8&hu zgD#&9$B`B*6~*&A8g_K|%Tv{5IOh^{&@eu@j>eNGX5@Iv-{rTtq7 zB{f?5-%1Q2BgfCQ!HTCQrXb19^f*twWJscw_XX_*;XAmy<688kj15_kd zrg%=xS5$N*qlr9;=xjpLJ1Nv5=x&9yQ#S}7!ZVB!mr6QHFCdlLzzEHK44zm%y(-q2 zIihU~ty#xvj7xqIoO+X1h*BPHc$Y9Qo`em+fMl+a>fI7wq~od7ai(D@n=O`&+>cUY zW*k6Jymi>4p@i?*GBe)>>88PkZr|3psOs-=q&EpV4!SQN3Xit78j(V6J)VlM2|#O- zeen6igc5^%e|lt5 zL!=jDXaH}HU?86IzKuS=jjTvK1Y#Q`@*+)#Af>@!pne+x6tF_(WxB=hdyOeFDWTuD z>-Fu!C@!949f2->n!S_(<=c6pu7v^8gxHWcxgFnFpE_uuR4`Tx-#FMnuNY?W z#8JusY;QKXC@E=p%x%Ksq4NBYsCO*!UgXk%E8r6OCL$9)+EfD?S%V%@E-*&ZVKO8G zggFO*+-ue8C1*M8gs1d2K$z2w1+@4otSLb=Emw37g$`t*Co^$K%Kd>6$s~YM_lRlzF4lQ zbX`>wnjwEDG5^SV<#%epB_fqn~lN;EppB4Z?hN)g`AEXN3RJnVuK zgLXO1@ZyALa%<^X-V}cVMEPXS0-kJZh%53ctOLS!s8t`!N{`fZi)TpKR<*~9iYU)T z?W+)4h8`nE9v(Ul53b*>_f-!^1WQV^=~r|QMHdLqZ>3CL z614ej8(Iukb1AAXdq$%-ONV1tq9K@#`c8nR)q1gHc*d}1Mr<*0g_u_g1DPpRAu*Pq z8N**N`)W~mH@&(-uL)@=!OzO7sg4K|Am}d=-U4UJ0t?S$#d~<^7_5G52I&?&{qmv+ z8hLxqcHC>NsgK?M?>t`Vp|-yDgFfr~zBl|N(NI$rNFZV|;U>MGt>1D-&xreuCXg&Y zV4)YlQC$~7hyddsK;w8w2dUl5abg!)5S%w7R~_jtC$jy^7fa z%ez{)oZv)IDAM~^C|UhBB1b2Ro+&0goCnRP_}I=z>}6|EZ-RsZ*>uDlCIF}so@$2x zc*1@32x!PzE;5#LHV}0ILY#~upNvoDJ*zj#inf6~s2qKq5!X)ljb0=x2rnDr*@o3u zoOJY*74}DJFqs(v2;~h2c(i(9bPlPTg4(PgUYmk`tb>j4_kl)7;h%;F({Z>=H7{15 zBrm}%Qi7oX6Vh66)u=F=OmywqpzNItRglok4wD?3jqKZ&v@I4A(ai_Ue&AHl2?R-# zikIAT%RmrW_@E&ol3}6YsDNnv0D@qW zJ;3+rwaqR%&$LZ55IR9f5jx3QnoNO}&1eUwo8&O0ij~6Z7)_q{s747Dh#Z=sJnnoK z$gf4QiI>MSMsHG1GX5Bu z5~rU|mu$Fiy<^mqr}IJIfOGJ_bwc zuOPQ?edINPmQjp}wns-Udp zLJCZe;xqc;EW#l>i^AzK<{j;)Nc~fsp(@)up+6As96T0>1$6T71I8@mVt&xV86#+&mt& zS(XF@MV@$u5Ns)xA+r8_6&<@b&vQZ&^O|R0p zw4__leC`bWI|=n}G!n!DVI=U!>!l8h74h19^wKlX{bpdG)WE&%^$53UiXC=vAqOcy zcVIKahb?c8ht2oB!bV6CYj^e)7*N=mpbI_?UOwkUVokgdTWI(|E5v~00k%|bpzbwN z+u~lX)|5J#FJ-x9r4e|_wm}_@9 zMhIn*0f+#)Ca4F6M*2ilNL1911XO|az>iM*R2v}#FYkT2B~d(!1VS-3JZ3El2UMVe z6|)qo%kN)Op1Z`);-~Q+jBk|XC)ZMF%Bik z))PqfO>m)saRSH7&i*@ZXOxPKB|cB8=g4jimb@v++N1?)y+g$C13>*|$VZCV!`a&k ziH?}VBO|&%5QKNqJ_7D_G1tFaAtx0g4z^tC4_#@Gckk2PHuE^^tx^p)=Q`l~?lX(X zxuabB;n$4?#I2vf?kxc=Bg4r^X%+#F_T+QfyqYPn(S}HQ`>yGir{ELgdI(A?%#aFo zA*62K0`N*GQaSDk@Ut zT=~sYL61d>RpEI9rY7Djz=rL60mQk8@d>l1JENhcLMZ~H)do#-%V?sBUxoOiC2XW( zvJ)?i=oEcSRE0`kkn~W2znhBF6%wX41?O? z)%!C^V>EQ-*&60bcLmZXc0LLfJgDj%by5j-S4IYlpNjoascu&W=(O60g!C$v)+qg{ zonu}4^GzNvEb>}n=gb>xaw z^v~<3p$`E~<+FxU(?2;sG1IQx@UPi-fD@n&GunzYK&UT!GpkHnh-C zn2b5CO(Cud-C>jHAL6w2=NT!f9@8ffiYb77n7-E3rKE5rT)!1KD=qU{%o4d62DkvA ze;A3;@(C9`9>CHgI$W1T45@?yTpGy(FpZ#Q#F=N^A_L(mcuMMUv{J@U;3rxE#LmaY zBC?qco+r=XlPcUdsoZwJ5Tt3vLJ6DCO6;s^G~?UGzHaMnst0IfF=jwC|;q9Mms)tbrAb4>aG7TwGr z3mT120fr6^V!(40^7>VXP!xRWo+>bbli*2ul|h=T5T0gh>QwhVA~buC(C141CGQHO%IxEjlfe+vTq+Ci*x4zNj{<3Q&|>Ul%3_@=|K))sEJjG1R2$L5*Ti zF;Pq|Ek!|U!lJG6q^=X<3M7E{X0S#NM*xcPY6qRihC~Ey&p?I6W+*4)4q_raY{*J4 zjdI=DB*4Oay+o!rb{MowkP^(oWCOt635A%ueGaG0psB)*jNPDL0y$BxZJ=?Roj?e4 zFfu9#3BHyO%pI+?%mQOB2WC&_ikgPn0wch-#L)w@Fz^bHaYk}5mwcpuIHG9GkSvmO z-6k!3%Vk3BW{NN*AvGin)_@=?EaP>xc_q-L<(C=1kLVq)1cK5~TNgHz%77df<6A1N zu-S}W2bAFEOvFO|Rd36TNEBk{@EUs#$7LTCjsq8~jLt}8bOb@+VaXAGbv%^k>43mt!mnZEokk&;Pm#{)0Gz7^ z(O7BrrQZjkG^1WMU8KVu&iM3F^|+`h-#NIxH(_6V$TgB(8ZG;J)Vii%MdUSy6RYF{ z;;@FAUf;ga0Qq$({WIA^P7|8Qs|t#umx_C{jNxfMl_8F0e1N6ucLmgNnCPA5mvFt# z3&hQ#Y-}UYVdNt>oc3lzgLsdoRQOH=Oe2NCOuIU}Q zI=n4$)=wfHDmtKN3-bJX;pt=U&r+^8yFUO+H(LFLhuO(NY6Uo0@#8jQh@Ko*LQ=Xo zF{$({JocUuN7Xio<1-*-wJPMK(KH^=v1G|j4>c-0T?>Mc;j_hWcZWr=;OMOG0k26( zPPJM{Rup!S_PMB;g^9WFIraaf6eyo9< zHYS$Q*y}pLi`L3AcgX{dg3a!(!A_*g(P>^iNNO6aF5wM9P(}JF)Cq(O>1IyVn8M&3 zg)y?Au30jUzOl$gvYQco9#-ZKWjZPA%3oEN55Q9?fR4>8XhngNiaoVZiPD1aMG{?l zJmM8Jxve2w@!Jo)v=s~h8=Rb~VFB}*cUGvV^iF8C&y7xW_AGA(<1D){>bf5mMOB$c zcKTHU>Q1n{ScK%0xL)$Zp1!=QKwn-Ei`rTa13QC)_`;eMyNIvi)7ZuJKg{x* zc;80mk5(070c3PjYiZZYC#vBLR+$uyY1cgnp4*uoQGg>@y=K#b<5i zQI+9AcP+4$hgNpvuV%ftl6T5G|68BpQ#X@R+FvQ{*6ECv^cEi3XQjt`Yi# z_W@!+%~v6!CqWNX67;RlwWiFHwR;xXUcH=OY|iU5wL?1vyRoG^=UGbeK=lCo1DQ@s zRr#2dSH-~7%`@&wxkIs)EKB*+&Va`+K~>Y$$ABgW8MA#lytV;BOErh+xPa*@rM+|gP^Q7%hWC!HlUgRn8>sRy_s z`51loO^Ht%Dam2Cx|+zMQ2<_K~dBv z)df=D6adb2!U7@Q4~6}Np{EWRmMrpaA>xgDzJ_v)>$uJ{<&$h}jkxokG|wIBiLcg^ zn8%oP<=|Fp1L>Ntn1d9$9(?i{zL=-{kJ>uo@3pjTCd8W?yww(`R3$d zI3|Fvlr&+}WT8Qb>^AEw#S?Uu2eRB*qbX!5_>IF11YDTb^zwXnez;3Ox7u#{h=^S|3F;CudEt)r_1uGUA0%0UN;lp9byF zgpq!&cT$8>T5_liXqJmWm^I_d_ARp9&!}B8$s`DDR5O<1E0BC-WI5RqDfZk3O$!8s zhCq1H%yo1`(D5gI6qM^m?q87Fk#?h$^C2a$O~4)wuo9-SwW*baNn5k>F$s#wSJQ<& z+nEn=XQP9tQ$@@~X@SZKq6U1>klN~u{N@vJNiWBcp&c9!DSgJ;lt-3rE3J5%IgF7l z1E{*H0{huX^Sem|No|4qL7v-ttqul>#Xu{G2Q;nL&@{*q9ogxCK>j3?4WV7kzQ)SWnY38tCsC|p^25dO&Y%Y z(r*ATxUR3!F=K{REK#BHT2yb1J8z>TfKhmfQ}Q}hx~!$pxE!0D(kmvNUk)kx*W$v? zh+9e?wkUOZar7!;qsZau#8onq&~P+yV~IYCy^LdZwE#*zV>R9oUO_xWZgXIMCaGcz zAe1mEiRTY44@ApfQ~6v&WWyu~7$_Mjp;>3fUn_~!8s21FaF#7zTK7rvDLpbM8mXJL zP;2~vGAkS?fGD~Z1XH!qQV0kVK*(MR28*D@=T6fF>_ZMT0um98o&}yM-vT8A>2T>S zT^KmV;cecT@>9I*XNl%G z&gN?48c&}6!OlfuNlXvt8n7G&%uX#hN4RFjLl9IP} z@LgWy)!*GR1|lv*$P)%YY--iQ zOj7wVRQv-Gzb#AB%;f&c3hBP4Z32#?Nfrtasdlrl7+OxBHDKI1*$VWDq<5~`+K_^v z>*I~kI5@<4?z(fnIi2X^Ok(#Ne$y^+QQuvib;kbDSAsAi+H5N|)Cet76s91VY@&pf z(H_;)=~*KtEZ1$|K^e*;>CRW6DO#e(ykENrfJNKs>x za8FT&1Vg|hLkbN+iYbkERY~!n+C;s8a8mn%k!gqnZ6OFMChY8=r^SP^q4(G-2@gz_ zvr_Az=2R9U1k`Rt<#G!Vtrf&q)B>yxQ#Km)91O~X!;+6kM|e@z%*O%n*x~J2McRWT zg{pPTzDZ6x+@uF^owL)*>*YL{Npq5b>|QI8EI)F%=)Ti2t1xKbWk@+$y!&Z#gj8W?F&mzJ(A7 z8YjzUk_B!a7c1xa!d`Oyp00x81I$cBc9$I^-7TKMm=DXwK><*z(1v)2wa7TVR)k_z zJ|n)SxwYHp3-NbT^j`wHjci$mO4I<4!-era8b&O(%;>*X4{A&D0p5mci1RNI8DQ)t z906%Ik)rf1G@c7I`%`HYT7;})@3|IwS&$)A<`Rbjb#RGPD7dMyy)q!QCeOhvt*$Ve z%YJqt{nv+-O&2f(rgET@v_~`FTf&prr11RE->$M2OyP?0Ks#I;7kA5VOyJ|91Qch8 zB*XMsJT&=bJ5T3Q=mVjP4lCA1Xq#K3M|K4h0I@mcfk#XVSy#KH$%{r|<>LHqU1HFO ztfaWE3%oRQCbL4bC#R9Dg^7u!CJsgdiOutgj2%G7!pKAIWg(xOI;a5UL+MN-1s)JC z(+KeaP~m`85NLpmoVe4Hm~87WG;aY^~{pG z(s{8OqB}R0sHj7swt>be&heaD*l=VGQApaWPn0e|DNqxTfi$jH*~B$>?qS7RRFR<$ z4e|Wi=7-qhL1Rfw~T`dLm#q7<3A%P@qMW$u#S^P zw@z9f>VZ)~%=sq(lsE-MbH>62!yH0OXk#h>(V8oA9!LtZf!9_P0m2J~PMB>zf_XP3 zO>)yq8H}lp#aM+vg%xQ{nfIIZtE$UCH5E z?>~LXpS|PG_qO}3xaS*do_)+g2XmRJYCb~}yP9?2^PYwQrJD*RNGuSE8&+`9Q2tIU zEP2IGNXJPdzuMj*kz=iKZ$jc2;V;MDQ!7#}yit|M12CM{YUg~U&YxJl6yVF2`cvl` z1A+n%a8qa&GO}wyP3x-|Kqg({rMiaU*}17o)eRwcV~&A}TdOxTZl%__Vdjz#F;Vo@ zz%e46{DKZVVIz2*5Ldi#=auH++u`B+#2q6l06^)+DbyZ-;}c)Km zo9)3s%cY>q5Hc8~4zyV|k6WHoI;vk#L9xMyW+1zFK&v1MRL=oLT4baOTH%F6PMDmN zh|wtOh06(|AU()TiTu@Yp5VMM~5osPSA#k&@BE*@q!4Z@E^aCNZ8Ppzmk};sM zQ603dMM2>1qv<(MQ~Mc7%6No*IYrMHo1m?GhH7+a1~rG6={v=V`@1XA`M{4 zRR%K{H@Pd-Z$vkIwi+BlZf{BrU!qk}K^{jIhS1cNw$|m@ID&S8bcB$u!M^gl4U|&a zk3&~#1de0XEfRzvHKQ}3g+$ZNl&(xooCijI z<8fi6nN5*-&Ypn7K(q>)u!DgH8Jtn&Y6c=HmPloplz&lH5uV@~zl6F-EXUSN}&)<<1Qv;nW5QF2xthTE$wljH^ zHu;A{vZvO@8u_e3Rjk(CWPnba#5Cw3gOmpNfha;!JiuZJ;8Y{vTpgg=+N8kw*;Yq% z6MlQvn2(>wa*tF?p)AO!=TfLO#(kv5l|6)el?Ylsr^U_@T=e5;J53X}FvFdC-br{@ zLOnIg4%j`FCRuRDHq%KUuSQy;8*p}dXPez3Uii&O<=LcFg-FayEmeLKL&JR?g;>yl z=cV_#Hz+B!^b;Cae*41if>uhfZF(cz1w{heL{d`3=@1};+LK);HYJT{B1l%2Gma!+ zl?J9Y(B6XsC1#}v46e0k!NC~YWXgr+>H2&diB>qxNy#8AvV2oGmY5fkZ9;-0ZUZ_f z>V@Ufj3y0@;)Lk{wR#x&dnwJ+%&;7hv3%p8LZRy$mKRYvu!F4Pu<@Ev z@y^$Tcf|rX&9l`@v`GwjogJRKj)dDKgW-2ntZGd1s}p$~ZrBb5*Ouvift)9QA8oQ% zn%8{Y6f7@z9mNb}3S27eiHZo}a%SDmyt8i=P4!D}%ba0@Cs}Q@qLew0qq|O~WzWE} zknsh$uMZ}I323WhD@eMbo!J)P@7m^WYak7#VILf!%zp4LOy6R-mguI5+N+*7DpT1y3*{d2B1%D##4_2PF^2Sr!BoTvh72E zR*8HPAONOZDE!BVDnldV(RQOp0>MlnaENws%ATuiJ-2@@0S_ZBaHHCQ(&9AKdDOep ze=3g+SB=Ny6X<3j$0u-LwSqmmWmy*qMt$gJKB8uiQFz;7--w3Emv<4mAJp=&kF?;W zNQqE!+}GF%)G3V@pMSungxRQ3K*Qg!$M;LHjz-> zOgh7O{(d3jj+zjkNgL}#3n*IV+`40y+kowY6sDC`ScH!wlbo$Vo%zUe0(k**!<|$K zKw}icWRG%8($uQ|5ntakv`5W(Ss`H7#_Cc?g5SQK;JF<~W-)?4pp|^EDD-ahZOzB{N`y zC1W0|04TwM0vSErZAUOr6yd_Ax_fvwt1D;+3!x?HzTc6`$TlsX35|78XMlh!_AZm# zQ5?NWaP2S*PbDrqoc5#A(^s_kXj%JPh>*s4ciFYngz+kiy=fMo6qmIL?8xFUDe^+` zbZC;96Rkumt5~?Xjvz?l=?oYttZmG2tSz~s`nz;bx^@S*gQdBFS8%CE*ht{mL_V;T z$q;vM`y zJ##3WL3FZg!F!|`P@*UaN3q>R*GI>Q0ydG#aqCo(=_5b3weU<2aX3B9*a9ZS%n~6m zLF#UviS|Iml?bH;sRx*&=fJ*A9x73;p=R{gT~B~`8JlK1QOp@4!9 z9Hi!3Ht78t5Au~yZF%BbwkR)<9TdRlg!{Y_t5xr1n-9j#+a_iLnH!a&6E4+A^XvM& zTG3P*H)?$zDGy`aS1}YWBtIN4RRzD2JeIQ6;?^kxG2#s(_RC7etHu+0gcNvw4l&vU z`9vCw`=I_Yvo0RqAjjTjK&!g&fsh*LsJcQVZK?%QkEbuQoWSpy2f`8@GA+{OL_~LNqU(x>Wo{xms}$$qk(+8R zLY^><0t{lt)A;~nOjDF3aDc~SO|WB2F;4A`4UUkrEYDtPY;aX7RQ}x2P}6V}5P5Z< z6||ecb=DYzjFF864?17F2+IIQ;%~jCstE%KlqhfU&`4uVKh5q7YWZ1xby(G;{59{_ xBHCE1TNVb+zE+x_tZTND_AqVEz5B27iXtceKt)g&{-^(oxgwk>NLc*h3V^#Ix3d5M diff --git a/tiledb/sm/misc/mgc_dict.cc b/tiledb/sm/misc/mgc_dict.cc index 5a42f253e851..1bafe99fabbd 100644 --- a/tiledb/sm/misc/mgc_dict.cc +++ b/tiledb/sm/misc/mgc_dict.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,55 +28,38 @@ #include "mgc_dict.h" -#include "tiledb/sm/compressors/util/gzip_wrappers.h" +#include "magic_mgc.zst.h_" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/compressors/zstd_compressor.h" #include #include #include -namespace tiledb { -namespace sm { +namespace tiledb::sm::magic_dict { -static const char magic_mgc_compressed_bytes[] = { -#include "magic_mgc_gzipped.bin" -}; - -shared_ptr magic_dict::expanded_buffer_; - -void* magic_dict::uncompressed_magic_dict_ = nullptr; - -static magic_dict magic_dict_object; - -magic_dict::magic_dict() { -} - -void magic_dict::prepare_data() { - if (uncompressed_magic_dict_) - return; - - expanded_buffer_ = make_shared(HERE()); - gzip_decompress( - expanded_buffer_, - reinterpret_cast(&magic_mgc_compressed_bytes[0])); - - uncompressed_magic_dict_ = expanded_buffer_.get()->data(); +void prepare_data(span expanded_buffer) { + ConstBuffer input(magic_mgc_compressed_bytes, magic_mgc_compressed_size); + PreallocatedBuffer output(expanded_buffer.data(), expanded_buffer.size()); + ZStd::ZSTD_Decompress_Context ctx; + ZStd::decompress(ctx, input, output); } -int magic_dict::magic_mgc_embedded_load(magic_t magic) { - if (!uncompressed_magic_dict_) - prepare_data(); +int magic_mgc_embedded_load(magic_t magic) { + auto buffer = expanded_buffer(); - void* data[1] = {uncompressed_magic_dict_}; - size_t sizes[1] = {expanded_buffer_->size()}; + void* data = const_cast(buffer.data()); + size_t size = buffer.size(); // zero ok, non-zero error - return magic_load_buffers(magic, &data[0], &sizes[0], 1); + return magic_load_buffers(magic, &data, &size, 1); } -const shared_ptr magic_dict::expanded_buffer() { - if (!uncompressed_magic_dict_) - prepare_data(); - return expanded_buffer_; +span expanded_buffer() { + static std::vector expanded_buffer(magic_mgc_decompressed_size); + static std::once_flag once_flag; + // Thread-safe initialization of the expanded data. + std::call_once(once_flag, [&]() { prepare_data(expanded_buffer); }); + return expanded_buffer; } -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::magic_dict diff --git a/tiledb/sm/misc/mgc_dict.h b/tiledb/sm/misc/mgc_dict.h index aa2b53fea15c..65bb020ff3bb 100644 --- a/tiledb/sm/misc/mgc_dict.h +++ b/tiledb/sm/misc/mgc_dict.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,49 +29,26 @@ #include "magic.h" #include "tiledb/common/common.h" -#include "tiledb/sm/misc/types.h" -namespace tiledb { -namespace sm { +namespace tiledb::sm::magic_dict { -using tiledb::common::Status; - -class magic_dict { - public: - /** Default constructor */ - magic_dict(); - - /** - * Have libmagic load data from our embedded version. - * - * @param magic - libmagic object obtained from magic_open() - * @return the value libmgaic returns from magic_load_buffers(). - */ - static int magic_mgc_embedded_load(magic_t magic); - - /** - * Provides access to the internally expanded data. - * - * @return a shared pointer to the internal ByteVecValue holding - * the expanded data. - */ - static const shared_ptr expanded_buffer(); - - private: - /** - * decompress and init data items - */ - static void prepare_data(); - - /** - * raw pointer to expanded bytes for use with magic_load_buffers(), - * points to the data held within expanded_buffer_ - */ - static void* uncompressed_magic_dict_; +/** + * Have libmagic load data from our embedded version. + * + * @param magic - libmagic object obtained from magic_open() + * @return the value libmgaic returns from magic_load_buffers(). + */ +int magic_mgc_embedded_load(magic_t magic); - /** holds the expanded data until application exits. */ - static shared_ptr expanded_buffer_; -}; +/** + * Provides access to the internally expanded data. + * + * Data is stored in the library in a compressed form (approx. 270KB) and gets + * decompressed (approx. 7MB) on the first call to this function. Subsequent + * calls to this function will reuse the decompressed buffer. + * + * @return a span to the internal buffer holding the expanded data. + */ +span expanded_buffer(); -} // namespace sm -} // namespace tiledb +} // namespace tiledb::sm::magic_dict diff --git a/tiledb/sm/misc/test/CMakeLists.txt b/tiledb/sm/misc/test/CMakeLists.txt index a0dac49341cb..8064459cd00a 100644 --- a/tiledb/sm/misc/test/CMakeLists.txt +++ b/tiledb/sm/misc/test/CMakeLists.txt @@ -38,3 +38,11 @@ commence(unit_test misc) unit_math.cc ) conclude(unit_test) + +commence(unit_test mgc_dict) + this_target_object_libraries(mgc_dict) + this_target_compile_definitions(TILEDB_PATH_TO_MAGIC_MGC=\"${libmagic_DICTIONARY}\") + this_target_sources( + unit_mgc_dict.cc + ) +conclude(unit_test) diff --git a/tiledb/sm/misc/test/compile_mgc_dict_main.cc b/tiledb/sm/misc/test/compile_mgc_dict_main.cc new file mode 100644 index 000000000000..977a8c9b167b --- /dev/null +++ b/tiledb/sm/misc/test/compile_mgc_dict_main.cc @@ -0,0 +1,34 @@ +/** + * @file compile_mgc_dict_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../mgc_dict.h" + +int main() { + (void)tiledb::sm::magic_dict::expanded_buffer(); + return 0; +} diff --git a/tiledb/sm/misc/test/unit_mgc_dict.cc b/tiledb/sm/misc/test/unit_mgc_dict.cc index 579cf5a780a5..c9ec25298e3b 100644 --- a/tiledb/sm/misc/test/unit_mgc_dict.cc +++ b/tiledb/sm/misc/test/unit_mgc_dict.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,66 +32,36 @@ * magic checks return same values using both embedded and external data. */ -#include "tiledb/common/common.h" -#include "tiledb/sm/buffer/buffer.h" -#include "tiledb/sm/filter/filter_buffer.h" -#include "tiledb/sm/filter/filter_storage.h" +#include #include "tiledb/sm/misc/mgc_dict.h" -#include "tiledb/sm/misc/types.h" -#include #include -#include +#include #include #include -#ifdef _WIN32 -#include -#include -#endif - #ifndef TILEDB_PATH_TO_MAGIC_MGC #error "TILEDB_PATH_TO_MAGIC_MGC not defined!" #endif -using tiledb::sm::magic_dict; - -int check_embedded_data_validity() { - FILE* infile = nullptr; - infile = fopen(TILEDB_PATH_TO_MAGIC_MGC, "rb"); - if (!infile) { - fprintf(stderr, "ERROR: Unable to open %s\n", TILEDB_PATH_TO_MAGIC_MGC); - return 1; - } - - fseek(infile, 0L, SEEK_END); - uint64_t magic_mgc_len = ftell(infile); - fseek(infile, 0L, SEEK_SET); +using namespace tiledb::sm; - char* magic_mgc_data = tdb_new_array(char, magic_mgc_len); - if (fread(magic_mgc_data, 1, magic_mgc_len, infile) != magic_mgc_len) { - fprintf(stderr, "ERROR reading data from %s\n", TILEDB_PATH_TO_MAGIC_MGC); - return 4; +TEST_CASE("Test embedded data validity", "[mgc_dict][embedded_validity]") { + std::vector magic_mgc_data; + { + std::ifstream infile(TILEDB_PATH_TO_MAGIC_MGC, std::ios::binary); + magic_mgc_data.assign( + std::istreambuf_iterator(infile), + std::istreambuf_iterator()); } - fclose(infile); auto magic_mgc_embedded_data = tiledb::sm::magic_dict::expanded_buffer(); - if (magic_mgc_embedded_data->size() != magic_mgc_len) { - fprintf( - stderr, - "ERROR magic.mgc data len (%" PRIu64 - ") does not match embedded data length (%" PRIu64 ")\n", - magic_mgc_len, - static_cast(magic_mgc_embedded_data->size())); - return 7; - } - - if (memcmp(magic_mgc_data, magic_mgc_embedded_data->data(), magic_mgc_len)) { - fprintf(stderr, "ERROR magic.mgc data different from embedded data\n"); - return 10; - } - - return 0; + REQUIRE(magic_mgc_data.size() == magic_mgc_embedded_data.size()); + REQUIRE( + memcmp( + magic_mgc_data.data(), + magic_mgc_embedded_data.data(), + magic_mgc_data.size()) == 0); } char empty_txt[] = {""}; // further below, treated differently from others here @@ -224,282 +194,182 @@ char text_txt[] = { '\x6c', '\x69', '\x6e', '\x65', '\x73', '\x2e', '\x0a', }; -struct file_data_sizes_s { - const char* file_name; - char* file_data; - uint64_t file_data_len; -} file_data_sizes1[] = - {{"empty_text", empty_txt, strlen(empty_txt)}, - {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, - {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, - {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, - {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, - {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, - {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, - {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, - {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, - {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, - {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, - {"quickstart_dense_csv", - quickstart_dense_csv, - sizeof(quickstart_dense_csv)}, - {"quickstart_dense_csv_gz", - quickstart_dense_csv_gz, - sizeof(quickstart_dense_csv_gz)}, - {"text_txt", text_txt, sizeof(text_txt)}, +struct FileData { + const std::string_view file_name; + span file_data; - {0, 0, 0}}, - file_data_sizes2[] = { - {"text_txt", text_txt, sizeof(text_txt)}, - {"quickstart_dense_csv_gz", - quickstart_dense_csv_gz, - sizeof(quickstart_dense_csv_gz)}, - {"quickstart_dense_csv", - quickstart_dense_csv, - sizeof(quickstart_dense_csv)}, - {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, - {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, - {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, - {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, - {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, - {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, - {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, - {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, - {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, - {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, - {"empty_text", empty_txt, strlen(empty_txt)}, - - {0, 0, 0}}; - -int embedded_vs_external_identifications() { - int errcnt = 0; + constexpr FileData( + std::string_view file_name, const char* file_data, size_t file_data_len) + : file_name(file_name) + , file_data(file_data, file_data_len) { + } +}; +std::vector file_data = { + {"empty_text", empty_txt, strlen(empty_txt)}, + {"fileapi0_csv", fileapi0_csv, sizeof(fileapi0_csv)}, + {"fileapi1_csv", fileapi1_csv, sizeof(fileapi1_csv)}, + {"fileapi2_csv", fileapi2_csv, sizeof(fileapi2_csv)}, + {"fileapi3_csv", fileapi3_csv, sizeof(fileapi3_csv)}, + {"fileapi4_csv", fileapi4_csv, sizeof(fileapi4_csv)}, + {"fileapi5_csv", fileapi5_csv, sizeof(fileapi5_csv)}, + {"fileapi6_csv", fileapi6_csv, sizeof(fileapi6_csv)}, + {"fileapi7_csv", fileapi7_csv, sizeof(fileapi7_csv)}, + {"fileapi8_csv", fileapi8_csv, sizeof(fileapi8_csv)}, + {"fileapi9_csv", fileapi9_csv, sizeof(fileapi9_csv)}, + {"quickstart_dense_csv", + quickstart_dense_csv, + sizeof(quickstart_dense_csv)}, + {"quickstart_dense_csv_gz", + quickstart_dense_csv_gz, + sizeof(quickstart_dense_csv_gz)}, + {"text_txt", text_txt, sizeof(text_txt)}}; + +TEST_CASE("Test embedded data validity", "[mgc_dict][embedded_vs_external]") { magic_t magic_mimeenc_embedded; magic_t magic_mimeenc_external; magic_t magic_mimetyp_embedded; magic_t magic_mimetyp_external; - auto magic_opens = [&]() { - auto magic_debug_options = 0; // MAGIC_DEBUG | MAGIC_CHECK | MAGIC_ERROR; - magic_mimeenc_embedded = - magic_open(MAGIC_MIME_ENCODING | magic_debug_options); - if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimeenc_embedded); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = - "(magic_error() returned 0, unexpected error loading embedded " - "data "; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 1; + auto magic_debug_options = 0; // MAGIC_DEBUG | MAGIC_CHECK | MAGIC_ERROR; + magic_mimeenc_embedded = + magic_open(MAGIC_MIME_ENCODING | magic_debug_options); + if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimeenc_embedded); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimeenc_embedded); + if (!err) { + err = + "(magic_error() returned 0, unexpected error loading embedded " + "data "; } - magic_mimeenc_external = - magic_open(MAGIC_MIME_ENCODING | magic_debug_options); - if (auto rval = - magic_load(magic_mimeenc_external, TILEDB_PATH_TO_MAGIC_MGC); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimeenc_external); - if (!err) { - err = - "(magic_error() returned 0, try setting env var 'MAGIC' to " - "location " - "of magic.mgc or " - "alternate!)"; - } - magic_close(magic_mimeenc_external); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 2; + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimeenc_external = + magic_open(MAGIC_MIME_ENCODING | magic_debug_options); + if (auto rval = magic_load(magic_mimeenc_external, TILEDB_PATH_TO_MAGIC_MGC); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimeenc_external); + if (!err) { + err = + "(magic_error() returned 0, try setting env var 'MAGIC' to " + "location " + "of magic.mgc or " + "alternate!)"; + } + magic_close(magic_mimeenc_external); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimetyp_embedded = magic_open(MAGIC_MIME_TYPE | magic_debug_options); + if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimetyp_embedded); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimetyp_embedded); + if (!err) { + err = + "(magic_error() returned 0, unexpected error loading embedded " + "data "; + } + magic_close(magic_mimetyp_embedded); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + magic_mimetyp_external = magic_open(MAGIC_MIME_TYPE | magic_debug_options); + if (auto rval = magic_load(magic_mimetyp_external, TILEDB_PATH_TO_MAGIC_MGC); + rval != 0) { + auto str_rval = std::to_string(rval); + auto err = magic_error(magic_mimetyp_external); + if (!err) { + err = + "(magic_error() returned 0, try setting env var 'MAGIC' to " + "location " + "of magic.mgc or " + "alternate!)"; } - magic_mimetyp_embedded = magic_open(MAGIC_MIME_TYPE | magic_debug_options); - if (auto rval = magic_dict::magic_mgc_embedded_load(magic_mimetyp_embedded); - rval != 0) { - auto str_rval = std::to_string(rval); - auto err = magic_error(magic_mimetyp_embedded); + magic_close(magic_mimetyp_external); + throw std::runtime_error( + std::string("cannot load magic database - ") + str_rval + err); + } + + for (auto& file_item : file_data) { + const char* mime_type_embedded = nullptr; + const char* mime_type_external = nullptr; + const char* mime_enc_embedded = nullptr; + const char* mime_enc_external = nullptr; + + mime_type_embedded = magic_buffer( + magic_mimetyp_embedded, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_type_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); + auto err = magic_error(magic_mimeenc_embedded); if (!err) { - err = - "(magic_error() returned 0, unexpected error loading embedded " - "data "; + err = "(magic_buffer(..._embedded) returned 0!)"; } - magic_close(magic_mimetyp_embedded); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 1; + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot access mime_type - ") + str_rval + err); } - magic_mimetyp_external = magic_open(MAGIC_MIME_TYPE | magic_debug_options); - if (auto rval = - magic_load(magic_mimetyp_external, TILEDB_PATH_TO_MAGIC_MGC); - rval != 0) { - auto str_rval = std::to_string(rval); + mime_type_external = magic_buffer( + magic_mimetyp_external, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_type_external) { + auto str_rval = std::to_string(magic_errno(magic_mimetyp_external)); auto err = magic_error(magic_mimetyp_external); if (!err) { - err = - "(magic_error() returned 0, try setting env var 'MAGIC' to " - "location " - "of magic.mgc or " - "alternate!)"; + err = "(magic_buffer(..._external) returned 0!)"; } magic_close(magic_mimetyp_external); - std::cerr << std::string("cannot load magic database - ") + str_rval + - err; - return 2; + throw std::runtime_error( + std::string("cannot access mime_type - ") + str_rval + err); } - return 0; - }; - auto magic_closes = [&]() { - magic_close(magic_mimeenc_embedded); - magic_close(magic_mimeenc_external); - magic_close(magic_mimetyp_embedded); - magic_close(magic_mimetyp_external); - }; + CHECK( + std::string_view(mime_type_embedded) == + std::string_view(mime_type_external)); - auto proc_list = [&](file_data_sizes_s file_data_sizes[], - bool global_open_close = true) { - if (global_open_close) { - if (auto rval = magic_opens(); rval != 0) { - return rval; + mime_enc_embedded = magic_buffer( + magic_mimeenc_embedded, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_enc_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); + auto err = magic_error(magic_mimeenc_embedded); + if (!err) { + err = "(magic_buffer(..._embedded) returned 0!)"; } + magic_close(magic_mimeenc_embedded); + throw std::runtime_error( + std::string("cannot access mime_encoding - ") + str_rval + err); } - - auto file_item = &file_data_sizes[0]; - while (file_item->file_data) { - if (!global_open_close) { - if (auto rval = magic_opens(); rval != 0) { - return rval; - } - } - const char* mime_type_embedded = nullptr; - const char* mime_type_external = nullptr; - const char* mime_enc_embedded = nullptr; - const char* mime_enc_external = nullptr; - - mime_type_embedded = magic_buffer( - magic_mimetyp_embedded, - file_item->file_data, - file_item->file_data_len); - if (!mime_type_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot access mime_type - ") + str_rval + err; - return 2; - } - mime_type_external = magic_buffer( - magic_mimetyp_external, - file_item->file_data, - file_item->file_data_len); - if (!mime_type_external) { - auto str_rval = std::to_string(magic_errno(magic_mimetyp_external)); - auto err = magic_error(magic_mimetyp_external); - if (!err) { - err = "(magic_buffer(..._external) returned 0!)"; - } - magic_close(magic_mimetyp_external); - std::cerr << std::string("cannot access mime_type - ") + str_rval + err; - return 2; - } - - if (strcmp(mime_type_embedded, mime_type_external)) { - fprintf( - stderr, - "ERROR mismatch (%s) mime type embedded (%s) vs external (%s)\n", - file_item->file_name, - mime_type_embedded, - mime_type_external); - ++errcnt; - } - - mime_enc_embedded = magic_buffer( - magic_mimeenc_embedded, - file_item->file_data, - file_item->file_data_len); - if (!mime_enc_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_embedded)); - auto err = magic_error(magic_mimeenc_embedded); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_embedded); - std::cerr << std::string("cannot access mime_encoding - ") + str_rval + - err; - return 2; - } - mime_enc_external = magic_buffer( - magic_mimeenc_external, - file_item->file_data, - file_item->file_data_len); - if (!mime_enc_embedded) { - auto str_rval = std::to_string(magic_errno(magic_mimeenc_external)); - auto err = magic_error(magic_mimeenc_external); - if (!err) { - err = "(magic_buffer(..._embedded) returned 0!)"; - } - magic_close(magic_mimeenc_external); - std::cerr << std::string("cannot access mime_encoding - ") + str_rval + - err; - return 2; - } - - if (strcmp(mime_enc_embedded, mime_enc_external)) { - fprintf( - stderr, - "ERROR mismatch (%s) mime encoding embedded (%s) vs external " - "(%s)\n", - file_item->file_name, - mime_enc_embedded, - mime_enc_external); - ++errcnt; - } - file_item++; - if (!global_open_close) { - magic_closes(); + mime_enc_external = magic_buffer( + magic_mimeenc_external, + file_item.file_data.data(), + file_item.file_data.size()); + if (!mime_enc_embedded) { + auto str_rval = std::to_string(magic_errno(magic_mimeenc_external)); + auto err = magic_error(magic_mimeenc_external); + if (!err) { + err = "(magic_buffer(..._embedded) returned 0!)"; } - } - if (global_open_close) { - magic_closes(); + magic_close(magic_mimeenc_external); + throw std::runtime_error( + std::string("cannot access mime_encoding - ") + str_rval + err); } - return 0; - }; - - if (auto rval = proc_list(file_data_sizes1, true); rval != 0) - return rval; - if (auto rval = proc_list(file_data_sizes2, true); rval != 0) - return rval; - - if (auto rval = proc_list(file_data_sizes1, false); rval != 0) - return rval; - if (auto rval = proc_list(file_data_sizes2, false); rval != 0) - return rval; - - if (errcnt) { - fprintf( - stderr, "%d mismatch errors magic embedded vs external data\n", errcnt); + CHECK( + std::string_view(mime_enc_embedded) == + std::string_view(mime_enc_external)); } - return errcnt ? 99 : 0; -} - -int main() { - int failures = 0; - - failures |= check_embedded_data_validity(); - failures |= embedded_vs_external_identifications(); - - if (failures) { - fprintf(stderr, "\nERRORS mgc_dict validation\n"); - return 1; - } else { - fprintf(stderr, "NO errors encountered in mgc_dict validation\n"); - return 0; - } + magic_close(magic_mimeenc_embedded); + magic_close(magic_mimeenc_external); + magic_close(magic_mimetyp_embedded); + magic_close(magic_mimetyp_external); } From 571dbe25760ffabf565c16efa0f5289679cdb6dc Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 17 Jun 2024 11:13:46 -0400 Subject: [PATCH 440/456] Add a ContextResources& argument to the consolidation constructors. (#5074) Add a `ContextResources&` argument to the consolidation constructors. This is the first step in transitional work to replace `StorageManager` with `ContextResources` everywhere in the consolidation code. Future work has been documented appropriately. [sc-49373] --- TYPE: NO_HISTORY DESC: Add a `ContextResources&` argument to the consolidation constructors. --- tiledb/sm/c_api/tiledb.cc | 4 + .../consolidator/array_meta_consolidator.cc | 16 ++-- .../sm/consolidator/array_meta_consolidator.h | 17 +++- .../sm/consolidator/commits_consolidator.cc | 15 +-- tiledb/sm/consolidator/commits_consolidator.h | 16 +++- tiledb/sm/consolidator/consolidator.cc | 45 +++++---- tiledb/sm/consolidator/consolidator.h | 61 ++++++++++-- .../sm/consolidator/fragment_consolidator.cc | 96 +++++++++---------- .../sm/consolidator/fragment_consolidator.h | 15 ++- .../fragment_meta_consolidator.cc | 14 +-- .../consolidator/fragment_meta_consolidator.h | 16 +++- .../consolidator/group_meta_consolidator.cc | 14 +-- .../sm/consolidator/group_meta_consolidator.h | 17 +++- tiledb/sm/group/group.cc | 8 +- 14 files changed, 235 insertions(+), 119 deletions(-) diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 7b10b8f755e2..4b5a2c9f0892 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2665,6 +2665,7 @@ int32_t tiledb_array_consolidate( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { api::ensure_config_is_valid_if_present(config); tiledb::sm::Consolidator::array_consolidate( + ctx->resources(), array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, @@ -2684,6 +2685,7 @@ int32_t tiledb_array_consolidate_with_key( // Sanity checks tiledb::sm::Consolidator::array_consolidate( + ctx->resources(), array_uri, static_cast(encryption_type), encryption_key, @@ -2710,6 +2712,7 @@ int32_t tiledb_array_consolidate_fragments( } tiledb::sm::Consolidator::fragments_consolidate( + ctx->resources(), array_uri, tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, @@ -2724,6 +2727,7 @@ int32_t tiledb_array_consolidate_fragments( int32_t tiledb_array_vacuum( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { tiledb::sm::Consolidator::array_vacuum( + ctx->resources(), array_uri, (config == nullptr) ? ctx->config() : config->config(), ctx->storage_manager()); diff --git a/tiledb/sm/consolidator/array_meta_consolidator.cc b/tiledb/sm/consolidator/array_meta_consolidator.cc index 52c8356f3afa..0608227adcc5 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.cc +++ b/tiledb/sm/consolidator/array_meta_consolidator.cc @@ -47,8 +47,10 @@ namespace tiledb::sm { /* ****************************** */ ArrayMetaConsolidator::ArrayMetaConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw std::logic_error(st.message()); @@ -69,8 +71,8 @@ Status ArrayMetaConsolidator::consolidate( // Open array for reading auto array_uri = URI(array_name); - Array array_for_reads(storage_manager_->resources(), array_uri); - RETURN_NOT_OK(array_for_reads.open( + Array array_for_reads(resources_, array_uri); + throw_if_not_ok(array_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_, @@ -79,7 +81,7 @@ Status ArrayMetaConsolidator::consolidate( key_length)); // Open array for writing - Array array_for_writes(storage_manager_->resources(), array_uri); + Array array_for_writes(resources_, array_uri); RETURN_NOT_OK_ELSE( array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length), @@ -144,10 +146,10 @@ Status ArrayMetaConsolidator::set_config(const Config& config) { Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); diff --git a/tiledb/sm/consolidator/array_meta_consolidator.h b/tiledb/sm/consolidator/array_meta_consolidator.h index 0393db98e709..a8ad9d1a7312 100644 --- a/tiledb/sm/consolidator/array_meta_consolidator.h +++ b/tiledb/sm/consolidator/array_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,11 +55,21 @@ class ArrayMetaConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * ArrayMetaConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit ArrayMetaConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~ArrayMetaConsolidator() = default; diff --git a/tiledb/sm/consolidator/commits_consolidator.cc b/tiledb/sm/consolidator/commits_consolidator.cc index d0859aa9582f..e5ef4d42ec17 100644 --- a/tiledb/sm/consolidator/commits_consolidator.cc +++ b/tiledb/sm/consolidator/commits_consolidator.cc @@ -49,8 +49,9 @@ namespace tiledb::sm { /* CONSTRUCTOR */ /* ****************************** */ -CommitsConsolidator::CommitsConsolidator(StorageManager* storage_manager) - : Consolidator(storage_manager) { +CommitsConsolidator::CommitsConsolidator( + ContextResources& resources, StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { } /* ****************************** */ @@ -68,16 +69,16 @@ Status CommitsConsolidator::consolidate( // Open array for writing auto array_uri = URI(array_name); - Array array_for_writes(storage_manager_->resources(), array_uri); - RETURN_NOT_OK(array_for_writes.open( + Array array_for_writes(resources_, array_uri); + throw_if_not_ok(array_for_writes.open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Ensure write version is at least 12. auto write_version = array_for_writes.array_schema_latest().write_version(); - RETURN_NOT_OK(array_for_writes.close()); + throw_if_not_ok(array_for_writes.close()); if (write_version < 12) { - return logger_->status(Status_ConsolidatorError( - "Array version should be at least 12 to consolidate commits.")); + throw ConsolidatorException( + "Array version should be at least 12 to consolidate commits."); } // Get the array uri to consolidate from the array directory. diff --git a/tiledb/sm/consolidator/commits_consolidator.h b/tiledb/sm/consolidator/commits_consolidator.h index b5c1100b4ef9..ba1bf226864a 100644 --- a/tiledb/sm/consolidator/commits_consolidator.h +++ b/tiledb/sm/consolidator/commits_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,9 +55,18 @@ class CommitsConsolidator : public Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * CommitsConsolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit CommitsConsolidator(StorageManager* storage_manager); + explicit CommitsConsolidator( + ContextResources& resources, StorageManager* storage_manager); /** Destructor. */ ~CommitsConsolidator() = default; diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index b0e711b2b2be..4cd544147252 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -55,22 +55,26 @@ namespace tiledb::sm { /** Factory function to create the consolidator depending on mode. */ shared_ptr Consolidator::create( + ContextResources& resources, const ConsolidationMode mode, const Config& config, StorageManager* storage_manager) { switch (mode) { case ConsolidationMode::FRAGMENT_META: - return make_shared(HERE(), storage_manager); + return make_shared( + HERE(), resources, storage_manager); case ConsolidationMode::FRAGMENT: - return make_shared(HERE(), config, storage_manager); + return make_shared( + HERE(), resources, config, storage_manager); case ConsolidationMode::ARRAY_META: return make_shared( - HERE(), config, storage_manager); + HERE(), resources, config, storage_manager); case ConsolidationMode::COMMITS: - return make_shared(HERE(), storage_manager); + return make_shared( + HERE(), resources, storage_manager); case ConsolidationMode::GROUP_META: return make_shared( - HERE(), config, storage_manager); + HERE(), resources, config, storage_manager); default: return nullptr; } @@ -105,8 +109,9 @@ ConsolidationMode Consolidator::mode_from_config( /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ -Consolidator::Consolidator(StorageManager* storage_manager) - : resources_(storage_manager->resources()) +Consolidator::Consolidator( + ContextResources& resources, StorageManager* storage_manager) + : resources_(resources) , storage_manager_(storage_manager) , consolidator_memory_tracker_(resources_.create_memory_tracker()) , stats_(resources_.stats().create_child("Consolidator")) @@ -125,8 +130,7 @@ Status Consolidator::consolidate( [[maybe_unused]] EncryptionType encryption_type, [[maybe_unused]] const void* encryption_key, [[maybe_unused]] uint32_t key_length) { - return logger_->status( - Status_ConsolidatorError("Cannot consolidate; Invalid object")); + throw ConsolidatorException("Cannot consolidate; Invalid object"); } void Consolidator::vacuum([[maybe_unused]] const char* array_name) { @@ -134,6 +138,7 @@ void Consolidator::vacuum([[maybe_unused]] const char* array_name) { } void Consolidator::array_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -148,8 +153,7 @@ void Consolidator::array_consolidate( // Check if array exists ObjectType obj_type; - throw_if_not_ok( - object_type(storage_manager->resources(), array_uri, &obj_type)); + throw_if_not_ok(object_type(resources, array_uri, &obj_type)); if (obj_type != ObjectType::ARRAY) { throw ConsolidatorException( @@ -158,8 +162,7 @@ void Consolidator::array_consolidate( if (array_uri.is_tiledb()) { throw_if_not_ok( - storage_manager->resources().rest_client()->post_consolidation_to_rest( - array_uri, config)); + resources.rest_client()->post_consolidation_to_rest(array_uri, config)); } else { // Get encryption key from config std::string encryption_key_from_cfg; @@ -190,13 +193,15 @@ void Consolidator::array_consolidate( // Consolidate auto mode = Consolidator::mode_from_config(config); - auto consolidator = Consolidator::create(mode, config, storage_manager); + auto consolidator = + Consolidator::create(resources, mode, config, storage_manager); throw_if_not_ok(consolidator->consolidate( array_name, encryption_type, encryption_key, key_length)); } } void Consolidator::fragments_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -212,8 +217,7 @@ void Consolidator::fragments_consolidate( // Check if array exists ObjectType obj_type; - throw_if_not_ok( - object_type(storage_manager->resources(), array_uri, &obj_type)); + throw_if_not_ok(object_type(resources, array_uri, &obj_type)); if (obj_type != ObjectType::ARRAY) { throw ConsolidatorException( @@ -249,7 +253,7 @@ void Consolidator::fragments_consolidate( // Consolidate auto consolidator = Consolidator::create( - ConsolidationMode::FRAGMENT, config, storage_manager); + resources, ConsolidationMode::FRAGMENT, config, storage_manager); auto fragment_consolidator = dynamic_cast(consolidator.get()); throw_if_not_ok(fragment_consolidator->consolidate_fragments( @@ -315,19 +319,20 @@ void Consolidator::write_consolidated_commits_file( } void Consolidator::array_vacuum( + ContextResources& resources, const char* array_name, const Config& config, StorageManager* storage_manager) { URI array_uri(array_name); if (array_uri.is_tiledb()) { throw_if_not_ok( - storage_manager->resources().rest_client()->post_vacuum_to_rest( - array_uri, config)); + resources.rest_client()->post_vacuum_to_rest(array_uri, config)); return; } auto mode = Consolidator::mode_from_config(config, true); - auto consolidator = Consolidator::create(mode, config, storage_manager); + auto consolidator = + Consolidator::create(resources, mode, config, storage_manager); consolidator->vacuum(array_name); } diff --git a/tiledb/sm/consolidator/consolidator.h b/tiledb/sm/consolidator/consolidator.h index 1c7d10ae2388..dcbdc37ad2a8 100644 --- a/tiledb/sm/consolidator/consolidator.h +++ b/tiledb/sm/consolidator/consolidator.h @@ -37,6 +37,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include @@ -76,13 +77,22 @@ class Consolidator { /** * Factory method to make a new Consolidator instance given the config mode. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be create(ContextResources&, ...). + * + * @param resources The context resources. * @param mode Consolidation mode. * @param config Configuration parameters for the consolidation * (`nullptr` means default). - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) * @return New Consolidator instance or nullptr on error. */ static shared_ptr create( + ContextResources& resources, const ConsolidationMode mode, const Config& config, StorageManager* storage_manager); @@ -134,6 +144,14 @@ class Consolidator { /** * Consolidates the fragments of an array into a single one. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * array_consolidate(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be consolidated. * @param encryption_type The encryption type of the array * @param encryption_key If the array is encrypted, the private encryption @@ -142,9 +160,11 @@ class Consolidator { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void array_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -155,6 +175,14 @@ class Consolidator { /** * Consolidates the fragments of an array into a single one. * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * fragments_consolidate(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be consolidated. * @param encryption_type The encryption type of the array * @param encryption_key If the array is encrypted, the private encryption @@ -164,9 +192,11 @@ class Consolidator { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void fragments_consolidate( + ContextResources& resources, const char* array_name, EncryptionType encryption_type, const void* encryption_key, @@ -194,11 +224,21 @@ class Consolidator { * metadata. Note that this will coarsen the granularity of time traveling * (see docs for more information). * + * @section Maturity Notes + * This is a transitional method in the sense that we are working + * on removing the dependency of the Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * array_vacuum(ContextResources&, ...). + * + * @param resources The context resources. * @param array_name The name of the array to be vacuumed. * @param config Configuration parameters for vacuuming. - * @param storage_manager The storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ static void array_vacuum( + ContextResources& resources, const char* array_name, const Config& config, StorageManager* storage_manager); @@ -223,9 +263,18 @@ class Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * Constructs a Consolidator object given a ContextResources reference. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Consolidator class on StorageManager. For + * now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be Consolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit Consolidator(StorageManager* storage_manager); + explicit Consolidator( + ContextResources& resources, StorageManager* storage_manager); /** * Checks if the array is remote. diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 93d224f6ed34..0494b156d464 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -187,8 +187,10 @@ void FragmentConsolidationWorkspace::resize_buffers( /* ****************************** */ FragmentConsolidator::FragmentConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw FragmentConsolidatorException(st.message()); @@ -209,15 +211,14 @@ Status FragmentConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - auto array_for_reads{make_shared( - HERE(), storage_manager_->resources(), URI(array_name))}; - RETURN_NOT_OK(array_for_reads->open_without_fragments( + auto array_for_reads{make_shared(HERE(), resources_, URI(array_name))}; + throw_if_not_ok(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing - auto array_for_writes{make_shared( - HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; - RETURN_NOT_OK(array_for_writes->open( + auto array_for_writes{ + make_shared(HERE(), resources_, array_for_reads->array_uri())}; + throw_if_not_ok(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Disable consolidation with timestamps on older arrays. @@ -232,7 +233,7 @@ Status FragmentConsolidator::consolidate( // must be fetched (even before `config_.timestamp_start_`), // to compute the anterior ND range that can help determine // which dense fragments are consolidatable. - FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); + FragmentInfo fragment_info(URI(array_name), resources_); auto st = fragment_info.load( array_for_reads->array_directory(), config_.timestamp_start_, @@ -305,7 +306,7 @@ Status FragmentConsolidator::consolidate( RETURN_NOT_OK_ELSE( array_for_reads->close(), throw_if_not_ok(array_for_writes->close())); - RETURN_NOT_OK(array_for_writes->close()); + throw_if_not_ok(array_for_writes->close()); stats_->add_counter("consolidate_step_num", step); @@ -321,15 +322,14 @@ Status FragmentConsolidator::consolidate_fragments( auto timer_se = stats_->start_timer("consolidate_frags"); // Open array for reading - auto array_for_reads{make_shared( - HERE(), storage_manager_->resources(), URI(array_name))}; - RETURN_NOT_OK(array_for_reads->open_without_fragments( + auto array_for_reads{make_shared(HERE(), resources_, URI(array_name))}; + throw_if_not_ok(array_for_reads->open_without_fragments( encryption_type, encryption_key, key_length)); // Open array for writing - auto array_for_writes{make_shared( - HERE(), storage_manager_->resources(), array_for_reads->array_uri())}; - RETURN_NOT_OK(array_for_writes->open( + auto array_for_writes{ + make_shared(HERE(), resources_, array_for_reads->array_uri())}; + throw_if_not_ok(array_for_writes->open( QueryType::WRITE, encryption_type, encryption_key, key_length)); // Disable consolidation with timestamps on older arrays. @@ -344,7 +344,7 @@ Status FragmentConsolidator::consolidate_fragments( } // Get all fragment info - FragmentInfo fragment_info(URI(array_name), storage_manager_->resources()); + FragmentInfo fragment_info(URI(array_name), resources_); auto st = fragment_info.load( array_for_reads->array_directory(), 0, @@ -383,8 +383,8 @@ Status FragmentConsolidator::consolidate_fragments( } if (count != fragment_uris.size()) { - return logger_->status(Status_ConsolidatorError( - "Cannot consolidate; Not all fragments could be found")); + throw FragmentConsolidatorException( + "Cannot consolidate; Not all fragments could be found"); } FragmentConsolidationWorkspace cw(consolidator_memory_tracker_); @@ -429,7 +429,7 @@ void FragmentConsolidator::vacuum(const char* array_name) { // Get the fragment URIs and vacuum file URIs to be vacuumed ArrayDirectory array_dir( - storage_manager_->resources(), + resources_, URI(array_name), 0, std::numeric_limits::max(), @@ -583,8 +583,8 @@ Status FragmentConsolidator::consolidate_internal( vac_uri = array_for_reads->array_directory().get_vacuum_uri(*new_fragment_uri); } catch (std::exception& e) { - std::throw_with_nested( - std::logic_error("[FragmentConsolidator::consolidate_internal] ")); + FragmentConsolidatorException( + "Internal consolidation failed with exception" + std::string(e.what())); } // Read from one array and write to the other @@ -669,14 +669,14 @@ Status FragmentConsolidator::create_queries( // Create read query query_r = tdb_unique_ptr(tdb_new( Query, storage_manager_, array_for_reads, nullopt, read_memory_budget)); - RETURN_NOT_OK(query_r->set_layout(Layout::GLOBAL_ORDER)); + throw_if_not_ok(query_r->set_layout(Layout::GLOBAL_ORDER)); // Dense consolidation will do a tile aligned read. if (dense) { NDRange read_subarray = subarray; auto& domain{array_for_reads->array_schema_latest().domain()}; domain.expand_to_tiles(&read_subarray); - RETURN_NOT_OK(query_r->set_subarray_unsafe(read_subarray)); + throw_if_not_ok(query_r->set_subarray_unsafe(read_subarray)); } // Enable consolidation with timestamps on the reader, if applicable. @@ -699,11 +699,11 @@ Status FragmentConsolidator::create_queries( array_for_writes, fragment_name, write_memory_budget)); - RETURN_NOT_OK(query_w->set_layout(Layout::GLOBAL_ORDER)); - RETURN_NOT_OK(query_w->disable_checks_consolidation()); + throw_if_not_ok(query_w->set_layout(Layout::GLOBAL_ORDER)); + throw_if_not_ok(query_w->disable_checks_consolidation()); query_w->set_fragment_size(config_.max_fragment_size_); if (array_for_reads->array_schema_latest().dense()) { - RETURN_NOT_OK(query_w->set_subarray_unsafe(subarray)); + throw_if_not_ok(query_w->set_subarray_unsafe(subarray)); } // Set the processed conditions on new fragment. @@ -931,11 +931,11 @@ Status FragmentConsolidator::set_config(const Config& config) { merged_config.inherit(config); bool found = false; config_.amplification_ = 0.0f; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.amplification", &config_.amplification_, &found)); assert(found); config_.steps_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.steps", &config_.steps_, &found)); assert(found); config_.buffer_size_ = 0; @@ -946,54 +946,54 @@ Status FragmentConsolidator::set_config(const Config& config) { "The `sm.consolidation.buffer_size configuration setting has been " "deprecated. Set consolidation buffer sizes using the newer " "`sm.mem.consolidation.buffers_weight` setting."); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.buffer_size", &config_.buffer_size_, &found)); assert(found); } config_.total_budget_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.total_budget", &config_.total_budget_, &found)); assert(found); config_.buffers_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.buffers_weight", &config_.buffers_weight_, &found)); assert(found); config_.reader_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.reader_weight", &config_.reader_weight_, &found)); assert(found); config_.writer_weight_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.mem.consolidation.writer_weight", &config_.writer_weight_, &found)); assert(found); config_.max_fragment_size_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.max_fragment_size", &config_.max_fragment_size_, &found)); assert(found); config_.size_ratio_ = 0.0f; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_size_ratio", &config_.size_ratio_, &found)); assert(found); config_.purge_deleted_cells_ = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.purge_deleted_cells", &config_.purge_deleted_cells_, &found)); assert(found); config_.min_frags_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_min_frags", &config_.min_frags_, &found)); assert(found); config_.max_frags_ = 0; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.step_max_frags", &config_.max_frags_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); std::string reader = @@ -1005,17 +1005,17 @@ Status FragmentConsolidator::set_config(const Config& config) { // Sanity checks if (config_.min_frags_ > config_.max_frags_) - return logger_->status(Status_ConsolidatorError( + throw FragmentConsolidatorException( "Invalid configuration; Minimum fragments config parameter is larger " - "than the maximum")); + "than the maximum"); if (config_.size_ratio_ > 1.0f || config_.size_ratio_ < 0.0f) - return logger_->status(Status_ConsolidatorError( + throw FragmentConsolidatorException( "Invalid configuration; Step size ratio config parameter must be in " - "[0.0, 1.0]")); + "[0.0, 1.0]"); if (config_.amplification_ < 0) - return logger_->status( - Status_ConsolidatorError("Invalid configuration; Amplification config " - "parameter must be non-negative")); + throw FragmentConsolidatorException( + "Invalid configuration; Amplification config parameter must be " + "non-negative"); return Status::Ok(); } diff --git a/tiledb/sm/consolidator/fragment_consolidator.h b/tiledb/sm/consolidator/fragment_consolidator.h index 5b828b407141..3fcae353d7e7 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.h +++ b/tiledb/sm/consolidator/fragment_consolidator.h @@ -40,6 +40,7 @@ #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/misc/types.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" #include @@ -173,11 +174,21 @@ class FragmentConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * FragmentConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit FragmentConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~FragmentConsolidator() = default; diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.cc b/tiledb/sm/consolidator/fragment_meta_consolidator.cc index b67262b34942..2df652f9fcdd 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.cc @@ -52,8 +52,8 @@ namespace tiledb::sm { /* ****************************** */ FragmentMetaConsolidator::FragmentMetaConsolidator( - StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { } /* ****************************** */ @@ -70,8 +70,8 @@ Status FragmentMetaConsolidator::consolidate( check_array_uri(array_name); // Open array for reading - Array array(storage_manager_->resources(), URI(array_name)); - RETURN_NOT_OK( + Array array(resources_, URI(array_name)); + throw_if_not_ok( array.open(QueryType::READ, encryption_type, encryption_key, key_length)); // Include only fragments with footers / separate basic metadata @@ -129,7 +129,7 @@ Status FragmentMetaConsolidator::consolidate( return Status::Ok(); }); - RETURN_NOT_OK(status); + throw_if_not_ok(status); auto serialize_data = [&](Serializer& serializer, uint64_t offset) { // Write number of fragments @@ -166,10 +166,10 @@ Status FragmentMetaConsolidator::consolidate( serialize_data(serializer, offset); // Close array - RETURN_NOT_OK(array.close()); + throw_if_not_ok(array.close()); EncryptionKey enc_key; - RETURN_NOT_OK(enc_key.set_key(encryption_type, encryption_key, key_length)); + throw_if_not_ok(enc_key.set_key(encryption_type, encryption_key, key_length)); GenericTileIO tile_io(resources_, uri); [[maybe_unused]] uint64_t nbytes = 0; diff --git a/tiledb/sm/consolidator/fragment_meta_consolidator.h b/tiledb/sm/consolidator/fragment_meta_consolidator.h index eb5027b0c17a..01c1ef574d9e 100644 --- a/tiledb/sm/consolidator/fragment_meta_consolidator.h +++ b/tiledb/sm/consolidator/fragment_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -38,6 +38,7 @@ #include "tiledb/common/status.h" #include "tiledb/sm/array/array.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -54,9 +55,18 @@ class FragmentMetaConsolidator : public Consolidator { /** * Constructor. * - * @param storage_manager Storage manager. + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * FragmentMetaConsolidator(ContextResources&). + * + * @param resources The context resources. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ - explicit FragmentMetaConsolidator(StorageManager* storage_manager); + explicit FragmentMetaConsolidator( + ContextResources& resources, StorageManager* storage_manager); /** Destructor. */ ~FragmentMetaConsolidator() = default; diff --git a/tiledb/sm/consolidator/group_meta_consolidator.cc b/tiledb/sm/consolidator/group_meta_consolidator.cc index ccb6099bb4a5..d534e1e89fe4 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.cc +++ b/tiledb/sm/consolidator/group_meta_consolidator.cc @@ -49,8 +49,10 @@ namespace tiledb::sm { /* ****************************** */ GroupMetaConsolidator::GroupMetaConsolidator( - const Config& config, StorageManager* storage_manager) - : Consolidator(storage_manager) { + ContextResources& resources, + const Config& config, + StorageManager* storage_manager) + : Consolidator(resources, storage_manager) { auto st = set_config(config); if (!st.ok()) { throw std::logic_error(st.message()); @@ -68,12 +70,12 @@ Status GroupMetaConsolidator::consolidate( // Open group for reading auto group_uri = URI(group_name); - Group group_for_reads(storage_manager_->resources(), group_uri); + Group group_for_reads(resources_, group_uri); group_for_reads.open( QueryType::READ, config_.timestamp_start_, config_.timestamp_end_); // Open group for writing - Group group_for_writes(storage_manager_->resources(), group_uri); + Group group_for_writes(resources_, group_uri); try { group_for_writes.open(QueryType::WRITE); @@ -136,10 +138,10 @@ Status GroupMetaConsolidator::set_config(const Config& config) { Config merged_config = resources_.config(); merged_config.inherit(config); bool found = false; - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_start", &config_.timestamp_start_, &found)); assert(found); - RETURN_NOT_OK(merged_config.get( + throw_if_not_ok(merged_config.get( "sm.consolidation.timestamp_end", &config_.timestamp_end_, &found)); assert(found); diff --git a/tiledb/sm/consolidator/group_meta_consolidator.h b/tiledb/sm/consolidator/group_meta_consolidator.h index e83618e925bb..5137584ba9f3 100644 --- a/tiledb/sm/consolidator/group_meta_consolidator.h +++ b/tiledb/sm/consolidator/group_meta_consolidator.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -37,6 +37,7 @@ #include "tiledb/common/heap_memory.h" #include "tiledb/common/status.h" #include "tiledb/sm/consolidator/consolidator.h" +#include "tiledb/sm/storage_manager/context_resources.h" #include "tiledb/sm/storage_manager/storage_manager_declaration.h" using namespace tiledb::common; @@ -53,11 +54,21 @@ class GroupMetaConsolidator : public Consolidator { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of all Consolidator classes on StorageManager. + * For now we still need to keep the storage_manager argument, but once the + * dependency is gone the signature will be + * GroupMetaConsolidator(ContextResources&, const Config&). + * + * @param resources The context resources. * @param config Config. - * @param storage_manager Storage manager. + * @param storage_manager A StorageManager pointer. + * (this will go away in the near future) */ explicit GroupMetaConsolidator( - const Config& config, StorageManager* storage_manager); + ContextResources& resources, + const Config& config, + StorageManager* storage_manager); /** Destructor. */ ~GroupMetaConsolidator() = default; diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index 7701b7b6009a..df9897471647 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -587,8 +587,8 @@ Status Group::consolidate_metadata( // Consolidate // Encryption credentials are loaded by Group from config StorageManager sm(resources, resources.logger(), config); - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); + auto consolidator = Consolidator::create( + resources, ConsolidationMode::GROUP_META, config, &sm); return consolidator->consolidate( group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); } @@ -610,8 +610,8 @@ void Group::vacuum_metadata( } StorageManager sm(resources, resources.logger(), config); - auto consolidator = - Consolidator::create(ConsolidationMode::GROUP_META, config, &sm); + auto consolidator = Consolidator::create( + resources, ConsolidationMode::GROUP_META, config, &sm); consolidator->vacuum(group_name); } From 51ac61ffdb0b299335eca8093c37d0235b872453 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 17 Jun 2024 09:07:16 -0700 Subject: [PATCH 441/456] Add design doc diagram (#5086) This PR adds the active diagram associated with the external sort design documentation. It is in SVG format to allow portable editing with multiple tools, as well as to be more friendly to GitHub. It has been verified to open and be editable in InkScape, which is freely available in MacOS, Linux, and Windows. For compatibility, it is recommended that further editing of the diagram be done exclusively in InkScape. --- TYPE: NO_HISTORY DESC: Add SVG format design documentation diagram --- .../query/external_sort/doc/figs/design.svg | 6786 +++++++++++++++++ 1 file changed, 6786 insertions(+) create mode 100644 tiledb/sm/query/external_sort/doc/figs/design.svg diff --git a/tiledb/sm/query/external_sort/doc/figs/design.svg b/tiledb/sm/query/external_sort/doc/figs/design.svg new file mode 100644 index 000000000000..235a5945afb5 --- /dev/null +++ b/tiledb/sm/query/external_sort/doc/figs/design.svg @@ -0,0 +1,6786 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Case 0a: Initial query completes Sort buffers in memory + + + Case 0b: Initial query does complete but user makes no further queries + + + Case 0: Unordered, everything fits + + + + + Array + + + + + (In)complete query + + + + + + + + User- Specified Buffers + + + + + + + + User Processing + + + + + + + + User Output + + + + + + + + + + + + Data + + + + + Offsets + + + + + Data + + + + + TILEDB_ROW_MAJOR + + + + + + Case -1: Legacy Reader + + + + + Array + + + + + (In)complete query + + + + + + + + User-Specified Buffers + + + + + + + + In-Memory Sort + + + + + + + + User Processing + + + + + + + + Regularized Buffers + + + + + + + + proxy_sortpar_sortstd::sort + + + + + + zip_view + + + + + var_length_view + + + + + Data + + + + + Offsets + + + + + Data + + + + + + + + + + TILEDB_UNORDERED + + + + + User Output + + + + + + Case 1: Unordered, initial user query does not fit into memory budget + + + N elements per + + + M attributes + dimensions + + + proxy_sort will require single array of N size_t to hold permutation + + + Other sorts can be done in-place, with no additional memory required + + + Copying TileDB offset format data to Arrow for var_length_view will require additional N size_t per view + + + Regularizing with alt_var_length_view will require additional 2N size_t per view + + + Phase Ia: Incomplete query into user-specified buffersPhase 1b: Buffer regularizationPhase 1c: In-memory sortPhase Id: Abstract partitioningPhase 1e: Write blocks to dense array + + + A block view has a uniform number of elements per attribute /dimension, the total memory usage of which fits under a specified byte limit. + + + Sort entire user buffers + + + Once partitioned into blocks, each block will be internally sorted, and each block will be sorted with respect to other blocks + + + + + + Array + + + + + Incomplete query + + + + + + + + User-Specified Buffers + + + + + + + + In-Memory Sort + + + + + + + + Abstract Partitioning + + + + + + + + + + + + + + Regularized Buffers + + + + + + + + block_view + + + + + par_sortstd::sortproxy_sort + + + + + + zip_view + + + + + var_length_view + + + + + Data + + + + + Offsets + + + + + Data + + + + + + + + + + TILEDB_UNORDERED + + + + + + Dense Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temporary storage + + + + The temporary dense array is a “meta-block” in some sense, it contains a sorted set of sorted blocks (each of which, in turn, is a “meta-element”) + + + Only a minimal amount of memory per block is required to represent a block_view No copies of data are made. + + + Each block gets its own tile (and vice-versa) + + + Phase II: Merge sort blocks from temporary arrays + + + Additional memory is required to buffer input blocks, equal to the size of a block times the number of temporary arrays + + + Because of memory constraints, we may not be able to sort all of the previously-created blocks in one pass, so we may need to do multiple additional passes — and then passes over those passes + + + The size of this array will be the combined size of all of the input temp arrays sorted in this pass + + + + + + + + + + In-Memory Merge Sort + + + + + Abstract Partitioning + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Temp Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Dense Array + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + block_buffer + + + + + + User Buffers + + + + + + + + + Schema?- Dimensions are 0 to M - M = num_blocks * block_size + + + Metadata?- Store size of each block + + + tiledb::Group?- each metablock is an array + + + group├── 0│ ├── 0│ ├── 1│ ├── ...│ └── n├── 1│ ├── 0│ ├── 1│ ├── ...│ └── n├── ...└── m ├── 0 ├── 1 ├── ... └── n + + + group└── iter └── metablockgroup└── iter └── iter └── metablock + + + The temporary dense array stores everything that was in the original user buffers (but sorted and tiled into blocks) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + User buffers (zipped) + + + chunk_view (of zipped), variable number of cells + + + Data blocks (wrapping chunks), constant bytes per block and constant cells per block (pad chunks) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Array (one block per tile) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Zoom in to temporary storage array format + + + Array = “metablock”, one per phase I pass + + + Case 1: One block from each temp array will fit into memory, only one Phase II pass needed + + + Case 2: One block from each temp array will not fit into memory, we need multiple Phase II passes, hierarchical + + + (Save chunk sizes as metadata) + + From d593c7d15c285127ee4170c2276ec338f13bc104 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:40:07 -0400 Subject: [PATCH 442/456] Migrate APIs out of StorageManager: object_iter_*. (#5060) Migrate APIs out of `StorageManager`: `object_iter_*`. [sc-46738] [sc-46739] [sc-46740] [sc-46741] [sc-46742] --- TYPE: NO_HISTORY DESC: Migrate APIs out of `StorageManager`: `object_iter_*.` --- tiledb/CMakeLists.txt | 1 + tiledb/sm/c_api/tiledb.cc | 26 +- tiledb/sm/object/object_iter.cc | 225 ++++++++++++++++++ tiledb/sm/object/object_iter.h | 149 ++++++++++++ tiledb/sm/storage_manager/storage_manager.cc | 157 ------------ .../storage_manager_canonical.h | 97 -------- 6 files changed, 388 insertions(+), 267 deletions(-) create mode 100644 tiledb/sm/object/object_iter.cc create mode 100644 tiledb/sm/object/object_iter.h diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 98e4eda41f35..2f2323de54f6 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -244,6 +244,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/utils.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object_iter.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/object/object_mutex.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes_and_updates/deletes_and_updates.cc diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 4b5a2c9f0892..325fb153af36 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -73,6 +73,7 @@ #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/tdb_time.h" #include "tiledb/sm/object/object.h" +#include "tiledb/sm/object/object_iter.h" #include "tiledb/sm/query/query.h" #include "tiledb/sm/query/query_condition.h" #include "tiledb/sm/rest/rest_client.h" @@ -3123,9 +3124,8 @@ int32_t tiledb_object_walk( } // Create an object iterator - tiledb::sm::StorageManager::ObjectIter* obj_iter; - throw_if_not_ok(ctx->storage_manager()->object_iter_begin( - &obj_iter, path, static_cast(order))); + tiledb::sm::ObjectIter* obj_iter = tiledb::sm::object_iter_begin( + ctx->resources(), path, static_cast(order)); // For as long as there is another object and the callback indicates to // continue, walk over the TileDB objects in the path @@ -3136,9 +3136,9 @@ int32_t tiledb_object_walk( do { if (SAVE_ERROR_CATCH( ctx, - ctx->storage_manager()->object_iter_next( - obj_iter, &obj_name, &obj_type, &has_next))) { - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_next( + ctx->resources(), obj_iter, &obj_name, &obj_type, &has_next))) { + tiledb::sm::object_iter_free(obj_iter); return TILEDB_ERR; } if (!has_next) @@ -3147,7 +3147,7 @@ int32_t tiledb_object_walk( } while (rc == 1); // Clean up - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_free(obj_iter); if (rc == -1) return TILEDB_ERR; @@ -3170,8 +3170,8 @@ int32_t tiledb_object_ls( } // Create an object iterator - tiledb::sm::StorageManager::ObjectIter* obj_iter; - throw_if_not_ok(ctx->storage_manager()->object_iter_begin(&obj_iter, path)); + tiledb::sm::ObjectIter* obj_iter = + tiledb::sm::object_iter_begin(ctx->resources(), path); // For as long as there is another object and the callback indicates to // continue, walk over the TileDB objects in the path @@ -3182,9 +3182,9 @@ int32_t tiledb_object_ls( do { if (SAVE_ERROR_CATCH( ctx, - ctx->storage_manager()->object_iter_next( - obj_iter, &obj_name, &obj_type, &has_next))) { - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_next( + ctx->resources(), obj_iter, &obj_name, &obj_type, &has_next))) { + tiledb::sm::object_iter_free(obj_iter); return TILEDB_ERR; } if (!has_next) @@ -3193,7 +3193,7 @@ int32_t tiledb_object_ls( } while (rc == 1); // Clean up - ctx->storage_manager()->object_iter_free(obj_iter); + tiledb::sm::object_iter_free(obj_iter); if (rc == -1) return TILEDB_ERR; diff --git a/tiledb/sm/object/object_iter.cc b/tiledb/sm/object/object_iter.cc new file mode 100644 index 000000000000..e428ce2ab70a --- /dev/null +++ b/tiledb/sm/object/object_iter.cc @@ -0,0 +1,225 @@ +/** + * @file object_iter.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements object iterator functions. + */ + +#include "tiledb/sm/object/object_iter.h" +#include "tiledb/sm/enums/object_type.h" +#include "tiledb/sm/object/object.h" + +namespace tiledb::sm { + +class ObjectIterException : public StatusException { + public: + explicit ObjectIterException(const std::string& message) + : StatusException("ObjectIter", message) { + } +}; + +ObjectIter* object_iter_begin( + ContextResources& resources, const char* path, WalkOrder order) { + // Sanity check + URI path_uri(path); + if (path_uri.is_invalid()) { + throw ObjectIterException( + "Cannot create object iterator; Invalid input path"); + } + + // Get all contents of path + std::vector uris; + throw_if_not_ok(resources.vfs().ls(path_uri, &uris)); + + // Create a new object iterator + ObjectIter* obj_iter = tdb_new(ObjectIter); + obj_iter->order_ = order; + obj_iter->recursive_ = true; + + // Include the uris that are TileDB objects in the iterator state + ObjectType obj_type; + for (auto& uri : uris) { + try { + throw_if_not_ok(object_type(resources, uri, &obj_type)); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_back(uri); + if (order == WalkOrder::POSTORDER) + obj_iter->expanded_.push_back(false); + } + } catch (...) { + tdb_delete(obj_iter); + throw; + } + } + + return obj_iter; +} + +ObjectIter* object_iter_begin(ContextResources& resources, const char* path) { + // Sanity check + URI path_uri(path); + if (path_uri.is_invalid()) { + throw ObjectIterException( + "Cannot create object iterator; Invalid input path"); + } + + // Get all contents of path + std::vector uris; + throw_if_not_ok(resources.vfs().ls(path_uri, &uris)); + + // Create a new object iterator + ObjectIter* obj_iter = tdb_new(ObjectIter); + obj_iter->order_ = WalkOrder::PREORDER; + obj_iter->recursive_ = false; + + // Include the uris that are TileDB objects in the iterator state + ObjectType obj_type; + for (auto& uri : uris) { + try { + throw_if_not_ok(object_type(resources, uri, &obj_type)); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_back(uri); + } + } catch (...) { + tdb_delete(obj_iter); + throw; + } + } + + return obj_iter; +} + +void object_iter_free(ObjectIter* obj_iter) { + tdb_delete(obj_iter); +} + +Status object_iter_next( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Handle case there is no next + if (obj_iter->objs_.empty()) { + *has_next = false; + return Status::Ok(); + } + + // Retrieve next object + switch (obj_iter->order_) { + case WalkOrder::PREORDER: + RETURN_NOT_OK( + object_iter_next_preorder(resources, obj_iter, path, type, has_next)); + break; + case WalkOrder::POSTORDER: + RETURN_NOT_OK(object_iter_next_postorder( + resources, obj_iter, path, type, has_next)); + break; + } + + return Status::Ok(); +} + +Status object_iter_next_postorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Get all contents of the next URI recursively till the bottom, + // if the front of the list has not been expanded + if (obj_iter->expanded_.front() == false) { + uint64_t obj_num; + do { + obj_num = obj_iter->objs_.size(); + std::vector uris; + throw_if_not_ok(resources.vfs().ls(obj_iter->objs_.front(), &uris)); + obj_iter->expanded_.front() = true; + + // Push the new TileDB objects in the front of the iterator's list + ObjectType obj_type; + for (auto it = uris.rbegin(); it != uris.rend(); ++it) { + throw_if_not_ok(object_type(resources, *it, &obj_type)); + if (obj_type != ObjectType::INVALID) { + obj_iter->objs_.push_front(*it); + obj_iter->expanded_.push_front(false); + } + } + } while (obj_num != obj_iter->objs_.size()); + } + + // Prepare the values to be returned + URI front_uri = obj_iter->objs_.front(); + obj_iter->next_ = front_uri.to_string(); + throw_if_not_ok(object_type(resources, front_uri, type)); + *path = obj_iter->next_.c_str(); + *has_next = true; + + // Pop the front (next URI) of the iterator's object list + obj_iter->objs_.pop_front(); + obj_iter->expanded_.pop_front(); + + return Status::Ok(); +} + +Status object_iter_next_preorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next) { + // Prepare the values to be returned + URI front_uri = obj_iter->objs_.front(); + obj_iter->next_ = front_uri.to_string(); + throw_if_not_ok(object_type(resources, front_uri, type)); + *path = obj_iter->next_.c_str(); + *has_next = true; + + // Pop the front (next URI) of the iterator's object list + obj_iter->objs_.pop_front(); + + // Return if no recursion is needed + if (!obj_iter->recursive_) + return Status::Ok(); + + // Get all contents of the next URI + std::vector uris; + throw_if_not_ok(resources.vfs().ls(front_uri, &uris)); + + // Push the new TileDB objects in the front of the iterator's list + ObjectType obj_type; + for (auto it = uris.rbegin(); it != uris.rend(); ++it) { + throw_if_not_ok(object_type(resources, *it, &obj_type)); + if (obj_type != ObjectType::INVALID) + obj_iter->objs_.push_front(*it); + } + + return Status::Ok(); +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/object/object_iter.h b/tiledb/sm/object/object_iter.h new file mode 100644 index 000000000000..b95a4e1ffc72 --- /dev/null +++ b/tiledb/sm/object/object_iter.h @@ -0,0 +1,149 @@ +/** + * @file object_iter.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines object iterator functions. + */ + +#ifndef TILEDB_OBJECT_ITER_H +#define TILEDB_OBJECT_ITER_H + +#include "tiledb/common/common.h" +#include "tiledb/sm/enums/object_type.h" +#include "tiledb/sm/enums/walk_order.h" +#include "tiledb/sm/filesystem/uri.h" +#include "tiledb/sm/storage_manager/context_resources.h" + +using namespace tiledb::common; + +namespace tiledb::sm { + +/** Enables iteration over TileDB objects in a path. */ +struct ObjectIter { + public: + /** + * There is a one-to-one correspondence between `expanded_` and `objs_`. + * An `expanded_` value is `true` if the corresponding `objs_` path + * has been expanded to the paths it contains in a post ored traversal. + * This is not used in a preorder traversal. + */ + std::list expanded_; + + /** The next URI in string format. */ + std::string next_; + + /** The next objects to be visited. */ + std::list objs_; + + /** The traversal order of the iterator. */ + WalkOrder order_; + + /** `True` if the iterator will recursively visit the directory tree. */ + bool recursive_; +}; + +/** + * Creates a new object iterator for the input path. The iteration + * in this case will be recursive in the entire directory tree rooted + * at `path`. + * + * @param resources The context resources. + * @param path The path the iterator will target at. + * @param order The traversal order of the iterator. + * @return The created object iterator. + */ +ObjectIter* object_iter_begin( + ContextResources& resources, const char* path, WalkOrder order); + +/** + * Creates a new object iterator for the input path. The iteration will + * not be recursive, and only the children of `path` will be visited. + * + * @param resources The context resources. + * @param path The path the iterator will target at. + * @return The created object iterator. + */ +ObjectIter* object_iter_begin(ContextResources& resources, const char* path); + +/** Frees the object iterator. */ +void object_iter_free(ObjectIter* obj_iter); + +/** + * Retrieves the next object path and type. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +/** + * Retrieves the next object in the post-order traversal. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next_postorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +/** + * Retrieves the next object in the post-order traversal. + * + * @param resources The context resources. + * @param obj_iter The object iterator. + * @param path The object path that is retrieved. + * @param type The object type that is retrieved. + * @param has_next True if an object path was retrieved and false otherwise. + * @return Status + */ +Status object_iter_next_preorder( + ContextResources& resources, + ObjectIter* obj_iter, + const char** path, + ObjectType* type, + bool* has_next); + +} // namespace tiledb::sm + +#endif // TILEDB_OBJECT_ITER_H diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 0bf601483aa7..76f3ce85ebc4 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -199,163 +199,6 @@ void StorageManagerCanonical::increment_in_progress() { queries_in_progress_cv_.notify_all(); } -Status StorageManagerCanonical::object_iter_begin( - ObjectIter** obj_iter, const char* path, WalkOrder order) { - // Sanity check - URI path_uri(path); - if (path_uri.is_invalid()) { - throw StorageManagerException( - "Cannot create object iterator; Invalid input path"); - } - - // Get all contents of path - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); - - // Create a new object iterator - *obj_iter = tdb_new(ObjectIter); - (*obj_iter)->order_ = order; - (*obj_iter)->recursive_ = true; - - // Include the uris that are TileDB objects in the iterator state - ObjectType obj_type; - for (auto& uri : uris) { - RETURN_NOT_OK_ELSE( - object_type(resources_, uri, &obj_type), tdb_delete(*obj_iter)); - if (obj_type != ObjectType::INVALID) { - (*obj_iter)->objs_.push_back(uri); - if (order == WalkOrder::POSTORDER) - (*obj_iter)->expanded_.push_back(false); - } - } - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_begin( - ObjectIter** obj_iter, const char* path) { - // Sanity check - URI path_uri(path); - if (path_uri.is_invalid()) { - throw StorageManagerException( - "Cannot create object iterator; Invalid input path"); - } - - // Get all contents of path - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(path_uri, &uris)); - - // Create a new object iterator - *obj_iter = tdb_new(ObjectIter); - (*obj_iter)->order_ = WalkOrder::PREORDER; - (*obj_iter)->recursive_ = false; - - // Include the uris that are TileDB objects in the iterator state - ObjectType obj_type; - for (auto& uri : uris) { - throw_if_not_ok(object_type(resources_, uri, &obj_type)); - if (obj_type != ObjectType::INVALID) { - (*obj_iter)->objs_.push_back(uri); - } - } - - return Status::Ok(); -} - -void StorageManagerCanonical::object_iter_free(ObjectIter* obj_iter) { - tdb_delete(obj_iter); -} - -Status StorageManagerCanonical::object_iter_next( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Handle case there is no next - if (obj_iter->objs_.empty()) { - *has_next = false; - return Status::Ok(); - } - - // Retrieve next object - switch (obj_iter->order_) { - case WalkOrder::PREORDER: - RETURN_NOT_OK(object_iter_next_preorder(obj_iter, path, type, has_next)); - break; - case WalkOrder::POSTORDER: - RETURN_NOT_OK(object_iter_next_postorder(obj_iter, path, type, has_next)); - break; - } - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_next_postorder( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Get all contents of the next URI recursively till the bottom, - // if the front of the list has not been expanded - if (obj_iter->expanded_.front() == false) { - uint64_t obj_num; - do { - obj_num = obj_iter->objs_.size(); - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(obj_iter->objs_.front(), &uris)); - obj_iter->expanded_.front() = true; - - // Push the new TileDB objects in the front of the iterator's list - ObjectType obj_type; - for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources_, *it, &obj_type)); - if (obj_type != ObjectType::INVALID) { - obj_iter->objs_.push_front(*it); - obj_iter->expanded_.push_front(false); - } - } - } while (obj_num != obj_iter->objs_.size()); - } - - // Prepare the values to be returned - URI front_uri = obj_iter->objs_.front(); - obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources_, front_uri, type)); - *path = obj_iter->next_.c_str(); - *has_next = true; - - // Pop the front (next URI) of the iterator's object list - obj_iter->objs_.pop_front(); - obj_iter->expanded_.pop_front(); - - return Status::Ok(); -} - -Status StorageManagerCanonical::object_iter_next_preorder( - ObjectIter* obj_iter, const char** path, ObjectType* type, bool* has_next) { - // Prepare the values to be returned - URI front_uri = obj_iter->objs_.front(); - obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources_, front_uri, type)); - *path = obj_iter->next_.c_str(); - *has_next = true; - - // Pop the front (next URI) of the iterator's object list - obj_iter->objs_.pop_front(); - - // Return if no recursion is needed - if (!obj_iter->recursive_) - return Status::Ok(); - - // Get all contents of the next URI - std::vector uris; - throw_if_not_ok(resources_.vfs().ls(front_uri, &uris)); - - // Push the new TileDB objects in the front of the iterator's list - ObjectType obj_type; - for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources_, *it, &obj_type)); - if (obj_type != ObjectType::INVALID) - obj_iter->objs_.push_front(*it); - } - - return Status::Ok(); -} - Status StorageManagerCanonical::query_submit(Query* query) { // Process the query QueryInProgress in_progress(this); diff --git a/tiledb/sm/storage_manager/storage_manager_canonical.h b/tiledb/sm/storage_manager/storage_manager_canonical.h index 67ab83edb02a..16381e51bd60 100644 --- a/tiledb/sm/storage_manager/storage_manager_canonical.h +++ b/tiledb/sm/storage_manager/storage_manager_canonical.h @@ -74,30 +74,6 @@ enum class EncryptionType : uint8_t; /** The storage manager that manages pretty much everything in TileDB. */ class StorageManagerCanonical { public: - /* ********************************* */ - /* TYPE DEFINITIONS */ - /* ********************************* */ - - /** Enables iteration over TileDB objects in a path. */ - class ObjectIter { - public: - /** - * There is a one-to-one correspondence between `expanded_` and `objs_`. - * An `expanded_` value is `true` if the corresponding `objs_` path - * has been expanded to the paths it contains in a post ored traversal. - * This is not used in a preorder traversal. - */ - std::list expanded_; - /** The next URI in string format. */ - std::string next_; - /** The next objects to be visited. */ - std::list objs_; - /** The traversal order of the iterator. */ - WalkOrder order_; - /** `True` if the iterator will recursively visit the directory tree. */ - bool recursive_; - }; - /* ********************************* */ /* CONSTRUCTORS & DESTRUCTORS */ /* ********************************* */ @@ -151,79 +127,6 @@ class StorageManagerCanonical { /** Returns the current map of any set tags. */ const std::unordered_map& tags() const; - /** - * Creates a new object iterator for the input path. The iteration - * in this case will be recursive in the entire directory tree rooted - * at `path`. - * - * @param obj_iter The object iterator to be created (memory is allocated for - * it by the function). - * @param path The path the iterator will target at. - * @param order The traversal order of the iterator. - * @return Status - */ - Status object_iter_begin( - ObjectIter** obj_iter, const char* path, WalkOrder order); - - /** - * Creates a new object iterator for the input path. The iteration will - * not be recursive, and only the children of `path` will be visited. - * - * @param obj_iter The object iterator to be created (memory is allocated for - * it by the function). - * @param path The path the iterator will target at. - * @return Status - */ - Status object_iter_begin(ObjectIter** obj_iter, const char* path); - - /** Frees the object iterator. */ - void object_iter_free(ObjectIter* obj_iter); - - /** - * Retrieves the next object path and type. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - - /** - * Retrieves the next object in the post-order traversal. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next_postorder( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - - /** - * Retrieves the next object in the post-order traversal. - * - * @param obj_iter The object iterator. - * @param path The object path that is retrieved. - * @param type The object type that is retrieved. - * @param has_next True if an object path was retrieved and false otherwise. - * @return Status - */ - Status object_iter_next_preorder( - ObjectIter* obj_iter, - const char** path, - ObjectType* type, - bool* has_next); - /** Submits a query for (sync) execution. */ Status query_submit(Query* query); From 25933f92b938a27896ef13549cfb4e0d3a351914 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 17 Jun 2024 12:32:53 -0700 Subject: [PATCH 443/456] Add external_sort/README.md (#5093) Create README.md to document algorithm and array directory structure. Should be integrated with design document in Notion. [sc-49555] --- TYPE: NO_HISTORY DESC: Create README.md to document algorithm and directory structure. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- tiledb/sm/query/external_sort/README.md | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tiledb/sm/query/external_sort/README.md diff --git a/tiledb/sm/query/external_sort/README.md b/tiledb/sm/query/external_sort/README.md new file mode 100644 index 000000000000..65c9176d1fd4 --- /dev/null +++ b/tiledb/sm/query/external_sort/README.md @@ -0,0 +1,68 @@ + +### Definitions +* An *input block* is a fixed-size block of memory +* A *block* contains the sorted data of filled user buffers and is stored in a single temporary array. The tiles of the array are sized so that a single tile will fit completely into an input block. +* A *big block* is a set of blocks (arrays) that are sorted with respect to each other and is stored in a temporary group. +* A *pass* sorts a set of big blocks to create a new big block. +* An *iteration* comprises multiple passes, each of which sorts a subset of big blocks and produces a new set of big blocks. An iteration corresponds to the data in one query + + +### Algorithm: +#### Initialize: +* Iterate over incomplete queries to create a set of big blocks, each of which corresponds to one incomplete query and each of which consists of one block +#### Process +* Iterator until there is only one big block + * Partition the big blocks into subsets equal to the number of input blocks in the input block buffer + * Until all subsets are processed (new pass) + * Open a new big block + * Until input blocks are exhausted + * Merge sort input blocks into user buffers + * When an input block is depleted, read a new tile from the coresponding big block + * When user buffers are full, flush to a new block within the new big block + +### Directory Structure +The overall temp hierarchical structure from processing a complete query is thus envisioned to look like this: +``` +query-results +├── iter_000 (group, set of big blocks, corresponding to one query) +│ ├── pass_000 (group, one big block, set of blocks) +│ │ ├── block_000 (one array, sorted) +│ │ │ ├── attr_0 +│ │ │ │ ├── attr_0_data +│ │ │ │ ├── attr_0_offsets +│ │ │ │ └── attr_N_validity +│ │ │ ├── attr_1 +│ │ │ ... +│ │ │ └── attr_N +│ │ ├── block_001 +│ │ │ ... +│ │ └── block_M +│ ├── pass_001 +│ ... +│ └── pass_L +├── iter_001 +... +└── iter_K +``` + +#### Scenario 0: +There is enough memory to that entire query can be loaded into the user buffers. + +In this case, only the initialization step is required. The user buffers are sorted and comprise one big block and one block. +All of the sorted data fits in memory and can be used by the user, or it can be written to a single array that is completely sorted. + +(Since there will be no subsequent processing, the array does not have to be tiled.) In this case, there are is the initial iteration, one pass, one big block, one block, and one (final) array. + +#### Scenario 1: +There is not enough memory to load the entire query into the user buffers, there are more input blocks in the input block buffer than there are big blocks produced by the initialization step. + +In this case, there will be a sequence of incomplete queries. In the initialization step, each incomplete query creates a new big block, with one block inside of it. + +On the next iteration, all the big blocks can be processed in a single pass to create a single new big block. All the blocks in the big block will be sorted and will be sorted with respect to each other. This big block can be consolidated into a single (final) array, or it can fill user buffers directly. + + #### Scenario 3: +There is not enough memory to load the entire query into the user buffers. After initialization, there are more temporary big blocks than then are input blocks in the input block buffer. + + In this case, subsets of big blocks are processed at a time to create new big blocks, with each subset creating a new big block. If there are still more big blocks than there are input blocks in the input block buffer, the process is repeated until only a single output big buffer is created. Thi big block can be consolidated into a single (final) array, or it can be used to fill user buffers directly. + + From 40af4f699f0a3ff4c4c168d0eaf0777b9eb3f7f8 Mon Sep 17 00:00:00 2001 From: Andrew Lumsdaine Date: Mon, 17 Jun 2024 21:49:29 -0700 Subject: [PATCH 444/456] Implement bin partitioning function (#5092) To write tiles into a temporary array that will fit into fixed-sized blocks on reading, this PR implements a bin partitioning function that returns a `std::vector` of cell partitions, such that each group of cells has a total byte count less than a specified number of bytes. Also returned is a `std::vector` of the bin sizes (in number of bytes). This is the first file to go into the new sm/external_sort, so it also creates the directory and populates it with a test subdirectory and appropriate CMakeLists.txt files. A previous PR creates a doc subdirectory and includes the svg image of our design for external sort. --- TYPE: FEATURE DESC: Implement a partitioning function to partition cells to fit into fixed size bins --------- Co-authored-by: Luc Rancourt --- tiledb/sm/query/CMakeLists.txt | 1 + tiledb/sm/query/external_sort/CMakeLists.txt | 29 ++++++ tiledb/sm/query/external_sort/partition.h | 98 +++++++++++++++++++ .../query/external_sort/test/CMakeLists.txt | 33 +++++++ .../external_sort/test/unit_partition.cc | 77 +++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 tiledb/sm/query/external_sort/CMakeLists.txt create mode 100644 tiledb/sm/query/external_sort/partition.h create mode 100644 tiledb/sm/query/external_sort/test/CMakeLists.txt create mode 100644 tiledb/sm/query/external_sort/test/unit_partition.cc diff --git a/tiledb/sm/query/CMakeLists.txt b/tiledb/sm/query/CMakeLists.txt index cccb774ebad9..4084966eaa4d 100644 --- a/tiledb/sm/query/CMakeLists.txt +++ b/tiledb/sm/query/CMakeLists.txt @@ -27,6 +27,7 @@ include(common NO_POLICY_SCOPE) add_subdirectory(ast) add_subdirectory(deletes_and_updates) +add_subdirectory(external_sort) add_subdirectory(readers) # diff --git a/tiledb/sm/query/external_sort/CMakeLists.txt b/tiledb/sm/query/external_sort/CMakeLists.txt new file mode 100644 index 000000000000..ef98a1193d3a --- /dev/null +++ b/tiledb/sm/query/external_sort/CMakeLists.txt @@ -0,0 +1,29 @@ +# +# tiledb/sm/query/external_sort/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(common NO_POLICY_SCOPE) + +add_test_subdirectory() diff --git a/tiledb/sm/query/external_sort/partition.h b/tiledb/sm/query/external_sort/partition.h new file mode 100644 index 000000000000..08e7e79aae3f --- /dev/null +++ b/tiledb/sm/query/external_sort/partition.h @@ -0,0 +1,98 @@ +/** + * @file tiledb/sm/query/external_sort/partition.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + */ + +#ifndef TILEDB_PARTITION_H +#define TILEDB_PARTITION_H + +#include +#include +#include +#include + +/** + * @brief Partition a list of sizes into bins that are less than or equal to a + * given number of bytes. The sizes are the number of elements in each cell, + * which are assumed to be of type `char`. + * @param bin_size The maximum number of bytes in a bin. + * @param num_cells The total number of cells to be partitioned. + * @param fixed_bytes_per_cell The number of fixed bytes per cell. This + * includes all of the non varlength elements in each cell, including the + * elements that specify the sizes. + * @param sizes The number of varlength elements in each cell. These are + * assumed to correspond to chars, so the number of bytes in each cell is + * the same as the number of elements in the cell. + * @return + */ +auto bin_partition( + size_t bin_size, + size_t num_cells, + size_t fixed_bytes_per_cell, + std::list::iterator>& sizes) { + assert(bin_size > 0); + assert(num_cells > 0); + assert(fixed_bytes_per_cell > 0); + + size_t current_index{0}; + // size_t next_index{0}; + size_t current_size{0}; + size_t next_size{0}; + + auto offsets_begin = sizes.begin(); + auto offsets_end = sizes.end(); + + std::vector bins{0}; + std::vector bin_sizes; + + while (true) { + next_size = current_size + fixed_bytes_per_cell; + for (auto o = offsets_begin; o != offsets_end; ++o) { + next_size += (*o)[current_index] * sizeof(char); + } + if (next_size > bin_size) { + bins.push_back(current_index); + bin_sizes.push_back(current_size); + + next_size = current_size = 0; + continue; + } else { + current_size = next_size; + } + if (++current_index == num_cells) { + bins.push_back(num_cells); + bin_sizes.push_back(current_size); + break; + } + } + + return std::make_tuple(std::move(bins), std::move(bin_sizes)); +} + +#endif // TILEDB_PARTITION_H diff --git a/tiledb/sm/query/external_sort/test/CMakeLists.txt b/tiledb/sm/query/external_sort/test/CMakeLists.txt new file mode 100644 index 000000000000..c47f50e8eb4b --- /dev/null +++ b/tiledb/sm/query/external_sort/test/CMakeLists.txt @@ -0,0 +1,33 @@ +# +# tiledb/sm/query/external_sort/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test partition) +this_target_sources( + unit_partition.cc +) +conclude(unit_test) \ No newline at end of file diff --git a/tiledb/sm/query/external_sort/test/unit_partition.cc b/tiledb/sm/query/external_sort/test/unit_partition.cc new file mode 100644 index 000000000000..f3ee50f21814 --- /dev/null +++ b/tiledb/sm/query/external_sort/test/unit_partition.cc @@ -0,0 +1,77 @@ +/** + * @file unit_partition.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements unit tests for the alt_var_length_view class. + */ + +#include +#include +#include "tiledb/common/util/var_length_util.h" +#include "tiledb/sm/query/external_sort/partition.h" + +TEST_CASE("partition: Null test", "[partition][null_test]") { + REQUIRE(true); +} + +TEST_CASE("partition: sized", "[partition]") { + std::vector o{8, 6, 7, 5, 3, 0, 9}; + std::vector p{3, 1, 4, 1, 5, 9, 2}; + + REQUIRE(o.size() == p.size()); + size_t num_cells = size(o); + size_t bin_size = 256; + auto fixed_bytes_per_cell = 24; + + std::vector o_bytes{64, 48, 56, 40, 24, 0, 72}; + std::vector p_bytes{24, 8, 32, 8, 40, 72, 16}; + for (size_t i = 0; i < num_cells; ++i) { + o_bytes[i] *= 8; + p_bytes[i] *= 8; + o[i] *= 8; + p[i] *= 8; + } + std::vector sum_bytes(num_cells); + for (size_t i = 0; i < num_cells; ++i) { + sum_bytes[i] = o_bytes[i] + p_bytes[i] + fixed_bytes_per_cell; + } + std::vector byte_offsets(num_cells + 1); + lengths_to_offsets(sum_bytes, byte_offsets); + // {112, 192, /**/ 304, 376, /**/ 464, 560, /**/ 672}; + // {112, 192, /**/ 112, 184, /**/ 88, 184, /**/ 112}; + + std::list::iterator> sizes{begin(o), begin(p)}; + + auto&& [x, y] = + bin_partition(bin_size, num_cells, fixed_bytes_per_cell, sizes); + std::vector expected_bins{0, 2, 4, 6, 7}; + std::vector expected_sizes{192, 184, 184, 112}; + + CHECK(x == expected_bins); + CHECK(y == expected_sizes); +} From 65ee4d2d80a7b9b2796ec11215c7a87f23d39a12 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 18 Jun 2024 11:15:34 +0300 Subject: [PATCH 445/456] Disable nightly testing of HDFS and log a warning when HDFS is enabled. (#5085) [SC-49481](https://app.shortcut.com/tiledb-inc/story/49481/disable-testing-hdfs-in-nightlies) [SC-49482](https://app.shortcut.com/tiledb-inc/story/49482/add-deprecation-warning-when-configuring-with-hdfs-enabled) This PR disables testing HDFS in nightlies. The runs have been very brittle for years when more tests were added in and with the recent addition of many tests to all VFSes, it is now impossible to get a successful run. As we are planning to fully remove support for HDFS in TileDB version 2.28, we have made the decision to not investigate the failures. The Core will still be built with HDFS enabled to catch build-time regressions. Furthermore, the build script was updated to emit a warning when configuring with `-DTILEDB_HDFS=ON`. --- TYPE: DEPRECATION DESC: The HDFS backend is no longer officially tested by TileDB. As announced before, it is scheduled to be removed in version 2.28, to be released in Q4 2024. --- .github/workflows/ci-linux_mac.yml | 7 +++++++ .github/workflows/nightly-test.yml | 5 +++-- cmake/Options/BuildOptions.cmake | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index d33495ab7ab6..ed2fbf4e3101 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -61,6 +61,11 @@ on: description: 'CMake generator' required: false type: string + build_only: + default: false + description: 'Whether to only build TileDB and not run tests' + required: false + type: boolean env: BACKWARDS_COMPATIBILITY_ARRAYS: OFF @@ -176,6 +181,7 @@ jobs: - name: 'Test libtiledb' id: test + if: ${{ !inputs.build_only }} shell: bash env: ASAN_OPTIONS: ${{ inputs.asan && 'detect_leaks=0' || '' }} @@ -266,6 +272,7 @@ jobs: /cores/ - name: 'Test status check' + if: ${{ !inputs.build_only }} run: | # tiledb_unit is configured to set a variable TILEDB_CI_SUCCESS=1 # following the test run. If this variable is not set, the build should fail. diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index 79ebf4512bdc..0e2403f4671f 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -65,7 +65,7 @@ jobs: run: | cmake --build build --target check --config ${{ matrix.config || 'Release' }} - test_hdfs: + test_hdfs_build: uses: ./.github/workflows/ci-linux_mac.yml with: ci_backend: HDFS @@ -73,6 +73,7 @@ jobs: matrix_compiler_cc: 'gcc-13' matrix_compiler_cxx: 'g++-13' timeout: 300 + build_only: true bootstrap_args: '--enable-hdfs --enable-static-tiledb --disable-werror' create_issue_on_fail: @@ -81,7 +82,7 @@ jobs: runs-on: ubuntu-latest needs: - test - - test_hdfs + - test_hdfs_build if: failure() || cancelled() steps: - name: Checkout TileDB `dev` diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 8415c3b02295..d428dde1e101 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -53,6 +53,10 @@ if (NOT TILEDB_VCPKG) message(FATAL_ERROR "Disabling TILEDB_VCPKG is not supported. To disable automatically downloading vcpkg, enable the TILEDB_DISABLE_AUTO_VCPKG option, or set ENV{TILEDB_DISABLE_AUTO_VCPKG} to any value.") endif() +if (TILEDB_HDFS) + message(DEPRECATION "The HDFS storage backend is deprecated and receiving build-only official validation. It will be removed in TileDB 2.28.") +endif() + # enable assertions by default for debug builds if (CMAKE_BUILD_TYPE STREQUAL "Debug") set(TILEDB_ASSERTIONS TRUE) From 4044e84879a6395c27c93cba05a8f31f6a57aa0c Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Tue, 18 Jun 2024 16:05:36 +0300 Subject: [PATCH 446/456] Fix check for out of bounds dimension in Dimension::dimension_ptr. (#5094) Fix domain::dimension_ptr(int) out of bounds check. Also adding a unit test to make sure the issue doesn't regress. --- TYPE: BUG DESC: Fix check for out of bounds dimension in Dimension::dimension_ptr. --- tiledb/sm/array_schema/domain.h | 2 +- tiledb/sm/array_schema/test/CMakeLists.txt | 1 + tiledb/sm/array_schema/test/unit_domain.cc | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tiledb/sm/array_schema/domain.h b/tiledb/sm/array_schema/domain.h index 40ecc0e31aa9..397cbdf0d83d 100644 --- a/tiledb/sm/array_schema/domain.h +++ b/tiledb/sm/array_schema/domain.h @@ -227,7 +227,7 @@ class Domain { * @return non-null pointer to the dimension */ inline const Dimension* dimension_ptr(dimension_size_type i) const { - if (i > dim_num_) { + if (i >= dim_num_) { throw std::invalid_argument("invalid dimension index"); } return dimension_ptrs_[i]; diff --git a/tiledb/sm/array_schema/test/CMakeLists.txt b/tiledb/sm/array_schema/test/CMakeLists.txt index 987264201750..9ed957e082b9 100644 --- a/tiledb/sm/array_schema/test/CMakeLists.txt +++ b/tiledb/sm/array_schema/test/CMakeLists.txt @@ -36,6 +36,7 @@ commence(unit_test array_schema) unit_attribute.cc unit_dimension.cc unit_dimension_label.cc + unit_domain.cc unit_domain_data.cc unit_tile_domain.cc unit_current_domain.cc diff --git a/tiledb/sm/array_schema/test/unit_domain.cc b/tiledb/sm/array_schema/test/unit_domain.cc index 27ea16fb0cf8..2d2f7ac00486 100644 --- a/tiledb/sm/array_schema/test/unit_domain.cc +++ b/tiledb/sm/array_schema/test/unit_domain.cc @@ -38,9 +38,12 @@ #include +#include "src/mem_helpers.h" + using namespace tiledb; using namespace tiledb::common; using namespace tiledb::sm; +using namespace tiledb::test; template inline T& dom_buffer_offset(void* p) { @@ -112,8 +115,14 @@ TEST_CASE("Domain: Test deserialization", "[domain][deserialize]") { dom_buffer_offset(p) = null_tile_extent2; Deserializer deserializer(&serialized_buffer, sizeof(serialized_buffer)); - auto dom{Domain::deserialize( - deserializer, 10, Layout::ROW_MAJOR, Layout::ROW_MAJOR)}; + FilterPipeline fp; + auto dom{tiledb::sm::Domain::deserialize( + deserializer, + 10, + Layout::ROW_MAJOR, + Layout::ROW_MAJOR, + fp, + tiledb::test::get_test_memory_tracker())}; CHECK(dom->dim_num() == dim_num); auto dim1{dom->dimension_ptr("d1")}; @@ -128,3 +137,8 @@ TEST_CASE("Domain: Test deserialization", "[domain][deserialize]") { CHECK(dim2->cell_val_num() == cell_val_num2); CHECK(dim2->filters().size() == num_filters2); } + +TEST_CASE("Domain: Test dimension_ptr is not oob", "[domain][oob]") { + auto d = tiledb::sm::Domain(tiledb::test::get_test_memory_tracker()); + REQUIRE_THROWS(d.dimension_ptr(0)); +} From f8e8ce9e5ba4f74439077d7306a7512d4cbb0192 Mon Sep 17 00:00:00 2001 From: Robert Bindar Date: Tue, 18 Jun 2024 20:44:33 +0300 Subject: [PATCH 447/456] Current Domain C API implementation. (#5096) This implement the CAPI for the current domain feature which defines an area of the domain that is currently in use for the array. The information lives as part of the schema of the array and can be evolved to expand the domain using the schema evolution APIs. The current domain is currently set through an N-dimensional rectangle but support for different shape types might be supported in the future. [sc-47337] --- TYPE: CAPI DESC: Current Domain C API implementation. --------- Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: Luc Rancourt --- CMakeLists.txt | 3 + test/src/unit-capi-array_schema.cc | 329 ++++++++++++++++++ tiledb/api/c_api/CMakeLists.txt | 6 + .../api/c_api/current_domain/CMakeLists.txt | 41 +++ .../current_domain/current_domain_api.cc | 155 +++++++++ .../current_domain/current_domain_api_enum.h | 34 ++ ...current_domain_api_external_experimental.h | 185 ++++++++++ .../current_domain_api_internal.h | 83 +++++ .../c_api/current_domain/test/CMakeLists.txt | 36 ++ .../test/compile_capi_current_domain_main.cc | 41 +++ .../test/unit_capi_current_domain.cc | 153 ++++++++ tiledb/api/c_api/ndrectangle/CMakeLists.txt | 41 +++ .../api/c_api/ndrectangle/ndrectangle_api.cc | 238 +++++++++++++ .../ndrectangle_api_external_experimental.h | 205 +++++++++++ .../ndrectangle/ndrectangle_api_internal.h | 94 +++++ .../api/c_api/ndrectangle/test/CMakeLists.txt | 36 ++ .../test/compile_capi_ndrectangle_main.cc | 40 +++ .../ndrectangle/test/unit_capi_ndrectangle.cc | 190 ++++++++++ tiledb/sm/array_schema/array_schema.cc | 11 +- tiledb/sm/array_schema/array_schema.h | 11 +- .../sm/array_schema/array_schema_evolution.cc | 6 +- .../sm/array_schema/array_schema_evolution.h | 8 +- tiledb/sm/array_schema/current_domain.cc | 8 +- tiledb/sm/array_schema/current_domain.h | 15 +- tiledb/sm/array_schema/ndrectangle.cc | 2 +- tiledb/sm/array_schema/ndrectangle.h | 2 +- .../array_schema/test/unit_current_domain.cc | 16 +- tiledb/sm/c_api/tiledb.cc | 79 +++++ tiledb/sm/c_api/tiledb_enum.h | 4 - tiledb/sm/c_api/tiledb_experimental.h | 89 +++++ tiledb/sm/enums/current_domain_type.h | 2 +- 31 files changed, 2120 insertions(+), 43 deletions(-) create mode 100644 tiledb/api/c_api/current_domain/CMakeLists.txt create mode 100644 tiledb/api/c_api/current_domain/current_domain_api.cc create mode 100644 tiledb/api/c_api/current_domain/current_domain_api_enum.h create mode 100644 tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h create mode 100644 tiledb/api/c_api/current_domain/current_domain_api_internal.h create mode 100644 tiledb/api/c_api/current_domain/test/CMakeLists.txt create mode 100644 tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc create mode 100644 tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc create mode 100644 tiledb/api/c_api/ndrectangle/CMakeLists.txt create mode 100644 tiledb/api/c_api/ndrectangle/ndrectangle_api.cc create mode 100644 tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h create mode 100644 tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h create mode 100644 tiledb/api/c_api/ndrectangle/test/CMakeLists.txt create mode 100644 tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc create mode 100644 tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index c178e9829f7a..78cf75781774 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -339,6 +339,8 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer_list/buffer_list_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/config/config_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/context/context_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_enum.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/data_order/data_order_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/data_order/data_order_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_enum.h" @@ -354,6 +356,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter/filter_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filter_list/filter_list_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/group/group_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/object/object_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/query/query_api_enum.h" diff --git a/test/src/unit-capi-array_schema.cc b/test/src/unit-capi-array_schema.cc index 5166195b92aa..6d9c7cfc14d6 100644 --- a/test/src/unit-capi-array_schema.cc +++ b/test/src/unit-capi-array_schema.cc @@ -2444,3 +2444,332 @@ TEST_CASE_METHOD( remove_temp_dir( array_uri + "/" + tiledb::sm::constants::array_schema_dir_name); } + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][args]") { + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + CHECK( + tiledb_array_schema_set_current_domain(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_set_current_domain(ctx_, nullptr, nullptr) == + TILEDB_ERR); + tiledb_array_schema_t* array_schema = nullptr; + REQUIRE( + tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &array_schema) == + TILEDB_OK); + CHECK( + tiledb_array_schema_set_current_domain(ctx_, array_schema, nullptr) == + TILEDB_ERR); + + CHECK( + tiledb_array_schema_get_current_domain(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_get_current_domain(ctx_, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_array_schema_get_current_domain(ctx_, array_schema, nullptr) == + TILEDB_ERR); + + CHECK( + tiledb_array_schema_evolution_expand_current_domain( + nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_array_schema_evolution_expand_current_domain( + ctx_, nullptr, nullptr) == TILEDB_ERR); + tiledb_array_schema_evolution_t* evo; + REQUIRE(tiledb_array_schema_evolution_alloc(ctx_, &evo) == TILEDB_OK); + CHECK( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, nullptr) == + TILEDB_ERR); + + tiledb_array_schema_evolution_free(&evo); + tiledb_array_schema_free(&array_schema); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); +} + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][get_set]") { + tiledb_array_schema_t* schema; + REQUIRE(tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &schema) == TILEDB_OK); + + tiledb_current_domain_t* crd = nullptr; + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + uint32_t is_empty = 0; + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 1); + + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + + tiledb_dimension_t* d1; + REQUIRE( + tiledb_dimension_alloc( + ctx_, "d1", TILEDB_INT64, &DIM_DOMAIN[0], &TILE_EXTENTS[0], &d1) == + TILEDB_OK); + + tiledb_domain_t* domain; + REQUIRE(tiledb_domain_alloc(ctx_, &domain) == TILEDB_OK); + REQUIRE(tiledb_domain_add_dimension(ctx_, domain, d1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_set_domain(ctx_, schema, domain) == TILEDB_OK); + + tiledb_attribute_t* attr1; + REQUIRE( + tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &attr1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_add_attribute(ctx_, schema, attr1) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_capacity(ctx_, schema, CAPACITY) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_cell_order(ctx_, schema, CELL_ORDER) == + TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_tile_order(ctx_, schema, TILE_ORDER) == + TILEDB_OK); + + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); + + SupportedFsLocal local_fs; + std::string array_name = + local_fs.file_prefix() + local_fs.temp_dir() + ARRAY_NAME; + create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + + // No range was set on the ndrectangle, can't create array + CHECK(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_ERR); + + tiledb_error_t* err; + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + const char* errmsg; + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals("TileDB internal: This current domain has no " + "range specified for dimension idx: 0")); + + tiledb_range_t range; + int64_t min = 2; + int64_t max = 100; + range.min = &min; + range.min_size = sizeof(int64_t); + range.max = &max; + range.max_size = sizeof(int64_t); + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + + // Range is out of schema domain bounds + CHECK(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_ERR); + + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "TileDB internal: This array current domain has ranges past the " + "boundaries of the array schema domain")); + + max = 5; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_attribute_free(&attr1); + tiledb_dimension_free(&d1); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&schema); + + // Open array, read back current domain from schema and check + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); + REQUIRE(tiledb_array_get_schema(ctx_, array, &schema) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + tiledb_range_t outrange; + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == + TILEDB_OK); + CHECK(*(int64_t*)outrange.min == min); + CHECK(*(int64_t*)outrange.max == max); + CHECK(outrange.min_size == range.min_size); + CHECK(outrange.max_size == range.max_size); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_free(&schema); + REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); + tiledb_array_free(&array); + + delete_array(array_name); + remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); +} + +TEST_CASE_METHOD( + ArraySchemaFx, + "C API: Test CurrentDomain schema APIs", + "[capi][array-schema][current_domain][evolution]") { + tiledb_array_schema_t* schema; + REQUIRE(tiledb_array_schema_alloc(ctx_, TILEDB_SPARSE, &schema) == TILEDB_OK); + + tiledb_dimension_t* d1; + REQUIRE( + tiledb_dimension_alloc( + ctx_, "d1", TILEDB_INT64, &DIM_DOMAIN[0], &TILE_EXTENTS[0], &d1) == + TILEDB_OK); + + tiledb_domain_t* domain; + REQUIRE(tiledb_domain_alloc(ctx_, &domain) == TILEDB_OK); + REQUIRE(tiledb_domain_add_dimension(ctx_, domain, d1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_set_domain(ctx_, schema, domain) == TILEDB_OK); + + tiledb_attribute_t* attr1; + REQUIRE( + tiledb_attribute_alloc(ctx_, "a1", TILEDB_INT32, &attr1) == TILEDB_OK); + REQUIRE(tiledb_array_schema_add_attribute(ctx_, schema, attr1) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_set_capacity(ctx_, schema, CAPACITY) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_cell_order(ctx_, schema, CELL_ORDER) == + TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_tile_order(ctx_, schema, TILE_ORDER) == + TILEDB_OK); + + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + + tiledb_range_t range; + int64_t min = 2; + int64_t max = 5; + range.min = &min; + range.min_size = sizeof(int64_t); + range.max = &max; + range.max_size = sizeof(int64_t); + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_set_current_domain(ctx_, schema, crd) == TILEDB_OK); + + SupportedFsLocal local_fs; + std::string array_name = + local_fs.file_prefix() + local_fs.temp_dir() + ARRAY_NAME; + create_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); + + REQUIRE(tiledb_array_create(ctx_, array_name.c_str(), schema) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + + // Evolve the schema + tiledb_array_schema_evolution_t* evo; + REQUIRE(tiledb_array_schema_evolution_alloc(ctx_, &evo) == TILEDB_OK); + + // Expansion with empty domain is an error + REQUIRE(tiledb_current_domain_create(ctx_, &crd) == TILEDB_OK); + CHECK( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, crd) == + TILEDB_ERR); + + tiledb_error_t* err; + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + const char* errmsg; + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "ArraySchemaEvolution: Unable to expand the array current domain, " + "the new current domain specified is empty")); + + REQUIRE(tiledb_ndrectangle_alloc(ctx_, domain, &ndr) == TILEDB_OK); + max = 3; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE( + tiledb_array_schema_evolution_expand_current_domain(ctx_, evo, crd) == + TILEDB_OK); + + // The shape is smaller here so it should fail. + CHECK(tiledb_array_evolve(ctx_, array_name.c_str(), evo) == TILEDB_ERR); + + CHECK(tiledb_ctx_get_last_error(ctx_, &err) == TILEDB_OK); + + CHECK(tiledb_error_message(err, &errmsg) == TILEDB_OK); + CHECK_THAT( + errmsg, + Catch::Matchers::Equals( + "ArraySchema: The current domain of an array can only be expanded, " + "please adjust your new current domain object.")); + + max = 7; + REQUIRE( + tiledb_ndrectangle_set_range_for_name(ctx_, ndr, "d1", &range) == + TILEDB_OK); + REQUIRE(tiledb_array_evolve(ctx_, array_name.c_str(), evo) == TILEDB_OK); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_evolution_free(&evo); + tiledb_attribute_free(&attr1); + tiledb_dimension_free(&d1); + tiledb_domain_free(&domain); + tiledb_array_schema_free(&schema); + + // Open array, read back current domain from schema and check + tiledb_array_t* array; + REQUIRE(tiledb_array_alloc(ctx_, array_name.c_str(), &array) == TILEDB_OK); + REQUIRE(tiledb_array_open(ctx_, array, TILEDB_READ) == TILEDB_OK); + REQUIRE(tiledb_array_get_schema(ctx_, array, &schema) == TILEDB_OK); + REQUIRE( + tiledb_array_schema_get_current_domain(ctx_, schema, &crd) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &ndr) == TILEDB_OK); + tiledb_range_t outrange; + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx_, ndr, "d1", &outrange) == + TILEDB_OK); + CHECK(*(int64_t*)outrange.min == min); + CHECK(*(int64_t*)outrange.max == max); + CHECK(outrange.min_size == range.min_size); + CHECK(outrange.max_size == range.max_size); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); + REQUIRE(tiledb_current_domain_free(&crd) == TILEDB_OK); + tiledb_array_schema_free(&schema); + REQUIRE(tiledb_array_close(ctx_, array) == TILEDB_OK); + tiledb_array_free(&array); + + delete_array(array_name); + remove_temp_dir(local_fs.file_prefix() + local_fs.temp_dir()); +} diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 11d62110bcde..e511ee25a457 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -58,6 +58,9 @@ add_subdirectory(buffer) # `buffer_list`: depends on `buffer` add_subdirectory(buffer_list) +# `current_domain`: depends on `ndrectangle` +add_subdirectory(current_domain) + # `error`: no dependencies add_subdirectory(error) @@ -100,6 +103,9 @@ add_subdirectory(attribute) # `group`: depends on `config`, `context` add_subdirectory(group) +# `ndrectangle`: depends on `domain` +add_subdirectory(ndrectangle) + # `object`: no dependencies add_subdirectory(object) diff --git a/tiledb/api/c_api/current_domain/CMakeLists.txt b/tiledb/api/c_api/current_domain/CMakeLists.txt new file mode 100644 index 000000000000..22b56da1a597 --- /dev/null +++ b/tiledb/api/c_api/current_domain/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/api/c_api/current_domain/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + current_domain_api.cc + ) +gather_sources(${SOURCES}) + +commence(object_library capi_current_domain) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(baseline current_domain capi_context_stub) +conclude(object_library) + +add_test_subdirectory() + diff --git a/tiledb/api/c_api/current_domain/current_domain_api.cc b/tiledb/api/c_api/current_domain/current_domain_api.cc new file mode 100644 index 000000000000..49b219a21c30 --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api.cc @@ -0,0 +1,155 @@ +/** + * @file tiledb/api/c_api/current_domain/current_domain_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the current domain C API of TileDB. + **/ + +#include "current_domain_api_external_experimental.h" +#include "current_domain_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/domain/domain_api_internal.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::api { + +capi_return_t tiledb_current_domain_create( + tiledb_ctx_t* ctx, tiledb_current_domain_t** current_domain) { + ensure_context_is_valid(ctx); + ensure_output_pointer_is_valid(current_domain); + + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *current_domain = tiledb_current_domain_handle_t::make_handle( + memory_tracker, sm::constants::current_domain_version); + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_free( + tiledb_current_domain_t** current_domain) { + ensure_output_pointer_is_valid(current_domain); + ensure_handle_is_valid(*current_domain); + tiledb_current_domain_handle_t::break_handle(*current_domain); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_set_ndrectangle( + tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t* ndr) { + ensure_handle_is_valid(current_domain); + ensure_handle_is_valid(ndr); + + current_domain->current_domain()->set_ndrectangle(ndr->ndrectangle()); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_ndrectangle( + tiledb_current_domain_t* current_domain, tiledb_ndrectangle_t** ndr) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(ndr); + + *ndr = tiledb_ndrectangle_handle_t::make_handle( + current_domain->current_domain()->ndrectangle()); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_is_empty( + tiledb_current_domain_t* current_domain, uint32_t* is_empty) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(is_empty); + + *is_empty = current_domain->current_domain()->empty(); + + return TILEDB_OK; +} + +capi_return_t tiledb_current_domain_get_type( + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) { + ensure_handle_is_valid(current_domain); + ensure_output_pointer_is_valid(type); + + *type = static_cast( + current_domain->current_domain()->type()); + + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_plain; +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + current_domain_create, + tiledb_ctx_t* ctx, + tiledb_current_domain_t** current_domain) { + return api_entry_with_context( + ctx, current_domain); +} + +CAPI_INTERFACE(current_domain_free, tiledb_current_domain_t** current_domain) { + return api_entry_plain( + current_domain); +} + +CAPI_INTERFACE( + current_domain_set_ndrectangle, + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t* ndr) { + return api_entry_plain( + current_domain, ndr); +} + +CAPI_INTERFACE( + current_domain_get_ndrectangle, + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t** ndr) { + return api_entry_plain( + current_domain, ndr); +} + +CAPI_INTERFACE( + current_domain_get_is_empty, + tiledb_current_domain_t* current_domain, + uint32_t* is_empty) { + return api_entry_plain( + current_domain, is_empty); +} + +CAPI_INTERFACE( + current_domain_get_type, + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) { + return api_entry_plain( + current_domain, type); +} diff --git a/tiledb/api/c_api/current_domain/current_domain_api_enum.h b/tiledb/api/c_api/current_domain/current_domain_api_enum.h new file mode 100644 index 000000000000..1d3154ed68c5 --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_enum.h @@ -0,0 +1,34 @@ +/* + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * NOTE: The values of these enums are serialized to the array schema and/or + * fragment metadata. Therefore, the values below should never change, + * otherwise backwards compatibility breaks. + */ + +// clang-format off +#ifdef TILEDB_CURRENT_DOMAIN_TYPE_ENUM + TILEDB_CURRENT_DOMAIN_TYPE_ENUM(NDRECTANGLE) = 0, +#endif diff --git a/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h new file mode 100644 index 000000000000..cc3db794f1ad --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h @@ -0,0 +1,185 @@ +/** + * @file + * tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the current_domain experimental C API for TileDB. + */ + +#ifndef TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H +#define TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H + +#include "tiledb/api/c_api/api_external_common.h" +#include "tiledb/api/c_api/context/context_api_external.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** The current domain type */ +typedef enum { +#define TILEDB_CURRENT_DOMAIN_TYPE_ENUM(id) TILEDB_##id +#include "tiledb/api/c_api/current_domain/current_domain_api_enum.h" +#undef TILEDB_CURRENT_DOMAIN_TYPE_ENUM +} tiledb_current_domain_type_t; + +typedef struct tiledb_current_domain_handle_t tiledb_current_domain_t; + +/** + * Create a current domain object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param ctx The TileDB context + * @param current_domain The current domain to be allocated + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_create( + tiledb_ctx_t* ctx, + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + +/** + * Free the resources associated with a current domain object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param current_domain The current domain to be freed + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_free( + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + +/** + * Set a N-dimensional rectangle representation on a current domain. + * Error if the current domain passed is not empty. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim", &range); + * + * tiledb_current_domain_set_ndrectangle(current_domain, ndr); + * + * tiledb_ndrectangle_free(&ndr); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param current_domain The current domain to modify + * @param ndr The N-dimensional rectangle to be set + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_set_ndrectangle( + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t* ndr) TILEDB_NOEXCEPT; + +/** + * Get the N-dimensional rectangle associated with the current domain object. + * Error if the current domain is empty or a different representation is set. + * + * It is the responsability of the caller to free the resources associated + * with the ndrectangle when the handle isn't needed anymore. + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_current_domain_get_ndrectangle(current_domain, &ndr); + * @endcode + * + * @param current_domain The current domain to query + * @param ndr The N-dimensional rectangle of the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_ndrectangle( + tiledb_current_domain_t* current_domain, + tiledb_ndrectangle_t** ndr) TILEDB_NOEXCEPT; + +/** + * Query if the current domain object is empty or not. + * + * **Example:** + * + * @code{.c} + * uint32_t empty = 0; + * tiledb_current_domain_get_is_empty(current_domain, &empty); + * @endcode + * + * @param current_domain The current domain to query + * @param is_empty True if nothing is set on the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_is_empty( + tiledb_current_domain_t* current_domain, + uint32_t* is_empty) TILEDB_NOEXCEPT; + +/** + * Query the type of current domain set on the object + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_type_t type; + * tiledb_current_domain_get_type(current_domain, &type); + * @endcode + * + * @param current_domain The current domain to query + * @param type The type of representation set on the current domain + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_current_domain_get_type( + tiledb_current_domain_t* current_domain, + tiledb_current_domain_type_t* type) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_CURRENT_DOMAIN_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/current_domain/current_domain_api_internal.h b/tiledb/api/c_api/current_domain/current_domain_api_internal.h new file mode 100644 index 000000000000..18d145745471 --- /dev/null +++ b/tiledb/api/c_api/current_domain/current_domain_api_internal.h @@ -0,0 +1,83 @@ +/** + * @file tiledb/api/c_api/current_domain/current_domain_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internal C API implementation of the Current Domain + */ + +#ifndef TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H +#define TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H + +#include "tiledb/api/c_api_support/argument_validation.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/sm/array_schema/current_domain.h" + +struct tiledb_current_domain_handle_t + : public tiledb::api::CAPIHandle { + /** + * Type name + */ + static constexpr std::string_view object_type_name{"tiledb_current_domain_t"}; + + private: + std::shared_ptr current_domain_; + + public: + /** + * Default constructor doesn't make sense + */ + tiledb_current_domain_handle_t() = delete; + + /** + * Constructs a handle with an empty CurrentDomain instance. + * @param memory_tracker The tracker to use in the internal CurrentDomain + * @param version The on-disk format version of the CurrentDomain + */ + explicit tiledb_current_domain_handle_t( + shared_ptr memory_tracker, + format_version_t version) + : current_domain_{make_shared( + HERE(), memory_tracker, version)} { + } + + /** + * Constructor. + * @param current_domain A CurrentDomain instance to assign to this handle + */ + explicit tiledb_current_domain_handle_t( + std::shared_ptr current_domain) + : current_domain_(current_domain) { + } + + [[nodiscard]] inline shared_ptr current_domain() + const { + return current_domain_; + } +}; + +#endif // TILEDB_CAPI_CURRENT_DOMAIN_INTERNAL_H diff --git a/tiledb/api/c_api/current_domain/test/CMakeLists.txt b/tiledb/api/c_api/current_domain/test/CMakeLists.txt new file mode 100644 index 000000000000..a9af423b2dc4 --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# tiledb/api/c_api/current_domain/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_current_domain) + this_target_sources(unit_capi_current_domain.cc) + if (NOT MSVC) + target_compile_options(unit_capi_current_domain PRIVATE -Wno-deprecated-declarations) + endif() + this_target_object_libraries(capi_current_domain) + this_target_link_libraries(tiledb_test_support_lib) +conclude(unit_test) diff --git a/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc new file mode 100644 index 000000000000..f2f52fd19dfe --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc @@ -0,0 +1,41 @@ +/** + * @file + * tiledb/api/c_api/current_domain/test/compile_capi_current_domain_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../current_domain_api_external_experimental.h" + +int main() { + tiledb_current_domain_create(nullptr, nullptr); + tiledb_current_domain_free(nullptr); + tiledb_current_domain_set_ndrectangle(nullptr, nullptr); + tiledb_current_domain_get_ndrectangle(nullptr, nullptr); + tiledb_current_domain_get_is_empty(nullptr, nullptr); + tiledb_current_domain_get_type(nullptr, nullptr); + + return 0; +} diff --git a/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc new file mode 100644 index 000000000000..d34db5fc5299 --- /dev/null +++ b/tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc @@ -0,0 +1,153 @@ +/** + * @file tiledb/api/c_api/current_domain/test/unit_capi_current_domain.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the current_domain C API. + */ + +#include +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_internal.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" + +using namespace tiledb::test; + +struct CAPICurrentDomainFx : TemporaryDirectoryFixture { + CAPICurrentDomainFx() + : array_name_(temp_dir_ + "curent_domain_array") { + create_domain(); + } + ~CAPICurrentDomainFx() { + free_domain(); + } + void create_domain(); + void free_domain(); + + std::string array_name_; + tiledb_dimension_t *d1_, *d2_; + tiledb_domain_t* domain_; +}; + +void CAPICurrentDomainFx::create_domain() { + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + int rc = tiledb_dimension_alloc( + ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_dimension_alloc( + ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2_); + CHECK(rc == TILEDB_OK); + + // Create domain + rc = tiledb_domain_alloc(ctx, &domain_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d2_); + CHECK(rc == TILEDB_OK); +} + +void CAPICurrentDomainFx::free_domain() { + tiledb_dimension_free(&d1_); + tiledb_dimension_free(&d2_); + tiledb_domain_free(&domain_); +} + +TEST_CASE_METHOD( + CAPICurrentDomainFx, + "C API: argument validation", + "[capi][current_domain][args]") { + CHECK( + tiledb_current_domain_create(nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK(tiledb_current_domain_create(ctx, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_free(nullptr) == TILEDB_ERR); + tiledb_current_domain_t* crd = nullptr; + CHECK(tiledb_current_domain_free(&crd) == TILEDB_ERR); + + crd = nullptr; + CHECK(tiledb_current_domain_create(ctx, &crd) == TILEDB_OK); + + CHECK(tiledb_current_domain_set_ndrectangle(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_set_ndrectangle(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_ndrectangle(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_ndrectangle(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_is_empty(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_is_empty(crd, nullptr) == TILEDB_ERR); + + CHECK(tiledb_current_domain_get_type(nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_current_domain_get_type(crd, nullptr) == TILEDB_ERR); + + tiledb_current_domain_free(&crd); +} + +TEST_CASE_METHOD( + CAPICurrentDomainFx, + "C API: Setting ND rectangles works", + "[capi][current_domain][ndr]") { + tiledb_current_domain_t* crd = nullptr; + REQUIRE(tiledb_current_domain_create(ctx, &crd) == TILEDB_OK); + + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + + uint32_t is_empty = 0; + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 1); + + tiledb_current_domain_type_t type; + CHECK(tiledb_current_domain_get_type(crd, &type) == TILEDB_ERR); + + REQUIRE(tiledb_current_domain_set_ndrectangle(crd, ndr) == TILEDB_OK); + + REQUIRE(tiledb_current_domain_get_is_empty(crd, &is_empty) == TILEDB_OK); + CHECK(is_empty == 0); + + REQUIRE(tiledb_current_domain_get_type(crd, &type) == TILEDB_OK); + CHECK(type == TILEDB_NDRECTANGLE); + + tiledb_ndrectangle_t* out_ndr = nullptr; + REQUIRE(tiledb_current_domain_get_ndrectangle(crd, &out_ndr) == TILEDB_OK); + CHECK(out_ndr != nullptr); + + // Verify that they point to the same tiledb::sm::NDRectangle instance. + CHECK(ndr->ndrectangle().get() == out_ndr->ndrectangle().get()); + + tiledb_ndrectangle_free(&out_ndr); + tiledb_ndrectangle_free(&ndr); + tiledb_current_domain_free(&crd); +} diff --git a/tiledb/api/c_api/ndrectangle/CMakeLists.txt b/tiledb/api/c_api/ndrectangle/CMakeLists.txt new file mode 100644 index 000000000000..6e22000eebb5 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# tiledb/api/c_api/ndrectangle/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +include(common NO_POLICY_SCOPE) +include(object_library) + +list(APPEND SOURCES + ndrectangle_api.cc + ) +gather_sources(${SOURCES}) + +commence(object_library capi_ndrectangle) + this_target_sources(${SOURCES}) + this_target_link_libraries(export) + this_target_object_libraries(baseline ndrectangle capi_context_stub) +conclude(object_library) + +add_test_subdirectory() + diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc new file mode 100644 index 000000000000..5acd46226e54 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api.cc @@ -0,0 +1,238 @@ +/** + * @file tiledb/api/c_api/ndrectangle/ndrectangle_api.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines the NDRectangle C API of TileDB. + **/ + +#include "ndrectangle_api_external_experimental.h" +#include "ndrectangle_api_internal.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/dimension/dimension_api_internal.h" +#include "tiledb/api/c_api/domain/domain_api_internal.h" +#include "tiledb/api/c_api_support/c_api_support.h" +#include "tiledb/common/memory_tracker.h" + +namespace tiledb::api { + +/** + * Ensure the argument is a valid range pointer + * + * @param range A range struct + */ +inline void ensure_range_ptr_is_valid(tiledb_range_t* range) { + ensure_output_pointer_is_valid(range); +} + +/** + * Ensure the argument is a valid char pointer + * + * @param name A char pointer + */ +inline void ensure_dim_name_is_valid(const char* name) { + if (!name) { + throw CAPIStatusException("argument `name` may not be nullptr"); + } +} + +/** + * Lays out an internal Range into a tiledb_range_t + * + * @param r The internal sm Range + * @param range The C API range + */ +inline void internal_range_to_range(const Range& r, tiledb_range_t* range) { + if (r.var_size()) { + range->min_size = r.start_size(); + range->max_size = r.end_size(); + range->min = static_cast(r.start_str().data()); + range->max = static_cast(r.end_str().data()); + } else { + range->min_size = r.size() / 2; + range->max_size = r.size() / 2; + range->min = static_cast(r.start_fixed()); + range->max = static_cast(r.end_fixed()); + } +} + +capi_return_t tiledb_ndrectangle_alloc( + tiledb_ctx_t* ctx, tiledb_domain_t* domain, tiledb_ndrectangle_t** ndr) { + ensure_context_is_valid(ctx); + ensure_domain_is_valid(domain); + ensure_output_pointer_is_valid(ndr); + + auto memory_tracker = ctx->resources().create_memory_tracker(); + memory_tracker->set_type(tiledb::sm::MemoryTrackerType::ARRAY_CREATE); + *ndr = tiledb_ndrectangle_handle_t::make_handle( + memory_tracker, domain->copy_domain()); + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_free(tiledb_ndrectangle_t** ndr) { + ensure_output_pointer_is_valid(ndr); + ensure_handle_is_valid(*ndr); + tiledb_ndrectangle_handle_t::break_handle(*ndr); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_range_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_dim_name_is_valid(name); + ensure_range_ptr_is_valid(range); + + auto& r = ndr->ndrectangle()->get_range_for_name(name); + + internal_range_to_range(r, range); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_get_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_range_ptr_is_valid(range); + + auto& r = ndr->ndrectangle()->get_range(idx); + internal_range_to_range(r, range); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_set_range_for_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_dim_name_is_valid(name); + ensure_range_ptr_is_valid(range); + + Range r; + auto idx = ndr->ndrectangle()->domain()->get_dimension_index(name); + if (ndr->ndrectangle()->domain()->dimension_ptr(idx)->var_size()) { + r.set_range_var(range->min, range->min_size, range->max, range->max_size); + } else { + r.set_range_fixed(range->min, range->max, range->min_size); + } + + ndr->ndrectangle()->set_range_for_name(r, name); + + return TILEDB_OK; +} + +capi_return_t tiledb_ndrectangle_set_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + ensure_context_is_valid(ctx); + ensure_handle_is_valid(ndr); + ensure_range_ptr_is_valid(range); + + Range r; + if (ndr->ndrectangle()->domain()->dimension_ptr(idx)->var_size()) { + r.set_range_var(range->min, range->min_size, range->max, range->max_size); + } else { + r.set_range_fixed(range->min, range->max, range->min_size); + } + + ndr->ndrectangle()->set_range(r, idx); + + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_plain; +using tiledb::api::api_entry_with_context; + +CAPI_INTERFACE( + ndrectangle_alloc, + tiledb_ctx_t* ctx, + tiledb_domain_t* domain, + tiledb_ndrectangle_t** ndr) { + return api_entry_with_context( + ctx, domain, ndr); +} + +CAPI_INTERFACE(ndrectangle_free, tiledb_ndrectangle_t** ndr) { + return api_entry_plain(ndr); +} + +CAPI_INTERFACE( + ndrectangle_get_range_from_name, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + return api_entry_with_context< + tiledb::api::tiledb_ndrectangle_get_range_from_name>( + ctx, ndr, name, range); +} + +CAPI_INTERFACE( + ndrectangle_get_range, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + return api_entry_with_context( + ctx, ndr, idx, range); +} + +CAPI_INTERFACE( + ndrectangle_set_range_for_name, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) { + return api_entry_with_context< + tiledb::api::tiledb_ndrectangle_set_range_for_name>( + ctx, ndr, name, range); +} + +CAPI_INTERFACE( + ndrectangle_set_range, + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) { + return api_entry_with_context( + ctx, ndr, idx, range); +} diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h new file mode 100644 index 000000000000..e0f80ea22b87 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h @@ -0,0 +1,205 @@ +/** + * @file + * tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the NDRectangle experimental C API for TileDB. + */ + +#ifndef TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H +#define TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H + +#include "tiledb/api/c_api/api_external_common.h" +#include "tiledb/api/c_api/context/context_api_external.h" +#include "tiledb/api/c_api/domain/domain_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** C API struct to specify the limits of a dimension for an ND rectangle. */ +typedef struct { + const void* min; + uint64_t min_size; + const void* max; + uint64_t max_size; +} tiledb_range_t; + +typedef struct tiledb_ndrectangle_handle_t tiledb_ndrectangle_t; + +/** + * Allocate an N-dimensional rectangle given a TileDB array schema domain. + * The resulted rectangle will maintain the same number of dimensions as the + * array schema domain. + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * tiledb_ndrectangle_free(&ndr); + * @endcode + * + * @param ctx The TileDB context + * @param domain The TileDB array schema domain + * @param ndr The n-dimensional rectangle to be allocated + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_alloc( + tiledb_ctx_t* ctx, + tiledb_domain_t* domain, + tiledb_ndrectangle_t** ndr) TILEDB_NOEXCEPT; + +/** + * Free the resources associated with the N-dimensional rectangle arg + * + * **Example:** + * + * @code{.c} + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * tiledb_ndrectangle_free(&ndr); + * @endcode + * + * @param ndr The n-dimensional rectangle to be freed + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_free(tiledb_ndrectangle_t** ndr) + TILEDB_NOEXCEPT; + +/** + * Get the range set on an N-dimensional rectangle for a dimension name + * + * The pointers within the returned range struct point to resources tied to + * the lifetime of the `tiledb_ndrectangle_t` object, it is not the + * responsibility of the caller to free those resources, attempting + * to do so results in undefined behavior. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * tiledb_ndrectangle_get_range_from_name(ctx, ndr, "dim", &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param name The name of the dimension for which range should be returned + * @param range The range returned as output argument + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_range_from_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Get the range set on an N-dimensional rectangle for a dimension index. + * + * The pointers within the returned range struct point to resources tied to + * the lifetime of the `tiledb_ndrectangle_t` object, it is not the + * responsibility of the caller to free those resources, attempting + * to do so results in undefined behavior. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * tiledb_ndrectangle_get_range_from_name(ctx, ndr, 1, &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param idx The index of the dimension for which range should be returned + * @param range The range returned as output argument + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_get_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Set the range on an N-dimensional rectangle for a dimension name + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim", &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param name The name of the dimension for which range should be set + * @param range The range to be set on the ND rectangle + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_set_range_for_name( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + const char* name, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +/** + * Set the range on an N-dimensional rectangle for dimension at idx. + * + * **Example:** + * + * @code{.c} + * tiledb_range_t range; + * range.min = &min; + * range.min_size = sizeof(min); + * range.max = &max; + * range.max_size = sizeof(max); + * tiledb_ndrectangle_set_range(ctx, ndr, 1, &range); + * @endcode + * + * @param ctx The TileDB context + * @param ndr The n-dimensional rectangle to be queried + * @param idx The index of the dimension for which range should be set + * @param range The range to be set on the ND rectangle + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_ndrectangle_set_range( + tiledb_ctx_t* ctx, + tiledb_ndrectangle_t* ndr, + uint32_t idx, + tiledb_range_t* range) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_NDRECTANGLE_API_EXTERNAL_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h b/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h new file mode 100644 index 000000000000..a418003a7bec --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h @@ -0,0 +1,94 @@ +/** + * @file tiledb/api/c_api/ndrectangle/ndrectangle_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the internal C API implementation of a ND rectangle + */ + +#ifndef TILEDB_CAPI_NDRECTANGLE_INTERNAL_H +#define TILEDB_CAPI_NDRECTANGLE_INTERNAL_H + +#include "tiledb/api/c_api_support/argument_validation.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/sm/array_schema/ndrectangle.h" + +struct tiledb_ndrectangle_handle_t + : public tiledb::api::CAPIHandle { + /** + * Type name + */ + static constexpr std::string_view object_type_name{"tiledb_ndrectangle_t"}; + + private: + std::shared_ptr ndrectangle_; + + public: + /** + * Default constructor doesn't make sense + */ + tiledb_ndrectangle_handle_t() = delete; + + /** + * Constructs a handle with a NDRectangle instance. + * @param memory_tracker The memory tracker to use in the internal NDRectangle + * @param domain The ArraySchema domain used for internal validations + */ + explicit tiledb_ndrectangle_handle_t( + shared_ptr memory_tracker, + shared_ptr domain) + : ndrectangle_{make_shared( + HERE(), memory_tracker, domain)} { + } + + /** + * Ordinary constructor. + * @param ndrectangle An internal tiledb NDRectangle instance + */ + explicit tiledb_ndrectangle_handle_t( + shared_ptr ndrectangle) + : ndrectangle_(ndrectangle) { + } + + /** + * Get the internal instance of NDRectangle + * @returns The internal NDRectangle + */ + [[nodiscard]] inline shared_ptr ndrectangle() const { + return ndrectangle_; + } + + /** + * Set the internal NDRectangle instance to be managed by this handle + * @param ndr The NDRectangle to manage + */ + inline void set_ndrectangle(shared_ptr ndr) { + ndrectangle_ = ndr; + } +}; + +#endif // TILEDB_CAPI_NDRECTANGLE_INTERNAL_H diff --git a/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt b/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt new file mode 100644 index 000000000000..d656225aff96 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# tiledb/api/c_api/ndrectangle/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2024 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_ndrectangle) + this_target_sources(unit_capi_ndrectangle.cc) + if (NOT MSVC) + target_compile_options(unit_capi_ndrectangle PRIVATE -Wno-deprecated-declarations) + endif() + this_target_object_libraries(capi_ndrectangle) + this_target_link_libraries(tiledb_test_support_lib) +conclude(unit_test) diff --git a/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc b/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc new file mode 100644 index 000000000000..ad3c40135197 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc @@ -0,0 +1,40 @@ +/** + * @file tiledb/api/c_api/ndrectangle/test/compile_capi_ndrectangle_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../ndrectangle_api_external_experimental.h" + +int main() { + tiledb_ndrectangle_alloc(nullptr, nullptr, nullptr); + tiledb_ndrectangle_free(nullptr); + tiledb_ndrectangle_get_range_from_name(nullptr, nullptr, nullptr, nullptr); + tiledb_ndrectangle_get_range(nullptr, nullptr, 0, nullptr); + tiledb_ndrectangle_set_range_for_name(nullptr, nullptr, nullptr, nullptr); + tiledb_ndrectangle_set_range(nullptr, nullptr, 0, nullptr); + + return 0; +} diff --git a/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc new file mode 100644 index 000000000000..49221b0b3c99 --- /dev/null +++ b/tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc @@ -0,0 +1,190 @@ +/** + * @file tiledb/api/c_api/ndrectangle/test/unit_capi_ndrectangle.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the NDRectangle C API. + */ + +#include +#include "test/support/src/helpers.h" +#include "test/support/src/vfs_helpers.h" +#include "tiledb/api/c_api/context/context_api_internal.h" +#include "tiledb/api/c_api/dimension/dimension_api_external.h" +#include "tiledb/api/c_api/ndrectangle/ndrectangle_api_external_experimental.h" +#include "tiledb/api/c_api/vfs/vfs_api_internal.h" + +using namespace tiledb::test; + +struct CAPINDRectangleFx : TemporaryDirectoryFixture { + CAPINDRectangleFx() { + create_domain(); + } + ~CAPINDRectangleFx() { + free_domain(); + } + void create_domain(); + void free_domain(); + + tiledb_dimension_t *d1_, *d2_; + tiledb_domain_t* domain_; +}; + +void CAPINDRectangleFx::create_domain() { + // Create dimensions + uint64_t tile_extents[] = {2, 2}; + uint64_t dim_domain[] = {1, 10, 1, 10}; + + int rc = tiledb_dimension_alloc( + ctx, "d1", TILEDB_UINT64, &dim_domain[0], &tile_extents[0], &d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_dimension_alloc( + ctx, "d2", TILEDB_UINT64, &dim_domain[2], &tile_extents[1], &d2_); + CHECK(rc == TILEDB_OK); + + // Create domain + rc = tiledb_domain_alloc(ctx, &domain_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d1_); + CHECK(rc == TILEDB_OK); + rc = tiledb_domain_add_dimension(ctx, domain_, d2_); + CHECK(rc == TILEDB_OK); +} + +void CAPINDRectangleFx::free_domain() { + tiledb_dimension_free(&d1_); + tiledb_dimension_free(&d2_); + tiledb_domain_free(&domain_); +} + +TEST_CASE_METHOD( + CAPINDRectangleFx, + "C API: argument validation", + "[capi][ndrectangle][args]") { + CHECK( + tiledb_ndrectangle_alloc(nullptr, nullptr, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_alloc(ctx, nullptr, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_alloc(ctx, domain_, nullptr) == TILEDB_ERR); + + CHECK(tiledb_ndrectangle_free(nullptr) == TILEDB_ERR); + tiledb_ndrectangle_t* ndr = nullptr; + CHECK(tiledb_ndrectangle_free(&ndr) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_range_from_name( + nullptr, nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, nullptr, nullptr, nullptr) == + TILEDB_ERR); + + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "dim1", nullptr) == + TILEDB_ERR); + tiledb_range_t range; + CHECK( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "doesntexist", &range) == + TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_get_range(nullptr, nullptr, 0, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_get_range(ctx, nullptr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_range(ctx, ndr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_get_range(ctx, ndr, 2, &range) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_set_range(nullptr, nullptr, 0, nullptr) == + TILEDB_INVALID_CONTEXT); + CHECK(tiledb_ndrectangle_set_range(ctx, nullptr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 0, nullptr) == TILEDB_ERR); + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 2, &range) == TILEDB_ERR); + + CHECK( + tiledb_ndrectangle_set_range_for_name( + nullptr, nullptr, nullptr, nullptr) == TILEDB_INVALID_CONTEXT); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, nullptr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, nullptr, nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim1", nullptr) == + TILEDB_ERR); + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "doesntexist", &range) == + TILEDB_ERR); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); +} + +TEST_CASE_METHOD( + CAPINDRectangleFx, + "C API: setting and getting ranges works", + "[capi][ndrectangle][range]") { + tiledb_ndrectangle_t* ndr = nullptr; + REQUIRE(tiledb_ndrectangle_alloc(ctx, domain_, &ndr) == TILEDB_OK); + + tiledb_range_t range; + uint64_t min = 2; + uint64_t max = 5; + range.min = &min; + range.min_size = sizeof(uint64_t); + range.max = &max; + range.max_size = sizeof(uint64_t); + + CHECK( + tiledb_ndrectangle_set_range_for_name(ctx, ndr, "d1", &range) == + TILEDB_OK); + + CHECK(tiledb_ndrectangle_set_range(ctx, ndr, 1, &range) == TILEDB_OK); + + tiledb_range_t out_range_d1; + + REQUIRE( + tiledb_ndrectangle_get_range_from_name(ctx, ndr, "d1", &out_range_d1) == + TILEDB_OK); + CHECK(range.min_size == out_range_d1.min_size); + CHECK(range.min_size == out_range_d1.min_size); + CHECK(std::memcmp(range.min, out_range_d1.min, range.min_size) == 0); + CHECK(std::memcmp(range.max, out_range_d1.max, range.max_size) == 0); + + tiledb_range_t out_range_d2; + REQUIRE( + tiledb_ndrectangle_get_range(ctx, ndr, 1, &out_range_d2) == TILEDB_OK); + CHECK(range.min_size == out_range_d2.min_size); + CHECK(range.min_size == out_range_d2.min_size); + CHECK(std::memcmp(range.min, out_range_d2.min, range.min_size) == 0); + CHECK(std::memcmp(range.max, out_range_d2.max, range.max_size) == 0); + + REQUIRE(tiledb_ndrectangle_free(&ndr) == TILEDB_OK); +} diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 4b006374087c..19c7c8b20c75 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -144,7 +144,7 @@ ArraySchema::ArraySchema( FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , uri_(uri) @@ -1437,7 +1437,7 @@ shared_ptr ArraySchema::deserialize( // Load the array current domain, if this is an older array, it'll get by // default an empty current domain object - auto current_domain = make_shared( + auto current_domain = make_shared( memory_tracker, constants::current_domain_version); if (version >= constants::current_domain_min_format_version) { current_domain = @@ -1815,7 +1815,7 @@ void ArraySchema::generate_uri( } void ArraySchema::expand_current_domain( - shared_ptr new_current_domain) { + shared_ptr new_current_domain) { if (new_current_domain == nullptr) { throw ArraySchemaException( "The argument specified for current domain expansion is nullptr."); @@ -1835,12 +1835,11 @@ void ArraySchema::expand_current_domain( current_domain_ = new_current_domain; } -shared_ptr ArraySchema::get_current_domain() const { +shared_ptr ArraySchema::get_current_domain() const { return current_domain_; } -void ArraySchema::set_current_domain( - shared_ptr current_domain) { +void ArraySchema::set_current_domain(shared_ptr current_domain) { if (current_domain == nullptr) { throw ArraySchemaException( "The argument specified for setting the current domain on the " diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index a07101cf19a8..5edc323d5fd7 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -143,7 +143,7 @@ class ArraySchema { FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker); /** @@ -593,18 +593,17 @@ class ArraySchema { * * @param new_current_domain The new array current domain we want to expand to */ - void expand_current_domain( - shared_ptr new_current_domain); + void expand_current_domain(shared_ptr new_current_domain); /** * Set the array current domain on the schema * * @param current_domain The array current domain we want to set on the schema */ - void set_current_domain(shared_ptr current_domain); + void set_current_domain(shared_ptr current_domain); /** Array current domain accessor */ - shared_ptr get_current_domain() const; + shared_ptr get_current_domain() const; private: /* ********************************* */ @@ -722,7 +721,7 @@ class ArraySchema { FilterPipeline coords_filters_; /** The array current domain */ - shared_ptr current_domain_; + shared_ptr current_domain_; /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index 2cafe3180a09..09aab9ff3290 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -92,7 +92,7 @@ ArraySchemaEvolution::ArraySchemaEvolution( enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker) : memory_tracker_(memory_tracker) , attributes_to_add_map_( @@ -379,7 +379,7 @@ std::pair ArraySchemaEvolution::timestamp_range() const { } void ArraySchemaEvolution::expand_current_domain( - shared_ptr current_domain) { + shared_ptr current_domain) { if (current_domain == nullptr) { throw ArraySchemaEvolutionException( "Cannot expand the array current domain; Input current domain is null"); @@ -395,7 +395,7 @@ void ArraySchemaEvolution::expand_current_domain( current_domain_to_expand_ = current_domain; } -shared_ptr ArraySchemaEvolution::current_domain_to_expand() +shared_ptr ArraySchemaEvolution::current_domain_to_expand() const { std::lock_guard lock(mtx_); diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index bdec153e483b..8959178128e4 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -92,7 +92,7 @@ class ArraySchemaEvolution { enmrs_to_extend, std::unordered_set enmrs_to_drop, std::pair timestamp_range, - shared_ptr current_domain, + shared_ptr current_domain, shared_ptr memory_tracker); DISABLE_COPY_AND_COPY_ASSIGN(ArraySchemaEvolution); @@ -196,12 +196,12 @@ class ArraySchemaEvolution { * * @param current_domain The new current domain to expand to */ - void expand_current_domain(shared_ptr current_domain); + void expand_current_domain(shared_ptr current_domain); /** * Accessor for the current domain we want to expand to */ - shared_ptr current_domain_to_expand() const; + shared_ptr current_domain_to_expand() const; private: /* ********************************* */ @@ -241,7 +241,7 @@ class ArraySchemaEvolution { std::pair timestamp_range_; /** The array current domain to expand */ - shared_ptr current_domain_to_expand_; + shared_ptr current_domain_to_expand_; /** Mutex for thread-safety. */ mutable std::mutex mtx_; diff --git a/tiledb/sm/array_schema/current_domain.cc b/tiledb/sm/array_schema/current_domain.cc index 9aad114d5a1e..489d0977557d 100644 --- a/tiledb/sm/array_schema/current_domain.cc +++ b/tiledb/sm/array_schema/current_domain.cc @@ -54,12 +54,12 @@ CurrentDomain::CurrentDomain( CurrentDomain::CurrentDomain( shared_ptr memory_tracker, format_version_t version, - shared_ptr ndr) + shared_ptr ndr) : CurrentDomain(memory_tracker, version) { set_ndrectangle(ndr); } -shared_ptr CurrentDomain::deserialize( +shared_ptr CurrentDomain::deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain) { @@ -149,7 +149,7 @@ void CurrentDomain::dump(FILE* out) const { } } -void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { +void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { if (!empty_) { throw std::logic_error( "Setting a rectangle on a non-empty CurrentDomain object is not " @@ -160,7 +160,7 @@ void CurrentDomain::set_ndrectangle(std::shared_ptr ndr) { empty_ = false; } -shared_ptr CurrentDomain::ndrectangle() const { +shared_ptr CurrentDomain::ndrectangle() const { if (empty_ || type_ != CurrentDomainType::NDRECTANGLE) { throw std::logic_error( "It's not possible to get the ndrectangle from this current domain if " diff --git a/tiledb/sm/array_schema/current_domain.h b/tiledb/sm/array_schema/current_domain.h index b73f3d2314d9..90e1c6084caa 100644 --- a/tiledb/sm/array_schema/current_domain.h +++ b/tiledb/sm/array_schema/current_domain.h @@ -77,7 +77,7 @@ class CurrentDomain { CurrentDomain( shared_ptr memory_tracker, format_version_t version, - shared_ptr ndr); + shared_ptr ndr); /** Destructor. */ ~CurrentDomain() = default; @@ -101,7 +101,7 @@ class CurrentDomain { * @param domain The array schema domain. * @return A new CurrentDomain. */ - static shared_ptr deserialize( + static shared_ptr deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain); @@ -117,6 +117,11 @@ class CurrentDomain { * @return Returns the type of current domain stored in this instance */ CurrentDomainType type() const { + if (empty_) { + throw std::logic_error( + "It's not possible to read the type, this CurrentDomain instance is " + "empty."); + } return type_; } @@ -147,14 +152,14 @@ class CurrentDomain { * * @param ndr A NDRectangle to be set on this CurrentDomain object. */ - void set_ndrectangle(std::shared_ptr ndr); + void set_ndrectangle(std::shared_ptr ndr); /** * Throws if the current domain doesn't have a NDRectangle set * * @return Returns the ndrectangle set on a current domain. */ - shared_ptr ndrectangle() const; + shared_ptr ndrectangle() const; /** * Checks if the argument fully contains this current domain. @@ -198,7 +203,7 @@ class CurrentDomain { bool empty_; /** The ndrectangle current domain */ - shared_ptr ndrectangle_; + shared_ptr ndrectangle_; /** The format version of this CurrentDomain */ format_version_t version_; diff --git a/tiledb/sm/array_schema/ndrectangle.cc b/tiledb/sm/array_schema/ndrectangle.cc index 009f844b28b9..bc0df55855ce 100644 --- a/tiledb/sm/array_schema/ndrectangle.cc +++ b/tiledb/sm/array_schema/ndrectangle.cc @@ -77,7 +77,7 @@ NDRange NDRectangle::deserialize_ndranges( return nd; } -shared_ptr NDRectangle::deserialize( +shared_ptr NDRectangle::deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain) { diff --git a/tiledb/sm/array_schema/ndrectangle.h b/tiledb/sm/array_schema/ndrectangle.h index 84a4c1a2a817..4ea35c1a3d14 100644 --- a/tiledb/sm/array_schema/ndrectangle.h +++ b/tiledb/sm/array_schema/ndrectangle.h @@ -99,7 +99,7 @@ class NDRectangle { * @param domain The domain from array schema * @return A new ndrectangle. */ - static shared_ptr deserialize( + static shared_ptr deserialize( Deserializer& deserializer, shared_ptr memory_tracker, shared_ptr domain); diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc index 077ddc0a0c60..71a8d02b2609 100644 --- a/tiledb/sm/array_schema/test/unit_current_domain.cc +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -58,7 +58,7 @@ class CurrentDomainFx { CurrentDomainFx(); ~CurrentDomainFx(); - shared_ptr create_current_domain( + shared_ptr create_current_domain( const NDRange& ranges, shared_ptr schema, shared_ptr ndrectangle = nullptr, @@ -68,9 +68,9 @@ class CurrentDomainFx { shared_ptr schema, const NDRange& ranges); storage_size_t calculate_serialized_size( - shared_ptr current_domain); + shared_ptr current_domain); shared_ptr serialize_to_tile( - shared_ptr current_domain); + shared_ptr current_domain); shared_ptr create_schema(); @@ -81,7 +81,7 @@ class CurrentDomainFx { shared_ptr get_array_directory(); shared_ptr get_array_schema_latest(); void check_current_domains_equal( - shared_ptr s1, shared_ptr s2); + shared_ptr s1, shared_ptr s2); void rm_array(); @@ -117,7 +117,7 @@ void CurrentDomainFx::rm_array() { } template -shared_ptr CurrentDomainFx::create_current_domain( +shared_ptr CurrentDomainFx::create_current_domain( const NDRange& ranges, shared_ptr schema, shared_ptr ndrectangle, @@ -158,7 +158,7 @@ void CurrentDomainFx::check_storage_serialization( template void CurrentDomainFx::check_current_domains_equal( - shared_ptr s1, shared_ptr s2) { + shared_ptr s1, shared_ptr s2) { REQUIRE(s1->empty() == s2->empty()); REQUIRE(s1->type() == s2->type()); REQUIRE(s1->version() == s2->version()); @@ -168,7 +168,7 @@ void CurrentDomainFx::check_current_domains_equal( template storage_size_t CurrentDomainFx::calculate_serialized_size( - shared_ptr current_domain) { + shared_ptr current_domain) { storage_size_t num_bytes = 0; // uint32_t - version @@ -200,7 +200,7 @@ storage_size_t CurrentDomainFx::calculate_serialized_size( template shared_ptr CurrentDomainFx::serialize_to_tile( - shared_ptr current_domain) { + shared_ptr current_domain) { SizeComputationSerializer size_serializer; current_domain->serialize(size_serializer); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 325fb153af36..1c34346a6268 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -40,6 +40,7 @@ #include "tiledb/api/c_api/buffer/buffer_api_internal.h" #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" #include "tiledb/api/c_api/domain/domain_api_internal.h" #include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" @@ -785,6 +786,40 @@ int32_t tiledb_array_schema_has_attribute( return TILEDB_OK; } +int32_t tiledb_array_schema_set_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_handle_is_valid(current_domain); + + array_schema->array_schema_->set_current_domain( + current_domain->current_domain()); + + return TILEDB_OK; +} + +int32_t tiledb_array_schema_get_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_output_pointer_is_valid(current_domain); + + // There is always a current domain on an ArraySchema instance, + // when none was set explicitly, there is an empty current domain. + *current_domain = tiledb_current_domain_handle_t::make_handle( + array_schema->array_schema_->get_current_domain()); + + return TILEDB_OK; +} + /* ********************************* */ /* SCHEMA EVOLUTION */ /* ********************************* */ @@ -907,6 +942,22 @@ capi_return_t tiledb_array_schema_evolution_drop_enumeration( return TILEDB_OK; } +capi_return_t tiledb_array_schema_evolution_expand_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) { + if (sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_handle_is_valid(expanded_domain); + + array_schema_evolution->array_schema_evolution_->expand_current_domain( + expanded_domain->current_domain()); + + return TILEDB_OK; +} + int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, @@ -5544,6 +5595,24 @@ CAPI_INTERFACE( ctx, array_schema, name, has_attr); } +CAPI_INTERFACE( + array_schema_set_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) { + return api_entry( + ctx, array_schema, current_domain); +} + +CAPI_INTERFACE( + array_schema_get_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) { + return api_entry( + ctx, array_schema, current_domain); +} + /* ********************************* */ /* SCHEMA EVOLUTION */ /* ********************************* */ @@ -5609,6 +5678,16 @@ CAPI_INTERFACE( ctx, array_schema_evolution, enumeration_name); } +CAPI_INTERFACE( + array_schema_evolution_expand_current_domain, + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) { + return api_entry< + tiledb::api::tiledb_array_schema_evolution_expand_current_domain>( + ctx, array_schema_evolution, expanded_domain); +} + TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( tiledb_ctx_t* ctx, tiledb_array_schema_evolution_t* array_schema_evolution, diff --git a/tiledb/sm/c_api/tiledb_enum.h b/tiledb/sm/c_api/tiledb_enum.h index 016809bc5d92..9fb10866c03e 100644 --- a/tiledb/sm/c_api/tiledb_enum.h +++ b/tiledb/sm/c_api/tiledb_enum.h @@ -132,7 +132,3 @@ /** application/pdf*/ TILEDB_MIME_TYPE_ENUM(MIME_PDF) = 2, #endif - -#ifdef TILEDB_CURRENT_DOMAIN_TYPE_ENUM - TILEDB_CURRENT_DOMAIN_TYPE_ENUM(NDRECTANGLE) = 0, -#endif diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 56bd92901856..691ce1f5586b 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -42,6 +42,7 @@ * API sections */ #include "tiledb/api/c_api/attribute/attribute_api_external_experimental.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_external_experimental.h" #include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/query_aggregate/query_aggregate_api_external_experimental.h" #include "tiledb/api/c_api/query_field/query_field_api_external_experimental.h" @@ -295,6 +296,49 @@ TILEDB_EXPORT int32_t tiledb_array_schema_evolution_set_timestamp_range( uint64_t lo, uint64_t hi) TILEDB_NOEXCEPT; +/** + * Expands the current domain during array schema evolution. + * TileDB will enforce that the new current domain is expanding + * on the current one and not contracting during `tiledb_array_evolve`. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *new_domain; + * tiledb_current_domain_create(ctx, &new_domain); + * tiledb_ndrectangle_t *ndr; + * tiledb_ndrectangle_alloc(ctx, domain, &ndr); + * + * tiledb_range_t range; + * range.min = &expanded_min; + * range.min_size = sizeof(expanded_min); + * range.max = &expanded_max; + * range.max_size = sizeof(expanded_max); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim1", &range); + * tiledb_ndrectangle_set_range_for_name(ctx, ndr, "dim2", &range); + * + * tiledb_current_domain_set_ndrectangle(new_domain, ndr); + * + * tiledb_array_schema_evolution_expand_current_domain(ctx, + * array_schema_evolution, new_domain); + * + * tiledb_array_evolve(ctx, array_uri, array_schema_evolution); + * + * tiledb_ndrectangle_free(&ndr); + * tiledb_current_domain_free(&new_domain); + * + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema_evolution The schema evolution. + * @param expanded_domain The current domain we want to expand the schema to. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_array_schema_evolution_expand_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_evolution_t* array_schema_evolution, + tiledb_current_domain_t* expanded_domain) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY SCHEMA */ /* ********************************* */ @@ -354,6 +398,51 @@ TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( tiledb_array_schema_t* array_schema, tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; +/** + * Sets the current domain on the array schema + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_current_domain_create(ctx, ¤t_domain); + * tiledb_array_schema_set_current_domain(ctx, array_schema, current_domain); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param current_domain The current domain to set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_set_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t* current_domain) TILEDB_NOEXCEPT; + +/** + * Gets the current domain set on the array schema or + * creates an empty current domain if none was set. + * It is the responsability of the caller to free the resources associated + * with the current domain when the handle isn't needed anymore. + * + * **Example:** + * + * @code{.c} + * tiledb_current_domain_t *current_domain; + * tiledb_array_schema_get_current_domain(ctx, array_schema, ¤t_domain); + * tiledb_current_domain_free(¤t_domain); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param current_domain The current domain set on the schema + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_get_current_domain( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_current_domain_t** current_domain) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY */ /* ********************************* */ diff --git a/tiledb/sm/enums/current_domain_type.h b/tiledb/sm/enums/current_domain_type.h index 38c9f8492b91..0c35f6a98f44 100644 --- a/tiledb/sm/enums/current_domain_type.h +++ b/tiledb/sm/enums/current_domain_type.h @@ -46,7 +46,7 @@ namespace sm { /** A current domain type. */ enum class CurrentDomainType : uint8_t { #define TILEDB_CURRENT_DOMAIN_TYPE_ENUM(id) id -#include "tiledb/sm/c_api/tiledb_enum.h" +#include "tiledb/api/c_api/current_domain/current_domain_api_enum.h" #undef TILEDB_CURRENT_DOMAIN_TYPE_ENUM }; From e95cf44ed4bc06fe1bdf43bcc6e5fc1f3a4afd28 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 18 Jun 2024 14:17:41 -0400 Subject: [PATCH 448/456] Add CODEOWNERS file to flag configuration changes for documentation follow-up (#5099) Add CODEOWNERS file to flag configuration changes for documentation follow-up [sc-49693] --- TYPE: NO_HISTORY --------- Co-authored-by: Nick Vigilante --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..aadb977758a7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Flag all PR configuration changes for documentation follow-up. +/tiledb/sm/cpp_api/config.h @TileDB-Inc/tiledb-docs +/tiledb/api/c_api/config/config_api_external.h @TileDB-Inc/tiledb-docs From 392270851c0e39925f43ec22810498e6dd09334e Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:32:05 +0200 Subject: [PATCH 449/456] Fix task graph tests in nightlies. (#5100) Some task graph tests now run in the nightlies and have been failing for two reasons. Some test cases were not closing tags properly and some tests were ran with --durations=yes when the executable doesn't support the option because they are not catch2 executables. [sc-49724] --- TYPE: NO_HISTORY DESC: Fix task graph tests in nightlies. --- .../tiledb/common/dag/edge/test/unit_edge.cc | 10 +++++----- .../execution/test/unit_scheduler_sieve.cc | 6 +++++- .../common/dag/graph/test/unit_graph_sieve.cc | 6 +++++- .../common/dag/nodes/test/CMakeLists.txt | 2 +- .../common/dag/ports/test/CMakeLists.txt | 2 +- .../dag/utility/test/unit_concurrent_map.cc | 2 +- .../dag/utility/test/unit_concurrent_set.cc | 20 +++++++++---------- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/experimental/tiledb/common/dag/edge/test/unit_edge.cc b/experimental/tiledb/common/dag/edge/test/unit_edge.cc index fe2fdf46e4dd..e8ea4c223817 100644 --- a/experimental/tiledb/common/dag/edge/test/unit_edge.cc +++ b/experimental/tiledb/common/dag/edge/test/unit_edge.cc @@ -54,7 +54,7 @@ using namespace tiledb::common; /** * Attach an `Edge` to a `Source` and a `Sink`, two stage */ -TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -63,7 +63,7 @@ TEST_CASE("Edge: Attach a Source and Sink with a two stage Edge", "[edge") { /** * Attach an `Edge` to a `Source` and a `Sink` */ -TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -72,7 +72,7 @@ TEST_CASE("Edge: Attach a Source and Sink with an Edge", "[edge") { /** * Attach an `Edge` to a `Source` and a `Sink, using CTAD` */ -TEST_CASE("Edge: Attach a Source and Sink with an Edge, using CTAD", "[edge") { +TEST_CASE("Edge: Attach a Source and Sink with an Edge, using CTAD", "[edge]") { Source left; Sink right; Edge mid(left, right); @@ -589,7 +589,7 @@ TEST_CASE("Edge: Async pass n integers", "[edge]") { /** * Attach an `Edge` to a `ProducerNode` and a `ConsumerNode` */ -TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge") { +TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge]") { ProducerNode left([]() { return 0UL; }); ConsumerNode right([](size_t) {}); @@ -600,7 +600,7 @@ TEST_CASE("Edge: Attach a Producer and Consumer with an Edge", "[edge") { * Attach an `Edge` to a `ProducerNode` and a `ConsumerNode, using CTAD` */ TEST_CASE( - "Edge: Attach a Producer and Consumer with an Edge, using CTAD", "[edge") { + "Edge: Attach a Producer and Consumer with an Edge, using CTAD", "[edge]") { ProducerNode left([]() { return 0UL; }); ConsumerNode right([](size_t) {}); diff --git a/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc b/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc index 4955d5f6a48e..d227484a8efa 100644 --- a/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc +++ b/experimental/tiledb/common/dag/execution/test/unit_scheduler_sieve.cc @@ -548,6 +548,9 @@ int main(int argc, char* argv[]) { size_t stages = 2; size_t trips = 2; + // This is unused until we migrate to catch2. + bool durations = false; + auto cli = Opt(block_size, "block_size")["-b"]["--block_size"]("Block size") | Opt(width, "width")["-w"]["--width"]("Width") | Opt(number, "number")["-n"]["--number"]("Number") | @@ -556,7 +559,8 @@ int main(int argc, char* argv[]) { Opt(scheduler, "scheduler")["-s"]["--scheduler"]( "Scheduler (bountiful, duffs, throw_catch, or frugal") | Opt(stages, "stages")["-t"]["--stages"]("Stages (2 or 3)") | - Opt(trips, "trips")["-p"]["--trips"]("Trips"); + Opt(trips, "trips")["-p"]["--trips"]("Trips") | + Opt(durations, "durations")["-d"]["--durations"]("Durations"); auto result = cli.parse(Args(argc, argv)); if (!result) { diff --git a/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc b/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc index b0e9f6cd5b21..78fc626126f5 100644 --- a/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc +++ b/experimental/tiledb/common/dag/graph/test/unit_graph_sieve.cc @@ -533,6 +533,9 @@ int main(int argc, char* argv[]) { size_t stages = 2; size_t trips = 2; + // This is unused until we migrate to catch2. + bool durations = false; + auto cli = Opt(block_size, "block_size")["-b"]["--block_size"]("Block size") | Opt(width, "width")["-w"]["--width"]("Width") | Opt(number, "number")["-n"]["--number"]("Number") | @@ -541,7 +544,8 @@ int main(int argc, char* argv[]) { Opt(scheduler, "scheduler")["-s"]["--scheduler"]( "Scheduler (bountiful, duffs, throw_catch, or frugal") | Opt(stages, "stages")["-t"]["--stages"]("Stages (2 or 3)") | - Opt(trips, "trips")["-p"]["--trips"]("Trips"); + Opt(trips, "trips")["-p"]["--trips"]("Trips") | + Opt(durations, "durations")["-d"]["--durations"]("Durations"); auto result = cli.parse(Args(argc, argv)); if (!result) { diff --git a/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt b/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt index 9ef986294bc7..20f853da2f9f 100644 --- a/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt +++ b/experimental/tiledb/common/dag/nodes/test/CMakeLists.txt @@ -57,7 +57,7 @@ target_sources(unit_nodes_sieve PUBLIC # add_test( NAME "unit_nodes_sieve" - COMMAND $ --durations=yes + COMMAND $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_dependencies(all_unit_tests "unit_nodes_sieve") diff --git a/experimental/tiledb/common/dag/ports/test/CMakeLists.txt b/experimental/tiledb/common/dag/ports/test/CMakeLists.txt index cae840acad10..104b98511d29 100644 --- a/experimental/tiledb/common/dag/ports/test/CMakeLists.txt +++ b/experimental/tiledb/common/dag/ports/test/CMakeLists.txt @@ -52,7 +52,7 @@ target_sources(unit_ports_sieve PUBLIC # add_test( NAME "unit_ports_sieve" - COMMAND $ --durations=yes + COMMAND $ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) add_dependencies(all_unit_tests "unit_ports_sieve") diff --git a/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc b/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc index 11fcb04ae586..0a8638907c4d 100644 --- a/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc +++ b/experimental/tiledb/common/dag/utility/test/unit_concurrent_map.cc @@ -1,5 +1,5 @@ /** - * @file unit_concurrent_set.cc + * @file unit_concurrent_map.cc * * @section LICENSE * diff --git a/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc b/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc index f93aae7b6ca2..56abcd66c8df 100644 --- a/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc +++ b/experimental/tiledb/common/dag/utility/test/unit_concurrent_set.cc @@ -39,10 +39,10 @@ namespace tiledb::common {} using namespace tiledb::common; -TEST_CASE("ConcurrentSet: Construct", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Construct", "[concurrent_set]") { } -TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set]") { ConcurrentSet numbers; CHECK(numbers.empty() == true); @@ -51,19 +51,19 @@ TEST_CASE("ConcurrentSet: Test empty", "[concurrent_set") { CHECK(numbers.empty() == false); } -TEST_CASE("ConcurrentSet: Test size", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test size", "[concurrent_set]") { ConcurrentSet numbers{1, 3, 5, 7, 11}; CHECK(numbers.size() == 5); } -TEST_CASE("ConcurrentSet: Test clear", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test clear", "[concurrent_set]") { ConcurrentSet numbers{1, 2, 3}; CHECK(numbers.size() == 3); numbers.clear(); CHECK(numbers.empty()); } -TEST_CASE("ConcurrentSet: Test insert", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test insert", "[concurrent_set]") { ConcurrentSet set; auto result_1 = set.insert(3); @@ -110,11 +110,11 @@ size_t set_emplace() { return set.size(); } -TEST_CASE("ConcurrentSet: Test emplace", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test emplace", "[concurrent_set]") { CHECK(set_emplace() == nof_operations * nof_operations * nof_operations); } -TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set]") { ConcurrentSet numbers = {1, 2, 3, 4, 1, 2, 3, 4}; CHECK(numbers.size() == 4); @@ -132,7 +132,7 @@ TEST_CASE("ConcurrentSet: Test erase", "[concurrent_set") { CHECK(numbers.erase(2) == 0); } -TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set]") { { ConcurrentSet a1{3, 1, 3, 2, 7}, a2{5, 4, 5}; @@ -178,7 +178,7 @@ TEST_CASE("ConcurrentSet: Test swap", "[concurrent_set") { } } -TEST_CASE("ConcurrentSet: Test extract", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test extract", "[concurrent_set]") { ConcurrentSet cont0{1, 2, 3}; ConcurrentSet cont1{1, 2, 3}; ConcurrentSet cont2{2, 3}; @@ -217,7 +217,7 @@ bool operator<(const FatKey& fk1, const FatKey& fk2) { return fk1.x < fk2.x; } -TEST_CASE("ConcurrentSet: Test find", "[concurrent_set") { +TEST_CASE("ConcurrentSet: Test find", "[concurrent_set]") { // simple comparison demo ConcurrentSet example = {1, 2, 3, 4}; From db496483ceb93d9614699aca6dcd03232d07a6e3 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 19 Jun 2024 04:00:24 -0400 Subject: [PATCH 450/456] Un-Status APIs which were previously migrated out of StorageManager. (#5081) Un-`Status` APIs which were previously migrated out of `StorageManager`. [sc-46098] --- TYPE: NO_HISTORY DESC: Un-`Status` APIs which were previously migrated out of `StorageManager`. --- test/src/unit-cppapi-fill_values.cc | 2 +- test/src/unit-cppapi-update-queries.cc | 1 - test/src/unit-cppapi-webp-filter.cc | 2 +- test/src/unit-enumerations.cc | 6 +- test/src/unit-request-handlers.cc | 2 +- tiledb/api/c_api/group/group_api.cc | 6 +- tiledb/sm/array/array.cc | 96 +++++++++---------- tiledb/sm/array/array.h | 12 +-- tiledb/sm/array/test/unit_consistency.cc | 6 +- tiledb/sm/array/test/unit_consistency.h | 4 +- tiledb/sm/array_schema/auxiliary.cc | 4 +- tiledb/sm/array_schema/auxiliary.h | 3 +- .../array_schema/test/unit_current_domain.cc | 6 +- tiledb/sm/c_api/tiledb.cc | 40 ++++---- tiledb/sm/consolidator/consolidator.cc | 10 +- tiledb/sm/group/group.cc | 29 ++---- tiledb/sm/group/group.h | 6 +- tiledb/sm/group/group_details.cc | 7 +- tiledb/sm/group/group_details.h | 3 +- tiledb/sm/object/object.cc | 68 ++++++------- tiledb/sm/object/object.h | 18 ++-- tiledb/sm/object/object_iter.cc | 12 +-- tiledb/sm/query_plan/test/unit_query_plan.cc | 2 +- .../sm/subarray/test/unit_add_ranges_list.cc | 3 +- 24 files changed, 148 insertions(+), 200 deletions(-) diff --git a/test/src/unit-cppapi-fill_values.cc b/test/src/unit-cppapi-fill_values.cc index 47c1cfe1cb31..b4f6e8ea7c9e 100644 --- a/test/src/unit-cppapi-fill_values.cc +++ b/test/src/unit-cppapi-fill_values.cc @@ -98,7 +98,7 @@ void create_array_1d( schema.set_domain(domain); schema.add_attributes(a1, a2, a3); - CHECK_NOTHROW(Array::create(array_name, schema)); + Array::create(array_name, schema); } void write_array_1d_partial( diff --git a/test/src/unit-cppapi-update-queries.cc b/test/src/unit-cppapi-update-queries.cc index 897eb91eb11f..ddcfa7e13347 100644 --- a/test/src/unit-cppapi-update-queries.cc +++ b/test/src/unit-cppapi-update-queries.cc @@ -217,7 +217,6 @@ void UpdatesFx::check_update_conditions( // Compare to negated condition. auto cmp = qcs[i].ptr()->query_condition_->negated_condition(); CHECK(tiledb::test::ast_equal(conditions->at(i).ast(), cmp.ast())); - auto& loaded_update_values = update_values->at(i); for (uint64_t j = 0; j < uvs[i].size(); j++) { CHECK(uvs[i][j].field_name() == loaded_update_values[j].field_name()); diff --git a/test/src/unit-cppapi-webp-filter.cc b/test/src/unit-cppapi-webp-filter.cc index 259e668da309..129d26035b14 100644 --- a/test/src/unit-cppapi-webp-filter.cc +++ b/test/src/unit-cppapi-webp-filter.cc @@ -314,7 +314,7 @@ TEMPLATE_LIST_TEST_CASE( "WebP filter can only be applied to dense arrays")); } - REQUIRE_NOTHROW(Array::create(webp_array_name, valid_schema)); + Array::create(webp_array_name, valid_schema); if (vfs.is_dir(webp_array_name)) { vfs.remove_dir(webp_array_name); diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index e8aee8b2df7f..46a81bcf853f 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2064,11 +2064,11 @@ TEST_CASE_METHOD( auto ase = make_shared(HERE(), memory_tracker_); ase->extend_enumeration(new_enmr); - throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + tiledb::sm::Array::evolve_array_schema( ctx_.resources(), array->array_uri(), ase.get(), - array->get_encryption_key())); + array->get_encryption_key()); // Check that we can not rewrite the query condition. array = get_array(QueryType::READ); @@ -2881,7 +2881,7 @@ shared_ptr EnumerationFx::create_schema() { void EnumerationFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } shared_ptr EnumerationFx::get_array(QueryType type) { diff --git a/test/src/unit-request-handlers.cc b/test/src/unit-request-handlers.cc index b4702f87f30f..b7368840d7a3 100644 --- a/test/src/unit-request-handlers.cc +++ b/test/src/unit-request-handlers.cc @@ -357,7 +357,7 @@ RequestHandlerFx::~RequestHandlerFx() { void RequestHandlerFx::create_array() { auto schema = create_schema(); - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } void RequestHandlerFx::delete_array() { diff --git a/tiledb/api/c_api/group/group_api.cc b/tiledb/api/c_api/group/group_api.cc index ba4b15371c29..03ad16cc12d7 100644 --- a/tiledb/api/c_api/group/group_api.cc +++ b/tiledb/api/c_api/group/group_api.cc @@ -81,8 +81,7 @@ capi_return_t tiledb_group_create( tiledb_ctx_handle_t* ctx, const char* group_uri) { ensure_group_uri_argument_is_valid(group_uri); - throw_if_not_ok( - tiledb::sm::Group::create(ctx->resources(), tiledb::sm::URI(group_uri))); + tiledb::sm::Group::create(ctx->resources(), tiledb::sm::URI(group_uri)); return TILEDB_OK; } @@ -576,8 +575,7 @@ capi_return_t tiledb_group_consolidate_metadata( ensure_group_uri_argument_is_valid(group_uri); auto cfg = (config == nullptr) ? ctx->config() : config->config(); - throw_if_not_ok(tiledb::sm::Group::consolidate_metadata( - ctx->resources(), group_uri, cfg)); + tiledb::sm::Group::consolidate_metadata(ctx->resources(), group_uri, cfg); return TILEDB_OK; } diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 45cbe3a41bca..6e9d7722ef2e 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -159,12 +159,12 @@ OpenedArray::load_delete_and_update_conditions() { throw_if_not_ok(conditions[i].check(array_schema_latest())); return Status::Ok(); }); - RETURN_NOT_OK_TUPLE(status, nullopt, nullopt); + throw_if_not_ok(status); return {Status::Ok(), conditions, update_values}; } -Status Array::evolve_array_schema( +void Array::evolve_array_schema( ContextResources& resources, const URI& array_uri, ArraySchemaEvolution* schema_evolution, @@ -175,8 +175,10 @@ Status Array::evolve_array_schema( } if (array_uri.is_tiledb()) { - return resources.rest_client()->post_array_schema_evolution_to_rest( - array_uri, schema_evolution); + throw_if_not_ok( + resources.rest_client()->post_array_schema_evolution_to_rest( + array_uri, schema_evolution)); + return; } // Load URIs from the array directory @@ -218,15 +220,12 @@ Status Array::evolve_array_schema( // Evolve schema auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); - - Status st = - store_array_schema(resources, array_schema_evolved, encryption_key); - if (!st.ok()) { - throw ArrayException( - "Cannot evolve schema; Not able to store evolved array schema."); + try { + store_array_schema(resources, array_schema_evolved, encryption_key); + } catch (...) { + std::throw_with_nested(ArrayException( + "Cannot evolve schema; Not able to store evolved array schema.")); } - - return Status::Ok(); } const URI& Array::array_uri() const { @@ -241,7 +240,7 @@ const EncryptionKey* Array::encryption_key() const { return opened_array_->encryption_key(); } -Status Array::create( +void Array::create( ContextResources& resources, const URI& array_uri, const shared_ptr& array_schema, @@ -306,41 +305,39 @@ Status Array::create( array_uri.join_path(constants::array_dimension_labels_dir_name); throw_if_not_ok(resources.vfs().create_dir(array_dimension_labels_uri)); - // Get encryption key from config - Status st; - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - resources.config().get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - resources.config().get("sm.encryption_type", &found); - assert(found); - auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); - throw_if_not_ok(st_enc); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - throw_if_not_ok( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + // Store the array schema + try { + // Get encryption key from config + if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { + bool found = false; + std::string encryption_key_from_cfg = + resources.config().get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + resources.config().get("sm.encryption_type", &found); + assert(found); + auto&& [st_enc, etc] = encryption_type_enum(encryption_type_from_cfg); + throw_if_not_ok(st_enc); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + throw_if_not_ok( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0)); + } else { + throw_if_not_ok(encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + static_cast(encryption_key_from_cfg.size()))); + } + store_array_schema(resources, array_schema, encryption_key_cfg); } else { - throw_if_not_ok(encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - static_cast(encryption_key_from_cfg.size()))); + store_array_schema(resources, array_schema, encryption_key); } - st = store_array_schema(resources, array_schema, encryption_key_cfg); - } else { - st = store_array_schema(resources, array_schema, encryption_key); - } - - if (!st.ok()) { + } catch (...) { throw_if_not_ok(resources.vfs().remove_dir(array_uri)); - return st; + throw; } - - return Status::Ok(); } // Used in Consolidator @@ -833,7 +830,7 @@ void Array::delete_fragments_list(const std::vector& fragment_uris) { } } -Status Array::encryption_type( +void Array::encryption_type( ContextResources& resources, const URI& uri, EncryptionType* encryption_type) { @@ -859,8 +856,6 @@ Status Array::encryption_type( auto&& header = GenericTileIO::read_generic_tile_header( resources, array_dir->latest_array_schema_uri(), 0); *encryption_type = static_cast(header.encryption_type); - - return Status::Ok(); } shared_ptr Array::get_enumeration( @@ -2113,7 +2108,7 @@ const ArrayDirectory& Array::load_array_directory() { return array_directory(); } -Status Array::upgrade_version( +void Array::upgrade_version( ContextResources& resources, const URI& array_uri, const Config& override_config) { @@ -2168,8 +2163,7 @@ Status Array::upgrade_version( throw_if_not_ok(resources.vfs().create_dir(array_schema_dir_uri)); // Store array schema - throw_if_not_ok( - store_array_schema(resources, array_schema, encryption_key_cfg)); + store_array_schema(resources, array_schema, encryption_key_cfg); // Create commit directory if necessary URI array_commit_uri = @@ -2186,8 +2180,6 @@ Status Array::upgrade_version( array_uri.join_path(constants::array_fragment_meta_dir_name); throw_if_not_ok(resources.vfs().create_dir(array_fragment_metadata_uri)); } - - return Status::Ok(); } Status Array::compute_non_empty_domain() { diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index c1e6ce1a9b34..3394939b42ef 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -375,9 +375,8 @@ class Array { * @param array_uri The uri of the array whose schema is to be evolved. * @param schema_evolution The schema evolution. * @param encryption_key The encryption key to use. - * @return Status */ - static Status evolve_array_schema( + static void evolve_array_schema( ContextResources& resources, const URI& array_uri, ArraySchemaEvolution* array_schema, @@ -397,9 +396,8 @@ class Array { * @param array_uri The URI of the array to be created. * @param array_schema The array schema. * @param encryption_key The encryption key to use. - * @return Status */ - static Status create( + static void create( ContextResources& resources, const URI& array_uri, const shared_ptr& array_schema, @@ -578,9 +576,8 @@ class Array { * @param resources The context resources. * @param uri The URI of the array. * @param encryption_type Set to the encryption type. - * @return Status */ - static Status encryption_type( + static void encryption_type( ContextResources& resources, const URI& uri, EncryptionType* encryption_type); @@ -1058,9 +1055,8 @@ class Array { * @param config Configuration parameters for the upgrade * (`nullptr` means default, which will use the config associated with * this instance). - * @return Status */ - static Status upgrade_version( + static void upgrade_version( ContextResources& resources, const URI& uri, const Config& config); private: diff --git a/tiledb/sm/array/test/unit_consistency.cc b/tiledb/sm/array/test/unit_consistency.cc index 1aeb1522eb90..d494e43c3b3e 100644 --- a/tiledb/sm/array/test/unit_consistency.cc +++ b/tiledb/sm/array/test/unit_consistency.cc @@ -168,7 +168,7 @@ TEST_CASE( WhiteboxConsistencyController x; const URI uri = URI("whitebox_single_array"); - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); @@ -193,7 +193,7 @@ TEST_CASE( "[ConsistencyController][array][vector]") { WhiteboxConsistencyController x; - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); @@ -235,7 +235,7 @@ TEST_CASE( WhiteboxConsistencyController x; const URI uri = URI("whitebox_modify_exclusive"); - // Create a StorageManager + // Create a ContextResources Config config; auto logger = make_shared(HERE(), "foo"); ContextResources resources(config, logger, 1, 1, ""); diff --git a/tiledb/sm/array/test/unit_consistency.h b/tiledb/sm/array/test/unit_consistency.h index 426fcfabb533..38a0e621c18d 100644 --- a/tiledb/sm/array/test/unit_consistency.h +++ b/tiledb/sm/array/test/unit_consistency.h @@ -1,5 +1,5 @@ /** - * @file tiledb/sm/array/test/unit_consistency.cc + * @file tiledb/sm/array/test/unit_consistency.h * * @section LICENSE * @@ -124,7 +124,7 @@ class WhiteboxConsistencyController : public ConsistencyController { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - throw_if_not_ok(Array::create(resources, uri, schema, key)); + Array::create(resources, uri, schema, key); tdb_unique_ptr array(new Array{resources, uri, *this}); return array; diff --git a/tiledb/sm/array_schema/auxiliary.cc b/tiledb/sm/array_schema/auxiliary.cc index 0d05fc878b18..af595d5f6b4c 100644 --- a/tiledb/sm/array_schema/auxiliary.cc +++ b/tiledb/sm/array_schema/auxiliary.cc @@ -51,7 +51,7 @@ namespace tiledb::sm { /* API */ /* ********************************* */ -Status store_array_schema( +void store_array_schema( ContextResources& resources, const shared_ptr& array_schema, const EncryptionKey& encryption_key) { @@ -122,8 +122,6 @@ Status store_array_schema( auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); GenericTileIO::store_data(resources, abs_enmr_uri, tile, encryption_key); } - - return Status::Ok(); } } // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/auxiliary.h b/tiledb/sm/array_schema/auxiliary.h index 85919a9e8704..e2e76335eafd 100644 --- a/tiledb/sm/array_schema/auxiliary.h +++ b/tiledb/sm/array_schema/auxiliary.h @@ -60,9 +60,8 @@ class EncryptionKey; * @param resources The context resources. * @param array_schema The array schema to be stored. * @param encryption_key The encryption key to use. - * @return Status */ -Status store_array_schema( +void store_array_schema( ContextResources& resources, const shared_ptr& array_schema, const EncryptionKey& encryption_key); diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc index 71a8d02b2609..df1663cd4541 100644 --- a/tiledb/sm/array_schema/test/unit_current_domain.cc +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -261,7 +261,7 @@ shared_ptr CurrentDomainFx::create_schema_var() { template void CurrentDomainFx::create_array(shared_ptr schema) { - throw_if_not_ok(Array::create(ctx_.resources(), uri_, schema, enc_key_)); + Array::create(ctx_.resources(), uri_, schema, enc_key_); } template @@ -524,8 +524,8 @@ TEMPLATE_LIST_TEST_CASE_METHOD( // Persist evolved schema and read it back, check it shows the correct // rectangle - throw_if_not_ok(Array::evolve_array_schema( - this->ctx_.resources(), this->uri_, ase.get(), this->enc_key_)); + Array::evolve_array_schema( + this->ctx_.resources(), this->uri_, ase.get(), this->enc_key_); // Read it back auto new_schema = this->get_array_schema_latest(); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 1c34346a6268..d3be1ffa9756 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -2613,8 +2613,8 @@ int32_t tiledb_array_create( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the array - throw_if_not_ok(tiledb::sm::Array::create( - ctx->resources(), uri, array_schema->array_schema_, key)); + tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2632,11 +2632,11 @@ int32_t tiledb_array_create( } // Create the dimension label array with the same key. - throw_if_not_ok(tiledb::sm::Array::create( + tiledb::sm::Array::create( ctx->resources(), dim_label_ref.uri(uri), dim_label_ref.schema(), - key)); + key); } } return TILEDB_OK; @@ -2684,8 +2684,8 @@ int32_t tiledb_array_create_with_key( key_length)); // Create the array - throw_if_not_ok(tiledb::sm::Array::create( - ctx->resources(), uri, array_schema->array_schema_, key)); + tiledb::sm::Array::create( + ctx->resources(), uri, array_schema->array_schema_, key); // Create any dimension labels in the array. for (tiledb::sm::ArraySchema::dimension_label_size_type ilabel{0}; @@ -2703,11 +2703,11 @@ int32_t tiledb_array_create_with_key( } // Create the dimension label array with the same key. - throw_if_not_ok(tiledb::sm::Array::create( + tiledb::sm::Array::create( ctx->resources(), dim_label_ref.uri(uri), dim_label_ref.schema(), - key)); + key); } } return TILEDB_OK; @@ -2944,8 +2944,8 @@ int32_t tiledb_array_encryption_type( // Get encryption type tiledb::sm::EncryptionType enc; - throw_if_not_ok(sm::Array::encryption_type( - ctx->resources(), tiledb::sm::URI(array_uri), &enc)); + sm::Array::encryption_type( + ctx->resources(), tiledb::sm::URI(array_uri), &enc); *encryption_type = static_cast(enc); return TILEDB_OK; @@ -3071,11 +3071,11 @@ int32_t tiledb_array_evolve( throw_if_not_ok( key.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Evolve schema - throw_if_not_ok(tiledb::sm::Array::evolve_array_schema( + tiledb::sm::Array::evolve_array_schema( ctx->resources(), uri, array_schema_evolution->array_schema_evolution_, - key)); + key); // Success return TILEDB_OK; @@ -3126,10 +3126,10 @@ int32_t tiledb_array_upgrade_version( } // Upgrade version - throw_if_not_ok(tiledb::sm::Array::upgrade_version( + tiledb::sm::Array::upgrade_version( ctx->resources(), uri, - (config == nullptr) ? ctx->config() : config->config())); + (config == nullptr) ? ctx->config() : config->config()); return TILEDB_OK; } @@ -3140,22 +3140,20 @@ int32_t tiledb_array_upgrade_version( int32_t tiledb_object_type( tiledb_ctx_t* ctx, const char* path, tiledb_object_t* type) { - auto uri = tiledb::sm::URI(path); - tiledb::sm::ObjectType object_type; - throw_if_not_ok(tiledb::sm::object_type(ctx->resources(), uri, &object_type)); - - *type = static_cast(object_type); + ensure_output_pointer_is_valid(type); + *type = static_cast( + tiledb::sm::object_type(ctx->resources(), tiledb::sm::URI(path))); return TILEDB_OK; } int32_t tiledb_object_remove(tiledb_ctx_t* ctx, const char* path) { - throw_if_not_ok(object_remove(ctx->resources(), path)); + object_remove(ctx->resources(), path); return TILEDB_OK; } int32_t tiledb_object_move( tiledb_ctx_t* ctx, const char* old_path, const char* new_path) { - throw_if_not_ok(object_move(ctx->resources(), old_path, new_path)); + object_move(ctx->resources(), old_path, new_path); return TILEDB_OK; } diff --git a/tiledb/sm/consolidator/consolidator.cc b/tiledb/sm/consolidator/consolidator.cc index 4cd544147252..a003a20fc5b6 100644 --- a/tiledb/sm/consolidator/consolidator.cc +++ b/tiledb/sm/consolidator/consolidator.cc @@ -152,10 +152,7 @@ void Consolidator::array_consolidate( } // Check if array exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { + if (object_type(resources, array_uri) != ObjectType::ARRAY) { throw ConsolidatorException( "Cannot consolidate array; Array does not exist"); } @@ -216,10 +213,7 @@ void Consolidator::fragments_consolidate( } // Check if array exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, array_uri, &obj_type)); - - if (obj_type != ObjectType::ARRAY) { + if (object_type(resources, array_uri) != ObjectType::ARRAY) { throw ConsolidatorException( "Cannot consolidate array; Array does not exist"); } diff --git a/tiledb/sm/group/group.cc b/tiledb/sm/group/group.cc index df9897471647..bcc3f40f39f8 100644 --- a/tiledb/sm/group/group.cc +++ b/tiledb/sm/group/group.cc @@ -82,16 +82,14 @@ Group::Group(ContextResources& resources, const URI& group_uri) memory_tracker_->set_type(MemoryTrackerType::GROUP); } -Status Group::create(ContextResources& resources, const URI& uri) { +void Group::create(ContextResources& resources, const URI& uri) { // Create group URI if (uri.is_invalid()) throw GroupException( "Cannot create group '" + uri.to_string() + "'; Invalid group URI"); // Check if group exists - bool exists; - throw_if_not_ok(is_group(resources, uri, &exists)); - if (exists) { + if (is_group(resources, uri)) { throw GroupException( "Cannot create group; Group '" + uri.to_string() + "' already exists"); } @@ -101,7 +99,7 @@ Status Group::create(ContextResources& resources, const URI& uri) { Group group(resources, uri); throw_if_not_ok( resources.rest_client()->post_group_create_to_rest(uri, &group)); - return Status::Ok(); + return; } // Create group directory @@ -118,7 +116,6 @@ Status Group::create(ContextResources& resources, const URI& uri) { // Create group detail folder throw_if_not_ok(resources.vfs().create_dir( uri.join_path(constants::group_detail_dir_name))); - return Status::Ok(); } void Group::open( @@ -245,11 +242,11 @@ void Group::close_for_writes() { if (group_details()->is_modified()) { const URI& group_detail_folder_uri = group_detail_uri(); auto group_detail_uri = generate_detail_uri(); - throw_if_not_ok(group_details()->store( + group_details()->store( resources_, group_detail_folder_uri, group_detail_uri, - *encryption_key())); + *encryption_key()); } } @@ -568,7 +565,7 @@ void Group::set_metadata_loaded(const bool metadata_loaded) { metadata_loaded_ = metadata_loaded; } -Status Group::consolidate_metadata( +void Group::consolidate_metadata( ContextResources& resources, const char* group_name, const Config& config) { // Check group URI URI group_uri(group_name); @@ -576,10 +573,7 @@ Status Group::consolidate_metadata( throw GroupException("Cannot consolidate group metadata; Invalid URI"); } // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { + if (object_type(resources, group_uri) != ObjectType::GROUP) { throw GroupException( "Cannot consolidate group metadata; Group does not exist"); } @@ -589,8 +583,8 @@ Status Group::consolidate_metadata( StorageManager sm(resources, resources.logger(), config); auto consolidator = Consolidator::create( resources, ConsolidationMode::GROUP_META, config, &sm); - return consolidator->consolidate( - group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0); + throw_if_not_ok(consolidator->consolidate( + group_name, EncryptionType::NO_ENCRYPTION, nullptr, 0)); } void Group::vacuum_metadata( @@ -602,10 +596,7 @@ void Group::vacuum_metadata( } // Check if group exists - ObjectType obj_type; - throw_if_not_ok(object_type(resources, group_uri, &obj_type)); - - if (obj_type != ObjectType::GROUP) { + if (object_type(resources, group_uri) != ObjectType::GROUP) { throw GroupException("Cannot vacuum group metadata; Group does not exist"); } diff --git a/tiledb/sm/group/group.h b/tiledb/sm/group/group.h index 2445d876fa3d..b2ebb2f022bd 100644 --- a/tiledb/sm/group/group.h +++ b/tiledb/sm/group/group.h @@ -74,9 +74,8 @@ class Group { * * @param resources The context resources. * @param uri The URI of the group to be created. - * @return Status */ - static Status create(ContextResources& resources, const URI& uri); + static void create(ContextResources& resources, const URI& uri); /** Returns the group directory object. */ const shared_ptr group_directory() const; @@ -236,9 +235,8 @@ class Group { * @param config Configuration parameters for the consolidation * (`nullptr` means default, which will use the config associated with * this instance). - * @return Status */ - static Status consolidate_metadata( + static void consolidate_metadata( ContextResources& resources, const char* group_name, const Config& config); diff --git a/tiledb/sm/group/group_details.cc b/tiledb/sm/group/group_details.cc index 3c5e9174bd54..cff44effcc5b 100644 --- a/tiledb/sm/group/group_details.cc +++ b/tiledb/sm/group/group_details.cc @@ -92,8 +92,7 @@ void GroupDetails::mark_member_for_addition( absolute_group_member_uri = group_uri_.join_path(group_member_uri.to_string()); } - ObjectType type = ObjectType::INVALID; - throw_if_not_ok(object_type(resources, absolute_group_member_uri, &type)); + ObjectType type = object_type(resources, absolute_group_member_uri); if (type == ObjectType::INVALID) { throw GroupDetailsException( "Cannot add group member " + absolute_group_member_uri.to_string() + @@ -203,7 +202,7 @@ GroupDetails::members() const { return members_; } -Status GroupDetails::store( +void GroupDetails::store( ContextResources& resources, const URI& group_detail_folder_uri, const URI& group_detail_uri, @@ -231,8 +230,6 @@ Status GroupDetails::store( throw_if_not_ok(vfs.create_dir(group_detail_folder_uri)); } GenericTileIO::store_data(resources, group_detail_uri, tile, encryption_key); - - return Status::Ok(); } std::optional> GroupDetails::deserialize( diff --git a/tiledb/sm/group/group_details.h b/tiledb/sm/group/group_details.h index cf95bbd0838a..71c91cbdb40e 100644 --- a/tiledb/sm/group/group_details.h +++ b/tiledb/sm/group/group_details.h @@ -139,9 +139,8 @@ class GroupDetails { * @param group_detail_folder_uri group details folder * @param group_detail_uri uri for detail file to write * @param encryption_key encryption key for at-rest encryption - * @return status */ - Status store( + void store( ContextResources& resources, const URI& group_detail_folder_uri, const URI& group_detail_uri, diff --git a/tiledb/sm/object/object.cc b/tiledb/sm/object/object.cc index 2bcbbe5e9564..a353504d0a47 100644 --- a/tiledb/sm/object/object.cc +++ b/tiledb/sm/object/object.cc @@ -56,7 +56,6 @@ bool is_array(ContextResources& resources, const URI& uri) { auto&& [st, exists] = resources.rest_client()->check_array_exists_from_rest(uri); throw_if_not_ok(st); - assert(exists.has_value()); return exists.value(); } else { // Check if the schema directory exists or not @@ -77,32 +76,32 @@ bool is_array(ContextResources& resources, const URI& uri) { } } -Status is_group(ContextResources& resources, const URI& uri, bool* is_group) { +bool is_group(ContextResources& resources, const URI& uri) { // Handle remote array if (uri.is_tiledb()) { auto&& [st, exists] = resources.rest_client()->check_group_exists_from_rest(uri); throw_if_not_ok(st); - *is_group = *exists; + return exists.value(); } else { // Check for new group details directory auto& vfs = resources.vfs(); - throw_if_not_ok( - vfs.is_dir(uri.join_path(constants::group_detail_dir_name), is_group)); + bool dir_exists = false; + throw_if_not_ok(vfs.is_dir( + uri.join_path(constants::group_detail_dir_name), &dir_exists)); - if (*is_group) { - return Status::Ok(); + if (dir_exists) { + return true; } // Fall back to older group file to check for legacy (pre-format 12) groups throw_if_not_ok( - vfs.is_file(uri.join_path(constants::group_filename), is_group)); + vfs.is_file(uri.join_path(constants::group_filename), &dir_exists)); + return dir_exists; } - return Status::Ok(); } -Status object_type( - ContextResources& resources, const URI& uri, ObjectType* type) { +ObjectType object_type(ContextResources& resources, const URI& uri) { URI dir_uri = uri; if (uri.is_s3() || uri.is_azure() || uri.is_gcs()) { // Always add a trailing '/' in the S3/Azure/GCS case so that listing the @@ -116,62 +115,57 @@ Status object_type( bool is_dir = false; throw_if_not_ok(resources.vfs().is_dir(uri, &is_dir)); if (!is_dir) { - *type = ObjectType::INVALID; - return Status::Ok(); + return ObjectType::INVALID; } } - bool exists = is_array(resources, uri); - if (exists) { - *type = ObjectType::ARRAY; - return Status::Ok(); - } - throw_if_not_ok(is_group(resources, uri, &exists)); - if (exists) { - *type = ObjectType::GROUP; - return Status::Ok(); + if (is_array(resources, uri)) { + return ObjectType::ARRAY; + } + if (is_group(resources, uri)) { + return ObjectType::GROUP; } - *type = ObjectType::INVALID; - return Status::Ok(); + return ObjectType::INVALID; } -Status object_move( +void object_move( ContextResources& resources, const char* old_path, const char* new_path) { auto old_uri = URI(old_path); - if (old_uri.is_invalid()) + if (old_uri.is_invalid()) { throw ObjectException( "Cannot move object '" + std::string(old_path) + "'; Invalid URI"); + } auto new_uri = URI(new_path); - if (new_uri.is_invalid()) + if (new_uri.is_invalid()) { throw ObjectException( "Cannot move object to '" + std::string(new_path) + "'; Invalid URI"); + } - ObjectType obj_type; - throw_if_not_ok(object_type(resources, old_uri, &obj_type)); - if (obj_type == ObjectType::INVALID) + if (object_type(resources, old_uri) == ObjectType::INVALID) { throw ObjectException( "Cannot move object '" + std::string(old_path) + "'; Invalid TileDB object"); + } - return resources.vfs().move_dir(old_uri, new_uri); + throw_if_not_ok(resources.vfs().move_dir(old_uri, new_uri)); } -Status object_remove(ContextResources& resources, const char* path) { +void object_remove(ContextResources& resources, const char* path) { auto uri = URI(path); - if (uri.is_invalid()) + if (uri.is_invalid()) { throw ObjectException( "Cannot remove object '" + std::string(path) + "'; Invalid URI"); + } - ObjectType obj_type; - throw_if_not_ok(object_type(resources, uri, &obj_type)); - if (obj_type == ObjectType::INVALID) + if (object_type(resources, uri) == ObjectType::INVALID) { throw ObjectException( "Cannot remove object '" + std::string(path) + "'; Invalid TileDB object"); + } - return resources.vfs().remove_dir(uri); + throw_if_not_ok(resources.vfs().remove_dir(uri)); } } // namespace tiledb::sm diff --git a/tiledb/sm/object/object.h b/tiledb/sm/object/object.h index 108eb3ca0e07..c3edb73de6b1 100644 --- a/tiledb/sm/object/object.h +++ b/tiledb/sm/object/object.h @@ -63,20 +63,18 @@ bool is_array(ContextResources& resources, const URI& uri); * @param uri the URI to be checked. * @param is_group Set to `true` if the URI is a group and `false` * otherwise. - * @return Status + * @return bool */ -Status is_group(ContextResources& resources, const URI& uri, bool* is_group); +bool is_group(ContextResources& resources, const URI& uri); /** * Returns the tiledb object type * * @param resources the context resources. - * @param uri Path to TileDB object resource - * @param type The ObjectType to be retrieved. - * @return Status + * @param uri Path to TileDB object resource. + * @return ObjectType */ -Status object_type( - ContextResources& resources, const URI& uri, ObjectType* type); +ObjectType object_type(ContextResources& resources, const URI& uri); /** * Moves a TileDB object. If `new_path` exists, it will be overwritten. @@ -84,9 +82,8 @@ Status object_type( * @param resources the context resources. * @param old_path the old path of the object. * @param new_path the new path of the object. - * @return Status */ -Status object_move( +void object_move( ContextResources& resources, const char* old_path, const char* new_path); /** @@ -94,9 +91,8 @@ Status object_move( * * @param resources the context resources. * @param path the path to the object to be removed. - * @return Status */ -Status object_remove(ContextResources& resources, const char* path); +void object_remove(ContextResources& resources, const char* path); } // namespace tiledb::sm diff --git a/tiledb/sm/object/object_iter.cc b/tiledb/sm/object/object_iter.cc index e428ce2ab70a..43599d6de221 100644 --- a/tiledb/sm/object/object_iter.cc +++ b/tiledb/sm/object/object_iter.cc @@ -65,7 +65,7 @@ ObjectIter* object_iter_begin( ObjectType obj_type; for (auto& uri : uris) { try { - throw_if_not_ok(object_type(resources, uri, &obj_type)); + obj_type = object_type(resources, uri); if (obj_type != ObjectType::INVALID) { obj_iter->objs_.push_back(uri); if (order == WalkOrder::POSTORDER) @@ -101,7 +101,7 @@ ObjectIter* object_iter_begin(ContextResources& resources, const char* path) { ObjectType obj_type; for (auto& uri : uris) { try { - throw_if_not_ok(object_type(resources, uri, &obj_type)); + obj_type = object_type(resources, uri); if (obj_type != ObjectType::INVALID) { obj_iter->objs_.push_back(uri); } @@ -164,7 +164,7 @@ Status object_iter_next_postorder( // Push the new TileDB objects in the front of the iterator's list ObjectType obj_type; for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources, *it, &obj_type)); + obj_type = object_type(resources, *it); if (obj_type != ObjectType::INVALID) { obj_iter->objs_.push_front(*it); obj_iter->expanded_.push_front(false); @@ -176,7 +176,7 @@ Status object_iter_next_postorder( // Prepare the values to be returned URI front_uri = obj_iter->objs_.front(); obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources, front_uri, type)); + *type = object_type(resources, front_uri); *path = obj_iter->next_.c_str(); *has_next = true; @@ -196,7 +196,7 @@ Status object_iter_next_preorder( // Prepare the values to be returned URI front_uri = obj_iter->objs_.front(); obj_iter->next_ = front_uri.to_string(); - throw_if_not_ok(object_type(resources, front_uri, type)); + *type = object_type(resources, front_uri); *path = obj_iter->next_.c_str(); *has_next = true; @@ -214,7 +214,7 @@ Status object_iter_next_preorder( // Push the new TileDB objects in the front of the iterator's list ObjectType obj_type; for (auto it = uris.rbegin(); it != uris.rend(); ++it) { - throw_if_not_ok(object_type(resources, *it, &obj_type)); + obj_type = object_type(resources, *it); if (obj_type != ObjectType::INVALID) obj_iter->objs_.push_front(*it); } diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 74fb4b8449cd..115334f626e4 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -95,7 +95,7 @@ tdb_unique_ptr QueryPlanFx::create_array(const URI uri) { throw_if_not_ok(key.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); // Create the (empty) array on disk. - throw_if_not_ok(Array::create(resources_, uri, schema, key)); + Array::create(resources_, uri, schema, key); tdb_unique_ptr array(new Array{resources_, uri}); return array; diff --git a/tiledb/sm/subarray/test/unit_add_ranges_list.cc b/tiledb/sm/subarray/test/unit_add_ranges_list.cc index 063ffccaa1e4..06fcf8453389 100644 --- a/tiledb/sm/subarray/test/unit_add_ranges_list.cc +++ b/tiledb/sm/subarray/test/unit_add_ranges_list.cc @@ -91,8 +91,7 @@ TEST_CASE("Subarray::add_ranges_list", "[subarray]") { tiledb::sm::Array a(ctx.resources(), tiledb::sm::URI{"mem://junk"}); tiledb::sm::EncryptionKey ek; CHECK(ek.set_key(tiledb::sm::EncryptionType::NO_ENCRYPTION, nullptr, 0).ok()); - CHECK(tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek) - .ok()); + tiledb::sm::Array::create(ctx.resources(), a.array_uri(), sp_as, ek); CHECK(a.open( tiledb::sm::QueryType::READ, tiledb::sm::EncryptionType::NO_ENCRYPTION, From e67c0126f0e04205a107b930bc272b7664a5617c Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:17:50 +0200 Subject: [PATCH 451/456] Fix ASAN nightlies with failure in unit_query. (#5102) After moving more tests to run in the ASAN nightlies, we encountered an issue in unit_query where it doesn't properly setup a test offset buffer with an extra offset and tries to read the extra offset when copying over to the tile. [sc-49797] --- TYPE: NO_HISTORY DESC: Fix ASAN nightlies with failure in unit_query. --- tiledb/sm/query/test/unit_query_condition.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tiledb/sm/query/test/unit_query_condition.cc b/tiledb/sm/query/test/unit_query_condition.cc index ca66dbe7eacc..a10809236f63 100644 --- a/tiledb/sm/query/test/unit_query_condition.cc +++ b/tiledb/sm/query/test/unit_query_condition.cc @@ -4591,6 +4591,7 @@ TEST_CASE( offsets.push_back(curr_offset); curr_offset += elem.size(); } + offsets.push_back(curr_offset); // Initialize the result tile. ResultTile::TileSizes tile_sizes( From 379ec58745ecd03dab9fd8963cd0b4f625cbdfb6 Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:56:10 +0200 Subject: [PATCH 452/456] Fix ASAN nightlies with failure in unit_cppapi_string. (#5103) After moving more tests to run in the ASAN nightlies, we encountered an issue in unit_cppapi_string where a handle is used after being freed. This test is doing this by design so this PR disables the test when ASAN is enabled. [sc-49797] --- TYPE: NO_HISTORY DESC: Fix ASAN nightlies with failure in unit_cppapi_string. --- tiledb/api/cpp_api_support/test/CMakeLists.txt | 4 ++++ tiledb/api/cpp_api_support/test/unit_cppapi_string.cc | 2 ++ tiledb/sm/array_schema/test/unit_current_domain.cc | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tiledb/api/cpp_api_support/test/CMakeLists.txt b/tiledb/api/cpp_api_support/test/CMakeLists.txt index 30acaadf619e..4b0097f0a5ec 100644 --- a/tiledb/api/cpp_api_support/test/CMakeLists.txt +++ b/tiledb/api/cpp_api_support/test/CMakeLists.txt @@ -30,3 +30,7 @@ commence(unit_test cppapi_string) this_target_sources(unit_cppapi_string.cc) this_target_object_libraries(capi_string) conclude(unit_test) + +if(TILEDB_SANITIZER) + target_compile_definitions(unit_cppapi_string PRIVATE -DHAVE_SANITIZER) +endif() diff --git a/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc index ffddb3f5012d..0157f99f68d0 100644 --- a/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc +++ b/tiledb/api/cpp_api_support/test/unit_cppapi_string.cc @@ -71,6 +71,7 @@ TEST_CASE( REQUIRE(result == test_string); } +#ifndef HAVE_SANITIZER TEST_CASE( "CAPIString: Test that accessing freed handle fails", "[capi_string][freed_handle]") { @@ -82,6 +83,7 @@ TEST_CASE( size_t length = 0; REQUIRE(tiledb_string_view(handle_copy, &chars, &length) == TILEDB_ERR); } +#endif TEST_CASE( "CAPIString: Test convert_to_string with null handle", "[capi_string]") { diff --git a/tiledb/sm/array_schema/test/unit_current_domain.cc b/tiledb/sm/array_schema/test/unit_current_domain.cc index df1663cd4541..54dfe96b074b 100644 --- a/tiledb/sm/array_schema/test/unit_current_domain.cc +++ b/tiledb/sm/array_schema/test/unit_current_domain.cc @@ -1,5 +1,5 @@ /** - * @file unit-current_domain.cc + * @file unit_current_domain.cc * * @section LICENSE * From cdf0cf340b74b30ac0ce36c7ded9190327709673 Mon Sep 17 00:00:00 2001 From: Beka Davis <31743465+bekadavis9@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:26:10 -0400 Subject: [PATCH 453/456] Add a ContextResources& parameter to the Query constructor. (#5098) Add a `ContextResources&` parameter to the `Query` constructor. Add a `ContextResources&` parameter to the `DimensionLabelQuery` constructor. Add a `ContextResources&` parameter to the `ArrayDimensionLabelQuery` constructor. Update `DimensionLabelQueryStatusException` -> `DimensionLabelQueryException`. [sc-49688] --- TYPE: NO_HISTORY DESC: Add a `ContextResources&` parameter to the `Query` constructor. --- test/src/unit-enumerations.cc | 8 ++--- tiledb/sm/c_api/tiledb.cc | 11 ++++--- tiledb/sm/c_api/tiledb_filestore.cc | 15 ++++++--- .../sm/consolidator/fragment_consolidator.cc | 8 ++++- .../array_dimension_label_queries.cc | 33 ++++++++++--------- .../array_dimension_label_queries.h | 8 +++++ .../dimension_label/dimension_label_query.cc | 22 +++++++------ .../dimension_label/dimension_label_query.h | 22 +++++++++++-- tiledb/sm/query/query.cc | 4 ++- tiledb/sm/query/query.h | 9 +++++ tiledb/sm/query_plan/test/unit_query_plan.cc | 2 +- 11 files changed, 97 insertions(+), 45 deletions(-) diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc index 46a81bcf853f..3e1a1dedaba4 100644 --- a/test/src/unit-enumerations.cc +++ b/test/src/unit-enumerations.cc @@ -2398,10 +2398,10 @@ TEST_CASE_METHOD( auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); qc1.set_use_enumeration(false); - Query q1(ctx_.storage_manager(), array); + Query q1(ctx_.resources(), ctx_.storage_manager(), array); throw_if_not_ok(q1.set_condition(qc1)); - Query q2(ctx_.storage_manager(), array); + Query q2(ctx_.resources(), ctx_.storage_manager(), array); ser_des_query(&q1, &q2, client_side, ser_type); auto qc2 = q2.condition(); @@ -2434,10 +2434,10 @@ TEST_CASE_METHOD( throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3)); - Query q1(ctx_.storage_manager(), array); + Query q1(ctx_.resources(), ctx_.storage_manager(), array); throw_if_not_ok(q1.set_condition(qc3)); - Query q2(ctx_.storage_manager(), array); + Query q2(ctx_.resources(), ctx_.storage_manager(), array); ser_des_query(&q1, &q2, client_side, ser_type); auto qc4 = q2.condition(); diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index d3be1ffa9756..ba6253036988 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -1027,8 +1027,8 @@ int32_t tiledb_query_alloc( } // Create query - (*query)->query_ = new (std::nothrow) - tiledb::sm::Query(ctx->storage_manager(), array->array_); + (*query)->query_ = new (std::nothrow) tiledb::sm::Query( + ctx->resources(), ctx->storage_manager(), array->array_); if ((*query)->query_ == nullptr) { auto st = Status_Error( "Failed to allocate TileDB query object; Memory allocation failed"); @@ -3791,8 +3791,8 @@ int32_t tiledb_deserialize_query_and_array( } // Create query - (*query)->query_ = new (std::nothrow) - tiledb::sm::Query(ctx->storage_manager(), (*array)->array_); + (*query)->query_ = new (std::nothrow) tiledb::sm::Query( + ctx->resources(), ctx->storage_manager(), (*array)->array_); if ((*query)->query_ == nullptr) { auto st = Status_Error( "Failed to allocate TileDB query object; Memory allocation failed"); @@ -4370,7 +4370,8 @@ capi_return_t tiledb_handle_query_plan_request( api::ensure_buffer_is_valid(request); api::ensure_buffer_is_valid(response); - tiledb::sm::Query query(ctx->storage_manager(), array->array_); + tiledb::sm::Query query( + ctx->resources(), ctx->storage_manager(), array->array_); tiledb::sm::serialization::deserialize_query_plan_request( static_cast(serialization_type), request->buffer(), diff --git a/tiledb/sm/c_api/tiledb_filestore.cc b/tiledb/sm/c_api/tiledb_filestore.cc index cc2d4d1ce031..d0541879c2a8 100644 --- a/tiledb/sm/c_api/tiledb_filestore.cc +++ b/tiledb/sm/c_api/tiledb_filestore.cc @@ -274,7 +274,8 @@ int32_t tiledb_filestore_uri_import( auto buffer_size = get_buffer_size_from_config(context.resources().config(), tile_extent); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::GLOBAL_ORDER)); std::vector buffer(buffer_size); @@ -296,7 +297,8 @@ int32_t tiledb_filestore_uri_import( query.set_subarray(subarray); auto tiledb_cloud_fix = [&](uint64_t start, uint64_t end) { - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray_cloud_fix( array.get(), nullptr, context.resources().logger(), true); @@ -429,7 +431,8 @@ int32_t tiledb_filestore_uri_export( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); subarray.add_range(0, std::move(subarray_range)); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); @@ -532,7 +535,8 @@ int32_t tiledb_filestore_buffer_import( static_cast(0), ""); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); tiledb::sm::Subarray subarray( @@ -601,7 +605,8 @@ int32_t tiledb_filestore_buffer_export( static_cast(subarray_range_arr), sizeof(uint64_t) * 2); subarray.add_range(0, std::move(subarray_range)); - tiledb::sm::Query query(context.storage_manager(), array); + tiledb::sm::Query query( + context.resources(), context.storage_manager(), array); throw_if_not_ok(query.set_layout(tiledb::sm::Layout::ROW_MAJOR)); query.set_subarray(subarray); uint64_t size_tmp = size; diff --git a/tiledb/sm/consolidator/fragment_consolidator.cc b/tiledb/sm/consolidator/fragment_consolidator.cc index 0494b156d464..2b8941f2a1dc 100644 --- a/tiledb/sm/consolidator/fragment_consolidator.cc +++ b/tiledb/sm/consolidator/fragment_consolidator.cc @@ -668,7 +668,12 @@ Status FragmentConsolidator::create_queries( // Create read query query_r = tdb_unique_ptr(tdb_new( - Query, storage_manager_, array_for_reads, nullopt, read_memory_budget)); + Query, + resources_, + storage_manager_, + array_for_reads, + nullopt, + read_memory_budget)); throw_if_not_ok(query_r->set_layout(Layout::GLOBAL_ORDER)); // Dense consolidation will do a tile aligned read. @@ -695,6 +700,7 @@ Status FragmentConsolidator::create_queries( // Create write query query_w = tdb_unique_ptr(tdb_new( Query, + resources_, storage_manager_, array_for_writes, fragment_name, diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc index 6330d25d098a..4f961970d5a7 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.cc @@ -50,13 +50,14 @@ using namespace tiledb::common; namespace tiledb::sm { ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( + ContextResources& resources, StorageManager* storage_manager, Array* array, const Subarray& subarray, const std::unordered_map& label_buffers, const std::unordered_map& array_buffers, const optional& fragment_name) - : resources_(storage_manager->resources()) + : resources_(resources) , storage_manager_(storage_manager) , label_range_queries_by_dim_idx_(subarray.dim_num(), nullptr) , label_data_queries_by_dim_idx_(subarray.dim_num()) @@ -79,7 +80,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( } else { // Cannot both read label ranges and write label data on the same write. if (subarray.has_label_ranges()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Cannot set both label " "buffer and label range on a write query."); } @@ -105,7 +106,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( case (QueryType::UPDATE): case (QueryType::MODIFY_EXCLUSIVE): if (!label_buffers.empty() || subarray.has_label_ranges()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Query type " + query_type_str(array->get_query_type()) + " is not supported for dimension labels."); @@ -113,7 +114,7 @@ ArrayDimensionLabelQueries::ArrayDimensionLabelQueries( break; default: - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to add dimension label queries. Unknown query type " + query_type_str(array->get_query_type()) + "."); } @@ -132,7 +133,7 @@ bool ArrayDimensionLabelQueries::completed() const { DimensionLabelQuery* ArrayDimensionLabelQueries::get_range_query( ArrayDimensionLabelQueries::dimension_size_type dim_idx) const { if (!has_range_query(dim_idx)) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "No dimension label range query for dimension at index " + std::to_string(dim_idx)); } @@ -142,7 +143,7 @@ DimensionLabelQuery* ArrayDimensionLabelQueries::get_range_query( std::vector ArrayDimensionLabelQueries::get_data_query( ArrayDimensionLabelQueries::dimension_size_type dim_idx) const { if (!has_data_query(dim_idx)) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "No dimension label data query for dimension at index " + std::to_string(dim_idx)); } @@ -161,7 +162,7 @@ void ArrayDimensionLabelQueries::process_data_queries() { throw_if_not_ok(query->process()); return Status::Ok(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to process data query for label '" + query->dim_label_name() + "'. " + err.what()); } @@ -182,7 +183,7 @@ void ArrayDimensionLabelQueries::process_range_queries(Query* parent_query) { range_query->init(); throw_if_not_ok(range_query->process()); if (!range_query->completed()) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Range query for label '" + range_query->dim_label_name() + "' failed to complete."); } @@ -202,7 +203,7 @@ void ArrayDimensionLabelQueries::process_range_queries(Query* parent_query) { } return Status::Ok(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to process and update index ranges for label '" + range_query->dim_label_name() + "'. " + err.what()); } @@ -249,13 +250,14 @@ void ArrayDimensionLabelQueries::add_read_queries( // Create the range query. range_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, label_ranges)); label_range_queries_by_dim_idx_[dim_idx] = range_queries_.back().get(); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the query to read range data from label '" + label_name + "'. " + err.what()); } @@ -281,6 +283,7 @@ void ArrayDimensionLabelQueries::add_read_queries( // Create the data query. data_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, @@ -291,7 +294,7 @@ void ArrayDimensionLabelQueries::add_read_queries( label_data_queries_by_dim_idx_[dim_label_ref.dimension_index()].push_back( data_queries_.back().get()); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the data query for label '" + label_name + "'. " + err.what()); } @@ -312,7 +315,7 @@ void ArrayDimensionLabelQueries::add_write_queries( // Verify that this subarray is not set to use labels. if (subarray.has_label_ranges(dim_label_ref.dimension_index())) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Cannot write label data when subarray is set by label range."); } @@ -332,6 +335,7 @@ void ArrayDimensionLabelQueries::add_write_queries( // Create the dimension label query. data_queries_.emplace_back(tdb_new( DimensionLabelQuery, + resources_, storage_manager_, dim_label, dim_label_ref, @@ -344,7 +348,7 @@ void ArrayDimensionLabelQueries::add_write_queries( label_data_queries_by_dim_idx_[dim_label_ref.dimension_index()].push_back( data_queries_.back().get()); } catch (const StatusException& err) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Failed to initialize the data query for label '" + label_name + "'. " + err.what()); } @@ -358,8 +362,7 @@ shared_ptr ArrayDimensionLabelQueries::open_dimension_label( const QueryType& query_type) { // Create the dimension label. auto label_iter = dimension_labels_.try_emplace( - dim_label_name, - make_shared(HERE(), storage_manager_->resources(), dim_label_uri)); + dim_label_name, make_shared(HERE(), resources_, dim_label_uri)); const auto dim_label = label_iter.first->second; // Open the dimension label with the same timestamps and encryption as the diff --git a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h index 7746956ee85c..0d73b42eed13 100644 --- a/tiledb/sm/query/dimension_label/array_dimension_label_queries.h +++ b/tiledb/sm/query/dimension_label/array_dimension_label_queries.h @@ -69,6 +69,13 @@ class ArrayDimensionLabelQueries { /** * Constructor. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * ArrayDimensionLabelQueries(ContextResources&, Array*, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param array Parent array the dimension labels are defined on. * @param subarray Subarray for the query on the parent array. @@ -78,6 +85,7 @@ class ArrayDimensionLabelQueries { * @param fragment_name Optional fragment name for writing fragments. */ ArrayDimensionLabelQueries( + ContextResources& resources, StorageManager* storage_manager, Array* array, const Subarray& subarray, diff --git a/tiledb/sm/query/dimension_label/dimension_label_query.cc b/tiledb/sm/query/dimension_label/dimension_label_query.cc index e6563af7f140..bf810731feec 100644 --- a/tiledb/sm/query/dimension_label/dimension_label_query.cc +++ b/tiledb/sm/query/dimension_label/dimension_label_query.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,6 +50,7 @@ using namespace tiledb::common; namespace tiledb::sm { DimensionLabelQuery::DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, @@ -57,7 +58,7 @@ DimensionLabelQuery::DimensionLabelQuery( const QueryBuffer& label_buffer, const QueryBuffer& index_buffer, optional fragment_name) - : Query(storage_manager, dim_label, fragment_name) + : Query(resources, storage_manager, dim_label, fragment_name) , dim_label_name_{dim_label_ref.name()} { switch (dim_label->get_query_type()) { case (QueryType::READ): @@ -92,7 +93,7 @@ DimensionLabelQuery::DimensionLabelQuery( default: // Invalid label order type. - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Unrecognized label order " + data_order_str(dim_label_ref.label_order())); } @@ -100,18 +101,19 @@ DimensionLabelQuery::DimensionLabelQuery( } default: - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Query type " + query_type_str(dim_label->get_query_type()) + " not supported for dimension label queries."); } } DimensionLabelQuery::DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, const std::vector& label_ranges) - : Query(storage_manager, dim_label, nullopt) + : Query(resources, storage_manager, dim_label, nullopt) , dim_label_name_{dim_label_ref.name()} , index_data_{IndexDataCreate::make_index_data( array_schema().dimension_ptr(0)->type(), @@ -123,12 +125,12 @@ DimensionLabelQuery::DimensionLabelQuery( case (DataOrder::DECREASING_DATA): break; case (DataOrder::UNORDERED_DATA): - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Support for reading ranges from unordered labels is not yet " "implemented."); default: // Invalid label order type. - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Unrecognized label order " + data_order_str(dim_label_ref.label_order())); } @@ -194,7 +196,7 @@ void DimensionLabelQuery::initialize_ordered_write_query( Subarray subarray{*this->subarray()}; subarray.set_ranges_for_dim(0, parent_subarray.ranges_for_dim(dim_idx)); if (subarray.range_num() > 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Dimension label writes can only be set for a single range."); } set_subarray(subarray); @@ -210,7 +212,7 @@ void DimensionLabelQuery::initialize_ordered_write_query( subarray.set_coalesce_ranges(true); subarray.add_point_ranges(0, index_buffer.buffer_, count); if (subarray.range_num() > 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "The dimension data must contain consecutive points when writing to " "a dimension label."); } @@ -231,7 +233,7 @@ void DimensionLabelQuery::initialize_unordered_write_query( if (!parent_subarray.is_default(dim_idx)) { const auto& ranges = parent_subarray.ranges_for_dim(dim_idx); if (ranges.size() != 1) { - throw DimensionLabelQueryStatusException( + throw DimensionLabelQueryException( "Dimension label writes can only be set for a single range."); } } diff --git a/tiledb/sm/query/dimension_label/dimension_label_query.h b/tiledb/sm/query/dimension_label/dimension_label_query.h index aa03665be6ca..5f8ae54b997f 100644 --- a/tiledb/sm/query/dimension_label/dimension_label_query.h +++ b/tiledb/sm/query/dimension_label/dimension_label_query.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -50,9 +50,9 @@ class QueryBuffer; class Subarray; /** Class for dimension label query status exceptions. */ -class DimensionLabelQueryStatusException : public StatusException { +class DimensionLabelQueryException : public StatusException { public: - explicit DimensionLabelQueryStatusException(const std::string& msg) + explicit DimensionLabelQueryException(const std::string& msg) : StatusException("DimensionLabelQuery", msg) { } }; @@ -66,6 +66,13 @@ class DimensionLabelQuery : public Query { /** * Constructor for queries to read or write label data. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * DimensionLabelQuery(ContextResources&, shared_ptr, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param dim_label Opened dimension label for the query. * @param dim_label_ref Description of the dimension label. @@ -76,6 +83,7 @@ class DimensionLabelQuery : public Query { * @param fragment_name Name to use when writing the fragment. */ DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, @@ -87,12 +95,20 @@ class DimensionLabelQuery : public Query { /** * Constructor for range queries. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * DimensionLabelQuery(ContextResources&, shared_ptr, ...). + * + * @param resources The context resources. * @param storage_manager Storage manager object. * @param dim_label Opened dimension label for the query. * @param dim_label_ref Description of the dimension label. * @param label_ranges Label ranges to read index ranges from. */ DimensionLabelQuery( + ContextResources& resources, StorageManager* storage_manager, shared_ptr dim_label, const DimensionLabel& dim_label_ref, diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index f9bb85d6d001..6a1d06f0f8c5 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -82,11 +82,12 @@ static uint64_t get_effective_memory_budget( /* ****************************** */ Query::Query( + ContextResources& resources, StorageManager* storage_manager, shared_ptr array, optional fragment_name, optional memory_budget) - : resources_(storage_manager->resources()) + : resources_(resources) , stats_(resources_.stats().create_child("Query")) , logger_(resources_.logger()->clone("Query", ++logger_id_)) , query_memory_tracker_(resources_.memory_tracker_manager().create_tracker( @@ -710,6 +711,7 @@ void Query::init() { // Initialize the dimension label queries. dim_label_queries_ = tdb_unique_ptr(tdb_new( ArrayDimensionLabelQueries, + resources_, storage_manager_, array_, subarray_, diff --git a/tiledb/sm/query/query.h b/tiledb/sm/query/query.h index eb23873819c9..b61dda308522 100644 --- a/tiledb/sm/query/query.h +++ b/tiledb/sm/query/query.h @@ -138,8 +138,16 @@ class Query { * case the query will be used as writes and the given URI should be used * for the name of the new fragment to be created. * + * This is a transitional constructor in the sense that we are working + * on removing the dependency of the Query class on StorageManager. + * For now, we still need to keep the storage_manager argument, but once the + * dependency is gone, the signature will be + * Query(ContextResources&, shared_ptr, ...). + * * @note Array must be a properly opened array. * + * @param resources The context resources. + * @param storage_manager Storage manager object. * @param array The array that is being queried. * @param fragment_uri The full URI for the new fragment. Only used for * writes. @@ -149,6 +157,7 @@ class Query { * will be obtained from the sm.mem.total_budget config option. */ Query( + ContextResources& resources, StorageManager* storage_manager, shared_ptr array, optional fragment_name = nullopt, diff --git a/tiledb/sm/query_plan/test/unit_query_plan.cc b/tiledb/sm/query_plan/test/unit_query_plan.cc index 115334f626e4..7379e9c2fc22 100644 --- a/tiledb/sm/query_plan/test/unit_query_plan.cc +++ b/tiledb/sm/query_plan/test/unit_query_plan.cc @@ -126,7 +126,7 @@ TEST_CASE_METHOD(QueryPlanFx, "Query plan dump_json", "[query_plan][dump]") { REQUIRE(st.ok()); shared_ptr array_shared = std::move(array); - Query query(sm_.get(), array_shared); + Query query(resources_, sm_.get(), array_shared); REQUIRE(query.set_layout(Layout::ROW_MAJOR).ok()); stats::Stats stats("foo"); From 6539f14e138cd3b7822b473ba8e3d9afd52e3774 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 19 Jun 2024 18:14:56 +0300 Subject: [PATCH 454/456] Add support for custom headers for REST via config. (#5104) [SC-49141](https://app.shortcut.com/tiledb-inc/story/49141/add-support-for-custom-rest-headers-via-config) Similar to S3 custom header support (#4400) this PR adds support for custom headers in REST request. This allows users to set header via a config parameter instead of the current `tildb_context_set_tag`. The context set tag does not work in all cases because its not always possible to use the same context, i.e SOMA and VCF python APIs can't pass a TileDB-Py ctx to their respective C++ code. Big thanks to @Shelnutt2 for implementing this. I added a test and documentation. --- TYPE: CONFIG DESC: Add `rest.custom_headers.*` config option to set custom headers on REST requests. --------- Co-authored-by: Seth Shelnutt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Co-authored-by: Luc Rancourt --- test/src/unit-curl.cc | 16 ++++++++++++++++ tiledb/api/c_api/config/config_api_external.h | 4 ++++ tiledb/sm/cpp_api/config.h | 4 ++++ tiledb/sm/misc/constants.cc | 3 +++ tiledb/sm/misc/constants.h | 3 +++ tiledb/sm/rest/rest_client.cc | 18 ++++++++++++++++++ tiledb/sm/rest/rest_client.h | 10 ++++++++++ 7 files changed, 58 insertions(+) diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index 9241941ad2fb..7f40e8b31ec5 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -128,3 +128,19 @@ TEST_CASE( .ok()); CHECK(rest_client.rest_server() == "http://localhost:8080"); } + +TEST_CASE( + "RestClient: Ensure custom headers are set", + "[rest-client][custom-headers]") { + tiledb::sm::Config cfg; + REQUIRE(cfg.set("rest.custom_headers.abc", "def").ok()); + REQUIRE(cfg.set("rest.custom_headers.ghi", "jkl").ok()); + + ContextResources resources( + cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); + // We copy because the [] operator is not const-qualified. + auto extra_headers = resources.rest_client()->extra_headers(); + CHECK(extra_headers.size() == 2); + CHECK(extra_headers["abc"] == "def"); + CHECK(extra_headers["ghi"] == "jkl"); +} diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 161a5a75ae6e..19d96f180ff7 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -709,6 +709,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * - `rest.capnp_traversal_limit`
* CAPNP traversal limit used in the deserialization of messages(bytes)
* **Default**: 536870912 (512MB) + * - `rest.custom_headers.*`
+ * (Optional) Prefix for custom headers on REST requests. For each custom + * header, use "rest.custom_headers.header_key" = "header_value"
+ * **Optional. No Default** * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the filestore * API. The size should be bigger than the minimum tile size filestore diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index cd1c2110ad9c..e74608070784 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -889,6 +889,10 @@ class Config { * CAPNP traversal limit used in the deserialization of messages(bytes) *
* **Default**: 536870912 (512MB) + * - `rest.custom_headers.*`
+ * (Optional) Prefix for custom headers on REST requests. For each custom + * header, use "rest.custom_headers.header_key" = "header_value"
+ * **Optional. No Default** * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the * filestore API. The size should be bigger than the minimum tile size diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 8feb0542c065..438442a2b639 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -773,6 +773,9 @@ const unsigned concurrent_attr_reads = 2; /** The redirection header key in REST response. */ extern const std::string redirection_header_key = "location"; +/** The config key prefix for REST custom headers. */ +const std::string rest_header_prefix = "rest.custom_headers."; + /** String describing MIME_AUTODETECT. */ const std::string mime_autodetect_str = "AUTODETECT"; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index c675b8de4d35..5a854d404b8d 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -760,6 +760,9 @@ const void* fill_value(Datatype type); /** The redirection header key in REST response. */ extern const std::string redirection_header_key; +/** The REST custom headers config key prefix. */ +extern const std::string rest_header_prefix; + /** Delimiter for lists passed as config parameter */ extern const std::string config_delimiter; diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index 8c7b8f177489..ae7575317e53 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -60,6 +60,7 @@ #include "tiledb/common/logger.h" #include "tiledb/common/memory_tracker.h" #include "tiledb/sm/array/array.h" +#include "tiledb/sm/config/config_iter.h" #include "tiledb/sm/enums/query_type.h" #include "tiledb/sm/group/group.h" #include "tiledb/sm/misc/constants.h" @@ -148,6 +149,8 @@ Status RestClient::init( RETURN_NOT_OK(config_->get( "rest.resubmit_incomplete", &resubmit_incomplete_, &found)); + load_headers(*config_); + return Status::Ok(); } @@ -1661,6 +1664,17 @@ RestClient::post_consolidation_plan_from_rest( serialization_type_, returned_data); } +void RestClient::load_headers(const Config& cfg) { + for (auto iter = ConfigIter(cfg, constants::rest_header_prefix); !iter.end(); + iter.next()) { + auto key = iter.param(); + if (key.size() == 0) { + continue; + } + extra_headers_[key] = iter.value(); + } +} + #else RestClient::RestClient() { @@ -1832,6 +1846,10 @@ RestClient::post_consolidation_plan_from_rest( throw RestClientDisabledException(); } +void RestClient::load_headers(const Config&) { + throw RestClientDisabledException(); +} + #endif // TILEDB_SERIALIZATION } // namespace tiledb::sm diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index 0f7cfae63f5f..d572cff96c3b 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -418,6 +418,13 @@ class RestClient { std::vector> post_consolidation_plan_from_rest( const URI& uri, const Config& config, uint64_t fragment_size); + /** + * Constant accessor to `extra_headers_` member variable. + */ + const std::unordered_map& extra_headers() const { + return extra_headers_; + } + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -566,6 +573,9 @@ class RestClient { * @return Status */ Status ensure_json_null_delimited_string(Buffer* buffer); + + /** Load all custom headers from the given config. */ + void load_headers(const Config& cfg); }; } // namespace tiledb::sm From bdfc19c0aa746a265fd32b2872b0304621afbb2d Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 19 Jun 2024 20:08:59 +0300 Subject: [PATCH 455/456] Add config option to override the payer namespace for REST requests. (#5105) [SC-49142](https://app.shortcut.com/tiledb-inc/story/49142/add-support-for-rest-x-payer-via-config) This PR introduces a new config option that allows users to set the namespace to charge for REST requests when it cannot be automatically determined or when the user needs to set a specific override. Big thanks to @Shelnutt2 for implementing this. I added a test and made some small changes on top of it. --- TYPE: CONFIG DESC: Add `rest.payer_namespace` config option to set the namespace to be charged for REST requests. --------- Co-authored-by: Seth Shelnutt Co-authored-by: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Co-authored-by: eric-hughes-tiledb <82400964+eric-hughes-tiledb@users.noreply.github.com> Co-authored-by: Luc Rancourt --- test/src/unit-capi-config.cc | 1 + test/src/unit-curl.cc | 20 ++++++++++++++----- tiledb/api/c_api/config/config_api_external.h | 3 +++ tiledb/sm/config/config.cc | 2 ++ tiledb/sm/config/config.h | 3 +++ tiledb/sm/cpp_api/config.h | 3 +++ tiledb/sm/rest/rest_client.cc | 6 ++++++ tiledb/sm/rest/rest_client.h | 2 +- 8 files changed, 34 insertions(+), 6 deletions(-) diff --git a/test/src/unit-capi-config.cc b/test/src/unit-capi-config.cc index 5f8abc0260d8..b7835cb4e02c 100644 --- a/test/src/unit-capi-config.cc +++ b/test/src/unit-capi-config.cc @@ -615,6 +615,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") { all_param_values["rest.load_enumerations_on_array_open"] = "false"; all_param_values["rest.use_refactored_array_open"] = "true"; all_param_values["rest.use_refactored_array_open_and_query_submit"] = "true"; + all_param_values["rest.payer_namespace"] = ""; all_param_values["sm.allow_separate_attribute_writes"] = "false"; all_param_values["sm.allow_updates_experimental"] = "false"; all_param_values["sm.encryption_key"] = ""; diff --git a/test/src/unit-curl.cc b/test/src/unit-curl.cc index 7f40e8b31ec5..e3e79017bc77 100644 --- a/test/src/unit-curl.cc +++ b/test/src/unit-curl.cc @@ -138,9 +138,19 @@ TEST_CASE( ContextResources resources( cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); - // We copy because the [] operator is not const-qualified. - auto extra_headers = resources.rest_client()->extra_headers(); - CHECK(extra_headers.size() == 2); - CHECK(extra_headers["abc"] == "def"); - CHECK(extra_headers["ghi"] == "jkl"); + const auto& extra_headers = resources.rest_client()->extra_headers(); + CHECK(extra_headers.at("abc") == "def"); + CHECK(extra_headers.at("ghi") == "jkl"); +} + +TEST_CASE( + "RestClient: Ensure payer namespace is set", + "[rest-client][payer-namespace]") { + tiledb::sm::Config cfg; + REQUIRE(cfg.set("rest.payer_namespace", "foo").ok()); + + ContextResources resources( + cfg, tiledb::test::g_helper_logger(), 1, 1, "test"); + const auto& extra_headers = resources.rest_client()->extra_headers(); + CHECK(extra_headers.at("X-Payer") == "foo"); } diff --git a/tiledb/api/c_api/config/config_api_external.h b/tiledb/api/c_api/config/config_api_external.h index 19d96f180ff7..8cb17bdf53e3 100644 --- a/tiledb/api/c_api/config/config_api_external.h +++ b/tiledb/api/c_api/config/config_api_external.h @@ -713,6 +713,9 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT; * (Optional) Prefix for custom headers on REST requests. For each custom * header, use "rest.custom_headers.header_key" = "header_value"
* **Optional. No Default** + * - `rest.payer_namespace`
+ * The namespace that should be charged for the request.
+ * **Default**: no default set * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the filestore * API. The size should be bigger than the minimum tile size filestore diff --git a/tiledb/sm/config/config.cc b/tiledb/sm/config/config.cc index 4a5e677295c8..325140d2125d 100644 --- a/tiledb/sm/config/config.cc +++ b/tiledb/sm/config/config.cc @@ -96,6 +96,7 @@ const std::string Config::REST_LOAD_METADATA_ON_ARRAY_OPEN = "true"; const std::string Config::REST_LOAD_NON_EMPTY_DOMAIN_ON_ARRAY_OPEN = "true"; const std::string Config::REST_USE_REFACTORED_ARRAY_OPEN = "false"; const std::string Config::REST_USE_REFACTORED_QUERY_SUBMIT = "false"; +const std::string Config::REST_PAYER_NAMESPACE = ""; const std::string Config::SM_ALLOW_SEPARATE_ATTRIBUTE_WRITES = "false"; const std::string Config::SM_ALLOW_UPDATES_EXPERIMENTAL = "false"; const std::string Config::SM_ENCRYPTION_KEY = ""; @@ -270,6 +271,7 @@ const std::map default_config_values = { std::make_pair( "rest.use_refactored_array_open_and_query_submit", Config::REST_USE_REFACTORED_QUERY_SUBMIT), + std::make_pair("rest.payer_namespace", Config::REST_PAYER_NAMESPACE), std::make_pair( "config.env_var_prefix", Config::CONFIG_ENVIRONMENT_VARIABLE_PREFIX), std::make_pair("config.logging_level", Config::CONFIG_LOGGING_LEVEL), diff --git a/tiledb/sm/config/config.h b/tiledb/sm/config/config.h index 3dd6fa760f1e..0ed2e5e050d1 100644 --- a/tiledb/sm/config/config.h +++ b/tiledb/sm/config/config.h @@ -125,6 +125,9 @@ class Config { /** Refactored query submit is disabled by default */ static const std::string REST_USE_REFACTORED_QUERY_SUBMIT; + /** The namespace that should be charged for the request. */ + static const std::string REST_PAYER_NAMESPACE; + /** The prefix to use for checking for parameter environmental variables. */ static const std::string CONFIG_ENVIRONMENT_VARIABLE_PREFIX; diff --git a/tiledb/sm/cpp_api/config.h b/tiledb/sm/cpp_api/config.h index e74608070784..e447683267b3 100644 --- a/tiledb/sm/cpp_api/config.h +++ b/tiledb/sm/cpp_api/config.h @@ -893,6 +893,9 @@ class Config { * (Optional) Prefix for custom headers on REST requests. For each custom * header, use "rest.custom_headers.header_key" = "header_value"
* **Optional. No Default** + * - `rest.payer_namespace`
+ * The namespace that should be charged for the request.
+ * **Default**: no default set * - `filestore.buffer_size`
* Specifies the size in bytes of the internal buffers used in the * filestore API. The size should be bigger than the minimum tile size diff --git a/tiledb/sm/rest/rest_client.cc b/tiledb/sm/rest/rest_client.cc index ae7575317e53..b7408fd6ffd4 100644 --- a/tiledb/sm/rest/rest_client.cc +++ b/tiledb/sm/rest/rest_client.cc @@ -151,6 +151,12 @@ Status RestClient::init( load_headers(*config_); + if (auto payer = + config_->get("rest.payer_namespace", Config::must_find); + !payer.empty()) { + extra_headers_["X-Payer"] = std::move(payer); + } + return Status::Ok(); } diff --git a/tiledb/sm/rest/rest_client.h b/tiledb/sm/rest/rest_client.h index d572cff96c3b..84b2535a820f 100644 --- a/tiledb/sm/rest/rest_client.h +++ b/tiledb/sm/rest/rest_client.h @@ -574,7 +574,7 @@ class RestClient { */ Status ensure_json_null_delimited_string(Buffer* buffer); - /** Load all custom headers from the given config. */ + /** Load all custom headers from the given config. */ void load_headers(const Config& cfg); }; From 5482f6c69a3279038bc47b65a916fdeb32a4d86b Mon Sep 17 00:00:00 2001 From: KiterLuc <67824247+KiterLuc@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:30:04 +0200 Subject: [PATCH 456/456] Fix wrong type in unit aggregators. (#5132) For string types, we used sizeof(std::string) instead of sizeof(char) in unit aggregators, which only causes problems in some rare build configuration. [sc-49853] --- TYPE: NO_HISTORY DESC: Fix wrong type in unit aggregators. --- .../sm/query/readers/aggregators/test/unit_aggregators.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc index 511d7c43b5ef..0378409565bc 100644 --- a/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc +++ b/tiledb/sm/query/readers/aggregators/test/unit_aggregators.cc @@ -481,17 +481,17 @@ void basic_aggregation_test(std::vector expected_results) { 10, 10, &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), zero.data()); auto tile_metadata = TileMetadata( 10, 5, &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), &fixed_data[0], - sizeof(fixed_data[0]), + sizeof(typename fixed_data_type::value_type), full_tile_sum.data()); if (aggregator.has_value()) { // Regular attribute.